[PATCH] avoid allocating pte_chains for unshared pages

Patch from David McCracken.  It is an optimisation to the rmap
pte_chains.

In the common case where a page is mapped by only a single pte, we
don't need to allocate a pte_chain structure.  Just make the page's
pte_chain pointer point straight at that pte and flag this with
PG_direct.

diff --git a/include/linux/mm.h b/include/linux/mm.h
index 0c0b6d41dbb0460445ce7c3b9ab354fcccede429..c470238148b61d4403f885049216018f09726ef4 100644 (file)
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -157,8 +157,11 @@ struct page {
                                           updated asynchronously */
        struct list_head lru;           /* Pageout list, eg. active_list;
                                           protected by pagemap_lru_lock !! */
-       struct pte_chain * pte_chain;   /* Reverse pte mapping pointer.
+       union {
+               struct pte_chain * chain;       /* Reverse pte mapping pointer.
                                         * protected by PG_chainlock */
+               pte_t            * direct;
+       } pte;
        unsigned long private;          /* mapping-private opaque data */
 
        /*

diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h
index 7cdd56c8cc3e3b1dfad8ea87a0e38b046a4bec9d..a1efdff4d23c963dfc8057ec0cefb68ff607bdc8 100644 (file)
--- a/include/linux/page-flags.h
+++ b/include/linux/page-flags.h
@@ -64,8 +64,10 @@
 
 #define PG_private             12      /* Has something at ->private */
 #define PG_writeback           13      /* Page is under writeback */
-#define PG_nosave              15      /* Used for system suspend/resume */
-#define PG_chainlock           16      /* lock bit for ->pte_chain */
+#define PG_nosave              14      /* Used for system suspend/resume */
+#define PG_chainlock           15      /* lock bit for ->pte_chain */
+
+#define PG_direct              16      /* ->pte_chain points directly at pte */
 
 /*
  * Global page accounting.  One instance per CPU.
@@ -217,6 +219,12 @@ extern void get_page_state(struct page_state *ret);
 #define ClearPageNosave(page)          clear_bit(PG_nosave, &(page)->flags)
 #define TestClearPageNosave(page)      test_and_clear_bit(PG_nosave, &(page)->flags)
 
+#define PageDirect(page)       test_bit(PG_direct, &(page)->flags)
+#define SetPageDirect(page)    set_bit(PG_direct, &(page)->flags)
+#define TestSetPageDirect(page)        test_and_set_bit(PG_direct, &(page)->flags)
+#define ClearPageDirect(page)          clear_bit(PG_direct, &(page)->flags)
+#define TestClearPageDirect(page)      test_and_clear_bit(PG_direct, &(page)->flags)
+
 /*
  * inlines for acquisition and release of PG_chainlock
  */

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index e34b71e2db7c823800a4f00b9d34c06c6817d556..a5b6e175632d11f8d99276ffc2b23dd86d263478 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -92,7 +92,7 @@ static void __free_pages_ok (struct page *page, unsigned int order)
        BUG_ON(PageLRU(page));
        BUG_ON(PageActive(page));
        BUG_ON(PageWriteback(page));
-       BUG_ON(page->pte_chain != NULL);
+       BUG_ON(page->pte.chain != NULL);
        if (PageDirty(page))
                ClearPageDirty(page);
        BUG_ON(page_count(page) != 0);

diff --git a/mm/rmap.c b/mm/rmap.c
index e41198daee51a90d94734c673da89c0c4ee78f29..7d7c2a621944f93bf00994327b871be1aa0c9df2 100644 (file)
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -13,7 +13,7 @@
 
 /*
  * Locking:
- * - the page->pte_chain is protected by the PG_chainlock bit,
+ * - the page->pte.chain is protected by the PG_chainlock bit,
  *   which nests within the pagemap_lru_lock, then the
  *   mm->page_table_lock, and then the page lock.
  * - because swapout locking is opposite to the locking order
@@ -71,10 +71,15 @@ int page_referenced(struct page * page)
        if (TestClearPageReferenced(page))
                referenced++;
 
-       /* Check all the page tables mapping this page. */
-       for (pc = page->pte_chain; pc; pc = pc->next) {
-               if (ptep_test_and_clear_young(pc->ptep))
+       if (PageDirect(page)) {
+               if (ptep_test_and_clear_young(page->pte.direct))
                        referenced++;
+       } else {
+               /* Check all the page tables mapping this page. */
+               for (pc = page->pte.chain; pc; pc = pc->next) {
+                       if (ptep_test_and_clear_young(pc->ptep))
+                               referenced++;
+               }
        }
        return referenced;
 }
@@ -108,22 +113,39 @@ void page_add_rmap(struct page * page, pte_t * ptep)
        pte_chain_lock(page);
        {
                struct pte_chain * pc;
-               for (pc = page->pte_chain; pc; pc = pc->next) {
-                       if (pc->ptep == ptep)
+               if (PageDirect(page)) {
+                       if (page->pte.direct == ptep)
                                BUG();
+               } else {
+                       for (pc = page->pte.chain; pc; pc = pc->next) {
+                               if (pc->ptep == ptep)
+                                       BUG();
+                       }
                }
        }
        pte_chain_unlock(page);
 #endif
 
-       pte_chain = pte_chain_alloc();
-
        pte_chain_lock(page);
 
-       /* Hook up the pte_chain to the page. */
-       pte_chain->ptep = ptep;
-       pte_chain->next = page->pte_chain;
-       page->pte_chain = pte_chain;
+       if (PageDirect(page)) {
+               /* Convert a direct pointer into a pte_chain */
+               pte_chain = pte_chain_alloc();
+               pte_chain->ptep = page->pte.direct;
+               pte_chain->next = NULL;
+               page->pte.chain = pte_chain;
+               ClearPageDirect(page);
+       }
+       if (page->pte.chain) {
+               /* Hook up the pte_chain to the page. */
+               pte_chain = pte_chain_alloc();
+               pte_chain->ptep = ptep;
+               pte_chain->next = page->pte.chain;
+               page->pte.chain = pte_chain;
+       } else {
+               page->pte.direct = ptep;
+               SetPageDirect(page);
+       }
 
        pte_chain_unlock(page);
 }
@@ -149,18 +171,38 @@ void page_remove_rmap(struct page * page, pte_t * ptep)
                return;
 
        pte_chain_lock(page);
-       for (pc = page->pte_chain; pc; prev_pc = pc, pc = pc->next) {
-               if (pc->ptep == ptep) {
-                       pte_chain_free(pc, prev_pc, page);
+
+       if (PageDirect(page)) {
+               if (page->pte.direct == ptep) {
+                       page->pte.direct = NULL;
+                       ClearPageDirect(page);
                        goto out;
                }
+       } else {
+               for (pc = page->pte.chain; pc; prev_pc = pc, pc = pc->next) {
+                       if (pc->ptep == ptep) {
+                               pte_chain_free(pc, prev_pc, page);
+                               /* Check whether we can convert to direct */
+                               pc = page->pte.chain;
+                               if (!pc->next) {
+                                       page->pte.direct = pc->ptep;
+                                       SetPageDirect(page);
+                                       pte_chain_free(pc, NULL, NULL);
+                               }
+                               goto out;
+                       }
+               }
        }
 #ifdef DEBUG_RMAP
        /* Not found. This should NEVER happen! */
        printk(KERN_ERR "page_remove_rmap: pte_chain %p not present.\n", ptep);
        printk(KERN_ERR "page_remove_rmap: only found: ");
-       for (pc = page->pte_chain; pc; pc = pc->next)
-               printk("%p ", pc->ptep);
+       if (PageDirect(page)) {
+               printk("%p ", page->pte.direct);
+       } else {
+               for (pc = page->pte.chain; pc; pc = pc->next)
+                       printk("%p ", pc->ptep);
+       }
        printk("\n");
        printk(KERN_ERR "page_remove_rmap: driver cleared PG_reserved ?\n");
 #endif
@@ -270,25 +312,41 @@ int try_to_unmap(struct page * page)
        if (!page->mapping)
                BUG();
 
-       for (pc = page->pte_chain; pc; pc = next_pc) {
-               next_pc = pc->next;
-               switch (try_to_unmap_one(page, pc->ptep)) {
-                       case SWAP_SUCCESS:
-                               /* Free the pte_chain struct. */
-                               pte_chain_free(pc, prev_pc, page);
-                               break;
-                       case SWAP_AGAIN:
-                               /* Skip this pte, remembering status. */
-                               prev_pc = pc;
-                               ret = SWAP_AGAIN;
-                               continue;
-                       case SWAP_FAIL:
-                               return SWAP_FAIL;
-                       case SWAP_ERROR:
-                               return SWAP_ERROR;
+       if (PageDirect(page)) {
+               ret = try_to_unmap_one(page, page->pte.direct);
+               if (ret == SWAP_SUCCESS) {
+                       page->pte.direct = NULL;
+                       ClearPageDirect(page);
+               }
+       } else {                
+               for (pc = page->pte.chain; pc; pc = next_pc) {
+                       next_pc = pc->next;
+                       switch (try_to_unmap_one(page, pc->ptep)) {
+                               case SWAP_SUCCESS:
+                                       /* Free the pte_chain struct. */
+                                       pte_chain_free(pc, prev_pc, page);
+                                       break;
+                               case SWAP_AGAIN:
+                                       /* Skip this pte, remembering status. */
+                                       prev_pc = pc;
+                                       ret = SWAP_AGAIN;
+                                       continue;
+                               case SWAP_FAIL:
+                                       ret = SWAP_FAIL;
+                                       break;
+                               case SWAP_ERROR:
+                                       ret = SWAP_ERROR;
+                                       break;
+                       }
+               }
+               /* Check whether we can convert to direct pte pointer */
+               pc = page->pte.chain;
+               if (pc && !pc->next) {
+                       page->pte.direct = pc->ptep;
+                       SetPageDirect(page);
+                       pte_chain_free(pc, NULL, NULL);
                }
        }
-
        return ret;
 }
 
@@ -336,7 +394,7 @@ static inline void pte_chain_free(struct pte_chain * pte_chain,
        if (prev_pte_chain)
                prev_pte_chain->next = pte_chain->next;
        else if (page)
-               page->pte_chain = pte_chain->next;
+               page->pte.chain = pte_chain->next;
 
        spin_lock(&pte_chain_freelist_lock);
        pte_chain_push(pte_chain);

diff --git a/mm/vmscan.c b/mm/vmscan.c
index 76e5a723eda917d08529825a4bf410a8ff3ac937..925c82063e8b380984a2dedc44e1f2ebc8da534c 100644 (file)
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -48,7 +48,7 @@ static inline int page_mapping_inuse(struct page * page)
        struct address_space *mapping = page->mapping;
 
        /* Page is in somebody's page tables. */
-       if (page->pte_chain)
+       if (page->pte.chain)
                return 1;
 
        /* XXX: does this happen ? */
@@ -151,7 +151,7 @@ shrink_cache(int nr_pages, zone_t *classzone,
                 *
                 * XXX: implement swap clustering ?
                 */
-               if (page->pte_chain && !page->mapping && !PagePrivate(page)) {
+               if (page->pte.chain && !page->mapping && !PagePrivate(page)) {
                        page_cache_get(page);
                        pte_chain_unlock(page);
                        spin_unlock(&pagemap_lru_lock);
@@ -171,7 +171,7 @@ shrink_cache(int nr_pages, zone_t *classzone,
                 * The page is mapped into the page tables of one or more
                 * processes. Try to unmap it here.
                 */
-               if (page->pte_chain) {
+               if (page->pte.chain) {
                        switch (try_to_unmap(page)) {
                                case SWAP_ERROR:
                                case SWAP_FAIL:
@@ -340,7 +340,7 @@ static void refill_inactive(int nr_pages)
                entry = entry->prev;
 
                pte_chain_lock(page);
-               if (page->pte_chain && page_referenced(page)) {
+               if (page->pte.chain && page_referenced(page)) {
                        list_del(&page->lru);
                        list_add(&page->lru, &active_list);
                        pte_chain_unlock(page);