X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=mm%2Frmap.c;h=653b994581e13a6d516380c3c6a32363669fcabe;hb=6a77f38946aaee1cd85eeec6cf4229b204c15071;hp=c5363bf4131df12ad4571dd2d033608ef7472746;hpb=87fc8d1bb10cd459024a742c6a10961fefcef18f;p=linux-2.6.git diff --git a/mm/rmap.c b/mm/rmap.c index c5363bf41..653b99458 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -35,6 +35,7 @@ * mm->page_table_lock * zone->lru_lock (in mark_page_accessed) * swap_list_lock (in swap_free etc's swap_info_get) + * mmlist_lock (in mmput, drain_mmlist and others) * swap_device_lock (in swap_duplicate, swap_info_get) * mapping->private_lock (in __set_page_dirty_buffers) * inode_lock (in set_page_dirty's __mark_inode_dirty) @@ -50,6 +51,7 @@ #include #include #include +#include #include #include #include @@ -120,14 +122,7 @@ int anon_vma_prepare(struct vm_area_struct *vma) void __anon_vma_merge(struct vm_area_struct *vma, struct vm_area_struct *next) { - if (!vma->anon_vma) { - BUG_ON(!next->anon_vma); - vma->anon_vma = next->anon_vma; - list_add(&vma->anon_vma_node, &next->anon_vma_node); - } else { - /* if they're both non-null they must be the same */ - BUG_ON(vma->anon_vma != next->anon_vma); - } + BUG_ON(vma->anon_vma != next->anon_vma); list_del(&next->anon_vma_node); } @@ -254,11 +249,12 @@ unsigned long page_address_in_vma(struct page *page, struct vm_area_struct *vma) * repeatedly from either page_referenced_anon or page_referenced_file. */ static int page_referenced_one(struct page *page, - struct vm_area_struct *vma, unsigned int *mapcount) + struct vm_area_struct *vma, unsigned int *mapcount, int ignore_token) { struct mm_struct *mm = vma->vm_mm; unsigned long address; pgd_t *pgd; + pud_t *pud; pmd_t *pmd; pte_t *pte; int referenced = 0; @@ -275,7 +271,11 @@ static int page_referenced_one(struct page *page, if (!pgd_present(*pgd)) goto out_unlock; - pmd = pmd_offset(pgd, address); + pud = pud_offset(pgd, address); + if (!pud_present(*pud)) + goto out_unlock; + + pmd = pmd_offset(pud, address); if (!pmd_present(*pmd)) goto out_unlock; @@ -289,7 +289,7 @@ static int page_referenced_one(struct page *page, if (ptep_clear_flush_young(vma, address, pte)) referenced++; - if (mm != current->mm && has_swap_token(mm)) + if (mm != current->mm && !ignore_token && has_swap_token(mm)) referenced++; (*mapcount)--; @@ -302,7 +302,7 @@ out: return referenced; } -static int page_referenced_anon(struct page *page) +static int page_referenced_anon(struct page *page, int ignore_token) { unsigned int mapcount; struct anon_vma *anon_vma; @@ -315,7 +315,8 @@ static int page_referenced_anon(struct page *page) mapcount = page_mapcount(page); list_for_each_entry(vma, &anon_vma->head, anon_vma_node) { - referenced += page_referenced_one(page, vma, &mapcount); + referenced += page_referenced_one(page, vma, &mapcount, + ignore_token); if (!mapcount) break; } @@ -334,7 +335,7 @@ static int page_referenced_anon(struct page *page) * * This function is only called from page_referenced for object-based pages. */ -static int page_referenced_file(struct page *page) +static int page_referenced_file(struct page *page, int ignore_token) { unsigned int mapcount; struct address_space *mapping = page->mapping; @@ -372,7 +373,8 @@ static int page_referenced_file(struct page *page) referenced++; break; } - referenced += page_referenced_one(page, vma, &mapcount); + referenced += page_referenced_one(page, vma, &mapcount, + ignore_token); if (!mapcount) break; } @@ -389,10 +391,13 @@ static int page_referenced_file(struct page *page) * Quick test_and_clear_referenced for all mappings to a page, * returns the number of ptes which referenced the page. */ -int page_referenced(struct page *page, int is_locked) +int page_referenced(struct page *page, int is_locked, int ignore_token) { int referenced = 0; + if (!swap_token_default_timeout) + ignore_token = 1; + if (page_test_and_clear_young(page)) referenced++; @@ -401,14 +406,15 @@ int page_referenced(struct page *page, int is_locked) if (page_mapped(page) && page->mapping) { if (PageAnon(page)) - referenced += page_referenced_anon(page); + referenced += page_referenced_anon(page, ignore_token); else if (is_locked) - referenced += page_referenced_file(page); + referenced += page_referenced_file(page, ignore_token); else if (TestSetPageLocked(page)) referenced++; else { if (page->mapping) - referenced += page_referenced_file(page); + referenced += page_referenced_file(page, + ignore_token); unlock_page(page); } } @@ -432,6 +438,9 @@ void page_add_anon_rmap(struct page *page, BUG_ON(PageReserved(page)); BUG_ON(!anon_vma); + // vma->vm_mm->anon_rss++; + vx_anonpages_inc(vma->vm_mm); + anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON; index = (address - vma->vm_start) >> PAGE_SHIFT; index += vma->vm_pgoff; @@ -497,6 +506,7 @@ static int try_to_unmap_one(struct page *page, struct vm_area_struct *vma) struct mm_struct *mm = vma->vm_mm; unsigned long address; pgd_t *pgd; + pud_t *pud; pmd_t *pmd; pte_t *pte; pte_t pteval; @@ -518,7 +528,11 @@ static int try_to_unmap_one(struct page *page, struct vm_area_struct *vma) if (!pgd_present(*pgd)) goto out_unlock; - pmd = pmd_offset(pgd, address); + pud = pud_offset(pgd, address); + if (!pud_present(*pud)) + goto out_unlock; + + pmd = pmd_offset(pud, address); if (!pmd_present(*pmd)) goto out_unlock; @@ -577,12 +591,20 @@ static int try_to_unmap_one(struct page *page, struct vm_area_struct *vma) */ BUG_ON(!PageSwapCache(page)); swap_duplicate(entry); + if (list_empty(&mm->mmlist)) { + spin_lock(&mmlist_lock); + list_add(&mm->mmlist, &init_mm.mmlist); + spin_unlock(&mmlist_lock); + } set_pte(pte, swp_entry_to_pte(entry)); BUG_ON(pte_file(*pte)); + // mm->anon_rss--; + vx_anonpages_dec(mm); } // mm->rss--; vx_rsspages_dec(mm); + acct_update_integrals(); page_remove_rmap(page); page_cache_release(page); @@ -621,6 +643,7 @@ static void try_to_unmap_cluster(unsigned long cursor, { struct mm_struct *mm = vma->vm_mm; pgd_t *pgd; + pud_t *pud; pmd_t *pmd; pte_t *pte; pte_t pteval; @@ -646,7 +669,11 @@ static void try_to_unmap_cluster(unsigned long cursor, if (!pgd_present(*pgd)) goto out_unlock; - pmd = pmd_offset(pgd, address); + pud = pud_offset(pgd, address); + if (!pud_present(*pud)) + goto out_unlock; + + pmd = pmd_offset(pud, address); if (!pmd_present(*pmd)) goto out_unlock; @@ -682,6 +709,7 @@ static void try_to_unmap_cluster(unsigned long cursor, page_remove_rmap(page); page_cache_release(page); + acct_update_integrals(); // mm->rss--; vx_rsspages_dec(mm); (*mapcount)--;