X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=mm%2Fswapfile.c;h=8297157378d26c506bfb627319918a6b7720779c;hb=refs%2Fheads%2Fvserver;hp=50c3a777c9f491181521064c0cf8a16a6e33b28f;hpb=c7b5ebbddf7bcd3651947760f423e3783bbe6573;p=linux-2.6.git diff --git a/mm/swapfile.c b/mm/swapfile.c index 50c3a777c..829715737 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -5,7 +5,6 @@ * Swap reorganised 29.12.95, Stephen Tweedie */ -#include #include #include #include @@ -25,6 +24,9 @@ #include #include #include +#include +#include +#include #include #include @@ -32,13 +34,11 @@ #include #include -spinlock_t swaplock = SPIN_LOCK_UNLOCKED; +DEFINE_SPINLOCK(swap_lock); unsigned int nr_swapfiles; long total_swap_pages; static int swap_overflow; -EXPORT_SYMBOL(total_swap_pages); - static const char Bad_file[] = "Bad swap file entry "; static const char Unused_file[] = "Unused swap file entry "; static const char Bad_offset[] = "Bad swap offset entry "; @@ -46,25 +46,23 @@ static const char Unused_offset[] = "Unused swap offset entry "; struct swap_list_t swap_list = {-1, -1}; -struct swap_info_struct swap_info[MAX_SWAPFILES]; +static struct swap_info_struct swap_info[MAX_SWAPFILES]; -static DECLARE_MUTEX(swapon_sem); +static DEFINE_MUTEX(swapon_mutex); /* * We need this because the bdev->unplug_fn can sleep and we cannot - * hold swap_list_lock while calling the unplug_fn. And swap_list_lock - * cannot be turned into a semaphore. + * hold swap_lock while calling the unplug_fn. And swap_lock + * cannot be turned into a mutex. */ static DECLARE_RWSEM(swap_unplug_sem); -#define SWAPFILE_CLUSTER 256 - void swap_unplug_io_fn(struct backing_dev_info *unused_bdi, struct page *page) { swp_entry_t entry; down_read(&swap_unplug_sem); - entry.val = page->private; + entry.val = page_private(page); if (PageSwapCache(page)) { struct block_device *bdev = swap_info[swp_type(entry)].bdev; struct backing_dev_info *bdi; @@ -72,129 +70,168 @@ void swap_unplug_io_fn(struct backing_dev_info *unused_bdi, struct page *page) /* * If the page is removed from swapcache from under us (with a * racy try_to_unuse/swapoff) we need an additional reference - * count to avoid reading garbage from page->private above. If - * the WARN_ON triggers during a swapoff it maybe the race + * count to avoid reading garbage from page_private(page) above. + * If the WARN_ON triggers during a swapoff it maybe the race * condition and it's harmless. However if it triggers without * swapoff it signals a problem. */ WARN_ON(page_count(page) <= 1); bdi = bdev->bd_inode->i_mapping->backing_dev_info; - bdi->unplug_io_fn(bdi, page); + blk_run_backing_dev(bdi, page); } up_read(&swap_unplug_sem); } -static inline int scan_swap_map(struct swap_info_struct *si) +#define SWAPFILE_CLUSTER 256 +#define LATENCY_LIMIT 256 + +static inline unsigned long scan_swap_map(struct swap_info_struct *si) { - unsigned long offset; + unsigned long offset, last_in_cluster; + int latency_ration = LATENCY_LIMIT; + /* - * We try to cluster swap pages by allocating them - * sequentially in swap. Once we've allocated - * SWAPFILE_CLUSTER pages this way, however, we resort to - * first-free allocation, starting a new cluster. This - * prevents us from scattering swap pages all over the entire - * swap partition, so that we reduce overall disk seek times - * between swap pages. -- sct */ - if (si->cluster_nr) { - while (si->cluster_next <= si->highest_bit) { - offset = si->cluster_next++; + * We try to cluster swap pages by allocating them sequentially + * in swap. Once we've allocated SWAPFILE_CLUSTER pages this + * way, however, we resort to first-free allocation, starting + * a new cluster. This prevents us from scattering swap pages + * all over the entire swap partition, so that we reduce + * overall disk seek times between swap pages. -- sct + * But we do now try to find an empty cluster. -Andrea + */ + + si->flags += SWP_SCANNING; + if (unlikely(!si->cluster_nr)) { + si->cluster_nr = SWAPFILE_CLUSTER - 1; + if (si->pages - si->inuse_pages < SWAPFILE_CLUSTER) + goto lowest; + spin_unlock(&swap_lock); + + offset = si->lowest_bit; + last_in_cluster = offset + SWAPFILE_CLUSTER - 1; + + /* Locate the first empty (unaligned) cluster */ + for (; last_in_cluster <= si->highest_bit; offset++) { if (si->swap_map[offset]) - continue; - si->cluster_nr--; - goto got_page; - } - } - si->cluster_nr = SWAPFILE_CLUSTER; - - /* try to find an empty (even not aligned) cluster. */ - offset = si->lowest_bit; - check_next_cluster: - if (offset+SWAPFILE_CLUSTER-1 <= si->highest_bit) - { - unsigned long nr; - for (nr = offset; nr < offset+SWAPFILE_CLUSTER; nr++) - if (si->swap_map[nr]) - { - offset = nr+1; - goto check_next_cluster; + last_in_cluster = offset + SWAPFILE_CLUSTER; + else if (offset == last_in_cluster) { + spin_lock(&swap_lock); + si->cluster_next = offset-SWAPFILE_CLUSTER+1; + goto cluster; } - /* We found a completly empty cluster, so start - * using it. - */ - goto got_page; + if (unlikely(--latency_ration < 0)) { + cond_resched(); + latency_ration = LATENCY_LIMIT; + } + } + spin_lock(&swap_lock); + goto lowest; } - /* No luck, so now go finegrined as usual. -Andrea */ - for (offset = si->lowest_bit; offset <= si->highest_bit ; offset++) { - if (si->swap_map[offset]) - continue; - si->lowest_bit = offset+1; - got_page: + + si->cluster_nr--; +cluster: + offset = si->cluster_next; + if (offset > si->highest_bit) +lowest: offset = si->lowest_bit; +checks: if (!(si->flags & SWP_WRITEOK)) + goto no_page; + if (!si->highest_bit) + goto no_page; + if (!si->swap_map[offset]) { if (offset == si->lowest_bit) si->lowest_bit++; if (offset == si->highest_bit) si->highest_bit--; - if (si->lowest_bit > si->highest_bit) { + si->inuse_pages++; + if (si->inuse_pages == si->pages) { si->lowest_bit = si->max; si->highest_bit = 0; } si->swap_map[offset] = 1; - si->inuse_pages++; - nr_swap_pages--; - si->cluster_next = offset+1; + si->cluster_next = offset + 1; + si->flags -= SWP_SCANNING; return offset; } - si->lowest_bit = si->max; - si->highest_bit = 0; + + spin_unlock(&swap_lock); + while (++offset <= si->highest_bit) { + if (!si->swap_map[offset]) { + spin_lock(&swap_lock); + goto checks; + } + if (unlikely(--latency_ration < 0)) { + cond_resched(); + latency_ration = LATENCY_LIMIT; + } + } + spin_lock(&swap_lock); + goto lowest; + +no_page: + si->flags -= SWP_SCANNING; return 0; } swp_entry_t get_swap_page(void) { - struct swap_info_struct * p; - unsigned long offset; - swp_entry_t entry; - int type, wrapped = 0; + struct swap_info_struct *si; + pgoff_t offset; + int type, next; + int wrapped = 0; - entry.val = 0; /* Out of memory */ - swap_list_lock(); - type = swap_list.next; - if (type < 0) - goto out; + spin_lock(&swap_lock); if (nr_swap_pages <= 0) - goto out; + goto noswap; + nr_swap_pages--; + + for (type = swap_list.next; type >= 0 && wrapped < 2; type = next) { + si = swap_info + type; + next = si->next; + if (next < 0 || + (!wrapped && si->prio != swap_info[next].prio)) { + next = swap_list.head; + wrapped++; + } - while (1) { - p = &swap_info[type]; - if ((p->flags & SWP_ACTIVE) == SWP_ACTIVE) { - swap_device_lock(p); - offset = scan_swap_map(p); - swap_device_unlock(p); - if (offset) { - entry = swp_entry(type,offset); - type = swap_info[type].next; - if (type < 0 || - p->prio != swap_info[type].prio) { - swap_list.next = swap_list.head; - } else { - swap_list.next = type; - } - goto out; - } + if (!si->highest_bit) + continue; + if (!(si->flags & SWP_WRITEOK)) + continue; + + swap_list.next = next; + offset = scan_swap_map(si); + if (offset) { + spin_unlock(&swap_lock); + return swp_entry(type, offset); } - type = p->next; - if (!wrapped) { - if (type < 0 || p->prio != swap_info[type].prio) { - type = swap_list.head; - wrapped = 1; - } - } else - if (type < 0) - goto out; /* out of swap space */ + next = swap_list.next; } -out: - swap_list_unlock(); - return entry; + + nr_swap_pages++; +noswap: + spin_unlock(&swap_lock); + return (swp_entry_t) {0}; +} + +swp_entry_t get_swap_page_of_type(int type) +{ + struct swap_info_struct *si; + pgoff_t offset; + + spin_lock(&swap_lock); + si = swap_info + type; + if (si->flags & SWP_WRITEOK) { + nr_swap_pages--; + offset = scan_swap_map(si); + if (offset) { + spin_unlock(&swap_lock); + return swp_entry(type, offset); + } + nr_swap_pages++; + } + spin_unlock(&swap_lock); + return (swp_entry_t) {0}; } static struct swap_info_struct * swap_info_get(swp_entry_t entry) @@ -215,10 +252,7 @@ static struct swap_info_struct * swap_info_get(swp_entry_t entry) goto bad_offset; if (!p->swap_map[offset]) goto bad_free; - swap_list_lock(); - if (p->prio > swap_info[swap_list.next].prio) - swap_list.next = type; - swap_device_lock(p); + spin_lock(&swap_lock); return p; bad_free: @@ -236,12 +270,6 @@ out: return NULL; } -static void swap_info_put(struct swap_info_struct * p) -{ - swap_device_unlock(p); - swap_list_unlock(); -} - static int swap_entry_free(struct swap_info_struct *p, unsigned long offset) { int count = p->swap_map[offset]; @@ -254,6 +282,8 @@ static int swap_entry_free(struct swap_info_struct *p, unsigned long offset) p->lowest_bit = offset; if (offset > p->highest_bit) p->highest_bit = offset; + if (p->prio > swap_info[swap_list.next].prio) + swap_list.next = p - swap_info; nr_swap_pages++; p->inuse_pages--; } @@ -272,66 +302,42 @@ void swap_free(swp_entry_t entry) p = swap_info_get(entry); if (p) { swap_entry_free(p, swp_offset(entry)); - swap_info_put(p); + spin_unlock(&swap_lock); } } /* - * Check if we're the only user of a swap page, - * when the page is locked. + * How many references to page are currently swapped out? */ -static int exclusive_swap_page(struct page *page) +static inline int page_swapcount(struct page *page) { - int retval = 0; - struct swap_info_struct * p; + int count = 0; + struct swap_info_struct *p; swp_entry_t entry; - entry.val = page->private; + entry.val = page_private(page); p = swap_info_get(entry); if (p) { - /* Is the only swap cache user the cache itself? */ - if (p->swap_map[swp_offset(entry)] == 1) { - /* Recheck the page count with the swapcache lock held.. */ - spin_lock_irq(&swapper_space.tree_lock); - if (page_count(page) == 2) - retval = 1; - spin_unlock_irq(&swapper_space.tree_lock); - } - swap_info_put(p); + /* Subtract the 1 for the swap cache itself */ + count = p->swap_map[swp_offset(entry)] - 1; + spin_unlock(&swap_lock); } - return retval; + return count; } /* * We can use this swap cache entry directly * if there are no other references to it. - * - * Here "exclusive_swap_page()" does the real - * work, but we opportunistically check whether - * we need to get all the locks first.. */ int can_share_swap_page(struct page *page) { - int retval = 0; + int count; - if (!PageLocked(page)) - BUG(); - switch (page_count(page)) { - case 3: - if (!PagePrivate(page)) - break; - /* Fallthrough */ - case 2: - if (!PageSwapCache(page)) - break; - retval = exclusive_swap_page(page); - break; - case 1: - if (PageReserved(page)) - break; - retval = 1; - } - return retval; + BUG_ON(!PageLocked(page)); + count = page_mapcount(page); + if (count <= 1 && PageSwapCache(page)) + count += page_swapcount(page); + return count == 1; } /* @@ -354,7 +360,7 @@ int remove_exclusive_swap_page(struct page *page) if (page_count(page) != 2) /* 2: us + cache */ return 0; - entry.val = page->private; + entry.val = page_private(page); p = swap_info_get(entry); if (!p) return 0; @@ -363,15 +369,15 @@ int remove_exclusive_swap_page(struct page *page) retval = 0; if (p->swap_map[swp_offset(entry)] == 1) { /* Recheck the page count with the swapcache lock held.. */ - spin_lock_irq(&swapper_space.tree_lock); + write_lock_irq(&swapper_space.tree_lock); if ((page_count(page) == 2) && !PageWriteback(page)) { __delete_from_swap_cache(page); SetPageDirty(page); retval = 1; } - spin_unlock_irq(&swapper_space.tree_lock); + write_unlock_irq(&swapper_space.tree_lock); } - swap_info_put(p); + spin_unlock(&swap_lock); if (retval) { swap_free(entry); @@ -390,26 +396,29 @@ void free_swap_and_cache(swp_entry_t entry) struct swap_info_struct * p; struct page *page = NULL; + if (is_migration_entry(entry)) + return; + p = swap_info_get(entry); if (p) { if (swap_entry_free(p, swp_offset(entry)) == 1) { - spin_lock_irq(&swapper_space.tree_lock); - page = radix_tree_lookup(&swapper_space.page_tree, - entry.val); - if (page && TestSetPageLocked(page)) + page = find_get_page(&swapper_space, entry.val); + if (page && unlikely(TestSetPageLocked(page))) { + page_cache_release(page); page = NULL; - spin_unlock_irq(&swapper_space.tree_lock); + } } - swap_info_put(p); + spin_unlock(&swap_lock); } if (page) { int one_user; BUG_ON(PagePrivate(page)); - page_cache_get(page); one_user = (page_count(page) == 2); /* Only cache user (+us), or swap space full? Free it! */ - if (!PageWriteback(page) && (one_user || vm_swap_full())) { + /* Also recheck PageSwapCache after page is locked (above) */ + if (PageSwapCache(page) && !PageWriteback(page) && + (one_user || vm_swap_full())) { delete_from_swap_cache(page); SetPageDirty(page); } @@ -418,170 +427,214 @@ void free_swap_and_cache(swp_entry_t entry) } } +#ifdef CONFIG_SOFTWARE_SUSPEND /* - * The swap entry has been read in advance, and we return 1 to indicate - * that the page has been used or is no longer needed. + * Find the swap type that corresponds to given device (if any). + * + * @offset - number of the PAGE_SIZE-sized block of the device, starting + * from 0, in which the swap header is expected to be located. * - * Always set the resulting pte to be nowrite (the same as COW pages - * after one process has exited). We don't know just how many PTEs will - * share this swap entry, so be cautious and let do_wp_page work out - * what to do if a write is requested later. + * This is needed for the suspend to disk (aka swsusp). */ -/* vma->vm_mm->page_table_lock is held */ -static void -unuse_pte(struct vm_area_struct *vma, unsigned long address, pte_t *dir, - swp_entry_t entry, struct page *page) +int swap_type_of(dev_t device, sector_t offset, struct block_device **bdev_p) { - // vma->vm_mm->rss++; - vx_rsspages_inc(vma->vm_mm); + struct block_device *bdev = NULL; + int i; + + if (device) + bdev = bdget(device); + + spin_lock(&swap_lock); + for (i = 0; i < nr_swapfiles; i++) { + struct swap_info_struct *sis = swap_info + i; + + if (!(sis->flags & SWP_WRITEOK)) + continue; + + if (!bdev) { + if (bdev_p) + *bdev_p = sis->bdev; + + spin_unlock(&swap_lock); + return i; + } + if (bdev == sis->bdev) { + struct swap_extent *se; + + se = list_entry(sis->extent_list.next, + struct swap_extent, list); + if (se->start_block == offset) { + if (bdev_p) + *bdev_p = sis->bdev; + + spin_unlock(&swap_lock); + bdput(bdev); + return i; + } + } + } + spin_unlock(&swap_lock); + if (bdev) + bdput(bdev); + + return -ENODEV; +} + +/* + * Return either the total number of swap pages of given type, or the number + * of free pages of that type (depending on @free) + * + * This is needed for software suspend + */ +unsigned int count_swap_pages(int type, int free) +{ + unsigned int n = 0; + + if (type < nr_swapfiles) { + spin_lock(&swap_lock); + if (swap_info[type].flags & SWP_WRITEOK) { + n = swap_info[type].pages; + if (free) + n -= swap_info[type].inuse_pages; + } + spin_unlock(&swap_lock); + } + return n; +} +#endif + +/* + * No need to decide whether this PTE shares the swap entry with others, + * just let do_wp_page work it out if a write is requested later - to + * force COW, vm_page_prot omits write permission from any private vma. + */ +static void unuse_pte(struct vm_area_struct *vma, pte_t *pte, + unsigned long addr, swp_entry_t entry, struct page *page) +{ + inc_mm_counter(vma->vm_mm, anon_rss); get_page(page); - set_pte(dir, pte_mkold(mk_pte(page, vma->vm_page_prot))); - page_add_anon_rmap(page, vma, address); + set_pte_at(vma->vm_mm, addr, pte, + pte_mkold(mk_pte(page, vma->vm_page_prot))); + page_add_anon_rmap(page, vma, addr); swap_free(entry); + /* + * Move the page to the active list so it is not + * immediately swapped out again after swapon. + */ + activate_page(page); } -/* vma->vm_mm->page_table_lock is held */ -static unsigned long unuse_pmd(struct vm_area_struct * vma, pmd_t *dir, - unsigned long address, unsigned long size, unsigned long offset, - swp_entry_t entry, struct page *page) +static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd, + unsigned long addr, unsigned long end, + swp_entry_t entry, struct page *page) { - pte_t * pte; - unsigned long end; pte_t swp_pte = swp_entry_to_pte(entry); + pte_t *pte; + spinlock_t *ptl; + int found = 0; - if (pmd_none(*dir)) - return 0; - if (pmd_bad(*dir)) { - pmd_ERROR(*dir); - pmd_clear(dir); - return 0; - } - pte = pte_offset_map(dir, address); - offset += address & PMD_MASK; - address &= ~PMD_MASK; - end = address + size; - if (end > PMD_SIZE) - end = PMD_SIZE; + pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); do { /* * swapoff spends a _lot_ of time in this loop! * Test inline before going to call unuse_pte. */ if (unlikely(pte_same(*pte, swp_pte))) { - unuse_pte(vma, offset + address, pte, entry, page); - pte_unmap(pte); + unuse_pte(vma, pte++, addr, entry, page); + found = 1; + break; + } + } while (pte++, addr += PAGE_SIZE, addr != end); + pte_unmap_unlock(pte - 1, ptl); + return found; +} - /* - * Move the page to the active list so it is not - * immediately swapped out again after swapon. - */ - activate_page(page); +static inline int unuse_pmd_range(struct vm_area_struct *vma, pud_t *pud, + unsigned long addr, unsigned long end, + swp_entry_t entry, struct page *page) +{ + pmd_t *pmd; + unsigned long next; - /* add 1 since address may be 0 */ - return 1 + offset + address; - } - address += PAGE_SIZE; - pte++; - } while (address && (address < end)); - pte_unmap(pte - 1); + pmd = pmd_offset(pud, addr); + do { + next = pmd_addr_end(addr, end); + if (pmd_none_or_clear_bad(pmd)) + continue; + if (unuse_pte_range(vma, pmd, addr, next, entry, page)) + return 1; + } while (pmd++, addr = next, addr != end); return 0; } -/* vma->vm_mm->page_table_lock is held */ -static unsigned long unuse_pgd(struct vm_area_struct * vma, pgd_t *dir, - unsigned long address, unsigned long size, - swp_entry_t entry, struct page *page) +static inline int unuse_pud_range(struct vm_area_struct *vma, pgd_t *pgd, + unsigned long addr, unsigned long end, + swp_entry_t entry, struct page *page) { - pmd_t * pmd; - unsigned long offset, end; - unsigned long foundaddr; + pud_t *pud; + unsigned long next; - if (pgd_none(*dir)) - return 0; - if (pgd_bad(*dir)) { - pgd_ERROR(*dir); - pgd_clear(dir); - return 0; - } - pmd = pmd_offset(dir, address); - offset = address & PGDIR_MASK; - address &= ~PGDIR_MASK; - end = address + size; - if (end > PGDIR_SIZE) - end = PGDIR_SIZE; - if (address >= end) - BUG(); + pud = pud_offset(pgd, addr); do { - foundaddr = unuse_pmd(vma, pmd, address, end - address, - offset, entry, page); - if (foundaddr) - return foundaddr; - address = (address + PMD_SIZE) & PMD_MASK; - pmd++; - } while (address && (address < end)); + next = pud_addr_end(addr, end); + if (pud_none_or_clear_bad(pud)) + continue; + if (unuse_pmd_range(vma, pud, addr, next, entry, page)) + return 1; + } while (pud++, addr = next, addr != end); return 0; } -/* vma->vm_mm->page_table_lock is held */ -static unsigned long unuse_vma(struct vm_area_struct * vma, - swp_entry_t entry, struct page *page) +static int unuse_vma(struct vm_area_struct *vma, + swp_entry_t entry, struct page *page) { - pgd_t *pgdir; - unsigned long start, end; - unsigned long foundaddr; + pgd_t *pgd; + unsigned long addr, end, next; if (page->mapping) { - start = page_address_in_vma(page, vma); - if (start == -EFAULT) + addr = page_address_in_vma(page, vma); + if (addr == -EFAULT) return 0; else - end = start + PAGE_SIZE; + end = addr + PAGE_SIZE; } else { - start = vma->vm_start; + addr = vma->vm_start; end = vma->vm_end; } - pgdir = pgd_offset(vma->vm_mm, start); + + pgd = pgd_offset(vma->vm_mm, addr); do { - foundaddr = unuse_pgd(vma, pgdir, start, end - start, - entry, page); - if (foundaddr) - return foundaddr; - start = (start + PGDIR_SIZE) & PGDIR_MASK; - pgdir++; - } while (start && (start < end)); + next = pgd_addr_end(addr, end); + if (pgd_none_or_clear_bad(pgd)) + continue; + if (unuse_pud_range(vma, pgd, addr, next, entry, page)) + return 1; + } while (pgd++, addr = next, addr != end); return 0; } -static int unuse_process(struct mm_struct * mm, - swp_entry_t entry, struct page* page) +static int unuse_mm(struct mm_struct *mm, + swp_entry_t entry, struct page *page) { - struct vm_area_struct* vma; - unsigned long foundaddr = 0; + struct vm_area_struct *vma; - /* - * Go through process' page directory. - */ if (!down_read_trylock(&mm->mmap_sem)) { /* - * Our reference to the page stops try_to_unmap_one from - * unmapping its ptes, so swapoff can make progress. + * Activate page so shrink_cache is unlikely to unmap its + * ptes while lock is dropped, so swapoff can make progress. */ + activate_page(page); unlock_page(page); down_read(&mm->mmap_sem); lock_page(page); } - spin_lock(&mm->page_table_lock); for (vma = mm->mmap; vma; vma = vma->vm_next) { - if (vma->anon_vma) { - foundaddr = unuse_vma(vma, entry, page); - if (foundaddr) - break; - } + if (vma->anon_vma && unuse_vma(vma, entry, page)) + break; } - spin_unlock(&mm->page_table_lock); up_read(&mm->mmap_sem); /* - * Currently unuse_process cannot fail, but leave error handling + * Currently unuse_mm cannot fail, but leave error handling * at call sites for now, since we change it from time to time. */ return 0; @@ -591,17 +644,18 @@ static int unuse_process(struct mm_struct * mm, * Scan swap_map from current position to next entry still in use. * Recycle to start on reaching the end, returning 0 when empty. */ -static int find_next_to_unuse(struct swap_info_struct *si, int prev) +static unsigned int find_next_to_unuse(struct swap_info_struct *si, + unsigned int prev) { - int max = si->max; - int i = prev; + unsigned int max = si->max; + unsigned int i = prev; int count; /* - * No need for swap_device_lock(si) here: we're just looking + * No need for swap_lock here: we're just looking * for whether an entry is in use, not modifying it; false * hits are okay, and sys_swapoff() has already prevented new - * allocations from this area (while holding swap_list_lock()). + * allocations from this area (while holding swap_lock). */ for (;;) { if (++i >= max) { @@ -637,7 +691,7 @@ static int try_to_unuse(unsigned int type) unsigned short swcount; struct page *page; swp_entry_t entry; - int i = 0; + unsigned int i = 0; int retval = 0; int reset_overflow = 0; int shmem; @@ -650,11 +704,12 @@ static int try_to_unuse(unsigned int type) * * A simpler strategy would be to start at the last mm we * freed the previous entry from; but that would take less - * advantage of mmlist ordering (now preserved by swap_out()), - * which clusters forked address spaces together, most recent - * child immediately after parent. If we race with dup_mmap(), - * we very much want to resolve parent before child, otherwise - * we may miss some entries: using last mm would invert that. + * advantage of mmlist ordering, which clusters forked mms + * together, child after parent. If we race with dup_mmap(), we + * prefer to resolve parent before child, lest we miss entries + * duplicated after we scanned child: using last mm would invert + * that. Though it's only a serious concern when an overflowed + * swap count is reset from SWAP_MAP_MAX, preventing a rescan. */ start_mm = &init_mm; atomic_inc(&init_mm.mm_users); @@ -662,15 +717,7 @@ static int try_to_unuse(unsigned int type) /* * Keep on scanning until all entries have gone. Usually, * one pass through swap_map is enough, but not necessarily: - * mmput() removes mm from mmlist before exit_mmap() and its - * zap_page_range(). That's not too bad, those entries are - * on their way out, and handled faster there than here. - * do_munmap() behaves similarly, taking the range out of mm's - * vma list before zap_page_range(). But unfortunately, when - * unmapping a part of a vma, it takes the whole out first, - * then reinserts what's left after (might even reschedule if - * open() method called) - so swap entries may be invisible - * to swapoff for a while, then reappear - but that is rare. + * there are races when an instance of an entry might be missed. */ while ((i = find_next_to_unuse(si, i)) != 0) { if (signal_pending(current)) { @@ -722,7 +769,7 @@ static int try_to_unuse(unsigned int type) wait_on_page_writeback(page); /* - * Remove all references to entry, without blocking. + * Remove all references to entry. * Whenever we reach init_mm, there's no address space * to search, but use it as a reminder to search shmem. */ @@ -732,7 +779,7 @@ static int try_to_unuse(unsigned int type) if (start_mm == &init_mm) shmem = shmem_unuse(entry, page); else - retval = unuse_process(start_mm, entry, page); + retval = unuse_mm(start_mm, entry, page); } if (*swap_map > 1) { int set_start_mm = (*swap_map >= swcount); @@ -747,7 +794,8 @@ static int try_to_unuse(unsigned int type) while (*swap_map > 1 && !retval && (p = p->next) != &start_mm->mmlist) { mm = list_entry(p, struct mm_struct, mmlist); - atomic_inc(&mm->mm_users); + if (!atomic_inc_not_zero(&mm->mm_users)) + continue; spin_unlock(&mmlist_lock); mmput(prev_mm); prev_mm = mm; @@ -761,7 +809,7 @@ static int try_to_unuse(unsigned int type) set_start_mm = 1; shmem = shmem_unuse(entry, page); } else - retval = unuse_process(mm, entry, page); + retval = unuse_mm(mm, entry, page); if (set_start_mm && *swap_map < swcount) { mmput(new_start_mm); atomic_inc(&mm->mm_users); @@ -795,9 +843,9 @@ static int try_to_unuse(unsigned int type) * report them; but do report if we reset SWAP_MAP_MAX. */ if (*swap_map == SWAP_MAP_MAX) { - swap_device_lock(si); + spin_lock(&swap_lock); *swap_map = 1; - swap_device_unlock(si); + spin_unlock(&swap_lock); reset_overflow = 1; } @@ -860,6 +908,26 @@ static int try_to_unuse(unsigned int type) return retval; } +/* + * After a successful try_to_unuse, if no swap is now in use, we know + * we can empty the mmlist. swap_lock must be held on entry and exit. + * Note that mmlist_lock nests inside swap_lock, and an mm must be + * added to the mmlist just after page_duplicate - before would be racy. + */ +static void drain_mmlist(void) +{ + struct list_head *p, *next; + unsigned int i; + + for (i = 0; i < nr_swapfiles; i++) + if (swap_info[i].inuse_pages) + return; + spin_lock(&mmlist_lock); + list_for_each_safe(p, next, &init_mm.mmlist) + list_del_init(p); + spin_unlock(&mmlist_lock); +} + /* * Use this swapdev's extent info to locate the (PAGE_SIZE) block which * corresponds to page offset `offset'. @@ -876,15 +944,32 @@ sector_t map_swap_page(struct swap_info_struct *sis, pgoff_t offset) offset < (se->start_page + se->nr_pages)) { return se->start_block + (offset - se->start_page); } - lh = se->list.prev; + lh = se->list.next; if (lh == &sis->extent_list) - lh = lh->prev; + lh = lh->next; se = list_entry(lh, struct swap_extent, list); sis->curr_swap_extent = se; BUG_ON(se == start_se); /* It *must* be present */ } } +#ifdef CONFIG_SOFTWARE_SUSPEND +/* + * Get the (PAGE_SIZE) block corresponding to given offset on the swapdev + * corresponding to given index in swap_info (swap type). + */ +sector_t swapdev_block(int swap_type, pgoff_t offset) +{ + struct swap_info_struct *sis; + + if (swap_type >= nr_swapfiles) + return 0; + + sis = swap_info + swap_type; + return (sis->flags & SWP_WRITEOK) ? map_swap_page(sis, offset) : 0; +} +#endif /* CONFIG_SOFTWARE_SUSPEND */ + /* * Free all of a swapdev's extent information */ @@ -898,15 +983,13 @@ static void destroy_swap_extents(struct swap_info_struct *sis) list_del(&se->list); kfree(se); } - sis->nr_extents = 0; } /* * Add a block range (and the corresponding page range) into this swapdev's - * extent list. The extent list is kept sorted in block order. + * extent list. The extent list is kept sorted in page order. * - * This function rather assumes that it is called in ascending sector_t order. - * It doesn't look for extent coalescing opportunities. + * This function rather assumes that it is called in ascending page order. */ static int add_swap_extent(struct swap_info_struct *sis, unsigned long start_page, @@ -916,16 +999,15 @@ add_swap_extent(struct swap_info_struct *sis, unsigned long start_page, struct swap_extent *new_se; struct list_head *lh; - lh = sis->extent_list.next; /* The highest-addressed block */ - while (lh != &sis->extent_list) { + lh = sis->extent_list.prev; /* The highest page extent */ + if (lh != &sis->extent_list) { se = list_entry(lh, struct swap_extent, list); - if (se->start_block + se->nr_pages == start_block && - se->start_page + se->nr_pages == start_page) { + BUG_ON(se->start_page + se->nr_pages != start_page); + if (se->start_block + se->nr_pages == start_block) { /* Merge it */ se->nr_pages += nr_pages; return 0; } - lh = lh->next; } /* @@ -938,16 +1020,8 @@ add_swap_extent(struct swap_info_struct *sis, unsigned long start_page, new_se->nr_pages = nr_pages; new_se->start_block = start_block; - lh = sis->extent_list.prev; /* The lowest block */ - while (lh != &sis->extent_list) { - se = list_entry(lh, struct swap_extent, list); - if (se->start_block > start_block) - break; - lh = lh->prev; - } - list_add_tail(&new_se->list, lh); - sis->nr_extents++; - return 0; + list_add_tail(&new_se->list, &sis->extent_list); + return 1; } /* @@ -970,7 +1044,7 @@ add_swap_extent(struct swap_info_struct *sis, unsigned long start_page, * requirements, they are simply tossed out - we will never use those blocks * for swapping. * - * For S_ISREG swapfiles we hold i_sem across the life of the swapon. This + * For S_ISREG swapfiles we set S_SWAPFILE across the life of the swapon. This * prevents root from shooting her foot off by ftruncating an in-use swapfile, * which will scribble on the fs. * @@ -981,7 +1055,7 @@ add_swap_extent(struct swap_info_struct *sis, unsigned long start_page, * This is extremely effective. The average number of iterations in * map_swap_page() has been measured at about 0.3 per page. - akpm. */ -static int setup_swap_extents(struct swap_info_struct *sis) +static int setup_swap_extents(struct swap_info_struct *sis, sector_t *span) { struct inode *inode; unsigned blocks_per_page; @@ -989,11 +1063,15 @@ static int setup_swap_extents(struct swap_info_struct *sis) unsigned blkbits; sector_t probe_block; sector_t last_block; + sector_t lowest_block = -1; + sector_t highest_block = 0; + int nr_extents = 0; int ret; inode = sis->swap_file->f_mapping->host; if (S_ISBLK(inode->i_mode)) { ret = add_swap_extent(sis, 0, sis->max, 0); + *span = sis->pages; goto done; } @@ -1038,22 +1116,32 @@ static int setup_swap_extents(struct swap_info_struct *sis) } } + first_block >>= (PAGE_SHIFT - blkbits); + if (page_no) { /* exclude the header page */ + if (first_block < lowest_block) + lowest_block = first_block; + if (first_block > highest_block) + highest_block = first_block; + } + /* * We found a PAGE_SIZE-length, PAGE_SIZE-aligned run of blocks */ - ret = add_swap_extent(sis, page_no, 1, - first_block >> (PAGE_SHIFT - blkbits)); - if (ret) + ret = add_swap_extent(sis, page_no, 1, first_block); + if (ret < 0) goto out; + nr_extents += ret; page_no++; probe_block += blocks_per_page; reprobe: continue; } - ret = 0; + ret = nr_extents; + *span = 1 + highest_block - lowest_block; if (page_no == 0) - ret = -EINVAL; + page_no = 1; /* force Empty message */ sis->max = page_no; + sis->pages = page_no - 1; sis->highest_bit = page_no - 1; done: sis->curr_swap_extent = list_entry(sis->extent_list.prev, @@ -1075,7 +1163,7 @@ int page_queue_congested(struct page *page) BUG_ON(!PageLocked(page)); /* It pins the swap_info_struct */ if (PageSwapCache(page)) { - swp_entry_t entry = { .val = page->private }; + swp_entry_t entry = { .val = page_private(page) }; struct swap_info_struct *sis; sis = get_swap_info_struct(swp_type(entry)); @@ -1113,7 +1201,7 @@ asmlinkage long sys_swapoff(const char __user * specialfile) mapping = victim->f_mapping; prev = -1; - swap_list_lock(); + spin_lock(&swap_lock); for (type = swap_list.head; type >= 0; type = swap_info[type].next) { p = swap_info + type; if ((p->flags & SWP_ACTIVE) == SWP_ACTIVE) { @@ -1124,14 +1212,14 @@ asmlinkage long sys_swapoff(const char __user * specialfile) } if (type < 0) { err = -EINVAL; - swap_list_unlock(); + spin_unlock(&swap_lock); goto out_dput; } if (!security_vm_enough_memory(p->pages)) vm_unacct_memory(p->pages); else { err = -ENOMEM; - swap_list_unlock(); + spin_unlock(&swap_lock); goto out_dput; } if (prev < 0) { @@ -1146,18 +1234,15 @@ asmlinkage long sys_swapoff(const char __user * specialfile) nr_swap_pages -= p->pages; total_swap_pages -= p->pages; p->flags &= ~SWP_WRITEOK; - swap_list_unlock(); + spin_unlock(&swap_lock); + current->flags |= PF_SWAPOFF; err = try_to_unuse(type); current->flags &= ~PF_SWAPOFF; - /* wait for any unplug function to finish */ - down_write(&swap_unplug_sem); - up_write(&swap_unplug_sem); - if (err) { /* re-insert swap space back into swap_list */ - swap_list_lock(); + spin_lock(&swap_lock); for (prev = -1, i = swap_list.head; i >= 0; prev = i, i = swap_info[i].next) if (p->prio >= swap_info[i].prio) break; @@ -1169,22 +1254,35 @@ asmlinkage long sys_swapoff(const char __user * specialfile) nr_swap_pages += p->pages; total_swap_pages += p->pages; p->flags |= SWP_WRITEOK; - swap_list_unlock(); + spin_unlock(&swap_lock); goto out_dput; } - down(&swapon_sem); - swap_list_lock(); - swap_device_lock(p); + + /* wait for any unplug function to finish */ + down_write(&swap_unplug_sem); + up_write(&swap_unplug_sem); + + destroy_swap_extents(p); + mutex_lock(&swapon_mutex); + spin_lock(&swap_lock); + drain_mmlist(); + + /* wait for anyone still in scan_swap_map */ + p->highest_bit = 0; /* cuts scans short */ + while (p->flags >= SWP_SCANNING) { + spin_unlock(&swap_lock); + schedule_timeout_uninterruptible(1); + spin_lock(&swap_lock); + } + swap_file = p->swap_file; p->swap_file = NULL; p->max = 0; swap_map = p->swap_map; p->swap_map = NULL; p->flags = 0; - destroy_swap_extents(p); - swap_device_unlock(p); - swap_list_unlock(); - up(&swapon_sem); + spin_unlock(&swap_lock); + mutex_unlock(&swapon_mutex); vfree(swap_map); inode = mapping->host; if (S_ISBLK(inode->i_mode)) { @@ -1192,9 +1290,9 @@ asmlinkage long sys_swapoff(const char __user * specialfile) set_blocksize(bdev, p->old_block_size); bd_release(bdev); } else { - down(&inode->i_sem); + mutex_lock(&inode->i_mutex); inode->i_flags &= ~S_SWAPFILE; - up(&inode->i_sem); + mutex_unlock(&inode->i_mutex); } filp_close(swap_file, NULL); err = 0; @@ -1213,12 +1311,15 @@ static void *swap_start(struct seq_file *swap, loff_t *pos) int i; loff_t l = *pos; - down(&swapon_sem); + mutex_lock(&swapon_mutex); + + if (!l) + return SEQ_START_TOKEN; for (i = 0; i < nr_swapfiles; i++, ptr++) { if (!(ptr->flags & SWP_USED) || !ptr->swap_map) continue; - if (!l--) + if (!--l) return ptr; } @@ -1227,10 +1328,17 @@ static void *swap_start(struct seq_file *swap, loff_t *pos) static void *swap_next(struct seq_file *swap, void *v, loff_t *pos) { - struct swap_info_struct *ptr = v; + struct swap_info_struct *ptr; struct swap_info_struct *endptr = swap_info + nr_swapfiles; - for (++ptr; ptr < endptr; ptr++) { + if (v == SEQ_START_TOKEN) + ptr = swap_info; + else { + ptr = v; + ptr++; + } + + for (; ptr < endptr; ptr++) { if (!(ptr->flags & SWP_USED) || !ptr->swap_map) continue; ++*pos; @@ -1242,7 +1350,7 @@ static void *swap_next(struct seq_file *swap, void *v, loff_t *pos) static void swap_stop(struct seq_file *swap, void *v) { - up(&swapon_sem); + mutex_unlock(&swapon_mutex); } static int swap_show(struct seq_file *swap, void *v) @@ -1251,14 +1359,16 @@ static int swap_show(struct seq_file *swap, void *v) struct file *file; int len; - if (v == swap_info) - seq_puts(swap, "Filename\t\t\t\tType\t\tSize\tUsed\tPriority\n"); + if (ptr == SEQ_START_TOKEN) { + seq_puts(swap,"Filename\t\t\t\tType\t\tSize\tUsed\tPriority\n"); + return 0; + } file = ptr->swap_file; - len = seq_path(swap, file->f_vfsmnt, file->f_dentry, " \t\n\\"); - seq_printf(swap, "%*s%s\t%d\t%ld\t%d\n", + len = seq_path(swap, file->f_path.mnt, file->f_path.dentry, " \t\n\\"); + seq_printf(swap, "%*s%s\t%u\t%u\t%d\n", len < 40 ? 40 - len : 1, " ", - S_ISBLK(file->f_dentry->d_inode->i_mode) ? + S_ISBLK(file->f_path.dentry->d_inode->i_mode) ? "partition" : "file\t", ptr->pages << (PAGE_SHIFT - 10), ptr->inuse_pages << (PAGE_SHIFT - 10), @@ -1266,7 +1376,7 @@ static int swap_show(struct seq_file *swap, void *v) return 0; } -static struct seq_operations swaps_op = { +static const struct seq_operations swaps_op = { .start = swap_start, .next = swap_next, .stop = swap_stop, @@ -1278,7 +1388,7 @@ static int swaps_open(struct inode *inode, struct file *file) return seq_open(file, &swaps_op); } -static struct file_operations proc_swaps_operations = { +static const struct file_operations proc_swaps_operations = { .open = swaps_open, .read = seq_read, .llseek = seq_lseek, @@ -1315,7 +1425,9 @@ asmlinkage long sys_swapon(const char __user * specialfile, int swap_flags) static int least_priority; union swap_header *swap_header = NULL; int swap_header_version; - int nr_good_pages = 0; + unsigned int nr_good_pages = 0; + int nr_extents = 0; + sector_t span; unsigned long maxpages = 1; int swapfilesize; unsigned short *swap_map; @@ -1325,33 +1437,20 @@ asmlinkage long sys_swapon(const char __user * specialfile, int swap_flags) if (!capable(CAP_SYS_ADMIN)) return -EPERM; - swap_list_lock(); + spin_lock(&swap_lock); p = swap_info; for (type = 0 ; type < nr_swapfiles ; type++,p++) if (!(p->flags & SWP_USED)) break; error = -EPERM; - /* - * Test if adding another swap device is possible. There are - * two limiting factors: 1) the number of bits for the swap - * type swp_entry_t definition and 2) the number of bits for - * the swap type in the swap ptes as defined by the different - * architectures. To honor both limitations a swap entry - * with swap offset 0 and swap type ~0UL is created, encoded - * to a swap pte, decoded to a swp_entry_t again and finally - * the swap type part is extracted. This will mask all bits - * from the initial ~0UL that can't be encoded in either the - * swp_entry_t or the architecture definition of a swap pte. - */ - if (type > swp_type(pte_to_swp_entry(swp_entry_to_pte(swp_entry(~0UL,0))))) { - swap_list_unlock(); + if (type >= MAX_SWAPFILES) { + spin_unlock(&swap_lock); goto out; } if (type >= nr_swapfiles) nr_swapfiles = type+1; INIT_LIST_HEAD(&p->extent_list); p->flags = SWP_USED; - p->nr_extents = 0; p->swap_file = NULL; p->old_block_size = 0; p->swap_map = NULL; @@ -1359,7 +1458,6 @@ asmlinkage long sys_swapon(const char __user * specialfile, int swap_flags) p->highest_bit = 0; p->cluster_nr = 0; p->inuse_pages = 0; - p->sdev_lock = SPIN_LOCK_UNLOCKED; p->next = -1; if (swap_flags & SWAP_FLAG_PREFER) { p->prio = @@ -1367,7 +1465,7 @@ asmlinkage long sys_swapon(const char __user * specialfile, int swap_flags) } else { p->prio = --least_priority; } - swap_list_unlock(); + spin_unlock(&swap_lock); name = getname(specialfile); error = PTR_ERR(name); if (IS_ERR(name)) { @@ -1401,6 +1499,7 @@ asmlinkage long sys_swapon(const char __user * specialfile, int swap_flags) error = bd_claim(bdev, sys_swapon); if (error < 0) { bdev = NULL; + error = -EINVAL; goto bad_swap; } p->old_block_size = block_size(bdev); @@ -1410,7 +1509,7 @@ asmlinkage long sys_swapon(const char __user * specialfile, int swap_flags) p->bdev = bdev; } else if (S_ISREG(inode->i_mode)) { p->bdev = inode->i_sb->s_bdev; - down(&inode->i_sem); + mutex_lock(&inode->i_mutex); did_down = 1; if (IS_SWAPFILE(inode)) { error = -EBUSY; @@ -1429,8 +1528,7 @@ asmlinkage long sys_swapon(const char __user * specialfile, int swap_flags) error = -EINVAL; goto bad_swap; } - page = read_cache_page(mapping, 0, - (filler_t *)mapping->a_ops->readpage, swap_file); + page = read_mapping_page(mapping, 0, swap_file); if (IS_ERR(page)) { error = PTR_ERR(page); goto bad_swap; @@ -1446,7 +1544,7 @@ asmlinkage long sys_swapon(const char __user * specialfile, int swap_flags) else if (!memcmp("SWAPSPACE2",swap_header->magic.magic,10)) swap_header_version = 2; else { - printk("Unable to find swap-space signature\n"); + printk(KERN_ERR "Unable to find swap-space signature\n"); error = -EINVAL; goto bad_swap; } @@ -1469,6 +1567,8 @@ asmlinkage long sys_swapon(const char __user * specialfile, int swap_flags) } p->lowest_bit = 1; + p->cluster_next = 1; + /* * Find out how many pages are allowed for a single swap * device. There are two limiting factors: 1) the number of @@ -1489,9 +1589,18 @@ asmlinkage long sys_swapon(const char __user * specialfile, int swap_flags) p->highest_bit = maxpages - 1; error = -EINVAL; + if (!maxpages) + goto bad_swap; + if (swapfilesize && maxpages > swapfilesize) { + printk(KERN_WARNING + "Swap area shorter than signature indicates\n"); + goto bad_swap; + } + if (swap_header->info.nr_badpages && S_ISREG(inode->i_mode)) + goto bad_swap; if (swap_header->info.nr_badpages > MAX_SWAP_BADPAGES) goto bad_swap; - + /* OK, set up the swap map and apply the bad block list */ if (!(p->swap_map = vmalloc(maxpages * sizeof(short)))) { error = -ENOMEM; @@ -1500,48 +1609,47 @@ asmlinkage long sys_swapon(const char __user * specialfile, int swap_flags) error = 0; memset(p->swap_map, 0, maxpages * sizeof(short)); - for (i=0; iinfo.nr_badpages; i++) { - int page = swap_header->info.badpages[i]; - if (page <= 0 || page >= swap_header->info.last_page) + for (i = 0; i < swap_header->info.nr_badpages; i++) { + int page_nr = swap_header->info.badpages[i]; + if (page_nr <= 0 || page_nr >= swap_header->info.last_page) error = -EINVAL; else - p->swap_map[page] = SWAP_MAP_BAD; + p->swap_map[page_nr] = SWAP_MAP_BAD; } nr_good_pages = swap_header->info.last_page - swap_header->info.nr_badpages - 1 /* header page */; - if (error) + if (error) goto bad_swap; } - - if (swapfilesize && maxpages > swapfilesize) { - printk(KERN_WARNING - "Swap area shorter than signature indicates\n"); - error = -EINVAL; - goto bad_swap; + + if (nr_good_pages) { + p->swap_map[0] = SWAP_MAP_BAD; + p->max = maxpages; + p->pages = nr_good_pages; + nr_extents = setup_swap_extents(p, &span); + if (nr_extents < 0) { + error = nr_extents; + goto bad_swap; + } + nr_good_pages = p->pages; } if (!nr_good_pages) { printk(KERN_WARNING "Empty swap-file\n"); error = -EINVAL; goto bad_swap; } - p->swap_map[0] = SWAP_MAP_BAD; - p->max = maxpages; - p->pages = nr_good_pages; - - error = setup_swap_extents(p); - if (error) - goto bad_swap; - down(&swapon_sem); - swap_list_lock(); - swap_device_lock(p); + mutex_lock(&swapon_mutex); + spin_lock(&swap_lock); p->flags = SWP_ACTIVE; nr_swap_pages += nr_good_pages; total_swap_pages += nr_good_pages; - printk(KERN_INFO "Adding %dk swap on %s. Priority:%d extents:%d\n", - nr_good_pages<<(PAGE_SHIFT-10), name, - p->prio, p->nr_extents); + + printk(KERN_INFO "Adding %uk swap on %s. " + "Priority:%d extents:%d across:%lluk\n", + nr_good_pages<<(PAGE_SHIFT-10), name, p->prio, + nr_extents, (unsigned long long)span<<(PAGE_SHIFT-10)); /* insert swap space into swap_list: */ prev = -1; @@ -1557,9 +1665,8 @@ asmlinkage long sys_swapon(const char __user * specialfile, int swap_flags) } else { swap_info[prev].next = p - swap_info; } - swap_device_unlock(p); - swap_list_unlock(); - up(&swapon_sem); + spin_unlock(&swap_lock); + mutex_unlock(&swapon_mutex); error = 0; goto out; bad_swap: @@ -1567,18 +1674,17 @@ bad_swap: set_blocksize(bdev, p->old_block_size); bd_release(bdev); } + destroy_swap_extents(p); bad_swap_2: - swap_list_lock(); + spin_lock(&swap_lock); swap_map = p->swap_map; p->swap_file = NULL; p->swap_map = NULL; p->flags = 0; if (!(swap_flags & SWAP_FLAG_PREFER)) ++least_priority; - swap_list_unlock(); - destroy_swap_extents(p); - if (swap_map) - vfree(swap_map); + spin_unlock(&swap_lock); + vfree(swap_map); if (swap_file) filp_close(swap_file, NULL); out: @@ -1591,7 +1697,7 @@ out: if (did_down) { if (!error) inode->i_flags |= S_SWAPFILE; - up(&inode->i_sem); + mutex_unlock(&inode->i_mutex); } return error; } @@ -1601,7 +1707,7 @@ void si_swapinfo(struct sysinfo *val) unsigned int i; unsigned long nr_to_be_unused = 0; - swap_list_lock(); + spin_lock(&swap_lock); for (i = 0; i < nr_swapfiles; i++) { if (!(swap_info[i].flags & SWP_USED) || (swap_info[i].flags & SWP_WRITEOK)) @@ -1610,7 +1716,7 @@ void si_swapinfo(struct sysinfo *val) } val->freeswap = nr_swap_pages + nr_to_be_unused; val->totalswap = total_swap_pages + nr_to_be_unused; - swap_list_unlock(); + spin_unlock(&swap_lock); if (vx_flags(VXF_VIRT_MEM, 0)) vx_vsi_swapinfo(val); } @@ -1627,13 +1733,16 @@ int swap_duplicate(swp_entry_t entry) unsigned long offset, type; int result = 0; + if (is_migration_entry(entry)) + return 1; + type = swp_type(entry); if (type >= nr_swapfiles) goto bad_file; p = type + swap_info; offset = swp_offset(entry); - swap_device_lock(p); + spin_lock(&swap_lock); if (offset < p->max && p->swap_map[offset]) { if (p->swap_map[offset] < SWAP_MAP_MAX - 1) { p->swap_map[offset]++; @@ -1645,7 +1754,7 @@ int swap_duplicate(swp_entry_t entry) result = 1; } } - swap_device_unlock(p); + spin_unlock(&swap_lock); out: return result; @@ -1661,23 +1770,24 @@ get_swap_info_struct(unsigned type) } /* - * swap_device_lock prevents swap_map being freed. Don't grab an extra + * swap_lock prevents swap_map being freed. Don't grab an extra * reference on the swaphandle, it doesn't matter if it becomes unused. */ int valid_swaphandles(swp_entry_t entry, unsigned long *offset) { - int ret = 0, i = 1 << page_cluster; + int our_page_cluster = page_cluster; + int ret = 0, i = 1 << our_page_cluster; unsigned long toff; struct swap_info_struct *swapdev = swp_type(entry) + swap_info; - if (!page_cluster) /* no readahead */ + if (!our_page_cluster) /* no readahead */ return 0; - toff = (swp_offset(entry) >> page_cluster) << page_cluster; + toff = (swp_offset(entry) >> our_page_cluster) << our_page_cluster; if (!toff) /* first page is swap header */ toff++, i--; *offset = toff; - swap_device_lock(swapdev); + spin_lock(&swap_lock); do { /* Don't read-ahead past the end of the swap area */ if (toff >= swapdev->max) @@ -1690,6 +1800,6 @@ int valid_swaphandles(swp_entry_t entry, unsigned long *offset) toff++; ret++; } while (--i); - swap_device_unlock(swapdev); + spin_unlock(&swap_lock); return ret; }