vserver 1.9.5.x5

[linux-2.6.git] / mm / msync.c

diff --git a/mm/msync.c b/mm/msync.c
index 2aaad04..b387bac 100644 (file)
--- a/mm/msync.c
+++ b/mm/msync.c
@@ -12,6 +12,7 @@
 #include <linux/mm.h>
 #include <linux/mman.h>
 #include <linux/hugetlb.h>
+#include <linux/syscalls.h>
 
 #include <asm/pgtable.h>
 #include <asm/tlbflush.h>
@@ -66,13 +67,39 @@ static int filemap_sync_pte_range(pmd_t * pmd,
        return error;
 }
 
-static inline int filemap_sync_pmd_range(pgd_t * pgd,
+static inline int filemap_sync_pmd_range(pud_t * pud,
        unsigned long address, unsigned long end, 
        struct vm_area_struct *vma, unsigned int flags)
 {
        pmd_t * pmd;
        int error;
 
+       if (pud_none(*pud))
+               return 0;
+       if (pud_bad(*pud)) {
+               pud_ERROR(*pud);
+               pud_clear(pud);
+               return 0;
+       }
+       pmd = pmd_offset(pud, address);
+       if ((address & PUD_MASK) != (end & PUD_MASK))
+               end = (address & PUD_MASK) + PUD_SIZE;
+       error = 0;
+       do {
+               error |= filemap_sync_pte_range(pmd, address, end, vma, flags);
+               address = (address + PMD_SIZE) & PMD_MASK;
+               pmd++;
+       } while (address && (address < end));
+       return error;
+}
+
+static inline int filemap_sync_pud_range(pgd_t *pgd,
+       unsigned long address, unsigned long end,
+       struct vm_area_struct *vma, unsigned int flags)
+{
+       pud_t *pud;
+       int error;
+
        if (pgd_none(*pgd))
                return 0;
        if (pgd_bad(*pgd)) {
@@ -80,23 +107,25 @@ static inline int filemap_sync_pmd_range(pgd_t * pgd,
                pgd_clear(pgd);
                return 0;
        }
-       pmd = pmd_offset(pgd, address);
+       pud = pud_offset(pgd, address);
        if ((address & PGDIR_MASK) != (end & PGDIR_MASK))
                end = (address & PGDIR_MASK) + PGDIR_SIZE;
        error = 0;
        do {
-               error |= filemap_sync_pte_range(pmd, address, end, vma, flags);
-               address = (address + PMD_SIZE) & PMD_MASK;
-               pmd++;
+               error |= filemap_sync_pmd_range(pud, address, end, vma, flags);
+               address = (address + PUD_SIZE) & PUD_MASK;
+               pud++;
        } while (address && (address < end));
        return error;
 }
 
-static int filemap_sync(struct vm_area_struct * vma, unsigned long address,
-       size_t size, unsigned int flags)
+static int __filemap_sync(struct vm_area_struct *vma, unsigned long address,
+                       size_t size, unsigned int flags)
 {
-       pgd_t * dir;
+       pgd_t *pgd;
        unsigned long end = address + size;
+       unsigned long next;
+       int i;
        int error = 0;
 
        /* Aquire the lock early; it may be possible to avoid dropping
@@ -104,7 +133,7 @@ static int filemap_sync(struct vm_area_struct * vma, unsigned long address,
         */
        spin_lock(&vma->vm_mm->page_table_lock);
 
-       dir = pgd_offset(vma->vm_mm, address);
+       pgd = pgd_offset(vma->vm_mm, address);
        flush_cache_range(vma, address, end);
 
        /* For hugepages we can't go walking the page table normally,
@@ -115,11 +144,14 @@ static int filemap_sync(struct vm_area_struct * vma, unsigned long address,
 
        if (address >= end)
                BUG();
-       do {
-               error |= filemap_sync_pmd_range(dir, address, end, vma, flags);
-               address = (address + PGDIR_SIZE) & PGDIR_MASK;
-               dir++;
-       } while (address && (address < end));
+       for (i = pgd_index(address); i <= pgd_index(end-1); i++) {
+               next = (address + PGDIR_SIZE) & PGDIR_MASK;
+               if (next <= address || next > end)
+                       next = end;
+               error |= filemap_sync_pud_range(pgd, address, next, vma, flags);
+               address = next;
+               pgd++;
+       }
        /*
         * Why flush ? filemap_sync_pte already flushed the tlbs with the
         * dirty bits.
@@ -131,6 +163,31 @@ static int filemap_sync(struct vm_area_struct * vma, unsigned long address,
        return error;
 }
 
+#ifdef CONFIG_PREEMPT
+static int filemap_sync(struct vm_area_struct *vma, unsigned long address,
+                       size_t size, unsigned int flags)
+{
+       const size_t chunk = 64 * 1024; /* bytes */
+       int error = 0;
+
+       while (size) {
+               size_t sz = min(size, chunk);
+
+               error |= __filemap_sync(vma, address, sz, flags);
+               cond_resched();
+               address += sz;
+               size -= sz;
+       }
+       return error;
+}
+#else
+static int filemap_sync(struct vm_area_struct *vma, unsigned long address,
+                       size_t size, unsigned int flags)
+{
+       return __filemap_sync(vma, address, size, flags);
+}
+#endif
+
 /*
  * MS_SYNC syncs the entire file - including mappings.
  *
@@ -158,9 +215,12 @@ static int msync_interval(struct vm_area_struct * vma,
                        struct address_space *mapping = file->f_mapping;
                        int err;
 
-                       down(&mapping->host->i_sem);
                        ret = filemap_fdatawrite(mapping);
                        if (file->f_op && file->f_op->fsync) {
+                               /*
+                                * We don't take i_sem here because mmap_sem
+                                * is already held.
+                                */
                                err = file->f_op->fsync(file,file->f_dentry,1);
                                if (err && !ret)
                                        ret = err;
@@ -168,7 +228,6 @@ static int msync_interval(struct vm_area_struct * vma,
                        err = filemap_fdatawait(mapping);
                        if (!ret)
                                ret = err;
-                       up(&mapping->host->i_sem);
                }
        }
        return ret;
@@ -180,6 +239,9 @@ asmlinkage long sys_msync(unsigned long start, size_t len, int flags)
        struct vm_area_struct * vma;
        int unmapped_error, error = -EINVAL;
 
+       if (flags & MS_SYNC)
+               current->flags |= PF_SYNCWRITE;
+
        down_read(&current->mm->mmap_sem);
        if (flags & ~(MS_ASYNC | MS_INVALIDATE | MS_SYNC))
                goto out;
@@ -230,5 +292,6 @@ asmlinkage long sys_msync(unsigned long start, size_t len, int flags)
        }
 out:
        up_read(&current->mm->mmap_sem);
+       current->flags &= ~PF_SYNCWRITE;
        return error;
 }