mm/filemap.c

   1 /*
   2  *      linux/mm/filemap.c
   3  *
   4  * Copyright (C) 1994-1999  Linus Torvalds
   5  */
   6
   7 /*
   8  * This file handles the generic file mmap semantics used by
   9  * most "normal" filesystems (but you don't /have/ to use this:
  10  * the NFS filesystem used to do this differently, for example)
  11  */
  12 #include <linux/config.h>
  13 #include <linux/module.h>
  14 #include <linux/slab.h>
  15 #include <linux/compiler.h>
  16 #include <linux/fs.h>
  17 #include <linux/aio.h>
  18 #include <linux/kernel_stat.h>
  19 #include <linux/mm.h>
  20 #include <linux/swap.h>
  21 #include <linux/mman.h>
  22 #include <linux/pagemap.h>
  23 #include <linux/file.h>
  24 #include <linux/uio.h>
  25 #include <linux/hash.h>
  26 #include <linux/writeback.h>
  27 #include <linux/pagevec.h>
  28 #include <linux/blkdev.h>
  29 #include <linux/security.h>
  30 /*
  31  * This is needed for the following functions:
  32  *  - try_to_release_page
  33  *  - block_invalidatepage
  34  *  - generic_osync_inode
  35  *
  36  * FIXME: remove all knowledge of the buffer layer from the core VM
  37  */
  38 #include <linux/buffer_head.h> /* for generic_osync_inode */
  39
  40 #include <asm/uaccess.h>
  41 #include <asm/mman.h>
  42
  43 /*
  44  * Shared mappings implemented 30.11.1994. It's not fully working yet,
  45  * though.
  46  *
  47  * Shared mappings now work. 15.8.1995  Bruno.
  48  *
  49  * finished 'unifying' the page and buffer cache and SMP-threaded the
  50  * page-cache, 21.05.1999, Ingo Molnar <mingo@redhat.com>
  51  *
  52  * SMP-threaded pagemap-LRU 1999, Andrea Arcangeli <andrea@suse.de>
  53  */
  54
  55 /*
  56  * Lock ordering:
  57  *
  58  *  ->i_mmap_lock               (vmtruncate)
  59  *    ->private_lock            (__free_pte->__set_page_dirty_buffers)
  60  *      ->swap_list_lock
  61  *        ->swap_device_lock    (exclusive_swap_page, others)
  62  *          ->mapping->tree_lock
  63  *
  64  *  ->i_sem
  65  *    ->i_mmap_lock             (truncate->unmap_mapping_range)
  66  *
  67  *  ->mmap_sem
  68  *    ->i_mmap_lock
  69  *      ->page_table_lock       (various places, mainly in mmap.c)
  70  *        ->mapping->tree_lock  (arch-dependent flush_dcache_mmap_lock)
  71  *
  72  *  ->mmap_sem
  73  *    ->lock_page               (access_process_vm)
  74  *
  75  *  ->mmap_sem
  76  *    ->i_sem                   (msync)
  77  *
  78  *  ->i_sem
  79  *    ->i_alloc_sem             (various)
  80  *
  81  *  ->inode_lock
  82  *    ->sb_lock                 (fs/fs-writeback.c)
  83  *    ->mapping->tree_lock      (__sync_single_inode)
  84  *
  85  *  ->page_table_lock
  86  *    ->swap_device_lock        (try_to_unmap_one)
  87  *    ->private_lock            (try_to_unmap_one)
  88  *    ->tree_lock               (try_to_unmap_one)
  89  *    ->zone.lru_lock           (follow_page->mark_page_accessed)
  90  *
  91  *  ->task->proc_lock
  92  *    ->dcache_lock             (proc_pid_lookup)
  93  */
  94
  95 /*
  96  * Remove a page from the page cache and free it. Caller has to make
  97  * sure the page is locked and that nobody else uses it - or that usage
  98  * is safe.  The caller must hold a write_lock on the mapping's tree_lock.
  99  */
 100 void __remove_from_page_cache(struct page *page)
 101 {
 102         struct address_space *mapping = page->mapping;
 103
 104         radix_tree_delete(&mapping->page_tree, page->index);
 105         page->mapping = NULL;
 106         mapping->nrpages--;
 107         pagecache_acct(-1);
 108 }
 109
 110 void remove_from_page_cache(struct page *page)
 111 {
 112         struct address_space *mapping = page->mapping;
 113
 114         if (unlikely(!PageLocked(page)))
 115                 PAGE_BUG(page);
 116
 117         spin_lock_irq(&mapping->tree_lock);
 118         __remove_from_page_cache(page);
 119         spin_unlock_irq(&mapping->tree_lock);
 120 }
 121
 122 static inline int sync_page(struct page *page)
 123 {
 124         struct address_space *mapping;
 125
 126         /*
 127          * FIXME, fercrissake.  What is this barrier here for?
 128          */
 129         smp_mb();
 130         mapping = page_mapping(page);
 131         if (mapping && mapping->a_ops && mapping->a_ops->sync_page)
 132                 return mapping->a_ops->sync_page(page);
 133         return 0;
 134 }
 135
 136 /**
 137  * filemap_fdatawrite - start writeback against all of a mapping's dirty pages
 138  * @mapping: address space structure to write
 139  *
 140  * If sync_mode is WB_SYNC_ALL then this is a "data integrity" operation, as
 141  * opposed to a regular memory * cleansing writeback.  The difference between
 142  * these two operations is that if a dirty page/buffer is encountered, it must
 143  * be waited upon, and not just skipped over.
 144  */
 145 static int __filemap_fdatawrite(struct address_space *mapping, int sync_mode)
 146 {
 147         int ret;
 148         struct writeback_control wbc = {
 149                 .sync_mode = sync_mode,
 150                 .nr_to_write = mapping->nrpages * 2,
 151         };
 152
 153         if (mapping->backing_dev_info->memory_backed)
 154                 return 0;
 155
 156         ret = do_writepages(mapping, &wbc);
 157         return ret;
 158 }
 159
 160 int filemap_fdatawrite(struct address_space *mapping)
 161 {
 162         return __filemap_fdatawrite(mapping, WB_SYNC_ALL);
 163 }
 164 EXPORT_SYMBOL(filemap_fdatawrite);
 165
 166 /*
 167  * This is a mostly non-blocking flush.  Not suitable for data-integrity
 168  * purposes - I/O may not be started against all dirty pages.
 169  */
 170 int filemap_flush(struct address_space *mapping)
 171 {
 172         return __filemap_fdatawrite(mapping, WB_SYNC_NONE);
 173 }
 174 EXPORT_SYMBOL(filemap_flush);
 175
 176 /*
 177  * Wait for writeback to complete against pages indexed by start->end
 178  * inclusive
 179  */
 180 static int wait_on_page_writeback_range(struct address_space *mapping,
 181                                 pgoff_t start, pgoff_t end)
 182 {
 183         struct pagevec pvec;
 184         int nr_pages;
 185         int ret = 0;
 186         pgoff_t index;
 187
 188         if (end < start)
 189                 return 0;
 190
 191         pagevec_init(&pvec, 0);
 192         index = start;
 193         while ((nr_pages = pagevec_lookup_tag(&pvec, mapping, &index,
 194                         PAGECACHE_TAG_WRITEBACK,
 195                         min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1))) {
 196                 unsigned i;
 197
 198                 for (i = 0; i < nr_pages; i++) {
 199                         struct page *page = pvec.pages[i];
 200
 201                         wait_on_page_writeback(page);
 202                         if (PageError(page))
 203                                 ret = -EIO;
 204                 }
 205                 pagevec_release(&pvec);
 206                 cond_resched();
 207         }
 208
 209         /* Check for outstanding write errors */
 210         if (test_and_clear_bit(AS_ENOSPC, &mapping->flags))
 211                 ret = -ENOSPC;
 212         if (test_and_clear_bit(AS_EIO, &mapping->flags))
 213                 ret = -EIO;
 214
 215         return ret;
 216 }
 217
 218 /**
 219  * filemap_fdatawait - walk the list of under-writeback pages of the given
 220  *     address space and wait for all of them.
 221  *
 222  * @mapping: address space structure to wait for
 223  */
 224 int filemap_fdatawait(struct address_space *mapping)
 225 {
 226         return wait_on_page_writeback_range(mapping, 0, -1);
 227 }
 228
 229 EXPORT_SYMBOL(filemap_fdatawait);
 230
 231 int filemap_write_and_wait(struct address_space *mapping)
 232 {
 233         int retval = 0;
 234
 235         if (mapping->nrpages) {
 236                 retval = filemap_fdatawrite(mapping);
 237                 if (retval == 0)
 238                         retval = filemap_fdatawait(mapping);
 239         }
 240         return retval;
 241 }
 242
 243 /*
 244  * This function is used to add newly allocated pagecache pages:
 245  * the page is new, so we can just run SetPageLocked() against it.
 246  * The other page state flags were set by rmqueue().
 247  *
 248  * This function does not add the page to the LRU.  The caller must do that.
 249  */
 250 int add_to_page_cache(struct page *page, struct address_space *mapping,
 251                 pgoff_t offset, int gfp_mask)
 252 {
 253         int error = radix_tree_preload(gfp_mask & ~__GFP_HIGHMEM);
 254
 255         if (error == 0) {
 256                 spin_lock_irq(&mapping->tree_lock);
 257                 error = radix_tree_insert(&mapping->page_tree, offset, page);
 258                 if (!error) {
 259                         page_cache_get(page);
 260                         SetPageLocked(page);
 261                         page->mapping = mapping;
 262                         page->index = offset;
 263                         mapping->nrpages++;
 264                         pagecache_acct(1);
 265                 }
 266                 spin_unlock_irq(&mapping->tree_lock);
 267                 radix_tree_preload_end();
 268         }
 269         return error;
 270 }
 271
 272 EXPORT_SYMBOL(add_to_page_cache);
 273
 274 int add_to_page_cache_lru(struct page *page, struct address_space *mapping,
 275                                 pgoff_t offset, int gfp_mask)
 276 {
 277         int ret = add_to_page_cache(page, mapping, offset, gfp_mask);
 278         if (ret == 0)
 279                 lru_cache_add(page);
 280         return ret;
 281 }
 282
 283 /*
 284  * In order to wait for pages to become available there must be
 285  * waitqueues associated with pages. By using a hash table of
 286  * waitqueues where the bucket discipline is to maintain all
 287  * waiters on the same queue and wake all when any of the pages
 288  * become available, and for the woken contexts to check to be
 289  * sure the appropriate page became available, this saves space
 290  * at a cost of "thundering herd" phenomena during rare hash
 291  * collisions.
 292  */
 293 struct page_wait_queue {
 294         struct page *page;
 295         int bit;
 296         wait_queue_t wait;
 297 };
 298
 299 static int page_wake_function(wait_queue_t *wait, unsigned mode, int sync, void *key)
 300 {
 301         struct page *page = key;
 302         struct page_wait_queue *wq;
 303
 304         wq = container_of(wait, struct page_wait_queue, wait);
 305         if (wq->page != page || test_bit(wq->bit, &page->flags))
 306                 return 0;
 307         else
 308                 return autoremove_wake_function(wait, mode, sync, NULL);
 309 }
 310
 311 #define __DEFINE_PAGE_WAIT(name, p, b, f)                               \
 312         struct page_wait_queue name = {                                 \
 313                 .page   = p,                                            \
 314                 .bit    = b,                                            \
 315                 .wait   = {                                             \
 316                         .task   = current,                              \
 317                         .func   = page_wake_function,                   \
 318                         .flags  = f,                                    \
 319                         .task_list = LIST_HEAD_INIT(name.wait.task_list),\
 320                 },                                                      \
 321         }
 322
 323 #define DEFINE_PAGE_WAIT(name, p, b)    __DEFINE_PAGE_WAIT(name, p, b, 0)
 324 #define DEFINE_PAGE_WAIT_EXCLUSIVE(name, p, b)                          \
 325                 __DEFINE_PAGE_WAIT(name, p, b, WQ_FLAG_EXCLUSIVE)
 326
 327 static wait_queue_head_t *page_waitqueue(struct page *page)
 328 {
 329         const struct zone *zone = page_zone(page);
 330
 331         return &zone->wait_table[hash_ptr(page, zone->wait_table_bits)];
 332 }
 333
 334 static void wake_up_page(struct page *page)
 335 {
 336         const unsigned int mode = TASK_UNINTERRUPTIBLE | TASK_INTERRUPTIBLE;
 337         wait_queue_head_t *waitqueue = page_waitqueue(page);
 338
 339         if (waitqueue_active(waitqueue))
 340                 __wake_up(waitqueue, mode, 1, page);
 341 }
 342
 343 void fastcall wait_on_page_bit(struct page *page, int bit_nr)
 344 {
 345         wait_queue_head_t *waitqueue = page_waitqueue(page);
 346         DEFINE_PAGE_WAIT(wait, page, bit_nr);
 347
 348         do {
 349                 prepare_to_wait(waitqueue, &wait.wait, TASK_UNINTERRUPTIBLE);
 350                 if (test_bit(bit_nr, &page->flags)) {
 351                         sync_page(page);
 352                         io_schedule();
 353                 }
 354         } while (test_bit(bit_nr, &page->flags));
 355         finish_wait(waitqueue, &wait.wait);
 356 }
 357
 358 EXPORT_SYMBOL(wait_on_page_bit);
 359
 360 /**
 361  * unlock_page() - unlock a locked page
 362  *
 363  * @page: the page
 364  *
 365  * Unlocks the page and wakes up sleepers in ___wait_on_page_locked().
 366  * Also wakes sleepers in wait_on_page_writeback() because the wakeup
 367  * mechananism between PageLocked pages and PageWriteback pages is shared.
 368  * But that's OK - sleepers in wait_on_page_writeback() just go back to sleep.
 369  *
 370  * The first mb is necessary to safely close the critical section opened by the
 371  * TestSetPageLocked(), the second mb is necessary to enforce ordering between
 372  * the clear_bit and the read of the waitqueue (to avoid SMP races with a
 373  * parallel wait_on_page_locked()).
 374  */
 375 void fastcall unlock_page(struct page *page)
 376 {
 377         smp_mb__before_clear_bit();
 378         if (!TestClearPageLocked(page))
 379                 BUG();
 380         smp_mb__after_clear_bit();
 381         wake_up_page(page);
 382 }
 383
 384 EXPORT_SYMBOL(unlock_page);
 385 EXPORT_SYMBOL(lock_page);
 386
 387 /*
 388  * End writeback against a page.
 389  */
 390 void end_page_writeback(struct page *page)
 391 {
 392         if (!TestClearPageReclaim(page) || rotate_reclaimable_page(page)) {
 393                 if (!test_clear_page_writeback(page))
 394                         BUG();
 395                 smp_mb__after_clear_bit();
 396         }
 397         wake_up_page(page);
 398 }
 399
 400 EXPORT_SYMBOL(end_page_writeback);
 401
 402 /*
 403  * Get a lock on the page, assuming we need to sleep to get it.
 404  *
 405  * Ugly: running sync_page() in state TASK_UNINTERRUPTIBLE is scary.  If some
 406  * random driver's requestfn sets TASK_RUNNING, we could busywait.  However
 407  * chances are that on the second loop, the block layer's plug list is empty,
 408  * so sync_page() will then return in state TASK_UNINTERRUPTIBLE.
 409  */
 410 void fastcall __lock_page(struct page *page)
 411 {
 412         wait_queue_head_t *wqh = page_waitqueue(page);
 413         DEFINE_PAGE_WAIT_EXCLUSIVE(wait, page, PG_locked);
 414
 415         while (TestSetPageLocked(page)) {
 416                 prepare_to_wait_exclusive(wqh, &wait.wait, TASK_UNINTERRUPTIBLE);
 417                 if (PageLocked(page)) {
 418                         sync_page(page);
 419                         io_schedule();
 420                 }
 421         }
 422         finish_wait(wqh, &wait.wait);
 423 }
 424
 425 EXPORT_SYMBOL(__lock_page);
 426
 427 /*
 428  * a rather lightweight function, finding and getting a reference to a
 429  * hashed page atomically.
 430  */
 431 struct page * find_get_page(struct address_space *mapping, unsigned long offset)
 432 {
 433         struct page *page;
 434
 435         /*
 436          * We scan the hash list read-only. Addition to and removal from
 437          * the hash-list needs a held write-lock.
 438          */
 439         spin_lock_irq(&mapping->tree_lock);
 440         page = radix_tree_lookup(&mapping->page_tree, offset);
 441         if (page)
 442                 page_cache_get(page);
 443         spin_unlock_irq(&mapping->tree_lock);
 444         return page;
 445 }
 446
 447 EXPORT_SYMBOL(find_get_page);
 448
 449 /*
 450  * Same as above, but trylock it instead of incrementing the count.
 451  */
 452 struct page *find_trylock_page(struct address_space *mapping, unsigned long offset)
 453 {
 454         struct page *page;
 455
 456         spin_lock_irq(&mapping->tree_lock);
 457         page = radix_tree_lookup(&mapping->page_tree, offset);
 458         if (page && TestSetPageLocked(page))
 459                 page = NULL;
 460         spin_unlock_irq(&mapping->tree_lock);
 461         return page;
 462 }
 463
 464 EXPORT_SYMBOL(find_trylock_page);
 465
 466 /**
 467  * find_lock_page - locate, pin and lock a pagecache page
 468  *
 469  * @mapping - the address_space to search
 470  * @offset - the page index
 471  *
 472  * Locates the desired pagecache page, locks it, increments its reference
 473  * count and returns its address.
 474  *
 475  * Returns zero if the page was not present. find_lock_page() may sleep.
 476  */
 477 struct page *find_lock_page(struct address_space *mapping,
 478                                 unsigned long offset)
 479 {
 480         struct page *page;
 481
 482         spin_lock_irq(&mapping->tree_lock);
 483 repeat:
 484         page = radix_tree_lookup(&mapping->page_tree, offset);
 485         if (page) {
 486                 page_cache_get(page);
 487                 if (TestSetPageLocked(page)) {
 488                         spin_unlock_irq(&mapping->tree_lock);
 489                         lock_page(page);
 490                         spin_lock_irq(&mapping->tree_lock);
 491
 492                         /* Has the page been truncated while we slept? */
 493                         if (page->mapping != mapping || page->index != offset) {
 494                                 unlock_page(page);
 495                                 page_cache_release(page);
 496                                 goto repeat;
 497                         }
 498                 }
 499         }
 500         spin_unlock_irq(&mapping->tree_lock);
 501         return page;
 502 }
 503
 504 EXPORT_SYMBOL(find_lock_page);
 505
 506 /**
 507  * find_or_create_page - locate or add a pagecache page
 508  *
 509  * @mapping - the page's address_space
 510  * @index - the page's index into the mapping
 511  * @gfp_mask - page allocation mode
 512  *
 513  * Locates a page in the pagecache.  If the page is not present, a new page
 514  * is allocated using @gfp_mask and is added to the pagecache and to the VM's
 515  * LRU list.  The returned page is locked and has its reference count
 516  * incremented.
 517  *
 518  * find_or_create_page() may sleep, even if @gfp_flags specifies an atomic
 519  * allocation!
 520  *
 521  * find_or_create_page() returns the desired page's address, or zero on
 522  * memory exhaustion.
 523  */
 524 struct page *find_or_create_page(struct address_space *mapping,
 525                 unsigned long index, unsigned int gfp_mask)
 526 {
 527         struct page *page, *cached_page = NULL;
 528         int err;
 529 repeat:
 530         page = find_lock_page(mapping, index);
 531         if (!page) {
 532                 if (!cached_page) {
 533                         cached_page = alloc_page(gfp_mask);
 534                         if (!cached_page)
 535                                 return NULL;
 536                 }
 537                 err = add_to_page_cache_lru(cached_page, mapping,
 538                                         index, gfp_mask);
 539                 if (!err) {
 540                         page = cached_page;
 541                         cached_page = NULL;
 542                 } else if (err == -EEXIST)
 543                         goto repeat;
 544         }
 545         if (cached_page)
 546                 page_cache_release(cached_page);
 547         return page;
 548 }
 549
 550 EXPORT_SYMBOL(find_or_create_page);
 551
 552 /**
 553  * find_get_pages - gang pagecache lookup
 554  * @mapping:    The address_space to search
 555  * @start:      The starting page index
 556  * @nr_pages:   The maximum number of pages
 557  * @pages:      Where the resulting pages are placed
 558  *
 559  * find_get_pages() will search for and return a group of up to
 560  * @nr_pages pages in the mapping.  The pages are placed at @pages.
 561  * find_get_pages() takes a reference against the returned pages.
 562  *
 563  * The search returns a group of mapping-contiguous pages with ascending
 564  * indexes.  There may be holes in the indices due to not-present pages.
 565  *
 566  * find_get_pages() returns the number of pages which were found.
 567  */
 568 unsigned find_get_pages(struct address_space *mapping, pgoff_t start,
 569                             unsigned int nr_pages, struct page **pages)
 570 {
 571         unsigned int i;
 572         unsigned int ret;
 573
 574         spin_lock_irq(&mapping->tree_lock);
 575         ret = radix_tree_gang_lookup(&mapping->page_tree,
 576                                 (void **)pages, start, nr_pages);
 577         for (i = 0; i < ret; i++)
 578                 page_cache_get(pages[i]);
 579         spin_unlock_irq(&mapping->tree_lock);
 580         return ret;
 581 }
 582
 583 /*
 584  * Like find_get_pages, except we only return pages which are tagged with
 585  * `tag'.   We update *index to index the next page for the traversal.
 586  */
 587 unsigned find_get_pages_tag(struct address_space *mapping, pgoff_t *index,
 588                         int tag, unsigned int nr_pages, struct page **pages)
 589 {
 590         unsigned int i;
 591         unsigned int ret;
 592
 593         spin_lock_irq(&mapping->tree_lock);
 594         ret = radix_tree_gang_lookup_tag(&mapping->page_tree,
 595                                 (void **)pages, *index, nr_pages, tag);
 596         for (i = 0; i < ret; i++)
 597                 page_cache_get(pages[i]);
 598         if (ret)
 599                 *index = pages[ret - 1]->index + 1;
 600         spin_unlock_irq(&mapping->tree_lock);
 601         return ret;
 602 }
 603
 604 /*
 605  * Same as grab_cache_page, but do not wait if the page is unavailable.
 606  * This is intended for speculative data generators, where the data can
 607  * be regenerated if the page couldn't be grabbed.  This routine should
 608  * be safe to call while holding the lock for another page.
 609  *
 610  * Clear __GFP_FS when allocating the page to avoid recursion into the fs
 611  * and deadlock against the caller's locked page.
 612  */
 613 struct page *
 614 grab_cache_page_nowait(struct address_space *mapping, unsigned long index)
 615 {
 616         struct page *page = find_get_page(mapping, index);
 617         int gfp_mask;
 618
 619         if (page) {
 620                 if (!TestSetPageLocked(page))
 621                         return page;
 622                 page_cache_release(page);
 623                 return NULL;
 624         }
 625         gfp_mask = mapping_gfp_mask(mapping) & ~__GFP_FS;
 626         page = alloc_pages(gfp_mask, 0);
 627         if (page && add_to_page_cache_lru(page, mapping, index, gfp_mask)) {
 628                 page_cache_release(page);
 629                 page = NULL;
 630         }
 631         return page;
 632 }
 633
 634 EXPORT_SYMBOL(grab_cache_page_nowait);
 635
 636 /*
 637  * This is a generic file read routine, and uses the
 638  * mapping->a_ops->readpage() function for the actual low-level
 639  * stuff.
 640  *
 641  * This is really ugly. But the goto's actually try to clarify some
 642  * of the logic when it comes to error handling etc.
 643  * - note the struct file * is only passed for the use of readpage
 644  */
 645 void do_generic_mapping_read(struct address_space *mapping,
 646                              struct file_ra_state *_ra,
 647                              struct file * filp,
 648                              loff_t *ppos,
 649                              read_descriptor_t * desc,
 650                              read_actor_t actor,
 651                              int nonblock)
 652 {
 653         struct inode *inode = mapping->host;
 654         unsigned long index, end_index, offset;
 655         loff_t isize;
 656         struct page *cached_page;
 657         int error;
 658         struct file_ra_state ra = *_ra;
 659
 660         cached_page = NULL;
 661         index = *ppos >> PAGE_CACHE_SHIFT;
 662         offset = *ppos & ~PAGE_CACHE_MASK;
 663
 664         isize = i_size_read(inode);
 665         end_index = isize >> PAGE_CACHE_SHIFT;
 666         if (index > end_index)
 667                 goto out;
 668
 669         for (;;) {
 670                 struct page *page;
 671                 unsigned long nr, ret;
 672
 673                 cond_resched();
 674                 page_cache_readahead(mapping, &ra, filp, index);
 675
 676 find_page:
 677                 page = find_get_page(mapping, index);
 678                 if (unlikely(page == NULL)) {
 679                         if (nonblock) {
 680                                 desc->error = -EWOULDBLOCKIO;
 681                                 break;
 682                         }
 683                         handle_ra_miss(mapping, &ra, index);
 684                         goto no_cached_page;
 685                 }
 686                 if (!PageUptodate(page)) {
 687                         if (nonblock) {
 688                                 page_cache_release(page);
 689                                 desc->error = -EWOULDBLOCKIO;
 690                                 break;
 691                         }
 692                         goto page_not_up_to_date;
 693                 }
 694 page_ok:
 695                 /* nr is the maximum number of bytes to copy from this page */
 696                 nr = PAGE_CACHE_SIZE;
 697                 if (index == end_index) {
 698                         nr = isize & ~PAGE_CACHE_MASK;
 699                         if (nr <= offset) {
 700                                 page_cache_release(page);
 701                                 goto out;
 702                         }
 703                 }
 704                 nr = nr - offset;
 705
 706                 /* If users can be writing to this page using arbitrary
 707                  * virtual addresses, take care about potential aliasing
 708                  * before reading the page on the kernel side.
 709                  */
 710                 if (mapping_writably_mapped(mapping))
 711                         flush_dcache_page(page);
 712
 713                 /*
 714                  * Mark the page accessed if we read the beginning.
 715                  */
 716                 if (!offset)
 717                         mark_page_accessed(page);
 718
 719                 /*
 720                  * Ok, we have the page, and it's up-to-date, so
 721                  * now we can copy it to user space...
 722                  *
 723                  * The actor routine returns how many bytes were actually used..
 724                  * NOTE! This may not be the same as how much of a user buffer
 725                  * we filled up (we may be padding etc), so we can only update
 726                  * "pos" here (the actor routine has to update the user buffer
 727                  * pointers and the remaining count).
 728                  */
 729                 ret = actor(desc, page, offset, nr);
 730                 offset += ret;
 731                 index += offset >> PAGE_CACHE_SHIFT;
 732                 offset &= ~PAGE_CACHE_MASK;
 733
 734                 page_cache_release(page);
 735                 if (ret == nr && desc->count)
 736                         continue;
 737                 goto out;
 738
 739 page_not_up_to_date:
 740                 /* Get exclusive access to the page ... */
 741                 lock_page(page);
 742
 743                 /* Did it get unhashed before we got the lock? */
 744                 if (!page->mapping) {
 745                         unlock_page(page);
 746                         page_cache_release(page);
 747                         continue;
 748                 }
 749
 750                 /* Did somebody else fill it already? */
 751                 if (PageUptodate(page)) {
 752                         unlock_page(page);
 753                         goto page_ok;
 754                 }
 755
 756 readpage:
 757                 /* Start the actual read. The read will unlock the page. */
 758                 error = mapping->a_ops->readpage(filp, page);
 759
 760                 if (unlikely(error))
 761                         goto readpage_error;
 762
 763                 if (!PageUptodate(page)) {
 764                         wait_on_page_locked(page);
 765                         if (!PageUptodate(page)) {
 766                                 error = -EIO;
 767                                 goto readpage_error;
 768                         }
 769                 }
 770
 771                 /*
 772                  * i_size must be checked after we have done ->readpage.
 773                  *
 774                  * Checking i_size after the readpage allows us to calculate
 775                  * the correct value for "nr", which means the zero-filled
 776                  * part of the page is not copied back to userspace (unless
 777                  * another truncate extends the file - this is desired though).
 778                  */
 779                 isize = i_size_read(inode);
 780                 end_index = isize >> PAGE_CACHE_SHIFT;
 781                 if (index > end_index) {
 782                         page_cache_release(page);
 783                         goto out;
 784                 }
 785                 goto page_ok;
 786
 787 readpage_error:
 788                 /* UHHUH! A synchronous read error occurred. Report it */
 789                 desc->error = error;
 790                 page_cache_release(page);
 791                 goto out;
 792
 793 no_cached_page:
 794                 /*
 795                  * Ok, it wasn't cached, so we need to create a new
 796                  * page..
 797                  */
 798                 if (!cached_page) {
 799                         cached_page = page_cache_alloc_cold(mapping);
 800                         if (!cached_page) {
 801                                 desc->error = -ENOMEM;
 802                                 goto out;
 803                         }
 804                 }
 805                 error = add_to_page_cache_lru(cached_page, mapping,
 806                                                 index, GFP_KERNEL);
 807                 if (error) {
 808                         if (error == -EEXIST)
 809                                 goto find_page;
 810                         desc->error = error;
 811                         goto out;
 812                 }
 813                 page = cached_page;
 814                 cached_page = NULL;
 815                 goto readpage;
 816         }
 817
 818 out:
 819         *_ra = ra;
 820
 821         *ppos = ((loff_t) index << PAGE_CACHE_SHIFT) + offset;
 822         if (cached_page)
 823                 page_cache_release(cached_page);
 824         file_accessed(filp);
 825 }
 826
 827 EXPORT_SYMBOL(do_generic_mapping_read);
 828
 829 int file_read_actor(read_descriptor_t *desc, struct page *page,
 830                         unsigned long offset, unsigned long size)
 831 {
 832         char *kaddr;
 833         unsigned long left, count = desc->count;
 834
 835         if (size > count)
 836                 size = count;
 837
 838         /*
 839          * Faults on the destination of a read are common, so do it before
 840          * taking the kmap.
 841          */
 842         if (!fault_in_pages_writeable(desc->buf, size)) {
 843                 kaddr = kmap_atomic(page, KM_USER0);
 844                 left = __copy_to_user(desc->buf, kaddr + offset, size);
 845                 kunmap_atomic(kaddr, KM_USER0);
 846                 if (left == 0)
 847                         goto success;
 848         }
 849
 850         /* Do it the slow way */
 851         kaddr = kmap(page);
 852         left = __copy_to_user(desc->buf, kaddr + offset, size);
 853         kunmap(page);
 854
 855         if (left) {
 856                 size -= left;
 857                 desc->error = -EFAULT;
 858         }
 859 success:
 860         desc->count = count - size;
 861         desc->written += size;
 862         desc->buf += size;
 863         return size;
 864 }
 865
 866 /*
 867  * This is the "read()" routine for all filesystems
 868  * that can use the page cache directly.
 869  */
 870 ssize_t
 871 __generic_file_aio_read(struct kiocb *iocb, const struct iovec *iov,
 872                 unsigned long nr_segs, loff_t *ppos)
 873 {
 874         struct file *filp = iocb->ki_filp;
 875         ssize_t retval;
 876         unsigned long seg;
 877         size_t count;
 878
 879         count = 0;
 880         for (seg = 0; seg < nr_segs; seg++) {
 881                 const struct iovec *iv = &iov[seg];
 882
 883                 /*
 884                  * If any segment has a negative length, or the cumulative
 885                  * length ever wraps negative then return -EINVAL.
 886                  */
 887                 count += iv->iov_len;
 888                 if (unlikely((ssize_t)(count|iv->iov_len) < 0))
 889                         return -EINVAL;
 890                 if (access_ok(VERIFY_WRITE, iv->iov_base, iv->iov_len))
 891                         continue;
 892                 if (seg == 0)
 893                         return -EFAULT;
 894                 nr_segs = seg;
 895                 count -= iv->iov_len;   /* This segment is no good */
 896                 break;
 897         }
 898
 899         /* coalesce the iovecs and go direct-to-BIO for O_DIRECT */
 900         if (filp->f_flags & O_DIRECT) {
 901                 loff_t pos = *ppos, size;
 902                 struct address_space *mapping;
 903                 struct inode *inode;
 904
 905                 mapping = filp->f_mapping;
 906                 inode = mapping->host;
 907                 retval = 0;
 908                 if (!count)
 909                         goto out; /* skip atime */
 910                 size = i_size_read(inode);
 911                 if (pos < size) {
 912                         retval = generic_file_direct_IO(READ, iocb,
 913                                                 iov, pos, nr_segs);
 914                         if (retval >= 0 && !is_sync_kiocb(iocb))
 915                                 retval = -EIOCBQUEUED;
 916                         if (retval > 0)
 917                                 *ppos = pos + retval;
 918                 }
 919                 file_accessed(filp);
 920                 goto out;
 921         }
 922
 923         retval = 0;
 924         if (count) {
 925                 for (seg = 0; seg < nr_segs; seg++) {
 926                         read_descriptor_t desc;
 927
 928                         desc.written = 0;
 929                         desc.buf = iov[seg].iov_base;
 930                         desc.count = iov[seg].iov_len;
 931                         if (desc.count == 0)
 932                                 continue;
 933                         desc.error = 0;
 934                         do_generic_file_read(filp,ppos,&desc,file_read_actor,0);
 935                         retval += desc.written;
 936                         if (!retval) {
 937                                 retval = desc.error;
 938                                 break;
 939                         }
 940                 }
 941         }
 942 out:
 943         return retval;
 944 }
 945
 946 EXPORT_SYMBOL(__generic_file_aio_read);
 947
 948 ssize_t
 949 generic_file_aio_read(struct kiocb *iocb, char __user *buf, size_t count, loff_t pos)
 950 {
 951         struct iovec local_iov = { .iov_base = buf, .iov_len = count };
 952
 953         BUG_ON(iocb->ki_pos != pos);
 954         return __generic_file_aio_read(iocb, &local_iov, 1, &iocb->ki_pos);
 955 }
 956
 957 EXPORT_SYMBOL(generic_file_aio_read);
 958
 959 ssize_t
 960 generic_file_read(struct file *filp, char __user *buf, size_t count, loff_t *ppos)
 961 {
 962         struct iovec local_iov = { .iov_base = buf, .iov_len = count };
 963         struct kiocb kiocb;
 964         ssize_t ret;
 965
 966         init_sync_kiocb(&kiocb, filp);
 967         ret = __generic_file_aio_read(&kiocb, &local_iov, 1, ppos);
 968         if (-EIOCBQUEUED == ret)
 969                 ret = wait_on_sync_kiocb(&kiocb);
 970         return ret;
 971 }
 972
 973 EXPORT_SYMBOL(generic_file_read);
 974
 975 int file_send_actor(read_descriptor_t * desc, struct page *page, unsigned long offset, unsigned long size)
 976 {
 977         ssize_t written;
 978         unsigned long count = desc->count;
 979         struct file *file = (struct file *) desc->buf;
 980
 981         if (size > count)
 982                 size = count;
 983
 984         written = file->f_op->sendpage(file, page, offset,
 985                                        size, &file->f_pos, size<count);
 986         if (written < 0) {
 987                 desc->error = written;
 988                 written = 0;
 989         }
 990         desc->count = count - written;
 991         desc->written += written;
 992         return written;
 993 }
 994
 995 ssize_t generic_file_sendfile(struct file *in_file, loff_t *ppos,
 996                          size_t count, read_actor_t actor, void __user *target)
 997 {
 998         read_descriptor_t desc;
 999
1000         if (!count)
1001                 return 0;
1002
1003         desc.written = 0;
1004         desc.count = count;
1005         desc.buf = target;
1006         desc.error = 0;
1007
1008         do_generic_file_read(in_file, ppos, &desc, actor, 0);
1009         if (desc.written)
1010                 return desc.written;
1011         return desc.error;
1012 }
1013
1014 EXPORT_SYMBOL(generic_file_sendfile);
1015
1016 static ssize_t
1017 do_readahead(struct address_space *mapping, struct file *filp,
1018              unsigned long index, unsigned long nr)
1019 {
1020         if (!mapping || !mapping->a_ops || !mapping->a_ops->readpage)
1021                 return -EINVAL;
1022
1023         force_page_cache_readahead(mapping, filp, index,
1024                                         max_sane_readahead(nr));
1025         return 0;
1026 }
1027
1028 asmlinkage ssize_t sys_readahead(int fd, loff_t offset, size_t count)
1029 {
1030         ssize_t ret;
1031         struct file *file;
1032
1033         ret = -EBADF;
1034         file = fget(fd);
1035         if (file) {
1036                 if (file->f_mode & FMODE_READ) {
1037                         struct address_space *mapping = file->f_mapping;
1038                         unsigned long start = offset >> PAGE_CACHE_SHIFT;
1039                         unsigned long end = (offset + count - 1) >> PAGE_CACHE_SHIFT;
1040                         unsigned long len = end - start + 1;
1041                         ret = do_readahead(mapping, file, start, len);
1042                 }
1043                 fput(file);
1044         }
1045         return ret;
1046 }
1047
1048 #ifdef CONFIG_MMU
1049 /*
1050  * This adds the requested page to the page cache if it isn't already there,
1051  * and schedules an I/O to read in its contents from disk.
1052  */
1053 static int FASTCALL(page_cache_read(struct file * file, unsigned long offset));
1054 static int fastcall page_cache_read(struct file * file, unsigned long offset)
1055 {
1056         struct address_space *mapping = file->f_mapping;
1057         struct page *page;
1058         int error;
1059
1060         page = page_cache_alloc_cold(mapping);
1061         if (!page)
1062                 return -ENOMEM;
1063
1064         error = add_to_page_cache_lru(page, mapping, offset, GFP_KERNEL);
1065         if (!error) {
1066                 error = mapping->a_ops->readpage(file, page);
1067                 page_cache_release(page);
1068                 return error;
1069         }
1070
1071         /*
1072          * We arrive here in the unlikely event that someone
1073          * raced with us and added our page to the cache first
1074          * or we are out of memory for radix-tree nodes.
1075          */
1076         page_cache_release(page);
1077         return error == -EEXIST ? 0 : error;
1078 }
1079
1080 #define MMAP_LOTSAMISS  (100)
1081
1082 /*
1083  * filemap_nopage() is invoked via the vma operations vector for a
1084  * mapped memory region to read in file data during a page fault.
1085  *
1086  * The goto's are kind of ugly, but this streamlines the normal case of having
1087  * it in the page cache, and handles the special cases reasonably without
1088  * having a lot of duplicated code.
1089  */
1090 struct page * filemap_nopage(struct vm_area_struct * area, unsigned long address, int *type)
1091 {
1092         int error;
1093         struct file *file = area->vm_file;
1094         struct address_space *mapping = file->f_mapping;
1095         struct file_ra_state *ra = &file->f_ra;
1096         struct inode *inode = mapping->host;
1097         struct page *page;
1098         unsigned long size, pgoff, endoff;
1099         int did_readaround = 0, majmin = VM_FAULT_MINOR;
1100
1101         pgoff = ((address - area->vm_start) >> PAGE_CACHE_SHIFT) + area->vm_pgoff;
1102         endoff = ((area->vm_end - area->vm_start) >> PAGE_CACHE_SHIFT) + area->vm_pgoff;
1103
1104 retry_all:
1105         size = (i_size_read(inode) + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
1106         if (pgoff >= size)
1107                 goto outside_data_content;
1108
1109         /* If we don't want any read-ahead, don't bother */
1110         if (VM_RandomReadHint(area))
1111                 goto no_cached_page;
1112
1113         /*
1114          * The "size" of the file, as far as mmap is concerned, isn't bigger
1115          * than the mapping
1116          */
1117         if (size > endoff)
1118                 size = endoff;
1119
1120         /*
1121          * The readahead code wants to be told about each and every page
1122          * so it can build and shrink its windows appropriately
1123          *
1124          * For sequential accesses, we use the generic readahead logic.
1125          */
1126         if (VM_SequentialReadHint(area))
1127                 page_cache_readahead(mapping, ra, file, pgoff);
1128
1129         /*
1130          * Do we have something in the page cache already?
1131          */
1132 retry_find:
1133         page = find_get_page(mapping, pgoff);
1134         if (!page) {
1135                 unsigned long ra_pages;
1136
1137                 if (VM_SequentialReadHint(area)) {
1138                         handle_ra_miss(mapping, ra, pgoff);
1139                         goto no_cached_page;
1140                 }
1141                 ra->mmap_miss++;
1142
1143                 /*
1144                  * Do we miss much more than hit in this file? If so,
1145                  * stop bothering with read-ahead. It will only hurt.
1146                  */
1147                 if (ra->mmap_miss > ra->mmap_hit + MMAP_LOTSAMISS)
1148                         goto no_cached_page;
1149
1150                 /*
1151                  * To keep the pgmajfault counter straight, we need to
1152                  * check did_readaround, as this is an inner loop.
1153                  */
1154                 if (!did_readaround) {
1155                         majmin = VM_FAULT_MAJOR;
1156                         inc_page_state(pgmajfault);
1157                 }
1158                 did_readaround = 1;
1159                 ra_pages = max_sane_readahead(file->f_ra.ra_pages);
1160                 if (ra_pages) {
1161                         long start;
1162
1163                         start = pgoff - ra_pages / 2;
1164                         if (pgoff < 0)
1165                                 pgoff = 0;
1166                         do_page_cache_readahead(mapping, file, pgoff, ra_pages);
1167                 }
1168                 page = find_get_page(mapping, pgoff);
1169                 if (!page)
1170                         goto no_cached_page;
1171         }
1172
1173         if (!did_readaround)
1174                 ra->mmap_hit++;
1175
1176         /*
1177          * Ok, found a page in the page cache, now we need to check
1178          * that it's up-to-date.
1179          */
1180         if (!PageUptodate(page))
1181                 goto page_not_uptodate;
1182
1183 success:
1184         /*
1185          * Found the page and have a reference on it.
1186          */
1187         mark_page_accessed(page);
1188         if (type)
1189                 *type = majmin;
1190         return page;
1191
1192 outside_data_content:
1193         /*
1194          * An external ptracer can access pages that normally aren't
1195          * accessible..
1196          */
1197         if (area->vm_mm == current->mm)
1198                 return NULL;
1199         /* Fall through to the non-read-ahead case */
1200 no_cached_page:
1201         /*
1202          * We're only likely to ever get here if MADV_RANDOM is in
1203          * effect.
1204          */
1205         error = page_cache_read(file, pgoff);
1206
1207         /*
1208          * The page we want has now been added to the page cache.
1209          * In the unlikely event that someone removed it in the
1210          * meantime, we'll just come back here and read it again.
1211          */
1212         if (error >= 0)
1213                 goto retry_find;
1214
1215         /*
1216          * An error return from page_cache_read can result if the
1217          * system is low on memory, or a problem occurs while trying
1218          * to schedule I/O.
1219          */
1220         if (error == -ENOMEM)
1221                 return NOPAGE_OOM;
1222         return NULL;
1223
1224 page_not_uptodate:
1225         if (!did_readaround) {
1226                 majmin = VM_FAULT_MAJOR;
1227                 inc_page_state(pgmajfault);
1228         }
1229         lock_page(page);
1230
1231         /* Did it get unhashed while we waited for it? */
1232         if (!page->mapping) {
1233                 unlock_page(page);
1234                 page_cache_release(page);
1235                 goto retry_all;
1236         }
1237
1238         /* Did somebody else get it up-to-date? */
1239         if (PageUptodate(page)) {
1240                 unlock_page(page);
1241                 goto success;
1242         }
1243
1244         if (!mapping->a_ops->readpage(file, page)) {
1245                 wait_on_page_locked(page);
1246                 if (PageUptodate(page))
1247                         goto success;
1248         }
1249
1250         /*
1251          * Umm, take care of errors if the page isn't up-to-date.
1252          * Try to re-read it _once_. We do this synchronously,
1253          * because there really aren't any performance issues here
1254          * and we need to check for errors.
1255          */
1256         lock_page(page);
1257
1258         /* Somebody truncated the page on us? */
1259         if (!page->mapping) {
1260                 unlock_page(page);
1261                 page_cache_release(page);
1262                 goto retry_all;
1263         }
1264
1265         /* Somebody else successfully read it in? */
1266         if (PageUptodate(page)) {
1267                 unlock_page(page);
1268                 goto success;
1269         }
1270         ClearPageError(page);
1271         if (!mapping->a_ops->readpage(file, page)) {
1272                 wait_on_page_locked(page);
1273                 if (PageUptodate(page))
1274                         goto success;
1275         }
1276
1277         /*
1278          * Things didn't work out. Return zero to tell the
1279          * mm layer so, possibly freeing the page cache page first.
1280          */
1281         page_cache_release(page);
1282         return NULL;
1283 }
1284
1285 EXPORT_SYMBOL(filemap_nopage);
1286
1287 static struct page * filemap_getpage(struct file *file, unsigned long pgoff,
1288                                         int nonblock)
1289 {
1290         struct address_space *mapping = file->f_mapping;
1291         struct page *page;
1292         int error;
1293
1294         /*
1295          * Do we have something in the page cache already?
1296          */
1297 retry_find:
1298         page = find_get_page(mapping, pgoff);
1299         if (!page) {
1300                 if (nonblock)
1301                         return NULL;
1302                 goto no_cached_page;
1303         }
1304
1305         /*
1306          * Ok, found a page in the page cache, now we need to check
1307          * that it's up-to-date.
1308          */
1309         if (!PageUptodate(page))
1310                 goto page_not_uptodate;
1311
1312 success:
1313         /*
1314          * Found the page and have a reference on it.
1315          */
1316         mark_page_accessed(page);
1317         return page;
1318
1319 no_cached_page:
1320         error = page_cache_read(file, pgoff);
1321
1322         /*
1323          * The page we want has now been added to the page cache.
1324          * In the unlikely event that someone removed it in the
1325          * meantime, we'll just come back here and read it again.
1326          */
1327         if (error >= 0)
1328                 goto retry_find;
1329
1330         /*
1331          * An error return from page_cache_read can result if the
1332          * system is low on memory, or a problem occurs while trying
1333          * to schedule I/O.
1334          */
1335         return NULL;
1336
1337 page_not_uptodate:
1338         lock_page(page);
1339
1340         /* Did it get unhashed while we waited for it? */
1341         if (!page->mapping) {
1342                 unlock_page(page);
1343                 goto err;
1344         }
1345
1346         /* Did somebody else get it up-to-date? */
1347         if (PageUptodate(page)) {
1348                 unlock_page(page);
1349                 goto success;
1350         }
1351
1352         if (!mapping->a_ops->readpage(file, page)) {
1353                 wait_on_page_locked(page);
1354                 if (PageUptodate(page))
1355                         goto success;
1356         }
1357
1358         /*
1359          * Umm, take care of errors if the page isn't up-to-date.
1360          * Try to re-read it _once_. We do this synchronously,
1361          * because there really aren't any performance issues here
1362          * and we need to check for errors.
1363          */
1364         lock_page(page);
1365
1366         /* Somebody truncated the page on us? */
1367         if (!page->mapping) {
1368                 unlock_page(page);
1369                 goto err;
1370         }
1371         /* Somebody else successfully read it in? */
1372         if (PageUptodate(page)) {
1373                 unlock_page(page);
1374                 goto success;
1375         }
1376
1377         ClearPageError(page);
1378         if (!mapping->a_ops->readpage(file, page)) {
1379                 wait_on_page_locked(page);
1380                 if (PageUptodate(page))
1381                         goto success;
1382         }
1383
1384         /*
1385          * Things didn't work out. Return zero to tell the
1386          * mm layer so, possibly freeing the page cache page first.
1387          */
1388 err:
1389         page_cache_release(page);
1390
1391         return NULL;
1392 }
1393
1394 static int filemap_populate(struct vm_area_struct *vma,
1395                         unsigned long addr,
1396                         unsigned long len,
1397                         pgprot_t prot,
1398                         unsigned long pgoff,
1399                         int nonblock)
1400 {
1401         struct file *file = vma->vm_file;
1402         struct address_space *mapping = file->f_mapping;
1403         struct inode *inode = mapping->host;
1404         unsigned long size;
1405         struct mm_struct *mm = vma->vm_mm;
1406         struct page *page;
1407         int err;
1408
1409         if (!nonblock)
1410                 force_page_cache_readahead(mapping, vma->vm_file,
1411                                         pgoff, len >> PAGE_CACHE_SHIFT);
1412
1413 repeat:
1414         size = (i_size_read(inode) + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
1415         if (pgoff + (len >> PAGE_CACHE_SHIFT) > size)
1416                 return -EINVAL;
1417
1418         page = filemap_getpage(file, pgoff, nonblock);
1419         if (!page && !nonblock)
1420                 return -ENOMEM;
1421         if (page) {
1422                 err = install_page(mm, vma, addr, page, prot);
1423                 if (err) {
1424                         page_cache_release(page);
1425                         return err;
1426                 }
1427         } else {
1428                 /*
1429                  * If a nonlinear mapping then store the file page offset
1430                  * in the pte.
1431                  */
1432                 if (pgoff != linear_page_index(vma, addr)) {
1433                         err = install_file_pte(mm, vma, addr, pgoff, prot);
1434                         if (err)
1435                                 return err;
1436                 }
1437         }
1438
1439         len -= PAGE_SIZE;
1440         addr += PAGE_SIZE;
1441         pgoff++;
1442         if (len)
1443                 goto repeat;
1444
1445         return 0;
1446 }
1447
1448 static struct vm_operations_struct generic_file_vm_ops = {
1449         .nopage         = filemap_nopage,
1450         .populate       = filemap_populate,
1451 };
1452
1453 /* This is used for a general mmap of a disk file */
1454
1455 int generic_file_mmap(struct file * file, struct vm_area_struct * vma)
1456 {
1457         struct address_space *mapping = file->f_mapping;
1458
1459         if (!mapping->a_ops->readpage)
1460                 return -ENOEXEC;
1461         file_accessed(file);
1462         vma->vm_ops = &generic_file_vm_ops;
1463         return 0;
1464 }
1465
1466 /*
1467  * This is for filesystems which do not implement ->writepage.
1468  */
1469 int generic_file_readonly_mmap(struct file *file, struct vm_area_struct *vma)
1470 {
1471         if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE))
1472                 return -EINVAL;
1473         return generic_file_mmap(file, vma);
1474 }
1475 #else
1476 int generic_file_mmap(struct file * file, struct vm_area_struct * vma)
1477 {
1478         return -ENOSYS;
1479 }
1480 int generic_file_readonly_mmap(struct file * file, struct vm_area_struct * vma)
1481 {
1482         return -ENOSYS;
1483 }
1484 #endif /* CONFIG_MMU */
1485
1486 EXPORT_SYMBOL(generic_file_mmap);
1487 EXPORT_SYMBOL(generic_file_readonly_mmap);
1488
1489 static inline struct page *__read_cache_page(struct address_space *mapping,
1490                                 unsigned long index,
1491                                 int (*filler)(void *,struct page*),
1492                                 void *data)
1493 {
1494         struct page *page, *cached_page = NULL;
1495         int err;
1496 repeat:
1497         page = find_get_page(mapping, index);
1498         if (!page) {
1499                 if (!cached_page) {
1500                         cached_page = page_cache_alloc_cold(mapping);
1501                         if (!cached_page)
1502                                 return ERR_PTR(-ENOMEM);
1503                 }
1504                 err = add_to_page_cache_lru(cached_page, mapping,
1505                                         index, GFP_KERNEL);
1506                 if (err == -EEXIST)
1507                         goto repeat;
1508                 if (err < 0) {
1509                         /* Presumably ENOMEM for radix tree node */
1510                         page_cache_release(cached_page);
1511                         return ERR_PTR(err);
1512                 }
1513                 page = cached_page;
1514                 cached_page = NULL;
1515                 err = filler(data, page);
1516                 if (err < 0) {
1517                         page_cache_release(page);
1518                         page = ERR_PTR(err);
1519                 }
1520         }
1521         if (cached_page)
1522                 page_cache_release(cached_page);
1523         return page;
1524 }
1525
1526 /*
1527  * Read into the page cache. If a page already exists,
1528  * and PageUptodate() is not set, try to fill the page.
1529  */
1530 struct page *read_cache_page(struct address_space *mapping,
1531                                 unsigned long index,
1532                                 int (*filler)(void *,struct page*),
1533                                 void *data)
1534 {
1535         struct page *page;
1536         int err;
1537
1538 retry:
1539         page = __read_cache_page(mapping, index, filler, data);
1540         if (IS_ERR(page))
1541                 goto out;
1542         mark_page_accessed(page);
1543         if (PageUptodate(page))
1544                 goto out;
1545
1546         lock_page(page);
1547         if (!page->mapping) {
1548                 unlock_page(page);
1549                 page_cache_release(page);
1550                 goto retry;
1551         }
1552         if (PageUptodate(page)) {
1553                 unlock_page(page);
1554                 goto out;
1555         }
1556         err = filler(data, page);
1557         if (err < 0) {
1558                 page_cache_release(page);
1559                 page = ERR_PTR(err);
1560         }
1561  out:
1562         return page;
1563 }
1564
1565 EXPORT_SYMBOL(read_cache_page);
1566
1567 /*
1568  * If the page was newly created, increment its refcount and add it to the
1569  * caller's lru-buffering pagevec.  This function is specifically for
1570  * generic_file_write().
1571  */
1572 static inline struct page *
1573 __grab_cache_page(struct address_space *mapping, unsigned long index,
1574                         struct page **cached_page, struct pagevec *lru_pvec)
1575 {
1576         int err;
1577         struct page *page;
1578 repeat:
1579         page = find_lock_page(mapping, index);
1580         if (!page) {
1581                 if (!*cached_page) {
1582                         *cached_page = page_cache_alloc(mapping);
1583                         if (!*cached_page)
1584                                 return NULL;
1585                 }
1586                 err = add_to_page_cache(*cached_page, mapping,
1587                                         index, GFP_KERNEL);
1588                 if (err == -EEXIST)
1589                         goto repeat;
1590                 if (err == 0) {
1591                         page = *cached_page;
1592                         page_cache_get(page);
1593                         if (!pagevec_add(lru_pvec, page))
1594                                 __pagevec_lru_add(lru_pvec);
1595                         *cached_page = NULL;
1596                 }
1597         }
1598         return page;
1599 }
1600
1601 /*
1602  * The logic we want is
1603  *
1604  *      if suid or (sgid and xgrp)
1605  *              remove privs
1606  */
1607 int remove_suid(struct dentry *dentry)
1608 {
1609         mode_t mode = dentry->d_inode->i_mode;
1610         int kill = 0;
1611         int result = 0;
1612
1613         /* suid always must be killed */
1614         if (unlikely(mode & S_ISUID))
1615                 kill = ATTR_KILL_SUID;
1616
1617         /*
1618          * sgid without any exec bits is just a mandatory locking mark; leave
1619          * it alone.  If some exec bits are set, it's a real sgid; kill it.
1620          */
1621         if (unlikely((mode & S_ISGID) && (mode & S_IXGRP)))
1622                 kill |= ATTR_KILL_SGID;
1623
1624         if (unlikely(kill && !capable(CAP_FSETID))) {
1625                 struct iattr newattrs;
1626
1627                 newattrs.ia_valid = ATTR_FORCE | kill;
1628                 result = notify_change(dentry, &newattrs);
1629         }
1630         return result;
1631 }
1632 EXPORT_SYMBOL(remove_suid);
1633
1634 /*
1635  * Copy as much as we can into the page and return the number of bytes which
1636  * were sucessfully copied.  If a fault is encountered then clear the page
1637  * out to (offset+bytes) and return the number of bytes which were copied.
1638  */
1639 static inline size_t
1640 filemap_copy_from_user(struct page *page, unsigned long offset,
1641                         const char __user *buf, unsigned bytes)
1642 {
1643         char *kaddr;
1644         int left;
1645
1646         kaddr = kmap_atomic(page, KM_USER0);
1647         left = __copy_from_user(kaddr + offset, buf, bytes);
1648         kunmap_atomic(kaddr, KM_USER0);
1649
1650         if (left != 0) {
1651                 /* Do it the slow way */
1652                 kaddr = kmap(page);
1653                 left = __copy_from_user(kaddr + offset, buf, bytes);
1654                 kunmap(page);
1655         }
1656         return bytes - left;
1657 }
1658
1659 static size_t
1660 __filemap_copy_from_user_iovec(char *vaddr,
1661                         const struct iovec *iov, size_t base, size_t bytes)
1662 {
1663         size_t copied = 0, left = 0;
1664
1665         while (bytes) {
1666                 char __user *buf = iov->iov_base + base;
1667                 int copy = min(bytes, iov->iov_len - base);
1668
1669                 base = 0;
1670                 left = __copy_from_user(vaddr, buf, copy);
1671                 copied += copy;
1672                 bytes -= copy;
1673                 vaddr += copy;
1674                 iov++;
1675
1676                 if (unlikely(left)) {
1677                         /* zero the rest of the target like __copy_from_user */
1678                         if (bytes)
1679                                 memset(vaddr, 0, bytes);
1680                         break;
1681                 }
1682         }
1683         return copied - left;
1684 }
1685
1686 /*
1687  * This has the same sideeffects and return value as filemap_copy_from_user().
1688  * The difference is that on a fault we need to memset the remainder of the
1689  * page (out to offset+bytes), to emulate filemap_copy_from_user()'s
1690  * single-segment behaviour.
1691  */
1692 static inline size_t
1693 filemap_copy_from_user_iovec(struct page *page, unsigned long offset,
1694                         const struct iovec *iov, size_t base, size_t bytes)
1695 {
1696         char *kaddr;
1697         size_t copied;
1698
1699         kaddr = kmap_atomic(page, KM_USER0);
1700         copied = __filemap_copy_from_user_iovec(kaddr + offset, iov,
1701                                                 base, bytes);
1702         kunmap_atomic(kaddr, KM_USER0);
1703         if (copied != bytes) {
1704                 kaddr = kmap(page);
1705                 copied = __filemap_copy_from_user_iovec(kaddr + offset, iov,
1706                                                         base, bytes);
1707                 kunmap(page);
1708         }
1709         return copied;
1710 }
1711
1712 static inline void
1713 filemap_set_next_iovec(const struct iovec **iovp, size_t *basep, size_t bytes)
1714 {
1715         const struct iovec *iov = *iovp;
1716         size_t base = *basep;
1717
1718         while (bytes) {
1719                 int copy = min(bytes, iov->iov_len - base);
1720
1721                 bytes -= copy;
1722                 base += copy;
1723                 if (iov->iov_len == base) {
1724                         iov++;
1725                         base = 0;
1726                 }
1727         }
1728         *iovp = iov;
1729         *basep = base;
1730 }
1731
1732 /*
1733  * Performs necessary checks before doing a write
1734  *
1735  * Can adjust writing position aor amount of bytes to write.
1736  * Returns appropriate error code that caller should return or
1737  * zero in case that write should be allowed.
1738  */
1739 inline int generic_write_checks(struct file *file, loff_t *pos, size_t *count, int isblk)
1740 {
1741         struct inode *inode = file->f_mapping->host;
1742         unsigned long limit = current->rlim[RLIMIT_FSIZE].rlim_cur;
1743
1744         if (unlikely(*pos < 0))
1745                 return -EINVAL;
1746
1747         if (unlikely(file->f_error)) {
1748                 int err = file->f_error;
1749                 file->f_error = 0;
1750                 return err;
1751         }
1752
1753         if (!isblk) {
1754                 /* FIXME: this is for backwards compatibility with 2.4 */
1755                 if (file->f_flags & O_APPEND)
1756                         *pos = i_size_read(inode);
1757
1758                 if (limit != RLIM_INFINITY) {
1759                         if (*pos >= limit) {
1760                                 send_sig(SIGXFSZ, current, 0);
1761                                 return -EFBIG;
1762                         }
1763                         if (*count > limit - (typeof(limit))*pos) {
1764                                 *count = limit - (typeof(limit))*pos;
1765                         }
1766                 }
1767         }
1768
1769         /*
1770          * LFS rule
1771          */
1772         if (unlikely(*pos + *count > MAX_NON_LFS &&
1773                                 !(file->f_flags & O_LARGEFILE))) {
1774                 if (*pos >= MAX_NON_LFS) {
1775                         send_sig(SIGXFSZ, current, 0);
1776                         return -EFBIG;
1777                 }
1778                 if (*count > MAX_NON_LFS - (unsigned long)*pos) {
1779                         *count = MAX_NON_LFS - (unsigned long)*pos;
1780                 }
1781         }
1782
1783         /*
1784          * Are we about to exceed the fs block limit ?
1785          *
1786          * If we have written data it becomes a short write.  If we have
1787          * exceeded without writing data we send a signal and return EFBIG.
1788          * Linus frestrict idea will clean these up nicely..
1789          */
1790         if (likely(!isblk)) {
1791                 if (unlikely(*pos >= inode->i_sb->s_maxbytes)) {
1792                         if (*count || *pos > inode->i_sb->s_maxbytes) {
1793                                 send_sig(SIGXFSZ, current, 0);
1794                                 return -EFBIG;
1795                         }
1796                         /* zero-length writes at ->s_maxbytes are OK */
1797                 }
1798
1799                 if (unlikely(*pos + *count > inode->i_sb->s_maxbytes))
1800                         *count = inode->i_sb->s_maxbytes - *pos;
1801         } else {
1802                 loff_t isize;
1803                 if (bdev_read_only(I_BDEV(inode)))
1804                         return -EPERM;
1805                 isize = i_size_read(inode);
1806                 if (*pos >= isize) {
1807                         if (*count || *pos > isize)
1808                                 return -ENOSPC;
1809                 }
1810
1811                 if (*pos + *count > isize)
1812                         *count = isize - *pos;
1813         }
1814         return 0;
1815 }
1816
1817 EXPORT_SYMBOL(generic_write_checks);
1818
1819 /*
1820  * Write to a file through the page cache.
1821  * Called under i_sem for S_ISREG files.
1822  *
1823  * We put everything into the page cache prior to writing it. This is not a
1824  * problem when writing full pages. With partial pages, however, we first have
1825  * to read the data into the cache, then dirty the page, and finally schedule
1826  * it for writing by marking it dirty.
1827  *                                                      okir@monad.swb.de
1828  */
1829 ssize_t
1830 generic_file_aio_write_nolock(struct kiocb *iocb, const struct iovec *iov,
1831                                 unsigned long nr_segs, loff_t *ppos)
1832 {
1833         struct file *file = iocb->ki_filp;
1834         struct address_space * mapping = file->f_mapping;
1835         struct address_space_operations *a_ops = mapping->a_ops;
1836         size_t ocount;          /* original count */
1837         size_t count;           /* after file limit checks */
1838         struct inode    *inode = mapping->host;
1839         long            status = 0;
1840         loff_t          pos;
1841         struct page     *page;
1842         struct page     *cached_page = NULL;
1843         const int       isblk = S_ISBLK(inode->i_mode);
1844         ssize_t         written;
1845         ssize_t         err;
1846         size_t          bytes;
1847         struct pagevec  lru_pvec;
1848         const struct iovec *cur_iov = iov; /* current iovec */
1849         size_t          iov_base = 0;      /* offset in the current iovec */
1850         unsigned long   seg;
1851         char __user     *buf;
1852
1853         ocount = 0;
1854         for (seg = 0; seg < nr_segs; seg++) {
1855                 const struct iovec *iv = &iov[seg];
1856
1857                 /*
1858                  * If any segment has a negative length, or the cumulative
1859                  * length ever wraps negative then return -EINVAL.
1860                  */
1861                 ocount += iv->iov_len;
1862                 if (unlikely((ssize_t)(ocount|iv->iov_len) < 0))
1863                         return -EINVAL;
1864                 if (access_ok(VERIFY_READ, iv->iov_base, iv->iov_len))
1865                         continue;
1866                 if (seg == 0)
1867                         return -EFAULT;
1868                 nr_segs = seg;
1869                 ocount -= iv->iov_len;  /* This segment is no good */
1870                 break;
1871         }
1872
1873         count = ocount;
1874         pos = *ppos;
1875         pagevec_init(&lru_pvec, 0);
1876
1877         /* We can write back this queue in page reclaim */
1878         current->backing_dev_info = mapping->backing_dev_info;
1879         written = 0;
1880
1881         err = generic_write_checks(file, &pos, &count, isblk);
1882         if (err)
1883                 goto out;
1884
1885         if (count == 0)
1886                 goto out;
1887
1888         err = remove_suid(file->f_dentry);
1889         if (err)
1890                 goto out;
1891
1892         inode_update_time(inode, 1);
1893
1894         /* coalesce the iovecs and go direct-to-BIO for O_DIRECT */
1895         if (unlikely(file->f_flags & O_DIRECT)) {
1896                 if (count != ocount)
1897                         nr_segs = iov_shorten((struct iovec *)iov,
1898                                                 nr_segs, count);
1899                 written = generic_file_direct_IO(WRITE, iocb,
1900                                         iov, pos, nr_segs);
1901                 if (written > 0) {
1902                         loff_t end = pos + written;
1903                         if (end > i_size_read(inode) && !isblk) {
1904                                 i_size_write(inode,  end);
1905                                 mark_inode_dirty(inode);
1906                         }
1907                         *ppos = end;
1908                 }
1909                 /*
1910                  * Sync the fs metadata but not the minor inode changes and
1911                  * of course not the data as we did direct DMA for the IO.
1912                  * i_sem is held, which protects generic_osync_inode() from
1913                  * livelocking.
1914                  */
1915                 if (written >= 0 && file->f_flags & O_SYNC)
1916                         status = generic_osync_inode(inode, mapping, OSYNC_METADATA);
1917                 if (written == count && !is_sync_kiocb(iocb))
1918                         written = -EIOCBQUEUED;
1919                 if (written < 0 || written == count)
1920                         goto out_status;
1921                 /*
1922                  * direct-io write to a hole: fall through to buffered I/O
1923                  * for completing the rest of the request.
1924                  */
1925                 pos += written;
1926                 count -= written;
1927         }
1928
1929         buf = iov->iov_base + written;  /* handle partial DIO write */
1930         do {
1931                 unsigned long index;
1932                 unsigned long offset;
1933                 size_t copied;
1934
1935                 offset = (pos & (PAGE_CACHE_SIZE -1)); /* Within page */
1936                 index = pos >> PAGE_CACHE_SHIFT;
1937                 bytes = PAGE_CACHE_SIZE - offset;
1938                 if (bytes > count)
1939                         bytes = count;
1940
1941                 /*
1942                  * Bring in the user page that we will copy from _first_.
1943                  * Otherwise there's a nasty deadlock on copying from the
1944                  * same page as we're writing to, without it being marked
1945                  * up-to-date.
1946                  */
1947                 fault_in_pages_readable(buf, bytes);
1948
1949                 page = __grab_cache_page(mapping,index,&cached_page,&lru_pvec);
1950                 if (!page) {
1951                         status = -ENOMEM;
1952                         break;
1953                 }
1954
1955                 status = a_ops->prepare_write(file, page, offset, offset+bytes);
1956                 if (unlikely(status)) {
1957                         loff_t isize = i_size_read(inode);
1958                         /*
1959                          * prepare_write() may have instantiated a few blocks
1960                          * outside i_size.  Trim these off again.
1961                          */
1962                         unlock_page(page);
1963                         page_cache_release(page);
1964                         if (pos + bytes > isize)
1965                                 vmtruncate(inode, isize);
1966                         break;
1967                 }
1968                 if (likely(nr_segs == 1))
1969                         copied = filemap_copy_from_user(page, offset,
1970                                                         buf, bytes);
1971                 else
1972                         copied = filemap_copy_from_user_iovec(page, offset,
1973                                                 cur_iov, iov_base, bytes);
1974                 flush_dcache_page(page);
1975                 status = a_ops->commit_write(file, page, offset, offset+bytes);
1976                 if (likely(copied > 0)) {
1977                         if (!status)
1978                                 status = copied;
1979
1980                         if (status >= 0) {
1981                                 written += status;
1982                                 count -= status;
1983                                 pos += status;
1984                                 buf += status;
1985                                 if (unlikely(nr_segs > 1))
1986                                         filemap_set_next_iovec(&cur_iov,
1987                                                         &iov_base, status);
1988                         }
1989                 }
1990                 if (unlikely(copied != bytes))
1991                         if (status >= 0)
1992                                 status = -EFAULT;
1993                 unlock_page(page);
1994                 mark_page_accessed(page);
1995                 page_cache_release(page);
1996                 if (status < 0)
1997                         break;
1998                 balance_dirty_pages_ratelimited(mapping);
1999                 cond_resched();
2000         } while (count);
2001         *ppos = pos;
2002
2003         if (cached_page)
2004                 page_cache_release(cached_page);
2005
2006         /*
2007          * For now, when the user asks for O_SYNC, we'll actually give O_DSYNC
2008          */
2009         if (status >= 0) {
2010                 if ((file->f_flags & O_SYNC) || IS_SYNC(inode))
2011                         status = generic_osync_inode(inode, mapping,
2012                                         OSYNC_METADATA|OSYNC_DATA);
2013         }
2014
2015         /*
2016          * If we get here for O_DIRECT writes then we must have fallen through
2017          * to buffered writes (block instantiation inside i_size).  So we sync
2018          * the file data here, to try to honour O_DIRECT expectations.
2019          */
2020         if (unlikely(file->f_flags & O_DIRECT) && written)
2021                 status = filemap_write_and_wait(mapping);
2022
2023 out_status:
2024         err = written ? written : status;
2025 out:
2026         pagevec_lru_add(&lru_pvec);
2027         current->backing_dev_info = 0;
2028         return err;
2029 }
2030
2031 EXPORT_SYMBOL(generic_file_aio_write_nolock);
2032
2033 ssize_t
2034 generic_file_write_nolock(struct file *file, const struct iovec *iov,
2035                                 unsigned long nr_segs, loff_t *ppos)
2036 {
2037         struct kiocb kiocb;
2038         ssize_t ret;
2039
2040         init_sync_kiocb(&kiocb, file);
2041         ret = generic_file_aio_write_nolock(&kiocb, iov, nr_segs, ppos);
2042         if (-EIOCBQUEUED == ret)
2043                 ret = wait_on_sync_kiocb(&kiocb);
2044         return ret;
2045 }
2046
2047 EXPORT_SYMBOL(generic_file_write_nolock);
2048
2049 ssize_t generic_file_aio_write(struct kiocb *iocb, const char __user *buf,
2050                                size_t count, loff_t pos)
2051 {
2052         struct file *file = iocb->ki_filp;
2053         struct inode *inode = file->f_mapping->host;
2054         ssize_t err;
2055         struct iovec local_iov = { .iov_base = (void __user *)buf, .iov_len = count };
2056
2057         BUG_ON(iocb->ki_pos != pos);
2058
2059         down(&inode->i_sem);
2060         err = generic_file_aio_write_nolock(iocb, &local_iov, 1,
2061                                                 &iocb->ki_pos);
2062         up(&inode->i_sem);
2063
2064         return err;
2065 }
2066
2067 EXPORT_SYMBOL(generic_file_aio_write);
2068
2069 ssize_t generic_file_write(struct file *file, const char __user *buf,
2070                            size_t count, loff_t *ppos)
2071 {
2072         struct inode    *inode = file->f_mapping->host;
2073         ssize_t         err;
2074         struct iovec local_iov = { .iov_base = (void __user *)buf, .iov_len = count };
2075
2076         down(&inode->i_sem);
2077         err = generic_file_write_nolock(file, &local_iov, 1, ppos);
2078         up(&inode->i_sem);
2079
2080         return err;
2081 }
2082
2083 EXPORT_SYMBOL(generic_file_write);
2084
2085 ssize_t generic_file_readv(struct file *filp, const struct iovec *iov,
2086                         unsigned long nr_segs, loff_t *ppos)
2087 {
2088         struct kiocb kiocb;
2089         ssize_t ret;
2090
2091         init_sync_kiocb(&kiocb, filp);
2092         ret = __generic_file_aio_read(&kiocb, iov, nr_segs, ppos);
2093         if (-EIOCBQUEUED == ret)
2094                 ret = wait_on_sync_kiocb(&kiocb);
2095         return ret;
2096 }
2097
2098 EXPORT_SYMBOL(generic_file_readv);
2099
2100 ssize_t generic_file_writev(struct file *file, const struct iovec *iov,
2101                         unsigned long nr_segs, loff_t * ppos)
2102 {
2103         struct inode *inode = file->f_mapping->host;
2104         ssize_t ret;
2105
2106         down(&inode->i_sem);
2107         ret = generic_file_write_nolock(file, iov, nr_segs, ppos);
2108         up(&inode->i_sem);
2109         return ret;
2110 }
2111
2112 EXPORT_SYMBOL(generic_file_writev);
2113
2114 /*
2115  * Called under i_sem for writes to S_ISREG files
2116  */
2117 ssize_t
2118 generic_file_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov,
2119         loff_t offset, unsigned long nr_segs)
2120 {
2121         struct file *file = iocb->ki_filp;
2122         struct address_space *mapping = file->f_mapping;
2123         ssize_t retval;
2124
2125         retval = filemap_write_and_wait(mapping);
2126         if (retval == 0) {
2127                 retval = mapping->a_ops->direct_IO(rw, iocb, iov,
2128                                                 offset, nr_segs);
2129                 if (rw == WRITE && mapping->nrpages)
2130                         invalidate_inode_pages2(mapping);
2131         }
2132         return retval;
2133 }
2134
2135 EXPORT_SYMBOL_GPL(generic_file_direct_IO);