mm/filemap.c

   1 /*
   2  *      linux/mm/filemap.c
   3  *
   4  * Copyright (C) 1994-1999  Linus Torvalds
   5  */
   6
   7 /*
   8  * This file handles the generic file mmap semantics used by
   9  * most "normal" filesystems (but you don't /have/ to use this:
  10  * the NFS filesystem used to do this differently, for example)
  11  */
  12 #include <linux/config.h>
  13 #include <linux/module.h>
  14 #include <linux/slab.h>
  15 #include <linux/compiler.h>
  16 #include <linux/fs.h>
  17 #include <linux/aio.h>
  18 #include <linux/kernel_stat.h>
  19 #include <linux/mm.h>
  20 #include <linux/swap.h>
  21 #include <linux/mman.h>
  22 #include <linux/pagemap.h>
  23 #include <linux/file.h>
  24 #include <linux/uio.h>
  25 #include <linux/hash.h>
  26 #include <linux/writeback.h>
  27 #include <linux/pagevec.h>
  28 #include <linux/blkdev.h>
  29 #include <linux/security.h>
  30 /*
  31  * This is needed for the following functions:
  32  *  - try_to_release_page
  33  *  - block_invalidatepage
  34  *  - generic_osync_inode
  35  *
  36  * FIXME: remove all knowledge of the buffer layer from the core VM
  37  */
  38 #include <linux/buffer_head.h> /* for generic_osync_inode */
  39
  40 #include <asm/uaccess.h>
  41 #include <asm/mman.h>
  42
  43 /*
  44  * Shared mappings implemented 30.11.1994. It's not fully working yet,
  45  * though.
  46  *
  47  * Shared mappings now work. 15.8.1995  Bruno.
  48  *
  49  * finished 'unifying' the page and buffer cache and SMP-threaded the
  50  * page-cache, 21.05.1999, Ingo Molnar <mingo@redhat.com>
  51  *
  52  * SMP-threaded pagemap-LRU 1999, Andrea Arcangeli <andrea@suse.de>
  53  */
  54
  55 /*
  56  * Lock ordering:
  57  *
  58  *  ->i_mmap_lock               (vmtruncate)
  59  *    ->private_lock            (__free_pte->__set_page_dirty_buffers)
  60  *      ->swap_list_lock
  61  *        ->swap_device_lock    (exclusive_swap_page, others)
  62  *          ->mapping->tree_lock
  63  *
  64  *  ->i_sem
  65  *    ->i_mmap_lock             (truncate->unmap_mapping_range)
  66  *
  67  *  ->mmap_sem
  68  *    ->i_mmap_lock
  69  *      ->page_table_lock       (various places, mainly in mmap.c)
  70  *        ->mapping->tree_lock  (arch-dependent flush_dcache_mmap_lock)
  71  *
  72  *  ->mmap_sem
  73  *    ->lock_page               (access_process_vm)
  74  *
  75  *  ->mmap_sem
  76  *    ->i_sem                   (msync)
  77  *
  78  *  ->i_sem
  79  *    ->i_alloc_sem             (various)
  80  *
  81  *  ->inode_lock
  82  *    ->sb_lock                 (fs/fs-writeback.c)
  83  *    ->mapping->tree_lock      (__sync_single_inode)
  84  *
  85  *  ->page_table_lock
  86  *    ->swap_device_lock        (try_to_unmap_one)
  87  *    ->private_lock            (try_to_unmap_one)
  88  *    ->tree_lock               (try_to_unmap_one)
  89  *    ->zone.lru_lock           (follow_page->mark_page_accessed)
  90  *
  91  *  ->task->proc_lock
  92  *    ->dcache_lock             (proc_pid_lookup)
  93  */
  94
  95 /*
  96  * Remove a page from the page cache and free it. Caller has to make
  97  * sure the page is locked and that nobody else uses it - or that usage
  98  * is safe.  The caller must hold a write_lock on the mapping's tree_lock.
  99  */
 100 void __remove_from_page_cache(struct page *page)
 101 {
 102         struct address_space *mapping = page->mapping;
 103
 104         radix_tree_delete(&mapping->page_tree, page->index);
 105         page->mapping = NULL;
 106         mapping->nrpages--;
 107         pagecache_acct(-1);
 108 }
 109
 110 void remove_from_page_cache(struct page *page)
 111 {
 112         struct address_space *mapping = page->mapping;
 113
 114         if (unlikely(!PageLocked(page)))
 115                 PAGE_BUG(page);
 116
 117         spin_lock_irq(&mapping->tree_lock);
 118         __remove_from_page_cache(page);
 119         spin_unlock_irq(&mapping->tree_lock);
 120 }
 121
 122 static inline int sync_page(struct page *page)
 123 {
 124         struct address_space *mapping;
 125
 126         /*
 127          * FIXME, fercrissake.  What is this barrier here for?
 128          */
 129         smp_mb();
 130         mapping = page_mapping(page);
 131         if (mapping && mapping->a_ops && mapping->a_ops->sync_page)
 132                 return mapping->a_ops->sync_page(page);
 133         return 0;
 134 }
 135
 136 /**
 137  * filemap_fdatawrite - start writeback against all of a mapping's dirty pages
 138  * @mapping: address space structure to write
 139  *
 140  * If sync_mode is WB_SYNC_ALL then this is a "data integrity" operation, as
 141  * opposed to a regular memory * cleansing writeback.  The difference between
 142  * these two operations is that if a dirty page/buffer is encountered, it must
 143  * be waited upon, and not just skipped over.
 144  */
 145 static int __filemap_fdatawrite(struct address_space *mapping, int sync_mode)
 146 {
 147         int ret;
 148         struct writeback_control wbc = {
 149                 .sync_mode = sync_mode,
 150                 .nr_to_write = mapping->nrpages * 2,
 151         };
 152
 153         if (mapping->backing_dev_info->memory_backed)
 154                 return 0;
 155
 156         ret = do_writepages(mapping, &wbc);
 157         return ret;
 158 }
 159
 160 int filemap_fdatawrite(struct address_space *mapping)
 161 {
 162         return __filemap_fdatawrite(mapping, WB_SYNC_ALL);
 163 }
 164 EXPORT_SYMBOL(filemap_fdatawrite);
 165
 166 /*
 167  * This is a mostly non-blocking flush.  Not suitable for data-integrity
 168  * purposes - I/O may not be started against all dirty pages.
 169  */
 170 int filemap_flush(struct address_space *mapping)
 171 {
 172         return __filemap_fdatawrite(mapping, WB_SYNC_NONE);
 173 }
 174 EXPORT_SYMBOL(filemap_flush);
 175
 176 /*
 177  * Wait for writeback to complete against pages indexed by start->end
 178  * inclusive
 179  */
 180 static int wait_on_page_writeback_range(struct address_space *mapping,
 181                                 pgoff_t start, pgoff_t end)
 182 {
 183         struct pagevec pvec;
 184         int nr_pages;
 185         int ret = 0;
 186         pgoff_t index;
 187
 188         if (end < start)
 189                 return 0;
 190
 191         pagevec_init(&pvec, 0);
 192         index = start;
 193         while ((nr_pages = pagevec_lookup_tag(&pvec, mapping, &index,
 194                         PAGECACHE_TAG_WRITEBACK,
 195                         min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1))) {
 196                 unsigned i;
 197
 198                 for (i = 0; i < nr_pages; i++) {
 199                         struct page *page = pvec.pages[i];
 200
 201                         wait_on_page_writeback(page);
 202                         if (PageError(page))
 203                                 ret = -EIO;
 204                 }
 205                 pagevec_release(&pvec);
 206                 cond_resched();
 207         }
 208
 209         /* Check for outstanding write errors */
 210         if (test_and_clear_bit(AS_ENOSPC, &mapping->flags))
 211                 ret = -ENOSPC;
 212         if (test_and_clear_bit(AS_EIO, &mapping->flags))
 213                 ret = -EIO;
 214
 215         return ret;
 216 }
 217
 218 /**
 219  * filemap_fdatawait - walk the list of under-writeback pages of the given
 220  *     address space and wait for all of them.
 221  *
 222  * @mapping: address space structure to wait for
 223  */
 224 int filemap_fdatawait(struct address_space *mapping)
 225 {
 226         return wait_on_page_writeback_range(mapping, 0, -1);
 227 }
 228
 229 EXPORT_SYMBOL(filemap_fdatawait);
 230
 231 int filemap_write_and_wait(struct address_space *mapping)
 232 {
 233         int retval = 0;
 234
 235         if (mapping->nrpages) {
 236                 retval = filemap_fdatawrite(mapping);
 237                 if (retval == 0)
 238                         retval = filemap_fdatawait(mapping);
 239         }
 240         return retval;
 241 }
 242
 243 /*
 244  * This function is used to add newly allocated pagecache pages:
 245  * the page is new, so we can just run SetPageLocked() against it.
 246  * The other page state flags were set by rmqueue().
 247  *
 248  * This function does not add the page to the LRU.  The caller must do that.
 249  */
 250 int add_to_page_cache(struct page *page, struct address_space *mapping,
 251                 pgoff_t offset, int gfp_mask)
 252 {
 253         int error = radix_tree_preload(gfp_mask & ~__GFP_HIGHMEM);
 254
 255         if (error == 0) {
 256                 spin_lock_irq(&mapping->tree_lock);
 257                 error = radix_tree_insert(&mapping->page_tree, offset, page);
 258                 if (!error) {
 259                         page_cache_get(page);
 260                         SetPageLocked(page);
 261                         page->mapping = mapping;
 262                         page->index = offset;
 263                         mapping->nrpages++;
 264                         pagecache_acct(1);
 265                 }
 266                 spin_unlock_irq(&mapping->tree_lock);
 267                 radix_tree_preload_end();
 268         }
 269         return error;
 270 }
 271
 272 EXPORT_SYMBOL(add_to_page_cache);
 273
 274 int add_to_page_cache_lru(struct page *page, struct address_space *mapping,
 275                                 pgoff_t offset, int gfp_mask)
 276 {
 277         int ret = add_to_page_cache(page, mapping, offset, gfp_mask);
 278         if (ret == 0)
 279                 lru_cache_add(page);
 280         return ret;
 281 }
 282
 283 /*
 284  * In order to wait for pages to become available there must be
 285  * waitqueues associated with pages. By using a hash table of
 286  * waitqueues where the bucket discipline is to maintain all
 287  * waiters on the same queue and wake all when any of the pages
 288  * become available, and for the woken contexts to check to be
 289  * sure the appropriate page became available, this saves space
 290  * at a cost of "thundering herd" phenomena during rare hash
 291  * collisions.
 292  */
 293 struct page_wait_queue {
 294         struct page *page;
 295         int bit;
 296         wait_queue_t wait;
 297 };
 298
 299 static int page_wake_function(wait_queue_t *wait, unsigned mode, int sync, void *key)
 300 {
 301         struct page *page = key;
 302         struct page_wait_queue *wq;
 303
 304         wq = container_of(wait, struct page_wait_queue, wait);
 305         if (wq->page != page || test_bit(wq->bit, &page->flags))
 306                 return 0;
 307         else
 308                 return autoremove_wake_function(wait, mode, sync, NULL);
 309 }
 310
 311 #define __DEFINE_PAGE_WAIT(name, p, b, f)                               \
 312         struct page_wait_queue name = {                                 \
 313                 .page   = p,                                            \
 314                 .bit    = b,                                            \
 315                 .wait   = {                                             \
 316                         .task   = current,                              \
 317                         .func   = page_wake_function,                   \
 318                         .flags  = f,                                    \
 319                         .task_list = LIST_HEAD_INIT(name.wait.task_list),\
 320                 },                                                      \
 321         }
 322
 323 #define DEFINE_PAGE_WAIT(name, p, b)    __DEFINE_PAGE_WAIT(name, p, b, 0)
 324 #define DEFINE_PAGE_WAIT_EXCLUSIVE(name, p, b)                          \
 325                 __DEFINE_PAGE_WAIT(name, p, b, WQ_FLAG_EXCLUSIVE)
 326
 327 static wait_queue_head_t *page_waitqueue(struct page *page)
 328 {
 329         const struct zone *zone = page_zone(page);
 330
 331         return &zone->wait_table[hash_ptr(page, zone->wait_table_bits)];
 332 }
 333
 334 static void wake_up_page(struct page *page)
 335 {
 336         const unsigned int mode = TASK_UNINTERRUPTIBLE | TASK_INTERRUPTIBLE;
 337         wait_queue_head_t *waitqueue = page_waitqueue(page);
 338
 339         if (waitqueue_active(waitqueue))
 340                 __wake_up(waitqueue, mode, 1, page);
 341 }
 342
 343 void fastcall wait_on_page_bit(struct page *page, int bit_nr)
 344 {
 345         wait_queue_head_t *waitqueue = page_waitqueue(page);
 346         DEFINE_PAGE_WAIT(wait, page, bit_nr);
 347
 348         do {
 349                 prepare_to_wait(waitqueue, &wait.wait, TASK_UNINTERRUPTIBLE);
 350                 if (test_bit(bit_nr, &page->flags)) {
 351                         sync_page(page);
 352                         io_schedule();
 353                 }
 354         } while (test_bit(bit_nr, &page->flags));
 355         finish_wait(waitqueue, &wait.wait);
 356 }
 357
 358 EXPORT_SYMBOL(wait_on_page_bit);
 359
 360 /**
 361  * unlock_page() - unlock a locked page
 362  *
 363  * @page: the page
 364  *
 365  * Unlocks the page and wakes up sleepers in ___wait_on_page_locked().
 366  * Also wakes sleepers in wait_on_page_writeback() because the wakeup
 367  * mechananism between PageLocked pages and PageWriteback pages is shared.
 368  * But that's OK - sleepers in wait_on_page_writeback() just go back to sleep.
 369  *
 370  * The first mb is necessary to safely close the critical section opened by the
 371  * TestSetPageLocked(), the second mb is necessary to enforce ordering between
 372  * the clear_bit and the read of the waitqueue (to avoid SMP races with a
 373  * parallel wait_on_page_locked()).
 374  */
 375 void fastcall unlock_page(struct page *page)
 376 {
 377         smp_mb__before_clear_bit();
 378         if (!TestClearPageLocked(page))
 379                 BUG();
 380         smp_mb__after_clear_bit();
 381         wake_up_page(page);
 382 }
 383
 384 EXPORT_SYMBOL(unlock_page);
 385 EXPORT_SYMBOL(lock_page);
 386
 387 /*
 388  * End writeback against a page.
 389  */
 390 void end_page_writeback(struct page *page)
 391 {
 392         if (!TestClearPageReclaim(page) || rotate_reclaimable_page(page)) {
 393                 if (!test_clear_page_writeback(page))
 394                         BUG();
 395                 smp_mb__after_clear_bit();
 396         }
 397         wake_up_page(page);
 398 }
 399
 400 EXPORT_SYMBOL(end_page_writeback);
 401
 402 /*
 403  * Get a lock on the page, assuming we need to sleep to get it.
 404  *
 405  * Ugly: running sync_page() in state TASK_UNINTERRUPTIBLE is scary.  If some
 406  * random driver's requestfn sets TASK_RUNNING, we could busywait.  However
 407  * chances are that on the second loop, the block layer's plug list is empty,
 408  * so sync_page() will then return in state TASK_UNINTERRUPTIBLE.
 409  */
 410 void fastcall __lock_page(struct page *page)
 411 {
 412         wait_queue_head_t *wqh = page_waitqueue(page);
 413         DEFINE_PAGE_WAIT_EXCLUSIVE(wait, page, PG_locked);
 414
 415         while (TestSetPageLocked(page)) {
 416                 prepare_to_wait_exclusive(wqh, &wait.wait, TASK_UNINTERRUPTIBLE);
 417                 if (PageLocked(page)) {
 418                         sync_page(page);
 419                         io_schedule();
 420                 }
 421         }
 422         finish_wait(wqh, &wait.wait);
 423 }
 424
 425 EXPORT_SYMBOL(__lock_page);
 426
 427 /*
 428  * a rather lightweight function, finding and getting a reference to a
 429  * hashed page atomically.
 430  */
 431 struct page * find_get_page(struct address_space *mapping, unsigned long offset)
 432 {
 433         struct page *page;
 434
 435         /*
 436          * We scan the hash list read-only. Addition to and removal from
 437          * the hash-list needs a held write-lock.
 438          */
 439         spin_lock_irq(&mapping->tree_lock);
 440         page = radix_tree_lookup(&mapping->page_tree, offset);
 441         if (page)
 442                 page_cache_get(page);
 443         spin_unlock_irq(&mapping->tree_lock);
 444         return page;
 445 }
 446
 447 EXPORT_SYMBOL(find_get_page);
 448
 449 /*
 450  * Same as above, but trylock it instead of incrementing the count.
 451  */
 452 struct page *find_trylock_page(struct address_space *mapping, unsigned long offset)
 453 {
 454         struct page *page;
 455
 456         spin_lock_irq(&mapping->tree_lock);
 457         page = radix_tree_lookup(&mapping->page_tree, offset);
 458         if (page && TestSetPageLocked(page))
 459                 page = NULL;
 460         spin_unlock_irq(&mapping->tree_lock);
 461         return page;
 462 }
 463
 464 EXPORT_SYMBOL(find_trylock_page);
 465
 466 /**
 467  * find_lock_page - locate, pin and lock a pagecache page
 468  *
 469  * @mapping - the address_space to search
 470  * @offset - the page index
 471  *
 472  * Locates the desired pagecache page, locks it, increments its reference
 473  * count and returns its address.
 474  *
 475  * Returns zero if the page was not present. find_lock_page() may sleep.
 476  */
 477 struct page *find_lock_page(struct address_space *mapping,
 478                                 unsigned long offset)
 479 {
 480         struct page *page;
 481
 482         spin_lock_irq(&mapping->tree_lock);
 483 repeat:
 484         page = radix_tree_lookup(&mapping->page_tree, offset);
 485         if (page) {
 486                 page_cache_get(page);
 487                 if (TestSetPageLocked(page)) {
 488                         spin_unlock_irq(&mapping->tree_lock);
 489                         lock_page(page);
 490                         spin_lock_irq(&mapping->tree_lock);
 491
 492                         /* Has the page been truncated while we slept? */
 493                         if (page->mapping != mapping || page->index != offset) {
 494                                 unlock_page(page);
 495                                 page_cache_release(page);
 496                                 goto repeat;
 497                         }
 498                 }
 499         }
 500         spin_unlock_irq(&mapping->tree_lock);
 501         return page;
 502 }
 503
 504 EXPORT_SYMBOL(find_lock_page);
 505
 506 /**
 507  * find_or_create_page - locate or add a pagecache page
 508  *
 509  * @mapping - the page's address_space
 510  * @index - the page's index into the mapping
 511  * @gfp_mask - page allocation mode
 512  *
 513  * Locates a page in the pagecache.  If the page is not present, a new page
 514  * is allocated using @gfp_mask and is added to the pagecache and to the VM's
 515  * LRU list.  The returned page is locked and has its reference count
 516  * incremented.
 517  *
 518  * find_or_create_page() may sleep, even if @gfp_flags specifies an atomic
 519  * allocation!
 520  *
 521  * find_or_create_page() returns the desired page's address, or zero on
 522  * memory exhaustion.
 523  */
 524 struct page *find_or_create_page(struct address_space *mapping,
 525                 unsigned long index, unsigned int gfp_mask)
 526 {
 527         struct page *page, *cached_page = NULL;
 528         int err;
 529 repeat:
 530         page = find_lock_page(mapping, index);
 531         if (!page) {
 532                 if (!cached_page) {
 533                         cached_page = alloc_page(gfp_mask);
 534                         if (!cached_page)
 535                                 return NULL;
 536                 }
 537                 err = add_to_page_cache_lru(cached_page, mapping,
 538                                         index, gfp_mask);
 539                 if (!err) {
 540                         page = cached_page;
 541                         cached_page = NULL;
 542                 } else if (err == -EEXIST)
 543                         goto repeat;
 544         }
 545         if (cached_page)
 546                 page_cache_release(cached_page);
 547         return page;
 548 }
 549
 550 EXPORT_SYMBOL(find_or_create_page);
 551
 552 /**
 553  * find_get_pages - gang pagecache lookup
 554  * @mapping:    The address_space to search
 555  * @start:      The starting page index
 556  * @nr_pages:   The maximum number of pages
 557  * @pages:      Where the resulting pages are placed
 558  *
 559  * find_get_pages() will search for and return a group of up to
 560  * @nr_pages pages in the mapping.  The pages are placed at @pages.
 561  * find_get_pages() takes a reference against the returned pages.
 562  *
 563  * The search returns a group of mapping-contiguous pages with ascending
 564  * indexes.  There may be holes in the indices due to not-present pages.
 565  *
 566  * find_get_pages() returns the number of pages which were found.
 567  */
 568 unsigned find_get_pages(struct address_space *mapping, pgoff_t start,
 569                             unsigned int nr_pages, struct page **pages)
 570 {
 571         unsigned int i;
 572         unsigned int ret;
 573
 574         spin_lock_irq(&mapping->tree_lock);
 575         ret = radix_tree_gang_lookup(&mapping->page_tree,
 576                                 (void **)pages, start, nr_pages);
 577         for (i = 0; i < ret; i++)
 578                 page_cache_get(pages[i]);
 579         spin_unlock_irq(&mapping->tree_lock);
 580         return ret;
 581 }
 582
 583 /*
 584  * Like find_get_pages, except we only return pages which are tagged with
 585  * `tag'.   We update *index to index the next page for the traversal.
 586  */
 587 unsigned find_get_pages_tag(struct address_space *mapping, pgoff_t *index,
 588                         int tag, unsigned int nr_pages, struct page **pages)
 589 {
 590         unsigned int i;
 591         unsigned int ret;
 592
 593         spin_lock_irq(&mapping->tree_lock);
 594         ret = radix_tree_gang_lookup_tag(&mapping->page_tree,
 595                                 (void **)pages, *index, nr_pages, tag);
 596         for (i = 0; i < ret; i++)
 597                 page_cache_get(pages[i]);
 598         if (ret)
 599                 *index = pages[ret - 1]->index + 1;
 600         spin_unlock_irq(&mapping->tree_lock);
 601         return ret;
 602 }
 603
 604 /*
 605  * Same as grab_cache_page, but do not wait if the page is unavailable.
 606  * This is intended for speculative data generators, where the data can
 607  * be regenerated if the page couldn't be grabbed.  This routine should
 608  * be safe to call while holding the lock for another page.
 609  *
 610  * Clear __GFP_FS when allocating the page to avoid recursion into the fs
 611  * and deadlock against the caller's locked page.
 612  */
 613 struct page *
 614 grab_cache_page_nowait(struct address_space *mapping, unsigned long index)
 615 {
 616         struct page *page = find_get_page(mapping, index);
 617         int gfp_mask;
 618
 619         if (page) {
 620                 if (!TestSetPageLocked(page))
 621                         return page;
 622                 page_cache_release(page);
 623                 return NULL;
 624         }
 625         gfp_mask = mapping_gfp_mask(mapping) & ~__GFP_FS;
 626         page = alloc_pages(gfp_mask, 0);
 627         if (page && add_to_page_cache_lru(page, mapping, index, gfp_mask)) {
 628                 page_cache_release(page);
 629                 page = NULL;
 630         }
 631         return page;
 632 }
 633
 634 EXPORT_SYMBOL(grab_cache_page_nowait);
 635
 636 /*
 637  * This is a generic file read routine, and uses the
 638  * mapping->a_ops->readpage() function for the actual low-level
 639  * stuff.
 640  *
 641  * This is really ugly. But the goto's actually try to clarify some
 642  * of the logic when it comes to error handling etc.
 643  * - note the struct file * is only passed for the use of readpage
 644  */
 645 void do_generic_mapping_read(struct address_space *mapping,
 646                              struct file_ra_state *_ra,
 647                              struct file * filp,
 648                              loff_t *ppos,
 649                              read_descriptor_t * desc,
 650                              read_actor_t actor)
 651 {
 652         struct inode *inode = mapping->host;
 653         unsigned long index, offset;
 654         struct page *cached_page;
 655         int error;
 656         struct file_ra_state ra = *_ra;
 657
 658         cached_page = NULL;
 659         index = *ppos >> PAGE_CACHE_SHIFT;
 660         offset = *ppos & ~PAGE_CACHE_MASK;
 661
 662         for (;;) {
 663                 struct page *page;
 664                 unsigned long end_index, nr, ret;
 665                 loff_t isize = i_size_read(inode);
 666
 667                 end_index = isize >> PAGE_CACHE_SHIFT;
 668
 669                 if (index > end_index)
 670                         break;
 671                 nr = PAGE_CACHE_SIZE;
 672                 if (index == end_index) {
 673                         nr = isize & ~PAGE_CACHE_MASK;
 674                         if (nr <= offset)
 675                                 break;
 676                 }
 677
 678                 cond_resched();
 679                 page_cache_readahead(mapping, &ra, filp, index);
 680
 681                 nr = nr - offset;
 682 find_page:
 683                 page = find_get_page(mapping, index);
 684                 if (unlikely(page == NULL)) {
 685                         handle_ra_miss(mapping, &ra, index);
 686                         goto no_cached_page;
 687                 }
 688                 if (!PageUptodate(page))
 689                         goto page_not_up_to_date;
 690 page_ok:
 691                 /* If users can be writing to this page using arbitrary
 692                  * virtual addresses, take care about potential aliasing
 693                  * before reading the page on the kernel side.
 694                  */
 695                 if (mapping_writably_mapped(mapping))
 696                         flush_dcache_page(page);
 697
 698                 /*
 699                  * Mark the page accessed if we read the beginning.
 700                  */
 701                 if (!offset)
 702                         mark_page_accessed(page);
 703
 704                 /*
 705                  * Ok, we have the page, and it's up-to-date, so
 706                  * now we can copy it to user space...
 707                  *
 708                  * The actor routine returns how many bytes were actually used..
 709                  * NOTE! This may not be the same as how much of a user buffer
 710                  * we filled up (we may be padding etc), so we can only update
 711                  * "pos" here (the actor routine has to update the user buffer
 712                  * pointers and the remaining count).
 713                  */
 714                 ret = actor(desc, page, offset, nr);
 715                 offset += ret;
 716                 index += offset >> PAGE_CACHE_SHIFT;
 717                 offset &= ~PAGE_CACHE_MASK;
 718
 719                 page_cache_release(page);
 720                 if (ret == nr && desc->count)
 721                         continue;
 722                 break;
 723
 724 page_not_up_to_date:
 725                 /* Get exclusive access to the page ... */
 726                 lock_page(page);
 727
 728                 /* Did it get unhashed before we got the lock? */
 729                 if (!page->mapping) {
 730                         unlock_page(page);
 731                         page_cache_release(page);
 732                         continue;
 733                 }
 734
 735                 /* Did somebody else fill it already? */
 736                 if (PageUptodate(page)) {
 737                         unlock_page(page);
 738                         goto page_ok;
 739                 }
 740
 741 readpage:
 742                 /* ... and start the actual read. The read will unlock the page. */
 743                 error = mapping->a_ops->readpage(filp, page);
 744
 745                 if (!error) {
 746                         if (PageUptodate(page))
 747                                 goto page_ok;
 748                         wait_on_page_locked(page);
 749                         if (PageUptodate(page))
 750                                 goto page_ok;
 751                         error = -EIO;
 752                 }
 753
 754                 /* UHHUH! A synchronous read error occurred. Report it */
 755                 desc->error = error;
 756                 page_cache_release(page);
 757                 break;
 758
 759 no_cached_page:
 760                 /*
 761                  * Ok, it wasn't cached, so we need to create a new
 762                  * page..
 763                  */
 764                 if (!cached_page) {
 765                         cached_page = page_cache_alloc_cold(mapping);
 766                         if (!cached_page) {
 767                                 desc->error = -ENOMEM;
 768                                 break;
 769                         }
 770                 }
 771                 error = add_to_page_cache_lru(cached_page, mapping,
 772                                                 index, GFP_KERNEL);
 773                 if (error) {
 774                         if (error == -EEXIST)
 775                                 goto find_page;
 776                         desc->error = error;
 777                         break;
 778                 }
 779                 page = cached_page;
 780                 cached_page = NULL;
 781                 goto readpage;
 782         }
 783
 784         *_ra = ra;
 785
 786         *ppos = ((loff_t) index << PAGE_CACHE_SHIFT) + offset;
 787         if (cached_page)
 788                 page_cache_release(cached_page);
 789         file_accessed(filp);
 790 }
 791
 792 EXPORT_SYMBOL(do_generic_mapping_read);
 793
 794 int file_read_actor(read_descriptor_t *desc, struct page *page,
 795                         unsigned long offset, unsigned long size)
 796 {
 797         char *kaddr;
 798         unsigned long left, count = desc->count;
 799
 800         if (size > count)
 801                 size = count;
 802
 803         /*
 804          * Faults on the destination of a read are common, so do it before
 805          * taking the kmap.
 806          */
 807         if (!fault_in_pages_writeable(desc->buf, size)) {
 808                 kaddr = kmap_atomic(page, KM_USER0);
 809                 left = __copy_to_user(desc->buf, kaddr + offset, size);
 810                 kunmap_atomic(kaddr, KM_USER0);
 811                 if (left == 0)
 812                         goto success;
 813         }
 814
 815         /* Do it the slow way */
 816         kaddr = kmap(page);
 817         left = __copy_to_user(desc->buf, kaddr + offset, size);
 818         kunmap(page);
 819
 820         if (left) {
 821                 size -= left;
 822                 desc->error = -EFAULT;
 823         }
 824 success:
 825         desc->count = count - size;
 826         desc->written += size;
 827         desc->buf += size;
 828         return size;
 829 }
 830
 831 /*
 832  * This is the "read()" routine for all filesystems
 833  * that can use the page cache directly.
 834  */
 835 ssize_t
 836 __generic_file_aio_read(struct kiocb *iocb, const struct iovec *iov,
 837                 unsigned long nr_segs, loff_t *ppos)
 838 {
 839         struct file *filp = iocb->ki_filp;
 840         ssize_t retval;
 841         unsigned long seg;
 842         size_t count;
 843
 844         count = 0;
 845         for (seg = 0; seg < nr_segs; seg++) {
 846                 const struct iovec *iv = &iov[seg];
 847
 848                 /*
 849                  * If any segment has a negative length, or the cumulative
 850                  * length ever wraps negative then return -EINVAL.
 851                  */
 852                 count += iv->iov_len;
 853                 if (unlikely((ssize_t)(count|iv->iov_len) < 0))
 854                         return -EINVAL;
 855                 if (access_ok(VERIFY_WRITE, iv->iov_base, iv->iov_len))
 856                         continue;
 857                 if (seg == 0)
 858                         return -EFAULT;
 859                 nr_segs = seg;
 860                 count -= iv->iov_len;   /* This segment is no good */
 861                 break;
 862         }
 863
 864         /* coalesce the iovecs and go direct-to-BIO for O_DIRECT */
 865         if (filp->f_flags & O_DIRECT) {
 866                 loff_t pos = *ppos, size;
 867                 struct address_space *mapping;
 868                 struct inode *inode;
 869
 870                 mapping = filp->f_mapping;
 871                 inode = mapping->host;
 872                 retval = 0;
 873                 if (!count)
 874                         goto out; /* skip atime */
 875                 size = i_size_read(inode);
 876                 if (pos < size) {
 877                         retval = generic_file_direct_IO(READ, iocb,
 878                                                 iov, pos, nr_segs);
 879                         if (retval >= 0 && !is_sync_kiocb(iocb))
 880                                 retval = -EIOCBQUEUED;
 881                         if (retval > 0)
 882                                 *ppos = pos + retval;
 883                 }
 884                 file_accessed(filp);
 885                 goto out;
 886         }
 887
 888         retval = 0;
 889         if (count) {
 890                 for (seg = 0; seg < nr_segs; seg++) {
 891                         read_descriptor_t desc;
 892
 893                         desc.written = 0;
 894                         desc.buf = iov[seg].iov_base;
 895                         desc.count = iov[seg].iov_len;
 896                         if (desc.count == 0)
 897                                 continue;
 898                         desc.error = 0;
 899                         do_generic_file_read(filp,ppos,&desc,file_read_actor);
 900                         retval += desc.written;
 901                         if (!retval) {
 902                                 retval = desc.error;
 903                                 break;
 904                         }
 905                 }
 906         }
 907 out:
 908         return retval;
 909 }
 910
 911 EXPORT_SYMBOL(__generic_file_aio_read);
 912
 913 ssize_t
 914 generic_file_aio_read(struct kiocb *iocb, char __user *buf, size_t count, loff_t pos)
 915 {
 916         struct iovec local_iov = { .iov_base = buf, .iov_len = count };
 917
 918         BUG_ON(iocb->ki_pos != pos);
 919         return __generic_file_aio_read(iocb, &local_iov, 1, &iocb->ki_pos);
 920 }
 921
 922 EXPORT_SYMBOL(generic_file_aio_read);
 923
 924 ssize_t
 925 generic_file_read(struct file *filp, char __user *buf, size_t count, loff_t *ppos)
 926 {
 927         struct iovec local_iov = { .iov_base = buf, .iov_len = count };
 928         struct kiocb kiocb;
 929         ssize_t ret;
 930
 931         init_sync_kiocb(&kiocb, filp);
 932         ret = __generic_file_aio_read(&kiocb, &local_iov, 1, ppos);
 933         if (-EIOCBQUEUED == ret)
 934                 ret = wait_on_sync_kiocb(&kiocb);
 935         return ret;
 936 }
 937
 938 EXPORT_SYMBOL(generic_file_read);
 939
 940 int file_send_actor(read_descriptor_t * desc, struct page *page, unsigned long offset, unsigned long size)
 941 {
 942         ssize_t written;
 943         unsigned long count = desc->count;
 944         struct file *file = (struct file *) desc->buf;
 945
 946         if (size > count)
 947                 size = count;
 948
 949         written = file->f_op->sendpage(file, page, offset,
 950                                        size, &file->f_pos, size<count);
 951         if (written < 0) {
 952                 desc->error = written;
 953                 written = 0;
 954         }
 955         desc->count = count - written;
 956         desc->written += written;
 957         return written;
 958 }
 959
 960 ssize_t generic_file_sendfile(struct file *in_file, loff_t *ppos,
 961                          size_t count, read_actor_t actor, void __user *target)
 962 {
 963         read_descriptor_t desc;
 964
 965         if (!count)
 966                 return 0;
 967
 968         desc.written = 0;
 969         desc.count = count;
 970         desc.buf = target;
 971         desc.error = 0;
 972
 973         do_generic_file_read(in_file, ppos, &desc, actor);
 974         if (desc.written)
 975                 return desc.written;
 976         return desc.error;
 977 }
 978
 979 EXPORT_SYMBOL(generic_file_sendfile);
 980
 981 static ssize_t
 982 do_readahead(struct address_space *mapping, struct file *filp,
 983              unsigned long index, unsigned long nr)
 984 {
 985         if (!mapping || !mapping->a_ops || !mapping->a_ops->readpage)
 986                 return -EINVAL;
 987
 988         force_page_cache_readahead(mapping, filp, index,
 989                                         max_sane_readahead(nr));
 990         return 0;
 991 }
 992
 993 asmlinkage ssize_t sys_readahead(int fd, loff_t offset, size_t count)
 994 {
 995         ssize_t ret;
 996         struct file *file;
 997
 998         ret = -EBADF;
 999         file = fget(fd);
1000         if (file) {
1001                 if (file->f_mode & FMODE_READ) {
1002                         struct address_space *mapping = file->f_mapping;
1003                         unsigned long start = offset >> PAGE_CACHE_SHIFT;
1004                         unsigned long end = (offset + count - 1) >> PAGE_CACHE_SHIFT;
1005                         unsigned long len = end - start + 1;
1006                         ret = do_readahead(mapping, file, start, len);
1007                 }
1008                 fput(file);
1009         }
1010         return ret;
1011 }
1012
1013 #ifdef CONFIG_MMU
1014 /*
1015  * This adds the requested page to the page cache if it isn't already there,
1016  * and schedules an I/O to read in its contents from disk.
1017  */
1018 static int FASTCALL(page_cache_read(struct file * file, unsigned long offset));
1019 static int fastcall page_cache_read(struct file * file, unsigned long offset)
1020 {
1021         struct address_space *mapping = file->f_mapping;
1022         struct page *page;
1023         int error;
1024
1025         page = page_cache_alloc_cold(mapping);
1026         if (!page)
1027                 return -ENOMEM;
1028
1029         error = add_to_page_cache_lru(page, mapping, offset, GFP_KERNEL);
1030         if (!error) {
1031                 error = mapping->a_ops->readpage(file, page);
1032                 page_cache_release(page);
1033                 return error;
1034         }
1035
1036         /*
1037          * We arrive here in the unlikely event that someone
1038          * raced with us and added our page to the cache first
1039          * or we are out of memory for radix-tree nodes.
1040          */
1041         page_cache_release(page);
1042         return error == -EEXIST ? 0 : error;
1043 }
1044
1045 #define MMAP_LOTSAMISS  (100)
1046
1047 /*
1048  * filemap_nopage() is invoked via the vma operations vector for a
1049  * mapped memory region to read in file data during a page fault.
1050  *
1051  * The goto's are kind of ugly, but this streamlines the normal case of having
1052  * it in the page cache, and handles the special cases reasonably without
1053  * having a lot of duplicated code.
1054  */
1055 struct page * filemap_nopage(struct vm_area_struct * area, unsigned long address, int *type)
1056 {
1057         int error;
1058         struct file *file = area->vm_file;
1059         struct address_space *mapping = file->f_mapping;
1060         struct file_ra_state *ra = &file->f_ra;
1061         struct inode *inode = mapping->host;
1062         struct page *page;
1063         unsigned long size, pgoff, endoff;
1064         int did_readaround = 0, majmin = VM_FAULT_MINOR;
1065
1066         pgoff = ((address - area->vm_start) >> PAGE_CACHE_SHIFT) + area->vm_pgoff;
1067         endoff = ((area->vm_end - area->vm_start) >> PAGE_CACHE_SHIFT) + area->vm_pgoff;
1068
1069 retry_all:
1070         size = (i_size_read(inode) + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
1071         if (pgoff >= size)
1072                 goto outside_data_content;
1073
1074         /* If we don't want any read-ahead, don't bother */
1075         if (VM_RandomReadHint(area))
1076                 goto no_cached_page;
1077
1078         /*
1079          * The "size" of the file, as far as mmap is concerned, isn't bigger
1080          * than the mapping
1081          */
1082         if (size > endoff)
1083                 size = endoff;
1084
1085         /*
1086          * The readahead code wants to be told about each and every page
1087          * so it can build and shrink its windows appropriately
1088          *
1089          * For sequential accesses, we use the generic readahead logic.
1090          */
1091         if (VM_SequentialReadHint(area))
1092                 page_cache_readahead(mapping, ra, file, pgoff);
1093
1094         /*
1095          * Do we have something in the page cache already?
1096          */
1097 retry_find:
1098         page = find_get_page(mapping, pgoff);
1099         if (!page) {
1100                 unsigned long ra_pages;
1101
1102                 if (VM_SequentialReadHint(area)) {
1103                         handle_ra_miss(mapping, ra, pgoff);
1104                         goto no_cached_page;
1105                 }
1106                 ra->mmap_miss++;
1107
1108                 /*
1109                  * Do we miss much more than hit in this file? If so,
1110                  * stop bothering with read-ahead. It will only hurt.
1111                  */
1112                 if (ra->mmap_miss > ra->mmap_hit + MMAP_LOTSAMISS)
1113                         goto no_cached_page;
1114
1115                 /*
1116                  * To keep the pgmajfault counter straight, we need to
1117                  * check did_readaround, as this is an inner loop.
1118                  */
1119                 if (!did_readaround) {
1120                         majmin = VM_FAULT_MAJOR;
1121                         inc_page_state(pgmajfault);
1122                 }
1123                 did_readaround = 1;
1124                 ra_pages = max_sane_readahead(file->f_ra.ra_pages);
1125                 if (ra_pages) {
1126                         long start;
1127
1128                         start = pgoff - ra_pages / 2;
1129                         if (pgoff < 0)
1130                                 pgoff = 0;
1131                         do_page_cache_readahead(mapping, file, pgoff, ra_pages);
1132                 }
1133                 page = find_get_page(mapping, pgoff);
1134                 if (!page)
1135                         goto no_cached_page;
1136         }
1137
1138         if (!did_readaround)
1139                 ra->mmap_hit++;
1140
1141         /*
1142          * Ok, found a page in the page cache, now we need to check
1143          * that it's up-to-date.
1144          */
1145         if (!PageUptodate(page))
1146                 goto page_not_uptodate;
1147
1148 success:
1149         /*
1150          * Found the page and have a reference on it.
1151          */
1152         mark_page_accessed(page);
1153         if (type)
1154                 *type = majmin;
1155         return page;
1156
1157 outside_data_content:
1158         /*
1159          * An external ptracer can access pages that normally aren't
1160          * accessible..
1161          */
1162         if (area->vm_mm == current->mm)
1163                 return NULL;
1164         /* Fall through to the non-read-ahead case */
1165 no_cached_page:
1166         /*
1167          * We're only likely to ever get here if MADV_RANDOM is in
1168          * effect.
1169          */
1170         error = page_cache_read(file, pgoff);
1171
1172         /*
1173          * The page we want has now been added to the page cache.
1174          * In the unlikely event that someone removed it in the
1175          * meantime, we'll just come back here and read it again.
1176          */
1177         if (error >= 0)
1178                 goto retry_find;
1179
1180         /*
1181          * An error return from page_cache_read can result if the
1182          * system is low on memory, or a problem occurs while trying
1183          * to schedule I/O.
1184          */
1185         if (error == -ENOMEM)
1186                 return NOPAGE_OOM;
1187         return NULL;
1188
1189 page_not_uptodate:
1190         if (!did_readaround) {
1191                 majmin = VM_FAULT_MAJOR;
1192                 inc_page_state(pgmajfault);
1193         }
1194         lock_page(page);
1195
1196         /* Did it get unhashed while we waited for it? */
1197         if (!page->mapping) {
1198                 unlock_page(page);
1199                 page_cache_release(page);
1200                 goto retry_all;
1201         }
1202
1203         /* Did somebody else get it up-to-date? */
1204         if (PageUptodate(page)) {
1205                 unlock_page(page);
1206                 goto success;
1207         }
1208
1209         if (!mapping->a_ops->readpage(file, page)) {
1210                 wait_on_page_locked(page);
1211                 if (PageUptodate(page))
1212                         goto success;
1213         }
1214
1215         /*
1216          * Umm, take care of errors if the page isn't up-to-date.
1217          * Try to re-read it _once_. We do this synchronously,
1218          * because there really aren't any performance issues here
1219          * and we need to check for errors.
1220          */
1221         lock_page(page);
1222
1223         /* Somebody truncated the page on us? */
1224         if (!page->mapping) {
1225                 unlock_page(page);
1226                 page_cache_release(page);
1227                 goto retry_all;
1228         }
1229
1230         /* Somebody else successfully read it in? */
1231         if (PageUptodate(page)) {
1232                 unlock_page(page);
1233                 goto success;
1234         }
1235         ClearPageError(page);
1236         if (!mapping->a_ops->readpage(file, page)) {
1237                 wait_on_page_locked(page);
1238                 if (PageUptodate(page))
1239                         goto success;
1240         }
1241
1242         /*
1243          * Things didn't work out. Return zero to tell the
1244          * mm layer so, possibly freeing the page cache page first.
1245          */
1246         page_cache_release(page);
1247         return NULL;
1248 }
1249
1250 EXPORT_SYMBOL(filemap_nopage);
1251
1252 static struct page * filemap_getpage(struct file *file, unsigned long pgoff,
1253                                         int nonblock)
1254 {
1255         struct address_space *mapping = file->f_mapping;
1256         struct page *page;
1257         int error;
1258
1259         /*
1260          * Do we have something in the page cache already?
1261          */
1262 retry_find:
1263         page = find_get_page(mapping, pgoff);
1264         if (!page) {
1265                 if (nonblock)
1266                         return NULL;
1267                 goto no_cached_page;
1268         }
1269
1270         /*
1271          * Ok, found a page in the page cache, now we need to check
1272          * that it's up-to-date.
1273          */
1274         if (!PageUptodate(page))
1275                 goto page_not_uptodate;
1276
1277 success:
1278         /*
1279          * Found the page and have a reference on it.
1280          */
1281         mark_page_accessed(page);
1282         return page;
1283
1284 no_cached_page:
1285         error = page_cache_read(file, pgoff);
1286
1287         /*
1288          * The page we want has now been added to the page cache.
1289          * In the unlikely event that someone removed it in the
1290          * meantime, we'll just come back here and read it again.
1291          */
1292         if (error >= 0)
1293                 goto retry_find;
1294
1295         /*
1296          * An error return from page_cache_read can result if the
1297          * system is low on memory, or a problem occurs while trying
1298          * to schedule I/O.
1299          */
1300         return NULL;
1301
1302 page_not_uptodate:
1303         lock_page(page);
1304
1305         /* Did it get unhashed while we waited for it? */
1306         if (!page->mapping) {
1307                 unlock_page(page);
1308                 goto err;
1309         }
1310
1311         /* Did somebody else get it up-to-date? */
1312         if (PageUptodate(page)) {
1313                 unlock_page(page);
1314                 goto success;
1315         }
1316
1317         if (!mapping->a_ops->readpage(file, page)) {
1318                 wait_on_page_locked(page);
1319                 if (PageUptodate(page))
1320                         goto success;
1321         }
1322
1323         /*
1324          * Umm, take care of errors if the page isn't up-to-date.
1325          * Try to re-read it _once_. We do this synchronously,
1326          * because there really aren't any performance issues here
1327          * and we need to check for errors.
1328          */
1329         lock_page(page);
1330
1331         /* Somebody truncated the page on us? */
1332         if (!page->mapping) {
1333                 unlock_page(page);
1334                 goto err;
1335         }
1336         /* Somebody else successfully read it in? */
1337         if (PageUptodate(page)) {
1338                 unlock_page(page);
1339                 goto success;
1340         }
1341
1342         ClearPageError(page);
1343         if (!mapping->a_ops->readpage(file, page)) {
1344                 wait_on_page_locked(page);
1345                 if (PageUptodate(page))
1346                         goto success;
1347         }
1348
1349         /*
1350          * Things didn't work out. Return zero to tell the
1351          * mm layer so, possibly freeing the page cache page first.
1352          */
1353 err:
1354         page_cache_release(page);
1355
1356         return NULL;
1357 }
1358
1359 static int filemap_populate(struct vm_area_struct *vma,
1360                         unsigned long addr,
1361                         unsigned long len,
1362                         pgprot_t prot,
1363                         unsigned long pgoff,
1364                         int nonblock)
1365 {
1366         struct file *file = vma->vm_file;
1367         struct address_space *mapping = file->f_mapping;
1368         struct inode *inode = mapping->host;
1369         unsigned long size;
1370         struct mm_struct *mm = vma->vm_mm;
1371         struct page *page;
1372         int err;
1373
1374         if (!nonblock)
1375                 force_page_cache_readahead(mapping, vma->vm_file,
1376                                         pgoff, len >> PAGE_CACHE_SHIFT);
1377
1378 repeat:
1379         size = (i_size_read(inode) + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
1380         if (pgoff + (len >> PAGE_CACHE_SHIFT) > size)
1381                 return -EINVAL;
1382
1383         page = filemap_getpage(file, pgoff, nonblock);
1384         if (!page && !nonblock)
1385                 return -ENOMEM;
1386         if (page) {
1387                 err = install_page(mm, vma, addr, page, prot);
1388                 if (err) {
1389                         page_cache_release(page);
1390                         return err;
1391                 }
1392         } else {
1393                 /*
1394                  * If a nonlinear mapping then store the file page offset
1395                  * in the pte.
1396                  */
1397                 if (pgoff != linear_page_index(vma, addr)) {
1398                         err = install_file_pte(mm, vma, addr, pgoff, prot);
1399                         if (err)
1400                                 return err;
1401                 }
1402         }
1403
1404         len -= PAGE_SIZE;
1405         addr += PAGE_SIZE;
1406         pgoff++;
1407         if (len)
1408                 goto repeat;
1409
1410         return 0;
1411 }
1412
1413 static struct vm_operations_struct generic_file_vm_ops = {
1414         .nopage         = filemap_nopage,
1415         .populate       = filemap_populate,
1416 };
1417
1418 /* This is used for a general mmap of a disk file */
1419
1420 int generic_file_mmap(struct file * file, struct vm_area_struct * vma)
1421 {
1422         struct address_space *mapping = file->f_mapping;
1423
1424         if (!mapping->a_ops->readpage)
1425                 return -ENOEXEC;
1426         file_accessed(file);
1427         vma->vm_ops = &generic_file_vm_ops;
1428         return 0;
1429 }
1430
1431 /*
1432  * This is for filesystems which do not implement ->writepage.
1433  */
1434 int generic_file_readonly_mmap(struct file *file, struct vm_area_struct *vma)
1435 {
1436         if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE))
1437                 return -EINVAL;
1438         return generic_file_mmap(file, vma);
1439 }
1440 #else
1441 int generic_file_mmap(struct file * file, struct vm_area_struct * vma)
1442 {
1443         return -ENOSYS;
1444 }
1445 int generic_file_readonly_mmap(struct file * file, struct vm_area_struct * vma)
1446 {
1447         return -ENOSYS;
1448 }
1449 #endif /* CONFIG_MMU */
1450
1451 EXPORT_SYMBOL(generic_file_mmap);
1452 EXPORT_SYMBOL(generic_file_readonly_mmap);
1453
1454 static inline struct page *__read_cache_page(struct address_space *mapping,
1455                                 unsigned long index,
1456                                 int (*filler)(void *,struct page*),
1457                                 void *data)
1458 {
1459         struct page *page, *cached_page = NULL;
1460         int err;
1461 repeat:
1462         page = find_get_page(mapping, index);
1463         if (!page) {
1464                 if (!cached_page) {
1465                         cached_page = page_cache_alloc_cold(mapping);
1466                         if (!cached_page)
1467                                 return ERR_PTR(-ENOMEM);
1468                 }
1469                 err = add_to_page_cache_lru(cached_page, mapping,
1470                                         index, GFP_KERNEL);
1471                 if (err == -EEXIST)
1472                         goto repeat;
1473                 if (err < 0) {
1474                         /* Presumably ENOMEM for radix tree node */
1475                         page_cache_release(cached_page);
1476                         return ERR_PTR(err);
1477                 }
1478                 page = cached_page;
1479                 cached_page = NULL;
1480                 err = filler(data, page);
1481                 if (err < 0) {
1482                         page_cache_release(page);
1483                         page = ERR_PTR(err);
1484                 }
1485         }
1486         if (cached_page)
1487                 page_cache_release(cached_page);
1488         return page;
1489 }
1490
1491 /*
1492  * Read into the page cache. If a page already exists,
1493  * and PageUptodate() is not set, try to fill the page.
1494  */
1495 struct page *read_cache_page(struct address_space *mapping,
1496                                 unsigned long index,
1497                                 int (*filler)(void *,struct page*),
1498                                 void *data)
1499 {
1500         struct page *page;
1501         int err;
1502
1503 retry:
1504         page = __read_cache_page(mapping, index, filler, data);
1505         if (IS_ERR(page))
1506                 goto out;
1507         mark_page_accessed(page);
1508         if (PageUptodate(page))
1509                 goto out;
1510
1511         lock_page(page);
1512         if (!page->mapping) {
1513                 unlock_page(page);
1514                 page_cache_release(page);
1515                 goto retry;
1516         }
1517         if (PageUptodate(page)) {
1518                 unlock_page(page);
1519                 goto out;
1520         }
1521         err = filler(data, page);
1522         if (err < 0) {
1523                 page_cache_release(page);
1524                 page = ERR_PTR(err);
1525         }
1526  out:
1527         return page;
1528 }
1529
1530 EXPORT_SYMBOL(read_cache_page);
1531
1532 /*
1533  * If the page was newly created, increment its refcount and add it to the
1534  * caller's lru-buffering pagevec.  This function is specifically for
1535  * generic_file_write().
1536  */
1537 static inline struct page *
1538 __grab_cache_page(struct address_space *mapping, unsigned long index,
1539                         struct page **cached_page, struct pagevec *lru_pvec)
1540 {
1541         int err;
1542         struct page *page;
1543 repeat:
1544         page = find_lock_page(mapping, index);
1545         if (!page) {
1546                 if (!*cached_page) {
1547                         *cached_page = page_cache_alloc(mapping);
1548                         if (!*cached_page)
1549                                 return NULL;
1550                 }
1551                 err = add_to_page_cache(*cached_page, mapping,
1552                                         index, GFP_KERNEL);
1553                 if (err == -EEXIST)
1554                         goto repeat;
1555                 if (err == 0) {
1556                         page = *cached_page;
1557                         page_cache_get(page);
1558                         if (!pagevec_add(lru_pvec, page))
1559                                 __pagevec_lru_add(lru_pvec);
1560                         *cached_page = NULL;
1561                 }
1562         }
1563         return page;
1564 }
1565
1566 /*
1567  * The logic we want is
1568  *
1569  *      if suid or (sgid and xgrp)
1570  *              remove privs
1571  */
1572 int remove_suid(struct dentry *dentry)
1573 {
1574         mode_t mode = dentry->d_inode->i_mode;
1575         int kill = 0;
1576         int result = 0;
1577
1578         /* suid always must be killed */
1579         if (unlikely(mode & S_ISUID))
1580                 kill = ATTR_KILL_SUID;
1581
1582         /*
1583          * sgid without any exec bits is just a mandatory locking mark; leave
1584          * it alone.  If some exec bits are set, it's a real sgid; kill it.
1585          */
1586         if (unlikely((mode & S_ISGID) && (mode & S_IXGRP)))
1587                 kill |= ATTR_KILL_SGID;
1588
1589         if (unlikely(kill && !capable(CAP_FSETID))) {
1590                 struct iattr newattrs;
1591
1592                 newattrs.ia_valid = ATTR_FORCE | kill;
1593                 result = notify_change(dentry, &newattrs);
1594         }
1595         return result;
1596 }
1597 EXPORT_SYMBOL(remove_suid);
1598
1599 /*
1600  * Copy as much as we can into the page and return the number of bytes which
1601  * were sucessfully copied.  If a fault is encountered then clear the page
1602  * out to (offset+bytes) and return the number of bytes which were copied.
1603  */
1604 static inline size_t
1605 filemap_copy_from_user(struct page *page, unsigned long offset,
1606                         const char __user *buf, unsigned bytes)
1607 {
1608         char *kaddr;
1609         int left;
1610
1611         kaddr = kmap_atomic(page, KM_USER0);
1612         left = __copy_from_user(kaddr + offset, buf, bytes);
1613         kunmap_atomic(kaddr, KM_USER0);
1614
1615         if (left != 0) {
1616                 /* Do it the slow way */
1617                 kaddr = kmap(page);
1618                 left = __copy_from_user(kaddr + offset, buf, bytes);
1619                 kunmap(page);
1620         }
1621         return bytes - left;
1622 }
1623
1624 static size_t
1625 __filemap_copy_from_user_iovec(char *vaddr,
1626                         const struct iovec *iov, size_t base, size_t bytes)
1627 {
1628         size_t copied = 0, left = 0;
1629
1630         while (bytes) {
1631                 char __user *buf = iov->iov_base + base;
1632                 int copy = min(bytes, iov->iov_len - base);
1633
1634                 base = 0;
1635                 left = __copy_from_user(vaddr, buf, copy);
1636                 copied += copy;
1637                 bytes -= copy;
1638                 vaddr += copy;
1639                 iov++;
1640
1641                 if (unlikely(left)) {
1642                         /* zero the rest of the target like __copy_from_user */
1643                         if (bytes)
1644                                 memset(vaddr, 0, bytes);
1645                         break;
1646                 }
1647         }
1648         return copied - left;
1649 }
1650
1651 /*
1652  * This has the same sideeffects and return value as filemap_copy_from_user().
1653  * The difference is that on a fault we need to memset the remainder of the
1654  * page (out to offset+bytes), to emulate filemap_copy_from_user()'s
1655  * single-segment behaviour.
1656  */
1657 static inline size_t
1658 filemap_copy_from_user_iovec(struct page *page, unsigned long offset,
1659                         const struct iovec *iov, size_t base, size_t bytes)
1660 {
1661         char *kaddr;
1662         size_t copied;
1663
1664         kaddr = kmap_atomic(page, KM_USER0);
1665         copied = __filemap_copy_from_user_iovec(kaddr + offset, iov,
1666                                                 base, bytes);
1667         kunmap_atomic(kaddr, KM_USER0);
1668         if (copied != bytes) {
1669                 kaddr = kmap(page);
1670                 copied = __filemap_copy_from_user_iovec(kaddr + offset, iov,
1671                                                         base, bytes);
1672                 kunmap(page);
1673         }
1674         return copied;
1675 }
1676
1677 static inline void
1678 filemap_set_next_iovec(const struct iovec **iovp, size_t *basep, size_t bytes)
1679 {
1680         const struct iovec *iov = *iovp;
1681         size_t base = *basep;
1682
1683         while (bytes) {
1684                 int copy = min(bytes, iov->iov_len - base);
1685
1686                 bytes -= copy;
1687                 base += copy;
1688                 if (iov->iov_len == base) {
1689                         iov++;
1690                         base = 0;
1691                 }
1692         }
1693         *iovp = iov;
1694         *basep = base;
1695 }
1696
1697 /*
1698  * Performs necessary checks before doing a write
1699  *
1700  * Can adjust writing position aor amount of bytes to write.
1701  * Returns appropriate error code that caller should return or
1702  * zero in case that write should be allowed.
1703  */
1704 inline int generic_write_checks(struct file *file, loff_t *pos, size_t *count, int isblk)
1705 {
1706         struct inode *inode = file->f_mapping->host;
1707         unsigned long limit = current->rlim[RLIMIT_FSIZE].rlim_cur;
1708
1709         if (unlikely(*pos < 0))
1710                 return -EINVAL;
1711
1712         if (unlikely(file->f_error)) {
1713                 int err = file->f_error;
1714                 file->f_error = 0;
1715                 return err;
1716         }
1717
1718         if (!isblk) {
1719                 /* FIXME: this is for backwards compatibility with 2.4 */
1720                 if (file->f_flags & O_APPEND)
1721                         *pos = i_size_read(inode);
1722
1723                 if (limit != RLIM_INFINITY) {
1724                         if (*pos >= limit) {
1725                                 send_sig(SIGXFSZ, current, 0);
1726                                 return -EFBIG;
1727                         }
1728                         if (*count > limit - (typeof(limit))*pos) {
1729                                 *count = limit - (typeof(limit))*pos;
1730                         }
1731                 }
1732         }
1733
1734         /*
1735          * LFS rule
1736          */
1737         if (unlikely(*pos + *count > MAX_NON_LFS &&
1738                                 !(file->f_flags & O_LARGEFILE))) {
1739                 if (*pos >= MAX_NON_LFS) {
1740                         send_sig(SIGXFSZ, current, 0);
1741                         return -EFBIG;
1742                 }
1743                 if (*count > MAX_NON_LFS - (unsigned long)*pos) {
1744                         *count = MAX_NON_LFS - (unsigned long)*pos;
1745                 }
1746         }
1747
1748         /*
1749          * Are we about to exceed the fs block limit ?
1750          *
1751          * If we have written data it becomes a short write.  If we have
1752          * exceeded without writing data we send a signal and return EFBIG.
1753          * Linus frestrict idea will clean these up nicely..
1754          */
1755         if (likely(!isblk)) {
1756                 if (unlikely(*pos >= inode->i_sb->s_maxbytes)) {
1757                         if (*count || *pos > inode->i_sb->s_maxbytes) {
1758                                 send_sig(SIGXFSZ, current, 0);
1759                                 return -EFBIG;
1760                         }
1761                         /* zero-length writes at ->s_maxbytes are OK */
1762                 }
1763
1764                 if (unlikely(*pos + *count > inode->i_sb->s_maxbytes))
1765                         *count = inode->i_sb->s_maxbytes - *pos;
1766         } else {
1767                 loff_t isize;
1768                 if (bdev_read_only(I_BDEV(inode)))
1769                         return -EPERM;
1770                 isize = i_size_read(inode);
1771                 if (*pos >= isize) {
1772                         if (*count || *pos > isize)
1773                                 return -ENOSPC;
1774                 }
1775
1776                 if (*pos + *count > isize)
1777                         *count = isize - *pos;
1778         }
1779         return 0;
1780 }
1781
1782 EXPORT_SYMBOL(generic_write_checks);
1783
1784 /*
1785  * Write to a file through the page cache.
1786  * Called under i_sem for S_ISREG files.
1787  *
1788  * We put everything into the page cache prior to writing it. This is not a
1789  * problem when writing full pages. With partial pages, however, we first have
1790  * to read the data into the cache, then dirty the page, and finally schedule
1791  * it for writing by marking it dirty.
1792  *                                                      okir@monad.swb.de
1793  */
1794 ssize_t
1795 generic_file_aio_write_nolock(struct kiocb *iocb, const struct iovec *iov,
1796                                 unsigned long nr_segs, loff_t *ppos)
1797 {
1798         struct file *file = iocb->ki_filp;
1799         struct address_space * mapping = file->f_mapping;
1800         struct address_space_operations *a_ops = mapping->a_ops;
1801         size_t ocount;          /* original count */
1802         size_t count;           /* after file limit checks */
1803         struct inode    *inode = mapping->host;
1804         long            status = 0;
1805         loff_t          pos;
1806         struct page     *page;
1807         struct page     *cached_page = NULL;
1808         const int       isblk = S_ISBLK(inode->i_mode);
1809         ssize_t         written;
1810         ssize_t         err;
1811         size_t          bytes;
1812         struct pagevec  lru_pvec;
1813         const struct iovec *cur_iov = iov; /* current iovec */
1814         size_t          iov_base = 0;      /* offset in the current iovec */
1815         unsigned long   seg;
1816         char __user     *buf;
1817
1818         ocount = 0;
1819         for (seg = 0; seg < nr_segs; seg++) {
1820                 const struct iovec *iv = &iov[seg];
1821
1822                 /*
1823                  * If any segment has a negative length, or the cumulative
1824                  * length ever wraps negative then return -EINVAL.
1825                  */
1826                 ocount += iv->iov_len;
1827                 if (unlikely((ssize_t)(ocount|iv->iov_len) < 0))
1828                         return -EINVAL;
1829                 if (access_ok(VERIFY_READ, iv->iov_base, iv->iov_len))
1830                         continue;
1831                 if (seg == 0)
1832                         return -EFAULT;
1833                 nr_segs = seg;
1834                 ocount -= iv->iov_len;  /* This segment is no good */
1835                 break;
1836         }
1837
1838         count = ocount;
1839         pos = *ppos;
1840         pagevec_init(&lru_pvec, 0);
1841
1842         /* We can write back this queue in page reclaim */
1843         current->backing_dev_info = mapping->backing_dev_info;
1844         written = 0;
1845
1846         err = generic_write_checks(file, &pos, &count, isblk);
1847         if (err)
1848                 goto out;
1849
1850         if (count == 0)
1851                 goto out;
1852
1853         err = remove_suid(file->f_dentry);
1854         if (err)
1855                 goto out;
1856
1857         inode_update_time(inode, 1);
1858
1859         /* coalesce the iovecs and go direct-to-BIO for O_DIRECT */
1860         if (unlikely(file->f_flags & O_DIRECT)) {
1861                 if (count != ocount)
1862                         nr_segs = iov_shorten((struct iovec *)iov,
1863                                                 nr_segs, count);
1864                 written = generic_file_direct_IO(WRITE, iocb,
1865                                         iov, pos, nr_segs);
1866                 if (written > 0) {
1867                         loff_t end = pos + written;
1868                         if (end > i_size_read(inode) && !isblk) {
1869                                 i_size_write(inode,  end);
1870                                 mark_inode_dirty(inode);
1871                         }
1872                         *ppos = end;
1873                 }
1874                 /*
1875                  * Sync the fs metadata but not the minor inode changes and
1876                  * of course not the data as we did direct DMA for the IO.
1877                  * i_sem is held, which protects generic_osync_inode() from
1878                  * livelocking.
1879                  */
1880                 if (written >= 0 && file->f_flags & O_SYNC)
1881                         status = generic_osync_inode(inode, mapping, OSYNC_METADATA);
1882                 if (written == count && !is_sync_kiocb(iocb))
1883                         written = -EIOCBQUEUED;
1884                 if (written < 0 || written == count)
1885                         goto out_status;
1886                 /*
1887                  * direct-io write to a hole: fall through to buffered I/O
1888                  * for completing the rest of the request.
1889                  */
1890                 pos += written;
1891                 count -= written;
1892         }
1893
1894         buf = iov->iov_base;
1895         do {
1896                 unsigned long index;
1897                 unsigned long offset;
1898                 size_t copied;
1899
1900                 offset = (pos & (PAGE_CACHE_SIZE -1)); /* Within page */
1901                 index = pos >> PAGE_CACHE_SHIFT;
1902                 bytes = PAGE_CACHE_SIZE - offset;
1903                 if (bytes > count)
1904                         bytes = count;
1905
1906                 /*
1907                  * Bring in the user page that we will copy from _first_.
1908                  * Otherwise there's a nasty deadlock on copying from the
1909                  * same page as we're writing to, without it being marked
1910                  * up-to-date.
1911                  */
1912                 fault_in_pages_readable(buf, bytes);
1913
1914                 page = __grab_cache_page(mapping,index,&cached_page,&lru_pvec);
1915                 if (!page) {
1916                         status = -ENOMEM;
1917                         break;
1918                 }
1919
1920                 status = a_ops->prepare_write(file, page, offset, offset+bytes);
1921                 if (unlikely(status)) {
1922                         loff_t isize = i_size_read(inode);
1923                         /*
1924                          * prepare_write() may have instantiated a few blocks
1925                          * outside i_size.  Trim these off again.
1926                          */
1927                         unlock_page(page);
1928                         page_cache_release(page);
1929                         if (pos + bytes > isize)
1930                                 vmtruncate(inode, isize);
1931                         break;
1932                 }
1933                 if (likely(nr_segs == 1))
1934                         copied = filemap_copy_from_user(page, offset,
1935                                                         buf, bytes);
1936                 else
1937                         copied = filemap_copy_from_user_iovec(page, offset,
1938                                                 cur_iov, iov_base, bytes);
1939                 flush_dcache_page(page);
1940                 status = a_ops->commit_write(file, page, offset, offset+bytes);
1941                 if (likely(copied > 0)) {
1942                         if (!status)
1943                                 status = copied;
1944
1945                         if (status >= 0) {
1946                                 written += status;
1947                                 count -= status;
1948                                 pos += status;
1949                                 buf += status;
1950                                 if (unlikely(nr_segs > 1))
1951                                         filemap_set_next_iovec(&cur_iov,
1952                                                         &iov_base, status);
1953                         }
1954                 }
1955                 if (unlikely(copied != bytes))
1956                         if (status >= 0)
1957                                 status = -EFAULT;
1958                 unlock_page(page);
1959                 mark_page_accessed(page);
1960                 page_cache_release(page);
1961                 if (status < 0)
1962                         break;
1963                 balance_dirty_pages_ratelimited(mapping);
1964                 cond_resched();
1965         } while (count);
1966         *ppos = pos;
1967
1968         if (cached_page)
1969                 page_cache_release(cached_page);
1970
1971         /*
1972          * For now, when the user asks for O_SYNC, we'll actually give O_DSYNC
1973          */
1974         if (status >= 0) {
1975                 if ((file->f_flags & O_SYNC) || IS_SYNC(inode))
1976                         status = generic_osync_inode(inode, mapping,
1977                                         OSYNC_METADATA|OSYNC_DATA);
1978         }
1979
1980         /*
1981          * If we get here for O_DIRECT writes then we must have fallen through
1982          * to buffered writes (block instantiation inside i_size).  So we sync
1983          * the file data here, to try to honour O_DIRECT expectations.
1984          */
1985         if (unlikely(file->f_flags & O_DIRECT) && written)
1986                 status = filemap_write_and_wait(mapping);
1987
1988 out_status:
1989         err = written ? written : status;
1990 out:
1991         pagevec_lru_add(&lru_pvec);
1992         current->backing_dev_info = 0;
1993         return err;
1994 }
1995
1996 EXPORT_SYMBOL(generic_file_aio_write_nolock);
1997
1998 ssize_t
1999 generic_file_write_nolock(struct file *file, const struct iovec *iov,
2000                                 unsigned long nr_segs, loff_t *ppos)
2001 {
2002         struct kiocb kiocb;
2003         ssize_t ret;
2004
2005         init_sync_kiocb(&kiocb, file);
2006         ret = generic_file_aio_write_nolock(&kiocb, iov, nr_segs, ppos);
2007         if (-EIOCBQUEUED == ret)
2008                 ret = wait_on_sync_kiocb(&kiocb);
2009         return ret;
2010 }
2011
2012 EXPORT_SYMBOL(generic_file_write_nolock);
2013
2014 ssize_t generic_file_aio_write(struct kiocb *iocb, const char __user *buf,
2015                                size_t count, loff_t pos)
2016 {
2017         struct file *file = iocb->ki_filp;
2018         struct inode *inode = file->f_mapping->host;
2019         ssize_t err;
2020         struct iovec local_iov = { .iov_base = (void __user *)buf, .iov_len = count };
2021
2022         BUG_ON(iocb->ki_pos != pos);
2023
2024         down(&inode->i_sem);
2025         err = generic_file_aio_write_nolock(iocb, &local_iov, 1,
2026                                                 &iocb->ki_pos);
2027         up(&inode->i_sem);
2028
2029         return err;
2030 }
2031
2032 EXPORT_SYMBOL(generic_file_aio_write);
2033
2034 ssize_t generic_file_write(struct file *file, const char __user *buf,
2035                            size_t count, loff_t *ppos)
2036 {
2037         struct inode    *inode = file->f_mapping->host;
2038         ssize_t         err;
2039         struct iovec local_iov = { .iov_base = (void __user *)buf, .iov_len = count };
2040
2041         down(&inode->i_sem);
2042         err = generic_file_write_nolock(file, &local_iov, 1, ppos);
2043         up(&inode->i_sem);
2044
2045         return err;
2046 }
2047
2048 EXPORT_SYMBOL(generic_file_write);
2049
2050 ssize_t generic_file_readv(struct file *filp, const struct iovec *iov,
2051                         unsigned long nr_segs, loff_t *ppos)
2052 {
2053         struct kiocb kiocb;
2054         ssize_t ret;
2055
2056         init_sync_kiocb(&kiocb, filp);
2057         ret = __generic_file_aio_read(&kiocb, iov, nr_segs, ppos);
2058         if (-EIOCBQUEUED == ret)
2059                 ret = wait_on_sync_kiocb(&kiocb);
2060         return ret;
2061 }
2062
2063 EXPORT_SYMBOL(generic_file_readv);
2064
2065 ssize_t generic_file_writev(struct file *file, const struct iovec *iov,
2066                         unsigned long nr_segs, loff_t * ppos)
2067 {
2068         struct inode *inode = file->f_mapping->host;
2069         ssize_t ret;
2070
2071         down(&inode->i_sem);
2072         ret = generic_file_write_nolock(file, iov, nr_segs, ppos);
2073         up(&inode->i_sem);
2074         return ret;
2075 }
2076
2077 EXPORT_SYMBOL(generic_file_writev);
2078
2079 /*
2080  * Called under i_sem for writes to S_ISREG files
2081  */
2082 ssize_t
2083 generic_file_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov,
2084         loff_t offset, unsigned long nr_segs)
2085 {
2086         struct file *file = iocb->ki_filp;
2087         struct address_space *mapping = file->f_mapping;
2088         ssize_t retval;
2089
2090         retval = filemap_write_and_wait(mapping);
2091         if (retval == 0) {
2092                 retval = mapping->a_ops->direct_IO(rw, iocb, iov,
2093                                                 offset, nr_segs);
2094                 if (rw == WRITE && mapping->nrpages)
2095                         invalidate_inode_pages2(mapping);
2096         }
2097         return retval;
2098 }
2099
2100 EXPORT_SYMBOL_GPL(generic_file_direct_IO);