mm/filemap.c

   1 /*
   2  *      linux/mm/filemap.c
   3  *
   4  * Copyright (C) 1994-1999  Linus Torvalds
   5  */
   6
   7 /*
   8  * This file handles the generic file mmap semantics used by
   9  * most "normal" filesystems (but you don't /have/ to use this:
  10  * the NFS filesystem used to do this differently, for example)
  11  */
  12 #include <linux/config.h>
  13 #include <linux/module.h>
  14 #include <linux/slab.h>
  15 #include <linux/compiler.h>
  16 #include <linux/fs.h>
  17 #include <linux/aio.h>
  18 #include <linux/kernel_stat.h>
  19 #include <linux/mm.h>
  20 #include <linux/swap.h>
  21 #include <linux/mman.h>
  22 #include <linux/pagemap.h>
  23 #include <linux/file.h>
  24 #include <linux/uio.h>
  25 #include <linux/hash.h>
  26 #include <linux/writeback.h>
  27 #include <linux/pagevec.h>
  28 #include <linux/blkdev.h>
  29 #include <linux/security.h>
  30 /*
  31  * This is needed for the following functions:
  32  *  - try_to_release_page
  33  *  - block_invalidatepage
  34  *  - generic_osync_inode
  35  *
  36  * FIXME: remove all knowledge of the buffer layer from the core VM
  37  */
  38 #include <linux/buffer_head.h> /* for generic_osync_inode */
  39
  40 #include <asm/uaccess.h>
  41 #include <asm/mman.h>
  42
  43 /*
  44  * Shared mappings implemented 30.11.1994. It's not fully working yet,
  45  * though.
  46  *
  47  * Shared mappings now work. 15.8.1995  Bruno.
  48  *
  49  * finished 'unifying' the page and buffer cache and SMP-threaded the
  50  * page-cache, 21.05.1999, Ingo Molnar <mingo@redhat.com>
  51  *
  52  * SMP-threaded pagemap-LRU 1999, Andrea Arcangeli <andrea@suse.de>
  53  */
  54
  55 /*
  56  * Lock ordering:
  57  *
  58  *  ->i_mmap_lock               (vmtruncate)
  59  *    ->private_lock            (__free_pte->__set_page_dirty_buffers)
  60  *      ->swap_list_lock
  61  *        ->swap_device_lock    (exclusive_swap_page, others)
  62  *          ->mapping->tree_lock
  63  *
  64  *  ->i_sem
  65  *    ->i_mmap_lock             (truncate->unmap_mapping_range)
  66  *
  67  *  ->mmap_sem
  68  *    ->i_mmap_lock
  69  *      ->page_table_lock       (various places, mainly in mmap.c)
  70  *        ->mapping->tree_lock  (arch-dependent flush_dcache_mmap_lock)
  71  *
  72  *  ->mmap_sem
  73  *    ->lock_page               (access_process_vm)
  74  *
  75  *  ->mmap_sem
  76  *    ->i_sem                   (msync)
  77  *
  78  *  ->i_sem
  79  *    ->i_alloc_sem             (various)
  80  *
  81  *  ->inode_lock
  82  *    ->sb_lock                 (fs/fs-writeback.c)
  83  *    ->mapping->tree_lock      (__sync_single_inode)
  84  *
  85  *  ->page_table_lock
  86  *    ->swap_device_lock        (try_to_unmap_one)
  87  *    ->private_lock            (try_to_unmap_one)
  88  *    ->tree_lock               (try_to_unmap_one)
  89  *    ->zone.lru_lock           (follow_page->mark_page_accessed)
  90  *
  91  *  ->task->proc_lock
  92  *    ->dcache_lock             (proc_pid_lookup)
  93  */
  94
  95 /*
  96  * Remove a page from the page cache and free it. Caller has to make
  97  * sure the page is locked and that nobody else uses it - or that usage
  98  * is safe.  The caller must hold a write_lock on the mapping's tree_lock.
  99  */
 100 void __remove_from_page_cache(struct page *page)
 101 {
 102         struct address_space *mapping = page->mapping;
 103
 104         radix_tree_delete(&mapping->page_tree, page->index);
 105         page->mapping = NULL;
 106         mapping->nrpages--;
 107         pagecache_acct(-1);
 108 }
 109
 110 void remove_from_page_cache(struct page *page)
 111 {
 112         struct address_space *mapping = page->mapping;
 113
 114         if (unlikely(!PageLocked(page)))
 115                 PAGE_BUG(page);
 116
 117         spin_lock_irq(&mapping->tree_lock);
 118         __remove_from_page_cache(page);
 119         spin_unlock_irq(&mapping->tree_lock);
 120 }
 121
 122 static inline int sync_page(struct page *page)
 123 {
 124         struct address_space *mapping;
 125
 126         /*
 127          * FIXME, fercrissake.  What is this barrier here for?
 128          */
 129         smp_mb();
 130         mapping = page_mapping(page);
 131         if (mapping && mapping->a_ops && mapping->a_ops->sync_page)
 132                 return mapping->a_ops->sync_page(page);
 133         return 0;
 134 }
 135
 136 /**
 137  * filemap_fdatawrite - start writeback against all of a mapping's dirty pages
 138  * @mapping: address space structure to write
 139  *
 140  * If sync_mode is WB_SYNC_ALL then this is a "data integrity" operation, as
 141  * opposed to a regular memory * cleansing writeback.  The difference between
 142  * these two operations is that if a dirty page/buffer is encountered, it must
 143  * be waited upon, and not just skipped over.
 144  */
 145 static int __filemap_fdatawrite(struct address_space *mapping, int sync_mode)
 146 {
 147         int ret;
 148         struct writeback_control wbc = {
 149                 .sync_mode = sync_mode,
 150                 .nr_to_write = mapping->nrpages * 2,
 151         };
 152
 153         if (mapping->backing_dev_info->memory_backed)
 154                 return 0;
 155
 156         ret = do_writepages(mapping, &wbc);
 157         return ret;
 158 }
 159
 160 int filemap_fdatawrite(struct address_space *mapping)
 161 {
 162         return __filemap_fdatawrite(mapping, WB_SYNC_ALL);
 163 }
 164 EXPORT_SYMBOL(filemap_fdatawrite);
 165
 166 /*
 167  * This is a mostly non-blocking flush.  Not suitable for data-integrity
 168  * purposes - I/O may not be started against all dirty pages.
 169  */
 170 int filemap_flush(struct address_space *mapping)
 171 {
 172         return __filemap_fdatawrite(mapping, WB_SYNC_NONE);
 173 }
 174 EXPORT_SYMBOL(filemap_flush);
 175
 176 /*
 177  * Wait for writeback to complete against pages indexed by start->end
 178  * inclusive
 179  */
 180 static int wait_on_page_writeback_range(struct address_space *mapping,
 181                                 pgoff_t start, pgoff_t end)
 182 {
 183         struct pagevec pvec;
 184         int nr_pages;
 185         int ret = 0;
 186         pgoff_t index;
 187
 188         if (end < start)
 189                 return 0;
 190
 191         pagevec_init(&pvec, 0);
 192         index = start;
 193         while ((nr_pages = pagevec_lookup_tag(&pvec, mapping, &index,
 194                         PAGECACHE_TAG_WRITEBACK,
 195                         min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1))) {
 196                 unsigned i;
 197
 198                 for (i = 0; i < nr_pages; i++) {
 199                         struct page *page = pvec.pages[i];
 200
 201                         wait_on_page_writeback(page);
 202                         if (PageError(page))
 203                                 ret = -EIO;
 204                 }
 205                 pagevec_release(&pvec);
 206                 cond_resched();
 207         }
 208
 209         /* Check for outstanding write errors */
 210         if (test_and_clear_bit(AS_ENOSPC, &mapping->flags))
 211                 ret = -ENOSPC;
 212         if (test_and_clear_bit(AS_EIO, &mapping->flags))
 213                 ret = -EIO;
 214
 215         return ret;
 216 }
 217
 218 /**
 219  * filemap_fdatawait - walk the list of under-writeback pages of the given
 220  *     address space and wait for all of them.
 221  *
 222  * @mapping: address space structure to wait for
 223  */
 224 int filemap_fdatawait(struct address_space *mapping)
 225 {
 226         return wait_on_page_writeback_range(mapping, 0, -1);
 227 }
 228
 229 EXPORT_SYMBOL(filemap_fdatawait);
 230
 231 int filemap_write_and_wait(struct address_space *mapping)
 232 {
 233         int retval = 0;
 234
 235         if (mapping->nrpages) {
 236                 retval = filemap_fdatawrite(mapping);
 237                 if (retval == 0)
 238                         retval = filemap_fdatawait(mapping);
 239         }
 240         return retval;
 241 }
 242
 243 /*
 244  * This function is used to add newly allocated pagecache pages:
 245  * the page is new, so we can just run SetPageLocked() against it.
 246  * The other page state flags were set by rmqueue().
 247  *
 248  * This function does not add the page to the LRU.  The caller must do that.
 249  */
 250 int add_to_page_cache(struct page *page, struct address_space *mapping,
 251                 pgoff_t offset, int gfp_mask)
 252 {
 253         int error = radix_tree_preload(gfp_mask & ~__GFP_HIGHMEM);
 254
 255         if (error == 0) {
 256                 spin_lock_irq(&mapping->tree_lock);
 257                 error = radix_tree_insert(&mapping->page_tree, offset, page);
 258                 if (!error) {
 259                         page_cache_get(page);
 260                         SetPageLocked(page);
 261                         page->mapping = mapping;
 262                         page->index = offset;
 263                         mapping->nrpages++;
 264                         pagecache_acct(1);
 265                 }
 266                 spin_unlock_irq(&mapping->tree_lock);
 267                 radix_tree_preload_end();
 268         }
 269         return error;
 270 }
 271
 272 EXPORT_SYMBOL(add_to_page_cache);
 273
 274 int add_to_page_cache_lru(struct page *page, struct address_space *mapping,
 275                                 pgoff_t offset, int gfp_mask)
 276 {
 277         int ret = add_to_page_cache(page, mapping, offset, gfp_mask);
 278         if (ret == 0)
 279                 lru_cache_add(page);
 280         return ret;
 281 }
 282
 283 /*
 284  * In order to wait for pages to become available there must be
 285  * waitqueues associated with pages. By using a hash table of
 286  * waitqueues where the bucket discipline is to maintain all
 287  * waiters on the same queue and wake all when any of the pages
 288  * become available, and for the woken contexts to check to be
 289  * sure the appropriate page became available, this saves space
 290  * at a cost of "thundering herd" phenomena during rare hash
 291  * collisions.
 292  */
 293 struct page_wait_queue {
 294         struct page *page;
 295         int bit;
 296         wait_queue_t wait;
 297 };
 298
 299 static int page_wake_function(wait_queue_t *wait, unsigned mode, int sync, void *key)
 300 {
 301         struct page *page = key;
 302         struct page_wait_queue *wq;
 303
 304         wq = container_of(wait, struct page_wait_queue, wait);
 305         if (wq->page != page || test_bit(wq->bit, &page->flags))
 306                 return 0;
 307         else
 308                 return autoremove_wake_function(wait, mode, sync, NULL);
 309 }
 310
 311 #define __DEFINE_PAGE_WAIT(name, p, b, f)                               \
 312         struct page_wait_queue name = {                                 \
 313                 .page   = p,                                            \
 314                 .bit    = b,                                            \
 315                 .wait   = {                                             \
 316                         .task   = current,                              \
 317                         .func   = page_wake_function,                   \
 318                         .flags  = f,                                    \
 319                         .task_list = LIST_HEAD_INIT(name.wait.task_list),\
 320                 },                                                      \
 321         }
 322
 323 #define DEFINE_PAGE_WAIT(name, p, b)    __DEFINE_PAGE_WAIT(name, p, b, 0)
 324 #define DEFINE_PAGE_WAIT_EXCLUSIVE(name, p, b)                          \
 325                 __DEFINE_PAGE_WAIT(name, p, b, WQ_FLAG_EXCLUSIVE)
 326
 327 static wait_queue_head_t *page_waitqueue(struct page *page)
 328 {
 329         const struct zone *zone = page_zone(page);
 330
 331         return &zone->wait_table[hash_ptr(page, zone->wait_table_bits)];
 332 }
 333
 334 static void wake_up_page(struct page *page)
 335 {
 336         const unsigned int mode = TASK_UNINTERRUPTIBLE | TASK_INTERRUPTIBLE;
 337         wait_queue_head_t *waitqueue = page_waitqueue(page);
 338
 339         if (waitqueue_active(waitqueue))
 340                 __wake_up(waitqueue, mode, 1, page);
 341 }
 342
 343 void fastcall wait_on_page_bit(struct page *page, int bit_nr)
 344 {
 345         wait_queue_head_t *waitqueue = page_waitqueue(page);
 346         DEFINE_PAGE_WAIT(wait, page, bit_nr);
 347
 348         do {
 349                 prepare_to_wait(waitqueue, &wait.wait, TASK_UNINTERRUPTIBLE);
 350                 if (test_bit(bit_nr, &page->flags)) {
 351                         sync_page(page);
 352                         io_schedule();
 353                 }
 354         } while (test_bit(bit_nr, &page->flags));
 355         finish_wait(waitqueue, &wait.wait);
 356 }
 357
 358 EXPORT_SYMBOL(wait_on_page_bit);
 359
 360 /**
 361  * unlock_page() - unlock a locked page
 362  *
 363  * @page: the page
 364  *
 365  * Unlocks the page and wakes up sleepers in ___wait_on_page_locked().
 366  * Also wakes sleepers in wait_on_page_writeback() because the wakeup
 367  * mechananism between PageLocked pages and PageWriteback pages is shared.
 368  * But that's OK - sleepers in wait_on_page_writeback() just go back to sleep.
 369  *
 370  * The first mb is necessary to safely close the critical section opened by the
 371  * TestSetPageLocked(), the second mb is necessary to enforce ordering between
 372  * the clear_bit and the read of the waitqueue (to avoid SMP races with a
 373  * parallel wait_on_page_locked()).
 374  */
 375 void fastcall unlock_page(struct page *page)
 376 {
 377         smp_mb__before_clear_bit();
 378         if (!TestClearPageLocked(page))
 379                 BUG();
 380         smp_mb__after_clear_bit();
 381         wake_up_page(page);
 382 }
 383
 384 EXPORT_SYMBOL(unlock_page);
 385 EXPORT_SYMBOL(lock_page);
 386
 387 /*
 388  * End writeback against a page.
 389  */
 390 void end_page_writeback(struct page *page)
 391 {
 392         if (!TestClearPageReclaim(page) || rotate_reclaimable_page(page)) {
 393                 if (!test_clear_page_writeback(page))
 394                         BUG();
 395                 smp_mb__after_clear_bit();
 396         }
 397         wake_up_page(page);
 398 }
 399
 400 EXPORT_SYMBOL(end_page_writeback);
 401
 402 /*
 403  * Get a lock on the page, assuming we need to sleep to get it.
 404  *
 405  * Ugly: running sync_page() in state TASK_UNINTERRUPTIBLE is scary.  If some
 406  * random driver's requestfn sets TASK_RUNNING, we could busywait.  However
 407  * chances are that on the second loop, the block layer's plug list is empty,
 408  * so sync_page() will then return in state TASK_UNINTERRUPTIBLE.
 409  */
 410 void fastcall __lock_page(struct page *page)
 411 {
 412         wait_queue_head_t *wqh = page_waitqueue(page);
 413         DEFINE_PAGE_WAIT_EXCLUSIVE(wait, page, PG_locked);
 414
 415         while (TestSetPageLocked(page)) {
 416                 prepare_to_wait_exclusive(wqh, &wait.wait, TASK_UNINTERRUPTIBLE);
 417                 if (PageLocked(page)) {
 418                         sync_page(page);
 419                         io_schedule();
 420                 }
 421         }
 422         finish_wait(wqh, &wait.wait);
 423 }
 424
 425 EXPORT_SYMBOL(__lock_page);
 426
 427 /*
 428  * a rather lightweight function, finding and getting a reference to a
 429  * hashed page atomically.
 430  */
 431 struct page * find_get_page(struct address_space *mapping, unsigned long offset)
 432 {
 433         struct page *page;
 434
 435         /*
 436          * We scan the hash list read-only. Addition to and removal from
 437          * the hash-list needs a held write-lock.
 438          */
 439         spin_lock_irq(&mapping->tree_lock);
 440         page = radix_tree_lookup(&mapping->page_tree, offset);
 441         if (page)
 442                 page_cache_get(page);
 443         spin_unlock_irq(&mapping->tree_lock);
 444         return page;
 445 }
 446
 447 EXPORT_SYMBOL(find_get_page);
 448
 449 /*
 450  * Same as above, but trylock it instead of incrementing the count.
 451  */
 452 struct page *find_trylock_page(struct address_space *mapping, unsigned long offset)
 453 {
 454         struct page *page;
 455
 456         spin_lock_irq(&mapping->tree_lock);
 457         page = radix_tree_lookup(&mapping->page_tree, offset);
 458         if (page && TestSetPageLocked(page))
 459                 page = NULL;
 460         spin_unlock_irq(&mapping->tree_lock);
 461         return page;
 462 }
 463
 464 EXPORT_SYMBOL(find_trylock_page);
 465
 466 /**
 467  * find_lock_page - locate, pin and lock a pagecache page
 468  *
 469  * @mapping - the address_space to search
 470  * @offset - the page index
 471  *
 472  * Locates the desired pagecache page, locks it, increments its reference
 473  * count and returns its address.
 474  *
 475  * Returns zero if the page was not present. find_lock_page() may sleep.
 476  */
 477 struct page *find_lock_page(struct address_space *mapping,
 478                                 unsigned long offset)
 479 {
 480         struct page *page;
 481
 482         spin_lock_irq(&mapping->tree_lock);
 483 repeat:
 484         page = radix_tree_lookup(&mapping->page_tree, offset);
 485         if (page) {
 486                 page_cache_get(page);
 487                 if (TestSetPageLocked(page)) {
 488                         spin_unlock_irq(&mapping->tree_lock);
 489                         lock_page(page);
 490                         spin_lock_irq(&mapping->tree_lock);
 491
 492                         /* Has the page been truncated while we slept? */
 493                         if (page->mapping != mapping || page->index != offset) {
 494                                 unlock_page(page);
 495                                 page_cache_release(page);
 496                                 goto repeat;
 497                         }
 498                 }
 499         }
 500         spin_unlock_irq(&mapping->tree_lock);
 501         return page;
 502 }
 503
 504 EXPORT_SYMBOL(find_lock_page);
 505
 506 /**
 507  * find_or_create_page - locate or add a pagecache page
 508  *
 509  * @mapping - the page's address_space
 510  * @index - the page's index into the mapping
 511  * @gfp_mask - page allocation mode
 512  *
 513  * Locates a page in the pagecache.  If the page is not present, a new page
 514  * is allocated using @gfp_mask and is added to the pagecache and to the VM's
 515  * LRU list.  The returned page is locked and has its reference count
 516  * incremented.
 517  *
 518  * find_or_create_page() may sleep, even if @gfp_flags specifies an atomic
 519  * allocation!
 520  *
 521  * find_or_create_page() returns the desired page's address, or zero on
 522  * memory exhaustion.
 523  */
 524 struct page *find_or_create_page(struct address_space *mapping,
 525                 unsigned long index, unsigned int gfp_mask)
 526 {
 527         struct page *page, *cached_page = NULL;
 528         int err;
 529 repeat:
 530         page = find_lock_page(mapping, index);
 531         if (!page) {
 532                 if (!cached_page) {
 533                         cached_page = alloc_page(gfp_mask);
 534                         if (!cached_page)
 535                                 return NULL;
 536                 }
 537                 err = add_to_page_cache_lru(cached_page, mapping,
 538                                         index, gfp_mask);
 539                 if (!err) {
 540                         page = cached_page;
 541                         cached_page = NULL;
 542                 } else if (err == -EEXIST)
 543                         goto repeat;
 544         }
 545         if (cached_page)
 546                 page_cache_release(cached_page);
 547         return page;
 548 }
 549
 550 EXPORT_SYMBOL(find_or_create_page);
 551
 552 /**
 553  * find_get_pages - gang pagecache lookup
 554  * @mapping:    The address_space to search
 555  * @start:      The starting page index
 556  * @nr_pages:   The maximum number of pages
 557  * @pages:      Where the resulting pages are placed
 558  *
 559  * find_get_pages() will search for and return a group of up to
 560  * @nr_pages pages in the mapping.  The pages are placed at @pages.
 561  * find_get_pages() takes a reference against the returned pages.
 562  *
 563  * The search returns a group of mapping-contiguous pages with ascending
 564  * indexes.  There may be holes in the indices due to not-present pages.
 565  *
 566  * find_get_pages() returns the number of pages which were found.
 567  */
 568 unsigned find_get_pages(struct address_space *mapping, pgoff_t start,
 569                             unsigned int nr_pages, struct page **pages)
 570 {
 571         unsigned int i;
 572         unsigned int ret;
 573
 574         spin_lock_irq(&mapping->tree_lock);
 575         ret = radix_tree_gang_lookup(&mapping->page_tree,
 576                                 (void **)pages, start, nr_pages);
 577         for (i = 0; i < ret; i++)
 578                 page_cache_get(pages[i]);
 579         spin_unlock_irq(&mapping->tree_lock);
 580         return ret;
 581 }
 582
 583 /*
 584  * Like find_get_pages, except we only return pages which are tagged with
 585  * `tag'.   We update *index to index the next page for the traversal.
 586  */
 587 unsigned find_get_pages_tag(struct address_space *mapping, pgoff_t *index,
 588                         int tag, unsigned int nr_pages, struct page **pages)
 589 {
 590         unsigned int i;
 591         unsigned int ret;
 592
 593         spin_lock_irq(&mapping->tree_lock);
 594         ret = radix_tree_gang_lookup_tag(&mapping->page_tree,
 595                                 (void **)pages, *index, nr_pages, tag);
 596         for (i = 0; i < ret; i++)
 597                 page_cache_get(pages[i]);
 598         if (ret)
 599                 *index = pages[ret - 1]->index + 1;
 600         spin_unlock_irq(&mapping->tree_lock);
 601         return ret;
 602 }
 603
 604 /*
 605  * Same as grab_cache_page, but do not wait if the page is unavailable.
 606  * This is intended for speculative data generators, where the data can
 607  * be regenerated if the page couldn't be grabbed.  This routine should
 608  * be safe to call while holding the lock for another page.
 609  *
 610  * Clear __GFP_FS when allocating the page to avoid recursion into the fs
 611  * and deadlock against the caller's locked page.
 612  */
 613 struct page *
 614 grab_cache_page_nowait(struct address_space *mapping, unsigned long index)
 615 {
 616         struct page *page = find_get_page(mapping, index);
 617         int gfp_mask;
 618
 619         if (page) {
 620                 if (!TestSetPageLocked(page))
 621                         return page;
 622                 page_cache_release(page);
 623                 return NULL;
 624         }
 625         gfp_mask = mapping_gfp_mask(mapping) & ~__GFP_FS;
 626         page = alloc_pages(gfp_mask, 0);
 627         if (page && add_to_page_cache_lru(page, mapping, index, gfp_mask)) {
 628                 page_cache_release(page);
 629                 page = NULL;
 630         }
 631         return page;
 632 }
 633
 634 EXPORT_SYMBOL(grab_cache_page_nowait);
 635
 636 /*
 637  * This is a generic file read routine, and uses the
 638  * mapping->a_ops->readpage() function for the actual low-level
 639  * stuff.
 640  *
 641  * This is really ugly. But the goto's actually try to clarify some
 642  * of the logic when it comes to error handling etc.
 643  * - note the struct file * is only passed for the use of readpage
 644  */
 645 void do_generic_mapping_read(struct address_space *mapping,
 646                              struct file_ra_state *_ra,
 647                              struct file * filp,
 648                              loff_t *ppos,
 649                              read_descriptor_t * desc,
 650                              read_actor_t actor,
 651                              int nonblock)
 652 {
 653         struct inode *inode = mapping->host;
 654         unsigned long index, offset;
 655         struct page *cached_page;
 656         int error;
 657         struct file_ra_state ra = *_ra;
 658
 659         cached_page = NULL;
 660         index = *ppos >> PAGE_CACHE_SHIFT;
 661         offset = *ppos & ~PAGE_CACHE_MASK;
 662
 663         for (;;) {
 664                 struct page *page;
 665                 unsigned long end_index, nr, ret;
 666                 loff_t isize = i_size_read(inode);
 667
 668                 end_index = isize >> PAGE_CACHE_SHIFT;
 669
 670                 if (index > end_index)
 671                         break;
 672                 nr = PAGE_CACHE_SIZE;
 673                 if (index == end_index) {
 674                         nr = isize & ~PAGE_CACHE_MASK;
 675                         if (nr <= offset)
 676                                 break;
 677                 }
 678
 679                 cond_resched();
 680                 page_cache_readahead(mapping, &ra, filp, index);
 681
 682                 nr = nr - offset;
 683 find_page:
 684                 page = find_get_page(mapping, index);
 685                 if (unlikely(page == NULL)) {
 686                         if (nonblock) {
 687                                 desc->error = -EWOULDBLOCKIO;
 688                                 break;
 689                         }
 690                         handle_ra_miss(mapping, &ra, index);
 691                         goto no_cached_page;
 692                 }
 693                 if (!PageUptodate(page)) {
 694                         if (nonblock) {
 695                                 page_cache_release(page);
 696                                 desc->error = -EWOULDBLOCKIO;
 697                                 break;
 698                         }
 699                         goto page_not_up_to_date;
 700                 }
 701 page_ok:
 702                 /* If users can be writing to this page using arbitrary
 703                  * virtual addresses, take care about potential aliasing
 704                  * before reading the page on the kernel side.
 705                  */
 706                 if (mapping_writably_mapped(mapping))
 707                         flush_dcache_page(page);
 708
 709                 /*
 710                  * Mark the page accessed if we read the beginning.
 711                  */
 712                 if (!offset)
 713                         mark_page_accessed(page);
 714
 715                 /*
 716                  * Ok, we have the page, and it's up-to-date, so
 717                  * now we can copy it to user space...
 718                  *
 719                  * The actor routine returns how many bytes were actually used..
 720                  * NOTE! This may not be the same as how much of a user buffer
 721                  * we filled up (we may be padding etc), so we can only update
 722                  * "pos" here (the actor routine has to update the user buffer
 723                  * pointers and the remaining count).
 724                  */
 725                 ret = actor(desc, page, offset, nr);
 726                 offset += ret;
 727                 index += offset >> PAGE_CACHE_SHIFT;
 728                 offset &= ~PAGE_CACHE_MASK;
 729
 730                 page_cache_release(page);
 731                 if (ret == nr && desc->count)
 732                         continue;
 733                 break;
 734
 735 page_not_up_to_date:
 736                 /* Get exclusive access to the page ... */
 737                 lock_page(page);
 738
 739                 /* Did it get unhashed before we got the lock? */
 740                 if (!page->mapping) {
 741                         unlock_page(page);
 742                         page_cache_release(page);
 743                         continue;
 744                 }
 745
 746                 /* Did somebody else fill it already? */
 747                 if (PageUptodate(page)) {
 748                         unlock_page(page);
 749                         goto page_ok;
 750                 }
 751
 752 readpage:
 753                 /* ... and start the actual read. The read will unlock the page. */
 754                 error = mapping->a_ops->readpage(filp, page);
 755
 756                 if (!error) {
 757                         if (PageUptodate(page))
 758                                 goto page_ok;
 759                         wait_on_page_locked(page);
 760                         if (PageUptodate(page))
 761                                 goto page_ok;
 762                         error = -EIO;
 763                 }
 764
 765                 /* UHHUH! A synchronous read error occurred. Report it */
 766                 desc->error = error;
 767                 page_cache_release(page);
 768                 break;
 769
 770 no_cached_page:
 771                 /*
 772                  * Ok, it wasn't cached, so we need to create a new
 773                  * page..
 774                  */
 775                 if (!cached_page) {
 776                         cached_page = page_cache_alloc_cold(mapping);
 777                         if (!cached_page) {
 778                                 desc->error = -ENOMEM;
 779                                 break;
 780                         }
 781                 }
 782                 error = add_to_page_cache_lru(cached_page, mapping,
 783                                                 index, GFP_KERNEL);
 784                 if (error) {
 785                         if (error == -EEXIST)
 786                                 goto find_page;
 787                         desc->error = error;
 788                         break;
 789                 }
 790                 page = cached_page;
 791                 cached_page = NULL;
 792                 goto readpage;
 793         }
 794
 795         *_ra = ra;
 796
 797         *ppos = ((loff_t) index << PAGE_CACHE_SHIFT) + offset;
 798         if (cached_page)
 799                 page_cache_release(cached_page);
 800         file_accessed(filp);
 801 }
 802
 803 EXPORT_SYMBOL(do_generic_mapping_read);
 804
 805 int file_read_actor(read_descriptor_t *desc, struct page *page,
 806                         unsigned long offset, unsigned long size)
 807 {
 808         char *kaddr;
 809         unsigned long left, count = desc->count;
 810
 811         if (size > count)
 812                 size = count;
 813
 814         /*
 815          * Faults on the destination of a read are common, so do it before
 816          * taking the kmap.
 817          */
 818         if (!fault_in_pages_writeable(desc->buf, size)) {
 819                 kaddr = kmap_atomic(page, KM_USER0);
 820                 left = __copy_to_user(desc->buf, kaddr + offset, size);
 821                 kunmap_atomic(kaddr, KM_USER0);
 822                 if (left == 0)
 823                         goto success;
 824         }
 825
 826         /* Do it the slow way */
 827         kaddr = kmap(page);
 828         left = __copy_to_user(desc->buf, kaddr + offset, size);
 829         kunmap(page);
 830
 831         if (left) {
 832                 size -= left;
 833                 desc->error = -EFAULT;
 834         }
 835 success:
 836         desc->count = count - size;
 837         desc->written += size;
 838         desc->buf += size;
 839         return size;
 840 }
 841
 842 /*
 843  * This is the "read()" routine for all filesystems
 844  * that can use the page cache directly.
 845  */
 846 ssize_t
 847 __generic_file_aio_read(struct kiocb *iocb, const struct iovec *iov,
 848                 unsigned long nr_segs, loff_t *ppos)
 849 {
 850         struct file *filp = iocb->ki_filp;
 851         ssize_t retval;
 852         unsigned long seg;
 853         size_t count;
 854
 855         count = 0;
 856         for (seg = 0; seg < nr_segs; seg++) {
 857                 const struct iovec *iv = &iov[seg];
 858
 859                 /*
 860                  * If any segment has a negative length, or the cumulative
 861                  * length ever wraps negative then return -EINVAL.
 862                  */
 863                 count += iv->iov_len;
 864                 if (unlikely((ssize_t)(count|iv->iov_len) < 0))
 865                         return -EINVAL;
 866                 if (access_ok(VERIFY_WRITE, iv->iov_base, iv->iov_len))
 867                         continue;
 868                 if (seg == 0)
 869                         return -EFAULT;
 870                 nr_segs = seg;
 871                 count -= iv->iov_len;   /* This segment is no good */
 872                 break;
 873         }
 874
 875         /* coalesce the iovecs and go direct-to-BIO for O_DIRECT */
 876         if (filp->f_flags & O_DIRECT) {
 877                 loff_t pos = *ppos, size;
 878                 struct address_space *mapping;
 879                 struct inode *inode;
 880
 881                 mapping = filp->f_mapping;
 882                 inode = mapping->host;
 883                 retval = 0;
 884                 if (!count)
 885                         goto out; /* skip atime */
 886                 size = i_size_read(inode);
 887                 if (pos < size) {
 888                         retval = generic_file_direct_IO(READ, iocb,
 889                                                 iov, pos, nr_segs);
 890                         if (retval >= 0 && !is_sync_kiocb(iocb))
 891                                 retval = -EIOCBQUEUED;
 892                         if (retval > 0)
 893                                 *ppos = pos + retval;
 894                 }
 895                 file_accessed(filp);
 896                 goto out;
 897         }
 898
 899         retval = 0;
 900         if (count) {
 901                 for (seg = 0; seg < nr_segs; seg++) {
 902                         read_descriptor_t desc;
 903
 904                         desc.written = 0;
 905                         desc.buf = iov[seg].iov_base;
 906                         desc.count = iov[seg].iov_len;
 907                         if (desc.count == 0)
 908                                 continue;
 909                         desc.error = 0;
 910                         do_generic_file_read(filp,ppos,&desc,file_read_actor,0);
 911                         retval += desc.written;
 912                         if (!retval) {
 913                                 retval = desc.error;
 914                                 break;
 915                         }
 916                 }
 917         }
 918 out:
 919         return retval;
 920 }
 921
 922 EXPORT_SYMBOL(__generic_file_aio_read);
 923
 924 ssize_t
 925 generic_file_aio_read(struct kiocb *iocb, char __user *buf, size_t count, loff_t pos)
 926 {
 927         struct iovec local_iov = { .iov_base = buf, .iov_len = count };
 928
 929         BUG_ON(iocb->ki_pos != pos);
 930         return __generic_file_aio_read(iocb, &local_iov, 1, &iocb->ki_pos);
 931 }
 932
 933 EXPORT_SYMBOL(generic_file_aio_read);
 934
 935 ssize_t
 936 generic_file_read(struct file *filp, char __user *buf, size_t count, loff_t *ppos)
 937 {
 938         struct iovec local_iov = { .iov_base = buf, .iov_len = count };
 939         struct kiocb kiocb;
 940         ssize_t ret;
 941
 942         init_sync_kiocb(&kiocb, filp);
 943         ret = __generic_file_aio_read(&kiocb, &local_iov, 1, ppos);
 944         if (-EIOCBQUEUED == ret)
 945                 ret = wait_on_sync_kiocb(&kiocb);
 946         return ret;
 947 }
 948
 949 EXPORT_SYMBOL(generic_file_read);
 950
 951 int file_send_actor(read_descriptor_t * desc, struct page *page, unsigned long offset, unsigned long size)
 952 {
 953         ssize_t written;
 954         unsigned long count = desc->count;
 955         struct file *file = (struct file *) desc->buf;
 956
 957         if (size > count)
 958                 size = count;
 959
 960         written = file->f_op->sendpage(file, page, offset,
 961                                        size, &file->f_pos, size<count);
 962         if (written < 0) {
 963                 desc->error = written;
 964                 written = 0;
 965         }
 966         desc->count = count - written;
 967         desc->written += written;
 968         return written;
 969 }
 970
 971 ssize_t generic_file_sendfile(struct file *in_file, loff_t *ppos,
 972                          size_t count, read_actor_t actor, void __user *target)
 973 {
 974         read_descriptor_t desc;
 975
 976         if (!count)
 977                 return 0;
 978
 979         desc.written = 0;
 980         desc.count = count;
 981         desc.buf = target;
 982         desc.error = 0;
 983
 984         do_generic_file_read(in_file, ppos, &desc, actor, 0);
 985         if (desc.written)
 986                 return desc.written;
 987         return desc.error;
 988 }
 989
 990 EXPORT_SYMBOL(generic_file_sendfile);
 991
 992 static ssize_t
 993 do_readahead(struct address_space *mapping, struct file *filp,
 994              unsigned long index, unsigned long nr)
 995 {
 996         if (!mapping || !mapping->a_ops || !mapping->a_ops->readpage)
 997                 return -EINVAL;
 998
 999         force_page_cache_readahead(mapping, filp, index,
1000                                         max_sane_readahead(nr));
1001         return 0;
1002 }
1003
1004 asmlinkage ssize_t sys_readahead(int fd, loff_t offset, size_t count)
1005 {
1006         ssize_t ret;
1007         struct file *file;
1008
1009         ret = -EBADF;
1010         file = fget(fd);
1011         if (file) {
1012                 if (file->f_mode & FMODE_READ) {
1013                         struct address_space *mapping = file->f_mapping;
1014                         unsigned long start = offset >> PAGE_CACHE_SHIFT;
1015                         unsigned long end = (offset + count - 1) >> PAGE_CACHE_SHIFT;
1016                         unsigned long len = end - start + 1;
1017                         ret = do_readahead(mapping, file, start, len);
1018                 }
1019                 fput(file);
1020         }
1021         return ret;
1022 }
1023
1024 #ifdef CONFIG_MMU
1025 /*
1026  * This adds the requested page to the page cache if it isn't already there,
1027  * and schedules an I/O to read in its contents from disk.
1028  */
1029 static int FASTCALL(page_cache_read(struct file * file, unsigned long offset));
1030 static int fastcall page_cache_read(struct file * file, unsigned long offset)
1031 {
1032         struct address_space *mapping = file->f_mapping;
1033         struct page *page;
1034         int error;
1035
1036         page = page_cache_alloc_cold(mapping);
1037         if (!page)
1038                 return -ENOMEM;
1039
1040         error = add_to_page_cache_lru(page, mapping, offset, GFP_KERNEL);
1041         if (!error) {
1042                 error = mapping->a_ops->readpage(file, page);
1043                 page_cache_release(page);
1044                 return error;
1045         }
1046
1047         /*
1048          * We arrive here in the unlikely event that someone
1049          * raced with us and added our page to the cache first
1050          * or we are out of memory for radix-tree nodes.
1051          */
1052         page_cache_release(page);
1053         return error == -EEXIST ? 0 : error;
1054 }
1055
1056 #define MMAP_LOTSAMISS  (100)
1057
1058 /*
1059  * filemap_nopage() is invoked via the vma operations vector for a
1060  * mapped memory region to read in file data during a page fault.
1061  *
1062  * The goto's are kind of ugly, but this streamlines the normal case of having
1063  * it in the page cache, and handles the special cases reasonably without
1064  * having a lot of duplicated code.
1065  */
1066 struct page * filemap_nopage(struct vm_area_struct * area, unsigned long address, int *type)
1067 {
1068         int error;
1069         struct file *file = area->vm_file;
1070         struct address_space *mapping = file->f_mapping;
1071         struct file_ra_state *ra = &file->f_ra;
1072         struct inode *inode = mapping->host;
1073         struct page *page;
1074         unsigned long size, pgoff, endoff;
1075         int did_readaround = 0, majmin = VM_FAULT_MINOR;
1076
1077         pgoff = ((address - area->vm_start) >> PAGE_CACHE_SHIFT) + area->vm_pgoff;
1078         endoff = ((area->vm_end - area->vm_start) >> PAGE_CACHE_SHIFT) + area->vm_pgoff;
1079
1080 retry_all:
1081         size = (i_size_read(inode) + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
1082         if (pgoff >= size)
1083                 goto outside_data_content;
1084
1085         /* If we don't want any read-ahead, don't bother */
1086         if (VM_RandomReadHint(area))
1087                 goto no_cached_page;
1088
1089         /*
1090          * The "size" of the file, as far as mmap is concerned, isn't bigger
1091          * than the mapping
1092          */
1093         if (size > endoff)
1094                 size = endoff;
1095
1096         /*
1097          * The readahead code wants to be told about each and every page
1098          * so it can build and shrink its windows appropriately
1099          *
1100          * For sequential accesses, we use the generic readahead logic.
1101          */
1102         if (VM_SequentialReadHint(area))
1103                 page_cache_readahead(mapping, ra, file, pgoff);
1104
1105         /*
1106          * Do we have something in the page cache already?
1107          */
1108 retry_find:
1109         page = find_get_page(mapping, pgoff);
1110         if (!page) {
1111                 unsigned long ra_pages;
1112
1113                 if (VM_SequentialReadHint(area)) {
1114                         handle_ra_miss(mapping, ra, pgoff);
1115                         goto no_cached_page;
1116                 }
1117                 ra->mmap_miss++;
1118
1119                 /*
1120                  * Do we miss much more than hit in this file? If so,
1121                  * stop bothering with read-ahead. It will only hurt.
1122                  */
1123                 if (ra->mmap_miss > ra->mmap_hit + MMAP_LOTSAMISS)
1124                         goto no_cached_page;
1125
1126                 /*
1127                  * To keep the pgmajfault counter straight, we need to
1128                  * check did_readaround, as this is an inner loop.
1129                  */
1130                 if (!did_readaround) {
1131                         majmin = VM_FAULT_MAJOR;
1132                         inc_page_state(pgmajfault);
1133                 }
1134                 did_readaround = 1;
1135                 ra_pages = max_sane_readahead(file->f_ra.ra_pages);
1136                 if (ra_pages) {
1137                         long start;
1138
1139                         start = pgoff - ra_pages / 2;
1140                         if (pgoff < 0)
1141                                 pgoff = 0;
1142                         do_page_cache_readahead(mapping, file, pgoff, ra_pages);
1143                 }
1144                 page = find_get_page(mapping, pgoff);
1145                 if (!page)
1146                         goto no_cached_page;
1147         }
1148
1149         if (!did_readaround)
1150                 ra->mmap_hit++;
1151
1152         /*
1153          * Ok, found a page in the page cache, now we need to check
1154          * that it's up-to-date.
1155          */
1156         if (!PageUptodate(page))
1157                 goto page_not_uptodate;
1158
1159 success:
1160         /*
1161          * Found the page and have a reference on it.
1162          */
1163         mark_page_accessed(page);
1164         if (type)
1165                 *type = majmin;
1166         return page;
1167
1168 outside_data_content:
1169         /*
1170          * An external ptracer can access pages that normally aren't
1171          * accessible..
1172          */
1173         if (area->vm_mm == current->mm)
1174                 return NULL;
1175         /* Fall through to the non-read-ahead case */
1176 no_cached_page:
1177         /*
1178          * We're only likely to ever get here if MADV_RANDOM is in
1179          * effect.
1180          */
1181         error = page_cache_read(file, pgoff);
1182
1183         /*
1184          * The page we want has now been added to the page cache.
1185          * In the unlikely event that someone removed it in the
1186          * meantime, we'll just come back here and read it again.
1187          */
1188         if (error >= 0)
1189                 goto retry_find;
1190
1191         /*
1192          * An error return from page_cache_read can result if the
1193          * system is low on memory, or a problem occurs while trying
1194          * to schedule I/O.
1195          */
1196         if (error == -ENOMEM)
1197                 return NOPAGE_OOM;
1198         return NULL;
1199
1200 page_not_uptodate:
1201         if (!did_readaround) {
1202                 majmin = VM_FAULT_MAJOR;
1203                 inc_page_state(pgmajfault);
1204         }
1205         lock_page(page);
1206
1207         /* Did it get unhashed while we waited for it? */
1208         if (!page->mapping) {
1209                 unlock_page(page);
1210                 page_cache_release(page);
1211                 goto retry_all;
1212         }
1213
1214         /* Did somebody else get it up-to-date? */
1215         if (PageUptodate(page)) {
1216                 unlock_page(page);
1217                 goto success;
1218         }
1219
1220         if (!mapping->a_ops->readpage(file, page)) {
1221                 wait_on_page_locked(page);
1222                 if (PageUptodate(page))
1223                         goto success;
1224         }
1225
1226         /*
1227          * Umm, take care of errors if the page isn't up-to-date.
1228          * Try to re-read it _once_. We do this synchronously,
1229          * because there really aren't any performance issues here
1230          * and we need to check for errors.
1231          */
1232         lock_page(page);
1233
1234         /* Somebody truncated the page on us? */
1235         if (!page->mapping) {
1236                 unlock_page(page);
1237                 page_cache_release(page);
1238                 goto retry_all;
1239         }
1240
1241         /* Somebody else successfully read it in? */
1242         if (PageUptodate(page)) {
1243                 unlock_page(page);
1244                 goto success;
1245         }
1246         ClearPageError(page);
1247         if (!mapping->a_ops->readpage(file, page)) {
1248                 wait_on_page_locked(page);
1249                 if (PageUptodate(page))
1250                         goto success;
1251         }
1252
1253         /*
1254          * Things didn't work out. Return zero to tell the
1255          * mm layer so, possibly freeing the page cache page first.
1256          */
1257         page_cache_release(page);
1258         return NULL;
1259 }
1260
1261 EXPORT_SYMBOL(filemap_nopage);
1262
1263 static struct page * filemap_getpage(struct file *file, unsigned long pgoff,
1264                                         int nonblock)
1265 {
1266         struct address_space *mapping = file->f_mapping;
1267         struct page *page;
1268         int error;
1269
1270         /*
1271          * Do we have something in the page cache already?
1272          */
1273 retry_find:
1274         page = find_get_page(mapping, pgoff);
1275         if (!page) {
1276                 if (nonblock)
1277                         return NULL;
1278                 goto no_cached_page;
1279         }
1280
1281         /*
1282          * Ok, found a page in the page cache, now we need to check
1283          * that it's up-to-date.
1284          */
1285         if (!PageUptodate(page))
1286                 goto page_not_uptodate;
1287
1288 success:
1289         /*
1290          * Found the page and have a reference on it.
1291          */
1292         mark_page_accessed(page);
1293         return page;
1294
1295 no_cached_page:
1296         error = page_cache_read(file, pgoff);
1297
1298         /*
1299          * The page we want has now been added to the page cache.
1300          * In the unlikely event that someone removed it in the
1301          * meantime, we'll just come back here and read it again.
1302          */
1303         if (error >= 0)
1304                 goto retry_find;
1305
1306         /*
1307          * An error return from page_cache_read can result if the
1308          * system is low on memory, or a problem occurs while trying
1309          * to schedule I/O.
1310          */
1311         return NULL;
1312
1313 page_not_uptodate:
1314         lock_page(page);
1315
1316         /* Did it get unhashed while we waited for it? */
1317         if (!page->mapping) {
1318                 unlock_page(page);
1319                 goto err;
1320         }
1321
1322         /* Did somebody else get it up-to-date? */
1323         if (PageUptodate(page)) {
1324                 unlock_page(page);
1325                 goto success;
1326         }
1327
1328         if (!mapping->a_ops->readpage(file, page)) {
1329                 wait_on_page_locked(page);
1330                 if (PageUptodate(page))
1331                         goto success;
1332         }
1333
1334         /*
1335          * Umm, take care of errors if the page isn't up-to-date.
1336          * Try to re-read it _once_. We do this synchronously,
1337          * because there really aren't any performance issues here
1338          * and we need to check for errors.
1339          */
1340         lock_page(page);
1341
1342         /* Somebody truncated the page on us? */
1343         if (!page->mapping) {
1344                 unlock_page(page);
1345                 goto err;
1346         }
1347         /* Somebody else successfully read it in? */
1348         if (PageUptodate(page)) {
1349                 unlock_page(page);
1350                 goto success;
1351         }
1352
1353         ClearPageError(page);
1354         if (!mapping->a_ops->readpage(file, page)) {
1355                 wait_on_page_locked(page);
1356                 if (PageUptodate(page))
1357                         goto success;
1358         }
1359
1360         /*
1361          * Things didn't work out. Return zero to tell the
1362          * mm layer so, possibly freeing the page cache page first.
1363          */
1364 err:
1365         page_cache_release(page);
1366
1367         return NULL;
1368 }
1369
1370 static int filemap_populate(struct vm_area_struct *vma,
1371                         unsigned long addr,
1372                         unsigned long len,
1373                         pgprot_t prot,
1374                         unsigned long pgoff,
1375                         int nonblock)
1376 {
1377         struct file *file = vma->vm_file;
1378         struct address_space *mapping = file->f_mapping;
1379         struct inode *inode = mapping->host;
1380         unsigned long size;
1381         struct mm_struct *mm = vma->vm_mm;
1382         struct page *page;
1383         int err;
1384
1385         if (!nonblock)
1386                 force_page_cache_readahead(mapping, vma->vm_file,
1387                                         pgoff, len >> PAGE_CACHE_SHIFT);
1388
1389 repeat:
1390         size = (i_size_read(inode) + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
1391         if (pgoff + (len >> PAGE_CACHE_SHIFT) > size)
1392                 return -EINVAL;
1393
1394         page = filemap_getpage(file, pgoff, nonblock);
1395         if (!page && !nonblock)
1396                 return -ENOMEM;
1397         if (page) {
1398                 err = install_page(mm, vma, addr, page, prot);
1399                 if (err) {
1400                         page_cache_release(page);
1401                         return err;
1402                 }
1403         } else {
1404                 /*
1405                  * If a nonlinear mapping then store the file page offset
1406                  * in the pte.
1407                  */
1408                 if (pgoff != linear_page_index(vma, addr)) {
1409                         err = install_file_pte(mm, vma, addr, pgoff, prot);
1410                         if (err)
1411                                 return err;
1412                 }
1413         }
1414
1415         len -= PAGE_SIZE;
1416         addr += PAGE_SIZE;
1417         pgoff++;
1418         if (len)
1419                 goto repeat;
1420
1421         return 0;
1422 }
1423
1424 static struct vm_operations_struct generic_file_vm_ops = {
1425         .nopage         = filemap_nopage,
1426         .populate       = filemap_populate,
1427 };
1428
1429 /* This is used for a general mmap of a disk file */
1430
1431 int generic_file_mmap(struct file * file, struct vm_area_struct * vma)
1432 {
1433         struct address_space *mapping = file->f_mapping;
1434
1435         if (!mapping->a_ops->readpage)
1436                 return -ENOEXEC;
1437         file_accessed(file);
1438         vma->vm_ops = &generic_file_vm_ops;
1439         return 0;
1440 }
1441
1442 /*
1443  * This is for filesystems which do not implement ->writepage.
1444  */
1445 int generic_file_readonly_mmap(struct file *file, struct vm_area_struct *vma)
1446 {
1447         if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE))
1448                 return -EINVAL;
1449         return generic_file_mmap(file, vma);
1450 }
1451 #else
1452 int generic_file_mmap(struct file * file, struct vm_area_struct * vma)
1453 {
1454         return -ENOSYS;
1455 }
1456 int generic_file_readonly_mmap(struct file * file, struct vm_area_struct * vma)
1457 {
1458         return -ENOSYS;
1459 }
1460 #endif /* CONFIG_MMU */
1461
1462 EXPORT_SYMBOL(generic_file_mmap);
1463 EXPORT_SYMBOL(generic_file_readonly_mmap);
1464
1465 static inline struct page *__read_cache_page(struct address_space *mapping,
1466                                 unsigned long index,
1467                                 int (*filler)(void *,struct page*),
1468                                 void *data)
1469 {
1470         struct page *page, *cached_page = NULL;
1471         int err;
1472 repeat:
1473         page = find_get_page(mapping, index);
1474         if (!page) {
1475                 if (!cached_page) {
1476                         cached_page = page_cache_alloc_cold(mapping);
1477                         if (!cached_page)
1478                                 return ERR_PTR(-ENOMEM);
1479                 }
1480                 err = add_to_page_cache_lru(cached_page, mapping,
1481                                         index, GFP_KERNEL);
1482                 if (err == -EEXIST)
1483                         goto repeat;
1484                 if (err < 0) {
1485                         /* Presumably ENOMEM for radix tree node */
1486                         page_cache_release(cached_page);
1487                         return ERR_PTR(err);
1488                 }
1489                 page = cached_page;
1490                 cached_page = NULL;
1491                 err = filler(data, page);
1492                 if (err < 0) {
1493                         page_cache_release(page);
1494                         page = ERR_PTR(err);
1495                 }
1496         }
1497         if (cached_page)
1498                 page_cache_release(cached_page);
1499         return page;
1500 }
1501
1502 /*
1503  * Read into the page cache. If a page already exists,
1504  * and PageUptodate() is not set, try to fill the page.
1505  */
1506 struct page *read_cache_page(struct address_space *mapping,
1507                                 unsigned long index,
1508                                 int (*filler)(void *,struct page*),
1509                                 void *data)
1510 {
1511         struct page *page;
1512         int err;
1513
1514 retry:
1515         page = __read_cache_page(mapping, index, filler, data);
1516         if (IS_ERR(page))
1517                 goto out;
1518         mark_page_accessed(page);
1519         if (PageUptodate(page))
1520                 goto out;
1521
1522         lock_page(page);
1523         if (!page->mapping) {
1524                 unlock_page(page);
1525                 page_cache_release(page);
1526                 goto retry;
1527         }
1528         if (PageUptodate(page)) {
1529                 unlock_page(page);
1530                 goto out;
1531         }
1532         err = filler(data, page);
1533         if (err < 0) {
1534                 page_cache_release(page);
1535                 page = ERR_PTR(err);
1536         }
1537  out:
1538         return page;
1539 }
1540
1541 EXPORT_SYMBOL(read_cache_page);
1542
1543 /*
1544  * If the page was newly created, increment its refcount and add it to the
1545  * caller's lru-buffering pagevec.  This function is specifically for
1546  * generic_file_write().
1547  */
1548 static inline struct page *
1549 __grab_cache_page(struct address_space *mapping, unsigned long index,
1550                         struct page **cached_page, struct pagevec *lru_pvec)
1551 {
1552         int err;
1553         struct page *page;
1554 repeat:
1555         page = find_lock_page(mapping, index);
1556         if (!page) {
1557                 if (!*cached_page) {
1558                         *cached_page = page_cache_alloc(mapping);
1559                         if (!*cached_page)
1560                                 return NULL;
1561                 }
1562                 err = add_to_page_cache(*cached_page, mapping,
1563                                         index, GFP_KERNEL);
1564                 if (err == -EEXIST)
1565                         goto repeat;
1566                 if (err == 0) {
1567                         page = *cached_page;
1568                         page_cache_get(page);
1569                         if (!pagevec_add(lru_pvec, page))
1570                                 __pagevec_lru_add(lru_pvec);
1571                         *cached_page = NULL;
1572                 }
1573         }
1574         return page;
1575 }
1576
1577 /*
1578  * The logic we want is
1579  *
1580  *      if suid or (sgid and xgrp)
1581  *              remove privs
1582  */
1583 int remove_suid(struct dentry *dentry)
1584 {
1585         mode_t mode = dentry->d_inode->i_mode;
1586         int kill = 0;
1587         int result = 0;
1588
1589         /* suid always must be killed */
1590         if (unlikely(mode & S_ISUID))
1591                 kill = ATTR_KILL_SUID;
1592
1593         /*
1594          * sgid without any exec bits is just a mandatory locking mark; leave
1595          * it alone.  If some exec bits are set, it's a real sgid; kill it.
1596          */
1597         if (unlikely((mode & S_ISGID) && (mode & S_IXGRP)))
1598                 kill |= ATTR_KILL_SGID;
1599
1600         if (unlikely(kill && !capable(CAP_FSETID))) {
1601                 struct iattr newattrs;
1602
1603                 newattrs.ia_valid = ATTR_FORCE | kill;
1604                 result = notify_change(dentry, &newattrs);
1605         }
1606         return result;
1607 }
1608 EXPORT_SYMBOL(remove_suid);
1609
1610 /*
1611  * Copy as much as we can into the page and return the number of bytes which
1612  * were sucessfully copied.  If a fault is encountered then clear the page
1613  * out to (offset+bytes) and return the number of bytes which were copied.
1614  */
1615 static inline size_t
1616 filemap_copy_from_user(struct page *page, unsigned long offset,
1617                         const char __user *buf, unsigned bytes)
1618 {
1619         char *kaddr;
1620         int left;
1621
1622         kaddr = kmap_atomic(page, KM_USER0);
1623         left = __copy_from_user(kaddr + offset, buf, bytes);
1624         kunmap_atomic(kaddr, KM_USER0);
1625
1626         if (left != 0) {
1627                 /* Do it the slow way */
1628                 kaddr = kmap(page);
1629                 left = __copy_from_user(kaddr + offset, buf, bytes);
1630                 kunmap(page);
1631         }
1632         return bytes - left;
1633 }
1634
1635 static size_t
1636 __filemap_copy_from_user_iovec(char *vaddr,
1637                         const struct iovec *iov, size_t base, size_t bytes)
1638 {
1639         size_t copied = 0, left = 0;
1640
1641         while (bytes) {
1642                 char __user *buf = iov->iov_base + base;
1643                 int copy = min(bytes, iov->iov_len - base);
1644
1645                 base = 0;
1646                 left = __copy_from_user(vaddr, buf, copy);
1647                 copied += copy;
1648                 bytes -= copy;
1649                 vaddr += copy;
1650                 iov++;
1651
1652                 if (unlikely(left)) {
1653                         /* zero the rest of the target like __copy_from_user */
1654                         if (bytes)
1655                                 memset(vaddr, 0, bytes);
1656                         break;
1657                 }
1658         }
1659         return copied - left;
1660 }
1661
1662 /*
1663  * This has the same sideeffects and return value as filemap_copy_from_user().
1664  * The difference is that on a fault we need to memset the remainder of the
1665  * page (out to offset+bytes), to emulate filemap_copy_from_user()'s
1666  * single-segment behaviour.
1667  */
1668 static inline size_t
1669 filemap_copy_from_user_iovec(struct page *page, unsigned long offset,
1670                         const struct iovec *iov, size_t base, size_t bytes)
1671 {
1672         char *kaddr;
1673         size_t copied;
1674
1675         kaddr = kmap_atomic(page, KM_USER0);
1676         copied = __filemap_copy_from_user_iovec(kaddr + offset, iov,
1677                                                 base, bytes);
1678         kunmap_atomic(kaddr, KM_USER0);
1679         if (copied != bytes) {
1680                 kaddr = kmap(page);
1681                 copied = __filemap_copy_from_user_iovec(kaddr + offset, iov,
1682                                                         base, bytes);
1683                 kunmap(page);
1684         }
1685         return copied;
1686 }
1687
1688 static inline void
1689 filemap_set_next_iovec(const struct iovec **iovp, size_t *basep, size_t bytes)
1690 {
1691         const struct iovec *iov = *iovp;
1692         size_t base = *basep;
1693
1694         while (bytes) {
1695                 int copy = min(bytes, iov->iov_len - base);
1696
1697                 bytes -= copy;
1698                 base += copy;
1699                 if (iov->iov_len == base) {
1700                         iov++;
1701                         base = 0;
1702                 }
1703         }
1704         *iovp = iov;
1705         *basep = base;
1706 }
1707
1708 /*
1709  * Performs necessary checks before doing a write
1710  *
1711  * Can adjust writing position aor amount of bytes to write.
1712  * Returns appropriate error code that caller should return or
1713  * zero in case that write should be allowed.
1714  */
1715 inline int generic_write_checks(struct file *file, loff_t *pos, size_t *count, int isblk)
1716 {
1717         struct inode *inode = file->f_mapping->host;
1718         unsigned long limit = current->rlim[RLIMIT_FSIZE].rlim_cur;
1719
1720         if (unlikely(*pos < 0))
1721                 return -EINVAL;
1722
1723         if (unlikely(file->f_error)) {
1724                 int err = file->f_error;
1725                 file->f_error = 0;
1726                 return err;
1727         }
1728
1729         if (!isblk) {
1730                 /* FIXME: this is for backwards compatibility with 2.4 */
1731                 if (file->f_flags & O_APPEND)
1732                         *pos = i_size_read(inode);
1733
1734                 if (limit != RLIM_INFINITY) {
1735                         if (*pos >= limit) {
1736                                 send_sig(SIGXFSZ, current, 0);
1737                                 return -EFBIG;
1738                         }
1739                         if (*count > limit - (typeof(limit))*pos) {
1740                                 *count = limit - (typeof(limit))*pos;
1741                         }
1742                 }
1743         }
1744
1745         /*
1746          * LFS rule
1747          */
1748         if (unlikely(*pos + *count > MAX_NON_LFS &&
1749                                 !(file->f_flags & O_LARGEFILE))) {
1750                 if (*pos >= MAX_NON_LFS) {
1751                         send_sig(SIGXFSZ, current, 0);
1752                         return -EFBIG;
1753                 }
1754                 if (*count > MAX_NON_LFS - (unsigned long)*pos) {
1755                         *count = MAX_NON_LFS - (unsigned long)*pos;
1756                 }
1757         }
1758
1759         /*
1760          * Are we about to exceed the fs block limit ?
1761          *
1762          * If we have written data it becomes a short write.  If we have
1763          * exceeded without writing data we send a signal and return EFBIG.
1764          * Linus frestrict idea will clean these up nicely..
1765          */
1766         if (likely(!isblk)) {
1767                 if (unlikely(*pos >= inode->i_sb->s_maxbytes)) {
1768                         if (*count || *pos > inode->i_sb->s_maxbytes) {
1769                                 send_sig(SIGXFSZ, current, 0);
1770                                 return -EFBIG;
1771                         }
1772                         /* zero-length writes at ->s_maxbytes are OK */
1773                 }
1774
1775                 if (unlikely(*pos + *count > inode->i_sb->s_maxbytes))
1776                         *count = inode->i_sb->s_maxbytes - *pos;
1777         } else {
1778                 loff_t isize;
1779                 if (bdev_read_only(I_BDEV(inode)))
1780                         return -EPERM;
1781                 isize = i_size_read(inode);
1782                 if (*pos >= isize) {
1783                         if (*count || *pos > isize)
1784                                 return -ENOSPC;
1785                 }
1786
1787                 if (*pos + *count > isize)
1788                         *count = isize - *pos;
1789         }
1790         return 0;
1791 }
1792
1793 EXPORT_SYMBOL(generic_write_checks);
1794
1795 /*
1796  * Write to a file through the page cache.
1797  * Called under i_sem for S_ISREG files.
1798  *
1799  * We put everything into the page cache prior to writing it. This is not a
1800  * problem when writing full pages. With partial pages, however, we first have
1801  * to read the data into the cache, then dirty the page, and finally schedule
1802  * it for writing by marking it dirty.
1803  *                                                      okir@monad.swb.de
1804  */
1805 ssize_t
1806 generic_file_aio_write_nolock(struct kiocb *iocb, const struct iovec *iov,
1807                                 unsigned long nr_segs, loff_t *ppos)
1808 {
1809         struct file *file = iocb->ki_filp;
1810         struct address_space * mapping = file->f_mapping;
1811         struct address_space_operations *a_ops = mapping->a_ops;
1812         size_t ocount;          /* original count */
1813         size_t count;           /* after file limit checks */
1814         struct inode    *inode = mapping->host;
1815         long            status = 0;
1816         loff_t          pos;
1817         struct page     *page;
1818         struct page     *cached_page = NULL;
1819         const int       isblk = S_ISBLK(inode->i_mode);
1820         ssize_t         written;
1821         ssize_t         err;
1822         size_t          bytes;
1823         struct pagevec  lru_pvec;
1824         const struct iovec *cur_iov = iov; /* current iovec */
1825         size_t          iov_base = 0;      /* offset in the current iovec */
1826         unsigned long   seg;
1827         char __user     *buf;
1828
1829         ocount = 0;
1830         for (seg = 0; seg < nr_segs; seg++) {
1831                 const struct iovec *iv = &iov[seg];
1832
1833                 /*
1834                  * If any segment has a negative length, or the cumulative
1835                  * length ever wraps negative then return -EINVAL.
1836                  */
1837                 ocount += iv->iov_len;
1838                 if (unlikely((ssize_t)(ocount|iv->iov_len) < 0))
1839                         return -EINVAL;
1840                 if (access_ok(VERIFY_READ, iv->iov_base, iv->iov_len))
1841                         continue;
1842                 if (seg == 0)
1843                         return -EFAULT;
1844                 nr_segs = seg;
1845                 ocount -= iv->iov_len;  /* This segment is no good */
1846                 break;
1847         }
1848
1849         count = ocount;
1850         pos = *ppos;
1851         pagevec_init(&lru_pvec, 0);
1852
1853         /* We can write back this queue in page reclaim */
1854         current->backing_dev_info = mapping->backing_dev_info;
1855         written = 0;
1856
1857         err = generic_write_checks(file, &pos, &count, isblk);
1858         if (err)
1859                 goto out;
1860
1861         if (count == 0)
1862                 goto out;
1863
1864         err = remove_suid(file->f_dentry);
1865         if (err)
1866                 goto out;
1867
1868         inode_update_time(inode, 1);
1869
1870         /* coalesce the iovecs and go direct-to-BIO for O_DIRECT */
1871         if (unlikely(file->f_flags & O_DIRECT)) {
1872                 if (count != ocount)
1873                         nr_segs = iov_shorten((struct iovec *)iov,
1874                                                 nr_segs, count);
1875                 written = generic_file_direct_IO(WRITE, iocb,
1876                                         iov, pos, nr_segs);
1877                 if (written > 0) {
1878                         loff_t end = pos + written;
1879                         if (end > i_size_read(inode) && !isblk) {
1880                                 i_size_write(inode,  end);
1881                                 mark_inode_dirty(inode);
1882                         }
1883                         *ppos = end;
1884                 }
1885                 /*
1886                  * Sync the fs metadata but not the minor inode changes and
1887                  * of course not the data as we did direct DMA for the IO.
1888                  * i_sem is held, which protects generic_osync_inode() from
1889                  * livelocking.
1890                  */
1891                 if (written >= 0 && file->f_flags & O_SYNC)
1892                         status = generic_osync_inode(inode, mapping, OSYNC_METADATA);
1893                 if (written == count && !is_sync_kiocb(iocb))
1894                         written = -EIOCBQUEUED;
1895                 if (written < 0 || written == count)
1896                         goto out_status;
1897                 /*
1898                  * direct-io write to a hole: fall through to buffered I/O
1899                  * for completing the rest of the request.
1900                  */
1901                 pos += written;
1902                 count -= written;
1903         }
1904
1905         buf = iov->iov_base;
1906         do {
1907                 unsigned long index;
1908                 unsigned long offset;
1909                 size_t copied;
1910
1911                 offset = (pos & (PAGE_CACHE_SIZE -1)); /* Within page */
1912                 index = pos >> PAGE_CACHE_SHIFT;
1913                 bytes = PAGE_CACHE_SIZE - offset;
1914                 if (bytes > count)
1915                         bytes = count;
1916
1917                 /*
1918                  * Bring in the user page that we will copy from _first_.
1919                  * Otherwise there's a nasty deadlock on copying from the
1920                  * same page as we're writing to, without it being marked
1921                  * up-to-date.
1922                  */
1923                 fault_in_pages_readable(buf, bytes);
1924
1925                 page = __grab_cache_page(mapping,index,&cached_page,&lru_pvec);
1926                 if (!page) {
1927                         status = -ENOMEM;
1928                         break;
1929                 }
1930
1931                 status = a_ops->prepare_write(file, page, offset, offset+bytes);
1932                 if (unlikely(status)) {
1933                         loff_t isize = i_size_read(inode);
1934                         /*
1935                          * prepare_write() may have instantiated a few blocks
1936                          * outside i_size.  Trim these off again.
1937                          */
1938                         unlock_page(page);
1939                         page_cache_release(page);
1940                         if (pos + bytes > isize)
1941                                 vmtruncate(inode, isize);
1942                         break;
1943                 }
1944                 if (likely(nr_segs == 1))
1945                         copied = filemap_copy_from_user(page, offset,
1946                                                         buf, bytes);
1947                 else
1948                         copied = filemap_copy_from_user_iovec(page, offset,
1949                                                 cur_iov, iov_base, bytes);
1950                 flush_dcache_page(page);
1951                 status = a_ops->commit_write(file, page, offset, offset+bytes);
1952                 if (likely(copied > 0)) {
1953                         if (!status)
1954                                 status = copied;
1955
1956                         if (status >= 0) {
1957                                 written += status;
1958                                 count -= status;
1959                                 pos += status;
1960                                 buf += status;
1961                                 if (unlikely(nr_segs > 1))
1962                                         filemap_set_next_iovec(&cur_iov,
1963                                                         &iov_base, status);
1964                         }
1965                 }
1966                 if (unlikely(copied != bytes))
1967                         if (status >= 0)
1968                                 status = -EFAULT;
1969                 unlock_page(page);
1970                 mark_page_accessed(page);
1971                 page_cache_release(page);
1972                 if (status < 0)
1973                         break;
1974                 balance_dirty_pages_ratelimited(mapping);
1975                 cond_resched();
1976         } while (count);
1977         *ppos = pos;
1978
1979         if (cached_page)
1980                 page_cache_release(cached_page);
1981
1982         /*
1983          * For now, when the user asks for O_SYNC, we'll actually give O_DSYNC
1984          */
1985         if (status >= 0) {
1986                 if ((file->f_flags & O_SYNC) || IS_SYNC(inode))
1987                         status = generic_osync_inode(inode, mapping,
1988                                         OSYNC_METADATA|OSYNC_DATA);
1989         }
1990
1991         /*
1992          * If we get here for O_DIRECT writes then we must have fallen through
1993          * to buffered writes (block instantiation inside i_size).  So we sync
1994          * the file data here, to try to honour O_DIRECT expectations.
1995          */
1996         if (unlikely(file->f_flags & O_DIRECT) && written)
1997                 status = filemap_write_and_wait(mapping);
1998
1999 out_status:
2000         err = written ? written : status;
2001 out:
2002         pagevec_lru_add(&lru_pvec);
2003         current->backing_dev_info = 0;
2004         return err;
2005 }
2006
2007 EXPORT_SYMBOL(generic_file_aio_write_nolock);
2008
2009 ssize_t
2010 generic_file_write_nolock(struct file *file, const struct iovec *iov,
2011                                 unsigned long nr_segs, loff_t *ppos)
2012 {
2013         struct kiocb kiocb;
2014         ssize_t ret;
2015
2016         init_sync_kiocb(&kiocb, file);
2017         ret = generic_file_aio_write_nolock(&kiocb, iov, nr_segs, ppos);
2018         if (-EIOCBQUEUED == ret)
2019                 ret = wait_on_sync_kiocb(&kiocb);
2020         return ret;
2021 }
2022
2023 EXPORT_SYMBOL(generic_file_write_nolock);
2024
2025 ssize_t generic_file_aio_write(struct kiocb *iocb, const char __user *buf,
2026                                size_t count, loff_t pos)
2027 {
2028         struct file *file = iocb->ki_filp;
2029         struct inode *inode = file->f_mapping->host;
2030         ssize_t err;
2031         struct iovec local_iov = { .iov_base = (void __user *)buf, .iov_len = count };
2032
2033         BUG_ON(iocb->ki_pos != pos);
2034
2035         down(&inode->i_sem);
2036         err = generic_file_aio_write_nolock(iocb, &local_iov, 1,
2037                                                 &iocb->ki_pos);
2038         up(&inode->i_sem);
2039
2040         return err;
2041 }
2042
2043 EXPORT_SYMBOL(generic_file_aio_write);
2044
2045 ssize_t generic_file_write(struct file *file, const char __user *buf,
2046                            size_t count, loff_t *ppos)
2047 {
2048         struct inode    *inode = file->f_mapping->host;
2049         ssize_t         err;
2050         struct iovec local_iov = { .iov_base = (void __user *)buf, .iov_len = count };
2051
2052         down(&inode->i_sem);
2053         err = generic_file_write_nolock(file, &local_iov, 1, ppos);
2054         up(&inode->i_sem);
2055
2056         return err;
2057 }
2058
2059 EXPORT_SYMBOL(generic_file_write);
2060
2061 ssize_t generic_file_readv(struct file *filp, const struct iovec *iov,
2062                         unsigned long nr_segs, loff_t *ppos)
2063 {
2064         struct kiocb kiocb;
2065         ssize_t ret;
2066
2067         init_sync_kiocb(&kiocb, filp);
2068         ret = __generic_file_aio_read(&kiocb, iov, nr_segs, ppos);
2069         if (-EIOCBQUEUED == ret)
2070                 ret = wait_on_sync_kiocb(&kiocb);
2071         return ret;
2072 }
2073
2074 EXPORT_SYMBOL(generic_file_readv);
2075
2076 ssize_t generic_file_writev(struct file *file, const struct iovec *iov,
2077                         unsigned long nr_segs, loff_t * ppos)
2078 {
2079         struct inode *inode = file->f_mapping->host;
2080         ssize_t ret;
2081
2082         down(&inode->i_sem);
2083         ret = generic_file_write_nolock(file, iov, nr_segs, ppos);
2084         up(&inode->i_sem);
2085         return ret;
2086 }
2087
2088 EXPORT_SYMBOL(generic_file_writev);
2089
2090 /*
2091  * Called under i_sem for writes to S_ISREG files
2092  */
2093 ssize_t
2094 generic_file_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov,
2095         loff_t offset, unsigned long nr_segs)
2096 {
2097         struct file *file = iocb->ki_filp;
2098         struct address_space *mapping = file->f_mapping;
2099         ssize_t retval;
2100
2101         retval = filemap_write_and_wait(mapping);
2102         if (retval == 0) {
2103                 retval = mapping->a_ops->direct_IO(rw, iocb, iov,
2104                                                 offset, nr_segs);
2105                 if (rw == WRITE && mapping->nrpages)
2106                         invalidate_inode_pages2(mapping);
2107         }
2108         return retval;
2109 }
2110
2111 EXPORT_SYMBOL_GPL(generic_file_direct_IO);