fs/xfs/linux-2.6/xfs_buf.c

   1 /*
   2  * Copyright (c) 2000-2004 Silicon Graphics, Inc.  All Rights Reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms of version 2 of the GNU General Public License as
   6  * published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it would be useful, but
   9  * WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
  11  *
  12  * Further, this software is distributed without any warranty that it is
  13  * free of the rightful claim of any third person regarding infringement
  14  * or the like.  Any license provided herein, whether implied or
  15  * otherwise, applies only to this software file.  Patent licenses, if
  16  * any, provided herein do not apply to combinations of this program with
  17  * other software, or any other product whatsoever.
  18  *
  19  * You should have received a copy of the GNU General Public License along
  20  * with this program; if not, write the Free Software Foundation, Inc., 59
  21  * Temple Place - Suite 330, Boston MA 02111-1307, USA.
  22  *
  23  * Contact information: Silicon Graphics, Inc., 1600 Amphitheatre Pkwy,
  24  * Mountain View, CA  94043, or:
  25  *
  26  * http://www.sgi.com
  27  *
  28  * For further information regarding this notice, see:
  29  *
  30  * http://oss.sgi.com/projects/GenInfo/SGIGPLNoticeExplan/
  31  */
  32
  33 /*
  34  *      The xfs_buf.c code provides an abstract buffer cache model on top
  35  *      of the Linux page cache.  Cached metadata blocks for a file system
  36  *      are hashed to the inode for the block device.  xfs_buf.c assembles
  37  *      buffers (xfs_buf_t) on demand to aggregate such cached pages for I/O.
  38  *
  39  *      Written by Steve Lord, Jim Mostek, Russell Cattelan
  40  *                  and Rajagopal Ananthanarayanan ("ananth") at SGI.
  41  *
  42  */
  43
  44 #include <linux/stddef.h>
  45 #include <linux/errno.h>
  46 #include <linux/slab.h>
  47 #include <linux/pagemap.h>
  48 #include <linux/init.h>
  49 #include <linux/vmalloc.h>
  50 #include <linux/bio.h>
  51 #include <linux/sysctl.h>
  52 #include <linux/proc_fs.h>
  53 #include <linux/workqueue.h>
  54 #include <linux/suspend.h>
  55 #include <linux/percpu.h>
  56
  57 #include "xfs_linux.h"
  58
  59 #ifndef GFP_READAHEAD
  60 #define GFP_READAHEAD   (__GFP_NOWARN|__GFP_NORETRY)
  61 #endif
  62
  63 /*
  64  * File wide globals
  65  */
  66
  67 STATIC kmem_cache_t *pagebuf_cache;
  68 STATIC void pagebuf_daemon_wakeup(void);
  69 STATIC void pagebuf_delwri_queue(xfs_buf_t *, int);
  70 STATIC struct workqueue_struct *pagebuf_logio_workqueue;
  71 STATIC struct workqueue_struct *pagebuf_dataio_workqueue;
  72
  73 /*
  74  * Pagebuf debugging
  75  */
  76
  77 #ifdef PAGEBUF_TRACE
  78 void
  79 pagebuf_trace(
  80         xfs_buf_t       *pb,
  81         char            *id,
  82         void            *data,
  83         void            *ra)
  84 {
  85         ktrace_enter(pagebuf_trace_buf,
  86                 pb, id,
  87                 (void *)(unsigned long)pb->pb_flags,
  88                 (void *)(unsigned long)pb->pb_hold.counter,
  89                 (void *)(unsigned long)pb->pb_sema.count.counter,
  90                 (void *)current,
  91                 data, ra,
  92                 (void *)(unsigned long)((pb->pb_file_offset>>32) & 0xffffffff),
  93                 (void *)(unsigned long)(pb->pb_file_offset & 0xffffffff),
  94                 (void *)(unsigned long)pb->pb_buffer_length,
  95                 NULL, NULL, NULL, NULL, NULL);
  96 }
  97 ktrace_t *pagebuf_trace_buf;
  98 #define PAGEBUF_TRACE_SIZE      4096
  99 #define PB_TRACE(pb, id, data)  \
 100         pagebuf_trace(pb, id, (void *)data, (void *)__builtin_return_address(0))
 101 #else
 102 #define PB_TRACE(pb, id, data)  do { } while (0)
 103 #endif
 104
 105 #ifdef PAGEBUF_LOCK_TRACKING
 106 # define PB_SET_OWNER(pb)       ((pb)->pb_last_holder = current->pid)
 107 # define PB_CLEAR_OWNER(pb)     ((pb)->pb_last_holder = -1)
 108 # define PB_GET_OWNER(pb)       ((pb)->pb_last_holder)
 109 #else
 110 # define PB_SET_OWNER(pb)       do { } while (0)
 111 # define PB_CLEAR_OWNER(pb)     do { } while (0)
 112 # define PB_GET_OWNER(pb)       do { } while (0)
 113 #endif
 114
 115 /*
 116  * Pagebuf allocation / freeing.
 117  */
 118
 119 #define pb_to_gfp(flags) \
 120         (((flags) & PBF_READ_AHEAD) ? GFP_READAHEAD : \
 121          ((flags) & PBF_DONT_BLOCK) ? GFP_NOFS : GFP_KERNEL)
 122
 123 #define pb_to_km(flags) \
 124          (((flags) & PBF_DONT_BLOCK) ? KM_NOFS : KM_SLEEP)
 125
 126
 127 #define pagebuf_allocate(flags) \
 128         kmem_zone_alloc(pagebuf_cache, pb_to_km(flags))
 129 #define pagebuf_deallocate(pb) \
 130         kmem_zone_free(pagebuf_cache, (pb));
 131
 132 /*
 133  * Pagebuf hashing
 134  */
 135
 136 #define NBITS   8
 137 #define NHASH   (1<<NBITS)
 138
 139 typedef struct {
 140         struct list_head        pb_hash;
 141         spinlock_t              pb_hash_lock;
 142 } pb_hash_t;
 143
 144 STATIC pb_hash_t        pbhash[NHASH];
 145 #define pb_hash(pb)     &pbhash[pb->pb_hash_index]
 146
 147 STATIC int
 148 _bhash(
 149         struct block_device *bdev,
 150         loff_t          base)
 151 {
 152         int             bit, hval;
 153
 154         base >>= 9;
 155         base ^= (unsigned long)bdev / L1_CACHE_BYTES;
 156         for (bit = hval = 0; base && bit < sizeof(base) * 8; bit += NBITS) {
 157                 hval ^= (int)base & (NHASH-1);
 158                 base >>= NBITS;
 159         }
 160         return hval;
 161 }
 162
 163 /*
 164  * Mapping of multi-page buffers into contiguous virtual space
 165  */
 166
 167 typedef struct a_list {
 168         void            *vm_addr;
 169         struct a_list   *next;
 170 } a_list_t;
 171
 172 STATIC a_list_t         *as_free_head;
 173 STATIC int              as_list_len;
 174 STATIC spinlock_t       as_lock = SPIN_LOCK_UNLOCKED;
 175
 176 /*
 177  * Try to batch vunmaps because they are costly.
 178  */
 179 STATIC void
 180 free_address(
 181         void            *addr)
 182 {
 183         a_list_t        *aentry;
 184
 185         aentry = kmalloc(sizeof(a_list_t), GFP_ATOMIC);
 186         if (aentry) {
 187                 spin_lock(&as_lock);
 188                 aentry->next = as_free_head;
 189                 aentry->vm_addr = addr;
 190                 as_free_head = aentry;
 191                 as_list_len++;
 192                 spin_unlock(&as_lock);
 193         } else {
 194                 vunmap(addr);
 195         }
 196 }
 197
 198 STATIC void
 199 purge_addresses(void)
 200 {
 201         a_list_t        *aentry, *old;
 202
 203         if (as_free_head == NULL)
 204                 return;
 205
 206         spin_lock(&as_lock);
 207         aentry = as_free_head;
 208         as_free_head = NULL;
 209         as_list_len = 0;
 210         spin_unlock(&as_lock);
 211
 212         while ((old = aentry) != NULL) {
 213                 vunmap(aentry->vm_addr);
 214                 aentry = aentry->next;
 215                 kfree(old);
 216         }
 217 }
 218
 219 /*
 220  *      Internal pagebuf object manipulation
 221  */
 222
 223 STATIC void
 224 _pagebuf_initialize(
 225         xfs_buf_t               *pb,
 226         xfs_buftarg_t           *target,
 227         loff_t                  range_base,
 228         size_t                  range_length,
 229         page_buf_flags_t        flags)
 230 {
 231         /*
 232          * We don't want certain flags to appear in pb->pb_flags.
 233          */
 234         flags &= ~(PBF_LOCK|PBF_MAPPED|PBF_DONT_BLOCK|PBF_READ_AHEAD);
 235
 236         memset(pb, 0, sizeof(xfs_buf_t));
 237         atomic_set(&pb->pb_hold, 1);
 238         init_MUTEX_LOCKED(&pb->pb_iodonesema);
 239         INIT_LIST_HEAD(&pb->pb_list);
 240         INIT_LIST_HEAD(&pb->pb_hash_list);
 241         init_MUTEX_LOCKED(&pb->pb_sema); /* held, no waiters */
 242         PB_SET_OWNER(pb);
 243         pb->pb_target = target;
 244         pb->pb_file_offset = range_base;
 245         /*
 246          * Set buffer_length and count_desired to the same value initially.
 247          * I/O routines should use count_desired, which will be the same in
 248          * most cases but may be reset (e.g. XFS recovery).
 249          */
 250         pb->pb_buffer_length = pb->pb_count_desired = range_length;
 251         pb->pb_flags = flags | PBF_NONE;
 252         pb->pb_bn = XFS_BUF_DADDR_NULL;
 253         atomic_set(&pb->pb_pin_count, 0);
 254         init_waitqueue_head(&pb->pb_waiters);
 255
 256         XFS_STATS_INC(pb_create);
 257         PB_TRACE(pb, "initialize", target);
 258 }
 259
 260 /*
 261  * Allocate a page array capable of holding a specified number
 262  * of pages, and point the page buf at it.
 263  */
 264 STATIC int
 265 _pagebuf_get_pages(
 266         xfs_buf_t               *pb,
 267         int                     page_count,
 268         page_buf_flags_t        flags)
 269 {
 270         /* Make sure that we have a page list */
 271         if (pb->pb_pages == NULL) {
 272                 pb->pb_offset = page_buf_poff(pb->pb_file_offset);
 273                 pb->pb_page_count = page_count;
 274                 if (page_count <= PB_PAGES) {
 275                         pb->pb_pages = pb->pb_page_array;
 276                 } else {
 277                         pb->pb_pages = kmem_alloc(sizeof(struct page *) *
 278                                         page_count, pb_to_km(flags));
 279                         if (pb->pb_pages == NULL)
 280                                 return -ENOMEM;
 281                 }
 282                 memset(pb->pb_pages, 0, sizeof(struct page *) * page_count);
 283         }
 284         return 0;
 285 }
 286
 287 /*
 288  *      Frees pb_pages if it was malloced.
 289  */
 290 STATIC void
 291 _pagebuf_free_pages(
 292         xfs_buf_t       *bp)
 293 {
 294         if (bp->pb_pages != bp->pb_page_array) {
 295                 kmem_free(bp->pb_pages,
 296                           bp->pb_page_count * sizeof(struct page *));
 297         }
 298 }
 299
 300 /*
 301  *      Releases the specified buffer.
 302  *
 303  *      The modification state of any associated pages is left unchanged.
 304  *      The buffer most not be on any hash - use pagebuf_rele instead for
 305  *      hashed and refcounted buffers
 306  */
 307 void
 308 pagebuf_free(
 309         xfs_buf_t               *bp)
 310 {
 311         PB_TRACE(bp, "free", 0);
 312
 313         ASSERT(list_empty(&bp->pb_hash_list));
 314
 315         if (bp->pb_flags & _PBF_PAGE_CACHE) {
 316                 uint            i;
 317
 318                 if ((bp->pb_flags & PBF_MAPPED) && (bp->pb_page_count > 1))
 319                         free_address(bp->pb_addr - bp->pb_offset);
 320
 321                 for (i = 0; i < bp->pb_page_count; i++)
 322                         page_cache_release(bp->pb_pages[i]);
 323                 _pagebuf_free_pages(bp);
 324         } else if (bp->pb_flags & _PBF_KMEM_ALLOC) {
 325                  /*
 326                   * XXX(hch): bp->pb_count_desired might be incorrect (see
 327                   * pagebuf_associate_memory for details), but fortunately
 328                   * the Linux version of kmem_free ignores the len argument..
 329                   */
 330                 kmem_free(bp->pb_addr, bp->pb_count_desired);
 331                 _pagebuf_free_pages(bp);
 332         }
 333
 334         pagebuf_deallocate(bp);
 335 }
 336
 337 /*
 338  *      Finds all pages for buffer in question and builds it's page list.
 339  */
 340 STATIC int
 341 _pagebuf_lookup_pages(
 342         xfs_buf_t               *bp,
 343         uint                    flags)
 344 {
 345         struct address_space    *mapping = bp->pb_target->pbr_mapping;
 346         unsigned int            sectorshift = bp->pb_target->pbr_sshift;
 347         size_t                  blocksize = bp->pb_target->pbr_bsize;
 348         size_t                  size = bp->pb_count_desired;
 349         size_t                  nbytes, offset;
 350         int                     gfp_mask = pb_to_gfp(flags);
 351         unsigned short          page_count, i;
 352         pgoff_t                 first;
 353         loff_t                  end;
 354         int                     error;
 355
 356         end = bp->pb_file_offset + bp->pb_buffer_length;
 357         page_count = page_buf_btoc(end) - page_buf_btoct(bp->pb_file_offset);
 358
 359         error = _pagebuf_get_pages(bp, page_count, flags);
 360         if (unlikely(error))
 361                 return error;
 362         bp->pb_flags |= _PBF_PAGE_CACHE;
 363
 364         offset = bp->pb_offset;
 365         first = bp->pb_file_offset >> PAGE_CACHE_SHIFT;
 366
 367         for (i = 0; i < bp->pb_page_count; i++) {
 368                 struct page     *page;
 369                 uint            retries = 0;
 370
 371               retry:
 372                 page = find_or_create_page(mapping, first + i, gfp_mask);
 373                 if (unlikely(page == NULL)) {
 374                         if (flags & PBF_READ_AHEAD) {
 375                                 bp->pb_page_count = i;
 376                                 for (i = 0; i < bp->pb_page_count; i++)
 377                                         unlock_page(bp->pb_pages[i]);
 378                                 return -ENOMEM;
 379                         }
 380
 381                         /*
 382                          * This could deadlock.
 383                          *
 384                          * But until all the XFS lowlevel code is revamped to
 385                          * handle buffer allocation failures we can't do much.
 386                          */
 387                         if (!(++retries % 100)) {
 388                                 printk(KERN_ERR "possibly deadlocking in %s\n",
 389                                                 __FUNCTION__);
 390                         }
 391
 392                         XFS_STATS_INC(pb_page_retries);
 393                         pagebuf_daemon_wakeup();
 394                         set_current_state(TASK_UNINTERRUPTIBLE);
 395                         schedule_timeout(10);
 396                         goto retry;
 397                 }
 398
 399                 XFS_STATS_INC(pb_page_found);
 400
 401                 nbytes = min_t(size_t, size, PAGE_CACHE_SIZE - offset);
 402                 size -= nbytes;
 403
 404                 if (!PageUptodate(page)) {
 405                         page_count--;
 406                         if (blocksize == PAGE_CACHE_SIZE) {
 407                                 if (flags & PBF_READ)
 408                                         bp->pb_locked = 1;
 409                         } else if (!PagePrivate(page)) {
 410                                 unsigned long   j, range;
 411
 412                                 /*
 413                                  * In this case page->private holds a bitmap
 414                                  * of uptodate sectors within the page
 415                                  */
 416                                 ASSERT(blocksize < PAGE_CACHE_SIZE);
 417                                 range = (offset + nbytes) >> sectorshift;
 418                                 for (j = offset >> sectorshift; j < range; j++)
 419                                         if (!test_bit(j, &page->private))
 420                                                 break;
 421                                 if (j == range)
 422                                         page_count++;
 423                         }
 424                 }
 425
 426                 bp->pb_pages[i] = page;
 427                 offset = 0;
 428         }
 429
 430         if (!bp->pb_locked) {
 431                 for (i = 0; i < bp->pb_page_count; i++)
 432                         unlock_page(bp->pb_pages[i]);
 433         }
 434
 435         if (page_count) {
 436                 /* if we have any uptodate pages, mark that in the buffer */
 437                 bp->pb_flags &= ~PBF_NONE;
 438
 439                 /* if some pages aren't uptodate, mark that in the buffer */
 440                 if (page_count != bp->pb_page_count)
 441                         bp->pb_flags |= PBF_PARTIAL;
 442         }
 443
 444         PB_TRACE(bp, "lookup_pages", (long)page_count);
 445         return error;
 446 }
 447
 448 /*
 449  *      Map buffer into kernel address-space if nessecary.
 450  */
 451 STATIC int
 452 _pagebuf_map_pages(
 453         xfs_buf_t               *bp,
 454         uint                    flags)
 455 {
 456         /* A single page buffer is always mappable */
 457         if (bp->pb_page_count == 1) {
 458                 bp->pb_addr = page_address(bp->pb_pages[0]) + bp->pb_offset;
 459                 bp->pb_flags |= PBF_MAPPED;
 460         } else if (flags & PBF_MAPPED) {
 461                 if (as_list_len > 64)
 462                         purge_addresses();
 463                 bp->pb_addr = vmap(bp->pb_pages, bp->pb_page_count,
 464                                 VM_MAP, PAGE_KERNEL);
 465                 if (unlikely(bp->pb_addr == NULL))
 466                         return -ENOMEM;
 467                 bp->pb_addr += bp->pb_offset;
 468                 bp->pb_flags |= PBF_MAPPED;
 469         }
 470
 471         return 0;
 472 }
 473
 474 /*
 475  *      Finding and Reading Buffers
 476  */
 477
 478 /*
 479  *      _pagebuf_find
 480  *
 481  *      Looks up, and creates if absent, a lockable buffer for
 482  *      a given range of an inode.  The buffer is returned
 483  *      locked.  If other overlapping buffers exist, they are
 484  *      released before the new buffer is created and locked,
 485  *      which may imply that this call will block until those buffers
 486  *      are unlocked.  No I/O is implied by this call.
 487  */
 488 STATIC xfs_buf_t *
 489 _pagebuf_find(                          /* find buffer for block        */
 490         xfs_buftarg_t           *target,/* target for block             */
 491         loff_t                  ioff,   /* starting offset of range     */
 492         size_t                  isize,  /* length of range              */
 493         page_buf_flags_t        flags,  /* PBF_TRYLOCK                  */
 494         xfs_buf_t               *new_pb)/* newly allocated buffer       */
 495 {
 496         loff_t                  range_base;
 497         size_t                  range_length;
 498         int                     hval;
 499         pb_hash_t               *h;
 500         xfs_buf_t               *pb, *n;
 501         int                     not_locked;
 502
 503         range_base = (ioff << BBSHIFT);
 504         range_length = (isize << BBSHIFT);
 505
 506         /* Ensure we never do IOs smaller than the sector size */
 507         BUG_ON(range_length < (1 << target->pbr_sshift));
 508
 509         /* Ensure we never do IOs that are not sector aligned */
 510         BUG_ON(range_base & (loff_t)target->pbr_smask);
 511
 512         hval = _bhash(target->pbr_bdev, range_base);
 513         h = &pbhash[hval];
 514
 515         spin_lock(&h->pb_hash_lock);
 516         list_for_each_entry_safe(pb, n, &h->pb_hash, pb_hash_list) {
 517                 if (pb->pb_target == target &&
 518                     pb->pb_file_offset == range_base &&
 519                     pb->pb_buffer_length == range_length) {
 520                         /* If we look at something bring it to the
 521                          * front of the list for next time
 522                          */
 523                         atomic_inc(&pb->pb_hold);
 524                         list_move(&pb->pb_hash_list, &h->pb_hash);
 525                         goto found;
 526                 }
 527         }
 528
 529         /* No match found */
 530         if (new_pb) {
 531                 _pagebuf_initialize(new_pb, target, range_base,
 532                                 range_length, flags);
 533                 new_pb->pb_hash_index = hval;
 534                 list_add(&new_pb->pb_hash_list, &h->pb_hash);
 535         } else {
 536                 XFS_STATS_INC(pb_miss_locked);
 537         }
 538
 539         spin_unlock(&h->pb_hash_lock);
 540         return (new_pb);
 541
 542 found:
 543         spin_unlock(&h->pb_hash_lock);
 544
 545         /* Attempt to get the semaphore without sleeping,
 546          * if this does not work then we need to drop the
 547          * spinlock and do a hard attempt on the semaphore.
 548          */
 549         not_locked = down_trylock(&pb->pb_sema);
 550         if (not_locked) {
 551                 if (!(flags & PBF_TRYLOCK)) {
 552                         /* wait for buffer ownership */
 553                         PB_TRACE(pb, "get_lock", 0);
 554                         pagebuf_lock(pb);
 555                         XFS_STATS_INC(pb_get_locked_waited);
 556                 } else {
 557                         /* We asked for a trylock and failed, no need
 558                          * to look at file offset and length here, we
 559                          * know that this pagebuf at least overlaps our
 560                          * pagebuf and is locked, therefore our buffer
 561                          * either does not exist, or is this buffer
 562                          */
 563
 564                         pagebuf_rele(pb);
 565                         XFS_STATS_INC(pb_busy_locked);
 566                         return (NULL);
 567                 }
 568         } else {
 569                 /* trylock worked */
 570                 PB_SET_OWNER(pb);
 571         }
 572
 573         if (pb->pb_flags & PBF_STALE)
 574                 pb->pb_flags &= PBF_MAPPED;
 575         PB_TRACE(pb, "got_lock", 0);
 576         XFS_STATS_INC(pb_get_locked);
 577         return (pb);
 578 }
 579
 580
 581 /*
 582  *      pagebuf_find
 583  *
 584  *      pagebuf_find returns a buffer matching the specified range of
 585  *      data for the specified target, if any of the relevant blocks
 586  *      are in memory.  The buffer may have unallocated holes, if
 587  *      some, but not all, of the blocks are in memory.  Even where
 588  *      pages are present in the buffer, not all of every page may be
 589  *      valid.
 590  */
 591 xfs_buf_t *
 592 pagebuf_find(                           /* find buffer for block        */
 593                                         /* if the block is in memory    */
 594         xfs_buftarg_t           *target,/* target for block             */
 595         loff_t                  ioff,   /* starting offset of range     */
 596         size_t                  isize,  /* length of range              */
 597         page_buf_flags_t        flags)  /* PBF_TRYLOCK                  */
 598 {
 599         return _pagebuf_find(target, ioff, isize, flags, NULL);
 600 }
 601
 602 /*
 603  *      pagebuf_get
 604  *
 605  *      pagebuf_get assembles a buffer covering the specified range.
 606  *      Some or all of the blocks in the range may be valid.  Storage
 607  *      in memory for all portions of the buffer will be allocated,
 608  *      although backing storage may not be.  If PBF_READ is set in
 609  *      flags, pagebuf_iostart is called also.
 610  */
 611 xfs_buf_t *
 612 pagebuf_get(                            /* allocate a buffer            */
 613         xfs_buftarg_t           *target,/* target for buffer            */
 614         loff_t                  ioff,   /* starting offset of range     */
 615         size_t                  isize,  /* length of range              */
 616         page_buf_flags_t        flags)  /* PBF_TRYLOCK                  */
 617 {
 618         xfs_buf_t               *pb, *new_pb;
 619         int                     error = 0, i;
 620
 621         new_pb = pagebuf_allocate(flags);
 622         if (unlikely(!new_pb))
 623                 return NULL;
 624
 625         pb = _pagebuf_find(target, ioff, isize, flags, new_pb);
 626         if (pb == new_pb) {
 627                 error = _pagebuf_lookup_pages(pb, flags);
 628                 if (unlikely(error)) {
 629                         printk(KERN_WARNING
 630                                "pagebuf_get: failed to lookup pages\n");
 631                         goto no_buffer;
 632                 }
 633         } else {
 634                 pagebuf_deallocate(new_pb);
 635                 if (unlikely(pb == NULL))
 636                         return NULL;
 637         }
 638
 639         for (i = 0; i < pb->pb_page_count; i++)
 640                 mark_page_accessed(pb->pb_pages[i]);
 641
 642         if (!(pb->pb_flags & PBF_MAPPED)) {
 643                 error = _pagebuf_map_pages(pb, flags);
 644                 if (unlikely(error)) {
 645                         printk(KERN_WARNING
 646                                "pagebuf_get: failed to map pages\n");
 647                         goto no_buffer;
 648                 }
 649         }
 650
 651         XFS_STATS_INC(pb_get);
 652
 653         /*
 654          * Always fill in the block number now, the mapped cases can do
 655          * their own overlay of this later.
 656          */
 657         pb->pb_bn = ioff;
 658         pb->pb_count_desired = pb->pb_buffer_length;
 659
 660         if (flags & PBF_READ) {
 661                 if (PBF_NOT_DONE(pb)) {
 662                         PB_TRACE(pb, "get_read", (unsigned long)flags);
 663                         XFS_STATS_INC(pb_get_read);
 664                         pagebuf_iostart(pb, flags);
 665                 } else if (flags & PBF_ASYNC) {
 666                         PB_TRACE(pb, "get_read_async", (unsigned long)flags);
 667                         /*
 668                          * Read ahead call which is already satisfied,
 669                          * drop the buffer
 670                          */
 671                         goto no_buffer;
 672                 } else {
 673                         PB_TRACE(pb, "get_read_done", (unsigned long)flags);
 674                         /* We do not want read in the flags */
 675                         pb->pb_flags &= ~PBF_READ;
 676                 }
 677         } else {
 678                 PB_TRACE(pb, "get_write", (unsigned long)flags);
 679         }
 680
 681         return pb;
 682
 683 no_buffer:
 684         if (flags & (PBF_LOCK | PBF_TRYLOCK))
 685                 pagebuf_unlock(pb);
 686         pagebuf_rele(pb);
 687         return NULL;
 688 }
 689
 690 /*
 691  * Create a skeletal pagebuf (no pages associated with it).
 692  */
 693 xfs_buf_t *
 694 pagebuf_lookup(
 695         xfs_buftarg_t           *target,
 696         loff_t                  ioff,
 697         size_t                  isize,
 698         page_buf_flags_t        flags)
 699 {
 700         xfs_buf_t               *pb;
 701
 702         pb = pagebuf_allocate(flags);
 703         if (pb) {
 704                 _pagebuf_initialize(pb, target, ioff, isize, flags);
 705         }
 706         return pb;
 707 }
 708
 709 /*
 710  * If we are not low on memory then do the readahead in a deadlock
 711  * safe manner.
 712  */
 713 void
 714 pagebuf_readahead(
 715         xfs_buftarg_t           *target,
 716         loff_t                  ioff,
 717         size_t                  isize,
 718         page_buf_flags_t        flags)
 719 {
 720         struct backing_dev_info *bdi;
 721
 722         bdi = target->pbr_mapping->backing_dev_info;
 723         if (bdi_read_congested(bdi))
 724                 return;
 725         if (bdi_write_congested(bdi))
 726                 return;
 727
 728         flags |= (PBF_TRYLOCK|PBF_READ|PBF_ASYNC|PBF_READ_AHEAD);
 729         pagebuf_get(target, ioff, isize, flags);
 730 }
 731
 732 xfs_buf_t *
 733 pagebuf_get_empty(
 734         size_t                  len,
 735         xfs_buftarg_t           *target)
 736 {
 737         xfs_buf_t               *pb;
 738
 739         pb = pagebuf_allocate(0);
 740         if (pb)
 741                 _pagebuf_initialize(pb, target, 0, len, 0);
 742         return pb;
 743 }
 744
 745 static inline struct page *
 746 mem_to_page(
 747         void                    *addr)
 748 {
 749         if (((unsigned long)addr < VMALLOC_START) ||
 750             ((unsigned long)addr >= VMALLOC_END)) {
 751                 return virt_to_page(addr);
 752         } else {
 753                 return vmalloc_to_page(addr);
 754         }
 755 }
 756
 757 int
 758 pagebuf_associate_memory(
 759         xfs_buf_t               *pb,
 760         void                    *mem,
 761         size_t                  len)
 762 {
 763         int                     rval;
 764         int                     i = 0;
 765         size_t                  ptr;
 766         size_t                  end, end_cur;
 767         off_t                   offset;
 768         int                     page_count;
 769
 770         page_count = PAGE_CACHE_ALIGN(len) >> PAGE_CACHE_SHIFT;
 771         offset = (off_t) mem - ((off_t)mem & PAGE_CACHE_MASK);
 772         if (offset && (len > PAGE_CACHE_SIZE))
 773                 page_count++;
 774
 775         /* Free any previous set of page pointers */
 776         if (pb->pb_pages)
 777                 _pagebuf_free_pages(pb);
 778
 779         pb->pb_pages = NULL;
 780         pb->pb_addr = mem;
 781
 782         rval = _pagebuf_get_pages(pb, page_count, 0);
 783         if (rval)
 784                 return rval;
 785
 786         pb->pb_offset = offset;
 787         ptr = (size_t) mem & PAGE_CACHE_MASK;
 788         end = PAGE_CACHE_ALIGN((size_t) mem + len);
 789         end_cur = end;
 790         /* set up first page */
 791         pb->pb_pages[0] = mem_to_page(mem);
 792
 793         ptr += PAGE_CACHE_SIZE;
 794         pb->pb_page_count = ++i;
 795         while (ptr < end) {
 796                 pb->pb_pages[i] = mem_to_page((void *)ptr);
 797                 pb->pb_page_count = ++i;
 798                 ptr += PAGE_CACHE_SIZE;
 799         }
 800         pb->pb_locked = 0;
 801
 802         pb->pb_count_desired = pb->pb_buffer_length = len;
 803         pb->pb_flags |= PBF_MAPPED;
 804
 805         return 0;
 806 }
 807
 808 xfs_buf_t *
 809 pagebuf_get_no_daddr(
 810         size_t                  len,
 811         xfs_buftarg_t           *target)
 812 {
 813         size_t                  malloc_len = len;
 814         xfs_buf_t               *bp;
 815         void                    *data;
 816         int                     error;
 817
 818         bp = pagebuf_allocate(0);
 819         if (unlikely(bp == NULL))
 820                 goto fail;
 821         _pagebuf_initialize(bp, target, 0, len, PBF_FORCEIO);
 822
 823  try_again:
 824         data = kmem_alloc(malloc_len, KM_SLEEP | KM_MAYFAIL);
 825         if (unlikely(data == NULL))
 826                 goto fail_free_buf;
 827
 828         /* check whether alignment matches.. */
 829         if ((__psunsigned_t)data !=
 830             ((__psunsigned_t)data & ~target->pbr_smask)) {
 831                 /* .. else double the size and try again */
 832                 kmem_free(data, malloc_len);
 833                 malloc_len <<= 1;
 834                 goto try_again;
 835         }
 836
 837         error = pagebuf_associate_memory(bp, data, len);
 838         if (error)
 839                 goto fail_free_mem;
 840         bp->pb_flags |= _PBF_KMEM_ALLOC;
 841
 842         pagebuf_unlock(bp);
 843
 844         PB_TRACE(bp, "no_daddr", data);
 845         return bp;
 846  fail_free_mem:
 847         kmem_free(data, malloc_len);
 848  fail_free_buf:
 849         pagebuf_free(bp);
 850  fail:
 851         return NULL;
 852 }
 853
 854 /*
 855  *      pagebuf_hold
 856  *
 857  *      Increment reference count on buffer, to hold the buffer concurrently
 858  *      with another thread which may release (free) the buffer asynchronously.
 859  *
 860  *      Must hold the buffer already to call this function.
 861  */
 862 void
 863 pagebuf_hold(
 864         xfs_buf_t               *pb)
 865 {
 866         atomic_inc(&pb->pb_hold);
 867         PB_TRACE(pb, "hold", 0);
 868 }
 869
 870 /*
 871  *      pagebuf_rele
 872  *
 873  *      pagebuf_rele releases a hold on the specified buffer.  If the
 874  *      the hold count is 1, pagebuf_rele calls pagebuf_free.
 875  */
 876 void
 877 pagebuf_rele(
 878         xfs_buf_t               *pb)
 879 {
 880         pb_hash_t               *hash = pb_hash(pb);
 881
 882         PB_TRACE(pb, "rele", pb->pb_relse);
 883
 884         if (atomic_dec_and_lock(&pb->pb_hold, &hash->pb_hash_lock)) {
 885                 int             do_free = 1;
 886
 887                 if (pb->pb_relse) {
 888                         atomic_inc(&pb->pb_hold);
 889                         spin_unlock(&hash->pb_hash_lock);
 890                         (*(pb->pb_relse)) (pb);
 891                         spin_lock(&hash->pb_hash_lock);
 892                         do_free = 0;
 893                 }
 894
 895                 if (pb->pb_flags & PBF_DELWRI) {
 896                         pb->pb_flags |= PBF_ASYNC;
 897                         atomic_inc(&pb->pb_hold);
 898                         pagebuf_delwri_queue(pb, 0);
 899                         do_free = 0;
 900                 } else if (pb->pb_flags & PBF_FS_MANAGED) {
 901                         do_free = 0;
 902                 }
 903
 904                 if (do_free) {
 905                         list_del_init(&pb->pb_hash_list);
 906                         spin_unlock(&hash->pb_hash_lock);
 907                         pagebuf_free(pb);
 908                 } else {
 909                         spin_unlock(&hash->pb_hash_lock);
 910                 }
 911         }
 912 }
 913
 914
 915 /*
 916  *      Mutual exclusion on buffers.  Locking model:
 917  *
 918  *      Buffers associated with inodes for which buffer locking
 919  *      is not enabled are not protected by semaphores, and are
 920  *      assumed to be exclusively owned by the caller.  There is a
 921  *      spinlock in the buffer, used by the caller when concurrent
 922  *      access is possible.
 923  */
 924
 925 /*
 926  *      pagebuf_cond_lock
 927  *
 928  *      pagebuf_cond_lock locks a buffer object, if it is not already locked.
 929  *      Note that this in no way
 930  *      locks the underlying pages, so it is only useful for synchronizing
 931  *      concurrent use of page buffer objects, not for synchronizing independent
 932  *      access to the underlying pages.
 933  */
 934 int
 935 pagebuf_cond_lock(                      /* lock buffer, if not locked   */
 936                                         /* returns -EBUSY if locked)    */
 937         xfs_buf_t               *pb)
 938 {
 939         int                     locked;
 940
 941         locked = down_trylock(&pb->pb_sema) == 0;
 942         if (locked) {
 943                 PB_SET_OWNER(pb);
 944         }
 945         PB_TRACE(pb, "cond_lock", (long)locked);
 946         return(locked ? 0 : -EBUSY);
 947 }
 948
 949 /*
 950  *      pagebuf_lock_value
 951  *
 952  *      Return lock value for a pagebuf
 953  */
 954 int
 955 pagebuf_lock_value(
 956         xfs_buf_t               *pb)
 957 {
 958         return(atomic_read(&pb->pb_sema.count));
 959 }
 960
 961 /*
 962  *      pagebuf_lock
 963  *
 964  *      pagebuf_lock locks a buffer object.  Note that this in no way
 965  *      locks the underlying pages, so it is only useful for synchronizing
 966  *      concurrent use of page buffer objects, not for synchronizing independent
 967  *      access to the underlying pages.
 968  */
 969 int
 970 pagebuf_lock(
 971         xfs_buf_t               *pb)
 972 {
 973         PB_TRACE(pb, "lock", 0);
 974         if (atomic_read(&pb->pb_io_remaining))
 975                 blk_run_address_space(pb->pb_target->pbr_mapping);
 976         down(&pb->pb_sema);
 977         PB_SET_OWNER(pb);
 978         PB_TRACE(pb, "locked", 0);
 979         return 0;
 980 }
 981
 982 /*
 983  *      pagebuf_unlock
 984  *
 985  *      pagebuf_unlock releases the lock on the buffer object created by
 986  *      pagebuf_lock or pagebuf_cond_lock (not any
 987  *      pinning of underlying pages created by pagebuf_pin).
 988  */
 989 void
 990 pagebuf_unlock(                         /* unlock buffer                */
 991         xfs_buf_t               *pb)    /* buffer to unlock             */
 992 {
 993         PB_CLEAR_OWNER(pb);
 994         up(&pb->pb_sema);
 995         PB_TRACE(pb, "unlock", 0);
 996 }
 997
 998
 999 /*
1000  *      Pinning Buffer Storage in Memory
1001  */
1002
1003 /*
1004  *      pagebuf_pin
1005  *
1006  *      pagebuf_pin locks all of the memory represented by a buffer in
1007  *      memory.  Multiple calls to pagebuf_pin and pagebuf_unpin, for
1008  *      the same or different buffers affecting a given page, will
1009  *      properly count the number of outstanding "pin" requests.  The
1010  *      buffer may be released after the pagebuf_pin and a different
1011  *      buffer used when calling pagebuf_unpin, if desired.
1012  *      pagebuf_pin should be used by the file system when it wants be
1013  *      assured that no attempt will be made to force the affected
1014  *      memory to disk.  It does not assure that a given logical page
1015  *      will not be moved to a different physical page.
1016  */
1017 void
1018 pagebuf_pin(
1019         xfs_buf_t               *pb)
1020 {
1021         atomic_inc(&pb->pb_pin_count);
1022         PB_TRACE(pb, "pin", (long)pb->pb_pin_count.counter);
1023 }
1024
1025 /*
1026  *      pagebuf_unpin
1027  *
1028  *      pagebuf_unpin reverses the locking of memory performed by
1029  *      pagebuf_pin.  Note that both functions affected the logical
1030  *      pages associated with the buffer, not the buffer itself.
1031  */
1032 void
1033 pagebuf_unpin(
1034         xfs_buf_t               *pb)
1035 {
1036         if (atomic_dec_and_test(&pb->pb_pin_count)) {
1037                 wake_up_all(&pb->pb_waiters);
1038         }
1039         PB_TRACE(pb, "unpin", (long)pb->pb_pin_count.counter);
1040 }
1041
1042 int
1043 pagebuf_ispin(
1044         xfs_buf_t               *pb)
1045 {
1046         return atomic_read(&pb->pb_pin_count);
1047 }
1048
1049 /*
1050  *      pagebuf_wait_unpin
1051  *
1052  *      pagebuf_wait_unpin waits until all of the memory associated
1053  *      with the buffer is not longer locked in memory.  It returns
1054  *      immediately if none of the affected pages are locked.
1055  */
1056 static inline void
1057 _pagebuf_wait_unpin(
1058         xfs_buf_t               *pb)
1059 {
1060         DECLARE_WAITQUEUE       (wait, current);
1061
1062         if (atomic_read(&pb->pb_pin_count) == 0)
1063                 return;
1064
1065         add_wait_queue(&pb->pb_waiters, &wait);
1066         for (;;) {
1067                 set_current_state(TASK_UNINTERRUPTIBLE);
1068                 if (atomic_read(&pb->pb_pin_count) == 0)
1069                         break;
1070                 if (atomic_read(&pb->pb_io_remaining))
1071                         blk_run_address_space(pb->pb_target->pbr_mapping);
1072                 schedule();
1073         }
1074         remove_wait_queue(&pb->pb_waiters, &wait);
1075         set_current_state(TASK_RUNNING);
1076 }
1077
1078 /*
1079  *      Buffer Utility Routines
1080  */
1081
1082 /*
1083  *      pagebuf_iodone
1084  *
1085  *      pagebuf_iodone marks a buffer for which I/O is in progress
1086  *      done with respect to that I/O.  The pb_iodone routine, if
1087  *      present, will be called as a side-effect.
1088  */
1089 void
1090 pagebuf_iodone_work(
1091         void                    *v)
1092 {
1093         xfs_buf_t               *bp = (xfs_buf_t *)v;
1094
1095         if (bp->pb_iodone)
1096                 (*(bp->pb_iodone))(bp);
1097         else if (bp->pb_flags & PBF_ASYNC)
1098                 xfs_buf_relse(bp);
1099 }
1100
1101 void
1102 pagebuf_iodone(
1103         xfs_buf_t               *pb,
1104         int                     dataio,
1105         int                     schedule)
1106 {
1107         pb->pb_flags &= ~(PBF_READ | PBF_WRITE);
1108         if (pb->pb_error == 0) {
1109                 pb->pb_flags &= ~(PBF_PARTIAL | PBF_NONE);
1110         }
1111
1112         PB_TRACE(pb, "iodone", pb->pb_iodone);
1113
1114         if ((pb->pb_iodone) || (pb->pb_flags & PBF_ASYNC)) {
1115                 if (schedule) {
1116                         INIT_WORK(&pb->pb_iodone_work, pagebuf_iodone_work, pb);
1117                         queue_work(dataio ? pagebuf_dataio_workqueue :
1118                                 pagebuf_logio_workqueue, &pb->pb_iodone_work);
1119                 } else {
1120                         pagebuf_iodone_work(pb);
1121                 }
1122         } else {
1123                 up(&pb->pb_iodonesema);
1124         }
1125 }
1126
1127 /*
1128  *      pagebuf_ioerror
1129  *
1130  *      pagebuf_ioerror sets the error code for a buffer.
1131  */
1132 void
1133 pagebuf_ioerror(                        /* mark/clear buffer error flag */
1134         xfs_buf_t               *pb,    /* buffer to mark               */
1135         int                     error)  /* error to store (0 if none)   */
1136 {
1137         ASSERT(error >= 0 && error <= 0xffff);
1138         pb->pb_error = (unsigned short)error;
1139         PB_TRACE(pb, "ioerror", (unsigned long)error);
1140 }
1141
1142 /*
1143  *      pagebuf_iostart
1144  *
1145  *      pagebuf_iostart initiates I/O on a buffer, based on the flags supplied.
1146  *      If necessary, it will arrange for any disk space allocation required,
1147  *      and it will break up the request if the block mappings require it.
1148  *      The pb_iodone routine in the buffer supplied will only be called
1149  *      when all of the subsidiary I/O requests, if any, have been completed.
1150  *      pagebuf_iostart calls the pagebuf_ioinitiate routine or
1151  *      pagebuf_iorequest, if the former routine is not defined, to start
1152  *      the I/O on a given low-level request.
1153  */
1154 int
1155 pagebuf_iostart(                        /* start I/O on a buffer          */
1156         xfs_buf_t               *pb,    /* buffer to start                */
1157         page_buf_flags_t        flags)  /* PBF_LOCK, PBF_ASYNC, PBF_READ, */
1158                                         /* PBF_WRITE, PBF_DELWRI,         */
1159                                         /* PBF_DONT_BLOCK                 */
1160 {
1161         int                     status = 0;
1162
1163         PB_TRACE(pb, "iostart", (unsigned long)flags);
1164
1165         if (flags & PBF_DELWRI) {
1166                 pb->pb_flags &= ~(PBF_READ | PBF_WRITE | PBF_ASYNC);
1167                 pb->pb_flags |= flags & (PBF_DELWRI | PBF_ASYNC);
1168                 pagebuf_delwri_queue(pb, 1);
1169                 return status;
1170         }
1171
1172         pb->pb_flags &= ~(PBF_READ | PBF_WRITE | PBF_ASYNC | PBF_DELWRI | \
1173                         PBF_READ_AHEAD | _PBF_RUN_QUEUES);
1174         pb->pb_flags |= flags & (PBF_READ | PBF_WRITE | PBF_ASYNC | \
1175                         PBF_READ_AHEAD | _PBF_RUN_QUEUES);
1176
1177         BUG_ON(pb->pb_bn == XFS_BUF_DADDR_NULL);
1178
1179         /* For writes allow an alternate strategy routine to precede
1180          * the actual I/O request (which may not be issued at all in
1181          * a shutdown situation, for example).
1182          */
1183         status = (flags & PBF_WRITE) ?
1184                 pagebuf_iostrategy(pb) : pagebuf_iorequest(pb);
1185
1186         /* Wait for I/O if we are not an async request.
1187          * Note: async I/O request completion will release the buffer,
1188          * and that can already be done by this point.  So using the
1189          * buffer pointer from here on, after async I/O, is invalid.
1190          */
1191         if (!status && !(flags & PBF_ASYNC))
1192                 status = pagebuf_iowait(pb);
1193
1194         return status;
1195 }
1196
1197 /*
1198  * Helper routine for pagebuf_iorequest
1199  */
1200
1201 STATIC __inline__ int
1202 _pagebuf_iolocked(
1203         xfs_buf_t               *pb)
1204 {
1205         ASSERT(pb->pb_flags & (PBF_READ|PBF_WRITE));
1206         if (pb->pb_flags & PBF_READ)
1207                 return pb->pb_locked;
1208         return 0;
1209 }
1210
1211 STATIC __inline__ void
1212 _pagebuf_iodone(
1213         xfs_buf_t               *pb,
1214         int                     schedule)
1215 {
1216         if (atomic_dec_and_test(&pb->pb_io_remaining) == 1) {
1217                 pb->pb_locked = 0;
1218                 pagebuf_iodone(pb, (pb->pb_flags & PBF_FS_DATAIOD), schedule);
1219         }
1220 }
1221
1222 STATIC int
1223 bio_end_io_pagebuf(
1224         struct bio              *bio,
1225         unsigned int            bytes_done,
1226         int                     error)
1227 {
1228         xfs_buf_t               *pb = (xfs_buf_t *)bio->bi_private;
1229         unsigned int            i, blocksize = pb->pb_target->pbr_bsize;
1230         unsigned int            sectorshift = pb->pb_target->pbr_sshift;
1231         struct bio_vec          *bvec = bio->bi_io_vec;
1232
1233         if (bio->bi_size)
1234                 return 1;
1235
1236         if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
1237                 pb->pb_error = EIO;
1238
1239         for (i = 0; i < bio->bi_vcnt; i++, bvec++) {
1240                 struct page     *page = bvec->bv_page;
1241
1242                 if (pb->pb_error) {
1243                         SetPageError(page);
1244                 } else if (blocksize == PAGE_CACHE_SIZE) {
1245                         SetPageUptodate(page);
1246                 } else if (!PagePrivate(page) &&
1247                                 (pb->pb_flags & _PBF_PAGE_CACHE)) {
1248                         unsigned long   j, range;
1249
1250                         ASSERT(blocksize < PAGE_CACHE_SIZE);
1251                         range = (bvec->bv_offset + bvec->bv_len) >> sectorshift;
1252                         for (j = bvec->bv_offset >> sectorshift; j < range; j++)
1253                                 set_bit(j, &page->private);
1254                         if (page->private == (unsigned long)(PAGE_CACHE_SIZE-1))
1255                                 SetPageUptodate(page);
1256                 }
1257
1258                 if (_pagebuf_iolocked(pb)) {
1259                         unlock_page(page);
1260                 }
1261         }
1262
1263         _pagebuf_iodone(pb, 1);
1264         bio_put(bio);
1265         return 0;
1266 }
1267
1268 void
1269 _pagebuf_ioapply(
1270         xfs_buf_t               *pb)
1271 {
1272         int                     i, map_i, total_nr_pages, nr_pages;
1273         struct bio              *bio;
1274         int                     offset = pb->pb_offset;
1275         int                     size = pb->pb_count_desired;
1276         sector_t                sector = pb->pb_bn;
1277         unsigned int            blocksize = pb->pb_target->pbr_bsize;
1278         int                     locking = _pagebuf_iolocked(pb);
1279
1280         total_nr_pages = pb->pb_page_count;
1281         map_i = 0;
1282
1283         /* Special code path for reading a sub page size pagebuf in --
1284          * we populate up the whole page, and hence the other metadata
1285          * in the same page.  This optimization is only valid when the
1286          * filesystem block size and the page size are equal.
1287          */
1288         if ((pb->pb_buffer_length < PAGE_CACHE_SIZE) &&
1289             (pb->pb_flags & PBF_READ) && locking &&
1290             (blocksize == PAGE_CACHE_SIZE)) {
1291                 bio = bio_alloc(GFP_NOIO, 1);
1292
1293                 bio->bi_bdev = pb->pb_target->pbr_bdev;
1294                 bio->bi_sector = sector - (offset >> BBSHIFT);
1295                 bio->bi_end_io = bio_end_io_pagebuf;
1296                 bio->bi_private = pb;
1297
1298                 bio_add_page(bio, pb->pb_pages[0], PAGE_CACHE_SIZE, 0);
1299                 size = 0;
1300
1301                 atomic_inc(&pb->pb_io_remaining);
1302
1303                 goto submit_io;
1304         }
1305
1306         /* Lock down the pages which we need to for the request */
1307         if (locking && (pb->pb_flags & PBF_WRITE) && (pb->pb_locked == 0)) {
1308                 for (i = 0; size; i++) {
1309                         int             nbytes = PAGE_CACHE_SIZE - offset;
1310                         struct page     *page = pb->pb_pages[i];
1311
1312                         if (nbytes > size)
1313                                 nbytes = size;
1314
1315                         lock_page(page);
1316
1317                         size -= nbytes;
1318                         offset = 0;
1319                 }
1320                 offset = pb->pb_offset;
1321                 size = pb->pb_count_desired;
1322         }
1323
1324 next_chunk:
1325         atomic_inc(&pb->pb_io_remaining);
1326         nr_pages = BIO_MAX_SECTORS >> (PAGE_SHIFT - BBSHIFT);
1327         if (nr_pages > total_nr_pages)
1328                 nr_pages = total_nr_pages;
1329
1330         bio = bio_alloc(GFP_NOIO, nr_pages);
1331         bio->bi_bdev = pb->pb_target->pbr_bdev;
1332         bio->bi_sector = sector;
1333         bio->bi_end_io = bio_end_io_pagebuf;
1334         bio->bi_private = pb;
1335
1336         for (; size && nr_pages; nr_pages--, map_i++) {
1337                 int     nbytes = PAGE_CACHE_SIZE - offset;
1338
1339                 if (nbytes > size)
1340                         nbytes = size;
1341
1342                 if (bio_add_page(bio, pb->pb_pages[map_i],
1343                                         nbytes, offset) < nbytes)
1344                         break;
1345
1346                 offset = 0;
1347                 sector += nbytes >> BBSHIFT;
1348                 size -= nbytes;
1349                 total_nr_pages--;
1350         }
1351
1352 submit_io:
1353         if (likely(bio->bi_size)) {
1354                 submit_bio((pb->pb_flags & PBF_READ) ? READ : WRITE, bio);
1355                 if (size)
1356                         goto next_chunk;
1357         } else {
1358                 bio_put(bio);
1359                 pagebuf_ioerror(pb, EIO);
1360         }
1361
1362         if (pb->pb_flags & _PBF_RUN_QUEUES) {
1363                 pb->pb_flags &= ~_PBF_RUN_QUEUES;
1364                 if (atomic_read(&pb->pb_io_remaining) > 1)
1365                         blk_run_address_space(pb->pb_target->pbr_mapping);
1366         }
1367 }
1368
1369 /*
1370  *      pagebuf_iorequest -- the core I/O request routine.
1371  */
1372 int
1373 pagebuf_iorequest(                      /* start real I/O               */
1374         xfs_buf_t               *pb)    /* buffer to convey to device   */
1375 {
1376         PB_TRACE(pb, "iorequest", 0);
1377
1378         if (pb->pb_flags & PBF_DELWRI) {
1379                 pagebuf_delwri_queue(pb, 1);
1380                 return 0;
1381         }
1382
1383         if (pb->pb_flags & PBF_WRITE) {
1384                 _pagebuf_wait_unpin(pb);
1385         }
1386
1387         pagebuf_hold(pb);
1388
1389         /* Set the count to 1 initially, this will stop an I/O
1390          * completion callout which happens before we have started
1391          * all the I/O from calling pagebuf_iodone too early.
1392          */
1393         atomic_set(&pb->pb_io_remaining, 1);
1394         _pagebuf_ioapply(pb);
1395         _pagebuf_iodone(pb, 0);
1396
1397         pagebuf_rele(pb);
1398         return 0;
1399 }
1400
1401 /*
1402  *      pagebuf_iowait
1403  *
1404  *      pagebuf_iowait waits for I/O to complete on the buffer supplied.
1405  *      It returns immediately if no I/O is pending.  In any case, it returns
1406  *      the error code, if any, or 0 if there is no error.
1407  */
1408 int
1409 pagebuf_iowait(
1410         xfs_buf_t               *pb)
1411 {
1412         PB_TRACE(pb, "iowait", 0);
1413         if (atomic_read(&pb->pb_io_remaining))
1414                 blk_run_address_space(pb->pb_target->pbr_mapping);
1415         down(&pb->pb_iodonesema);
1416         PB_TRACE(pb, "iowaited", (long)pb->pb_error);
1417         return pb->pb_error;
1418 }
1419
1420 caddr_t
1421 pagebuf_offset(
1422         xfs_buf_t               *pb,
1423         size_t                  offset)
1424 {
1425         struct page             *page;
1426
1427         offset += pb->pb_offset;
1428
1429         page = pb->pb_pages[offset >> PAGE_CACHE_SHIFT];
1430         return (caddr_t) page_address(page) + (offset & (PAGE_CACHE_SIZE - 1));
1431 }
1432
1433 /*
1434  *      pagebuf_iomove
1435  *
1436  *      Move data into or out of a buffer.
1437  */
1438 void
1439 pagebuf_iomove(
1440         xfs_buf_t               *pb,    /* buffer to process            */
1441         size_t                  boff,   /* starting buffer offset       */
1442         size_t                  bsize,  /* length to copy               */
1443         caddr_t                 data,   /* data address                 */
1444         page_buf_rw_t           mode)   /* read/write flag              */
1445 {
1446         size_t                  bend, cpoff, csize;
1447         struct page             *page;
1448
1449         bend = boff + bsize;
1450         while (boff < bend) {
1451                 page = pb->pb_pages[page_buf_btoct(boff + pb->pb_offset)];
1452                 cpoff = page_buf_poff(boff + pb->pb_offset);
1453                 csize = min_t(size_t,
1454                               PAGE_CACHE_SIZE-cpoff, pb->pb_count_desired-boff);
1455
1456                 ASSERT(((csize + cpoff) <= PAGE_CACHE_SIZE));
1457
1458                 switch (mode) {
1459                 case PBRW_ZERO:
1460                         memset(page_address(page) + cpoff, 0, csize);
1461                         break;
1462                 case PBRW_READ:
1463                         memcpy(data, page_address(page) + cpoff, csize);
1464                         break;
1465                 case PBRW_WRITE:
1466                         memcpy(page_address(page) + cpoff, data, csize);
1467                 }
1468
1469                 boff += csize;
1470                 data += csize;
1471         }
1472 }
1473
1474 /*
1475  *      Handling of buftargs.
1476  */
1477
1478 void
1479 xfs_free_buftarg(
1480         xfs_buftarg_t           *btp,
1481         int                     external)
1482 {
1483         xfs_flush_buftarg(btp, 1);
1484         if (external)
1485                 xfs_blkdev_put(btp->pbr_bdev);
1486         kmem_free(btp, sizeof(*btp));
1487 }
1488
1489 void
1490 xfs_incore_relse(
1491         xfs_buftarg_t           *btp,
1492         int                     delwri_only,
1493         int                     wait)
1494 {
1495         invalidate_bdev(btp->pbr_bdev, 1);
1496         truncate_inode_pages(btp->pbr_mapping, 0LL);
1497 }
1498
1499 void
1500 xfs_setsize_buftarg(
1501         xfs_buftarg_t           *btp,
1502         unsigned int            blocksize,
1503         unsigned int            sectorsize)
1504 {
1505         btp->pbr_bsize = blocksize;
1506         btp->pbr_sshift = ffs(sectorsize) - 1;
1507         btp->pbr_smask = sectorsize - 1;
1508
1509         if (set_blocksize(btp->pbr_bdev, sectorsize)) {
1510                 printk(KERN_WARNING
1511                         "XFS: Cannot set_blocksize to %u on device %s\n",
1512                         sectorsize, XFS_BUFTARG_NAME(btp));
1513         }
1514 }
1515
1516 xfs_buftarg_t *
1517 xfs_alloc_buftarg(
1518         struct block_device     *bdev)
1519 {
1520         xfs_buftarg_t           *btp;
1521
1522         btp = kmem_zalloc(sizeof(*btp), KM_SLEEP);
1523
1524         btp->pbr_dev =  bdev->bd_dev;
1525         btp->pbr_bdev = bdev;
1526         btp->pbr_mapping = bdev->bd_inode->i_mapping;
1527         xfs_setsize_buftarg(btp, PAGE_CACHE_SIZE, bdev_hardsect_size(bdev));
1528
1529         return btp;
1530 }
1531
1532
1533 /*
1534  * Pagebuf delayed write buffer handling
1535  */
1536
1537 STATIC LIST_HEAD(pbd_delwrite_queue);
1538 STATIC spinlock_t pbd_delwrite_lock = SPIN_LOCK_UNLOCKED;
1539
1540 STATIC void
1541 pagebuf_delwri_queue(
1542         xfs_buf_t               *pb,
1543         int                     unlock)
1544 {
1545         PB_TRACE(pb, "delwri_q", (long)unlock);
1546         ASSERT(pb->pb_flags & PBF_DELWRI);
1547
1548         spin_lock(&pbd_delwrite_lock);
1549         /* If already in the queue, dequeue and place at tail */
1550         if (!list_empty(&pb->pb_list)) {
1551                 if (unlock) {
1552                         atomic_dec(&pb->pb_hold);
1553                 }
1554                 list_del(&pb->pb_list);
1555         }
1556
1557         list_add_tail(&pb->pb_list, &pbd_delwrite_queue);
1558         pb->pb_queuetime = jiffies;
1559         spin_unlock(&pbd_delwrite_lock);
1560
1561         if (unlock)
1562                 pagebuf_unlock(pb);
1563 }
1564
1565 void
1566 pagebuf_delwri_dequeue(
1567         xfs_buf_t               *pb)
1568 {
1569         PB_TRACE(pb, "delwri_uq", 0);
1570         spin_lock(&pbd_delwrite_lock);
1571         list_del_init(&pb->pb_list);
1572         pb->pb_flags &= ~PBF_DELWRI;
1573         spin_unlock(&pbd_delwrite_lock);
1574 }
1575
1576 STATIC void
1577 pagebuf_runall_queues(
1578         struct workqueue_struct *queue)
1579 {
1580         flush_workqueue(queue);
1581 }
1582
1583 /* Defines for pagebuf daemon */
1584 STATIC DECLARE_COMPLETION(pagebuf_daemon_done);
1585 STATIC struct task_struct *pagebuf_daemon_task;
1586 STATIC int pagebuf_daemon_active;
1587 STATIC int force_flush;
1588
1589 STATIC void
1590 pagebuf_daemon_wakeup(void)
1591 {
1592         force_flush = 1;
1593         barrier();
1594         wake_up_process(pagebuf_daemon_task);
1595 }
1596
1597 STATIC int
1598 pagebuf_daemon(
1599         void                    *data)
1600 {
1601         struct list_head        tmp;
1602         unsigned long           age;
1603         xfs_buf_t               *pb, *n;
1604
1605         /*  Set up the thread  */
1606         daemonize("xfsbufd");
1607         current->flags |= PF_MEMALLOC;
1608
1609         pagebuf_daemon_task = current;
1610         pagebuf_daemon_active = 1;
1611         barrier();
1612
1613         INIT_LIST_HEAD(&tmp);
1614         do {
1615                 /* swsusp */
1616                 if (current->flags & PF_FREEZE)
1617                         refrigerator(PF_FREEZE);
1618
1619                 set_current_state(TASK_INTERRUPTIBLE);
1620                 schedule_timeout((xfs_buf_timer_centisecs * HZ) / 100);
1621
1622                 age = (xfs_buf_age_centisecs * HZ) / 100;
1623                 spin_lock(&pbd_delwrite_lock);
1624                 list_for_each_entry_safe(pb, n, &pbd_delwrite_queue, pb_list) {
1625                         PB_TRACE(pb, "walkq1", (long)pagebuf_ispin(pb));
1626                         ASSERT(pb->pb_flags & PBF_DELWRI);
1627
1628                         if (!pagebuf_ispin(pb) && !pagebuf_cond_lock(pb)) {
1629                                 if (!force_flush &&
1630                                     time_before(jiffies,
1631                                                 pb->pb_queuetime + age)) {
1632                                         pagebuf_unlock(pb);
1633                                         break;
1634                                 }
1635
1636                                 pb->pb_flags &= ~PBF_DELWRI;
1637                                 pb->pb_flags |= PBF_WRITE;
1638                                 list_move(&pb->pb_list, &tmp);
1639                         }
1640                 }
1641                 spin_unlock(&pbd_delwrite_lock);
1642
1643                 while (!list_empty(&tmp)) {
1644                         pb = list_entry(tmp.next, xfs_buf_t, pb_list);
1645                         list_del_init(&pb->pb_list);
1646                         pagebuf_iostrategy(pb);
1647                         blk_run_address_space(pb->pb_target->pbr_mapping);
1648                 }
1649
1650                 if (as_list_len > 0)
1651                         purge_addresses();
1652
1653                 force_flush = 0;
1654         } while (pagebuf_daemon_active);
1655
1656         complete_and_exit(&pagebuf_daemon_done, 0);
1657 }
1658
1659 /*
1660  * Go through all incore buffers, and release buffers if they belong to
1661  * the given device. This is used in filesystem error handling to
1662  * preserve the consistency of its metadata.
1663  */
1664 int
1665 xfs_flush_buftarg(
1666         xfs_buftarg_t           *target,
1667         int                     wait)
1668 {
1669         struct list_head        tmp;
1670         xfs_buf_t               *pb, *n;
1671         int                     pincount = 0;
1672
1673         pagebuf_runall_queues(pagebuf_dataio_workqueue);
1674         pagebuf_runall_queues(pagebuf_logio_workqueue);
1675
1676         INIT_LIST_HEAD(&tmp);
1677         spin_lock(&pbd_delwrite_lock);
1678         list_for_each_entry_safe(pb, n, &pbd_delwrite_queue, pb_list) {
1679
1680                 if (pb->pb_target != target)
1681                         continue;
1682
1683                 ASSERT(pb->pb_flags & PBF_DELWRI);
1684                 PB_TRACE(pb, "walkq2", (long)pagebuf_ispin(pb));
1685                 if (pagebuf_ispin(pb)) {
1686                         pincount++;
1687                         continue;
1688                 }
1689
1690                 pb->pb_flags &= ~PBF_DELWRI;
1691                 pb->pb_flags |= PBF_WRITE;
1692                 list_move(&pb->pb_list, &tmp);
1693         }
1694         spin_unlock(&pbd_delwrite_lock);
1695
1696         /*
1697          * Dropped the delayed write list lock, now walk the temporary list
1698          */
1699         list_for_each_entry_safe(pb, n, &tmp, pb_list) {
1700                 if (wait)
1701                         pb->pb_flags &= ~PBF_ASYNC;
1702                 else
1703                         list_del_init(&pb->pb_list);
1704
1705                 pagebuf_lock(pb);
1706                 pagebuf_iostrategy(pb);
1707         }
1708
1709         /*
1710          * Remaining list items must be flushed before returning
1711          */
1712         while (!list_empty(&tmp)) {
1713                 pb = list_entry(tmp.next, xfs_buf_t, pb_list);
1714
1715                 list_del_init(&pb->pb_list);
1716                 xfs_iowait(pb);
1717                 xfs_buf_relse(pb);
1718         }
1719
1720         if (wait)
1721                 blk_run_address_space(target->pbr_mapping);
1722
1723         return pincount;
1724 }
1725
1726 STATIC int
1727 pagebuf_daemon_start(void)
1728 {
1729         int             rval;
1730
1731         pagebuf_logio_workqueue = create_workqueue("xfslogd");
1732         if (!pagebuf_logio_workqueue)
1733                 return -ENOMEM;
1734
1735         pagebuf_dataio_workqueue = create_workqueue("xfsdatad");
1736         if (!pagebuf_dataio_workqueue) {
1737                 destroy_workqueue(pagebuf_logio_workqueue);
1738                 return -ENOMEM;
1739         }
1740
1741         rval = kernel_thread(pagebuf_daemon, NULL, CLONE_FS|CLONE_FILES);
1742         if (rval < 0) {
1743                 destroy_workqueue(pagebuf_logio_workqueue);
1744                 destroy_workqueue(pagebuf_dataio_workqueue);
1745         }
1746
1747         return rval;
1748 }
1749
1750 /*
1751  * pagebuf_daemon_stop
1752  *
1753  * Note: do not mark as __exit, it is called from pagebuf_terminate.
1754  */
1755 STATIC void
1756 pagebuf_daemon_stop(void)
1757 {
1758         pagebuf_daemon_active = 0;
1759         barrier();
1760         wait_for_completion(&pagebuf_daemon_done);
1761
1762         destroy_workqueue(pagebuf_logio_workqueue);
1763         destroy_workqueue(pagebuf_dataio_workqueue);
1764 }
1765
1766 /*
1767  *      Initialization and Termination
1768  */
1769
1770 int __init
1771 pagebuf_init(void)
1772 {
1773         int                     i;
1774
1775         pagebuf_cache = kmem_cache_create("xfs_buf_t", sizeof(xfs_buf_t), 0,
1776                         SLAB_HWCACHE_ALIGN, NULL, NULL);
1777         if (pagebuf_cache == NULL) {
1778                 printk("pagebuf: couldn't init pagebuf cache\n");
1779                 pagebuf_terminate();
1780                 return -ENOMEM;
1781         }
1782
1783         for (i = 0; i < NHASH; i++) {
1784                 spin_lock_init(&pbhash[i].pb_hash_lock);
1785                 INIT_LIST_HEAD(&pbhash[i].pb_hash);
1786         }
1787
1788 #ifdef PAGEBUF_TRACE
1789         pagebuf_trace_buf = ktrace_alloc(PAGEBUF_TRACE_SIZE, KM_SLEEP);
1790 #endif
1791
1792         pagebuf_daemon_start();
1793         return 0;
1794 }
1795
1796
1797 /*
1798  *      pagebuf_terminate.
1799  *
1800  *      Note: do not mark as __exit, this is also called from the __init code.
1801  */
1802 void
1803 pagebuf_terminate(void)
1804 {
1805         pagebuf_daemon_stop();
1806
1807 #ifdef PAGEBUF_TRACE
1808         ktrace_free(pagebuf_trace_buf);
1809 #endif
1810
1811         kmem_cache_destroy(pagebuf_cache);
1812 }