fs/xfs/linux-2.6/xfs_buf.c

   1 /*
   2  * Copyright (c) 2000-2004 Silicon Graphics, Inc.  All Rights Reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms of version 2 of the GNU General Public License as
   6  * published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it would be useful, but
   9  * WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
  11  *
  12  * Further, this software is distributed without any warranty that it is
  13  * free of the rightful claim of any third person regarding infringement
  14  * or the like.  Any license provided herein, whether implied or
  15  * otherwise, applies only to this software file.  Patent licenses, if
  16  * any, provided herein do not apply to combinations of this program with
  17  * other software, or any other product whatsoever.
  18  *
  19  * You should have received a copy of the GNU General Public License along
  20  * with this program; if not, write the Free Software Foundation, Inc., 59
  21  * Temple Place - Suite 330, Boston MA 02111-1307, USA.
  22  *
  23  * Contact information: Silicon Graphics, Inc., 1600 Amphitheatre Pkwy,
  24  * Mountain View, CA  94043, or:
  25  *
  26  * http://www.sgi.com
  27  *
  28  * For further information regarding this notice, see:
  29  *
  30  * http://oss.sgi.com/projects/GenInfo/SGIGPLNoticeExplan/
  31  */
  32
  33 /*
  34  *      The xfs_buf.c code provides an abstract buffer cache model on top
  35  *      of the Linux page cache.  Cached metadata blocks for a file system
  36  *      are hashed to the inode for the block device.  xfs_buf.c assembles
  37  *      buffers (xfs_buf_t) on demand to aggregate such cached pages for I/O.
  38  *
  39  *      Written by Steve Lord, Jim Mostek, Russell Cattelan
  40  *                  and Rajagopal Ananthanarayanan ("ananth") at SGI.
  41  *
  42  */
  43
  44 #include <linux/stddef.h>
  45 #include <linux/errno.h>
  46 #include <linux/slab.h>
  47 #include <linux/pagemap.h>
  48 #include <linux/init.h>
  49 #include <linux/vmalloc.h>
  50 #include <linux/bio.h>
  51 #include <linux/sysctl.h>
  52 #include <linux/proc_fs.h>
  53 #include <linux/workqueue.h>
  54 #include <linux/suspend.h>
  55 #include <linux/percpu.h>
  56 #include <linux/blkdev.h>
  57
  58 #include "xfs_linux.h"
  59
  60 /*
  61  * File wide globals
  62  */
  63
  64 STATIC kmem_cache_t *pagebuf_cache;
  65 STATIC kmem_shaker_t pagebuf_shake;
  66 STATIC int pagebuf_daemon_wakeup(int, unsigned int);
  67 STATIC void pagebuf_delwri_queue(xfs_buf_t *, int);
  68 STATIC struct workqueue_struct *pagebuf_logio_workqueue;
  69 STATIC struct workqueue_struct *pagebuf_dataio_workqueue;
  70
  71 /*
  72  * Pagebuf debugging
  73  */
  74
  75 #ifdef PAGEBUF_TRACE
  76 void
  77 pagebuf_trace(
  78         xfs_buf_t       *pb,
  79         char            *id,
  80         void            *data,
  81         void            *ra)
  82 {
  83         ktrace_enter(pagebuf_trace_buf,
  84                 pb, id,
  85                 (void *)(unsigned long)pb->pb_flags,
  86                 (void *)(unsigned long)pb->pb_hold.counter,
  87                 (void *)(unsigned long)pb->pb_sema.count.counter,
  88                 (void *)current,
  89                 data, ra,
  90                 (void *)(unsigned long)((pb->pb_file_offset>>32) & 0xffffffff),
  91                 (void *)(unsigned long)(pb->pb_file_offset & 0xffffffff),
  92                 (void *)(unsigned long)pb->pb_buffer_length,
  93                 NULL, NULL, NULL, NULL, NULL);
  94 }
  95 ktrace_t *pagebuf_trace_buf;
  96 #define PAGEBUF_TRACE_SIZE      4096
  97 #define PB_TRACE(pb, id, data)  \
  98         pagebuf_trace(pb, id, (void *)data, (void *)__builtin_return_address(0))
  99 #else
 100 #define PB_TRACE(pb, id, data)  do { } while (0)
 101 #endif
 102
 103 #ifdef PAGEBUF_LOCK_TRACKING
 104 # define PB_SET_OWNER(pb)       ((pb)->pb_last_holder = current->pid)
 105 # define PB_CLEAR_OWNER(pb)     ((pb)->pb_last_holder = -1)
 106 # define PB_GET_OWNER(pb)       ((pb)->pb_last_holder)
 107 #else
 108 # define PB_SET_OWNER(pb)       do { } while (0)
 109 # define PB_CLEAR_OWNER(pb)     do { } while (0)
 110 # define PB_GET_OWNER(pb)       do { } while (0)
 111 #endif
 112
 113 /*
 114  * Pagebuf allocation / freeing.
 115  */
 116
 117 #define pb_to_gfp(flags) \
 118         ((((flags) & PBF_READ_AHEAD) ? __GFP_NORETRY : \
 119           ((flags) & PBF_DONT_BLOCK) ? GFP_NOFS : GFP_KERNEL) | __GFP_NOWARN)
 120
 121 #define pb_to_km(flags) \
 122          (((flags) & PBF_DONT_BLOCK) ? KM_NOFS : KM_SLEEP)
 123
 124
 125 #define pagebuf_allocate(flags) \
 126         kmem_zone_alloc(pagebuf_cache, pb_to_km(flags))
 127 #define pagebuf_deallocate(pb) \
 128         kmem_zone_free(pagebuf_cache, (pb));
 129
 130 /*
 131  * Pagebuf hashing
 132  */
 133
 134 #define NBITS   8
 135 #define NHASH   (1<<NBITS)
 136
 137 typedef struct {
 138         struct list_head        pb_hash;
 139         spinlock_t              pb_hash_lock;
 140 } pb_hash_t;
 141
 142 STATIC pb_hash_t        pbhash[NHASH];
 143 #define pb_hash(pb)     &pbhash[pb->pb_hash_index]
 144
 145 STATIC int
 146 _bhash(
 147         struct block_device *bdev,
 148         loff_t          base)
 149 {
 150         int             bit, hval;
 151
 152         base >>= 9;
 153         base ^= (unsigned long)bdev / L1_CACHE_BYTES;
 154         for (bit = hval = 0; base && bit < sizeof(base) * 8; bit += NBITS) {
 155                 hval ^= (int)base & (NHASH-1);
 156                 base >>= NBITS;
 157         }
 158         return hval;
 159 }
 160
 161 /*
 162  * Mapping of multi-page buffers into contiguous virtual space
 163  */
 164
 165 typedef struct a_list {
 166         void            *vm_addr;
 167         struct a_list   *next;
 168 } a_list_t;
 169
 170 STATIC a_list_t         *as_free_head;
 171 STATIC int              as_list_len;
 172 STATIC spinlock_t       as_lock = SPIN_LOCK_UNLOCKED;
 173
 174 /*
 175  * Try to batch vunmaps because they are costly.
 176  */
 177 STATIC void
 178 free_address(
 179         void            *addr)
 180 {
 181         a_list_t        *aentry;
 182
 183         aentry = kmalloc(sizeof(a_list_t), GFP_ATOMIC);
 184         if (aentry) {
 185                 spin_lock(&as_lock);
 186                 aentry->next = as_free_head;
 187                 aentry->vm_addr = addr;
 188                 as_free_head = aentry;
 189                 as_list_len++;
 190                 spin_unlock(&as_lock);
 191         } else {
 192                 vunmap(addr);
 193         }
 194 }
 195
 196 STATIC void
 197 purge_addresses(void)
 198 {
 199         a_list_t        *aentry, *old;
 200
 201         if (as_free_head == NULL)
 202                 return;
 203
 204         spin_lock(&as_lock);
 205         aentry = as_free_head;
 206         as_free_head = NULL;
 207         as_list_len = 0;
 208         spin_unlock(&as_lock);
 209
 210         while ((old = aentry) != NULL) {
 211                 vunmap(aentry->vm_addr);
 212                 aentry = aentry->next;
 213                 kfree(old);
 214         }
 215 }
 216
 217 /*
 218  *      Internal pagebuf object manipulation
 219  */
 220
 221 STATIC void
 222 _pagebuf_initialize(
 223         xfs_buf_t               *pb,
 224         xfs_buftarg_t           *target,
 225         loff_t                  range_base,
 226         size_t                  range_length,
 227         page_buf_flags_t        flags)
 228 {
 229         /*
 230          * We don't want certain flags to appear in pb->pb_flags.
 231          */
 232         flags &= ~(PBF_LOCK|PBF_MAPPED|PBF_DONT_BLOCK|PBF_READ_AHEAD);
 233
 234         memset(pb, 0, sizeof(xfs_buf_t));
 235         atomic_set(&pb->pb_hold, 1);
 236         init_MUTEX_LOCKED(&pb->pb_iodonesema);
 237         INIT_LIST_HEAD(&pb->pb_list);
 238         INIT_LIST_HEAD(&pb->pb_hash_list);
 239         init_MUTEX_LOCKED(&pb->pb_sema); /* held, no waiters */
 240         PB_SET_OWNER(pb);
 241         pb->pb_target = target;
 242         pb->pb_file_offset = range_base;
 243         /*
 244          * Set buffer_length and count_desired to the same value initially.
 245          * I/O routines should use count_desired, which will be the same in
 246          * most cases but may be reset (e.g. XFS recovery).
 247          */
 248         pb->pb_buffer_length = pb->pb_count_desired = range_length;
 249         pb->pb_flags = flags | PBF_NONE;
 250         pb->pb_bn = XFS_BUF_DADDR_NULL;
 251         atomic_set(&pb->pb_pin_count, 0);
 252         init_waitqueue_head(&pb->pb_waiters);
 253
 254         XFS_STATS_INC(pb_create);
 255         PB_TRACE(pb, "initialize", target);
 256 }
 257
 258 /*
 259  * Allocate a page array capable of holding a specified number
 260  * of pages, and point the page buf at it.
 261  */
 262 STATIC int
 263 _pagebuf_get_pages(
 264         xfs_buf_t               *pb,
 265         int                     page_count,
 266         page_buf_flags_t        flags)
 267 {
 268         /* Make sure that we have a page list */
 269         if (pb->pb_pages == NULL) {
 270                 pb->pb_offset = page_buf_poff(pb->pb_file_offset);
 271                 pb->pb_page_count = page_count;
 272                 if (page_count <= PB_PAGES) {
 273                         pb->pb_pages = pb->pb_page_array;
 274                 } else {
 275                         pb->pb_pages = kmem_alloc(sizeof(struct page *) *
 276                                         page_count, pb_to_km(flags));
 277                         if (pb->pb_pages == NULL)
 278                                 return -ENOMEM;
 279                 }
 280                 memset(pb->pb_pages, 0, sizeof(struct page *) * page_count);
 281         }
 282         return 0;
 283 }
 284
 285 /*
 286  *      Frees pb_pages if it was malloced.
 287  */
 288 STATIC void
 289 _pagebuf_free_pages(
 290         xfs_buf_t       *bp)
 291 {
 292         if (bp->pb_pages != bp->pb_page_array) {
 293                 kmem_free(bp->pb_pages,
 294                           bp->pb_page_count * sizeof(struct page *));
 295         }
 296 }
 297
 298 /*
 299  *      Releases the specified buffer.
 300  *
 301  *      The modification state of any associated pages is left unchanged.
 302  *      The buffer most not be on any hash - use pagebuf_rele instead for
 303  *      hashed and refcounted buffers
 304  */
 305 void
 306 pagebuf_free(
 307         xfs_buf_t               *bp)
 308 {
 309         PB_TRACE(bp, "free", 0);
 310
 311         ASSERT(list_empty(&bp->pb_hash_list));
 312
 313         if (bp->pb_flags & _PBF_PAGE_CACHE) {
 314                 uint            i;
 315
 316                 if ((bp->pb_flags & PBF_MAPPED) && (bp->pb_page_count > 1))
 317                         free_address(bp->pb_addr - bp->pb_offset);
 318
 319                 for (i = 0; i < bp->pb_page_count; i++)
 320                         page_cache_release(bp->pb_pages[i]);
 321                 _pagebuf_free_pages(bp);
 322         } else if (bp->pb_flags & _PBF_KMEM_ALLOC) {
 323                  /*
 324                   * XXX(hch): bp->pb_count_desired might be incorrect (see
 325                   * pagebuf_associate_memory for details), but fortunately
 326                   * the Linux version of kmem_free ignores the len argument..
 327                   */
 328                 kmem_free(bp->pb_addr, bp->pb_count_desired);
 329                 _pagebuf_free_pages(bp);
 330         }
 331
 332         pagebuf_deallocate(bp);
 333 }
 334
 335 /*
 336  *      Finds all pages for buffer in question and builds it's page list.
 337  */
 338 STATIC int
 339 _pagebuf_lookup_pages(
 340         xfs_buf_t               *bp,
 341         uint                    flags)
 342 {
 343         struct address_space    *mapping = bp->pb_target->pbr_mapping;
 344         unsigned int            sectorshift = bp->pb_target->pbr_sshift;
 345         size_t                  blocksize = bp->pb_target->pbr_bsize;
 346         size_t                  size = bp->pb_count_desired;
 347         size_t                  nbytes, offset;
 348         int                     gfp_mask = pb_to_gfp(flags);
 349         unsigned short          page_count, i;
 350         pgoff_t                 first;
 351         loff_t                  end;
 352         int                     error;
 353
 354         end = bp->pb_file_offset + bp->pb_buffer_length;
 355         page_count = page_buf_btoc(end) - page_buf_btoct(bp->pb_file_offset);
 356
 357         error = _pagebuf_get_pages(bp, page_count, flags);
 358         if (unlikely(error))
 359                 return error;
 360         bp->pb_flags |= _PBF_PAGE_CACHE;
 361
 362         offset = bp->pb_offset;
 363         first = bp->pb_file_offset >> PAGE_CACHE_SHIFT;
 364
 365         for (i = 0; i < bp->pb_page_count; i++) {
 366                 struct page     *page;
 367                 uint            retries = 0;
 368
 369               retry:
 370                 page = find_or_create_page(mapping, first + i, gfp_mask);
 371                 if (unlikely(page == NULL)) {
 372                         if (flags & PBF_READ_AHEAD) {
 373                                 bp->pb_page_count = i;
 374                                 for (i = 0; i < bp->pb_page_count; i++)
 375                                         unlock_page(bp->pb_pages[i]);
 376                                 return -ENOMEM;
 377                         }
 378
 379                         /*
 380                          * This could deadlock.
 381                          *
 382                          * But until all the XFS lowlevel code is revamped to
 383                          * handle buffer allocation failures we can't do much.
 384                          */
 385                         if (!(++retries % 100))
 386                                 printk(KERN_ERR
 387                                         "XFS: possible memory allocation "
 388                                         "deadlock in %s (mode:0x%x)\n",
 389                                         __FUNCTION__, gfp_mask);
 390
 391                         XFS_STATS_INC(pb_page_retries);
 392                         pagebuf_daemon_wakeup(0, gfp_mask);
 393                         blk_congestion_wait(WRITE, HZ/50);
 394                         goto retry;
 395                 }
 396
 397                 XFS_STATS_INC(pb_page_found);
 398
 399                 nbytes = min_t(size_t, size, PAGE_CACHE_SIZE - offset);
 400                 size -= nbytes;
 401
 402                 if (!PageUptodate(page)) {
 403                         page_count--;
 404                         if (blocksize == PAGE_CACHE_SIZE) {
 405                                 if (flags & PBF_READ)
 406                                         bp->pb_locked = 1;
 407                         } else if (!PagePrivate(page)) {
 408                                 unsigned long   j, range;
 409
 410                                 /*
 411                                  * In this case page->private holds a bitmap
 412                                  * of uptodate sectors within the page
 413                                  */
 414                                 ASSERT(blocksize < PAGE_CACHE_SIZE);
 415                                 range = (offset + nbytes) >> sectorshift;
 416                                 for (j = offset >> sectorshift; j < range; j++)
 417                                         if (!test_bit(j, &page->private))
 418                                                 break;
 419                                 if (j == range)
 420                                         page_count++;
 421                         }
 422                 }
 423
 424                 bp->pb_pages[i] = page;
 425                 offset = 0;
 426         }
 427
 428         if (!bp->pb_locked) {
 429                 for (i = 0; i < bp->pb_page_count; i++)
 430                         unlock_page(bp->pb_pages[i]);
 431         }
 432
 433         if (page_count) {
 434                 /* if we have any uptodate pages, mark that in the buffer */
 435                 bp->pb_flags &= ~PBF_NONE;
 436
 437                 /* if some pages aren't uptodate, mark that in the buffer */
 438                 if (page_count != bp->pb_page_count)
 439                         bp->pb_flags |= PBF_PARTIAL;
 440         }
 441
 442         PB_TRACE(bp, "lookup_pages", (long)page_count);
 443         return error;
 444 }
 445
 446 /*
 447  *      Map buffer into kernel address-space if nessecary.
 448  */
 449 STATIC int
 450 _pagebuf_map_pages(
 451         xfs_buf_t               *bp,
 452         uint                    flags)
 453 {
 454         /* A single page buffer is always mappable */
 455         if (bp->pb_page_count == 1) {
 456                 bp->pb_addr = page_address(bp->pb_pages[0]) + bp->pb_offset;
 457                 bp->pb_flags |= PBF_MAPPED;
 458         } else if (flags & PBF_MAPPED) {
 459                 if (as_list_len > 64)
 460                         purge_addresses();
 461                 bp->pb_addr = vmap(bp->pb_pages, bp->pb_page_count,
 462                                 VM_MAP, PAGE_KERNEL);
 463                 if (unlikely(bp->pb_addr == NULL))
 464                         return -ENOMEM;
 465                 bp->pb_addr += bp->pb_offset;
 466                 bp->pb_flags |= PBF_MAPPED;
 467         }
 468
 469         return 0;
 470 }
 471
 472 /*
 473  *      Finding and Reading Buffers
 474  */
 475
 476 /*
 477  *      _pagebuf_find
 478  *
 479  *      Looks up, and creates if absent, a lockable buffer for
 480  *      a given range of an inode.  The buffer is returned
 481  *      locked.  If other overlapping buffers exist, they are
 482  *      released before the new buffer is created and locked,
 483  *      which may imply that this call will block until those buffers
 484  *      are unlocked.  No I/O is implied by this call.
 485  */
 486 xfs_buf_t *
 487 _pagebuf_find(                          /* find buffer for block        */
 488         xfs_buftarg_t           *target,/* target for block             */
 489         loff_t                  ioff,   /* starting offset of range     */
 490         size_t                  isize,  /* length of range              */
 491         page_buf_flags_t        flags,  /* PBF_TRYLOCK                  */
 492         xfs_buf_t               *new_pb)/* newly allocated buffer       */
 493 {
 494         loff_t                  range_base;
 495         size_t                  range_length;
 496         int                     hval;
 497         pb_hash_t               *h;
 498         xfs_buf_t               *pb, *n;
 499         int                     not_locked;
 500
 501         range_base = (ioff << BBSHIFT);
 502         range_length = (isize << BBSHIFT);
 503
 504         /* Ensure we never do IOs smaller than the sector size */
 505         BUG_ON(range_length < (1 << target->pbr_sshift));
 506
 507         /* Ensure we never do IOs that are not sector aligned */
 508         BUG_ON(range_base & (loff_t)target->pbr_smask);
 509
 510         hval = _bhash(target->pbr_bdev, range_base);
 511         h = &pbhash[hval];
 512
 513         spin_lock(&h->pb_hash_lock);
 514         list_for_each_entry_safe(pb, n, &h->pb_hash, pb_hash_list) {
 515                 if (pb->pb_target == target &&
 516                     pb->pb_file_offset == range_base &&
 517                     pb->pb_buffer_length == range_length) {
 518                         /* If we look at something bring it to the
 519                          * front of the list for next time
 520                          */
 521                         atomic_inc(&pb->pb_hold);
 522                         list_move(&pb->pb_hash_list, &h->pb_hash);
 523                         goto found;
 524                 }
 525         }
 526
 527         /* No match found */
 528         if (new_pb) {
 529                 _pagebuf_initialize(new_pb, target, range_base,
 530                                 range_length, flags);
 531                 new_pb->pb_hash_index = hval;
 532                 list_add(&new_pb->pb_hash_list, &h->pb_hash);
 533         } else {
 534                 XFS_STATS_INC(pb_miss_locked);
 535         }
 536
 537         spin_unlock(&h->pb_hash_lock);
 538         return (new_pb);
 539
 540 found:
 541         spin_unlock(&h->pb_hash_lock);
 542
 543         /* Attempt to get the semaphore without sleeping,
 544          * if this does not work then we need to drop the
 545          * spinlock and do a hard attempt on the semaphore.
 546          */
 547         not_locked = down_trylock(&pb->pb_sema);
 548         if (not_locked) {
 549                 if (!(flags & PBF_TRYLOCK)) {
 550                         /* wait for buffer ownership */
 551                         PB_TRACE(pb, "get_lock", 0);
 552                         pagebuf_lock(pb);
 553                         XFS_STATS_INC(pb_get_locked_waited);
 554                 } else {
 555                         /* We asked for a trylock and failed, no need
 556                          * to look at file offset and length here, we
 557                          * know that this pagebuf at least overlaps our
 558                          * pagebuf and is locked, therefore our buffer
 559                          * either does not exist, or is this buffer
 560                          */
 561
 562                         pagebuf_rele(pb);
 563                         XFS_STATS_INC(pb_busy_locked);
 564                         return (NULL);
 565                 }
 566         } else {
 567                 /* trylock worked */
 568                 PB_SET_OWNER(pb);
 569         }
 570
 571         if (pb->pb_flags & PBF_STALE)
 572                 pb->pb_flags &= PBF_MAPPED;
 573         PB_TRACE(pb, "got_lock", 0);
 574         XFS_STATS_INC(pb_get_locked);
 575         return (pb);
 576 }
 577
 578 /*
 579  *      xfs_buf_get_flags assembles a buffer covering the specified range.
 580  *
 581  *      Storage in memory for all portions of the buffer will be allocated,
 582  *      although backing storage may not be.
 583  */
 584 xfs_buf_t *
 585 xfs_buf_get_flags(                      /* allocate a buffer            */
 586         xfs_buftarg_t           *target,/* target for buffer            */
 587         loff_t                  ioff,   /* starting offset of range     */
 588         size_t                  isize,  /* length of range              */
 589         page_buf_flags_t        flags)  /* PBF_TRYLOCK                  */
 590 {
 591         xfs_buf_t               *pb, *new_pb;
 592         int                     error = 0, i;
 593
 594         new_pb = pagebuf_allocate(flags);
 595         if (unlikely(!new_pb))
 596                 return NULL;
 597
 598         pb = _pagebuf_find(target, ioff, isize, flags, new_pb);
 599         if (pb == new_pb) {
 600                 error = _pagebuf_lookup_pages(pb, flags);
 601                 if (error)
 602                         goto no_buffer;
 603         } else {
 604                 pagebuf_deallocate(new_pb);
 605                 if (unlikely(pb == NULL))
 606                         return NULL;
 607         }
 608
 609         for (i = 0; i < pb->pb_page_count; i++)
 610                 mark_page_accessed(pb->pb_pages[i]);
 611
 612         if (!(pb->pb_flags & PBF_MAPPED)) {
 613                 error = _pagebuf_map_pages(pb, flags);
 614                 if (unlikely(error)) {
 615                         printk(KERN_WARNING "%s: failed to map pages\n",
 616                                         __FUNCTION__);
 617                         goto no_buffer;
 618                 }
 619         }
 620
 621         XFS_STATS_INC(pb_get);
 622
 623         /*
 624          * Always fill in the block number now, the mapped cases can do
 625          * their own overlay of this later.
 626          */
 627         pb->pb_bn = ioff;
 628         pb->pb_count_desired = pb->pb_buffer_length;
 629
 630         PB_TRACE(pb, "get", (unsigned long)flags);
 631         return pb;
 632
 633  no_buffer:
 634         if (flags & (PBF_LOCK | PBF_TRYLOCK))
 635                 pagebuf_unlock(pb);
 636         pagebuf_rele(pb);
 637         return NULL;
 638 }
 639
 640 xfs_buf_t *
 641 xfs_buf_read_flags(
 642         xfs_buftarg_t           *target,
 643         loff_t                  ioff,
 644         size_t                  isize,
 645         page_buf_flags_t        flags)
 646 {
 647         xfs_buf_t               *pb;
 648
 649         flags |= PBF_READ;
 650
 651         pb = xfs_buf_get_flags(target, ioff, isize, flags);
 652         if (pb) {
 653                 if (PBF_NOT_DONE(pb)) {
 654                         PB_TRACE(pb, "read", (unsigned long)flags);
 655                         XFS_STATS_INC(pb_get_read);
 656                         pagebuf_iostart(pb, flags);
 657                 } else if (flags & PBF_ASYNC) {
 658                         PB_TRACE(pb, "read_async", (unsigned long)flags);
 659                         /*
 660                          * Read ahead call which is already satisfied,
 661                          * drop the buffer
 662                          */
 663                         goto no_buffer;
 664                 } else {
 665                         PB_TRACE(pb, "read_done", (unsigned long)flags);
 666                         /* We do not want read in the flags */
 667                         pb->pb_flags &= ~PBF_READ;
 668                 }
 669         }
 670
 671         return pb;
 672
 673  no_buffer:
 674         if (flags & (PBF_LOCK | PBF_TRYLOCK))
 675                 pagebuf_unlock(pb);
 676         pagebuf_rele(pb);
 677         return NULL;
 678 }
 679
 680 /*
 681  * Create a skeletal pagebuf (no pages associated with it).
 682  */
 683 xfs_buf_t *
 684 pagebuf_lookup(
 685         xfs_buftarg_t           *target,
 686         loff_t                  ioff,
 687         size_t                  isize,
 688         page_buf_flags_t        flags)
 689 {
 690         xfs_buf_t               *pb;
 691
 692         pb = pagebuf_allocate(flags);
 693         if (pb) {
 694                 _pagebuf_initialize(pb, target, ioff, isize, flags);
 695         }
 696         return pb;
 697 }
 698
 699 /*
 700  * If we are not low on memory then do the readahead in a deadlock
 701  * safe manner.
 702  */
 703 void
 704 pagebuf_readahead(
 705         xfs_buftarg_t           *target,
 706         loff_t                  ioff,
 707         size_t                  isize,
 708         page_buf_flags_t        flags)
 709 {
 710         struct backing_dev_info *bdi;
 711
 712         bdi = target->pbr_mapping->backing_dev_info;
 713         if (bdi_read_congested(bdi))
 714                 return;
 715         if (bdi_write_congested(bdi))
 716                 return;
 717
 718         flags |= (PBF_TRYLOCK|PBF_ASYNC|PBF_READ_AHEAD);
 719         xfs_buf_read_flags(target, ioff, isize, flags);
 720 }
 721
 722 xfs_buf_t *
 723 pagebuf_get_empty(
 724         size_t                  len,
 725         xfs_buftarg_t           *target)
 726 {
 727         xfs_buf_t               *pb;
 728
 729         pb = pagebuf_allocate(0);
 730         if (pb)
 731                 _pagebuf_initialize(pb, target, 0, len, 0);
 732         return pb;
 733 }
 734
 735 static inline struct page *
 736 mem_to_page(
 737         void                    *addr)
 738 {
 739         if (((unsigned long)addr < VMALLOC_START) ||
 740             ((unsigned long)addr >= VMALLOC_END)) {
 741                 return virt_to_page(addr);
 742         } else {
 743                 return vmalloc_to_page(addr);
 744         }
 745 }
 746
 747 int
 748 pagebuf_associate_memory(
 749         xfs_buf_t               *pb,
 750         void                    *mem,
 751         size_t                  len)
 752 {
 753         int                     rval;
 754         int                     i = 0;
 755         size_t                  ptr;
 756         size_t                  end, end_cur;
 757         off_t                   offset;
 758         int                     page_count;
 759
 760         page_count = PAGE_CACHE_ALIGN(len) >> PAGE_CACHE_SHIFT;
 761         offset = (off_t) mem - ((off_t)mem & PAGE_CACHE_MASK);
 762         if (offset && (len > PAGE_CACHE_SIZE))
 763                 page_count++;
 764
 765         /* Free any previous set of page pointers */
 766         if (pb->pb_pages)
 767                 _pagebuf_free_pages(pb);
 768
 769         pb->pb_pages = NULL;
 770         pb->pb_addr = mem;
 771
 772         rval = _pagebuf_get_pages(pb, page_count, 0);
 773         if (rval)
 774                 return rval;
 775
 776         pb->pb_offset = offset;
 777         ptr = (size_t) mem & PAGE_CACHE_MASK;
 778         end = PAGE_CACHE_ALIGN((size_t) mem + len);
 779         end_cur = end;
 780         /* set up first page */
 781         pb->pb_pages[0] = mem_to_page(mem);
 782
 783         ptr += PAGE_CACHE_SIZE;
 784         pb->pb_page_count = ++i;
 785         while (ptr < end) {
 786                 pb->pb_pages[i] = mem_to_page((void *)ptr);
 787                 pb->pb_page_count = ++i;
 788                 ptr += PAGE_CACHE_SIZE;
 789         }
 790         pb->pb_locked = 0;
 791
 792         pb->pb_count_desired = pb->pb_buffer_length = len;
 793         pb->pb_flags |= PBF_MAPPED;
 794
 795         return 0;
 796 }
 797
 798 xfs_buf_t *
 799 pagebuf_get_no_daddr(
 800         size_t                  len,
 801         xfs_buftarg_t           *target)
 802 {
 803         size_t                  malloc_len = len;
 804         xfs_buf_t               *bp;
 805         void                    *data;
 806         int                     error;
 807
 808         bp = pagebuf_allocate(0);
 809         if (unlikely(bp == NULL))
 810                 goto fail;
 811         _pagebuf_initialize(bp, target, 0, len, PBF_FORCEIO);
 812
 813  try_again:
 814         data = kmem_alloc(malloc_len, KM_SLEEP | KM_MAYFAIL);
 815         if (unlikely(data == NULL))
 816                 goto fail_free_buf;
 817
 818         /* check whether alignment matches.. */
 819         if ((__psunsigned_t)data !=
 820             ((__psunsigned_t)data & ~target->pbr_smask)) {
 821                 /* .. else double the size and try again */
 822                 kmem_free(data, malloc_len);
 823                 malloc_len <<= 1;
 824                 goto try_again;
 825         }
 826
 827         error = pagebuf_associate_memory(bp, data, len);
 828         if (error)
 829                 goto fail_free_mem;
 830         bp->pb_flags |= _PBF_KMEM_ALLOC;
 831
 832         pagebuf_unlock(bp);
 833
 834         PB_TRACE(bp, "no_daddr", data);
 835         return bp;
 836  fail_free_mem:
 837         kmem_free(data, malloc_len);
 838  fail_free_buf:
 839         pagebuf_free(bp);
 840  fail:
 841         return NULL;
 842 }
 843
 844 /*
 845  *      pagebuf_hold
 846  *
 847  *      Increment reference count on buffer, to hold the buffer concurrently
 848  *      with another thread which may release (free) the buffer asynchronously.
 849  *
 850  *      Must hold the buffer already to call this function.
 851  */
 852 void
 853 pagebuf_hold(
 854         xfs_buf_t               *pb)
 855 {
 856         atomic_inc(&pb->pb_hold);
 857         PB_TRACE(pb, "hold", 0);
 858 }
 859
 860 /*
 861  *      pagebuf_rele
 862  *
 863  *      pagebuf_rele releases a hold on the specified buffer.  If the
 864  *      the hold count is 1, pagebuf_rele calls pagebuf_free.
 865  */
 866 void
 867 pagebuf_rele(
 868         xfs_buf_t               *pb)
 869 {
 870         pb_hash_t               *hash = pb_hash(pb);
 871
 872         PB_TRACE(pb, "rele", pb->pb_relse);
 873
 874         if (atomic_dec_and_lock(&pb->pb_hold, &hash->pb_hash_lock)) {
 875                 int             do_free = 1;
 876
 877                 if (pb->pb_relse) {
 878                         atomic_inc(&pb->pb_hold);
 879                         spin_unlock(&hash->pb_hash_lock);
 880                         (*(pb->pb_relse)) (pb);
 881                         spin_lock(&hash->pb_hash_lock);
 882                         do_free = 0;
 883                 }
 884
 885                 if (pb->pb_flags & PBF_DELWRI) {
 886                         pb->pb_flags |= PBF_ASYNC;
 887                         atomic_inc(&pb->pb_hold);
 888                         pagebuf_delwri_queue(pb, 0);
 889                         do_free = 0;
 890                 } else if (pb->pb_flags & PBF_FS_MANAGED) {
 891                         do_free = 0;
 892                 }
 893
 894                 if (do_free) {
 895                         list_del_init(&pb->pb_hash_list);
 896                         spin_unlock(&hash->pb_hash_lock);
 897                         pagebuf_free(pb);
 898                 } else {
 899                         spin_unlock(&hash->pb_hash_lock);
 900                 }
 901         }
 902 }
 903
 904
 905 /*
 906  *      Mutual exclusion on buffers.  Locking model:
 907  *
 908  *      Buffers associated with inodes for which buffer locking
 909  *      is not enabled are not protected by semaphores, and are
 910  *      assumed to be exclusively owned by the caller.  There is a
 911  *      spinlock in the buffer, used by the caller when concurrent
 912  *      access is possible.
 913  */
 914
 915 /*
 916  *      pagebuf_cond_lock
 917  *
 918  *      pagebuf_cond_lock locks a buffer object, if it is not already locked.
 919  *      Note that this in no way
 920  *      locks the underlying pages, so it is only useful for synchronizing
 921  *      concurrent use of page buffer objects, not for synchronizing independent
 922  *      access to the underlying pages.
 923  */
 924 int
 925 pagebuf_cond_lock(                      /* lock buffer, if not locked   */
 926                                         /* returns -EBUSY if locked)    */
 927         xfs_buf_t               *pb)
 928 {
 929         int                     locked;
 930
 931         locked = down_trylock(&pb->pb_sema) == 0;
 932         if (locked) {
 933                 PB_SET_OWNER(pb);
 934         }
 935         PB_TRACE(pb, "cond_lock", (long)locked);
 936         return(locked ? 0 : -EBUSY);
 937 }
 938
 939 /*
 940  *      pagebuf_lock_value
 941  *
 942  *      Return lock value for a pagebuf
 943  */
 944 int
 945 pagebuf_lock_value(
 946         xfs_buf_t               *pb)
 947 {
 948         return(atomic_read(&pb->pb_sema.count));
 949 }
 950
 951 /*
 952  *      pagebuf_lock
 953  *
 954  *      pagebuf_lock locks a buffer object.  Note that this in no way
 955  *      locks the underlying pages, so it is only useful for synchronizing
 956  *      concurrent use of page buffer objects, not for synchronizing independent
 957  *      access to the underlying pages.
 958  */
 959 int
 960 pagebuf_lock(
 961         xfs_buf_t               *pb)
 962 {
 963         PB_TRACE(pb, "lock", 0);
 964         if (atomic_read(&pb->pb_io_remaining))
 965                 blk_run_address_space(pb->pb_target->pbr_mapping);
 966         down(&pb->pb_sema);
 967         PB_SET_OWNER(pb);
 968         PB_TRACE(pb, "locked", 0);
 969         return 0;
 970 }
 971
 972 /*
 973  *      pagebuf_unlock
 974  *
 975  *      pagebuf_unlock releases the lock on the buffer object created by
 976  *      pagebuf_lock or pagebuf_cond_lock (not any
 977  *      pinning of underlying pages created by pagebuf_pin).
 978  */
 979 void
 980 pagebuf_unlock(                         /* unlock buffer                */
 981         xfs_buf_t               *pb)    /* buffer to unlock             */
 982 {
 983         PB_CLEAR_OWNER(pb);
 984         up(&pb->pb_sema);
 985         PB_TRACE(pb, "unlock", 0);
 986 }
 987
 988
 989 /*
 990  *      Pinning Buffer Storage in Memory
 991  */
 992
 993 /*
 994  *      pagebuf_pin
 995  *
 996  *      pagebuf_pin locks all of the memory represented by a buffer in
 997  *      memory.  Multiple calls to pagebuf_pin and pagebuf_unpin, for
 998  *      the same or different buffers affecting a given page, will
 999  *      properly count the number of outstanding "pin" requests.  The
1000  *      buffer may be released after the pagebuf_pin and a different
1001  *      buffer used when calling pagebuf_unpin, if desired.
1002  *      pagebuf_pin should be used by the file system when it wants be
1003  *      assured that no attempt will be made to force the affected
1004  *      memory to disk.  It does not assure that a given logical page
1005  *      will not be moved to a different physical page.
1006  */
1007 void
1008 pagebuf_pin(
1009         xfs_buf_t               *pb)
1010 {
1011         atomic_inc(&pb->pb_pin_count);
1012         PB_TRACE(pb, "pin", (long)pb->pb_pin_count.counter);
1013 }
1014
1015 /*
1016  *      pagebuf_unpin
1017  *
1018  *      pagebuf_unpin reverses the locking of memory performed by
1019  *      pagebuf_pin.  Note that both functions affected the logical
1020  *      pages associated with the buffer, not the buffer itself.
1021  */
1022 void
1023 pagebuf_unpin(
1024         xfs_buf_t               *pb)
1025 {
1026         if (atomic_dec_and_test(&pb->pb_pin_count)) {
1027                 wake_up_all(&pb->pb_waiters);
1028         }
1029         PB_TRACE(pb, "unpin", (long)pb->pb_pin_count.counter);
1030 }
1031
1032 int
1033 pagebuf_ispin(
1034         xfs_buf_t               *pb)
1035 {
1036         return atomic_read(&pb->pb_pin_count);
1037 }
1038
1039 /*
1040  *      pagebuf_wait_unpin
1041  *
1042  *      pagebuf_wait_unpin waits until all of the memory associated
1043  *      with the buffer is not longer locked in memory.  It returns
1044  *      immediately if none of the affected pages are locked.
1045  */
1046 static inline void
1047 _pagebuf_wait_unpin(
1048         xfs_buf_t               *pb)
1049 {
1050         DECLARE_WAITQUEUE       (wait, current);
1051
1052         if (atomic_read(&pb->pb_pin_count) == 0)
1053                 return;
1054
1055         add_wait_queue(&pb->pb_waiters, &wait);
1056         for (;;) {
1057                 set_current_state(TASK_UNINTERRUPTIBLE);
1058                 if (atomic_read(&pb->pb_pin_count) == 0)
1059                         break;
1060                 if (atomic_read(&pb->pb_io_remaining))
1061                         blk_run_address_space(pb->pb_target->pbr_mapping);
1062                 schedule();
1063         }
1064         remove_wait_queue(&pb->pb_waiters, &wait);
1065         set_current_state(TASK_RUNNING);
1066 }
1067
1068 /*
1069  *      Buffer Utility Routines
1070  */
1071
1072 /*
1073  *      pagebuf_iodone
1074  *
1075  *      pagebuf_iodone marks a buffer for which I/O is in progress
1076  *      done with respect to that I/O.  The pb_iodone routine, if
1077  *      present, will be called as a side-effect.
1078  */
1079 STATIC void
1080 pagebuf_iodone_work(
1081         void                    *v)
1082 {
1083         xfs_buf_t               *bp = (xfs_buf_t *)v;
1084
1085         if (bp->pb_iodone)
1086                 (*(bp->pb_iodone))(bp);
1087         else if (bp->pb_flags & PBF_ASYNC)
1088                 xfs_buf_relse(bp);
1089 }
1090
1091 void
1092 pagebuf_iodone(
1093         xfs_buf_t               *pb,
1094         int                     dataio,
1095         int                     schedule)
1096 {
1097         pb->pb_flags &= ~(PBF_READ | PBF_WRITE);
1098         if (pb->pb_error == 0) {
1099                 pb->pb_flags &= ~(PBF_PARTIAL | PBF_NONE);
1100         }
1101
1102         PB_TRACE(pb, "iodone", pb->pb_iodone);
1103
1104         if ((pb->pb_iodone) || (pb->pb_flags & PBF_ASYNC)) {
1105                 if (schedule) {
1106                         INIT_WORK(&pb->pb_iodone_work, pagebuf_iodone_work, pb);
1107                         queue_work(dataio ? pagebuf_dataio_workqueue :
1108                                 pagebuf_logio_workqueue, &pb->pb_iodone_work);
1109                 } else {
1110                         pagebuf_iodone_work(pb);
1111                 }
1112         } else {
1113                 up(&pb->pb_iodonesema);
1114         }
1115 }
1116
1117 /*
1118  *      pagebuf_ioerror
1119  *
1120  *      pagebuf_ioerror sets the error code for a buffer.
1121  */
1122 void
1123 pagebuf_ioerror(                        /* mark/clear buffer error flag */
1124         xfs_buf_t               *pb,    /* buffer to mark               */
1125         int                     error)  /* error to store (0 if none)   */
1126 {
1127         ASSERT(error >= 0 && error <= 0xffff);
1128         pb->pb_error = (unsigned short)error;
1129         PB_TRACE(pb, "ioerror", (unsigned long)error);
1130 }
1131
1132 /*
1133  *      pagebuf_iostart
1134  *
1135  *      pagebuf_iostart initiates I/O on a buffer, based on the flags supplied.
1136  *      If necessary, it will arrange for any disk space allocation required,
1137  *      and it will break up the request if the block mappings require it.
1138  *      The pb_iodone routine in the buffer supplied will only be called
1139  *      when all of the subsidiary I/O requests, if any, have been completed.
1140  *      pagebuf_iostart calls the pagebuf_ioinitiate routine or
1141  *      pagebuf_iorequest, if the former routine is not defined, to start
1142  *      the I/O on a given low-level request.
1143  */
1144 int
1145 pagebuf_iostart(                        /* start I/O on a buffer          */
1146         xfs_buf_t               *pb,    /* buffer to start                */
1147         page_buf_flags_t        flags)  /* PBF_LOCK, PBF_ASYNC, PBF_READ, */
1148                                         /* PBF_WRITE, PBF_DELWRI,         */
1149                                         /* PBF_DONT_BLOCK                 */
1150 {
1151         int                     status = 0;
1152
1153         PB_TRACE(pb, "iostart", (unsigned long)flags);
1154
1155         if (flags & PBF_DELWRI) {
1156                 pb->pb_flags &= ~(PBF_READ | PBF_WRITE | PBF_ASYNC);
1157                 pb->pb_flags |= flags & (PBF_DELWRI | PBF_ASYNC);
1158                 pagebuf_delwri_queue(pb, 1);
1159                 return status;
1160         }
1161
1162         pb->pb_flags &= ~(PBF_READ | PBF_WRITE | PBF_ASYNC | PBF_DELWRI | \
1163                         PBF_READ_AHEAD | _PBF_RUN_QUEUES);
1164         pb->pb_flags |= flags & (PBF_READ | PBF_WRITE | PBF_ASYNC | \
1165                         PBF_READ_AHEAD | _PBF_RUN_QUEUES);
1166
1167         BUG_ON(pb->pb_bn == XFS_BUF_DADDR_NULL);
1168
1169         /* For writes allow an alternate strategy routine to precede
1170          * the actual I/O request (which may not be issued at all in
1171          * a shutdown situation, for example).
1172          */
1173         status = (flags & PBF_WRITE) ?
1174                 pagebuf_iostrategy(pb) : pagebuf_iorequest(pb);
1175
1176         /* Wait for I/O if we are not an async request.
1177          * Note: async I/O request completion will release the buffer,
1178          * and that can already be done by this point.  So using the
1179          * buffer pointer from here on, after async I/O, is invalid.
1180          */
1181         if (!status && !(flags & PBF_ASYNC))
1182                 status = pagebuf_iowait(pb);
1183
1184         return status;
1185 }
1186
1187 /*
1188  * Helper routine for pagebuf_iorequest
1189  */
1190
1191 STATIC __inline__ int
1192 _pagebuf_iolocked(
1193         xfs_buf_t               *pb)
1194 {
1195         ASSERT(pb->pb_flags & (PBF_READ|PBF_WRITE));
1196         if (pb->pb_flags & PBF_READ)
1197                 return pb->pb_locked;
1198         return 0;
1199 }
1200
1201 STATIC __inline__ void
1202 _pagebuf_iodone(
1203         xfs_buf_t               *pb,
1204         int                     schedule)
1205 {
1206         if (atomic_dec_and_test(&pb->pb_io_remaining) == 1) {
1207                 pb->pb_locked = 0;
1208                 pagebuf_iodone(pb, (pb->pb_flags & PBF_FS_DATAIOD), schedule);
1209         }
1210 }
1211
1212 STATIC int
1213 bio_end_io_pagebuf(
1214         struct bio              *bio,
1215         unsigned int            bytes_done,
1216         int                     error)
1217 {
1218         xfs_buf_t               *pb = (xfs_buf_t *)bio->bi_private;
1219         unsigned int            i, blocksize = pb->pb_target->pbr_bsize;
1220         unsigned int            sectorshift = pb->pb_target->pbr_sshift;
1221         struct bio_vec          *bvec = bio->bi_io_vec;
1222
1223         if (bio->bi_size)
1224                 return 1;
1225
1226         if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
1227                 pb->pb_error = EIO;
1228
1229         for (i = 0; i < bio->bi_vcnt; i++, bvec++) {
1230                 struct page     *page = bvec->bv_page;
1231
1232                 if (pb->pb_error) {
1233                         SetPageError(page);
1234                 } else if (blocksize == PAGE_CACHE_SIZE) {
1235                         SetPageUptodate(page);
1236                 } else if (!PagePrivate(page) &&
1237                                 (pb->pb_flags & _PBF_PAGE_CACHE)) {
1238                         unsigned long   j, range;
1239
1240                         ASSERT(blocksize < PAGE_CACHE_SIZE);
1241                         range = (bvec->bv_offset + bvec->bv_len) >> sectorshift;
1242                         for (j = bvec->bv_offset >> sectorshift; j < range; j++)
1243                                 set_bit(j, &page->private);
1244                         if (page->private == (unsigned long)(PAGE_CACHE_SIZE-1))
1245                                 SetPageUptodate(page);
1246                 }
1247
1248                 if (_pagebuf_iolocked(pb)) {
1249                         unlock_page(page);
1250                 }
1251         }
1252
1253         _pagebuf_iodone(pb, 1);
1254         bio_put(bio);
1255         return 0;
1256 }
1257
1258 STATIC void
1259 _pagebuf_ioapply(
1260         xfs_buf_t               *pb)
1261 {
1262         int                     i, map_i, total_nr_pages, nr_pages;
1263         struct bio              *bio;
1264         int                     offset = pb->pb_offset;
1265         int                     size = pb->pb_count_desired;
1266         sector_t                sector = pb->pb_bn;
1267         unsigned int            blocksize = pb->pb_target->pbr_bsize;
1268         int                     locking = _pagebuf_iolocked(pb);
1269
1270         total_nr_pages = pb->pb_page_count;
1271         map_i = 0;
1272
1273         /* Special code path for reading a sub page size pagebuf in --
1274          * we populate up the whole page, and hence the other metadata
1275          * in the same page.  This optimization is only valid when the
1276          * filesystem block size and the page size are equal.
1277          */
1278         if ((pb->pb_buffer_length < PAGE_CACHE_SIZE) &&
1279             (pb->pb_flags & PBF_READ) && locking &&
1280             (blocksize == PAGE_CACHE_SIZE)) {
1281                 bio = bio_alloc(GFP_NOIO, 1);
1282
1283                 bio->bi_bdev = pb->pb_target->pbr_bdev;
1284                 bio->bi_sector = sector - (offset >> BBSHIFT);
1285                 bio->bi_end_io = bio_end_io_pagebuf;
1286                 bio->bi_private = pb;
1287
1288                 bio_add_page(bio, pb->pb_pages[0], PAGE_CACHE_SIZE, 0);
1289                 size = 0;
1290
1291                 atomic_inc(&pb->pb_io_remaining);
1292
1293                 goto submit_io;
1294         }
1295
1296         /* Lock down the pages which we need to for the request */
1297         if (locking && (pb->pb_flags & PBF_WRITE) && (pb->pb_locked == 0)) {
1298                 for (i = 0; size; i++) {
1299                         int             nbytes = PAGE_CACHE_SIZE - offset;
1300                         struct page     *page = pb->pb_pages[i];
1301
1302                         if (nbytes > size)
1303                                 nbytes = size;
1304
1305                         lock_page(page);
1306
1307                         size -= nbytes;
1308                         offset = 0;
1309                 }
1310                 offset = pb->pb_offset;
1311                 size = pb->pb_count_desired;
1312         }
1313
1314 next_chunk:
1315         atomic_inc(&pb->pb_io_remaining);
1316         nr_pages = BIO_MAX_SECTORS >> (PAGE_SHIFT - BBSHIFT);
1317         if (nr_pages > total_nr_pages)
1318                 nr_pages = total_nr_pages;
1319
1320         bio = bio_alloc(GFP_NOIO, nr_pages);
1321         bio->bi_bdev = pb->pb_target->pbr_bdev;
1322         bio->bi_sector = sector;
1323         bio->bi_end_io = bio_end_io_pagebuf;
1324         bio->bi_private = pb;
1325
1326         for (; size && nr_pages; nr_pages--, map_i++) {
1327                 int     nbytes = PAGE_CACHE_SIZE - offset;
1328
1329                 if (nbytes > size)
1330                         nbytes = size;
1331
1332                 if (bio_add_page(bio, pb->pb_pages[map_i],
1333                                         nbytes, offset) < nbytes)
1334                         break;
1335
1336                 offset = 0;
1337                 sector += nbytes >> BBSHIFT;
1338                 size -= nbytes;
1339                 total_nr_pages--;
1340         }
1341
1342 submit_io:
1343         if (likely(bio->bi_size)) {
1344                 submit_bio((pb->pb_flags & PBF_READ) ? READ : WRITE, bio);
1345                 if (size)
1346                         goto next_chunk;
1347         } else {
1348                 bio_put(bio);
1349                 pagebuf_ioerror(pb, EIO);
1350         }
1351
1352         if (pb->pb_flags & _PBF_RUN_QUEUES) {
1353                 pb->pb_flags &= ~_PBF_RUN_QUEUES;
1354                 if (atomic_read(&pb->pb_io_remaining) > 1)
1355                         blk_run_address_space(pb->pb_target->pbr_mapping);
1356         }
1357 }
1358
1359 /*
1360  *      pagebuf_iorequest -- the core I/O request routine.
1361  */
1362 int
1363 pagebuf_iorequest(                      /* start real I/O               */
1364         xfs_buf_t               *pb)    /* buffer to convey to device   */
1365 {
1366         PB_TRACE(pb, "iorequest", 0);
1367
1368         if (pb->pb_flags & PBF_DELWRI) {
1369                 pagebuf_delwri_queue(pb, 1);
1370                 return 0;
1371         }
1372
1373         if (pb->pb_flags & PBF_WRITE) {
1374                 _pagebuf_wait_unpin(pb);
1375         }
1376
1377         pagebuf_hold(pb);
1378
1379         /* Set the count to 1 initially, this will stop an I/O
1380          * completion callout which happens before we have started
1381          * all the I/O from calling pagebuf_iodone too early.
1382          */
1383         atomic_set(&pb->pb_io_remaining, 1);
1384         _pagebuf_ioapply(pb);
1385         _pagebuf_iodone(pb, 0);
1386
1387         pagebuf_rele(pb);
1388         return 0;
1389 }
1390
1391 /*
1392  *      pagebuf_iowait
1393  *
1394  *      pagebuf_iowait waits for I/O to complete on the buffer supplied.
1395  *      It returns immediately if no I/O is pending.  In any case, it returns
1396  *      the error code, if any, or 0 if there is no error.
1397  */
1398 int
1399 pagebuf_iowait(
1400         xfs_buf_t               *pb)
1401 {
1402         PB_TRACE(pb, "iowait", 0);
1403         if (atomic_read(&pb->pb_io_remaining))
1404                 blk_run_address_space(pb->pb_target->pbr_mapping);
1405         down(&pb->pb_iodonesema);
1406         PB_TRACE(pb, "iowaited", (long)pb->pb_error);
1407         return pb->pb_error;
1408 }
1409
1410 caddr_t
1411 pagebuf_offset(
1412         xfs_buf_t               *pb,
1413         size_t                  offset)
1414 {
1415         struct page             *page;
1416
1417         offset += pb->pb_offset;
1418
1419         page = pb->pb_pages[offset >> PAGE_CACHE_SHIFT];
1420         return (caddr_t) page_address(page) + (offset & (PAGE_CACHE_SIZE - 1));
1421 }
1422
1423 /*
1424  *      pagebuf_iomove
1425  *
1426  *      Move data into or out of a buffer.
1427  */
1428 void
1429 pagebuf_iomove(
1430         xfs_buf_t               *pb,    /* buffer to process            */
1431         size_t                  boff,   /* starting buffer offset       */
1432         size_t                  bsize,  /* length to copy               */
1433         caddr_t                 data,   /* data address                 */
1434         page_buf_rw_t           mode)   /* read/write flag              */
1435 {
1436         size_t                  bend, cpoff, csize;
1437         struct page             *page;
1438
1439         bend = boff + bsize;
1440         while (boff < bend) {
1441                 page = pb->pb_pages[page_buf_btoct(boff + pb->pb_offset)];
1442                 cpoff = page_buf_poff(boff + pb->pb_offset);
1443                 csize = min_t(size_t,
1444                               PAGE_CACHE_SIZE-cpoff, pb->pb_count_desired-boff);
1445
1446                 ASSERT(((csize + cpoff) <= PAGE_CACHE_SIZE));
1447
1448                 switch (mode) {
1449                 case PBRW_ZERO:
1450                         memset(page_address(page) + cpoff, 0, csize);
1451                         break;
1452                 case PBRW_READ:
1453                         memcpy(data, page_address(page) + cpoff, csize);
1454                         break;
1455                 case PBRW_WRITE:
1456                         memcpy(page_address(page) + cpoff, data, csize);
1457                 }
1458
1459                 boff += csize;
1460                 data += csize;
1461         }
1462 }
1463
1464 /*
1465  *      Handling of buftargs.
1466  */
1467
1468 /*
1469  * Wait for any bufs with callbacks that have been submitted but
1470  * have not yet returned... walk the hash list for the target.
1471  */
1472 void
1473 xfs_wait_buftarg(
1474         xfs_buftarg_t *target)
1475 {
1476         xfs_buf_t       *pb, *n;
1477         pb_hash_t       *h;
1478         int             i;
1479
1480         for (i = 0; i < NHASH; i++) {
1481                 h = &pbhash[i];
1482 again:
1483                 spin_lock(&h->pb_hash_lock);
1484                 list_for_each_entry_safe(pb, n, &h->pb_hash, pb_hash_list) {
1485                         if (pb->pb_target == target &&
1486                                         !(pb->pb_flags & PBF_FS_MANAGED)) {
1487                                 spin_unlock(&h->pb_hash_lock);
1488                                 delay(100);
1489                                 goto again;
1490                         }
1491                 }
1492                 spin_unlock(&h->pb_hash_lock);
1493         }
1494 }
1495
1496 void
1497 xfs_free_buftarg(
1498         xfs_buftarg_t           *btp,
1499         int                     external)
1500 {
1501         xfs_flush_buftarg(btp, 1);
1502         if (external)
1503                 xfs_blkdev_put(btp->pbr_bdev);
1504         iput(btp->pbr_mapping->host);
1505         kmem_free(btp, sizeof(*btp));
1506 }
1507
1508 void
1509 xfs_incore_relse(
1510         xfs_buftarg_t           *btp,
1511         int                     delwri_only,
1512         int                     wait)
1513 {
1514         invalidate_bdev(btp->pbr_bdev, 1);
1515         truncate_inode_pages(btp->pbr_mapping, 0LL);
1516 }
1517
1518 int
1519 xfs_setsize_buftarg(
1520         xfs_buftarg_t           *btp,
1521         unsigned int            blocksize,
1522         unsigned int            sectorsize)
1523 {
1524         btp->pbr_bsize = blocksize;
1525         btp->pbr_sshift = ffs(sectorsize) - 1;
1526         btp->pbr_smask = sectorsize - 1;
1527
1528         if (set_blocksize(btp->pbr_bdev, sectorsize)) {
1529                 printk(KERN_WARNING
1530                         "XFS: Cannot set_blocksize to %u on device %s\n",
1531                         sectorsize, XFS_BUFTARG_NAME(btp));
1532                 return EINVAL;
1533         }
1534         return 0;
1535 }
1536
1537 STATIC int
1538 xfs_mapping_buftarg(
1539         xfs_buftarg_t           *btp,
1540         struct block_device     *bdev)
1541 {
1542         struct backing_dev_info *bdi;
1543         struct inode            *inode;
1544         struct address_space    *mapping;
1545         static struct address_space_operations mapping_aops = {
1546                 .sync_page = block_sync_page,
1547         };
1548
1549         inode = new_inode(bdev->bd_inode->i_sb);
1550         if (!inode) {
1551                 printk(KERN_WARNING
1552                         "XFS: Cannot allocate mapping inode for device %s\n",
1553                         XFS_BUFTARG_NAME(btp));
1554                 return ENOMEM;
1555         }
1556         inode->i_mode = S_IFBLK;
1557         inode->i_bdev = bdev;
1558         inode->i_rdev = bdev->bd_dev;
1559         bdi = blk_get_backing_dev_info(bdev);
1560         if (!bdi)
1561                 bdi = &default_backing_dev_info;
1562         mapping = &inode->i_data;
1563         mapping->a_ops = &mapping_aops;
1564         mapping->backing_dev_info = bdi;
1565         mapping_set_gfp_mask(mapping, GFP_KERNEL);
1566         btp->pbr_mapping = mapping;
1567         return 0;
1568 }
1569
1570 xfs_buftarg_t *
1571 xfs_alloc_buftarg(
1572         struct block_device     *bdev)
1573 {
1574         xfs_buftarg_t           *btp;
1575
1576         btp = kmem_zalloc(sizeof(*btp), KM_SLEEP);
1577
1578         btp->pbr_dev =  bdev->bd_dev;
1579         btp->pbr_bdev = bdev;
1580         if (xfs_setsize_buftarg(btp, PAGE_CACHE_SIZE, bdev_hardsect_size(bdev)))
1581                 goto error;
1582         if (xfs_mapping_buftarg(btp, bdev))
1583                 goto error;
1584         return btp;
1585
1586 error:
1587         kmem_free(btp, sizeof(*btp));
1588         return NULL;
1589 }
1590
1591
1592 /*
1593  * Pagebuf delayed write buffer handling
1594  */
1595
1596 STATIC LIST_HEAD(pbd_delwrite_queue);
1597 STATIC spinlock_t pbd_delwrite_lock = SPIN_LOCK_UNLOCKED;
1598
1599 STATIC void
1600 pagebuf_delwri_queue(
1601         xfs_buf_t               *pb,
1602         int                     unlock)
1603 {
1604         PB_TRACE(pb, "delwri_q", (long)unlock);
1605         ASSERT(pb->pb_flags & PBF_DELWRI);
1606
1607         spin_lock(&pbd_delwrite_lock);
1608         /* If already in the queue, dequeue and place at tail */
1609         if (!list_empty(&pb->pb_list)) {
1610                 if (unlock) {
1611                         atomic_dec(&pb->pb_hold);
1612                 }
1613                 list_del(&pb->pb_list);
1614         }
1615
1616         list_add_tail(&pb->pb_list, &pbd_delwrite_queue);
1617         pb->pb_queuetime = jiffies;
1618         spin_unlock(&pbd_delwrite_lock);
1619
1620         if (unlock)
1621                 pagebuf_unlock(pb);
1622 }
1623
1624 void
1625 pagebuf_delwri_dequeue(
1626         xfs_buf_t               *pb)
1627 {
1628         int                     dequeued = 0;
1629
1630         spin_lock(&pbd_delwrite_lock);
1631         if ((pb->pb_flags & PBF_DELWRI) && !list_empty(&pb->pb_list)) {
1632                 list_del_init(&pb->pb_list);
1633                 dequeued = 1;
1634         }
1635         pb->pb_flags &= ~PBF_DELWRI;
1636         spin_unlock(&pbd_delwrite_lock);
1637
1638         if (dequeued)
1639                 pagebuf_rele(pb);
1640
1641         PB_TRACE(pb, "delwri_dq", (long)dequeued);
1642 }
1643
1644 STATIC void
1645 pagebuf_runall_queues(
1646         struct workqueue_struct *queue)
1647 {
1648         flush_workqueue(queue);
1649 }
1650
1651 /* Defines for pagebuf daemon */
1652 STATIC DECLARE_COMPLETION(pagebuf_daemon_done);
1653 STATIC struct task_struct *pagebuf_daemon_task;
1654 STATIC int pagebuf_daemon_active;
1655 STATIC int force_flush;
1656
1657
1658 STATIC int
1659 pagebuf_daemon_wakeup(
1660         int                     priority,
1661         unsigned int            mask)
1662 {
1663         force_flush = 1;
1664         barrier();
1665         wake_up_process(pagebuf_daemon_task);
1666         return 0;
1667 }
1668
1669 STATIC int
1670 pagebuf_daemon(
1671         void                    *data)
1672 {
1673         struct list_head        tmp;
1674         unsigned long           age;
1675         xfs_buftarg_t           *target;
1676         xfs_buf_t               *pb, *n;
1677
1678         /*  Set up the thread  */
1679         daemonize("xfsbufd");
1680         current->flags |= PF_MEMALLOC;
1681
1682         pagebuf_daemon_task = current;
1683         pagebuf_daemon_active = 1;
1684         barrier();
1685
1686         INIT_LIST_HEAD(&tmp);
1687         do {
1688                 /* swsusp */
1689                 if (current->flags & PF_FREEZE)
1690                         refrigerator(PF_FREEZE);
1691
1692                 set_current_state(TASK_INTERRUPTIBLE);
1693                 schedule_timeout((xfs_buf_timer_centisecs * HZ) / 100);
1694
1695                 age = (xfs_buf_age_centisecs * HZ) / 100;
1696                 spin_lock(&pbd_delwrite_lock);
1697                 list_for_each_entry_safe(pb, n, &pbd_delwrite_queue, pb_list) {
1698                         PB_TRACE(pb, "walkq1", (long)pagebuf_ispin(pb));
1699                         ASSERT(pb->pb_flags & PBF_DELWRI);
1700
1701                         if (!pagebuf_ispin(pb) && !pagebuf_cond_lock(pb)) {
1702                                 if (!force_flush &&
1703                                     time_before(jiffies,
1704                                                 pb->pb_queuetime + age)) {
1705                                         pagebuf_unlock(pb);
1706                                         break;
1707                                 }
1708
1709                                 pb->pb_flags &= ~PBF_DELWRI;
1710                                 pb->pb_flags |= PBF_WRITE;
1711                                 list_move(&pb->pb_list, &tmp);
1712                         }
1713                 }
1714                 spin_unlock(&pbd_delwrite_lock);
1715
1716                 while (!list_empty(&tmp)) {
1717                         pb = list_entry(tmp.next, xfs_buf_t, pb_list);
1718                         target = pb->pb_target;
1719
1720                         list_del_init(&pb->pb_list);
1721                         pagebuf_iostrategy(pb);
1722
1723                         blk_run_address_space(target->pbr_mapping);
1724                 }
1725
1726                 if (as_list_len > 0)
1727                         purge_addresses();
1728
1729                 force_flush = 0;
1730         } while (pagebuf_daemon_active);
1731
1732         complete_and_exit(&pagebuf_daemon_done, 0);
1733 }
1734
1735 /*
1736  * Go through all incore buffers, and release buffers if they belong to
1737  * the given device. This is used in filesystem error handling to
1738  * preserve the consistency of its metadata.
1739  */
1740 int
1741 xfs_flush_buftarg(
1742         xfs_buftarg_t           *target,
1743         int                     wait)
1744 {
1745         struct list_head        tmp;
1746         xfs_buf_t               *pb, *n;
1747         int                     pincount = 0;
1748
1749         pagebuf_runall_queues(pagebuf_dataio_workqueue);
1750         pagebuf_runall_queues(pagebuf_logio_workqueue);
1751
1752         INIT_LIST_HEAD(&tmp);
1753         spin_lock(&pbd_delwrite_lock);
1754         list_for_each_entry_safe(pb, n, &pbd_delwrite_queue, pb_list) {
1755
1756                 if (pb->pb_target != target)
1757                         continue;
1758
1759                 ASSERT(pb->pb_flags & PBF_DELWRI);
1760                 PB_TRACE(pb, "walkq2", (long)pagebuf_ispin(pb));
1761                 if (pagebuf_ispin(pb)) {
1762                         pincount++;
1763                         continue;
1764                 }
1765
1766                 pb->pb_flags &= ~PBF_DELWRI;
1767                 pb->pb_flags |= PBF_WRITE;
1768                 list_move(&pb->pb_list, &tmp);
1769         }
1770         spin_unlock(&pbd_delwrite_lock);
1771
1772         /*
1773          * Dropped the delayed write list lock, now walk the temporary list
1774          */
1775         list_for_each_entry_safe(pb, n, &tmp, pb_list) {
1776                 if (wait)
1777                         pb->pb_flags &= ~PBF_ASYNC;
1778                 else
1779                         list_del_init(&pb->pb_list);
1780
1781                 pagebuf_lock(pb);
1782                 pagebuf_iostrategy(pb);
1783         }
1784
1785         /*
1786          * Remaining list items must be flushed before returning
1787          */
1788         while (!list_empty(&tmp)) {
1789                 pb = list_entry(tmp.next, xfs_buf_t, pb_list);
1790
1791                 list_del_init(&pb->pb_list);
1792                 xfs_iowait(pb);
1793                 xfs_buf_relse(pb);
1794         }
1795
1796         if (wait)
1797                 blk_run_address_space(target->pbr_mapping);
1798
1799         return pincount;
1800 }
1801
1802 STATIC int
1803 pagebuf_daemon_start(void)
1804 {
1805         int             rval;
1806
1807         pagebuf_logio_workqueue = create_workqueue("xfslogd");
1808         if (!pagebuf_logio_workqueue)
1809                 return -ENOMEM;
1810
1811         pagebuf_dataio_workqueue = create_workqueue("xfsdatad");
1812         if (!pagebuf_dataio_workqueue) {
1813                 destroy_workqueue(pagebuf_logio_workqueue);
1814                 return -ENOMEM;
1815         }
1816
1817         rval = kernel_thread(pagebuf_daemon, NULL, CLONE_FS|CLONE_FILES);
1818         if (rval < 0) {
1819                 destroy_workqueue(pagebuf_logio_workqueue);
1820                 destroy_workqueue(pagebuf_dataio_workqueue);
1821         }
1822
1823         return rval;
1824 }
1825
1826 /*
1827  * pagebuf_daemon_stop
1828  *
1829  * Note: do not mark as __exit, it is called from pagebuf_terminate.
1830  */
1831 STATIC void
1832 pagebuf_daemon_stop(void)
1833 {
1834         pagebuf_daemon_active = 0;
1835         barrier();
1836         wait_for_completion(&pagebuf_daemon_done);
1837
1838         destroy_workqueue(pagebuf_logio_workqueue);
1839         destroy_workqueue(pagebuf_dataio_workqueue);
1840 }
1841
1842 /*
1843  *      Initialization and Termination
1844  */
1845
1846 int __init
1847 pagebuf_init(void)
1848 {
1849         int                     i;
1850
1851         pagebuf_cache = kmem_cache_create("xfs_buf_t", sizeof(xfs_buf_t), 0,
1852                         SLAB_HWCACHE_ALIGN, NULL, NULL);
1853         if (pagebuf_cache == NULL) {
1854                 printk("XFS: couldn't init xfs_buf_t cache\n");
1855                 pagebuf_terminate();
1856                 return -ENOMEM;
1857         }
1858
1859 #ifdef PAGEBUF_TRACE
1860         pagebuf_trace_buf = ktrace_alloc(PAGEBUF_TRACE_SIZE, KM_SLEEP);
1861 #endif
1862
1863         pagebuf_daemon_start();
1864
1865         pagebuf_shake = kmem_shake_register(pagebuf_daemon_wakeup);
1866         if (pagebuf_shake == NULL) {
1867                 pagebuf_terminate();
1868                 return -ENOMEM;
1869         }
1870
1871         for (i = 0; i < NHASH; i++) {
1872                 spin_lock_init(&pbhash[i].pb_hash_lock);
1873                 INIT_LIST_HEAD(&pbhash[i].pb_hash);
1874         }
1875
1876         return 0;
1877 }
1878
1879
1880 /*
1881  *      pagebuf_terminate.
1882  *
1883  *      Note: do not mark as __exit, this is also called from the __init code.
1884  */
1885 void
1886 pagebuf_terminate(void)
1887 {
1888         pagebuf_daemon_stop();
1889
1890 #ifdef PAGEBUF_TRACE
1891         ktrace_free(pagebuf_trace_buf);
1892 #endif
1893
1894         kmem_zone_destroy(pagebuf_cache);
1895         kmem_shake_deregister(pagebuf_shake);
1896 }