fs/xfs/linux-2.6/xfs_aops.c

   1 /*
   2  * Copyright (c) 2000-2005 Silicon Graphics, Inc.  All Rights Reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms of version 2 of the GNU General Public License as
   6  * published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it would be useful, but
   9  * WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
  11  *
  12  * Further, this software is distributed without any warranty that it is
  13  * free of the rightful claim of any third person regarding infringement
  14  * or the like.  Any license provided herein, whether implied or
  15  * otherwise, applies only to this software file.  Patent licenses, if
  16  * any, provided herein do not apply to combinations of this program with
  17  * other software, or any other product whatsoever.
  18  *
  19  * You should have received a copy of the GNU General Public License along
  20  * with this program; if not, write the Free Software Foundation, Inc., 59
  21  * Temple Place - Suite 330, Boston MA 02111-1307, USA.
  22  *
  23  * Contact information: Silicon Graphics, Inc., 1600 Amphitheatre Pkwy,
  24  * Mountain View, CA  94043, or:
  25  *
  26  * http://www.sgi.com
  27  *
  28  * For further information regarding this notice, see:
  29  *
  30  * http://oss.sgi.com/projects/GenInfo/SGIGPLNoticeExplan/
  31  */
  32
  33 #include "xfs.h"
  34 #include "xfs_inum.h"
  35 #include "xfs_log.h"
  36 #include "xfs_sb.h"
  37 #include "xfs_dir.h"
  38 #include "xfs_dir2.h"
  39 #include "xfs_trans.h"
  40 #include "xfs_dmapi.h"
  41 #include "xfs_mount.h"
  42 #include "xfs_bmap_btree.h"
  43 #include "xfs_alloc_btree.h"
  44 #include "xfs_ialloc_btree.h"
  45 #include "xfs_alloc.h"
  46 #include "xfs_btree.h"
  47 #include "xfs_attr_sf.h"
  48 #include "xfs_dir_sf.h"
  49 #include "xfs_dir2_sf.h"
  50 #include "xfs_dinode.h"
  51 #include "xfs_inode.h"
  52 #include "xfs_error.h"
  53 #include "xfs_rw.h"
  54 #include "xfs_iomap.h"
  55 #include <linux/mpage.h>
  56 #include <linux/writeback.h>
  57
  58 STATIC void xfs_count_page_state(struct page *, int *, int *, int *);
  59 STATIC void xfs_convert_page(struct inode *, struct page *, xfs_iomap_t *,
  60                 struct writeback_control *wbc, void *, int, int);
  61
  62 #if defined(XFS_RW_TRACE)
  63 void
  64 xfs_page_trace(
  65         int             tag,
  66         struct inode    *inode,
  67         struct page     *page,
  68         int             mask)
  69 {
  70         xfs_inode_t     *ip;
  71         bhv_desc_t      *bdp;
  72         vnode_t         *vp = LINVFS_GET_VP(inode);
  73         loff_t          isize = i_size_read(inode);
  74         loff_t          offset = (loff_t)page->index << PAGE_CACHE_SHIFT;
  75         int             delalloc = -1, unmapped = -1, unwritten = -1;
  76
  77         if (page_has_buffers(page))
  78                 xfs_count_page_state(page, &delalloc, &unmapped, &unwritten);
  79
  80         bdp = vn_bhv_lookup(VN_BHV_HEAD(vp), &xfs_vnodeops);
  81         ip = XFS_BHVTOI(bdp);
  82         if (!ip->i_rwtrace)
  83                 return;
  84
  85         ktrace_enter(ip->i_rwtrace,
  86                 (void *)((unsigned long)tag),
  87                 (void *)ip,
  88                 (void *)inode,
  89                 (void *)page,
  90                 (void *)((unsigned long)mask),
  91                 (void *)((unsigned long)((ip->i_d.di_size >> 32) & 0xffffffff)),
  92                 (void *)((unsigned long)(ip->i_d.di_size & 0xffffffff)),
  93                 (void *)((unsigned long)((isize >> 32) & 0xffffffff)),
  94                 (void *)((unsigned long)(isize & 0xffffffff)),
  95                 (void *)((unsigned long)((offset >> 32) & 0xffffffff)),
  96                 (void *)((unsigned long)(offset & 0xffffffff)),
  97                 (void *)((unsigned long)delalloc),
  98                 (void *)((unsigned long)unmapped),
  99                 (void *)((unsigned long)unwritten),
 100                 (void *)NULL,
 101                 (void *)NULL);
 102 }
 103 #else
 104 #define xfs_page_trace(tag, inode, page, mask)
 105 #endif
 106
 107 void
 108 linvfs_unwritten_done(
 109         struct buffer_head      *bh,
 110         int                     uptodate)
 111 {
 112         xfs_buf_t               *pb = (xfs_buf_t *)bh->b_private;
 113
 114         ASSERT(buffer_unwritten(bh));
 115         bh->b_end_io = NULL;
 116         clear_buffer_unwritten(bh);
 117         if (!uptodate)
 118                 pagebuf_ioerror(pb, EIO);
 119         if (atomic_dec_and_test(&pb->pb_io_remaining) == 1) {
 120                 pagebuf_iodone(pb, 1, 1);
 121         }
 122         end_buffer_async_write(bh, uptodate);
 123 }
 124
 125 /*
 126  * Issue transactions to convert a buffer range from unwritten
 127  * to written extents (buffered IO).
 128  */
 129 STATIC void
 130 linvfs_unwritten_convert(
 131         xfs_buf_t       *bp)
 132 {
 133         vnode_t         *vp = XFS_BUF_FSPRIVATE(bp, vnode_t *);
 134         int             error;
 135
 136         BUG_ON(atomic_read(&bp->pb_hold) < 1);
 137         VOP_BMAP(vp, XFS_BUF_OFFSET(bp), XFS_BUF_SIZE(bp),
 138                         BMAPI_UNWRITTEN, NULL, NULL, error);
 139         XFS_BUF_SET_FSPRIVATE(bp, NULL);
 140         XFS_BUF_CLR_IODONE_FUNC(bp);
 141         XFS_BUF_UNDATAIO(bp);
 142         iput(LINVFS_GET_IP(vp));
 143         pagebuf_iodone(bp, 0, 0);
 144 }
 145
 146 /*
 147  * Issue transactions to convert a buffer range from unwritten
 148  * to written extents (direct IO).
 149  */
 150 STATIC void
 151 linvfs_unwritten_convert_direct(
 152         struct inode    *inode,
 153         loff_t          offset,
 154         ssize_t         size,
 155         void            *private)
 156 {
 157         ASSERT(!private || inode == (struct inode *)private);
 158
 159         /* private indicates an unwritten extent lay beneath this IO */
 160         if (private && size > 0) {
 161                 vnode_t *vp = LINVFS_GET_VP(inode);
 162                 int     error;
 163
 164                 VOP_BMAP(vp, offset, size, BMAPI_UNWRITTEN, NULL, NULL, error);
 165         }
 166 }
 167
 168 STATIC int
 169 xfs_map_blocks(
 170         struct inode            *inode,
 171         loff_t                  offset,
 172         ssize_t                 count,
 173         xfs_iomap_t             *mapp,
 174         int                     flags)
 175 {
 176         vnode_t                 *vp = LINVFS_GET_VP(inode);
 177         int                     error, nmaps = 1;
 178
 179         VOP_BMAP(vp, offset, count, flags, mapp, &nmaps, error);
 180         if (!error && (flags & (BMAPI_WRITE|BMAPI_ALLOCATE)))
 181                 VMODIFY(vp);
 182         return -error;
 183 }
 184
 185 /*
 186  * Finds the corresponding mapping in block @map array of the
 187  * given @offset within a @page.
 188  */
 189 STATIC xfs_iomap_t *
 190 xfs_offset_to_map(
 191         struct page             *page,
 192         xfs_iomap_t             *iomapp,
 193         unsigned long           offset)
 194 {
 195         loff_t                  full_offset;    /* offset from start of file */
 196
 197         ASSERT(offset < PAGE_CACHE_SIZE);
 198
 199         full_offset = page->index;              /* NB: using 64bit number */
 200         full_offset <<= PAGE_CACHE_SHIFT;       /* offset from file start */
 201         full_offset += offset;                  /* offset from page start */
 202
 203         if (full_offset < iomapp->iomap_offset)
 204                 return NULL;
 205         if (iomapp->iomap_offset + (iomapp->iomap_bsize -1) >= full_offset)
 206                 return iomapp;
 207         return NULL;
 208 }
 209
 210 STATIC void
 211 xfs_map_at_offset(
 212         struct page             *page,
 213         struct buffer_head      *bh,
 214         unsigned long           offset,
 215         int                     block_bits,
 216         xfs_iomap_t             *iomapp)
 217 {
 218         xfs_daddr_t             bn;
 219         loff_t                  delta;
 220         int                     sector_shift;
 221
 222         ASSERT(!(iomapp->iomap_flags & IOMAP_HOLE));
 223         ASSERT(!(iomapp->iomap_flags & IOMAP_DELAY));
 224         ASSERT(iomapp->iomap_bn != IOMAP_DADDR_NULL);
 225
 226         delta = page->index;
 227         delta <<= PAGE_CACHE_SHIFT;
 228         delta += offset;
 229         delta -= iomapp->iomap_offset;
 230         delta >>= block_bits;
 231
 232         sector_shift = block_bits - BBSHIFT;
 233         bn = iomapp->iomap_bn >> sector_shift;
 234         bn += delta;
 235         BUG_ON(!bn && !(iomapp->iomap_flags & IOMAP_REALTIME));
 236         ASSERT((bn << sector_shift) >= iomapp->iomap_bn);
 237
 238         lock_buffer(bh);
 239         bh->b_blocknr = bn;
 240         bh->b_bdev = iomapp->iomap_target->pbr_bdev;
 241         set_buffer_mapped(bh);
 242         clear_buffer_delay(bh);
 243 }
 244
 245 /*
 246  * Look for a page at index which is unlocked and contains our
 247  * unwritten extent flagged buffers at its head.  Returns page
 248  * locked and with an extra reference count, and length of the
 249  * unwritten extent component on this page that we can write,
 250  * in units of filesystem blocks.
 251  */
 252 STATIC struct page *
 253 xfs_probe_unwritten_page(
 254         struct address_space    *mapping,
 255         pgoff_t                 index,
 256         xfs_iomap_t             *iomapp,
 257         xfs_buf_t               *pb,
 258         unsigned long           max_offset,
 259         unsigned long           *fsbs,
 260         unsigned int            bbits)
 261 {
 262         struct page             *page;
 263
 264         page = find_trylock_page(mapping, index);
 265         if (!page)
 266                 return NULL;
 267         if (PageWriteback(page))
 268                 goto out;
 269
 270         if (page->mapping && page_has_buffers(page)) {
 271                 struct buffer_head      *bh, *head;
 272                 unsigned long           p_offset = 0;
 273
 274                 *fsbs = 0;
 275                 bh = head = page_buffers(page);
 276                 do {
 277                         if (!buffer_unwritten(bh) || !buffer_uptodate(bh))
 278                                 break;
 279                         if (!xfs_offset_to_map(page, iomapp, p_offset))
 280                                 break;
 281                         if (p_offset >= max_offset)
 282                                 break;
 283                         xfs_map_at_offset(page, bh, p_offset, bbits, iomapp);
 284                         set_buffer_unwritten_io(bh);
 285                         bh->b_private = pb;
 286                         p_offset += bh->b_size;
 287                         (*fsbs)++;
 288                 } while ((bh = bh->b_this_page) != head);
 289
 290                 if (p_offset)
 291                         return page;
 292         }
 293
 294 out:
 295         unlock_page(page);
 296         return NULL;
 297 }
 298
 299 /*
 300  * Look for a page at index which is unlocked and not mapped
 301  * yet - clustering for mmap write case.
 302  */
 303 STATIC unsigned int
 304 xfs_probe_unmapped_page(
 305         struct address_space    *mapping,
 306         pgoff_t                 index,
 307         unsigned int            pg_offset)
 308 {
 309         struct page             *page;
 310         int                     ret = 0;
 311
 312         page = find_trylock_page(mapping, index);
 313         if (!page)
 314                 return 0;
 315         if (PageWriteback(page))
 316                 goto out;
 317
 318         if (page->mapping && PageDirty(page)) {
 319                 if (page_has_buffers(page)) {
 320                         struct buffer_head      *bh, *head;
 321
 322                         bh = head = page_buffers(page);
 323                         do {
 324                                 if (buffer_mapped(bh) || !buffer_uptodate(bh))
 325                                         break;
 326                                 ret += bh->b_size;
 327                                 if (ret >= pg_offset)
 328                                         break;
 329                         } while ((bh = bh->b_this_page) != head);
 330                 } else
 331                         ret = PAGE_CACHE_SIZE;
 332         }
 333
 334 out:
 335         unlock_page(page);
 336         return ret;
 337 }
 338
 339 STATIC unsigned int
 340 xfs_probe_unmapped_cluster(
 341         struct inode            *inode,
 342         struct page             *startpage,
 343         struct buffer_head      *bh,
 344         struct buffer_head      *head)
 345 {
 346         pgoff_t                 tindex, tlast, tloff;
 347         unsigned int            pg_offset, len, total = 0;
 348         struct address_space    *mapping = inode->i_mapping;
 349
 350         /* First sum forwards in this page */
 351         do {
 352                 if (buffer_mapped(bh))
 353                         break;
 354                 total += bh->b_size;
 355         } while ((bh = bh->b_this_page) != head);
 356
 357         /* If we reached the end of the page, sum forwards in
 358          * following pages.
 359          */
 360         if (bh == head) {
 361                 tlast = i_size_read(inode) >> PAGE_CACHE_SHIFT;
 362                 /* Prune this back to avoid pathological behavior */
 363                 tloff = min(tlast, startpage->index + 64);
 364                 for (tindex = startpage->index + 1; tindex < tloff; tindex++) {
 365                         len = xfs_probe_unmapped_page(mapping, tindex,
 366                                                         PAGE_CACHE_SIZE);
 367                         if (!len)
 368                                 return total;
 369                         total += len;
 370                 }
 371                 if (tindex == tlast &&
 372                     (pg_offset = i_size_read(inode) & (PAGE_CACHE_SIZE - 1))) {
 373                         total += xfs_probe_unmapped_page(mapping,
 374                                                         tindex, pg_offset);
 375                 }
 376         }
 377         return total;
 378 }
 379
 380 /*
 381  * Probe for a given page (index) in the inode and test if it is delayed
 382  * and without unwritten buffers.  Returns page locked and with an extra
 383  * reference count.
 384  */
 385 STATIC struct page *
 386 xfs_probe_delalloc_page(
 387         struct inode            *inode,
 388         pgoff_t                 index)
 389 {
 390         struct page             *page;
 391
 392         page = find_trylock_page(inode->i_mapping, index);
 393         if (!page)
 394                 return NULL;
 395         if (PageWriteback(page))
 396                 goto out;
 397
 398         if (page->mapping && page_has_buffers(page)) {
 399                 struct buffer_head      *bh, *head;
 400                 int                     acceptable = 0;
 401
 402                 bh = head = page_buffers(page);
 403                 do {
 404                         if (buffer_unwritten(bh)) {
 405                                 acceptable = 0;
 406                                 break;
 407                         } else if (buffer_delay(bh)) {
 408                                 acceptable = 1;
 409                         }
 410                 } while ((bh = bh->b_this_page) != head);
 411
 412                 if (acceptable)
 413                         return page;
 414         }
 415
 416 out:
 417         unlock_page(page);
 418         return NULL;
 419 }
 420
 421 STATIC int
 422 xfs_map_unwritten(
 423         struct inode            *inode,
 424         struct page             *start_page,
 425         struct buffer_head      *head,
 426         struct buffer_head      *curr,
 427         unsigned long           p_offset,
 428         int                     block_bits,
 429         xfs_iomap_t             *iomapp,
 430         struct writeback_control *wbc,
 431         int                     startio,
 432         int                     all_bh)
 433 {
 434         struct buffer_head      *bh = curr;
 435         xfs_iomap_t             *tmp;
 436         xfs_buf_t               *pb;
 437         loff_t                  offset, size;
 438         unsigned long           nblocks = 0;
 439
 440         offset = start_page->index;
 441         offset <<= PAGE_CACHE_SHIFT;
 442         offset += p_offset;
 443
 444         /* get an "empty" pagebuf to manage IO completion
 445          * Proper values will be set before returning */
 446         pb = pagebuf_lookup(iomapp->iomap_target, 0, 0, 0);
 447         if (!pb)
 448                 return -EAGAIN;
 449
 450         /* Take a reference to the inode to prevent it from
 451          * being reclaimed while we have outstanding unwritten
 452          * extent IO on it.
 453          */
 454         if ((igrab(inode)) != inode) {
 455                 pagebuf_free(pb);
 456                 return -EAGAIN;
 457         }
 458
 459         /* Set the count to 1 initially, this will stop an I/O
 460          * completion callout which happens before we have started
 461          * all the I/O from calling pagebuf_iodone too early.
 462          */
 463         atomic_set(&pb->pb_io_remaining, 1);
 464
 465         /* First map forwards in the page consecutive buffers
 466          * covering this unwritten extent
 467          */
 468         do {
 469                 if (!buffer_unwritten(bh))
 470                         break;
 471                 tmp = xfs_offset_to_map(start_page, iomapp, p_offset);
 472                 if (!tmp)
 473                         break;
 474                 xfs_map_at_offset(start_page, bh, p_offset, block_bits, iomapp);
 475                 set_buffer_unwritten_io(bh);
 476                 bh->b_private = pb;
 477                 p_offset += bh->b_size;
 478                 nblocks++;
 479         } while ((bh = bh->b_this_page) != head);
 480
 481         atomic_add(nblocks, &pb->pb_io_remaining);
 482
 483         /* If we reached the end of the page, map forwards in any
 484          * following pages which are also covered by this extent.
 485          */
 486         if (bh == head) {
 487                 struct address_space    *mapping = inode->i_mapping;
 488                 pgoff_t                 tindex, tloff, tlast;
 489                 unsigned long           bs;
 490                 unsigned int            pg_offset, bbits = inode->i_blkbits;
 491                 struct page             *page;
 492
 493                 tlast = i_size_read(inode) >> PAGE_CACHE_SHIFT;
 494                 tloff = (iomapp->iomap_offset + iomapp->iomap_bsize) >> PAGE_CACHE_SHIFT;
 495                 tloff = min(tlast, tloff);
 496                 for (tindex = start_page->index + 1; tindex < tloff; tindex++) {
 497                         page = xfs_probe_unwritten_page(mapping,
 498                                                 tindex, iomapp, pb,
 499                                                 PAGE_CACHE_SIZE, &bs, bbits);
 500                         if (!page)
 501                                 break;
 502                         nblocks += bs;
 503                         atomic_add(bs, &pb->pb_io_remaining);
 504                         xfs_convert_page(inode, page, iomapp, wbc, pb,
 505                                                         startio, all_bh);
 506                         /* stop if converting the next page might add
 507                          * enough blocks that the corresponding byte
 508                          * count won't fit in our ulong page buf length */
 509                         if (nblocks >= ((ULONG_MAX - PAGE_SIZE) >> block_bits))
 510                                 goto enough;
 511                 }
 512
 513                 if (tindex == tlast &&
 514                     (pg_offset = (i_size_read(inode) & (PAGE_CACHE_SIZE - 1)))) {
 515                         page = xfs_probe_unwritten_page(mapping,
 516                                                         tindex, iomapp, pb,
 517                                                         pg_offset, &bs, bbits);
 518                         if (page) {
 519                                 nblocks += bs;
 520                                 atomic_add(bs, &pb->pb_io_remaining);
 521                                 xfs_convert_page(inode, page, iomapp, wbc, pb,
 522                                                         startio, all_bh);
 523                                 if (nblocks >= ((ULONG_MAX - PAGE_SIZE) >> block_bits))
 524                                         goto enough;
 525                         }
 526                 }
 527         }
 528
 529 enough:
 530         size = nblocks;         /* NB: using 64bit number here */
 531         size <<= block_bits;    /* convert fsb's to byte range */
 532
 533         XFS_BUF_DATAIO(pb);
 534         XFS_BUF_ASYNC(pb);
 535         XFS_BUF_SET_SIZE(pb, size);
 536         XFS_BUF_SET_COUNT(pb, size);
 537         XFS_BUF_SET_OFFSET(pb, offset);
 538         XFS_BUF_SET_FSPRIVATE(pb, LINVFS_GET_VP(inode));
 539         XFS_BUF_SET_IODONE_FUNC(pb, linvfs_unwritten_convert);
 540
 541         if (atomic_dec_and_test(&pb->pb_io_remaining) == 1) {
 542                 pagebuf_iodone(pb, 1, 1);
 543         }
 544
 545         return 0;
 546 }
 547
 548 STATIC void
 549 xfs_submit_page(
 550         struct page             *page,
 551         struct writeback_control *wbc,
 552         struct buffer_head      *bh_arr[],
 553         int                     bh_count,
 554         int                     probed_page,
 555         int                     clear_dirty)
 556 {
 557         struct buffer_head      *bh;
 558         int                     i;
 559
 560         BUG_ON(PageWriteback(page));
 561         if (bh_count)
 562                 set_page_writeback(page);
 563         if (clear_dirty)
 564                 clear_page_dirty(page);
 565         unlock_page(page);
 566
 567         if (bh_count) {
 568                 for (i = 0; i < bh_count; i++) {
 569                         bh = bh_arr[i];
 570                         mark_buffer_async_write(bh);
 571                         if (buffer_unwritten(bh))
 572                                 set_buffer_unwritten_io(bh);
 573                         set_buffer_uptodate(bh);
 574                         clear_buffer_dirty(bh);
 575                 }
 576
 577                 for (i = 0; i < bh_count; i++)
 578                         submit_bh(WRITE, bh_arr[i]);
 579
 580                 if (probed_page && clear_dirty)
 581                         wbc->nr_to_write--;     /* Wrote an "extra" page */
 582         }
 583 }
 584
 585 /*
 586  * Allocate & map buffers for page given the extent map. Write it out.
 587  * except for the original page of a writepage, this is called on
 588  * delalloc/unwritten pages only, for the original page it is possible
 589  * that the page has no mapping at all.
 590  */
 591 STATIC void
 592 xfs_convert_page(
 593         struct inode            *inode,
 594         struct page             *page,
 595         xfs_iomap_t             *iomapp,
 596         struct writeback_control *wbc,
 597         void                    *private,
 598         int                     startio,
 599         int                     all_bh)
 600 {
 601         struct buffer_head      *bh_arr[MAX_BUF_PER_PAGE], *bh, *head;
 602         xfs_iomap_t             *mp = iomapp, *tmp;
 603         unsigned long           offset, end_offset;
 604         int                     index = 0;
 605         int                     bbits = inode->i_blkbits;
 606         int                     len, page_dirty;
 607
 608         end_offset = (i_size_read(inode) & (PAGE_CACHE_SIZE - 1));
 609
 610         /*
 611          * page_dirty is initially a count of buffers on the page before
 612          * EOF and is decrememted as we move each into a cleanable state.
 613          */
 614         len = 1 << inode->i_blkbits;
 615         end_offset = max(end_offset, PAGE_CACHE_SIZE);
 616         end_offset = roundup(end_offset, len);
 617         page_dirty = end_offset / len;
 618
 619         offset = 0;
 620         bh = head = page_buffers(page);
 621         do {
 622                 if (offset >= end_offset)
 623                         break;
 624                 if (!(PageUptodate(page) || buffer_uptodate(bh)))
 625                         continue;
 626                 if (buffer_mapped(bh) && all_bh &&
 627                     !(buffer_unwritten(bh) || buffer_delay(bh))) {
 628                         if (startio) {
 629                                 lock_buffer(bh);
 630                                 bh_arr[index++] = bh;
 631                                 page_dirty--;
 632                         }
 633                         continue;
 634                 }
 635                 tmp = xfs_offset_to_map(page, mp, offset);
 636                 if (!tmp)
 637                         continue;
 638                 ASSERT(!(tmp->iomap_flags & IOMAP_HOLE));
 639                 ASSERT(!(tmp->iomap_flags & IOMAP_DELAY));
 640
 641                 /* If this is a new unwritten extent buffer (i.e. one
 642                  * that we haven't passed in private data for, we must
 643                  * now map this buffer too.
 644                  */
 645                 if (buffer_unwritten(bh) && !bh->b_end_io) {
 646                         ASSERT(tmp->iomap_flags & IOMAP_UNWRITTEN);
 647                         xfs_map_unwritten(inode, page, head, bh, offset,
 648                                         bbits, tmp, wbc, startio, all_bh);
 649                 } else if (! (buffer_unwritten(bh) && buffer_locked(bh))) {
 650                         xfs_map_at_offset(page, bh, offset, bbits, tmp);
 651                         if (buffer_unwritten(bh)) {
 652                                 set_buffer_unwritten_io(bh);
 653                                 bh->b_private = private;
 654                                 ASSERT(private);
 655                         }
 656                 }
 657                 if (startio) {
 658                         bh_arr[index++] = bh;
 659                 } else {
 660                         set_buffer_dirty(bh);
 661                         unlock_buffer(bh);
 662                         mark_buffer_dirty(bh);
 663                 }
 664                 page_dirty--;
 665         } while (offset += len, (bh = bh->b_this_page) != head);
 666
 667         if (startio && index) {
 668                 xfs_submit_page(page, wbc, bh_arr, index, 1, !page_dirty);
 669         } else {
 670                 unlock_page(page);
 671         }
 672 }
 673
 674 /*
 675  * Convert & write out a cluster of pages in the same extent as defined
 676  * by mp and following the start page.
 677  */
 678 STATIC void
 679 xfs_cluster_write(
 680         struct inode            *inode,
 681         pgoff_t                 tindex,
 682         xfs_iomap_t             *iomapp,
 683         struct writeback_control *wbc,
 684         int                     startio,
 685         int                     all_bh,
 686         pgoff_t                 tlast)
 687 {
 688         struct page             *page;
 689
 690         for (; tindex <= tlast; tindex++) {
 691                 page = xfs_probe_delalloc_page(inode, tindex);
 692                 if (!page)
 693                         break;
 694                 xfs_convert_page(inode, page, iomapp, wbc, NULL,
 695                                 startio, all_bh);
 696         }
 697 }
 698
 699 /*
 700  * Calling this without startio set means we are being asked to make a dirty
 701  * page ready for freeing it's buffers.  When called with startio set then
 702  * we are coming from writepage.
 703  *
 704  * When called with startio set it is important that we write the WHOLE
 705  * page if possible.
 706  * The bh->b_state's cannot know if any of the blocks or which block for
 707  * that matter are dirty due to mmap writes, and therefore bh uptodate is
 708  * only vaild if the page itself isn't completely uptodate.  Some layers
 709  * may clear the page dirty flag prior to calling write page, under the
 710  * assumption the entire page will be written out; by not writing out the
 711  * whole page the page can be reused before all valid dirty data is
 712  * written out.  Note: in the case of a page that has been dirty'd by
 713  * mapwrite and but partially setup by block_prepare_write the
 714  * bh->b_states's will not agree and only ones setup by BPW/BCW will have
 715  * valid state, thus the whole page must be written out thing.
 716  */
 717
 718 STATIC int
 719 xfs_page_state_convert(
 720         struct inode    *inode,
 721         struct page     *page,
 722         struct writeback_control *wbc,
 723         int             startio,
 724         int             unmapped) /* also implies page uptodate */
 725 {
 726         struct buffer_head      *bh_arr[MAX_BUF_PER_PAGE], *bh, *head;
 727         xfs_iomap_t             *iomp, iomap;
 728         loff_t                  offset;
 729         unsigned long           p_offset = 0;
 730         __uint64_t              end_offset;
 731         pgoff_t                 end_index, last_index, tlast;
 732         int                     len, err, i, cnt = 0, uptodate = 1;
 733         int                     flags;
 734         int                     page_dirty;
 735
 736         /* wait for other IO threads? */
 737         flags = (startio && wbc->sync_mode != WB_SYNC_NONE) ? 0 : BMAPI_TRYLOCK;
 738
 739         /* Is this page beyond the end of the file? */
 740         offset = i_size_read(inode);
 741         end_index = offset >> PAGE_CACHE_SHIFT;
 742         last_index = (offset - 1) >> PAGE_CACHE_SHIFT;
 743         if (page->index >= end_index) {
 744                 if ((page->index >= end_index + 1) ||
 745                     !(i_size_read(inode) & (PAGE_CACHE_SIZE - 1))) {
 746                         err = -EIO;
 747                         goto error;
 748                 }
 749         }
 750
 751         end_offset = min_t(unsigned long long,
 752                         (loff_t)(page->index + 1) << PAGE_CACHE_SHIFT, offset);
 753         offset = (loff_t)page->index << PAGE_CACHE_SHIFT;
 754
 755         /*
 756          * page_dirty is initially a count of buffers on the page before
 757          * EOF and is decrememted as we move each into a cleanable state.
 758          */
 759         len = 1 << inode->i_blkbits;
 760         p_offset = max(p_offset, PAGE_CACHE_SIZE);
 761         p_offset = roundup(p_offset, len);
 762         page_dirty = p_offset / len;
 763
 764         iomp = NULL;
 765         p_offset = 0;
 766         bh = head = page_buffers(page);
 767
 768         do {
 769                 if (offset >= end_offset)
 770                         break;
 771                 if (!buffer_uptodate(bh))
 772                         uptodate = 0;
 773                 if (!(PageUptodate(page) || buffer_uptodate(bh)) && !startio)
 774                         continue;
 775
 776                 if (iomp) {
 777                         iomp = xfs_offset_to_map(page, &iomap, p_offset);
 778                 }
 779
 780                 /*
 781                  * First case, map an unwritten extent and prepare for
 782                  * extent state conversion transaction on completion.
 783                  */
 784                 if (buffer_unwritten(bh)) {
 785                         if (!startio)
 786                                 continue;
 787                         if (!iomp) {
 788                                 err = xfs_map_blocks(inode, offset, len, &iomap,
 789                                                 BMAPI_READ|BMAPI_IGNSTATE);
 790                                 if (err) {
 791                                         goto error;
 792                                 }
 793                                 iomp = xfs_offset_to_map(page, &iomap,
 794                                                                 p_offset);
 795                         }
 796                         if (iomp) {
 797                                 if (!bh->b_end_io) {
 798                                         err = xfs_map_unwritten(inode, page,
 799                                                         head, bh, p_offset,
 800                                                         inode->i_blkbits, iomp,
 801                                                         wbc, startio, unmapped);
 802                                         if (err) {
 803                                                 goto error;
 804                                         }
 805                                 } else {
 806                                         set_bit(BH_Lock, &bh->b_state);
 807                                 }
 808                                 BUG_ON(!buffer_locked(bh));
 809                                 bh_arr[cnt++] = bh;
 810                                 page_dirty--;
 811                         }
 812                 /*
 813                  * Second case, allocate space for a delalloc buffer.
 814                  * We can return EAGAIN here in the release page case.
 815                  */
 816                 } else if (buffer_delay(bh)) {
 817                         if (!iomp) {
 818                                 err = xfs_map_blocks(inode, offset, len, &iomap,
 819                                                 BMAPI_ALLOCATE | flags);
 820                                 if (err) {
 821                                         goto error;
 822                                 }
 823                                 iomp = xfs_offset_to_map(page, &iomap,
 824                                                                 p_offset);
 825                         }
 826                         if (iomp) {
 827                                 xfs_map_at_offset(page, bh, p_offset,
 828                                                 inode->i_blkbits, iomp);
 829                                 if (startio) {
 830                                         bh_arr[cnt++] = bh;
 831                                 } else {
 832                                         set_buffer_dirty(bh);
 833                                         unlock_buffer(bh);
 834                                         mark_buffer_dirty(bh);
 835                                 }
 836                                 page_dirty--;
 837                         }
 838                 } else if ((buffer_uptodate(bh) || PageUptodate(page)) &&
 839                            (unmapped || startio)) {
 840
 841                         if (!buffer_mapped(bh)) {
 842                                 int     size;
 843
 844                                 /*
 845                                  * Getting here implies an unmapped buffer
 846                                  * was found, and we are in a path where we
 847                                  * need to write the whole page out.
 848                                  */
 849                                 if (!iomp) {
 850                                         size = xfs_probe_unmapped_cluster(
 851                                                         inode, page, bh, head);
 852                                         err = xfs_map_blocks(inode, offset,
 853                                                         size, &iomap,
 854                                                         BMAPI_WRITE|BMAPI_MMAP);
 855                                         if (err) {
 856                                                 goto error;
 857                                         }
 858                                         iomp = xfs_offset_to_map(page, &iomap,
 859                                                                      p_offset);
 860                                 }
 861                                 if (iomp) {
 862                                         xfs_map_at_offset(page,
 863                                                         bh, p_offset,
 864                                                         inode->i_blkbits, iomp);
 865                                         if (startio) {
 866                                                 bh_arr[cnt++] = bh;
 867                                         } else {
 868                                                 set_buffer_dirty(bh);
 869                                                 unlock_buffer(bh);
 870                                                 mark_buffer_dirty(bh);
 871                                         }
 872                                         page_dirty--;
 873                                 }
 874                         } else if (startio) {
 875                                 if (buffer_uptodate(bh) &&
 876                                     !test_and_set_bit(BH_Lock, &bh->b_state)) {
 877                                         bh_arr[cnt++] = bh;
 878                                         page_dirty--;
 879                                 }
 880                         }
 881                 }
 882         } while (offset += len, p_offset += len,
 883                 ((bh = bh->b_this_page) != head));
 884
 885         if (uptodate && bh == head)
 886                 SetPageUptodate(page);
 887
 888         if (startio) {
 889                 xfs_submit_page(page, wbc, bh_arr, cnt, 0, !page_dirty);
 890         }
 891
 892         if (iomp) {
 893                 offset = (iomp->iomap_offset + iomp->iomap_bsize - 1) >>
 894                                         PAGE_CACHE_SHIFT;
 895                 tlast = min_t(pgoff_t, offset, last_index);
 896                 xfs_cluster_write(inode, page->index + 1, iomp, wbc,
 897                                         startio, unmapped, tlast);
 898         }
 899
 900         return page_dirty;
 901
 902 error:
 903         for (i = 0; i < cnt; i++) {
 904                 unlock_buffer(bh_arr[i]);
 905         }
 906
 907         /*
 908          * If it's delalloc and we have nowhere to put it,
 909          * throw it away, unless the lower layers told
 910          * us to try again.
 911          */
 912         if (err != -EAGAIN) {
 913                 if (!unmapped) {
 914                         block_invalidatepage(page, 0);
 915                 }
 916                 ClearPageUptodate(page);
 917         }
 918         return err;
 919 }
 920
 921 STATIC int
 922 __linvfs_get_block(
 923         struct inode            *inode,
 924         sector_t                iblock,
 925         unsigned long           blocks,
 926         struct buffer_head      *bh_result,
 927         int                     create,
 928         int                     direct,
 929         bmapi_flags_t           flags)
 930 {
 931         vnode_t                 *vp = LINVFS_GET_VP(inode);
 932         xfs_iomap_t             iomap;
 933         int                     retpbbm = 1;
 934         int                     error;
 935         ssize_t                 size;
 936         loff_t                  offset = (loff_t)iblock << inode->i_blkbits;
 937
 938         if (blocks)
 939                 size = blocks << inode->i_blkbits;
 940         else
 941                 size = 1 << inode->i_blkbits;
 942
 943         VOP_BMAP(vp, offset, size,
 944                 create ? flags : BMAPI_READ, &iomap, &retpbbm, error);
 945         if (error)
 946                 return -error;
 947
 948         if (retpbbm == 0)
 949                 return 0;
 950
 951         if (iomap.iomap_bn != IOMAP_DADDR_NULL) {
 952                 xfs_daddr_t             bn;
 953                 loff_t                  delta;
 954
 955                 /* For unwritten extents do not report a disk address on
 956                  * the read case (treat as if we're reading into a hole).
 957                  */
 958                 if (create || !(iomap.iomap_flags & IOMAP_UNWRITTEN)) {
 959                         delta = offset - iomap.iomap_offset;
 960                         delta >>= inode->i_blkbits;
 961
 962                         bn = iomap.iomap_bn >> (inode->i_blkbits - BBSHIFT);
 963                         bn += delta;
 964                         BUG_ON(!bn && !(iomap.iomap_flags & IOMAP_REALTIME));
 965                         bh_result->b_blocknr = bn;
 966                         set_buffer_mapped(bh_result);
 967                 }
 968                 if (create && (iomap.iomap_flags & IOMAP_UNWRITTEN)) {
 969                         if (direct)
 970                                 bh_result->b_private = inode;
 971                         set_buffer_unwritten(bh_result);
 972                         set_buffer_delay(bh_result);
 973                 }
 974         }
 975
 976         /* If this is a realtime file, data might be on a new device */
 977         bh_result->b_bdev = iomap.iomap_target->pbr_bdev;
 978
 979         /* If we previously allocated a block out beyond eof and
 980          * we are now coming back to use it then we will need to
 981          * flag it as new even if it has a disk address.
 982          */
 983         if (create &&
 984             ((!buffer_mapped(bh_result) && !buffer_uptodate(bh_result)) ||
 985              (offset >= i_size_read(inode)) || (iomap.iomap_flags & IOMAP_NEW))) {
 986                 set_buffer_new(bh_result);
 987         }
 988
 989         if (iomap.iomap_flags & IOMAP_DELAY) {
 990                 BUG_ON(direct);
 991                 if (create) {
 992                         set_buffer_uptodate(bh_result);
 993                         set_buffer_mapped(bh_result);
 994                         set_buffer_delay(bh_result);
 995                 }
 996         }
 997
 998         if (blocks) {
 999                 bh_result->b_size = (ssize_t)min(
1000                         (loff_t)(iomap.iomap_bsize - iomap.iomap_delta),
1001                         (loff_t)(blocks << inode->i_blkbits));
1002         }
1003
1004         return 0;
1005 }
1006
1007 int
1008 linvfs_get_block(
1009         struct inode            *inode,
1010         sector_t                iblock,
1011         struct buffer_head      *bh_result,
1012         int                     create)
1013 {
1014         return __linvfs_get_block(inode, iblock, 0, bh_result,
1015                                         create, 0, BMAPI_WRITE);
1016 }
1017
1018 STATIC int
1019 linvfs_get_blocks_direct(
1020         struct inode            *inode,
1021         sector_t                iblock,
1022         unsigned long           max_blocks,
1023         struct buffer_head      *bh_result,
1024         int                     create)
1025 {
1026         return __linvfs_get_block(inode, iblock, max_blocks, bh_result,
1027                                         create, 1, BMAPI_WRITE|BMAPI_DIRECT);
1028 }
1029
1030 STATIC ssize_t
1031 linvfs_direct_IO(
1032         int                     rw,
1033         struct kiocb            *iocb,
1034         const struct iovec      *iov,
1035         loff_t                  offset,
1036         unsigned long           nr_segs)
1037 {
1038         struct file     *file = iocb->ki_filp;
1039         struct inode    *inode = file->f_mapping->host;
1040         vnode_t         *vp = LINVFS_GET_VP(inode);
1041         xfs_iomap_t     iomap;
1042         int             maps = 1;
1043         int             error;
1044
1045         VOP_BMAP(vp, offset, 0, BMAPI_DEVICE, &iomap, &maps, error);
1046         if (error)
1047                 return -error;
1048
1049         return blockdev_direct_IO_own_locking(rw, iocb, inode,
1050                 iomap.iomap_target->pbr_bdev,
1051                 iov, offset, nr_segs,
1052                 linvfs_get_blocks_direct,
1053                 linvfs_unwritten_convert_direct);
1054 }
1055
1056
1057 STATIC sector_t
1058 linvfs_bmap(
1059         struct address_space    *mapping,
1060         sector_t                block)
1061 {
1062         struct inode            *inode = (struct inode *)mapping->host;
1063         vnode_t                 *vp = LINVFS_GET_VP(inode);
1064         int                     error;
1065
1066         vn_trace_entry(vp, "linvfs_bmap", (inst_t *)__return_address);
1067
1068         VOP_RWLOCK(vp, VRWLOCK_READ);
1069         VOP_FLUSH_PAGES(vp, (xfs_off_t)0, -1, 0, FI_REMAPF, error);
1070         VOP_RWUNLOCK(vp, VRWLOCK_READ);
1071         return generic_block_bmap(mapping, block, linvfs_get_block);
1072 }
1073
1074 STATIC int
1075 linvfs_readpage(
1076         struct file             *unused,
1077         struct page             *page)
1078 {
1079         return mpage_readpage(page, linvfs_get_block);
1080 }
1081
1082 STATIC int
1083 linvfs_readpages(
1084         struct file             *unused,
1085         struct address_space    *mapping,
1086         struct list_head        *pages,
1087         unsigned                nr_pages)
1088 {
1089         return mpage_readpages(mapping, pages, nr_pages, linvfs_get_block);
1090 }
1091
1092 STATIC void
1093 xfs_count_page_state(
1094         struct page             *page,
1095         int                     *delalloc,
1096         int                     *unmapped,
1097         int                     *unwritten)
1098 {
1099         struct buffer_head      *bh, *head;
1100
1101         *delalloc = *unmapped = *unwritten = 0;
1102
1103         bh = head = page_buffers(page);
1104         do {
1105                 if (buffer_uptodate(bh) && !buffer_mapped(bh))
1106                         (*unmapped) = 1;
1107                 else if (buffer_unwritten(bh) && !buffer_delay(bh))
1108                         clear_buffer_unwritten(bh);
1109                 else if (buffer_unwritten(bh))
1110                         (*unwritten) = 1;
1111                 else if (buffer_delay(bh))
1112                         (*delalloc) = 1;
1113         } while ((bh = bh->b_this_page) != head);
1114 }
1115
1116
1117 /*
1118  * writepage: Called from one of two places:
1119  *
1120  * 1. we are flushing a delalloc buffer head.
1121  *
1122  * 2. we are writing out a dirty page. Typically the page dirty
1123  *    state is cleared before we get here. In this case is it
1124  *    conceivable we have no buffer heads.
1125  *
1126  * For delalloc space on the page we need to allocate space and
1127  * flush it. For unmapped buffer heads on the page we should
1128  * allocate space if the page is uptodate. For any other dirty
1129  * buffer heads on the page we should flush them.
1130  *
1131  * If we detect that a transaction would be required to flush
1132  * the page, we have to check the process flags first, if we
1133  * are already in a transaction or disk I/O during allocations
1134  * is off, we need to fail the writepage and redirty the page.
1135  */
1136
1137 STATIC int
1138 linvfs_writepage(
1139         struct page             *page,
1140         struct writeback_control *wbc)
1141 {
1142         int                     error;
1143         int                     need_trans;
1144         int                     delalloc, unmapped, unwritten;
1145         struct inode            *inode = page->mapping->host;
1146
1147         xfs_page_trace(XFS_WRITEPAGE_ENTER, inode, page, 0);
1148
1149         /*
1150          * We need a transaction if:
1151          *  1. There are delalloc buffers on the page
1152          *  2. The page is uptodate and we have unmapped buffers
1153          *  3. The page is uptodate and we have no buffers
1154          *  4. There are unwritten buffers on the page
1155          */
1156
1157         if (!page_has_buffers(page)) {
1158                 unmapped = 1;
1159                 need_trans = 1;
1160         } else {
1161                 xfs_count_page_state(page, &delalloc, &unmapped, &unwritten);
1162                 if (!PageUptodate(page))
1163                         unmapped = 0;
1164                 need_trans = delalloc + unmapped + unwritten;
1165         }
1166
1167         /*
1168          * If we need a transaction and the process flags say
1169          * we are already in a transaction, or no IO is allowed
1170          * then mark the page dirty again and leave the page
1171          * as is.
1172          */
1173         if (PFLAGS_TEST_FSTRANS() && need_trans)
1174                 goto out_fail;
1175
1176         /*
1177          * Delay hooking up buffer heads until we have
1178          * made our go/no-go decision.
1179          */
1180         if (!page_has_buffers(page))
1181                 create_empty_buffers(page, 1 << inode->i_blkbits, 0);
1182
1183         /*
1184          * Convert delayed allocate, unwritten or unmapped space
1185          * to real space and flush out to disk.
1186          */
1187         error = xfs_page_state_convert(inode, page, wbc, 1, unmapped);
1188         if (error == -EAGAIN)
1189                 goto out_fail;
1190         if (unlikely(error < 0))
1191                 goto out_unlock;
1192
1193         return 0;
1194
1195 out_fail:
1196         redirty_page_for_writepage(wbc, page);
1197         unlock_page(page);
1198         return 0;
1199 out_unlock:
1200         unlock_page(page);
1201         return error;
1202 }
1203
1204 /*
1205  * Called to move a page into cleanable state - and from there
1206  * to be released. Possibly the page is already clean. We always
1207  * have buffer heads in this call.
1208  *
1209  * Returns 0 if the page is ok to release, 1 otherwise.
1210  *
1211  * Possible scenarios are:
1212  *
1213  * 1. We are being called to release a page which has been written
1214  *    to via regular I/O. buffer heads will be dirty and possibly
1215  *    delalloc. If no delalloc buffer heads in this case then we
1216  *    can just return zero.
1217  *
1218  * 2. We are called to release a page which has been written via
1219  *    mmap, all we need to do is ensure there is no delalloc
1220  *    state in the buffer heads, if not we can let the caller
1221  *    free them and we should come back later via writepage.
1222  */
1223 STATIC int
1224 linvfs_release_page(
1225         struct page             *page,
1226         int                     gfp_mask)
1227 {
1228         struct inode            *inode = page->mapping->host;
1229         int                     dirty, delalloc, unmapped, unwritten;
1230         struct writeback_control wbc = {
1231                 .sync_mode = WB_SYNC_ALL,
1232                 .nr_to_write = 1,
1233         };
1234
1235         xfs_page_trace(XFS_RELEASEPAGE_ENTER, inode, page, gfp_mask);
1236
1237         xfs_count_page_state(page, &delalloc, &unmapped, &unwritten);
1238         if (!delalloc && !unwritten)
1239                 goto free_buffers;
1240
1241         if (!(gfp_mask & __GFP_FS))
1242                 return 0;
1243
1244         /* If we are already inside a transaction or the thread cannot
1245          * do I/O, we cannot release this page.
1246          */
1247         if (PFLAGS_TEST_FSTRANS())
1248                 return 0;
1249
1250         /*
1251          * Convert delalloc space to real space, do not flush the
1252          * data out to disk, that will be done by the caller.
1253          * Never need to allocate space here - we will always
1254          * come back to writepage in that case.
1255          */
1256         dirty = xfs_page_state_convert(inode, page, &wbc, 0, 0);
1257         if (dirty == 0 && !unwritten)
1258                 goto free_buffers;
1259         return 0;
1260
1261 free_buffers:
1262         return try_to_free_buffers(page);
1263 }
1264
1265 STATIC int
1266 linvfs_prepare_write(
1267         struct file             *file,
1268         struct page             *page,
1269         unsigned int            from,
1270         unsigned int            to)
1271 {
1272         return block_prepare_write(page, from, to, linvfs_get_block);
1273 }
1274
1275 struct address_space_operations linvfs_aops = {
1276         .readpage               = linvfs_readpage,
1277         .readpages              = linvfs_readpages,
1278         .writepage              = linvfs_writepage,
1279         .sync_page              = block_sync_page,
1280         .releasepage            = linvfs_release_page,
1281         .prepare_write          = linvfs_prepare_write,
1282         .commit_write           = generic_commit_write,
1283         .bmap                   = linvfs_bmap,
1284         .direct_IO              = linvfs_direct_IO,
1285 };