mm/hugetlb.c

   1 /*
   2  * Generic hugetlb support.
   3  * (C) William Irwin, April 2004
   4  */
   5 #include <linux/gfp.h>
   6 #include <linux/list.h>
   7 #include <linux/init.h>
   8 #include <linux/module.h>
   9 #include <linux/mm.h>
  10 #include <linux/sysctl.h>
  11 #include <linux/highmem.h>
  12 #include <linux/nodemask.h>
  13 #include <linux/pagemap.h>
  14 #include <linux/mempolicy.h>
  15 #include <linux/cpuset.h>
  16 #include <linux/mutex.h>
  17 #include <linux/vs_base.h>
  18 #include <linux/vs_memory.h>
  19
  20 #include <asm/page.h>
  21 #include <asm/pgtable.h>
  22
  23 #include <linux/hugetlb.h>
  24 #include "internal.h"
  25
  26 const unsigned long hugetlb_zero = 0, hugetlb_infinity = ~0UL;
  27 static unsigned long nr_huge_pages, free_huge_pages, reserved_huge_pages;
  28 unsigned long max_huge_pages;
  29 static struct list_head hugepage_freelists[MAX_NUMNODES];
  30 static unsigned int nr_huge_pages_node[MAX_NUMNODES];
  31 static unsigned int free_huge_pages_node[MAX_NUMNODES];
  32 /*
  33  * Protects updates to hugepage_freelists, nr_huge_pages, and free_huge_pages
  34  */
  35 static DEFINE_SPINLOCK(hugetlb_lock);
  36
  37 static void clear_huge_page(struct page *page, unsigned long addr)
  38 {
  39         int i;
  40
  41         might_sleep();
  42         for (i = 0; i < (HPAGE_SIZE/PAGE_SIZE); i++) {
  43                 cond_resched();
  44                 clear_user_highpage(page + i, addr);
  45         }
  46 }
  47
  48 static void copy_huge_page(struct page *dst, struct page *src,
  49                            unsigned long addr)
  50 {
  51         int i;
  52
  53         might_sleep();
  54         for (i = 0; i < HPAGE_SIZE/PAGE_SIZE; i++) {
  55                 cond_resched();
  56                 copy_user_highpage(dst + i, src + i, addr + i*PAGE_SIZE);
  57         }
  58 }
  59
  60 static void enqueue_huge_page(struct page *page)
  61 {
  62         int nid = page_to_nid(page);
  63         list_add(&page->lru, &hugepage_freelists[nid]);
  64         free_huge_pages++;
  65         free_huge_pages_node[nid]++;
  66 }
  67
  68 static struct page *dequeue_huge_page(struct vm_area_struct *vma,
  69                                 unsigned long address)
  70 {
  71         int nid = numa_node_id();
  72         struct page *page = NULL;
  73         struct zonelist *zonelist = huge_zonelist(vma, address);
  74         struct zone **z;
  75
  76         for (z = zonelist->zones; *z; z++) {
  77                 nid = (*z)->zone_pgdat->node_id;
  78                 if (cpuset_zone_allowed(*z, GFP_HIGHUSER) &&
  79                     !list_empty(&hugepage_freelists[nid]))
  80                         break;
  81         }
  82
  83         if (*z) {
  84                 page = list_entry(hugepage_freelists[nid].next,
  85                                   struct page, lru);
  86                 list_del(&page->lru);
  87                 free_huge_pages--;
  88                 free_huge_pages_node[nid]--;
  89         }
  90         return page;
  91 }
  92
  93 static void free_huge_page(struct page *page)
  94 {
  95         BUG_ON(page_count(page));
  96
  97         INIT_LIST_HEAD(&page->lru);
  98
  99         spin_lock(&hugetlb_lock);
 100         enqueue_huge_page(page);
 101         spin_unlock(&hugetlb_lock);
 102 }
 103
 104 static int alloc_fresh_huge_page(void)
 105 {
 106         static int nid = 0;
 107         struct page *page;
 108         page = alloc_pages_node(nid, GFP_HIGHUSER|__GFP_COMP|__GFP_NOWARN,
 109                                         HUGETLB_PAGE_ORDER);
 110         nid = next_node(nid, node_online_map);
 111         if (nid == MAX_NUMNODES)
 112                 nid = first_node(node_online_map);
 113         if (page) {
 114                 page[1].lru.next = (void *)free_huge_page;      /* dtor */
 115                 spin_lock(&hugetlb_lock);
 116                 nr_huge_pages++;
 117                 nr_huge_pages_node[page_to_nid(page)]++;
 118                 spin_unlock(&hugetlb_lock);
 119                 put_page(page); /* free it into the hugepage allocator */
 120                 return 1;
 121         }
 122         return 0;
 123 }
 124
 125 static struct page *alloc_huge_page(struct vm_area_struct *vma,
 126                                     unsigned long addr)
 127 {
 128         struct inode *inode = vma->vm_file->f_dentry->d_inode;
 129         struct page *page;
 130         int use_reserve = 0;
 131         unsigned long idx;
 132
 133         spin_lock(&hugetlb_lock);
 134
 135         if (vma->vm_flags & VM_MAYSHARE) {
 136
 137                 /* idx = radix tree index, i.e. offset into file in
 138                  * HPAGE_SIZE units */
 139                 idx = ((addr - vma->vm_start) >> HPAGE_SHIFT)
 140                         + (vma->vm_pgoff >> (HPAGE_SHIFT - PAGE_SHIFT));
 141
 142                 /* The hugetlbfs specific inode info stores the number
 143                  * of "guaranteed available" (huge) pages.  That is,
 144                  * the first 'prereserved_hpages' pages of the inode
 145                  * are either already instantiated, or have been
 146                  * pre-reserved (by hugetlb_reserve_for_inode()). Here
 147                  * we're in the process of instantiating the page, so
 148                  * we use this to determine whether to draw from the
 149                  * pre-reserved pool or the truly free pool. */
 150                 if (idx < HUGETLBFS_I(inode)->prereserved_hpages)
 151                         use_reserve = 1;
 152         }
 153
 154         if (!use_reserve) {
 155                 if (free_huge_pages <= reserved_huge_pages)
 156                         goto fail;
 157         } else {
 158                 BUG_ON(reserved_huge_pages == 0);
 159                 reserved_huge_pages--;
 160         }
 161
 162         page = dequeue_huge_page(vma, addr);
 163         if (!page)
 164                 goto fail;
 165
 166         spin_unlock(&hugetlb_lock);
 167         set_page_refcounted(page);
 168         return page;
 169
 170  fail:
 171         WARN_ON(use_reserve); /* reserved allocations shouldn't fail */
 172         spin_unlock(&hugetlb_lock);
 173         return NULL;
 174 }
 175
 176 /* hugetlb_extend_reservation()
 177  *
 178  * Ensure that at least 'atleast' hugepages are, and will remain,
 179  * available to instantiate the first 'atleast' pages of the given
 180  * inode.  If the inode doesn't already have this many pages reserved
 181  * or instantiated, set aside some hugepages in the reserved pool to
 182  * satisfy later faults (or fail now if there aren't enough, rather
 183  * than getting the SIGBUS later).
 184  */
 185 int hugetlb_extend_reservation(struct hugetlbfs_inode_info *info,
 186                                unsigned long atleast)
 187 {
 188         struct inode *inode = &info->vfs_inode;
 189         unsigned long change_in_reserve = 0;
 190         int ret = 0;
 191
 192         spin_lock(&hugetlb_lock);
 193         read_lock_irq(&inode->i_mapping->tree_lock);
 194
 195         if (info->prereserved_hpages >= atleast)
 196                 goto out;
 197
 198         /* Because we always call this on shared mappings, none of the
 199          * pages beyond info->prereserved_hpages can have been
 200          * instantiated, so we need to reserve all of them now. */
 201         change_in_reserve = atleast - info->prereserved_hpages;
 202
 203         if ((reserved_huge_pages + change_in_reserve) > free_huge_pages) {
 204                 ret = -ENOMEM;
 205                 goto out;
 206         }
 207
 208         reserved_huge_pages += change_in_reserve;
 209         info->prereserved_hpages = atleast;
 210
 211  out:
 212         read_unlock_irq(&inode->i_mapping->tree_lock);
 213         spin_unlock(&hugetlb_lock);
 214
 215         return ret;
 216 }
 217
 218 /* hugetlb_truncate_reservation()
 219  *
 220  * This returns pages reserved for the given inode to the general free
 221  * hugepage pool.  If the inode has any pages prereserved, but not
 222  * instantiated, beyond offset (atmost << HPAGE_SIZE), then release
 223  * them.
 224  */
 225 void hugetlb_truncate_reservation(struct hugetlbfs_inode_info *info,
 226                                   unsigned long atmost)
 227 {
 228         struct inode *inode = &info->vfs_inode;
 229         struct address_space *mapping = inode->i_mapping;
 230         unsigned long idx;
 231         unsigned long change_in_reserve = 0;
 232         struct page *page;
 233
 234         spin_lock(&hugetlb_lock);
 235         read_lock_irq(&inode->i_mapping->tree_lock);
 236
 237         if (info->prereserved_hpages <= atmost)
 238                 goto out;
 239
 240         /* Count pages which were reserved, but not instantiated, and
 241          * which we can now release. */
 242         for (idx = atmost; idx < info->prereserved_hpages; idx++) {
 243                 page = radix_tree_lookup(&mapping->page_tree, idx);
 244                 if (!page)
 245                         /* Pages which are already instantiated can't
 246                          * be unreserved (and in fact have already
 247                          * been removed from the reserved pool) */
 248                         change_in_reserve++;
 249         }
 250
 251         BUG_ON(reserved_huge_pages < change_in_reserve);
 252         reserved_huge_pages -= change_in_reserve;
 253         info->prereserved_hpages = atmost;
 254
 255  out:
 256         read_unlock_irq(&inode->i_mapping->tree_lock);
 257         spin_unlock(&hugetlb_lock);
 258 }
 259
 260 static int __init hugetlb_init(void)
 261 {
 262         unsigned long i;
 263
 264         if (HPAGE_SHIFT == 0)
 265                 return 0;
 266
 267         for (i = 0; i < MAX_NUMNODES; ++i)
 268                 INIT_LIST_HEAD(&hugepage_freelists[i]);
 269
 270         for (i = 0; i < max_huge_pages; ++i) {
 271                 if (!alloc_fresh_huge_page())
 272                         break;
 273         }
 274         max_huge_pages = free_huge_pages = nr_huge_pages = i;
 275         printk("Total HugeTLB memory allocated, %ld\n", free_huge_pages);
 276         return 0;
 277 }
 278 module_init(hugetlb_init);
 279
 280 static int __init hugetlb_setup(char *s)
 281 {
 282         if (sscanf(s, "%lu", &max_huge_pages) <= 0)
 283                 max_huge_pages = 0;
 284         return 1;
 285 }
 286 __setup("hugepages=", hugetlb_setup);
 287
 288 #ifdef CONFIG_SYSCTL
 289 static void update_and_free_page(struct page *page)
 290 {
 291         int i;
 292         nr_huge_pages--;
 293         nr_huge_pages_node[page_zone(page)->zone_pgdat->node_id]--;
 294         for (i = 0; i < (HPAGE_SIZE / PAGE_SIZE); i++) {
 295                 page[i].flags &= ~(1 << PG_locked | 1 << PG_error | 1 << PG_referenced |
 296                                 1 << PG_dirty | 1 << PG_active | 1 << PG_reserved |
 297                                 1 << PG_private | 1<< PG_writeback);
 298         }
 299         page[1].lru.next = NULL;
 300         set_page_refcounted(page);
 301         __free_pages(page, HUGETLB_PAGE_ORDER);
 302 }
 303
 304 #ifdef CONFIG_HIGHMEM
 305 static void try_to_free_low(unsigned long count)
 306 {
 307         int i, nid;
 308         for (i = 0; i < MAX_NUMNODES; ++i) {
 309                 struct page *page, *next;
 310                 list_for_each_entry_safe(page, next, &hugepage_freelists[i], lru) {
 311                         if (PageHighMem(page))
 312                                 continue;
 313                         list_del(&page->lru);
 314                         update_and_free_page(page);
 315                         nid = page_zone(page)->zone_pgdat->node_id;
 316                         free_huge_pages--;
 317                         free_huge_pages_node[nid]--;
 318                         if (count >= nr_huge_pages)
 319                                 return;
 320                 }
 321         }
 322 }
 323 #else
 324 static inline void try_to_free_low(unsigned long count)
 325 {
 326 }
 327 #endif
 328
 329 static unsigned long set_max_huge_pages(unsigned long count)
 330 {
 331         while (count > nr_huge_pages) {
 332                 if (!alloc_fresh_huge_page())
 333                         return nr_huge_pages;
 334         }
 335         if (count >= nr_huge_pages)
 336                 return nr_huge_pages;
 337
 338         spin_lock(&hugetlb_lock);
 339         count = max(count, reserved_huge_pages);
 340         try_to_free_low(count);
 341         while (count < nr_huge_pages) {
 342                 struct page *page = dequeue_huge_page(NULL, 0);
 343                 if (!page)
 344                         break;
 345                 update_and_free_page(page);
 346         }
 347         spin_unlock(&hugetlb_lock);
 348         return nr_huge_pages;
 349 }
 350
 351 int hugetlb_sysctl_handler(struct ctl_table *table, int write,
 352                            struct file *file, void __user *buffer,
 353                            size_t *length, loff_t *ppos)
 354 {
 355         proc_doulongvec_minmax(table, write, file, buffer, length, ppos);
 356         max_huge_pages = set_max_huge_pages(max_huge_pages);
 357         return 0;
 358 }
 359 #endif /* CONFIG_SYSCTL */
 360
 361 int hugetlb_report_meminfo(char *buf)
 362 {
 363         return sprintf(buf,
 364                         "HugePages_Total: %5lu\n"
 365                         "HugePages_Free:  %5lu\n"
 366                         "HugePages_Rsvd:  %5lu\n"
 367                         "Hugepagesize:    %5lu kB\n",
 368                         nr_huge_pages,
 369                         free_huge_pages,
 370                         reserved_huge_pages,
 371                         HPAGE_SIZE/1024);
 372 }
 373
 374 int hugetlb_report_node_meminfo(int nid, char *buf)
 375 {
 376         return sprintf(buf,
 377                 "Node %d HugePages_Total: %5u\n"
 378                 "Node %d HugePages_Free:  %5u\n",
 379                 nid, nr_huge_pages_node[nid],
 380                 nid, free_huge_pages_node[nid]);
 381 }
 382
 383 /* Return the number pages of memory we physically have, in PAGE_SIZE units. */
 384 unsigned long hugetlb_total_pages(void)
 385 {
 386         return nr_huge_pages * (HPAGE_SIZE / PAGE_SIZE);
 387 }
 388
 389 /*
 390  * We cannot handle pagefaults against hugetlb pages at all.  They cause
 391  * handle_mm_fault() to try to instantiate regular-sized pages in the
 392  * hugegpage VMA.  do_page_fault() is supposed to trap this, so BUG is we get
 393  * this far.
 394  */
 395 static struct page *hugetlb_nopage(struct vm_area_struct *vma,
 396                                 unsigned long address, int *unused)
 397 {
 398         BUG();
 399         return NULL;
 400 }
 401
 402 struct vm_operations_struct hugetlb_vm_ops = {
 403         .nopage = hugetlb_nopage,
 404 };
 405
 406 static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page,
 407                                 int writable)
 408 {
 409         pte_t entry;
 410
 411         if (writable) {
 412                 entry =
 413                     pte_mkwrite(pte_mkdirty(mk_pte(page, vma->vm_page_prot)));
 414         } else {
 415                 entry = pte_wrprotect(mk_pte(page, vma->vm_page_prot));
 416         }
 417         entry = pte_mkyoung(entry);
 418         entry = pte_mkhuge(entry);
 419
 420         return entry;
 421 }
 422
 423 static void set_huge_ptep_writable(struct vm_area_struct *vma,
 424                                    unsigned long address, pte_t *ptep)
 425 {
 426         pte_t entry;
 427
 428         entry = pte_mkwrite(pte_mkdirty(*ptep));
 429         ptep_set_access_flags(vma, address, ptep, entry, 1);
 430         update_mmu_cache(vma, address, entry);
 431         lazy_mmu_prot_update(entry);
 432 }
 433
 434
 435 int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
 436                             struct vm_area_struct *vma)
 437 {
 438         pte_t *src_pte, *dst_pte, entry;
 439         struct page *ptepage;
 440         unsigned long addr;
 441         int cow;
 442
 443         cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
 444
 445         for (addr = vma->vm_start; addr < vma->vm_end; addr += HPAGE_SIZE) {
 446                 src_pte = huge_pte_offset(src, addr);
 447                 if (!src_pte)
 448                         continue;
 449                 dst_pte = huge_pte_alloc(dst, addr);
 450                 if (!dst_pte)
 451                         goto nomem;
 452                 spin_lock(&dst->page_table_lock);
 453                 spin_lock(&src->page_table_lock);
 454                 if (!pte_none(*src_pte)) {
 455                         if (cow)
 456                                 ptep_set_wrprotect(src, addr, src_pte);
 457                         entry = *src_pte;
 458                         ptepage = pte_page(entry);
 459                         get_page(ptepage);
 460                         add_mm_counter(dst, file_rss, HPAGE_SIZE / PAGE_SIZE);
 461                         set_huge_pte_at(dst, addr, dst_pte, entry);
 462                 }
 463                 spin_unlock(&src->page_table_lock);
 464                 spin_unlock(&dst->page_table_lock);
 465         }
 466         return 0;
 467
 468 nomem:
 469         return -ENOMEM;
 470 }
 471
 472 void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
 473                           unsigned long end)
 474 {
 475         struct mm_struct *mm = vma->vm_mm;
 476         unsigned long address;
 477         pte_t *ptep;
 478         pte_t pte;
 479         struct page *page;
 480
 481         WARN_ON(!is_vm_hugetlb_page(vma));
 482         BUG_ON(start & ~HPAGE_MASK);
 483         BUG_ON(end & ~HPAGE_MASK);
 484
 485         spin_lock(&mm->page_table_lock);
 486
 487         /* Update high watermark before we lower rss */
 488         update_hiwater_rss(mm);
 489
 490         for (address = start; address < end; address += HPAGE_SIZE) {
 491                 ptep = huge_pte_offset(mm, address);
 492                 if (!ptep)
 493                         continue;
 494
 495                 pte = huge_ptep_get_and_clear(mm, address, ptep);
 496                 if (pte_none(pte))
 497                         continue;
 498
 499                 page = pte_page(pte);
 500                 put_page(page);
 501                 add_mm_counter(mm, file_rss, (int) -(HPAGE_SIZE / PAGE_SIZE));
 502         }
 503
 504         spin_unlock(&mm->page_table_lock);
 505         flush_tlb_range(vma, start, end);
 506 }
 507
 508 static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma,
 509                         unsigned long address, pte_t *ptep, pte_t pte)
 510 {
 511         struct page *old_page, *new_page;
 512         int avoidcopy;
 513
 514         old_page = pte_page(pte);
 515
 516         /* If no-one else is actually using this page, avoid the copy
 517          * and just make the page writable */
 518         avoidcopy = (page_count(old_page) == 1);
 519         if (avoidcopy) {
 520                 set_huge_ptep_writable(vma, address, ptep);
 521                 return VM_FAULT_MINOR;
 522         }
 523
 524         page_cache_get(old_page);
 525         new_page = alloc_huge_page(vma, address);
 526
 527         if (!new_page) {
 528                 page_cache_release(old_page);
 529                 return VM_FAULT_OOM;
 530         }
 531
 532         spin_unlock(&mm->page_table_lock);
 533         copy_huge_page(new_page, old_page, address);
 534         spin_lock(&mm->page_table_lock);
 535
 536         ptep = huge_pte_offset(mm, address & HPAGE_MASK);
 537         if (likely(pte_same(*ptep, pte))) {
 538                 /* Break COW */
 539                 set_huge_pte_at(mm, address, ptep,
 540                                 make_huge_pte(vma, new_page, 1));
 541                 /* Make the old page be freed below */
 542                 new_page = old_page;
 543         }
 544         page_cache_release(new_page);
 545         page_cache_release(old_page);
 546         return VM_FAULT_MINOR;
 547 }
 548
 549 int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma,
 550                         unsigned long address, pte_t *ptep, int write_access)
 551 {
 552         int ret = VM_FAULT_SIGBUS;
 553         unsigned long idx;
 554         unsigned long size;
 555         struct page *page;
 556         struct address_space *mapping;
 557         pte_t new_pte;
 558
 559         mapping = vma->vm_file->f_mapping;
 560         idx = ((address - vma->vm_start) >> HPAGE_SHIFT)
 561                 + (vma->vm_pgoff >> (HPAGE_SHIFT - PAGE_SHIFT));
 562
 563         /*
 564          * Use page lock to guard against racing truncation
 565          * before we get page_table_lock.
 566          */
 567 retry:
 568         page = find_lock_page(mapping, idx);
 569         if (!page) {
 570                 if (hugetlb_get_quota(mapping))
 571                         goto out;
 572                 page = alloc_huge_page(vma, address);
 573                 if (!page) {
 574                         hugetlb_put_quota(mapping);
 575                         ret = VM_FAULT_OOM;
 576                         goto out;
 577                 }
 578                 clear_huge_page(page, address);
 579
 580                 if (vma->vm_flags & VM_SHARED) {
 581                         int err;
 582
 583                         err = add_to_page_cache(page, mapping, idx, GFP_KERNEL);
 584                         if (err) {
 585                                 put_page(page);
 586                                 hugetlb_put_quota(mapping);
 587                                 if (err == -EEXIST)
 588                                         goto retry;
 589                                 goto out;
 590                         }
 591                 } else
 592                         lock_page(page);
 593         }
 594
 595         spin_lock(&mm->page_table_lock);
 596         size = i_size_read(mapping->host) >> HPAGE_SHIFT;
 597         if (idx >= size)
 598                 goto backout;
 599
 600         ret = VM_FAULT_MINOR;
 601         if (!pte_none(*ptep))
 602                 goto backout;
 603
 604         add_mm_counter(mm, file_rss, HPAGE_SIZE / PAGE_SIZE);
 605         new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE)
 606                                 && (vma->vm_flags & VM_SHARED)));
 607         set_huge_pte_at(mm, address, ptep, new_pte);
 608
 609         if (write_access && !(vma->vm_flags & VM_SHARED)) {
 610                 /* Optimization, do the COW without a second fault */
 611                 ret = hugetlb_cow(mm, vma, address, ptep, new_pte);
 612         }
 613
 614         spin_unlock(&mm->page_table_lock);
 615         unlock_page(page);
 616 out:
 617         return ret;
 618
 619 backout:
 620         spin_unlock(&mm->page_table_lock);
 621         hugetlb_put_quota(mapping);
 622         unlock_page(page);
 623         put_page(page);
 624         goto out;
 625 }
 626
 627 int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
 628                         unsigned long address, int write_access)
 629 {
 630         pte_t *ptep;
 631         pte_t entry;
 632         int ret;
 633         static DEFINE_MUTEX(hugetlb_instantiation_mutex);
 634
 635         ptep = huge_pte_alloc(mm, address);
 636         if (!ptep)
 637                 return VM_FAULT_OOM;
 638
 639         /*
 640          * Serialize hugepage allocation and instantiation, so that we don't
 641          * get spurious allocation failures if two CPUs race to instantiate
 642          * the same page in the page cache.
 643          */
 644         mutex_lock(&hugetlb_instantiation_mutex);
 645         entry = *ptep;
 646         if (pte_none(entry)) {
 647                 ret = hugetlb_no_page(mm, vma, address, ptep, write_access);
 648                 mutex_unlock(&hugetlb_instantiation_mutex);
 649                 return ret;
 650         }
 651
 652         ret = VM_FAULT_MINOR;
 653
 654         spin_lock(&mm->page_table_lock);
 655         /* Check for a racing update before calling hugetlb_cow */
 656         if (likely(pte_same(entry, *ptep)))
 657                 if (write_access && !pte_write(entry))
 658                         ret = hugetlb_cow(mm, vma, address, ptep, entry);
 659         spin_unlock(&mm->page_table_lock);
 660         mutex_unlock(&hugetlb_instantiation_mutex);
 661
 662         return ret;
 663 }
 664
 665 int follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
 666                         struct page **pages, struct vm_area_struct **vmas,
 667                         unsigned long *position, int *length, int i)
 668 {
 669         unsigned long pfn_offset;
 670         unsigned long vaddr = *position;
 671         int remainder = *length;
 672
 673         spin_lock(&mm->page_table_lock);
 674         while (vaddr < vma->vm_end && remainder) {
 675                 pte_t *pte;
 676                 struct page *page;
 677
 678                 /*
 679                  * Some archs (sparc64, sh*) have multiple pte_ts to
 680                  * each hugepage.  We have to make * sure we get the
 681                  * first, for the page indexing below to work.
 682                  */
 683                 pte = huge_pte_offset(mm, vaddr & HPAGE_MASK);
 684
 685                 if (!pte || pte_none(*pte)) {
 686                         int ret;
 687
 688                         spin_unlock(&mm->page_table_lock);
 689                         ret = hugetlb_fault(mm, vma, vaddr, 0);
 690                         spin_lock(&mm->page_table_lock);
 691                         if (ret == VM_FAULT_MINOR)
 692                                 continue;
 693
 694                         remainder = 0;
 695                         if (!i)
 696                                 i = -EFAULT;
 697                         break;
 698                 }
 699
 700                 pfn_offset = (vaddr & ~HPAGE_MASK) >> PAGE_SHIFT;
 701                 page = pte_page(*pte);
 702 same_page:
 703                 if (pages) {
 704                         get_page(page);
 705                         pages[i] = page + pfn_offset;
 706                 }
 707
 708                 if (vmas)
 709                         vmas[i] = vma;
 710
 711                 vaddr += PAGE_SIZE;
 712                 ++pfn_offset;
 713                 --remainder;
 714                 ++i;
 715                 if (vaddr < vma->vm_end && remainder &&
 716                                 pfn_offset < HPAGE_SIZE/PAGE_SIZE) {
 717                         /*
 718                          * We use pfn_offset to avoid touching the pageframes
 719                          * of this compound page.
 720                          */
 721                         goto same_page;
 722                 }
 723         }
 724         spin_unlock(&mm->page_table_lock);
 725         *length = remainder;
 726         *position = vaddr;
 727
 728         return i;
 729 }
 730
 731 void hugetlb_change_protection(struct vm_area_struct *vma,
 732                 unsigned long address, unsigned long end, pgprot_t newprot)
 733 {
 734         struct mm_struct *mm = vma->vm_mm;
 735         unsigned long start = address;
 736         pte_t *ptep;
 737         pte_t pte;
 738
 739         BUG_ON(address >= end);
 740         flush_cache_range(vma, address, end);
 741
 742         spin_lock(&mm->page_table_lock);
 743         for (; address < end; address += HPAGE_SIZE) {
 744                 ptep = huge_pte_offset(mm, address);
 745                 if (!ptep)
 746                         continue;
 747                 if (!pte_none(*ptep)) {
 748                         pte = huge_ptep_get_and_clear(mm, address, ptep);
 749                         pte = pte_mkhuge(pte_modify(pte, newprot));
 750                         set_huge_pte_at(mm, address, ptep, pte);
 751                         lazy_mmu_prot_update(pte);
 752                 }
 753         }
 754         spin_unlock(&mm->page_table_lock);
 755
 756         flush_tlb_range(vma, start, end);
 757 }
 758