mm/hugetlb.c

   1 /*
   2  * Generic hugetlb support.
   3  * (C) William Irwin, April 2004
   4  */
   5 #include <linux/gfp.h>
   6 #include <linux/list.h>
   7 #include <linux/init.h>
   8 #include <linux/module.h>
   9 #include <linux/mm.h>
  10 #include <linux/hugetlb.h>
  11 #include <linux/sysctl.h>
  12 #include <linux/highmem.h>
  13
  14 const unsigned long hugetlb_zero = 0, hugetlb_infinity = ~0UL;
  15 static unsigned long nr_huge_pages, free_huge_pages;
  16 unsigned long max_huge_pages;
  17 static struct list_head hugepage_freelists[MAX_NUMNODES];
  18 static unsigned int nr_huge_pages_node[MAX_NUMNODES];
  19 static unsigned int free_huge_pages_node[MAX_NUMNODES];
  20 static spinlock_t hugetlb_lock = SPIN_LOCK_UNLOCKED;
  21
  22 static void enqueue_huge_page(struct page *page)
  23 {
  24         int nid = page_zone(page)->zone_pgdat->node_id;
  25         list_add(&page->lru, &hugepage_freelists[nid]);
  26         free_huge_pages++;
  27         free_huge_pages_node[nid]++;
  28 }
  29
  30 static struct page *dequeue_huge_page(void)
  31 {
  32         int nid = numa_node_id();
  33         struct page *page = NULL;
  34
  35         if (list_empty(&hugepage_freelists[nid])) {
  36                 for (nid = 0; nid < MAX_NUMNODES; ++nid)
  37                         if (!list_empty(&hugepage_freelists[nid]))
  38                                 break;
  39         }
  40         if (nid >= 0 && nid < MAX_NUMNODES &&
  41             !list_empty(&hugepage_freelists[nid])) {
  42                 page = list_entry(hugepage_freelists[nid].next,
  43                                   struct page, lru);
  44                 list_del(&page->lru);
  45                 free_huge_pages--;
  46                 free_huge_pages_node[nid]--;
  47         }
  48         return page;
  49 }
  50
  51 static struct page *alloc_fresh_huge_page(void)
  52 {
  53         static int nid = 0;
  54         struct page *page;
  55         page = alloc_pages_node(nid, GFP_HIGHUSER|__GFP_COMP,
  56                                         HUGETLB_PAGE_ORDER);
  57         nid = (nid + 1) % numnodes;
  58         if (page) {
  59                 nr_huge_pages++;
  60                 nr_huge_pages_node[page_zone(page)->zone_pgdat->node_id]++;
  61         }
  62         return page;
  63 }
  64
  65 void free_huge_page(struct page *page)
  66 {
  67         BUG_ON(page_count(page));
  68
  69         INIT_LIST_HEAD(&page->lru);
  70         page[1].mapping = NULL;
  71
  72         spin_lock(&hugetlb_lock);
  73         enqueue_huge_page(page);
  74         spin_unlock(&hugetlb_lock);
  75 }
  76
  77 struct page *alloc_huge_page(void)
  78 {
  79         struct page *page;
  80         int i;
  81
  82         spin_lock(&hugetlb_lock);
  83         page = dequeue_huge_page();
  84         if (!page) {
  85                 spin_unlock(&hugetlb_lock);
  86                 return NULL;
  87         }
  88         spin_unlock(&hugetlb_lock);
  89         set_page_count(page, 1);
  90         page[1].mapping = (void *)free_huge_page;
  91         for (i = 0; i < (HPAGE_SIZE/PAGE_SIZE); ++i)
  92                 clear_highpage(&page[i]);
  93         return page;
  94 }
  95
  96 static int __init hugetlb_init(void)
  97 {
  98         unsigned long i;
  99         struct page *page;
 100
 101         for (i = 0; i < MAX_NUMNODES; ++i)
 102                 INIT_LIST_HEAD(&hugepage_freelists[i]);
 103
 104         for (i = 0; i < max_huge_pages; ++i) {
 105                 page = alloc_fresh_huge_page();
 106                 if (!page)
 107                         break;
 108                 spin_lock(&hugetlb_lock);
 109                 enqueue_huge_page(page);
 110                 spin_unlock(&hugetlb_lock);
 111         }
 112         max_huge_pages = free_huge_pages = nr_huge_pages = i;
 113         printk("Total HugeTLB memory allocated, %ld\n", free_huge_pages);
 114         return 0;
 115 }
 116 module_init(hugetlb_init);
 117
 118 static int __init hugetlb_setup(char *s)
 119 {
 120         if (sscanf(s, "%lu", &max_huge_pages) <= 0)
 121                 max_huge_pages = 0;
 122         return 1;
 123 }
 124 __setup("hugepages=", hugetlb_setup);
 125
 126 static void update_and_free_page(struct page *page)
 127 {
 128         int i;
 129         nr_huge_pages--;
 130         nr_huge_pages_node[page_zone(page)->zone_pgdat->node_id]--;
 131         for (i = 0; i < (HPAGE_SIZE / PAGE_SIZE); i++) {
 132                 page[i].flags &= ~(1 << PG_locked | 1 << PG_error | 1 << PG_referenced |
 133                                 1 << PG_dirty | 1 << PG_active | 1 << PG_reserved |
 134                                 1 << PG_private | 1<< PG_writeback);
 135                 set_page_count(&page[i], 0);
 136         }
 137         set_page_count(page, 1);
 138         __free_pages(page, HUGETLB_PAGE_ORDER);
 139 }
 140
 141 #ifdef CONFIG_HIGHMEM
 142 static void try_to_free_low(unsigned long count)
 143 {
 144         int i, nid;
 145         for (i = 0; i < MAX_NUMNODES; ++i) {
 146                 struct page *page, *next;
 147                 list_for_each_entry_safe(page, next, &hugepage_freelists[i], lru) {
 148                         if (PageHighMem(page))
 149                                 continue;
 150                         list_del(&page->lru);
 151                         update_and_free_page(page);
 152                         nid = page_zone(page)->zone_pgdat->node_id;
 153                         free_huge_pages--;
 154                         free_huge_pages_node[nid]--;
 155                         if (count >= nr_huge_pages)
 156                                 return;
 157                 }
 158         }
 159 }
 160 #else
 161 static inline void try_to_free_low(unsigned long count)
 162 {
 163 }
 164 #endif
 165
 166 static unsigned long set_max_huge_pages(unsigned long count)
 167 {
 168         while (count > nr_huge_pages) {
 169                 struct page *page = alloc_fresh_huge_page();
 170                 if (!page)
 171                         return nr_huge_pages;
 172                 spin_lock(&hugetlb_lock);
 173                 enqueue_huge_page(page);
 174                 spin_unlock(&hugetlb_lock);
 175         }
 176         if (count >= nr_huge_pages)
 177                 return nr_huge_pages;
 178
 179         spin_lock(&hugetlb_lock);
 180         try_to_free_low(count);
 181         while (count < nr_huge_pages) {
 182                 struct page *page = dequeue_huge_page();
 183                 if (!page)
 184                         break;
 185                 update_and_free_page(page);
 186         }
 187         spin_unlock(&hugetlb_lock);
 188         return nr_huge_pages;
 189 }
 190
 191 #ifdef CONFIG_SYSCTL
 192 int hugetlb_sysctl_handler(struct ctl_table *table, int write,
 193                            struct file *file, void __user *buffer,
 194                            size_t *length, loff_t *ppos)
 195 {
 196         proc_doulongvec_minmax(table, write, file, buffer, length, ppos);
 197         max_huge_pages = set_max_huge_pages(max_huge_pages);
 198         return 0;
 199 }
 200 #endif /* CONFIG_SYSCTL */
 201
 202 int hugetlb_report_meminfo(char *buf)
 203 {
 204         return sprintf(buf,
 205                         "HugePages_Total: %5lu\n"
 206                         "HugePages_Free:  %5lu\n"
 207                         "Hugepagesize:    %5lu kB\n",
 208                         nr_huge_pages,
 209                         free_huge_pages,
 210                         HPAGE_SIZE/1024);
 211 }
 212
 213 int hugetlb_report_node_meminfo(int nid, char *buf)
 214 {
 215         return sprintf(buf,
 216                 "Node %d HugePages_Total: %5u\n"
 217                 "Node %d HugePages_Free:  %5u\n",
 218                 nid, nr_huge_pages_node[nid],
 219                 nid, free_huge_pages_node[nid]);
 220 }
 221
 222 int is_hugepage_mem_enough(size_t size)
 223 {
 224         return (size + ~HPAGE_MASK)/HPAGE_SIZE <= free_huge_pages;
 225 }
 226
 227 /* Return the number pages of memory we physically have, in PAGE_SIZE units. */
 228 unsigned long hugetlb_total_pages(void)
 229 {
 230         return nr_huge_pages * (HPAGE_SIZE / PAGE_SIZE);
 231 }
 232 EXPORT_SYMBOL(hugetlb_total_pages);
 233
 234 /*
 235  * We cannot handle pagefaults against hugetlb pages at all.  They cause
 236  * handle_mm_fault() to try to instantiate regular-sized pages in the
 237  * hugegpage VMA.  do_page_fault() is supposed to trap this, so BUG is we get
 238  * this far.
 239  */
 240 static struct page *hugetlb_nopage(struct vm_area_struct *vma,
 241                                 unsigned long address, int *unused)
 242 {
 243         BUG();
 244         return NULL;
 245 }
 246
 247 struct vm_operations_struct hugetlb_vm_ops = {
 248         .nopage = hugetlb_nopage,
 249 };
 250
 251 void zap_hugepage_range(struct vm_area_struct *vma,
 252                         unsigned long start, unsigned long length)
 253 {
 254         struct mm_struct *mm = vma->vm_mm;
 255
 256         spin_lock(&mm->page_table_lock);
 257         unmap_hugepage_range(vma, start, start + length);
 258         spin_unlock(&mm->page_table_lock);
 259 }