arch/ppc64/mm/numa.c

   1 /*
   2  * pSeries NUMA support
   3  *
   4  * Copyright (C) 2002 Anton Blanchard <anton@au.ibm.com>, IBM
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License
   8  * as published by the Free Software Foundation; either version
   9  * 2 of the License, or (at your option) any later version.
  10  */
  11 #include <linux/threads.h>
  12 #include <linux/bootmem.h>
  13 #include <linux/init.h>
  14 #include <linux/mm.h>
  15 #include <linux/mmzone.h>
  16 #include <linux/module.h>
  17 #include <linux/cpu.h>
  18 #include <linux/notifier.h>
  19 #include <asm/lmb.h>
  20 #include <asm/machdep.h>
  21 #include <asm/abs_addr.h>
  22
  23 static int numa_enabled = 1;
  24
  25 static int numa_debug;
  26 #define dbg(args...) if (numa_debug) { printk(KERN_INFO args); }
  27
  28 #ifdef DEBUG_NUMA
  29 #define ARRAY_INITIALISER -1
  30 #else
  31 #define ARRAY_INITIALISER 0
  32 #endif
  33
  34 int numa_cpu_lookup_table[NR_CPUS] = { [ 0 ... (NR_CPUS - 1)] =
  35         ARRAY_INITIALISER};
  36 char *numa_memory_lookup_table;
  37 cpumask_t numa_cpumask_lookup_table[MAX_NUMNODES];
  38 int nr_cpus_in_node[MAX_NUMNODES] = { [0 ... (MAX_NUMNODES -1)] = 0};
  39
  40 struct pglist_data *node_data[MAX_NUMNODES];
  41 bootmem_data_t __initdata plat_node_bdata[MAX_NUMNODES];
  42 static unsigned long node0_io_hole_size;
  43 static int min_common_depth;
  44
  45 /*
  46  * We need somewhere to store start/span for each node until we have
  47  * allocated the real node_data structures.
  48  */
  49 static struct {
  50         unsigned long node_start_pfn;
  51         unsigned long node_spanned_pages;
  52 } init_node_data[MAX_NUMNODES] __initdata;
  53
  54 EXPORT_SYMBOL(node_data);
  55 EXPORT_SYMBOL(numa_cpu_lookup_table);
  56 EXPORT_SYMBOL(numa_memory_lookup_table);
  57 EXPORT_SYMBOL(numa_cpumask_lookup_table);
  58 EXPORT_SYMBOL(nr_cpus_in_node);
  59
  60 static inline void map_cpu_to_node(int cpu, int node)
  61 {
  62         numa_cpu_lookup_table[cpu] = node;
  63         if (!(cpu_isset(cpu, numa_cpumask_lookup_table[node]))) {
  64                 cpu_set(cpu, numa_cpumask_lookup_table[node]);
  65                 nr_cpus_in_node[node]++;
  66         }
  67 }
  68
  69 #ifdef CONFIG_HOTPLUG_CPU
  70 static void unmap_cpu_from_node(unsigned long cpu)
  71 {
  72         int node = numa_cpu_lookup_table[cpu];
  73
  74         dbg("removing cpu %lu from node %d\n", cpu, node);
  75
  76         if (cpu_isset(cpu, numa_cpumask_lookup_table[node])) {
  77                 cpu_clear(cpu, numa_cpumask_lookup_table[node]);
  78                 nr_cpus_in_node[node]--;
  79         } else {
  80                 printk(KERN_ERR "WARNING: cpu %lu not found in node %d\n",
  81                        cpu, node);
  82         }
  83 }
  84 #endif /* CONFIG_HOTPLUG_CPU */
  85
  86 static struct device_node * __devinit find_cpu_node(unsigned int cpu)
  87 {
  88         unsigned int hw_cpuid = get_hard_smp_processor_id(cpu);
  89         struct device_node *cpu_node = NULL;
  90         unsigned int *interrupt_server, *reg;
  91         int len;
  92
  93         while ((cpu_node = of_find_node_by_type(cpu_node, "cpu")) != NULL) {
  94                 /* Try interrupt server first */
  95                 interrupt_server = (unsigned int *)get_property(cpu_node,
  96                                         "ibm,ppc-interrupt-server#s", &len);
  97
  98                 len = len / sizeof(u32);
  99
 100                 if (interrupt_server && (len > 0)) {
 101                         while (len--) {
 102                                 if (interrupt_server[len] == hw_cpuid)
 103                                         return cpu_node;
 104                         }
 105                 } else {
 106                         reg = (unsigned int *)get_property(cpu_node,
 107                                                            "reg", &len);
 108                         if (reg && (len > 0) && (reg[0] == hw_cpuid))
 109                                 return cpu_node;
 110                 }
 111         }
 112
 113         return NULL;
 114 }
 115
 116 /* must hold reference to node during call */
 117 static int *of_get_associativity(struct device_node *dev)
 118 {
 119         return (unsigned int *)get_property(dev, "ibm,associativity", NULL);
 120 }
 121
 122 static int of_node_numa_domain(struct device_node *device)
 123 {
 124         int numa_domain;
 125         unsigned int *tmp;
 126
 127         if (min_common_depth == -1)
 128                 return 0;
 129
 130         tmp = of_get_associativity(device);
 131         if (tmp && (tmp[0] >= min_common_depth)) {
 132                 numa_domain = tmp[min_common_depth];
 133         } else {
 134                 dbg("WARNING: no NUMA information for %s\n",
 135                     device->full_name);
 136                 numa_domain = 0;
 137         }
 138         return numa_domain;
 139 }
 140
 141 /*
 142  * In theory, the "ibm,associativity" property may contain multiple
 143  * associativity lists because a resource may be multiply connected
 144  * into the machine.  This resource then has different associativity
 145  * characteristics relative to its multiple connections.  We ignore
 146  * this for now.  We also assume that all cpu and memory sets have
 147  * their distances represented at a common level.  This won't be
 148  * true for heirarchical NUMA.
 149  *
 150  * In any case the ibm,associativity-reference-points should give
 151  * the correct depth for a normal NUMA system.
 152  *
 153  * - Dave Hansen <haveblue@us.ibm.com>
 154  */
 155 static int __init find_min_common_depth(void)
 156 {
 157         int depth;
 158         unsigned int *ref_points;
 159         struct device_node *rtas_root;
 160         unsigned int len;
 161
 162         rtas_root = of_find_node_by_path("/rtas");
 163
 164         if (!rtas_root)
 165                 return -1;
 166
 167         /*
 168          * this property is 2 32-bit integers, each representing a level of
 169          * depth in the associativity nodes.  The first is for an SMP
 170          * configuration (should be all 0's) and the second is for a normal
 171          * NUMA configuration.
 172          */
 173         ref_points = (unsigned int *)get_property(rtas_root,
 174                         "ibm,associativity-reference-points", &len);
 175
 176         if ((len >= 1) && ref_points) {
 177                 depth = ref_points[1];
 178         } else {
 179                 dbg("WARNING: could not find NUMA "
 180                     "associativity reference point\n");
 181                 depth = -1;
 182         }
 183         of_node_put(rtas_root);
 184
 185         return depth;
 186 }
 187
 188 static unsigned long read_cell_ul(struct device_node *device, unsigned int **buf)
 189 {
 190         int i;
 191         unsigned long result = 0;
 192
 193         i = prom_n_size_cells(device);
 194         /* bug on i>2 ?? */
 195         while (i--) {
 196                 result = (result << 32) | **buf;
 197                 (*buf)++;
 198         }
 199         return result;
 200 }
 201
 202 /*
 203  * Figure out to which domain a cpu belongs and stick it there.
 204  * Return the id of the domain used.
 205  */
 206 static int numa_setup_cpu(unsigned long lcpu)
 207 {
 208         int numa_domain = 0;
 209         struct device_node *cpu = find_cpu_node(lcpu);
 210
 211         if (!cpu) {
 212                 WARN_ON(1);
 213                 goto out;
 214         }
 215
 216         numa_domain = of_node_numa_domain(cpu);
 217
 218         if (numa_domain >= MAX_NUMNODES) {
 219                 /*
 220                  * POWER4 LPAR uses 0xffff as invalid node,
 221                  * dont warn in this case.
 222                  */
 223                 if (numa_domain != 0xffff)
 224                         printk(KERN_ERR "WARNING: cpu %ld "
 225                                "maps to invalid NUMA node %d\n",
 226                                lcpu, numa_domain);
 227                 numa_domain = 0;
 228         }
 229 out:
 230         node_set_online(numa_domain);
 231
 232         map_cpu_to_node(lcpu, numa_domain);
 233
 234         of_node_put(cpu);
 235
 236         return numa_domain;
 237 }
 238
 239 static int cpu_numa_callback(struct notifier_block *nfb,
 240                              unsigned long action,
 241                              void *hcpu)
 242 {
 243         unsigned long lcpu = (unsigned long)hcpu;
 244         int ret = NOTIFY_DONE;
 245
 246         switch (action) {
 247         case CPU_UP_PREPARE:
 248                 if (min_common_depth == -1 || !numa_enabled)
 249                         map_cpu_to_node(lcpu, 0);
 250                 else
 251                         numa_setup_cpu(lcpu);
 252                 ret = NOTIFY_OK;
 253                 break;
 254 #ifdef CONFIG_HOTPLUG_CPU
 255         case CPU_DEAD:
 256         case CPU_UP_CANCELED:
 257                 unmap_cpu_from_node(lcpu);
 258                 break;
 259                 ret = NOTIFY_OK;
 260 #endif
 261         }
 262         return ret;
 263 }
 264
 265 static int __init parse_numa_properties(void)
 266 {
 267         struct device_node *memory = NULL;
 268         int max_domain = 0;
 269         long entries = lmb_end_of_DRAM() >> MEMORY_INCREMENT_SHIFT;
 270         unsigned long i;
 271
 272         if (numa_enabled == 0) {
 273                 printk(KERN_WARNING "NUMA disabled by user\n");
 274                 return -1;
 275         }
 276
 277         numa_memory_lookup_table =
 278                 (char *)abs_to_virt(lmb_alloc(entries * sizeof(char), 1));
 279         memset(numa_memory_lookup_table, 0, entries * sizeof(char));
 280
 281         for (i = 0; i < entries ; i++)
 282                 numa_memory_lookup_table[i] = ARRAY_INITIALISER;
 283
 284         min_common_depth = find_min_common_depth();
 285
 286         dbg("NUMA associativity depth for CPU/Memory: %d\n", min_common_depth);
 287         if (min_common_depth < 0)
 288                 return min_common_depth;
 289
 290         max_domain = numa_setup_cpu(boot_cpuid);
 291
 292         memory = NULL;
 293         while ((memory = of_find_node_by_type(memory, "memory")) != NULL) {
 294                 unsigned long start;
 295                 unsigned long size;
 296                 int numa_domain;
 297                 int ranges;
 298                 unsigned int *memcell_buf;
 299                 unsigned int len;
 300
 301                 memcell_buf = (unsigned int *)get_property(memory, "reg", &len);
 302                 if (!memcell_buf || len <= 0)
 303                         continue;
 304
 305                 ranges = memory->n_addrs;
 306 new_range:
 307                 /* these are order-sensitive, and modify the buffer pointer */
 308                 start = read_cell_ul(memory, &memcell_buf);
 309                 size = read_cell_ul(memory, &memcell_buf);
 310
 311                 start = _ALIGN_DOWN(start, MEMORY_INCREMENT);
 312                 size = _ALIGN_UP(size, MEMORY_INCREMENT);
 313
 314                 numa_domain = of_node_numa_domain(memory);
 315
 316                 if (numa_domain >= MAX_NUMNODES) {
 317                         if (numa_domain != 0xffff)
 318                                 printk(KERN_ERR "WARNING: memory at %lx maps "
 319                                        "to invalid NUMA node %d\n", start,
 320                                        numa_domain);
 321                         numa_domain = 0;
 322                 }
 323
 324                 node_set_online(numa_domain);
 325
 326                 if (max_domain < numa_domain)
 327                         max_domain = numa_domain;
 328
 329                 /*
 330                  * For backwards compatibility, OF splits the first node
 331                  * into two regions (the first being 0-4GB). Check for
 332                  * this simple case and complain if there is a gap in
 333                  * memory
 334                  */
 335                 if (init_node_data[numa_domain].node_spanned_pages) {
 336                         unsigned long shouldstart =
 337                                 init_node_data[numa_domain].node_start_pfn +
 338                                 init_node_data[numa_domain].node_spanned_pages;
 339                         if (shouldstart != (start / PAGE_SIZE)) {
 340                                 printk(KERN_ERR "WARNING: Hole in node, "
 341                                                 "disabling region start %lx "
 342                                                 "length %lx\n", start, size);
 343                                 continue;
 344                         }
 345                         init_node_data[numa_domain].node_spanned_pages +=
 346                                 size / PAGE_SIZE;
 347                 } else {
 348                         init_node_data[numa_domain].node_start_pfn =
 349                                 start / PAGE_SIZE;
 350                         init_node_data[numa_domain].node_spanned_pages =
 351                                 size / PAGE_SIZE;
 352                 }
 353
 354                 for (i = start ; i < (start+size); i += MEMORY_INCREMENT)
 355                         numa_memory_lookup_table[i >> MEMORY_INCREMENT_SHIFT] =
 356                                 numa_domain;
 357
 358                 ranges--;
 359                 if (ranges)
 360                         goto new_range;
 361         }
 362
 363         numnodes = max_domain + 1;
 364
 365         return 0;
 366 }
 367
 368 static void __init setup_nonnuma(void)
 369 {
 370         unsigned long top_of_ram = lmb_end_of_DRAM();
 371         unsigned long total_ram = lmb_phys_mem_size();
 372         unsigned long i;
 373
 374         printk(KERN_INFO "Top of RAM: 0x%lx, Total RAM: 0x%lx\n",
 375                top_of_ram, total_ram);
 376         printk(KERN_INFO "Memory hole size: %ldMB\n",
 377                (top_of_ram - total_ram) >> 20);
 378
 379         if (!numa_memory_lookup_table) {
 380                 long entries = top_of_ram >> MEMORY_INCREMENT_SHIFT;
 381                 numa_memory_lookup_table =
 382                         (char *)abs_to_virt(lmb_alloc(entries * sizeof(char), 1));
 383                 memset(numa_memory_lookup_table, 0, entries * sizeof(char));
 384                 for (i = 0; i < entries ; i++)
 385                         numa_memory_lookup_table[i] = ARRAY_INITIALISER;
 386         }
 387
 388         map_cpu_to_node(boot_cpuid, 0);
 389
 390         node_set_online(0);
 391
 392         init_node_data[0].node_start_pfn = 0;
 393         init_node_data[0].node_spanned_pages = lmb_end_of_DRAM() / PAGE_SIZE;
 394
 395         for (i = 0 ; i < top_of_ram; i += MEMORY_INCREMENT)
 396                 numa_memory_lookup_table[i >> MEMORY_INCREMENT_SHIFT] = 0;
 397
 398         node0_io_hole_size = top_of_ram - total_ram;
 399 }
 400
 401 static void __init dump_numa_topology(void)
 402 {
 403         unsigned int node;
 404         unsigned int count;
 405
 406         if (min_common_depth == -1 || !numa_enabled)
 407                 return;
 408
 409         for (node = 0; node < MAX_NUMNODES; node++) {
 410                 unsigned long i;
 411
 412                 if (!node_online(node))
 413                         continue;
 414
 415                 printk(KERN_INFO "Node %d Memory:", node);
 416
 417                 count = 0;
 418
 419                 for (i = 0; i < lmb_end_of_DRAM(); i += MEMORY_INCREMENT) {
 420                         if (numa_memory_lookup_table[i >> MEMORY_INCREMENT_SHIFT] == node) {
 421                                 if (count == 0)
 422                                         printk(" 0x%lx", i);
 423                                 ++count;
 424                         } else {
 425                                 if (count > 0)
 426                                         printk("-0x%lx", i);
 427                                 count = 0;
 428                         }
 429                 }
 430
 431                 if (count > 0)
 432                         printk("-0x%lx", i);
 433                 printk("\n");
 434         }
 435         return;
 436 }
 437
 438 /*
 439  * Allocate some memory, satisfying the lmb or bootmem allocator where
 440  * required. nid is the preferred node and end is the physical address of
 441  * the highest address in the node.
 442  *
 443  * Returns the physical address of the memory.
 444  */
 445 static unsigned long careful_allocation(int nid, unsigned long size,
 446                                         unsigned long align, unsigned long end)
 447 {
 448         unsigned long ret = lmb_alloc_base(size, align, end);
 449
 450         /* retry over all memory */
 451         if (!ret)
 452                 ret = lmb_alloc_base(size, align, lmb_end_of_DRAM());
 453
 454         if (!ret)
 455                 panic("numa.c: cannot allocate %lu bytes on node %d",
 456                       size, nid);
 457
 458         /*
 459          * If the memory came from a previously allocated node, we must
 460          * retry with the bootmem allocator.
 461          */
 462         if (pa_to_nid(ret) < nid) {
 463                 nid = pa_to_nid(ret);
 464                 ret = (unsigned long)__alloc_bootmem_node(NODE_DATA(nid),
 465                                 size, align, 0);
 466
 467                 if (!ret)
 468                         panic("numa.c: cannot allocate %lu bytes on node %d",
 469                               size, nid);
 470
 471                 ret = virt_to_abs(ret);
 472
 473                 dbg("alloc_bootmem %lx %lx\n", ret, size);
 474         }
 475
 476         return ret;
 477 }
 478
 479 void __init do_init_bootmem(void)
 480 {
 481         int nid;
 482
 483         min_low_pfn = 0;
 484         max_low_pfn = lmb_end_of_DRAM() >> PAGE_SHIFT;
 485         max_pfn = max_low_pfn;
 486
 487         if (parse_numa_properties())
 488                 setup_nonnuma();
 489         else
 490                 dump_numa_topology();
 491         /*
 492          * This must run before the sched domains notifier.
 493          */
 494         hotcpu_notifier(cpu_numa_callback, 1);
 495
 496         for (nid = 0; nid < numnodes; nid++) {
 497                 unsigned long start_paddr, end_paddr;
 498                 int i;
 499                 unsigned long bootmem_paddr;
 500                 unsigned long bootmap_pages;
 501
 502                 start_paddr = init_node_data[nid].node_start_pfn * PAGE_SIZE;
 503                 end_paddr = start_paddr + (init_node_data[nid].node_spanned_pages * PAGE_SIZE);
 504
 505                 /* Allocate the node structure node local if possible */
 506                 NODE_DATA(nid) = (struct pglist_data *)careful_allocation(nid,
 507                                         sizeof(struct pglist_data),
 508                                         SMP_CACHE_BYTES, end_paddr);
 509                 NODE_DATA(nid) = abs_to_virt(NODE_DATA(nid));
 510                 memset(NODE_DATA(nid), 0, sizeof(struct pglist_data));
 511
 512                 dbg("node %d\n", nid);
 513                 dbg("NODE_DATA() = %p\n", NODE_DATA(nid));
 514
 515                 NODE_DATA(nid)->bdata = &plat_node_bdata[nid];
 516                 NODE_DATA(nid)->node_start_pfn =
 517                         init_node_data[nid].node_start_pfn;
 518                 NODE_DATA(nid)->node_spanned_pages =
 519                         init_node_data[nid].node_spanned_pages;
 520
 521                 if (init_node_data[nid].node_spanned_pages == 0)
 522                         continue;
 523
 524                 dbg("start_paddr = %lx\n", start_paddr);
 525                 dbg("end_paddr = %lx\n", end_paddr);
 526
 527                 bootmap_pages = bootmem_bootmap_pages((end_paddr - start_paddr) >> PAGE_SHIFT);
 528
 529                 bootmem_paddr = careful_allocation(nid,
 530                                 bootmap_pages << PAGE_SHIFT,
 531                                 PAGE_SIZE, end_paddr);
 532                 memset(abs_to_virt(bootmem_paddr), 0,
 533                        bootmap_pages << PAGE_SHIFT);
 534                 dbg("bootmap_paddr = %lx\n", bootmem_paddr);
 535
 536                 init_bootmem_node(NODE_DATA(nid), bootmem_paddr >> PAGE_SHIFT,
 537                                   start_paddr >> PAGE_SHIFT,
 538                                   end_paddr >> PAGE_SHIFT);
 539
 540                 for (i = 0; i < lmb.memory.cnt; i++) {
 541                         unsigned long physbase, size;
 542
 543                         physbase = lmb.memory.region[i].physbase;
 544                         size = lmb.memory.region[i].size;
 545
 546                         if (physbase < end_paddr &&
 547                             (physbase+size) > start_paddr) {
 548                                 /* overlaps */
 549                                 if (physbase < start_paddr) {
 550                                         size -= start_paddr - physbase;
 551                                         physbase = start_paddr;
 552                                 }
 553
 554                                 if (size > end_paddr - physbase)
 555                                         size = end_paddr - physbase;
 556
 557                                 dbg("free_bootmem %lx %lx\n", physbase, size);
 558                                 free_bootmem_node(NODE_DATA(nid), physbase,
 559                                                   size);
 560                         }
 561                 }
 562
 563                 for (i = 0; i < lmb.reserved.cnt; i++) {
 564                         unsigned long physbase = lmb.reserved.region[i].physbase;
 565                         unsigned long size = lmb.reserved.region[i].size;
 566
 567                         if (physbase < end_paddr &&
 568                             (physbase+size) > start_paddr) {
 569                                 /* overlaps */
 570                                 if (physbase < start_paddr) {
 571                                         size -= start_paddr - physbase;
 572                                         physbase = start_paddr;
 573                                 }
 574
 575                                 if (size > end_paddr - physbase)
 576                                         size = end_paddr - physbase;
 577
 578                                 dbg("reserve_bootmem %lx %lx\n", physbase,
 579                                     size);
 580                                 reserve_bootmem_node(NODE_DATA(nid), physbase,
 581                                                      size);
 582                         }
 583                 }
 584         }
 585 }
 586
 587 void __init paging_init(void)
 588 {
 589         unsigned long zones_size[MAX_NR_ZONES];
 590         unsigned long zholes_size[MAX_NR_ZONES];
 591         int nid;
 592
 593         memset(zones_size, 0, sizeof(zones_size));
 594         memset(zholes_size, 0, sizeof(zholes_size));
 595
 596         for (nid = 0; nid < numnodes; nid++) {
 597                 unsigned long start_pfn;
 598                 unsigned long end_pfn;
 599
 600                 start_pfn = plat_node_bdata[nid].node_boot_start >> PAGE_SHIFT;
 601                 end_pfn = plat_node_bdata[nid].node_low_pfn;
 602
 603                 zones_size[ZONE_DMA] = end_pfn - start_pfn;
 604                 zholes_size[ZONE_DMA] = 0;
 605                 if (nid == 0)
 606                         zholes_size[ZONE_DMA] = node0_io_hole_size >> PAGE_SHIFT;
 607
 608                 dbg("free_area_init node %d %lx %lx (hole: %lx)\n", nid,
 609                     zones_size[ZONE_DMA], start_pfn, zholes_size[ZONE_DMA]);
 610
 611                 free_area_init_node(nid, NODE_DATA(nid), zones_size,
 612                                                         start_pfn, zholes_size);
 613         }
 614 }
 615
 616 static int __init early_numa(char *p)
 617 {
 618         if (!p)
 619                 return 0;
 620
 621         if (strstr(p, "off"))
 622                 numa_enabled = 0;
 623
 624         if (strstr(p, "debug"))
 625                 numa_debug = 1;
 626
 627         return 0;
 628 }
 629 early_param("numa", early_numa);