/*
- * Copyright (C) 2001-2005 Silicon Graphics, Inc. All rights reserved.
+ * Copyright (C) 2001-2006 Silicon Graphics, Inc. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms of version 2 of the GNU General Public License
#include <asm/tlbflush.h>
#include <asm/sn/arch.h>
-#define DEBUG 0
-#if DEBUG
-#define dprintk printk
-#else
-#define dprintk(x...) do { } while (0)
-#endif
+extern void __init efi_memmap_walk_uc(efi_freemem_callback_t, void *);
-void __init efi_memmap_walk_uc (efi_freemem_callback_t callback);
+struct uncached_pool {
+ struct gen_pool *pool;
+ struct mutex add_chunk_mutex; /* serialize adding a converted chunk */
+ int nchunks_added; /* #of converted chunks added to pool */
+ atomic_t status; /* smp called function's return status*/
+};
-#define MAX_UNCACHED_GRANULES 5
-static int allocated_granules;
+#define MAX_CONVERTED_CHUNKS_PER_NODE 2
-struct gen_pool *uncached_pool[MAX_NUMNODES];
+struct uncached_pool uncached_pools[MAX_NUMNODES];
static void uncached_ipi_visibility(void *data)
{
int status;
+ struct uncached_pool *uc_pool = (struct uncached_pool *)data;
status = ia64_pal_prefetch_visibility(PAL_VISIBILITY_PHYSICAL);
if ((status != PAL_VISIBILITY_OK) &&
(status != PAL_VISIBILITY_OK_REMOTE_NEEDED))
- printk(KERN_DEBUG "pal_prefetch_visibility() returns %i on "
- "CPU %i\n", status, raw_smp_processor_id());
+ atomic_inc(&uc_pool->status);
}
static void uncached_ipi_mc_drain(void *data)
{
int status;
+ struct uncached_pool *uc_pool = (struct uncached_pool *)data;
+
status = ia64_pal_mc_drain();
- if (status)
- printk(KERN_WARNING "ia64_pal_mc_drain() failed with %i on "
- "CPU %i\n", status, raw_smp_processor_id());
+ if (status != PAL_STATUS_SUCCESS)
+ atomic_inc(&uc_pool->status);
}
-static unsigned long
-uncached_get_new_chunk(struct gen_pool *poolp)
+/*
+ * Add a new chunk of uncached memory pages to the specified pool.
+ *
+ * @pool: pool to add new chunk of uncached memory to
+ * @nid: node id of node to allocate memory from, or -1
+ *
+ * This is accomplished by first allocating a granule of cached memory pages
+ * and then converting them to uncached memory pages.
+ */
+static int uncached_add_chunk(struct uncached_pool *uc_pool, int nid)
{
struct page *page;
- void *tmp;
- int status, i;
- unsigned long addr, node;
+ int status, i, nchunks_added = uc_pool->nchunks_added;
+ unsigned long c_addr, uc_addr;
- if (allocated_granules >= MAX_UNCACHED_GRANULES)
+ if (mutex_lock_interruptible(&uc_pool->add_chunk_mutex) != 0)
+ return -1; /* interrupted by a signal */
+
+ if (uc_pool->nchunks_added > nchunks_added) {
+ /* someone added a new chunk while we were waiting */
+ mutex_unlock(&uc_pool->add_chunk_mutex);
return 0;
+ }
+
+ if (uc_pool->nchunks_added >= MAX_CONVERTED_CHUNKS_PER_NODE) {
+ mutex_unlock(&uc_pool->add_chunk_mutex);
+ return -1;
+ }
+
+ /* attempt to allocate a granule's worth of cached memory pages */
- node = poolp->private;
- page = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO,
+ page = alloc_pages_node(nid, GFP_KERNEL | __GFP_ZERO | GFP_THISNODE,
IA64_GRANULE_SHIFT-PAGE_SHIFT);
+ if (!page) {
+ mutex_unlock(&uc_pool->add_chunk_mutex);
+ return -1;
+ }
- dprintk(KERN_INFO "get_new_chunk page %p, addr %lx\n",
- page, (unsigned long)(page-vmem_map) << PAGE_SHIFT);
+ /* convert the memory pages from cached to uncached */
- /*
- * Do magic if no mem on local node! XXX
- */
- if (!page)
- return 0;
- tmp = page_address(page);
+ c_addr = (unsigned long)page_address(page);
+ uc_addr = c_addr - PAGE_OFFSET + __IA64_UNCACHED_OFFSET;
/*
* There's a small race here where it's possible for someone to
for (i = 0; i < (IA64_GRANULE_SIZE / PAGE_SIZE); i++)
SetPageUncached(&page[i]);
- flush_tlb_kernel_range(tmp, tmp + IA64_GRANULE_SIZE);
+ flush_tlb_kernel_range(uc_addr, uc_adddr + IA64_GRANULE_SIZE);
status = ia64_pal_prefetch_visibility(PAL_VISIBILITY_PHYSICAL);
+ if (status == PAL_VISIBILITY_OK_REMOTE_NEEDED) {
+ atomic_set(&uc_pool->status, 0);
+ status = smp_call_function(uncached_ipi_visibility, uc_pool,
+ 0, 1);
+ if (status || atomic_read(&uc_pool->status))
+ goto failed;
+ } else if (status != PAL_VISIBILITY_OK)
+ goto failed;
- dprintk(KERN_INFO "pal_prefetch_visibility() returns %i on cpu %i\n",
- status, raw_smp_processor_id());
-
- if (!status) {
- status = smp_call_function(uncached_ipi_visibility, NULL, 0, 1);
- if (status)
- printk(KERN_WARNING "smp_call_function failed for "
- "uncached_ipi_visibility! (%i)\n", status);
- }
+ preempt_disable();
if (ia64_platform_is("sn2"))
- sn_flush_all_caches((unsigned long)tmp, IA64_GRANULE_SIZE);
+ sn_flush_all_caches(uc_addr, IA64_GRANULE_SIZE);
else
- flush_icache_range((unsigned long)tmp,
- (unsigned long)tmp+IA64_GRANULE_SIZE);
+ flush_icache_range(uc_addr, uc_addr + IA64_GRANULE_SIZE);
- ia64_pal_mc_drain();
- status = smp_call_function(uncached_ipi_mc_drain, NULL, 0, 1);
+ /* flush the just introduced uncached translation from the TLB */
+ local_flush_tlb_all();
+
+ preempt_enable();
+
+ status = ia64_pal_mc_drain();
+ if (status != PAL_STATUS_SUCCESS)
+ goto failed;
+ atomic_set(&uc_pool->status, 0);
+ status = smp_call_function(uncached_ipi_mc_drain, uc_pool, 0, 1);
+ if (status || atomic_read(&uc_pool->status))
+ goto failed;
+
+ /*
+ * The chunk of memory pages has been converted to uncached so now we
+ * can add it to the pool.
+ */
+ status = gen_pool_add(uc_pool->pool, uc_addr, IA64_GRANULE_SIZE, nid);
if (status)
- printk(KERN_WARNING "smp_call_function failed for "
- "uncached_ipi_mc_drain! (%i)\n", status);
+ goto failed;
+
+ uc_pool->nchunks_added++;
+ mutex_unlock(&uc_pool->add_chunk_mutex);
+ return 0;
- addr = (unsigned long)tmp - PAGE_OFFSET + __IA64_UNCACHED_OFFSET;
+ /* failed to convert or add the chunk so give it back to the kernel */
+failed:
+ for (i = 0; i < (IA64_GRANULE_SIZE / PAGE_SIZE); i++)
+ ClearPageUncached(&page[i]);
- allocated_granules++;
- return addr;
+ free_pages(c_addr, IA64_GRANULE_SHIFT-PAGE_SHIFT);
+ mutex_unlock(&uc_pool->add_chunk_mutex);
+ return -1;
}
/*
* uncached_alloc_page
*
+ * @starting_nid: node id of node to start with, or -1
+ *
* Allocate 1 uncached page. Allocates on the requested node. If no
* uncached pages are available on the requested node, roundrobin starting
- * with higher nodes.
+ * with the next higher node.
*/
-unsigned long
-uncached_alloc_page(int nid)
+unsigned long uncached_alloc_page(int starting_nid)
{
- unsigned long maddr;
+ unsigned long uc_addr;
+ struct uncached_pool *uc_pool;
+ int nid;
- maddr = gen_pool_alloc(uncached_pool[nid], PAGE_SIZE);
+ if (unlikely(starting_nid >= MAX_NUMNODES))
+ return 0;
- dprintk(KERN_DEBUG "uncached_alloc_page returns %lx on node %i\n",
- maddr, nid);
+ if (starting_nid < 0)
+ starting_nid = numa_node_id();
+ nid = starting_nid;
- /*
- * If no memory is availble on our local node, try the
- * remaining nodes in the system.
- */
- if (!maddr) {
- int i;
-
- for (i = MAX_NUMNODES - 1; i >= 0; i--) {
- if (i == nid || !node_online(i))
- continue;
- maddr = gen_pool_alloc(uncached_pool[i], PAGE_SIZE);
- dprintk(KERN_DEBUG "uncached_alloc_page alternate search "
- "returns %lx on node %i\n", maddr, i);
- if (maddr) {
- break;
- }
- }
- }
+ do {
+ if (!node_online(nid))
+ continue;
+ uc_pool = &uncached_pools[nid];
+ if (uc_pool->pool == NULL)
+ continue;
+ do {
+ uc_addr = gen_pool_alloc(uc_pool->pool, PAGE_SIZE);
+ if (uc_addr != 0)
+ return uc_addr;
+ } while (uncached_add_chunk(uc_pool, nid) == 0);
- return maddr;
+ } while ((nid = (nid + 1) % MAX_NUMNODES) != starting_nid);
+
+ return 0;
}
EXPORT_SYMBOL(uncached_alloc_page);
/*
* uncached_free_page
*
+ * @uc_addr: uncached address of page to free
+ *
* Free a single uncached page.
*/
-void
-uncached_free_page(unsigned long maddr)
+void uncached_free_page(unsigned long uc_addr)
{
- int node;
+ int nid = paddr_to_nid(uc_addr - __IA64_UNCACHED_OFFSET);
+ struct gen_pool *pool = uncached_pools[nid].pool;
- node = paddr_to_nid(maddr - __IA64_UNCACHED_OFFSET);
+ if (unlikely(pool == NULL))
+ return;
- dprintk(KERN_DEBUG "uncached_free_page(%lx) on node %i\n", maddr, node);
+ if ((uc_addr & (0XFUL << 60)) != __IA64_UNCACHED_OFFSET)
+ panic("uncached_free_page invalid address %lx\n", uc_addr);
- if ((maddr & (0XFUL << 60)) != __IA64_UNCACHED_OFFSET)
- panic("uncached_free_page invalid address %lx\n", maddr);
-
- gen_pool_free(uncached_pool[node], maddr, PAGE_SIZE);
+ gen_pool_free(pool, uc_addr, PAGE_SIZE);
}
EXPORT_SYMBOL(uncached_free_page);
/*
* uncached_build_memmap,
*
+ * @uc_start: uncached starting address of a chunk of uncached memory
+ * @uc_end: uncached ending address of a chunk of uncached memory
+ * @arg: ignored, (NULL argument passed in on call to efi_memmap_walk_uc())
+ *
* Called at boot time to build a map of pages that can be used for
* memory special operations.
*/
-static int __init
-uncached_build_memmap(unsigned long start, unsigned long end, void *arg)
+static int __init uncached_build_memmap(unsigned long uc_start,
+ unsigned long uc_end, void *arg)
{
- long length = end - start;
- int node;
-
- dprintk(KERN_ERR "uncached_build_memmap(%lx %lx)\n", start, end);
+ int nid = paddr_to_nid(uc_start - __IA64_UNCACHED_OFFSET);
+ struct gen_pool *pool = uncached_pools[nid].pool;
+ size_t size = uc_end - uc_start;
touch_softlockup_watchdog();
- memset((char *)start, 0, length);
- node = paddr_to_nid(start - __IA64_UNCACHED_OFFSET);
-
- for (; start < end ; start += PAGE_SIZE) {
- dprintk(KERN_INFO "sticking %lx into the pool!\n", start);
- gen_pool_free(uncached_pool[node], start, PAGE_SIZE);
+ if (pool != NULL) {
+ memset((char *)uc_start, 0, size);
+ (void) gen_pool_add(pool, uc_start, size, nid);
}
-
return 0;
}
-static int __init uncached_init(void) {
- int i;
+static int __init uncached_init(void)
+{
+ int nid;
- for (i = 0; i < MAX_NUMNODES; i++) {
- if (!node_online(i))
- continue;
- uncached_pool[i] = gen_pool_create(0, IA64_GRANULE_SHIFT,
- &uncached_get_new_chunk, i);
+ for_each_online_node(nid) {
+ uncached_pools[nid].pool = gen_pool_create(PAGE_SHIFT, nid);
+ mutex_init(&uncached_pools[nid].add_chunk_mutex);
}
- efi_memmap_walk_uc(uncached_build_memmap);
-
+ efi_memmap_walk_uc(uncached_build_memmap, NULL);
return 0;
}
git://git.onelab.eu / linux-2.6.git / blobdiff