X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=arch%2Fia64%2Fsn%2Fkernel%2Fbte.c;h=ff1c55601178b2c2ff7071b26f03bf64f5638865;hb=refs%2Fheads%2Fvserver;hp=e040fc3c445aa4bd122643cb779cd595b6eaabad;hpb=9bf4aaab3e101692164d49b7ca357651eb691cb6;p=linux-2.6.git diff --git a/arch/ia64/sn/kernel/bte.c b/arch/ia64/sn/kernel/bte.c index e040fc3c4..ff1c55601 100644 --- a/arch/ia64/sn/kernel/bte.c +++ b/arch/ia64/sn/kernel/bte.c @@ -3,19 +3,18 @@ * License. See the file "COPYING" in the main directory of this archive * for more details. * - * Copyright (c) 2000-2003 Silicon Graphics, Inc. All Rights Reserved. + * Copyright (c) 2000-2006 Silicon Graphics, Inc. All Rights Reserved. */ -#include #include -#include #include #include #include #include #include -#include +#include #include +#include #include #include @@ -29,24 +28,35 @@ /* two interfaces on two btes */ #define MAX_INTERFACES_TO_TRY 4 +#define MAX_NODES_TO_TRY 2 -static struct bteinfo_s * -bte_if_on_node(nasid_t nasid, int interface) +static struct bteinfo_s *bte_if_on_node(nasid_t nasid, int interface) { nodepda_t *tmp_nodepda; + if (nasid_to_cnodeid(nasid) == -1) + return (struct bteinfo_s *)NULL; + tmp_nodepda = NODEPDA(nasid_to_cnodeid(nasid)); return &tmp_nodepda->bte_if[interface]; } +static inline void bte_start_transfer(struct bteinfo_s *bte, u64 len, u64 mode) +{ + if (is_shub2()) { + BTE_CTRL_STORE(bte, (IBLS_BUSY | ((len) | (mode) << 24))); + } else { + BTE_LNSTAT_STORE(bte, len); + BTE_CTRL_STORE(bte, mode); + } +} /************************************************************************ * Block Transfer Engine copy related functions. * ***********************************************************************/ - /* * bte_copy(src, dest, len, mode, notification) * @@ -66,17 +76,19 @@ bte_if_on_node(nasid_t nasid, int interface) * NOTE: This function requires src, dest, and len to * be cacheline aligned. */ -bte_result_t -bte_copy(u64 src, u64 dest, u64 len, u64 mode, void *notification) +bte_result_t bte_copy(u64 src, u64 dest, u64 len, u64 mode, void *notification) { u64 transfer_size; u64 transfer_stat; + u64 notif_phys_addr; struct bteinfo_s *bte; bte_result_t bte_status; unsigned long irq_flags; - struct bteinfo_s *btes_to_try[MAX_INTERFACES_TO_TRY]; - int bte_if_index; - + unsigned long itc_end = 0; + int nasid_to_try[MAX_NODES_TO_TRY]; + int my_nasid = cpuid_to_nasid(raw_smp_processor_id()); + int bte_if_index, nasid_index; + int bte_first, btes_per_node = BTES_PER_NODE; BTE_PRINTK(("bte_copy(0x%lx, 0x%lx, 0x%lx, 0x%lx, 0x%p)\n", src, dest, len, mode, notification)); @@ -85,58 +97,69 @@ bte_copy(u64 src, u64 dest, u64 len, u64 mode, void *notification) return BTE_SUCCESS; } - ASSERT(!((len & L1_CACHE_MASK) || - (src & L1_CACHE_MASK) || (dest & L1_CACHE_MASK))); - ASSERT(len < ((BTE_LEN_MASK + 1) << L1_CACHE_SHIFT)); + BUG_ON((len & L1_CACHE_MASK) || + (src & L1_CACHE_MASK) || (dest & L1_CACHE_MASK)); + BUG_ON(!(len < ((BTE_LEN_MASK + 1) << L1_CACHE_SHIFT))); + + /* + * Start with interface corresponding to cpu number + */ + bte_first = raw_smp_processor_id() % btes_per_node; if (mode & BTE_USE_DEST) { /* try remote then local */ - btes_to_try[0] = bte_if_on_node(NASID_GET(dest), 0); - btes_to_try[1] = bte_if_on_node(NASID_GET(dest), 1); + nasid_to_try[0] = NASID_GET(dest); if (mode & BTE_USE_ANY) { - btes_to_try[2] = bte_if_on_node(get_nasid(), 0); - btes_to_try[3] = bte_if_on_node(get_nasid(), 1); + nasid_to_try[1] = my_nasid; } else { - btes_to_try[2] = NULL; - btes_to_try[3] = NULL; + nasid_to_try[1] = (int)NULL; } } else { /* try local then remote */ - btes_to_try[0] = bte_if_on_node(get_nasid(), 0); - btes_to_try[1] = bte_if_on_node(get_nasid(), 1); + nasid_to_try[0] = my_nasid; if (mode & BTE_USE_ANY) { - btes_to_try[2] = bte_if_on_node(NASID_GET(dest), 0); - btes_to_try[3] = bte_if_on_node(NASID_GET(dest), 1); + nasid_to_try[1] = NASID_GET(dest); } else { - btes_to_try[2] = NULL; - btes_to_try[3] = NULL; + nasid_to_try[1] = (int)NULL; } } +retry_bteop: do { local_irq_save(irq_flags); - bte_if_index = 0; + bte_if_index = bte_first; + nasid_index = 0; /* Attempt to lock one of the BTE interfaces. */ - while (bte_if_index < MAX_INTERFACES_TO_TRY) { - bte = btes_to_try[bte_if_index++]; + while (nasid_index < MAX_NODES_TO_TRY) { + bte = bte_if_on_node(nasid_to_try[nasid_index],bte_if_index); if (bte == NULL) { + nasid_index++; continue; } if (spin_trylock(&bte->spinlock)) { - if ((*bte->most_rcnt_na & BTE_ACTIVE) || + if (!(*bte->most_rcnt_na & BTE_WORD_AVAILABLE) || (BTE_LNSTAT_LOAD(bte) & BTE_ACTIVE)) { /* Got the lock but BTE still busy */ spin_unlock(&bte->spinlock); - bte = NULL; } else { /* we got the lock and it's not busy */ break; } } + + bte_if_index = (bte_if_index + 1) % btes_per_node; /* Next interface */ + if (bte_if_index == bte_first) { + /* + * We've tried all interfaces on this node + */ + nasid_index++; + } + + bte = NULL; } if (bte != NULL) { @@ -150,7 +173,6 @@ bte_copy(u64 src, u64 dest, u64 len, u64 mode, void *notification) } } while (1); - if (notification == NULL) { /* User does not want to be notified. */ bte->most_rcnt_na = &bte->notify; @@ -162,55 +184,62 @@ bte_copy(u64 src, u64 dest, u64 len, u64 mode, void *notification) transfer_size = ((len >> L1_CACHE_SHIFT) & BTE_LEN_MASK); /* Initialize the notification to a known value. */ - *bte->most_rcnt_na = -1L; - - /* Set the status reg busy bit and transfer length */ - BTE_PRINTKV(("IBLS = 0x%lx\n", IBLS_BUSY | transfer_size)); - BTE_LNSTAT_STORE(bte, IBLS_BUSY | transfer_size); + *bte->most_rcnt_na = BTE_WORD_BUSY; + notif_phys_addr = (u64)bte->most_rcnt_na; /* Set the source and destination registers */ - BTE_PRINTKV(("IBSA = 0x%lx)\n", (TO_PHYS(src)))); - BTE_SRC_STORE(bte, TO_PHYS(src)); - BTE_PRINTKV(("IBDA = 0x%lx)\n", (TO_PHYS(dest)))); - BTE_DEST_STORE(bte, TO_PHYS(dest)); + BTE_PRINTKV(("IBSA = 0x%lx)\n", src)); + BTE_SRC_STORE(bte, src); + BTE_PRINTKV(("IBDA = 0x%lx)\n", dest)); + BTE_DEST_STORE(bte, dest); /* Set the notification register */ - BTE_PRINTKV(("IBNA = 0x%lx)\n", - TO_PHYS(ia64_tpa((unsigned long)bte->most_rcnt_na)))); - BTE_NOTIF_STORE(bte, TO_PHYS(ia64_tpa((unsigned long)bte->most_rcnt_na))); - + BTE_PRINTKV(("IBNA = 0x%lx)\n", notif_phys_addr)); + BTE_NOTIF_STORE(bte, notif_phys_addr); /* Initiate the transfer */ BTE_PRINTK(("IBCT = 0x%lx)\n", BTE_VALID_MODE(mode))); - BTE_CTRL_STORE(bte, BTE_VALID_MODE(mode)); + bte_start_transfer(bte, transfer_size, BTE_VALID_MODE(mode)); - spin_unlock_irqrestore(&bte->spinlock, irq_flags); + itc_end = ia64_get_itc() + (40000000 * local_cpu_data->cyc_per_usec); + spin_unlock_irqrestore(&bte->spinlock, irq_flags); if (notification != NULL) { return BTE_SUCCESS; } - while ((transfer_stat = *bte->most_rcnt_na) == -1UL) { + while ((transfer_stat = *bte->most_rcnt_na) == BTE_WORD_BUSY) { + cpu_relax(); + if (ia64_get_itc() > itc_end) { + BTE_PRINTK(("BTE timeout nasid 0x%x bte%d IBLS = 0x%lx na 0x%lx\n", + NASID_GET(bte->bte_base_addr), bte->bte_num, + BTE_LNSTAT_LOAD(bte), *bte->most_rcnt_na) ); + bte->bte_error_count++; + bte->bh_error = IBLS_ERROR; + bte_error_handler((unsigned long)NODEPDA(bte->bte_cnode)); + *bte->most_rcnt_na = BTE_WORD_AVAILABLE; + goto retry_bteop; + } } - BTE_PRINTKV((" Delay Done. IBLS = 0x%lx, most_rcnt_na = 0x%lx\n", - BTE_LNSTAT_LOAD(bte), *bte->most_rcnt_na)); + BTE_LNSTAT_LOAD(bte), *bte->most_rcnt_na)); if (transfer_stat & IBLS_ERROR) { bte_status = transfer_stat & ~IBLS_ERROR; - *bte->most_rcnt_na = 0L; } else { bte_status = BTE_SUCCESS; } + *bte->most_rcnt_na = BTE_WORD_AVAILABLE; + BTE_PRINTK(("Returning status is 0x%lx and most_rcnt_na is 0x%lx\n", - BTE_LNSTAT_LOAD(bte), *bte->most_rcnt_na)); + BTE_LNSTAT_LOAD(bte), *bte->most_rcnt_na)); return bte_status; } -EXPORT_SYMBOL(bte_copy); +EXPORT_SYMBOL(bte_copy); /* * bte_unaligned_copy(src, dest, len, mode) @@ -228,8 +257,7 @@ EXPORT_SYMBOL(bte_copy); * NOTE: If the source, dest, and len are all cache line aligned, * then it would be _FAR_ preferrable to use bte_copy instead. */ -bte_result_t -bte_unaligned_copy(u64 src, u64 dest, u64 len, u64 mode) +bte_result_t bte_unaligned_copy(u64 src, u64 dest, u64 len, u64 mode) { int destFirstCacheOffset; u64 headBteSource; @@ -249,12 +277,11 @@ bte_unaligned_copy(u64 src, u64 dest, u64 len, u64 mode) } /* temporary buffer used during unaligned transfers */ - bteBlock_unaligned = kmalloc(len + 3 * L1_CACHE_BYTES, - GFP_KERNEL | GFP_DMA); + bteBlock_unaligned = kmalloc(len + 3 * L1_CACHE_BYTES, GFP_KERNEL); if (bteBlock_unaligned == NULL) { return BTEFAIL_NOTAVAIL; } - bteBlock = (char *) L1_CACHE_ALIGN((u64) bteBlock_unaligned); + bteBlock = (char *)L1_CACHE_ALIGN((u64) bteBlock_unaligned); headBcopySrcOffset = src & L1_CACHE_MASK; destFirstCacheOffset = dest & L1_CACHE_MASK; @@ -302,15 +329,13 @@ bte_unaligned_copy(u64 src, u64 dest, u64 len, u64 mode) } if (len > headBcopyLen) { - footBcopyLen = - (len - headBcopyLen) & L1_CACHE_MASK; + footBcopyLen = (len - headBcopyLen) & L1_CACHE_MASK; footBteLen = L1_CACHE_BYTES; footBteSource = src + len - footBcopyLen; footBcopyDest = dest + len - footBcopyLen; - if (footBcopyDest == - (headBcopyDest + headBcopyLen)) { + if (footBcopyDest == (headBcopyDest + headBcopyLen)) { /* * We have two contigous bcopy * blocks. Merge them. @@ -326,9 +351,8 @@ bte_unaligned_copy(u64 src, u64 dest, u64 len, u64 mode) return rv; } - memcpy(__va(footBcopyDest), - (char *) bteBlock, footBcopyLen); + (char *)bteBlock, footBcopyLen); } } else { footBcopyLen = 0; @@ -350,7 +374,6 @@ bte_unaligned_copy(u64 src, u64 dest, u64 len, u64 mode) } } else { - /* * The transfer is not symetric, we will * allocate a buffer large enough for all the @@ -358,53 +381,48 @@ bte_unaligned_copy(u64 src, u64 dest, u64 len, u64 mode) * bcopy to the destination. */ - /* Add the leader from source */ - headBteLen = len + (src & L1_CACHE_MASK); - /* Add the trailing bytes from footer. */ - headBteLen += - L1_CACHE_BYTES - (headBteLen & L1_CACHE_MASK); - headBteSource = src & ~L1_CACHE_MASK; headBcopySrcOffset = src & L1_CACHE_MASK; headBcopyDest = dest; headBcopyLen = len; + + headBteSource = src - headBcopySrcOffset; + /* Add the leading and trailing bytes from source */ + headBteLen = L1_CACHE_ALIGN(len + headBcopySrcOffset); } if (headBcopyLen > 0) { rv = bte_copy(headBteSource, - ia64_tpa((unsigned long)bteBlock), headBteLen, mode, NULL); + ia64_tpa((unsigned long)bteBlock), headBteLen, + mode, NULL); if (rv != BTE_SUCCESS) { kfree(bteBlock_unaligned); return rv; } - memcpy(__va(headBcopyDest), ((char *) bteBlock + - headBcopySrcOffset), - headBcopyLen); + memcpy(__va(headBcopyDest), ((char *)bteBlock + + headBcopySrcOffset), headBcopyLen); } kfree(bteBlock_unaligned); return BTE_SUCCESS; } -EXPORT_SYMBOL(bte_unaligned_copy); +EXPORT_SYMBOL(bte_unaligned_copy); /************************************************************************ * Block Transfer Engine initialization functions. * ***********************************************************************/ - /* * bte_init_node(nodepda, cnode) * * Initialize the nodepda structure with BTE base addresses and * spinlocks. */ -void -bte_init_node(nodepda_t * mynodepda, cnodeid_t cnode) +void bte_init_node(nodepda_t * mynodepda, cnodeid_t cnode) { int i; - /* * Indicate that all the block transfer engines on this node * are available. @@ -418,12 +436,19 @@ bte_init_node(nodepda_t * mynodepda, cnodeid_t cnode) spin_lock_init(&mynodepda->bte_recovery_lock); init_timer(&mynodepda->bte_recovery_timer); mynodepda->bte_recovery_timer.function = bte_error_handler; - mynodepda->bte_recovery_timer.data = (unsigned long) mynodepda; + mynodepda->bte_recovery_timer.data = (unsigned long)mynodepda; for (i = 0; i < BTES_PER_NODE; i++) { - (u64) mynodepda->bte_if[i].bte_base_addr = - REMOTE_HUB_ADDR(cnodeid_to_nasid(cnode), - (i == 0 ? IIO_IBLS0 : IIO_IBLS1)); + u64 *base_addr; + + /* Which link status register should we use? */ + base_addr = (u64 *) + REMOTE_HUB_ADDR(cnodeid_to_nasid(cnode), BTE_BASE_ADDR(i)); + mynodepda->bte_if[i].bte_base_addr = base_addr; + mynodepda->bte_if[i].bte_source_addr = BTE_SOURCE_ADDR(base_addr); + mynodepda->bte_if[i].bte_destination_addr = BTE_DEST_ADDR(base_addr); + mynodepda->bte_if[i].bte_control_addr = BTE_CTRL_ADDR(base_addr); + mynodepda->bte_if[i].bte_notify_addr = BTE_NOTIF_ADDR(base_addr); /* * Initialize the notification and spinlock @@ -431,7 +456,7 @@ bte_init_node(nodepda_t * mynodepda, cnodeid_t cnode) */ mynodepda->bte_if[i].most_rcnt_na = &(mynodepda->bte_if[i].notify); - mynodepda->bte_if[i].notify = 0L; + mynodepda->bte_if[i].notify = BTE_WORD_AVAILABLE; spin_lock_init(&mynodepda->bte_if[i].spinlock); mynodepda->bte_if[i].bte_cnode = cnode;