2 * String handling functions for PowerPC.
4 * Copyright (C) 1996 Paul Mackerras.
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
11 #include <linux/config.h>
12 #include <asm/processor.h>
13 #include <asm/cache.h>
14 #include <asm/errno.h>
15 #include <asm/ppc_asm.h>
17 #define COPY_16_BYTES \
27 #define COPY_16_BYTES_WITHEX(n) \
45 #define COPY_16_BYTES_EXCODE(n) \
47 addi r5,r5,-(16 * n); \
50 addi r5,r5,-(16 * n); \
52 .section __ex_table,"a"; \
54 .long 8 ## n ## 0b,9 ## n ## 0b; \
55 .long 8 ## n ## 1b,9 ## n ## 0b; \
56 .long 8 ## n ## 2b,9 ## n ## 0b; \
57 .long 8 ## n ## 3b,9 ## n ## 0b; \
58 .long 8 ## n ## 4b,9 ## n ## 1b; \
59 .long 8 ## n ## 5b,9 ## n ## 1b; \
60 .long 8 ## n ## 6b,9 ## n ## 1b; \
61 .long 8 ## n ## 7b,9 ## n ## 1b; \
65 .stabs "arch/ppc/lib/",N_SO,0,0,0f
66 .stabs "string.S",N_SO,0,0,0f
68 CACHELINE_BYTES = L1_CACHE_LINE_SIZE
69 LG_CACHELINE_BYTES = LG_L1_CACHE_LINE_SIZE
70 CACHELINE_MASK = (L1_CACHE_LINE_SIZE-1)
81 /* This clears out any unused part of the destination buffer,
82 just as the libc version does. -- paulus */
92 bdnzf 2,1b /* dec ctr, branch if ctr != 0 && !cr0.eq */
93 bnelr /* if we didn't hit a null char, we're done */
95 cmpwi 0,r5,0 /* any space left in destination buffer? */
96 beqlr /* we know r0 == 0 here */
97 2: stbu r0,1(r6) /* clear it out if so */
134 * Use dcbz on the complete cache lines in the destination
135 * to set them to zero. This requires that the destination
136 * area is cacheable. -- paulus
138 _GLOBAL(cacheable_memzero)
149 clrlwi r7,r6,32-LG_CACHELINE_BYTES
151 srwi r9,r8,LG_CACHELINE_BYTES
152 addic. r9,r9,-1 /* total number of complete cachelines */
154 xori r0,r7,CACHELINE_MASK & ~3
162 #if !defined(CONFIG_8xx)
169 #if CACHE_LINE_SIZE >= 32
174 #endif /* CACHE_LINE_SIZE */
176 addi r6,r6,CACHELINE_BYTES
178 clrlwi r5,r8,32-LG_CACHELINE_BYTES
226 * This version uses dcbz on the complete cache lines in the
227 * destination area to reduce memory traffic. This requires that
228 * the destination area is cacheable.
229 * We only use this version if the source and dest don't overlap.
232 _GLOBAL(cacheable_memcpy)
233 add r7,r3,r5 /* test if the src & dst overlap */
237 crand 0,0,4 /* cr0.lt &= cr1.lt */
238 blt memcpy /* if regions overlap */
243 andi. r0,r0,CACHELINE_MASK /* # bytes to start of cache line */
246 cmplw 0,r5,r0 /* is this more than total to do? */
247 blt 63f /* if not much to do */
248 andi. r8,r0,3 /* get it word-aligned first */
252 70: lbz r9,4(r4) /* do some bytes */
260 72: lwzu r9,4(r4) /* do some words */
264 58: srwi. r0,r5,LG_CACHELINE_BYTES /* # complete cachelines */
265 clrlwi r5,r5,32-LG_CACHELINE_BYTES
270 #if !defined(CONFIG_8xx)
274 #if L1_CACHE_LINE_SIZE >= 32
276 #if L1_CACHE_LINE_SIZE >= 64
279 #if L1_CACHE_LINE_SIZE >= 128
315 beq 2f /* if less than 8 bytes to do */
316 andi. r0,r6,3 /* get dest word aligned */
347 rlwinm. r7,r5,32-3,3,31
352 _GLOBAL(backwards_memcpy)
353 rlwinm. r7,r5,32-3,3,31 /* r0 = r5 >> 3 */
383 rlwinm. r7,r5,32-3,3,31
414 _GLOBAL(__copy_tofrom_user)
418 andi. r0,r0,CACHELINE_MASK /* # bytes to start of cache line */
421 cmplw 0,r5,r0 /* is this more than total to do? */
422 blt 63f /* if not much to do */
423 andi. r8,r0,3 /* get it word-aligned first */
426 70: lbz r9,4(r4) /* do some bytes */
435 72: lwzu r9,4(r4) /* do some words */
439 58: srwi. r0,r5,LG_CACHELINE_BYTES /* # complete cachelines */
440 clrlwi r5,r5,32-LG_CACHELINE_BYTES
444 #if !defined(CONFIG_8xx)
445 /* Here we decide how far ahead to prefetch the source */
446 #if MAX_COPY_PREFETCH > 1
447 /* Heuristically, for large transfers we prefetch
448 MAX_COPY_PREFETCH cachelines ahead. For small transfers
449 we prefetch 1 cacheline ahead. */
450 cmpwi r0,MAX_COPY_PREFETCH
454 li r7,MAX_COPY_PREFETCH
457 addi r3,r3,CACHELINE_BYTES
459 #else /* MAX_COPY_PREFETCH == 1 */
460 li r3,CACHELINE_BYTES + 4
462 #endif /* MAX_COPY_PREFETCH */
463 #endif /* CONFIG_8xx */
467 #if !defined(CONFIG_8xx)
471 /* had to move these to keep extable in order */
472 .section __ex_table,"a"
478 #if !defined(CONFIG_8xx)
482 /* the main body of the cacheline loop */
483 COPY_16_BYTES_WITHEX(0)
484 #if L1_CACHE_LINE_SIZE >= 32
485 COPY_16_BYTES_WITHEX(1)
486 #if L1_CACHE_LINE_SIZE >= 64
487 COPY_16_BYTES_WITHEX(2)
488 COPY_16_BYTES_WITHEX(3)
489 #if L1_CACHE_LINE_SIZE >= 128
490 COPY_16_BYTES_WITHEX(4)
491 COPY_16_BYTES_WITHEX(5)
492 COPY_16_BYTES_WITHEX(6)
493 COPY_16_BYTES_WITHEX(7)
517 /* read fault, initial single-byte copy */
520 /* write fault, initial single-byte copy */
525 /* read fault, initial word copy */
528 /* write fault, initial word copy */
534 * this stuff handles faults in the cacheline loop and branches to either
535 * 104f (if in read part) or 105f (if in write part), after updating r5
537 COPY_16_BYTES_EXCODE(0)
538 #if L1_CACHE_LINE_SIZE >= 32
539 COPY_16_BYTES_EXCODE(1)
540 #if L1_CACHE_LINE_SIZE >= 64
541 COPY_16_BYTES_EXCODE(2)
542 COPY_16_BYTES_EXCODE(3)
543 #if L1_CACHE_LINE_SIZE >= 128
544 COPY_16_BYTES_EXCODE(4)
545 COPY_16_BYTES_EXCODE(5)
546 COPY_16_BYTES_EXCODE(6)
547 COPY_16_BYTES_EXCODE(7)
552 /* read fault in cacheline loop */
555 /* fault on dcbz (effectively a write fault) */
556 /* or write fault in cacheline loop */
558 92: li r3,LG_CACHELINE_BYTES
560 /* read fault in final word loop */
563 /* write fault in final word loop */
568 /* read fault in final byte loop */
571 /* write fault in final byte loop */
576 * At this stage the number of bytes not copied is
577 * r5 + (ctr << r3), and r9 is 0 for read or 1 for write.
582 beq 120f /* shouldn't happen */
585 /* for a read fault, first try to continue the copy one byte at a time */
592 /* then clear out the destination: r3 bytes starting at 4(r6) */
608 .section __ex_table,"a"
620 _GLOBAL(__clear_user)
626 /* clear a single word */
629 /* clear word sized chunks */
639 /* clear byte sized chunks */
656 .section __ex_table,"a"
663 _GLOBAL(__strncpy_from_user)
672 bdnzf 2,1b /* dec ctr, branch if ctr != 0 && !cr0.eq */
680 .section __ex_table,"a"
685 /* r3 = str, r4 = len (> 0), r5 = top (highest addr) */
686 _GLOBAL(__strnlen_user)
688 subf r6,r7,r5 /* top+1 - str */
692 0: mtctr r6 /* ctr = min(len, top - str) */
693 1: lbzu r0,1(r7) /* get next byte */
695 bdnzf 2,1b /* loop if --ctr != 0 && byte != 0 */
697 subf r3,r3,r7 /* number of bytes we have looked at */
698 beqlr /* return if we found a 0 byte */
699 cmpw 0,r3,r4 /* did we look at all len bytes? */
700 blt 99f /* if not, must have hit top */
701 addi r3,r4,1 /* return len + 1 to indicate no null found */
703 99: li r3,0 /* bad address, return 0 */
706 .section __ex_table,"a"