1 /* U3memcpy.S: UltraSparc-III optimized memcpy.
3 * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)
7 #include <asm/visasm.h>
10 #define ASI_BLK_P 0xf0
13 #define VISEntryHalf rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs; \
14 clr %g1; clr %g2; clr %g3; subcc %g0, %g0, %g0;
15 #define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs
17 #define VISEntryHalf rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs
18 #define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs
31 #define EX_RETVAL(x) x
35 #define LOAD(type,addr,dest) type [addr], dest
39 #define STORE(type,src,addr) type src, [addr]
43 #define STORE_BLK(src,addr) stda src, [addr] ASI_BLK_P
47 #define FUNC_NAME U3memcpy
58 .register %g2,#scratch
59 .register %g3,#scratch
61 /* Special/non-trivial issues of this code:
63 * 1) %o5 is preserved from VISEntryHalf to VISExitHalf
64 * 2) Only low 32 FPU registers are used so that only the
65 * lower half of the FPU register set is dirtied by this
66 * code. This is especially important in the kernel.
67 * 3) This code never prefetches cachelines past the end
68 * of the source buffer.
74 /* The cheetah's flexible spine, oversized liver, enlarged heart,
75 * slender muscular body, and claws make it the swiftest hunter
76 * in Africa and the fastest animal on land. Can reach speeds
77 * of up to 2.4GB per second.
81 .type FUNC_NAME,#function
82 FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
99 /* Clobbers o5/g1/g2/g3/g7/icc/xcc. We must preserve
100 * o5 from here until we hit VISExitHalf.
104 /* Is 'dst' already aligned on an 64-byte boundary? */
108 /* Compute abs((dst & 0x3f) - 0x40) into %g2. This is the number
109 * of bytes to copy to make 'dst' 64-byte aligned. We pre-
110 * subtract this from 'len'.
120 1: subcc %g1, 0x1, %g1
121 EX_LD(LOAD(ldub, %o1 + 0x00, %o3))
122 EX_ST(STORE(stb, %o3, %o1 + %o4))
131 alignaddr %o1, %g0, %o1
133 EX_LD(LOAD(ldd, %o1, %f4))
134 1: EX_LD(LOAD(ldd, %o1 + 0x8, %f6))
137 faligndata %f4, %f6, %f0
138 EX_ST(STORE(std, %f0, %o0))
142 EX_LD(LOAD(ldd, %o1 + 0x8, %f4))
145 faligndata %f6, %f4, %f2
146 EX_ST(STORE(std, %f2, %o0))
150 3: LOAD(prefetch, %o1 + 0x000, #one_read)
151 LOAD(prefetch, %o1 + 0x040, #one_read)
152 andn %o2, (0x40 - 1), %o4
153 LOAD(prefetch, %o1 + 0x080, #one_read)
154 LOAD(prefetch, %o1 + 0x0c0, #one_read)
155 LOAD(prefetch, %o1 + 0x100, #one_read)
156 EX_LD(LOAD(ldd, %o1 + 0x000, %f0))
157 LOAD(prefetch, %o1 + 0x140, #one_read)
158 EX_LD(LOAD(ldd, %o1 + 0x008, %f2))
159 LOAD(prefetch, %o1 + 0x180, #one_read)
160 EX_LD(LOAD(ldd, %o1 + 0x010, %f4))
161 LOAD(prefetch, %o1 + 0x1c0, #one_read)
162 faligndata %f0, %f2, %f16
163 EX_LD(LOAD(ldd, %o1 + 0x018, %f6))
164 faligndata %f2, %f4, %f18
165 EX_LD(LOAD(ldd, %o1 + 0x020, %f8))
166 faligndata %f4, %f6, %f20
167 EX_LD(LOAD(ldd, %o1 + 0x028, %f10))
168 faligndata %f6, %f8, %f22
170 EX_LD(LOAD(ldd, %o1 + 0x030, %f12))
171 faligndata %f8, %f10, %f24
172 EX_LD(LOAD(ldd, %o1 + 0x038, %f14))
173 faligndata %f10, %f12, %f26
174 EX_LD(LOAD(ldd, %o1 + 0x040, %f0))
185 EX_LD(LOAD(ldd, %o1 + 0x008, %f2))
186 faligndata %f12, %f14, %f28
187 EX_LD(LOAD(ldd, %o1 + 0x010, %f4))
188 faligndata %f14, %f0, %f30
189 EX_ST(STORE_BLK(%f16, %o0))
190 EX_LD(LOAD(ldd, %o1 + 0x018, %f6))
191 faligndata %f0, %f2, %f16
194 EX_LD(LOAD(ldd, %o1 + 0x020, %f8))
195 faligndata %f2, %f4, %f18
196 EX_LD(LOAD(ldd, %o1 + 0x028, %f10))
197 faligndata %f4, %f6, %f20
198 EX_LD(LOAD(ldd, %o1 + 0x030, %f12))
200 faligndata %f6, %f8, %f22
201 EX_LD(LOAD(ldd, %o1 + 0x038, %f14))
203 faligndata %f8, %f10, %f24
204 EX_LD(LOAD(ldd, %o1 + 0x040, %f0))
205 LOAD(prefetch, %o1 + 0x1c0, #one_read)
206 faligndata %f10, %f12, %f26
210 /* Finally we copy the last full 64-byte block. */
212 EX_LD(LOAD(ldd, %o1 + 0x008, %f2))
213 faligndata %f12, %f14, %f28
214 EX_LD(LOAD(ldd, %o1 + 0x010, %f4))
215 faligndata %f14, %f0, %f30
216 EX_ST(STORE_BLK(%f16, %o0))
217 EX_LD(LOAD(ldd, %o1 + 0x018, %f6))
218 faligndata %f0, %f2, %f16
219 EX_LD(LOAD(ldd, %o1 + 0x020, %f8))
220 faligndata %f2, %f4, %f18
221 EX_LD(LOAD(ldd, %o1 + 0x028, %f10))
222 faligndata %f4, %f6, %f20
223 EX_LD(LOAD(ldd, %o1 + 0x030, %f12))
224 faligndata %f6, %f8, %f22
225 EX_LD(LOAD(ldd, %o1 + 0x038, %f14))
226 faligndata %f8, %f10, %f24
230 EX_LD(LOAD(ldd, %o1 + 0x040, %f0))
231 1: faligndata %f10, %f12, %f26
232 faligndata %f12, %f14, %f28
233 faligndata %f14, %f0, %f30
234 EX_ST(STORE_BLK(%f16, %o0))
239 /* Now we copy the (len modulo 64) bytes at the end.
240 * Note how we borrow the %f0 loaded above.
242 * Also notice how this code is careful not to perform a
243 * load past the end of the src buffer.
254 EX_LD(LOAD(ldd, %o1 + 0x00, %f0))
256 1: EX_LD(LOAD(ldd, %o1 + 0x08, %f2))
259 faligndata %f0, %f2, %f8
260 EX_ST(STORE(std, %f8, %o0))
263 EX_LD(LOAD(ldd, %o1 + 0x08, %f0))
266 faligndata %f2, %f0, %f8
267 EX_ST(STORE(std, %f8, %o0))
271 /* If anything is left, we copy it one byte at a time.
272 * Note that %g1 is (src & 0x3) saved above before the
273 * alignaddr was performed.
287 EX_LD(LOAD(ldx, %o1, %o5))
288 EX_ST(STORE(stx, %o5, %o1 + %o3))
291 1: andcc %o2, 0x4, %g0
294 EX_LD(LOAD(lduw, %o1, %o5))
295 EX_ST(STORE(stw, %o5, %o1 + %o3))
298 1: andcc %o2, 0x2, %g0
301 EX_LD(LOAD(lduh, %o1, %o5))
302 EX_ST(STORE(sth, %o5, %o1 + %o3))
305 1: andcc %o2, 0x1, %g0
308 EX_LD(LOAD(ldub, %o1, %o5))
310 EX_ST(STORE(stb, %o5, %o1 + %o3))
313 70: /* 16 < len <= 64 */
320 1: subcc %o4, 0x10, %o4
321 EX_LD(LOAD(ldx, %o1 + 0x00, %o5))
322 EX_LD(LOAD(ldx, %o1 + 0x08, %g1))
323 EX_ST(STORE(stx, %o5, %o1 + %o3))
325 EX_ST(STORE(stx, %g1, %o1 + %o3))
328 73: andcc %o2, 0x8, %g0
332 EX_LD(LOAD(ldx, %o1, %o5))
333 EX_ST(STORE(stx, %o5, %o1 + %o3))
335 1: andcc %o2, 0x4, %g0
339 EX_LD(LOAD(lduw, %o1, %o5))
340 EX_ST(STORE(stw, %o5, %o1 + %o3))
356 EX_LD(LOAD(ldub, %o1, %o5))
357 EX_ST(STORE(stb, %o5, %o1 + %o3))
373 EX_LD(LOAD(ldx, %o1, %g2))
377 1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3))
382 EX_ST(STORE(stx, %o5, %o0))
395 80: /* 0 < len <= 16 */
402 EX_LD(LOAD(lduw, %o1, %g1))
403 EX_ST(STORE(stw, %g1, %o1 + %o3))
408 mov EX_RETVAL(%g5), %o0
413 EX_LD(LOAD(ldub, %o1, %g1))
414 EX_ST(STORE(stb, %g1, %o1 + %o3))
418 mov EX_RETVAL(%g5), %o0
420 .size FUNC_NAME, .-FUNC_NAME