1 /* U1memcpy.S: UltraSPARC-I/II/IIi/IIe optimized memcpy.
3 * Copyright (C) 1997, 2004 David S. Miller (davem@redhat.com)
4 * Copyright (C) 1996, 1997, 1998, 1999 Jakub Jelinek (jj@ultra.linux.cz)
8 #include <asm/visasm.h>
11 #define ASI_BLK_P 0xf0
14 #define VISEntry rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs; \
15 clr %g1; clr %g2; clr %g3; subcc %g0, %g0, %g0;
16 #define VISExit and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs
18 #define VISEntry rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs
19 #define VISExit and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs
32 #define EX_RETVAL(x) x
36 #define LOAD(type,addr,dest) type [addr], dest
40 #define LOAD_BLK(addr,dest) ldda [addr] ASI_BLK_P, dest
44 #define STORE(type,src,addr) type src, [addr]
48 #define STORE_BLK(src,addr) stda src, [addr] ASI_BLK_P
52 #define FUNC_NAME memcpy
63 #define FREG_FROB(f1, f2, f3, f4, f5, f6, f7, f8, f9) \
64 faligndata %f1, %f2, %f48; \
65 faligndata %f2, %f3, %f50; \
66 faligndata %f3, %f4, %f52; \
67 faligndata %f4, %f5, %f54; \
68 faligndata %f5, %f6, %f56; \
69 faligndata %f6, %f7, %f58; \
70 faligndata %f7, %f8, %f60; \
71 faligndata %f8, %f9, %f62;
73 #define MAIN_LOOP_CHUNK(src, dest, fdest, fsrc, len, jmptgt) \
74 EX_LD(LOAD_BLK(%src, %fdest)); \
75 EX_ST(STORE_BLK(%fsrc, %dest)); \
76 add %src, 0x40, %src; \
77 subcc %len, 0x40, %len; \
79 add %dest, 0x40, %dest; \
81 #define LOOP_CHUNK1(src, dest, len, branch_dest) \
82 MAIN_LOOP_CHUNK(src, dest, f0, f48, len, branch_dest)
83 #define LOOP_CHUNK2(src, dest, len, branch_dest) \
84 MAIN_LOOP_CHUNK(src, dest, f16, f48, len, branch_dest)
85 #define LOOP_CHUNK3(src, dest, len, branch_dest) \
86 MAIN_LOOP_CHUNK(src, dest, f32, f48, len, branch_dest)
88 #define STORE_SYNC(dest, fsrc) \
89 EX_ST(STORE_BLK(%fsrc, %dest)); \
90 add %dest, 0x40, %dest;
92 #define STORE_JUMP(dest, fsrc, target) \
93 EX_ST(STORE_BLK(%fsrc, %dest)); \
94 add %dest, 0x40, %dest; \
97 #define FINISH_VISCHUNK(dest, f0, f1, left) \
98 subcc %left, 8, %left;\
100 faligndata %f0, %f1, %f48; \
101 EX_ST(STORE(std, %f48, %dest)); \
104 #define UNEVEN_VISCHUNK_LAST(dest, f0, f1, left) \
105 subcc %left, 8, %left; \
109 #define UNEVEN_VISCHUNK(dest, f0, f1, left) \
110 UNEVEN_VISCHUNK_LAST(dest, f0, f1, left) \
113 .register %g2,#scratch
114 .register %g3,#scratch
120 .type FUNC_NAME,#function
121 FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
138 /* Clobbers o5/g1/g2/g3/g7/icc/xcc. */
141 /* Is 'dst' already aligned on an 64-byte boundary? */
145 /* Compute abs((dst & 0x3f) - 0x40) into %g2. This is the number
146 * of bytes to copy to make 'dst' 64-byte aligned. We pre-
147 * subtract this from 'len'.
157 1: subcc %g1, 0x1, %g1
158 EX_LD(LOAD(ldub, %o1 + 0x00, %o3))
159 EX_ST(STORE(stb, %o3, %o1 + %o4))
168 alignaddr %o1, %g0, %o1
170 EX_LD(LOAD(ldd, %o1, %f4))
171 1: EX_LD(LOAD(ldd, %o1 + 0x8, %f6))
174 faligndata %f4, %f6, %f0
175 EX_ST(STORE(std, %f0, %o0))
179 EX_LD(LOAD(ldd, %o1 + 0x8, %f4))
182 faligndata %f6, %f4, %f0
183 EX_ST(STORE(std, %f0, %o0))
187 /* Destination is 64-byte aligned. */
189 membar #LoadStore | #StoreStore | #StoreLoad
193 andncc %o4, (0x40 - 1), %o4
196 andn %o1, (0x40 - 1), %o1
206 EX_LD(LOAD_BLK(%o1, %f0))
209 EX_LD(LOAD_BLK(%o1, %f16))
212 EX_LD(LOAD_BLK(%o1, %f32))
215 /* There are 8 instances of the unrolled loop,
216 * one for each possible alignment of the
217 * source buffer. Each loop instance is 452
226 add %o3, %lo(1f - 1b), %o3
231 1: FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16)
232 LOOP_CHUNK1(o1, o0, o4, 1f)
233 FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32)
234 LOOP_CHUNK2(o1, o0, o4, 2f)
235 FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0)
236 LOOP_CHUNK3(o1, o0, o4, 3f)
238 faligndata %f0, %f2, %f48
239 1: FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32)
240 STORE_SYNC(o0, f48) membar #Sync
241 FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0)
242 STORE_JUMP(o0, f48, 40f) membar #Sync
243 2: FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0)
244 STORE_SYNC(o0, f48) membar #Sync
245 FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16)
246 STORE_JUMP(o0, f48, 48f) membar #Sync
247 3: FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16)
248 STORE_SYNC(o0, f48) membar #Sync
249 FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32)
250 STORE_JUMP(o0, f48, 56f) membar #Sync
252 1: FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18)
253 LOOP_CHUNK1(o1, o0, o4, 1f)
254 FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34)
255 LOOP_CHUNK2(o1, o0, o4, 2f)
256 FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2)
257 LOOP_CHUNK3(o1, o0, o4, 3f)
259 faligndata %f2, %f4, %f48
260 1: FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34)
261 STORE_SYNC(o0, f48) membar #Sync
262 FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2)
263 STORE_JUMP(o0, f48, 41f) membar #Sync
264 2: FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2)
265 STORE_SYNC(o0, f48) membar #Sync
266 FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18)
267 STORE_JUMP(o0, f48, 49f) membar #Sync
268 3: FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18)
269 STORE_SYNC(o0, f48) membar #Sync
270 FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34)
271 STORE_JUMP(o0, f48, 57f) membar #Sync
273 1: FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20)
274 LOOP_CHUNK1(o1, o0, o4, 1f)
275 FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36)
276 LOOP_CHUNK2(o1, o0, o4, 2f)
277 FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4)
278 LOOP_CHUNK3(o1, o0, o4, 3f)
280 faligndata %f4, %f6, %f48
281 1: FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36)
282 STORE_SYNC(o0, f48) membar #Sync
283 FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4)
284 STORE_JUMP(o0, f48, 42f) membar #Sync
285 2: FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4)
286 STORE_SYNC(o0, f48) membar #Sync
287 FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20)
288 STORE_JUMP(o0, f48, 50f) membar #Sync
289 3: FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20)
290 STORE_SYNC(o0, f48) membar #Sync
291 FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36)
292 STORE_JUMP(o0, f48, 58f) membar #Sync
294 1: FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22)
295 LOOP_CHUNK1(o1, o0, o4, 1f)
296 FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38)
297 LOOP_CHUNK2(o1, o0, o4, 2f)
298 FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6)
299 LOOP_CHUNK3(o1, o0, o4, 3f)
301 faligndata %f6, %f8, %f48
302 1: FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38)
303 STORE_SYNC(o0, f48) membar #Sync
304 FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6)
305 STORE_JUMP(o0, f48, 43f) membar #Sync
306 2: FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6)
307 STORE_SYNC(o0, f48) membar #Sync
308 FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22)
309 STORE_JUMP(o0, f48, 51f) membar #Sync
310 3: FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22)
311 STORE_SYNC(o0, f48) membar #Sync
312 FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38)
313 STORE_JUMP(o0, f48, 59f) membar #Sync
315 1: FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24)
316 LOOP_CHUNK1(o1, o0, o4, 1f)
317 FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40)
318 LOOP_CHUNK2(o1, o0, o4, 2f)
319 FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8)
320 LOOP_CHUNK3(o1, o0, o4, 3f)
322 faligndata %f8, %f10, %f48
323 1: FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40)
324 STORE_SYNC(o0, f48) membar #Sync
325 FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8)
326 STORE_JUMP(o0, f48, 44f) membar #Sync
327 2: FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8)
328 STORE_SYNC(o0, f48) membar #Sync
329 FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24)
330 STORE_JUMP(o0, f48, 52f) membar #Sync
331 3: FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24)
332 STORE_SYNC(o0, f48) membar #Sync
333 FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40)
334 STORE_JUMP(o0, f48, 60f) membar #Sync
336 1: FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26)
337 LOOP_CHUNK1(o1, o0, o4, 1f)
338 FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42)
339 LOOP_CHUNK2(o1, o0, o4, 2f)
340 FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10)
341 LOOP_CHUNK3(o1, o0, o4, 3f)
343 faligndata %f10, %f12, %f48
344 1: FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42)
345 STORE_SYNC(o0, f48) membar #Sync
346 FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10)
347 STORE_JUMP(o0, f48, 45f) membar #Sync
348 2: FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10)
349 STORE_SYNC(o0, f48) membar #Sync
350 FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26)
351 STORE_JUMP(o0, f48, 53f) membar #Sync
352 3: FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26)
353 STORE_SYNC(o0, f48) membar #Sync
354 FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42)
355 STORE_JUMP(o0, f48, 61f) membar #Sync
357 1: FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28)
358 LOOP_CHUNK1(o1, o0, o4, 1f)
359 FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44)
360 LOOP_CHUNK2(o1, o0, o4, 2f)
361 FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12)
362 LOOP_CHUNK3(o1, o0, o4, 3f)
364 faligndata %f12, %f14, %f48
365 1: FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44)
366 STORE_SYNC(o0, f48) membar #Sync
367 FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12)
368 STORE_JUMP(o0, f48, 46f) membar #Sync
369 2: FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12)
370 STORE_SYNC(o0, f48) membar #Sync
371 FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28)
372 STORE_JUMP(o0, f48, 54f) membar #Sync
373 3: FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28)
374 STORE_SYNC(o0, f48) membar #Sync
375 FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44)
376 STORE_JUMP(o0, f48, 62f) membar #Sync
378 1: FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30)
379 LOOP_CHUNK1(o1, o0, o4, 1f)
380 FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46)
381 LOOP_CHUNK2(o1, o0, o4, 2f)
382 FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14)
383 LOOP_CHUNK3(o1, o0, o4, 3f)
385 faligndata %f14, %f16, %f48
386 1: FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46)
387 STORE_SYNC(o0, f48) membar #Sync
388 FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14)
389 STORE_JUMP(o0, f48, 47f) membar #Sync
390 2: FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14)
391 STORE_SYNC(o0, f48) membar #Sync
392 FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30)
393 STORE_JUMP(o0, f48, 55f) membar #Sync
394 3: FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30)
395 STORE_SYNC(o0, f48) membar #Sync
396 FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46)
397 STORE_JUMP(o0, f48, 63f) membar #Sync
399 40: FINISH_VISCHUNK(o0, f0, f2, g3)
400 41: FINISH_VISCHUNK(o0, f2, f4, g3)
401 42: FINISH_VISCHUNK(o0, f4, f6, g3)
402 43: FINISH_VISCHUNK(o0, f6, f8, g3)
403 44: FINISH_VISCHUNK(o0, f8, f10, g3)
404 45: FINISH_VISCHUNK(o0, f10, f12, g3)
405 46: FINISH_VISCHUNK(o0, f12, f14, g3)
406 47: UNEVEN_VISCHUNK(o0, f14, f0, g3)
407 48: FINISH_VISCHUNK(o0, f16, f18, g3)
408 49: FINISH_VISCHUNK(o0, f18, f20, g3)
409 50: FINISH_VISCHUNK(o0, f20, f22, g3)
410 51: FINISH_VISCHUNK(o0, f22, f24, g3)
411 52: FINISH_VISCHUNK(o0, f24, f26, g3)
412 53: FINISH_VISCHUNK(o0, f26, f28, g3)
413 54: FINISH_VISCHUNK(o0, f28, f30, g3)
414 55: UNEVEN_VISCHUNK(o0, f30, f0, g3)
415 56: FINISH_VISCHUNK(o0, f32, f34, g3)
416 57: FINISH_VISCHUNK(o0, f34, f36, g3)
417 58: FINISH_VISCHUNK(o0, f36, f38, g3)
418 59: FINISH_VISCHUNK(o0, f38, f40, g3)
419 60: FINISH_VISCHUNK(o0, f40, f42, g3)
420 61: FINISH_VISCHUNK(o0, f42, f44, g3)
421 62: FINISH_VISCHUNK(o0, f44, f46, g3)
422 63: UNEVEN_VISCHUNK_LAST(o0, f46, f0, g3)
424 93: EX_LD(LOAD(ldd, %o1, %f2))
427 faligndata %f0, %f2, %f8
428 EX_ST(STORE(std, %f8, %o0))
431 EX_LD(LOAD(ldd, %o1, %f0))
434 faligndata %f2, %f0, %f8
435 EX_ST(STORE(std, %f8, %o0))
442 1: EX_LD(LOAD(ldub, %o1, %o3))
445 EX_ST(STORE(stb, %o3, %o0))
449 2: membar #StoreLoad | #StoreStore
452 mov EX_RETVAL(%g5), %o0
455 70: /* 16 < len <= (5 * 64) */
459 72: andn %o2, 0xf, %o4
461 1: EX_LD(LOAD(ldx, %o1 + 0x00, %o5))
462 EX_LD(LOAD(ldx, %o1 + 0x08, %g1))
464 EX_ST(STORE(stx, %o5, %o1 + %o3))
466 EX_ST(STORE(stx, %g1, %o1 + %o3))
469 73: andcc %o2, 0x8, %g0
472 EX_LD(LOAD(ldx, %o1, %o5))
474 EX_ST(STORE(stx, %o5, %o1 + %o3))
476 1: andcc %o2, 0x4, %g0
479 EX_LD(LOAD(lduw, %o1, %o5))
481 EX_ST(STORE(stw, %o5, %o1 + %o3))
489 75: andcc %o0, 0x7, %g1
495 1: EX_LD(LOAD(ldub, %o1, %o5))
497 EX_ST(STORE(stb, %o5, %o1 + %o3))
513 EX_LD(LOAD(ldx, %o1, %g2))
517 1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3))
522 EX_ST(STORE(stx, %o5, %o0))
535 80: /* 0 < len <= 16 */
540 1: EX_LD(LOAD(lduw, %o1, %g1))
542 EX_ST(STORE(stw, %g1, %o1 + %o3))
547 mov EX_RETVAL(%g5), %o0
550 90: EX_LD(LOAD(ldub, %o1, %g1))
552 EX_ST(STORE(stb, %g1, %o1 + %o3))
556 mov EX_RETVAL(%g5), %o0
558 .size FUNC_NAME, .-FUNC_NAME