1 /* U1memcpy.S: UltraSPARC-I/II/IIi/IIe optimized memcpy.
3 * Copyright (C) 1997, 2004 David S. Miller (davem@redhat.com)
4 * Copyright (C) 1996, 1997, 1998, 1999 Jakub Jelinek (jj@ultra.linux.cz)
8 #include <asm/visasm.h>
11 #define ASI_BLK_P 0xf0
14 #define VISEntry rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs; \
15 clr %g1; clr %g2; clr %g3; subcc %g0, %g0, %g0;
16 #define VISExit and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs
18 #define VISEntry rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs
19 #define VISExit and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs
32 #define EX_RETVAL(x) x
36 #define LOAD(type,addr,dest) type [addr], dest
40 #define LOAD_BLK(addr,dest) ldda [addr] ASI_BLK_P, dest
44 #define STORE(type,src,addr) type src, [addr]
48 #define STORE_BLK(src,addr) stda src, [addr] ASI_BLK_P
52 #define FUNC_NAME memcpy
63 #define FREG_FROB(f1, f2, f3, f4, f5, f6, f7, f8, f9) \
64 faligndata %f1, %f2, %f48; \
65 faligndata %f2, %f3, %f50; \
66 faligndata %f3, %f4, %f52; \
67 faligndata %f4, %f5, %f54; \
68 faligndata %f5, %f6, %f56; \
69 faligndata %f6, %f7, %f58; \
70 faligndata %f7, %f8, %f60; \
71 faligndata %f8, %f9, %f62;
73 #define MAIN_LOOP_CHUNK(src, dest, fdest, fsrc, len, jmptgt) \
74 EX_LD(LOAD_BLK(%src, %fdest)); \
75 EX_ST(STORE_BLK(%fsrc, %dest)); \
76 add %src, 0x40, %src; \
77 subcc %len, 0x40, %len; \
79 add %dest, 0x40, %dest; \
81 #define LOOP_CHUNK1(src, dest, len, branch_dest) \
82 MAIN_LOOP_CHUNK(src, dest, f0, f48, len, branch_dest)
83 #define LOOP_CHUNK2(src, dest, len, branch_dest) \
84 MAIN_LOOP_CHUNK(src, dest, f16, f48, len, branch_dest)
85 #define LOOP_CHUNK3(src, dest, len, branch_dest) \
86 MAIN_LOOP_CHUNK(src, dest, f32, f48, len, branch_dest)
88 #define STORE_SYNC(dest, fsrc) \
89 EX_ST(STORE_BLK(%fsrc, %dest)); \
90 add %dest, 0x40, %dest;
92 #define STORE_JUMP(dest, fsrc, target) \
93 EX_ST(STORE_BLK(%fsrc, %dest)); \
94 add %dest, 0x40, %dest; \
97 #define FINISH_VISCHUNK(dest, f0, f1, left) \
98 subcc %left, 8, %left;\
100 faligndata %f0, %f1, %f48; \
101 EX_ST(STORE(std, %f48, %dest)); \
104 #define UNEVEN_VISCHUNK_LAST(dest, f0, f1, left) \
105 subcc %left, 8, %left; \
109 #define UNEVEN_VISCHUNK(dest, f0, f1, left) \
110 UNEVEN_VISCHUNK_LAST(dest, f0, f1, left) \
113 .register %g2,#scratch
114 .register %g3,#scratch
120 .type FUNC_NAME,#function
121 FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
135 /* Clobbers o5/g1/g2/g3/g7/icc/xcc. */
138 /* Is 'dst' already aligned on an 64-byte boundary? */
142 /* Compute abs((dst & 0x3f) - 0x40) into %g2. This is the number
143 * of bytes to copy to make 'dst' 64-byte aligned. We pre-
144 * subtract this from 'len'.
154 1: subcc %g1, 0x1, %g1
155 EX_LD(LOAD(ldub, %o1 + 0x00, %o3))
156 EX_ST(STORE(stb, %o3, %o1 + %o4))
165 alignaddr %o1, %g0, %o1
167 EX_LD(LOAD(ldd, %o1, %f4))
168 1: EX_LD(LOAD(ldd, %o1 + 0x8, %f6))
171 faligndata %f4, %f6, %f0
172 EX_ST(STORE(std, %f0, %o0))
176 EX_LD(LOAD(ldd, %o1 + 0x8, %f4))
179 faligndata %f6, %f4, %f0
180 EX_ST(STORE(std, %f0, %o0))
184 /* Destination is 64-byte aligned. */
186 membar #LoadStore | #StoreStore | #StoreLoad
190 andncc %o4, (0x40 - 1), %o4
193 andn %o1, (0x40 - 1), %o1
203 EX_LD(LOAD_BLK(%o1, %f0))
206 EX_LD(LOAD_BLK(%o1, %f16))
209 EX_LD(LOAD_BLK(%o1, %f32))
212 /* There are 8 instances of the unrolled loop,
213 * one for each possible alignment of the
214 * source buffer. Each loop instance is 452
223 add %o3, %lo(1f - 1b), %o3
228 1: FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16)
229 LOOP_CHUNK1(o1, o0, o4, 1f)
230 FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32)
231 LOOP_CHUNK2(o1, o0, o4, 2f)
232 FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0)
233 LOOP_CHUNK3(o1, o0, o4, 3f)
235 faligndata %f0, %f2, %f48
236 1: FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32)
237 STORE_SYNC(o0, f48) membar #Sync
238 FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0)
239 STORE_JUMP(o0, f48, 40f) membar #Sync
240 2: FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0)
241 STORE_SYNC(o0, f48) membar #Sync
242 FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16)
243 STORE_JUMP(o0, f48, 48f) membar #Sync
244 3: FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16)
245 STORE_SYNC(o0, f48) membar #Sync
246 FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32)
247 STORE_JUMP(o0, f48, 56f) membar #Sync
249 1: FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18)
250 LOOP_CHUNK1(o1, o0, o4, 1f)
251 FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34)
252 LOOP_CHUNK2(o1, o0, o4, 2f)
253 FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2)
254 LOOP_CHUNK3(o1, o0, o4, 3f)
256 faligndata %f2, %f4, %f48
257 1: FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34)
258 STORE_SYNC(o0, f48) membar #Sync
259 FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2)
260 STORE_JUMP(o0, f48, 41f) membar #Sync
261 2: FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2)
262 STORE_SYNC(o0, f48) membar #Sync
263 FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18)
264 STORE_JUMP(o0, f48, 49f) membar #Sync
265 3: FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18)
266 STORE_SYNC(o0, f48) membar #Sync
267 FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34)
268 STORE_JUMP(o0, f48, 57f) membar #Sync
270 1: FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20)
271 LOOP_CHUNK1(o1, o0, o4, 1f)
272 FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36)
273 LOOP_CHUNK2(o1, o0, o4, 2f)
274 FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4)
275 LOOP_CHUNK3(o1, o0, o4, 3f)
277 faligndata %f4, %f6, %f48
278 1: FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36)
279 STORE_SYNC(o0, f48) membar #Sync
280 FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4)
281 STORE_JUMP(o0, f48, 42f) membar #Sync
282 2: FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4)
283 STORE_SYNC(o0, f48) membar #Sync
284 FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20)
285 STORE_JUMP(o0, f48, 50f) membar #Sync
286 3: FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20)
287 STORE_SYNC(o0, f48) membar #Sync
288 FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36)
289 STORE_JUMP(o0, f48, 58f) membar #Sync
291 1: FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22)
292 LOOP_CHUNK1(o1, o0, o4, 1f)
293 FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38)
294 LOOP_CHUNK2(o1, o0, o4, 2f)
295 FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6)
296 LOOP_CHUNK3(o1, o0, o4, 3f)
298 faligndata %f6, %f8, %f48
299 1: FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38)
300 STORE_SYNC(o0, f48) membar #Sync
301 FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6)
302 STORE_JUMP(o0, f48, 43f) membar #Sync
303 2: FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6)
304 STORE_SYNC(o0, f48) membar #Sync
305 FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22)
306 STORE_JUMP(o0, f48, 51f) membar #Sync
307 3: FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22)
308 STORE_SYNC(o0, f48) membar #Sync
309 FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38)
310 STORE_JUMP(o0, f48, 59f) membar #Sync
312 1: FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24)
313 LOOP_CHUNK1(o1, o0, o4, 1f)
314 FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40)
315 LOOP_CHUNK2(o1, o0, o4, 2f)
316 FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8)
317 LOOP_CHUNK3(o1, o0, o4, 3f)
319 faligndata %f8, %f10, %f48
320 1: FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40)
321 STORE_SYNC(o0, f48) membar #Sync
322 FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8)
323 STORE_JUMP(o0, f48, 44f) membar #Sync
324 2: FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8)
325 STORE_SYNC(o0, f48) membar #Sync
326 FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24)
327 STORE_JUMP(o0, f48, 52f) membar #Sync
328 3: FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24)
329 STORE_SYNC(o0, f48) membar #Sync
330 FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40)
331 STORE_JUMP(o0, f48, 60f) membar #Sync
333 1: FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26)
334 LOOP_CHUNK1(o1, o0, o4, 1f)
335 FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42)
336 LOOP_CHUNK2(o1, o0, o4, 2f)
337 FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10)
338 LOOP_CHUNK3(o1, o0, o4, 3f)
340 faligndata %f10, %f12, %f48
341 1: FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42)
342 STORE_SYNC(o0, f48) membar #Sync
343 FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10)
344 STORE_JUMP(o0, f48, 45f) membar #Sync
345 2: FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10)
346 STORE_SYNC(o0, f48) membar #Sync
347 FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26)
348 STORE_JUMP(o0, f48, 53f) membar #Sync
349 3: FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26)
350 STORE_SYNC(o0, f48) membar #Sync
351 FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42)
352 STORE_JUMP(o0, f48, 61f) membar #Sync
354 1: FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28)
355 LOOP_CHUNK1(o1, o0, o4, 1f)
356 FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44)
357 LOOP_CHUNK2(o1, o0, o4, 2f)
358 FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12)
359 LOOP_CHUNK3(o1, o0, o4, 3f)
361 faligndata %f12, %f14, %f48
362 1: FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44)
363 STORE_SYNC(o0, f48) membar #Sync
364 FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12)
365 STORE_JUMP(o0, f48, 46f) membar #Sync
366 2: FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12)
367 STORE_SYNC(o0, f48) membar #Sync
368 FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28)
369 STORE_JUMP(o0, f48, 54f) membar #Sync
370 3: FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28)
371 STORE_SYNC(o0, f48) membar #Sync
372 FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44)
373 STORE_JUMP(o0, f48, 62f) membar #Sync
375 1: FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30)
376 LOOP_CHUNK1(o1, o0, o4, 1f)
377 FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46)
378 LOOP_CHUNK2(o1, o0, o4, 2f)
379 FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14)
380 LOOP_CHUNK3(o1, o0, o4, 3f)
382 faligndata %f14, %f16, %f48
383 1: FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46)
384 STORE_SYNC(o0, f48) membar #Sync
385 FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14)
386 STORE_JUMP(o0, f48, 47f) membar #Sync
387 2: FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14)
388 STORE_SYNC(o0, f48) membar #Sync
389 FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30)
390 STORE_JUMP(o0, f48, 55f) membar #Sync
391 3: FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30)
392 STORE_SYNC(o0, f48) membar #Sync
393 FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46)
394 STORE_JUMP(o0, f48, 63f) membar #Sync
396 40: FINISH_VISCHUNK(o0, f0, f2, g3)
397 41: FINISH_VISCHUNK(o0, f2, f4, g3)
398 42: FINISH_VISCHUNK(o0, f4, f6, g3)
399 43: FINISH_VISCHUNK(o0, f6, f8, g3)
400 44: FINISH_VISCHUNK(o0, f8, f10, g3)
401 45: FINISH_VISCHUNK(o0, f10, f12, g3)
402 46: FINISH_VISCHUNK(o0, f12, f14, g3)
403 47: UNEVEN_VISCHUNK(o0, f14, f0, g3)
404 48: FINISH_VISCHUNK(o0, f16, f18, g3)
405 49: FINISH_VISCHUNK(o0, f18, f20, g3)
406 50: FINISH_VISCHUNK(o0, f20, f22, g3)
407 51: FINISH_VISCHUNK(o0, f22, f24, g3)
408 52: FINISH_VISCHUNK(o0, f24, f26, g3)
409 53: FINISH_VISCHUNK(o0, f26, f28, g3)
410 54: FINISH_VISCHUNK(o0, f28, f30, g3)
411 55: UNEVEN_VISCHUNK(o0, f30, f0, g3)
412 56: FINISH_VISCHUNK(o0, f32, f34, g3)
413 57: FINISH_VISCHUNK(o0, f34, f36, g3)
414 58: FINISH_VISCHUNK(o0, f36, f38, g3)
415 59: FINISH_VISCHUNK(o0, f38, f40, g3)
416 60: FINISH_VISCHUNK(o0, f40, f42, g3)
417 61: FINISH_VISCHUNK(o0, f42, f44, g3)
418 62: FINISH_VISCHUNK(o0, f44, f46, g3)
419 63: UNEVEN_VISCHUNK_LAST(o0, f46, f0, g3)
421 93: EX_LD(LOAD(ldd, %o1, %f2))
424 faligndata %f0, %f2, %f8
425 EX_ST(STORE(std, %f8, %o0))
428 EX_LD(LOAD(ldd, %o1, %f0))
431 faligndata %f2, %f0, %f8
432 EX_ST(STORE(std, %f8, %o0))
439 1: EX_LD(LOAD(ldub, %o1, %o3))
442 EX_ST(STORE(stb, %o3, %o0))
446 2: membar #StoreLoad | #StoreStore
449 mov EX_RETVAL(%g5), %o0
452 70: /* 16 < len <= (5 * 64) */
456 72: andn %o2, 0xf, %o4
458 1: EX_LD(LOAD(ldx, %o1 + 0x00, %o5))
459 EX_LD(LOAD(ldx, %o1 + 0x08, %g1))
461 EX_ST(STORE(stx, %o5, %o1 + %o3))
463 EX_ST(STORE(stx, %g1, %o1 + %o3))
466 73: andcc %o2, 0x8, %g0
469 EX_LD(LOAD(ldx, %o1, %o5))
471 EX_ST(STORE(stx, %o5, %o1 + %o3))
473 1: andcc %o2, 0x4, %g0
476 EX_LD(LOAD(lduw, %o1, %o5))
478 EX_ST(STORE(stw, %o5, %o1 + %o3))
486 75: andcc %o0, 0x7, %g1
492 1: EX_LD(LOAD(ldub, %o1, %o5))
494 EX_ST(STORE(stb, %o5, %o1 + %o3))
510 EX_LD(LOAD(ldx, %o1, %g2))
514 1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3))
519 EX_ST(STORE(stx, %o5, %o0))
532 80: /* 0 < len <= 16 */
537 1: EX_LD(LOAD(lduw, %o1, %g1))
539 EX_ST(STORE(stw, %g1, %o1 + %o3))
544 mov EX_RETVAL(%g5), %o0
547 90: EX_LD(LOAD(ldub, %o1, %g1))
549 EX_ST(STORE(stb, %g1, %o1 + %o3))
553 mov EX_RETVAL(%g5), %o0
555 .size FUNC_NAME, .-FUNC_NAME