1 /* $Id: VIScsumcopyusr.S,v 1.2 2000/02/20 23:21:40 davem Exp $
2 * VIScsumcopyusr.S: High bandwidth IP checksumming with simultaneous
3 * copying utilizing the UltraSparc Visual Instruction Set.
5 * Copyright (C) 1997, 1999 Jakub Jelinek (jj@ultra.linux.cz)
6 * Copyright (C) 2000 David S. Miller (davem@redhat.com)
8 * Based on older sparc32/sparc64 checksum.S, which is:
10 * Copyright(C) 1995 Linus Torvalds
11 * Copyright(C) 1995 Miguel de Icaza
12 * Copyright(C) 1996,1997 David S. Miller
14 * Linux/Alpha checksum c-code
15 * Linux/ix86 inline checksum assembly
16 * RFC1071 Computing the Internet Checksum (esp. Jacobsons m68k code)
17 * David Mosberger-Tang for optimized reference c-code
18 * BSD4.4 portable checksum routine
22 #define STACKOFF 0x7ff+128
31 #include <asm/visasm.h>
32 #include <asm/thread_info.h>
34 #define ASI_BLK_XOR1 (ASI_BLK_P ^ (ASI_BLK_P >> 3) ^ ASI_P)
35 #define ASI_BLK_OR (ASI_BLK_P & ~ASI_P)
38 #define ASI_BLK_P 0xf0
42 #define ASI_BLK_XOR (ASI_BLK_P ^ ASI_P)
58 /* Dobrou noc, SunSoft engineers. Spete sladce.
59 * This has a couple of tricks in and those
60 * tricks are UltraLinux trade secrets :))
61 * Once AGAIN, the SunSoft engineers are caught
62 * asleep at the keyboard :)).
63 * The main loop does about 20 superscalar cycles
64 * per 64bytes checksummed/copied.
68 ldda [%src] ASI_BLK_P, %O0 /* Load Group */
71 stda %f48, [%dst] %asi /* Store */
74 #define STBLK_XORASI(tmpreg1,tmpreg2) \
75 stda %f48, [%dst] %asi /* Store */; \
77 srl %tmpreg1, 3, %tmpreg2; \
78 xor %tmpreg1, ASI_BLK_XOR1, %tmpreg1; \
79 wr %tmpreg1, %tmpreg2, %asi;
81 #define STBLK_XORASI(tmpreg1,tmpreg2) \
82 stda %f48, [%dst] %asi /* Store */; \
84 wr %tmpreg1, ASI_BLK_XOR, %asi;
88 stda %fx, [%dst + off] %asi /* Store */
94 #define DO_THE_TRICK(f0,f2,f4,f6,f8,f10,f12,f14,F0,F2,F4,F6,F8,F10,F12,F14,DUMMY1,A0,A2,A4,A6,A8,A10,A12,A14,B14,DUMMY2,LOAD,STORE1,STORE2,STORE3,STORE4,STORE5,STORE6,STORE7,STORE8,DUMMY3,BRANCH...) \
95 LOAD /* Load (Group) */; \
96 faligndata %A14, %F0, %A14 /* FPA Group */; \
98 STORE1 /* Store (optional) */; \
99 faligndata %F0, %F2, %A0 /* FPA Group */; \
100 srl %x5, 1, %x5 /* IEU0 */; \
101 add %sum, %x4, %sum /* IEU1 */; \
102 fpadd32 %F0, %f0, %F0 /* FPA Group */; \
103 inc %x6 /* IEU0 */; \
104 STORE2 /* Store (optional) */; \
105 faligndata %F2, %F4, %A2 /* FPA Group */; \
106 srl %x6, 1, %x6 /* IEU0 */; \
107 add %sum, %x5, %sum /* IEU1 */; \
108 fpadd32 %F2, %f2, %F2 /* FPA Group */; \
109 add %src, 64, %src /* IEU0 */; \
110 fcmpgt32 %f0, %F0, %x1 /* FPM */; \
111 add %dst, 64, %dst /* IEU1 Group */; \
112 inc %x7 /* IEU0 */; \
113 STORE3 /* Store (optional) */; \
114 faligndata %F4, %F6, %A4 /* FPA */; \
115 fpadd32 %F4, %f4, %F4 /* FPA Group */; \
116 add %sum, %x6, %sum /* IEU1 */; \
117 fcmpgt32 %f2, %F2, %x2 /* FPM */; \
118 srl %x7, 1, %x7 /* IEU0 Group */; \
119 inc %x8 /* IEU1 */; \
120 STORE4 /* Store (optional) */; \
121 faligndata %F6, %F8, %A6 /* FPA */; \
122 fpadd32 %F6, %f6, %F6 /* FPA Group */; \
123 srl %x8, 1, %x8 /* IEU0 */; \
124 fcmpgt32 %f4, %F4, %x3 /* FPM */; \
125 add %sum, %x7, %sum /* IEU0 Group */; \
126 inc %x1 /* IEU1 */; \
127 STORE5 /* Store (optional) */; \
128 faligndata %F8, %F10, %A8 /* FPA */; \
129 fpadd32 %F8, %f8, %F8 /* FPA Group */; \
130 srl %x1, 1, %x1 /* IEU0 */; \
131 fcmpgt32 %f6, %F6, %x4 /* FPM */; \
132 add %sum, %x8, %sum /* IEU0 Group */; \
133 inc %x2 /* IEU1 */; \
134 STORE6 /* Store (optional) */; \
135 faligndata %F10, %F12, %A10 /* FPA */; \
136 fpadd32 %F10, %f10, %F10 /* FPA Group */; \
137 srl %x2, 1, %x2 /* IEU0 */; \
138 fcmpgt32 %f8, %F8, %x5 /* FPM */; \
139 add %sum, %x1, %sum /* IEU0 Group */; \
140 inc %x3 /* IEU1 */; \
141 STORE7 /* Store (optional) */; \
142 faligndata %F12, %F14, %A12 /* FPA */; \
143 fpadd32 %F12, %f12, %F12 /* FPA Group */; \
144 srl %x3, 1, %x3 /* IEU0 */; \
145 fcmpgt32 %f10, %F10, %x6 /* FPM */; \
146 add %sum, %x2, %sum /* IEU0 Group */; \
147 inc %x4 /* IEU1 */; \
148 STORE8 /* Store (optional) */; \
149 fmovd %F14, %B14 /* FPA */; \
150 fpadd32 %F14, %f14, %F14 /* FPA Group */; \
151 srl %x4, 1, %x4 /* IEU0 */; \
152 fcmpgt32 %f12, %F12, %x7 /* FPM */; \
153 add %sum, %x3, %sum /* IEU0 Group */; \
154 subcc %len, 64, %len /* IEU1 */; \
156 fcmpgt32 %f14, %F14, %x8 /* FPM Group */;
158 #define END_THE_TRICK(f0,f2,f4,f6,f8,f10,f12,f14,FA,FB,S0,S1,S2,S3,T0,T1,U0,fz) \
159 inc %x5 /* IEU0 Group */; \
160 fpadd32 %f2, %f0, %S0 /* FPA */; \
161 add %sum, %x4, %sum /* IEU1 */; \
162 srl %x5, 1, %x5 /* IEU0 Group */; \
163 fpadd32 %f6, %f4, %S1 /* FPA */; \
164 inc %x6 /* IEU1 */; \
165 fpadd32 %f10, %f8, %S2 /* FPA Group */; \
166 add %sum, %x5, %sum /* IEU0 */; \
167 fcmpgt32 %f0, %S0, %x1 /* FPM */; \
168 fpadd32 %f14, %f12, %S3 /* FPA Group */; \
169 srl %x6, 1, %x6 /* IEU0 */; \
170 fcmpgt32 %f4, %S1, %x2 /* FPM */; \
171 add %sum, %x6, %sum /* IEU0 Group */; \
172 fzero %fz /* FPA */; \
173 fcmpgt32 %f8, %S2, %x3 /* FPM */; \
174 inc %x7 /* IEU0 Group */; \
175 inc %x8 /* IEU1 */; \
176 srl %x7, 1, %x7 /* IEU0 Group */; \
177 inc %x1 /* IEU1 */; \
178 fpadd32 %S0, %S1, %T0 /* FPA */; \
179 fpadd32 %S2, %S3, %T1 /* FPA Group */; \
180 add %sum, %x7, %sum /* IEU0 */; \
181 fcmpgt32 %f12, %S3, %x4 /* FPM */; \
182 srl %x8, 1, %x8 /* IEU0 Group */; \
183 inc %x2 /* IEU1 */; \
184 srl %x1, 1, %x1 /* IEU0 Group */; \
185 add %sum, %x8, %sum /* IEU1 */; \
186 add %sum, %x1, %sum /* IEU0 Group */; \
187 fcmpgt32 %S0, %T0, %x5 /* FPM */; \
188 srl %x2, 1, %x2 /* IEU0 Group */; \
189 fcmpgt32 %S2, %T1, %x6 /* FPM */; \
190 inc %x3 /* IEU0 Group */; \
191 add %sum, %x2, %sum /* IEU1 */; \
192 srl %x3, 1, %x3 /* IEU0 Group */; \
193 inc %x4 /* IEU1 */; \
194 fpadd32 %T0, %T1, %U0 /* FPA Group */; \
195 add %sum, %x3, %sum /* IEU0 */; \
196 fcmpgt32 %fz, %f2, %x7 /* FPM */; \
197 srl %x4, 1, %x4 /* IEU0 Group */; \
198 fcmpgt32 %fz, %f6, %x8 /* FPM */; \
199 inc %x5 /* IEU0 Group */; \
200 add %sum, %x4, %sum /* IEU1 */; \
201 srl %x5, 1, %x5 /* IEU0 Group */; \
202 fcmpgt32 %fz, %f10, %x1 /* FPM */; \
203 inc %x6 /* IEU0 Group */; \
204 add %sum, %x5, %sum /* IEU1 */; \
205 fmovd %FA, %FB /* FPA Group */; \
206 fcmpgt32 %fz, %f14, %x2 /* FPM */; \
207 srl %x6, 1, %x6 /* IEU0 Group */; \
208 ba,pt %xcc, ett /* CTI */; \
211 #define END_THE_TRICK1(f0,f2,f4,f6,f8,f10,f12,f14,FA,FB) \
212 END_THE_TRICK(f0,f2,f4,f6,f8,f10,f12,f14,FA,FB,f48,f50,f52,f54,f56,f58,f60,f62)
214 #define END_THE_TRICK2(S0,S1,S2,S3,T0,T1,U0,U1,V0,fz) \
215 fpadd32 %U0, %U1, %V0 /* FPA Group */; \
216 srl %x7, 1, %x7 /* IEU0 */; \
217 add %sum, %x6, %sum /* IEU1 */; \
218 std %V0, [%sp + STACKOFF] /* Store Group */; \
219 inc %x8 /* IEU0 */; \
220 sub %sum, %x7, %sum /* IEU1 */; \
221 srl %x8, 1, %x8 /* IEU0 Group */; \
222 fcmpgt32 %fz, %S1, %x3 /* FPM */; \
223 inc %x1 /* IEU0 Group */; \
224 fcmpgt32 %fz, %S3, %x4 /* FPM */; \
225 srl %x1, 1, %x1 /* IEU0 Group */; \
226 sub %sum, %x8, %sum /* IEU1 */; \
227 ldx [%sp + STACKOFF], %x8 /* Load Group */; \
228 inc %x2 /* IEU0 */; \
229 sub %sum, %x1, %sum /* IEU1 */; \
230 srl %x2, 1, %x2 /* IEU0 Group */; \
231 fcmpgt32 %fz, %T1, %x5 /* FPM */; \
232 inc %x3 /* IEU0 Group */; \
233 fcmpgt32 %T0, %U0, %x6 /* FPM */; \
234 srl %x3, 1, %x3 /* IEU0 Group */; \
235 sub %sum, %x2, %sum /* IEU1 */; \
236 inc %x4 /* IEU0 Group */; \
237 sub %sum, %x3, %sum /* IEU1 */; \
238 srl %x4, 1, %x4 /* IEU0 Group */; \
239 fcmpgt32 %fz, %U1, %x7 /* FPM */; \
240 inc %x5 /* IEU0 Group */; \
241 fcmpgt32 %U0, %V0, %x1 /* FPM */; \
242 srl %x5, 1, %x5 /* IEU0 Group */; \
243 sub %sum, %x4, %sum /* IEU1 */; \
244 sub %sum, %x5, %sum /* IEU0 Group */; \
245 fcmpgt32 %fz, %V0, %x2 /* FPM */; \
246 inc %x6 /* IEU0 Group */; \
247 inc %x7 /* IEU1 */; \
248 srl %x6, 1, %x6 /* IEU0 Group */; \
249 inc %x1 /* IEU1 */; \
250 srl %x7, 1, %x7 /* IEU0 Group */; \
251 add %sum, %x6, %sum /* IEU1 */; \
252 srl %x1, 1, %x1 /* IEU0 Group */; \
253 sub %sum, %x7, %sum /* IEU1 */; \
254 inc %x2 /* IEU0 Group */; \
255 add %sum, %x1, %sum /* IEU1 */; \
256 srl %x2, 1, %x2 /* IEU0 Group */; \
257 sub %sum, %x2, %sum /* IEU0 Group */; \
258 addcc %sum, %x8, %sum /* IEU1 Group */; \
259 bcs,a,pn %xcc, 33f /* CTI */; \
260 add %sum, 1, %sum /* IEU0 (Group) */; \
264 .globl csum_partial_copy_user_vis
266 /* %asi should be either ASI_P or ASI_AIUS for csum_partial_copy resp.
267 * csum_partial_copy_from_user
268 * This assumes that !((%src^%dst)&3) && !((%src|%dst)&1) && %len >= 256
270 csum_partial_copy_user_vis:
271 andcc %dst, 7, %g0 /* IEU1 Group */
272 be,pt %icc, 4f /* CTI */
273 and %dst, 0x38, %o4 /* IEU0 */
274 mov 1, %g5 /* IEU0 Group */
275 andcc %dst, 2, %g0 /* IEU1 */
276 be,pt %icc, 1f /* CTI */
277 and %dst, 4, %g7 /* IEU0 Group */
278 lduh [%src], %g2 /* Load */
279 sub %len, 2, %len /* IEU0 Group */
280 add %dst, 2, %dst /* IEU1 */
281 andcc %dst, 4, %g7 /* IEU1 Group */
282 sll %g5, 16, %g5 /* IEU0 */
283 stha %g2, [%dst - 2] %asi /* Store Group */
284 sll %g2, 16, %g2 /* IEU0 */
285 add %src, 2, %src /* IEU1 */
286 addcc %g2, %sum, %sum /* IEU1 Group */
287 bcs,a,pn %icc, 1f /* CTI */
288 add %sum, %g5, %sum /* IEU0 */
289 1: lduw [%src], %g2 /* Load */
290 brz,a,pn %g7, 4f /* CTI+IEU1 Group */
291 and %dst, 0x38, %o4 /* IEU0 */
292 add %dst, 4, %dst /* IEU0 Group */
293 sub %len, 4, %len /* IEU1 */
294 addcc %g2, %sum, %sum /* IEU1 Group */
295 bcs,a,pn %icc, 1f /* CTI */
296 add %sum, 1, %sum /* IEU0 */
297 1: and %dst, 0x38, %o4 /* IEU0 Group */
298 stwa %g2, [%dst - 4] %asi /* Store */
299 add %src, 4, %src /* IEU1 */
304 mov %src, %g7 /* IEU1 Group */
306 alignaddr %src, %g0, %src /* Single Group */
307 subcc %g7, %src, %g7 /* IEU1 Group */
308 be,pt %xcc, 1f /* CTI */
309 mov 0x40, %g1 /* IEU0 */
310 lduw [%src], %g2 /* Load Group */
311 subcc %sum, %g2, %sum /* IEU1 Group+load stall*/
312 bcs,a,pn %icc, 1f /* CTI */
313 sub %sum, 1, %sum /* IEU0 */
314 1: srl %sum, 0, %sum /* IEU0 Group */
316 brz,pn %o4, 3f /* CTI+IEU1 Group */
317 sub %g1, %o4, %g1 /* IEU0 */
318 ldd [%src], %f0 /* Load */
319 clr %o4 /* IEU0 Group */
320 andcc %dst, 8, %g0 /* IEU1 */
321 be,pn %icc, 1f /* CTI */
322 ldd [%src + 8], %f2 /* Load Group */
323 add %src, 8, %src /* IEU0 */
324 sub %len, 8, %len /* IEU1 */
325 fpadd32 %f0, %f48, %f50 /* FPA */
326 addcc %dst, 8, %dst /* IEU1 Group */
327 faligndata %f0, %f2, %f16 /* FPA */
328 fcmpgt32 %f48, %f50, %o4 /* FPM Group */
329 fmovd %f2, %f0 /* FPA Group */
330 ldd [%src + 8], %f2 /* Load */
331 stda %f16, [%dst - 8] %asi /* Store */
332 fmovd %f50, %f48 /* FPA */
333 1: andcc %g1, 0x10, %g0 /* IEU1 Group */
334 be,pn %icc, 1f /* CTI */
335 and %g1, 0x20, %g1 /* IEU0 */
336 fpadd32 %f0, %f48, %f50 /* FPA */
337 ldd [%src + 16], %f4 /* Load Group */
338 add %src, 16, %src /* IEU0 */
339 add %dst, 16, %dst /* IEU1 */
340 faligndata %f0, %f2, %f16 /* FPA */
341 fcmpgt32 %f48, %f50, %g5 /* FPM Group */
342 sub %len, 16, %len /* IEU0 */
344 stda %f16, [%dst - 16] %asi /* Store Group */
345 fpadd32 %f2, %f50, %f48 /* FPA */
346 srl %o4, 1, %o5 /* IEU0 */
347 faligndata %f2, %f4, %f18 /* FPA Group */
348 stda %f18, [%dst - 8] %asi /* Store */
349 fcmpgt32 %f50, %f48, %o4 /* FPM Group */
350 add %o5, %sum, %sum /* IEU0 */
351 ldd [%src + 8], %f2 /* Load */
352 fmovd %f4, %f0 /* FPA */
353 1: brz,a,pn %g1, 4f /* CTI+IEU1 Group */
354 rd %asi, %g2 /* LSU Group + 4 bubbles*/
356 fpadd32 %f0, %f48, %f50 /* FPA */
357 ldd [%src + 16], %f4 /* Load Group */
358 srl %g5, 1, %g5 /* IEU0 */
359 add %dst, 32, %dst /* IEU1 */
360 faligndata %f0, %f2, %f16 /* FPA */
361 fcmpgt32 %f48, %f50, %o5 /* FPM Group */
363 ldd [%src + 24], %f6 /* Load */
364 srl %o4, 1, %o4 /* IEU0 Group */
365 add %g5, %sum, %sum /* IEU1 */
366 ldd [%src + 32], %f8 /* Load */
367 fpadd32 %f2, %f50, %f48 /* FPA */
368 faligndata %f2, %f4, %f18 /* FPA Group */
369 sub %len, 32, %len /* IEU0 */
370 stda %f16, [%dst - 32] %asi /* Store */
371 fcmpgt32 %f50, %f48, %g3 /* FPM Group */
373 add %o4, %sum, %sum /* IEU1 */
374 fpadd32 %f4, %f48, %f50 /* FPA */
375 faligndata %f4, %f6, %f20 /* FPA Group */
376 srl %o5, 1, %o5 /* IEU0 */
377 fcmpgt32 %f48, %f50, %g5 /* FPM Group */
378 add %o5, %sum, %sum /* IEU0 */
379 stda %f18, [%dst - 24] %asi /* Store */
380 fpadd32 %f6, %f50, %f48 /* FPA */
381 inc %g3 /* IEU0 Group */
382 stda %f20, [%dst - 16] %asi /* Store */
383 add %src, 32, %src /* IEU1 */
384 faligndata %f6, %f8, %f22 /* FPA */
385 fcmpgt32 %f50, %f48, %o4 /* FPM Group */
386 srl %g3, 1, %g3 /* IEU0 */
387 stda %f22, [%dst - 8] %asi /* Store */
388 add %g3, %sum, %sum /* IEU0 Group */
389 3: rd %asi, %g2 /* LSU Group + 4 bubbles*/
391 4: sethi %hi(vis0s), %g7 /* IEU0 Group */
392 or %g2, ASI_BLK_OR, %g2 /* IEU1 */
394 4: rd %pc, %g7 /* LSU Group + 4 bubbles*/
396 inc %g5 /* IEU0 Group */
397 and %src, 0x38, %g3 /* IEU1 */
398 membar #StoreLoad /* LSU Group */
399 srl %g5, 1, %g5 /* IEU0 */
401 sll %g3, 8, %g3 /* IEU0 Group */
402 sub %len, 0xc0, %len /* IEU1 */
403 addcc %g5, %sum, %sum /* IEU1 Group */
404 srl %o4, 1, %o4 /* IEU0 */
405 add %g7, %g3, %g7 /* IEU0 Group */
406 add %o4, %sum, %sum /* IEU1 */
408 jmpl %g7 + %lo(vis0s), %g0 /* CTI+IEU1 Group */
410 jmpl %g7 + (vis0s - 4b), %g0 /* CTI+IEU1 Group */
415 vis0s: wr %g2, ASI_BLK_XOR, %asi /* LSU Group */
416 ldda [%src] ASI_BLK_P, %f0 /* Load Group */
417 add %src, 64, %src /* IEU0 Group */
418 ldda [%src] ASI_BLK_P, %f16 /* Load Group */
419 add %src, 64, %src /* IEU0 Group */
420 fmovd %f48, %f62 /* FPA Group f0 available*/
421 faligndata %f0, %f2, %f48 /* FPA Group f2 available*/
422 fcmpgt32 %f32, %f2, %x1 /* FPM Group f4 available*/
423 fpadd32 %f0, %f62, %f0 /* FPA */
424 fcmpgt32 %f32, %f4, %x2 /* FPM Group f6 available*/
425 faligndata %f2, %f4, %f50 /* FPA */
426 fcmpgt32 %f62, %f0, %x3 /* FPM Group f8 available*/
427 faligndata %f4, %f6, %f52 /* FPA */
428 fcmpgt32 %f32, %f6, %x4 /* FPM Group f10 available*/
430 faligndata %f6, %f8, %f54 /* FPA */
431 fcmpgt32 %f32, %f8, %x5 /* FPM Group f12 available*/
432 srl %x1, 1, %x1 /* IEU0 */
434 faligndata %f8, %f10, %f56 /* FPA */
435 fcmpgt32 %f32, %f10, %x6 /* FPM Group f14 available*/
436 srl %x2, 1, %x2 /* IEU0 */
437 add %sum, %x1, %sum /* IEU1 */
438 faligndata %f10, %f12, %f58 /* FPA */
439 fcmpgt32 %f32, %f12, %x7 /* FPM Group */
441 add %sum, %x2, %sum /* IEU1 */
442 faligndata %f12, %f14, %f60 /* FPA */
443 fcmpgt32 %f32, %f14, %x8 /* FPM Group */
444 srl %x3, 1, %x3 /* IEU0 */
446 fmovd %f14, %f62 /* FPA */
447 srl %x4, 1, %x4 /* IEU0 Group */
448 add %sum, %x3, %sum /* IEU1 */
449 vis0: DO_THE_TRICK( f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,
450 ,f48,f50,f52,f54,f56,f58,f60,f62,f62,
451 ,LDBLK(f32), STBLK,,,,,,,,
452 ,bcs,pn %icc, vis0e1)
453 DO_THE_TRICK( f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
454 ,f48,f50,f52,f54,f56,f58,f60,f62,f62,
455 ,LDBLK(f0), STBLK,,,,,,,,
456 ,bcs,pn %icc, vis0e2)
457 DO_THE_TRICK( f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,
458 ,f48,f50,f52,f54,f56,f58,f60,f62,f62,
459 ,LDBLK(f16), STBLK,,,,,,,,
461 vis0e3: DO_THE_TRICK( f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,
462 ,f48,f50,f52,f54,f56,f58,f60,f62,f32,
463 ,SYNC, STBLK_XORASI(x1,x2),ST(f48,64),ST(f50,8),ST(f52,16),ST(f54,24),ST(f56,32),ST(f58,40),ST(f60,48),
464 ,add %dst, 56, %dst; add %len, 192 - 8*8, %len; ba,pt %icc, e2)
465 vis0e1: DO_THE_TRICK( f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
466 ,f48,f50,f52,f54,f56,f58,f60,f62,f0,
467 ,SYNC, STBLK_XORASI(x1,x2),ST(f48,64),ST(f50,8),ST(f52,16),ST(f54,24),ST(f56,32),ST(f58,40),ST(f60,48),
468 ,add %dst, 56, %dst; add %len, 192 - 8*8, %len; ba,pt %icc, e3)
469 vis0e2: DO_THE_TRICK( f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,
470 ,f48,f50,f52,f54,f56,f58,f60,f62,f16,
471 ,SYNC, STBLK_XORASI(x1,x2),ST(f48,64),ST(f50,8),ST(f52,16),ST(f54,24),ST(f56,32),ST(f58,40),ST(f60,48),
472 ,add %dst, 56, %dst; add %len, 192 - 8*8, %len; ba,pt %icc, e1)
474 vis1s: wr %g2, ASI_BLK_XOR, %asi /* LSU Group */
475 sub %src, 8, %src /* IEU0 Group */
476 ldda [%src] ASI_BLK_P, %f0 /* Load Group */
477 add %src, 64, %src /* IEU0 Group */
478 ldda [%src] ASI_BLK_P, %f16 /* Load Group */
479 add %src, 64, %src /* IEU0 Group */
480 fmovd %f0, %f58 /* FPA Group */
481 fmovd %f48, %f0 /* FPA Group */
482 fcmpgt32 %f32, %f2, %x2 /* FPM Group */
483 faligndata %f2, %f4, %f48 /* FPA */
484 fcmpgt32 %f32, %f4, %x3 /* FPM Group */
485 faligndata %f4, %f6, %f50 /* FPA */
486 fcmpgt32 %f32, %f6, %x4 /* FPM Group */
487 faligndata %f6, %f8, %f52 /* FPA */
488 fcmpgt32 %f32, %f8, %x5 /* FPM Group */
490 faligndata %f8, %f10, %f54 /* FPA */
491 fcmpgt32 %f32, %f10, %x6 /* FPM Group */
492 srl %x2, 1, %x2 /* IEU0 */
493 faligndata %f10, %f12, %f56 /* FPA */
494 fcmpgt32 %f32, %f12, %x7 /* FPM Group */
496 add %sum, %x2, %sum /* IEU1 */
497 faligndata %f12, %f14, %f58 /* FPA */
498 fcmpgt32 %f32, %f14, %x8 /* FPM Group */
499 srl %x3, 1, %x3 /* IEU0 */
501 fmovd %f14, %f60 /* FPA */
502 srl %x4, 1, %x4 /* IEU0 Group */
503 add %sum, %x3, %sum /* IEU1 */
504 vis1: DO_THE_TRICK( f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,
505 ,f62,f48,f50,f52,f54,f56,f58,f60,f60,
506 ,LDBLK(f32), ,STBLK,,,,,,,
507 ,bcs,pn %icc, vis1e1)
508 DO_THE_TRICK( f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
509 ,f62,f48,f50,f52,f54,f56,f58,f60,f60,
510 ,LDBLK(f0), ,STBLK,,,,,,,
511 ,bcs,pn %icc, vis1e2)
512 DO_THE_TRICK( f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,
513 ,f62,f48,f50,f52,f54,f56,f58,f60,f60,
514 ,LDBLK(f16), ,STBLK,,,,,,,
516 vis1e3: DO_THE_TRICK( f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,
517 ,f62,f48,f50,f52,f54,f56,f58,f60,f32,
518 ,SYNC, ,STBLK_XORASI(x1,x2),ST(f48,0),ST(f50,8),ST(f52,16),ST(f54,24),ST(f56,32),ST(f58,40),
519 ,add %dst, 48, %dst; add %len, 192 - 7*8, %len; ba,pt %icc, e2)
520 vis1e1: DO_THE_TRICK( f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
521 ,f62,f48,f50,f52,f54,f56,f58,f60,f0,
522 ,SYNC, ,STBLK_XORASI(x1,x2),ST(f48,0),ST(f50,8),ST(f52,16),ST(f54,24),ST(f56,32),ST(f58,40),
523 ,add %dst, 48, %dst; add %len, 192 - 7*8, %len; ba,pt %icc, e3)
524 vis1e2: DO_THE_TRICK( f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,
525 ,f62,f48,f50,f52,f54,f56,f58,f60,f16,
526 ,SYNC, ,STBLK_XORASI(x1,x2),ST(f48,0),ST(f50,8),ST(f52,16),ST(f54,24),ST(f56,32),ST(f58,40),
527 ,add %dst, 48, %dst; add %len, 192 - 7*8, %len; ba,pt %icc, e1)
529 vis2s: wr %g2, ASI_BLK_XOR, %asi /* LSU Group */
530 sub %src, 16, %src /* IEU0 Group */
531 ldda [%src] ASI_BLK_P, %f0 /* Load Group */
532 add %src, 64, %src /* IEU0 Group */
533 ldda [%src] ASI_BLK_P, %f16 /* Load Group */
534 add %src, 64, %src /* IEU0 Group */
535 fmovd %f0, %f56 /* FPA Group */
536 fmovd %f48, %f0 /* FPA Group */
537 sub %dst, 64, %dst /* IEU0 */
538 fpsub32 %f2, %f2, %f2 /* FPA Group */
539 fcmpgt32 %f32, %f4, %x3 /* FPM Group */
540 faligndata %f4, %f6, %f48 /* FPA */
541 fcmpgt32 %f32, %f6, %x4 /* FPM Group */
542 faligndata %f6, %f8, %f50 /* FPA */
543 fcmpgt32 %f32, %f8, %x5 /* FPM Group */
544 faligndata %f8, %f10, %f52 /* FPA */
545 fcmpgt32 %f32, %f10, %x6 /* FPM Group */
546 faligndata %f10, %f12, %f54 /* FPA */
547 fcmpgt32 %f32, %f12, %x7 /* FPM Group */
549 faligndata %f12, %f14, %f56 /* FPA */
550 fcmpgt32 %f32, %f14, %x8 /* FPM Group */
551 srl %x3, 1, %x3 /* IEU0 */
553 fmovd %f14, %f58 /* FPA */
554 srl %x4, 1, %x4 /* IEU0 Group */
555 add %sum, %x3, %sum /* IEU1 */
556 vis2: DO_THE_TRICK( f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,
557 ,f60,f62,f48,f50,f52,f54,f56,f58,f58,
558 ,LDBLK(f32), ,,STBLK,,,,,,
559 ,bcs,pn %icc, vis2e1)
560 DO_THE_TRICK( f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
561 ,f60,f62,f48,f50,f52,f54,f56,f58,f58,
562 ,LDBLK(f0), ,,STBLK,,,,,,
563 ,bcs,pn %icc, vis2e2)
564 DO_THE_TRICK( f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,
565 ,f60,f62,f48,f50,f52,f54,f56,f58,f58,
566 ,LDBLK(f16), ,,STBLK,,,,,,
568 vis2e3: DO_THE_TRICK( f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,
569 ,f60,f62,f48,f50,f52,f54,f56,f58,f32,
570 ,SYNC, ,,STBLK_XORASI(x2,x3),ST(f48,64),ST(f50,72),ST(f52,80),ST(f54,88),ST(f56,96),
571 ,add %dst, 104, %dst; add %len, 192 - 6*8, %len; ba,pt %icc, e2)
572 vis2e1: DO_THE_TRICK( f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
573 ,f60,f62,f48,f50,f52,f54,f56,f58,f0,
574 ,SYNC, ,,STBLK_XORASI(x2,x3),ST(f48,64),ST(f50,72),ST(f52,80),ST(f54,88),ST(f56,96),
575 ,add %dst, 104, %dst; add %len, 192 - 6*8, %len; ba,pt %icc, e3)
576 vis2e2: DO_THE_TRICK( f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,
577 ,f60,f62,f48,f50,f52,f54,f56,f58,f16,
578 ,SYNC, ,,STBLK_XORASI(x2,x3),ST(f48,64),ST(f50,72),ST(f52,80),ST(f54,88),ST(f56,96),
579 ,add %dst, 104, %dst; add %len, 192 - 6*8, %len; ba,pt %icc, e1)
581 vis3s: wr %g2, ASI_BLK_XOR, %asi /* LSU Group */
582 sub %src, 24, %src /* IEU0 Group */
583 ldda [%src] ASI_BLK_P, %f0 /* Load Group */
584 add %src, 64, %src /* IEU0 Group */
585 ldda [%src] ASI_BLK_P, %f16 /* Load Group */
586 add %src, 64, %src /* IEU0 Group */
587 fmovd %f0, %f54 /* FPA Group */
588 fmovd %f48, %f0 /* FPA Group */
589 sub %dst, 64, %dst /* IEU0 */
590 fpsub32 %f2, %f2, %f2 /* FPA Group */
591 fpsub32 %f4, %f4, %f4 /* FPA Group */
592 fcmpgt32 %f32, %f6, %x4 /* FPM Group */
593 faligndata %f6, %f8, %f48 /* FPA */
594 fcmpgt32 %f32, %f8, %x5 /* FPM Group */
595 faligndata %f8, %f10, %f50 /* FPA */
596 fcmpgt32 %f32, %f10, %x6 /* FPM Group */
597 faligndata %f10, %f12, %f52 /* FPA */
598 fcmpgt32 %f32, %f12, %x7 /* FPM Group */
599 faligndata %f12, %f14, %f54 /* FPA */
600 fcmpgt32 %f32, %f14, %x8 /* FPM Group */
601 fmovd %f14, %f56 /* FPA */
603 srl %x4, 1, %x4 /* IEU0 Group */
604 vis3: DO_THE_TRICK( f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,
605 ,f58,f60,f62,f48,f50,f52,f54,f56,f56,
606 ,LDBLK(f32), ,,,STBLK,,,,,
607 ,bcs,pn %icc, vis3e1)
608 DO_THE_TRICK( f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
609 ,f58,f60,f62,f48,f50,f52,f54,f56,f56,
610 ,LDBLK(f0), ,,,STBLK,,,,,
611 ,bcs,pn %icc, vis3e2)
612 DO_THE_TRICK( f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,
613 ,f58,f60,f62,f48,f50,f52,f54,f56,f56,
614 ,LDBLK(f16), ,,,STBLK,,,,,
616 vis3e3: DO_THE_TRICK( f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,
617 ,f58,f60,f62,f48,f50,f52,f54,f56,f32,
618 ,SYNC, ,,,STBLK_XORASI(x3,x4),ST(f48,64),ST(f50,72),ST(f52,80),ST(f54,88),
619 ,add %dst, 96, %dst; add %len, 192 - 5*8, %len; ba,pt %icc, e2)
620 vis3e1: DO_THE_TRICK( f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
621 ,f58,f60,f62,f48,f50,f52,f54,f56,f0,
622 ,SYNC, ,,,STBLK_XORASI(x3,x4),ST(f48,64),ST(f50,72),ST(f52,80),ST(f54,88),
623 ,add %dst, 96, %dst; add %len, 192 - 5*8, %len; ba,pt %icc, e3)
624 vis3e2: DO_THE_TRICK( f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,
625 ,f58,f60,f62,f48,f50,f52,f54,f56,f16,
626 ,SYNC, ,,,STBLK_XORASI(x3,x4),ST(f48,64),ST(f50,72),ST(f52,80),ST(f54,88),
627 ,add %dst, 96, %dst; add %len, 192 - 5*8, %len; ba,pt %icc, e1)
629 vis4s: wr %g2, ASI_BLK_XOR, %asi /* LSU Group */
630 sub %src, 32, %src /* IEU0 Group */
631 ldda [%src] ASI_BLK_P, %f0 /* Load Group */
632 add %src, 64, %src /* IEU0 Group */
633 ldda [%src] ASI_BLK_P, %f16 /* Load Group */
634 add %src, 64, %src /* IEU0 Group */
635 fmovd %f0, %f52 /* FPA Group */
636 fmovd %f48, %f0 /* FPA Group */
637 sub %dst, 64, %dst /* IEU0 */
638 fpsub32 %f2, %f2, %f2 /* FPA Group */
639 fpsub32 %f4, %f4, %f4 /* FPA Group */
640 fpsub32 %f6, %f6, %f6 /* FPA Group */
642 fcmpgt32 %f32, %f8, %x5 /* FPM Group */
643 faligndata %f8, %f10, %f48 /* FPA */
644 fcmpgt32 %f32, %f10, %x6 /* FPM Group */
645 faligndata %f10, %f12, %f50 /* FPA */
646 fcmpgt32 %f32, %f12, %x7 /* FPM Group */
647 faligndata %f12, %f14, %f52 /* FPA */
648 fcmpgt32 %f32, %f14, %x8 /* FPM Group */
649 fmovd %f14, %f54 /* FPA */
650 vis4: DO_THE_TRICK( f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,
651 ,f56,f58,f60,f62,f48,f50,f52,f54,f54,
652 ,LDBLK(f32), ,,,,STBLK,,,,
653 ,bcs,pn %icc, vis4e1)
654 DO_THE_TRICK( f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
655 ,f56,f58,f60,f62,f48,f50,f52,f54,f54,
656 ,LDBLK(f0), ,,,,STBLK,,,,
657 ,bcs,pn %icc, vis4e2)
658 DO_THE_TRICK( f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,
659 ,f56,f58,f60,f62,f48,f50,f52,f54,f54,
660 ,LDBLK(f16), ,,,,STBLK,,,,
662 vis4e3: DO_THE_TRICK( f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,
663 ,f56,f58,f60,f62,f48,f50,f52,f54,f32,
664 ,SYNC, ,,,,STBLK_XORASI(x4,x5),ST(f48,64),ST(f50,72),ST(f52,80),
665 ,add %dst, 88, %dst; add %len, 192 - 4*8, %len; ba,pt %icc, e2)
666 vis4e1: DO_THE_TRICK( f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
667 ,f56,f58,f60,f62,f48,f50,f52,f54,f0,
668 ,SYNC, ,,,,STBLK_XORASI(x4,x5),ST(f48,64),ST(f50,72),ST(f52,80),
669 ,add %dst, 88, %dst; add %len, 192 - 4*8, %len; ba,pt %icc, e3)
670 vis4e2: DO_THE_TRICK( f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,
671 ,f56,f58,f60,f62,f48,f50,f52,f54,f16,
672 ,SYNC, ,,,,STBLK_XORASI(x4,x5),ST(f48,64),ST(f50,72),ST(f52,80),
673 ,add %dst, 88, %dst; add %len, 192 - 4*8, %len; ba,pt %icc, e1)
675 vis5s: ldd [%src+0], %f10 /* Load Group */
676 ldd [%src+8], %f12 /* Load Group */
677 ldd [%src+16], %f14 /* Load Group */
678 add %src, 24, %src /* IEU0 Group */
679 wr %g2, ASI_BLK_XOR, %asi /* LSU Group */
680 ldda [%src] ASI_BLK_P, %f16 /* Load Group */
681 add %src, 64, %src /* IEU0 Group */
682 fmovd %f48, %f0 /* FPA Group */
683 fmuld %f32, %f32, %f2 /* FPM */
685 faddd %f32, %f32, %f4 /* FPA Group */
686 fmuld %f32, %f32, %f6 /* FPM */
688 faddd %f32, %f32, %f8 /* FPA Group */
689 fcmpgt32 %f32, %f10, %x6 /* FPM Group */
690 sub %dst, 64, %dst /* IEU0 */
691 faligndata %f10, %f12, %f48 /* FPA */
692 fcmpgt32 %f32, %f12, %x7 /* FPM Group */
693 faligndata %f12, %f14, %f50 /* FPA */
694 fcmpgt32 %f32, %f14, %x8 /* FPM Group */
695 fmovd %f14, %f52 /* FPA */
696 vis5: DO_THE_TRICK( f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,
697 ,f54,f56,f58,f60,f62,f48,f50,f52,f52,
698 ,LDBLK(f32), ,,,,,STBLK,,,
699 ,bcs,pn %icc, vis5e1)
700 DO_THE_TRICK( f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
701 ,f54,f56,f58,f60,f62,f48,f50,f52,f52,
702 ,LDBLK(f0), ,,,,,STBLK,,,
703 ,bcs,pn %icc, vis5e2)
704 DO_THE_TRICK( f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,
705 ,f54,f56,f58,f60,f62,f48,f50,f52,f52,
706 ,LDBLK(f16), ,,,,,STBLK,,,
708 vis5e3: DO_THE_TRICK( f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,
709 ,f54,f56,f58,f60,f62,f48,f50,f52,f32,
710 ,SYNC, ,,,,,STBLK_XORASI(x5,x6),ST(f48,64),ST(f50,72),
711 ,add %dst, 80, %dst; add %len, 192 - 3*8, %len; ba,pt %icc, e2)
712 vis5e1: DO_THE_TRICK( f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
713 ,f54,f56,f58,f60,f62,f48,f50,f52,f0,
714 ,SYNC, ,,,,,STBLK_XORASI(x5,x6),ST(f48,64),ST(f50,72),
715 ,add %dst, 80, %dst; add %len, 192 - 3*8, %len; ba,pt %icc, e3)
716 vis5e2: DO_THE_TRICK( f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,
717 ,f54,f56,f58,f60,f62,f48,f50,f52,f16,
718 ,SYNC, ,,,,,STBLK_XORASI(x5,x6),ST(f48,64),ST(f50,72),
719 ,add %dst, 80, %dst; add %len, 192 - 3*8, %len; ba,pt %icc, e1)
721 vis6s: ldd [%src+0], %f12 /* Load Group */
722 ldd [%src+8], %f14 /* Load Group */
723 add %src, 16, %src /* IEU0 Group */
724 wr %g2, ASI_BLK_XOR, %asi /* LSU Group */
725 ldda [%src] ASI_BLK_P, %f16 /* Load Group */
726 add %src, 64, %src /* IEU0 Group */
727 fmovd %f48, %f0 /* FPA Group */
728 fmuld %f32, %f32, %f2 /* FPM */
730 faddd %f32, %f32, %f4 /* FPA Group */
731 fmuld %f32, %f32, %f6 /* FPM */
733 faddd %f32, %f32, %f8 /* FPA Group */
734 fmuld %f32, %f32, %f10 /* FPM */
736 fcmpgt32 %f32, %f12, %x7 /* FPM Group */
737 sub %dst, 64, %dst /* IEU0 */
738 fcmpgt32 %f32, %f14, %x8 /* FPM Group */
739 faligndata %f12, %f14, %f48 /* FPA */
740 fmovd %f14, %f50 /* FPA Group */
741 vis6: DO_THE_TRICK( f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,
742 ,f52,f54,f56,f58,f60,f62,f48,f50,f50,
743 ,LDBLK(f32), ,,,,,,STBLK,,
744 ,bcs,pn %icc, vis6e1)
745 DO_THE_TRICK( f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
746 ,f52,f54,f56,f58,f60,f62,f48,f50,f50,
747 ,LDBLK(f0), ,,,,,,STBLK,,
748 ,bcs,pn %icc, vis6e2)
749 DO_THE_TRICK( f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,
750 ,f52,f54,f56,f58,f60,f62,f48,f50,f50,
751 ,LDBLK(f16), ,,,,,,STBLK,,
753 vis6e3: DO_THE_TRICK( f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,
754 ,f52,f54,f56,f58,f60,f62,f48,f50,f32,
755 ,SYNC, ,,,,,,STBLK_XORASI(x6,x7),ST(f48,64),
756 ,add %dst, 72, %dst; add %len, 192 - 2*8, %len; ba,pt %icc, e2)
757 vis6e1: DO_THE_TRICK( f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
758 ,f52,f54,f56,f58,f60,f62,f48,f50,f0,
759 ,SYNC, ,,,,,,STBLK_XORASI(x6,x7),ST(f48,64),
760 ,add %dst, 72, %dst; add %len, 192 - 2*8, %len; ba,pt %icc, e3)
761 vis6e2: DO_THE_TRICK( f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,
762 ,f52,f54,f56,f58,f60,f62,f48,f50,f16,
763 ,SYNC, ,,,,,,STBLK_XORASI(x6,x7),ST(f48,64),
764 ,add %dst, 72, %dst; add %len, 192 - 2*8, %len; ba,pt %icc, e1)
766 vis7s: ldd [%src+0], %f14 /* Load Group */
767 add %src, 8, %src /* IEU0 Group */
768 wr %g2, ASI_BLK_XOR, %asi /* LSU Group */
769 ldda [%src] ASI_BLK_P, %f16 /* Load Group */
770 add %src, 64, %src /* IEU0 Group */
771 fmovd %f48, %f0 /* FPA Group */
772 fmuld %f32, %f32, %f2 /* FPM */
774 faddd %f32, %f32, %f4 /* FPA Group */
775 fmuld %f32, %f32, %f6 /* FPM */
777 faddd %f32, %f32, %f8 /* FPA Group */
778 fmuld %f32, %f32, %f10 /* FPM */
780 faddd %f32, %f32, %f12 /* FPA Group */
782 fcmpgt32 %f32, %f14, %x8 /* FPM Group */
783 sub %dst, 64, %dst /* IEU0 */
784 fmovd %f14, %f48 /* FPA */
785 vis7: DO_THE_TRICK( f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,
786 ,f50,f52,f54,f56,f58,f60,f62,f48,f48,
787 ,LDBLK(f32), ,,,,,,,STBLK,
788 ,bcs,pn %icc, vis7e1)
789 DO_THE_TRICK( f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
790 ,f50,f52,f54,f56,f58,f60,f62,f48,f48,
791 ,LDBLK(f0), ,,,,,,,STBLK,
792 ,bcs,pn %icc, vis7e2)
793 DO_THE_TRICK( f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,
794 ,f50,f52,f54,f56,f58,f60,f62,f48,f48,
795 ,LDBLK(f16), ,,,,,,,STBLK,
797 vis7e3: DO_THE_TRICK( f0,f2,f4,f6,f8,f10,f12,f14,f16,f18,f20,f22,f24,f26,f28,f30,
798 ,f50,f52,f54,f56,f58,f60,f62,f48,f32,
799 ,SYNC, ,,,,,,,STBLK_XORASI(x7,x8),
800 ,add %dst, 64, %dst; add %len, 192 - 1*8, %len; ba,pt %icc, e2)
801 vis7e1: DO_THE_TRICK( f16,f18,f20,f22,f24,f26,f28,f30,f32,f34,f36,f38,f40,f42,f44,f46,
802 ,f50,f52,f54,f56,f58,f60,f62,f48,f0,
803 ,SYNC, ,,,,,,,STBLK_XORASI(x7,x8),
804 ,add %dst, 64, %dst; add %len, 192 - 1*8, %len; ba,pt %icc, e3)
805 vis7e2: DO_THE_TRICK( f32,f34,f36,f38,f40,f42,f44,f46,f0,f2,f4,f6,f8,f10,f12,f14,
806 ,f50,f52,f54,f56,f58,f60,f62,f48,f16,
807 ,SYNC, ,,,,,,,STBLK_XORASI(x7,x8),
808 ,add %dst, 64, %dst; add %len, 192 - 1*8, %len; ba,pt %icc, e1)
809 e1: END_THE_TRICK1( f0,f2,f4,f6,f8,f10,f12,f14,f16,f6)
810 e2: END_THE_TRICK1( f16,f18,f20,f22,f24,f26,f28,f30,f32,f6)
811 e3: END_THE_TRICK1( f32,f34,f36,f38,f40,f42,f44,f46,f0,f6)
812 ett: rd %gsr, %x3 /* LSU Group+4bubbles */
813 andcc %x3, 7, %x3 /* IEU1 Group */
814 add %dst, 8, %dst /* IEU0 */
815 bne,pn %icc, 1f /* CTI */
817 brz,a,pn %len, 2f /* CTI+IEU1 Group */
818 stda %f6, [%dst - 8] %asi /* Store */
819 1: cmp %len, 8 /* IEU1 */
820 blu,pn %icc, 3f /* CTI */
821 sub %src, 64, %src /* IEU0 Group */
822 1: ldd [%src], %f2 /* Load Group */
823 fpadd32 %f10, %f2, %f12 /* FPA Group+load stall*/
824 add %src, 8, %src /* IEU0 */
825 add %dst, 8, %dst /* IEU1 */
826 faligndata %f6, %f2, %f14 /* FPA Group */
827 fcmpgt32 %f10, %f12, %x5 /* FPM Group */
828 stda %f14, [%dst - 16] %asi /* Store */
829 fmovd %f2, %f6 /* FPA */
830 fmovd %f12, %f10 /* FPA Group */
831 sub %len, 8, %len /* IEU1 */
832 fzero %f16 /* FPA Group - FPU nop */
833 fzero %f18 /* FPA Group - FPU nop */
835 srl %x5, 1, %x5 /* IEU0 Group (regdep) */
836 cmp %len, 8 /* IEU1 */
837 bgeu,pt %icc, 1b /* CTI */
838 add %x5, %sum, %sum /* IEU0 Group */
839 3: brz,a,pt %x3, 2f /* CTI+IEU1 */
840 stda %f6, [%dst - 8] %asi /* Store Group */
841 sta %f7, [%dst - 8] %asi /* Store Group */
842 sub %dst, 4, %dst /* IEU0 */
843 add %len, 4, %len /* IEU1 */
846 sub %sp, 8, %sp /* IEU0 Group */
848 END_THE_TRICK2( f48,f50,f52,f54,f56,f58,f60,f10,f12,f62)
849 membar #Sync /* LSU Group */
852 add %sp, 8, %sp /* IEU0 Group */
854 23: brnz,pn %len, 26f /* CTI+IEU1 Group */
855 24: sllx %sum, 32, %g1 /* IEU0 */
856 25: addcc %sum, %g1, %src /* IEU1 Group */
857 srlx %src, 32, %src /* IEU0 Group (regdep) */
858 bcs,a,pn %xcc, 1f /* CTI */
859 add %src, 1, %src /* IEU1 */
861 1: retl /* CTI Group brk forced*/
862 srl %src, 0, %src /* IEU0 */
864 1: retl /* CTI Group brk forced*/
865 ldx [%g6 + TI_TASK], %g4 /* Load */
867 26: andcc %len, 8, %g0 /* IEU1 Group */
868 be,pn %icc, 1f /* CTI */
869 lduw [%src], %o4 /* Load */
870 lduw [%src+4], %g2 /* Load Group */
871 add %src, 8, %src /* IEU0 */
872 add %dst, 8, %dst /* IEU1 */
873 sllx %o4, 32, %g5 /* IEU0 Group */
874 stwa %o4, [%dst - 8] %asi /* Store */
875 or %g5, %g2, %g5 /* IEU0 Group */
876 stwa %g2, [%dst - 4] %asi /* Store */
877 addcc %g5, %sum, %sum /* IEU1 Group */
878 bcs,a,pn %xcc, 1f /* CTI */
879 add %sum, 1, %sum /* IEU0 */
880 1: andcc %len, 4, %g0 /* IEU1 Group */
881 be,a,pn %icc, 1f /* CTI */
883 lduw [%src], %g7 /* Load */
884 add %src, 4, %src /* IEU0 Group */
885 add %dst, 4, %dst /* IEU1 */
886 sllx %g7, 32, %g2 /* IEU0 Group */
887 stwa %g7, [%dst - 4] %asi /* Store */
888 1: andcc %len, 2, %g0 /* IEU1 */
889 be,a,pn %icc, 1f /* CTI */
890 clr %g3 /* IEU0 Group */
891 lduh [%src], %g7 /* Load */
892 add %src, 2, %src /* IEU1 */
893 add %dst, 2, %dst /* IEU0 Group */
894 sll %g7, 16, %g3 /* IEU0 Group */
895 stha %g7, [%dst - 2] %asi /* Store */
896 1: andcc %len, 1, %g0 /* IEU1 */
897 be,a,pn %icc, 1f /* CTI */
898 clr %o5 /* IEU0 Group */
899 ldub [%src], %g7 /* Load */
900 sll %g7, 8, %o5 /* IEU0 Group */
901 stba %g7, [%dst] %asi /* Store */
902 1: or %g2, %g3, %g3 /* IEU1 */
903 or %o5, %g3, %g3 /* IEU0 Group (regdep) */
904 addcc %g3, %sum, %sum /* IEU1 Group (regdep) */
905 bcs,a,pn %xcc, 1f /* CTI */
906 add %sum, 1, %sum /* IEU0 */
907 1: ba,pt %xcc, 25b /* CTI Group */
908 sllx %sum, 32, %g1 /* IEU0 */
915 .word csum_partial_copy_user_vis, 0, end, cpc_handler