1 /* $Id: blockops.S,v 1.42 2002/02/09 19:49:30 davem Exp $
2 * blockops.S: UltraSparc block zero optimized routines.
4 * Copyright (C) 1996, 1998, 1999, 2000 David S. Miller (davem@redhat.com)
5 * Copyright (C) 1997 Jakub Jelinek (jakub@redhat.com)
9 #include <asm/visasm.h>
10 #include <asm/thread_info.h>
13 #include <asm/spitfire.h>
14 #include <asm/pgtable.h>
16 #define TOUCH(reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7) \
17 fmovd %reg0, %f48; fmovd %reg1, %f50; \
18 fmovd %reg2, %f52; fmovd %reg3, %f54; \
19 fmovd %reg4, %f56; fmovd %reg5, %f58; \
20 fmovd %reg6, %f60; fmovd %reg7, %f62;
22 #define DCACHE_SIZE (PAGE_SIZE * 2)
23 #define TLBTEMP_ENT1 (60 << 3)
24 #define TLBTEMP_ENT2 (61 << 3)
25 #define TLBTEMP_ENTSZ (1 << 3)
27 #if (PAGE_SHIFT == 13) || (PAGE_SHIFT == 19)
28 #define PAGE_SIZE_REM 0x80
29 #elif (PAGE_SHIFT == 16) || (PAGE_SHIFT == 22)
30 #define PAGE_SIZE_REM 0x100
32 #error Wrong PAGE_SHIFT specified
39 .type copy_user_page,@function
40 copy_user_page: /* %o0=dest, %o1=src, %o2=vaddr */
42 sethi %hi(PAGE_SIZE), %g3
43 sethi %uhi(PAGE_OFFSET), %g2
47 sethi %hi(TLBTEMP_BASE), %o3
48 sethi %uhi(_PAGE_VALID | _PAGE_SZBITS), %g3
51 mov TLB_TAG_ACCESS, %o2
52 or %g3, (_PAGE_CP | _PAGE_CV | _PAGE_P | _PAGE_L | _PAGE_W), %g3
53 sethi %hi(DCACHE_SIZE), %o1
58 #define FIX_INSN_1 0x96102060 /* mov (12 << 3), %o3 */
62 wrpr %g3, PSTATE_IE, %pstate
64 /* Do this now, before loading the fixed TLB entries for copying,
65 * so we do not risk a multiple TLB match condition later when
66 * restoring those entries.
68 ldx [%g6 + TI_FLAGS], %g3
70 /* Spitfire Errata #32 workaround */
71 mov PRIMARY_CONTEXT, %o4
72 stxa %g0, [%o4] ASI_DMMU
75 ldxa [%o3] ASI_DTLB_TAG_READ, %o4
77 /* Spitfire Errata #32 workaround */
78 mov PRIMARY_CONTEXT, %o5
79 stxa %g0, [%o5] ASI_DMMU
82 ldxa [%o3] ASI_DTLB_DATA_ACCESS, %g0
83 ldxa [%o3] ASI_DTLB_DATA_ACCESS, %o5
84 stxa %o0, [%o2] ASI_DMMU
85 stxa %g1, [%o3] ASI_DTLB_DATA_ACCESS
87 add %o3, (TLBTEMP_ENTSZ), %o3
89 /* Spitfire Errata #32 workaround */
90 mov PRIMARY_CONTEXT, %g5
91 stxa %g0, [%g5] ASI_DMMU
94 ldxa [%o3] ASI_DTLB_TAG_READ, %g5
96 /* Spitfire Errata #32 workaround */
97 mov PRIMARY_CONTEXT, %g7
98 stxa %g0, [%g7] ASI_DMMU
101 ldxa [%o3] ASI_DTLB_DATA_ACCESS, %g0
102 ldxa [%o3] ASI_DTLB_DATA_ACCESS, %g7
103 stxa %o1, [%o2] ASI_DMMU
104 stxa %g2, [%o3] ASI_DTLB_DATA_ACCESS
107 andcc %g3, _TIF_BLKCOMMIT, %g0
108 bne,pn %xcc, copy_page_using_blkcommit
111 BRANCH_IF_ANY_CHEETAH(g3,o2,cheetah_copy_user_page)
112 ba,pt %xcc, spitfire_copy_user_page
115 cheetah_copy_user_page:
116 .globl cheetah_copy_user_page_nop_1_6
117 cheetah_copy_user_page_nop_1_6:
118 ldxa [%g0] ASI_DCU_CONTROL_REG, %g3
119 sethi %uhi(DCU_PE), %o2
122 stxa %o2, [%g0] ASI_DCU_CONTROL_REG ! Enable P-cache
125 sethi %hi((PAGE_SIZE/64)-7), %o2 ! A0 Group
126 prefetch [%o1 + 0x000], #one_read ! MS
127 or %o2, %lo((PAGE_SIZE/64)-7), %o2 ! A1 Group
128 prefetch [%o1 + 0x040], #one_read ! MS
129 prefetch [%o1 + 0x080], #one_read ! MS Group
130 prefetch [%o1 + 0x0c0], #one_read ! MS Group
131 ldd [%o1 + 0x000], %f0 ! MS Group
132 prefetch [%o1 + 0x100], #one_read ! MS Group
133 ldd [%o1 + 0x008], %f2 ! AX
134 prefetch [%o1 + 0x140], #one_read ! MS Group
135 ldd [%o1 + 0x010], %f4 ! AX
136 prefetch [%o1 + 0x180], #one_read ! MS Group
137 fmovd %f0, %f32 ! FGA Group
138 ldd [%o1 + 0x018], %f6 ! AX
139 fmovd %f2, %f34 ! FGA Group
140 ldd [%o1 + 0x020], %f8 ! MS
141 fmovd %f4, %f36 ! FGA Group
142 ldd [%o1 + 0x028], %f10 ! AX
143 membar #StoreStore ! MS
144 fmovd %f6, %f38 ! FGA Group
145 ldd [%o1 + 0x030], %f12 ! MS
146 fmovd %f8, %f40 ! FGA Group
147 ldd [%o1 + 0x038], %f14 ! AX
148 fmovd %f10, %f42 ! FGA Group
149 ldd [%o1 + 0x040], %f16 ! MS
150 1: ldd [%o1 + 0x048], %f2 ! AX (Group)
151 fmovd %f12, %f44 ! FGA
152 ldd [%o1 + 0x050], %f4 ! MS
153 fmovd %f14, %f46 ! FGA Group
154 stda %f32, [%o0] ASI_BLK_P ! MS
155 ldd [%o1 + 0x058], %f6 ! AX
156 fmovd %f16, %f32 ! FGA Group (8-cycle stall)
157 ldd [%o1 + 0x060], %f8 ! MS
158 fmovd %f2, %f34 ! FGA Group
159 ldd [%o1 + 0x068], %f10 ! AX
160 fmovd %f4, %f36 ! FGA Group
161 ldd [%o1 + 0x070], %f12 ! MS
162 fmovd %f6, %f38 ! FGA Group
163 ldd [%o1 + 0x078], %f14 ! AX
164 fmovd %f8, %f40 ! FGA Group
165 ldd [%o1 + 0x080], %f16 ! AX
166 prefetch [%o1 + 0x180], #one_read ! MS
167 fmovd %f10, %f42 ! FGA Group
168 subcc %o2, 1, %o2 ! A0
169 add %o0, 0x40, %o0 ! A1
171 add %o1, 0x40, %o1 ! A0 Group
173 mov 5, %o2 ! A0 Group
174 1: ldd [%o1 + 0x048], %f2 ! AX
175 fmovd %f12, %f44 ! FGA
176 ldd [%o1 + 0x050], %f4 ! MS
177 fmovd %f14, %f46 ! FGA Group
178 stda %f32, [%o0] ASI_BLK_P ! MS
179 ldd [%o1 + 0x058], %f6 ! AX
180 fmovd %f16, %f32 ! FGA Group (8-cycle stall)
181 ldd [%o1 + 0x060], %f8 ! MS
182 fmovd %f2, %f34 ! FGA Group
183 ldd [%o1 + 0x068], %f10 ! AX
184 fmovd %f4, %f36 ! FGA Group
185 ldd [%o1 + 0x070], %f12 ! MS
186 fmovd %f6, %f38 ! FGA Group
187 ldd [%o1 + 0x078], %f14 ! AX
188 fmovd %f8, %f40 ! FGA Group
189 ldd [%o1 + 0x080], %f16 ! MS
190 fmovd %f10, %f42 ! FGA Group
191 subcc %o2, 1, %o2 ! A0
192 add %o0, 0x40, %o0 ! A1
194 add %o1, 0x40, %o1 ! A0 Group
196 ldd [%o1 + 0x048], %f2 ! AX
197 fmovd %f12, %f44 ! FGA
198 ldd [%o1 + 0x050], %f4 ! MS
199 fmovd %f14, %f46 ! FGA Group
200 stda %f32, [%o0] ASI_BLK_P ! MS
201 ldd [%o1 + 0x058], %f6 ! AX
202 fmovd %f16, %f32 ! FGA Group (8-cycle stall)
203 ldd [%o1 + 0x060], %f8 ! MS
204 fmovd %f2, %f34 ! FGA Group
205 ldd [%o1 + 0x068], %f10 ! AX
206 fmovd %f4, %f36 ! FGA Group
207 ldd [%o1 + 0x070], %f12 ! MS
208 fmovd %f6, %f38 ! FGA Group
209 add %o0, 0x40, %o0 ! A0
210 ldd [%o1 + 0x078], %f14 ! AX
211 fmovd %f8, %f40 ! FGA Group
212 fmovd %f10, %f42 ! FGA Group
213 fmovd %f12, %f44 ! FGA Group
214 fmovd %f14, %f46 ! FGA Group
215 stda %f32, [%o0] ASI_BLK_P ! MS
216 .globl cheetah_copy_user_page_nop_2_3
217 cheetah_copy_user_page_nop_2_3:
218 mov PRIMARY_CONTEXT, %o2
219 stxa %g0, [%o2] ASI_DMMU ! Flush P-cache
220 stxa %g3, [%g0] ASI_DCU_CONTROL_REG ! Disable P-cache
221 ba,a,pt %xcc, copy_user_page_continue
223 spitfire_copy_user_page:
224 ldda [%o1] ASI_BLK_P, %f0
226 ldda [%o1] ASI_BLK_P, %f16
228 sethi %hi(PAGE_SIZE), %o2
229 1: TOUCH(f0, f2, f4, f6, f8, f10, f12, f14)
230 ldda [%o1] ASI_BLK_P, %f32
231 stda %f48, [%o0] ASI_BLK_P
235 TOUCH(f16, f18, f20, f22, f24, f26, f28, f30)
236 ldda [%o1] ASI_BLK_P, %f0
237 stda %f48, [%o0] ASI_BLK_P
241 TOUCH(f32, f34, f36, f38, f40, f42, f44, f46)
242 ldda [%o1] ASI_BLK_P, %f16
243 stda %f48, [%o0] ASI_BLK_P
246 cmp %o2, PAGE_SIZE_REM
249 #if (PAGE_SHIFT == 16) || (PAGE_SHIFT == 22)
250 TOUCH(f0, f2, f4, f6, f8, f10, f12, f14)
251 ldda [%o1] ASI_BLK_P, %f32
252 stda %f48, [%o0] ASI_BLK_P
256 TOUCH(f16, f18, f20, f22, f24, f26, f28, f30)
257 ldda [%o1] ASI_BLK_P, %f0
258 stda %f48, [%o0] ASI_BLK_P
263 stda %f32, [%o0] ASI_BLK_P
265 stda %f0, [%o0] ASI_BLK_P
268 stda %f0, [%o0] ASI_BLK_P
270 stda %f16, [%o0] ASI_BLK_P
272 copy_user_page_continue:
276 mov TLB_TAG_ACCESS, %o2
277 stxa %g5, [%o2] ASI_DMMU
278 stxa %g7, [%o3] ASI_DTLB_DATA_ACCESS
280 sub %o3, (TLBTEMP_ENTSZ), %o3
281 stxa %o4, [%o2] ASI_DMMU
282 stxa %o5, [%o3] ASI_DTLB_DATA_ACCESS
286 wrpr %g3, PSTATE_IE, %pstate
288 copy_page_using_blkcommit:
289 membar #LoadStore | #StoreStore | #StoreLoad
290 ldda [%o1] ASI_BLK_P, %f0
292 ldda [%o1] ASI_BLK_P, %f16
294 sethi %hi(PAGE_SIZE), %o2
295 1: TOUCH(f0, f2, f4, f6, f8, f10, f12, f14)
296 ldda [%o1] ASI_BLK_P, %f32
297 stda %f48, [%o0] ASI_BLK_COMMIT_P
301 TOUCH(f16, f18, f20, f22, f24, f26, f28, f30)
302 ldda [%o1] ASI_BLK_P, %f0
303 stda %f48, [%o0] ASI_BLK_COMMIT_P
307 TOUCH(f32, f34, f36, f38, f40, f42, f44, f46)
308 ldda [%o1] ASI_BLK_P, %f16
309 stda %f48, [%o0] ASI_BLK_COMMIT_P
312 cmp %o2, PAGE_SIZE_REM
315 #if (PAGE_SHIFT == 16) || (PAGE_SHIFT == 22)
316 TOUCH(f0, f2, f4, f6, f8, f10, f12, f14)
317 ldda [%o1] ASI_BLK_P, %f32
318 stda %f48, [%o0] ASI_BLK_COMMIT_P
322 TOUCH(f16, f18, f20, f22, f24, f26, f28, f30)
323 ldda [%o1] ASI_BLK_P, %f0
324 stda %f48, [%o0] ASI_BLK_COMMIT_P
329 stda %f32, [%o0] ASI_BLK_COMMIT_P
331 ba,pt %xcc, copy_user_page_continue
332 stda %f0, [%o0] ASI_BLK_COMMIT_P
335 stda %f0, [%o0] ASI_BLK_COMMIT_P
337 ba,pt %xcc, copy_user_page_continue
338 stda %f16, [%o0] ASI_BLK_COMMIT_P
343 .type _clear_page,@function
344 _clear_page: /* %o0=dest */
346 ba,pt %xcc, clear_page_common
350 .globl clear_user_page
351 .type clear_user_page,@function
352 clear_user_page: /* %o0=dest, %o1=vaddr */
354 sethi %hi(PAGE_SIZE), %g3
355 sethi %uhi(PAGE_OFFSET), %g2
359 mov TLB_TAG_ACCESS, %o2
360 sethi %uhi(_PAGE_VALID | _PAGE_SZBITS), %g3
361 sethi %hi(TLBTEMP_BASE), %o3
363 or %g3, (_PAGE_CP | _PAGE_CV | _PAGE_P | _PAGE_L | _PAGE_W), %g3
366 #define FIX_INSN_2 0x96102068 /* mov (13 << 3), %o3 */
368 mov TLBTEMP_ENT2, %o3
370 wrpr %g3, PSTATE_IE, %pstate
372 /* Spitfire Errata #32 workaround */
373 mov PRIMARY_CONTEXT, %g5
374 stxa %g0, [%g5] ASI_DMMU
377 ldxa [%o3] ASI_DTLB_TAG_READ, %g5
379 /* Spitfire Errata #32 workaround */
380 mov PRIMARY_CONTEXT, %g7
381 stxa %g0, [%g7] ASI_DMMU
384 ldxa [%o3] ASI_DTLB_DATA_ACCESS, %g0
385 ldxa [%o3] ASI_DTLB_DATA_ACCESS, %g7
386 stxa %o0, [%o2] ASI_DMMU
387 stxa %g1, [%o3] ASI_DTLB_DATA_ACCESS
393 membar #StoreLoad | #StoreStore | #LoadStore ! LSU Group
394 fzero %f0 ! FPA Group
395 sethi %hi(PAGE_SIZE/256), %o1 ! IEU0
396 fzero %f2 ! FPA Group
397 or %o1, %lo(PAGE_SIZE/256), %o1 ! IEU0
398 faddd %f0, %f2, %f4 ! FPA Group
399 fmuld %f0, %f2, %f6 ! FPM
400 faddd %f0, %f2, %f8 ! FPA Group
401 fmuld %f0, %f2, %f10 ! FPM
403 faddd %f0, %f2, %f12 ! FPA Group
404 fmuld %f0, %f2, %f14 ! FPM
405 1: stda %f0, [%o0 + %g0] ASI_BLK_P ! Store Group
406 add %o0, 0x40, %o0 ! IEU0
407 stda %f0, [%o0 + %g0] ASI_BLK_P ! Store Group
408 add %o0, 0x40, %o0 ! IEU0
409 stda %f0, [%o0 + %g0] ASI_BLK_P ! Store Group
411 add %o0, 0x40, %o0 ! IEU0 Group
412 stda %f0, [%o0 + %g0] ASI_BLK_P ! Store Group
413 subcc %o1, 1, %o1 ! IEU1
414 bne,pt %icc, 1b ! CTI
415 add %o0, 0x40, %o0 ! IEU0 Group
416 membar #Sync ! LSU Group
426 stxa %g5, [%o2] ASI_DMMU
427 stxa %g7, [%o3] ASI_DTLB_DATA_ACCESS
430 wrpr %g3, 0x0, %pstate
432 .globl cheetah_patch_pgcopyops
433 cheetah_patch_pgcopyops:
434 sethi %hi(FIX_INSN_1), %g1
435 or %g1, %lo(FIX_INSN_1), %g1
436 sethi %hi(cheetah_patch_1), %g2
437 or %g2, %lo(cheetah_patch_1), %g2
440 sethi %hi(FIX_INSN_2), %g1
441 or %g1, %lo(FIX_INSN_2), %g1
442 sethi %hi(cheetah_patch_2), %g2
443 or %g2, %lo(cheetah_patch_2), %g2