X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=arch%2Fia64%2Fkernel%2Fgate.S;h=30c2dc3d348857ef1ab510b2631e1036c7c940d7;hb=97bf2856c6014879bd04983a3e9dfcdac1e7fe85;hp=36e8a24ff4cdb681af07119aab7b7ad5b2ab10d8;hpb=5273a3df6485dc2ad6aa7ddd441b9a21970f003b;p=linux-2.6.git diff --git a/arch/ia64/kernel/gate.S b/arch/ia64/kernel/gate.S index 36e8a24ff..30c2dc3d3 100644 --- a/arch/ia64/kernel/gate.S +++ b/arch/ia64/kernel/gate.S @@ -6,14 +6,15 @@ * David Mosberger-Tang */ -#include - #include #include -#include +#include #include #include #include +#ifdef CONFIG_XEN_IA64_VDSO_PARAVIRT +# include +#endif /* * We can't easily refer to symbols inside the kernel. To avoid full runtime relocation, @@ -33,6 +34,52 @@ [1:](pr)brl.cond.sptk 0; \ .xdata4 ".data.patch.brl_fsys_bubble_down", 1b-. +#ifdef CONFIG_XEN_IA64_VDSO_PARAVIRT + // The page in which hyperprivop lives must be pinned by ITR. + // However vDSO area isn't pinned. So issuing hyperprivop + // from vDSO page causes trouble that Kevin pointed out. + // After clearing vpsr.ic, the vcpu is pre-empted and the itlb + // is flushed. Then vcpu get cpu again, tlb miss fault occures. + // However it results in nested dtlb fault because vpsr.ic is off. + // To avoid such a situation, we jump into the kernel text area + // which is pinned, and then issue hyperprivop and return back + // to vDSO page. + // This is Dan Magenheimer's idea. + + // Currently is_running_on_xen() is defined as running_on_xen. + // If is_running_on_xen() is a real function, we must update + // according to it. + .section ".data.patch.running_on_xen", "a" + .previous +#define LOAD_RUNNING_ON_XEN(reg) \ +[1:] movl reg=0; \ + .xdata4 ".data.patch.running_on_xen", 1b-. + + .section ".data.patch.brl_xen_rsm_be_i", "a" + .previous +#define BRL_COND_XEN_RSM_BE_I(pr) \ +[1:](pr)brl.cond.sptk 0; \ + .xdata4 ".data.patch.brl_xen_rsm_be_i", 1b-. + + .section ".data.patch.brl_xen_get_psr", "a" + .previous +#define BRL_COND_XEN_GET_PSR(pr) \ +[1:](pr)brl.cond.sptk 0; \ + .xdata4 ".data.patch.brl_xen_get_psr", 1b-. + + .section ".data.patch.brl_xen_ssm_i_0", "a" + .previous +#define BRL_COND_XEN_SSM_I_0(pr) \ +[1:](pr)brl.cond.sptk 0; \ + .xdata4 ".data.patch.brl_xen_ssm_i_0", 1b-. + + .section ".data.patch.brl_xen_ssm_i_1", "a" + .previous +#define BRL_COND_XEN_SSM_I_1(pr) \ +[1:](pr)brl.cond.sptk 0; \ + .xdata4 ".data.patch.brl_xen_ssm_i_1", 1b-. +#endif + GLOBAL_ENTRY(__kernel_syscall_via_break) .prologue .altrp b6 @@ -72,35 +119,91 @@ GLOBAL_ENTRY(__kernel_syscall_via_epc) * bundle get executed. The remaining code must be safe even if * they do not get executed. */ - adds r17=-1024,r15 - mov r10=0 // default to successful syscall execution - epc + adds r17=-1024,r15 // A + mov r10=0 // A default to successful syscall execution + epc // B causes split-issue } ;; - rsm psr.be // note: on McKinley "rsm psr.be/srlz.d" is slightly faster than "rum psr.be" - LOAD_FSYSCALL_TABLE(r14) - - mov r16=IA64_KR(CURRENT) // 12 cycle read latency - mov r19=NR_syscalls-1 +#ifdef CONFIG_XEN_IA64_VDSO_PARAVIRT + // r20 = 1 + // r22 = &vcpu->evtchn_mask + // r23 = &vpsr.ic + // r24 = &vcpu->pending_interruption + // r25 = tmp + // r28 = &running_on_xen + // r30 = running_on_xen + // r31 = tmp + // p11 = tmp + // p12 = running_on_xen + // p13 = !running_on_xen + // p14 = tmp + // p15 = tmp +#define isXen p12 +#define isRaw p13 + LOAD_RUNNING_ON_XEN(r28) + movl r22=XSI_PSR_I_ADDR + movl r23=XSI_PSR_IC + movl r24=XSI_PSR_I_ADDR+(XSI_PEND_OFS-XSI_PSR_I_ADDR_OFS) + mov r20=1 + ;; + ld4 r30=[r28] + ;; + cmp.ne isXen,isRaw=r0,r30 + ;; +(isRaw) rsm psr.be | psr.i + BRL_COND_XEN_RSM_BE_I(isXen) + .global .vdso_rsm_be_i_ret +.vdso_rsm_be_i_ret: +#else + rsm psr.be | psr.i // M2 (5 cyc to srlz.d) +#endif + LOAD_FSYSCALL_TABLE(r14) // X + ;; + mov r16=IA64_KR(CURRENT) // M2 (12 cyc) + shladd r18=r17,3,r14 // A + mov r19=NR_syscalls-1 // A + ;; + lfetch [r18] // M0|1 +#ifdef CONFIG_XEN_IA64_VDSO_PARAVIRT +(isRaw) mov r29=psr + BRL_COND_XEN_GET_PSR(isXen) + .global .vdso_get_psr_ret +.vdso_get_psr_ret: +#else + mov r29=psr // M2 (12 cyc) +#endif + // If r17 is a NaT, p6 will be zero + cmp.geu p6,p7=r19,r17 // A (sysnr > 0 && sysnr < 1024+NR_syscalls)? ;; - shladd r18=r17,3,r14 - - srlz.d - cmp.ne p8,p0=r0,r0 // p8 <- FALSE - /* Note: if r17 is a NaT, p6 will be set to zero. */ - cmp.geu p6,p7=r19,r17 // (syscall > 0 && syscall < 1024+NR_syscalls)? - ;; -(p6) ld8 r18=[r18] - mov r29=psr // read psr (12 cyc load latency) - add r14=-8,r14 // r14 <- addr of fsys_bubble_down entry - ;; -(p6) mov b7=r18 -(p6) tbit.z p8,p0=r18,0 -(p8) br.dptk.many b7 + mov r21=ar.fpsr // M2 (12 cyc) + tnat.nz p10,p9=r15 // I0 + mov.i r26=ar.pfs // I0 (would stall anyhow due to srlz.d...) + ;; + srlz.d // M0 (forces split-issue) ensure PSR.BE==0 +(p6) ld8 r18=[r18] // M0|1 + nop.i 0 + ;; + nop.m 0 +(p6) tbit.z.unc p8,p0=r18,0 // I0 (dual-issues with "mov b7=r18"!) +#ifdef CONFIG_XEN_IA64_VDSO_PARAVIRT + ;; + // p14 = running_on_xen && p8 + // p15 = !running_on_xen && p8 +(p8) cmp.ne.unc p14,p15=r0,r30 + ;; +(p15) ssm psr.i + BRL_COND_XEN_SSM_I_0(p14) + .global .vdso_ssm_i_0_ret +.vdso_ssm_i_0_ret: +#else + nop.i 0 + ;; +(p8) ssm psr.i +#endif +(p6) mov b7=r18 // I0 +(p8) br.dptk.many b7 // B - mov r27=ar.rsc - mov r21=ar.fpsr - mov r26=ar.pfs + mov r27=ar.rsc // M2 (12 cyc) /* * brl.cond doesn't work as intended because the linker would convert this branch * into a branch to a PLT. Perhaps there will be a way to avoid this with some @@ -108,6 +211,8 @@ GLOBAL_ENTRY(__kernel_syscall_via_epc) * instead. */ #ifdef CONFIG_ITANIUM +(p6) add r14=-8,r14 // r14 <- addr of fsys_bubble_down entry + ;; (p6) ld8 r14=[r14] // r14 <- fsys_bubble_down ;; (p6) mov b7=r14 @@ -115,9 +220,22 @@ GLOBAL_ENTRY(__kernel_syscall_via_epc) #else BRL_COND_FSYS_BUBBLE_DOWN(p6) #endif - +#ifdef CONFIG_XEN_IA64_VDSO_PARAVIRT +(isRaw) ssm psr.i + BRL_COND_XEN_SSM_I_1(isXen) + .global .vdso_ssm_i_1_ret +.vdso_ssm_i_1_ret: +#else + ssm psr.i +#endif mov r10=-1 - mov r8=ENOSYS +(p10) mov r8=EINVAL +#ifdef CONFIG_XEN_IA64_VDSO_PARAVIRT + dv_serialize_data // shut up gas warning. + // we know xen_hyper_ssm_i_0 or xen_hyper_ssm_i_1 + // doesn't change p9 and p10 +#endif +(p9) mov r8=ENOSYS FSYS_RETURN END(__kernel_syscall_via_epc)