include/asm-xen/asm-x86_64/system.h

   1 #ifndef __ASM_SYSTEM_H
   2 #define __ASM_SYSTEM_H
   3
   4 #include <linux/config.h>
   5 #include <linux/kernel.h>
   6 #include <asm/segment.h>
   7 #include <asm/synch_bitops.h>
   8 #include <asm-xen/hypervisor.h>
   9 #include <asm-xen/xen-public/arch-x86_64.h>
  10
  11 #ifdef __KERNEL__
  12
  13 #ifdef CONFIG_SMP
  14 #define LOCK_PREFIX "lock ; "
  15 #else
  16 #define LOCK_PREFIX ""
  17 #endif
  18
  19 #define __STR(x) #x
  20 #define STR(x) __STR(x)
  21
  22 #define __SAVE(reg,offset) "movq %%" #reg ",(14-" #offset ")*8(%%rsp)\n\t"
  23 #define __RESTORE(reg,offset) "movq (14-" #offset ")*8(%%rsp),%%" #reg "\n\t"
  24
  25 /* frame pointer must be last for get_wchan */
  26 #define SAVE_CONTEXT    "pushfq ; pushq %%rbp ; movq %%rsi,%%rbp\n\t"
  27 #define RESTORE_CONTEXT "movq %%rbp,%%rsi ; popq %%rbp ; popfq\n\t"
  28
  29 #define __EXTRA_CLOBBER  \
  30         ,"rcx","rbx","rdx","r8","r9","r10","r11","r12","r13","r14","r15"
  31
  32 #define switch_to(prev,next,last) \
  33         asm volatile(SAVE_CONTEXT                                                   \
  34                      "movq %%rsp,%P[threadrsp](%[prev])\n\t" /* save RSP */       \
  35                      "movq %P[threadrsp](%[next]),%%rsp\n\t" /* restore RSP */    \
  36                      "call __switch_to\n\t"                                       \
  37                      ".globl thread_return\n"                                   \
  38                      "thread_return:\n\t"                                           \
  39                      "movq %%gs:%P[pda_pcurrent],%%rsi\n\t"                       \
  40                      "movq %P[thread_info](%%rsi),%%r8\n\t"                       \
  41                      LOCK "btr  %[tif_fork],%P[ti_flags](%%r8)\n\t"               \
  42                      "movq %%rax,%%rdi\n\t"                                       \
  43                      "jc   ret_from_fork\n\t"                                     \
  44                      RESTORE_CONTEXT                                                \
  45                      : "=a" (last)                                                \
  46                      : [next] "S" (next), [prev] "D" (prev),                      \
  47                        [threadrsp] "i" (offsetof(struct task_struct, thread.rsp)), \
  48                        [ti_flags] "i" (offsetof(struct thread_info, flags)),\
  49                        [tif_fork] "i" (TIF_FORK),                         \
  50                        [thread_info] "i" (offsetof(struct task_struct, thread_info)), \
  51                        [pda_pcurrent] "i" (offsetof(struct x8664_pda, pcurrent))   \
  52                      : "memory", "cc" __EXTRA_CLOBBER)
  53
  54
  55 extern void load_gs_index(unsigned);
  56
  57 /*
  58  * Load a segment. Fall back on loading the zero
  59  * segment if something goes wrong..
  60  */
  61 #define loadsegment(seg,value)  \
  62         asm volatile("\n"                       \
  63                 "1:\t"                          \
  64                 "movl %k0,%%" #seg "\n"         \
  65                 "2:\n"                          \
  66                 ".section .fixup,\"ax\"\n"      \
  67                 "3:\t"                          \
  68                 "movl %1,%%" #seg "\n\t"        \
  69                 "jmp 2b\n"                      \
  70                 ".previous\n"                   \
  71                 ".section __ex_table,\"a\"\n\t" \
  72                 ".align 8\n\t"                  \
  73                 ".quad 1b,3b\n"                 \
  74                 ".previous"                     \
  75                 : :"r" (value), "r" (0))
  76
  77 #define set_debug(value,register) \
  78                 __asm__("movq %0,%%db" #register  \
  79                 : /* no output */ \
  80                 :"r" ((unsigned long) value))
  81
  82
  83 #ifdef __KERNEL__
  84 struct alt_instr {
  85         __u8 *instr;            /* original instruction */
  86         __u8 *replacement;
  87         __u8  cpuid;            /* cpuid bit set for replacement */
  88         __u8  instrlen;         /* length of original instruction */
  89         __u8  replacementlen;   /* length of new instruction, <= instrlen */
  90         __u8  pad[5];
  91 };
  92 #endif
  93
  94 /*
  95  * Alternative instructions for different CPU types or capabilities.
  96  *
  97  * This allows to use optimized instructions even on generic binary
  98  * kernels.
  99  *
 100  * length of oldinstr must be longer or equal the length of newinstr
 101  * It can be padded with nops as needed.
 102  *
 103  * For non barrier like inlines please define new variants
 104  * without volatile and memory clobber.
 105  */
 106 #define alternative(oldinstr, newinstr, feature)        \
 107         asm volatile ("661:\n\t" oldinstr "\n662:\n"                 \
 108                       ".section .altinstructions,\"a\"\n"            \
 109                       "  .align 8\n"                                   \
 110                       "  .quad 661b\n"            /* label */          \
 111                       "  .quad 663f\n"            /* new instruction */ \
 112                       "  .byte %c0\n"             /* feature bit */    \
 113                       "  .byte 662b-661b\n"       /* sourcelen */      \
 114                       "  .byte 664f-663f\n"       /* replacementlen */ \
 115                       ".previous\n"                                     \
 116                       ".section .altinstr_replacement,\"ax\"\n"         \
 117                       "663:\n\t" newinstr "\n664:\n"   /* replacement */ \
 118                       ".previous" :: "i" (feature) : "memory")
 119
 120 /*
 121  * Alternative inline assembly with input.
 122  *
 123  * Pecularities:
 124  * No memory clobber here.
 125  * Argument numbers start with 1.
 126  * Best is to use constraints that are fixed size (like (%1) ... "r")
 127  * If you use variable sized constraints like "m" or "g" in the
 128  * replacement maake sure to pad to the worst case length.
 129  */
 130 #define alternative_input(oldinstr, newinstr, feature, input...)        \
 131         asm volatile ("661:\n\t" oldinstr "\n662:\n"                    \
 132                       ".section .altinstructions,\"a\"\n"               \
 133                       "  .align 8\n"                                    \
 134                       "  .quad 661b\n"            /* label */           \
 135                       "  .quad 663f\n"            /* new instruction */ \
 136                       "  .byte %c0\n"             /* feature bit */     \
 137                       "  .byte 662b-661b\n"       /* sourcelen */       \
 138                       "  .byte 664f-663f\n"       /* replacementlen */  \
 139                       ".previous\n"                                     \
 140                       ".section .altinstr_replacement,\"ax\"\n"         \
 141                       "663:\n\t" newinstr "\n664:\n"   /* replacement */ \
 142                       ".previous" :: "i" (feature), ##input)
 143
 144 /*
 145  * Clear and set 'TS' bit respectively
 146  */
 147 #define clts() (HYPERVISOR_fpu_taskswitch(0))
 148 static inline unsigned long read_cr0(void)
 149 {
 150         BUG();
 151 }
 152
 153 static inline void write_cr0(unsigned long val)
 154 {
 155         BUG();
 156 }
 157
 158 static inline unsigned long read_cr3(void)
 159 {
 160         BUG();
 161 }
 162
 163 static inline unsigned long read_cr4(void)
 164 {
 165         BUG();
 166 }
 167
 168 static inline void write_cr4(unsigned long val)
 169 {
 170         BUG();
 171 }
 172 #define stts() (HYPERVISOR_fpu_taskswitch(1))
 173
 174 #define wbinvd() \
 175         __asm__ __volatile__ ("wbinvd": : :"memory");
 176
 177 #endif  /* __KERNEL__ */
 178
 179 #define nop() __asm__ __volatile__ ("nop")
 180
 181 #define xchg(ptr,v) ((__typeof__(*(ptr)))__xchg((unsigned long)(v),(ptr),sizeof(*(ptr))))
 182
 183 #define tas(ptr) (xchg((ptr),1))
 184
 185 #define __xg(x) ((volatile long *)(x))
 186
 187 extern inline void set_64bit(volatile unsigned long *ptr, unsigned long val)
 188 {
 189         *ptr = val;
 190 }
 191
 192 #define _set_64bit set_64bit
 193
 194 /*
 195  * Note: no "lock" prefix even on SMP: xchg always implies lock anyway
 196  * Note 2: xchg has side effect, so that attribute volatile is necessary,
 197  *        but generally the primitive is invalid, *ptr is output argument. --ANK
 198  */
 199 static inline unsigned long __xchg(unsigned long x, volatile void * ptr, int size)
 200 {
 201         switch (size) {
 202                 case 1:
 203                         __asm__ __volatile__("xchgb %b0,%1"
 204                                 :"=q" (x)
 205                                 :"m" (*__xg(ptr)), "0" (x)
 206                                 :"memory");
 207                         break;
 208                 case 2:
 209                         __asm__ __volatile__("xchgw %w0,%1"
 210                                 :"=r" (x)
 211                                 :"m" (*__xg(ptr)), "0" (x)
 212                                 :"memory");
 213                         break;
 214                 case 4:
 215                         __asm__ __volatile__("xchgl %k0,%1"
 216                                 :"=r" (x)
 217                                 :"m" (*__xg(ptr)), "0" (x)
 218                                 :"memory");
 219                         break;
 220                 case 8:
 221                         __asm__ __volatile__("xchgq %0,%1"
 222                                 :"=r" (x)
 223                                 :"m" (*__xg(ptr)), "0" (x)
 224                                 :"memory");
 225                         break;
 226         }
 227         return x;
 228 }
 229
 230 /*
 231  * Atomic compare and exchange.  Compare OLD with MEM, if identical,
 232  * store NEW in MEM.  Return the initial value in MEM.  Success is
 233  * indicated by comparing RETURN with OLD.
 234  */
 235
 236 #define __HAVE_ARCH_CMPXCHG 1
 237
 238 static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,
 239                                       unsigned long new, int size)
 240 {
 241         unsigned long prev;
 242         switch (size) {
 243         case 1:
 244                 __asm__ __volatile__(LOCK_PREFIX "cmpxchgb %b1,%2"
 245                                      : "=a"(prev)
 246                                      : "q"(new), "m"(*__xg(ptr)), "0"(old)
 247                                      : "memory");
 248                 return prev;
 249         case 2:
 250                 __asm__ __volatile__(LOCK_PREFIX "cmpxchgw %w1,%2"
 251                                      : "=a"(prev)
 252                                      : "q"(new), "m"(*__xg(ptr)), "0"(old)
 253                                      : "memory");
 254                 return prev;
 255         case 4:
 256                 __asm__ __volatile__(LOCK_PREFIX "cmpxchgl %k1,%2"
 257                                      : "=a"(prev)
 258                                      : "q"(new), "m"(*__xg(ptr)), "0"(old)
 259                                      : "memory");
 260                 return prev;
 261         case 8:
 262                 __asm__ __volatile__(LOCK_PREFIX "cmpxchgq %1,%2"
 263                                      : "=a"(prev)
 264                                      : "q"(new), "m"(*__xg(ptr)), "0"(old)
 265                                      : "memory");
 266                 return prev;
 267         }
 268         return old;
 269 }
 270
 271 #define cmpxchg(ptr,o,n)\
 272         ((__typeof__(*(ptr)))__cmpxchg((ptr),(unsigned long)(o),\
 273                                         (unsigned long)(n),sizeof(*(ptr))))
 274
 275 #ifdef CONFIG_SMP
 276 #define smp_mb()        mb()
 277 #define smp_rmb()       rmb()
 278 #define smp_wmb()       wmb()
 279 #define smp_read_barrier_depends()      do {} while(0)
 280 #else
 281 #define smp_mb()        barrier()
 282 #define smp_rmb()       barrier()
 283 #define smp_wmb()       barrier()
 284 #define smp_read_barrier_depends()      do {} while(0)
 285 #endif
 286
 287
 288 /*
 289  * Force strict CPU ordering.
 290  * And yes, this is required on UP too when we're talking
 291  * to devices.
 292  */
 293 #define mb()    asm volatile("mfence":::"memory")
 294 #define rmb()   asm volatile("lfence":::"memory")
 295
 296 #ifdef CONFIG_UNORDERED_IO
 297 #define wmb()   asm volatile("sfence" ::: "memory")
 298 #else
 299 #define wmb()   asm volatile("" ::: "memory")
 300 #endif
 301 #define read_barrier_depends()  do {} while(0)
 302 #define set_mb(var, value) do { xchg(&var, value); } while (0)
 303 #define set_wmb(var, value) do { var = value; wmb(); } while (0)
 304
 305 #define warn_if_not_ulong(x) do { unsigned long foo; (void) (&(x) == &foo); } while (0)
 306
 307
 308 /*
 309  * The use of 'barrier' in the following reflects their use as local-lock
 310  * operations. Reentrancy must be prevented (e.g., __cli()) /before/ following
 311  * critical operations are executed. All critical operations must complete
 312  * /before/ reentrancy is permitted (e.g., __sti()). Alpha architecture also
 313  * includes these barriers, for example.
 314  */
 315
 316 #define __cli()                                                         \
 317 do {                                                                    \
 318         vcpu_info_t *_vcpu;                                             \
 319         preempt_disable();                                              \
 320         _vcpu = &HYPERVISOR_shared_info->vcpu_data[smp_processor_id()]; \
 321         _vcpu->evtchn_upcall_mask = 1;                                  \
 322         preempt_enable_no_resched();                                    \
 323         barrier();                                                      \
 324 } while (0)
 325
 326 #define __sti()                                                         \
 327 do {                                                                    \
 328         vcpu_info_t *_vcpu;                                             \
 329         barrier();                                                      \
 330         preempt_disable();                                              \
 331         _vcpu = &HYPERVISOR_shared_info->vcpu_data[smp_processor_id()]; \
 332         _vcpu->evtchn_upcall_mask = 0;                                  \
 333         barrier(); /* unmask then check (avoid races) */                \
 334         if ( unlikely(_vcpu->evtchn_upcall_pending) )                   \
 335                 force_evtchn_callback();                                \
 336         preempt_enable();                                               \
 337 } while (0)
 338
 339 #define __save_flags(x)                                                 \
 340 do {                                                                    \
 341         vcpu_info_t *_vcpu;                                             \
 342         _vcpu = &HYPERVISOR_shared_info->vcpu_data[smp_processor_id()]; \
 343         (x) = _vcpu->evtchn_upcall_mask;                                \
 344 } while (0)
 345
 346 #define __restore_flags(x)                                              \
 347 do {                                                                    \
 348         vcpu_info_t *_vcpu;                                             \
 349         barrier();                                                      \
 350         preempt_disable();                                              \
 351         _vcpu = &HYPERVISOR_shared_info->vcpu_data[smp_processor_id()]; \
 352         if ((_vcpu->evtchn_upcall_mask = (x)) == 0) {                   \
 353                 barrier(); /* unmask then check (avoid races) */        \
 354                 if ( unlikely(_vcpu->evtchn_upcall_pending) )           \
 355                         force_evtchn_callback();                        \
 356                 preempt_enable();                                       \
 357         } else                                                          \
 358                 preempt_enable_no_resched();                            \
 359 } while (0)
 360
 361 #define safe_halt()             ((void)0)
 362
 363 #define __save_and_cli(x)                                               \
 364 do {                                                                    \
 365         vcpu_info_t *_vcpu;                                             \
 366         preempt_disable();                                              \
 367         _vcpu = &HYPERVISOR_shared_info->vcpu_data[smp_processor_id()]; \
 368         (x) = _vcpu->evtchn_upcall_mask;                                \
 369         _vcpu->evtchn_upcall_mask = 1;                                  \
 370         preempt_enable_no_resched();                                    \
 371         barrier();                                                      \
 372 } while (0)
 373
 374 void cpu_idle_wait(void);
 375
 376 #define local_irq_save(x)       __save_and_cli(x)
 377 #define local_irq_restore(x)    __restore_flags(x)
 378 #define local_save_flags(x)     __save_flags(x)
 379 #define local_irq_disable()     __cli()
 380 #define local_irq_enable()      __sti()
 381
 382 #define irqs_disabled()                 \
 383     HYPERVISOR_shared_info->vcpu_data[smp_processor_id()].evtchn_upcall_mask
 384
 385 /*
 386  * disable hlt during certain critical i/o operations
 387  */
 388 #define HAVE_DISABLE_HLT
 389 void disable_hlt(void);
 390 void enable_hlt(void);
 391
 392 #define HAVE_EAT_KEY
 393 void eat_key(void);
 394
 395 #endif