include/asm-x86_64/uaccess.h

   1 #ifndef __X86_64_UACCESS_H
   2 #define __X86_64_UACCESS_H
   3
   4 /*
   5  * User space memory access functions
   6  */
   7 #include <linux/config.h>
   8 #include <linux/compiler.h>
   9 #include <linux/errno.h>
  10 #include <linux/sched.h>
  11 #include <linux/prefetch.h>
  12 #include <asm/page.h>
  13
  14 #define VERIFY_READ 0
  15 #define VERIFY_WRITE 1
  16
  17 /*
  18  * The fs value determines whether argument validity checking should be
  19  * performed or not.  If get_fs() == USER_DS, checking is performed, with
  20  * get_fs() == KERNEL_DS, checking is bypassed.
  21  *
  22  * For historical reasons, these macros are grossly misnamed.
  23  */
  24
  25 #define MAKE_MM_SEG(s)  ((mm_segment_t) { (s) })
  26
  27 #define KERNEL_DS       MAKE_MM_SEG(0xFFFFFFFFFFFFFFFFUL)
  28 #define USER_DS         MAKE_MM_SEG(PAGE_OFFSET)
  29
  30 #define get_ds()        (KERNEL_DS)
  31 #define get_fs()        (current_thread_info()->addr_limit)
  32 #define set_fs(x)       (current_thread_info()->addr_limit = (x))
  33
  34 #define segment_eq(a,b) ((a).seg == (b).seg)
  35
  36 #ifdef __CHECKER__
  37 #define CHECK_UPTR(ptr) do {                            \
  38         __typeof__(*(ptr)) *__dummy_check_uptr =        \
  39                 (void __user *)&__dummy_check_uptr;     \
  40 } while(0)
  41 #else
  42 #define CHECK_UPTR(ptr)
  43 #endif
  44
  45
  46 #define __addr_ok(addr) (!((unsigned long)(addr) & (current_thread_info()->addr_limit.seg)))
  47
  48 /*
  49  * Uhhuh, this needs 65-bit arithmetic. We have a carry..
  50  */
  51 #define __range_not_ok(addr,size) ({ \
  52         unsigned long flag,sum; \
  53         CHECK_UPTR(addr);       \
  54         asm("# range_ok\n\r" \
  55                 "addq %3,%1 ; sbbq %0,%0 ; cmpq %1,%4 ; sbbq $0,%0"  \
  56                 :"=&r" (flag), "=r" (sum) \
  57                 :"1" (addr),"g" ((long)(size)),"g" (current_thread_info()->addr_limit.seg)); \
  58         flag; })
  59
  60 #define access_ok(type, addr, size) (__range_not_ok(addr,size) == 0)
  61
  62 extern inline int verify_area(int type, const void __user * addr, unsigned long size)
  63 {
  64         return access_ok(type,addr,size) ? 0 : -EFAULT;
  65 }
  66
  67
  68 /*
  69  * The exception table consists of pairs of addresses: the first is the
  70  * address of an instruction that is allowed to fault, and the second is
  71  * the address at which the program should continue.  No registers are
  72  * modified, so it is entirely up to the continuation code to figure out
  73  * what to do.
  74  *
  75  * All the routines below use bits of fixup code that are out of line
  76  * with the main instruction path.  This means when everything is well,
  77  * we don't even have to jump over them.  Further, they do not intrude
  78  * on our cache or tlb entries.
  79  */
  80
  81 struct exception_table_entry
  82 {
  83         unsigned long insn, fixup;
  84 };
  85
  86
  87 /*
  88  * These are the main single-value transfer routines.  They automatically
  89  * use the right size if we just have the right pointer type.
  90  *
  91  * This gets kind of ugly. We want to return _two_ values in "get_user()"
  92  * and yet we don't want to do any pointers, because that is too much
  93  * of a performance impact. Thus we have a few rather ugly macros here,
  94  * and hide all the ugliness from the user.
  95  *
  96  * The "__xxx" versions of the user access functions are versions that
  97  * do not verify the address space, that must have been done previously
  98  * with a separate "access_ok()" call (this is used when we do multiple
  99  * accesses to the same area of user memory).
 100  */
 101
 102 extern void __get_user_1(void);
 103 extern void __get_user_2(void);
 104 extern void __get_user_4(void);
 105 extern void __get_user_8(void);
 106
 107 #define __get_user_x(size,ret,x,ptr) \
 108         __asm__ __volatile__("call __get_user_" #size \
 109                 :"=a" (ret),"=d" (x) \
 110                 :"0" (ptr) \
 111                 :"rbx")
 112
 113 /* Careful: we have to cast the result to the type of the pointer for sign reasons */
 114 #define get_user(x,ptr)                                                 \
 115 ({      long __val_gu;                                                  \
 116         int __ret_gu;                                                   \
 117         CHECK_UPTR(ptr);                                                \
 118         switch(sizeof (*(ptr))) {                                       \
 119         case 1:  __get_user_x(1,__ret_gu,__val_gu,ptr); break;          \
 120         case 2:  __get_user_x(2,__ret_gu,__val_gu,ptr); break;          \
 121         case 4:  __get_user_x(4,__ret_gu,__val_gu,ptr); break;          \
 122         case 8:  __get_user_x(8,__ret_gu,__val_gu,ptr); break;          \
 123         default: __get_user_bad(); break;                               \
 124         }                                                               \
 125         (x) = (__typeof__(*(ptr)))__val_gu;                             \
 126         __ret_gu;                                                       \
 127 })
 128
 129 extern void __put_user_1(void);
 130 extern void __put_user_2(void);
 131 extern void __put_user_4(void);
 132 extern void __put_user_8(void);
 133
 134 extern void __put_user_bad(void);
 135
 136 #define __put_user_x(size,ret,x,ptr)                                    \
 137         __asm__ __volatile__("call __put_user_" #size                   \
 138                 :"=a" (ret)                                             \
 139                 :"0" (ptr),"d" (x)                                      \
 140                 :"rbx")
 141
 142 #define put_user(x,ptr)                                                 \
 143   __put_user_check((__typeof__(*(ptr)))(x),(ptr),sizeof(*(ptr)))
 144
 145 #define __get_user(x,ptr) \
 146   __get_user_nocheck((x),(ptr),sizeof(*(ptr)))
 147 #define __put_user(x,ptr) \
 148   __put_user_nocheck((__typeof__(*(ptr)))(x),(ptr),sizeof(*(ptr)))
 149
 150 #define __put_user_nocheck(x,ptr,size)                  \
 151 ({                                                      \
 152         int __pu_err;                                   \
 153         CHECK_UPTR(ptr);                                \
 154         __put_user_size((x),(ptr),(size),__pu_err);     \
 155         __pu_err;                                       \
 156 })
 157
 158
 159 #define __put_user_check(x,ptr,size)                    \
 160 ({                                                      \
 161         int __pu_err = -EFAULT;                         \
 162         __typeof__(*(ptr)) *__pu_addr = (ptr);          \
 163         if (likely(access_ok(VERIFY_WRITE,__pu_addr,size)))     \
 164                 __put_user_size((x),__pu_addr,(size),__pu_err); \
 165         __pu_err;                                       \
 166 })
 167
 168 #define __put_user_size(x,ptr,size,retval)                              \
 169 do {                                                                    \
 170         retval = 0;                                                     \
 171         switch (size) {                                                 \
 172           case 1: __put_user_asm(x,ptr,retval,"b","b","iq",-EFAULT); break;\
 173           case 2: __put_user_asm(x,ptr,retval,"w","w","ir",-EFAULT); break;\
 174           case 4: __put_user_asm(x,ptr,retval,"l","k","ir",-EFAULT); break;\
 175           case 8: __put_user_asm(x,ptr,retval,"q","","ir",-EFAULT); break;\
 176           default: __put_user_bad();                                    \
 177         }                                                               \
 178 } while (0)
 179
 180 /* FIXME: this hack is definitely wrong -AK */
 181 struct __large_struct { unsigned long buf[100]; };
 182 #define __m(x) (*(struct __large_struct *)(x))
 183
 184 /*
 185  * Tell gcc we read from memory instead of writing: this is because
 186  * we do not write to any memory gcc knows about, so there are no
 187  * aliasing issues.
 188  */
 189 #define __put_user_asm(x, addr, err, itype, rtype, ltype, errno)        \
 190         __asm__ __volatile__(                                   \
 191                 "1:     mov"itype" %"rtype"1,%2\n"              \
 192                 "2:\n"                                          \
 193                 ".section .fixup,\"ax\"\n"                      \
 194                 "3:     mov %3,%0\n"                            \
 195                 "       jmp 2b\n"                               \
 196                 ".previous\n"                                   \
 197                 ".section __ex_table,\"a\"\n"                   \
 198                 "       .align 8\n"                             \
 199                 "       .quad 1b,3b\n"                          \
 200                 ".previous"                                     \
 201                 : "=r"(err)                                     \
 202                 : ltype (x), "m"(__m(addr)), "i"(errno), "0"(err))
 203
 204
 205 #define __get_user_nocheck(x,ptr,size)                          \
 206 ({                                                              \
 207         int __gu_err;                                           \
 208         long __gu_val;                                          \
 209         CHECK_UPTR(ptr);                                        \
 210         __get_user_size(__gu_val,(ptr),(size),__gu_err);        \
 211         (x) = (__typeof__(*(ptr)))__gu_val;                     \
 212         __gu_err;                                               \
 213 })
 214
 215 extern int __get_user_bad(void);
 216
 217 #define __get_user_size(x,ptr,size,retval)                              \
 218 do {                                                                    \
 219         retval = 0;                                                     \
 220         switch (size) {                                                 \
 221           case 1: __get_user_asm(x,ptr,retval,"b","b","=q",-EFAULT); break;\
 222           case 2: __get_user_asm(x,ptr,retval,"w","w","=r",-EFAULT); break;\
 223           case 4: __get_user_asm(x,ptr,retval,"l","k","=r",-EFAULT); break;\
 224           case 8: __get_user_asm(x,ptr,retval,"q","","=r",-EFAULT); break;\
 225           default: (x) = __get_user_bad();                              \
 226         }                                                               \
 227 } while (0)
 228
 229 #define __get_user_asm(x, addr, err, itype, rtype, ltype, errno)        \
 230         __asm__ __volatile__(                                   \
 231                 "1:     mov"itype" %2,%"rtype"1\n"              \
 232                 "2:\n"                                          \
 233                 ".section .fixup,\"ax\"\n"                      \
 234                 "3:     mov %3,%0\n"                            \
 235                 "       xor"itype" %"rtype"1,%"rtype"1\n"       \
 236                 "       jmp 2b\n"                               \
 237                 ".previous\n"                                   \
 238                 ".section __ex_table,\"a\"\n"                   \
 239                 "       .align 8\n"                             \
 240                 "       .quad 1b,3b\n"                          \
 241                 ".previous"                                     \
 242                 : "=r"(err), ltype (x)                          \
 243                 : "m"(__m(addr)), "i"(errno), "0"(err))
 244
 245 /*
 246  * Copy To/From Userspace
 247  */
 248
 249 /* Handles exceptions in both to and from, but doesn't do access_ok */
 250 extern unsigned long copy_user_generic(void *to, const void *from, unsigned len);
 251
 252 extern unsigned long copy_to_user(void __user *to, const void *from, unsigned len);
 253 extern unsigned long copy_from_user(void *to, const void __user *from, unsigned len);
 254 extern unsigned long copy_in_user(void __user *to, const void __user *from, unsigned len);
 255
 256 static inline int __copy_from_user(void *dst, const void __user *src, unsigned size)
 257 {
 258        int ret = 0;
 259         if (!__builtin_constant_p(size))
 260                 return copy_user_generic(dst,(void *)src,size);
 261         switch (size) {
 262         case 1:__get_user_asm(*(u8*)dst,(u8 *)src,ret,"b","b","=q",1);
 263                 return ret;
 264         case 2:__get_user_asm(*(u16*)dst,(u16*)src,ret,"w","w","=r",2);
 265                 return ret;
 266         case 4:__get_user_asm(*(u32*)dst,(u32*)src,ret,"l","k","=r",4);
 267                 return ret;
 268         case 8:__get_user_asm(*(u64*)dst,(u64*)src,ret,"q","","=r",8);
 269                 return ret;
 270         case 10:
 271                 __get_user_asm(*(u64*)dst,(u64*)src,ret,"q","","=r",16);
 272                 if (unlikely(ret)) return ret;
 273                 __get_user_asm(*(u16*)(8+(char*)dst),(u16*)(8+(char*)src),ret,"w","w","=r",2);
 274                 return ret;
 275         case 16:
 276                 __get_user_asm(*(u64*)dst,(u64*)src,ret,"q","","=r",16);
 277                 if (unlikely(ret)) return ret;
 278                 __get_user_asm(*(u64*)(8+(char*)dst),(u64*)(8+(char*)src),ret,"q","","=r",8);
 279                 return ret;
 280         default:
 281                 return copy_user_generic(dst,(void *)src,size);
 282         }
 283 }
 284
 285 static inline int __copy_to_user(void __user *dst, const void *src, unsigned size)
 286 {
 287        int ret = 0;
 288         if (!__builtin_constant_p(size))
 289                 return copy_user_generic((void *)dst,src,size);
 290         switch (size) {
 291         case 1:__put_user_asm(*(u8*)src,(u8 *)dst,ret,"b","b","iq",1);
 292                 return ret;
 293         case 2:__put_user_asm(*(u16*)src,(u16*)dst,ret,"w","w","ir",2);
 294                 return ret;
 295         case 4:__put_user_asm(*(u32*)src,(u32*)dst,ret,"l","k","ir",4);
 296                 return ret;
 297         case 8:__put_user_asm(*(u64*)src,(u64*)dst,ret,"q","","ir",8);
 298                 return ret;
 299         case 10:
 300                 __put_user_asm(*(u64*)src,(u64*)dst,ret,"q","","ir",10);
 301                 if (unlikely(ret)) return ret;
 302                 asm("":::"memory");
 303                 __put_user_asm(4[(u16*)src],4+(u16*)dst,ret,"w","w","ir",2);
 304                 return ret;
 305         case 16:
 306                 __put_user_asm(*(u64*)src,(u64*)dst,ret,"q","","ir",16);
 307                 if (unlikely(ret)) return ret;
 308                 asm("":::"memory");
 309                 __put_user_asm(1[(u64*)src],1+(u64*)dst,ret,"q","","ir",8);
 310                 return ret;
 311         default:
 312                 return copy_user_generic((void *)dst,src,size);
 313         }
 314 }
 315
 316
 317 static inline int __copy_in_user(void __user *dst, const void __user *src, unsigned size)
 318 {
 319        int ret = 0;
 320         if (!__builtin_constant_p(size))
 321                 return copy_user_generic((void *)dst,(void *)src,size);
 322         switch (size) {
 323         case 1: {
 324                 u8 tmp;
 325                 __get_user_asm(tmp,(u8 *)src,ret,"b","b","=q",1);
 326                 if (likely(!ret))
 327                         __put_user_asm(tmp,(u8 *)dst,ret,"b","b","iq",1);
 328                 return ret;
 329         }
 330         case 2: {
 331                 u16 tmp;
 332                 __get_user_asm(tmp,(u16 *)src,ret,"w","w","=r",2);
 333                 if (likely(!ret))
 334                         __put_user_asm(tmp,(u16 *)dst,ret,"w","w","ir",2);
 335                 return ret;
 336         }
 337
 338         case 4: {
 339                 u32 tmp;
 340                 __get_user_asm(tmp,(u32 *)src,ret,"l","k","=r",4);
 341                 if (likely(!ret))
 342                         __put_user_asm(tmp,(u32 *)dst,ret,"l","k","ir",4);
 343                 return ret;
 344         }
 345         case 8: {
 346                 u64 tmp;
 347                 __get_user_asm(tmp,(u64 *)src,ret,"q","","=r",8);
 348                 if (likely(!ret))
 349                         __put_user_asm(tmp,(u64 *)dst,ret,"q","","ir",8);
 350                 return ret;
 351         }
 352         default:
 353                 return copy_user_generic((void *)dst,(void *)src,size);
 354         }
 355 }
 356
 357 long strncpy_from_user(char *dst, const char __user *src, long count);
 358 long __strncpy_from_user(char *dst, const char __user *src, long count);
 359 long strnlen_user(const char __user *str, long n);
 360 long strlen_user(const char __user *str);
 361 unsigned long clear_user(void __user *mem, unsigned long len);
 362 unsigned long __clear_user(void __user *mem, unsigned long len);
 363
 364 #endif /* __X86_64_UACCESS_H */