/*
* User space memory access functions
*/
-#include <linux/config.h>
#include <linux/errno.h>
#include <linux/thread_info.h>
#include <linux/prefetch.h>
__chk_user_ptr(addr); \
asm("addl %3,%1 ; sbbl %0,%0; cmpl %1,%4; sbbl $0,%0" \
:"=&r" (flag), "=r" (sum) \
- :"1" (addr),"g" ((int)(size)),"g" (current_thread_info()->addr_limit.seg)); \
+ :"1" (addr),"g" ((int)(size)),"rm" (current_thread_info()->addr_limit.seg)); \
flag; })
/**
*/
#define access_ok(type,addr,size) (likely(__range_ok(addr,size) == 0))
-/**
- * verify_area: - Obsolete, use access_ok()
- * @type: Type of access: %VERIFY_READ or %VERIFY_WRITE
- * @addr: User space pointer to start of block to check
- * @size: Size of block to check
- *
- * Context: User context only. This function may sleep.
- *
- * This function has been replaced by access_ok().
- *
- * Checks if a pointer to a block of memory in user space is valid.
- *
- * Returns zero if the memory block may be valid, -EFAULT
- * if it is definitely invalid.
- *
- * See access_ok() for more details.
- */
-static inline int verify_area(int type, const void __user * addr, unsigned long size)
-{
- return access_ok(type,addr,size) ? 0 : -EFAULT;
-}
-
-
/*
* The exception table consists of pairs of addresses: the first is the
* address of an instruction that is allowed to fault, and the second is
extern void __put_user_bad(void);
+/*
+ * Strange magic calling convention: pointer in %ecx,
+ * value in %eax(:%edx), return value in %eax, no clobbers.
+ */
+extern void __put_user_1(void);
+extern void __put_user_2(void);
+extern void __put_user_4(void);
+extern void __put_user_8(void);
+
+#define __put_user_1(x, ptr) __asm__ __volatile__("call __put_user_1":"=a" (__ret_pu):"0" ((typeof(*(ptr)))(x)), "c" (ptr))
+#define __put_user_2(x, ptr) __asm__ __volatile__("call __put_user_2":"=a" (__ret_pu):"0" ((typeof(*(ptr)))(x)), "c" (ptr))
+#define __put_user_4(x, ptr) __asm__ __volatile__("call __put_user_4":"=a" (__ret_pu):"0" ((typeof(*(ptr)))(x)), "c" (ptr))
+#define __put_user_8(x, ptr) __asm__ __volatile__("call __put_user_8":"=a" (__ret_pu):"A" ((typeof(*(ptr)))(x)), "c" (ptr))
+#define __put_user_X(x, ptr) __asm__ __volatile__("call __put_user_X":"=a" (__ret_pu):"c" (ptr))
+
/**
* put_user: - Write a simple value into user space.
* @x: Value to copy to user space.
*
* Returns zero on success, or -EFAULT on error.
*/
-#define put_user(x,ptr) \
- __put_user_check((__typeof__(*(ptr)))(x),(ptr),sizeof(*(ptr)))
+#ifdef CONFIG_X86_WP_WORKS_OK
+
+#define put_user(x,ptr) \
+({ int __ret_pu; \
+ __typeof__(*(ptr)) __pu_val; \
+ __chk_user_ptr(ptr); \
+ __pu_val = x; \
+ switch(sizeof(*(ptr))) { \
+ case 1: __put_user_1(__pu_val, ptr); break; \
+ case 2: __put_user_2(__pu_val, ptr); break; \
+ case 4: __put_user_4(__pu_val, ptr); break; \
+ case 8: __put_user_8(__pu_val, ptr); break; \
+ default:__put_user_X(__pu_val, ptr); break; \
+ } \
+ __ret_pu; \
+})
+#else
+#define put_user(x,ptr) \
+({ \
+ int __ret_pu; \
+ __typeof__(*(ptr)) __pus_tmp = x; \
+ __ret_pu=0; \
+ if(unlikely(__copy_to_user_ll(ptr, &__pus_tmp, \
+ sizeof(*(ptr))) != 0)) \
+ __ret_pu=-EFAULT; \
+ __ret_pu; \
+ })
+
+
+#endif
/**
* __get_user: - Get a simple variable from user space, with less checking.
})
-#define __put_user_check(x,ptr,size) \
-({ \
- long __pu_err = -EFAULT; \
- __typeof__(*(ptr)) __user *__pu_addr = (ptr); \
- might_sleep(); \
- if (access_ok(VERIFY_WRITE,__pu_addr,size)) \
- __put_user_size((x),__pu_addr,(size),__pu_err,-EFAULT); \
- __pu_err; \
-})
-
#define __put_user_u64(x, addr, err) \
__asm__ __volatile__( \
"1: movl %%eax,0(%2)\n" \
const void *from, unsigned long n);
unsigned long __must_check __copy_from_user_ll(void *to,
const void __user *from, unsigned long n);
+unsigned long __must_check __copy_from_user_ll_nozero(void *to,
+ const void __user *from, unsigned long n);
+unsigned long __must_check __copy_from_user_ll_nocache(void *to,
+ const void __user *from, unsigned long n);
+unsigned long __must_check __copy_from_user_ll_nocache_nozero(void *to,
+ const void __user *from, unsigned long n);
/*
* Here we special-case 1, 2 and 4-byte copy_*_user invocations. On a fault
* anything, so this is accurate.
*/
+static __always_inline unsigned long __must_check
+__copy_to_user_inatomic(void __user *to, const void *from, unsigned long n)
+{
+ if (__builtin_constant_p(n)) {
+ unsigned long ret;
+
+ switch (n) {
+ case 1:
+ __put_user_size(*(u8 *)from, (u8 __user *)to, 1, ret, 1);
+ return ret;
+ case 2:
+ __put_user_size(*(u16 *)from, (u16 __user *)to, 2, ret, 2);
+ return ret;
+ case 4:
+ __put_user_size(*(u32 *)from, (u32 __user *)to, 4, ret, 4);
+ return ret;
+ }
+ }
+ return __copy_to_user_ll(to, from, n);
+}
+
/**
* __copy_to_user: - Copy a block of data into user space, with less checking.
* @to: Destination address, in user space.
* Returns number of bytes that could not be copied.
* On success, this will be zero.
*/
-static inline unsigned long __must_check
-__copy_to_user_inatomic(void __user *to, const void *from, unsigned long n)
+static __always_inline unsigned long __must_check
+__copy_to_user(void __user *to, const void *from, unsigned long n)
{
+ might_sleep();
+ return __copy_to_user_inatomic(to, from, n);
+}
+
+static __always_inline unsigned long
+__copy_from_user_inatomic(void *to, const void __user *from, unsigned long n)
+{
+ /* Avoid zeroing the tail if the copy fails..
+ * If 'n' is constant and 1, 2, or 4, we do still zero on a failure,
+ * but as the zeroing behaviour is only significant when n is not
+ * constant, that shouldn't be a problem.
+ */
if (__builtin_constant_p(n)) {
unsigned long ret;
switch (n) {
case 1:
- __put_user_size(*(u8 *)from, (u8 __user *)to, 1, ret, 1);
+ __get_user_size(*(u8 *)to, from, 1, ret, 1);
return ret;
case 2:
- __put_user_size(*(u16 *)from, (u16 __user *)to, 2, ret, 2);
+ __get_user_size(*(u16 *)to, from, 2, ret, 2);
return ret;
case 4:
- __put_user_size(*(u32 *)from, (u32 __user *)to, 4, ret, 4);
+ __get_user_size(*(u32 *)to, from, 4, ret, 4);
return ret;
}
}
- return __copy_to_user_ll(to, from, n);
-}
-
-static inline unsigned long __must_check
-__copy_to_user(void __user *to, const void *from, unsigned long n)
-{
- might_sleep();
- return __copy_to_user_inatomic(to, from, n);
+ return __copy_from_user_ll_nozero(to, from, n);
}
/**
*
* If some data could not be copied, this function will pad the copied
* data to the requested size using zero bytes.
+ *
+ * An alternate version - __copy_from_user_inatomic() - may be called from
+ * atomic context and will fail rather than sleep. In this case the
+ * uncopied bytes will *NOT* be padded with zeros. See fs/filemap.h
+ * for explanation of why this is needed.
*/
-static inline unsigned long
-__copy_from_user_inatomic(void *to, const void __user *from, unsigned long n)
+static __always_inline unsigned long
+__copy_from_user(void *to, const void __user *from, unsigned long n)
{
+ might_sleep();
if (__builtin_constant_p(n)) {
unsigned long ret;
return __copy_from_user_ll(to, from, n);
}
-static inline unsigned long
-__copy_from_user(void *to, const void __user *from, unsigned long n)
+#define ARCH_HAS_NOCACHE_UACCESS
+
+static __always_inline unsigned long __copy_from_user_nocache(void *to,
+ const void __user *from, unsigned long n)
{
- might_sleep();
- return __copy_from_user_inatomic(to, from, n);
+ might_sleep();
+ if (__builtin_constant_p(n)) {
+ unsigned long ret;
+
+ switch (n) {
+ case 1:
+ __get_user_size(*(u8 *)to, from, 1, ret, 1);
+ return ret;
+ case 2:
+ __get_user_size(*(u16 *)to, from, 2, ret, 2);
+ return ret;
+ case 4:
+ __get_user_size(*(u32 *)to, from, 4, ret, 4);
+ return ret;
+ }
+ }
+ return __copy_from_user_ll_nocache(to, from, n);
+}
+
+static __always_inline unsigned long
+__copy_from_user_inatomic_nocache(void *to, const void __user *from, unsigned long n)
+{
+ return __copy_from_user_ll_nocache_nozero(to, from, n);
}
+
unsigned long __must_check copy_to_user(void __user *to,
const void *from, unsigned long n);
unsigned long __must_check copy_from_user(void *to,
git://git.onelab.eu / linux-2.6.git / blobdiff