2 * User address space access functions.
3 * The non inlined parts of asm-i386/uaccess.h are here.
5 * Copyright 1997 Andi Kleen <ak@muc.de>
6 * Copyright 1997 Linus Torvalds
8 #include <linux/config.h>
10 #include <linux/highmem.h>
11 #include <linux/blkdev.h>
12 #include <asm/uaccess.h>
15 static inline int __movsl_is_ok(unsigned long a1, unsigned long a2, unsigned long n)
17 #ifdef CONFIG_X86_INTEL_USERCOPY
18 if (n >= 64 && ((a1 ^ a2) & movsl_mask.mask))
23 #define movsl_is_ok(a1,a2,n) \
24 __movsl_is_ok((unsigned long)(a1),(unsigned long)(a2),(n))
27 * Copy a null terminated string from userspace.
30 #define __do_strncpy_from_user(dst,src,count,res) \
32 int __d0, __d1, __d2; \
34 __asm__ __volatile__( \
39 " testb %%al,%%al\n" \
45 ".section .fixup,\"ax\"\n" \
49 ".section __ex_table,\"a\"\n" \
53 : "=d"(res), "=c"(count), "=&a" (__d0), "=&S" (__d1), \
55 : "i"(-EFAULT), "0"(count), "1"(count), "3"(src), "4"(dst) \
60 * __strncpy_from_user: - Copy a NUL terminated string from userspace, with less checking.
61 * @dst: Destination address, in kernel space. This buffer must be at
62 * least @count bytes long.
63 * @src: Source address, in user space.
64 * @count: Maximum number of bytes to copy, including the trailing NUL.
66 * Copies a NUL-terminated string from userspace to kernel space.
67 * Caller must check the specified block with access_ok() before calling
70 * On success, returns the length of the string (not including the trailing
73 * If access to userspace fails, returns -EFAULT (some data may have been
76 * If @count is smaller than the length of the string, copies @count bytes
80 __direct_strncpy_from_user(char *dst, const char __user *src, long count)
83 __do_strncpy_from_user(dst, src, count, res);
88 * strncpy_from_user: - Copy a NUL terminated string from userspace.
89 * @dst: Destination address, in kernel space. This buffer must be at
90 * least @count bytes long.
91 * @src: Source address, in user space.
92 * @count: Maximum number of bytes to copy, including the trailing NUL.
94 * Copies a NUL-terminated string from userspace to kernel space.
96 * On success, returns the length of the string (not including the trailing
99 * If access to userspace fails, returns -EFAULT (some data may have been
102 * If @count is smaller than the length of the string, copies @count bytes
103 * and returns @count.
106 direct_strncpy_from_user(char *dst, const char __user *src, long count)
109 if (access_ok(VERIFY_READ, src, 1))
110 __do_strncpy_from_user(dst, src, count, res);
119 #define __do_clear_user(addr,size) \
123 __asm__ __volatile__( \
128 ".section .fixup,\"ax\"\n" \
129 "3: lea 0(%2,%0,4),%0\n" \
132 ".section __ex_table,\"a\"\n" \
137 : "=&c"(size), "=&D" (__d0) \
138 : "r"(size & 3), "0"(size / 4), "1"(addr), "a"(0)); \
142 * clear_user: - Zero a block of memory in user space.
143 * @to: Destination address, in user space.
144 * @n: Number of bytes to zero.
146 * Zero a block of memory in user space.
148 * Returns number of bytes that could not be cleared.
149 * On success, this will be zero.
152 direct_clear_user(void __user *to, unsigned long n)
155 if (access_ok(VERIFY_WRITE, to, n))
156 __do_clear_user(to, n);
161 * __clear_user: - Zero a block of memory in user space, with less checking.
162 * @to: Destination address, in user space.
163 * @n: Number of bytes to zero.
165 * Zero a block of memory in user space. Caller must check
166 * the specified block with access_ok() before calling this function.
168 * Returns number of bytes that could not be cleared.
169 * On success, this will be zero.
172 __direct_clear_user(void __user *to, unsigned long n)
174 __do_clear_user(to, n);
179 * strlen_user: - Get the size of a string in user space.
180 * @s: The string to measure.
181 * @n: The maximum valid length
183 * Get the size of a NUL-terminated string in user space.
185 * Returns the size of the string INCLUDING the terminating NUL.
186 * On exception, returns 0.
187 * If the string is too long, returns a value greater than @n.
189 long direct_strnlen_user(const char __user *s, long n)
191 unsigned long mask = -__addr_ok(s);
192 unsigned long res, tmp;
196 __asm__ __volatile__(
205 ".section .fixup,\"ax\"\n"
206 "2: xorl %%eax,%%eax\n"
211 ".section __ex_table,\"a\"\n"
215 :"=r" (n), "=D" (s), "=a" (res), "=c" (tmp)
216 :"0" (n), "1" (s), "2" (0), "3" (mask)
221 #ifdef CONFIG_X86_INTEL_USERCOPY
223 __copy_user_intel(void __user *to, const void *from, unsigned long size)
226 __asm__ __volatile__(
228 "1: movl 32(%4), %%eax\n"
231 "2: movl 64(%4), %%eax\n"
233 "3: movl 0(%4), %%eax\n"
234 "4: movl 4(%4), %%edx\n"
235 "5: movl %%eax, 0(%3)\n"
236 "6: movl %%edx, 4(%3)\n"
237 "7: movl 8(%4), %%eax\n"
238 "8: movl 12(%4),%%edx\n"
239 "9: movl %%eax, 8(%3)\n"
240 "10: movl %%edx, 12(%3)\n"
241 "11: movl 16(%4), %%eax\n"
242 "12: movl 20(%4), %%edx\n"
243 "13: movl %%eax, 16(%3)\n"
244 "14: movl %%edx, 20(%3)\n"
245 "15: movl 24(%4), %%eax\n"
246 "16: movl 28(%4), %%edx\n"
247 "17: movl %%eax, 24(%3)\n"
248 "18: movl %%edx, 28(%3)\n"
249 "19: movl 32(%4), %%eax\n"
250 "20: movl 36(%4), %%edx\n"
251 "21: movl %%eax, 32(%3)\n"
252 "22: movl %%edx, 36(%3)\n"
253 "23: movl 40(%4), %%eax\n"
254 "24: movl 44(%4), %%edx\n"
255 "25: movl %%eax, 40(%3)\n"
256 "26: movl %%edx, 44(%3)\n"
257 "27: movl 48(%4), %%eax\n"
258 "28: movl 52(%4), %%edx\n"
259 "29: movl %%eax, 48(%3)\n"
260 "30: movl %%edx, 52(%3)\n"
261 "31: movl 56(%4), %%eax\n"
262 "32: movl 60(%4), %%edx\n"
263 "33: movl %%eax, 56(%3)\n"
264 "34: movl %%edx, 60(%3)\n"
270 "35: movl %0, %%eax\n"
275 "36: movl %%eax, %0\n"
278 ".section .fixup,\"ax\"\n"
279 "101: lea 0(%%eax,%0,4),%0\n"
282 ".section __ex_table,\"a\"\n"
323 : "=&c"(size), "=&D" (d0), "=&S" (d1)
324 : "1"(to), "2"(from), "0"(size)
325 : "eax", "edx", "memory");
330 __copy_user_zeroing_intel(void *to, const void __user *from, unsigned long size)
333 __asm__ __volatile__(
335 "0: movl 32(%4), %%eax\n"
338 "1: movl 64(%4), %%eax\n"
340 "2: movl 0(%4), %%eax\n"
341 "21: movl 4(%4), %%edx\n"
342 " movl %%eax, 0(%3)\n"
343 " movl %%edx, 4(%3)\n"
344 "3: movl 8(%4), %%eax\n"
345 "31: movl 12(%4),%%edx\n"
346 " movl %%eax, 8(%3)\n"
347 " movl %%edx, 12(%3)\n"
348 "4: movl 16(%4), %%eax\n"
349 "41: movl 20(%4), %%edx\n"
350 " movl %%eax, 16(%3)\n"
351 " movl %%edx, 20(%3)\n"
352 "10: movl 24(%4), %%eax\n"
353 "51: movl 28(%4), %%edx\n"
354 " movl %%eax, 24(%3)\n"
355 " movl %%edx, 28(%3)\n"
356 "11: movl 32(%4), %%eax\n"
357 "61: movl 36(%4), %%edx\n"
358 " movl %%eax, 32(%3)\n"
359 " movl %%edx, 36(%3)\n"
360 "12: movl 40(%4), %%eax\n"
361 "71: movl 44(%4), %%edx\n"
362 " movl %%eax, 40(%3)\n"
363 " movl %%edx, 44(%3)\n"
364 "13: movl 48(%4), %%eax\n"
365 "81: movl 52(%4), %%edx\n"
366 " movl %%eax, 48(%3)\n"
367 " movl %%edx, 52(%3)\n"
368 "14: movl 56(%4), %%eax\n"
369 "91: movl 60(%4), %%edx\n"
370 " movl %%eax, 56(%3)\n"
371 " movl %%edx, 60(%3)\n"
377 "5: movl %0, %%eax\n"
385 ".section .fixup,\"ax\"\n"
386 "9: lea 0(%%eax,%0,4),%0\n"
389 " xorl %%eax,%%eax\n"
395 ".section __ex_table,\"a\"\n"
418 : "=&c"(size), "=&D" (d0), "=&S" (d1)
419 : "1"(to), "2"(from), "0"(size)
420 : "eax", "edx", "memory");
425 * Leave these declared but undefined. They should not be any references to
429 __copy_user_zeroing_intel(void *to, const void __user *from, unsigned long size);
431 __copy_user_intel(void __user *to, const void *from, unsigned long size);
432 #endif /* CONFIG_X86_INTEL_USERCOPY */
434 /* Generic arbitrary sized copy. */
435 #define __copy_user(to,from,size) \
437 int __d0, __d1, __d2; \
438 __asm__ __volatile__( \
454 ".section .fixup,\"ax\"\n" \
457 "3: lea 0(%3,%0,4),%0\n" \
460 ".section __ex_table,\"a\"\n" \
466 : "=&c"(size), "=&D" (__d0), "=&S" (__d1), "=r"(__d2) \
467 : "3"(size), "0"(size), "1"(to), "2"(from) \
471 #define __copy_user_zeroing(to,from,size) \
473 int __d0, __d1, __d2; \
474 __asm__ __volatile__( \
490 ".section .fixup,\"ax\"\n" \
493 "3: lea 0(%3,%0,4),%0\n" \
496 " xorl %%eax,%%eax\n" \
502 ".section __ex_table,\"a\"\n" \
508 : "=&c"(size), "=&D" (__d0), "=&S" (__d1), "=r"(__d2) \
509 : "3"(size), "0"(size), "1"(to), "2"(from) \
514 unsigned long __copy_to_user_ll(void __user *to, const void *from, unsigned long n)
516 #ifndef CONFIG_X86_WP_WORKS_OK
517 if (unlikely(boot_cpu_data.wp_works_ok == 0) &&
518 ((unsigned long )to) < TASK_SIZE) {
520 * CPU does not honor the WP bit when writing
521 * from supervisory mode, and due to preemption or SMP,
522 * the page tables can change at any time.
523 * Do it manually. Manfred <manfred@colorfullife.com>
526 unsigned long offset = ((unsigned long)to)%PAGE_SIZE;
527 unsigned long len = PAGE_SIZE - offset;
536 down_read(¤t->mm->mmap_sem);
537 retval = get_user_pages(current, current->mm,
538 (unsigned long )to, 1, 1, 0, &pg, NULL);
540 if (retval == -ENOMEM && current->pid == 1) {
541 up_read(¤t->mm->mmap_sem);
542 blk_congestion_wait(WRITE, HZ/50);
547 up_read(¤t->mm->mmap_sem);
551 maddr = kmap_atomic(pg, KM_USER0);
552 memcpy(maddr + offset, from, len);
553 kunmap_atomic(maddr, KM_USER0);
554 set_page_dirty_lock(pg);
556 up_read(¤t->mm->mmap_sem);
565 if (movsl_is_ok(to, from, n))
566 __copy_user(to, from, n);
568 n = __copy_user_intel(to, from, n);
572 unsigned long __copy_from_user_ll(void *to, const void __user *from, unsigned long n)
574 if (movsl_is_ok(to, from, n))
575 __copy_user_zeroing(to, from, n);
577 n = __copy_user_zeroing_intel(to, from, n);