2 * csum_partial_copy - do IP checksumming and copy
4 * (C) Copyright 1996 Linus Torvalds
5 * accellerated versions (and 21264 assembly versions ) contributed by
6 * Rick Gorton <rick.gorton@alpha-processor.com>
8 * Don't look at this too closely - you'll go mad. The things
9 * we do for performance..
12 #include <linux/types.h>
13 #include <linux/string.h>
14 #include <asm/uaccess.h>
18 __asm__ __volatile__("ldq_u %0,%1":"=r" (x):"m" (*(const unsigned long *)(y)))
21 __asm__ __volatile__("stq_u %1,%0":"=m" (*(unsigned long *)(y)):"r" (x))
23 #define extql(x,y,z) \
24 __asm__ __volatile__("extql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
26 #define extqh(x,y,z) \
27 __asm__ __volatile__("extqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
29 #define mskql(x,y,z) \
30 __asm__ __volatile__("mskql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
32 #define mskqh(x,y,z) \
33 __asm__ __volatile__("mskqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
35 #define insql(x,y,z) \
36 __asm__ __volatile__("insql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
38 #define insqh(x,y,z) \
39 __asm__ __volatile__("insqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
42 #define __get_user_u(x,ptr) \
45 __asm__ __volatile__( \
48 ".section __ex_table,\"a\"\n" \
50 " lda %0,2b-1b(%1)\n" \
52 : "=r"(x), "=r"(__guu_err) \
53 : "m"(__m(ptr)), "1"(0)); \
57 #define __put_user_u(x,ptr) \
60 __asm__ __volatile__( \
63 ".section __ex_table,\"a\"\n" \
65 " lda $31,2b-1b(%0)\n" \
68 : "m"(__m(addr)), "rJ"(x), "0"(0)); \
73 static inline unsigned short from64to16(unsigned long x)
75 /* Using extract instructions is a bit more efficient
76 than the original shift/bitmask version. */
85 tmp_v.ul = (unsigned long) in_v.ui[0] + (unsigned long) in_v.ui[1];
87 /* Since the bits of tmp_v.sh[3] are going to always be zero,
88 we don't have to bother to add that in. */
89 out_v.ul = (unsigned long) tmp_v.us[0] + (unsigned long) tmp_v.us[1]
90 + (unsigned long) tmp_v.us[2];
92 /* Similarly, out_v.us[2] is always zero for the final add. */
93 return out_v.us[0] + out_v.us[1];
99 * Ok. This isn't fun, but this is the EASY case.
101 static inline unsigned long
102 csum_partial_cfu_aligned(const unsigned long __user *src, unsigned long *dst,
103 long len, unsigned long checksum,
106 unsigned long carry = 0;
111 err |= __get_user(word, src);
116 carry = checksum < word;
123 unsigned long word, tmp;
124 err |= __get_user(word, src);
126 mskql(word, len, word);
128 mskqh(tmp, len, tmp);
129 carry = checksum < word;
133 if (err) *errp = err;
138 * This is even less fun, but this is still reasonably
141 static inline unsigned long
142 csum_partial_cfu_dest_aligned(const unsigned long __user *src,
145 long len, unsigned long checksum,
149 unsigned long word, carry;
150 unsigned long lastsrc = 7+len+(unsigned long)src;
153 err |= __get_user_u(first,src);
156 unsigned long second;
158 err |= __get_user_u(second, src+1);
159 extql(first, soff, word);
162 extqh(second, soff, first);
169 carry = checksum < word;
175 unsigned long second;
176 err |= __get_user_u(second, lastsrc);
178 extql(first, soff, word);
179 extqh(second, soff, first);
181 mskql(word, len, word);
183 mskqh(tmp, len, tmp);
184 carry = checksum < word;
188 if (err) *errp = err;
193 * This is slightly less fun than the above..
195 static inline unsigned long
196 csum_partial_cfu_src_aligned(const unsigned long __user *src,
199 long len, unsigned long checksum,
200 unsigned long partial_dest,
203 unsigned long carry = 0;
205 unsigned long second_dest;
208 mskql(partial_dest, doff, partial_dest);
210 err |= __get_user(word, src);
212 insql(word, doff, second_dest);
214 stq_u(partial_dest | second_dest, dst);
217 insqh(word, doff, partial_dest);
218 carry = checksum < word;
224 err |= __get_user(word, src);
225 mskql(word, len, word);
228 insql(word, doff, second_dest);
230 carry = checksum < word;
231 partial_dest |= second_dest;
233 stq_u(partial_dest, dst);
236 insqh(word, doff, partial_dest);
240 ldq_u(second_dest, dst);
241 mskqh(second_dest, doff, second_dest);
242 stq_u(partial_dest | second_dest, dst);
245 if (err) *errp = err;
250 * This is so totally un-fun that it's frightening. Don't
251 * look at this too closely, you'll go blind.
253 static inline unsigned long
254 csum_partial_cfu_unaligned(const unsigned long * src, unsigned long * dst,
255 unsigned long soff, unsigned long doff,
256 long len, unsigned long checksum,
257 unsigned long partial_dest,
260 unsigned long carry = 0;
262 unsigned long lastsrc;
265 err |= __get_user_u(first, src);
266 lastsrc = 7+len+(unsigned long)src;
267 mskql(partial_dest, doff, partial_dest);
269 unsigned long second, word;
270 unsigned long second_dest;
272 err |= __get_user_u(second, src+1);
273 extql(first, soff, word);
276 extqh(second, soff, first);
280 insql(word, doff, second_dest);
282 stq_u(partial_dest | second_dest, dst);
283 carry = checksum < word;
284 insqh(word, doff, partial_dest);
290 unsigned long second, word;
291 unsigned long second_dest;
293 err |= __get_user_u(second, lastsrc);
294 extql(first, soff, word);
295 extqh(second, soff, first);
298 mskql(word, len-doff, word);
300 insql(word, doff, second_dest);
301 carry = checksum < word;
302 stq_u(partial_dest | second_dest, dst);
304 ldq_u(second_dest, dst+1);
305 insqh(word, doff, partial_dest);
306 mskqh(second_dest, len, second_dest);
307 stq_u(partial_dest | second_dest, dst+1);
311 unsigned long second, word;
312 unsigned long second_dest;
314 err |= __get_user_u(second, lastsrc);
315 extql(first, soff, word);
316 extqh(second, soff, first);
318 ldq_u(second_dest, dst);
319 mskql(word, len-doff, word);
321 mskqh(second_dest, len, second_dest);
322 carry = checksum < word;
323 insql(word, doff, word);
324 stq_u(partial_dest | word | second_dest, dst);
327 if (err) *errp = err;
332 do_csum_partial_copy_from_user(const char __user *src, char *dst, int len,
333 unsigned int sum, int *errp)
335 unsigned long checksum = (unsigned) sum;
336 unsigned long soff = 7 & (unsigned long) src;
337 unsigned long doff = 7 & (unsigned long) dst;
342 checksum = csum_partial_cfu_aligned(
343 (const unsigned long __user *) src,
344 (unsigned long *) dst,
345 len-8, checksum, errp);
347 checksum = csum_partial_cfu_dest_aligned(
348 (const unsigned long __user *) src,
349 (unsigned long *) dst,
350 soff, len-8, checksum, errp);
352 unsigned long partial_dest;
353 ldq_u(partial_dest, dst);
355 checksum = csum_partial_cfu_src_aligned(
356 (const unsigned long __user *) src,
357 (unsigned long *) dst,
358 doff, len-8, checksum,
361 checksum = csum_partial_cfu_unaligned(
362 (const unsigned long __user *) src,
363 (unsigned long *) dst,
364 soff, doff, len-8, checksum,
367 checksum = from64to16 (checksum);
373 csum_partial_copy_from_user(const char __user *src, char *dst, int len,
374 unsigned int sum, int *errp)
376 if (!access_ok(VERIFY_READ, src, len)) {
382 return do_csum_partial_copy_from_user(src, dst, len, sum, errp);
386 csum_partial_copy_nocheck(const char *src, char *dst, int len, unsigned int sum)
388 return do_csum_partial_copy_from_user(src, dst, len, sum, NULL);