2 * csum_partial_copy - do IP checksumming and copy
4 * (C) Copyright 1996 Linus Torvalds
5 * accellerated versions (and 21264 assembly versions ) contributed by
6 * Rick Gorton <rick.gorton@alpha-processor.com>
8 * Don't look at this too closely - you'll go mad. The things
9 * we do for performance..
12 #include <linux/types.h>
13 #include <linux/string.h>
14 #include <asm/uaccess.h>
18 __asm__ __volatile__("ldq_u %0,%1":"=r" (x):"m" (*(const unsigned long *)(y)))
21 __asm__ __volatile__("stq_u %1,%0":"=m" (*(unsigned long *)(y)):"r" (x))
23 #define extql(x,y,z) \
24 __asm__ __volatile__("extql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
26 #define extqh(x,y,z) \
27 __asm__ __volatile__("extqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
29 #define mskql(x,y,z) \
30 __asm__ __volatile__("mskql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
32 #define mskqh(x,y,z) \
33 __asm__ __volatile__("mskqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
35 #define insql(x,y,z) \
36 __asm__ __volatile__("insql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
38 #define insqh(x,y,z) \
39 __asm__ __volatile__("insqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
42 #define __get_user_u(x,ptr) \
45 __asm__ __volatile__( \
48 ".section __ex_table,\"a\"\n" \
50 " lda %0,2b-1b(%1)\n" \
52 : "=r"(x), "=r"(__guu_err) \
53 : "m"(__m(ptr)), "1"(0)); \
57 #define __put_user_u(x,ptr) \
60 __asm__ __volatile__( \
63 ".section __ex_table,\"a\"\n" \
65 " lda $31,2b-1b(%0)\n" \
68 : "m"(__m(addr)), "rJ"(x), "0"(0)); \
73 static inline unsigned short from64to16(unsigned long x)
75 /* Using extract instructions is a bit more efficient
76 than the original shift/bitmask version. */
85 tmp_v.ul = (unsigned long) in_v.ui[0] + (unsigned long) in_v.ui[1];
87 /* Since the bits of tmp_v.sh[3] are going to always be zero,
88 we don't have to bother to add that in. */
89 out_v.ul = (unsigned long) tmp_v.us[0] + (unsigned long) tmp_v.us[1]
90 + (unsigned long) tmp_v.us[2];
92 /* Similarly, out_v.us[2] is always zero for the final add. */
93 return out_v.us[0] + out_v.us[1];
99 * Ok. This isn't fun, but this is the EASY case.
101 static inline unsigned long
102 csum_partial_cfu_aligned(const unsigned long *src, unsigned long *dst,
103 long len, unsigned long checksum,
106 unsigned long carry = 0;
111 err |= __get_user(word, src);
116 carry = checksum < word;
123 unsigned long word, tmp;
124 err |= __get_user(word, src);
126 mskql(word, len, word);
128 mskqh(tmp, len, tmp);
129 carry = checksum < word;
133 if (err) *errp = err;
138 * This is even less fun, but this is still reasonably
141 static inline unsigned long
142 csum_partial_cfu_dest_aligned(const unsigned long *src, unsigned long *dst,
144 long len, unsigned long checksum,
148 unsigned long word, carry;
149 unsigned long lastsrc = 7+len+(unsigned long)src;
152 err |= __get_user_u(first,src);
155 unsigned long second;
157 err |= __get_user_u(second, src+1);
158 extql(first, soff, word);
161 extqh(second, soff, first);
168 carry = checksum < word;
174 unsigned long second;
175 err |= __get_user_u(second, lastsrc);
177 extql(first, soff, word);
178 extqh(second, soff, first);
180 mskql(word, len, word);
182 mskqh(tmp, len, tmp);
183 carry = checksum < word;
187 if (err) *errp = err;
192 * This is slightly less fun than the above..
194 static inline unsigned long
195 csum_partial_cfu_src_aligned(const unsigned long *src, unsigned long *dst,
197 long len, unsigned long checksum,
198 unsigned long partial_dest,
201 unsigned long carry = 0;
203 unsigned long second_dest;
206 mskql(partial_dest, doff, partial_dest);
208 err |= __get_user(word, src);
210 insql(word, doff, second_dest);
212 stq_u(partial_dest | second_dest, dst);
215 insqh(word, doff, partial_dest);
216 carry = checksum < word;
222 err |= __get_user(word, src);
223 mskql(word, len, word);
226 insql(word, doff, second_dest);
228 carry = checksum < word;
229 partial_dest |= second_dest;
231 stq_u(partial_dest, dst);
234 insqh(word, doff, partial_dest);
238 ldq_u(second_dest, dst);
239 mskqh(second_dest, doff, second_dest);
240 stq_u(partial_dest | second_dest, dst);
243 if (err) *errp = err;
248 * This is so totally un-fun that it's frightening. Don't
249 * look at this too closely, you'll go blind.
251 static inline unsigned long
252 csum_partial_cfu_unaligned(const unsigned long * src, unsigned long * dst,
253 unsigned long soff, unsigned long doff,
254 long len, unsigned long checksum,
255 unsigned long partial_dest,
258 unsigned long carry = 0;
260 unsigned long lastsrc;
263 err |= __get_user_u(first, src);
264 lastsrc = 7+len+(unsigned long)src;
265 mskql(partial_dest, doff, partial_dest);
267 unsigned long second, word;
268 unsigned long second_dest;
270 err |= __get_user_u(second, src+1);
271 extql(first, soff, word);
274 extqh(second, soff, first);
278 insql(word, doff, second_dest);
280 stq_u(partial_dest | second_dest, dst);
281 carry = checksum < word;
282 insqh(word, doff, partial_dest);
288 unsigned long second, word;
289 unsigned long second_dest;
291 err |= __get_user_u(second, lastsrc);
292 extql(first, soff, word);
293 extqh(second, soff, first);
296 mskql(word, len-doff, word);
298 insql(word, doff, second_dest);
299 carry = checksum < word;
300 stq_u(partial_dest | second_dest, dst);
302 ldq_u(second_dest, dst+1);
303 insqh(word, doff, partial_dest);
304 mskqh(second_dest, len, second_dest);
305 stq_u(partial_dest | second_dest, dst+1);
309 unsigned long second, word;
310 unsigned long second_dest;
312 err |= __get_user_u(second, lastsrc);
313 extql(first, soff, word);
314 extqh(second, soff, first);
316 ldq_u(second_dest, dst);
317 mskql(word, len-doff, word);
319 mskqh(second_dest, len, second_dest);
320 carry = checksum < word;
321 insql(word, doff, word);
322 stq_u(partial_dest | word | second_dest, dst);
325 if (err) *errp = err;
330 do_csum_partial_copy_from_user(const char *src, char *dst, int len,
331 unsigned int sum, int *errp)
333 unsigned long checksum = (unsigned) sum;
334 unsigned long soff = 7 & (unsigned long) src;
335 unsigned long doff = 7 & (unsigned long) dst;
340 checksum = csum_partial_cfu_aligned(
341 (const unsigned long *) src,
342 (unsigned long *) dst,
343 len-8, checksum, errp);
345 checksum = csum_partial_cfu_dest_aligned(
346 (const unsigned long *) src,
347 (unsigned long *) dst,
348 soff, len-8, checksum, errp);
350 unsigned long partial_dest;
351 ldq_u(partial_dest, dst);
353 checksum = csum_partial_cfu_src_aligned(
354 (const unsigned long *) src,
355 (unsigned long *) dst,
356 doff, len-8, checksum,
359 checksum = csum_partial_cfu_unaligned(
360 (const unsigned long *) src,
361 (unsigned long *) dst,
362 soff, doff, len-8, checksum,
365 checksum = from64to16 (checksum);
371 csum_partial_copy_from_user(const char *src, char *dst, int len,
372 unsigned int sum, int *errp)
374 if (!access_ok(src, len, VERIFY_READ)) {
380 return do_csum_partial_copy_from_user(src, dst, len, sum, errp);
384 csum_partial_copy_nocheck(const char *src, char *dst, int len, unsigned int sum)
386 return do_csum_partial_copy_from_user(src, dst, len, sum, NULL);