arch/alpha/lib/csum_partial_copy.c

   1 /*
   2  * csum_partial_copy - do IP checksumming and copy
   3  *
   4  * (C) Copyright 1996 Linus Torvalds
   5  * accellerated versions (and 21264 assembly versions ) contributed by
   6  *      Rick Gorton     <rick.gorton@alpha-processor.com>
   7  *
   8  * Don't look at this too closely - you'll go mad. The things
   9  * we do for performance..
  10  */
  11
  12 #include <linux/types.h>
  13 #include <linux/string.h>
  14 #include <asm/uaccess.h>
  15
  16
  17 #define ldq_u(x,y) \
  18 __asm__ __volatile__("ldq_u %0,%1":"=r" (x):"m" (*(const unsigned long *)(y)))
  19
  20 #define stq_u(x,y) \
  21 __asm__ __volatile__("stq_u %1,%0":"=m" (*(unsigned long *)(y)):"r" (x))
  22
  23 #define extql(x,y,z) \
  24 __asm__ __volatile__("extql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
  25
  26 #define extqh(x,y,z) \
  27 __asm__ __volatile__("extqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
  28
  29 #define mskql(x,y,z) \
  30 __asm__ __volatile__("mskql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
  31
  32 #define mskqh(x,y,z) \
  33 __asm__ __volatile__("mskqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
  34
  35 #define insql(x,y,z) \
  36 __asm__ __volatile__("insql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
  37
  38 #define insqh(x,y,z) \
  39 __asm__ __volatile__("insqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
  40
  41
  42 #define __get_user_u(x,ptr)                             \
  43 ({                                                      \
  44         long __guu_err;                                 \
  45         __asm__ __volatile__(                           \
  46         "1:     ldq_u %0,%2\n"                          \
  47         "2:\n"                                          \
  48         ".section __ex_table,\"a\"\n"                   \
  49         "       .long 1b - .\n"                         \
  50         "       lda %0,2b-1b(%1)\n"                     \
  51         ".previous"                                     \
  52                 : "=r"(x), "=r"(__guu_err)              \
  53                 : "m"(__m(ptr)), "1"(0));               \
  54         __guu_err;                                      \
  55 })
  56
  57 #define __put_user_u(x,ptr)                             \
  58 ({                                                      \
  59         long __puu_err;                                 \
  60         __asm__ __volatile__(                           \
  61         "1:     stq_u %2,%1\n"                          \
  62         "2:\n"                                          \
  63         ".section __ex_table,\"a\"\n"                   \
  64         "       .long 1b - ."                           \
  65         "       lda $31,2b-1b(%0)\n"                    \
  66         ".previous"                                     \
  67                 : "=r"(__puu_err)                       \
  68                 : "m"(__m(addr)), "rJ"(x), "0"(0));     \
  69         __puu_err;                                      \
  70 })
  71
  72
  73 static inline unsigned short from64to16(unsigned long x)
  74 {
  75         /* Using extract instructions is a bit more efficient
  76            than the original shift/bitmask version.  */
  77
  78         union {
  79                 unsigned long   ul;
  80                 unsigned int    ui[2];
  81                 unsigned short  us[4];
  82         } in_v, tmp_v, out_v;
  83
  84         in_v.ul = x;
  85         tmp_v.ul = (unsigned long) in_v.ui[0] + (unsigned long) in_v.ui[1];
  86
  87         /* Since the bits of tmp_v.sh[3] are going to always be zero,
  88            we don't have to bother to add that in.  */
  89         out_v.ul = (unsigned long) tmp_v.us[0] + (unsigned long) tmp_v.us[1]
  90                         + (unsigned long) tmp_v.us[2];
  91
  92         /* Similarly, out_v.us[2] is always zero for the final add.  */
  93         return out_v.us[0] + out_v.us[1];
  94 }
  95
  96
  97
  98 /*
  99  * Ok. This isn't fun, but this is the EASY case.
 100  */
 101 static inline unsigned long
 102 csum_partial_cfu_aligned(const unsigned long __user *src, unsigned long *dst,
 103                          long len, unsigned long checksum,
 104                          int *errp)
 105 {
 106         unsigned long carry = 0;
 107         int err = 0;
 108
 109         while (len >= 0) {
 110                 unsigned long word;
 111                 err |= __get_user(word, src);
 112                 checksum += carry;
 113                 src++;
 114                 checksum += word;
 115                 len -= 8;
 116                 carry = checksum < word;
 117                 *dst = word;
 118                 dst++;
 119         }
 120         len += 8;
 121         checksum += carry;
 122         if (len) {
 123                 unsigned long word, tmp;
 124                 err |= __get_user(word, src);
 125                 tmp = *dst;
 126                 mskql(word, len, word);
 127                 checksum += word;
 128                 mskqh(tmp, len, tmp);
 129                 carry = checksum < word;
 130                 *dst = word | tmp;
 131                 checksum += carry;
 132         }
 133         if (err) *errp = err;
 134         return checksum;
 135 }
 136
 137 /*
 138  * This is even less fun, but this is still reasonably
 139  * easy.
 140  */
 141 static inline unsigned long
 142 csum_partial_cfu_dest_aligned(const unsigned long __user *src,
 143                               unsigned long *dst,
 144                               unsigned long soff,
 145                               long len, unsigned long checksum,
 146                               int *errp)
 147 {
 148         unsigned long first;
 149         unsigned long word, carry;
 150         unsigned long lastsrc = 7+len+(unsigned long)src;
 151         int err = 0;
 152
 153         err |= __get_user_u(first,src);
 154         carry = 0;
 155         while (len >= 0) {
 156                 unsigned long second;
 157
 158                 err |= __get_user_u(second, src+1);
 159                 extql(first, soff, word);
 160                 len -= 8;
 161                 src++;
 162                 extqh(second, soff, first);
 163                 checksum += carry;
 164                 word |= first;
 165                 first = second;
 166                 checksum += word;
 167                 *dst = word;
 168                 dst++;
 169                 carry = checksum < word;
 170         }
 171         len += 8;
 172         checksum += carry;
 173         if (len) {
 174                 unsigned long tmp;
 175                 unsigned long second;
 176                 err |= __get_user_u(second, lastsrc);
 177                 tmp = *dst;
 178                 extql(first, soff, word);
 179                 extqh(second, soff, first);
 180                 word |= first;
 181                 mskql(word, len, word);
 182                 checksum += word;
 183                 mskqh(tmp, len, tmp);
 184                 carry = checksum < word;
 185                 *dst = word | tmp;
 186                 checksum += carry;
 187         }
 188         if (err) *errp = err;
 189         return checksum;
 190 }
 191
 192 /*
 193  * This is slightly less fun than the above..
 194  */
 195 static inline unsigned long
 196 csum_partial_cfu_src_aligned(const unsigned long *src, unsigned long *dst,
 197                              unsigned long doff,
 198                              long len, unsigned long checksum,
 199                              unsigned long partial_dest,
 200                              int *errp)
 201 {
 202         unsigned long carry = 0;
 203         unsigned long word;
 204         unsigned long second_dest;
 205         int err = 0;
 206
 207         mskql(partial_dest, doff, partial_dest);
 208         while (len >= 0) {
 209                 err |= __get_user(word, src);
 210                 len -= 8;
 211                 insql(word, doff, second_dest);
 212                 checksum += carry;
 213                 stq_u(partial_dest | second_dest, dst);
 214                 src++;
 215                 checksum += word;
 216                 insqh(word, doff, partial_dest);
 217                 carry = checksum < word;
 218                 dst++;
 219         }
 220         len += 8;
 221         if (len) {
 222                 checksum += carry;
 223                 err |= __get_user(word, src);
 224                 mskql(word, len, word);
 225                 len -= 8;
 226                 checksum += word;
 227                 insql(word, doff, second_dest);
 228                 len += doff;
 229                 carry = checksum < word;
 230                 partial_dest |= second_dest;
 231                 if (len >= 0) {
 232                         stq_u(partial_dest, dst);
 233                         if (!len) goto out;
 234                         dst++;
 235                         insqh(word, doff, partial_dest);
 236                 }
 237                 doff = len;
 238         }
 239         ldq_u(second_dest, dst);
 240         mskqh(second_dest, doff, second_dest);
 241         stq_u(partial_dest | second_dest, dst);
 242 out:
 243         checksum += carry;
 244         if (err) *errp = err;
 245         return checksum;
 246 }
 247
 248 /*
 249  * This is so totally un-fun that it's frightening. Don't
 250  * look at this too closely, you'll go blind.
 251  */
 252 static inline unsigned long
 253 csum_partial_cfu_unaligned(const unsigned long * src, unsigned long * dst,
 254                            unsigned long soff, unsigned long doff,
 255                            long len, unsigned long checksum,
 256                            unsigned long partial_dest,
 257                            int *errp)
 258 {
 259         unsigned long carry = 0;
 260         unsigned long first;
 261         unsigned long lastsrc;
 262         int err = 0;
 263
 264         err |= __get_user_u(first, src);
 265         lastsrc = 7+len+(unsigned long)src;
 266         mskql(partial_dest, doff, partial_dest);
 267         while (len >= 0) {
 268                 unsigned long second, word;
 269                 unsigned long second_dest;
 270
 271                 err |= __get_user_u(second, src+1);
 272                 extql(first, soff, word);
 273                 checksum += carry;
 274                 len -= 8;
 275                 extqh(second, soff, first);
 276                 src++;
 277                 word |= first;
 278                 first = second;
 279                 insql(word, doff, second_dest);
 280                 checksum += word;
 281                 stq_u(partial_dest | second_dest, dst);
 282                 carry = checksum < word;
 283                 insqh(word, doff, partial_dest);
 284                 dst++;
 285         }
 286         len += doff;
 287         checksum += carry;
 288         if (len >= 0) {
 289                 unsigned long second, word;
 290                 unsigned long second_dest;
 291
 292                 err |= __get_user_u(second, lastsrc);
 293                 extql(first, soff, word);
 294                 extqh(second, soff, first);
 295                 word |= first;
 296                 first = second;
 297                 mskql(word, len-doff, word);
 298                 checksum += word;
 299                 insql(word, doff, second_dest);
 300                 carry = checksum < word;
 301                 stq_u(partial_dest | second_dest, dst);
 302                 if (len) {
 303                         ldq_u(second_dest, dst+1);
 304                         insqh(word, doff, partial_dest);
 305                         mskqh(second_dest, len, second_dest);
 306                         stq_u(partial_dest | second_dest, dst+1);
 307                 }
 308                 checksum += carry;
 309         } else {
 310                 unsigned long second, word;
 311                 unsigned long second_dest;
 312
 313                 err |= __get_user_u(second, lastsrc);
 314                 extql(first, soff, word);
 315                 extqh(second, soff, first);
 316                 word |= first;
 317                 ldq_u(second_dest, dst);
 318                 mskql(word, len-doff, word);
 319                 checksum += word;
 320                 mskqh(second_dest, len, second_dest);
 321                 carry = checksum < word;
 322                 insql(word, doff, word);
 323                 stq_u(partial_dest | word | second_dest, dst);
 324                 checksum += carry;
 325         }
 326         if (err) *errp = err;
 327         return checksum;
 328 }
 329
 330 static unsigned int
 331 do_csum_partial_copy_from_user(const char __user *src, char *dst, int len,
 332                                unsigned int sum, int *errp)
 333 {
 334         unsigned long checksum = (unsigned) sum;
 335         unsigned long soff = 7 & (unsigned long) src;
 336         unsigned long doff = 7 & (unsigned long) dst;
 337
 338         if (len) {
 339                 if (!doff) {
 340                         if (!soff)
 341                                 checksum = csum_partial_cfu_aligned(
 342                                         (const unsigned long __user *) src,
 343                                         (unsigned long *) dst,
 344                                         len-8, checksum, errp);
 345                         else
 346                                 checksum = csum_partial_cfu_dest_aligned(
 347                                         (const unsigned long __user *) src,
 348                                         (unsigned long *) dst,
 349                                         soff, len-8, checksum, errp);
 350                 } else {
 351                         unsigned long partial_dest;
 352                         ldq_u(partial_dest, dst);
 353                         if (!soff)
 354                                 checksum = csum_partial_cfu_src_aligned(
 355                                         (const unsigned long __user *) src,
 356                                         (unsigned long *) dst,
 357                                         doff, len-8, checksum,
 358                                         partial_dest, errp);
 359                         else
 360                                 checksum = csum_partial_cfu_unaligned(
 361                                         (const unsigned long __user *) src,
 362                                         (unsigned long *) dst,
 363                                         soff, doff, len-8, checksum,
 364                                         partial_dest, errp);
 365                 }
 366                 checksum = from64to16 (checksum);
 367         }
 368         return checksum;
 369 }
 370
 371 unsigned int
 372 csum_partial_copy_from_user(const char __user *src, char *dst, int len,
 373                             unsigned int sum, int *errp)
 374 {
 375         if (!access_ok(src, len, VERIFY_READ)) {
 376                 *errp = -EFAULT;
 377                 memset(dst, 0, len);
 378                 return sum;
 379         }
 380
 381         return do_csum_partial_copy_from_user(src, dst, len, sum, errp);
 382 }
 383
 384 unsigned int
 385 csum_partial_copy_nocheck(const char *src, char *dst, int len, unsigned int sum)
 386 {
 387         return do_csum_partial_copy_from_user(src, dst, len, sum, NULL);
 388 }