arch/cris/arch-v32/drivers/cryptocop.c

   1 /* $Id: cryptocop.c,v 1.13 2005/04/21 17:27:55 henriken Exp $
   2  *
   3  * Stream co-processor driver for the ETRAX FS
   4  *
   5  *    Copyright (C) 2003-2005  Axis Communications AB
   6  */
   7
   8 #include <linux/init.h>
   9 #include <linux/sched.h>
  10 #include <linux/module.h>
  11 #include <linux/slab.h>
  12 #include <linux/string.h>
  13 #include <linux/fs.h>
  14 #include <linux/mm.h>
  15 #include <linux/spinlock.h>
  16 #include <linux/stddef.h>
  17
  18 #include <asm/uaccess.h>
  19 #include <asm/io.h>
  20 #include <asm/atomic.h>
  21
  22 #include <linux/list.h>
  23 #include <linux/interrupt.h>
  24
  25 #include <asm/signal.h>
  26 #include <asm/irq.h>
  27
  28 #include <asm/arch/dma.h>
  29 #include <asm/arch/hwregs/dma.h>
  30 #include <asm/arch/hwregs/reg_map.h>
  31 #include <asm/arch/hwregs/reg_rdwr.h>
  32 #include <asm/arch/hwregs/intr_vect_defs.h>
  33
  34 #include <asm/arch/hwregs/strcop.h>
  35 #include <asm/arch/hwregs/strcop_defs.h>
  36 #include <asm/arch/cryptocop.h>
  37
  38
  39
  40 #define DESCR_ALLOC_PAD  (31)
  41
  42 struct cryptocop_dma_desc {
  43         char *free_buf; /* If non-null will be kfreed in free_cdesc() */
  44         dma_descr_data *dma_descr;
  45
  46         unsigned char dma_descr_buf[sizeof(dma_descr_data) + DESCR_ALLOC_PAD];
  47
  48         unsigned int from_pool:1; /* If 1 'allocated' from the descriptor pool. */
  49         struct cryptocop_dma_desc *next;
  50 };
  51
  52
  53 struct cryptocop_int_operation{
  54         void                        *alloc_ptr;
  55         cryptocop_session_id        sid;
  56
  57         dma_descr_context           ctx_out;
  58         dma_descr_context           ctx_in;
  59
  60         /* DMA descriptors allocated by driver. */
  61         struct cryptocop_dma_desc   *cdesc_out;
  62         struct cryptocop_dma_desc   *cdesc_in;
  63
  64         /* Strcop config to use. */
  65         cryptocop_3des_mode         tdes_mode;
  66         cryptocop_csum_type         csum_mode;
  67
  68         /* DMA descrs provided by consumer. */
  69         dma_descr_data              *ddesc_out;
  70         dma_descr_data              *ddesc_in;
  71 };
  72
  73
  74 struct cryptocop_tfrm_ctx {
  75         cryptocop_tfrm_id tid;
  76         unsigned int blocklength;
  77
  78         unsigned int start_ix;
  79
  80         struct cryptocop_tfrm_cfg *tcfg;
  81         struct cryptocop_transform_ctx *tctx;
  82
  83         unsigned char previous_src;
  84         unsigned char current_src;
  85
  86         /* Values to use in metadata out. */
  87         unsigned char hash_conf;
  88         unsigned char hash_mode;
  89         unsigned char ciph_conf;
  90         unsigned char cbcmode;
  91         unsigned char decrypt;
  92
  93         unsigned int requires_padding:1;
  94         unsigned int strict_block_length:1;
  95         unsigned int active:1;
  96         unsigned int done:1;
  97         size_t consumed;
  98         size_t produced;
  99
 100         /* Pad (input) descriptors to put in the DMA out list when the transform
 101          * output is put on the DMA in list. */
 102         struct cryptocop_dma_desc *pad_descs;
 103
 104         struct cryptocop_tfrm_ctx *prev_src;
 105         struct cryptocop_tfrm_ctx *curr_src;
 106
 107         /* Mapping to HW. */
 108         unsigned char unit_no;
 109 };
 110
 111
 112 struct cryptocop_private{
 113         cryptocop_session_id sid;
 114         struct cryptocop_private *next;
 115 };
 116
 117 /* Session list. */
 118
 119 struct cryptocop_transform_ctx{
 120         struct cryptocop_transform_init init;
 121         unsigned char dec_key[CRYPTOCOP_MAX_KEY_LENGTH];
 122         unsigned int dec_key_set:1;
 123
 124         struct cryptocop_transform_ctx *next;
 125 };
 126
 127
 128 struct cryptocop_session{
 129         cryptocop_session_id sid;
 130
 131         struct cryptocop_transform_ctx *tfrm_ctx;
 132
 133         struct cryptocop_session *next;
 134 };
 135
 136 /* Priority levels for jobs sent to the cryptocop.  Checksum operations from
 137    kernel have highest priority since TCPIP stack processing must not
 138    be a bottleneck. */
 139 typedef enum {
 140         cryptocop_prio_kernel_csum = 0,
 141         cryptocop_prio_kernel = 1,
 142         cryptocop_prio_user = 2,
 143         cryptocop_prio_no_prios = 3
 144 } cryptocop_queue_priority;
 145
 146 struct cryptocop_prio_queue{
 147         struct list_head jobs;
 148         cryptocop_queue_priority prio;
 149 };
 150
 151 struct cryptocop_prio_job{
 152         struct list_head node;
 153         cryptocop_queue_priority prio;
 154
 155         struct cryptocop_operation *oper;
 156         struct cryptocop_int_operation *iop;
 157 };
 158
 159 struct ioctl_job_cb_ctx {
 160         unsigned int processed:1;
 161 };
 162
 163
 164 static struct cryptocop_session *cryptocop_sessions = NULL;
 165 spinlock_t cryptocop_sessions_lock;
 166
 167 /* Next Session ID to assign. */
 168 static cryptocop_session_id next_sid = 1;
 169
 170 /* Pad for checksum. */
 171 static const char csum_zero_pad[1] = {0x00};
 172
 173 /* Trash buffer for mem2mem operations. */
 174 #define MEM2MEM_DISCARD_BUF_LENGTH  (512)
 175 static unsigned char mem2mem_discard_buf[MEM2MEM_DISCARD_BUF_LENGTH];
 176
 177 /* Descriptor pool. */
 178 /* FIXME Tweak this value. */
 179 #define CRYPTOCOP_DESCRIPTOR_POOL_SIZE   (100)
 180 static struct cryptocop_dma_desc descr_pool[CRYPTOCOP_DESCRIPTOR_POOL_SIZE];
 181 static struct cryptocop_dma_desc *descr_pool_free_list;
 182 static int descr_pool_no_free;
 183 static spinlock_t descr_pool_lock;
 184
 185 /* Lock to stop cryptocop to start processing of a new operation. The holder
 186    of this lock MUST call cryptocop_start_job() after it is unlocked. */
 187 spinlock_t cryptocop_process_lock;
 188
 189 static struct cryptocop_prio_queue cryptocop_job_queues[cryptocop_prio_no_prios];
 190 static spinlock_t cryptocop_job_queue_lock;
 191 static struct cryptocop_prio_job *cryptocop_running_job = NULL;
 192 static spinlock_t running_job_lock;
 193
 194 /* The interrupt handler appends completed jobs to this list. The scehduled
 195  * tasklet removes them upon sending the response to the crypto consumer. */
 196 static struct list_head cryptocop_completed_jobs;
 197 static spinlock_t cryptocop_completed_jobs_lock;
 198
 199 DECLARE_WAIT_QUEUE_HEAD(cryptocop_ioc_process_wq);
 200
 201
 202 /** Local functions. **/
 203
 204 static int cryptocop_open(struct inode *, struct file *);
 205
 206 static int cryptocop_release(struct inode *, struct file *);
 207
 208 static int cryptocop_ioctl(struct inode *inode, struct file *file,
 209                            unsigned int cmd, unsigned long arg);
 210
 211 static void cryptocop_start_job(void);
 212
 213 static int cryptocop_job_queue_insert(cryptocop_queue_priority prio, struct cryptocop_operation *operation);
 214 static int cryptocop_job_setup(struct cryptocop_prio_job **pj, struct cryptocop_operation *operation);
 215
 216 static int cryptocop_job_queue_init(void);
 217 static void cryptocop_job_queue_close(void);
 218
 219 static int create_md5_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length);
 220
 221 static int create_sha1_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length);
 222
 223 static int transform_ok(struct cryptocop_transform_init *tinit);
 224
 225 static struct cryptocop_session *get_session(cryptocop_session_id sid);
 226
 227 static struct cryptocop_transform_ctx *get_transform_ctx(struct cryptocop_session *sess, cryptocop_tfrm_id tid);
 228
 229 static void delete_internal_operation(struct cryptocop_int_operation *iop);
 230
 231 static void get_aes_decrypt_key(unsigned char *dec_key, const unsigned  char *key, unsigned int keylength);
 232
 233 static int init_stream_coprocessor(void);
 234
 235 static void __exit exit_stream_coprocessor(void);
 236
 237 /*#define LDEBUG*/
 238 #ifdef LDEBUG
 239 #define DEBUG(s) s
 240 #define DEBUG_API(s) s
 241 static void print_cryptocop_operation(struct cryptocop_operation *cop);
 242 static void print_dma_descriptors(struct cryptocop_int_operation *iop);
 243 static void print_strcop_crypto_op(struct strcop_crypto_op *cop);
 244 static void print_lock_status(void);
 245 static void print_user_dma_lists(struct cryptocop_dma_list_operation *dma_op);
 246 #define assert(s) do{if (!(s)) panic(#s);} while(0);
 247 #else
 248 #define DEBUG(s)
 249 #define DEBUG_API(s)
 250 #define assert(s)
 251 #endif
 252
 253
 254 /* Transform constants. */
 255 #define DES_BLOCK_LENGTH   (8)
 256 #define AES_BLOCK_LENGTH   (16)
 257 #define MD5_BLOCK_LENGTH   (64)
 258 #define SHA1_BLOCK_LENGTH  (64)
 259 #define CSUM_BLOCK_LENGTH  (2)
 260 #define MD5_STATE_LENGTH   (16)
 261 #define SHA1_STATE_LENGTH  (20)
 262
 263 /* The device number. */
 264 #define CRYPTOCOP_MAJOR    (254)
 265 #define CRYPTOCOP_MINOR    (0)
 266
 267
 268
 269 struct file_operations cryptocop_fops = {
 270         owner: THIS_MODULE,
 271         open: cryptocop_open,
 272         release: cryptocop_release,
 273         ioctl: cryptocop_ioctl
 274 };
 275
 276
 277 static void free_cdesc(struct cryptocop_dma_desc *cdesc)
 278 {
 279         DEBUG(printk("free_cdesc: cdesc 0x%p, from_pool=%d\n", cdesc, cdesc->from_pool));
 280         kfree(cdesc->free_buf);
 281
 282         if (cdesc->from_pool) {
 283                 unsigned long int flags;
 284                 spin_lock_irqsave(&descr_pool_lock, flags);
 285                 cdesc->next = descr_pool_free_list;
 286                 descr_pool_free_list = cdesc;
 287                 ++descr_pool_no_free;
 288                 spin_unlock_irqrestore(&descr_pool_lock, flags);
 289         } else {
 290                 kfree(cdesc);
 291         }
 292 }
 293
 294
 295 static struct cryptocop_dma_desc *alloc_cdesc(int alloc_flag)
 296 {
 297         int use_pool = (alloc_flag & GFP_ATOMIC) ? 1 : 0;
 298         struct cryptocop_dma_desc *cdesc;
 299
 300         if (use_pool) {
 301                 unsigned long int flags;
 302                 spin_lock_irqsave(&descr_pool_lock, flags);
 303                 if (!descr_pool_free_list) {
 304                         spin_unlock_irqrestore(&descr_pool_lock, flags);
 305                         DEBUG_API(printk("alloc_cdesc: pool is empty\n"));
 306                         return NULL;
 307                 }
 308                 cdesc = descr_pool_free_list;
 309                 descr_pool_free_list = descr_pool_free_list->next;
 310                 --descr_pool_no_free;
 311                 spin_unlock_irqrestore(&descr_pool_lock, flags);
 312                 cdesc->from_pool = 1;
 313         } else {
 314                 cdesc = kmalloc(sizeof(struct cryptocop_dma_desc), alloc_flag);
 315                 if (!cdesc) {
 316                         DEBUG_API(printk("alloc_cdesc: kmalloc\n"));
 317                         return NULL;
 318                 }
 319                 cdesc->from_pool = 0;
 320         }
 321         cdesc->dma_descr = (dma_descr_data*)(((unsigned long int)cdesc + offsetof(struct cryptocop_dma_desc, dma_descr_buf) + DESCR_ALLOC_PAD) & ~0x0000001F);
 322
 323         cdesc->next = NULL;
 324
 325         cdesc->free_buf = NULL;
 326         cdesc->dma_descr->out_eop = 0;
 327         cdesc->dma_descr->in_eop = 0;
 328         cdesc->dma_descr->intr = 0;
 329         cdesc->dma_descr->eol = 0;
 330         cdesc->dma_descr->wait = 0;
 331         cdesc->dma_descr->buf = NULL;
 332         cdesc->dma_descr->after = NULL;
 333
 334         DEBUG_API(printk("alloc_cdesc: return 0x%p, cdesc->dma_descr=0x%p, from_pool=%d\n", cdesc, cdesc->dma_descr, cdesc->from_pool));
 335         return cdesc;
 336 }
 337
 338
 339 static void setup_descr_chain(struct cryptocop_dma_desc *cd)
 340 {
 341         DEBUG(printk("setup_descr_chain: entering\n"));
 342         while (cd) {
 343                 if (cd->next) {
 344                         cd->dma_descr->next = (dma_descr_data*)virt_to_phys(cd->next->dma_descr);
 345                 } else {
 346                         cd->dma_descr->next = NULL;
 347                 }
 348                 cd = cd->next;
 349         }
 350         DEBUG(printk("setup_descr_chain: exit\n"));
 351 }
 352
 353
 354 /* Create a pad descriptor for the transform.
 355  * Return -1 for error, 0 if pad created. */
 356 static int create_pad_descriptor(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **pad_desc, int alloc_flag)
 357 {
 358         struct cryptocop_dma_desc        *cdesc = NULL;
 359         int                              error = 0;
 360         struct strcop_meta_out           mo = {
 361                 .ciphsel = src_none,
 362                 .hashsel = src_none,
 363                 .csumsel = src_none
 364         };
 365         char                             *pad;
 366         size_t                           plen;
 367
 368         DEBUG(printk("create_pad_descriptor: start.\n"));
 369         /* Setup pad descriptor. */
 370
 371         DEBUG(printk("create_pad_descriptor: setting up padding.\n"));
 372         cdesc = alloc_cdesc(alloc_flag);
 373         if (!cdesc){
 374                 DEBUG_API(printk("create_pad_descriptor: alloc pad desc\n"));
 375                 goto error_cleanup;
 376         }
 377         switch (tc->unit_no) {
 378         case src_md5:
 379                 error = create_md5_pad(alloc_flag, tc->consumed, &pad, &plen);
 380                 if (error){
 381                         DEBUG_API(printk("create_pad_descriptor: create_md5_pad_failed\n"));
 382                         goto error_cleanup;
 383                 }
 384                 cdesc->free_buf = pad;
 385                 mo.hashsel = src_dma;
 386                 mo.hashconf = tc->hash_conf;
 387                 mo.hashmode = tc->hash_mode;
 388                 break;
 389         case src_sha1:
 390                 error = create_sha1_pad(alloc_flag, tc->consumed, &pad, &plen);
 391                 if (error){
 392                         DEBUG_API(printk("create_pad_descriptor: create_sha1_pad_failed\n"));
 393                         goto error_cleanup;
 394                 }
 395                 cdesc->free_buf = pad;
 396                 mo.hashsel = src_dma;
 397                 mo.hashconf = tc->hash_conf;
 398                 mo.hashmode = tc->hash_mode;
 399                 break;
 400         case src_csum:
 401                 if (tc->consumed % tc->blocklength){
 402                         pad = (char*)csum_zero_pad;
 403                         plen = 1;
 404                 } else {
 405                         pad = (char*)cdesc; /* Use any pointer. */
 406                         plen = 0;
 407                 }
 408                 mo.csumsel = src_dma;
 409                 break;
 410         }
 411         cdesc->dma_descr->wait = 1;
 412         cdesc->dma_descr->out_eop = 1; /* Since this is a pad output is pushed.  EOP is ok here since the padded unit is the only one active. */
 413         cdesc->dma_descr->buf = (char*)virt_to_phys((char*)pad);
 414         cdesc->dma_descr->after = cdesc->dma_descr->buf + plen;
 415
 416         cdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
 417         *pad_desc = cdesc;
 418
 419         return 0;
 420
 421  error_cleanup:
 422         if (cdesc) free_cdesc(cdesc);
 423         return -1;
 424 }
 425
 426
 427 static int setup_key_dl_desc(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **kd, int alloc_flag)
 428 {
 429         struct cryptocop_dma_desc  *key_desc = alloc_cdesc(alloc_flag);
 430         struct strcop_meta_out     mo = {0};
 431
 432         DEBUG(printk("setup_key_dl_desc\n"));
 433
 434         if (!key_desc) {
 435                 DEBUG_API(printk("setup_key_dl_desc: failed descriptor allocation.\n"));
 436                 return -ENOMEM;
 437         }
 438
 439         /* Download key. */
 440         if ((tc->tctx->init.alg == cryptocop_alg_aes) && (tc->tcfg->flags & CRYPTOCOP_DECRYPT)) {
 441                 /* Precook the AES decrypt key. */
 442                 if (!tc->tctx->dec_key_set){
 443                         get_aes_decrypt_key(tc->tctx->dec_key, tc->tctx->init.key, tc->tctx->init.keylen);
 444                         tc->tctx->dec_key_set = 1;
 445                 }
 446                 key_desc->dma_descr->buf = (char*)virt_to_phys(tc->tctx->dec_key);
 447                 key_desc->dma_descr->after = key_desc->dma_descr->buf + tc->tctx->init.keylen/8;
 448         } else {
 449                 key_desc->dma_descr->buf = (char*)virt_to_phys(tc->tctx->init.key);
 450                 key_desc->dma_descr->after = key_desc->dma_descr->buf + tc->tctx->init.keylen/8;
 451         }
 452         /* Setup metadata. */
 453         mo.dlkey = 1;
 454         switch (tc->tctx->init.keylen) {
 455         case 64:
 456                 mo.decrypt = 0;
 457                 mo.hashmode = 0;
 458                 break;
 459         case 128:
 460                 mo.decrypt = 0;
 461                 mo.hashmode = 1;
 462                 break;
 463         case 192:
 464                 mo.decrypt = 1;
 465                 mo.hashmode = 0;
 466                 break;
 467         case 256:
 468                 mo.decrypt = 1;
 469                 mo.hashmode = 1;
 470                 break;
 471         default:
 472                 break;
 473         }
 474         mo.ciphsel = mo.hashsel = mo.csumsel = src_none;
 475         key_desc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
 476
 477         key_desc->dma_descr->out_eop = 1;
 478         key_desc->dma_descr->wait = 1;
 479         key_desc->dma_descr->intr = 0;
 480
 481         *kd = key_desc;
 482         return 0;
 483 }
 484
 485 static int setup_cipher_iv_desc(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **id, int alloc_flag)
 486 {
 487         struct cryptocop_dma_desc  *iv_desc = alloc_cdesc(alloc_flag);
 488         struct strcop_meta_out     mo = {0};
 489
 490         DEBUG(printk("setup_cipher_iv_desc\n"));
 491
 492         if (!iv_desc) {
 493                 DEBUG_API(printk("setup_cipher_iv_desc: failed CBC IV descriptor allocation.\n"));
 494                 return -ENOMEM;
 495         }
 496         /* Download IV. */
 497         iv_desc->dma_descr->buf = (char*)virt_to_phys(tc->tcfg->iv);
 498         iv_desc->dma_descr->after = iv_desc->dma_descr->buf + tc->blocklength;
 499
 500         /* Setup metadata. */
 501         mo.hashsel = mo.csumsel = src_none;
 502         mo.ciphsel = src_dma;
 503         mo.ciphconf = tc->ciph_conf;
 504         mo.cbcmode = tc->cbcmode;
 505
 506         iv_desc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
 507
 508         iv_desc->dma_descr->out_eop = 0;
 509         iv_desc->dma_descr->wait = 1;
 510         iv_desc->dma_descr->intr = 0;
 511
 512         *id = iv_desc;
 513         return 0;
 514 }
 515
 516 /* Map the ouput length of the transform to operation output starting on the inject index. */
 517 static int create_input_descriptors(struct cryptocop_operation *operation, struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **id, int alloc_flag)
 518 {
 519         int                        err = 0;
 520         struct cryptocop_dma_desc  head = {0};
 521         struct cryptocop_dma_desc  *outdesc = &head;
 522         size_t                     iov_offset = 0;
 523         size_t                     out_ix = 0;
 524         int                        outiov_ix = 0;
 525         struct strcop_meta_in      mi = {0};
 526
 527         size_t                     out_length = tc->produced;
 528         int                        rem_length;
 529         int                        dlength;
 530
 531         assert(out_length != 0);
 532         if (((tc->produced + tc->tcfg->inject_ix) > operation->tfrm_op.outlen) || (tc->produced && (operation->tfrm_op.outlen == 0))) {
 533                 DEBUG_API(printk("create_input_descriptors: operation outdata too small\n"));
 534                 return -EINVAL;
 535         }
 536         /* Traverse the out iovec until the result inject index is reached. */
 537         while ((outiov_ix < operation->tfrm_op.outcount) && ((out_ix + operation->tfrm_op.outdata[outiov_ix].iov_len) <= tc->tcfg->inject_ix)){
 538                 out_ix += operation->tfrm_op.outdata[outiov_ix].iov_len;
 539                 outiov_ix++;
 540         }
 541         if (outiov_ix >= operation->tfrm_op.outcount){
 542                 DEBUG_API(printk("create_input_descriptors: operation outdata too small\n"));
 543                 return -EINVAL;
 544         }
 545         iov_offset = tc->tcfg->inject_ix - out_ix;
 546         mi.dmasel = tc->unit_no;
 547
 548         /* Setup the output descriptors. */
 549         while ((out_length > 0) && (outiov_ix < operation->tfrm_op.outcount)) {
 550                 outdesc->next = alloc_cdesc(alloc_flag);
 551                 if (!outdesc->next) {
 552                         DEBUG_API(printk("create_input_descriptors: alloc_cdesc\n"));
 553                         err = -ENOMEM;
 554                         goto error_cleanup;
 555                 }
 556                 outdesc = outdesc->next;
 557                 rem_length = operation->tfrm_op.outdata[outiov_ix].iov_len - iov_offset;
 558                 dlength = (out_length < rem_length) ? out_length : rem_length;
 559
 560                 DEBUG(printk("create_input_descriptors:\n"
 561                              "outiov_ix=%d, rem_length=%d, dlength=%d\n"
 562                              "iov_offset=%d, outdata[outiov_ix].iov_len=%d\n"
 563                              "outcount=%d, outiov_ix=%d\n",
 564                              outiov_ix, rem_length, dlength, iov_offset, operation->tfrm_op.outdata[outiov_ix].iov_len, operation->tfrm_op.outcount, outiov_ix));
 565
 566                 outdesc->dma_descr->buf = (char*)virt_to_phys(operation->tfrm_op.outdata[outiov_ix].iov_base + iov_offset);
 567                 outdesc->dma_descr->after = outdesc->dma_descr->buf + dlength;
 568                 outdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
 569
 570                 out_length -= dlength;
 571                 iov_offset += dlength;
 572                 if (iov_offset >= operation->tfrm_op.outdata[outiov_ix].iov_len) {
 573                         iov_offset = 0;
 574                         ++outiov_ix;
 575                 }
 576         }
 577         if (out_length > 0){
 578                 DEBUG_API(printk("create_input_descriptors: not enough room for output, %d remained\n", out_length));
 579                 err = -EINVAL;
 580                 goto error_cleanup;
 581         }
 582         /* Set sync in last descriptor. */
 583         mi.sync = 1;
 584         outdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
 585
 586         *id = head.next;
 587         return 0;
 588
 589  error_cleanup:
 590         while (head.next) {
 591                 outdesc = head.next->next;
 592                 free_cdesc(head.next);
 593                 head.next = outdesc;
 594         }
 595         return err;
 596 }
 597
 598
 599 static int create_output_descriptors(struct cryptocop_operation *operation, int *iniov_ix, int *iniov_offset, size_t desc_len, struct cryptocop_dma_desc **current_out_cdesc, struct strcop_meta_out *meta_out, int alloc_flag)
 600 {
 601         while (desc_len != 0) {
 602                 struct cryptocop_dma_desc  *cdesc;
 603                 int                        rem_length = operation->tfrm_op.indata[*iniov_ix].iov_len - *iniov_offset;
 604                 int                        dlength = (desc_len < rem_length) ? desc_len : rem_length;
 605
 606                 cdesc = alloc_cdesc(alloc_flag);
 607                 if (!cdesc) {
 608                         DEBUG_API(printk("create_output_descriptors: alloc_cdesc\n"));
 609                         return -ENOMEM;
 610                 }
 611                 (*current_out_cdesc)->next = cdesc;
 612                 (*current_out_cdesc) = cdesc;
 613
 614                 cdesc->free_buf = NULL;
 615
 616                 cdesc->dma_descr->buf = (char*)virt_to_phys(operation->tfrm_op.indata[*iniov_ix].iov_base + *iniov_offset);
 617                 cdesc->dma_descr->after = cdesc->dma_descr->buf + dlength;
 618
 619                 desc_len -= dlength;
 620                 *iniov_offset += dlength;
 621                 assert(desc_len >= 0);
 622                 if (*iniov_offset >= operation->tfrm_op.indata[*iniov_ix].iov_len) {
 623                         *iniov_offset = 0;
 624                         ++(*iniov_ix);
 625                         if (*iniov_ix > operation->tfrm_op.incount) {
 626                                 DEBUG_API(printk("create_output_descriptors: not enough indata in operation."));
 627                                 return  -EINVAL;
 628                         }
 629                 }
 630                 cdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, (*meta_out));
 631         } /* while (desc_len != 0) */
 632         /* Last DMA descriptor gets a 'wait' bit to signal expected change in metadata. */
 633         (*current_out_cdesc)->dma_descr->wait = 1; /* This will set extraneous WAIT in some situations, e.g. when padding hashes and checksums. */
 634
 635         return 0;
 636 }
 637
 638
 639 static int append_input_descriptors(struct cryptocop_operation *operation, struct cryptocop_dma_desc **current_in_cdesc, struct cryptocop_dma_desc **current_out_cdesc, struct cryptocop_tfrm_ctx *tc, int alloc_flag)
 640 {
 641         DEBUG(printk("append_input_descriptors, tc=0x%p, unit_no=%d\n", tc, tc->unit_no));
 642         if (tc->tcfg) {
 643                 int                        failed = 0;
 644                 struct cryptocop_dma_desc  *idescs = NULL;
 645                 DEBUG(printk("append_input_descriptors: pushing output, consumed %d produced %d bytes.\n", tc->consumed, tc->produced));
 646                 if (tc->pad_descs) {
 647                         DEBUG(printk("append_input_descriptors: append pad descriptors to DMA out list.\n"));
 648                         while (tc->pad_descs) {
 649                                 DEBUG(printk("append descriptor 0x%p\n", tc->pad_descs));
 650                                 (*current_out_cdesc)->next = tc->pad_descs;
 651                                 tc->pad_descs = tc->pad_descs->next;
 652                                 (*current_out_cdesc) = (*current_out_cdesc)->next;
 653                         }
 654                 }
 655
 656                 /* Setup and append output descriptors to DMA in list. */
 657                 if (tc->unit_no == src_dma){
 658                         /* mem2mem.  Setup DMA in descriptors to discard all input prior to the requested mem2mem data. */
 659                         struct strcop_meta_in mi = {.sync = 0, .dmasel = src_dma};
 660                         unsigned int start_ix = tc->start_ix;
 661                         while (start_ix){
 662                                 unsigned int desclen = start_ix < MEM2MEM_DISCARD_BUF_LENGTH ? start_ix : MEM2MEM_DISCARD_BUF_LENGTH;
 663                                 (*current_in_cdesc)->next = alloc_cdesc(alloc_flag);
 664                                 if (!(*current_in_cdesc)->next){
 665                                         DEBUG_API(printk("append_input_descriptors: alloc_cdesc mem2mem discard failed\n"));
 666                                         return -ENOMEM;
 667                                 }
 668                                 (*current_in_cdesc) = (*current_in_cdesc)->next;
 669                                 (*current_in_cdesc)->dma_descr->buf = (char*)virt_to_phys(mem2mem_discard_buf);
 670                                 (*current_in_cdesc)->dma_descr->after = (*current_in_cdesc)->dma_descr->buf + desclen;
 671                                 (*current_in_cdesc)->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
 672                                 start_ix -= desclen;
 673                         }
 674                         mi.sync = 1;
 675                         (*current_in_cdesc)->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
 676                 }
 677
 678                 failed = create_input_descriptors(operation, tc, &idescs, alloc_flag);
 679                 if (failed){
 680                         DEBUG_API(printk("append_input_descriptors: output descriptor setup failed\n"));
 681                         return failed;
 682                 }
 683                 DEBUG(printk("append_input_descriptors: append output descriptors to DMA in list.\n"));
 684                 while (idescs) {
 685                         DEBUG(printk("append descriptor 0x%p\n", idescs));
 686                         (*current_in_cdesc)->next = idescs;
 687                         idescs = idescs->next;
 688                         (*current_in_cdesc) = (*current_in_cdesc)->next;
 689                 }
 690         }
 691         return 0;
 692 }
 693
 694
 695
 696 static int cryptocop_setup_dma_list(struct cryptocop_operation *operation, struct cryptocop_int_operation **int_op, int alloc_flag)
 697 {
 698         struct cryptocop_session *sess;
 699         struct cryptocop_transform_ctx *tctx;
 700
 701         struct cryptocop_tfrm_ctx digest_ctx = {
 702                 .previous_src = src_none,
 703                 .current_src = src_none,
 704                 .start_ix = 0,
 705                 .requires_padding = 1,
 706                 .strict_block_length = 0,
 707                 .hash_conf = 0,
 708                 .hash_mode = 0,
 709                 .ciph_conf = 0,
 710                 .cbcmode = 0,
 711                 .decrypt = 0,
 712                 .consumed = 0,
 713                 .produced = 0,
 714                 .pad_descs = NULL,
 715                 .active = 0,
 716                 .done = 0,
 717                 .prev_src = NULL,
 718                 .curr_src = NULL,
 719                 .tcfg = NULL};
 720         struct cryptocop_tfrm_ctx cipher_ctx = {
 721                 .previous_src = src_none,
 722                 .current_src = src_none,
 723                 .start_ix = 0,
 724                 .requires_padding = 0,
 725                 .strict_block_length = 1,
 726                 .hash_conf = 0,
 727                 .hash_mode = 0,
 728                 .ciph_conf = 0,
 729                 .cbcmode = 0,
 730                 .decrypt = 0,
 731                 .consumed = 0,
 732                 .produced = 0,
 733                 .pad_descs = NULL,
 734                 .active = 0,
 735                 .done = 0,
 736                 .prev_src = NULL,
 737                 .curr_src = NULL,
 738                 .tcfg = NULL};
 739         struct cryptocop_tfrm_ctx csum_ctx = {
 740                 .previous_src = src_none,
 741                 .current_src = src_none,
 742                 .start_ix = 0,
 743                 .blocklength = 2,
 744                 .requires_padding = 1,
 745                 .strict_block_length = 0,
 746                 .hash_conf = 0,
 747                 .hash_mode = 0,
 748                 .ciph_conf = 0,
 749                 .cbcmode = 0,
 750                 .decrypt = 0,
 751                 .consumed = 0,
 752                 .produced = 0,
 753                 .pad_descs = NULL,
 754                 .active = 0,
 755                 .done = 0,
 756                 .tcfg = NULL,
 757                 .prev_src = NULL,
 758                 .curr_src = NULL,
 759                 .unit_no = src_csum};
 760         struct cryptocop_tfrm_cfg *tcfg = operation->tfrm_op.tfrm_cfg;
 761
 762         unsigned int indata_ix = 0;
 763
 764         /* iovec accounting. */
 765         int iniov_ix = 0;
 766         int iniov_offset = 0;
 767
 768         /* Operation descriptor cfg traversal pointer. */
 769         struct cryptocop_desc *odsc;
 770
 771         int failed = 0;
 772         /* List heads for allocated descriptors. */
 773         struct cryptocop_dma_desc out_cdesc_head = {0};
 774         struct cryptocop_dma_desc in_cdesc_head = {0};
 775
 776         struct cryptocop_dma_desc *current_out_cdesc = &out_cdesc_head;
 777         struct cryptocop_dma_desc *current_in_cdesc = &in_cdesc_head;
 778
 779         struct cryptocop_tfrm_ctx *output_tc = NULL;
 780         void                      *iop_alloc_ptr;
 781
 782         assert(operation != NULL);
 783         assert(int_op != NULL);
 784
 785         DEBUG(printk("cryptocop_setup_dma_list: start\n"));
 786         DEBUG(print_cryptocop_operation(operation));
 787
 788         sess = get_session(operation->sid);
 789         if (!sess) {
 790                 DEBUG_API(printk("cryptocop_setup_dma_list: no session found for operation.\n"));
 791                 failed = -EINVAL;
 792                 goto error_cleanup;
 793         }
 794         iop_alloc_ptr = kmalloc(DESCR_ALLOC_PAD + sizeof(struct cryptocop_int_operation), alloc_flag);
 795         if (!iop_alloc_ptr) {
 796                 DEBUG_API(printk("cryptocop_setup_dma_list:  kmalloc cryptocop_int_operation\n"));
 797                 failed = -ENOMEM;
 798                 goto error_cleanup;
 799         }
 800         (*int_op) = (struct cryptocop_int_operation*)(((unsigned long int)(iop_alloc_ptr + DESCR_ALLOC_PAD + offsetof(struct cryptocop_int_operation, ctx_out)) & ~0x0000001F) - offsetof(struct cryptocop_int_operation, ctx_out));
 801         DEBUG(memset((*int_op), 0xff, sizeof(struct cryptocop_int_operation)));
 802         (*int_op)->alloc_ptr = iop_alloc_ptr;
 803         DEBUG(printk("cryptocop_setup_dma_list: *int_op=0x%p, alloc_ptr=0x%p\n", *int_op, (*int_op)->alloc_ptr));
 804
 805         (*int_op)->sid = operation->sid;
 806         (*int_op)->cdesc_out = NULL;
 807         (*int_op)->cdesc_in = NULL;
 808         (*int_op)->tdes_mode = cryptocop_3des_ede;
 809         (*int_op)->csum_mode = cryptocop_csum_le;
 810         (*int_op)->ddesc_out = NULL;
 811         (*int_op)->ddesc_in = NULL;
 812
 813         /* Scan operation->tfrm_op.tfrm_cfg for bad configuration and set up the local contexts. */
 814         if (!tcfg) {
 815                 DEBUG_API(printk("cryptocop_setup_dma_list: no configured transforms in operation.\n"));
 816                 failed = -EINVAL;
 817                 goto error_cleanup;
 818         }
 819         while (tcfg) {
 820                 tctx = get_transform_ctx(sess, tcfg->tid);
 821                 if (!tctx) {
 822                         DEBUG_API(printk("cryptocop_setup_dma_list: no transform id %d in session.\n", tcfg->tid));
 823                         failed = -EINVAL;
 824                         goto error_cleanup;
 825                 }
 826                 if (tcfg->inject_ix > operation->tfrm_op.outlen){
 827                         DEBUG_API(printk("cryptocop_setup_dma_list: transform id %d inject_ix (%d) > operation->tfrm_op.outlen(%d)", tcfg->tid, tcfg->inject_ix, operation->tfrm_op.outlen));
 828                         failed = -EINVAL;
 829                         goto error_cleanup;
 830                 }
 831                 switch (tctx->init.alg){
 832                 case cryptocop_alg_mem2mem:
 833                         if (cipher_ctx.tcfg != NULL){
 834                                 DEBUG_API(printk("cryptocop_setup_dma_list: multiple ciphers in operation.\n"));
 835                                 failed = -EINVAL;
 836                                 goto error_cleanup;
 837                         }
 838                         /* mem2mem is handled as a NULL cipher. */
 839                         cipher_ctx.cbcmode = 0;
 840                         cipher_ctx.decrypt = 0;
 841                         cipher_ctx.blocklength = 1;
 842                         cipher_ctx.ciph_conf = 0;
 843                         cipher_ctx.unit_no = src_dma;
 844                         cipher_ctx.tcfg = tcfg;
 845                         cipher_ctx.tctx = tctx;
 846                         break;
 847                 case cryptocop_alg_des:
 848                 case cryptocop_alg_3des:
 849                 case cryptocop_alg_aes:
 850                         /* cipher */
 851                         if (cipher_ctx.tcfg != NULL){
 852                                 DEBUG_API(printk("cryptocop_setup_dma_list: multiple ciphers in operation.\n"));
 853                                 failed = -EINVAL;
 854                                 goto error_cleanup;
 855                         }
 856                         cipher_ctx.tcfg = tcfg;
 857                         cipher_ctx.tctx = tctx;
 858                         if (cipher_ctx.tcfg->flags & CRYPTOCOP_DECRYPT){
 859                                 cipher_ctx.decrypt = 1;
 860                         }
 861                         switch (tctx->init.cipher_mode) {
 862                         case cryptocop_cipher_mode_ecb:
 863                                 cipher_ctx.cbcmode = 0;
 864                                 break;
 865                         case cryptocop_cipher_mode_cbc:
 866                                 cipher_ctx.cbcmode = 1;
 867                                 break;
 868                         default:
 869                                 DEBUG_API(printk("cryptocop_setup_dma_list: cipher_ctx, bad cipher mode==%d\n", tctx->init.cipher_mode));
 870                                 failed = -EINVAL;
 871                                 goto error_cleanup;
 872                         }
 873                         DEBUG(printk("cryptocop_setup_dma_list: cipher_ctx, set CBC mode==%d\n", cipher_ctx.cbcmode));
 874                         switch (tctx->init.alg){
 875                         case cryptocop_alg_des:
 876                                 cipher_ctx.ciph_conf = 0;
 877                                 cipher_ctx.unit_no = src_des;
 878                                 cipher_ctx.blocklength = DES_BLOCK_LENGTH;
 879                                 break;
 880                         case cryptocop_alg_3des:
 881                                 cipher_ctx.ciph_conf = 1;
 882                                 cipher_ctx.unit_no = src_des;
 883                                 cipher_ctx.blocklength = DES_BLOCK_LENGTH;
 884                                 break;
 885                         case cryptocop_alg_aes:
 886                                 cipher_ctx.ciph_conf = 2;
 887                                 cipher_ctx.unit_no = src_aes;
 888                                 cipher_ctx.blocklength = AES_BLOCK_LENGTH;
 889                                 break;
 890                         default:
 891                                 panic("cryptocop_setup_dma_list: impossible algorithm %d\n", tctx->init.alg);
 892                         }
 893                         (*int_op)->tdes_mode = tctx->init.tdes_mode;
 894                         break;
 895                 case cryptocop_alg_md5:
 896                 case cryptocop_alg_sha1:
 897                         /* digest */
 898                         if (digest_ctx.tcfg != NULL){
 899                                 DEBUG_API(printk("cryptocop_setup_dma_list: multiple digests in operation.\n"));
 900                                 failed = -EINVAL;
 901                                 goto error_cleanup;
 902                         }
 903                         digest_ctx.tcfg = tcfg;
 904                         digest_ctx.tctx = tctx;
 905                         digest_ctx.hash_mode = 0; /* Don't use explicit IV in this API. */
 906                         switch (tctx->init.alg){
 907                         case cryptocop_alg_md5:
 908                                 digest_ctx.blocklength = MD5_BLOCK_LENGTH;
 909                                 digest_ctx.unit_no = src_md5;
 910                                 digest_ctx.hash_conf = 1; /* 1 => MD-5 */
 911                                 break;
 912                         case cryptocop_alg_sha1:
 913                                 digest_ctx.blocklength = SHA1_BLOCK_LENGTH;
 914                                 digest_ctx.unit_no = src_sha1;
 915                                 digest_ctx.hash_conf = 0; /* 0 => SHA-1 */
 916                                 break;
 917                         default:
 918                                 panic("cryptocop_setup_dma_list: impossible digest algorithm\n");
 919                         }
 920                         break;
 921                 case cryptocop_alg_csum:
 922                         /* digest */
 923                         if (csum_ctx.tcfg != NULL){
 924                                 DEBUG_API(printk("cryptocop_setup_dma_list: multiple checksums in operation.\n"));
 925                                 failed = -EINVAL;
 926                                 goto error_cleanup;
 927                         }
 928                         (*int_op)->csum_mode = tctx->init.csum_mode;
 929                         csum_ctx.tcfg = tcfg;
 930                         csum_ctx.tctx = tctx;
 931                         break;
 932                 default:
 933                         /* no algorithm. */
 934                         DEBUG_API(printk("cryptocop_setup_dma_list: invalid algorithm %d specified in tfrm %d.\n", tctx->init.alg, tcfg->tid));
 935                         failed = -EINVAL;
 936                         goto error_cleanup;
 937                 }
 938                 tcfg = tcfg->next;
 939         }
 940         /* Download key if a cipher is used. */
 941         if (cipher_ctx.tcfg && (cipher_ctx.tctx->init.alg != cryptocop_alg_mem2mem)){
 942                 struct cryptocop_dma_desc  *key_desc = NULL;
 943
 944                 failed = setup_key_dl_desc(&cipher_ctx, &key_desc, alloc_flag);
 945                 if (failed) {
 946                         DEBUG_API(printk("cryptocop_setup_dma_list: setup key dl\n"));
 947                         goto error_cleanup;
 948                 }
 949                 current_out_cdesc->next = key_desc;
 950                 current_out_cdesc = key_desc;
 951                 indata_ix += (unsigned int)(key_desc->dma_descr->after - key_desc->dma_descr->buf);
 952
 953                 /* Download explicit IV if a cipher is used and CBC mode and explicit IV selected. */
 954                 if ((cipher_ctx.tctx->init.cipher_mode == cryptocop_cipher_mode_cbc) && (cipher_ctx.tcfg->flags & CRYPTOCOP_EXPLICIT_IV)) {
 955                         struct cryptocop_dma_desc  *iv_desc = NULL;
 956
 957                         DEBUG(printk("cryptocop_setup_dma_list: setup cipher CBC IV descriptor.\n"));
 958
 959                         failed = setup_cipher_iv_desc(&cipher_ctx, &iv_desc, alloc_flag);
 960                         if (failed) {
 961                                 DEBUG_API(printk("cryptocop_setup_dma_list: CBC IV descriptor.\n"));
 962                                 goto error_cleanup;
 963                         }
 964                         current_out_cdesc->next = iv_desc;
 965                         current_out_cdesc = iv_desc;
 966                         indata_ix += (unsigned int)(iv_desc->dma_descr->after - iv_desc->dma_descr->buf);
 967                 }
 968         }
 969
 970         /* Process descriptors. */
 971         odsc = operation->tfrm_op.desc;
 972         while (odsc) {
 973                 struct cryptocop_desc_cfg   *dcfg = odsc->cfg;
 974                 struct strcop_meta_out      meta_out = {0};
 975                 size_t                      desc_len = odsc->length;
 976                 int                         active_count, eop_needed_count;
 977
 978                 output_tc = NULL;
 979
 980                 DEBUG(printk("cryptocop_setup_dma_list: parsing an operation descriptor\n"));
 981
 982                 while (dcfg) {
 983                         struct cryptocop_tfrm_ctx  *tc = NULL;
 984
 985                         DEBUG(printk("cryptocop_setup_dma_list: parsing an operation descriptor configuration.\n"));
 986                         /* Get the local context for the transform and mark it as the output unit if it produces output. */
 987                         if (digest_ctx.tcfg && (digest_ctx.tcfg->tid == dcfg->tid)){
 988                                 tc = &digest_ctx;
 989                         } else if (cipher_ctx.tcfg && (cipher_ctx.tcfg->tid == dcfg->tid)){
 990                                 tc = &cipher_ctx;
 991                         } else if (csum_ctx.tcfg && (csum_ctx.tcfg->tid == dcfg->tid)){
 992                                 tc = &csum_ctx;
 993                         }
 994                         if (!tc) {
 995                                 DEBUG_API(printk("cryptocop_setup_dma_list: invalid transform %d specified in descriptor.\n", dcfg->tid));
 996                                 failed = -EINVAL;
 997                                 goto error_cleanup;
 998                         }
 999                         if (tc->done) {
1000                                 DEBUG_API(printk("cryptocop_setup_dma_list: completed transform %d reused.\n", dcfg->tid));
1001                                 failed = -EINVAL;
1002                                 goto error_cleanup;
1003                         }
1004                         if (!tc->active) {
1005                                 tc->start_ix = indata_ix;
1006                                 tc->active = 1;
1007                         }
1008
1009                         tc->previous_src = tc->current_src;
1010                         tc->prev_src = tc->curr_src;
1011                         /* Map source unit id to DMA source config. */
1012                         switch (dcfg->src){
1013                         case cryptocop_source_dma:
1014                                 tc->current_src = src_dma;
1015                                 break;
1016                         case cryptocop_source_des:
1017                                 tc->current_src = src_des;
1018                                 break;
1019                         case cryptocop_source_3des:
1020                                 tc->current_src = src_des;
1021                                 break;
1022                         case cryptocop_source_aes:
1023                                 tc->current_src = src_aes;
1024                                 break;
1025                         case cryptocop_source_md5:
1026                         case cryptocop_source_sha1:
1027                         case cryptocop_source_csum:
1028                         case cryptocop_source_none:
1029                         default:
1030                                 /* We do not allow using accumulating style units (SHA-1, MD5, checksum) as sources to other units.
1031                                  */
1032                                 DEBUG_API(printk("cryptocop_setup_dma_list: bad unit source configured %d.\n", dcfg->src));
1033                                 failed = -EINVAL;
1034                                 goto error_cleanup;
1035                         }
1036                         if (tc->current_src != src_dma) {
1037                                 /* Find the unit we are sourcing from. */
1038                                 if (digest_ctx.unit_no == tc->current_src){
1039                                         tc->curr_src = &digest_ctx;
1040                                 } else if (cipher_ctx.unit_no == tc->current_src){
1041                                         tc->curr_src = &cipher_ctx;
1042                                 } else if (csum_ctx.unit_no == tc->current_src){
1043                                         tc->curr_src = &csum_ctx;
1044                                 }
1045                                 if ((tc->curr_src == tc) && (tc->unit_no != src_dma)){
1046                                         DEBUG_API(printk("cryptocop_setup_dma_list: unit %d configured to source from itself.\n", tc->unit_no));
1047                                         failed = -EINVAL;
1048                                         goto error_cleanup;
1049                                 }
1050                         } else {
1051                                 tc->curr_src = NULL;
1052                         }
1053
1054                         /* Detect source switch. */
1055                         DEBUG(printk("cryptocop_setup_dma_list: tc->active=%d tc->unit_no=%d tc->current_src=%d tc->previous_src=%d, tc->curr_src=0x%p, tc->prev_srv=0x%p\n", tc->active, tc->unit_no, tc->current_src, tc->previous_src, tc->curr_src, tc->prev_src));
1056                         if (tc->active && (tc->current_src != tc->previous_src)) {
1057                                 /* Only allow source switch when both the old source unit and the new one have
1058                                  * no pending data to process (i.e. the consumed length must be a multiple of the
1059                                  * transform blocklength). */
1060                                 /* Note: if the src == NULL we are actually sourcing from DMA out. */
1061                                 if (((tc->prev_src != NULL) && (tc->prev_src->consumed % tc->prev_src->blocklength)) ||
1062                                     ((tc->curr_src != NULL) && (tc->curr_src->consumed % tc->curr_src->blocklength)))
1063                                 {
1064                                         DEBUG_API(printk("cryptocop_setup_dma_list: can only disconnect from or connect to a unit on a multiple of the blocklength, old: cons=%d, prod=%d, block=%d, new: cons=%d prod=%d, block=%d.\n", tc->prev_src ? tc->prev_src->consumed : INT_MIN, tc->prev_src ? tc->prev_src->produced : INT_MIN, tc->prev_src ? tc->prev_src->blocklength : INT_MIN, tc->curr_src ? tc->curr_src->consumed : INT_MIN, tc->curr_src ? tc->curr_src->produced : INT_MIN, tc->curr_src ? tc->curr_src->blocklength : INT_MIN));
1065                                         failed = -EINVAL;
1066                                         goto error_cleanup;
1067                                 }
1068                         }
1069                         /* Detect unit deactivation. */
1070                         if (dcfg->last) {
1071                                 /* Length check of this is handled below. */
1072                                 tc->done = 1;
1073                         }
1074                         dcfg = dcfg->next;
1075                 } /* while (dcfg) */
1076                 DEBUG(printk("cryptocop_setup_dma_list: parsing operation descriptor configuration complete.\n"));
1077
1078                 if (cipher_ctx.active && (cipher_ctx.curr_src != NULL) && !cipher_ctx.curr_src->active){
1079                         DEBUG_API(printk("cryptocop_setup_dma_list: cipher source from inactive unit %d\n", cipher_ctx.curr_src->unit_no));
1080                         failed = -EINVAL;
1081                         goto error_cleanup;
1082                 }
1083                 if (digest_ctx.active && (digest_ctx.curr_src != NULL) && !digest_ctx.curr_src->active){
1084                         DEBUG_API(printk("cryptocop_setup_dma_list: digest source from inactive unit %d\n", digest_ctx.curr_src->unit_no));
1085                         failed = -EINVAL;
1086                         goto error_cleanup;
1087                 }
1088                 if (csum_ctx.active && (csum_ctx.curr_src != NULL) && !csum_ctx.curr_src->active){
1089                         DEBUG_API(printk("cryptocop_setup_dma_list: cipher source from inactive unit %d\n", csum_ctx.curr_src->unit_no));
1090                         failed = -EINVAL;
1091                         goto error_cleanup;
1092                 }
1093
1094                 /* Update consumed and produced lengths.
1095
1096                    The consumed length accounting here is actually cheating.  If a unit source from DMA (or any
1097                    other unit that process data in blocks of one octet) it is correct, but if it source from a
1098                    block processing unit, i.e. a cipher, it will be temporarily incorrect at some times.  However
1099                    since it is only allowed--by the HW--to change source to or from a block processing unit at times where that
1100                    unit has processed an exact multiple of its block length the end result will be correct.
1101                    Beware that if the source change restriction change this code will need to be (much) reworked.
1102                 */
1103                 DEBUG(printk("cryptocop_setup_dma_list: desc->length=%d, desc_len=%d.\n", odsc->length, desc_len));
1104
1105                 if (csum_ctx.active) {
1106                         csum_ctx.consumed += desc_len;
1107                         if (csum_ctx.done) {
1108                                 csum_ctx.produced = 2;
1109                         }
1110                         DEBUG(printk("cryptocop_setup_dma_list: csum_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", csum_ctx.consumed, csum_ctx.produced, csum_ctx.blocklength));
1111                 }
1112                 if (digest_ctx.active) {
1113                         digest_ctx.consumed += desc_len;
1114                         if (digest_ctx.done) {
1115                                 if (digest_ctx.unit_no == src_md5) {
1116                                         digest_ctx.produced = MD5_STATE_LENGTH;
1117                                 } else {
1118                                         digest_ctx.produced = SHA1_STATE_LENGTH;
1119                                 }
1120                         }
1121                         DEBUG(printk("cryptocop_setup_dma_list: digest_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", digest_ctx.consumed, digest_ctx.produced, digest_ctx.blocklength));
1122                 }
1123                 if (cipher_ctx.active) {
1124                         /* Ciphers are allowed only to source from DMA out.  That is filtered above. */
1125                         assert(cipher_ctx.current_src == src_dma);
1126                         cipher_ctx.consumed += desc_len;
1127                         cipher_ctx.produced = cipher_ctx.blocklength * (cipher_ctx.consumed / cipher_ctx.blocklength);
1128                         if (cipher_ctx.cbcmode && !(cipher_ctx.tcfg->flags & CRYPTOCOP_EXPLICIT_IV) && cipher_ctx.produced){
1129                                 cipher_ctx.produced -= cipher_ctx.blocklength; /* Compensate for CBC iv. */
1130                         }
1131                         DEBUG(printk("cryptocop_setup_dma_list: cipher_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", cipher_ctx.consumed, cipher_ctx.produced, cipher_ctx.blocklength));
1132                 }
1133
1134                 /* Setup the DMA out descriptors. */
1135                 /* Configure the metadata. */
1136                 active_count = 0;
1137                 eop_needed_count = 0;
1138                 if (cipher_ctx.active) {
1139                         ++active_count;
1140                         if (cipher_ctx.unit_no == src_dma){
1141                                 /* mem2mem */
1142                                 meta_out.ciphsel = src_none;
1143                         } else {
1144                                 meta_out.ciphsel = cipher_ctx.current_src;
1145                         }
1146                         meta_out.ciphconf = cipher_ctx.ciph_conf;
1147                         meta_out.cbcmode = cipher_ctx.cbcmode;
1148                         meta_out.decrypt = cipher_ctx.decrypt;
1149                         DEBUG(printk("set ciphsel=%d ciphconf=%d cbcmode=%d decrypt=%d\n", meta_out.ciphsel, meta_out.ciphconf, meta_out.cbcmode, meta_out.decrypt));
1150                         if (cipher_ctx.done) ++eop_needed_count;
1151                 } else {
1152                         meta_out.ciphsel = src_none;
1153                 }
1154
1155                 if (digest_ctx.active) {
1156                         ++active_count;
1157                         meta_out.hashsel = digest_ctx.current_src;
1158                         meta_out.hashconf = digest_ctx.hash_conf;
1159                         meta_out.hashmode = 0; /* Explicit mode is not used here. */
1160                         DEBUG(printk("set hashsel=%d hashconf=%d hashmode=%d\n", meta_out.hashsel, meta_out.hashconf, meta_out.hashmode));
1161                         if (digest_ctx.done) {
1162                                 assert(digest_ctx.pad_descs == NULL);
1163                                 failed = create_pad_descriptor(&digest_ctx, &digest_ctx.pad_descs, alloc_flag);
1164                                 if (failed) {
1165                                         DEBUG_API(printk("cryptocop_setup_dma_list: failed digest pad creation.\n"));
1166                                         goto error_cleanup;
1167                                 }
1168                         }
1169                 } else {
1170                         meta_out.hashsel = src_none;
1171                 }
1172
1173                 if (csum_ctx.active) {
1174                         ++active_count;
1175                         meta_out.csumsel = csum_ctx.current_src;
1176                         if (csum_ctx.done) {
1177                                 assert(csum_ctx.pad_descs == NULL);
1178                                 failed = create_pad_descriptor(&csum_ctx, &csum_ctx.pad_descs, alloc_flag);
1179                                 if (failed) {
1180                                         DEBUG_API(printk("cryptocop_setup_dma_list: failed csum pad creation.\n"));
1181                                         goto error_cleanup;
1182                                 }
1183                         }
1184                 } else {
1185                         meta_out.csumsel = src_none;
1186                 }
1187                 DEBUG(printk("cryptocop_setup_dma_list: %d eop needed, %d active units\n", eop_needed_count, active_count));
1188                 /* Setup DMA out descriptors for the indata. */
1189                 failed = create_output_descriptors(operation, &iniov_ix, &iniov_offset, desc_len, &current_out_cdesc, &meta_out, alloc_flag);
1190                 if (failed) {
1191                         DEBUG_API(printk("cryptocop_setup_dma_list: create_output_descriptors %d\n", failed));
1192                         goto error_cleanup;
1193                 }
1194                 /* Setup out EOP.  If there are active units that are not done here they cannot get an EOP
1195                  * so we ust setup a zero length descriptor to DMA to signal EOP only to done units.
1196                  * If there is a pad descriptor EOP for the padded unit will be EOPed by it.
1197                  */
1198                 assert(active_count >= eop_needed_count);
1199                 assert((eop_needed_count == 0) || (eop_needed_count == 1));
1200                 if (eop_needed_count) {
1201                         /* This means that the bulk operation (cipeher/m2m) is terminated. */
1202                         if (active_count > 1) {
1203                                 /* Use zero length EOP descriptor. */
1204                                 struct cryptocop_dma_desc *ed = alloc_cdesc(alloc_flag);
1205                                 struct strcop_meta_out    ed_mo = {0};
1206                                 if (!ed) {
1207                                         DEBUG_API(printk("cryptocop_setup_dma_list: alloc EOP descriptor for cipher\n"));
1208                                         failed = -ENOMEM;
1209                                         goto error_cleanup;
1210                                 }
1211
1212                                 assert(cipher_ctx.active && cipher_ctx.done);
1213
1214                                 if (cipher_ctx.unit_no == src_dma){
1215                                         /* mem2mem */
1216                                         ed_mo.ciphsel = src_none;
1217                                 } else {
1218                                         ed_mo.ciphsel = cipher_ctx.current_src;
1219                                 }
1220                                 ed_mo.ciphconf = cipher_ctx.ciph_conf;
1221                                 ed_mo.cbcmode = cipher_ctx.cbcmode;
1222                                 ed_mo.decrypt = cipher_ctx.decrypt;
1223
1224                                 ed->free_buf = NULL;
1225                                 ed->dma_descr->wait = 1;
1226                                 ed->dma_descr->out_eop = 1;
1227
1228                                 ed->dma_descr->buf = (char*)virt_to_phys(&ed); /* Use any valid physical address for zero length descriptor. */
1229                                 ed->dma_descr->after = ed->dma_descr->buf;
1230                                 ed->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, ed_mo);
1231                                 current_out_cdesc->next = ed;
1232                                 current_out_cdesc = ed;
1233                         } else {
1234                                 /* Set EOP in the current out descriptor since the only active module is
1235                                  * the one needing the EOP. */
1236
1237                                 current_out_cdesc->dma_descr->out_eop = 1;
1238                         }
1239                 }
1240
1241                 if (cipher_ctx.done && cipher_ctx.active) cipher_ctx.active = 0;
1242                 if (digest_ctx.done && digest_ctx.active) digest_ctx.active = 0;
1243                 if (csum_ctx.done && csum_ctx.active) csum_ctx.active = 0;
1244                 indata_ix += odsc->length;
1245                 odsc = odsc->next;
1246         } /* while (odsc) */ /* Process descriptors. */
1247         DEBUG(printk("cryptocop_setup_dma_list: done parsing operation descriptors\n"));
1248         if (cipher_ctx.tcfg && (cipher_ctx.active || !cipher_ctx.done)){
1249                 DEBUG_API(printk("cryptocop_setup_dma_list: cipher operation not terminated.\n"));
1250                 failed = -EINVAL;
1251                 goto error_cleanup;
1252         }
1253         if (digest_ctx.tcfg && (digest_ctx.active || !digest_ctx.done)){
1254                 DEBUG_API(printk("cryptocop_setup_dma_list: digest operation not terminated.\n"));
1255                 failed = -EINVAL;
1256                 goto error_cleanup;
1257         }
1258         if (csum_ctx.tcfg && (csum_ctx.active || !csum_ctx.done)){
1259                 DEBUG_API(printk("cryptocop_setup_dma_list: csum operation not terminated.\n"));
1260                 failed = -EINVAL;
1261                 goto error_cleanup;
1262         }
1263
1264         failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &cipher_ctx, alloc_flag);
1265         if (failed){
1266                 DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1267                 goto error_cleanup;
1268         }
1269         failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &digest_ctx, alloc_flag);
1270         if (failed){
1271                 DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1272                 goto error_cleanup;
1273         }
1274         failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &csum_ctx, alloc_flag);
1275         if (failed){
1276                 DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1277                 goto error_cleanup;
1278         }
1279
1280         DEBUG(printk("cryptocop_setup_dma_list: int_op=0x%p, *int_op=0x%p\n", int_op, *int_op));
1281         (*int_op)->cdesc_out = out_cdesc_head.next;
1282         (*int_op)->cdesc_in = in_cdesc_head.next;
1283         DEBUG(printk("cryptocop_setup_dma_list: out_cdesc_head=0x%p in_cdesc_head=0x%p\n", (*int_op)->cdesc_out, (*int_op)->cdesc_in));
1284
1285         setup_descr_chain(out_cdesc_head.next);
1286         setup_descr_chain(in_cdesc_head.next);
1287
1288         /* Last but not least: mark the last DMA in descriptor for a INTR and EOL and the the
1289          * last DMA out descriptor for EOL.
1290          */
1291         current_in_cdesc->dma_descr->intr = 1;
1292         current_in_cdesc->dma_descr->eol = 1;
1293         current_out_cdesc->dma_descr->eol = 1;
1294
1295         /* Setup DMA contexts. */
1296         (*int_op)->ctx_out.next = NULL;
1297         (*int_op)->ctx_out.eol = 1;
1298         (*int_op)->ctx_out.intr = 0;
1299         (*int_op)->ctx_out.store_mode = 0;
1300         (*int_op)->ctx_out.en = 0;
1301         (*int_op)->ctx_out.dis = 0;
1302         (*int_op)->ctx_out.md0 = 0;
1303         (*int_op)->ctx_out.md1 = 0;
1304         (*int_op)->ctx_out.md2 = 0;
1305         (*int_op)->ctx_out.md3 = 0;
1306         (*int_op)->ctx_out.md4 = 0;
1307         (*int_op)->ctx_out.saved_data = (dma_descr_data*)virt_to_phys((*int_op)->cdesc_out->dma_descr);
1308         (*int_op)->ctx_out.saved_data_buf = (*int_op)->cdesc_out->dma_descr->buf; /* Already physical address. */
1309
1310         (*int_op)->ctx_in.next = NULL;
1311         (*int_op)->ctx_in.eol = 1;
1312         (*int_op)->ctx_in.intr = 0;
1313         (*int_op)->ctx_in.store_mode = 0;
1314         (*int_op)->ctx_in.en = 0;
1315         (*int_op)->ctx_in.dis = 0;
1316         (*int_op)->ctx_in.md0 = 0;
1317         (*int_op)->ctx_in.md1 = 0;
1318         (*int_op)->ctx_in.md2 = 0;
1319         (*int_op)->ctx_in.md3 = 0;
1320         (*int_op)->ctx_in.md4 = 0;
1321
1322         (*int_op)->ctx_in.saved_data = (dma_descr_data*)virt_to_phys((*int_op)->cdesc_in->dma_descr);
1323         (*int_op)->ctx_in.saved_data_buf = (*int_op)->cdesc_in->dma_descr->buf; /* Already physical address. */
1324
1325         DEBUG(printk("cryptocop_setup_dma_list: done\n"));
1326         return 0;
1327
1328 error_cleanup:
1329         {
1330                 /* Free all allocated resources. */
1331                 struct cryptocop_dma_desc *tmp_cdesc;
1332                 while (digest_ctx.pad_descs){
1333                         tmp_cdesc = digest_ctx.pad_descs->next;
1334                         free_cdesc(digest_ctx.pad_descs);
1335                         digest_ctx.pad_descs = tmp_cdesc;
1336                 }
1337                 while (csum_ctx.pad_descs){
1338                         tmp_cdesc = csum_ctx.pad_descs->next;
1339                         free_cdesc(csum_ctx.pad_descs);
1340                         csum_ctx.pad_descs = tmp_cdesc;
1341                 }
1342                 assert(cipher_ctx.pad_descs == NULL); /* The ciphers are never padded. */
1343
1344                 if (*int_op != NULL) delete_internal_operation(*int_op);
1345         }
1346         DEBUG_API(printk("cryptocop_setup_dma_list: done with error %d\n", failed));
1347         return failed;
1348 }
1349
1350
1351 static void delete_internal_operation(struct cryptocop_int_operation *iop)
1352 {
1353         void                      *ptr = iop->alloc_ptr;
1354         struct cryptocop_dma_desc *cd = iop->cdesc_out;
1355         struct cryptocop_dma_desc *next;
1356
1357         DEBUG(printk("delete_internal_operation: iop=0x%p, alloc_ptr=0x%p\n", iop, ptr));
1358
1359         while (cd) {
1360                 next = cd->next;
1361                 free_cdesc(cd);
1362                 cd = next;
1363         }
1364         cd = iop->cdesc_in;
1365         while (cd) {
1366                 next = cd->next;
1367                 free_cdesc(cd);
1368                 cd = next;
1369         }
1370         kfree(ptr);
1371 }
1372
1373 #define MD5_MIN_PAD_LENGTH (9)
1374 #define MD5_PAD_LENGTH_FIELD_LENGTH (8)
1375
1376 static int create_md5_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length)
1377 {
1378         size_t                  padlen = MD5_BLOCK_LENGTH - (hashed_length % MD5_BLOCK_LENGTH);
1379         unsigned char           *p;
1380         int                     i;
1381         unsigned long long int  bit_length = hashed_length << 3;
1382
1383         if (padlen < MD5_MIN_PAD_LENGTH) padlen += MD5_BLOCK_LENGTH;
1384
1385         p = kmalloc(padlen, alloc_flag);
1386         if (!pad) return -ENOMEM;
1387
1388         *p = 0x80;
1389         memset(p+1, 0, padlen - 1);
1390
1391         DEBUG(printk("create_md5_pad: hashed_length=%lld bits == %lld bytes\n", bit_length, hashed_length));
1392
1393         i = padlen - MD5_PAD_LENGTH_FIELD_LENGTH;
1394         while (bit_length != 0){
1395                 p[i++] = bit_length % 0x100;
1396                 bit_length >>= 8;
1397         }
1398
1399         *pad = (char*)p;
1400         *pad_length = padlen;
1401
1402         return 0;
1403 }
1404
1405 #define SHA1_MIN_PAD_LENGTH (9)
1406 #define SHA1_PAD_LENGTH_FIELD_LENGTH (8)
1407
1408 static int create_sha1_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length)
1409 {
1410         size_t                  padlen = SHA1_BLOCK_LENGTH - (hashed_length % SHA1_BLOCK_LENGTH);
1411         unsigned char           *p;
1412         int                     i;
1413         unsigned long long int  bit_length = hashed_length << 3;
1414
1415         if (padlen < SHA1_MIN_PAD_LENGTH) padlen += SHA1_BLOCK_LENGTH;
1416
1417         p = kmalloc(padlen, alloc_flag);
1418         if (!pad) return -ENOMEM;
1419
1420         *p = 0x80;
1421         memset(p+1, 0, padlen - 1);
1422
1423         DEBUG(printk("create_sha1_pad: hashed_length=%lld bits == %lld bytes\n", bit_length, hashed_length));
1424
1425         i = padlen - 1;
1426         while (bit_length != 0){
1427                 p[i--] = bit_length % 0x100;
1428                 bit_length >>= 8;
1429         }
1430
1431         *pad = (char*)p;
1432         *pad_length = padlen;
1433
1434         return 0;
1435 }
1436
1437
1438 static int transform_ok(struct cryptocop_transform_init *tinit)
1439 {
1440         switch (tinit->alg){
1441         case cryptocop_alg_csum:
1442                 switch (tinit->csum_mode){
1443                 case cryptocop_csum_le:
1444                 case cryptocop_csum_be:
1445                         break;
1446                 default:
1447                         DEBUG_API(printk("transform_ok: Bad mode set for csum transform\n"));
1448                         return -EINVAL;
1449                 }
1450         case cryptocop_alg_mem2mem:
1451         case cryptocop_alg_md5:
1452         case cryptocop_alg_sha1:
1453                 if (tinit->keylen != 0) {
1454                         DEBUG_API(printk("transform_ok: non-zero keylength, %d, for a digest/csum algorithm\n", tinit->keylen));
1455                         return -EINVAL; /* This check is a bit strict. */
1456                 }
1457                 break;
1458         case cryptocop_alg_des:
1459                 if (tinit->keylen != 64) {
1460                         DEBUG_API(printk("transform_ok: keylen %d invalid for DES\n", tinit->keylen));
1461                         return -EINVAL;
1462                 }
1463                 break;
1464         case cryptocop_alg_3des:
1465                 if (tinit->keylen != 192) {
1466                         DEBUG_API(printk("transform_ok: keylen %d invalid for 3DES\n", tinit->keylen));
1467                         return -EINVAL;
1468                 }
1469                 break;
1470         case cryptocop_alg_aes:
1471                 if (tinit->keylen != 128 && tinit->keylen != 192 && tinit->keylen != 256) {
1472                         DEBUG_API(printk("transform_ok: keylen %d invalid for AES\n", tinit->keylen));
1473                         return -EINVAL;
1474                 }
1475                 break;
1476         case cryptocop_no_alg:
1477         default:
1478                 DEBUG_API(printk("transform_ok: no such algorithm %d\n", tinit->alg));
1479                 return -EINVAL;
1480         }
1481
1482         switch (tinit->alg){
1483         case cryptocop_alg_des:
1484         case cryptocop_alg_3des:
1485         case cryptocop_alg_aes:
1486                 if (tinit->cipher_mode != cryptocop_cipher_mode_ecb && tinit->cipher_mode != cryptocop_cipher_mode_cbc) return -EINVAL;
1487         default:
1488                  break;
1489         }
1490         return 0;
1491 }
1492
1493
1494 int cryptocop_new_session(cryptocop_session_id *sid, struct cryptocop_transform_init *tinit, int alloc_flag)
1495 {
1496         struct cryptocop_session         *sess;
1497         struct cryptocop_transform_init  *tfrm_in = tinit;
1498         struct cryptocop_transform_init  *tmp_in;
1499         int                              no_tfrms = 0;
1500         int                              i;
1501         unsigned long int                flags;
1502
1503         init_stream_coprocessor(); /* For safety if we are called early */
1504
1505         while (tfrm_in){
1506                 int err;
1507                 ++no_tfrms;
1508                 if ((err = transform_ok(tfrm_in))) {
1509                         DEBUG_API(printk("cryptocop_new_session, bad transform\n"));
1510                         return err;
1511                 }
1512                 tfrm_in = tfrm_in->next;
1513         }
1514         if (0 == no_tfrms) {
1515                 DEBUG_API(printk("cryptocop_new_session, no transforms specified\n"));
1516                 return -EINVAL;
1517         }
1518
1519         sess = kmalloc(sizeof(struct cryptocop_session), alloc_flag);
1520         if (!sess){
1521                 DEBUG_API(printk("cryptocop_new_session, kmalloc cryptocop_session\n"));
1522                 return -ENOMEM;
1523         }
1524
1525         sess->tfrm_ctx = kmalloc(no_tfrms * sizeof(struct cryptocop_transform_ctx), alloc_flag);
1526         if (!sess->tfrm_ctx) {
1527                 DEBUG_API(printk("cryptocop_new_session, kmalloc cryptocop_transform_ctx\n"));
1528                 kfree(sess);
1529                 return -ENOMEM;
1530         }
1531
1532         tfrm_in = tinit;
1533         for (i = 0; i < no_tfrms; i++){
1534                 tmp_in = tfrm_in->next;
1535                 while (tmp_in){
1536                         if (tmp_in->tid == tfrm_in->tid) {
1537                                 DEBUG_API(printk("cryptocop_new_session, duplicate transform ids\n"));
1538                                 kfree(sess->tfrm_ctx);
1539                                 kfree(sess);
1540                                 return -EINVAL;
1541                         }
1542                         tmp_in = tmp_in->next;
1543                 }
1544                 memcpy(&sess->tfrm_ctx[i].init, tfrm_in, sizeof(struct cryptocop_transform_init));
1545                 sess->tfrm_ctx[i].dec_key_set = 0;
1546                 sess->tfrm_ctx[i].next = &sess->tfrm_ctx[i] + 1;
1547
1548                 tfrm_in = tfrm_in->next;
1549         }
1550         sess->tfrm_ctx[i-1].next = NULL;
1551
1552         spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1553         sess->sid = next_sid;
1554         next_sid++;
1555         /* TODO If we are really paranoid we should do duplicate check to handle sid wraparound.
1556          *      OTOH 2^64 is a really large number of session. */
1557         if (next_sid == 0) next_sid = 1;
1558
1559         /* Prepend to session list. */
1560         sess->next = cryptocop_sessions;
1561         cryptocop_sessions = sess;
1562         spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1563         *sid = sess->sid;
1564         return 0;
1565 }
1566
1567
1568 int cryptocop_free_session(cryptocop_session_id sid)
1569 {
1570         struct cryptocop_transform_ctx    *tc;
1571         struct cryptocop_session          *sess = NULL;
1572         struct cryptocop_session          *psess = NULL;
1573         unsigned long int                 flags;
1574         int                               i;
1575         LIST_HEAD(remove_list);
1576         struct list_head                  *node, *tmp;
1577         struct cryptocop_prio_job         *pj;
1578
1579         DEBUG(printk("cryptocop_free_session: sid=%lld\n", sid));
1580
1581         spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1582         sess = cryptocop_sessions;
1583         while (sess && sess->sid != sid){
1584                 psess = sess;
1585                 sess = sess->next;
1586         }
1587         if (sess){
1588                 if (psess){
1589                         psess->next = sess->next;
1590                 } else {
1591                         cryptocop_sessions = sess->next;
1592                 }
1593         }
1594         spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1595
1596         if (!sess) return -EINVAL;
1597
1598         /* Remove queued jobs. */
1599         spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
1600
1601         for (i = 0; i < cryptocop_prio_no_prios; i++){
1602                 if (!list_empty(&(cryptocop_job_queues[i].jobs))){
1603                         list_for_each_safe(node, tmp, &(cryptocop_job_queues[i].jobs)) {
1604                                 pj = list_entry(node, struct cryptocop_prio_job, node);
1605                                 if (pj->oper->sid == sid) {
1606                                         list_move_tail(node, &remove_list);
1607                                 }
1608                         }
1609                 }
1610         }
1611         spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
1612
1613         list_for_each_safe(node, tmp, &remove_list) {
1614                 list_del(node);
1615                 pj = list_entry(node, struct cryptocop_prio_job, node);
1616                 pj->oper->operation_status = -EAGAIN;  /* EAGAIN is not ideal for job/session terminated but it's the best choice I know of. */
1617                 DEBUG(printk("cryptocop_free_session: pj=0x%p, pj->oper=0x%p, pj->iop=0x%p\n", pj, pj->oper, pj->iop));
1618                 pj->oper->cb(pj->oper, pj->oper->cb_data);
1619                 delete_internal_operation(pj->iop);
1620                 kfree(pj);
1621         }
1622
1623         tc = sess->tfrm_ctx;
1624         /* Erase keying data. */
1625         while (tc){
1626                 DEBUG(printk("cryptocop_free_session: memset keys, tfrm id=%d\n", tc->init.tid));
1627                 memset(tc->init.key, 0xff, CRYPTOCOP_MAX_KEY_LENGTH);
1628                 memset(tc->dec_key, 0xff, CRYPTOCOP_MAX_KEY_LENGTH);
1629                 tc = tc->next;
1630         }
1631         kfree(sess->tfrm_ctx);
1632         kfree(sess);
1633
1634         return 0;
1635 }
1636
1637 static struct cryptocop_session *get_session(cryptocop_session_id sid)
1638 {
1639         struct cryptocop_session    *sess;
1640         unsigned long int           flags;
1641
1642         spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1643         sess = cryptocop_sessions;
1644         while (sess && (sess->sid != sid)){
1645                 sess = sess->next;
1646         }
1647         spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1648
1649         return sess;
1650 }
1651
1652 static struct cryptocop_transform_ctx *get_transform_ctx(struct cryptocop_session *sess, cryptocop_tfrm_id tid)
1653 {
1654         struct cryptocop_transform_ctx *tc = sess->tfrm_ctx;
1655
1656         DEBUG(printk("get_transform_ctx, sess=0x%p, tid=%d\n", sess, tid));
1657         assert(sess != NULL);
1658         while (tc && tc->init.tid != tid){
1659                 DEBUG(printk("tc=0x%p, tc->next=0x%p\n", tc, tc->next));
1660                 tc = tc->next;
1661         }
1662         DEBUG(printk("get_transform_ctx, returning tc=0x%p\n", tc));
1663         return tc;
1664 }
1665
1666
1667
1668 /* The AES s-transform matrix (s-box). */
1669 static const u8 aes_sbox[256] = {
1670         99,  124, 119, 123, 242, 107, 111, 197, 48,  1,   103, 43,  254, 215, 171, 118,
1671         202, 130, 201, 125, 250, 89,  71,  240, 173, 212, 162, 175, 156, 164, 114, 192,
1672         183, 253, 147, 38,  54,  63,  247, 204, 52,  165, 229, 241, 113, 216, 49,  21,
1673         4,   199, 35,  195, 24,  150, 5,   154, 7,   18,  128, 226, 235, 39,  178, 117,
1674         9,   131, 44,  26,  27,  110, 90,  160, 82,  59,  214, 179, 41,  227, 47,  132,
1675         83,  209, 0,   237, 32,  252, 177, 91,  106, 203, 190, 57,  74,  76,  88,  207,
1676         208, 239, 170, 251, 67,  77,  51,  133, 69,  249, 2,   127, 80,  60,  159, 168,
1677         81,  163, 64,  143, 146, 157, 56,  245, 188, 182, 218, 33,  16,  255, 243, 210,
1678         205, 12,  19,  236, 95,  151, 68,  23,  196, 167, 126, 61,  100, 93,  25,  115,
1679         96,  129, 79,  220, 34,  42,  144, 136, 70,  238, 184, 20,  222, 94,  11,  219,
1680         224, 50,  58,  10,  73,  6,   36,  92,  194, 211, 172, 98,  145, 149, 228, 121,
1681         231, 200, 55,  109, 141, 213, 78,  169, 108, 86,  244, 234, 101, 122, 174, 8,
1682         186, 120, 37,  46,  28,  166, 180, 198, 232, 221, 116, 31,  75,  189, 139, 138,
1683         112, 62,  181, 102, 72,  3,   246, 14,  97,  53,  87,  185, 134, 193, 29,  158,
1684         225, 248, 152, 17,  105, 217, 142, 148, 155, 30,  135, 233, 206, 85,  40,  223,
1685         140, 161, 137, 13,  191, 230, 66,  104, 65,  153, 45,  15,  176, 84,  187, 22
1686 };
1687
1688 /* AES has a 32 bit word round constants for each round in the
1689  * key schedule.  round_constant[i] is really Rcon[i+1] in FIPS187.
1690  */
1691 static u32 round_constant[11] = {
1692         0x01000000, 0x02000000, 0x04000000, 0x08000000,
1693         0x10000000, 0x20000000, 0x40000000, 0x80000000,
1694         0x1B000000, 0x36000000, 0x6C000000
1695 };
1696
1697 /* Apply the s-box to each of the four occtets in w. */
1698 static u32 aes_ks_subword(const u32 w)
1699 {
1700         u8 bytes[4];
1701
1702         *(u32*)(&bytes[0]) = w;
1703         bytes[0] = aes_sbox[bytes[0]];
1704         bytes[1] = aes_sbox[bytes[1]];
1705         bytes[2] = aes_sbox[bytes[2]];
1706         bytes[3] = aes_sbox[bytes[3]];
1707         return *(u32*)(&bytes[0]);
1708 }
1709
1710 /* The encrypt (forward) Rijndael key schedule algorithm pseudo code:
1711  * (Note that AES words are 32 bit long)
1712  *
1713  * KeyExpansion(byte key[4*Nk], word w[Nb*(Nr+1)], Nk){
1714  * word temp
1715  * i = 0
1716  * while (i < Nk) {
1717  *   w[i] = word(key[4*i, 4*i + 1, 4*i + 2, 4*i + 3])
1718  *   i = i + 1
1719  * }
1720  * i = Nk
1721  *
1722  * while (i < (Nb * (Nr + 1))) {
1723  *   temp = w[i - 1]
1724  *   if ((i mod Nk) == 0) {
1725  *     temp = SubWord(RotWord(temp)) xor Rcon[i/Nk]
1726  *   }
1727  *   else if ((Nk > 6) && ((i mod Nk) == 4)) {
1728  *     temp = SubWord(temp)
1729  *   }
1730  *   w[i] = w[i - Nk] xor temp
1731  * }
1732  * RotWord(t) does a 8 bit cyclic shift left on a 32 bit word.
1733  * SubWord(t) applies the AES s-box individually to each octet
1734  * in a 32 bit word.
1735  *
1736  * For AES Nk can have the values 4, 6, and 8 (corresponding to
1737  * values for Nr of 10, 12, and 14).  Nb is always 4.
1738  *
1739  * To construct w[i], w[i - 1] and w[i - Nk] must be
1740  * available.  Consequently we must keep a state of the last Nk words
1741  * to be able to create the last round keys.
1742  */
1743 static void get_aes_decrypt_key(unsigned char *dec_key, const unsigned  char *key, unsigned int keylength)
1744 {
1745         u32 temp;
1746         u32 w_ring[8]; /* nk is max 8, use elements 0..(nk - 1) as a ringbuffer */
1747         u8  w_last_ix;
1748         int i;
1749         u8  nr, nk;
1750
1751         switch (keylength){
1752         case 128:
1753                 nk = 4;
1754                 nr = 10;
1755                 break;
1756         case 192:
1757                 nk = 6;
1758                 nr = 12;
1759                 break;
1760         case 256:
1761                 nk = 8;
1762                 nr = 14;
1763                 break;
1764         default:
1765                 panic("stream co-processor: bad aes key length in get_aes_decrypt_key\n");
1766         };
1767
1768         /* Need to do host byte order correction here since key is byte oriented and the
1769          * kx algorithm is word (u32) oriented. */
1770         for (i = 0; i < nk; i+=1) {
1771                 w_ring[i] = be32_to_cpu(*(u32*)&key[4*i]);
1772         }
1773
1774         i = (int)nk;
1775         w_last_ix = i - 1;
1776         while (i < (4 * (nr + 2))) {
1777                 temp = w_ring[w_last_ix];
1778                 if (!(i % nk)) {
1779                         /* RotWord(temp) */
1780                         temp = (temp << 8) | (temp >> 24);
1781                         temp = aes_ks_subword(temp);
1782                         temp ^= round_constant[i/nk - 1];
1783                 } else if ((nk > 6) && ((i % nk) == 4)) {
1784                         temp = aes_ks_subword(temp);
1785                 }
1786                 w_last_ix = (w_last_ix + 1) % nk; /* This is the same as (i-Nk) mod Nk */
1787                 temp ^= w_ring[w_last_ix];
1788                 w_ring[w_last_ix] = temp;
1789
1790                 /* We need the round keys for round Nr+1 and Nr+2 (round key
1791                  * Nr+2 is the round key beyond the last one used when
1792                  * encrypting).  Rounds are numbered starting from 0, Nr=10
1793                  * implies 11 rounds are used in encryption/decryption.
1794                  */
1795                 if (i >= (4 * nr)) {
1796                         /* Need to do host byte order correction here, the key
1797                          * is byte oriented. */
1798                         *(u32*)dec_key = cpu_to_be32(temp);
1799                         dec_key += 4;
1800                 }
1801                 ++i;
1802         }
1803 }
1804
1805
1806 /**** Job/operation management. ****/
1807
1808 int cryptocop_job_queue_insert_csum(struct cryptocop_operation *operation)
1809 {
1810         return cryptocop_job_queue_insert(cryptocop_prio_kernel_csum, operation);
1811 }
1812
1813 int cryptocop_job_queue_insert_crypto(struct cryptocop_operation *operation)
1814 {
1815         return cryptocop_job_queue_insert(cryptocop_prio_kernel, operation);
1816 }
1817
1818 int cryptocop_job_queue_insert_user_job(struct cryptocop_operation *operation)
1819 {
1820         return cryptocop_job_queue_insert(cryptocop_prio_user, operation);
1821 }
1822
1823 static int cryptocop_job_queue_insert(cryptocop_queue_priority prio, struct cryptocop_operation *operation)
1824 {
1825         int                           ret;
1826         struct cryptocop_prio_job     *pj = NULL;
1827         unsigned long int             flags;
1828
1829         DEBUG(printk("cryptocop_job_queue_insert(%d, 0x%p)\n", prio, operation));
1830
1831         if (!operation || !operation->cb){
1832                 DEBUG_API(printk("cryptocop_job_queue_insert oper=0x%p, NULL operation or callback\n", operation));
1833                 return -EINVAL;
1834         }
1835
1836         if ((ret = cryptocop_job_setup(&pj, operation)) != 0){
1837                 DEBUG_API(printk("cryptocop_job_queue_insert: job setup failed\n"));
1838                 return ret;
1839         }
1840         assert(pj != NULL);
1841
1842         spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
1843         list_add_tail(&pj->node, &cryptocop_job_queues[prio].jobs);
1844         spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
1845
1846         /* Make sure a job is running */
1847         cryptocop_start_job();
1848         return 0;
1849 }
1850
1851 static void cryptocop_do_tasklet(unsigned long unused);
1852 DECLARE_TASKLET (cryptocop_tasklet, cryptocop_do_tasklet, 0);
1853
1854 static void cryptocop_do_tasklet(unsigned long unused)
1855 {
1856         struct list_head             *node;
1857         struct cryptocop_prio_job    *pj = NULL;
1858         unsigned long                flags;
1859
1860         DEBUG(printk("cryptocop_do_tasklet: entering\n"));
1861
1862         do {
1863                 spin_lock_irqsave(&cryptocop_completed_jobs_lock, flags);
1864                 if (!list_empty(&cryptocop_completed_jobs)){
1865                         node = cryptocop_completed_jobs.next;
1866                         list_del(node);
1867                         pj = list_entry(node, struct cryptocop_prio_job, node);
1868                 } else {
1869                         pj = NULL;
1870                 }
1871                 spin_unlock_irqrestore(&cryptocop_completed_jobs_lock, flags);
1872                 if (pj) {
1873                         assert(pj->oper != NULL);
1874
1875                         /* Notify consumer of operation completeness. */
1876                         DEBUG(printk("cryptocop_do_tasklet: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
1877
1878                         pj->oper->operation_status = 0; /* Job is completed. */
1879                         pj->oper->cb(pj->oper, pj->oper->cb_data);
1880                         delete_internal_operation(pj->iop);
1881                         kfree(pj);
1882                 }
1883         } while (pj != NULL);
1884
1885         DEBUG(printk("cryptocop_do_tasklet: exiting\n"));
1886 }
1887
1888 static irqreturn_t
1889 dma_done_interrupt(int irq, void *dev_id, struct pt_regs * regs)
1890 {
1891         struct cryptocop_prio_job *done_job;
1892         reg_dma_rw_ack_intr ack_intr = {
1893                 .data = 1,
1894         };
1895
1896         REG_WR (dma, regi_dma9, rw_ack_intr, ack_intr);
1897
1898         DEBUG(printk("cryptocop DMA done\n"));
1899
1900         spin_lock(&running_job_lock);
1901         if (cryptocop_running_job == NULL){
1902                 printk("stream co-processor got interrupt when not busy\n");
1903                 spin_unlock(&running_job_lock);
1904                 return IRQ_HANDLED;
1905         }
1906         done_job = cryptocop_running_job;
1907         cryptocop_running_job = NULL;
1908         spin_unlock(&running_job_lock);
1909
1910         /* Start processing a job. */
1911         if (!spin_trylock(&cryptocop_process_lock)){
1912                 DEBUG(printk("cryptocop irq handler, not starting a job\n"));
1913         } else {
1914                 cryptocop_start_job();
1915                 spin_unlock(&cryptocop_process_lock);
1916         }
1917
1918         done_job->oper->operation_status = 0; /* Job is completed. */
1919         if (done_job->oper->fast_callback){
1920                 /* This operation wants callback from interrupt. */
1921                 done_job->oper->cb(done_job->oper, done_job->oper->cb_data);
1922                 delete_internal_operation(done_job->iop);
1923                 kfree(done_job);
1924         } else {
1925                 spin_lock(&cryptocop_completed_jobs_lock);
1926                 list_add_tail(&(done_job->node), &cryptocop_completed_jobs);
1927                 spin_unlock(&cryptocop_completed_jobs_lock);
1928                 tasklet_schedule(&cryptocop_tasklet);
1929         }
1930
1931         DEBUG(printk("cryptocop leave irq handler\n"));
1932         return IRQ_HANDLED;
1933 }
1934
1935
1936 /* Setup interrupts and DMA channels. */
1937 static int init_cryptocop(void)
1938 {
1939         unsigned long          flags;
1940         reg_intr_vect_rw_mask  intr_mask;
1941         reg_dma_rw_cfg         dma_cfg = {.en = 1};
1942         reg_dma_rw_intr_mask   intr_mask_in = {.data = regk_dma_yes}; /* Only want descriptor interrupts from the DMA in channel. */
1943         reg_dma_rw_ack_intr    ack_intr = {.data = 1,.in_eop = 1 };
1944         reg_strcop_rw_cfg      strcop_cfg = {
1945                 .ipend = regk_strcop_little,
1946                 .td1 = regk_strcop_e,
1947                 .td2 = regk_strcop_d,
1948                 .td3 = regk_strcop_e,
1949                 .ignore_sync = 0,
1950                 .en = 1
1951         };
1952
1953         if (request_irq(DMA9_INTR_VECT, dma_done_interrupt, 0, "stream co-processor DMA", NULL)) panic("request_irq stream co-processor irq dma9");
1954
1955         (void)crisv32_request_dma(8, "strcop", DMA_PANIC_ON_ERROR, 0, dma_strp);
1956         (void)crisv32_request_dma(9, "strcop", DMA_PANIC_ON_ERROR, 0, dma_strp);
1957
1958         local_irq_save(flags);
1959
1960         /* Reset and enable the cryptocop. */
1961         strcop_cfg.en = 0;
1962         REG_WR(strcop, regi_strcop, rw_cfg, strcop_cfg);
1963         strcop_cfg.en = 1;
1964         REG_WR(strcop, regi_strcop, rw_cfg, strcop_cfg);
1965
1966         /* Enable DMA9 interrupt */
1967         intr_mask = REG_RD(intr_vect, regi_irq, rw_mask);
1968         intr_mask.dma9 = 1;
1969         REG_WR(intr_vect, regi_irq, rw_mask, intr_mask);
1970
1971         /* Enable DMAs. */
1972         REG_WR(dma, regi_dma9, rw_cfg, dma_cfg); /* input DMA */
1973         REG_WR(dma, regi_dma8, rw_cfg, dma_cfg); /* output DMA */
1974
1975         /* Set up wordsize = 4 for DMAs. */
1976         DMA_WR_CMD (regi_dma8, regk_dma_set_w_size4);
1977         DMA_WR_CMD (regi_dma9, regk_dma_set_w_size4);
1978
1979         /* Enable interrupts. */
1980         REG_WR(dma, regi_dma9, rw_intr_mask, intr_mask_in);
1981
1982         /* Clear intr ack. */
1983         REG_WR(dma, regi_dma9, rw_ack_intr, ack_intr);
1984
1985         local_irq_restore(flags);
1986
1987         return 0;
1988 }
1989
1990 /* Free used cryptocop hw resources (interrupt and DMA channels). */
1991 static void release_cryptocop(void)
1992 {
1993         unsigned long          flags;
1994         reg_intr_vect_rw_mask  intr_mask;
1995         reg_dma_rw_cfg         dma_cfg = {.en = 0};
1996         reg_dma_rw_intr_mask   intr_mask_in = {0};
1997         reg_dma_rw_ack_intr    ack_intr = {.data = 1,.in_eop = 1 };
1998
1999         local_irq_save(flags);
2000
2001         /* Clear intr ack. */
2002         REG_WR(dma, regi_dma9, rw_ack_intr, ack_intr);
2003
2004         /* Disable DMA9 interrupt */
2005         intr_mask = REG_RD(intr_vect, regi_irq, rw_mask);
2006         intr_mask.dma9 = 0;
2007         REG_WR(intr_vect, regi_irq, rw_mask, intr_mask);
2008
2009         /* Disable DMAs. */
2010         REG_WR(dma, regi_dma9, rw_cfg, dma_cfg); /* input DMA */
2011         REG_WR(dma, regi_dma8, rw_cfg, dma_cfg); /* output DMA */
2012
2013         /* Disable interrupts. */
2014         REG_WR(dma, regi_dma9, rw_intr_mask, intr_mask_in);
2015
2016         local_irq_restore(flags);
2017
2018         free_irq(DMA9_INTR_VECT, NULL);
2019
2020         (void)crisv32_free_dma(8);
2021         (void)crisv32_free_dma(9);
2022 }
2023
2024
2025 /* Init job queue. */
2026 static int cryptocop_job_queue_init(void)
2027 {
2028         int i;
2029
2030         INIT_LIST_HEAD(&cryptocop_completed_jobs);
2031
2032         for (i = 0; i < cryptocop_prio_no_prios; i++){
2033                 cryptocop_job_queues[i].prio = (cryptocop_queue_priority)i;
2034                 INIT_LIST_HEAD(&cryptocop_job_queues[i].jobs);
2035         }
2036         return 0;
2037 }
2038
2039
2040 static void cryptocop_job_queue_close(void)
2041 {
2042         struct list_head               *node, *tmp;
2043         struct cryptocop_prio_job      *pj = NULL;
2044         unsigned long int              process_flags, flags;
2045         int                            i;
2046
2047         /* FIXME: This is as yet untested code. */
2048
2049         /* Stop strcop from getting an operation to process while we are closing the
2050            module. */
2051         spin_lock_irqsave(&cryptocop_process_lock, process_flags);
2052
2053         /* Empty the job queue. */
2054         spin_lock_irqsave(&cryptocop_process_lock, process_flags);
2055         for (i = 0; i < cryptocop_prio_no_prios; i++){
2056                 if (!list_empty(&(cryptocop_job_queues[i].jobs))){
2057                         list_for_each_safe(node, tmp, &(cryptocop_job_queues[i].jobs)) {
2058                                 pj = list_entry(node, struct cryptocop_prio_job, node);
2059                                 list_del(node);
2060
2061                                 /* Call callback to notify consumer of job removal. */
2062                                 DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2063                                 pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2064                                 pj->oper->cb(pj->oper, pj->oper->cb_data);
2065
2066                                 delete_internal_operation(pj->iop);
2067                                 kfree(pj);
2068                         }
2069                 }
2070         }
2071         spin_unlock_irqrestore(&cryptocop_process_lock, process_flags);
2072
2073         /* Remove the running job, if any. */
2074         spin_lock_irqsave(&running_job_lock, flags);
2075         if (cryptocop_running_job){
2076                 reg_strcop_rw_cfg rw_cfg;
2077                 reg_dma_rw_cfg    dma_out_cfg, dma_in_cfg;
2078
2079                 /* Stop DMA. */
2080                 dma_out_cfg = REG_RD(dma, regi_dma8, rw_cfg);
2081                 dma_out_cfg.en = regk_dma_no;
2082                 REG_WR(dma, regi_dma8, rw_cfg, dma_out_cfg);
2083
2084                 dma_in_cfg = REG_RD(dma, regi_dma9, rw_cfg);
2085                 dma_in_cfg.en = regk_dma_no;
2086                 REG_WR(dma, regi_dma9, rw_cfg, dma_in_cfg);
2087
2088                 /* Disble the cryptocop. */
2089                 rw_cfg = REG_RD(strcop, regi_strcop, rw_cfg);
2090                 rw_cfg.en = 0;
2091                 REG_WR(strcop, regi_strcop, rw_cfg, rw_cfg);
2092
2093                 pj = cryptocop_running_job;
2094                 cryptocop_running_job = NULL;
2095
2096                 /* Call callback to notify consumer of job removal. */
2097                 DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2098                 pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2099                 pj->oper->cb(pj->oper, pj->oper->cb_data);
2100
2101                 delete_internal_operation(pj->iop);
2102                 kfree(pj);
2103         }
2104         spin_unlock_irqrestore(&running_job_lock, flags);
2105
2106         /* Remove completed jobs, if any. */
2107         spin_lock_irqsave(&cryptocop_completed_jobs_lock, flags);
2108
2109         list_for_each_safe(node, tmp, &cryptocop_completed_jobs) {
2110                 pj = list_entry(node, struct cryptocop_prio_job, node);
2111                 list_del(node);
2112                 /* Call callback to notify consumer of job removal. */
2113                 DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2114                 pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2115                 pj->oper->cb(pj->oper, pj->oper->cb_data);
2116
2117                 delete_internal_operation(pj->iop);
2118                 kfree(pj);
2119         }
2120         spin_unlock_irqrestore(&cryptocop_completed_jobs_lock, flags);
2121 }
2122
2123
2124 static void cryptocop_start_job(void)
2125 {
2126         int                          i;
2127         struct cryptocop_prio_job    *pj;
2128         unsigned long int            flags;
2129         unsigned long int            running_job_flags;
2130         reg_strcop_rw_cfg            rw_cfg = {.en = 1, .ignore_sync = 0};
2131
2132         DEBUG(printk("cryptocop_start_job: entering\n"));
2133
2134         spin_lock_irqsave(&running_job_lock, running_job_flags);
2135         if (cryptocop_running_job != NULL){
2136                 /* Already running. */
2137                 DEBUG(printk("cryptocop_start_job: already running, exit\n"));
2138                 spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2139                 return;
2140         }
2141         spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
2142
2143         /* Check the queues in priority order. */
2144         for (i = cryptocop_prio_kernel_csum; (i < cryptocop_prio_no_prios) && list_empty(&cryptocop_job_queues[i].jobs); i++);
2145         if (i == cryptocop_prio_no_prios) {
2146                 spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
2147                 spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2148                 DEBUG(printk("cryptocop_start_job: no jobs to run\n"));
2149                 return; /* No jobs to run */
2150         }
2151         DEBUG(printk("starting job for prio %d\n", i));
2152
2153         /* TODO: Do not starve lower priority jobs.  Let in a lower
2154          * prio job for every N-th processed higher prio job or some
2155          * other scheduling policy.  This could reasonably be
2156          * tweakable since the optimal balance would depend on the
2157          * type of load on the system. */
2158
2159         /* Pull the DMA lists from the job and start the DMA client. */
2160         pj = list_entry(cryptocop_job_queues[i].jobs.next, struct cryptocop_prio_job, node);
2161         list_del(&pj->node);
2162         spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
2163         cryptocop_running_job = pj;
2164
2165         /* Set config register (3DES and CSUM modes). */
2166         switch (pj->iop->tdes_mode){
2167         case cryptocop_3des_eee:
2168                 rw_cfg.td1 = regk_strcop_e;
2169                 rw_cfg.td2 = regk_strcop_e;
2170                 rw_cfg.td3 = regk_strcop_e;
2171                 break;
2172         case cryptocop_3des_eed:
2173                 rw_cfg.td1 = regk_strcop_e;
2174                 rw_cfg.td2 = regk_strcop_e;
2175                 rw_cfg.td3 = regk_strcop_d;
2176                 break;
2177         case cryptocop_3des_ede:
2178                 rw_cfg.td1 = regk_strcop_e;
2179                 rw_cfg.td2 = regk_strcop_d;
2180                 rw_cfg.td3 = regk_strcop_e;
2181                 break;
2182         case cryptocop_3des_edd:
2183                 rw_cfg.td1 = regk_strcop_e;
2184                 rw_cfg.td2 = regk_strcop_d;
2185                 rw_cfg.td3 = regk_strcop_d;
2186                 break;
2187         case cryptocop_3des_dee:
2188                 rw_cfg.td1 = regk_strcop_d;
2189                 rw_cfg.td2 = regk_strcop_e;
2190                 rw_cfg.td3 = regk_strcop_e;
2191                 break;
2192         case cryptocop_3des_ded:
2193                 rw_cfg.td1 = regk_strcop_d;
2194                 rw_cfg.td2 = regk_strcop_e;
2195                 rw_cfg.td3 = regk_strcop_d;
2196                 break;
2197         case cryptocop_3des_dde:
2198                 rw_cfg.td1 = regk_strcop_d;
2199                 rw_cfg.td2 = regk_strcop_d;
2200                 rw_cfg.td3 = regk_strcop_e;
2201                 break;
2202         case cryptocop_3des_ddd:
2203                 rw_cfg.td1 = regk_strcop_d;
2204                 rw_cfg.td2 = regk_strcop_d;
2205                 rw_cfg.td3 = regk_strcop_d;
2206                 break;
2207         default:
2208                 DEBUG(printk("cryptocop_setup_dma_list: bad 3DES mode\n"));
2209         }
2210         switch (pj->iop->csum_mode){
2211         case cryptocop_csum_le:
2212                 rw_cfg.ipend = regk_strcop_little;
2213                 break;
2214         case cryptocop_csum_be:
2215                 rw_cfg.ipend = regk_strcop_big;
2216                 break;
2217         default:
2218                 DEBUG(printk("cryptocop_setup_dma_list: bad checksum mode\n"));
2219         }
2220         REG_WR(strcop, regi_strcop, rw_cfg, rw_cfg);
2221
2222         DEBUG(printk("cryptocop_start_job: starting DMA, new cryptocop_running_job=0x%p\n"
2223                      "ctx_in: 0x%p, phys: 0x%p\n"
2224                      "ctx_out: 0x%p, phys: 0x%p\n",
2225                      pj,
2226                      &pj->iop->ctx_in, (char*)virt_to_phys(&pj->iop->ctx_in),
2227                      &pj->iop->ctx_out, (char*)virt_to_phys(&pj->iop->ctx_out)));
2228
2229         /* Start input DMA. */
2230         DMA_START_CONTEXT(regi_dma9, virt_to_phys(&pj->iop->ctx_in));
2231
2232         /* Start output DMA. */
2233         DMA_START_CONTEXT(regi_dma8, virt_to_phys(&pj->iop->ctx_out));
2234
2235         spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2236         DEBUG(printk("cryptocop_start_job: exiting\n"));
2237 }
2238
2239
2240 static int cryptocop_job_setup(struct cryptocop_prio_job **pj, struct cryptocop_operation *operation)
2241 {
2242         int  err;
2243         int  alloc_flag = operation->in_interrupt ? GFP_ATOMIC : GFP_KERNEL;
2244         void *iop_alloc_ptr = NULL;
2245
2246         *pj = kmalloc(sizeof (struct cryptocop_prio_job), alloc_flag);
2247         if (!*pj) return -ENOMEM;
2248
2249         DEBUG(printk("cryptocop_job_setup: operation=0x%p\n", operation));
2250
2251         (*pj)->oper = operation;
2252         DEBUG(printk("cryptocop_job_setup, cb=0x%p cb_data=0x%p\n",  (*pj)->oper->cb, (*pj)->oper->cb_data));
2253
2254         if (operation->use_dmalists) {
2255                 DEBUG(print_user_dma_lists(&operation->list_op));
2256                 if (!operation->list_op.inlist || !operation->list_op.outlist || !operation->list_op.out_data_buf || !operation->list_op.in_data_buf){
2257                         DEBUG_API(printk("cryptocop_job_setup: bad indata (use_dmalists)\n"));
2258                         kfree(*pj);
2259                         return -EINVAL;
2260                 }
2261                 iop_alloc_ptr = kmalloc(DESCR_ALLOC_PAD + sizeof(struct cryptocop_int_operation), alloc_flag);
2262                 if (!iop_alloc_ptr) {
2263                         DEBUG_API(printk("cryptocop_job_setup: kmalloc cryptocop_int_operation\n"));
2264                         kfree(*pj);
2265                         return -ENOMEM;
2266                 }
2267                 (*pj)->iop = (struct cryptocop_int_operation*)(((unsigned long int)(iop_alloc_ptr + DESCR_ALLOC_PAD + offsetof(struct cryptocop_int_operation, ctx_out)) & ~0x0000001F) - offsetof(struct cryptocop_int_operation, ctx_out));
2268                 DEBUG(memset((*pj)->iop, 0xff, sizeof(struct cryptocop_int_operation)));
2269                 (*pj)->iop->alloc_ptr = iop_alloc_ptr;
2270                 (*pj)->iop->sid = operation->sid;
2271                 (*pj)->iop->cdesc_out = NULL;
2272                 (*pj)->iop->cdesc_in = NULL;
2273                 (*pj)->iop->tdes_mode = operation->list_op.tdes_mode;
2274                 (*pj)->iop->csum_mode = operation->list_op.csum_mode;
2275                 (*pj)->iop->ddesc_out = operation->list_op.outlist;
2276                 (*pj)->iop->ddesc_in = operation->list_op.inlist;
2277
2278                 /* Setup DMA contexts. */
2279                 (*pj)->iop->ctx_out.next = NULL;
2280                 (*pj)->iop->ctx_out.eol = 1;
2281                 (*pj)->iop->ctx_out.saved_data = operation->list_op.outlist;
2282                 (*pj)->iop->ctx_out.saved_data_buf = operation->list_op.out_data_buf;
2283
2284                 (*pj)->iop->ctx_in.next = NULL;
2285                 (*pj)->iop->ctx_in.eol = 1;
2286                 (*pj)->iop->ctx_in.saved_data = operation->list_op.inlist;
2287                 (*pj)->iop->ctx_in.saved_data_buf = operation->list_op.in_data_buf;
2288         } else {
2289                 if ((err = cryptocop_setup_dma_list(operation, &(*pj)->iop, alloc_flag))) {
2290                         DEBUG_API(printk("cryptocop_job_setup: cryptocop_setup_dma_list failed %d\n", err));
2291                         kfree(*pj);
2292                         return err;
2293                 }
2294         }
2295         DEBUG(print_dma_descriptors((*pj)->iop));
2296
2297         DEBUG(printk("cryptocop_job_setup, DMA list setup successful\n"));
2298
2299         return 0;
2300 }
2301
2302
2303 static int cryptocop_open(struct inode *inode, struct file *filp)
2304 {
2305         int p = iminor(inode);
2306
2307         if (p != CRYPTOCOP_MINOR) return -EINVAL;
2308
2309         filp->private_data = NULL;
2310         return 0;
2311 }
2312
2313
2314 static int cryptocop_release(struct inode *inode, struct file *filp)
2315 {
2316         struct cryptocop_private *dev = filp->private_data;
2317         struct cryptocop_private *dev_next;
2318
2319         while (dev){
2320                 dev_next = dev->next;
2321                 if (dev->sid != CRYPTOCOP_SESSION_ID_NONE) {
2322                         (void)cryptocop_free_session(dev->sid);
2323                 }
2324                 kfree(dev);
2325                 dev = dev_next;
2326         }
2327
2328         return 0;
2329 }
2330
2331
2332 static int cryptocop_ioctl_close_session(struct inode *inode, struct file *filp,
2333                                          unsigned int cmd, unsigned long arg)
2334 {
2335         struct cryptocop_private  *dev = filp->private_data;
2336         struct cryptocop_private  *prev_dev = NULL;
2337         struct strcop_session_op  *sess_op = (struct strcop_session_op *)arg;
2338         struct strcop_session_op  sop;
2339         int                       err;
2340
2341         DEBUG(printk("cryptocop_ioctl_close_session\n"));
2342
2343         if (!access_ok(VERIFY_READ, sess_op, sizeof(struct strcop_session_op)))
2344                 return -EFAULT;
2345         err = copy_from_user(&sop, sess_op, sizeof(struct strcop_session_op));
2346         if (err) return -EFAULT;
2347
2348         while (dev && (dev->sid != sop.ses_id)) {
2349                 prev_dev = dev;
2350                 dev = dev->next;
2351         }
2352         if (dev){
2353                 if (prev_dev){
2354                         prev_dev->next = dev->next;
2355                 } else {
2356                         filp->private_data = dev->next;
2357                 }
2358                 err = cryptocop_free_session(dev->sid);
2359                 if (err) return -EFAULT;
2360         } else {
2361                 DEBUG_API(printk("cryptocop_ioctl_close_session: session %lld not found\n", sop.ses_id));
2362                 return -EINVAL;
2363         }
2364         return 0;
2365 }
2366
2367
2368 static void ioctl_process_job_callback(struct cryptocop_operation *op, void*cb_data)
2369 {
2370         struct ioctl_job_cb_ctx *jc = (struct ioctl_job_cb_ctx *)cb_data;
2371
2372         DEBUG(printk("ioctl_process_job_callback: op=0x%p, cb_data=0x%p\n", op, cb_data));
2373
2374         jc->processed = 1;
2375         wake_up(&cryptocop_ioc_process_wq);
2376 }
2377
2378
2379 #define CRYPTOCOP_IOCTL_CIPHER_TID  (1)
2380 #define CRYPTOCOP_IOCTL_DIGEST_TID  (2)
2381 #define CRYPTOCOP_IOCTL_CSUM_TID    (3)
2382
2383 static size_t first_cfg_change_ix(struct strcop_crypto_op *crp_op)
2384 {
2385         size_t ch_ix = 0;
2386
2387         if (crp_op->do_cipher) ch_ix = crp_op->cipher_start;
2388         if (crp_op->do_digest && (crp_op->digest_start < ch_ix)) ch_ix = crp_op->digest_start;
2389         if (crp_op->do_csum && (crp_op->csum_start < ch_ix)) ch_ix = crp_op->csum_start;
2390
2391         DEBUG(printk("first_cfg_change_ix: ix=%d\n", ch_ix));
2392         return ch_ix;
2393 }
2394
2395
2396 static size_t next_cfg_change_ix(struct strcop_crypto_op *crp_op, size_t ix)
2397 {
2398         size_t ch_ix = INT_MAX;
2399         size_t tmp_ix = 0;
2400
2401         if (crp_op->do_cipher && ((crp_op->cipher_start + crp_op->cipher_len) > ix)){
2402                 if (crp_op->cipher_start > ix) {
2403                         ch_ix = crp_op->cipher_start;
2404                 } else {
2405                         ch_ix = crp_op->cipher_start + crp_op->cipher_len;
2406                 }
2407         }
2408         if (crp_op->do_digest && ((crp_op->digest_start + crp_op->digest_len) > ix)){
2409                 if (crp_op->digest_start > ix) {
2410                         tmp_ix = crp_op->digest_start;
2411                 } else {
2412                         tmp_ix = crp_op->digest_start + crp_op->digest_len;
2413                 }
2414                 if (tmp_ix < ch_ix) ch_ix = tmp_ix;
2415         }
2416         if (crp_op->do_csum && ((crp_op->csum_start + crp_op->csum_len) > ix)){
2417                 if (crp_op->csum_start > ix) {
2418                         tmp_ix = crp_op->csum_start;
2419                 } else {
2420                         tmp_ix = crp_op->csum_start + crp_op->csum_len;
2421                 }
2422                 if (tmp_ix < ch_ix) ch_ix = tmp_ix;
2423         }
2424         if (ch_ix == INT_MAX) ch_ix = ix;
2425         DEBUG(printk("next_cfg_change_ix prev ix=%d, next ix=%d\n", ix, ch_ix));
2426         return ch_ix;
2427 }
2428
2429
2430 /* Map map_length bytes from the pages starting on *pageix and *pageoffset to iovecs starting on *iovix.
2431  * Return -1 for ok, 0 for fail. */
2432 static int map_pages_to_iovec(struct iovec *iov, int iovlen, int *iovix, struct page **pages, int nopages, int *pageix, int *pageoffset, int map_length )
2433 {
2434         int tmplen;
2435
2436         assert(iov != NULL);
2437         assert(iovix != NULL);
2438         assert(pages != NULL);
2439         assert(pageix != NULL);
2440         assert(pageoffset != NULL);
2441
2442         DEBUG(printk("map_pages_to_iovec, map_length=%d, iovlen=%d, *iovix=%d, nopages=%d, *pageix=%d, *pageoffset=%d\n", map_length, iovlen, *iovix, nopages, *pageix, *pageoffset));
2443
2444         while (map_length > 0){
2445                 DEBUG(printk("map_pages_to_iovec, map_length=%d, iovlen=%d, *iovix=%d, nopages=%d, *pageix=%d, *pageoffset=%d\n", map_length, iovlen, *iovix, nopages, *pageix, *pageoffset));
2446                 if (*iovix >= iovlen){
2447                         DEBUG_API(printk("map_page_to_iovec: *iovix=%d >= iovlen=%d\n", *iovix, iovlen));
2448                         return 0;
2449                 }
2450                 if (*pageix >= nopages){
2451                         DEBUG_API(printk("map_page_to_iovec: *pageix=%d >= nopages=%d\n", *pageix, nopages));
2452                         return 0;
2453                 }
2454                 iov[*iovix].iov_base = (unsigned char*)page_address(pages[*pageix]) + *pageoffset;
2455                 tmplen = PAGE_SIZE - *pageoffset;
2456                 if (tmplen < map_length){
2457                         (*pageoffset) = 0;
2458                         (*pageix)++;
2459                 } else {
2460                         tmplen = map_length;
2461                         (*pageoffset) += map_length;
2462                 }
2463                 DEBUG(printk("mapping %d bytes from page %d (or %d) to iovec %d\n", tmplen, *pageix, *pageix-1, *iovix));
2464                 iov[*iovix].iov_len = tmplen;
2465                 map_length -= tmplen;
2466                 (*iovix)++;
2467         }
2468         DEBUG(printk("map_page_to_iovec, exit, *iovix=%d\n", *iovix));
2469         return -1;
2470 }
2471
2472
2473
2474 static int cryptocop_ioctl_process(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
2475 {
2476         int                             i;
2477         struct cryptocop_private        *dev = filp->private_data;
2478         struct strcop_crypto_op         *crp_oper = (struct strcop_crypto_op *)arg;
2479         struct strcop_crypto_op         oper = {0};
2480         int                             err = 0;
2481         struct cryptocop_operation      *cop = NULL;
2482
2483         struct ioctl_job_cb_ctx         *jc = NULL;
2484
2485         struct page                     **inpages = NULL;
2486         struct page                     **outpages = NULL;
2487         int                             noinpages = 0;
2488         int                             nooutpages = 0;
2489
2490         struct cryptocop_desc           descs[5]; /* Max 5 descriptors are needed, there are three transforms that
2491                                                    * can get connected/disconnected on different places in the indata. */
2492         struct cryptocop_desc_cfg       dcfgs[5*3];
2493         int                             desc_ix = 0;
2494         int                             dcfg_ix = 0;
2495         struct cryptocop_tfrm_cfg       ciph_tcfg = {0};
2496         struct cryptocop_tfrm_cfg       digest_tcfg = {0};
2497         struct cryptocop_tfrm_cfg       csum_tcfg = {0};
2498
2499         unsigned char                   *digest_result = NULL;
2500         int                             digest_length = 0;
2501         int                             cblocklen = 0;
2502         unsigned char                   csum_result[CSUM_BLOCK_LENGTH];
2503         struct cryptocop_session        *sess;
2504
2505         int    iovlen = 0;
2506         int    iovix = 0;
2507         int    pageix = 0;
2508         int    pageoffset = 0;
2509
2510         size_t prev_ix = 0;
2511         size_t next_ix;
2512
2513         int    cipher_active, digest_active, csum_active;
2514         int    end_digest, end_csum;
2515         int    digest_done = 0;
2516         int    cipher_done = 0;
2517         int    csum_done = 0;
2518
2519         DEBUG(printk("cryptocop_ioctl_process\n"));
2520
2521         if (!access_ok(VERIFY_WRITE, crp_oper, sizeof(struct strcop_crypto_op))){
2522                 DEBUG_API(printk("cryptocop_ioctl_process: !access_ok crp_oper!\n"));
2523                 return -EFAULT;
2524         }
2525         if (copy_from_user(&oper, crp_oper, sizeof(struct strcop_crypto_op))) {
2526                 DEBUG_API(printk("cryptocop_ioctl_process: copy_from_user\n"));
2527                 return -EFAULT;
2528         }
2529         DEBUG(print_strcop_crypto_op(&oper));
2530
2531         while (dev && dev->sid != oper.ses_id) dev = dev->next;
2532         if (!dev){
2533                 DEBUG_API(printk("cryptocop_ioctl_process: session %lld not found\n", oper.ses_id));
2534                 return -EINVAL;
2535         }
2536
2537         /* Check buffers. */
2538         if (((oper.indata + oper.inlen) < oper.indata) || ((oper.cipher_outdata + oper.cipher_outlen) < oper.cipher_outdata)){
2539                 DEBUG_API(printk("cryptocop_ioctl_process: user buffers wrapped around, bad user!\n"));
2540                 return -EINVAL;
2541         }
2542
2543         if (!access_ok(VERIFY_WRITE, oper.cipher_outdata, oper.cipher_outlen)){
2544                 DEBUG_API(printk("cryptocop_ioctl_process: !access_ok out data!\n"));
2545                 return -EFAULT;
2546         }
2547         if (!access_ok(VERIFY_READ, oper.indata, oper.inlen)){
2548                 DEBUG_API(printk("cryptocop_ioctl_process: !access_ok in data!\n"));
2549                 return -EFAULT;
2550         }
2551
2552         cop = kmalloc(sizeof(struct cryptocop_operation), GFP_KERNEL);
2553         if (!cop) {
2554                 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc\n"));
2555                 return -ENOMEM;
2556         }
2557         jc = kmalloc(sizeof(struct ioctl_job_cb_ctx), GFP_KERNEL);
2558         if (!jc) {
2559                 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc\n"));
2560                 err = -ENOMEM;
2561                 goto error_cleanup;
2562         }
2563         jc->processed = 0;
2564
2565         cop->cb_data = jc;
2566         cop->cb = ioctl_process_job_callback;
2567         cop->operation_status = 0;
2568         cop->use_dmalists = 0;
2569         cop->in_interrupt = 0;
2570         cop->fast_callback = 0;
2571         cop->tfrm_op.tfrm_cfg = NULL;
2572         cop->tfrm_op.desc = NULL;
2573         cop->tfrm_op.indata = NULL;
2574         cop->tfrm_op.incount = 0;
2575         cop->tfrm_op.inlen = 0;
2576         cop->tfrm_op.outdata = NULL;
2577         cop->tfrm_op.outcount = 0;
2578         cop->tfrm_op.outlen = 0;
2579
2580         sess = get_session(oper.ses_id);
2581         if (!sess){
2582                 DEBUG_API(printk("cryptocop_ioctl_process: bad session id.\n"));
2583                 kfree(cop);
2584                 kfree(jc);
2585                 return -EINVAL;
2586         }
2587
2588         if (oper.do_cipher) {
2589                 unsigned int                    cipher_outlen = 0;
2590                 struct cryptocop_transform_ctx  *tc = get_transform_ctx(sess, CRYPTOCOP_IOCTL_CIPHER_TID);
2591                 if (!tc) {
2592                         DEBUG_API(printk("cryptocop_ioctl_process: no cipher transform in session.\n"));
2593                         err = -EINVAL;
2594                         goto error_cleanup;
2595                 }
2596                 ciph_tcfg.tid = CRYPTOCOP_IOCTL_CIPHER_TID;
2597                 ciph_tcfg.inject_ix = 0;
2598                 ciph_tcfg.flags = 0;
2599                 if ((oper.cipher_start < 0) || (oper.cipher_len <= 0) || (oper.cipher_start > oper.inlen) || ((oper.cipher_start + oper.cipher_len) > oper.inlen)){
2600                         DEBUG_API(printk("cryptocop_ioctl_process: bad cipher length\n"));
2601                         kfree(cop);
2602                         kfree(jc);
2603                         return -EINVAL;
2604                 }
2605                 cblocklen = tc->init.alg == cryptocop_alg_aes ? AES_BLOCK_LENGTH : DES_BLOCK_LENGTH;
2606                 if (oper.cipher_len % cblocklen) {
2607                         kfree(cop);
2608                         kfree(jc);
2609                         DEBUG_API(printk("cryptocop_ioctl_process: cipher inlength not multiple of block length.\n"));
2610                         return -EINVAL;
2611                 }
2612                 cipher_outlen = oper.cipher_len;
2613                 if (tc->init.cipher_mode == cryptocop_cipher_mode_cbc){
2614                         if (oper.cipher_explicit) {
2615                                 ciph_tcfg.flags |= CRYPTOCOP_EXPLICIT_IV;
2616                                 memcpy(ciph_tcfg.iv, oper.cipher_iv, cblocklen);
2617                         } else {
2618                                 cipher_outlen = oper.cipher_len - cblocklen;
2619                         }
2620                 } else {
2621                         if (oper.cipher_explicit){
2622                                 kfree(cop);
2623                                 kfree(jc);
2624                                 DEBUG_API(printk("cryptocop_ioctl_process: explicit_iv when not CBC mode\n"));
2625                                 return -EINVAL;
2626                         }
2627                 }
2628                 if (oper.cipher_outlen != cipher_outlen) {
2629                         kfree(cop);
2630                         kfree(jc);
2631                         DEBUG_API(printk("cryptocop_ioctl_process: cipher_outlen incorrect, should be %d not %d.\n", cipher_outlen, oper.cipher_outlen));
2632                         return -EINVAL;
2633                 }
2634
2635                 if (oper.decrypt){
2636                         ciph_tcfg.flags |= CRYPTOCOP_DECRYPT;
2637                 } else {
2638                         ciph_tcfg.flags |= CRYPTOCOP_ENCRYPT;
2639                 }
2640                 ciph_tcfg.next = cop->tfrm_op.tfrm_cfg;
2641                 cop->tfrm_op.tfrm_cfg = &ciph_tcfg;
2642         }
2643         if (oper.do_digest){
2644                 struct cryptocop_transform_ctx *tc = get_transform_ctx(sess, CRYPTOCOP_IOCTL_DIGEST_TID);
2645                 if (!tc) {
2646                         DEBUG_API(printk("cryptocop_ioctl_process: no digest transform in session.\n"));
2647                         err = -EINVAL;
2648                         goto error_cleanup;
2649                 }
2650                 digest_length = tc->init.alg == cryptocop_alg_md5 ? 16 : 20;
2651                 digest_result = kmalloc(digest_length, GFP_KERNEL);
2652                 if (!digest_result) {
2653                         DEBUG_API(printk("cryptocop_ioctl_process: kmalloc digest_result\n"));
2654                         err = -EINVAL;
2655                         goto error_cleanup;
2656                 }
2657                 DEBUG(memset(digest_result, 0xff, digest_length));
2658
2659                 digest_tcfg.tid = CRYPTOCOP_IOCTL_DIGEST_TID;
2660                 digest_tcfg.inject_ix = 0;
2661                 ciph_tcfg.inject_ix += digest_length;
2662                 if ((oper.digest_start < 0) || (oper.digest_len <= 0) || (oper.digest_start > oper.inlen) || ((oper.digest_start + oper.digest_len) > oper.inlen)){
2663                         DEBUG_API(printk("cryptocop_ioctl_process: bad digest length\n"));
2664                         err = -EINVAL;
2665                         goto error_cleanup;
2666                 }
2667
2668                 digest_tcfg.next = cop->tfrm_op.tfrm_cfg;
2669                 cop->tfrm_op.tfrm_cfg = &digest_tcfg;
2670         }
2671         if (oper.do_csum){
2672                 csum_tcfg.tid = CRYPTOCOP_IOCTL_CSUM_TID;
2673                 csum_tcfg.inject_ix = digest_length;
2674                 ciph_tcfg.inject_ix += 2;
2675
2676                 if ((oper.csum_start < 0) || (oper.csum_len <= 0) || (oper.csum_start > oper.inlen) || ((oper.csum_start + oper.csum_len) > oper.inlen)){
2677                         DEBUG_API(printk("cryptocop_ioctl_process: bad csum length\n"));
2678                         kfree(cop);
2679                         kfree(jc);
2680                         return -EINVAL;
2681                 }
2682
2683                 csum_tcfg.next = cop->tfrm_op.tfrm_cfg;
2684                 cop->tfrm_op.tfrm_cfg = &csum_tcfg;
2685         }
2686
2687         prev_ix = first_cfg_change_ix(&oper);
2688         if (prev_ix > oper.inlen) {
2689                 DEBUG_API(printk("cryptocop_ioctl_process: length mismatch\n"));
2690                 nooutpages = noinpages = 0;
2691                 err = -EINVAL;
2692                 goto error_cleanup;
2693         }
2694         DEBUG(printk("cryptocop_ioctl_process: inlen=%d, cipher_outlen=%d\n", oper.inlen, oper.cipher_outlen));
2695
2696         /* Map user pages for in and out data of the operation. */
2697         noinpages = (((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK) + oper.inlen - 1 - prev_ix + ~PAGE_MASK) >> PAGE_SHIFT;
2698         DEBUG(printk("cryptocop_ioctl_process: noinpages=%d\n", noinpages));
2699         inpages = kmalloc(noinpages * sizeof(struct page*), GFP_KERNEL);
2700         if (!inpages){
2701                 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc inpages\n"));
2702                 nooutpages = noinpages = 0;
2703                 err = -ENOMEM;
2704                 goto error_cleanup;
2705         }
2706         if (oper.do_cipher){
2707                 nooutpages = (((unsigned long int)oper.cipher_outdata & ~PAGE_MASK) + oper.cipher_outlen - 1 + ~PAGE_MASK) >> PAGE_SHIFT;
2708                 DEBUG(printk("cryptocop_ioctl_process: nooutpages=%d\n", nooutpages));
2709                 outpages = kmalloc(nooutpages * sizeof(struct page*), GFP_KERNEL);
2710                 if (!outpages){
2711                         DEBUG_API(printk("cryptocop_ioctl_process: kmalloc outpages\n"));
2712                         nooutpages = noinpages = 0;
2713                         err = -ENOMEM;
2714                         goto error_cleanup;
2715                 }
2716         }
2717
2718         /* Acquire the mm page semaphore. */
2719         down_read(&current->mm->mmap_sem);
2720
2721         err = get_user_pages(current,
2722                              current->mm,
2723                              (unsigned long int)(oper.indata + prev_ix),
2724                              noinpages,
2725                              0,  /* read access only for in data */
2726                              0, /* no force */
2727                              inpages,
2728                              NULL);
2729
2730         if (err < 0) {
2731                 up_read(&current->mm->mmap_sem);
2732                 nooutpages = noinpages = 0;
2733                 DEBUG_API(printk("cryptocop_ioctl_process: get_user_pages indata\n"));
2734                 goto error_cleanup;
2735         }
2736         noinpages = err;
2737         if (oper.do_cipher){
2738                 err = get_user_pages(current,
2739                                      current->mm,
2740                                      (unsigned long int)oper.cipher_outdata,
2741                                      nooutpages,
2742                                      1, /* write access for out data */
2743                                      0, /* no force */
2744                                      outpages,
2745                                      NULL);
2746                 up_read(&current->mm->mmap_sem);
2747                 if (err < 0) {
2748                         nooutpages = 0;
2749                         DEBUG_API(printk("cryptocop_ioctl_process: get_user_pages outdata\n"));
2750                         goto error_cleanup;
2751                 }
2752                 nooutpages = err;
2753         } else {
2754                 up_read(&current->mm->mmap_sem);
2755         }
2756
2757         /* Add 6 to nooutpages to make room for possibly inserted buffers for storing digest and
2758          * csum output and splits when units are (dis-)connected. */
2759         cop->tfrm_op.indata = kmalloc((noinpages) * sizeof(struct iovec), GFP_KERNEL);
2760         cop->tfrm_op.outdata = kmalloc((6 + nooutpages) * sizeof(struct iovec), GFP_KERNEL);
2761         if (!cop->tfrm_op.indata || !cop->tfrm_op.outdata) {
2762                 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc iovecs\n"));
2763                 err = -ENOMEM;
2764                 goto error_cleanup;
2765         }
2766
2767         cop->tfrm_op.inlen = oper.inlen - prev_ix;
2768         cop->tfrm_op.outlen = 0;
2769         if (oper.do_cipher) cop->tfrm_op.outlen += oper.cipher_outlen;
2770         if (oper.do_digest) cop->tfrm_op.outlen += digest_length;
2771         if (oper.do_csum) cop->tfrm_op.outlen += 2;
2772
2773         /* Setup the in iovecs. */
2774         cop->tfrm_op.incount = noinpages;
2775         if (noinpages > 1){
2776                 size_t tmplen = cop->tfrm_op.inlen;
2777
2778                 cop->tfrm_op.indata[0].iov_len = PAGE_SIZE - ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2779                 cop->tfrm_op.indata[0].iov_base = (unsigned char*)page_address(inpages[0]) + ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2780                 tmplen -= cop->tfrm_op.indata[0].iov_len;
2781                 for (i = 1; i<noinpages; i++){
2782                         cop->tfrm_op.indata[i].iov_len = tmplen < PAGE_SIZE ? tmplen : PAGE_SIZE;
2783                         cop->tfrm_op.indata[i].iov_base = (unsigned char*)page_address(inpages[i]);
2784                         tmplen -= PAGE_SIZE;
2785                 }
2786         } else {
2787                 cop->tfrm_op.indata[0].iov_len = oper.inlen - prev_ix;
2788                 cop->tfrm_op.indata[0].iov_base = (unsigned char*)page_address(inpages[0]) + ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2789         }
2790
2791         iovlen = nooutpages + 6;
2792         pageoffset = oper.do_cipher ? ((unsigned long int)oper.cipher_outdata & ~PAGE_MASK) : 0;
2793
2794         next_ix = next_cfg_change_ix(&oper, prev_ix);
2795         if (prev_ix == next_ix){
2796                 DEBUG_API(printk("cryptocop_ioctl_process: length configuration broken.\n"));
2797                 err = -EINVAL;  /* This should be impossible barring bugs. */
2798                 goto error_cleanup;
2799         }
2800         while (prev_ix != next_ix){
2801                 end_digest = end_csum = cipher_active = digest_active = csum_active = 0;
2802                 descs[desc_ix].cfg = NULL;
2803                 descs[desc_ix].length = next_ix - prev_ix;
2804
2805                 if (oper.do_cipher && (oper.cipher_start < next_ix) && (prev_ix < (oper.cipher_start + oper.cipher_len))) {
2806                         dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_CIPHER_TID;
2807                         dcfgs[dcfg_ix].src = cryptocop_source_dma;
2808                         cipher_active = 1;
2809
2810                         if (next_ix == (oper.cipher_start + oper.cipher_len)){
2811                                 cipher_done = 1;
2812                                 dcfgs[dcfg_ix].last = 1;
2813                         } else {
2814                                 dcfgs[dcfg_ix].last = 0;
2815                         }
2816                         dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2817                         descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2818                         ++dcfg_ix;
2819                 }
2820                 if (oper.do_digest && (oper.digest_start < next_ix) && (prev_ix < (oper.digest_start + oper.digest_len))) {
2821                         digest_active = 1;
2822                         dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_DIGEST_TID;
2823                         dcfgs[dcfg_ix].src = cryptocop_source_dma;
2824                         if (next_ix == (oper.digest_start + oper.digest_len)){
2825                                 assert(!digest_done);
2826                                 digest_done = 1;
2827                                 dcfgs[dcfg_ix].last = 1;
2828                         } else {
2829                                 dcfgs[dcfg_ix].last = 0;
2830                         }
2831                         dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2832                         descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2833                         ++dcfg_ix;
2834                 }
2835                 if (oper.do_csum && (oper.csum_start < next_ix) && (prev_ix < (oper.csum_start + oper.csum_len))){
2836                         csum_active = 1;
2837                         dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_CSUM_TID;
2838                         dcfgs[dcfg_ix].src = cryptocop_source_dma;
2839                         if (next_ix == (oper.csum_start + oper.csum_len)){
2840                                 csum_done = 1;
2841                                 dcfgs[dcfg_ix].last = 1;
2842                         } else {
2843                                 dcfgs[dcfg_ix].last = 0;
2844                         }
2845                         dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2846                         descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2847                         ++dcfg_ix;
2848                 }
2849                 if (!descs[desc_ix].cfg){
2850                         DEBUG_API(printk("cryptocop_ioctl_process: data segment %d (%d to %d) had no active transforms\n", desc_ix, prev_ix, next_ix));
2851                         err = -EINVAL;
2852                         goto error_cleanup;
2853                 }
2854                 descs[desc_ix].next = &(descs[desc_ix]) + 1;
2855                 ++desc_ix;
2856                 prev_ix = next_ix;
2857                 next_ix = next_cfg_change_ix(&oper, prev_ix);
2858         }
2859         if (desc_ix > 0){
2860                 descs[desc_ix-1].next = NULL;
2861         } else {
2862                 descs[0].next = NULL;
2863         }
2864         if (oper.do_digest) {
2865                 DEBUG(printk("cryptocop_ioctl_process: mapping %d byte digest output to iovec %d\n", digest_length, iovix));
2866                 /* Add outdata iovec, length == <length of type of digest> */
2867                 cop->tfrm_op.outdata[iovix].iov_base = digest_result;
2868                 cop->tfrm_op.outdata[iovix].iov_len = digest_length;
2869                 ++iovix;
2870         }
2871         if (oper.do_csum) {
2872                 /* Add outdata iovec, length == 2, the length of csum. */
2873                 DEBUG(printk("cryptocop_ioctl_process: mapping 2 byte csum output to iovec %d\n", iovix));
2874                 /* Add outdata iovec, length == <length of type of digest> */
2875                 cop->tfrm_op.outdata[iovix].iov_base = csum_result;
2876                 cop->tfrm_op.outdata[iovix].iov_len = 2;
2877                 ++iovix;
2878         }
2879         if (oper.do_cipher) {
2880                 if (!map_pages_to_iovec(cop->tfrm_op.outdata, iovlen, &iovix, outpages, nooutpages, &pageix, &pageoffset, oper.cipher_outlen)){
2881                         DEBUG_API(printk("cryptocop_ioctl_process: failed to map pages to iovec.\n"));
2882                         err = -ENOSYS; /* This should be impossible barring bugs. */
2883                         goto error_cleanup;
2884                 }
2885         }
2886         DEBUG(printk("cryptocop_ioctl_process: setting cop->tfrm_op.outcount %d\n", iovix));
2887         cop->tfrm_op.outcount = iovix;
2888         assert(iovix <= (nooutpages + 6));
2889
2890         cop->sid = oper.ses_id;
2891         cop->tfrm_op.desc = &descs[0];
2892
2893         DEBUG(printk("cryptocop_ioctl_process: inserting job, cb_data=0x%p\n", cop->cb_data));
2894
2895         if ((err = cryptocop_job_queue_insert_user_job(cop)) != 0) {
2896                 DEBUG_API(printk("cryptocop_ioctl_process: insert job %d\n", err));
2897                 err = -EINVAL;
2898                 goto error_cleanup;
2899         }
2900
2901         DEBUG(printk("cryptocop_ioctl_process: begin wait for result\n"));
2902
2903         wait_event(cryptocop_ioc_process_wq, (jc->processed != 0));
2904         DEBUG(printk("cryptocop_ioctl_process: end wait for result\n"));
2905         if (!jc->processed){
2906                 printk(KERN_WARNING "cryptocop_ioctl_process: job not processed at completion\n");
2907                 err = -EIO;
2908                 goto error_cleanup;
2909         }
2910
2911         /* Job process done.  Cipher output should already be correct in job so no post processing of outdata. */
2912         DEBUG(printk("cryptocop_ioctl_process: operation_status = %d\n", cop->operation_status));
2913         if (cop->operation_status == 0){
2914                 if (oper.do_digest){
2915                         DEBUG(printk("cryptocop_ioctl_process: copy %d bytes digest to user\n", digest_length));
2916                         err = copy_to_user((unsigned char*)crp_oper + offsetof(struct strcop_crypto_op, digest), digest_result, digest_length);
2917                         if (0 != err){
2918                                 DEBUG_API(printk("cryptocop_ioctl_process: copy_to_user, digest length %d, err %d\n", digest_length, err));
2919                                 err = -EFAULT;
2920                                 goto error_cleanup;
2921                         }
2922                 }
2923                 if (oper.do_csum){
2924                         DEBUG(printk("cryptocop_ioctl_process: copy 2 bytes checksum to user\n"));
2925                         err = copy_to_user((unsigned char*)crp_oper + offsetof(struct strcop_crypto_op, csum), csum_result, 2);
2926                         if (0 != err){
2927                                 DEBUG_API(printk("cryptocop_ioctl_process: copy_to_user, csum, err %d\n", err));
2928                                 err = -EFAULT;
2929                                 goto error_cleanup;
2930                         }
2931                 }
2932                 err = 0;
2933         } else {
2934                 DEBUG(printk("cryptocop_ioctl_process: returning err = operation_status = %d\n", cop->operation_status));
2935                 err = cop->operation_status;
2936         }
2937
2938  error_cleanup:
2939         /* Release page caches. */
2940         for (i = 0; i < noinpages; i++){
2941                 put_page(inpages[i]);
2942         }
2943         for (i = 0; i < nooutpages; i++){
2944                 int spdl_err;
2945                 /* Mark output pages dirty. */
2946                 spdl_err = set_page_dirty_lock(outpages[i]);
2947                 DEBUG(if (spdl_err < 0)printk("cryptocop_ioctl_process: set_page_dirty_lock returned %d\n", spdl_err));
2948         }
2949         for (i = 0; i < nooutpages; i++){
2950                 put_page(outpages[i]);
2951         }
2952
2953         kfree(digest_result);
2954         kfree(inpages);
2955         kfree(outpages);
2956         if (cop){
2957                 kfree(cop->tfrm_op.indata);
2958                 kfree(cop->tfrm_op.outdata);
2959                 kfree(cop);
2960         }
2961         kfree(jc);
2962
2963         DEBUG(print_lock_status());
2964
2965         return err;
2966 }
2967
2968
2969 static int cryptocop_ioctl_create_session(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
2970 {
2971         cryptocop_session_id             sid;
2972         int                              err;
2973         struct cryptocop_private         *dev;
2974         struct strcop_session_op         *sess_op = (struct strcop_session_op *)arg;
2975         struct strcop_session_op         sop;
2976         struct cryptocop_transform_init  *tis = NULL;
2977         struct cryptocop_transform_init  ti_cipher = {0};
2978         struct cryptocop_transform_init  ti_digest = {0};
2979         struct cryptocop_transform_init  ti_csum = {0};
2980
2981         if (!access_ok(VERIFY_WRITE, sess_op, sizeof(struct strcop_session_op)))
2982                 return -EFAULT;
2983         err = copy_from_user(&sop, sess_op, sizeof(struct strcop_session_op));
2984         if (err) return -EFAULT;
2985         if (sop.cipher != cryptocop_cipher_none) {
2986                 if (!access_ok(VERIFY_READ, sop.key, sop.keylen)) return -EFAULT;
2987         }
2988         DEBUG(printk("cryptocop_ioctl_create_session, sess_op:\n"));
2989
2990         DEBUG(printk("\tcipher:%d\n"
2991                      "\tcipher_mode:%d\n"
2992                      "\tdigest:%d\n"
2993                      "\tcsum:%d\n",
2994                      (int)sop.cipher,
2995                      (int)sop.cmode,
2996                      (int)sop.digest,
2997                      (int)sop.csum));
2998
2999         if (sop.cipher != cryptocop_cipher_none){
3000                 /* Init the cipher. */
3001                 switch (sop.cipher){
3002                 case cryptocop_cipher_des:
3003                         ti_cipher.alg = cryptocop_alg_des;
3004                         break;
3005                 case cryptocop_cipher_3des:
3006                         ti_cipher.alg = cryptocop_alg_3des;
3007                         break;
3008                 case cryptocop_cipher_aes:
3009                         ti_cipher.alg = cryptocop_alg_aes;
3010                         break;
3011                 default:
3012                         DEBUG_API(printk("create session, bad cipher algorithm %d\n", sop.cipher));
3013                         return -EINVAL;
3014                 };
3015                 DEBUG(printk("setting cipher transform %d\n", ti_cipher.alg));
3016                 copy_from_user(ti_cipher.key, sop.key, sop.keylen/8);
3017                 ti_cipher.keylen = sop.keylen;
3018                 switch (sop.cmode){
3019                 case cryptocop_cipher_mode_cbc:
3020                 case cryptocop_cipher_mode_ecb:
3021                         ti_cipher.cipher_mode = sop.cmode;
3022                         break;
3023                 default:
3024                         DEBUG_API(printk("create session, bad cipher mode %d\n", sop.cmode));
3025                         return -EINVAL;
3026                 }
3027                 DEBUG(printk("cryptocop_ioctl_create_session: setting CBC mode %d\n", ti_cipher.cipher_mode));
3028                 switch (sop.des3_mode){
3029                 case cryptocop_3des_eee:
3030                 case cryptocop_3des_eed:
3031                 case cryptocop_3des_ede:
3032                 case cryptocop_3des_edd:
3033                 case cryptocop_3des_dee:
3034                 case cryptocop_3des_ded:
3035                 case cryptocop_3des_dde:
3036                 case cryptocop_3des_ddd:
3037                         ti_cipher.tdes_mode = sop.des3_mode;
3038                         break;
3039                 default:
3040                         DEBUG_API(printk("create session, bad 3DES mode %d\n", sop.des3_mode));
3041                         return -EINVAL;
3042                 }
3043                 ti_cipher.tid = CRYPTOCOP_IOCTL_CIPHER_TID;
3044                 ti_cipher.next = tis;
3045                 tis = &ti_cipher;
3046         } /* if (sop.cipher != cryptocop_cipher_none) */
3047         if (sop.digest != cryptocop_digest_none){
3048                 DEBUG(printk("setting digest transform\n"));
3049                 switch (sop.digest){
3050                 case cryptocop_digest_md5:
3051                         ti_digest.alg = cryptocop_alg_md5;
3052                         break;
3053                 case cryptocop_digest_sha1:
3054                         ti_digest.alg = cryptocop_alg_sha1;
3055                         break;
3056                 default:
3057                         DEBUG_API(printk("create session, bad digest algorithm %d\n", sop.digest));
3058                         return -EINVAL;
3059                 }
3060                 ti_digest.tid = CRYPTOCOP_IOCTL_DIGEST_TID;
3061                 ti_digest.next = tis;
3062                 tis = &ti_digest;
3063         } /* if (sop.digest != cryptocop_digest_none) */
3064         if (sop.csum != cryptocop_csum_none){
3065                 DEBUG(printk("setting csum transform\n"));
3066                 switch (sop.csum){
3067                 case cryptocop_csum_le:
3068                 case cryptocop_csum_be:
3069                         ti_csum.csum_mode = sop.csum;
3070                         break;
3071                 default:
3072                         DEBUG_API(printk("create session, bad checksum algorithm %d\n", sop.csum));
3073                         return -EINVAL;
3074                 }
3075                 ti_csum.alg = cryptocop_alg_csum;
3076                 ti_csum.tid = CRYPTOCOP_IOCTL_CSUM_TID;
3077                 ti_csum.next = tis;
3078                 tis = &ti_csum;
3079         } /* (sop.csum != cryptocop_csum_none) */
3080         dev = kmalloc(sizeof(struct cryptocop_private), GFP_KERNEL);
3081         if (!dev){
3082                 DEBUG_API(printk("create session, alloc dev\n"));
3083                 return -ENOMEM;
3084         }
3085
3086         err = cryptocop_new_session(&sid, tis, GFP_KERNEL);
3087         DEBUG({ if (err) printk("create session, cryptocop_new_session %d\n", err);});
3088
3089         if (err) {
3090                 kfree(dev);
3091                 return err;
3092         }
3093         sess_op->ses_id = sid;
3094         dev->sid = sid;
3095         dev->next = filp->private_data;
3096         filp->private_data = dev;
3097
3098         return 0;
3099 }
3100
3101 static int cryptocop_ioctl(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
3102 {
3103         int err = 0;
3104         if (_IOC_TYPE(cmd) != ETRAXCRYPTOCOP_IOCTYPE) {
3105                 DEBUG_API(printk("cryptocop_ioctl: wrong type\n"));
3106                 return -ENOTTY;
3107         }
3108         if (_IOC_NR(cmd) > CRYPTOCOP_IO_MAXNR){
3109                 return -ENOTTY;
3110         }
3111         /* Access check of the argument.  Some commands, e.g. create session and process op,
3112            needs additional checks.  Those are handled in the command handling functions. */
3113         if (_IOC_DIR(cmd) & _IOC_READ)
3114                 err = !access_ok(VERIFY_WRITE, (void *)arg, _IOC_SIZE(cmd));
3115         else if (_IOC_DIR(cmd) & _IOC_WRITE)
3116                 err = !access_ok(VERIFY_READ, (void *)arg, _IOC_SIZE(cmd));
3117         if (err) return -EFAULT;
3118
3119         switch (cmd) {
3120         case CRYPTOCOP_IO_CREATE_SESSION:
3121                 return cryptocop_ioctl_create_session(inode, filp, cmd, arg);
3122         case CRYPTOCOP_IO_CLOSE_SESSION:
3123                 return cryptocop_ioctl_close_session(inode, filp, cmd, arg);
3124         case CRYPTOCOP_IO_PROCESS_OP:
3125                 return cryptocop_ioctl_process(inode, filp, cmd, arg);
3126         default:
3127                 DEBUG_API(printk("cryptocop_ioctl: unknown command\n"));
3128                 return -ENOTTY;
3129         }
3130         return 0;
3131 }
3132
3133
3134 #ifdef LDEBUG
3135 static void print_dma_descriptors(struct cryptocop_int_operation *iop)
3136 {
3137         struct cryptocop_dma_desc *cdesc_out = iop->cdesc_out;
3138         struct cryptocop_dma_desc *cdesc_in = iop->cdesc_in;
3139         int                       i;
3140
3141         printk("print_dma_descriptors start\n");
3142
3143         printk("iop:\n");
3144         printk("\tsid: 0x%lld\n", iop->sid);
3145
3146         printk("\tcdesc_out: 0x%p\n", iop->cdesc_out);
3147         printk("\tcdesc_in: 0x%p\n", iop->cdesc_in);
3148         printk("\tddesc_out: 0x%p\n", iop->ddesc_out);
3149         printk("\tddesc_in: 0x%p\n", iop->ddesc_in);
3150
3151         printk("\niop->ctx_out: 0x%p phys: 0x%p\n", &iop->ctx_out, (char*)virt_to_phys(&iop->ctx_out));
3152         printk("\tnext: 0x%p\n"
3153                "\tsaved_data: 0x%p\n"
3154                "\tsaved_data_buf: 0x%p\n",
3155                iop->ctx_out.next,
3156                iop->ctx_out.saved_data,
3157                iop->ctx_out.saved_data_buf);
3158
3159         printk("\niop->ctx_in: 0x%p phys: 0x%p\n", &iop->ctx_in, (char*)virt_to_phys(&iop->ctx_in));
3160         printk("\tnext: 0x%p\n"
3161                "\tsaved_data: 0x%p\n"
3162                "\tsaved_data_buf: 0x%p\n",
3163                iop->ctx_in.next,
3164                iop->ctx_in.saved_data,
3165                iop->ctx_in.saved_data_buf);
3166
3167         i = 0;
3168         while (cdesc_out) {
3169                 dma_descr_data *td;
3170                 printk("cdesc_out %d, desc=0x%p\n", i, cdesc_out->dma_descr);
3171                 printk("\n\tvirt_to_phys(desc): 0x%p\n", (char*)virt_to_phys(cdesc_out->dma_descr));
3172                 td = cdesc_out->dma_descr;
3173                 printk("\n\tbuf: 0x%p\n"
3174                        "\tafter: 0x%p\n"
3175                        "\tmd: 0x%04x\n"
3176                        "\tnext: 0x%p\n",
3177                        td->buf,
3178                        td->after,
3179                        td->md,
3180                        td->next);
3181                 printk("flags:\n"
3182                        "\twait:\t%d\n"
3183                        "\teol:\t%d\n"
3184                        "\touteop:\t%d\n"
3185                        "\tineop:\t%d\n"
3186                        "\tintr:\t%d\n",
3187                        td->wait,
3188                        td->eol,
3189                        td->out_eop,
3190                        td->in_eop,
3191                        td->intr);
3192                 cdesc_out = cdesc_out->next;
3193                 i++;
3194         }
3195         i = 0;
3196         while (cdesc_in) {
3197                 dma_descr_data *td;
3198                 printk("cdesc_in %d, desc=0x%p\n", i, cdesc_in->dma_descr);
3199                 printk("\n\tvirt_to_phys(desc): 0x%p\n", (char*)virt_to_phys(cdesc_in->dma_descr));
3200                 td = cdesc_in->dma_descr;
3201                 printk("\n\tbuf: 0x%p\n"
3202                        "\tafter: 0x%p\n"
3203                        "\tmd: 0x%04x\n"
3204                        "\tnext: 0x%p\n",
3205                        td->buf,
3206                        td->after,
3207                        td->md,
3208                        td->next);
3209                 printk("flags:\n"
3210                        "\twait:\t%d\n"
3211                        "\teol:\t%d\n"
3212                        "\touteop:\t%d\n"
3213                        "\tineop:\t%d\n"
3214                        "\tintr:\t%d\n",
3215                        td->wait,
3216                        td->eol,
3217                        td->out_eop,
3218                        td->in_eop,
3219                        td->intr);
3220                 cdesc_in = cdesc_in->next;
3221                 i++;
3222         }
3223
3224         printk("print_dma_descriptors end\n");
3225 }
3226
3227
3228 static void print_strcop_crypto_op(struct strcop_crypto_op *cop)
3229 {
3230         printk("print_strcop_crypto_op, 0x%p\n", cop);
3231
3232         /* Indata. */
3233         printk("indata=0x%p\n"
3234                "inlen=%d\n"
3235                "do_cipher=%d\n"
3236                "decrypt=%d\n"
3237                "cipher_explicit=%d\n"
3238                "cipher_start=%d\n"
3239                "cipher_len=%d\n"
3240                "outdata=0x%p\n"
3241                "outlen=%d\n",
3242                cop->indata,
3243                cop->inlen,
3244                cop->do_cipher,
3245                cop->decrypt,
3246                cop->cipher_explicit,
3247                cop->cipher_start,
3248                cop->cipher_len,
3249                cop->cipher_outdata,
3250                cop->cipher_outlen);
3251
3252         printk("do_digest=%d\n"
3253                "digest_start=%d\n"
3254                "digest_len=%d\n",
3255                cop->do_digest,
3256                cop->digest_start,
3257                cop->digest_len);
3258
3259         printk("do_csum=%d\n"
3260                "csum_start=%d\n"
3261                "csum_len=%d\n",
3262                cop->do_csum,
3263                cop->csum_start,
3264                cop->csum_len);
3265 }
3266
3267 static void print_cryptocop_operation(struct cryptocop_operation *cop)
3268 {
3269         struct cryptocop_desc      *d;
3270         struct cryptocop_tfrm_cfg  *tc;
3271         struct cryptocop_desc_cfg  *dc;
3272         int                        i;
3273
3274         printk("print_cryptocop_operation, cop=0x%p\n\n", cop);
3275         printk("sid: %lld\n", cop->sid);
3276         printk("operation_status=%d\n"
3277                "use_dmalists=%d\n"
3278                "in_interrupt=%d\n"
3279                "fast_callback=%d\n",
3280                cop->operation_status,
3281                cop->use_dmalists,
3282                cop->in_interrupt,
3283                cop->fast_callback);
3284
3285         if (cop->use_dmalists){
3286                 print_user_dma_lists(&cop->list_op);
3287         } else {
3288                 printk("cop->tfrm_op\n"
3289                        "tfrm_cfg=0x%p\n"
3290                        "desc=0x%p\n"
3291                        "indata=0x%p\n"
3292                        "incount=%d\n"
3293                        "inlen=%d\n"
3294                        "outdata=0x%p\n"
3295                        "outcount=%d\n"
3296                        "outlen=%d\n\n",
3297                        cop->tfrm_op.tfrm_cfg,
3298                        cop->tfrm_op.desc,
3299                        cop->tfrm_op.indata,
3300                        cop->tfrm_op.incount,
3301                        cop->tfrm_op.inlen,
3302                        cop->tfrm_op.outdata,
3303                        cop->tfrm_op.outcount,
3304                        cop->tfrm_op.outlen);
3305
3306                 tc = cop->tfrm_op.tfrm_cfg;
3307                 while (tc){
3308                         printk("tfrm_cfg, 0x%p\n"
3309                                "tid=%d\n"
3310                                "flags=%d\n"
3311                                "inject_ix=%d\n"
3312                                "next=0x%p\n",
3313                                tc,
3314                                tc->tid,
3315                                tc->flags,
3316                                tc->inject_ix,
3317                                tc->next);
3318                         tc = tc->next;
3319                 }
3320                 d = cop->tfrm_op.desc;
3321                 while (d){
3322                         printk("\n======================desc, 0x%p\n"
3323                                "length=%d\n"
3324                                "cfg=0x%p\n"
3325                                "next=0x%p\n",
3326                                d,
3327                                d->length,
3328                                d->cfg,
3329                                d->next);
3330                         dc = d->cfg;
3331                         while (dc){
3332                                 printk("=========desc_cfg, 0x%p\n"
3333                                        "tid=%d\n"
3334                                        "src=%d\n"
3335                                        "last=%d\n"
3336                                        "next=0x%p\n",
3337                                        dc,
3338                                        dc->tid,
3339                                        dc->src,
3340                                        dc->last,
3341                                        dc->next);
3342                                 dc = dc->next;
3343                         }
3344                         d = d->next;
3345                 }
3346                 printk("\n====iniov\n");
3347                 for (i = 0; i < cop->tfrm_op.incount; i++){
3348                         printk("indata[%d]\n"
3349                                "base=0x%p\n"
3350                                "len=%d\n",
3351                                i,
3352                                cop->tfrm_op.indata[i].iov_base,
3353                                cop->tfrm_op.indata[i].iov_len);
3354                 }
3355                 printk("\n====outiov\n");
3356                 for (i = 0; i < cop->tfrm_op.outcount; i++){
3357                         printk("outdata[%d]\n"
3358                                "base=0x%p\n"
3359                                "len=%d\n",
3360                                i,
3361                                cop->tfrm_op.outdata[i].iov_base,
3362                                cop->tfrm_op.outdata[i].iov_len);
3363                 }
3364         }
3365         printk("------------end print_cryptocop_operation\n");
3366 }
3367
3368
3369 static void print_user_dma_lists(struct cryptocop_dma_list_operation *dma_op)
3370 {
3371         dma_descr_data *dd;
3372         int i;
3373
3374         printk("print_user_dma_lists, dma_op=0x%p\n", dma_op);
3375
3376         printk("out_data_buf = 0x%p, phys_to_virt(out_data_buf) = 0x%p\n", dma_op->out_data_buf, phys_to_virt((unsigned long int)dma_op->out_data_buf));
3377         printk("in_data_buf = 0x%p, phys_to_virt(in_data_buf) = 0x%p\n", dma_op->in_data_buf, phys_to_virt((unsigned long int)dma_op->in_data_buf));
3378
3379         printk("##############outlist\n");
3380         dd = phys_to_virt((unsigned long int)dma_op->outlist);
3381         i = 0;
3382         while (dd != NULL) {
3383                 printk("#%d phys_to_virt(desc) 0x%p\n", i, dd);
3384                 printk("\n\tbuf: 0x%p\n"
3385                        "\tafter: 0x%p\n"
3386                        "\tmd: 0x%04x\n"
3387                        "\tnext: 0x%p\n",
3388                        dd->buf,
3389                        dd->after,
3390                        dd->md,
3391                        dd->next);
3392                 printk("flags:\n"
3393                        "\twait:\t%d\n"
3394                        "\teol:\t%d\n"
3395                        "\touteop:\t%d\n"
3396                        "\tineop:\t%d\n"
3397                        "\tintr:\t%d\n",
3398                        dd->wait,
3399                        dd->eol,
3400                        dd->out_eop,
3401                        dd->in_eop,
3402                        dd->intr);
3403                 if (dd->eol)
3404                         dd = NULL;
3405                 else
3406                         dd = phys_to_virt((unsigned long int)dd->next);
3407                 ++i;
3408         }
3409
3410         printk("##############inlist\n");
3411         dd = phys_to_virt((unsigned long int)dma_op->inlist);
3412         i = 0;
3413         while (dd != NULL) {
3414                 printk("#%d phys_to_virt(desc) 0x%p\n", i, dd);
3415                 printk("\n\tbuf: 0x%p\n"
3416                        "\tafter: 0x%p\n"
3417                        "\tmd: 0x%04x\n"
3418                        "\tnext: 0x%p\n",
3419                        dd->buf,
3420                        dd->after,
3421                        dd->md,
3422                        dd->next);
3423                 printk("flags:\n"
3424                        "\twait:\t%d\n"
3425                        "\teol:\t%d\n"
3426                        "\touteop:\t%d\n"
3427                        "\tineop:\t%d\n"
3428                        "\tintr:\t%d\n",
3429                        dd->wait,
3430                        dd->eol,
3431                        dd->out_eop,
3432                        dd->in_eop,
3433                        dd->intr);
3434                 if (dd->eol)
3435                         dd = NULL;
3436                 else
3437                         dd = phys_to_virt((unsigned long int)dd->next);
3438                 ++i;
3439         }
3440 }
3441
3442
3443 static void print_lock_status(void)
3444 {
3445         printk("**********************print_lock_status\n");
3446         printk("cryptocop_completed_jobs_lock %d\n", spin_is_locked(&cryptocop_completed_jobs_lock));
3447         printk("cryptocop_job_queue_lock %d\n", spin_is_locked(&cryptocop_job_queue_lock));
3448         printk("descr_pool_lock %d\n", spin_is_locked(&descr_pool_lock));
3449         printk("cryptocop_sessions_lock %d\n", spin_is_locked(cryptocop_sessions_lock));
3450         printk("running_job_lock %d\n", spin_is_locked(running_job_lock));
3451         printk("cryptocop_process_lock %d\n", spin_is_locked(cryptocop_process_lock));
3452 }
3453 #endif /* LDEBUG */
3454
3455
3456 static const char cryptocop_name[] = "ETRAX FS stream co-processor";
3457
3458 static int init_stream_coprocessor(void)
3459 {
3460         int err;
3461         int i;
3462         static int initialized = 0;
3463
3464         if (initialized)
3465                 return 0;
3466
3467         initialized = 1;
3468
3469         printk("ETRAX FS stream co-processor driver v0.01, (c) 2003 Axis Communications AB\n");
3470
3471         err = register_chrdev(CRYPTOCOP_MAJOR, cryptocop_name, &cryptocop_fops);
3472         if (err < 0) {
3473                 printk(KERN_ERR "stream co-processor: could not get major number.\n");
3474                 return err;
3475         }
3476
3477         err = init_cryptocop();
3478         if (err) {
3479                 (void)unregister_chrdev(CRYPTOCOP_MAJOR, cryptocop_name);
3480                 return err;
3481         }
3482         err = cryptocop_job_queue_init();
3483         if (err) {
3484                 release_cryptocop();
3485                 (void)unregister_chrdev(CRYPTOCOP_MAJOR, cryptocop_name);
3486                 return err;
3487         }
3488         /* Init the descriptor pool. */
3489         for (i = 0; i < CRYPTOCOP_DESCRIPTOR_POOL_SIZE - 1; i++) {
3490                 descr_pool[i].from_pool = 1;
3491                 descr_pool[i].next = &descr_pool[i + 1];
3492         }
3493         descr_pool[i].from_pool = 1;
3494         descr_pool[i].next = NULL;
3495         descr_pool_free_list = &descr_pool[0];
3496         descr_pool_no_free = CRYPTOCOP_DESCRIPTOR_POOL_SIZE;
3497
3498         spin_lock_init(&cryptocop_completed_jobs_lock);
3499         spin_lock_init(&cryptocop_job_queue_lock);
3500         spin_lock_init(&descr_pool_lock);
3501         spin_lock_init(&cryptocop_sessions_lock);
3502         spin_lock_init(&running_job_lock);
3503         spin_lock_init(&cryptocop_process_lock);
3504
3505         cryptocop_sessions = NULL;
3506         next_sid = 1;
3507
3508         cryptocop_running_job = NULL;
3509
3510         printk("stream co-processor: init done.\n");
3511         return 0;
3512 }
3513
3514 static void __exit exit_stream_coprocessor(void)
3515 {
3516         release_cryptocop();
3517         cryptocop_job_queue_close();
3518 }
3519
3520 module_init(init_stream_coprocessor);
3521 module_exit(exit_stream_coprocessor);
3522