* ---------------------------------------------------------------------------
*/
+#include <crypto/algapi.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/types.h>
#include <linux/errno.h>
-#include <linux/crypto.h>
#include <linux/interrupt.h>
+#include <linux/kernel.h>
#include <asm/byteorder.h>
#include "padlock.h"
#define AES_EXTENDED_KEY_SIZE 64 /* in uint32_t units */
#define AES_EXTENDED_KEY_SIZE_B (AES_EXTENDED_KEY_SIZE * sizeof(uint32_t))
+/* Control word. */
+struct cword {
+ unsigned int __attribute__ ((__packed__))
+ rounds:4,
+ algo:3,
+ keygen:1,
+ interm:1,
+ encdec:1,
+ ksize:2;
+} __attribute__ ((__aligned__(PADLOCK_ALIGNMENT)));
+
+/* Whenever making any changes to the following
+ * structure *make sure* you keep E, d_data
+ * and cword aligned on 16 Bytes boundaries!!! */
struct aes_ctx {
- uint32_t e_data[AES_EXTENDED_KEY_SIZE+4];
- uint32_t d_data[AES_EXTENDED_KEY_SIZE+4];
- uint32_t *E;
- uint32_t *D;
+ struct {
+ struct cword encrypt;
+ struct cword decrypt;
+ } cword;
+ u32 *D;
int key_length;
+ u32 E[AES_EXTENDED_KEY_SIZE]
+ __attribute__ ((__aligned__(PADLOCK_ALIGNMENT)));
+ u32 d_data[AES_EXTENDED_KEY_SIZE]
+ __attribute__ ((__aligned__(PADLOCK_ALIGNMENT)));
};
/* ====== Key management routines ====== */
return x >> (n << 3);
}
-#define uint32_t_in(x) le32_to_cpu(*(const uint32_t *)(x))
-#define uint32_t_out(to, from) (*(uint32_t *)(to) = cpu_to_le32(from))
-
#define E_KEY ctx->E
#define D_KEY ctx->D
return 0;
}
-static int
-aes_set_key(void *ctx_arg, const uint8_t *in_key, unsigned int key_len, uint32_t *flags)
+static inline struct aes_ctx *aes_ctx_common(void *ctx)
+{
+ unsigned long addr = (unsigned long)ctx;
+ unsigned long align = PADLOCK_ALIGNMENT;
+
+ if (align <= crypto_tfm_ctx_alignment())
+ align = 1;
+ return (struct aes_ctx *)ALIGN(addr, align);
+}
+
+static inline struct aes_ctx *aes_ctx(struct crypto_tfm *tfm)
{
- struct aes_ctx *ctx = ctx_arg;
+ return aes_ctx_common(crypto_tfm_ctx(tfm));
+}
+
+static inline struct aes_ctx *blk_aes_ctx(struct crypto_blkcipher *tfm)
+{
+ return aes_ctx_common(crypto_blkcipher_ctx(tfm));
+}
+
+static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
+ unsigned int key_len)
+{
+ struct aes_ctx *ctx = aes_ctx(tfm);
+ const __le32 *key = (const __le32 *)in_key;
+ u32 *flags = &tfm->crt_flags;
uint32_t i, t, u, v, w;
uint32_t P[AES_EXTENDED_KEY_SIZE];
uint32_t rounds;
- if (key_len != 16 && key_len != 24 && key_len != 32) {
+ if (key_len % 8) {
*flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
return -EINVAL;
}
ctx->key_length = key_len;
- ctx->E = ctx->e_data;
- ctx->D = ctx->d_data;
+ /*
+ * If the hardware is capable of generating the extended key
+ * itself we must supply the plain key for both encryption
+ * and decryption.
+ */
+ ctx->D = ctx->E;
- /* Ensure 16-Bytes alignmentation of keys for VIA PadLock. */
- if ((int)(ctx->e_data) & 0x0F)
- ctx->E += 4 - (((int)(ctx->e_data) & 0x0F) / sizeof (ctx->e_data[0]));
+ E_KEY[0] = le32_to_cpu(key[0]);
+ E_KEY[1] = le32_to_cpu(key[1]);
+ E_KEY[2] = le32_to_cpu(key[2]);
+ E_KEY[3] = le32_to_cpu(key[3]);
- if ((int)(ctx->d_data) & 0x0F)
- ctx->D += 4 - (((int)(ctx->d_data) & 0x0F) / sizeof (ctx->d_data[0]));
+ /* Prepare control words. */
+ memset(&ctx->cword, 0, sizeof(ctx->cword));
- E_KEY[0] = uint32_t_in (in_key);
- E_KEY[1] = uint32_t_in (in_key + 4);
- E_KEY[2] = uint32_t_in (in_key + 8);
- E_KEY[3] = uint32_t_in (in_key + 12);
+ ctx->cword.decrypt.encdec = 1;
+ ctx->cword.encrypt.rounds = 10 + (key_len - 16) / 4;
+ ctx->cword.decrypt.rounds = ctx->cword.encrypt.rounds;
+ ctx->cword.encrypt.ksize = (key_len - 16) / 8;
+ ctx->cword.decrypt.ksize = ctx->cword.encrypt.ksize;
/* Don't generate extended keys if the hardware can do it. */
if (aes_hw_extkey_available(key_len))
return 0;
+ ctx->D = ctx->d_data;
+ ctx->cword.encrypt.keygen = 1;
+ ctx->cword.decrypt.keygen = 1;
+
switch (key_len) {
case 16:
t = E_KEY[3];
break;
case 24:
- E_KEY[4] = uint32_t_in (in_key + 16);
- t = E_KEY[5] = uint32_t_in (in_key + 20);
+ E_KEY[4] = le32_to_cpu(key[4]);
+ t = E_KEY[5] = le32_to_cpu(key[5]);
for (i = 0; i < 8; ++i)
loop6 (i);
break;
case 32:
- E_KEY[4] = uint32_t_in (in_key + 16);
- E_KEY[5] = uint32_t_in (in_key + 20);
- E_KEY[6] = uint32_t_in (in_key + 24);
- t = E_KEY[7] = uint32_t_in (in_key + 28);
+ E_KEY[4] = le32_to_cpu(key[4]);
+ E_KEY[5] = le32_to_cpu(key[5]);
+ E_KEY[6] = le32_to_cpu(key[6]);
+ t = E_KEY[7] = le32_to_cpu(key[7]);
for (i = 0; i < 7; ++i)
loop8 (i);
break;
/* ====== Encryption/decryption routines ====== */
-/* This is the real call to PadLock. */
-static inline void
-padlock_xcrypt_ecb(uint8_t *input, uint8_t *output, uint8_t *key,
- void *control_word, uint32_t count)
+/* These are the real call to PadLock. */
+static inline void padlock_xcrypt_ecb(const u8 *input, u8 *output, void *key,
+ void *control_word, u32 count)
{
asm volatile ("pushfl; popfl"); /* enforce key reload. */
asm volatile (".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */
: "d"(control_word), "b"(key), "c"(count));
}
-static void
-aes_padlock(void *ctx_arg, uint8_t *out_arg, const uint8_t *in_arg, int encdec)
+static inline u8 *padlock_xcrypt_cbc(const u8 *input, u8 *output, void *key,
+ u8 *iv, void *control_word, u32 count)
{
- /* Don't blindly modify this structure - the items must
- fit on 16-Bytes boundaries! */
- struct padlock_xcrypt_data {
- uint8_t buf[AES_BLOCK_SIZE];
- union cword cword;
- };
-
- struct aes_ctx *ctx = ctx_arg;
- char bigbuf[sizeof(struct padlock_xcrypt_data) + 16];
- struct padlock_xcrypt_data *data;
- void *key;
-
- /* Place 'data' at the first 16-Bytes aligned address in 'bigbuf'. */
- if (((long)bigbuf) & 0x0F)
- data = (void*)(bigbuf + 16 - ((long)bigbuf & 0x0F));
- else
- data = (void*)bigbuf;
-
- /* Prepare Control word. */
- memset (data, 0, sizeof(struct padlock_xcrypt_data));
- data->cword.b.encdec = !encdec; /* in the rest of cryptoapi ENC=1/DEC=0 */
- data->cword.b.rounds = 10 + (ctx->key_length - 16) / 4;
- data->cword.b.ksize = (ctx->key_length - 16) / 8;
-
- /* Is the hardware capable to generate the extended key? */
- if (!aes_hw_extkey_available(ctx->key_length))
- data->cword.b.keygen = 1;
-
- /* ctx->E starts with a plain key - if the hardware is capable
- to generate the extended key itself we must supply
- the plain key for both Encryption and Decryption. */
- if (encdec == CRYPTO_DIR_ENCRYPT || data->cword.b.keygen == 0)
- key = ctx->E;
- else
- key = ctx->D;
-
- memcpy(data->buf, in_arg, AES_BLOCK_SIZE);
- padlock_xcrypt_ecb(data->buf, data->buf, key, &data->cword, 1);
- memcpy(out_arg, data->buf, AES_BLOCK_SIZE);
+ /* Enforce key reload. */
+ asm volatile ("pushfl; popfl");
+ /* rep xcryptcbc */
+ asm volatile (".byte 0xf3,0x0f,0xa7,0xd0"
+ : "+S" (input), "+D" (output), "+a" (iv)
+ : "d" (control_word), "b" (key), "c" (count));
+ return iv;
}
-static void
-aes_encrypt(void *ctx_arg, uint8_t *out, const uint8_t *in)
+static void aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
{
- aes_padlock(ctx_arg, out, in, CRYPTO_DIR_ENCRYPT);
+ struct aes_ctx *ctx = aes_ctx(tfm);
+ padlock_xcrypt_ecb(in, out, ctx->E, &ctx->cword.encrypt, 1);
}
-static void
-aes_decrypt(void *ctx_arg, uint8_t *out, const uint8_t *in)
+static void aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
{
- aes_padlock(ctx_arg, out, in, CRYPTO_DIR_DECRYPT);
+ struct aes_ctx *ctx = aes_ctx(tfm);
+ padlock_xcrypt_ecb(in, out, ctx->D, &ctx->cword.decrypt, 1);
}
static struct crypto_alg aes_alg = {
.cra_name = "aes",
+ .cra_driver_name = "aes-padlock",
+ .cra_priority = PADLOCK_CRA_PRIORITY,
.cra_flags = CRYPTO_ALG_TYPE_CIPHER,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct aes_ctx),
+ .cra_alignmask = PADLOCK_ALIGNMENT - 1,
.cra_module = THIS_MODULE,
.cra_list = LIST_HEAD_INIT(aes_alg.cra_list),
.cra_u = {
.cia_max_keysize = AES_MAX_KEY_SIZE,
.cia_setkey = aes_set_key,
.cia_encrypt = aes_encrypt,
- .cia_decrypt = aes_decrypt
+ .cia_decrypt = aes_decrypt,
}
}
};
-int __init padlock_init_aes(void)
+static int ecb_aes_encrypt(struct blkcipher_desc *desc,
+ struct scatterlist *dst, struct scatterlist *src,
+ unsigned int nbytes)
{
- printk(KERN_NOTICE PFX "Using VIA PadLock ACE for AES algorithm.\n");
+ struct aes_ctx *ctx = blk_aes_ctx(desc->tfm);
+ struct blkcipher_walk walk;
+ int err;
+
+ blkcipher_walk_init(&walk, dst, src, nbytes);
+ err = blkcipher_walk_virt(desc, &walk);
+
+ while ((nbytes = walk.nbytes)) {
+ padlock_xcrypt_ecb(walk.src.virt.addr, walk.dst.virt.addr,
+ ctx->E, &ctx->cword.encrypt,
+ nbytes / AES_BLOCK_SIZE);
+ nbytes &= AES_BLOCK_SIZE - 1;
+ err = blkcipher_walk_done(desc, &walk, nbytes);
+ }
+
+ return err;
+}
+
+static int ecb_aes_decrypt(struct blkcipher_desc *desc,
+ struct scatterlist *dst, struct scatterlist *src,
+ unsigned int nbytes)
+{
+ struct aes_ctx *ctx = blk_aes_ctx(desc->tfm);
+ struct blkcipher_walk walk;
+ int err;
+
+ blkcipher_walk_init(&walk, dst, src, nbytes);
+ err = blkcipher_walk_virt(desc, &walk);
+
+ while ((nbytes = walk.nbytes)) {
+ padlock_xcrypt_ecb(walk.src.virt.addr, walk.dst.virt.addr,
+ ctx->D, &ctx->cword.decrypt,
+ nbytes / AES_BLOCK_SIZE);
+ nbytes &= AES_BLOCK_SIZE - 1;
+ err = blkcipher_walk_done(desc, &walk, nbytes);
+ }
+
+ return err;
+}
+
+static struct crypto_alg ecb_aes_alg = {
+ .cra_name = "ecb(aes)",
+ .cra_driver_name = "ecb-aes-padlock",
+ .cra_priority = PADLOCK_COMPOSITE_PRIORITY,
+ .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct aes_ctx),
+ .cra_alignmask = PADLOCK_ALIGNMENT - 1,
+ .cra_type = &crypto_blkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_list = LIST_HEAD_INIT(ecb_aes_alg.cra_list),
+ .cra_u = {
+ .blkcipher = {
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .setkey = aes_set_key,
+ .encrypt = ecb_aes_encrypt,
+ .decrypt = ecb_aes_decrypt,
+ }
+ }
+};
+
+static int cbc_aes_encrypt(struct blkcipher_desc *desc,
+ struct scatterlist *dst, struct scatterlist *src,
+ unsigned int nbytes)
+{
+ struct aes_ctx *ctx = blk_aes_ctx(desc->tfm);
+ struct blkcipher_walk walk;
+ int err;
+
+ blkcipher_walk_init(&walk, dst, src, nbytes);
+ err = blkcipher_walk_virt(desc, &walk);
+
+ while ((nbytes = walk.nbytes)) {
+ u8 *iv = padlock_xcrypt_cbc(walk.src.virt.addr,
+ walk.dst.virt.addr, ctx->E,
+ walk.iv, &ctx->cword.encrypt,
+ nbytes / AES_BLOCK_SIZE);
+ memcpy(walk.iv, iv, AES_BLOCK_SIZE);
+ nbytes &= AES_BLOCK_SIZE - 1;
+ err = blkcipher_walk_done(desc, &walk, nbytes);
+ }
+
+ return err;
+}
+
+static int cbc_aes_decrypt(struct blkcipher_desc *desc,
+ struct scatterlist *dst, struct scatterlist *src,
+ unsigned int nbytes)
+{
+ struct aes_ctx *ctx = blk_aes_ctx(desc->tfm);
+ struct blkcipher_walk walk;
+ int err;
+
+ blkcipher_walk_init(&walk, dst, src, nbytes);
+ err = blkcipher_walk_virt(desc, &walk);
+
+ while ((nbytes = walk.nbytes)) {
+ padlock_xcrypt_cbc(walk.src.virt.addr, walk.dst.virt.addr,
+ ctx->D, walk.iv, &ctx->cword.decrypt,
+ nbytes / AES_BLOCK_SIZE);
+ nbytes &= AES_BLOCK_SIZE - 1;
+ err = blkcipher_walk_done(desc, &walk, nbytes);
+ }
+
+ return err;
+}
+
+static struct crypto_alg cbc_aes_alg = {
+ .cra_name = "cbc(aes)",
+ .cra_driver_name = "cbc-aes-padlock",
+ .cra_priority = PADLOCK_COMPOSITE_PRIORITY,
+ .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct aes_ctx),
+ .cra_alignmask = PADLOCK_ALIGNMENT - 1,
+ .cra_type = &crypto_blkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_list = LIST_HEAD_INIT(cbc_aes_alg.cra_list),
+ .cra_u = {
+ .blkcipher = {
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .setkey = aes_set_key,
+ .encrypt = cbc_aes_encrypt,
+ .decrypt = cbc_aes_decrypt,
+ }
+ }
+};
+
+static int __init padlock_init(void)
+{
+ int ret;
+
+ if (!cpu_has_xcrypt) {
+ printk(KERN_ERR PFX "VIA PadLock not detected.\n");
+ return -ENODEV;
+ }
+
+ if (!cpu_has_xcrypt_enabled) {
+ printk(KERN_ERR PFX "VIA PadLock detected, but not enabled. Hmm, strange...\n");
+ return -ENODEV;
+ }
gen_tabs();
- return crypto_register_alg(&aes_alg);
+ if ((ret = crypto_register_alg(&aes_alg)))
+ goto aes_err;
+
+ if ((ret = crypto_register_alg(&ecb_aes_alg)))
+ goto ecb_aes_err;
+
+ if ((ret = crypto_register_alg(&cbc_aes_alg)))
+ goto cbc_aes_err;
+
+ printk(KERN_NOTICE PFX "Using VIA PadLock ACE for AES algorithm.\n");
+
+out:
+ return ret;
+
+cbc_aes_err:
+ crypto_unregister_alg(&ecb_aes_alg);
+ecb_aes_err:
+ crypto_unregister_alg(&aes_alg);
+aes_err:
+ printk(KERN_ERR PFX "VIA PadLock AES initialization failed.\n");
+ goto out;
}
-void __exit padlock_fini_aes(void)
+static void __exit padlock_fini(void)
{
+ crypto_unregister_alg(&cbc_aes_alg);
+ crypto_unregister_alg(&ecb_aes_alg);
crypto_unregister_alg(&aes_alg);
}
+
+module_init(padlock_init);
+module_exit(padlock_fini);
+
+MODULE_DESCRIPTION("VIA PadLock AES algorithm support");
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Michal Ludvig");
+
+MODULE_ALIAS("aes-padlock");
git://git.onelab.eu / linux-2.6.git / blobdiff