diff options
author | Stephen Rothwell <sfr@canb.auug.org.au> | 2010-01-18 14:26:42 +1100 |
---|---|---|
committer | Stephen Rothwell <sfr@canb.auug.org.au> | 2010-01-18 14:26:42 +1100 |
commit | a21030ec6223c043a55a7a6e1934a8c947beee5e (patch) | |
tree | 4d2c1017eca5de7a1c7f84bbee99c14ff200a0ad /crypto | |
parent | a66aad20d0ee2c6d73a1ac73dc2b2ce08efabd45 (diff) | |
parent | 7d6f75eb21b84cdc5dfb09789974f02b42a89058 (diff) |
Merge remote branch 'crypto/master'
Diffstat (limited to 'crypto')
-rw-r--r-- | crypto/Kconfig | 10 | ||||
-rw-r--r-- | crypto/Makefile | 1 | ||||
-rw-r--r-- | crypto/gcm.c | 287 | ||||
-rw-r--r-- | crypto/md5.c | 40 | ||||
-rw-r--r-- | crypto/pcrypt.c | 445 | ||||
-rw-r--r-- | crypto/testmgr.c | 84 |
6 files changed, 851 insertions, 16 deletions
diff --git a/crypto/Kconfig b/crypto/Kconfig index 81c185a6971f..6a2e295ee227 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -114,6 +114,16 @@ config CRYPTO_NULL help These are 'Null' algorithms, used by IPsec, which do nothing. +config CRYPTO_PCRYPT + tristate "Parallel crypto engine (EXPERIMENTAL)" + depends on SMP && EXPERIMENTAL + select PADATA + select CRYPTO_MANAGER + select CRYPTO_AEAD + help + This converts an arbitrary crypto algorithm into a parallel + algorithm that executes in kernel threads. + config CRYPTO_WORKQUEUE tristate diff --git a/crypto/Makefile b/crypto/Makefile index 9e8f61908cb5..d7e6441df7fe 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -56,6 +56,7 @@ obj-$(CONFIG_CRYPTO_XTS) += xts.o obj-$(CONFIG_CRYPTO_CTR) += ctr.o obj-$(CONFIG_CRYPTO_GCM) += gcm.o obj-$(CONFIG_CRYPTO_CCM) += ccm.o +obj-$(CONFIG_CRYPTO_PCRYPT) += pcrypt.o obj-$(CONFIG_CRYPTO_CRYPTD) += cryptd.o obj-$(CONFIG_CRYPTO_DES) += des_generic.o obj-$(CONFIG_CRYPTO_FCRYPT) += fcrypt.o diff --git a/crypto/gcm.c b/crypto/gcm.c index c6547130624c..2f5fbba6576c 100644 --- a/crypto/gcm.c +++ b/crypto/gcm.c @@ -37,6 +37,19 @@ struct crypto_rfc4106_ctx { u8 nonce[4]; }; +struct crypto_rfc4543_ctx { + struct crypto_aead *child; + u8 nonce[4]; +}; + +struct crypto_rfc4543_req_ctx { + u8 auth_tag[16]; + struct scatterlist cipher[1]; + struct scatterlist payload[2]; + struct scatterlist assoc[2]; + struct aead_request subreq; +}; + struct crypto_gcm_ghash_ctx { unsigned int cryptlen; struct scatterlist *src; @@ -1047,6 +1060,272 @@ static struct crypto_template crypto_rfc4106_tmpl = { .module = THIS_MODULE, }; +static inline struct crypto_rfc4543_req_ctx *crypto_rfc4543_reqctx( + struct aead_request *req) +{ + unsigned long align = crypto_aead_alignmask(crypto_aead_reqtfm(req)); + + return (void *)PTR_ALIGN((u8 *)aead_request_ctx(req), align + 1); +} + +static int crypto_rfc4543_setkey(struct crypto_aead *parent, const u8 *key, + unsigned int keylen) +{ + struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(parent); + struct crypto_aead *child = ctx->child; + int err; + + if (keylen < 4) + return -EINVAL; + + keylen -= 4; + memcpy(ctx->nonce, key + keylen, 4); + + crypto_aead_clear_flags(child, CRYPTO_TFM_REQ_MASK); + crypto_aead_set_flags(child, crypto_aead_get_flags(parent) & + CRYPTO_TFM_REQ_MASK); + err = crypto_aead_setkey(child, key, keylen); + crypto_aead_set_flags(parent, crypto_aead_get_flags(child) & + CRYPTO_TFM_RES_MASK); + + return err; +} + +static int crypto_rfc4543_setauthsize(struct crypto_aead *parent, + unsigned int authsize) +{ + struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(parent); + + if (authsize != 16) + return -EINVAL; + + return crypto_aead_setauthsize(ctx->child, authsize); +} + +/* this is the same as crypto_authenc_chain */ +static void crypto_rfc4543_chain(struct scatterlist *head, + struct scatterlist *sg, int chain) +{ + if (chain) { + head->length += sg->length; + sg = scatterwalk_sg_next(sg); + } + + if (sg) + scatterwalk_sg_chain(head, 2, sg); + else + sg_mark_end(head); +} + +static struct aead_request *crypto_rfc4543_crypt(struct aead_request *req, + int enc) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(aead); + struct crypto_rfc4543_req_ctx *rctx = crypto_rfc4543_reqctx(req); + struct aead_request *subreq = &rctx->subreq; + struct scatterlist *dst = req->dst; + struct scatterlist *cipher = rctx->cipher; + struct scatterlist *payload = rctx->payload; + struct scatterlist *assoc = rctx->assoc; + unsigned int authsize = crypto_aead_authsize(aead); + unsigned int assoclen = req->assoclen; + struct page *dstp; + u8 *vdst; + u8 *iv = PTR_ALIGN((u8 *)(rctx + 1) + crypto_aead_reqsize(ctx->child), + crypto_aead_alignmask(ctx->child) + 1); + + memcpy(iv, ctx->nonce, 4); + memcpy(iv + 4, req->iv, 8); + + /* construct cipher/plaintext */ + if (enc) + memset(rctx->auth_tag, 0, authsize); + else + scatterwalk_map_and_copy(rctx->auth_tag, dst, + req->cryptlen - authsize, + authsize, 0); + + sg_init_one(cipher, rctx->auth_tag, authsize); + + /* construct the aad */ + dstp = sg_page(dst); + vdst = PageHighMem(dstp) ? NULL : page_address(dstp) + dst->offset; + + sg_init_table(payload, 2); + sg_set_buf(payload, req->iv, 8); + crypto_rfc4543_chain(payload, dst, vdst == req->iv + 8); + assoclen += 8 + req->cryptlen - (enc ? 0 : authsize); + + sg_init_table(assoc, 2); + sg_set_page(assoc, sg_page(req->assoc), req->assoc->length, + req->assoc->offset); + crypto_rfc4543_chain(assoc, payload, 0); + + aead_request_set_tfm(subreq, ctx->child); + aead_request_set_callback(subreq, req->base.flags, req->base.complete, + req->base.data); + aead_request_set_crypt(subreq, cipher, cipher, enc ? 0 : authsize, iv); + aead_request_set_assoc(subreq, assoc, assoclen); + + return subreq; +} + +static int crypto_rfc4543_encrypt(struct aead_request *req) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct crypto_rfc4543_req_ctx *rctx = crypto_rfc4543_reqctx(req); + struct aead_request *subreq; + int err; + + subreq = crypto_rfc4543_crypt(req, 1); + err = crypto_aead_encrypt(subreq); + if (err) + return err; + + scatterwalk_map_and_copy(rctx->auth_tag, req->dst, req->cryptlen, + crypto_aead_authsize(aead), 1); + + return 0; +} + +static int crypto_rfc4543_decrypt(struct aead_request *req) +{ + req = crypto_rfc4543_crypt(req, 0); + + return crypto_aead_decrypt(req); +} + +static int crypto_rfc4543_init_tfm(struct crypto_tfm *tfm) +{ + struct crypto_instance *inst = (void *)tfm->__crt_alg; + struct crypto_aead_spawn *spawn = crypto_instance_ctx(inst); + struct crypto_rfc4543_ctx *ctx = crypto_tfm_ctx(tfm); + struct crypto_aead *aead; + unsigned long align; + + aead = crypto_spawn_aead(spawn); + if (IS_ERR(aead)) + return PTR_ERR(aead); + + ctx->child = aead; + + align = crypto_aead_alignmask(aead); + align &= ~(crypto_tfm_ctx_alignment() - 1); + tfm->crt_aead.reqsize = sizeof(struct crypto_rfc4543_req_ctx) + + ALIGN(crypto_aead_reqsize(aead), + crypto_tfm_ctx_alignment()) + + align + 16; + + return 0; +} + +static void crypto_rfc4543_exit_tfm(struct crypto_tfm *tfm) +{ + struct crypto_rfc4543_ctx *ctx = crypto_tfm_ctx(tfm); + + crypto_free_aead(ctx->child); +} + +static struct crypto_instance *crypto_rfc4543_alloc(struct rtattr **tb) +{ + struct crypto_attr_type *algt; + struct crypto_instance *inst; + struct crypto_aead_spawn *spawn; + struct crypto_alg *alg; + const char *ccm_name; + int err; + + algt = crypto_get_attr_type(tb); + err = PTR_ERR(algt); + if (IS_ERR(algt)) + return ERR_PTR(err); + + if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask) + return ERR_PTR(-EINVAL); + + ccm_name = crypto_attr_alg_name(tb[1]); + err = PTR_ERR(ccm_name); + if (IS_ERR(ccm_name)) + return ERR_PTR(err); + + inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL); + if (!inst) + return ERR_PTR(-ENOMEM); + + spawn = crypto_instance_ctx(inst); + crypto_set_aead_spawn(spawn, inst); + err = crypto_grab_aead(spawn, ccm_name, 0, + crypto_requires_sync(algt->type, algt->mask)); + if (err) + goto out_free_inst; + + alg = crypto_aead_spawn_alg(spawn); + + err = -EINVAL; + + /* We only support 16-byte blocks. */ + if (alg->cra_aead.ivsize != 16) + goto out_drop_alg; + + /* Not a stream cipher? */ + if (alg->cra_blocksize != 1) + goto out_drop_alg; + + err = -ENAMETOOLONG; + if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME, + "rfc4543(%s)", alg->cra_name) >= CRYPTO_MAX_ALG_NAME || + snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, + "rfc4543(%s)", alg->cra_driver_name) >= + CRYPTO_MAX_ALG_NAME) + goto out_drop_alg; + + inst->alg.cra_flags = CRYPTO_ALG_TYPE_AEAD; + inst->alg.cra_flags |= alg->cra_flags & CRYPTO_ALG_ASYNC; + inst->alg.cra_priority = alg->cra_priority; + inst->alg.cra_blocksize = 1; + inst->alg.cra_alignmask = alg->cra_alignmask; + inst->alg.cra_type = &crypto_nivaead_type; + + inst->alg.cra_aead.ivsize = 8; + inst->alg.cra_aead.maxauthsize = 16; + + inst->alg.cra_ctxsize = sizeof(struct crypto_rfc4543_ctx); + + inst->alg.cra_init = crypto_rfc4543_init_tfm; + inst->alg.cra_exit = crypto_rfc4543_exit_tfm; + + inst->alg.cra_aead.setkey = crypto_rfc4543_setkey; + inst->alg.cra_aead.setauthsize = crypto_rfc4543_setauthsize; + inst->alg.cra_aead.encrypt = crypto_rfc4543_encrypt; + inst->alg.cra_aead.decrypt = crypto_rfc4543_decrypt; + + inst->alg.cra_aead.geniv = "seqiv"; + +out: + return inst; + +out_drop_alg: + crypto_drop_aead(spawn); +out_free_inst: + kfree(inst); + inst = ERR_PTR(err); + goto out; +} + +static void crypto_rfc4543_free(struct crypto_instance *inst) +{ + crypto_drop_spawn(crypto_instance_ctx(inst)); + kfree(inst); +} + +static struct crypto_template crypto_rfc4543_tmpl = { + .name = "rfc4543", + .alloc = crypto_rfc4543_alloc, + .free = crypto_rfc4543_free, + .module = THIS_MODULE, +}; + static int __init crypto_gcm_module_init(void) { int err; @@ -1067,8 +1346,14 @@ static int __init crypto_gcm_module_init(void) if (err) goto out_undo_gcm; + err = crypto_register_template(&crypto_rfc4543_tmpl); + if (err) + goto out_undo_rfc4106; + return 0; +out_undo_rfc4106: + crypto_unregister_template(&crypto_rfc4106_tmpl); out_undo_gcm: crypto_unregister_template(&crypto_gcm_tmpl); out_undo_base: @@ -1081,6 +1366,7 @@ out: static void __exit crypto_gcm_module_exit(void) { kfree(gcm_zeroes); + crypto_unregister_template(&crypto_rfc4543_tmpl); crypto_unregister_template(&crypto_rfc4106_tmpl); crypto_unregister_template(&crypto_gcm_tmpl); crypto_unregister_template(&crypto_gcm_base_tmpl); @@ -1094,3 +1380,4 @@ MODULE_DESCRIPTION("Galois/Counter Mode"); MODULE_AUTHOR("Mikko Herranen <mh1@iki.fi>"); MODULE_ALIAS("gcm_base"); MODULE_ALIAS("rfc4106"); +MODULE_ALIAS("rfc4543"); diff --git a/crypto/md5.c b/crypto/md5.c index 83eb52961750..9fda213a592e 100644 --- a/crypto/md5.c +++ b/crypto/md5.c @@ -16,17 +16,13 @@ * */ #include <crypto/internal/hash.h> +#include <crypto/md5.h> #include <linux/init.h> #include <linux/module.h> #include <linux/string.h> #include <linux/types.h> #include <asm/byteorder.h> -#define MD5_DIGEST_SIZE 16 -#define MD5_HMAC_BLOCK_SIZE 64 -#define MD5_BLOCK_WORDS 16 -#define MD5_HASH_WORDS 4 - #define F1(x, y, z) (z ^ (x & (y ^ z))) #define F2(x, y, z) F1(z, x, y) #define F3(x, y, z) (x ^ y ^ z) @@ -35,12 +31,6 @@ #define MD5STEP(f, w, x, y, z, in, s) \ (w += f(x, y, z) + in, w = (w<<s | w>>(32-s)) + x) -struct md5_ctx { - u32 hash[MD5_HASH_WORDS]; - u32 block[MD5_BLOCK_WORDS]; - u64 byte_count; -}; - static void md5_transform(u32 *hash, u32 const *in) { u32 a, b, c, d; @@ -141,7 +131,7 @@ static inline void cpu_to_le32_array(u32 *buf, unsigned int words) } } -static inline void md5_transform_helper(struct md5_ctx *ctx) +static inline void md5_transform_helper(struct md5_state *ctx) { le32_to_cpu_array(ctx->block, sizeof(ctx->block) / sizeof(u32)); md5_transform(ctx->hash, ctx->block); @@ -149,7 +139,7 @@ static inline void md5_transform_helper(struct md5_ctx *ctx) static int md5_init(struct shash_desc *desc) { - struct md5_ctx *mctx = shash_desc_ctx(desc); + struct md5_state *mctx = shash_desc_ctx(desc); mctx->hash[0] = 0x67452301; mctx->hash[1] = 0xefcdab89; @@ -162,7 +152,7 @@ static int md5_init(struct shash_desc *desc) static int md5_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - struct md5_ctx *mctx = shash_desc_ctx(desc); + struct md5_state *mctx = shash_desc_ctx(desc); const u32 avail = sizeof(mctx->block) - (mctx->byte_count & 0x3f); mctx->byte_count += len; @@ -194,7 +184,7 @@ static int md5_update(struct shash_desc *desc, const u8 *data, unsigned int len) static int md5_final(struct shash_desc *desc, u8 *out) { - struct md5_ctx *mctx = shash_desc_ctx(desc); + struct md5_state *mctx = shash_desc_ctx(desc); const unsigned int offset = mctx->byte_count & 0x3f; char *p = (char *)mctx->block + offset; int padding = 56 - (offset + 1); @@ -220,12 +210,30 @@ static int md5_final(struct shash_desc *desc, u8 *out) return 0; } +static int md5_export(struct shash_desc *desc, void *out) +{ + struct md5_state *ctx = shash_desc_ctx(desc); + + memcpy(out, ctx, sizeof(*ctx)); + return 0; +} + +static int md5_import(struct shash_desc *desc, const void *in) +{ + struct md5_state *ctx = shash_desc_ctx(desc); + + memcpy(ctx, in, sizeof(*ctx)); + return 0; +} + static struct shash_alg alg = { .digestsize = MD5_DIGEST_SIZE, .init = md5_init, .update = md5_update, .final = md5_final, - .descsize = sizeof(struct md5_ctx), + .export = md5_export, + .import = md5_import, + .descsize = sizeof(struct md5_state), .base = { .cra_name = "md5", .cra_flags = CRYPTO_ALG_TYPE_SHASH, diff --git a/crypto/pcrypt.c b/crypto/pcrypt.c new file mode 100644 index 000000000000..b9527d05e17d --- /dev/null +++ b/crypto/pcrypt.c @@ -0,0 +1,445 @@ +/* + * pcrypt - Parallel crypto wrapper. + * + * Copyright (C) 2009 secunet Security Networks AG + * Copyright (C) 2009 Steffen Klassert <steffen.klassert@secunet.com> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include <crypto/algapi.h> +#include <crypto/internal/aead.h> +#include <linux/err.h> +#include <linux/init.h> +#include <linux/module.h> +#include <linux/slab.h> +#include <crypto/pcrypt.h> + +static struct padata_instance *pcrypt_enc_padata; +static struct padata_instance *pcrypt_dec_padata; +static struct workqueue_struct *encwq; +static struct workqueue_struct *decwq; + +struct pcrypt_instance_ctx { + struct crypto_spawn spawn; + unsigned int tfm_count; +}; + +struct pcrypt_aead_ctx { + struct crypto_aead *child; + unsigned int cb_cpu; +}; + +static int pcrypt_do_parallel(struct padata_priv *padata, unsigned int *cb_cpu, + struct padata_instance *pinst) +{ + unsigned int cpu_index, cpu, i; + + cpu = *cb_cpu; + + if (cpumask_test_cpu(cpu, cpu_active_mask)) + goto out; + + cpu_index = cpu % cpumask_weight(cpu_active_mask); + + cpu = cpumask_first(cpu_active_mask); + for (i = 0; i < cpu_index; i++) + cpu = cpumask_next(cpu, cpu_active_mask); + + *cb_cpu = cpu; + +out: + return padata_do_parallel(pinst, padata, cpu); +} + +static int pcrypt_aead_setkey(struct crypto_aead *parent, + const u8 *key, unsigned int keylen) +{ + struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(parent); + + return crypto_aead_setkey(ctx->child, key, keylen); +} + +static int pcrypt_aead_setauthsize(struct crypto_aead *parent, + unsigned int authsize) +{ + struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(parent); + + return crypto_aead_setauthsize(ctx->child, authsize); +} + +static void pcrypt_aead_serial(struct padata_priv *padata) +{ + struct pcrypt_request *preq = pcrypt_padata_request(padata); + struct aead_request *req = pcrypt_request_ctx(preq); + + aead_request_complete(req->base.data, padata->info); +} + +static void pcrypt_aead_giv_serial(struct padata_priv *padata) +{ + struct pcrypt_request *preq = pcrypt_padata_request(padata); + struct aead_givcrypt_request *req = pcrypt_request_ctx(preq); + + aead_request_complete(req->areq.base.data, padata->info); +} + +static void pcrypt_aead_done(struct crypto_async_request *areq, int err) +{ + struct aead_request *req = areq->data; + struct pcrypt_request *preq = aead_request_ctx(req); + struct padata_priv *padata = pcrypt_request_padata(preq); + + padata->info = err; + req->base.flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + + padata_do_serial(padata); +} + +static void pcrypt_aead_enc(struct padata_priv *padata) +{ + struct pcrypt_request *preq = pcrypt_padata_request(padata); + struct aead_request *req = pcrypt_request_ctx(preq); + + padata->info = crypto_aead_encrypt(req); + + if (padata->info) + return; + + padata_do_serial(padata); +} + +static int pcrypt_aead_encrypt(struct aead_request *req) +{ + int err; + struct pcrypt_request *preq = aead_request_ctx(req); + struct aead_request *creq = pcrypt_request_ctx(preq); + struct padata_priv *padata = pcrypt_request_padata(preq); + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(aead); + u32 flags = aead_request_flags(req); + + memset(padata, 0, sizeof(struct padata_priv)); + + padata->parallel = pcrypt_aead_enc; + padata->serial = pcrypt_aead_serial; + + aead_request_set_tfm(creq, ctx->child); + aead_request_set_callback(creq, flags & ~CRYPTO_TFM_REQ_MAY_SLEEP, + pcrypt_aead_done, req); + aead_request_set_crypt(creq, req->src, req->dst, + req->cryptlen, req->iv); + aead_request_set_assoc(creq, req->assoc, req->assoclen); + + err = pcrypt_do_parallel(padata, &ctx->cb_cpu, pcrypt_enc_padata); + if (err) + return err; + else + err = crypto_aead_encrypt(creq); + + return err; +} + +static void pcrypt_aead_dec(struct padata_priv *padata) +{ + struct pcrypt_request *preq = pcrypt_padata_request(padata); + struct aead_request *req = pcrypt_request_ctx(preq); + + padata->info = crypto_aead_decrypt(req); + + if (padata->info) + return; + + padata_do_serial(padata); +} + +static int pcrypt_aead_decrypt(struct aead_request *req) +{ + int err; + struct pcrypt_request *preq = aead_request_ctx(req); + struct aead_request *creq = pcrypt_request_ctx(preq); + struct padata_priv *padata = pcrypt_request_padata(preq); + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(aead); + u32 flags = aead_request_flags(req); + + memset(padata, 0, sizeof(struct padata_priv)); + + padata->parallel = pcrypt_aead_dec; + padata->serial = pcrypt_aead_serial; + + aead_request_set_tfm(creq, ctx->child); + aead_request_set_callback(creq, flags & ~CRYPTO_TFM_REQ_MAY_SLEEP, + pcrypt_aead_done, req); + aead_request_set_crypt(creq, req->src, req->dst, + req->cryptlen, req->iv); + aead_request_set_assoc(creq, req->assoc, req->assoclen); + + err = pcrypt_do_parallel(padata, &ctx->cb_cpu, pcrypt_dec_padata); + if (err) + return err; + else + err = crypto_aead_decrypt(creq); + + return err; +} + +static void pcrypt_aead_givenc(struct padata_priv *padata) +{ + struct pcrypt_request *preq = pcrypt_padata_request(padata); + struct aead_givcrypt_request *req = pcrypt_request_ctx(preq); + + padata->info = crypto_aead_givencrypt(req); + + if (padata->info) + return; + + padata_do_serial(padata); +} + +static int pcrypt_aead_givencrypt(struct aead_givcrypt_request *req) +{ + int err; + struct aead_request *areq = &req->areq; + struct pcrypt_request *preq = aead_request_ctx(areq); + struct aead_givcrypt_request *creq = pcrypt_request_ctx(preq); + struct padata_priv *padata = pcrypt_request_padata(preq); + struct crypto_aead *aead = aead_givcrypt_reqtfm(req); + struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(aead); + u32 flags = aead_request_flags(areq); + + memset(padata, 0, sizeof(struct padata_priv)); + + padata->parallel = pcrypt_aead_givenc; + padata->serial = pcrypt_aead_giv_serial; + + aead_givcrypt_set_tfm(creq, ctx->child); + aead_givcrypt_set_callback(creq, flags & ~CRYPTO_TFM_REQ_MAY_SLEEP, + pcrypt_aead_done, areq); + aead_givcrypt_set_crypt(creq, areq->src, areq->dst, + areq->cryptlen, areq->iv); + aead_givcrypt_set_assoc(creq, areq->assoc, areq->assoclen); + aead_givcrypt_set_giv(creq, req->giv, req->seq); + + err = pcrypt_do_parallel(padata, &ctx->cb_cpu, pcrypt_enc_padata); + if (err) + return err; + else + err = crypto_aead_givencrypt(creq); + + return err; +} + +static int pcrypt_aead_init_tfm(struct crypto_tfm *tfm) +{ + int cpu, cpu_index; + struct crypto_instance *inst = crypto_tfm_alg_instance(tfm); + struct pcrypt_instance_ctx *ictx = crypto_instance_ctx(inst); + struct pcrypt_aead_ctx *ctx = crypto_tfm_ctx(tfm); + struct crypto_aead *cipher; + + ictx->tfm_count++; + + cpu_index = ictx->tfm_count % cpumask_weight(cpu_active_mask); + + ctx->cb_cpu = cpumask_first(cpu_active_mask); + for (cpu = 0; cpu < cpu_index; cpu++) + ctx->cb_cpu = cpumask_next(ctx->cb_cpu, cpu_active_mask); + + cipher = crypto_spawn_aead(crypto_instance_ctx(inst)); + + if (IS_ERR(cipher)) + return PTR_ERR(cipher); + + ctx->child = cipher; + tfm->crt_aead.reqsize = sizeof(struct pcrypt_request) + + sizeof(struct aead_givcrypt_request) + + crypto_aead_reqsize(cipher); + + return 0; +} + +static void pcrypt_aead_exit_tfm(struct crypto_tfm *tfm) +{ + struct pcrypt_aead_ctx *ctx = crypto_tfm_ctx(tfm); + + crypto_free_aead(ctx->child); +} + +static struct crypto_instance *pcrypt_alloc_instance(struct crypto_alg *alg) +{ + struct crypto_instance *inst; + struct pcrypt_instance_ctx *ctx; + int err; + + inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); + if (!inst) { + inst = ERR_PTR(-ENOMEM); + goto out; + } + + err = -ENAMETOOLONG; + if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, + "pcrypt(%s)", alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME) + goto out_free_inst; + + memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME); + + ctx = crypto_instance_ctx(inst); + err = crypto_init_spawn(&ctx->spawn, alg, inst, + CRYPTO_ALG_TYPE_MASK); + if (err) + goto out_free_inst; + + inst->alg.cra_priority = alg->cra_priority + 100; + inst->alg.cra_blocksize = alg->cra_blocksize; + inst->alg.cra_alignmask = alg->cra_alignmask; + +out: + return inst; + +out_free_inst: + kfree(inst); + inst = ERR_PTR(err); + goto out; +} + +static struct crypto_instance *pcrypt_alloc_aead(struct rtattr **tb) +{ + struct crypto_instance *inst; + struct crypto_alg *alg; + struct crypto_attr_type *algt; + + algt = crypto_get_attr_type(tb); + + alg = crypto_get_attr_alg(tb, algt->type, + (algt->mask & CRYPTO_ALG_TYPE_MASK)); + if (IS_ERR(alg)) + return ERR_CAST(alg); + + inst = pcrypt_alloc_instance(alg); + if (IS_ERR(inst)) + goto out_put_alg; + + inst->alg.cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC; + inst->alg.cra_type = &crypto_aead_type; + + inst->alg.cra_aead.ivsize = alg->cra_aead.ivsize; + inst->alg.cra_aead.geniv = alg->cra_aead.geniv; + inst->alg.cra_aead.maxauthsize = alg->cra_aead.maxauthsize; + + inst->alg.cra_ctxsize = sizeof(struct pcrypt_aead_ctx); + + inst->alg.cra_init = pcrypt_aead_init_tfm; + inst->alg.cra_exit = pcrypt_aead_exit_tfm; + + inst->alg.cra_aead.setkey = pcrypt_aead_setkey; + inst->alg.cra_aead.setauthsize = pcrypt_aead_setauthsize; + inst->alg.cra_aead.encrypt = pcrypt_aead_encrypt; + inst->alg.cra_aead.decrypt = pcrypt_aead_decrypt; + inst->alg.cra_aead.givencrypt = pcrypt_aead_givencrypt; + +out_put_alg: + crypto_mod_put(alg); + return inst; +} + +static struct crypto_instance *pcrypt_alloc(struct rtattr **tb) +{ + struct crypto_attr_type *algt; + + algt = crypto_get_attr_type(tb); + if (IS_ERR(algt)) + return ERR_CAST(algt); + + switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) { + case CRYPTO_ALG_TYPE_AEAD: + return pcrypt_alloc_aead(tb); + } + + return ERR_PTR(-EINVAL); +} + +static void pcrypt_free(struct crypto_instance *inst) +{ + struct pcrypt_instance_ctx *ctx = crypto_instance_ctx(inst); + + crypto_drop_spawn(&ctx->spawn); + kfree(inst); +} + +static struct crypto_template pcrypt_tmpl = { + .name = "pcrypt", + .alloc = pcrypt_alloc, + .free = pcrypt_free, + .module = THIS_MODULE, +}; + +static int __init pcrypt_init(void) +{ + encwq = create_workqueue("pencrypt"); + if (!encwq) + goto err; + + decwq = create_workqueue("pdecrypt"); + if (!decwq) + goto err_destroy_encwq; + + + pcrypt_enc_padata = padata_alloc(cpu_possible_mask, encwq); + if (!pcrypt_enc_padata) + goto err_destroy_decwq; + + pcrypt_dec_padata = padata_alloc(cpu_possible_mask, decwq); + if (!pcrypt_dec_padata) + goto err_free_padata; + + padata_start(pcrypt_enc_padata); + padata_start(pcrypt_dec_padata); + + return crypto_register_template(&pcrypt_tmpl); + +err_free_padata: + padata_free(pcrypt_enc_padata); + +err_destroy_decwq: + destroy_workqueue(decwq); + +err_destroy_encwq: + destroy_workqueue(encwq); + +err: + return -ENOMEM; +} + +static void __exit pcrypt_exit(void) +{ + padata_stop(pcrypt_enc_padata); + padata_stop(pcrypt_dec_padata); + + destroy_workqueue(encwq); + destroy_workqueue(decwq); + + padata_free(pcrypt_enc_padata); + padata_free(pcrypt_dec_padata); + + crypto_unregister_template(&pcrypt_tmpl); +} + +module_init(pcrypt_init); +module_exit(pcrypt_exit); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Steffen Klassert <steffen.klassert@secunet.com>"); +MODULE_DESCRIPTION("Parallel crypto wrapper"); diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 7620bfce92f2..c494d7610be1 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -1477,9 +1477,54 @@ static int alg_test_cprng(const struct alg_test_desc *desc, const char *driver, return err; } +static int alg_test_null(const struct alg_test_desc *desc, + const char *driver, u32 type, u32 mask) +{ + return 0; +} + /* Please keep this list sorted by algorithm name. */ static const struct alg_test_desc alg_test_descs[] = { { + .alg = "__driver-cbc-aes-aesni", + .test = alg_test_null, + .suite = { + .cipher = { + .enc = { + .vecs = NULL, + .count = 0 + }, + .dec = { + .vecs = NULL, + .count = 0 + } + } + } + }, { + .alg = "__driver-ecb-aes-aesni", + .test = alg_test_null, + .suite = { + .cipher = { + .enc = { + .vecs = NULL, + .count = 0 + }, + .dec = { + .vecs = NULL, + .count = 0 + } + } + } + }, { + .alg = "__ghash-pclmulqdqni", + .test = alg_test_null, + .suite = { + .hash = { + .vecs = NULL, + .count = 0 + } + } + }, { .alg = "ansi_cprng", .test = alg_test_cprng, .fips_allowed = 1, @@ -1623,6 +1668,30 @@ static const struct alg_test_desc alg_test_descs[] = { } } }, { + .alg = "cryptd(__driver-ecb-aes-aesni)", + .test = alg_test_null, + .suite = { + .cipher = { + .enc = { + .vecs = NULL, + .count = 0 + }, + .dec = { + .vecs = NULL, + .count = 0 + } + } + } + }, { + .alg = "cryptd(__ghash-pclmulqdqni)", + .test = alg_test_null, + .suite = { + .hash = { + .vecs = NULL, + .count = 0 + } + } + }, { .alg = "ctr(aes)", .test = alg_test_skcipher, .fips_allowed = 1, @@ -1669,6 +1738,21 @@ static const struct alg_test_desc alg_test_descs[] = { } } }, { + .alg = "ecb(__aes-aesni)", + .test = alg_test_null, + .suite = { + .cipher = { + .enc = { + .vecs = NULL, + .count = 0 + }, + .dec = { + .vecs = NULL, + .count = 0 + } + } + } + }, { .alg = "ecb(aes)", .test = alg_test_skcipher, .fips_allowed = 1, |