diff options
Diffstat (limited to 'drivers/crypto')
160 files changed, 13336 insertions, 5277 deletions
diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 193204dfbf3a..47ec920d5b71 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -199,22 +199,8 @@ config CRYPTO_CRC32_S390 It is available with IBM z13 or later. -config CRYPTO_DEV_MV_CESA - tristate "Marvell's Cryptographic Engine" - depends on PLAT_ORION - select CRYPTO_AES - select CRYPTO_BLKCIPHER - select CRYPTO_HASH - select SRAM - help - This driver allows you to utilize the Cryptographic Engines and - Security Accelerator (CESA) which can be found on the Marvell Orion - and Kirkwood SoCs, such as QNAP's TS-209. - - Currently the driver supports AES in ECB and CBC mode without DMA. - config CRYPTO_DEV_MARVELL_CESA - tristate "New Marvell's Cryptographic Engine driver" + tristate "Marvell's Cryptographic Engine driver" depends on PLAT_ORION || ARCH_MVEBU select CRYPTO_AES select CRYPTO_DES @@ -223,12 +209,10 @@ config CRYPTO_DEV_MARVELL_CESA select SRAM help This driver allows you to utilize the Cryptographic Engines and - Security Accelerator (CESA) which can be found on the Armada 370. + Security Accelerator (CESA) which can be found on MVEBU and ORION + platforms. This driver supports CPU offload through DMA transfers. - This driver is aimed at replacing the mv_cesa driver. This will only - happen once it has received proper testing. - config CRYPTO_DEV_NIAGARA2 tristate "Niagara2 Stream Processing Unit driver" select CRYPTO_DES @@ -315,6 +299,10 @@ config CRYPTO_DEV_PPC4XX tristate "Driver AMCC PPC4xx crypto accelerator" depends on PPC && 4xx select CRYPTO_HASH + select CRYPTO_AEAD + select CRYPTO_AES + select CRYPTO_CCM + select CRYPTO_GCM select CRYPTO_BLKCIPHER help This option allows you to have support for AMCC crypto acceleration. @@ -439,6 +427,20 @@ config CRYPTO_DEV_S5P Select this to offload Samsung S5PV210 or S5PC110, Exynos from AES algorithms execution. +config CRYPTO_DEV_EXYNOS_HASH + bool "Support for Samsung Exynos HASH accelerator" + depends on CRYPTO_DEV_S5P + depends on !CRYPTO_DEV_EXYNOS_RNG && CRYPTO_DEV_EXYNOS_RNG!=m + select CRYPTO_SHA1 + select CRYPTO_MD5 + select CRYPTO_SHA256 + help + Select this to offload Exynos from HASH MD5/SHA1/SHA256. + This will select software SHA1, MD5 and SHA256 as they are + needed for small and zero-size messages. + HASH algorithms will be disabled if EXYNOS_RNG + is enabled due to hw conflict. + config CRYPTO_DEV_NX bool "Support for IBM PowerPC Nest (NX) cryptographic acceleration" depends on PPC64 @@ -525,12 +527,26 @@ config CRYPTO_DEV_ATMEL_SHA To compile this driver as a module, choose M here: the module will be called atmel-sha. +config CRYPTO_DEV_ATMEL_ECC + tristate "Support for Microchip / Atmel ECC hw accelerator" + depends on ARCH_AT91 || COMPILE_TEST + depends on I2C + select CRYPTO_ECDH + select CRC16 + help + Microhip / Atmel ECC hw accelerator. + Select this if you want to use the Microchip / Atmel module for + ECDH algorithm. + + To compile this driver as a module, choose M here: the module + will be called atmel-ecc. + config CRYPTO_DEV_CCP - bool "Support for AMD Cryptographic Coprocessor" + bool "Support for AMD Secure Processor" depends on ((X86 && PCI) || (ARM64 && (OF_ADDRESS || ACPI))) && HAS_IOMEM help - The AMD Cryptographic Coprocessor provides hardware offload support - for encryption, hashing and related operations. + The AMD Secure Processor provides support for the Cryptographic Coprocessor + (CCP) and the Platform Security Processor (PSP) devices. if CRYPTO_DEV_CCP source "drivers/crypto/ccp/Kconfig" @@ -616,6 +632,14 @@ config CRYPTO_DEV_SUN4I_SS To compile this driver as a module, choose M here: the module will be called sun4i-ss. +config CRYPTO_DEV_SUN4I_SS_PRNG + bool "Support for Allwinner Security System PRNG" + depends on CRYPTO_DEV_SUN4I_SS + select CRYPTO_RNG + help + Select this option if you want to provide kernel-side support for + the Pseudo-Random Number Generator found in the Security System. + config CRYPTO_DEV_ROCKCHIP tristate "Rockchip's Cryptographic Engine driver" depends on OF && ARCH_ROCKCHIP @@ -655,7 +679,7 @@ source "drivers/crypto/virtio/Kconfig" config CRYPTO_DEV_BCM_SPU tristate "Broadcom symmetric crypto/hash acceleration support" depends on ARCH_BCM_IPROC - depends on BCM_PDC_MBOX + depends on MAILBOX default m select CRYPTO_DES select CRYPTO_MD5 @@ -686,4 +710,25 @@ config CRYPTO_DEV_SAFEXCEL chain mode, AES cipher mode and SHA1/SHA224/SHA256/SHA512 hash algorithms. +config CRYPTO_DEV_ARTPEC6 + tristate "Support for Axis ARTPEC-6/7 hardware crypto acceleration." + depends on ARM && (ARCH_ARTPEC || COMPILE_TEST) + depends on HAS_DMA + depends on OF + select CRYPTO_AEAD + select CRYPTO_AES + select CRYPTO_ALGAPI + select CRYPTO_BLKCIPHER + select CRYPTO_CTR + select CRYPTO_HASH + select CRYPTO_SHA1 + select CRYPTO_SHA256 + select CRYPTO_SHA384 + select CRYPTO_SHA512 + help + Enables the driver for the on-chip crypto accelerator + of Axis ARTPEC SoCs. + + To compile this driver as a module, choose M here. + endif # CRYPTO_HW diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile index 2c555a3393b2..2513d13ea2c4 100644 --- a/drivers/crypto/Makefile +++ b/drivers/crypto/Makefile @@ -1,6 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_CRYPTO_DEV_ATMEL_AES) += atmel-aes.o obj-$(CONFIG_CRYPTO_DEV_ATMEL_SHA) += atmel-sha.o obj-$(CONFIG_CRYPTO_DEV_ATMEL_TDES) += atmel-tdes.o +obj-$(CONFIG_CRYPTO_DEV_ATMEL_ECC) += atmel-ecc.o obj-$(CONFIG_CRYPTO_DEV_BFIN_CRC) += bfin_crc.o obj-$(CONFIG_CRYPTO_DEV_CAVIUM_ZIP) += cavium/ obj-$(CONFIG_CRYPTO_DEV_CCP) += ccp/ @@ -13,7 +15,6 @@ obj-$(CONFIG_CRYPTO_DEV_GEODE) += geode-aes.o obj-$(CONFIG_CRYPTO_DEV_HIFN_795X) += hifn_795x.o obj-$(CONFIG_CRYPTO_DEV_IMGTEC_HASH) += img-hash.o obj-$(CONFIG_CRYPTO_DEV_IXP4XX) += ixp4xx_crypto.o -obj-$(CONFIG_CRYPTO_DEV_MV_CESA) += mv_cesa.o obj-$(CONFIG_CRYPTO_DEV_MARVELL_CESA) += marvell/ obj-$(CONFIG_CRYPTO_DEV_MEDIATEK) += mediatek/ obj-$(CONFIG_CRYPTO_DEV_MXS_DCP) += mxs-dcp.o @@ -35,7 +36,7 @@ obj-$(CONFIG_CRYPTO_DEV_QCE) += qce/ obj-$(CONFIG_CRYPTO_DEV_ROCKCHIP) += rockchip/ obj-$(CONFIG_CRYPTO_DEV_S5P) += s5p-sss.o obj-$(CONFIG_CRYPTO_DEV_SAHARA) += sahara.o -obj-$(CONFIG_CRYPTO_DEV_STM32) += stm32/ +obj-$(CONFIG_ARCH_STM32) += stm32/ obj-$(CONFIG_CRYPTO_DEV_SUN4I_SS) += sunxi-ss/ obj-$(CONFIG_CRYPTO_DEV_TALITOS) += talitos.o obj-$(CONFIG_CRYPTO_DEV_UX500) += ux500/ @@ -43,3 +44,4 @@ obj-$(CONFIG_CRYPTO_DEV_VIRTIO) += virtio/ obj-$(CONFIG_CRYPTO_DEV_VMX) += vmx/ obj-$(CONFIG_CRYPTO_DEV_BCM_SPU) += bcm/ obj-$(CONFIG_CRYPTO_DEV_SAFEXCEL) += inside-secure/ +obj-$(CONFIG_CRYPTO_DEV_ARTPEC6) += axis/ diff --git a/drivers/crypto/amcc/Makefile b/drivers/crypto/amcc/Makefile index b95539928fdf..e33c185fc163 100644 --- a/drivers/crypto/amcc/Makefile +++ b/drivers/crypto/amcc/Makefile @@ -1,3 +1,3 @@ obj-$(CONFIG_CRYPTO_DEV_PPC4XX) += crypto4xx.o -crypto4xx-y := crypto4xx_core.o crypto4xx_alg.o crypto4xx_sa.o +crypto4xx-y := crypto4xx_core.o crypto4xx_alg.o crypto4xx-$(CONFIG_HW_RANDOM_PPC4XX) += crypto4xx_trng.o diff --git a/drivers/crypto/amcc/crypto4xx_alg.c b/drivers/crypto/amcc/crypto4xx_alg.c index 4afca3968773..eeaf27859d80 100644 --- a/drivers/crypto/amcc/crypto4xx_alg.c +++ b/drivers/crypto/amcc/crypto4xx_alg.c @@ -26,11 +26,14 @@ #include <crypto/internal/hash.h> #include <linux/dma-mapping.h> #include <crypto/algapi.h> +#include <crypto/aead.h> #include <crypto/aes.h> +#include <crypto/gcm.h> #include <crypto/sha.h> +#include <crypto/ctr.h> #include "crypto4xx_reg_def.h" -#include "crypto4xx_sa.h" #include "crypto4xx_core.h" +#include "crypto4xx_sa.h" static void set_dynamic_sa_command_0(struct dynamic_sa_ctl *sa, u32 save_h, u32 save_iv, u32 ld_h, u32 ld_iv, @@ -62,6 +65,7 @@ static void set_dynamic_sa_command_1(struct dynamic_sa_ctl *sa, u32 cm, sa->sa_command_1.bf.crypto_mode9_8 = cm & 3; sa->sa_command_1.bf.feedback_mode = cfb, sa->sa_command_1.bf.sa_rev = 1; + sa->sa_command_1.bf.hmac_muting = hmac_mc; sa->sa_command_1.bf.extended_seq_num = esn; sa->sa_command_1.bf.seq_num_mask = sn_mask; sa->sa_command_1.bf.mutable_bit_proc = mute; @@ -73,29 +77,29 @@ static void set_dynamic_sa_command_1(struct dynamic_sa_ctl *sa, u32 cm, int crypto4xx_encrypt(struct ablkcipher_request *req) { struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + unsigned int ivlen = crypto_ablkcipher_ivsize( + crypto_ablkcipher_reqtfm(req)); + __le32 iv[ivlen]; - ctx->direction = DIR_OUTBOUND; - ctx->hash_final = 0; - ctx->is_hash = 0; - ctx->pd_ctl = 0x1; + if (ivlen) + crypto4xx_memcpy_to_le32(iv, req->info, ivlen); return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst, - req->nbytes, req->info, - get_dynamic_sa_iv_size(ctx)); + req->nbytes, iv, ivlen, ctx->sa_out, ctx->sa_len, 0); } int crypto4xx_decrypt(struct ablkcipher_request *req) { struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + unsigned int ivlen = crypto_ablkcipher_ivsize( + crypto_ablkcipher_reqtfm(req)); + __le32 iv[ivlen]; - ctx->direction = DIR_INBOUND; - ctx->hash_final = 0; - ctx->is_hash = 0; - ctx->pd_ctl = 1; + if (ivlen) + crypto4xx_memcpy_to_le32(iv, req->info, ivlen); return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst, - req->nbytes, req->info, - get_dynamic_sa_iv_size(ctx)); + req->nbytes, iv, ivlen, ctx->sa_in, ctx->sa_len, 0); } /** @@ -120,23 +124,15 @@ static int crypto4xx_setkey_aes(struct crypto_ablkcipher *cipher, } /* Create SA */ - if (ctx->sa_in_dma_addr || ctx->sa_out_dma_addr) + if (ctx->sa_in || ctx->sa_out) crypto4xx_free_sa(ctx); rc = crypto4xx_alloc_sa(ctx, SA_AES128_LEN + (keylen-16) / 4); if (rc) return rc; - if (ctx->state_record_dma_addr == 0) { - rc = crypto4xx_alloc_state_record(ctx); - if (rc) { - crypto4xx_free_sa(ctx); - return rc; - } - } /* Setup SA */ - sa = (struct dynamic_sa_ctl *) ctx->sa_in; - ctx->hash_final = 0; + sa = ctx->sa_in; set_dynamic_sa_command_0(sa, SA_NOT_SAVE_HASH, SA_NOT_SAVE_IV, SA_LOAD_HASH_FROM_SA, SA_LOAD_IV_FROM_STATE, @@ -150,18 +146,13 @@ static int crypto4xx_setkey_aes(struct crypto_ablkcipher *cipher, SA_SEQ_MASK_OFF, SA_MC_ENABLE, SA_NOT_COPY_PAD, SA_NOT_COPY_PAYLOAD, SA_NOT_COPY_HDR); - crypto4xx_memcpy_le(ctx->sa_in + get_dynamic_sa_offset_key_field(ctx), - key, keylen); - sa->sa_contents = SA_AES_CONTENTS | (keylen << 2); + crypto4xx_memcpy_to_le32(get_dynamic_sa_key_field(sa), + key, keylen); + sa->sa_contents.w = SA_AES_CONTENTS | (keylen << 2); sa->sa_command_1.bf.key_len = keylen >> 3; - ctx->is_hash = 0; - ctx->direction = DIR_INBOUND; - memcpy(ctx->sa_in + get_dynamic_sa_offset_state_ptr_field(ctx), - (void *)&ctx->state_record_dma_addr, 4); - ctx->offset_to_sr_ptr = get_dynamic_sa_offset_state_ptr_field(ctx); memcpy(ctx->sa_out, ctx->sa_in, ctx->sa_len * 4); - sa = (struct dynamic_sa_ctl *) ctx->sa_out; + sa = ctx->sa_out; sa->sa_command_0.bf.dir = DIR_OUTBOUND; return 0; @@ -174,6 +165,396 @@ int crypto4xx_setkey_aes_cbc(struct crypto_ablkcipher *cipher, CRYPTO_FEEDBACK_MODE_NO_FB); } +int crypto4xx_setkey_aes_cfb(struct crypto_ablkcipher *cipher, + const u8 *key, unsigned int keylen) +{ + return crypto4xx_setkey_aes(cipher, key, keylen, CRYPTO_MODE_CFB, + CRYPTO_FEEDBACK_MODE_128BIT_CFB); +} + +int crypto4xx_setkey_aes_ecb(struct crypto_ablkcipher *cipher, + const u8 *key, unsigned int keylen) +{ + return crypto4xx_setkey_aes(cipher, key, keylen, CRYPTO_MODE_ECB, + CRYPTO_FEEDBACK_MODE_NO_FB); +} + +int crypto4xx_setkey_aes_ofb(struct crypto_ablkcipher *cipher, + const u8 *key, unsigned int keylen) +{ + return crypto4xx_setkey_aes(cipher, key, keylen, CRYPTO_MODE_OFB, + CRYPTO_FEEDBACK_MODE_64BIT_OFB); +} + +int crypto4xx_setkey_rfc3686(struct crypto_ablkcipher *cipher, + const u8 *key, unsigned int keylen) +{ + struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); + struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm); + int rc; + + rc = crypto4xx_setkey_aes(cipher, key, keylen - CTR_RFC3686_NONCE_SIZE, + CRYPTO_MODE_CTR, CRYPTO_FEEDBACK_MODE_NO_FB); + if (rc) + return rc; + + ctx->iv_nonce = cpu_to_le32p((u32 *)&key[keylen - + CTR_RFC3686_NONCE_SIZE]); + + return 0; +} + +int crypto4xx_rfc3686_encrypt(struct ablkcipher_request *req) +{ + struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + __le32 iv[AES_IV_SIZE / 4] = { + ctx->iv_nonce, + cpu_to_le32p((u32 *) req->info), + cpu_to_le32p((u32 *) (req->info + 4)), + cpu_to_le32(1) }; + + return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst, + req->nbytes, iv, AES_IV_SIZE, + ctx->sa_out, ctx->sa_len, 0); +} + +int crypto4xx_rfc3686_decrypt(struct ablkcipher_request *req) +{ + struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + __le32 iv[AES_IV_SIZE / 4] = { + ctx->iv_nonce, + cpu_to_le32p((u32 *) req->info), + cpu_to_le32p((u32 *) (req->info + 4)), + cpu_to_le32(1) }; + + return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst, + req->nbytes, iv, AES_IV_SIZE, + ctx->sa_out, ctx->sa_len, 0); +} + +static inline bool crypto4xx_aead_need_fallback(struct aead_request *req, + bool is_ccm, bool decrypt) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(req); + + /* authsize has to be a multiple of 4 */ + if (aead->authsize & 3) + return true; + + /* + * hardware does not handle cases where cryptlen + * is less than a block + */ + if (req->cryptlen < AES_BLOCK_SIZE) + return true; + + /* assoc len needs to be a multiple of 4 */ + if (req->assoclen & 0x3) + return true; + + /* CCM supports only counter field length of 2 and 4 bytes */ + if (is_ccm && !(req->iv[0] == 1 || req->iv[0] == 3)) + return true; + + /* CCM - fix CBC MAC mismatch in special case */ + if (is_ccm && decrypt && !req->assoclen) + return true; + + return false; +} + +static int crypto4xx_aead_fallback(struct aead_request *req, + struct crypto4xx_ctx *ctx, bool do_decrypt) +{ + char aead_req_data[sizeof(struct aead_request) + + crypto_aead_reqsize(ctx->sw_cipher.aead)] + __aligned(__alignof__(struct aead_request)); + + struct aead_request *subreq = (void *) aead_req_data; + + memset(subreq, 0, sizeof(aead_req_data)); + + aead_request_set_tfm(subreq, ctx->sw_cipher.aead); + aead_request_set_callback(subreq, req->base.flags, + req->base.complete, req->base.data); + aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, + req->iv); + aead_request_set_ad(subreq, req->assoclen); + return do_decrypt ? crypto_aead_decrypt(subreq) : + crypto_aead_encrypt(subreq); +} + +static int crypto4xx_setup_fallback(struct crypto4xx_ctx *ctx, + struct crypto_aead *cipher, + const u8 *key, + unsigned int keylen) +{ + int rc; + + crypto_aead_clear_flags(ctx->sw_cipher.aead, CRYPTO_TFM_REQ_MASK); + crypto_aead_set_flags(ctx->sw_cipher.aead, + crypto_aead_get_flags(cipher) & CRYPTO_TFM_REQ_MASK); + rc = crypto_aead_setkey(ctx->sw_cipher.aead, key, keylen); + crypto_aead_clear_flags(cipher, CRYPTO_TFM_RES_MASK); + crypto_aead_set_flags(cipher, + crypto_aead_get_flags(ctx->sw_cipher.aead) & + CRYPTO_TFM_RES_MASK); + + return rc; +} + +/** + * AES-CCM Functions + */ + +int crypto4xx_setkey_aes_ccm(struct crypto_aead *cipher, const u8 *key, + unsigned int keylen) +{ + struct crypto_tfm *tfm = crypto_aead_tfm(cipher); + struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm); + struct dynamic_sa_ctl *sa; + int rc = 0; + + rc = crypto4xx_setup_fallback(ctx, cipher, key, keylen); + if (rc) + return rc; + + if (ctx->sa_in || ctx->sa_out) + crypto4xx_free_sa(ctx); + + rc = crypto4xx_alloc_sa(ctx, SA_AES128_CCM_LEN + (keylen - 16) / 4); + if (rc) + return rc; + + /* Setup SA */ + sa = (struct dynamic_sa_ctl *) ctx->sa_in; + sa->sa_contents.w = SA_AES_CCM_CONTENTS | (keylen << 2); + + set_dynamic_sa_command_0(sa, SA_NOT_SAVE_HASH, SA_NOT_SAVE_IV, + SA_LOAD_HASH_FROM_SA, SA_LOAD_IV_FROM_STATE, + SA_NO_HEADER_PROC, SA_HASH_ALG_CBC_MAC, + SA_CIPHER_ALG_AES, + SA_PAD_TYPE_ZERO, SA_OP_GROUP_BASIC, + SA_OPCODE_HASH_DECRYPT, DIR_INBOUND); + + set_dynamic_sa_command_1(sa, CRYPTO_MODE_CTR, SA_HASH_MODE_HASH, + CRYPTO_FEEDBACK_MODE_NO_FB, SA_EXTENDED_SN_OFF, + SA_SEQ_MASK_OFF, SA_MC_ENABLE, + SA_NOT_COPY_PAD, SA_COPY_PAYLOAD, + SA_NOT_COPY_HDR); + + sa->sa_command_1.bf.key_len = keylen >> 3; + + crypto4xx_memcpy_to_le32(get_dynamic_sa_key_field(sa), key, keylen); + + memcpy(ctx->sa_out, ctx->sa_in, ctx->sa_len * 4); + sa = (struct dynamic_sa_ctl *) ctx->sa_out; + + set_dynamic_sa_command_0(sa, SA_SAVE_HASH, SA_NOT_SAVE_IV, + SA_LOAD_HASH_FROM_SA, SA_LOAD_IV_FROM_STATE, + SA_NO_HEADER_PROC, SA_HASH_ALG_CBC_MAC, + SA_CIPHER_ALG_AES, + SA_PAD_TYPE_ZERO, SA_OP_GROUP_BASIC, + SA_OPCODE_ENCRYPT_HASH, DIR_OUTBOUND); + + set_dynamic_sa_command_1(sa, CRYPTO_MODE_CTR, SA_HASH_MODE_HASH, + CRYPTO_FEEDBACK_MODE_NO_FB, SA_EXTENDED_SN_OFF, + SA_SEQ_MASK_OFF, SA_MC_ENABLE, + SA_COPY_PAD, SA_COPY_PAYLOAD, + SA_NOT_COPY_HDR); + + sa->sa_command_1.bf.key_len = keylen >> 3; + return 0; +} + +static int crypto4xx_crypt_aes_ccm(struct aead_request *req, bool decrypt) +{ + struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct crypto_aead *aead = crypto_aead_reqtfm(req); + unsigned int len = req->cryptlen; + __le32 iv[16]; + u32 tmp_sa[ctx->sa_len * 4]; + struct dynamic_sa_ctl *sa = (struct dynamic_sa_ctl *)tmp_sa; + + if (crypto4xx_aead_need_fallback(req, true, decrypt)) + return crypto4xx_aead_fallback(req, ctx, decrypt); + + if (decrypt) + len -= crypto_aead_authsize(aead); + + memcpy(tmp_sa, decrypt ? ctx->sa_in : ctx->sa_out, sizeof(tmp_sa)); + sa->sa_command_0.bf.digest_len = crypto_aead_authsize(aead) >> 2; + + if (req->iv[0] == 1) { + /* CRYPTO_MODE_AES_ICM */ + sa->sa_command_1.bf.crypto_mode9_8 = 1; + } + + iv[3] = cpu_to_le32(0); + crypto4xx_memcpy_to_le32(iv, req->iv, 16 - (req->iv[0] + 1)); + + return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst, + len, iv, sizeof(iv), + sa, ctx->sa_len, req->assoclen); +} + +int crypto4xx_encrypt_aes_ccm(struct aead_request *req) +{ + return crypto4xx_crypt_aes_ccm(req, false); +} + +int crypto4xx_decrypt_aes_ccm(struct aead_request *req) +{ + return crypto4xx_crypt_aes_ccm(req, true); +} + +int crypto4xx_setauthsize_aead(struct crypto_aead *cipher, + unsigned int authsize) +{ + struct crypto_tfm *tfm = crypto_aead_tfm(cipher); + struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm); + + return crypto_aead_setauthsize(ctx->sw_cipher.aead, authsize); +} + +/** + * AES-GCM Functions + */ + +static int crypto4xx_aes_gcm_validate_keylen(unsigned int keylen) +{ + switch (keylen) { + case 16: + case 24: + case 32: + return 0; + default: + return -EINVAL; + } +} + +static int crypto4xx_compute_gcm_hash_key_sw(__le32 *hash_start, const u8 *key, + unsigned int keylen) +{ + struct crypto_cipher *aes_tfm = NULL; + uint8_t src[16] = { 0 }; + int rc = 0; + + aes_tfm = crypto_alloc_cipher("aes", 0, CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(aes_tfm)) { + rc = PTR_ERR(aes_tfm); + pr_warn("could not load aes cipher driver: %d\n", rc); + return rc; + } + + rc = crypto_cipher_setkey(aes_tfm, key, keylen); + if (rc) { + pr_err("setkey() failed: %d\n", rc); + goto out; + } + + crypto_cipher_encrypt_one(aes_tfm, src, src); + crypto4xx_memcpy_to_le32(hash_start, src, 16); +out: + crypto_free_cipher(aes_tfm); + return rc; +} + +int crypto4xx_setkey_aes_gcm(struct crypto_aead *cipher, + const u8 *key, unsigned int keylen) +{ + struct crypto_tfm *tfm = crypto_aead_tfm(cipher); + struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm); + struct dynamic_sa_ctl *sa; + int rc = 0; + + if (crypto4xx_aes_gcm_validate_keylen(keylen) != 0) { + crypto_aead_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + + rc = crypto4xx_setup_fallback(ctx, cipher, key, keylen); + if (rc) + return rc; + + if (ctx->sa_in || ctx->sa_out) + crypto4xx_free_sa(ctx); + + rc = crypto4xx_alloc_sa(ctx, SA_AES128_GCM_LEN + (keylen - 16) / 4); + if (rc) + return rc; + + sa = (struct dynamic_sa_ctl *) ctx->sa_in; + + sa->sa_contents.w = SA_AES_GCM_CONTENTS | (keylen << 2); + set_dynamic_sa_command_0(sa, SA_SAVE_HASH, SA_NOT_SAVE_IV, + SA_LOAD_HASH_FROM_SA, SA_LOAD_IV_FROM_STATE, + SA_NO_HEADER_PROC, SA_HASH_ALG_GHASH, + SA_CIPHER_ALG_AES, SA_PAD_TYPE_ZERO, + SA_OP_GROUP_BASIC, SA_OPCODE_HASH_DECRYPT, + DIR_INBOUND); + set_dynamic_sa_command_1(sa, CRYPTO_MODE_CTR, SA_HASH_MODE_HASH, + CRYPTO_FEEDBACK_MODE_NO_FB, SA_EXTENDED_SN_OFF, + SA_SEQ_MASK_ON, SA_MC_DISABLE, + SA_NOT_COPY_PAD, SA_COPY_PAYLOAD, + SA_NOT_COPY_HDR); + + sa->sa_command_1.bf.key_len = keylen >> 3; + + crypto4xx_memcpy_to_le32(get_dynamic_sa_key_field(sa), + key, keylen); + + rc = crypto4xx_compute_gcm_hash_key_sw(get_dynamic_sa_inner_digest(sa), + key, keylen); + if (rc) { + pr_err("GCM hash key setting failed = %d\n", rc); + goto err; + } + + memcpy(ctx->sa_out, ctx->sa_in, ctx->sa_len * 4); + sa = (struct dynamic_sa_ctl *) ctx->sa_out; + sa->sa_command_0.bf.dir = DIR_OUTBOUND; + sa->sa_command_0.bf.opcode = SA_OPCODE_ENCRYPT_HASH; + + return 0; +err: + crypto4xx_free_sa(ctx); + return rc; +} + +static inline int crypto4xx_crypt_aes_gcm(struct aead_request *req, + bool decrypt) +{ + struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + unsigned int len = req->cryptlen; + __le32 iv[4]; + + if (crypto4xx_aead_need_fallback(req, false, decrypt)) + return crypto4xx_aead_fallback(req, ctx, decrypt); + + crypto4xx_memcpy_to_le32(iv, req->iv, GCM_AES_IV_SIZE); + iv[3] = cpu_to_le32(1); + + if (decrypt) + len -= crypto_aead_authsize(crypto_aead_reqtfm(req)); + + return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst, + len, iv, sizeof(iv), + decrypt ? ctx->sa_in : ctx->sa_out, + ctx->sa_len, req->assoclen); +} + +int crypto4xx_encrypt_aes_gcm(struct aead_request *req) +{ + return crypto4xx_crypt_aes_gcm(req, false); +} + +int crypto4xx_decrypt_aes_gcm(struct aead_request *req) +{ + return crypto4xx_crypt_aes_gcm(req, true); +} + /** * HASH SHA1 Functions */ @@ -183,53 +564,39 @@ static int crypto4xx_hash_alg_init(struct crypto_tfm *tfm, unsigned char hm) { struct crypto_alg *alg = tfm->__crt_alg; - struct crypto4xx_alg *my_alg = crypto_alg_to_crypto4xx_alg(alg); + struct crypto4xx_alg *my_alg; struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm); - struct dynamic_sa_ctl *sa; - struct dynamic_sa_hash160 *sa_in; + struct dynamic_sa_hash160 *sa; int rc; + my_alg = container_of(__crypto_ahash_alg(alg), struct crypto4xx_alg, + alg.u.hash); ctx->dev = my_alg->dev; - ctx->is_hash = 1; - ctx->hash_final = 0; /* Create SA */ - if (ctx->sa_in_dma_addr || ctx->sa_out_dma_addr) + if (ctx->sa_in || ctx->sa_out) crypto4xx_free_sa(ctx); rc = crypto4xx_alloc_sa(ctx, sa_len); if (rc) return rc; - if (ctx->state_record_dma_addr == 0) { - crypto4xx_alloc_state_record(ctx); - if (!ctx->state_record_dma_addr) { - crypto4xx_free_sa(ctx); - return -ENOMEM; - } - } - crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), sizeof(struct crypto4xx_ctx)); - sa = (struct dynamic_sa_ctl *) ctx->sa_in; - set_dynamic_sa_command_0(sa, SA_SAVE_HASH, SA_NOT_SAVE_IV, + sa = (struct dynamic_sa_hash160 *)ctx->sa_in; + set_dynamic_sa_command_0(&sa->ctrl, SA_SAVE_HASH, SA_NOT_SAVE_IV, SA_NOT_LOAD_HASH, SA_LOAD_IV_FROM_SA, SA_NO_HEADER_PROC, ha, SA_CIPHER_ALG_NULL, SA_PAD_TYPE_ZERO, SA_OP_GROUP_BASIC, SA_OPCODE_HASH, DIR_INBOUND); - set_dynamic_sa_command_1(sa, 0, SA_HASH_MODE_HASH, + set_dynamic_sa_command_1(&sa->ctrl, 0, SA_HASH_MODE_HASH, CRYPTO_FEEDBACK_MODE_NO_FB, SA_EXTENDED_SN_OFF, SA_SEQ_MASK_OFF, SA_MC_ENABLE, SA_NOT_COPY_PAD, SA_NOT_COPY_PAYLOAD, SA_NOT_COPY_HDR); - ctx->direction = DIR_INBOUND; - sa->sa_contents = SA_HASH160_CONTENTS; - sa_in = (struct dynamic_sa_hash160 *) ctx->sa_in; /* Need to zero hash digest in SA */ - memset(sa_in->inner_digest, 0, sizeof(sa_in->inner_digest)); - memset(sa_in->outer_digest, 0, sizeof(sa_in->outer_digest)); - sa_in->state_ptr = ctx->state_record_dma_addr; - ctx->offset_to_sr_ptr = get_dynamic_sa_offset_state_ptr_field(ctx); + memset(sa->inner_digest, 0, sizeof(sa->inner_digest)); + memset(sa->outer_digest, 0, sizeof(sa->outer_digest)); return 0; } @@ -240,29 +607,27 @@ int crypto4xx_hash_init(struct ahash_request *req) int ds; struct dynamic_sa_ctl *sa; - sa = (struct dynamic_sa_ctl *) ctx->sa_in; + sa = ctx->sa_in; ds = crypto_ahash_digestsize( __crypto_ahash_cast(req->base.tfm)); sa->sa_command_0.bf.digest_len = ds >> 2; sa->sa_command_0.bf.load_hash_state = SA_LOAD_HASH_FROM_SA; - ctx->is_hash = 1; - ctx->direction = DIR_INBOUND; return 0; } int crypto4xx_hash_update(struct ahash_request *req) { + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct scatterlist dst; + unsigned int ds = crypto_ahash_digestsize(ahash); - ctx->is_hash = 1; - ctx->hash_final = 0; - ctx->pd_ctl = 0x11; - ctx->direction = DIR_INBOUND; + sg_init_one(&dst, req->result, ds); - return crypto4xx_build_pd(&req->base, ctx, req->src, - (struct scatterlist *) req->result, - req->nbytes, NULL, 0); + return crypto4xx_build_pd(&req->base, ctx, req->src, &dst, + req->nbytes, NULL, 0, ctx->sa_in, + ctx->sa_len, 0); } int crypto4xx_hash_final(struct ahash_request *req) @@ -272,15 +637,16 @@ int crypto4xx_hash_final(struct ahash_request *req) int crypto4xx_hash_digest(struct ahash_request *req) { + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct scatterlist dst; + unsigned int ds = crypto_ahash_digestsize(ahash); - ctx->hash_final = 1; - ctx->pd_ctl = 0x11; - ctx->direction = DIR_INBOUND; + sg_init_one(&dst, req->result, ds); - return crypto4xx_build_pd(&req->base, ctx, req->src, - (struct scatterlist *) req->result, - req->nbytes, NULL, 0); + return crypto4xx_build_pd(&req->base, ctx, req->src, &dst, + req->nbytes, NULL, 0, ctx->sa_in, + ctx->sa_len, 0); } /** @@ -291,5 +657,3 @@ int crypto4xx_sha1_alg_init(struct crypto_tfm *tfm) return crypto4xx_hash_alg_init(tfm, SA_HASH160_LEN, SA_HASH_ALG_SHA1, SA_HASH_MODE_HASH); } - - diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index 65dc78b91dea..c44954e274bc 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -35,8 +35,14 @@ #include <asm/dcr.h> #include <asm/dcr-regs.h> #include <asm/cacheflush.h> +#include <crypto/aead.h> #include <crypto/aes.h> +#include <crypto/ctr.h> +#include <crypto/gcm.h> #include <crypto/sha.h> +#include <crypto/scatterwalk.h> +#include <crypto/internal/aead.h> +#include <crypto/internal/skcipher.h> #include "crypto4xx_reg_def.h" #include "crypto4xx_core.h" #include "crypto4xx_sa.h" @@ -127,21 +133,17 @@ static void crypto4xx_hw_init(struct crypto4xx_device *dev) int crypto4xx_alloc_sa(struct crypto4xx_ctx *ctx, u32 size) { - ctx->sa_in = dma_alloc_coherent(ctx->dev->core_dev->device, size * 4, - &ctx->sa_in_dma_addr, GFP_ATOMIC); + ctx->sa_in = kzalloc(size * 4, GFP_ATOMIC); if (ctx->sa_in == NULL) return -ENOMEM; - ctx->sa_out = dma_alloc_coherent(ctx->dev->core_dev->device, size * 4, - &ctx->sa_out_dma_addr, GFP_ATOMIC); + ctx->sa_out = kzalloc(size * 4, GFP_ATOMIC); if (ctx->sa_out == NULL) { - dma_free_coherent(ctx->dev->core_dev->device, size * 4, - ctx->sa_in, ctx->sa_in_dma_addr); + kfree(ctx->sa_in); + ctx->sa_in = NULL; return -ENOMEM; } - memset(ctx->sa_in, 0, size * 4); - memset(ctx->sa_out, 0, size * 4); ctx->sa_len = size; return 0; @@ -149,40 +151,13 @@ int crypto4xx_alloc_sa(struct crypto4xx_ctx *ctx, u32 size) void crypto4xx_free_sa(struct crypto4xx_ctx *ctx) { - if (ctx->sa_in != NULL) - dma_free_coherent(ctx->dev->core_dev->device, ctx->sa_len * 4, - ctx->sa_in, ctx->sa_in_dma_addr); - if (ctx->sa_out != NULL) - dma_free_coherent(ctx->dev->core_dev->device, ctx->sa_len * 4, - ctx->sa_out, ctx->sa_out_dma_addr); - - ctx->sa_in_dma_addr = 0; - ctx->sa_out_dma_addr = 0; + kfree(ctx->sa_in); + ctx->sa_in = NULL; + kfree(ctx->sa_out); + ctx->sa_out = NULL; ctx->sa_len = 0; } -u32 crypto4xx_alloc_state_record(struct crypto4xx_ctx *ctx) -{ - ctx->state_record = dma_alloc_coherent(ctx->dev->core_dev->device, - sizeof(struct sa_state_record), - &ctx->state_record_dma_addr, GFP_ATOMIC); - if (!ctx->state_record_dma_addr) - return -ENOMEM; - memset(ctx->state_record, 0, sizeof(struct sa_state_record)); - - return 0; -} - -void crypto4xx_free_state_record(struct crypto4xx_ctx *ctx) -{ - if (ctx->state_record != NULL) - dma_free_coherent(ctx->dev->core_dev->device, - sizeof(struct sa_state_record), - ctx->state_record, - ctx->state_record_dma_addr); - ctx->state_record_dma_addr = 0; -} - /** * alloc memory for the gather ring * no need to alloc buf for the ring @@ -191,7 +166,6 @@ void crypto4xx_free_state_record(struct crypto4xx_ctx *ctx) static u32 crypto4xx_build_pdr(struct crypto4xx_device *dev) { int i; - struct pd_uinfo *pd_uinfo; dev->pdr = dma_alloc_coherent(dev->core_dev->device, sizeof(struct ce_pd) * PPC4XX_NUM_PD, &dev->pdr_pa, GFP_ATOMIC); @@ -207,9 +181,9 @@ static u32 crypto4xx_build_pdr(struct crypto4xx_device *dev) dev->pdr_pa); return -ENOMEM; } - memset(dev->pdr, 0, sizeof(struct ce_pd) * PPC4XX_NUM_PD); + memset(dev->pdr, 0, sizeof(struct ce_pd) * PPC4XX_NUM_PD); dev->shadow_sa_pool = dma_alloc_coherent(dev->core_dev->device, - 256 * PPC4XX_NUM_PD, + sizeof(union shadow_sa_buf) * PPC4XX_NUM_PD, &dev->shadow_sa_pool_pa, GFP_ATOMIC); if (!dev->shadow_sa_pool) @@ -221,16 +195,17 @@ static u32 crypto4xx_build_pdr(struct crypto4xx_device *dev) if (!dev->shadow_sr_pool) return -ENOMEM; for (i = 0; i < PPC4XX_NUM_PD; i++) { - pd_uinfo = (struct pd_uinfo *) (dev->pdr_uinfo + - sizeof(struct pd_uinfo) * i); + struct ce_pd *pd = &dev->pdr[i]; + struct pd_uinfo *pd_uinfo = &dev->pdr_uinfo[i]; + + pd->sa = dev->shadow_sa_pool_pa + + sizeof(union shadow_sa_buf) * i; /* alloc 256 bytes which is enough for any kind of dynamic sa */ - pd_uinfo->sa_va = dev->shadow_sa_pool + 256 * i; - pd_uinfo->sa_pa = dev->shadow_sa_pool_pa + 256 * i; + pd_uinfo->sa_va = &dev->shadow_sa_pool[i].sa; /* alloc state record */ - pd_uinfo->sr_va = dev->shadow_sr_pool + - sizeof(struct sa_state_record) * i; + pd_uinfo->sr_va = &dev->shadow_sr_pool[i]; pd_uinfo->sr_pa = dev->shadow_sr_pool_pa + sizeof(struct sa_state_record) * i; } @@ -240,13 +215,16 @@ static u32 crypto4xx_build_pdr(struct crypto4xx_device *dev) static void crypto4xx_destroy_pdr(struct crypto4xx_device *dev) { - if (dev->pdr != NULL) + if (dev->pdr) dma_free_coherent(dev->core_dev->device, sizeof(struct ce_pd) * PPC4XX_NUM_PD, dev->pdr, dev->pdr_pa); + if (dev->shadow_sa_pool) - dma_free_coherent(dev->core_dev->device, 256 * PPC4XX_NUM_PD, - dev->shadow_sa_pool, dev->shadow_sa_pool_pa); + dma_free_coherent(dev->core_dev->device, + sizeof(union shadow_sa_buf) * PPC4XX_NUM_PD, + dev->shadow_sa_pool, dev->shadow_sa_pool_pa); + if (dev->shadow_sr_pool) dma_free_coherent(dev->core_dev->device, sizeof(struct sa_state_record) * PPC4XX_NUM_PD, @@ -273,28 +251,21 @@ static u32 crypto4xx_get_pd_from_pdr_nolock(struct crypto4xx_device *dev) static u32 crypto4xx_put_pd_to_pdr(struct crypto4xx_device *dev, u32 idx) { - struct pd_uinfo *pd_uinfo; + struct pd_uinfo *pd_uinfo = &dev->pdr_uinfo[idx]; + u32 tail; unsigned long flags; - pd_uinfo = (struct pd_uinfo *)(dev->pdr_uinfo + - sizeof(struct pd_uinfo) * idx); spin_lock_irqsave(&dev->core_dev->lock, flags); + pd_uinfo->state = PD_ENTRY_FREE; + if (dev->pdr_tail != PPC4XX_LAST_PD) dev->pdr_tail++; else dev->pdr_tail = 0; - pd_uinfo->state = PD_ENTRY_FREE; + tail = dev->pdr_tail; spin_unlock_irqrestore(&dev->core_dev->lock, flags); - return 0; -} - -static struct ce_pd *crypto4xx_get_pdp(struct crypto4xx_device *dev, - dma_addr_t *pd_dma, u32 idx) -{ - *pd_dma = dev->pdr_pa + sizeof(struct ce_pd) * idx; - - return dev->pdr + sizeof(struct ce_pd) * idx; + return tail; } /** @@ -326,10 +297,11 @@ static inline void crypto4xx_destroy_gdr(struct crypto4xx_device *dev) * when this function is called. * preemption or interrupt must be disabled */ -u32 crypto4xx_get_n_gd(struct crypto4xx_device *dev, int n) +static u32 crypto4xx_get_n_gd(struct crypto4xx_device *dev, int n) { u32 retval; u32 tmp; + if (n >= PPC4XX_NUM_GD) return ERING_WAS_FULL; @@ -372,7 +344,7 @@ static inline struct ce_gd *crypto4xx_get_gdp(struct crypto4xx_device *dev, { *gd_dma = dev->gdr_pa + sizeof(struct ce_gd) * idx; - return (struct ce_gd *) (dev->gdr + sizeof(struct ce_gd) * idx); + return &dev->gdr[idx]; } /** @@ -383,7 +355,6 @@ static inline struct ce_gd *crypto4xx_get_gdp(struct crypto4xx_device *dev, static u32 crypto4xx_build_sdr(struct crypto4xx_device *dev) { int i; - struct ce_sd *sd_array; /* alloc memory for scatter descriptor ring */ dev->sdr = dma_alloc_coherent(dev->core_dev->device, @@ -392,10 +363,9 @@ static u32 crypto4xx_build_sdr(struct crypto4xx_device *dev) if (!dev->sdr) return -ENOMEM; - dev->scatter_buffer_size = PPC4XX_SD_BUFFER_SIZE; dev->scatter_buffer_va = dma_alloc_coherent(dev->core_dev->device, - dev->scatter_buffer_size * PPC4XX_NUM_SD, + PPC4XX_SD_BUFFER_SIZE * PPC4XX_NUM_SD, &dev->scatter_buffer_pa, GFP_ATOMIC); if (!dev->scatter_buffer_va) { dma_free_coherent(dev->core_dev->device, @@ -404,11 +374,9 @@ static u32 crypto4xx_build_sdr(struct crypto4xx_device *dev) return -ENOMEM; } - sd_array = dev->sdr; - for (i = 0; i < PPC4XX_NUM_SD; i++) { - sd_array[i].ptr = dev->scatter_buffer_pa + - dev->scatter_buffer_size * i; + dev->sdr[i].ptr = dev->scatter_buffer_pa + + PPC4XX_SD_BUFFER_SIZE * i; } return 0; @@ -416,14 +384,14 @@ static u32 crypto4xx_build_sdr(struct crypto4xx_device *dev) static void crypto4xx_destroy_sdr(struct crypto4xx_device *dev) { - if (dev->sdr != NULL) + if (dev->sdr) dma_free_coherent(dev->core_dev->device, sizeof(struct ce_sd) * PPC4XX_NUM_SD, dev->sdr, dev->sdr_pa); - if (dev->scatter_buffer_va != NULL) + if (dev->scatter_buffer_va) dma_free_coherent(dev->core_dev->device, - dev->scatter_buffer_size * PPC4XX_NUM_SD, + PPC4XX_SD_BUFFER_SIZE * PPC4XX_NUM_SD, dev->scatter_buffer_va, dev->scatter_buffer_pa); } @@ -477,63 +445,7 @@ static inline struct ce_sd *crypto4xx_get_sdp(struct crypto4xx_device *dev, { *sd_dma = dev->sdr_pa + sizeof(struct ce_sd) * idx; - return (struct ce_sd *)(dev->sdr + sizeof(struct ce_sd) * idx); -} - -static u32 crypto4xx_fill_one_page(struct crypto4xx_device *dev, - dma_addr_t *addr, u32 *length, - u32 *idx, u32 *offset, u32 *nbytes) -{ - u32 len; - - if (*length > dev->scatter_buffer_size) { - memcpy(phys_to_virt(*addr), - dev->scatter_buffer_va + - *idx * dev->scatter_buffer_size + *offset, - dev->scatter_buffer_size); - *offset = 0; - *length -= dev->scatter_buffer_size; - *nbytes -= dev->scatter_buffer_size; - if (*idx == PPC4XX_LAST_SD) - *idx = 0; - else - (*idx)++; - *addr = *addr + dev->scatter_buffer_size; - return 1; - } else if (*length < dev->scatter_buffer_size) { - memcpy(phys_to_virt(*addr), - dev->scatter_buffer_va + - *idx * dev->scatter_buffer_size + *offset, *length); - if ((*offset + *length) == dev->scatter_buffer_size) { - if (*idx == PPC4XX_LAST_SD) - *idx = 0; - else - (*idx)++; - *nbytes -= *length; - *offset = 0; - } else { - *nbytes -= *length; - *offset += *length; - } - - return 0; - } else { - len = (*nbytes <= dev->scatter_buffer_size) ? - (*nbytes) : dev->scatter_buffer_size; - memcpy(phys_to_virt(*addr), - dev->scatter_buffer_va + - *idx * dev->scatter_buffer_size + *offset, - len); - *offset = 0; - *nbytes -= len; - - if (*idx == PPC4XX_LAST_SD) - *idx = 0; - else - (*idx)++; - - return 0; - } + return &dev->sdr[idx]; } static void crypto4xx_copy_pkt_to_dst(struct crypto4xx_device *dev, @@ -542,66 +454,52 @@ static void crypto4xx_copy_pkt_to_dst(struct crypto4xx_device *dev, u32 nbytes, struct scatterlist *dst) { - dma_addr_t addr; - u32 this_sd; - u32 offset; - u32 len; - u32 i; - u32 sg_len; - struct scatterlist *sg; + unsigned int first_sd = pd_uinfo->first_sd; + unsigned int last_sd; + unsigned int overflow = 0; + unsigned int to_copy; + unsigned int dst_start = 0; - this_sd = pd_uinfo->first_sd; - offset = 0; - i = 0; + /* + * Because the scatter buffers are all neatly organized in one + * big continuous ringbuffer; scatterwalk_map_and_copy() can + * be instructed to copy a range of buffers in one go. + */ + + last_sd = (first_sd + pd_uinfo->num_sd); + if (last_sd > PPC4XX_LAST_SD) { + last_sd = PPC4XX_LAST_SD; + overflow = last_sd % PPC4XX_NUM_SD; + } while (nbytes) { - sg = &dst[i]; - sg_len = sg->length; - addr = dma_map_page(dev->core_dev->device, sg_page(sg), - sg->offset, sg->length, DMA_TO_DEVICE); - - if (offset == 0) { - len = (nbytes <= sg->length) ? nbytes : sg->length; - while (crypto4xx_fill_one_page(dev, &addr, &len, - &this_sd, &offset, &nbytes)) - ; - if (!nbytes) - return; - i++; - } else { - len = (nbytes <= (dev->scatter_buffer_size - offset)) ? - nbytes : (dev->scatter_buffer_size - offset); - len = (sg->length < len) ? sg->length : len; - while (crypto4xx_fill_one_page(dev, &addr, &len, - &this_sd, &offset, &nbytes)) - ; - if (!nbytes) - return; - sg_len -= len; - if (sg_len) { - addr += len; - while (crypto4xx_fill_one_page(dev, &addr, - &sg_len, &this_sd, &offset, &nbytes)) - ; - } - i++; + void *buf = dev->scatter_buffer_va + + first_sd * PPC4XX_SD_BUFFER_SIZE; + + to_copy = min(nbytes, PPC4XX_SD_BUFFER_SIZE * + (1 + last_sd - first_sd)); + scatterwalk_map_and_copy(buf, dst, dst_start, to_copy, 1); + nbytes -= to_copy; + + if (overflow) { + first_sd = 0; + last_sd = overflow; + dst_start += to_copy; + overflow = 0; } } } -static u32 crypto4xx_copy_digest_to_dst(struct pd_uinfo *pd_uinfo, +static void crypto4xx_copy_digest_to_dst(void *dst, + struct pd_uinfo *pd_uinfo, struct crypto4xx_ctx *ctx) { struct dynamic_sa_ctl *sa = (struct dynamic_sa_ctl *) ctx->sa_in; - struct sa_state_record *state_record = - (struct sa_state_record *) pd_uinfo->sr_va; if (sa->sa_command_0.bf.hash_alg == SA_HASH_ALG_SHA1) { - memcpy((void *) pd_uinfo->dest_va, state_record->save_digest, + memcpy(dst, pd_uinfo->sr_va->save_digest, SA_HASH_ALG_SHA1_DIGEST_SIZE); } - - return 0; } static void crypto4xx_ret_sg_desc(struct crypto4xx_device *dev, @@ -623,7 +521,7 @@ static void crypto4xx_ret_sg_desc(struct crypto4xx_device *dev, } } -static u32 crypto4xx_ablkcipher_done(struct crypto4xx_device *dev, +static void crypto4xx_ablkcipher_done(struct crypto4xx_device *dev, struct pd_uinfo *pd_uinfo, struct ce_pd *pd) { @@ -644,13 +542,13 @@ static u32 crypto4xx_ablkcipher_done(struct crypto4xx_device *dev, dst->offset, dst->length, DMA_FROM_DEVICE); } crypto4xx_ret_sg_desc(dev, pd_uinfo); - if (ablk_req->base.complete != NULL) - ablk_req->base.complete(&ablk_req->base, 0); - return 0; + if (pd_uinfo->state & PD_ENTRY_BUSY) + ablkcipher_request_complete(ablk_req, -EINPROGRESS); + ablkcipher_request_complete(ablk_req, 0); } -static u32 crypto4xx_ahash_done(struct crypto4xx_device *dev, +static void crypto4xx_ahash_done(struct crypto4xx_device *dev, struct pd_uinfo *pd_uinfo) { struct crypto4xx_ctx *ctx; @@ -659,62 +557,93 @@ static u32 crypto4xx_ahash_done(struct crypto4xx_device *dev, ahash_req = ahash_request_cast(pd_uinfo->async_req); ctx = crypto_tfm_ctx(ahash_req->base.tfm); - crypto4xx_copy_digest_to_dst(pd_uinfo, + crypto4xx_copy_digest_to_dst(ahash_req->result, pd_uinfo, crypto_tfm_ctx(ahash_req->base.tfm)); crypto4xx_ret_sg_desc(dev, pd_uinfo); - /* call user provided callback function x */ - if (ahash_req->base.complete != NULL) - ahash_req->base.complete(&ahash_req->base, 0); - return 0; + if (pd_uinfo->state & PD_ENTRY_BUSY) + ahash_request_complete(ahash_req, -EINPROGRESS); + ahash_request_complete(ahash_req, 0); } -static u32 crypto4xx_pd_done(struct crypto4xx_device *dev, u32 idx) +static void crypto4xx_aead_done(struct crypto4xx_device *dev, + struct pd_uinfo *pd_uinfo, + struct ce_pd *pd) { - struct ce_pd *pd; - struct pd_uinfo *pd_uinfo; + struct aead_request *aead_req; + struct crypto4xx_ctx *ctx; + struct scatterlist *dst = pd_uinfo->dest_va; + int err = 0; - pd = dev->pdr + sizeof(struct ce_pd)*idx; - pd_uinfo = dev->pdr_uinfo + sizeof(struct pd_uinfo)*idx; - if (crypto_tfm_alg_type(pd_uinfo->async_req->tfm) == - CRYPTO_ALG_TYPE_ABLKCIPHER) - return crypto4xx_ablkcipher_done(dev, pd_uinfo, pd); - else - return crypto4xx_ahash_done(dev, pd_uinfo); + aead_req = container_of(pd_uinfo->async_req, struct aead_request, + base); + ctx = crypto_tfm_ctx(aead_req->base.tfm); + + if (pd_uinfo->using_sd) { + crypto4xx_copy_pkt_to_dst(dev, pd, pd_uinfo, + pd->pd_ctl_len.bf.pkt_len, + dst); + } else { + __dma_sync_page(sg_page(dst), dst->offset, dst->length, + DMA_FROM_DEVICE); + } + + if (pd_uinfo->sa_va->sa_command_0.bf.dir == DIR_OUTBOUND) { + /* append icv at the end */ + size_t cp_len = crypto_aead_authsize( + crypto_aead_reqtfm(aead_req)); + u32 icv[cp_len]; + + crypto4xx_memcpy_from_le32(icv, pd_uinfo->sr_va->save_digest, + cp_len); + + scatterwalk_map_and_copy(icv, dst, aead_req->cryptlen, + cp_len, 1); + } + + crypto4xx_ret_sg_desc(dev, pd_uinfo); + + if (pd->pd_ctl.bf.status & 0xff) { + if (pd->pd_ctl.bf.status & 0x1) { + /* authentication error */ + err = -EBADMSG; + } else { + if (!__ratelimit(&dev->aead_ratelimit)) { + if (pd->pd_ctl.bf.status & 2) + pr_err("pad fail error\n"); + if (pd->pd_ctl.bf.status & 4) + pr_err("seqnum fail\n"); + if (pd->pd_ctl.bf.status & 8) + pr_err("error _notify\n"); + pr_err("aead return err status = 0x%02x\n", + pd->pd_ctl.bf.status & 0xff); + pr_err("pd pad_ctl = 0x%08x\n", + pd->pd_ctl.bf.pd_pad_ctl); + } + err = -EINVAL; + } + } + + if (pd_uinfo->state & PD_ENTRY_BUSY) + aead_request_complete(aead_req, -EINPROGRESS); + + aead_request_complete(aead_req, err); } -/** - * Note: Only use this function to copy items that is word aligned. - */ -void crypto4xx_memcpy_le(unsigned int *dst, - const unsigned char *buf, - int len) +static void crypto4xx_pd_done(struct crypto4xx_device *dev, u32 idx) { - u8 *tmp; - for (; len >= 4; buf += 4, len -= 4) - *dst++ = cpu_to_le32(*(unsigned int *) buf); - - tmp = (u8 *)dst; - switch (len) { - case 3: - *tmp++ = 0; - *tmp++ = *(buf+2); - *tmp++ = *(buf+1); - *tmp++ = *buf; - break; - case 2: - *tmp++ = 0; - *tmp++ = 0; - *tmp++ = *(buf+1); - *tmp++ = *buf; + struct ce_pd *pd = &dev->pdr[idx]; + struct pd_uinfo *pd_uinfo = &dev->pdr_uinfo[idx]; + + switch (crypto_tfm_alg_type(pd_uinfo->async_req->tfm)) { + case CRYPTO_ALG_TYPE_ABLKCIPHER: + crypto4xx_ablkcipher_done(dev, pd_uinfo, pd); break; - case 1: - *tmp++ = 0; - *tmp++ = 0; - *tmp++ = 0; - *tmp++ = *buf; + case CRYPTO_ALG_TYPE_AEAD: + crypto4xx_aead_done(dev, pd_uinfo, pd); break; - default: + case CRYPTO_ALG_TYPE_AHASH: + crypto4xx_ahash_done(dev, pd_uinfo); break; } } @@ -729,17 +658,6 @@ static void crypto4xx_stop_all(struct crypto4xx_core_device *core_dev) kfree(core_dev); } -void crypto4xx_return_pd(struct crypto4xx_device *dev, - u32 pd_entry, struct ce_pd *pd, - struct pd_uinfo *pd_uinfo) -{ - /* irq should be already disabled */ - dev->pdr_head = pd_entry; - pd->pd_ctl.w = 0; - pd->pd_ctl_len.w = 0; - pd_uinfo->state = PD_ENTRY_FREE; -} - static u32 get_next_gd(u32 current) { if (current != PPC4XX_LAST_GD) @@ -756,17 +674,19 @@ static u32 get_next_sd(u32 current) return 0; } -u32 crypto4xx_build_pd(struct crypto_async_request *req, +int crypto4xx_build_pd(struct crypto_async_request *req, struct crypto4xx_ctx *ctx, struct scatterlist *src, struct scatterlist *dst, - unsigned int datalen, - void *iv, u32 iv_len) + const unsigned int datalen, + const __le32 *iv, const u32 iv_len, + const struct dynamic_sa_ctl *req_sa, + const unsigned int sa_len, + const unsigned int assoclen) { + struct scatterlist _dst[2]; struct crypto4xx_device *dev = ctx->dev; - dma_addr_t addr, pd_dma, sd_dma, gd_dma; struct dynamic_sa_ctl *sa; - struct scatterlist *sg; struct ce_gd *gd; struct ce_pd *pd; u32 num_gd, num_sd; @@ -774,22 +694,30 @@ u32 crypto4xx_build_pd(struct crypto_async_request *req, u32 fst_sd = 0xffffffff; u32 pd_entry; unsigned long flags; - struct pd_uinfo *pd_uinfo = NULL; - unsigned int nbytes = datalen, idx; - unsigned int ivlen = 0; + struct pd_uinfo *pd_uinfo; + unsigned int nbytes = datalen; + size_t offset_to_sr_ptr; u32 gd_idx = 0; + int tmp; + bool is_busy; - /* figure how many gd is needed */ - num_gd = sg_nents_for_len(src, datalen); - if ((int)num_gd < 0) { + /* figure how many gd are needed */ + tmp = sg_nents_for_len(src, assoclen + datalen); + if (tmp < 0) { dev_err(dev->core_dev->device, "Invalid number of src SG.\n"); - return -EINVAL; + return tmp; } - if (num_gd == 1) - num_gd = 0; + if (tmp == 1) + tmp = 0; + num_gd = tmp; - /* figure how many sd is needed */ - if (sg_is_last(dst) || ctx->is_hash) { + if (assoclen) { + nbytes += assoclen; + dst = scatterwalk_ffwd(_dst, dst, assoclen); + } + + /* figure how many sd are needed */ + if (sg_is_last(dst)) { num_sd = 0; } else { if (datalen > PPC4XX_SD_BUFFER_SIZE) { @@ -808,6 +736,31 @@ u32 crypto4xx_build_pd(struct crypto_async_request *req, * already got must be return the original place. */ spin_lock_irqsave(&dev->core_dev->lock, flags); + /* + * Let the caller know to slow down, once more than 13/16ths = 81% + * of the available data contexts are being used simultaneously. + * + * With PPC4XX_NUM_PD = 256, this will leave a "backlog queue" for + * 31 more contexts. Before new requests have to be rejected. + */ + if (req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG) { + is_busy = ((dev->pdr_head - dev->pdr_tail) % PPC4XX_NUM_PD) >= + ((PPC4XX_NUM_PD * 13) / 16); + } else { + /* + * To fix contention issues between ipsec (no blacklog) and + * dm-crypto (backlog) reserve 32 entries for "no backlog" + * data contexts. + */ + is_busy = ((dev->pdr_head - dev->pdr_tail) % PPC4XX_NUM_PD) >= + ((PPC4XX_NUM_PD * 15) / 16); + + if (is_busy) { + spin_unlock_irqrestore(&dev->core_dev->lock, flags); + return -EBUSY; + } + } + if (num_gd) { fst_gd = crypto4xx_get_n_gd(dev, num_gd); if (fst_gd == ERING_WAS_FULL) { @@ -835,38 +788,28 @@ u32 crypto4xx_build_pd(struct crypto_async_request *req, } spin_unlock_irqrestore(&dev->core_dev->lock, flags); - pd_uinfo = (struct pd_uinfo *)(dev->pdr_uinfo + - sizeof(struct pd_uinfo) * pd_entry); - pd = crypto4xx_get_pdp(dev, &pd_dma, pd_entry); + pd = &dev->pdr[pd_entry]; + pd->sa_len = sa_len; + + pd_uinfo = &dev->pdr_uinfo[pd_entry]; pd_uinfo->async_req = req; pd_uinfo->num_gd = num_gd; pd_uinfo->num_sd = num_sd; - if (iv_len || ctx->is_hash) { - ivlen = iv_len; - pd->sa = pd_uinfo->sa_pa; - sa = (struct dynamic_sa_ctl *) pd_uinfo->sa_va; - if (ctx->direction == DIR_INBOUND) - memcpy(sa, ctx->sa_in, ctx->sa_len * 4); - else - memcpy(sa, ctx->sa_out, ctx->sa_len * 4); + if (iv_len) + memcpy(pd_uinfo->sr_va->save_iv, iv, iv_len); - memcpy((void *) sa + ctx->offset_to_sr_ptr, - &pd_uinfo->sr_pa, 4); + sa = pd_uinfo->sa_va; + memcpy(sa, req_sa, sa_len * 4); + + sa->sa_command_1.bf.hash_crypto_offset = (assoclen >> 2); + offset_to_sr_ptr = get_dynamic_sa_offset_state_ptr_field(sa); + *(u32 *)((unsigned long)sa + offset_to_sr_ptr) = pd_uinfo->sr_pa; - if (iv_len) - crypto4xx_memcpy_le(pd_uinfo->sr_va, iv, iv_len); - } else { - if (ctx->direction == DIR_INBOUND) { - pd->sa = ctx->sa_in_dma_addr; - sa = (struct dynamic_sa_ctl *) ctx->sa_in; - } else { - pd->sa = ctx->sa_out_dma_addr; - sa = (struct dynamic_sa_ctl *) ctx->sa_out; - } - } - pd->sa_len = ctx->sa_len; if (num_gd) { + dma_addr_t gd_dma; + struct scatterlist *sg; + /* get first gd we are going to use */ gd_idx = fst_gd; pd_uinfo->first_gd = fst_gd; @@ -875,27 +818,30 @@ u32 crypto4xx_build_pd(struct crypto_async_request *req, pd->src = gd_dma; /* enable gather */ sa->sa_command_0.bf.gather = 1; - idx = 0; - src = &src[0]; /* walk the sg, and setup gather array */ + + sg = src; while (nbytes) { - sg = &src[idx]; - addr = dma_map_page(dev->core_dev->device, sg_page(sg), - sg->offset, sg->length, DMA_TO_DEVICE); - gd->ptr = addr; - gd->ctl_len.len = sg->length; + size_t len; + + len = min(sg->length, nbytes); + gd->ptr = dma_map_page(dev->core_dev->device, + sg_page(sg), sg->offset, len, DMA_TO_DEVICE); + gd->ctl_len.len = len; gd->ctl_len.done = 0; gd->ctl_len.ready = 1; - if (sg->length >= nbytes) + if (len >= nbytes) break; + nbytes -= sg->length; gd_idx = get_next_gd(gd_idx); gd = crypto4xx_get_gdp(dev, &gd_dma, gd_idx); - idx++; + sg = sg_next(sg); } } else { pd->src = (u32)dma_map_page(dev->core_dev->device, sg_page(src), - src->offset, src->length, DMA_TO_DEVICE); + src->offset, min(nbytes, src->length), + DMA_TO_DEVICE); /* * Disable gather in sa command */ @@ -906,25 +852,24 @@ u32 crypto4xx_build_pd(struct crypto_async_request *req, pd_uinfo->first_gd = 0xffffffff; pd_uinfo->num_gd = 0; } - if (ctx->is_hash || sg_is_last(dst)) { + if (sg_is_last(dst)) { /* * we know application give us dst a whole piece of memory * no need to use scatter ring. - * In case of is_hash, the icv is always at end of src data. */ pd_uinfo->using_sd = 0; pd_uinfo->first_sd = 0xffffffff; pd_uinfo->num_sd = 0; pd_uinfo->dest_va = dst; sa->sa_command_0.bf.scatter = 0; - if (ctx->is_hash) - pd->dest = virt_to_phys((void *)dst); - else - pd->dest = (u32)dma_map_page(dev->core_dev->device, - sg_page(dst), dst->offset, - dst->length, DMA_TO_DEVICE); + pd->dest = (u32)dma_map_page(dev->core_dev->device, + sg_page(dst), dst->offset, + min(datalen, dst->length), + DMA_TO_DEVICE); } else { + dma_addr_t sd_dma; struct ce_sd *sd = NULL; + u32 sd_idx = fst_sd; nbytes = datalen; sa->sa_command_0.bf.scatter = 1; @@ -938,7 +883,6 @@ u32 crypto4xx_build_pd(struct crypto_async_request *req, sd->ctl.done = 0; sd->ctl.rdy = 1; /* sd->ptr should be setup by sd_init routine*/ - idx = 0; if (nbytes >= PPC4XX_SD_BUFFER_SIZE) nbytes -= PPC4XX_SD_BUFFER_SIZE; else @@ -949,67 +893,97 @@ u32 crypto4xx_build_pd(struct crypto_async_request *req, /* setup scatter descriptor */ sd->ctl.done = 0; sd->ctl.rdy = 1; - if (nbytes >= PPC4XX_SD_BUFFER_SIZE) + if (nbytes >= PPC4XX_SD_BUFFER_SIZE) { nbytes -= PPC4XX_SD_BUFFER_SIZE; - else + } else { /* * SD entry can hold PPC4XX_SD_BUFFER_SIZE, * which is more than nbytes, so done. */ nbytes = 0; + } } } - sa->sa_command_1.bf.hash_crypto_offset = 0; - pd->pd_ctl.w = ctx->pd_ctl; - pd->pd_ctl_len.w = 0x00400000 | (ctx->bypass << 24) | datalen; - pd_uinfo->state = PD_ENTRY_INUSE; + pd->pd_ctl.w = PD_CTL_HOST_READY | + ((crypto_tfm_alg_type(req->tfm) == CRYPTO_ALG_TYPE_AHASH) | + (crypto_tfm_alg_type(req->tfm) == CRYPTO_ALG_TYPE_AEAD) ? + PD_CTL_HASH_FINAL : 0); + pd->pd_ctl_len.w = 0x00400000 | (assoclen + datalen); + pd_uinfo->state = PD_ENTRY_INUSE | (is_busy ? PD_ENTRY_BUSY : 0); + wmb(); /* write any value to push engine to read a pd */ + writel(0, dev->ce_base + CRYPTO4XX_INT_DESCR_RD); writel(1, dev->ce_base + CRYPTO4XX_INT_DESCR_RD); - return -EINPROGRESS; + return is_busy ? -EBUSY : -EINPROGRESS; } /** * Algorithm Registration Functions */ -static int crypto4xx_alg_init(struct crypto_tfm *tfm) +static void crypto4xx_ctx_init(struct crypto4xx_alg *amcc_alg, + struct crypto4xx_ctx *ctx) { - struct crypto_alg *alg = tfm->__crt_alg; - struct crypto4xx_alg *amcc_alg = crypto_alg_to_crypto4xx_alg(alg); - struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm); - ctx->dev = amcc_alg->dev; ctx->sa_in = NULL; ctx->sa_out = NULL; - ctx->sa_in_dma_addr = 0; - ctx->sa_out_dma_addr = 0; ctx->sa_len = 0; +} - switch (alg->cra_flags & CRYPTO_ALG_TYPE_MASK) { - default: - tfm->crt_ablkcipher.reqsize = sizeof(struct crypto4xx_ctx); - break; - case CRYPTO_ALG_TYPE_AHASH: - crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), - sizeof(struct crypto4xx_ctx)); - break; - } +static int crypto4xx_ablk_init(struct crypto_tfm *tfm) +{ + struct crypto_alg *alg = tfm->__crt_alg; + struct crypto4xx_alg *amcc_alg; + struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm); + amcc_alg = container_of(alg, struct crypto4xx_alg, alg.u.cipher); + crypto4xx_ctx_init(amcc_alg, ctx); + tfm->crt_ablkcipher.reqsize = sizeof(struct crypto4xx_ctx); return 0; } -static void crypto4xx_alg_exit(struct crypto_tfm *tfm) +static void crypto4xx_common_exit(struct crypto4xx_ctx *ctx) { - struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm); - crypto4xx_free_sa(ctx); - crypto4xx_free_state_record(ctx); } -int crypto4xx_register_alg(struct crypto4xx_device *sec_dev, - struct crypto4xx_alg_common *crypto_alg, - int array_size) +static void crypto4xx_ablk_exit(struct crypto_tfm *tfm) +{ + crypto4xx_common_exit(crypto_tfm_ctx(tfm)); +} + +static int crypto4xx_aead_init(struct crypto_aead *tfm) +{ + struct aead_alg *alg = crypto_aead_alg(tfm); + struct crypto4xx_ctx *ctx = crypto_aead_ctx(tfm); + struct crypto4xx_alg *amcc_alg; + + ctx->sw_cipher.aead = crypto_alloc_aead(alg->base.cra_name, 0, + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_ASYNC); + if (IS_ERR(ctx->sw_cipher.aead)) + return PTR_ERR(ctx->sw_cipher.aead); + + amcc_alg = container_of(alg, struct crypto4xx_alg, alg.u.aead); + crypto4xx_ctx_init(amcc_alg, ctx); + crypto_aead_set_reqsize(tfm, sizeof(struct aead_request) + + max(sizeof(struct crypto4xx_ctx), 32 + + crypto_aead_reqsize(ctx->sw_cipher.aead))); + return 0; +} + +static void crypto4xx_aead_exit(struct crypto_aead *tfm) +{ + struct crypto4xx_ctx *ctx = crypto_aead_ctx(tfm); + + crypto4xx_common_exit(ctx); + crypto_free_aead(ctx->sw_cipher.aead); +} + +static int crypto4xx_register_alg(struct crypto4xx_device *sec_dev, + struct crypto4xx_alg_common *crypto_alg, + int array_size) { struct crypto4xx_alg *alg; int i; @@ -1024,6 +998,10 @@ int crypto4xx_register_alg(struct crypto4xx_device *sec_dev, alg->dev = sec_dev; switch (alg->alg.type) { + case CRYPTO_ALG_TYPE_AEAD: + rc = crypto_register_aead(&alg->alg.u.aead); + break; + case CRYPTO_ALG_TYPE_AHASH: rc = crypto_register_ahash(&alg->alg.u.hash); break; @@ -1033,12 +1011,10 @@ int crypto4xx_register_alg(struct crypto4xx_device *sec_dev, break; } - if (rc) { - list_del(&alg->entry); + if (rc) kfree(alg); - } else { + else list_add_tail(&alg->entry, &sec_dev->alg_list); - } } return 0; @@ -1055,6 +1031,10 @@ static void crypto4xx_unregister_alg(struct crypto4xx_device *sec_dev) crypto_unregister_ahash(&alg->alg.u.hash); break; + case CRYPTO_ALG_TYPE_AEAD: + crypto_unregister_aead(&alg->alg.u.aead); + break; + default: crypto_unregister_alg(&alg->alg.u.cipher); } @@ -1068,25 +1048,23 @@ static void crypto4xx_bh_tasklet_cb(unsigned long data) struct crypto4xx_core_device *core_dev = dev_get_drvdata(dev); struct pd_uinfo *pd_uinfo; struct ce_pd *pd; - u32 tail; - - while (core_dev->dev->pdr_head != core_dev->dev->pdr_tail) { - tail = core_dev->dev->pdr_tail; - pd_uinfo = core_dev->dev->pdr_uinfo + - sizeof(struct pd_uinfo)*tail; - pd = core_dev->dev->pdr + sizeof(struct ce_pd) * tail; - if ((pd_uinfo->state == PD_ENTRY_INUSE) && - pd->pd_ctl.bf.pe_done && - !pd->pd_ctl.bf.host_ready) { - pd->pd_ctl.bf.pe_done = 0; + u32 tail = core_dev->dev->pdr_tail; + u32 head = core_dev->dev->pdr_head; + + do { + pd_uinfo = &core_dev->dev->pdr_uinfo[tail]; + pd = &core_dev->dev->pdr[tail]; + if ((pd_uinfo->state & PD_ENTRY_INUSE) && + ((READ_ONCE(pd->pd_ctl.w) & + (PD_CTL_PE_DONE | PD_CTL_HOST_READY)) == + PD_CTL_PE_DONE)) { crypto4xx_pd_done(core_dev->dev, tail); - crypto4xx_put_pd_to_pdr(core_dev->dev, tail); - pd_uinfo->state = PD_ENTRY_FREE; + tail = crypto4xx_put_pd_to_pdr(core_dev->dev, tail); } else { /* if tail not done, break */ break; } - } + } while (head != tail); } /** @@ -1110,18 +1088,20 @@ static irqreturn_t crypto4xx_ce_interrupt_handler(int irq, void *data) /** * Supported Crypto Algorithms */ -struct crypto4xx_alg_common crypto4xx_alg[] = { +static struct crypto4xx_alg_common crypto4xx_alg[] = { /* Crypto AES modes */ { .type = CRYPTO_ALG_TYPE_ABLKCIPHER, .u.cipher = { .cra_name = "cbc(aes)", .cra_driver_name = "cbc-aes-ppc4xx", .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY, - .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct crypto4xx_ctx), .cra_type = &crypto_ablkcipher_type, - .cra_init = crypto4xx_alg_init, - .cra_exit = crypto4xx_alg_exit, + .cra_init = crypto4xx_ablk_init, + .cra_exit = crypto4xx_ablk_exit, .cra_module = THIS_MODULE, .cra_u = { .ablkcipher = { @@ -1134,6 +1114,147 @@ struct crypto4xx_alg_common crypto4xx_alg[] = { } } }}, + { .type = CRYPTO_ALG_TYPE_ABLKCIPHER, .u.cipher = { + .cra_name = "cfb(aes)", + .cra_driver_name = "cfb-aes-ppc4xx", + .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct crypto4xx_ctx), + .cra_type = &crypto_ablkcipher_type, + .cra_init = crypto4xx_ablk_init, + .cra_exit = crypto4xx_ablk_exit, + .cra_module = THIS_MODULE, + .cra_u = { + .ablkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_IV_SIZE, + .setkey = crypto4xx_setkey_aes_cfb, + .encrypt = crypto4xx_encrypt, + .decrypt = crypto4xx_decrypt, + } + } + } }, + { .type = CRYPTO_ALG_TYPE_ABLKCIPHER, .u.cipher = { + .cra_name = "rfc3686(ctr(aes))", + .cra_driver_name = "rfc3686-ctr-aes-ppc4xx", + .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct crypto4xx_ctx), + .cra_type = &crypto_ablkcipher_type, + .cra_init = crypto4xx_ablk_init, + .cra_exit = crypto4xx_ablk_exit, + .cra_module = THIS_MODULE, + .cra_u = { + .ablkcipher = { + .min_keysize = AES_MIN_KEY_SIZE + + CTR_RFC3686_NONCE_SIZE, + .max_keysize = AES_MAX_KEY_SIZE + + CTR_RFC3686_NONCE_SIZE, + .ivsize = CTR_RFC3686_IV_SIZE, + .setkey = crypto4xx_setkey_rfc3686, + .encrypt = crypto4xx_rfc3686_encrypt, + .decrypt = crypto4xx_rfc3686_decrypt, + } + } + } }, + { .type = CRYPTO_ALG_TYPE_ABLKCIPHER, .u.cipher = { + .cra_name = "ecb(aes)", + .cra_driver_name = "ecb-aes-ppc4xx", + .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct crypto4xx_ctx), + .cra_type = &crypto_ablkcipher_type, + .cra_init = crypto4xx_ablk_init, + .cra_exit = crypto4xx_ablk_exit, + .cra_module = THIS_MODULE, + .cra_u = { + .ablkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = crypto4xx_setkey_aes_ecb, + .encrypt = crypto4xx_encrypt, + .decrypt = crypto4xx_decrypt, + } + } + } }, + { .type = CRYPTO_ALG_TYPE_ABLKCIPHER, .u.cipher = { + .cra_name = "ofb(aes)", + .cra_driver_name = "ofb-aes-ppc4xx", + .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct crypto4xx_ctx), + .cra_type = &crypto_ablkcipher_type, + .cra_init = crypto4xx_ablk_init, + .cra_exit = crypto4xx_ablk_exit, + .cra_module = THIS_MODULE, + .cra_u = { + .ablkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_IV_SIZE, + .setkey = crypto4xx_setkey_aes_ofb, + .encrypt = crypto4xx_encrypt, + .decrypt = crypto4xx_decrypt, + } + } + } }, + + /* AEAD */ + { .type = CRYPTO_ALG_TYPE_AEAD, .u.aead = { + .setkey = crypto4xx_setkey_aes_ccm, + .setauthsize = crypto4xx_setauthsize_aead, + .encrypt = crypto4xx_encrypt_aes_ccm, + .decrypt = crypto4xx_decrypt_aes_ccm, + .init = crypto4xx_aead_init, + .exit = crypto4xx_aead_exit, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = 16, + .base = { + .cra_name = "ccm(aes)", + .cra_driver_name = "ccm-aes-ppc4xx", + .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct crypto4xx_ctx), + .cra_module = THIS_MODULE, + }, + } }, + { .type = CRYPTO_ALG_TYPE_AEAD, .u.aead = { + .setkey = crypto4xx_setkey_aes_gcm, + .setauthsize = crypto4xx_setauthsize_aead, + .encrypt = crypto4xx_encrypt_aes_gcm, + .decrypt = crypto4xx_decrypt_aes_gcm, + .init = crypto4xx_aead_init, + .exit = crypto4xx_aead_exit, + .ivsize = GCM_AES_IV_SIZE, + .maxauthsize = 16, + .base = { + .cra_name = "gcm(aes)", + .cra_driver_name = "gcm-aes-ppc4xx", + .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct crypto4xx_ctx), + .cra_module = THIS_MODULE, + }, + } }, }; /** @@ -1187,13 +1308,14 @@ static int crypto4xx_probe(struct platform_device *ofdev) core_dev->device = dev; spin_lock_init(&core_dev->lock); INIT_LIST_HEAD(&core_dev->dev->alg_list); + ratelimit_default_init(&core_dev->dev->aead_ratelimit); rc = crypto4xx_build_pdr(core_dev->dev); if (rc) goto err_build_pdr; rc = crypto4xx_build_gdr(core_dev->dev); if (rc) - goto err_build_gdr; + goto err_build_pdr; rc = crypto4xx_build_sdr(core_dev->dev); if (rc) @@ -1236,12 +1358,11 @@ err_iomap: err_request_irq: irq_dispose_mapping(core_dev->irq); tasklet_kill(&core_dev->tasklet); - crypto4xx_destroy_sdr(core_dev->dev); err_build_sdr: + crypto4xx_destroy_sdr(core_dev->dev); crypto4xx_destroy_gdr(core_dev->dev); -err_build_gdr: - crypto4xx_destroy_pdr(core_dev->dev); err_build_pdr: + crypto4xx_destroy_pdr(core_dev->dev); kfree(core_dev->dev); err_alloc_dev: kfree(core_dev); diff --git a/drivers/crypto/amcc/crypto4xx_core.h b/drivers/crypto/amcc/crypto4xx_core.h index ecfdcfe3698d..8ac3bd37203b 100644 --- a/drivers/crypto/amcc/crypto4xx_core.h +++ b/drivers/crypto/amcc/crypto4xx_core.h @@ -22,7 +22,11 @@ #ifndef __CRYPTO4XX_CORE_H__ #define __CRYPTO4XX_CORE_H__ +#include <linux/ratelimit.h> #include <crypto/internal/hash.h> +#include <crypto/internal/aead.h> +#include "crypto4xx_reg_def.h" +#include "crypto4xx_sa.h" #define MODULE_NAME "crypto4xx" @@ -34,20 +38,28 @@ #define PPC405EX_CE_RESET 0x00000008 #define CRYPTO4XX_CRYPTO_PRIORITY 300 -#define PPC4XX_LAST_PD 63 -#define PPC4XX_NUM_PD 64 -#define PPC4XX_LAST_GD 1023 +#define PPC4XX_NUM_PD 256 +#define PPC4XX_LAST_PD (PPC4XX_NUM_PD - 1) #define PPC4XX_NUM_GD 1024 -#define PPC4XX_LAST_SD 63 -#define PPC4XX_NUM_SD 64 +#define PPC4XX_LAST_GD (PPC4XX_NUM_GD - 1) +#define PPC4XX_NUM_SD 256 +#define PPC4XX_LAST_SD (PPC4XX_NUM_SD - 1) #define PPC4XX_SD_BUFFER_SIZE 2048 -#define PD_ENTRY_INUSE 1 +#define PD_ENTRY_BUSY BIT(1) +#define PD_ENTRY_INUSE BIT(0) #define PD_ENTRY_FREE 0 #define ERING_WAS_FULL 0xffffffff struct crypto4xx_device; +union shadow_sa_buf { + struct dynamic_sa_ctl sa; + + /* alloc 256 bytes which is enough for any kind of dynamic sa */ + u8 buf[256]; +} __packed; + struct pd_uinfo { struct crypto4xx_device *dev; u32 state; @@ -60,9 +72,8 @@ struct pd_uinfo { used by this packet */ u32 num_sd; /* number of scatter discriptors used by this packet */ - void *sa_va; /* shadow sa, when using cp from ctx->sa */ - u32 sa_pa; - void *sr_va; /* state record for shadow sa */ + struct dynamic_sa_ctl *sa_va; /* shadow sa */ + struct sa_state_record *sr_va; /* state record for shadow sa */ u32 sr_pa; struct scatterlist *dest_va; struct crypto_async_request *async_req; /* base crypto request @@ -72,27 +83,21 @@ struct pd_uinfo { struct crypto4xx_device { struct crypto4xx_core_device *core_dev; char *name; - u64 ce_phy_address; void __iomem *ce_base; void __iomem *trng_base; - void *pdr; /* base address of packet - descriptor ring */ - dma_addr_t pdr_pa; /* physical address used to - program ce pdr_base_register */ - void *gdr; /* gather descriptor ring */ - dma_addr_t gdr_pa; /* physical address used to - program ce gdr_base_register */ - void *sdr; /* scatter descriptor ring */ - dma_addr_t sdr_pa; /* physical address used to - program ce sdr_base_register */ + struct ce_pd *pdr; /* base address of packet descriptor ring */ + dma_addr_t pdr_pa; /* physical address of pdr_base_register */ + struct ce_gd *gdr; /* gather descriptor ring */ + dma_addr_t gdr_pa; /* physical address of gdr_base_register */ + struct ce_sd *sdr; /* scatter descriptor ring */ + dma_addr_t sdr_pa; /* physical address of sdr_base_register */ void *scatter_buffer_va; dma_addr_t scatter_buffer_pa; - u32 scatter_buffer_size; - void *shadow_sa_pool; /* pool of memory for sa in pd_uinfo */ + union shadow_sa_buf *shadow_sa_pool; dma_addr_t shadow_sa_pool_pa; - void *shadow_sr_pool; /* pool of memory for sr in pd_uinfo */ + struct sa_state_record *shadow_sr_pool; dma_addr_t shadow_sr_pool_pa; u32 pdr_tail; u32 pdr_head; @@ -100,9 +105,10 @@ struct crypto4xx_device { u32 gdr_head; u32 sdr_tail; u32 sdr_head; - void *pdr_uinfo; + struct pd_uinfo *pdr_uinfo; struct list_head alg_list; /* List of algorithm supported by this device */ + struct ratelimit_state aead_ratelimit; }; struct crypto4xx_core_device { @@ -118,30 +124,13 @@ struct crypto4xx_core_device { struct crypto4xx_ctx { struct crypto4xx_device *dev; - void *sa_in; - dma_addr_t sa_in_dma_addr; - void *sa_out; - dma_addr_t sa_out_dma_addr; - void *state_record; - dma_addr_t state_record_dma_addr; + struct dynamic_sa_ctl *sa_in; + struct dynamic_sa_ctl *sa_out; + __le32 iv_nonce; u32 sa_len; - u32 offset_to_sr_ptr; /* offset to state ptr, in dynamic sa */ - u32 direction; - u32 next_hdr; - u32 save_iv; - u32 pd_ctl_len; - u32 pd_ctl; - u32 bypass; - u32 is_hash; - u32 hash_final; -}; - -struct crypto4xx_req_ctx { - struct crypto4xx_device *dev; /* Device in which - operation to send to */ - void *sa; - u32 sa_dma_addr; - u16 sa_len; + union { + struct crypto_aead *aead; + } sw_cipher; }; struct crypto4xx_alg_common { @@ -149,6 +138,7 @@ struct crypto4xx_alg_common { union { struct crypto_alg cipher; struct ahash_alg hash; + struct aead_alg aead; } u; }; @@ -158,43 +148,90 @@ struct crypto4xx_alg { struct crypto4xx_device *dev; }; -static inline struct crypto4xx_alg *crypto_alg_to_crypto4xx_alg( - struct crypto_alg *x) +int crypto4xx_alloc_sa(struct crypto4xx_ctx *ctx, u32 size); +void crypto4xx_free_sa(struct crypto4xx_ctx *ctx); +void crypto4xx_free_ctx(struct crypto4xx_ctx *ctx); +int crypto4xx_build_pd(struct crypto_async_request *req, + struct crypto4xx_ctx *ctx, + struct scatterlist *src, + struct scatterlist *dst, + const unsigned int datalen, + const __le32 *iv, const u32 iv_len, + const struct dynamic_sa_ctl *sa, + const unsigned int sa_len, + const unsigned int assoclen); +int crypto4xx_setkey_aes_cbc(struct crypto_ablkcipher *cipher, + const u8 *key, unsigned int keylen); +int crypto4xx_setkey_aes_cfb(struct crypto_ablkcipher *cipher, + const u8 *key, unsigned int keylen); +int crypto4xx_setkey_aes_ecb(struct crypto_ablkcipher *cipher, + const u8 *key, unsigned int keylen); +int crypto4xx_setkey_aes_ofb(struct crypto_ablkcipher *cipher, + const u8 *key, unsigned int keylen); +int crypto4xx_setkey_rfc3686(struct crypto_ablkcipher *cipher, + const u8 *key, unsigned int keylen); +int crypto4xx_encrypt(struct ablkcipher_request *req); +int crypto4xx_decrypt(struct ablkcipher_request *req); +int crypto4xx_rfc3686_encrypt(struct ablkcipher_request *req); +int crypto4xx_rfc3686_decrypt(struct ablkcipher_request *req); +int crypto4xx_sha1_alg_init(struct crypto_tfm *tfm); +int crypto4xx_hash_digest(struct ahash_request *req); +int crypto4xx_hash_final(struct ahash_request *req); +int crypto4xx_hash_update(struct ahash_request *req); +int crypto4xx_hash_init(struct ahash_request *req); + +/** + * Note: Only use this function to copy items that is word aligned. + */ +static inline void crypto4xx_memcpy_swab32(u32 *dst, const void *buf, + size_t len) { - switch (x->cra_flags & CRYPTO_ALG_TYPE_MASK) { - case CRYPTO_ALG_TYPE_AHASH: - return container_of(__crypto_ahash_alg(x), - struct crypto4xx_alg, alg.u.hash); + for (; len >= 4; buf += 4, len -= 4) + *dst++ = __swab32p((u32 *) buf); + + if (len) { + const u8 *tmp = (u8 *)buf; + + switch (len) { + case 3: + *dst = (tmp[2] << 16) | + (tmp[1] << 8) | + tmp[0]; + break; + case 2: + *dst = (tmp[1] << 8) | + tmp[0]; + break; + case 1: + *dst = tmp[0]; + break; + default: + break; + } } +} - return container_of(x, struct crypto4xx_alg, alg.u.cipher); +static inline void crypto4xx_memcpy_from_le32(u32 *dst, const void *buf, + size_t len) +{ + crypto4xx_memcpy_swab32(dst, buf, len); } -extern int crypto4xx_alloc_sa(struct crypto4xx_ctx *ctx, u32 size); -extern void crypto4xx_free_sa(struct crypto4xx_ctx *ctx); -extern u32 crypto4xx_alloc_sa_rctx(struct crypto4xx_ctx *ctx, - struct crypto4xx_ctx *rctx); -extern void crypto4xx_free_sa_rctx(struct crypto4xx_ctx *rctx); -extern void crypto4xx_free_ctx(struct crypto4xx_ctx *ctx); -extern u32 crypto4xx_alloc_state_record(struct crypto4xx_ctx *ctx); -extern u32 get_dynamic_sa_offset_state_ptr_field(struct crypto4xx_ctx *ctx); -extern u32 get_dynamic_sa_offset_key_field(struct crypto4xx_ctx *ctx); -extern u32 get_dynamic_sa_iv_size(struct crypto4xx_ctx *ctx); -extern void crypto4xx_memcpy_le(unsigned int *dst, - const unsigned char *buf, int len); -extern u32 crypto4xx_build_pd(struct crypto_async_request *req, - struct crypto4xx_ctx *ctx, - struct scatterlist *src, - struct scatterlist *dst, - unsigned int datalen, - void *iv, u32 iv_len); -extern int crypto4xx_setkey_aes_cbc(struct crypto_ablkcipher *cipher, - const u8 *key, unsigned int keylen); -extern int crypto4xx_encrypt(struct ablkcipher_request *req); -extern int crypto4xx_decrypt(struct ablkcipher_request *req); -extern int crypto4xx_sha1_alg_init(struct crypto_tfm *tfm); -extern int crypto4xx_hash_digest(struct ahash_request *req); -extern int crypto4xx_hash_final(struct ahash_request *req); -extern int crypto4xx_hash_update(struct ahash_request *req); -extern int crypto4xx_hash_init(struct ahash_request *req); +static inline void crypto4xx_memcpy_to_le32(__le32 *dst, const void *buf, + size_t len) +{ + crypto4xx_memcpy_swab32((u32 *)dst, buf, len); +} + +int crypto4xx_setauthsize_aead(struct crypto_aead *ciper, + unsigned int authsize); +int crypto4xx_setkey_aes_ccm(struct crypto_aead *cipher, + const u8 *key, unsigned int keylen); +int crypto4xx_encrypt_aes_ccm(struct aead_request *req); +int crypto4xx_decrypt_aes_ccm(struct aead_request *req); +int crypto4xx_setkey_aes_gcm(struct crypto_aead *cipher, + const u8 *key, unsigned int keylen); +int crypto4xx_encrypt_aes_gcm(struct aead_request *req); +int crypto4xx_decrypt_aes_gcm(struct aead_request *req); + #endif diff --git a/drivers/crypto/amcc/crypto4xx_reg_def.h b/drivers/crypto/amcc/crypto4xx_reg_def.h index 279b8725559f..0a22ec5d1a96 100644 --- a/drivers/crypto/amcc/crypto4xx_reg_def.h +++ b/drivers/crypto/amcc/crypto4xx_reg_def.h @@ -261,6 +261,9 @@ union ce_pd_ctl { } bf; u32 w; } __attribute__((packed)); +#define PD_CTL_HASH_FINAL BIT(4) +#define PD_CTL_PE_DONE BIT(1) +#define PD_CTL_HOST_READY BIT(0) union ce_pd_ctl_len { struct { diff --git a/drivers/crypto/amcc/crypto4xx_sa.c b/drivers/crypto/amcc/crypto4xx_sa.c deleted file mode 100644 index 69182e2cc3ea..000000000000 --- a/drivers/crypto/amcc/crypto4xx_sa.c +++ /dev/null @@ -1,85 +0,0 @@ -/** - * AMCC SoC PPC4xx Crypto Driver - * - * Copyright (c) 2008 Applied Micro Circuits Corporation. - * All rights reserved. James Hsiao <jhsiao@amcc.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * @file crypto4xx_sa.c - * - * This file implements the security context - * associate format. - */ -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/moduleparam.h> -#include <linux/mod_devicetable.h> -#include <linux/interrupt.h> -#include <linux/spinlock_types.h> -#include <linux/highmem.h> -#include <linux/scatterlist.h> -#include <linux/crypto.h> -#include <crypto/algapi.h> -#include <crypto/des.h> -#include "crypto4xx_reg_def.h" -#include "crypto4xx_sa.h" -#include "crypto4xx_core.h" - -u32 get_dynamic_sa_offset_state_ptr_field(struct crypto4xx_ctx *ctx) -{ - u32 offset; - union dynamic_sa_contents cts; - - if (ctx->direction == DIR_INBOUND) - cts.w = ((struct dynamic_sa_ctl *) ctx->sa_in)->sa_contents; - else - cts.w = ((struct dynamic_sa_ctl *) ctx->sa_out)->sa_contents; - offset = cts.bf.key_size - + cts.bf.inner_size - + cts.bf.outer_size - + cts.bf.spi - + cts.bf.seq_num0 - + cts.bf.seq_num1 - + cts.bf.seq_num_mask0 - + cts.bf.seq_num_mask1 - + cts.bf.seq_num_mask2 - + cts.bf.seq_num_mask3 - + cts.bf.iv0 - + cts.bf.iv1 - + cts.bf.iv2 - + cts.bf.iv3; - - return sizeof(struct dynamic_sa_ctl) + offset * 4; -} - -u32 get_dynamic_sa_iv_size(struct crypto4xx_ctx *ctx) -{ - union dynamic_sa_contents cts; - - if (ctx->direction == DIR_INBOUND) - cts.w = ((struct dynamic_sa_ctl *) ctx->sa_in)->sa_contents; - else - cts.w = ((struct dynamic_sa_ctl *) ctx->sa_out)->sa_contents; - return (cts.bf.iv0 + cts.bf.iv1 + cts.bf.iv2 + cts.bf.iv3) * 4; -} - -u32 get_dynamic_sa_offset_key_field(struct crypto4xx_ctx *ctx) -{ - union dynamic_sa_contents cts; - - if (ctx->direction == DIR_INBOUND) - cts.w = ((struct dynamic_sa_ctl *) ctx->sa_in)->sa_contents; - else - cts.w = ((struct dynamic_sa_ctl *) ctx->sa_out)->sa_contents; - - return sizeof(struct dynamic_sa_ctl); -} diff --git a/drivers/crypto/amcc/crypto4xx_sa.h b/drivers/crypto/amcc/crypto4xx_sa.h index 1352d58d4e34..a4d403528db5 100644 --- a/drivers/crypto/amcc/crypto4xx_sa.h +++ b/drivers/crypto/amcc/crypto4xx_sa.h @@ -55,6 +55,8 @@ union dynamic_sa_contents { #define SA_OP_GROUP_BASIC 0 #define SA_OPCODE_ENCRYPT 0 #define SA_OPCODE_DECRYPT 0 +#define SA_OPCODE_ENCRYPT_HASH 1 +#define SA_OPCODE_HASH_DECRYPT 1 #define SA_OPCODE_HASH 3 #define SA_CIPHER_ALG_DES 0 #define SA_CIPHER_ALG_3DES 1 @@ -65,6 +67,8 @@ union dynamic_sa_contents { #define SA_HASH_ALG_MD5 0 #define SA_HASH_ALG_SHA1 1 +#define SA_HASH_ALG_GHASH 12 +#define SA_HASH_ALG_CBC_MAC 14 #define SA_HASH_ALG_NULL 15 #define SA_HASH_ALG_SHA1_DIGEST_SIZE 20 @@ -112,6 +116,9 @@ union sa_command_0 { #define CRYPTO_MODE_ECB 0 #define CRYPTO_MODE_CBC 1 +#define CRYPTO_MODE_OFB 2 +#define CRYPTO_MODE_CFB 3 +#define CRYPTO_MODE_CTR 4 #define CRYPTO_FEEDBACK_MODE_NO_FB 0 #define CRYPTO_FEEDBACK_MODE_64BIT_OFB 0 @@ -169,7 +176,7 @@ union sa_command_1 { } __attribute__((packed)); struct dynamic_sa_ctl { - u32 sa_contents; + union dynamic_sa_contents sa_contents; union sa_command_0 sa_command_0; union sa_command_1 sa_command_1; } __attribute__((packed)); @@ -178,9 +185,12 @@ struct dynamic_sa_ctl { * State Record for Security Association (SA) */ struct sa_state_record { - u32 save_iv[4]; - u32 save_hash_byte_cnt[2]; - u32 save_digest[16]; + __le32 save_iv[4]; + __le32 save_hash_byte_cnt[2]; + union { + u32 save_digest[16]; /* for MD5/SHA */ + __le32 save_digest_le32[16]; /* GHASH / CBC */ + }; } __attribute__((packed)); /** @@ -189,8 +199,8 @@ struct sa_state_record { */ struct dynamic_sa_aes128 { struct dynamic_sa_ctl ctrl; - u32 key[4]; - u32 iv[4]; /* for CBC, OFC, and CFB mode */ + __le32 key[4]; + __le32 iv[4]; /* for CBC, OFC, and CFB mode */ u32 state_ptr; u32 reserved; } __attribute__((packed)); @@ -203,8 +213,8 @@ struct dynamic_sa_aes128 { */ struct dynamic_sa_aes192 { struct dynamic_sa_ctl ctrl; - u32 key[6]; - u32 iv[4]; /* for CBC, OFC, and CFB mode */ + __le32 key[6]; + __le32 iv[4]; /* for CBC, OFC, and CFB mode */ u32 state_ptr; u32 reserved; } __attribute__((packed)); @@ -217,8 +227,8 @@ struct dynamic_sa_aes192 { */ struct dynamic_sa_aes256 { struct dynamic_sa_ctl ctrl; - u32 key[8]; - u32 iv[4]; /* for CBC, OFC, and CFB mode */ + __le32 key[8]; + __le32 iv[4]; /* for CBC, OFC, and CFB mode */ u32 state_ptr; u32 reserved; } __attribute__((packed)); @@ -228,16 +238,81 @@ struct dynamic_sa_aes256 { #define SA_AES_CONTENTS 0x3e000002 /** + * Security Association (SA) for AES128 CCM + */ +struct dynamic_sa_aes128_ccm { + struct dynamic_sa_ctl ctrl; + __le32 key[4]; + __le32 iv[4]; + u32 state_ptr; + u32 reserved; +} __packed; +#define SA_AES128_CCM_LEN (sizeof(struct dynamic_sa_aes128_ccm)/4) +#define SA_AES128_CCM_CONTENTS 0x3e000042 +#define SA_AES_CCM_CONTENTS 0x3e000002 + +/** + * Security Association (SA) for AES128_GCM + */ +struct dynamic_sa_aes128_gcm { + struct dynamic_sa_ctl ctrl; + __le32 key[4]; + __le32 inner_digest[4]; + __le32 iv[4]; + u32 state_ptr; + u32 reserved; +} __packed; + +#define SA_AES128_GCM_LEN (sizeof(struct dynamic_sa_aes128_gcm)/4) +#define SA_AES128_GCM_CONTENTS 0x3e000442 +#define SA_AES_GCM_CONTENTS 0x3e000402 + +/** * Security Association (SA) for HASH160: HMAC-SHA1 */ struct dynamic_sa_hash160 { struct dynamic_sa_ctl ctrl; - u32 inner_digest[5]; - u32 outer_digest[5]; + __le32 inner_digest[5]; + __le32 outer_digest[5]; u32 state_ptr; u32 reserved; } __attribute__((packed)); #define SA_HASH160_LEN (sizeof(struct dynamic_sa_hash160)/4) #define SA_HASH160_CONTENTS 0x2000a502 +static inline u32 +get_dynamic_sa_offset_state_ptr_field(struct dynamic_sa_ctl *cts) +{ + u32 offset; + + offset = cts->sa_contents.bf.key_size + + cts->sa_contents.bf.inner_size + + cts->sa_contents.bf.outer_size + + cts->sa_contents.bf.spi + + cts->sa_contents.bf.seq_num0 + + cts->sa_contents.bf.seq_num1 + + cts->sa_contents.bf.seq_num_mask0 + + cts->sa_contents.bf.seq_num_mask1 + + cts->sa_contents.bf.seq_num_mask2 + + cts->sa_contents.bf.seq_num_mask3 + + cts->sa_contents.bf.iv0 + + cts->sa_contents.bf.iv1 + + cts->sa_contents.bf.iv2 + + cts->sa_contents.bf.iv3; + + return sizeof(struct dynamic_sa_ctl) + offset * 4; +} + +static inline __le32 *get_dynamic_sa_key_field(struct dynamic_sa_ctl *cts) +{ + return (__le32 *) ((unsigned long)cts + sizeof(struct dynamic_sa_ctl)); +} + +static inline __le32 *get_dynamic_sa_inner_digest(struct dynamic_sa_ctl *cts) +{ + return (__le32 *) ((unsigned long)cts + + sizeof(struct dynamic_sa_ctl) + + cts->sa_contents.bf.key_size * 4); +} + #endif diff --git a/drivers/crypto/atmel-aes-regs.h b/drivers/crypto/atmel-aes-regs.h index 7694679802b3..62f9d3038757 100644 --- a/drivers/crypto/atmel-aes-regs.h +++ b/drivers/crypto/atmel-aes-regs.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __ATMEL_AES_REGS_H__ #define __ATMEL_AES_REGS_H__ diff --git a/drivers/crypto/atmel-aes.c b/drivers/crypto/atmel-aes.c index 29e20c37f3a6..691c6465b71e 100644 --- a/drivers/crypto/atmel-aes.c +++ b/drivers/crypto/atmel-aes.c @@ -36,6 +36,7 @@ #include <crypto/scatterwalk.h> #include <crypto/algapi.h> #include <crypto/aes.h> +#include <crypto/gcm.h> #include <crypto/xts.h> #include <crypto/internal/aead.h> #include <linux/platform_data/crypto-atmel.h> @@ -76,12 +77,11 @@ AES_FLAGS_ENCRYPT | \ AES_FLAGS_GTAGEN) -#define AES_FLAGS_INIT BIT(2) #define AES_FLAGS_BUSY BIT(3) #define AES_FLAGS_DUMP_REG BIT(4) #define AES_FLAGS_OWN_SHA BIT(5) -#define AES_FLAGS_PERSISTENT (AES_FLAGS_INIT | AES_FLAGS_BUSY) +#define AES_FLAGS_PERSISTENT AES_FLAGS_BUSY #define ATMEL_AES_QUEUE_LENGTH 50 @@ -110,6 +110,7 @@ struct atmel_aes_base_ctx { int keylen; u32 key[AES_KEYSIZE_256 / sizeof(u32)]; u16 block_size; + bool is_aead; }; struct atmel_aes_ctx { @@ -156,6 +157,7 @@ struct atmel_aes_authenc_ctx { struct atmel_aes_reqctx { unsigned long mode; + u32 lastc[AES_BLOCK_SIZE / sizeof(u32)]; }; #ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC @@ -448,11 +450,8 @@ static int atmel_aes_hw_init(struct atmel_aes_dev *dd) if (err) return err; - if (!(dd->flags & AES_FLAGS_INIT)) { - atmel_aes_write(dd, AES_CR, AES_CR_SWRST); - atmel_aes_write(dd, AES_MR, 0xE << AES_MR_CKEY_OFFSET); - dd->flags |= AES_FLAGS_INIT; - } + atmel_aes_write(dd, AES_CR, AES_CR_SWRST); + atmel_aes_write(dd, AES_MR, 0xE << AES_MR_CKEY_OFFSET); return 0; } @@ -497,12 +496,34 @@ static void atmel_aes_authenc_complete(struct atmel_aes_dev *dd, int err); static inline int atmel_aes_complete(struct atmel_aes_dev *dd, int err) { #ifdef CONFIG_CRYPTO_DEV_ATMEL_AUTHENC - atmel_aes_authenc_complete(dd, err); + if (dd->ctx->is_aead) + atmel_aes_authenc_complete(dd, err); #endif clk_disable(dd->iclk); dd->flags &= ~AES_FLAGS_BUSY; + if (!dd->ctx->is_aead) { + struct ablkcipher_request *req = + ablkcipher_request_cast(dd->areq); + struct atmel_aes_reqctx *rctx = ablkcipher_request_ctx(req); + struct crypto_ablkcipher *ablkcipher = + crypto_ablkcipher_reqtfm(req); + int ivsize = crypto_ablkcipher_ivsize(ablkcipher); + + if (rctx->mode & AES_FLAGS_ENCRYPT) { + scatterwalk_map_and_copy(req->info, req->dst, + req->nbytes - ivsize, ivsize, 0); + } else { + if (req->src == req->dst) { + memcpy(req->info, rctx->lastc, ivsize); + } else { + scatterwalk_map_and_copy(req->info, req->src, + req->nbytes - ivsize, ivsize, 0); + } + } + } + if (dd->is_async) dd->areq->complete(dd->areq, err); @@ -1071,11 +1092,11 @@ static int atmel_aes_ctr_start(struct atmel_aes_dev *dd) static int atmel_aes_crypt(struct ablkcipher_request *req, unsigned long mode) { - struct atmel_aes_base_ctx *ctx; + struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); + struct atmel_aes_base_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); struct atmel_aes_reqctx *rctx; struct atmel_aes_dev *dd; - ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req)); switch (mode & AES_FLAGS_OPMODE_MASK) { case AES_FLAGS_CFB8: ctx->block_size = CFB8_BLOCK_SIZE; @@ -1097,6 +1118,7 @@ static int atmel_aes_crypt(struct ablkcipher_request *req, unsigned long mode) ctx->block_size = AES_BLOCK_SIZE; break; } + ctx->is_aead = false; dd = atmel_aes_find_dev(ctx); if (!dd) @@ -1105,6 +1127,13 @@ static int atmel_aes_crypt(struct ablkcipher_request *req, unsigned long mode) rctx = ablkcipher_request_ctx(req); rctx->mode = mode; + if (!(mode & AES_FLAGS_ENCRYPT) && (req->src == req->dst)) { + int ivsize = crypto_ablkcipher_ivsize(ablkcipher); + + scatterwalk_map_and_copy(rctx->lastc, req->src, + (req->nbytes - ivsize), ivsize, 0); + } + return atmel_aes_handle_queue(dd, &req->base); } @@ -1236,10 +1265,6 @@ static int atmel_aes_ctr_cra_init(struct crypto_tfm *tfm) return 0; } -static void atmel_aes_cra_exit(struct crypto_tfm *tfm) -{ -} - static struct crypto_alg aes_algs[] = { { .cra_name = "ecb(aes)", @@ -1252,7 +1277,6 @@ static struct crypto_alg aes_algs[] = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_aes_cra_init, - .cra_exit = atmel_aes_cra_exit, .cra_u.ablkcipher = { .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, @@ -1272,7 +1296,6 @@ static struct crypto_alg aes_algs[] = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_aes_cra_init, - .cra_exit = atmel_aes_cra_exit, .cra_u.ablkcipher = { .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, @@ -1293,7 +1316,6 @@ static struct crypto_alg aes_algs[] = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_aes_cra_init, - .cra_exit = atmel_aes_cra_exit, .cra_u.ablkcipher = { .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, @@ -1314,7 +1336,6 @@ static struct crypto_alg aes_algs[] = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_aes_cra_init, - .cra_exit = atmel_aes_cra_exit, .cra_u.ablkcipher = { .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, @@ -1335,7 +1356,6 @@ static struct crypto_alg aes_algs[] = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_aes_cra_init, - .cra_exit = atmel_aes_cra_exit, .cra_u.ablkcipher = { .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, @@ -1356,7 +1376,6 @@ static struct crypto_alg aes_algs[] = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_aes_cra_init, - .cra_exit = atmel_aes_cra_exit, .cra_u.ablkcipher = { .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, @@ -1377,7 +1396,6 @@ static struct crypto_alg aes_algs[] = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_aes_cra_init, - .cra_exit = atmel_aes_cra_exit, .cra_u.ablkcipher = { .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, @@ -1398,7 +1416,6 @@ static struct crypto_alg aes_algs[] = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_aes_ctr_cra_init, - .cra_exit = atmel_aes_cra_exit, .cra_u.ablkcipher = { .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, @@ -1421,7 +1438,6 @@ static struct crypto_alg aes_cfb64_alg = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_aes_cra_init, - .cra_exit = atmel_aes_cra_exit, .cra_u.ablkcipher = { .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, @@ -1532,7 +1548,7 @@ static int atmel_aes_gcm_start(struct atmel_aes_dev *dd) if (err) return atmel_aes_complete(dd, err); - if (likely(ivsize == 12)) { + if (likely(ivsize == GCM_AES_IV_SIZE)) { memcpy(ctx->j0, iv, ivsize); ctx->j0[3] = cpu_to_be32(1); return atmel_aes_gcm_process(dd); @@ -1739,6 +1755,7 @@ static int atmel_aes_gcm_crypt(struct aead_request *req, ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); ctx->block_size = AES_BLOCK_SIZE; + ctx->is_aead = true; dd = atmel_aes_find_dev(ctx); if (!dd) @@ -1808,19 +1825,13 @@ static int atmel_aes_gcm_init(struct crypto_aead *tfm) return 0; } -static void atmel_aes_gcm_exit(struct crypto_aead *tfm) -{ - -} - static struct aead_alg aes_gcm_alg = { .setkey = atmel_aes_gcm_setkey, .setauthsize = atmel_aes_gcm_setauthsize, .encrypt = atmel_aes_gcm_encrypt, .decrypt = atmel_aes_gcm_decrypt, .init = atmel_aes_gcm_init, - .exit = atmel_aes_gcm_exit, - .ivsize = 12, + .ivsize = GCM_AES_IV_SIZE, .maxauthsize = AES_BLOCK_SIZE, .base = { @@ -1955,7 +1966,6 @@ static struct crypto_alg aes_xts_alg = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_aes_xts_cra_init, - .cra_exit = atmel_aes_cra_exit, .cra_u.ablkcipher = { .min_keysize = 2 * AES_MIN_KEY_SIZE, .max_keysize = 2 * AES_MAX_KEY_SIZE, @@ -2223,6 +2233,7 @@ static int atmel_aes_authenc_crypt(struct aead_request *req, rctx->base.mode = mode; ctx->block_size = AES_BLOCK_SIZE; + ctx->is_aead = true; dd = atmel_aes_find_dev(ctx); if (!dd) @@ -2382,7 +2393,6 @@ static int atmel_aes_dma_init(struct atmel_aes_dev *dd, struct crypto_platform_data *pdata) { struct at_dma_slave *slave; - int err = -ENOMEM; dma_cap_mask_t mask; dma_cap_zero(mask); @@ -2407,7 +2417,7 @@ err_dma_out: dma_release_channel(dd->src.chan); err_dma_in: dev_warn(dd->dev, "no DMA channel available\n"); - return err; + return -ENODEV; } static void atmel_aes_dma_cleanup(struct atmel_aes_dev *dd) @@ -2658,8 +2668,6 @@ static int atmel_aes_probe(struct platform_device *pdev) crypto_init_queue(&aes_dd->queue, ATMEL_AES_QUEUE_LENGTH); - aes_dd->irq = -1; - /* Get the base address */ aes_res = platform_get_resource(pdev, IORESOURCE_MEM, 0); if (!aes_res) { diff --git a/drivers/crypto/atmel-ecc.c b/drivers/crypto/atmel-ecc.c new file mode 100644 index 000000000000..e66f18a0ddd0 --- /dev/null +++ b/drivers/crypto/atmel-ecc.c @@ -0,0 +1,781 @@ +/* + * Microchip / Atmel ECC (I2C) driver. + * + * Copyright (c) 2017, Microchip Technology Inc. + * Author: Tudor Ambarus <tudor.ambarus@microchip.com> + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include <linux/bitrev.h> +#include <linux/crc16.h> +#include <linux/delay.h> +#include <linux/device.h> +#include <linux/err.h> +#include <linux/errno.h> +#include <linux/i2c.h> +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/of_device.h> +#include <linux/scatterlist.h> +#include <linux/slab.h> +#include <linux/workqueue.h> +#include <crypto/internal/kpp.h> +#include <crypto/ecdh.h> +#include <crypto/kpp.h> +#include "atmel-ecc.h" + +/* Used for binding tfm objects to i2c clients. */ +struct atmel_ecc_driver_data { + struct list_head i2c_client_list; + spinlock_t i2c_list_lock; +} ____cacheline_aligned; + +static struct atmel_ecc_driver_data driver_data; + +/** + * atmel_ecc_i2c_client_priv - i2c_client private data + * @client : pointer to i2c client device + * @i2c_client_list_node: part of i2c_client_list + * @lock : lock for sending i2c commands + * @wake_token : wake token array of zeros + * @wake_token_sz : size in bytes of the wake_token + * @tfm_count : number of active crypto transformations on i2c client + * + * Reads and writes from/to the i2c client are sequential. The first byte + * transmitted to the device is treated as the byte size. Any attempt to send + * more than this number of bytes will cause the device to not ACK those bytes. + * After the host writes a single command byte to the input buffer, reads are + * prohibited until after the device completes command execution. Use a mutex + * when sending i2c commands. + */ +struct atmel_ecc_i2c_client_priv { + struct i2c_client *client; + struct list_head i2c_client_list_node; + struct mutex lock; + u8 wake_token[WAKE_TOKEN_MAX_SIZE]; + size_t wake_token_sz; + atomic_t tfm_count ____cacheline_aligned; +}; + +/** + * atmel_ecdh_ctx - transformation context + * @client : pointer to i2c client device + * @fallback : used for unsupported curves or when user wants to use its own + * private key. + * @public_key : generated when calling set_secret(). It's the responsibility + * of the user to not call set_secret() while + * generate_public_key() or compute_shared_secret() are in flight. + * @curve_id : elliptic curve id + * @n_sz : size in bytes of the n prime + * @do_fallback: true when the device doesn't support the curve or when the user + * wants to use its own private key. + */ +struct atmel_ecdh_ctx { + struct i2c_client *client; + struct crypto_kpp *fallback; + const u8 *public_key; + unsigned int curve_id; + size_t n_sz; + bool do_fallback; +}; + +/** + * atmel_ecc_work_data - data structure representing the work + * @ctx : transformation context. + * @cbk : pointer to a callback function to be invoked upon completion of this + * request. This has the form: + * callback(struct atmel_ecc_work_data *work_data, void *areq, u8 status) + * where: + * @work_data: data structure representing the work + * @areq : optional pointer to an argument passed with the original + * request. + * @status : status returned from the i2c client device or i2c error. + * @areq: optional pointer to a user argument for use at callback time. + * @work: describes the task to be executed. + * @cmd : structure used for communicating with the device. + */ +struct atmel_ecc_work_data { + struct atmel_ecdh_ctx *ctx; + void (*cbk)(struct atmel_ecc_work_data *work_data, void *areq, + int status); + void *areq; + struct work_struct work; + struct atmel_ecc_cmd cmd; +}; + +static u16 atmel_ecc_crc16(u16 crc, const u8 *buffer, size_t len) +{ + return cpu_to_le16(bitrev16(crc16(crc, buffer, len))); +} + +/** + * atmel_ecc_checksum() - Generate 16-bit CRC as required by ATMEL ECC. + * CRC16 verification of the count, opcode, param1, param2 and data bytes. + * The checksum is saved in little-endian format in the least significant + * two bytes of the command. CRC polynomial is 0x8005 and the initial register + * value should be zero. + * + * @cmd : structure used for communicating with the device. + */ +static void atmel_ecc_checksum(struct atmel_ecc_cmd *cmd) +{ + u8 *data = &cmd->count; + size_t len = cmd->count - CRC_SIZE; + u16 *crc16 = (u16 *)(data + len); + + *crc16 = atmel_ecc_crc16(0, data, len); +} + +static void atmel_ecc_init_read_cmd(struct atmel_ecc_cmd *cmd) +{ + cmd->word_addr = COMMAND; + cmd->opcode = OPCODE_READ; + /* + * Read the word from Configuration zone that contains the lock bytes + * (UserExtra, Selector, LockValue, LockConfig). + */ + cmd->param1 = CONFIG_ZONE; + cmd->param2 = DEVICE_LOCK_ADDR; + cmd->count = READ_COUNT; + + atmel_ecc_checksum(cmd); + + cmd->msecs = MAX_EXEC_TIME_READ; + cmd->rxsize = READ_RSP_SIZE; +} + +static void atmel_ecc_init_genkey_cmd(struct atmel_ecc_cmd *cmd, u16 keyid) +{ + cmd->word_addr = COMMAND; + cmd->count = GENKEY_COUNT; + cmd->opcode = OPCODE_GENKEY; + cmd->param1 = GENKEY_MODE_PRIVATE; + /* a random private key will be generated and stored in slot keyID */ + cmd->param2 = cpu_to_le16(keyid); + + atmel_ecc_checksum(cmd); + + cmd->msecs = MAX_EXEC_TIME_GENKEY; + cmd->rxsize = GENKEY_RSP_SIZE; +} + +static int atmel_ecc_init_ecdh_cmd(struct atmel_ecc_cmd *cmd, + struct scatterlist *pubkey) +{ + size_t copied; + + cmd->word_addr = COMMAND; + cmd->count = ECDH_COUNT; + cmd->opcode = OPCODE_ECDH; + cmd->param1 = ECDH_PREFIX_MODE; + /* private key slot */ + cmd->param2 = cpu_to_le16(DATA_SLOT_2); + + /* + * The device only supports NIST P256 ECC keys. The public key size will + * always be the same. Use a macro for the key size to avoid unnecessary + * computations. + */ + copied = sg_copy_to_buffer(pubkey, 1, cmd->data, ATMEL_ECC_PUBKEY_SIZE); + if (copied != ATMEL_ECC_PUBKEY_SIZE) + return -EINVAL; + + atmel_ecc_checksum(cmd); + + cmd->msecs = MAX_EXEC_TIME_ECDH; + cmd->rxsize = ECDH_RSP_SIZE; + + return 0; +} + +/* + * After wake and after execution of a command, there will be error, status, or + * result bytes in the device's output register that can be retrieved by the + * system. When the length of that group is four bytes, the codes returned are + * detailed in error_list. + */ +static int atmel_ecc_status(struct device *dev, u8 *status) +{ + size_t err_list_len = ARRAY_SIZE(error_list); + int i; + u8 err_id = status[1]; + + if (*status != STATUS_SIZE) + return 0; + + if (err_id == STATUS_WAKE_SUCCESSFUL || err_id == STATUS_NOERR) + return 0; + + for (i = 0; i < err_list_len; i++) + if (error_list[i].value == err_id) + break; + + /* if err_id is not in the error_list then ignore it */ + if (i != err_list_len) { + dev_err(dev, "%02x: %s:\n", err_id, error_list[i].error_text); + return err_id; + } + + return 0; +} + +static int atmel_ecc_wakeup(struct i2c_client *client) +{ + struct atmel_ecc_i2c_client_priv *i2c_priv = i2c_get_clientdata(client); + u8 status[STATUS_RSP_SIZE]; + int ret; + + /* + * The device ignores any levels or transitions on the SCL pin when the + * device is idle, asleep or during waking up. Don't check for error + * when waking up the device. + */ + i2c_master_send(client, i2c_priv->wake_token, i2c_priv->wake_token_sz); + + /* + * Wait to wake the device. Typical execution times for ecdh and genkey + * are around tens of milliseconds. Delta is chosen to 50 microseconds. + */ + usleep_range(TWHI_MIN, TWHI_MAX); + + ret = i2c_master_recv(client, status, STATUS_SIZE); + if (ret < 0) + return ret; + + return atmel_ecc_status(&client->dev, status); +} + +static int atmel_ecc_sleep(struct i2c_client *client) +{ + u8 sleep = SLEEP_TOKEN; + + return i2c_master_send(client, &sleep, 1); +} + +static void atmel_ecdh_done(struct atmel_ecc_work_data *work_data, void *areq, + int status) +{ + struct kpp_request *req = areq; + struct atmel_ecdh_ctx *ctx = work_data->ctx; + struct atmel_ecc_cmd *cmd = &work_data->cmd; + size_t copied; + size_t n_sz = ctx->n_sz; + + if (status) + goto free_work_data; + + /* copy the shared secret */ + copied = sg_copy_from_buffer(req->dst, 1, &cmd->data[RSP_DATA_IDX], + n_sz); + if (copied != n_sz) + status = -EINVAL; + + /* fall through */ +free_work_data: + kzfree(work_data); + kpp_request_complete(req, status); +} + +/* + * atmel_ecc_send_receive() - send a command to the device and receive its + * response. + * @client: i2c client device + * @cmd : structure used to communicate with the device + * + * After the device receives a Wake token, a watchdog counter starts within the + * device. After the watchdog timer expires, the device enters sleep mode + * regardless of whether some I/O transmission or command execution is in + * progress. If a command is attempted when insufficient time remains prior to + * watchdog timer execution, the device will return the watchdog timeout error + * code without attempting to execute the command. There is no way to reset the + * counter other than to put the device into sleep or idle mode and then + * wake it up again. + */ +static int atmel_ecc_send_receive(struct i2c_client *client, + struct atmel_ecc_cmd *cmd) +{ + struct atmel_ecc_i2c_client_priv *i2c_priv = i2c_get_clientdata(client); + int ret; + + mutex_lock(&i2c_priv->lock); + + ret = atmel_ecc_wakeup(client); + if (ret) + goto err; + + /* send the command */ + ret = i2c_master_send(client, (u8 *)cmd, cmd->count + WORD_ADDR_SIZE); + if (ret < 0) + goto err; + + /* delay the appropriate amount of time for command to execute */ + msleep(cmd->msecs); + + /* receive the response */ + ret = i2c_master_recv(client, cmd->data, cmd->rxsize); + if (ret < 0) + goto err; + + /* put the device into low-power mode */ + ret = atmel_ecc_sleep(client); + if (ret < 0) + goto err; + + mutex_unlock(&i2c_priv->lock); + return atmel_ecc_status(&client->dev, cmd->data); +err: + mutex_unlock(&i2c_priv->lock); + return ret; +} + +static void atmel_ecc_work_handler(struct work_struct *work) +{ + struct atmel_ecc_work_data *work_data = + container_of(work, struct atmel_ecc_work_data, work); + struct atmel_ecc_cmd *cmd = &work_data->cmd; + struct i2c_client *client = work_data->ctx->client; + int status; + + status = atmel_ecc_send_receive(client, cmd); + work_data->cbk(work_data, work_data->areq, status); +} + +static void atmel_ecc_enqueue(struct atmel_ecc_work_data *work_data, + void (*cbk)(struct atmel_ecc_work_data *work_data, + void *areq, int status), + void *areq) +{ + work_data->cbk = (void *)cbk; + work_data->areq = areq; + + INIT_WORK(&work_data->work, atmel_ecc_work_handler); + schedule_work(&work_data->work); +} + +static unsigned int atmel_ecdh_supported_curve(unsigned int curve_id) +{ + if (curve_id == ECC_CURVE_NIST_P256) + return ATMEL_ECC_NIST_P256_N_SIZE; + + return 0; +} + +/* + * A random private key is generated and stored in the device. The device + * returns the pair public key. + */ +static int atmel_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf, + unsigned int len) +{ + struct atmel_ecdh_ctx *ctx = kpp_tfm_ctx(tfm); + struct atmel_ecc_cmd *cmd; + void *public_key; + struct ecdh params; + int ret = -ENOMEM; + + /* free the old public key, if any */ + kfree(ctx->public_key); + /* make sure you don't free the old public key twice */ + ctx->public_key = NULL; + + if (crypto_ecdh_decode_key(buf, len, ¶ms) < 0) { + dev_err(&ctx->client->dev, "crypto_ecdh_decode_key failed\n"); + return -EINVAL; + } + + ctx->n_sz = atmel_ecdh_supported_curve(params.curve_id); + if (!ctx->n_sz || params.key_size) { + /* fallback to ecdh software implementation */ + ctx->do_fallback = true; + return crypto_kpp_set_secret(ctx->fallback, buf, len); + } + + cmd = kmalloc(sizeof(*cmd), GFP_KERNEL); + if (!cmd) + return -ENOMEM; + + /* + * The device only supports NIST P256 ECC keys. The public key size will + * always be the same. Use a macro for the key size to avoid unnecessary + * computations. + */ + public_key = kmalloc(ATMEL_ECC_PUBKEY_SIZE, GFP_KERNEL); + if (!public_key) + goto free_cmd; + + ctx->do_fallback = false; + ctx->curve_id = params.curve_id; + + atmel_ecc_init_genkey_cmd(cmd, DATA_SLOT_2); + + ret = atmel_ecc_send_receive(ctx->client, cmd); + if (ret) + goto free_public_key; + + /* save the public key */ + memcpy(public_key, &cmd->data[RSP_DATA_IDX], ATMEL_ECC_PUBKEY_SIZE); + ctx->public_key = public_key; + + kfree(cmd); + return 0; + +free_public_key: + kfree(public_key); +free_cmd: + kfree(cmd); + return ret; +} + +static int atmel_ecdh_generate_public_key(struct kpp_request *req) +{ + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); + struct atmel_ecdh_ctx *ctx = kpp_tfm_ctx(tfm); + size_t copied; + int ret = 0; + + if (ctx->do_fallback) { + kpp_request_set_tfm(req, ctx->fallback); + return crypto_kpp_generate_public_key(req); + } + + /* public key was saved at private key generation */ + copied = sg_copy_from_buffer(req->dst, 1, ctx->public_key, + ATMEL_ECC_PUBKEY_SIZE); + if (copied != ATMEL_ECC_PUBKEY_SIZE) + ret = -EINVAL; + + return ret; +} + +static int atmel_ecdh_compute_shared_secret(struct kpp_request *req) +{ + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); + struct atmel_ecdh_ctx *ctx = kpp_tfm_ctx(tfm); + struct atmel_ecc_work_data *work_data; + gfp_t gfp; + int ret; + + if (ctx->do_fallback) { + kpp_request_set_tfm(req, ctx->fallback); + return crypto_kpp_compute_shared_secret(req); + } + + gfp = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : + GFP_ATOMIC; + + work_data = kmalloc(sizeof(*work_data), gfp); + if (!work_data) + return -ENOMEM; + + work_data->ctx = ctx; + + ret = atmel_ecc_init_ecdh_cmd(&work_data->cmd, req->src); + if (ret) + goto free_work_data; + + atmel_ecc_enqueue(work_data, atmel_ecdh_done, req); + + return -EINPROGRESS; + +free_work_data: + kfree(work_data); + return ret; +} + +static struct i2c_client *atmel_ecc_i2c_client_alloc(void) +{ + struct atmel_ecc_i2c_client_priv *i2c_priv, *min_i2c_priv = NULL; + struct i2c_client *client = ERR_PTR(-ENODEV); + int min_tfm_cnt = INT_MAX; + int tfm_cnt; + + spin_lock(&driver_data.i2c_list_lock); + + if (list_empty(&driver_data.i2c_client_list)) { + spin_unlock(&driver_data.i2c_list_lock); + return ERR_PTR(-ENODEV); + } + + list_for_each_entry(i2c_priv, &driver_data.i2c_client_list, + i2c_client_list_node) { + tfm_cnt = atomic_read(&i2c_priv->tfm_count); + if (tfm_cnt < min_tfm_cnt) { + min_tfm_cnt = tfm_cnt; + min_i2c_priv = i2c_priv; + } + if (!min_tfm_cnt) + break; + } + + if (min_i2c_priv) { + atomic_inc(&min_i2c_priv->tfm_count); + client = min_i2c_priv->client; + } + + spin_unlock(&driver_data.i2c_list_lock); + + return client; +} + +static void atmel_ecc_i2c_client_free(struct i2c_client *client) +{ + struct atmel_ecc_i2c_client_priv *i2c_priv = i2c_get_clientdata(client); + + atomic_dec(&i2c_priv->tfm_count); +} + +static int atmel_ecdh_init_tfm(struct crypto_kpp *tfm) +{ + const char *alg = kpp_alg_name(tfm); + struct crypto_kpp *fallback; + struct atmel_ecdh_ctx *ctx = kpp_tfm_ctx(tfm); + + ctx->client = atmel_ecc_i2c_client_alloc(); + if (IS_ERR(ctx->client)) { + pr_err("tfm - i2c_client binding failed\n"); + return PTR_ERR(ctx->client); + } + + fallback = crypto_alloc_kpp(alg, 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(fallback)) { + dev_err(&ctx->client->dev, "Failed to allocate transformation for '%s': %ld\n", + alg, PTR_ERR(fallback)); + return PTR_ERR(fallback); + } + + crypto_kpp_set_flags(fallback, crypto_kpp_get_flags(tfm)); + + dev_info(&ctx->client->dev, "Using '%s' as fallback implementation.\n", + crypto_tfm_alg_driver_name(crypto_kpp_tfm(fallback))); + + ctx->fallback = fallback; + + return 0; +} + +static void atmel_ecdh_exit_tfm(struct crypto_kpp *tfm) +{ + struct atmel_ecdh_ctx *ctx = kpp_tfm_ctx(tfm); + + kfree(ctx->public_key); + crypto_free_kpp(ctx->fallback); + atmel_ecc_i2c_client_free(ctx->client); +} + +static unsigned int atmel_ecdh_max_size(struct crypto_kpp *tfm) +{ + struct atmel_ecdh_ctx *ctx = kpp_tfm_ctx(tfm); + + if (ctx->fallback) + return crypto_kpp_maxsize(ctx->fallback); + + /* + * The device only supports NIST P256 ECC keys. The public key size will + * always be the same. Use a macro for the key size to avoid unnecessary + * computations. + */ + return ATMEL_ECC_PUBKEY_SIZE; +} + +static struct kpp_alg atmel_ecdh = { + .set_secret = atmel_ecdh_set_secret, + .generate_public_key = atmel_ecdh_generate_public_key, + .compute_shared_secret = atmel_ecdh_compute_shared_secret, + .init = atmel_ecdh_init_tfm, + .exit = atmel_ecdh_exit_tfm, + .max_size = atmel_ecdh_max_size, + .base = { + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, + .cra_name = "ecdh", + .cra_driver_name = "atmel-ecdh", + .cra_priority = ATMEL_ECC_PRIORITY, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct atmel_ecdh_ctx), + }, +}; + +static inline size_t atmel_ecc_wake_token_sz(u32 bus_clk_rate) +{ + u32 no_of_bits = DIV_ROUND_UP(TWLO_USEC * bus_clk_rate, USEC_PER_SEC); + + /* return the size of the wake_token in bytes */ + return DIV_ROUND_UP(no_of_bits, 8); +} + +static int device_sanity_check(struct i2c_client *client) +{ + struct atmel_ecc_cmd *cmd; + int ret; + + cmd = kmalloc(sizeof(*cmd), GFP_KERNEL); + if (!cmd) + return -ENOMEM; + + atmel_ecc_init_read_cmd(cmd); + + ret = atmel_ecc_send_receive(client, cmd); + if (ret) + goto free_cmd; + + /* + * It is vital that the Configuration, Data and OTP zones be locked + * prior to release into the field of the system containing the device. + * Failure to lock these zones may permit modification of any secret + * keys and may lead to other security problems. + */ + if (cmd->data[LOCK_CONFIG_IDX] || cmd->data[LOCK_VALUE_IDX]) { + dev_err(&client->dev, "Configuration or Data and OTP zones are unlocked!\n"); + ret = -ENOTSUPP; + } + + /* fall through */ +free_cmd: + kfree(cmd); + return ret; +} + +static int atmel_ecc_probe(struct i2c_client *client, + const struct i2c_device_id *id) +{ + struct atmel_ecc_i2c_client_priv *i2c_priv; + struct device *dev = &client->dev; + int ret; + u32 bus_clk_rate; + + if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) { + dev_err(dev, "I2C_FUNC_I2C not supported\n"); + return -ENODEV; + } + + ret = of_property_read_u32(client->adapter->dev.of_node, + "clock-frequency", &bus_clk_rate); + if (ret) { + dev_err(dev, "of: failed to read clock-frequency property\n"); + return ret; + } + + if (bus_clk_rate > 1000000L) { + dev_err(dev, "%d exceeds maximum supported clock frequency (1MHz)\n", + bus_clk_rate); + return -EINVAL; + } + + i2c_priv = devm_kmalloc(dev, sizeof(*i2c_priv), GFP_KERNEL); + if (!i2c_priv) + return -ENOMEM; + + i2c_priv->client = client; + mutex_init(&i2c_priv->lock); + + /* + * WAKE_TOKEN_MAX_SIZE was calculated for the maximum bus_clk_rate - + * 1MHz. The previous bus_clk_rate check ensures us that wake_token_sz + * will always be smaller than or equal to WAKE_TOKEN_MAX_SIZE. + */ + i2c_priv->wake_token_sz = atmel_ecc_wake_token_sz(bus_clk_rate); + + memset(i2c_priv->wake_token, 0, sizeof(i2c_priv->wake_token)); + + atomic_set(&i2c_priv->tfm_count, 0); + + i2c_set_clientdata(client, i2c_priv); + + ret = device_sanity_check(client); + if (ret) + return ret; + + spin_lock(&driver_data.i2c_list_lock); + list_add_tail(&i2c_priv->i2c_client_list_node, + &driver_data.i2c_client_list); + spin_unlock(&driver_data.i2c_list_lock); + + ret = crypto_register_kpp(&atmel_ecdh); + if (ret) { + spin_lock(&driver_data.i2c_list_lock); + list_del(&i2c_priv->i2c_client_list_node); + spin_unlock(&driver_data.i2c_list_lock); + + dev_err(dev, "%s alg registration failed\n", + atmel_ecdh.base.cra_driver_name); + } else { + dev_info(dev, "atmel ecc algorithms registered in /proc/crypto\n"); + } + + return ret; +} + +static int atmel_ecc_remove(struct i2c_client *client) +{ + struct atmel_ecc_i2c_client_priv *i2c_priv = i2c_get_clientdata(client); + + /* Return EBUSY if i2c client already allocated. */ + if (atomic_read(&i2c_priv->tfm_count)) { + dev_err(&client->dev, "Device is busy\n"); + return -EBUSY; + } + + crypto_unregister_kpp(&atmel_ecdh); + + spin_lock(&driver_data.i2c_list_lock); + list_del(&i2c_priv->i2c_client_list_node); + spin_unlock(&driver_data.i2c_list_lock); + + return 0; +} + +#ifdef CONFIG_OF +static const struct of_device_id atmel_ecc_dt_ids[] = { + { + .compatible = "atmel,atecc508a", + }, { + /* sentinel */ + } +}; +MODULE_DEVICE_TABLE(of, atmel_ecc_dt_ids); +#endif + +static const struct i2c_device_id atmel_ecc_id[] = { + { "atecc508a", 0 }, + { } +}; +MODULE_DEVICE_TABLE(i2c, atmel_ecc_id); + +static struct i2c_driver atmel_ecc_driver = { + .driver = { + .name = "atmel-ecc", + .of_match_table = of_match_ptr(atmel_ecc_dt_ids), + }, + .probe = atmel_ecc_probe, + .remove = atmel_ecc_remove, + .id_table = atmel_ecc_id, +}; + +static int __init atmel_ecc_init(void) +{ + spin_lock_init(&driver_data.i2c_list_lock); + INIT_LIST_HEAD(&driver_data.i2c_client_list); + return i2c_add_driver(&atmel_ecc_driver); +} + +static void __exit atmel_ecc_exit(void) +{ + flush_scheduled_work(); + i2c_del_driver(&atmel_ecc_driver); +} + +module_init(atmel_ecc_init); +module_exit(atmel_ecc_exit); + +MODULE_AUTHOR("Tudor Ambarus <tudor.ambarus@microchip.com>"); +MODULE_DESCRIPTION("Microchip / Atmel ECC (I2C) driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/crypto/atmel-ecc.h b/drivers/crypto/atmel-ecc.h new file mode 100644 index 000000000000..25232c8abcc2 --- /dev/null +++ b/drivers/crypto/atmel-ecc.h @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2017, Microchip Technology Inc. + * Author: Tudor Ambarus <tudor.ambarus@microchip.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see <http://www.gnu.org/licenses/>. + * + */ + +#ifndef __ATMEL_ECC_H__ +#define __ATMEL_ECC_H__ + +#define ATMEL_ECC_PRIORITY 300 + +#define COMMAND 0x03 /* packet function */ +#define SLEEP_TOKEN 0x01 +#define WAKE_TOKEN_MAX_SIZE 8 + +/* Definitions of Data and Command sizes */ +#define WORD_ADDR_SIZE 1 +#define COUNT_SIZE 1 +#define CRC_SIZE 2 +#define CMD_OVERHEAD_SIZE (COUNT_SIZE + CRC_SIZE) + +/* size in bytes of the n prime */ +#define ATMEL_ECC_NIST_P256_N_SIZE 32 +#define ATMEL_ECC_PUBKEY_SIZE (2 * ATMEL_ECC_NIST_P256_N_SIZE) + +#define STATUS_RSP_SIZE 4 +#define ECDH_RSP_SIZE (32 + CMD_OVERHEAD_SIZE) +#define GENKEY_RSP_SIZE (ATMEL_ECC_PUBKEY_SIZE + \ + CMD_OVERHEAD_SIZE) +#define READ_RSP_SIZE (4 + CMD_OVERHEAD_SIZE) +#define MAX_RSP_SIZE GENKEY_RSP_SIZE + +/** + * atmel_ecc_cmd - structure used for communicating with the device. + * @word_addr: indicates the function of the packet sent to the device. This + * byte should have a value of COMMAND for normal operation. + * @count : number of bytes to be transferred to (or from) the device. + * @opcode : the command code. + * @param1 : the first parameter; always present. + * @param2 : the second parameter; always present. + * @data : optional remaining input data. Includes a 2-byte CRC. + * @rxsize : size of the data received from i2c client. + * @msecs : command execution time in milliseconds + */ +struct atmel_ecc_cmd { + u8 word_addr; + u8 count; + u8 opcode; + u8 param1; + u16 param2; + u8 data[MAX_RSP_SIZE]; + u8 msecs; + u16 rxsize; +} __packed; + +/* Status/Error codes */ +#define STATUS_SIZE 0x04 +#define STATUS_NOERR 0x00 +#define STATUS_WAKE_SUCCESSFUL 0x11 + +static const struct { + u8 value; + const char *error_text; +} error_list[] = { + { 0x01, "CheckMac or Verify miscompare" }, + { 0x03, "Parse Error" }, + { 0x05, "ECC Fault" }, + { 0x0F, "Execution Error" }, + { 0xEE, "Watchdog about to expire" }, + { 0xFF, "CRC or other communication error" }, +}; + +/* Definitions for eeprom organization */ +#define CONFIG_ZONE 0 + +/* Definitions for Indexes common to all commands */ +#define RSP_DATA_IDX 1 /* buffer index of data in response */ +#define DATA_SLOT_2 2 /* used for ECDH private key */ + +/* Definitions for the device lock state */ +#define DEVICE_LOCK_ADDR 0x15 +#define LOCK_VALUE_IDX (RSP_DATA_IDX + 2) +#define LOCK_CONFIG_IDX (RSP_DATA_IDX + 3) + +/* + * Wake High delay to data communication (microseconds). SDA should be stable + * high for this entire duration. + */ +#define TWHI_MIN 1500 +#define TWHI_MAX 1550 + +/* Wake Low duration */ +#define TWLO_USEC 60 + +/* Command execution time (milliseconds) */ +#define MAX_EXEC_TIME_ECDH 58 +#define MAX_EXEC_TIME_GENKEY 115 +#define MAX_EXEC_TIME_READ 1 + +/* Command opcode */ +#define OPCODE_ECDH 0x43 +#define OPCODE_GENKEY 0x40 +#define OPCODE_READ 0x02 + +/* Definitions for the READ Command */ +#define READ_COUNT 7 + +/* Definitions for the GenKey Command */ +#define GENKEY_COUNT 7 +#define GENKEY_MODE_PRIVATE 0x04 + +/* Definitions for the ECDH Command */ +#define ECDH_COUNT 71 +#define ECDH_PREFIX_MODE 0x00 + +#endif /* __ATMEL_ECC_H__ */ diff --git a/drivers/crypto/atmel-sha-regs.h b/drivers/crypto/atmel-sha-regs.h index 1b0eba4a2706..b2b5e634e80f 100644 --- a/drivers/crypto/atmel-sha-regs.h +++ b/drivers/crypto/atmel-sha-regs.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __ATMEL_SHA_REGS_H__ #define __ATMEL_SHA_REGS_H__ diff --git a/drivers/crypto/atmel-sha.c b/drivers/crypto/atmel-sha.c index dad4e5bad827..8874aa5ca0f7 100644 --- a/drivers/crypto/atmel-sha.c +++ b/drivers/crypto/atmel-sha.c @@ -2628,7 +2628,6 @@ static bool atmel_sha_filter(struct dma_chan *chan, void *slave) static int atmel_sha_dma_init(struct atmel_sha_dev *dd, struct crypto_platform_data *pdata) { - int err = -ENOMEM; dma_cap_mask_t mask_in; /* Try to grab DMA channel */ @@ -2639,7 +2638,7 @@ static int atmel_sha_dma_init(struct atmel_sha_dev *dd, atmel_sha_filter, &pdata->dma_slave->rxdata, dd->dev, "tx"); if (!dd->dma_lch_in.chan) { dev_warn(dd->dev, "no DMA channel available\n"); - return err; + return -ENODEV; } dd->dma_lch_in.dma_conf.direction = DMA_MEM_TO_DEV; @@ -2778,8 +2777,6 @@ static int atmel_sha_probe(struct platform_device *pdev) crypto_init_queue(&sha_dd->queue, ATMEL_SHA_QUEUE_LENGTH); - sha_dd->irq = -1; - /* Get the base address */ sha_res = platform_get_resource(pdev, IORESOURCE_MEM, 0); if (!sha_res) { @@ -2883,7 +2880,7 @@ sha_dd_err: static int atmel_sha_remove(struct platform_device *pdev) { - static struct atmel_sha_dev *sha_dd; + struct atmel_sha_dev *sha_dd; sha_dd = platform_get_drvdata(pdev); if (!sha_dd) diff --git a/drivers/crypto/atmel-tdes-regs.h b/drivers/crypto/atmel-tdes-regs.h index f86734d0fda4..fbd905720dfa 100644 --- a/drivers/crypto/atmel-tdes-regs.h +++ b/drivers/crypto/atmel-tdes-regs.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __ATMEL_TDES_REGS_H__ #define __ATMEL_TDES_REGS_H__ diff --git a/drivers/crypto/atmel-tdes.c b/drivers/crypto/atmel-tdes.c index b25f1b3c981f..592124f8382b 100644 --- a/drivers/crypto/atmel-tdes.c +++ b/drivers/crypto/atmel-tdes.c @@ -720,7 +720,6 @@ static bool atmel_tdes_filter(struct dma_chan *chan, void *slave) static int atmel_tdes_dma_init(struct atmel_tdes_dev *dd, struct crypto_platform_data *pdata) { - int err = -ENOMEM; dma_cap_mask_t mask; dma_cap_zero(mask); @@ -765,7 +764,7 @@ err_dma_out: dma_release_channel(dd->dma_lch_in.chan); err_dma_in: dev_warn(dd->dev, "no DMA channel available\n"); - return err; + return -ENODEV; } static void atmel_tdes_dma_cleanup(struct atmel_tdes_dev *dd) @@ -912,10 +911,6 @@ static int atmel_tdes_cra_init(struct crypto_tfm *tfm) return 0; } -static void atmel_tdes_cra_exit(struct crypto_tfm *tfm) -{ -} - static struct crypto_alg tdes_algs[] = { { .cra_name = "ecb(des)", @@ -928,7 +923,6 @@ static struct crypto_alg tdes_algs[] = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_tdes_cra_init, - .cra_exit = atmel_tdes_cra_exit, .cra_u.ablkcipher = { .min_keysize = DES_KEY_SIZE, .max_keysize = DES_KEY_SIZE, @@ -948,7 +942,6 @@ static struct crypto_alg tdes_algs[] = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_tdes_cra_init, - .cra_exit = atmel_tdes_cra_exit, .cra_u.ablkcipher = { .min_keysize = DES_KEY_SIZE, .max_keysize = DES_KEY_SIZE, @@ -969,7 +962,6 @@ static struct crypto_alg tdes_algs[] = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_tdes_cra_init, - .cra_exit = atmel_tdes_cra_exit, .cra_u.ablkcipher = { .min_keysize = DES_KEY_SIZE, .max_keysize = DES_KEY_SIZE, @@ -990,7 +982,6 @@ static struct crypto_alg tdes_algs[] = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_tdes_cra_init, - .cra_exit = atmel_tdes_cra_exit, .cra_u.ablkcipher = { .min_keysize = DES_KEY_SIZE, .max_keysize = DES_KEY_SIZE, @@ -1011,7 +1002,6 @@ static struct crypto_alg tdes_algs[] = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_tdes_cra_init, - .cra_exit = atmel_tdes_cra_exit, .cra_u.ablkcipher = { .min_keysize = DES_KEY_SIZE, .max_keysize = DES_KEY_SIZE, @@ -1032,7 +1022,6 @@ static struct crypto_alg tdes_algs[] = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_tdes_cra_init, - .cra_exit = atmel_tdes_cra_exit, .cra_u.ablkcipher = { .min_keysize = DES_KEY_SIZE, .max_keysize = DES_KEY_SIZE, @@ -1053,7 +1042,6 @@ static struct crypto_alg tdes_algs[] = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_tdes_cra_init, - .cra_exit = atmel_tdes_cra_exit, .cra_u.ablkcipher = { .min_keysize = DES_KEY_SIZE, .max_keysize = DES_KEY_SIZE, @@ -1074,7 +1062,6 @@ static struct crypto_alg tdes_algs[] = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_tdes_cra_init, - .cra_exit = atmel_tdes_cra_exit, .cra_u.ablkcipher = { .min_keysize = 2 * DES_KEY_SIZE, .max_keysize = 3 * DES_KEY_SIZE, @@ -1094,7 +1081,6 @@ static struct crypto_alg tdes_algs[] = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_tdes_cra_init, - .cra_exit = atmel_tdes_cra_exit, .cra_u.ablkcipher = { .min_keysize = 2*DES_KEY_SIZE, .max_keysize = 3*DES_KEY_SIZE, @@ -1115,7 +1101,6 @@ static struct crypto_alg tdes_algs[] = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_tdes_cra_init, - .cra_exit = atmel_tdes_cra_exit, .cra_u.ablkcipher = { .min_keysize = 2*DES_KEY_SIZE, .max_keysize = 2*DES_KEY_SIZE, @@ -1136,7 +1121,6 @@ static struct crypto_alg tdes_algs[] = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_tdes_cra_init, - .cra_exit = atmel_tdes_cra_exit, .cra_u.ablkcipher = { .min_keysize = 2*DES_KEY_SIZE, .max_keysize = 2*DES_KEY_SIZE, @@ -1157,7 +1141,6 @@ static struct crypto_alg tdes_algs[] = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_tdes_cra_init, - .cra_exit = atmel_tdes_cra_exit, .cra_u.ablkcipher = { .min_keysize = 2*DES_KEY_SIZE, .max_keysize = 2*DES_KEY_SIZE, @@ -1178,7 +1161,6 @@ static struct crypto_alg tdes_algs[] = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_tdes_cra_init, - .cra_exit = atmel_tdes_cra_exit, .cra_u.ablkcipher = { .min_keysize = 2*DES_KEY_SIZE, .max_keysize = 2*DES_KEY_SIZE, @@ -1199,7 +1181,6 @@ static struct crypto_alg tdes_algs[] = { .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, .cra_init = atmel_tdes_cra_init, - .cra_exit = atmel_tdes_cra_exit, .cra_u.ablkcipher = { .min_keysize = 2*DES_KEY_SIZE, .max_keysize = 3*DES_KEY_SIZE, @@ -1382,8 +1363,6 @@ static int atmel_tdes_probe(struct platform_device *pdev) crypto_init_queue(&tdes_dd->queue, ATMEL_TDES_QUEUE_LENGTH); - tdes_dd->irq = -1; - /* Get the base address */ tdes_res = platform_get_resource(pdev, IORESOURCE_MEM, 0); if (!tdes_res) { @@ -1487,7 +1466,7 @@ tdes_dd_err: static int atmel_tdes_remove(struct platform_device *pdev) { - static struct atmel_tdes_dev *tdes_dd; + struct atmel_tdes_dev *tdes_dd; tdes_dd = platform_get_drvdata(pdev); if (!tdes_dd) diff --git a/drivers/crypto/axis/Makefile b/drivers/crypto/axis/Makefile new file mode 100644 index 000000000000..be9a84a4b667 --- /dev/null +++ b/drivers/crypto/axis/Makefile @@ -0,0 +1 @@ +obj-$(CONFIG_CRYPTO_DEV_ARTPEC6) := artpec6_crypto.o diff --git a/drivers/crypto/axis/artpec6_crypto.c b/drivers/crypto/axis/artpec6_crypto.c new file mode 100644 index 000000000000..456278440863 --- /dev/null +++ b/drivers/crypto/axis/artpec6_crypto.c @@ -0,0 +1,3192 @@ +/* + * Driver for ARTPEC-6 crypto block using the kernel asynchronous crypto api. + * + * Copyright (C) 2014-2017 Axis Communications AB + */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/bitfield.h> +#include <linux/crypto.h> +#include <linux/debugfs.h> +#include <linux/delay.h> +#include <linux/dma-mapping.h> +#include <linux/fault-inject.h> +#include <linux/init.h> +#include <linux/interrupt.h> +#include <linux/kernel.h> +#include <linux/list.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/platform_device.h> +#include <linux/scatterlist.h> +#include <linux/slab.h> + +#include <crypto/aes.h> +#include <crypto/internal/aead.h> +#include <crypto/internal/hash.h> +#include <crypto/internal/skcipher.h> +#include <crypto/scatterwalk.h> +#include <crypto/sha.h> +#include <crypto/xts.h> + +/* Max length of a line in all cache levels for Artpec SoCs. */ +#define ARTPEC_CACHE_LINE_MAX 32 + +#define PDMA_OUT_CFG 0x0000 +#define PDMA_OUT_BUF_CFG 0x0004 +#define PDMA_OUT_CMD 0x0008 +#define PDMA_OUT_DESCRQ_PUSH 0x0010 +#define PDMA_OUT_DESCRQ_STAT 0x0014 + +#define A6_PDMA_IN_CFG 0x0028 +#define A6_PDMA_IN_BUF_CFG 0x002c +#define A6_PDMA_IN_CMD 0x0030 +#define A6_PDMA_IN_STATQ_PUSH 0x0038 +#define A6_PDMA_IN_DESCRQ_PUSH 0x0044 +#define A6_PDMA_IN_DESCRQ_STAT 0x0048 +#define A6_PDMA_INTR_MASK 0x0068 +#define A6_PDMA_ACK_INTR 0x006c +#define A6_PDMA_MASKED_INTR 0x0074 + +#define A7_PDMA_IN_CFG 0x002c +#define A7_PDMA_IN_BUF_CFG 0x0030 +#define A7_PDMA_IN_CMD 0x0034 +#define A7_PDMA_IN_STATQ_PUSH 0x003c +#define A7_PDMA_IN_DESCRQ_PUSH 0x0048 +#define A7_PDMA_IN_DESCRQ_STAT 0x004C +#define A7_PDMA_INTR_MASK 0x006c +#define A7_PDMA_ACK_INTR 0x0070 +#define A7_PDMA_MASKED_INTR 0x0078 + +#define PDMA_OUT_CFG_EN BIT(0) + +#define PDMA_OUT_BUF_CFG_DATA_BUF_SIZE GENMASK(4, 0) +#define PDMA_OUT_BUF_CFG_DESCR_BUF_SIZE GENMASK(9, 5) + +#define PDMA_OUT_CMD_START BIT(0) +#define A6_PDMA_OUT_CMD_STOP BIT(3) +#define A7_PDMA_OUT_CMD_STOP BIT(2) + +#define PDMA_OUT_DESCRQ_PUSH_LEN GENMASK(5, 0) +#define PDMA_OUT_DESCRQ_PUSH_ADDR GENMASK(31, 6) + +#define PDMA_OUT_DESCRQ_STAT_LEVEL GENMASK(3, 0) +#define PDMA_OUT_DESCRQ_STAT_SIZE GENMASK(7, 4) + +#define PDMA_IN_CFG_EN BIT(0) + +#define PDMA_IN_BUF_CFG_DATA_BUF_SIZE GENMASK(4, 0) +#define PDMA_IN_BUF_CFG_DESCR_BUF_SIZE GENMASK(9, 5) +#define PDMA_IN_BUF_CFG_STAT_BUF_SIZE GENMASK(14, 10) + +#define PDMA_IN_CMD_START BIT(0) +#define A6_PDMA_IN_CMD_FLUSH_STAT BIT(2) +#define A6_PDMA_IN_CMD_STOP BIT(3) +#define A7_PDMA_IN_CMD_FLUSH_STAT BIT(1) +#define A7_PDMA_IN_CMD_STOP BIT(2) + +#define PDMA_IN_STATQ_PUSH_LEN GENMASK(5, 0) +#define PDMA_IN_STATQ_PUSH_ADDR GENMASK(31, 6) + +#define PDMA_IN_DESCRQ_PUSH_LEN GENMASK(5, 0) +#define PDMA_IN_DESCRQ_PUSH_ADDR GENMASK(31, 6) + +#define PDMA_IN_DESCRQ_STAT_LEVEL GENMASK(3, 0) +#define PDMA_IN_DESCRQ_STAT_SIZE GENMASK(7, 4) + +#define A6_PDMA_INTR_MASK_IN_DATA BIT(2) +#define A6_PDMA_INTR_MASK_IN_EOP BIT(3) +#define A6_PDMA_INTR_MASK_IN_EOP_FLUSH BIT(4) + +#define A7_PDMA_INTR_MASK_IN_DATA BIT(3) +#define A7_PDMA_INTR_MASK_IN_EOP BIT(4) +#define A7_PDMA_INTR_MASK_IN_EOP_FLUSH BIT(5) + +#define A6_CRY_MD_OPER GENMASK(19, 16) + +#define A6_CRY_MD_HASH_SEL_CTX GENMASK(21, 20) +#define A6_CRY_MD_HASH_HMAC_FIN BIT(23) + +#define A6_CRY_MD_CIPHER_LEN GENMASK(21, 20) +#define A6_CRY_MD_CIPHER_DECR BIT(22) +#define A6_CRY_MD_CIPHER_TWEAK BIT(23) +#define A6_CRY_MD_CIPHER_DSEQ BIT(24) + +#define A7_CRY_MD_OPER GENMASK(11, 8) + +#define A7_CRY_MD_HASH_SEL_CTX GENMASK(13, 12) +#define A7_CRY_MD_HASH_HMAC_FIN BIT(15) + +#define A7_CRY_MD_CIPHER_LEN GENMASK(13, 12) +#define A7_CRY_MD_CIPHER_DECR BIT(14) +#define A7_CRY_MD_CIPHER_TWEAK BIT(15) +#define A7_CRY_MD_CIPHER_DSEQ BIT(16) + +/* DMA metadata constants */ +#define regk_crypto_aes_cbc 0x00000002 +#define regk_crypto_aes_ctr 0x00000003 +#define regk_crypto_aes_ecb 0x00000001 +#define regk_crypto_aes_gcm 0x00000004 +#define regk_crypto_aes_xts 0x00000005 +#define regk_crypto_cache 0x00000002 +#define a6_regk_crypto_dlkey 0x0000000a +#define a7_regk_crypto_dlkey 0x0000000e +#define regk_crypto_ext 0x00000001 +#define regk_crypto_hmac_sha1 0x00000007 +#define regk_crypto_hmac_sha256 0x00000009 +#define regk_crypto_hmac_sha384 0x0000000b +#define regk_crypto_hmac_sha512 0x0000000d +#define regk_crypto_init 0x00000000 +#define regk_crypto_key_128 0x00000000 +#define regk_crypto_key_192 0x00000001 +#define regk_crypto_key_256 0x00000002 +#define regk_crypto_null 0x00000000 +#define regk_crypto_sha1 0x00000006 +#define regk_crypto_sha256 0x00000008 +#define regk_crypto_sha384 0x0000000a +#define regk_crypto_sha512 0x0000000c + +/* DMA descriptor structures */ +struct pdma_descr_ctrl { + unsigned char short_descr : 1; + unsigned char pad1 : 1; + unsigned char eop : 1; + unsigned char intr : 1; + unsigned char short_len : 3; + unsigned char pad2 : 1; +} __packed; + +struct pdma_data_descr { + unsigned int len : 24; + unsigned int buf : 32; +} __packed; + +struct pdma_short_descr { + unsigned char data[7]; +} __packed; + +struct pdma_descr { + struct pdma_descr_ctrl ctrl; + union { + struct pdma_data_descr data; + struct pdma_short_descr shrt; + }; +}; + +struct pdma_stat_descr { + unsigned char pad1 : 1; + unsigned char pad2 : 1; + unsigned char eop : 1; + unsigned char pad3 : 5; + unsigned int len : 24; +}; + +/* Each descriptor array can hold max 64 entries */ +#define PDMA_DESCR_COUNT 64 + +#define MODULE_NAME "Artpec-6 CA" + +/* Hash modes (including HMAC variants) */ +#define ARTPEC6_CRYPTO_HASH_SHA1 1 +#define ARTPEC6_CRYPTO_HASH_SHA256 2 +#define ARTPEC6_CRYPTO_HASH_SHA384 3 +#define ARTPEC6_CRYPTO_HASH_SHA512 4 + +/* Crypto modes */ +#define ARTPEC6_CRYPTO_CIPHER_AES_ECB 1 +#define ARTPEC6_CRYPTO_CIPHER_AES_CBC 2 +#define ARTPEC6_CRYPTO_CIPHER_AES_CTR 3 +#define ARTPEC6_CRYPTO_CIPHER_AES_XTS 5 + +/* The PDMA is a DMA-engine tightly coupled with a ciphering engine. + * It operates on a descriptor array with up to 64 descriptor entries. + * The arrays must be 64 byte aligned in memory. + * + * The ciphering unit has no registers and is completely controlled by + * a 4-byte metadata that is inserted at the beginning of each dma packet. + * + * A dma packet is a sequence of descriptors terminated by setting the .eop + * field in the final descriptor of the packet. + * + * Multiple packets are used for providing context data, key data and + * the plain/ciphertext. + * + * PDMA Descriptors (Array) + * +------+------+------+~~+-------+------+---- + * | 0 | 1 | 2 |~~| 11 EOP| 12 | .... + * +--+---+--+---+----+-+~~+-------+----+-+---- + * | | | | | + * | | | | | + * __|__ +-------++-------++-------+ +----+ + * | MD | |Payload||Payload||Payload| | MD | + * +-----+ +-------++-------++-------+ +----+ + */ + +struct artpec6_crypto_bounce_buffer { + struct list_head list; + size_t length; + struct scatterlist *sg; + size_t offset; + /* buf is aligned to ARTPEC_CACHE_LINE_MAX and + * holds up to ARTPEC_CACHE_LINE_MAX bytes data. + */ + void *buf; +}; + +struct artpec6_crypto_dma_map { + dma_addr_t dma_addr; + size_t size; + enum dma_data_direction dir; +}; + +struct artpec6_crypto_dma_descriptors { + struct pdma_descr out[PDMA_DESCR_COUNT] __aligned(64); + struct pdma_descr in[PDMA_DESCR_COUNT] __aligned(64); + u32 stat[PDMA_DESCR_COUNT] __aligned(64); + struct list_head bounce_buffers; + /* Enough maps for all out/in buffers, and all three descr. arrays */ + struct artpec6_crypto_dma_map maps[PDMA_DESCR_COUNT * 2 + 2]; + dma_addr_t out_dma_addr; + dma_addr_t in_dma_addr; + dma_addr_t stat_dma_addr; + size_t out_cnt; + size_t in_cnt; + size_t map_count; +}; + +enum artpec6_crypto_variant { + ARTPEC6_CRYPTO, + ARTPEC7_CRYPTO, +}; + +struct artpec6_crypto { + void __iomem *base; + spinlock_t queue_lock; + struct list_head queue; /* waiting for pdma fifo space */ + struct list_head pending; /* submitted to pdma fifo */ + struct tasklet_struct task; + struct kmem_cache *dma_cache; + int pending_count; + struct timer_list timer; + enum artpec6_crypto_variant variant; + void *pad_buffer; /* cache-aligned block padding buffer */ + void *zero_buffer; +}; + +enum artpec6_crypto_hash_flags { + HASH_FLAG_INIT_CTX = 2, + HASH_FLAG_UPDATE = 4, + HASH_FLAG_FINALIZE = 8, + HASH_FLAG_HMAC = 16, + HASH_FLAG_UPDATE_KEY = 32, +}; + +struct artpec6_crypto_req_common { + struct list_head list; + struct artpec6_crypto_dma_descriptors *dma; + struct crypto_async_request *req; + void (*complete)(struct crypto_async_request *req); + gfp_t gfp_flags; +}; + +struct artpec6_hash_request_context { + char partial_buffer[SHA512_BLOCK_SIZE]; + char partial_buffer_out[SHA512_BLOCK_SIZE]; + char key_buffer[SHA512_BLOCK_SIZE]; + char pad_buffer[SHA512_BLOCK_SIZE + 32]; + unsigned char digeststate[SHA512_DIGEST_SIZE]; + size_t partial_bytes; + u64 digcnt; + u32 key_md; + u32 hash_md; + enum artpec6_crypto_hash_flags hash_flags; + struct artpec6_crypto_req_common common; +}; + +struct artpec6_hash_export_state { + char partial_buffer[SHA512_BLOCK_SIZE]; + unsigned char digeststate[SHA512_DIGEST_SIZE]; + size_t partial_bytes; + u64 digcnt; + int oper; + unsigned int hash_flags; +}; + +struct artpec6_hashalg_context { + char hmac_key[SHA512_BLOCK_SIZE]; + size_t hmac_key_length; + struct crypto_shash *child_hash; +}; + +struct artpec6_crypto_request_context { + u32 cipher_md; + bool decrypt; + struct artpec6_crypto_req_common common; +}; + +struct artpec6_cryptotfm_context { + unsigned char aes_key[2*AES_MAX_KEY_SIZE]; + size_t key_length; + u32 key_md; + int crypto_type; + struct crypto_skcipher *fallback; +}; + +struct artpec6_crypto_aead_hw_ctx { + __be64 aad_length_bits; + __be64 text_length_bits; + __u8 J0[AES_BLOCK_SIZE]; +}; + +struct artpec6_crypto_aead_req_ctx { + struct artpec6_crypto_aead_hw_ctx hw_ctx; + u32 cipher_md; + bool decrypt; + struct artpec6_crypto_req_common common; + __u8 decryption_tag[AES_BLOCK_SIZE] ____cacheline_aligned; +}; + +/* The crypto framework makes it hard to avoid this global. */ +static struct device *artpec6_crypto_dev; + +#ifdef CONFIG_FAULT_INJECTION +static DECLARE_FAULT_ATTR(artpec6_crypto_fail_status_read); +static DECLARE_FAULT_ATTR(artpec6_crypto_fail_dma_array_full); +#endif + +enum { + ARTPEC6_CRYPTO_PREPARE_HASH_NO_START, + ARTPEC6_CRYPTO_PREPARE_HASH_START, +}; + +static int artpec6_crypto_prepare_aead(struct aead_request *areq); +static int artpec6_crypto_prepare_crypto(struct skcipher_request *areq); +static int artpec6_crypto_prepare_hash(struct ahash_request *areq); + +static void +artpec6_crypto_complete_crypto(struct crypto_async_request *req); +static void +artpec6_crypto_complete_cbc_encrypt(struct crypto_async_request *req); +static void +artpec6_crypto_complete_cbc_decrypt(struct crypto_async_request *req); +static void +artpec6_crypto_complete_aead(struct crypto_async_request *req); +static void +artpec6_crypto_complete_hash(struct crypto_async_request *req); + +static int +artpec6_crypto_common_destroy(struct artpec6_crypto_req_common *common); + +static void +artpec6_crypto_start_dma(struct artpec6_crypto_req_common *common); + +struct artpec6_crypto_walk { + struct scatterlist *sg; + size_t offset; +}; + +static void artpec6_crypto_walk_init(struct artpec6_crypto_walk *awalk, + struct scatterlist *sg) +{ + awalk->sg = sg; + awalk->offset = 0; +} + +static size_t artpec6_crypto_walk_advance(struct artpec6_crypto_walk *awalk, + size_t nbytes) +{ + while (nbytes && awalk->sg) { + size_t piece; + + WARN_ON(awalk->offset > awalk->sg->length); + + piece = min(nbytes, (size_t)awalk->sg->length - awalk->offset); + nbytes -= piece; + awalk->offset += piece; + if (awalk->offset == awalk->sg->length) { + awalk->sg = sg_next(awalk->sg); + awalk->offset = 0; + } + + } + + return nbytes; +} + +static size_t +artpec6_crypto_walk_chunklen(const struct artpec6_crypto_walk *awalk) +{ + WARN_ON(awalk->sg->length == awalk->offset); + + return awalk->sg->length - awalk->offset; +} + +static dma_addr_t +artpec6_crypto_walk_chunk_phys(const struct artpec6_crypto_walk *awalk) +{ + return sg_phys(awalk->sg) + awalk->offset; +} + +static void +artpec6_crypto_copy_bounce_buffers(struct artpec6_crypto_req_common *common) +{ + struct artpec6_crypto_dma_descriptors *dma = common->dma; + struct artpec6_crypto_bounce_buffer *b; + struct artpec6_crypto_bounce_buffer *next; + + list_for_each_entry_safe(b, next, &dma->bounce_buffers, list) { + pr_debug("bounce entry %p: %zu bytes @ %zu from %p\n", + b, b->length, b->offset, b->buf); + sg_pcopy_from_buffer(b->sg, + 1, + b->buf, + b->length, + b->offset); + + list_del(&b->list); + kfree(b); + } +} + +static inline bool artpec6_crypto_busy(void) +{ + struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); + int fifo_count = ac->pending_count; + + return fifo_count > 6; +} + +static int artpec6_crypto_submit(struct artpec6_crypto_req_common *req) +{ + struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); + int ret = -EBUSY; + + spin_lock_bh(&ac->queue_lock); + + if (!artpec6_crypto_busy()) { + list_add_tail(&req->list, &ac->pending); + artpec6_crypto_start_dma(req); + ret = -EINPROGRESS; + } else if (req->req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG) { + list_add_tail(&req->list, &ac->queue); + } else { + artpec6_crypto_common_destroy(req); + } + + spin_unlock_bh(&ac->queue_lock); + + return ret; +} + +static void artpec6_crypto_start_dma(struct artpec6_crypto_req_common *common) +{ + struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); + enum artpec6_crypto_variant variant = ac->variant; + void __iomem *base = ac->base; + struct artpec6_crypto_dma_descriptors *dma = common->dma; + u32 ind, statd, outd; + + /* Make descriptor content visible to the DMA before starting it. */ + wmb(); + + ind = FIELD_PREP(PDMA_IN_DESCRQ_PUSH_LEN, dma->in_cnt - 1) | + FIELD_PREP(PDMA_IN_DESCRQ_PUSH_ADDR, dma->in_dma_addr >> 6); + + statd = FIELD_PREP(PDMA_IN_STATQ_PUSH_LEN, dma->in_cnt - 1) | + FIELD_PREP(PDMA_IN_STATQ_PUSH_ADDR, dma->stat_dma_addr >> 6); + + outd = FIELD_PREP(PDMA_OUT_DESCRQ_PUSH_LEN, dma->out_cnt - 1) | + FIELD_PREP(PDMA_OUT_DESCRQ_PUSH_ADDR, dma->out_dma_addr >> 6); + + if (variant == ARTPEC6_CRYPTO) { + writel_relaxed(ind, base + A6_PDMA_IN_DESCRQ_PUSH); + writel_relaxed(statd, base + A6_PDMA_IN_STATQ_PUSH); + writel_relaxed(PDMA_IN_CMD_START, base + A6_PDMA_IN_CMD); + } else { + writel_relaxed(ind, base + A7_PDMA_IN_DESCRQ_PUSH); + writel_relaxed(statd, base + A7_PDMA_IN_STATQ_PUSH); + writel_relaxed(PDMA_IN_CMD_START, base + A7_PDMA_IN_CMD); + } + + writel_relaxed(outd, base + PDMA_OUT_DESCRQ_PUSH); + writel_relaxed(PDMA_OUT_CMD_START, base + PDMA_OUT_CMD); + + ac->pending_count++; +} + +static void +artpec6_crypto_init_dma_operation(struct artpec6_crypto_req_common *common) +{ + struct artpec6_crypto_dma_descriptors *dma = common->dma; + + dma->out_cnt = 0; + dma->in_cnt = 0; + dma->map_count = 0; + INIT_LIST_HEAD(&dma->bounce_buffers); +} + +static bool fault_inject_dma_descr(void) +{ +#ifdef CONFIG_FAULT_INJECTION + return should_fail(&artpec6_crypto_fail_dma_array_full, 1); +#else + return false; +#endif +} + +/** artpec6_crypto_setup_out_descr_phys - Setup an out channel with a + * physical address + * + * @addr: The physical address of the data buffer + * @len: The length of the data buffer + * @eop: True if this is the last buffer in the packet + * + * @return 0 on success or -ENOSPC if there are no more descriptors available + */ +static int +artpec6_crypto_setup_out_descr_phys(struct artpec6_crypto_req_common *common, + dma_addr_t addr, size_t len, bool eop) +{ + struct artpec6_crypto_dma_descriptors *dma = common->dma; + struct pdma_descr *d; + + if (dma->out_cnt >= PDMA_DESCR_COUNT || + fault_inject_dma_descr()) { + pr_err("No free OUT DMA descriptors available!\n"); + return -ENOSPC; + } + + d = &dma->out[dma->out_cnt++]; + memset(d, 0, sizeof(*d)); + + d->ctrl.short_descr = 0; + d->ctrl.eop = eop; + d->data.len = len; + d->data.buf = addr; + return 0; +} + +/** artpec6_crypto_setup_out_descr_short - Setup a short out descriptor + * + * @dst: The virtual address of the data + * @len: The length of the data, must be between 1 to 7 bytes + * @eop: True if this is the last buffer in the packet + * + * @return 0 on success + * -ENOSPC if no more descriptors are available + * -EINVAL if the data length exceeds 7 bytes + */ +static int +artpec6_crypto_setup_out_descr_short(struct artpec6_crypto_req_common *common, + void *dst, unsigned int len, bool eop) +{ + struct artpec6_crypto_dma_descriptors *dma = common->dma; + struct pdma_descr *d; + + if (dma->out_cnt >= PDMA_DESCR_COUNT || + fault_inject_dma_descr()) { + pr_err("No free OUT DMA descriptors available!\n"); + return -ENOSPC; + } else if (len > 7 || len < 1) { + return -EINVAL; + } + d = &dma->out[dma->out_cnt++]; + memset(d, 0, sizeof(*d)); + + d->ctrl.short_descr = 1; + d->ctrl.short_len = len; + d->ctrl.eop = eop; + memcpy(d->shrt.data, dst, len); + return 0; +} + +static int artpec6_crypto_dma_map_page(struct artpec6_crypto_req_common *common, + struct page *page, size_t offset, + size_t size, + enum dma_data_direction dir, + dma_addr_t *dma_addr_out) +{ + struct artpec6_crypto_dma_descriptors *dma = common->dma; + struct device *dev = artpec6_crypto_dev; + struct artpec6_crypto_dma_map *map; + dma_addr_t dma_addr; + + *dma_addr_out = 0; + + if (dma->map_count >= ARRAY_SIZE(dma->maps)) + return -ENOMEM; + + dma_addr = dma_map_page(dev, page, offset, size, dir); + if (dma_mapping_error(dev, dma_addr)) + return -ENOMEM; + + map = &dma->maps[dma->map_count++]; + map->size = size; + map->dma_addr = dma_addr; + map->dir = dir; + + *dma_addr_out = dma_addr; + + return 0; +} + +static int +artpec6_crypto_dma_map_single(struct artpec6_crypto_req_common *common, + void *ptr, size_t size, + enum dma_data_direction dir, + dma_addr_t *dma_addr_out) +{ + struct page *page = virt_to_page(ptr); + size_t offset = (uintptr_t)ptr & ~PAGE_MASK; + + return artpec6_crypto_dma_map_page(common, page, offset, size, dir, + dma_addr_out); +} + +static int +artpec6_crypto_dma_map_descs(struct artpec6_crypto_req_common *common) +{ + struct artpec6_crypto_dma_descriptors *dma = common->dma; + int ret; + + ret = artpec6_crypto_dma_map_single(common, dma->in, + sizeof(dma->in[0]) * dma->in_cnt, + DMA_TO_DEVICE, &dma->in_dma_addr); + if (ret) + return ret; + + ret = artpec6_crypto_dma_map_single(common, dma->out, + sizeof(dma->out[0]) * dma->out_cnt, + DMA_TO_DEVICE, &dma->out_dma_addr); + if (ret) + return ret; + + /* We only read one stat descriptor */ + dma->stat[dma->in_cnt - 1] = 0; + + /* + * DMA_BIDIRECTIONAL since we need our zeroing of the stat descriptor + * to be written. + */ + return artpec6_crypto_dma_map_single(common, + dma->stat + dma->in_cnt - 1, + sizeof(dma->stat[0]), + DMA_BIDIRECTIONAL, + &dma->stat_dma_addr); +} + +static void +artpec6_crypto_dma_unmap_all(struct artpec6_crypto_req_common *common) +{ + struct artpec6_crypto_dma_descriptors *dma = common->dma; + struct device *dev = artpec6_crypto_dev; + int i; + + for (i = 0; i < dma->map_count; i++) { + struct artpec6_crypto_dma_map *map = &dma->maps[i]; + + dma_unmap_page(dev, map->dma_addr, map->size, map->dir); + } + + dma->map_count = 0; +} + +/** artpec6_crypto_setup_out_descr - Setup an out descriptor + * + * @dst: The virtual address of the data + * @len: The length of the data + * @eop: True if this is the last buffer in the packet + * @use_short: If this is true and the data length is 7 bytes or less then + * a short descriptor will be used + * + * @return 0 on success + * Any errors from artpec6_crypto_setup_out_descr_short() or + * setup_out_descr_phys() + */ +static int +artpec6_crypto_setup_out_descr(struct artpec6_crypto_req_common *common, + void *dst, unsigned int len, bool eop, + bool use_short) +{ + if (use_short && len < 7) { + return artpec6_crypto_setup_out_descr_short(common, dst, len, + eop); + } else { + int ret; + dma_addr_t dma_addr; + + ret = artpec6_crypto_dma_map_single(common, dst, len, + DMA_TO_DEVICE, + &dma_addr); + if (ret) + return ret; + + return artpec6_crypto_setup_out_descr_phys(common, dma_addr, + len, eop); + } +} + +/** artpec6_crypto_setup_in_descr_phys - Setup an in channel with a + * physical address + * + * @addr: The physical address of the data buffer + * @len: The length of the data buffer + * @intr: True if an interrupt should be fired after HW processing of this + * descriptor + * + */ +static int +artpec6_crypto_setup_in_descr_phys(struct artpec6_crypto_req_common *common, + dma_addr_t addr, unsigned int len, bool intr) +{ + struct artpec6_crypto_dma_descriptors *dma = common->dma; + struct pdma_descr *d; + + if (dma->in_cnt >= PDMA_DESCR_COUNT || + fault_inject_dma_descr()) { + pr_err("No free IN DMA descriptors available!\n"); + return -ENOSPC; + } + d = &dma->in[dma->in_cnt++]; + memset(d, 0, sizeof(*d)); + + d->ctrl.intr = intr; + d->data.len = len; + d->data.buf = addr; + return 0; +} + +/** artpec6_crypto_setup_in_descr - Setup an in channel descriptor + * + * @buffer: The virtual address to of the data buffer + * @len: The length of the data buffer + * @last: If this is the last data buffer in the request (i.e. an interrupt + * is needed + * + * Short descriptors are not used for the in channel + */ +static int +artpec6_crypto_setup_in_descr(struct artpec6_crypto_req_common *common, + void *buffer, unsigned int len, bool last) +{ + dma_addr_t dma_addr; + int ret; + + ret = artpec6_crypto_dma_map_single(common, buffer, len, + DMA_FROM_DEVICE, &dma_addr); + if (ret) + return ret; + + return artpec6_crypto_setup_in_descr_phys(common, dma_addr, len, last); +} + +static struct artpec6_crypto_bounce_buffer * +artpec6_crypto_alloc_bounce(gfp_t flags) +{ + void *base; + size_t alloc_size = sizeof(struct artpec6_crypto_bounce_buffer) + + 2 * ARTPEC_CACHE_LINE_MAX; + struct artpec6_crypto_bounce_buffer *bbuf = kzalloc(alloc_size, flags); + + if (!bbuf) + return NULL; + + base = bbuf + 1; + bbuf->buf = PTR_ALIGN(base, ARTPEC_CACHE_LINE_MAX); + return bbuf; +} + +static int setup_bounce_buffer_in(struct artpec6_crypto_req_common *common, + struct artpec6_crypto_walk *walk, size_t size) +{ + struct artpec6_crypto_bounce_buffer *bbuf; + int ret; + + bbuf = artpec6_crypto_alloc_bounce(common->gfp_flags); + if (!bbuf) + return -ENOMEM; + + bbuf->length = size; + bbuf->sg = walk->sg; + bbuf->offset = walk->offset; + + ret = artpec6_crypto_setup_in_descr(common, bbuf->buf, size, false); + if (ret) { + kfree(bbuf); + return ret; + } + + pr_debug("BOUNCE %zu offset %zu\n", size, walk->offset); + list_add_tail(&bbuf->list, &common->dma->bounce_buffers); + return 0; +} + +static int +artpec6_crypto_setup_sg_descrs_in(struct artpec6_crypto_req_common *common, + struct artpec6_crypto_walk *walk, + size_t count) +{ + size_t chunk; + int ret; + dma_addr_t addr; + + while (walk->sg && count) { + chunk = min(count, artpec6_crypto_walk_chunklen(walk)); + addr = artpec6_crypto_walk_chunk_phys(walk); + + /* When destination buffers are not aligned to the cache line + * size we need bounce buffers. The DMA-API requires that the + * entire line is owned by the DMA buffer and this holds also + * for the case when coherent DMA is used. + */ + if (!IS_ALIGNED(addr, ARTPEC_CACHE_LINE_MAX)) { + chunk = min_t(dma_addr_t, chunk, + ALIGN(addr, ARTPEC_CACHE_LINE_MAX) - + addr); + + pr_debug("CHUNK-b %pad:%zu\n", &addr, chunk); + ret = setup_bounce_buffer_in(common, walk, chunk); + } else if (chunk < ARTPEC_CACHE_LINE_MAX) { + pr_debug("CHUNK-b %pad:%zu\n", &addr, chunk); + ret = setup_bounce_buffer_in(common, walk, chunk); + } else { + dma_addr_t dma_addr; + + chunk = chunk & ~(ARTPEC_CACHE_LINE_MAX-1); + + pr_debug("CHUNK %pad:%zu\n", &addr, chunk); + + ret = artpec6_crypto_dma_map_page(common, + sg_page(walk->sg), + walk->sg->offset + + walk->offset, + chunk, + DMA_FROM_DEVICE, + &dma_addr); + if (ret) + return ret; + + ret = artpec6_crypto_setup_in_descr_phys(common, + dma_addr, + chunk, false); + } + + if (ret) + return ret; + + count = count - chunk; + artpec6_crypto_walk_advance(walk, chunk); + } + + if (count) + pr_err("EOL unexpected %zu bytes left\n", count); + + return count ? -EINVAL : 0; +} + +static int +artpec6_crypto_setup_sg_descrs_out(struct artpec6_crypto_req_common *common, + struct artpec6_crypto_walk *walk, + size_t count) +{ + size_t chunk; + int ret; + dma_addr_t addr; + + while (walk->sg && count) { + chunk = min(count, artpec6_crypto_walk_chunklen(walk)); + addr = artpec6_crypto_walk_chunk_phys(walk); + + pr_debug("OUT-CHUNK %pad:%zu\n", &addr, chunk); + + if (addr & 3) { + char buf[3]; + + chunk = min_t(size_t, chunk, (4-(addr&3))); + + sg_pcopy_to_buffer(walk->sg, 1, buf, chunk, + walk->offset); + + ret = artpec6_crypto_setup_out_descr_short(common, buf, + chunk, + false); + } else { + dma_addr_t dma_addr; + + ret = artpec6_crypto_dma_map_page(common, + sg_page(walk->sg), + walk->sg->offset + + walk->offset, + chunk, + DMA_TO_DEVICE, + &dma_addr); + if (ret) + return ret; + + ret = artpec6_crypto_setup_out_descr_phys(common, + dma_addr, + chunk, false); + } + + if (ret) + return ret; + + count = count - chunk; + artpec6_crypto_walk_advance(walk, chunk); + } + + if (count) + pr_err("EOL unexpected %zu bytes left\n", count); + + return count ? -EINVAL : 0; +} + + +/** artpec6_crypto_terminate_out_descrs - Set the EOP on the last out descriptor + * + * If the out descriptor list is non-empty, then the eop flag on the + * last used out descriptor will be set. + * + * @return 0 on success + * -EINVAL if the out descriptor is empty or has overflown + */ +static int +artpec6_crypto_terminate_out_descrs(struct artpec6_crypto_req_common *common) +{ + struct artpec6_crypto_dma_descriptors *dma = common->dma; + struct pdma_descr *d; + + if (!dma->out_cnt || dma->out_cnt > PDMA_DESCR_COUNT) { + pr_err("%s: OUT descriptor list is %s\n", + MODULE_NAME, dma->out_cnt ? "empty" : "full"); + return -EINVAL; + + } + + d = &dma->out[dma->out_cnt-1]; + d->ctrl.eop = 1; + + return 0; +} + +/** artpec6_crypto_terminate_in_descrs - Set the interrupt flag on the last + * in descriptor + * + * See artpec6_crypto_terminate_out_descrs() for return values + */ +static int +artpec6_crypto_terminate_in_descrs(struct artpec6_crypto_req_common *common) +{ + struct artpec6_crypto_dma_descriptors *dma = common->dma; + struct pdma_descr *d; + + if (!dma->in_cnt || dma->in_cnt > PDMA_DESCR_COUNT) { + pr_err("%s: IN descriptor list is %s\n", + MODULE_NAME, dma->in_cnt ? "empty" : "full"); + return -EINVAL; + } + + d = &dma->in[dma->in_cnt-1]; + d->ctrl.intr = 1; + return 0; +} + +/** create_hash_pad - Create a Secure Hash conformant pad + * + * @dst: The destination buffer to write the pad. Must be at least 64 bytes + * @dgstlen: The total length of the hash digest in bytes + * @bitcount: The total length of the digest in bits + * + * @return The total number of padding bytes written to @dst + */ +static size_t +create_hash_pad(int oper, unsigned char *dst, u64 dgstlen, u64 bitcount) +{ + unsigned int mod, target, diff, pad_bytes, size_bytes; + __be64 bits = __cpu_to_be64(bitcount); + + switch (oper) { + case regk_crypto_sha1: + case regk_crypto_sha256: + case regk_crypto_hmac_sha1: + case regk_crypto_hmac_sha256: + target = 448 / 8; + mod = 512 / 8; + size_bytes = 8; + break; + default: + target = 896 / 8; + mod = 1024 / 8; + size_bytes = 16; + break; + } + + target -= 1; + diff = dgstlen & (mod - 1); + pad_bytes = diff > target ? target + mod - diff : target - diff; + + memset(dst + 1, 0, pad_bytes); + dst[0] = 0x80; + + if (size_bytes == 16) { + memset(dst + 1 + pad_bytes, 0, 8); + memcpy(dst + 1 + pad_bytes + 8, &bits, 8); + } else { + memcpy(dst + 1 + pad_bytes, &bits, 8); + } + + return pad_bytes + size_bytes + 1; +} + +static int artpec6_crypto_common_init(struct artpec6_crypto_req_common *common, + struct crypto_async_request *parent, + void (*complete)(struct crypto_async_request *req), + struct scatterlist *dstsg, unsigned int nbytes) +{ + gfp_t flags; + struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); + + flags = (parent->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + + common->gfp_flags = flags; + common->dma = kmem_cache_alloc(ac->dma_cache, flags); + if (!common->dma) + return -ENOMEM; + + common->req = parent; + common->complete = complete; + return 0; +} + +static void +artpec6_crypto_bounce_destroy(struct artpec6_crypto_dma_descriptors *dma) +{ + struct artpec6_crypto_bounce_buffer *b; + struct artpec6_crypto_bounce_buffer *next; + + list_for_each_entry_safe(b, next, &dma->bounce_buffers, list) { + kfree(b); + } +} + +static int +artpec6_crypto_common_destroy(struct artpec6_crypto_req_common *common) +{ + struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); + + artpec6_crypto_dma_unmap_all(common); + artpec6_crypto_bounce_destroy(common->dma); + kmem_cache_free(ac->dma_cache, common->dma); + common->dma = NULL; + return 0; +} + +/* + * Ciphering functions. + */ +static int artpec6_crypto_encrypt(struct skcipher_request *req) +{ + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); + struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher); + struct artpec6_crypto_request_context *req_ctx = NULL; + void (*complete)(struct crypto_async_request *req); + int ret; + + req_ctx = skcipher_request_ctx(req); + + switch (ctx->crypto_type) { + case ARTPEC6_CRYPTO_CIPHER_AES_CBC: + case ARTPEC6_CRYPTO_CIPHER_AES_ECB: + case ARTPEC6_CRYPTO_CIPHER_AES_XTS: + req_ctx->decrypt = 0; + break; + default: + break; + } + + switch (ctx->crypto_type) { + case ARTPEC6_CRYPTO_CIPHER_AES_CBC: + complete = artpec6_crypto_complete_cbc_encrypt; + break; + default: + complete = artpec6_crypto_complete_crypto; + break; + } + + ret = artpec6_crypto_common_init(&req_ctx->common, + &req->base, + complete, + req->dst, req->cryptlen); + if (ret) + return ret; + + ret = artpec6_crypto_prepare_crypto(req); + if (ret) { + artpec6_crypto_common_destroy(&req_ctx->common); + return ret; + } + + return artpec6_crypto_submit(&req_ctx->common); +} + +static int artpec6_crypto_decrypt(struct skcipher_request *req) +{ + int ret; + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); + struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher); + struct artpec6_crypto_request_context *req_ctx = NULL; + void (*complete)(struct crypto_async_request *req); + + req_ctx = skcipher_request_ctx(req); + + switch (ctx->crypto_type) { + case ARTPEC6_CRYPTO_CIPHER_AES_CBC: + case ARTPEC6_CRYPTO_CIPHER_AES_ECB: + case ARTPEC6_CRYPTO_CIPHER_AES_XTS: + req_ctx->decrypt = 1; + break; + default: + break; + } + + + switch (ctx->crypto_type) { + case ARTPEC6_CRYPTO_CIPHER_AES_CBC: + complete = artpec6_crypto_complete_cbc_decrypt; + break; + default: + complete = artpec6_crypto_complete_crypto; + break; + } + + ret = artpec6_crypto_common_init(&req_ctx->common, &req->base, + complete, + req->dst, req->cryptlen); + if (ret) + return ret; + + ret = artpec6_crypto_prepare_crypto(req); + if (ret) { + artpec6_crypto_common_destroy(&req_ctx->common); + return ret; + } + + return artpec6_crypto_submit(&req_ctx->common); +} + +static int +artpec6_crypto_ctr_crypt(struct skcipher_request *req, bool encrypt) +{ + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); + struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher); + size_t iv_len = crypto_skcipher_ivsize(cipher); + unsigned int counter = be32_to_cpup((__be32 *) + (req->iv + iv_len - 4)); + unsigned int nblks = ALIGN(req->cryptlen, AES_BLOCK_SIZE) / + AES_BLOCK_SIZE; + + /* + * The hardware uses only the last 32-bits as the counter while the + * kernel tests (aes_ctr_enc_tv_template[4] for example) expect that + * the whole IV is a counter. So fallback if the counter is going to + * overlow. + */ + if (counter + nblks < counter) { + int ret; + + pr_debug("counter %x will overflow (nblks %u), falling back\n", + counter, counter + nblks); + + ret = crypto_skcipher_setkey(ctx->fallback, ctx->aes_key, + ctx->key_length); + if (ret) + return ret; + + { + SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback); + + skcipher_request_set_tfm(subreq, ctx->fallback); + skcipher_request_set_callback(subreq, req->base.flags, + NULL, NULL); + skcipher_request_set_crypt(subreq, req->src, req->dst, + req->cryptlen, req->iv); + ret = encrypt ? crypto_skcipher_encrypt(subreq) + : crypto_skcipher_decrypt(subreq); + skcipher_request_zero(subreq); + } + return ret; + } + + return encrypt ? artpec6_crypto_encrypt(req) + : artpec6_crypto_decrypt(req); +} + +static int artpec6_crypto_ctr_encrypt(struct skcipher_request *req) +{ + return artpec6_crypto_ctr_crypt(req, true); +} + +static int artpec6_crypto_ctr_decrypt(struct skcipher_request *req) +{ + return artpec6_crypto_ctr_crypt(req, false); +} + +/* + * AEAD functions + */ +static int artpec6_crypto_aead_init(struct crypto_aead *tfm) +{ + struct artpec6_cryptotfm_context *tfm_ctx = crypto_aead_ctx(tfm); + + memset(tfm_ctx, 0, sizeof(*tfm_ctx)); + + crypto_aead_set_reqsize(tfm, + sizeof(struct artpec6_crypto_aead_req_ctx)); + + return 0; +} + +static int artpec6_crypto_aead_set_key(struct crypto_aead *tfm, const u8 *key, + unsigned int len) +{ + struct artpec6_cryptotfm_context *ctx = crypto_tfm_ctx(&tfm->base); + + if (len != 16 && len != 24 && len != 32) { + crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -1; + } + + ctx->key_length = len; + + memcpy(ctx->aes_key, key, len); + return 0; +} + +static int artpec6_crypto_aead_encrypt(struct aead_request *req) +{ + int ret; + struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(req); + + req_ctx->decrypt = false; + ret = artpec6_crypto_common_init(&req_ctx->common, &req->base, + artpec6_crypto_complete_aead, + NULL, 0); + if (ret) + return ret; + + ret = artpec6_crypto_prepare_aead(req); + if (ret) { + artpec6_crypto_common_destroy(&req_ctx->common); + return ret; + } + + return artpec6_crypto_submit(&req_ctx->common); +} + +static int artpec6_crypto_aead_decrypt(struct aead_request *req) +{ + int ret; + struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(req); + + req_ctx->decrypt = true; + if (req->cryptlen < AES_BLOCK_SIZE) + return -EINVAL; + + ret = artpec6_crypto_common_init(&req_ctx->common, + &req->base, + artpec6_crypto_complete_aead, + NULL, 0); + if (ret) + return ret; + + ret = artpec6_crypto_prepare_aead(req); + if (ret) { + artpec6_crypto_common_destroy(&req_ctx->common); + return ret; + } + + return artpec6_crypto_submit(&req_ctx->common); +} + +static int artpec6_crypto_prepare_hash(struct ahash_request *areq) +{ + struct artpec6_hashalg_context *ctx = crypto_tfm_ctx(areq->base.tfm); + struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(areq); + size_t digestsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(areq)); + size_t contextsize = digestsize == SHA384_DIGEST_SIZE ? + SHA512_DIGEST_SIZE : digestsize; + size_t blocksize = crypto_tfm_alg_blocksize( + crypto_ahash_tfm(crypto_ahash_reqtfm(areq))); + struct artpec6_crypto_req_common *common = &req_ctx->common; + struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); + enum artpec6_crypto_variant variant = ac->variant; + u32 sel_ctx; + bool ext_ctx = false; + bool run_hw = false; + int error = 0; + + artpec6_crypto_init_dma_operation(common); + + /* Upload HMAC key, must be first the first packet */ + if (req_ctx->hash_flags & HASH_FLAG_HMAC) { + if (variant == ARTPEC6_CRYPTO) { + req_ctx->key_md = FIELD_PREP(A6_CRY_MD_OPER, + a6_regk_crypto_dlkey); + } else { + req_ctx->key_md = FIELD_PREP(A7_CRY_MD_OPER, + a7_regk_crypto_dlkey); + } + + /* Copy and pad up the key */ + memcpy(req_ctx->key_buffer, ctx->hmac_key, + ctx->hmac_key_length); + memset(req_ctx->key_buffer + ctx->hmac_key_length, 0, + blocksize - ctx->hmac_key_length); + + error = artpec6_crypto_setup_out_descr(common, + (void *)&req_ctx->key_md, + sizeof(req_ctx->key_md), false, false); + if (error) + return error; + + error = artpec6_crypto_setup_out_descr(common, + req_ctx->key_buffer, blocksize, + true, false); + if (error) + return error; + } + + if (!(req_ctx->hash_flags & HASH_FLAG_INIT_CTX)) { + /* Restore context */ + sel_ctx = regk_crypto_ext; + ext_ctx = true; + } else { + sel_ctx = regk_crypto_init; + } + + if (variant == ARTPEC6_CRYPTO) { + req_ctx->hash_md &= ~A6_CRY_MD_HASH_SEL_CTX; + req_ctx->hash_md |= FIELD_PREP(A6_CRY_MD_HASH_SEL_CTX, sel_ctx); + + /* If this is the final round, set the final flag */ + if (req_ctx->hash_flags & HASH_FLAG_FINALIZE) + req_ctx->hash_md |= A6_CRY_MD_HASH_HMAC_FIN; + } else { + req_ctx->hash_md &= ~A7_CRY_MD_HASH_SEL_CTX; + req_ctx->hash_md |= FIELD_PREP(A7_CRY_MD_HASH_SEL_CTX, sel_ctx); + + /* If this is the final round, set the final flag */ + if (req_ctx->hash_flags & HASH_FLAG_FINALIZE) + req_ctx->hash_md |= A7_CRY_MD_HASH_HMAC_FIN; + } + + /* Setup up metadata descriptors */ + error = artpec6_crypto_setup_out_descr(common, + (void *)&req_ctx->hash_md, + sizeof(req_ctx->hash_md), false, false); + if (error) + return error; + + error = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false); + if (error) + return error; + + if (ext_ctx) { + error = artpec6_crypto_setup_out_descr(common, + req_ctx->digeststate, + contextsize, false, false); + + if (error) + return error; + } + + if (req_ctx->hash_flags & HASH_FLAG_UPDATE) { + size_t done_bytes = 0; + size_t total_bytes = areq->nbytes + req_ctx->partial_bytes; + size_t ready_bytes = round_down(total_bytes, blocksize); + struct artpec6_crypto_walk walk; + + run_hw = ready_bytes > 0; + if (req_ctx->partial_bytes && ready_bytes) { + /* We have a partial buffer and will at least some bytes + * to the HW. Empty this partial buffer before tackling + * the SG lists + */ + memcpy(req_ctx->partial_buffer_out, + req_ctx->partial_buffer, + req_ctx->partial_bytes); + + error = artpec6_crypto_setup_out_descr(common, + req_ctx->partial_buffer_out, + req_ctx->partial_bytes, + false, true); + if (error) + return error; + + /* Reset partial buffer */ + done_bytes += req_ctx->partial_bytes; + req_ctx->partial_bytes = 0; + } + + artpec6_crypto_walk_init(&walk, areq->src); + + error = artpec6_crypto_setup_sg_descrs_out(common, &walk, + ready_bytes - + done_bytes); + if (error) + return error; + + if (walk.sg) { + size_t sg_skip = ready_bytes - done_bytes; + size_t sg_rem = areq->nbytes - sg_skip; + + sg_pcopy_to_buffer(areq->src, sg_nents(areq->src), + req_ctx->partial_buffer + + req_ctx->partial_bytes, + sg_rem, sg_skip); + + req_ctx->partial_bytes += sg_rem; + } + + req_ctx->digcnt += ready_bytes; + req_ctx->hash_flags &= ~(HASH_FLAG_UPDATE); + } + + /* Finalize */ + if (req_ctx->hash_flags & HASH_FLAG_FINALIZE) { + bool needtrim = contextsize != digestsize; + size_t hash_pad_len; + u64 digest_bits; + u32 oper; + + if (variant == ARTPEC6_CRYPTO) + oper = FIELD_GET(A6_CRY_MD_OPER, req_ctx->hash_md); + else + oper = FIELD_GET(A7_CRY_MD_OPER, req_ctx->hash_md); + + /* Write out the partial buffer if present */ + if (req_ctx->partial_bytes) { + memcpy(req_ctx->partial_buffer_out, + req_ctx->partial_buffer, + req_ctx->partial_bytes); + error = artpec6_crypto_setup_out_descr(common, + req_ctx->partial_buffer_out, + req_ctx->partial_bytes, + false, true); + if (error) + return error; + + req_ctx->digcnt += req_ctx->partial_bytes; + req_ctx->partial_bytes = 0; + } + + if (req_ctx->hash_flags & HASH_FLAG_HMAC) + digest_bits = 8 * (req_ctx->digcnt + blocksize); + else + digest_bits = 8 * req_ctx->digcnt; + + /* Add the hash pad */ + hash_pad_len = create_hash_pad(oper, req_ctx->pad_buffer, + req_ctx->digcnt, digest_bits); + error = artpec6_crypto_setup_out_descr(common, + req_ctx->pad_buffer, + hash_pad_len, false, + true); + req_ctx->digcnt = 0; + + if (error) + return error; + + /* Descriptor for the final result */ + error = artpec6_crypto_setup_in_descr(common, areq->result, + digestsize, + !needtrim); + if (error) + return error; + + if (needtrim) { + /* Discard the extra context bytes for SHA-384 */ + error = artpec6_crypto_setup_in_descr(common, + req_ctx->partial_buffer, + digestsize - contextsize, true); + if (error) + return error; + } + + } else { /* This is not the final operation for this request */ + if (!run_hw) + return ARTPEC6_CRYPTO_PREPARE_HASH_NO_START; + + /* Save the result to the context */ + error = artpec6_crypto_setup_in_descr(common, + req_ctx->digeststate, + contextsize, false); + if (error) + return error; + /* fall through */ + } + + req_ctx->hash_flags &= ~(HASH_FLAG_INIT_CTX | HASH_FLAG_UPDATE | + HASH_FLAG_FINALIZE); + + error = artpec6_crypto_terminate_in_descrs(common); + if (error) + return error; + + error = artpec6_crypto_terminate_out_descrs(common); + if (error) + return error; + + error = artpec6_crypto_dma_map_descs(common); + if (error) + return error; + + return ARTPEC6_CRYPTO_PREPARE_HASH_START; +} + + +static int artpec6_crypto_aes_ecb_init(struct crypto_skcipher *tfm) +{ + struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); + + tfm->reqsize = sizeof(struct artpec6_crypto_request_context); + ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_ECB; + + return 0; +} + +static int artpec6_crypto_aes_ctr_init(struct crypto_skcipher *tfm) +{ + struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); + + ctx->fallback = crypto_alloc_skcipher(crypto_tfm_alg_name(&tfm->base), + 0, + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->fallback)) + return PTR_ERR(ctx->fallback); + + tfm->reqsize = sizeof(struct artpec6_crypto_request_context); + ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_CTR; + + return 0; +} + +static int artpec6_crypto_aes_cbc_init(struct crypto_skcipher *tfm) +{ + struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); + + tfm->reqsize = sizeof(struct artpec6_crypto_request_context); + ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_CBC; + + return 0; +} + +static int artpec6_crypto_aes_xts_init(struct crypto_skcipher *tfm) +{ + struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); + + tfm->reqsize = sizeof(struct artpec6_crypto_request_context); + ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_XTS; + + return 0; +} + +static void artpec6_crypto_aes_exit(struct crypto_skcipher *tfm) +{ + struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); + + memset(ctx, 0, sizeof(*ctx)); +} + +static void artpec6_crypto_aes_ctr_exit(struct crypto_skcipher *tfm) +{ + struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); + + crypto_free_skcipher(ctx->fallback); + artpec6_crypto_aes_exit(tfm); +} + +static int +artpec6_crypto_cipher_set_key(struct crypto_skcipher *cipher, const u8 *key, + unsigned int keylen) +{ + struct artpec6_cryptotfm_context *ctx = + crypto_skcipher_ctx(cipher); + + switch (keylen) { + case 16: + case 24: + case 32: + break; + default: + crypto_skcipher_set_flags(cipher, + CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + + memcpy(ctx->aes_key, key, keylen); + ctx->key_length = keylen; + return 0; +} + +static int +artpec6_crypto_xts_set_key(struct crypto_skcipher *cipher, const u8 *key, + unsigned int keylen) +{ + struct artpec6_cryptotfm_context *ctx = + crypto_skcipher_ctx(cipher); + int ret; + + ret = xts_check_key(&cipher->base, key, keylen); + if (ret) + return ret; + + switch (keylen) { + case 32: + case 48: + case 64: + break; + default: + crypto_skcipher_set_flags(cipher, + CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + + memcpy(ctx->aes_key, key, keylen); + ctx->key_length = keylen; + return 0; +} + +/** artpec6_crypto_process_crypto - Prepare an async block cipher crypto request + * + * @req: The asynch request to process + * + * @return 0 if the dma job was successfully prepared + * <0 on error + * + * This function sets up the PDMA descriptors for a block cipher request. + * + * The required padding is added for AES-CTR using a statically defined + * buffer. + * + * The PDMA descriptor list will be as follows: + * + * OUT: [KEY_MD][KEY][EOP]<CIPHER_MD>[IV]<data_0>...[data_n][AES-CTR_pad]<eop> + * IN: <CIPHER_MD><data_0>...[data_n]<intr> + * + */ +static int artpec6_crypto_prepare_crypto(struct skcipher_request *areq) +{ + int ret; + struct artpec6_crypto_walk walk; + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq); + struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher); + struct artpec6_crypto_request_context *req_ctx = NULL; + size_t iv_len = crypto_skcipher_ivsize(cipher); + struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); + enum artpec6_crypto_variant variant = ac->variant; + struct artpec6_crypto_req_common *common; + bool cipher_decr = false; + size_t cipher_klen; + u32 cipher_len = 0; /* Same as regk_crypto_key_128 for NULL crypto */ + u32 oper; + + req_ctx = skcipher_request_ctx(areq); + common = &req_ctx->common; + + artpec6_crypto_init_dma_operation(common); + + if (variant == ARTPEC6_CRYPTO) + ctx->key_md = FIELD_PREP(A6_CRY_MD_OPER, a6_regk_crypto_dlkey); + else + ctx->key_md = FIELD_PREP(A7_CRY_MD_OPER, a7_regk_crypto_dlkey); + + ret = artpec6_crypto_setup_out_descr(common, (void *)&ctx->key_md, + sizeof(ctx->key_md), false, false); + if (ret) + return ret; + + ret = artpec6_crypto_setup_out_descr(common, ctx->aes_key, + ctx->key_length, true, false); + if (ret) + return ret; + + req_ctx->cipher_md = 0; + + if (ctx->crypto_type == ARTPEC6_CRYPTO_CIPHER_AES_XTS) + cipher_klen = ctx->key_length/2; + else + cipher_klen = ctx->key_length; + + /* Metadata */ + switch (cipher_klen) { + case 16: + cipher_len = regk_crypto_key_128; + break; + case 24: + cipher_len = regk_crypto_key_192; + break; + case 32: + cipher_len = regk_crypto_key_256; + break; + default: + pr_err("%s: Invalid key length %d!\n", + MODULE_NAME, ctx->key_length); + return -EINVAL; + } + + switch (ctx->crypto_type) { + case ARTPEC6_CRYPTO_CIPHER_AES_ECB: + oper = regk_crypto_aes_ecb; + cipher_decr = req_ctx->decrypt; + break; + + case ARTPEC6_CRYPTO_CIPHER_AES_CBC: + oper = regk_crypto_aes_cbc; + cipher_decr = req_ctx->decrypt; + break; + + case ARTPEC6_CRYPTO_CIPHER_AES_CTR: + oper = regk_crypto_aes_ctr; + cipher_decr = false; + break; + + case ARTPEC6_CRYPTO_CIPHER_AES_XTS: + oper = regk_crypto_aes_xts; + cipher_decr = req_ctx->decrypt; + + if (variant == ARTPEC6_CRYPTO) + req_ctx->cipher_md |= A6_CRY_MD_CIPHER_DSEQ; + else + req_ctx->cipher_md |= A7_CRY_MD_CIPHER_DSEQ; + break; + + default: + pr_err("%s: Invalid cipher mode %d!\n", + MODULE_NAME, ctx->crypto_type); + return -EINVAL; + } + + if (variant == ARTPEC6_CRYPTO) { + req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_OPER, oper); + req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_CIPHER_LEN, + cipher_len); + if (cipher_decr) + req_ctx->cipher_md |= A6_CRY_MD_CIPHER_DECR; + } else { + req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_OPER, oper); + req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_CIPHER_LEN, + cipher_len); + if (cipher_decr) + req_ctx->cipher_md |= A7_CRY_MD_CIPHER_DECR; + } + + ret = artpec6_crypto_setup_out_descr(common, + &req_ctx->cipher_md, + sizeof(req_ctx->cipher_md), + false, false); + if (ret) + return ret; + + ret = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false); + if (ret) + return ret; + + if (iv_len) { + ret = artpec6_crypto_setup_out_descr(common, areq->iv, iv_len, + false, false); + if (ret) + return ret; + } + /* Data out */ + artpec6_crypto_walk_init(&walk, areq->src); + ret = artpec6_crypto_setup_sg_descrs_out(common, &walk, areq->cryptlen); + if (ret) + return ret; + + /* Data in */ + artpec6_crypto_walk_init(&walk, areq->dst); + ret = artpec6_crypto_setup_sg_descrs_in(common, &walk, areq->cryptlen); + if (ret) + return ret; + + /* CTR-mode padding required by the HW. */ + if (ctx->crypto_type == ARTPEC6_CRYPTO_CIPHER_AES_CTR || + ctx->crypto_type == ARTPEC6_CRYPTO_CIPHER_AES_XTS) { + size_t pad = ALIGN(areq->cryptlen, AES_BLOCK_SIZE) - + areq->cryptlen; + + if (pad) { + ret = artpec6_crypto_setup_out_descr(common, + ac->pad_buffer, + pad, false, false); + if (ret) + return ret; + + ret = artpec6_crypto_setup_in_descr(common, + ac->pad_buffer, pad, + false); + if (ret) + return ret; + } + } + + ret = artpec6_crypto_terminate_out_descrs(common); + if (ret) + return ret; + + ret = artpec6_crypto_terminate_in_descrs(common); + if (ret) + return ret; + + return artpec6_crypto_dma_map_descs(common); +} + +static int artpec6_crypto_prepare_aead(struct aead_request *areq) +{ + size_t count; + int ret; + size_t input_length; + struct artpec6_cryptotfm_context *ctx = crypto_tfm_ctx(areq->base.tfm); + struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(areq); + struct crypto_aead *cipher = crypto_aead_reqtfm(areq); + struct artpec6_crypto_req_common *common = &req_ctx->common; + struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); + enum artpec6_crypto_variant variant = ac->variant; + u32 md_cipher_len; + + artpec6_crypto_init_dma_operation(common); + + /* Key */ + if (variant == ARTPEC6_CRYPTO) { + ctx->key_md = FIELD_PREP(A6_CRY_MD_OPER, + a6_regk_crypto_dlkey); + } else { + ctx->key_md = FIELD_PREP(A7_CRY_MD_OPER, + a7_regk_crypto_dlkey); + } + ret = artpec6_crypto_setup_out_descr(common, (void *)&ctx->key_md, + sizeof(ctx->key_md), false, false); + if (ret) + return ret; + + ret = artpec6_crypto_setup_out_descr(common, ctx->aes_key, + ctx->key_length, true, false); + if (ret) + return ret; + + req_ctx->cipher_md = 0; + + switch (ctx->key_length) { + case 16: + md_cipher_len = regk_crypto_key_128; + break; + case 24: + md_cipher_len = regk_crypto_key_192; + break; + case 32: + md_cipher_len = regk_crypto_key_256; + break; + default: + return -EINVAL; + } + + if (variant == ARTPEC6_CRYPTO) { + req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_OPER, + regk_crypto_aes_gcm); + req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_CIPHER_LEN, + md_cipher_len); + if (req_ctx->decrypt) + req_ctx->cipher_md |= A6_CRY_MD_CIPHER_DECR; + } else { + req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_OPER, + regk_crypto_aes_gcm); + req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_CIPHER_LEN, + md_cipher_len); + if (req_ctx->decrypt) + req_ctx->cipher_md |= A7_CRY_MD_CIPHER_DECR; + } + + ret = artpec6_crypto_setup_out_descr(common, + (void *) &req_ctx->cipher_md, + sizeof(req_ctx->cipher_md), false, + false); + if (ret) + return ret; + + ret = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false); + if (ret) + return ret; + + /* For the decryption, cryptlen includes the tag. */ + input_length = areq->cryptlen; + if (req_ctx->decrypt) + input_length -= AES_BLOCK_SIZE; + + /* Prepare the context buffer */ + req_ctx->hw_ctx.aad_length_bits = + __cpu_to_be64(8*areq->assoclen); + + req_ctx->hw_ctx.text_length_bits = + __cpu_to_be64(8*input_length); + + memcpy(req_ctx->hw_ctx.J0, areq->iv, crypto_aead_ivsize(cipher)); + // The HW omits the initial increment of the counter field. + crypto_inc(req_ctx->hw_ctx.J0+12, 4); + + ret = artpec6_crypto_setup_out_descr(common, &req_ctx->hw_ctx, + sizeof(struct artpec6_crypto_aead_hw_ctx), false, false); + if (ret) + return ret; + + { + struct artpec6_crypto_walk walk; + + artpec6_crypto_walk_init(&walk, areq->src); + + /* Associated data */ + count = areq->assoclen; + ret = artpec6_crypto_setup_sg_descrs_out(common, &walk, count); + if (ret) + return ret; + + if (!IS_ALIGNED(areq->assoclen, 16)) { + size_t assoc_pad = 16 - (areq->assoclen % 16); + /* The HW mandates zero padding here */ + ret = artpec6_crypto_setup_out_descr(common, + ac->zero_buffer, + assoc_pad, false, + false); + if (ret) + return ret; + } + + /* Data to crypto */ + count = input_length; + ret = artpec6_crypto_setup_sg_descrs_out(common, &walk, count); + if (ret) + return ret; + + if (!IS_ALIGNED(input_length, 16)) { + size_t crypto_pad = 16 - (input_length % 16); + /* The HW mandates zero padding here */ + ret = artpec6_crypto_setup_out_descr(common, + ac->zero_buffer, + crypto_pad, + false, + false); + if (ret) + return ret; + } + } + + /* Data from crypto */ + { + struct artpec6_crypto_walk walk; + size_t output_len = areq->cryptlen; + + if (req_ctx->decrypt) + output_len -= AES_BLOCK_SIZE; + + artpec6_crypto_walk_init(&walk, areq->dst); + + /* skip associated data in the output */ + count = artpec6_crypto_walk_advance(&walk, areq->assoclen); + if (count) + return -EINVAL; + + count = output_len; + ret = artpec6_crypto_setup_sg_descrs_in(common, &walk, count); + if (ret) + return ret; + + /* Put padding between the cryptotext and the auth tag */ + if (!IS_ALIGNED(output_len, 16)) { + size_t crypto_pad = 16 - (output_len % 16); + + ret = artpec6_crypto_setup_in_descr(common, + ac->pad_buffer, + crypto_pad, false); + if (ret) + return ret; + } + + /* The authentication tag shall follow immediately after + * the output ciphertext. For decryption it is put in a context + * buffer for later compare against the input tag. + */ + count = AES_BLOCK_SIZE; + + if (req_ctx->decrypt) { + ret = artpec6_crypto_setup_in_descr(common, + req_ctx->decryption_tag, count, false); + if (ret) + return ret; + + } else { + ret = artpec6_crypto_setup_sg_descrs_in(common, &walk, + count); + if (ret) + return ret; + } + + } + + ret = artpec6_crypto_terminate_in_descrs(common); + if (ret) + return ret; + + ret = artpec6_crypto_terminate_out_descrs(common); + if (ret) + return ret; + + return artpec6_crypto_dma_map_descs(common); +} + +static void artpec6_crypto_process_queue(struct artpec6_crypto *ac) +{ + struct artpec6_crypto_req_common *req; + + while (!list_empty(&ac->queue) && !artpec6_crypto_busy()) { + req = list_first_entry(&ac->queue, + struct artpec6_crypto_req_common, + list); + list_move_tail(&req->list, &ac->pending); + artpec6_crypto_start_dma(req); + + req->req->complete(req->req, -EINPROGRESS); + } + + /* + * In some cases, the hardware can raise an in_eop_flush interrupt + * before actually updating the status, so we have an timer which will + * recheck the status on timeout. Since the cases are expected to be + * very rare, we use a relatively large timeout value. There should be + * no noticeable negative effect if we timeout spuriously. + */ + if (ac->pending_count) + mod_timer(&ac->timer, jiffies + msecs_to_jiffies(100)); + else + del_timer(&ac->timer); +} + +static void artpec6_crypto_timeout(struct timer_list *t) +{ + struct artpec6_crypto *ac = from_timer(ac, t, timer); + + dev_info_ratelimited(artpec6_crypto_dev, "timeout\n"); + + tasklet_schedule(&ac->task); +} + +static void artpec6_crypto_task(unsigned long data) +{ + struct artpec6_crypto *ac = (struct artpec6_crypto *)data; + struct artpec6_crypto_req_common *req; + struct artpec6_crypto_req_common *n; + + if (list_empty(&ac->pending)) { + pr_debug("Spurious IRQ\n"); + return; + } + + spin_lock_bh(&ac->queue_lock); + + list_for_each_entry_safe(req, n, &ac->pending, list) { + struct artpec6_crypto_dma_descriptors *dma = req->dma; + u32 stat; + + dma_sync_single_for_cpu(artpec6_crypto_dev, dma->stat_dma_addr, + sizeof(dma->stat[0]), + DMA_BIDIRECTIONAL); + + stat = req->dma->stat[req->dma->in_cnt-1]; + + /* A non-zero final status descriptor indicates + * this job has finished. + */ + pr_debug("Request %p status is %X\n", req, stat); + if (!stat) + break; + + /* Allow testing of timeout handling with fault injection */ +#ifdef CONFIG_FAULT_INJECTION + if (should_fail(&artpec6_crypto_fail_status_read, 1)) + continue; +#endif + + pr_debug("Completing request %p\n", req); + + list_del(&req->list); + + artpec6_crypto_dma_unmap_all(req); + artpec6_crypto_copy_bounce_buffers(req); + + ac->pending_count--; + artpec6_crypto_common_destroy(req); + req->complete(req->req); + } + + artpec6_crypto_process_queue(ac); + + spin_unlock_bh(&ac->queue_lock); +} + +static void artpec6_crypto_complete_crypto(struct crypto_async_request *req) +{ + req->complete(req, 0); +} + +static void +artpec6_crypto_complete_cbc_decrypt(struct crypto_async_request *req) +{ + struct skcipher_request *cipher_req = container_of(req, + struct skcipher_request, base); + + scatterwalk_map_and_copy(cipher_req->iv, cipher_req->src, + cipher_req->cryptlen - AES_BLOCK_SIZE, + AES_BLOCK_SIZE, 0); + req->complete(req, 0); +} + +static void +artpec6_crypto_complete_cbc_encrypt(struct crypto_async_request *req) +{ + struct skcipher_request *cipher_req = container_of(req, + struct skcipher_request, base); + + scatterwalk_map_and_copy(cipher_req->iv, cipher_req->dst, + cipher_req->cryptlen - AES_BLOCK_SIZE, + AES_BLOCK_SIZE, 0); + req->complete(req, 0); +} + +static void artpec6_crypto_complete_aead(struct crypto_async_request *req) +{ + int result = 0; + + /* Verify GCM hashtag. */ + struct aead_request *areq = container_of(req, + struct aead_request, base); + struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(areq); + + if (req_ctx->decrypt) { + u8 input_tag[AES_BLOCK_SIZE]; + + sg_pcopy_to_buffer(areq->src, + sg_nents(areq->src), + input_tag, + AES_BLOCK_SIZE, + areq->assoclen + areq->cryptlen - + AES_BLOCK_SIZE); + + if (memcmp(req_ctx->decryption_tag, + input_tag, + AES_BLOCK_SIZE)) { + pr_debug("***EBADMSG:\n"); + print_hex_dump_debug("ref:", DUMP_PREFIX_ADDRESS, 32, 1, + input_tag, AES_BLOCK_SIZE, true); + print_hex_dump_debug("out:", DUMP_PREFIX_ADDRESS, 32, 1, + req_ctx->decryption_tag, + AES_BLOCK_SIZE, true); + + result = -EBADMSG; + } + } + + req->complete(req, result); +} + +static void artpec6_crypto_complete_hash(struct crypto_async_request *req) +{ + req->complete(req, 0); +} + + +/*------------------- Hash functions -----------------------------------------*/ +static int +artpec6_crypto_hash_set_key(struct crypto_ahash *tfm, + const u8 *key, unsigned int keylen) +{ + struct artpec6_hashalg_context *tfm_ctx = crypto_tfm_ctx(&tfm->base); + size_t blocksize; + int ret; + + if (!keylen) { + pr_err("Invalid length (%d) of HMAC key\n", + keylen); + return -EINVAL; + } + + memset(tfm_ctx->hmac_key, 0, sizeof(tfm_ctx->hmac_key)); + + blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); + + if (keylen > blocksize) { + SHASH_DESC_ON_STACK(hdesc, tfm_ctx->child_hash); + + hdesc->tfm = tfm_ctx->child_hash; + hdesc->flags = crypto_ahash_get_flags(tfm) & + CRYPTO_TFM_REQ_MAY_SLEEP; + + tfm_ctx->hmac_key_length = blocksize; + ret = crypto_shash_digest(hdesc, key, keylen, + tfm_ctx->hmac_key); + if (ret) + return ret; + + } else { + memcpy(tfm_ctx->hmac_key, key, keylen); + tfm_ctx->hmac_key_length = keylen; + } + + return 0; +} + +static int +artpec6_crypto_init_hash(struct ahash_request *req, u8 type, int hmac) +{ + struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); + enum artpec6_crypto_variant variant = ac->variant; + struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); + u32 oper; + + memset(req_ctx, 0, sizeof(*req_ctx)); + + req_ctx->hash_flags = HASH_FLAG_INIT_CTX; + if (hmac) + req_ctx->hash_flags |= (HASH_FLAG_HMAC | HASH_FLAG_UPDATE_KEY); + + switch (type) { + case ARTPEC6_CRYPTO_HASH_SHA1: + oper = hmac ? regk_crypto_hmac_sha1 : regk_crypto_sha1; + break; + case ARTPEC6_CRYPTO_HASH_SHA256: + oper = hmac ? regk_crypto_hmac_sha256 : regk_crypto_sha256; + break; + case ARTPEC6_CRYPTO_HASH_SHA384: + oper = hmac ? regk_crypto_hmac_sha384 : regk_crypto_sha384; + break; + case ARTPEC6_CRYPTO_HASH_SHA512: + oper = hmac ? regk_crypto_hmac_sha512 : regk_crypto_sha512; + break; + + default: + pr_err("%s: Unsupported hash type 0x%x\n", MODULE_NAME, type); + return -EINVAL; + } + + if (variant == ARTPEC6_CRYPTO) + req_ctx->hash_md = FIELD_PREP(A6_CRY_MD_OPER, oper); + else + req_ctx->hash_md = FIELD_PREP(A7_CRY_MD_OPER, oper); + + return 0; +} + +static int artpec6_crypto_prepare_submit_hash(struct ahash_request *req) +{ + struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); + int ret; + + if (!req_ctx->common.dma) { + ret = artpec6_crypto_common_init(&req_ctx->common, + &req->base, + artpec6_crypto_complete_hash, + NULL, 0); + + if (ret) + return ret; + } + + ret = artpec6_crypto_prepare_hash(req); + switch (ret) { + case ARTPEC6_CRYPTO_PREPARE_HASH_START: + ret = artpec6_crypto_submit(&req_ctx->common); + break; + + case ARTPEC6_CRYPTO_PREPARE_HASH_NO_START: + ret = 0; + /* Fallthrough */ + + default: + artpec6_crypto_common_destroy(&req_ctx->common); + break; + } + + return ret; +} + +static int artpec6_crypto_hash_final(struct ahash_request *req) +{ + struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); + + req_ctx->hash_flags |= HASH_FLAG_FINALIZE; + + return artpec6_crypto_prepare_submit_hash(req); +} + +static int artpec6_crypto_hash_update(struct ahash_request *req) +{ + struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); + + req_ctx->hash_flags |= HASH_FLAG_UPDATE; + + return artpec6_crypto_prepare_submit_hash(req); +} + +static int artpec6_crypto_sha1_init(struct ahash_request *req) +{ + return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA1, 0); +} + +static int artpec6_crypto_sha1_digest(struct ahash_request *req) +{ + struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); + + artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA1, 0); + + req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE; + + return artpec6_crypto_prepare_submit_hash(req); +} + +static int artpec6_crypto_sha256_init(struct ahash_request *req) +{ + return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 0); +} + +static int artpec6_crypto_sha256_digest(struct ahash_request *req) +{ + struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); + + artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 0); + req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE; + + return artpec6_crypto_prepare_submit_hash(req); +} + +static int __maybe_unused artpec6_crypto_sha384_init(struct ahash_request *req) +{ + return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA384, 0); +} + +static int __maybe_unused +artpec6_crypto_sha384_digest(struct ahash_request *req) +{ + struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); + + artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA384, 0); + req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE; + + return artpec6_crypto_prepare_submit_hash(req); +} + +static int artpec6_crypto_sha512_init(struct ahash_request *req) +{ + return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA512, 0); +} + +static int artpec6_crypto_sha512_digest(struct ahash_request *req) +{ + struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); + + artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA512, 0); + req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE; + + return artpec6_crypto_prepare_submit_hash(req); +} + +static int artpec6_crypto_hmac_sha256_init(struct ahash_request *req) +{ + return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 1); +} + +static int __maybe_unused +artpec6_crypto_hmac_sha384_init(struct ahash_request *req) +{ + return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA384, 1); +} + +static int artpec6_crypto_hmac_sha512_init(struct ahash_request *req) +{ + return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA512, 1); +} + +static int artpec6_crypto_hmac_sha256_digest(struct ahash_request *req) +{ + struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); + + artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 1); + req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE; + + return artpec6_crypto_prepare_submit_hash(req); +} + +static int __maybe_unused +artpec6_crypto_hmac_sha384_digest(struct ahash_request *req) +{ + struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); + + artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA384, 1); + req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE; + + return artpec6_crypto_prepare_submit_hash(req); +} + +static int artpec6_crypto_hmac_sha512_digest(struct ahash_request *req) +{ + struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req); + + artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA512, 1); + req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE; + + return artpec6_crypto_prepare_submit_hash(req); +} + +static int artpec6_crypto_ahash_init_common(struct crypto_tfm *tfm, + const char *base_hash_name) +{ + struct artpec6_hashalg_context *tfm_ctx = crypto_tfm_ctx(tfm); + + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct artpec6_hash_request_context)); + memset(tfm_ctx, 0, sizeof(*tfm_ctx)); + + if (base_hash_name) { + struct crypto_shash *child; + + child = crypto_alloc_shash(base_hash_name, 0, + CRYPTO_ALG_NEED_FALLBACK); + + if (IS_ERR(child)) + return PTR_ERR(child); + + tfm_ctx->child_hash = child; + } + + return 0; +} + +static int artpec6_crypto_ahash_init(struct crypto_tfm *tfm) +{ + return artpec6_crypto_ahash_init_common(tfm, NULL); +} + +static int artpec6_crypto_ahash_init_hmac_sha256(struct crypto_tfm *tfm) +{ + return artpec6_crypto_ahash_init_common(tfm, "sha256"); +} + +static int __maybe_unused +artpec6_crypto_ahash_init_hmac_sha384(struct crypto_tfm *tfm) +{ + return artpec6_crypto_ahash_init_common(tfm, "sha384"); +} + +static int artpec6_crypto_ahash_init_hmac_sha512(struct crypto_tfm *tfm) +{ + return artpec6_crypto_ahash_init_common(tfm, "sha512"); +} + +static void artpec6_crypto_ahash_exit(struct crypto_tfm *tfm) +{ + struct artpec6_hashalg_context *tfm_ctx = crypto_tfm_ctx(tfm); + + if (tfm_ctx->child_hash) + crypto_free_shash(tfm_ctx->child_hash); + + memset(tfm_ctx->hmac_key, 0, sizeof(tfm_ctx->hmac_key)); + tfm_ctx->hmac_key_length = 0; +} + +static int artpec6_crypto_hash_export(struct ahash_request *req, void *out) +{ + const struct artpec6_hash_request_context *ctx = ahash_request_ctx(req); + struct artpec6_hash_export_state *state = out; + struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); + enum artpec6_crypto_variant variant = ac->variant; + + BUILD_BUG_ON(sizeof(state->partial_buffer) != + sizeof(ctx->partial_buffer)); + BUILD_BUG_ON(sizeof(state->digeststate) != sizeof(ctx->digeststate)); + + state->digcnt = ctx->digcnt; + state->partial_bytes = ctx->partial_bytes; + state->hash_flags = ctx->hash_flags; + + if (variant == ARTPEC6_CRYPTO) + state->oper = FIELD_GET(A6_CRY_MD_OPER, ctx->hash_md); + else + state->oper = FIELD_GET(A7_CRY_MD_OPER, ctx->hash_md); + + memcpy(state->partial_buffer, ctx->partial_buffer, + sizeof(state->partial_buffer)); + memcpy(state->digeststate, ctx->digeststate, + sizeof(state->digeststate)); + + return 0; +} + +static int artpec6_crypto_hash_import(struct ahash_request *req, const void *in) +{ + struct artpec6_hash_request_context *ctx = ahash_request_ctx(req); + const struct artpec6_hash_export_state *state = in; + struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev); + enum artpec6_crypto_variant variant = ac->variant; + + memset(ctx, 0, sizeof(*ctx)); + + ctx->digcnt = state->digcnt; + ctx->partial_bytes = state->partial_bytes; + ctx->hash_flags = state->hash_flags; + + if (variant == ARTPEC6_CRYPTO) + ctx->hash_md = FIELD_PREP(A6_CRY_MD_OPER, state->oper); + else + ctx->hash_md = FIELD_PREP(A7_CRY_MD_OPER, state->oper); + + memcpy(ctx->partial_buffer, state->partial_buffer, + sizeof(state->partial_buffer)); + memcpy(ctx->digeststate, state->digeststate, + sizeof(state->digeststate)); + + return 0; +} + +static int init_crypto_hw(struct artpec6_crypto *ac) +{ + enum artpec6_crypto_variant variant = ac->variant; + void __iomem *base = ac->base; + u32 out_descr_buf_size; + u32 out_data_buf_size; + u32 in_data_buf_size; + u32 in_descr_buf_size; + u32 in_stat_buf_size; + u32 in, out; + + /* + * The PDMA unit contains 1984 bytes of internal memory for the OUT + * channels and 1024 bytes for the IN channel. This is an elastic + * memory used to internally store the descriptors and data. The values + * ares specified in 64 byte incremements. Trustzone buffers are not + * used at this stage. + */ + out_data_buf_size = 16; /* 1024 bytes for data */ + out_descr_buf_size = 15; /* 960 bytes for descriptors */ + in_data_buf_size = 8; /* 512 bytes for data */ + in_descr_buf_size = 4; /* 256 bytes for descriptors */ + in_stat_buf_size = 4; /* 256 bytes for stat descrs */ + + BUILD_BUG_ON_MSG((out_data_buf_size + + out_descr_buf_size) * 64 > 1984, + "Invalid OUT configuration"); + + BUILD_BUG_ON_MSG((in_data_buf_size + + in_descr_buf_size + + in_stat_buf_size) * 64 > 1024, + "Invalid IN configuration"); + + in = FIELD_PREP(PDMA_IN_BUF_CFG_DATA_BUF_SIZE, in_data_buf_size) | + FIELD_PREP(PDMA_IN_BUF_CFG_DESCR_BUF_SIZE, in_descr_buf_size) | + FIELD_PREP(PDMA_IN_BUF_CFG_STAT_BUF_SIZE, in_stat_buf_size); + + out = FIELD_PREP(PDMA_OUT_BUF_CFG_DATA_BUF_SIZE, out_data_buf_size) | + FIELD_PREP(PDMA_OUT_BUF_CFG_DESCR_BUF_SIZE, out_descr_buf_size); + + writel_relaxed(out, base + PDMA_OUT_BUF_CFG); + writel_relaxed(PDMA_OUT_CFG_EN, base + PDMA_OUT_CFG); + + if (variant == ARTPEC6_CRYPTO) { + writel_relaxed(in, base + A6_PDMA_IN_BUF_CFG); + writel_relaxed(PDMA_IN_CFG_EN, base + A6_PDMA_IN_CFG); + writel_relaxed(A6_PDMA_INTR_MASK_IN_DATA | + A6_PDMA_INTR_MASK_IN_EOP_FLUSH, + base + A6_PDMA_INTR_MASK); + } else { + writel_relaxed(in, base + A7_PDMA_IN_BUF_CFG); + writel_relaxed(PDMA_IN_CFG_EN, base + A7_PDMA_IN_CFG); + writel_relaxed(A7_PDMA_INTR_MASK_IN_DATA | + A7_PDMA_INTR_MASK_IN_EOP_FLUSH, + base + A7_PDMA_INTR_MASK); + } + + return 0; +} + +static void artpec6_crypto_disable_hw(struct artpec6_crypto *ac) +{ + enum artpec6_crypto_variant variant = ac->variant; + void __iomem *base = ac->base; + + if (variant == ARTPEC6_CRYPTO) { + writel_relaxed(A6_PDMA_IN_CMD_STOP, base + A6_PDMA_IN_CMD); + writel_relaxed(0, base + A6_PDMA_IN_CFG); + writel_relaxed(A6_PDMA_OUT_CMD_STOP, base + PDMA_OUT_CMD); + } else { + writel_relaxed(A7_PDMA_IN_CMD_STOP, base + A7_PDMA_IN_CMD); + writel_relaxed(0, base + A7_PDMA_IN_CFG); + writel_relaxed(A7_PDMA_OUT_CMD_STOP, base + PDMA_OUT_CMD); + } + + writel_relaxed(0, base + PDMA_OUT_CFG); + +} + +static irqreturn_t artpec6_crypto_irq(int irq, void *dev_id) +{ + struct artpec6_crypto *ac = dev_id; + enum artpec6_crypto_variant variant = ac->variant; + void __iomem *base = ac->base; + u32 mask_in_data, mask_in_eop_flush; + u32 in_cmd_flush_stat, in_cmd_reg; + u32 ack_intr_reg; + u32 ack = 0; + u32 intr; + + if (variant == ARTPEC6_CRYPTO) { + intr = readl_relaxed(base + A6_PDMA_MASKED_INTR); + mask_in_data = A6_PDMA_INTR_MASK_IN_DATA; + mask_in_eop_flush = A6_PDMA_INTR_MASK_IN_EOP_FLUSH; + in_cmd_flush_stat = A6_PDMA_IN_CMD_FLUSH_STAT; + in_cmd_reg = A6_PDMA_IN_CMD; + ack_intr_reg = A6_PDMA_ACK_INTR; + } else { + intr = readl_relaxed(base + A7_PDMA_MASKED_INTR); + mask_in_data = A7_PDMA_INTR_MASK_IN_DATA; + mask_in_eop_flush = A7_PDMA_INTR_MASK_IN_EOP_FLUSH; + in_cmd_flush_stat = A7_PDMA_IN_CMD_FLUSH_STAT; + in_cmd_reg = A7_PDMA_IN_CMD; + ack_intr_reg = A7_PDMA_ACK_INTR; + } + + /* We get two interrupt notifications from each job. + * The in_data means all data was sent to memory and then + * we request a status flush command to write the per-job + * status to its status vector. This ensures that the + * tasklet can detect exactly how many submitted jobs + * that have finished. + */ + if (intr & mask_in_data) + ack |= mask_in_data; + + if (intr & mask_in_eop_flush) + ack |= mask_in_eop_flush; + else + writel_relaxed(in_cmd_flush_stat, base + in_cmd_reg); + + writel_relaxed(ack, base + ack_intr_reg); + + if (intr & mask_in_eop_flush) + tasklet_schedule(&ac->task); + + return IRQ_HANDLED; +} + +/*------------------- Algorithm definitions ----------------------------------*/ + +/* Hashes */ +static struct ahash_alg hash_algos[] = { + /* SHA-1 */ + { + .init = artpec6_crypto_sha1_init, + .update = artpec6_crypto_hash_update, + .final = artpec6_crypto_hash_final, + .digest = artpec6_crypto_sha1_digest, + .import = artpec6_crypto_hash_import, + .export = artpec6_crypto_hash_export, + .halg.digestsize = SHA1_DIGEST_SIZE, + .halg.statesize = sizeof(struct artpec6_hash_export_state), + .halg.base = { + .cra_name = "sha1", + .cra_driver_name = "artpec-sha1", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct artpec6_hashalg_context), + .cra_alignmask = 3, + .cra_module = THIS_MODULE, + .cra_init = artpec6_crypto_ahash_init, + .cra_exit = artpec6_crypto_ahash_exit, + } + }, + /* SHA-256 */ + { + .init = artpec6_crypto_sha256_init, + .update = artpec6_crypto_hash_update, + .final = artpec6_crypto_hash_final, + .digest = artpec6_crypto_sha256_digest, + .import = artpec6_crypto_hash_import, + .export = artpec6_crypto_hash_export, + .halg.digestsize = SHA256_DIGEST_SIZE, + .halg.statesize = sizeof(struct artpec6_hash_export_state), + .halg.base = { + .cra_name = "sha256", + .cra_driver_name = "artpec-sha256", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct artpec6_hashalg_context), + .cra_alignmask = 3, + .cra_module = THIS_MODULE, + .cra_init = artpec6_crypto_ahash_init, + .cra_exit = artpec6_crypto_ahash_exit, + } + }, + /* HMAC SHA-256 */ + { + .init = artpec6_crypto_hmac_sha256_init, + .update = artpec6_crypto_hash_update, + .final = artpec6_crypto_hash_final, + .digest = artpec6_crypto_hmac_sha256_digest, + .import = artpec6_crypto_hash_import, + .export = artpec6_crypto_hash_export, + .setkey = artpec6_crypto_hash_set_key, + .halg.digestsize = SHA256_DIGEST_SIZE, + .halg.statesize = sizeof(struct artpec6_hash_export_state), + .halg.base = { + .cra_name = "hmac(sha256)", + .cra_driver_name = "artpec-hmac-sha256", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct artpec6_hashalg_context), + .cra_alignmask = 3, + .cra_module = THIS_MODULE, + .cra_init = artpec6_crypto_ahash_init_hmac_sha256, + .cra_exit = artpec6_crypto_ahash_exit, + } + }, +}; + +static struct ahash_alg artpec7_hash_algos[] = { + /* SHA-384 */ + { + .init = artpec6_crypto_sha384_init, + .update = artpec6_crypto_hash_update, + .final = artpec6_crypto_hash_final, + .digest = artpec6_crypto_sha384_digest, + .import = artpec6_crypto_hash_import, + .export = artpec6_crypto_hash_export, + .halg.digestsize = SHA384_DIGEST_SIZE, + .halg.statesize = sizeof(struct artpec6_hash_export_state), + .halg.base = { + .cra_name = "sha384", + .cra_driver_name = "artpec-sha384", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA384_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct artpec6_hashalg_context), + .cra_alignmask = 3, + .cra_module = THIS_MODULE, + .cra_init = artpec6_crypto_ahash_init, + .cra_exit = artpec6_crypto_ahash_exit, + } + }, + /* HMAC SHA-384 */ + { + .init = artpec6_crypto_hmac_sha384_init, + .update = artpec6_crypto_hash_update, + .final = artpec6_crypto_hash_final, + .digest = artpec6_crypto_hmac_sha384_digest, + .import = artpec6_crypto_hash_import, + .export = artpec6_crypto_hash_export, + .setkey = artpec6_crypto_hash_set_key, + .halg.digestsize = SHA384_DIGEST_SIZE, + .halg.statesize = sizeof(struct artpec6_hash_export_state), + .halg.base = { + .cra_name = "hmac(sha384)", + .cra_driver_name = "artpec-hmac-sha384", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA384_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct artpec6_hashalg_context), + .cra_alignmask = 3, + .cra_module = THIS_MODULE, + .cra_init = artpec6_crypto_ahash_init_hmac_sha384, + .cra_exit = artpec6_crypto_ahash_exit, + } + }, + /* SHA-512 */ + { + .init = artpec6_crypto_sha512_init, + .update = artpec6_crypto_hash_update, + .final = artpec6_crypto_hash_final, + .digest = artpec6_crypto_sha512_digest, + .import = artpec6_crypto_hash_import, + .export = artpec6_crypto_hash_export, + .halg.digestsize = SHA512_DIGEST_SIZE, + .halg.statesize = sizeof(struct artpec6_hash_export_state), + .halg.base = { + .cra_name = "sha512", + .cra_driver_name = "artpec-sha512", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct artpec6_hashalg_context), + .cra_alignmask = 3, + .cra_module = THIS_MODULE, + .cra_init = artpec6_crypto_ahash_init, + .cra_exit = artpec6_crypto_ahash_exit, + } + }, + /* HMAC SHA-512 */ + { + .init = artpec6_crypto_hmac_sha512_init, + .update = artpec6_crypto_hash_update, + .final = artpec6_crypto_hash_final, + .digest = artpec6_crypto_hmac_sha512_digest, + .import = artpec6_crypto_hash_import, + .export = artpec6_crypto_hash_export, + .setkey = artpec6_crypto_hash_set_key, + .halg.digestsize = SHA512_DIGEST_SIZE, + .halg.statesize = sizeof(struct artpec6_hash_export_state), + .halg.base = { + .cra_name = "hmac(sha512)", + .cra_driver_name = "artpec-hmac-sha512", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct artpec6_hashalg_context), + .cra_alignmask = 3, + .cra_module = THIS_MODULE, + .cra_init = artpec6_crypto_ahash_init_hmac_sha512, + .cra_exit = artpec6_crypto_ahash_exit, + } + }, +}; + +/* Crypto */ +static struct skcipher_alg crypto_algos[] = { + /* AES - ECB */ + { + .base = { + .cra_name = "ecb(aes)", + .cra_driver_name = "artpec6-ecb-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | + CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context), + .cra_alignmask = 3, + .cra_module = THIS_MODULE, + }, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = artpec6_crypto_cipher_set_key, + .encrypt = artpec6_crypto_encrypt, + .decrypt = artpec6_crypto_decrypt, + .init = artpec6_crypto_aes_ecb_init, + .exit = artpec6_crypto_aes_exit, + }, + /* AES - CTR */ + { + .base = { + .cra_name = "ctr(aes)", + .cra_driver_name = "artpec6-ctr-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context), + .cra_alignmask = 3, + .cra_module = THIS_MODULE, + }, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = artpec6_crypto_cipher_set_key, + .encrypt = artpec6_crypto_ctr_encrypt, + .decrypt = artpec6_crypto_ctr_decrypt, + .init = artpec6_crypto_aes_ctr_init, + .exit = artpec6_crypto_aes_ctr_exit, + }, + /* AES - CBC */ + { + .base = { + .cra_name = "cbc(aes)", + .cra_driver_name = "artpec6-cbc-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | + CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context), + .cra_alignmask = 3, + .cra_module = THIS_MODULE, + }, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = artpec6_crypto_cipher_set_key, + .encrypt = artpec6_crypto_encrypt, + .decrypt = artpec6_crypto_decrypt, + .init = artpec6_crypto_aes_cbc_init, + .exit = artpec6_crypto_aes_exit + }, + /* AES - XTS */ + { + .base = { + .cra_name = "xts(aes)", + .cra_driver_name = "artpec6-xts-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | + CRYPTO_ALG_ASYNC, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context), + .cra_alignmask = 3, + .cra_module = THIS_MODULE, + }, + .min_keysize = 2*AES_MIN_KEY_SIZE, + .max_keysize = 2*AES_MAX_KEY_SIZE, + .ivsize = 16, + .setkey = artpec6_crypto_xts_set_key, + .encrypt = artpec6_crypto_encrypt, + .decrypt = artpec6_crypto_decrypt, + .init = artpec6_crypto_aes_xts_init, + .exit = artpec6_crypto_aes_exit, + }, +}; + +static struct aead_alg aead_algos[] = { + { + .init = artpec6_crypto_aead_init, + .setkey = artpec6_crypto_aead_set_key, + .encrypt = artpec6_crypto_aead_encrypt, + .decrypt = artpec6_crypto_aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + + .base = { + .cra_name = "gcm(aes)", + .cra_driver_name = "artpec-gcm-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context), + .cra_alignmask = 3, + .cra_module = THIS_MODULE, + }, + } +}; + +#ifdef CONFIG_DEBUG_FS + +struct dbgfs_u32 { + char *name; + mode_t mode; + u32 *flag; + char *desc; +}; + +static struct dentry *dbgfs_root; + +static void artpec6_crypto_init_debugfs(void) +{ + dbgfs_root = debugfs_create_dir("artpec6_crypto", NULL); + + if (!dbgfs_root || IS_ERR(dbgfs_root)) { + dbgfs_root = NULL; + pr_err("%s: Could not initialise debugfs!\n", MODULE_NAME); + return; + } + +#ifdef CONFIG_FAULT_INJECTION + fault_create_debugfs_attr("fail_status_read", dbgfs_root, + &artpec6_crypto_fail_status_read); + + fault_create_debugfs_attr("fail_dma_array_full", dbgfs_root, + &artpec6_crypto_fail_dma_array_full); +#endif +} + +static void artpec6_crypto_free_debugfs(void) +{ + if (!dbgfs_root) + return; + + debugfs_remove_recursive(dbgfs_root); + dbgfs_root = NULL; +} +#endif + +static const struct of_device_id artpec6_crypto_of_match[] = { + { .compatible = "axis,artpec6-crypto", .data = (void *)ARTPEC6_CRYPTO }, + { .compatible = "axis,artpec7-crypto", .data = (void *)ARTPEC7_CRYPTO }, + {} +}; +MODULE_DEVICE_TABLE(of, artpec6_crypto_of_match); + +static int artpec6_crypto_probe(struct platform_device *pdev) +{ + const struct of_device_id *match; + enum artpec6_crypto_variant variant; + struct artpec6_crypto *ac; + struct device *dev = &pdev->dev; + void __iomem *base; + struct resource *res; + int irq; + int err; + + if (artpec6_crypto_dev) + return -ENODEV; + + match = of_match_node(artpec6_crypto_of_match, dev->of_node); + if (!match) + return -EINVAL; + + variant = (enum artpec6_crypto_variant)match->data; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res) + return -ENODEV; + + base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(base)) + return PTR_ERR(base); + + irq = platform_get_irq(pdev, 0); + if (irq < 0) + return -ENODEV; + + ac = devm_kzalloc(&pdev->dev, sizeof(struct artpec6_crypto), + GFP_KERNEL); + if (!ac) + return -ENOMEM; + + platform_set_drvdata(pdev, ac); + ac->variant = variant; + + spin_lock_init(&ac->queue_lock); + INIT_LIST_HEAD(&ac->queue); + INIT_LIST_HEAD(&ac->pending); + timer_setup(&ac->timer, artpec6_crypto_timeout, 0); + + ac->base = base; + + ac->dma_cache = kmem_cache_create("artpec6_crypto_dma", + sizeof(struct artpec6_crypto_dma_descriptors), + 64, + 0, + NULL); + if (!ac->dma_cache) + return -ENOMEM; + +#ifdef CONFIG_DEBUG_FS + artpec6_crypto_init_debugfs(); +#endif + + tasklet_init(&ac->task, artpec6_crypto_task, + (unsigned long)ac); + + ac->pad_buffer = devm_kzalloc(&pdev->dev, 2 * ARTPEC_CACHE_LINE_MAX, + GFP_KERNEL); + if (!ac->pad_buffer) + return -ENOMEM; + ac->pad_buffer = PTR_ALIGN(ac->pad_buffer, ARTPEC_CACHE_LINE_MAX); + + ac->zero_buffer = devm_kzalloc(&pdev->dev, 2 * ARTPEC_CACHE_LINE_MAX, + GFP_KERNEL); + if (!ac->zero_buffer) + return -ENOMEM; + ac->zero_buffer = PTR_ALIGN(ac->zero_buffer, ARTPEC_CACHE_LINE_MAX); + + err = init_crypto_hw(ac); + if (err) + goto free_cache; + + err = devm_request_irq(&pdev->dev, irq, artpec6_crypto_irq, 0, + "artpec6-crypto", ac); + if (err) + goto disable_hw; + + artpec6_crypto_dev = &pdev->dev; + + err = crypto_register_ahashes(hash_algos, ARRAY_SIZE(hash_algos)); + if (err) { + dev_err(dev, "Failed to register ahashes\n"); + goto disable_hw; + } + + if (variant != ARTPEC6_CRYPTO) { + err = crypto_register_ahashes(artpec7_hash_algos, + ARRAY_SIZE(artpec7_hash_algos)); + if (err) { + dev_err(dev, "Failed to register ahashes\n"); + goto unregister_ahashes; + } + } + + err = crypto_register_skciphers(crypto_algos, ARRAY_SIZE(crypto_algos)); + if (err) { + dev_err(dev, "Failed to register ciphers\n"); + goto unregister_a7_ahashes; + } + + err = crypto_register_aeads(aead_algos, ARRAY_SIZE(aead_algos)); + if (err) { + dev_err(dev, "Failed to register aeads\n"); + goto unregister_algs; + } + + return 0; + +unregister_algs: + crypto_unregister_skciphers(crypto_algos, ARRAY_SIZE(crypto_algos)); +unregister_a7_ahashes: + if (variant != ARTPEC6_CRYPTO) + crypto_unregister_ahashes(artpec7_hash_algos, + ARRAY_SIZE(artpec7_hash_algos)); +unregister_ahashes: + crypto_unregister_ahashes(hash_algos, ARRAY_SIZE(hash_algos)); +disable_hw: + artpec6_crypto_disable_hw(ac); +free_cache: + kmem_cache_destroy(ac->dma_cache); + return err; +} + +static int artpec6_crypto_remove(struct platform_device *pdev) +{ + struct artpec6_crypto *ac = platform_get_drvdata(pdev); + int irq = platform_get_irq(pdev, 0); + + crypto_unregister_ahashes(hash_algos, ARRAY_SIZE(hash_algos)); + if (ac->variant != ARTPEC6_CRYPTO) + crypto_unregister_ahashes(artpec7_hash_algos, + ARRAY_SIZE(artpec7_hash_algos)); + crypto_unregister_skciphers(crypto_algos, ARRAY_SIZE(crypto_algos)); + crypto_unregister_aeads(aead_algos, ARRAY_SIZE(aead_algos)); + + tasklet_disable(&ac->task); + devm_free_irq(&pdev->dev, irq, ac); + tasklet_kill(&ac->task); + del_timer_sync(&ac->timer); + + artpec6_crypto_disable_hw(ac); + + kmem_cache_destroy(ac->dma_cache); +#ifdef CONFIG_DEBUG_FS + artpec6_crypto_free_debugfs(); +#endif + return 0; +} + +static struct platform_driver artpec6_crypto_driver = { + .probe = artpec6_crypto_probe, + .remove = artpec6_crypto_remove, + .driver = { + .name = "artpec6-crypto", + .owner = THIS_MODULE, + .of_match_table = artpec6_crypto_of_match, + }, +}; + +module_platform_driver(artpec6_crypto_driver); + +MODULE_AUTHOR("Axis Communications AB"); +MODULE_DESCRIPTION("ARTPEC-6 Crypto driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/crypto/bcm/cipher.c b/drivers/crypto/bcm/cipher.c index 9cfd36c1bcb6..ce70b44d0fb6 100644 --- a/drivers/crypto/bcm/cipher.c +++ b/drivers/crypto/bcm/cipher.c @@ -90,8 +90,6 @@ static int aead_pri = 150; module_param(aead_pri, int, 0644); MODULE_PARM_DESC(aead_pri, "Priority for AEAD algos"); -#define MAX_SPUS 16 - /* A type 3 BCM header, expected to precede the SPU header for SPU-M. * Bits 3 and 4 in the first byte encode the channel number (the dma ringset). * 0x60 - ring 0 @@ -120,7 +118,7 @@ static u8 select_channel(void) { u8 chan_idx = atomic_inc_return(&iproc_priv.next_chan); - return chan_idx % iproc_priv.spu.num_spu; + return chan_idx % iproc_priv.spu.num_chan; } /** @@ -258,6 +256,44 @@ spu_ablkcipher_tx_sg_create(struct brcm_message *mssg, return 0; } +static int mailbox_send_message(struct brcm_message *mssg, u32 flags, + u8 chan_idx) +{ + int err; + int retry_cnt = 0; + struct device *dev = &(iproc_priv.pdev->dev); + + err = mbox_send_message(iproc_priv.mbox[chan_idx], mssg); + if (flags & CRYPTO_TFM_REQ_MAY_SLEEP) { + while ((err == -ENOBUFS) && (retry_cnt < SPU_MB_RETRY_MAX)) { + /* + * Mailbox queue is full. Since MAY_SLEEP is set, assume + * not in atomic context and we can wait and try again. + */ + retry_cnt++; + usleep_range(MBOX_SLEEP_MIN, MBOX_SLEEP_MAX); + err = mbox_send_message(iproc_priv.mbox[chan_idx], + mssg); + atomic_inc(&iproc_priv.mb_no_spc); + } + } + if (err < 0) { + atomic_inc(&iproc_priv.mb_send_fail); + return err; + } + + /* Check error returned by mailbox controller */ + err = mssg->error; + if (unlikely(err < 0)) { + dev_err(dev, "message error %d", err); + /* Signal txdone for mailbox channel */ + } + + /* Signal txdone for mailbox channel */ + mbox_client_txdone(iproc_priv.mbox[chan_idx], err); + return err; +} + /** * handle_ablkcipher_req() - Submit as much of a block cipher request as fits in * a single SPU request message, starting at the current position in the request @@ -295,7 +331,6 @@ static int handle_ablkcipher_req(struct iproc_reqctx_s *rctx) u32 pad_len; /* total length of all padding */ bool update_key = false; struct brcm_message *mssg; /* mailbox message */ - int retry_cnt = 0; /* number of entries in src and dst sg in mailbox message. */ u8 rx_frag_num = 2; /* response header and STATUS */ @@ -464,24 +499,9 @@ static int handle_ablkcipher_req(struct iproc_reqctx_s *rctx) if (err) return err; - err = mbox_send_message(iproc_priv.mbox[rctx->chan_idx], mssg); - if (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) { - while ((err == -ENOBUFS) && (retry_cnt < SPU_MB_RETRY_MAX)) { - /* - * Mailbox queue is full. Since MAY_SLEEP is set, assume - * not in atomic context and we can wait and try again. - */ - retry_cnt++; - usleep_range(MBOX_SLEEP_MIN, MBOX_SLEEP_MAX); - err = mbox_send_message(iproc_priv.mbox[rctx->chan_idx], - mssg); - atomic_inc(&iproc_priv.mb_no_spc); - } - } - if (unlikely(err < 0)) { - atomic_inc(&iproc_priv.mb_send_fail); + err = mailbox_send_message(mssg, req->base.flags, rctx->chan_idx); + if (unlikely(err < 0)) return err; - } return -EINPROGRESS; } @@ -712,7 +732,6 @@ static int handle_ahash_req(struct iproc_reqctx_s *rctx) u32 spu_hdr_len; unsigned int digestsize; u16 rem = 0; - int retry_cnt = 0; /* * number of entries in src and dst sg. Always includes SPU msg header. @@ -906,24 +925,10 @@ static int handle_ahash_req(struct iproc_reqctx_s *rctx) if (err) return err; - err = mbox_send_message(iproc_priv.mbox[rctx->chan_idx], mssg); - if (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) { - while ((err == -ENOBUFS) && (retry_cnt < SPU_MB_RETRY_MAX)) { - /* - * Mailbox queue is full. Since MAY_SLEEP is set, assume - * not in atomic context and we can wait and try again. - */ - retry_cnt++; - usleep_range(MBOX_SLEEP_MIN, MBOX_SLEEP_MAX); - err = mbox_send_message(iproc_priv.mbox[rctx->chan_idx], - mssg); - atomic_inc(&iproc_priv.mb_no_spc); - } - } - if (err < 0) { - atomic_inc(&iproc_priv.mb_send_fail); + err = mailbox_send_message(mssg, req->base.flags, rctx->chan_idx); + if (unlikely(err < 0)) return err; - } + return -EINPROGRESS; } @@ -1322,7 +1327,6 @@ static int handle_aead_req(struct iproc_reqctx_s *rctx) int assoc_nents = 0; bool incl_icv = false; unsigned int digestsize = ctx->digestsize; - int retry_cnt = 0; /* number of entries in src and dst sg. Always includes SPU msg header. */ @@ -1369,11 +1373,11 @@ static int handle_aead_req(struct iproc_reqctx_s *rctx) * expects AAD to include just SPI and seqno. So * subtract off the IV len. */ - aead_parms.assoc_size -= GCM_ESP_IV_SIZE; + aead_parms.assoc_size -= GCM_RFC4106_IV_SIZE; if (rctx->is_encrypt) { aead_parms.return_iv = true; - aead_parms.ret_iv_len = GCM_ESP_IV_SIZE; + aead_parms.ret_iv_len = GCM_RFC4106_IV_SIZE; aead_parms.ret_iv_off = GCM_ESP_SALT_SIZE; } } else { @@ -1560,24 +1564,9 @@ static int handle_aead_req(struct iproc_reqctx_s *rctx) if (err) return err; - err = mbox_send_message(iproc_priv.mbox[rctx->chan_idx], mssg); - if (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) { - while ((err == -ENOBUFS) && (retry_cnt < SPU_MB_RETRY_MAX)) { - /* - * Mailbox queue is full. Since MAY_SLEEP is set, assume - * not in atomic context and we can wait and try again. - */ - retry_cnt++; - usleep_range(MBOX_SLEEP_MIN, MBOX_SLEEP_MAX); - err = mbox_send_message(iproc_priv.mbox[rctx->chan_idx], - mssg); - atomic_inc(&iproc_priv.mb_no_spc); - } - } - if (err < 0) { - atomic_inc(&iproc_priv.mb_send_fail); + err = mailbox_send_message(mssg, req->base.flags, rctx->chan_idx); + if (unlikely(err < 0)) return err; - } return -EINPROGRESS; } @@ -3257,7 +3246,7 @@ static struct iproc_alg_s driver_algs[] = { .cra_flags = CRYPTO_ALG_NEED_FALLBACK }, .setkey = aead_gcm_esp_setkey, - .ivsize = GCM_ESP_IV_SIZE, + .ivsize = GCM_RFC4106_IV_SIZE, .maxauthsize = AES_BLOCK_SIZE, }, .cipher_info = { @@ -3303,7 +3292,7 @@ static struct iproc_alg_s driver_algs[] = { .cra_flags = CRYPTO_ALG_NEED_FALLBACK }, .setkey = rfc4543_gcm_esp_setkey, - .ivsize = GCM_ESP_IV_SIZE, + .ivsize = GCM_RFC4106_IV_SIZE, .maxauthsize = AES_BLOCK_SIZE, }, .cipher_info = { @@ -4528,35 +4517,48 @@ static void spu_functions_register(struct device *dev, */ static int spu_mb_init(struct device *dev) { - struct mbox_client *mcl = &iproc_priv.mcl[iproc_priv.spu.num_spu]; - int err; + struct mbox_client *mcl = &iproc_priv.mcl; + int err, i; + + iproc_priv.mbox = devm_kcalloc(dev, iproc_priv.spu.num_chan, + sizeof(struct mbox_chan *), GFP_KERNEL); + if (!iproc_priv.mbox) + return -ENOMEM; mcl->dev = dev; mcl->tx_block = false; mcl->tx_tout = 0; - mcl->knows_txdone = false; + mcl->knows_txdone = true; mcl->rx_callback = spu_rx_callback; mcl->tx_done = NULL; - iproc_priv.mbox[iproc_priv.spu.num_spu] = - mbox_request_channel(mcl, 0); - if (IS_ERR(iproc_priv.mbox[iproc_priv.spu.num_spu])) { - err = (int)PTR_ERR(iproc_priv.mbox[iproc_priv.spu.num_spu]); - dev_err(dev, - "Mbox channel %d request failed with err %d", - iproc_priv.spu.num_spu, err); - iproc_priv.mbox[iproc_priv.spu.num_spu] = NULL; - return err; + for (i = 0; i < iproc_priv.spu.num_chan; i++) { + iproc_priv.mbox[i] = mbox_request_channel(mcl, i); + if (IS_ERR(iproc_priv.mbox[i])) { + err = (int)PTR_ERR(iproc_priv.mbox[i]); + dev_err(dev, + "Mbox channel %d request failed with err %d", + i, err); + iproc_priv.mbox[i] = NULL; + goto free_channels; + } } return 0; +free_channels: + for (i = 0; i < iproc_priv.spu.num_chan; i++) { + if (iproc_priv.mbox[i]) + mbox_free_channel(iproc_priv.mbox[i]); + } + + return err; } static void spu_mb_release(struct platform_device *pdev) { int i; - for (i = 0; i < iproc_priv.spu.num_spu; i++) + for (i = 0; i < iproc_priv.spu.num_chan; i++) mbox_free_channel(iproc_priv.mbox[i]); } @@ -4567,7 +4569,7 @@ static void spu_counters_init(void) atomic_set(&iproc_priv.session_count, 0); atomic_set(&iproc_priv.stream_count, 0); - atomic_set(&iproc_priv.next_chan, (int)iproc_priv.spu.num_spu); + atomic_set(&iproc_priv.next_chan, (int)iproc_priv.spu.num_chan); atomic64_set(&iproc_priv.bytes_in, 0); atomic64_set(&iproc_priv.bytes_out, 0); for (i = 0; i < SPU_OP_NUM; i++) { @@ -4807,49 +4809,37 @@ static int spu_dt_read(struct platform_device *pdev) struct device *dev = &pdev->dev; struct spu_hw *spu = &iproc_priv.spu; struct resource *spu_ctrl_regs; - const struct of_device_id *match; const struct spu_type_subtype *matched_spu_type; - void __iomem *spu_reg_vbase[MAX_SPUS]; - int err; + struct device_node *dn = pdev->dev.of_node; + int err, i; - match = of_match_device(of_match_ptr(bcm_spu_dt_ids), dev); - matched_spu_type = match->data; + /* Count number of mailbox channels */ + spu->num_chan = of_count_phandle_with_args(dn, "mboxes", "#mbox-cells"); - if (iproc_priv.spu.num_spu > 1) { - /* If this is 2nd or later SPU, make sure it's same type */ - if ((spu->spu_type != matched_spu_type->type) || - (spu->spu_subtype != matched_spu_type->subtype)) { - err = -EINVAL; - dev_err(&pdev->dev, "Multiple SPU types not allowed"); - return err; - } - } else { - /* Record type of first SPU */ - spu->spu_type = matched_spu_type->type; - spu->spu_subtype = matched_spu_type->subtype; + matched_spu_type = of_device_get_match_data(dev); + if (!matched_spu_type) { + dev_err(&pdev->dev, "Failed to match device\n"); + return -ENODEV; } - /* Get and map SPU registers */ - spu_ctrl_regs = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!spu_ctrl_regs) { - err = -EINVAL; - dev_err(&pdev->dev, "Invalid/missing registers for SPU\n"); - return err; - } - - spu_reg_vbase[iproc_priv.spu.num_spu] = - devm_ioremap_resource(dev, spu_ctrl_regs); - if (IS_ERR(spu_reg_vbase[iproc_priv.spu.num_spu])) { - err = PTR_ERR(spu_reg_vbase[iproc_priv.spu.num_spu]); - dev_err(&pdev->dev, "Failed to map registers: %d\n", - err); - spu_reg_vbase[iproc_priv.spu.num_spu] = NULL; - return err; - } + spu->spu_type = matched_spu_type->type; + spu->spu_subtype = matched_spu_type->subtype; - dev_dbg(dev, "SPU %d detected.", iproc_priv.spu.num_spu); + i = 0; + for (i = 0; (i < MAX_SPUS) && ((spu_ctrl_regs = + platform_get_resource(pdev, IORESOURCE_MEM, i)) != NULL); i++) { - spu->reg_vbase[iproc_priv.spu.num_spu] = spu_reg_vbase; + spu->reg_vbase[i] = devm_ioremap_resource(dev, spu_ctrl_regs); + if (IS_ERR(spu->reg_vbase[i])) { + err = PTR_ERR(spu->reg_vbase[i]); + dev_err(&pdev->dev, "Failed to map registers: %d\n", + err); + spu->reg_vbase[i] = NULL; + return err; + } + } + spu->num_spu = i; + dev_dbg(dev, "Device has %d SPUs", spu->num_spu); return 0; } @@ -4860,8 +4850,8 @@ int bcm_spu_probe(struct platform_device *pdev) struct spu_hw *spu = &iproc_priv.spu; int err = 0; - iproc_priv.pdev[iproc_priv.spu.num_spu] = pdev; - platform_set_drvdata(iproc_priv.pdev[iproc_priv.spu.num_spu], + iproc_priv.pdev = pdev; + platform_set_drvdata(iproc_priv.pdev, &iproc_priv); err = spu_dt_read(pdev); @@ -4872,12 +4862,6 @@ int bcm_spu_probe(struct platform_device *pdev) if (err < 0) goto failure; - iproc_priv.spu.num_spu++; - - /* If already initialized, we've just added another SPU and are done */ - if (iproc_priv.inited) - return 0; - if (spu->spu_type == SPU_TYPE_SPUM) iproc_priv.bcm_hdr_len = 8; else if (spu->spu_type == SPU_TYPE_SPU2) @@ -4893,8 +4877,6 @@ int bcm_spu_probe(struct platform_device *pdev) if (err < 0) goto fail_reg; - iproc_priv.inited = true; - return 0; fail_reg: diff --git a/drivers/crypto/bcm/cipher.h b/drivers/crypto/bcm/cipher.h index 51dca529ce8f..763c425c41ca 100644 --- a/drivers/crypto/bcm/cipher.h +++ b/drivers/crypto/bcm/cipher.h @@ -23,6 +23,7 @@ #include <crypto/aes.h> #include <crypto/internal/hash.h> #include <crypto/aead.h> +#include <crypto/gcm.h> #include <crypto/sha.h> #include <crypto/sha3.h> @@ -39,8 +40,6 @@ #define ARC4_STATE_SIZE 4 #define CCM_AES_IV_SIZE 16 -#define GCM_AES_IV_SIZE 12 -#define GCM_ESP_IV_SIZE 8 #define CCM_ESP_IV_SIZE 8 #define RFC4543_ICV_SIZE 16 @@ -427,10 +426,13 @@ struct spu_hw { /* The number of SPUs on this platform */ u32 num_spu; + + /* The number of SPU channels on this platform */ + u32 num_chan; }; struct device_private { - struct platform_device *pdev[MAX_SPUS]; + struct platform_device *pdev; struct spu_hw spu; @@ -470,12 +472,10 @@ struct device_private { /* Number of ICV check failures for AEAD messages */ atomic_t bad_icv; - struct mbox_client mcl[MAX_SPUS]; - /* Array of mailbox channel pointers, one for each channel */ - struct mbox_chan *mbox[MAX_SPUS]; + struct mbox_client mcl; - /* Driver initialized */ - bool inited; + /* Array of mailbox channel pointers, one for each channel */ + struct mbox_chan **mbox; }; extern struct device_private iproc_priv; diff --git a/drivers/crypto/bcm/spu2.c b/drivers/crypto/bcm/spu2.c index ef04c9748317..bf7ac621c591 100644 --- a/drivers/crypto/bcm/spu2.c +++ b/drivers/crypto/bcm/spu2.c @@ -302,6 +302,7 @@ spu2_hash_xlate(enum hash_alg hash_alg, enum hash_mode hash_mode, break; case HASH_ALG_SHA3_512: *spu2_type = SPU2_HASH_TYPE_SHA3_512; + break; case HASH_ALG_LAST: default: err = -EINVAL; diff --git a/drivers/crypto/bcm/util.c b/drivers/crypto/bcm/util.c index 430c5570ea87..d543c010ccd9 100644 --- a/drivers/crypto/bcm/util.c +++ b/drivers/crypto/bcm/util.c @@ -271,7 +271,7 @@ int do_shash(unsigned char *name, unsigned char *result, hash = crypto_alloc_shash(name, 0, 0); if (IS_ERR(hash)) { rc = PTR_ERR(hash); - pr_err("%s: Crypto %s allocation error %d", __func__, name, rc); + pr_err("%s: Crypto %s allocation error %d\n", __func__, name, rc); return rc; } @@ -279,7 +279,7 @@ int do_shash(unsigned char *name, unsigned char *result, sdesc = kmalloc(size, GFP_KERNEL); if (!sdesc) { rc = -ENOMEM; - pr_err("%s: Memory allocation failure", __func__); + pr_err("%s: Memory allocation failure\n", __func__); goto do_shash_err; } sdesc->shash.tfm = hash; @@ -288,31 +288,31 @@ int do_shash(unsigned char *name, unsigned char *result, if (key_len > 0) { rc = crypto_shash_setkey(hash, key, key_len); if (rc) { - pr_err("%s: Could not setkey %s shash", __func__, name); + pr_err("%s: Could not setkey %s shash\n", __func__, name); goto do_shash_err; } } rc = crypto_shash_init(&sdesc->shash); if (rc) { - pr_err("%s: Could not init %s shash", __func__, name); + pr_err("%s: Could not init %s shash\n", __func__, name); goto do_shash_err; } rc = crypto_shash_update(&sdesc->shash, data1, data1_len); if (rc) { - pr_err("%s: Could not update1", __func__); + pr_err("%s: Could not update1\n", __func__); goto do_shash_err; } if (data2 && data2_len) { rc = crypto_shash_update(&sdesc->shash, data2, data2_len); if (rc) { - pr_err("%s: Could not update2", __func__); + pr_err("%s: Could not update2\n", __func__); goto do_shash_err; } } rc = crypto_shash_final(&sdesc->shash, result); if (rc) - pr_err("%s: Could not generate %s hash", __func__, name); + pr_err("%s: Could not generate %s hash\n", __func__, name); do_shash_err: crypto_free_shash(hash); diff --git a/drivers/crypto/caam/Kconfig b/drivers/crypto/caam/Kconfig index e36aeacd7635..1eb852765469 100644 --- a/drivers/crypto/caam/Kconfig +++ b/drivers/crypto/caam/Kconfig @@ -1,6 +1,7 @@ config CRYPTO_DEV_FSL_CAAM tristate "Freescale CAAM-Multicore driver backend" depends on FSL_SOC || ARCH_MXC || ARCH_LAYERSCAPE + select SOC_BUS help Enables the driver module for Freescale's Cryptographic Accelerator and Assurance Module (CAAM), also known as the SEC version 4 (SEC4). @@ -141,10 +142,6 @@ config CRYPTO_DEV_FSL_CAAM_RNG_API To compile this as a module, choose M here: the module will be called caamrng. -config CRYPTO_DEV_FSL_CAAM_IMX - def_bool SOC_IMX6 || SOC_IMX7D - depends on CRYPTO_DEV_FSL_CAAM - config CRYPTO_DEV_FSL_CAAM_DEBUG bool "Enable debug output in CAAM driver" depends on CRYPTO_DEV_FSL_CAAM diff --git a/drivers/crypto/caam/Makefile b/drivers/crypto/caam/Makefile index 9e2e98856b9b..cb652ee7dfc8 100644 --- a/drivers/crypto/caam/Makefile +++ b/drivers/crypto/caam/Makefile @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0 # # Makefile for the CAAM backend and dependent components # diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index 0488b7f81dcf..baa8dd52472d 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -81,40 +81,6 @@ #define debug(format, arg...) #endif -#ifdef DEBUG -#include <linux/highmem.h> - -static void dbg_dump_sg(const char *level, const char *prefix_str, - int prefix_type, int rowsize, int groupsize, - struct scatterlist *sg, size_t tlen, bool ascii) -{ - struct scatterlist *it; - void *it_page; - size_t len; - void *buf; - - for (it = sg; it != NULL && tlen > 0 ; it = sg_next(sg)) { - /* - * make sure the scatterlist's page - * has a valid virtual memory mapping - */ - it_page = kmap_atomic(sg_page(it)); - if (unlikely(!it_page)) { - printk(KERN_ERR "dbg_dump_sg: kmap failed\n"); - return; - } - - buf = it_page + it->offset; - len = min_t(size_t, tlen, it->length); - print_hex_dump(level, prefix_str, prefix_type, rowsize, - groupsize, buf, len, ascii); - tlen -= len; - - kunmap_atomic(it_page); - } -} -#endif - static struct list_head alg_list; struct caam_alg_entry { @@ -898,10 +864,10 @@ static void ablkcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err, print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, req->info, edesc->src_nents > 1 ? 100 : ivsize, 1); - dbg_dump_sg(KERN_ERR, "dst @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->dst, - edesc->dst_nents > 1 ? 100 : req->nbytes, 1); #endif + caam_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->dst, + edesc->dst_nents > 1 ? 100 : req->nbytes, 1); ablkcipher_unmap(jrdev, edesc, req); @@ -937,10 +903,10 @@ static void ablkcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err, print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, req->info, ivsize, 1); - dbg_dump_sg(KERN_ERR, "dst @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->dst, - edesc->dst_nents > 1 ? 100 : req->nbytes, 1); #endif + caam_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->dst, + edesc->dst_nents > 1 ? 100 : req->nbytes, 1); ablkcipher_unmap(jrdev, edesc, req); @@ -1026,7 +992,7 @@ static void init_gcm_job(struct aead_request *req, struct caam_ctx *ctx = crypto_aead_ctx(aead); unsigned int ivsize = crypto_aead_ivsize(aead); u32 *desc = edesc->hw_desc; - bool generic_gcm = (ivsize == 12); + bool generic_gcm = (ivsize == GCM_AES_IV_SIZE); unsigned int last; init_aead_job(req, edesc, all_contig, encrypt); @@ -1038,7 +1004,7 @@ static void init_gcm_job(struct aead_request *req, /* Read GCM IV */ append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE | - FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | 12 | last); + FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | GCM_AES_IV_SIZE | last); /* Append Salt */ if (!generic_gcm) append_data(desc, ctx->key + ctx->cdata.keylen, 4); @@ -1107,10 +1073,10 @@ static void init_ablkcipher_job(u32 *sh_desc, dma_addr_t ptr, ivsize, 1); pr_err("asked=%d, nbytes%d\n", (int)edesc->src_nents > 1 ? 100 : req->nbytes, req->nbytes); - dbg_dump_sg(KERN_ERR, "src @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->src, - edesc->src_nents > 1 ? 100 : req->nbytes, 1); #endif + caam_dump_sg(KERN_ERR, "src @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->src, + edesc->src_nents > 1 ? 100 : req->nbytes, 1); len = desc_len(sh_desc); init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); @@ -1164,10 +1130,10 @@ static void init_ablkcipher_giv_job(u32 *sh_desc, dma_addr_t ptr, print_hex_dump(KERN_ERR, "presciv@" __stringify(__LINE__) ": ", DUMP_PREFIX_ADDRESS, 16, 4, req->info, ivsize, 1); - dbg_dump_sg(KERN_ERR, "src @" __stringify(__LINE__) ": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->src, - edesc->src_nents > 1 ? 100 : req->nbytes, 1); #endif + caam_dump_sg(KERN_ERR, "src @" __stringify(__LINE__) ": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->src, + edesc->src_nents > 1 ? 100 : req->nbytes, 1); len = desc_len(sh_desc); init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); @@ -1449,11 +1415,9 @@ static int aead_decrypt(struct aead_request *req) u32 *desc; int ret = 0; -#ifdef DEBUG - dbg_dump_sg(KERN_ERR, "dec src@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->src, - req->assoclen + req->cryptlen, 1); -#endif + caam_dump_sg(KERN_ERR, "dec src@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->src, + req->assoclen + req->cryptlen, 1); /* allocate extended descriptor */ edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN, @@ -1989,7 +1953,7 @@ static struct caam_aead_alg driver_aeads[] = { .setauthsize = rfc4106_setauthsize, .encrypt = ipsec_gcm_encrypt, .decrypt = ipsec_gcm_decrypt, - .ivsize = 8, + .ivsize = GCM_RFC4106_IV_SIZE, .maxauthsize = AES_BLOCK_SIZE, }, .caam = { @@ -2007,7 +1971,7 @@ static struct caam_aead_alg driver_aeads[] = { .setauthsize = rfc4543_setauthsize, .encrypt = ipsec_gcm_encrypt, .decrypt = ipsec_gcm_decrypt, - .ivsize = 8, + .ivsize = GCM_RFC4543_IV_SIZE, .maxauthsize = AES_BLOCK_SIZE, }, .caam = { @@ -2026,7 +1990,7 @@ static struct caam_aead_alg driver_aeads[] = { .setauthsize = gcm_setauthsize, .encrypt = gcm_encrypt, .decrypt = gcm_decrypt, - .ivsize = 12, + .ivsize = GCM_AES_IV_SIZE, .maxauthsize = AES_BLOCK_SIZE, }, .caam = { diff --git a/drivers/crypto/caam/caamalg_desc.c b/drivers/crypto/caam/caamalg_desc.c index 6f9c7ec0e339..530c14ee32de 100644 --- a/drivers/crypto/caam/caamalg_desc.c +++ b/drivers/crypto/caam/caamalg_desc.c @@ -599,7 +599,7 @@ void cnstr_shdsc_gcm_encap(u32 * const desc, struct alginfo *cdata, /* skip key loading if they are loaded due to sharing */ key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | - JUMP_COND_SHRD | JUMP_COND_SELF); + JUMP_COND_SHRD); if (cdata->key_inline) append_key_as_imm(desc, cdata->key_virt, cdata->keylen, cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); @@ -688,8 +688,7 @@ void cnstr_shdsc_gcm_decap(u32 * const desc, struct alginfo *cdata, /* skip key loading if they are loaded due to sharing */ key_jump_cmd = append_jump(desc, JUMP_JSL | - JUMP_TEST_ALL | JUMP_COND_SHRD | - JUMP_COND_SELF); + JUMP_TEST_ALL | JUMP_COND_SHRD); if (cdata->key_inline) append_key_as_imm(desc, cdata->key_virt, cdata->keylen, cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); diff --git a/drivers/crypto/caam/caamalg_desc.h b/drivers/crypto/caam/caamalg_desc.h index 8731e4a7ff05..e412ec8f7005 100644 --- a/drivers/crypto/caam/caamalg_desc.h +++ b/drivers/crypto/caam/caamalg_desc.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Shared descriptors for aead, ablkcipher algorithms * diff --git a/drivers/crypto/caam/caamalg_qi.c b/drivers/crypto/caam/caamalg_qi.c index 78c4c0485c58..f9f08fce4356 100644 --- a/drivers/crypto/caam/caamalg_qi.c +++ b/drivers/crypto/caam/caamalg_qi.c @@ -7,12 +7,11 @@ */ #include "compat.h" - +#include "ctrl.h" #include "regs.h" #include "intern.h" #include "desc_constr.h" #include "error.h" -#include "sg_sw_sec4.h" #include "sg_sw_qm.h" #include "key_gen.h" #include "qi.h" @@ -399,6 +398,7 @@ badkey: * @iv_dma: dma address of iv for checking continuity and link table * @qm_sg_bytes: length of dma mapped h/w link table * @qm_sg_dma: bus physical mapped address of h/w link table + * @assoclen: associated data length, in CAAM endianness * @assoclen_dma: bus physical mapped address of req->assoclen * @drv_req: driver-specific request structure * @sgt: the h/w link table @@ -409,8 +409,12 @@ struct aead_edesc { dma_addr_t iv_dma; int qm_sg_bytes; dma_addr_t qm_sg_dma; + unsigned int assoclen; dma_addr_t assoclen_dma; struct caam_drv_req drv_req; +#define CAAM_QI_MAX_AEAD_SG \ + ((CAAM_QI_MEMCACHE_SIZE - offsetof(struct aead_edesc, sgt)) / \ + sizeof(struct qm_sg_entry)) struct qm_sg_entry sgt[0]; }; @@ -431,6 +435,9 @@ struct ablkcipher_edesc { int qm_sg_bytes; dma_addr_t qm_sg_dma; struct caam_drv_req drv_req; +#define CAAM_QI_MAX_ABLKCIPHER_SG \ + ((CAAM_QI_MEMCACHE_SIZE - offsetof(struct ablkcipher_edesc, sgt)) / \ + sizeof(struct qm_sg_entry)) struct qm_sg_entry sgt[0]; }; @@ -660,6 +667,14 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, */ qm_sg_ents = 1 + !!ivsize + mapped_src_nents + (mapped_dst_nents > 1 ? mapped_dst_nents : 0); + if (unlikely(qm_sg_ents > CAAM_QI_MAX_AEAD_SG)) { + dev_err(qidev, "Insufficient S/G entries: %d > %lu\n", + qm_sg_ents, CAAM_QI_MAX_AEAD_SG); + caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, + iv_dma, ivsize, op_type, 0, 0); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } sg_table = &edesc->sgt[0]; qm_sg_bytes = qm_sg_ents * sizeof(*sg_table); @@ -670,7 +685,8 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, edesc->drv_req.cbk = aead_done; edesc->drv_req.drv_ctx = drv_ctx; - edesc->assoclen_dma = dma_map_single(qidev, &req->assoclen, 4, + edesc->assoclen = cpu_to_caam32(req->assoclen); + edesc->assoclen_dma = dma_map_single(qidev, &edesc->assoclen, 4, DMA_TO_DEVICE); if (dma_mapping_error(qidev, edesc->assoclen_dma)) { dev_err(qidev, "unable to map assoclen\n"); @@ -776,9 +792,9 @@ static void ablkcipher_done(struct caam_drv_req *drv_req, u32 status) struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); struct caam_ctx *caam_ctx = crypto_ablkcipher_ctx(ablkcipher); struct device *qidev = caam_ctx->qidev; -#ifdef DEBUG int ivsize = crypto_ablkcipher_ivsize(ablkcipher); +#ifdef DEBUG dev_err(qidev, "%s %d: status 0x%x\n", __func__, __LINE__, status); #endif @@ -791,14 +807,21 @@ static void ablkcipher_done(struct caam_drv_req *drv_req, u32 status) print_hex_dump(KERN_ERR, "dstiv @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, req->info, edesc->src_nents > 1 ? 100 : ivsize, 1); - dbg_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->dst, - edesc->dst_nents > 1 ? 100 : req->nbytes, 1); + caam_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->dst, + edesc->dst_nents > 1 ? 100 : req->nbytes, 1); #endif ablkcipher_unmap(qidev, edesc, req); qi_cache_free(edesc); + /* + * The crypto API expects us to set the IV (req->info) to the last + * ciphertext block. This is used e.g. by the CTS mode. + */ + scatterwalk_map_and_copy(req->info, req->dst, req->nbytes - ivsize, + ivsize, 0); + ablkcipher_request_complete(req, status); } @@ -880,6 +903,15 @@ static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request } dst_sg_idx = qm_sg_ents; + qm_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0; + if (unlikely(qm_sg_ents > CAAM_QI_MAX_ABLKCIPHER_SG)) { + dev_err(qidev, "Insufficient S/G entries: %d > %lu\n", + qm_sg_ents, CAAM_QI_MAX_ABLKCIPHER_SG); + caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, + iv_dma, ivsize, op_type, 0, 0); + return ERR_PTR(-ENOMEM); + } + /* allocate space for base edesc and link tables */ edesc = qi_cache_alloc(GFP_DMA | flags); if (unlikely(!edesc)) { @@ -892,7 +924,6 @@ static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request edesc->src_nents = src_nents; edesc->dst_nents = dst_nents; edesc->iv_dma = iv_dma; - qm_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0; sg_table = &edesc->sgt[0]; edesc->qm_sg_bytes = qm_sg_ents * sizeof(*sg_table); edesc->drv_req.app_ctx = req; @@ -1026,6 +1057,14 @@ static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc( qm_sg_ents += 1 + mapped_dst_nents; } + if (unlikely(qm_sg_ents > CAAM_QI_MAX_ABLKCIPHER_SG)) { + dev_err(qidev, "Insufficient S/G entries: %d > %lu\n", + qm_sg_ents, CAAM_QI_MAX_ABLKCIPHER_SG); + caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, + iv_dma, ivsize, GIVENCRYPT, 0, 0); + return ERR_PTR(-ENOMEM); + } + /* allocate space for base edesc and link tables */ edesc = qi_cache_alloc(GFP_DMA | flags); if (!edesc) { @@ -1968,7 +2007,7 @@ static struct caam_aead_alg driver_aeads[] = { .cra_name = "echainiv(authenc(hmac(sha256)," "cbc(des)))", .cra_driver_name = "echainiv-authenc-" - "hmac-sha256-cbc-desi-" + "hmac-sha256-cbc-des-" "caam-qi", .cra_blocksize = DES_BLOCK_SIZE, }, @@ -2273,6 +2312,11 @@ static int __init caam_qi_algapi_init(void) if (!priv || !priv->qi_present) return -ENODEV; + if (caam_dpaa2) { + dev_info(ctrldev, "caam/qi frontend driver not suitable for DPAA 2.x, aborting...\n"); + return -ENODEV; + } + INIT_LIST_HEAD(&alg_list); /* diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c index 910ec61cae09..616720a04e7a 100644 --- a/drivers/crypto/caam/caamhash.c +++ b/drivers/crypto/caam/caamhash.c @@ -218,7 +218,7 @@ static inline int buf_map_to_sec4_sg(struct device *jrdev, } /* Map state->caam_ctx, and add it to link table */ -static inline int ctx_map_to_sec4_sg(u32 *desc, struct device *jrdev, +static inline int ctx_map_to_sec4_sg(struct device *jrdev, struct caam_hash_state *state, int ctx_len, struct sec4_sg_entry *sec4_sg, u32 flag) { @@ -773,7 +773,7 @@ static int ahash_update_ctx(struct ahash_request *req) edesc->src_nents = src_nents; edesc->sec4_sg_bytes = sec4_sg_bytes; - ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len, + ret = ctx_map_to_sec4_sg(jrdev, state, ctx->ctx_len, edesc->sec4_sg, DMA_BIDIRECTIONAL); if (ret) goto unmap_ctx; @@ -791,8 +791,8 @@ static int ahash_update_ctx(struct ahash_request *req) to_hash - *buflen, *next_buflen, 0); } else { - (edesc->sec4_sg + sec4_sg_src_index - 1)->len |= - cpu_to_caam32(SEC4_SG_LEN_FIN); + sg_to_sec4_set_last(edesc->sec4_sg + sec4_sg_src_index - + 1); } desc = edesc->hw_desc; @@ -871,9 +871,8 @@ static int ahash_final_ctx(struct ahash_request *req) desc = edesc->hw_desc; edesc->sec4_sg_bytes = sec4_sg_bytes; - edesc->src_nents = 0; - ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len, + ret = ctx_map_to_sec4_sg(jrdev, state, ctx->ctx_len, edesc->sec4_sg, DMA_TO_DEVICE); if (ret) goto unmap_ctx; @@ -882,8 +881,7 @@ static int ahash_final_ctx(struct ahash_request *req) if (ret) goto unmap_ctx; - (edesc->sec4_sg + sec4_sg_src_index - 1)->len |= - cpu_to_caam32(SEC4_SG_LEN_FIN); + sg_to_sec4_set_last(edesc->sec4_sg + sec4_sg_src_index - 1); edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, sec4_sg_bytes, DMA_TO_DEVICE); @@ -968,7 +966,7 @@ static int ahash_finup_ctx(struct ahash_request *req) edesc->src_nents = src_nents; - ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len, + ret = ctx_map_to_sec4_sg(jrdev, state, ctx->ctx_len, edesc->sec4_sg, DMA_TO_DEVICE); if (ret) goto unmap_ctx; @@ -1124,7 +1122,6 @@ static int ahash_final_no_ctx(struct ahash_request *req) dev_err(jrdev, "unable to map dst\n"); goto unmap; } - edesc->src_nents = 0; #ifdef DEBUG print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", @@ -1206,7 +1203,6 @@ static int ahash_update_no_ctx(struct ahash_request *req) edesc->src_nents = src_nents; edesc->sec4_sg_bytes = sec4_sg_bytes; - edesc->dst_dma = 0; ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, state); if (ret) @@ -1418,7 +1414,6 @@ static int ahash_update_first(struct ahash_request *req) } edesc->src_nents = src_nents; - edesc->dst_dma = 0; ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 0, 0, to_hash); diff --git a/drivers/crypto/caam/caampkc.h b/drivers/crypto/caam/caampkc.h index 87ab75e9df43..fd145c46eae1 100644 --- a/drivers/crypto/caam/caampkc.h +++ b/drivers/crypto/caam/caampkc.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * caam - Freescale FSL CAAM support for Public Key Cryptography descriptors * diff --git a/drivers/crypto/caam/caamrng.c b/drivers/crypto/caam/caamrng.c index 41398da3edf4..fde07d4ff019 100644 --- a/drivers/crypto/caam/caamrng.c +++ b/drivers/crypto/caam/caamrng.c @@ -285,11 +285,7 @@ static int caam_init_rng(struct caam_rng_ctx *ctx, struct device *jrdev) if (err) return err; - err = caam_init_buf(ctx, 1); - if (err) - return err; - - return 0; + return caam_init_buf(ctx, 1); } static struct hwrng caam_rng = { diff --git a/drivers/crypto/caam/compat.h b/drivers/crypto/caam/compat.h index 7149cd2492e0..1c71e0cd5098 100644 --- a/drivers/crypto/caam/compat.h +++ b/drivers/crypto/caam/compat.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright 2008-2011 Freescale Semiconductor, Inc. */ @@ -31,6 +32,7 @@ #include <crypto/aes.h> #include <crypto/ctr.h> #include <crypto/des.h> +#include <crypto/gcm.h> #include <crypto/sha.h> #include <crypto/md5.h> #include <crypto/internal/aead.h> diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c index dd353e342c12..027e121c6f70 100644 --- a/drivers/crypto/caam/ctrl.c +++ b/drivers/crypto/caam/ctrl.c @@ -7,6 +7,7 @@ #include <linux/device.h> #include <linux/of_address.h> #include <linux/of_irq.h> +#include <linux/sys_soc.h> #include "compat.h" #include "regs.h" @@ -17,6 +18,10 @@ bool caam_little_end; EXPORT_SYMBOL(caam_little_end); +bool caam_dpaa2; +EXPORT_SYMBOL(caam_dpaa2); +bool caam_imx; +EXPORT_SYMBOL(caam_imx); #ifdef CONFIG_CAAM_QI #include "qi.h" @@ -26,19 +31,11 @@ EXPORT_SYMBOL(caam_little_end); * i.MX targets tend to have clock control subsystems that can * enable/disable clocking to our device. */ -#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_IMX static inline struct clk *caam_drv_identify_clk(struct device *dev, char *clk_name) { - return devm_clk_get(dev, clk_name); + return caam_imx ? devm_clk_get(dev, clk_name) : NULL; } -#else -static inline struct clk *caam_drv_identify_clk(struct device *dev, - char *clk_name) -{ - return NULL; -} -#endif /* * Descriptor to instantiate RNG State Handle 0 in normal mode and @@ -319,8 +316,11 @@ static int caam_remove(struct platform_device *pdev) caam_qi_shutdown(ctrlpriv->qidev); #endif - /* De-initialize RNG state handles initialized by this driver. */ - if (ctrlpriv->rng4_sh_init) + /* + * De-initialize RNG state handles initialized by this driver. + * In case of DPAA 2.x, RNG is managed by MC firmware. + */ + if (!caam_dpaa2 && ctrlpriv->rng4_sh_init) deinstantiate_rng(ctrldev, ctrlpriv->rng4_sh_init); /* Shut down debug views */ @@ -425,6 +425,10 @@ static int caam_probe(struct platform_device *pdev) { int ret, ring, gen_sk, ent_delay = RTSDCTL_ENT_DLY_MIN; u64 caam_id; + static const struct soc_device_attribute imx_soc[] = { + {.family = "Freescale i.MX"}, + {}, + }; struct device *dev; struct device_node *nprop, *np; struct caam_ctrl __iomem *ctrl; @@ -444,9 +448,10 @@ static int caam_probe(struct platform_device *pdev) dev = &pdev->dev; dev_set_drvdata(dev, ctrlpriv); - ctrlpriv->pdev = pdev; nprop = pdev->dev.of_node; + caam_imx = (bool)soc_device_match(imx_soc); + /* Enable clocking */ clk = caam_drv_identify_clk(&pdev->dev, "ipg"); if (IS_ERR(clk)) { @@ -553,12 +558,17 @@ static int caam_probe(struct platform_device *pdev) /* * Enable DECO watchdogs and, if this is a PHYS_ADDR_T_64BIT kernel, - * long pointers in master configuration register + * long pointers in master configuration register. + * In case of DPAA 2.x, Management Complex firmware performs + * the configuration. */ - clrsetbits_32(&ctrl->mcr, MCFGR_AWCACHE_MASK | MCFGR_LONG_PTR, - MCFGR_AWCACHE_CACH | MCFGR_AWCACHE_BUFF | - MCFGR_WDENABLE | MCFGR_LARGE_BURST | - (sizeof(dma_addr_t) == sizeof(u64) ? MCFGR_LONG_PTR : 0)); + caam_dpaa2 = !!(comp_params & CTPR_MS_DPAA2); + if (!caam_dpaa2) + clrsetbits_32(&ctrl->mcr, MCFGR_AWCACHE_MASK | MCFGR_LONG_PTR, + MCFGR_AWCACHE_CACH | MCFGR_AWCACHE_BUFF | + MCFGR_WDENABLE | MCFGR_LARGE_BURST | + (sizeof(dma_addr_t) == sizeof(u64) ? + MCFGR_LONG_PTR : 0)); /* * Read the Compile Time paramters and SCFGR to determine @@ -587,7 +597,9 @@ static int caam_probe(struct platform_device *pdev) JRSTART_JR3_START); if (sizeof(dma_addr_t) == sizeof(u64)) { - if (of_device_is_compatible(nprop, "fsl,sec-v5.0")) + if (caam_dpaa2) + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(49)); + else if (of_device_is_compatible(nprop, "fsl,sec-v5.0")) ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(40)); else ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(36)); @@ -630,11 +642,9 @@ static int caam_probe(struct platform_device *pdev) ring++; } - /* Check to see if QI present. If so, enable */ - ctrlpriv->qi_present = - !!(rd_reg32(&ctrl->perfmon.comp_parms_ms) & - CTPR_MS_QI_MASK); - if (ctrlpriv->qi_present) { + /* Check to see if (DPAA 1.x) QI present. If so, enable */ + ctrlpriv->qi_present = !!(comp_params & CTPR_MS_QI_MASK); + if (ctrlpriv->qi_present && !caam_dpaa2) { ctrlpriv->qi = (struct caam_queue_if __iomem __force *) ((__force uint8_t *)ctrl + BLOCK_OFFSET * QI_BLOCK_NUMBER @@ -662,8 +672,10 @@ static int caam_probe(struct platform_device *pdev) /* * If SEC has RNG version >= 4 and RNG state handle has not been * already instantiated, do RNG instantiation + * In case of DPAA 2.x, RNG is managed by MC firmware. */ - if ((cha_vid_ls & CHA_ID_LS_RNG_MASK) >> CHA_ID_LS_RNG_SHIFT >= 4) { + if (!caam_dpaa2 && + (cha_vid_ls & CHA_ID_LS_RNG_MASK) >> CHA_ID_LS_RNG_SHIFT >= 4) { ctrlpriv->rng4_sh_init = rd_reg32(&ctrl->r4tst[0].rdsta); /* @@ -731,63 +743,43 @@ static int caam_probe(struct platform_device *pdev) /* Report "alive" for developer to see */ dev_info(dev, "device ID = 0x%016llx (Era %d)\n", caam_id, caam_get_era()); - dev_info(dev, "job rings = %d, qi = %d\n", - ctrlpriv->total_jobrs, ctrlpriv->qi_present); + dev_info(dev, "job rings = %d, qi = %d, dpaa2 = %s\n", + ctrlpriv->total_jobrs, ctrlpriv->qi_present, + caam_dpaa2 ? "yes" : "no"); #ifdef CONFIG_DEBUG_FS - - ctrlpriv->ctl_rq_dequeued = - debugfs_create_file("rq_dequeued", - S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->req_dequeued, - &caam_fops_u64_ro); - ctrlpriv->ctl_ob_enc_req = - debugfs_create_file("ob_rq_encrypted", - S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->ob_enc_req, - &caam_fops_u64_ro); - ctrlpriv->ctl_ib_dec_req = - debugfs_create_file("ib_rq_decrypted", - S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->ib_dec_req, - &caam_fops_u64_ro); - ctrlpriv->ctl_ob_enc_bytes = - debugfs_create_file("ob_bytes_encrypted", - S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->ob_enc_bytes, - &caam_fops_u64_ro); - ctrlpriv->ctl_ob_prot_bytes = - debugfs_create_file("ob_bytes_protected", - S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->ob_prot_bytes, - &caam_fops_u64_ro); - ctrlpriv->ctl_ib_dec_bytes = - debugfs_create_file("ib_bytes_decrypted", - S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->ib_dec_bytes, - &caam_fops_u64_ro); - ctrlpriv->ctl_ib_valid_bytes = - debugfs_create_file("ib_bytes_validated", - S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->ib_valid_bytes, - &caam_fops_u64_ro); + debugfs_create_file("rq_dequeued", S_IRUSR | S_IRGRP | S_IROTH, + ctrlpriv->ctl, &perfmon->req_dequeued, + &caam_fops_u64_ro); + debugfs_create_file("ob_rq_encrypted", S_IRUSR | S_IRGRP | S_IROTH, + ctrlpriv->ctl, &perfmon->ob_enc_req, + &caam_fops_u64_ro); + debugfs_create_file("ib_rq_decrypted", S_IRUSR | S_IRGRP | S_IROTH, + ctrlpriv->ctl, &perfmon->ib_dec_req, + &caam_fops_u64_ro); + debugfs_create_file("ob_bytes_encrypted", S_IRUSR | S_IRGRP | S_IROTH, + ctrlpriv->ctl, &perfmon->ob_enc_bytes, + &caam_fops_u64_ro); + debugfs_create_file("ob_bytes_protected", S_IRUSR | S_IRGRP | S_IROTH, + ctrlpriv->ctl, &perfmon->ob_prot_bytes, + &caam_fops_u64_ro); + debugfs_create_file("ib_bytes_decrypted", S_IRUSR | S_IRGRP | S_IROTH, + ctrlpriv->ctl, &perfmon->ib_dec_bytes, + &caam_fops_u64_ro); + debugfs_create_file("ib_bytes_validated", S_IRUSR | S_IRGRP | S_IROTH, + ctrlpriv->ctl, &perfmon->ib_valid_bytes, + &caam_fops_u64_ro); /* Controller level - global status values */ - ctrlpriv->ctl_faultaddr = - debugfs_create_file("fault_addr", - S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->faultaddr, - &caam_fops_u32_ro); - ctrlpriv->ctl_faultdetail = - debugfs_create_file("fault_detail", - S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->faultdetail, - &caam_fops_u32_ro); - ctrlpriv->ctl_faultstatus = - debugfs_create_file("fault_status", - S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->status, - &caam_fops_u32_ro); + debugfs_create_file("fault_addr", S_IRUSR | S_IRGRP | S_IROTH, + ctrlpriv->ctl, &perfmon->faultaddr, + &caam_fops_u32_ro); + debugfs_create_file("fault_detail", S_IRUSR | S_IRGRP | S_IROTH, + ctrlpriv->ctl, &perfmon->faultdetail, + &caam_fops_u32_ro); + debugfs_create_file("fault_status", S_IRUSR | S_IRGRP | S_IROTH, + ctrlpriv->ctl, &perfmon->status, + &caam_fops_u32_ro); /* Internal covering keys (useful in non-secure mode only) */ ctrlpriv->ctl_kek_wrap.data = (__force void *)&ctrlpriv->ctrl->kek[0]; diff --git a/drivers/crypto/caam/ctrl.h b/drivers/crypto/caam/ctrl.h index cac5402a46eb..be693a2cc25e 100644 --- a/drivers/crypto/caam/ctrl.h +++ b/drivers/crypto/caam/ctrl.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * CAAM control-plane driver backend public-level include definitions * @@ -10,4 +11,6 @@ /* Prototypes for backend-level services exposed to APIs */ int caam_get_era(void); +extern bool caam_dpaa2; + #endif /* CTRL_H */ diff --git a/drivers/crypto/caam/desc.h b/drivers/crypto/caam/desc.h index 2e6766a1573f..8142de7ba050 100644 --- a/drivers/crypto/caam/desc.h +++ b/drivers/crypto/caam/desc.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * CAAM descriptor composition header * Definitions to support CAAM descriptor instruction generation @@ -1439,7 +1440,7 @@ #define MATH_SRC1_REG2 (0x02 << MATH_SRC1_SHIFT) #define MATH_SRC1_REG3 (0x03 << MATH_SRC1_SHIFT) #define MATH_SRC1_IMM (0x04 << MATH_SRC1_SHIFT) -#define MATH_SRC1_DPOVRD (0x07 << MATH_SRC0_SHIFT) +#define MATH_SRC1_DPOVRD (0x07 << MATH_SRC1_SHIFT) #define MATH_SRC1_INFIFO (0x0a << MATH_SRC1_SHIFT) #define MATH_SRC1_OUTFIFO (0x0b << MATH_SRC1_SHIFT) #define MATH_SRC1_ONE (0x0c << MATH_SRC1_SHIFT) diff --git a/drivers/crypto/caam/desc_constr.h b/drivers/crypto/caam/desc_constr.h index d8e83ca104e0..ba1ca0806f0a 100644 --- a/drivers/crypto/caam/desc_constr.h +++ b/drivers/crypto/caam/desc_constr.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * caam descriptor construction helper functions * diff --git a/drivers/crypto/caam/error.c b/drivers/crypto/caam/error.c index 6f44ccb55c63..8da88beb1abb 100644 --- a/drivers/crypto/caam/error.c +++ b/drivers/crypto/caam/error.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * CAAM Error Reporting * @@ -9,6 +10,46 @@ #include "desc.h" #include "error.h" +#ifdef DEBUG +#include <linux/highmem.h> + +void caam_dump_sg(const char *level, const char *prefix_str, int prefix_type, + int rowsize, int groupsize, struct scatterlist *sg, + size_t tlen, bool ascii) +{ + struct scatterlist *it; + void *it_page; + size_t len; + void *buf; + + for (it = sg; it && tlen > 0 ; it = sg_next(sg)) { + /* + * make sure the scatterlist's page + * has a valid virtual memory mapping + */ + it_page = kmap_atomic(sg_page(it)); + if (unlikely(!it_page)) { + pr_err("caam_dump_sg: kmap failed\n"); + return; + } + + buf = it_page + it->offset; + len = min_t(size_t, tlen, it->length); + print_hex_dump(level, prefix_str, prefix_type, rowsize, + groupsize, buf, len, ascii); + tlen -= len; + + kunmap_atomic(it_page); + } +} +#else +void caam_dump_sg(const char *level, const char *prefix_str, int prefix_type, + int rowsize, int groupsize, struct scatterlist *sg, + size_t tlen, bool ascii) +{} +#endif /* DEBUG */ +EXPORT_SYMBOL(caam_dump_sg); + static const struct { u8 value; const char *error_text; diff --git a/drivers/crypto/caam/error.h b/drivers/crypto/caam/error.h index b6350b0d9153..5aa332bac4b0 100644 --- a/drivers/crypto/caam/error.h +++ b/drivers/crypto/caam/error.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * CAAM Error Reporting code header * @@ -8,4 +9,8 @@ #define CAAM_ERROR_H #define CAAM_ERROR_STR_MAX 302 void caam_jr_strstatus(struct device *jrdev, u32 status); + +void caam_dump_sg(const char *level, const char *prefix_str, int prefix_type, + int rowsize, int groupsize, struct scatterlist *sg, + size_t tlen, bool ascii); #endif /* CAAM_ERROR_H */ diff --git a/drivers/crypto/caam/intern.h b/drivers/crypto/caam/intern.h index 85b6c5835b8f..91f1107276e5 100644 --- a/drivers/crypto/caam/intern.h +++ b/drivers/crypto/caam/intern.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * CAAM/SEC 4.x driver backend * Private/internal definitions between modules @@ -64,12 +65,9 @@ struct caam_drv_private_jr { * Driver-private storage for a single CAAM block instance */ struct caam_drv_private { - - struct device *dev; #ifdef CONFIG_CAAM_QI struct device *qidev; #endif - struct platform_device *pdev; /* Physical-presence section */ struct caam_ctrl __iomem *ctrl; /* controller region */ @@ -105,16 +103,8 @@ struct caam_drv_private { #ifdef CONFIG_DEBUG_FS struct dentry *dfs_root; struct dentry *ctl; /* controller dir */ - struct dentry *ctl_rq_dequeued, *ctl_ob_enc_req, *ctl_ib_dec_req; - struct dentry *ctl_ob_enc_bytes, *ctl_ob_prot_bytes; - struct dentry *ctl_ib_dec_bytes, *ctl_ib_valid_bytes; - struct dentry *ctl_faultaddr, *ctl_faultdetail, *ctl_faultstatus; - struct debugfs_blob_wrapper ctl_kek_wrap, ctl_tkek_wrap, ctl_tdsk_wrap; struct dentry *ctl_kek, *ctl_tkek, *ctl_tdsk; -#ifdef CONFIG_CAAM_QI - struct dentry *qi_congested; -#endif #endif }; diff --git a/drivers/crypto/caam/jr.c b/drivers/crypto/caam/jr.c index 1ccfb317d468..f4f258075b89 100644 --- a/drivers/crypto/caam/jr.c +++ b/drivers/crypto/caam/jr.c @@ -9,6 +9,7 @@ #include <linux/of_address.h> #include "compat.h" +#include "ctrl.h" #include "regs.h" #include "jr.h" #include "desc.h" @@ -171,7 +172,7 @@ static void caam_jr_dequeue(unsigned long devarg) while (rd_reg32(&jrp->rregs->outring_used)) { - head = ACCESS_ONCE(jrp->head); + head = READ_ONCE(jrp->head); spin_lock(&jrp->outlock); @@ -340,7 +341,7 @@ int caam_jr_enqueue(struct device *dev, u32 *desc, spin_lock_bh(&jrp->inplock); head = jrp->head; - tail = ACCESS_ONCE(jrp->tail); + tail = READ_ONCE(jrp->tail); if (!rd_reg32(&jrp->rregs->inpring_avail) || CIRC_SPACE(head, tail, JOBR_DEPTH) <= 0) { @@ -499,7 +500,11 @@ static int caam_jr_probe(struct platform_device *pdev) jrpriv->rregs = (struct caam_job_ring __iomem __force *)ctrl; if (sizeof(dma_addr_t) == sizeof(u64)) { - if (of_device_is_compatible(nprop, "fsl,sec-v5.0-job-ring")) + if (caam_dpaa2) + error = dma_set_mask_and_coherent(jrdev, + DMA_BIT_MASK(49)); + else if (of_device_is_compatible(nprop, + "fsl,sec-v5.0-job-ring")) error = dma_set_mask_and_coherent(jrdev, DMA_BIT_MASK(40)); else diff --git a/drivers/crypto/caam/jr.h b/drivers/crypto/caam/jr.h index 97113a6d6c58..eab611530f36 100644 --- a/drivers/crypto/caam/jr.h +++ b/drivers/crypto/caam/jr.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * CAAM public-level include definitions for the JobR backend * diff --git a/drivers/crypto/caam/key_gen.c b/drivers/crypto/caam/key_gen.c index c425d4adaf2a..8c79c3a153dc 100644 --- a/drivers/crypto/caam/key_gen.c +++ b/drivers/crypto/caam/key_gen.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * CAAM/SEC 4.x functions for handling key-generation jobs * diff --git a/drivers/crypto/caam/key_gen.h b/drivers/crypto/caam/key_gen.h index 4628f389eb64..5db055c25bd2 100644 --- a/drivers/crypto/caam/key_gen.h +++ b/drivers/crypto/caam/key_gen.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * CAAM/SEC 4.x definitions for handling key-generation jobs * diff --git a/drivers/crypto/caam/pdb.h b/drivers/crypto/caam/pdb.h index 31e59963f4d2..810f0bef0652 100644 --- a/drivers/crypto/caam/pdb.h +++ b/drivers/crypto/caam/pdb.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * CAAM Protocol Data Block (PDB) definition header file * diff --git a/drivers/crypto/caam/pkc_desc.c b/drivers/crypto/caam/pkc_desc.c index 9e2ce6fe2e43..2a8d87ea94bf 100644 --- a/drivers/crypto/caam/pkc_desc.c +++ b/drivers/crypto/caam/pkc_desc.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * caam - Freescale FSL CAAM support for Public Key Cryptography descriptors * diff --git a/drivers/crypto/caam/qi.c b/drivers/crypto/caam/qi.c index 1990ed460c46..f9a44f485aac 100644 --- a/drivers/crypto/caam/qi.c +++ b/drivers/crypto/caam/qi.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * CAAM/SEC 4.x QI transport/backend driver * Queue Interface backend functionality @@ -24,9 +25,6 @@ */ #define MAX_RSP_FQ_BACKLOG_PER_CPU 256 -/* Length of a single buffer in the QI driver memory cache */ -#define CAAM_QI_MEMCACHE_SIZE 512 - #define CAAM_QI_ENQUEUE_RETRIES 10000 #define CAAM_NAPI_WEIGHT 63 @@ -55,6 +53,7 @@ struct caam_qi_pcpu_priv { } ____cacheline_aligned; static DEFINE_PER_CPU(struct caam_qi_pcpu_priv, pcpu_qipriv); +static DEFINE_PER_CPU(int, last_cpu); /* * caam_qi_priv - CAAM QI backend private params @@ -203,8 +202,8 @@ static struct qman_fq *create_caam_req_fq(struct device *qidev, goto init_req_fq_fail; } - dev_info(qidev, "Allocated request FQ %u for CPU %u\n", req_fq->fqid, - smp_processor_id()); + dev_dbg(qidev, "Allocated request FQ %u for CPU %u\n", req_fq->fqid, + smp_processor_id()); return req_fq; init_req_fq_fail: @@ -277,6 +276,7 @@ empty_fq: dev_err(qidev, "OOS of FQID: %u failed\n", fq->fqid); qman_destroy_fq(fq); + kfree(fq); return ret; } @@ -342,8 +342,7 @@ int caam_drv_ctx_update(struct caam_drv_ctx *drv_ctx, u32 *sh_desc) drv_ctx->req_fq = old_fq; if (kill_fq(qidev, new_fq)) - dev_warn(qidev, "New CAAM FQ: %u kill failed\n", - new_fq->fqid); + dev_warn(qidev, "New CAAM FQ kill failed\n"); return ret; } @@ -373,10 +372,9 @@ int caam_drv_ctx_update(struct caam_drv_ctx *drv_ctx, u32 *sh_desc) drv_ctx->req_fq = old_fq; if (kill_fq(qidev, new_fq)) - dev_warn(qidev, "New CAAM FQ: %u kill failed\n", - new_fq->fqid); + dev_warn(qidev, "New CAAM FQ kill failed\n"); } else if (kill_fq(qidev, old_fq)) { - dev_warn(qidev, "Old CAAM FQ: %u kill failed\n", old_fq->fqid); + dev_warn(qidev, "Old CAAM FQ kill failed\n"); } return 0; @@ -392,7 +390,6 @@ struct caam_drv_ctx *caam_drv_ctx_init(struct device *qidev, dma_addr_t hwdesc; struct caam_drv_ctx *drv_ctx; const cpumask_t *cpus = qman_affine_cpus(); - static DEFINE_PER_CPU(int, last_cpu); num_words = desc_len(sh_desc); if (num_words > MAX_SDLEN) { @@ -511,7 +508,6 @@ int caam_qi_shutdown(struct device *qidev) if (kill_fq(qidev, per_cpu(pcpu_qipriv.rsp_fq, i))) dev_err(qidev, "Rsp FQ kill failed, cpu: %d\n", i); - kfree(per_cpu(pcpu_qipriv.rsp_fq, i)); } /* @@ -646,7 +642,7 @@ static int alloc_rsp_fq_cpu(struct device *qidev, unsigned int cpu) per_cpu(pcpu_qipriv.rsp_fq, cpu) = fq; - dev_info(qidev, "Allocated response FQ %u for CPU %u", fq->fqid, cpu); + dev_dbg(qidev, "Allocated response FQ %u for CPU %u", fq->fqid, cpu); return 0; } @@ -679,7 +675,7 @@ static int init_cgr(struct device *qidev) return ret; } - dev_info(qidev, "Congestion threshold set to %llu\n", val); + dev_dbg(qidev, "Congestion threshold set to %llu\n", val); return 0; } @@ -737,6 +733,7 @@ int caam_qi_init(struct platform_device *caam_pdev) qi_pdev = platform_device_register_full(&qi_pdev_info); if (IS_ERR(qi_pdev)) return PTR_ERR(qi_pdev); + set_dma_ops(&qi_pdev->dev, get_dma_ops(ctrldev)); ctrlpriv = dev_get_drvdata(ctrldev); qidev = &qi_pdev->dev; @@ -795,10 +792,8 @@ int caam_qi_init(struct platform_device *caam_pdev) /* Done with the CGRs; restore the cpus allowed mask */ set_cpus_allowed_ptr(current, &old_cpumask); #ifdef CONFIG_DEBUG_FS - ctrlpriv->qi_congested = debugfs_create_file("qi_congested", 0444, - ctrlpriv->ctl, - ×_congested, - &caam_fops_u64_ro); + debugfs_create_file("qi_congested", 0444, ctrlpriv->ctl, + ×_congested, &caam_fops_u64_ro); #endif dev_info(qidev, "Linux CAAM Queue I/F driver initialised\n"); return 0; diff --git a/drivers/crypto/caam/qi.h b/drivers/crypto/caam/qi.h index 33b0433f5f22..357b69f57072 100644 --- a/drivers/crypto/caam/qi.h +++ b/drivers/crypto/caam/qi.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Public definitions for the CAAM/QI (Queue Interface) backend. * @@ -39,6 +40,9 @@ */ #define MAX_SDLEN ((CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN) / CAAM_CMD_SZ) +/* Length of a single buffer in the QI driver memory cache */ +#define CAAM_QI_MEMCACHE_SIZE 768 + extern bool caam_congested __read_mostly; /* diff --git a/drivers/crypto/caam/regs.h b/drivers/crypto/caam/regs.h index 84d2f838a063..fee363865d88 100644 --- a/drivers/crypto/caam/regs.h +++ b/drivers/crypto/caam/regs.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * CAAM hardware register-level view * @@ -67,6 +68,7 @@ */ extern bool caam_little_end; +extern bool caam_imx; #define caam_to_cpu(len) \ static inline u##len caam##len ## _to_cpu(u##len val) \ @@ -154,13 +156,10 @@ static inline u64 rd_reg64(void __iomem *reg) #else /* CONFIG_64BIT */ static inline void wr_reg64(void __iomem *reg, u64 data) { -#ifndef CONFIG_CRYPTO_DEV_FSL_CAAM_IMX - if (caam_little_end) { + if (!caam_imx && caam_little_end) { wr_reg32((u32 __iomem *)(reg) + 1, data >> 32); wr_reg32((u32 __iomem *)(reg), data); - } else -#endif - { + } else { wr_reg32((u32 __iomem *)(reg), data >> 32); wr_reg32((u32 __iomem *)(reg) + 1, data); } @@ -168,41 +167,40 @@ static inline void wr_reg64(void __iomem *reg, u64 data) static inline u64 rd_reg64(void __iomem *reg) { -#ifndef CONFIG_CRYPTO_DEV_FSL_CAAM_IMX - if (caam_little_end) + if (!caam_imx && caam_little_end) return ((u64)rd_reg32((u32 __iomem *)(reg) + 1) << 32 | (u64)rd_reg32((u32 __iomem *)(reg))); - else -#endif - return ((u64)rd_reg32((u32 __iomem *)(reg)) << 32 | - (u64)rd_reg32((u32 __iomem *)(reg) + 1)); + + return ((u64)rd_reg32((u32 __iomem *)(reg)) << 32 | + (u64)rd_reg32((u32 __iomem *)(reg) + 1)); } #endif /* CONFIG_64BIT */ +static inline u64 cpu_to_caam_dma64(dma_addr_t value) +{ + if (caam_imx) + return (((u64)cpu_to_caam32(lower_32_bits(value)) << 32) | + (u64)cpu_to_caam32(upper_32_bits(value))); + + return cpu_to_caam64(value); +} + +static inline u64 caam_dma64_to_cpu(u64 value) +{ + if (caam_imx) + return (((u64)caam32_to_cpu(lower_32_bits(value)) << 32) | + (u64)caam32_to_cpu(upper_32_bits(value))); + + return caam64_to_cpu(value); +} + #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT -#ifdef CONFIG_SOC_IMX7D -#define cpu_to_caam_dma(value) \ - (((u64)cpu_to_caam32(lower_32_bits(value)) << 32) | \ - (u64)cpu_to_caam32(upper_32_bits(value))) -#define caam_dma_to_cpu(value) \ - (((u64)caam32_to_cpu(lower_32_bits(value)) << 32) | \ - (u64)caam32_to_cpu(upper_32_bits(value))) -#else -#define cpu_to_caam_dma(value) cpu_to_caam64(value) -#define caam_dma_to_cpu(value) caam64_to_cpu(value) -#endif /* CONFIG_SOC_IMX7D */ +#define cpu_to_caam_dma(value) cpu_to_caam_dma64(value) +#define caam_dma_to_cpu(value) caam_dma64_to_cpu(value) #else #define cpu_to_caam_dma(value) cpu_to_caam32(value) #define caam_dma_to_cpu(value) caam32_to_cpu(value) -#endif /* CONFIG_ARCH_DMA_ADDR_T_64BIT */ - -#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_IMX -#define cpu_to_caam_dma64(value) \ - (((u64)cpu_to_caam32(lower_32_bits(value)) << 32) | \ - (u64)cpu_to_caam32(upper_32_bits(value))) -#else -#define cpu_to_caam_dma64(value) cpu_to_caam64(value) -#endif +#endif /* CONFIG_ARCH_DMA_ADDR_T_64BIT */ /* * jr_outentry @@ -293,6 +291,7 @@ struct caam_perfmon { u32 cha_rev_ls; /* CRNR - CHA Rev No. Least significant half*/ #define CTPR_MS_QI_SHIFT 25 #define CTPR_MS_QI_MASK (0x1ull << CTPR_MS_QI_SHIFT) +#define CTPR_MS_DPAA2 BIT(13) #define CTPR_MS_VIRT_EN_INCL 0x00000001 #define CTPR_MS_VIRT_EN_POR 0x00000002 #define CTPR_MS_PG_SZ_MASK 0x10 diff --git a/drivers/crypto/caam/sg_sw_qm2.h b/drivers/crypto/caam/sg_sw_qm2.h new file mode 100644 index 000000000000..31b440757146 --- /dev/null +++ b/drivers/crypto/caam/sg_sw_qm2.h @@ -0,0 +1,81 @@ +/* + * Copyright 2015-2016 Freescale Semiconductor, Inc. + * Copyright 2017 NXP + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the names of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _SG_SW_QM2_H_ +#define _SG_SW_QM2_H_ + +#include "../../../drivers/staging/fsl-mc/include/dpaa2-fd.h" + +static inline void dma_to_qm_sg_one(struct dpaa2_sg_entry *qm_sg_ptr, + dma_addr_t dma, u32 len, u16 offset) +{ + dpaa2_sg_set_addr(qm_sg_ptr, dma); + dpaa2_sg_set_format(qm_sg_ptr, dpaa2_sg_single); + dpaa2_sg_set_final(qm_sg_ptr, false); + dpaa2_sg_set_len(qm_sg_ptr, len); + dpaa2_sg_set_bpid(qm_sg_ptr, 0); + dpaa2_sg_set_offset(qm_sg_ptr, offset); +} + +/* + * convert scatterlist to h/w link table format + * but does not have final bit; instead, returns last entry + */ +static inline struct dpaa2_sg_entry * +sg_to_qm_sg(struct scatterlist *sg, int sg_count, + struct dpaa2_sg_entry *qm_sg_ptr, u16 offset) +{ + while (sg_count && sg) { + dma_to_qm_sg_one(qm_sg_ptr, sg_dma_address(sg), + sg_dma_len(sg), offset); + qm_sg_ptr++; + sg = sg_next(sg); + sg_count--; + } + return qm_sg_ptr - 1; +} + +/* + * convert scatterlist to h/w link table format + * scatterlist must have been previously dma mapped + */ +static inline void sg_to_qm_sg_last(struct scatterlist *sg, int sg_count, + struct dpaa2_sg_entry *qm_sg_ptr, + u16 offset) +{ + qm_sg_ptr = sg_to_qm_sg(sg, sg_count, qm_sg_ptr, offset); + dpaa2_sg_set_final(qm_sg_ptr, true); +} + +#endif /* _SG_SW_QM2_H_ */ diff --git a/drivers/crypto/caam/sg_sw_sec4.h b/drivers/crypto/caam/sg_sw_sec4.h index c6adad09c972..e586ffab8358 100644 --- a/drivers/crypto/caam/sg_sw_sec4.h +++ b/drivers/crypto/caam/sg_sw_sec4.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * CAAM/SEC 4.x functions for using scatterlists in caam driver * @@ -5,7 +6,13 @@ * */ +#ifndef _SG_SW_SEC4_H_ +#define _SG_SW_SEC4_H_ + +#include "ctrl.h" #include "regs.h" +#include "sg_sw_qm2.h" +#include "../../../drivers/staging/fsl-mc/include/dpaa2-fd.h" struct sec4_sg_entry { u64 ptr; @@ -19,9 +26,15 @@ struct sec4_sg_entry { static inline void dma_to_sec4_sg_one(struct sec4_sg_entry *sec4_sg_ptr, dma_addr_t dma, u32 len, u16 offset) { - sec4_sg_ptr->ptr = cpu_to_caam_dma64(dma); - sec4_sg_ptr->len = cpu_to_caam32(len); - sec4_sg_ptr->bpid_offset = cpu_to_caam32(offset & SEC4_SG_OFFSET_MASK); + if (caam_dpaa2) { + dma_to_qm_sg_one((struct dpaa2_sg_entry *)sec4_sg_ptr, dma, len, + offset); + } else { + sec4_sg_ptr->ptr = cpu_to_caam_dma64(dma); + sec4_sg_ptr->len = cpu_to_caam32(len); + sec4_sg_ptr->bpid_offset = cpu_to_caam32(offset & + SEC4_SG_OFFSET_MASK); + } #ifdef DEBUG print_hex_dump(KERN_ERR, "sec4_sg_ptr@: ", DUMP_PREFIX_ADDRESS, 16, 4, sec4_sg_ptr, @@ -47,6 +60,14 @@ sg_to_sec4_sg(struct scatterlist *sg, int sg_count, return sec4_sg_ptr - 1; } +static inline void sg_to_sec4_set_last(struct sec4_sg_entry *sec4_sg_ptr) +{ + if (caam_dpaa2) + dpaa2_sg_set_final((struct dpaa2_sg_entry *)sec4_sg_ptr, true); + else + sec4_sg_ptr->len |= cpu_to_caam32(SEC4_SG_LEN_FIN); +} + /* * convert scatterlist to h/w link table format * scatterlist must have been previously dma mapped @@ -56,20 +77,7 @@ static inline void sg_to_sec4_sg_last(struct scatterlist *sg, int sg_count, u16 offset) { sec4_sg_ptr = sg_to_sec4_sg(sg, sg_count, sec4_sg_ptr, offset); - sec4_sg_ptr->len |= cpu_to_caam32(SEC4_SG_LEN_FIN); + sg_to_sec4_set_last(sec4_sg_ptr); } -static inline struct sec4_sg_entry *sg_to_sec4_sg_len( - struct scatterlist *sg, unsigned int total, - struct sec4_sg_entry *sec4_sg_ptr) -{ - do { - unsigned int len = min(sg_dma_len(sg), total); - - dma_to_sec4_sg_one(sec4_sg_ptr, sg_dma_address(sg), len, 0); - sec4_sg_ptr++; - sg = sg_next(sg); - total -= len; - } while (total); - return sec4_sg_ptr - 1; -} +#endif /* _SG_SW_SEC4_H_ */ diff --git a/drivers/crypto/cavium/cpt/cptpf_main.c b/drivers/crypto/cavium/cpt/cptpf_main.c index 4119c40e7c4b..34a6d8bf229e 100644 --- a/drivers/crypto/cavium/cpt/cptpf_main.c +++ b/drivers/crypto/cavium/cpt/cptpf_main.c @@ -268,8 +268,10 @@ static int cpt_ucode_load_fw(struct cpt_device *cpt, const u8 *fw, bool is_ae) mcode = &cpt->mcode[cpt->next_mc_idx]; memcpy(mcode->version, (u8 *)fw_entry->data, CPT_UCODE_VERSION_SZ); mcode->code_size = ntohl(ucode->code_length) * 2; - if (!mcode->code_size) - return -EINVAL; + if (!mcode->code_size) { + ret = -EINVAL; + goto fw_release; + } mcode->is_ae = is_ae; mcode->core_mask = 0ULL; @@ -280,7 +282,8 @@ static int cpt_ucode_load_fw(struct cpt_device *cpt, const u8 *fw, bool is_ae) &mcode->phys_base, GFP_KERNEL); if (!mcode->code) { dev_err(dev, "Unable to allocate space for microcode"); - return -ENOMEM; + ret = -ENOMEM; + goto fw_release; } memcpy((void *)mcode->code, (void *)(fw_entry->data + sizeof(*ucode)), @@ -302,12 +305,14 @@ static int cpt_ucode_load_fw(struct cpt_device *cpt, const u8 *fw, bool is_ae) ret = do_cpt_init(cpt, mcode); if (ret) { dev_err(dev, "do_cpt_init failed with ret: %d\n", ret); - return ret; + goto fw_release; } dev_info(dev, "Microcode Loaded %s\n", mcode->version); mcode->is_mc_valid = 1; cpt->next_mc_idx++; + +fw_release: release_firmware(fw_entry); return ret; diff --git a/drivers/crypto/cavium/nitrox/Makefile b/drivers/crypto/cavium/nitrox/Makefile index 5af2e4368267..45b7379e8e30 100644 --- a/drivers/crypto/cavium/nitrox/Makefile +++ b/drivers/crypto/cavium/nitrox/Makefile @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_CRYPTO_DEV_NITROX_CNN55XX) += n5pf.o n5pf-objs := nitrox_main.o \ diff --git a/drivers/crypto/cavium/nitrox/nitrox_algs.c b/drivers/crypto/cavium/nitrox/nitrox_algs.c index ce330278ef8a..2ae6124e5da6 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_algs.c +++ b/drivers/crypto/cavium/nitrox/nitrox_algs.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 #include <linux/crypto.h> #include <linux/kernel.h> #include <linux/module.h> diff --git a/drivers/crypto/cavium/nitrox/nitrox_common.h b/drivers/crypto/cavium/nitrox/nitrox_common.h index 4888c7823a5f..312f72801af6 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_common.h +++ b/drivers/crypto/cavium/nitrox/nitrox_common.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __NITROX_COMMON_H #define __NITROX_COMMON_H diff --git a/drivers/crypto/cavium/nitrox/nitrox_csr.h b/drivers/crypto/cavium/nitrox/nitrox_csr.h index 30b04c4c6076..9dcb7fdbe0a7 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_csr.h +++ b/drivers/crypto/cavium/nitrox/nitrox_csr.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __NITROX_CSR_H #define __NITROX_CSR_H diff --git a/drivers/crypto/cavium/nitrox/nitrox_dev.h b/drivers/crypto/cavium/nitrox/nitrox_dev.h index 57858b04f165..9a476bb6d4c7 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_dev.h +++ b/drivers/crypto/cavium/nitrox/nitrox_dev.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __NITROX_DEV_H #define __NITROX_DEV_H diff --git a/drivers/crypto/cavium/nitrox/nitrox_hal.c b/drivers/crypto/cavium/nitrox/nitrox_hal.c index f0655f82fa7d..ab4ccf2f9e77 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_hal.c +++ b/drivers/crypto/cavium/nitrox/nitrox_hal.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 #include <linux/delay.h> #include "nitrox_dev.h" @@ -126,7 +127,7 @@ void nitrox_config_pkt_input_rings(struct nitrox_device *ndev) * size and interrupt threshold. */ offset = NPS_PKT_IN_INSTR_BADDRX(i); - nitrox_write_csr(ndev, NPS_PKT_IN_INSTR_BADDRX(i), cmdq->dma); + nitrox_write_csr(ndev, offset, cmdq->dma); /* configure ring size */ offset = NPS_PKT_IN_INSTR_RSIZEX(i); diff --git a/drivers/crypto/cavium/nitrox/nitrox_isr.c b/drivers/crypto/cavium/nitrox/nitrox_isr.c index 71f934871a89..dbead5f45df3 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_isr.c +++ b/drivers/crypto/cavium/nitrox/nitrox_isr.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 #include <linux/pci.h> #include <linux/printk.h> #include <linux/slab.h> diff --git a/drivers/crypto/cavium/nitrox/nitrox_lib.c b/drivers/crypto/cavium/nitrox/nitrox_lib.c index b4a391adb9b6..4fdc921ba611 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_lib.c +++ b/drivers/crypto/cavium/nitrox/nitrox_lib.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 #include <linux/cpumask.h> #include <linux/dma-mapping.h> #include <linux/dmapool.h> diff --git a/drivers/crypto/cavium/nitrox/nitrox_main.c b/drivers/crypto/cavium/nitrox/nitrox_main.c index ae44a464cd2d..fee7cb2ce747 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_main.c +++ b/drivers/crypto/cavium/nitrox/nitrox_main.c @@ -18,8 +18,9 @@ #define SE_GROUP 0 #define DRIVER_VERSION "1.0" +#define FW_DIR "cavium/" /* SE microcode */ -#define SE_FW "cnn55xx_se.fw" +#define SE_FW FW_DIR "cnn55xx_se.fw" static const char nitrox_driver_name[] = "CNN55XX"; @@ -512,8 +513,10 @@ static int nitrox_probe(struct pci_dev *pdev, pci_set_master(pdev); ndev = kzalloc(sizeof(*ndev), GFP_KERNEL); - if (!ndev) + if (!ndev) { + err = -ENOMEM; goto ndev_fail; + } pci_set_drvdata(pdev, ndev); ndev->pdev = pdev; diff --git a/drivers/crypto/cavium/nitrox/nitrox_req.h b/drivers/crypto/cavium/nitrox/nitrox_req.h index 74f4c20dc87d..d091b6f5f5dd 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_req.h +++ b/drivers/crypto/cavium/nitrox/nitrox_req.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __NITROX_REQ_H #define __NITROX_REQ_H diff --git a/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c b/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c index 4bb4377c5ac0..4addc238a6ef 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c +++ b/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 #include <linux/gfp.h> #include <linux/workqueue.h> #include <crypto/internal/skcipher.h> diff --git a/drivers/crypto/cavium/zip/Makefile b/drivers/crypto/cavium/zip/Makefile index b2f3baaff757..020d189d793d 100644 --- a/drivers/crypto/cavium/zip/Makefile +++ b/drivers/crypto/cavium/zip/Makefile @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0 # # Makefile for Cavium's ZIP Driver. # diff --git a/drivers/crypto/ccp/Kconfig b/drivers/crypto/ccp/Kconfig index 2238f77aa248..6d626606b9c5 100644 --- a/drivers/crypto/ccp/Kconfig +++ b/drivers/crypto/ccp/Kconfig @@ -1,25 +1,33 @@ config CRYPTO_DEV_CCP_DD - tristate "Cryptographic Coprocessor device driver" - depends on CRYPTO_DEV_CCP + tristate "Secure Processor device driver" default m + help + Provides AMD Secure Processor device driver. + If you choose 'M' here, this module will be called ccp. + +config CRYPTO_DEV_SP_CCP + bool "Cryptographic Coprocessor device" + default y + depends on CRYPTO_DEV_CCP_DD select HW_RANDOM select DMA_ENGINE select DMADEVICES select CRYPTO_SHA1 select CRYPTO_SHA256 help - Provides the interface to use the AMD Cryptographic Coprocessor - which can be used to offload encryption operations such as SHA, - AES and more. If you choose 'M' here, this module will be called - ccp. + Provides the support for AMD Cryptographic Coprocessor (CCP) device + which can be used to offload encryption operations such as SHA, AES + and more. config CRYPTO_DEV_CCP_CRYPTO tristate "Encryption and hashing offload support" - depends on CRYPTO_DEV_CCP_DD default m + depends on CRYPTO_DEV_CCP_DD + depends on CRYPTO_DEV_SP_CCP select CRYPTO_HASH select CRYPTO_BLKCIPHER select CRYPTO_AUTHENC + select CRYPTO_RSA help Support for using the cryptographic API with the AMD Cryptographic Coprocessor. This module supports offload of SHA and AES algorithms. diff --git a/drivers/crypto/ccp/Makefile b/drivers/crypto/ccp/Makefile index 59493fd3a751..c4ce726b931e 100644 --- a/drivers/crypto/ccp/Makefile +++ b/drivers/crypto/ccp/Makefile @@ -1,12 +1,13 @@ +# SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_CRYPTO_DEV_CCP_DD) += ccp.o -ccp-objs := ccp-dev.o \ +ccp-objs := sp-dev.o sp-platform.o +ccp-$(CONFIG_CRYPTO_DEV_SP_CCP) += ccp-dev.o \ ccp-ops.o \ ccp-dev-v3.o \ ccp-dev-v5.o \ - ccp-platform.o \ ccp-dmaengine.o \ ccp-debugfs.o -ccp-$(CONFIG_PCI) += ccp-pci.o +ccp-$(CONFIG_PCI) += sp-pci.o obj-$(CONFIG_CRYPTO_DEV_CCP_CRYPTO) += ccp-crypto.o ccp-crypto-objs := ccp-crypto-main.o \ @@ -15,4 +16,5 @@ ccp-crypto-objs := ccp-crypto-main.o \ ccp-crypto-aes-xts.o \ ccp-crypto-aes-galois.o \ ccp-crypto-des3.o \ + ccp-crypto-rsa.o \ ccp-crypto-sha.o diff --git a/drivers/crypto/ccp/ccp-crypto-aes-galois.c b/drivers/crypto/ccp/ccp-crypto-aes-galois.c index 38ee6f348ea9..ff02b713c6f6 100644 --- a/drivers/crypto/ccp/ccp-crypto-aes-galois.c +++ b/drivers/crypto/ccp/ccp-crypto-aes-galois.c @@ -1,7 +1,7 @@ /* * AMD Cryptographic Coprocessor (CCP) AES GCM crypto API support * - * Copyright (C) 2016 Advanced Micro Devices, Inc. + * Copyright (C) 2016,2017 Advanced Micro Devices, Inc. * * Author: Gary R Hook <gary.hook@amd.com> * @@ -19,13 +19,12 @@ #include <crypto/algapi.h> #include <crypto/aes.h> #include <crypto/ctr.h> +#include <crypto/gcm.h> #include <crypto/scatterwalk.h> #include <linux/delay.h> #include "ccp-crypto.h" -#define AES_GCM_IVSIZE 12 - static int ccp_aes_gcm_complete(struct crypto_async_request *async_req, int ret) { return ret; @@ -95,9 +94,9 @@ static int ccp_aes_gcm_crypt(struct aead_request *req, bool encrypt) */ /* Prepare the IV: 12 bytes + an integer (counter) */ - memcpy(rctx->iv, req->iv, AES_GCM_IVSIZE); + memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE); for (i = 0; i < 3; i++) - rctx->iv[i + AES_GCM_IVSIZE] = 0; + rctx->iv[i + GCM_AES_IV_SIZE] = 0; rctx->iv[AES_BLOCK_SIZE - 1] = 1; /* Set up a scatterlist for the IV */ @@ -160,7 +159,7 @@ static struct aead_alg ccp_aes_gcm_defaults = { .encrypt = ccp_aes_gcm_encrypt, .decrypt = ccp_aes_gcm_decrypt, .init = ccp_aes_gcm_cra_init, - .ivsize = AES_GCM_IVSIZE, + .ivsize = GCM_AES_IV_SIZE, .maxauthsize = AES_BLOCK_SIZE, .base = { .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | diff --git a/drivers/crypto/ccp/ccp-crypto-aes-xts.c b/drivers/crypto/ccp/ccp-crypto-aes-xts.c index 58a4244b4752..94b5bcf5b628 100644 --- a/drivers/crypto/ccp/ccp-crypto-aes-xts.c +++ b/drivers/crypto/ccp/ccp-crypto-aes-xts.c @@ -1,8 +1,9 @@ /* * AMD Cryptographic Coprocessor (CCP) AES XTS crypto API support * - * Copyright (C) 2013 Advanced Micro Devices, Inc. + * Copyright (C) 2013,2017 Advanced Micro Devices, Inc. * + * Author: Gary R Hook <gary.hook@amd.com> * Author: Tom Lendacky <thomas.lendacky@amd.com> * * This program is free software; you can redistribute it and/or modify @@ -15,6 +16,7 @@ #include <linux/delay.h> #include <linux/scatterlist.h> #include <crypto/aes.h> +#include <crypto/xts.h> #include <crypto/internal/skcipher.h> #include <crypto/scatterwalk.h> @@ -37,46 +39,26 @@ struct ccp_unit_size_map { u32 value; }; -static struct ccp_unit_size_map unit_size_map[] = { +static struct ccp_unit_size_map xts_unit_sizes[] = { { - .size = 4096, - .value = CCP_XTS_AES_UNIT_SIZE_4096, - }, - { - .size = 2048, - .value = CCP_XTS_AES_UNIT_SIZE_2048, - }, - { - .size = 1024, - .value = CCP_XTS_AES_UNIT_SIZE_1024, + .size = 16, + .value = CCP_XTS_AES_UNIT_SIZE_16, }, { - .size = 512, + .size = 512, .value = CCP_XTS_AES_UNIT_SIZE_512, }, { - .size = 256, - .value = CCP_XTS_AES_UNIT_SIZE__LAST, - }, - { - .size = 128, - .value = CCP_XTS_AES_UNIT_SIZE__LAST, - }, - { - .size = 64, - .value = CCP_XTS_AES_UNIT_SIZE__LAST, - }, - { - .size = 32, - .value = CCP_XTS_AES_UNIT_SIZE__LAST, + .size = 1024, + .value = CCP_XTS_AES_UNIT_SIZE_1024, }, { - .size = 16, - .value = CCP_XTS_AES_UNIT_SIZE_16, + .size = 2048, + .value = CCP_XTS_AES_UNIT_SIZE_2048, }, { - .size = 1, - .value = CCP_XTS_AES_UNIT_SIZE__LAST, + .size = 4096, + .value = CCP_XTS_AES_UNIT_SIZE_4096, }, }; @@ -96,15 +78,26 @@ static int ccp_aes_xts_complete(struct crypto_async_request *async_req, int ret) static int ccp_aes_xts_setkey(struct crypto_ablkcipher *tfm, const u8 *key, unsigned int key_len) { - struct ccp_ctx *ctx = crypto_tfm_ctx(crypto_ablkcipher_tfm(tfm)); + struct crypto_tfm *xfm = crypto_ablkcipher_tfm(tfm); + struct ccp_ctx *ctx = crypto_tfm_ctx(xfm); + unsigned int ccpversion = ccp_version(); + int ret; - /* Only support 128-bit AES key with a 128-bit Tweak key, - * otherwise use the fallback + ret = xts_check_key(xfm, key, key_len); + if (ret) + return ret; + + /* Version 3 devices support 128-bit keys; version 5 devices can + * accommodate 128- and 256-bit keys. */ switch (key_len) { case AES_KEYSIZE_128 * 2: memcpy(ctx->u.aes.key, key, key_len); break; + case AES_KEYSIZE_256 * 2: + if (ccpversion > CCP_VERSION(3, 0)) + memcpy(ctx->u.aes.key, key, key_len); + break; } ctx->u.aes.key_len = key_len / 2; sg_init_one(&ctx->u.aes.key_sg, ctx->u.aes.key, key_len); @@ -117,6 +110,8 @@ static int ccp_aes_xts_crypt(struct ablkcipher_request *req, { struct ccp_ctx *ctx = crypto_tfm_ctx(req->base.tfm); struct ccp_aes_req_ctx *rctx = ablkcipher_request_ctx(req); + unsigned int ccpversion = ccp_version(); + unsigned int fallback = 0; unsigned int unit; u32 unit_size; int ret; @@ -130,18 +125,32 @@ static int ccp_aes_xts_crypt(struct ablkcipher_request *req, if (!req->info) return -EINVAL; + /* Check conditions under which the CCP can fulfill a request. The + * device can handle input plaintext of a length that is a multiple + * of the unit_size, bug the crypto implementation only supports + * the unit_size being equal to the input length. This limits the + * number of scenarios we can handle. + */ unit_size = CCP_XTS_AES_UNIT_SIZE__LAST; - if (req->nbytes <= unit_size_map[0].size) { - for (unit = 0; unit < ARRAY_SIZE(unit_size_map); unit++) { - if (!(req->nbytes & (unit_size_map[unit].size - 1))) { - unit_size = unit_size_map[unit].value; - break; - } + for (unit = 0; unit < ARRAY_SIZE(xts_unit_sizes); unit++) { + if (req->nbytes == xts_unit_sizes[unit].size) { + unit_size = unit; + break; } } - - if ((unit_size == CCP_XTS_AES_UNIT_SIZE__LAST) || - (ctx->u.aes.key_len != AES_KEYSIZE_128)) { + /* The CCP has restrictions on block sizes. Also, a version 3 device + * only supports AES-128 operations; version 5 CCPs support both + * AES-128 and -256 operations. + */ + if (unit_size == CCP_XTS_AES_UNIT_SIZE__LAST) + fallback = 1; + if ((ccpversion < CCP_VERSION(5, 0)) && + (ctx->u.aes.key_len != AES_KEYSIZE_128)) + fallback = 1; + if ((ctx->u.aes.key_len != AES_KEYSIZE_128) && + (ctx->u.aes.key_len != AES_KEYSIZE_256)) + fallback = 1; + if (fallback) { SKCIPHER_REQUEST_ON_STACK(subreq, ctx->u.aes.tfm_skcipher); /* Use the fallback to process the request for any @@ -164,6 +173,7 @@ static int ccp_aes_xts_crypt(struct ablkcipher_request *req, memset(&rctx->cmd, 0, sizeof(rctx->cmd)); INIT_LIST_HEAD(&rctx->cmd.entry); rctx->cmd.engine = CCP_ENGINE_XTS_AES_128; + rctx->cmd.u.xts.type = CCP_AES_TYPE_128; rctx->cmd.u.xts.action = (encrypt) ? CCP_AES_ACTION_ENCRYPT : CCP_AES_ACTION_DECRYPT; rctx->cmd.u.xts.unit_size = unit_size; diff --git a/drivers/crypto/ccp/ccp-crypto-des3.c b/drivers/crypto/ccp/ccp-crypto-des3.c index 5af7347ae03c..ae87b741f9d5 100644 --- a/drivers/crypto/ccp/ccp-crypto-des3.c +++ b/drivers/crypto/ccp/ccp-crypto-des3.c @@ -1,7 +1,7 @@ /* * AMD Cryptographic Coprocessor (CCP) DES3 crypto API support * - * Copyright (C) 2016 Advanced Micro Devices, Inc. + * Copyright (C) 2016,2017 Advanced Micro Devices, Inc. * * Author: Gary R Hook <ghook@amd.com> * diff --git a/drivers/crypto/ccp/ccp-crypto-main.c b/drivers/crypto/ccp/ccp-crypto-main.c index 8dccbddabef1..b95d19974aa6 100644 --- a/drivers/crypto/ccp/ccp-crypto-main.c +++ b/drivers/crypto/ccp/ccp-crypto-main.c @@ -1,7 +1,7 @@ /* * AMD Cryptographic Coprocessor (CCP) crypto API support * - * Copyright (C) 2013 Advanced Micro Devices, Inc. + * Copyright (C) 2013,2017 Advanced Micro Devices, Inc. * * Author: Tom Lendacky <thomas.lendacky@amd.com> * @@ -17,6 +17,7 @@ #include <linux/ccp.h> #include <linux/scatterlist.h> #include <crypto/internal/hash.h> +#include <crypto/internal/akcipher.h> #include "ccp-crypto.h" @@ -37,10 +38,15 @@ static unsigned int des3_disable; module_param(des3_disable, uint, 0444); MODULE_PARM_DESC(des3_disable, "Disable use of 3DES - any non-zero value"); +static unsigned int rsa_disable; +module_param(rsa_disable, uint, 0444); +MODULE_PARM_DESC(rsa_disable, "Disable use of RSA - any non-zero value"); + /* List heads for the supported algorithms */ static LIST_HEAD(hash_algs); static LIST_HEAD(cipher_algs); static LIST_HEAD(aead_algs); +static LIST_HEAD(akcipher_algs); /* For any tfm, requests for that tfm must be returned on the order * received. With multiple queues available, the CCP can process more @@ -216,9 +222,10 @@ static int ccp_crypto_enqueue_cmd(struct ccp_crypto_cmd *crypto_cmd) /* Check if the cmd can/should be queued */ if (req_queue.cmd_count >= CCP_CRYPTO_MAX_QLEN) { - ret = -EBUSY; - if (!(crypto_cmd->cmd->flags & CCP_CMD_MAY_BACKLOG)) + if (!(crypto_cmd->cmd->flags & CCP_CMD_MAY_BACKLOG)) { + ret = -ENOSPC; goto e_lock; + } } /* Look for an entry with the same tfm. If there is a cmd @@ -237,9 +244,6 @@ static int ccp_crypto_enqueue_cmd(struct ccp_crypto_cmd *crypto_cmd) ret = ccp_enqueue_cmd(crypto_cmd->cmd); if (!ccp_crypto_success(ret)) goto e_lock; /* Error, don't queue it */ - if ((ret == -EBUSY) && - !(crypto_cmd->cmd->flags & CCP_CMD_MAY_BACKLOG)) - goto e_lock; /* Not backlogging, don't queue it */ } if (req_queue.cmd_count >= CCP_CRYPTO_MAX_QLEN) { @@ -358,6 +362,12 @@ static int ccp_register_algs(void) return ret; } + if (!rsa_disable) { + ret = ccp_register_rsa_algs(&akcipher_algs); + if (ret) + return ret; + } + return 0; } @@ -366,6 +376,7 @@ static void ccp_unregister_algs(void) struct ccp_crypto_ahash_alg *ahash_alg, *ahash_tmp; struct ccp_crypto_ablkcipher_alg *ablk_alg, *ablk_tmp; struct ccp_crypto_aead *aead_alg, *aead_tmp; + struct ccp_crypto_akcipher_alg *akc_alg, *akc_tmp; list_for_each_entry_safe(ahash_alg, ahash_tmp, &hash_algs, entry) { crypto_unregister_ahash(&ahash_alg->alg); @@ -384,6 +395,12 @@ static void ccp_unregister_algs(void) list_del(&aead_alg->entry); kfree(aead_alg); } + + list_for_each_entry_safe(akc_alg, akc_tmp, &akcipher_algs, entry) { + crypto_unregister_akcipher(&akc_alg->alg); + list_del(&akc_alg->entry); + kfree(akc_alg); + } } static int ccp_crypto_init(void) diff --git a/drivers/crypto/ccp/ccp-crypto-rsa.c b/drivers/crypto/ccp/ccp-crypto-rsa.c new file mode 100644 index 000000000000..e6db8672d89c --- /dev/null +++ b/drivers/crypto/ccp/ccp-crypto-rsa.c @@ -0,0 +1,299 @@ +/* + * AMD Cryptographic Coprocessor (CCP) RSA crypto API support + * + * Copyright (C) 2017 Advanced Micro Devices, Inc. + * + * Author: Gary R Hook <gary.hook@amd.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/module.h> +#include <linux/sched.h> +#include <linux/scatterlist.h> +#include <linux/crypto.h> +#include <crypto/algapi.h> +#include <crypto/internal/rsa.h> +#include <crypto/internal/akcipher.h> +#include <crypto/akcipher.h> +#include <crypto/scatterwalk.h> + +#include "ccp-crypto.h" + +static inline struct akcipher_request *akcipher_request_cast( + struct crypto_async_request *req) +{ + return container_of(req, struct akcipher_request, base); +} + +static inline int ccp_copy_and_save_keypart(u8 **kpbuf, unsigned int *kplen, + const u8 *buf, size_t sz) +{ + int nskip; + + for (nskip = 0; nskip < sz; nskip++) + if (buf[nskip]) + break; + *kplen = sz - nskip; + *kpbuf = kzalloc(*kplen, GFP_KERNEL); + if (!*kpbuf) + return -ENOMEM; + memcpy(*kpbuf, buf + nskip, *kplen); + + return 0; +} + +static int ccp_rsa_complete(struct crypto_async_request *async_req, int ret) +{ + struct akcipher_request *req = akcipher_request_cast(async_req); + struct ccp_rsa_req_ctx *rctx = akcipher_request_ctx(req); + + if (ret) + return ret; + + req->dst_len = rctx->cmd.u.rsa.key_size >> 3; + + return 0; +} + +static unsigned int ccp_rsa_maxsize(struct crypto_akcipher *tfm) +{ + if (ccp_version() > CCP_VERSION(3, 0)) + return CCP5_RSA_MAXMOD; + else + return CCP_RSA_MAXMOD; +} + +static int ccp_rsa_crypt(struct akcipher_request *req, bool encrypt) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct ccp_ctx *ctx = akcipher_tfm_ctx(tfm); + struct ccp_rsa_req_ctx *rctx = akcipher_request_ctx(req); + int ret = 0; + + memset(&rctx->cmd, 0, sizeof(rctx->cmd)); + INIT_LIST_HEAD(&rctx->cmd.entry); + rctx->cmd.engine = CCP_ENGINE_RSA; + + rctx->cmd.u.rsa.key_size = ctx->u.rsa.key_len; /* in bits */ + if (encrypt) { + rctx->cmd.u.rsa.exp = &ctx->u.rsa.e_sg; + rctx->cmd.u.rsa.exp_len = ctx->u.rsa.e_len; + } else { + rctx->cmd.u.rsa.exp = &ctx->u.rsa.d_sg; + rctx->cmd.u.rsa.exp_len = ctx->u.rsa.d_len; + } + rctx->cmd.u.rsa.mod = &ctx->u.rsa.n_sg; + rctx->cmd.u.rsa.mod_len = ctx->u.rsa.n_len; + rctx->cmd.u.rsa.src = req->src; + rctx->cmd.u.rsa.src_len = req->src_len; + rctx->cmd.u.rsa.dst = req->dst; + + ret = ccp_crypto_enqueue_request(&req->base, &rctx->cmd); + + return ret; +} + +static int ccp_rsa_encrypt(struct akcipher_request *req) +{ + return ccp_rsa_crypt(req, true); +} + +static int ccp_rsa_decrypt(struct akcipher_request *req) +{ + return ccp_rsa_crypt(req, false); +} + +static int ccp_check_key_length(unsigned int len) +{ + /* In bits */ + if (len < 8 || len > 4096) + return -EINVAL; + return 0; +} + +static void ccp_rsa_free_key_bufs(struct ccp_ctx *ctx) +{ + /* Clean up old key data */ + kzfree(ctx->u.rsa.e_buf); + ctx->u.rsa.e_buf = NULL; + ctx->u.rsa.e_len = 0; + kzfree(ctx->u.rsa.n_buf); + ctx->u.rsa.n_buf = NULL; + ctx->u.rsa.n_len = 0; + kzfree(ctx->u.rsa.d_buf); + ctx->u.rsa.d_buf = NULL; + ctx->u.rsa.d_len = 0; +} + +static int ccp_rsa_setkey(struct crypto_akcipher *tfm, const void *key, + unsigned int keylen, bool private) +{ + struct ccp_ctx *ctx = akcipher_tfm_ctx(tfm); + struct rsa_key raw_key; + int ret; + + ccp_rsa_free_key_bufs(ctx); + memset(&raw_key, 0, sizeof(raw_key)); + + /* Code borrowed from crypto/rsa.c */ + if (private) + ret = rsa_parse_priv_key(&raw_key, key, keylen); + else + ret = rsa_parse_pub_key(&raw_key, key, keylen); + if (ret) + goto n_key; + + ret = ccp_copy_and_save_keypart(&ctx->u.rsa.n_buf, &ctx->u.rsa.n_len, + raw_key.n, raw_key.n_sz); + if (ret) + goto key_err; + sg_init_one(&ctx->u.rsa.n_sg, ctx->u.rsa.n_buf, ctx->u.rsa.n_len); + + ctx->u.rsa.key_len = ctx->u.rsa.n_len << 3; /* convert to bits */ + if (ccp_check_key_length(ctx->u.rsa.key_len)) { + ret = -EINVAL; + goto key_err; + } + + ret = ccp_copy_and_save_keypart(&ctx->u.rsa.e_buf, &ctx->u.rsa.e_len, + raw_key.e, raw_key.e_sz); + if (ret) + goto key_err; + sg_init_one(&ctx->u.rsa.e_sg, ctx->u.rsa.e_buf, ctx->u.rsa.e_len); + + if (private) { + ret = ccp_copy_and_save_keypart(&ctx->u.rsa.d_buf, + &ctx->u.rsa.d_len, + raw_key.d, raw_key.d_sz); + if (ret) + goto key_err; + sg_init_one(&ctx->u.rsa.d_sg, + ctx->u.rsa.d_buf, ctx->u.rsa.d_len); + } + + return 0; + +key_err: + ccp_rsa_free_key_bufs(ctx); + +n_key: + return ret; +} + +static int ccp_rsa_setprivkey(struct crypto_akcipher *tfm, const void *key, + unsigned int keylen) +{ + return ccp_rsa_setkey(tfm, key, keylen, true); +} + +static int ccp_rsa_setpubkey(struct crypto_akcipher *tfm, const void *key, + unsigned int keylen) +{ + return ccp_rsa_setkey(tfm, key, keylen, false); +} + +static int ccp_rsa_init_tfm(struct crypto_akcipher *tfm) +{ + struct ccp_ctx *ctx = akcipher_tfm_ctx(tfm); + + akcipher_set_reqsize(tfm, sizeof(struct ccp_rsa_req_ctx)); + ctx->complete = ccp_rsa_complete; + + return 0; +} + +static void ccp_rsa_exit_tfm(struct crypto_akcipher *tfm) +{ + struct ccp_ctx *ctx = crypto_tfm_ctx(&tfm->base); + + ccp_rsa_free_key_bufs(ctx); +} + +static struct akcipher_alg ccp_rsa_defaults = { + .encrypt = ccp_rsa_encrypt, + .decrypt = ccp_rsa_decrypt, + .sign = ccp_rsa_decrypt, + .verify = ccp_rsa_encrypt, + .set_pub_key = ccp_rsa_setpubkey, + .set_priv_key = ccp_rsa_setprivkey, + .max_size = ccp_rsa_maxsize, + .init = ccp_rsa_init_tfm, + .exit = ccp_rsa_exit_tfm, + .base = { + .cra_name = "rsa", + .cra_driver_name = "rsa-ccp", + .cra_priority = CCP_CRA_PRIORITY, + .cra_module = THIS_MODULE, + .cra_ctxsize = 2 * sizeof(struct ccp_ctx), + }, +}; + +struct ccp_rsa_def { + unsigned int version; + const char *name; + const char *driver_name; + unsigned int reqsize; + struct akcipher_alg *alg_defaults; +}; + +static struct ccp_rsa_def rsa_algs[] = { + { + .version = CCP_VERSION(3, 0), + .name = "rsa", + .driver_name = "rsa-ccp", + .reqsize = sizeof(struct ccp_rsa_req_ctx), + .alg_defaults = &ccp_rsa_defaults, + } +}; + +int ccp_register_rsa_alg(struct list_head *head, const struct ccp_rsa_def *def) +{ + struct ccp_crypto_akcipher_alg *ccp_alg; + struct akcipher_alg *alg; + int ret; + + ccp_alg = kzalloc(sizeof(*ccp_alg), GFP_KERNEL); + if (!ccp_alg) + return -ENOMEM; + + INIT_LIST_HEAD(&ccp_alg->entry); + + alg = &ccp_alg->alg; + *alg = *def->alg_defaults; + snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", def->name); + snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", + def->driver_name); + ret = crypto_register_akcipher(alg); + if (ret) { + pr_err("%s akcipher algorithm registration error (%d)\n", + alg->base.cra_name, ret); + kfree(ccp_alg); + return ret; + } + + list_add(&ccp_alg->entry, head); + + return 0; +} + +int ccp_register_rsa_algs(struct list_head *head) +{ + int i, ret; + unsigned int ccpversion = ccp_version(); + + /* Register the RSA algorithm in standard mode + * This works for CCP v3 and later + */ + for (i = 0; i < ARRAY_SIZE(rsa_algs); i++) { + if (rsa_algs[i].version > ccpversion) + continue; + ret = ccp_register_rsa_alg(head, &rsa_algs[i]); + if (ret) + return ret; + } + + return 0; +} diff --git a/drivers/crypto/ccp/ccp-crypto-sha.c b/drivers/crypto/ccp/ccp-crypto-sha.c index ce97b3868f4a..8b9b16d433f7 100644 --- a/drivers/crypto/ccp/ccp-crypto-sha.c +++ b/drivers/crypto/ccp/ccp-crypto-sha.c @@ -1,7 +1,7 @@ /* * AMD Cryptographic Coprocessor (CCP) SHA crypto API support * - * Copyright (C) 2013,2016 Advanced Micro Devices, Inc. + * Copyright (C) 2013,2017 Advanced Micro Devices, Inc. * * Author: Tom Lendacky <thomas.lendacky@amd.com> * Author: Gary R Hook <gary.hook@amd.com> diff --git a/drivers/crypto/ccp/ccp-crypto.h b/drivers/crypto/ccp/ccp-crypto.h index dd5bf15f06e5..b9fd090c46c2 100644 --- a/drivers/crypto/ccp/ccp-crypto.h +++ b/drivers/crypto/ccp/ccp-crypto.h @@ -1,7 +1,7 @@ /* * AMD Cryptographic Coprocessor (CCP) crypto API support * - * Copyright (C) 2013 Advanced Micro Devices, Inc. + * Copyright (C) 2013,2017 Advanced Micro Devices, Inc. * * Author: Tom Lendacky <thomas.lendacky@amd.com> * @@ -24,6 +24,8 @@ #include <crypto/ctr.h> #include <crypto/hash.h> #include <crypto/sha.h> +#include <crypto/akcipher.h> +#include <crypto/internal/rsa.h> #define CCP_LOG_LEVEL KERN_INFO @@ -58,6 +60,12 @@ struct ccp_crypto_ahash_alg { struct ahash_alg alg; }; +struct ccp_crypto_akcipher_alg { + struct list_head entry; + + struct akcipher_alg alg; +}; + static inline struct ccp_crypto_ablkcipher_alg * ccp_crypto_ablkcipher_alg(struct crypto_tfm *tfm) { @@ -91,7 +99,7 @@ struct ccp_aes_ctx { struct scatterlist key_sg; unsigned int key_len; - u8 key[AES_MAX_KEY_SIZE]; + u8 key[AES_MAX_KEY_SIZE * 2]; u8 nonce[CTR_RFC3686_NONCE_SIZE]; @@ -227,12 +235,35 @@ struct ccp_sha_exp_ctx { u8 buf[MAX_SHA_BLOCK_SIZE]; }; +/***** RSA related defines *****/ + +struct ccp_rsa_ctx { + unsigned int key_len; /* in bits */ + struct scatterlist e_sg; + u8 *e_buf; + unsigned int e_len; + struct scatterlist n_sg; + u8 *n_buf; + unsigned int n_len; + struct scatterlist d_sg; + u8 *d_buf; + unsigned int d_len; +}; + +struct ccp_rsa_req_ctx { + struct ccp_cmd cmd; +}; + +#define CCP_RSA_MAXMOD (4 * 1024 / 8) +#define CCP5_RSA_MAXMOD (16 * 1024 / 8) + /***** Common Context Structure *****/ struct ccp_ctx { int (*complete)(struct crypto_async_request *req, int ret); union { struct ccp_aes_ctx aes; + struct ccp_rsa_ctx rsa; struct ccp_sha_ctx sha; struct ccp_des3_ctx des3; } u; @@ -249,5 +280,6 @@ int ccp_register_aes_xts_algs(struct list_head *head); int ccp_register_aes_aeads(struct list_head *head); int ccp_register_sha_algs(struct list_head *head); int ccp_register_des3_algs(struct list_head *head); +int ccp_register_rsa_algs(struct list_head *head); #endif diff --git a/drivers/crypto/ccp/ccp-debugfs.c b/drivers/crypto/ccp/ccp-debugfs.c index 3cd6c83754e0..59d4ca4e72d8 100644 --- a/drivers/crypto/ccp/ccp-debugfs.c +++ b/drivers/crypto/ccp/ccp-debugfs.c @@ -305,19 +305,19 @@ void ccp5_debugfs_setup(struct ccp_device *ccp) ccp->debugfs_instance = debugfs_create_dir(ccp->name, ccp_debugfs_dir); if (!ccp->debugfs_instance) - return; + goto err; debugfs_info = debugfs_create_file("info", 0400, ccp->debugfs_instance, ccp, &ccp_debugfs_info_ops); if (!debugfs_info) - return; + goto err; debugfs_stats = debugfs_create_file("stats", 0600, ccp->debugfs_instance, ccp, &ccp_debugfs_stats_ops); if (!debugfs_stats) - return; + goto err; for (i = 0; i < ccp->cmd_q_count; i++) { cmd_q = &ccp->cmd_q[i]; @@ -327,15 +327,20 @@ void ccp5_debugfs_setup(struct ccp_device *ccp) debugfs_q_instance = debugfs_create_dir(name, ccp->debugfs_instance); if (!debugfs_q_instance) - return; + goto err; debugfs_q_stats = debugfs_create_file("stats", 0600, debugfs_q_instance, cmd_q, &ccp_debugfs_queue_ops); if (!debugfs_q_stats) - return; + goto err; } + + return; + +err: + debugfs_remove_recursive(ccp->debugfs_instance); } void ccp5_debugfs_destroy(void) diff --git a/drivers/crypto/ccp/ccp-dev-v3.c b/drivers/crypto/ccp/ccp-dev-v3.c index 367c2e30656f..240bebbcb8ac 100644 --- a/drivers/crypto/ccp/ccp-dev-v3.c +++ b/drivers/crypto/ccp/ccp-dev-v3.c @@ -1,7 +1,7 @@ /* * AMD Cryptographic Coprocessor (CCP) driver * - * Copyright (C) 2013,2016 Advanced Micro Devices, Inc. + * Copyright (C) 2013,2017 Advanced Micro Devices, Inc. * * Author: Tom Lendacky <thomas.lendacky@amd.com> * Author: Gary R Hook <gary.hook@amd.com> @@ -359,8 +359,7 @@ static void ccp_irq_bh(unsigned long data) static irqreturn_t ccp_irq_handler(int irq, void *data) { - struct device *dev = data; - struct ccp_device *ccp = dev_get_drvdata(dev); + struct ccp_device *ccp = (struct ccp_device *)data; ccp_disable_queue_interrupts(ccp); if (ccp->use_tasklet) @@ -454,7 +453,7 @@ static int ccp_init(struct ccp_device *ccp) iowrite32(ccp->qim, ccp->io_regs + IRQ_STATUS_REG); /* Request an irq */ - ret = ccp->get_irq(ccp); + ret = sp_request_ccp_irq(ccp->sp, ccp_irq_handler, ccp->name, ccp); if (ret) { dev_err(dev, "unable to allocate an IRQ\n"); goto e_pool; @@ -511,7 +510,7 @@ e_kthread: if (ccp->cmd_q[i].kthread) kthread_stop(ccp->cmd_q[i].kthread); - ccp->free_irq(ccp); + sp_free_ccp_irq(ccp->sp, ccp); e_pool: for (i = 0; i < ccp->cmd_q_count; i++) @@ -550,7 +549,7 @@ static void ccp_destroy(struct ccp_device *ccp) if (ccp->cmd_q[i].kthread) kthread_stop(ccp->cmd_q[i].kthread); - ccp->free_irq(ccp); + sp_free_ccp_irq(ccp->sp, ccp); for (i = 0; i < ccp->cmd_q_count; i++) dma_pool_destroy(ccp->cmd_q[i].dma_pool); @@ -586,10 +585,17 @@ static const struct ccp_actions ccp3_actions = { .irqhandler = ccp_irq_handler, }; +const struct ccp_vdata ccpv3_platform = { + .version = CCP_VERSION(3, 0), + .setup = NULL, + .perform = &ccp3_actions, + .offset = 0, +}; + const struct ccp_vdata ccpv3 = { .version = CCP_VERSION(3, 0), .setup = NULL, .perform = &ccp3_actions, - .bar = 2, .offset = 0x20000, + .rsamax = CCP_RSA_MAX_WIDTH, }; diff --git a/drivers/crypto/ccp/ccp-dev-v5.c b/drivers/crypto/ccp/ccp-dev-v5.c index b10d2d2075cb..44a4d2779b15 100644 --- a/drivers/crypto/ccp/ccp-dev-v5.c +++ b/drivers/crypto/ccp/ccp-dev-v5.c @@ -1,7 +1,7 @@ /* * AMD Cryptographic Coprocessor (CCP) driver * - * Copyright (C) 2016 Advanced Micro Devices, Inc. + * Copyright (C) 2016,2017 Advanced Micro Devices, Inc. * * Author: Gary R Hook <gary.hook@amd.com> * @@ -145,6 +145,7 @@ union ccp_function { #define CCP_AES_MODE(p) ((p)->aes.mode) #define CCP_AES_TYPE(p) ((p)->aes.type) #define CCP_XTS_SIZE(p) ((p)->aes_xts.size) +#define CCP_XTS_TYPE(p) ((p)->aes_xts.type) #define CCP_XTS_ENCRYPT(p) ((p)->aes_xts.encrypt) #define CCP_DES3_SIZE(p) ((p)->des3.size) #define CCP_DES3_ENCRYPT(p) ((p)->des3.encrypt) @@ -344,6 +345,7 @@ static int ccp5_perform_xts_aes(struct ccp_op *op) CCP5_CMD_PROT(&desc) = 0; function.raw = 0; + CCP_XTS_TYPE(&function) = op->u.xts.type; CCP_XTS_ENCRYPT(&function) = op->u.xts.action; CCP_XTS_SIZE(&function) = op->u.xts.unit_size; CCP5_CMD_FUNCTION(&desc) = function.raw; @@ -469,7 +471,7 @@ static int ccp5_perform_rsa(struct ccp_op *op) CCP5_CMD_PROT(&desc) = 0; function.raw = 0; - CCP_RSA_SIZE(&function) = op->u.rsa.mod_size >> 3; + CCP_RSA_SIZE(&function) = (op->u.rsa.mod_size + 7) >> 3; CCP5_CMD_FUNCTION(&desc) = function.raw; CCP5_CMD_LEN(&desc) = op->u.rsa.input_len; @@ -484,10 +486,10 @@ static int ccp5_perform_rsa(struct ccp_op *op) CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma); CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM; - /* Exponent is in LSB memory */ - CCP5_CMD_KEY_LO(&desc) = op->sb_key * LSB_ITEM_SIZE; - CCP5_CMD_KEY_HI(&desc) = 0; - CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB; + /* Key (Exponent) is in external memory */ + CCP5_CMD_KEY_LO(&desc) = ccp_addr_lo(&op->exp.u.dma); + CCP5_CMD_KEY_HI(&desc) = ccp_addr_hi(&op->exp.u.dma); + CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SYSTEM; return ccp5_do_cmd(&desc, op->cmd_q); } @@ -769,8 +771,7 @@ static void ccp5_irq_bh(unsigned long data) static irqreturn_t ccp5_irq_handler(int irq, void *data) { - struct device *dev = data; - struct ccp_device *ccp = dev_get_drvdata(dev); + struct ccp_device *ccp = (struct ccp_device *)data; ccp5_disable_queue_interrupts(ccp); ccp->total_interrupts++; @@ -787,13 +788,12 @@ static int ccp5_init(struct ccp_device *ccp) struct ccp_cmd_queue *cmd_q; struct dma_pool *dma_pool; char dma_pool_name[MAX_DMAPOOL_NAME_LEN]; - unsigned int qmr, qim, i; + unsigned int qmr, i; u64 status; u32 status_lo, status_hi; int ret; /* Find available queues */ - qim = 0; qmr = ioread32(ccp->io_regs + Q_MASK_REG); for (i = 0; i < MAX_HW_QUEUES; i++) { @@ -881,7 +881,7 @@ static int ccp5_init(struct ccp_device *ccp) dev_dbg(dev, "Requesting an IRQ...\n"); /* Request an irq */ - ret = ccp->get_irq(ccp); + ret = sp_request_ccp_irq(ccp->sp, ccp5_irq_handler, ccp->name, ccp); if (ret) { dev_err(dev, "unable to allocate an IRQ\n"); goto e_pool; @@ -987,7 +987,7 @@ e_kthread: kthread_stop(ccp->cmd_q[i].kthread); e_irq: - ccp->free_irq(ccp); + sp_free_ccp_irq(ccp->sp, ccp); e_pool: for (i = 0; i < ccp->cmd_q_count; i++) @@ -1037,7 +1037,7 @@ static void ccp5_destroy(struct ccp_device *ccp) if (ccp->cmd_q[i].kthread) kthread_stop(ccp->cmd_q[i].kthread); - ccp->free_irq(ccp); + sp_free_ccp_irq(ccp->sp, ccp); for (i = 0; i < ccp->cmd_q_count; i++) { cmd_q = &ccp->cmd_q[i]; @@ -1106,15 +1106,14 @@ static const struct ccp_actions ccp5_actions = { .init = ccp5_init, .destroy = ccp5_destroy, .get_free_slots = ccp5_get_free_slots, - .irqhandler = ccp5_irq_handler, }; const struct ccp_vdata ccpv5a = { .version = CCP_VERSION(5, 0), .setup = ccp5_config, .perform = &ccp5_actions, - .bar = 2, .offset = 0x0, + .rsamax = CCP5_RSA_MAX_WIDTH, }; const struct ccp_vdata ccpv5b = { @@ -1122,6 +1121,6 @@ const struct ccp_vdata ccpv5b = { .dma_chan_attr = DMA_PRIVATE, .setup = ccp5other_config, .perform = &ccp5_actions, - .bar = 2, .offset = 0x0, + .rsamax = CCP5_RSA_MAX_WIDTH, }; diff --git a/drivers/crypto/ccp/ccp-dev.c b/drivers/crypto/ccp/ccp-dev.c index 2506b5025700..1b5035d56288 100644 --- a/drivers/crypto/ccp/ccp-dev.c +++ b/drivers/crypto/ccp/ccp-dev.c @@ -1,7 +1,7 @@ /* * AMD Cryptographic Coprocessor (CCP) driver * - * Copyright (C) 2013,2016 Advanced Micro Devices, Inc. + * Copyright (C) 2013,2017 Advanced Micro Devices, Inc. * * Author: Tom Lendacky <thomas.lendacky@amd.com> * Author: Gary R Hook <gary.hook@amd.com> @@ -11,7 +11,6 @@ * published by the Free Software Foundation. */ -#include <linux/module.h> #include <linux/kernel.h> #include <linux/kthread.h> #include <linux/sched.h> @@ -30,12 +29,6 @@ #include "ccp-dev.h" -MODULE_AUTHOR("Tom Lendacky <thomas.lendacky@amd.com>"); -MODULE_AUTHOR("Gary R Hook <gary.hook@amd.com>"); -MODULE_LICENSE("GPL"); -MODULE_VERSION("1.1.0"); -MODULE_DESCRIPTION("AMD Cryptographic Coprocessor driver"); - struct ccp_tasklet_data { struct completion completion; struct ccp_cmd *cmd; @@ -111,13 +104,6 @@ static LIST_HEAD(ccp_units); static DEFINE_SPINLOCK(ccp_rr_lock); static struct ccp_device *ccp_rr; -/* Ever-increasing value to produce unique unit numbers */ -static atomic_t ccp_unit_ordinal; -static unsigned int ccp_increment_unit_ordinal(void) -{ - return atomic_inc_return(&ccp_unit_ordinal); -} - /** * ccp_add_device - add a CCP device to the list * @@ -306,9 +292,12 @@ int ccp_enqueue_cmd(struct ccp_cmd *cmd) i = ccp->cmd_q_count; if (ccp->cmd_count >= MAX_CMD_QLEN) { - ret = -EBUSY; - if (cmd->flags & CCP_CMD_MAY_BACKLOG) + if (cmd->flags & CCP_CMD_MAY_BACKLOG) { + ret = -EBUSY; list_add_tail(&cmd->entry, &ccp->backlog); + } else { + ret = -ENOSPC; + } } else { ret = -EINPROGRESS; ccp->cmd_count++; @@ -415,6 +404,7 @@ static void ccp_do_cmd_complete(unsigned long data) struct ccp_cmd *cmd = tdata->cmd; cmd->callback(cmd->data, cmd->ret); + complete(&tdata->completion); } @@ -464,14 +454,17 @@ int ccp_cmd_queue_thread(void *data) * * @dev: device struct of the CCP */ -struct ccp_device *ccp_alloc_struct(struct device *dev) +struct ccp_device *ccp_alloc_struct(struct sp_device *sp) { + struct device *dev = sp->dev; struct ccp_device *ccp; ccp = devm_kzalloc(dev, sizeof(*ccp), GFP_KERNEL); if (!ccp) return NULL; ccp->dev = dev; + ccp->sp = sp; + ccp->axcache = sp->axcache; INIT_LIST_HEAD(&ccp->cmd); INIT_LIST_HEAD(&ccp->backlog); @@ -486,9 +479,8 @@ struct ccp_device *ccp_alloc_struct(struct device *dev) init_waitqueue_head(&ccp->sb_queue); init_waitqueue_head(&ccp->suspend_queue); - ccp->ord = ccp_increment_unit_ordinal(); - snprintf(ccp->name, MAX_CCP_NAME_LEN, "ccp-%u", ccp->ord); - snprintf(ccp->rngname, MAX_CCP_NAME_LEN, "ccp-%u-rng", ccp->ord); + snprintf(ccp->name, MAX_CCP_NAME_LEN, "ccp-%u", sp->ord); + snprintf(ccp->rngname, MAX_CCP_NAME_LEN, "ccp-%u-rng", sp->ord); return ccp; } @@ -538,55 +530,100 @@ bool ccp_queues_suspended(struct ccp_device *ccp) return ccp->cmd_q_count == suspended; } -#endif -static int __init ccp_mod_init(void) +int ccp_dev_suspend(struct sp_device *sp, pm_message_t state) { -#ifdef CONFIG_X86 - int ret; + struct ccp_device *ccp = sp->ccp_data; + unsigned long flags; + unsigned int i; - ret = ccp_pci_init(); - if (ret) - return ret; + spin_lock_irqsave(&ccp->cmd_lock, flags); - /* Don't leave the driver loaded if init failed */ - if (ccp_present() != 0) { - ccp_pci_exit(); - return -ENODEV; + ccp->suspending = 1; + + /* Wake all the queue kthreads to prepare for suspend */ + for (i = 0; i < ccp->cmd_q_count; i++) + wake_up_process(ccp->cmd_q[i].kthread); + + spin_unlock_irqrestore(&ccp->cmd_lock, flags); + + /* Wait for all queue kthreads to say they're done */ + while (!ccp_queues_suspended(ccp)) + wait_event_interruptible(ccp->suspend_queue, + ccp_queues_suspended(ccp)); + + return 0; +} + +int ccp_dev_resume(struct sp_device *sp) +{ + struct ccp_device *ccp = sp->ccp_data; + unsigned long flags; + unsigned int i; + + spin_lock_irqsave(&ccp->cmd_lock, flags); + + ccp->suspending = 0; + + /* Wake up all the kthreads */ + for (i = 0; i < ccp->cmd_q_count; i++) { + ccp->cmd_q[i].suspended = 0; + wake_up_process(ccp->cmd_q[i].kthread); } + spin_unlock_irqrestore(&ccp->cmd_lock, flags); + return 0; +} #endif -#ifdef CONFIG_ARM64 +int ccp_dev_init(struct sp_device *sp) +{ + struct device *dev = sp->dev; + struct ccp_device *ccp; int ret; - ret = ccp_platform_init(); + ret = -ENOMEM; + ccp = ccp_alloc_struct(sp); + if (!ccp) + goto e_err; + sp->ccp_data = ccp; + + ccp->vdata = (struct ccp_vdata *)sp->dev_vdata->ccp_vdata; + if (!ccp->vdata || !ccp->vdata->version) { + ret = -ENODEV; + dev_err(dev, "missing driver data\n"); + goto e_err; + } + + ccp->use_tasklet = sp->use_tasklet; + + ccp->io_regs = sp->io_map + ccp->vdata->offset; + if (ccp->vdata->setup) + ccp->vdata->setup(ccp); + + ret = ccp->vdata->perform->init(ccp); if (ret) - return ret; + goto e_err; - /* Don't leave the driver loaded if init failed */ - if (ccp_present() != 0) { - ccp_platform_exit(); - return -ENODEV; - } + dev_notice(dev, "ccp enabled\n"); return 0; -#endif - return -ENODEV; +e_err: + sp->ccp_data = NULL; + + dev_notice(dev, "ccp initialization failed\n"); + + return ret; } -static void __exit ccp_mod_exit(void) +void ccp_dev_destroy(struct sp_device *sp) { -#ifdef CONFIG_X86 - ccp_pci_exit(); -#endif + struct ccp_device *ccp = sp->ccp_data; -#ifdef CONFIG_ARM64 - ccp_platform_exit(); -#endif -} + if (!ccp) + return; -module_init(ccp_mod_init); -module_exit(ccp_mod_exit); + ccp->vdata->perform->destroy(ccp); +} diff --git a/drivers/crypto/ccp/ccp-dev.h b/drivers/crypto/ccp/ccp-dev.h index a70154ac7405..6810b65c1939 100644 --- a/drivers/crypto/ccp/ccp-dev.h +++ b/drivers/crypto/ccp/ccp-dev.h @@ -1,7 +1,7 @@ /* * AMD Cryptographic Coprocessor (CCP) driver * - * Copyright (C) 2013,2016 Advanced Micro Devices, Inc. + * Copyright (C) 2013,2017 Advanced Micro Devices, Inc. * * Author: Tom Lendacky <thomas.lendacky@amd.com> * Author: Gary R Hook <gary.hook@amd.com> @@ -27,6 +27,8 @@ #include <linux/irqreturn.h> #include <linux/dmaengine.h> +#include "sp-dev.h" + #define MAX_CCP_NAME_LEN 16 #define MAX_DMAPOOL_NAME_LEN 32 @@ -192,6 +194,7 @@ #define CCP_AES_CTX_SB_COUNT 1 #define CCP_XTS_AES_KEY_SB_COUNT 1 +#define CCP5_XTS_AES_KEY_SB_COUNT 2 #define CCP_XTS_AES_CTX_SB_COUNT 1 #define CCP_DES3_KEY_SB_COUNT 1 @@ -200,6 +203,7 @@ #define CCP_SHA_SB_COUNT 1 #define CCP_RSA_MAX_WIDTH 4096 +#define CCP5_RSA_MAX_WIDTH 16384 #define CCP_PASSTHRU_BLOCKSIZE 256 #define CCP_PASSTHRU_MASKSIZE 32 @@ -344,12 +348,11 @@ struct ccp_device { char rngname[MAX_CCP_NAME_LEN]; struct device *dev; + struct sp_device *sp; /* Bus specific device information */ void *dev_specific; - int (*get_irq)(struct ccp_device *ccp); - void (*free_irq)(struct ccp_device *ccp); unsigned int qim; unsigned int irq; bool use_tasklet; @@ -362,7 +365,6 @@ struct ccp_device { * them. */ struct mutex req_mutex ____cacheline_aligned; - void __iomem *io_map; void __iomem *io_regs; /* Master lists that all cmds are queued on. Because there can be @@ -497,6 +499,7 @@ struct ccp_aes_op { }; struct ccp_xts_aes_op { + enum ccp_aes_type type; enum ccp_aes_action action; enum ccp_xts_aes_unit_size unit_size; }; @@ -626,18 +629,12 @@ struct ccp5_desc { struct dword7 dw7; }; -int ccp_pci_init(void); -void ccp_pci_exit(void); - -int ccp_platform_init(void); -void ccp_platform_exit(void); - void ccp_add_device(struct ccp_device *ccp); void ccp_del_device(struct ccp_device *ccp); extern void ccp_log_error(struct ccp_device *, int); -struct ccp_device *ccp_alloc_struct(struct device *dev); +struct ccp_device *ccp_alloc_struct(struct sp_device *sp); bool ccp_queues_suspended(struct ccp_device *ccp); int ccp_cmd_queue_thread(void *data); int ccp_trng_read(struct hwrng *rng, void *data, size_t max, bool wait); @@ -669,16 +666,7 @@ struct ccp_actions { irqreturn_t (*irqhandler)(int, void *); }; -/* Structure to hold CCP version-specific values */ -struct ccp_vdata { - const unsigned int version; - const unsigned int dma_chan_attr; - void (*setup)(struct ccp_device *); - const struct ccp_actions *perform; - const unsigned int bar; - const unsigned int offset; -}; - +extern const struct ccp_vdata ccpv3_platform; extern const struct ccp_vdata ccpv3; extern const struct ccp_vdata ccpv5a; extern const struct ccp_vdata ccpv5b; diff --git a/drivers/crypto/ccp/ccp-dmaengine.c b/drivers/crypto/ccp/ccp-dmaengine.c index e00be01fbf5a..8b9da58459df 100644 --- a/drivers/crypto/ccp/ccp-dmaengine.c +++ b/drivers/crypto/ccp/ccp-dmaengine.c @@ -1,7 +1,7 @@ /* * AMD Cryptographic Coprocessor (CCP) driver * - * Copyright (C) 2016 Advanced Micro Devices, Inc. + * Copyright (C) 2016,2017 Advanced Micro Devices, Inc. * * Author: Gary R Hook <gary.hook@amd.com> * @@ -223,6 +223,7 @@ static struct ccp_dma_desc *ccp_handle_active_desc(struct ccp_dma_chan *chan, desc->tx_desc.cookie, desc->status); dma_cookie_complete(tx_desc); + dma_descriptor_unmap(tx_desc); } desc = __ccp_next_dma_desc(chan, desc); @@ -230,9 +231,7 @@ static struct ccp_dma_desc *ccp_handle_active_desc(struct ccp_dma_chan *chan, spin_unlock_irqrestore(&chan->lock, flags); if (tx_desc) { - if (tx_desc->callback && - (tx_desc->flags & DMA_PREP_INTERRUPT)) - tx_desc->callback(tx_desc->callback_param); + dmaengine_desc_get_callback_invoke(tx_desc, NULL); dma_run_dependencies(tx_desc); } @@ -502,27 +501,6 @@ static struct dma_async_tx_descriptor *ccp_prep_dma_memcpy( return &desc->tx_desc; } -static struct dma_async_tx_descriptor *ccp_prep_dma_sg( - struct dma_chan *dma_chan, struct scatterlist *dst_sg, - unsigned int dst_nents, struct scatterlist *src_sg, - unsigned int src_nents, unsigned long flags) -{ - struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, - dma_chan); - struct ccp_dma_desc *desc; - - dev_dbg(chan->ccp->dev, - "%s - src=%p, src_nents=%u dst=%p, dst_nents=%u, flags=%#lx\n", - __func__, src_sg, src_nents, dst_sg, dst_nents, flags); - - desc = ccp_create_desc(dma_chan, dst_sg, dst_nents, src_sg, src_nents, - flags); - if (!desc) - return NULL; - - return &desc->tx_desc; -} - static struct dma_async_tx_descriptor *ccp_prep_dma_interrupt( struct dma_chan *dma_chan, unsigned long flags) { @@ -704,7 +682,6 @@ int ccp_dmaengine_register(struct ccp_device *ccp) dma_dev->directions = DMA_MEM_TO_MEM; dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); - dma_cap_set(DMA_SG, dma_dev->cap_mask); dma_cap_set(DMA_INTERRUPT, dma_dev->cap_mask); /* The DMA channels for this device can be set to public or private, @@ -740,7 +717,6 @@ int ccp_dmaengine_register(struct ccp_device *ccp) dma_dev->device_free_chan_resources = ccp_free_chan_resources; dma_dev->device_prep_dma_memcpy = ccp_prep_dma_memcpy; - dma_dev->device_prep_dma_sg = ccp_prep_dma_sg; dma_dev->device_prep_dma_interrupt = ccp_prep_dma_interrupt; dma_dev->device_issue_pending = ccp_issue_pending; dma_dev->device_tx_status = ccp_tx_status; diff --git a/drivers/crypto/ccp/ccp-ops.c b/drivers/crypto/ccp/ccp-ops.c index c0dfdacbdff5..406b95329b3d 100644 --- a/drivers/crypto/ccp/ccp-ops.c +++ b/drivers/crypto/ccp/ccp-ops.c @@ -1,7 +1,7 @@ /* * AMD Cryptographic Coprocessor (CCP) driver * - * Copyright (C) 2013,2016 Advanced Micro Devices, Inc. + * Copyright (C) 2013,2017 Advanced Micro Devices, Inc. * * Author: Tom Lendacky <thomas.lendacky@amd.com> * Author: Gary R Hook <gary.hook@amd.com> @@ -168,7 +168,7 @@ static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa, wa->dma.address = dma_map_single(wa->dev, wa->address, len, dir); - if (!wa->dma.address) + if (dma_mapping_error(wa->dev, wa->dma.address)) return -ENOMEM; wa->dma.length = len; @@ -1038,6 +1038,8 @@ static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_op op; unsigned int unit_size, dm_offset; bool in_place = false; + unsigned int sb_count; + enum ccp_aes_type aestype; int ret; switch (xts->unit_size) { @@ -1061,7 +1063,11 @@ static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q, return -EINVAL; } - if (xts->key_len != AES_KEYSIZE_128) + if (xts->key_len == AES_KEYSIZE_128) + aestype = CCP_AES_TYPE_128; + else if (xts->key_len == AES_KEYSIZE_256) + aestype = CCP_AES_TYPE_256; + else return -EINVAL; if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1))) @@ -1083,23 +1089,44 @@ static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q, op.sb_key = cmd_q->sb_key; op.sb_ctx = cmd_q->sb_ctx; op.init = 1; + op.u.xts.type = aestype; op.u.xts.action = xts->action; op.u.xts.unit_size = xts->unit_size; - /* All supported key sizes fit in a single (32-byte) SB entry - * and must be in little endian format. Use the 256-bit byte - * swap passthru option to convert from big endian to little - * endian. + /* A version 3 device only supports 128-bit keys, which fits into a + * single SB entry. A version 5 device uses a 512-bit vector, so two + * SB entries. */ + if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) + sb_count = CCP_XTS_AES_KEY_SB_COUNT; + else + sb_count = CCP5_XTS_AES_KEY_SB_COUNT; ret = ccp_init_dm_workarea(&key, cmd_q, - CCP_XTS_AES_KEY_SB_COUNT * CCP_SB_BYTES, + sb_count * CCP_SB_BYTES, DMA_TO_DEVICE); if (ret) return ret; - dm_offset = CCP_SB_BYTES - AES_KEYSIZE_128; - ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len); - ccp_set_dm_area(&key, 0, xts->key, dm_offset, xts->key_len); + if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) { + /* All supported key sizes must be in little endian format. + * Use the 256-bit byte swap passthru option to convert from + * big endian to little endian. + */ + dm_offset = CCP_SB_BYTES - AES_KEYSIZE_128; + ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len); + ccp_set_dm_area(&key, 0, xts->key, xts->key_len, xts->key_len); + } else { + /* Version 5 CCPs use a 512-bit space for the key: each portion + * occupies 256 bits, or one entire slot, and is zero-padded. + */ + unsigned int pad; + + dm_offset = CCP_SB_BYTES; + pad = dm_offset - xts->key_len; + ccp_set_dm_area(&key, pad, xts->key, 0, xts->key_len); + ccp_set_dm_area(&key, dm_offset + pad, xts->key, xts->key_len, + xts->key_len); + } ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key, CCP_PASSTHRU_BYTESWAP_256BIT); if (ret) { @@ -1731,42 +1758,53 @@ e_ctx: static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) { struct ccp_rsa_engine *rsa = &cmd->u.rsa; - struct ccp_dm_workarea exp, src; - struct ccp_data dst; + struct ccp_dm_workarea exp, src, dst; struct ccp_op op; unsigned int sb_count, i_len, o_len; int ret; - if (rsa->key_size > CCP_RSA_MAX_WIDTH) + /* Check against the maximum allowable size, in bits */ + if (rsa->key_size > cmd_q->ccp->vdata->rsamax) return -EINVAL; if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst) return -EINVAL; + memset(&op, 0, sizeof(op)); + op.cmd_q = cmd_q; + op.jobid = CCP_NEW_JOBID(cmd_q->ccp); + /* The RSA modulus must precede the message being acted upon, so * it must be copied to a DMA area where the message and the * modulus can be concatenated. Therefore the input buffer * length required is twice the output buffer length (which - * must be a multiple of 256-bits). + * must be a multiple of 256-bits). Compute o_len, i_len in bytes. + * Buffer sizes must be a multiple of 32 bytes; rounding up may be + * required. */ - o_len = ((rsa->key_size + 255) / 256) * 32; + o_len = 32 * ((rsa->key_size + 255) / 256); i_len = o_len * 2; - sb_count = o_len / CCP_SB_BYTES; - - memset(&op, 0, sizeof(op)); - op.cmd_q = cmd_q; - op.jobid = ccp_gen_jobid(cmd_q->ccp); - op.sb_key = cmd_q->ccp->vdata->perform->sballoc(cmd_q, sb_count); - - if (!op.sb_key) - return -EIO; + sb_count = 0; + if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) { + /* sb_count is the number of storage block slots required + * for the modulus. + */ + sb_count = o_len / CCP_SB_BYTES; + op.sb_key = cmd_q->ccp->vdata->perform->sballoc(cmd_q, + sb_count); + if (!op.sb_key) + return -EIO; + } else { + /* A version 5 device allows a modulus size that will not fit + * in the LSB, so the command will transfer it from memory. + * Set the sb key to the default, even though it's not used. + */ + op.sb_key = cmd_q->sb_key; + } - /* The RSA exponent may span multiple (32-byte) SB entries and must - * be in little endian format. Reverse copy each 32-byte chunk - * of the exponent (En chunk to E0 chunk, E(n-1) chunk to E1 chunk) - * and each byte within that chunk and do not perform any byte swap - * operations on the passthru operation. + /* The RSA exponent must be in little endian format. Reverse its + * byte order. */ ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE); if (ret) @@ -1775,11 +1813,22 @@ static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) ret = ccp_reverse_set_dm_area(&exp, 0, rsa->exp, 0, rsa->exp_len); if (ret) goto e_exp; - ret = ccp_copy_to_sb(cmd_q, &exp, op.jobid, op.sb_key, - CCP_PASSTHRU_BYTESWAP_NOOP); - if (ret) { - cmd->engine_error = cmd_q->cmd_error; - goto e_exp; + + if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) { + /* Copy the exponent to the local storage block, using + * as many 32-byte blocks as were allocated above. It's + * already little endian, so no further change is required. + */ + ret = ccp_copy_to_sb(cmd_q, &exp, op.jobid, op.sb_key, + CCP_PASSTHRU_BYTESWAP_NOOP); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_exp; + } + } else { + /* The exponent can be retrieved from memory via DMA. */ + op.exp.u.dma.address = exp.dma.address; + op.exp.u.dma.offset = 0; } /* Concatenate the modulus and the message. Both the modulus and @@ -1798,8 +1847,7 @@ static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) goto e_src; /* Prepare the output area for the operation */ - ret = ccp_init_data(&dst, cmd_q, rsa->dst, rsa->mod_len, - o_len, DMA_FROM_DEVICE); + ret = ccp_init_dm_workarea(&dst, cmd_q, o_len, DMA_FROM_DEVICE); if (ret) goto e_src; @@ -1807,7 +1855,7 @@ static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) op.src.u.dma.address = src.dma.address; op.src.u.dma.offset = 0; op.src.u.dma.length = i_len; - op.dst.u.dma.address = dst.dm_wa.dma.address; + op.dst.u.dma.address = dst.dma.address; op.dst.u.dma.offset = 0; op.dst.u.dma.length = o_len; @@ -1820,10 +1868,10 @@ static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) goto e_dst; } - ccp_reverse_get_dm_area(&dst.dm_wa, 0, rsa->dst, 0, rsa->mod_len); + ccp_reverse_get_dm_area(&dst, 0, rsa->dst, 0, rsa->mod_len); e_dst: - ccp_free_data(&dst, cmd_q); + ccp_dm_free(&dst); e_src: ccp_dm_free(&src); @@ -1832,7 +1880,8 @@ e_exp: ccp_dm_free(&exp); e_sb: - cmd_q->ccp->vdata->perform->sbfree(cmd_q, op.sb_key, sb_count); + if (sb_count) + cmd_q->ccp->vdata->perform->sbfree(cmd_q, op.sb_key, sb_count); return ret; } @@ -1992,7 +2041,7 @@ static int ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q, memset(&op, 0, sizeof(op)); op.cmd_q = cmd_q; - op.jobid = ccp_gen_jobid(cmd_q->ccp); + op.jobid = CCP_NEW_JOBID(cmd_q->ccp); if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { /* Load the mask */ diff --git a/drivers/crypto/ccp/ccp-pci.c b/drivers/crypto/ccp/ccp-pci.c deleted file mode 100644 index e880d4cf4ada..000000000000 --- a/drivers/crypto/ccp/ccp-pci.c +++ /dev/null @@ -1,356 +0,0 @@ -/* - * AMD Cryptographic Coprocessor (CCP) driver - * - * Copyright (C) 2013,2016 Advanced Micro Devices, Inc. - * - * Author: Tom Lendacky <thomas.lendacky@amd.com> - * Author: Gary R Hook <gary.hook@amd.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - */ - -#include <linux/module.h> -#include <linux/kernel.h> -#include <linux/device.h> -#include <linux/pci.h> -#include <linux/pci_ids.h> -#include <linux/dma-mapping.h> -#include <linux/kthread.h> -#include <linux/sched.h> -#include <linux/interrupt.h> -#include <linux/spinlock.h> -#include <linux/delay.h> -#include <linux/ccp.h> - -#include "ccp-dev.h" - -#define MSIX_VECTORS 2 - -struct ccp_msix { - u32 vector; - char name[16]; -}; - -struct ccp_pci { - int msix_count; - struct ccp_msix msix[MSIX_VECTORS]; -}; - -static int ccp_get_msix_irqs(struct ccp_device *ccp) -{ - struct ccp_pci *ccp_pci = ccp->dev_specific; - struct device *dev = ccp->dev; - struct pci_dev *pdev = to_pci_dev(dev); - struct msix_entry msix_entry[MSIX_VECTORS]; - unsigned int name_len = sizeof(ccp_pci->msix[0].name) - 1; - int v, ret; - - for (v = 0; v < ARRAY_SIZE(msix_entry); v++) - msix_entry[v].entry = v; - - ret = pci_enable_msix_range(pdev, msix_entry, 1, v); - if (ret < 0) - return ret; - - ccp_pci->msix_count = ret; - for (v = 0; v < ccp_pci->msix_count; v++) { - /* Set the interrupt names and request the irqs */ - snprintf(ccp_pci->msix[v].name, name_len, "%s-%u", - ccp->name, v); - ccp_pci->msix[v].vector = msix_entry[v].vector; - ret = request_irq(ccp_pci->msix[v].vector, - ccp->vdata->perform->irqhandler, - 0, ccp_pci->msix[v].name, dev); - if (ret) { - dev_notice(dev, "unable to allocate MSI-X IRQ (%d)\n", - ret); - goto e_irq; - } - } - ccp->use_tasklet = true; - - return 0; - -e_irq: - while (v--) - free_irq(ccp_pci->msix[v].vector, dev); - - pci_disable_msix(pdev); - - ccp_pci->msix_count = 0; - - return ret; -} - -static int ccp_get_msi_irq(struct ccp_device *ccp) -{ - struct device *dev = ccp->dev; - struct pci_dev *pdev = to_pci_dev(dev); - int ret; - - ret = pci_enable_msi(pdev); - if (ret) - return ret; - - ccp->irq = pdev->irq; - ret = request_irq(ccp->irq, ccp->vdata->perform->irqhandler, 0, - ccp->name, dev); - if (ret) { - dev_notice(dev, "unable to allocate MSI IRQ (%d)\n", ret); - goto e_msi; - } - ccp->use_tasklet = true; - - return 0; - -e_msi: - pci_disable_msi(pdev); - - return ret; -} - -static int ccp_get_irqs(struct ccp_device *ccp) -{ - struct device *dev = ccp->dev; - int ret; - - ret = ccp_get_msix_irqs(ccp); - if (!ret) - return 0; - - /* Couldn't get MSI-X vectors, try MSI */ - dev_notice(dev, "could not enable MSI-X (%d), trying MSI\n", ret); - ret = ccp_get_msi_irq(ccp); - if (!ret) - return 0; - - /* Couldn't get MSI interrupt */ - dev_notice(dev, "could not enable MSI (%d)\n", ret); - - return ret; -} - -static void ccp_free_irqs(struct ccp_device *ccp) -{ - struct ccp_pci *ccp_pci = ccp->dev_specific; - struct device *dev = ccp->dev; - struct pci_dev *pdev = to_pci_dev(dev); - - if (ccp_pci->msix_count) { - while (ccp_pci->msix_count--) - free_irq(ccp_pci->msix[ccp_pci->msix_count].vector, - dev); - pci_disable_msix(pdev); - } else if (ccp->irq) { - free_irq(ccp->irq, dev); - pci_disable_msi(pdev); - } - ccp->irq = 0; -} - -static int ccp_find_mmio_area(struct ccp_device *ccp) -{ - struct device *dev = ccp->dev; - struct pci_dev *pdev = to_pci_dev(dev); - resource_size_t io_len; - unsigned long io_flags; - - io_flags = pci_resource_flags(pdev, ccp->vdata->bar); - io_len = pci_resource_len(pdev, ccp->vdata->bar); - if ((io_flags & IORESOURCE_MEM) && - (io_len >= (ccp->vdata->offset + 0x800))) - return ccp->vdata->bar; - - return -EIO; -} - -static int ccp_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) -{ - struct ccp_device *ccp; - struct ccp_pci *ccp_pci; - struct device *dev = &pdev->dev; - unsigned int bar; - int ret; - - ret = -ENOMEM; - ccp = ccp_alloc_struct(dev); - if (!ccp) - goto e_err; - - ccp_pci = devm_kzalloc(dev, sizeof(*ccp_pci), GFP_KERNEL); - if (!ccp_pci) - goto e_err; - - ccp->dev_specific = ccp_pci; - ccp->vdata = (struct ccp_vdata *)id->driver_data; - if (!ccp->vdata || !ccp->vdata->version) { - ret = -ENODEV; - dev_err(dev, "missing driver data\n"); - goto e_err; - } - ccp->get_irq = ccp_get_irqs; - ccp->free_irq = ccp_free_irqs; - - ret = pci_request_regions(pdev, "ccp"); - if (ret) { - dev_err(dev, "pci_request_regions failed (%d)\n", ret); - goto e_err; - } - - ret = pci_enable_device(pdev); - if (ret) { - dev_err(dev, "pci_enable_device failed (%d)\n", ret); - goto e_regions; - } - - pci_set_master(pdev); - - ret = ccp_find_mmio_area(ccp); - if (ret < 0) - goto e_device; - bar = ret; - - ret = -EIO; - ccp->io_map = pci_iomap(pdev, bar, 0); - if (!ccp->io_map) { - dev_err(dev, "pci_iomap failed\n"); - goto e_device; - } - ccp->io_regs = ccp->io_map + ccp->vdata->offset; - - ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48)); - if (ret) { - ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); - if (ret) { - dev_err(dev, "dma_set_mask_and_coherent failed (%d)\n", - ret); - goto e_iomap; - } - } - - dev_set_drvdata(dev, ccp); - - if (ccp->vdata->setup) - ccp->vdata->setup(ccp); - - ret = ccp->vdata->perform->init(ccp); - if (ret) - goto e_iomap; - - dev_notice(dev, "enabled\n"); - - return 0; - -e_iomap: - pci_iounmap(pdev, ccp->io_map); - -e_device: - pci_disable_device(pdev); - -e_regions: - pci_release_regions(pdev); - -e_err: - dev_notice(dev, "initialization failed\n"); - return ret; -} - -static void ccp_pci_remove(struct pci_dev *pdev) -{ - struct device *dev = &pdev->dev; - struct ccp_device *ccp = dev_get_drvdata(dev); - - if (!ccp) - return; - - ccp->vdata->perform->destroy(ccp); - - pci_iounmap(pdev, ccp->io_map); - - pci_disable_device(pdev); - - pci_release_regions(pdev); - - dev_notice(dev, "disabled\n"); -} - -#ifdef CONFIG_PM -static int ccp_pci_suspend(struct pci_dev *pdev, pm_message_t state) -{ - struct device *dev = &pdev->dev; - struct ccp_device *ccp = dev_get_drvdata(dev); - unsigned long flags; - unsigned int i; - - spin_lock_irqsave(&ccp->cmd_lock, flags); - - ccp->suspending = 1; - - /* Wake all the queue kthreads to prepare for suspend */ - for (i = 0; i < ccp->cmd_q_count; i++) - wake_up_process(ccp->cmd_q[i].kthread); - - spin_unlock_irqrestore(&ccp->cmd_lock, flags); - - /* Wait for all queue kthreads to say they're done */ - while (!ccp_queues_suspended(ccp)) - wait_event_interruptible(ccp->suspend_queue, - ccp_queues_suspended(ccp)); - - return 0; -} - -static int ccp_pci_resume(struct pci_dev *pdev) -{ - struct device *dev = &pdev->dev; - struct ccp_device *ccp = dev_get_drvdata(dev); - unsigned long flags; - unsigned int i; - - spin_lock_irqsave(&ccp->cmd_lock, flags); - - ccp->suspending = 0; - - /* Wake up all the kthreads */ - for (i = 0; i < ccp->cmd_q_count; i++) { - ccp->cmd_q[i].suspended = 0; - wake_up_process(ccp->cmd_q[i].kthread); - } - - spin_unlock_irqrestore(&ccp->cmd_lock, flags); - - return 0; -} -#endif - -static const struct pci_device_id ccp_pci_table[] = { - { PCI_VDEVICE(AMD, 0x1537), (kernel_ulong_t)&ccpv3 }, - { PCI_VDEVICE(AMD, 0x1456), (kernel_ulong_t)&ccpv5a }, - { PCI_VDEVICE(AMD, 0x1468), (kernel_ulong_t)&ccpv5b }, - /* Last entry must be zero */ - { 0, } -}; -MODULE_DEVICE_TABLE(pci, ccp_pci_table); - -static struct pci_driver ccp_pci_driver = { - .name = "ccp", - .id_table = ccp_pci_table, - .probe = ccp_pci_probe, - .remove = ccp_pci_remove, -#ifdef CONFIG_PM - .suspend = ccp_pci_suspend, - .resume = ccp_pci_resume, -#endif -}; - -int ccp_pci_init(void) -{ - return pci_register_driver(&ccp_pci_driver); -} - -void ccp_pci_exit(void) -{ - pci_unregister_driver(&ccp_pci_driver); -} diff --git a/drivers/crypto/ccp/ccp-platform.c b/drivers/crypto/ccp/ccp-platform.c deleted file mode 100644 index e26969e601ad..000000000000 --- a/drivers/crypto/ccp/ccp-platform.c +++ /dev/null @@ -1,293 +0,0 @@ -/* - * AMD Cryptographic Coprocessor (CCP) driver - * - * Copyright (C) 2014,2016 Advanced Micro Devices, Inc. - * - * Author: Tom Lendacky <thomas.lendacky@amd.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - */ - -#include <linux/module.h> -#include <linux/kernel.h> -#include <linux/device.h> -#include <linux/platform_device.h> -#include <linux/ioport.h> -#include <linux/dma-mapping.h> -#include <linux/kthread.h> -#include <linux/sched.h> -#include <linux/interrupt.h> -#include <linux/spinlock.h> -#include <linux/delay.h> -#include <linux/ccp.h> -#include <linux/of.h> -#include <linux/of_address.h> -#include <linux/acpi.h> - -#include "ccp-dev.h" - -struct ccp_platform { - int coherent; -}; - -static const struct acpi_device_id ccp_acpi_match[]; -static const struct of_device_id ccp_of_match[]; - -static struct ccp_vdata *ccp_get_of_version(struct platform_device *pdev) -{ -#ifdef CONFIG_OF - const struct of_device_id *match; - - match = of_match_node(ccp_of_match, pdev->dev.of_node); - if (match && match->data) - return (struct ccp_vdata *)match->data; -#endif - return NULL; -} - -static struct ccp_vdata *ccp_get_acpi_version(struct platform_device *pdev) -{ -#ifdef CONFIG_ACPI - const struct acpi_device_id *match; - - match = acpi_match_device(ccp_acpi_match, &pdev->dev); - if (match && match->driver_data) - return (struct ccp_vdata *)match->driver_data; -#endif - return NULL; -} - -static int ccp_get_irq(struct ccp_device *ccp) -{ - struct device *dev = ccp->dev; - struct platform_device *pdev = to_platform_device(dev); - int ret; - - ret = platform_get_irq(pdev, 0); - if (ret < 0) - return ret; - - ccp->irq = ret; - ret = request_irq(ccp->irq, ccp->vdata->perform->irqhandler, 0, - ccp->name, dev); - if (ret) { - dev_notice(dev, "unable to allocate IRQ (%d)\n", ret); - return ret; - } - - return 0; -} - -static int ccp_get_irqs(struct ccp_device *ccp) -{ - struct device *dev = ccp->dev; - int ret; - - ret = ccp_get_irq(ccp); - if (!ret) - return 0; - - /* Couldn't get an interrupt */ - dev_notice(dev, "could not enable interrupts (%d)\n", ret); - - return ret; -} - -static void ccp_free_irqs(struct ccp_device *ccp) -{ - struct device *dev = ccp->dev; - - free_irq(ccp->irq, dev); -} - -static struct resource *ccp_find_mmio_area(struct ccp_device *ccp) -{ - struct device *dev = ccp->dev; - struct platform_device *pdev = to_platform_device(dev); - struct resource *ior; - - ior = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (ior && (resource_size(ior) >= 0x800)) - return ior; - - return NULL; -} - -static int ccp_platform_probe(struct platform_device *pdev) -{ - struct ccp_device *ccp; - struct ccp_platform *ccp_platform; - struct device *dev = &pdev->dev; - enum dev_dma_attr attr; - struct resource *ior; - int ret; - - ret = -ENOMEM; - ccp = ccp_alloc_struct(dev); - if (!ccp) - goto e_err; - - ccp_platform = devm_kzalloc(dev, sizeof(*ccp_platform), GFP_KERNEL); - if (!ccp_platform) - goto e_err; - - ccp->dev_specific = ccp_platform; - ccp->vdata = pdev->dev.of_node ? ccp_get_of_version(pdev) - : ccp_get_acpi_version(pdev); - if (!ccp->vdata || !ccp->vdata->version) { - ret = -ENODEV; - dev_err(dev, "missing driver data\n"); - goto e_err; - } - ccp->get_irq = ccp_get_irqs; - ccp->free_irq = ccp_free_irqs; - - ior = ccp_find_mmio_area(ccp); - ccp->io_map = devm_ioremap_resource(dev, ior); - if (IS_ERR(ccp->io_map)) { - ret = PTR_ERR(ccp->io_map); - goto e_err; - } - ccp->io_regs = ccp->io_map; - - attr = device_get_dma_attr(dev); - if (attr == DEV_DMA_NOT_SUPPORTED) { - dev_err(dev, "DMA is not supported"); - goto e_err; - } - - ccp_platform->coherent = (attr == DEV_DMA_COHERENT); - if (ccp_platform->coherent) - ccp->axcache = CACHE_WB_NO_ALLOC; - else - ccp->axcache = CACHE_NONE; - - ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48)); - if (ret) { - dev_err(dev, "dma_set_mask_and_coherent failed (%d)\n", ret); - goto e_err; - } - - dev_set_drvdata(dev, ccp); - - ret = ccp->vdata->perform->init(ccp); - if (ret) - goto e_err; - - dev_notice(dev, "enabled\n"); - - return 0; - -e_err: - dev_notice(dev, "initialization failed\n"); - return ret; -} - -static int ccp_platform_remove(struct platform_device *pdev) -{ - struct device *dev = &pdev->dev; - struct ccp_device *ccp = dev_get_drvdata(dev); - - ccp->vdata->perform->destroy(ccp); - - dev_notice(dev, "disabled\n"); - - return 0; -} - -#ifdef CONFIG_PM -static int ccp_platform_suspend(struct platform_device *pdev, - pm_message_t state) -{ - struct device *dev = &pdev->dev; - struct ccp_device *ccp = dev_get_drvdata(dev); - unsigned long flags; - unsigned int i; - - spin_lock_irqsave(&ccp->cmd_lock, flags); - - ccp->suspending = 1; - - /* Wake all the queue kthreads to prepare for suspend */ - for (i = 0; i < ccp->cmd_q_count; i++) - wake_up_process(ccp->cmd_q[i].kthread); - - spin_unlock_irqrestore(&ccp->cmd_lock, flags); - - /* Wait for all queue kthreads to say they're done */ - while (!ccp_queues_suspended(ccp)) - wait_event_interruptible(ccp->suspend_queue, - ccp_queues_suspended(ccp)); - - return 0; -} - -static int ccp_platform_resume(struct platform_device *pdev) -{ - struct device *dev = &pdev->dev; - struct ccp_device *ccp = dev_get_drvdata(dev); - unsigned long flags; - unsigned int i; - - spin_lock_irqsave(&ccp->cmd_lock, flags); - - ccp->suspending = 0; - - /* Wake up all the kthreads */ - for (i = 0; i < ccp->cmd_q_count; i++) { - ccp->cmd_q[i].suspended = 0; - wake_up_process(ccp->cmd_q[i].kthread); - } - - spin_unlock_irqrestore(&ccp->cmd_lock, flags); - - return 0; -} -#endif - -#ifdef CONFIG_ACPI -static const struct acpi_device_id ccp_acpi_match[] = { - { "AMDI0C00", (kernel_ulong_t)&ccpv3 }, - { }, -}; -MODULE_DEVICE_TABLE(acpi, ccp_acpi_match); -#endif - -#ifdef CONFIG_OF -static const struct of_device_id ccp_of_match[] = { - { .compatible = "amd,ccp-seattle-v1a", - .data = (const void *)&ccpv3 }, - { }, -}; -MODULE_DEVICE_TABLE(of, ccp_of_match); -#endif - -static struct platform_driver ccp_platform_driver = { - .driver = { - .name = "ccp", -#ifdef CONFIG_ACPI - .acpi_match_table = ccp_acpi_match, -#endif -#ifdef CONFIG_OF - .of_match_table = ccp_of_match, -#endif - }, - .probe = ccp_platform_probe, - .remove = ccp_platform_remove, -#ifdef CONFIG_PM - .suspend = ccp_platform_suspend, - .resume = ccp_platform_resume, -#endif -}; - -int ccp_platform_init(void) -{ - return platform_driver_register(&ccp_platform_driver); -} - -void ccp_platform_exit(void) -{ - platform_driver_unregister(&ccp_platform_driver); -} diff --git a/drivers/crypto/ccp/sp-dev.c b/drivers/crypto/ccp/sp-dev.c new file mode 100644 index 000000000000..bef387c8abfd --- /dev/null +++ b/drivers/crypto/ccp/sp-dev.c @@ -0,0 +1,277 @@ +/* + * AMD Secure Processor driver + * + * Copyright (C) 2017 Advanced Micro Devices, Inc. + * + * Author: Tom Lendacky <thomas.lendacky@amd.com> + * Author: Gary R Hook <gary.hook@amd.com> + * Author: Brijesh Singh <brijesh.singh@amd.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/kthread.h> +#include <linux/sched.h> +#include <linux/interrupt.h> +#include <linux/spinlock.h> +#include <linux/spinlock_types.h> +#include <linux/types.h> +#include <linux/ccp.h> + +#include "ccp-dev.h" +#include "sp-dev.h" + +MODULE_AUTHOR("Tom Lendacky <thomas.lendacky@amd.com>"); +MODULE_AUTHOR("Gary R Hook <gary.hook@amd.com>"); +MODULE_LICENSE("GPL"); +MODULE_VERSION("1.1.0"); +MODULE_DESCRIPTION("AMD Secure Processor driver"); + +/* List of SPs, SP count, read-write access lock, and access functions + * + * Lock structure: get sp_unit_lock for reading whenever we need to + * examine the SP list. + */ +static DEFINE_RWLOCK(sp_unit_lock); +static LIST_HEAD(sp_units); + +/* Ever-increasing value to produce unique unit numbers */ +static atomic_t sp_ordinal; + +static void sp_add_device(struct sp_device *sp) +{ + unsigned long flags; + + write_lock_irqsave(&sp_unit_lock, flags); + + list_add_tail(&sp->entry, &sp_units); + + write_unlock_irqrestore(&sp_unit_lock, flags); +} + +static void sp_del_device(struct sp_device *sp) +{ + unsigned long flags; + + write_lock_irqsave(&sp_unit_lock, flags); + + list_del(&sp->entry); + + write_unlock_irqrestore(&sp_unit_lock, flags); +} + +static irqreturn_t sp_irq_handler(int irq, void *data) +{ + struct sp_device *sp = data; + + if (sp->ccp_irq_handler) + sp->ccp_irq_handler(irq, sp->ccp_irq_data); + + if (sp->psp_irq_handler) + sp->psp_irq_handler(irq, sp->psp_irq_data); + + return IRQ_HANDLED; +} + +int sp_request_ccp_irq(struct sp_device *sp, irq_handler_t handler, + const char *name, void *data) +{ + int ret; + + if ((sp->psp_irq == sp->ccp_irq) && sp->dev_vdata->psp_vdata) { + /* Need a common routine to manage all interrupts */ + sp->ccp_irq_data = data; + sp->ccp_irq_handler = handler; + + if (!sp->irq_registered) { + ret = request_irq(sp->ccp_irq, sp_irq_handler, 0, + sp->name, sp); + if (ret) + return ret; + + sp->irq_registered = true; + } + } else { + /* Each sub-device can manage it's own interrupt */ + ret = request_irq(sp->ccp_irq, handler, 0, name, data); + if (ret) + return ret; + } + + return 0; +} + +int sp_request_psp_irq(struct sp_device *sp, irq_handler_t handler, + const char *name, void *data) +{ + int ret; + + if ((sp->psp_irq == sp->ccp_irq) && sp->dev_vdata->ccp_vdata) { + /* Need a common routine to manage all interrupts */ + sp->psp_irq_data = data; + sp->psp_irq_handler = handler; + + if (!sp->irq_registered) { + ret = request_irq(sp->psp_irq, sp_irq_handler, 0, + sp->name, sp); + if (ret) + return ret; + + sp->irq_registered = true; + } + } else { + /* Each sub-device can manage it's own interrupt */ + ret = request_irq(sp->psp_irq, handler, 0, name, data); + if (ret) + return ret; + } + + return 0; +} + +void sp_free_ccp_irq(struct sp_device *sp, void *data) +{ + if ((sp->psp_irq == sp->ccp_irq) && sp->dev_vdata->psp_vdata) { + /* Using common routine to manage all interrupts */ + if (!sp->psp_irq_handler) { + /* Nothing else using it, so free it */ + free_irq(sp->ccp_irq, sp); + + sp->irq_registered = false; + } + + sp->ccp_irq_handler = NULL; + sp->ccp_irq_data = NULL; + } else { + /* Each sub-device can manage it's own interrupt */ + free_irq(sp->ccp_irq, data); + } +} + +void sp_free_psp_irq(struct sp_device *sp, void *data) +{ + if ((sp->psp_irq == sp->ccp_irq) && sp->dev_vdata->ccp_vdata) { + /* Using common routine to manage all interrupts */ + if (!sp->ccp_irq_handler) { + /* Nothing else using it, so free it */ + free_irq(sp->psp_irq, sp); + + sp->irq_registered = false; + } + + sp->psp_irq_handler = NULL; + sp->psp_irq_data = NULL; + } else { + /* Each sub-device can manage it's own interrupt */ + free_irq(sp->psp_irq, data); + } +} + +/** + * sp_alloc_struct - allocate and initialize the sp_device struct + * + * @dev: device struct of the SP + */ +struct sp_device *sp_alloc_struct(struct device *dev) +{ + struct sp_device *sp; + + sp = devm_kzalloc(dev, sizeof(*sp), GFP_KERNEL); + if (!sp) + return NULL; + + sp->dev = dev; + sp->ord = atomic_inc_return(&sp_ordinal); + snprintf(sp->name, SP_MAX_NAME_LEN, "sp-%u", sp->ord); + + return sp; +} + +int sp_init(struct sp_device *sp) +{ + sp_add_device(sp); + + if (sp->dev_vdata->ccp_vdata) + ccp_dev_init(sp); + + return 0; +} + +void sp_destroy(struct sp_device *sp) +{ + if (sp->dev_vdata->ccp_vdata) + ccp_dev_destroy(sp); + + sp_del_device(sp); +} + +#ifdef CONFIG_PM +int sp_suspend(struct sp_device *sp, pm_message_t state) +{ + int ret; + + if (sp->dev_vdata->ccp_vdata) { + ret = ccp_dev_suspend(sp, state); + if (ret) + return ret; + } + + return 0; +} + +int sp_resume(struct sp_device *sp) +{ + int ret; + + if (sp->dev_vdata->ccp_vdata) { + ret = ccp_dev_resume(sp); + if (ret) + return ret; + } + + return 0; +} +#endif + +static int __init sp_mod_init(void) +{ +#ifdef CONFIG_X86 + int ret; + + ret = sp_pci_init(); + if (ret) + return ret; + + return 0; +#endif + +#ifdef CONFIG_ARM64 + int ret; + + ret = sp_platform_init(); + if (ret) + return ret; + + return 0; +#endif + + return -ENODEV; +} + +static void __exit sp_mod_exit(void) +{ +#ifdef CONFIG_X86 + sp_pci_exit(); +#endif + +#ifdef CONFIG_ARM64 + sp_platform_exit(); +#endif +} + +module_init(sp_mod_init); +module_exit(sp_mod_exit); diff --git a/drivers/crypto/ccp/sp-dev.h b/drivers/crypto/ccp/sp-dev.h new file mode 100644 index 000000000000..5ab486ade1ad --- /dev/null +++ b/drivers/crypto/ccp/sp-dev.h @@ -0,0 +1,133 @@ +/* + * AMD Secure Processor driver + * + * Copyright (C) 2017 Advanced Micro Devices, Inc. + * + * Author: Tom Lendacky <thomas.lendacky@amd.com> + * Author: Gary R Hook <gary.hook@amd.com> + * Author: Brijesh Singh <brijesh.singh@amd.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#ifndef __SP_DEV_H__ +#define __SP_DEV_H__ + +#include <linux/device.h> +#include <linux/pci.h> +#include <linux/spinlock.h> +#include <linux/mutex.h> +#include <linux/list.h> +#include <linux/wait.h> +#include <linux/dmapool.h> +#include <linux/hw_random.h> +#include <linux/bitops.h> +#include <linux/interrupt.h> +#include <linux/irqreturn.h> + +#define SP_MAX_NAME_LEN 32 + +#define CACHE_NONE 0x00 +#define CACHE_WB_NO_ALLOC 0xb7 + +/* Structure to hold CCP device data */ +struct ccp_device; +struct ccp_vdata { + const unsigned int version; + const unsigned int dma_chan_attr; + void (*setup)(struct ccp_device *); + const struct ccp_actions *perform; + const unsigned int offset; + const unsigned int rsamax; +}; +/* Structure to hold SP device data */ +struct sp_dev_vdata { + const unsigned int bar; + + const struct ccp_vdata *ccp_vdata; + void *psp_vdata; +}; + +struct sp_device { + struct list_head entry; + + struct device *dev; + + struct sp_dev_vdata *dev_vdata; + unsigned int ord; + char name[SP_MAX_NAME_LEN]; + + /* Bus specific device information */ + void *dev_specific; + + /* I/O area used for device communication. */ + void __iomem *io_map; + + /* DMA caching attribute support */ + unsigned int axcache; + + bool irq_registered; + bool use_tasklet; + + unsigned int ccp_irq; + irq_handler_t ccp_irq_handler; + void *ccp_irq_data; + + unsigned int psp_irq; + irq_handler_t psp_irq_handler; + void *psp_irq_data; + + void *ccp_data; + void *psp_data; +}; + +int sp_pci_init(void); +void sp_pci_exit(void); + +int sp_platform_init(void); +void sp_platform_exit(void); + +struct sp_device *sp_alloc_struct(struct device *dev); + +int sp_init(struct sp_device *sp); +void sp_destroy(struct sp_device *sp); +struct sp_device *sp_get_master(void); + +int sp_suspend(struct sp_device *sp, pm_message_t state); +int sp_resume(struct sp_device *sp); +int sp_request_ccp_irq(struct sp_device *sp, irq_handler_t handler, + const char *name, void *data); +void sp_free_ccp_irq(struct sp_device *sp, void *data); +int sp_request_psp_irq(struct sp_device *sp, irq_handler_t handler, + const char *name, void *data); +void sp_free_psp_irq(struct sp_device *sp, void *data); + +#ifdef CONFIG_CRYPTO_DEV_SP_CCP + +int ccp_dev_init(struct sp_device *sp); +void ccp_dev_destroy(struct sp_device *sp); + +int ccp_dev_suspend(struct sp_device *sp, pm_message_t state); +int ccp_dev_resume(struct sp_device *sp); + +#else /* !CONFIG_CRYPTO_DEV_SP_CCP */ + +static inline int ccp_dev_init(struct sp_device *sp) +{ + return 0; +} +static inline void ccp_dev_destroy(struct sp_device *sp) { } + +static inline int ccp_dev_suspend(struct sp_device *sp, pm_message_t state) +{ + return 0; +} +static inline int ccp_dev_resume(struct sp_device *sp) +{ + return 0; +} +#endif /* CONFIG_CRYPTO_DEV_SP_CCP */ + +#endif diff --git a/drivers/crypto/ccp/sp-pci.c b/drivers/crypto/ccp/sp-pci.c new file mode 100644 index 000000000000..9859aa683a28 --- /dev/null +++ b/drivers/crypto/ccp/sp-pci.c @@ -0,0 +1,276 @@ +/* + * AMD Secure Processor device driver + * + * Copyright (C) 2013,2016 Advanced Micro Devices, Inc. + * + * Author: Tom Lendacky <thomas.lendacky@amd.com> + * Author: Gary R Hook <gary.hook@amd.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/device.h> +#include <linux/pci.h> +#include <linux/pci_ids.h> +#include <linux/dma-mapping.h> +#include <linux/kthread.h> +#include <linux/sched.h> +#include <linux/interrupt.h> +#include <linux/spinlock.h> +#include <linux/delay.h> +#include <linux/ccp.h> + +#include "ccp-dev.h" + +#define MSIX_VECTORS 2 + +struct sp_pci { + int msix_count; + struct msix_entry msix_entry[MSIX_VECTORS]; +}; + +static int sp_get_msix_irqs(struct sp_device *sp) +{ + struct sp_pci *sp_pci = sp->dev_specific; + struct device *dev = sp->dev; + struct pci_dev *pdev = to_pci_dev(dev); + int v, ret; + + for (v = 0; v < ARRAY_SIZE(sp_pci->msix_entry); v++) + sp_pci->msix_entry[v].entry = v; + + ret = pci_enable_msix_range(pdev, sp_pci->msix_entry, 1, v); + if (ret < 0) + return ret; + + sp_pci->msix_count = ret; + sp->use_tasklet = true; + + sp->psp_irq = sp_pci->msix_entry[0].vector; + sp->ccp_irq = (sp_pci->msix_count > 1) ? sp_pci->msix_entry[1].vector + : sp_pci->msix_entry[0].vector; + return 0; +} + +static int sp_get_msi_irq(struct sp_device *sp) +{ + struct device *dev = sp->dev; + struct pci_dev *pdev = to_pci_dev(dev); + int ret; + + ret = pci_enable_msi(pdev); + if (ret) + return ret; + + sp->ccp_irq = pdev->irq; + sp->psp_irq = pdev->irq; + + return 0; +} + +static int sp_get_irqs(struct sp_device *sp) +{ + struct device *dev = sp->dev; + int ret; + + ret = sp_get_msix_irqs(sp); + if (!ret) + return 0; + + /* Couldn't get MSI-X vectors, try MSI */ + dev_notice(dev, "could not enable MSI-X (%d), trying MSI\n", ret); + ret = sp_get_msi_irq(sp); + if (!ret) + return 0; + + /* Couldn't get MSI interrupt */ + dev_notice(dev, "could not enable MSI (%d)\n", ret); + + return ret; +} + +static void sp_free_irqs(struct sp_device *sp) +{ + struct sp_pci *sp_pci = sp->dev_specific; + struct device *dev = sp->dev; + struct pci_dev *pdev = to_pci_dev(dev); + + if (sp_pci->msix_count) + pci_disable_msix(pdev); + else if (sp->psp_irq) + pci_disable_msi(pdev); + + sp->ccp_irq = 0; + sp->psp_irq = 0; +} + +static int sp_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct sp_device *sp; + struct sp_pci *sp_pci; + struct device *dev = &pdev->dev; + void __iomem * const *iomap_table; + int bar_mask; + int ret; + + ret = -ENOMEM; + sp = sp_alloc_struct(dev); + if (!sp) + goto e_err; + + sp_pci = devm_kzalloc(dev, sizeof(*sp_pci), GFP_KERNEL); + if (!sp_pci) + goto e_err; + + sp->dev_specific = sp_pci; + sp->dev_vdata = (struct sp_dev_vdata *)id->driver_data; + if (!sp->dev_vdata) { + ret = -ENODEV; + dev_err(dev, "missing driver data\n"); + goto e_err; + } + + ret = pcim_enable_device(pdev); + if (ret) { + dev_err(dev, "pcim_enable_device failed (%d)\n", ret); + goto e_err; + } + + bar_mask = pci_select_bars(pdev, IORESOURCE_MEM); + ret = pcim_iomap_regions(pdev, bar_mask, "ccp"); + if (ret) { + dev_err(dev, "pcim_iomap_regions failed (%d)\n", ret); + goto e_err; + } + + iomap_table = pcim_iomap_table(pdev); + if (!iomap_table) { + dev_err(dev, "pcim_iomap_table failed\n"); + ret = -ENOMEM; + goto e_err; + } + + sp->io_map = iomap_table[sp->dev_vdata->bar]; + if (!sp->io_map) { + dev_err(dev, "ioremap failed\n"); + ret = -ENOMEM; + goto e_err; + } + + ret = sp_get_irqs(sp); + if (ret) + goto e_err; + + pci_set_master(pdev); + + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48)); + if (ret) { + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); + if (ret) { + dev_err(dev, "dma_set_mask_and_coherent failed (%d)\n", + ret); + goto e_err; + } + } + + dev_set_drvdata(dev, sp); + + ret = sp_init(sp); + if (ret) + goto e_err; + + dev_notice(dev, "enabled\n"); + + return 0; + +e_err: + dev_notice(dev, "initialization failed\n"); + return ret; +} + +static void sp_pci_remove(struct pci_dev *pdev) +{ + struct device *dev = &pdev->dev; + struct sp_device *sp = dev_get_drvdata(dev); + + if (!sp) + return; + + sp_destroy(sp); + + sp_free_irqs(sp); + + dev_notice(dev, "disabled\n"); +} + +#ifdef CONFIG_PM +static int sp_pci_suspend(struct pci_dev *pdev, pm_message_t state) +{ + struct device *dev = &pdev->dev; + struct sp_device *sp = dev_get_drvdata(dev); + + return sp_suspend(sp, state); +} + +static int sp_pci_resume(struct pci_dev *pdev) +{ + struct device *dev = &pdev->dev; + struct sp_device *sp = dev_get_drvdata(dev); + + return sp_resume(sp); +} +#endif + +static const struct sp_dev_vdata dev_vdata[] = { + { + .bar = 2, +#ifdef CONFIG_CRYPTO_DEV_SP_CCP + .ccp_vdata = &ccpv3, +#endif + }, + { + .bar = 2, +#ifdef CONFIG_CRYPTO_DEV_SP_CCP + .ccp_vdata = &ccpv5a, +#endif + }, + { + .bar = 2, +#ifdef CONFIG_CRYPTO_DEV_SP_CCP + .ccp_vdata = &ccpv5b, +#endif + }, +}; +static const struct pci_device_id sp_pci_table[] = { + { PCI_VDEVICE(AMD, 0x1537), (kernel_ulong_t)&dev_vdata[0] }, + { PCI_VDEVICE(AMD, 0x1456), (kernel_ulong_t)&dev_vdata[1] }, + { PCI_VDEVICE(AMD, 0x1468), (kernel_ulong_t)&dev_vdata[2] }, + /* Last entry must be zero */ + { 0, } +}; +MODULE_DEVICE_TABLE(pci, sp_pci_table); + +static struct pci_driver sp_pci_driver = { + .name = "ccp", + .id_table = sp_pci_table, + .probe = sp_pci_probe, + .remove = sp_pci_remove, +#ifdef CONFIG_PM + .suspend = sp_pci_suspend, + .resume = sp_pci_resume, +#endif +}; + +int sp_pci_init(void) +{ + return pci_register_driver(&sp_pci_driver); +} + +void sp_pci_exit(void) +{ + pci_unregister_driver(&sp_pci_driver); +} diff --git a/drivers/crypto/ccp/sp-platform.c b/drivers/crypto/ccp/sp-platform.c new file mode 100644 index 000000000000..71734f254fd1 --- /dev/null +++ b/drivers/crypto/ccp/sp-platform.c @@ -0,0 +1,256 @@ +/* + * AMD Secure Processor device driver + * + * Copyright (C) 2014,2016 Advanced Micro Devices, Inc. + * + * Author: Tom Lendacky <thomas.lendacky@amd.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/device.h> +#include <linux/platform_device.h> +#include <linux/ioport.h> +#include <linux/dma-mapping.h> +#include <linux/kthread.h> +#include <linux/sched.h> +#include <linux/interrupt.h> +#include <linux/spinlock.h> +#include <linux/delay.h> +#include <linux/ccp.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include <linux/acpi.h> + +#include "ccp-dev.h" + +struct sp_platform { + int coherent; + unsigned int irq_count; +}; + +static const struct acpi_device_id sp_acpi_match[]; +static const struct of_device_id sp_of_match[]; + +static struct sp_dev_vdata *sp_get_of_version(struct platform_device *pdev) +{ +#ifdef CONFIG_OF + const struct of_device_id *match; + + match = of_match_node(sp_of_match, pdev->dev.of_node); + if (match && match->data) + return (struct sp_dev_vdata *)match->data; +#endif + return NULL; +} + +static struct sp_dev_vdata *sp_get_acpi_version(struct platform_device *pdev) +{ +#ifdef CONFIG_ACPI + const struct acpi_device_id *match; + + match = acpi_match_device(sp_acpi_match, &pdev->dev); + if (match && match->driver_data) + return (struct sp_dev_vdata *)match->driver_data; +#endif + return NULL; +} + +static int sp_get_irqs(struct sp_device *sp) +{ + struct sp_platform *sp_platform = sp->dev_specific; + struct device *dev = sp->dev; + struct platform_device *pdev = to_platform_device(dev); + unsigned int i, count; + int ret; + + for (i = 0, count = 0; i < pdev->num_resources; i++) { + struct resource *res = &pdev->resource[i]; + + if (resource_type(res) == IORESOURCE_IRQ) + count++; + } + + sp_platform->irq_count = count; + + ret = platform_get_irq(pdev, 0); + if (ret < 0) { + dev_notice(dev, "unable to get IRQ (%d)\n", ret); + return ret; + } + + sp->psp_irq = ret; + if (count == 1) { + sp->ccp_irq = ret; + } else { + ret = platform_get_irq(pdev, 1); + if (ret < 0) { + dev_notice(dev, "unable to get IRQ (%d)\n", ret); + return ret; + } + + sp->ccp_irq = ret; + } + + return 0; +} + +static int sp_platform_probe(struct platform_device *pdev) +{ + struct sp_device *sp; + struct sp_platform *sp_platform; + struct device *dev = &pdev->dev; + enum dev_dma_attr attr; + struct resource *ior; + int ret; + + ret = -ENOMEM; + sp = sp_alloc_struct(dev); + if (!sp) + goto e_err; + + sp_platform = devm_kzalloc(dev, sizeof(*sp_platform), GFP_KERNEL); + if (!sp_platform) + goto e_err; + + sp->dev_specific = sp_platform; + sp->dev_vdata = pdev->dev.of_node ? sp_get_of_version(pdev) + : sp_get_acpi_version(pdev); + if (!sp->dev_vdata) { + ret = -ENODEV; + dev_err(dev, "missing driver data\n"); + goto e_err; + } + + ior = platform_get_resource(pdev, IORESOURCE_MEM, 0); + sp->io_map = devm_ioremap_resource(dev, ior); + if (IS_ERR(sp->io_map)) { + ret = PTR_ERR(sp->io_map); + goto e_err; + } + + attr = device_get_dma_attr(dev); + if (attr == DEV_DMA_NOT_SUPPORTED) { + dev_err(dev, "DMA is not supported"); + goto e_err; + } + + sp_platform->coherent = (attr == DEV_DMA_COHERENT); + if (sp_platform->coherent) + sp->axcache = CACHE_WB_NO_ALLOC; + else + sp->axcache = CACHE_NONE; + + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48)); + if (ret) { + dev_err(dev, "dma_set_mask_and_coherent failed (%d)\n", ret); + goto e_err; + } + + ret = sp_get_irqs(sp); + if (ret) + goto e_err; + + dev_set_drvdata(dev, sp); + + ret = sp_init(sp); + if (ret) + goto e_err; + + dev_notice(dev, "enabled\n"); + + return 0; + +e_err: + dev_notice(dev, "initialization failed\n"); + return ret; +} + +static int sp_platform_remove(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct sp_device *sp = dev_get_drvdata(dev); + + sp_destroy(sp); + + dev_notice(dev, "disabled\n"); + + return 0; +} + +#ifdef CONFIG_PM +static int sp_platform_suspend(struct platform_device *pdev, + pm_message_t state) +{ + struct device *dev = &pdev->dev; + struct sp_device *sp = dev_get_drvdata(dev); + + return sp_suspend(sp, state); +} + +static int sp_platform_resume(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct sp_device *sp = dev_get_drvdata(dev); + + return sp_resume(sp); +} +#endif + +static const struct sp_dev_vdata dev_vdata[] = { + { + .bar = 0, +#ifdef CONFIG_CRYPTO_DEV_SP_CCP + .ccp_vdata = &ccpv3_platform, +#endif + }, +}; + +#ifdef CONFIG_ACPI +static const struct acpi_device_id sp_acpi_match[] = { + { "AMDI0C00", (kernel_ulong_t)&dev_vdata[0] }, + { }, +}; +MODULE_DEVICE_TABLE(acpi, sp_acpi_match); +#endif + +#ifdef CONFIG_OF +static const struct of_device_id sp_of_match[] = { + { .compatible = "amd,ccp-seattle-v1a", + .data = (const void *)&dev_vdata[0] }, + { }, +}; +MODULE_DEVICE_TABLE(of, sp_of_match); +#endif + +static struct platform_driver sp_platform_driver = { + .driver = { + .name = "ccp", +#ifdef CONFIG_ACPI + .acpi_match_table = sp_acpi_match, +#endif +#ifdef CONFIG_OF + .of_match_table = sp_of_match, +#endif + }, + .probe = sp_platform_probe, + .remove = sp_platform_remove, +#ifdef CONFIG_PM + .suspend = sp_platform_suspend, + .resume = sp_platform_resume, +#endif +}; + +int sp_platform_init(void) +{ + return platform_driver_register(&sp_platform_driver); +} + +void sp_platform_exit(void) +{ + platform_driver_unregister(&sp_platform_driver); +} diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index 0e8160701833..4eed7171e2ae 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -53,6 +53,7 @@ #include <crypto/aes.h> #include <crypto/algapi.h> #include <crypto/hash.h> +#include <crypto/gcm.h> #include <crypto/sha.h> #include <crypto/authenc.h> #include <crypto/ctr.h> @@ -70,6 +71,8 @@ #include "chcr_algo.h" #include "chcr_crypto.h" +#define IV AES_BLOCK_SIZE + static inline struct chcr_aead_ctx *AEAD_CTX(struct chcr_context *ctx) { return ctx->crypto_ctx->aeadctx; @@ -102,7 +105,7 @@ static inline struct uld_ctx *ULD_CTX(struct chcr_context *ctx) static inline int is_ofld_imm(const struct sk_buff *skb) { - return (skb->len <= CRYPTO_MAX_IMM_TX_PKT_LEN); + return (skb->len <= SGE_MAX_WR_LEN); } /* @@ -117,6 +120,92 @@ static inline unsigned int sgl_len(unsigned int n) return (3 * n) / 2 + (n & 1) + 2; } +static int sg_nents_xlen(struct scatterlist *sg, unsigned int reqlen, + unsigned int entlen, + unsigned int skip) +{ + int nents = 0; + unsigned int less; + unsigned int skip_len = 0; + + while (sg && skip) { + if (sg_dma_len(sg) <= skip) { + skip -= sg_dma_len(sg); + skip_len = 0; + sg = sg_next(sg); + } else { + skip_len = skip; + skip = 0; + } + } + + while (sg && reqlen) { + less = min(reqlen, sg_dma_len(sg) - skip_len); + nents += DIV_ROUND_UP(less, entlen); + reqlen -= less; + skip_len = 0; + sg = sg_next(sg); + } + return nents; +} + +static inline void chcr_handle_ahash_resp(struct ahash_request *req, + unsigned char *input, + int err) +{ + struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req); + int digestsize, updated_digestsize; + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct uld_ctx *u_ctx = ULD_CTX(h_ctx(tfm)); + + if (input == NULL) + goto out; + reqctx = ahash_request_ctx(req); + digestsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(req)); + if (reqctx->is_sg_map) + chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req); + if (reqctx->dma_addr) + dma_unmap_single(&u_ctx->lldi.pdev->dev, reqctx->dma_addr, + reqctx->dma_len, DMA_TO_DEVICE); + reqctx->dma_addr = 0; + updated_digestsize = digestsize; + if (digestsize == SHA224_DIGEST_SIZE) + updated_digestsize = SHA256_DIGEST_SIZE; + else if (digestsize == SHA384_DIGEST_SIZE) + updated_digestsize = SHA512_DIGEST_SIZE; + if (reqctx->result == 1) { + reqctx->result = 0; + memcpy(req->result, input + sizeof(struct cpl_fw6_pld), + digestsize); + } else { + memcpy(reqctx->partial_hash, input + sizeof(struct cpl_fw6_pld), + updated_digestsize); + } +out: + req->base.complete(&req->base, err); + + } + +static inline void chcr_handle_aead_resp(struct aead_request *req, + unsigned char *input, + int err) +{ + struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct uld_ctx *u_ctx = ULD_CTX(a_ctx(tfm)); + + + chcr_aead_dma_unmap(&u_ctx->lldi.pdev->dev, req, reqctx->op); + if (reqctx->b0_dma) + dma_unmap_single(&u_ctx->lldi.pdev->dev, reqctx->b0_dma, + reqctx->b0_len, DMA_BIDIRECTIONAL); + if (reqctx->verify == VERIFY_SW) { + chcr_verify_tag(req, input, &err); + reqctx->verify = VERIFY_HW; +} + req->base.complete(&req->base, err); + +} static void chcr_verify_tag(struct aead_request *req, u8 *input, int *err) { u8 temp[SHA512_DIGEST_SIZE]; @@ -151,29 +240,11 @@ int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input, { struct crypto_tfm *tfm = req->tfm; struct chcr_context *ctx = crypto_tfm_ctx(tfm); - struct uld_ctx *u_ctx = ULD_CTX(ctx); - struct chcr_req_ctx ctx_req; - unsigned int digestsize, updated_digestsize; struct adapter *adap = padap(ctx->dev); switch (tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK) { case CRYPTO_ALG_TYPE_AEAD: - ctx_req.req.aead_req = aead_request_cast(req); - ctx_req.ctx.reqctx = aead_request_ctx(ctx_req.req.aead_req); - dma_unmap_sg(&u_ctx->lldi.pdev->dev, ctx_req.ctx.reqctx->dst, - ctx_req.ctx.reqctx->dst_nents, DMA_FROM_DEVICE); - if (ctx_req.ctx.reqctx->skb) { - kfree_skb(ctx_req.ctx.reqctx->skb); - ctx_req.ctx.reqctx->skb = NULL; - } - free_new_sg(ctx_req.ctx.reqctx->newdstsg); - ctx_req.ctx.reqctx->newdstsg = NULL; - if (ctx_req.ctx.reqctx->verify == VERIFY_SW) { - chcr_verify_tag(ctx_req.req.aead_req, input, - &err); - ctx_req.ctx.reqctx->verify = VERIFY_HW; - } - ctx_req.req.aead_req->base.complete(req, err); + chcr_handle_aead_resp(aead_request_cast(req), input, err); break; case CRYPTO_ALG_TYPE_ABLKCIPHER: @@ -182,60 +253,13 @@ int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input, break; case CRYPTO_ALG_TYPE_AHASH: - ctx_req.req.ahash_req = ahash_request_cast(req); - ctx_req.ctx.ahash_ctx = - ahash_request_ctx(ctx_req.req.ahash_req); - digestsize = - crypto_ahash_digestsize(crypto_ahash_reqtfm( - ctx_req.req.ahash_req)); - updated_digestsize = digestsize; - if (digestsize == SHA224_DIGEST_SIZE) - updated_digestsize = SHA256_DIGEST_SIZE; - else if (digestsize == SHA384_DIGEST_SIZE) - updated_digestsize = SHA512_DIGEST_SIZE; - if (ctx_req.ctx.ahash_ctx->skb) { - kfree_skb(ctx_req.ctx.ahash_ctx->skb); - ctx_req.ctx.ahash_ctx->skb = NULL; - } - if (ctx_req.ctx.ahash_ctx->result == 1) { - ctx_req.ctx.ahash_ctx->result = 0; - memcpy(ctx_req.req.ahash_req->result, input + - sizeof(struct cpl_fw6_pld), - digestsize); - } else { - memcpy(ctx_req.ctx.ahash_ctx->partial_hash, input + - sizeof(struct cpl_fw6_pld), - updated_digestsize); + chcr_handle_ahash_resp(ahash_request_cast(req), input, err); } - ctx_req.req.ahash_req->base.complete(req, err); - break; - } atomic_inc(&adap->chcr_stats.complete); return err; } -/* - * calc_tx_flits_ofld - calculate # of flits for an offload packet - * @skb: the packet - * Returns the number of flits needed for the given offload packet. - * These packets are already fully constructed and no additional headers - * will be added. - */ -static inline unsigned int calc_tx_flits_ofld(const struct sk_buff *skb) -{ - unsigned int flits, cnt; - - if (is_ofld_imm(skb)) - return DIV_ROUND_UP(skb->len, 8); - - flits = skb_transport_offset(skb) / 8; /* headers */ - cnt = skb_shinfo(skb)->nr_frags; - if (skb_tail_pointer(skb) != skb_transport_header(skb)) - cnt++; - return flits + sgl_len(cnt); -} - -static inline void get_aes_decrypt_key(unsigned char *dec_key, +static void get_aes_decrypt_key(unsigned char *dec_key, const unsigned char *key, unsigned int keylength) { @@ -382,13 +406,19 @@ static inline int is_hmac(struct crypto_tfm *tfm) return 0; } -static void write_phys_cpl(struct cpl_rx_phys_dsgl *phys_cpl, - struct scatterlist *sg, - struct phys_sge_parm *sg_param) +static inline void dsgl_walk_init(struct dsgl_walk *walk, + struct cpl_rx_phys_dsgl *dsgl) { - struct phys_sge_pairs *to; - unsigned int len = 0, left_size = sg_param->obsize; - unsigned int nents = sg_param->nents, i, j = 0; + walk->dsgl = dsgl; + walk->nents = 0; + walk->to = (struct phys_sge_pairs *)(dsgl + 1); +} + +static inline void dsgl_walk_end(struct dsgl_walk *walk, unsigned short qid) +{ + struct cpl_rx_phys_dsgl *phys_cpl; + + phys_cpl = walk->dsgl; phys_cpl->op_to_tid = htonl(CPL_RX_PHYS_DSGL_OPCODE_V(CPL_RX_PHYS_DSGL) | CPL_RX_PHYS_DSGL_ISRDMA_V(0)); @@ -398,38 +428,171 @@ static void write_phys_cpl(struct cpl_rx_phys_dsgl *phys_cpl, CPL_RX_PHYS_DSGL_PCITPHNTENB_V(0) | CPL_RX_PHYS_DSGL_PCITPHNT_V(0) | CPL_RX_PHYS_DSGL_DCAID_V(0) | - CPL_RX_PHYS_DSGL_NOOFSGENTR_V(nents)); + CPL_RX_PHYS_DSGL_NOOFSGENTR_V(walk->nents)); phys_cpl->rss_hdr_int.opcode = CPL_RX_PHYS_ADDR; - phys_cpl->rss_hdr_int.qid = htons(sg_param->qid); + phys_cpl->rss_hdr_int.qid = htons(qid); phys_cpl->rss_hdr_int.hash_val = 0; - to = (struct phys_sge_pairs *)((unsigned char *)phys_cpl + - sizeof(struct cpl_rx_phys_dsgl)); - for (i = 0; nents && left_size; to++) { - for (j = 0; j < 8 && nents && left_size; j++, nents--) { - len = min(left_size, sg_dma_len(sg)); - to->len[j] = htons(len); - to->addr[j] = cpu_to_be64(sg_dma_address(sg)); - left_size -= len; +} + +static inline void dsgl_walk_add_page(struct dsgl_walk *walk, + size_t size, + dma_addr_t *addr) +{ + int j; + + if (!size) + return; + j = walk->nents; + walk->to->len[j % 8] = htons(size); + walk->to->addr[j % 8] = cpu_to_be64(*addr); + j++; + if ((j % 8) == 0) + walk->to++; + walk->nents = j; +} + +static void dsgl_walk_add_sg(struct dsgl_walk *walk, + struct scatterlist *sg, + unsigned int slen, + unsigned int skip) +{ + int skip_len = 0; + unsigned int left_size = slen, len = 0; + unsigned int j = walk->nents; + int offset, ent_len; + + if (!slen) + return; + while (sg && skip) { + if (sg_dma_len(sg) <= skip) { + skip -= sg_dma_len(sg); + skip_len = 0; sg = sg_next(sg); + } else { + skip_len = skip; + skip = 0; } } + + while (left_size && sg) { + len = min_t(u32, left_size, sg_dma_len(sg) - skip_len); + offset = 0; + while (len) { + ent_len = min_t(u32, len, CHCR_DST_SG_SIZE); + walk->to->len[j % 8] = htons(ent_len); + walk->to->addr[j % 8] = cpu_to_be64(sg_dma_address(sg) + + offset + skip_len); + offset += ent_len; + len -= ent_len; + j++; + if ((j % 8) == 0) + walk->to++; + } + walk->last_sg = sg; + walk->last_sg_len = min_t(u32, left_size, sg_dma_len(sg) - + skip_len) + skip_len; + left_size -= min_t(u32, left_size, sg_dma_len(sg) - skip_len); + skip_len = 0; + sg = sg_next(sg); + } + walk->nents = j; +} + +static inline void ulptx_walk_init(struct ulptx_walk *walk, + struct ulptx_sgl *ulp) +{ + walk->sgl = ulp; + walk->nents = 0; + walk->pair_idx = 0; + walk->pair = ulp->sge; + walk->last_sg = NULL; + walk->last_sg_len = 0; } -static inline int map_writesg_phys_cpl(struct device *dev, - struct cpl_rx_phys_dsgl *phys_cpl, +static inline void ulptx_walk_end(struct ulptx_walk *walk) +{ + walk->sgl->cmd_nsge = htonl(ULPTX_CMD_V(ULP_TX_SC_DSGL) | + ULPTX_NSGE_V(walk->nents)); +} + + +static inline void ulptx_walk_add_page(struct ulptx_walk *walk, + size_t size, + dma_addr_t *addr) +{ + if (!size) + return; + + if (walk->nents == 0) { + walk->sgl->len0 = cpu_to_be32(size); + walk->sgl->addr0 = cpu_to_be64(*addr); + } else { + walk->pair->addr[walk->pair_idx] = cpu_to_be64(*addr); + walk->pair->len[walk->pair_idx] = cpu_to_be32(size); + walk->pair_idx = !walk->pair_idx; + if (!walk->pair_idx) + walk->pair++; + } + walk->nents++; +} + +static void ulptx_walk_add_sg(struct ulptx_walk *walk, struct scatterlist *sg, - struct phys_sge_parm *sg_param) + unsigned int len, + unsigned int skip) { - if (!sg || !sg_param->nents) - return -EINVAL; + int small; + int skip_len = 0; + unsigned int sgmin; - sg_param->nents = dma_map_sg(dev, sg, sg_param->nents, DMA_FROM_DEVICE); - if (sg_param->nents == 0) { - pr_err("CHCR : DMA mapping failed\n"); - return -EINVAL; + if (!len) + return; + + while (sg && skip) { + if (sg_dma_len(sg) <= skip) { + skip -= sg_dma_len(sg); + skip_len = 0; + sg = sg_next(sg); + } else { + skip_len = skip; + skip = 0; + } + } + if (walk->nents == 0) { + small = min_t(unsigned int, sg_dma_len(sg) - skip_len, len); + sgmin = min_t(unsigned int, small, CHCR_SRC_SG_SIZE); + walk->sgl->len0 = cpu_to_be32(sgmin); + walk->sgl->addr0 = cpu_to_be64(sg_dma_address(sg) + skip_len); + walk->nents++; + len -= sgmin; + walk->last_sg = sg; + walk->last_sg_len = sgmin + skip_len; + skip_len += sgmin; + if (sg_dma_len(sg) == skip_len) { + sg = sg_next(sg); + skip_len = 0; + } + } + + while (sg && len) { + small = min(sg_dma_len(sg) - skip_len, len); + sgmin = min_t(unsigned int, small, CHCR_SRC_SG_SIZE); + walk->pair->len[walk->pair_idx] = cpu_to_be32(sgmin); + walk->pair->addr[walk->pair_idx] = + cpu_to_be64(sg_dma_address(sg) + skip_len); + walk->pair_idx = !walk->pair_idx; + walk->nents++; + if (!walk->pair_idx) + walk->pair++; + len -= sgmin; + skip_len += sgmin; + walk->last_sg = sg; + walk->last_sg_len = skip_len; + if (sg_dma_len(sg) == skip_len) { + sg = sg_next(sg); + skip_len = 0; + } } - write_phys_cpl(phys_cpl, sg, sg_param); - return 0; } static inline int get_aead_subtype(struct crypto_aead *aead) @@ -449,45 +612,6 @@ static inline int get_cryptoalg_subtype(struct crypto_tfm *tfm) return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK; } -static inline void write_buffer_to_skb(struct sk_buff *skb, - unsigned int *frags, - char *bfr, - u8 bfr_len) -{ - skb->len += bfr_len; - skb->data_len += bfr_len; - skb->truesize += bfr_len; - get_page(virt_to_page(bfr)); - skb_fill_page_desc(skb, *frags, virt_to_page(bfr), - offset_in_page(bfr), bfr_len); - (*frags)++; -} - - -static inline void -write_sg_to_skb(struct sk_buff *skb, unsigned int *frags, - struct scatterlist *sg, unsigned int count) -{ - struct page *spage; - unsigned int page_len; - - skb->len += count; - skb->data_len += count; - skb->truesize += count; - - while (count > 0) { - if (!sg || (!(sg->length))) - break; - spage = sg_page(sg); - get_page(spage); - page_len = min(sg->length, count); - skb_fill_page_desc(skb, *frags, spage, sg->offset, page_len); - (*frags)++; - count -= page_len; - sg = sg_next(sg); - } -} - static int cxgb4_is_crypto_q_full(struct net_device *dev, unsigned int idx) { struct adapter *adap = netdev2adap(dev); @@ -524,30 +648,46 @@ static int chcr_sg_ent_in_wr(struct scatterlist *src, struct scatterlist *dst, unsigned int minsg, unsigned int space, - short int *sent, - short int *dent) + unsigned int srcskip, + unsigned int dstskip) { int srclen = 0, dstlen = 0; - int srcsg = minsg, dstsg = 0; + int srcsg = minsg, dstsg = minsg; + int offset = 0, less; + + if (sg_dma_len(src) == srcskip) { + src = sg_next(src); + srcskip = 0; + } - *sent = 0; - *dent = 0; - while (src && dst && ((srcsg + 1) <= MAX_SKB_FRAGS) && + if (sg_dma_len(dst) == dstskip) { + dst = sg_next(dst); + dstskip = 0; + } + + while (src && dst && space > (sgl_ent_len[srcsg + 1] + dsgl_ent_len[dstsg])) { - srclen += src->length; + srclen += (sg_dma_len(src) - srcskip); srcsg++; + offset = 0; while (dst && ((dstsg + 1) <= MAX_DSGL_ENT) && space > (sgl_ent_len[srcsg] + dsgl_ent_len[dstsg + 1])) { if (srclen <= dstlen) break; - dstlen += dst->length; - dst = sg_next(dst); + less = min_t(unsigned int, sg_dma_len(dst) - offset - + dstskip, CHCR_DST_SG_SIZE); + dstlen += less; + offset += less; + if (offset == sg_dma_len(dst)) { + dst = sg_next(dst); + offset = 0; + } dstsg++; + dstskip = 0; } src = sg_next(src); + srcskip = 0; } - *sent = srcsg - minsg; - *dent = dstsg; return min(srclen, dstlen); } @@ -576,47 +716,35 @@ static int chcr_cipher_fallback(struct crypto_skcipher *cipher, } static inline void create_wreq(struct chcr_context *ctx, struct chcr_wr *chcr_req, - void *req, struct sk_buff *skb, - int kctx_len, int hash_sz, - int is_iv, + struct crypto_async_request *req, + unsigned int imm, + int hash_sz, + unsigned int len16, unsigned int sc_len, unsigned int lcb) { struct uld_ctx *u_ctx = ULD_CTX(ctx); - int iv_loc = IV_DSGL; int qid = u_ctx->lldi.rxq_ids[ctx->rx_qidx]; - unsigned int immdatalen = 0, nr_frags = 0; - if (is_ofld_imm(skb)) { - immdatalen = skb->data_len; - iv_loc = IV_IMMEDIATE; - } else { - nr_frags = skb_shinfo(skb)->nr_frags; - } - chcr_req->wreq.op_to_cctx_size = FILL_WR_OP_CCTX_SIZE(immdatalen, - ((sizeof(chcr_req->key_ctx) + kctx_len) >> 4)); + chcr_req->wreq.op_to_cctx_size = FILL_WR_OP_CCTX_SIZE; chcr_req->wreq.pld_size_hash_size = - htonl(FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE_V(sgl_lengths[nr_frags]) | - FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_V(hash_sz)); + htonl(FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_V(hash_sz)); chcr_req->wreq.len16_pkd = - htonl(FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP( - (calc_tx_flits_ofld(skb) * 8), 16))); + htonl(FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP(len16, 16))); chcr_req->wreq.cookie = cpu_to_be64((uintptr_t)req); chcr_req->wreq.rx_chid_to_rx_q_id = FILL_WR_RX_Q_ID(ctx->dev->rx_channel_id, qid, - is_iv ? iv_loc : IV_NOP, !!lcb, - ctx->tx_qidx); + !!lcb, ctx->tx_qidx); chcr_req->ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(ctx->dev->tx_channel_id, qid); - chcr_req->ulptx.len = htonl((DIV_ROUND_UP((calc_tx_flits_ofld(skb) * 8), - 16) - ((sizeof(chcr_req->wreq)) >> 4))); + chcr_req->ulptx.len = htonl((DIV_ROUND_UP(len16, 16) - + ((sizeof(chcr_req->wreq)) >> 4))); - chcr_req->sc_imm.cmd_more = FILL_CMD_MORE(immdatalen); + chcr_req->sc_imm.cmd_more = FILL_CMD_MORE(!imm); chcr_req->sc_imm.len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) + - sizeof(chcr_req->key_ctx) + - kctx_len + sc_len + immdatalen); + sizeof(chcr_req->key_ctx) + sc_len); } /** @@ -629,47 +757,52 @@ static inline void create_wreq(struct chcr_context *ctx, static struct sk_buff *create_cipher_wr(struct cipher_wr_param *wrparam) { struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(wrparam->req); - struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); - struct uld_ctx *u_ctx = ULD_CTX(ctx); - struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm)); struct sk_buff *skb = NULL; struct chcr_wr *chcr_req; struct cpl_rx_phys_dsgl *phys_cpl; + struct ulptx_sgl *ulptx; struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(wrparam->req); - struct phys_sge_parm sg_param; - unsigned int frags = 0, transhdr_len, phys_dsgl; + unsigned int temp = 0, transhdr_len, dst_size; int error; - unsigned int ivsize = AES_BLOCK_SIZE, kctx_len; + int nents; + unsigned int kctx_len; gfp_t flags = wrparam->req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; - struct adapter *adap = padap(ctx->dev); - - phys_dsgl = get_space_for_phys_dsgl(reqctx->dst_nents); + struct adapter *adap = padap(c_ctx(tfm)->dev); + nents = sg_nents_xlen(reqctx->dstsg, wrparam->bytes, CHCR_DST_SG_SIZE, + reqctx->dst_ofst); + dst_size = get_space_for_phys_dsgl(nents + 1); kctx_len = (DIV_ROUND_UP(ablkctx->enckey_len, 16) * 16); - transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, phys_dsgl); - skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); + transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); + nents = sg_nents_xlen(reqctx->srcsg, wrparam->bytes, + CHCR_SRC_SG_SIZE, reqctx->src_ofst); + temp = reqctx->imm ? (DIV_ROUND_UP((IV + wrparam->req->nbytes), 16) + * 16) : (sgl_len(nents + MIN_CIPHER_SG) * 8); + transhdr_len += temp; + transhdr_len = DIV_ROUND_UP(transhdr_len, 16) * 16; + skb = alloc_skb(SGE_MAX_WR_LEN, flags); if (!skb) { error = -ENOMEM; goto err; } - skb_reserve(skb, sizeof(struct sge_opaque_hdr)); chcr_req = __skb_put_zero(skb, transhdr_len); chcr_req->sec_cpl.op_ivinsrtofst = - FILL_SEC_CPL_OP_IVINSR(ctx->dev->rx_channel_id, 2, 1); + FILL_SEC_CPL_OP_IVINSR(c_ctx(tfm)->dev->rx_channel_id, 2, 1); - chcr_req->sec_cpl.pldlen = htonl(ivsize + wrparam->bytes); + chcr_req->sec_cpl.pldlen = htonl(IV + wrparam->bytes); chcr_req->sec_cpl.aadstart_cipherstop_hi = - FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, ivsize + 1, 0); + FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, IV + 1, 0); chcr_req->sec_cpl.cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(0, 0, 0, 0); chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(reqctx->op, 0, ablkctx->ciph_mode, - 0, 0, ivsize >> 1); + 0, 0, IV >> 1); chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 0, - 0, 1, phys_dsgl); + 0, 0, dst_size); chcr_req->key_ctx.ctx_hdr = ablkctx->key_ctx_hdr; if ((reqctx->op == CHCR_DECRYPT_OP) && @@ -694,26 +827,18 @@ static struct sk_buff *create_cipher_wr(struct cipher_wr_param *wrparam) } } phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); - sg_param.nents = reqctx->dst_nents; - sg_param.obsize = wrparam->bytes; - sg_param.qid = wrparam->qid; - error = map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, - reqctx->dst, &sg_param); - if (error) - goto map_fail1; + ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size); + chcr_add_cipher_src_ent(wrparam->req, ulptx, wrparam); + chcr_add_cipher_dst_ent(wrparam->req, phys_cpl, wrparam, wrparam->qid); - skb_set_transport_header(skb, transhdr_len); - write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize); - write_sg_to_skb(skb, &frags, wrparam->srcsg, wrparam->bytes); atomic_inc(&adap->chcr_stats.cipher_rqst); - create_wreq(ctx, chcr_req, &(wrparam->req->base), skb, kctx_len, 0, 1, - sizeof(struct cpl_rx_phys_dsgl) + phys_dsgl, + temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + kctx_len + +(reqctx->imm ? (IV + wrparam->bytes) : 0); + create_wreq(c_ctx(tfm), chcr_req, &(wrparam->req->base), reqctx->imm, 0, + transhdr_len, temp, ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC); reqctx->skb = skb; - skb_get(skb); return skb; -map_fail1: - kfree_skb(skb); err: return ERR_PTR(error); } @@ -738,8 +863,7 @@ static int chcr_cipher_fallback_setkey(struct crypto_ablkcipher *cipher, unsigned int keylen) { struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); - struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher); - struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); int err = 0; crypto_skcipher_clear_flags(ablkctx->sw_cipher, CRYPTO_TFM_REQ_MASK); @@ -757,8 +881,7 @@ static int chcr_aes_cbc_setkey(struct crypto_ablkcipher *cipher, const u8 *key, unsigned int keylen) { - struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher); - struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); unsigned int ck_size, context_size; u16 alignment = 0; int err; @@ -790,8 +913,7 @@ static int chcr_aes_ctr_setkey(struct crypto_ablkcipher *cipher, const u8 *key, unsigned int keylen) { - struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher); - struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); unsigned int ck_size, context_size; u16 alignment = 0; int err; @@ -822,8 +944,7 @@ static int chcr_aes_rfc3686_setkey(struct crypto_ablkcipher *cipher, const u8 *key, unsigned int keylen) { - struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher); - struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); unsigned int ck_size, context_size; u16 alignment = 0; int err; @@ -890,25 +1011,28 @@ static unsigned int adjust_ctr_overflow(u8 *iv, u32 bytes) static int chcr_update_tweak(struct ablkcipher_request *req, u8 *iv) { struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); - struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); - struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm)); struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); struct crypto_cipher *cipher; int ret, i; u8 *key; unsigned int keylen; + int round = reqctx->last_req_len / AES_BLOCK_SIZE; + int round8 = round / 8; cipher = ablkctx->aes_generic; - memcpy(iv, req->info, AES_BLOCK_SIZE); + memcpy(iv, reqctx->iv, AES_BLOCK_SIZE); keylen = ablkctx->enckey_len / 2; key = ablkctx->key + keylen; ret = crypto_cipher_setkey(cipher, key, keylen); if (ret) goto out; + /*H/W sends the encrypted IV in dsgl when AADIVDROP bit is 0*/ + for (i = 0; i < round8; i++) + gf128mul_x8_ble((le128 *)iv, (le128 *)iv); - crypto_cipher_encrypt_one(cipher, iv, iv); - for (i = 0; i < (reqctx->processed / AES_BLOCK_SIZE); i++) + for (i = 0; i < (round % 8); i++) gf128mul_x_ble((le128 *)iv, (le128 *)iv); crypto_cipher_decrypt_one(cipher, iv, iv); @@ -982,65 +1106,60 @@ static int chcr_handle_cipher_resp(struct ablkcipher_request *req, unsigned char *input, int err) { struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); - struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); - struct uld_ctx *u_ctx = ULD_CTX(ctx); - struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm)); + struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm)); struct sk_buff *skb; struct cpl_fw6_pld *fw6_pld = (struct cpl_fw6_pld *)input; struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); struct cipher_wr_param wrparam; int bytes; - dma_unmap_sg(&u_ctx->lldi.pdev->dev, reqctx->dst, reqctx->dst_nents, - DMA_FROM_DEVICE); - - if (reqctx->skb) { - kfree_skb(reqctx->skb); - reqctx->skb = NULL; - } if (err) - goto complete; - + goto unmap; if (req->nbytes == reqctx->processed) { + chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, + req); err = chcr_final_cipher_iv(req, fw6_pld, req->info); goto complete; } if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], - ctx->tx_qidx))) { + c_ctx(tfm)->tx_qidx))) { if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { err = -EBUSY; - goto complete; + goto unmap; } } - wrparam.srcsg = scatterwalk_ffwd(reqctx->srcffwd, req->src, - reqctx->processed); - reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, reqctx->dstsg, - reqctx->processed); - if (!wrparam.srcsg || !reqctx->dst) { - pr_err("Input sg list length less that nbytes\n"); - err = -EINVAL; - goto complete; - } - bytes = chcr_sg_ent_in_wr(wrparam.srcsg, reqctx->dst, 1, - SPACE_LEFT(ablkctx->enckey_len), - &wrparam.snent, &reqctx->dst_nents); + if (!reqctx->imm) { + bytes = chcr_sg_ent_in_wr(reqctx->srcsg, reqctx->dstsg, 1, + SPACE_LEFT(ablkctx->enckey_len), + reqctx->src_ofst, reqctx->dst_ofst); if ((bytes + reqctx->processed) >= req->nbytes) bytes = req->nbytes - reqctx->processed; else bytes = ROUND_16(bytes); + } else { + /*CTR mode counter overfloa*/ + bytes = req->nbytes - reqctx->processed; + } + dma_sync_single_for_cpu(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, + reqctx->iv_dma, IV, DMA_BIDIRECTIONAL); err = chcr_update_cipher_iv(req, fw6_pld, reqctx->iv); + dma_sync_single_for_device(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, + reqctx->iv_dma, IV, DMA_BIDIRECTIONAL); if (err) - goto complete; + goto unmap; if (unlikely(bytes == 0)) { + chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, + req); err = chcr_cipher_fallback(ablkctx->sw_cipher, req->base.flags, - wrparam.srcsg, - reqctx->dst, - req->nbytes - reqctx->processed, - reqctx->iv, + req->src, + req->dst, + req->nbytes, + req->info, reqctx->op); goto complete; } @@ -1048,23 +1167,24 @@ static int chcr_handle_cipher_resp(struct ablkcipher_request *req, if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) == CRYPTO_ALG_SUB_TYPE_CTR) bytes = adjust_ctr_overflow(reqctx->iv, bytes); - reqctx->processed += bytes; - wrparam.qid = u_ctx->lldi.rxq_ids[ctx->rx_qidx]; + wrparam.qid = u_ctx->lldi.rxq_ids[c_ctx(tfm)->rx_qidx]; wrparam.req = req; wrparam.bytes = bytes; skb = create_cipher_wr(&wrparam); if (IS_ERR(skb)) { pr_err("chcr : %s : Failed to form WR. No memory\n", __func__); err = PTR_ERR(skb); - goto complete; + goto unmap; } skb->dev = u_ctx->lldi.ports[0]; - set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); + set_wr_txq(skb, CPL_PRIORITY_DATA, c_ctx(tfm)->tx_qidx); chcr_send_wr(skb); + reqctx->last_req_len = bytes; + reqctx->processed += bytes; return 0; +unmap: + chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req); complete: - free_new_sg(reqctx->newdstsg); - reqctx->newdstsg = NULL; req->base.complete(&req->base, err); return err; } @@ -1077,12 +1197,10 @@ static int process_cipher(struct ablkcipher_request *req, struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); unsigned int ivsize = crypto_ablkcipher_ivsize(tfm); struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); - struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); - struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm)); struct cipher_wr_param wrparam; - int bytes, nents, err = -EINVAL; + int bytes, err = -EINVAL; - reqctx->newdstsg = NULL; reqctx->processed = 0; if (!req->info) goto error; @@ -1093,25 +1211,41 @@ static int process_cipher(struct ablkcipher_request *req, ablkctx->enckey_len, req->nbytes, ivsize); goto error; } - wrparam.srcsg = req->src; - if (is_newsg(req->dst, &nents)) { - reqctx->newdstsg = alloc_new_sg(req->dst, nents); - if (IS_ERR(reqctx->newdstsg)) - return PTR_ERR(reqctx->newdstsg); - reqctx->dstsg = reqctx->newdstsg; + chcr_cipher_dma_map(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req); + if (req->nbytes < (SGE_MAX_WR_LEN - (sizeof(struct chcr_wr) + + AES_MIN_KEY_SIZE + + sizeof(struct cpl_rx_phys_dsgl) + + /*Min dsgl size*/ + 32))) { + /* Can be sent as Imm*/ + unsigned int dnents = 0, transhdr_len, phys_dsgl, kctx_len; + + dnents = sg_nents_xlen(req->dst, req->nbytes, + CHCR_DST_SG_SIZE, 0); + dnents += 1; // IV + phys_dsgl = get_space_for_phys_dsgl(dnents); + kctx_len = (DIV_ROUND_UP(ablkctx->enckey_len, 16) * 16); + transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, phys_dsgl); + reqctx->imm = (transhdr_len + IV + req->nbytes) <= + SGE_MAX_WR_LEN; + bytes = IV + req->nbytes; + } else { - reqctx->dstsg = req->dst; + reqctx->imm = 0; } - bytes = chcr_sg_ent_in_wr(wrparam.srcsg, reqctx->dstsg, MIN_CIPHER_SG, - SPACE_LEFT(ablkctx->enckey_len), - &wrparam.snent, - &reqctx->dst_nents); + + if (!reqctx->imm) { + bytes = chcr_sg_ent_in_wr(req->src, req->dst, + MIN_CIPHER_SG, + SPACE_LEFT(ablkctx->enckey_len), + 0, 0); if ((bytes + reqctx->processed) >= req->nbytes) bytes = req->nbytes - reqctx->processed; else bytes = ROUND_16(bytes); - if (unlikely(bytes > req->nbytes)) + } else { bytes = req->nbytes; + } if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) == CRYPTO_ALG_SUB_TYPE_CTR) { bytes = adjust_ctr_overflow(req->info, bytes); @@ -1128,9 +1262,11 @@ static int process_cipher(struct ablkcipher_request *req, } else { - memcpy(reqctx->iv, req->info, ivsize); + memcpy(reqctx->iv, req->info, IV); } if (unlikely(bytes == 0)) { + chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, + req); err = chcr_cipher_fallback(ablkctx->sw_cipher, req->base.flags, req->src, @@ -1140,45 +1276,48 @@ static int process_cipher(struct ablkcipher_request *req, op_type); goto error; } - reqctx->processed = bytes; - reqctx->dst = reqctx->dstsg; reqctx->op = op_type; + reqctx->srcsg = req->src; + reqctx->dstsg = req->dst; + reqctx->src_ofst = 0; + reqctx->dst_ofst = 0; wrparam.qid = qid; wrparam.req = req; wrparam.bytes = bytes; *skb = create_cipher_wr(&wrparam); if (IS_ERR(*skb)) { err = PTR_ERR(*skb); - goto error; + goto unmap; } + reqctx->processed = bytes; + reqctx->last_req_len = bytes; return 0; +unmap: + chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req); error: - free_new_sg(reqctx->newdstsg); - reqctx->newdstsg = NULL; return err; } static int chcr_aes_encrypt(struct ablkcipher_request *req) { struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); - struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); struct sk_buff *skb = NULL; int err; - struct uld_ctx *u_ctx = ULD_CTX(ctx); + struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm)); if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], - ctx->tx_qidx))) { + c_ctx(tfm)->tx_qidx))) { if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) return -EBUSY; } - err = process_cipher(req, u_ctx->lldi.rxq_ids[ctx->rx_qidx], &skb, - CHCR_ENCRYPT_OP); + err = process_cipher(req, u_ctx->lldi.rxq_ids[c_ctx(tfm)->rx_qidx], + &skb, CHCR_ENCRYPT_OP); if (err || !skb) return err; skb->dev = u_ctx->lldi.ports[0]; - set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); + set_wr_txq(skb, CPL_PRIORITY_DATA, c_ctx(tfm)->tx_qidx); chcr_send_wr(skb); return -EINPROGRESS; } @@ -1186,23 +1325,22 @@ static int chcr_aes_encrypt(struct ablkcipher_request *req) static int chcr_aes_decrypt(struct ablkcipher_request *req) { struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); - struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm); - struct uld_ctx *u_ctx = ULD_CTX(ctx); + struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm)); struct sk_buff *skb = NULL; int err; if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], - ctx->tx_qidx))) { + c_ctx(tfm)->tx_qidx))) { if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) return -EBUSY; } - err = process_cipher(req, u_ctx->lldi.rxq_ids[ctx->rx_qidx], &skb, - CHCR_DECRYPT_OP); + err = process_cipher(req, u_ctx->lldi.rxq_ids[c_ctx(tfm)->rx_qidx], + &skb, CHCR_DECRYPT_OP); if (err || !skb) return err; skb->dev = u_ctx->lldi.ports[0]; - set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); + set_wr_txq(skb, CPL_PRIORITY_DATA, c_ctx(tfm)->tx_qidx); chcr_send_wr(skb); return -EINPROGRESS; } @@ -1350,17 +1488,19 @@ static struct sk_buff *create_hash_wr(struct ahash_request *req, { struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); - struct hmac_ctx *hmacctx = HMAC_CTX(ctx); + struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(tfm)); struct sk_buff *skb = NULL; + struct uld_ctx *u_ctx = ULD_CTX(h_ctx(tfm)); struct chcr_wr *chcr_req; - unsigned int frags = 0, transhdr_len, iopad_alignment = 0; + struct ulptx_sgl *ulptx; + unsigned int nents = 0, transhdr_len, iopad_alignment = 0; unsigned int digestsize = crypto_ahash_digestsize(tfm); - unsigned int kctx_len = 0; + unsigned int kctx_len = 0, temp = 0; u8 hash_size_in_response = 0; gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; - struct adapter *adap = padap(ctx->dev); + struct adapter *adap = padap(h_ctx(tfm)->dev); + int error = 0; iopad_alignment = KEYCTX_ALIGN_PAD(digestsize); kctx_len = param->alg_prm.result_size + iopad_alignment; @@ -1372,15 +1512,22 @@ static struct sk_buff *create_hash_wr(struct ahash_request *req, else hash_size_in_response = param->alg_prm.result_size; transhdr_len = HASH_TRANSHDR_SIZE(kctx_len); - skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); + req_ctx->imm = (transhdr_len + param->bfr_len + param->sg_len) <= + SGE_MAX_WR_LEN; + nents = sg_nents_xlen(req->src, param->sg_len, CHCR_SRC_SG_SIZE, 0); + nents += param->bfr_len ? 1 : 0; + transhdr_len += req_ctx->imm ? (DIV_ROUND_UP((param->bfr_len + + param->sg_len), 16) * 16) : + (sgl_len(nents) * 8); + transhdr_len = DIV_ROUND_UP(transhdr_len, 16) * 16; + + skb = alloc_skb(SGE_MAX_WR_LEN, flags); if (!skb) - return skb; - - skb_reserve(skb, sizeof(struct sge_opaque_hdr)); + return ERR_PTR(-ENOMEM); chcr_req = __skb_put_zero(skb, transhdr_len); chcr_req->sec_cpl.op_ivinsrtofst = - FILL_SEC_CPL_OP_IVINSR(ctx->dev->rx_channel_id, 2, 0); + FILL_SEC_CPL_OP_IVINSR(h_ctx(tfm)->dev->rx_channel_id, 2, 0); chcr_req->sec_cpl.pldlen = htonl(param->bfr_len + param->sg_len); chcr_req->sec_cpl.aadstart_cipherstop_hi = @@ -1409,37 +1556,52 @@ static struct sk_buff *create_hash_wr(struct ahash_request *req, ((kctx_len + sizeof(chcr_req->key_ctx)) >> 4)); chcr_req->sec_cpl.scmd1 = cpu_to_be64((u64)param->scmd1); - - skb_set_transport_header(skb, transhdr_len); - if (param->bfr_len != 0) - write_buffer_to_skb(skb, &frags, req_ctx->reqbfr, - param->bfr_len); - if (param->sg_len != 0) - write_sg_to_skb(skb, &frags, req->src, param->sg_len); + ulptx = (struct ulptx_sgl *)((u8 *)(chcr_req + 1) + kctx_len + + DUMMY_BYTES); + if (param->bfr_len != 0) { + req_ctx->dma_addr = dma_map_single(&u_ctx->lldi.pdev->dev, + req_ctx->reqbfr, param->bfr_len, + DMA_TO_DEVICE); + if (dma_mapping_error(&u_ctx->lldi.pdev->dev, + req_ctx->dma_addr)) { + error = -ENOMEM; + goto err; + } + req_ctx->dma_len = param->bfr_len; + } else { + req_ctx->dma_addr = 0; + } + chcr_add_hash_src_ent(req, ulptx, param); + /* Request upto max wr size */ + temp = kctx_len + DUMMY_BYTES + (req_ctx->imm ? (param->sg_len + + param->bfr_len) : 0); atomic_inc(&adap->chcr_stats.digest_rqst); - create_wreq(ctx, chcr_req, &req->base, skb, kctx_len, - hash_size_in_response, 0, DUMMY_BYTES, 0); + create_wreq(h_ctx(tfm), chcr_req, &req->base, req_ctx->imm, + hash_size_in_response, transhdr_len, + temp, 0); req_ctx->skb = skb; - skb_get(skb); return skb; +err: + kfree_skb(skb); + return ERR_PTR(error); } static int chcr_ahash_update(struct ahash_request *req) { struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); - struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm)); struct uld_ctx *u_ctx = NULL; struct sk_buff *skb; u8 remainder = 0, bs; unsigned int nbytes = req->nbytes; struct hash_wr_param params; + int error; bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); - u_ctx = ULD_CTX(ctx); + u_ctx = ULD_CTX(h_ctx(rtfm)); if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], - ctx->tx_qidx))) { + h_ctx(rtfm)->tx_qidx))) { if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) return -EBUSY; } @@ -1453,7 +1615,9 @@ static int chcr_ahash_update(struct ahash_request *req) req_ctx->reqlen += nbytes; return 0; } - + error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req); + if (error) + return -ENOMEM; params.opad_needed = 0; params.more = 1; params.last = 0; @@ -1464,25 +1628,27 @@ static int chcr_ahash_update(struct ahash_request *req) req_ctx->result = 0; req_ctx->data_len += params.sg_len + params.bfr_len; skb = create_hash_wr(req, ¶ms); - if (!skb) - return -ENOMEM; + if (IS_ERR(skb)) { + error = PTR_ERR(skb); + goto unmap; + } if (remainder) { - u8 *temp; /* Swap buffers */ - temp = req_ctx->reqbfr; - req_ctx->reqbfr = req_ctx->skbfr; - req_ctx->skbfr = temp; + swap(req_ctx->reqbfr, req_ctx->skbfr); sg_pcopy_to_buffer(req->src, sg_nents(req->src), req_ctx->reqbfr, remainder, req->nbytes - remainder); } req_ctx->reqlen = remainder; skb->dev = u_ctx->lldi.ports[0]; - set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); + set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx); chcr_send_wr(skb); return -EINPROGRESS; +unmap: + chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req); + return error; } static void create_last_hash_block(char *bfr_ptr, unsigned int bs, u64 scmd1) @@ -1499,13 +1665,12 @@ static int chcr_ahash_final(struct ahash_request *req) { struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); - struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm)); struct hash_wr_param params; struct sk_buff *skb; struct uld_ctx *u_ctx = NULL; u8 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); - u_ctx = ULD_CTX(ctx); + u_ctx = ULD_CTX(h_ctx(rtfm)); if (is_hmac(crypto_ahash_tfm(rtfm))) params.opad_needed = 1; else @@ -1528,11 +1693,11 @@ static int chcr_ahash_final(struct ahash_request *req) params.more = 0; } skb = create_hash_wr(req, ¶ms); - if (!skb) - return -ENOMEM; + if (IS_ERR(skb)) + return PTR_ERR(skb); skb->dev = u_ctx->lldi.ports[0]; - set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); + set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx); chcr_send_wr(skb); return -EINPROGRESS; } @@ -1541,17 +1706,17 @@ static int chcr_ahash_finup(struct ahash_request *req) { struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); - struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm)); struct uld_ctx *u_ctx = NULL; struct sk_buff *skb; struct hash_wr_param params; u8 bs; + int error; bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); - u_ctx = ULD_CTX(ctx); + u_ctx = ULD_CTX(h_ctx(rtfm)); if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], - ctx->tx_qidx))) { + h_ctx(rtfm)->tx_qidx))) { if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) return -EBUSY; } @@ -1577,34 +1742,41 @@ static int chcr_ahash_finup(struct ahash_request *req) params.last = 1; params.more = 0; } - - skb = create_hash_wr(req, ¶ms); - if (!skb) + error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req); + if (error) return -ENOMEM; + skb = create_hash_wr(req, ¶ms); + if (IS_ERR(skb)) { + error = PTR_ERR(skb); + goto unmap; + } skb->dev = u_ctx->lldi.ports[0]; - set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); + set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx); chcr_send_wr(skb); return -EINPROGRESS; +unmap: + chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req); + return error; } static int chcr_ahash_digest(struct ahash_request *req) { struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req); - struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm)); struct uld_ctx *u_ctx = NULL; struct sk_buff *skb; struct hash_wr_param params; u8 bs; + int error; rtfm->init(req); bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); - u_ctx = ULD_CTX(ctx); + u_ctx = ULD_CTX(h_ctx(rtfm)); if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], - ctx->tx_qidx))) { + h_ctx(rtfm)->tx_qidx))) { if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) return -EBUSY; } @@ -1613,6 +1785,9 @@ static int chcr_ahash_digest(struct ahash_request *req) params.opad_needed = 1; else params.opad_needed = 0; + error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req); + if (error) + return -ENOMEM; params.last = 0; params.more = 0; @@ -1630,13 +1805,17 @@ static int chcr_ahash_digest(struct ahash_request *req) } skb = create_hash_wr(req, ¶ms); - if (!skb) - return -ENOMEM; - + if (IS_ERR(skb)) { + error = PTR_ERR(skb); + goto unmap; + } skb->dev = u_ctx->lldi.ports[0]; - set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); + set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx); chcr_send_wr(skb); return -EINPROGRESS; +unmap: + chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req); + return error; } static int chcr_ahash_export(struct ahash_request *areq, void *out) @@ -1646,6 +1825,8 @@ static int chcr_ahash_export(struct ahash_request *areq, void *out) state->reqlen = req_ctx->reqlen; state->data_len = req_ctx->data_len; + state->is_sg_map = 0; + state->result = 0; memcpy(state->bfr1, req_ctx->reqbfr, req_ctx->reqlen); memcpy(state->partial_hash, req_ctx->partial_hash, CHCR_HASH_MAX_DIGEST_SIZE); @@ -1661,6 +1842,8 @@ static int chcr_ahash_import(struct ahash_request *areq, const void *in) req_ctx->data_len = state->data_len; req_ctx->reqbfr = req_ctx->bfr1; req_ctx->skbfr = req_ctx->bfr2; + req_ctx->is_sg_map = 0; + req_ctx->result = 0; memcpy(req_ctx->bfr1, state->bfr1, CHCR_HASH_MAX_BLOCK_SIZE_128); memcpy(req_ctx->partial_hash, state->partial_hash, CHCR_HASH_MAX_DIGEST_SIZE); @@ -1670,8 +1853,7 @@ static int chcr_ahash_import(struct ahash_request *areq, const void *in) static int chcr_ahash_setkey(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen) { - struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); - struct hmac_ctx *hmacctx = HMAC_CTX(ctx); + struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(tfm)); unsigned int digestsize = crypto_ahash_digestsize(tfm); unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); unsigned int i, err = 0, updated_digestsize; @@ -1724,8 +1906,7 @@ out: static int chcr_aes_xts_setkey(struct crypto_ablkcipher *cipher, const u8 *key, unsigned int key_len) { - struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher); - struct ablk_ctx *ablkctx = ABLK_CTX(ctx); + struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher)); unsigned short context_size = 0; int err; @@ -1764,6 +1945,7 @@ static int chcr_sha_init(struct ahash_request *areq) req_ctx->skbfr = req_ctx->bfr2; req_ctx->skb = NULL; req_ctx->result = 0; + req_ctx->is_sg_map = 0; copy_hash_init_values(req_ctx->partial_hash, digestsize); return 0; } @@ -1779,8 +1961,7 @@ static int chcr_hmac_init(struct ahash_request *areq) { struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); struct crypto_ahash *rtfm = crypto_ahash_reqtfm(areq); - struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm)); - struct hmac_ctx *hmacctx = HMAC_CTX(ctx); + struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(rtfm)); unsigned int digestsize = crypto_ahash_digestsize(rtfm); unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm)); @@ -1826,86 +2007,48 @@ static void chcr_hmac_cra_exit(struct crypto_tfm *tfm) } } -static int is_newsg(struct scatterlist *sgl, unsigned int *newents) -{ - int nents = 0; - int ret = 0; - - while (sgl) { - if (sgl->length > CHCR_SG_SIZE) - ret = 1; - nents += DIV_ROUND_UP(sgl->length, CHCR_SG_SIZE); - sgl = sg_next(sgl); - } - *newents = nents; - return ret; -} - -static inline void free_new_sg(struct scatterlist *sgl) +static int chcr_aead_common_init(struct aead_request *req, + unsigned short op_type) { - kfree(sgl); -} - -static struct scatterlist *alloc_new_sg(struct scatterlist *sgl, - unsigned int nents) -{ - struct scatterlist *newsg, *sg; - int i, len, processed = 0; - struct page *spage; - int offset; + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); + struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); + int error = -EINVAL; + unsigned int dst_size; + unsigned int authsize = crypto_aead_authsize(tfm); - newsg = kmalloc_array(nents, sizeof(struct scatterlist), GFP_KERNEL); - if (!newsg) - return ERR_PTR(-ENOMEM); - sg = newsg; - sg_init_table(sg, nents); - offset = sgl->offset; - spage = sg_page(sgl); - for (i = 0; i < nents; i++) { - len = min_t(u32, sgl->length - processed, CHCR_SG_SIZE); - sg_set_page(sg, spage, len, offset); - processed += len; - offset += len; - if (offset >= PAGE_SIZE) { - offset = offset % PAGE_SIZE; - spage++; - } - if (processed == sgl->length) { - processed = 0; - sgl = sg_next(sgl); - if (!sgl) - break; - spage = sg_page(sgl); - offset = sgl->offset; - } - sg = sg_next(sg); + dst_size = req->assoclen + req->cryptlen + (op_type ? + -authsize : authsize); + /* validate key size */ + if (aeadctx->enckey_len == 0) + goto err; + if (op_type && req->cryptlen < authsize) + goto err; + error = chcr_aead_dma_map(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, + op_type); + if (error) { + error = -ENOMEM; + goto err; } - return newsg; + reqctx->aad_nents = sg_nents_xlen(req->src, req->assoclen, + CHCR_SRC_SG_SIZE, 0); + reqctx->src_nents = sg_nents_xlen(req->src, req->cryptlen, + CHCR_SRC_SG_SIZE, req->assoclen); + return 0; +err: + return error; } -static int chcr_copy_assoc(struct aead_request *req, - struct chcr_aead_ctx *ctx) -{ - SKCIPHER_REQUEST_ON_STACK(skreq, ctx->null); - - skcipher_request_set_tfm(skreq, ctx->null); - skcipher_request_set_callback(skreq, aead_request_flags(req), - NULL, NULL); - skcipher_request_set_crypt(skreq, req->src, req->dst, req->assoclen, - NULL); - - return crypto_skcipher_encrypt(skreq); -} -static int chcr_aead_need_fallback(struct aead_request *req, int src_nent, +static int chcr_aead_need_fallback(struct aead_request *req, int dst_nents, int aadmax, int wrlen, unsigned short op_type) { unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req)); if (((req->cryptlen - (op_type ? authsize : 0)) == 0) || + dst_nents > MAX_DSGL_ENT || (req->assoclen > aadmax) || - (src_nent > MAX_SKB_FRAGS) || - (wrlen > MAX_WR_SIZE)) + (wrlen > SGE_MAX_WR_LEN)) return 1; return 0; } @@ -1913,8 +2056,7 @@ static int chcr_aead_need_fallback(struct aead_request *req, int src_nent, static int chcr_aead_fallback(struct aead_request *req, unsigned short op_type) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); - struct chcr_context *ctx = crypto_aead_ctx(tfm); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); struct aead_request *subreq = aead_request_ctx(req); aead_request_set_tfm(subreq, aeadctx->sw_cipher); @@ -1933,96 +2075,75 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, unsigned short op_type) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); - struct chcr_context *ctx = crypto_aead_ctx(tfm); - struct uld_ctx *u_ctx = ULD_CTX(ctx); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx); struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); struct sk_buff *skb = NULL; struct chcr_wr *chcr_req; struct cpl_rx_phys_dsgl *phys_cpl; - struct phys_sge_parm sg_param; - struct scatterlist *src; - unsigned int frags = 0, transhdr_len; - unsigned int ivsize = crypto_aead_ivsize(tfm), dst_size = 0; - unsigned int kctx_len = 0, nents; - unsigned short stop_offset = 0; + struct ulptx_sgl *ulptx; + unsigned int transhdr_len; + unsigned int dst_size = 0, temp; + unsigned int kctx_len = 0, dnents; unsigned int assoclen = req->assoclen; unsigned int authsize = crypto_aead_authsize(tfm); - int error = -EINVAL, src_nent; + int error = -EINVAL; int null = 0; gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; - struct adapter *adap = padap(ctx->dev); + struct adapter *adap = padap(a_ctx(tfm)->dev); - reqctx->newdstsg = NULL; - dst_size = req->assoclen + req->cryptlen + (op_type ? -authsize : - authsize); - if (aeadctx->enckey_len == 0 || (req->cryptlen <= 0)) - goto err; - - if (op_type && req->cryptlen < crypto_aead_authsize(tfm)) - goto err; - src_nent = sg_nents_for_len(req->src, req->assoclen + req->cryptlen); - if (src_nent < 0) - goto err; - src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen); + if (req->cryptlen == 0) + return NULL; - if (req->src != req->dst) { - error = chcr_copy_assoc(req, aeadctx); - if (error) - return ERR_PTR(error); - } - if (dst_size && is_newsg(req->dst, &nents)) { - reqctx->newdstsg = alloc_new_sg(req->dst, nents); - if (IS_ERR(reqctx->newdstsg)) - return ERR_CAST(reqctx->newdstsg); - reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, - reqctx->newdstsg, req->assoclen); - } else { - if (req->src == req->dst) - reqctx->dst = src; - else - reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, - req->dst, req->assoclen); - } + reqctx->b0_dma = 0; if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_NULL) { null = 1; assoclen = 0; } - reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen + - (op_type ? -authsize : authsize)); - if (reqctx->dst_nents < 0) { - pr_err("AUTHENC:Invalid Destination sg entries\n"); - error = -EINVAL; - goto err; + dst_size = assoclen + req->cryptlen + (op_type ? -authsize : + authsize); + error = chcr_aead_common_init(req, op_type); + if (error) + return ERR_PTR(error); + if (dst_size) { + dnents = sg_nents_xlen(req->dst, assoclen, CHCR_DST_SG_SIZE, 0); + dnents += sg_nents_xlen(req->dst, req->cryptlen + + (op_type ? -authsize : authsize), CHCR_DST_SG_SIZE, + req->assoclen); + dnents += MIN_AUTH_SG; // For IV + } else { + dnents = 0; } - dst_size = get_space_for_phys_dsgl(reqctx->dst_nents); + + dst_size = get_space_for_phys_dsgl(dnents); kctx_len = (ntohl(KEY_CONTEXT_CTX_LEN_V(aeadctx->key_ctx_hdr)) << 4) - sizeof(chcr_req->key_ctx); transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); - if (chcr_aead_need_fallback(req, src_nent + MIN_AUTH_SG, - T6_MAX_AAD_SIZE, - transhdr_len + (sgl_len(src_nent + MIN_AUTH_SG) * 8), - op_type)) { + reqctx->imm = (transhdr_len + assoclen + IV + req->cryptlen) < + SGE_MAX_WR_LEN; + temp = reqctx->imm ? (DIV_ROUND_UP((assoclen + IV + req->cryptlen), 16) + * 16) : (sgl_len(reqctx->src_nents + reqctx->aad_nents + + MIN_GCM_SG) * 8); + transhdr_len += temp; + transhdr_len = DIV_ROUND_UP(transhdr_len, 16) * 16; + + if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE, + transhdr_len, op_type)) { atomic_inc(&adap->chcr_stats.fallback); - free_new_sg(reqctx->newdstsg); - reqctx->newdstsg = NULL; + chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, + op_type); return ERR_PTR(chcr_aead_fallback(req, op_type)); } - skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); + skb = alloc_skb(SGE_MAX_WR_LEN, flags); if (!skb) { error = -ENOMEM; goto err; } - /* LLD is going to write the sge hdr. */ - skb_reserve(skb, sizeof(struct sge_opaque_hdr)); - - /* Write WR */ chcr_req = __skb_put_zero(skb, transhdr_len); - stop_offset = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize; + temp = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize; /* * Input order is AAD,IV and Payload. where IV should be included as @@ -2030,24 +2151,24 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, * to the hardware spec */ chcr_req->sec_cpl.op_ivinsrtofst = - FILL_SEC_CPL_OP_IVINSR(ctx->dev->rx_channel_id, 2, - (ivsize ? (assoclen + 1) : 0)); - chcr_req->sec_cpl.pldlen = htonl(assoclen + ivsize + req->cryptlen); + FILL_SEC_CPL_OP_IVINSR(a_ctx(tfm)->dev->rx_channel_id, 2, + assoclen + 1); + chcr_req->sec_cpl.pldlen = htonl(assoclen + IV + req->cryptlen); chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( assoclen ? 1 : 0, assoclen, - assoclen + ivsize + 1, - (stop_offset & 0x1F0) >> 4); + assoclen + IV + 1, + (temp & 0x1F0) >> 4); chcr_req->sec_cpl.cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT( - stop_offset & 0xF, - null ? 0 : assoclen + ivsize + 1, - stop_offset, stop_offset); + temp & 0xF, + null ? 0 : assoclen + IV + 1, + temp, temp); chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type, (op_type == CHCR_ENCRYPT_OP) ? 1 : 0, CHCR_SCMD_CIPHER_MODE_AES_CBC, actx->auth_mode, aeadctx->hmac_ctrl, - ivsize >> 1); + IV >> 1); chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, - 0, 1, dst_size); + 0, 0, dst_size); chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr; if (op_type == CHCR_ENCRYPT_OP) @@ -2060,41 +2181,312 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req, memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) << 4), actx->h_iopad, kctx_len - (DIV_ROUND_UP(aeadctx->enckey_len, 16) << 4)); - + memcpy(reqctx->iv, req->iv, IV); phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); - sg_param.nents = reqctx->dst_nents; - sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize); - sg_param.qid = qid; - error = map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, - reqctx->dst, &sg_param); - if (error) - goto dstmap_fail; - - skb_set_transport_header(skb, transhdr_len); - - if (assoclen) { - /* AAD buffer in */ - write_sg_to_skb(skb, &frags, req->src, assoclen); - - } - write_buffer_to_skb(skb, &frags, req->iv, ivsize); - write_sg_to_skb(skb, &frags, src, req->cryptlen); + ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size); + chcr_add_aead_dst_ent(req, phys_cpl, assoclen, op_type, qid); + chcr_add_aead_src_ent(req, ulptx, assoclen, op_type); atomic_inc(&adap->chcr_stats.cipher_rqst); - create_wreq(ctx, chcr_req, &req->base, skb, kctx_len, size, 1, - sizeof(struct cpl_rx_phys_dsgl) + dst_size, 0); + temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + + kctx_len + (reqctx->imm ? (assoclen + IV + req->cryptlen) : 0); + create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, size, + transhdr_len, temp, 0); reqctx->skb = skb; - skb_get(skb); + reqctx->op = op_type; return skb; -dstmap_fail: - /* ivmap_fail: */ - kfree_skb(skb); err: - free_new_sg(reqctx->newdstsg); - reqctx->newdstsg = NULL; + chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, + op_type); + return ERR_PTR(error); } +static int chcr_aead_dma_map(struct device *dev, + struct aead_request *req, + unsigned short op_type) +{ + int error; + struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + unsigned int authsize = crypto_aead_authsize(tfm); + int dst_size; + + dst_size = req->assoclen + req->cryptlen + (op_type ? + -authsize : authsize); + if (!req->cryptlen || !dst_size) + return 0; + reqctx->iv_dma = dma_map_single(dev, reqctx->iv, IV, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(dev, reqctx->iv_dma)) + return -ENOMEM; + + if (req->src == req->dst) { + error = dma_map_sg(dev, req->src, sg_nents(req->src), + DMA_BIDIRECTIONAL); + if (!error) + goto err; + } else { + error = dma_map_sg(dev, req->src, sg_nents(req->src), + DMA_TO_DEVICE); + if (!error) + goto err; + error = dma_map_sg(dev, req->dst, sg_nents(req->dst), + DMA_FROM_DEVICE); + if (!error) { + dma_unmap_sg(dev, req->src, sg_nents(req->src), + DMA_TO_DEVICE); + goto err; + } + } + + return 0; +err: + dma_unmap_single(dev, reqctx->iv_dma, IV, DMA_BIDIRECTIONAL); + return -ENOMEM; +} + +static void chcr_aead_dma_unmap(struct device *dev, + struct aead_request *req, + unsigned short op_type) +{ + struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + unsigned int authsize = crypto_aead_authsize(tfm); + int dst_size; + + dst_size = req->assoclen + req->cryptlen + (op_type ? + -authsize : authsize); + if (!req->cryptlen || !dst_size) + return; + + dma_unmap_single(dev, reqctx->iv_dma, IV, + DMA_BIDIRECTIONAL); + if (req->src == req->dst) { + dma_unmap_sg(dev, req->src, sg_nents(req->src), + DMA_BIDIRECTIONAL); + } else { + dma_unmap_sg(dev, req->src, sg_nents(req->src), + DMA_TO_DEVICE); + dma_unmap_sg(dev, req->dst, sg_nents(req->dst), + DMA_FROM_DEVICE); + } +} + +static inline void chcr_add_aead_src_ent(struct aead_request *req, + struct ulptx_sgl *ulptx, + unsigned int assoclen, + unsigned short op_type) +{ + struct ulptx_walk ulp_walk; + struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); + + if (reqctx->imm) { + u8 *buf = (u8 *)ulptx; + + if (reqctx->b0_dma) { + memcpy(buf, reqctx->scratch_pad, reqctx->b0_len); + buf += reqctx->b0_len; + } + sg_pcopy_to_buffer(req->src, sg_nents(req->src), + buf, assoclen, 0); + buf += assoclen; + memcpy(buf, reqctx->iv, IV); + buf += IV; + sg_pcopy_to_buffer(req->src, sg_nents(req->src), + buf, req->cryptlen, req->assoclen); + } else { + ulptx_walk_init(&ulp_walk, ulptx); + if (reqctx->b0_dma) + ulptx_walk_add_page(&ulp_walk, reqctx->b0_len, + &reqctx->b0_dma); + ulptx_walk_add_sg(&ulp_walk, req->src, assoclen, 0); + ulptx_walk_add_page(&ulp_walk, IV, &reqctx->iv_dma); + ulptx_walk_add_sg(&ulp_walk, req->src, req->cryptlen, + req->assoclen); + ulptx_walk_end(&ulp_walk); + } +} + +static inline void chcr_add_aead_dst_ent(struct aead_request *req, + struct cpl_rx_phys_dsgl *phys_cpl, + unsigned int assoclen, + unsigned short op_type, + unsigned short qid) +{ + struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct dsgl_walk dsgl_walk; + unsigned int authsize = crypto_aead_authsize(tfm); + u32 temp; + + dsgl_walk_init(&dsgl_walk, phys_cpl); + if (reqctx->b0_dma) + dsgl_walk_add_page(&dsgl_walk, reqctx->b0_len, &reqctx->b0_dma); + dsgl_walk_add_sg(&dsgl_walk, req->dst, assoclen, 0); + dsgl_walk_add_page(&dsgl_walk, IV, &reqctx->iv_dma); + temp = req->cryptlen + (op_type ? -authsize : authsize); + dsgl_walk_add_sg(&dsgl_walk, req->dst, temp, req->assoclen); + dsgl_walk_end(&dsgl_walk, qid); +} + +static inline void chcr_add_cipher_src_ent(struct ablkcipher_request *req, + struct ulptx_sgl *ulptx, + struct cipher_wr_param *wrparam) +{ + struct ulptx_walk ulp_walk; + struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); + + if (reqctx->imm) { + u8 *buf = (u8 *)ulptx; + + memcpy(buf, reqctx->iv, IV); + buf += IV; + sg_pcopy_to_buffer(req->src, sg_nents(req->src), + buf, wrparam->bytes, reqctx->processed); + } else { + ulptx_walk_init(&ulp_walk, ulptx); + ulptx_walk_add_page(&ulp_walk, IV, &reqctx->iv_dma); + ulptx_walk_add_sg(&ulp_walk, reqctx->srcsg, wrparam->bytes, + reqctx->src_ofst); + reqctx->srcsg = ulp_walk.last_sg; + reqctx->src_ofst = ulp_walk.last_sg_len; + ulptx_walk_end(&ulp_walk); + } +} + +static inline void chcr_add_cipher_dst_ent(struct ablkcipher_request *req, + struct cpl_rx_phys_dsgl *phys_cpl, + struct cipher_wr_param *wrparam, + unsigned short qid) +{ + struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); + struct dsgl_walk dsgl_walk; + + dsgl_walk_init(&dsgl_walk, phys_cpl); + dsgl_walk_add_page(&dsgl_walk, IV, &reqctx->iv_dma); + dsgl_walk_add_sg(&dsgl_walk, reqctx->dstsg, wrparam->bytes, + reqctx->dst_ofst); + reqctx->dstsg = dsgl_walk.last_sg; + reqctx->dst_ofst = dsgl_walk.last_sg_len; + + dsgl_walk_end(&dsgl_walk, qid); +} + +static inline void chcr_add_hash_src_ent(struct ahash_request *req, + struct ulptx_sgl *ulptx, + struct hash_wr_param *param) +{ + struct ulptx_walk ulp_walk; + struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req); + + if (reqctx->imm) { + u8 *buf = (u8 *)ulptx; + + if (param->bfr_len) { + memcpy(buf, reqctx->reqbfr, param->bfr_len); + buf += param->bfr_len; + } + sg_pcopy_to_buffer(req->src, sg_nents(req->src), + buf, param->sg_len, 0); + } else { + ulptx_walk_init(&ulp_walk, ulptx); + if (param->bfr_len) + ulptx_walk_add_page(&ulp_walk, param->bfr_len, + &reqctx->dma_addr); + ulptx_walk_add_sg(&ulp_walk, req->src, param->sg_len, + 0); +// reqctx->srcsg = ulp_walk.last_sg; +// reqctx->src_ofst = ulp_walk.last_sg_len; + ulptx_walk_end(&ulp_walk); + } +} + + +static inline int chcr_hash_dma_map(struct device *dev, + struct ahash_request *req) +{ + struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); + int error = 0; + + if (!req->nbytes) + return 0; + error = dma_map_sg(dev, req->src, sg_nents(req->src), + DMA_TO_DEVICE); + if (!error) + return error; + req_ctx->is_sg_map = 1; + return 0; +} + +static inline void chcr_hash_dma_unmap(struct device *dev, + struct ahash_request *req) +{ + struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req); + + if (!req->nbytes) + return; + + dma_unmap_sg(dev, req->src, sg_nents(req->src), + DMA_TO_DEVICE); + req_ctx->is_sg_map = 0; + +} + + +static int chcr_cipher_dma_map(struct device *dev, + struct ablkcipher_request *req) +{ + int error; + struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); + + reqctx->iv_dma = dma_map_single(dev, reqctx->iv, IV, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(dev, reqctx->iv_dma)) + return -ENOMEM; + + if (req->src == req->dst) { + error = dma_map_sg(dev, req->src, sg_nents(req->src), + DMA_BIDIRECTIONAL); + if (!error) + goto err; + } else { + error = dma_map_sg(dev, req->src, sg_nents(req->src), + DMA_TO_DEVICE); + if (!error) + goto err; + error = dma_map_sg(dev, req->dst, sg_nents(req->dst), + DMA_FROM_DEVICE); + if (!error) { + dma_unmap_sg(dev, req->src, sg_nents(req->src), + DMA_TO_DEVICE); + goto err; + } + } + + return 0; +err: + dma_unmap_single(dev, reqctx->iv_dma, IV, DMA_BIDIRECTIONAL); + return -ENOMEM; +} +static void chcr_cipher_dma_unmap(struct device *dev, + struct ablkcipher_request *req) +{ + struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req); + + dma_unmap_single(dev, reqctx->iv_dma, IV, + DMA_BIDIRECTIONAL); + if (req->src == req->dst) { + dma_unmap_sg(dev, req->src, sg_nents(req->src), + DMA_BIDIRECTIONAL); + } else { + dma_unmap_sg(dev, req->src, sg_nents(req->src), + DMA_TO_DEVICE); + dma_unmap_sg(dev, req->dst, sg_nents(req->dst), + DMA_FROM_DEVICE); + } +} + static int set_msg_len(u8 *block, unsigned int msglen, int csize) { __be32 data; @@ -2179,15 +2571,13 @@ static int ccm_format_packet(struct aead_request *req, static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl, unsigned int dst_size, struct aead_request *req, - unsigned short op_type, - struct chcr_context *chcrctx) + unsigned short op_type) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); - unsigned int ivsize = AES_BLOCK_SIZE; + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); unsigned int cipher_mode = CHCR_SCMD_CIPHER_MODE_AES_CCM; unsigned int mac_mode = CHCR_SCMD_AUTH_MODE_CBCMAC; - unsigned int c_id = chcrctx->dev->rx_channel_id; + unsigned int c_id = a_ctx(tfm)->dev->rx_channel_id; unsigned int ccm_xtra; unsigned char tag_offset = 0, auth_offset = 0; unsigned int assoclen; @@ -2200,7 +2590,7 @@ static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl, ((assoclen) ? CCM_AAD_FIELD_SIZE : 0); auth_offset = req->cryptlen ? - (assoclen + ivsize + 1 + ccm_xtra) : 0; + (assoclen + IV + 1 + ccm_xtra) : 0; if (op_type == CHCR_DECRYPT_OP) { if (crypto_aead_authsize(tfm) != req->cryptlen) tag_offset = crypto_aead_authsize(tfm); @@ -2210,14 +2600,13 @@ static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl, sec_cpl->op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR(c_id, - 2, (ivsize ? (assoclen + 1) : 0) + - ccm_xtra); + 2, assoclen + 1 + ccm_xtra); sec_cpl->pldlen = - htonl(assoclen + ivsize + req->cryptlen + ccm_xtra); + htonl(assoclen + IV + req->cryptlen + ccm_xtra); /* For CCM there wil be b0 always. So AAD start will be 1 always */ sec_cpl->aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( 1, assoclen + ccm_xtra, assoclen - + ivsize + 1 + ccm_xtra, 0); + + IV + 1 + ccm_xtra, 0); sec_cpl->cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(0, auth_offset, tag_offset, @@ -2226,10 +2615,10 @@ static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl, sec_cpl->seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type, (op_type == CHCR_ENCRYPT_OP) ? 0 : 1, cipher_mode, mac_mode, - aeadctx->hmac_ctrl, ivsize >> 1); + aeadctx->hmac_ctrl, IV >> 1); sec_cpl->ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 0, - 1, dst_size); + 0, dst_size); } int aead_ccm_validate_input(unsigned short op_type, @@ -2249,131 +2638,83 @@ int aead_ccm_validate_input(unsigned short op_type, return -EINVAL; } } - if (aeadctx->enckey_len == 0) { - pr_err("CCM: Encryption key not set\n"); - return -EINVAL; - } return 0; } -unsigned int fill_aead_req_fields(struct sk_buff *skb, - struct aead_request *req, - struct scatterlist *src, - unsigned int ivsize, - struct chcr_aead_ctx *aeadctx) -{ - unsigned int frags = 0; - struct crypto_aead *tfm = crypto_aead_reqtfm(req); - struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); - /* b0 and aad length(if available) */ - - write_buffer_to_skb(skb, &frags, reqctx->scratch_pad, CCM_B0_SIZE + - (req->assoclen ? CCM_AAD_FIELD_SIZE : 0)); - if (req->assoclen) { - if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) - write_sg_to_skb(skb, &frags, req->src, - req->assoclen - 8); - else - write_sg_to_skb(skb, &frags, req->src, req->assoclen); - } - write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize); - if (req->cryptlen) - write_sg_to_skb(skb, &frags, src, req->cryptlen); - - return frags; -} - static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, unsigned short qid, int size, unsigned short op_type) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); - struct chcr_context *ctx = crypto_aead_ctx(tfm); - struct uld_ctx *u_ctx = ULD_CTX(ctx); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); struct sk_buff *skb = NULL; struct chcr_wr *chcr_req; struct cpl_rx_phys_dsgl *phys_cpl; - struct phys_sge_parm sg_param; - struct scatterlist *src; - unsigned int frags = 0, transhdr_len, ivsize = AES_BLOCK_SIZE; - unsigned int dst_size = 0, kctx_len, nents; - unsigned int sub_type; + struct ulptx_sgl *ulptx; + unsigned int transhdr_len; + unsigned int dst_size = 0, kctx_len, dnents, temp; + unsigned int sub_type, assoclen = req->assoclen; unsigned int authsize = crypto_aead_authsize(tfm); - int error = -EINVAL, src_nent; + int error = -EINVAL; gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; - struct adapter *adap = padap(ctx->dev); + struct adapter *adap = padap(a_ctx(tfm)->dev); - dst_size = req->assoclen + req->cryptlen + (op_type ? -authsize : + reqctx->b0_dma = 0; + sub_type = get_aead_subtype(tfm); + if (sub_type == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) + assoclen -= 8; + dst_size = assoclen + req->cryptlen + (op_type ? -authsize : authsize); - reqctx->newdstsg = NULL; - if (op_type && req->cryptlen < crypto_aead_authsize(tfm)) - goto err; - src_nent = sg_nents_for_len(req->src, req->assoclen + req->cryptlen); - if (src_nent < 0) - goto err; + error = chcr_aead_common_init(req, op_type); + if (error) + return ERR_PTR(error); - sub_type = get_aead_subtype(tfm); - src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen); - if (req->src != req->dst) { - error = chcr_copy_assoc(req, aeadctx); - if (error) { - pr_err("AAD copy to destination buffer fails\n"); - return ERR_PTR(error); - } - } - if (dst_size && is_newsg(req->dst, &nents)) { - reqctx->newdstsg = alloc_new_sg(req->dst, nents); - if (IS_ERR(reqctx->newdstsg)) - return ERR_CAST(reqctx->newdstsg); - reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, - reqctx->newdstsg, req->assoclen); - } else { - if (req->src == req->dst) - reqctx->dst = src; - else - reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, - req->dst, req->assoclen); - } - reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen + - (op_type ? -authsize : authsize)); - if (reqctx->dst_nents < 0) { - pr_err("CCM:Invalid Destination sg entries\n"); - error = -EINVAL; - goto err; - } + + reqctx->b0_len = CCM_B0_SIZE + (assoclen ? CCM_AAD_FIELD_SIZE : 0); error = aead_ccm_validate_input(op_type, req, aeadctx, sub_type); if (error) goto err; - - dst_size = get_space_for_phys_dsgl(reqctx->dst_nents); + if (dst_size) { + dnents = sg_nents_xlen(req->dst, assoclen, CHCR_DST_SG_SIZE, 0); + dnents += sg_nents_xlen(req->dst, req->cryptlen + + (op_type ? -authsize : authsize), + CHCR_DST_SG_SIZE, req->assoclen); + dnents += MIN_CCM_SG; // For IV and B0 + } else { + dnents = 0; + } + dst_size = get_space_for_phys_dsgl(dnents); kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) * 2; transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); - if (chcr_aead_need_fallback(req, src_nent + MIN_CCM_SG, - T6_MAX_AAD_SIZE - 18, - transhdr_len + (sgl_len(src_nent + MIN_CCM_SG) * 8), - op_type)) { + reqctx->imm = (transhdr_len + assoclen + IV + req->cryptlen + + reqctx->b0_len) <= SGE_MAX_WR_LEN; + temp = reqctx->imm ? (DIV_ROUND_UP((assoclen + IV + req->cryptlen + + reqctx->b0_len), 16) * 16) : + (sgl_len(reqctx->src_nents + reqctx->aad_nents + + MIN_CCM_SG) * 8); + transhdr_len += temp; + transhdr_len = DIV_ROUND_UP(transhdr_len, 16) * 16; + + if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE - + reqctx->b0_len, transhdr_len, op_type)) { atomic_inc(&adap->chcr_stats.fallback); - free_new_sg(reqctx->newdstsg); - reqctx->newdstsg = NULL; + chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, + op_type); return ERR_PTR(chcr_aead_fallback(req, op_type)); } - - skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); + skb = alloc_skb(SGE_MAX_WR_LEN, flags); if (!skb) { error = -ENOMEM; goto err; } - skb_reserve(skb, sizeof(struct sge_opaque_hdr)); - - chcr_req = __skb_put_zero(skb, transhdr_len); + chcr_req = (struct chcr_wr *) __skb_put_zero(skb, transhdr_len); - fill_sec_cpl_for_aead(&chcr_req->sec_cpl, dst_size, req, op_type, ctx); + fill_sec_cpl_for_aead(&chcr_req->sec_cpl, dst_size, req, op_type); chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr; memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len); @@ -2381,31 +2722,37 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req, 16), aeadctx->key, aeadctx->enckey_len); phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); + ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size); error = ccm_format_packet(req, aeadctx, sub_type, op_type); if (error) goto dstmap_fail; - sg_param.nents = reqctx->dst_nents; - sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize); - sg_param.qid = qid; - error = map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, - reqctx->dst, &sg_param); - if (error) + reqctx->b0_dma = dma_map_single(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, + &reqctx->scratch_pad, reqctx->b0_len, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, + reqctx->b0_dma)) { + error = -ENOMEM; goto dstmap_fail; + } + + chcr_add_aead_dst_ent(req, phys_cpl, assoclen, op_type, qid); + chcr_add_aead_src_ent(req, ulptx, assoclen, op_type); - skb_set_transport_header(skb, transhdr_len); - frags = fill_aead_req_fields(skb, req, src, ivsize, aeadctx); atomic_inc(&adap->chcr_stats.aead_rqst); - create_wreq(ctx, chcr_req, &req->base, skb, kctx_len, 0, 1, - sizeof(struct cpl_rx_phys_dsgl) + dst_size, 0); + temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + + kctx_len + (reqctx->imm ? (assoclen + IV + req->cryptlen + + reqctx->b0_len) : 0); + create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, 0, + transhdr_len, temp, 0); reqctx->skb = skb; - skb_get(skb); + reqctx->op = op_type; + return skb; dstmap_fail: kfree_skb(skb); err: - free_new_sg(reqctx->newdstsg); - reqctx->newdstsg = NULL; + chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, op_type); return ERR_PTR(error); } @@ -2415,115 +2762,84 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, unsigned short op_type) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); - struct chcr_context *ctx = crypto_aead_ctx(tfm); - struct uld_ctx *u_ctx = ULD_CTX(ctx); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); struct sk_buff *skb = NULL; struct chcr_wr *chcr_req; struct cpl_rx_phys_dsgl *phys_cpl; - struct phys_sge_parm sg_param; - struct scatterlist *src; - unsigned int frags = 0, transhdr_len; - unsigned int ivsize = AES_BLOCK_SIZE; - unsigned int dst_size = 0, kctx_len, nents, assoclen = req->assoclen; - unsigned char tag_offset = 0; + struct ulptx_sgl *ulptx; + unsigned int transhdr_len, dnents = 0; + unsigned int dst_size = 0, temp = 0, kctx_len, assoclen = req->assoclen; unsigned int authsize = crypto_aead_authsize(tfm); - int error = -EINVAL, src_nent; + int error = -EINVAL; gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; - struct adapter *adap = padap(ctx->dev); - - reqctx->newdstsg = NULL; - dst_size = assoclen + req->cryptlen + (op_type ? -authsize : - authsize); - /* validate key size */ - if (aeadctx->enckey_len == 0) - goto err; + struct adapter *adap = padap(a_ctx(tfm)->dev); - if (op_type && req->cryptlen < crypto_aead_authsize(tfm)) - goto err; - src_nent = sg_nents_for_len(req->src, assoclen + req->cryptlen); - if (src_nent < 0) - goto err; + if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) + assoclen = req->assoclen - 8; - src = scatterwalk_ffwd(reqctx->srcffwd, req->src, assoclen); - if (req->src != req->dst) { - error = chcr_copy_assoc(req, aeadctx); + reqctx->b0_dma = 0; + dst_size = assoclen + req->cryptlen + (op_type ? -authsize : authsize); + error = chcr_aead_common_init(req, op_type); if (error) return ERR_PTR(error); - } - - if (dst_size && is_newsg(req->dst, &nents)) { - reqctx->newdstsg = alloc_new_sg(req->dst, nents); - if (IS_ERR(reqctx->newdstsg)) - return ERR_CAST(reqctx->newdstsg); - reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, - reqctx->newdstsg, assoclen); + if (dst_size) { + dnents = sg_nents_xlen(req->dst, assoclen, CHCR_DST_SG_SIZE, 0); + dnents += sg_nents_xlen(req->dst, + req->cryptlen + (op_type ? -authsize : authsize), + CHCR_DST_SG_SIZE, req->assoclen); + dnents += MIN_GCM_SG; // For IV } else { - if (req->src == req->dst) - reqctx->dst = src; - else - reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, - req->dst, assoclen); - } - - reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen + - (op_type ? -authsize : authsize)); - if (reqctx->dst_nents < 0) { - pr_err("GCM:Invalid Destination sg entries\n"); - error = -EINVAL; - goto err; + dnents = 0; } - - - dst_size = get_space_for_phys_dsgl(reqctx->dst_nents); + dst_size = get_space_for_phys_dsgl(dnents); kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) + AEAD_H_SIZE; transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size); - if (chcr_aead_need_fallback(req, src_nent + MIN_GCM_SG, - T6_MAX_AAD_SIZE, - transhdr_len + (sgl_len(src_nent + MIN_GCM_SG) * 8), - op_type)) { + reqctx->imm = (transhdr_len + assoclen + IV + req->cryptlen) <= + SGE_MAX_WR_LEN; + temp = reqctx->imm ? (DIV_ROUND_UP((assoclen + IV + + req->cryptlen), 16) * 16) : (sgl_len(reqctx->src_nents + + reqctx->aad_nents + MIN_GCM_SG) * 8); + transhdr_len += temp; + transhdr_len = DIV_ROUND_UP(transhdr_len, 16) * 16; + if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE, + transhdr_len, op_type)) { atomic_inc(&adap->chcr_stats.fallback); - free_new_sg(reqctx->newdstsg); - reqctx->newdstsg = NULL; + chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, + op_type); return ERR_PTR(chcr_aead_fallback(req, op_type)); } - skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags); + skb = alloc_skb(SGE_MAX_WR_LEN, flags); if (!skb) { error = -ENOMEM; goto err; } - /* NIC driver is going to write the sge hdr. */ - skb_reserve(skb, sizeof(struct sge_opaque_hdr)); - chcr_req = __skb_put_zero(skb, transhdr_len); - if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) - assoclen = req->assoclen - 8; - - tag_offset = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize; + //Offset of tag from end + temp = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize; chcr_req->sec_cpl.op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR( - ctx->dev->rx_channel_id, 2, (ivsize ? - (assoclen + 1) : 0)); + a_ctx(tfm)->dev->rx_channel_id, 2, + (assoclen + 1)); chcr_req->sec_cpl.pldlen = - htonl(assoclen + ivsize + req->cryptlen); + htonl(assoclen + IV + req->cryptlen); chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI( assoclen ? 1 : 0, assoclen, - assoclen + ivsize + 1, 0); + assoclen + IV + 1, 0); chcr_req->sec_cpl.cipherstop_lo_authinsert = - FILL_SEC_CPL_AUTHINSERT(0, assoclen + ivsize + 1, - tag_offset, tag_offset); + FILL_SEC_CPL_AUTHINSERT(0, assoclen + IV + 1, + temp, temp); chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type, (op_type == CHCR_ENCRYPT_OP) ? 1 : 0, CHCR_SCMD_CIPHER_MODE_AES_GCM, CHCR_SCMD_AUTH_MODE_GHASH, - aeadctx->hmac_ctrl, ivsize >> 1); + aeadctx->hmac_ctrl, IV >> 1); chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, - 0, 1, dst_size); + 0, 0, dst_size); chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr; memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len); memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) * @@ -2534,39 +2850,28 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req, if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) { memcpy(reqctx->iv, aeadctx->salt, 4); - memcpy(reqctx->iv + 4, req->iv, 8); + memcpy(reqctx->iv + 4, req->iv, GCM_RFC4106_IV_SIZE); } else { - memcpy(reqctx->iv, req->iv, 12); + memcpy(reqctx->iv, req->iv, GCM_AES_IV_SIZE); } *((unsigned int *)(reqctx->iv + 12)) = htonl(0x01); phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len); - sg_param.nents = reqctx->dst_nents; - sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize); - sg_param.qid = qid; - error = map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, - reqctx->dst, &sg_param); - if (error) - goto dstmap_fail; + ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size); - skb_set_transport_header(skb, transhdr_len); - write_sg_to_skb(skb, &frags, req->src, assoclen); - write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize); - write_sg_to_skb(skb, &frags, src, req->cryptlen); + chcr_add_aead_dst_ent(req, phys_cpl, assoclen, op_type, qid); + chcr_add_aead_src_ent(req, ulptx, assoclen, op_type); atomic_inc(&adap->chcr_stats.aead_rqst); - create_wreq(ctx, chcr_req, &req->base, skb, kctx_len, size, 1, - sizeof(struct cpl_rx_phys_dsgl) + dst_size, - reqctx->verify); + temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + + kctx_len + (reqctx->imm ? (assoclen + IV + req->cryptlen) : 0); + create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, size, + transhdr_len, temp, reqctx->verify); reqctx->skb = skb; - skb_get(skb); + reqctx->op = op_type; return skb; -dstmap_fail: - /* ivmap_fail: */ - kfree_skb(skb); err: - free_new_sg(reqctx->newdstsg); - reqctx->newdstsg = NULL; + chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, op_type); return ERR_PTR(error); } @@ -2574,8 +2879,7 @@ err: static int chcr_aead_cra_init(struct crypto_aead *tfm) { - struct chcr_context *ctx = crypto_aead_ctx(tfm); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); struct aead_alg *alg = crypto_aead_alg(tfm); aeadctx->sw_cipher = crypto_alloc_aead(alg->base.cra_name, 0, @@ -2586,25 +2890,20 @@ static int chcr_aead_cra_init(struct crypto_aead *tfm) crypto_aead_set_reqsize(tfm, max(sizeof(struct chcr_aead_reqctx), sizeof(struct aead_request) + crypto_aead_reqsize(aeadctx->sw_cipher))); - aeadctx->null = crypto_get_default_null_skcipher(); - if (IS_ERR(aeadctx->null)) - return PTR_ERR(aeadctx->null); - return chcr_device_init(ctx); + return chcr_device_init(a_ctx(tfm)); } static void chcr_aead_cra_exit(struct crypto_aead *tfm) { - struct chcr_context *ctx = crypto_aead_ctx(tfm); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); - crypto_put_default_null_skcipher(); crypto_free_aead(aeadctx->sw_cipher); } static int chcr_authenc_null_setauthsize(struct crypto_aead *tfm, unsigned int authsize) { - struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NOP; aeadctx->mayverify = VERIFY_HW; @@ -2613,7 +2912,7 @@ static int chcr_authenc_null_setauthsize(struct crypto_aead *tfm, static int chcr_authenc_setauthsize(struct crypto_aead *tfm, unsigned int authsize) { - struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); u32 maxauth = crypto_aead_maxauthsize(tfm); /*SHA1 authsize in ipsec is 12 instead of 10 i.e maxauthsize / 2 is not @@ -2651,7 +2950,7 @@ static int chcr_authenc_setauthsize(struct crypto_aead *tfm, static int chcr_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) { - struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); switch (authsize) { case ICV_4: @@ -2691,7 +2990,7 @@ static int chcr_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) static int chcr_4106_4309_setauthsize(struct crypto_aead *tfm, unsigned int authsize) { - struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); switch (authsize) { case ICV_8: @@ -2717,7 +3016,7 @@ static int chcr_4106_4309_setauthsize(struct crypto_aead *tfm, static int chcr_ccm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) { - struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); switch (authsize) { case ICV_4: @@ -2760,8 +3059,7 @@ static int chcr_ccm_common_setkey(struct crypto_aead *aead, const u8 *key, unsigned int keylen) { - struct chcr_context *ctx = crypto_aead_ctx(aead); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead)); unsigned char ck_size, mk_size; int key_ctx_size = 0; @@ -2794,8 +3092,7 @@ static int chcr_aead_ccm_setkey(struct crypto_aead *aead, const u8 *key, unsigned int keylen) { - struct chcr_context *ctx = crypto_aead_ctx(aead); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead)); int error; crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK); @@ -2813,8 +3110,7 @@ static int chcr_aead_ccm_setkey(struct crypto_aead *aead, static int chcr_aead_rfc4309_setkey(struct crypto_aead *aead, const u8 *key, unsigned int keylen) { - struct chcr_context *ctx = crypto_aead_ctx(aead); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead)); int error; if (keylen < 3) { @@ -2840,8 +3136,7 @@ static int chcr_aead_rfc4309_setkey(struct crypto_aead *aead, const u8 *key, static int chcr_gcm_setkey(struct crypto_aead *aead, const u8 *key, unsigned int keylen) { - struct chcr_context *ctx = crypto_aead_ctx(aead); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead)); struct chcr_gcm_ctx *gctx = GCM_CTX(aeadctx); struct crypto_cipher *cipher; unsigned int ck_size; @@ -2913,8 +3208,7 @@ out: static int chcr_authenc_setkey(struct crypto_aead *authenc, const u8 *key, unsigned int keylen) { - struct chcr_context *ctx = crypto_aead_ctx(authenc); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(authenc)); struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx); /* it contains auth and cipher key both*/ struct crypto_authenc_keys keys; @@ -3034,8 +3328,7 @@ out: static int chcr_aead_digest_null_setkey(struct crypto_aead *authenc, const u8 *key, unsigned int keylen) { - struct chcr_context *ctx = crypto_aead_ctx(authenc); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(authenc)); struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx); struct crypto_authenc_keys keys; int err; @@ -3107,7 +3400,7 @@ static int chcr_aead_encrypt(struct aead_request *req) static int chcr_aead_decrypt(struct aead_request *req) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); - struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm)); + struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm)); struct chcr_aead_reqctx *reqctx = aead_request_ctx(req); int size; @@ -3140,30 +3433,29 @@ static int chcr_aead_op(struct aead_request *req, create_wr_t create_wr_fn) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); - struct chcr_context *ctx = crypto_aead_ctx(tfm); struct uld_ctx *u_ctx; struct sk_buff *skb; - if (!ctx->dev) { + if (!a_ctx(tfm)->dev) { pr_err("chcr : %s : No crypto device.\n", __func__); return -ENXIO; } - u_ctx = ULD_CTX(ctx); + u_ctx = ULD_CTX(a_ctx(tfm)); if (cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0], - ctx->tx_qidx)) { + a_ctx(tfm)->tx_qidx)) { if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) return -EBUSY; } /* Form a WR from req */ - skb = create_wr_fn(req, u_ctx->lldi.rxq_ids[ctx->rx_qidx], size, + skb = create_wr_fn(req, u_ctx->lldi.rxq_ids[a_ctx(tfm)->rx_qidx], size, op_type); if (IS_ERR(skb) || !skb) return PTR_ERR(skb); skb->dev = u_ctx->lldi.ports[0]; - set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx); + set_wr_txq(skb, CPL_PRIORITY_DATA, a_ctx(tfm)->tx_qidx); chcr_send_wr(skb); return -EINPROGRESS; } @@ -3385,7 +3677,7 @@ static struct chcr_alg_template driver_algs[] = { sizeof(struct chcr_aead_ctx) + sizeof(struct chcr_gcm_ctx), }, - .ivsize = 12, + .ivsize = GCM_AES_IV_SIZE, .maxauthsize = GHASH_DIGEST_SIZE, .setkey = chcr_gcm_setkey, .setauthsize = chcr_gcm_setauthsize, @@ -3405,7 +3697,7 @@ static struct chcr_alg_template driver_algs[] = { sizeof(struct chcr_gcm_ctx), }, - .ivsize = 8, + .ivsize = GCM_RFC4106_IV_SIZE, .maxauthsize = GHASH_DIGEST_SIZE, .setkey = chcr_gcm_setkey, .setauthsize = chcr_4106_4309_setauthsize, diff --git a/drivers/crypto/chelsio/chcr_algo.h b/drivers/crypto/chelsio/chcr_algo.h index 583008de51a3..96c9335ee728 100644 --- a/drivers/crypto/chelsio/chcr_algo.h +++ b/drivers/crypto/chelsio/chcr_algo.h @@ -176,21 +176,21 @@ KEY_CONTEXT_SALT_PRESENT_V(1) | \ KEY_CONTEXT_CTX_LEN_V((ctx_len))) -#define FILL_WR_OP_CCTX_SIZE(len, ctx_len) \ +#define FILL_WR_OP_CCTX_SIZE \ htonl( \ FW_CRYPTO_LOOKASIDE_WR_OPCODE_V( \ FW_CRYPTO_LOOKASIDE_WR) | \ FW_CRYPTO_LOOKASIDE_WR_COMPL_V(0) | \ - FW_CRYPTO_LOOKASIDE_WR_IMM_LEN_V((len)) | \ - FW_CRYPTO_LOOKASIDE_WR_CCTX_LOC_V(1) | \ - FW_CRYPTO_LOOKASIDE_WR_CCTX_SIZE_V((ctx_len))) + FW_CRYPTO_LOOKASIDE_WR_IMM_LEN_V((0)) | \ + FW_CRYPTO_LOOKASIDE_WR_CCTX_LOC_V(0) | \ + FW_CRYPTO_LOOKASIDE_WR_CCTX_SIZE_V(0)) -#define FILL_WR_RX_Q_ID(cid, qid, wr_iv, lcb, fid) \ +#define FILL_WR_RX_Q_ID(cid, qid, lcb, fid) \ htonl( \ FW_CRYPTO_LOOKASIDE_WR_RX_CHID_V((cid)) | \ FW_CRYPTO_LOOKASIDE_WR_RX_Q_ID_V((qid)) | \ FW_CRYPTO_LOOKASIDE_WR_LCB_V((lcb)) | \ - FW_CRYPTO_LOOKASIDE_WR_IV_V((wr_iv)) | \ + FW_CRYPTO_LOOKASIDE_WR_IV_V((IV_NOP)) | \ FW_CRYPTO_LOOKASIDE_WR_FQIDX_V(fid)) #define FILL_ULPTX_CMD_DEST(cid, qid) \ @@ -214,27 +214,22 @@ calc_tx_flits_ofld(skb) * 8), 16))) #define FILL_CMD_MORE(immdatalen) htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) |\ - ULP_TX_SC_MORE_V((immdatalen) ? 0 : 1)) - + ULP_TX_SC_MORE_V((immdatalen))) #define MAX_NK 8 -#define CRYPTO_MAX_IMM_TX_PKT_LEN 256 -#define MAX_WR_SIZE 512 #define ROUND_16(bytes) ((bytes) & 0xFFFFFFF0) #define MAX_DSGL_ENT 32 -#define MAX_DIGEST_SKB_SGE (MAX_SKB_FRAGS - 2) #define MIN_CIPHER_SG 1 /* IV */ -#define MIN_AUTH_SG 2 /*IV + AAD*/ -#define MIN_GCM_SG 2 /* IV + AAD*/ +#define MIN_AUTH_SG 1 /* IV */ +#define MIN_GCM_SG 1 /* IV */ #define MIN_DIGEST_SG 1 /*Partial Buffer*/ -#define MIN_CCM_SG 3 /*IV+AAD+B0*/ +#define MIN_CCM_SG 2 /*IV+B0*/ #define SPACE_LEFT(len) \ - ((MAX_WR_SIZE - WR_MIN_LEN - (len))) + ((SGE_MAX_WR_LEN - WR_MIN_LEN - (len))) -unsigned int sgl_ent_len[] = {0, 0, 16, 24, 40, - 48, 64, 72, 88, - 96, 112, 120, 136, - 144, 160, 168, 184, - 192}; +unsigned int sgl_ent_len[] = {0, 0, 16, 24, 40, 48, 64, 72, 88, + 96, 112, 120, 136, 144, 160, 168, 184, + 192, 208, 216, 232, 240, 256, 264, 280, + 288, 304, 312, 328, 336, 352, 360, 376}; unsigned int dsgl_ent_len[] = {0, 32, 32, 48, 48, 64, 64, 80, 80, 112, 112, 128, 128, 144, 144, 160, 160, 192, 192, 208, 208, 224, 224, 240, 240, @@ -258,10 +253,8 @@ struct hash_wr_param { struct cipher_wr_param { struct ablkcipher_request *req; - struct scatterlist *srcsg; char *iv; int bytes; - short int snent; unsigned short qid; }; enum { @@ -299,31 +292,11 @@ enum { ICV_16 = 16 }; -struct hash_op_params { - unsigned char mk_size; - unsigned char pad_align; - unsigned char auth_mode; - char hash_name[MAX_HASH_NAME]; - unsigned short block_size; - unsigned short word_size; - unsigned short ipad_size; -}; - struct phys_sge_pairs { __be16 len[8]; __be64 addr[8]; }; -struct phys_sge_parm { - unsigned int nents; - unsigned int obsize; - unsigned short qid; -}; - -struct crypto_result { - struct completion completion; - int err; -}; static const u32 sha1_init[SHA1_DIGEST_SIZE / 4] = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4, diff --git a/drivers/crypto/chelsio/chcr_core.c b/drivers/crypto/chelsio/chcr_core.c index b6dd9cbe815f..f5a2624081dc 100644 --- a/drivers/crypto/chelsio/chcr_core.c +++ b/drivers/crypto/chelsio/chcr_core.c @@ -154,15 +154,15 @@ static void *chcr_uld_add(const struct cxgb4_lld_info *lld) struct uld_ctx *u_ctx; /* Create the device and add it in the device list */ + if (!(lld->ulp_crypto & ULP_CRYPTO_LOOKASIDE)) + return ERR_PTR(-EOPNOTSUPP); + + /* Create the device and add it in the device list */ u_ctx = kzalloc(sizeof(*u_ctx), GFP_KERNEL); if (!u_ctx) { u_ctx = ERR_PTR(-ENOMEM); goto out; } - if (!(lld->ulp_crypto & ULP_CRYPTO_LOOKASIDE)) { - u_ctx = ERR_PTR(-ENOMEM); - goto out; - } u_ctx->lldi = *lld; out: return u_ctx; @@ -224,7 +224,7 @@ static int chcr_uld_state_change(void *handle, enum cxgb4_state state) static int __init chcr_crypto_init(void) { if (cxgb4_register_uld(CXGB4_ULD_CRYPTO, &chcr_uld_info)) - pr_err("ULD register fail: No chcr crypto support in cxgb4"); + pr_err("ULD register fail: No chcr crypto support in cxgb4\n"); return 0; } diff --git a/drivers/crypto/chelsio/chcr_core.h b/drivers/crypto/chelsio/chcr_core.h index c9a19b2a1e9f..94e7412f6164 100644 --- a/drivers/crypto/chelsio/chcr_core.h +++ b/drivers/crypto/chelsio/chcr_core.h @@ -89,7 +89,7 @@ struct uld_ctx { struct chcr_dev *dev; }; -struct uld_ctx * assign_chcr_device(void); +struct uld_ctx *assign_chcr_device(void); int chcr_send_wr(struct sk_buff *skb); int start_crypto(void); int stop_crypto(void); diff --git a/drivers/crypto/chelsio/chcr_crypto.h b/drivers/crypto/chelsio/chcr_crypto.h index 30af1ee17b87..94a87e3ad9bc 100644 --- a/drivers/crypto/chelsio/chcr_crypto.h +++ b/drivers/crypto/chelsio/chcr_crypto.h @@ -149,9 +149,23 @@ #define CHCR_HASH_MAX_BLOCK_SIZE_64 64 #define CHCR_HASH_MAX_BLOCK_SIZE_128 128 -#define CHCR_SG_SIZE 2048 +#define CHCR_SRC_SG_SIZE (0x10000 - sizeof(int)) +#define CHCR_DST_SG_SIZE 2048 -/* Aligned to 128 bit boundary */ +static inline struct chcr_context *a_ctx(struct crypto_aead *tfm) +{ + return crypto_aead_ctx(tfm); +} + +static inline struct chcr_context *c_ctx(struct crypto_ablkcipher *tfm) +{ + return crypto_ablkcipher_ctx(tfm); +} + +static inline struct chcr_context *h_ctx(struct crypto_ahash *tfm) +{ + return crypto_tfm_ctx(crypto_ahash_tfm(tfm)); +} struct ablk_ctx { struct crypto_skcipher *sw_cipher; @@ -165,16 +179,39 @@ struct ablk_ctx { }; struct chcr_aead_reqctx { struct sk_buff *skb; - struct scatterlist *dst; - struct scatterlist *newdstsg; - struct scatterlist srcffwd[2]; - struct scatterlist dstffwd[2]; + dma_addr_t iv_dma; + dma_addr_t b0_dma; + unsigned int b0_len; + unsigned int op; + short int aad_nents; + short int src_nents; short int dst_nents; + u16 imm; u16 verify; u8 iv[CHCR_MAX_CRYPTO_IV_LEN]; unsigned char scratch_pad[MAX_SCRATCH_PAD_SIZE]; }; +struct ulptx_walk { + struct ulptx_sgl *sgl; + unsigned int nents; + unsigned int pair_idx; + unsigned int last_sg_len; + struct scatterlist *last_sg; + struct ulptx_sge_pair *pair; + +}; + +struct dsgl_walk { + unsigned int nents; + unsigned int last_sg_len; + struct scatterlist *last_sg; + struct cpl_rx_phys_dsgl *dsgl; + struct phys_sge_pairs *to; +}; + + + struct chcr_gcm_ctx { u8 ghash_h[AEAD_H_SIZE]; }; @@ -195,7 +232,6 @@ struct __aead_ctx { struct chcr_aead_ctx { __be32 key_ctx_hdr; unsigned int enckey_len; - struct crypto_skcipher *null; struct crypto_aead *sw_cipher; u8 salt[MAX_SALT]; u8 key[CHCR_AES_MAX_KEY_LEN]; @@ -231,8 +267,11 @@ struct chcr_ahash_req_ctx { u8 bfr2[CHCR_HASH_MAX_BLOCK_SIZE_128]; u8 *reqbfr; u8 *skbfr; + dma_addr_t dma_addr; + u32 dma_len; u8 reqlen; - /* DMA the partial hash in it */ + u8 imm; + u8 is_sg_map; u8 partial_hash[CHCR_HASH_MAX_DIGEST_SIZE]; u64 data_len; /* Data len till time */ /* SKB which is being sent to the hardware for processing */ @@ -241,14 +280,15 @@ struct chcr_ahash_req_ctx { struct chcr_blkcipher_req_ctx { struct sk_buff *skb; - struct scatterlist srcffwd[2]; - struct scatterlist dstffwd[2]; struct scatterlist *dstsg; - struct scatterlist *dst; - struct scatterlist *newdstsg; unsigned int processed; + unsigned int last_req_len; + struct scatterlist *srcsg; + unsigned int src_ofst; + unsigned int dst_ofst; unsigned int op; - short int dst_nents; + dma_addr_t iv_dma; + u16 imm; u8 iv[CHCR_MAX_CRYPTO_IV_LEN]; }; @@ -262,24 +302,6 @@ struct chcr_alg_template { } alg; }; -struct chcr_req_ctx { - union { - struct ahash_request *ahash_req; - struct aead_request *aead_req; - struct ablkcipher_request *ablk_req; - } req; - union { - struct chcr_ahash_req_ctx *ahash_ctx; - struct chcr_aead_reqctx *reqctx; - struct chcr_blkcipher_req_ctx *ablk_ctx; - } ctx; -}; - -struct sge_opaque_hdr { - void *dev; - dma_addr_t addr[MAX_SKB_FRAGS + 1]; -}; - typedef struct sk_buff *(*create_wr_t)(struct aead_request *req, unsigned short qid, int size, @@ -290,10 +312,39 @@ static int chcr_aead_op(struct aead_request *req_base, int size, create_wr_t create_wr_fn); static inline int get_aead_subtype(struct crypto_aead *aead); -static int is_newsg(struct scatterlist *sgl, unsigned int *newents); -static struct scatterlist *alloc_new_sg(struct scatterlist *sgl, - unsigned int nents); -static inline void free_new_sg(struct scatterlist *sgl); static int chcr_handle_cipher_resp(struct ablkcipher_request *req, unsigned char *input, int err); +static void chcr_verify_tag(struct aead_request *req, u8 *input, int *err); +static int chcr_aead_dma_map(struct device *dev, struct aead_request *req, + unsigned short op_type); +static void chcr_aead_dma_unmap(struct device *dev, struct aead_request + *req, unsigned short op_type); +static inline void chcr_add_aead_dst_ent(struct aead_request *req, + struct cpl_rx_phys_dsgl *phys_cpl, + unsigned int assoclen, + unsigned short op_type, + unsigned short qid); +static inline void chcr_add_aead_src_ent(struct aead_request *req, + struct ulptx_sgl *ulptx, + unsigned int assoclen, + unsigned short op_type); +static inline void chcr_add_cipher_src_ent(struct ablkcipher_request *req, + struct ulptx_sgl *ulptx, + struct cipher_wr_param *wrparam); +static int chcr_cipher_dma_map(struct device *dev, + struct ablkcipher_request *req); +static void chcr_cipher_dma_unmap(struct device *dev, + struct ablkcipher_request *req); +static inline void chcr_add_cipher_dst_ent(struct ablkcipher_request *req, + struct cpl_rx_phys_dsgl *phys_cpl, + struct cipher_wr_param *wrparam, + unsigned short qid); +int sg_nents_len_skip(struct scatterlist *sg, u64 len, u64 skip); +static inline void chcr_add_hash_src_ent(struct ahash_request *req, + struct ulptx_sgl *ulptx, + struct hash_wr_param *param); +static inline int chcr_hash_dma_map(struct device *dev, + struct ahash_request *req); +static inline void chcr_hash_dma_unmap(struct device *dev, + struct ahash_request *req); #endif /* __CHCR_CRYPTO_H__ */ diff --git a/drivers/crypto/geode-aes.c b/drivers/crypto/geode-aes.c index fe538e5287a5..eb2a0a73cbed 100644 --- a/drivers/crypto/geode-aes.c +++ b/drivers/crypto/geode-aes.c @@ -1,10 +1,10 @@ /* Copyright (C) 2004-2006, Advanced Micro Devices, Inc. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - */ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + */ #include <linux/module.h> #include <linux/kernel.h> @@ -30,6 +30,7 @@ static inline void _writefield(u32 offset, void *value) { int i; + for (i = 0; i < 4; i++) iowrite32(((u32 *) value)[i], _iobase + offset + (i * 4)); } @@ -39,6 +40,7 @@ static inline void _readfield(u32 offset, void *value) { int i; + for (i = 0; i < 4; i++) ((u32 *) value)[i] = ioread32(_iobase + offset + (i * 4)); } @@ -515,6 +517,7 @@ static void geode_aes_remove(struct pci_dev *dev) static int geode_aes_probe(struct pci_dev *dev, const struct pci_device_id *id) { int ret; + ret = pci_enable_device(dev); if (ret) return ret; @@ -570,7 +573,7 @@ static int geode_aes_probe(struct pci_dev *dev, const struct pci_device_id *id) } static struct pci_device_id geode_aes_tbl[] = { - { PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_LX_AES), } , + { PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_LX_AES), }, { 0, } }; diff --git a/drivers/crypto/img-hash.c b/drivers/crypto/img-hash.c index 0c6a917a9ab8..b87000a0a01c 100644 --- a/drivers/crypto/img-hash.c +++ b/drivers/crypto/img-hash.c @@ -1054,7 +1054,7 @@ res_err: static int img_hash_remove(struct platform_device *pdev) { - static struct img_hash_dev *hdev; + struct img_hash_dev *hdev; hdev = platform_get_drvdata(pdev); spin_lock(&img_hash.lock); diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index e7f87ac12685..89ba9e85c0f3 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -773,7 +773,6 @@ static int safexcel_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; struct resource *res; struct safexcel_crypto_priv *priv; - u64 dma_mask; int i, ret; priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); @@ -802,9 +801,7 @@ static int safexcel_probe(struct platform_device *pdev) return -EPROBE_DEFER; } - if (of_property_read_u64(dev->of_node, "dma-mask", &dma_mask)) - dma_mask = DMA_BIT_MASK(64); - ret = dma_set_mask_and_coherent(dev, dma_mask); + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); if (ret) goto err_clk; @@ -842,9 +839,10 @@ static int safexcel_probe(struct platform_device *pdev) snprintf(irq_name, 6, "ring%d", i); irq = safexcel_request_ring_irq(pdev, irq_name, safexcel_irq_ring, ring_irq); - - if (irq < 0) + if (irq < 0) { + ret = irq; goto err_clk; + } priv->ring[i].work_data.priv = priv; priv->ring[i].work_data.ring = i; diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c index d2207ac5ba19..5438552bc6d7 100644 --- a/drivers/crypto/inside-secure/safexcel_cipher.c +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -386,7 +386,7 @@ static int safexcel_cipher_exit_inv(struct crypto_tfm *tfm) struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); struct safexcel_crypto_priv *priv = ctx->priv; struct skcipher_request req; - struct safexcel_inv_result result = { 0 }; + struct safexcel_inv_result result = {}; int ring = ctx->base.ring; memset(&req, 0, sizeof(struct skcipher_request)); diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index 8527a5899a2f..74feb6227101 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -308,10 +308,8 @@ unmap_cache: ctx->base.cache_sz = 0; } free_cache: - if (ctx->base.cache) { - kfree(ctx->base.cache); - ctx->base.cache = NULL; - } + kfree(ctx->base.cache); + ctx->base.cache = NULL; unlock: spin_unlock_bh(&priv->ring[ring].egress_lock); @@ -419,7 +417,7 @@ static int safexcel_ahash_exit_inv(struct crypto_tfm *tfm) struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); struct safexcel_crypto_priv *priv = ctx->priv; struct ahash_request req; - struct safexcel_inv_result result = { 0 }; + struct safexcel_inv_result result = {}; int ring = ctx->base.ring; memset(&req, 0, sizeof(struct ahash_request)); @@ -883,10 +881,7 @@ static int safexcel_hmac_sha1_setkey(struct crypto_ahash *tfm, const u8 *key, if (ret) return ret; - memcpy(ctx->ipad, &istate.state, SHA1_DIGEST_SIZE); - memcpy(ctx->opad, &ostate.state, SHA1_DIGEST_SIZE); - - for (i = 0; i < ARRAY_SIZE(istate.state); i++) { + for (i = 0; i < SHA1_DIGEST_SIZE / sizeof(u32); i++) { if (ctx->ipad[i] != le32_to_cpu(istate.state[i]) || ctx->opad[i] != le32_to_cpu(ostate.state[i])) { ctx->base.needs_inv = true; @@ -894,6 +889,9 @@ static int safexcel_hmac_sha1_setkey(struct crypto_ahash *tfm, const u8 *key, } } + memcpy(ctx->ipad, &istate.state, SHA1_DIGEST_SIZE); + memcpy(ctx->opad, &ostate.state, SHA1_DIGEST_SIZE); + return 0; } diff --git a/drivers/crypto/ixp4xx_crypto.c b/drivers/crypto/ixp4xx_crypto.c index 427cbe012729..8705b28eb02c 100644 --- a/drivers/crypto/ixp4xx_crypto.c +++ b/drivers/crypto/ixp4xx_crypto.c @@ -534,7 +534,6 @@ static void release_ixp_crypto(struct device *dev) NPE_QLEN_TOTAL * sizeof( struct crypt_ctl), crypt_virt, crypt_phys); } - return; } static void reset_sa_dir(struct ix_sa_dir *dir) @@ -1073,7 +1072,7 @@ static int aead_perform(struct aead_request *req, int encrypt, req_ctx->hmac_virt = dma_pool_alloc(buffer_pool, flags, &crypt->icv_rev_aes); if (unlikely(!req_ctx->hmac_virt)) - goto free_buf_src; + goto free_buf_dst; if (!encrypt) { scatterwalk_map_and_copy(req_ctx->hmac_virt, req->src, cryptlen, authsize, 0); @@ -1088,10 +1087,10 @@ static int aead_perform(struct aead_request *req, int encrypt, BUG_ON(qmgr_stat_overflow(SEND_QID)); return -EINPROGRESS; -free_buf_src: - free_buf_chain(dev, req_ctx->src, crypt->src_buf); free_buf_dst: free_buf_chain(dev, req_ctx->dst, crypt->dst_buf); +free_buf_src: + free_buf_chain(dev, req_ctx->src, crypt->src_buf); crypt->ctl_flags = CTL_FLAG_UNUSED; return -ENOMEM; } diff --git a/drivers/crypto/marvell/cesa.c b/drivers/crypto/marvell/cesa.c index 6e7a5c77a00a..293832488cc9 100644 --- a/drivers/crypto/marvell/cesa.c +++ b/drivers/crypto/marvell/cesa.c @@ -34,10 +34,6 @@ /* Limit of the crypto queue before reaching the backlog */ #define CESA_CRYPTO_DEFAULT_MAX_QLEN 128 -static int allhwsupport = !IS_ENABLED(CONFIG_CRYPTO_DEV_MV_CESA); -module_param_named(allhwsupport, allhwsupport, int, 0444); -MODULE_PARM_DESC(allhwsupport, "Enable support for all hardware (even it if overlaps with the mv_cesa driver)"); - struct mv_cesa_dev *cesa_dev; struct crypto_async_request * @@ -76,8 +72,6 @@ static void mv_cesa_rearm_engine(struct mv_cesa_engine *engine) ctx = crypto_tfm_ctx(req->tfm); ctx->ops->step(req); - - return; } static int mv_cesa_std_process(struct mv_cesa_engine *engine, u32 status) @@ -183,8 +177,7 @@ int mv_cesa_queue_req(struct crypto_async_request *req, spin_lock_bh(&engine->lock); ret = crypto_enqueue_request(&engine->queue, req); if ((mv_cesa_req_get_type(creq) == CESA_DMA_REQ) && - (ret == -EINPROGRESS || - (ret == -EBUSY && req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG))) + (ret == -EINPROGRESS || ret == -EBUSY)) mv_cesa_tdma_chain(engine, creq); spin_unlock_bh(&engine->lock); @@ -202,7 +195,7 @@ static int mv_cesa_add_algs(struct mv_cesa_dev *cesa) int i, j; for (i = 0; i < cesa->caps->ncipher_algs; i++) { - ret = crypto_register_alg(cesa->caps->cipher_algs[i]); + ret = crypto_register_skcipher(cesa->caps->cipher_algs[i]); if (ret) goto err_unregister_crypto; } @@ -222,7 +215,7 @@ err_unregister_ahash: err_unregister_crypto: for (j = 0; j < i; j++) - crypto_unregister_alg(cesa->caps->cipher_algs[j]); + crypto_unregister_skcipher(cesa->caps->cipher_algs[j]); return ret; } @@ -235,10 +228,10 @@ static void mv_cesa_remove_algs(struct mv_cesa_dev *cesa) crypto_unregister_ahash(cesa->caps->ahash_algs[i]); for (i = 0; i < cesa->caps->ncipher_algs; i++) - crypto_unregister_alg(cesa->caps->cipher_algs[i]); + crypto_unregister_skcipher(cesa->caps->cipher_algs[i]); } -static struct crypto_alg *orion_cipher_algs[] = { +static struct skcipher_alg *orion_cipher_algs[] = { &mv_cesa_ecb_des_alg, &mv_cesa_cbc_des_alg, &mv_cesa_ecb_des3_ede_alg, @@ -254,7 +247,7 @@ static struct ahash_alg *orion_ahash_algs[] = { &mv_ahmac_sha1_alg, }; -static struct crypto_alg *armada_370_cipher_algs[] = { +static struct skcipher_alg *armada_370_cipher_algs[] = { &mv_cesa_ecb_des_alg, &mv_cesa_cbc_des_alg, &mv_cesa_ecb_des3_ede_alg, @@ -459,9 +452,6 @@ static int mv_cesa_probe(struct platform_device *pdev) caps = match->data; } - if ((caps == &orion_caps || caps == &kirkwood_caps) && !allhwsupport) - return -ENOTSUPP; - cesa = devm_kzalloc(dev, sizeof(*cesa), GFP_KERNEL); if (!cesa) return -ENOMEM; @@ -599,9 +589,16 @@ static int mv_cesa_remove(struct platform_device *pdev) return 0; } +static const struct platform_device_id mv_cesa_plat_id_table[] = { + { .name = "mv_crypto" }, + { /* sentinel */ }, +}; +MODULE_DEVICE_TABLE(platform, mv_cesa_plat_id_table); + static struct platform_driver marvell_cesa = { .probe = mv_cesa_probe, .remove = mv_cesa_remove, + .id_table = mv_cesa_plat_id_table, .driver = { .name = "marvell-cesa", .of_match_table = mv_cesa_of_match_table, diff --git a/drivers/crypto/marvell/cesa.h b/drivers/crypto/marvell/cesa.h index b7872f62f674..d63a6ee905c9 100644 --- a/drivers/crypto/marvell/cesa.h +++ b/drivers/crypto/marvell/cesa.h @@ -1,9 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __MARVELL_CESA_H__ #define __MARVELL_CESA_H__ #include <crypto/algapi.h> #include <crypto/hash.h> #include <crypto/internal/hash.h> +#include <crypto/internal/skcipher.h> #include <linux/crypto.h> #include <linux/dmapool.h> @@ -372,7 +374,7 @@ struct mv_cesa_engine; struct mv_cesa_caps { int nengines; bool has_tdma; - struct crypto_alg **cipher_algs; + struct skcipher_alg **cipher_algs; int ncipher_algs; struct ahash_alg **ahash_algs; int nahash_algs; @@ -538,12 +540,12 @@ struct mv_cesa_sg_std_iter { }; /** - * struct mv_cesa_ablkcipher_std_req - cipher standard request + * struct mv_cesa_skcipher_std_req - cipher standard request * @op: operation context * @offset: current operation offset * @size: size of the crypto operation */ -struct mv_cesa_ablkcipher_std_req { +struct mv_cesa_skcipher_std_req { struct mv_cesa_op_ctx op; unsigned int offset; unsigned int size; @@ -551,14 +553,14 @@ struct mv_cesa_ablkcipher_std_req { }; /** - * struct mv_cesa_ablkcipher_req - cipher request + * struct mv_cesa_skcipher_req - cipher request * @req: type specific request information * @src_nents: number of entries in the src sg list * @dst_nents: number of entries in the dest sg list */ -struct mv_cesa_ablkcipher_req { +struct mv_cesa_skcipher_req { struct mv_cesa_req base; - struct mv_cesa_ablkcipher_std_req std; + struct mv_cesa_skcipher_std_req std; int src_nents; int dst_nents; }; @@ -763,7 +765,7 @@ static inline int mv_cesa_req_needs_cleanup(struct crypto_async_request *req, * the backlog and will be processed later. There's no need to * clean it up. */ - if (ret == -EBUSY && req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG) + if (ret == -EBUSY) return false; /* Request wasn't queued, we need to clean it up */ @@ -868,11 +870,11 @@ extern struct ahash_alg mv_ahmac_md5_alg; extern struct ahash_alg mv_ahmac_sha1_alg; extern struct ahash_alg mv_ahmac_sha256_alg; -extern struct crypto_alg mv_cesa_ecb_des_alg; -extern struct crypto_alg mv_cesa_cbc_des_alg; -extern struct crypto_alg mv_cesa_ecb_des3_ede_alg; -extern struct crypto_alg mv_cesa_cbc_des3_ede_alg; -extern struct crypto_alg mv_cesa_ecb_aes_alg; -extern struct crypto_alg mv_cesa_cbc_aes_alg; +extern struct skcipher_alg mv_cesa_ecb_des_alg; +extern struct skcipher_alg mv_cesa_cbc_des_alg; +extern struct skcipher_alg mv_cesa_ecb_des3_ede_alg; +extern struct skcipher_alg mv_cesa_cbc_des3_ede_alg; +extern struct skcipher_alg mv_cesa_ecb_aes_alg; +extern struct skcipher_alg mv_cesa_cbc_aes_alg; #endif /* __MARVELL_CESA_H__ */ diff --git a/drivers/crypto/marvell/cipher.c b/drivers/crypto/marvell/cipher.c index 098871a22a54..0ae84ec9e21c 100644 --- a/drivers/crypto/marvell/cipher.c +++ b/drivers/crypto/marvell/cipher.c @@ -32,23 +32,23 @@ struct mv_cesa_aes_ctx { struct crypto_aes_ctx aes; }; -struct mv_cesa_ablkcipher_dma_iter { +struct mv_cesa_skcipher_dma_iter { struct mv_cesa_dma_iter base; struct mv_cesa_sg_dma_iter src; struct mv_cesa_sg_dma_iter dst; }; static inline void -mv_cesa_ablkcipher_req_iter_init(struct mv_cesa_ablkcipher_dma_iter *iter, - struct ablkcipher_request *req) +mv_cesa_skcipher_req_iter_init(struct mv_cesa_skcipher_dma_iter *iter, + struct skcipher_request *req) { - mv_cesa_req_dma_iter_init(&iter->base, req->nbytes); + mv_cesa_req_dma_iter_init(&iter->base, req->cryptlen); mv_cesa_sg_dma_iter_init(&iter->src, req->src, DMA_TO_DEVICE); mv_cesa_sg_dma_iter_init(&iter->dst, req->dst, DMA_FROM_DEVICE); } static inline bool -mv_cesa_ablkcipher_req_iter_next_op(struct mv_cesa_ablkcipher_dma_iter *iter) +mv_cesa_skcipher_req_iter_next_op(struct mv_cesa_skcipher_dma_iter *iter) { iter->src.op_offset = 0; iter->dst.op_offset = 0; @@ -57,9 +57,9 @@ mv_cesa_ablkcipher_req_iter_next_op(struct mv_cesa_ablkcipher_dma_iter *iter) } static inline void -mv_cesa_ablkcipher_dma_cleanup(struct ablkcipher_request *req) +mv_cesa_skcipher_dma_cleanup(struct skcipher_request *req) { - struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req); + struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(req); if (req->dst != req->src) { dma_unmap_sg(cesa_dev->dev, req->dst, creq->dst_nents, @@ -73,20 +73,20 @@ mv_cesa_ablkcipher_dma_cleanup(struct ablkcipher_request *req) mv_cesa_dma_cleanup(&creq->base); } -static inline void mv_cesa_ablkcipher_cleanup(struct ablkcipher_request *req) +static inline void mv_cesa_skcipher_cleanup(struct skcipher_request *req) { - struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req); + struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(req); if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ) - mv_cesa_ablkcipher_dma_cleanup(req); + mv_cesa_skcipher_dma_cleanup(req); } -static void mv_cesa_ablkcipher_std_step(struct ablkcipher_request *req) +static void mv_cesa_skcipher_std_step(struct skcipher_request *req) { - struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req); - struct mv_cesa_ablkcipher_std_req *sreq = &creq->std; + struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(req); + struct mv_cesa_skcipher_std_req *sreq = &creq->std; struct mv_cesa_engine *engine = creq->base.engine; - size_t len = min_t(size_t, req->nbytes - sreq->offset, + size_t len = min_t(size_t, req->cryptlen - sreq->offset, CESA_SA_SRAM_PAYLOAD_SIZE); mv_cesa_adjust_op(engine, &sreq->op); @@ -114,11 +114,11 @@ static void mv_cesa_ablkcipher_std_step(struct ablkcipher_request *req) writel(CESA_SA_CMD_EN_CESA_SA_ACCL0, engine->regs + CESA_SA_CMD); } -static int mv_cesa_ablkcipher_std_process(struct ablkcipher_request *req, - u32 status) +static int mv_cesa_skcipher_std_process(struct skcipher_request *req, + u32 status) { - struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req); - struct mv_cesa_ablkcipher_std_req *sreq = &creq->std; + struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(req); + struct mv_cesa_skcipher_std_req *sreq = &creq->std; struct mv_cesa_engine *engine = creq->base.engine; size_t len; @@ -127,122 +127,130 @@ static int mv_cesa_ablkcipher_std_process(struct ablkcipher_request *req, sreq->size, sreq->offset); sreq->offset += len; - if (sreq->offset < req->nbytes) + if (sreq->offset < req->cryptlen) return -EINPROGRESS; return 0; } -static int mv_cesa_ablkcipher_process(struct crypto_async_request *req, - u32 status) +static int mv_cesa_skcipher_process(struct crypto_async_request *req, + u32 status) { - struct ablkcipher_request *ablkreq = ablkcipher_request_cast(req); - struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(ablkreq); + struct skcipher_request *skreq = skcipher_request_cast(req); + struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(skreq); struct mv_cesa_req *basereq = &creq->base; if (mv_cesa_req_get_type(basereq) == CESA_STD_REQ) - return mv_cesa_ablkcipher_std_process(ablkreq, status); + return mv_cesa_skcipher_std_process(skreq, status); return mv_cesa_dma_process(basereq, status); } -static void mv_cesa_ablkcipher_step(struct crypto_async_request *req) +static void mv_cesa_skcipher_step(struct crypto_async_request *req) { - struct ablkcipher_request *ablkreq = ablkcipher_request_cast(req); - struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(ablkreq); + struct skcipher_request *skreq = skcipher_request_cast(req); + struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(skreq); if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ) mv_cesa_dma_step(&creq->base); else - mv_cesa_ablkcipher_std_step(ablkreq); + mv_cesa_skcipher_std_step(skreq); } static inline void -mv_cesa_ablkcipher_dma_prepare(struct ablkcipher_request *req) +mv_cesa_skcipher_dma_prepare(struct skcipher_request *req) { - struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req); + struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(req); struct mv_cesa_req *basereq = &creq->base; mv_cesa_dma_prepare(basereq, basereq->engine); } static inline void -mv_cesa_ablkcipher_std_prepare(struct ablkcipher_request *req) +mv_cesa_skcipher_std_prepare(struct skcipher_request *req) { - struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req); - struct mv_cesa_ablkcipher_std_req *sreq = &creq->std; + struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(req); + struct mv_cesa_skcipher_std_req *sreq = &creq->std; sreq->size = 0; sreq->offset = 0; } -static inline void mv_cesa_ablkcipher_prepare(struct crypto_async_request *req, - struct mv_cesa_engine *engine) +static inline void mv_cesa_skcipher_prepare(struct crypto_async_request *req, + struct mv_cesa_engine *engine) { - struct ablkcipher_request *ablkreq = ablkcipher_request_cast(req); - struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(ablkreq); + struct skcipher_request *skreq = skcipher_request_cast(req); + struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(skreq); creq->base.engine = engine; if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ) - mv_cesa_ablkcipher_dma_prepare(ablkreq); + mv_cesa_skcipher_dma_prepare(skreq); else - mv_cesa_ablkcipher_std_prepare(ablkreq); + mv_cesa_skcipher_std_prepare(skreq); } static inline void -mv_cesa_ablkcipher_req_cleanup(struct crypto_async_request *req) +mv_cesa_skcipher_req_cleanup(struct crypto_async_request *req) { - struct ablkcipher_request *ablkreq = ablkcipher_request_cast(req); + struct skcipher_request *skreq = skcipher_request_cast(req); - mv_cesa_ablkcipher_cleanup(ablkreq); + mv_cesa_skcipher_cleanup(skreq); } static void -mv_cesa_ablkcipher_complete(struct crypto_async_request *req) +mv_cesa_skcipher_complete(struct crypto_async_request *req) { - struct ablkcipher_request *ablkreq = ablkcipher_request_cast(req); - struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(ablkreq); + struct skcipher_request *skreq = skcipher_request_cast(req); + struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(skreq); struct mv_cesa_engine *engine = creq->base.engine; unsigned int ivsize; - atomic_sub(ablkreq->nbytes, &engine->load); - ivsize = crypto_ablkcipher_ivsize(crypto_ablkcipher_reqtfm(ablkreq)); + atomic_sub(skreq->cryptlen, &engine->load); + ivsize = crypto_skcipher_ivsize(crypto_skcipher_reqtfm(skreq)); if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ) { struct mv_cesa_req *basereq; basereq = &creq->base; - memcpy(ablkreq->info, basereq->chain.last->op->ctx.blkcipher.iv, + memcpy(skreq->iv, basereq->chain.last->op->ctx.blkcipher.iv, ivsize); } else { - memcpy_fromio(ablkreq->info, + memcpy_fromio(skreq->iv, engine->sram + CESA_SA_CRYPT_IV_SRAM_OFFSET, ivsize); } } -static const struct mv_cesa_req_ops mv_cesa_ablkcipher_req_ops = { - .step = mv_cesa_ablkcipher_step, - .process = mv_cesa_ablkcipher_process, - .cleanup = mv_cesa_ablkcipher_req_cleanup, - .complete = mv_cesa_ablkcipher_complete, +static const struct mv_cesa_req_ops mv_cesa_skcipher_req_ops = { + .step = mv_cesa_skcipher_step, + .process = mv_cesa_skcipher_process, + .cleanup = mv_cesa_skcipher_req_cleanup, + .complete = mv_cesa_skcipher_complete, }; -static int mv_cesa_ablkcipher_cra_init(struct crypto_tfm *tfm) +static void mv_cesa_skcipher_cra_exit(struct crypto_tfm *tfm) { - struct mv_cesa_aes_ctx *ctx = crypto_tfm_ctx(tfm); + void *ctx = crypto_tfm_ctx(tfm); + + memzero_explicit(ctx, tfm->__crt_alg->cra_ctxsize); +} + +static int mv_cesa_skcipher_cra_init(struct crypto_tfm *tfm) +{ + struct mv_cesa_ctx *ctx = crypto_tfm_ctx(tfm); - ctx->base.ops = &mv_cesa_ablkcipher_req_ops; + ctx->ops = &mv_cesa_skcipher_req_ops; - tfm->crt_ablkcipher.reqsize = sizeof(struct mv_cesa_ablkcipher_req); + crypto_skcipher_set_reqsize(__crypto_skcipher_cast(tfm), + sizeof(struct mv_cesa_skcipher_req)); return 0; } -static int mv_cesa_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key, +static int mv_cesa_aes_setkey(struct crypto_skcipher *cipher, const u8 *key, unsigned int len) { - struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); + struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher); struct mv_cesa_aes_ctx *ctx = crypto_tfm_ctx(tfm); int remaining; int offset; @@ -251,7 +259,7 @@ static int mv_cesa_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key, ret = crypto_aes_expand_key(&ctx->aes, key, len); if (ret) { - crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); + crypto_skcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); return ret; } @@ -264,16 +272,16 @@ static int mv_cesa_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key, return 0; } -static int mv_cesa_des_setkey(struct crypto_ablkcipher *cipher, const u8 *key, +static int mv_cesa_des_setkey(struct crypto_skcipher *cipher, const u8 *key, unsigned int len) { - struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); + struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher); struct mv_cesa_des_ctx *ctx = crypto_tfm_ctx(tfm); u32 tmp[DES_EXPKEY_WORDS]; int ret; if (len != DES_KEY_SIZE) { - crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); + crypto_skcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); return -EINVAL; } @@ -288,14 +296,14 @@ static int mv_cesa_des_setkey(struct crypto_ablkcipher *cipher, const u8 *key, return 0; } -static int mv_cesa_des3_ede_setkey(struct crypto_ablkcipher *cipher, +static int mv_cesa_des3_ede_setkey(struct crypto_skcipher *cipher, const u8 *key, unsigned int len) { - struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); + struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher); struct mv_cesa_des_ctx *ctx = crypto_tfm_ctx(tfm); if (len != DES3_EDE_KEY_SIZE) { - crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); + crypto_skcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); return -EINVAL; } @@ -304,14 +312,14 @@ static int mv_cesa_des3_ede_setkey(struct crypto_ablkcipher *cipher, return 0; } -static int mv_cesa_ablkcipher_dma_req_init(struct ablkcipher_request *req, - const struct mv_cesa_op_ctx *op_templ) +static int mv_cesa_skcipher_dma_req_init(struct skcipher_request *req, + const struct mv_cesa_op_ctx *op_templ) { - struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req); + struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(req); gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : GFP_ATOMIC; struct mv_cesa_req *basereq = &creq->base; - struct mv_cesa_ablkcipher_dma_iter iter; + struct mv_cesa_skcipher_dma_iter iter; bool skip_ctx = false; int ret; unsigned int ivsize; @@ -339,7 +347,7 @@ static int mv_cesa_ablkcipher_dma_req_init(struct ablkcipher_request *req, } mv_cesa_tdma_desc_iter_init(&basereq->chain); - mv_cesa_ablkcipher_req_iter_init(&iter, req); + mv_cesa_skcipher_req_iter_init(&iter, req); do { struct mv_cesa_op_ctx *op; @@ -370,10 +378,10 @@ static int mv_cesa_ablkcipher_dma_req_init(struct ablkcipher_request *req, if (ret) goto err_free_tdma; - } while (mv_cesa_ablkcipher_req_iter_next_op(&iter)); + } while (mv_cesa_skcipher_req_iter_next_op(&iter)); /* Add output data for IV */ - ivsize = crypto_ablkcipher_ivsize(crypto_ablkcipher_reqtfm(req)); + ivsize = crypto_skcipher_ivsize(crypto_skcipher_reqtfm(req)); ret = mv_cesa_dma_add_result_op(&basereq->chain, CESA_SA_CFG_SRAM_OFFSET, CESA_SA_DATA_SRAM_OFFSET, CESA_TDMA_SRC_IN_SRAM, flags); @@ -399,11 +407,11 @@ err_unmap_src: } static inline int -mv_cesa_ablkcipher_std_req_init(struct ablkcipher_request *req, - const struct mv_cesa_op_ctx *op_templ) +mv_cesa_skcipher_std_req_init(struct skcipher_request *req, + const struct mv_cesa_op_ctx *op_templ) { - struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req); - struct mv_cesa_ablkcipher_std_req *sreq = &creq->std; + struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(req); + struct mv_cesa_skcipher_std_req *sreq = &creq->std; struct mv_cesa_req *basereq = &creq->base; sreq->op = *op_templ; @@ -414,23 +422,23 @@ mv_cesa_ablkcipher_std_req_init(struct ablkcipher_request *req, return 0; } -static int mv_cesa_ablkcipher_req_init(struct ablkcipher_request *req, - struct mv_cesa_op_ctx *tmpl) +static int mv_cesa_skcipher_req_init(struct skcipher_request *req, + struct mv_cesa_op_ctx *tmpl) { - struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req); - struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); - unsigned int blksize = crypto_ablkcipher_blocksize(tfm); + struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(req); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + unsigned int blksize = crypto_skcipher_blocksize(tfm); int ret; - if (!IS_ALIGNED(req->nbytes, blksize)) + if (!IS_ALIGNED(req->cryptlen, blksize)) return -EINVAL; - creq->src_nents = sg_nents_for_len(req->src, req->nbytes); + creq->src_nents = sg_nents_for_len(req->src, req->cryptlen); if (creq->src_nents < 0) { dev_err(cesa_dev->dev, "Invalid number of src SG"); return creq->src_nents; } - creq->dst_nents = sg_nents_for_len(req->dst, req->nbytes); + creq->dst_nents = sg_nents_for_len(req->dst, req->cryptlen); if (creq->dst_nents < 0) { dev_err(cesa_dev->dev, "Invalid number of dst SG"); return creq->dst_nents; @@ -440,36 +448,36 @@ static int mv_cesa_ablkcipher_req_init(struct ablkcipher_request *req, CESA_SA_DESC_CFG_OP_MSK); if (cesa_dev->caps->has_tdma) - ret = mv_cesa_ablkcipher_dma_req_init(req, tmpl); + ret = mv_cesa_skcipher_dma_req_init(req, tmpl); else - ret = mv_cesa_ablkcipher_std_req_init(req, tmpl); + ret = mv_cesa_skcipher_std_req_init(req, tmpl); return ret; } -static int mv_cesa_ablkcipher_queue_req(struct ablkcipher_request *req, - struct mv_cesa_op_ctx *tmpl) +static int mv_cesa_skcipher_queue_req(struct skcipher_request *req, + struct mv_cesa_op_ctx *tmpl) { int ret; - struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req); + struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(req); struct mv_cesa_engine *engine; - ret = mv_cesa_ablkcipher_req_init(req, tmpl); + ret = mv_cesa_skcipher_req_init(req, tmpl); if (ret) return ret; - engine = mv_cesa_select_engine(req->nbytes); - mv_cesa_ablkcipher_prepare(&req->base, engine); + engine = mv_cesa_select_engine(req->cryptlen); + mv_cesa_skcipher_prepare(&req->base, engine); ret = mv_cesa_queue_req(&req->base, &creq->base); if (mv_cesa_req_needs_cleanup(&req->base, ret)) - mv_cesa_ablkcipher_cleanup(req); + mv_cesa_skcipher_cleanup(req); return ret; } -static int mv_cesa_des_op(struct ablkcipher_request *req, +static int mv_cesa_des_op(struct skcipher_request *req, struct mv_cesa_op_ctx *tmpl) { struct mv_cesa_des_ctx *ctx = crypto_tfm_ctx(req->base.tfm); @@ -479,10 +487,10 @@ static int mv_cesa_des_op(struct ablkcipher_request *req, memcpy(tmpl->ctx.blkcipher.key, ctx->key, DES_KEY_SIZE); - return mv_cesa_ablkcipher_queue_req(req, tmpl); + return mv_cesa_skcipher_queue_req(req, tmpl); } -static int mv_cesa_ecb_des_encrypt(struct ablkcipher_request *req) +static int mv_cesa_ecb_des_encrypt(struct skcipher_request *req) { struct mv_cesa_op_ctx tmpl; @@ -493,7 +501,7 @@ static int mv_cesa_ecb_des_encrypt(struct ablkcipher_request *req) return mv_cesa_des_op(req, &tmpl); } -static int mv_cesa_ecb_des_decrypt(struct ablkcipher_request *req) +static int mv_cesa_ecb_des_decrypt(struct skcipher_request *req) { struct mv_cesa_op_ctx tmpl; @@ -504,41 +512,38 @@ static int mv_cesa_ecb_des_decrypt(struct ablkcipher_request *req) return mv_cesa_des_op(req, &tmpl); } -struct crypto_alg mv_cesa_ecb_des_alg = { - .cra_name = "ecb(des)", - .cra_driver_name = "mv-ecb-des", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | - CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, - .cra_blocksize = DES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct mv_cesa_des_ctx), - .cra_alignmask = 0, - .cra_type = &crypto_ablkcipher_type, - .cra_module = THIS_MODULE, - .cra_init = mv_cesa_ablkcipher_cra_init, - .cra_u = { - .ablkcipher = { - .min_keysize = DES_KEY_SIZE, - .max_keysize = DES_KEY_SIZE, - .setkey = mv_cesa_des_setkey, - .encrypt = mv_cesa_ecb_des_encrypt, - .decrypt = mv_cesa_ecb_des_decrypt, - }, +struct skcipher_alg mv_cesa_ecb_des_alg = { + .setkey = mv_cesa_des_setkey, + .encrypt = mv_cesa_ecb_des_encrypt, + .decrypt = mv_cesa_ecb_des_decrypt, + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .base = { + .cra_name = "ecb(des)", + .cra_driver_name = "mv-ecb-des", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct mv_cesa_des_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = mv_cesa_skcipher_cra_init, + .cra_exit = mv_cesa_skcipher_cra_exit, }, }; -static int mv_cesa_cbc_des_op(struct ablkcipher_request *req, +static int mv_cesa_cbc_des_op(struct skcipher_request *req, struct mv_cesa_op_ctx *tmpl) { mv_cesa_update_op_cfg(tmpl, CESA_SA_DESC_CFG_CRYPTCM_CBC, CESA_SA_DESC_CFG_CRYPTCM_MSK); - memcpy(tmpl->ctx.blkcipher.iv, req->info, DES_BLOCK_SIZE); + memcpy(tmpl->ctx.blkcipher.iv, req->iv, DES_BLOCK_SIZE); return mv_cesa_des_op(req, tmpl); } -static int mv_cesa_cbc_des_encrypt(struct ablkcipher_request *req) +static int mv_cesa_cbc_des_encrypt(struct skcipher_request *req) { struct mv_cesa_op_ctx tmpl; @@ -547,7 +552,7 @@ static int mv_cesa_cbc_des_encrypt(struct ablkcipher_request *req) return mv_cesa_cbc_des_op(req, &tmpl); } -static int mv_cesa_cbc_des_decrypt(struct ablkcipher_request *req) +static int mv_cesa_cbc_des_decrypt(struct skcipher_request *req) { struct mv_cesa_op_ctx tmpl; @@ -556,31 +561,28 @@ static int mv_cesa_cbc_des_decrypt(struct ablkcipher_request *req) return mv_cesa_cbc_des_op(req, &tmpl); } -struct crypto_alg mv_cesa_cbc_des_alg = { - .cra_name = "cbc(des)", - .cra_driver_name = "mv-cbc-des", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | - CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, - .cra_blocksize = DES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct mv_cesa_des_ctx), - .cra_alignmask = 0, - .cra_type = &crypto_ablkcipher_type, - .cra_module = THIS_MODULE, - .cra_init = mv_cesa_ablkcipher_cra_init, - .cra_u = { - .ablkcipher = { - .min_keysize = DES_KEY_SIZE, - .max_keysize = DES_KEY_SIZE, - .ivsize = DES_BLOCK_SIZE, - .setkey = mv_cesa_des_setkey, - .encrypt = mv_cesa_cbc_des_encrypt, - .decrypt = mv_cesa_cbc_des_decrypt, - }, +struct skcipher_alg mv_cesa_cbc_des_alg = { + .setkey = mv_cesa_des_setkey, + .encrypt = mv_cesa_cbc_des_encrypt, + .decrypt = mv_cesa_cbc_des_decrypt, + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + .base = { + .cra_name = "cbc(des)", + .cra_driver_name = "mv-cbc-des", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct mv_cesa_des_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = mv_cesa_skcipher_cra_init, + .cra_exit = mv_cesa_skcipher_cra_exit, }, }; -static int mv_cesa_des3_op(struct ablkcipher_request *req, +static int mv_cesa_des3_op(struct skcipher_request *req, struct mv_cesa_op_ctx *tmpl) { struct mv_cesa_des3_ctx *ctx = crypto_tfm_ctx(req->base.tfm); @@ -590,10 +592,10 @@ static int mv_cesa_des3_op(struct ablkcipher_request *req, memcpy(tmpl->ctx.blkcipher.key, ctx->key, DES3_EDE_KEY_SIZE); - return mv_cesa_ablkcipher_queue_req(req, tmpl); + return mv_cesa_skcipher_queue_req(req, tmpl); } -static int mv_cesa_ecb_des3_ede_encrypt(struct ablkcipher_request *req) +static int mv_cesa_ecb_des3_ede_encrypt(struct skcipher_request *req) { struct mv_cesa_op_ctx tmpl; @@ -605,7 +607,7 @@ static int mv_cesa_ecb_des3_ede_encrypt(struct ablkcipher_request *req) return mv_cesa_des3_op(req, &tmpl); } -static int mv_cesa_ecb_des3_ede_decrypt(struct ablkcipher_request *req) +static int mv_cesa_ecb_des3_ede_decrypt(struct skcipher_request *req) { struct mv_cesa_op_ctx tmpl; @@ -617,39 +619,36 @@ static int mv_cesa_ecb_des3_ede_decrypt(struct ablkcipher_request *req) return mv_cesa_des3_op(req, &tmpl); } -struct crypto_alg mv_cesa_ecb_des3_ede_alg = { - .cra_name = "ecb(des3_ede)", - .cra_driver_name = "mv-ecb-des3-ede", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | - CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, - .cra_blocksize = DES3_EDE_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct mv_cesa_des3_ctx), - .cra_alignmask = 0, - .cra_type = &crypto_ablkcipher_type, - .cra_module = THIS_MODULE, - .cra_init = mv_cesa_ablkcipher_cra_init, - .cra_u = { - .ablkcipher = { - .min_keysize = DES3_EDE_KEY_SIZE, - .max_keysize = DES3_EDE_KEY_SIZE, - .ivsize = DES3_EDE_BLOCK_SIZE, - .setkey = mv_cesa_des3_ede_setkey, - .encrypt = mv_cesa_ecb_des3_ede_encrypt, - .decrypt = mv_cesa_ecb_des3_ede_decrypt, - }, +struct skcipher_alg mv_cesa_ecb_des3_ede_alg = { + .setkey = mv_cesa_des3_ede_setkey, + .encrypt = mv_cesa_ecb_des3_ede_encrypt, + .decrypt = mv_cesa_ecb_des3_ede_decrypt, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + .base = { + .cra_name = "ecb(des3_ede)", + .cra_driver_name = "mv-ecb-des3-ede", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct mv_cesa_des3_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = mv_cesa_skcipher_cra_init, + .cra_exit = mv_cesa_skcipher_cra_exit, }, }; -static int mv_cesa_cbc_des3_op(struct ablkcipher_request *req, +static int mv_cesa_cbc_des3_op(struct skcipher_request *req, struct mv_cesa_op_ctx *tmpl) { - memcpy(tmpl->ctx.blkcipher.iv, req->info, DES3_EDE_BLOCK_SIZE); + memcpy(tmpl->ctx.blkcipher.iv, req->iv, DES3_EDE_BLOCK_SIZE); return mv_cesa_des3_op(req, tmpl); } -static int mv_cesa_cbc_des3_ede_encrypt(struct ablkcipher_request *req) +static int mv_cesa_cbc_des3_ede_encrypt(struct skcipher_request *req) { struct mv_cesa_op_ctx tmpl; @@ -661,7 +660,7 @@ static int mv_cesa_cbc_des3_ede_encrypt(struct ablkcipher_request *req) return mv_cesa_cbc_des3_op(req, &tmpl); } -static int mv_cesa_cbc_des3_ede_decrypt(struct ablkcipher_request *req) +static int mv_cesa_cbc_des3_ede_decrypt(struct skcipher_request *req) { struct mv_cesa_op_ctx tmpl; @@ -673,31 +672,28 @@ static int mv_cesa_cbc_des3_ede_decrypt(struct ablkcipher_request *req) return mv_cesa_cbc_des3_op(req, &tmpl); } -struct crypto_alg mv_cesa_cbc_des3_ede_alg = { - .cra_name = "cbc(des3_ede)", - .cra_driver_name = "mv-cbc-des3-ede", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | - CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, - .cra_blocksize = DES3_EDE_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct mv_cesa_des3_ctx), - .cra_alignmask = 0, - .cra_type = &crypto_ablkcipher_type, - .cra_module = THIS_MODULE, - .cra_init = mv_cesa_ablkcipher_cra_init, - .cra_u = { - .ablkcipher = { - .min_keysize = DES3_EDE_KEY_SIZE, - .max_keysize = DES3_EDE_KEY_SIZE, - .ivsize = DES3_EDE_BLOCK_SIZE, - .setkey = mv_cesa_des3_ede_setkey, - .encrypt = mv_cesa_cbc_des3_ede_encrypt, - .decrypt = mv_cesa_cbc_des3_ede_decrypt, - }, +struct skcipher_alg mv_cesa_cbc_des3_ede_alg = { + .setkey = mv_cesa_des3_ede_setkey, + .encrypt = mv_cesa_cbc_des3_ede_encrypt, + .decrypt = mv_cesa_cbc_des3_ede_decrypt, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + .base = { + .cra_name = "cbc(des3_ede)", + .cra_driver_name = "mv-cbc-des3-ede", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct mv_cesa_des3_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = mv_cesa_skcipher_cra_init, + .cra_exit = mv_cesa_skcipher_cra_exit, }, }; -static int mv_cesa_aes_op(struct ablkcipher_request *req, +static int mv_cesa_aes_op(struct skcipher_request *req, struct mv_cesa_op_ctx *tmpl) { struct mv_cesa_aes_ctx *ctx = crypto_tfm_ctx(req->base.tfm); @@ -724,10 +720,10 @@ static int mv_cesa_aes_op(struct ablkcipher_request *req, CESA_SA_DESC_CFG_CRYPTM_MSK | CESA_SA_DESC_CFG_AES_LEN_MSK); - return mv_cesa_ablkcipher_queue_req(req, tmpl); + return mv_cesa_skcipher_queue_req(req, tmpl); } -static int mv_cesa_ecb_aes_encrypt(struct ablkcipher_request *req) +static int mv_cesa_ecb_aes_encrypt(struct skcipher_request *req) { struct mv_cesa_op_ctx tmpl; @@ -738,7 +734,7 @@ static int mv_cesa_ecb_aes_encrypt(struct ablkcipher_request *req) return mv_cesa_aes_op(req, &tmpl); } -static int mv_cesa_ecb_aes_decrypt(struct ablkcipher_request *req) +static int mv_cesa_ecb_aes_decrypt(struct skcipher_request *req) { struct mv_cesa_op_ctx tmpl; @@ -749,40 +745,37 @@ static int mv_cesa_ecb_aes_decrypt(struct ablkcipher_request *req) return mv_cesa_aes_op(req, &tmpl); } -struct crypto_alg mv_cesa_ecb_aes_alg = { - .cra_name = "ecb(aes)", - .cra_driver_name = "mv-ecb-aes", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | - CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct mv_cesa_aes_ctx), - .cra_alignmask = 0, - .cra_type = &crypto_ablkcipher_type, - .cra_module = THIS_MODULE, - .cra_init = mv_cesa_ablkcipher_cra_init, - .cra_u = { - .ablkcipher = { - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .setkey = mv_cesa_aes_setkey, - .encrypt = mv_cesa_ecb_aes_encrypt, - .decrypt = mv_cesa_ecb_aes_decrypt, - }, +struct skcipher_alg mv_cesa_ecb_aes_alg = { + .setkey = mv_cesa_aes_setkey, + .encrypt = mv_cesa_ecb_aes_encrypt, + .decrypt = mv_cesa_ecb_aes_decrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .base = { + .cra_name = "ecb(aes)", + .cra_driver_name = "mv-ecb-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct mv_cesa_aes_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = mv_cesa_skcipher_cra_init, + .cra_exit = mv_cesa_skcipher_cra_exit, }, }; -static int mv_cesa_cbc_aes_op(struct ablkcipher_request *req, +static int mv_cesa_cbc_aes_op(struct skcipher_request *req, struct mv_cesa_op_ctx *tmpl) { mv_cesa_update_op_cfg(tmpl, CESA_SA_DESC_CFG_CRYPTCM_CBC, CESA_SA_DESC_CFG_CRYPTCM_MSK); - memcpy(tmpl->ctx.blkcipher.iv, req->info, AES_BLOCK_SIZE); + memcpy(tmpl->ctx.blkcipher.iv, req->iv, AES_BLOCK_SIZE); return mv_cesa_aes_op(req, tmpl); } -static int mv_cesa_cbc_aes_encrypt(struct ablkcipher_request *req) +static int mv_cesa_cbc_aes_encrypt(struct skcipher_request *req) { struct mv_cesa_op_ctx tmpl; @@ -791,7 +784,7 @@ static int mv_cesa_cbc_aes_encrypt(struct ablkcipher_request *req) return mv_cesa_cbc_aes_op(req, &tmpl); } -static int mv_cesa_cbc_aes_decrypt(struct ablkcipher_request *req) +static int mv_cesa_cbc_aes_decrypt(struct skcipher_request *req) { struct mv_cesa_op_ctx tmpl; @@ -800,26 +793,23 @@ static int mv_cesa_cbc_aes_decrypt(struct ablkcipher_request *req) return mv_cesa_cbc_aes_op(req, &tmpl); } -struct crypto_alg mv_cesa_cbc_aes_alg = { - .cra_name = "cbc(aes)", - .cra_driver_name = "mv-cbc-aes", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | - CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct mv_cesa_aes_ctx), - .cra_alignmask = 0, - .cra_type = &crypto_ablkcipher_type, - .cra_module = THIS_MODULE, - .cra_init = mv_cesa_ablkcipher_cra_init, - .cra_u = { - .ablkcipher = { - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - .setkey = mv_cesa_aes_setkey, - .encrypt = mv_cesa_cbc_aes_encrypt, - .decrypt = mv_cesa_cbc_aes_decrypt, - }, +struct skcipher_alg mv_cesa_cbc_aes_alg = { + .setkey = mv_cesa_aes_setkey, + .encrypt = mv_cesa_cbc_aes_encrypt, + .decrypt = mv_cesa_cbc_aes_decrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .base = { + .cra_name = "cbc(aes)", + .cra_driver_name = "mv-cbc-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct mv_cesa_aes_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = mv_cesa_skcipher_cra_init, + .cra_exit = mv_cesa_skcipher_cra_exit, }, }; diff --git a/drivers/crypto/marvell/tdma.c b/drivers/crypto/marvell/tdma.c index c76375ff376d..d0ef171c18df 100644 --- a/drivers/crypto/marvell/tdma.c +++ b/drivers/crypto/marvell/tdma.c @@ -304,10 +304,7 @@ int mv_cesa_dma_add_dummy_launch(struct mv_cesa_tdma_chain *chain, gfp_t flags) struct mv_cesa_tdma_desc *tdma; tdma = mv_cesa_dma_add_desc(chain, flags); - if (IS_ERR(tdma)) - return PTR_ERR(tdma); - - return 0; + return PTR_ERR_OR_ZERO(tdma); } int mv_cesa_dma_add_dummy_end(struct mv_cesa_tdma_chain *chain, gfp_t flags) diff --git a/drivers/crypto/mediatek/mtk-aes.c b/drivers/crypto/mediatek/mtk-aes.c index 9e845e866dec..c2058cf59f57 100644 --- a/drivers/crypto/mediatek/mtk-aes.c +++ b/drivers/crypto/mediatek/mtk-aes.c @@ -13,6 +13,7 @@ */ #include <crypto/aes.h> +#include <crypto/gcm.h> #include "mtk-platform.h" #define AES_QUEUE_SIZE 512 @@ -137,11 +138,6 @@ struct mtk_aes_gcm_ctx { struct crypto_skcipher *ctr; }; -struct mtk_aes_gcm_setkey_result { - int err; - struct completion completion; -}; - struct mtk_aes_drv { struct list_head dev_list; /* Device list lock */ @@ -928,25 +924,19 @@ static int mtk_aes_gcm_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) static int mtk_aes_gcm_crypt(struct aead_request *req, u64 mode) { struct mtk_aes_base_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + struct mtk_aes_gcm_ctx *gctx = mtk_aes_gcm_ctx_cast(ctx); struct mtk_aes_reqctx *rctx = aead_request_ctx(req); + /* Empty messages are not supported yet */ + if (!gctx->textlen && !req->assoclen) + return -EINVAL; + rctx->mode = AES_FLAGS_GCM | mode; return mtk_aes_handle_queue(ctx->cryp, !!(mode & AES_FLAGS_ENCRYPT), &req->base); } -static void mtk_gcm_setkey_done(struct crypto_async_request *req, int err) -{ - struct mtk_aes_gcm_setkey_result *result = req->data; - - if (err == -EINPROGRESS) - return; - - result->err = err; - complete(&result->completion); -} - /* * Because of the hardware limitation, we need to pre-calculate key(H) * for the GHASH operation. The result of the encryption operation @@ -962,7 +952,7 @@ static int mtk_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key, u32 hash[4]; u8 iv[8]; - struct mtk_aes_gcm_setkey_result result; + struct crypto_wait wait; struct scatterlist sg[1]; struct skcipher_request req; @@ -1002,22 +992,17 @@ static int mtk_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key, if (!data) return -ENOMEM; - init_completion(&data->result.completion); + crypto_init_wait(&data->wait); sg_init_one(data->sg, &data->hash, AES_BLOCK_SIZE); skcipher_request_set_tfm(&data->req, ctr); skcipher_request_set_callback(&data->req, CRYPTO_TFM_REQ_MAY_SLEEP | CRYPTO_TFM_REQ_MAY_BACKLOG, - mtk_gcm_setkey_done, &data->result); + crypto_req_done, &data->wait); skcipher_request_set_crypt(&data->req, data->sg, data->sg, AES_BLOCK_SIZE, data->iv); - err = crypto_skcipher_encrypt(&data->req); - if (err == -EINPROGRESS || err == -EBUSY) { - err = wait_for_completion_interruptible( - &data->result.completion); - if (!err) - err = data->result.err; - } + err = crypto_wait_req(crypto_skcipher_encrypt(&data->req), + &data->wait); if (err) goto out; @@ -1098,7 +1083,7 @@ static struct aead_alg aes_gcm_alg = { .decrypt = mtk_aes_gcm_decrypt, .init = mtk_aes_gcm_init, .exit = mtk_aes_gcm_exit, - .ivsize = 12, + .ivsize = GCM_AES_IV_SIZE, .maxauthsize = AES_BLOCK_SIZE, .base = { diff --git a/drivers/crypto/mediatek/mtk-platform.c b/drivers/crypto/mediatek/mtk-platform.c index 000b6500a22d..b182e941b0cd 100644 --- a/drivers/crypto/mediatek/mtk-platform.c +++ b/drivers/crypto/mediatek/mtk-platform.c @@ -500,7 +500,7 @@ static int mtk_crypto_probe(struct platform_device *pdev) cryp->irq[i] = platform_get_irq(pdev, i); if (cryp->irq[i] < 0) { dev_err(cryp->dev, "no IRQ:%d resource info\n", i); - return -ENXIO; + return cryp->irq[i]; } } diff --git a/drivers/crypto/mv_cesa.c b/drivers/crypto/mv_cesa.c deleted file mode 100644 index bf25f415eea6..000000000000 --- a/drivers/crypto/mv_cesa.c +++ /dev/null @@ -1,1216 +0,0 @@ -/* - * Support for Marvell's crypto engine which can be found on some Orion5X - * boards. - * - * Author: Sebastian Andrzej Siewior < sebastian at breakpoint dot cc > - * License: GPLv2 - * - */ -#include <crypto/aes.h> -#include <crypto/algapi.h> -#include <linux/crypto.h> -#include <linux/genalloc.h> -#include <linux/interrupt.h> -#include <linux/io.h> -#include <linux/kthread.h> -#include <linux/platform_device.h> -#include <linux/scatterlist.h> -#include <linux/slab.h> -#include <linux/module.h> -#include <linux/clk.h> -#include <crypto/hmac.h> -#include <crypto/internal/hash.h> -#include <crypto/sha.h> -#include <linux/of.h> -#include <linux/of_platform.h> -#include <linux/of_irq.h> - -#include "mv_cesa.h" - -#define MV_CESA "MV-CESA:" -#define MAX_HW_HASH_SIZE 0xFFFF -#define MV_CESA_EXPIRE 500 /* msec */ - -#define MV_CESA_DEFAULT_SRAM_SIZE 2048 - -/* - * STM: - * /---------------------------------------\ - * | | request complete - * \./ | - * IDLE -> new request -> BUSY -> done -> DEQUEUE - * /°\ | - * | | more scatter entries - * \________________/ - */ -enum engine_status { - ENGINE_IDLE, - ENGINE_BUSY, - ENGINE_W_DEQUEUE, -}; - -/** - * struct req_progress - used for every crypt request - * @src_sg_it: sg iterator for src - * @dst_sg_it: sg iterator for dst - * @sg_src_left: bytes left in src to process (scatter list) - * @src_start: offset to add to src start position (scatter list) - * @crypt_len: length of current hw crypt/hash process - * @hw_nbytes: total bytes to process in hw for this request - * @copy_back: whether to copy data back (crypt) or not (hash) - * @sg_dst_left: bytes left dst to process in this scatter list - * @dst_start: offset to add to dst start position (scatter list) - * @hw_processed_bytes: number of bytes processed by hw (request). - * - * sg helper are used to iterate over the scatterlist. Since the size of the - * SRAM may be less than the scatter size, this struct struct is used to keep - * track of progress within current scatterlist. - */ -struct req_progress { - struct sg_mapping_iter src_sg_it; - struct sg_mapping_iter dst_sg_it; - void (*complete) (void); - void (*process) (int is_first); - - /* src mostly */ - int sg_src_left; - int src_start; - int crypt_len; - int hw_nbytes; - /* dst mostly */ - int copy_back; - int sg_dst_left; - int dst_start; - int hw_processed_bytes; -}; - -struct crypto_priv { - void __iomem *reg; - void __iomem *sram; - struct gen_pool *sram_pool; - dma_addr_t sram_dma; - int irq; - struct clk *clk; - struct task_struct *queue_th; - - /* the lock protects queue and eng_st */ - spinlock_t lock; - struct crypto_queue queue; - enum engine_status eng_st; - struct timer_list completion_timer; - struct crypto_async_request *cur_req; - struct req_progress p; - int max_req_size; - int sram_size; - int has_sha1; - int has_hmac_sha1; -}; - -static struct crypto_priv *cpg; - -struct mv_ctx { - u8 aes_enc_key[AES_KEY_LEN]; - u32 aes_dec_key[8]; - int key_len; - u32 need_calc_aes_dkey; -}; - -enum crypto_op { - COP_AES_ECB, - COP_AES_CBC, -}; - -struct mv_req_ctx { - enum crypto_op op; - int decrypt; -}; - -enum hash_op { - COP_SHA1, - COP_HMAC_SHA1 -}; - -struct mv_tfm_hash_ctx { - struct crypto_shash *fallback; - struct crypto_shash *base_hash; - u32 ivs[2 * SHA1_DIGEST_SIZE / 4]; - int count_add; - enum hash_op op; -}; - -struct mv_req_hash_ctx { - u64 count; - u32 state[SHA1_DIGEST_SIZE / 4]; - u8 buffer[SHA1_BLOCK_SIZE]; - int first_hash; /* marks that we don't have previous state */ - int last_chunk; /* marks that this is the 'final' request */ - int extra_bytes; /* unprocessed bytes in buffer */ - enum hash_op op; - int count_add; -}; - -static void mv_completion_timer_callback(unsigned long unused) -{ - int active = readl(cpg->reg + SEC_ACCEL_CMD) & SEC_CMD_EN_SEC_ACCL0; - - printk(KERN_ERR MV_CESA - "completion timer expired (CESA %sactive), cleaning up.\n", - active ? "" : "in"); - - del_timer(&cpg->completion_timer); - writel(SEC_CMD_DISABLE_SEC, cpg->reg + SEC_ACCEL_CMD); - while(readl(cpg->reg + SEC_ACCEL_CMD) & SEC_CMD_DISABLE_SEC) - printk(KERN_INFO MV_CESA "%s: waiting for engine finishing\n", __func__); - cpg->eng_st = ENGINE_W_DEQUEUE; - wake_up_process(cpg->queue_th); -} - -static void mv_setup_timer(void) -{ - setup_timer(&cpg->completion_timer, &mv_completion_timer_callback, 0); - mod_timer(&cpg->completion_timer, - jiffies + msecs_to_jiffies(MV_CESA_EXPIRE)); -} - -static void compute_aes_dec_key(struct mv_ctx *ctx) -{ - struct crypto_aes_ctx gen_aes_key; - int key_pos; - - if (!ctx->need_calc_aes_dkey) - return; - - crypto_aes_expand_key(&gen_aes_key, ctx->aes_enc_key, ctx->key_len); - - key_pos = ctx->key_len + 24; - memcpy(ctx->aes_dec_key, &gen_aes_key.key_enc[key_pos], 4 * 4); - switch (ctx->key_len) { - case AES_KEYSIZE_256: - key_pos -= 2; - /* fall */ - case AES_KEYSIZE_192: - key_pos -= 2; - memcpy(&ctx->aes_dec_key[4], &gen_aes_key.key_enc[key_pos], - 4 * 4); - break; - } - ctx->need_calc_aes_dkey = 0; -} - -static int mv_setkey_aes(struct crypto_ablkcipher *cipher, const u8 *key, - unsigned int len) -{ - struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); - struct mv_ctx *ctx = crypto_tfm_ctx(tfm); - - switch (len) { - case AES_KEYSIZE_128: - case AES_KEYSIZE_192: - case AES_KEYSIZE_256: - break; - default: - crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); - return -EINVAL; - } - ctx->key_len = len; - ctx->need_calc_aes_dkey = 1; - - memcpy(ctx->aes_enc_key, key, AES_KEY_LEN); - return 0; -} - -static void copy_src_to_buf(struct req_progress *p, char *dbuf, int len) -{ - int ret; - void *sbuf; - int copy_len; - - while (len) { - if (!p->sg_src_left) { - ret = sg_miter_next(&p->src_sg_it); - BUG_ON(!ret); - p->sg_src_left = p->src_sg_it.length; - p->src_start = 0; - } - - sbuf = p->src_sg_it.addr + p->src_start; - - copy_len = min(p->sg_src_left, len); - memcpy(dbuf, sbuf, copy_len); - - p->src_start += copy_len; - p->sg_src_left -= copy_len; - - len -= copy_len; - dbuf += copy_len; - } -} - -static void setup_data_in(void) -{ - struct req_progress *p = &cpg->p; - int data_in_sram = - min(p->hw_nbytes - p->hw_processed_bytes, cpg->max_req_size); - copy_src_to_buf(p, cpg->sram + SRAM_DATA_IN_START + p->crypt_len, - data_in_sram - p->crypt_len); - p->crypt_len = data_in_sram; -} - -static void mv_process_current_q(int first_block) -{ - struct ablkcipher_request *req = ablkcipher_request_cast(cpg->cur_req); - struct mv_ctx *ctx = crypto_tfm_ctx(req->base.tfm); - struct mv_req_ctx *req_ctx = ablkcipher_request_ctx(req); - struct sec_accel_config op; - - switch (req_ctx->op) { - case COP_AES_ECB: - op.config = CFG_OP_CRYPT_ONLY | CFG_ENCM_AES | CFG_ENC_MODE_ECB; - break; - case COP_AES_CBC: - default: - op.config = CFG_OP_CRYPT_ONLY | CFG_ENCM_AES | CFG_ENC_MODE_CBC; - op.enc_iv = ENC_IV_POINT(SRAM_DATA_IV) | - ENC_IV_BUF_POINT(SRAM_DATA_IV_BUF); - if (first_block) - memcpy(cpg->sram + SRAM_DATA_IV, req->info, 16); - break; - } - if (req_ctx->decrypt) { - op.config |= CFG_DIR_DEC; - memcpy(cpg->sram + SRAM_DATA_KEY_P, ctx->aes_dec_key, - AES_KEY_LEN); - } else { - op.config |= CFG_DIR_ENC; - memcpy(cpg->sram + SRAM_DATA_KEY_P, ctx->aes_enc_key, - AES_KEY_LEN); - } - - switch (ctx->key_len) { - case AES_KEYSIZE_128: - op.config |= CFG_AES_LEN_128; - break; - case AES_KEYSIZE_192: - op.config |= CFG_AES_LEN_192; - break; - case AES_KEYSIZE_256: - op.config |= CFG_AES_LEN_256; - break; - } - op.enc_p = ENC_P_SRC(SRAM_DATA_IN_START) | - ENC_P_DST(SRAM_DATA_OUT_START); - op.enc_key_p = SRAM_DATA_KEY_P; - - setup_data_in(); - op.enc_len = cpg->p.crypt_len; - memcpy(cpg->sram + SRAM_CONFIG, &op, - sizeof(struct sec_accel_config)); - - /* GO */ - mv_setup_timer(); - writel(SEC_CMD_EN_SEC_ACCL0, cpg->reg + SEC_ACCEL_CMD); -} - -static void mv_crypto_algo_completion(void) -{ - struct ablkcipher_request *req = ablkcipher_request_cast(cpg->cur_req); - struct mv_req_ctx *req_ctx = ablkcipher_request_ctx(req); - - sg_miter_stop(&cpg->p.src_sg_it); - sg_miter_stop(&cpg->p.dst_sg_it); - - if (req_ctx->op != COP_AES_CBC) - return ; - - memcpy(req->info, cpg->sram + SRAM_DATA_IV_BUF, 16); -} - -static void mv_process_hash_current(int first_block) -{ - struct ahash_request *req = ahash_request_cast(cpg->cur_req); - const struct mv_tfm_hash_ctx *tfm_ctx = crypto_tfm_ctx(req->base.tfm); - struct mv_req_hash_ctx *req_ctx = ahash_request_ctx(req); - struct req_progress *p = &cpg->p; - struct sec_accel_config op = { 0 }; - int is_last; - - switch (req_ctx->op) { - case COP_SHA1: - default: - op.config = CFG_OP_MAC_ONLY | CFG_MACM_SHA1; - break; - case COP_HMAC_SHA1: - op.config = CFG_OP_MAC_ONLY | CFG_MACM_HMAC_SHA1; - memcpy(cpg->sram + SRAM_HMAC_IV_IN, - tfm_ctx->ivs, sizeof(tfm_ctx->ivs)); - break; - } - - op.mac_src_p = - MAC_SRC_DATA_P(SRAM_DATA_IN_START) | MAC_SRC_TOTAL_LEN((u32) - req_ctx-> - count); - - setup_data_in(); - - op.mac_digest = - MAC_DIGEST_P(SRAM_DIGEST_BUF) | MAC_FRAG_LEN(p->crypt_len); - op.mac_iv = - MAC_INNER_IV_P(SRAM_HMAC_IV_IN) | - MAC_OUTER_IV_P(SRAM_HMAC_IV_OUT); - - is_last = req_ctx->last_chunk - && (p->hw_processed_bytes + p->crypt_len >= p->hw_nbytes) - && (req_ctx->count <= MAX_HW_HASH_SIZE); - if (req_ctx->first_hash) { - if (is_last) - op.config |= CFG_NOT_FRAG; - else - op.config |= CFG_FIRST_FRAG; - - req_ctx->first_hash = 0; - } else { - if (is_last) - op.config |= CFG_LAST_FRAG; - else - op.config |= CFG_MID_FRAG; - - if (first_block) { - writel(req_ctx->state[0], cpg->reg + DIGEST_INITIAL_VAL_A); - writel(req_ctx->state[1], cpg->reg + DIGEST_INITIAL_VAL_B); - writel(req_ctx->state[2], cpg->reg + DIGEST_INITIAL_VAL_C); - writel(req_ctx->state[3], cpg->reg + DIGEST_INITIAL_VAL_D); - writel(req_ctx->state[4], cpg->reg + DIGEST_INITIAL_VAL_E); - } - } - - memcpy(cpg->sram + SRAM_CONFIG, &op, sizeof(struct sec_accel_config)); - - /* GO */ - mv_setup_timer(); - writel(SEC_CMD_EN_SEC_ACCL0, cpg->reg + SEC_ACCEL_CMD); -} - -static inline int mv_hash_import_sha1_ctx(const struct mv_req_hash_ctx *ctx, - struct shash_desc *desc) -{ - int i; - struct sha1_state shash_state; - - shash_state.count = ctx->count + ctx->count_add; - for (i = 0; i < 5; i++) - shash_state.state[i] = ctx->state[i]; - memcpy(shash_state.buffer, ctx->buffer, sizeof(shash_state.buffer)); - return crypto_shash_import(desc, &shash_state); -} - -static int mv_hash_final_fallback(struct ahash_request *req) -{ - const struct mv_tfm_hash_ctx *tfm_ctx = crypto_tfm_ctx(req->base.tfm); - struct mv_req_hash_ctx *req_ctx = ahash_request_ctx(req); - SHASH_DESC_ON_STACK(shash, tfm_ctx->fallback); - int rc; - - shash->tfm = tfm_ctx->fallback; - shash->flags = CRYPTO_TFM_REQ_MAY_SLEEP; - if (unlikely(req_ctx->first_hash)) { - crypto_shash_init(shash); - crypto_shash_update(shash, req_ctx->buffer, - req_ctx->extra_bytes); - } else { - /* only SHA1 for now.... - */ - rc = mv_hash_import_sha1_ctx(req_ctx, shash); - if (rc) - goto out; - } - rc = crypto_shash_final(shash, req->result); -out: - return rc; -} - -static void mv_save_digest_state(struct mv_req_hash_ctx *ctx) -{ - ctx->state[0] = readl(cpg->reg + DIGEST_INITIAL_VAL_A); - ctx->state[1] = readl(cpg->reg + DIGEST_INITIAL_VAL_B); - ctx->state[2] = readl(cpg->reg + DIGEST_INITIAL_VAL_C); - ctx->state[3] = readl(cpg->reg + DIGEST_INITIAL_VAL_D); - ctx->state[4] = readl(cpg->reg + DIGEST_INITIAL_VAL_E); -} - -static void mv_hash_algo_completion(void) -{ - struct ahash_request *req = ahash_request_cast(cpg->cur_req); - struct mv_req_hash_ctx *ctx = ahash_request_ctx(req); - - if (ctx->extra_bytes) - copy_src_to_buf(&cpg->p, ctx->buffer, ctx->extra_bytes); - sg_miter_stop(&cpg->p.src_sg_it); - - if (likely(ctx->last_chunk)) { - if (likely(ctx->count <= MAX_HW_HASH_SIZE)) { - memcpy(req->result, cpg->sram + SRAM_DIGEST_BUF, - crypto_ahash_digestsize(crypto_ahash_reqtfm - (req))); - } else { - mv_save_digest_state(ctx); - mv_hash_final_fallback(req); - } - } else { - mv_save_digest_state(ctx); - } -} - -static void dequeue_complete_req(void) -{ - struct crypto_async_request *req = cpg->cur_req; - void *buf; - int ret; - cpg->p.hw_processed_bytes += cpg->p.crypt_len; - if (cpg->p.copy_back) { - int need_copy_len = cpg->p.crypt_len; - int sram_offset = 0; - do { - int dst_copy; - - if (!cpg->p.sg_dst_left) { - ret = sg_miter_next(&cpg->p.dst_sg_it); - BUG_ON(!ret); - cpg->p.sg_dst_left = cpg->p.dst_sg_it.length; - cpg->p.dst_start = 0; - } - - buf = cpg->p.dst_sg_it.addr; - buf += cpg->p.dst_start; - - dst_copy = min(need_copy_len, cpg->p.sg_dst_left); - - memcpy(buf, - cpg->sram + SRAM_DATA_OUT_START + sram_offset, - dst_copy); - sram_offset += dst_copy; - cpg->p.sg_dst_left -= dst_copy; - need_copy_len -= dst_copy; - cpg->p.dst_start += dst_copy; - } while (need_copy_len > 0); - } - - cpg->p.crypt_len = 0; - - BUG_ON(cpg->eng_st != ENGINE_W_DEQUEUE); - if (cpg->p.hw_processed_bytes < cpg->p.hw_nbytes) { - /* process next scatter list entry */ - cpg->eng_st = ENGINE_BUSY; - cpg->p.process(0); - } else { - cpg->p.complete(); - cpg->eng_st = ENGINE_IDLE; - local_bh_disable(); - req->complete(req, 0); - local_bh_enable(); - } -} - -static int count_sgs(struct scatterlist *sl, unsigned int total_bytes) -{ - int i = 0; - size_t cur_len; - - while (sl) { - cur_len = sl[i].length; - ++i; - if (total_bytes > cur_len) - total_bytes -= cur_len; - else - break; - } - - return i; -} - -static void mv_start_new_crypt_req(struct ablkcipher_request *req) -{ - struct req_progress *p = &cpg->p; - int num_sgs; - - cpg->cur_req = &req->base; - memset(p, 0, sizeof(struct req_progress)); - p->hw_nbytes = req->nbytes; - p->complete = mv_crypto_algo_completion; - p->process = mv_process_current_q; - p->copy_back = 1; - - num_sgs = count_sgs(req->src, req->nbytes); - sg_miter_start(&p->src_sg_it, req->src, num_sgs, SG_MITER_FROM_SG); - - num_sgs = count_sgs(req->dst, req->nbytes); - sg_miter_start(&p->dst_sg_it, req->dst, num_sgs, SG_MITER_TO_SG); - - mv_process_current_q(1); -} - -static void mv_start_new_hash_req(struct ahash_request *req) -{ - struct req_progress *p = &cpg->p; - struct mv_req_hash_ctx *ctx = ahash_request_ctx(req); - int num_sgs, hw_bytes, old_extra_bytes, rc; - cpg->cur_req = &req->base; - memset(p, 0, sizeof(struct req_progress)); - hw_bytes = req->nbytes + ctx->extra_bytes; - old_extra_bytes = ctx->extra_bytes; - - ctx->extra_bytes = hw_bytes % SHA1_BLOCK_SIZE; - if (ctx->extra_bytes != 0 - && (!ctx->last_chunk || ctx->count > MAX_HW_HASH_SIZE)) - hw_bytes -= ctx->extra_bytes; - else - ctx->extra_bytes = 0; - - num_sgs = count_sgs(req->src, req->nbytes); - sg_miter_start(&p->src_sg_it, req->src, num_sgs, SG_MITER_FROM_SG); - - if (hw_bytes) { - p->hw_nbytes = hw_bytes; - p->complete = mv_hash_algo_completion; - p->process = mv_process_hash_current; - - if (unlikely(old_extra_bytes)) { - memcpy(cpg->sram + SRAM_DATA_IN_START, ctx->buffer, - old_extra_bytes); - p->crypt_len = old_extra_bytes; - } - - mv_process_hash_current(1); - } else { - copy_src_to_buf(p, ctx->buffer + old_extra_bytes, - ctx->extra_bytes - old_extra_bytes); - sg_miter_stop(&p->src_sg_it); - if (ctx->last_chunk) - rc = mv_hash_final_fallback(req); - else - rc = 0; - cpg->eng_st = ENGINE_IDLE; - local_bh_disable(); - req->base.complete(&req->base, rc); - local_bh_enable(); - } -} - -static int queue_manag(void *data) -{ - cpg->eng_st = ENGINE_IDLE; - do { - struct crypto_async_request *async_req = NULL; - struct crypto_async_request *backlog = NULL; - - __set_current_state(TASK_INTERRUPTIBLE); - - if (cpg->eng_st == ENGINE_W_DEQUEUE) - dequeue_complete_req(); - - spin_lock_irq(&cpg->lock); - if (cpg->eng_st == ENGINE_IDLE) { - backlog = crypto_get_backlog(&cpg->queue); - async_req = crypto_dequeue_request(&cpg->queue); - if (async_req) { - BUG_ON(cpg->eng_st != ENGINE_IDLE); - cpg->eng_st = ENGINE_BUSY; - } - } - spin_unlock_irq(&cpg->lock); - - if (backlog) { - backlog->complete(backlog, -EINPROGRESS); - backlog = NULL; - } - - if (async_req) { - if (crypto_tfm_alg_type(async_req->tfm) != - CRYPTO_ALG_TYPE_AHASH) { - struct ablkcipher_request *req = - ablkcipher_request_cast(async_req); - mv_start_new_crypt_req(req); - } else { - struct ahash_request *req = - ahash_request_cast(async_req); - mv_start_new_hash_req(req); - } - async_req = NULL; - } - - schedule(); - - } while (!kthread_should_stop()); - return 0; -} - -static int mv_handle_req(struct crypto_async_request *req) -{ - unsigned long flags; - int ret; - - spin_lock_irqsave(&cpg->lock, flags); - ret = crypto_enqueue_request(&cpg->queue, req); - spin_unlock_irqrestore(&cpg->lock, flags); - wake_up_process(cpg->queue_th); - return ret; -} - -static int mv_enc_aes_ecb(struct ablkcipher_request *req) -{ - struct mv_req_ctx *req_ctx = ablkcipher_request_ctx(req); - - req_ctx->op = COP_AES_ECB; - req_ctx->decrypt = 0; - - return mv_handle_req(&req->base); -} - -static int mv_dec_aes_ecb(struct ablkcipher_request *req) -{ - struct mv_ctx *ctx = crypto_tfm_ctx(req->base.tfm); - struct mv_req_ctx *req_ctx = ablkcipher_request_ctx(req); - - req_ctx->op = COP_AES_ECB; - req_ctx->decrypt = 1; - - compute_aes_dec_key(ctx); - return mv_handle_req(&req->base); -} - -static int mv_enc_aes_cbc(struct ablkcipher_request *req) -{ - struct mv_req_ctx *req_ctx = ablkcipher_request_ctx(req); - - req_ctx->op = COP_AES_CBC; - req_ctx->decrypt = 0; - - return mv_handle_req(&req->base); -} - -static int mv_dec_aes_cbc(struct ablkcipher_request *req) -{ - struct mv_ctx *ctx = crypto_tfm_ctx(req->base.tfm); - struct mv_req_ctx *req_ctx = ablkcipher_request_ctx(req); - - req_ctx->op = COP_AES_CBC; - req_ctx->decrypt = 1; - - compute_aes_dec_key(ctx); - return mv_handle_req(&req->base); -} - -static int mv_cra_init(struct crypto_tfm *tfm) -{ - tfm->crt_ablkcipher.reqsize = sizeof(struct mv_req_ctx); - return 0; -} - -static void mv_init_hash_req_ctx(struct mv_req_hash_ctx *ctx, int op, - int is_last, unsigned int req_len, - int count_add) -{ - memset(ctx, 0, sizeof(*ctx)); - ctx->op = op; - ctx->count = req_len; - ctx->first_hash = 1; - ctx->last_chunk = is_last; - ctx->count_add = count_add; -} - -static void mv_update_hash_req_ctx(struct mv_req_hash_ctx *ctx, int is_last, - unsigned req_len) -{ - ctx->last_chunk = is_last; - ctx->count += req_len; -} - -static int mv_hash_init(struct ahash_request *req) -{ - const struct mv_tfm_hash_ctx *tfm_ctx = crypto_tfm_ctx(req->base.tfm); - mv_init_hash_req_ctx(ahash_request_ctx(req), tfm_ctx->op, 0, 0, - tfm_ctx->count_add); - return 0; -} - -static int mv_hash_update(struct ahash_request *req) -{ - if (!req->nbytes) - return 0; - - mv_update_hash_req_ctx(ahash_request_ctx(req), 0, req->nbytes); - return mv_handle_req(&req->base); -} - -static int mv_hash_final(struct ahash_request *req) -{ - struct mv_req_hash_ctx *ctx = ahash_request_ctx(req); - - ahash_request_set_crypt(req, NULL, req->result, 0); - mv_update_hash_req_ctx(ctx, 1, 0); - return mv_handle_req(&req->base); -} - -static int mv_hash_finup(struct ahash_request *req) -{ - mv_update_hash_req_ctx(ahash_request_ctx(req), 1, req->nbytes); - return mv_handle_req(&req->base); -} - -static int mv_hash_digest(struct ahash_request *req) -{ - const struct mv_tfm_hash_ctx *tfm_ctx = crypto_tfm_ctx(req->base.tfm); - mv_init_hash_req_ctx(ahash_request_ctx(req), tfm_ctx->op, 1, - req->nbytes, tfm_ctx->count_add); - return mv_handle_req(&req->base); -} - -static void mv_hash_init_ivs(struct mv_tfm_hash_ctx *ctx, const void *istate, - const void *ostate) -{ - const struct sha1_state *isha1_state = istate, *osha1_state = ostate; - int i; - for (i = 0; i < 5; i++) { - ctx->ivs[i] = cpu_to_be32(isha1_state->state[i]); - ctx->ivs[i + 5] = cpu_to_be32(osha1_state->state[i]); - } -} - -static int mv_hash_setkey(struct crypto_ahash *tfm, const u8 * key, - unsigned int keylen) -{ - int rc; - struct mv_tfm_hash_ctx *ctx = crypto_tfm_ctx(&tfm->base); - int bs, ds, ss; - - if (!ctx->base_hash) - return 0; - - rc = crypto_shash_setkey(ctx->fallback, key, keylen); - if (rc) - return rc; - - /* Can't see a way to extract the ipad/opad from the fallback tfm - so I'm basically copying code from the hmac module */ - bs = crypto_shash_blocksize(ctx->base_hash); - ds = crypto_shash_digestsize(ctx->base_hash); - ss = crypto_shash_statesize(ctx->base_hash); - - { - SHASH_DESC_ON_STACK(shash, ctx->base_hash); - - unsigned int i; - char ipad[ss]; - char opad[ss]; - - shash->tfm = ctx->base_hash; - shash->flags = crypto_shash_get_flags(ctx->base_hash) & - CRYPTO_TFM_REQ_MAY_SLEEP; - - if (keylen > bs) { - int err; - - err = - crypto_shash_digest(shash, key, keylen, ipad); - if (err) - return err; - - keylen = ds; - } else - memcpy(ipad, key, keylen); - - memset(ipad + keylen, 0, bs - keylen); - memcpy(opad, ipad, bs); - - for (i = 0; i < bs; i++) { - ipad[i] ^= HMAC_IPAD_VALUE; - opad[i] ^= HMAC_OPAD_VALUE; - } - - rc = crypto_shash_init(shash) ? : - crypto_shash_update(shash, ipad, bs) ? : - crypto_shash_export(shash, ipad) ? : - crypto_shash_init(shash) ? : - crypto_shash_update(shash, opad, bs) ? : - crypto_shash_export(shash, opad); - - if (rc == 0) - mv_hash_init_ivs(ctx, ipad, opad); - - return rc; - } -} - -static int mv_cra_hash_init(struct crypto_tfm *tfm, const char *base_hash_name, - enum hash_op op, int count_add) -{ - const char *fallback_driver_name = crypto_tfm_alg_name(tfm); - struct mv_tfm_hash_ctx *ctx = crypto_tfm_ctx(tfm); - struct crypto_shash *fallback_tfm = NULL; - struct crypto_shash *base_hash = NULL; - int err = -ENOMEM; - - ctx->op = op; - ctx->count_add = count_add; - - /* Allocate a fallback and abort if it failed. */ - fallback_tfm = crypto_alloc_shash(fallback_driver_name, 0, - CRYPTO_ALG_NEED_FALLBACK); - if (IS_ERR(fallback_tfm)) { - printk(KERN_WARNING MV_CESA - "Fallback driver '%s' could not be loaded!\n", - fallback_driver_name); - err = PTR_ERR(fallback_tfm); - goto out; - } - ctx->fallback = fallback_tfm; - - if (base_hash_name) { - /* Allocate a hash to compute the ipad/opad of hmac. */ - base_hash = crypto_alloc_shash(base_hash_name, 0, - CRYPTO_ALG_NEED_FALLBACK); - if (IS_ERR(base_hash)) { - printk(KERN_WARNING MV_CESA - "Base driver '%s' could not be loaded!\n", - base_hash_name); - err = PTR_ERR(base_hash); - goto err_bad_base; - } - } - ctx->base_hash = base_hash; - - crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), - sizeof(struct mv_req_hash_ctx) + - crypto_shash_descsize(ctx->fallback)); - return 0; -err_bad_base: - crypto_free_shash(fallback_tfm); -out: - return err; -} - -static void mv_cra_hash_exit(struct crypto_tfm *tfm) -{ - struct mv_tfm_hash_ctx *ctx = crypto_tfm_ctx(tfm); - - crypto_free_shash(ctx->fallback); - if (ctx->base_hash) - crypto_free_shash(ctx->base_hash); -} - -static int mv_cra_hash_sha1_init(struct crypto_tfm *tfm) -{ - return mv_cra_hash_init(tfm, NULL, COP_SHA1, 0); -} - -static int mv_cra_hash_hmac_sha1_init(struct crypto_tfm *tfm) -{ - return mv_cra_hash_init(tfm, "sha1", COP_HMAC_SHA1, SHA1_BLOCK_SIZE); -} - -static irqreturn_t crypto_int(int irq, void *priv) -{ - u32 val; - - val = readl(cpg->reg + SEC_ACCEL_INT_STATUS); - if (!(val & SEC_INT_ACCEL0_DONE)) - return IRQ_NONE; - - if (!del_timer(&cpg->completion_timer)) { - printk(KERN_WARNING MV_CESA - "got an interrupt but no pending timer?\n"); - } - val &= ~SEC_INT_ACCEL0_DONE; - writel(val, cpg->reg + FPGA_INT_STATUS); - writel(val, cpg->reg + SEC_ACCEL_INT_STATUS); - BUG_ON(cpg->eng_st != ENGINE_BUSY); - cpg->eng_st = ENGINE_W_DEQUEUE; - wake_up_process(cpg->queue_th); - return IRQ_HANDLED; -} - -static struct crypto_alg mv_aes_alg_ecb = { - .cra_name = "ecb(aes)", - .cra_driver_name = "mv-ecb-aes", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | - CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, - .cra_blocksize = 16, - .cra_ctxsize = sizeof(struct mv_ctx), - .cra_alignmask = 0, - .cra_type = &crypto_ablkcipher_type, - .cra_module = THIS_MODULE, - .cra_init = mv_cra_init, - .cra_u = { - .ablkcipher = { - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .setkey = mv_setkey_aes, - .encrypt = mv_enc_aes_ecb, - .decrypt = mv_dec_aes_ecb, - }, - }, -}; - -static struct crypto_alg mv_aes_alg_cbc = { - .cra_name = "cbc(aes)", - .cra_driver_name = "mv-cbc-aes", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | - CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct mv_ctx), - .cra_alignmask = 0, - .cra_type = &crypto_ablkcipher_type, - .cra_module = THIS_MODULE, - .cra_init = mv_cra_init, - .cra_u = { - .ablkcipher = { - .ivsize = AES_BLOCK_SIZE, - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .setkey = mv_setkey_aes, - .encrypt = mv_enc_aes_cbc, - .decrypt = mv_dec_aes_cbc, - }, - }, -}; - -static struct ahash_alg mv_sha1_alg = { - .init = mv_hash_init, - .update = mv_hash_update, - .final = mv_hash_final, - .finup = mv_hash_finup, - .digest = mv_hash_digest, - .halg = { - .digestsize = SHA1_DIGEST_SIZE, - .base = { - .cra_name = "sha1", - .cra_driver_name = "mv-sha1", - .cra_priority = 300, - .cra_flags = - CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY | - CRYPTO_ALG_NEED_FALLBACK, - .cra_blocksize = SHA1_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct mv_tfm_hash_ctx), - .cra_init = mv_cra_hash_sha1_init, - .cra_exit = mv_cra_hash_exit, - .cra_module = THIS_MODULE, - } - } -}; - -static struct ahash_alg mv_hmac_sha1_alg = { - .init = mv_hash_init, - .update = mv_hash_update, - .final = mv_hash_final, - .finup = mv_hash_finup, - .digest = mv_hash_digest, - .setkey = mv_hash_setkey, - .halg = { - .digestsize = SHA1_DIGEST_SIZE, - .base = { - .cra_name = "hmac(sha1)", - .cra_driver_name = "mv-hmac-sha1", - .cra_priority = 300, - .cra_flags = - CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY | - CRYPTO_ALG_NEED_FALLBACK, - .cra_blocksize = SHA1_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct mv_tfm_hash_ctx), - .cra_init = mv_cra_hash_hmac_sha1_init, - .cra_exit = mv_cra_hash_exit, - .cra_module = THIS_MODULE, - } - } -}; - -static int mv_cesa_get_sram(struct platform_device *pdev, - struct crypto_priv *cp) -{ - struct resource *res; - u32 sram_size = MV_CESA_DEFAULT_SRAM_SIZE; - - of_property_read_u32(pdev->dev.of_node, "marvell,crypto-sram-size", - &sram_size); - - cp->sram_size = sram_size; - cp->sram_pool = of_gen_pool_get(pdev->dev.of_node, - "marvell,crypto-srams", 0); - if (cp->sram_pool) { - cp->sram = gen_pool_dma_alloc(cp->sram_pool, sram_size, - &cp->sram_dma); - if (cp->sram) - return 0; - - return -ENOMEM; - } - - res = platform_get_resource_byname(pdev, IORESOURCE_MEM, - "sram"); - if (!res || resource_size(res) < cp->sram_size) - return -EINVAL; - - cp->sram = devm_ioremap_resource(&pdev->dev, res); - if (IS_ERR(cp->sram)) - return PTR_ERR(cp->sram); - - return 0; -} - -static int mv_probe(struct platform_device *pdev) -{ - struct crypto_priv *cp; - struct resource *res; - int irq; - int ret; - - if (cpg) { - printk(KERN_ERR MV_CESA "Second crypto dev?\n"); - return -EEXIST; - } - - res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "regs"); - if (!res) - return -ENXIO; - - cp = devm_kzalloc(&pdev->dev, sizeof(*cp), GFP_KERNEL); - if (!cp) - return -ENOMEM; - - spin_lock_init(&cp->lock); - crypto_init_queue(&cp->queue, 50); - cp->reg = devm_ioremap_resource(&pdev->dev, res); - if (IS_ERR(cp->reg)) { - ret = PTR_ERR(cp->reg); - goto err; - } - - ret = mv_cesa_get_sram(pdev, cp); - if (ret) - goto err; - - cp->max_req_size = cp->sram_size - SRAM_CFG_SPACE; - - irq = platform_get_irq(pdev, 0); - if (irq < 0) { - ret = irq; - goto err; - } - cp->irq = irq; - - platform_set_drvdata(pdev, cp); - cpg = cp; - - cp->queue_th = kthread_run(queue_manag, cp, "mv_crypto"); - if (IS_ERR(cp->queue_th)) { - ret = PTR_ERR(cp->queue_th); - goto err; - } - - ret = request_irq(irq, crypto_int, 0, dev_name(&pdev->dev), - cp); - if (ret) - goto err_thread; - - /* Not all platforms can gate the clock, so it is not - an error if the clock does not exists. */ - cp->clk = clk_get(&pdev->dev, NULL); - if (!IS_ERR(cp->clk)) - clk_prepare_enable(cp->clk); - - writel(0, cpg->reg + SEC_ACCEL_INT_STATUS); - writel(SEC_INT_ACCEL0_DONE, cpg->reg + SEC_ACCEL_INT_MASK); - writel(SEC_CFG_STOP_DIG_ERR, cpg->reg + SEC_ACCEL_CFG); - writel(SRAM_CONFIG, cpg->reg + SEC_ACCEL_DESC_P0); - - ret = crypto_register_alg(&mv_aes_alg_ecb); - if (ret) { - printk(KERN_WARNING MV_CESA - "Could not register aes-ecb driver\n"); - goto err_irq; - } - - ret = crypto_register_alg(&mv_aes_alg_cbc); - if (ret) { - printk(KERN_WARNING MV_CESA - "Could not register aes-cbc driver\n"); - goto err_unreg_ecb; - } - - ret = crypto_register_ahash(&mv_sha1_alg); - if (ret == 0) - cpg->has_sha1 = 1; - else - printk(KERN_WARNING MV_CESA "Could not register sha1 driver\n"); - - ret = crypto_register_ahash(&mv_hmac_sha1_alg); - if (ret == 0) { - cpg->has_hmac_sha1 = 1; - } else { - printk(KERN_WARNING MV_CESA - "Could not register hmac-sha1 driver\n"); - } - - return 0; -err_unreg_ecb: - crypto_unregister_alg(&mv_aes_alg_ecb); -err_irq: - free_irq(irq, cp); - if (!IS_ERR(cp->clk)) { - clk_disable_unprepare(cp->clk); - clk_put(cp->clk); - } -err_thread: - kthread_stop(cp->queue_th); -err: - cpg = NULL; - return ret; -} - -static int mv_remove(struct platform_device *pdev) -{ - struct crypto_priv *cp = platform_get_drvdata(pdev); - - crypto_unregister_alg(&mv_aes_alg_ecb); - crypto_unregister_alg(&mv_aes_alg_cbc); - if (cp->has_sha1) - crypto_unregister_ahash(&mv_sha1_alg); - if (cp->has_hmac_sha1) - crypto_unregister_ahash(&mv_hmac_sha1_alg); - kthread_stop(cp->queue_th); - free_irq(cp->irq, cp); - memset(cp->sram, 0, cp->sram_size); - - if (!IS_ERR(cp->clk)) { - clk_disable_unprepare(cp->clk); - clk_put(cp->clk); - } - - cpg = NULL; - return 0; -} - -static const struct of_device_id mv_cesa_of_match_table[] = { - { .compatible = "marvell,orion-crypto", }, - { .compatible = "marvell,kirkwood-crypto", }, - { .compatible = "marvell,dove-crypto", }, - {} -}; -MODULE_DEVICE_TABLE(of, mv_cesa_of_match_table); - -static struct platform_driver marvell_crypto = { - .probe = mv_probe, - .remove = mv_remove, - .driver = { - .name = "mv_crypto", - .of_match_table = mv_cesa_of_match_table, - }, -}; -MODULE_ALIAS("platform:mv_crypto"); - -module_platform_driver(marvell_crypto); - -MODULE_AUTHOR("Sebastian Andrzej Siewior <sebastian@breakpoint.cc>"); -MODULE_DESCRIPTION("Support for Marvell's cryptographic engine"); -MODULE_LICENSE("GPL"); diff --git a/drivers/crypto/mv_cesa.h b/drivers/crypto/mv_cesa.h deleted file mode 100644 index 9249d3ed184b..000000000000 --- a/drivers/crypto/mv_cesa.h +++ /dev/null @@ -1,150 +0,0 @@ -#ifndef __MV_CRYPTO_H__ -#define __MV_CRYPTO_H__ - -#define DIGEST_INITIAL_VAL_A 0xdd00 -#define DIGEST_INITIAL_VAL_B 0xdd04 -#define DIGEST_INITIAL_VAL_C 0xdd08 -#define DIGEST_INITIAL_VAL_D 0xdd0c -#define DIGEST_INITIAL_VAL_E 0xdd10 -#define DES_CMD_REG 0xdd58 - -#define SEC_ACCEL_CMD 0xde00 -#define SEC_CMD_EN_SEC_ACCL0 (1 << 0) -#define SEC_CMD_EN_SEC_ACCL1 (1 << 1) -#define SEC_CMD_DISABLE_SEC (1 << 2) - -#define SEC_ACCEL_DESC_P0 0xde04 -#define SEC_DESC_P0_PTR(x) (x) - -#define SEC_ACCEL_DESC_P1 0xde14 -#define SEC_DESC_P1_PTR(x) (x) - -#define SEC_ACCEL_CFG 0xde08 -#define SEC_CFG_STOP_DIG_ERR (1 << 0) -#define SEC_CFG_CH0_W_IDMA (1 << 7) -#define SEC_CFG_CH1_W_IDMA (1 << 8) -#define SEC_CFG_ACT_CH0_IDMA (1 << 9) -#define SEC_CFG_ACT_CH1_IDMA (1 << 10) - -#define SEC_ACCEL_STATUS 0xde0c -#define SEC_ST_ACT_0 (1 << 0) -#define SEC_ST_ACT_1 (1 << 1) - -/* - * FPGA_INT_STATUS looks like a FPGA leftover and is documented only in Errata - * 4.12. It looks like that it was part of an IRQ-controller in FPGA and - * someone forgot to remove it while switching to the core and moving to - * SEC_ACCEL_INT_STATUS. - */ -#define FPGA_INT_STATUS 0xdd68 -#define SEC_ACCEL_INT_STATUS 0xde20 -#define SEC_INT_AUTH_DONE (1 << 0) -#define SEC_INT_DES_E_DONE (1 << 1) -#define SEC_INT_AES_E_DONE (1 << 2) -#define SEC_INT_AES_D_DONE (1 << 3) -#define SEC_INT_ENC_DONE (1 << 4) -#define SEC_INT_ACCEL0_DONE (1 << 5) -#define SEC_INT_ACCEL1_DONE (1 << 6) -#define SEC_INT_ACC0_IDMA_DONE (1 << 7) -#define SEC_INT_ACC1_IDMA_DONE (1 << 8) - -#define SEC_ACCEL_INT_MASK 0xde24 - -#define AES_KEY_LEN (8 * 4) - -struct sec_accel_config { - - u32 config; -#define CFG_OP_MAC_ONLY 0 -#define CFG_OP_CRYPT_ONLY 1 -#define CFG_OP_MAC_CRYPT 2 -#define CFG_OP_CRYPT_MAC 3 -#define CFG_MACM_MD5 (4 << 4) -#define CFG_MACM_SHA1 (5 << 4) -#define CFG_MACM_HMAC_MD5 (6 << 4) -#define CFG_MACM_HMAC_SHA1 (7 << 4) -#define CFG_ENCM_DES (1 << 8) -#define CFG_ENCM_3DES (2 << 8) -#define CFG_ENCM_AES (3 << 8) -#define CFG_DIR_ENC (0 << 12) -#define CFG_DIR_DEC (1 << 12) -#define CFG_ENC_MODE_ECB (0 << 16) -#define CFG_ENC_MODE_CBC (1 << 16) -#define CFG_3DES_EEE (0 << 20) -#define CFG_3DES_EDE (1 << 20) -#define CFG_AES_LEN_128 (0 << 24) -#define CFG_AES_LEN_192 (1 << 24) -#define CFG_AES_LEN_256 (2 << 24) -#define CFG_NOT_FRAG (0 << 30) -#define CFG_FIRST_FRAG (1 << 30) -#define CFG_LAST_FRAG (2 << 30) -#define CFG_MID_FRAG (3 << 30) - - u32 enc_p; -#define ENC_P_SRC(x) (x) -#define ENC_P_DST(x) ((x) << 16) - - u32 enc_len; -#define ENC_LEN(x) (x) - - u32 enc_key_p; -#define ENC_KEY_P(x) (x) - - u32 enc_iv; -#define ENC_IV_POINT(x) ((x) << 0) -#define ENC_IV_BUF_POINT(x) ((x) << 16) - - u32 mac_src_p; -#define MAC_SRC_DATA_P(x) (x) -#define MAC_SRC_TOTAL_LEN(x) ((x) << 16) - - u32 mac_digest; -#define MAC_DIGEST_P(x) (x) -#define MAC_FRAG_LEN(x) ((x) << 16) - u32 mac_iv; -#define MAC_INNER_IV_P(x) (x) -#define MAC_OUTER_IV_P(x) ((x) << 16) -}__attribute__ ((packed)); - /* - * /-----------\ 0 - * | ACCEL CFG | 4 * 8 - * |-----------| 0x20 - * | CRYPT KEY | 8 * 4 - * |-----------| 0x40 - * | IV IN | 4 * 4 - * |-----------| 0x40 (inplace) - * | IV BUF | 4 * 4 - * |-----------| 0x80 - * | DATA IN | 16 * x (max ->max_req_size) - * |-----------| 0x80 (inplace operation) - * | DATA OUT | 16 * x (max ->max_req_size) - * \-----------/ SRAM size - */ - - /* Hashing memory map: - * /-----------\ 0 - * | ACCEL CFG | 4 * 8 - * |-----------| 0x20 - * | Inner IV | 5 * 4 - * |-----------| 0x34 - * | Outer IV | 5 * 4 - * |-----------| 0x48 - * | Output BUF| 5 * 4 - * |-----------| 0x80 - * | DATA IN | 64 * x (max ->max_req_size) - * \-----------/ SRAM size - */ -#define SRAM_CONFIG 0x00 -#define SRAM_DATA_KEY_P 0x20 -#define SRAM_DATA_IV 0x40 -#define SRAM_DATA_IV_BUF 0x40 -#define SRAM_DATA_IN_START 0x80 -#define SRAM_DATA_OUT_START 0x80 - -#define SRAM_HMAC_IV_IN 0x20 -#define SRAM_HMAC_IV_OUT 0x34 -#define SRAM_DIGEST_BUF 0x48 - -#define SRAM_CFG_SPACE 0x80 - -#endif diff --git a/drivers/crypto/mxc-scc.c b/drivers/crypto/mxc-scc.c index ee4be1b0d30b..e01c46387df8 100644 --- a/drivers/crypto/mxc-scc.c +++ b/drivers/crypto/mxc-scc.c @@ -708,8 +708,8 @@ static int mxc_scc_probe(struct platform_device *pdev) for (i = 0; i < 2; i++) { irq = platform_get_irq(pdev, i); if (irq < 0) { - dev_err(dev, "failed to get irq resource\n"); - ret = -EINVAL; + dev_err(dev, "failed to get irq resource: %d\n", irq); + ret = irq; goto err_out; } diff --git a/drivers/crypto/mxs-dcp.c b/drivers/crypto/mxs-dcp.c index 625ee50fd78b..764be3e6933c 100644 --- a/drivers/crypto/mxs-dcp.c +++ b/drivers/crypto/mxs-dcp.c @@ -908,12 +908,16 @@ static int mxs_dcp_probe(struct platform_device *pdev) iores = platform_get_resource(pdev, IORESOURCE_MEM, 0); dcp_vmi_irq = platform_get_irq(pdev, 0); - if (dcp_vmi_irq < 0) + if (dcp_vmi_irq < 0) { + dev_err(dev, "Failed to get IRQ: (%d)!\n", dcp_vmi_irq); return dcp_vmi_irq; + } dcp_irq = platform_get_irq(pdev, 1); - if (dcp_irq < 0) + if (dcp_irq < 0) { + dev_err(dev, "Failed to get IRQ: (%d)!\n", dcp_irq); return dcp_irq; + } sdcp = devm_kzalloc(dev, sizeof(*sdcp), GFP_KERNEL); if (!sdcp) diff --git a/drivers/crypto/n2_asm.S b/drivers/crypto/n2_asm.S index f7c793745a1e..9a67dbf340f4 100644 --- a/drivers/crypto/n2_asm.S +++ b/drivers/crypto/n2_asm.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* n2_asm.S: Hypervisor calls for NCS support. * * Copyright (C) 2009 David S. Miller <davem@davemloft.net> diff --git a/drivers/crypto/n2_core.c b/drivers/crypto/n2_core.c index 269451375b63..48de52cf2ecc 100644 --- a/drivers/crypto/n2_core.c +++ b/drivers/crypto/n2_core.c @@ -1730,8 +1730,8 @@ static int spu_mdesc_walk_arcs(struct mdesc_handle *mdesc, continue; id = mdesc_get_property(mdesc, tgt, "id", NULL); if (table[*id] != NULL) { - dev_err(&dev->dev, "%s: SPU cpu slot already set.\n", - dev->dev.of_node->full_name); + dev_err(&dev->dev, "%pOF: SPU cpu slot already set.\n", + dev->dev.of_node); return -EINVAL; } cpumask_set_cpu(*id, &p->sharing); @@ -1751,8 +1751,8 @@ static int handle_exec_unit(struct spu_mdesc_info *ip, struct list_head *list, p = kzalloc(sizeof(struct spu_queue), GFP_KERNEL); if (!p) { - dev_err(&dev->dev, "%s: Could not allocate SPU queue.\n", - dev->dev.of_node->full_name); + dev_err(&dev->dev, "%pOF: Could not allocate SPU queue.\n", + dev->dev.of_node); return -ENOMEM; } @@ -1962,10 +1962,8 @@ static struct n2_crypto *alloc_n2cp(void) static void free_n2cp(struct n2_crypto *np) { - if (np->cwq_info.ino_table) { - kfree(np->cwq_info.ino_table); - np->cwq_info.ino_table = NULL; - } + kfree(np->cwq_info.ino_table); + np->cwq_info.ino_table = NULL; kfree(np); } @@ -1981,41 +1979,39 @@ static void n2_spu_driver_version(void) static int n2_crypto_probe(struct platform_device *dev) { struct mdesc_handle *mdesc; - const char *full_name; struct n2_crypto *np; int err; n2_spu_driver_version(); - full_name = dev->dev.of_node->full_name; - pr_info("Found N2CP at %s\n", full_name); + pr_info("Found N2CP at %pOF\n", dev->dev.of_node); np = alloc_n2cp(); if (!np) { - dev_err(&dev->dev, "%s: Unable to allocate n2cp.\n", - full_name); + dev_err(&dev->dev, "%pOF: Unable to allocate n2cp.\n", + dev->dev.of_node); return -ENOMEM; } err = grab_global_resources(); if (err) { - dev_err(&dev->dev, "%s: Unable to grab " - "global resources.\n", full_name); + dev_err(&dev->dev, "%pOF: Unable to grab global resources.\n", + dev->dev.of_node); goto out_free_n2cp; } mdesc = mdesc_grab(); if (!mdesc) { - dev_err(&dev->dev, "%s: Unable to grab MDESC.\n", - full_name); + dev_err(&dev->dev, "%pOF: Unable to grab MDESC.\n", + dev->dev.of_node); err = -ENODEV; goto out_free_global; } err = grab_mdesc_irq_props(mdesc, dev, &np->cwq_info, "n2cp"); if (err) { - dev_err(&dev->dev, "%s: Unable to grab IRQ props.\n", - full_name); + dev_err(&dev->dev, "%pOF: Unable to grab IRQ props.\n", + dev->dev.of_node); mdesc_release(mdesc); goto out_free_global; } @@ -2026,15 +2022,15 @@ static int n2_crypto_probe(struct platform_device *dev) mdesc_release(mdesc); if (err) { - dev_err(&dev->dev, "%s: CWQ MDESC scan failed.\n", - full_name); + dev_err(&dev->dev, "%pOF: CWQ MDESC scan failed.\n", + dev->dev.of_node); goto out_free_global; } err = n2_register_algs(); if (err) { - dev_err(&dev->dev, "%s: Unable to register algorithms.\n", - full_name); + dev_err(&dev->dev, "%pOF: Unable to register algorithms.\n", + dev->dev.of_node); goto out_free_spu_list; } @@ -2081,10 +2077,8 @@ static struct n2_mau *alloc_ncp(void) static void free_ncp(struct n2_mau *mp) { - if (mp->mau_info.ino_table) { - kfree(mp->mau_info.ino_table); - mp->mau_info.ino_table = NULL; - } + kfree(mp->mau_info.ino_table); + mp->mau_info.ino_table = NULL; kfree(mp); } @@ -2092,42 +2086,40 @@ static void free_ncp(struct n2_mau *mp) static int n2_mau_probe(struct platform_device *dev) { struct mdesc_handle *mdesc; - const char *full_name; struct n2_mau *mp; int err; n2_spu_driver_version(); - full_name = dev->dev.of_node->full_name; - pr_info("Found NCP at %s\n", full_name); + pr_info("Found NCP at %pOF\n", dev->dev.of_node); mp = alloc_ncp(); if (!mp) { - dev_err(&dev->dev, "%s: Unable to allocate ncp.\n", - full_name); + dev_err(&dev->dev, "%pOF: Unable to allocate ncp.\n", + dev->dev.of_node); return -ENOMEM; } err = grab_global_resources(); if (err) { - dev_err(&dev->dev, "%s: Unable to grab " - "global resources.\n", full_name); + dev_err(&dev->dev, "%pOF: Unable to grab global resources.\n", + dev->dev.of_node); goto out_free_ncp; } mdesc = mdesc_grab(); if (!mdesc) { - dev_err(&dev->dev, "%s: Unable to grab MDESC.\n", - full_name); + dev_err(&dev->dev, "%pOF: Unable to grab MDESC.\n", + dev->dev.of_node); err = -ENODEV; goto out_free_global; } err = grab_mdesc_irq_props(mdesc, dev, &mp->mau_info, "ncp"); if (err) { - dev_err(&dev->dev, "%s: Unable to grab IRQ props.\n", - full_name); + dev_err(&dev->dev, "%pOF: Unable to grab IRQ props.\n", + dev->dev.of_node); mdesc_release(mdesc); goto out_free_global; } @@ -2138,8 +2130,8 @@ static int n2_mau_probe(struct platform_device *dev) mdesc_release(mdesc); if (err) { - dev_err(&dev->dev, "%s: MAU MDESC scan failed.\n", - full_name); + dev_err(&dev->dev, "%pOF: MAU MDESC scan failed.\n", + dev->dev.of_node); goto out_free_global; } diff --git a/drivers/crypto/n2_core.h b/drivers/crypto/n2_core.h index 4bcbbeae98f5..2406763b0306 100644 --- a/drivers/crypto/n2_core.h +++ b/drivers/crypto/n2_core.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _N2_CORE_H #define _N2_CORE_H diff --git a/drivers/crypto/nx/Kconfig b/drivers/crypto/nx/Kconfig index ad7552a6998c..cd5dda9c48f4 100644 --- a/drivers/crypto/nx/Kconfig +++ b/drivers/crypto/nx/Kconfig @@ -38,6 +38,7 @@ config CRYPTO_DEV_NX_COMPRESS_PSERIES config CRYPTO_DEV_NX_COMPRESS_POWERNV tristate "Compression acceleration support on PowerNV platform" depends on PPC_POWERNV + depends on PPC_VAS default y help Support for PowerPC Nest (NX) compression acceleration. This diff --git a/drivers/crypto/nx/Makefile b/drivers/crypto/nx/Makefile index b727821c8ed4..015155da59c2 100644 --- a/drivers/crypto/nx/Makefile +++ b/drivers/crypto/nx/Makefile @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_CRYPTO_DEV_NX_ENCRYPT) += nx-crypto.o nx-crypto-objs := nx.o \ nx_debugfs.o \ diff --git a/drivers/crypto/nx/nx-842-powernv.c b/drivers/crypto/nx/nx-842-powernv.c index 1710f80a09ec..f2246a5abcf6 100644 --- a/drivers/crypto/nx/nx-842-powernv.c +++ b/drivers/crypto/nx/nx-842-powernv.c @@ -22,6 +22,8 @@ #include <asm/prom.h> #include <asm/icswx.h> +#include <asm/vas.h> +#include <asm/reg.h> MODULE_LICENSE("GPL"); MODULE_AUTHOR("Dan Streetman <ddstreet@ieee.org>"); @@ -31,6 +33,9 @@ MODULE_ALIAS_CRYPTO("842-nx"); #define WORKMEM_ALIGN (CRB_ALIGN) #define CSB_WAIT_MAX (5000) /* ms */ +#define VAS_RETRIES (10) +/* # of requests allowed per RxFIFO at a time. 0 for unlimited */ +#define MAX_CREDITS_PER_RXFIFO (1024) struct nx842_workmem { /* Below fields must be properly aligned */ @@ -47,13 +52,27 @@ struct nx842_workmem { struct nx842_coproc { unsigned int chip_id; unsigned int ct; - unsigned int ci; + unsigned int ci; /* Coprocessor instance, used with icswx */ + struct { + struct vas_window *rxwin; + int id; + } vas; struct list_head list; }; +/* + * Send the request to NX engine on the chip for the corresponding CPU + * where the process is executing. Use with VAS function. + */ +static DEFINE_PER_CPU(struct vas_window *, cpu_txwin); + /* no cpu hotplug on powernv, so this list never changes after init */ static LIST_HEAD(nx842_coprocs); -static unsigned int nx842_ct; +static unsigned int nx842_ct; /* used in icswx function */ + +static int (*nx842_powernv_exec)(const unsigned char *in, + unsigned int inlen, unsigned char *out, + unsigned int *outlenp, void *workmem, int fc); /** * setup_indirect_dde - Setup an indirect DDE @@ -173,7 +192,7 @@ static int wait_for_csb(struct nx842_workmem *wmem, ktime_t start = wmem->start, now = ktime_get(); ktime_t timeout = ktime_add_ms(start, CSB_WAIT_MAX); - while (!(ACCESS_ONCE(csb->flags) & CSB_V)) { + while (!(READ_ONCE(csb->flags) & CSB_V)) { cpu_relax(); now = ktime_get(); if (ktime_after(now, timeout)) @@ -238,6 +257,13 @@ static int wait_for_csb(struct nx842_workmem *wmem, case CSB_CC_TEMPL_OVERFLOW: CSB_ERR(csb, "Compressed data template shows data past end"); return -EINVAL; + case CSB_CC_EXCEED_BYTE_COUNT: /* P9 or later */ + /* + * DDE byte count exceeds the limit specified in Maximum + * byte count register. + */ + CSB_ERR(csb, "DDE byte count exceeds the limit"); + return -EINVAL; /* these should not happen */ case CSB_CC_INVALID_ALIGN: @@ -279,9 +305,17 @@ static int wait_for_csb(struct nx842_workmem *wmem, CSB_ERR(csb, "Too many DDEs in DDL"); return -EINVAL; case CSB_CC_TRANSPORT: + case CSB_CC_INVALID_CRB: /* P9 or later */ /* shouldn't happen, we setup CRB correctly */ CSB_ERR(csb, "Invalid CRB"); return -EINVAL; + case CSB_CC_INVALID_DDE: /* P9 or later */ + /* + * shouldn't happen, setup_direct/indirect_dde creates + * DDE right + */ + CSB_ERR(csb, "Invalid DDE"); + return -EINVAL; case CSB_CC_SEGMENTED_DDL: /* shouldn't happen, setup_ddl creates DDL right */ CSB_ERR(csb, "Segmented DDL error"); @@ -325,6 +359,9 @@ static int wait_for_csb(struct nx842_workmem *wmem, case CSB_CC_HW: CSB_ERR(csb, "Correctable hardware error"); return -EPROTO; + case CSB_CC_HW_EXPIRED_TIMER: /* P9 or later */ + CSB_ERR(csb, "Job did not finish within allowed time"); + return -EPROTO; default: CSB_ERR(csb, "Invalid CC %d", csb->cc); @@ -353,8 +390,42 @@ static int wait_for_csb(struct nx842_workmem *wmem, return 0; } +static int nx842_config_crb(const unsigned char *in, unsigned int inlen, + unsigned char *out, unsigned int outlen, + struct nx842_workmem *wmem) +{ + struct coprocessor_request_block *crb; + struct coprocessor_status_block *csb; + u64 csb_addr; + int ret; + + crb = &wmem->crb; + csb = &crb->csb; + + /* Clear any previous values */ + memset(crb, 0, sizeof(*crb)); + + /* set up DDLs */ + ret = setup_ddl(&crb->source, wmem->ddl_in, + (unsigned char *)in, inlen, true); + if (ret) + return ret; + + ret = setup_ddl(&crb->target, wmem->ddl_out, + out, outlen, false); + if (ret) + return ret; + + /* set up CRB's CSB addr */ + csb_addr = nx842_get_pa(csb) & CRB_CSB_ADDRESS; + csb_addr |= CRB_CSB_AT; /* Addrs are phys */ + crb->csb_addr = cpu_to_be64(csb_addr); + + return 0; +} + /** - * nx842_powernv_function - compress/decompress data using the 842 algorithm + * nx842_exec_icswx - compress/decompress data using the 842 algorithm * * (De)compression provided by the NX842 coprocessor on IBM PowerNV systems. * This compresses or decompresses the provided input buffer into the provided @@ -384,7 +455,7 @@ static int wait_for_csb(struct nx842_workmem *wmem, * -ETIMEDOUT hardware did not complete operation in reasonable time * -EINTR operation was aborted */ -static int nx842_powernv_function(const unsigned char *in, unsigned int inlen, +static int nx842_exec_icswx(const unsigned char *in, unsigned int inlen, unsigned char *out, unsigned int *outlenp, void *workmem, int fc) { @@ -392,7 +463,6 @@ static int nx842_powernv_function(const unsigned char *in, unsigned int inlen, struct coprocessor_status_block *csb; struct nx842_workmem *wmem; int ret; - u64 csb_addr; u32 ccw; unsigned int outlen = *outlenp; @@ -406,32 +476,18 @@ static int nx842_powernv_function(const unsigned char *in, unsigned int inlen, return -ENODEV; } - crb = &wmem->crb; - csb = &crb->csb; - - /* Clear any previous values */ - memset(crb, 0, sizeof(*crb)); - - /* set up DDLs */ - ret = setup_ddl(&crb->source, wmem->ddl_in, - (unsigned char *)in, inlen, true); - if (ret) - return ret; - ret = setup_ddl(&crb->target, wmem->ddl_out, - out, outlen, false); + ret = nx842_config_crb(in, inlen, out, outlen, wmem); if (ret) return ret; + crb = &wmem->crb; + csb = &crb->csb; + /* set up CCW */ ccw = 0; - ccw = SET_FIELD(ccw, CCW_CT, nx842_ct); - ccw = SET_FIELD(ccw, CCW_CI_842, 0); /* use 0 for hw auto-selection */ - ccw = SET_FIELD(ccw, CCW_FC_842, fc); - - /* set up CRB's CSB addr */ - csb_addr = nx842_get_pa(csb) & CRB_CSB_ADDRESS; - csb_addr |= CRB_CSB_AT; /* Addrs are phys */ - crb->csb_addr = cpu_to_be64(csb_addr); + ccw = SET_FIELD(CCW_CT, ccw, nx842_ct); + ccw = SET_FIELD(CCW_CI_842, ccw, 0); /* use 0 for hw auto-selection */ + ccw = SET_FIELD(CCW_FC_842, ccw, fc); wmem->start = ktime_get(); @@ -471,6 +527,99 @@ static int nx842_powernv_function(const unsigned char *in, unsigned int inlen, } /** + * nx842_exec_vas - compress/decompress data using the 842 algorithm + * + * (De)compression provided by the NX842 coprocessor on IBM PowerNV systems. + * This compresses or decompresses the provided input buffer into the provided + * output buffer. + * + * Upon return from this function @outlen contains the length of the + * output data. If there is an error then @outlen will be 0 and an + * error will be specified by the return code from this function. + * + * The @workmem buffer should only be used by one function call at a time. + * + * @in: input buffer pointer + * @inlen: input buffer size + * @out: output buffer pointer + * @outlenp: output buffer size pointer + * @workmem: working memory buffer pointer, size determined by + * nx842_powernv_driver.workmem_size + * @fc: function code, see CCW Function Codes in nx-842.h + * + * Returns: + * 0 Success, output of length @outlenp stored in the buffer + * at @out + * -ENODEV Hardware unavailable + * -ENOSPC Output buffer is to small + * -EMSGSIZE Input buffer too large + * -EINVAL buffer constraints do not fix nx842_constraints + * -EPROTO hardware error during operation + * -ETIMEDOUT hardware did not complete operation in reasonable time + * -EINTR operation was aborted + */ +static int nx842_exec_vas(const unsigned char *in, unsigned int inlen, + unsigned char *out, unsigned int *outlenp, + void *workmem, int fc) +{ + struct coprocessor_request_block *crb; + struct coprocessor_status_block *csb; + struct nx842_workmem *wmem; + struct vas_window *txwin; + int ret, i = 0; + u32 ccw; + unsigned int outlen = *outlenp; + + wmem = PTR_ALIGN(workmem, WORKMEM_ALIGN); + + *outlenp = 0; + + crb = &wmem->crb; + csb = &crb->csb; + + ret = nx842_config_crb(in, inlen, out, outlen, wmem); + if (ret) + return ret; + + ccw = 0; + ccw = SET_FIELD(CCW_FC_842, ccw, fc); + crb->ccw = cpu_to_be32(ccw); + + do { + wmem->start = ktime_get(); + preempt_disable(); + txwin = this_cpu_read(cpu_txwin); + + /* + * VAS copy CRB into L2 cache. Refer <asm/vas.h>. + * @crb and @offset. + */ + vas_copy_crb(crb, 0); + + /* + * VAS paste previously copied CRB to NX. + * @txwin, @offset and @last (must be true). + */ + ret = vas_paste_crb(txwin, 0, 1); + preempt_enable(); + /* + * Retry copy/paste function for VAS failures. + */ + } while (ret && (i++ < VAS_RETRIES)); + + if (ret) { + pr_err_ratelimited("VAS copy/paste failed\n"); + return ret; + } + + ret = wait_for_csb(wmem, csb); + if (!ret) + *outlenp = be32_to_cpu(csb->count); + + return ret; +} + +/** * nx842_powernv_compress - Compress data using the 842 algorithm * * Compression provided by the NX842 coprocessor on IBM PowerNV systems. @@ -488,13 +637,13 @@ static int nx842_powernv_function(const unsigned char *in, unsigned int inlen, * @workmem: working memory buffer pointer, size determined by * nx842_powernv_driver.workmem_size * - * Returns: see @nx842_powernv_function() + * Returns: see @nx842_powernv_exec() */ static int nx842_powernv_compress(const unsigned char *in, unsigned int inlen, unsigned char *out, unsigned int *outlenp, void *wmem) { - return nx842_powernv_function(in, inlen, out, outlenp, + return nx842_powernv_exec(in, inlen, out, outlenp, wmem, CCW_FC_842_COMP_CRC); } @@ -516,16 +665,223 @@ static int nx842_powernv_compress(const unsigned char *in, unsigned int inlen, * @workmem: working memory buffer pointer, size determined by * nx842_powernv_driver.workmem_size * - * Returns: see @nx842_powernv_function() + * Returns: see @nx842_powernv_exec() */ static int nx842_powernv_decompress(const unsigned char *in, unsigned int inlen, unsigned char *out, unsigned int *outlenp, void *wmem) { - return nx842_powernv_function(in, inlen, out, outlenp, + return nx842_powernv_exec(in, inlen, out, outlenp, wmem, CCW_FC_842_DECOMP_CRC); } +static inline void nx842_add_coprocs_list(struct nx842_coproc *coproc, + int chipid) +{ + coproc->chip_id = chipid; + INIT_LIST_HEAD(&coproc->list); + list_add(&coproc->list, &nx842_coprocs); +} + +static struct vas_window *nx842_alloc_txwin(struct nx842_coproc *coproc) +{ + struct vas_window *txwin = NULL; + struct vas_tx_win_attr txattr; + + /* + * Kernel requests will be high priority. So open send + * windows only for high priority RxFIFO entries. + */ + vas_init_tx_win_attr(&txattr, coproc->ct); + txattr.lpid = 0; /* lpid is 0 for kernel requests */ + txattr.pid = 0; /* pid is 0 for kernel requests */ + + /* + * Open a VAS send window which is used to send request to NX. + */ + txwin = vas_tx_win_open(coproc->vas.id, coproc->ct, &txattr); + if (IS_ERR(txwin)) + pr_err("ibm,nx-842: Can not open TX window: %ld\n", + PTR_ERR(txwin)); + + return txwin; +} + +/* + * Identify chip ID for each CPU, open send wndow for the corresponding NX + * engine and save txwin in percpu cpu_txwin. + * cpu_txwin is used in copy/paste operation for each compression / + * decompression request. + */ +static int nx842_open_percpu_txwins(void) +{ + struct nx842_coproc *coproc, *n; + unsigned int i, chip_id; + + for_each_possible_cpu(i) { + struct vas_window *txwin = NULL; + + chip_id = cpu_to_chip_id(i); + + list_for_each_entry_safe(coproc, n, &nx842_coprocs, list) { + /* + * Kernel requests use only high priority FIFOs. So + * open send windows for these FIFOs. + */ + + if (coproc->ct != VAS_COP_TYPE_842_HIPRI) + continue; + + if (coproc->chip_id == chip_id) { + txwin = nx842_alloc_txwin(coproc); + if (IS_ERR(txwin)) + return PTR_ERR(txwin); + + per_cpu(cpu_txwin, i) = txwin; + break; + } + } + + if (!per_cpu(cpu_txwin, i)) { + /* shoudn't happen, Each chip will have NX engine */ + pr_err("NX engine is not availavle for CPU %d\n", i); + return -EINVAL; + } + } + + return 0; +} + +static int __init vas_cfg_coproc_info(struct device_node *dn, int chip_id, + int vasid) +{ + struct vas_window *rxwin = NULL; + struct vas_rx_win_attr rxattr; + struct nx842_coproc *coproc; + u32 lpid, pid, tid, fifo_size; + u64 rx_fifo; + const char *priority; + int ret; + + ret = of_property_read_u64(dn, "rx-fifo-address", &rx_fifo); + if (ret) { + pr_err("Missing rx-fifo-address property\n"); + return ret; + } + + ret = of_property_read_u32(dn, "rx-fifo-size", &fifo_size); + if (ret) { + pr_err("Missing rx-fifo-size property\n"); + return ret; + } + + ret = of_property_read_u32(dn, "lpid", &lpid); + if (ret) { + pr_err("Missing lpid property\n"); + return ret; + } + + ret = of_property_read_u32(dn, "pid", &pid); + if (ret) { + pr_err("Missing pid property\n"); + return ret; + } + + ret = of_property_read_u32(dn, "tid", &tid); + if (ret) { + pr_err("Missing tid property\n"); + return ret; + } + + ret = of_property_read_string(dn, "priority", &priority); + if (ret) { + pr_err("Missing priority property\n"); + return ret; + } + + coproc = kzalloc(sizeof(*coproc), GFP_KERNEL); + if (!coproc) + return -ENOMEM; + + if (!strcmp(priority, "High")) + coproc->ct = VAS_COP_TYPE_842_HIPRI; + else if (!strcmp(priority, "Normal")) + coproc->ct = VAS_COP_TYPE_842; + else { + pr_err("Invalid RxFIFO priority value\n"); + ret = -EINVAL; + goto err_out; + } + + vas_init_rx_win_attr(&rxattr, coproc->ct); + rxattr.rx_fifo = (void *)rx_fifo; + rxattr.rx_fifo_size = fifo_size; + rxattr.lnotify_lpid = lpid; + rxattr.lnotify_pid = pid; + rxattr.lnotify_tid = tid; + rxattr.wcreds_max = MAX_CREDITS_PER_RXFIFO; + + /* + * Open a VAS receice window which is used to configure RxFIFO + * for NX. + */ + rxwin = vas_rx_win_open(vasid, coproc->ct, &rxattr); + if (IS_ERR(rxwin)) { + ret = PTR_ERR(rxwin); + pr_err("setting RxFIFO with VAS failed: %d\n", + ret); + goto err_out; + } + + coproc->vas.rxwin = rxwin; + coproc->vas.id = vasid; + nx842_add_coprocs_list(coproc, chip_id); + + return 0; + +err_out: + kfree(coproc); + return ret; +} + + +static int __init nx842_powernv_probe_vas(struct device_node *pn) +{ + struct device_node *dn; + int chip_id, vasid, ret = 0; + int nx_fifo_found = 0; + + chip_id = of_get_ibm_chip_id(pn); + if (chip_id < 0) { + pr_err("ibm,chip-id missing\n"); + return -EINVAL; + } + + vasid = chip_to_vas_id(chip_id); + if (vasid < 0) { + pr_err("Unable to map chip_id %d to vasid\n", chip_id); + return -EINVAL; + } + + for_each_child_of_node(pn, dn) { + if (of_device_is_compatible(dn, "ibm,p9-nx-842")) { + ret = vas_cfg_coproc_info(dn, chip_id, vasid); + if (ret) { + of_node_put(dn); + return ret; + } + nx_fifo_found++; + } + } + + if (!nx_fifo_found) { + pr_err("NX842 FIFO nodes are missing\n"); + ret = -EINVAL; + } + + return ret; +} + static int __init nx842_powernv_probe(struct device_node *dn) { struct nx842_coproc *coproc; @@ -552,11 +908,9 @@ static int __init nx842_powernv_probe(struct device_node *dn) if (!coproc) return -ENOMEM; - coproc->chip_id = chip_id; coproc->ct = ct; coproc->ci = ci; - INIT_LIST_HEAD(&coproc->list); - list_add(&coproc->list, &nx842_coprocs); + nx842_add_coprocs_list(coproc, chip_id); pr_info("coprocessor found on chip %d, CT %d CI %d\n", chip_id, ct, ci); @@ -569,6 +923,32 @@ static int __init nx842_powernv_probe(struct device_node *dn) return 0; } +static void nx842_delete_coprocs(void) +{ + struct nx842_coproc *coproc, *n; + struct vas_window *txwin; + int i; + + /* + * close percpu txwins that are opened for the corresponding coproc. + */ + for_each_possible_cpu(i) { + txwin = per_cpu(cpu_txwin, i); + if (txwin) + vas_win_close(txwin); + + per_cpu(cpu_txwin, i) = 0; + } + + list_for_each_entry_safe(coproc, n, &nx842_coprocs, list) { + if (coproc->vas.rxwin) + vas_win_close(coproc->vas.rxwin); + + list_del(&coproc->list); + kfree(coproc); + } +} + static struct nx842_constraints nx842_powernv_constraints = { .alignment = DDE_BUFFER_ALIGN, .multiple = DDE_BUFFER_LAST_MULT, @@ -618,21 +998,35 @@ static __init int nx842_powernv_init(void) BUILD_BUG_ON(DDE_BUFFER_ALIGN % DDE_BUFFER_SIZE_MULT); BUILD_BUG_ON(DDE_BUFFER_SIZE_MULT % DDE_BUFFER_LAST_MULT); - for_each_compatible_node(dn, NULL, "ibm,power-nx") - nx842_powernv_probe(dn); + for_each_compatible_node(dn, NULL, "ibm,power9-nx") { + ret = nx842_powernv_probe_vas(dn); + if (ret) { + nx842_delete_coprocs(); + return ret; + } + } - if (!nx842_ct) - return -ENODEV; + if (list_empty(&nx842_coprocs)) { + for_each_compatible_node(dn, NULL, "ibm,power-nx") + nx842_powernv_probe(dn); - ret = crypto_register_alg(&nx842_powernv_alg); - if (ret) { - struct nx842_coproc *coproc, *n; + if (!nx842_ct) + return -ENODEV; - list_for_each_entry_safe(coproc, n, &nx842_coprocs, list) { - list_del(&coproc->list); - kfree(coproc); + nx842_powernv_exec = nx842_exec_icswx; + } else { + ret = nx842_open_percpu_txwins(); + if (ret) { + nx842_delete_coprocs(); + return ret; } + nx842_powernv_exec = nx842_exec_vas; + } + + ret = crypto_register_alg(&nx842_powernv_alg); + if (ret) { + nx842_delete_coprocs(); return ret; } @@ -642,13 +1036,8 @@ module_init(nx842_powernv_init); static void __exit nx842_powernv_exit(void) { - struct nx842_coproc *coproc, *n; - crypto_unregister_alg(&nx842_powernv_alg); - list_for_each_entry_safe(coproc, n, &nx842_coprocs, list) { - list_del(&coproc->list); - kfree(coproc); - } + nx842_delete_coprocs(); } module_exit(nx842_powernv_exit); diff --git a/drivers/crypto/nx/nx-842-pseries.c b/drivers/crypto/nx/nx-842-pseries.c index cddc6d8b55d9..bf52cd1d7fca 100644 --- a/drivers/crypto/nx/nx-842-pseries.c +++ b/drivers/crypto/nx/nx-842-pseries.c @@ -1082,7 +1082,7 @@ static int nx842_remove(struct vio_dev *viodev) return 0; } -static struct vio_device_id nx842_vio_driver_ids[] = { +static const struct vio_device_id nx842_vio_driver_ids[] = { {"ibm,compression-v1", "ibm,compression"}, {"", ""}, }; diff --git a/drivers/crypto/nx/nx-842.h b/drivers/crypto/nx/nx-842.h index a4eee3bba937..b66f19ac600f 100644 --- a/drivers/crypto/nx/nx-842.h +++ b/drivers/crypto/nx/nx-842.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __NX_842_H__ #define __NX_842_H__ @@ -76,9 +77,17 @@ #define CSB_CC_DECRYPT_OVERFLOW (64) /* asym crypt codes */ #define CSB_CC_MINV_OVERFLOW (128) +/* + * HW error - Job did not finish in the maximum time allowed. + * Job terminated. + */ +#define CSB_CC_HW_EXPIRED_TIMER (224) /* These are reserved for hypervisor use */ #define CSB_CC_HYP_RESERVE_START (240) #define CSB_CC_HYP_RESERVE_END (253) +#define CSB_CC_HYP_RESERVE_P9_END (251) +/* No valid interrupt server (P9 or later). */ +#define CSB_CC_HYP_RESERVE_NO_INTR_SERVER (252) #define CSB_CC_HYP_NO_HW (254) #define CSB_CC_HYP_HANG_ABORTED (255) @@ -100,11 +109,6 @@ static inline unsigned long nx842_get_pa(void *addr) return page_to_phys(vmalloc_to_page(addr)) + offset_in_page(addr); } -/* Get/Set bit fields */ -#define MASK_LSH(m) (__builtin_ffsl(m) - 1) -#define GET_FIELD(v, m) (((v) & (m)) >> MASK_LSH(m)) -#define SET_FIELD(v, m, val) (((v) & ~(m)) | (((val) << MASK_LSH(m)) & (m))) - /** * This provides the driver's constraints. Different nx842 implementations * may have varying requirements. The constraints are: diff --git a/drivers/crypto/nx/nx-aes-gcm.c b/drivers/crypto/nx/nx-aes-gcm.c index abd465f479c4..a810596b97c2 100644 --- a/drivers/crypto/nx/nx-aes-gcm.c +++ b/drivers/crypto/nx/nx-aes-gcm.c @@ -22,6 +22,7 @@ #include <crypto/internal/aead.h> #include <crypto/aes.h> #include <crypto/algapi.h> +#include <crypto/gcm.h> #include <crypto/scatterwalk.h> #include <linux/module.h> #include <linux/types.h> @@ -433,7 +434,7 @@ static int gcm_aes_nx_encrypt(struct aead_request *req) struct nx_gcm_rctx *rctx = aead_request_ctx(req); char *iv = rctx->iv; - memcpy(iv, req->iv, 12); + memcpy(iv, req->iv, GCM_AES_IV_SIZE); return gcm_aes_nx_crypt(req, 1, req->assoclen); } @@ -443,7 +444,7 @@ static int gcm_aes_nx_decrypt(struct aead_request *req) struct nx_gcm_rctx *rctx = aead_request_ctx(req); char *iv = rctx->iv; - memcpy(iv, req->iv, 12); + memcpy(iv, req->iv, GCM_AES_IV_SIZE); return gcm_aes_nx_crypt(req, 0, req->assoclen); } @@ -498,7 +499,7 @@ struct aead_alg nx_gcm_aes_alg = { }, .init = nx_crypto_ctx_aes_gcm_init, .exit = nx_crypto_ctx_aead_exit, - .ivsize = 12, + .ivsize = GCM_AES_IV_SIZE, .maxauthsize = AES_BLOCK_SIZE, .setkey = gcm_aes_nx_set_key, .encrypt = gcm_aes_nx_encrypt, @@ -516,7 +517,7 @@ struct aead_alg nx_gcm4106_aes_alg = { }, .init = nx_crypto_ctx_aes_gcm_init, .exit = nx_crypto_ctx_aead_exit, - .ivsize = 8, + .ivsize = GCM_RFC4106_IV_SIZE, .maxauthsize = AES_BLOCK_SIZE, .setkey = gcm4106_aes_nx_set_key, .setauthsize = gcm4106_aes_nx_setauthsize, diff --git a/drivers/crypto/nx/nx.c b/drivers/crypto/nx/nx.c index 036057abb257..3a5e31be4764 100644 --- a/drivers/crypto/nx/nx.c +++ b/drivers/crypto/nx/nx.c @@ -833,7 +833,7 @@ static void __exit nx_fini(void) vio_unregister_driver(&nx_driver.viodriver); } -static struct vio_device_id nx_crypto_driver_ids[] = { +static const struct vio_device_id nx_crypto_driver_ids[] = { { "ibm,sym-encryption-v1", "ibm,sym-encryption" }, { "", "" } }; diff --git a/drivers/crypto/nx/nx.h b/drivers/crypto/nx/nx.h index 9347878d4f30..c3e54af18645 100644 --- a/drivers/crypto/nx/nx.h +++ b/drivers/crypto/nx/nx.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __NX_H__ #define __NX_H__ diff --git a/drivers/crypto/nx/nx_csbcpb.h b/drivers/crypto/nx/nx_csbcpb.h index a304f956d6f8..493f8490ff94 100644 --- a/drivers/crypto/nx/nx_csbcpb.h +++ b/drivers/crypto/nx/nx_csbcpb.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __NX_CSBCPB_H__ #define __NX_CSBCPB_H__ diff --git a/drivers/crypto/omap-aes-gcm.c b/drivers/crypto/omap-aes-gcm.c index 7d4f8a4be6d8..0cc3b65d7162 100644 --- a/drivers/crypto/omap-aes-gcm.c +++ b/drivers/crypto/omap-aes-gcm.c @@ -18,6 +18,7 @@ #include <linux/omap-dma.h> #include <linux/interrupt.h> #include <crypto/aes.h> +#include <crypto/gcm.h> #include <crypto/scatterwalk.h> #include <crypto/skcipher.h> #include <crypto/internal/aead.h> @@ -186,7 +187,7 @@ static int do_encrypt_iv(struct aead_request *req, u32 *tag, u32 *iv) sk_req = skcipher_request_alloc(ctx->ctr, GFP_KERNEL); if (!sk_req) { pr_err("skcipher: Failed to allocate request\n"); - return -1; + return -ENOMEM; } init_completion(&result.completion); @@ -214,7 +215,7 @@ static int do_encrypt_iv(struct aead_request *req, u32 *tag, u32 *iv) } /* fall through */ default: - pr_err("Encryption of IV failed for GCM mode"); + pr_err("Encryption of IV failed for GCM mode\n"); break; } @@ -311,7 +312,7 @@ static int omap_aes_gcm_crypt(struct aead_request *req, unsigned long mode) int err, assoclen; memset(rctx->auth_tag, 0, sizeof(rctx->auth_tag)); - memcpy(rctx->iv + 12, &counter, 4); + memcpy(rctx->iv + GCM_AES_IV_SIZE, &counter, 4); err = do_encrypt_iv(req, (u32 *)rctx->auth_tag, (u32 *)rctx->iv); if (err) @@ -339,7 +340,7 @@ int omap_aes_gcm_encrypt(struct aead_request *req) { struct omap_aes_reqctx *rctx = aead_request_ctx(req); - memcpy(rctx->iv, req->iv, 12); + memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE); return omap_aes_gcm_crypt(req, FLAGS_ENCRYPT | FLAGS_GCM); } @@ -347,7 +348,7 @@ int omap_aes_gcm_decrypt(struct aead_request *req) { struct omap_aes_reqctx *rctx = aead_request_ctx(req); - memcpy(rctx->iv, req->iv, 12); + memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE); return omap_aes_gcm_crypt(req, FLAGS_GCM); } diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index 5120a17731d0..fbec0a2e76dd 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -35,6 +35,7 @@ #include <linux/interrupt.h> #include <crypto/scatterwalk.h> #include <crypto/aes.h> +#include <crypto/gcm.h> #include <crypto/engine.h> #include <crypto/internal/skcipher.h> #include <crypto/internal/aead.h> @@ -767,7 +768,7 @@ static struct aead_alg algs_aead_gcm[] = { }, .init = omap_aes_gcm_cra_init, .exit = omap_aes_gcm_cra_exit, - .ivsize = 12, + .ivsize = GCM_AES_IV_SIZE, .maxauthsize = AES_BLOCK_SIZE, .setkey = omap_aes_gcm_setkey, .encrypt = omap_aes_gcm_encrypt, @@ -788,7 +789,7 @@ static struct aead_alg algs_aead_gcm[] = { .init = omap_aes_gcm_cra_init, .exit = omap_aes_gcm_cra_exit, .maxauthsize = AES_BLOCK_SIZE, - .ivsize = 8, + .ivsize = GCM_RFC4106_IV_SIZE, .setkey = omap_aes_4106gcm_setkey, .encrypt = omap_aes_4106gcm_encrypt, .decrypt = omap_aes_4106gcm_decrypt, @@ -974,11 +975,10 @@ static int omap_aes_get_res_of(struct omap_aes_dev *dd, struct device *dev, struct resource *res) { struct device_node *node = dev->of_node; - const struct of_device_id *match; int err = 0; - match = of_match_device(of_match_ptr(omap_aes_of_match), dev); - if (!match) { + dd->pdata = of_device_get_match_data(dev); + if (!dd->pdata) { dev_err(dev, "no compatible OF match\n"); err = -EINVAL; goto err; @@ -991,8 +991,6 @@ static int omap_aes_get_res_of(struct omap_aes_dev *dd, goto err; } - dd->pdata = match->data; - err: return err; } @@ -1095,6 +1093,7 @@ static int omap_aes_probe(struct platform_device *pdev) irq = platform_get_irq(pdev, 0); if (irq < 0) { dev_err(dev, "can't get IRQ resource\n"); + err = irq; goto err_irq; } diff --git a/drivers/crypto/omap-des.c b/drivers/crypto/omap-des.c index 0bcab00e0ff5..ebc5c0f11f03 100644 --- a/drivers/crypto/omap-des.c +++ b/drivers/crypto/omap-des.c @@ -928,16 +928,13 @@ MODULE_DEVICE_TABLE(of, omap_des_of_match); static int omap_des_get_of(struct omap_des_dev *dd, struct platform_device *pdev) { - const struct of_device_id *match; - match = of_match_device(of_match_ptr(omap_des_of_match), &pdev->dev); - if (!match) { + dd->pdata = of_device_get_match_data(&pdev->dev); + if (!dd->pdata) { dev_err(&pdev->dev, "no compatible OF match\n"); return -EINVAL; } - dd->pdata = match->data; - return 0; } #else @@ -1023,7 +1020,8 @@ static int omap_des_probe(struct platform_device *pdev) irq = platform_get_irq(pdev, 0); if (irq < 0) { - dev_err(dev, "can't get IRQ resource\n"); + dev_err(dev, "can't get IRQ resource: %d\n", irq); + err = irq; goto err_irq; } diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c index 9ad9d399daf1..86b89ace836f 100644 --- a/drivers/crypto/omap-sham.c +++ b/drivers/crypto/omap-sham.c @@ -1944,11 +1944,10 @@ static int omap_sham_get_res_of(struct omap_sham_dev *dd, struct device *dev, struct resource *res) { struct device_node *node = dev->of_node; - const struct of_device_id *match; int err = 0; - match = of_match_device(of_match_ptr(omap_sham_of_match), dev); - if (!match) { + dd->pdata = of_device_get_match_data(dev); + if (!dd->pdata) { dev_err(dev, "no compatible OF match\n"); err = -EINVAL; goto err; @@ -1968,8 +1967,6 @@ static int omap_sham_get_res_of(struct omap_sham_dev *dd, goto err; } - dd->pdata = match->data; - err: return err; } @@ -2133,7 +2130,7 @@ data_err: static int omap_sham_remove(struct platform_device *pdev) { - static struct omap_sham_dev *dd; + struct omap_sham_dev *dd; int i, j; dd = platform_get_drvdata(pdev); diff --git a/drivers/crypto/padlock-aes.c b/drivers/crypto/padlock-aes.c index b3869748cc6b..4b6642a25df5 100644 --- a/drivers/crypto/padlock-aes.c +++ b/drivers/crypto/padlock-aes.c @@ -482,7 +482,7 @@ static struct crypto_alg cbc_aes_alg = { } }; -static struct x86_cpu_id padlock_cpu_id[] = { +static const struct x86_cpu_id padlock_cpu_id[] = { X86_FEATURE_MATCH(X86_FEATURE_XCRYPT), {} }; diff --git a/drivers/crypto/padlock-sha.c b/drivers/crypto/padlock-sha.c index bc72d20c32c3..d32c79328876 100644 --- a/drivers/crypto/padlock-sha.c +++ b/drivers/crypto/padlock-sha.c @@ -509,7 +509,7 @@ static struct shash_alg sha256_alg_nano = { } }; -static struct x86_cpu_id padlock_sha_ids[] = { +static const struct x86_cpu_id padlock_sha_ids[] = { X86_FEATURE_MATCH(X86_FEATURE_PHE), {} }; diff --git a/drivers/crypto/picoxcell_crypto.c b/drivers/crypto/picoxcell_crypto.c index b6f14844702e..5a6dc53b2b9d 100644 --- a/drivers/crypto/picoxcell_crypto.c +++ b/drivers/crypto/picoxcell_crypto.c @@ -1125,9 +1125,9 @@ static irqreturn_t spacc_spacc_irq(int irq, void *dev) return IRQ_HANDLED; } -static void spacc_packet_timeout(unsigned long data) +static void spacc_packet_timeout(struct timer_list *t) { - struct spacc_engine *engine = (struct spacc_engine *)data; + struct spacc_engine *engine = from_timer(engine, t, packet_timeout); spacc_process_done(engine); } @@ -1714,8 +1714,7 @@ static int spacc_probe(struct platform_device *pdev) writel(SPA_IRQ_EN_STAT_EN | SPA_IRQ_EN_GLBL_EN, engine->regs + SPA_IRQ_EN_REG_OFFSET); - setup_timer(&engine->packet_timeout, spacc_packet_timeout, - (unsigned long)engine); + timer_setup(&engine->packet_timeout, spacc_packet_timeout, 0); INIT_LIST_HEAD(&engine->pending); INIT_LIST_HEAD(&engine->completed); diff --git a/drivers/crypto/qat/Makefile b/drivers/crypto/qat/Makefile index 8265106f1c8e..7dd15e751d02 100644 --- a/drivers/crypto/qat/Makefile +++ b/drivers/crypto/qat/Makefile @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_CRYPTO_DEV_QAT) += qat_common/ obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCC) += qat_dh895xcc/ obj-$(CONFIG_CRYPTO_DEV_QAT_C3XXX) += qat_c3xxx/ diff --git a/drivers/crypto/qat/qat_common/Makefile b/drivers/crypto/qat/qat_common/Makefile index 92fb6ffdc062..47a8e3d8b81a 100644 --- a/drivers/crypto/qat/qat_common/Makefile +++ b/drivers/crypto/qat/qat_common/Makefile @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_CRYPTO_DEV_QAT) += intel_qat.o intel_qat-objs := adf_cfg.o \ adf_isr.o \ diff --git a/drivers/crypto/qat/qat_common/adf_aer.c b/drivers/crypto/qat/qat_common/adf_aer.c index d3e25c37dc33..da8a2d3b5e9a 100644 --- a/drivers/crypto/qat/qat_common/adf_aer.c +++ b/drivers/crypto/qat/qat_common/adf_aer.c @@ -208,7 +208,7 @@ static pci_ers_result_t adf_slot_reset(struct pci_dev *pdev) static void adf_resume(struct pci_dev *pdev) { dev_info(&pdev->dev, "Acceleration driver reset completed\n"); - dev_info(&pdev->dev, "Device is up and runnig\n"); + dev_info(&pdev->dev, "Device is up and running\n"); } static const struct pci_error_handlers adf_err_handler = { diff --git a/drivers/crypto/qat/qat_common/adf_dev_mgr.c b/drivers/crypto/qat/qat_common/adf_dev_mgr.c index 8afac52677a6..2d06409bd3c4 100644 --- a/drivers/crypto/qat/qat_common/adf_dev_mgr.c +++ b/drivers/crypto/qat/qat_common/adf_dev_mgr.c @@ -228,11 +228,8 @@ int adf_devmgr_add_dev(struct adf_accel_dev *accel_dev, list_add_tail(&map->list, &vfs_table); } else if (accel_dev->is_vf && pf) { /* VF on host */ - struct adf_accel_vf_info *vf_info; struct vf_id_map *map; - vf_info = pf->pf.vf_info + adf_get_vf_id(accel_dev); - map = adf_find_vf(adf_get_vf_num(accel_dev)); if (map) { struct vf_id_map *next; diff --git a/drivers/crypto/qat/qat_common/qat_asym_algs.c b/drivers/crypto/qat/qat_common/qat_asym_algs.c index 6f5dd68449c6..13c52d6bf630 100644 --- a/drivers/crypto/qat/qat_common/qat_asym_algs.c +++ b/drivers/crypto/qat/qat_common/qat_asym_algs.c @@ -443,9 +443,6 @@ static int qat_dh_set_params(struct qat_dh_ctx *ctx, struct dh *params) struct qat_crypto_instance *inst = ctx->inst; struct device *dev = &GET_DEV(inst->accel_dev); - if (unlikely(!params->p || !params->g)) - return -EINVAL; - if (qat_dh_check_params_length(params->p_size << 3)) return -EINVAL; @@ -462,11 +459,8 @@ static int qat_dh_set_params(struct qat_dh_ctx *ctx, struct dh *params) } ctx->g = dma_zalloc_coherent(dev, ctx->p_size, &ctx->dma_g, GFP_KERNEL); - if (!ctx->g) { - dma_free_coherent(dev, ctx->p_size, ctx->p, ctx->dma_p); - ctx->p = NULL; + if (!ctx->g) return -ENOMEM; - } memcpy(ctx->g + (ctx->p_size - params->g_size), params->g, params->g_size); @@ -507,18 +501,22 @@ static int qat_dh_set_secret(struct crypto_kpp *tfm, const void *buf, ret = qat_dh_set_params(ctx, ¶ms); if (ret < 0) - return ret; + goto err_clear_ctx; ctx->xa = dma_zalloc_coherent(dev, ctx->p_size, &ctx->dma_xa, GFP_KERNEL); if (!ctx->xa) { - qat_dh_clear_ctx(dev, ctx); - return -ENOMEM; + ret = -ENOMEM; + goto err_clear_ctx; } memcpy(ctx->xa + (ctx->p_size - params.key_size), params.key, params.key_size); return 0; + +err_clear_ctx: + qat_dh_clear_ctx(dev, ctx); + return ret; } static unsigned int qat_dh_max_size(struct crypto_kpp *tfm) diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index e2454d90d949..98d22c2096e3 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -567,26 +567,26 @@ qat_uclo_check_image_compat(struct icp_qat_uof_encap_obj *encap_uof_obj, code_page->imp_expr_tab_offset); if (uc_var_tab->entry_num || imp_var_tab->entry_num || imp_expr_tab->entry_num) { - pr_err("QAT: UOF can't contain imported variable to be parsed"); + pr_err("QAT: UOF can't contain imported variable to be parsed\n"); return -EINVAL; } neigh_reg_tab = (struct icp_qat_uof_objtable *) (encap_uof_obj->beg_uof + code_page->neigh_reg_tab_offset); if (neigh_reg_tab->entry_num) { - pr_err("QAT: UOF can't contain shared control store feature"); + pr_err("QAT: UOF can't contain shared control store feature\n"); return -EINVAL; } if (image->numpages > 1) { - pr_err("QAT: UOF can't contain multiple pages"); + pr_err("QAT: UOF can't contain multiple pages\n"); return -EINVAL; } if (ICP_QAT_SHARED_USTORE_MODE(image->ae_mode)) { - pr_err("QAT: UOF can't use shared control store feature"); + pr_err("QAT: UOF can't use shared control store feature\n"); return -EFAULT; } if (RELOADABLE_CTX_SHARED_MODE(image->ae_mode)) { - pr_err("QAT: UOF can't use reloadable feature"); + pr_err("QAT: UOF can't use reloadable feature\n"); return -EFAULT; } return 0; @@ -702,7 +702,7 @@ static int qat_uclo_map_ae(struct icp_qat_fw_loader_handle *handle, int max_ae) } } if (!mflag) { - pr_err("QAT: uimage uses AE not set"); + pr_err("QAT: uimage uses AE not set\n"); return -EINVAL; } return 0; @@ -791,6 +791,7 @@ static int qat_uclo_init_reg(struct icp_qat_fw_loader_handle *handle, case ICP_GPA_ABS: case ICP_GPB_ABS: ctx_mask = 0; + /* fall through */ case ICP_GPA_REL: case ICP_GPB_REL: return qat_hal_init_gpr(handle, ae, ctx_mask, reg_type, @@ -800,6 +801,7 @@ static int qat_uclo_init_reg(struct icp_qat_fw_loader_handle *handle, case ICP_SR_RD_ABS: case ICP_DR_RD_ABS: ctx_mask = 0; + /* fall through */ case ICP_SR_REL: case ICP_DR_REL: case ICP_SR_RD_REL: @@ -809,6 +811,7 @@ static int qat_uclo_init_reg(struct icp_qat_fw_loader_handle *handle, case ICP_SR_WR_ABS: case ICP_DR_WR_ABS: ctx_mask = 0; + /* fall through */ case ICP_SR_WR_REL: case ICP_DR_WR_REL: return qat_hal_init_wr_xfer(handle, ae, ctx_mask, reg_type, diff --git a/drivers/crypto/qce/Makefile b/drivers/crypto/qce/Makefile index 348dc3173afa..19a7f899acff 100644 --- a/drivers/crypto/qce/Makefile +++ b/drivers/crypto/qce/Makefile @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_CRYPTO_DEV_QCE) += qcrypto.o qcrypto-objs := core.o \ common.o \ diff --git a/drivers/crypto/qce/ablkcipher.c b/drivers/crypto/qce/ablkcipher.c index b04b42f48366..ea4d96bf47e8 100644 --- a/drivers/crypto/qce/ablkcipher.c +++ b/drivers/crypto/qce/ablkcipher.c @@ -248,10 +248,7 @@ static int qce_ablkcipher_init(struct crypto_tfm *tfm) ctx->fallback = crypto_alloc_skcipher(crypto_tfm_alg_name(tfm), 0, CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK); - if (IS_ERR(ctx->fallback)) - return PTR_ERR(ctx->fallback); - - return 0; + return PTR_ERR_OR_ZERO(ctx->fallback); } static void qce_ablkcipher_exit(struct crypto_tfm *tfm) diff --git a/drivers/crypto/qce/sha.c b/drivers/crypto/qce/sha.c index 47e114ac09d0..53227d70d397 100644 --- a/drivers/crypto/qce/sha.c +++ b/drivers/crypto/qce/sha.c @@ -349,28 +349,12 @@ static int qce_ahash_digest(struct ahash_request *req) return qce->async_req_enqueue(tmpl->qce, &req->base); } -struct qce_ahash_result { - struct completion completion; - int error; -}; - -static void qce_digest_complete(struct crypto_async_request *req, int error) -{ - struct qce_ahash_result *result = req->data; - - if (error == -EINPROGRESS) - return; - - result->error = error; - complete(&result->completion); -} - static int qce_ahash_hmac_setkey(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen) { unsigned int digestsize = crypto_ahash_digestsize(tfm); struct qce_sha_ctx *ctx = crypto_tfm_ctx(&tfm->base); - struct qce_ahash_result result; + struct crypto_wait wait; struct ahash_request *req; struct scatterlist sg; unsigned int blocksize; @@ -405,9 +389,9 @@ static int qce_ahash_hmac_setkey(struct crypto_ahash *tfm, const u8 *key, goto err_free_ahash; } - init_completion(&result.completion); + crypto_init_wait(&wait); ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, - qce_digest_complete, &result); + crypto_req_done, &wait); crypto_ahash_clear_flags(ahash_tfm, ~0); buf = kzalloc(keylen + QCE_MAX_ALIGN_SIZE, GFP_KERNEL); @@ -420,13 +404,7 @@ static int qce_ahash_hmac_setkey(struct crypto_ahash *tfm, const u8 *key, sg_init_one(&sg, buf, keylen); ahash_request_set_crypt(req, &sg, ctx->authkey, keylen); - ret = crypto_ahash_digest(req); - if (ret == -EINPROGRESS || ret == -EBUSY) { - ret = wait_for_completion_interruptible(&result.completion); - if (!ret) - ret = result.error; - } - + ret = crypto_wait_req(crypto_ahash_digest(req), &wait); if (ret) crypto_ahash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); diff --git a/drivers/crypto/rockchip/rk3288_crypto.c b/drivers/crypto/rockchip/rk3288_crypto.c index d0f80c6241f9..c9d622abd90c 100644 --- a/drivers/crypto/rockchip/rk3288_crypto.c +++ b/drivers/crypto/rockchip/rk3288_crypto.c @@ -169,50 +169,82 @@ static irqreturn_t rk_crypto_irq_handle(int irq, void *dev_id) { struct rk_crypto_info *dev = platform_get_drvdata(dev_id); u32 interrupt_status; - int err = 0; spin_lock(&dev->lock); interrupt_status = CRYPTO_READ(dev, RK_CRYPTO_INTSTS); CRYPTO_WRITE(dev, RK_CRYPTO_INTSTS, interrupt_status); + if (interrupt_status & 0x0a) { dev_warn(dev->dev, "DMA Error\n"); - err = -EFAULT; - } else if (interrupt_status & 0x05) { - err = dev->update(dev); + dev->err = -EFAULT; } - if (err) - dev->complete(dev, err); + tasklet_schedule(&dev->done_task); + spin_unlock(&dev->lock); return IRQ_HANDLED; } -static void rk_crypto_tasklet_cb(unsigned long data) +static int rk_crypto_enqueue(struct rk_crypto_info *dev, + struct crypto_async_request *async_req) +{ + unsigned long flags; + int ret; + + spin_lock_irqsave(&dev->lock, flags); + ret = crypto_enqueue_request(&dev->queue, async_req); + if (dev->busy) { + spin_unlock_irqrestore(&dev->lock, flags); + return ret; + } + dev->busy = true; + spin_unlock_irqrestore(&dev->lock, flags); + tasklet_schedule(&dev->queue_task); + + return ret; +} + +static void rk_crypto_queue_task_cb(unsigned long data) { struct rk_crypto_info *dev = (struct rk_crypto_info *)data; struct crypto_async_request *async_req, *backlog; unsigned long flags; int err = 0; + dev->err = 0; spin_lock_irqsave(&dev->lock, flags); backlog = crypto_get_backlog(&dev->queue); async_req = crypto_dequeue_request(&dev->queue); - spin_unlock_irqrestore(&dev->lock, flags); + if (!async_req) { - dev_err(dev->dev, "async_req is NULL !!\n"); + dev->busy = false; + spin_unlock_irqrestore(&dev->lock, flags); return; } + spin_unlock_irqrestore(&dev->lock, flags); + if (backlog) { backlog->complete(backlog, -EINPROGRESS); backlog = NULL; } - if (crypto_tfm_alg_type(async_req->tfm) == CRYPTO_ALG_TYPE_ABLKCIPHER) - dev->ablk_req = ablkcipher_request_cast(async_req); - else - dev->ahash_req = ahash_request_cast(async_req); + dev->async_req = async_req; err = dev->start(dev); if (err) - dev->complete(dev, err); + dev->complete(dev->async_req, err); +} + +static void rk_crypto_done_task_cb(unsigned long data) +{ + struct rk_crypto_info *dev = (struct rk_crypto_info *)data; + + if (dev->err) { + dev->complete(dev->async_req, dev->err); + return; + } + + dev->err = dev->update(dev); + if (dev->err) + dev->complete(dev->async_req, dev->err); } static struct rk_crypto_tmp *rk_cipher_algs[] = { @@ -361,14 +393,18 @@ static int rk_crypto_probe(struct platform_device *pdev) crypto_info->dev = &pdev->dev; platform_set_drvdata(pdev, crypto_info); - tasklet_init(&crypto_info->crypto_tasklet, - rk_crypto_tasklet_cb, (unsigned long)crypto_info); + tasklet_init(&crypto_info->queue_task, + rk_crypto_queue_task_cb, (unsigned long)crypto_info); + tasklet_init(&crypto_info->done_task, + rk_crypto_done_task_cb, (unsigned long)crypto_info); crypto_init_queue(&crypto_info->queue, 50); crypto_info->enable_clk = rk_crypto_enable_clk; crypto_info->disable_clk = rk_crypto_disable_clk; crypto_info->load_data = rk_load_data; crypto_info->unload_data = rk_unload_data; + crypto_info->enqueue = rk_crypto_enqueue; + crypto_info->busy = false; err = rk_crypto_register(crypto_info); if (err) { @@ -380,7 +416,8 @@ static int rk_crypto_probe(struct platform_device *pdev) return 0; err_register_alg: - tasklet_kill(&crypto_info->crypto_tasklet); + tasklet_kill(&crypto_info->queue_task); + tasklet_kill(&crypto_info->done_task); err_crypto: return err; } @@ -390,7 +427,8 @@ static int rk_crypto_remove(struct platform_device *pdev) struct rk_crypto_info *crypto_tmp = platform_get_drvdata(pdev); rk_crypto_unregister(); - tasklet_kill(&crypto_tmp->crypto_tasklet); + tasklet_kill(&crypto_tmp->done_task); + tasklet_kill(&crypto_tmp->queue_task); return 0; } diff --git a/drivers/crypto/rockchip/rk3288_crypto.h b/drivers/crypto/rockchip/rk3288_crypto.h index d7b71fea320b..d5fb4013fb42 100644 --- a/drivers/crypto/rockchip/rk3288_crypto.h +++ b/drivers/crypto/rockchip/rk3288_crypto.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __RK3288_CRYPTO_H__ #define __RK3288_CRYPTO_H__ @@ -190,9 +191,10 @@ struct rk_crypto_info { void __iomem *reg; int irq; struct crypto_queue queue; - struct tasklet_struct crypto_tasklet; - struct ablkcipher_request *ablk_req; - struct ahash_request *ahash_req; + struct tasklet_struct queue_task; + struct tasklet_struct done_task; + struct crypto_async_request *async_req; + int err; /* device lock */ spinlock_t lock; @@ -208,18 +210,20 @@ struct rk_crypto_info { size_t nents; unsigned int total; unsigned int count; - u32 mode; dma_addr_t addr_in; dma_addr_t addr_out; + bool busy; int (*start)(struct rk_crypto_info *dev); int (*update)(struct rk_crypto_info *dev); - void (*complete)(struct rk_crypto_info *dev, int err); + void (*complete)(struct crypto_async_request *base, int err); int (*enable_clk)(struct rk_crypto_info *dev); void (*disable_clk)(struct rk_crypto_info *dev); int (*load_data)(struct rk_crypto_info *dev, struct scatterlist *sg_src, struct scatterlist *sg_dst); void (*unload_data)(struct rk_crypto_info *dev); + int (*enqueue)(struct rk_crypto_info *dev, + struct crypto_async_request *async_req); }; /* the private variable of hash */ @@ -232,12 +236,14 @@ struct rk_ahash_ctx { /* the privete variable of hash for fallback */ struct rk_ahash_rctx { struct ahash_request fallback_req; + u32 mode; }; /* the private variable of cipher */ struct rk_cipher_ctx { struct rk_crypto_info *dev; unsigned int keylen; + u32 mode; }; enum alg_type { diff --git a/drivers/crypto/rockchip/rk3288_crypto_ablkcipher.c b/drivers/crypto/rockchip/rk3288_crypto_ablkcipher.c index b5a3afe222e4..639c15c5364b 100644 --- a/drivers/crypto/rockchip/rk3288_crypto_ablkcipher.c +++ b/drivers/crypto/rockchip/rk3288_crypto_ablkcipher.c @@ -15,35 +15,19 @@ #define RK_CRYPTO_DEC BIT(0) -static void rk_crypto_complete(struct rk_crypto_info *dev, int err) +static void rk_crypto_complete(struct crypto_async_request *base, int err) { - if (dev->ablk_req->base.complete) - dev->ablk_req->base.complete(&dev->ablk_req->base, err); + if (base->complete) + base->complete(base, err); } static int rk_handle_req(struct rk_crypto_info *dev, struct ablkcipher_request *req) { - unsigned long flags; - int err; - if (!IS_ALIGNED(req->nbytes, dev->align_size)) return -EINVAL; - - dev->left_bytes = req->nbytes; - dev->total = req->nbytes; - dev->sg_src = req->src; - dev->first = req->src; - dev->nents = sg_nents(req->src); - dev->sg_dst = req->dst; - dev->aligned = 1; - dev->ablk_req = req; - - spin_lock_irqsave(&dev->lock, flags); - err = ablkcipher_enqueue_request(&dev->queue, req); - spin_unlock_irqrestore(&dev->lock, flags); - tasklet_schedule(&dev->crypto_tasklet); - return err; + else + return dev->enqueue(dev, &req->base); } static int rk_aes_setkey(struct crypto_ablkcipher *cipher, @@ -93,7 +77,7 @@ static int rk_aes_ecb_encrypt(struct ablkcipher_request *req) struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm); struct rk_crypto_info *dev = ctx->dev; - dev->mode = RK_CRYPTO_AES_ECB_MODE; + ctx->mode = RK_CRYPTO_AES_ECB_MODE; return rk_handle_req(dev, req); } @@ -103,7 +87,7 @@ static int rk_aes_ecb_decrypt(struct ablkcipher_request *req) struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm); struct rk_crypto_info *dev = ctx->dev; - dev->mode = RK_CRYPTO_AES_ECB_MODE | RK_CRYPTO_DEC; + ctx->mode = RK_CRYPTO_AES_ECB_MODE | RK_CRYPTO_DEC; return rk_handle_req(dev, req); } @@ -113,7 +97,7 @@ static int rk_aes_cbc_encrypt(struct ablkcipher_request *req) struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm); struct rk_crypto_info *dev = ctx->dev; - dev->mode = RK_CRYPTO_AES_CBC_MODE; + ctx->mode = RK_CRYPTO_AES_CBC_MODE; return rk_handle_req(dev, req); } @@ -123,7 +107,7 @@ static int rk_aes_cbc_decrypt(struct ablkcipher_request *req) struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm); struct rk_crypto_info *dev = ctx->dev; - dev->mode = RK_CRYPTO_AES_CBC_MODE | RK_CRYPTO_DEC; + ctx->mode = RK_CRYPTO_AES_CBC_MODE | RK_CRYPTO_DEC; return rk_handle_req(dev, req); } @@ -133,7 +117,7 @@ static int rk_des_ecb_encrypt(struct ablkcipher_request *req) struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm); struct rk_crypto_info *dev = ctx->dev; - dev->mode = 0; + ctx->mode = 0; return rk_handle_req(dev, req); } @@ -143,7 +127,7 @@ static int rk_des_ecb_decrypt(struct ablkcipher_request *req) struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm); struct rk_crypto_info *dev = ctx->dev; - dev->mode = RK_CRYPTO_DEC; + ctx->mode = RK_CRYPTO_DEC; return rk_handle_req(dev, req); } @@ -153,7 +137,7 @@ static int rk_des_cbc_encrypt(struct ablkcipher_request *req) struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm); struct rk_crypto_info *dev = ctx->dev; - dev->mode = RK_CRYPTO_TDES_CHAINMODE_CBC; + ctx->mode = RK_CRYPTO_TDES_CHAINMODE_CBC; return rk_handle_req(dev, req); } @@ -163,7 +147,7 @@ static int rk_des_cbc_decrypt(struct ablkcipher_request *req) struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm); struct rk_crypto_info *dev = ctx->dev; - dev->mode = RK_CRYPTO_TDES_CHAINMODE_CBC | RK_CRYPTO_DEC; + ctx->mode = RK_CRYPTO_TDES_CHAINMODE_CBC | RK_CRYPTO_DEC; return rk_handle_req(dev, req); } @@ -173,7 +157,7 @@ static int rk_des3_ede_ecb_encrypt(struct ablkcipher_request *req) struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm); struct rk_crypto_info *dev = ctx->dev; - dev->mode = RK_CRYPTO_TDES_SELECT; + ctx->mode = RK_CRYPTO_TDES_SELECT; return rk_handle_req(dev, req); } @@ -183,7 +167,7 @@ static int rk_des3_ede_ecb_decrypt(struct ablkcipher_request *req) struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm); struct rk_crypto_info *dev = ctx->dev; - dev->mode = RK_CRYPTO_TDES_SELECT | RK_CRYPTO_DEC; + ctx->mode = RK_CRYPTO_TDES_SELECT | RK_CRYPTO_DEC; return rk_handle_req(dev, req); } @@ -193,7 +177,7 @@ static int rk_des3_ede_cbc_encrypt(struct ablkcipher_request *req) struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm); struct rk_crypto_info *dev = ctx->dev; - dev->mode = RK_CRYPTO_TDES_SELECT | RK_CRYPTO_TDES_CHAINMODE_CBC; + ctx->mode = RK_CRYPTO_TDES_SELECT | RK_CRYPTO_TDES_CHAINMODE_CBC; return rk_handle_req(dev, req); } @@ -203,15 +187,16 @@ static int rk_des3_ede_cbc_decrypt(struct ablkcipher_request *req) struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm); struct rk_crypto_info *dev = ctx->dev; - dev->mode = RK_CRYPTO_TDES_SELECT | RK_CRYPTO_TDES_CHAINMODE_CBC | + ctx->mode = RK_CRYPTO_TDES_SELECT | RK_CRYPTO_TDES_CHAINMODE_CBC | RK_CRYPTO_DEC; return rk_handle_req(dev, req); } static void rk_ablk_hw_init(struct rk_crypto_info *dev) { - struct crypto_ablkcipher *cipher = - crypto_ablkcipher_reqtfm(dev->ablk_req); + struct ablkcipher_request *req = + ablkcipher_request_cast(dev->async_req); + struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(req); struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(cipher); u32 ivsize, block, conf_reg = 0; @@ -220,25 +205,23 @@ static void rk_ablk_hw_init(struct rk_crypto_info *dev) ivsize = crypto_ablkcipher_ivsize(cipher); if (block == DES_BLOCK_SIZE) { - dev->mode |= RK_CRYPTO_TDES_FIFO_MODE | + ctx->mode |= RK_CRYPTO_TDES_FIFO_MODE | RK_CRYPTO_TDES_BYTESWAP_KEY | RK_CRYPTO_TDES_BYTESWAP_IV; - CRYPTO_WRITE(dev, RK_CRYPTO_TDES_CTRL, dev->mode); - memcpy_toio(dev->reg + RK_CRYPTO_TDES_IV_0, - dev->ablk_req->info, ivsize); + CRYPTO_WRITE(dev, RK_CRYPTO_TDES_CTRL, ctx->mode); + memcpy_toio(dev->reg + RK_CRYPTO_TDES_IV_0, req->info, ivsize); conf_reg = RK_CRYPTO_DESSEL; } else { - dev->mode |= RK_CRYPTO_AES_FIFO_MODE | + ctx->mode |= RK_CRYPTO_AES_FIFO_MODE | RK_CRYPTO_AES_KEY_CHANGE | RK_CRYPTO_AES_BYTESWAP_KEY | RK_CRYPTO_AES_BYTESWAP_IV; if (ctx->keylen == AES_KEYSIZE_192) - dev->mode |= RK_CRYPTO_AES_192BIT_key; + ctx->mode |= RK_CRYPTO_AES_192BIT_key; else if (ctx->keylen == AES_KEYSIZE_256) - dev->mode |= RK_CRYPTO_AES_256BIT_key; - CRYPTO_WRITE(dev, RK_CRYPTO_AES_CTRL, dev->mode); - memcpy_toio(dev->reg + RK_CRYPTO_AES_IV_0, - dev->ablk_req->info, ivsize); + ctx->mode |= RK_CRYPTO_AES_256BIT_key; + CRYPTO_WRITE(dev, RK_CRYPTO_AES_CTRL, ctx->mode); + memcpy_toio(dev->reg + RK_CRYPTO_AES_IV_0, req->info, ivsize); } conf_reg |= RK_CRYPTO_BYTESWAP_BTFIFO | RK_CRYPTO_BYTESWAP_BRFIFO; @@ -268,8 +251,18 @@ static int rk_set_data_start(struct rk_crypto_info *dev) static int rk_ablk_start(struct rk_crypto_info *dev) { + struct ablkcipher_request *req = + ablkcipher_request_cast(dev->async_req); unsigned long flags; - int err; + int err = 0; + + dev->left_bytes = req->nbytes; + dev->total = req->nbytes; + dev->sg_src = req->src; + dev->first = req->src; + dev->nents = sg_nents(req->src); + dev->sg_dst = req->dst; + dev->aligned = 1; spin_lock_irqsave(&dev->lock, flags); rk_ablk_hw_init(dev); @@ -280,15 +273,16 @@ static int rk_ablk_start(struct rk_crypto_info *dev) static void rk_iv_copyback(struct rk_crypto_info *dev) { - struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(dev->ablk_req); + struct ablkcipher_request *req = + ablkcipher_request_cast(dev->async_req); + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); u32 ivsize = crypto_ablkcipher_ivsize(tfm); if (ivsize == DES_BLOCK_SIZE) - memcpy_fromio(dev->ablk_req->info, - dev->reg + RK_CRYPTO_TDES_IV_0, ivsize); + memcpy_fromio(req->info, dev->reg + RK_CRYPTO_TDES_IV_0, + ivsize); else if (ivsize == AES_BLOCK_SIZE) - memcpy_fromio(dev->ablk_req->info, - dev->reg + RK_CRYPTO_AES_IV_0, ivsize); + memcpy_fromio(req->info, dev->reg + RK_CRYPTO_AES_IV_0, ivsize); } /* return: @@ -298,10 +292,12 @@ static void rk_iv_copyback(struct rk_crypto_info *dev) static int rk_ablk_rx(struct rk_crypto_info *dev) { int err = 0; + struct ablkcipher_request *req = + ablkcipher_request_cast(dev->async_req); dev->unload_data(dev); if (!dev->aligned) { - if (!sg_pcopy_from_buffer(dev->ablk_req->dst, dev->nents, + if (!sg_pcopy_from_buffer(req->dst, dev->nents, dev->addr_vir, dev->count, dev->total - dev->left_bytes - dev->count)) { @@ -324,7 +320,8 @@ static int rk_ablk_rx(struct rk_crypto_info *dev) } else { rk_iv_copyback(dev); /* here show the calculation is over without any err */ - dev->complete(dev, 0); + dev->complete(dev->async_req, 0); + tasklet_schedule(&dev->queue_task); } out_rx: return err; diff --git a/drivers/crypto/rockchip/rk3288_crypto_ahash.c b/drivers/crypto/rockchip/rk3288_crypto_ahash.c index 718588219f75..821a506b9e17 100644 --- a/drivers/crypto/rockchip/rk3288_crypto_ahash.c +++ b/drivers/crypto/rockchip/rk3288_crypto_ahash.c @@ -40,14 +40,16 @@ static int zero_message_process(struct ahash_request *req) return 0; } -static void rk_ahash_crypto_complete(struct rk_crypto_info *dev, int err) +static void rk_ahash_crypto_complete(struct crypto_async_request *base, int err) { - if (dev->ahash_req->base.complete) - dev->ahash_req->base.complete(&dev->ahash_req->base, err); + if (base->complete) + base->complete(base, err); } static void rk_ahash_reg_init(struct rk_crypto_info *dev) { + struct ahash_request *req = ahash_request_cast(dev->async_req); + struct rk_ahash_rctx *rctx = ahash_request_ctx(req); int reg_status = 0; reg_status = CRYPTO_READ(dev, RK_CRYPTO_CTRL) | @@ -67,7 +69,7 @@ static void rk_ahash_reg_init(struct rk_crypto_info *dev) CRYPTO_WRITE(dev, RK_CRYPTO_INTSTS, RK_CRYPTO_HRDMA_ERR_INT | RK_CRYPTO_HRDMA_DONE_INT); - CRYPTO_WRITE(dev, RK_CRYPTO_HASH_CTRL, dev->mode | + CRYPTO_WRITE(dev, RK_CRYPTO_HASH_CTRL, rctx->mode | RK_CRYPTO_HASH_SWAP_DO); CRYPTO_WRITE(dev, RK_CRYPTO_CONF, RK_CRYPTO_BYTESWAP_HRFIFO | @@ -164,64 +166,13 @@ static int rk_ahash_export(struct ahash_request *req, void *out) static int rk_ahash_digest(struct ahash_request *req) { - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct rk_ahash_ctx *tctx = crypto_tfm_ctx(req->base.tfm); - struct rk_crypto_info *dev = NULL; - unsigned long flags; - int ret; + struct rk_crypto_info *dev = tctx->dev; if (!req->nbytes) return zero_message_process(req); - - dev = tctx->dev; - dev->total = req->nbytes; - dev->left_bytes = req->nbytes; - dev->aligned = 0; - dev->mode = 0; - dev->align_size = 4; - dev->sg_dst = NULL; - dev->sg_src = req->src; - dev->first = req->src; - dev->nents = sg_nents(req->src); - - switch (crypto_ahash_digestsize(tfm)) { - case SHA1_DIGEST_SIZE: - dev->mode = RK_CRYPTO_HASH_SHA1; - break; - case SHA256_DIGEST_SIZE: - dev->mode = RK_CRYPTO_HASH_SHA256; - break; - case MD5_DIGEST_SIZE: - dev->mode = RK_CRYPTO_HASH_MD5; - break; - default: - return -EINVAL; - } - - rk_ahash_reg_init(dev); - - spin_lock_irqsave(&dev->lock, flags); - ret = crypto_enqueue_request(&dev->queue, &req->base); - spin_unlock_irqrestore(&dev->lock, flags); - - tasklet_schedule(&dev->crypto_tasklet); - - /* - * it will take some time to process date after last dma transmission. - * - * waiting time is relative with the last date len, - * so cannot set a fixed time here. - * 10-50 makes system not call here frequently wasting - * efficiency, and make it response quickly when dma - * complete. - */ - while (!CRYPTO_READ(dev, RK_CRYPTO_HASH_STS)) - usleep_range(10, 50); - - memcpy_fromio(req->result, dev->reg + RK_CRYPTO_HASH_DOUT_0, - crypto_ahash_digestsize(tfm)); - - return 0; + else + return dev->enqueue(dev, &req->base); } static void crypto_ahash_dma_start(struct rk_crypto_info *dev) @@ -244,12 +195,45 @@ static int rk_ahash_set_data_start(struct rk_crypto_info *dev) static int rk_ahash_start(struct rk_crypto_info *dev) { + struct ahash_request *req = ahash_request_cast(dev->async_req); + struct crypto_ahash *tfm; + struct rk_ahash_rctx *rctx; + + dev->total = req->nbytes; + dev->left_bytes = req->nbytes; + dev->aligned = 0; + dev->align_size = 4; + dev->sg_dst = NULL; + dev->sg_src = req->src; + dev->first = req->src; + dev->nents = sg_nents(req->src); + rctx = ahash_request_ctx(req); + rctx->mode = 0; + + tfm = crypto_ahash_reqtfm(req); + switch (crypto_ahash_digestsize(tfm)) { + case SHA1_DIGEST_SIZE: + rctx->mode = RK_CRYPTO_HASH_SHA1; + break; + case SHA256_DIGEST_SIZE: + rctx->mode = RK_CRYPTO_HASH_SHA256; + break; + case MD5_DIGEST_SIZE: + rctx->mode = RK_CRYPTO_HASH_MD5; + break; + default: + return -EINVAL; + } + + rk_ahash_reg_init(dev); return rk_ahash_set_data_start(dev); } static int rk_ahash_crypto_rx(struct rk_crypto_info *dev) { int err = 0; + struct ahash_request *req = ahash_request_cast(dev->async_req); + struct crypto_ahash *tfm; dev->unload_data(dev); if (dev->left_bytes) { @@ -264,7 +248,24 @@ static int rk_ahash_crypto_rx(struct rk_crypto_info *dev) } err = rk_ahash_set_data_start(dev); } else { - dev->complete(dev, 0); + /* + * it will take some time to process date after last dma + * transmission. + * + * waiting time is relative with the last date len, + * so cannot set a fixed time here. + * 10us makes system not call here frequently wasting + * efficiency, and make it response quickly when dma + * complete. + */ + while (!CRYPTO_READ(dev, RK_CRYPTO_HASH_STS)) + udelay(10); + + tfm = crypto_ahash_reqtfm(req); + memcpy_fromio(req->result, dev->reg + RK_CRYPTO_HASH_DOUT_0, + crypto_ahash_digestsize(tfm)); + dev->complete(dev->async_req, 0); + tasklet_schedule(&dev->queue_task); } out_rx: diff --git a/drivers/crypto/s5p-sss.c b/drivers/crypto/s5p-sss.c index 7ac657f46d15..142c6020cec7 100644 --- a/drivers/crypto/s5p-sss.c +++ b/drivers/crypto/s5p-sss.c @@ -1,14 +1,16 @@ /* * Cryptographic API. * - * Support for Samsung S5PV210 HW acceleration. + * Support for Samsung S5PV210 and Exynos HW acceleration. * * Copyright (C) 2011 NetUP Inc. All rights reserved. + * Copyright (c) 2017 Samsung Electronics Co., Ltd. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as published * by the Free Software Foundation. * + * Hash part based on omap-sham.c driver. */ #include <linux/clk.h> @@ -30,98 +32,112 @@ #include <crypto/algapi.h> #include <crypto/scatterwalk.h> -#define _SBF(s, v) ((v) << (s)) +#include <crypto/hash.h> +#include <crypto/md5.h> +#include <crypto/sha.h> +#include <crypto/internal/hash.h> + +#define _SBF(s, v) ((v) << (s)) /* Feed control registers */ -#define SSS_REG_FCINTSTAT 0x0000 -#define SSS_FCINTSTAT_BRDMAINT BIT(3) -#define SSS_FCINTSTAT_BTDMAINT BIT(2) -#define SSS_FCINTSTAT_HRDMAINT BIT(1) -#define SSS_FCINTSTAT_PKDMAINT BIT(0) - -#define SSS_REG_FCINTENSET 0x0004 -#define SSS_FCINTENSET_BRDMAINTENSET BIT(3) -#define SSS_FCINTENSET_BTDMAINTENSET BIT(2) -#define SSS_FCINTENSET_HRDMAINTENSET BIT(1) -#define SSS_FCINTENSET_PKDMAINTENSET BIT(0) - -#define SSS_REG_FCINTENCLR 0x0008 -#define SSS_FCINTENCLR_BRDMAINTENCLR BIT(3) -#define SSS_FCINTENCLR_BTDMAINTENCLR BIT(2) -#define SSS_FCINTENCLR_HRDMAINTENCLR BIT(1) -#define SSS_FCINTENCLR_PKDMAINTENCLR BIT(0) - -#define SSS_REG_FCINTPEND 0x000C -#define SSS_FCINTPEND_BRDMAINTP BIT(3) -#define SSS_FCINTPEND_BTDMAINTP BIT(2) -#define SSS_FCINTPEND_HRDMAINTP BIT(1) -#define SSS_FCINTPEND_PKDMAINTP BIT(0) - -#define SSS_REG_FCFIFOSTAT 0x0010 -#define SSS_FCFIFOSTAT_BRFIFOFUL BIT(7) -#define SSS_FCFIFOSTAT_BRFIFOEMP BIT(6) -#define SSS_FCFIFOSTAT_BTFIFOFUL BIT(5) -#define SSS_FCFIFOSTAT_BTFIFOEMP BIT(4) -#define SSS_FCFIFOSTAT_HRFIFOFUL BIT(3) -#define SSS_FCFIFOSTAT_HRFIFOEMP BIT(2) -#define SSS_FCFIFOSTAT_PKFIFOFUL BIT(1) -#define SSS_FCFIFOSTAT_PKFIFOEMP BIT(0) - -#define SSS_REG_FCFIFOCTRL 0x0014 -#define SSS_FCFIFOCTRL_DESSEL BIT(2) -#define SSS_HASHIN_INDEPENDENT _SBF(0, 0x00) -#define SSS_HASHIN_CIPHER_INPUT _SBF(0, 0x01) -#define SSS_HASHIN_CIPHER_OUTPUT _SBF(0, 0x02) - -#define SSS_REG_FCBRDMAS 0x0020 -#define SSS_REG_FCBRDMAL 0x0024 -#define SSS_REG_FCBRDMAC 0x0028 -#define SSS_FCBRDMAC_BYTESWAP BIT(1) -#define SSS_FCBRDMAC_FLUSH BIT(0) - -#define SSS_REG_FCBTDMAS 0x0030 -#define SSS_REG_FCBTDMAL 0x0034 -#define SSS_REG_FCBTDMAC 0x0038 -#define SSS_FCBTDMAC_BYTESWAP BIT(1) -#define SSS_FCBTDMAC_FLUSH BIT(0) - -#define SSS_REG_FCHRDMAS 0x0040 -#define SSS_REG_FCHRDMAL 0x0044 -#define SSS_REG_FCHRDMAC 0x0048 -#define SSS_FCHRDMAC_BYTESWAP BIT(1) -#define SSS_FCHRDMAC_FLUSH BIT(0) - -#define SSS_REG_FCPKDMAS 0x0050 -#define SSS_REG_FCPKDMAL 0x0054 -#define SSS_REG_FCPKDMAC 0x0058 -#define SSS_FCPKDMAC_BYTESWAP BIT(3) -#define SSS_FCPKDMAC_DESCEND BIT(2) -#define SSS_FCPKDMAC_TRANSMIT BIT(1) -#define SSS_FCPKDMAC_FLUSH BIT(0) - -#define SSS_REG_FCPKDMAO 0x005C +#define SSS_REG_FCINTSTAT 0x0000 +#define SSS_FCINTSTAT_HPARTINT BIT(7) +#define SSS_FCINTSTAT_HDONEINT BIT(5) +#define SSS_FCINTSTAT_BRDMAINT BIT(3) +#define SSS_FCINTSTAT_BTDMAINT BIT(2) +#define SSS_FCINTSTAT_HRDMAINT BIT(1) +#define SSS_FCINTSTAT_PKDMAINT BIT(0) + +#define SSS_REG_FCINTENSET 0x0004 +#define SSS_FCINTENSET_HPARTINTENSET BIT(7) +#define SSS_FCINTENSET_HDONEINTENSET BIT(5) +#define SSS_FCINTENSET_BRDMAINTENSET BIT(3) +#define SSS_FCINTENSET_BTDMAINTENSET BIT(2) +#define SSS_FCINTENSET_HRDMAINTENSET BIT(1) +#define SSS_FCINTENSET_PKDMAINTENSET BIT(0) + +#define SSS_REG_FCINTENCLR 0x0008 +#define SSS_FCINTENCLR_HPARTINTENCLR BIT(7) +#define SSS_FCINTENCLR_HDONEINTENCLR BIT(5) +#define SSS_FCINTENCLR_BRDMAINTENCLR BIT(3) +#define SSS_FCINTENCLR_BTDMAINTENCLR BIT(2) +#define SSS_FCINTENCLR_HRDMAINTENCLR BIT(1) +#define SSS_FCINTENCLR_PKDMAINTENCLR BIT(0) + +#define SSS_REG_FCINTPEND 0x000C +#define SSS_FCINTPEND_HPARTINTP BIT(7) +#define SSS_FCINTPEND_HDONEINTP BIT(5) +#define SSS_FCINTPEND_BRDMAINTP BIT(3) +#define SSS_FCINTPEND_BTDMAINTP BIT(2) +#define SSS_FCINTPEND_HRDMAINTP BIT(1) +#define SSS_FCINTPEND_PKDMAINTP BIT(0) + +#define SSS_REG_FCFIFOSTAT 0x0010 +#define SSS_FCFIFOSTAT_BRFIFOFUL BIT(7) +#define SSS_FCFIFOSTAT_BRFIFOEMP BIT(6) +#define SSS_FCFIFOSTAT_BTFIFOFUL BIT(5) +#define SSS_FCFIFOSTAT_BTFIFOEMP BIT(4) +#define SSS_FCFIFOSTAT_HRFIFOFUL BIT(3) +#define SSS_FCFIFOSTAT_HRFIFOEMP BIT(2) +#define SSS_FCFIFOSTAT_PKFIFOFUL BIT(1) +#define SSS_FCFIFOSTAT_PKFIFOEMP BIT(0) + +#define SSS_REG_FCFIFOCTRL 0x0014 +#define SSS_FCFIFOCTRL_DESSEL BIT(2) +#define SSS_HASHIN_INDEPENDENT _SBF(0, 0x00) +#define SSS_HASHIN_CIPHER_INPUT _SBF(0, 0x01) +#define SSS_HASHIN_CIPHER_OUTPUT _SBF(0, 0x02) +#define SSS_HASHIN_MASK _SBF(0, 0x03) + +#define SSS_REG_FCBRDMAS 0x0020 +#define SSS_REG_FCBRDMAL 0x0024 +#define SSS_REG_FCBRDMAC 0x0028 +#define SSS_FCBRDMAC_BYTESWAP BIT(1) +#define SSS_FCBRDMAC_FLUSH BIT(0) + +#define SSS_REG_FCBTDMAS 0x0030 +#define SSS_REG_FCBTDMAL 0x0034 +#define SSS_REG_FCBTDMAC 0x0038 +#define SSS_FCBTDMAC_BYTESWAP BIT(1) +#define SSS_FCBTDMAC_FLUSH BIT(0) + +#define SSS_REG_FCHRDMAS 0x0040 +#define SSS_REG_FCHRDMAL 0x0044 +#define SSS_REG_FCHRDMAC 0x0048 +#define SSS_FCHRDMAC_BYTESWAP BIT(1) +#define SSS_FCHRDMAC_FLUSH BIT(0) + +#define SSS_REG_FCPKDMAS 0x0050 +#define SSS_REG_FCPKDMAL 0x0054 +#define SSS_REG_FCPKDMAC 0x0058 +#define SSS_FCPKDMAC_BYTESWAP BIT(3) +#define SSS_FCPKDMAC_DESCEND BIT(2) +#define SSS_FCPKDMAC_TRANSMIT BIT(1) +#define SSS_FCPKDMAC_FLUSH BIT(0) + +#define SSS_REG_FCPKDMAO 0x005C /* AES registers */ #define SSS_REG_AES_CONTROL 0x00 -#define SSS_AES_BYTESWAP_DI BIT(11) -#define SSS_AES_BYTESWAP_DO BIT(10) -#define SSS_AES_BYTESWAP_IV BIT(9) -#define SSS_AES_BYTESWAP_CNT BIT(8) -#define SSS_AES_BYTESWAP_KEY BIT(7) -#define SSS_AES_KEY_CHANGE_MODE BIT(6) -#define SSS_AES_KEY_SIZE_128 _SBF(4, 0x00) -#define SSS_AES_KEY_SIZE_192 _SBF(4, 0x01) -#define SSS_AES_KEY_SIZE_256 _SBF(4, 0x02) -#define SSS_AES_FIFO_MODE BIT(3) -#define SSS_AES_CHAIN_MODE_ECB _SBF(1, 0x00) -#define SSS_AES_CHAIN_MODE_CBC _SBF(1, 0x01) -#define SSS_AES_CHAIN_MODE_CTR _SBF(1, 0x02) -#define SSS_AES_MODE_DECRYPT BIT(0) +#define SSS_AES_BYTESWAP_DI BIT(11) +#define SSS_AES_BYTESWAP_DO BIT(10) +#define SSS_AES_BYTESWAP_IV BIT(9) +#define SSS_AES_BYTESWAP_CNT BIT(8) +#define SSS_AES_BYTESWAP_KEY BIT(7) +#define SSS_AES_KEY_CHANGE_MODE BIT(6) +#define SSS_AES_KEY_SIZE_128 _SBF(4, 0x00) +#define SSS_AES_KEY_SIZE_192 _SBF(4, 0x01) +#define SSS_AES_KEY_SIZE_256 _SBF(4, 0x02) +#define SSS_AES_FIFO_MODE BIT(3) +#define SSS_AES_CHAIN_MODE_ECB _SBF(1, 0x00) +#define SSS_AES_CHAIN_MODE_CBC _SBF(1, 0x01) +#define SSS_AES_CHAIN_MODE_CTR _SBF(1, 0x02) +#define SSS_AES_MODE_DECRYPT BIT(0) #define SSS_REG_AES_STATUS 0x04 -#define SSS_AES_BUSY BIT(2) -#define SSS_AES_INPUT_READY BIT(1) -#define SSS_AES_OUTPUT_READY BIT(0) +#define SSS_AES_BUSY BIT(2) +#define SSS_AES_INPUT_READY BIT(1) +#define SSS_AES_OUTPUT_READY BIT(0) #define SSS_REG_AES_IN_DATA(s) (0x10 + (s << 2)) #define SSS_REG_AES_OUT_DATA(s) (0x20 + (s << 2)) @@ -129,26 +145,97 @@ #define SSS_REG_AES_CNT_DATA(s) (0x40 + (s << 2)) #define SSS_REG_AES_KEY_DATA(s) (0x80 + (s << 2)) -#define SSS_REG(dev, reg) ((dev)->ioaddr + (SSS_REG_##reg)) -#define SSS_READ(dev, reg) __raw_readl(SSS_REG(dev, reg)) -#define SSS_WRITE(dev, reg, val) __raw_writel((val), SSS_REG(dev, reg)) +#define SSS_REG(dev, reg) ((dev)->ioaddr + (SSS_REG_##reg)) +#define SSS_READ(dev, reg) __raw_readl(SSS_REG(dev, reg)) +#define SSS_WRITE(dev, reg, val) __raw_writel((val), SSS_REG(dev, reg)) -#define SSS_AES_REG(dev, reg) ((dev)->aes_ioaddr + SSS_REG_##reg) +#define SSS_AES_REG(dev, reg) ((dev)->aes_ioaddr + SSS_REG_##reg) #define SSS_AES_WRITE(dev, reg, val) __raw_writel((val), \ SSS_AES_REG(dev, reg)) /* HW engine modes */ -#define FLAGS_AES_DECRYPT BIT(0) -#define FLAGS_AES_MODE_MASK _SBF(1, 0x03) -#define FLAGS_AES_CBC _SBF(1, 0x01) -#define FLAGS_AES_CTR _SBF(1, 0x02) +#define FLAGS_AES_DECRYPT BIT(0) +#define FLAGS_AES_MODE_MASK _SBF(1, 0x03) +#define FLAGS_AES_CBC _SBF(1, 0x01) +#define FLAGS_AES_CTR _SBF(1, 0x02) + +#define AES_KEY_LEN 16 +#define CRYPTO_QUEUE_LEN 1 + +/* HASH registers */ +#define SSS_REG_HASH_CTRL 0x00 + +#define SSS_HASH_USER_IV_EN BIT(5) +#define SSS_HASH_INIT_BIT BIT(4) +#define SSS_HASH_ENGINE_SHA1 _SBF(1, 0x00) +#define SSS_HASH_ENGINE_MD5 _SBF(1, 0x01) +#define SSS_HASH_ENGINE_SHA256 _SBF(1, 0x02) + +#define SSS_HASH_ENGINE_MASK _SBF(1, 0x03) + +#define SSS_REG_HASH_CTRL_PAUSE 0x04 + +#define SSS_HASH_PAUSE BIT(0) + +#define SSS_REG_HASH_CTRL_FIFO 0x08 + +#define SSS_HASH_FIFO_MODE_DMA BIT(0) +#define SSS_HASH_FIFO_MODE_CPU 0 + +#define SSS_REG_HASH_CTRL_SWAP 0x0C + +#define SSS_HASH_BYTESWAP_DI BIT(3) +#define SSS_HASH_BYTESWAP_DO BIT(2) +#define SSS_HASH_BYTESWAP_IV BIT(1) +#define SSS_HASH_BYTESWAP_KEY BIT(0) + +#define SSS_REG_HASH_STATUS 0x10 + +#define SSS_HASH_STATUS_MSG_DONE BIT(6) +#define SSS_HASH_STATUS_PARTIAL_DONE BIT(4) +#define SSS_HASH_STATUS_BUFFER_READY BIT(0) + +#define SSS_REG_HASH_MSG_SIZE_LOW 0x20 +#define SSS_REG_HASH_MSG_SIZE_HIGH 0x24 + +#define SSS_REG_HASH_PRE_MSG_SIZE_LOW 0x28 +#define SSS_REG_HASH_PRE_MSG_SIZE_HIGH 0x2C -#define AES_KEY_LEN 16 -#define CRYPTO_QUEUE_LEN 1 +#define SSS_REG_HASH_IV(s) (0xB0 + ((s) << 2)) +#define SSS_REG_HASH_OUT(s) (0x100 + ((s) << 2)) + +#define HASH_BLOCK_SIZE 64 +#define HASH_REG_SIZEOF 4 +#define HASH_MD5_MAX_REG (MD5_DIGEST_SIZE / HASH_REG_SIZEOF) +#define HASH_SHA1_MAX_REG (SHA1_DIGEST_SIZE / HASH_REG_SIZEOF) +#define HASH_SHA256_MAX_REG (SHA256_DIGEST_SIZE / HASH_REG_SIZEOF) + +/* + * HASH bit numbers, used by device, setting in dev->hash_flags with + * functions set_bit(), clear_bit() or tested with test_bit() or BIT(), + * to keep HASH state BUSY or FREE, or to signal state from irq_handler + * to hash_tasklet. SGS keep track of allocated memory for scatterlist + */ +#define HASH_FLAGS_BUSY 0 +#define HASH_FLAGS_FINAL 1 +#define HASH_FLAGS_DMA_ACTIVE 2 +#define HASH_FLAGS_OUTPUT_READY 3 +#define HASH_FLAGS_DMA_READY 4 +#define HASH_FLAGS_SGS_COPIED 5 +#define HASH_FLAGS_SGS_ALLOCED 6 + +/* HASH HW constants */ +#define BUFLEN HASH_BLOCK_SIZE + +#define SSS_HASH_DMA_LEN_ALIGN 8 +#define SSS_HASH_DMA_ALIGN_MASK (SSS_HASH_DMA_LEN_ALIGN - 1) + +#define SSS_HASH_QUEUE_LENGTH 10 /** * struct samsung_aes_variant - platform specific SSS driver data * @aes_offset: AES register offset from SSS module's base. + * @hash_offset: HASH register offset from SSS module's base. * * Specifies platform specific configuration of SSS module. * Note: A structure for driver specific platform data is used for future @@ -156,6 +243,7 @@ */ struct samsung_aes_variant { unsigned int aes_offset; + unsigned int hash_offset; }; struct s5p_aes_reqctx { @@ -195,6 +283,19 @@ struct s5p_aes_ctx { * protects against concurrent access to these fields. * @lock: Lock for protecting both access to device hardware registers * and fields related to current request (including the busy field). + * @res: Resources for hash. + * @io_hash_base: Per-variant offset for HASH block IO memory. + * @hash_lock: Lock for protecting hash_req, hash_queue and hash_flags + * variable. + * @hash_flags: Flags for current HASH op. + * @hash_queue: Async hash queue. + * @hash_tasklet: New HASH request scheduling job. + * @xmit_buf: Buffer for current HASH request transfer into SSS block. + * @hash_req: Current request sending to SSS HASH block. + * @hash_sg_iter: Scatterlist transferred through DMA into SSS HASH block. + * @hash_sg_cnt: Counter for hash_sg_iter. + * + * @use_hash: true if HASH algs enabled */ struct s5p_aes_dev { struct device *dev; @@ -215,16 +316,83 @@ struct s5p_aes_dev { struct crypto_queue queue; bool busy; spinlock_t lock; + + struct resource *res; + void __iomem *io_hash_base; + + spinlock_t hash_lock; /* protect hash_ vars */ + unsigned long hash_flags; + struct crypto_queue hash_queue; + struct tasklet_struct hash_tasklet; + + u8 xmit_buf[BUFLEN]; + struct ahash_request *hash_req; + struct scatterlist *hash_sg_iter; + unsigned int hash_sg_cnt; + + bool use_hash; }; -static struct s5p_aes_dev *s5p_dev; +/** + * struct s5p_hash_reqctx - HASH request context + * @dd: Associated device + * @op_update: Current request operation (OP_UPDATE or OP_FINAL) + * @digcnt: Number of bytes processed by HW (without buffer[] ones) + * @digest: Digest message or IV for partial result + * @nregs: Number of HW registers for digest or IV read/write + * @engine: Bits for selecting type of HASH in SSS block + * @sg: sg for DMA transfer + * @sg_len: Length of sg for DMA transfer + * @sgl[]: sg for joining buffer and req->src scatterlist + * @skip: Skip offset in req->src for current op + * @total: Total number of bytes for current request + * @finup: Keep state for finup or final. + * @error: Keep track of error. + * @bufcnt: Number of bytes holded in buffer[] + * @buffer[]: For byte(s) from end of req->src in UPDATE op + */ +struct s5p_hash_reqctx { + struct s5p_aes_dev *dd; + bool op_update; + + u64 digcnt; + u8 digest[SHA256_DIGEST_SIZE]; + + unsigned int nregs; /* digest_size / sizeof(reg) */ + u32 engine; + + struct scatterlist *sg; + unsigned int sg_len; + struct scatterlist sgl[2]; + unsigned int skip; + unsigned int total; + bool finup; + bool error; + + u32 bufcnt; + u8 buffer[0]; +}; + +/** + * struct s5p_hash_ctx - HASH transformation context + * @dd: Associated device + * @flags: Bits for algorithm HASH. + * @fallback: Software transformation for zero message or size < BUFLEN. + */ +struct s5p_hash_ctx { + struct s5p_aes_dev *dd; + unsigned long flags; + struct crypto_shash *fallback; +}; static const struct samsung_aes_variant s5p_aes_data = { .aes_offset = 0x4000, + .hash_offset = 0x6000, }; static const struct samsung_aes_variant exynos_aes_data = { .aes_offset = 0x200, + .hash_offset = 0x400, }; static const struct of_device_id s5p_sss_dt_match[] = { @@ -254,6 +422,8 @@ static inline struct samsung_aes_variant *find_s5p_sss_version platform_get_device_id(pdev)->driver_data; } +static struct s5p_aes_dev *s5p_dev; + static void s5p_set_dma_indata(struct s5p_aes_dev *dev, struct scatterlist *sg) { SSS_WRITE(dev, FCBRDMAS, sg_dma_address(sg)); @@ -436,15 +606,65 @@ static int s5p_aes_rx(struct s5p_aes_dev *dev/*, bool *set_dma*/) return ret; } +static inline u32 s5p_hash_read(struct s5p_aes_dev *dd, u32 offset) +{ + return __raw_readl(dd->io_hash_base + offset); +} + +static inline void s5p_hash_write(struct s5p_aes_dev *dd, + u32 offset, u32 value) +{ + __raw_writel(value, dd->io_hash_base + offset); +} + +/** + * s5p_set_dma_hashdata() - start DMA with sg + * @dev: device + * @sg: scatterlist ready to DMA transmit + */ +static void s5p_set_dma_hashdata(struct s5p_aes_dev *dev, + struct scatterlist *sg) +{ + dev->hash_sg_cnt--; + SSS_WRITE(dev, FCHRDMAS, sg_dma_address(sg)); + SSS_WRITE(dev, FCHRDMAL, sg_dma_len(sg)); /* DMA starts */ +} + +/** + * s5p_hash_rx() - get next hash_sg_iter + * @dev: device + * + * Return: + * 2 if there is no more data and it is UPDATE op + * 1 if new receiving (input) data is ready and can be written to device + * 0 if there is no more data and it is FINAL op + */ +static int s5p_hash_rx(struct s5p_aes_dev *dev) +{ + if (dev->hash_sg_cnt > 0) { + dev->hash_sg_iter = sg_next(dev->hash_sg_iter); + return 1; + } + + set_bit(HASH_FLAGS_DMA_READY, &dev->hash_flags); + if (test_bit(HASH_FLAGS_FINAL, &dev->hash_flags)) + return 0; + + return 2; +} + static irqreturn_t s5p_aes_interrupt(int irq, void *dev_id) { struct platform_device *pdev = dev_id; struct s5p_aes_dev *dev = platform_get_drvdata(pdev); int err_dma_tx = 0; int err_dma_rx = 0; + int err_dma_hx = 0; bool tx_end = false; + bool hx_end = false; unsigned long flags; uint32_t status; + u32 st_bits; int err; spin_lock_irqsave(&dev->lock, flags); @@ -456,6 +676,8 @@ static irqreturn_t s5p_aes_interrupt(int irq, void *dev_id) * * If there is no more data in tx scatter list, call s5p_aes_complete() * and schedule new tasklet. + * + * Handle hx interrupt. If there is still data map next entry. */ status = SSS_READ(dev, FCINTSTAT); if (status & SSS_FCINTSTAT_BRDMAINT) @@ -467,7 +689,29 @@ static irqreturn_t s5p_aes_interrupt(int irq, void *dev_id) err_dma_tx = s5p_aes_tx(dev); } - SSS_WRITE(dev, FCINTPEND, status); + if (status & SSS_FCINTSTAT_HRDMAINT) + err_dma_hx = s5p_hash_rx(dev); + + st_bits = status & (SSS_FCINTSTAT_BRDMAINT | SSS_FCINTSTAT_BTDMAINT | + SSS_FCINTSTAT_HRDMAINT); + /* clear DMA bits */ + SSS_WRITE(dev, FCINTPEND, st_bits); + + /* clear HASH irq bits */ + if (status & (SSS_FCINTSTAT_HDONEINT | SSS_FCINTSTAT_HPARTINT)) { + /* cannot have both HPART and HDONE */ + if (status & SSS_FCINTSTAT_HPARTINT) + st_bits = SSS_HASH_STATUS_PARTIAL_DONE; + + if (status & SSS_FCINTSTAT_HDONEINT) + st_bits = SSS_HASH_STATUS_MSG_DONE; + + set_bit(HASH_FLAGS_OUTPUT_READY, &dev->hash_flags); + s5p_hash_write(dev, SSS_REG_HASH_STATUS, st_bits); + hx_end = true; + /* when DONE or PART, do not handle HASH DMA */ + err_dma_hx = 0; + } if (err_dma_rx < 0) { err = err_dma_rx; @@ -480,6 +724,8 @@ static irqreturn_t s5p_aes_interrupt(int irq, void *dev_id) if (tx_end) { s5p_sg_done(dev); + if (err_dma_hx == 1) + s5p_set_dma_hashdata(dev, dev->hash_sg_iter); spin_unlock_irqrestore(&dev->lock, flags); @@ -497,21 +743,1100 @@ static irqreturn_t s5p_aes_interrupt(int irq, void *dev_id) s5p_set_dma_outdata(dev, dev->sg_dst); if (err_dma_rx == 1) s5p_set_dma_indata(dev, dev->sg_src); + if (err_dma_hx == 1) + s5p_set_dma_hashdata(dev, dev->hash_sg_iter); spin_unlock_irqrestore(&dev->lock, flags); } - return IRQ_HANDLED; + goto hash_irq_end; error: s5p_sg_done(dev); dev->busy = false; + if (err_dma_hx == 1) + s5p_set_dma_hashdata(dev, dev->hash_sg_iter); + spin_unlock_irqrestore(&dev->lock, flags); s5p_aes_complete(dev, err); +hash_irq_end: + /* + * Note about else if: + * when hash_sg_iter reaches end and its UPDATE op, + * issue SSS_HASH_PAUSE and wait for HPART irq + */ + if (hx_end) + tasklet_schedule(&dev->hash_tasklet); + else if (err_dma_hx == 2) + s5p_hash_write(dev, SSS_REG_HASH_CTRL_PAUSE, + SSS_HASH_PAUSE); + return IRQ_HANDLED; } +/** + * s5p_hash_read_msg() - read message or IV from HW + * @req: AHASH request + */ +static void s5p_hash_read_msg(struct ahash_request *req) +{ + struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); + struct s5p_aes_dev *dd = ctx->dd; + u32 *hash = (u32 *)ctx->digest; + unsigned int i; + + for (i = 0; i < ctx->nregs; i++) + hash[i] = s5p_hash_read(dd, SSS_REG_HASH_OUT(i)); +} + +/** + * s5p_hash_write_ctx_iv() - write IV for next partial/finup op. + * @dd: device + * @ctx: request context + */ +static void s5p_hash_write_ctx_iv(struct s5p_aes_dev *dd, + struct s5p_hash_reqctx *ctx) +{ + u32 *hash = (u32 *)ctx->digest; + unsigned int i; + + for (i = 0; i < ctx->nregs; i++) + s5p_hash_write(dd, SSS_REG_HASH_IV(i), hash[i]); +} + +/** + * s5p_hash_write_iv() - write IV for next partial/finup op. + * @req: AHASH request + */ +static void s5p_hash_write_iv(struct ahash_request *req) +{ + struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); + + s5p_hash_write_ctx_iv(ctx->dd, ctx); +} + +/** + * s5p_hash_copy_result() - copy digest into req->result + * @req: AHASH request + */ +static void s5p_hash_copy_result(struct ahash_request *req) +{ + struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); + + if (!req->result) + return; + + memcpy(req->result, ctx->digest, ctx->nregs * HASH_REG_SIZEOF); +} + +/** + * s5p_hash_dma_flush() - flush HASH DMA + * @dev: secss device + */ +static void s5p_hash_dma_flush(struct s5p_aes_dev *dev) +{ + SSS_WRITE(dev, FCHRDMAC, SSS_FCHRDMAC_FLUSH); +} + +/** + * s5p_hash_dma_enable() - enable DMA mode for HASH + * @dev: secss device + * + * enable DMA mode for HASH + */ +static void s5p_hash_dma_enable(struct s5p_aes_dev *dev) +{ + s5p_hash_write(dev, SSS_REG_HASH_CTRL_FIFO, SSS_HASH_FIFO_MODE_DMA); +} + +/** + * s5p_hash_irq_disable() - disable irq HASH signals + * @dev: secss device + * @flags: bitfield with irq's to be disabled + */ +static void s5p_hash_irq_disable(struct s5p_aes_dev *dev, u32 flags) +{ + SSS_WRITE(dev, FCINTENCLR, flags); +} + +/** + * s5p_hash_irq_enable() - enable irq signals + * @dev: secss device + * @flags: bitfield with irq's to be enabled + */ +static void s5p_hash_irq_enable(struct s5p_aes_dev *dev, int flags) +{ + SSS_WRITE(dev, FCINTENSET, flags); +} + +/** + * s5p_hash_set_flow() - set flow inside SecSS AES/DES with/without HASH + * @dev: secss device + * @hashflow: HASH stream flow with/without crypto AES/DES + */ +static void s5p_hash_set_flow(struct s5p_aes_dev *dev, u32 hashflow) +{ + unsigned long flags; + u32 flow; + + spin_lock_irqsave(&dev->lock, flags); + + flow = SSS_READ(dev, FCFIFOCTRL); + flow &= ~SSS_HASHIN_MASK; + flow |= hashflow; + SSS_WRITE(dev, FCFIFOCTRL, flow); + + spin_unlock_irqrestore(&dev->lock, flags); +} + +/** + * s5p_ahash_dma_init() - enable DMA and set HASH flow inside SecSS + * @dev: secss device + * @hashflow: HASH stream flow with/without AES/DES + * + * flush HASH DMA and enable DMA, set HASH stream flow inside SecSS HW, + * enable HASH irq's HRDMA, HDONE, HPART + */ +static void s5p_ahash_dma_init(struct s5p_aes_dev *dev, u32 hashflow) +{ + s5p_hash_irq_disable(dev, SSS_FCINTENCLR_HRDMAINTENCLR | + SSS_FCINTENCLR_HDONEINTENCLR | + SSS_FCINTENCLR_HPARTINTENCLR); + s5p_hash_dma_flush(dev); + + s5p_hash_dma_enable(dev); + s5p_hash_set_flow(dev, hashflow & SSS_HASHIN_MASK); + s5p_hash_irq_enable(dev, SSS_FCINTENSET_HRDMAINTENSET | + SSS_FCINTENSET_HDONEINTENSET | + SSS_FCINTENSET_HPARTINTENSET); +} + +/** + * s5p_hash_write_ctrl() - prepare HASH block in SecSS for processing + * @dd: secss device + * @length: length for request + * @final: true if final op + * + * Prepare SSS HASH block for processing bytes in DMA mode. If it is called + * after previous updates, fill up IV words. For final, calculate and set + * lengths for HASH so SecSS can finalize hash. For partial, set SSS HASH + * length as 2^63 so it will be never reached and set to zero prelow and + * prehigh. + * + * This function does not start DMA transfer. + */ +static void s5p_hash_write_ctrl(struct s5p_aes_dev *dd, size_t length, + bool final) +{ + struct s5p_hash_reqctx *ctx = ahash_request_ctx(dd->hash_req); + u32 prelow, prehigh, low, high; + u32 configflags, swapflags; + u64 tmplen; + + configflags = ctx->engine | SSS_HASH_INIT_BIT; + + if (likely(ctx->digcnt)) { + s5p_hash_write_ctx_iv(dd, ctx); + configflags |= SSS_HASH_USER_IV_EN; + } + + if (final) { + /* number of bytes for last part */ + low = length; + high = 0; + /* total number of bits prev hashed */ + tmplen = ctx->digcnt * 8; + prelow = (u32)tmplen; + prehigh = (u32)(tmplen >> 32); + } else { + prelow = 0; + prehigh = 0; + low = 0; + high = BIT(31); + } + + swapflags = SSS_HASH_BYTESWAP_DI | SSS_HASH_BYTESWAP_DO | + SSS_HASH_BYTESWAP_IV | SSS_HASH_BYTESWAP_KEY; + + s5p_hash_write(dd, SSS_REG_HASH_MSG_SIZE_LOW, low); + s5p_hash_write(dd, SSS_REG_HASH_MSG_SIZE_HIGH, high); + s5p_hash_write(dd, SSS_REG_HASH_PRE_MSG_SIZE_LOW, prelow); + s5p_hash_write(dd, SSS_REG_HASH_PRE_MSG_SIZE_HIGH, prehigh); + + s5p_hash_write(dd, SSS_REG_HASH_CTRL_SWAP, swapflags); + s5p_hash_write(dd, SSS_REG_HASH_CTRL, configflags); +} + +/** + * s5p_hash_xmit_dma() - start DMA hash processing + * @dd: secss device + * @length: length for request + * @final: true if final op + * + * Update digcnt here, as it is needed for finup/final op. + */ +static int s5p_hash_xmit_dma(struct s5p_aes_dev *dd, size_t length, + bool final) +{ + struct s5p_hash_reqctx *ctx = ahash_request_ctx(dd->hash_req); + unsigned int cnt; + + cnt = dma_map_sg(dd->dev, ctx->sg, ctx->sg_len, DMA_TO_DEVICE); + if (!cnt) { + dev_err(dd->dev, "dma_map_sg error\n"); + ctx->error = true; + return -EINVAL; + } + + set_bit(HASH_FLAGS_DMA_ACTIVE, &dd->hash_flags); + dd->hash_sg_iter = ctx->sg; + dd->hash_sg_cnt = cnt; + s5p_hash_write_ctrl(dd, length, final); + ctx->digcnt += length; + ctx->total -= length; + + /* catch last interrupt */ + if (final) + set_bit(HASH_FLAGS_FINAL, &dd->hash_flags); + + s5p_set_dma_hashdata(dd, dd->hash_sg_iter); /* DMA starts */ + + return -EINPROGRESS; +} + +/** + * s5p_hash_copy_sgs() - copy request's bytes into new buffer + * @ctx: request context + * @sg: source scatterlist request + * @new_len: number of bytes to process from sg + * + * Allocate new buffer, copy data for HASH into it. If there was xmit_buf + * filled, copy it first, then copy data from sg into it. Prepare one sgl[0] + * with allocated buffer. + * + * Set bit in dd->hash_flag so we can free it after irq ends processing. + */ +static int s5p_hash_copy_sgs(struct s5p_hash_reqctx *ctx, + struct scatterlist *sg, unsigned int new_len) +{ + unsigned int pages, len; + void *buf; + + len = new_len + ctx->bufcnt; + pages = get_order(len); + + buf = (void *)__get_free_pages(GFP_ATOMIC, pages); + if (!buf) { + dev_err(ctx->dd->dev, "alloc pages for unaligned case.\n"); + ctx->error = true; + return -ENOMEM; + } + + if (ctx->bufcnt) + memcpy(buf, ctx->dd->xmit_buf, ctx->bufcnt); + + scatterwalk_map_and_copy(buf + ctx->bufcnt, sg, ctx->skip, + new_len, 0); + sg_init_table(ctx->sgl, 1); + sg_set_buf(ctx->sgl, buf, len); + ctx->sg = ctx->sgl; + ctx->sg_len = 1; + ctx->bufcnt = 0; + ctx->skip = 0; + set_bit(HASH_FLAGS_SGS_COPIED, &ctx->dd->hash_flags); + + return 0; +} + +/** + * s5p_hash_copy_sg_lists() - copy sg list and make fixes in copy + * @ctx: request context + * @sg: source scatterlist request + * @new_len: number of bytes to process from sg + * + * Allocate new scatterlist table, copy data for HASH into it. If there was + * xmit_buf filled, prepare it first, then copy page, length and offset from + * source sg into it, adjusting begin and/or end for skip offset and + * hash_later value. + * + * Resulting sg table will be assigned to ctx->sg. Set flag so we can free + * it after irq ends processing. + */ +static int s5p_hash_copy_sg_lists(struct s5p_hash_reqctx *ctx, + struct scatterlist *sg, unsigned int new_len) +{ + unsigned int skip = ctx->skip, n = sg_nents(sg); + struct scatterlist *tmp; + unsigned int len; + + if (ctx->bufcnt) + n++; + + ctx->sg = kmalloc_array(n, sizeof(*sg), GFP_KERNEL); + if (!ctx->sg) { + ctx->error = true; + return -ENOMEM; + } + + sg_init_table(ctx->sg, n); + + tmp = ctx->sg; + + ctx->sg_len = 0; + + if (ctx->bufcnt) { + sg_set_buf(tmp, ctx->dd->xmit_buf, ctx->bufcnt); + tmp = sg_next(tmp); + ctx->sg_len++; + } + + while (sg && skip >= sg->length) { + skip -= sg->length; + sg = sg_next(sg); + } + + while (sg && new_len) { + len = sg->length - skip; + if (new_len < len) + len = new_len; + + new_len -= len; + sg_set_page(tmp, sg_page(sg), len, sg->offset + skip); + skip = 0; + if (new_len <= 0) + sg_mark_end(tmp); + + tmp = sg_next(tmp); + ctx->sg_len++; + sg = sg_next(sg); + } + + set_bit(HASH_FLAGS_SGS_ALLOCED, &ctx->dd->hash_flags); + + return 0; +} + +/** + * s5p_hash_prepare_sgs() - prepare sg for processing + * @ctx: request context + * @sg: source scatterlist request + * @nbytes: number of bytes to process from sg + * @final: final flag + * + * Check two conditions: (1) if buffers in sg have len aligned data, and (2) + * sg table have good aligned elements (list_ok). If one of this checks fails, + * then either (1) allocates new buffer for data with s5p_hash_copy_sgs, copy + * data into this buffer and prepare request in sgl, or (2) allocates new sg + * table and prepare sg elements. + * + * For digest or finup all conditions can be good, and we may not need any + * fixes. + */ +static int s5p_hash_prepare_sgs(struct s5p_hash_reqctx *ctx, + struct scatterlist *sg, + unsigned int new_len, bool final) +{ + unsigned int skip = ctx->skip, nbytes = new_len, n = 0; + bool aligned = true, list_ok = true; + struct scatterlist *sg_tmp = sg; + + if (!sg || !sg->length || !new_len) + return 0; + + if (skip || !final) + list_ok = false; + + while (nbytes > 0 && sg_tmp) { + n++; + if (skip >= sg_tmp->length) { + skip -= sg_tmp->length; + if (!sg_tmp->length) { + aligned = false; + break; + } + } else { + if (!IS_ALIGNED(sg_tmp->length - skip, BUFLEN)) { + aligned = false; + break; + } + + if (nbytes < sg_tmp->length - skip) { + list_ok = false; + break; + } + + nbytes -= sg_tmp->length - skip; + skip = 0; + } + + sg_tmp = sg_next(sg_tmp); + } + + if (!aligned) + return s5p_hash_copy_sgs(ctx, sg, new_len); + else if (!list_ok) + return s5p_hash_copy_sg_lists(ctx, sg, new_len); + + /* + * Have aligned data from previous operation and/or current + * Note: will enter here only if (digest or finup) and aligned + */ + if (ctx->bufcnt) { + ctx->sg_len = n; + sg_init_table(ctx->sgl, 2); + sg_set_buf(ctx->sgl, ctx->dd->xmit_buf, ctx->bufcnt); + sg_chain(ctx->sgl, 2, sg); + ctx->sg = ctx->sgl; + ctx->sg_len++; + } else { + ctx->sg = sg; + ctx->sg_len = n; + } + + return 0; +} + +/** + * s5p_hash_prepare_request() - prepare request for processing + * @req: AHASH request + * @update: true if UPDATE op + * + * Note 1: we can have update flag _and_ final flag at the same time. + * Note 2: we enter here when digcnt > BUFLEN (=HASH_BLOCK_SIZE) or + * either req->nbytes or ctx->bufcnt + req->nbytes is > BUFLEN or + * we have final op + */ +static int s5p_hash_prepare_request(struct ahash_request *req, bool update) +{ + struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); + bool final = ctx->finup; + int xmit_len, hash_later, nbytes; + int ret; + + if (!req) + return 0; + + if (update) + nbytes = req->nbytes; + else + nbytes = 0; + + ctx->total = nbytes + ctx->bufcnt; + if (!ctx->total) + return 0; + + if (nbytes && (!IS_ALIGNED(ctx->bufcnt, BUFLEN))) { + /* bytes left from previous request, so fill up to BUFLEN */ + int len = BUFLEN - ctx->bufcnt % BUFLEN; + + if (len > nbytes) + len = nbytes; + + scatterwalk_map_and_copy(ctx->buffer + ctx->bufcnt, req->src, + 0, len, 0); + ctx->bufcnt += len; + nbytes -= len; + ctx->skip = len; + } else { + ctx->skip = 0; + } + + if (ctx->bufcnt) + memcpy(ctx->dd->xmit_buf, ctx->buffer, ctx->bufcnt); + + xmit_len = ctx->total; + if (final) { + hash_later = 0; + } else { + if (IS_ALIGNED(xmit_len, BUFLEN)) + xmit_len -= BUFLEN; + else + xmit_len -= xmit_len & (BUFLEN - 1); + + hash_later = ctx->total - xmit_len; + /* copy hash_later bytes from end of req->src */ + /* previous bytes are in xmit_buf, so no overwrite */ + scatterwalk_map_and_copy(ctx->buffer, req->src, + req->nbytes - hash_later, + hash_later, 0); + } + + if (xmit_len > BUFLEN) { + ret = s5p_hash_prepare_sgs(ctx, req->src, nbytes - hash_later, + final); + if (ret) + return ret; + } else { + /* have buffered data only */ + if (unlikely(!ctx->bufcnt)) { + /* first update didn't fill up buffer */ + scatterwalk_map_and_copy(ctx->dd->xmit_buf, req->src, + 0, xmit_len, 0); + } + + sg_init_table(ctx->sgl, 1); + sg_set_buf(ctx->sgl, ctx->dd->xmit_buf, xmit_len); + + ctx->sg = ctx->sgl; + ctx->sg_len = 1; + } + + ctx->bufcnt = hash_later; + if (!final) + ctx->total = xmit_len; + + return 0; +} + +/** + * s5p_hash_update_dma_stop() - unmap DMA + * @dd: secss device + * + * Unmap scatterlist ctx->sg. + */ +static void s5p_hash_update_dma_stop(struct s5p_aes_dev *dd) +{ + struct s5p_hash_reqctx *ctx = ahash_request_ctx(dd->hash_req); + + dma_unmap_sg(dd->dev, ctx->sg, ctx->sg_len, DMA_TO_DEVICE); + clear_bit(HASH_FLAGS_DMA_ACTIVE, &dd->hash_flags); +} + +/** + * s5p_hash_finish() - copy calculated digest to crypto layer + * @req: AHASH request + */ +static void s5p_hash_finish(struct ahash_request *req) +{ + struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); + struct s5p_aes_dev *dd = ctx->dd; + + if (ctx->digcnt) + s5p_hash_copy_result(req); + + dev_dbg(dd->dev, "hash_finish digcnt: %lld\n", ctx->digcnt); +} + +/** + * s5p_hash_finish_req() - finish request + * @req: AHASH request + * @err: error + */ +static void s5p_hash_finish_req(struct ahash_request *req, int err) +{ + struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); + struct s5p_aes_dev *dd = ctx->dd; + unsigned long flags; + + if (test_bit(HASH_FLAGS_SGS_COPIED, &dd->hash_flags)) + free_pages((unsigned long)sg_virt(ctx->sg), + get_order(ctx->sg->length)); + + if (test_bit(HASH_FLAGS_SGS_ALLOCED, &dd->hash_flags)) + kfree(ctx->sg); + + ctx->sg = NULL; + dd->hash_flags &= ~(BIT(HASH_FLAGS_SGS_ALLOCED) | + BIT(HASH_FLAGS_SGS_COPIED)); + + if (!err && !ctx->error) { + s5p_hash_read_msg(req); + if (test_bit(HASH_FLAGS_FINAL, &dd->hash_flags)) + s5p_hash_finish(req); + } else { + ctx->error = true; + } + + spin_lock_irqsave(&dd->hash_lock, flags); + dd->hash_flags &= ~(BIT(HASH_FLAGS_BUSY) | BIT(HASH_FLAGS_FINAL) | + BIT(HASH_FLAGS_DMA_READY) | + BIT(HASH_FLAGS_OUTPUT_READY)); + spin_unlock_irqrestore(&dd->hash_lock, flags); + + if (req->base.complete) + req->base.complete(&req->base, err); +} + +/** + * s5p_hash_handle_queue() - handle hash queue + * @dd: device s5p_aes_dev + * @req: AHASH request + * + * If req!=NULL enqueue it on dd->queue, if FLAGS_BUSY is not set on the + * device then processes the first request from the dd->queue + * + * Returns: see s5p_hash_final below. + */ +static int s5p_hash_handle_queue(struct s5p_aes_dev *dd, + struct ahash_request *req) +{ + struct crypto_async_request *async_req, *backlog; + struct s5p_hash_reqctx *ctx; + unsigned long flags; + int err = 0, ret = 0; + +retry: + spin_lock_irqsave(&dd->hash_lock, flags); + if (req) + ret = ahash_enqueue_request(&dd->hash_queue, req); + + if (test_bit(HASH_FLAGS_BUSY, &dd->hash_flags)) { + spin_unlock_irqrestore(&dd->hash_lock, flags); + return ret; + } + + backlog = crypto_get_backlog(&dd->hash_queue); + async_req = crypto_dequeue_request(&dd->hash_queue); + if (async_req) + set_bit(HASH_FLAGS_BUSY, &dd->hash_flags); + + spin_unlock_irqrestore(&dd->hash_lock, flags); + + if (!async_req) + return ret; + + if (backlog) + backlog->complete(backlog, -EINPROGRESS); + + req = ahash_request_cast(async_req); + dd->hash_req = req; + ctx = ahash_request_ctx(req); + + err = s5p_hash_prepare_request(req, ctx->op_update); + if (err || !ctx->total) + goto out; + + dev_dbg(dd->dev, "handling new req, op_update: %u, nbytes: %d\n", + ctx->op_update, req->nbytes); + + s5p_ahash_dma_init(dd, SSS_HASHIN_INDEPENDENT); + if (ctx->digcnt) + s5p_hash_write_iv(req); /* restore hash IV */ + + if (ctx->op_update) { /* HASH_OP_UPDATE */ + err = s5p_hash_xmit_dma(dd, ctx->total, ctx->finup); + if (err != -EINPROGRESS && ctx->finup && !ctx->error) + /* no final() after finup() */ + err = s5p_hash_xmit_dma(dd, ctx->total, true); + } else { /* HASH_OP_FINAL */ + err = s5p_hash_xmit_dma(dd, ctx->total, true); + } +out: + if (err != -EINPROGRESS) { + /* hash_tasklet_cb will not finish it, so do it here */ + s5p_hash_finish_req(req, err); + req = NULL; + + /* + * Execute next request immediately if there is anything + * in queue. + */ + goto retry; + } + + return ret; +} + +/** + * s5p_hash_tasklet_cb() - hash tasklet + * @data: ptr to s5p_aes_dev + */ +static void s5p_hash_tasklet_cb(unsigned long data) +{ + struct s5p_aes_dev *dd = (struct s5p_aes_dev *)data; + + if (!test_bit(HASH_FLAGS_BUSY, &dd->hash_flags)) { + s5p_hash_handle_queue(dd, NULL); + return; + } + + if (test_bit(HASH_FLAGS_DMA_READY, &dd->hash_flags)) { + if (test_and_clear_bit(HASH_FLAGS_DMA_ACTIVE, + &dd->hash_flags)) { + s5p_hash_update_dma_stop(dd); + } + + if (test_and_clear_bit(HASH_FLAGS_OUTPUT_READY, + &dd->hash_flags)) { + /* hash or semi-hash ready */ + clear_bit(HASH_FLAGS_DMA_READY, &dd->hash_flags); + goto finish; + } + } + + return; + +finish: + /* finish curent request */ + s5p_hash_finish_req(dd->hash_req, 0); + + /* If we are not busy, process next req */ + if (!test_bit(HASH_FLAGS_BUSY, &dd->hash_flags)) + s5p_hash_handle_queue(dd, NULL); +} + +/** + * s5p_hash_enqueue() - enqueue request + * @req: AHASH request + * @op: operation UPDATE (true) or FINAL (false) + * + * Returns: see s5p_hash_final below. + */ +static int s5p_hash_enqueue(struct ahash_request *req, bool op) +{ + struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); + struct s5p_hash_ctx *tctx = crypto_tfm_ctx(req->base.tfm); + + ctx->op_update = op; + + return s5p_hash_handle_queue(tctx->dd, req); +} + +/** + * s5p_hash_update() - process the hash input data + * @req: AHASH request + * + * If request will fit in buffer, copy it and return immediately + * else enqueue it with OP_UPDATE. + * + * Returns: see s5p_hash_final below. + */ +static int s5p_hash_update(struct ahash_request *req) +{ + struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); + + if (!req->nbytes) + return 0; + + if (ctx->bufcnt + req->nbytes <= BUFLEN) { + scatterwalk_map_and_copy(ctx->buffer + ctx->bufcnt, req->src, + 0, req->nbytes, 0); + ctx->bufcnt += req->nbytes; + return 0; + } + + return s5p_hash_enqueue(req, true); /* HASH_OP_UPDATE */ +} + +/** + * s5p_hash_shash_digest() - calculate shash digest + * @tfm: crypto transformation + * @flags: tfm flags + * @data: input data + * @len: length of data + * @out: output buffer + */ +static int s5p_hash_shash_digest(struct crypto_shash *tfm, u32 flags, + const u8 *data, unsigned int len, u8 *out) +{ + SHASH_DESC_ON_STACK(shash, tfm); + + shash->tfm = tfm; + shash->flags = flags & ~CRYPTO_TFM_REQ_MAY_SLEEP; + + return crypto_shash_digest(shash, data, len, out); +} + +/** + * s5p_hash_final_shash() - calculate shash digest + * @req: AHASH request + */ +static int s5p_hash_final_shash(struct ahash_request *req) +{ + struct s5p_hash_ctx *tctx = crypto_tfm_ctx(req->base.tfm); + struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); + + return s5p_hash_shash_digest(tctx->fallback, req->base.flags, + ctx->buffer, ctx->bufcnt, req->result); +} + +/** + * s5p_hash_final() - close up hash and calculate digest + * @req: AHASH request + * + * Note: in final req->src do not have any data, and req->nbytes can be + * non-zero. + * + * If there were no input data processed yet and the buffered hash data is + * less than BUFLEN (64) then calculate the final hash immediately by using + * SW algorithm fallback. + * + * Otherwise enqueues the current AHASH request with OP_FINAL operation op + * and finalize hash message in HW. Note that if digcnt!=0 then there were + * previous update op, so there are always some buffered bytes in ctx->buffer, + * which means that ctx->bufcnt!=0 + * + * Returns: + * 0 if the request has been processed immediately, + * -EINPROGRESS if the operation has been queued for later execution or is set + * to processing by HW, + * -EBUSY if queue is full and request should be resubmitted later, + * other negative values denotes an error. + */ +static int s5p_hash_final(struct ahash_request *req) +{ + struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); + + ctx->finup = true; + if (ctx->error) + return -EINVAL; /* uncompleted hash is not needed */ + + if (!ctx->digcnt && ctx->bufcnt < BUFLEN) + return s5p_hash_final_shash(req); + + return s5p_hash_enqueue(req, false); /* HASH_OP_FINAL */ +} + +/** + * s5p_hash_finup() - process last req->src and calculate digest + * @req: AHASH request containing the last update data + * + * Return values: see s5p_hash_final above. + */ +static int s5p_hash_finup(struct ahash_request *req) +{ + struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); + int err1, err2; + + ctx->finup = true; + + err1 = s5p_hash_update(req); + if (err1 == -EINPROGRESS || err1 == -EBUSY) + return err1; + + /* + * final() has to be always called to cleanup resources even if + * update() failed, except EINPROGRESS or calculate digest for small + * size + */ + err2 = s5p_hash_final(req); + + return err1 ?: err2; +} + +/** + * s5p_hash_init() - initialize AHASH request contex + * @req: AHASH request + * + * Init async hash request context. + */ +static int s5p_hash_init(struct ahash_request *req) +{ + struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct s5p_hash_ctx *tctx = crypto_ahash_ctx(tfm); + + ctx->dd = tctx->dd; + ctx->error = false; + ctx->finup = false; + ctx->bufcnt = 0; + ctx->digcnt = 0; + ctx->total = 0; + ctx->skip = 0; + + dev_dbg(tctx->dd->dev, "init: digest size: %d\n", + crypto_ahash_digestsize(tfm)); + + switch (crypto_ahash_digestsize(tfm)) { + case MD5_DIGEST_SIZE: + ctx->engine = SSS_HASH_ENGINE_MD5; + ctx->nregs = HASH_MD5_MAX_REG; + break; + case SHA1_DIGEST_SIZE: + ctx->engine = SSS_HASH_ENGINE_SHA1; + ctx->nregs = HASH_SHA1_MAX_REG; + break; + case SHA256_DIGEST_SIZE: + ctx->engine = SSS_HASH_ENGINE_SHA256; + ctx->nregs = HASH_SHA256_MAX_REG; + break; + default: + ctx->error = true; + return -EINVAL; + } + + return 0; +} + +/** + * s5p_hash_digest - calculate digest from req->src + * @req: AHASH request + * + * Return values: see s5p_hash_final above. + */ +static int s5p_hash_digest(struct ahash_request *req) +{ + return s5p_hash_init(req) ?: s5p_hash_finup(req); +} + +/** + * s5p_hash_cra_init_alg - init crypto alg transformation + * @tfm: crypto transformation + */ +static int s5p_hash_cra_init_alg(struct crypto_tfm *tfm) +{ + struct s5p_hash_ctx *tctx = crypto_tfm_ctx(tfm); + const char *alg_name = crypto_tfm_alg_name(tfm); + + tctx->dd = s5p_dev; + /* Allocate a fallback and abort if it failed. */ + tctx->fallback = crypto_alloc_shash(alg_name, 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(tctx->fallback)) { + pr_err("fallback alloc fails for '%s'\n", alg_name); + return PTR_ERR(tctx->fallback); + } + + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct s5p_hash_reqctx) + BUFLEN); + + return 0; +} + +/** + * s5p_hash_cra_init - init crypto tfm + * @tfm: crypto transformation + */ +static int s5p_hash_cra_init(struct crypto_tfm *tfm) +{ + return s5p_hash_cra_init_alg(tfm); +} + +/** + * s5p_hash_cra_exit - exit crypto tfm + * @tfm: crypto transformation + * + * free allocated fallback + */ +static void s5p_hash_cra_exit(struct crypto_tfm *tfm) +{ + struct s5p_hash_ctx *tctx = crypto_tfm_ctx(tfm); + + crypto_free_shash(tctx->fallback); + tctx->fallback = NULL; +} + +/** + * s5p_hash_export - export hash state + * @req: AHASH request + * @out: buffer for exported state + */ +static int s5p_hash_export(struct ahash_request *req, void *out) +{ + struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); + + memcpy(out, ctx, sizeof(*ctx) + ctx->bufcnt); + + return 0; +} + +/** + * s5p_hash_import - import hash state + * @req: AHASH request + * @in: buffer with state to be imported from + */ +static int s5p_hash_import(struct ahash_request *req, const void *in) +{ + struct s5p_hash_reqctx *ctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct s5p_hash_ctx *tctx = crypto_ahash_ctx(tfm); + const struct s5p_hash_reqctx *ctx_in = in; + + memcpy(ctx, in, sizeof(*ctx) + BUFLEN); + if (ctx_in->bufcnt > BUFLEN) { + ctx->error = true; + return -EINVAL; + } + + ctx->dd = tctx->dd; + ctx->error = false; + + return 0; +} + +static struct ahash_alg algs_sha1_md5_sha256[] = { +{ + .init = s5p_hash_init, + .update = s5p_hash_update, + .final = s5p_hash_final, + .finup = s5p_hash_finup, + .digest = s5p_hash_digest, + .export = s5p_hash_export, + .import = s5p_hash_import, + .halg.statesize = sizeof(struct s5p_hash_reqctx) + BUFLEN, + .halg.digestsize = SHA1_DIGEST_SIZE, + .halg.base = { + .cra_name = "sha1", + .cra_driver_name = "exynos-sha1", + .cra_priority = 100, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = HASH_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct s5p_hash_ctx), + .cra_alignmask = SSS_HASH_DMA_ALIGN_MASK, + .cra_module = THIS_MODULE, + .cra_init = s5p_hash_cra_init, + .cra_exit = s5p_hash_cra_exit, + } +}, +{ + .init = s5p_hash_init, + .update = s5p_hash_update, + .final = s5p_hash_final, + .finup = s5p_hash_finup, + .digest = s5p_hash_digest, + .export = s5p_hash_export, + .import = s5p_hash_import, + .halg.statesize = sizeof(struct s5p_hash_reqctx) + BUFLEN, + .halg.digestsize = MD5_DIGEST_SIZE, + .halg.base = { + .cra_name = "md5", + .cra_driver_name = "exynos-md5", + .cra_priority = 100, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = HASH_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct s5p_hash_ctx), + .cra_alignmask = SSS_HASH_DMA_ALIGN_MASK, + .cra_module = THIS_MODULE, + .cra_init = s5p_hash_cra_init, + .cra_exit = s5p_hash_cra_exit, + } +}, +{ + .init = s5p_hash_init, + .update = s5p_hash_update, + .final = s5p_hash_final, + .finup = s5p_hash_finup, + .digest = s5p_hash_digest, + .export = s5p_hash_export, + .import = s5p_hash_import, + .halg.statesize = sizeof(struct s5p_hash_reqctx) + BUFLEN, + .halg.digestsize = SHA256_DIGEST_SIZE, + .halg.base = { + .cra_name = "sha256", + .cra_driver_name = "exynos-sha256", + .cra_priority = 100, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = HASH_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct s5p_hash_ctx), + .cra_alignmask = SSS_HASH_DMA_ALIGN_MASK, + .cra_module = THIS_MODULE, + .cra_init = s5p_hash_cra_init, + .cra_exit = s5p_hash_cra_exit, + } +} + +}; + static void s5p_set_aes(struct s5p_aes_dev *dev, uint8_t *key, uint8_t *iv, unsigned int keylen) { @@ -829,6 +2154,7 @@ static int s5p_aes_probe(struct platform_device *pdev) struct samsung_aes_variant *variant; struct s5p_aes_dev *pdata; struct resource *res; + unsigned int hash_i; if (s5p_dev) return -EEXIST; @@ -837,12 +2163,34 @@ static int s5p_aes_probe(struct platform_device *pdev) if (!pdata) return -ENOMEM; + variant = find_s5p_sss_version(pdev); res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - pdata->ioaddr = devm_ioremap_resource(&pdev->dev, res); - if (IS_ERR(pdata->ioaddr)) - return PTR_ERR(pdata->ioaddr); - variant = find_s5p_sss_version(pdev); + /* + * Note: HASH and PRNG uses the same registers in secss, avoid + * overwrite each other. This will drop HASH when CONFIG_EXYNOS_RNG + * is enabled in config. We need larger size for HASH registers in + * secss, current describe only AES/DES + */ + if (IS_ENABLED(CONFIG_CRYPTO_DEV_EXYNOS_HASH)) { + if (variant == &exynos_aes_data) { + res->end += 0x300; + pdata->use_hash = true; + } + } + + pdata->res = res; + pdata->ioaddr = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(pdata->ioaddr)) { + if (!pdata->use_hash) + return PTR_ERR(pdata->ioaddr); + /* try AES without HASH */ + res->end -= 0x300; + pdata->use_hash = false; + pdata->ioaddr = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(pdata->ioaddr)) + return PTR_ERR(pdata->ioaddr); + } pdata->clk = devm_clk_get(dev, "secss"); if (IS_ERR(pdata->clk)) { @@ -857,8 +2205,10 @@ static int s5p_aes_probe(struct platform_device *pdev) } spin_lock_init(&pdata->lock); + spin_lock_init(&pdata->hash_lock); pdata->aes_ioaddr = pdata->ioaddr + variant->aes_offset; + pdata->io_hash_base = pdata->ioaddr + variant->hash_offset; pdata->irq_fc = platform_get_irq(pdev, 0); if (pdata->irq_fc < 0) { @@ -888,12 +2238,40 @@ static int s5p_aes_probe(struct platform_device *pdev) goto err_algs; } + if (pdata->use_hash) { + tasklet_init(&pdata->hash_tasklet, s5p_hash_tasklet_cb, + (unsigned long)pdata); + crypto_init_queue(&pdata->hash_queue, SSS_HASH_QUEUE_LENGTH); + + for (hash_i = 0; hash_i < ARRAY_SIZE(algs_sha1_md5_sha256); + hash_i++) { + struct ahash_alg *alg; + + alg = &algs_sha1_md5_sha256[hash_i]; + err = crypto_register_ahash(alg); + if (err) { + dev_err(dev, "can't register '%s': %d\n", + alg->halg.base.cra_driver_name, err); + goto err_hash; + } + } + } + dev_info(dev, "s5p-sss driver registered\n"); return 0; +err_hash: + for (j = hash_i - 1; j >= 0; j--) + crypto_unregister_ahash(&algs_sha1_md5_sha256[j]); + + tasklet_kill(&pdata->hash_tasklet); + res->end -= 0x300; + err_algs: - dev_err(dev, "can't register '%s': %d\n", algs[i].cra_name, err); + if (i < ARRAY_SIZE(algs)) + dev_err(dev, "can't register '%s': %d\n", algs[i].cra_name, + err); for (j = 0; j < i; j++) crypto_unregister_alg(&algs[j]); @@ -920,9 +2298,16 @@ static int s5p_aes_remove(struct platform_device *pdev) crypto_unregister_alg(&algs[i]); tasklet_kill(&pdata->tasklet); + if (pdata->use_hash) { + for (i = ARRAY_SIZE(algs_sha1_md5_sha256) - 1; i >= 0; i--) + crypto_unregister_ahash(&algs_sha1_md5_sha256[i]); - clk_disable_unprepare(pdata->clk); + pdata->res->end -= 0x300; + tasklet_kill(&pdata->hash_tasklet); + pdata->use_hash = false; + } + clk_disable_unprepare(pdata->clk); s5p_dev = NULL; return 0; @@ -942,3 +2327,4 @@ module_platform_driver(s5p_aes_crypto); MODULE_DESCRIPTION("S5PV210 AES hw acceleration support."); MODULE_LICENSE("GPL v2"); MODULE_AUTHOR("Vladimir Zapolskiy <vzapolskiy@gmail.com>"); +MODULE_AUTHOR("Kamil Konieczny <k.konieczny@partner.samsung.com>"); diff --git a/drivers/crypto/sahara.c b/drivers/crypto/sahara.c index 1d9ecd368b5b..08e7bdcaa6e3 100644 --- a/drivers/crypto/sahara.c +++ b/drivers/crypto/sahara.c @@ -202,7 +202,6 @@ struct sahara_dev { struct completion dma_completion; struct sahara_ctx *ctx; - spinlock_t lock; struct crypto_queue queue; unsigned long flags; @@ -543,10 +542,10 @@ static int sahara_hw_descriptor_create(struct sahara_dev *dev) unmap_out: dma_unmap_sg(dev->device, dev->out_sg, dev->nb_out_sg, - DMA_TO_DEVICE); + DMA_FROM_DEVICE); unmap_in: dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg, - DMA_FROM_DEVICE); + DMA_TO_DEVICE); return -EINVAL; } @@ -594,9 +593,9 @@ static int sahara_aes_process(struct ablkcipher_request *req) } dma_unmap_sg(dev->device, dev->out_sg, dev->nb_out_sg, - DMA_TO_DEVICE); - dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg, DMA_FROM_DEVICE); + dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg, + DMA_TO_DEVICE); return 0; } @@ -1376,13 +1375,13 @@ static void sahara_unregister_algs(struct sahara_dev *dev) crypto_unregister_ahash(&sha_v4_algs[i]); } -static struct platform_device_id sahara_platform_ids[] = { +static const struct platform_device_id sahara_platform_ids[] = { { .name = "sahara-imx27" }, { /* sentinel */ } }; MODULE_DEVICE_TABLE(platform, sahara_platform_ids); -static struct of_device_id sahara_dt_ids[] = { +static const struct of_device_id sahara_dt_ids[] = { { .compatible = "fsl,imx53-sahara" }, { .compatible = "fsl,imx27-sahara" }, { /* sentinel */ } @@ -1487,7 +1486,6 @@ static int sahara_probe(struct platform_device *pdev) crypto_init_queue(&dev->queue, SAHARA_QUEUE_LENGTH); - spin_lock_init(&dev->lock); mutex_init(&dev->queue_mutex); dev_ptr = dev; diff --git a/drivers/crypto/stm32/Kconfig b/drivers/crypto/stm32/Kconfig index 09b4ec87c212..602332e02729 100644 --- a/drivers/crypto/stm32/Kconfig +++ b/drivers/crypto/stm32/Kconfig @@ -1,7 +1,20 @@ -config CRYPTO_DEV_STM32 - tristate "Support for STM32 crypto accelerators" +config CRC_DEV_STM32 + tristate "Support for STM32 crc accelerators" depends on ARCH_STM32 select CRYPTO_HASH help This enables support for the CRC32 hw accelerator which can be found - on STMicroelectronis STM32 SOC. + on STMicroelectronics STM32 SOC. + +config HASH_DEV_STM32 + tristate "Support for STM32 hash accelerators" + depends on ARCH_STM32 + depends on HAS_DMA + select CRYPTO_HASH + select CRYPTO_MD5 + select CRYPTO_SHA1 + select CRYPTO_SHA256 + select CRYPTO_ENGINE + help + This enables support for the HASH hw accelerator which can be found + on STMicroelectronics STM32 SOC. diff --git a/drivers/crypto/stm32/Makefile b/drivers/crypto/stm32/Makefile index 73b4c6e47f5f..73cd56cad0cc 100644 --- a/drivers/crypto/stm32/Makefile +++ b/drivers/crypto/stm32/Makefile @@ -1,2 +1,2 @@ -obj-$(CONFIG_CRYPTO_DEV_STM32) += stm32_cryp.o -stm32_cryp-objs := stm32_crc32.o +obj-$(CONFIG_CRC_DEV_STM32) += stm32_crc32.o +obj-$(CONFIG_HASH_DEV_STM32) += stm32-hash.o
\ No newline at end of file diff --git a/drivers/crypto/stm32/stm32-hash.c b/drivers/crypto/stm32/stm32-hash.c new file mode 100644 index 000000000000..4ca4a264a833 --- /dev/null +++ b/drivers/crypto/stm32/stm32-hash.c @@ -0,0 +1,1568 @@ +/* + * This file is part of STM32 Crypto driver for Linux. + * + * Copyright (C) 2017, STMicroelectronics - All Rights Reserved + * Author(s): Lionel DEBIEVE <lionel.debieve@st.com> for STMicroelectronics. + * + * License terms: GPL V2.0. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published by + * the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see <http://www.gnu.org/licenses/>. + * + */ + +#include <linux/clk.h> +#include <linux/crypto.h> +#include <linux/delay.h> +#include <linux/dmaengine.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/iopoll.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/of_device.h> +#include <linux/platform_device.h> +#include <linux/reset.h> + +#include <crypto/engine.h> +#include <crypto/hash.h> +#include <crypto/md5.h> +#include <crypto/scatterwalk.h> +#include <crypto/sha.h> +#include <crypto/internal/hash.h> + +#define HASH_CR 0x00 +#define HASH_DIN 0x04 +#define HASH_STR 0x08 +#define HASH_IMR 0x20 +#define HASH_SR 0x24 +#define HASH_CSR(x) (0x0F8 + ((x) * 0x04)) +#define HASH_HREG(x) (0x310 + ((x) * 0x04)) +#define HASH_HWCFGR 0x3F0 +#define HASH_VER 0x3F4 +#define HASH_ID 0x3F8 + +/* Control Register */ +#define HASH_CR_INIT BIT(2) +#define HASH_CR_DMAE BIT(3) +#define HASH_CR_DATATYPE_POS 4 +#define HASH_CR_MODE BIT(6) +#define HASH_CR_MDMAT BIT(13) +#define HASH_CR_DMAA BIT(14) +#define HASH_CR_LKEY BIT(16) + +#define HASH_CR_ALGO_SHA1 0x0 +#define HASH_CR_ALGO_MD5 0x80 +#define HASH_CR_ALGO_SHA224 0x40000 +#define HASH_CR_ALGO_SHA256 0x40080 + +/* Interrupt */ +#define HASH_DINIE BIT(0) +#define HASH_DCIE BIT(1) + +/* Interrupt Mask */ +#define HASH_MASK_CALC_COMPLETION BIT(0) +#define HASH_MASK_DATA_INPUT BIT(1) + +/* Context swap register */ +#define HASH_CSR_REGISTER_NUMBER 53 + +/* Status Flags */ +#define HASH_SR_DATA_INPUT_READY BIT(0) +#define HASH_SR_OUTPUT_READY BIT(1) +#define HASH_SR_DMA_ACTIVE BIT(2) +#define HASH_SR_BUSY BIT(3) + +/* STR Register */ +#define HASH_STR_NBLW_MASK GENMASK(4, 0) +#define HASH_STR_DCAL BIT(8) + +#define HASH_FLAGS_INIT BIT(0) +#define HASH_FLAGS_OUTPUT_READY BIT(1) +#define HASH_FLAGS_CPU BIT(2) +#define HASH_FLAGS_DMA_READY BIT(3) +#define HASH_FLAGS_DMA_ACTIVE BIT(4) +#define HASH_FLAGS_HMAC_INIT BIT(5) +#define HASH_FLAGS_HMAC_FINAL BIT(6) +#define HASH_FLAGS_HMAC_KEY BIT(7) + +#define HASH_FLAGS_FINAL BIT(15) +#define HASH_FLAGS_FINUP BIT(16) +#define HASH_FLAGS_ALGO_MASK GENMASK(21, 18) +#define HASH_FLAGS_MD5 BIT(18) +#define HASH_FLAGS_SHA1 BIT(19) +#define HASH_FLAGS_SHA224 BIT(20) +#define HASH_FLAGS_SHA256 BIT(21) +#define HASH_FLAGS_ERRORS BIT(22) +#define HASH_FLAGS_HMAC BIT(23) + +#define HASH_OP_UPDATE 1 +#define HASH_OP_FINAL 2 + +enum stm32_hash_data_format { + HASH_DATA_32_BITS = 0x0, + HASH_DATA_16_BITS = 0x1, + HASH_DATA_8_BITS = 0x2, + HASH_DATA_1_BIT = 0x3 +}; + +#define HASH_BUFLEN 256 +#define HASH_LONG_KEY 64 +#define HASH_MAX_KEY_SIZE (SHA256_BLOCK_SIZE * 8) +#define HASH_QUEUE_LENGTH 16 +#define HASH_DMA_THRESHOLD 50 + +struct stm32_hash_ctx { + struct stm32_hash_dev *hdev; + unsigned long flags; + + u8 key[HASH_MAX_KEY_SIZE]; + int keylen; +}; + +struct stm32_hash_request_ctx { + struct stm32_hash_dev *hdev; + unsigned long flags; + unsigned long op; + + u8 digest[SHA256_DIGEST_SIZE] __aligned(sizeof(u32)); + size_t digcnt; + size_t bufcnt; + size_t buflen; + + /* DMA */ + struct scatterlist *sg; + unsigned int offset; + unsigned int total; + struct scatterlist sg_key; + + dma_addr_t dma_addr; + size_t dma_ct; + int nents; + + u8 data_type; + + u8 buffer[HASH_BUFLEN] __aligned(sizeof(u32)); + + /* Export Context */ + u32 *hw_context; +}; + +struct stm32_hash_algs_info { + struct ahash_alg *algs_list; + size_t size; +}; + +struct stm32_hash_pdata { + struct stm32_hash_algs_info *algs_info; + size_t algs_info_size; +}; + +struct stm32_hash_dev { + struct list_head list; + struct device *dev; + struct clk *clk; + struct reset_control *rst; + void __iomem *io_base; + phys_addr_t phys_base; + u32 dma_mode; + u32 dma_maxburst; + + spinlock_t lock; /* lock to protect queue */ + + struct ahash_request *req; + struct crypto_engine *engine; + + int err; + unsigned long flags; + + struct dma_chan *dma_lch; + struct completion dma_completion; + + const struct stm32_hash_pdata *pdata; +}; + +struct stm32_hash_drv { + struct list_head dev_list; + spinlock_t lock; /* List protection access */ +}; + +static struct stm32_hash_drv stm32_hash = { + .dev_list = LIST_HEAD_INIT(stm32_hash.dev_list), + .lock = __SPIN_LOCK_UNLOCKED(stm32_hash.lock), +}; + +static void stm32_hash_dma_callback(void *param); + +static inline u32 stm32_hash_read(struct stm32_hash_dev *hdev, u32 offset) +{ + return readl_relaxed(hdev->io_base + offset); +} + +static inline void stm32_hash_write(struct stm32_hash_dev *hdev, + u32 offset, u32 value) +{ + writel_relaxed(value, hdev->io_base + offset); +} + +static inline int stm32_hash_wait_busy(struct stm32_hash_dev *hdev) +{ + u32 status; + + return readl_relaxed_poll_timeout(hdev->io_base + HASH_SR, status, + !(status & HASH_SR_BUSY), 10, 10000); +} + +static void stm32_hash_set_nblw(struct stm32_hash_dev *hdev, int length) +{ + u32 reg; + + reg = stm32_hash_read(hdev, HASH_STR); + reg &= ~(HASH_STR_NBLW_MASK); + reg |= (8U * ((length) % 4U)); + stm32_hash_write(hdev, HASH_STR, reg); +} + +static int stm32_hash_write_key(struct stm32_hash_dev *hdev) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(hdev->req); + struct stm32_hash_ctx *ctx = crypto_ahash_ctx(tfm); + u32 reg; + int keylen = ctx->keylen; + void *key = ctx->key; + + if (keylen) { + stm32_hash_set_nblw(hdev, keylen); + + while (keylen > 0) { + stm32_hash_write(hdev, HASH_DIN, *(u32 *)key); + keylen -= 4; + key += 4; + } + + reg = stm32_hash_read(hdev, HASH_STR); + reg |= HASH_STR_DCAL; + stm32_hash_write(hdev, HASH_STR, reg); + + return -EINPROGRESS; + } + + return 0; +} + +static void stm32_hash_write_ctrl(struct stm32_hash_dev *hdev) +{ + struct stm32_hash_request_ctx *rctx = ahash_request_ctx(hdev->req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(hdev->req); + struct stm32_hash_ctx *ctx = crypto_ahash_ctx(tfm); + + u32 reg = HASH_CR_INIT; + + if (!(hdev->flags & HASH_FLAGS_INIT)) { + switch (rctx->flags & HASH_FLAGS_ALGO_MASK) { + case HASH_FLAGS_MD5: + reg |= HASH_CR_ALGO_MD5; + break; + case HASH_FLAGS_SHA1: + reg |= HASH_CR_ALGO_SHA1; + break; + case HASH_FLAGS_SHA224: + reg |= HASH_CR_ALGO_SHA224; + break; + case HASH_FLAGS_SHA256: + reg |= HASH_CR_ALGO_SHA256; + break; + default: + reg |= HASH_CR_ALGO_MD5; + } + + reg |= (rctx->data_type << HASH_CR_DATATYPE_POS); + + if (rctx->flags & HASH_FLAGS_HMAC) { + hdev->flags |= HASH_FLAGS_HMAC; + reg |= HASH_CR_MODE; + if (ctx->keylen > HASH_LONG_KEY) + reg |= HASH_CR_LKEY; + } + + stm32_hash_write(hdev, HASH_IMR, HASH_DCIE); + + stm32_hash_write(hdev, HASH_CR, reg); + + hdev->flags |= HASH_FLAGS_INIT; + + dev_dbg(hdev->dev, "Write Control %x\n", reg); + } +} + +static void stm32_hash_append_sg(struct stm32_hash_request_ctx *rctx) +{ + size_t count; + + while ((rctx->bufcnt < rctx->buflen) && rctx->total) { + count = min(rctx->sg->length - rctx->offset, rctx->total); + count = min(count, rctx->buflen - rctx->bufcnt); + + if (count <= 0) { + if ((rctx->sg->length == 0) && !sg_is_last(rctx->sg)) { + rctx->sg = sg_next(rctx->sg); + continue; + } else { + break; + } + } + + scatterwalk_map_and_copy(rctx->buffer + rctx->bufcnt, rctx->sg, + rctx->offset, count, 0); + + rctx->bufcnt += count; + rctx->offset += count; + rctx->total -= count; + + if (rctx->offset == rctx->sg->length) { + rctx->sg = sg_next(rctx->sg); + if (rctx->sg) + rctx->offset = 0; + else + rctx->total = 0; + } + } +} + +static int stm32_hash_xmit_cpu(struct stm32_hash_dev *hdev, + const u8 *buf, size_t length, int final) +{ + unsigned int count, len32; + const u32 *buffer = (const u32 *)buf; + u32 reg; + + if (final) + hdev->flags |= HASH_FLAGS_FINAL; + + len32 = DIV_ROUND_UP(length, sizeof(u32)); + + dev_dbg(hdev->dev, "%s: length: %d, final: %x len32 %i\n", + __func__, length, final, len32); + + hdev->flags |= HASH_FLAGS_CPU; + + stm32_hash_write_ctrl(hdev); + + if (stm32_hash_wait_busy(hdev)) + return -ETIMEDOUT; + + if ((hdev->flags & HASH_FLAGS_HMAC) && + (hdev->flags & ~HASH_FLAGS_HMAC_KEY)) { + hdev->flags |= HASH_FLAGS_HMAC_KEY; + stm32_hash_write_key(hdev); + if (stm32_hash_wait_busy(hdev)) + return -ETIMEDOUT; + } + + for (count = 0; count < len32; count++) + stm32_hash_write(hdev, HASH_DIN, buffer[count]); + + if (final) { + stm32_hash_set_nblw(hdev, length); + reg = stm32_hash_read(hdev, HASH_STR); + reg |= HASH_STR_DCAL; + stm32_hash_write(hdev, HASH_STR, reg); + if (hdev->flags & HASH_FLAGS_HMAC) { + if (stm32_hash_wait_busy(hdev)) + return -ETIMEDOUT; + stm32_hash_write_key(hdev); + } + return -EINPROGRESS; + } + + return 0; +} + +static int stm32_hash_update_cpu(struct stm32_hash_dev *hdev) +{ + struct stm32_hash_request_ctx *rctx = ahash_request_ctx(hdev->req); + int bufcnt, err = 0, final; + + dev_dbg(hdev->dev, "%s flags %lx\n", __func__, rctx->flags); + + final = (rctx->flags & HASH_FLAGS_FINUP); + + while ((rctx->total >= rctx->buflen) || + (rctx->bufcnt + rctx->total >= rctx->buflen)) { + stm32_hash_append_sg(rctx); + bufcnt = rctx->bufcnt; + rctx->bufcnt = 0; + err = stm32_hash_xmit_cpu(hdev, rctx->buffer, bufcnt, 0); + } + + stm32_hash_append_sg(rctx); + + if (final) { + bufcnt = rctx->bufcnt; + rctx->bufcnt = 0; + err = stm32_hash_xmit_cpu(hdev, rctx->buffer, bufcnt, + (rctx->flags & HASH_FLAGS_FINUP)); + } + + return err; +} + +static int stm32_hash_xmit_dma(struct stm32_hash_dev *hdev, + struct scatterlist *sg, int length, int mdma) +{ + struct dma_async_tx_descriptor *in_desc; + dma_cookie_t cookie; + u32 reg; + int err; + + in_desc = dmaengine_prep_slave_sg(hdev->dma_lch, sg, 1, + DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | + DMA_CTRL_ACK); + if (!in_desc) { + dev_err(hdev->dev, "dmaengine_prep_slave error\n"); + return -ENOMEM; + } + + reinit_completion(&hdev->dma_completion); + in_desc->callback = stm32_hash_dma_callback; + in_desc->callback_param = hdev; + + hdev->flags |= HASH_FLAGS_FINAL; + hdev->flags |= HASH_FLAGS_DMA_ACTIVE; + + reg = stm32_hash_read(hdev, HASH_CR); + + if (mdma) + reg |= HASH_CR_MDMAT; + else + reg &= ~HASH_CR_MDMAT; + + reg |= HASH_CR_DMAE; + + stm32_hash_write(hdev, HASH_CR, reg); + + stm32_hash_set_nblw(hdev, length); + + cookie = dmaengine_submit(in_desc); + err = dma_submit_error(cookie); + if (err) + return -ENOMEM; + + dma_async_issue_pending(hdev->dma_lch); + + if (!wait_for_completion_interruptible_timeout(&hdev->dma_completion, + msecs_to_jiffies(100))) + err = -ETIMEDOUT; + + if (dma_async_is_tx_complete(hdev->dma_lch, cookie, + NULL, NULL) != DMA_COMPLETE) + err = -ETIMEDOUT; + + if (err) { + dev_err(hdev->dev, "DMA Error %i\n", err); + dmaengine_terminate_all(hdev->dma_lch); + return err; + } + + return -EINPROGRESS; +} + +static void stm32_hash_dma_callback(void *param) +{ + struct stm32_hash_dev *hdev = param; + + complete(&hdev->dma_completion); + + hdev->flags |= HASH_FLAGS_DMA_READY; +} + +static int stm32_hash_hmac_dma_send(struct stm32_hash_dev *hdev) +{ + struct stm32_hash_request_ctx *rctx = ahash_request_ctx(hdev->req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(hdev->req); + struct stm32_hash_ctx *ctx = crypto_ahash_ctx(tfm); + int err; + + if (ctx->keylen < HASH_DMA_THRESHOLD || (hdev->dma_mode == 1)) { + err = stm32_hash_write_key(hdev); + if (stm32_hash_wait_busy(hdev)) + return -ETIMEDOUT; + } else { + if (!(hdev->flags & HASH_FLAGS_HMAC_KEY)) + sg_init_one(&rctx->sg_key, ctx->key, + ALIGN(ctx->keylen, sizeof(u32))); + + rctx->dma_ct = dma_map_sg(hdev->dev, &rctx->sg_key, 1, + DMA_TO_DEVICE); + if (rctx->dma_ct == 0) { + dev_err(hdev->dev, "dma_map_sg error\n"); + return -ENOMEM; + } + + err = stm32_hash_xmit_dma(hdev, &rctx->sg_key, ctx->keylen, 0); + + dma_unmap_sg(hdev->dev, &rctx->sg_key, 1, DMA_TO_DEVICE); + } + + return err; +} + +static int stm32_hash_dma_init(struct stm32_hash_dev *hdev) +{ + struct dma_slave_config dma_conf; + int err; + + memset(&dma_conf, 0, sizeof(dma_conf)); + + dma_conf.direction = DMA_MEM_TO_DEV; + dma_conf.dst_addr = hdev->phys_base + HASH_DIN; + dma_conf.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + dma_conf.src_maxburst = hdev->dma_maxburst; + dma_conf.dst_maxburst = hdev->dma_maxburst; + dma_conf.device_fc = false; + + hdev->dma_lch = dma_request_slave_channel(hdev->dev, "in"); + if (!hdev->dma_lch) { + dev_err(hdev->dev, "Couldn't acquire a slave DMA channel.\n"); + return -EBUSY; + } + + err = dmaengine_slave_config(hdev->dma_lch, &dma_conf); + if (err) { + dma_release_channel(hdev->dma_lch); + hdev->dma_lch = NULL; + dev_err(hdev->dev, "Couldn't configure DMA slave.\n"); + return err; + } + + init_completion(&hdev->dma_completion); + + return 0; +} + +static int stm32_hash_dma_send(struct stm32_hash_dev *hdev) +{ + struct stm32_hash_request_ctx *rctx = ahash_request_ctx(hdev->req); + struct scatterlist sg[1], *tsg; + int err = 0, len = 0, reg, ncp = 0; + unsigned int i; + u32 *buffer = (void *)rctx->buffer; + + rctx->sg = hdev->req->src; + rctx->total = hdev->req->nbytes; + + rctx->nents = sg_nents(rctx->sg); + + if (rctx->nents < 0) + return -EINVAL; + + stm32_hash_write_ctrl(hdev); + + if (hdev->flags & HASH_FLAGS_HMAC) { + err = stm32_hash_hmac_dma_send(hdev); + if (err != -EINPROGRESS) + return err; + } + + for_each_sg(rctx->sg, tsg, rctx->nents, i) { + len = sg->length; + + sg[0] = *tsg; + if (sg_is_last(sg)) { + if (hdev->dma_mode == 1) { + len = (ALIGN(sg->length, 16) - 16); + + ncp = sg_pcopy_to_buffer( + rctx->sg, rctx->nents, + rctx->buffer, sg->length - len, + rctx->total - sg->length + len); + + sg->length = len; + } else { + if (!(IS_ALIGNED(sg->length, sizeof(u32)))) { + len = sg->length; + sg->length = ALIGN(sg->length, + sizeof(u32)); + } + } + } + + rctx->dma_ct = dma_map_sg(hdev->dev, sg, 1, + DMA_TO_DEVICE); + if (rctx->dma_ct == 0) { + dev_err(hdev->dev, "dma_map_sg error\n"); + return -ENOMEM; + } + + err = stm32_hash_xmit_dma(hdev, sg, len, + !sg_is_last(sg)); + + dma_unmap_sg(hdev->dev, sg, 1, DMA_TO_DEVICE); + + if (err == -ENOMEM) + return err; + } + + if (hdev->dma_mode == 1) { + if (stm32_hash_wait_busy(hdev)) + return -ETIMEDOUT; + reg = stm32_hash_read(hdev, HASH_CR); + reg &= ~HASH_CR_DMAE; + reg |= HASH_CR_DMAA; + stm32_hash_write(hdev, HASH_CR, reg); + + if (ncp) { + memset(buffer + ncp, 0, + DIV_ROUND_UP(ncp, sizeof(u32)) - ncp); + writesl(hdev->io_base + HASH_DIN, buffer, + DIV_ROUND_UP(ncp, sizeof(u32))); + } + stm32_hash_set_nblw(hdev, DIV_ROUND_UP(ncp, sizeof(u32))); + reg = stm32_hash_read(hdev, HASH_STR); + reg |= HASH_STR_DCAL; + stm32_hash_write(hdev, HASH_STR, reg); + err = -EINPROGRESS; + } + + if (hdev->flags & HASH_FLAGS_HMAC) { + if (stm32_hash_wait_busy(hdev)) + return -ETIMEDOUT; + err = stm32_hash_hmac_dma_send(hdev); + } + + return err; +} + +static struct stm32_hash_dev *stm32_hash_find_dev(struct stm32_hash_ctx *ctx) +{ + struct stm32_hash_dev *hdev = NULL, *tmp; + + spin_lock_bh(&stm32_hash.lock); + if (!ctx->hdev) { + list_for_each_entry(tmp, &stm32_hash.dev_list, list) { + hdev = tmp; + break; + } + ctx->hdev = hdev; + } else { + hdev = ctx->hdev; + } + + spin_unlock_bh(&stm32_hash.lock); + + return hdev; +} + +static bool stm32_hash_dma_aligned_data(struct ahash_request *req) +{ + struct scatterlist *sg; + struct stm32_hash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req)); + struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx); + int i; + + if (req->nbytes <= HASH_DMA_THRESHOLD) + return false; + + if (sg_nents(req->src) > 1) { + if (hdev->dma_mode == 1) + return false; + for_each_sg(req->src, sg, sg_nents(req->src), i) { + if ((!IS_ALIGNED(sg->length, sizeof(u32))) && + (!sg_is_last(sg))) + return false; + } + } + + if (req->src->offset % 4) + return false; + + return true; +} + +static int stm32_hash_init(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct stm32_hash_ctx *ctx = crypto_ahash_ctx(tfm); + struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req); + struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx); + + rctx->hdev = hdev; + + rctx->flags = HASH_FLAGS_CPU; + + rctx->digcnt = crypto_ahash_digestsize(tfm); + switch (rctx->digcnt) { + case MD5_DIGEST_SIZE: + rctx->flags |= HASH_FLAGS_MD5; + break; + case SHA1_DIGEST_SIZE: + rctx->flags |= HASH_FLAGS_SHA1; + break; + case SHA224_DIGEST_SIZE: + rctx->flags |= HASH_FLAGS_SHA224; + break; + case SHA256_DIGEST_SIZE: + rctx->flags |= HASH_FLAGS_SHA256; + break; + default: + return -EINVAL; + } + + rctx->bufcnt = 0; + rctx->buflen = HASH_BUFLEN; + rctx->total = 0; + rctx->offset = 0; + rctx->data_type = HASH_DATA_8_BITS; + + memset(rctx->buffer, 0, HASH_BUFLEN); + + if (ctx->flags & HASH_FLAGS_HMAC) + rctx->flags |= HASH_FLAGS_HMAC; + + dev_dbg(hdev->dev, "%s Flags %lx\n", __func__, rctx->flags); + + return 0; +} + +static int stm32_hash_update_req(struct stm32_hash_dev *hdev) +{ + return stm32_hash_update_cpu(hdev); +} + +static int stm32_hash_final_req(struct stm32_hash_dev *hdev) +{ + struct ahash_request *req = hdev->req; + struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req); + int err; + + if (!(rctx->flags & HASH_FLAGS_CPU)) + err = stm32_hash_dma_send(hdev); + else + err = stm32_hash_xmit_cpu(hdev, rctx->buffer, rctx->bufcnt, 1); + + rctx->bufcnt = 0; + + return err; +} + +static void stm32_hash_copy_hash(struct ahash_request *req) +{ + struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req); + u32 *hash = (u32 *)rctx->digest; + unsigned int i, hashsize; + + switch (rctx->flags & HASH_FLAGS_ALGO_MASK) { + case HASH_FLAGS_MD5: + hashsize = MD5_DIGEST_SIZE; + break; + case HASH_FLAGS_SHA1: + hashsize = SHA1_DIGEST_SIZE; + break; + case HASH_FLAGS_SHA224: + hashsize = SHA224_DIGEST_SIZE; + break; + case HASH_FLAGS_SHA256: + hashsize = SHA256_DIGEST_SIZE; + break; + default: + return; + } + + for (i = 0; i < hashsize / sizeof(u32); i++) + hash[i] = be32_to_cpu(stm32_hash_read(rctx->hdev, + HASH_HREG(i))); +} + +static int stm32_hash_finish(struct ahash_request *req) +{ + struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req); + + if (!req->result) + return -EINVAL; + + memcpy(req->result, rctx->digest, rctx->digcnt); + + return 0; +} + +static void stm32_hash_finish_req(struct ahash_request *req, int err) +{ + struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req); + struct stm32_hash_dev *hdev = rctx->hdev; + + if (!err && (HASH_FLAGS_FINAL & hdev->flags)) { + stm32_hash_copy_hash(req); + err = stm32_hash_finish(req); + hdev->flags &= ~(HASH_FLAGS_FINAL | HASH_FLAGS_CPU | + HASH_FLAGS_INIT | HASH_FLAGS_DMA_READY | + HASH_FLAGS_OUTPUT_READY | HASH_FLAGS_HMAC | + HASH_FLAGS_HMAC_INIT | HASH_FLAGS_HMAC_FINAL | + HASH_FLAGS_HMAC_KEY); + } else { + rctx->flags |= HASH_FLAGS_ERRORS; + } + + crypto_finalize_hash_request(hdev->engine, req, err); +} + +static int stm32_hash_hw_init(struct stm32_hash_dev *hdev, + struct stm32_hash_request_ctx *rctx) +{ + if (!(HASH_FLAGS_INIT & hdev->flags)) { + stm32_hash_write(hdev, HASH_CR, HASH_CR_INIT); + stm32_hash_write(hdev, HASH_STR, 0); + stm32_hash_write(hdev, HASH_DIN, 0); + stm32_hash_write(hdev, HASH_IMR, 0); + hdev->err = 0; + } + + return 0; +} + +static int stm32_hash_handle_queue(struct stm32_hash_dev *hdev, + struct ahash_request *req) +{ + return crypto_transfer_hash_request_to_engine(hdev->engine, req); +} + +static int stm32_hash_prepare_req(struct crypto_engine *engine, + struct ahash_request *req) +{ + struct stm32_hash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req)); + struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx); + struct stm32_hash_request_ctx *rctx; + + if (!hdev) + return -ENODEV; + + hdev->req = req; + + rctx = ahash_request_ctx(req); + + dev_dbg(hdev->dev, "processing new req, op: %lu, nbytes %d\n", + rctx->op, req->nbytes); + + return stm32_hash_hw_init(hdev, rctx); +} + +static int stm32_hash_one_request(struct crypto_engine *engine, + struct ahash_request *req) +{ + struct stm32_hash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req)); + struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx); + struct stm32_hash_request_ctx *rctx; + int err = 0; + + if (!hdev) + return -ENODEV; + + hdev->req = req; + + rctx = ahash_request_ctx(req); + + if (rctx->op == HASH_OP_UPDATE) + err = stm32_hash_update_req(hdev); + else if (rctx->op == HASH_OP_FINAL) + err = stm32_hash_final_req(hdev); + + if (err != -EINPROGRESS) + /* done task will not finish it, so do it here */ + stm32_hash_finish_req(req, err); + + return 0; +} + +static int stm32_hash_enqueue(struct ahash_request *req, unsigned int op) +{ + struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req); + struct stm32_hash_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct stm32_hash_dev *hdev = ctx->hdev; + + rctx->op = op; + + return stm32_hash_handle_queue(hdev, req); +} + +static int stm32_hash_update(struct ahash_request *req) +{ + struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req); + + if (!req->nbytes || !(rctx->flags & HASH_FLAGS_CPU)) + return 0; + + rctx->total = req->nbytes; + rctx->sg = req->src; + rctx->offset = 0; + + if ((rctx->bufcnt + rctx->total < rctx->buflen)) { + stm32_hash_append_sg(rctx); + return 0; + } + + return stm32_hash_enqueue(req, HASH_OP_UPDATE); +} + +static int stm32_hash_final(struct ahash_request *req) +{ + struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req); + + rctx->flags |= HASH_FLAGS_FINUP; + + return stm32_hash_enqueue(req, HASH_OP_FINAL); +} + +static int stm32_hash_finup(struct ahash_request *req) +{ + struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req); + struct stm32_hash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req)); + struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx); + int err1, err2; + + rctx->flags |= HASH_FLAGS_FINUP; + + if (hdev->dma_lch && stm32_hash_dma_aligned_data(req)) + rctx->flags &= ~HASH_FLAGS_CPU; + + err1 = stm32_hash_update(req); + + if (err1 == -EINPROGRESS || err1 == -EBUSY) + return err1; + + /* + * final() has to be always called to cleanup resources + * even if update() failed, except EINPROGRESS + */ + err2 = stm32_hash_final(req); + + return err1 ?: err2; +} + +static int stm32_hash_digest(struct ahash_request *req) +{ + return stm32_hash_init(req) ?: stm32_hash_finup(req); +} + +static int stm32_hash_export(struct ahash_request *req, void *out) +{ + struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req); + struct stm32_hash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req)); + struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx); + u32 *preg; + unsigned int i; + + while (!(stm32_hash_read(hdev, HASH_SR) & HASH_SR_DATA_INPUT_READY)) + cpu_relax(); + + rctx->hw_context = kmalloc(sizeof(u32) * (3 + HASH_CSR_REGISTER_NUMBER), + GFP_KERNEL); + + preg = rctx->hw_context; + + *preg++ = stm32_hash_read(hdev, HASH_IMR); + *preg++ = stm32_hash_read(hdev, HASH_STR); + *preg++ = stm32_hash_read(hdev, HASH_CR); + for (i = 0; i < HASH_CSR_REGISTER_NUMBER; i++) + *preg++ = stm32_hash_read(hdev, HASH_CSR(i)); + + memcpy(out, rctx, sizeof(*rctx)); + + return 0; +} + +static int stm32_hash_import(struct ahash_request *req, const void *in) +{ + struct stm32_hash_request_ctx *rctx = ahash_request_ctx(req); + struct stm32_hash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req)); + struct stm32_hash_dev *hdev = stm32_hash_find_dev(ctx); + const u32 *preg = in; + u32 reg; + unsigned int i; + + memcpy(rctx, in, sizeof(*rctx)); + + preg = rctx->hw_context; + + stm32_hash_write(hdev, HASH_IMR, *preg++); + stm32_hash_write(hdev, HASH_STR, *preg++); + stm32_hash_write(hdev, HASH_CR, *preg); + reg = *preg++ | HASH_CR_INIT; + stm32_hash_write(hdev, HASH_CR, reg); + + for (i = 0; i < HASH_CSR_REGISTER_NUMBER; i++) + stm32_hash_write(hdev, HASH_CSR(i), *preg++); + + kfree(rctx->hw_context); + + return 0; +} + +static int stm32_hash_setkey(struct crypto_ahash *tfm, + const u8 *key, unsigned int keylen) +{ + struct stm32_hash_ctx *ctx = crypto_ahash_ctx(tfm); + + if (keylen <= HASH_MAX_KEY_SIZE) { + memcpy(ctx->key, key, keylen); + ctx->keylen = keylen; + } else { + return -ENOMEM; + } + + return 0; +} + +static int stm32_hash_cra_init_algs(struct crypto_tfm *tfm, + const char *algs_hmac_name) +{ + struct stm32_hash_ctx *ctx = crypto_tfm_ctx(tfm); + + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct stm32_hash_request_ctx)); + + ctx->keylen = 0; + + if (algs_hmac_name) + ctx->flags |= HASH_FLAGS_HMAC; + + return 0; +} + +static int stm32_hash_cra_init(struct crypto_tfm *tfm) +{ + return stm32_hash_cra_init_algs(tfm, NULL); +} + +static int stm32_hash_cra_md5_init(struct crypto_tfm *tfm) +{ + return stm32_hash_cra_init_algs(tfm, "md5"); +} + +static int stm32_hash_cra_sha1_init(struct crypto_tfm *tfm) +{ + return stm32_hash_cra_init_algs(tfm, "sha1"); +} + +static int stm32_hash_cra_sha224_init(struct crypto_tfm *tfm) +{ + return stm32_hash_cra_init_algs(tfm, "sha224"); +} + +static int stm32_hash_cra_sha256_init(struct crypto_tfm *tfm) +{ + return stm32_hash_cra_init_algs(tfm, "sha256"); +} + +static irqreturn_t stm32_hash_irq_thread(int irq, void *dev_id) +{ + struct stm32_hash_dev *hdev = dev_id; + + if (HASH_FLAGS_CPU & hdev->flags) { + if (HASH_FLAGS_OUTPUT_READY & hdev->flags) { + hdev->flags &= ~HASH_FLAGS_OUTPUT_READY; + goto finish; + } + } else if (HASH_FLAGS_DMA_READY & hdev->flags) { + if (HASH_FLAGS_DMA_ACTIVE & hdev->flags) { + hdev->flags &= ~HASH_FLAGS_DMA_ACTIVE; + goto finish; + } + } + + return IRQ_HANDLED; + +finish: + /* Finish current request */ + stm32_hash_finish_req(hdev->req, 0); + + return IRQ_HANDLED; +} + +static irqreturn_t stm32_hash_irq_handler(int irq, void *dev_id) +{ + struct stm32_hash_dev *hdev = dev_id; + u32 reg; + + reg = stm32_hash_read(hdev, HASH_SR); + if (reg & HASH_SR_OUTPUT_READY) { + reg &= ~HASH_SR_OUTPUT_READY; + stm32_hash_write(hdev, HASH_SR, reg); + hdev->flags |= HASH_FLAGS_OUTPUT_READY; + return IRQ_WAKE_THREAD; + } + + return IRQ_NONE; +} + +static struct ahash_alg algs_md5_sha1[] = { + { + .init = stm32_hash_init, + .update = stm32_hash_update, + .final = stm32_hash_final, + .finup = stm32_hash_finup, + .digest = stm32_hash_digest, + .export = stm32_hash_export, + .import = stm32_hash_import, + .halg = { + .digestsize = MD5_DIGEST_SIZE, + .statesize = sizeof(struct stm32_hash_request_ctx), + .base = { + .cra_name = "md5", + .cra_driver_name = "stm32-md5", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = MD5_HMAC_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct stm32_hash_ctx), + .cra_alignmask = 3, + .cra_init = stm32_hash_cra_init, + .cra_module = THIS_MODULE, + } + } + }, + { + .init = stm32_hash_init, + .update = stm32_hash_update, + .final = stm32_hash_final, + .finup = stm32_hash_finup, + .digest = stm32_hash_digest, + .export = stm32_hash_export, + .import = stm32_hash_import, + .setkey = stm32_hash_setkey, + .halg = { + .digestsize = MD5_DIGEST_SIZE, + .statesize = sizeof(struct stm32_hash_request_ctx), + .base = { + .cra_name = "hmac(md5)", + .cra_driver_name = "stm32-hmac-md5", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = MD5_HMAC_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct stm32_hash_ctx), + .cra_alignmask = 3, + .cra_init = stm32_hash_cra_md5_init, + .cra_module = THIS_MODULE, + } + } + }, + { + .init = stm32_hash_init, + .update = stm32_hash_update, + .final = stm32_hash_final, + .finup = stm32_hash_finup, + .digest = stm32_hash_digest, + .export = stm32_hash_export, + .import = stm32_hash_import, + .halg = { + .digestsize = SHA1_DIGEST_SIZE, + .statesize = sizeof(struct stm32_hash_request_ctx), + .base = { + .cra_name = "sha1", + .cra_driver_name = "stm32-sha1", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct stm32_hash_ctx), + .cra_alignmask = 3, + .cra_init = stm32_hash_cra_init, + .cra_module = THIS_MODULE, + } + } + }, + { + .init = stm32_hash_init, + .update = stm32_hash_update, + .final = stm32_hash_final, + .finup = stm32_hash_finup, + .digest = stm32_hash_digest, + .export = stm32_hash_export, + .import = stm32_hash_import, + .setkey = stm32_hash_setkey, + .halg = { + .digestsize = SHA1_DIGEST_SIZE, + .statesize = sizeof(struct stm32_hash_request_ctx), + .base = { + .cra_name = "hmac(sha1)", + .cra_driver_name = "stm32-hmac-sha1", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct stm32_hash_ctx), + .cra_alignmask = 3, + .cra_init = stm32_hash_cra_sha1_init, + .cra_module = THIS_MODULE, + } + } + }, +}; + +static struct ahash_alg algs_sha224_sha256[] = { + { + .init = stm32_hash_init, + .update = stm32_hash_update, + .final = stm32_hash_final, + .finup = stm32_hash_finup, + .digest = stm32_hash_digest, + .export = stm32_hash_export, + .import = stm32_hash_import, + .halg = { + .digestsize = SHA224_DIGEST_SIZE, + .statesize = sizeof(struct stm32_hash_request_ctx), + .base = { + .cra_name = "sha224", + .cra_driver_name = "stm32-sha224", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct stm32_hash_ctx), + .cra_alignmask = 3, + .cra_init = stm32_hash_cra_init, + .cra_module = THIS_MODULE, + } + } + }, + { + .init = stm32_hash_init, + .update = stm32_hash_update, + .final = stm32_hash_final, + .finup = stm32_hash_finup, + .digest = stm32_hash_digest, + .setkey = stm32_hash_setkey, + .export = stm32_hash_export, + .import = stm32_hash_import, + .halg = { + .digestsize = SHA224_DIGEST_SIZE, + .statesize = sizeof(struct stm32_hash_request_ctx), + .base = { + .cra_name = "hmac(sha224)", + .cra_driver_name = "stm32-hmac-sha224", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct stm32_hash_ctx), + .cra_alignmask = 3, + .cra_init = stm32_hash_cra_sha224_init, + .cra_module = THIS_MODULE, + } + } + }, + { + .init = stm32_hash_init, + .update = stm32_hash_update, + .final = stm32_hash_final, + .finup = stm32_hash_finup, + .digest = stm32_hash_digest, + .export = stm32_hash_export, + .import = stm32_hash_import, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct stm32_hash_request_ctx), + .base = { + .cra_name = "sha256", + .cra_driver_name = "stm32-sha256", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct stm32_hash_ctx), + .cra_alignmask = 3, + .cra_init = stm32_hash_cra_init, + .cra_module = THIS_MODULE, + } + } + }, + { + .init = stm32_hash_init, + .update = stm32_hash_update, + .final = stm32_hash_final, + .finup = stm32_hash_finup, + .digest = stm32_hash_digest, + .export = stm32_hash_export, + .import = stm32_hash_import, + .setkey = stm32_hash_setkey, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct stm32_hash_request_ctx), + .base = { + .cra_name = "hmac(sha256)", + .cra_driver_name = "stm32-hmac-sha256", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct stm32_hash_ctx), + .cra_alignmask = 3, + .cra_init = stm32_hash_cra_sha256_init, + .cra_module = THIS_MODULE, + } + } + }, +}; + +static int stm32_hash_register_algs(struct stm32_hash_dev *hdev) +{ + unsigned int i, j; + int err; + + for (i = 0; i < hdev->pdata->algs_info_size; i++) { + for (j = 0; j < hdev->pdata->algs_info[i].size; j++) { + err = crypto_register_ahash( + &hdev->pdata->algs_info[i].algs_list[j]); + if (err) + goto err_algs; + } + } + + return 0; +err_algs: + dev_err(hdev->dev, "Algo %d : %d failed\n", i, j); + for (; i--; ) { + for (; j--;) + crypto_unregister_ahash( + &hdev->pdata->algs_info[i].algs_list[j]); + } + + return err; +} + +static int stm32_hash_unregister_algs(struct stm32_hash_dev *hdev) +{ + unsigned int i, j; + + for (i = 0; i < hdev->pdata->algs_info_size; i++) { + for (j = 0; j < hdev->pdata->algs_info[i].size; j++) + crypto_unregister_ahash( + &hdev->pdata->algs_info[i].algs_list[j]); + } + + return 0; +} + +static struct stm32_hash_algs_info stm32_hash_algs_info_stm32f4[] = { + { + .algs_list = algs_md5_sha1, + .size = ARRAY_SIZE(algs_md5_sha1), + }, +}; + +static const struct stm32_hash_pdata stm32_hash_pdata_stm32f4 = { + .algs_info = stm32_hash_algs_info_stm32f4, + .algs_info_size = ARRAY_SIZE(stm32_hash_algs_info_stm32f4), +}; + +static struct stm32_hash_algs_info stm32_hash_algs_info_stm32f7[] = { + { + .algs_list = algs_md5_sha1, + .size = ARRAY_SIZE(algs_md5_sha1), + }, + { + .algs_list = algs_sha224_sha256, + .size = ARRAY_SIZE(algs_sha224_sha256), + }, +}; + +static const struct stm32_hash_pdata stm32_hash_pdata_stm32f7 = { + .algs_info = stm32_hash_algs_info_stm32f7, + .algs_info_size = ARRAY_SIZE(stm32_hash_algs_info_stm32f7), +}; + +static const struct of_device_id stm32_hash_of_match[] = { + { + .compatible = "st,stm32f456-hash", + .data = &stm32_hash_pdata_stm32f4, + }, + { + .compatible = "st,stm32f756-hash", + .data = &stm32_hash_pdata_stm32f7, + }, + {}, +}; + +MODULE_DEVICE_TABLE(of, stm32_hash_of_match); + +static int stm32_hash_get_of_match(struct stm32_hash_dev *hdev, + struct device *dev) +{ + int err; + + hdev->pdata = of_device_get_match_data(dev); + if (!hdev->pdata) { + dev_err(dev, "no compatible OF match\n"); + return -EINVAL; + } + + err = of_property_read_u32(dev->of_node, "dma-maxburst", + &hdev->dma_maxburst); + + return err; +} + +static int stm32_hash_probe(struct platform_device *pdev) +{ + struct stm32_hash_dev *hdev; + struct device *dev = &pdev->dev; + struct resource *res; + int ret, irq; + + hdev = devm_kzalloc(dev, sizeof(*hdev), GFP_KERNEL); + if (!hdev) + return -ENOMEM; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + hdev->io_base = devm_ioremap_resource(dev, res); + if (IS_ERR(hdev->io_base)) + return PTR_ERR(hdev->io_base); + + hdev->phys_base = res->start; + + ret = stm32_hash_get_of_match(hdev, dev); + if (ret) + return ret; + + irq = platform_get_irq(pdev, 0); + if (irq < 0) { + dev_err(dev, "Cannot get IRQ resource\n"); + return irq; + } + + ret = devm_request_threaded_irq(dev, irq, stm32_hash_irq_handler, + stm32_hash_irq_thread, IRQF_ONESHOT, + dev_name(dev), hdev); + if (ret) { + dev_err(dev, "Cannot grab IRQ\n"); + return ret; + } + + hdev->clk = devm_clk_get(&pdev->dev, NULL); + if (IS_ERR(hdev->clk)) { + dev_err(dev, "failed to get clock for hash (%lu)\n", + PTR_ERR(hdev->clk)); + return PTR_ERR(hdev->clk); + } + + ret = clk_prepare_enable(hdev->clk); + if (ret) { + dev_err(dev, "failed to enable hash clock (%d)\n", ret); + return ret; + } + + hdev->rst = devm_reset_control_get(&pdev->dev, NULL); + if (!IS_ERR(hdev->rst)) { + reset_control_assert(hdev->rst); + udelay(2); + reset_control_deassert(hdev->rst); + } + + hdev->dev = dev; + + platform_set_drvdata(pdev, hdev); + + ret = stm32_hash_dma_init(hdev); + if (ret) + dev_dbg(dev, "DMA mode not available\n"); + + spin_lock(&stm32_hash.lock); + list_add_tail(&hdev->list, &stm32_hash.dev_list); + spin_unlock(&stm32_hash.lock); + + /* Initialize crypto engine */ + hdev->engine = crypto_engine_alloc_init(dev, 1); + if (!hdev->engine) { + ret = -ENOMEM; + goto err_engine; + } + + hdev->engine->prepare_hash_request = stm32_hash_prepare_req; + hdev->engine->hash_one_request = stm32_hash_one_request; + + ret = crypto_engine_start(hdev->engine); + if (ret) + goto err_engine_start; + + hdev->dma_mode = stm32_hash_read(hdev, HASH_HWCFGR); + + /* Register algos */ + ret = stm32_hash_register_algs(hdev); + if (ret) + goto err_algs; + + dev_info(dev, "Init HASH done HW ver %x DMA mode %u\n", + stm32_hash_read(hdev, HASH_VER), hdev->dma_mode); + + return 0; + +err_algs: +err_engine_start: + crypto_engine_exit(hdev->engine); +err_engine: + spin_lock(&stm32_hash.lock); + list_del(&hdev->list); + spin_unlock(&stm32_hash.lock); + + if (hdev->dma_lch) + dma_release_channel(hdev->dma_lch); + + clk_disable_unprepare(hdev->clk); + + return ret; +} + +static int stm32_hash_remove(struct platform_device *pdev) +{ + static struct stm32_hash_dev *hdev; + + hdev = platform_get_drvdata(pdev); + if (!hdev) + return -ENODEV; + + stm32_hash_unregister_algs(hdev); + + crypto_engine_exit(hdev->engine); + + spin_lock(&stm32_hash.lock); + list_del(&hdev->list); + spin_unlock(&stm32_hash.lock); + + if (hdev->dma_lch) + dma_release_channel(hdev->dma_lch); + + clk_disable_unprepare(hdev->clk); + + return 0; +} + +static struct platform_driver stm32_hash_driver = { + .probe = stm32_hash_probe, + .remove = stm32_hash_remove, + .driver = { + .name = "stm32-hash", + .of_match_table = stm32_hash_of_match, + } +}; + +module_platform_driver(stm32_hash_driver); + +MODULE_DESCRIPTION("STM32 SHA1/224/256 & MD5 (HMAC) hw accelerator driver"); +MODULE_AUTHOR("Lionel Debieve <lionel.debieve@st.com>"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/crypto/stm32/stm32_crc32.c b/drivers/crypto/stm32/stm32_crc32.c index ec83b1e6bfe8..090582baecfe 100644 --- a/drivers/crypto/stm32/stm32_crc32.c +++ b/drivers/crypto/stm32/stm32_crc32.c @@ -107,12 +107,12 @@ static int stm32_crc_init(struct shash_desc *desc) spin_unlock_bh(&crc_list.lock); /* Reset, set key, poly and configure in bit reverse mode */ - writel(bitrev32(mctx->key), ctx->crc->regs + CRC_INIT); - writel(bitrev32(mctx->poly), ctx->crc->regs + CRC_POL); - writel(CRC_CR_RESET | CRC_CR_REVERSE, ctx->crc->regs + CRC_CR); + writel_relaxed(bitrev32(mctx->key), ctx->crc->regs + CRC_INIT); + writel_relaxed(bitrev32(mctx->poly), ctx->crc->regs + CRC_POL); + writel_relaxed(CRC_CR_RESET | CRC_CR_REVERSE, ctx->crc->regs + CRC_CR); /* Store partial result */ - ctx->partial = readl(ctx->crc->regs + CRC_DR); + ctx->partial = readl_relaxed(ctx->crc->regs + CRC_DR); ctx->crc->nb_pending_bytes = 0; return 0; @@ -135,7 +135,8 @@ static int stm32_crc_update(struct shash_desc *desc, const u8 *d8, if (crc->nb_pending_bytes == sizeof(u32)) { /* Process completed pending data */ - writel(*(u32 *)crc->pending_data, crc->regs + CRC_DR); + writel_relaxed(*(u32 *)crc->pending_data, + crc->regs + CRC_DR); crc->nb_pending_bytes = 0; } } @@ -143,10 +144,10 @@ static int stm32_crc_update(struct shash_desc *desc, const u8 *d8, d32 = (u32 *)d8; for (i = 0; i < length >> 2; i++) /* Process 32 bits data */ - writel(*(d32++), crc->regs + CRC_DR); + writel_relaxed(*(d32++), crc->regs + CRC_DR); /* Store partial result */ - ctx->partial = readl(crc->regs + CRC_DR); + ctx->partial = readl_relaxed(crc->regs + CRC_DR); /* Check for pending data (non 32 bits) */ length &= 3; @@ -295,7 +296,7 @@ static int stm32_crc_remove(struct platform_device *pdev) list_del(&crc->list); spin_unlock(&crc_list.lock); - crypto_unregister_shash(algs); + crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); clk_disable_unprepare(crc->clk); diff --git a/drivers/crypto/sunxi-ss/Makefile b/drivers/crypto/sunxi-ss/Makefile index 8f4c7a273141..ccb893219079 100644 --- a/drivers/crypto/sunxi-ss/Makefile +++ b/drivers/crypto/sunxi-ss/Makefile @@ -1,2 +1,3 @@ obj-$(CONFIG_CRYPTO_DEV_SUN4I_SS) += sun4i-ss.o sun4i-ss-y += sun4i-ss-core.o sun4i-ss-hash.o sun4i-ss-cipher.o +sun4i-ss-$(CONFIG_CRYPTO_DEV_SUN4I_SS_PRNG) += sun4i-ss-prng.o diff --git a/drivers/crypto/sunxi-ss/sun4i-ss-core.c b/drivers/crypto/sunxi-ss/sun4i-ss-core.c index 02ad8256e900..1547cbe13dc2 100644 --- a/drivers/crypto/sunxi-ss/sun4i-ss-core.c +++ b/drivers/crypto/sunxi-ss/sun4i-ss-core.c @@ -213,6 +213,23 @@ static struct sun4i_ss_alg_template ss_algs[] = { } } }, +#ifdef CONFIG_CRYPTO_DEV_SUN4I_SS_PRNG +{ + .type = CRYPTO_ALG_TYPE_RNG, + .alg.rng = { + .base = { + .cra_name = "stdrng", + .cra_driver_name = "sun4i_ss_rng", + .cra_priority = 300, + .cra_ctxsize = 0, + .cra_module = THIS_MODULE, + }, + .generate = sun4i_ss_prng_generate, + .seed = sun4i_ss_prng_seed, + .seedsize = SS_SEED_LEN / BITS_PER_BYTE, + } +}, +#endif }; static int sun4i_ss_probe(struct platform_device *pdev) @@ -355,6 +372,13 @@ static int sun4i_ss_probe(struct platform_device *pdev) goto error_alg; } break; + case CRYPTO_ALG_TYPE_RNG: + err = crypto_register_rng(&ss_algs[i].alg.rng); + if (err) { + dev_err(ss->dev, "Fail to register %s\n", + ss_algs[i].alg.rng.base.cra_name); + } + break; } } platform_set_drvdata(pdev, ss); @@ -369,6 +393,9 @@ error_alg: case CRYPTO_ALG_TYPE_AHASH: crypto_unregister_ahash(&ss_algs[i].alg.hash); break; + case CRYPTO_ALG_TYPE_RNG: + crypto_unregister_rng(&ss_algs[i].alg.rng); + break; } } if (ss->reset) @@ -393,6 +420,9 @@ static int sun4i_ss_remove(struct platform_device *pdev) case CRYPTO_ALG_TYPE_AHASH: crypto_unregister_ahash(&ss_algs[i].alg.hash); break; + case CRYPTO_ALG_TYPE_RNG: + crypto_unregister_rng(&ss_algs[i].alg.rng); + break; } } diff --git a/drivers/crypto/sunxi-ss/sun4i-ss-prng.c b/drivers/crypto/sunxi-ss/sun4i-ss-prng.c new file mode 100644 index 000000000000..0d01d1624252 --- /dev/null +++ b/drivers/crypto/sunxi-ss/sun4i-ss-prng.c @@ -0,0 +1,56 @@ +#include "sun4i-ss.h" + +int sun4i_ss_prng_seed(struct crypto_rng *tfm, const u8 *seed, + unsigned int slen) +{ + struct sun4i_ss_alg_template *algt; + struct rng_alg *alg = crypto_rng_alg(tfm); + + algt = container_of(alg, struct sun4i_ss_alg_template, alg.rng); + memcpy(algt->ss->seed, seed, slen); + + return 0; +} + +int sun4i_ss_prng_generate(struct crypto_rng *tfm, const u8 *src, + unsigned int slen, u8 *dst, unsigned int dlen) +{ + struct sun4i_ss_alg_template *algt; + struct rng_alg *alg = crypto_rng_alg(tfm); + int i; + u32 v; + u32 *data = (u32 *)dst; + const u32 mode = SS_OP_PRNG | SS_PRNG_CONTINUE | SS_ENABLED; + size_t len; + struct sun4i_ss_ctx *ss; + unsigned int todo = (dlen / 4) * 4; + + algt = container_of(alg, struct sun4i_ss_alg_template, alg.rng); + ss = algt->ss; + + spin_lock(&ss->slock); + + writel(mode, ss->base + SS_CTL); + + while (todo > 0) { + /* write the seed */ + for (i = 0; i < SS_SEED_LEN / BITS_PER_LONG; i++) + writel(ss->seed[i], ss->base + SS_KEY0 + i * 4); + + /* Read the random data */ + len = min_t(size_t, SS_DATA_LEN / BITS_PER_BYTE, todo); + readsl(ss->base + SS_TXFIFO, data, len / 4); + data += len / 4; + todo -= len; + + /* Update the seed */ + for (i = 0; i < SS_SEED_LEN / BITS_PER_LONG; i++) { + v = readl(ss->base + SS_KEY0 + i * 4); + ss->seed[i] = v; + } + } + + writel(0, ss->base + SS_CTL); + spin_unlock(&ss->slock); + return dlen; +} diff --git a/drivers/crypto/sunxi-ss/sun4i-ss.h b/drivers/crypto/sunxi-ss/sun4i-ss.h index a0e1efc1cb2a..f3ac90692ac6 100644 --- a/drivers/crypto/sunxi-ss/sun4i-ss.h +++ b/drivers/crypto/sunxi-ss/sun4i-ss.h @@ -32,6 +32,7 @@ #include <crypto/aes.h> #include <crypto/des.h> #include <crypto/internal/rng.h> +#include <crypto/rng.h> #define SS_CTL 0x00 #define SS_KEY0 0x04 @@ -127,6 +128,9 @@ #define SS_RXFIFO_EMP_INT_ENABLE (1 << 2) #define SS_TXFIFO_AVA_INT_ENABLE (1 << 0) +#define SS_SEED_LEN 192 +#define SS_DATA_LEN 160 + struct sun4i_ss_ctx { void __iomem *base; int irq; @@ -136,6 +140,9 @@ struct sun4i_ss_ctx { struct device *dev; struct resource *res; spinlock_t slock; /* control the use of the device */ +#ifdef CONFIG_CRYPTO_DEV_SUN4I_SS_PRNG + u32 seed[SS_SEED_LEN / BITS_PER_LONG]; +#endif }; struct sun4i_ss_alg_template { @@ -144,6 +151,7 @@ struct sun4i_ss_alg_template { union { struct skcipher_alg crypto; struct ahash_alg hash; + struct rng_alg rng; } alg; struct sun4i_ss_ctx *ss; }; @@ -201,3 +209,6 @@ int sun4i_ss_des_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen); int sun4i_ss_des3_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen); +int sun4i_ss_prng_generate(struct crypto_rng *tfm, const u8 *src, + unsigned int slen, u8 *dst, unsigned int dlen); +int sun4i_ss_prng_seed(struct crypto_rng *tfm, const u8 *seed, unsigned int slen); diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index 79791c690858..9c80e0cb1664 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -56,29 +56,26 @@ #include "talitos.h" static void to_talitos_ptr(struct talitos_ptr *ptr, dma_addr_t dma_addr, - bool is_sec1) + unsigned int len, bool is_sec1) { ptr->ptr = cpu_to_be32(lower_32_bits(dma_addr)); - if (!is_sec1) + if (is_sec1) { + ptr->len1 = cpu_to_be16(len); + } else { + ptr->len = cpu_to_be16(len); ptr->eptr = upper_32_bits(dma_addr); + } } static void copy_talitos_ptr(struct talitos_ptr *dst_ptr, struct talitos_ptr *src_ptr, bool is_sec1) { dst_ptr->ptr = src_ptr->ptr; - if (!is_sec1) - dst_ptr->eptr = src_ptr->eptr; -} - -static void to_talitos_ptr_len(struct talitos_ptr *ptr, unsigned int len, - bool is_sec1) -{ if (is_sec1) { - ptr->res = 0; - ptr->len1 = cpu_to_be16(len); + dst_ptr->len1 = src_ptr->len1; } else { - ptr->len = cpu_to_be16(len); + dst_ptr->len = src_ptr->len; + dst_ptr->eptr = src_ptr->eptr; } } @@ -116,9 +113,7 @@ static void map_single_talitos_ptr(struct device *dev, struct talitos_private *priv = dev_get_drvdata(dev); bool is_sec1 = has_ftr_sec1(priv); - to_talitos_ptr_len(ptr, len, is_sec1); - to_talitos_ptr(ptr, dma_addr, is_sec1); - to_talitos_ptr_ext_set(ptr, 0, is_sec1); + to_talitos_ptr(ptr, dma_addr, len, is_sec1); } /* @@ -165,6 +160,10 @@ static int reset_channel(struct device *dev, int ch) /* set 36-bit addressing, done writeback enable and done IRQ enable */ setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, TALITOS_CCCR_LO_EAE | TALITOS_CCCR_LO_CDWE | TALITOS_CCCR_LO_CDIE); + /* enable chaining descriptors */ + if (is_sec1) + setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, + TALITOS_CCCR_LO_NE); /* and ICCR writeback, if available */ if (priv->features & TALITOS_FTR_HW_AUTH_CHECK) @@ -287,7 +286,6 @@ int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc, /* map descriptor and save caller data */ if (is_sec1) { desc->hdr1 = desc->hdr; - desc->next_desc = 0; request->dma_desc = dma_map_single(dev, &desc->hdr1, TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL); @@ -339,7 +337,12 @@ static void flush_channel(struct device *dev, int ch, int error, int reset_ch) /* descriptors with their done bits set don't get the error */ rmb(); - hdr = is_sec1 ? request->desc->hdr1 : request->desc->hdr; + if (!is_sec1) + hdr = request->desc->hdr; + else if (request->desc->next_desc) + hdr = (request->desc + 1)->hdr1; + else + hdr = request->desc->hdr1; if ((hdr & DESC_HDR_DONE) == DESC_HDR_DONE) status = 0; @@ -393,8 +396,6 @@ static void talitos1_done_##name(unsigned long data) \ \ if (ch_done_mask & 0x10000000) \ flush_channel(dev, 0, 0, 0); \ - if (priv->num_channels == 1) \ - goto out; \ if (ch_done_mask & 0x40000000) \ flush_channel(dev, 1, 0, 0); \ if (ch_done_mask & 0x00010000) \ @@ -402,7 +403,6 @@ static void talitos1_done_##name(unsigned long data) \ if (ch_done_mask & 0x00040000) \ flush_channel(dev, 3, 0, 0); \ \ -out: \ /* At this point, all completed channels have been processed */ \ /* Unmask done interrupts for channels completed later on. */ \ spin_lock_irqsave(&priv->reg_lock, flags); \ @@ -412,6 +412,7 @@ out: \ } DEF_TALITOS1_DONE(4ch, TALITOS1_ISR_4CHDONE) +DEF_TALITOS1_DONE(ch0, TALITOS1_ISR_CH_0_DONE) #define DEF_TALITOS2_DONE(name, ch_done_mask) \ static void talitos2_done_##name(unsigned long data) \ @@ -422,8 +423,6 @@ static void talitos2_done_##name(unsigned long data) \ \ if (ch_done_mask & 1) \ flush_channel(dev, 0, 0, 0); \ - if (priv->num_channels == 1) \ - goto out; \ if (ch_done_mask & (1 << 2)) \ flush_channel(dev, 1, 0, 0); \ if (ch_done_mask & (1 << 4)) \ @@ -431,7 +430,6 @@ static void talitos2_done_##name(unsigned long data) \ if (ch_done_mask & (1 << 6)) \ flush_channel(dev, 3, 0, 0); \ \ -out: \ /* At this point, all completed channels have been processed */ \ /* Unmask done interrupts for channels completed later on. */ \ spin_lock_irqsave(&priv->reg_lock, flags); \ @@ -441,6 +439,7 @@ out: \ } DEF_TALITOS2_DONE(4ch, TALITOS2_ISR_4CHDONE) +DEF_TALITOS2_DONE(ch0, TALITOS2_ISR_CH_0_DONE) DEF_TALITOS2_DONE(ch0_2, TALITOS2_ISR_CH_0_2_DONE) DEF_TALITOS2_DONE(ch1_3, TALITOS2_ISR_CH_1_3_DONE) @@ -464,7 +463,8 @@ static u32 current_desc_hdr(struct device *dev, int ch) tail = priv->chan[ch].tail; iter = tail; - while (priv->chan[ch].fifo[iter].dma_desc != cur_desc) { + while (priv->chan[ch].fifo[iter].dma_desc != cur_desc && + priv->chan[ch].fifo[iter].desc->next_desc != cur_desc) { iter = (iter + 1) & (priv->fifo_len - 1); if (iter == tail) { dev_err(dev, "couldn't locate current descriptor\n"); @@ -472,6 +472,9 @@ static u32 current_desc_hdr(struct device *dev, int ch) } } + if (priv->chan[ch].fifo[iter].desc->next_desc == cur_desc) + return (priv->chan[ch].fifo[iter].desc + 1)->hdr; + return priv->chan[ch].fifo[iter].desc->hdr; } @@ -825,9 +828,12 @@ struct talitos_ctx { __be32 desc_hdr_template; u8 key[TALITOS_MAX_KEY_SIZE]; u8 iv[TALITOS_MAX_IV_LENGTH]; + dma_addr_t dma_key; unsigned int keylen; unsigned int enckeylen; unsigned int authkeylen; + dma_addr_t dma_buf; + dma_addr_t dma_hw_context; }; #define HASH_MAX_BLOCK_SIZE SHA512_BLOCK_SIZE @@ -836,8 +842,8 @@ struct talitos_ctx { struct talitos_ahash_req_ctx { u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)]; unsigned int hw_context_size; - u8 buf[HASH_MAX_BLOCK_SIZE]; - u8 bufnext[HASH_MAX_BLOCK_SIZE]; + u8 buf[2][HASH_MAX_BLOCK_SIZE]; + int buf_idx; unsigned int swinit; unsigned int first; unsigned int last; @@ -861,6 +867,7 @@ static int aead_setkey(struct crypto_aead *authenc, const u8 *key, unsigned int keylen) { struct talitos_ctx *ctx = crypto_aead_ctx(authenc); + struct device *dev = ctx->dev; struct crypto_authenc_keys keys; if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) @@ -869,12 +876,17 @@ static int aead_setkey(struct crypto_aead *authenc, if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE) goto badkey; + if (ctx->keylen) + dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE); + memcpy(ctx->key, keys.authkey, keys.authkeylen); memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen); ctx->keylen = keys.authkeylen + keys.enckeylen; ctx->enckeylen = keys.enckeylen; ctx->authkeylen = keys.authkeylen; + ctx->dma_key = dma_map_single(dev, ctx->key, ctx->keylen, + DMA_TO_DEVICE); return 0; @@ -948,13 +960,13 @@ static void ipsec_esp_unmap(struct device *dev, struct crypto_aead *aead = crypto_aead_reqtfm(areq); struct talitos_ctx *ctx = crypto_aead_ctx(aead); unsigned int ivsize = crypto_aead_ivsize(aead); + bool is_ipsec_esp = edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP; + struct talitos_ptr *civ_ptr = &edesc->desc.ptr[is_ipsec_esp ? 2 : 3]; - if (edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP) + if (is_ipsec_esp) unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6], DMA_FROM_DEVICE); - unmap_single_talitos_ptr(dev, &edesc->desc.ptr[3], DMA_TO_DEVICE); - unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE); - unmap_single_talitos_ptr(dev, &edesc->desc.ptr[0], DMA_TO_DEVICE); + unmap_single_talitos_ptr(dev, civ_ptr, DMA_TO_DEVICE); talitos_sg_unmap(dev, edesc, areq->src, areq->dst, areq->cryptlen, areq->assoclen); @@ -963,7 +975,7 @@ static void ipsec_esp_unmap(struct device *dev, dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len, DMA_BIDIRECTIONAL); - if (!(edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP)) { + if (!is_ipsec_esp) { unsigned int dst_nents = edesc->dst_nents ? : 1; sg_pcopy_to_buffer(areq->dst, dst_nents, ctx->iv, ivsize, @@ -983,6 +995,7 @@ static void ipsec_esp_encrypt_done(struct device *dev, struct aead_request *areq = context; struct crypto_aead *authenc = crypto_aead_reqtfm(areq); unsigned int authsize = crypto_aead_authsize(authenc); + unsigned int ivsize = crypto_aead_ivsize(authenc); struct talitos_edesc *edesc; struct scatterlist *sg; void *icvdata; @@ -1003,6 +1016,8 @@ static void ipsec_esp_encrypt_done(struct device *dev, icvdata, authsize); } + dma_unmap_single(dev, edesc->iv_dma, ivsize, DMA_TO_DEVICE); + kfree(edesc); aead_request_complete(areq, err); @@ -1097,8 +1112,7 @@ static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count, len = cryptlen; to_talitos_ptr(link_tbl_ptr + count, - sg_dma_address(sg) + offset, 0); - to_talitos_ptr_len(link_tbl_ptr + count, len, 0); + sg_dma_address(sg) + offset, len, 0); to_talitos_ptr_ext_set(link_tbl_ptr + count, 0, 0); count++; cryptlen -= len; @@ -1116,7 +1130,7 @@ next: return count; } -int talitos_sg_map(struct device *dev, struct scatterlist *src, +static int talitos_sg_map(struct device *dev, struct scatterlist *src, unsigned int len, struct talitos_edesc *edesc, struct talitos_ptr *ptr, int sg_count, unsigned int offset, int tbl_off) @@ -1124,15 +1138,12 @@ int talitos_sg_map(struct device *dev, struct scatterlist *src, struct talitos_private *priv = dev_get_drvdata(dev); bool is_sec1 = has_ftr_sec1(priv); - to_talitos_ptr_len(ptr, len, is_sec1); - to_talitos_ptr_ext_set(ptr, 0, is_sec1); - if (sg_count == 1) { - to_talitos_ptr(ptr, sg_dma_address(src) + offset, is_sec1); + to_talitos_ptr(ptr, sg_dma_address(src) + offset, len, is_sec1); return sg_count; } if (is_sec1) { - to_talitos_ptr(ptr, edesc->dma_link_tbl + offset, is_sec1); + to_talitos_ptr(ptr, edesc->dma_link_tbl + offset, len, is_sec1); return sg_count; } sg_count = sg_to_link_tbl_offset(src, sg_count, offset, len, @@ -1143,7 +1154,7 @@ int talitos_sg_map(struct device *dev, struct scatterlist *src, return sg_count; } to_talitos_ptr(ptr, edesc->dma_link_tbl + - tbl_off * sizeof(struct talitos_ptr), is_sec1); + tbl_off * sizeof(struct talitos_ptr), len, is_sec1); to_talitos_ptr_ext_or(ptr, DESC_PTR_LNKTBL_JUMP, is_sec1); return sg_count; @@ -1170,10 +1181,12 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, bool sync_needed = false; struct talitos_private *priv = dev_get_drvdata(dev); bool is_sec1 = has_ftr_sec1(priv); + bool is_ipsec_esp = desc->hdr & DESC_HDR_TYPE_IPSEC_ESP; + struct talitos_ptr *civ_ptr = &desc->ptr[is_ipsec_esp ? 2 : 3]; + struct talitos_ptr *ckey_ptr = &desc->ptr[is_ipsec_esp ? 3 : 2]; /* hmac key */ - map_single_talitos_ptr(dev, &desc->ptr[0], ctx->authkeylen, &ctx->key, - DMA_TO_DEVICE); + to_talitos_ptr(&desc->ptr[0], ctx->dma_key, ctx->authkeylen, is_sec1); sg_count = edesc->src_nents ?: 1; if (is_sec1 && sg_count > 1) @@ -1194,25 +1207,11 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, } /* cipher iv */ - if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP) { - to_talitos_ptr(&desc->ptr[2], edesc->iv_dma, is_sec1); - to_talitos_ptr_len(&desc->ptr[2], ivsize, is_sec1); - to_talitos_ptr_ext_set(&desc->ptr[2], 0, is_sec1); - } else { - to_talitos_ptr(&desc->ptr[3], edesc->iv_dma, is_sec1); - to_talitos_ptr_len(&desc->ptr[3], ivsize, is_sec1); - to_talitos_ptr_ext_set(&desc->ptr[3], 0, is_sec1); - } + to_talitos_ptr(civ_ptr, edesc->iv_dma, ivsize, is_sec1); /* cipher key */ - if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP) - map_single_talitos_ptr(dev, &desc->ptr[3], ctx->enckeylen, - (char *)&ctx->key + ctx->authkeylen, - DMA_TO_DEVICE); - else - map_single_talitos_ptr(dev, &desc->ptr[2], ctx->enckeylen, - (char *)&ctx->key + ctx->authkeylen, - DMA_TO_DEVICE); + to_talitos_ptr(ckey_ptr, ctx->dma_key + ctx->authkeylen, + ctx->enckeylen, is_sec1); /* * cipher in @@ -1220,24 +1219,20 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, * extent is bytes of HMAC postpended to ciphertext, * typically 12 for ipsec */ - to_talitos_ptr_len(&desc->ptr[4], cryptlen, is_sec1); - to_talitos_ptr_ext_set(&desc->ptr[4], 0, is_sec1); - sg_link_tbl_len = cryptlen; - if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP) { + if (is_ipsec_esp) { to_talitos_ptr_ext_set(&desc->ptr[4], authsize, is_sec1); - if (edesc->desc.hdr & DESC_HDR_MODE1_MDEU_CICV) + if (desc->hdr & DESC_HDR_MODE1_MDEU_CICV) sg_link_tbl_len += authsize; } - sg_count = talitos_sg_map(dev, areq->src, cryptlen, edesc, - &desc->ptr[4], sg_count, areq->assoclen, - tbl_off); + ret = talitos_sg_map(dev, areq->src, sg_link_tbl_len, edesc, + &desc->ptr[4], sg_count, areq->assoclen, tbl_off); - if (sg_count > 1) { - tbl_off += sg_count; + if (ret > 1) { + tbl_off += ret; sync_needed = true; } @@ -1248,47 +1243,59 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE); } - sg_count = talitos_sg_map(dev, areq->dst, cryptlen, edesc, - &desc->ptr[5], sg_count, areq->assoclen, - tbl_off); + ret = talitos_sg_map(dev, areq->dst, cryptlen, edesc, &desc->ptr[5], + sg_count, areq->assoclen, tbl_off); - if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP) + if (is_ipsec_esp) to_talitos_ptr_ext_or(&desc->ptr[5], authsize, is_sec1); - if (sg_count > 1) { + /* ICV data */ + if (ret > 1) { + tbl_off += ret; edesc->icv_ool = true; sync_needed = true; - if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP) { + if (is_ipsec_esp) { struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off]; int offset = (edesc->src_nents + edesc->dst_nents + 2) * sizeof(struct talitos_ptr) + authsize; /* Add an entry to the link table for ICV data */ - tbl_ptr += sg_count - 1; - to_talitos_ptr_ext_set(tbl_ptr, 0, is_sec1); - tbl_ptr++; + to_talitos_ptr_ext_set(tbl_ptr - 1, 0, is_sec1); to_talitos_ptr_ext_set(tbl_ptr, DESC_PTR_LNKTBL_RETURN, is_sec1); - to_talitos_ptr_len(tbl_ptr, authsize, is_sec1); /* icv data follows link tables */ to_talitos_ptr(tbl_ptr, edesc->dma_link_tbl + offset, - is_sec1); + authsize, is_sec1); + } else { + dma_addr_t addr = edesc->dma_link_tbl; + + if (is_sec1) + addr += areq->assoclen + cryptlen; + else + addr += sizeof(struct talitos_ptr) * tbl_off; + + to_talitos_ptr(&desc->ptr[6], addr, authsize, is_sec1); + } + } else if (!is_ipsec_esp) { + ret = talitos_sg_map(dev, areq->dst, authsize, edesc, + &desc->ptr[6], sg_count, areq->assoclen + + cryptlen, + tbl_off); + if (ret > 1) { + tbl_off += ret; + edesc->icv_ool = true; + sync_needed = true; + } else { + edesc->icv_ool = false; } } else { edesc->icv_ool = false; } - /* ICV data */ - if (!(desc->hdr & DESC_HDR_TYPE_IPSEC_ESP)) { - to_talitos_ptr_len(&desc->ptr[6], authsize, is_sec1); - to_talitos_ptr(&desc->ptr[6], edesc->dma_link_tbl + - areq->assoclen + cryptlen, is_sec1); - } - /* iv out */ - if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP) + if (is_ipsec_esp) map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv, DMA_FROM_DEVICE); @@ -1387,22 +1394,31 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, alloc_len += icv_stashing ? authsize : 0; } + /* if its a ahash, add space for a second desc next to the first one */ + if (is_sec1 && !dst) + alloc_len += sizeof(struct talitos_desc); + edesc = kmalloc(alloc_len, GFP_DMA | flags); if (!edesc) { dev_err(dev, "could not allocate edescriptor\n"); err = ERR_PTR(-ENOMEM); goto error_sg; } + memset(&edesc->desc, 0, sizeof(edesc->desc)); edesc->src_nents = src_nents; edesc->dst_nents = dst_nents; edesc->iv_dma = iv_dma; edesc->dma_len = dma_len; - if (dma_len) - edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0], + if (dma_len) { + void *addr = &edesc->link_tbl[0]; + + if (is_sec1 && !dst) + addr += sizeof(struct talitos_desc); + edesc->dma_link_tbl = dma_map_single(dev, addr, edesc->dma_len, DMA_BIDIRECTIONAL); - + } return edesc; error_sg: if (iv_dma) @@ -1468,7 +1484,6 @@ static int aead_decrypt(struct aead_request *req) DESC_HDR_MODE1_MDEU_CICV; /* reset integrity check result bits */ - edesc->desc.hdr_lo = 0; return ipsec_esp(edesc, req, ipsec_esp_decrypt_hwauth_done); } @@ -1494,15 +1509,29 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *cipher, const u8 *key, unsigned int keylen) { struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher); + struct device *dev = ctx->dev; + u32 tmp[DES_EXPKEY_WORDS]; if (keylen > TALITOS_MAX_KEY_SIZE) { crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); return -EINVAL; } + if (unlikely(crypto_ablkcipher_get_flags(cipher) & + CRYPTO_TFM_REQ_WEAK_KEY) && + !des_ekey(tmp, key)) { + crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_WEAK_KEY); + return -EINVAL; + } + + if (ctx->keylen) + dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE); + memcpy(&ctx->key, key, keylen); ctx->keylen = keylen; + ctx->dma_key = dma_map_single(dev, ctx->key, keylen, DMA_TO_DEVICE); + return 0; } @@ -1513,7 +1542,6 @@ static void common_nonsnoop_unmap(struct device *dev, unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE); talitos_sg_unmap(dev, edesc, areq->src, areq->dst, areq->nbytes, 0); - unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE); unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], DMA_TO_DEVICE); if (edesc->dma_len) @@ -1555,16 +1583,12 @@ static int common_nonsnoop(struct talitos_edesc *edesc, bool is_sec1 = has_ftr_sec1(priv); /* first DWORD empty */ - desc->ptr[0] = zero_entry; /* cipher iv */ - to_talitos_ptr(&desc->ptr[1], edesc->iv_dma, is_sec1); - to_talitos_ptr_len(&desc->ptr[1], ivsize, is_sec1); - to_talitos_ptr_ext_set(&desc->ptr[1], 0, is_sec1); + to_talitos_ptr(&desc->ptr[1], edesc->iv_dma, ivsize, is_sec1); /* cipher key */ - map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen, - (char *)&ctx->key, DMA_TO_DEVICE); + to_talitos_ptr(&desc->ptr[2], ctx->dma_key, ctx->keylen, is_sec1); sg_count = edesc->src_nents ?: 1; if (is_sec1 && sg_count > 1) @@ -1599,7 +1623,6 @@ static int common_nonsnoop(struct talitos_edesc *edesc, DMA_FROM_DEVICE); /* last DWORD empty */ - desc->ptr[6] = zero_entry; if (sync_needed) dma_sync_single_for_device(dev, edesc->dma_link_tbl, @@ -1663,26 +1686,16 @@ static void common_nonsnoop_hash_unmap(struct device *dev, struct ahash_request *areq) { struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); - struct talitos_private *priv = dev_get_drvdata(dev); - bool is_sec1 = has_ftr_sec1(priv); - - unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE); talitos_sg_unmap(dev, edesc, req_ctx->psrc, NULL, 0, 0); - /* When using hashctx-in, must unmap it. */ - if (from_talitos_ptr_len(&edesc->desc.ptr[1], is_sec1)) - unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], - DMA_TO_DEVICE); - - if (from_talitos_ptr_len(&edesc->desc.ptr[2], is_sec1)) - unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], - DMA_TO_DEVICE); - if (edesc->dma_len) dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len, DMA_BIDIRECTIONAL); + if (edesc->desc.next_desc) + dma_unmap_single(dev, be32_to_cpu(edesc->desc.next_desc), + TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL); } static void ahash_done(struct device *dev, @@ -1696,7 +1709,7 @@ static void ahash_done(struct device *dev, if (!req_ctx->last && req_ctx->to_hash_later) { /* Position any partial block for next update/final/finup */ - memcpy(req_ctx->buf, req_ctx->bufnext, req_ctx->to_hash_later); + req_ctx->buf_idx = (req_ctx->buf_idx + 1) & 1; req_ctx->nbuf = req_ctx->to_hash_later; } common_nonsnoop_hash_unmap(dev, edesc, areq); @@ -1710,7 +1723,7 @@ static void ahash_done(struct device *dev, * SEC1 doesn't like hashing of 0 sized message, so we do the padding * ourself and submit a padded block */ -void talitos_handle_buggy_hash(struct talitos_ctx *ctx, +static void talitos_handle_buggy_hash(struct talitos_ctx *ctx, struct talitos_edesc *edesc, struct talitos_ptr *ptr) { @@ -1729,6 +1742,7 @@ void talitos_handle_buggy_hash(struct talitos_ctx *ctx, static int common_nonsnoop_hash(struct talitos_edesc *edesc, struct ahash_request *areq, unsigned int length, + unsigned int offset, void (*callback) (struct device *dev, struct talitos_desc *desc, void *context, int error)) @@ -1745,44 +1759,48 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, int sg_count; /* first DWORD empty */ - desc->ptr[0] = zero_entry; /* hash context in */ if (!req_ctx->first || req_ctx->swinit) { - map_single_talitos_ptr(dev, &desc->ptr[1], - req_ctx->hw_context_size, - (char *)req_ctx->hw_context, - DMA_TO_DEVICE); + to_talitos_ptr(&desc->ptr[1], ctx->dma_hw_context, + req_ctx->hw_context_size, is_sec1); req_ctx->swinit = 0; - } else { - desc->ptr[1] = zero_entry; - /* Indicate next op is not the first. */ - req_ctx->first = 0; } + /* Indicate next op is not the first. */ + req_ctx->first = 0; /* HMAC key */ if (ctx->keylen) - map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen, - (char *)&ctx->key, DMA_TO_DEVICE); - else - desc->ptr[2] = zero_entry; + to_talitos_ptr(&desc->ptr[2], ctx->dma_key, ctx->keylen, + is_sec1); + + if (is_sec1 && req_ctx->nbuf) + length -= req_ctx->nbuf; sg_count = edesc->src_nents ?: 1; if (is_sec1 && sg_count > 1) - sg_copy_to_buffer(areq->src, sg_count, edesc->buf, length); - else + sg_pcopy_to_buffer(req_ctx->psrc, sg_count, + edesc->buf + sizeof(struct talitos_desc), + length, req_ctx->nbuf); + else if (length) sg_count = dma_map_sg(dev, req_ctx->psrc, sg_count, DMA_TO_DEVICE); /* * data in */ - sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc, - &desc->ptr[3], sg_count, 0, 0); - if (sg_count > 1) - sync_needed = true; + if (is_sec1 && req_ctx->nbuf) { + dma_addr_t dma_buf = ctx->dma_buf + req_ctx->buf_idx * + HASH_MAX_BLOCK_SIZE; + + to_talitos_ptr(&desc->ptr[3], dma_buf, req_ctx->nbuf, is_sec1); + } else { + sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc, + &desc->ptr[3], sg_count, offset, 0); + if (sg_count > 1) + sync_needed = true; + } /* fifth DWORD empty */ - desc->ptr[4] = zero_entry; /* hash/HMAC out -or- hash context out */ if (req_ctx->last) @@ -1790,16 +1808,44 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, crypto_ahash_digestsize(tfm), areq->result, DMA_FROM_DEVICE); else - map_single_talitos_ptr(dev, &desc->ptr[5], - req_ctx->hw_context_size, - req_ctx->hw_context, DMA_FROM_DEVICE); + to_talitos_ptr(&desc->ptr[5], ctx->dma_hw_context, + req_ctx->hw_context_size, is_sec1); /* last DWORD empty */ - desc->ptr[6] = zero_entry; if (is_sec1 && from_talitos_ptr_len(&desc->ptr[3], true) == 0) talitos_handle_buggy_hash(ctx, edesc, &desc->ptr[3]); + if (is_sec1 && req_ctx->nbuf && length) { + struct talitos_desc *desc2 = desc + 1; + dma_addr_t next_desc; + + memset(desc2, 0, sizeof(*desc2)); + desc2->hdr = desc->hdr; + desc2->hdr &= ~DESC_HDR_MODE0_MDEU_INIT; + desc2->hdr1 = desc2->hdr; + desc->hdr &= ~DESC_HDR_MODE0_MDEU_PAD; + desc->hdr |= DESC_HDR_MODE0_MDEU_CONT; + desc->hdr &= ~DESC_HDR_DONE_NOTIFY; + + to_talitos_ptr(&desc2->ptr[1], ctx->dma_hw_context, + req_ctx->hw_context_size, is_sec1); + + copy_talitos_ptr(&desc2->ptr[2], &desc->ptr[2], is_sec1); + sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc, + &desc2->ptr[3], sg_count, offset, 0); + if (sg_count > 1) + sync_needed = true; + copy_talitos_ptr(&desc2->ptr[5], &desc->ptr[5], is_sec1); + if (req_ctx->last) + to_talitos_ptr(&desc->ptr[5], ctx->dma_hw_context, + req_ctx->hw_context_size, is_sec1); + + next_desc = dma_map_single(dev, &desc2->hdr1, TALITOS_DESC_SIZE, + DMA_BIDIRECTIONAL); + desc->next_desc = cpu_to_be32(next_desc); + } + if (sync_needed) dma_sync_single_for_device(dev, edesc->dma_link_tbl, edesc->dma_len, DMA_BIDIRECTIONAL); @@ -1818,6 +1864,11 @@ static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq, struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); + struct talitos_private *priv = dev_get_drvdata(ctx->dev); + bool is_sec1 = has_ftr_sec1(priv); + + if (is_sec1) + nbytes -= req_ctx->nbuf; return talitos_edesc_alloc(ctx->dev, req_ctx->psrc, NULL, NULL, 0, nbytes, 0, 0, 0, areq->base.flags, false); @@ -1826,17 +1877,35 @@ static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq, static int ahash_init(struct ahash_request *areq) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); + struct device *dev = ctx->dev; struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); + unsigned int size; + struct talitos_private *priv = dev_get_drvdata(dev); + bool is_sec1 = has_ftr_sec1(priv); /* Initialize the context */ + req_ctx->buf_idx = 0; req_ctx->nbuf = 0; req_ctx->first = 1; /* first indicates h/w must init its context */ req_ctx->swinit = 0; /* assume h/w init of context */ - req_ctx->hw_context_size = - (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE) + size = (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE) ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256 : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512; + req_ctx->hw_context_size = size; + if (ctx->dma_hw_context) + dma_unmap_single(dev, ctx->dma_hw_context, size, + DMA_BIDIRECTIONAL); + ctx->dma_hw_context = dma_map_single(dev, req_ctx->hw_context, size, + DMA_BIDIRECTIONAL); + if (ctx->dma_buf) + dma_unmap_single(dev, ctx->dma_buf, sizeof(req_ctx->buf), + DMA_TO_DEVICE); + if (is_sec1) + ctx->dma_buf = dma_map_single(dev, req_ctx->buf, + sizeof(req_ctx->buf), + DMA_TO_DEVICE); return 0; } @@ -1847,6 +1916,9 @@ static int ahash_init(struct ahash_request *areq) static int ahash_init_sha224_swinit(struct ahash_request *areq) { struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); + struct device *dev = ctx->dev; ahash_init(areq); req_ctx->swinit = 1;/* prevent h/w initting context with sha256 values*/ @@ -1864,6 +1936,9 @@ static int ahash_init_sha224_swinit(struct ahash_request *areq) req_ctx->hw_context[8] = 0; req_ctx->hw_context[9] = 0; + dma_sync_single_for_device(dev, ctx->dma_hw_context, + req_ctx->hw_context_size, DMA_TO_DEVICE); + return 0; } @@ -1879,6 +1954,11 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) unsigned int to_hash_later; unsigned int nsg; int nents; + struct device *dev = ctx->dev; + struct talitos_private *priv = dev_get_drvdata(dev); + bool is_sec1 = has_ftr_sec1(priv); + int offset = 0; + u8 *ctx_buf = req_ctx->buf[req_ctx->buf_idx]; if (!req_ctx->last && (nbytes + req_ctx->nbuf <= blocksize)) { /* Buffer up to one whole block */ @@ -1888,7 +1968,7 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) return nents; } sg_copy_to_buffer(areq->src, nents, - req_ctx->buf + req_ctx->nbuf, nbytes); + ctx_buf + req_ctx->nbuf, nbytes); req_ctx->nbuf += nbytes; return 0; } @@ -1909,13 +1989,27 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) } /* Chain in any previously buffered data */ - if (req_ctx->nbuf) { + if (!is_sec1 && req_ctx->nbuf) { nsg = (req_ctx->nbuf < nbytes_to_hash) ? 2 : 1; sg_init_table(req_ctx->bufsl, nsg); - sg_set_buf(req_ctx->bufsl, req_ctx->buf, req_ctx->nbuf); + sg_set_buf(req_ctx->bufsl, ctx_buf, req_ctx->nbuf); if (nsg > 1) sg_chain(req_ctx->bufsl, 2, areq->src); req_ctx->psrc = req_ctx->bufsl; + } else if (is_sec1 && req_ctx->nbuf && req_ctx->nbuf < blocksize) { + if (nbytes_to_hash > blocksize) + offset = blocksize - req_ctx->nbuf; + else + offset = nbytes_to_hash - req_ctx->nbuf; + nents = sg_nents_for_len(areq->src, offset); + if (nents < 0) { + dev_err(ctx->dev, "Invalid number of src SG.\n"); + return nents; + } + sg_copy_to_buffer(areq->src, nents, + ctx_buf + req_ctx->nbuf, offset); + req_ctx->nbuf += offset; + req_ctx->psrc = areq->src; } else req_ctx->psrc = areq->src; @@ -1926,7 +2020,7 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) return nents; } sg_pcopy_to_buffer(areq->src, nents, - req_ctx->bufnext, + req_ctx->buf[(req_ctx->buf_idx + 1) & 1], to_hash_later, nbytes - to_hash_later); } @@ -1948,6 +2042,13 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) /* request SEC to INIT hash. */ if (req_ctx->first && !req_ctx->swinit) edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT; + if (is_sec1) { + dma_addr_t dma_buf = ctx->dma_buf + req_ctx->buf_idx * + HASH_MAX_BLOCK_SIZE; + + dma_sync_single_for_device(dev, dma_buf, + req_ctx->nbuf, DMA_TO_DEVICE); + } /* When the tfm context has a keylen, it's an HMAC. * A first or last (ie. not middle) descriptor must request HMAC. @@ -1955,7 +2056,7 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) if (ctx->keylen && (req_ctx->first || req_ctx->last)) edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC; - return common_nonsnoop_hash(edesc, areq, nbytes_to_hash, + return common_nonsnoop_hash(edesc, areq, nbytes_to_hash, offset, ahash_done); } @@ -2001,10 +2102,15 @@ static int ahash_export(struct ahash_request *areq, void *out) { struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); struct talitos_export_state *export = out; + struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq); + struct talitos_ctx *ctx = crypto_ahash_ctx(ahash); + struct device *dev = ctx->dev; + dma_sync_single_for_cpu(dev, ctx->dma_hw_context, + req_ctx->hw_context_size, DMA_FROM_DEVICE); memcpy(export->hw_context, req_ctx->hw_context, req_ctx->hw_context_size); - memcpy(export->buf, req_ctx->buf, req_ctx->nbuf); + memcpy(export->buf, req_ctx->buf[req_ctx->buf_idx], req_ctx->nbuf); export->swinit = req_ctx->swinit; export->first = req_ctx->first; export->last = req_ctx->last; @@ -2019,15 +2125,32 @@ static int ahash_import(struct ahash_request *areq, const void *in) struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); const struct talitos_export_state *export = in; + unsigned int size; + struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); + struct device *dev = ctx->dev; + struct talitos_private *priv = dev_get_drvdata(dev); + bool is_sec1 = has_ftr_sec1(priv); memset(req_ctx, 0, sizeof(*req_ctx)); - req_ctx->hw_context_size = - (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE) + size = (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE) ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256 : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512; - memcpy(req_ctx->hw_context, export->hw_context, - req_ctx->hw_context_size); - memcpy(req_ctx->buf, export->buf, export->nbuf); + req_ctx->hw_context_size = size; + if (ctx->dma_hw_context) + dma_unmap_single(dev, ctx->dma_hw_context, size, + DMA_BIDIRECTIONAL); + + memcpy(req_ctx->hw_context, export->hw_context, size); + ctx->dma_hw_context = dma_map_single(dev, req_ctx->hw_context, size, + DMA_BIDIRECTIONAL); + if (ctx->dma_buf) + dma_unmap_single(dev, ctx->dma_buf, sizeof(req_ctx->buf), + DMA_TO_DEVICE); + memcpy(req_ctx->buf[0], export->buf, export->nbuf); + if (is_sec1) + ctx->dma_buf = dma_map_single(dev, req_ctx->buf, + sizeof(req_ctx->buf), + DMA_TO_DEVICE); req_ctx->swinit = export->swinit; req_ctx->first = export->first; req_ctx->last = export->last; @@ -2037,22 +2160,6 @@ static int ahash_import(struct ahash_request *areq, const void *in) return 0; } -struct keyhash_result { - struct completion completion; - int err; -}; - -static void keyhash_complete(struct crypto_async_request *req, int err) -{ - struct keyhash_result *res = req->data; - - if (err == -EINPROGRESS) - return; - - res->err = err; - complete(&res->completion); -} - static int keyhash(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen, u8 *hash) { @@ -2060,10 +2167,10 @@ static int keyhash(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen, struct scatterlist sg[1]; struct ahash_request *req; - struct keyhash_result hresult; + struct crypto_wait wait; int ret; - init_completion(&hresult.completion); + crypto_init_wait(&wait); req = ahash_request_alloc(tfm, GFP_KERNEL); if (!req) @@ -2072,25 +2179,13 @@ static int keyhash(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen, /* Keep tfm keylen == 0 during hash of the long key */ ctx->keylen = 0; ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, - keyhash_complete, &hresult); + crypto_req_done, &wait); sg_init_one(&sg[0], key, keylen); ahash_request_set_crypt(req, sg, hash, keylen); - ret = crypto_ahash_digest(req); - switch (ret) { - case 0: - break; - case -EINPROGRESS: - case -EBUSY: - ret = wait_for_completion_interruptible( - &hresult.completion); - if (!ret) - ret = hresult.err; - break; - default: - break; - } + ret = crypto_wait_req(crypto_ahash_digest(req), &wait); + ahash_request_free(req); return ret; @@ -2100,6 +2195,7 @@ static int ahash_setkey(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen) { struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); + struct device *dev = ctx->dev; unsigned int blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); unsigned int digestsize = crypto_ahash_digestsize(tfm); @@ -2122,7 +2218,11 @@ static int ahash_setkey(struct crypto_ahash *tfm, const u8 *key, memcpy(ctx->key, hash, digestsize); } + if (ctx->keylen) + dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE); + ctx->keylen = keysize; + ctx->dma_key = dma_map_single(dev, ctx->key, keysize, DMA_TO_DEVICE); return 0; } @@ -2614,7 +2714,7 @@ static struct talitos_alg_template driver_algs[] = { .ivsize = AES_BLOCK_SIZE, } }, - .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | + .desc_hdr_template = DESC_HDR_TYPE_AESU_CTR_NONSNOOP | DESC_HDR_SEL0_AESU | DESC_HDR_MODE0_AESU_CTR, }, @@ -2951,6 +3051,36 @@ static int talitos_cra_init_ahash(struct crypto_tfm *tfm) return 0; } +static void talitos_cra_exit(struct crypto_tfm *tfm) +{ + struct talitos_ctx *ctx = crypto_tfm_ctx(tfm); + struct device *dev = ctx->dev; + + if (ctx->keylen) + dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE); +} + +static void talitos_cra_exit_ahash(struct crypto_tfm *tfm) +{ + struct talitos_ctx *ctx = crypto_tfm_ctx(tfm); + struct device *dev = ctx->dev; + unsigned int size; + + talitos_cra_exit(tfm); + + size = (crypto_ahash_digestsize(__crypto_ahash_cast(tfm)) <= + SHA256_DIGEST_SIZE) + ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256 + : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512; + + if (ctx->dma_hw_context) + dma_unmap_single(dev, ctx->dma_hw_context, size, + DMA_BIDIRECTIONAL); + if (ctx->dma_buf) + dma_unmap_single(dev, ctx->dma_buf, HASH_MAX_BLOCK_SIZE * 2, + DMA_TO_DEVICE); +} + /* * given the alg's descriptor header template, determine whether descriptor * type and primary/secondary execution units required match the hw @@ -2989,17 +3119,11 @@ static int talitos_remove(struct platform_device *ofdev) break; } list_del(&t_alg->entry); - kfree(t_alg); } if (hw_supports(dev, DESC_HDR_SEL0_RNG)) talitos_unregister_rng(dev); - for (i = 0; priv->chan && i < priv->num_channels; i++) - kfree(priv->chan[i].fifo); - - kfree(priv->chan); - for (i = 0; i < 2; i++) if (priv->irq[i]) { free_irq(priv->irq[i], dev); @@ -3010,10 +3134,6 @@ static int talitos_remove(struct platform_device *ofdev) if (priv->irq[1]) tasklet_kill(&priv->done_task[1]); - iounmap(priv->reg); - - kfree(priv); - return 0; } @@ -3025,7 +3145,8 @@ static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev, struct talitos_crypto_alg *t_alg; struct crypto_alg *alg; - t_alg = kzalloc(sizeof(struct talitos_crypto_alg), GFP_KERNEL); + t_alg = devm_kzalloc(dev, sizeof(struct talitos_crypto_alg), + GFP_KERNEL); if (!t_alg) return ERR_PTR(-ENOMEM); @@ -3035,6 +3156,7 @@ static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev, case CRYPTO_ALG_TYPE_ABLKCIPHER: alg = &t_alg->algt.alg.crypto; alg->cra_init = talitos_cra_init; + alg->cra_exit = talitos_cra_exit; alg->cra_type = &crypto_ablkcipher_type; alg->cra_ablkcipher.setkey = ablkcipher_setkey; alg->cra_ablkcipher.encrypt = ablkcipher_encrypt; @@ -3043,27 +3165,35 @@ static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev, break; case CRYPTO_ALG_TYPE_AEAD: alg = &t_alg->algt.alg.aead.base; + alg->cra_exit = talitos_cra_exit; t_alg->algt.alg.aead.init = talitos_cra_init_aead; t_alg->algt.alg.aead.setkey = aead_setkey; t_alg->algt.alg.aead.encrypt = aead_encrypt; t_alg->algt.alg.aead.decrypt = aead_decrypt; + if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) && + !strncmp(alg->cra_name, "authenc(hmac(sha224)", 20)) { + devm_kfree(dev, t_alg); + return ERR_PTR(-ENOTSUPP); + } break; case CRYPTO_ALG_TYPE_AHASH: alg = &t_alg->algt.alg.hash.halg.base; alg->cra_init = talitos_cra_init_ahash; + alg->cra_exit = talitos_cra_exit_ahash; alg->cra_type = &crypto_ahash_type; t_alg->algt.alg.hash.init = ahash_init; t_alg->algt.alg.hash.update = ahash_update; t_alg->algt.alg.hash.final = ahash_final; t_alg->algt.alg.hash.finup = ahash_finup; t_alg->algt.alg.hash.digest = ahash_digest; - t_alg->algt.alg.hash.setkey = ahash_setkey; + if (!strncmp(alg->cra_name, "hmac", 4)) + t_alg->algt.alg.hash.setkey = ahash_setkey; t_alg->algt.alg.hash.import = ahash_import; t_alg->algt.alg.hash.export = ahash_export; if (!(priv->features & TALITOS_FTR_HMAC_OK) && !strncmp(alg->cra_name, "hmac", 4)) { - kfree(t_alg); + devm_kfree(dev, t_alg); return ERR_PTR(-ENOTSUPP); } if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) && @@ -3078,7 +3208,7 @@ static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev, break; default: dev_err(dev, "unknown algorithm type %d\n", t_alg->algt.type); - kfree(t_alg); + devm_kfree(dev, t_alg); return ERR_PTR(-EINVAL); } @@ -3155,11 +3285,11 @@ static int talitos_probe(struct platform_device *ofdev) struct device *dev = &ofdev->dev; struct device_node *np = ofdev->dev.of_node; struct talitos_private *priv; - const unsigned int *prop; int i, err; int stride; + struct resource *res; - priv = kzalloc(sizeof(struct talitos_private), GFP_KERNEL); + priv = devm_kzalloc(dev, sizeof(struct talitos_private), GFP_KERNEL); if (!priv) return -ENOMEM; @@ -3171,7 +3301,10 @@ static int talitos_probe(struct platform_device *ofdev) spin_lock_init(&priv->reg_lock); - priv->reg = of_iomap(np, 0); + res = platform_get_resource(ofdev, IORESOURCE_MEM, 0); + if (!res) + return -ENXIO; + priv->reg = devm_ioremap(dev, res->start, resource_size(res)); if (!priv->reg) { dev_err(dev, "failed to of_iomap\n"); err = -ENOMEM; @@ -3179,21 +3312,11 @@ static int talitos_probe(struct platform_device *ofdev) } /* get SEC version capabilities from device tree */ - prop = of_get_property(np, "fsl,num-channels", NULL); - if (prop) - priv->num_channels = *prop; - - prop = of_get_property(np, "fsl,channel-fifo-len", NULL); - if (prop) - priv->chfifo_len = *prop; - - prop = of_get_property(np, "fsl,exec-units-mask", NULL); - if (prop) - priv->exec_units = *prop; - - prop = of_get_property(np, "fsl,descriptor-types-mask", NULL); - if (prop) - priv->desc_types = *prop; + of_property_read_u32(np, "fsl,num-channels", &priv->num_channels); + of_property_read_u32(np, "fsl,channel-fifo-len", &priv->chfifo_len); + of_property_read_u32(np, "fsl,exec-units-mask", &priv->exec_units); + of_property_read_u32(np, "fsl,descriptor-types-mask", + &priv->desc_types); if (!is_power_of_2(priv->num_channels) || !priv->chfifo_len || !priv->exec_units || !priv->desc_types) { @@ -3243,22 +3366,29 @@ static int talitos_probe(struct platform_device *ofdev) goto err_out; if (of_device_is_compatible(np, "fsl,sec1.0")) { - tasklet_init(&priv->done_task[0], talitos1_done_4ch, - (unsigned long)dev); - } else { - if (!priv->irq[1]) { - tasklet_init(&priv->done_task[0], talitos2_done_4ch, + if (priv->num_channels == 1) + tasklet_init(&priv->done_task[0], talitos1_done_ch0, (unsigned long)dev); - } else { + else + tasklet_init(&priv->done_task[0], talitos1_done_4ch, + (unsigned long)dev); + } else { + if (priv->irq[1]) { tasklet_init(&priv->done_task[0], talitos2_done_ch0_2, (unsigned long)dev); tasklet_init(&priv->done_task[1], talitos2_done_ch1_3, (unsigned long)dev); + } else if (priv->num_channels == 1) { + tasklet_init(&priv->done_task[0], talitos2_done_ch0, + (unsigned long)dev); + } else { + tasklet_init(&priv->done_task[0], talitos2_done_4ch, + (unsigned long)dev); } } - priv->chan = kzalloc(sizeof(struct talitos_channel) * - priv->num_channels, GFP_KERNEL); + priv->chan = devm_kzalloc(dev, sizeof(struct talitos_channel) * + priv->num_channels, GFP_KERNEL); if (!priv->chan) { dev_err(dev, "failed to allocate channel management space\n"); err = -ENOMEM; @@ -3275,8 +3405,9 @@ static int talitos_probe(struct platform_device *ofdev) spin_lock_init(&priv->chan[i].head_lock); spin_lock_init(&priv->chan[i].tail_lock); - priv->chan[i].fifo = kzalloc(sizeof(struct talitos_request) * - priv->fifo_len, GFP_KERNEL); + priv->chan[i].fifo = devm_kzalloc(dev, + sizeof(struct talitos_request) * + priv->fifo_len, GFP_KERNEL); if (!priv->chan[i].fifo) { dev_err(dev, "failed to allocate request fifo %d\n", i); err = -ENOMEM; @@ -3342,7 +3473,7 @@ static int talitos_probe(struct platform_device *ofdev) if (err) { dev_err(dev, "%s alg registration failed\n", alg->cra_driver_name); - kfree(t_alg); + devm_kfree(dev, t_alg); } else list_add_tail(&t_alg->entry, &priv->alg_list); } diff --git a/drivers/crypto/talitos.h b/drivers/crypto/talitos.h index 8dd8f40e2771..a65a63e0d6c1 100644 --- a/drivers/crypto/talitos.h +++ b/drivers/crypto/talitos.h @@ -52,8 +52,6 @@ struct talitos_ptr { __be32 ptr; /* address */ }; -static const struct talitos_ptr zero_entry; - /* descriptor */ struct talitos_desc { __be32 hdr; /* header high bits */ @@ -210,9 +208,13 @@ static inline bool has_ftr_sec1(struct talitos_private *priv) #define TALITOS_ISR 0x1010 /* interrupt status register */ #define TALITOS1_ISR_4CHERR ISR1_FORMAT(0xa) /* 4 ch errors mask */ #define TALITOS1_ISR_4CHDONE ISR1_FORMAT(0x5) /* 4 ch done mask */ +#define TALITOS1_ISR_CH_0_ERR (2 << 28) /* ch 0 errors mask */ +#define TALITOS1_ISR_CH_0_DONE (1 << 28) /* ch 0 done mask */ #define TALITOS1_ISR_TEA_ERR 0x00000040 #define TALITOS2_ISR_4CHERR ISR2_FORMAT(0xa) /* 4 ch errors mask */ #define TALITOS2_ISR_4CHDONE ISR2_FORMAT(0x5) /* 4 ch done mask */ +#define TALITOS2_ISR_CH_0_ERR 2 /* ch 0 errors mask */ +#define TALITOS2_ISR_CH_0_DONE 1 /* ch 0 done mask */ #define TALITOS2_ISR_CH_0_2_ERR ISR2_FORMAT(0x2) /* ch 0, 2 err mask */ #define TALITOS2_ISR_CH_0_2_DONE ISR2_FORMAT(0x1) /* ch 0, 2 done mask */ #define TALITOS2_ISR_CH_1_3_ERR ISR2_FORMAT(0x8) /* ch 1, 3 err mask */ @@ -234,6 +236,7 @@ static inline bool has_ftr_sec1(struct talitos_private *priv) #define TALITOS_CCCR_LO_IWSE 0x80 /* chan. ICCR writeback enab. */ #define TALITOS_CCCR_LO_EAE 0x20 /* extended address enable */ #define TALITOS_CCCR_LO_CDWE 0x10 /* chan. done writeback enab. */ +#define TALITOS_CCCR_LO_NE 0x8 /* fetch next descriptor enab. */ #define TALITOS_CCCR_LO_NT 0x4 /* notification type */ #define TALITOS_CCCR_LO_CDIE 0x2 /* channel done IRQ enable */ #define TALITOS1_CCCR_LO_RESET 0x1 /* channel reset on SEC1 */ diff --git a/drivers/crypto/ux500/cryp/cryp_core.c b/drivers/crypto/ux500/cryp/cryp_core.c index 790f7cadc1ed..765f53e548ab 100644 --- a/drivers/crypto/ux500/cryp/cryp_core.c +++ b/drivers/crypto/ux500/cryp/cryp_core.c @@ -1751,7 +1751,6 @@ static void __exit ux500_cryp_mod_fini(void) { pr_debug("[%s] is called!", __func__); platform_driver_unregister(&cryp_driver); - return; } module_init(ux500_cryp_mod_init); diff --git a/drivers/crypto/virtio/Makefile b/drivers/crypto/virtio/Makefile index dd342c947ff9..cbfccccfa135 100644 --- a/drivers/crypto/virtio/Makefile +++ b/drivers/crypto/virtio/Makefile @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_CRYPTO_DEV_VIRTIO) += virtio_crypto.o virtio_crypto-objs := \ virtio_crypto_algs.o \ diff --git a/drivers/crypto/virtio/virtio_crypto_algs.c b/drivers/crypto/virtio/virtio_crypto_algs.c index 49defda4e03d..abe8c15450df 100644 --- a/drivers/crypto/virtio/virtio_crypto_algs.c +++ b/drivers/crypto/virtio/virtio_crypto_algs.c @@ -27,12 +27,68 @@ #include <uapi/linux/virtio_crypto.h> #include "virtio_crypto_common.h" + +struct virtio_crypto_ablkcipher_ctx { + struct virtio_crypto *vcrypto; + struct crypto_tfm *tfm; + + struct virtio_crypto_sym_session_info enc_sess_info; + struct virtio_crypto_sym_session_info dec_sess_info; +}; + +struct virtio_crypto_sym_request { + struct virtio_crypto_request base; + + /* Cipher or aead */ + uint32_t type; + struct virtio_crypto_ablkcipher_ctx *ablkcipher_ctx; + struct ablkcipher_request *ablkcipher_req; + uint8_t *iv; + /* Encryption? */ + bool encrypt; +}; + /* * The algs_lock protects the below global virtio_crypto_active_devs * and crypto algorithms registion. */ static DEFINE_MUTEX(algs_lock); static unsigned int virtio_crypto_active_devs; +static void virtio_crypto_ablkcipher_finalize_req( + struct virtio_crypto_sym_request *vc_sym_req, + struct ablkcipher_request *req, + int err); + +static void virtio_crypto_dataq_sym_callback + (struct virtio_crypto_request *vc_req, int len) +{ + struct virtio_crypto_sym_request *vc_sym_req = + container_of(vc_req, struct virtio_crypto_sym_request, base); + struct ablkcipher_request *ablk_req; + int error; + + /* Finish the encrypt or decrypt process */ + if (vc_sym_req->type == VIRTIO_CRYPTO_SYM_OP_CIPHER) { + switch (vc_req->status) { + case VIRTIO_CRYPTO_OK: + error = 0; + break; + case VIRTIO_CRYPTO_INVSESS: + case VIRTIO_CRYPTO_ERR: + error = -EINVAL; + break; + case VIRTIO_CRYPTO_BADMSG: + error = -EBADMSG; + break; + default: + error = -EIO; + break; + } + ablk_req = vc_sym_req->ablkcipher_req; + virtio_crypto_ablkcipher_finalize_req(vc_sym_req, + ablk_req, error); + } +} static u64 virtio_crypto_alg_sg_nents_length(struct scatterlist *sg) { @@ -263,7 +319,7 @@ static int virtio_crypto_ablkcipher_setkey(struct crypto_ablkcipher *tfm, struct virtio_crypto *vcrypto = virtcrypto_get_dev_node(node); if (!vcrypto) { - pr_err("virtio_crypto: Could not find a virtio device in the system"); + pr_err("virtio_crypto: Could not find a virtio device in the system\n"); return -ENODEV; } @@ -286,13 +342,14 @@ static int virtio_crypto_ablkcipher_setkey(struct crypto_ablkcipher *tfm, } static int -__virtio_crypto_ablkcipher_do_req(struct virtio_crypto_request *vc_req, +__virtio_crypto_ablkcipher_do_req(struct virtio_crypto_sym_request *vc_sym_req, struct ablkcipher_request *req, struct data_queue *data_vq) { struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); + struct virtio_crypto_ablkcipher_ctx *ctx = vc_sym_req->ablkcipher_ctx; + struct virtio_crypto_request *vc_req = &vc_sym_req->base; unsigned int ivsize = crypto_ablkcipher_ivsize(tfm); - struct virtio_crypto_ablkcipher_ctx *ctx = vc_req->ablkcipher_ctx; struct virtio_crypto *vcrypto = ctx->vcrypto; struct virtio_crypto_op_data_req *req_data; int src_nents, dst_nents; @@ -326,9 +383,9 @@ __virtio_crypto_ablkcipher_do_req(struct virtio_crypto_request *vc_req, } vc_req->req_data = req_data; - vc_req->type = VIRTIO_CRYPTO_SYM_OP_CIPHER; + vc_sym_req->type = VIRTIO_CRYPTO_SYM_OP_CIPHER; /* Head of operation */ - if (vc_req->encrypt) { + if (vc_sym_req->encrypt) { req_data->header.session_id = cpu_to_le64(ctx->enc_sess_info.session_id); req_data->header.opcode = @@ -383,7 +440,7 @@ __virtio_crypto_ablkcipher_do_req(struct virtio_crypto_request *vc_req, memcpy(iv, req->info, ivsize); sg_init_one(&iv_sg, iv, ivsize); sgs[num_out++] = &iv_sg; - vc_req->iv = iv; + vc_sym_req->iv = iv; /* Source data */ for (i = 0; i < src_nents; i++) @@ -421,15 +478,18 @@ static int virtio_crypto_ablkcipher_encrypt(struct ablkcipher_request *req) { struct crypto_ablkcipher *atfm = crypto_ablkcipher_reqtfm(req); struct virtio_crypto_ablkcipher_ctx *ctx = crypto_ablkcipher_ctx(atfm); - struct virtio_crypto_request *vc_req = ablkcipher_request_ctx(req); + struct virtio_crypto_sym_request *vc_sym_req = + ablkcipher_request_ctx(req); + struct virtio_crypto_request *vc_req = &vc_sym_req->base; struct virtio_crypto *vcrypto = ctx->vcrypto; /* Use the first data virtqueue as default */ struct data_queue *data_vq = &vcrypto->data_vq[0]; - vc_req->ablkcipher_ctx = ctx; - vc_req->ablkcipher_req = req; - vc_req->encrypt = true; vc_req->dataq = data_vq; + vc_req->alg_cb = virtio_crypto_dataq_sym_callback; + vc_sym_req->ablkcipher_ctx = ctx; + vc_sym_req->ablkcipher_req = req; + vc_sym_req->encrypt = true; return crypto_transfer_cipher_request_to_engine(data_vq->engine, req); } @@ -438,16 +498,18 @@ static int virtio_crypto_ablkcipher_decrypt(struct ablkcipher_request *req) { struct crypto_ablkcipher *atfm = crypto_ablkcipher_reqtfm(req); struct virtio_crypto_ablkcipher_ctx *ctx = crypto_ablkcipher_ctx(atfm); - struct virtio_crypto_request *vc_req = ablkcipher_request_ctx(req); + struct virtio_crypto_sym_request *vc_sym_req = + ablkcipher_request_ctx(req); + struct virtio_crypto_request *vc_req = &vc_sym_req->base; struct virtio_crypto *vcrypto = ctx->vcrypto; /* Use the first data virtqueue as default */ struct data_queue *data_vq = &vcrypto->data_vq[0]; - vc_req->ablkcipher_ctx = ctx; - vc_req->ablkcipher_req = req; - - vc_req->encrypt = false; vc_req->dataq = data_vq; + vc_req->alg_cb = virtio_crypto_dataq_sym_callback; + vc_sym_req->ablkcipher_ctx = ctx; + vc_sym_req->ablkcipher_req = req; + vc_sym_req->encrypt = false; return crypto_transfer_cipher_request_to_engine(data_vq->engine, req); } @@ -456,7 +518,7 @@ static int virtio_crypto_ablkcipher_init(struct crypto_tfm *tfm) { struct virtio_crypto_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm); - tfm->crt_ablkcipher.reqsize = sizeof(struct virtio_crypto_request); + tfm->crt_ablkcipher.reqsize = sizeof(struct virtio_crypto_sym_request); ctx->tfm = tfm; return 0; @@ -479,11 +541,13 @@ int virtio_crypto_ablkcipher_crypt_req( struct crypto_engine *engine, struct ablkcipher_request *req) { - struct virtio_crypto_request *vc_req = ablkcipher_request_ctx(req); + struct virtio_crypto_sym_request *vc_sym_req = + ablkcipher_request_ctx(req); + struct virtio_crypto_request *vc_req = &vc_sym_req->base; struct data_queue *data_vq = vc_req->dataq; int ret; - ret = __virtio_crypto_ablkcipher_do_req(vc_req, req, data_vq); + ret = __virtio_crypto_ablkcipher_do_req(vc_sym_req, req, data_vq); if (ret < 0) return ret; @@ -492,14 +556,15 @@ int virtio_crypto_ablkcipher_crypt_req( return 0; } -void virtio_crypto_ablkcipher_finalize_req( - struct virtio_crypto_request *vc_req, +static void virtio_crypto_ablkcipher_finalize_req( + struct virtio_crypto_sym_request *vc_sym_req, struct ablkcipher_request *req, int err) { - crypto_finalize_cipher_request(vc_req->dataq->engine, req, err); - - virtcrypto_clear_request(vc_req); + crypto_finalize_cipher_request(vc_sym_req->base.dataq->engine, + req, err); + kzfree(vc_sym_req->iv); + virtcrypto_clear_request(&vc_sym_req->base); } static struct crypto_alg virtio_crypto_algs[] = { { diff --git a/drivers/crypto/virtio/virtio_crypto_common.h b/drivers/crypto/virtio/virtio_crypto_common.h index da6d8c0ea407..e976539a05d9 100644 --- a/drivers/crypto/virtio/virtio_crypto_common.h +++ b/drivers/crypto/virtio/virtio_crypto_common.h @@ -83,26 +83,16 @@ struct virtio_crypto_sym_session_info { __u64 session_id; }; -struct virtio_crypto_ablkcipher_ctx { - struct virtio_crypto *vcrypto; - struct crypto_tfm *tfm; - - struct virtio_crypto_sym_session_info enc_sess_info; - struct virtio_crypto_sym_session_info dec_sess_info; -}; +struct virtio_crypto_request; +typedef void (*virtio_crypto_data_callback) + (struct virtio_crypto_request *vc_req, int len); struct virtio_crypto_request { - /* Cipher or aead */ - uint32_t type; uint8_t status; - struct virtio_crypto_ablkcipher_ctx *ablkcipher_ctx; - struct ablkcipher_request *ablkcipher_req; struct virtio_crypto_op_data_req *req_data; struct scatterlist **sgs; - uint8_t *iv; - /* Encryption? */ - bool encrypt; struct data_queue *dataq; + virtio_crypto_data_callback alg_cb; }; int virtcrypto_devmgr_add_dev(struct virtio_crypto *vcrypto_dev); @@ -119,10 +109,6 @@ void virtcrypto_dev_stop(struct virtio_crypto *vcrypto); int virtio_crypto_ablkcipher_crypt_req( struct crypto_engine *engine, struct ablkcipher_request *req); -void virtio_crypto_ablkcipher_finalize_req( - struct virtio_crypto_request *vc_req, - struct ablkcipher_request *req, - int err); void virtcrypto_clear_request(struct virtio_crypto_request *vc_req); diff --git a/drivers/crypto/virtio/virtio_crypto_core.c b/drivers/crypto/virtio/virtio_crypto_core.c index a111cd72797b..ff1410a32c2b 100644 --- a/drivers/crypto/virtio/virtio_crypto_core.c +++ b/drivers/crypto/virtio/virtio_crypto_core.c @@ -29,7 +29,6 @@ void virtcrypto_clear_request(struct virtio_crypto_request *vc_req) { if (vc_req) { - kzfree(vc_req->iv); kzfree(vc_req->req_data); kfree(vc_req->sgs); } @@ -41,40 +40,18 @@ static void virtcrypto_dataq_callback(struct virtqueue *vq) struct virtio_crypto_request *vc_req; unsigned long flags; unsigned int len; - struct ablkcipher_request *ablk_req; - int error; unsigned int qid = vq->index; spin_lock_irqsave(&vcrypto->data_vq[qid].lock, flags); do { virtqueue_disable_cb(vq); while ((vc_req = virtqueue_get_buf(vq, &len)) != NULL) { - if (vc_req->type == VIRTIO_CRYPTO_SYM_OP_CIPHER) { - switch (vc_req->status) { - case VIRTIO_CRYPTO_OK: - error = 0; - break; - case VIRTIO_CRYPTO_INVSESS: - case VIRTIO_CRYPTO_ERR: - error = -EINVAL; - break; - case VIRTIO_CRYPTO_BADMSG: - error = -EBADMSG; - break; - default: - error = -EIO; - break; - } - ablk_req = vc_req->ablkcipher_req; - - spin_unlock_irqrestore( - &vcrypto->data_vq[qid].lock, flags); - /* Finish the encrypt or decrypt process */ - virtio_crypto_ablkcipher_finalize_req(vc_req, - ablk_req, error); - spin_lock_irqsave( - &vcrypto->data_vq[qid].lock, flags); - } + spin_unlock_irqrestore( + &vcrypto->data_vq[qid].lock, flags); + if (vc_req->alg_cb) + vc_req->alg_cb(vc_req, len); + spin_lock_irqsave( + &vcrypto->data_vq[qid].lock, flags); } } while (!virtqueue_enable_cb(vq)); spin_unlock_irqrestore(&vcrypto->data_vq[qid].lock, flags); @@ -270,7 +247,7 @@ static int virtcrypto_update_status(struct virtio_crypto *vcrypto) return -EPERM; } - dev_info(&vcrypto->vdev->dev, "Accelerator is ready\n"); + dev_info(&vcrypto->vdev->dev, "Accelerator device is ready\n"); } else { virtcrypto_dev_stop(vcrypto); dev_info(&vcrypto->vdev->dev, "Accelerator is not ready\n"); diff --git a/drivers/crypto/vmx/Makefile b/drivers/crypto/vmx/Makefile index 55f7c392582f..cab32cfec9c4 100644 --- a/drivers/crypto/vmx/Makefile +++ b/drivers/crypto/vmx/Makefile @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_CRYPTO_DEV_VMX_ENCRYPT) += vmx-crypto.o vmx-crypto-objs := vmx.o aesp8-ppc.o ghashp8-ppc.o aes.o aes_cbc.o aes_ctr.o aes_xts.o ghash.o diff --git a/drivers/crypto/vmx/aes_ctr.c b/drivers/crypto/vmx/aes_ctr.c index 9c26d9e8dbea..fc60d00a2e84 100644 --- a/drivers/crypto/vmx/aes_ctr.c +++ b/drivers/crypto/vmx/aes_ctr.c @@ -27,21 +27,23 @@ #include <asm/switch_to.h> #include <crypto/aes.h> #include <crypto/scatterwalk.h> +#include <crypto/skcipher.h> + #include "aesp8-ppc.h" struct p8_aes_ctr_ctx { - struct crypto_blkcipher *fallback; + struct crypto_skcipher *fallback; struct aes_key enc_key; }; static int p8_aes_ctr_init(struct crypto_tfm *tfm) { const char *alg = crypto_tfm_alg_name(tfm); - struct crypto_blkcipher *fallback; + struct crypto_skcipher *fallback; struct p8_aes_ctr_ctx *ctx = crypto_tfm_ctx(tfm); - fallback = - crypto_alloc_blkcipher(alg, 0, CRYPTO_ALG_NEED_FALLBACK); + fallback = crypto_alloc_skcipher(alg, 0, + CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK); if (IS_ERR(fallback)) { printk(KERN_ERR "Failed to allocate transformation for '%s': %ld\n", @@ -49,11 +51,11 @@ static int p8_aes_ctr_init(struct crypto_tfm *tfm) return PTR_ERR(fallback); } printk(KERN_INFO "Using '%s' as fallback implementation.\n", - crypto_tfm_alg_driver_name((struct crypto_tfm *) fallback)); + crypto_skcipher_driver_name(fallback)); - crypto_blkcipher_set_flags( + crypto_skcipher_set_flags( fallback, - crypto_blkcipher_get_flags((struct crypto_blkcipher *)tfm)); + crypto_skcipher_get_flags((struct crypto_skcipher *)tfm)); ctx->fallback = fallback; return 0; @@ -64,7 +66,7 @@ static void p8_aes_ctr_exit(struct crypto_tfm *tfm) struct p8_aes_ctr_ctx *ctx = crypto_tfm_ctx(tfm); if (ctx->fallback) { - crypto_free_blkcipher(ctx->fallback); + crypto_free_skcipher(ctx->fallback); ctx->fallback = NULL; } } @@ -83,7 +85,7 @@ static int p8_aes_ctr_setkey(struct crypto_tfm *tfm, const u8 *key, pagefault_enable(); preempt_enable(); - ret += crypto_blkcipher_setkey(ctx->fallback, key, keylen); + ret += crypto_skcipher_setkey(ctx->fallback, key, keylen); return ret; } @@ -104,8 +106,7 @@ static void p8_aes_ctr_final(struct p8_aes_ctr_ctx *ctx, pagefault_enable(); preempt_enable(); - crypto_xor(keystream, src, nbytes); - memcpy(dst, keystream, nbytes); + crypto_xor_cpy(dst, keystream, src, nbytes); crypto_inc(ctrblk, AES_BLOCK_SIZE); } @@ -118,15 +119,14 @@ static int p8_aes_ctr_crypt(struct blkcipher_desc *desc, struct blkcipher_walk walk; struct p8_aes_ctr_ctx *ctx = crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm)); - struct blkcipher_desc fallback_desc = { - .tfm = ctx->fallback, - .info = desc->info, - .flags = desc->flags - }; if (in_interrupt()) { - ret = crypto_blkcipher_encrypt(&fallback_desc, dst, src, - nbytes); + SKCIPHER_REQUEST_ON_STACK(req, ctx->fallback); + skcipher_request_set_tfm(req, ctx->fallback); + skcipher_request_set_callback(req, desc->flags, NULL, NULL); + skcipher_request_set_crypt(req, src, dst, nbytes, desc->info); + ret = crypto_skcipher_encrypt(req); + skcipher_request_zero(req); } else { blkcipher_walk_init(&walk, dst, src, nbytes); ret = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE); diff --git a/drivers/crypto/vmx/aesp8-ppc.h b/drivers/crypto/vmx/aesp8-ppc.h index 01972e16a6c0..349646b73754 100644 --- a/drivers/crypto/vmx/aesp8-ppc.h +++ b/drivers/crypto/vmx/aesp8-ppc.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #include <linux/types.h> #include <crypto/aes.h> diff --git a/drivers/crypto/vmx/ppc-xlate.pl b/drivers/crypto/vmx/ppc-xlate.pl index b18e67d0e065..36db2ef09e5b 100644 --- a/drivers/crypto/vmx/ppc-xlate.pl +++ b/drivers/crypto/vmx/ppc-xlate.pl @@ -1,4 +1,5 @@ #!/usr/bin/env perl +# SPDX-License-Identifier: GPL-2.0 # PowerPC assembler distiller by <appro>. |