From e09b41010ba33a20a87472ee821fa407a5b8da36 Mon Sep 17 00:00:00 2001 From: José Pekkarinen Date: Mon, 11 Apr 2016 10:41:07 +0300 Subject: These changes are the raw update to linux-4.4.6-rt14. Kernel sources are taken from kernel.org, and rt patch from the rt wiki download page. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit During the rebasing, the following patch collided: Force tick interrupt and get rid of softirq magic(I70131fb85). Collisions have been removed because its logic was found on the source already. Change-Id: I7f57a4081d9deaa0d9ccfc41a6c8daccdee3b769 Signed-off-by: José Pekkarinen --- kernel/drivers/crypto/Kconfig | 111 +- kernel/drivers/crypto/Makefile | 2 + kernel/drivers/crypto/amcc/crypto4xx_core.c | 25 +- kernel/drivers/crypto/atmel-aes.c | 44 +- kernel/drivers/crypto/atmel-sha.c | 50 +- kernel/drivers/crypto/atmel-tdes.c | 35 +- kernel/drivers/crypto/bfin_crc.c | 28 +- kernel/drivers/crypto/caam/Kconfig | 15 +- kernel/drivers/crypto/caam/caamalg.c | 3414 +++++++++++--------- kernel/drivers/crypto/caam/caamhash.c | 205 +- kernel/drivers/crypto/caam/caamrng.c | 26 +- kernel/drivers/crypto/caam/compat.h | 3 +- kernel/drivers/crypto/caam/ctrl.c | 156 +- kernel/drivers/crypto/caam/desc.h | 24 +- kernel/drivers/crypto/caam/desc_constr.h | 2 +- kernel/drivers/crypto/caam/intern.h | 5 + kernel/drivers/crypto/caam/jr.c | 30 +- kernel/drivers/crypto/caam/regs.h | 94 +- kernel/drivers/crypto/caam/sg_sw_sec4.h | 61 +- kernel/drivers/crypto/ccp/Kconfig | 14 +- kernel/drivers/crypto/ccp/ccp-crypto-aes-cmac.c | 20 +- kernel/drivers/crypto/ccp/ccp-crypto-main.c | 6 +- kernel/drivers/crypto/ccp/ccp-crypto-sha.c | 13 + kernel/drivers/crypto/ccp/ccp-ops.c | 117 +- kernel/drivers/crypto/ccp/ccp-pci.c | 2 +- kernel/drivers/crypto/ccp/ccp-platform.c | 83 +- kernel/drivers/crypto/hifn_795x.c | 2 - kernel/drivers/crypto/img-hash.c | 2 +- kernel/drivers/crypto/ixp4xx_crypto.c | 317 +- kernel/drivers/crypto/marvell/Makefile | 2 + kernel/drivers/crypto/marvell/cesa.c | 546 ++++ kernel/drivers/crypto/marvell/cesa.h | 825 +++++ kernel/drivers/crypto/marvell/cipher.c | 797 +++++ kernel/drivers/crypto/marvell/hash.c | 1420 ++++++++ kernel/drivers/crypto/marvell/tdma.c | 226 ++ kernel/drivers/crypto/mv_cesa.c | 73 +- kernel/drivers/crypto/n2_core.c | 10 +- kernel/drivers/crypto/nx/Kconfig | 54 +- kernel/drivers/crypto/nx/Makefile | 5 +- kernel/drivers/crypto/nx/nx-842-powernv.c | 653 ++++ kernel/drivers/crypto/nx/nx-842-pseries.c | 1147 +++++++ kernel/drivers/crypto/nx/nx-842.c | 1874 +++-------- kernel/drivers/crypto/nx/nx-842.h | 185 ++ kernel/drivers/crypto/nx/nx-aes-ccm.c | 153 +- kernel/drivers/crypto/nx/nx-aes-ctr.c | 21 - kernel/drivers/crypto/nx/nx-aes-gcm.c | 175 +- kernel/drivers/crypto/nx/nx.c | 173 +- kernel/drivers/crypto/nx/nx.h | 16 +- kernel/drivers/crypto/omap-aes.c | 86 +- kernel/drivers/crypto/omap-sham.c | 29 +- kernel/drivers/crypto/padlock-aes.c | 2 +- kernel/drivers/crypto/padlock-sha.c | 2 +- kernel/drivers/crypto/picoxcell_crypto.c | 697 ++-- kernel/drivers/crypto/qat/Kconfig | 21 +- kernel/drivers/crypto/qat/Makefile | 1 + kernel/drivers/crypto/qat/qat_common/.gitignore | 1 + kernel/drivers/crypto/qat/qat_common/Makefile | 14 + .../crypto/qat/qat_common/adf_accel_devices.h | 47 +- .../crypto/qat/qat_common/adf_accel_engine.c | 47 +- kernel/drivers/crypto/qat/qat_common/adf_admin.c | 290 ++ kernel/drivers/crypto/qat/qat_common/adf_aer.c | 8 +- kernel/drivers/crypto/qat/qat_common/adf_cfg.c | 9 +- .../drivers/crypto/qat/qat_common/adf_cfg_common.h | 3 +- .../drivers/crypto/qat/qat_common/adf_cfg_user.h | 12 - .../drivers/crypto/qat/qat_common/adf_common_drv.h | 60 +- kernel/drivers/crypto/qat/qat_common/adf_ctl_drv.c | 15 +- kernel/drivers/crypto/qat/qat_common/adf_dev_mgr.c | 286 +- .../drivers/crypto/qat/qat_common/adf_hw_arbiter.c | 168 + kernel/drivers/crypto/qat/qat_common/adf_init.c | 106 +- .../drivers/crypto/qat/qat_common/adf_pf2vf_msg.c | 438 +++ .../drivers/crypto/qat/qat_common/adf_pf2vf_msg.h | 146 + kernel/drivers/crypto/qat/qat_common/adf_sriov.c | 306 ++ .../drivers/crypto/qat/qat_common/adf_transport.c | 15 +- .../qat/qat_common/adf_transport_access_macros.h | 5 +- .../crypto/qat/qat_common/adf_transport_debug.c | 16 +- kernel/drivers/crypto/qat/qat_common/icp_qat_fw.h | 2 + .../drivers/crypto/qat/qat_common/icp_qat_fw_pke.h | 112 + kernel/drivers/crypto/qat/qat_common/qat_algs.c | 471 +-- .../drivers/crypto/qat/qat_common/qat_asym_algs.c | 755 +++++ kernel/drivers/crypto/qat/qat_common/qat_crypto.c | 101 +- kernel/drivers/crypto/qat/qat_common/qat_crypto.h | 2 - kernel/drivers/crypto/qat/qat_common/qat_hal.c | 19 +- .../crypto/qat/qat_common/qat_rsaprivkey.asn1 | 11 + .../crypto/qat/qat_common/qat_rsapubkey.asn1 | 4 + kernel/drivers/crypto/qat/qat_common/qat_uclo.c | 27 +- kernel/drivers/crypto/qat/qat_dh895xcc/Makefile | 5 +- kernel/drivers/crypto/qat/qat_dh895xcc/adf_admin.c | 145 - .../crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c | 38 +- .../crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h | 12 +- kernel/drivers/crypto/qat/qat_dh895xcc/adf_drv.c | 102 +- kernel/drivers/crypto/qat/qat_dh895xcc/adf_drv.h | 9 - .../crypto/qat/qat_dh895xcc/adf_hw_arbiter.c | 159 - kernel/drivers/crypto/qat/qat_dh895xcc/adf_isr.c | 139 +- kernel/drivers/crypto/qat/qat_dh895xcc/qat_admin.c | 107 - kernel/drivers/crypto/qat/qat_dh895xccvf/Makefile | 5 + .../qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c | 172 + .../qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.h | 68 + kernel/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c | 393 +++ kernel/drivers/crypto/qat/qat_dh895xccvf/adf_drv.h | 57 + kernel/drivers/crypto/qat/qat_dh895xccvf/adf_isr.c | 258 ++ kernel/drivers/crypto/qce/ablkcipher.c | 30 +- kernel/drivers/crypto/qce/cipher.h | 4 - kernel/drivers/crypto/qce/dma.c | 52 - kernel/drivers/crypto/qce/dma.h | 5 - kernel/drivers/crypto/qce/sha.c | 20 +- kernel/drivers/crypto/qce/sha.h | 2 - kernel/drivers/crypto/sahara.c | 167 +- kernel/drivers/crypto/sunxi-ss/Makefile | 2 + kernel/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c | 542 ++++ kernel/drivers/crypto/sunxi-ss/sun4i-ss-core.c | 427 +++ kernel/drivers/crypto/sunxi-ss/sun4i-ss-hash.c | 492 +++ kernel/drivers/crypto/sunxi-ss/sun4i-ss.h | 201 ++ kernel/drivers/crypto/talitos.c | 1407 ++++---- kernel/drivers/crypto/talitos.h | 161 +- kernel/drivers/crypto/ux500/Kconfig | 4 +- kernel/drivers/crypto/ux500/cryp/cryp_core.c | 71 +- kernel/drivers/crypto/ux500/hash/hash_core.c | 50 +- kernel/drivers/crypto/vmx/Kconfig | 2 +- kernel/drivers/crypto/vmx/Makefile | 2 +- kernel/drivers/crypto/vmx/aes.c | 178 +- kernel/drivers/crypto/vmx/aes_cbc.c | 252 +- kernel/drivers/crypto/vmx/aes_ctr.c | 237 +- kernel/drivers/crypto/vmx/aesp8-ppc.h | 15 +- kernel/drivers/crypto/vmx/aesp8-ppc.pl | 34 +- kernel/drivers/crypto/vmx/ghash.c | 298 +- kernel/drivers/crypto/vmx/ppc-xlate.pl | 1 + kernel/drivers/crypto/vmx/vmx.c | 68 +- 127 files changed, 17080 insertions(+), 6908 deletions(-) create mode 100644 kernel/drivers/crypto/marvell/Makefile create mode 100644 kernel/drivers/crypto/marvell/cesa.c create mode 100644 kernel/drivers/crypto/marvell/cesa.h create mode 100644 kernel/drivers/crypto/marvell/cipher.c create mode 100644 kernel/drivers/crypto/marvell/hash.c create mode 100644 kernel/drivers/crypto/marvell/tdma.c create mode 100644 kernel/drivers/crypto/nx/nx-842-powernv.c create mode 100644 kernel/drivers/crypto/nx/nx-842-pseries.c create mode 100644 kernel/drivers/crypto/nx/nx-842.h create mode 100644 kernel/drivers/crypto/qat/qat_common/.gitignore create mode 100644 kernel/drivers/crypto/qat/qat_common/adf_admin.c create mode 100644 kernel/drivers/crypto/qat/qat_common/adf_hw_arbiter.c create mode 100644 kernel/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c create mode 100644 kernel/drivers/crypto/qat/qat_common/adf_pf2vf_msg.h create mode 100644 kernel/drivers/crypto/qat/qat_common/adf_sriov.c create mode 100644 kernel/drivers/crypto/qat/qat_common/icp_qat_fw_pke.h create mode 100644 kernel/drivers/crypto/qat/qat_common/qat_asym_algs.c create mode 100644 kernel/drivers/crypto/qat/qat_common/qat_rsaprivkey.asn1 create mode 100644 kernel/drivers/crypto/qat/qat_common/qat_rsapubkey.asn1 delete mode 100644 kernel/drivers/crypto/qat/qat_dh895xcc/adf_admin.c delete mode 100644 kernel/drivers/crypto/qat/qat_dh895xcc/adf_hw_arbiter.c delete mode 100644 kernel/drivers/crypto/qat/qat_dh895xcc/qat_admin.c create mode 100644 kernel/drivers/crypto/qat/qat_dh895xccvf/Makefile create mode 100644 kernel/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c create mode 100644 kernel/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.h create mode 100644 kernel/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c create mode 100644 kernel/drivers/crypto/qat/qat_dh895xccvf/adf_drv.h create mode 100644 kernel/drivers/crypto/qat/qat_dh895xccvf/adf_isr.c create mode 100644 kernel/drivers/crypto/sunxi-ss/Makefile create mode 100644 kernel/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c create mode 100644 kernel/drivers/crypto/sunxi-ss/sun4i-ss-core.c create mode 100644 kernel/drivers/crypto/sunxi-ss/sun4i-ss-hash.c create mode 100644 kernel/drivers/crypto/sunxi-ss/sun4i-ss.h (limited to 'kernel/drivers/crypto') diff --git a/kernel/drivers/crypto/Kconfig b/kernel/drivers/crypto/Kconfig index 033c0c86f..2569e0433 100644 --- a/kernel/drivers/crypto/Kconfig +++ b/kernel/drivers/crypto/Kconfig @@ -162,10 +162,10 @@ config CRYPTO_GHASH_S390 config CRYPTO_DEV_MV_CESA tristate "Marvell's Cryptographic Engine" depends on PLAT_ORION - select CRYPTO_ALGAPI select CRYPTO_AES - select CRYPTO_BLKCIPHER2 + select CRYPTO_BLKCIPHER select CRYPTO_HASH + select SRAM help This driver allows you to utilize the Cryptographic Engines and Security Accelerator (CESA) which can be found on the Marvell Orion @@ -173,10 +173,27 @@ config CRYPTO_DEV_MV_CESA Currently the driver supports AES in ECB and CBC mode without DMA. +config CRYPTO_DEV_MARVELL_CESA + tristate "New Marvell's Cryptographic Engine driver" + depends on PLAT_ORION || ARCH_MVEBU + select CRYPTO_AES + select CRYPTO_DES + select CRYPTO_BLKCIPHER + select CRYPTO_HASH + select SRAM + help + This driver allows you to utilize the Cryptographic Engines and + Security Accelerator (CESA) which can be found on the Armada 370. + This driver supports CPU offload through DMA transfers. + + This driver is aimed at replacing the mv_cesa driver. This will only + happen once it has received proper testing. + config CRYPTO_DEV_NIAGARA2 tristate "Niagara2 Stream Processing Unit driver" select CRYPTO_DES - select CRYPTO_ALGAPI + select CRYPTO_BLKCIPHER + select CRYPTO_HASH depends on SPARC64 help Each core of a Niagara2 processor contains a Stream @@ -189,7 +206,6 @@ config CRYPTO_DEV_NIAGARA2 config CRYPTO_DEV_HIFN_795X tristate "Driver HIFN 795x crypto accelerator chips" select CRYPTO_DES - select CRYPTO_ALGAPI select CRYPTO_BLKCIPHER select HW_RANDOM if CRYPTO_DEV_HIFN_795X_RNG depends on PCI @@ -208,8 +224,10 @@ source drivers/crypto/caam/Kconfig config CRYPTO_DEV_TALITOS tristate "Talitos Freescale Security Engine (SEC)" - select CRYPTO_ALGAPI + select CRYPTO_AEAD select CRYPTO_AUTHENC + select CRYPTO_BLKCIPHER + select CRYPTO_HASH select HW_RANDOM depends on FSL_SOC help @@ -222,11 +240,29 @@ config CRYPTO_DEV_TALITOS To compile this driver as a module, choose M here: the module will be called talitos. +config CRYPTO_DEV_TALITOS1 + bool "SEC1 (SEC 1.0 and SEC Lite 1.2)" + depends on CRYPTO_DEV_TALITOS + depends on PPC_8xx || PPC_82xx + default y + help + Say 'Y' here to use the Freescale Security Engine (SEC) version 1.0 + found on MPC82xx or the Freescale Security Engine (SEC Lite) + version 1.2 found on MPC8xx + +config CRYPTO_DEV_TALITOS2 + bool "SEC2+ (SEC version 2.0 or upper)" + depends on CRYPTO_DEV_TALITOS + default y if !PPC_8xx + help + Say 'Y' here to use the Freescale Security Engine (SEC) + version 2 and following as found on MPC83xx, MPC85xx, etc ... + config CRYPTO_DEV_IXP4XX tristate "Driver for IXP4xx crypto hardware acceleration" depends on ARCH_IXP4XX && IXP4XX_QMGR && IXP4XX_NPE select CRYPTO_DES - select CRYPTO_ALGAPI + select CRYPTO_AEAD select CRYPTO_AUTHENC select CRYPTO_BLKCIPHER help @@ -236,7 +272,6 @@ config CRYPTO_DEV_PPC4XX tristate "Driver AMCC PPC4xx crypto accelerator" depends on PPC && 4xx select CRYPTO_HASH - select CRYPTO_ALGAPI select CRYPTO_BLKCIPHER help This option allows you to have support for AMCC crypto acceleration. @@ -257,7 +292,7 @@ config CRYPTO_DEV_OMAP_AES tristate "Support for OMAP AES hw engine" depends on ARCH_OMAP2 || ARCH_OMAP3 || ARCH_OMAP2PLUS select CRYPTO_AES - select CRYPTO_BLKCIPHER2 + select CRYPTO_BLKCIPHER help OMAP processors have AES module accelerator. Select this if you want to use the OMAP module for AES algorithms. @@ -266,7 +301,7 @@ config CRYPTO_DEV_OMAP_DES tristate "Support for OMAP DES3DES hw engine" depends on ARCH_OMAP2PLUS select CRYPTO_DES - select CRYPTO_BLKCIPHER2 + select CRYPTO_BLKCIPHER help OMAP processors have DES/3DES module accelerator. Select this if you want to use the OMAP module for DES and 3DES algorithms. Currently @@ -276,9 +311,10 @@ config CRYPTO_DEV_OMAP_DES config CRYPTO_DEV_PICOXCELL tristate "Support for picoXcell IPSEC and Layer2 crypto engines" depends on ARCH_PICOXCELL && HAVE_CLK + select CRYPTO_AEAD select CRYPTO_AES select CRYPTO_AUTHENC - select CRYPTO_ALGAPI + select CRYPTO_BLKCIPHER select CRYPTO_DES select CRYPTO_CBC select CRYPTO_ECB @@ -304,7 +340,6 @@ config CRYPTO_DEV_S5P tristate "Support for Samsung S5PV210/Exynos crypto accelerator" depends on ARCH_S5PV210 || ARCH_EXYNOS select CRYPTO_AES - select CRYPTO_ALGAPI select CRYPTO_BLKCIPHER help This option allows you to have support for S5P crypto acceleration. @@ -312,11 +347,13 @@ config CRYPTO_DEV_S5P algorithms execution. config CRYPTO_DEV_NX - bool "Support for IBM Power7+ in-Nest cryptographic acceleration" - depends on PPC64 && IBMVIO && !CPU_LITTLE_ENDIAN - default n + bool "Support for IBM PowerPC Nest (NX) cryptographic acceleration" + depends on PPC64 help - Support for Power7+ in-Nest cryptographic acceleration. + This enables support for the NX hardware cryptographic accelerator + coprocessor that is in IBM PowerPC P7+ or later processors. This + does not actually enable any drivers, it only allows you to select + which acceleration type (encryption and/or compression) to enable. if CRYPTO_DEV_NX source "drivers/crypto/nx/Kconfig" @@ -325,7 +362,6 @@ endif config CRYPTO_DEV_UX500 tristate "Driver for ST-Ericsson UX500 crypto hardware acceleration" depends on ARCH_U8500 - select CRYPTO_ALGAPI help Driver for ST-Ericsson UX500 crypto engine. @@ -343,10 +379,7 @@ config CRYPTO_DEV_BFIN_CRC config CRYPTO_DEV_ATMEL_AES tristate "Support for Atmel AES hw accelerator" depends on ARCH_AT91 - select CRYPTO_CBC - select CRYPTO_ECB select CRYPTO_AES - select CRYPTO_ALGAPI select CRYPTO_BLKCIPHER select AT_HDMAC help @@ -361,9 +394,6 @@ config CRYPTO_DEV_ATMEL_TDES tristate "Support for Atmel DES/TDES hw accelerator" depends on ARCH_AT91 select CRYPTO_DES - select CRYPTO_CBC - select CRYPTO_ECB - select CRYPTO_ALGAPI select CRYPTO_BLKCIPHER help Some Atmel processors have DES/TDES hw accelerator. @@ -376,10 +406,7 @@ config CRYPTO_DEV_ATMEL_TDES config CRYPTO_DEV_ATMEL_SHA tristate "Support for Atmel SHA hw accelerator" depends on ARCH_AT91 - select CRYPTO_SHA1 - select CRYPTO_SHA256 - select CRYPTO_SHA512 - select CRYPTO_ALGAPI + select CRYPTO_HASH help Some Atmel processors have SHA1/SHA224/SHA256/SHA384/SHA512 hw accelerator. @@ -392,9 +419,8 @@ config CRYPTO_DEV_ATMEL_SHA config CRYPTO_DEV_CCP bool "Support for AMD Cryptographic Coprocessor" depends on ((X86 && PCI) || (ARM64 && (OF_ADDRESS || ACPI))) && HAS_IOMEM - default n help - The AMD Cryptographic Coprocessor provides hardware support + The AMD Cryptographic Coprocessor provides hardware offload support for encryption, hashing and related operations. if CRYPTO_DEV_CCP @@ -403,14 +429,13 @@ endif config CRYPTO_DEV_MXS_DCP tristate "Support for Freescale MXS DCP" - depends on ARCH_MXS - select CRYPTO_SHA1 - select CRYPTO_SHA256 + depends on (ARCH_MXS || ARCH_MXC) + select STMP_DEVICE select CRYPTO_CBC select CRYPTO_ECB select CRYPTO_AES select CRYPTO_BLKCIPHER - select CRYPTO_ALGAPI + select CRYPTO_HASH help The Freescale i.MX23/i.MX28 has SHA1/SHA256 and AES128 CBC/ECB co-processor on the die. @@ -429,7 +454,6 @@ config CRYPTO_DEV_QCE select CRYPTO_CBC select CRYPTO_XTS select CRYPTO_CTR - select CRYPTO_ALGAPI select CRYPTO_BLKCIPHER help This driver supports Qualcomm crypto engine accelerator @@ -438,8 +462,7 @@ config CRYPTO_DEV_QCE config CRYPTO_DEV_VMX bool "Support for VMX cryptographic acceleration instructions" - depends on PPC64 - default n + depends on PPC64 && VSX help Support for VMX cryptographic acceleration instructions. @@ -449,7 +472,6 @@ config CRYPTO_DEV_IMGTEC_HASH tristate "Imagination Technologies hardware hash accelerator" depends on MIPS || COMPILE_TEST depends on HAS_DMA - select CRYPTO_ALGAPI select CRYPTO_MD5 select CRYPTO_SHA1 select CRYPTO_SHA256 @@ -459,4 +481,21 @@ config CRYPTO_DEV_IMGTEC_HASH hardware hash accelerator. Supporting MD5/SHA1/SHA224/SHA256 hashing algorithms. +config CRYPTO_DEV_SUN4I_SS + tristate "Support for Allwinner Security System cryptographic accelerator" + depends on ARCH_SUNXI + select CRYPTO_MD5 + select CRYPTO_SHA1 + select CRYPTO_AES + select CRYPTO_DES + select CRYPTO_BLKCIPHER + help + Some Allwinner SoC have a crypto accelerator named + Security System. Select this if you want to use it. + The Security System handle AES/DES/3DES ciphers in CBC mode + and SHA1 and MD5 hash algorithms. + + To compile this driver as a module, choose M here: the module + will be called sun4i-ss. + endif # CRYPTO_HW diff --git a/kernel/drivers/crypto/Makefile b/kernel/drivers/crypto/Makefile index fb84be7e6..c3ced6fbd 100644 --- a/kernel/drivers/crypto/Makefile +++ b/kernel/drivers/crypto/Makefile @@ -9,6 +9,7 @@ obj-$(CONFIG_CRYPTO_DEV_HIFN_795X) += hifn_795x.o obj-$(CONFIG_CRYPTO_DEV_IMGTEC_HASH) += img-hash.o obj-$(CONFIG_CRYPTO_DEV_IXP4XX) += ixp4xx_crypto.o obj-$(CONFIG_CRYPTO_DEV_MV_CESA) += mv_cesa.o +obj-$(CONFIG_CRYPTO_DEV_MARVELL_CESA) += marvell/ obj-$(CONFIG_CRYPTO_DEV_MXS_DCP) += mxs-dcp.o obj-$(CONFIG_CRYPTO_DEV_NIAGARA2) += n2_crypto.o n2_crypto-y := n2_core.o n2_asm.o @@ -27,3 +28,4 @@ obj-$(CONFIG_CRYPTO_DEV_UX500) += ux500/ obj-$(CONFIG_CRYPTO_DEV_QAT) += qat/ obj-$(CONFIG_CRYPTO_DEV_QCE) += qce/ obj-$(CONFIG_CRYPTO_DEV_VMX) += vmx/ +obj-$(CONFIG_CRYPTO_DEV_SUN4I_SS) += sunxi-ss/ diff --git a/kernel/drivers/crypto/amcc/crypto4xx_core.c b/kernel/drivers/crypto/amcc/crypto4xx_core.c index 3b28e8c3d..58a630e55 100644 --- a/kernel/drivers/crypto/amcc/crypto4xx_core.c +++ b/kernel/drivers/crypto/amcc/crypto4xx_core.c @@ -740,26 +740,6 @@ void crypto4xx_return_pd(struct crypto4xx_device *dev, pd_uinfo->state = PD_ENTRY_FREE; } -/* - * derive number of elements in scatterlist - * Shamlessly copy from talitos.c - */ -static int get_sg_count(struct scatterlist *sg_list, int nbytes) -{ - struct scatterlist *sg = sg_list; - int sg_nents = 0; - - while (nbytes) { - sg_nents++; - if (sg->length > nbytes) - break; - nbytes -= sg->length; - sg = sg_next(sg); - } - - return sg_nents; -} - static u32 get_next_gd(u32 current) { if (current != PPC4XX_LAST_GD) @@ -800,7 +780,7 @@ u32 crypto4xx_build_pd(struct crypto_async_request *req, u32 gd_idx = 0; /* figure how many gd is needed */ - num_gd = get_sg_count(src, datalen); + num_gd = sg_nents_for_len(src, datalen); if (num_gd == 1) num_gd = 0; @@ -1113,7 +1093,7 @@ static irqreturn_t crypto4xx_ce_interrupt_handler(int irq, void *data) struct device *dev = (struct device *)data; struct crypto4xx_core_device *core_dev = dev_get_drvdata(dev); - if (core_dev->dev->ce_base == 0) + if (!core_dev->dev->ce_base) return 0; writel(PPC4XX_INTERRUPT_CLR, @@ -1284,6 +1264,7 @@ static const struct of_device_id crypto4xx_match[] = { { .compatible = "amcc,ppc4xx-crypto",}, { }, }; +MODULE_DEVICE_TABLE(of, crypto4xx_match); static struct platform_driver crypto4xx_driver = { .driver = { diff --git a/kernel/drivers/crypto/atmel-aes.c b/kernel/drivers/crypto/atmel-aes.c index 0f9a9dc06..fb16d812c 100644 --- a/kernel/drivers/crypto/atmel-aes.c +++ b/kernel/drivers/crypto/atmel-aes.c @@ -260,7 +260,11 @@ static struct atmel_aes_dev *atmel_aes_find_dev(struct atmel_aes_ctx *ctx) static int atmel_aes_hw_init(struct atmel_aes_dev *dd) { - clk_prepare_enable(dd->iclk); + int err; + + err = clk_prepare_enable(dd->iclk); + if (err) + return err; if (!(dd->flags & AES_FLAGS_INIT)) { atmel_aes_write(dd, AES_CR, AES_CR_SWRST); @@ -1320,7 +1324,6 @@ static int atmel_aes_probe(struct platform_device *pdev) struct crypto_platform_data *pdata; struct device *dev = &pdev->dev; struct resource *aes_res; - unsigned long aes_phys_size; int err; pdata = pdev->dev.platform_data; @@ -1337,7 +1340,7 @@ static int atmel_aes_probe(struct platform_device *pdev) goto aes_dd_err; } - aes_dd = kzalloc(sizeof(struct atmel_aes_dev), GFP_KERNEL); + aes_dd = devm_kzalloc(&pdev->dev, sizeof(*aes_dd), GFP_KERNEL); if (aes_dd == NULL) { dev_err(dev, "unable to alloc data struct.\n"); err = -ENOMEM; @@ -1368,36 +1371,35 @@ static int atmel_aes_probe(struct platform_device *pdev) goto res_err; } aes_dd->phys_base = aes_res->start; - aes_phys_size = resource_size(aes_res); /* Get the IRQ */ aes_dd->irq = platform_get_irq(pdev, 0); if (aes_dd->irq < 0) { dev_err(dev, "no IRQ resource info\n"); err = aes_dd->irq; - goto aes_irq_err; + goto res_err; } - err = request_irq(aes_dd->irq, atmel_aes_irq, IRQF_SHARED, "atmel-aes", - aes_dd); + err = devm_request_irq(&pdev->dev, aes_dd->irq, atmel_aes_irq, + IRQF_SHARED, "atmel-aes", aes_dd); if (err) { dev_err(dev, "unable to request aes irq.\n"); - goto aes_irq_err; + goto res_err; } /* Initializing the clock */ - aes_dd->iclk = clk_get(&pdev->dev, "aes_clk"); + aes_dd->iclk = devm_clk_get(&pdev->dev, "aes_clk"); if (IS_ERR(aes_dd->iclk)) { dev_err(dev, "clock initialization failed.\n"); err = PTR_ERR(aes_dd->iclk); - goto clk_err; + goto res_err; } - aes_dd->io_base = ioremap(aes_dd->phys_base, aes_phys_size); + aes_dd->io_base = devm_ioremap_resource(&pdev->dev, aes_res); if (!aes_dd->io_base) { dev_err(dev, "can't ioremap\n"); err = -ENOMEM; - goto aes_io_err; + goto res_err; } atmel_aes_hw_version_init(aes_dd); @@ -1434,17 +1436,9 @@ err_algs: err_aes_dma: atmel_aes_buff_cleanup(aes_dd); err_aes_buff: - iounmap(aes_dd->io_base); -aes_io_err: - clk_put(aes_dd->iclk); -clk_err: - free_irq(aes_dd->irq, aes_dd); -aes_irq_err: res_err: tasklet_kill(&aes_dd->done_task); tasklet_kill(&aes_dd->queue_task); - kfree(aes_dd); - aes_dd = NULL; aes_dd_err: dev_err(dev, "initialization failed.\n"); @@ -1469,16 +1463,6 @@ static int atmel_aes_remove(struct platform_device *pdev) atmel_aes_dma_cleanup(aes_dd); - iounmap(aes_dd->io_base); - - clk_put(aes_dd->iclk); - - if (aes_dd->irq > 0) - free_irq(aes_dd->irq, aes_dd); - - kfree(aes_dd); - aes_dd = NULL; - return 0; } diff --git a/kernel/drivers/crypto/atmel-sha.c b/kernel/drivers/crypto/atmel-sha.c index 5b35433c5..3178f84d2 100644 --- a/kernel/drivers/crypto/atmel-sha.c +++ b/kernel/drivers/crypto/atmel-sha.c @@ -783,7 +783,7 @@ static void atmel_sha_finish_req(struct ahash_request *req, int err) dd->flags &= ~(SHA_FLAGS_BUSY | SHA_FLAGS_FINAL | SHA_FLAGS_CPU | SHA_FLAGS_DMA_READY | SHA_FLAGS_OUTPUT_READY); - clk_disable_unprepare(dd->iclk); + clk_disable(dd->iclk); if (req->base.complete) req->base.complete(&req->base, err); @@ -794,7 +794,11 @@ static void atmel_sha_finish_req(struct ahash_request *req, int err) static int atmel_sha_hw_init(struct atmel_sha_dev *dd) { - clk_prepare_enable(dd->iclk); + int err; + + err = clk_enable(dd->iclk); + if (err) + return err; if (!(SHA_FLAGS_INIT & dd->flags)) { atmel_sha_write(dd, SHA_CR, SHA_CR_SWRST); @@ -819,7 +823,7 @@ static void atmel_sha_hw_version_init(struct atmel_sha_dev *dd) dev_info(dd->dev, "version: 0x%x\n", dd->hw_version); - clk_disable_unprepare(dd->iclk); + clk_disable(dd->iclk); } static int atmel_sha_handle_queue(struct atmel_sha_dev *dd, @@ -1345,11 +1349,9 @@ static int atmel_sha_probe(struct platform_device *pdev) struct crypto_platform_data *pdata; struct device *dev = &pdev->dev; struct resource *sha_res; - unsigned long sha_phys_size; int err; - sha_dd = devm_kzalloc(&pdev->dev, sizeof(struct atmel_sha_dev), - GFP_KERNEL); + sha_dd = devm_kzalloc(&pdev->dev, sizeof(*sha_dd), GFP_KERNEL); if (sha_dd == NULL) { dev_err(dev, "unable to alloc data struct.\n"); err = -ENOMEM; @@ -1378,7 +1380,6 @@ static int atmel_sha_probe(struct platform_device *pdev) goto res_err; } sha_dd->phys_base = sha_res->start; - sha_phys_size = resource_size(sha_res); /* Get the IRQ */ sha_dd->irq = platform_get_irq(pdev, 0); @@ -1388,28 +1389,32 @@ static int atmel_sha_probe(struct platform_device *pdev) goto res_err; } - err = request_irq(sha_dd->irq, atmel_sha_irq, IRQF_SHARED, "atmel-sha", - sha_dd); + err = devm_request_irq(&pdev->dev, sha_dd->irq, atmel_sha_irq, + IRQF_SHARED, "atmel-sha", sha_dd); if (err) { dev_err(dev, "unable to request sha irq.\n"); goto res_err; } /* Initializing the clock */ - sha_dd->iclk = clk_get(&pdev->dev, "sha_clk"); + sha_dd->iclk = devm_clk_get(&pdev->dev, "sha_clk"); if (IS_ERR(sha_dd->iclk)) { dev_err(dev, "clock initialization failed.\n"); err = PTR_ERR(sha_dd->iclk); - goto clk_err; + goto res_err; } - sha_dd->io_base = ioremap(sha_dd->phys_base, sha_phys_size); + sha_dd->io_base = devm_ioremap_resource(&pdev->dev, sha_res); if (!sha_dd->io_base) { dev_err(dev, "can't ioremap\n"); err = -ENOMEM; - goto sha_io_err; + goto res_err; } + err = clk_prepare(sha_dd->iclk); + if (err) + goto res_err; + atmel_sha_hw_version_init(sha_dd); atmel_sha_get_cap(sha_dd); @@ -1421,12 +1426,12 @@ static int atmel_sha_probe(struct platform_device *pdev) if (IS_ERR(pdata)) { dev_err(&pdev->dev, "platform data not available\n"); err = PTR_ERR(pdata); - goto err_pdata; + goto iclk_unprepare; } } if (!pdata->dma_slave) { err = -ENXIO; - goto err_pdata; + goto iclk_unprepare; } err = atmel_sha_dma_init(sha_dd, pdata); if (err) @@ -1457,12 +1462,8 @@ err_algs: if (sha_dd->caps.has_dma) atmel_sha_dma_cleanup(sha_dd); err_sha_dma: -err_pdata: - iounmap(sha_dd->io_base); -sha_io_err: - clk_put(sha_dd->iclk); -clk_err: - free_irq(sha_dd->irq, sha_dd); +iclk_unprepare: + clk_unprepare(sha_dd->iclk); res_err: tasklet_kill(&sha_dd->done_task); sha_dd_err: @@ -1489,12 +1490,7 @@ static int atmel_sha_remove(struct platform_device *pdev) if (sha_dd->caps.has_dma) atmel_sha_dma_cleanup(sha_dd); - iounmap(sha_dd->io_base); - - clk_put(sha_dd->iclk); - - if (sha_dd->irq >= 0) - free_irq(sha_dd->irq, sha_dd); + clk_unprepare(sha_dd->iclk); return 0; } diff --git a/kernel/drivers/crypto/atmel-tdes.c b/kernel/drivers/crypto/atmel-tdes.c index ca2999709..2c7a628d0 100644 --- a/kernel/drivers/crypto/atmel-tdes.c +++ b/kernel/drivers/crypto/atmel-tdes.c @@ -218,7 +218,11 @@ static struct atmel_tdes_dev *atmel_tdes_find_dev(struct atmel_tdes_ctx *ctx) static int atmel_tdes_hw_init(struct atmel_tdes_dev *dd) { - clk_prepare_enable(dd->iclk); + int err; + + err = clk_prepare_enable(dd->iclk); + if (err) + return err; if (!(dd->flags & TDES_FLAGS_INIT)) { atmel_tdes_write(dd, TDES_CR, TDES_CR_SWRST); @@ -1355,7 +1359,6 @@ static int atmel_tdes_probe(struct platform_device *pdev) struct crypto_platform_data *pdata; struct device *dev = &pdev->dev; struct resource *tdes_res; - unsigned long tdes_phys_size; int err; tdes_dd = devm_kmalloc(&pdev->dev, sizeof(*tdes_dd), GFP_KERNEL); @@ -1389,7 +1392,6 @@ static int atmel_tdes_probe(struct platform_device *pdev) goto res_err; } tdes_dd->phys_base = tdes_res->start; - tdes_phys_size = resource_size(tdes_res); /* Get the IRQ */ tdes_dd->irq = platform_get_irq(pdev, 0); @@ -1399,26 +1401,26 @@ static int atmel_tdes_probe(struct platform_device *pdev) goto res_err; } - err = request_irq(tdes_dd->irq, atmel_tdes_irq, IRQF_SHARED, - "atmel-tdes", tdes_dd); + err = devm_request_irq(&pdev->dev, tdes_dd->irq, atmel_tdes_irq, + IRQF_SHARED, "atmel-tdes", tdes_dd); if (err) { dev_err(dev, "unable to request tdes irq.\n"); - goto tdes_irq_err; + goto res_err; } /* Initializing the clock */ - tdes_dd->iclk = clk_get(&pdev->dev, "tdes_clk"); + tdes_dd->iclk = devm_clk_get(&pdev->dev, "tdes_clk"); if (IS_ERR(tdes_dd->iclk)) { dev_err(dev, "clock initialization failed.\n"); err = PTR_ERR(tdes_dd->iclk); - goto clk_err; + goto res_err; } - tdes_dd->io_base = ioremap(tdes_dd->phys_base, tdes_phys_size); + tdes_dd->io_base = devm_ioremap_resource(&pdev->dev, tdes_res); if (!tdes_dd->io_base) { dev_err(dev, "can't ioremap\n"); err = -ENOMEM; - goto tdes_io_err; + goto res_err; } atmel_tdes_hw_version_init(tdes_dd); @@ -1474,12 +1476,6 @@ err_tdes_dma: err_pdata: atmel_tdes_buff_cleanup(tdes_dd); err_tdes_buff: - iounmap(tdes_dd->io_base); -tdes_io_err: - clk_put(tdes_dd->iclk); -clk_err: - free_irq(tdes_dd->irq, tdes_dd); -tdes_irq_err: res_err: tasklet_kill(&tdes_dd->done_task); tasklet_kill(&tdes_dd->queue_task); @@ -1510,13 +1506,6 @@ static int atmel_tdes_remove(struct platform_device *pdev) atmel_tdes_buff_cleanup(tdes_dd); - iounmap(tdes_dd->io_base); - - clk_put(tdes_dd->iclk); - - if (tdes_dd->irq >= 0) - free_irq(tdes_dd->irq, tdes_dd); - return 0; } diff --git a/kernel/drivers/crypto/bfin_crc.c b/kernel/drivers/crypto/bfin_crc.c index d9af9403a..95b73968c 100644 --- a/kernel/drivers/crypto/bfin_crc.c +++ b/kernel/drivers/crypto/bfin_crc.c @@ -96,26 +96,6 @@ struct bfin_crypto_crc_ctx { u32 key; }; - -/* - * derive number of elements in scatterlist - */ -static int sg_count(struct scatterlist *sg_list) -{ - struct scatterlist *sg = sg_list; - int sg_nents = 1; - - if (sg_list == NULL) - return 0; - - while (!sg_is_last(sg)) { - sg_nents++; - sg = sg_next(sg); - } - - return sg_nents; -} - /* * get element in scatter list by given index */ @@ -160,7 +140,7 @@ static int bfin_crypto_crc_init(struct ahash_request *req) } spin_unlock_bh(&crc_list.lock); - if (sg_count(req->src) > CRC_MAX_DMA_DESC) { + if (sg_nents(req->src) > CRC_MAX_DMA_DESC) { dev_dbg(ctx->crc->dev, "init: requested sg list is too big > %d\n", CRC_MAX_DMA_DESC); return -EINVAL; @@ -370,14 +350,14 @@ static int bfin_crypto_crc_handle_queue(struct bfin_crypto_crc *crc, sg_init_table(ctx->bufsl, nsg); sg_set_buf(ctx->bufsl, ctx->buflast, ctx->buflast_len); if (nsg > 1) - scatterwalk_sg_chain(ctx->bufsl, nsg, - req->src); + sg_chain(ctx->bufsl, nsg, req->src); ctx->sg = ctx->bufsl; } else ctx->sg = req->src; /* Chop crc buffer size to multiple of 32 bit */ - nsg = ctx->sg_nents = sg_count(ctx->sg); + nsg = sg_nents(ctx->sg); + ctx->sg_nents = nsg; ctx->sg_buflen = ctx->buflast_len + req->nbytes; ctx->bufnext_len = ctx->sg_buflen % 4; ctx->sg_buflen &= ~0x3; diff --git a/kernel/drivers/crypto/caam/Kconfig b/kernel/drivers/crypto/caam/Kconfig index e7555ff4c..5652a5341 100644 --- a/kernel/drivers/crypto/caam/Kconfig +++ b/kernel/drivers/crypto/caam/Kconfig @@ -1,6 +1,6 @@ config CRYPTO_DEV_FSL_CAAM tristate "Freescale CAAM-Multicore driver backend" - depends on FSL_SOC + depends on FSL_SOC || ARCH_MXC help Enables the driver module for Freescale's Cryptographic Accelerator and Assurance Module (CAAM), also known as the SEC version 4 (SEC4). @@ -45,7 +45,6 @@ config CRYPTO_DEV_FSL_CAAM_RINGSIZE config CRYPTO_DEV_FSL_CAAM_INTC bool "Job Ring interrupt coalescing" depends on CRYPTO_DEV_FSL_CAAM_JR - default n help Enable the Job Ring's interrupt coalescing feature. @@ -77,8 +76,9 @@ config CRYPTO_DEV_FSL_CAAM_CRYPTO_API tristate "Register algorithm implementations with the Crypto API" depends on CRYPTO_DEV_FSL_CAAM && CRYPTO_DEV_FSL_CAAM_JR default y - select CRYPTO_ALGAPI + select CRYPTO_AEAD select CRYPTO_AUTHENC + select CRYPTO_BLKCIPHER help Selecting this will offload crypto for users of the scatterlist crypto API (such as the linux native IPSec @@ -112,10 +112,17 @@ config CRYPTO_DEV_FSL_CAAM_RNG_API To compile this as a module, choose M here: the module will be called caamrng. +config CRYPTO_DEV_FSL_CAAM_IMX + def_bool SOC_IMX6 || SOC_IMX7D + depends on CRYPTO_DEV_FSL_CAAM + +config CRYPTO_DEV_FSL_CAAM_LE + def_bool CRYPTO_DEV_FSL_CAAM_IMX || SOC_LS1021A + depends on CRYPTO_DEV_FSL_CAAM + config CRYPTO_DEV_FSL_CAAM_DEBUG bool "Enable debug output in CAAM driver" depends on CRYPTO_DEV_FSL_CAAM - default n help Selecting this will enable printing of various debug information in the CAAM driver. diff --git a/kernel/drivers/crypto/caam/caamalg.c b/kernel/drivers/crypto/caam/caamalg.c index 29071a156..ea8189f4b 100644 --- a/kernel/drivers/crypto/caam/caamalg.c +++ b/kernel/drivers/crypto/caam/caamalg.c @@ -65,32 +65,36 @@ /* max IV is max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */ #define CAAM_MAX_IV_LENGTH 16 +#define AEAD_DESC_JOB_IO_LEN (DESC_JOB_IO_LEN + CAAM_CMD_SZ * 2) +#define GCM_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \ + CAAM_CMD_SZ * 4) +#define AUTHENC_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \ + CAAM_CMD_SZ * 5) + /* length of descriptors text */ #define DESC_AEAD_BASE (4 * CAAM_CMD_SZ) -#define DESC_AEAD_ENC_LEN (DESC_AEAD_BASE + 15 * CAAM_CMD_SZ) -#define DESC_AEAD_DEC_LEN (DESC_AEAD_BASE + 18 * CAAM_CMD_SZ) -#define DESC_AEAD_GIVENC_LEN (DESC_AEAD_ENC_LEN + 7 * CAAM_CMD_SZ) +#define DESC_AEAD_ENC_LEN (DESC_AEAD_BASE + 11 * CAAM_CMD_SZ) +#define DESC_AEAD_DEC_LEN (DESC_AEAD_BASE + 15 * CAAM_CMD_SZ) +#define DESC_AEAD_GIVENC_LEN (DESC_AEAD_ENC_LEN + 9 * CAAM_CMD_SZ) /* Note: Nonce is counted in enckeylen */ -#define DESC_AEAD_CTR_RFC3686_LEN (6 * CAAM_CMD_SZ) +#define DESC_AEAD_CTR_RFC3686_LEN (4 * CAAM_CMD_SZ) #define DESC_AEAD_NULL_BASE (3 * CAAM_CMD_SZ) -#define DESC_AEAD_NULL_ENC_LEN (DESC_AEAD_NULL_BASE + 14 * CAAM_CMD_SZ) -#define DESC_AEAD_NULL_DEC_LEN (DESC_AEAD_NULL_BASE + 17 * CAAM_CMD_SZ) +#define DESC_AEAD_NULL_ENC_LEN (DESC_AEAD_NULL_BASE + 11 * CAAM_CMD_SZ) +#define DESC_AEAD_NULL_DEC_LEN (DESC_AEAD_NULL_BASE + 13 * CAAM_CMD_SZ) #define DESC_GCM_BASE (3 * CAAM_CMD_SZ) -#define DESC_GCM_ENC_LEN (DESC_GCM_BASE + 23 * CAAM_CMD_SZ) -#define DESC_GCM_DEC_LEN (DESC_GCM_BASE + 19 * CAAM_CMD_SZ) +#define DESC_GCM_ENC_LEN (DESC_GCM_BASE + 16 * CAAM_CMD_SZ) +#define DESC_GCM_DEC_LEN (DESC_GCM_BASE + 12 * CAAM_CMD_SZ) #define DESC_RFC4106_BASE (3 * CAAM_CMD_SZ) -#define DESC_RFC4106_ENC_LEN (DESC_RFC4106_BASE + 15 * CAAM_CMD_SZ) -#define DESC_RFC4106_DEC_LEN (DESC_RFC4106_BASE + 14 * CAAM_CMD_SZ) -#define DESC_RFC4106_GIVENC_LEN (DESC_RFC4106_BASE + 21 * CAAM_CMD_SZ) +#define DESC_RFC4106_ENC_LEN (DESC_RFC4106_BASE + 13 * CAAM_CMD_SZ) +#define DESC_RFC4106_DEC_LEN (DESC_RFC4106_BASE + 13 * CAAM_CMD_SZ) #define DESC_RFC4543_BASE (3 * CAAM_CMD_SZ) -#define DESC_RFC4543_ENC_LEN (DESC_RFC4543_BASE + 25 * CAAM_CMD_SZ) -#define DESC_RFC4543_DEC_LEN (DESC_RFC4543_BASE + 27 * CAAM_CMD_SZ) -#define DESC_RFC4543_GIVENC_LEN (DESC_RFC4543_BASE + 30 * CAAM_CMD_SZ) +#define DESC_RFC4543_ENC_LEN (DESC_RFC4543_BASE + 11 * CAAM_CMD_SZ) +#define DESC_RFC4543_DEC_LEN (DESC_RFC4543_BASE + 12 * CAAM_CMD_SZ) #define DESC_ABLKCIPHER_BASE (3 * CAAM_CMD_SZ) #define DESC_ABLKCIPHER_ENC_LEN (DESC_ABLKCIPHER_BASE + \ @@ -98,8 +102,7 @@ #define DESC_ABLKCIPHER_DEC_LEN (DESC_ABLKCIPHER_BASE + \ 15 * CAAM_CMD_SZ) -#define DESC_MAX_USED_BYTES (DESC_RFC4543_GIVENC_LEN + \ - CAAM_MAX_KEY_SIZE) +#define DESC_MAX_USED_BYTES (CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN) #define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ) #ifdef DEBUG @@ -110,6 +113,20 @@ #endif static struct list_head alg_list; +struct caam_alg_entry { + int class1_alg_type; + int class2_alg_type; + int alg_op; + bool rfc3686; + bool geniv; +}; + +struct caam_aead_alg { + struct aead_alg aead; + struct caam_alg_entry caam; + bool registered; +}; + /* Set DK bit in class 1 operation if shared */ static inline void append_dec_op1(u32 *desc, u32 type) { @@ -143,18 +160,6 @@ static inline void aead_append_src_dst(u32 *desc, u32 msg_type) KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH); } -/* - * For aead encrypt and decrypt, read iv for both classes - */ -static inline void aead_append_ld_iv(u32 *desc, int ivsize, int ivoffset) -{ - append_seq_load(desc, ivsize, LDST_CLASS_1_CCB | - LDST_SRCDST_BYTE_CONTEXT | - (ivoffset << LDST_OFFSET_SHIFT)); - append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO | - (ivoffset << MOVE_OFFSET_SHIFT) | ivsize); -} - /* * For ablkcipher encrypt and decrypt, read from req->src and * write to req->dst @@ -168,13 +173,6 @@ static inline void ablkcipher_append_src_dst(u32 *desc) append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); } -/* - * If all data, including src (with assoc and iv) or dst (with iv only) are - * contiguous - */ -#define GIV_SRC_CONTIG 1 -#define GIV_DST_CONTIG (1 << 1) - /* * per-session context */ @@ -258,7 +256,6 @@ static void init_sh_desc_key_aead(u32 *desc, struct caam_ctx *ctx, static int aead_null_set_sh_desc(struct crypto_aead *aead) { - struct aead_tfm *tfm = &aead->base.crt_aead; struct caam_ctx *ctx = crypto_aead_ctx(aead); struct device *jrdev = ctx->jrdev; bool keys_fit_inline = false; @@ -269,7 +266,7 @@ static int aead_null_set_sh_desc(struct crypto_aead *aead) * Job Descriptor and Shared Descriptors * must all fit into the 64-word Descriptor h/w Buffer */ - if (DESC_AEAD_NULL_ENC_LEN + DESC_JOB_IO_LEN + + if (DESC_AEAD_NULL_ENC_LEN + AEAD_DESC_JOB_IO_LEN + ctx->split_key_pad_len <= CAAM_DESC_BYTES_MAX) keys_fit_inline = true; @@ -290,20 +287,10 @@ static int aead_null_set_sh_desc(struct crypto_aead *aead) KEY_DEST_MDHA_SPLIT | KEY_ENC); set_jump_tgt_here(desc, key_jump_cmd); - /* cryptlen = seqoutlen - authsize */ - append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize); - - /* - * NULL encryption; IV is zero - * assoclen = (assoclen + cryptlen) - cryptlen - */ - append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG3, CAAM_CMD_SZ); - - /* read assoc before reading payload */ - append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | - KEY_VLF); + /* assoclen + cryptlen = seqinlen */ + append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ); - /* Prepare to read and write cryptlen bytes */ + /* Prepare to read and write cryptlen + assoclen bytes */ append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); @@ -381,18 +368,10 @@ static int aead_null_set_sh_desc(struct crypto_aead *aead) append_operation(desc, ctx->class2_alg_type | OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); - /* assoclen + cryptlen = seqinlen - ivsize - authsize */ - append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM, - ctx->authsize + tfm->ivsize); - /* assoclen = (assoclen + cryptlen) - cryptlen */ + /* assoclen + cryptlen = seqoutlen */ append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ); - append_math_sub(desc, VARSEQINLEN, REG3, REG2, CAAM_CMD_SZ); - /* read assoc before reading payload */ - append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | - KEY_VLF); - - /* Prepare to read and write cryptlen bytes */ + /* Prepare to read and write cryptlen + assoclen bytes */ append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ); append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ); @@ -449,10 +428,10 @@ static int aead_null_set_sh_desc(struct crypto_aead *aead) static int aead_set_sh_desc(struct crypto_aead *aead) { - struct aead_tfm *tfm = &aead->base.crt_aead; + struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), + struct caam_aead_alg, aead); + unsigned int ivsize = crypto_aead_ivsize(aead); struct caam_ctx *ctx = crypto_aead_ctx(aead); - struct crypto_tfm *ctfm = crypto_aead_tfm(aead); - const char *alg_name = crypto_tfm_alg_name(ctfm); struct device *jrdev = ctx->jrdev; bool keys_fit_inline; u32 geniv, moveiv; @@ -460,11 +439,7 @@ static int aead_set_sh_desc(struct crypto_aead *aead) u32 *desc; const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == OP_ALG_AAI_CTR_MOD128); - const bool is_rfc3686 = (ctr_mode && - (strstr(alg_name, "rfc3686") != NULL)); - - if (!ctx->authsize) - return 0; + const bool is_rfc3686 = alg->caam.rfc3686; /* NULL encryption / decryption */ if (!ctx->enckeylen) @@ -485,12 +460,15 @@ static int aead_set_sh_desc(struct crypto_aead *aead) if (is_rfc3686) ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; + if (alg->caam.geniv) + goto skip_enc; + /* * Job Descriptor and Shared Descriptors * must all fit into the 64-word Descriptor h/w Buffer */ keys_fit_inline = false; - if (DESC_AEAD_ENC_LEN + DESC_JOB_IO_LEN + + if (DESC_AEAD_ENC_LEN + AUTHENC_DESC_JOB_IO_LEN + ctx->split_key_pad_len + ctx->enckeylen + (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <= CAAM_DESC_BYTES_MAX) @@ -506,19 +484,16 @@ static int aead_set_sh_desc(struct crypto_aead *aead) append_operation(desc, ctx->class2_alg_type | OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); - /* cryptlen = seqoutlen - authsize */ - append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize); - - /* assoclen + cryptlen = seqinlen - ivsize */ - append_math_sub_imm_u32(desc, REG2, SEQINLEN, IMM, tfm->ivsize); + /* Read and write assoclen bytes */ + append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); + append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); - /* assoclen = (assoclen + cryptlen) - cryptlen */ - append_math_sub(desc, VARSEQINLEN, REG2, REG3, CAAM_CMD_SZ); + /* Skip assoc data */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); /* read assoc before reading payload */ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | - KEY_VLF); - aead_append_ld_iv(desc, tfm->ivsize, ctx1_iv_off); + FIFOLDST_VLF); /* Load Counter into CONTEXT1 reg */ if (is_rfc3686) @@ -533,8 +508,8 @@ static int aead_set_sh_desc(struct crypto_aead *aead) OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); /* Read and write cryptlen bytes */ - append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); - append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); + append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); + append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2); /* Write ICV */ @@ -554,12 +529,13 @@ static int aead_set_sh_desc(struct crypto_aead *aead) desc_bytes(desc), 1); #endif +skip_enc: /* * Job Descriptor and Shared Descriptors * must all fit into the 64-word Descriptor h/w Buffer */ keys_fit_inline = false; - if (DESC_AEAD_DEC_LEN + DESC_JOB_IO_LEN + + if (DESC_AEAD_DEC_LEN + AUTHENC_DESC_JOB_IO_LEN + ctx->split_key_pad_len + ctx->enckeylen + (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <= CAAM_DESC_BYTES_MAX) @@ -575,19 +551,17 @@ static int aead_set_sh_desc(struct crypto_aead *aead) append_operation(desc, ctx->class2_alg_type | OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); - /* assoclen + cryptlen = seqinlen - ivsize - authsize */ - append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM, - ctx->authsize + tfm->ivsize); - /* assoclen = (assoclen + cryptlen) - cryptlen */ - append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ); - append_math_sub(desc, VARSEQINLEN, REG3, REG2, CAAM_CMD_SZ); + /* Read and write assoclen bytes */ + append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); + append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); + + /* Skip assoc data */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); /* read assoc before reading payload */ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | KEY_VLF); - aead_append_ld_iv(desc, tfm->ivsize, ctx1_iv_off); - /* Load Counter into CONTEXT1 reg */ if (is_rfc3686) append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM | @@ -604,8 +578,8 @@ static int aead_set_sh_desc(struct crypto_aead *aead) append_dec_op1(desc, ctx->class1_alg_type); /* Read and write cryptlen bytes */ - append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ); - append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ); + append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); + append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); aead_append_src_dst(desc, FIFOLD_TYPE_MSG); /* Load ICV */ @@ -625,12 +599,15 @@ static int aead_set_sh_desc(struct crypto_aead *aead) desc_bytes(desc), 1); #endif + if (!alg->caam.geniv) + goto skip_givenc; + /* * Job Descriptor and Shared Descriptors * must all fit into the 64-word Descriptor h/w Buffer */ keys_fit_inline = false; - if (DESC_AEAD_GIVENC_LEN + DESC_JOB_IO_LEN + + if (DESC_AEAD_GIVENC_LEN + AUTHENC_DESC_JOB_IO_LEN + ctx->split_key_pad_len + ctx->enckeylen + (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <= CAAM_DESC_BYTES_MAX) @@ -642,23 +619,27 @@ static int aead_set_sh_desc(struct crypto_aead *aead) /* Note: Context registers are saved. */ init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686); + if (is_rfc3686) + goto copy_iv; + /* Generate IV */ geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO | NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | - NFIFOENTRY_PTYPE_RND | (tfm->ivsize << NFIFOENTRY_DLEN_SHIFT); + NFIFOENTRY_PTYPE_RND | (ivsize << NFIFOENTRY_DLEN_SHIFT); append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB | LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); append_move(desc, MOVE_WAITCOMP | MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX | (ctx1_iv_off << MOVE_OFFSET_SHIFT) | - (tfm->ivsize << MOVE_LEN_SHIFT)); + (ivsize << MOVE_LEN_SHIFT)); append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); +copy_iv: /* Copy IV to class 1 context */ append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO | (ctx1_iv_off << MOVE_OFFSET_SHIFT) | - (tfm->ivsize << MOVE_LEN_SHIFT)); + (ivsize << MOVE_LEN_SHIFT)); /* Return to encryption */ append_operation(desc, ctx->class2_alg_type | @@ -667,8 +648,12 @@ static int aead_set_sh_desc(struct crypto_aead *aead) /* ivsize + cryptlen = seqoutlen - authsize */ append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize); - /* assoclen = seqinlen - (ivsize + cryptlen) */ - append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG3, CAAM_CMD_SZ); + /* Read and write assoclen bytes */ + append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); + append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); + + /* Skip assoc data */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); /* read assoc before reading payload */ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | @@ -676,10 +661,10 @@ static int aead_set_sh_desc(struct crypto_aead *aead) /* Copy iv from outfifo to class 2 fifo */ moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 | - NFIFOENTRY_DTYPE_MSG | (tfm->ivsize << NFIFOENTRY_DLEN_SHIFT); + NFIFOENTRY_DTYPE_MSG | (ivsize << NFIFOENTRY_DLEN_SHIFT); append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB | LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); - append_load_imm_u32(desc, tfm->ivsize, LDST_CLASS_2_CCB | + append_load_imm_u32(desc, ivsize, LDST_CLASS_2_CCB | LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM); /* Load Counter into CONTEXT1 reg */ @@ -698,7 +683,7 @@ static int aead_set_sh_desc(struct crypto_aead *aead) append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); /* Not need to reload iv */ - append_seq_fifo_load(desc, tfm->ivsize, + append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_SKIP); /* Will read cryptlen */ @@ -709,9 +694,9 @@ static int aead_set_sh_desc(struct crypto_aead *aead) append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB | LDST_SRCDST_BYTE_CONTEXT); - ctx->sh_desc_givenc_dma = dma_map_single(jrdev, desc, - desc_bytes(desc), - DMA_TO_DEVICE); + ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, + desc_bytes(desc), + DMA_TO_DEVICE); if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) { dev_err(jrdev, "unable to map shared descriptor\n"); return -ENOMEM; @@ -722,6 +707,7 @@ static int aead_set_sh_desc(struct crypto_aead *aead) desc_bytes(desc), 1); #endif +skip_givenc: return 0; } @@ -738,7 +724,6 @@ static int aead_setauthsize(struct crypto_aead *authenc, static int gcm_set_sh_desc(struct crypto_aead *aead) { - struct aead_tfm *tfm = &aead->base.crt_aead; struct caam_ctx *ctx = crypto_aead_ctx(aead); struct device *jrdev = ctx->jrdev; bool keys_fit_inline = false; @@ -754,7 +739,7 @@ static int gcm_set_sh_desc(struct crypto_aead *aead) * Job Descriptor and Shared Descriptor * must fit into the 64-word Descriptor h/w Buffer */ - if (DESC_GCM_ENC_LEN + DESC_JOB_IO_LEN + + if (DESC_GCM_ENC_LEN + GCM_DESC_JOB_IO_LEN + ctx->enckeylen <= CAAM_DESC_BYTES_MAX) keys_fit_inline = true; @@ -777,34 +762,34 @@ static int gcm_set_sh_desc(struct crypto_aead *aead) append_operation(desc, ctx->class1_alg_type | OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); - /* cryptlen = seqoutlen - authsize */ - append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize); + /* if assoclen + cryptlen is ZERO, skip to ICV write */ + append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); + zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL | + JUMP_COND_MATH_Z); + + /* if assoclen is ZERO, skip reading the assoc data */ + append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); + zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL | + JUMP_COND_MATH_Z); + + append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); - /* assoclen + cryptlen = seqinlen - ivsize */ - append_math_sub_imm_u32(desc, REG2, SEQINLEN, IMM, tfm->ivsize); + /* skip assoc data */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); - /* assoclen = (assoclen + cryptlen) - cryptlen */ - append_math_sub(desc, REG1, REG2, REG3, CAAM_CMD_SZ); + /* cryptlen = seqinlen - assoclen */ + append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ); /* if cryptlen is ZERO jump to zero-payload commands */ - append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_MATH_Z); - /* read IV */ - append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_CLASS1 | - FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); - - /* if assoclen is ZERO, skip reading the assoc data */ - append_math_add(desc, VARSEQINLEN, ZERO, REG1, CAAM_CMD_SZ); - zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL | - JUMP_COND_MATH_Z); /* read assoc data */ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); set_jump_tgt_here(desc, zero_assoc_jump_cmd1); - append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); + append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); /* write encrypted data */ append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); @@ -814,31 +799,17 @@ static int gcm_set_sh_desc(struct crypto_aead *aead) FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); /* jump the zero-payload commands */ - append_jump(desc, JUMP_TEST_ALL | 7); + append_jump(desc, JUMP_TEST_ALL | 2); /* zero-payload commands */ set_jump_tgt_here(desc, zero_payload_jump_cmd); - /* if assoclen is ZERO, jump to IV reading - is the only input data */ - append_math_add(desc, VARSEQINLEN, ZERO, REG1, CAAM_CMD_SZ); - zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL | - JUMP_COND_MATH_Z); - /* read IV */ - append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_CLASS1 | - FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); - /* read assoc data */ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1); - /* jump to ICV writing */ - append_jump(desc, JUMP_TEST_ALL | 2); - - /* read IV - is the only input data */ + /* There is no input data */ set_jump_tgt_here(desc, zero_assoc_jump_cmd2); - append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_CLASS1 | - FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | - FIFOLD_TYPE_LAST1); /* write ICV */ append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB | @@ -862,7 +833,7 @@ static int gcm_set_sh_desc(struct crypto_aead *aead) * must all fit into the 64-word Descriptor h/w Buffer */ keys_fit_inline = false; - if (DESC_GCM_DEC_LEN + DESC_JOB_IO_LEN + + if (DESC_GCM_DEC_LEN + GCM_DESC_JOB_IO_LEN + ctx->enckeylen <= CAAM_DESC_BYTES_MAX) keys_fit_inline = true; @@ -886,33 +857,30 @@ static int gcm_set_sh_desc(struct crypto_aead *aead) append_operation(desc, ctx->class1_alg_type | OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); - /* assoclen + cryptlen = seqinlen - ivsize - icvsize */ - append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM, - ctx->authsize + tfm->ivsize); - - /* assoclen = (assoclen + cryptlen) - cryptlen */ - append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ); - append_math_sub(desc, REG1, REG3, REG2, CAAM_CMD_SZ); + /* if assoclen is ZERO, skip reading the assoc data */ + append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); + zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL | + JUMP_COND_MATH_Z); - /* read IV */ - append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_CLASS1 | - FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); + append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); - /* jump to zero-payload command if cryptlen is zero */ - append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ); - zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL | - JUMP_COND_MATH_Z); + /* skip assoc data */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); - append_math_add(desc, VARSEQINLEN, ZERO, REG1, CAAM_CMD_SZ); - /* if asoclen is ZERO, skip reading assoc data */ - zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL | - JUMP_COND_MATH_Z); /* read assoc data */ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); + set_jump_tgt_here(desc, zero_assoc_jump_cmd1); - append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ); + /* cryptlen = seqoutlen - assoclen */ + append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); + + /* jump to zero-payload command if cryptlen is zero */ + zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL | + JUMP_COND_MATH_Z); + + append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); /* store encrypted data */ append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); @@ -921,21 +889,9 @@ static int gcm_set_sh_desc(struct crypto_aead *aead) append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); - /* jump the zero-payload commands */ - append_jump(desc, JUMP_TEST_ALL | 4); - /* zero-payload command */ set_jump_tgt_here(desc, zero_payload_jump_cmd); - /* if assoclen is ZERO, jump to ICV reading */ - append_math_add(desc, VARSEQINLEN, ZERO, REG1, CAAM_CMD_SZ); - zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL | - JUMP_COND_MATH_Z); - /* read assoc data */ - append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | - FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); - set_jump_tgt_here(desc, zero_assoc_jump_cmd2); - /* read ICV */ append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); @@ -968,13 +924,11 @@ static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize) static int rfc4106_set_sh_desc(struct crypto_aead *aead) { - struct aead_tfm *tfm = &aead->base.crt_aead; struct caam_ctx *ctx = crypto_aead_ctx(aead); struct device *jrdev = ctx->jrdev; bool keys_fit_inline = false; - u32 *key_jump_cmd, *move_cmd, *write_iv_cmd; + u32 *key_jump_cmd; u32 *desc; - u32 geniv; if (!ctx->enckeylen || !ctx->authsize) return 0; @@ -984,7 +938,7 @@ static int rfc4106_set_sh_desc(struct crypto_aead *aead) * Job Descriptor and Shared Descriptor * must fit into the 64-word Descriptor h/w Buffer */ - if (DESC_RFC4106_ENC_LEN + DESC_JOB_IO_LEN + + if (DESC_RFC4106_ENC_LEN + GCM_DESC_JOB_IO_LEN + ctx->enckeylen <= CAAM_DESC_BYTES_MAX) keys_fit_inline = true; @@ -1007,29 +961,27 @@ static int rfc4106_set_sh_desc(struct crypto_aead *aead) append_operation(desc, ctx->class1_alg_type | OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); - /* cryptlen = seqoutlen - authsize */ - append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize); + append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, 8); append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); - /* assoclen + cryptlen = seqinlen - ivsize */ - append_math_sub_imm_u32(desc, REG2, SEQINLEN, IMM, tfm->ivsize); - - /* assoclen = (assoclen + cryptlen) - cryptlen */ - append_math_sub(desc, VARSEQINLEN, REG2, REG3, CAAM_CMD_SZ); - - /* Read Salt */ - append_fifo_load_as_imm(desc, (void *)(ctx->key + ctx->enckeylen), - 4, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_IV); - /* Read AES-GCM-ESP IV */ - append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_CLASS1 | - FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); - /* Read assoc data */ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); + /* Skip IV */ + append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); + /* Will read cryptlen bytes */ - append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); + append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); + + /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG); + + /* Skip assoc data */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); + + /* cryptlen = seqoutlen - assoclen */ + append_math_sub(desc, VARSEQOUTLEN, VARSEQINLEN, REG0, CAAM_CMD_SZ); /* Write encrypted data */ append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); @@ -1083,30 +1035,27 @@ static int rfc4106_set_sh_desc(struct crypto_aead *aead) append_operation(desc, ctx->class1_alg_type | OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); - /* assoclen + cryptlen = seqinlen - ivsize - icvsize */ - append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM, - ctx->authsize + tfm->ivsize); - - /* assoclen = (assoclen + cryptlen) - cryptlen */ - append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ); - append_math_sub(desc, VARSEQINLEN, REG3, REG2, CAAM_CMD_SZ); - - /* Will write cryptlen bytes */ - append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); - - /* Read Salt */ - append_fifo_load_as_imm(desc, (void *)(ctx->key + ctx->enckeylen), - 4, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_IV); - /* Read AES-GCM-ESP IV */ - append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_CLASS1 | - FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); + append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, 8); + append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); /* Read assoc data */ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); + /* Skip IV */ + append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); + /* Will read cryptlen bytes */ - append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ); + append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG3, CAAM_CMD_SZ); + + /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG); + + /* Skip assoc data */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); + + /* Will write cryptlen bytes */ + append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); /* Store payload data */ append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); @@ -1132,107 +1081,6 @@ static int rfc4106_set_sh_desc(struct crypto_aead *aead) desc_bytes(desc), 1); #endif - /* - * Job Descriptor and Shared Descriptors - * must all fit into the 64-word Descriptor h/w Buffer - */ - keys_fit_inline = false; - if (DESC_RFC4106_GIVENC_LEN + DESC_JOB_IO_LEN + - ctx->split_key_pad_len + ctx->enckeylen <= - CAAM_DESC_BYTES_MAX) - keys_fit_inline = true; - - /* rfc4106_givencrypt shared descriptor */ - desc = ctx->sh_desc_givenc; - - init_sh_desc(desc, HDR_SHARE_SERIAL); - - /* Skip key loading if it is loaded due to sharing */ - key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | - JUMP_COND_SHRD); - if (keys_fit_inline) - append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, - ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); - else - append_key(desc, ctx->key_dma, ctx->enckeylen, - CLASS_1 | KEY_DEST_CLASS_REG); - set_jump_tgt_here(desc, key_jump_cmd); - - /* Generate IV */ - geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO | - NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | - NFIFOENTRY_PTYPE_RND | (tfm->ivsize << NFIFOENTRY_DLEN_SHIFT); - append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB | - LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); - append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); - move_cmd = append_move(desc, MOVE_SRC_INFIFO | MOVE_DEST_DESCBUF | - (tfm->ivsize << MOVE_LEN_SHIFT)); - append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); - - /* Copy generated IV to OFIFO */ - write_iv_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_OUTFIFO | - (tfm->ivsize << MOVE_LEN_SHIFT)); - - /* Class 1 operation */ - append_operation(desc, ctx->class1_alg_type | - OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); - - /* ivsize + cryptlen = seqoutlen - authsize */ - append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize); - - /* assoclen = seqinlen - (ivsize + cryptlen) */ - append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG3, CAAM_CMD_SZ); - - /* Will write ivsize + cryptlen */ - append_math_add(desc, VARSEQOUTLEN, REG3, REG0, CAAM_CMD_SZ); - - /* Read Salt and generated IV */ - append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_IV | - FIFOLD_TYPE_FLUSH1 | IMMEDIATE | 12); - /* Append Salt */ - append_data(desc, (void *)(ctx->key + ctx->enckeylen), 4); - set_move_tgt_here(desc, move_cmd); - set_move_tgt_here(desc, write_iv_cmd); - /* Blank commands. Will be overwritten by generated IV. */ - append_cmd(desc, 0x00000000); - append_cmd(desc, 0x00000000); - /* End of blank commands */ - - /* No need to reload iv */ - append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_SKIP); - - /* Read assoc data */ - append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | - FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); - - /* Will read cryptlen */ - append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); - - /* Store generated IV and encrypted data */ - append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); - - /* Read payload data */ - append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | - FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); - - /* Write ICV */ - append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB | - LDST_SRCDST_BYTE_CONTEXT); - - ctx->sh_desc_givenc_dma = dma_map_single(jrdev, desc, - desc_bytes(desc), - DMA_TO_DEVICE); - if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) { - dev_err(jrdev, "unable to map shared descriptor\n"); - return -ENOMEM; - } -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "rfc4106 givenc shdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, - desc_bytes(desc), 1); -#endif - return 0; } @@ -1249,14 +1097,12 @@ static int rfc4106_setauthsize(struct crypto_aead *authenc, static int rfc4543_set_sh_desc(struct crypto_aead *aead) { - struct aead_tfm *tfm = &aead->base.crt_aead; struct caam_ctx *ctx = crypto_aead_ctx(aead); struct device *jrdev = ctx->jrdev; bool keys_fit_inline = false; - u32 *key_jump_cmd, *write_iv_cmd, *write_aad_cmd; + u32 *key_jump_cmd; u32 *read_move_cmd, *write_move_cmd; u32 *desc; - u32 geniv; if (!ctx->enckeylen || !ctx->authsize) return 0; @@ -1266,7 +1112,7 @@ static int rfc4543_set_sh_desc(struct crypto_aead *aead) * Job Descriptor and Shared Descriptor * must fit into the 64-word Descriptor h/w Buffer */ - if (DESC_RFC4543_ENC_LEN + DESC_JOB_IO_LEN + + if (DESC_RFC4543_ENC_LEN + GCM_DESC_JOB_IO_LEN + ctx->enckeylen <= CAAM_DESC_BYTES_MAX) keys_fit_inline = true; @@ -1289,48 +1135,8 @@ static int rfc4543_set_sh_desc(struct crypto_aead *aead) append_operation(desc, ctx->class1_alg_type | OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); - /* Load AES-GMAC ESP IV into Math1 register */ - append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_WORD_DECO_MATH1 | - LDST_CLASS_DECO | tfm->ivsize); - - /* Wait the DMA transaction to finish */ - append_jump(desc, JUMP_TEST_ALL | JUMP_COND_CALM | - (1 << JUMP_OFFSET_SHIFT)); - - /* Overwrite blank immediate AES-GMAC ESP IV data */ - write_iv_cmd = append_move(desc, MOVE_SRC_MATH1 | MOVE_DEST_DESCBUF | - (tfm->ivsize << MOVE_LEN_SHIFT)); - - /* Overwrite blank immediate AAD data */ - write_aad_cmd = append_move(desc, MOVE_SRC_MATH1 | MOVE_DEST_DESCBUF | - (tfm->ivsize << MOVE_LEN_SHIFT)); - - /* cryptlen = seqoutlen - authsize */ - append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize); - - /* assoclen = (seqinlen - ivsize) - cryptlen */ - append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG3, CAAM_CMD_SZ); - - /* Read Salt and AES-GMAC ESP IV */ - append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE | - FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | (4 + tfm->ivsize)); - /* Append Salt */ - append_data(desc, (void *)(ctx->key + ctx->enckeylen), 4); - set_move_tgt_here(desc, write_iv_cmd); - /* Blank commands. Will be overwritten by AES-GMAC ESP IV. */ - append_cmd(desc, 0x00000000); - append_cmd(desc, 0x00000000); - /* End of blank commands */ - - /* Read assoc data */ - append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | - FIFOLD_TYPE_AAD); - - /* Will read cryptlen bytes */ - append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); - - /* Will write cryptlen bytes */ - append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); + /* assoclen + cryptlen = seqinlen */ + append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ); /* * MOVE_LEN opcode is not available in all SEC HW revisions, @@ -1342,16 +1148,13 @@ static int rfc4543_set_sh_desc(struct crypto_aead *aead) write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF | (0x8 << MOVE_LEN_SHIFT)); - /* Authenticate AES-GMAC ESP IV */ - append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE | - FIFOLD_TYPE_AAD | tfm->ivsize); - set_move_tgt_here(desc, write_aad_cmd); - /* Blank commands. Will be overwritten by AES-GMAC ESP IV. */ - append_cmd(desc, 0x00000000); - append_cmd(desc, 0x00000000); - /* End of blank commands */ + /* Will read assoclen + cryptlen bytes */ + append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); - /* Read and write cryptlen bytes */ + /* Will write assoclen + cryptlen bytes */ + append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); + + /* Read and write assoclen + cryptlen bytes */ aead_append_src_dst(desc, FIFOLD_TYPE_AAD); set_move_tgt_here(desc, read_move_cmd); @@ -1382,7 +1185,7 @@ static int rfc4543_set_sh_desc(struct crypto_aead *aead) * must all fit into the 64-word Descriptor h/w Buffer */ keys_fit_inline = false; - if (DESC_RFC4543_DEC_LEN + DESC_JOB_IO_LEN + + if (DESC_RFC4543_DEC_LEN + GCM_DESC_JOB_IO_LEN + ctx->enckeylen <= CAAM_DESC_BYTES_MAX) keys_fit_inline = true; @@ -1405,28 +1208,8 @@ static int rfc4543_set_sh_desc(struct crypto_aead *aead) append_operation(desc, ctx->class1_alg_type | OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); - /* Load AES-GMAC ESP IV into Math1 register */ - append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_WORD_DECO_MATH1 | - LDST_CLASS_DECO | tfm->ivsize); - - /* Wait the DMA transaction to finish */ - append_jump(desc, JUMP_TEST_ALL | JUMP_COND_CALM | - (1 << JUMP_OFFSET_SHIFT)); - - /* assoclen + cryptlen = (seqinlen - ivsize) - icvsize */ - append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM, ctx->authsize); - - /* Overwrite blank immediate AES-GMAC ESP IV data */ - write_iv_cmd = append_move(desc, MOVE_SRC_MATH1 | MOVE_DEST_DESCBUF | - (tfm->ivsize << MOVE_LEN_SHIFT)); - - /* Overwrite blank immediate AAD data */ - write_aad_cmd = append_move(desc, MOVE_SRC_MATH1 | MOVE_DEST_DESCBUF | - (tfm->ivsize << MOVE_LEN_SHIFT)); - - /* assoclen = (assoclen + cryptlen) - cryptlen */ - append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ); - append_math_sub(desc, VARSEQINLEN, REG3, REG2, CAAM_CMD_SZ); + /* assoclen + cryptlen = seqoutlen */ + append_math_sub(desc, REG3, SEQOUTLEN, REG0, CAAM_CMD_SZ); /* * MOVE_LEN opcode is not available in all SEC HW revisions, @@ -1438,40 +1221,16 @@ static int rfc4543_set_sh_desc(struct crypto_aead *aead) write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF | (0x8 << MOVE_LEN_SHIFT)); - /* Read Salt and AES-GMAC ESP IV */ - append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE | - FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | (4 + tfm->ivsize)); - /* Append Salt */ - append_data(desc, (void *)(ctx->key + ctx->enckeylen), 4); - set_move_tgt_here(desc, write_iv_cmd); - /* Blank commands. Will be overwritten by AES-GMAC ESP IV. */ - append_cmd(desc, 0x00000000); - append_cmd(desc, 0x00000000); - /* End of blank commands */ - - /* Read assoc data */ - append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | - FIFOLD_TYPE_AAD); - - /* Will read cryptlen bytes */ - append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ); - - /* Will write cryptlen bytes */ - append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ); + /* Will read assoclen + cryptlen bytes */ + append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); - /* Authenticate AES-GMAC ESP IV */ - append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE | - FIFOLD_TYPE_AAD | tfm->ivsize); - set_move_tgt_here(desc, write_aad_cmd); - /* Blank commands. Will be overwritten by AES-GMAC ESP IV. */ - append_cmd(desc, 0x00000000); - append_cmd(desc, 0x00000000); - /* End of blank commands */ + /* Will write assoclen + cryptlen bytes */ + append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); /* Store payload data */ append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); - /* In-snoop cryptlen data */ + /* In-snoop assoclen + cryptlen data */ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | FIFOLDST_VLF | FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST2FLUSH1); @@ -1499,135 +1258,6 @@ static int rfc4543_set_sh_desc(struct crypto_aead *aead) desc_bytes(desc), 1); #endif - /* - * Job Descriptor and Shared Descriptors - * must all fit into the 64-word Descriptor h/w Buffer - */ - keys_fit_inline = false; - if (DESC_RFC4543_GIVENC_LEN + DESC_JOB_IO_LEN + - ctx->enckeylen <= CAAM_DESC_BYTES_MAX) - keys_fit_inline = true; - - /* rfc4543_givencrypt shared descriptor */ - desc = ctx->sh_desc_givenc; - - init_sh_desc(desc, HDR_SHARE_SERIAL); - - /* Skip key loading if it is loaded due to sharing */ - key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | - JUMP_COND_SHRD); - if (keys_fit_inline) - append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, - ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); - else - append_key(desc, ctx->key_dma, ctx->enckeylen, - CLASS_1 | KEY_DEST_CLASS_REG); - set_jump_tgt_here(desc, key_jump_cmd); - - /* Generate IV */ - geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO | - NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | - NFIFOENTRY_PTYPE_RND | (tfm->ivsize << NFIFOENTRY_DLEN_SHIFT); - append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB | - LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); - append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); - /* Move generated IV to Math1 register */ - append_move(desc, MOVE_SRC_INFIFO | MOVE_DEST_MATH1 | - (tfm->ivsize << MOVE_LEN_SHIFT)); - append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); - - /* Overwrite blank immediate AES-GMAC IV data */ - write_iv_cmd = append_move(desc, MOVE_SRC_MATH1 | MOVE_DEST_DESCBUF | - (tfm->ivsize << MOVE_LEN_SHIFT)); - - /* Overwrite blank immediate AAD data */ - write_aad_cmd = append_move(desc, MOVE_SRC_MATH1 | MOVE_DEST_DESCBUF | - (tfm->ivsize << MOVE_LEN_SHIFT)); - - /* Copy generated IV to OFIFO */ - append_move(desc, MOVE_SRC_MATH1 | MOVE_DEST_OUTFIFO | - (tfm->ivsize << MOVE_LEN_SHIFT)); - - /* Class 1 operation */ - append_operation(desc, ctx->class1_alg_type | - OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); - - /* ivsize + cryptlen = seqoutlen - authsize */ - append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize); - - /* assoclen = seqinlen - (ivsize + cryptlen) */ - append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG3, CAAM_CMD_SZ); - - /* Will write ivsize + cryptlen */ - append_math_add(desc, VARSEQOUTLEN, REG3, REG0, CAAM_CMD_SZ); - - /* - * MOVE_LEN opcode is not available in all SEC HW revisions, - * thus need to do some magic, i.e. self-patch the descriptor - * buffer. - */ - read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 | - (0x6 << MOVE_LEN_SHIFT)); - write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF | - (0x8 << MOVE_LEN_SHIFT)); - - /* Read Salt and AES-GMAC generated IV */ - append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE | - FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | (4 + tfm->ivsize)); - /* Append Salt */ - append_data(desc, (void *)(ctx->key + ctx->enckeylen), 4); - set_move_tgt_here(desc, write_iv_cmd); - /* Blank commands. Will be overwritten by AES-GMAC generated IV. */ - append_cmd(desc, 0x00000000); - append_cmd(desc, 0x00000000); - /* End of blank commands */ - - /* No need to reload iv */ - append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_SKIP); - - /* Read assoc data */ - append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | - FIFOLD_TYPE_AAD); - - /* Will read cryptlen */ - append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); - - /* Authenticate AES-GMAC IV */ - append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE | - FIFOLD_TYPE_AAD | tfm->ivsize); - set_move_tgt_here(desc, write_aad_cmd); - /* Blank commands. Will be overwritten by AES-GMAC IV. */ - append_cmd(desc, 0x00000000); - append_cmd(desc, 0x00000000); - /* End of blank commands */ - - /* Read and write cryptlen bytes */ - aead_append_src_dst(desc, FIFOLD_TYPE_AAD); - - set_move_tgt_here(desc, read_move_cmd); - set_move_tgt_here(desc, write_move_cmd); - append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); - /* Move payload data to OFIFO */ - append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO); - - /* Write ICV */ - append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB | - LDST_SRCDST_BYTE_CONTEXT); - - ctx->sh_desc_givenc_dma = dma_map_single(jrdev, desc, - desc_bytes(desc), - DMA_TO_DEVICE); - if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) { - dev_err(jrdev, "unable to map shared descriptor\n"); - return -ENOMEM; - } -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "rfc4543 givenc shdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, - desc_bytes(desc), 1); -#endif - return 0; } @@ -2075,14 +1705,131 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, return ret; } +static int xts_ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, + const u8 *key, unsigned int keylen) +{ + struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); + struct device *jrdev = ctx->jrdev; + u32 *key_jump_cmd, *desc; + __be64 sector_size = cpu_to_be64(512); + + if (keylen != 2 * AES_MIN_KEY_SIZE && keylen != 2 * AES_MAX_KEY_SIZE) { + crypto_ablkcipher_set_flags(ablkcipher, + CRYPTO_TFM_RES_BAD_KEY_LEN); + dev_err(jrdev, "key size mismatch\n"); + return -EINVAL; + } + + memcpy(ctx->key, key, keylen); + ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen, DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, ctx->key_dma)) { + dev_err(jrdev, "unable to map key i/o memory\n"); + return -ENOMEM; + } + ctx->enckeylen = keylen; + + /* xts_ablkcipher_encrypt shared descriptor */ + desc = ctx->sh_desc_enc; + init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); + /* Skip if already shared */ + key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_SHRD); + + /* Load class1 keys only */ + append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, + ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); + + /* Load sector size with index 40 bytes (0x28) */ + append_cmd(desc, CMD_LOAD | IMMEDIATE | LDST_SRCDST_BYTE_CONTEXT | + LDST_CLASS_1_CCB | (0x28 << LDST_OFFSET_SHIFT) | 8); + append_data(desc, (void *)§or_size, 8); + + set_jump_tgt_here(desc, key_jump_cmd); + + /* + * create sequence for loading the sector index + * Upper 8B of IV - will be used as sector index + * Lower 8B of IV - will be discarded + */ + append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT | + LDST_CLASS_1_CCB | (0x20 << LDST_OFFSET_SHIFT) | 8); + append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); + + /* Load operation */ + append_operation(desc, ctx->class1_alg_type | OP_ALG_AS_INITFINAL | + OP_ALG_ENCRYPT); + + /* Perform operation */ + ablkcipher_append_src_dst(desc); + + ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, desc_bytes(desc), + DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { + dev_err(jrdev, "unable to map shared descriptor\n"); + return -ENOMEM; + } +#ifdef DEBUG + print_hex_dump(KERN_ERR, + "xts ablkcipher enc shdesc@" __stringify(__LINE__) ": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); +#endif + + /* xts_ablkcipher_decrypt shared descriptor */ + desc = ctx->sh_desc_dec; + + init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); + /* Skip if already shared */ + key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_SHRD); + + /* Load class1 key only */ + append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, + ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); + + /* Load sector size with index 40 bytes (0x28) */ + append_cmd(desc, CMD_LOAD | IMMEDIATE | LDST_SRCDST_BYTE_CONTEXT | + LDST_CLASS_1_CCB | (0x28 << LDST_OFFSET_SHIFT) | 8); + append_data(desc, (void *)§or_size, 8); + + set_jump_tgt_here(desc, key_jump_cmd); + + /* + * create sequence for loading the sector index + * Upper 8B of IV - will be used as sector index + * Lower 8B of IV - will be discarded + */ + append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT | + LDST_CLASS_1_CCB | (0x20 << LDST_OFFSET_SHIFT) | 8); + append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); + + /* Load operation */ + append_dec_op1(desc, ctx->class1_alg_type); + + /* Perform operation */ + ablkcipher_append_src_dst(desc); + + ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, desc_bytes(desc), + DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { + dma_unmap_single(jrdev, ctx->sh_desc_enc_dma, + desc_bytes(ctx->sh_desc_enc), DMA_TO_DEVICE); + dev_err(jrdev, "unable to map shared descriptor\n"); + return -ENOMEM; + } +#ifdef DEBUG + print_hex_dump(KERN_ERR, + "xts ablkcipher dec shdesc@" __stringify(__LINE__) ": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); +#endif + + return 0; +} + /* * aead_edesc - s/w-extended aead descriptor * @assoc_nents: number of segments in associated data (SPI+Seq) scatterlist - * @assoc_chained: if source is chained * @src_nents: number of segments in input scatterlist - * @src_chained: if source is chained * @dst_nents: number of segments in output scatterlist - * @dst_chained: if destination is chained * @iv_dma: dma address of iv for checking continuity and link table * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE) * @sec4_sg_bytes: length of dma mapped sec4_sg space @@ -2091,24 +1838,19 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, */ struct aead_edesc { int assoc_nents; - bool assoc_chained; int src_nents; - bool src_chained; int dst_nents; - bool dst_chained; dma_addr_t iv_dma; int sec4_sg_bytes; dma_addr_t sec4_sg_dma; struct sec4_sg_entry *sec4_sg; - u32 hw_desc[0]; + u32 hw_desc[]; }; /* * ablkcipher_edesc - s/w-extended ablkcipher descriptor * @src_nents: number of segments in input scatterlist - * @src_chained: if source is chained * @dst_nents: number of segments in output scatterlist - * @dst_chained: if destination is chained * @iv_dma: dma address of iv for checking continuity and link table * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE) * @sec4_sg_bytes: length of dma mapped sec4_sg space @@ -2117,9 +1859,7 @@ struct aead_edesc { */ struct ablkcipher_edesc { int src_nents; - bool src_chained; int dst_nents; - bool dst_chained; dma_addr_t iv_dma; int sec4_sg_bytes; dma_addr_t sec4_sg_dma; @@ -2129,18 +1869,15 @@ struct ablkcipher_edesc { static void caam_unmap(struct device *dev, struct scatterlist *src, struct scatterlist *dst, int src_nents, - bool src_chained, int dst_nents, bool dst_chained, + int dst_nents, dma_addr_t iv_dma, int ivsize, dma_addr_t sec4_sg_dma, int sec4_sg_bytes) { if (dst != src) { - dma_unmap_sg_chained(dev, src, src_nents ? : 1, DMA_TO_DEVICE, - src_chained); - dma_unmap_sg_chained(dev, dst, dst_nents ? : 1, DMA_FROM_DEVICE, - dst_chained); + dma_unmap_sg(dev, src, src_nents ? : 1, DMA_TO_DEVICE); + dma_unmap_sg(dev, dst, dst_nents ? : 1, DMA_FROM_DEVICE); } else { - dma_unmap_sg_chained(dev, src, src_nents ? : 1, - DMA_BIDIRECTIONAL, src_chained); + dma_unmap_sg(dev, src, src_nents ? : 1, DMA_BIDIRECTIONAL); } if (iv_dma) @@ -2154,15 +1891,8 @@ static void aead_unmap(struct device *dev, struct aead_edesc *edesc, struct aead_request *req) { - struct crypto_aead *aead = crypto_aead_reqtfm(req); - int ivsize = crypto_aead_ivsize(aead); - - dma_unmap_sg_chained(dev, req->assoc, edesc->assoc_nents, - DMA_TO_DEVICE, edesc->assoc_chained); - caam_unmap(dev, req->src, req->dst, - edesc->src_nents, edesc->src_chained, edesc->dst_nents, - edesc->dst_chained, edesc->iv_dma, ivsize, + edesc->src_nents, edesc->dst_nents, 0, 0, edesc->sec4_sg_dma, edesc->sec4_sg_bytes); } @@ -2174,8 +1904,8 @@ static void ablkcipher_unmap(struct device *dev, int ivsize = crypto_ablkcipher_ivsize(ablkcipher); caam_unmap(dev, req->src, req->dst, - edesc->src_nents, edesc->src_chained, edesc->dst_nents, - edesc->dst_chained, edesc->iv_dma, ivsize, + edesc->src_nents, edesc->dst_nents, + edesc->iv_dma, ivsize, edesc->sec4_sg_dma, edesc->sec4_sg_bytes); } @@ -2184,35 +1914,18 @@ static void aead_encrypt_done(struct device *jrdev, u32 *desc, u32 err, { struct aead_request *req = context; struct aead_edesc *edesc; -#ifdef DEBUG - struct crypto_aead *aead = crypto_aead_reqtfm(req); - struct caam_ctx *ctx = crypto_aead_ctx(aead); - int ivsize = crypto_aead_ivsize(aead); +#ifdef DEBUG dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); #endif - edesc = (struct aead_edesc *)((char *)desc - - offsetof(struct aead_edesc, hw_desc)); + edesc = container_of(desc, struct aead_edesc, hw_desc[0]); if (err) caam_jr_strstatus(jrdev, err); aead_unmap(jrdev, edesc, req); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "assoc @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc), - req->assoclen , 1); - print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src) - ivsize, - edesc->src_nents ? 100 : ivsize, 1); - print_hex_dump(KERN_ERR, "dst @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), - edesc->src_nents ? 100 : req->cryptlen + - ctx->authsize + 4, 1); -#endif - kfree(edesc); aead_request_complete(req, err); @@ -2223,25 +1936,12 @@ static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err, { struct aead_request *req = context; struct aead_edesc *edesc; -#ifdef DEBUG - struct crypto_aead *aead = crypto_aead_reqtfm(req); - struct caam_ctx *ctx = crypto_aead_ctx(aead); - int ivsize = crypto_aead_ivsize(aead); +#ifdef DEBUG dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); #endif - edesc = (struct aead_edesc *)((char *)desc - - offsetof(struct aead_edesc, hw_desc)); - -#ifdef DEBUG - print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->iv, - ivsize, 1); - print_hex_dump(KERN_ERR, "dst @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->dst), - req->cryptlen - ctx->authsize, 1); -#endif + edesc = container_of(desc, struct aead_edesc, hw_desc[0]); if (err) caam_jr_strstatus(jrdev, err); @@ -2254,21 +1954,6 @@ static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err, if ((err & JRSTA_CCBERR_ERRID_MASK) == JRSTA_CCBERR_ERRID_ICVCHK) err = -EBADMSG; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "iphdrout@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, - ((char *)sg_virt(req->assoc) - sizeof(struct iphdr)), - sizeof(struct iphdr) + req->assoclen + - ((req->cryptlen > 1500) ? 1500 : req->cryptlen) + - ctx->authsize + 36, 1); - if (!err && edesc->sec4_sg_bytes) { - struct scatterlist *sg = sg_last(req->src, edesc->src_nents); - print_hex_dump(KERN_ERR, "sglastout@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(sg), - sg->length + ctx->authsize + 16, 1); - } -#endif - kfree(edesc); aead_request_complete(req, err); @@ -2342,71 +2027,42 @@ static void ablkcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err, /* * Fill in aead job descriptor */ -static void init_aead_job(u32 *sh_desc, dma_addr_t ptr, +static void init_aead_job(struct aead_request *req, struct aead_edesc *edesc, - struct aead_request *req, bool all_contig, bool encrypt) { struct crypto_aead *aead = crypto_aead_reqtfm(req); struct caam_ctx *ctx = crypto_aead_ctx(aead); - int ivsize = crypto_aead_ivsize(aead); int authsize = ctx->authsize; u32 *desc = edesc->hw_desc; - u32 out_options = 0, in_options; + u32 out_options, in_options; dma_addr_t dst_dma, src_dma; int len, sec4_sg_index = 0; - bool is_gcm = false; - -#ifdef DEBUG - debug("assoclen %d cryptlen %d authsize %d\n", - req->assoclen, req->cryptlen, authsize); - print_hex_dump(KERN_ERR, "assoc @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc), - req->assoclen , 1); - print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->iv, - edesc->src_nents ? 100 : ivsize, 1); - print_hex_dump(KERN_ERR, "src @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), - edesc->src_nents ? 100 : req->cryptlen, 1); - print_hex_dump(KERN_ERR, "shrdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, sh_desc, - desc_bytes(sh_desc), 1); -#endif + dma_addr_t ptr; + u32 *sh_desc; - if (((ctx->class1_alg_type & OP_ALG_ALGSEL_MASK) == - OP_ALG_ALGSEL_AES) && - ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == OP_ALG_AAI_GCM)) - is_gcm = true; + sh_desc = encrypt ? ctx->sh_desc_enc : ctx->sh_desc_dec; + ptr = encrypt ? ctx->sh_desc_enc_dma : ctx->sh_desc_dec_dma; len = desc_len(sh_desc); init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); if (all_contig) { - if (is_gcm) - src_dma = edesc->iv_dma; - else - src_dma = sg_dma_address(req->assoc); + src_dma = sg_dma_address(req->src); in_options = 0; } else { src_dma = edesc->sec4_sg_dma; - sec4_sg_index += (edesc->assoc_nents ? : 1) + 1 + - (edesc->src_nents ? : 1); + sec4_sg_index += edesc->src_nents; in_options = LDST_SGF; } - append_seq_in_ptr(desc, src_dma, req->assoclen + ivsize + req->cryptlen, + append_seq_in_ptr(desc, src_dma, req->assoclen + req->cryptlen, in_options); - if (likely(req->src == req->dst)) { - if (all_contig) { - dst_dma = sg_dma_address(req->src); - } else { - dst_dma = src_dma + sizeof(struct sec4_sg_entry) * - ((edesc->assoc_nents ? : 1) + 1); - out_options = LDST_SGF; - } - } else { + dst_dma = src_dma; + out_options = in_options; + + if (unlikely(req->src != req->dst)) { if (!edesc->dst_nents) { dst_dma = sg_dma_address(req->dst); } else { @@ -2416,96 +2072,94 @@ static void init_aead_job(u32 *sh_desc, dma_addr_t ptr, out_options = LDST_SGF; } } + if (encrypt) - append_seq_out_ptr(desc, dst_dma, req->cryptlen + authsize, + append_seq_out_ptr(desc, dst_dma, + req->assoclen + req->cryptlen + authsize, out_options); else - append_seq_out_ptr(desc, dst_dma, req->cryptlen - authsize, + append_seq_out_ptr(desc, dst_dma, + req->assoclen + req->cryptlen - authsize, out_options); + + /* REG3 = assoclen */ + append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen); } -/* - * Fill in aead givencrypt job descriptor - */ -static void init_aead_giv_job(u32 *sh_desc, dma_addr_t ptr, - struct aead_edesc *edesc, - struct aead_request *req, - int contig) +static void init_gcm_job(struct aead_request *req, + struct aead_edesc *edesc, + bool all_contig, bool encrypt) { struct crypto_aead *aead = crypto_aead_reqtfm(req); struct caam_ctx *ctx = crypto_aead_ctx(aead); - int ivsize = crypto_aead_ivsize(aead); - int authsize = ctx->authsize; + unsigned int ivsize = crypto_aead_ivsize(aead); u32 *desc = edesc->hw_desc; - u32 out_options = 0, in_options; - dma_addr_t dst_dma, src_dma; - int len, sec4_sg_index = 0; - bool is_gcm = false; - -#ifdef DEBUG - debug("assoclen %d cryptlen %d authsize %d\n", - req->assoclen, req->cryptlen, authsize); - print_hex_dump(KERN_ERR, "assoc @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc), - req->assoclen , 1); - print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1); - print_hex_dump(KERN_ERR, "src @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), - edesc->src_nents > 1 ? 100 : req->cryptlen, 1); - print_hex_dump(KERN_ERR, "shrdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, sh_desc, - desc_bytes(sh_desc), 1); -#endif - - if (((ctx->class1_alg_type & OP_ALG_ALGSEL_MASK) == - OP_ALG_ALGSEL_AES) && - ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == OP_ALG_AAI_GCM)) - is_gcm = true; + bool generic_gcm = (ivsize == 12); + unsigned int last; - len = desc_len(sh_desc); - init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); - - if (contig & GIV_SRC_CONTIG) { - if (is_gcm) - src_dma = edesc->iv_dma; - else - src_dma = sg_dma_address(req->assoc); - in_options = 0; - } else { - src_dma = edesc->sec4_sg_dma; - sec4_sg_index += edesc->assoc_nents + 1 + edesc->src_nents; - in_options = LDST_SGF; - } - append_seq_in_ptr(desc, src_dma, req->assoclen + ivsize + req->cryptlen, - in_options); + init_aead_job(req, edesc, all_contig, encrypt); - if (contig & GIV_DST_CONTIG) { - dst_dma = edesc->iv_dma; - } else { - if (likely(req->src == req->dst)) { - dst_dma = src_dma + sizeof(struct sec4_sg_entry) * - (edesc->assoc_nents + - (is_gcm ? 1 + edesc->src_nents : 0)); - out_options = LDST_SGF; - } else { - dst_dma = edesc->sec4_sg_dma + - sec4_sg_index * - sizeof(struct sec4_sg_entry); - out_options = LDST_SGF; - } - } + /* BUG This should not be specific to generic GCM. */ + last = 0; + if (encrypt && generic_gcm && !(req->assoclen + req->cryptlen)) + last = FIFOLD_TYPE_LAST1; - append_seq_out_ptr(desc, dst_dma, ivsize + req->cryptlen + authsize, - out_options); + /* Read GCM IV */ + append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE | + FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | 12 | last); + /* Append Salt */ + if (!generic_gcm) + append_data(desc, ctx->key + ctx->enckeylen, 4); + /* Append IV */ + append_data(desc, req->iv, ivsize); + /* End of blank commands */ } -/* - * Fill in ablkcipher job descriptor - */ -static void init_ablkcipher_job(u32 *sh_desc, dma_addr_t ptr, - struct ablkcipher_edesc *edesc, - struct ablkcipher_request *req, +static void init_authenc_job(struct aead_request *req, + struct aead_edesc *edesc, + bool all_contig, bool encrypt) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), + struct caam_aead_alg, aead); + unsigned int ivsize = crypto_aead_ivsize(aead); + struct caam_ctx *ctx = crypto_aead_ctx(aead); + const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == + OP_ALG_AAI_CTR_MOD128); + const bool is_rfc3686 = alg->caam.rfc3686; + u32 *desc = edesc->hw_desc; + u32 ivoffset = 0; + + /* + * AES-CTR needs to load IV in CONTEXT1 reg + * at an offset of 128bits (16bytes) + * CONTEXT1[255:128] = IV + */ + if (ctr_mode) + ivoffset = 16; + + /* + * RFC3686 specific: + * CONTEXT1[255:128] = {NONCE, IV, COUNTER} + */ + if (is_rfc3686) + ivoffset = 16 + CTR_RFC3686_NONCE_SIZE; + + init_aead_job(req, edesc, all_contig, encrypt); + + if (ivsize && (is_rfc3686 || !(alg->caam.geniv && encrypt))) + append_load_as_imm(desc, req->iv, ivsize, + LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT | + (ivoffset << LDST_OFFSET_SHIFT)); +} + +/* + * Fill in ablkcipher job descriptor + */ +static void init_ablkcipher_job(u32 *sh_desc, dma_addr_t ptr, + struct ablkcipher_edesc *edesc, + struct ablkcipher_request *req, bool iv_contig) { struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); @@ -2617,73 +2271,29 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, struct device *jrdev = ctx->jrdev; gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; - int assoc_nents, src_nents, dst_nents = 0; + int src_nents, dst_nents = 0; struct aead_edesc *edesc; - dma_addr_t iv_dma = 0; int sgc; bool all_contig = true; - bool assoc_chained = false, src_chained = false, dst_chained = false; - int ivsize = crypto_aead_ivsize(aead); int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes; unsigned int authsize = ctx->authsize; - bool is_gcm = false; - - assoc_nents = sg_count(req->assoc, req->assoclen, &assoc_chained); if (unlikely(req->dst != req->src)) { - src_nents = sg_count(req->src, req->cryptlen, &src_chained); + src_nents = sg_count(req->src, req->assoclen + req->cryptlen); dst_nents = sg_count(req->dst, - req->cryptlen + - (encrypt ? authsize : (-authsize)), - &dst_chained); + req->assoclen + req->cryptlen + + (encrypt ? authsize : (-authsize))); } else { src_nents = sg_count(req->src, - req->cryptlen + - (encrypt ? authsize : 0), - &src_chained); - } - - sgc = dma_map_sg_chained(jrdev, req->assoc, assoc_nents ? : 1, - DMA_TO_DEVICE, assoc_chained); - if (likely(req->src == req->dst)) { - sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, - DMA_BIDIRECTIONAL, src_chained); - } else { - sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, - DMA_TO_DEVICE, src_chained); - sgc = dma_map_sg_chained(jrdev, req->dst, dst_nents ? : 1, - DMA_FROM_DEVICE, dst_chained); - } - - iv_dma = dma_map_single(jrdev, req->iv, ivsize, DMA_TO_DEVICE); - if (dma_mapping_error(jrdev, iv_dma)) { - dev_err(jrdev, "unable to map IV\n"); - return ERR_PTR(-ENOMEM); + req->assoclen + req->cryptlen + + (encrypt ? authsize : 0)); } - if (((ctx->class1_alg_type & OP_ALG_ALGSEL_MASK) == - OP_ALG_ALGSEL_AES) && - ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == OP_ALG_AAI_GCM)) - is_gcm = true; - - /* - * Check if data are contiguous. - * GCM expected input sequence: IV, AAD, text - * All other - expected input sequence: AAD, IV, text - */ - if (is_gcm) - all_contig = (!assoc_nents && - iv_dma + ivsize == sg_dma_address(req->assoc) && - !src_nents && sg_dma_address(req->assoc) + - req->assoclen == sg_dma_address(req->src)); - else - all_contig = (!assoc_nents && sg_dma_address(req->assoc) + - req->assoclen == iv_dma && !src_nents && - iv_dma + ivsize == sg_dma_address(req->src)); + /* Check if data are contiguous. */ + all_contig = !src_nents; if (!all_contig) { - assoc_nents = assoc_nents ? : 1; src_nents = src_nents ? : 1; - sec4_sg_len = assoc_nents + 1 + src_nents; + sec4_sg_len = src_nents; } sec4_sg_len += dst_nents; @@ -2691,68 +2301,76 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry); /* allocate space for base edesc and hw desc commands, link tables */ - edesc = kmalloc(sizeof(struct aead_edesc) + desc_bytes + - sec4_sg_bytes, GFP_DMA | flags); + edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes, + GFP_DMA | flags); if (!edesc) { dev_err(jrdev, "could not allocate extended descriptor\n"); return ERR_PTR(-ENOMEM); } - edesc->assoc_nents = assoc_nents; - edesc->assoc_chained = assoc_chained; + if (likely(req->src == req->dst)) { + sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, + DMA_BIDIRECTIONAL); + if (unlikely(!sgc)) { + dev_err(jrdev, "unable to map source\n"); + kfree(edesc); + return ERR_PTR(-ENOMEM); + } + } else { + sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, + DMA_TO_DEVICE); + if (unlikely(!sgc)) { + dev_err(jrdev, "unable to map source\n"); + kfree(edesc); + return ERR_PTR(-ENOMEM); + } + + sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1, + DMA_FROM_DEVICE); + if (unlikely(!sgc)) { + dev_err(jrdev, "unable to map destination\n"); + dma_unmap_sg(jrdev, req->src, src_nents ? : 1, + DMA_TO_DEVICE); + kfree(edesc); + return ERR_PTR(-ENOMEM); + } + } + edesc->src_nents = src_nents; - edesc->src_chained = src_chained; edesc->dst_nents = dst_nents; - edesc->dst_chained = dst_chained; - edesc->iv_dma = iv_dma; - edesc->sec4_sg_bytes = sec4_sg_bytes; edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) + desc_bytes; *all_contig_ptr = all_contig; sec4_sg_index = 0; if (!all_contig) { - if (!is_gcm) { - sg_to_sec4_sg(req->assoc, - assoc_nents, - edesc->sec4_sg + - sec4_sg_index, 0); - sec4_sg_index += assoc_nents; - } - - dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index, - iv_dma, ivsize, 0); - sec4_sg_index += 1; - - if (is_gcm) { - sg_to_sec4_sg(req->assoc, - assoc_nents, - edesc->sec4_sg + - sec4_sg_index, 0); - sec4_sg_index += assoc_nents; - } - - sg_to_sec4_sg_last(req->src, - src_nents, - edesc->sec4_sg + - sec4_sg_index, 0); + sg_to_sec4_sg_last(req->src, src_nents, + edesc->sec4_sg + sec4_sg_index, 0); sec4_sg_index += src_nents; } if (dst_nents) { sg_to_sec4_sg_last(req->dst, dst_nents, edesc->sec4_sg + sec4_sg_index, 0); } + + if (!sec4_sg_bytes) + return edesc; + edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, sec4_sg_bytes, DMA_TO_DEVICE); if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { dev_err(jrdev, "unable to map S/G table\n"); + aead_unmap(jrdev, edesc, req); + kfree(edesc); return ERR_PTR(-ENOMEM); } + edesc->sec4_sg_bytes = sec4_sg_bytes; + return edesc; } -static int aead_encrypt(struct aead_request *req) +static int gcm_encrypt(struct aead_request *req) { struct aead_edesc *edesc; struct crypto_aead *aead = crypto_aead_reqtfm(req); @@ -2763,14 +2381,12 @@ static int aead_encrypt(struct aead_request *req) int ret = 0; /* allocate extended descriptor */ - edesc = aead_edesc_alloc(req, DESC_JOB_IO_LEN * - CAAM_CMD_SZ, &all_contig, true); + edesc = aead_edesc_alloc(req, GCM_DESC_JOB_IO_LEN, &all_contig, true); if (IS_ERR(edesc)) return PTR_ERR(edesc); /* Create and submit job descriptor */ - init_aead_job(ctx->sh_desc_enc, ctx->sh_desc_enc_dma, edesc, req, - all_contig, true); + init_gcm_job(req, edesc, all_contig, true); #ifdef DEBUG print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, @@ -2789,7 +2405,15 @@ static int aead_encrypt(struct aead_request *req) return ret; } -static int aead_decrypt(struct aead_request *req) +static int ipsec_gcm_encrypt(struct aead_request *req) +{ + if (req->assoclen < 8) + return -EINVAL; + + return gcm_encrypt(req); +} + +static int aead_encrypt(struct aead_request *req) { struct aead_edesc *edesc; struct crypto_aead *aead = crypto_aead_reqtfm(req); @@ -2800,20 +2424,13 @@ static int aead_decrypt(struct aead_request *req) int ret = 0; /* allocate extended descriptor */ - edesc = aead_edesc_alloc(req, DESC_JOB_IO_LEN * - CAAM_CMD_SZ, &all_contig, false); + edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN, + &all_contig, true); if (IS_ERR(edesc)) return PTR_ERR(edesc); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "dec src@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), - req->cryptlen, 1); -#endif - - /* Create and submit job descriptor*/ - init_aead_job(ctx->sh_desc_dec, - ctx->sh_desc_dec_dma, edesc, req, all_contig, false); + /* Create and submit job descriptor */ + init_authenc_job(req, edesc, all_contig, true); #ifdef DEBUG print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, @@ -2821,7 +2438,7 @@ static int aead_decrypt(struct aead_request *req) #endif desc = edesc->hw_desc; - ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req); + ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req); if (!ret) { ret = -EINPROGRESS; } else { @@ -2832,200 +2449,73 @@ static int aead_decrypt(struct aead_request *req) return ret; } -/* - * allocate and map the aead extended descriptor for aead givencrypt - */ -static struct aead_edesc *aead_giv_edesc_alloc(struct aead_givcrypt_request - *greq, int desc_bytes, - u32 *contig_ptr) +static int gcm_decrypt(struct aead_request *req) { - struct aead_request *req = &greq->areq; + struct aead_edesc *edesc; struct crypto_aead *aead = crypto_aead_reqtfm(req); struct caam_ctx *ctx = crypto_aead_ctx(aead); struct device *jrdev = ctx->jrdev; - gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | - CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; - int assoc_nents, src_nents, dst_nents = 0; - struct aead_edesc *edesc; - dma_addr_t iv_dma = 0; - int sgc; - u32 contig = GIV_SRC_CONTIG | GIV_DST_CONTIG; - int ivsize = crypto_aead_ivsize(aead); - bool assoc_chained = false, src_chained = false, dst_chained = false; - int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes; - bool is_gcm = false; - - assoc_nents = sg_count(req->assoc, req->assoclen, &assoc_chained); - src_nents = sg_count(req->src, req->cryptlen, &src_chained); - - if (unlikely(req->dst != req->src)) - dst_nents = sg_count(req->dst, req->cryptlen + ctx->authsize, - &dst_chained); - - sgc = dma_map_sg_chained(jrdev, req->assoc, assoc_nents ? : 1, - DMA_TO_DEVICE, assoc_chained); - if (likely(req->src == req->dst)) { - sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, - DMA_BIDIRECTIONAL, src_chained); - } else { - sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, - DMA_TO_DEVICE, src_chained); - sgc = dma_map_sg_chained(jrdev, req->dst, dst_nents ? : 1, - DMA_FROM_DEVICE, dst_chained); - } - - iv_dma = dma_map_single(jrdev, greq->giv, ivsize, DMA_TO_DEVICE); - if (dma_mapping_error(jrdev, iv_dma)) { - dev_err(jrdev, "unable to map IV\n"); - return ERR_PTR(-ENOMEM); - } + bool all_contig; + u32 *desc; + int ret = 0; - if (((ctx->class1_alg_type & OP_ALG_ALGSEL_MASK) == - OP_ALG_ALGSEL_AES) && - ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == OP_ALG_AAI_GCM)) - is_gcm = true; + /* allocate extended descriptor */ + edesc = aead_edesc_alloc(req, GCM_DESC_JOB_IO_LEN, &all_contig, false); + if (IS_ERR(edesc)) + return PTR_ERR(edesc); - /* - * Check if data are contiguous. - * GCM expected input sequence: IV, AAD, text - * All other - expected input sequence: AAD, IV, text - */ + /* Create and submit job descriptor*/ + init_gcm_job(req, edesc, all_contig, false); +#ifdef DEBUG + print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, + desc_bytes(edesc->hw_desc), 1); +#endif - if (is_gcm) { - if (assoc_nents || iv_dma + ivsize != - sg_dma_address(req->assoc) || src_nents || - sg_dma_address(req->assoc) + req->assoclen != - sg_dma_address(req->src)) - contig &= ~GIV_SRC_CONTIG; + desc = edesc->hw_desc; + ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req); + if (!ret) { + ret = -EINPROGRESS; } else { - if (assoc_nents || - sg_dma_address(req->assoc) + req->assoclen != iv_dma || - src_nents || iv_dma + ivsize != sg_dma_address(req->src)) - contig &= ~GIV_SRC_CONTIG; - } - - if (dst_nents || iv_dma + ivsize != sg_dma_address(req->dst)) - contig &= ~GIV_DST_CONTIG; - - if (!(contig & GIV_SRC_CONTIG)) { - assoc_nents = assoc_nents ? : 1; - src_nents = src_nents ? : 1; - sec4_sg_len += assoc_nents + 1 + src_nents; - if (req->src == req->dst && - (src_nents || iv_dma + ivsize != sg_dma_address(req->src))) - contig &= ~GIV_DST_CONTIG; - } - - /* - * Add new sg entries for GCM output sequence. - * Expected output sequence: IV, encrypted text. - */ - if (is_gcm && req->src == req->dst && !(contig & GIV_DST_CONTIG)) - sec4_sg_len += 1 + src_nents; - - if (unlikely(req->src != req->dst)) { - dst_nents = dst_nents ? : 1; - sec4_sg_len += 1 + dst_nents; - } - - sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry); - - /* allocate space for base edesc and hw desc commands, link tables */ - edesc = kmalloc(sizeof(struct aead_edesc) + desc_bytes + - sec4_sg_bytes, GFP_DMA | flags); - if (!edesc) { - dev_err(jrdev, "could not allocate extended descriptor\n"); - return ERR_PTR(-ENOMEM); - } - - edesc->assoc_nents = assoc_nents; - edesc->assoc_chained = assoc_chained; - edesc->src_nents = src_nents; - edesc->src_chained = src_chained; - edesc->dst_nents = dst_nents; - edesc->dst_chained = dst_chained; - edesc->iv_dma = iv_dma; - edesc->sec4_sg_bytes = sec4_sg_bytes; - edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) + - desc_bytes; - *contig_ptr = contig; - - sec4_sg_index = 0; - if (!(contig & GIV_SRC_CONTIG)) { - if (!is_gcm) { - sg_to_sec4_sg(req->assoc, assoc_nents, - edesc->sec4_sg + sec4_sg_index, 0); - sec4_sg_index += assoc_nents; - } - - dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index, - iv_dma, ivsize, 0); - sec4_sg_index += 1; - - if (is_gcm) { - sg_to_sec4_sg(req->assoc, assoc_nents, - edesc->sec4_sg + sec4_sg_index, 0); - sec4_sg_index += assoc_nents; - } - - sg_to_sec4_sg_last(req->src, src_nents, - edesc->sec4_sg + - sec4_sg_index, 0); - sec4_sg_index += src_nents; + aead_unmap(jrdev, edesc, req); + kfree(edesc); } - if (is_gcm && req->src == req->dst && !(contig & GIV_DST_CONTIG)) { - dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index, - iv_dma, ivsize, 0); - sec4_sg_index += 1; - sg_to_sec4_sg_last(req->src, src_nents, - edesc->sec4_sg + sec4_sg_index, 0); - } + return ret; +} - if (unlikely(req->src != req->dst && !(contig & GIV_DST_CONTIG))) { - dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index, - iv_dma, ivsize, 0); - sec4_sg_index += 1; - sg_to_sec4_sg_last(req->dst, dst_nents, - edesc->sec4_sg + sec4_sg_index, 0); - } - edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, - sec4_sg_bytes, DMA_TO_DEVICE); - if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { - dev_err(jrdev, "unable to map S/G table\n"); - return ERR_PTR(-ENOMEM); - } +static int ipsec_gcm_decrypt(struct aead_request *req) +{ + if (req->assoclen < 8) + return -EINVAL; - return edesc; + return gcm_decrypt(req); } -static int aead_givencrypt(struct aead_givcrypt_request *areq) +static int aead_decrypt(struct aead_request *req) { - struct aead_request *req = &areq->areq; struct aead_edesc *edesc; struct crypto_aead *aead = crypto_aead_reqtfm(req); struct caam_ctx *ctx = crypto_aead_ctx(aead); struct device *jrdev = ctx->jrdev; - u32 contig; + bool all_contig; u32 *desc; int ret = 0; /* allocate extended descriptor */ - edesc = aead_giv_edesc_alloc(areq, DESC_JOB_IO_LEN * - CAAM_CMD_SZ, &contig); - + edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN, + &all_contig, false); if (IS_ERR(edesc)) return PTR_ERR(edesc); #ifdef DEBUG - print_hex_dump(KERN_ERR, "giv src@"__stringify(__LINE__)": ", + print_hex_dump(KERN_ERR, "dec src@"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src), - req->cryptlen, 1); + req->assoclen + req->cryptlen, 1); #endif /* Create and submit job descriptor*/ - init_aead_giv_job(ctx->sh_desc_givenc, - ctx->sh_desc_givenc_dma, edesc, req, contig); + init_authenc_job(req, edesc, all_contig, false); #ifdef DEBUG print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, @@ -3033,7 +2523,7 @@ static int aead_givencrypt(struct aead_givcrypt_request *areq) #endif desc = edesc->hw_desc; - ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req); + ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req); if (!ret) { ret = -EINPROGRESS; } else { @@ -3044,9 +2534,18 @@ static int aead_givencrypt(struct aead_givcrypt_request *areq) return ret; } -static int aead_null_givencrypt(struct aead_givcrypt_request *areq) +static int aead_givdecrypt(struct aead_request *req) { - return aead_encrypt(&areq->areq); + struct crypto_aead *aead = crypto_aead_reqtfm(req); + unsigned int ivsize = crypto_aead_ivsize(aead); + + if (req->cryptlen < ivsize) + return -EINVAL; + + req->cryptlen -= ivsize; + req->assoclen += ivsize; + + return aead_decrypt(req); } /* @@ -3068,22 +2567,21 @@ static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request bool iv_contig = false; int sgc; int ivsize = crypto_ablkcipher_ivsize(ablkcipher); - bool src_chained = false, dst_chained = false; int sec4_sg_index; - src_nents = sg_count(req->src, req->nbytes, &src_chained); + src_nents = sg_count(req->src, req->nbytes); if (req->dst != req->src) - dst_nents = sg_count(req->dst, req->nbytes, &dst_chained); + dst_nents = sg_count(req->dst, req->nbytes); if (likely(req->src == req->dst)) { - sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, - DMA_BIDIRECTIONAL, src_chained); + sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, + DMA_BIDIRECTIONAL); } else { - sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, - DMA_TO_DEVICE, src_chained); - sgc = dma_map_sg_chained(jrdev, req->dst, dst_nents ? : 1, - DMA_FROM_DEVICE, dst_chained); + sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, + DMA_TO_DEVICE); + sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1, + DMA_FROM_DEVICE); } iv_dma = dma_map_single(jrdev, req->info, ivsize, DMA_TO_DEVICE); @@ -3104,17 +2602,15 @@ static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request sizeof(struct sec4_sg_entry); /* allocate space for base edesc and hw desc commands, link tables */ - edesc = kmalloc(sizeof(struct ablkcipher_edesc) + desc_bytes + - sec4_sg_bytes, GFP_DMA | flags); + edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes, + GFP_DMA | flags); if (!edesc) { dev_err(jrdev, "could not allocate extended descriptor\n"); return ERR_PTR(-ENOMEM); } edesc->src_nents = src_nents; - edesc->src_chained = src_chained; edesc->dst_nents = dst_nents; - edesc->dst_chained = dst_chained; edesc->sec4_sg_bytes = sec4_sg_bytes; edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) + desc_bytes; @@ -3247,22 +2743,21 @@ static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc( bool iv_contig = false; int sgc; int ivsize = crypto_ablkcipher_ivsize(ablkcipher); - bool src_chained = false, dst_chained = false; int sec4_sg_index; - src_nents = sg_count(req->src, req->nbytes, &src_chained); + src_nents = sg_count(req->src, req->nbytes); if (unlikely(req->dst != req->src)) - dst_nents = sg_count(req->dst, req->nbytes, &dst_chained); + dst_nents = sg_count(req->dst, req->nbytes); if (likely(req->src == req->dst)) { - sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, - DMA_BIDIRECTIONAL, src_chained); + sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, + DMA_BIDIRECTIONAL); } else { - sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, - DMA_TO_DEVICE, src_chained); - sgc = dma_map_sg_chained(jrdev, req->dst, dst_nents ? : 1, - DMA_FROM_DEVICE, dst_chained); + sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, + DMA_TO_DEVICE); + sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1, + DMA_FROM_DEVICE); } /* @@ -3283,17 +2778,15 @@ static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc( sizeof(struct sec4_sg_entry); /* allocate space for base edesc and hw desc commands, link tables */ - edesc = kmalloc(sizeof(*edesc) + desc_bytes + - sec4_sg_bytes, GFP_DMA | flags); + edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes, + GFP_DMA | flags); if (!edesc) { dev_err(jrdev, "could not allocate extended descriptor\n"); return ERR_PTR(-ENOMEM); } edesc->src_nents = src_nents; - edesc->src_chained = src_chained; edesc->dst_nents = dst_nents; - edesc->dst_chained = dst_chained; edesc->sec4_sg_bytes = sec4_sg_bytes; edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) + desc_bytes; @@ -3379,11 +2872,6 @@ struct caam_alg_template { u32 type; union { struct ablkcipher_alg ablkcipher; - struct aead_alg aead; - struct blkcipher_alg blkcipher; - struct cipher_alg cipher; - struct compress_alg compress; - struct rng_alg rng; } template_u; u32 class1_alg_type; u32 class2_alg_type; @@ -3391,754 +2879,1454 @@ struct caam_alg_template { }; static struct caam_alg_template driver_algs[] = { - /* single-pass ipsec_esp descriptor */ - { - .name = "authenc(hmac(md5),ecb(cipher_null))", - .driver_name = "authenc-hmac-md5-ecb-cipher_null-caam", - .blocksize = NULL_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { - .setkey = aead_setkey, - .setauthsize = aead_setauthsize, - .encrypt = aead_encrypt, - .decrypt = aead_decrypt, - .givencrypt = aead_null_givencrypt, - .geniv = "", - .ivsize = NULL_IV_SIZE, - .maxauthsize = MD5_DIGEST_SIZE, - }, - .class1_alg_type = 0, - .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, - }, + /* ablkcipher descriptor */ { - .name = "authenc(hmac(sha1),ecb(cipher_null))", - .driver_name = "authenc-hmac-sha1-ecb-cipher_null-caam", - .blocksize = NULL_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { - .setkey = aead_setkey, - .setauthsize = aead_setauthsize, - .encrypt = aead_encrypt, - .decrypt = aead_decrypt, - .givencrypt = aead_null_givencrypt, + .name = "cbc(aes)", + .driver_name = "cbc-aes-caam", + .blocksize = AES_BLOCK_SIZE, + .type = CRYPTO_ALG_TYPE_GIVCIPHER, + .template_ablkcipher = { + .setkey = ablkcipher_setkey, + .encrypt = ablkcipher_encrypt, + .decrypt = ablkcipher_decrypt, + .givencrypt = ablkcipher_givencrypt, .geniv = "", - .ivsize = NULL_IV_SIZE, - .maxauthsize = SHA1_DIGEST_SIZE, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, }, - .class1_alg_type = 0, - .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, }, { - .name = "authenc(hmac(sha224),ecb(cipher_null))", - .driver_name = "authenc-hmac-sha224-ecb-cipher_null-caam", - .blocksize = NULL_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { - .setkey = aead_setkey, - .setauthsize = aead_setauthsize, - .encrypt = aead_encrypt, - .decrypt = aead_decrypt, - .givencrypt = aead_null_givencrypt, + .name = "cbc(des3_ede)", + .driver_name = "cbc-3des-caam", + .blocksize = DES3_EDE_BLOCK_SIZE, + .type = CRYPTO_ALG_TYPE_GIVCIPHER, + .template_ablkcipher = { + .setkey = ablkcipher_setkey, + .encrypt = ablkcipher_encrypt, + .decrypt = ablkcipher_decrypt, + .givencrypt = ablkcipher_givencrypt, .geniv = "", - .ivsize = NULL_IV_SIZE, - .maxauthsize = SHA224_DIGEST_SIZE, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, }, - .class1_alg_type = 0, - .class2_alg_type = OP_ALG_ALGSEL_SHA224 | - OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, }, { - .name = "authenc(hmac(sha256),ecb(cipher_null))", - .driver_name = "authenc-hmac-sha256-ecb-cipher_null-caam", - .blocksize = NULL_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { - .setkey = aead_setkey, - .setauthsize = aead_setauthsize, + .name = "cbc(des)", + .driver_name = "cbc-des-caam", + .blocksize = DES_BLOCK_SIZE, + .type = CRYPTO_ALG_TYPE_GIVCIPHER, + .template_ablkcipher = { + .setkey = ablkcipher_setkey, + .encrypt = ablkcipher_encrypt, + .decrypt = ablkcipher_decrypt, + .givencrypt = ablkcipher_givencrypt, + .geniv = "", + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + }, + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + }, + { + .name = "ctr(aes)", + .driver_name = "ctr-aes-caam", + .blocksize = 1, + .type = CRYPTO_ALG_TYPE_ABLKCIPHER, + .template_ablkcipher = { + .setkey = ablkcipher_setkey, + .encrypt = ablkcipher_encrypt, + .decrypt = ablkcipher_decrypt, + .geniv = "chainiv", + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + }, + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, + }, + { + .name = "rfc3686(ctr(aes))", + .driver_name = "rfc3686-ctr-aes-caam", + .blocksize = 1, + .type = CRYPTO_ALG_TYPE_GIVCIPHER, + .template_ablkcipher = { + .setkey = ablkcipher_setkey, + .encrypt = ablkcipher_encrypt, + .decrypt = ablkcipher_decrypt, + .givencrypt = ablkcipher_givencrypt, + .geniv = "", + .min_keysize = AES_MIN_KEY_SIZE + + CTR_RFC3686_NONCE_SIZE, + .max_keysize = AES_MAX_KEY_SIZE + + CTR_RFC3686_NONCE_SIZE, + .ivsize = CTR_RFC3686_IV_SIZE, + }, + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, + }, + { + .name = "xts(aes)", + .driver_name = "xts-aes-caam", + .blocksize = AES_BLOCK_SIZE, + .type = CRYPTO_ALG_TYPE_ABLKCIPHER, + .template_ablkcipher = { + .setkey = xts_ablkcipher_setkey, + .encrypt = ablkcipher_encrypt, + .decrypt = ablkcipher_decrypt, + .geniv = "eseqiv", + .min_keysize = 2 * AES_MIN_KEY_SIZE, + .max_keysize = 2 * AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + }, + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS, + }, +}; + +static struct caam_aead_alg driver_aeads[] = { + { + .aead = { + .base = { + .cra_name = "rfc4106(gcm(aes))", + .cra_driver_name = "rfc4106-gcm-aes-caam", + .cra_blocksize = 1, + }, + .setkey = rfc4106_setkey, + .setauthsize = rfc4106_setauthsize, + .encrypt = ipsec_gcm_encrypt, + .decrypt = ipsec_gcm_decrypt, + .ivsize = 8, + .maxauthsize = AES_BLOCK_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, + }, + }, + { + .aead = { + .base = { + .cra_name = "rfc4543(gcm(aes))", + .cra_driver_name = "rfc4543-gcm-aes-caam", + .cra_blocksize = 1, + }, + .setkey = rfc4543_setkey, + .setauthsize = rfc4543_setauthsize, + .encrypt = ipsec_gcm_encrypt, + .decrypt = ipsec_gcm_decrypt, + .ivsize = 8, + .maxauthsize = AES_BLOCK_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, + }, + }, + /* Galois Counter Mode */ + { + .aead = { + .base = { + .cra_name = "gcm(aes)", + .cra_driver_name = "gcm-aes-caam", + .cra_blocksize = 1, + }, + .setkey = gcm_setkey, + .setauthsize = gcm_setauthsize, + .encrypt = gcm_encrypt, + .decrypt = gcm_decrypt, + .ivsize = 12, + .maxauthsize = AES_BLOCK_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, + }, + }, + /* single-pass ipsec_esp descriptor */ + { + .aead = { + .base = { + .cra_name = "authenc(hmac(md5)," + "ecb(cipher_null))", + .cra_driver_name = "authenc-hmac-md5-" + "ecb-cipher_null-caam", + .cra_blocksize = NULL_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, - .givencrypt = aead_null_givencrypt, - .geniv = "", .ivsize = NULL_IV_SIZE, - .maxauthsize = SHA256_DIGEST_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, + }, + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha1)," + "ecb(cipher_null))", + .cra_driver_name = "authenc-hmac-sha1-" + "ecb-cipher_null-caam", + .cra_blocksize = NULL_BLOCK_SIZE, }, - .class1_alg_type = 0, - .class2_alg_type = OP_ALG_ALGSEL_SHA256 | - OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = NULL_IV_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, + }, }, { - .name = "authenc(hmac(sha384),ecb(cipher_null))", - .driver_name = "authenc-hmac-sha384-ecb-cipher_null-caam", - .blocksize = NULL_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha224)," + "ecb(cipher_null))", + .cra_driver_name = "authenc-hmac-sha224-" + "ecb-cipher_null-caam", + .cra_blocksize = NULL_BLOCK_SIZE, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, - .givencrypt = aead_null_givencrypt, - .geniv = "", .ivsize = NULL_IV_SIZE, - .maxauthsize = SHA384_DIGEST_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, + }, + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha256)," + "ecb(cipher_null))", + .cra_driver_name = "authenc-hmac-sha256-" + "ecb-cipher_null-caam", + .cra_blocksize = NULL_BLOCK_SIZE, }, - .class1_alg_type = 0, - .class2_alg_type = OP_ALG_ALGSEL_SHA384 | - OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = NULL_IV_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, + }, }, { - .name = "authenc(hmac(sha512),ecb(cipher_null))", - .driver_name = "authenc-hmac-sha512-ecb-cipher_null-caam", - .blocksize = NULL_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha384)," + "ecb(cipher_null))", + .cra_driver_name = "authenc-hmac-sha384-" + "ecb-cipher_null-caam", + .cra_blocksize = NULL_BLOCK_SIZE, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, - .givencrypt = aead_null_givencrypt, - .geniv = "", .ivsize = NULL_IV_SIZE, - .maxauthsize = SHA512_DIGEST_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, + }, + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha512)," + "ecb(cipher_null))", + .cra_driver_name = "authenc-hmac-sha512-" + "ecb-cipher_null-caam", + .cra_blocksize = NULL_BLOCK_SIZE, }, - .class1_alg_type = 0, - .class2_alg_type = OP_ALG_ALGSEL_SHA512 | - OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = NULL_IV_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, + }, }, { - .name = "authenc(hmac(md5),cbc(aes))", - .driver_name = "authenc-hmac-md5-cbc-aes-caam", - .blocksize = AES_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(md5),cbc(aes))", + .cra_driver_name = "authenc-hmac-md5-" + "cbc-aes-caam", + .cra_blocksize = AES_BLOCK_SIZE, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, - .givencrypt = aead_givencrypt, - .geniv = "", .ivsize = AES_BLOCK_SIZE, .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(md5)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-hmac-md5-" + "cbc-aes-caam", + .cra_blocksize = AES_BLOCK_SIZE, }, - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_givdecrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, + .geniv = true, + }, }, { - .name = "authenc(hmac(sha1),cbc(aes))", - .driver_name = "authenc-hmac-sha1-cbc-aes-caam", - .blocksize = AES_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha1),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha1-" + "cbc-aes-caam", + .cra_blocksize = AES_BLOCK_SIZE, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, - .givencrypt = aead_givencrypt, - .geniv = "", .ivsize = AES_BLOCK_SIZE, .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha1)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha1-cbc-aes-caam", + .cra_blocksize = AES_BLOCK_SIZE, }, - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_givdecrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, + .geniv = true, + }, }, { - .name = "authenc(hmac(sha224),cbc(aes))", - .driver_name = "authenc-hmac-sha224-cbc-aes-caam", - .blocksize = AES_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha224),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha224-" + "cbc-aes-caam", + .cra_blocksize = AES_BLOCK_SIZE, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, - .givencrypt = aead_givencrypt, - .geniv = "", .ivsize = AES_BLOCK_SIZE, .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha224)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha224-cbc-aes-caam", + .cra_blocksize = AES_BLOCK_SIZE, }, - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA224 | - OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_givdecrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, + .geniv = true, + }, }, { - .name = "authenc(hmac(sha256),cbc(aes))", - .driver_name = "authenc-hmac-sha256-cbc-aes-caam", - .blocksize = AES_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha256),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha256-" + "cbc-aes-caam", + .cra_blocksize = AES_BLOCK_SIZE, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, - .givencrypt = aead_givencrypt, - .geniv = "", .ivsize = AES_BLOCK_SIZE, .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha256)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha256-cbc-aes-caam", + .cra_blocksize = AES_BLOCK_SIZE, }, - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA256 | - OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_givdecrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, + .geniv = true, + }, }, { - .name = "authenc(hmac(sha384),cbc(aes))", - .driver_name = "authenc-hmac-sha384-cbc-aes-caam", - .blocksize = AES_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha384),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha384-" + "cbc-aes-caam", + .cra_blocksize = AES_BLOCK_SIZE, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, - .givencrypt = aead_givencrypt, - .geniv = "", .ivsize = AES_BLOCK_SIZE, .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha384)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha384-cbc-aes-caam", + .cra_blocksize = AES_BLOCK_SIZE, }, - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA384 | - OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_givdecrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, + .geniv = true, + }, }, - { - .name = "authenc(hmac(sha512),cbc(aes))", - .driver_name = "authenc-hmac-sha512-cbc-aes-caam", - .blocksize = AES_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha512),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha512-" + "cbc-aes-caam", + .cra_blocksize = AES_BLOCK_SIZE, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, - .givencrypt = aead_givencrypt, - .geniv = "", .ivsize = AES_BLOCK_SIZE, .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha512)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha512-cbc-aes-caam", + .cra_blocksize = AES_BLOCK_SIZE, }, - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA512 | - OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_givdecrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, + .geniv = true, + }, }, { - .name = "authenc(hmac(md5),cbc(des3_ede))", - .driver_name = "authenc-hmac-md5-cbc-des3_ede-caam", - .blocksize = DES3_EDE_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(md5),cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-md5-" + "cbc-des3_ede-caam", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, - .givencrypt = aead_givencrypt, - .geniv = "", .ivsize = DES3_EDE_BLOCK_SIZE, .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, + } + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(md5)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-hmac-md5-" + "cbc-des3_ede-caam", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_givdecrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, + .geniv = true, + } }, { - .name = "authenc(hmac(sha1),cbc(des3_ede))", - .driver_name = "authenc-hmac-sha1-cbc-des3_ede-caam", - .blocksize = DES3_EDE_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha1)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha1-" + "cbc-des3_ede-caam", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, - .givencrypt = aead_givencrypt, - .geniv = "", .ivsize = DES3_EDE_BLOCK_SIZE, .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha1)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha1-" + "cbc-des3_ede-caam", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_givdecrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, + .geniv = true, + }, }, { - .name = "authenc(hmac(sha224),cbc(des3_ede))", - .driver_name = "authenc-hmac-sha224-cbc-des3_ede-caam", - .blocksize = DES3_EDE_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha224)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha224-" + "cbc-des3_ede-caam", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, - .givencrypt = aead_givencrypt, - .geniv = "", .ivsize = DES3_EDE_BLOCK_SIZE, .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha224)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha224-" + "cbc-des3_ede-caam", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA224 | - OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_givdecrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, + .geniv = true, + }, }, { - .name = "authenc(hmac(sha256),cbc(des3_ede))", - .driver_name = "authenc-hmac-sha256-cbc-des3_ede-caam", - .blocksize = DES3_EDE_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha256)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha256-" + "cbc-des3_ede-caam", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, - .givencrypt = aead_givencrypt, - .geniv = "", .ivsize = DES3_EDE_BLOCK_SIZE, .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha256)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha256-" + "cbc-des3_ede-caam", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA256 | - OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_givdecrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, + .geniv = true, + }, }, { - .name = "authenc(hmac(sha384),cbc(des3_ede))", - .driver_name = "authenc-hmac-sha384-cbc-des3_ede-caam", - .blocksize = DES3_EDE_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha384)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha384-" + "cbc-des3_ede-caam", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, - .givencrypt = aead_givencrypt, - .geniv = "", .ivsize = DES3_EDE_BLOCK_SIZE, .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha384)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha384-" + "cbc-des3_ede-caam", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA384 | - OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_givdecrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, + .geniv = true, + }, }, { - .name = "authenc(hmac(sha512),cbc(des3_ede))", - .driver_name = "authenc-hmac-sha512-cbc-des3_ede-caam", - .blocksize = DES3_EDE_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha512)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha512-" + "cbc-des3_ede-caam", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, - .givencrypt = aead_givencrypt, - .geniv = "", .ivsize = DES3_EDE_BLOCK_SIZE, .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha512)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha512-" + "cbc-des3_ede-caam", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, }, - .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA512 | - OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_givdecrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, + .geniv = true, + }, }, { - .name = "authenc(hmac(md5),cbc(des))", - .driver_name = "authenc-hmac-md5-cbc-des-caam", - .blocksize = DES_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(md5),cbc(des))", + .cra_driver_name = "authenc-hmac-md5-" + "cbc-des-caam", + .cra_blocksize = DES_BLOCK_SIZE, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, - .givencrypt = aead_givencrypt, - .geniv = "", .ivsize = DES_BLOCK_SIZE, .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(md5)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-hmac-md5-" + "cbc-des-caam", + .cra_blocksize = DES_BLOCK_SIZE, }, - .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_givdecrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, + .geniv = true, + }, }, { - .name = "authenc(hmac(sha1),cbc(des))", - .driver_name = "authenc-hmac-sha1-cbc-des-caam", - .blocksize = DES_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha1),cbc(des))", + .cra_driver_name = "authenc-hmac-sha1-" + "cbc-des-caam", + .cra_blocksize = DES_BLOCK_SIZE, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, - .givencrypt = aead_givencrypt, - .geniv = "", .ivsize = DES_BLOCK_SIZE, .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha1)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha1-cbc-des-caam", + .cra_blocksize = DES_BLOCK_SIZE, }, - .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_givdecrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, + .geniv = true, + }, }, { - .name = "authenc(hmac(sha224),cbc(des))", - .driver_name = "authenc-hmac-sha224-cbc-des-caam", - .blocksize = DES_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha224),cbc(des))", + .cra_driver_name = "authenc-hmac-sha224-" + "cbc-des-caam", + .cra_blocksize = DES_BLOCK_SIZE, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, - .givencrypt = aead_givencrypt, - .geniv = "", .ivsize = DES_BLOCK_SIZE, .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha224)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha224-cbc-des-caam", + .cra_blocksize = DES_BLOCK_SIZE, }, - .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA224 | - OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_givdecrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, + .geniv = true, + }, + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha256),cbc(des))", + .cra_driver_name = "authenc-hmac-sha256-" + "cbc-des-caam", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha256)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha256-cbc-des-caam", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_givdecrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, + .geniv = true, + }, }, { - .name = "authenc(hmac(sha256),cbc(des))", - .driver_name = "authenc-hmac-sha256-cbc-des-caam", - .blocksize = DES_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha384),cbc(des))", + .cra_driver_name = "authenc-hmac-sha384-" + "cbc-des-caam", + .cra_blocksize = DES_BLOCK_SIZE, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, - .givencrypt = aead_givencrypt, - .geniv = "", .ivsize = DES_BLOCK_SIZE, - .maxauthsize = SHA256_DIGEST_SIZE, - }, - .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA256 | - OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, + }, }, { - .name = "authenc(hmac(sha384),cbc(des))", - .driver_name = "authenc-hmac-sha384-cbc-des-caam", - .blocksize = DES_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha384)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha384-cbc-des-caam", + .cra_blocksize = DES_BLOCK_SIZE, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, - .decrypt = aead_decrypt, - .givencrypt = aead_givencrypt, - .geniv = "", + .decrypt = aead_givdecrypt, .ivsize = DES_BLOCK_SIZE, .maxauthsize = SHA384_DIGEST_SIZE, - }, - .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA384 | - OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, + .geniv = true, + }, }, { - .name = "authenc(hmac(sha512),cbc(des))", - .driver_name = "authenc-hmac-sha512-cbc-des-caam", - .blocksize = DES_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha512),cbc(des))", + .cra_driver_name = "authenc-hmac-sha512-" + "cbc-des-caam", + .cra_blocksize = DES_BLOCK_SIZE, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, - .givencrypt = aead_givencrypt, - .geniv = "", .ivsize = DES_BLOCK_SIZE, .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha512)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha512-cbc-des-caam", + .cra_blocksize = DES_BLOCK_SIZE, }, - .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA512 | - OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_givdecrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, + .geniv = true, + }, }, { - .name = "authenc(hmac(md5),rfc3686(ctr(aes)))", - .driver_name = "authenc-hmac-md5-rfc3686-ctr-aes-caam", - .blocksize = 1, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(md5)," + "rfc3686(ctr(aes)))", + .cra_driver_name = "authenc-hmac-md5-" + "rfc3686-ctr-aes-caam", + .cra_blocksize = 1, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, - .givencrypt = aead_givencrypt, - .geniv = "", .ivsize = CTR_RFC3686_IV_SIZE, .maxauthsize = MD5_DIGEST_SIZE, - }, - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, - .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, + .rfc3686 = true, + }, }, { - .name = "authenc(hmac(sha1),rfc3686(ctr(aes)))", - .driver_name = "authenc-hmac-sha1-rfc3686-ctr-aes-caam", - .blocksize = 1, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "seqiv(authenc(" + "hmac(md5),rfc3686(ctr(aes))))", + .cra_driver_name = "seqiv-authenc-hmac-md5-" + "rfc3686-ctr-aes-caam", + .cra_blocksize = 1, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, - .decrypt = aead_decrypt, - .givencrypt = aead_givencrypt, - .geniv = "", + .decrypt = aead_givdecrypt, .ivsize = CTR_RFC3686_IV_SIZE, - .maxauthsize = SHA1_DIGEST_SIZE, - }, - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, - .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, + .rfc3686 = true, + .geniv = true, + }, }, { - .name = "authenc(hmac(sha224),rfc3686(ctr(aes)))", - .driver_name = "authenc-hmac-sha224-rfc3686-ctr-aes-caam", - .blocksize = 1, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha1)," + "rfc3686(ctr(aes)))", + .cra_driver_name = "authenc-hmac-sha1-" + "rfc3686-ctr-aes-caam", + .cra_blocksize = 1, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, - .givencrypt = aead_givencrypt, - .geniv = "", .ivsize = CTR_RFC3686_IV_SIZE, - .maxauthsize = SHA224_DIGEST_SIZE, - }, - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, - .class2_alg_type = OP_ALG_ALGSEL_SHA224 | - OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, + .rfc3686 = true, + }, }, { - .name = "authenc(hmac(sha256),rfc3686(ctr(aes)))", - .driver_name = "authenc-hmac-sha256-rfc3686-ctr-aes-caam", - .blocksize = 1, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "seqiv(authenc(" + "hmac(sha1),rfc3686(ctr(aes))))", + .cra_driver_name = "seqiv-authenc-hmac-sha1-" + "rfc3686-ctr-aes-caam", + .cra_blocksize = 1, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, - .decrypt = aead_decrypt, - .givencrypt = aead_givencrypt, - .geniv = "", + .decrypt = aead_givdecrypt, .ivsize = CTR_RFC3686_IV_SIZE, - .maxauthsize = SHA256_DIGEST_SIZE, - }, - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, - .class2_alg_type = OP_ALG_ALGSEL_SHA256 | - OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, + .rfc3686 = true, + .geniv = true, + }, }, { - .name = "authenc(hmac(sha384),rfc3686(ctr(aes)))", - .driver_name = "authenc-hmac-sha384-rfc3686-ctr-aes-caam", - .blocksize = 1, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha224)," + "rfc3686(ctr(aes)))", + .cra_driver_name = "authenc-hmac-sha224-" + "rfc3686-ctr-aes-caam", + .cra_blocksize = 1, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, - .givencrypt = aead_givencrypt, - .geniv = "", .ivsize = CTR_RFC3686_IV_SIZE, - .maxauthsize = SHA384_DIGEST_SIZE, - }, - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, - .class2_alg_type = OP_ALG_ALGSEL_SHA384 | - OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, + .rfc3686 = true, + }, }, { - .name = "authenc(hmac(sha512),rfc3686(ctr(aes)))", - .driver_name = "authenc-hmac-sha512-rfc3686-ctr-aes-caam", - .blocksize = 1, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { + .aead = { + .base = { + .cra_name = "seqiv(authenc(" + "hmac(sha224),rfc3686(ctr(aes))))", + .cra_driver_name = "seqiv-authenc-hmac-sha224-" + "rfc3686-ctr-aes-caam", + .cra_blocksize = 1, + }, .setkey = aead_setkey, .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, - .decrypt = aead_decrypt, - .givencrypt = aead_givencrypt, - .geniv = "", + .decrypt = aead_givdecrypt, .ivsize = CTR_RFC3686_IV_SIZE, - .maxauthsize = SHA512_DIGEST_SIZE, - }, - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, - .class2_alg_type = OP_ALG_ALGSEL_SHA512 | - OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, + .rfc3686 = true, + .geniv = true, + }, }, { - .name = "rfc4106(gcm(aes))", - .driver_name = "rfc4106-gcm-aes-caam", - .blocksize = 1, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { - .setkey = rfc4106_setkey, - .setauthsize = rfc4106_setauthsize, - .encrypt = aead_encrypt, - .decrypt = aead_decrypt, - .givencrypt = aead_givencrypt, - .geniv = "", - .ivsize = 8, - .maxauthsize = AES_BLOCK_SIZE, + .aead = { + .base = { + .cra_name = "authenc(hmac(sha256)," + "rfc3686(ctr(aes)))", + .cra_driver_name = "authenc-hmac-sha256-" + "rfc3686-ctr-aes-caam", + .cra_blocksize = 1, }, - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, - }, - { - .name = "rfc4543(gcm(aes))", - .driver_name = "rfc4543-gcm-aes-caam", - .blocksize = 1, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { - .setkey = rfc4543_setkey, - .setauthsize = rfc4543_setauthsize, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, .encrypt = aead_encrypt, .decrypt = aead_decrypt, - .givencrypt = aead_givencrypt, - .geniv = "", - .ivsize = 8, - .maxauthsize = AES_BLOCK_SIZE, - }, - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, + .rfc3686 = true, + }, }, - /* Galois Counter Mode */ { - .name = "gcm(aes)", - .driver_name = "gcm-aes-caam", - .blocksize = 1, - .type = CRYPTO_ALG_TYPE_AEAD, - .template_aead = { - .setkey = gcm_setkey, - .setauthsize = gcm_setauthsize, - .encrypt = aead_encrypt, - .decrypt = aead_decrypt, - .givencrypt = NULL, - .geniv = "", - .ivsize = 12, - .maxauthsize = AES_BLOCK_SIZE, + .aead = { + .base = { + .cra_name = "seqiv(authenc(hmac(sha256)," + "rfc3686(ctr(aes))))", + .cra_driver_name = "seqiv-authenc-hmac-sha256-" + "rfc3686-ctr-aes-caam", + .cra_blocksize = 1, }, - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_givdecrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, + .rfc3686 = true, + .geniv = true, + }, }, - /* ablkcipher descriptor */ { - .name = "cbc(aes)", - .driver_name = "cbc-aes-caam", - .blocksize = AES_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_GIVCIPHER, - .template_ablkcipher = { - .setkey = ablkcipher_setkey, - .encrypt = ablkcipher_encrypt, - .decrypt = ablkcipher_decrypt, - .givencrypt = ablkcipher_givencrypt, - .geniv = "", - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, + .aead = { + .base = { + .cra_name = "authenc(hmac(sha384)," + "rfc3686(ctr(aes)))", + .cra_driver_name = "authenc-hmac-sha384-" + "rfc3686-ctr-aes-caam", + .cra_blocksize = 1, }, - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, + .rfc3686 = true, + }, }, { - .name = "cbc(des3_ede)", - .driver_name = "cbc-3des-caam", - .blocksize = DES3_EDE_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_GIVCIPHER, - .template_ablkcipher = { - .setkey = ablkcipher_setkey, - .encrypt = ablkcipher_encrypt, - .decrypt = ablkcipher_decrypt, - .givencrypt = ablkcipher_givencrypt, - .geniv = "", - .min_keysize = DES3_EDE_KEY_SIZE, - .max_keysize = DES3_EDE_KEY_SIZE, - .ivsize = DES3_EDE_BLOCK_SIZE, + .aead = { + .base = { + .cra_name = "seqiv(authenc(hmac(sha384)," + "rfc3686(ctr(aes))))", + .cra_driver_name = "seqiv-authenc-hmac-sha384-" + "rfc3686-ctr-aes-caam", + .cra_blocksize = 1, }, - .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_givdecrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, + .rfc3686 = true, + .geniv = true, + }, }, { - .name = "cbc(des)", - .driver_name = "cbc-des-caam", - .blocksize = DES_BLOCK_SIZE, - .type = CRYPTO_ALG_TYPE_GIVCIPHER, - .template_ablkcipher = { - .setkey = ablkcipher_setkey, - .encrypt = ablkcipher_encrypt, - .decrypt = ablkcipher_decrypt, - .givencrypt = ablkcipher_givencrypt, - .geniv = "", - .min_keysize = DES_KEY_SIZE, - .max_keysize = DES_KEY_SIZE, - .ivsize = DES_BLOCK_SIZE, + .aead = { + .base = { + .cra_name = "authenc(hmac(sha512)," + "rfc3686(ctr(aes)))", + .cra_driver_name = "authenc-hmac-sha512-" + "rfc3686-ctr-aes-caam", + .cra_blocksize = 1, }, - .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, + .rfc3686 = true, + }, }, { - .name = "ctr(aes)", - .driver_name = "ctr-aes-caam", - .blocksize = 1, - .type = CRYPTO_ALG_TYPE_ABLKCIPHER, - .template_ablkcipher = { - .setkey = ablkcipher_setkey, - .encrypt = ablkcipher_encrypt, - .decrypt = ablkcipher_decrypt, - .geniv = "chainiv", - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, + .aead = { + .base = { + .cra_name = "seqiv(authenc(hmac(sha512)," + "rfc3686(ctr(aes))))", + .cra_driver_name = "seqiv-authenc-hmac-sha512-" + "rfc3686-ctr-aes-caam", + .cra_blocksize = 1, }, - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, - }, - { - .name = "rfc3686(ctr(aes))", - .driver_name = "rfc3686-ctr-aes-caam", - .blocksize = 1, - .type = CRYPTO_ALG_TYPE_GIVCIPHER, - .template_ablkcipher = { - .setkey = ablkcipher_setkey, - .encrypt = ablkcipher_encrypt, - .decrypt = ablkcipher_decrypt, - .givencrypt = ablkcipher_givencrypt, - .geniv = "", - .min_keysize = AES_MIN_KEY_SIZE + - CTR_RFC3686_NONCE_SIZE, - .max_keysize = AES_MAX_KEY_SIZE + - CTR_RFC3686_NONCE_SIZE, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_givdecrypt, .ivsize = CTR_RFC3686_IV_SIZE, - }, - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, - } + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, + .rfc3686 = true, + .geniv = true, + }, + }, }; struct caam_crypto_alg { - struct list_head entry; - int class1_alg_type; - int class2_alg_type; - int alg_op; struct crypto_alg crypto_alg; + struct list_head entry; + struct caam_alg_entry caam; }; -static int caam_cra_init(struct crypto_tfm *tfm) +static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam) { - struct crypto_alg *alg = tfm->__crt_alg; - struct caam_crypto_alg *caam_alg = - container_of(alg, struct caam_crypto_alg, crypto_alg); - struct caam_ctx *ctx = crypto_tfm_ctx(tfm); - ctx->jrdev = caam_jr_alloc(); if (IS_ERR(ctx->jrdev)) { pr_err("Job Ring Device allocation for transform failed\n"); @@ -4146,17 +4334,35 @@ static int caam_cra_init(struct crypto_tfm *tfm) } /* copy descriptor header template value */ - ctx->class1_alg_type = OP_TYPE_CLASS1_ALG | caam_alg->class1_alg_type; - ctx->class2_alg_type = OP_TYPE_CLASS2_ALG | caam_alg->class2_alg_type; - ctx->alg_op = OP_TYPE_CLASS2_ALG | caam_alg->alg_op; + ctx->class1_alg_type = OP_TYPE_CLASS1_ALG | caam->class1_alg_type; + ctx->class2_alg_type = OP_TYPE_CLASS2_ALG | caam->class2_alg_type; + ctx->alg_op = OP_TYPE_CLASS2_ALG | caam->alg_op; return 0; } -static void caam_cra_exit(struct crypto_tfm *tfm) +static int caam_cra_init(struct crypto_tfm *tfm) { + struct crypto_alg *alg = tfm->__crt_alg; + struct caam_crypto_alg *caam_alg = + container_of(alg, struct caam_crypto_alg, crypto_alg); struct caam_ctx *ctx = crypto_tfm_ctx(tfm); + return caam_init_common(ctx, &caam_alg->caam); +} + +static int caam_aead_init(struct crypto_aead *tfm) +{ + struct aead_alg *alg = crypto_aead_alg(tfm); + struct caam_aead_alg *caam_alg = + container_of(alg, struct caam_aead_alg, aead); + struct caam_ctx *ctx = crypto_aead_ctx(tfm); + + return caam_init_common(ctx, &caam_alg->caam); +} + +static void caam_exit_common(struct caam_ctx *ctx) +{ if (ctx->sh_desc_enc_dma && !dma_mapping_error(ctx->jrdev, ctx->sh_desc_enc_dma)) dma_unmap_single(ctx->jrdev, ctx->sh_desc_enc_dma, @@ -4179,10 +4385,28 @@ static void caam_cra_exit(struct crypto_tfm *tfm) caam_jr_free(ctx->jrdev); } +static void caam_cra_exit(struct crypto_tfm *tfm) +{ + caam_exit_common(crypto_tfm_ctx(tfm)); +} + +static void caam_aead_exit(struct crypto_aead *tfm) +{ + caam_exit_common(crypto_aead_ctx(tfm)); +} + static void __exit caam_algapi_exit(void) { struct caam_crypto_alg *t_alg, *n; + int i; + + for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { + struct caam_aead_alg *t_alg = driver_aeads + i; + + if (t_alg->registered) + crypto_unregister_aead(&t_alg->aead); + } if (!alg_list.next) return; @@ -4200,7 +4424,7 @@ static struct caam_crypto_alg *caam_alg_alloc(struct caam_alg_template struct caam_crypto_alg *t_alg; struct crypto_alg *alg; - t_alg = kzalloc(sizeof(struct caam_crypto_alg), GFP_KERNEL); + t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL); if (!t_alg) { pr_err("failed to allocate t_alg\n"); return ERR_PTR(-ENOMEM); @@ -4229,26 +4453,38 @@ static struct caam_crypto_alg *caam_alg_alloc(struct caam_alg_template alg->cra_type = &crypto_ablkcipher_type; alg->cra_ablkcipher = template->template_ablkcipher; break; - case CRYPTO_ALG_TYPE_AEAD: - alg->cra_type = &crypto_aead_type; - alg->cra_aead = template->template_aead; - break; } - t_alg->class1_alg_type = template->class1_alg_type; - t_alg->class2_alg_type = template->class2_alg_type; - t_alg->alg_op = template->alg_op; + t_alg->caam.class1_alg_type = template->class1_alg_type; + t_alg->caam.class2_alg_type = template->class2_alg_type; + t_alg->caam.alg_op = template->alg_op; return t_alg; } +static void caam_aead_alg_init(struct caam_aead_alg *t_alg) +{ + struct aead_alg *alg = &t_alg->aead; + + alg->base.cra_module = THIS_MODULE; + alg->base.cra_priority = CAAM_CRA_PRIORITY; + alg->base.cra_ctxsize = sizeof(struct caam_ctx); + alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; + + alg->init = caam_aead_init; + alg->exit = caam_aead_exit; +} + static int __init caam_algapi_init(void) { struct device_node *dev_node; struct platform_device *pdev; struct device *ctrldev; - void *priv; + struct caam_drv_private *priv; int i = 0, err = 0; + u32 cha_vid, cha_inst, des_inst, aes_inst, md_inst; + unsigned int md_limit = SHA512_DIGEST_SIZE; + bool registered = false; dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); if (!dev_node) { @@ -4277,16 +4513,39 @@ static int __init caam_algapi_init(void) INIT_LIST_HEAD(&alg_list); - /* register crypto algorithms the device supports */ + /* + * Register crypto algorithms the device supports. + * First, detect presence and attributes of DES, AES, and MD blocks. + */ + cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls); + cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls); + des_inst = (cha_inst & CHA_ID_LS_DES_MASK) >> CHA_ID_LS_DES_SHIFT; + aes_inst = (cha_inst & CHA_ID_LS_AES_MASK) >> CHA_ID_LS_AES_SHIFT; + md_inst = (cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT; + + /* If MD is present, limit digest size based on LP256 */ + if (md_inst && ((cha_vid & CHA_ID_LS_MD_MASK) == CHA_ID_LS_MD_LP256)) + md_limit = SHA256_DIGEST_SIZE; + for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { - /* TODO: check if h/w supports alg */ struct caam_crypto_alg *t_alg; + struct caam_alg_template *alg = driver_algs + i; + u32 alg_sel = alg->class1_alg_type & OP_ALG_ALGSEL_MASK; + + /* Skip DES algorithms if not supported by device */ + if (!des_inst && + ((alg_sel == OP_ALG_ALGSEL_3DES) || + (alg_sel == OP_ALG_ALGSEL_DES))) + continue; - t_alg = caam_alg_alloc(&driver_algs[i]); + /* Skip AES algorithms if not supported by device */ + if (!aes_inst && (alg_sel == OP_ALG_ALGSEL_AES)) + continue; + + t_alg = caam_alg_alloc(alg); if (IS_ERR(t_alg)) { err = PTR_ERR(t_alg); - pr_warn("%s alg allocation failed\n", - driver_algs[i].driver_name); + pr_warn("%s alg allocation failed\n", alg->driver_name); continue; } @@ -4295,10 +4554,61 @@ static int __init caam_algapi_init(void) pr_warn("%s alg registration failed\n", t_alg->crypto_alg.cra_driver_name); kfree(t_alg); - } else - list_add_tail(&t_alg->entry, &alg_list); + continue; + } + + list_add_tail(&t_alg->entry, &alg_list); + registered = true; } - if (!list_empty(&alg_list)) + + for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { + struct caam_aead_alg *t_alg = driver_aeads + i; + u32 c1_alg_sel = t_alg->caam.class1_alg_type & + OP_ALG_ALGSEL_MASK; + u32 c2_alg_sel = t_alg->caam.class2_alg_type & + OP_ALG_ALGSEL_MASK; + u32 alg_aai = t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK; + + /* Skip DES algorithms if not supported by device */ + if (!des_inst && + ((c1_alg_sel == OP_ALG_ALGSEL_3DES) || + (c1_alg_sel == OP_ALG_ALGSEL_DES))) + continue; + + /* Skip AES algorithms if not supported by device */ + if (!aes_inst && (c1_alg_sel == OP_ALG_ALGSEL_AES)) + continue; + + /* + * Check support for AES algorithms not available + * on LP devices. + */ + if ((cha_vid & CHA_ID_LS_AES_MASK) == CHA_ID_LS_AES_LP) + if (alg_aai == OP_ALG_AAI_GCM) + continue; + + /* + * Skip algorithms requiring message digests + * if MD or MD size is not supported by device. + */ + if (c2_alg_sel && + (!md_inst || (t_alg->aead.maxauthsize > md_limit))) + continue; + + caam_aead_alg_init(t_alg); + + err = crypto_register_aead(&t_alg->aead); + if (err) { + pr_warn("%s alg registration failed\n", + t_alg->aead.base.cra_driver_name); + continue; + } + + t_alg->registered = true; + registered = true; + } + + if (registered) pr_info("caam algorithms registered in /proc/crypto\n"); return err; diff --git a/kernel/drivers/crypto/caam/caamhash.c b/kernel/drivers/crypto/caam/caamhash.c index 0436997e0..49106ea42 100644 --- a/kernel/drivers/crypto/caam/caamhash.c +++ b/kernel/drivers/crypto/caam/caamhash.c @@ -127,13 +127,22 @@ struct caam_hash_state { int buflen_0; u8 buf_1[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned; int buflen_1; - u8 caam_ctx[MAX_CTX_LEN]; + u8 caam_ctx[MAX_CTX_LEN] ____cacheline_aligned; int (*update)(struct ahash_request *req); int (*final)(struct ahash_request *req); int (*finup)(struct ahash_request *req); int current_buf; }; +struct caam_export_state { + u8 buf[CAAM_MAX_HASH_BLOCK_SIZE]; + u8 caam_ctx[MAX_CTX_LEN]; + int buflen; + int (*update)(struct ahash_request *req); + int (*final)(struct ahash_request *req); + int (*finup)(struct ahash_request *req); +}; + /* Common job descriptor seq in/out ptr routines */ /* Map state->caam_ctx, and append seq_out_ptr command that points to it */ @@ -181,10 +190,9 @@ static inline dma_addr_t buf_map_to_sec4_sg(struct device *jrdev, /* Map req->src and put it in link table */ static inline void src_map_to_sec4_sg(struct device *jrdev, struct scatterlist *src, int src_nents, - struct sec4_sg_entry *sec4_sg, - bool chained) + struct sec4_sg_entry *sec4_sg) { - dma_map_sg_chained(jrdev, src, src_nents, DMA_TO_DEVICE, chained); + dma_map_sg(jrdev, src, src_nents, DMA_TO_DEVICE); sg_to_sec4_sg_last(src, src_nents, sec4_sg, 0); } @@ -585,7 +593,6 @@ badkey: * ahash_edesc - s/w-extended ahash descriptor * @dst_dma: physical mapped address of req->result * @sec4_sg_dma: physical mapped address of h/w link table - * @chained: if source is chained * @src_nents: number of segments in input scatterlist * @sec4_sg_bytes: length of dma mapped sec4_sg space * @sec4_sg: pointer to h/w link table @@ -594,7 +601,6 @@ badkey: struct ahash_edesc { dma_addr_t dst_dma; dma_addr_t sec4_sg_dma; - bool chained; int src_nents; int sec4_sg_bytes; struct sec4_sg_entry *sec4_sg; @@ -606,8 +612,7 @@ static inline void ahash_unmap(struct device *dev, struct ahash_request *req, int dst_len) { if (edesc->src_nents) - dma_unmap_sg_chained(dev, req->src, edesc->src_nents, - DMA_TO_DEVICE, edesc->chained); + dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE); if (edesc->dst_dma) dma_unmap_single(dev, edesc->dst_dma, dst_len, DMA_FROM_DEVICE); @@ -788,7 +793,6 @@ static int ahash_update_ctx(struct ahash_request *req) dma_addr_t ptr = ctx->sh_desc_update_dma; int src_nents, sec4_sg_bytes, sec4_sg_src_index; struct ahash_edesc *edesc; - bool chained = false; int ret = 0; int sh_len; @@ -797,8 +801,8 @@ static int ahash_update_ctx(struct ahash_request *req) to_hash = in_len - *next_buflen; if (to_hash) { - src_nents = __sg_count(req->src, req->nbytes - (*next_buflen), - &chained); + src_nents = sg_nents_for_len(req->src, + req->nbytes - (*next_buflen)); sec4_sg_src_index = 1 + (*buflen ? 1 : 0); sec4_sg_bytes = (sec4_sg_src_index + src_nents) * sizeof(struct sec4_sg_entry); @@ -807,7 +811,7 @@ static int ahash_update_ctx(struct ahash_request *req) * allocate space for base edesc and hw desc commands, * link tables */ - edesc = kmalloc(sizeof(struct ahash_edesc) + DESC_JOB_IO_LEN + + edesc = kzalloc(sizeof(*edesc) + DESC_JOB_IO_LEN + sec4_sg_bytes, GFP_DMA | flags); if (!edesc) { dev_err(jrdev, @@ -816,7 +820,6 @@ static int ahash_update_ctx(struct ahash_request *req) } edesc->src_nents = src_nents; - edesc->chained = chained; edesc->sec4_sg_bytes = sec4_sg_bytes; edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) + DESC_JOB_IO_LEN; @@ -833,19 +836,18 @@ static int ahash_update_ctx(struct ahash_request *req) if (src_nents) { src_map_to_sec4_sg(jrdev, req->src, src_nents, - edesc->sec4_sg + sec4_sg_src_index, - chained); - if (*next_buflen) { + edesc->sec4_sg + sec4_sg_src_index); + if (*next_buflen) scatterwalk_map_and_copy(next_buf, req->src, to_hash - *buflen, *next_buflen, 0); - state->current_buf = !state->current_buf; - } } else { (edesc->sec4_sg + sec4_sg_src_index - 1)->len |= SEC4_SG_LEN_FIN; } + state->current_buf = !state->current_buf; + sh_len = desc_len(sh_desc); desc = edesc->hw_desc; init_job_desc_shared(desc, ptr, sh_len, HDR_SHARE_DEFER | @@ -919,8 +921,8 @@ static int ahash_final_ctx(struct ahash_request *req) sec4_sg_bytes = sec4_sg_src_index * sizeof(struct sec4_sg_entry); /* allocate space for base edesc and hw desc commands, link tables */ - edesc = kmalloc(sizeof(struct ahash_edesc) + DESC_JOB_IO_LEN + - sec4_sg_bytes, GFP_DMA | flags); + edesc = kzalloc(sizeof(*edesc) + DESC_JOB_IO_LEN + sec4_sg_bytes, + GFP_DMA | flags); if (!edesc) { dev_err(jrdev, "could not allocate extended descriptor\n"); return -ENOMEM; @@ -996,18 +998,17 @@ static int ahash_finup_ctx(struct ahash_request *req) int src_nents; int digestsize = crypto_ahash_digestsize(ahash); struct ahash_edesc *edesc; - bool chained = false; int ret = 0; int sh_len; - src_nents = __sg_count(req->src, req->nbytes, &chained); + src_nents = sg_nents_for_len(req->src, req->nbytes); sec4_sg_src_index = 1 + (buflen ? 1 : 0); sec4_sg_bytes = (sec4_sg_src_index + src_nents) * sizeof(struct sec4_sg_entry); /* allocate space for base edesc and hw desc commands, link tables */ - edesc = kmalloc(sizeof(struct ahash_edesc) + DESC_JOB_IO_LEN + - sec4_sg_bytes, GFP_DMA | flags); + edesc = kzalloc(sizeof(*edesc) + DESC_JOB_IO_LEN + sec4_sg_bytes, + GFP_DMA | flags); if (!edesc) { dev_err(jrdev, "could not allocate extended descriptor\n"); return -ENOMEM; @@ -1018,7 +1019,6 @@ static int ahash_finup_ctx(struct ahash_request *req) init_job_desc_shared(desc, ptr, sh_len, HDR_SHARE_DEFER | HDR_REVERSE); edesc->src_nents = src_nents; - edesc->chained = chained; edesc->sec4_sg_bytes = sec4_sg_bytes; edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) + DESC_JOB_IO_LEN; @@ -1033,7 +1033,7 @@ static int ahash_finup_ctx(struct ahash_request *req) last_buflen); src_map_to_sec4_sg(jrdev, req->src, src_nents, edesc->sec4_sg + - sec4_sg_src_index, chained); + sec4_sg_src_index); edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, sec4_sg_bytes, DMA_TO_DEVICE); @@ -1081,19 +1081,17 @@ static int ahash_digest(struct ahash_request *req) int src_nents, sec4_sg_bytes; dma_addr_t src_dma; struct ahash_edesc *edesc; - bool chained = false; int ret = 0; u32 options; int sh_len; - src_nents = sg_count(req->src, req->nbytes, &chained); - dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, DMA_TO_DEVICE, - chained); + src_nents = sg_count(req->src, req->nbytes); + dma_map_sg(jrdev, req->src, src_nents ? : 1, DMA_TO_DEVICE); sec4_sg_bytes = src_nents * sizeof(struct sec4_sg_entry); /* allocate space for base edesc and hw desc commands, link tables */ - edesc = kmalloc(sizeof(struct ahash_edesc) + sec4_sg_bytes + - DESC_JOB_IO_LEN, GFP_DMA | flags); + edesc = kzalloc(sizeof(*edesc) + sec4_sg_bytes + DESC_JOB_IO_LEN, + GFP_DMA | flags); if (!edesc) { dev_err(jrdev, "could not allocate extended descriptor\n"); return -ENOMEM; @@ -1102,7 +1100,6 @@ static int ahash_digest(struct ahash_request *req) DESC_JOB_IO_LEN; edesc->sec4_sg_bytes = sec4_sg_bytes; edesc->src_nents = src_nents; - edesc->chained = chained; sh_len = desc_len(sh_desc); desc = edesc->hw_desc; @@ -1166,8 +1163,7 @@ static int ahash_final_no_ctx(struct ahash_request *req) int sh_len; /* allocate space for base edesc and hw desc commands, link tables */ - edesc = kmalloc(sizeof(struct ahash_edesc) + DESC_JOB_IO_LEN, - GFP_DMA | flags); + edesc = kzalloc(sizeof(*edesc) + DESC_JOB_IO_LEN, GFP_DMA | flags); if (!edesc) { dev_err(jrdev, "could not allocate extended descriptor\n"); return -ENOMEM; @@ -1229,7 +1225,6 @@ static int ahash_update_no_ctx(struct ahash_request *req) struct ahash_edesc *edesc; u32 *desc, *sh_desc = ctx->sh_desc_update_first; dma_addr_t ptr = ctx->sh_desc_update_first_dma; - bool chained = false; int ret = 0; int sh_len; @@ -1237,8 +1232,8 @@ static int ahash_update_no_ctx(struct ahash_request *req) to_hash = in_len - *next_buflen; if (to_hash) { - src_nents = __sg_count(req->src, req->nbytes - (*next_buflen), - &chained); + src_nents = sg_nents_for_len(req->src, + req->nbytes - (*next_buflen)); sec4_sg_bytes = (1 + src_nents) * sizeof(struct sec4_sg_entry); @@ -1246,7 +1241,7 @@ static int ahash_update_no_ctx(struct ahash_request *req) * allocate space for base edesc and hw desc commands, * link tables */ - edesc = kmalloc(sizeof(struct ahash_edesc) + DESC_JOB_IO_LEN + + edesc = kzalloc(sizeof(*edesc) + DESC_JOB_IO_LEN + sec4_sg_bytes, GFP_DMA | flags); if (!edesc) { dev_err(jrdev, @@ -1255,7 +1250,6 @@ static int ahash_update_no_ctx(struct ahash_request *req) } edesc->src_nents = src_nents; - edesc->chained = chained; edesc->sec4_sg_bytes = sec4_sg_bytes; edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) + DESC_JOB_IO_LEN; @@ -1264,14 +1258,15 @@ static int ahash_update_no_ctx(struct ahash_request *req) state->buf_dma = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, buf, *buflen); src_map_to_sec4_sg(jrdev, req->src, src_nents, - edesc->sec4_sg + 1, chained); + edesc->sec4_sg + 1); if (*next_buflen) { scatterwalk_map_and_copy(next_buf, req->src, to_hash - *buflen, *next_buflen, 0); - state->current_buf = !state->current_buf; } + state->current_buf = !state->current_buf; + sh_len = desc_len(sh_desc); desc = edesc->hw_desc; init_job_desc_shared(desc, ptr, sh_len, HDR_SHARE_DEFER | @@ -1343,18 +1338,17 @@ static int ahash_finup_no_ctx(struct ahash_request *req) int sec4_sg_bytes, sec4_sg_src_index, src_nents; int digestsize = crypto_ahash_digestsize(ahash); struct ahash_edesc *edesc; - bool chained = false; int sh_len; int ret = 0; - src_nents = __sg_count(req->src, req->nbytes, &chained); + src_nents = sg_nents_for_len(req->src, req->nbytes); sec4_sg_src_index = 2; sec4_sg_bytes = (sec4_sg_src_index + src_nents) * sizeof(struct sec4_sg_entry); /* allocate space for base edesc and hw desc commands, link tables */ - edesc = kmalloc(sizeof(struct ahash_edesc) + DESC_JOB_IO_LEN + - sec4_sg_bytes, GFP_DMA | flags); + edesc = kzalloc(sizeof(*edesc) + DESC_JOB_IO_LEN + sec4_sg_bytes, + GFP_DMA | flags); if (!edesc) { dev_err(jrdev, "could not allocate extended descriptor\n"); return -ENOMEM; @@ -1365,7 +1359,6 @@ static int ahash_finup_no_ctx(struct ahash_request *req) init_job_desc_shared(desc, ptr, sh_len, HDR_SHARE_DEFER | HDR_REVERSE); edesc->src_nents = src_nents; - edesc->chained = chained; edesc->sec4_sg_bytes = sec4_sg_bytes; edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) + DESC_JOB_IO_LEN; @@ -1374,8 +1367,7 @@ static int ahash_finup_no_ctx(struct ahash_request *req) state->buf_dma, buflen, last_buflen); - src_map_to_sec4_sg(jrdev, req->src, src_nents, edesc->sec4_sg + 1, - chained); + src_map_to_sec4_sg(jrdev, req->src, src_nents, edesc->sec4_sg + 1); edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, sec4_sg_bytes, DMA_TO_DEVICE); @@ -1429,7 +1421,6 @@ static int ahash_update_first(struct ahash_request *req) dma_addr_t src_dma; u32 options; struct ahash_edesc *edesc; - bool chained = false; int ret = 0; int sh_len; @@ -1438,17 +1429,15 @@ static int ahash_update_first(struct ahash_request *req) to_hash = req->nbytes - *next_buflen; if (to_hash) { - src_nents = sg_count(req->src, req->nbytes - (*next_buflen), - &chained); - dma_map_sg_chained(jrdev, req->src, src_nents ? : 1, - DMA_TO_DEVICE, chained); + src_nents = sg_count(req->src, req->nbytes - (*next_buflen)); + dma_map_sg(jrdev, req->src, src_nents ? : 1, DMA_TO_DEVICE); sec4_sg_bytes = src_nents * sizeof(struct sec4_sg_entry); /* * allocate space for base edesc and hw desc commands, * link tables */ - edesc = kmalloc(sizeof(struct ahash_edesc) + DESC_JOB_IO_LEN + + edesc = kzalloc(sizeof(*edesc) + DESC_JOB_IO_LEN + sec4_sg_bytes, GFP_DMA | flags); if (!edesc) { dev_err(jrdev, @@ -1457,7 +1446,6 @@ static int ahash_update_first(struct ahash_request *req) } edesc->src_nents = src_nents; - edesc->chained = chained; edesc->sec4_sg_bytes = sec4_sg_bytes; edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) + DESC_JOB_IO_LEN; @@ -1574,25 +1562,42 @@ static int ahash_final(struct ahash_request *req) static int ahash_export(struct ahash_request *req, void *out) { - struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); - struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); struct caam_hash_state *state = ahash_request_ctx(req); + struct caam_export_state *export = out; + int len; + u8 *buf; + + if (state->current_buf) { + buf = state->buf_1; + len = state->buflen_1; + } else { + buf = state->buf_0; + len = state->buflen_1; + } + + memcpy(export->buf, buf, len); + memcpy(export->caam_ctx, state->caam_ctx, sizeof(export->caam_ctx)); + export->buflen = len; + export->update = state->update; + export->final = state->final; + export->finup = state->finup; - memcpy(out, ctx, sizeof(struct caam_hash_ctx)); - memcpy(out + sizeof(struct caam_hash_ctx), state, - sizeof(struct caam_hash_state)); return 0; } static int ahash_import(struct ahash_request *req, const void *in) { - struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); - struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); struct caam_hash_state *state = ahash_request_ctx(req); + const struct caam_export_state *export = in; + + memset(state, 0, sizeof(*state)); + memcpy(state->buf_0, export->buf, export->buflen); + memcpy(state->caam_ctx, export->caam_ctx, sizeof(state->caam_ctx)); + state->buflen_0 = export->buflen; + state->update = export->update; + state->final = export->final; + state->finup = export->finup; - memcpy(ctx, in, sizeof(struct caam_hash_ctx)); - memcpy(state, in + sizeof(struct caam_hash_ctx), - sizeof(struct caam_hash_state)); return 0; } @@ -1626,8 +1631,9 @@ static struct caam_hash_template driver_hash[] = { .setkey = ahash_setkey, .halg = { .digestsize = SHA1_DIGEST_SIZE, - }, + .statesize = sizeof(struct caam_export_state), }, + }, .alg_type = OP_ALG_ALGSEL_SHA1, .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, }, { @@ -1647,8 +1653,9 @@ static struct caam_hash_template driver_hash[] = { .setkey = ahash_setkey, .halg = { .digestsize = SHA224_DIGEST_SIZE, - }, + .statesize = sizeof(struct caam_export_state), }, + }, .alg_type = OP_ALG_ALGSEL_SHA224, .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, }, { @@ -1668,8 +1675,9 @@ static struct caam_hash_template driver_hash[] = { .setkey = ahash_setkey, .halg = { .digestsize = SHA256_DIGEST_SIZE, - }, + .statesize = sizeof(struct caam_export_state), }, + }, .alg_type = OP_ALG_ALGSEL_SHA256, .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, }, { @@ -1689,8 +1697,9 @@ static struct caam_hash_template driver_hash[] = { .setkey = ahash_setkey, .halg = { .digestsize = SHA384_DIGEST_SIZE, - }, + .statesize = sizeof(struct caam_export_state), }, + }, .alg_type = OP_ALG_ALGSEL_SHA384, .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, }, { @@ -1710,8 +1719,9 @@ static struct caam_hash_template driver_hash[] = { .setkey = ahash_setkey, .halg = { .digestsize = SHA512_DIGEST_SIZE, - }, + .statesize = sizeof(struct caam_export_state), }, + }, .alg_type = OP_ALG_ALGSEL_SHA512, .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, }, { @@ -1731,8 +1741,9 @@ static struct caam_hash_template driver_hash[] = { .setkey = ahash_setkey, .halg = { .digestsize = MD5_DIGEST_SIZE, - }, + .statesize = sizeof(struct caam_export_state), }, + }, .alg_type = OP_ALG_ALGSEL_MD5, .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, }, @@ -1842,7 +1853,7 @@ caam_hash_alloc(struct caam_hash_template *template, struct ahash_alg *halg; struct crypto_alg *alg; - t_alg = kzalloc(sizeof(struct caam_hash_alg), GFP_KERNEL); + t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL); if (!t_alg) { pr_err("failed to allocate t_alg\n"); return ERR_PTR(-ENOMEM); @@ -1884,8 +1895,10 @@ static int __init caam_algapi_hash_init(void) struct device_node *dev_node; struct platform_device *pdev; struct device *ctrldev; - void *priv; int i = 0, err = 0; + struct caam_drv_private *priv; + unsigned int md_limit = SHA512_DIGEST_SIZE; + u32 cha_inst, cha_vid; dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); if (!dev_node) { @@ -1911,43 +1924,65 @@ static int __init caam_algapi_hash_init(void) if (!priv) return -ENODEV; + /* + * Register crypto algorithms the device supports. First, identify + * presence and attributes of MD block. + */ + cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls); + cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls); + + /* + * Skip registration of any hashing algorithms if MD block + * is not present. + */ + if (!((cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT)) + return -ENODEV; + + /* Limit digest size based on LP256 */ + if ((cha_vid & CHA_ID_LS_MD_MASK) == CHA_ID_LS_MD_LP256) + md_limit = SHA256_DIGEST_SIZE; + INIT_LIST_HEAD(&hash_list); /* register crypto algorithms the device supports */ for (i = 0; i < ARRAY_SIZE(driver_hash); i++) { - /* TODO: check if h/w supports alg */ struct caam_hash_alg *t_alg; + struct caam_hash_template *alg = driver_hash + i; + + /* If MD size is not supported by device, skip registration */ + if (alg->template_ahash.halg.digestsize > md_limit) + continue; /* register hmac version */ - t_alg = caam_hash_alloc(&driver_hash[i], true); + t_alg = caam_hash_alloc(alg, true); if (IS_ERR(t_alg)) { err = PTR_ERR(t_alg); - pr_warn("%s alg allocation failed\n", - driver_hash[i].driver_name); + pr_warn("%s alg allocation failed\n", alg->driver_name); continue; } err = crypto_register_ahash(&t_alg->ahash_alg); if (err) { - pr_warn("%s alg registration failed\n", - t_alg->ahash_alg.halg.base.cra_driver_name); + pr_warn("%s alg registration failed: %d\n", + t_alg->ahash_alg.halg.base.cra_driver_name, + err); kfree(t_alg); } else list_add_tail(&t_alg->entry, &hash_list); /* register unkeyed version */ - t_alg = caam_hash_alloc(&driver_hash[i], false); + t_alg = caam_hash_alloc(alg, false); if (IS_ERR(t_alg)) { err = PTR_ERR(t_alg); - pr_warn("%s alg allocation failed\n", - driver_hash[i].driver_name); + pr_warn("%s alg allocation failed\n", alg->driver_name); continue; } err = crypto_register_ahash(&t_alg->ahash_alg); if (err) { - pr_warn("%s alg registration failed\n", - t_alg->ahash_alg.halg.base.cra_driver_name); + pr_warn("%s alg registration failed: %d\n", + t_alg->ahash_alg.halg.base.cra_driver_name, + err); kfree(t_alg); } else list_add_tail(&t_alg->entry, &hash_list); diff --git a/kernel/drivers/crypto/caam/caamrng.c b/kernel/drivers/crypto/caam/caamrng.c index 509533720..9b92af2c7 100644 --- a/kernel/drivers/crypto/caam/caamrng.c +++ b/kernel/drivers/crypto/caam/caamrng.c @@ -108,6 +108,10 @@ static void rng_done(struct device *jrdev, u32 *desc, u32 err, void *context) atomic_set(&bd->empty, BUF_NOT_EMPTY); complete(&bd->filled); + + /* Buffer refilled, invalidate cache */ + dma_sync_single_for_cpu(jrdev, bd->addr, RN_BUF_SIZE, DMA_FROM_DEVICE); + #ifdef DEBUG print_hex_dump(KERN_ERR, "rng refreshed buf@: ", DUMP_PREFIX_ADDRESS, 16, 4, bd->buf, RN_BUF_SIZE, 1); @@ -311,7 +315,7 @@ static int __init caam_rng_init(void) struct device_node *dev_node; struct platform_device *pdev; struct device *ctrldev; - void *priv; + struct caam_drv_private *priv; int err; dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); @@ -338,20 +342,32 @@ static int __init caam_rng_init(void) if (!priv) return -ENODEV; + /* Check for an instantiated RNG before registration */ + if (!(rd_reg32(&priv->ctrl->perfmon.cha_num_ls) & CHA_ID_LS_RNG_MASK)) + return -ENODEV; + dev = caam_jr_alloc(); if (IS_ERR(dev)) { pr_err("Job Ring Device allocation for transform failed\n"); return PTR_ERR(dev); } - rng_ctx = kmalloc(sizeof(struct caam_rng_ctx), GFP_DMA); - if (!rng_ctx) - return -ENOMEM; + rng_ctx = kmalloc(sizeof(*rng_ctx), GFP_DMA); + if (!rng_ctx) { + err = -ENOMEM; + goto free_caam_alloc; + } err = caam_init_rng(rng_ctx, dev); if (err) - return err; + goto free_rng_ctx; dev_info(dev, "registering rng-caam\n"); return hwrng_register(&caam_rng); + +free_rng_ctx: + kfree(rng_ctx); +free_caam_alloc: + caam_jr_free(dev); + return err; } module_init(caam_rng_init); diff --git a/kernel/drivers/crypto/caam/compat.h b/kernel/drivers/crypto/caam/compat.h index acd7743e2..b6955ecdf 100644 --- a/kernel/drivers/crypto/caam/compat.h +++ b/kernel/drivers/crypto/caam/compat.h @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -32,7 +33,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/kernel/drivers/crypto/caam/ctrl.c b/kernel/drivers/crypto/caam/ctrl.c index efba4ccd4..69d4a1326 100644 --- a/kernel/drivers/crypto/caam/ctrl.c +++ b/kernel/drivers/crypto/caam/ctrl.c @@ -15,6 +15,24 @@ #include "desc_constr.h" #include "error.h" +/* + * i.MX targets tend to have clock control subsystems that can + * enable/disable clocking to our device. + */ +#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_IMX +static inline struct clk *caam_drv_identify_clk(struct device *dev, + char *clk_name) +{ + return devm_clk_get(dev, clk_name); +} +#else +static inline struct clk *caam_drv_identify_clk(struct device *dev, + char *clk_name) +{ + return NULL; +} +#endif + /* * Descriptor to instantiate RNG State Handle 0 in normal mode and * load the JDKEK, TDKEK and TDSK registers @@ -121,7 +139,7 @@ static inline int run_descriptor_deco0(struct device *ctrldev, u32 *desc, flags |= DECO_JQCR_FOUR; /* Instruct the DECO to execute it */ - wr_reg32(&deco->jr_ctl_hi, flags); + setbits32(&deco->jr_ctl_hi, flags); timeout = 10000000; do { @@ -175,7 +193,7 @@ static int instantiate_rng(struct device *ctrldev, int state_handle_mask, { struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctrldev); struct caam_ctrl __iomem *ctrl; - u32 *desc, status, rdsta_val; + u32 *desc, status = 0, rdsta_val; int ret = 0, sh_idx; ctrl = (struct caam_ctrl __iomem *)ctrlpriv->ctrl; @@ -207,7 +225,8 @@ static int instantiate_rng(struct device *ctrldev, int state_handle_mask, * CAAM eras), then try again. */ rdsta_val = rd_reg32(&ctrl->r4tst[0].rdsta) & RDSTA_IFMASK; - if (status || !(rdsta_val & (1 << sh_idx))) + if ((status && status != JRSTA_SSRC_JUMP_HALT_CC) || + !(rdsta_val & (1 << sh_idx))) ret = -EAGAIN; if (ret) break; @@ -279,7 +298,7 @@ static int caam_remove(struct platform_device *pdev) struct device *ctrldev; struct caam_drv_private *ctrlpriv; struct caam_ctrl __iomem *ctrl; - int ring, ret = 0; + int ring; ctrldev = &pdev->dev; ctrlpriv = dev_get_drvdata(ctrldev); @@ -301,9 +320,15 @@ static int caam_remove(struct platform_device *pdev) #endif /* Unmap controller region */ - iounmap(&ctrl); + iounmap(ctrl); - return ret; + /* shut clocks off before finalizing shutdown */ + clk_disable_unprepare(ctrlpriv->caam_ipg); + clk_disable_unprepare(ctrlpriv->caam_mem); + clk_disable_unprepare(ctrlpriv->caam_aclk); + clk_disable_unprepare(ctrlpriv->caam_emi_slow); + + return 0; } /* @@ -370,14 +395,14 @@ static void kick_trng(struct platform_device *pdev, int ent_delay) int caam_get_era(void) { struct device_node *caam_node; - for_each_compatible_node(caam_node, NULL, "fsl,sec-v4.0") { - const uint32_t *prop = (uint32_t *)of_get_property(caam_node, - "fsl,sec-era", - NULL); - return prop ? *prop : -ENOTSUPP; - } + int ret; + u32 prop; - return -ENOTSUPP; + caam_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); + ret = of_property_read_u32(caam_node, "fsl,sec-era", &prop); + of_node_put(caam_node); + + return IS_ERR_VALUE(ret) ? -ENOTSUPP : prop; } EXPORT_SYMBOL(caam_get_era); @@ -390,6 +415,7 @@ static int caam_probe(struct platform_device *pdev) struct device_node *nprop, *np; struct caam_ctrl __iomem *ctrl; struct caam_drv_private *ctrlpriv; + struct clk *clk; #ifdef CONFIG_DEBUG_FS struct caam_perfmon *perfmon; #endif @@ -398,8 +424,7 @@ static int caam_probe(struct platform_device *pdev) int pg_size; int BLOCK_OFFSET = 0; - ctrlpriv = devm_kzalloc(&pdev->dev, sizeof(struct caam_drv_private), - GFP_KERNEL); + ctrlpriv = devm_kzalloc(&pdev->dev, sizeof(*ctrlpriv), GFP_KERNEL); if (!ctrlpriv) return -ENOMEM; @@ -408,12 +433,76 @@ static int caam_probe(struct platform_device *pdev) ctrlpriv->pdev = pdev; nprop = pdev->dev.of_node; + /* Enable clocking */ + clk = caam_drv_identify_clk(&pdev->dev, "ipg"); + if (IS_ERR(clk)) { + ret = PTR_ERR(clk); + dev_err(&pdev->dev, + "can't identify CAAM ipg clk: %d\n", ret); + return ret; + } + ctrlpriv->caam_ipg = clk; + + clk = caam_drv_identify_clk(&pdev->dev, "mem"); + if (IS_ERR(clk)) { + ret = PTR_ERR(clk); + dev_err(&pdev->dev, + "can't identify CAAM mem clk: %d\n", ret); + return ret; + } + ctrlpriv->caam_mem = clk; + + clk = caam_drv_identify_clk(&pdev->dev, "aclk"); + if (IS_ERR(clk)) { + ret = PTR_ERR(clk); + dev_err(&pdev->dev, + "can't identify CAAM aclk clk: %d\n", ret); + return ret; + } + ctrlpriv->caam_aclk = clk; + + clk = caam_drv_identify_clk(&pdev->dev, "emi_slow"); + if (IS_ERR(clk)) { + ret = PTR_ERR(clk); + dev_err(&pdev->dev, + "can't identify CAAM emi_slow clk: %d\n", ret); + return ret; + } + ctrlpriv->caam_emi_slow = clk; + + ret = clk_prepare_enable(ctrlpriv->caam_ipg); + if (ret < 0) { + dev_err(&pdev->dev, "can't enable CAAM ipg clock: %d\n", ret); + return ret; + } + + ret = clk_prepare_enable(ctrlpriv->caam_mem); + if (ret < 0) { + dev_err(&pdev->dev, "can't enable CAAM secure mem clock: %d\n", + ret); + goto disable_caam_ipg; + } + + ret = clk_prepare_enable(ctrlpriv->caam_aclk); + if (ret < 0) { + dev_err(&pdev->dev, "can't enable CAAM aclk clock: %d\n", ret); + goto disable_caam_mem; + } + + ret = clk_prepare_enable(ctrlpriv->caam_emi_slow); + if (ret < 0) { + dev_err(&pdev->dev, "can't enable CAAM emi slow clock: %d\n", + ret); + goto disable_caam_aclk; + } + /* Get configuration properties from device tree */ /* First, get register page */ ctrl = of_iomap(nprop, 0); if (ctrl == NULL) { dev_err(dev, "caam: of_iomap() failed\n"); - return -ENOMEM; + ret = -ENOMEM; + goto disable_caam_emi_slow; } /* Finding the page size for using the CTPR_MS register */ comp_params = rd_reg32(&ctrl->perfmon.comp_parms_ms); @@ -444,8 +533,9 @@ static int caam_probe(struct platform_device *pdev) * Enable DECO watchdogs and, if this is a PHYS_ADDR_T_64BIT kernel, * long pointers in master configuration register */ - setbits32(&ctrl->mcr, MCFGR_WDENABLE | - (sizeof(dma_addr_t) == sizeof(u64) ? MCFGR_LONG_PTR : 0)); + clrsetbits_32(&ctrl->mcr, MCFGR_AWCACHE_MASK, MCFGR_AWCACHE_CACH | + MCFGR_AWCACHE_BUFF | MCFGR_WDENABLE | + (sizeof(dma_addr_t) == sizeof(u64) ? MCFGR_LONG_PTR : 0)); /* * Read the Compile Time paramters and SCFGR to determine @@ -492,12 +582,11 @@ static int caam_probe(struct platform_device *pdev) of_device_is_compatible(np, "fsl,sec4.0-job-ring")) rspec++; - ctrlpriv->jrpdev = devm_kzalloc(&pdev->dev, - sizeof(struct platform_device *) * rspec, - GFP_KERNEL); + ctrlpriv->jrpdev = devm_kcalloc(&pdev->dev, rspec, + sizeof(*ctrlpriv->jrpdev), GFP_KERNEL); if (ctrlpriv->jrpdev == NULL) { - iounmap(&ctrl); - return -ENOMEM; + ret = -ENOMEM; + goto iounmap_ctrl; } ring = 0; @@ -537,8 +626,8 @@ static int caam_probe(struct platform_device *pdev) /* If no QI and no rings specified, quit and go home */ if ((!ctrlpriv->qi_present) && (!ctrlpriv->total_jobrs)) { dev_err(dev, "no queues configured, terminating\n"); - caam_remove(pdev); - return -ENOMEM; + ret = -ENOMEM; + goto caam_remove; } cha_vid_ls = rd_reg32(&ctrl->perfmon.cha_id_ls); @@ -595,8 +684,7 @@ static int caam_probe(struct platform_device *pdev) } while ((ret == -EAGAIN) && (ent_delay < RTSDCTL_ENT_DLY_MAX)); if (ret) { dev_err(dev, "failed to instantiate RNG"); - caam_remove(pdev); - return ret; + goto caam_remove; } /* * Set handles init'ed by this module as the complement of the @@ -700,6 +788,20 @@ static int caam_probe(struct platform_device *pdev) &ctrlpriv->ctl_tdsk_wrap); #endif return 0; + +caam_remove: + caam_remove(pdev); +iounmap_ctrl: + iounmap(ctrl); +disable_caam_emi_slow: + clk_disable_unprepare(ctrlpriv->caam_emi_slow); +disable_caam_aclk: + clk_disable_unprepare(ctrlpriv->caam_aclk); +disable_caam_mem: + clk_disable_unprepare(ctrlpriv->caam_mem); +disable_caam_ipg: + clk_disable_unprepare(ctrlpriv->caam_ipg); + return ret; } static struct of_device_id caam_match[] = { diff --git a/kernel/drivers/crypto/caam/desc.h b/kernel/drivers/crypto/caam/desc.h index d397ff9d5..1e93c6af2 100644 --- a/kernel/drivers/crypto/caam/desc.h +++ b/kernel/drivers/crypto/caam/desc.h @@ -8,12 +8,29 @@ #ifndef DESC_H #define DESC_H +/* + * 16-byte hardware scatter/gather table + * An 8-byte table exists in the hardware spec, but has never been + * implemented to date. The 8/16 option is selected at RTL-compile-time. + * and this selection is visible in the Compile Time Parameters Register + */ + +#define SEC4_SG_LEN_EXT 0x80000000 /* Entry points to table */ +#define SEC4_SG_LEN_FIN 0x40000000 /* Last ent in table */ +#define SEC4_SG_BPID_MASK 0x000000ff +#define SEC4_SG_BPID_SHIFT 16 +#define SEC4_SG_LEN_MASK 0x3fffffff /* Excludes EXT and FINAL */ +#define SEC4_SG_OFFS_MASK 0x00001fff + struct sec4_sg_entry { +#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_IMX + u32 rsvd1; + dma_addr_t ptr; +#else u64 ptr; -#define SEC4_SG_LEN_FIN 0x40000000 -#define SEC4_SG_LEN_EXT 0x80000000 +#endif /* CONFIG_CRYPTO_DEV_FSL_CAAM_IMX */ u32 len; - u8 reserved; + u8 rsvd2; u8 buf_pool_id; u16 offset; }; @@ -1475,7 +1492,6 @@ struct sec4_sg_entry { #define JUMP_JSL (1 << JUMP_JSL_SHIFT) #define JUMP_TYPE_SHIFT 22 -#define JUMP_TYPE_MASK (0x03 << JUMP_TYPE_SHIFT) #define JUMP_TYPE_LOCAL (0x00 << JUMP_TYPE_SHIFT) #define JUMP_TYPE_NONLOCAL (0x01 << JUMP_TYPE_SHIFT) #define JUMP_TYPE_HALT (0x02 << JUMP_TYPE_SHIFT) diff --git a/kernel/drivers/crypto/caam/desc_constr.h b/kernel/drivers/crypto/caam/desc_constr.h index 9f79fd7bd..98d07de24 100644 --- a/kernel/drivers/crypto/caam/desc_constr.h +++ b/kernel/drivers/crypto/caam/desc_constr.h @@ -367,7 +367,7 @@ do { \ if (upper) \ append_u64(desc, data); \ else \ - append_u32(desc, data); \ + append_u32(desc, lower_32_bits(data)); \ } while (0) #define append_math_add_imm_u64(desc, dest, src0, src1, data) \ diff --git a/kernel/drivers/crypto/caam/intern.h b/kernel/drivers/crypto/caam/intern.h index 89b94cc9e..e2bcacc1a 100644 --- a/kernel/drivers/crypto/caam/intern.h +++ b/kernel/drivers/crypto/caam/intern.h @@ -91,6 +91,11 @@ struct caam_drv_private { Handles of the RNG4 block are initialized by this driver */ + struct clk *caam_ipg; + struct clk *caam_mem; + struct clk *caam_aclk; + struct clk *caam_emi_slow; + /* * debugfs entries for developer view into driver/device * variables at runtime. diff --git a/kernel/drivers/crypto/caam/jr.c b/kernel/drivers/crypto/caam/jr.c index b8b5d47ac..f7e0d8d4c 100644 --- a/kernel/drivers/crypto/caam/jr.c +++ b/kernel/drivers/crypto/caam/jr.c @@ -202,6 +202,13 @@ static void caam_jr_dequeue(unsigned long devarg) userdesc = jrp->entinfo[sw_idx].desc_addr_virt; userstatus = jrp->outring[hw_idx].jrstatus; + /* + * Make sure all information from the job has been obtained + * before telling CAAM that the job has been removed from the + * output ring. + */ + mb(); + /* set done */ wr_reg32(&jrp->rregs->outring_rmvd, 1); @@ -351,12 +358,23 @@ int caam_jr_enqueue(struct device *dev, u32 *desc, jrp->inpring[jrp->inp_ring_write_index] = desc_dma; + /* + * Guarantee that the descriptor's DMA address has been written to + * the next slot in the ring before the write index is updated, since + * other cores may update this index independently. + */ smp_wmb(); jrp->inp_ring_write_index = (jrp->inp_ring_write_index + 1) & (JOBR_DEPTH - 1); jrp->head = (head + 1) & (JOBR_DEPTH - 1); + /* + * Ensure that all job information has been written before + * notifying CAAM that a new job was added to the input ring. + */ + wmb(); + wr_reg32(&jrp->rregs->inpring_jobadd, 1); spin_unlock_bh(&jrp->inplock); @@ -392,18 +410,17 @@ static int caam_jr_init(struct device *dev) goto out_free_irq; error = -ENOMEM; - jrp->inpring = dma_alloc_coherent(dev, sizeof(dma_addr_t) * JOBR_DEPTH, - &inpbusaddr, GFP_KERNEL); + jrp->inpring = dma_alloc_coherent(dev, sizeof(*jrp->inpring) * + JOBR_DEPTH, &inpbusaddr, GFP_KERNEL); if (!jrp->inpring) goto out_free_irq; - jrp->outring = dma_alloc_coherent(dev, sizeof(struct jr_outentry) * + jrp->outring = dma_alloc_coherent(dev, sizeof(*jrp->outring) * JOBR_DEPTH, &outbusaddr, GFP_KERNEL); if (!jrp->outring) goto out_free_inpring; - jrp->entinfo = kzalloc(sizeof(struct caam_jrentry_info) * JOBR_DEPTH, - GFP_KERNEL); + jrp->entinfo = kcalloc(JOBR_DEPTH, sizeof(*jrp->entinfo), GFP_KERNEL); if (!jrp->entinfo) goto out_free_outring; @@ -461,8 +478,7 @@ static int caam_jr_probe(struct platform_device *pdev) int error; jrdev = &pdev->dev; - jrpriv = devm_kmalloc(jrdev, sizeof(struct caam_drv_private_jr), - GFP_KERNEL); + jrpriv = devm_kmalloc(jrdev, sizeof(*jrpriv), GFP_KERNEL); if (!jrpriv) return -ENOMEM; diff --git a/kernel/drivers/crypto/caam/regs.h b/kernel/drivers/crypto/caam/regs.h index 378ddc17f..a8a799756 100644 --- a/kernel/drivers/crypto/caam/regs.h +++ b/kernel/drivers/crypto/caam/regs.h @@ -65,9 +65,31 @@ * */ +#ifdef CONFIG_ARM +/* These are common macros for Power, put here for ARM */ +#define setbits32(_addr, _v) writel((readl(_addr) | (_v)), (_addr)) +#define clrbits32(_addr, _v) writel((readl(_addr) & ~(_v)), (_addr)) + +#define out_arch(type, endian, a, v) __raw_write##type(cpu_to_##endian(v), a) +#define in_arch(type, endian, a) endian##_to_cpu(__raw_read##type(a)) + +#define out_le32(a, v) out_arch(l, le32, a, v) +#define in_le32(a) in_arch(l, le32, a) + +#define out_be32(a, v) out_arch(l, be32, a, v) +#define in_be32(a) in_arch(l, be32, a) + +#define clrsetbits(type, addr, clear, set) \ + out_##type((addr), (in_##type(addr) & ~(clear)) | (set)) + +#define clrsetbits_be32(addr, clear, set) clrsetbits(be32, addr, clear, set) +#define clrsetbits_le32(addr, clear, set) clrsetbits(le32, addr, clear, set) +#endif + #ifdef __BIG_ENDIAN #define wr_reg32(reg, data) out_be32(reg, data) #define rd_reg32(reg) in_be32(reg) +#define clrsetbits_32(addr, clear, set) clrsetbits_be32(addr, clear, set) #ifdef CONFIG_64BIT #define wr_reg64(reg, data) out_be64(reg, data) #define rd_reg64(reg) in_be64(reg) @@ -76,6 +98,7 @@ #ifdef __LITTLE_ENDIAN #define wr_reg32(reg, data) __raw_writel(data, reg) #define rd_reg32(reg) __raw_readl(reg) +#define clrsetbits_32(addr, clear, set) clrsetbits_le32(addr, clear, set) #ifdef CONFIG_64BIT #define wr_reg64(reg, data) __raw_writeq(data, reg) #define rd_reg64(reg) __raw_readq(reg) @@ -83,35 +106,46 @@ #endif #endif -#ifndef CONFIG_64BIT -#ifdef __BIG_ENDIAN -static inline void wr_reg64(u64 __iomem *reg, u64 data) -{ - wr_reg32((u32 __iomem *)reg, (data & 0xffffffff00000000ull) >> 32); - wr_reg32((u32 __iomem *)reg + 1, data & 0x00000000ffffffffull); -} +/* + * The only users of these wr/rd_reg64 functions is the Job Ring (JR). + * The DMA address registers in the JR are handled differently depending on + * platform: + * + * 1. All BE CAAM platforms and i.MX platforms (LE CAAM): + * + * base + 0x0000 : most-significant 32 bits + * base + 0x0004 : least-significant 32 bits + * + * The 32-bit version of this core therefore has to write to base + 0x0004 + * to set the 32-bit wide DMA address. + * + * 2. All other LE CAAM platforms (LS1021A etc.) + * base + 0x0000 : least-significant 32 bits + * base + 0x0004 : most-significant 32 bits + */ -static inline u64 rd_reg64(u64 __iomem *reg) -{ - return (((u64)rd_reg32((u32 __iomem *)reg)) << 32) | - ((u64)rd_reg32((u32 __iomem *)reg + 1)); -} +#ifndef CONFIG_64BIT +#if !defined(CONFIG_CRYPTO_DEV_FSL_CAAM_LE) || \ + defined(CONFIG_CRYPTO_DEV_FSL_CAAM_IMX) +#define REG64_MS32(reg) ((u32 __iomem *)(reg)) +#define REG64_LS32(reg) ((u32 __iomem *)(reg) + 1) #else -#ifdef __LITTLE_ENDIAN +#define REG64_MS32(reg) ((u32 __iomem *)(reg) + 1) +#define REG64_LS32(reg) ((u32 __iomem *)(reg)) +#endif + static inline void wr_reg64(u64 __iomem *reg, u64 data) { - wr_reg32((u32 __iomem *)reg + 1, (data & 0xffffffff00000000ull) >> 32); - wr_reg32((u32 __iomem *)reg, data & 0x00000000ffffffffull); + wr_reg32(REG64_MS32(reg), data >> 32); + wr_reg32(REG64_LS32(reg), data); } static inline u64 rd_reg64(u64 __iomem *reg) { - return (((u64)rd_reg32((u32 __iomem *)reg + 1)) << 32) | - ((u64)rd_reg32((u32 __iomem *)reg)); + return ((u64)rd_reg32(REG64_MS32(reg)) << 32 | + (u64)rd_reg32(REG64_LS32(reg))); } #endif -#endif -#endif /* * jr_outentry @@ -133,18 +167,28 @@ struct jr_outentry { #define CHA_NUM_MS_DECONUM_SHIFT 24 #define CHA_NUM_MS_DECONUM_MASK (0xfull << CHA_NUM_MS_DECONUM_SHIFT) -/* CHA Version IDs */ +/* + * CHA version IDs / instantiation bitfields + * Defined for use with the cha_id fields in perfmon, but the same shift/mask + * selectors can be used to pull out the number of instantiated blocks within + * cha_num fields in perfmon because the locations are the same. + */ #define CHA_ID_LS_AES_SHIFT 0 -#define CHA_ID_LS_AES_MASK (0xfull << CHA_ID_LS_AES_SHIFT) +#define CHA_ID_LS_AES_MASK (0xfull << CHA_ID_LS_AES_SHIFT) +#define CHA_ID_LS_AES_LP (0x3ull << CHA_ID_LS_AES_SHIFT) +#define CHA_ID_LS_AES_HP (0x4ull << CHA_ID_LS_AES_SHIFT) #define CHA_ID_LS_DES_SHIFT 4 -#define CHA_ID_LS_DES_MASK (0xfull << CHA_ID_LS_DES_SHIFT) +#define CHA_ID_LS_DES_MASK (0xfull << CHA_ID_LS_DES_SHIFT) #define CHA_ID_LS_ARC4_SHIFT 8 #define CHA_ID_LS_ARC4_MASK (0xfull << CHA_ID_LS_ARC4_SHIFT) #define CHA_ID_LS_MD_SHIFT 12 #define CHA_ID_LS_MD_MASK (0xfull << CHA_ID_LS_MD_SHIFT) +#define CHA_ID_LS_MD_LP256 (0x0ull << CHA_ID_LS_MD_SHIFT) +#define CHA_ID_LS_MD_LP512 (0x1ull << CHA_ID_LS_MD_SHIFT) +#define CHA_ID_LS_MD_HP (0x2ull << CHA_ID_LS_MD_SHIFT) #define CHA_ID_LS_RNG_SHIFT 16 #define CHA_ID_LS_RNG_MASK (0xfull << CHA_ID_LS_RNG_SHIFT) @@ -395,10 +439,16 @@ struct caam_ctrl { /* AXI read cache control */ #define MCFGR_ARCACHE_SHIFT 12 #define MCFGR_ARCACHE_MASK (0xf << MCFGR_ARCACHE_SHIFT) +#define MCFGR_ARCACHE_BUFF (0x1 << MCFGR_ARCACHE_SHIFT) +#define MCFGR_ARCACHE_CACH (0x2 << MCFGR_ARCACHE_SHIFT) +#define MCFGR_ARCACHE_RALL (0x4 << MCFGR_ARCACHE_SHIFT) /* AXI write cache control */ #define MCFGR_AWCACHE_SHIFT 8 #define MCFGR_AWCACHE_MASK (0xf << MCFGR_AWCACHE_SHIFT) +#define MCFGR_AWCACHE_BUFF (0x1 << MCFGR_AWCACHE_SHIFT) +#define MCFGR_AWCACHE_CACH (0x2 << MCFGR_AWCACHE_SHIFT) +#define MCFGR_AWCACHE_WALL (0x8 << MCFGR_AWCACHE_SHIFT) /* AXI pipeline depth */ #define MCFGR_AXIPIPE_SHIFT 4 diff --git a/kernel/drivers/crypto/caam/sg_sw_sec4.h b/kernel/drivers/crypto/caam/sg_sw_sec4.h index 3b918218a..12ec6616e 100644 --- a/kernel/drivers/crypto/caam/sg_sw_sec4.h +++ b/kernel/drivers/crypto/caam/sg_sw_sec4.h @@ -15,7 +15,6 @@ static inline void dma_to_sec4_sg_one(struct sec4_sg_entry *sec4_sg_ptr, { sec4_sg_ptr->ptr = dma; sec4_sg_ptr->len = len; - sec4_sg_ptr->reserved = 0; sec4_sg_ptr->buf_pool_id = 0; sec4_sg_ptr->offset = offset; #ifdef DEBUG @@ -55,64 +54,28 @@ static inline void sg_to_sec4_sg_last(struct scatterlist *sg, int sg_count, sec4_sg_ptr->len |= SEC4_SG_LEN_FIN; } -/* count number of elements in scatterlist */ -static inline int __sg_count(struct scatterlist *sg_list, int nbytes, - bool *chained) +static inline struct sec4_sg_entry *sg_to_sec4_sg_len( + struct scatterlist *sg, unsigned int total, + struct sec4_sg_entry *sec4_sg_ptr) { - struct scatterlist *sg = sg_list; - int sg_nents = 0; + do { + unsigned int len = min(sg_dma_len(sg), total); - while (nbytes > 0) { - sg_nents++; - nbytes -= sg->length; - if (!sg_is_last(sg) && (sg + 1)->length == 0) - *chained = true; + dma_to_sec4_sg_one(sec4_sg_ptr, sg_dma_address(sg), len, 0); + sec4_sg_ptr++; sg = sg_next(sg); - } - - return sg_nents; + total -= len; + } while (total); + return sec4_sg_ptr - 1; } /* derive number of elements in scatterlist, but return 0 for 1 */ -static inline int sg_count(struct scatterlist *sg_list, int nbytes, - bool *chained) +static inline int sg_count(struct scatterlist *sg_list, int nbytes) { - int sg_nents = __sg_count(sg_list, nbytes, chained); + int sg_nents = sg_nents_for_len(sg_list, nbytes); if (likely(sg_nents == 1)) return 0; return sg_nents; } - -static int dma_map_sg_chained(struct device *dev, struct scatterlist *sg, - unsigned int nents, enum dma_data_direction dir, - bool chained) -{ - if (unlikely(chained)) { - int i; - for (i = 0; i < nents; i++) { - dma_map_sg(dev, sg, 1, dir); - sg = sg_next(sg); - } - } else { - dma_map_sg(dev, sg, nents, dir); - } - return nents; -} - -static int dma_unmap_sg_chained(struct device *dev, struct scatterlist *sg, - unsigned int nents, enum dma_data_direction dir, - bool chained) -{ - if (unlikely(chained)) { - int i; - for (i = 0; i < nents; i++) { - dma_unmap_sg(dev, sg, 1, dir); - sg = sg_next(sg); - } - } else { - dma_unmap_sg(dev, sg, nents, dir); - } - return nents; -} diff --git a/kernel/drivers/crypto/ccp/Kconfig b/kernel/drivers/crypto/ccp/Kconfig index 7639ffc36..3cd848106 100644 --- a/kernel/drivers/crypto/ccp/Kconfig +++ b/kernel/drivers/crypto/ccp/Kconfig @@ -5,20 +5,18 @@ config CRYPTO_DEV_CCP_DD select HW_RANDOM help Provides the interface to use the AMD Cryptographic Coprocessor - which can be used to accelerate or offload encryption operations - such as SHA, AES and more. If you choose 'M' here, this module - will be called ccp. + which can be used to offload encryption operations such as SHA, + AES and more. If you choose 'M' here, this module will be called + ccp. config CRYPTO_DEV_CCP_CRYPTO - tristate "Encryption and hashing acceleration support" + tristate "Encryption and hashing offload support" depends on CRYPTO_DEV_CCP_DD default m - select CRYPTO_ALGAPI select CRYPTO_HASH select CRYPTO_BLKCIPHER select CRYPTO_AUTHENC help Support for using the cryptographic API with the AMD Cryptographic - Coprocessor. This module supports acceleration and offload of SHA - and AES algorithms. If you choose 'M' here, this module will be - called ccp_crypto. + Coprocessor. This module supports offload of SHA and AES algorithms. + If you choose 'M' here, this module will be called ccp_crypto. diff --git a/kernel/drivers/crypto/ccp/ccp-crypto-aes-cmac.c b/kernel/drivers/crypto/ccp/ccp-crypto-aes-cmac.c index ea7e84469..d89f20c04 100644 --- a/kernel/drivers/crypto/ccp/ccp-crypto-aes-cmac.c +++ b/kernel/drivers/crypto/ccp/ccp-crypto-aes-cmac.c @@ -118,10 +118,19 @@ static int ccp_do_cmac_update(struct ahash_request *req, unsigned int nbytes, if (rctx->buf_count) { sg_init_one(&rctx->buf_sg, rctx->buf, rctx->buf_count); sg = ccp_crypto_sg_table_add(&rctx->data_sg, &rctx->buf_sg); + if (!sg) { + ret = -EINVAL; + goto e_free; + } } - if (nbytes) + if (nbytes) { sg = ccp_crypto_sg_table_add(&rctx->data_sg, req->src); + if (!sg) { + ret = -EINVAL; + goto e_free; + } + } if (need_pad) { int pad_length = block_size - (len & (block_size - 1)); @@ -132,6 +141,10 @@ static int ccp_do_cmac_update(struct ahash_request *req, unsigned int nbytes, rctx->pad[0] = 0x80; sg_init_one(&rctx->pad_sg, rctx->pad, pad_length); sg = ccp_crypto_sg_table_add(&rctx->data_sg, &rctx->pad_sg); + if (!sg) { + ret = -EINVAL; + goto e_free; + } } if (sg) { sg_mark_end(sg); @@ -162,6 +175,11 @@ static int ccp_do_cmac_update(struct ahash_request *req, unsigned int nbytes, ret = ccp_crypto_enqueue_request(&req->base, &rctx->cmd); + return ret; + +e_free: + sg_free_table(&rctx->data_sg); + return ret; } diff --git a/kernel/drivers/crypto/ccp/ccp-crypto-main.c b/kernel/drivers/crypto/ccp/ccp-crypto-main.c index bdec01ec6..e0380e59c 100644 --- a/kernel/drivers/crypto/ccp/ccp-crypto-main.c +++ b/kernel/drivers/crypto/ccp/ccp-crypto-main.c @@ -305,14 +305,16 @@ struct scatterlist *ccp_crypto_sg_table_add(struct sg_table *table, for (sg = table->sgl; sg; sg = sg_next(sg)) if (!sg_page(sg)) break; - BUG_ON(!sg); + if (WARN_ON(!sg)) + return NULL; for (; sg && sg_add; sg = sg_next(sg), sg_add = sg_next(sg_add)) { sg_set_page(sg, sg_page(sg_add), sg_add->length, sg_add->offset); sg_last = sg; } - BUG_ON(sg_add); + if (WARN_ON(sg_add)) + return NULL; return sg_last; } diff --git a/kernel/drivers/crypto/ccp/ccp-crypto-sha.c b/kernel/drivers/crypto/ccp/ccp-crypto-sha.c index 507b34e0c..d14b3f28e 100644 --- a/kernel/drivers/crypto/ccp/ccp-crypto-sha.c +++ b/kernel/drivers/crypto/ccp/ccp-crypto-sha.c @@ -107,7 +107,15 @@ static int ccp_do_sha_update(struct ahash_request *req, unsigned int nbytes, sg_init_one(&rctx->buf_sg, rctx->buf, rctx->buf_count); sg = ccp_crypto_sg_table_add(&rctx->data_sg, &rctx->buf_sg); + if (!sg) { + ret = -EINVAL; + goto e_free; + } sg = ccp_crypto_sg_table_add(&rctx->data_sg, req->src); + if (!sg) { + ret = -EINVAL; + goto e_free; + } sg_mark_end(sg); sg = rctx->data_sg.sgl; @@ -141,6 +149,11 @@ static int ccp_do_sha_update(struct ahash_request *req, unsigned int nbytes, ret = ccp_crypto_enqueue_request(&req->base, &rctx->cmd); + return ret; + +e_free: + sg_free_table(&rctx->data_sg); + return ret; } diff --git a/kernel/drivers/crypto/ccp/ccp-ops.c b/kernel/drivers/crypto/ccp/ccp-ops.c index 71f2e3c89..c6e883b29 100644 --- a/kernel/drivers/crypto/ccp/ccp-ops.c +++ b/kernel/drivers/crypto/ccp/ccp-ops.c @@ -52,8 +52,7 @@ struct ccp_dm_workarea { struct ccp_sg_workarea { struct scatterlist *sg; - unsigned int nents; - unsigned int length; + int nents; struct scatterlist *dma_sg; struct device *dma_dev; @@ -496,8 +495,10 @@ static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev, if (!sg) return 0; - wa->nents = sg_nents(sg); - wa->length = sg->length; + wa->nents = sg_nents_for_len(sg, len); + if (wa->nents < 0) + return wa->nents; + wa->bytes_left = len; wa->sg_used = 0; @@ -610,15 +611,16 @@ static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset, 1); } -static void ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa, - struct scatterlist *sg, - unsigned int len, unsigned int se_len, - bool sign_extend) +static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa, + struct scatterlist *sg, + unsigned int len, unsigned int se_len, + bool sign_extend) { unsigned int nbytes, sg_offset, dm_offset, ksb_len, i; u8 buffer[CCP_REVERSE_BUF_SIZE]; - BUG_ON(se_len > sizeof(buffer)); + if (WARN_ON(se_len > sizeof(buffer))) + return -EINVAL; sg_offset = len; dm_offset = 0; @@ -641,6 +643,8 @@ static void ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa, se_len - ksb_len); } } + + return 0; } static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa, @@ -1605,8 +1609,10 @@ static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) if (ret) goto e_ksb; - ccp_reverse_set_dm_area(&exp, rsa->exp, rsa->exp_len, CCP_KSB_BYTES, - false); + ret = ccp_reverse_set_dm_area(&exp, rsa->exp, rsa->exp_len, + CCP_KSB_BYTES, false); + if (ret) + goto e_exp; ret = ccp_copy_to_ksb(cmd_q, &exp, op.jobid, op.ksb_key, CCP_PASSTHRU_BYTESWAP_NOOP); if (ret) { @@ -1622,11 +1628,15 @@ static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) if (ret) goto e_exp; - ccp_reverse_set_dm_area(&src, rsa->mod, rsa->mod_len, CCP_KSB_BYTES, - false); + ret = ccp_reverse_set_dm_area(&src, rsa->mod, rsa->mod_len, + CCP_KSB_BYTES, false); + if (ret) + goto e_src; src.address += o_len; /* Adjust the address for the copy operation */ - ccp_reverse_set_dm_area(&src, rsa->src, rsa->src_len, CCP_KSB_BYTES, - false); + ret = ccp_reverse_set_dm_area(&src, rsa->src, rsa->src_len, + CCP_KSB_BYTES, false); + if (ret) + goto e_src; src.address -= o_len; /* Reset the address to original value */ /* Prepare the output area for the operation */ @@ -1840,21 +1850,27 @@ static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) save = src.address; /* Copy the ECC modulus */ - ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len, - CCP_ECC_OPERAND_SIZE, false); + ret = ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len, + CCP_ECC_OPERAND_SIZE, false); + if (ret) + goto e_src; src.address += CCP_ECC_OPERAND_SIZE; /* Copy the first operand */ - ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_1, - ecc->u.mm.operand_1_len, - CCP_ECC_OPERAND_SIZE, false); + ret = ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_1, + ecc->u.mm.operand_1_len, + CCP_ECC_OPERAND_SIZE, false); + if (ret) + goto e_src; src.address += CCP_ECC_OPERAND_SIZE; if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) { /* Copy the second operand */ - ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_2, - ecc->u.mm.operand_2_len, - CCP_ECC_OPERAND_SIZE, false); + ret = ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_2, + ecc->u.mm.operand_2_len, + CCP_ECC_OPERAND_SIZE, false); + if (ret) + goto e_src; src.address += CCP_ECC_OPERAND_SIZE; } @@ -1959,18 +1975,24 @@ static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) save = src.address; /* Copy the ECC modulus */ - ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len, - CCP_ECC_OPERAND_SIZE, false); + ret = ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len, + CCP_ECC_OPERAND_SIZE, false); + if (ret) + goto e_src; src.address += CCP_ECC_OPERAND_SIZE; /* Copy the first point X and Y coordinate */ - ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.x, - ecc->u.pm.point_1.x_len, - CCP_ECC_OPERAND_SIZE, false); + ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.x, + ecc->u.pm.point_1.x_len, + CCP_ECC_OPERAND_SIZE, false); + if (ret) + goto e_src; src.address += CCP_ECC_OPERAND_SIZE; - ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.y, - ecc->u.pm.point_1.y_len, - CCP_ECC_OPERAND_SIZE, false); + ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.y, + ecc->u.pm.point_1.y_len, + CCP_ECC_OPERAND_SIZE, false); + if (ret) + goto e_src; src.address += CCP_ECC_OPERAND_SIZE; /* Set the first point Z coordianate to 1 */ @@ -1979,13 +2001,17 @@ static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) { /* Copy the second point X and Y coordinate */ - ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.x, - ecc->u.pm.point_2.x_len, - CCP_ECC_OPERAND_SIZE, false); + ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.x, + ecc->u.pm.point_2.x_len, + CCP_ECC_OPERAND_SIZE, false); + if (ret) + goto e_src; src.address += CCP_ECC_OPERAND_SIZE; - ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.y, - ecc->u.pm.point_2.y_len, - CCP_ECC_OPERAND_SIZE, false); + ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.y, + ecc->u.pm.point_2.y_len, + CCP_ECC_OPERAND_SIZE, false); + if (ret) + goto e_src; src.address += CCP_ECC_OPERAND_SIZE; /* Set the second point Z coordianate to 1 */ @@ -1993,16 +2019,21 @@ static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) src.address += CCP_ECC_OPERAND_SIZE; } else { /* Copy the Domain "a" parameter */ - ccp_reverse_set_dm_area(&src, ecc->u.pm.domain_a, - ecc->u.pm.domain_a_len, - CCP_ECC_OPERAND_SIZE, false); + ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.domain_a, + ecc->u.pm.domain_a_len, + CCP_ECC_OPERAND_SIZE, false); + if (ret) + goto e_src; src.address += CCP_ECC_OPERAND_SIZE; if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) { /* Copy the scalar value */ - ccp_reverse_set_dm_area(&src, ecc->u.pm.scalar, - ecc->u.pm.scalar_len, - CCP_ECC_OPERAND_SIZE, false); + ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.scalar, + ecc->u.pm.scalar_len, + CCP_ECC_OPERAND_SIZE, + false); + if (ret) + goto e_src; src.address += CCP_ECC_OPERAND_SIZE; } } diff --git a/kernel/drivers/crypto/ccp/ccp-pci.c b/kernel/drivers/crypto/ccp/ccp-pci.c index af190d479..6ade02f04 100644 --- a/kernel/drivers/crypto/ccp/ccp-pci.c +++ b/kernel/drivers/crypto/ccp/ccp-pci.c @@ -319,7 +319,7 @@ static const struct pci_device_id ccp_pci_table[] = { MODULE_DEVICE_TABLE(pci, ccp_pci_table); static struct pci_driver ccp_pci_driver = { - .name = "AMD Cryptographic Coprocessor", + .name = "ccp", .id_table = ccp_pci_table, .probe = ccp_pci_probe, .remove = ccp_pci_remove, diff --git a/kernel/drivers/crypto/ccp/ccp-platform.c b/kernel/drivers/crypto/ccp/ccp-platform.c index b1c20b2b5..01b50cb4c 100644 --- a/kernel/drivers/crypto/ccp/ccp-platform.c +++ b/kernel/drivers/crypto/ccp/ccp-platform.c @@ -29,7 +29,6 @@ #include "ccp-dev.h" struct ccp_platform { - int use_acpi; int coherent; }; @@ -90,64 +89,12 @@ static struct resource *ccp_find_mmio_area(struct ccp_device *ccp) return NULL; } -#ifdef CONFIG_ACPI -static int ccp_acpi_support(struct ccp_device *ccp) -{ - struct ccp_platform *ccp_platform = ccp->dev_specific; - struct acpi_device *adev = ACPI_COMPANION(ccp->dev); - acpi_handle handle; - acpi_status status; - unsigned long long data; - int cca; - - /* Retrieve the device cache coherency value */ - handle = adev->handle; - do { - status = acpi_evaluate_integer(handle, "_CCA", NULL, &data); - if (!ACPI_FAILURE(status)) { - cca = data; - break; - } - } while (!ACPI_FAILURE(status)); - - if (ACPI_FAILURE(status)) { - dev_err(ccp->dev, "error obtaining acpi coherency value\n"); - return -EINVAL; - } - - ccp_platform->coherent = !!cca; - - return 0; -} -#else /* CONFIG_ACPI */ -static int ccp_acpi_support(struct ccp_device *ccp) -{ - return -EINVAL; -} -#endif - -#ifdef CONFIG_OF -static int ccp_of_support(struct ccp_device *ccp) -{ - struct ccp_platform *ccp_platform = ccp->dev_specific; - - ccp_platform->coherent = of_dma_is_coherent(ccp->dev->of_node); - - return 0; -} -#else -static int ccp_of_support(struct ccp_device *ccp) -{ - return -EINVAL; -} -#endif - static int ccp_platform_probe(struct platform_device *pdev) { struct ccp_device *ccp; struct ccp_platform *ccp_platform; struct device *dev = &pdev->dev; - struct acpi_device *adev = ACPI_COMPANION(dev); + enum dev_dma_attr attr; struct resource *ior; int ret; @@ -164,8 +111,6 @@ static int ccp_platform_probe(struct platform_device *pdev) ccp->get_irq = ccp_get_irqs; ccp->free_irq = ccp_free_irqs; - ccp_platform->use_acpi = (!adev || acpi_disabled) ? 0 : 1; - ior = ccp_find_mmio_area(ccp); ccp->io_map = devm_ioremap_resource(dev, ior); if (IS_ERR(ccp->io_map)) { @@ -174,26 +119,24 @@ static int ccp_platform_probe(struct platform_device *pdev) } ccp->io_regs = ccp->io_map; - if (!dev->dma_mask) - dev->dma_mask = &dev->coherent_dma_mask; - ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48)); - if (ret) { - dev_err(dev, "dma_set_mask_and_coherent failed (%d)\n", ret); + attr = device_get_dma_attr(dev); + if (attr == DEV_DMA_NOT_SUPPORTED) { + dev_err(dev, "DMA is not supported"); goto e_err; } - if (ccp_platform->use_acpi) - ret = ccp_acpi_support(ccp); - else - ret = ccp_of_support(ccp); - if (ret) - goto e_err; - + ccp_platform->coherent = (attr == DEV_DMA_COHERENT); if (ccp_platform->coherent) ccp->axcache = CACHE_WB_NO_ALLOC; else ccp->axcache = CACHE_NONE; + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48)); + if (ret) { + dev_err(dev, "dma_set_mask_and_coherent failed (%d)\n", ret); + goto e_err; + } + dev_set_drvdata(dev, ccp); ret = ccp_init(ccp); @@ -276,6 +219,7 @@ static const struct acpi_device_id ccp_acpi_match[] = { { "AMDI0C00", 0 }, { }, }; +MODULE_DEVICE_TABLE(acpi, ccp_acpi_match); #endif #ifdef CONFIG_OF @@ -283,11 +227,12 @@ static const struct of_device_id ccp_of_match[] = { { .compatible = "amd,ccp-seattle-v1a" }, { }, }; +MODULE_DEVICE_TABLE(of, ccp_of_match); #endif static struct platform_driver ccp_platform_driver = { .driver = { - .name = "AMD Cryptographic Coprocessor", + .name = "ccp", #ifdef CONFIG_ACPI .acpi_match_table = ccp_acpi_match, #endif diff --git a/kernel/drivers/crypto/hifn_795x.c b/kernel/drivers/crypto/hifn_795x.c index 8d2a77284..ca5c71ab4 100644 --- a/kernel/drivers/crypto/hifn_795x.c +++ b/kernel/drivers/crypto/hifn_795x.c @@ -36,8 +36,6 @@ #include #include -#include - //#define HIFN_DEBUG #ifdef HIFN_DEBUG diff --git a/kernel/drivers/crypto/img-hash.c b/kernel/drivers/crypto/img-hash.c index ad47d0d61..68e8aa90f 100644 --- a/kernel/drivers/crypto/img-hash.c +++ b/kernel/drivers/crypto/img-hash.c @@ -334,7 +334,7 @@ static int img_hash_dma_init(struct img_hash_dev *hdev) hdev->dma_lch = dma_request_slave_channel(hdev->dev, "tx"); if (!hdev->dma_lch) { - dev_err(hdev->dev, "Couldn't aquire a slave DMA channel.\n"); + dev_err(hdev->dev, "Couldn't acquire a slave DMA channel.\n"); return -EBUSY; } dma_conf.direction = DMA_MEM_TO_DEV; diff --git a/kernel/drivers/crypto/ixp4xx_crypto.c b/kernel/drivers/crypto/ixp4xx_crypto.c index ede9e9e3c..8f2790353 100644 --- a/kernel/drivers/crypto/ixp4xx_crypto.c +++ b/kernel/drivers/crypto/ixp4xx_crypto.c @@ -25,7 +25,7 @@ #include #include #include -#include +#include #include #include @@ -156,7 +156,8 @@ struct ablk_ctx { }; struct aead_ctx { - struct buffer_desc *buffer; + struct buffer_desc *src; + struct buffer_desc *dst; struct scatterlist ivlist; /* used when the hmac is not on one sg entry */ u8 *hmac_virt; @@ -198,6 +199,15 @@ struct ixp_alg { int registered; }; +struct ixp_aead_alg { + struct aead_alg crypto; + const struct ix_hash_algo *hash; + u32 cfg_enc; + u32 cfg_dec; + + int registered; +}; + static const struct ix_hash_algo hash_alg_md5 = { .cfgword = 0xAA010004, .icv = "\x01\x23\x45\x67\x89\xAB\xCD\xEF" @@ -339,11 +349,11 @@ static void finish_scattered_hmac(struct crypt_ctl *crypt) struct aead_ctx *req_ctx = aead_request_ctx(req); struct crypto_aead *tfm = crypto_aead_reqtfm(req); int authsize = crypto_aead_authsize(tfm); - int decryptlen = req->cryptlen - authsize; + int decryptlen = req->assoclen + req->cryptlen - authsize; if (req_ctx->encrypt) { scatterwalk_map_and_copy(req_ctx->hmac_virt, - req->src, decryptlen, authsize, 1); + req->dst, decryptlen, authsize, 1); } dma_pool_free(buffer_pool, req_ctx->hmac_virt, crypt->icv_rev_aes); } @@ -364,7 +374,8 @@ static void one_packet(dma_addr_t phys) struct aead_request *req = crypt->data.aead_req; struct aead_ctx *req_ctx = aead_request_ctx(req); - free_buf_chain(dev, req_ctx->buffer, crypt->src_buf); + free_buf_chain(dev, req_ctx->src, crypt->src_buf); + free_buf_chain(dev, req_ctx->dst, crypt->dst_buf); if (req_ctx->hmac_virt) { finish_scattered_hmac(crypt); } @@ -573,10 +584,10 @@ static int init_tfm_ablk(struct crypto_tfm *tfm) return init_tfm(tfm); } -static int init_tfm_aead(struct crypto_tfm *tfm) +static int init_tfm_aead(struct crypto_aead *tfm) { - tfm->crt_aead.reqsize = sizeof(struct aead_ctx); - return init_tfm(tfm); + crypto_aead_set_reqsize(tfm, sizeof(struct aead_ctx)); + return init_tfm(crypto_aead_tfm(tfm)); } static void exit_tfm(struct crypto_tfm *tfm) @@ -586,6 +597,11 @@ static void exit_tfm(struct crypto_tfm *tfm) free_sa_dir(&ctx->decrypt); } +static void exit_tfm_aead(struct crypto_aead *tfm) +{ + exit_tfm(crypto_aead_tfm(tfm)); +} + static int register_chain_var(struct crypto_tfm *tfm, u8 xpad, u32 target, int init_len, u32 ctx_addr, const u8 *key, int key_len) { @@ -968,24 +984,6 @@ static int ablk_rfc3686_crypt(struct ablkcipher_request *req) return ret; } -static int hmac_inconsistent(struct scatterlist *sg, unsigned start, - unsigned int nbytes) -{ - int offset = 0; - - if (!nbytes) - return 0; - - for (;;) { - if (start < offset + sg->length) - break; - - offset += sg->length; - sg = sg_next(sg); - } - return (start + nbytes > offset + sg->length); -} - static int aead_perform(struct aead_request *req, int encrypt, int cryptoffset, int eff_cryptlen, u8 *iv) { @@ -1001,6 +999,8 @@ static int aead_perform(struct aead_request *req, int encrypt, struct device *dev = &pdev->dev; gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; + enum dma_data_direction src_direction = DMA_BIDIRECTIONAL; + unsigned int lastlen; if (qmgr_stat_full(SEND_QID)) return -EAGAIN; @@ -1029,35 +1029,55 @@ static int aead_perform(struct aead_request *req, int encrypt, crypt->crypt_len = eff_cryptlen; crypt->auth_offs = 0; - crypt->auth_len = req->assoclen + ivsize + cryptlen; + crypt->auth_len = req->assoclen + cryptlen; BUG_ON(ivsize && !req->iv); memcpy(crypt->iv, req->iv, ivsize); + req_ctx->dst = NULL; + if (req->src != req->dst) { - BUG(); /* -ENOTSUP because of my laziness */ + struct buffer_desc dst_hook; + + crypt->mode |= NPE_OP_NOT_IN_PLACE; + src_direction = DMA_TO_DEVICE; + + buf = chainup_buffers(dev, req->dst, crypt->auth_len, + &dst_hook, flags, DMA_FROM_DEVICE); + req_ctx->dst = dst_hook.next; + crypt->dst_buf = dst_hook.phys_next; + + if (!buf) + goto free_buf_dst; + + if (encrypt) { + lastlen = buf->buf_len; + if (lastlen >= authsize) + crypt->icv_rev_aes = buf->phys_addr + + buf->buf_len - authsize; + } } - /* ASSOC data */ - buf = chainup_buffers(dev, req->assoc, req->assoclen, &src_hook, - flags, DMA_TO_DEVICE); - req_ctx->buffer = src_hook.next; + buf = chainup_buffers(dev, req->src, crypt->auth_len, + &src_hook, flags, src_direction); + req_ctx->src = src_hook.next; crypt->src_buf = src_hook.phys_next; if (!buf) - goto out; - /* IV */ - sg_init_table(&req_ctx->ivlist, 1); - sg_set_buf(&req_ctx->ivlist, iv, ivsize); - buf = chainup_buffers(dev, &req_ctx->ivlist, ivsize, buf, flags, - DMA_BIDIRECTIONAL); - if (!buf) - goto free_chain; - if (unlikely(hmac_inconsistent(req->src, cryptlen, authsize))) { + goto free_buf_src; + + if (!encrypt || !req_ctx->dst) { + lastlen = buf->buf_len; + if (lastlen >= authsize) + crypt->icv_rev_aes = buf->phys_addr + + buf->buf_len - authsize; + } + + if (unlikely(lastlen < authsize)) { /* The 12 hmac bytes are scattered, * we need to copy them into a safe buffer */ req_ctx->hmac_virt = dma_pool_alloc(buffer_pool, flags, &crypt->icv_rev_aes); if (unlikely(!req_ctx->hmac_virt)) - goto free_chain; + goto free_buf_src; if (!encrypt) { scatterwalk_map_and_copy(req_ctx->hmac_virt, req->src, cryptlen, authsize, 0); @@ -1066,27 +1086,16 @@ static int aead_perform(struct aead_request *req, int encrypt, } else { req_ctx->hmac_virt = NULL; } - /* Crypt */ - buf = chainup_buffers(dev, req->src, cryptlen + authsize, buf, flags, - DMA_BIDIRECTIONAL); - if (!buf) - goto free_hmac_virt; - if (!req_ctx->hmac_virt) { - crypt->icv_rev_aes = buf->phys_addr + buf->buf_len - authsize; - } crypt->ctl_flags |= CTL_FLAG_PERFORM_AEAD; qmgr_put_entry(SEND_QID, crypt_virt2phys(crypt)); BUG_ON(qmgr_stat_overflow(SEND_QID)); return -EINPROGRESS; -free_hmac_virt: - if (req_ctx->hmac_virt) { - dma_pool_free(buffer_pool, req_ctx->hmac_virt, - crypt->icv_rev_aes); - } -free_chain: - free_buf_chain(dev, req_ctx->buffer, crypt->src_buf); -out: + +free_buf_src: + free_buf_chain(dev, req_ctx->src, crypt->src_buf); +free_buf_dst: + free_buf_chain(dev, req_ctx->dst, crypt->dst_buf); crypt->ctl_flags = CTL_FLAG_UNUSED; return -ENOMEM; } @@ -1095,7 +1104,7 @@ static int aead_setup(struct crypto_aead *tfm, unsigned int authsize) { struct ixp_ctx *ctx = crypto_aead_ctx(tfm); u32 *flags = &tfm->base.crt_flags; - unsigned digest_len = crypto_aead_alg(tfm)->maxauthsize; + unsigned digest_len = crypto_aead_maxauthsize(tfm); int ret; if (!ctx->enckey_len && !ctx->authkey_len) @@ -1137,7 +1146,7 @@ out: static int aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize) { - int max = crypto_aead_alg(tfm)->maxauthsize >> 2; + int max = crypto_aead_maxauthsize(tfm) >> 2; if ((authsize>>2) < 1 || (authsize>>2) > max || (authsize & 3)) return -EINVAL; @@ -1172,40 +1181,12 @@ badkey: static int aead_encrypt(struct aead_request *req) { - unsigned ivsize = crypto_aead_ivsize(crypto_aead_reqtfm(req)); - return aead_perform(req, 1, req->assoclen + ivsize, - req->cryptlen, req->iv); + return aead_perform(req, 1, req->assoclen, req->cryptlen, req->iv); } static int aead_decrypt(struct aead_request *req) { - unsigned ivsize = crypto_aead_ivsize(crypto_aead_reqtfm(req)); - return aead_perform(req, 0, req->assoclen + ivsize, - req->cryptlen, req->iv); -} - -static int aead_givencrypt(struct aead_givcrypt_request *req) -{ - struct crypto_aead *tfm = aead_givcrypt_reqtfm(req); - struct ixp_ctx *ctx = crypto_aead_ctx(tfm); - unsigned len, ivsize = crypto_aead_ivsize(tfm); - __be64 seq; - - /* copied from eseqiv.c */ - if (!ctx->salted) { - get_random_bytes(ctx->salt, ivsize); - ctx->salted = 1; - } - memcpy(req->areq.iv, ctx->salt, ivsize); - len = ivsize; - if (ivsize > sizeof(u64)) { - memset(req->giv, 0, ivsize - sizeof(u64)); - len = sizeof(u64); - } - seq = cpu_to_be64(req->seq); - memcpy(req->giv + ivsize - len, &seq, len); - return aead_perform(&req->areq, 1, req->areq.assoclen, - req->areq.cryptlen +ivsize, req->giv); + return aead_perform(req, 0, req->assoclen, req->cryptlen, req->iv); } static struct ixp_alg ixp4xx_algos[] = { @@ -1318,80 +1299,77 @@ static struct ixp_alg ixp4xx_algos[] = { }, .cfg_enc = CIPH_ENCR | MOD_AES | MOD_CTR, .cfg_dec = CIPH_ENCR | MOD_AES | MOD_CTR, -}, { +} }; + +static struct ixp_aead_alg ixp4xx_aeads[] = { +{ .crypto = { - .cra_name = "authenc(hmac(md5),cbc(des))", - .cra_blocksize = DES_BLOCK_SIZE, - .cra_u = { .aead = { - .ivsize = DES_BLOCK_SIZE, - .maxauthsize = MD5_DIGEST_SIZE, - } - } + .base = { + .cra_name = "authenc(hmac(md5),cbc(des))", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, }, .hash = &hash_alg_md5, .cfg_enc = CIPH_ENCR | MOD_DES | MOD_CBC_ENC | KEYLEN_192, .cfg_dec = CIPH_DECR | MOD_DES | MOD_CBC_DEC | KEYLEN_192, }, { .crypto = { - .cra_name = "authenc(hmac(md5),cbc(des3_ede))", - .cra_blocksize = DES3_EDE_BLOCK_SIZE, - .cra_u = { .aead = { - .ivsize = DES3_EDE_BLOCK_SIZE, - .maxauthsize = MD5_DIGEST_SIZE, - } - } + .base = { + .cra_name = "authenc(hmac(md5),cbc(des3_ede))", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, }, .hash = &hash_alg_md5, .cfg_enc = CIPH_ENCR | MOD_3DES | MOD_CBC_ENC | KEYLEN_192, .cfg_dec = CIPH_DECR | MOD_3DES | MOD_CBC_DEC | KEYLEN_192, }, { .crypto = { - .cra_name = "authenc(hmac(sha1),cbc(des))", - .cra_blocksize = DES_BLOCK_SIZE, - .cra_u = { .aead = { + .base = { + .cra_name = "authenc(hmac(sha1),cbc(des))", + .cra_blocksize = DES_BLOCK_SIZE, + }, .ivsize = DES_BLOCK_SIZE, .maxauthsize = SHA1_DIGEST_SIZE, - } - } }, .hash = &hash_alg_sha1, .cfg_enc = CIPH_ENCR | MOD_DES | MOD_CBC_ENC | KEYLEN_192, .cfg_dec = CIPH_DECR | MOD_DES | MOD_CBC_DEC | KEYLEN_192, }, { .crypto = { - .cra_name = "authenc(hmac(sha1),cbc(des3_ede))", - .cra_blocksize = DES3_EDE_BLOCK_SIZE, - .cra_u = { .aead = { - .ivsize = DES3_EDE_BLOCK_SIZE, - .maxauthsize = SHA1_DIGEST_SIZE, - } - } + .base = { + .cra_name = "authenc(hmac(sha1),cbc(des3_ede))", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, }, .hash = &hash_alg_sha1, .cfg_enc = CIPH_ENCR | MOD_3DES | MOD_CBC_ENC | KEYLEN_192, .cfg_dec = CIPH_DECR | MOD_3DES | MOD_CBC_DEC | KEYLEN_192, }, { .crypto = { - .cra_name = "authenc(hmac(md5),cbc(aes))", - .cra_blocksize = AES_BLOCK_SIZE, - .cra_u = { .aead = { - .ivsize = AES_BLOCK_SIZE, - .maxauthsize = MD5_DIGEST_SIZE, - } - } + .base = { + .cra_name = "authenc(hmac(md5),cbc(aes))", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, }, .hash = &hash_alg_md5, .cfg_enc = CIPH_ENCR | MOD_AES | MOD_CBC_ENC, .cfg_dec = CIPH_DECR | MOD_AES | MOD_CBC_DEC, }, { .crypto = { - .cra_name = "authenc(hmac(sha1),cbc(aes))", - .cra_blocksize = AES_BLOCK_SIZE, - .cra_u = { .aead = { - .ivsize = AES_BLOCK_SIZE, - .maxauthsize = SHA1_DIGEST_SIZE, - } - } + .base = { + .cra_name = "authenc(hmac(sha1),cbc(aes))", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, }, .hash = &hash_alg_sha1, .cfg_enc = CIPH_ENCR | MOD_AES | MOD_CBC_ENC, @@ -1435,32 +1413,20 @@ static int __init ixp_module_init(void) if (!support_aes && (ixp4xx_algos[i].cfg_enc & MOD_AES)) { continue; } - if (!ixp4xx_algos[i].hash) { - /* block ciphers */ - cra->cra_type = &crypto_ablkcipher_type; - cra->cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | - CRYPTO_ALG_KERN_DRIVER_ONLY | - CRYPTO_ALG_ASYNC; - if (!cra->cra_ablkcipher.setkey) - cra->cra_ablkcipher.setkey = ablk_setkey; - if (!cra->cra_ablkcipher.encrypt) - cra->cra_ablkcipher.encrypt = ablk_encrypt; - if (!cra->cra_ablkcipher.decrypt) - cra->cra_ablkcipher.decrypt = ablk_decrypt; - cra->cra_init = init_tfm_ablk; - } else { - /* authenc */ - cra->cra_type = &crypto_aead_type; - cra->cra_flags = CRYPTO_ALG_TYPE_AEAD | - CRYPTO_ALG_KERN_DRIVER_ONLY | - CRYPTO_ALG_ASYNC; - cra->cra_aead.setkey = aead_setkey; - cra->cra_aead.setauthsize = aead_setauthsize; - cra->cra_aead.encrypt = aead_encrypt; - cra->cra_aead.decrypt = aead_decrypt; - cra->cra_aead.givencrypt = aead_givencrypt; - cra->cra_init = init_tfm_aead; - } + + /* block ciphers */ + cra->cra_type = &crypto_ablkcipher_type; + cra->cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC; + if (!cra->cra_ablkcipher.setkey) + cra->cra_ablkcipher.setkey = ablk_setkey; + if (!cra->cra_ablkcipher.encrypt) + cra->cra_ablkcipher.encrypt = ablk_encrypt; + if (!cra->cra_ablkcipher.decrypt) + cra->cra_ablkcipher.decrypt = ablk_decrypt; + cra->cra_init = init_tfm_ablk; + cra->cra_ctxsize = sizeof(struct ixp_ctx); cra->cra_module = THIS_MODULE; cra->cra_alignmask = 3; @@ -1472,6 +1438,38 @@ static int __init ixp_module_init(void) else ixp4xx_algos[i].registered = 1; } + + for (i = 0; i < ARRAY_SIZE(ixp4xx_aeads); i++) { + struct aead_alg *cra = &ixp4xx_aeads[i].crypto; + + if (snprintf(cra->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, + "%s"IXP_POSTFIX, cra->base.cra_name) >= + CRYPTO_MAX_ALG_NAME) + continue; + if (!support_aes && (ixp4xx_algos[i].cfg_enc & MOD_AES)) + continue; + + /* authenc */ + cra->base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC; + cra->setkey = aead_setkey; + cra->setauthsize = aead_setauthsize; + cra->encrypt = aead_encrypt; + cra->decrypt = aead_decrypt; + cra->init = init_tfm_aead; + cra->exit = exit_tfm_aead; + + cra->base.cra_ctxsize = sizeof(struct ixp_ctx); + cra->base.cra_module = THIS_MODULE; + cra->base.cra_alignmask = 3; + cra->base.cra_priority = 300; + + if (crypto_register_aead(cra)) + printk(KERN_ERR "Failed to register '%s'\n", + cra->base.cra_driver_name); + else + ixp4xx_aeads[i].registered = 1; + } return 0; } @@ -1480,6 +1478,11 @@ static void __exit ixp_module_exit(void) int num = ARRAY_SIZE(ixp4xx_algos); int i; + for (i = 0; i < ARRAY_SIZE(ixp4xx_aeads); i++) { + if (ixp4xx_aeads[i].registered) + crypto_unregister_aead(&ixp4xx_aeads[i].crypto); + } + for (i=0; i< num; i++) { if (ixp4xx_algos[i].registered) crypto_unregister_alg(&ixp4xx_algos[i].crypto); diff --git a/kernel/drivers/crypto/marvell/Makefile b/kernel/drivers/crypto/marvell/Makefile new file mode 100644 index 000000000..0c12b1357 --- /dev/null +++ b/kernel/drivers/crypto/marvell/Makefile @@ -0,0 +1,2 @@ +obj-$(CONFIG_CRYPTO_DEV_MARVELL_CESA) += marvell-cesa.o +marvell-cesa-objs := cesa.o cipher.o hash.o tdma.o diff --git a/kernel/drivers/crypto/marvell/cesa.c b/kernel/drivers/crypto/marvell/cesa.c new file mode 100644 index 000000000..c0656e7f3 --- /dev/null +++ b/kernel/drivers/crypto/marvell/cesa.c @@ -0,0 +1,546 @@ +/* + * Support for Marvell's Cryptographic Engine and Security Accelerator (CESA) + * that can be found on the following platform: Orion, Kirkwood, Armada. This + * driver supports the TDMA engine on platforms on which it is available. + * + * Author: Boris Brezillon + * Author: Arnaud Ebalard + * + * This work is based on an initial version written by + * Sebastian Andrzej Siewior < sebastian at breakpoint dot cc > + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cesa.h" + +static int allhwsupport = !IS_ENABLED(CONFIG_CRYPTO_DEV_MV_CESA); +module_param_named(allhwsupport, allhwsupport, int, 0444); +MODULE_PARM_DESC(allhwsupport, "Enable support for all hardware (even it if overlaps with the mv_cesa driver)"); + +struct mv_cesa_dev *cesa_dev; + +static void mv_cesa_dequeue_req_unlocked(struct mv_cesa_engine *engine) +{ + struct crypto_async_request *req, *backlog; + struct mv_cesa_ctx *ctx; + + spin_lock_bh(&cesa_dev->lock); + backlog = crypto_get_backlog(&cesa_dev->queue); + req = crypto_dequeue_request(&cesa_dev->queue); + engine->req = req; + spin_unlock_bh(&cesa_dev->lock); + + if (!req) + return; + + if (backlog) + backlog->complete(backlog, -EINPROGRESS); + + ctx = crypto_tfm_ctx(req->tfm); + ctx->ops->prepare(req, engine); + ctx->ops->step(req); +} + +static irqreturn_t mv_cesa_int(int irq, void *priv) +{ + struct mv_cesa_engine *engine = priv; + struct crypto_async_request *req; + struct mv_cesa_ctx *ctx; + u32 status, mask; + irqreturn_t ret = IRQ_NONE; + + while (true) { + int res; + + mask = mv_cesa_get_int_mask(engine); + status = readl(engine->regs + CESA_SA_INT_STATUS); + + if (!(status & mask)) + break; + + /* + * TODO: avoid clearing the FPGA_INT_STATUS if this not + * relevant on some platforms. + */ + writel(~status, engine->regs + CESA_SA_FPGA_INT_STATUS); + writel(~status, engine->regs + CESA_SA_INT_STATUS); + + ret = IRQ_HANDLED; + spin_lock_bh(&engine->lock); + req = engine->req; + spin_unlock_bh(&engine->lock); + if (req) { + ctx = crypto_tfm_ctx(req->tfm); + res = ctx->ops->process(req, status & mask); + if (res != -EINPROGRESS) { + spin_lock_bh(&engine->lock); + engine->req = NULL; + mv_cesa_dequeue_req_unlocked(engine); + spin_unlock_bh(&engine->lock); + ctx->ops->cleanup(req); + local_bh_disable(); + req->complete(req, res); + local_bh_enable(); + } else { + ctx->ops->step(req); + } + } + } + + return ret; +} + +int mv_cesa_queue_req(struct crypto_async_request *req) +{ + int ret; + int i; + + spin_lock_bh(&cesa_dev->lock); + ret = crypto_enqueue_request(&cesa_dev->queue, req); + spin_unlock_bh(&cesa_dev->lock); + + if (ret != -EINPROGRESS) + return ret; + + for (i = 0; i < cesa_dev->caps->nengines; i++) { + spin_lock_bh(&cesa_dev->engines[i].lock); + if (!cesa_dev->engines[i].req) + mv_cesa_dequeue_req_unlocked(&cesa_dev->engines[i]); + spin_unlock_bh(&cesa_dev->engines[i].lock); + } + + return -EINPROGRESS; +} + +static int mv_cesa_add_algs(struct mv_cesa_dev *cesa) +{ + int ret; + int i, j; + + for (i = 0; i < cesa->caps->ncipher_algs; i++) { + ret = crypto_register_alg(cesa->caps->cipher_algs[i]); + if (ret) + goto err_unregister_crypto; + } + + for (i = 0; i < cesa->caps->nahash_algs; i++) { + ret = crypto_register_ahash(cesa->caps->ahash_algs[i]); + if (ret) + goto err_unregister_ahash; + } + + return 0; + +err_unregister_ahash: + for (j = 0; j < i; j++) + crypto_unregister_ahash(cesa->caps->ahash_algs[j]); + i = cesa->caps->ncipher_algs; + +err_unregister_crypto: + for (j = 0; j < i; j++) + crypto_unregister_alg(cesa->caps->cipher_algs[j]); + + return ret; +} + +static void mv_cesa_remove_algs(struct mv_cesa_dev *cesa) +{ + int i; + + for (i = 0; i < cesa->caps->nahash_algs; i++) + crypto_unregister_ahash(cesa->caps->ahash_algs[i]); + + for (i = 0; i < cesa->caps->ncipher_algs; i++) + crypto_unregister_alg(cesa->caps->cipher_algs[i]); +} + +static struct crypto_alg *orion_cipher_algs[] = { + &mv_cesa_ecb_des_alg, + &mv_cesa_cbc_des_alg, + &mv_cesa_ecb_des3_ede_alg, + &mv_cesa_cbc_des3_ede_alg, + &mv_cesa_ecb_aes_alg, + &mv_cesa_cbc_aes_alg, +}; + +static struct ahash_alg *orion_ahash_algs[] = { + &mv_md5_alg, + &mv_sha1_alg, + &mv_ahmac_md5_alg, + &mv_ahmac_sha1_alg, +}; + +static struct crypto_alg *armada_370_cipher_algs[] = { + &mv_cesa_ecb_des_alg, + &mv_cesa_cbc_des_alg, + &mv_cesa_ecb_des3_ede_alg, + &mv_cesa_cbc_des3_ede_alg, + &mv_cesa_ecb_aes_alg, + &mv_cesa_cbc_aes_alg, +}; + +static struct ahash_alg *armada_370_ahash_algs[] = { + &mv_md5_alg, + &mv_sha1_alg, + &mv_sha256_alg, + &mv_ahmac_md5_alg, + &mv_ahmac_sha1_alg, + &mv_ahmac_sha256_alg, +}; + +static const struct mv_cesa_caps orion_caps = { + .nengines = 1, + .cipher_algs = orion_cipher_algs, + .ncipher_algs = ARRAY_SIZE(orion_cipher_algs), + .ahash_algs = orion_ahash_algs, + .nahash_algs = ARRAY_SIZE(orion_ahash_algs), + .has_tdma = false, +}; + +static const struct mv_cesa_caps kirkwood_caps = { + .nengines = 1, + .cipher_algs = orion_cipher_algs, + .ncipher_algs = ARRAY_SIZE(orion_cipher_algs), + .ahash_algs = orion_ahash_algs, + .nahash_algs = ARRAY_SIZE(orion_ahash_algs), + .has_tdma = true, +}; + +static const struct mv_cesa_caps armada_370_caps = { + .nengines = 1, + .cipher_algs = armada_370_cipher_algs, + .ncipher_algs = ARRAY_SIZE(armada_370_cipher_algs), + .ahash_algs = armada_370_ahash_algs, + .nahash_algs = ARRAY_SIZE(armada_370_ahash_algs), + .has_tdma = true, +}; + +static const struct mv_cesa_caps armada_xp_caps = { + .nengines = 2, + .cipher_algs = armada_370_cipher_algs, + .ncipher_algs = ARRAY_SIZE(armada_370_cipher_algs), + .ahash_algs = armada_370_ahash_algs, + .nahash_algs = ARRAY_SIZE(armada_370_ahash_algs), + .has_tdma = true, +}; + +static const struct of_device_id mv_cesa_of_match_table[] = { + { .compatible = "marvell,orion-crypto", .data = &orion_caps }, + { .compatible = "marvell,kirkwood-crypto", .data = &kirkwood_caps }, + { .compatible = "marvell,dove-crypto", .data = &kirkwood_caps }, + { .compatible = "marvell,armada-370-crypto", .data = &armada_370_caps }, + { .compatible = "marvell,armada-xp-crypto", .data = &armada_xp_caps }, + { .compatible = "marvell,armada-375-crypto", .data = &armada_xp_caps }, + { .compatible = "marvell,armada-38x-crypto", .data = &armada_xp_caps }, + {} +}; +MODULE_DEVICE_TABLE(of, mv_cesa_of_match_table); + +static void +mv_cesa_conf_mbus_windows(struct mv_cesa_engine *engine, + const struct mbus_dram_target_info *dram) +{ + void __iomem *iobase = engine->regs; + int i; + + for (i = 0; i < 4; i++) { + writel(0, iobase + CESA_TDMA_WINDOW_CTRL(i)); + writel(0, iobase + CESA_TDMA_WINDOW_BASE(i)); + } + + for (i = 0; i < dram->num_cs; i++) { + const struct mbus_dram_window *cs = dram->cs + i; + + writel(((cs->size - 1) & 0xffff0000) | + (cs->mbus_attr << 8) | + (dram->mbus_dram_target_id << 4) | 1, + iobase + CESA_TDMA_WINDOW_CTRL(i)); + writel(cs->base, iobase + CESA_TDMA_WINDOW_BASE(i)); + } +} + +static int mv_cesa_dev_dma_init(struct mv_cesa_dev *cesa) +{ + struct device *dev = cesa->dev; + struct mv_cesa_dev_dma *dma; + + if (!cesa->caps->has_tdma) + return 0; + + dma = devm_kzalloc(dev, sizeof(*dma), GFP_KERNEL); + if (!dma) + return -ENOMEM; + + dma->tdma_desc_pool = dmam_pool_create("tdma_desc", dev, + sizeof(struct mv_cesa_tdma_desc), + 16, 0); + if (!dma->tdma_desc_pool) + return -ENOMEM; + + dma->op_pool = dmam_pool_create("cesa_op", dev, + sizeof(struct mv_cesa_op_ctx), 16, 0); + if (!dma->op_pool) + return -ENOMEM; + + dma->cache_pool = dmam_pool_create("cesa_cache", dev, + CESA_MAX_HASH_BLOCK_SIZE, 1, 0); + if (!dma->cache_pool) + return -ENOMEM; + + dma->padding_pool = dmam_pool_create("cesa_padding", dev, 72, 1, 0); + if (!dma->padding_pool) + return -ENOMEM; + + cesa->dma = dma; + + return 0; +} + +static int mv_cesa_get_sram(struct platform_device *pdev, int idx) +{ + struct mv_cesa_dev *cesa = platform_get_drvdata(pdev); + struct mv_cesa_engine *engine = &cesa->engines[idx]; + const char *res_name = "sram"; + struct resource *res; + + engine->pool = of_gen_pool_get(cesa->dev->of_node, + "marvell,crypto-srams", idx); + if (engine->pool) { + engine->sram = gen_pool_dma_alloc(engine->pool, + cesa->sram_size, + &engine->sram_dma); + if (engine->sram) + return 0; + + engine->pool = NULL; + return -ENOMEM; + } + + if (cesa->caps->nengines > 1) { + if (!idx) + res_name = "sram0"; + else + res_name = "sram1"; + } + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, + res_name); + if (!res || resource_size(res) < cesa->sram_size) + return -EINVAL; + + engine->sram = devm_ioremap_resource(cesa->dev, res); + if (IS_ERR(engine->sram)) + return PTR_ERR(engine->sram); + + engine->sram_dma = phys_to_dma(cesa->dev, + (phys_addr_t)res->start); + + return 0; +} + +static void mv_cesa_put_sram(struct platform_device *pdev, int idx) +{ + struct mv_cesa_dev *cesa = platform_get_drvdata(pdev); + struct mv_cesa_engine *engine = &cesa->engines[idx]; + + if (!engine->pool) + return; + + gen_pool_free(engine->pool, (unsigned long)engine->sram, + cesa->sram_size); +} + +static int mv_cesa_probe(struct platform_device *pdev) +{ + const struct mv_cesa_caps *caps = &orion_caps; + const struct mbus_dram_target_info *dram; + const struct of_device_id *match; + struct device *dev = &pdev->dev; + struct mv_cesa_dev *cesa; + struct mv_cesa_engine *engines; + struct resource *res; + int irq, ret, i; + u32 sram_size; + + if (cesa_dev) { + dev_err(&pdev->dev, "Only one CESA device authorized\n"); + return -EEXIST; + } + + if (dev->of_node) { + match = of_match_node(mv_cesa_of_match_table, dev->of_node); + if (!match || !match->data) + return -ENOTSUPP; + + caps = match->data; + } + + if ((caps == &orion_caps || caps == &kirkwood_caps) && !allhwsupport) + return -ENOTSUPP; + + cesa = devm_kzalloc(dev, sizeof(*cesa), GFP_KERNEL); + if (!cesa) + return -ENOMEM; + + cesa->caps = caps; + cesa->dev = dev; + + sram_size = CESA_SA_DEFAULT_SRAM_SIZE; + of_property_read_u32(cesa->dev->of_node, "marvell,crypto-sram-size", + &sram_size); + if (sram_size < CESA_SA_MIN_SRAM_SIZE) + sram_size = CESA_SA_MIN_SRAM_SIZE; + + cesa->sram_size = sram_size; + cesa->engines = devm_kzalloc(dev, caps->nengines * sizeof(*engines), + GFP_KERNEL); + if (!cesa->engines) + return -ENOMEM; + + spin_lock_init(&cesa->lock); + crypto_init_queue(&cesa->queue, 50); + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "regs"); + cesa->regs = devm_ioremap_resource(dev, res); + if (IS_ERR(cesa->regs)) + return -ENOMEM; + + ret = mv_cesa_dev_dma_init(cesa); + if (ret) + return ret; + + dram = mv_mbus_dram_info_nooverlap(); + + platform_set_drvdata(pdev, cesa); + + for (i = 0; i < caps->nengines; i++) { + struct mv_cesa_engine *engine = &cesa->engines[i]; + char res_name[7]; + + engine->id = i; + spin_lock_init(&engine->lock); + + ret = mv_cesa_get_sram(pdev, i); + if (ret) + goto err_cleanup; + + irq = platform_get_irq(pdev, i); + if (irq < 0) { + ret = irq; + goto err_cleanup; + } + + /* + * Not all platforms can gate the CESA clocks: do not complain + * if the clock does not exist. + */ + snprintf(res_name, sizeof(res_name), "cesa%d", i); + engine->clk = devm_clk_get(dev, res_name); + if (IS_ERR(engine->clk)) { + engine->clk = devm_clk_get(dev, NULL); + if (IS_ERR(engine->clk)) + engine->clk = NULL; + } + + snprintf(res_name, sizeof(res_name), "cesaz%d", i); + engine->zclk = devm_clk_get(dev, res_name); + if (IS_ERR(engine->zclk)) + engine->zclk = NULL; + + ret = clk_prepare_enable(engine->clk); + if (ret) + goto err_cleanup; + + ret = clk_prepare_enable(engine->zclk); + if (ret) + goto err_cleanup; + + engine->regs = cesa->regs + CESA_ENGINE_OFF(i); + + if (dram && cesa->caps->has_tdma) + mv_cesa_conf_mbus_windows(&cesa->engines[i], dram); + + writel(0, cesa->engines[i].regs + CESA_SA_INT_STATUS); + writel(CESA_SA_CFG_STOP_DIG_ERR, + cesa->engines[i].regs + CESA_SA_CFG); + writel(engine->sram_dma & CESA_SA_SRAM_MSK, + cesa->engines[i].regs + CESA_SA_DESC_P0); + + ret = devm_request_threaded_irq(dev, irq, NULL, mv_cesa_int, + IRQF_ONESHOT, + dev_name(&pdev->dev), + &cesa->engines[i]); + if (ret) + goto err_cleanup; + } + + cesa_dev = cesa; + + ret = mv_cesa_add_algs(cesa); + if (ret) { + cesa_dev = NULL; + goto err_cleanup; + } + + dev_info(dev, "CESA device successfully registered\n"); + + return 0; + +err_cleanup: + for (i = 0; i < caps->nengines; i++) { + clk_disable_unprepare(cesa->engines[i].zclk); + clk_disable_unprepare(cesa->engines[i].clk); + mv_cesa_put_sram(pdev, i); + } + + return ret; +} + +static int mv_cesa_remove(struct platform_device *pdev) +{ + struct mv_cesa_dev *cesa = platform_get_drvdata(pdev); + int i; + + mv_cesa_remove_algs(cesa); + + for (i = 0; i < cesa->caps->nengines; i++) { + clk_disable_unprepare(cesa->engines[i].zclk); + clk_disable_unprepare(cesa->engines[i].clk); + mv_cesa_put_sram(pdev, i); + } + + return 0; +} + +static struct platform_driver marvell_cesa = { + .probe = mv_cesa_probe, + .remove = mv_cesa_remove, + .driver = { + .name = "marvell-cesa", + .of_match_table = mv_cesa_of_match_table, + }, +}; +module_platform_driver(marvell_cesa); + +MODULE_ALIAS("platform:mv_crypto"); +MODULE_AUTHOR("Boris Brezillon "); +MODULE_AUTHOR("Arnaud Ebalard "); +MODULE_DESCRIPTION("Support for Marvell's cryptographic engine"); +MODULE_LICENSE("GPL v2"); diff --git a/kernel/drivers/crypto/marvell/cesa.h b/kernel/drivers/crypto/marvell/cesa.h new file mode 100644 index 000000000..bd985e725 --- /dev/null +++ b/kernel/drivers/crypto/marvell/cesa.h @@ -0,0 +1,825 @@ +#ifndef __MARVELL_CESA_H__ +#define __MARVELL_CESA_H__ + +#include +#include +#include + +#include +#include + +#define CESA_ENGINE_OFF(i) (((i) * 0x2000)) + +#define CESA_TDMA_BYTE_CNT 0x800 +#define CESA_TDMA_SRC_ADDR 0x810 +#define CESA_TDMA_DST_ADDR 0x820 +#define CESA_TDMA_NEXT_ADDR 0x830 + +#define CESA_TDMA_CONTROL 0x840 +#define CESA_TDMA_DST_BURST GENMASK(2, 0) +#define CESA_TDMA_DST_BURST_32B 3 +#define CESA_TDMA_DST_BURST_128B 4 +#define CESA_TDMA_OUT_RD_EN BIT(4) +#define CESA_TDMA_SRC_BURST GENMASK(8, 6) +#define CESA_TDMA_SRC_BURST_32B (3 << 6) +#define CESA_TDMA_SRC_BURST_128B (4 << 6) +#define CESA_TDMA_CHAIN BIT(9) +#define CESA_TDMA_BYTE_SWAP BIT(11) +#define CESA_TDMA_NO_BYTE_SWAP BIT(11) +#define CESA_TDMA_EN BIT(12) +#define CESA_TDMA_FETCH_ND BIT(13) +#define CESA_TDMA_ACT BIT(14) + +#define CESA_TDMA_CUR 0x870 +#define CESA_TDMA_ERROR_CAUSE 0x8c8 +#define CESA_TDMA_ERROR_MSK 0x8cc + +#define CESA_TDMA_WINDOW_BASE(x) (((x) * 0x8) + 0xa00) +#define CESA_TDMA_WINDOW_CTRL(x) (((x) * 0x8) + 0xa04) + +#define CESA_IVDIG(x) (0xdd00 + ((x) * 4) + \ + (((x) < 5) ? 0 : 0x14)) + +#define CESA_SA_CMD 0xde00 +#define CESA_SA_CMD_EN_CESA_SA_ACCL0 BIT(0) +#define CESA_SA_CMD_EN_CESA_SA_ACCL1 BIT(1) +#define CESA_SA_CMD_DISABLE_SEC BIT(2) + +#define CESA_SA_DESC_P0 0xde04 + +#define CESA_SA_DESC_P1 0xde14 + +#define CESA_SA_CFG 0xde08 +#define CESA_SA_CFG_STOP_DIG_ERR GENMASK(1, 0) +#define CESA_SA_CFG_DIG_ERR_CONT 0 +#define CESA_SA_CFG_DIG_ERR_SKIP 1 +#define CESA_SA_CFG_DIG_ERR_STOP 3 +#define CESA_SA_CFG_CH0_W_IDMA BIT(7) +#define CESA_SA_CFG_CH1_W_IDMA BIT(8) +#define CESA_SA_CFG_ACT_CH0_IDMA BIT(9) +#define CESA_SA_CFG_ACT_CH1_IDMA BIT(10) +#define CESA_SA_CFG_MULTI_PKT BIT(11) +#define CESA_SA_CFG_PARA_DIS BIT(13) + +#define CESA_SA_ACCEL_STATUS 0xde0c +#define CESA_SA_ST_ACT_0 BIT(0) +#define CESA_SA_ST_ACT_1 BIT(1) + +/* + * CESA_SA_FPGA_INT_STATUS looks like a FPGA leftover and is documented only + * in Errata 4.12. It looks like that it was part of an IRQ-controller in FPGA + * and someone forgot to remove it while switching to the core and moving to + * CESA_SA_INT_STATUS. + */ +#define CESA_SA_FPGA_INT_STATUS 0xdd68 +#define CESA_SA_INT_STATUS 0xde20 +#define CESA_SA_INT_AUTH_DONE BIT(0) +#define CESA_SA_INT_DES_E_DONE BIT(1) +#define CESA_SA_INT_AES_E_DONE BIT(2) +#define CESA_SA_INT_AES_D_DONE BIT(3) +#define CESA_SA_INT_ENC_DONE BIT(4) +#define CESA_SA_INT_ACCEL0_DONE BIT(5) +#define CESA_SA_INT_ACCEL1_DONE BIT(6) +#define CESA_SA_INT_ACC0_IDMA_DONE BIT(7) +#define CESA_SA_INT_ACC1_IDMA_DONE BIT(8) +#define CESA_SA_INT_IDMA_DONE BIT(9) +#define CESA_SA_INT_IDMA_OWN_ERR BIT(10) + +#define CESA_SA_INT_MSK 0xde24 + +#define CESA_SA_DESC_CFG_OP_MAC_ONLY 0 +#define CESA_SA_DESC_CFG_OP_CRYPT_ONLY 1 +#define CESA_SA_DESC_CFG_OP_MAC_CRYPT 2 +#define CESA_SA_DESC_CFG_OP_CRYPT_MAC 3 +#define CESA_SA_DESC_CFG_OP_MSK GENMASK(1, 0) +#define CESA_SA_DESC_CFG_MACM_SHA256 (1 << 4) +#define CESA_SA_DESC_CFG_MACM_HMAC_SHA256 (3 << 4) +#define CESA_SA_DESC_CFG_MACM_MD5 (4 << 4) +#define CESA_SA_DESC_CFG_MACM_SHA1 (5 << 4) +#define CESA_SA_DESC_CFG_MACM_HMAC_MD5 (6 << 4) +#define CESA_SA_DESC_CFG_MACM_HMAC_SHA1 (7 << 4) +#define CESA_SA_DESC_CFG_MACM_MSK GENMASK(6, 4) +#define CESA_SA_DESC_CFG_CRYPTM_DES (1 << 8) +#define CESA_SA_DESC_CFG_CRYPTM_3DES (2 << 8) +#define CESA_SA_DESC_CFG_CRYPTM_AES (3 << 8) +#define CESA_SA_DESC_CFG_CRYPTM_MSK GENMASK(9, 8) +#define CESA_SA_DESC_CFG_DIR_ENC (0 << 12) +#define CESA_SA_DESC_CFG_DIR_DEC (1 << 12) +#define CESA_SA_DESC_CFG_CRYPTCM_ECB (0 << 16) +#define CESA_SA_DESC_CFG_CRYPTCM_CBC (1 << 16) +#define CESA_SA_DESC_CFG_CRYPTCM_MSK BIT(16) +#define CESA_SA_DESC_CFG_3DES_EEE (0 << 20) +#define CESA_SA_DESC_CFG_3DES_EDE (1 << 20) +#define CESA_SA_DESC_CFG_AES_LEN_128 (0 << 24) +#define CESA_SA_DESC_CFG_AES_LEN_192 (1 << 24) +#define CESA_SA_DESC_CFG_AES_LEN_256 (2 << 24) +#define CESA_SA_DESC_CFG_AES_LEN_MSK GENMASK(25, 24) +#define CESA_SA_DESC_CFG_NOT_FRAG (0 << 30) +#define CESA_SA_DESC_CFG_FIRST_FRAG (1 << 30) +#define CESA_SA_DESC_CFG_LAST_FRAG (2 << 30) +#define CESA_SA_DESC_CFG_MID_FRAG (3 << 30) +#define CESA_SA_DESC_CFG_FRAG_MSK GENMASK(31, 30) + +/* + * /-----------\ 0 + * | ACCEL CFG | 4 * 8 + * |-----------| 0x20 + * | CRYPT KEY | 8 * 4 + * |-----------| 0x40 + * | IV IN | 4 * 4 + * |-----------| 0x40 (inplace) + * | IV BUF | 4 * 4 + * |-----------| 0x80 + * | DATA IN | 16 * x (max ->max_req_size) + * |-----------| 0x80 (inplace operation) + * | DATA OUT | 16 * x (max ->max_req_size) + * \-----------/ SRAM size + */ + +/* + * Hashing memory map: + * /-----------\ 0 + * | ACCEL CFG | 4 * 8 + * |-----------| 0x20 + * | Inner IV | 8 * 4 + * |-----------| 0x40 + * | Outer IV | 8 * 4 + * |-----------| 0x60 + * | Output BUF| 8 * 4 + * |-----------| 0x80 + * | DATA IN | 64 * x (max ->max_req_size) + * \-----------/ SRAM size + */ + +#define CESA_SA_CFG_SRAM_OFFSET 0x00 +#define CESA_SA_DATA_SRAM_OFFSET 0x80 + +#define CESA_SA_CRYPT_KEY_SRAM_OFFSET 0x20 +#define CESA_SA_CRYPT_IV_SRAM_OFFSET 0x40 + +#define CESA_SA_MAC_IIV_SRAM_OFFSET 0x20 +#define CESA_SA_MAC_OIV_SRAM_OFFSET 0x40 +#define CESA_SA_MAC_DIG_SRAM_OFFSET 0x60 + +#define CESA_SA_DESC_CRYPT_DATA(offset) \ + cpu_to_le32((CESA_SA_DATA_SRAM_OFFSET + (offset)) | \ + ((CESA_SA_DATA_SRAM_OFFSET + (offset)) << 16)) + +#define CESA_SA_DESC_CRYPT_IV(offset) \ + cpu_to_le32((CESA_SA_CRYPT_IV_SRAM_OFFSET + (offset)) | \ + ((CESA_SA_CRYPT_IV_SRAM_OFFSET + (offset)) << 16)) + +#define CESA_SA_DESC_CRYPT_KEY(offset) \ + cpu_to_le32(CESA_SA_CRYPT_KEY_SRAM_OFFSET + (offset)) + +#define CESA_SA_DESC_MAC_DATA(offset) \ + cpu_to_le32(CESA_SA_DATA_SRAM_OFFSET + (offset)) +#define CESA_SA_DESC_MAC_DATA_MSK cpu_to_le32(GENMASK(15, 0)) + +#define CESA_SA_DESC_MAC_TOTAL_LEN(total_len) cpu_to_le32((total_len) << 16) +#define CESA_SA_DESC_MAC_TOTAL_LEN_MSK cpu_to_le32(GENMASK(31, 16)) + +#define CESA_SA_DESC_MAC_SRC_TOTAL_LEN_MAX 0xffff + +#define CESA_SA_DESC_MAC_DIGEST(offset) \ + cpu_to_le32(CESA_SA_MAC_DIG_SRAM_OFFSET + (offset)) +#define CESA_SA_DESC_MAC_DIGEST_MSK cpu_to_le32(GENMASK(15, 0)) + +#define CESA_SA_DESC_MAC_FRAG_LEN(frag_len) cpu_to_le32((frag_len) << 16) +#define CESA_SA_DESC_MAC_FRAG_LEN_MSK cpu_to_le32(GENMASK(31, 16)) + +#define CESA_SA_DESC_MAC_IV(offset) \ + cpu_to_le32((CESA_SA_MAC_IIV_SRAM_OFFSET + (offset)) | \ + ((CESA_SA_MAC_OIV_SRAM_OFFSET + (offset)) << 16)) + +#define CESA_SA_SRAM_SIZE 2048 +#define CESA_SA_SRAM_PAYLOAD_SIZE (cesa_dev->sram_size - \ + CESA_SA_DATA_SRAM_OFFSET) + +#define CESA_SA_DEFAULT_SRAM_SIZE 2048 +#define CESA_SA_MIN_SRAM_SIZE 1024 + +#define CESA_SA_SRAM_MSK (2048 - 1) + +#define CESA_MAX_HASH_BLOCK_SIZE 64 +#define CESA_HASH_BLOCK_SIZE_MSK (CESA_MAX_HASH_BLOCK_SIZE - 1) + +/** + * struct mv_cesa_sec_accel_desc - security accelerator descriptor + * @config: engine config + * @enc_p: input and output data pointers for a cipher operation + * @enc_len: cipher operation length + * @enc_key_p: cipher key pointer + * @enc_iv: cipher IV pointers + * @mac_src_p: input pointer and total hash length + * @mac_digest: digest pointer and hash operation length + * @mac_iv: hmac IV pointers + * + * Structure passed to the CESA engine to describe the crypto operation + * to be executed. + */ +struct mv_cesa_sec_accel_desc { + __le32 config; + __le32 enc_p; + __le32 enc_len; + __le32 enc_key_p; + __le32 enc_iv; + __le32 mac_src_p; + __le32 mac_digest; + __le32 mac_iv; +}; + +/** + * struct mv_cesa_blkcipher_op_ctx - cipher operation context + * @key: cipher key + * @iv: cipher IV + * + * Context associated to a cipher operation. + */ +struct mv_cesa_blkcipher_op_ctx { + u32 key[8]; + u32 iv[4]; +}; + +/** + * struct mv_cesa_hash_op_ctx - hash or hmac operation context + * @key: cipher key + * @iv: cipher IV + * + * Context associated to an hash or hmac operation. + */ +struct mv_cesa_hash_op_ctx { + u32 iv[16]; + u32 hash[8]; +}; + +/** + * struct mv_cesa_op_ctx - crypto operation context + * @desc: CESA descriptor + * @ctx: context associated to the crypto operation + * + * Context associated to a crypto operation. + */ +struct mv_cesa_op_ctx { + struct mv_cesa_sec_accel_desc desc; + union { + struct mv_cesa_blkcipher_op_ctx blkcipher; + struct mv_cesa_hash_op_ctx hash; + } ctx; +}; + +/* TDMA descriptor flags */ +#define CESA_TDMA_DST_IN_SRAM BIT(31) +#define CESA_TDMA_SRC_IN_SRAM BIT(30) +#define CESA_TDMA_TYPE_MSK GENMASK(29, 0) +#define CESA_TDMA_DUMMY 0 +#define CESA_TDMA_DATA 1 +#define CESA_TDMA_OP 2 + +/** + * struct mv_cesa_tdma_desc - TDMA descriptor + * @byte_cnt: number of bytes to transfer + * @src: DMA address of the source + * @dst: DMA address of the destination + * @next_dma: DMA address of the next TDMA descriptor + * @cur_dma: DMA address of this TDMA descriptor + * @next: pointer to the next TDMA descriptor + * @op: CESA operation attached to this TDMA descriptor + * @data: raw data attached to this TDMA descriptor + * @flags: flags describing the TDMA transfer. See the + * "TDMA descriptor flags" section above + * + * TDMA descriptor used to create a transfer chain describing a crypto + * operation. + */ +struct mv_cesa_tdma_desc { + __le32 byte_cnt; + __le32 src; + __le32 dst; + __le32 next_dma; + + /* Software state */ + dma_addr_t cur_dma; + struct mv_cesa_tdma_desc *next; + union { + struct mv_cesa_op_ctx *op; + void *data; + }; + u32 flags; +}; + +/** + * struct mv_cesa_sg_dma_iter - scatter-gather iterator + * @dir: transfer direction + * @sg: scatter list + * @offset: current position in the scatter list + * @op_offset: current position in the crypto operation + * + * Iterator used to iterate over a scatterlist while creating a TDMA chain for + * a crypto operation. + */ +struct mv_cesa_sg_dma_iter { + enum dma_data_direction dir; + struct scatterlist *sg; + unsigned int offset; + unsigned int op_offset; +}; + +/** + * struct mv_cesa_dma_iter - crypto operation iterator + * @len: the crypto operation length + * @offset: current position in the crypto operation + * @op_len: sub-operation length (the crypto engine can only act on 2kb + * chunks) + * + * Iterator used to create a TDMA chain for a given crypto operation. + */ +struct mv_cesa_dma_iter { + unsigned int len; + unsigned int offset; + unsigned int op_len; +}; + +/** + * struct mv_cesa_tdma_chain - TDMA chain + * @first: first entry in the TDMA chain + * @last: last entry in the TDMA chain + * + * Stores a TDMA chain for a specific crypto operation. + */ +struct mv_cesa_tdma_chain { + struct mv_cesa_tdma_desc *first; + struct mv_cesa_tdma_desc *last; +}; + +struct mv_cesa_engine; + +/** + * struct mv_cesa_caps - CESA device capabilities + * @engines: number of engines + * @has_tdma: whether this device has a TDMA block + * @cipher_algs: supported cipher algorithms + * @ncipher_algs: number of supported cipher algorithms + * @ahash_algs: supported hash algorithms + * @nahash_algs: number of supported hash algorithms + * + * Structure used to describe CESA device capabilities. + */ +struct mv_cesa_caps { + int nengines; + bool has_tdma; + struct crypto_alg **cipher_algs; + int ncipher_algs; + struct ahash_alg **ahash_algs; + int nahash_algs; +}; + +/** + * struct mv_cesa_dev_dma - DMA pools + * @tdma_desc_pool: TDMA desc pool + * @op_pool: crypto operation pool + * @cache_pool: data cache pool (used by hash implementation when the + * hash request is smaller than the hash block size) + * @padding_pool: padding pool (used by hash implementation when hardware + * padding cannot be used) + * + * Structure containing the different DMA pools used by this driver. + */ +struct mv_cesa_dev_dma { + struct dma_pool *tdma_desc_pool; + struct dma_pool *op_pool; + struct dma_pool *cache_pool; + struct dma_pool *padding_pool; +}; + +/** + * struct mv_cesa_dev - CESA device + * @caps: device capabilities + * @regs: device registers + * @sram_size: usable SRAM size + * @lock: device lock + * @queue: crypto request queue + * @engines: array of engines + * @dma: dma pools + * + * Structure storing CESA device information. + */ +struct mv_cesa_dev { + const struct mv_cesa_caps *caps; + void __iomem *regs; + struct device *dev; + unsigned int sram_size; + spinlock_t lock; + struct crypto_queue queue; + struct mv_cesa_engine *engines; + struct mv_cesa_dev_dma *dma; +}; + +/** + * struct mv_cesa_engine - CESA engine + * @id: engine id + * @regs: engine registers + * @sram: SRAM memory region + * @sram_dma: DMA address of the SRAM memory region + * @lock: engine lock + * @req: current crypto request + * @clk: engine clk + * @zclk: engine zclk + * @max_req_len: maximum chunk length (useful to create the TDMA chain) + * @int_mask: interrupt mask cache + * @pool: memory pool pointing to the memory region reserved in + * SRAM + * + * Structure storing CESA engine information. + */ +struct mv_cesa_engine { + int id; + void __iomem *regs; + void __iomem *sram; + dma_addr_t sram_dma; + spinlock_t lock; + struct crypto_async_request *req; + struct clk *clk; + struct clk *zclk; + size_t max_req_len; + u32 int_mask; + struct gen_pool *pool; +}; + +/** + * struct mv_cesa_req_ops - CESA request operations + * @prepare: prepare a request to be executed on the specified engine + * @process: process a request chunk result (should return 0 if the + * operation, -EINPROGRESS if it needs more steps or an error + * code) + * @step: launch the crypto operation on the next chunk + * @cleanup: cleanup the crypto request (release associated data) + */ +struct mv_cesa_req_ops { + void (*prepare)(struct crypto_async_request *req, + struct mv_cesa_engine *engine); + int (*process)(struct crypto_async_request *req, u32 status); + void (*step)(struct crypto_async_request *req); + void (*cleanup)(struct crypto_async_request *req); +}; + +/** + * struct mv_cesa_ctx - CESA operation context + * @ops: crypto operations + * + * Base context structure inherited by operation specific ones. + */ +struct mv_cesa_ctx { + const struct mv_cesa_req_ops *ops; +}; + +/** + * struct mv_cesa_hash_ctx - CESA hash operation context + * @base: base context structure + * + * Hash context structure. + */ +struct mv_cesa_hash_ctx { + struct mv_cesa_ctx base; +}; + +/** + * struct mv_cesa_hash_ctx - CESA hmac operation context + * @base: base context structure + * @iv: initialization vectors + * + * HMAC context structure. + */ +struct mv_cesa_hmac_ctx { + struct mv_cesa_ctx base; + u32 iv[16]; +}; + +/** + * enum mv_cesa_req_type - request type definitions + * @CESA_STD_REQ: standard request + * @CESA_DMA_REQ: DMA request + */ +enum mv_cesa_req_type { + CESA_STD_REQ, + CESA_DMA_REQ, +}; + +/** + * struct mv_cesa_req - CESA request + * @type: request type + * @engine: engine associated with this request + */ +struct mv_cesa_req { + enum mv_cesa_req_type type; + struct mv_cesa_engine *engine; +}; + +/** + * struct mv_cesa_tdma_req - CESA TDMA request + * @base: base information + * @chain: TDMA chain + */ +struct mv_cesa_tdma_req { + struct mv_cesa_req base; + struct mv_cesa_tdma_chain chain; +}; + +/** + * struct mv_cesa_sg_std_iter - CESA scatter-gather iterator for standard + * requests + * @iter: sg mapping iterator + * @offset: current offset in the SG entry mapped in memory + */ +struct mv_cesa_sg_std_iter { + struct sg_mapping_iter iter; + unsigned int offset; +}; + +/** + * struct mv_cesa_ablkcipher_std_req - cipher standard request + * @base: base information + * @op: operation context + * @offset: current operation offset + * @size: size of the crypto operation + */ +struct mv_cesa_ablkcipher_std_req { + struct mv_cesa_req base; + struct mv_cesa_op_ctx op; + unsigned int offset; + unsigned int size; + bool skip_ctx; +}; + +/** + * struct mv_cesa_ablkcipher_req - cipher request + * @req: type specific request information + * @src_nents: number of entries in the src sg list + * @dst_nents: number of entries in the dest sg list + */ +struct mv_cesa_ablkcipher_req { + union { + struct mv_cesa_req base; + struct mv_cesa_tdma_req dma; + struct mv_cesa_ablkcipher_std_req std; + } req; + int src_nents; + int dst_nents; +}; + +/** + * struct mv_cesa_ahash_std_req - standard hash request + * @base: base information + * @offset: current operation offset + */ +struct mv_cesa_ahash_std_req { + struct mv_cesa_req base; + unsigned int offset; +}; + +/** + * struct mv_cesa_ahash_dma_req - DMA hash request + * @base: base information + * @padding: padding buffer + * @padding_dma: DMA address of the padding buffer + * @cache_dma: DMA address of the cache buffer + */ +struct mv_cesa_ahash_dma_req { + struct mv_cesa_tdma_req base; + u8 *padding; + dma_addr_t padding_dma; + dma_addr_t cache_dma; +}; + +/** + * struct mv_cesa_ahash_req - hash request + * @req: type specific request information + * @cache: cache buffer + * @cache_ptr: write pointer in the cache buffer + * @len: hash total length + * @src_nents: number of entries in the scatterlist + * @last_req: define whether the current operation is the last one + * or not + * @state: hash state + */ +struct mv_cesa_ahash_req { + union { + struct mv_cesa_req base; + struct mv_cesa_ahash_dma_req dma; + struct mv_cesa_ahash_std_req std; + } req; + struct mv_cesa_op_ctx op_tmpl; + u8 *cache; + unsigned int cache_ptr; + u64 len; + int src_nents; + bool last_req; + bool algo_le; + u32 state[8]; +}; + +/* CESA functions */ + +extern struct mv_cesa_dev *cesa_dev; + +static inline void mv_cesa_update_op_cfg(struct mv_cesa_op_ctx *op, + u32 cfg, u32 mask) +{ + op->desc.config &= cpu_to_le32(~mask); + op->desc.config |= cpu_to_le32(cfg); +} + +static inline u32 mv_cesa_get_op_cfg(const struct mv_cesa_op_ctx *op) +{ + return le32_to_cpu(op->desc.config); +} + +static inline void mv_cesa_set_op_cfg(struct mv_cesa_op_ctx *op, u32 cfg) +{ + op->desc.config = cpu_to_le32(cfg); +} + +static inline void mv_cesa_adjust_op(struct mv_cesa_engine *engine, + struct mv_cesa_op_ctx *op) +{ + u32 offset = engine->sram_dma & CESA_SA_SRAM_MSK; + + op->desc.enc_p = CESA_SA_DESC_CRYPT_DATA(offset); + op->desc.enc_key_p = CESA_SA_DESC_CRYPT_KEY(offset); + op->desc.enc_iv = CESA_SA_DESC_CRYPT_IV(offset); + op->desc.mac_src_p &= ~CESA_SA_DESC_MAC_DATA_MSK; + op->desc.mac_src_p |= CESA_SA_DESC_MAC_DATA(offset); + op->desc.mac_digest &= ~CESA_SA_DESC_MAC_DIGEST_MSK; + op->desc.mac_digest |= CESA_SA_DESC_MAC_DIGEST(offset); + op->desc.mac_iv = CESA_SA_DESC_MAC_IV(offset); +} + +static inline void mv_cesa_set_crypt_op_len(struct mv_cesa_op_ctx *op, int len) +{ + op->desc.enc_len = cpu_to_le32(len); +} + +static inline void mv_cesa_set_mac_op_total_len(struct mv_cesa_op_ctx *op, + int len) +{ + op->desc.mac_src_p &= ~CESA_SA_DESC_MAC_TOTAL_LEN_MSK; + op->desc.mac_src_p |= CESA_SA_DESC_MAC_TOTAL_LEN(len); +} + +static inline void mv_cesa_set_mac_op_frag_len(struct mv_cesa_op_ctx *op, + int len) +{ + op->desc.mac_digest &= ~CESA_SA_DESC_MAC_FRAG_LEN_MSK; + op->desc.mac_digest |= CESA_SA_DESC_MAC_FRAG_LEN(len); +} + +static inline void mv_cesa_set_int_mask(struct mv_cesa_engine *engine, + u32 int_mask) +{ + if (int_mask == engine->int_mask) + return; + + writel_relaxed(int_mask, engine->regs + CESA_SA_INT_MSK); + engine->int_mask = int_mask; +} + +static inline u32 mv_cesa_get_int_mask(struct mv_cesa_engine *engine) +{ + return engine->int_mask; +} + +static inline bool mv_cesa_mac_op_is_first_frag(const struct mv_cesa_op_ctx *op) +{ + return (mv_cesa_get_op_cfg(op) & CESA_SA_DESC_CFG_FRAG_MSK) == + CESA_SA_DESC_CFG_FIRST_FRAG; +} + +int mv_cesa_queue_req(struct crypto_async_request *req); + +/* + * Helper function that indicates whether a crypto request needs to be + * cleaned up or not after being enqueued using mv_cesa_queue_req(). + */ +static inline int mv_cesa_req_needs_cleanup(struct crypto_async_request *req, + int ret) +{ + /* + * The queue still had some space, the request was queued + * normally, so there's no need to clean it up. + */ + if (ret == -EINPROGRESS) + return false; + + /* + * The queue had not space left, but since the request is + * flagged with CRYPTO_TFM_REQ_MAY_BACKLOG, it was added to + * the backlog and will be processed later. There's no need to + * clean it up. + */ + if (ret == -EBUSY && req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG) + return false; + + /* Request wasn't queued, we need to clean it up */ + return true; +} + +/* TDMA functions */ + +static inline void mv_cesa_req_dma_iter_init(struct mv_cesa_dma_iter *iter, + unsigned int len) +{ + iter->len = len; + iter->op_len = min(len, CESA_SA_SRAM_PAYLOAD_SIZE); + iter->offset = 0; +} + +static inline void mv_cesa_sg_dma_iter_init(struct mv_cesa_sg_dma_iter *iter, + struct scatterlist *sg, + enum dma_data_direction dir) +{ + iter->op_offset = 0; + iter->offset = 0; + iter->sg = sg; + iter->dir = dir; +} + +static inline unsigned int +mv_cesa_req_dma_iter_transfer_len(struct mv_cesa_dma_iter *iter, + struct mv_cesa_sg_dma_iter *sgiter) +{ + return min(iter->op_len - sgiter->op_offset, + sg_dma_len(sgiter->sg) - sgiter->offset); +} + +bool mv_cesa_req_dma_iter_next_transfer(struct mv_cesa_dma_iter *chain, + struct mv_cesa_sg_dma_iter *sgiter, + unsigned int len); + +static inline bool mv_cesa_req_dma_iter_next_op(struct mv_cesa_dma_iter *iter) +{ + iter->offset += iter->op_len; + iter->op_len = min(iter->len - iter->offset, + CESA_SA_SRAM_PAYLOAD_SIZE); + + return iter->op_len; +} + +void mv_cesa_dma_step(struct mv_cesa_tdma_req *dreq); + +static inline int mv_cesa_dma_process(struct mv_cesa_tdma_req *dreq, + u32 status) +{ + if (!(status & CESA_SA_INT_ACC0_IDMA_DONE)) + return -EINPROGRESS; + + if (status & CESA_SA_INT_IDMA_OWN_ERR) + return -EINVAL; + + return 0; +} + +void mv_cesa_dma_prepare(struct mv_cesa_tdma_req *dreq, + struct mv_cesa_engine *engine); + +void mv_cesa_dma_cleanup(struct mv_cesa_tdma_req *dreq); + +static inline void +mv_cesa_tdma_desc_iter_init(struct mv_cesa_tdma_chain *chain) +{ + memset(chain, 0, sizeof(*chain)); +} + +struct mv_cesa_op_ctx *mv_cesa_dma_add_op(struct mv_cesa_tdma_chain *chain, + const struct mv_cesa_op_ctx *op_templ, + bool skip_ctx, + gfp_t flags); + +int mv_cesa_dma_add_data_transfer(struct mv_cesa_tdma_chain *chain, + dma_addr_t dst, dma_addr_t src, u32 size, + u32 flags, gfp_t gfp_flags); + +int mv_cesa_dma_add_dummy_launch(struct mv_cesa_tdma_chain *chain, gfp_t flags); +int mv_cesa_dma_add_dummy_end(struct mv_cesa_tdma_chain *chain, gfp_t flags); + +int mv_cesa_dma_add_op_transfers(struct mv_cesa_tdma_chain *chain, + struct mv_cesa_dma_iter *dma_iter, + struct mv_cesa_sg_dma_iter *sgiter, + gfp_t gfp_flags); + +/* Algorithm definitions */ + +extern struct ahash_alg mv_md5_alg; +extern struct ahash_alg mv_sha1_alg; +extern struct ahash_alg mv_sha256_alg; +extern struct ahash_alg mv_ahmac_md5_alg; +extern struct ahash_alg mv_ahmac_sha1_alg; +extern struct ahash_alg mv_ahmac_sha256_alg; + +extern struct crypto_alg mv_cesa_ecb_des_alg; +extern struct crypto_alg mv_cesa_cbc_des_alg; +extern struct crypto_alg mv_cesa_ecb_des3_ede_alg; +extern struct crypto_alg mv_cesa_cbc_des3_ede_alg; +extern struct crypto_alg mv_cesa_ecb_aes_alg; +extern struct crypto_alg mv_cesa_cbc_aes_alg; + +#endif /* __MARVELL_CESA_H__ */ diff --git a/kernel/drivers/crypto/marvell/cipher.c b/kernel/drivers/crypto/marvell/cipher.c new file mode 100644 index 000000000..6edae64bb --- /dev/null +++ b/kernel/drivers/crypto/marvell/cipher.c @@ -0,0 +1,797 @@ +/* + * Cipher algorithms supported by the CESA: DES, 3DES and AES. + * + * Author: Boris Brezillon + * Author: Arnaud Ebalard + * + * This work is based on an initial version written by + * Sebastian Andrzej Siewior < sebastian at breakpoint dot cc > + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation. + */ + +#include +#include + +#include "cesa.h" + +struct mv_cesa_des_ctx { + struct mv_cesa_ctx base; + u8 key[DES_KEY_SIZE]; +}; + +struct mv_cesa_des3_ctx { + struct mv_cesa_ctx base; + u8 key[DES3_EDE_KEY_SIZE]; +}; + +struct mv_cesa_aes_ctx { + struct mv_cesa_ctx base; + struct crypto_aes_ctx aes; +}; + +struct mv_cesa_ablkcipher_dma_iter { + struct mv_cesa_dma_iter base; + struct mv_cesa_sg_dma_iter src; + struct mv_cesa_sg_dma_iter dst; +}; + +static inline void +mv_cesa_ablkcipher_req_iter_init(struct mv_cesa_ablkcipher_dma_iter *iter, + struct ablkcipher_request *req) +{ + mv_cesa_req_dma_iter_init(&iter->base, req->nbytes); + mv_cesa_sg_dma_iter_init(&iter->src, req->src, DMA_TO_DEVICE); + mv_cesa_sg_dma_iter_init(&iter->dst, req->dst, DMA_FROM_DEVICE); +} + +static inline bool +mv_cesa_ablkcipher_req_iter_next_op(struct mv_cesa_ablkcipher_dma_iter *iter) +{ + iter->src.op_offset = 0; + iter->dst.op_offset = 0; + + return mv_cesa_req_dma_iter_next_op(&iter->base); +} + +static inline void +mv_cesa_ablkcipher_dma_cleanup(struct ablkcipher_request *req) +{ + struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req); + + if (req->dst != req->src) { + dma_unmap_sg(cesa_dev->dev, req->dst, creq->dst_nents, + DMA_FROM_DEVICE); + dma_unmap_sg(cesa_dev->dev, req->src, creq->src_nents, + DMA_TO_DEVICE); + } else { + dma_unmap_sg(cesa_dev->dev, req->src, creq->src_nents, + DMA_BIDIRECTIONAL); + } + mv_cesa_dma_cleanup(&creq->req.dma); +} + +static inline void mv_cesa_ablkcipher_cleanup(struct ablkcipher_request *req) +{ + struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req); + + if (creq->req.base.type == CESA_DMA_REQ) + mv_cesa_ablkcipher_dma_cleanup(req); +} + +static void mv_cesa_ablkcipher_std_step(struct ablkcipher_request *req) +{ + struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req); + struct mv_cesa_ablkcipher_std_req *sreq = &creq->req.std; + struct mv_cesa_engine *engine = sreq->base.engine; + size_t len = min_t(size_t, req->nbytes - sreq->offset, + CESA_SA_SRAM_PAYLOAD_SIZE); + + len = sg_pcopy_to_buffer(req->src, creq->src_nents, + engine->sram + CESA_SA_DATA_SRAM_OFFSET, + len, sreq->offset); + + sreq->size = len; + mv_cesa_set_crypt_op_len(&sreq->op, len); + + /* FIXME: only update enc_len field */ + if (!sreq->skip_ctx) { + memcpy_toio(engine->sram, &sreq->op, sizeof(sreq->op)); + sreq->skip_ctx = true; + } else { + memcpy_toio(engine->sram, &sreq->op, sizeof(sreq->op.desc)); + } + + mv_cesa_set_int_mask(engine, CESA_SA_INT_ACCEL0_DONE); + writel_relaxed(CESA_SA_CFG_PARA_DIS, engine->regs + CESA_SA_CFG); + writel(CESA_SA_CMD_EN_CESA_SA_ACCL0, engine->regs + CESA_SA_CMD); +} + +static int mv_cesa_ablkcipher_std_process(struct ablkcipher_request *req, + u32 status) +{ + struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req); + struct mv_cesa_ablkcipher_std_req *sreq = &creq->req.std; + struct mv_cesa_engine *engine = sreq->base.engine; + size_t len; + + len = sg_pcopy_from_buffer(req->dst, creq->dst_nents, + engine->sram + CESA_SA_DATA_SRAM_OFFSET, + sreq->size, sreq->offset); + + sreq->offset += len; + if (sreq->offset < req->nbytes) + return -EINPROGRESS; + + return 0; +} + +static int mv_cesa_ablkcipher_process(struct crypto_async_request *req, + u32 status) +{ + struct ablkcipher_request *ablkreq = ablkcipher_request_cast(req); + struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(ablkreq); + struct mv_cesa_ablkcipher_std_req *sreq = &creq->req.std; + struct mv_cesa_engine *engine = sreq->base.engine; + int ret; + + if (creq->req.base.type == CESA_DMA_REQ) + ret = mv_cesa_dma_process(&creq->req.dma, status); + else + ret = mv_cesa_ablkcipher_std_process(ablkreq, status); + + if (ret) + return ret; + + memcpy_fromio(ablkreq->info, + engine->sram + CESA_SA_CRYPT_IV_SRAM_OFFSET, + crypto_ablkcipher_ivsize(crypto_ablkcipher_reqtfm(ablkreq))); + + return 0; +} + +static void mv_cesa_ablkcipher_step(struct crypto_async_request *req) +{ + struct ablkcipher_request *ablkreq = ablkcipher_request_cast(req); + struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(ablkreq); + + if (creq->req.base.type == CESA_DMA_REQ) + mv_cesa_dma_step(&creq->req.dma); + else + mv_cesa_ablkcipher_std_step(ablkreq); +} + +static inline void +mv_cesa_ablkcipher_dma_prepare(struct ablkcipher_request *req) +{ + struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req); + struct mv_cesa_tdma_req *dreq = &creq->req.dma; + + mv_cesa_dma_prepare(dreq, dreq->base.engine); +} + +static inline void +mv_cesa_ablkcipher_std_prepare(struct ablkcipher_request *req) +{ + struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req); + struct mv_cesa_ablkcipher_std_req *sreq = &creq->req.std; + struct mv_cesa_engine *engine = sreq->base.engine; + + sreq->size = 0; + sreq->offset = 0; + mv_cesa_adjust_op(engine, &sreq->op); + memcpy_toio(engine->sram, &sreq->op, sizeof(sreq->op)); +} + +static inline void mv_cesa_ablkcipher_prepare(struct crypto_async_request *req, + struct mv_cesa_engine *engine) +{ + struct ablkcipher_request *ablkreq = ablkcipher_request_cast(req); + struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(ablkreq); + creq->req.base.engine = engine; + + if (creq->req.base.type == CESA_DMA_REQ) + mv_cesa_ablkcipher_dma_prepare(ablkreq); + else + mv_cesa_ablkcipher_std_prepare(ablkreq); +} + +static inline void +mv_cesa_ablkcipher_req_cleanup(struct crypto_async_request *req) +{ + struct ablkcipher_request *ablkreq = ablkcipher_request_cast(req); + + mv_cesa_ablkcipher_cleanup(ablkreq); +} + +static const struct mv_cesa_req_ops mv_cesa_ablkcipher_req_ops = { + .step = mv_cesa_ablkcipher_step, + .process = mv_cesa_ablkcipher_process, + .prepare = mv_cesa_ablkcipher_prepare, + .cleanup = mv_cesa_ablkcipher_req_cleanup, +}; + +static int mv_cesa_ablkcipher_cra_init(struct crypto_tfm *tfm) +{ + struct mv_cesa_aes_ctx *ctx = crypto_tfm_ctx(tfm); + + ctx->base.ops = &mv_cesa_ablkcipher_req_ops; + + tfm->crt_ablkcipher.reqsize = sizeof(struct mv_cesa_ablkcipher_req); + + return 0; +} + +static int mv_cesa_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key, + unsigned int len) +{ + struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); + struct mv_cesa_aes_ctx *ctx = crypto_tfm_ctx(tfm); + int remaining; + int offset; + int ret; + int i; + + ret = crypto_aes_expand_key(&ctx->aes, key, len); + if (ret) { + crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); + return ret; + } + + remaining = (ctx->aes.key_length - 16) / 4; + offset = ctx->aes.key_length + 24 - remaining; + for (i = 0; i < remaining; i++) + ctx->aes.key_dec[4 + i] = + cpu_to_le32(ctx->aes.key_enc[offset + i]); + + return 0; +} + +static int mv_cesa_des_setkey(struct crypto_ablkcipher *cipher, const u8 *key, + unsigned int len) +{ + struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); + struct mv_cesa_des_ctx *ctx = crypto_tfm_ctx(tfm); + u32 tmp[DES_EXPKEY_WORDS]; + int ret; + + if (len != DES_KEY_SIZE) { + crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + + ret = des_ekey(tmp, key); + if (!ret && (tfm->crt_flags & CRYPTO_TFM_REQ_WEAK_KEY)) { + tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY; + return -EINVAL; + } + + memcpy(ctx->key, key, DES_KEY_SIZE); + + return 0; +} + +static int mv_cesa_des3_ede_setkey(struct crypto_ablkcipher *cipher, + const u8 *key, unsigned int len) +{ + struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); + struct mv_cesa_des_ctx *ctx = crypto_tfm_ctx(tfm); + + if (len != DES3_EDE_KEY_SIZE) { + crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + + memcpy(ctx->key, key, DES3_EDE_KEY_SIZE); + + return 0; +} + +static int mv_cesa_ablkcipher_dma_req_init(struct ablkcipher_request *req, + const struct mv_cesa_op_ctx *op_templ) +{ + struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req); + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + struct mv_cesa_tdma_req *dreq = &creq->req.dma; + struct mv_cesa_ablkcipher_dma_iter iter; + struct mv_cesa_tdma_chain chain; + bool skip_ctx = false; + int ret; + + dreq->base.type = CESA_DMA_REQ; + dreq->chain.first = NULL; + dreq->chain.last = NULL; + + if (req->src != req->dst) { + ret = dma_map_sg(cesa_dev->dev, req->src, creq->src_nents, + DMA_TO_DEVICE); + if (!ret) + return -ENOMEM; + + ret = dma_map_sg(cesa_dev->dev, req->dst, creq->dst_nents, + DMA_FROM_DEVICE); + if (!ret) { + ret = -ENOMEM; + goto err_unmap_src; + } + } else { + ret = dma_map_sg(cesa_dev->dev, req->src, creq->src_nents, + DMA_BIDIRECTIONAL); + if (!ret) + return -ENOMEM; + } + + mv_cesa_tdma_desc_iter_init(&chain); + mv_cesa_ablkcipher_req_iter_init(&iter, req); + + do { + struct mv_cesa_op_ctx *op; + + op = mv_cesa_dma_add_op(&chain, op_templ, skip_ctx, flags); + if (IS_ERR(op)) { + ret = PTR_ERR(op); + goto err_free_tdma; + } + skip_ctx = true; + + mv_cesa_set_crypt_op_len(op, iter.base.op_len); + + /* Add input transfers */ + ret = mv_cesa_dma_add_op_transfers(&chain, &iter.base, + &iter.src, flags); + if (ret) + goto err_free_tdma; + + /* Add dummy desc to launch the crypto operation */ + ret = mv_cesa_dma_add_dummy_launch(&chain, flags); + if (ret) + goto err_free_tdma; + + /* Add output transfers */ + ret = mv_cesa_dma_add_op_transfers(&chain, &iter.base, + &iter.dst, flags); + if (ret) + goto err_free_tdma; + + } while (mv_cesa_ablkcipher_req_iter_next_op(&iter)); + + dreq->chain = chain; + + return 0; + +err_free_tdma: + mv_cesa_dma_cleanup(dreq); + if (req->dst != req->src) + dma_unmap_sg(cesa_dev->dev, req->dst, creq->dst_nents, + DMA_FROM_DEVICE); + +err_unmap_src: + dma_unmap_sg(cesa_dev->dev, req->src, creq->src_nents, + req->dst != req->src ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL); + + return ret; +} + +static inline int +mv_cesa_ablkcipher_std_req_init(struct ablkcipher_request *req, + const struct mv_cesa_op_ctx *op_templ) +{ + struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req); + struct mv_cesa_ablkcipher_std_req *sreq = &creq->req.std; + + sreq->base.type = CESA_STD_REQ; + sreq->op = *op_templ; + sreq->skip_ctx = false; + + return 0; +} + +static int mv_cesa_ablkcipher_req_init(struct ablkcipher_request *req, + struct mv_cesa_op_ctx *tmpl) +{ + struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req); + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); + unsigned int blksize = crypto_ablkcipher_blocksize(tfm); + int ret; + + if (!IS_ALIGNED(req->nbytes, blksize)) + return -EINVAL; + + creq->src_nents = sg_nents_for_len(req->src, req->nbytes); + creq->dst_nents = sg_nents_for_len(req->dst, req->nbytes); + + mv_cesa_update_op_cfg(tmpl, CESA_SA_DESC_CFG_OP_CRYPT_ONLY, + CESA_SA_DESC_CFG_OP_MSK); + + /* TODO: add a threshold for DMA usage */ + if (cesa_dev->caps->has_tdma) + ret = mv_cesa_ablkcipher_dma_req_init(req, tmpl); + else + ret = mv_cesa_ablkcipher_std_req_init(req, tmpl); + + return ret; +} + +static int mv_cesa_des_op(struct ablkcipher_request *req, + struct mv_cesa_op_ctx *tmpl) +{ + struct mv_cesa_des_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + int ret; + + mv_cesa_update_op_cfg(tmpl, CESA_SA_DESC_CFG_CRYPTM_DES, + CESA_SA_DESC_CFG_CRYPTM_MSK); + + memcpy(tmpl->ctx.blkcipher.key, ctx->key, DES_KEY_SIZE); + + ret = mv_cesa_ablkcipher_req_init(req, tmpl); + if (ret) + return ret; + + ret = mv_cesa_queue_req(&req->base); + if (mv_cesa_req_needs_cleanup(&req->base, ret)) + mv_cesa_ablkcipher_cleanup(req); + + return ret; +} + +static int mv_cesa_ecb_des_encrypt(struct ablkcipher_request *req) +{ + struct mv_cesa_op_ctx tmpl; + + mv_cesa_set_op_cfg(&tmpl, + CESA_SA_DESC_CFG_CRYPTCM_ECB | + CESA_SA_DESC_CFG_DIR_ENC); + + return mv_cesa_des_op(req, &tmpl); +} + +static int mv_cesa_ecb_des_decrypt(struct ablkcipher_request *req) +{ + struct mv_cesa_op_ctx tmpl; + + mv_cesa_set_op_cfg(&tmpl, + CESA_SA_DESC_CFG_CRYPTCM_ECB | + CESA_SA_DESC_CFG_DIR_DEC); + + return mv_cesa_des_op(req, &tmpl); +} + +struct crypto_alg mv_cesa_ecb_des_alg = { + .cra_name = "ecb(des)", + .cra_driver_name = "mv-ecb-des", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct mv_cesa_des_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = mv_cesa_ablkcipher_cra_init, + .cra_u = { + .ablkcipher = { + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .setkey = mv_cesa_des_setkey, + .encrypt = mv_cesa_ecb_des_encrypt, + .decrypt = mv_cesa_ecb_des_decrypt, + }, + }, +}; + +static int mv_cesa_cbc_des_op(struct ablkcipher_request *req, + struct mv_cesa_op_ctx *tmpl) +{ + mv_cesa_update_op_cfg(tmpl, CESA_SA_DESC_CFG_CRYPTCM_CBC, + CESA_SA_DESC_CFG_CRYPTCM_MSK); + + memcpy(tmpl->ctx.blkcipher.iv, req->info, DES_BLOCK_SIZE); + + return mv_cesa_des_op(req, tmpl); +} + +static int mv_cesa_cbc_des_encrypt(struct ablkcipher_request *req) +{ + struct mv_cesa_op_ctx tmpl; + + mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_DIR_ENC); + + return mv_cesa_cbc_des_op(req, &tmpl); +} + +static int mv_cesa_cbc_des_decrypt(struct ablkcipher_request *req) +{ + struct mv_cesa_op_ctx tmpl; + + mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_DIR_DEC); + + return mv_cesa_cbc_des_op(req, &tmpl); +} + +struct crypto_alg mv_cesa_cbc_des_alg = { + .cra_name = "cbc(des)", + .cra_driver_name = "mv-cbc-des", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct mv_cesa_des_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = mv_cesa_ablkcipher_cra_init, + .cra_u = { + .ablkcipher = { + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + .setkey = mv_cesa_des_setkey, + .encrypt = mv_cesa_cbc_des_encrypt, + .decrypt = mv_cesa_cbc_des_decrypt, + }, + }, +}; + +static int mv_cesa_des3_op(struct ablkcipher_request *req, + struct mv_cesa_op_ctx *tmpl) +{ + struct mv_cesa_des3_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + int ret; + + mv_cesa_update_op_cfg(tmpl, CESA_SA_DESC_CFG_CRYPTM_3DES, + CESA_SA_DESC_CFG_CRYPTM_MSK); + + memcpy(tmpl->ctx.blkcipher.key, ctx->key, DES3_EDE_KEY_SIZE); + + ret = mv_cesa_ablkcipher_req_init(req, tmpl); + if (ret) + return ret; + + ret = mv_cesa_queue_req(&req->base); + if (mv_cesa_req_needs_cleanup(&req->base, ret)) + mv_cesa_ablkcipher_cleanup(req); + + return ret; +} + +static int mv_cesa_ecb_des3_ede_encrypt(struct ablkcipher_request *req) +{ + struct mv_cesa_op_ctx tmpl; + + mv_cesa_set_op_cfg(&tmpl, + CESA_SA_DESC_CFG_CRYPTCM_ECB | + CESA_SA_DESC_CFG_3DES_EDE | + CESA_SA_DESC_CFG_DIR_ENC); + + return mv_cesa_des3_op(req, &tmpl); +} + +static int mv_cesa_ecb_des3_ede_decrypt(struct ablkcipher_request *req) +{ + struct mv_cesa_op_ctx tmpl; + + mv_cesa_set_op_cfg(&tmpl, + CESA_SA_DESC_CFG_CRYPTCM_ECB | + CESA_SA_DESC_CFG_3DES_EDE | + CESA_SA_DESC_CFG_DIR_DEC); + + return mv_cesa_des3_op(req, &tmpl); +} + +struct crypto_alg mv_cesa_ecb_des3_ede_alg = { + .cra_name = "ecb(des3_ede)", + .cra_driver_name = "mv-ecb-des3-ede", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct mv_cesa_des3_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = mv_cesa_ablkcipher_cra_init, + .cra_u = { + .ablkcipher = { + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + .setkey = mv_cesa_des3_ede_setkey, + .encrypt = mv_cesa_ecb_des3_ede_encrypt, + .decrypt = mv_cesa_ecb_des3_ede_decrypt, + }, + }, +}; + +static int mv_cesa_cbc_des3_op(struct ablkcipher_request *req, + struct mv_cesa_op_ctx *tmpl) +{ + memcpy(tmpl->ctx.blkcipher.iv, req->info, DES3_EDE_BLOCK_SIZE); + + return mv_cesa_des3_op(req, tmpl); +} + +static int mv_cesa_cbc_des3_ede_encrypt(struct ablkcipher_request *req) +{ + struct mv_cesa_op_ctx tmpl; + + mv_cesa_set_op_cfg(&tmpl, + CESA_SA_DESC_CFG_CRYPTCM_CBC | + CESA_SA_DESC_CFG_3DES_EDE | + CESA_SA_DESC_CFG_DIR_ENC); + + return mv_cesa_cbc_des3_op(req, &tmpl); +} + +static int mv_cesa_cbc_des3_ede_decrypt(struct ablkcipher_request *req) +{ + struct mv_cesa_op_ctx tmpl; + + mv_cesa_set_op_cfg(&tmpl, + CESA_SA_DESC_CFG_CRYPTCM_CBC | + CESA_SA_DESC_CFG_3DES_EDE | + CESA_SA_DESC_CFG_DIR_DEC); + + return mv_cesa_cbc_des3_op(req, &tmpl); +} + +struct crypto_alg mv_cesa_cbc_des3_ede_alg = { + .cra_name = "cbc(des3_ede)", + .cra_driver_name = "mv-cbc-des3-ede", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct mv_cesa_des3_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = mv_cesa_ablkcipher_cra_init, + .cra_u = { + .ablkcipher = { + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + .setkey = mv_cesa_des3_ede_setkey, + .encrypt = mv_cesa_cbc_des3_ede_encrypt, + .decrypt = mv_cesa_cbc_des3_ede_decrypt, + }, + }, +}; + +static int mv_cesa_aes_op(struct ablkcipher_request *req, + struct mv_cesa_op_ctx *tmpl) +{ + struct mv_cesa_aes_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + int ret, i; + u32 *key; + u32 cfg; + + cfg = CESA_SA_DESC_CFG_CRYPTM_AES; + + if (mv_cesa_get_op_cfg(tmpl) & CESA_SA_DESC_CFG_DIR_DEC) + key = ctx->aes.key_dec; + else + key = ctx->aes.key_enc; + + for (i = 0; i < ctx->aes.key_length / sizeof(u32); i++) + tmpl->ctx.blkcipher.key[i] = cpu_to_le32(key[i]); + + if (ctx->aes.key_length == 24) + cfg |= CESA_SA_DESC_CFG_AES_LEN_192; + else if (ctx->aes.key_length == 32) + cfg |= CESA_SA_DESC_CFG_AES_LEN_256; + + mv_cesa_update_op_cfg(tmpl, cfg, + CESA_SA_DESC_CFG_CRYPTM_MSK | + CESA_SA_DESC_CFG_AES_LEN_MSK); + + ret = mv_cesa_ablkcipher_req_init(req, tmpl); + if (ret) + return ret; + + ret = mv_cesa_queue_req(&req->base); + if (mv_cesa_req_needs_cleanup(&req->base, ret)) + mv_cesa_ablkcipher_cleanup(req); + + return ret; +} + +static int mv_cesa_ecb_aes_encrypt(struct ablkcipher_request *req) +{ + struct mv_cesa_op_ctx tmpl; + + mv_cesa_set_op_cfg(&tmpl, + CESA_SA_DESC_CFG_CRYPTCM_ECB | + CESA_SA_DESC_CFG_DIR_ENC); + + return mv_cesa_aes_op(req, &tmpl); +} + +static int mv_cesa_ecb_aes_decrypt(struct ablkcipher_request *req) +{ + struct mv_cesa_op_ctx tmpl; + + mv_cesa_set_op_cfg(&tmpl, + CESA_SA_DESC_CFG_CRYPTCM_ECB | + CESA_SA_DESC_CFG_DIR_DEC); + + return mv_cesa_aes_op(req, &tmpl); +} + +struct crypto_alg mv_cesa_ecb_aes_alg = { + .cra_name = "ecb(aes)", + .cra_driver_name = "mv-ecb-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct mv_cesa_aes_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = mv_cesa_ablkcipher_cra_init, + .cra_u = { + .ablkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = mv_cesa_aes_setkey, + .encrypt = mv_cesa_ecb_aes_encrypt, + .decrypt = mv_cesa_ecb_aes_decrypt, + }, + }, +}; + +static int mv_cesa_cbc_aes_op(struct ablkcipher_request *req, + struct mv_cesa_op_ctx *tmpl) +{ + mv_cesa_update_op_cfg(tmpl, CESA_SA_DESC_CFG_CRYPTCM_CBC, + CESA_SA_DESC_CFG_CRYPTCM_MSK); + memcpy(tmpl->ctx.blkcipher.iv, req->info, AES_BLOCK_SIZE); + + return mv_cesa_aes_op(req, tmpl); +} + +static int mv_cesa_cbc_aes_encrypt(struct ablkcipher_request *req) +{ + struct mv_cesa_op_ctx tmpl; + + mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_DIR_ENC); + + return mv_cesa_cbc_aes_op(req, &tmpl); +} + +static int mv_cesa_cbc_aes_decrypt(struct ablkcipher_request *req) +{ + struct mv_cesa_op_ctx tmpl; + + mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_DIR_DEC); + + return mv_cesa_cbc_aes_op(req, &tmpl); +} + +struct crypto_alg mv_cesa_cbc_aes_alg = { + .cra_name = "cbc(aes)", + .cra_driver_name = "mv-cbc-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct mv_cesa_aes_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = mv_cesa_ablkcipher_cra_init, + .cra_u = { + .ablkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = mv_cesa_aes_setkey, + .encrypt = mv_cesa_cbc_aes_encrypt, + .decrypt = mv_cesa_cbc_aes_decrypt, + }, + }, +}; diff --git a/kernel/drivers/crypto/marvell/hash.c b/kernel/drivers/crypto/marvell/hash.c new file mode 100644 index 000000000..6ec55b4a0 --- /dev/null +++ b/kernel/drivers/crypto/marvell/hash.c @@ -0,0 +1,1420 @@ +/* + * Hash algorithms supported by the CESA: MD5, SHA1 and SHA256. + * + * Author: Boris Brezillon + * Author: Arnaud Ebalard + * + * This work is based on an initial version written by + * Sebastian Andrzej Siewior < sebastian at breakpoint dot cc > + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation. + */ + +#include +#include + +#include "cesa.h" + +struct mv_cesa_ahash_dma_iter { + struct mv_cesa_dma_iter base; + struct mv_cesa_sg_dma_iter src; +}; + +static inline void +mv_cesa_ahash_req_iter_init(struct mv_cesa_ahash_dma_iter *iter, + struct ahash_request *req) +{ + struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + unsigned int len = req->nbytes + creq->cache_ptr; + + if (!creq->last_req) + len &= ~CESA_HASH_BLOCK_SIZE_MSK; + + mv_cesa_req_dma_iter_init(&iter->base, len); + mv_cesa_sg_dma_iter_init(&iter->src, req->src, DMA_TO_DEVICE); + iter->src.op_offset = creq->cache_ptr; +} + +static inline bool +mv_cesa_ahash_req_iter_next_op(struct mv_cesa_ahash_dma_iter *iter) +{ + iter->src.op_offset = 0; + + return mv_cesa_req_dma_iter_next_op(&iter->base); +} + +static inline int mv_cesa_ahash_dma_alloc_cache(struct mv_cesa_ahash_req *creq, + gfp_t flags) +{ + struct mv_cesa_ahash_dma_req *dreq = &creq->req.dma; + + creq->cache = dma_pool_alloc(cesa_dev->dma->cache_pool, flags, + &dreq->cache_dma); + if (!creq->cache) + return -ENOMEM; + + return 0; +} + +static inline int mv_cesa_ahash_std_alloc_cache(struct mv_cesa_ahash_req *creq, + gfp_t flags) +{ + creq->cache = kzalloc(CESA_MAX_HASH_BLOCK_SIZE, flags); + if (!creq->cache) + return -ENOMEM; + + return 0; +} + +static int mv_cesa_ahash_alloc_cache(struct ahash_request *req) +{ + struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + int ret; + + if (creq->cache) + return 0; + + if (creq->req.base.type == CESA_DMA_REQ) + ret = mv_cesa_ahash_dma_alloc_cache(creq, flags); + else + ret = mv_cesa_ahash_std_alloc_cache(creq, flags); + + return ret; +} + +static inline void mv_cesa_ahash_dma_free_cache(struct mv_cesa_ahash_req *creq) +{ + dma_pool_free(cesa_dev->dma->cache_pool, creq->cache, + creq->req.dma.cache_dma); +} + +static inline void mv_cesa_ahash_std_free_cache(struct mv_cesa_ahash_req *creq) +{ + kfree(creq->cache); +} + +static void mv_cesa_ahash_free_cache(struct mv_cesa_ahash_req *creq) +{ + if (!creq->cache) + return; + + if (creq->req.base.type == CESA_DMA_REQ) + mv_cesa_ahash_dma_free_cache(creq); + else + mv_cesa_ahash_std_free_cache(creq); + + creq->cache = NULL; +} + +static int mv_cesa_ahash_dma_alloc_padding(struct mv_cesa_ahash_dma_req *req, + gfp_t flags) +{ + if (req->padding) + return 0; + + req->padding = dma_pool_alloc(cesa_dev->dma->padding_pool, flags, + &req->padding_dma); + if (!req->padding) + return -ENOMEM; + + return 0; +} + +static void mv_cesa_ahash_dma_free_padding(struct mv_cesa_ahash_dma_req *req) +{ + if (!req->padding) + return; + + dma_pool_free(cesa_dev->dma->padding_pool, req->padding, + req->padding_dma); + req->padding = NULL; +} + +static inline void mv_cesa_ahash_dma_last_cleanup(struct ahash_request *req) +{ + struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + + mv_cesa_ahash_dma_free_padding(&creq->req.dma); +} + +static inline void mv_cesa_ahash_dma_cleanup(struct ahash_request *req) +{ + struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + + dma_unmap_sg(cesa_dev->dev, req->src, creq->src_nents, DMA_TO_DEVICE); + mv_cesa_dma_cleanup(&creq->req.dma.base); +} + +static inline void mv_cesa_ahash_cleanup(struct ahash_request *req) +{ + struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + + if (creq->req.base.type == CESA_DMA_REQ) + mv_cesa_ahash_dma_cleanup(req); +} + +static void mv_cesa_ahash_last_cleanup(struct ahash_request *req) +{ + struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + + mv_cesa_ahash_free_cache(creq); + + if (creq->req.base.type == CESA_DMA_REQ) + mv_cesa_ahash_dma_last_cleanup(req); +} + +static int mv_cesa_ahash_pad_len(struct mv_cesa_ahash_req *creq) +{ + unsigned int index, padlen; + + index = creq->len & CESA_HASH_BLOCK_SIZE_MSK; + padlen = (index < 56) ? (56 - index) : (64 + 56 - index); + + return padlen; +} + +static int mv_cesa_ahash_pad_req(struct mv_cesa_ahash_req *creq, u8 *buf) +{ + unsigned int index, padlen; + + buf[0] = 0x80; + /* Pad out to 56 mod 64 */ + index = creq->len & CESA_HASH_BLOCK_SIZE_MSK; + padlen = mv_cesa_ahash_pad_len(creq); + memset(buf + 1, 0, padlen - 1); + + if (creq->algo_le) { + __le64 bits = cpu_to_le64(creq->len << 3); + memcpy(buf + padlen, &bits, sizeof(bits)); + } else { + __be64 bits = cpu_to_be64(creq->len << 3); + memcpy(buf + padlen, &bits, sizeof(bits)); + } + + return padlen + 8; +} + +static void mv_cesa_ahash_std_step(struct ahash_request *req) +{ + struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + struct mv_cesa_ahash_std_req *sreq = &creq->req.std; + struct mv_cesa_engine *engine = sreq->base.engine; + struct mv_cesa_op_ctx *op; + unsigned int new_cache_ptr = 0; + u32 frag_mode; + size_t len; + + if (creq->cache_ptr) + memcpy_toio(engine->sram + CESA_SA_DATA_SRAM_OFFSET, + creq->cache, creq->cache_ptr); + + len = min_t(size_t, req->nbytes + creq->cache_ptr - sreq->offset, + CESA_SA_SRAM_PAYLOAD_SIZE); + + if (!creq->last_req) { + new_cache_ptr = len & CESA_HASH_BLOCK_SIZE_MSK; + len &= ~CESA_HASH_BLOCK_SIZE_MSK; + } + + if (len - creq->cache_ptr) + sreq->offset += sg_pcopy_to_buffer(req->src, creq->src_nents, + engine->sram + + CESA_SA_DATA_SRAM_OFFSET + + creq->cache_ptr, + len - creq->cache_ptr, + sreq->offset); + + op = &creq->op_tmpl; + + frag_mode = mv_cesa_get_op_cfg(op) & CESA_SA_DESC_CFG_FRAG_MSK; + + if (creq->last_req && sreq->offset == req->nbytes && + creq->len <= CESA_SA_DESC_MAC_SRC_TOTAL_LEN_MAX) { + if (frag_mode == CESA_SA_DESC_CFG_FIRST_FRAG) + frag_mode = CESA_SA_DESC_CFG_NOT_FRAG; + else if (frag_mode == CESA_SA_DESC_CFG_MID_FRAG) + frag_mode = CESA_SA_DESC_CFG_LAST_FRAG; + } + + if (frag_mode == CESA_SA_DESC_CFG_NOT_FRAG || + frag_mode == CESA_SA_DESC_CFG_LAST_FRAG) { + if (len && + creq->len <= CESA_SA_DESC_MAC_SRC_TOTAL_LEN_MAX) { + mv_cesa_set_mac_op_total_len(op, creq->len); + } else { + int trailerlen = mv_cesa_ahash_pad_len(creq) + 8; + + if (len + trailerlen > CESA_SA_SRAM_PAYLOAD_SIZE) { + len &= CESA_HASH_BLOCK_SIZE_MSK; + new_cache_ptr = 64 - trailerlen; + memcpy_fromio(creq->cache, + engine->sram + + CESA_SA_DATA_SRAM_OFFSET + len, + new_cache_ptr); + } else { + len += mv_cesa_ahash_pad_req(creq, + engine->sram + len + + CESA_SA_DATA_SRAM_OFFSET); + } + + if (frag_mode == CESA_SA_DESC_CFG_LAST_FRAG) + frag_mode = CESA_SA_DESC_CFG_MID_FRAG; + else + frag_mode = CESA_SA_DESC_CFG_FIRST_FRAG; + } + } + + mv_cesa_set_mac_op_frag_len(op, len); + mv_cesa_update_op_cfg(op, frag_mode, CESA_SA_DESC_CFG_FRAG_MSK); + + /* FIXME: only update enc_len field */ + memcpy_toio(engine->sram, op, sizeof(*op)); + + if (frag_mode == CESA_SA_DESC_CFG_FIRST_FRAG) + mv_cesa_update_op_cfg(op, CESA_SA_DESC_CFG_MID_FRAG, + CESA_SA_DESC_CFG_FRAG_MSK); + + creq->cache_ptr = new_cache_ptr; + + mv_cesa_set_int_mask(engine, CESA_SA_INT_ACCEL0_DONE); + writel_relaxed(CESA_SA_CFG_PARA_DIS, engine->regs + CESA_SA_CFG); + writel(CESA_SA_CMD_EN_CESA_SA_ACCL0, engine->regs + CESA_SA_CMD); +} + +static int mv_cesa_ahash_std_process(struct ahash_request *req, u32 status) +{ + struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + struct mv_cesa_ahash_std_req *sreq = &creq->req.std; + + if (sreq->offset < (req->nbytes - creq->cache_ptr)) + return -EINPROGRESS; + + return 0; +} + +static inline void mv_cesa_ahash_dma_prepare(struct ahash_request *req) +{ + struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + struct mv_cesa_tdma_req *dreq = &creq->req.dma.base; + + mv_cesa_dma_prepare(dreq, dreq->base.engine); +} + +static void mv_cesa_ahash_std_prepare(struct ahash_request *req) +{ + struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + struct mv_cesa_ahash_std_req *sreq = &creq->req.std; + struct mv_cesa_engine *engine = sreq->base.engine; + + sreq->offset = 0; + mv_cesa_adjust_op(engine, &creq->op_tmpl); + memcpy_toio(engine->sram, &creq->op_tmpl, sizeof(creq->op_tmpl)); +} + +static void mv_cesa_ahash_step(struct crypto_async_request *req) +{ + struct ahash_request *ahashreq = ahash_request_cast(req); + struct mv_cesa_ahash_req *creq = ahash_request_ctx(ahashreq); + + if (creq->req.base.type == CESA_DMA_REQ) + mv_cesa_dma_step(&creq->req.dma.base); + else + mv_cesa_ahash_std_step(ahashreq); +} + +static int mv_cesa_ahash_process(struct crypto_async_request *req, u32 status) +{ + struct ahash_request *ahashreq = ahash_request_cast(req); + struct mv_cesa_ahash_req *creq = ahash_request_ctx(ahashreq); + struct mv_cesa_engine *engine = creq->req.base.engine; + unsigned int digsize; + int ret, i; + + if (creq->req.base.type == CESA_DMA_REQ) + ret = mv_cesa_dma_process(&creq->req.dma.base, status); + else + ret = mv_cesa_ahash_std_process(ahashreq, status); + + if (ret == -EINPROGRESS) + return ret; + + digsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(ahashreq)); + for (i = 0; i < digsize / 4; i++) + creq->state[i] = readl_relaxed(engine->regs + CESA_IVDIG(i)); + + if (creq->cache_ptr) + sg_pcopy_to_buffer(ahashreq->src, creq->src_nents, + creq->cache, + creq->cache_ptr, + ahashreq->nbytes - creq->cache_ptr); + + if (creq->last_req) { + /* + * Hardware's MD5 digest is in little endian format, but + * SHA in big endian format + */ + if (creq->algo_le) { + __le32 *result = (void *)ahashreq->result; + + for (i = 0; i < digsize / 4; i++) + result[i] = cpu_to_le32(creq->state[i]); + } else { + __be32 *result = (void *)ahashreq->result; + + for (i = 0; i < digsize / 4; i++) + result[i] = cpu_to_be32(creq->state[i]); + } + } + + return ret; +} + +static void mv_cesa_ahash_prepare(struct crypto_async_request *req, + struct mv_cesa_engine *engine) +{ + struct ahash_request *ahashreq = ahash_request_cast(req); + struct mv_cesa_ahash_req *creq = ahash_request_ctx(ahashreq); + unsigned int digsize; + int i; + + creq->req.base.engine = engine; + + if (creq->req.base.type == CESA_DMA_REQ) + mv_cesa_ahash_dma_prepare(ahashreq); + else + mv_cesa_ahash_std_prepare(ahashreq); + + digsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(ahashreq)); + for (i = 0; i < digsize / 4; i++) + writel_relaxed(creq->state[i], engine->regs + CESA_IVDIG(i)); +} + +static void mv_cesa_ahash_req_cleanup(struct crypto_async_request *req) +{ + struct ahash_request *ahashreq = ahash_request_cast(req); + struct mv_cesa_ahash_req *creq = ahash_request_ctx(ahashreq); + + if (creq->last_req) + mv_cesa_ahash_last_cleanup(ahashreq); + + mv_cesa_ahash_cleanup(ahashreq); +} + +static const struct mv_cesa_req_ops mv_cesa_ahash_req_ops = { + .step = mv_cesa_ahash_step, + .process = mv_cesa_ahash_process, + .prepare = mv_cesa_ahash_prepare, + .cleanup = mv_cesa_ahash_req_cleanup, +}; + +static int mv_cesa_ahash_init(struct ahash_request *req, + struct mv_cesa_op_ctx *tmpl, bool algo_le) +{ + struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + + memset(creq, 0, sizeof(*creq)); + mv_cesa_update_op_cfg(tmpl, + CESA_SA_DESC_CFG_OP_MAC_ONLY | + CESA_SA_DESC_CFG_FIRST_FRAG, + CESA_SA_DESC_CFG_OP_MSK | + CESA_SA_DESC_CFG_FRAG_MSK); + mv_cesa_set_mac_op_total_len(tmpl, 0); + mv_cesa_set_mac_op_frag_len(tmpl, 0); + creq->op_tmpl = *tmpl; + creq->len = 0; + creq->algo_le = algo_le; + + return 0; +} + +static inline int mv_cesa_ahash_cra_init(struct crypto_tfm *tfm) +{ + struct mv_cesa_hash_ctx *ctx = crypto_tfm_ctx(tfm); + + ctx->base.ops = &mv_cesa_ahash_req_ops; + + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct mv_cesa_ahash_req)); + return 0; +} + +static int mv_cesa_ahash_cache_req(struct ahash_request *req, bool *cached) +{ + struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + int ret; + + if (((creq->cache_ptr + req->nbytes) & CESA_HASH_BLOCK_SIZE_MSK) && + !creq->last_req) { + ret = mv_cesa_ahash_alloc_cache(req); + if (ret) + return ret; + } + + if (creq->cache_ptr + req->nbytes < 64 && !creq->last_req) { + *cached = true; + + if (!req->nbytes) + return 0; + + sg_pcopy_to_buffer(req->src, creq->src_nents, + creq->cache + creq->cache_ptr, + req->nbytes, 0); + + creq->cache_ptr += req->nbytes; + } + + return 0; +} + +static struct mv_cesa_op_ctx * +mv_cesa_dma_add_frag(struct mv_cesa_tdma_chain *chain, + struct mv_cesa_op_ctx *tmpl, unsigned int frag_len, + gfp_t flags) +{ + struct mv_cesa_op_ctx *op; + int ret; + + op = mv_cesa_dma_add_op(chain, tmpl, false, flags); + if (IS_ERR(op)) + return op; + + /* Set the operation block fragment length. */ + mv_cesa_set_mac_op_frag_len(op, frag_len); + + /* Append dummy desc to launch operation */ + ret = mv_cesa_dma_add_dummy_launch(chain, flags); + if (ret) + return ERR_PTR(ret); + + if (mv_cesa_mac_op_is_first_frag(tmpl)) + mv_cesa_update_op_cfg(tmpl, + CESA_SA_DESC_CFG_MID_FRAG, + CESA_SA_DESC_CFG_FRAG_MSK); + + return op; +} + +static int +mv_cesa_ahash_dma_add_cache(struct mv_cesa_tdma_chain *chain, + struct mv_cesa_ahash_dma_iter *dma_iter, + struct mv_cesa_ahash_req *creq, + gfp_t flags) +{ + struct mv_cesa_ahash_dma_req *ahashdreq = &creq->req.dma; + + if (!creq->cache_ptr) + return 0; + + return mv_cesa_dma_add_data_transfer(chain, + CESA_SA_DATA_SRAM_OFFSET, + ahashdreq->cache_dma, + creq->cache_ptr, + CESA_TDMA_DST_IN_SRAM, + flags); +} + +static struct mv_cesa_op_ctx * +mv_cesa_ahash_dma_last_req(struct mv_cesa_tdma_chain *chain, + struct mv_cesa_ahash_dma_iter *dma_iter, + struct mv_cesa_ahash_req *creq, + unsigned int frag_len, gfp_t flags) +{ + struct mv_cesa_ahash_dma_req *ahashdreq = &creq->req.dma; + unsigned int len, trailerlen, padoff = 0; + struct mv_cesa_op_ctx *op; + int ret; + + /* + * If the transfer is smaller than our maximum length, and we have + * some data outstanding, we can ask the engine to finish the hash. + */ + if (creq->len <= CESA_SA_DESC_MAC_SRC_TOTAL_LEN_MAX && frag_len) { + op = mv_cesa_dma_add_frag(chain, &creq->op_tmpl, frag_len, + flags); + if (IS_ERR(op)) + return op; + + mv_cesa_set_mac_op_total_len(op, creq->len); + mv_cesa_update_op_cfg(op, mv_cesa_mac_op_is_first_frag(op) ? + CESA_SA_DESC_CFG_NOT_FRAG : + CESA_SA_DESC_CFG_LAST_FRAG, + CESA_SA_DESC_CFG_FRAG_MSK); + + return op; + } + + /* + * The request is longer than the engine can handle, or we have + * no data outstanding. Manually generate the padding, adding it + * as a "mid" fragment. + */ + ret = mv_cesa_ahash_dma_alloc_padding(ahashdreq, flags); + if (ret) + return ERR_PTR(ret); + + trailerlen = mv_cesa_ahash_pad_req(creq, ahashdreq->padding); + + len = min(CESA_SA_SRAM_PAYLOAD_SIZE - frag_len, trailerlen); + if (len) { + ret = mv_cesa_dma_add_data_transfer(chain, + CESA_SA_DATA_SRAM_OFFSET + + frag_len, + ahashdreq->padding_dma, + len, CESA_TDMA_DST_IN_SRAM, + flags); + if (ret) + return ERR_PTR(ret); + + op = mv_cesa_dma_add_frag(chain, &creq->op_tmpl, frag_len + len, + flags); + if (IS_ERR(op)) + return op; + + if (len == trailerlen) + return op; + + padoff += len; + } + + ret = mv_cesa_dma_add_data_transfer(chain, + CESA_SA_DATA_SRAM_OFFSET, + ahashdreq->padding_dma + + padoff, + trailerlen - padoff, + CESA_TDMA_DST_IN_SRAM, + flags); + if (ret) + return ERR_PTR(ret); + + return mv_cesa_dma_add_frag(chain, &creq->op_tmpl, trailerlen - padoff, + flags); +} + +static int mv_cesa_ahash_dma_req_init(struct ahash_request *req) +{ + struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + struct mv_cesa_ahash_dma_req *ahashdreq = &creq->req.dma; + struct mv_cesa_tdma_req *dreq = &ahashdreq->base; + struct mv_cesa_ahash_dma_iter iter; + struct mv_cesa_op_ctx *op = NULL; + unsigned int frag_len; + int ret; + + dreq->chain.first = NULL; + dreq->chain.last = NULL; + + if (creq->src_nents) { + ret = dma_map_sg(cesa_dev->dev, req->src, creq->src_nents, + DMA_TO_DEVICE); + if (!ret) { + ret = -ENOMEM; + goto err; + } + } + + mv_cesa_tdma_desc_iter_init(&dreq->chain); + mv_cesa_ahash_req_iter_init(&iter, req); + + /* + * Add the cache (left-over data from a previous block) first. + * This will never overflow the SRAM size. + */ + ret = mv_cesa_ahash_dma_add_cache(&dreq->chain, &iter, creq, flags); + if (ret) + goto err_free_tdma; + + if (iter.src.sg) { + /* + * Add all the new data, inserting an operation block and + * launch command between each full SRAM block-worth of + * data. We intentionally do not add the final op block. + */ + while (true) { + ret = mv_cesa_dma_add_op_transfers(&dreq->chain, + &iter.base, + &iter.src, flags); + if (ret) + goto err_free_tdma; + + frag_len = iter.base.op_len; + + if (!mv_cesa_ahash_req_iter_next_op(&iter)) + break; + + op = mv_cesa_dma_add_frag(&dreq->chain, &creq->op_tmpl, + frag_len, flags); + if (IS_ERR(op)) { + ret = PTR_ERR(op); + goto err_free_tdma; + } + } + } else { + /* Account for the data that was in the cache. */ + frag_len = iter.base.op_len; + } + + /* + * At this point, frag_len indicates whether we have any data + * outstanding which needs an operation. Queue up the final + * operation, which depends whether this is the final request. + */ + if (creq->last_req) + op = mv_cesa_ahash_dma_last_req(&dreq->chain, &iter, creq, + frag_len, flags); + else if (frag_len) + op = mv_cesa_dma_add_frag(&dreq->chain, &creq->op_tmpl, + frag_len, flags); + + if (IS_ERR(op)) { + ret = PTR_ERR(op); + goto err_free_tdma; + } + + if (op) { + /* Add dummy desc to wait for crypto operation end */ + ret = mv_cesa_dma_add_dummy_end(&dreq->chain, flags); + if (ret) + goto err_free_tdma; + } + + if (!creq->last_req) + creq->cache_ptr = req->nbytes + creq->cache_ptr - + iter.base.len; + else + creq->cache_ptr = 0; + + return 0; + +err_free_tdma: + mv_cesa_dma_cleanup(dreq); + dma_unmap_sg(cesa_dev->dev, req->src, creq->src_nents, DMA_TO_DEVICE); + +err: + mv_cesa_ahash_last_cleanup(req); + + return ret; +} + +static int mv_cesa_ahash_req_init(struct ahash_request *req, bool *cached) +{ + struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + int ret; + + if (cesa_dev->caps->has_tdma) + creq->req.base.type = CESA_DMA_REQ; + else + creq->req.base.type = CESA_STD_REQ; + + creq->src_nents = sg_nents_for_len(req->src, req->nbytes); + + ret = mv_cesa_ahash_cache_req(req, cached); + if (ret) + return ret; + + if (*cached) + return 0; + + if (creq->req.base.type == CESA_DMA_REQ) + ret = mv_cesa_ahash_dma_req_init(req); + + return ret; +} + +static int mv_cesa_ahash_update(struct ahash_request *req) +{ + struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + bool cached = false; + int ret; + + creq->len += req->nbytes; + ret = mv_cesa_ahash_req_init(req, &cached); + if (ret) + return ret; + + if (cached) + return 0; + + ret = mv_cesa_queue_req(&req->base); + if (mv_cesa_req_needs_cleanup(&req->base, ret)) + mv_cesa_ahash_cleanup(req); + + return ret; +} + +static int mv_cesa_ahash_final(struct ahash_request *req) +{ + struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + struct mv_cesa_op_ctx *tmpl = &creq->op_tmpl; + bool cached = false; + int ret; + + mv_cesa_set_mac_op_total_len(tmpl, creq->len); + creq->last_req = true; + req->nbytes = 0; + + ret = mv_cesa_ahash_req_init(req, &cached); + if (ret) + return ret; + + if (cached) + return 0; + + ret = mv_cesa_queue_req(&req->base); + if (mv_cesa_req_needs_cleanup(&req->base, ret)) + mv_cesa_ahash_cleanup(req); + + return ret; +} + +static int mv_cesa_ahash_finup(struct ahash_request *req) +{ + struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + struct mv_cesa_op_ctx *tmpl = &creq->op_tmpl; + bool cached = false; + int ret; + + creq->len += req->nbytes; + mv_cesa_set_mac_op_total_len(tmpl, creq->len); + creq->last_req = true; + + ret = mv_cesa_ahash_req_init(req, &cached); + if (ret) + return ret; + + if (cached) + return 0; + + ret = mv_cesa_queue_req(&req->base); + if (mv_cesa_req_needs_cleanup(&req->base, ret)) + mv_cesa_ahash_cleanup(req); + + return ret; +} + +static int mv_cesa_ahash_export(struct ahash_request *req, void *hash, + u64 *len, void *cache) +{ + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + unsigned int digsize = crypto_ahash_digestsize(ahash); + unsigned int blocksize; + + blocksize = crypto_ahash_blocksize(ahash); + + *len = creq->len; + memcpy(hash, creq->state, digsize); + memset(cache, 0, blocksize); + if (creq->cache) + memcpy(cache, creq->cache, creq->cache_ptr); + + return 0; +} + +static int mv_cesa_ahash_import(struct ahash_request *req, const void *hash, + u64 len, const void *cache) +{ + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); + unsigned int digsize = crypto_ahash_digestsize(ahash); + unsigned int blocksize; + unsigned int cache_ptr; + int ret; + + ret = crypto_ahash_init(req); + if (ret) + return ret; + + blocksize = crypto_ahash_blocksize(ahash); + if (len >= blocksize) + mv_cesa_update_op_cfg(&creq->op_tmpl, + CESA_SA_DESC_CFG_MID_FRAG, + CESA_SA_DESC_CFG_FRAG_MSK); + + creq->len = len; + memcpy(creq->state, hash, digsize); + creq->cache_ptr = 0; + + cache_ptr = do_div(len, blocksize); + if (!cache_ptr) + return 0; + + ret = mv_cesa_ahash_alloc_cache(req); + if (ret) + return ret; + + memcpy(creq->cache, cache, cache_ptr); + creq->cache_ptr = cache_ptr; + + return 0; +} + +static int mv_cesa_md5_init(struct ahash_request *req) +{ + struct mv_cesa_op_ctx tmpl = { }; + + mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_MACM_MD5); + + mv_cesa_ahash_init(req, &tmpl, true); + + return 0; +} + +static int mv_cesa_md5_export(struct ahash_request *req, void *out) +{ + struct md5_state *out_state = out; + + return mv_cesa_ahash_export(req, out_state->hash, + &out_state->byte_count, out_state->block); +} + +static int mv_cesa_md5_import(struct ahash_request *req, const void *in) +{ + const struct md5_state *in_state = in; + + return mv_cesa_ahash_import(req, in_state->hash, in_state->byte_count, + in_state->block); +} + +static int mv_cesa_md5_digest(struct ahash_request *req) +{ + int ret; + + ret = mv_cesa_md5_init(req); + if (ret) + return ret; + + return mv_cesa_ahash_finup(req); +} + +struct ahash_alg mv_md5_alg = { + .init = mv_cesa_md5_init, + .update = mv_cesa_ahash_update, + .final = mv_cesa_ahash_final, + .finup = mv_cesa_ahash_finup, + .digest = mv_cesa_md5_digest, + .export = mv_cesa_md5_export, + .import = mv_cesa_md5_import, + .halg = { + .digestsize = MD5_DIGEST_SIZE, + .statesize = sizeof(struct md5_state), + .base = { + .cra_name = "md5", + .cra_driver_name = "mv-md5", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = MD5_HMAC_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct mv_cesa_hash_ctx), + .cra_init = mv_cesa_ahash_cra_init, + .cra_module = THIS_MODULE, + } + } +}; + +static int mv_cesa_sha1_init(struct ahash_request *req) +{ + struct mv_cesa_op_ctx tmpl = { }; + + mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_MACM_SHA1); + + mv_cesa_ahash_init(req, &tmpl, false); + + return 0; +} + +static int mv_cesa_sha1_export(struct ahash_request *req, void *out) +{ + struct sha1_state *out_state = out; + + return mv_cesa_ahash_export(req, out_state->state, &out_state->count, + out_state->buffer); +} + +static int mv_cesa_sha1_import(struct ahash_request *req, const void *in) +{ + const struct sha1_state *in_state = in; + + return mv_cesa_ahash_import(req, in_state->state, in_state->count, + in_state->buffer); +} + +static int mv_cesa_sha1_digest(struct ahash_request *req) +{ + int ret; + + ret = mv_cesa_sha1_init(req); + if (ret) + return ret; + + return mv_cesa_ahash_finup(req); +} + +struct ahash_alg mv_sha1_alg = { + .init = mv_cesa_sha1_init, + .update = mv_cesa_ahash_update, + .final = mv_cesa_ahash_final, + .finup = mv_cesa_ahash_finup, + .digest = mv_cesa_sha1_digest, + .export = mv_cesa_sha1_export, + .import = mv_cesa_sha1_import, + .halg = { + .digestsize = SHA1_DIGEST_SIZE, + .statesize = sizeof(struct sha1_state), + .base = { + .cra_name = "sha1", + .cra_driver_name = "mv-sha1", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct mv_cesa_hash_ctx), + .cra_init = mv_cesa_ahash_cra_init, + .cra_module = THIS_MODULE, + } + } +}; + +static int mv_cesa_sha256_init(struct ahash_request *req) +{ + struct mv_cesa_op_ctx tmpl = { }; + + mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_MACM_SHA256); + + mv_cesa_ahash_init(req, &tmpl, false); + + return 0; +} + +static int mv_cesa_sha256_digest(struct ahash_request *req) +{ + int ret; + + ret = mv_cesa_sha256_init(req); + if (ret) + return ret; + + return mv_cesa_ahash_finup(req); +} + +static int mv_cesa_sha256_export(struct ahash_request *req, void *out) +{ + struct sha256_state *out_state = out; + + return mv_cesa_ahash_export(req, out_state->state, &out_state->count, + out_state->buf); +} + +static int mv_cesa_sha256_import(struct ahash_request *req, const void *in) +{ + const struct sha256_state *in_state = in; + + return mv_cesa_ahash_import(req, in_state->state, in_state->count, + in_state->buf); +} + +struct ahash_alg mv_sha256_alg = { + .init = mv_cesa_sha256_init, + .update = mv_cesa_ahash_update, + .final = mv_cesa_ahash_final, + .finup = mv_cesa_ahash_finup, + .digest = mv_cesa_sha256_digest, + .export = mv_cesa_sha256_export, + .import = mv_cesa_sha256_import, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct sha256_state), + .base = { + .cra_name = "sha256", + .cra_driver_name = "mv-sha256", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct mv_cesa_hash_ctx), + .cra_init = mv_cesa_ahash_cra_init, + .cra_module = THIS_MODULE, + } + } +}; + +struct mv_cesa_ahash_result { + struct completion completion; + int error; +}; + +static void mv_cesa_hmac_ahash_complete(struct crypto_async_request *req, + int error) +{ + struct mv_cesa_ahash_result *result = req->data; + + if (error == -EINPROGRESS) + return; + + result->error = error; + complete(&result->completion); +} + +static int mv_cesa_ahmac_iv_state_init(struct ahash_request *req, u8 *pad, + void *state, unsigned int blocksize) +{ + struct mv_cesa_ahash_result result; + struct scatterlist sg; + int ret; + + ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + mv_cesa_hmac_ahash_complete, &result); + sg_init_one(&sg, pad, blocksize); + ahash_request_set_crypt(req, &sg, pad, blocksize); + init_completion(&result.completion); + + ret = crypto_ahash_init(req); + if (ret) + return ret; + + ret = crypto_ahash_update(req); + if (ret && ret != -EINPROGRESS) + return ret; + + wait_for_completion_interruptible(&result.completion); + if (result.error) + return result.error; + + ret = crypto_ahash_export(req, state); + if (ret) + return ret; + + return 0; +} + +static int mv_cesa_ahmac_pad_init(struct ahash_request *req, + const u8 *key, unsigned int keylen, + u8 *ipad, u8 *opad, + unsigned int blocksize) +{ + struct mv_cesa_ahash_result result; + struct scatterlist sg; + int ret; + int i; + + if (keylen <= blocksize) { + memcpy(ipad, key, keylen); + } else { + u8 *keydup = kmemdup(key, keylen, GFP_KERNEL); + + if (!keydup) + return -ENOMEM; + + ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + mv_cesa_hmac_ahash_complete, + &result); + sg_init_one(&sg, keydup, keylen); + ahash_request_set_crypt(req, &sg, ipad, keylen); + init_completion(&result.completion); + + ret = crypto_ahash_digest(req); + if (ret == -EINPROGRESS) { + wait_for_completion_interruptible(&result.completion); + ret = result.error; + } + + /* Set the memory region to 0 to avoid any leak. */ + memset(keydup, 0, keylen); + kfree(keydup); + + if (ret) + return ret; + + keylen = crypto_ahash_digestsize(crypto_ahash_reqtfm(req)); + } + + memset(ipad + keylen, 0, blocksize - keylen); + memcpy(opad, ipad, blocksize); + + for (i = 0; i < blocksize; i++) { + ipad[i] ^= 0x36; + opad[i] ^= 0x5c; + } + + return 0; +} + +static int mv_cesa_ahmac_setkey(const char *hash_alg_name, + const u8 *key, unsigned int keylen, + void *istate, void *ostate) +{ + struct ahash_request *req; + struct crypto_ahash *tfm; + unsigned int blocksize; + u8 *ipad = NULL; + u8 *opad; + int ret; + + tfm = crypto_alloc_ahash(hash_alg_name, CRYPTO_ALG_TYPE_AHASH, + CRYPTO_ALG_TYPE_AHASH_MASK); + if (IS_ERR(tfm)) + return PTR_ERR(tfm); + + req = ahash_request_alloc(tfm, GFP_KERNEL); + if (!req) { + ret = -ENOMEM; + goto free_ahash; + } + + crypto_ahash_clear_flags(tfm, ~0); + + blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); + + ipad = kzalloc(2 * blocksize, GFP_KERNEL); + if (!ipad) { + ret = -ENOMEM; + goto free_req; + } + + opad = ipad + blocksize; + + ret = mv_cesa_ahmac_pad_init(req, key, keylen, ipad, opad, blocksize); + if (ret) + goto free_ipad; + + ret = mv_cesa_ahmac_iv_state_init(req, ipad, istate, blocksize); + if (ret) + goto free_ipad; + + ret = mv_cesa_ahmac_iv_state_init(req, opad, ostate, blocksize); + +free_ipad: + kfree(ipad); +free_req: + ahash_request_free(req); +free_ahash: + crypto_free_ahash(tfm); + + return ret; +} + +static int mv_cesa_ahmac_cra_init(struct crypto_tfm *tfm) +{ + struct mv_cesa_hmac_ctx *ctx = crypto_tfm_ctx(tfm); + + ctx->base.ops = &mv_cesa_ahash_req_ops; + + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct mv_cesa_ahash_req)); + return 0; +} + +static int mv_cesa_ahmac_md5_init(struct ahash_request *req) +{ + struct mv_cesa_hmac_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct mv_cesa_op_ctx tmpl = { }; + + mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_MACM_HMAC_MD5); + memcpy(tmpl.ctx.hash.iv, ctx->iv, sizeof(ctx->iv)); + + mv_cesa_ahash_init(req, &tmpl, true); + + return 0; +} + +static int mv_cesa_ahmac_md5_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen) +{ + struct mv_cesa_hmac_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); + struct md5_state istate, ostate; + int ret, i; + + ret = mv_cesa_ahmac_setkey("mv-md5", key, keylen, &istate, &ostate); + if (ret) + return ret; + + for (i = 0; i < ARRAY_SIZE(istate.hash); i++) + ctx->iv[i] = be32_to_cpu(istate.hash[i]); + + for (i = 0; i < ARRAY_SIZE(ostate.hash); i++) + ctx->iv[i + 8] = be32_to_cpu(ostate.hash[i]); + + return 0; +} + +static int mv_cesa_ahmac_md5_digest(struct ahash_request *req) +{ + int ret; + + ret = mv_cesa_ahmac_md5_init(req); + if (ret) + return ret; + + return mv_cesa_ahash_finup(req); +} + +struct ahash_alg mv_ahmac_md5_alg = { + .init = mv_cesa_ahmac_md5_init, + .update = mv_cesa_ahash_update, + .final = mv_cesa_ahash_final, + .finup = mv_cesa_ahash_finup, + .digest = mv_cesa_ahmac_md5_digest, + .setkey = mv_cesa_ahmac_md5_setkey, + .export = mv_cesa_md5_export, + .import = mv_cesa_md5_import, + .halg = { + .digestsize = MD5_DIGEST_SIZE, + .statesize = sizeof(struct md5_state), + .base = { + .cra_name = "hmac(md5)", + .cra_driver_name = "mv-hmac-md5", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = MD5_HMAC_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct mv_cesa_hmac_ctx), + .cra_init = mv_cesa_ahmac_cra_init, + .cra_module = THIS_MODULE, + } + } +}; + +static int mv_cesa_ahmac_sha1_init(struct ahash_request *req) +{ + struct mv_cesa_hmac_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct mv_cesa_op_ctx tmpl = { }; + + mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_MACM_HMAC_SHA1); + memcpy(tmpl.ctx.hash.iv, ctx->iv, sizeof(ctx->iv)); + + mv_cesa_ahash_init(req, &tmpl, false); + + return 0; +} + +static int mv_cesa_ahmac_sha1_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen) +{ + struct mv_cesa_hmac_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); + struct sha1_state istate, ostate; + int ret, i; + + ret = mv_cesa_ahmac_setkey("mv-sha1", key, keylen, &istate, &ostate); + if (ret) + return ret; + + for (i = 0; i < ARRAY_SIZE(istate.state); i++) + ctx->iv[i] = be32_to_cpu(istate.state[i]); + + for (i = 0; i < ARRAY_SIZE(ostate.state); i++) + ctx->iv[i + 8] = be32_to_cpu(ostate.state[i]); + + return 0; +} + +static int mv_cesa_ahmac_sha1_digest(struct ahash_request *req) +{ + int ret; + + ret = mv_cesa_ahmac_sha1_init(req); + if (ret) + return ret; + + return mv_cesa_ahash_finup(req); +} + +struct ahash_alg mv_ahmac_sha1_alg = { + .init = mv_cesa_ahmac_sha1_init, + .update = mv_cesa_ahash_update, + .final = mv_cesa_ahash_final, + .finup = mv_cesa_ahash_finup, + .digest = mv_cesa_ahmac_sha1_digest, + .setkey = mv_cesa_ahmac_sha1_setkey, + .export = mv_cesa_sha1_export, + .import = mv_cesa_sha1_import, + .halg = { + .digestsize = SHA1_DIGEST_SIZE, + .statesize = sizeof(struct sha1_state), + .base = { + .cra_name = "hmac(sha1)", + .cra_driver_name = "mv-hmac-sha1", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct mv_cesa_hmac_ctx), + .cra_init = mv_cesa_ahmac_cra_init, + .cra_module = THIS_MODULE, + } + } +}; + +static int mv_cesa_ahmac_sha256_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen) +{ + struct mv_cesa_hmac_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); + struct sha256_state istate, ostate; + int ret, i; + + ret = mv_cesa_ahmac_setkey("mv-sha256", key, keylen, &istate, &ostate); + if (ret) + return ret; + + for (i = 0; i < ARRAY_SIZE(istate.state); i++) + ctx->iv[i] = be32_to_cpu(istate.state[i]); + + for (i = 0; i < ARRAY_SIZE(ostate.state); i++) + ctx->iv[i + 8] = be32_to_cpu(ostate.state[i]); + + return 0; +} + +static int mv_cesa_ahmac_sha256_init(struct ahash_request *req) +{ + struct mv_cesa_hmac_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct mv_cesa_op_ctx tmpl = { }; + + mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_MACM_HMAC_SHA256); + memcpy(tmpl.ctx.hash.iv, ctx->iv, sizeof(ctx->iv)); + + mv_cesa_ahash_init(req, &tmpl, false); + + return 0; +} + +static int mv_cesa_ahmac_sha256_digest(struct ahash_request *req) +{ + int ret; + + ret = mv_cesa_ahmac_sha256_init(req); + if (ret) + return ret; + + return mv_cesa_ahash_finup(req); +} + +struct ahash_alg mv_ahmac_sha256_alg = { + .init = mv_cesa_ahmac_sha256_init, + .update = mv_cesa_ahash_update, + .final = mv_cesa_ahash_final, + .finup = mv_cesa_ahash_finup, + .digest = mv_cesa_ahmac_sha256_digest, + .setkey = mv_cesa_ahmac_sha256_setkey, + .export = mv_cesa_sha256_export, + .import = mv_cesa_sha256_import, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct sha256_state), + .base = { + .cra_name = "hmac(sha256)", + .cra_driver_name = "mv-hmac-sha256", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct mv_cesa_hmac_ctx), + .cra_init = mv_cesa_ahmac_cra_init, + .cra_module = THIS_MODULE, + } + } +}; diff --git a/kernel/drivers/crypto/marvell/tdma.c b/kernel/drivers/crypto/marvell/tdma.c new file mode 100644 index 000000000..764279812 --- /dev/null +++ b/kernel/drivers/crypto/marvell/tdma.c @@ -0,0 +1,226 @@ +/* + * Provide TDMA helper functions used by cipher and hash algorithm + * implementations. + * + * Author: Boris Brezillon + * Author: Arnaud Ebalard + * + * This work is based on an initial version written by + * Sebastian Andrzej Siewior < sebastian at breakpoint dot cc > + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation. + */ + +#include "cesa.h" + +bool mv_cesa_req_dma_iter_next_transfer(struct mv_cesa_dma_iter *iter, + struct mv_cesa_sg_dma_iter *sgiter, + unsigned int len) +{ + if (!sgiter->sg) + return false; + + sgiter->op_offset += len; + sgiter->offset += len; + if (sgiter->offset == sg_dma_len(sgiter->sg)) { + if (sg_is_last(sgiter->sg)) + return false; + sgiter->offset = 0; + sgiter->sg = sg_next(sgiter->sg); + } + + if (sgiter->op_offset == iter->op_len) + return false; + + return true; +} + +void mv_cesa_dma_step(struct mv_cesa_tdma_req *dreq) +{ + struct mv_cesa_engine *engine = dreq->base.engine; + + writel_relaxed(0, engine->regs + CESA_SA_CFG); + + mv_cesa_set_int_mask(engine, CESA_SA_INT_ACC0_IDMA_DONE); + writel_relaxed(CESA_TDMA_DST_BURST_128B | CESA_TDMA_SRC_BURST_128B | + CESA_TDMA_NO_BYTE_SWAP | CESA_TDMA_EN, + engine->regs + CESA_TDMA_CONTROL); + + writel_relaxed(CESA_SA_CFG_ACT_CH0_IDMA | CESA_SA_CFG_MULTI_PKT | + CESA_SA_CFG_CH0_W_IDMA | CESA_SA_CFG_PARA_DIS, + engine->regs + CESA_SA_CFG); + writel_relaxed(dreq->chain.first->cur_dma, + engine->regs + CESA_TDMA_NEXT_ADDR); + writel(CESA_SA_CMD_EN_CESA_SA_ACCL0, engine->regs + CESA_SA_CMD); +} + +void mv_cesa_dma_cleanup(struct mv_cesa_tdma_req *dreq) +{ + struct mv_cesa_tdma_desc *tdma; + + for (tdma = dreq->chain.first; tdma;) { + struct mv_cesa_tdma_desc *old_tdma = tdma; + + if (tdma->flags & CESA_TDMA_OP) + dma_pool_free(cesa_dev->dma->op_pool, tdma->op, + le32_to_cpu(tdma->src)); + + tdma = tdma->next; + dma_pool_free(cesa_dev->dma->tdma_desc_pool, old_tdma, + old_tdma->cur_dma); + } + + dreq->chain.first = NULL; + dreq->chain.last = NULL; +} + +void mv_cesa_dma_prepare(struct mv_cesa_tdma_req *dreq, + struct mv_cesa_engine *engine) +{ + struct mv_cesa_tdma_desc *tdma; + + for (tdma = dreq->chain.first; tdma; tdma = tdma->next) { + if (tdma->flags & CESA_TDMA_DST_IN_SRAM) + tdma->dst = cpu_to_le32(tdma->dst + engine->sram_dma); + + if (tdma->flags & CESA_TDMA_SRC_IN_SRAM) + tdma->src = cpu_to_le32(tdma->src + engine->sram_dma); + + if (tdma->flags & CESA_TDMA_OP) + mv_cesa_adjust_op(engine, tdma->op); + } +} + +static struct mv_cesa_tdma_desc * +mv_cesa_dma_add_desc(struct mv_cesa_tdma_chain *chain, gfp_t flags) +{ + struct mv_cesa_tdma_desc *new_tdma = NULL; + dma_addr_t dma_handle; + + new_tdma = dma_pool_alloc(cesa_dev->dma->tdma_desc_pool, flags, + &dma_handle); + if (!new_tdma) + return ERR_PTR(-ENOMEM); + + memset(new_tdma, 0, sizeof(*new_tdma)); + new_tdma->cur_dma = dma_handle; + if (chain->last) { + chain->last->next_dma = cpu_to_le32(dma_handle); + chain->last->next = new_tdma; + } else { + chain->first = new_tdma; + } + + chain->last = new_tdma; + + return new_tdma; +} + +struct mv_cesa_op_ctx *mv_cesa_dma_add_op(struct mv_cesa_tdma_chain *chain, + const struct mv_cesa_op_ctx *op_templ, + bool skip_ctx, + gfp_t flags) +{ + struct mv_cesa_tdma_desc *tdma; + struct mv_cesa_op_ctx *op; + dma_addr_t dma_handle; + unsigned int size; + + tdma = mv_cesa_dma_add_desc(chain, flags); + if (IS_ERR(tdma)) + return ERR_CAST(tdma); + + op = dma_pool_alloc(cesa_dev->dma->op_pool, flags, &dma_handle); + if (!op) + return ERR_PTR(-ENOMEM); + + *op = *op_templ; + + size = skip_ctx ? sizeof(op->desc) : sizeof(*op); + + tdma = chain->last; + tdma->op = op; + tdma->byte_cnt = cpu_to_le32(size | BIT(31)); + tdma->src = cpu_to_le32(dma_handle); + tdma->flags = CESA_TDMA_DST_IN_SRAM | CESA_TDMA_OP; + + return op; +} + +int mv_cesa_dma_add_data_transfer(struct mv_cesa_tdma_chain *chain, + dma_addr_t dst, dma_addr_t src, u32 size, + u32 flags, gfp_t gfp_flags) +{ + struct mv_cesa_tdma_desc *tdma; + + tdma = mv_cesa_dma_add_desc(chain, gfp_flags); + if (IS_ERR(tdma)) + return PTR_ERR(tdma); + + tdma->byte_cnt = cpu_to_le32(size | BIT(31)); + tdma->src = src; + tdma->dst = dst; + + flags &= (CESA_TDMA_DST_IN_SRAM | CESA_TDMA_SRC_IN_SRAM); + tdma->flags = flags | CESA_TDMA_DATA; + + return 0; +} + +int mv_cesa_dma_add_dummy_launch(struct mv_cesa_tdma_chain *chain, gfp_t flags) +{ + struct mv_cesa_tdma_desc *tdma; + + tdma = mv_cesa_dma_add_desc(chain, flags); + if (IS_ERR(tdma)) + return PTR_ERR(tdma); + + return 0; +} + +int mv_cesa_dma_add_dummy_end(struct mv_cesa_tdma_chain *chain, gfp_t flags) +{ + struct mv_cesa_tdma_desc *tdma; + + tdma = mv_cesa_dma_add_desc(chain, flags); + if (IS_ERR(tdma)) + return PTR_ERR(tdma); + + tdma->byte_cnt = cpu_to_le32(BIT(31)); + + return 0; +} + +int mv_cesa_dma_add_op_transfers(struct mv_cesa_tdma_chain *chain, + struct mv_cesa_dma_iter *dma_iter, + struct mv_cesa_sg_dma_iter *sgiter, + gfp_t gfp_flags) +{ + u32 flags = sgiter->dir == DMA_TO_DEVICE ? + CESA_TDMA_DST_IN_SRAM : CESA_TDMA_SRC_IN_SRAM; + unsigned int len; + + do { + dma_addr_t dst, src; + int ret; + + len = mv_cesa_req_dma_iter_transfer_len(dma_iter, sgiter); + if (sgiter->dir == DMA_TO_DEVICE) { + dst = CESA_SA_DATA_SRAM_OFFSET + sgiter->op_offset; + src = sg_dma_address(sgiter->sg) + sgiter->offset; + } else { + dst = sg_dma_address(sgiter->sg) + sgiter->offset; + src = CESA_SA_DATA_SRAM_OFFSET + sgiter->op_offset; + } + + ret = mv_cesa_dma_add_data_transfer(chain, dst, src, len, + flags, gfp_flags); + if (ret) + return ret; + + } while (mv_cesa_req_dma_iter_next_transfer(dma_iter, sgiter, len)); + + return 0; +} diff --git a/kernel/drivers/crypto/mv_cesa.c b/kernel/drivers/crypto/mv_cesa.c index f91f15dde..e6b658fae 100644 --- a/kernel/drivers/crypto/mv_cesa.c +++ b/kernel/drivers/crypto/mv_cesa.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -29,6 +30,8 @@ #define MAX_HW_HASH_SIZE 0xFFFF #define MV_CESA_EXPIRE 500 /* msec */ +#define MV_CESA_DEFAULT_SRAM_SIZE 2048 + /* * STM: * /---------------------------------------\ @@ -83,6 +86,8 @@ struct req_progress { struct crypto_priv { void __iomem *reg; void __iomem *sram; + struct gen_pool *sram_pool; + dma_addr_t sram_dma; int irq; struct clk *clk; struct task_struct *queue_th; @@ -595,7 +600,7 @@ static int queue_manag(void *data) cpg->eng_st = ENGINE_IDLE; do { struct crypto_async_request *async_req = NULL; - struct crypto_async_request *backlog; + struct crypto_async_request *backlog = NULL; __set_current_state(TASK_INTERRUPTIBLE); @@ -1019,6 +1024,39 @@ static struct ahash_alg mv_hmac_sha1_alg = { } }; +static int mv_cesa_get_sram(struct platform_device *pdev, + struct crypto_priv *cp) +{ + struct resource *res; + u32 sram_size = MV_CESA_DEFAULT_SRAM_SIZE; + + of_property_read_u32(pdev->dev.of_node, "marvell,crypto-sram-size", + &sram_size); + + cp->sram_size = sram_size; + cp->sram_pool = of_gen_pool_get(pdev->dev.of_node, + "marvell,crypto-srams", 0); + if (cp->sram_pool) { + cp->sram = gen_pool_dma_alloc(cp->sram_pool, sram_size, + &cp->sram_dma); + if (cp->sram) + return 0; + + return -ENOMEM; + } + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, + "sram"); + if (!res || resource_size(res) < cp->sram_size) + return -EINVAL; + + cp->sram = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(cp->sram)) + return PTR_ERR(cp->sram); + + return 0; +} + static int mv_probe(struct platform_device *pdev) { struct crypto_priv *cp; @@ -1041,24 +1079,17 @@ static int mv_probe(struct platform_device *pdev) spin_lock_init(&cp->lock); crypto_init_queue(&cp->queue, 50); - cp->reg = ioremap(res->start, resource_size(res)); - if (!cp->reg) { - ret = -ENOMEM; + cp->reg = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(cp->reg)) { + ret = PTR_ERR(cp->reg); goto err; } - res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "sram"); - if (!res) { - ret = -ENXIO; - goto err_unmap_reg; - } - cp->sram_size = resource_size(res); + ret = mv_cesa_get_sram(pdev, cp); + if (ret) + goto err; + cp->max_req_size = cp->sram_size - SRAM_CFG_SPACE; - cp->sram = ioremap(res->start, cp->sram_size); - if (!cp->sram) { - ret = -ENOMEM; - goto err_unmap_reg; - } if (pdev->dev.of_node) irq = irq_of_parse_and_map(pdev->dev.of_node, 0); @@ -1066,7 +1097,7 @@ static int mv_probe(struct platform_device *pdev) irq = platform_get_irq(pdev, 0); if (irq < 0 || irq == NO_IRQ) { ret = irq; - goto err_unmap_sram; + goto err; } cp->irq = irq; @@ -1076,7 +1107,7 @@ static int mv_probe(struct platform_device *pdev) cp->queue_th = kthread_run(queue_manag, cp, "mv_crypto"); if (IS_ERR(cp->queue_th)) { ret = PTR_ERR(cp->queue_th); - goto err_unmap_sram; + goto err; } ret = request_irq(irq, crypto_int, 0, dev_name(&pdev->dev), @@ -1134,10 +1165,6 @@ err_irq: } err_thread: kthread_stop(cp->queue_th); -err_unmap_sram: - iounmap(cp->sram); -err_unmap_reg: - iounmap(cp->reg); err: kfree(cp); cpg = NULL; @@ -1157,8 +1184,6 @@ static int mv_remove(struct platform_device *pdev) kthread_stop(cp->queue_th); free_irq(cp->irq, cp); memset(cp->sram, 0, cp->sram_size); - iounmap(cp->sram); - iounmap(cp->reg); if (!IS_ERR(cp->clk)) { clk_disable_unprepare(cp->clk); @@ -1172,6 +1197,8 @@ static int mv_remove(struct platform_device *pdev) static const struct of_device_id mv_cesa_of_match_table[] = { { .compatible = "marvell,orion-crypto", }, + { .compatible = "marvell,kirkwood-crypto", }, + { .compatible = "marvell,dove-crypto", }, {} }; MODULE_DEVICE_TABLE(of, mv_cesa_of_match_table); diff --git a/kernel/drivers/crypto/n2_core.c b/kernel/drivers/crypto/n2_core.c index 10a9aeff1..5450880ab 100644 --- a/kernel/drivers/crypto/n2_core.c +++ b/kernel/drivers/crypto/n2_core.c @@ -34,7 +34,7 @@ #define DRV_MODULE_VERSION "0.2" #define DRV_MODULE_RELDATE "July 28, 2011" -static char version[] = +static const char version[] = DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n"; MODULE_AUTHOR("David S. Miller (davem@davemloft.net)"); @@ -1281,10 +1281,10 @@ static const char md5_zero[MD5_DIGEST_SIZE] = { 0xe9, 0x80, 0x09, 0x98, 0xec, 0xf8, 0x42, 0x7e, }; static const u32 md5_init[MD5_HASH_WORDS] = { - cpu_to_le32(0x67452301), - cpu_to_le32(0xefcdab89), - cpu_to_le32(0x98badcfe), - cpu_to_le32(0x10325476), + cpu_to_le32(MD5_H0), + cpu_to_le32(MD5_H1), + cpu_to_le32(MD5_H2), + cpu_to_le32(MD5_H3), }; static const char sha1_zero[SHA1_DIGEST_SIZE] = { 0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, 0x32, diff --git a/kernel/drivers/crypto/nx/Kconfig b/kernel/drivers/crypto/nx/Kconfig index f82616621..ad7552a69 100644 --- a/kernel/drivers/crypto/nx/Kconfig +++ b/kernel/drivers/crypto/nx/Kconfig @@ -1,26 +1,48 @@ + config CRYPTO_DEV_NX_ENCRYPT - tristate "Encryption acceleration support" - depends on PPC64 && IBMVIO + tristate "Encryption acceleration support on pSeries platform" + depends on PPC_PSERIES && IBMVIO && !CPU_LITTLE_ENDIAN default y select CRYPTO_AES - select CRYPTO_CBC - select CRYPTO_ECB select CRYPTO_CCM - select CRYPTO_GCM - select CRYPTO_AUTHENC - select CRYPTO_XCBC - select CRYPTO_SHA256 - select CRYPTO_SHA512 help - Support for Power7+ in-Nest encryption acceleration. This - module supports acceleration for AES and SHA2 algorithms. If you - choose 'M' here, this module will be called nx_crypto. + Support for PowerPC Nest (NX) encryption acceleration. This + module supports acceleration for AES and SHA2 algorithms on + the pSeries platform. If you choose 'M' here, this module + will be called nx_crypto. config CRYPTO_DEV_NX_COMPRESS tristate "Compression acceleration support" - depends on PPC64 && IBMVIO + default y + select CRYPTO_ALGAPI + select 842_DECOMPRESS + help + Support for PowerPC Nest (NX) compression acceleration. This + module supports acceleration for compressing memory with the 842 + algorithm using the cryptographic API. One of the platform + drivers must be selected also. If you choose 'M' here, this + module will be called nx_compress. + +if CRYPTO_DEV_NX_COMPRESS + +config CRYPTO_DEV_NX_COMPRESS_PSERIES + tristate "Compression acceleration support on pSeries platform" + depends on PPC_PSERIES && IBMVIO default y help - Support for Power7+ in-Nest compression acceleration. This - module supports acceleration for AES and SHA2 algorithms. If you - choose 'M' here, this module will be called nx_compress. + Support for PowerPC Nest (NX) compression acceleration. This + module supports acceleration for compressing memory with the 842 + algorithm. This supports NX hardware on the pSeries platform. + If you choose 'M' here, this module will be called nx_compress_pseries. + +config CRYPTO_DEV_NX_COMPRESS_POWERNV + tristate "Compression acceleration support on PowerNV platform" + depends on PPC_POWERNV + default y + help + Support for PowerPC Nest (NX) compression acceleration. This + module supports acceleration for compressing memory with the 842 + algorithm. This supports NX hardware on the PowerNV platform. + If you choose 'M' here, this module will be called nx_compress_powernv. + +endif diff --git a/kernel/drivers/crypto/nx/Makefile b/kernel/drivers/crypto/nx/Makefile index bb770ea45..b727821c8 100644 --- a/kernel/drivers/crypto/nx/Makefile +++ b/kernel/drivers/crypto/nx/Makefile @@ -10,5 +10,8 @@ nx-crypto-objs := nx.o \ nx-sha256.o \ nx-sha512.o -obj-$(CONFIG_CRYPTO_DEV_NX_COMPRESS) += nx-compress.o +obj-$(CONFIG_CRYPTO_DEV_NX_COMPRESS_PSERIES) += nx-compress-pseries.o nx-compress.o +obj-$(CONFIG_CRYPTO_DEV_NX_COMPRESS_POWERNV) += nx-compress-powernv.o nx-compress.o nx-compress-objs := nx-842.o +nx-compress-pseries-objs := nx-842-pseries.o +nx-compress-powernv-objs := nx-842-powernv.o diff --git a/kernel/drivers/crypto/nx/nx-842-powernv.c b/kernel/drivers/crypto/nx/nx-842-powernv.c new file mode 100644 index 000000000..9ef51fafd --- /dev/null +++ b/kernel/drivers/crypto/nx/nx-842-powernv.c @@ -0,0 +1,653 @@ +/* + * Driver for IBM PowerNV 842 compression accelerator + * + * Copyright (C) 2015 Dan Streetman, IBM Corp + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include "nx-842.h" + +#include + +#include +#include + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Dan Streetman "); +MODULE_DESCRIPTION("842 H/W Compression driver for IBM PowerNV processors"); +MODULE_ALIAS_CRYPTO("842"); +MODULE_ALIAS_CRYPTO("842-nx"); + +#define WORKMEM_ALIGN (CRB_ALIGN) +#define CSB_WAIT_MAX (5000) /* ms */ + +struct nx842_workmem { + /* Below fields must be properly aligned */ + struct coprocessor_request_block crb; /* CRB_ALIGN align */ + struct data_descriptor_entry ddl_in[DDL_LEN_MAX]; /* DDE_ALIGN align */ + struct data_descriptor_entry ddl_out[DDL_LEN_MAX]; /* DDE_ALIGN align */ + /* Above fields must be properly aligned */ + + ktime_t start; + + char padding[WORKMEM_ALIGN]; /* unused, to allow alignment */ +} __packed __aligned(WORKMEM_ALIGN); + +struct nx842_coproc { + unsigned int chip_id; + unsigned int ct; + unsigned int ci; + struct list_head list; +}; + +/* no cpu hotplug on powernv, so this list never changes after init */ +static LIST_HEAD(nx842_coprocs); +static unsigned int nx842_ct; + +/** + * setup_indirect_dde - Setup an indirect DDE + * + * The DDE is setup with the the DDE count, byte count, and address of + * first direct DDE in the list. + */ +static void setup_indirect_dde(struct data_descriptor_entry *dde, + struct data_descriptor_entry *ddl, + unsigned int dde_count, unsigned int byte_count) +{ + dde->flags = 0; + dde->count = dde_count; + dde->index = 0; + dde->length = cpu_to_be32(byte_count); + dde->address = cpu_to_be64(nx842_get_pa(ddl)); +} + +/** + * setup_direct_dde - Setup single DDE from buffer + * + * The DDE is setup with the buffer and length. The buffer must be properly + * aligned. The used length is returned. + * Returns: + * N Successfully set up DDE with N bytes + */ +static unsigned int setup_direct_dde(struct data_descriptor_entry *dde, + unsigned long pa, unsigned int len) +{ + unsigned int l = min_t(unsigned int, len, LEN_ON_PAGE(pa)); + + dde->flags = 0; + dde->count = 0; + dde->index = 0; + dde->length = cpu_to_be32(l); + dde->address = cpu_to_be64(pa); + + return l; +} + +/** + * setup_ddl - Setup DDL from buffer + * + * Returns: + * 0 Successfully set up DDL + */ +static int setup_ddl(struct data_descriptor_entry *dde, + struct data_descriptor_entry *ddl, + unsigned char *buf, unsigned int len, + bool in) +{ + unsigned long pa = nx842_get_pa(buf); + int i, ret, total_len = len; + + if (!IS_ALIGNED(pa, DDE_BUFFER_ALIGN)) { + pr_debug("%s buffer pa 0x%lx not 0x%x-byte aligned\n", + in ? "input" : "output", pa, DDE_BUFFER_ALIGN); + return -EINVAL; + } + + /* only need to check last mult; since buffer must be + * DDE_BUFFER_ALIGN aligned, and that is a multiple of + * DDE_BUFFER_SIZE_MULT, and pre-last page DDE buffers + * are guaranteed a multiple of DDE_BUFFER_SIZE_MULT. + */ + if (len % DDE_BUFFER_LAST_MULT) { + pr_debug("%s buffer len 0x%x not a multiple of 0x%x\n", + in ? "input" : "output", len, DDE_BUFFER_LAST_MULT); + if (in) + return -EINVAL; + len = round_down(len, DDE_BUFFER_LAST_MULT); + } + + /* use a single direct DDE */ + if (len <= LEN_ON_PAGE(pa)) { + ret = setup_direct_dde(dde, pa, len); + WARN_ON(ret < len); + return 0; + } + + /* use the DDL */ + for (i = 0; i < DDL_LEN_MAX && len > 0; i++) { + ret = setup_direct_dde(&ddl[i], pa, len); + buf += ret; + len -= ret; + pa = nx842_get_pa(buf); + } + + if (len > 0) { + pr_debug("0x%x total %s bytes 0x%x too many for DDL.\n", + total_len, in ? "input" : "output", len); + if (in) + return -EMSGSIZE; + total_len -= len; + } + setup_indirect_dde(dde, ddl, i, total_len); + + return 0; +} + +#define CSB_ERR(csb, msg, ...) \ + pr_err("ERROR: " msg " : %02x %02x %02x %02x %08x\n", \ + ##__VA_ARGS__, (csb)->flags, \ + (csb)->cs, (csb)->cc, (csb)->ce, \ + be32_to_cpu((csb)->count)) + +#define CSB_ERR_ADDR(csb, msg, ...) \ + CSB_ERR(csb, msg " at %lx", ##__VA_ARGS__, \ + (unsigned long)be64_to_cpu((csb)->address)) + +/** + * wait_for_csb + */ +static int wait_for_csb(struct nx842_workmem *wmem, + struct coprocessor_status_block *csb) +{ + ktime_t start = wmem->start, now = ktime_get(); + ktime_t timeout = ktime_add_ms(start, CSB_WAIT_MAX); + + while (!(ACCESS_ONCE(csb->flags) & CSB_V)) { + cpu_relax(); + now = ktime_get(); + if (ktime_after(now, timeout)) + break; + } + + /* hw has updated csb and output buffer */ + barrier(); + + /* check CSB flags */ + if (!(csb->flags & CSB_V)) { + CSB_ERR(csb, "CSB still not valid after %ld us, giving up", + (long)ktime_us_delta(now, start)); + return -ETIMEDOUT; + } + if (csb->flags & CSB_F) { + CSB_ERR(csb, "Invalid CSB format"); + return -EPROTO; + } + if (csb->flags & CSB_CH) { + CSB_ERR(csb, "Invalid CSB chaining state"); + return -EPROTO; + } + + /* verify CSB completion sequence is 0 */ + if (csb->cs) { + CSB_ERR(csb, "Invalid CSB completion sequence"); + return -EPROTO; + } + + /* check CSB Completion Code */ + switch (csb->cc) { + /* no error */ + case CSB_CC_SUCCESS: + break; + case CSB_CC_TPBC_GT_SPBC: + /* not an error, but the compressed data is + * larger than the uncompressed data :( + */ + break; + + /* input data errors */ + case CSB_CC_OPERAND_OVERLAP: + /* input and output buffers overlap */ + CSB_ERR(csb, "Operand Overlap error"); + return -EINVAL; + case CSB_CC_INVALID_OPERAND: + CSB_ERR(csb, "Invalid operand"); + return -EINVAL; + case CSB_CC_NOSPC: + /* output buffer too small */ + return -ENOSPC; + case CSB_CC_ABORT: + CSB_ERR(csb, "Function aborted"); + return -EINTR; + case CSB_CC_CRC_MISMATCH: + CSB_ERR(csb, "CRC mismatch"); + return -EINVAL; + case CSB_CC_TEMPL_INVALID: + CSB_ERR(csb, "Compressed data template invalid"); + return -EINVAL; + case CSB_CC_TEMPL_OVERFLOW: + CSB_ERR(csb, "Compressed data template shows data past end"); + return -EINVAL; + + /* these should not happen */ + case CSB_CC_INVALID_ALIGN: + /* setup_ddl should have detected this */ + CSB_ERR_ADDR(csb, "Invalid alignment"); + return -EINVAL; + case CSB_CC_DATA_LENGTH: + /* setup_ddl should have detected this */ + CSB_ERR(csb, "Invalid data length"); + return -EINVAL; + case CSB_CC_WR_TRANSLATION: + case CSB_CC_TRANSLATION: + case CSB_CC_TRANSLATION_DUP1: + case CSB_CC_TRANSLATION_DUP2: + case CSB_CC_TRANSLATION_DUP3: + case CSB_CC_TRANSLATION_DUP4: + case CSB_CC_TRANSLATION_DUP5: + case CSB_CC_TRANSLATION_DUP6: + /* should not happen, we use physical addrs */ + CSB_ERR_ADDR(csb, "Translation error"); + return -EPROTO; + case CSB_CC_WR_PROTECTION: + case CSB_CC_PROTECTION: + case CSB_CC_PROTECTION_DUP1: + case CSB_CC_PROTECTION_DUP2: + case CSB_CC_PROTECTION_DUP3: + case CSB_CC_PROTECTION_DUP4: + case CSB_CC_PROTECTION_DUP5: + case CSB_CC_PROTECTION_DUP6: + /* should not happen, we use physical addrs */ + CSB_ERR_ADDR(csb, "Protection error"); + return -EPROTO; + case CSB_CC_PRIVILEGE: + /* shouldn't happen, we're in HYP mode */ + CSB_ERR(csb, "Insufficient Privilege error"); + return -EPROTO; + case CSB_CC_EXCESSIVE_DDE: + /* shouldn't happen, setup_ddl doesn't use many dde's */ + CSB_ERR(csb, "Too many DDEs in DDL"); + return -EINVAL; + case CSB_CC_TRANSPORT: + /* shouldn't happen, we setup CRB correctly */ + CSB_ERR(csb, "Invalid CRB"); + return -EINVAL; + case CSB_CC_SEGMENTED_DDL: + /* shouldn't happen, setup_ddl creates DDL right */ + CSB_ERR(csb, "Segmented DDL error"); + return -EINVAL; + case CSB_CC_DDE_OVERFLOW: + /* shouldn't happen, setup_ddl creates DDL right */ + CSB_ERR(csb, "DDE overflow error"); + return -EINVAL; + case CSB_CC_SESSION: + /* should not happen with ICSWX */ + CSB_ERR(csb, "Session violation error"); + return -EPROTO; + case CSB_CC_CHAIN: + /* should not happen, we don't use chained CRBs */ + CSB_ERR(csb, "Chained CRB error"); + return -EPROTO; + case CSB_CC_SEQUENCE: + /* should not happen, we don't use chained CRBs */ + CSB_ERR(csb, "CRB seqeunce number error"); + return -EPROTO; + case CSB_CC_UNKNOWN_CODE: + CSB_ERR(csb, "Unknown subfunction code"); + return -EPROTO; + + /* hardware errors */ + case CSB_CC_RD_EXTERNAL: + case CSB_CC_RD_EXTERNAL_DUP1: + case CSB_CC_RD_EXTERNAL_DUP2: + case CSB_CC_RD_EXTERNAL_DUP3: + CSB_ERR_ADDR(csb, "Read error outside coprocessor"); + return -EPROTO; + case CSB_CC_WR_EXTERNAL: + CSB_ERR_ADDR(csb, "Write error outside coprocessor"); + return -EPROTO; + case CSB_CC_INTERNAL: + CSB_ERR(csb, "Internal error in coprocessor"); + return -EPROTO; + case CSB_CC_PROVISION: + CSB_ERR(csb, "Storage provision error"); + return -EPROTO; + case CSB_CC_HW: + CSB_ERR(csb, "Correctable hardware error"); + return -EPROTO; + + default: + CSB_ERR(csb, "Invalid CC %d", csb->cc); + return -EPROTO; + } + + /* check Completion Extension state */ + if (csb->ce & CSB_CE_TERMINATION) { + CSB_ERR(csb, "CSB request was terminated"); + return -EPROTO; + } + if (csb->ce & CSB_CE_INCOMPLETE) { + CSB_ERR(csb, "CSB request not complete"); + return -EPROTO; + } + if (!(csb->ce & CSB_CE_TPBC)) { + CSB_ERR(csb, "TPBC not provided, unknown target length"); + return -EPROTO; + } + + /* successful completion */ + pr_debug_ratelimited("Processed %u bytes in %lu us\n", + be32_to_cpu(csb->count), + (unsigned long)ktime_us_delta(now, start)); + + return 0; +} + +/** + * nx842_powernv_function - compress/decompress data using the 842 algorithm + * + * (De)compression provided by the NX842 coprocessor on IBM PowerNV systems. + * This compresses or decompresses the provided input buffer into the provided + * output buffer. + * + * Upon return from this function @outlen contains the length of the + * output data. If there is an error then @outlen will be 0 and an + * error will be specified by the return code from this function. + * + * The @workmem buffer should only be used by one function call at a time. + * + * @in: input buffer pointer + * @inlen: input buffer size + * @out: output buffer pointer + * @outlenp: output buffer size pointer + * @workmem: working memory buffer pointer, size determined by + * nx842_powernv_driver.workmem_size + * @fc: function code, see CCW Function Codes in nx-842.h + * + * Returns: + * 0 Success, output of length @outlenp stored in the buffer at @out + * -ENODEV Hardware unavailable + * -ENOSPC Output buffer is to small + * -EMSGSIZE Input buffer too large + * -EINVAL buffer constraints do not fix nx842_constraints + * -EPROTO hardware error during operation + * -ETIMEDOUT hardware did not complete operation in reasonable time + * -EINTR operation was aborted + */ +static int nx842_powernv_function(const unsigned char *in, unsigned int inlen, + unsigned char *out, unsigned int *outlenp, + void *workmem, int fc) +{ + struct coprocessor_request_block *crb; + struct coprocessor_status_block *csb; + struct nx842_workmem *wmem; + int ret; + u64 csb_addr; + u32 ccw; + unsigned int outlen = *outlenp; + + wmem = PTR_ALIGN(workmem, WORKMEM_ALIGN); + + *outlenp = 0; + + /* shoudn't happen, we don't load without a coproc */ + if (!nx842_ct) { + pr_err_ratelimited("coprocessor CT is 0"); + return -ENODEV; + } + + crb = &wmem->crb; + csb = &crb->csb; + + /* Clear any previous values */ + memset(crb, 0, sizeof(*crb)); + + /* set up DDLs */ + ret = setup_ddl(&crb->source, wmem->ddl_in, + (unsigned char *)in, inlen, true); + if (ret) + return ret; + ret = setup_ddl(&crb->target, wmem->ddl_out, + out, outlen, false); + if (ret) + return ret; + + /* set up CCW */ + ccw = 0; + ccw = SET_FIELD(ccw, CCW_CT, nx842_ct); + ccw = SET_FIELD(ccw, CCW_CI_842, 0); /* use 0 for hw auto-selection */ + ccw = SET_FIELD(ccw, CCW_FC_842, fc); + + /* set up CRB's CSB addr */ + csb_addr = nx842_get_pa(csb) & CRB_CSB_ADDRESS; + csb_addr |= CRB_CSB_AT; /* Addrs are phys */ + crb->csb_addr = cpu_to_be64(csb_addr); + + wmem->start = ktime_get(); + + /* do ICSWX */ + ret = icswx(cpu_to_be32(ccw), crb); + + pr_debug_ratelimited("icswx CR %x ccw %x crb->ccw %x\n", ret, + (unsigned int)ccw, + (unsigned int)be32_to_cpu(crb->ccw)); + + switch (ret) { + case ICSWX_INITIATED: + ret = wait_for_csb(wmem, csb); + break; + case ICSWX_BUSY: + pr_debug_ratelimited("842 Coprocessor busy\n"); + ret = -EBUSY; + break; + case ICSWX_REJECTED: + pr_err_ratelimited("ICSWX rejected\n"); + ret = -EPROTO; + break; + default: + pr_err_ratelimited("Invalid ICSWX return code %x\n", ret); + ret = -EPROTO; + break; + } + + if (!ret) + *outlenp = be32_to_cpu(csb->count); + + return ret; +} + +/** + * nx842_powernv_compress - Compress data using the 842 algorithm + * + * Compression provided by the NX842 coprocessor on IBM PowerNV systems. + * The input buffer is compressed and the result is stored in the + * provided output buffer. + * + * Upon return from this function @outlen contains the length of the + * compressed data. If there is an error then @outlen will be 0 and an + * error will be specified by the return code from this function. + * + * @in: input buffer pointer + * @inlen: input buffer size + * @out: output buffer pointer + * @outlenp: output buffer size pointer + * @workmem: working memory buffer pointer, size determined by + * nx842_powernv_driver.workmem_size + * + * Returns: see @nx842_powernv_function() + */ +static int nx842_powernv_compress(const unsigned char *in, unsigned int inlen, + unsigned char *out, unsigned int *outlenp, + void *wmem) +{ + return nx842_powernv_function(in, inlen, out, outlenp, + wmem, CCW_FC_842_COMP_CRC); +} + +/** + * nx842_powernv_decompress - Decompress data using the 842 algorithm + * + * Decompression provided by the NX842 coprocessor on IBM PowerNV systems. + * The input buffer is decompressed and the result is stored in the + * provided output buffer. + * + * Upon return from this function @outlen contains the length of the + * decompressed data. If there is an error then @outlen will be 0 and an + * error will be specified by the return code from this function. + * + * @in: input buffer pointer + * @inlen: input buffer size + * @out: output buffer pointer + * @outlenp: output buffer size pointer + * @workmem: working memory buffer pointer, size determined by + * nx842_powernv_driver.workmem_size + * + * Returns: see @nx842_powernv_function() + */ +static int nx842_powernv_decompress(const unsigned char *in, unsigned int inlen, + unsigned char *out, unsigned int *outlenp, + void *wmem) +{ + return nx842_powernv_function(in, inlen, out, outlenp, + wmem, CCW_FC_842_DECOMP_CRC); +} + +static int __init nx842_powernv_probe(struct device_node *dn) +{ + struct nx842_coproc *coproc; + struct property *ct_prop, *ci_prop; + unsigned int ct, ci; + int chip_id; + + chip_id = of_get_ibm_chip_id(dn); + if (chip_id < 0) { + pr_err("ibm,chip-id missing\n"); + return -EINVAL; + } + ct_prop = of_find_property(dn, "ibm,842-coprocessor-type", NULL); + if (!ct_prop) { + pr_err("ibm,842-coprocessor-type missing\n"); + return -EINVAL; + } + ct = be32_to_cpu(*(unsigned int *)ct_prop->value); + ci_prop = of_find_property(dn, "ibm,842-coprocessor-instance", NULL); + if (!ci_prop) { + pr_err("ibm,842-coprocessor-instance missing\n"); + return -EINVAL; + } + ci = be32_to_cpu(*(unsigned int *)ci_prop->value); + + coproc = kmalloc(sizeof(*coproc), GFP_KERNEL); + if (!coproc) + return -ENOMEM; + + coproc->chip_id = chip_id; + coproc->ct = ct; + coproc->ci = ci; + INIT_LIST_HEAD(&coproc->list); + list_add(&coproc->list, &nx842_coprocs); + + pr_info("coprocessor found on chip %d, CT %d CI %d\n", chip_id, ct, ci); + + if (!nx842_ct) + nx842_ct = ct; + else if (nx842_ct != ct) + pr_err("NX842 chip %d, CT %d != first found CT %d\n", + chip_id, ct, nx842_ct); + + return 0; +} + +static struct nx842_constraints nx842_powernv_constraints = { + .alignment = DDE_BUFFER_ALIGN, + .multiple = DDE_BUFFER_LAST_MULT, + .minimum = DDE_BUFFER_LAST_MULT, + .maximum = (DDL_LEN_MAX - 1) * PAGE_SIZE, +}; + +static struct nx842_driver nx842_powernv_driver = { + .name = KBUILD_MODNAME, + .owner = THIS_MODULE, + .workmem_size = sizeof(struct nx842_workmem), + .constraints = &nx842_powernv_constraints, + .compress = nx842_powernv_compress, + .decompress = nx842_powernv_decompress, +}; + +static int nx842_powernv_crypto_init(struct crypto_tfm *tfm) +{ + return nx842_crypto_init(tfm, &nx842_powernv_driver); +} + +static struct crypto_alg nx842_powernv_alg = { + .cra_name = "842", + .cra_driver_name = "842-nx", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_COMPRESS, + .cra_ctxsize = sizeof(struct nx842_crypto_ctx), + .cra_module = THIS_MODULE, + .cra_init = nx842_powernv_crypto_init, + .cra_exit = nx842_crypto_exit, + .cra_u = { .compress = { + .coa_compress = nx842_crypto_compress, + .coa_decompress = nx842_crypto_decompress } } +}; + +static __init int nx842_powernv_init(void) +{ + struct device_node *dn; + int ret; + + /* verify workmem size/align restrictions */ + BUILD_BUG_ON(WORKMEM_ALIGN % CRB_ALIGN); + BUILD_BUG_ON(CRB_ALIGN % DDE_ALIGN); + BUILD_BUG_ON(CRB_SIZE % DDE_ALIGN); + /* verify buffer size/align restrictions */ + BUILD_BUG_ON(PAGE_SIZE % DDE_BUFFER_ALIGN); + BUILD_BUG_ON(DDE_BUFFER_ALIGN % DDE_BUFFER_SIZE_MULT); + BUILD_BUG_ON(DDE_BUFFER_SIZE_MULT % DDE_BUFFER_LAST_MULT); + + for_each_compatible_node(dn, NULL, "ibm,power-nx") + nx842_powernv_probe(dn); + + if (!nx842_ct) + return -ENODEV; + + ret = crypto_register_alg(&nx842_powernv_alg); + if (ret) { + struct nx842_coproc *coproc, *n; + + list_for_each_entry_safe(coproc, n, &nx842_coprocs, list) { + list_del(&coproc->list); + kfree(coproc); + } + + return ret; + } + + return 0; +} +module_init(nx842_powernv_init); + +static void __exit nx842_powernv_exit(void) +{ + struct nx842_coproc *coproc, *n; + + crypto_unregister_alg(&nx842_powernv_alg); + + list_for_each_entry_safe(coproc, n, &nx842_coprocs, list) { + list_del(&coproc->list); + kfree(coproc); + } +} +module_exit(nx842_powernv_exit); diff --git a/kernel/drivers/crypto/nx/nx-842-pseries.c b/kernel/drivers/crypto/nx/nx-842-pseries.c new file mode 100644 index 000000000..cddc6d8b5 --- /dev/null +++ b/kernel/drivers/crypto/nx/nx-842-pseries.c @@ -0,0 +1,1147 @@ +/* + * Driver for IBM Power 842 compression accelerator + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + * Copyright (C) IBM Corporation, 2012 + * + * Authors: Robert Jennings + * Seth Jennings + */ + +#include + +#include "nx-842.h" +#include "nx_csbcpb.h" /* struct nx_csbcpb */ + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Robert Jennings "); +MODULE_DESCRIPTION("842 H/W Compression driver for IBM Power processors"); +MODULE_ALIAS_CRYPTO("842"); +MODULE_ALIAS_CRYPTO("842-nx"); + +static struct nx842_constraints nx842_pseries_constraints = { + .alignment = DDE_BUFFER_ALIGN, + .multiple = DDE_BUFFER_LAST_MULT, + .minimum = DDE_BUFFER_LAST_MULT, + .maximum = PAGE_SIZE, /* dynamic, max_sync_size */ +}; + +static int check_constraints(unsigned long buf, unsigned int *len, bool in) +{ + if (!IS_ALIGNED(buf, nx842_pseries_constraints.alignment)) { + pr_debug("%s buffer 0x%lx not aligned to 0x%x\n", + in ? "input" : "output", buf, + nx842_pseries_constraints.alignment); + return -EINVAL; + } + if (*len % nx842_pseries_constraints.multiple) { + pr_debug("%s buffer len 0x%x not multiple of 0x%x\n", + in ? "input" : "output", *len, + nx842_pseries_constraints.multiple); + if (in) + return -EINVAL; + *len = round_down(*len, nx842_pseries_constraints.multiple); + } + if (*len < nx842_pseries_constraints.minimum) { + pr_debug("%s buffer len 0x%x under minimum 0x%x\n", + in ? "input" : "output", *len, + nx842_pseries_constraints.minimum); + return -EINVAL; + } + if (*len > nx842_pseries_constraints.maximum) { + pr_debug("%s buffer len 0x%x over maximum 0x%x\n", + in ? "input" : "output", *len, + nx842_pseries_constraints.maximum); + if (in) + return -EINVAL; + *len = nx842_pseries_constraints.maximum; + } + return 0; +} + +/* I assume we need to align the CSB? */ +#define WORKMEM_ALIGN (256) + +struct nx842_workmem { + /* scatterlist */ + char slin[4096]; + char slout[4096]; + /* coprocessor status/parameter block */ + struct nx_csbcpb csbcpb; + + char padding[WORKMEM_ALIGN]; +} __aligned(WORKMEM_ALIGN); + +/* Macros for fields within nx_csbcpb */ +/* Check the valid bit within the csbcpb valid field */ +#define NX842_CSBCBP_VALID_CHK(x) (x & BIT_MASK(7)) + +/* CE macros operate on the completion_extension field bits in the csbcpb. + * CE0 0=full completion, 1=partial completion + * CE1 0=CE0 indicates completion, 1=termination (output may be modified) + * CE2 0=processed_bytes is source bytes, 1=processed_bytes is target bytes */ +#define NX842_CSBCPB_CE0(x) (x & BIT_MASK(7)) +#define NX842_CSBCPB_CE1(x) (x & BIT_MASK(6)) +#define NX842_CSBCPB_CE2(x) (x & BIT_MASK(5)) + +/* The NX unit accepts data only on 4K page boundaries */ +#define NX842_HW_PAGE_SIZE (4096) +#define NX842_HW_PAGE_MASK (~(NX842_HW_PAGE_SIZE-1)) + +struct ibm_nx842_counters { + atomic64_t comp_complete; + atomic64_t comp_failed; + atomic64_t decomp_complete; + atomic64_t decomp_failed; + atomic64_t swdecomp; + atomic64_t comp_times[32]; + atomic64_t decomp_times[32]; +}; + +static struct nx842_devdata { + struct vio_dev *vdev; + struct device *dev; + struct ibm_nx842_counters *counters; + unsigned int max_sg_len; + unsigned int max_sync_size; + unsigned int max_sync_sg; +} __rcu *devdata; +static DEFINE_SPINLOCK(devdata_mutex); + +#define NX842_COUNTER_INC(_x) \ +static inline void nx842_inc_##_x( \ + const struct nx842_devdata *dev) { \ + if (dev) \ + atomic64_inc(&dev->counters->_x); \ +} +NX842_COUNTER_INC(comp_complete); +NX842_COUNTER_INC(comp_failed); +NX842_COUNTER_INC(decomp_complete); +NX842_COUNTER_INC(decomp_failed); +NX842_COUNTER_INC(swdecomp); + +#define NX842_HIST_SLOTS 16 + +static void ibm_nx842_incr_hist(atomic64_t *times, unsigned int time) +{ + int bucket = fls(time); + + if (bucket) + bucket = min((NX842_HIST_SLOTS - 1), bucket - 1); + + atomic64_inc(×[bucket]); +} + +/* NX unit operation flags */ +#define NX842_OP_COMPRESS 0x0 +#define NX842_OP_CRC 0x1 +#define NX842_OP_DECOMPRESS 0x2 +#define NX842_OP_COMPRESS_CRC (NX842_OP_COMPRESS | NX842_OP_CRC) +#define NX842_OP_DECOMPRESS_CRC (NX842_OP_DECOMPRESS | NX842_OP_CRC) +#define NX842_OP_ASYNC (1<<23) +#define NX842_OP_NOTIFY (1<<22) +#define NX842_OP_NOTIFY_INT(x) ((x & 0xff)<<8) + +static unsigned long nx842_get_desired_dma(struct vio_dev *viodev) +{ + /* No use of DMA mappings within the driver. */ + return 0; +} + +struct nx842_slentry { + __be64 ptr; /* Real address (use __pa()) */ + __be64 len; +}; + +/* pHyp scatterlist entry */ +struct nx842_scatterlist { + int entry_nr; /* number of slentries */ + struct nx842_slentry *entries; /* ptr to array of slentries */ +}; + +/* Does not include sizeof(entry_nr) in the size */ +static inline unsigned long nx842_get_scatterlist_size( + struct nx842_scatterlist *sl) +{ + return sl->entry_nr * sizeof(struct nx842_slentry); +} + +static int nx842_build_scatterlist(unsigned long buf, int len, + struct nx842_scatterlist *sl) +{ + unsigned long entrylen; + struct nx842_slentry *entry; + + sl->entry_nr = 0; + + entry = sl->entries; + while (len) { + entry->ptr = cpu_to_be64(nx842_get_pa((void *)buf)); + entrylen = min_t(int, len, + LEN_ON_SIZE(buf, NX842_HW_PAGE_SIZE)); + entry->len = cpu_to_be64(entrylen); + + len -= entrylen; + buf += entrylen; + + sl->entry_nr++; + entry++; + } + + return 0; +} + +static int nx842_validate_result(struct device *dev, + struct cop_status_block *csb) +{ + /* The csb must be valid after returning from vio_h_cop_sync */ + if (!NX842_CSBCBP_VALID_CHK(csb->valid)) { + dev_err(dev, "%s: cspcbp not valid upon completion.\n", + __func__); + dev_dbg(dev, "valid:0x%02x cs:0x%02x cc:0x%02x ce:0x%02x\n", + csb->valid, + csb->crb_seq_number, + csb->completion_code, + csb->completion_extension); + dev_dbg(dev, "processed_bytes:%d address:0x%016lx\n", + be32_to_cpu(csb->processed_byte_count), + (unsigned long)be64_to_cpu(csb->address)); + return -EIO; + } + + /* Check return values from the hardware in the CSB */ + switch (csb->completion_code) { + case 0: /* Completed without error */ + break; + case 64: /* Compression ok, but output larger than input */ + dev_dbg(dev, "%s: output size larger than input size\n", + __func__); + break; + case 13: /* Output buffer too small */ + dev_dbg(dev, "%s: Out of space in output buffer\n", + __func__); + return -ENOSPC; + case 65: /* Calculated CRC doesn't match the passed value */ + dev_dbg(dev, "%s: CRC mismatch for decompression\n", + __func__); + return -EINVAL; + case 66: /* Input data contains an illegal template field */ + case 67: /* Template indicates data past the end of the input stream */ + dev_dbg(dev, "%s: Bad data for decompression (code:%d)\n", + __func__, csb->completion_code); + return -EINVAL; + default: + dev_dbg(dev, "%s: Unspecified error (code:%d)\n", + __func__, csb->completion_code); + return -EIO; + } + + /* Hardware sanity check */ + if (!NX842_CSBCPB_CE2(csb->completion_extension)) { + dev_err(dev, "%s: No error returned by hardware, but " + "data returned is unusable, contact support.\n" + "(Additional info: csbcbp->processed bytes " + "does not specify processed bytes for the " + "target buffer.)\n", __func__); + return -EIO; + } + + return 0; +} + +/** + * nx842_pseries_compress - Compress data using the 842 algorithm + * + * Compression provide by the NX842 coprocessor on IBM Power systems. + * The input buffer is compressed and the result is stored in the + * provided output buffer. + * + * Upon return from this function @outlen contains the length of the + * compressed data. If there is an error then @outlen will be 0 and an + * error will be specified by the return code from this function. + * + * @in: Pointer to input buffer + * @inlen: Length of input buffer + * @out: Pointer to output buffer + * @outlen: Length of output buffer + * @wrkmem: ptr to buffer for working memory, size determined by + * nx842_pseries_driver.workmem_size + * + * Returns: + * 0 Success, output of length @outlen stored in the buffer at @out + * -ENOMEM Unable to allocate internal buffers + * -ENOSPC Output buffer is to small + * -EIO Internal error + * -ENODEV Hardware unavailable + */ +static int nx842_pseries_compress(const unsigned char *in, unsigned int inlen, + unsigned char *out, unsigned int *outlen, + void *wmem) +{ + struct nx842_devdata *local_devdata; + struct device *dev = NULL; + struct nx842_workmem *workmem; + struct nx842_scatterlist slin, slout; + struct nx_csbcpb *csbcpb; + int ret = 0, max_sync_size; + unsigned long inbuf, outbuf; + struct vio_pfo_op op = { + .done = NULL, + .handle = 0, + .timeout = 0, + }; + unsigned long start = get_tb(); + + inbuf = (unsigned long)in; + if (check_constraints(inbuf, &inlen, true)) + return -EINVAL; + + outbuf = (unsigned long)out; + if (check_constraints(outbuf, outlen, false)) + return -EINVAL; + + rcu_read_lock(); + local_devdata = rcu_dereference(devdata); + if (!local_devdata || !local_devdata->dev) { + rcu_read_unlock(); + return -ENODEV; + } + max_sync_size = local_devdata->max_sync_size; + dev = local_devdata->dev; + + /* Init scatterlist */ + workmem = PTR_ALIGN(wmem, WORKMEM_ALIGN); + slin.entries = (struct nx842_slentry *)workmem->slin; + slout.entries = (struct nx842_slentry *)workmem->slout; + + /* Init operation */ + op.flags = NX842_OP_COMPRESS_CRC; + csbcpb = &workmem->csbcpb; + memset(csbcpb, 0, sizeof(*csbcpb)); + op.csbcpb = nx842_get_pa(csbcpb); + + if ((inbuf & NX842_HW_PAGE_MASK) == + ((inbuf + inlen - 1) & NX842_HW_PAGE_MASK)) { + /* Create direct DDE */ + op.in = nx842_get_pa((void *)inbuf); + op.inlen = inlen; + } else { + /* Create indirect DDE (scatterlist) */ + nx842_build_scatterlist(inbuf, inlen, &slin); + op.in = nx842_get_pa(slin.entries); + op.inlen = -nx842_get_scatterlist_size(&slin); + } + + if ((outbuf & NX842_HW_PAGE_MASK) == + ((outbuf + *outlen - 1) & NX842_HW_PAGE_MASK)) { + /* Create direct DDE */ + op.out = nx842_get_pa((void *)outbuf); + op.outlen = *outlen; + } else { + /* Create indirect DDE (scatterlist) */ + nx842_build_scatterlist(outbuf, *outlen, &slout); + op.out = nx842_get_pa(slout.entries); + op.outlen = -nx842_get_scatterlist_size(&slout); + } + + dev_dbg(dev, "%s: op.in %lx op.inlen %ld op.out %lx op.outlen %ld\n", + __func__, (unsigned long)op.in, (long)op.inlen, + (unsigned long)op.out, (long)op.outlen); + + /* Send request to pHyp */ + ret = vio_h_cop_sync(local_devdata->vdev, &op); + + /* Check for pHyp error */ + if (ret) { + dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n", + __func__, ret, op.hcall_err); + ret = -EIO; + goto unlock; + } + + /* Check for hardware error */ + ret = nx842_validate_result(dev, &csbcpb->csb); + if (ret) + goto unlock; + + *outlen = be32_to_cpu(csbcpb->csb.processed_byte_count); + dev_dbg(dev, "%s: processed_bytes=%d\n", __func__, *outlen); + +unlock: + if (ret) + nx842_inc_comp_failed(local_devdata); + else { + nx842_inc_comp_complete(local_devdata); + ibm_nx842_incr_hist(local_devdata->counters->comp_times, + (get_tb() - start) / tb_ticks_per_usec); + } + rcu_read_unlock(); + return ret; +} + +/** + * nx842_pseries_decompress - Decompress data using the 842 algorithm + * + * Decompression provide by the NX842 coprocessor on IBM Power systems. + * The input buffer is decompressed and the result is stored in the + * provided output buffer. The size allocated to the output buffer is + * provided by the caller of this function in @outlen. Upon return from + * this function @outlen contains the length of the decompressed data. + * If there is an error then @outlen will be 0 and an error will be + * specified by the return code from this function. + * + * @in: Pointer to input buffer + * @inlen: Length of input buffer + * @out: Pointer to output buffer + * @outlen: Length of output buffer + * @wrkmem: ptr to buffer for working memory, size determined by + * nx842_pseries_driver.workmem_size + * + * Returns: + * 0 Success, output of length @outlen stored in the buffer at @out + * -ENODEV Hardware decompression device is unavailable + * -ENOMEM Unable to allocate internal buffers + * -ENOSPC Output buffer is to small + * -EINVAL Bad input data encountered when attempting decompress + * -EIO Internal error + */ +static int nx842_pseries_decompress(const unsigned char *in, unsigned int inlen, + unsigned char *out, unsigned int *outlen, + void *wmem) +{ + struct nx842_devdata *local_devdata; + struct device *dev = NULL; + struct nx842_workmem *workmem; + struct nx842_scatterlist slin, slout; + struct nx_csbcpb *csbcpb; + int ret = 0, max_sync_size; + unsigned long inbuf, outbuf; + struct vio_pfo_op op = { + .done = NULL, + .handle = 0, + .timeout = 0, + }; + unsigned long start = get_tb(); + + /* Ensure page alignment and size */ + inbuf = (unsigned long)in; + if (check_constraints(inbuf, &inlen, true)) + return -EINVAL; + + outbuf = (unsigned long)out; + if (check_constraints(outbuf, outlen, false)) + return -EINVAL; + + rcu_read_lock(); + local_devdata = rcu_dereference(devdata); + if (!local_devdata || !local_devdata->dev) { + rcu_read_unlock(); + return -ENODEV; + } + max_sync_size = local_devdata->max_sync_size; + dev = local_devdata->dev; + + workmem = PTR_ALIGN(wmem, WORKMEM_ALIGN); + + /* Init scatterlist */ + slin.entries = (struct nx842_slentry *)workmem->slin; + slout.entries = (struct nx842_slentry *)workmem->slout; + + /* Init operation */ + op.flags = NX842_OP_DECOMPRESS_CRC; + csbcpb = &workmem->csbcpb; + memset(csbcpb, 0, sizeof(*csbcpb)); + op.csbcpb = nx842_get_pa(csbcpb); + + if ((inbuf & NX842_HW_PAGE_MASK) == + ((inbuf + inlen - 1) & NX842_HW_PAGE_MASK)) { + /* Create direct DDE */ + op.in = nx842_get_pa((void *)inbuf); + op.inlen = inlen; + } else { + /* Create indirect DDE (scatterlist) */ + nx842_build_scatterlist(inbuf, inlen, &slin); + op.in = nx842_get_pa(slin.entries); + op.inlen = -nx842_get_scatterlist_size(&slin); + } + + if ((outbuf & NX842_HW_PAGE_MASK) == + ((outbuf + *outlen - 1) & NX842_HW_PAGE_MASK)) { + /* Create direct DDE */ + op.out = nx842_get_pa((void *)outbuf); + op.outlen = *outlen; + } else { + /* Create indirect DDE (scatterlist) */ + nx842_build_scatterlist(outbuf, *outlen, &slout); + op.out = nx842_get_pa(slout.entries); + op.outlen = -nx842_get_scatterlist_size(&slout); + } + + dev_dbg(dev, "%s: op.in %lx op.inlen %ld op.out %lx op.outlen %ld\n", + __func__, (unsigned long)op.in, (long)op.inlen, + (unsigned long)op.out, (long)op.outlen); + + /* Send request to pHyp */ + ret = vio_h_cop_sync(local_devdata->vdev, &op); + + /* Check for pHyp error */ + if (ret) { + dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n", + __func__, ret, op.hcall_err); + goto unlock; + } + + /* Check for hardware error */ + ret = nx842_validate_result(dev, &csbcpb->csb); + if (ret) + goto unlock; + + *outlen = be32_to_cpu(csbcpb->csb.processed_byte_count); + +unlock: + if (ret) + /* decompress fail */ + nx842_inc_decomp_failed(local_devdata); + else { + nx842_inc_decomp_complete(local_devdata); + ibm_nx842_incr_hist(local_devdata->counters->decomp_times, + (get_tb() - start) / tb_ticks_per_usec); + } + + rcu_read_unlock(); + return ret; +} + +/** + * nx842_OF_set_defaults -- Set default (disabled) values for devdata + * + * @devdata - struct nx842_devdata to update + * + * Returns: + * 0 on success + * -ENOENT if @devdata ptr is NULL + */ +static int nx842_OF_set_defaults(struct nx842_devdata *devdata) +{ + if (devdata) { + devdata->max_sync_size = 0; + devdata->max_sync_sg = 0; + devdata->max_sg_len = 0; + return 0; + } else + return -ENOENT; +} + +/** + * nx842_OF_upd_status -- Check the device info from OF status prop + * + * The status property indicates if the accelerator is enabled. If the + * device is in the OF tree it indicates that the hardware is present. + * The status field indicates if the device is enabled when the status + * is 'okay'. Otherwise the device driver will be disabled. + * + * @prop - struct property point containing the maxsyncop for the update + * + * Returns: + * 0 - Device is available + * -ENODEV - Device is not available + */ +static int nx842_OF_upd_status(struct property *prop) +{ + const char *status = (const char *)prop->value; + + if (!strncmp(status, "okay", (size_t)prop->length)) + return 0; + if (!strncmp(status, "disabled", (size_t)prop->length)) + return -ENODEV; + dev_info(devdata->dev, "%s: unknown status '%s'\n", __func__, status); + + return -EINVAL; +} + +/** + * nx842_OF_upd_maxsglen -- Update the device info from OF maxsglen prop + * + * Definition of the 'ibm,max-sg-len' OF property: + * This field indicates the maximum byte length of a scatter list + * for the platform facility. It is a single cell encoded as with encode-int. + * + * Example: + * # od -x ibm,max-sg-len + * 0000000 0000 0ff0 + * + * In this example, the maximum byte length of a scatter list is + * 0x0ff0 (4,080). + * + * @devdata - struct nx842_devdata to update + * @prop - struct property point containing the maxsyncop for the update + * + * Returns: + * 0 on success + * -EINVAL on failure + */ +static int nx842_OF_upd_maxsglen(struct nx842_devdata *devdata, + struct property *prop) { + int ret = 0; + const unsigned int maxsglen = of_read_number(prop->value, 1); + + if (prop->length != sizeof(maxsglen)) { + dev_err(devdata->dev, "%s: unexpected format for ibm,max-sg-len property\n", __func__); + dev_dbg(devdata->dev, "%s: ibm,max-sg-len is %d bytes long, expected %lu bytes\n", __func__, + prop->length, sizeof(maxsglen)); + ret = -EINVAL; + } else { + devdata->max_sg_len = min_t(unsigned int, + maxsglen, NX842_HW_PAGE_SIZE); + } + + return ret; +} + +/** + * nx842_OF_upd_maxsyncop -- Update the device info from OF maxsyncop prop + * + * Definition of the 'ibm,max-sync-cop' OF property: + * Two series of cells. The first series of cells represents the maximums + * that can be synchronously compressed. The second series of cells + * represents the maximums that can be synchronously decompressed. + * 1. The first cell in each series contains the count of the number of + * data length, scatter list elements pairs that follow – each being + * of the form + * a. One cell data byte length + * b. One cell total number of scatter list elements + * + * Example: + * # od -x ibm,max-sync-cop + * 0000000 0000 0001 0000 1000 0000 01fe 0000 0001 + * 0000020 0000 1000 0000 01fe + * + * In this example, compression supports 0x1000 (4,096) data byte length + * and 0x1fe (510) total scatter list elements. Decompression supports + * 0x1000 (4,096) data byte length and 0x1f3 (510) total scatter list + * elements. + * + * @devdata - struct nx842_devdata to update + * @prop - struct property point containing the maxsyncop for the update + * + * Returns: + * 0 on success + * -EINVAL on failure + */ +static int nx842_OF_upd_maxsyncop(struct nx842_devdata *devdata, + struct property *prop) { + int ret = 0; + unsigned int comp_data_limit, decomp_data_limit; + unsigned int comp_sg_limit, decomp_sg_limit; + const struct maxsynccop_t { + __be32 comp_elements; + __be32 comp_data_limit; + __be32 comp_sg_limit; + __be32 decomp_elements; + __be32 decomp_data_limit; + __be32 decomp_sg_limit; + } *maxsynccop; + + if (prop->length != sizeof(*maxsynccop)) { + dev_err(devdata->dev, "%s: unexpected format for ibm,max-sync-cop property\n", __func__); + dev_dbg(devdata->dev, "%s: ibm,max-sync-cop is %d bytes long, expected %lu bytes\n", __func__, prop->length, + sizeof(*maxsynccop)); + ret = -EINVAL; + goto out; + } + + maxsynccop = (const struct maxsynccop_t *)prop->value; + comp_data_limit = be32_to_cpu(maxsynccop->comp_data_limit); + comp_sg_limit = be32_to_cpu(maxsynccop->comp_sg_limit); + decomp_data_limit = be32_to_cpu(maxsynccop->decomp_data_limit); + decomp_sg_limit = be32_to_cpu(maxsynccop->decomp_sg_limit); + + /* Use one limit rather than separate limits for compression and + * decompression. Set a maximum for this so as not to exceed the + * size that the header can support and round the value down to + * the hardware page size (4K) */ + devdata->max_sync_size = min(comp_data_limit, decomp_data_limit); + + devdata->max_sync_size = min_t(unsigned int, devdata->max_sync_size, + 65536); + + if (devdata->max_sync_size < 4096) { + dev_err(devdata->dev, "%s: hardware max data size (%u) is " + "less than the driver minimum, unable to use " + "the hardware device\n", + __func__, devdata->max_sync_size); + ret = -EINVAL; + goto out; + } + + nx842_pseries_constraints.maximum = devdata->max_sync_size; + + devdata->max_sync_sg = min(comp_sg_limit, decomp_sg_limit); + if (devdata->max_sync_sg < 1) { + dev_err(devdata->dev, "%s: hardware max sg size (%u) is " + "less than the driver minimum, unable to use " + "the hardware device\n", + __func__, devdata->max_sync_sg); + ret = -EINVAL; + goto out; + } + +out: + return ret; +} + +/** + * + * nx842_OF_upd -- Handle OF properties updates for the device. + * + * Set all properties from the OF tree. Optionally, a new property + * can be provided by the @new_prop pointer to overwrite an existing value. + * The device will remain disabled until all values are valid, this function + * will return an error for updates unless all values are valid. + * + * @new_prop: If not NULL, this property is being updated. If NULL, update + * all properties from the current values in the OF tree. + * + * Returns: + * 0 - Success + * -ENOMEM - Could not allocate memory for new devdata structure + * -EINVAL - property value not found, new_prop is not a recognized + * property for the device or property value is not valid. + * -ENODEV - Device is not available + */ +static int nx842_OF_upd(struct property *new_prop) +{ + struct nx842_devdata *old_devdata = NULL; + struct nx842_devdata *new_devdata = NULL; + struct device_node *of_node = NULL; + struct property *status = NULL; + struct property *maxsglen = NULL; + struct property *maxsyncop = NULL; + int ret = 0; + unsigned long flags; + + new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS); + if (!new_devdata) + return -ENOMEM; + + spin_lock_irqsave(&devdata_mutex, flags); + old_devdata = rcu_dereference_check(devdata, + lockdep_is_held(&devdata_mutex)); + if (old_devdata) + of_node = old_devdata->dev->of_node; + + if (!old_devdata || !of_node) { + pr_err("%s: device is not available\n", __func__); + spin_unlock_irqrestore(&devdata_mutex, flags); + kfree(new_devdata); + return -ENODEV; + } + + memcpy(new_devdata, old_devdata, sizeof(*old_devdata)); + new_devdata->counters = old_devdata->counters; + + /* Set ptrs for existing properties */ + status = of_find_property(of_node, "status", NULL); + maxsglen = of_find_property(of_node, "ibm,max-sg-len", NULL); + maxsyncop = of_find_property(of_node, "ibm,max-sync-cop", NULL); + if (!status || !maxsglen || !maxsyncop) { + dev_err(old_devdata->dev, "%s: Could not locate device properties\n", __func__); + ret = -EINVAL; + goto error_out; + } + + /* + * If this is a property update, there are only certain properties that + * we care about. Bail if it isn't in the below list + */ + if (new_prop && (strncmp(new_prop->name, "status", new_prop->length) || + strncmp(new_prop->name, "ibm,max-sg-len", new_prop->length) || + strncmp(new_prop->name, "ibm,max-sync-cop", new_prop->length))) + goto out; + + /* Perform property updates */ + ret = nx842_OF_upd_status(status); + if (ret) + goto error_out; + + ret = nx842_OF_upd_maxsglen(new_devdata, maxsglen); + if (ret) + goto error_out; + + ret = nx842_OF_upd_maxsyncop(new_devdata, maxsyncop); + if (ret) + goto error_out; + +out: + dev_info(old_devdata->dev, "%s: max_sync_size new:%u old:%u\n", + __func__, new_devdata->max_sync_size, + old_devdata->max_sync_size); + dev_info(old_devdata->dev, "%s: max_sync_sg new:%u old:%u\n", + __func__, new_devdata->max_sync_sg, + old_devdata->max_sync_sg); + dev_info(old_devdata->dev, "%s: max_sg_len new:%u old:%u\n", + __func__, new_devdata->max_sg_len, + old_devdata->max_sg_len); + + rcu_assign_pointer(devdata, new_devdata); + spin_unlock_irqrestore(&devdata_mutex, flags); + synchronize_rcu(); + dev_set_drvdata(new_devdata->dev, new_devdata); + kfree(old_devdata); + return 0; + +error_out: + if (new_devdata) { + dev_info(old_devdata->dev, "%s: device disabled\n", __func__); + nx842_OF_set_defaults(new_devdata); + rcu_assign_pointer(devdata, new_devdata); + spin_unlock_irqrestore(&devdata_mutex, flags); + synchronize_rcu(); + dev_set_drvdata(new_devdata->dev, new_devdata); + kfree(old_devdata); + } else { + dev_err(old_devdata->dev, "%s: could not update driver from hardware\n", __func__); + spin_unlock_irqrestore(&devdata_mutex, flags); + } + + if (!ret) + ret = -EINVAL; + return ret; +} + +/** + * nx842_OF_notifier - Process updates to OF properties for the device + * + * @np: notifier block + * @action: notifier action + * @update: struct pSeries_reconfig_prop_update pointer if action is + * PSERIES_UPDATE_PROPERTY + * + * Returns: + * NOTIFY_OK on success + * NOTIFY_BAD encoded with error number on failure, use + * notifier_to_errno() to decode this value + */ +static int nx842_OF_notifier(struct notifier_block *np, unsigned long action, + void *data) +{ + struct of_reconfig_data *upd = data; + struct nx842_devdata *local_devdata; + struct device_node *node = NULL; + + rcu_read_lock(); + local_devdata = rcu_dereference(devdata); + if (local_devdata) + node = local_devdata->dev->of_node; + + if (local_devdata && + action == OF_RECONFIG_UPDATE_PROPERTY && + !strcmp(upd->dn->name, node->name)) { + rcu_read_unlock(); + nx842_OF_upd(upd->prop); + } else + rcu_read_unlock(); + + return NOTIFY_OK; +} + +static struct notifier_block nx842_of_nb = { + .notifier_call = nx842_OF_notifier, +}; + +#define nx842_counter_read(_name) \ +static ssize_t nx842_##_name##_show(struct device *dev, \ + struct device_attribute *attr, \ + char *buf) { \ + struct nx842_devdata *local_devdata; \ + int p = 0; \ + rcu_read_lock(); \ + local_devdata = rcu_dereference(devdata); \ + if (local_devdata) \ + p = snprintf(buf, PAGE_SIZE, "%ld\n", \ + atomic64_read(&local_devdata->counters->_name)); \ + rcu_read_unlock(); \ + return p; \ +} + +#define NX842DEV_COUNTER_ATTR_RO(_name) \ + nx842_counter_read(_name); \ + static struct device_attribute dev_attr_##_name = __ATTR(_name, \ + 0444, \ + nx842_##_name##_show,\ + NULL); + +NX842DEV_COUNTER_ATTR_RO(comp_complete); +NX842DEV_COUNTER_ATTR_RO(comp_failed); +NX842DEV_COUNTER_ATTR_RO(decomp_complete); +NX842DEV_COUNTER_ATTR_RO(decomp_failed); +NX842DEV_COUNTER_ATTR_RO(swdecomp); + +static ssize_t nx842_timehist_show(struct device *, + struct device_attribute *, char *); + +static struct device_attribute dev_attr_comp_times = __ATTR(comp_times, 0444, + nx842_timehist_show, NULL); +static struct device_attribute dev_attr_decomp_times = __ATTR(decomp_times, + 0444, nx842_timehist_show, NULL); + +static ssize_t nx842_timehist_show(struct device *dev, + struct device_attribute *attr, char *buf) { + char *p = buf; + struct nx842_devdata *local_devdata; + atomic64_t *times; + int bytes_remain = PAGE_SIZE; + int bytes; + int i; + + rcu_read_lock(); + local_devdata = rcu_dereference(devdata); + if (!local_devdata) { + rcu_read_unlock(); + return 0; + } + + if (attr == &dev_attr_comp_times) + times = local_devdata->counters->comp_times; + else if (attr == &dev_attr_decomp_times) + times = local_devdata->counters->decomp_times; + else { + rcu_read_unlock(); + return 0; + } + + for (i = 0; i < (NX842_HIST_SLOTS - 2); i++) { + bytes = snprintf(p, bytes_remain, "%u-%uus:\t%ld\n", + i ? (2<<(i-1)) : 0, (2<counters = kzalloc(sizeof(*new_devdata->counters), + GFP_NOFS); + if (!new_devdata->counters) { + kfree(new_devdata); + return -ENOMEM; + } + + spin_lock_irqsave(&devdata_mutex, flags); + old_devdata = rcu_dereference_check(devdata, + lockdep_is_held(&devdata_mutex)); + + if (old_devdata && old_devdata->vdev != NULL) { + dev_err(&viodev->dev, "%s: Attempt to register more than one instance of the hardware\n", __func__); + ret = -1; + goto error_unlock; + } + + dev_set_drvdata(&viodev->dev, NULL); + + new_devdata->vdev = viodev; + new_devdata->dev = &viodev->dev; + nx842_OF_set_defaults(new_devdata); + + rcu_assign_pointer(devdata, new_devdata); + spin_unlock_irqrestore(&devdata_mutex, flags); + synchronize_rcu(); + kfree(old_devdata); + + of_reconfig_notifier_register(&nx842_of_nb); + + ret = nx842_OF_upd(NULL); + if (ret) + goto error; + + ret = crypto_register_alg(&nx842_pseries_alg); + if (ret) { + dev_err(&viodev->dev, "could not register comp alg: %d\n", ret); + goto error; + } + + rcu_read_lock(); + dev_set_drvdata(&viodev->dev, rcu_dereference(devdata)); + rcu_read_unlock(); + + if (sysfs_create_group(&viodev->dev.kobj, &nx842_attribute_group)) { + dev_err(&viodev->dev, "could not create sysfs device attributes\n"); + ret = -1; + goto error; + } + + return 0; + +error_unlock: + spin_unlock_irqrestore(&devdata_mutex, flags); + if (new_devdata) + kfree(new_devdata->counters); + kfree(new_devdata); +error: + return ret; +} + +static int nx842_remove(struct vio_dev *viodev) +{ + struct nx842_devdata *old_devdata; + unsigned long flags; + + pr_info("Removing IBM Power 842 compression device\n"); + sysfs_remove_group(&viodev->dev.kobj, &nx842_attribute_group); + + crypto_unregister_alg(&nx842_pseries_alg); + + spin_lock_irqsave(&devdata_mutex, flags); + old_devdata = rcu_dereference_check(devdata, + lockdep_is_held(&devdata_mutex)); + of_reconfig_notifier_unregister(&nx842_of_nb); + RCU_INIT_POINTER(devdata, NULL); + spin_unlock_irqrestore(&devdata_mutex, flags); + synchronize_rcu(); + dev_set_drvdata(&viodev->dev, NULL); + if (old_devdata) + kfree(old_devdata->counters); + kfree(old_devdata); + + return 0; +} + +static struct vio_device_id nx842_vio_driver_ids[] = { + {"ibm,compression-v1", "ibm,compression"}, + {"", ""}, +}; + +static struct vio_driver nx842_vio_driver = { + .name = KBUILD_MODNAME, + .probe = nx842_probe, + .remove = nx842_remove, + .get_desired_dma = nx842_get_desired_dma, + .id_table = nx842_vio_driver_ids, +}; + +static int __init nx842_pseries_init(void) +{ + struct nx842_devdata *new_devdata; + int ret; + + if (!of_find_compatible_node(NULL, NULL, "ibm,compression")) + return -ENODEV; + + RCU_INIT_POINTER(devdata, NULL); + new_devdata = kzalloc(sizeof(*new_devdata), GFP_KERNEL); + if (!new_devdata) { + pr_err("Could not allocate memory for device data\n"); + return -ENOMEM; + } + RCU_INIT_POINTER(devdata, new_devdata); + + ret = vio_register_driver(&nx842_vio_driver); + if (ret) { + pr_err("Could not register VIO driver %d\n", ret); + + kfree(new_devdata); + return ret; + } + + return 0; +} + +module_init(nx842_pseries_init); + +static void __exit nx842_pseries_exit(void) +{ + struct nx842_devdata *old_devdata; + unsigned long flags; + + crypto_unregister_alg(&nx842_pseries_alg); + + spin_lock_irqsave(&devdata_mutex, flags); + old_devdata = rcu_dereference_check(devdata, + lockdep_is_held(&devdata_mutex)); + RCU_INIT_POINTER(devdata, NULL); + spin_unlock_irqrestore(&devdata_mutex, flags); + synchronize_rcu(); + if (old_devdata && old_devdata->dev) + dev_set_drvdata(old_devdata->dev, NULL); + kfree(old_devdata); + vio_unregister_driver(&nx842_vio_driver); +} + +module_exit(nx842_pseries_exit); + diff --git a/kernel/drivers/crypto/nx/nx-842.c b/kernel/drivers/crypto/nx/nx-842.c index 887196e9b..046c1c454 100644 --- a/kernel/drivers/crypto/nx/nx-842.c +++ b/kernel/drivers/crypto/nx/nx-842.c @@ -1,5 +1,5 @@ /* - * Driver for IBM Power 842 compression accelerator + * Cryptographic API for the NX-842 hardware compression. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -11,1593 +11,521 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - * - * Copyright (C) IBM Corporation, 2012 - * - * Authors: Robert Jennings - * Seth Jennings + * Copyright (C) IBM Corporation, 2011-2015 + * + * Designer of the Power data compression engine: + * Bulent Abali + * + * Original Authors: Robert Jennings + * Seth Jennings + * + * Rewrite: Dan Streetman + * + * This is an interface to the NX-842 compression hardware in PowerPC + * processors. Most of the complexity of this drvier is due to the fact that + * the NX-842 compression hardware requires the input and output data buffers + * to be specifically aligned, to be a specific multiple in length, and within + * specific minimum and maximum lengths. Those restrictions, provided by the + * nx-842 driver via nx842_constraints, mean this driver must use bounce + * buffers and headers to correct misaligned in or out buffers, and to split + * input buffers that are too large. + * + * This driver will fall back to software decompression if the hardware + * decompression fails, so this driver's decompression should never fail as + * long as the provided compressed buffer is valid. Any compressed buffer + * created by this driver will have a header (except ones where the input + * perfectly matches the constraints); so users of this driver cannot simply + * pass a compressed buffer created by this driver over to the 842 software + * decompression library. Instead, users must use this driver to decompress; + * if the hardware fails or is unavailable, the compressed buffer will be + * parsed and the header removed, and the raw 842 buffer(s) passed to the 842 + * software decompression library. + * + * This does not fall back to software compression, however, since the caller + * of this function is specifically requesting hardware compression; if the + * hardware compression fails, the caller can fall back to software + * compression, and the raw 842 compressed buffer that the software compressor + * creates can be passed to this driver for hardware decompression; any + * buffer without our specific header magic is assumed to be a raw 842 buffer + * and passed directly to the hardware. Note that the software compression + * library will produce a compressed buffer that is incompatible with the + * hardware decompressor if the original input buffer length is not a multiple + * of 8; if such a compressed buffer is passed to this driver for + * decompression, the hardware will reject it and this driver will then pass + * it over to the software library for decompression. */ -#include -#include -#include -#include -#include - -#include -#include +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -#include "nx_csbcpb.h" /* struct nx_csbcpb */ - -#define MODULE_NAME "nx-compress" -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Robert Jennings "); -MODULE_DESCRIPTION("842 H/W Compression driver for IBM Power processors"); +#include +#include +#include -#define SHIFT_4K 12 -#define SHIFT_64K 16 -#define SIZE_4K (1UL << SHIFT_4K) -#define SIZE_64K (1UL << SHIFT_64K) +#include "nx-842.h" -/* IO buffer must be 128 byte aligned */ -#define IO_BUFFER_ALIGN 128 - -struct nx842_header { - int blocks_nr; /* number of compressed blocks */ - int offset; /* offset of the first block (from beginning of header) */ - int sizes[0]; /* size of compressed blocks */ +/* The first 5 bits of this magic are 0x1f, which is an invalid 842 5-bit + * template (see lib/842/842.h), so this magic number will never appear at + * the start of a raw 842 compressed buffer. That is important, as any buffer + * passed to us without this magic is assumed to be a raw 842 compressed + * buffer, and passed directly to the hardware to decompress. + */ +#define NX842_CRYPTO_MAGIC (0xf842) +#define NX842_CRYPTO_HEADER_SIZE(g) \ + (sizeof(struct nx842_crypto_header) + \ + sizeof(struct nx842_crypto_header_group) * (g)) +#define NX842_CRYPTO_HEADER_MAX_SIZE \ + NX842_CRYPTO_HEADER_SIZE(NX842_CRYPTO_GROUP_MAX) + +/* bounce buffer size */ +#define BOUNCE_BUFFER_ORDER (2) +#define BOUNCE_BUFFER_SIZE \ + ((unsigned int)(PAGE_SIZE << BOUNCE_BUFFER_ORDER)) + +/* try longer on comp because we can fallback to sw decomp if hw is busy */ +#define COMP_BUSY_TIMEOUT (250) /* ms */ +#define DECOMP_BUSY_TIMEOUT (50) /* ms */ + +struct nx842_crypto_param { + u8 *in; + unsigned int iremain; + u8 *out; + unsigned int oremain; + unsigned int ototal; }; -static inline int nx842_header_size(const struct nx842_header *hdr) +static int update_param(struct nx842_crypto_param *p, + unsigned int slen, unsigned int dlen) { - return sizeof(struct nx842_header) + - hdr->blocks_nr * sizeof(hdr->sizes[0]); -} - -/* Macros for fields within nx_csbcpb */ -/* Check the valid bit within the csbcpb valid field */ -#define NX842_CSBCBP_VALID_CHK(x) (x & BIT_MASK(7)) - -/* CE macros operate on the completion_extension field bits in the csbcpb. - * CE0 0=full completion, 1=partial completion - * CE1 0=CE0 indicates completion, 1=termination (output may be modified) - * CE2 0=processed_bytes is source bytes, 1=processed_bytes is target bytes */ -#define NX842_CSBCPB_CE0(x) (x & BIT_MASK(7)) -#define NX842_CSBCPB_CE1(x) (x & BIT_MASK(6)) -#define NX842_CSBCPB_CE2(x) (x & BIT_MASK(5)) - -/* The NX unit accepts data only on 4K page boundaries */ -#define NX842_HW_PAGE_SHIFT SHIFT_4K -#define NX842_HW_PAGE_SIZE (ASM_CONST(1) << NX842_HW_PAGE_SHIFT) -#define NX842_HW_PAGE_MASK (~(NX842_HW_PAGE_SIZE-1)) - -enum nx842_status { - UNAVAILABLE, - AVAILABLE -}; + if (p->iremain < slen) + return -EOVERFLOW; + if (p->oremain < dlen) + return -ENOSPC; -struct ibm_nx842_counters { - atomic64_t comp_complete; - atomic64_t comp_failed; - atomic64_t decomp_complete; - atomic64_t decomp_failed; - atomic64_t swdecomp; - atomic64_t comp_times[32]; - atomic64_t decomp_times[32]; -}; + p->in += slen; + p->iremain -= slen; + p->out += dlen; + p->oremain -= dlen; + p->ototal += dlen; -static struct nx842_devdata { - struct vio_dev *vdev; - struct device *dev; - struct ibm_nx842_counters *counters; - unsigned int max_sg_len; - unsigned int max_sync_size; - unsigned int max_sync_sg; - enum nx842_status status; -} __rcu *devdata; -static DEFINE_SPINLOCK(devdata_mutex); - -#define NX842_COUNTER_INC(_x) \ -static inline void nx842_inc_##_x( \ - const struct nx842_devdata *dev) { \ - if (dev) \ - atomic64_inc(&dev->counters->_x); \ + return 0; } -NX842_COUNTER_INC(comp_complete); -NX842_COUNTER_INC(comp_failed); -NX842_COUNTER_INC(decomp_complete); -NX842_COUNTER_INC(decomp_failed); -NX842_COUNTER_INC(swdecomp); - -#define NX842_HIST_SLOTS 16 -static void ibm_nx842_incr_hist(atomic64_t *times, unsigned int time) +int nx842_crypto_init(struct crypto_tfm *tfm, struct nx842_driver *driver) { - int bucket = fls(time); + struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm); - if (bucket) - bucket = min((NX842_HIST_SLOTS - 1), bucket - 1); - - atomic64_inc(×[bucket]); -} + spin_lock_init(&ctx->lock); + ctx->driver = driver; + ctx->wmem = kmalloc(driver->workmem_size, GFP_KERNEL); + ctx->sbounce = (u8 *)__get_free_pages(GFP_KERNEL, BOUNCE_BUFFER_ORDER); + ctx->dbounce = (u8 *)__get_free_pages(GFP_KERNEL, BOUNCE_BUFFER_ORDER); + if (!ctx->wmem || !ctx->sbounce || !ctx->dbounce) { + kfree(ctx->wmem); + free_page((unsigned long)ctx->sbounce); + free_page((unsigned long)ctx->dbounce); + return -ENOMEM; + } -/* NX unit operation flags */ -#define NX842_OP_COMPRESS 0x0 -#define NX842_OP_CRC 0x1 -#define NX842_OP_DECOMPRESS 0x2 -#define NX842_OP_COMPRESS_CRC (NX842_OP_COMPRESS | NX842_OP_CRC) -#define NX842_OP_DECOMPRESS_CRC (NX842_OP_DECOMPRESS | NX842_OP_CRC) -#define NX842_OP_ASYNC (1<<23) -#define NX842_OP_NOTIFY (1<<22) -#define NX842_OP_NOTIFY_INT(x) ((x & 0xff)<<8) - -static unsigned long nx842_get_desired_dma(struct vio_dev *viodev) -{ - /* No use of DMA mappings within the driver. */ return 0; } +EXPORT_SYMBOL_GPL(nx842_crypto_init); -struct nx842_slentry { - unsigned long ptr; /* Real address (use __pa()) */ - unsigned long len; -}; - -/* pHyp scatterlist entry */ -struct nx842_scatterlist { - int entry_nr; /* number of slentries */ - struct nx842_slentry *entries; /* ptr to array of slentries */ -}; - -/* Does not include sizeof(entry_nr) in the size */ -static inline unsigned long nx842_get_scatterlist_size( - struct nx842_scatterlist *sl) +void nx842_crypto_exit(struct crypto_tfm *tfm) { - return sl->entry_nr * sizeof(struct nx842_slentry); + struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm); + + kfree(ctx->wmem); + free_page((unsigned long)ctx->sbounce); + free_page((unsigned long)ctx->dbounce); } +EXPORT_SYMBOL_GPL(nx842_crypto_exit); -static inline unsigned long nx842_get_pa(void *addr) +static void check_constraints(struct nx842_constraints *c) { - if (is_vmalloc_addr(addr)) - return page_to_phys(vmalloc_to_page(addr)) - + offset_in_page(addr); - else - return __pa(addr); + /* limit maximum, to always have enough bounce buffer to decompress */ + if (c->maximum > BOUNCE_BUFFER_SIZE) + c->maximum = BOUNCE_BUFFER_SIZE; } -static int nx842_build_scatterlist(unsigned long buf, int len, - struct nx842_scatterlist *sl) +static int nx842_crypto_add_header(struct nx842_crypto_header *hdr, u8 *buf) { - unsigned long nextpage; - struct nx842_slentry *entry; - - sl->entry_nr = 0; - - entry = sl->entries; - while (len) { - entry->ptr = nx842_get_pa((void *)buf); - nextpage = ALIGN(buf + 1, NX842_HW_PAGE_SIZE); - if (nextpage < buf + len) { - /* we aren't at the end yet */ - if (IS_ALIGNED(buf, NX842_HW_PAGE_SIZE)) - /* we are in the middle (or beginning) */ - entry->len = NX842_HW_PAGE_SIZE; - else - /* we are at the beginning */ - entry->len = nextpage - buf; - } else { - /* at the end */ - entry->len = len; - } + int s = NX842_CRYPTO_HEADER_SIZE(hdr->groups); - len -= entry->len; - buf += entry->len; - sl->entry_nr++; - entry++; + /* compress should have added space for header */ + if (s > be16_to_cpu(hdr->group[0].padding)) { + pr_err("Internal error: no space for header\n"); + return -EINVAL; } - return 0; -} + memcpy(buf, hdr, s); -/* - * Working memory for software decompression - */ -struct sw842_fifo { - union { - char f8[256][8]; - char f4[512][4]; - }; - char f2[256][2]; - unsigned char f84_full; - unsigned char f2_full; - unsigned char f8_count; - unsigned char f2_count; - unsigned int f4_count; -}; + print_hex_dump_debug("header ", DUMP_PREFIX_OFFSET, 16, 1, buf, s, 0); -/* - * Working memory for crypto API - */ -struct nx842_workmem { - char bounce[PAGE_SIZE]; /* bounce buffer for decompression input */ - union { - /* hardware working memory */ - struct { - /* scatterlist */ - char slin[SIZE_4K]; - char slout[SIZE_4K]; - /* coprocessor status/parameter block */ - struct nx_csbcpb csbcpb; - }; - /* software working memory */ - struct sw842_fifo swfifo; /* software decompression fifo */ - }; -}; - -int nx842_get_workmem_size(void) -{ - return sizeof(struct nx842_workmem) + NX842_HW_PAGE_SIZE; + return 0; } -EXPORT_SYMBOL_GPL(nx842_get_workmem_size); -int nx842_get_workmem_size_aligned(void) +static int compress(struct nx842_crypto_ctx *ctx, + struct nx842_crypto_param *p, + struct nx842_crypto_header_group *g, + struct nx842_constraints *c, + u16 *ignore, + unsigned int hdrsize) { - return sizeof(struct nx842_workmem); -} -EXPORT_SYMBOL_GPL(nx842_get_workmem_size_aligned); + unsigned int slen = p->iremain, dlen = p->oremain, tmplen; + unsigned int adj_slen = slen; + u8 *src = p->in, *dst = p->out; + int ret, dskip = 0; + ktime_t timeout; -static int nx842_validate_result(struct device *dev, - struct cop_status_block *csb) -{ - /* The csb must be valid after returning from vio_h_cop_sync */ - if (!NX842_CSBCBP_VALID_CHK(csb->valid)) { - dev_err(dev, "%s: cspcbp not valid upon completion.\n", - __func__); - dev_dbg(dev, "valid:0x%02x cs:0x%02x cc:0x%02x ce:0x%02x\n", - csb->valid, - csb->crb_seq_number, - csb->completion_code, - csb->completion_extension); - dev_dbg(dev, "processed_bytes:%d address:0x%016lx\n", - csb->processed_byte_count, - (unsigned long)csb->address); - return -EIO; - } + if (p->iremain == 0) + return -EOVERFLOW; - /* Check return values from the hardware in the CSB */ - switch (csb->completion_code) { - case 0: /* Completed without error */ - break; - case 64: /* Target bytes > Source bytes during compression */ - case 13: /* Output buffer too small */ - dev_dbg(dev, "%s: Compression output larger than input\n", - __func__); + if (p->oremain == 0 || hdrsize + c->minimum > dlen) return -ENOSPC; - case 66: /* Input data contains an illegal template field */ - case 67: /* Template indicates data past the end of the input stream */ - dev_dbg(dev, "%s: Bad data for decompression (code:%d)\n", - __func__, csb->completion_code); - return -EINVAL; - default: - dev_dbg(dev, "%s: Unspecified error (code:%d)\n", - __func__, csb->completion_code); - return -EIO; - } - /* Hardware sanity check */ - if (!NX842_CSBCPB_CE2(csb->completion_extension)) { - dev_err(dev, "%s: No error returned by hardware, but " - "data returned is unusable, contact support.\n" - "(Additional info: csbcbp->processed bytes " - "does not specify processed bytes for the " - "target buffer.)\n", __func__); - return -EIO; + if (slen % c->multiple) + adj_slen = round_up(slen, c->multiple); + if (slen < c->minimum) + adj_slen = c->minimum; + if (slen > c->maximum) + adj_slen = slen = c->maximum; + if (adj_slen > slen || (u64)src % c->alignment) { + adj_slen = min(adj_slen, BOUNCE_BUFFER_SIZE); + slen = min(slen, BOUNCE_BUFFER_SIZE); + if (adj_slen > slen) + memset(ctx->sbounce + slen, 0, adj_slen - slen); + memcpy(ctx->sbounce, src, slen); + src = ctx->sbounce; + slen = adj_slen; + pr_debug("using comp sbounce buffer, len %x\n", slen); } - return 0; -} - -/** - * nx842_compress - Compress data using the 842 algorithm - * - * Compression provide by the NX842 coprocessor on IBM Power systems. - * The input buffer is compressed and the result is stored in the - * provided output buffer. - * - * Upon return from this function @outlen contains the length of the - * compressed data. If there is an error then @outlen will be 0 and an - * error will be specified by the return code from this function. - * - * @in: Pointer to input buffer, must be page aligned - * @inlen: Length of input buffer, must be PAGE_SIZE - * @out: Pointer to output buffer - * @outlen: Length of output buffer - * @wrkmem: ptr to buffer for working memory, size determined by - * nx842_get_workmem_size() - * - * Returns: - * 0 Success, output of length @outlen stored in the buffer at @out - * -ENOMEM Unable to allocate internal buffers - * -ENOSPC Output buffer is to small - * -EMSGSIZE XXX Difficult to describe this limitation - * -EIO Internal error - * -ENODEV Hardware unavailable - */ -int nx842_compress(const unsigned char *in, unsigned int inlen, - unsigned char *out, unsigned int *outlen, void *wmem) -{ - struct nx842_header *hdr; - struct nx842_devdata *local_devdata; - struct device *dev = NULL; - struct nx842_workmem *workmem; - struct nx842_scatterlist slin, slout; - struct nx_csbcpb *csbcpb; - int ret = 0, max_sync_size, i, bytesleft, size, hdrsize; - unsigned long inbuf, outbuf, padding; - struct vio_pfo_op op = { - .done = NULL, - .handle = 0, - .timeout = 0, - }; - unsigned long start_time = get_tb(); - - /* - * Make sure input buffer is 64k page aligned. This is assumed since - * this driver is designed for page compression only (for now). This - * is very nice since we can now use direct DDE(s) for the input and - * the alignment is guaranteed. - */ - inbuf = (unsigned long)in; - if (!IS_ALIGNED(inbuf, PAGE_SIZE) || inlen != PAGE_SIZE) - return -EINVAL; + dst += hdrsize; + dlen -= hdrsize; - rcu_read_lock(); - local_devdata = rcu_dereference(devdata); - if (!local_devdata || !local_devdata->dev) { - rcu_read_unlock(); - return -ENODEV; + if ((u64)dst % c->alignment) { + dskip = (int)(PTR_ALIGN(dst, c->alignment) - dst); + dst += dskip; + dlen -= dskip; + } + if (dlen % c->multiple) + dlen = round_down(dlen, c->multiple); + if (dlen < c->minimum) { +nospc: + dst = ctx->dbounce; + dlen = min(p->oremain, BOUNCE_BUFFER_SIZE); + dlen = round_down(dlen, c->multiple); + dskip = 0; + pr_debug("using comp dbounce buffer, len %x\n", dlen); } - max_sync_size = local_devdata->max_sync_size; - dev = local_devdata->dev; - - /* Create the header */ - hdr = (struct nx842_header *)out; - hdr->blocks_nr = PAGE_SIZE / max_sync_size; - hdrsize = nx842_header_size(hdr); - outbuf = (unsigned long)out + hdrsize; - bytesleft = *outlen - hdrsize; - - /* Init scatterlist */ - workmem = (struct nx842_workmem *)ALIGN((unsigned long)wmem, - NX842_HW_PAGE_SIZE); - slin.entries = (struct nx842_slentry *)workmem->slin; - slout.entries = (struct nx842_slentry *)workmem->slout; - - /* Init operation */ - op.flags = NX842_OP_COMPRESS; - csbcpb = &workmem->csbcpb; - memset(csbcpb, 0, sizeof(*csbcpb)); - op.csbcpb = nx842_get_pa(csbcpb); - op.out = nx842_get_pa(slout.entries); - - for (i = 0; i < hdr->blocks_nr; i++) { - /* - * Aligning the output blocks to 128 bytes does waste space, - * but it prevents the need for bounce buffers and memory - * copies. It also simplifies the code a lot. In the worst - * case (64k page, 4k max_sync_size), you lose up to - * (128*16)/64k = ~3% the compression factor. For 64k - * max_sync_size, the loss would be at most 128/64k = ~0.2%. + if (dlen > c->maximum) + dlen = c->maximum; + + tmplen = dlen; + timeout = ktime_add_ms(ktime_get(), COMP_BUSY_TIMEOUT); + do { + dlen = tmplen; /* reset dlen, if we're retrying */ + ret = ctx->driver->compress(src, slen, dst, &dlen, ctx->wmem); + /* possibly we should reduce the slen here, instead of + * retrying with the dbounce buffer? */ - padding = ALIGN(outbuf, IO_BUFFER_ALIGN) - outbuf; - outbuf += padding; - bytesleft -= padding; - if (i == 0) - /* save offset into first block in header */ - hdr->offset = padding + hdrsize; - - if (bytesleft <= 0) { - ret = -ENOSPC; - goto unlock; - } + if (ret == -ENOSPC && dst != ctx->dbounce) + goto nospc; + } while (ret == -EBUSY && ktime_before(ktime_get(), timeout)); + if (ret) + return ret; - /* - * NOTE: If the default max_sync_size is changed from 4k - * to 64k, remove the "likely" case below, since a - * scatterlist will always be needed. - */ - if (likely(max_sync_size == NX842_HW_PAGE_SIZE)) { - /* Create direct DDE */ - op.in = nx842_get_pa((void *)inbuf); - op.inlen = max_sync_size; - - } else { - /* Create indirect DDE (scatterlist) */ - nx842_build_scatterlist(inbuf, max_sync_size, &slin); - op.in = nx842_get_pa(slin.entries); - op.inlen = -nx842_get_scatterlist_size(&slin); - } + dskip += hdrsize; - /* - * If max_sync_size != NX842_HW_PAGE_SIZE, an indirect - * DDE is required for the outbuf. - * If max_sync_size == NX842_HW_PAGE_SIZE, outbuf must - * also be page aligned (1 in 128/4k=32 chance) in order - * to use a direct DDE. - * This is unlikely, just use an indirect DDE always. - */ - nx842_build_scatterlist(outbuf, - min(bytesleft, max_sync_size), &slout); - /* op.out set before loop */ - op.outlen = -nx842_get_scatterlist_size(&slout); - - /* Send request to pHyp */ - ret = vio_h_cop_sync(local_devdata->vdev, &op); - - /* Check for pHyp error */ - if (ret) { - dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n", - __func__, ret, op.hcall_err); - ret = -EIO; - goto unlock; - } - - /* Check for hardware error */ - ret = nx842_validate_result(dev, &csbcpb->csb); - if (ret && ret != -ENOSPC) - goto unlock; + if (dst == ctx->dbounce) + memcpy(p->out + dskip, dst, dlen); - /* Handle incompressible data */ - if (unlikely(ret == -ENOSPC)) { - if (bytesleft < max_sync_size) { - /* - * Not enough space left in the output buffer - * to store uncompressed block - */ - goto unlock; - } else { - /* Store incompressible block */ - memcpy((void *)outbuf, (void *)inbuf, - max_sync_size); - hdr->sizes[i] = -max_sync_size; - outbuf += max_sync_size; - bytesleft -= max_sync_size; - /* Reset ret, incompressible data handled */ - ret = 0; - } - } else { - /* Normal case, compression was successful */ - size = csbcpb->csb.processed_byte_count; - dev_dbg(dev, "%s: processed_bytes=%d\n", - __func__, size); - hdr->sizes[i] = size; - outbuf += size; - bytesleft -= size; - } + g->padding = cpu_to_be16(dskip); + g->compressed_length = cpu_to_be32(dlen); + g->uncompressed_length = cpu_to_be32(slen); - inbuf += max_sync_size; + if (p->iremain < slen) { + *ignore = slen - p->iremain; + slen = p->iremain; } - *outlen = (unsigned int)(outbuf - (unsigned long)out); + pr_debug("compress slen %x ignore %x dlen %x padding %x\n", + slen, *ignore, dlen, dskip); -unlock: - if (ret) - nx842_inc_comp_failed(local_devdata); - else { - nx842_inc_comp_complete(local_devdata); - ibm_nx842_incr_hist(local_devdata->counters->comp_times, - (get_tb() - start_time) / tb_ticks_per_usec); - } - rcu_read_unlock(); - return ret; + return update_param(p, slen, dskip + dlen); } -EXPORT_SYMBOL_GPL(nx842_compress); - -static int sw842_decompress(const unsigned char *, int, unsigned char *, int *, - const void *); -/** - * nx842_decompress - Decompress data using the 842 algorithm - * - * Decompression provide by the NX842 coprocessor on IBM Power systems. - * The input buffer is decompressed and the result is stored in the - * provided output buffer. The size allocated to the output buffer is - * provided by the caller of this function in @outlen. Upon return from - * this function @outlen contains the length of the decompressed data. - * If there is an error then @outlen will be 0 and an error will be - * specified by the return code from this function. - * - * @in: Pointer to input buffer, will use bounce buffer if not 128 byte - * aligned - * @inlen: Length of input buffer - * @out: Pointer to output buffer, must be page aligned - * @outlen: Length of output buffer, must be PAGE_SIZE - * @wrkmem: ptr to buffer for working memory, size determined by - * nx842_get_workmem_size() - * - * Returns: - * 0 Success, output of length @outlen stored in the buffer at @out - * -ENODEV Hardware decompression device is unavailable - * -ENOMEM Unable to allocate internal buffers - * -ENOSPC Output buffer is to small - * -EINVAL Bad input data encountered when attempting decompress - * -EIO Internal error - */ -int nx842_decompress(const unsigned char *in, unsigned int inlen, - unsigned char *out, unsigned int *outlen, void *wmem) +int nx842_crypto_compress(struct crypto_tfm *tfm, + const u8 *src, unsigned int slen, + u8 *dst, unsigned int *dlen) { - struct nx842_header *hdr; - struct nx842_devdata *local_devdata; - struct device *dev = NULL; - struct nx842_workmem *workmem; - struct nx842_scatterlist slin, slout; - struct nx_csbcpb *csbcpb; - int ret = 0, i, size, max_sync_size; - unsigned long inbuf, outbuf; - struct vio_pfo_op op = { - .done = NULL, - .handle = 0, - .timeout = 0, - }; - unsigned long start_time = get_tb(); - - /* Ensure page alignment and size */ - outbuf = (unsigned long)out; - if (!IS_ALIGNED(outbuf, PAGE_SIZE) || *outlen != PAGE_SIZE) - return -EINVAL; - - rcu_read_lock(); - local_devdata = rcu_dereference(devdata); - if (local_devdata) - dev = local_devdata->dev; - - /* Get header */ - hdr = (struct nx842_header *)in; + struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm); + struct nx842_crypto_header *hdr = &ctx->header; + struct nx842_crypto_param p; + struct nx842_constraints c = *ctx->driver->constraints; + unsigned int groups, hdrsize, h; + int ret, n; + bool add_header; + u16 ignore = 0; - workmem = (struct nx842_workmem *)ALIGN((unsigned long)wmem, - NX842_HW_PAGE_SIZE); + check_constraints(&c); - inbuf = (unsigned long)in + hdr->offset; - if (likely(!IS_ALIGNED(inbuf, IO_BUFFER_ALIGN))) { - /* Copy block(s) into bounce buffer for alignment */ - memcpy(workmem->bounce, in + hdr->offset, inlen - hdr->offset); - inbuf = (unsigned long)workmem->bounce; - } + p.in = (u8 *)src; + p.iremain = slen; + p.out = dst; + p.oremain = *dlen; + p.ototal = 0; - /* Init scatterlist */ - slin.entries = (struct nx842_slentry *)workmem->slin; - slout.entries = (struct nx842_slentry *)workmem->slout; + *dlen = 0; - /* Init operation */ - op.flags = NX842_OP_DECOMPRESS; - csbcpb = &workmem->csbcpb; - memset(csbcpb, 0, sizeof(*csbcpb)); - op.csbcpb = nx842_get_pa(csbcpb); + groups = min_t(unsigned int, NX842_CRYPTO_GROUP_MAX, + DIV_ROUND_UP(p.iremain, c.maximum)); + hdrsize = NX842_CRYPTO_HEADER_SIZE(groups); - /* - * max_sync_size may have changed since compression, - * so we can't read it from the device info. We need - * to derive it from hdr->blocks_nr. - */ - max_sync_size = PAGE_SIZE / hdr->blocks_nr; - - for (i = 0; i < hdr->blocks_nr; i++) { - /* Skip padding */ - inbuf = ALIGN(inbuf, IO_BUFFER_ALIGN); - - if (hdr->sizes[i] < 0) { - /* Negative sizes indicate uncompressed data blocks */ - size = abs(hdr->sizes[i]); - memcpy((void *)outbuf, (void *)inbuf, size); - outbuf += size; - inbuf += size; - continue; - } + spin_lock_bh(&ctx->lock); - if (!dev) - goto sw; + /* skip adding header if the buffers meet all constraints */ + add_header = (p.iremain % c.multiple || + p.iremain < c.minimum || + p.iremain > c.maximum || + (u64)p.in % c.alignment || + p.oremain % c.multiple || + p.oremain < c.minimum || + p.oremain > c.maximum || + (u64)p.out % c.alignment); - /* - * The better the compression, the more likely the "likely" - * case becomes. - */ - if (likely((inbuf & NX842_HW_PAGE_MASK) == - ((inbuf + hdr->sizes[i] - 1) & NX842_HW_PAGE_MASK))) { - /* Create direct DDE */ - op.in = nx842_get_pa((void *)inbuf); - op.inlen = hdr->sizes[i]; - } else { - /* Create indirect DDE (scatterlist) */ - nx842_build_scatterlist(inbuf, hdr->sizes[i] , &slin); - op.in = nx842_get_pa(slin.entries); - op.inlen = -nx842_get_scatterlist_size(&slin); - } + hdr->magic = cpu_to_be16(NX842_CRYPTO_MAGIC); + hdr->groups = 0; + hdr->ignore = 0; - /* - * NOTE: If the default max_sync_size is changed from 4k - * to 64k, remove the "likely" case below, since a - * scatterlist will always be needed. - */ - if (likely(max_sync_size == NX842_HW_PAGE_SIZE)) { - /* Create direct DDE */ - op.out = nx842_get_pa((void *)outbuf); - op.outlen = max_sync_size; - } else { - /* Create indirect DDE (scatterlist) */ - nx842_build_scatterlist(outbuf, max_sync_size, &slout); - op.out = nx842_get_pa(slout.entries); - op.outlen = -nx842_get_scatterlist_size(&slout); - } + while (p.iremain > 0) { + n = hdr->groups++; + ret = -ENOSPC; + if (hdr->groups > NX842_CRYPTO_GROUP_MAX) + goto unlock; - /* Send request to pHyp */ - ret = vio_h_cop_sync(local_devdata->vdev, &op); + /* header goes before first group */ + h = !n && add_header ? hdrsize : 0; - /* Check for pHyp error */ - if (ret) { - dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n", - __func__, ret, op.hcall_err); - dev = NULL; - goto sw; - } + if (ignore) + pr_warn("interal error, ignore is set %x\n", ignore); - /* Check for hardware error */ - ret = nx842_validate_result(dev, &csbcpb->csb); - if (ret) { - dev = NULL; - goto sw; - } - - /* HW decompression success */ - inbuf += hdr->sizes[i]; - outbuf += csbcpb->csb.processed_byte_count; - continue; - -sw: - /* software decompression */ - size = max_sync_size; - ret = sw842_decompress( - (unsigned char *)inbuf, hdr->sizes[i], - (unsigned char *)outbuf, &size, wmem); + ret = compress(ctx, &p, &hdr->group[n], &c, &ignore, h); if (ret) - pr_debug("%s: sw842_decompress failed with %d\n", - __func__, ret); - - if (ret) { - if (ret != -ENOSPC && ret != -EINVAL && - ret != -EMSGSIZE) - ret = -EIO; goto unlock; - } - - /* SW decompression success */ - inbuf += hdr->sizes[i]; - outbuf += size; } - *outlen = (unsigned int)(outbuf - (unsigned long)out); - -unlock: - if (ret) - /* decompress fail */ - nx842_inc_decomp_failed(local_devdata); - else { - if (!dev) - /* software decompress */ - nx842_inc_swdecomp(local_devdata); - nx842_inc_decomp_complete(local_devdata); - ibm_nx842_incr_hist(local_devdata->counters->decomp_times, - (get_tb() - start_time) / tb_ticks_per_usec); + if (!add_header && hdr->groups > 1) { + pr_err("Internal error: No header but multiple groups\n"); + ret = -EINVAL; + goto unlock; } - rcu_read_unlock(); - return ret; -} -EXPORT_SYMBOL_GPL(nx842_decompress); - -/** - * nx842_OF_set_defaults -- Set default (disabled) values for devdata - * - * @devdata - struct nx842_devdata to update - * - * Returns: - * 0 on success - * -ENOENT if @devdata ptr is NULL - */ -static int nx842_OF_set_defaults(struct nx842_devdata *devdata) -{ - if (devdata) { - devdata->max_sync_size = 0; - devdata->max_sync_sg = 0; - devdata->max_sg_len = 0; - devdata->status = UNAVAILABLE; - return 0; - } else - return -ENOENT; -} + /* ignore indicates the input stream needed to be padded */ + hdr->ignore = cpu_to_be16(ignore); + if (ignore) + pr_debug("marked %d bytes as ignore\n", ignore); -/** - * nx842_OF_upd_status -- Update the device info from OF status prop - * - * The status property indicates if the accelerator is enabled. If the - * device is in the OF tree it indicates that the hardware is present. - * The status field indicates if the device is enabled when the status - * is 'okay'. Otherwise the device driver will be disabled. - * - * @devdata - struct nx842_devdata to update - * @prop - struct property point containing the maxsyncop for the update - * - * Returns: - * 0 - Device is available - * -EINVAL - Device is not available - */ -static int nx842_OF_upd_status(struct nx842_devdata *devdata, - struct property *prop) { - int ret = 0; - const char *status = (const char *)prop->value; - - if (!strncmp(status, "okay", (size_t)prop->length)) { - devdata->status = AVAILABLE; - } else { - dev_info(devdata->dev, "%s: status '%s' is not 'okay'\n", - __func__, status); - devdata->status = UNAVAILABLE; - } + if (add_header) + ret = nx842_crypto_add_header(hdr, dst); + if (ret) + goto unlock; - return ret; -} + *dlen = p.ototal; -/** - * nx842_OF_upd_maxsglen -- Update the device info from OF maxsglen prop - * - * Definition of the 'ibm,max-sg-len' OF property: - * This field indicates the maximum byte length of a scatter list - * for the platform facility. It is a single cell encoded as with encode-int. - * - * Example: - * # od -x ibm,max-sg-len - * 0000000 0000 0ff0 - * - * In this example, the maximum byte length of a scatter list is - * 0x0ff0 (4,080). - * - * @devdata - struct nx842_devdata to update - * @prop - struct property point containing the maxsyncop for the update - * - * Returns: - * 0 on success - * -EINVAL on failure - */ -static int nx842_OF_upd_maxsglen(struct nx842_devdata *devdata, - struct property *prop) { - int ret = 0; - const int *maxsglen = prop->value; - - if (prop->length != sizeof(*maxsglen)) { - dev_err(devdata->dev, "%s: unexpected format for ibm,max-sg-len property\n", __func__); - dev_dbg(devdata->dev, "%s: ibm,max-sg-len is %d bytes long, expected %lu bytes\n", __func__, - prop->length, sizeof(*maxsglen)); - ret = -EINVAL; - } else { - devdata->max_sg_len = (unsigned int)min(*maxsglen, - (int)NX842_HW_PAGE_SIZE); - } + pr_debug("compress total slen %x dlen %x\n", slen, *dlen); +unlock: + spin_unlock_bh(&ctx->lock); return ret; } +EXPORT_SYMBOL_GPL(nx842_crypto_compress); -/** - * nx842_OF_upd_maxsyncop -- Update the device info from OF maxsyncop prop - * - * Definition of the 'ibm,max-sync-cop' OF property: - * Two series of cells. The first series of cells represents the maximums - * that can be synchronously compressed. The second series of cells - * represents the maximums that can be synchronously decompressed. - * 1. The first cell in each series contains the count of the number of - * data length, scatter list elements pairs that follow – each being - * of the form - * a. One cell data byte length - * b. One cell total number of scatter list elements - * - * Example: - * # od -x ibm,max-sync-cop - * 0000000 0000 0001 0000 1000 0000 01fe 0000 0001 - * 0000020 0000 1000 0000 01fe - * - * In this example, compression supports 0x1000 (4,096) data byte length - * and 0x1fe (510) total scatter list elements. Decompression supports - * 0x1000 (4,096) data byte length and 0x1f3 (510) total scatter list - * elements. - * - * @devdata - struct nx842_devdata to update - * @prop - struct property point containing the maxsyncop for the update - * - * Returns: - * 0 on success - * -EINVAL on failure - */ -static int nx842_OF_upd_maxsyncop(struct nx842_devdata *devdata, - struct property *prop) { - int ret = 0; - const struct maxsynccop_t { - int comp_elements; - int comp_data_limit; - int comp_sg_limit; - int decomp_elements; - int decomp_data_limit; - int decomp_sg_limit; - } *maxsynccop; - - if (prop->length != sizeof(*maxsynccop)) { - dev_err(devdata->dev, "%s: unexpected format for ibm,max-sync-cop property\n", __func__); - dev_dbg(devdata->dev, "%s: ibm,max-sync-cop is %d bytes long, expected %lu bytes\n", __func__, prop->length, - sizeof(*maxsynccop)); - ret = -EINVAL; - goto out; - } - - maxsynccop = (const struct maxsynccop_t *)prop->value; - - /* Use one limit rather than separate limits for compression and - * decompression. Set a maximum for this so as not to exceed the - * size that the header can support and round the value down to - * the hardware page size (4K) */ - devdata->max_sync_size = - (unsigned int)min(maxsynccop->comp_data_limit, - maxsynccop->decomp_data_limit); - - devdata->max_sync_size = min_t(unsigned int, devdata->max_sync_size, - SIZE_64K); +static int decompress(struct nx842_crypto_ctx *ctx, + struct nx842_crypto_param *p, + struct nx842_crypto_header_group *g, + struct nx842_constraints *c, + u16 ignore) +{ + unsigned int slen = be32_to_cpu(g->compressed_length); + unsigned int required_len = be32_to_cpu(g->uncompressed_length); + unsigned int dlen = p->oremain, tmplen; + unsigned int adj_slen = slen; + u8 *src = p->in, *dst = p->out; + u16 padding = be16_to_cpu(g->padding); + int ret, spadding = 0, dpadding = 0; + ktime_t timeout; - if (devdata->max_sync_size < SIZE_4K) { - dev_err(devdata->dev, "%s: hardware max data size (%u) is " - "less than the driver minimum, unable to use " - "the hardware device\n", - __func__, devdata->max_sync_size); - ret = -EINVAL; - goto out; - } + if (!slen || !required_len) + return -EINVAL; - devdata->max_sync_sg = (unsigned int)min(maxsynccop->comp_sg_limit, - maxsynccop->decomp_sg_limit); - if (devdata->max_sync_sg < 1) { - dev_err(devdata->dev, "%s: hardware max sg size (%u) is " - "less than the driver minimum, unable to use " - "the hardware device\n", - __func__, devdata->max_sync_sg); - ret = -EINVAL; - goto out; - } + if (p->iremain <= 0 || padding + slen > p->iremain) + return -EOVERFLOW; -out: - return ret; -} + if (p->oremain <= 0 || required_len - ignore > p->oremain) + return -ENOSPC; -/** - * - * nx842_OF_upd -- Handle OF properties updates for the device. - * - * Set all properties from the OF tree. Optionally, a new property - * can be provided by the @new_prop pointer to overwrite an existing value. - * The device will remain disabled until all values are valid, this function - * will return an error for updates unless all values are valid. - * - * @new_prop: If not NULL, this property is being updated. If NULL, update - * all properties from the current values in the OF tree. - * - * Returns: - * 0 - Success - * -ENOMEM - Could not allocate memory for new devdata structure - * -EINVAL - property value not found, new_prop is not a recognized - * property for the device or property value is not valid. - * -ENODEV - Device is not available - */ -static int nx842_OF_upd(struct property *new_prop) -{ - struct nx842_devdata *old_devdata = NULL; - struct nx842_devdata *new_devdata = NULL; - struct device_node *of_node = NULL; - struct property *status = NULL; - struct property *maxsglen = NULL; - struct property *maxsyncop = NULL; - int ret = 0; - unsigned long flags; - - spin_lock_irqsave(&devdata_mutex, flags); - old_devdata = rcu_dereference_check(devdata, - lockdep_is_held(&devdata_mutex)); - if (old_devdata) - of_node = old_devdata->dev->of_node; - - if (!old_devdata || !of_node) { - pr_err("%s: device is not available\n", __func__); - spin_unlock_irqrestore(&devdata_mutex, flags); - return -ENODEV; + src += padding; + + if (slen % c->multiple) + adj_slen = round_up(slen, c->multiple); + if (slen < c->minimum) + adj_slen = c->minimum; + if (slen > c->maximum) + goto usesw; + if (slen < adj_slen || (u64)src % c->alignment) { + /* we can append padding bytes because the 842 format defines + * an "end" template (see lib/842/842_decompress.c) and will + * ignore any bytes following it. + */ + if (slen < adj_slen) + memset(ctx->sbounce + slen, 0, adj_slen - slen); + memcpy(ctx->sbounce, src, slen); + src = ctx->sbounce; + spadding = adj_slen - slen; + slen = adj_slen; + pr_debug("using decomp sbounce buffer, len %x\n", slen); } - new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS); - if (!new_devdata) { - dev_err(old_devdata->dev, "%s: Could not allocate memory for device data\n", __func__); - ret = -ENOMEM; - goto error_out; + if (dlen % c->multiple) + dlen = round_down(dlen, c->multiple); + if (dlen < required_len || (u64)dst % c->alignment) { + dst = ctx->dbounce; + dlen = min(required_len, BOUNCE_BUFFER_SIZE); + pr_debug("using decomp dbounce buffer, len %x\n", dlen); } - - memcpy(new_devdata, old_devdata, sizeof(*old_devdata)); - new_devdata->counters = old_devdata->counters; - - /* Set ptrs for existing properties */ - status = of_find_property(of_node, "status", NULL); - maxsglen = of_find_property(of_node, "ibm,max-sg-len", NULL); - maxsyncop = of_find_property(of_node, "ibm,max-sync-cop", NULL); - if (!status || !maxsglen || !maxsyncop) { - dev_err(old_devdata->dev, "%s: Could not locate device properties\n", __func__); - ret = -EINVAL; - goto error_out; + if (dlen < c->minimum) + goto usesw; + if (dlen > c->maximum) + dlen = c->maximum; + + tmplen = dlen; + timeout = ktime_add_ms(ktime_get(), DECOMP_BUSY_TIMEOUT); + do { + dlen = tmplen; /* reset dlen, if we're retrying */ + ret = ctx->driver->decompress(src, slen, dst, &dlen, ctx->wmem); + } while (ret == -EBUSY && ktime_before(ktime_get(), timeout)); + if (ret) { +usesw: + /* reset everything, sw doesn't have constraints */ + src = p->in + padding; + slen = be32_to_cpu(g->compressed_length); + spadding = 0; + dst = p->out; + dlen = p->oremain; + dpadding = 0; + if (dlen < required_len) { /* have ignore bytes */ + dst = ctx->dbounce; + dlen = BOUNCE_BUFFER_SIZE; + } + pr_info_ratelimited("using software 842 decompression\n"); + ret = sw842_decompress(src, slen, dst, &dlen); } - - /* - * If this is a property update, there are only certain properties that - * we care about. Bail if it isn't in the below list - */ - if (new_prop && (strncmp(new_prop->name, "status", new_prop->length) || - strncmp(new_prop->name, "ibm,max-sg-len", new_prop->length) || - strncmp(new_prop->name, "ibm,max-sync-cop", new_prop->length))) - goto out; - - /* Perform property updates */ - ret = nx842_OF_upd_status(new_devdata, status); if (ret) - goto error_out; + return ret; - ret = nx842_OF_upd_maxsglen(new_devdata, maxsglen); - if (ret) - goto error_out; + slen -= spadding; - ret = nx842_OF_upd_maxsyncop(new_devdata, maxsyncop); - if (ret) - goto error_out; - -out: - dev_info(old_devdata->dev, "%s: max_sync_size new:%u old:%u\n", - __func__, new_devdata->max_sync_size, - old_devdata->max_sync_size); - dev_info(old_devdata->dev, "%s: max_sync_sg new:%u old:%u\n", - __func__, new_devdata->max_sync_sg, - old_devdata->max_sync_sg); - dev_info(old_devdata->dev, "%s: max_sg_len new:%u old:%u\n", - __func__, new_devdata->max_sg_len, - old_devdata->max_sg_len); - - rcu_assign_pointer(devdata, new_devdata); - spin_unlock_irqrestore(&devdata_mutex, flags); - synchronize_rcu(); - dev_set_drvdata(new_devdata->dev, new_devdata); - kfree(old_devdata); - return 0; + dlen -= ignore; + if (ignore) + pr_debug("ignoring last %x bytes\n", ignore); -error_out: - if (new_devdata) { - dev_info(old_devdata->dev, "%s: device disabled\n", __func__); - nx842_OF_set_defaults(new_devdata); - rcu_assign_pointer(devdata, new_devdata); - spin_unlock_irqrestore(&devdata_mutex, flags); - synchronize_rcu(); - dev_set_drvdata(new_devdata->dev, new_devdata); - kfree(old_devdata); - } else { - dev_err(old_devdata->dev, "%s: could not update driver from hardware\n", __func__); - spin_unlock_irqrestore(&devdata_mutex, flags); - } + if (dst == ctx->dbounce) + memcpy(p->out, dst, dlen); - if (!ret) - ret = -EINVAL; - return ret; + pr_debug("decompress slen %x padding %x dlen %x ignore %x\n", + slen, padding, dlen, ignore); + + return update_param(p, slen + padding, dlen); } -/** - * nx842_OF_notifier - Process updates to OF properties for the device - * - * @np: notifier block - * @action: notifier action - * @update: struct pSeries_reconfig_prop_update pointer if action is - * PSERIES_UPDATE_PROPERTY - * - * Returns: - * NOTIFY_OK on success - * NOTIFY_BAD encoded with error number on failure, use - * notifier_to_errno() to decode this value - */ -static int nx842_OF_notifier(struct notifier_block *np, unsigned long action, - void *data) +int nx842_crypto_decompress(struct crypto_tfm *tfm, + const u8 *src, unsigned int slen, + u8 *dst, unsigned int *dlen) { - struct of_reconfig_data *upd = data; - struct nx842_devdata *local_devdata; - struct device_node *node = NULL; - - rcu_read_lock(); - local_devdata = rcu_dereference(devdata); - if (local_devdata) - node = local_devdata->dev->of_node; - - if (local_devdata && - action == OF_RECONFIG_UPDATE_PROPERTY && - !strcmp(upd->dn->name, node->name)) { - rcu_read_unlock(); - nx842_OF_upd(upd->prop); - } else - rcu_read_unlock(); - - return NOTIFY_OK; -} + struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm); + struct nx842_crypto_header *hdr; + struct nx842_crypto_param p; + struct nx842_constraints c = *ctx->driver->constraints; + int n, ret, hdr_len; + u16 ignore = 0; -static struct notifier_block nx842_of_nb = { - .notifier_call = nx842_OF_notifier, -}; + check_constraints(&c); -#define nx842_counter_read(_name) \ -static ssize_t nx842_##_name##_show(struct device *dev, \ - struct device_attribute *attr, \ - char *buf) { \ - struct nx842_devdata *local_devdata; \ - int p = 0; \ - rcu_read_lock(); \ - local_devdata = rcu_dereference(devdata); \ - if (local_devdata) \ - p = snprintf(buf, PAGE_SIZE, "%ld\n", \ - atomic64_read(&local_devdata->counters->_name)); \ - rcu_read_unlock(); \ - return p; \ -} + p.in = (u8 *)src; + p.iremain = slen; + p.out = dst; + p.oremain = *dlen; + p.ototal = 0; -#define NX842DEV_COUNTER_ATTR_RO(_name) \ - nx842_counter_read(_name); \ - static struct device_attribute dev_attr_##_name = __ATTR(_name, \ - 0444, \ - nx842_##_name##_show,\ - NULL); - -NX842DEV_COUNTER_ATTR_RO(comp_complete); -NX842DEV_COUNTER_ATTR_RO(comp_failed); -NX842DEV_COUNTER_ATTR_RO(decomp_complete); -NX842DEV_COUNTER_ATTR_RO(decomp_failed); -NX842DEV_COUNTER_ATTR_RO(swdecomp); - -static ssize_t nx842_timehist_show(struct device *, - struct device_attribute *, char *); - -static struct device_attribute dev_attr_comp_times = __ATTR(comp_times, 0444, - nx842_timehist_show, NULL); -static struct device_attribute dev_attr_decomp_times = __ATTR(decomp_times, - 0444, nx842_timehist_show, NULL); - -static ssize_t nx842_timehist_show(struct device *dev, - struct device_attribute *attr, char *buf) { - char *p = buf; - struct nx842_devdata *local_devdata; - atomic64_t *times; - int bytes_remain = PAGE_SIZE; - int bytes; - int i; - - rcu_read_lock(); - local_devdata = rcu_dereference(devdata); - if (!local_devdata) { - rcu_read_unlock(); - return 0; - } + *dlen = 0; - if (attr == &dev_attr_comp_times) - times = local_devdata->counters->comp_times; - else if (attr == &dev_attr_decomp_times) - times = local_devdata->counters->decomp_times; - else { - rcu_read_unlock(); - return 0; - } + hdr = (struct nx842_crypto_header *)src; - for (i = 0; i < (NX842_HIST_SLOTS - 2); i++) { - bytes = snprintf(p, bytes_remain, "%u-%uus:\t%ld\n", - i ? (2<<(i-1)) : 0, (2<lock); -static struct attribute *nx842_sysfs_entries[] = { - &dev_attr_comp_complete.attr, - &dev_attr_comp_failed.attr, - &dev_attr_decomp_complete.attr, - &dev_attr_decomp_failed.attr, - &dev_attr_swdecomp.attr, - &dev_attr_comp_times.attr, - &dev_attr_decomp_times.attr, - NULL, -}; - -static struct attribute_group nx842_attribute_group = { - .name = NULL, /* put in device directory */ - .attrs = nx842_sysfs_entries, -}; - -static int __init nx842_probe(struct vio_dev *viodev, - const struct vio_device_id *id) -{ - struct nx842_devdata *old_devdata, *new_devdata = NULL; - unsigned long flags; - int ret = 0; - - spin_lock_irqsave(&devdata_mutex, flags); - old_devdata = rcu_dereference_check(devdata, - lockdep_is_held(&devdata_mutex)); - - if (old_devdata && old_devdata->vdev != NULL) { - dev_err(&viodev->dev, "%s: Attempt to register more than one instance of the hardware\n", __func__); - ret = -1; - goto error_unlock; - } + /* If it doesn't start with our header magic number, assume it's a raw + * 842 compressed buffer and pass it directly to the hardware driver + */ + if (be16_to_cpu(hdr->magic) != NX842_CRYPTO_MAGIC) { + struct nx842_crypto_header_group g = { + .padding = 0, + .compressed_length = cpu_to_be32(p.iremain), + .uncompressed_length = cpu_to_be32(p.oremain), + }; - dev_set_drvdata(&viodev->dev, NULL); + ret = decompress(ctx, &p, &g, &c, 0); + if (ret) + goto unlock; - new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS); - if (!new_devdata) { - dev_err(&viodev->dev, "%s: Could not allocate memory for device data\n", __func__); - ret = -ENOMEM; - goto error_unlock; + goto success; } - new_devdata->counters = kzalloc(sizeof(*new_devdata->counters), - GFP_NOFS); - if (!new_devdata->counters) { - dev_err(&viodev->dev, "%s: Could not allocate memory for performance counters\n", __func__); - ret = -ENOMEM; - goto error_unlock; + if (!hdr->groups) { + pr_err("header has no groups\n"); + ret = -EINVAL; + goto unlock; } - - new_devdata->vdev = viodev; - new_devdata->dev = &viodev->dev; - nx842_OF_set_defaults(new_devdata); - - rcu_assign_pointer(devdata, new_devdata); - spin_unlock_irqrestore(&devdata_mutex, flags); - synchronize_rcu(); - kfree(old_devdata); - - of_reconfig_notifier_register(&nx842_of_nb); - - ret = nx842_OF_upd(NULL); - if (ret && ret != -ENODEV) { - dev_err(&viodev->dev, "could not parse device tree. %d\n", ret); - ret = -1; - goto error; + if (hdr->groups > NX842_CRYPTO_GROUP_MAX) { + pr_err("header has too many groups %x, max %x\n", + hdr->groups, NX842_CRYPTO_GROUP_MAX); + ret = -EINVAL; + goto unlock; } - rcu_read_lock(); - dev_set_drvdata(&viodev->dev, rcu_dereference(devdata)); - rcu_read_unlock(); - - if (sysfs_create_group(&viodev->dev.kobj, &nx842_attribute_group)) { - dev_err(&viodev->dev, "could not create sysfs device attributes\n"); - ret = -1; - goto error; + hdr_len = NX842_CRYPTO_HEADER_SIZE(hdr->groups); + if (hdr_len > slen) { + ret = -EOVERFLOW; + goto unlock; } - return 0; + memcpy(&ctx->header, src, hdr_len); + hdr = &ctx->header; -error_unlock: - spin_unlock_irqrestore(&devdata_mutex, flags); - if (new_devdata) - kfree(new_devdata->counters); - kfree(new_devdata); -error: - return ret; -} - -static int __exit nx842_remove(struct vio_dev *viodev) -{ - struct nx842_devdata *old_devdata; - unsigned long flags; - - pr_info("Removing IBM Power 842 compression device\n"); - sysfs_remove_group(&viodev->dev.kobj, &nx842_attribute_group); - - spin_lock_irqsave(&devdata_mutex, flags); - old_devdata = rcu_dereference_check(devdata, - lockdep_is_held(&devdata_mutex)); - of_reconfig_notifier_unregister(&nx842_of_nb); - RCU_INIT_POINTER(devdata, NULL); - spin_unlock_irqrestore(&devdata_mutex, flags); - synchronize_rcu(); - dev_set_drvdata(&viodev->dev, NULL); - if (old_devdata) - kfree(old_devdata->counters); - kfree(old_devdata); - return 0; -} - -static struct vio_device_id nx842_driver_ids[] = { - {"ibm,compression-v1", "ibm,compression"}, - {"", ""}, -}; + for (n = 0; n < hdr->groups; n++) { + /* ignore applies to last group */ + if (n + 1 == hdr->groups) + ignore = be16_to_cpu(hdr->ignore); -static struct vio_driver nx842_driver = { - .name = MODULE_NAME, - .probe = nx842_probe, - .remove = __exit_p(nx842_remove), - .get_desired_dma = nx842_get_desired_dma, - .id_table = nx842_driver_ids, -}; - -static int __init nx842_init(void) -{ - struct nx842_devdata *new_devdata; - pr_info("Registering IBM Power 842 compression driver\n"); - - RCU_INIT_POINTER(devdata, NULL); - new_devdata = kzalloc(sizeof(*new_devdata), GFP_KERNEL); - if (!new_devdata) { - pr_err("Could not allocate memory for device data\n"); - return -ENOMEM; + ret = decompress(ctx, &p, &hdr->group[n], &c, ignore); + if (ret) + goto unlock; } - new_devdata->status = UNAVAILABLE; - RCU_INIT_POINTER(devdata, new_devdata); - return vio_register_driver(&nx842_driver); -} +success: + *dlen = p.ototal; -module_init(nx842_init); + pr_debug("decompress total slen %x dlen %x\n", slen, *dlen); -static void __exit nx842_exit(void) -{ - struct nx842_devdata *old_devdata; - unsigned long flags; - - pr_info("Exiting IBM Power 842 compression driver\n"); - spin_lock_irqsave(&devdata_mutex, flags); - old_devdata = rcu_dereference_check(devdata, - lockdep_is_held(&devdata_mutex)); - RCU_INIT_POINTER(devdata, NULL); - spin_unlock_irqrestore(&devdata_mutex, flags); - synchronize_rcu(); - if (old_devdata) - dev_set_drvdata(old_devdata->dev, NULL); - kfree(old_devdata); - vio_unregister_driver(&nx842_driver); -} - -module_exit(nx842_exit); - -/********************************* - * 842 software decompressor -*********************************/ -typedef int (*sw842_template_op)(const char **, int *, unsigned char **, - struct sw842_fifo *); - -static int sw842_data8(const char **, int *, unsigned char **, - struct sw842_fifo *); -static int sw842_data4(const char **, int *, unsigned char **, - struct sw842_fifo *); -static int sw842_data2(const char **, int *, unsigned char **, - struct sw842_fifo *); -static int sw842_ptr8(const char **, int *, unsigned char **, - struct sw842_fifo *); -static int sw842_ptr4(const char **, int *, unsigned char **, - struct sw842_fifo *); -static int sw842_ptr2(const char **, int *, unsigned char **, - struct sw842_fifo *); - -/* special templates */ -#define SW842_TMPL_REPEAT 0x1B -#define SW842_TMPL_ZEROS 0x1C -#define SW842_TMPL_EOF 0x1E - -static sw842_template_op sw842_tmpl_ops[26][4] = { - { sw842_data8, NULL}, /* 0 (00000) */ - { sw842_data4, sw842_data2, sw842_ptr2, NULL}, - { sw842_data4, sw842_ptr2, sw842_data2, NULL}, - { sw842_data4, sw842_ptr2, sw842_ptr2, NULL}, - { sw842_data4, sw842_ptr4, NULL}, - { sw842_data2, sw842_ptr2, sw842_data4, NULL}, - { sw842_data2, sw842_ptr2, sw842_data2, sw842_ptr2}, - { sw842_data2, sw842_ptr2, sw842_ptr2, sw842_data2}, - { sw842_data2, sw842_ptr2, sw842_ptr2, sw842_ptr2,}, - { sw842_data2, sw842_ptr2, sw842_ptr4, NULL}, - { sw842_ptr2, sw842_data2, sw842_data4, NULL}, /* 10 (01010) */ - { sw842_ptr2, sw842_data4, sw842_ptr2, NULL}, - { sw842_ptr2, sw842_data2, sw842_ptr2, sw842_data2}, - { sw842_ptr2, sw842_data2, sw842_ptr2, sw842_ptr2}, - { sw842_ptr2, sw842_data2, sw842_ptr4, NULL}, - { sw842_ptr2, sw842_ptr2, sw842_data4, NULL}, - { sw842_ptr2, sw842_ptr2, sw842_data2, sw842_ptr2}, - { sw842_ptr2, sw842_ptr2, sw842_ptr2, sw842_data2}, - { sw842_ptr2, sw842_ptr2, sw842_ptr2, sw842_ptr2}, - { sw842_ptr2, sw842_ptr2, sw842_ptr4, NULL}, - { sw842_ptr4, sw842_data4, NULL}, /* 20 (10100) */ - { sw842_ptr4, sw842_data2, sw842_ptr2, NULL}, - { sw842_ptr4, sw842_ptr2, sw842_data2, NULL}, - { sw842_ptr4, sw842_ptr2, sw842_ptr2, NULL}, - { sw842_ptr4, sw842_ptr4, NULL}, - { sw842_ptr8, NULL} -}; - -/* Software decompress helpers */ - -static uint8_t sw842_get_byte(const char *buf, int bit) -{ - uint8_t tmpl; - uint16_t tmp; - tmp = htons(*(uint16_t *)(buf)); - tmp = (uint16_t)(tmp << bit); - tmp = ntohs(tmp); - memcpy(&tmpl, &tmp, 1); - return tmpl; -} - -static uint8_t sw842_get_template(const char **buf, int *bit) -{ - uint8_t byte; - byte = sw842_get_byte(*buf, *bit); - byte = byte >> 3; - byte &= 0x1F; - *buf += (*bit + 5) / 8; - *bit = (*bit + 5) % 8; - return byte; -} - -/* repeat_count happens to be 5-bit too (like the template) */ -static uint8_t sw842_get_repeat_count(const char **buf, int *bit) -{ - uint8_t byte; - byte = sw842_get_byte(*buf, *bit); - byte = byte >> 2; - byte &= 0x3F; - *buf += (*bit + 6) / 8; - *bit = (*bit + 6) % 8; - return byte; -} - -static uint8_t sw842_get_ptr2(const char **buf, int *bit) -{ - uint8_t ptr; - ptr = sw842_get_byte(*buf, *bit); - (*buf)++; - return ptr; -} + ret = 0; -static uint16_t sw842_get_ptr4(const char **buf, int *bit, - struct sw842_fifo *fifo) -{ - uint16_t ptr; - ptr = htons(*(uint16_t *)(*buf)); - ptr = (uint16_t)(ptr << *bit); - ptr = ptr >> 7; - ptr &= 0x01FF; - *buf += (*bit + 9) / 8; - *bit = (*bit + 9) % 8; - return ptr; -} - -static uint8_t sw842_get_ptr8(const char **buf, int *bit, - struct sw842_fifo *fifo) -{ - return sw842_get_ptr2(buf, bit); -} - -/* Software decompress template ops */ - -static int sw842_data8(const char **inbuf, int *inbit, - unsigned char **outbuf, struct sw842_fifo *fifo) -{ - int ret; - - ret = sw842_data4(inbuf, inbit, outbuf, fifo); - if (ret) - return ret; - ret = sw842_data4(inbuf, inbit, outbuf, fifo); - return ret; -} - -static int sw842_data4(const char **inbuf, int *inbit, - unsigned char **outbuf, struct sw842_fifo *fifo) -{ - int ret; +unlock: + spin_unlock_bh(&ctx->lock); - ret = sw842_data2(inbuf, inbit, outbuf, fifo); - if (ret) - return ret; - ret = sw842_data2(inbuf, inbit, outbuf, fifo); return ret; } +EXPORT_SYMBOL_GPL(nx842_crypto_decompress); -static int sw842_data2(const char **inbuf, int *inbit, - unsigned char **outbuf, struct sw842_fifo *fifo) -{ - **outbuf = sw842_get_byte(*inbuf, *inbit); - (*inbuf)++; - (*outbuf)++; - **outbuf = sw842_get_byte(*inbuf, *inbit); - (*inbuf)++; - (*outbuf)++; - return 0; -} - -static int sw842_ptr8(const char **inbuf, int *inbit, - unsigned char **outbuf, struct sw842_fifo *fifo) -{ - uint8_t ptr; - ptr = sw842_get_ptr8(inbuf, inbit, fifo); - if (!fifo->f84_full && (ptr >= fifo->f8_count)) - return 1; - memcpy(*outbuf, fifo->f8[ptr], 8); - *outbuf += 8; - return 0; -} - -static int sw842_ptr4(const char **inbuf, int *inbit, - unsigned char **outbuf, struct sw842_fifo *fifo) -{ - uint16_t ptr; - ptr = sw842_get_ptr4(inbuf, inbit, fifo); - if (!fifo->f84_full && (ptr >= fifo->f4_count)) - return 1; - memcpy(*outbuf, fifo->f4[ptr], 4); - *outbuf += 4; - return 0; -} - -static int sw842_ptr2(const char **inbuf, int *inbit, - unsigned char **outbuf, struct sw842_fifo *fifo) -{ - uint8_t ptr; - ptr = sw842_get_ptr2(inbuf, inbit); - if (!fifo->f2_full && (ptr >= fifo->f2_count)) - return 1; - memcpy(*outbuf, fifo->f2[ptr], 2); - *outbuf += 2; - return 0; -} - -static void sw842_copy_to_fifo(const char *buf, struct sw842_fifo *fifo) -{ - unsigned char initial_f2count = fifo->f2_count; - - memcpy(fifo->f8[fifo->f8_count], buf, 8); - fifo->f4_count += 2; - fifo->f8_count += 1; - - if (!fifo->f84_full && fifo->f4_count >= 512) { - fifo->f84_full = 1; - fifo->f4_count /= 512; - } - - memcpy(fifo->f2[fifo->f2_count++], buf, 2); - memcpy(fifo->f2[fifo->f2_count++], buf + 2, 2); - memcpy(fifo->f2[fifo->f2_count++], buf + 4, 2); - memcpy(fifo->f2[fifo->f2_count++], buf + 6, 2); - if (fifo->f2_count < initial_f2count) - fifo->f2_full = 1; -} - -static int sw842_decompress(const unsigned char *src, int srclen, - unsigned char *dst, int *destlen, - const void *wrkmem) -{ - uint8_t tmpl; - const char *inbuf; - int inbit = 0; - unsigned char *outbuf, *outbuf_end, *origbuf, *prevbuf; - const char *inbuf_end; - sw842_template_op op; - int opindex; - int i, repeat_count; - struct sw842_fifo *fifo; - int ret = 0; - - fifo = &((struct nx842_workmem *)(wrkmem))->swfifo; - memset(fifo, 0, sizeof(*fifo)); - - origbuf = NULL; - inbuf = src; - inbuf_end = src + srclen; - outbuf = dst; - outbuf_end = dst + *destlen; - - while ((tmpl = sw842_get_template(&inbuf, &inbit)) != SW842_TMPL_EOF) { - if (inbuf >= inbuf_end) { - ret = -EINVAL; - goto out; - } - - opindex = 0; - prevbuf = origbuf; - origbuf = outbuf; - switch (tmpl) { - case SW842_TMPL_REPEAT: - if (prevbuf == NULL) { - ret = -EINVAL; - goto out; - } - - repeat_count = sw842_get_repeat_count(&inbuf, - &inbit) + 1; - - /* Did the repeat count advance past the end of input */ - if (inbuf > inbuf_end) { - ret = -EINVAL; - goto out; - } - - for (i = 0; i < repeat_count; i++) { - /* Would this overflow the output buffer */ - if ((outbuf + 8) > outbuf_end) { - ret = -ENOSPC; - goto out; - } - - memcpy(outbuf, prevbuf, 8); - sw842_copy_to_fifo(outbuf, fifo); - outbuf += 8; - } - break; - - case SW842_TMPL_ZEROS: - /* Would this overflow the output buffer */ - if ((outbuf + 8) > outbuf_end) { - ret = -ENOSPC; - goto out; - } - - memset(outbuf, 0, 8); - sw842_copy_to_fifo(outbuf, fifo); - outbuf += 8; - break; - - default: - if (tmpl > 25) { - ret = -EINVAL; - goto out; - } - - /* Does this go past the end of the input buffer */ - if ((inbuf + 2) > inbuf_end) { - ret = -EINVAL; - goto out; - } - - /* Would this overflow the output buffer */ - if ((outbuf + 8) > outbuf_end) { - ret = -ENOSPC; - goto out; - } - - while (opindex < 4 && - (op = sw842_tmpl_ops[tmpl][opindex++]) - != NULL) { - ret = (*op)(&inbuf, &inbit, &outbuf, fifo); - if (ret) { - ret = -EINVAL; - goto out; - } - sw842_copy_to_fifo(origbuf, fifo); - } - } - } - -out: - if (!ret) - *destlen = (unsigned int)(outbuf - dst); - else - *destlen = 0; - - return ret; -} +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("IBM PowerPC Nest (NX) 842 Hardware Compression Driver"); +MODULE_AUTHOR("Dan Streetman "); diff --git a/kernel/drivers/crypto/nx/nx-842.h b/kernel/drivers/crypto/nx/nx-842.h new file mode 100644 index 000000000..a4eee3bba --- /dev/null +++ b/kernel/drivers/crypto/nx/nx-842.h @@ -0,0 +1,185 @@ + +#ifndef __NX_842_H__ +#define __NX_842_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Restrictions on Data Descriptor List (DDL) and Entry (DDE) buffers + * + * From NX P8 workbook, sec 4.9.1 "842 details" + * Each DDE buffer is 128 byte aligned + * Each DDE buffer size is a multiple of 32 bytes (except the last) + * The last DDE buffer size is a multiple of 8 bytes + */ +#define DDE_BUFFER_ALIGN (128) +#define DDE_BUFFER_SIZE_MULT (32) +#define DDE_BUFFER_LAST_MULT (8) + +/* Arbitrary DDL length limit + * Allows max buffer size of MAX-1 to MAX pages + * (depending on alignment) + */ +#define DDL_LEN_MAX (17) + +/* CCW 842 CI/FC masks + * NX P8 workbook, section 4.3.1, figure 4-6 + * "CI/FC Boundary by NX CT type" + */ +#define CCW_CI_842 (0x00003ff8) +#define CCW_FC_842 (0x00000007) + +/* CCW Function Codes (FC) for 842 + * NX P8 workbook, section 4.9, table 4-28 + * "Function Code Definitions for 842 Memory Compression" + */ +#define CCW_FC_842_COMP_NOCRC (0) +#define CCW_FC_842_COMP_CRC (1) +#define CCW_FC_842_DECOMP_NOCRC (2) +#define CCW_FC_842_DECOMP_CRC (3) +#define CCW_FC_842_MOVE (4) + +/* CSB CC Error Types for 842 + * NX P8 workbook, section 4.10.3, table 4-30 + * "Reported Error Types Summary Table" + */ +/* These are all duplicates of existing codes defined in icswx.h. */ +#define CSB_CC_TRANSLATION_DUP1 (80) +#define CSB_CC_TRANSLATION_DUP2 (82) +#define CSB_CC_TRANSLATION_DUP3 (84) +#define CSB_CC_TRANSLATION_DUP4 (86) +#define CSB_CC_TRANSLATION_DUP5 (92) +#define CSB_CC_TRANSLATION_DUP6 (94) +#define CSB_CC_PROTECTION_DUP1 (81) +#define CSB_CC_PROTECTION_DUP2 (83) +#define CSB_CC_PROTECTION_DUP3 (85) +#define CSB_CC_PROTECTION_DUP4 (87) +#define CSB_CC_PROTECTION_DUP5 (93) +#define CSB_CC_PROTECTION_DUP6 (95) +#define CSB_CC_RD_EXTERNAL_DUP1 (89) +#define CSB_CC_RD_EXTERNAL_DUP2 (90) +#define CSB_CC_RD_EXTERNAL_DUP3 (91) +/* These are specific to NX */ +/* 842 codes */ +#define CSB_CC_TPBC_GT_SPBC (64) /* no error, but >1 comp ratio */ +#define CSB_CC_CRC_MISMATCH (65) /* decomp crc mismatch */ +#define CSB_CC_TEMPL_INVALID (66) /* decomp invalid template value */ +#define CSB_CC_TEMPL_OVERFLOW (67) /* decomp template shows data after end */ +/* sym crypt codes */ +#define CSB_CC_DECRYPT_OVERFLOW (64) +/* asym crypt codes */ +#define CSB_CC_MINV_OVERFLOW (128) +/* These are reserved for hypervisor use */ +#define CSB_CC_HYP_RESERVE_START (240) +#define CSB_CC_HYP_RESERVE_END (253) +#define CSB_CC_HYP_NO_HW (254) +#define CSB_CC_HYP_HANG_ABORTED (255) + +/* CCB Completion Modes (CM) for 842 + * NX P8 workbook, section 4.3, figure 4-5 + * "CRB Details - Normal Cop_Req (CL=00, C=1)" + */ +#define CCB_CM_EXTRA_WRITE (CCB_CM0_ALL_COMPLETIONS & CCB_CM12_STORE) +#define CCB_CM_INTERRUPT (CCB_CM0_ALL_COMPLETIONS & CCB_CM12_INTERRUPT) + +#define LEN_ON_SIZE(pa, size) ((size) - ((pa) & ((size) - 1))) +#define LEN_ON_PAGE(pa) LEN_ON_SIZE(pa, PAGE_SIZE) + +static inline unsigned long nx842_get_pa(void *addr) +{ + if (!is_vmalloc_addr(addr)) + return __pa(addr); + + return page_to_phys(vmalloc_to_page(addr)) + offset_in_page(addr); +} + +/* Get/Set bit fields */ +#define MASK_LSH(m) (__builtin_ffsl(m) - 1) +#define GET_FIELD(v, m) (((v) & (m)) >> MASK_LSH(m)) +#define SET_FIELD(v, m, val) (((v) & ~(m)) | (((val) << MASK_LSH(m)) & (m))) + +/** + * This provides the driver's constraints. Different nx842 implementations + * may have varying requirements. The constraints are: + * @alignment: All buffers should be aligned to this + * @multiple: All buffer lengths should be a multiple of this + * @minimum: Buffer lengths must not be less than this amount + * @maximum: Buffer lengths must not be more than this amount + * + * The constraints apply to all buffers and lengths, both input and output, + * for both compression and decompression, except for the minimum which + * only applies to compression input and decompression output; the + * compressed data can be less than the minimum constraint. It can be + * assumed that compressed data will always adhere to the multiple + * constraint. + * + * The driver may succeed even if these constraints are violated; + * however the driver can return failure or suffer reduced performance + * if any constraint is not met. + */ +struct nx842_constraints { + int alignment; + int multiple; + int minimum; + int maximum; +}; + +struct nx842_driver { + char *name; + struct module *owner; + size_t workmem_size; + + struct nx842_constraints *constraints; + + int (*compress)(const unsigned char *in, unsigned int in_len, + unsigned char *out, unsigned int *out_len, + void *wrkmem); + int (*decompress)(const unsigned char *in, unsigned int in_len, + unsigned char *out, unsigned int *out_len, + void *wrkmem); +}; + +struct nx842_crypto_header_group { + __be16 padding; /* unused bytes at start of group */ + __be32 compressed_length; /* compressed bytes in group */ + __be32 uncompressed_length; /* bytes after decompression */ +} __packed; + +struct nx842_crypto_header { + __be16 magic; /* NX842_CRYPTO_MAGIC */ + __be16 ignore; /* decompressed end bytes to ignore */ + u8 groups; /* total groups in this header */ + struct nx842_crypto_header_group group[]; +} __packed; + +#define NX842_CRYPTO_GROUP_MAX (0x20) + +struct nx842_crypto_ctx { + spinlock_t lock; + + u8 *wmem; + u8 *sbounce, *dbounce; + + struct nx842_crypto_header header; + struct nx842_crypto_header_group group[NX842_CRYPTO_GROUP_MAX]; + + struct nx842_driver *driver; +}; + +int nx842_crypto_init(struct crypto_tfm *tfm, struct nx842_driver *driver); +void nx842_crypto_exit(struct crypto_tfm *tfm); +int nx842_crypto_compress(struct crypto_tfm *tfm, + const u8 *src, unsigned int slen, + u8 *dst, unsigned int *dlen); +int nx842_crypto_decompress(struct crypto_tfm *tfm, + const u8 *src, unsigned int slen, + u8 *dst, unsigned int *dlen); + +#endif /* __NX_842_H__ */ diff --git a/kernel/drivers/crypto/nx/nx-aes-ccm.c b/kernel/drivers/crypto/nx/nx-aes-ccm.c index e4311ce0c..7038f364a 100644 --- a/kernel/drivers/crypto/nx/nx-aes-ccm.c +++ b/kernel/drivers/crypto/nx/nx-aes-ccm.c @@ -94,8 +94,6 @@ static int ccm_aes_nx_setauthsize(struct crypto_aead *tfm, return -EINVAL; } - crypto_aead_crt(tfm)->authsize = authsize; - return 0; } @@ -111,8 +109,6 @@ static int ccm4309_aes_nx_setauthsize(struct crypto_aead *tfm, return -EINVAL; } - crypto_aead_crt(tfm)->authsize = authsize; - return 0; } @@ -174,6 +170,7 @@ static int generate_pat(u8 *iv, struct nx_crypto_ctx *nx_ctx, unsigned int authsize, unsigned int nbytes, + unsigned int assoclen, u8 *out) { struct nx_sg *nx_insg = nx_ctx->in_sg; @@ -200,16 +197,16 @@ static int generate_pat(u8 *iv, * greater than 2^32. */ - if (!req->assoclen) { + if (!assoclen) { b0 = nx_ctx->csbcpb->cpb.aes_ccm.in_pat_or_b0; - } else if (req->assoclen <= 14) { + } else if (assoclen <= 14) { /* if associated data is 14 bytes or less, we do 1 GCM * operation on 2 AES blocks, B0 (stored in the csbcpb) and B1, * which is fed in through the source buffers here */ b0 = nx_ctx->csbcpb->cpb.aes_ccm.in_pat_or_b0; b1 = nx_ctx->priv.ccm.iauth_tag; - iauth_len = req->assoclen; - } else if (req->assoclen <= 65280) { + iauth_len = assoclen; + } else if (assoclen <= 65280) { /* if associated data is less than (2^16 - 2^8), we construct * B1 differently and feed in the associated data to a CCA * operation */ @@ -223,7 +220,7 @@ static int generate_pat(u8 *iv, } /* generate B0 */ - rc = generate_b0(iv, req->assoclen, authsize, nbytes, b0); + rc = generate_b0(iv, assoclen, authsize, nbytes, b0); if (rc) return rc; @@ -233,22 +230,22 @@ static int generate_pat(u8 *iv, */ if (b1) { memset(b1, 0, 16); - if (req->assoclen <= 65280) { - *(u16 *)b1 = (u16)req->assoclen; - scatterwalk_map_and_copy(b1 + 2, req->assoc, 0, + if (assoclen <= 65280) { + *(u16 *)b1 = assoclen; + scatterwalk_map_and_copy(b1 + 2, req->src, 0, iauth_len, SCATTERWALK_FROM_SG); } else { *(u16 *)b1 = (u16)(0xfffe); - *(u32 *)&b1[2] = (u32)req->assoclen; - scatterwalk_map_and_copy(b1 + 6, req->assoc, 0, + *(u32 *)&b1[2] = assoclen; + scatterwalk_map_and_copy(b1 + 6, req->src, 0, iauth_len, SCATTERWALK_FROM_SG); } } /* now copy any remaining AAD to scatterlist and call nx... */ - if (!req->assoclen) { + if (!assoclen) { return rc; - } else if (req->assoclen <= 14) { + } else if (assoclen <= 14) { unsigned int len = 16; nx_insg = nx_build_sg_list(nx_insg, b1, &len, nx_ctx->ap->sglen); @@ -280,7 +277,7 @@ static int generate_pat(u8 *iv, return rc; atomic_inc(&(nx_ctx->stats->aes_ops)); - atomic64_add(req->assoclen, &(nx_ctx->stats->aes_bytes)); + atomic64_add(assoclen, &nx_ctx->stats->aes_bytes); } else { unsigned int processed = 0, to_process; @@ -294,15 +291,15 @@ static int generate_pat(u8 *iv, nx_ctx->ap->databytelen/NX_PAGE_SIZE); do { - to_process = min_t(u32, req->assoclen - processed, + to_process = min_t(u32, assoclen - processed, nx_ctx->ap->databytelen); nx_insg = nx_walk_and_build(nx_ctx->in_sg, nx_ctx->ap->sglen, - req->assoc, processed, + req->src, processed, &to_process); - if ((to_process + processed) < req->assoclen) { + if ((to_process + processed) < assoclen) { NX_CPB_FDM(nx_ctx->csbcpb_aead) |= NX_FDM_INTERMEDIATE; } else { @@ -328,11 +325,10 @@ static int generate_pat(u8 *iv, NX_CPB_FDM(nx_ctx->csbcpb_aead) |= NX_FDM_CONTINUATION; atomic_inc(&(nx_ctx->stats->aes_ops)); - atomic64_add(req->assoclen, - &(nx_ctx->stats->aes_bytes)); + atomic64_add(assoclen, &nx_ctx->stats->aes_bytes); processed += to_process; - } while (processed < req->assoclen); + } while (processed < assoclen); result = nx_ctx->csbcpb_aead->cpb.aes_cca.out_pat_or_b0; } @@ -343,7 +339,8 @@ static int generate_pat(u8 *iv, } static int ccm_nx_decrypt(struct aead_request *req, - struct blkcipher_desc *desc) + struct blkcipher_desc *desc, + unsigned int assoclen) { struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm); struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; @@ -360,10 +357,10 @@ static int ccm_nx_decrypt(struct aead_request *req, /* copy out the auth tag to compare with later */ scatterwalk_map_and_copy(priv->oauth_tag, - req->src, nbytes, authsize, + req->src, nbytes + req->assoclen, authsize, SCATTERWALK_FROM_SG); - rc = generate_pat(desc->info, req, nx_ctx, authsize, nbytes, + rc = generate_pat(desc->info, req, nx_ctx, authsize, nbytes, assoclen, csbcpb->cpb.aes_ccm.in_pat_or_b0); if (rc) goto out; @@ -383,8 +380,8 @@ static int ccm_nx_decrypt(struct aead_request *req, NX_CPB_FDM(nx_ctx->csbcpb) &= ~NX_FDM_ENDE_ENCRYPT; rc = nx_build_sg_lists(nx_ctx, desc, req->dst, req->src, - &to_process, processed, - csbcpb->cpb.aes_ccm.iv_or_ctr); + &to_process, processed + req->assoclen, + csbcpb->cpb.aes_ccm.iv_or_ctr); if (rc) goto out; @@ -412,7 +409,7 @@ static int ccm_nx_decrypt(struct aead_request *req, processed += to_process; } while (processed < nbytes); - rc = memcmp(csbcpb->cpb.aes_ccm.out_pat_or_mac, priv->oauth_tag, + rc = crypto_memneq(csbcpb->cpb.aes_ccm.out_pat_or_mac, priv->oauth_tag, authsize) ? -EBADMSG : 0; out: spin_unlock_irqrestore(&nx_ctx->lock, irq_flags); @@ -420,7 +417,8 @@ out: } static int ccm_nx_encrypt(struct aead_request *req, - struct blkcipher_desc *desc) + struct blkcipher_desc *desc, + unsigned int assoclen) { struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm); struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; @@ -432,7 +430,7 @@ static int ccm_nx_encrypt(struct aead_request *req, spin_lock_irqsave(&nx_ctx->lock, irq_flags); - rc = generate_pat(desc->info, req, nx_ctx, authsize, nbytes, + rc = generate_pat(desc->info, req, nx_ctx, authsize, nbytes, assoclen, csbcpb->cpb.aes_ccm.in_pat_or_b0); if (rc) goto out; @@ -451,7 +449,7 @@ static int ccm_nx_encrypt(struct aead_request *req, NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT; rc = nx_build_sg_lists(nx_ctx, desc, req->dst, req->src, - &to_process, processed, + &to_process, processed + req->assoclen, csbcpb->cpb.aes_ccm.iv_or_ctr); if (rc) goto out; @@ -483,7 +481,7 @@ static int ccm_nx_encrypt(struct aead_request *req, /* copy out the auth tag */ scatterwalk_map_and_copy(csbcpb->cpb.aes_ccm.out_pat_or_mac, - req->dst, nbytes, authsize, + req->dst, nbytes + req->assoclen, authsize, SCATTERWALK_TO_SG); out: @@ -503,9 +501,8 @@ static int ccm4309_aes_nx_encrypt(struct aead_request *req) memcpy(iv + 4, req->iv, 8); desc.info = iv; - desc.tfm = (struct crypto_blkcipher *)req->base.tfm; - return ccm_nx_encrypt(req, &desc); + return ccm_nx_encrypt(req, &desc, req->assoclen - 8); } static int ccm_aes_nx_encrypt(struct aead_request *req) @@ -514,13 +511,12 @@ static int ccm_aes_nx_encrypt(struct aead_request *req) int rc; desc.info = req->iv; - desc.tfm = (struct crypto_blkcipher *)req->base.tfm; rc = crypto_ccm_check_iv(desc.info); if (rc) return rc; - return ccm_nx_encrypt(req, &desc); + return ccm_nx_encrypt(req, &desc, req->assoclen); } static int ccm4309_aes_nx_decrypt(struct aead_request *req) @@ -535,9 +531,8 @@ static int ccm4309_aes_nx_decrypt(struct aead_request *req) memcpy(iv + 4, req->iv, 8); desc.info = iv; - desc.tfm = (struct crypto_blkcipher *)req->base.tfm; - return ccm_nx_decrypt(req, &desc); + return ccm_nx_decrypt(req, &desc, req->assoclen - 8); } static int ccm_aes_nx_decrypt(struct aead_request *req) @@ -546,13 +541,12 @@ static int ccm_aes_nx_decrypt(struct aead_request *req) int rc; desc.info = req->iv; - desc.tfm = (struct crypto_blkcipher *)req->base.tfm; rc = crypto_ccm_check_iv(desc.info); if (rc) return rc; - return ccm_nx_decrypt(req, &desc); + return ccm_nx_decrypt(req, &desc, req->assoclen); } /* tell the block cipher walk routines that this is a stream cipher by @@ -560,47 +554,42 @@ static int ccm_aes_nx_decrypt(struct aead_request *req) * during encrypt/decrypt doesn't solve this problem, because it calls * blkcipher_walk_done under the covers, which doesn't use walk->blocksize, * but instead uses this tfm->blocksize. */ -struct crypto_alg nx_ccm_aes_alg = { - .cra_name = "ccm(aes)", - .cra_driver_name = "ccm-aes-nx", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_AEAD | - CRYPTO_ALG_NEED_FALLBACK, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct nx_crypto_ctx), - .cra_type = &crypto_aead_type, - .cra_module = THIS_MODULE, - .cra_init = nx_crypto_ctx_aes_ccm_init, - .cra_exit = nx_crypto_ctx_exit, - .cra_aead = { - .ivsize = AES_BLOCK_SIZE, - .maxauthsize = AES_BLOCK_SIZE, - .setkey = ccm_aes_nx_set_key, - .setauthsize = ccm_aes_nx_setauthsize, - .encrypt = ccm_aes_nx_encrypt, - .decrypt = ccm_aes_nx_decrypt, - } +struct aead_alg nx_ccm_aes_alg = { + .base = { + .cra_name = "ccm(aes)", + .cra_driver_name = "ccm-aes-nx", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct nx_crypto_ctx), + .cra_module = THIS_MODULE, + }, + .init = nx_crypto_ctx_aes_ccm_init, + .exit = nx_crypto_ctx_aead_exit, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + .setkey = ccm_aes_nx_set_key, + .setauthsize = ccm_aes_nx_setauthsize, + .encrypt = ccm_aes_nx_encrypt, + .decrypt = ccm_aes_nx_decrypt, }; -struct crypto_alg nx_ccm4309_aes_alg = { - .cra_name = "rfc4309(ccm(aes))", - .cra_driver_name = "rfc4309-ccm-aes-nx", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_AEAD | - CRYPTO_ALG_NEED_FALLBACK, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct nx_crypto_ctx), - .cra_type = &crypto_nivaead_type, - .cra_module = THIS_MODULE, - .cra_init = nx_crypto_ctx_aes_ccm_init, - .cra_exit = nx_crypto_ctx_exit, - .cra_aead = { - .ivsize = 8, - .maxauthsize = AES_BLOCK_SIZE, - .setkey = ccm4309_aes_nx_set_key, - .setauthsize = ccm4309_aes_nx_setauthsize, - .encrypt = ccm4309_aes_nx_encrypt, - .decrypt = ccm4309_aes_nx_decrypt, - .geniv = "seqiv", - } +struct aead_alg nx_ccm4309_aes_alg = { + .base = { + .cra_name = "rfc4309(ccm(aes))", + .cra_driver_name = "rfc4309-ccm-aes-nx", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct nx_crypto_ctx), + .cra_module = THIS_MODULE, + }, + .init = nx_crypto_ctx_aes_ccm_init, + .exit = nx_crypto_ctx_aead_exit, + .ivsize = 8, + .maxauthsize = AES_BLOCK_SIZE, + .setkey = ccm4309_aes_nx_set_key, + .setauthsize = ccm4309_aes_nx_setauthsize, + .encrypt = ccm4309_aes_nx_encrypt, + .decrypt = ccm4309_aes_nx_decrypt, }; diff --git a/kernel/drivers/crypto/nx/nx-aes-ctr.c b/kernel/drivers/crypto/nx/nx-aes-ctr.c index dd7e9f3f5..898c0a280 100644 --- a/kernel/drivers/crypto/nx/nx-aes-ctr.c +++ b/kernel/drivers/crypto/nx/nx-aes-ctr.c @@ -144,27 +144,6 @@ static int ctr3686_aes_nx_crypt(struct blkcipher_desc *desc, return ctr_aes_nx_crypt(desc, dst, src, nbytes); } -struct crypto_alg nx_ctr_aes_alg = { - .cra_name = "ctr(aes)", - .cra_driver_name = "ctr-aes-nx", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct nx_crypto_ctx), - .cra_type = &crypto_blkcipher_type, - .cra_module = THIS_MODULE, - .cra_init = nx_crypto_ctx_aes_ctr_init, - .cra_exit = nx_crypto_ctx_exit, - .cra_blkcipher = { - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - .setkey = ctr_aes_nx_set_key, - .encrypt = ctr_aes_nx_crypt, - .decrypt = ctr_aes_nx_crypt, - } -}; - struct crypto_alg nx_ctr3686_aes_alg = { .cra_name = "rfc3686(ctr(aes))", .cra_driver_name = "rfc3686-ctr-aes-nx", diff --git a/kernel/drivers/crypto/nx/nx-aes-gcm.c b/kernel/drivers/crypto/nx/nx-aes-gcm.c index c6ebeb644..abd465f47 100644 --- a/kernel/drivers/crypto/nx/nx-aes-gcm.c +++ b/kernel/drivers/crypto/nx/nx-aes-gcm.c @@ -25,7 +25,6 @@ #include #include #include -#include #include #include "nx_csbcpb.h" @@ -36,7 +35,7 @@ static int gcm_aes_nx_set_key(struct crypto_aead *tfm, const u8 *in_key, unsigned int key_len) { - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&tfm->base); + struct nx_crypto_ctx *nx_ctx = crypto_aead_ctx(tfm); struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; struct nx_csbcpb *csbcpb_aead = nx_ctx->csbcpb_aead; @@ -75,7 +74,7 @@ static int gcm4106_aes_nx_set_key(struct crypto_aead *tfm, const u8 *in_key, unsigned int key_len) { - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&tfm->base); + struct nx_crypto_ctx *nx_ctx = crypto_aead_ctx(tfm); char *nonce = nx_ctx->priv.gcm.nonce; int rc; @@ -93,17 +92,6 @@ out: return rc; } -static int gcm_aes_nx_setauthsize(struct crypto_aead *tfm, - unsigned int authsize) -{ - if (authsize > crypto_aead_alg(tfm)->maxauthsize) - return -EINVAL; - - crypto_aead_crt(tfm)->authsize = authsize; - - return 0; -} - static int gcm4106_aes_nx_setauthsize(struct crypto_aead *tfm, unsigned int authsize) { @@ -116,25 +104,24 @@ static int gcm4106_aes_nx_setauthsize(struct crypto_aead *tfm, return -EINVAL; } - crypto_aead_crt(tfm)->authsize = authsize; - return 0; } static int nx_gca(struct nx_crypto_ctx *nx_ctx, struct aead_request *req, - u8 *out) + u8 *out, + unsigned int assoclen) { int rc; struct nx_csbcpb *csbcpb_aead = nx_ctx->csbcpb_aead; struct scatter_walk walk; struct nx_sg *nx_sg = nx_ctx->in_sg; - unsigned int nbytes = req->assoclen; + unsigned int nbytes = assoclen; unsigned int processed = 0, to_process; unsigned int max_sg_len; if (nbytes <= AES_BLOCK_SIZE) { - scatterwalk_start(&walk, req->assoc); + scatterwalk_start(&walk, req->src); scatterwalk_copychunks(out, &walk, nbytes, SCATTERWALK_FROM_SG); scatterwalk_done(&walk, SCATTERWALK_FROM_SG, 0); return 0; @@ -159,7 +146,7 @@ static int nx_gca(struct nx_crypto_ctx *nx_ctx, NX_PAGE_SIZE * (max_sg_len - 1)); nx_sg = nx_walk_and_build(nx_ctx->in_sg, max_sg_len, - req->assoc, processed, &to_process); + req->src, processed, &to_process); if ((to_process + processed) < nbytes) NX_CPB_FDM(csbcpb_aead) |= NX_FDM_INTERMEDIATE; @@ -180,7 +167,7 @@ static int nx_gca(struct nx_crypto_ctx *nx_ctx, NX_CPB_FDM(csbcpb_aead) |= NX_FDM_CONTINUATION; atomic_inc(&(nx_ctx->stats->aes_ops)); - atomic64_add(req->assoclen, &(nx_ctx->stats->aes_bytes)); + atomic64_add(assoclen, &(nx_ctx->stats->aes_bytes)); processed += to_process; } while (processed < nbytes); @@ -190,13 +177,15 @@ static int nx_gca(struct nx_crypto_ctx *nx_ctx, return rc; } -static int gmac(struct aead_request *req, struct blkcipher_desc *desc) +static int gmac(struct aead_request *req, struct blkcipher_desc *desc, + unsigned int assoclen) { int rc; - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm); + struct nx_crypto_ctx *nx_ctx = + crypto_aead_ctx(crypto_aead_reqtfm(req)); struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; struct nx_sg *nx_sg; - unsigned int nbytes = req->assoclen; + unsigned int nbytes = assoclen; unsigned int processed = 0, to_process; unsigned int max_sg_len; @@ -225,7 +214,7 @@ static int gmac(struct aead_request *req, struct blkcipher_desc *desc) NX_PAGE_SIZE * (max_sg_len - 1)); nx_sg = nx_walk_and_build(nx_ctx->in_sg, max_sg_len, - req->assoc, processed, &to_process); + req->src, processed, &to_process); if ((to_process + processed) < nbytes) NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE; @@ -251,7 +240,7 @@ static int gmac(struct aead_request *req, struct blkcipher_desc *desc) NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION; atomic_inc(&(nx_ctx->stats->aes_ops)); - atomic64_add(req->assoclen, &(nx_ctx->stats->aes_bytes)); + atomic64_add(assoclen, &(nx_ctx->stats->aes_bytes)); processed += to_process; } while (processed < nbytes); @@ -266,7 +255,8 @@ static int gcm_empty(struct aead_request *req, struct blkcipher_desc *desc, int enc) { int rc; - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm); + struct nx_crypto_ctx *nx_ctx = + crypto_aead_ctx(crypto_aead_reqtfm(req)); struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; char out[AES_BLOCK_SIZE]; struct nx_sg *in_sg, *out_sg; @@ -327,9 +317,11 @@ out: return rc; } -static int gcm_aes_nx_crypt(struct aead_request *req, int enc) +static int gcm_aes_nx_crypt(struct aead_request *req, int enc, + unsigned int assoclen) { - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm); + struct nx_crypto_ctx *nx_ctx = + crypto_aead_ctx(crypto_aead_reqtfm(req)); struct nx_gcm_rctx *rctx = aead_request_ctx(req); struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; struct blkcipher_desc desc; @@ -345,10 +337,10 @@ static int gcm_aes_nx_crypt(struct aead_request *req, int enc) *(u32 *)(desc.info + NX_GCM_CTR_OFFSET) = 1; if (nbytes == 0) { - if (req->assoclen == 0) + if (assoclen == 0) rc = gcm_empty(req, &desc, enc); else - rc = gmac(req, &desc); + rc = gmac(req, &desc, assoclen); if (rc) goto out; else @@ -356,9 +348,10 @@ static int gcm_aes_nx_crypt(struct aead_request *req, int enc) } /* Process associated data */ - csbcpb->cpb.aes_gcm.bit_length_aad = req->assoclen * 8; - if (req->assoclen) { - rc = nx_gca(nx_ctx, req, csbcpb->cpb.aes_gcm.in_pat_or_aad); + csbcpb->cpb.aes_gcm.bit_length_aad = assoclen * 8; + if (assoclen) { + rc = nx_gca(nx_ctx, req, csbcpb->cpb.aes_gcm.in_pat_or_aad, + assoclen); if (rc) goto out; } @@ -376,9 +369,9 @@ static int gcm_aes_nx_crypt(struct aead_request *req, int enc) to_process = nbytes - processed; csbcpb->cpb.aes_gcm.bit_length_data = nbytes * 8; - desc.tfm = (struct crypto_blkcipher *) req->base.tfm; rc = nx_build_sg_lists(nx_ctx, &desc, req->dst, - req->src, &to_process, processed, + req->src, &to_process, + processed + req->assoclen, csbcpb->cpb.aes_gcm.iv_or_cnt); if (rc) @@ -413,18 +406,20 @@ static int gcm_aes_nx_crypt(struct aead_request *req, int enc) mac: if (enc) { /* copy out the auth tag */ - scatterwalk_map_and_copy(csbcpb->cpb.aes_gcm.out_pat_or_mac, - req->dst, nbytes, - crypto_aead_authsize(crypto_aead_reqtfm(req)), - SCATTERWALK_TO_SG); + scatterwalk_map_and_copy( + csbcpb->cpb.aes_gcm.out_pat_or_mac, + req->dst, req->assoclen + nbytes, + crypto_aead_authsize(crypto_aead_reqtfm(req)), + SCATTERWALK_TO_SG); } else { u8 *itag = nx_ctx->priv.gcm.iauth_tag; u8 *otag = csbcpb->cpb.aes_gcm.out_pat_or_mac; - scatterwalk_map_and_copy(itag, req->src, nbytes, - crypto_aead_authsize(crypto_aead_reqtfm(req)), - SCATTERWALK_FROM_SG); - rc = memcmp(itag, otag, + scatterwalk_map_and_copy( + itag, req->src, req->assoclen + nbytes, + crypto_aead_authsize(crypto_aead_reqtfm(req)), + SCATTERWALK_FROM_SG); + rc = crypto_memneq(itag, otag, crypto_aead_authsize(crypto_aead_reqtfm(req))) ? -EBADMSG : 0; } @@ -440,7 +435,7 @@ static int gcm_aes_nx_encrypt(struct aead_request *req) memcpy(iv, req->iv, 12); - return gcm_aes_nx_crypt(req, 1); + return gcm_aes_nx_crypt(req, 1, req->assoclen); } static int gcm_aes_nx_decrypt(struct aead_request *req) @@ -450,12 +445,13 @@ static int gcm_aes_nx_decrypt(struct aead_request *req) memcpy(iv, req->iv, 12); - return gcm_aes_nx_crypt(req, 0); + return gcm_aes_nx_crypt(req, 0, req->assoclen); } static int gcm4106_aes_nx_encrypt(struct aead_request *req) { - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm); + struct nx_crypto_ctx *nx_ctx = + crypto_aead_ctx(crypto_aead_reqtfm(req)); struct nx_gcm_rctx *rctx = aead_request_ctx(req); char *iv = rctx->iv; char *nonce = nx_ctx->priv.gcm.nonce; @@ -463,12 +459,16 @@ static int gcm4106_aes_nx_encrypt(struct aead_request *req) memcpy(iv, nonce, NX_GCM4106_NONCE_LEN); memcpy(iv + NX_GCM4106_NONCE_LEN, req->iv, 8); - return gcm_aes_nx_crypt(req, 1); + if (req->assoclen < 8) + return -EINVAL; + + return gcm_aes_nx_crypt(req, 1, req->assoclen - 8); } static int gcm4106_aes_nx_decrypt(struct aead_request *req) { - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm); + struct nx_crypto_ctx *nx_ctx = + crypto_aead_ctx(crypto_aead_reqtfm(req)); struct nx_gcm_rctx *rctx = aead_request_ctx(req); char *iv = rctx->iv; char *nonce = nx_ctx->priv.gcm.nonce; @@ -476,7 +476,10 @@ static int gcm4106_aes_nx_decrypt(struct aead_request *req) memcpy(iv, nonce, NX_GCM4106_NONCE_LEN); memcpy(iv + NX_GCM4106_NONCE_LEN, req->iv, 8); - return gcm_aes_nx_crypt(req, 0); + if (req->assoclen < 8) + return -EINVAL; + + return gcm_aes_nx_crypt(req, 0, req->assoclen - 8); } /* tell the block cipher walk routines that this is a stream cipher by @@ -484,45 +487,39 @@ static int gcm4106_aes_nx_decrypt(struct aead_request *req) * during encrypt/decrypt doesn't solve this problem, because it calls * blkcipher_walk_done under the covers, which doesn't use walk->blocksize, * but instead uses this tfm->blocksize. */ -struct crypto_alg nx_gcm_aes_alg = { - .cra_name = "gcm(aes)", - .cra_driver_name = "gcm-aes-nx", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_AEAD, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct nx_crypto_ctx), - .cra_type = &crypto_aead_type, - .cra_module = THIS_MODULE, - .cra_init = nx_crypto_ctx_aes_gcm_init, - .cra_exit = nx_crypto_ctx_exit, - .cra_aead = { - .ivsize = AES_BLOCK_SIZE, - .maxauthsize = AES_BLOCK_SIZE, - .setkey = gcm_aes_nx_set_key, - .setauthsize = gcm_aes_nx_setauthsize, - .encrypt = gcm_aes_nx_encrypt, - .decrypt = gcm_aes_nx_decrypt, - } +struct aead_alg nx_gcm_aes_alg = { + .base = { + .cra_name = "gcm(aes)", + .cra_driver_name = "gcm-aes-nx", + .cra_priority = 300, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct nx_crypto_ctx), + .cra_module = THIS_MODULE, + }, + .init = nx_crypto_ctx_aes_gcm_init, + .exit = nx_crypto_ctx_aead_exit, + .ivsize = 12, + .maxauthsize = AES_BLOCK_SIZE, + .setkey = gcm_aes_nx_set_key, + .encrypt = gcm_aes_nx_encrypt, + .decrypt = gcm_aes_nx_decrypt, }; -struct crypto_alg nx_gcm4106_aes_alg = { - .cra_name = "rfc4106(gcm(aes))", - .cra_driver_name = "rfc4106-gcm-aes-nx", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_AEAD, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct nx_crypto_ctx), - .cra_type = &crypto_nivaead_type, - .cra_module = THIS_MODULE, - .cra_init = nx_crypto_ctx_aes_gcm_init, - .cra_exit = nx_crypto_ctx_exit, - .cra_aead = { - .ivsize = 8, - .maxauthsize = AES_BLOCK_SIZE, - .geniv = "seqiv", - .setkey = gcm4106_aes_nx_set_key, - .setauthsize = gcm4106_aes_nx_setauthsize, - .encrypt = gcm4106_aes_nx_encrypt, - .decrypt = gcm4106_aes_nx_decrypt, - } +struct aead_alg nx_gcm4106_aes_alg = { + .base = { + .cra_name = "rfc4106(gcm(aes))", + .cra_driver_name = "rfc4106-gcm-aes-nx", + .cra_priority = 300, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct nx_crypto_ctx), + .cra_module = THIS_MODULE, + }, + .init = nx_crypto_ctx_aes_gcm_init, + .exit = nx_crypto_ctx_aead_exit, + .ivsize = 8, + .maxauthsize = AES_BLOCK_SIZE, + .setkey = gcm4106_aes_nx_set_key, + .setauthsize = gcm4106_aes_nx_setauthsize, + .encrypt = gcm4106_aes_nx_encrypt, + .decrypt = gcm4106_aes_nx_decrypt, }; diff --git a/kernel/drivers/crypto/nx/nx.c b/kernel/drivers/crypto/nx/nx.c index 737d33dc5..0794f1cc0 100644 --- a/kernel/drivers/crypto/nx/nx.c +++ b/kernel/drivers/crypto/nx/nx.c @@ -19,8 +19,8 @@ * Author: Kent Yoder */ +#include #include -#include #include #include #include @@ -29,10 +29,10 @@ #include #include #include -#include #include #include #include +#include #include #include @@ -399,6 +399,13 @@ static void nx_of_update_msc(struct device *dev, goto next_loop; } + if (!trip->sglen || trip->databytelen < NX_PAGE_SIZE) { + dev_warn(dev, "bogus sglen/databytelen: " + "%u/%u (ignored)\n", trip->sglen, + trip->databytelen); + goto next_loop; + } + switch (trip->keybitlen) { case 128: case 160: @@ -491,6 +498,72 @@ static void nx_of_init(struct device *dev, struct nx_of *props) nx_of_update_msc(dev, p, props); } +static bool nx_check_prop(struct device *dev, u32 fc, u32 mode, int slot) +{ + struct alg_props *props = &nx_driver.of.ap[fc][mode][slot]; + + if (!props->sglen || props->databytelen < NX_PAGE_SIZE) { + if (dev) + dev_warn(dev, "bogus sglen/databytelen for %u/%u/%u: " + "%u/%u (ignored)\n", fc, mode, slot, + props->sglen, props->databytelen); + return false; + } + + return true; +} + +static bool nx_check_props(struct device *dev, u32 fc, u32 mode) +{ + int i; + + for (i = 0; i < 3; i++) + if (!nx_check_prop(dev, fc, mode, i)) + return false; + + return true; +} + +static int nx_register_alg(struct crypto_alg *alg, u32 fc, u32 mode) +{ + return nx_check_props(&nx_driver.viodev->dev, fc, mode) ? + crypto_register_alg(alg) : 0; +} + +static int nx_register_aead(struct aead_alg *alg, u32 fc, u32 mode) +{ + return nx_check_props(&nx_driver.viodev->dev, fc, mode) ? + crypto_register_aead(alg) : 0; +} + +static int nx_register_shash(struct shash_alg *alg, u32 fc, u32 mode, int slot) +{ + return (slot >= 0 ? nx_check_prop(&nx_driver.viodev->dev, + fc, mode, slot) : + nx_check_props(&nx_driver.viodev->dev, fc, mode)) ? + crypto_register_shash(alg) : 0; +} + +static void nx_unregister_alg(struct crypto_alg *alg, u32 fc, u32 mode) +{ + if (nx_check_props(NULL, fc, mode)) + crypto_unregister_alg(alg); +} + +static void nx_unregister_aead(struct aead_alg *alg, u32 fc, u32 mode) +{ + if (nx_check_props(NULL, fc, mode)) + crypto_unregister_aead(alg); +} + +static void nx_unregister_shash(struct shash_alg *alg, u32 fc, u32 mode, + int slot) +{ + if (slot >= 0 ? nx_check_prop(NULL, fc, mode, slot) : + nx_check_props(NULL, fc, mode)) + crypto_unregister_shash(alg); +} + /** * nx_register_algs - register algorithms with the crypto API * @@ -515,72 +588,71 @@ static int nx_register_algs(void) nx_driver.of.status = NX_OKAY; - rc = crypto_register_alg(&nx_ecb_aes_alg); + rc = nx_register_alg(&nx_ecb_aes_alg, NX_FC_AES, NX_MODE_AES_ECB); if (rc) goto out; - rc = crypto_register_alg(&nx_cbc_aes_alg); + rc = nx_register_alg(&nx_cbc_aes_alg, NX_FC_AES, NX_MODE_AES_CBC); if (rc) goto out_unreg_ecb; - rc = crypto_register_alg(&nx_ctr_aes_alg); + rc = nx_register_alg(&nx_ctr3686_aes_alg, NX_FC_AES, NX_MODE_AES_CTR); if (rc) goto out_unreg_cbc; - rc = crypto_register_alg(&nx_ctr3686_aes_alg); - if (rc) - goto out_unreg_ctr; - - rc = crypto_register_alg(&nx_gcm_aes_alg); + rc = nx_register_aead(&nx_gcm_aes_alg, NX_FC_AES, NX_MODE_AES_GCM); if (rc) goto out_unreg_ctr3686; - rc = crypto_register_alg(&nx_gcm4106_aes_alg); + rc = nx_register_aead(&nx_gcm4106_aes_alg, NX_FC_AES, NX_MODE_AES_GCM); if (rc) goto out_unreg_gcm; - rc = crypto_register_alg(&nx_ccm_aes_alg); + rc = nx_register_aead(&nx_ccm_aes_alg, NX_FC_AES, NX_MODE_AES_CCM); if (rc) goto out_unreg_gcm4106; - rc = crypto_register_alg(&nx_ccm4309_aes_alg); + rc = nx_register_aead(&nx_ccm4309_aes_alg, NX_FC_AES, NX_MODE_AES_CCM); if (rc) goto out_unreg_ccm; - rc = crypto_register_shash(&nx_shash_sha256_alg); + rc = nx_register_shash(&nx_shash_sha256_alg, NX_FC_SHA, NX_MODE_SHA, + NX_PROPS_SHA256); if (rc) goto out_unreg_ccm4309; - rc = crypto_register_shash(&nx_shash_sha512_alg); + rc = nx_register_shash(&nx_shash_sha512_alg, NX_FC_SHA, NX_MODE_SHA, + NX_PROPS_SHA512); if (rc) goto out_unreg_s256; - rc = crypto_register_shash(&nx_shash_aes_xcbc_alg); + rc = nx_register_shash(&nx_shash_aes_xcbc_alg, + NX_FC_AES, NX_MODE_AES_XCBC_MAC, -1); if (rc) goto out_unreg_s512; goto out; out_unreg_s512: - crypto_unregister_shash(&nx_shash_sha512_alg); + nx_unregister_shash(&nx_shash_sha512_alg, NX_FC_SHA, NX_MODE_SHA, + NX_PROPS_SHA512); out_unreg_s256: - crypto_unregister_shash(&nx_shash_sha256_alg); + nx_unregister_shash(&nx_shash_sha256_alg, NX_FC_SHA, NX_MODE_SHA, + NX_PROPS_SHA256); out_unreg_ccm4309: - crypto_unregister_alg(&nx_ccm4309_aes_alg); + nx_unregister_aead(&nx_ccm4309_aes_alg, NX_FC_AES, NX_MODE_AES_CCM); out_unreg_ccm: - crypto_unregister_alg(&nx_ccm_aes_alg); + nx_unregister_aead(&nx_ccm_aes_alg, NX_FC_AES, NX_MODE_AES_CCM); out_unreg_gcm4106: - crypto_unregister_alg(&nx_gcm4106_aes_alg); + nx_unregister_aead(&nx_gcm4106_aes_alg, NX_FC_AES, NX_MODE_AES_GCM); out_unreg_gcm: - crypto_unregister_alg(&nx_gcm_aes_alg); + nx_unregister_aead(&nx_gcm_aes_alg, NX_FC_AES, NX_MODE_AES_GCM); out_unreg_ctr3686: - crypto_unregister_alg(&nx_ctr3686_aes_alg); -out_unreg_ctr: - crypto_unregister_alg(&nx_ctr_aes_alg); + nx_unregister_alg(&nx_ctr3686_aes_alg, NX_FC_AES, NX_MODE_AES_CTR); out_unreg_cbc: - crypto_unregister_alg(&nx_cbc_aes_alg); + nx_unregister_alg(&nx_cbc_aes_alg, NX_FC_AES, NX_MODE_AES_CBC); out_unreg_ecb: - crypto_unregister_alg(&nx_ecb_aes_alg); + nx_unregister_alg(&nx_ecb_aes_alg, NX_FC_AES, NX_MODE_AES_ECB); out: return rc; } @@ -633,17 +705,17 @@ static int nx_crypto_ctx_init(struct nx_crypto_ctx *nx_ctx, u32 fc, u32 mode) } /* entry points from the crypto tfm initializers */ -int nx_crypto_ctx_aes_ccm_init(struct crypto_tfm *tfm) +int nx_crypto_ctx_aes_ccm_init(struct crypto_aead *tfm) { - tfm->crt_aead.reqsize = sizeof(struct nx_ccm_rctx); - return nx_crypto_ctx_init(crypto_tfm_ctx(tfm), NX_FC_AES, + crypto_aead_set_reqsize(tfm, sizeof(struct nx_ccm_rctx)); + return nx_crypto_ctx_init(crypto_aead_ctx(tfm), NX_FC_AES, NX_MODE_AES_CCM); } -int nx_crypto_ctx_aes_gcm_init(struct crypto_tfm *tfm) +int nx_crypto_ctx_aes_gcm_init(struct crypto_aead *tfm) { - tfm->crt_aead.reqsize = sizeof(struct nx_gcm_rctx); - return nx_crypto_ctx_init(crypto_tfm_ctx(tfm), NX_FC_AES, + crypto_aead_set_reqsize(tfm, sizeof(struct nx_gcm_rctx)); + return nx_crypto_ctx_init(crypto_aead_ctx(tfm), NX_FC_AES, NX_MODE_AES_GCM); } @@ -695,6 +767,13 @@ void nx_crypto_ctx_exit(struct crypto_tfm *tfm) nx_ctx->out_sg = NULL; } +void nx_crypto_ctx_aead_exit(struct crypto_aead *tfm) +{ + struct nx_crypto_ctx *nx_ctx = crypto_aead_ctx(tfm); + + kzfree(nx_ctx->kmem); +} + static int nx_probe(struct vio_dev *viodev, const struct vio_device_id *id) { dev_dbg(&viodev->dev, "driver probed: %s resource id: 0x%x\n", @@ -721,17 +800,23 @@ static int nx_remove(struct vio_dev *viodev) if (nx_driver.of.status == NX_OKAY) { NX_DEBUGFS_FINI(&nx_driver); - crypto_unregister_alg(&nx_ccm_aes_alg); - crypto_unregister_alg(&nx_ccm4309_aes_alg); - crypto_unregister_alg(&nx_gcm_aes_alg); - crypto_unregister_alg(&nx_gcm4106_aes_alg); - crypto_unregister_alg(&nx_ctr_aes_alg); - crypto_unregister_alg(&nx_ctr3686_aes_alg); - crypto_unregister_alg(&nx_cbc_aes_alg); - crypto_unregister_alg(&nx_ecb_aes_alg); - crypto_unregister_shash(&nx_shash_sha256_alg); - crypto_unregister_shash(&nx_shash_sha512_alg); - crypto_unregister_shash(&nx_shash_aes_xcbc_alg); + nx_unregister_shash(&nx_shash_aes_xcbc_alg, + NX_FC_AES, NX_MODE_AES_XCBC_MAC, -1); + nx_unregister_shash(&nx_shash_sha512_alg, + NX_FC_SHA, NX_MODE_SHA, NX_PROPS_SHA256); + nx_unregister_shash(&nx_shash_sha256_alg, + NX_FC_SHA, NX_MODE_SHA, NX_PROPS_SHA512); + nx_unregister_aead(&nx_ccm4309_aes_alg, + NX_FC_AES, NX_MODE_AES_CCM); + nx_unregister_aead(&nx_ccm_aes_alg, NX_FC_AES, NX_MODE_AES_CCM); + nx_unregister_aead(&nx_gcm4106_aes_alg, + NX_FC_AES, NX_MODE_AES_GCM); + nx_unregister_aead(&nx_gcm_aes_alg, + NX_FC_AES, NX_MODE_AES_GCM); + nx_unregister_alg(&nx_ctr3686_aes_alg, + NX_FC_AES, NX_MODE_AES_CTR); + nx_unregister_alg(&nx_cbc_aes_alg, NX_FC_AES, NX_MODE_AES_CBC); + nx_unregister_alg(&nx_ecb_aes_alg, NX_FC_AES, NX_MODE_AES_ECB); } return 0; diff --git a/kernel/drivers/crypto/nx/nx.h b/kernel/drivers/crypto/nx/nx.h index c3ed83764..9347878d4 100644 --- a/kernel/drivers/crypto/nx/nx.h +++ b/kernel/drivers/crypto/nx/nx.h @@ -149,15 +149,18 @@ struct nx_crypto_ctx { } priv; }; +struct crypto_aead; + /* prototypes */ -int nx_crypto_ctx_aes_ccm_init(struct crypto_tfm *tfm); -int nx_crypto_ctx_aes_gcm_init(struct crypto_tfm *tfm); +int nx_crypto_ctx_aes_ccm_init(struct crypto_aead *tfm); +int nx_crypto_ctx_aes_gcm_init(struct crypto_aead *tfm); int nx_crypto_ctx_aes_xcbc_init(struct crypto_tfm *tfm); int nx_crypto_ctx_aes_ctr_init(struct crypto_tfm *tfm); int nx_crypto_ctx_aes_cbc_init(struct crypto_tfm *tfm); int nx_crypto_ctx_aes_ecb_init(struct crypto_tfm *tfm); int nx_crypto_ctx_sha_init(struct crypto_tfm *tfm); void nx_crypto_ctx_exit(struct crypto_tfm *tfm); +void nx_crypto_ctx_aead_exit(struct crypto_aead *tfm); void nx_ctx_init(struct nx_crypto_ctx *nx_ctx, unsigned int function); int nx_hcall_sync(struct nx_crypto_ctx *ctx, struct vio_pfo_op *op, u32 may_sleep); @@ -184,12 +187,11 @@ void nx_debugfs_fini(struct nx_crypto_driver *); extern struct crypto_alg nx_cbc_aes_alg; extern struct crypto_alg nx_ecb_aes_alg; -extern struct crypto_alg nx_gcm_aes_alg; -extern struct crypto_alg nx_gcm4106_aes_alg; -extern struct crypto_alg nx_ctr_aes_alg; +extern struct aead_alg nx_gcm_aes_alg; +extern struct aead_alg nx_gcm4106_aes_alg; extern struct crypto_alg nx_ctr3686_aes_alg; -extern struct crypto_alg nx_ccm_aes_alg; -extern struct crypto_alg nx_ccm4309_aes_alg; +extern struct aead_alg nx_ccm_aes_alg; +extern struct aead_alg nx_ccm4309_aes_alg; extern struct shash_alg nx_shash_aes_xcbc_alg; extern struct shash_alg nx_shash_sha512_alg; extern struct shash_alg nx_shash_sha256_alg; diff --git a/kernel/drivers/crypto/omap-aes.c b/kernel/drivers/crypto/omap-aes.c index 9a28b7e07..eba23147c 100644 --- a/kernel/drivers/crypto/omap-aes.c +++ b/kernel/drivers/crypto/omap-aes.c @@ -52,29 +52,30 @@ #define AES_REG_IV(dd, x) ((dd)->pdata->iv_ofs + ((x) * 0x04)) #define AES_REG_CTRL(dd) ((dd)->pdata->ctrl_ofs) -#define AES_REG_CTRL_CTR_WIDTH_MASK (3 << 7) -#define AES_REG_CTRL_CTR_WIDTH_32 (0 << 7) -#define AES_REG_CTRL_CTR_WIDTH_64 (1 << 7) -#define AES_REG_CTRL_CTR_WIDTH_96 (2 << 7) -#define AES_REG_CTRL_CTR_WIDTH_128 (3 << 7) -#define AES_REG_CTRL_CTR (1 << 6) -#define AES_REG_CTRL_CBC (1 << 5) -#define AES_REG_CTRL_KEY_SIZE (3 << 3) -#define AES_REG_CTRL_DIRECTION (1 << 2) -#define AES_REG_CTRL_INPUT_READY (1 << 1) -#define AES_REG_CTRL_OUTPUT_READY (1 << 0) +#define AES_REG_CTRL_CTR_WIDTH_MASK GENMASK(8, 7) +#define AES_REG_CTRL_CTR_WIDTH_32 0 +#define AES_REG_CTRL_CTR_WIDTH_64 BIT(7) +#define AES_REG_CTRL_CTR_WIDTH_96 BIT(8) +#define AES_REG_CTRL_CTR_WIDTH_128 GENMASK(8, 7) +#define AES_REG_CTRL_CTR BIT(6) +#define AES_REG_CTRL_CBC BIT(5) +#define AES_REG_CTRL_KEY_SIZE GENMASK(4, 3) +#define AES_REG_CTRL_DIRECTION BIT(2) +#define AES_REG_CTRL_INPUT_READY BIT(1) +#define AES_REG_CTRL_OUTPUT_READY BIT(0) +#define AES_REG_CTRL_MASK GENMASK(24, 2) #define AES_REG_DATA_N(dd, x) ((dd)->pdata->data_ofs + ((x) * 0x04)) #define AES_REG_REV(dd) ((dd)->pdata->rev_ofs) #define AES_REG_MASK(dd) ((dd)->pdata->mask_ofs) -#define AES_REG_MASK_SIDLE (1 << 6) -#define AES_REG_MASK_START (1 << 5) -#define AES_REG_MASK_DMA_OUT_EN (1 << 3) -#define AES_REG_MASK_DMA_IN_EN (1 << 2) -#define AES_REG_MASK_SOFTRESET (1 << 1) -#define AES_REG_AUTOIDLE (1 << 0) +#define AES_REG_MASK_SIDLE BIT(6) +#define AES_REG_MASK_START BIT(5) +#define AES_REG_MASK_DMA_OUT_EN BIT(3) +#define AES_REG_MASK_DMA_IN_EN BIT(2) +#define AES_REG_MASK_SOFTRESET BIT(1) +#define AES_REG_AUTOIDLE BIT(0) #define AES_REG_LENGTH_N(x) (0x54 + ((x) * 0x04)) @@ -254,7 +255,7 @@ static int omap_aes_write_ctrl(struct omap_aes_dev *dd) { unsigned int key32; int i, err; - u32 val, mask = 0; + u32 val; err = omap_aes_hw_init(dd); if (err) @@ -274,17 +275,13 @@ static int omap_aes_write_ctrl(struct omap_aes_dev *dd) val = FLD_VAL(((dd->ctx->keylen >> 3) - 1), 4, 3); if (dd->flags & FLAGS_CBC) val |= AES_REG_CTRL_CBC; - if (dd->flags & FLAGS_CTR) { + if (dd->flags & FLAGS_CTR) val |= AES_REG_CTRL_CTR | AES_REG_CTRL_CTR_WIDTH_128; - mask = AES_REG_CTRL_CTR | AES_REG_CTRL_CTR_WIDTH_MASK; - } + if (dd->flags & FLAGS_ENCRYPT) val |= AES_REG_CTRL_DIRECTION; - mask |= AES_REG_CTRL_CBC | AES_REG_CTRL_DIRECTION | - AES_REG_CTRL_KEY_SIZE; - - omap_aes_write_mask(dd, AES_REG_CTRL(dd), val, mask); + omap_aes_write_mask(dd, AES_REG_CTRL(dd), val, AES_REG_CTRL_MASK); return 0; } @@ -558,6 +555,9 @@ static int omap_aes_check_aligned(struct scatterlist *sg, int total) { int len = 0; + if (!IS_ALIGNED(total, AES_BLOCK_SIZE)) + return -EINVAL; + while (sg) { if (!IS_ALIGNED(sg->offset, 4)) return -1; @@ -577,9 +577,10 @@ static int omap_aes_check_aligned(struct scatterlist *sg, int total) static int omap_aes_copy_sgs(struct omap_aes_dev *dd) { void *buf_in, *buf_out; - int pages; + int pages, total; - pages = get_order(dd->total); + total = ALIGN(dd->total, AES_BLOCK_SIZE); + pages = get_order(total); buf_in = (void *)__get_free_pages(GFP_ATOMIC, pages); buf_out = (void *)__get_free_pages(GFP_ATOMIC, pages); @@ -594,11 +595,11 @@ static int omap_aes_copy_sgs(struct omap_aes_dev *dd) sg_copy_buf(buf_in, dd->in_sg, 0, dd->total, 0); sg_init_table(&dd->in_sgl, 1); - sg_set_buf(&dd->in_sgl, buf_in, dd->total); + sg_set_buf(&dd->in_sgl, buf_in, total); dd->in_sg = &dd->in_sgl; sg_init_table(&dd->out_sgl, 1); - sg_set_buf(&dd->out_sgl, buf_out, dd->total); + sg_set_buf(&dd->out_sgl, buf_out, total); dd->out_sg = &dd->out_sgl; return 0; @@ -611,7 +612,7 @@ static int omap_aes_handle_queue(struct omap_aes_dev *dd, struct omap_aes_ctx *ctx; struct omap_aes_reqctx *rctx; unsigned long flags; - int err, ret = 0; + int err, ret = 0, len; spin_lock_irqsave(&dd->lock, flags); if (req) @@ -650,8 +651,9 @@ static int omap_aes_handle_queue(struct omap_aes_dev *dd, dd->sgs_copied = 0; } - dd->in_sg_len = scatterwalk_bytes_sglen(dd->in_sg, dd->total); - dd->out_sg_len = scatterwalk_bytes_sglen(dd->out_sg, dd->total); + len = ALIGN(dd->total, AES_BLOCK_SIZE); + dd->in_sg_len = scatterwalk_bytes_sglen(dd->in_sg, len); + dd->out_sg_len = scatterwalk_bytes_sglen(dd->out_sg, len); BUG_ON(dd->in_sg_len < 0 || dd->out_sg_len < 0); rctx = ablkcipher_request_ctx(req); @@ -678,7 +680,7 @@ static void omap_aes_done_task(unsigned long data) { struct omap_aes_dev *dd = (struct omap_aes_dev *)data; void *buf_in, *buf_out; - int pages; + int pages, len; pr_debug("enter done_task\n"); @@ -697,7 +699,8 @@ static void omap_aes_done_task(unsigned long data) sg_copy_buf(buf_out, dd->orig_out, 0, dd->total_save, 1); - pages = get_order(dd->total_save); + len = ALIGN(dd->total_save, AES_BLOCK_SIZE); + pages = get_order(len); free_pages((unsigned long)buf_in, pages); free_pages((unsigned long)buf_out, pages); } @@ -726,11 +729,6 @@ static int omap_aes_crypt(struct ablkcipher_request *req, unsigned long mode) !!(mode & FLAGS_ENCRYPT), !!(mode & FLAGS_CBC)); - if (!IS_ALIGNED(req->nbytes, AES_BLOCK_SIZE)) { - pr_err("request size is not exact amount of AES blocks\n"); - return -EINVAL; - } - dd = omap_aes_find_dev(ctx); if (!dd) return -ENODEV; @@ -833,7 +831,7 @@ static struct crypto_alg algs_ecb_cbc[] = { { .cra_name = "ecb(aes)", .cra_driver_name = "ecb-aes-omap", - .cra_priority = 100, + .cra_priority = 300, .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, @@ -855,7 +853,7 @@ static struct crypto_alg algs_ecb_cbc[] = { { .cra_name = "cbc(aes)", .cra_driver_name = "cbc-aes-omap", - .cra_priority = 100, + .cra_priority = 300, .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, @@ -881,7 +879,7 @@ static struct crypto_alg algs_ctr[] = { { .cra_name = "ctr(aes)", .cra_driver_name = "ctr-aes-omap", - .cra_priority = 100, + .cra_priority = 300, .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC, @@ -1046,9 +1044,7 @@ static irqreturn_t omap_aes_irq(int irq, void *dev_id) } } - dd->total -= AES_BLOCK_SIZE; - - BUG_ON(dd->total < 0); + dd->total -= min_t(size_t, AES_BLOCK_SIZE, dd->total); /* Clear IRQ status */ status &= ~AES_REG_IRQ_DATA_OUT; diff --git a/kernel/drivers/crypto/omap-sham.c b/kernel/drivers/crypto/omap-sham.c index 4d63e0d4d..48adb2a09 100644 --- a/kernel/drivers/crypto/omap-sham.c +++ b/kernel/drivers/crypto/omap-sham.c @@ -362,7 +362,13 @@ static void omap_sham_copy_ready_hash(struct ahash_request *req) static int omap_sham_hw_init(struct omap_sham_dev *dd) { - pm_runtime_get_sync(dd->dev); + int err; + + err = pm_runtime_get_sync(dd->dev); + if (err < 0) { + dev_err(dd->dev, "failed to get sync: %d\n", err); + return err; + } if (!test_bit(FLAGS_INIT, &dd->flags)) { set_bit(FLAGS_INIT, &dd->flags); @@ -582,7 +588,7 @@ static int omap_sham_xmit_dma(struct omap_sham_dev *dd, dma_addr_t dma_addr, * the dmaengine may try to DMA the incorrect amount of data. */ sg_init_table(&ctx->sgl, 1); - ctx->sgl.page_link = ctx->sg->page_link; + sg_assign_page(&ctx->sgl, sg_page(ctx->sg)); ctx->sgl.offset = ctx->sg->offset; sg_dma_len(&ctx->sgl) = len32; sg_dma_address(&ctx->sgl) = sg_dma_address(ctx->sg); @@ -1792,6 +1798,10 @@ static const struct of_device_id omap_sham_of_match[] = { .compatible = "ti,omap2-sham", .data = &omap_sham_pdata_omap2, }, + { + .compatible = "ti,omap3-sham", + .data = &omap_sham_pdata_omap2, + }, { .compatible = "ti,omap4-sham", .data = &omap_sham_pdata_omap4, @@ -1947,7 +1957,13 @@ static int omap_sham_probe(struct platform_device *pdev) pm_runtime_enable(dev); pm_runtime_irq_safe(dev); - pm_runtime_get_sync(dev); + + err = pm_runtime_get_sync(dev); + if (err < 0) { + dev_err(dev, "failed to get sync: %d\n", err); + goto err_pm; + } + rev = omap_sham_read(dd, SHA_REG_REV(dd)); pm_runtime_put_sync(&pdev->dev); @@ -1977,6 +1993,7 @@ err_algs: for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--) crypto_unregister_ahash( &dd->pdata->algs_info[i].algs_list[j]); +err_pm: pm_runtime_disable(dev); if (dd->dma_lch) dma_release_channel(dd->dma_lch); @@ -2019,7 +2036,11 @@ static int omap_sham_suspend(struct device *dev) static int omap_sham_resume(struct device *dev) { - pm_runtime_get_sync(dev); + int err = pm_runtime_get_sync(dev); + if (err < 0) { + dev_err(dev, "failed to get sync: %d\n", err); + return err; + } return 0; } #endif diff --git a/kernel/drivers/crypto/padlock-aes.c b/kernel/drivers/crypto/padlock-aes.c index c178ed8c3..da2d6777b 100644 --- a/kernel/drivers/crypto/padlock-aes.c +++ b/kernel/drivers/crypto/padlock-aes.c @@ -22,7 +22,7 @@ #include #include #include -#include +#include /* * Number of data blocks actually fetched for each xcrypt insn. diff --git a/kernel/drivers/crypto/padlock-sha.c b/kernel/drivers/crypto/padlock-sha.c index 95f7d27ce..4e154c9b9 100644 --- a/kernel/drivers/crypto/padlock-sha.c +++ b/kernel/drivers/crypto/padlock-sha.c @@ -23,7 +23,7 @@ #include #include #include -#include +#include struct padlock_sha_desc { struct shash_desc fallback; diff --git a/kernel/drivers/crypto/picoxcell_crypto.c b/kernel/drivers/crypto/picoxcell_crypto.c index 5da5b98b8..615da961c 100644 --- a/kernel/drivers/crypto/picoxcell_crypto.c +++ b/kernel/drivers/crypto/picoxcell_crypto.c @@ -15,7 +15,7 @@ * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ -#include +#include #include #include #include @@ -40,6 +40,7 @@ #include #include #include +#include #include #include @@ -98,11 +99,16 @@ struct spacc_req { dma_addr_t src_addr, dst_addr; struct spacc_ddt *src_ddt, *dst_ddt; void (*complete)(struct spacc_req *req); +}; - /* AEAD specific bits. */ - u8 *giv; - size_t giv_len; - dma_addr_t giv_pa; +struct spacc_aead { + unsigned long ctrl_default; + unsigned long type; + struct aead_alg alg; + struct spacc_engine *engine; + struct list_head entry; + int key_offs; + int iv_offs; }; struct spacc_engine { @@ -120,6 +126,9 @@ struct spacc_engine { struct spacc_alg *algs; unsigned num_algs; struct list_head registered_algs; + struct spacc_aead *aeads; + unsigned num_aeads; + struct list_head registered_aeads; size_t cipher_pg_sz; size_t hash_pg_sz; const char *name; @@ -173,8 +182,6 @@ struct spacc_aead_ctx { u8 cipher_key_len; u8 hash_key_len; struct crypto_aead *sw_cipher; - size_t auth_size; - u8 salt[AES_BLOCK_SIZE]; }; static int spacc_ablk_submit(struct spacc_req *req); @@ -184,6 +191,11 @@ static inline struct spacc_alg *to_spacc_alg(struct crypto_alg *alg) return alg ? container_of(alg, struct spacc_alg, alg) : NULL; } +static inline struct spacc_aead *to_spacc_aead(struct aead_alg *alg) +{ + return container_of(alg, struct spacc_aead, alg); +} + static inline int spacc_fifo_cmd_full(struct spacc_engine *engine) { u32 fifo_stat = readl(engine->regs + SPA_FIFO_STAT_REG_OFFSET); @@ -261,18 +273,9 @@ static unsigned spacc_load_ctx(struct spacc_generic_ctx *ctx, } /* Count the number of scatterlist entries in a scatterlist. */ -static int sg_count(struct scatterlist *sg_list, int nbytes) +static inline int sg_count(struct scatterlist *sg_list, int nbytes) { - struct scatterlist *sg = sg_list; - int sg_nents = 0; - - while (nbytes > 0) { - ++sg_nents; - nbytes -= sg->length; - sg = sg_next(sg); - } - - return sg_nents; + return sg_nents_for_len(sg_list, nbytes); } static inline void ddt_set(struct spacc_ddt *ddt, dma_addr_t phys, size_t len) @@ -318,112 +321,117 @@ out: return NULL; } -static int spacc_aead_make_ddts(struct spacc_req *req, u8 *giv) +static int spacc_aead_make_ddts(struct aead_request *areq) { - struct aead_request *areq = container_of(req->req, struct aead_request, - base); + struct crypto_aead *aead = crypto_aead_reqtfm(areq); + struct spacc_req *req = aead_request_ctx(areq); struct spacc_engine *engine = req->engine; struct spacc_ddt *src_ddt, *dst_ddt; - unsigned ivsize = crypto_aead_ivsize(crypto_aead_reqtfm(areq)); - unsigned nents = sg_count(areq->src, areq->cryptlen); - dma_addr_t iv_addr; + unsigned total; + unsigned int src_nents, dst_nents; struct scatterlist *cur; - int i, dst_ents, src_ents, assoc_ents; - u8 *iv = giv ? giv : areq->iv; + int i, dst_ents, src_ents; + + total = areq->assoclen + areq->cryptlen; + if (req->is_encrypt) + total += crypto_aead_authsize(aead); + + src_nents = sg_count(areq->src, total); + if (src_nents + 1 > MAX_DDT_LEN) + return -E2BIG; + + dst_nents = 0; + if (areq->src != areq->dst) { + dst_nents = sg_count(areq->dst, total); + if (src_nents + 1 > MAX_DDT_LEN) + return -E2BIG; + } src_ddt = dma_pool_alloc(engine->req_pool, GFP_ATOMIC, &req->src_addr); if (!src_ddt) - return -ENOMEM; + goto err; dst_ddt = dma_pool_alloc(engine->req_pool, GFP_ATOMIC, &req->dst_addr); - if (!dst_ddt) { - dma_pool_free(engine->req_pool, src_ddt, req->src_addr); - return -ENOMEM; - } + if (!dst_ddt) + goto err_free_src; req->src_ddt = src_ddt; req->dst_ddt = dst_ddt; - assoc_ents = dma_map_sg(engine->dev, areq->assoc, - sg_count(areq->assoc, areq->assoclen), DMA_TO_DEVICE); - if (areq->src != areq->dst) { - src_ents = dma_map_sg(engine->dev, areq->src, nents, + if (dst_nents) { + src_ents = dma_map_sg(engine->dev, areq->src, src_nents, DMA_TO_DEVICE); - dst_ents = dma_map_sg(engine->dev, areq->dst, nents, + if (!src_ents) + goto err_free_dst; + + dst_ents = dma_map_sg(engine->dev, areq->dst, dst_nents, DMA_FROM_DEVICE); + + if (!dst_ents) { + dma_unmap_sg(engine->dev, areq->src, src_nents, + DMA_TO_DEVICE); + goto err_free_dst; + } } else { - src_ents = dma_map_sg(engine->dev, areq->src, nents, + src_ents = dma_map_sg(engine->dev, areq->src, src_nents, DMA_BIDIRECTIONAL); - dst_ents = 0; + if (!src_ents) + goto err_free_dst; + dst_ents = src_ents; } - /* - * Map the IV/GIV. For the GIV it needs to be bidirectional as it is - * formed by the crypto block and sent as the ESP IV for IPSEC. - */ - iv_addr = dma_map_single(engine->dev, iv, ivsize, - giv ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); - req->giv_pa = iv_addr; - - /* - * Map the associated data. For decryption we don't copy the - * associated data. - */ - for_each_sg(areq->assoc, cur, assoc_ents, i) { - ddt_set(src_ddt++, sg_dma_address(cur), sg_dma_len(cur)); - if (req->is_encrypt) - ddt_set(dst_ddt++, sg_dma_address(cur), - sg_dma_len(cur)); - } - ddt_set(src_ddt++, iv_addr, ivsize); - - if (giv || req->is_encrypt) - ddt_set(dst_ddt++, iv_addr, ivsize); - /* * Now map in the payload for the source and destination and terminate * with the NULL pointers. */ - for_each_sg(areq->src, cur, src_ents, i) { + for_each_sg(areq->src, cur, src_ents, i) ddt_set(src_ddt++, sg_dma_address(cur), sg_dma_len(cur)); - if (areq->src == areq->dst) - ddt_set(dst_ddt++, sg_dma_address(cur), - sg_dma_len(cur)); - } - for_each_sg(areq->dst, cur, dst_ents, i) - ddt_set(dst_ddt++, sg_dma_address(cur), - sg_dma_len(cur)); + /* For decryption we need to skip the associated data. */ + total = req->is_encrypt ? 0 : areq->assoclen; + for_each_sg(areq->dst, cur, dst_ents, i) { + unsigned len = sg_dma_len(cur); + + if (len <= total) { + total -= len; + continue; + } + + ddt_set(dst_ddt++, sg_dma_address(cur) + total, len - total); + } ddt_set(src_ddt, 0, 0); ddt_set(dst_ddt, 0, 0); return 0; + +err_free_dst: + dma_pool_free(engine->req_pool, dst_ddt, req->dst_addr); +err_free_src: + dma_pool_free(engine->req_pool, src_ddt, req->src_addr); +err: + return -ENOMEM; } static void spacc_aead_free_ddts(struct spacc_req *req) { struct aead_request *areq = container_of(req->req, struct aead_request, base); - struct spacc_alg *alg = to_spacc_alg(req->req->tfm->__crt_alg); - struct spacc_ablk_ctx *aead_ctx = crypto_tfm_ctx(req->req->tfm); + struct crypto_aead *aead = crypto_aead_reqtfm(areq); + unsigned total = areq->assoclen + areq->cryptlen + + (req->is_encrypt ? crypto_aead_authsize(aead) : 0); + struct spacc_aead_ctx *aead_ctx = crypto_aead_ctx(aead); struct spacc_engine *engine = aead_ctx->generic.engine; - unsigned ivsize = alg->alg.cra_aead.ivsize; - unsigned nents = sg_count(areq->src, areq->cryptlen); + unsigned nents = sg_count(areq->src, total); if (areq->src != areq->dst) { dma_unmap_sg(engine->dev, areq->src, nents, DMA_TO_DEVICE); dma_unmap_sg(engine->dev, areq->dst, - sg_count(areq->dst, areq->cryptlen), + sg_count(areq->dst, total), DMA_FROM_DEVICE); } else dma_unmap_sg(engine->dev, areq->src, nents, DMA_BIDIRECTIONAL); - dma_unmap_sg(engine->dev, areq->assoc, - sg_count(areq->assoc, areq->assoclen), DMA_TO_DEVICE); - - dma_unmap_single(engine->dev, req->giv_pa, ivsize, DMA_BIDIRECTIONAL); - dma_pool_free(engine->req_pool, req->src_ddt, req->src_addr); dma_pool_free(engine->req_pool, req->dst_ddt, req->dst_addr); } @@ -438,65 +446,22 @@ static void spacc_free_ddt(struct spacc_req *req, struct spacc_ddt *ddt, dma_pool_free(req->engine->req_pool, ddt, ddt_addr); } -/* - * Set key for a DES operation in an AEAD cipher. This also performs weak key - * checking if required. - */ -static int spacc_aead_des_setkey(struct crypto_aead *aead, const u8 *key, - unsigned int len) -{ - struct crypto_tfm *tfm = crypto_aead_tfm(aead); - struct spacc_aead_ctx *ctx = crypto_tfm_ctx(tfm); - u32 tmp[DES_EXPKEY_WORDS]; - - if (unlikely(!des_ekey(tmp, key)) && - (crypto_aead_get_flags(aead)) & CRYPTO_TFM_REQ_WEAK_KEY) { - tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY; - return -EINVAL; - } - - memcpy(ctx->cipher_key, key, len); - ctx->cipher_key_len = len; - - return 0; -} - -/* Set the key for the AES block cipher component of the AEAD transform. */ -static int spacc_aead_aes_setkey(struct crypto_aead *aead, const u8 *key, - unsigned int len) -{ - struct crypto_tfm *tfm = crypto_aead_tfm(aead); - struct spacc_aead_ctx *ctx = crypto_tfm_ctx(tfm); - - /* - * IPSec engine only supports 128 and 256 bit AES keys. If we get a - * request for any other size (192 bits) then we need to do a software - * fallback. - */ - if (len != AES_KEYSIZE_128 && len != AES_KEYSIZE_256) { - /* - * Set the fallback transform to use the same request flags as - * the hardware transform. - */ - ctx->sw_cipher->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK; - ctx->sw_cipher->base.crt_flags |= - tfm->crt_flags & CRYPTO_TFM_REQ_MASK; - return crypto_aead_setkey(ctx->sw_cipher, key, len); - } - - memcpy(ctx->cipher_key, key, len); - ctx->cipher_key_len = len; - - return 0; -} - static int spacc_aead_setkey(struct crypto_aead *tfm, const u8 *key, unsigned int keylen) { struct spacc_aead_ctx *ctx = crypto_aead_ctx(tfm); - struct spacc_alg *alg = to_spacc_alg(tfm->base.__crt_alg); struct crypto_authenc_keys keys; - int err = -EINVAL; + int err; + + crypto_aead_clear_flags(ctx->sw_cipher, CRYPTO_TFM_REQ_MASK); + crypto_aead_set_flags(ctx->sw_cipher, crypto_aead_get_flags(tfm) & + CRYPTO_TFM_REQ_MASK); + err = crypto_aead_setkey(ctx->sw_cipher, key, keylen); + crypto_aead_clear_flags(tfm, CRYPTO_TFM_RES_MASK); + crypto_aead_set_flags(tfm, crypto_aead_get_flags(ctx->sw_cipher) & + CRYPTO_TFM_RES_MASK); + if (err) + return err; if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) goto badkey; @@ -507,14 +472,8 @@ static int spacc_aead_setkey(struct crypto_aead *tfm, const u8 *key, if (keys.authkeylen > sizeof(ctx->hash_ctx)) goto badkey; - if ((alg->ctrl_default & SPACC_CRYPTO_ALG_MASK) == - SPA_CTRL_CIPH_ALG_AES) - err = spacc_aead_aes_setkey(tfm, keys.enckey, keys.enckeylen); - else - err = spacc_aead_des_setkey(tfm, keys.enckey, keys.enckeylen); - - if (err) - goto badkey; + memcpy(ctx->cipher_key, keys.enckey, keys.enckeylen); + ctx->cipher_key_len = keys.enckeylen; memcpy(ctx->hash_ctx, keys.authkey, keys.authkeylen); ctx->hash_key_len = keys.authkeylen; @@ -531,9 +490,7 @@ static int spacc_aead_setauthsize(struct crypto_aead *tfm, { struct spacc_aead_ctx *ctx = crypto_tfm_ctx(crypto_aead_tfm(tfm)); - ctx->auth_size = authsize; - - return 0; + return crypto_aead_setauthsize(ctx->sw_cipher, authsize); } /* @@ -541,15 +498,13 @@ static int spacc_aead_setauthsize(struct crypto_aead *tfm, * be completed in hardware because the hardware may not support certain key * sizes. In these cases we need to complete the request in software. */ -static int spacc_aead_need_fallback(struct spacc_req *req) +static int spacc_aead_need_fallback(struct aead_request *aead_req) { - struct aead_request *aead_req; - struct crypto_tfm *tfm = req->req->tfm; - struct crypto_alg *alg = req->req->tfm->__crt_alg; - struct spacc_alg *spacc_alg = to_spacc_alg(alg); - struct spacc_aead_ctx *ctx = crypto_tfm_ctx(tfm); + struct crypto_aead *aead = crypto_aead_reqtfm(aead_req); + struct aead_alg *alg = crypto_aead_alg(aead); + struct spacc_aead *spacc_alg = to_spacc_aead(alg); + struct spacc_aead_ctx *ctx = crypto_aead_ctx(aead); - aead_req = container_of(req->req, struct aead_request, base); /* * If we have a non-supported key-length, then we need to do a * software fallback. @@ -568,22 +523,17 @@ static int spacc_aead_do_fallback(struct aead_request *req, unsigned alg_type, { struct crypto_tfm *old_tfm = crypto_aead_tfm(crypto_aead_reqtfm(req)); struct spacc_aead_ctx *ctx = crypto_tfm_ctx(old_tfm); - int err; + struct aead_request *subreq = aead_request_ctx(req); - if (ctx->sw_cipher) { - /* - * Change the request to use the software fallback transform, - * and once the ciphering has completed, put the old transform - * back into the request. - */ - aead_request_set_tfm(req, ctx->sw_cipher); - err = is_encrypt ? crypto_aead_encrypt(req) : - crypto_aead_decrypt(req); - aead_request_set_tfm(req, __crypto_aead_cast(old_tfm)); - } else - err = -EINVAL; + aead_request_set_tfm(subreq, ctx->sw_cipher); + aead_request_set_callback(subreq, req->base.flags, + req->base.complete, req->base.data); + aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, + req->iv); + aead_request_set_ad(subreq, req->assoclen); - return err; + return is_encrypt ? crypto_aead_encrypt(subreq) : + crypto_aead_decrypt(subreq); } static void spacc_aead_complete(struct spacc_req *req) @@ -594,18 +544,19 @@ static void spacc_aead_complete(struct spacc_req *req) static int spacc_aead_submit(struct spacc_req *req) { - struct crypto_tfm *tfm = req->req->tfm; - struct spacc_aead_ctx *ctx = crypto_tfm_ctx(tfm); - struct crypto_alg *alg = req->req->tfm->__crt_alg; - struct spacc_alg *spacc_alg = to_spacc_alg(alg); - struct spacc_engine *engine = ctx->generic.engine; - u32 ctrl, proc_len, assoc_len; struct aead_request *aead_req = container_of(req->req, struct aead_request, base); + struct crypto_aead *aead = crypto_aead_reqtfm(aead_req); + unsigned int authsize = crypto_aead_authsize(aead); + struct spacc_aead_ctx *ctx = crypto_aead_ctx(aead); + struct aead_alg *alg = crypto_aead_alg(aead); + struct spacc_aead *spacc_alg = to_spacc_aead(alg); + struct spacc_engine *engine = ctx->generic.engine; + u32 ctrl, proc_len, assoc_len; req->result = -EINPROGRESS; req->ctx_id = spacc_load_ctx(&ctx->generic, ctx->cipher_key, - ctx->cipher_key_len, aead_req->iv, alg->cra_aead.ivsize, + ctx->cipher_key_len, aead_req->iv, crypto_aead_ivsize(aead), ctx->hash_ctx, ctx->hash_key_len); /* Set the source and destination DDT pointers. */ @@ -616,26 +567,16 @@ static int spacc_aead_submit(struct spacc_req *req) assoc_len = aead_req->assoclen; proc_len = aead_req->cryptlen + assoc_len; - /* - * If we aren't generating an IV, then we need to include the IV in the - * associated data so that it is included in the hash. - */ - if (!req->giv) { - assoc_len += crypto_aead_ivsize(crypto_aead_reqtfm(aead_req)); - proc_len += crypto_aead_ivsize(crypto_aead_reqtfm(aead_req)); - } else - proc_len += req->giv_len; - /* * If we are decrypting, we need to take the length of the ICV out of * the processing length. */ if (!req->is_encrypt) - proc_len -= ctx->auth_size; + proc_len -= authsize; writel(proc_len, engine->regs + SPA_PROC_LEN_REG_OFFSET); writel(assoc_len, engine->regs + SPA_AAD_LEN_REG_OFFSET); - writel(ctx->auth_size, engine->regs + SPA_ICV_LEN_REG_OFFSET); + writel(authsize, engine->regs + SPA_ICV_LEN_REG_OFFSET); writel(0, engine->regs + SPA_ICV_OFFSET_REG_OFFSET); writel(0, engine->regs + SPA_AUX_INFO_REG_OFFSET); @@ -674,32 +615,29 @@ static void spacc_push(struct spacc_engine *engine) /* * Setup an AEAD request for processing. This will configure the engine, load * the context and then start the packet processing. - * - * @giv Pointer to destination address for a generated IV. If the - * request does not need to generate an IV then this should be set to NULL. */ -static int spacc_aead_setup(struct aead_request *req, u8 *giv, +static int spacc_aead_setup(struct aead_request *req, unsigned alg_type, bool is_encrypt) { - struct crypto_alg *alg = req->base.tfm->__crt_alg; - struct spacc_engine *engine = to_spacc_alg(alg)->engine; + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct aead_alg *alg = crypto_aead_alg(aead); + struct spacc_engine *engine = to_spacc_aead(alg)->engine; struct spacc_req *dev_req = aead_request_ctx(req); - int err = -EINPROGRESS; + int err; unsigned long flags; - unsigned ivsize = crypto_aead_ivsize(crypto_aead_reqtfm(req)); - dev_req->giv = giv; - dev_req->giv_len = ivsize; dev_req->req = &req->base; dev_req->is_encrypt = is_encrypt; dev_req->result = -EBUSY; dev_req->engine = engine; dev_req->complete = spacc_aead_complete; - if (unlikely(spacc_aead_need_fallback(dev_req))) + if (unlikely(spacc_aead_need_fallback(req) || + ((err = spacc_aead_make_ddts(req)) == -E2BIG))) return spacc_aead_do_fallback(req, alg_type, is_encrypt); - spacc_aead_make_ddts(dev_req, dev_req->giv); + if (err) + goto out; err = -EINPROGRESS; spin_lock_irqsave(&engine->hw_lock, flags); @@ -728,69 +666,44 @@ out: static int spacc_aead_encrypt(struct aead_request *req) { struct crypto_aead *aead = crypto_aead_reqtfm(req); - struct crypto_tfm *tfm = crypto_aead_tfm(aead); - struct spacc_alg *alg = to_spacc_alg(tfm->__crt_alg); + struct spacc_aead *alg = to_spacc_aead(crypto_aead_alg(aead)); - return spacc_aead_setup(req, NULL, alg->type, 1); -} - -static int spacc_aead_givencrypt(struct aead_givcrypt_request *req) -{ - struct crypto_aead *tfm = aead_givcrypt_reqtfm(req); - struct spacc_aead_ctx *ctx = crypto_aead_ctx(tfm); - size_t ivsize = crypto_aead_ivsize(tfm); - struct spacc_alg *alg = to_spacc_alg(tfm->base.__crt_alg); - unsigned len; - __be64 seq; - - memcpy(req->areq.iv, ctx->salt, ivsize); - len = ivsize; - if (ivsize > sizeof(u64)) { - memset(req->giv, 0, ivsize - sizeof(u64)); - len = sizeof(u64); - } - seq = cpu_to_be64(req->seq); - memcpy(req->giv + ivsize - len, &seq, len); - - return spacc_aead_setup(&req->areq, req->giv, alg->type, 1); + return spacc_aead_setup(req, alg->type, 1); } static int spacc_aead_decrypt(struct aead_request *req) { struct crypto_aead *aead = crypto_aead_reqtfm(req); - struct crypto_tfm *tfm = crypto_aead_tfm(aead); - struct spacc_alg *alg = to_spacc_alg(tfm->__crt_alg); + struct spacc_aead *alg = to_spacc_aead(crypto_aead_alg(aead)); - return spacc_aead_setup(req, NULL, alg->type, 0); + return spacc_aead_setup(req, alg->type, 0); } /* * Initialise a new AEAD context. This is responsible for allocating the * fallback cipher and initialising the context. */ -static int spacc_aead_cra_init(struct crypto_tfm *tfm) +static int spacc_aead_cra_init(struct crypto_aead *tfm) { - struct spacc_aead_ctx *ctx = crypto_tfm_ctx(tfm); - struct crypto_alg *alg = tfm->__crt_alg; - struct spacc_alg *spacc_alg = to_spacc_alg(alg); + struct spacc_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct aead_alg *alg = crypto_aead_alg(tfm); + struct spacc_aead *spacc_alg = to_spacc_aead(alg); struct spacc_engine *engine = spacc_alg->engine; ctx->generic.flags = spacc_alg->type; ctx->generic.engine = engine; - ctx->sw_cipher = crypto_alloc_aead(alg->cra_name, 0, - CRYPTO_ALG_ASYNC | + ctx->sw_cipher = crypto_alloc_aead(alg->base.cra_name, 0, CRYPTO_ALG_NEED_FALLBACK); - if (IS_ERR(ctx->sw_cipher)) { - dev_warn(engine->dev, "failed to allocate fallback for %s\n", - alg->cra_name); - ctx->sw_cipher = NULL; - } + if (IS_ERR(ctx->sw_cipher)) + return PTR_ERR(ctx->sw_cipher); ctx->generic.key_offs = spacc_alg->key_offs; ctx->generic.iv_offs = spacc_alg->iv_offs; - get_random_bytes(ctx->salt, sizeof(ctx->salt)); - - tfm->crt_aead.reqsize = sizeof(struct spacc_req); + crypto_aead_set_reqsize( + tfm, + max(sizeof(struct spacc_req), + sizeof(struct aead_request) + + crypto_aead_reqsize(ctx->sw_cipher))); return 0; } @@ -799,13 +712,11 @@ static int spacc_aead_cra_init(struct crypto_tfm *tfm) * Destructor for an AEAD context. This is called when the transform is freed * and must free the fallback cipher. */ -static void spacc_aead_cra_exit(struct crypto_tfm *tfm) +static void spacc_aead_cra_exit(struct crypto_aead *tfm) { - struct spacc_aead_ctx *ctx = crypto_tfm_ctx(tfm); + struct spacc_aead_ctx *ctx = crypto_aead_ctx(tfm); - if (ctx->sw_cipher) - crypto_free_aead(ctx->sw_cipher); - ctx->sw_cipher = NULL; + crypto_free_aead(ctx->sw_cipher); } /* @@ -1457,180 +1368,188 @@ static struct spacc_alg ipsec_engine_algs[] = { .cra_exit = spacc_ablk_cra_exit, }, }, +}; + +static struct spacc_aead ipsec_engine_aeads[] = { { - .ctrl_default = SPA_CTRL_CIPH_ALG_AES | SPA_CTRL_CIPH_MODE_CBC | - SPA_CTRL_HASH_ALG_SHA | SPA_CTRL_HASH_MODE_HMAC, + .ctrl_default = SPA_CTRL_CIPH_ALG_AES | + SPA_CTRL_CIPH_MODE_CBC | + SPA_CTRL_HASH_ALG_SHA | + SPA_CTRL_HASH_MODE_HMAC, .key_offs = 0, .iv_offs = AES_MAX_KEY_SIZE, .alg = { - .cra_name = "authenc(hmac(sha1),cbc(aes))", - .cra_driver_name = "authenc-hmac-sha1-cbc-aes-picoxcell", - .cra_priority = SPACC_CRYPTO_ALG_PRIORITY, - .cra_flags = CRYPTO_ALG_TYPE_AEAD | - CRYPTO_ALG_ASYNC | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct spacc_aead_ctx), - .cra_type = &crypto_aead_type, - .cra_module = THIS_MODULE, - .cra_aead = { - .setkey = spacc_aead_setkey, - .setauthsize = spacc_aead_setauthsize, - .encrypt = spacc_aead_encrypt, - .decrypt = spacc_aead_decrypt, - .givencrypt = spacc_aead_givencrypt, - .ivsize = AES_BLOCK_SIZE, - .maxauthsize = SHA1_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha1),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha1-" + "cbc-aes-picoxcell", + .cra_priority = SPACC_CRYPTO_ALG_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct spacc_aead_ctx), + .cra_module = THIS_MODULE, }, - .cra_init = spacc_aead_cra_init, - .cra_exit = spacc_aead_cra_exit, + .setkey = spacc_aead_setkey, + .setauthsize = spacc_aead_setauthsize, + .encrypt = spacc_aead_encrypt, + .decrypt = spacc_aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + .init = spacc_aead_cra_init, + .exit = spacc_aead_cra_exit, }, }, { - .ctrl_default = SPA_CTRL_CIPH_ALG_AES | SPA_CTRL_CIPH_MODE_CBC | + .ctrl_default = SPA_CTRL_CIPH_ALG_AES | + SPA_CTRL_CIPH_MODE_CBC | SPA_CTRL_HASH_ALG_SHA256 | SPA_CTRL_HASH_MODE_HMAC, .key_offs = 0, .iv_offs = AES_MAX_KEY_SIZE, .alg = { - .cra_name = "authenc(hmac(sha256),cbc(aes))", - .cra_driver_name = "authenc-hmac-sha256-cbc-aes-picoxcell", - .cra_priority = SPACC_CRYPTO_ALG_PRIORITY, - .cra_flags = CRYPTO_ALG_TYPE_AEAD | - CRYPTO_ALG_ASYNC | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct spacc_aead_ctx), - .cra_type = &crypto_aead_type, - .cra_module = THIS_MODULE, - .cra_aead = { - .setkey = spacc_aead_setkey, - .setauthsize = spacc_aead_setauthsize, - .encrypt = spacc_aead_encrypt, - .decrypt = spacc_aead_decrypt, - .givencrypt = spacc_aead_givencrypt, - .ivsize = AES_BLOCK_SIZE, - .maxauthsize = SHA256_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha256),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha256-" + "cbc-aes-picoxcell", + .cra_priority = SPACC_CRYPTO_ALG_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct spacc_aead_ctx), + .cra_module = THIS_MODULE, }, - .cra_init = spacc_aead_cra_init, - .cra_exit = spacc_aead_cra_exit, + .setkey = spacc_aead_setkey, + .setauthsize = spacc_aead_setauthsize, + .encrypt = spacc_aead_encrypt, + .decrypt = spacc_aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + .init = spacc_aead_cra_init, + .exit = spacc_aead_cra_exit, }, }, { .key_offs = 0, .iv_offs = AES_MAX_KEY_SIZE, - .ctrl_default = SPA_CTRL_CIPH_ALG_AES | SPA_CTRL_CIPH_MODE_CBC | - SPA_CTRL_HASH_ALG_MD5 | SPA_CTRL_HASH_MODE_HMAC, + .ctrl_default = SPA_CTRL_CIPH_ALG_AES | + SPA_CTRL_CIPH_MODE_CBC | + SPA_CTRL_HASH_ALG_MD5 | + SPA_CTRL_HASH_MODE_HMAC, .alg = { - .cra_name = "authenc(hmac(md5),cbc(aes))", - .cra_driver_name = "authenc-hmac-md5-cbc-aes-picoxcell", - .cra_priority = SPACC_CRYPTO_ALG_PRIORITY, - .cra_flags = CRYPTO_ALG_TYPE_AEAD | - CRYPTO_ALG_ASYNC | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct spacc_aead_ctx), - .cra_type = &crypto_aead_type, - .cra_module = THIS_MODULE, - .cra_aead = { - .setkey = spacc_aead_setkey, - .setauthsize = spacc_aead_setauthsize, - .encrypt = spacc_aead_encrypt, - .decrypt = spacc_aead_decrypt, - .givencrypt = spacc_aead_givencrypt, - .ivsize = AES_BLOCK_SIZE, - .maxauthsize = MD5_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(md5),cbc(aes))", + .cra_driver_name = "authenc-hmac-md5-" + "cbc-aes-picoxcell", + .cra_priority = SPACC_CRYPTO_ALG_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct spacc_aead_ctx), + .cra_module = THIS_MODULE, }, - .cra_init = spacc_aead_cra_init, - .cra_exit = spacc_aead_cra_exit, + .setkey = spacc_aead_setkey, + .setauthsize = spacc_aead_setauthsize, + .encrypt = spacc_aead_encrypt, + .decrypt = spacc_aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + .init = spacc_aead_cra_init, + .exit = spacc_aead_cra_exit, }, }, { .key_offs = DES_BLOCK_SIZE, .iv_offs = 0, - .ctrl_default = SPA_CTRL_CIPH_ALG_DES | SPA_CTRL_CIPH_MODE_CBC | - SPA_CTRL_HASH_ALG_SHA | SPA_CTRL_HASH_MODE_HMAC, + .ctrl_default = SPA_CTRL_CIPH_ALG_DES | + SPA_CTRL_CIPH_MODE_CBC | + SPA_CTRL_HASH_ALG_SHA | + SPA_CTRL_HASH_MODE_HMAC, .alg = { - .cra_name = "authenc(hmac(sha1),cbc(des3_ede))", - .cra_driver_name = "authenc-hmac-sha1-cbc-3des-picoxcell", - .cra_priority = SPACC_CRYPTO_ALG_PRIORITY, - .cra_flags = CRYPTO_ALG_TYPE_AEAD | - CRYPTO_ALG_ASYNC | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = DES3_EDE_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct spacc_aead_ctx), - .cra_type = &crypto_aead_type, - .cra_module = THIS_MODULE, - .cra_aead = { - .setkey = spacc_aead_setkey, - .setauthsize = spacc_aead_setauthsize, - .encrypt = spacc_aead_encrypt, - .decrypt = spacc_aead_decrypt, - .givencrypt = spacc_aead_givencrypt, - .ivsize = DES3_EDE_BLOCK_SIZE, - .maxauthsize = SHA1_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha1),cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha1-" + "cbc-3des-picoxcell", + .cra_priority = SPACC_CRYPTO_ALG_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct spacc_aead_ctx), + .cra_module = THIS_MODULE, }, - .cra_init = spacc_aead_cra_init, - .cra_exit = spacc_aead_cra_exit, + .setkey = spacc_aead_setkey, + .setauthsize = spacc_aead_setauthsize, + .encrypt = spacc_aead_encrypt, + .decrypt = spacc_aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + .init = spacc_aead_cra_init, + .exit = spacc_aead_cra_exit, }, }, { .key_offs = DES_BLOCK_SIZE, .iv_offs = 0, - .ctrl_default = SPA_CTRL_CIPH_ALG_AES | SPA_CTRL_CIPH_MODE_CBC | + .ctrl_default = SPA_CTRL_CIPH_ALG_AES | + SPA_CTRL_CIPH_MODE_CBC | SPA_CTRL_HASH_ALG_SHA256 | SPA_CTRL_HASH_MODE_HMAC, .alg = { - .cra_name = "authenc(hmac(sha256),cbc(des3_ede))", - .cra_driver_name = "authenc-hmac-sha256-cbc-3des-picoxcell", - .cra_priority = SPACC_CRYPTO_ALG_PRIORITY, - .cra_flags = CRYPTO_ALG_TYPE_AEAD | - CRYPTO_ALG_ASYNC | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = DES3_EDE_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct spacc_aead_ctx), - .cra_type = &crypto_aead_type, - .cra_module = THIS_MODULE, - .cra_aead = { - .setkey = spacc_aead_setkey, - .setauthsize = spacc_aead_setauthsize, - .encrypt = spacc_aead_encrypt, - .decrypt = spacc_aead_decrypt, - .givencrypt = spacc_aead_givencrypt, - .ivsize = DES3_EDE_BLOCK_SIZE, - .maxauthsize = SHA256_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(sha256)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha256-" + "cbc-3des-picoxcell", + .cra_priority = SPACC_CRYPTO_ALG_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct spacc_aead_ctx), + .cra_module = THIS_MODULE, }, - .cra_init = spacc_aead_cra_init, - .cra_exit = spacc_aead_cra_exit, + .setkey = spacc_aead_setkey, + .setauthsize = spacc_aead_setauthsize, + .encrypt = spacc_aead_encrypt, + .decrypt = spacc_aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + .init = spacc_aead_cra_init, + .exit = spacc_aead_cra_exit, }, }, { .key_offs = DES_BLOCK_SIZE, .iv_offs = 0, - .ctrl_default = SPA_CTRL_CIPH_ALG_DES | SPA_CTRL_CIPH_MODE_CBC | - SPA_CTRL_HASH_ALG_MD5 | SPA_CTRL_HASH_MODE_HMAC, + .ctrl_default = SPA_CTRL_CIPH_ALG_DES | + SPA_CTRL_CIPH_MODE_CBC | + SPA_CTRL_HASH_ALG_MD5 | + SPA_CTRL_HASH_MODE_HMAC, .alg = { - .cra_name = "authenc(hmac(md5),cbc(des3_ede))", - .cra_driver_name = "authenc-hmac-md5-cbc-3des-picoxcell", - .cra_priority = SPACC_CRYPTO_ALG_PRIORITY, - .cra_flags = CRYPTO_ALG_TYPE_AEAD | - CRYPTO_ALG_ASYNC | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = DES3_EDE_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct spacc_aead_ctx), - .cra_type = &crypto_aead_type, - .cra_module = THIS_MODULE, - .cra_aead = { - .setkey = spacc_aead_setkey, - .setauthsize = spacc_aead_setauthsize, - .encrypt = spacc_aead_encrypt, - .decrypt = spacc_aead_decrypt, - .givencrypt = spacc_aead_givencrypt, - .ivsize = DES3_EDE_BLOCK_SIZE, - .maxauthsize = MD5_DIGEST_SIZE, + .base = { + .cra_name = "authenc(hmac(md5),cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-md5-" + "cbc-3des-picoxcell", + .cra_priority = SPACC_CRYPTO_ALG_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct spacc_aead_ctx), + .cra_module = THIS_MODULE, }, - .cra_init = spacc_aead_cra_init, - .cra_exit = spacc_aead_cra_exit, + .setkey = spacc_aead_setkey, + .setauthsize = spacc_aead_setauthsize, + .encrypt = spacc_aead_encrypt, + .decrypt = spacc_aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + .init = spacc_aead_cra_init, + .exit = spacc_aead_cra_exit, }, }, }; @@ -1672,6 +1591,7 @@ static const struct of_device_id spacc_of_id_table[] = { { .compatible = "picochip,spacc-l2" }, {} }; +MODULE_DEVICE_TABLE(of, spacc_of_id_table); #endif /* CONFIG_OF */ static bool spacc_is_compatible(struct platform_device *pdev, @@ -1706,6 +1626,8 @@ static int spacc_probe(struct platform_device *pdev) engine->fifo_sz = SPACC_CRYPTO_IPSEC_FIFO_SZ; engine->algs = ipsec_engine_algs; engine->num_algs = ARRAY_SIZE(ipsec_engine_algs); + engine->aeads = ipsec_engine_aeads; + engine->num_aeads = ARRAY_SIZE(ipsec_engine_aeads); } else if (spacc_is_compatible(pdev, "picochip,spacc-l2")) { engine->max_ctxs = SPACC_CRYPTO_L2_MAX_CTXS; engine->cipher_pg_sz = SPACC_CRYPTO_L2_CIPHER_PG_SZ; @@ -1754,15 +1676,15 @@ static int spacc_probe(struct platform_device *pdev) return PTR_ERR(engine->clk); } - if (clk_enable(engine->clk)) { - dev_info(&pdev->dev, "unable to enable clk\n"); + if (clk_prepare_enable(engine->clk)) { + dev_info(&pdev->dev, "unable to prepare/enable clk\n"); clk_put(engine->clk); return -EIO; } err = device_create_file(&pdev->dev, &dev_attr_stat_irq_thresh); if (err) { - clk_disable(engine->clk); + clk_disable_unprepare(engine->clk); clk_put(engine->clk); return err; } @@ -1814,23 +1736,46 @@ static int spacc_probe(struct platform_device *pdev) engine->algs[i].alg.cra_name); } + INIT_LIST_HEAD(&engine->registered_aeads); + for (i = 0; i < engine->num_aeads; ++i) { + engine->aeads[i].engine = engine; + err = crypto_register_aead(&engine->aeads[i].alg); + if (!err) { + list_add_tail(&engine->aeads[i].entry, + &engine->registered_aeads); + ret = 0; + } + if (err) + dev_err(engine->dev, "failed to register alg \"%s\"\n", + engine->aeads[i].alg.base.cra_name); + else + dev_dbg(engine->dev, "registered alg \"%s\"\n", + engine->aeads[i].alg.base.cra_name); + } + return ret; } static int spacc_remove(struct platform_device *pdev) { + struct spacc_aead *aead, *an; struct spacc_alg *alg, *next; struct spacc_engine *engine = platform_get_drvdata(pdev); del_timer_sync(&engine->packet_timeout); device_remove_file(&pdev->dev, &dev_attr_stat_irq_thresh); + list_for_each_entry_safe(aead, an, &engine->registered_aeads, entry) { + list_del(&aead->entry); + crypto_unregister_aead(&aead->alg); + } + list_for_each_entry_safe(alg, next, &engine->registered_algs, entry) { list_del(&alg->entry); crypto_unregister_alg(&alg->alg); } - clk_disable(engine->clk); + clk_disable_unprepare(engine->clk); clk_put(engine->clk); return 0; diff --git a/kernel/drivers/crypto/qat/Kconfig b/kernel/drivers/crypto/qat/Kconfig index 49bede2a9..eefccf7b8 100644 --- a/kernel/drivers/crypto/qat/Kconfig +++ b/kernel/drivers/crypto/qat/Kconfig @@ -2,18 +2,18 @@ config CRYPTO_DEV_QAT tristate select CRYPTO_AEAD select CRYPTO_AUTHENC - select CRYPTO_ALGAPI - select CRYPTO_AES - select CRYPTO_CBC + select CRYPTO_BLKCIPHER + select CRYPTO_AKCIPHER + select CRYPTO_HMAC select CRYPTO_SHA1 select CRYPTO_SHA256 select CRYPTO_SHA512 select FW_LOADER + select ASN1 config CRYPTO_DEV_QAT_DH895xCC tristate "Support for Intel(R) DH895xCC" depends on X86 && PCI - default n select CRYPTO_DEV_QAT help Support for Intel(R) DH895xcc with Intel(R) QuickAssist Technology @@ -21,3 +21,16 @@ config CRYPTO_DEV_QAT_DH895xCC To compile this as a module, choose M here: the module will be called qat_dh895xcc. + +config CRYPTO_DEV_QAT_DH895xCCVF + tristate "Support for Intel(R) DH895xCC Virtual Function" + depends on X86 && PCI + select PCI_IOV + select CRYPTO_DEV_QAT + + help + Support for Intel(R) DH895xcc with Intel(R) QuickAssist Technology + Virtual Function for accelerating crypto and compression workloads. + + To compile this as a module, choose M here: the module + will be called qat_dh895xccvf. diff --git a/kernel/drivers/crypto/qat/Makefile b/kernel/drivers/crypto/qat/Makefile index d11481be2..a3ce0b70e 100644 --- a/kernel/drivers/crypto/qat/Makefile +++ b/kernel/drivers/crypto/qat/Makefile @@ -1,2 +1,3 @@ obj-$(CONFIG_CRYPTO_DEV_QAT) += qat_common/ obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCC) += qat_dh895xcc/ +obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCCVF) += qat_dh895xccvf/ diff --git a/kernel/drivers/crypto/qat/qat_common/.gitignore b/kernel/drivers/crypto/qat/qat_common/.gitignore new file mode 100644 index 000000000..ee328374d --- /dev/null +++ b/kernel/drivers/crypto/qat/qat_common/.gitignore @@ -0,0 +1 @@ +*-asn1.[ch] diff --git a/kernel/drivers/crypto/qat/qat_common/Makefile b/kernel/drivers/crypto/qat/qat_common/Makefile index e0424dc38..9e9e196c6 100644 --- a/kernel/drivers/crypto/qat/qat_common/Makefile +++ b/kernel/drivers/crypto/qat/qat_common/Makefile @@ -1,3 +1,11 @@ +$(obj)/qat_rsapubkey-asn1.o: $(obj)/qat_rsapubkey-asn1.c \ + $(obj)/qat_rsapubkey-asn1.h +$(obj)/qat_rsaprivkey-asn1.o: $(obj)/qat_rsaprivkey-asn1.c \ + $(obj)/qat_rsaprivkey-asn1.h + +clean-files += qat_rsapubkey-asn1.c qat_rsapubkey-asn1.h +clean-files += qat_rsaprivkey-asn1.c qat_rsapvivkey-asn1.h + obj-$(CONFIG_CRYPTO_DEV_QAT) += intel_qat.o intel_qat-objs := adf_cfg.o \ adf_ctl_drv.o \ @@ -6,9 +14,15 @@ intel_qat-objs := adf_cfg.o \ adf_accel_engine.o \ adf_aer.o \ adf_transport.o \ + adf_admin.o \ + adf_hw_arbiter.o \ qat_crypto.o \ qat_algs.o \ + qat_rsapubkey-asn1.o \ + qat_rsaprivkey-asn1.o \ + qat_asym_algs.o \ qat_uclo.o \ qat_hal.o intel_qat-$(CONFIG_DEBUG_FS) += adf_transport_debug.o +intel_qat-$(CONFIG_PCI_IOV) += adf_sriov.o adf_pf2vf_msg.o diff --git a/kernel/drivers/crypto/qat/qat_common/adf_accel_devices.h b/kernel/drivers/crypto/qat/qat_common/adf_accel_devices.h index f22ce7169..ca853d50b 100644 --- a/kernel/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/kernel/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -46,14 +46,17 @@ */ #ifndef ADF_ACCEL_DEVICES_H_ #define ADF_ACCEL_DEVICES_H_ +#include #include #include -#include #include +#include #include "adf_cfg_common.h" #define ADF_DH895XCC_DEVICE_NAME "dh895xcc" +#define ADF_DH895XCCVF_DEVICE_NAME "dh895xccvf" #define ADF_DH895XCC_PCI_DEVICE_ID 0x435 +#define ADF_DH895XCCIOV_PCI_DEVICE_ID 0x443 #define ADF_PCI_MAX_BARS 3 #define ADF_DEVICE_NAME_LENGTH 32 #define ADF_ETR_MAX_RINGS_PER_BANK 16 @@ -80,6 +83,7 @@ struct adf_bar { struct adf_accel_msix { struct msix_entry *entries; char **names; + u32 num_entries; } __packed; struct adf_accel_pci { @@ -100,6 +104,7 @@ enum dev_sku_info { DEV_SKU_2, DEV_SKU_3, DEV_SKU_4, + DEV_SKU_VF, DEV_SKU_UNKNOWN, }; @@ -114,6 +119,8 @@ static inline const char *get_sku_info(enum dev_sku_info info) return "SKU3"; case DEV_SKU_4: return "SKU4"; + case DEV_SKU_VF: + return "SKUVF"; case DEV_SKU_UNKNOWN: default: break; @@ -136,23 +143,29 @@ struct adf_hw_device_data { struct adf_hw_device_class *dev_class; uint32_t (*get_accel_mask)(uint32_t fuse); uint32_t (*get_ae_mask)(uint32_t fuse); + uint32_t (*get_sram_bar_id)(struct adf_hw_device_data *self); uint32_t (*get_misc_bar_id)(struct adf_hw_device_data *self); uint32_t (*get_etr_bar_id)(struct adf_hw_device_data *self); uint32_t (*get_num_aes)(struct adf_hw_device_data *self); uint32_t (*get_num_accels)(struct adf_hw_device_data *self); + uint32_t (*get_pf2vf_offset)(uint32_t i); + uint32_t (*get_vintmsk_offset)(uint32_t i); enum dev_sku_info (*get_sku)(struct adf_hw_device_data *self); - void (*hw_arb_ring_enable)(struct adf_etr_ring_data *ring); - void (*hw_arb_ring_disable)(struct adf_etr_ring_data *ring); int (*alloc_irq)(struct adf_accel_dev *accel_dev); void (*free_irq)(struct adf_accel_dev *accel_dev); void (*enable_error_correction)(struct adf_accel_dev *accel_dev); int (*init_admin_comms)(struct adf_accel_dev *accel_dev); void (*exit_admin_comms)(struct adf_accel_dev *accel_dev); + int (*send_admin_init)(struct adf_accel_dev *accel_dev); int (*init_arb)(struct adf_accel_dev *accel_dev); void (*exit_arb)(struct adf_accel_dev *accel_dev); + void (*get_arb_mapping)(struct adf_accel_dev *accel_dev, + const uint32_t **cfg); + void (*disable_iov)(struct adf_accel_dev *accel_dev); void (*enable_ints)(struct adf_accel_dev *accel_dev); + int (*enable_vf2pf_comms)(struct adf_accel_dev *accel_dev); const char *fw_name; - uint32_t pci_dev_id; + const char *fw_mmp_name; uint32_t fuses; uint32_t accel_capabilities_mask; uint16_t accel_mask; @@ -164,6 +177,7 @@ struct adf_hw_device_data { uint8_t num_accel; uint8_t num_logical_accel; uint8_t num_engines; + uint8_t min_iov_compat_ver; } __packed; /* CSR write macro */ @@ -185,6 +199,16 @@ struct icp_qat_fw_loader_handle; struct adf_fw_loader_data { struct icp_qat_fw_loader_handle *fw_loader; const struct firmware *uof_fw; + const struct firmware *mmp_fw; +}; + +struct adf_accel_vf_info { + struct adf_accel_dev *accel_dev; + struct tasklet_struct vf2pf_bh_tasklet; + struct mutex pf2vf_lock; /* protect CSR access for PF2VF messages */ + struct ratelimit_state vf2pf_ratelimit; + u32 vf_nr; + bool init; }; struct adf_accel_dev { @@ -200,6 +224,21 @@ struct adf_accel_dev { struct list_head list; struct module *owner; struct adf_accel_pci accel_pci_dev; + union { + struct { + /* vf_info is non-zero when SR-IOV is init'ed */ + struct adf_accel_vf_info *vf_info; + } pf; + struct { + char *irq_name; + struct tasklet_struct pf2vf_bh_tasklet; + struct mutex vf2pf_lock; /* protect CSR access */ + struct completion iov_msg_completion; + uint8_t compatible; + uint8_t pf_version; + } vf; + }; + bool is_vf; uint8_t accel_id; } __packed; #endif diff --git a/kernel/drivers/crypto/qat/qat_common/adf_accel_engine.c b/kernel/drivers/crypto/qat/qat_common/adf_accel_engine.c index 7f8b66c91..20b08bdcb 100644 --- a/kernel/drivers/crypto/qat/qat_common/adf_accel_engine.c +++ b/kernel/drivers/crypto/qat/qat_common/adf_accel_engine.c @@ -55,24 +55,36 @@ int adf_ae_fw_load(struct adf_accel_dev *accel_dev) { struct adf_fw_loader_data *loader_data = accel_dev->fw_loader; struct adf_hw_device_data *hw_device = accel_dev->hw_device; - void *uof_addr; - uint32_t uof_size; + void *uof_addr, *mmp_addr; + u32 uof_size, mmp_size; + if (!hw_device->fw_name) + return 0; + + if (request_firmware(&loader_data->mmp_fw, hw_device->fw_mmp_name, + &accel_dev->accel_pci_dev.pci_dev->dev)) { + dev_err(&GET_DEV(accel_dev), "Failed to load MMP firmware %s\n", + hw_device->fw_mmp_name); + return -EFAULT; + } if (request_firmware(&loader_data->uof_fw, hw_device->fw_name, &accel_dev->accel_pci_dev.pci_dev->dev)) { - dev_err(&GET_DEV(accel_dev), "Failed to load firmware %s\n", + dev_err(&GET_DEV(accel_dev), "Failed to load UOF firmware %s\n", hw_device->fw_name); - return -EFAULT; + goto out_err; } uof_size = loader_data->uof_fw->size; uof_addr = (void *)loader_data->uof_fw->data; + mmp_size = loader_data->mmp_fw->size; + mmp_addr = (void *)loader_data->mmp_fw->data; + qat_uclo_wr_mimage(loader_data->fw_loader, mmp_addr, mmp_size); if (qat_uclo_map_uof_obj(loader_data->fw_loader, uof_addr, uof_size)) { dev_err(&GET_DEV(accel_dev), "Failed to map UOF\n"); goto out_err; } if (qat_uclo_wr_all_uimage(loader_data->fw_loader)) { - dev_err(&GET_DEV(accel_dev), "Failed to map UOF\n"); + dev_err(&GET_DEV(accel_dev), "Failed to load UOF\n"); goto out_err; } return 0; @@ -85,14 +97,17 @@ out_err: void adf_ae_fw_release(struct adf_accel_dev *accel_dev) { struct adf_fw_loader_data *loader_data = accel_dev->fw_loader; + struct adf_hw_device_data *hw_device = accel_dev->hw_device; + + if (!hw_device->fw_name) + return; qat_uclo_del_uof_obj(loader_data->fw_loader); qat_hal_deinit(loader_data->fw_loader); - - if (loader_data->uof_fw) - release_firmware(loader_data->uof_fw); - + release_firmware(loader_data->uof_fw); + release_firmware(loader_data->mmp_fw); loader_data->uof_fw = NULL; + loader_data->mmp_fw = NULL; loader_data->fw_loader = NULL; } @@ -102,6 +117,9 @@ int adf_ae_start(struct adf_accel_dev *accel_dev) struct adf_hw_device_data *hw_data = accel_dev->hw_device; uint32_t ae_ctr, ae, max_aes = GET_MAX_ACCELENGINES(accel_dev); + if (!hw_data->fw_name) + return 0; + for (ae = 0, ae_ctr = 0; ae < max_aes; ae++) { if (hw_data->ae_mask & (1 << ae)) { qat_hal_start(loader_data->fw_loader, ae, 0xFF); @@ -120,6 +138,9 @@ int adf_ae_stop(struct adf_accel_dev *accel_dev) struct adf_hw_device_data *hw_data = accel_dev->hw_device; uint32_t ae_ctr, ae, max_aes = GET_MAX_ACCELENGINES(accel_dev); + if (!hw_data->fw_name) + return 0; + for (ae = 0, ae_ctr = 0; ae < max_aes; ae++) { if (hw_data->ae_mask & (1 << ae)) { qat_hal_stop(loader_data->fw_loader, ae, 0xFF); @@ -146,6 +167,10 @@ static int adf_ae_reset(struct adf_accel_dev *accel_dev, int ae) int adf_ae_init(struct adf_accel_dev *accel_dev) { struct adf_fw_loader_data *loader_data; + struct adf_hw_device_data *hw_device = accel_dev->hw_device; + + if (!hw_device->fw_name) + return 0; loader_data = kzalloc(sizeof(*loader_data), GFP_KERNEL); if (!loader_data) @@ -169,6 +194,10 @@ int adf_ae_init(struct adf_accel_dev *accel_dev) int adf_ae_shutdown(struct adf_accel_dev *accel_dev) { struct adf_fw_loader_data *loader_data = accel_dev->fw_loader; + struct adf_hw_device_data *hw_device = accel_dev->hw_device; + + if (!hw_device->fw_name) + return 0; qat_hal_deinit(loader_data->fw_loader); kfree(accel_dev->fw_loader); diff --git a/kernel/drivers/crypto/qat/qat_common/adf_admin.c b/kernel/drivers/crypto/qat/qat_common/adf_admin.c new file mode 100644 index 000000000..147d755fe --- /dev/null +++ b/kernel/drivers/crypto/qat/qat_common/adf_admin.c @@ -0,0 +1,290 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#include +#include +#include +#include +#include +#include +#include "adf_accel_devices.h" +#include "icp_qat_fw_init_admin.h" + +/* Admin Messages Registers */ +#define ADF_DH895XCC_ADMINMSGUR_OFFSET (0x3A000 + 0x574) +#define ADF_DH895XCC_ADMINMSGLR_OFFSET (0x3A000 + 0x578) +#define ADF_DH895XCC_MAILBOX_BASE_OFFSET 0x20970 +#define ADF_DH895XCC_MAILBOX_STRIDE 0x1000 +#define ADF_ADMINMSG_LEN 32 + +static const u8 const_tab[1024] = { +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x11, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x03, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x01, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x13, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13, 0x02, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x13, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13, +0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x33, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0xfe, 0xdc, 0xba, 0x98, 0x76, +0x54, 0x32, 0x10, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x67, 0x45, 0x23, 0x01, 0xef, 0xcd, 0xab, +0x89, 0x98, 0xba, 0xdc, 0xfe, 0x10, 0x32, 0x54, 0x76, 0xc3, 0xd2, 0xe1, 0xf0, +0x00, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc1, 0x05, 0x9e, +0xd8, 0x36, 0x7c, 0xd5, 0x07, 0x30, 0x70, 0xdd, 0x17, 0xf7, 0x0e, 0x59, 0x39, +0xff, 0xc0, 0x0b, 0x31, 0x68, 0x58, 0x15, 0x11, 0x64, 0xf9, 0x8f, 0xa7, 0xbe, +0xfa, 0x4f, 0xa4, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6a, 0x09, 0xe6, 0x67, 0xbb, 0x67, 0xae, +0x85, 0x3c, 0x6e, 0xf3, 0x72, 0xa5, 0x4f, 0xf5, 0x3a, 0x51, 0x0e, 0x52, 0x7f, +0x9b, 0x05, 0x68, 0x8c, 0x1f, 0x83, 0xd9, 0xab, 0x5b, 0xe0, 0xcd, 0x19, 0x05, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0xcb, 0xbb, 0x9d, 0x5d, 0xc1, 0x05, 0x9e, 0xd8, 0x62, 0x9a, 0x29, +0x2a, 0x36, 0x7c, 0xd5, 0x07, 0x91, 0x59, 0x01, 0x5a, 0x30, 0x70, 0xdd, 0x17, +0x15, 0x2f, 0xec, 0xd8, 0xf7, 0x0e, 0x59, 0x39, 0x67, 0x33, 0x26, 0x67, 0xff, +0xc0, 0x0b, 0x31, 0x8e, 0xb4, 0x4a, 0x87, 0x68, 0x58, 0x15, 0x11, 0xdb, 0x0c, +0x2e, 0x0d, 0x64, 0xf9, 0x8f, 0xa7, 0x47, 0xb5, 0x48, 0x1d, 0xbe, 0xfa, 0x4f, +0xa4, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x6a, 0x09, 0xe6, 0x67, 0xf3, 0xbc, 0xc9, 0x08, 0xbb, +0x67, 0xae, 0x85, 0x84, 0xca, 0xa7, 0x3b, 0x3c, 0x6e, 0xf3, 0x72, 0xfe, 0x94, +0xf8, 0x2b, 0xa5, 0x4f, 0xf5, 0x3a, 0x5f, 0x1d, 0x36, 0xf1, 0x51, 0x0e, 0x52, +0x7f, 0xad, 0xe6, 0x82, 0xd1, 0x9b, 0x05, 0x68, 0x8c, 0x2b, 0x3e, 0x6c, 0x1f, +0x1f, 0x83, 0xd9, 0xab, 0xfb, 0x41, 0xbd, 0x6b, 0x5b, 0xe0, 0xcd, 0x19, 0x13, +0x7e, 0x21, 0x79, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; + +struct adf_admin_comms { + dma_addr_t phy_addr; + dma_addr_t const_tbl_addr; + void *virt_addr; + void __iomem *mailbox_addr; + struct mutex lock; /* protects adf_admin_comms struct */ +}; + +static int adf_put_admin_msg_sync(struct adf_accel_dev *accel_dev, u32 ae, + void *in, void *out) +{ + struct adf_admin_comms *admin = accel_dev->admin; + int offset = ae * ADF_ADMINMSG_LEN * 2; + void __iomem *mailbox = admin->mailbox_addr; + int mb_offset = ae * ADF_DH895XCC_MAILBOX_STRIDE; + int times, received; + + mutex_lock(&admin->lock); + + if (ADF_CSR_RD(mailbox, mb_offset) == 1) { + mutex_unlock(&admin->lock); + return -EAGAIN; + } + + memcpy(admin->virt_addr + offset, in, ADF_ADMINMSG_LEN); + ADF_CSR_WR(mailbox, mb_offset, 1); + received = 0; + for (times = 0; times < 50; times++) { + msleep(20); + if (ADF_CSR_RD(mailbox, mb_offset) == 0) { + received = 1; + break; + } + } + if (received) + memcpy(out, admin->virt_addr + offset + + ADF_ADMINMSG_LEN, ADF_ADMINMSG_LEN); + else + dev_err(&GET_DEV(accel_dev), + "Failed to send admin msg to accelerator\n"); + + mutex_unlock(&admin->lock); + return received ? 0 : -EFAULT; +} + +static int adf_send_admin_cmd(struct adf_accel_dev *accel_dev, int cmd) +{ + struct adf_hw_device_data *hw_device = accel_dev->hw_device; + struct icp_qat_fw_init_admin_req req; + struct icp_qat_fw_init_admin_resp resp; + int i; + + memset(&req, 0, sizeof(struct icp_qat_fw_init_admin_req)); + req.init_admin_cmd_id = cmd; + + if (cmd == ICP_QAT_FW_CONSTANTS_CFG) { + req.init_cfg_sz = 1024; + req.init_cfg_ptr = accel_dev->admin->const_tbl_addr; + } + for (i = 0; i < hw_device->get_num_aes(hw_device); i++) { + memset(&resp, 0, sizeof(struct icp_qat_fw_init_admin_resp)); + if (adf_put_admin_msg_sync(accel_dev, i, &req, &resp) || + resp.init_resp_hdr.status) + return -EFAULT; + } + return 0; +} + +/** + * adf_send_admin_init() - Function sends init message to FW + * @accel_dev: Pointer to acceleration device. + * + * Function sends admin init message to the FW + * + * Return: 0 on success, error code otherwise. + */ +int adf_send_admin_init(struct adf_accel_dev *accel_dev) +{ + int ret = adf_send_admin_cmd(accel_dev, ICP_QAT_FW_INIT_ME); + + if (ret) + return ret; + return adf_send_admin_cmd(accel_dev, ICP_QAT_FW_CONSTANTS_CFG); +} +EXPORT_SYMBOL_GPL(adf_send_admin_init); + +int adf_init_admin_comms(struct adf_accel_dev *accel_dev) +{ + struct adf_admin_comms *admin; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_bar *pmisc = + &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)]; + void __iomem *csr = pmisc->virt_addr; + void __iomem *mailbox = csr + ADF_DH895XCC_MAILBOX_BASE_OFFSET; + u64 reg_val; + + admin = kzalloc_node(sizeof(*accel_dev->admin), GFP_KERNEL, + dev_to_node(&GET_DEV(accel_dev))); + if (!admin) + return -ENOMEM; + admin->virt_addr = dma_zalloc_coherent(&GET_DEV(accel_dev), PAGE_SIZE, + &admin->phy_addr, GFP_KERNEL); + if (!admin->virt_addr) { + dev_err(&GET_DEV(accel_dev), "Failed to allocate dma buff\n"); + kfree(admin); + return -ENOMEM; + } + + admin->const_tbl_addr = dma_map_single(&GET_DEV(accel_dev), + (void *) const_tab, 1024, + DMA_TO_DEVICE); + + if (unlikely(dma_mapping_error(&GET_DEV(accel_dev), + admin->const_tbl_addr))) { + dma_free_coherent(&GET_DEV(accel_dev), PAGE_SIZE, + admin->virt_addr, admin->phy_addr); + kfree(admin); + return -ENOMEM; + } + reg_val = (u64)admin->phy_addr; + ADF_CSR_WR(csr, ADF_DH895XCC_ADMINMSGUR_OFFSET, reg_val >> 32); + ADF_CSR_WR(csr, ADF_DH895XCC_ADMINMSGLR_OFFSET, reg_val); + mutex_init(&admin->lock); + admin->mailbox_addr = mailbox; + accel_dev->admin = admin; + return 0; +} +EXPORT_SYMBOL_GPL(adf_init_admin_comms); + +void adf_exit_admin_comms(struct adf_accel_dev *accel_dev) +{ + struct adf_admin_comms *admin = accel_dev->admin; + + if (!admin) + return; + + if (admin->virt_addr) + dma_free_coherent(&GET_DEV(accel_dev), PAGE_SIZE, + admin->virt_addr, admin->phy_addr); + + dma_unmap_single(&GET_DEV(accel_dev), admin->const_tbl_addr, 1024, + DMA_TO_DEVICE); + mutex_destroy(&admin->lock); + kfree(admin); + accel_dev->admin = NULL; +} +EXPORT_SYMBOL_GPL(adf_exit_admin_comms); diff --git a/kernel/drivers/crypto/qat/qat_common/adf_aer.c b/kernel/drivers/crypto/qat/qat_common/adf_aer.c index 2dbc733b8..0a5ca0ba5 100644 --- a/kernel/drivers/crypto/qat/qat_common/adf_aer.c +++ b/kernel/drivers/crypto/qat/qat_common/adf_aer.c @@ -88,9 +88,15 @@ static void adf_dev_restore(struct adf_accel_dev *accel_dev) struct pci_dev *parent = pdev->bus->self; uint16_t bridge_ctl = 0; + if (accel_dev->is_vf) + return; + dev_info(&GET_DEV(accel_dev), "Resetting device qat_dev%d\n", accel_dev->accel_id); + if (!parent) + parent = pdev; + if (!pci_wait_for_pending_transaction(pdev)) dev_info(&GET_DEV(accel_dev), "Transaction still in progress. Proceeding\n"); @@ -206,7 +212,7 @@ static struct pci_error_handlers adf_err_handler = { * QAT acceleration device accel_dev. * To be used by QAT device specific drivers. * - * Return: 0 on success, error code othewise. + * Return: 0 on success, error code otherwise. */ int adf_enable_aer(struct adf_accel_dev *accel_dev, struct pci_driver *adf) { diff --git a/kernel/drivers/crypto/qat/qat_common/adf_cfg.c b/kernel/drivers/crypto/qat/qat_common/adf_cfg.c index ab65bc274..d08797905 100644 --- a/kernel/drivers/crypto/qat/qat_common/adf_cfg.c +++ b/kernel/drivers/crypto/qat/qat_common/adf_cfg.c @@ -123,7 +123,7 @@ static const struct file_operations qat_dev_cfg_fops = { * The table stores device specific config values. * To be used by QAT device specific drivers. * - * Return: 0 on success, error code othewise. + * Return: 0 on success, error code otherwise. */ int adf_cfg_dev_add(struct adf_accel_dev *accel_dev) { @@ -178,6 +178,9 @@ void adf_cfg_dev_remove(struct adf_accel_dev *accel_dev) { struct adf_cfg_device_data *dev_cfg_data = accel_dev->cfg; + if (!dev_cfg_data) + return; + down_write(&dev_cfg_data->lock); adf_cfg_section_del_all(&dev_cfg_data->sec_list); up_write(&dev_cfg_data->lock); @@ -276,7 +279,7 @@ static int adf_cfg_key_val_get(struct adf_accel_dev *accel_dev, * in the given acceleration device * To be used by QAT device specific drivers. * - * Return: 0 on success, error code othewise. + * Return: 0 on success, error code otherwise. */ int adf_cfg_add_key_value_param(struct adf_accel_dev *accel_dev, const char *section_name, @@ -327,7 +330,7 @@ EXPORT_SYMBOL_GPL(adf_cfg_add_key_value_param); * will be stored. * To be used by QAT device specific drivers. * - * Return: 0 on success, error code othewise. + * Return: 0 on success, error code otherwise. */ int adf_cfg_section_add(struct adf_accel_dev *accel_dev, const char *name) { diff --git a/kernel/drivers/crypto/qat/qat_common/adf_cfg_common.h b/kernel/drivers/crypto/qat/qat_common/adf_cfg_common.h index 88b82187a..c697fb1cd 100644 --- a/kernel/drivers/crypto/qat/qat_common/adf_cfg_common.h +++ b/kernel/drivers/crypto/qat/qat_common/adf_cfg_common.h @@ -60,7 +60,7 @@ #define ADF_CFG_NO_DEVICE 0xFF #define ADF_CFG_AFFINITY_WHATEVER 0xFF #define MAX_DEVICE_NAME_SIZE 32 -#define ADF_MAX_DEVICES 32 +#define ADF_MAX_DEVICES (32 * 32) enum adf_cfg_val_type { ADF_DEC, @@ -71,6 +71,7 @@ enum adf_cfg_val_type { enum adf_device_type { DEV_UNKNOWN = 0, DEV_DH895XCC, + DEV_DH895XCCVF, }; struct adf_dev_status_info { diff --git a/kernel/drivers/crypto/qat/qat_common/adf_cfg_user.h b/kernel/drivers/crypto/qat/qat_common/adf_cfg_user.h index 0c38a155a..ef5988afd 100644 --- a/kernel/drivers/crypto/qat/qat_common/adf_cfg_user.h +++ b/kernel/drivers/crypto/qat/qat_common/adf_cfg_user.h @@ -53,14 +53,6 @@ struct adf_user_cfg_key_val { char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; - union { - char *user_val_ptr; - uint64_t padding1; - }; - union { - struct adf_user_cfg_key_val *prev; - uint64_t padding2; - }; union { struct adf_user_cfg_key_val *next; uint64_t padding3; @@ -74,10 +66,6 @@ struct adf_user_cfg_section { struct adf_user_cfg_key_val *params; uint64_t padding1; }; - union { - struct adf_user_cfg_section *prev; - uint64_t padding2; - }; union { struct adf_user_cfg_section *next; uint64_t padding3; diff --git a/kernel/drivers/crypto/qat/qat_common/adf_common_drv.h b/kernel/drivers/crypto/qat/qat_common/adf_common_drv.h index 0666ee6a3..3f76bd495 100644 --- a/kernel/drivers/crypto/qat/qat_common/adf_common_drv.h +++ b/kernel/drivers/crypto/qat/qat_common/adf_common_drv.h @@ -53,6 +53,13 @@ #include "icp_qat_fw_loader_handle.h" #include "icp_qat_hal.h" +#define ADF_MAJOR_VERSION 0 +#define ADF_MINOR_VERSION 2 +#define ADF_BUILD_VERSION 0 +#define ADF_DRV_VERSION __stringify(ADF_MAJOR_VERSION) "." \ + __stringify(ADF_MINOR_VERSION) "." \ + __stringify(ADF_BUILD_VERSION) + #define ADF_STATUS_RESTARTING 0 #define ADF_STATUS_STARTING 1 #define ADF_STATUS_CONFIGURED 2 @@ -84,9 +91,13 @@ struct service_hndl { unsigned long start_status; char *name; struct list_head list; - int admin; }; +static inline int get_current_node(void) +{ + return topology_physical_package_id(smp_processor_id()); +} + int adf_service_register(struct service_hndl *service); int adf_service_unregister(struct service_hndl *service); @@ -95,13 +106,24 @@ int adf_dev_start(struct adf_accel_dev *accel_dev); int adf_dev_stop(struct adf_accel_dev *accel_dev); void adf_dev_shutdown(struct adf_accel_dev *accel_dev); +void adf_enable_pf2vf_interrupts(struct adf_accel_dev *accel_dev); +void adf_disable_pf2vf_interrupts(struct adf_accel_dev *accel_dev); +int adf_iov_putmsg(struct adf_accel_dev *accel_dev, u32 msg, u8 vf_nr); +void adf_pf2vf_notify_restarting(struct adf_accel_dev *accel_dev); +int adf_enable_vf2pf_comms(struct adf_accel_dev *accel_dev); +void adf_vf2pf_req_hndl(struct adf_accel_vf_info *vf_info); +void adf_devmgr_update_class_index(struct adf_hw_device_data *hw_data); +void adf_clean_vf_map(bool); + int adf_ctl_dev_register(void); void adf_ctl_dev_unregister(void); int adf_processes_dev_register(void); void adf_processes_dev_unregister(void); -int adf_devmgr_add_dev(struct adf_accel_dev *accel_dev); -void adf_devmgr_rm_dev(struct adf_accel_dev *accel_dev); +int adf_devmgr_add_dev(struct adf_accel_dev *accel_dev, + struct adf_accel_dev *pf); +void adf_devmgr_rm_dev(struct adf_accel_dev *accel_dev, + struct adf_accel_dev *pf); struct list_head *adf_devmgr_get_head(void); struct adf_accel_dev *adf_devmgr_get_dev_by_id(uint32_t id); struct adf_accel_dev *adf_devmgr_get_first(void); @@ -123,6 +145,12 @@ int adf_enable_aer(struct adf_accel_dev *accel_dev, struct pci_driver *adf); void adf_disable_aer(struct adf_accel_dev *accel_dev); int adf_init_aer(void); void adf_exit_aer(void); +int adf_init_admin_comms(struct adf_accel_dev *accel_dev); +void adf_exit_admin_comms(struct adf_accel_dev *accel_dev); +int adf_send_admin_init(struct adf_accel_dev *accel_dev); +int adf_init_arb(struct adf_accel_dev *accel_dev); +void adf_exit_arb(struct adf_accel_dev *accel_dev); +void adf_update_ring_arb(struct adf_etr_ring_data *ring); int adf_dev_get(struct adf_accel_dev *accel_dev); void adf_dev_put(struct adf_accel_dev *accel_dev); @@ -134,10 +162,11 @@ int qat_crypto_unregister(void); struct qat_crypto_instance *qat_crypto_get_instance_node(int node); void qat_crypto_put_instance(struct qat_crypto_instance *inst); void qat_alg_callback(void *resp); -int qat_algs_init(void); -void qat_algs_exit(void); +void qat_alg_asym_callback(void *resp); int qat_algs_register(void); -int qat_algs_unregister(void); +void qat_algs_unregister(void); +int qat_asym_algs_register(void); +void qat_asym_algs_unregister(void); int qat_hal_init(struct adf_accel_dev *accel_dev); void qat_hal_deinit(struct icp_qat_fw_loader_handle *handle); @@ -189,4 +218,23 @@ int qat_uclo_wr_all_uimage(struct icp_qat_fw_loader_handle *handle); void qat_uclo_del_uof_obj(struct icp_qat_fw_loader_handle *handle); int qat_uclo_map_uof_obj(struct icp_qat_fw_loader_handle *handle, void *addr_ptr, int mem_size); +void qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle, + void *addr_ptr, int mem_size); +#if defined(CONFIG_PCI_IOV) +int adf_sriov_configure(struct pci_dev *pdev, int numvfs); +void adf_disable_sriov(struct adf_accel_dev *accel_dev); +void adf_disable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, + uint32_t vf_mask); +void adf_enable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, + uint32_t vf_mask); +#else +static inline int adf_sriov_configure(struct pci_dev *pdev, int numvfs) +{ + return 0; +} + +static inline void adf_disable_sriov(struct adf_accel_dev *accel_dev) +{ +} +#endif #endif diff --git a/kernel/drivers/crypto/qat/qat_common/adf_ctl_drv.c b/kernel/drivers/crypto/qat/qat_common/adf_ctl_drv.c index cb5f066e9..473d36d91 100644 --- a/kernel/drivers/crypto/qat/qat_common/adf_ctl_drv.c +++ b/kernel/drivers/crypto/qat/qat_common/adf_ctl_drv.c @@ -198,7 +198,7 @@ static int adf_copy_key_value_data(struct adf_accel_dev *accel_dev, goto out_err; } - params_head = section_head->params; + params_head = section.params; while (params_head) { if (copy_from_user(&key_val, (void __user *)params_head, @@ -398,10 +398,9 @@ static int adf_ctl_ioctl_get_status(struct file *fp, unsigned int cmd, } accel_dev = adf_devmgr_get_dev_by_id(dev_info.accel_id); - if (!accel_dev) { - pr_err("QAT: Device %d not found\n", dev_info.accel_id); + if (!accel_dev) return -ENODEV; - } + hw_data = accel_dev->hw_device; dev_info.state = adf_dev_started(accel_dev) ? DEV_UP : DEV_DOWN; dev_info.num_ae = hw_data->get_num_aes(hw_data); @@ -464,9 +463,6 @@ static int __init adf_register_ctl_device_driver(void) { mutex_init(&adf_ctl_lock); - if (qat_algs_init()) - goto err_algs_init; - if (adf_chr_drv_create()) goto err_chr_dev; @@ -483,8 +479,6 @@ err_crypto_register: err_aer: adf_chr_drv_destroy(); err_chr_dev: - qat_algs_exit(); -err_algs_init: mutex_destroy(&adf_ctl_lock); return -EFAULT; } @@ -494,7 +488,7 @@ static void __exit adf_unregister_ctl_device_driver(void) adf_chr_drv_destroy(); adf_exit_aer(); qat_crypto_unregister(); - qat_algs_exit(); + adf_clean_vf_map(false); mutex_destroy(&adf_ctl_lock); } @@ -504,3 +498,4 @@ MODULE_LICENSE("Dual BSD/GPL"); MODULE_AUTHOR("Intel"); MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); MODULE_ALIAS_CRYPTO("intel_qat"); +MODULE_VERSION(ADF_DRV_VERSION); diff --git a/kernel/drivers/crypto/qat/qat_common/adf_dev_mgr.c b/kernel/drivers/crypto/qat/qat_common/adf_dev_mgr.c index 3f0ff9e7d..8dfdb8f90 100644 --- a/kernel/drivers/crypto/qat/qat_common/adf_dev_mgr.c +++ b/kernel/drivers/crypto/qat/qat_common/adf_dev_mgr.c @@ -50,21 +50,125 @@ #include "adf_common_drv.h" static LIST_HEAD(accel_table); +static LIST_HEAD(vfs_table); static DEFINE_MUTEX(table_lock); static uint32_t num_devices; +struct vf_id_map { + u32 bdf; + u32 id; + u32 fake_id; + bool attached; + struct list_head list; +}; + +static int adf_get_vf_id(struct adf_accel_dev *vf) +{ + return ((7 * (PCI_SLOT(accel_to_pci_dev(vf)->devfn) - 1)) + + PCI_FUNC(accel_to_pci_dev(vf)->devfn) + + (PCI_SLOT(accel_to_pci_dev(vf)->devfn) - 1)); +} + +static int adf_get_vf_num(struct adf_accel_dev *vf) +{ + return (accel_to_pci_dev(vf)->bus->number << 8) | adf_get_vf_id(vf); +} + +static struct vf_id_map *adf_find_vf(u32 bdf) +{ + struct list_head *itr; + + list_for_each(itr, &vfs_table) { + struct vf_id_map *ptr = + list_entry(itr, struct vf_id_map, list); + + if (ptr->bdf == bdf) + return ptr; + } + return NULL; +} + +static int adf_get_vf_real_id(u32 fake) +{ + struct list_head *itr; + + list_for_each(itr, &vfs_table) { + struct vf_id_map *ptr = + list_entry(itr, struct vf_id_map, list); + if (ptr->fake_id == fake) + return ptr->id; + } + return -1; +} + +/** + * adf_clean_vf_map() - Cleans VF id mapings + * + * Function cleans internal ids for virtual functions. + * @vf: flag indicating whether mappings is cleaned + * for vfs only or for vfs and pfs + */ +void adf_clean_vf_map(bool vf) +{ + struct vf_id_map *map; + struct list_head *ptr, *tmp; + + mutex_lock(&table_lock); + list_for_each_safe(ptr, tmp, &vfs_table) { + map = list_entry(ptr, struct vf_id_map, list); + if (map->bdf != -1) + num_devices--; + + if (vf && map->bdf == -1) + continue; + + list_del(ptr); + kfree(map); + } + mutex_unlock(&table_lock); +} +EXPORT_SYMBOL_GPL(adf_clean_vf_map); + +/** + * adf_devmgr_update_class_index() - Update internal index + * @hw_data: Pointer to internal device data. + * + * Function updates internal dev index for VFs + */ +void adf_devmgr_update_class_index(struct adf_hw_device_data *hw_data) +{ + struct adf_hw_device_class *class = hw_data->dev_class; + struct list_head *itr; + int i = 0; + + list_for_each(itr, &accel_table) { + struct adf_accel_dev *ptr = + list_entry(itr, struct adf_accel_dev, list); + + if (ptr->hw_device->dev_class == class) + ptr->hw_device->instance_id = i++; + + if (i == class->instances) + break; + } +} +EXPORT_SYMBOL_GPL(adf_devmgr_update_class_index); + /** * adf_devmgr_add_dev() - Add accel_dev to the acceleration framework * @accel_dev: Pointer to acceleration device. + * @pf: Corresponding PF if the accel_dev is a VF * * Function adds acceleration device to the acceleration framework. * To be used by QAT device specific drivers. * - * Return: 0 on success, error code othewise. + * Return: 0 on success, error code otherwise. */ -int adf_devmgr_add_dev(struct adf_accel_dev *accel_dev) +int adf_devmgr_add_dev(struct adf_accel_dev *accel_dev, + struct adf_accel_dev *pf) { struct list_head *itr; + int ret = 0; if (num_devices == ADF_MAX_DEVICES) { dev_err(&GET_DEV(accel_dev), "Only support up to %d devices\n", @@ -73,20 +177,77 @@ int adf_devmgr_add_dev(struct adf_accel_dev *accel_dev) } mutex_lock(&table_lock); - list_for_each(itr, &accel_table) { - struct adf_accel_dev *ptr = + atomic_set(&accel_dev->ref_count, 0); + + /* PF on host or VF on guest */ + if (!accel_dev->is_vf || (accel_dev->is_vf && !pf)) { + struct vf_id_map *map; + + list_for_each(itr, &accel_table) { + struct adf_accel_dev *ptr = list_entry(itr, struct adf_accel_dev, list); - if (ptr == accel_dev) { - mutex_unlock(&table_lock); - return -EEXIST; + if (ptr == accel_dev) { + ret = -EEXIST; + goto unlock; + } } + + list_add_tail(&accel_dev->list, &accel_table); + accel_dev->accel_id = num_devices++; + + map = kzalloc(sizeof(*map), GFP_KERNEL); + if (!map) { + ret = -ENOMEM; + goto unlock; + } + map->bdf = ~0; + map->id = accel_dev->accel_id; + map->fake_id = map->id; + map->attached = true; + list_add_tail(&map->list, &vfs_table); + } else if (accel_dev->is_vf && pf) { + /* VF on host */ + struct adf_accel_vf_info *vf_info; + struct vf_id_map *map; + + vf_info = pf->pf.vf_info + adf_get_vf_id(accel_dev); + + map = adf_find_vf(adf_get_vf_num(accel_dev)); + if (map) { + struct vf_id_map *next; + + accel_dev->accel_id = map->id; + list_add_tail(&accel_dev->list, &accel_table); + map->fake_id++; + map->attached = true; + next = list_next_entry(map, list); + while (next && &next->list != &vfs_table) { + next->fake_id++; + next = list_next_entry(next, list); + } + + ret = 0; + goto unlock; + } + + map = kzalloc(sizeof(*map), GFP_KERNEL); + if (!map) { + ret = -ENOMEM; + goto unlock; + } + + accel_dev->accel_id = num_devices++; + list_add_tail(&accel_dev->list, &accel_table); + map->bdf = adf_get_vf_num(accel_dev); + map->id = accel_dev->accel_id; + map->fake_id = map->id; + map->attached = true; + list_add_tail(&map->list, &vfs_table); } - atomic_set(&accel_dev->ref_count, 0); - list_add_tail(&accel_dev->list, &accel_table); - accel_dev->accel_id = num_devices++; +unlock: mutex_unlock(&table_lock); - return 0; + return ret; } EXPORT_SYMBOL_GPL(adf_devmgr_add_dev); @@ -98,17 +259,37 @@ struct list_head *adf_devmgr_get_head(void) /** * adf_devmgr_rm_dev() - Remove accel_dev from the acceleration framework. * @accel_dev: Pointer to acceleration device. + * @pf: Corresponding PF if the accel_dev is a VF * * Function removes acceleration device from the acceleration framework. * To be used by QAT device specific drivers. * * Return: void */ -void adf_devmgr_rm_dev(struct adf_accel_dev *accel_dev) +void adf_devmgr_rm_dev(struct adf_accel_dev *accel_dev, + struct adf_accel_dev *pf) { mutex_lock(&table_lock); + if (!accel_dev->is_vf || (accel_dev->is_vf && !pf)) { + num_devices--; + } else if (accel_dev->is_vf && pf) { + struct vf_id_map *map, *next; + + map = adf_find_vf(adf_get_vf_num(accel_dev)); + if (!map) { + dev_err(&GET_DEV(accel_dev), "Failed to find VF map\n"); + goto unlock; + } + map->fake_id--; + map->attached = false; + next = list_next_entry(map, list); + while (next && &next->list != &vfs_table) { + next->fake_id--; + next = list_next_entry(next, list); + } + } +unlock: list_del(&accel_dev->list); - num_devices--; mutex_unlock(&table_lock); } EXPORT_SYMBOL_GPL(adf_devmgr_rm_dev); @@ -154,17 +335,24 @@ EXPORT_SYMBOL_GPL(adf_devmgr_pci_to_accel_dev); struct adf_accel_dev *adf_devmgr_get_dev_by_id(uint32_t id) { struct list_head *itr; + int real_id; mutex_lock(&table_lock); + real_id = adf_get_vf_real_id(id); + if (real_id < 0) + goto unlock; + + id = real_id; + list_for_each(itr, &accel_table) { struct adf_accel_dev *ptr = list_entry(itr, struct adf_accel_dev, list); - if (ptr->accel_id == id) { mutex_unlock(&table_lock); return ptr; } } +unlock: mutex_unlock(&table_lock); return NULL; } @@ -180,21 +368,52 @@ int adf_devmgr_verify_id(uint32_t id) return -ENODEV; } -void adf_devmgr_get_num_dev(uint32_t *num) +static int adf_get_num_dettached_vfs(void) { struct list_head *itr; + int vfs = 0; - *num = 0; - list_for_each(itr, &accel_table) { - (*num)++; + mutex_lock(&table_lock); + list_for_each(itr, &vfs_table) { + struct vf_id_map *ptr = + list_entry(itr, struct vf_id_map, list); + if (ptr->bdf != ~0 && !ptr->attached) + vfs++; } + mutex_unlock(&table_lock); + return vfs; +} + +void adf_devmgr_get_num_dev(uint32_t *num) +{ + *num = num_devices - adf_get_num_dettached_vfs(); } +/** + * adf_dev_in_use() - Check whether accel_dev is currently in use + * @accel_dev: Pointer to acceleration device. + * + * To be used by QAT device specific drivers. + * + * Return: 1 when device is in use, 0 otherwise. + */ int adf_dev_in_use(struct adf_accel_dev *accel_dev) { return atomic_read(&accel_dev->ref_count) != 0; } +EXPORT_SYMBOL_GPL(adf_dev_in_use); +/** + * adf_dev_get() - Increment accel_dev reference count + * @accel_dev: Pointer to acceleration device. + * + * Increment the accel_dev refcount and if this is the first time + * incrementing it during this period the accel_dev is in use, + * increment the module refcount too. + * To be used by QAT device specific drivers. + * + * Return: 0 when successful, EFAULT when fail to bump module refcount + */ int adf_dev_get(struct adf_accel_dev *accel_dev) { if (atomic_add_return(1, &accel_dev->ref_count) == 1) @@ -202,19 +421,50 @@ int adf_dev_get(struct adf_accel_dev *accel_dev) return -EFAULT; return 0; } +EXPORT_SYMBOL_GPL(adf_dev_get); +/** + * adf_dev_put() - Decrement accel_dev reference count + * @accel_dev: Pointer to acceleration device. + * + * Decrement the accel_dev refcount and if this is the last time + * decrementing it during this period the accel_dev is in use, + * decrement the module refcount too. + * To be used by QAT device specific drivers. + * + * Return: void + */ void adf_dev_put(struct adf_accel_dev *accel_dev) { if (atomic_sub_return(1, &accel_dev->ref_count) == 0) module_put(accel_dev->owner); } +EXPORT_SYMBOL_GPL(adf_dev_put); +/** + * adf_devmgr_in_reset() - Check whether device is in reset + * @accel_dev: Pointer to acceleration device. + * + * To be used by QAT device specific drivers. + * + * Return: 1 when the device is being reset, 0 otherwise. + */ int adf_devmgr_in_reset(struct adf_accel_dev *accel_dev) { return test_bit(ADF_STATUS_RESTARTING, &accel_dev->status); } +EXPORT_SYMBOL_GPL(adf_devmgr_in_reset); +/** + * adf_dev_started() - Check whether device has started + * @accel_dev: Pointer to acceleration device. + * + * To be used by QAT device specific drivers. + * + * Return: 1 when the device has started, 0 otherwise + */ int adf_dev_started(struct adf_accel_dev *accel_dev) { return test_bit(ADF_STATUS_STARTED, &accel_dev->status); } +EXPORT_SYMBOL_GPL(adf_dev_started); diff --git a/kernel/drivers/crypto/qat/qat_common/adf_hw_arbiter.c b/kernel/drivers/crypto/qat/qat_common/adf_hw_arbiter.c new file mode 100644 index 000000000..6849422e0 --- /dev/null +++ b/kernel/drivers/crypto/qat/qat_common/adf_hw_arbiter.c @@ -0,0 +1,168 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#include "adf_accel_devices.h" +#include "adf_transport_internal.h" + +#define ADF_ARB_NUM 4 +#define ADF_ARB_REQ_RING_NUM 8 +#define ADF_ARB_REG_SIZE 0x4 +#define ADF_ARB_WTR_SIZE 0x20 +#define ADF_ARB_OFFSET 0x30000 +#define ADF_ARB_REG_SLOT 0x1000 +#define ADF_ARB_WTR_OFFSET 0x010 +#define ADF_ARB_RO_EN_OFFSET 0x090 +#define ADF_ARB_WQCFG_OFFSET 0x100 +#define ADF_ARB_WRK_2_SER_MAP_OFFSET 0x180 +#define ADF_ARB_RINGSRVARBEN_OFFSET 0x19C + +#define WRITE_CSR_ARB_RINGSRVARBEN(csr_addr, index, value) \ + ADF_CSR_WR(csr_addr, ADF_ARB_RINGSRVARBEN_OFFSET + \ + (ADF_ARB_REG_SLOT * index), value) + +#define WRITE_CSR_ARB_RESPORDERING(csr_addr, index, value) \ + ADF_CSR_WR(csr_addr, (ADF_ARB_OFFSET + \ + ADF_ARB_RO_EN_OFFSET) + (ADF_ARB_REG_SIZE * index), value) + +#define WRITE_CSR_ARB_WEIGHT(csr_addr, arb, index, value) \ + ADF_CSR_WR(csr_addr, (ADF_ARB_OFFSET + \ + ADF_ARB_WTR_OFFSET) + (ADF_ARB_WTR_SIZE * arb) + \ + (ADF_ARB_REG_SIZE * index), value) + +#define WRITE_CSR_ARB_SARCONFIG(csr_addr, index, value) \ + ADF_CSR_WR(csr_addr, ADF_ARB_OFFSET + \ + (ADF_ARB_REG_SIZE * index), value) + +#define WRITE_CSR_ARB_WRK_2_SER_MAP(csr_addr, index, value) \ + ADF_CSR_WR(csr_addr, (ADF_ARB_OFFSET + \ + ADF_ARB_WRK_2_SER_MAP_OFFSET) + \ + (ADF_ARB_REG_SIZE * index), value) + +#define WRITE_CSR_ARB_WQCFG(csr_addr, index, value) \ + ADF_CSR_WR(csr_addr, (ADF_ARB_OFFSET + \ + ADF_ARB_WQCFG_OFFSET) + (ADF_ARB_REG_SIZE * index), value) + +int adf_init_arb(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + void __iomem *csr = accel_dev->transport->banks[0].csr_addr; + u32 arb_cfg = 0x1 << 31 | 0x4 << 4 | 0x1; + u32 arb, i; + const u32 *thd_2_arb_cfg; + + /* Service arb configured for 32 bytes responses and + * ring flow control check enabled. */ + for (arb = 0; arb < ADF_ARB_NUM; arb++) + WRITE_CSR_ARB_SARCONFIG(csr, arb, arb_cfg); + + /* Setup service weighting */ + for (arb = 0; arb < ADF_ARB_NUM; arb++) + for (i = 0; i < ADF_ARB_REQ_RING_NUM; i++) + WRITE_CSR_ARB_WEIGHT(csr, arb, i, 0xFFFFFFFF); + + /* Setup ring response ordering */ + for (i = 0; i < ADF_ARB_REQ_RING_NUM; i++) + WRITE_CSR_ARB_RESPORDERING(csr, i, 0xFFFFFFFF); + + /* Setup worker queue registers */ + for (i = 0; i < hw_data->num_engines; i++) + WRITE_CSR_ARB_WQCFG(csr, i, i); + + /* Map worker threads to service arbiters */ + hw_data->get_arb_mapping(accel_dev, &thd_2_arb_cfg); + + if (!thd_2_arb_cfg) + return -EFAULT; + + for (i = 0; i < hw_data->num_engines; i++) + WRITE_CSR_ARB_WRK_2_SER_MAP(csr, i, *(thd_2_arb_cfg + i)); + + return 0; +} +EXPORT_SYMBOL_GPL(adf_init_arb); + +/** + * adf_update_ring_arb() - update ring arbitration rgister + * @accel_dev: Pointer to ring data. + * + * Function enables or disables rings for/from arbitration. + */ +void adf_update_ring_arb(struct adf_etr_ring_data *ring) +{ + WRITE_CSR_ARB_RINGSRVARBEN(ring->bank->csr_addr, + ring->bank->bank_number, + ring->bank->ring_mask & 0xFF); +} +EXPORT_SYMBOL_GPL(adf_update_ring_arb); + +void adf_exit_arb(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + void __iomem *csr; + unsigned int i; + + if (!accel_dev->transport) + return; + + csr = accel_dev->transport->banks[0].csr_addr; + + /* Reset arbiter configuration */ + for (i = 0; i < ADF_ARB_NUM; i++) + WRITE_CSR_ARB_SARCONFIG(csr, i, 0); + + /* Shutdown work queue */ + for (i = 0; i < hw_data->num_engines; i++) + WRITE_CSR_ARB_WQCFG(csr, i, 0); + + /* Unmap worker threads to service arbiters */ + for (i = 0; i < hw_data->num_engines; i++) + WRITE_CSR_ARB_WRK_2_SER_MAP(csr, i, 0); + + /* Disable arbitration on all rings */ + for (i = 0; i < GET_MAX_BANKS(accel_dev); i++) + WRITE_CSR_ARB_RINGSRVARBEN(csr, i, 0); +} +EXPORT_SYMBOL_GPL(adf_exit_arb); diff --git a/kernel/drivers/crypto/qat/qat_common/adf_init.c b/kernel/drivers/crypto/qat/qat_common/adf_init.c index 245f43237..d873eeecc 100644 --- a/kernel/drivers/crypto/qat/qat_common/adf_init.c +++ b/kernel/drivers/crypto/qat/qat_common/adf_init.c @@ -69,7 +69,7 @@ static void adf_service_add(struct service_hndl *service) * Function adds the acceleration service to the acceleration framework. * To be used by QAT device specific drivers. * - * Return: 0 on success, error code othewise. + * Return: 0 on success, error code otherwise. */ int adf_service_register(struct service_hndl *service) { @@ -94,7 +94,7 @@ static void adf_service_remove(struct service_hndl *service) * Function remove the acceleration service from the acceleration framework. * To be used by QAT device specific drivers. * - * Return: 0 on success, error code othewise. + * Return: 0 on success, error code otherwise. */ int adf_service_unregister(struct service_hndl *service) { @@ -114,7 +114,7 @@ EXPORT_SYMBOL_GPL(adf_service_unregister); * Initialize the ring data structures and the admin comms and arbitration * services. * - * Return: 0 on success, error code othewise. + * Return: 0 on success, error code otherwise. */ int adf_dev_init(struct adf_accel_dev *accel_dev) { @@ -177,20 +177,6 @@ int adf_dev_init(struct adf_accel_dev *accel_dev) */ list_for_each(list_itr, &service_table) { service = list_entry(list_itr, struct service_hndl, list); - if (!service->admin) - continue; - if (service->event_hld(accel_dev, ADF_EVENT_INIT)) { - dev_err(&GET_DEV(accel_dev), - "Failed to initialise service %s\n", - service->name); - return -EFAULT; - } - set_bit(accel_dev->accel_id, &service->init_status); - } - list_for_each(list_itr, &service_table) { - service = list_entry(list_itr, struct service_hndl, list); - if (service->admin) - continue; if (service->event_hld(accel_dev, ADF_EVENT_INIT)) { dev_err(&GET_DEV(accel_dev), "Failed to initialise service %s\n", @@ -201,6 +187,7 @@ int adf_dev_init(struct adf_accel_dev *accel_dev) } hw_data->enable_error_correction(accel_dev); + hw_data->enable_vf2pf_comms(accel_dev); return 0; } @@ -214,10 +201,11 @@ EXPORT_SYMBOL_GPL(adf_dev_init); * is ready to be used. * To be used by QAT device specific drivers. * - * Return: 0 on success, error code othewise. + * Return: 0 on success, error code otherwise. */ int adf_dev_start(struct adf_accel_dev *accel_dev) { + struct adf_hw_device_data *hw_data = accel_dev->hw_device; struct service_hndl *service; struct list_head *list_itr; @@ -229,22 +217,13 @@ int adf_dev_start(struct adf_accel_dev *accel_dev) } set_bit(ADF_STATUS_AE_STARTED, &accel_dev->status); - list_for_each(list_itr, &service_table) { - service = list_entry(list_itr, struct service_hndl, list); - if (!service->admin) - continue; - if (service->event_hld(accel_dev, ADF_EVENT_START)) { - dev_err(&GET_DEV(accel_dev), - "Failed to start service %s\n", - service->name); - return -EFAULT; - } - set_bit(accel_dev->accel_id, &service->start_status); + if (hw_data->send_admin_init(accel_dev)) { + dev_err(&GET_DEV(accel_dev), "Failed to send init message\n"); + return -EFAULT; } + list_for_each(list_itr, &service_table) { service = list_entry(list_itr, struct service_hndl, list); - if (service->admin) - continue; if (service->event_hld(accel_dev, ADF_EVENT_START)) { dev_err(&GET_DEV(accel_dev), "Failed to start service %s\n", @@ -257,7 +236,8 @@ int adf_dev_start(struct adf_accel_dev *accel_dev) clear_bit(ADF_STATUS_STARTING, &accel_dev->status); set_bit(ADF_STATUS_STARTED, &accel_dev->status); - if (qat_algs_register()) { + if (!list_empty(&accel_dev->crypto_list) && + (qat_algs_register() || qat_asym_algs_register())) { dev_err(&GET_DEV(accel_dev), "Failed to register crypto algs\n"); set_bit(ADF_STATUS_STARTING, &accel_dev->status); @@ -276,7 +256,7 @@ EXPORT_SYMBOL_GPL(adf_dev_start); * is shuting down. * To be used by QAT device specific drivers. * - * Return: 0 on success, error code othewise. + * Return: 0 on success, error code otherwise. */ int adf_dev_stop(struct adf_accel_dev *accel_dev) { @@ -292,14 +272,13 @@ int adf_dev_stop(struct adf_accel_dev *accel_dev) clear_bit(ADF_STATUS_STARTING, &accel_dev->status); clear_bit(ADF_STATUS_STARTED, &accel_dev->status); - if (qat_algs_unregister()) - dev_err(&GET_DEV(accel_dev), - "Failed to unregister crypto algs\n"); + if (!list_empty(&accel_dev->crypto_list)) { + qat_algs_unregister(); + qat_asym_algs_unregister(); + } list_for_each(list_itr, &service_table) { service = list_entry(list_itr, struct service_hndl, list); - if (service->admin) - continue; if (!test_bit(accel_dev->accel_id, &service->start_status)) continue; ret = service->event_hld(accel_dev, ADF_EVENT_STOP); @@ -310,19 +289,6 @@ int adf_dev_stop(struct adf_accel_dev *accel_dev) clear_bit(accel_dev->accel_id, &service->start_status); } } - list_for_each(list_itr, &service_table) { - service = list_entry(list_itr, struct service_hndl, list); - if (!service->admin) - continue; - if (!test_bit(accel_dev->accel_id, &service->start_status)) - continue; - if (service->event_hld(accel_dev, ADF_EVENT_STOP)) - dev_err(&GET_DEV(accel_dev), - "Failed to shutdown service %s\n", - service->name); - else - clear_bit(accel_dev->accel_id, &service->start_status); - } if (wait) msleep(100); @@ -373,21 +339,6 @@ void adf_dev_shutdown(struct adf_accel_dev *accel_dev) list_for_each(list_itr, &service_table) { service = list_entry(list_itr, struct service_hndl, list); - if (service->admin) - continue; - if (!test_bit(accel_dev->accel_id, &service->init_status)) - continue; - if (service->event_hld(accel_dev, ADF_EVENT_SHUTDOWN)) - dev_err(&GET_DEV(accel_dev), - "Failed to shutdown service %s\n", - service->name); - else - clear_bit(accel_dev->accel_id, &service->init_status); - } - list_for_each(list_itr, &service_table) { - service = list_entry(list_itr, struct service_hndl, list); - if (!service->admin) - continue; if (!test_bit(accel_dev->accel_id, &service->init_status)) continue; if (service->event_hld(accel_dev, ADF_EVENT_SHUTDOWN)) @@ -413,6 +364,7 @@ void adf_dev_shutdown(struct adf_accel_dev *accel_dev) if (hw_data->exit_admin_comms) hw_data->exit_admin_comms(accel_dev); + hw_data->disable_iov(accel_dev); adf_cleanup_etr_data(accel_dev); } EXPORT_SYMBOL_GPL(adf_dev_shutdown); @@ -424,17 +376,6 @@ int adf_dev_restarting_notify(struct adf_accel_dev *accel_dev) list_for_each(list_itr, &service_table) { service = list_entry(list_itr, struct service_hndl, list); - if (service->admin) - continue; - if (service->event_hld(accel_dev, ADF_EVENT_RESTARTING)) - dev_err(&GET_DEV(accel_dev), - "Failed to restart service %s.\n", - service->name); - } - list_for_each(list_itr, &service_table) { - service = list_entry(list_itr, struct service_hndl, list); - if (!service->admin) - continue; if (service->event_hld(accel_dev, ADF_EVENT_RESTARTING)) dev_err(&GET_DEV(accel_dev), "Failed to restart service %s.\n", @@ -450,17 +391,6 @@ int adf_dev_restarted_notify(struct adf_accel_dev *accel_dev) list_for_each(list_itr, &service_table) { service = list_entry(list_itr, struct service_hndl, list); - if (service->admin) - continue; - if (service->event_hld(accel_dev, ADF_EVENT_RESTARTED)) - dev_err(&GET_DEV(accel_dev), - "Failed to restart service %s.\n", - service->name); - } - list_for_each(list_itr, &service_table) { - service = list_entry(list_itr, struct service_hndl, list); - if (!service->admin) - continue; if (service->event_hld(accel_dev, ADF_EVENT_RESTARTED)) dev_err(&GET_DEV(accel_dev), "Failed to restart service %s.\n", diff --git a/kernel/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c b/kernel/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c new file mode 100644 index 000000000..5fdbad809 --- /dev/null +++ b/kernel/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c @@ -0,0 +1,438 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2015 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2015 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#include +#include +#include +#include "adf_accel_devices.h" +#include "adf_common_drv.h" +#include "adf_pf2vf_msg.h" + +#define ADF_DH895XCC_EP_OFFSET 0x3A000 +#define ADF_DH895XCC_ERRMSK3 (ADF_DH895XCC_EP_OFFSET + 0x1C) +#define ADF_DH895XCC_ERRMSK3_VF2PF_L_MASK(vf_mask) ((vf_mask & 0xFFFF) << 9) +#define ADF_DH895XCC_ERRMSK5 (ADF_DH895XCC_EP_OFFSET + 0xDC) +#define ADF_DH895XCC_ERRMSK5_VF2PF_U_MASK(vf_mask) (vf_mask >> 16) + +/** + * adf_enable_pf2vf_interrupts() - Enable PF to VF interrupts + * @accel_dev: Pointer to acceleration device. + * + * Function enables PF to VF interrupts + */ +void adf_enable_pf2vf_interrupts(struct adf_accel_dev *accel_dev) +{ + struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + void __iomem *pmisc_bar_addr = + pci_info->pci_bars[hw_data->get_misc_bar_id(hw_data)].virt_addr; + + ADF_CSR_WR(pmisc_bar_addr, hw_data->get_vintmsk_offset(0), 0x0); +} +EXPORT_SYMBOL_GPL(adf_enable_pf2vf_interrupts); + +/** + * adf_disable_pf2vf_interrupts() - Disable PF to VF interrupts + * @accel_dev: Pointer to acceleration device. + * + * Function disables PF to VF interrupts + */ +void adf_disable_pf2vf_interrupts(struct adf_accel_dev *accel_dev) +{ + struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + void __iomem *pmisc_bar_addr = + pci_info->pci_bars[hw_data->get_misc_bar_id(hw_data)].virt_addr; + + ADF_CSR_WR(pmisc_bar_addr, hw_data->get_vintmsk_offset(0), 0x2); +} +EXPORT_SYMBOL_GPL(adf_disable_pf2vf_interrupts); + +void adf_enable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, + u32 vf_mask) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_bar *pmisc = + &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)]; + void __iomem *pmisc_addr = pmisc->virt_addr; + u32 reg; + + /* Enable VF2PF Messaging Ints - VFs 1 through 16 per vf_mask[15:0] */ + if (vf_mask & 0xFFFF) { + reg = ADF_CSR_RD(pmisc_addr, ADF_DH895XCC_ERRMSK3); + reg &= ~ADF_DH895XCC_ERRMSK3_VF2PF_L_MASK(vf_mask); + ADF_CSR_WR(pmisc_addr, ADF_DH895XCC_ERRMSK3, reg); + } + + /* Enable VF2PF Messaging Ints - VFs 17 through 32 per vf_mask[31:16] */ + if (vf_mask >> 16) { + reg = ADF_CSR_RD(pmisc_addr, ADF_DH895XCC_ERRMSK5); + reg &= ~ADF_DH895XCC_ERRMSK5_VF2PF_U_MASK(vf_mask); + ADF_CSR_WR(pmisc_addr, ADF_DH895XCC_ERRMSK5, reg); + } +} + +/** + * adf_disable_pf2vf_interrupts() - Disable VF to PF interrupts + * @accel_dev: Pointer to acceleration device. + * + * Function disables VF to PF interrupts + */ +void adf_disable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 vf_mask) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_bar *pmisc = + &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)]; + void __iomem *pmisc_addr = pmisc->virt_addr; + u32 reg; + + /* Disable VF2PF interrupts for VFs 1 through 16 per vf_mask[15:0] */ + if (vf_mask & 0xFFFF) { + reg = ADF_CSR_RD(pmisc_addr, ADF_DH895XCC_ERRMSK3) | + ADF_DH895XCC_ERRMSK3_VF2PF_L_MASK(vf_mask); + ADF_CSR_WR(pmisc_addr, ADF_DH895XCC_ERRMSK3, reg); + } + + /* Disable VF2PF interrupts for VFs 17 through 32 per vf_mask[31:16] */ + if (vf_mask >> 16) { + reg = ADF_CSR_RD(pmisc_addr, ADF_DH895XCC_ERRMSK5) | + ADF_DH895XCC_ERRMSK5_VF2PF_U_MASK(vf_mask); + ADF_CSR_WR(pmisc_addr, ADF_DH895XCC_ERRMSK5, reg); + } +} +EXPORT_SYMBOL_GPL(adf_disable_vf2pf_interrupts); + +static int __adf_iov_putmsg(struct adf_accel_dev *accel_dev, u32 msg, u8 vf_nr) +{ + struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + void __iomem *pmisc_bar_addr = + pci_info->pci_bars[hw_data->get_misc_bar_id(hw_data)].virt_addr; + u32 val, pf2vf_offset, count = 0; + u32 local_in_use_mask, local_in_use_pattern; + u32 remote_in_use_mask, remote_in_use_pattern; + struct mutex *lock; /* lock preventing concurrent acces of CSR */ + u32 int_bit; + int ret = 0; + + if (accel_dev->is_vf) { + pf2vf_offset = hw_data->get_pf2vf_offset(0); + lock = &accel_dev->vf.vf2pf_lock; + local_in_use_mask = ADF_VF2PF_IN_USE_BY_VF_MASK; + local_in_use_pattern = ADF_VF2PF_IN_USE_BY_VF; + remote_in_use_mask = ADF_PF2VF_IN_USE_BY_PF_MASK; + remote_in_use_pattern = ADF_PF2VF_IN_USE_BY_PF; + int_bit = ADF_VF2PF_INT; + } else { + pf2vf_offset = hw_data->get_pf2vf_offset(vf_nr); + lock = &accel_dev->pf.vf_info[vf_nr].pf2vf_lock; + local_in_use_mask = ADF_PF2VF_IN_USE_BY_PF_MASK; + local_in_use_pattern = ADF_PF2VF_IN_USE_BY_PF; + remote_in_use_mask = ADF_VF2PF_IN_USE_BY_VF_MASK; + remote_in_use_pattern = ADF_VF2PF_IN_USE_BY_VF; + int_bit = ADF_PF2VF_INT; + } + + mutex_lock(lock); + + /* Check if PF2VF CSR is in use by remote function */ + val = ADF_CSR_RD(pmisc_bar_addr, pf2vf_offset); + if ((val & remote_in_use_mask) == remote_in_use_pattern) { + dev_dbg(&GET_DEV(accel_dev), + "PF2VF CSR in use by remote function\n"); + ret = -EBUSY; + goto out; + } + + /* Attempt to get ownership of PF2VF CSR */ + msg &= ~local_in_use_mask; + msg |= local_in_use_pattern; + ADF_CSR_WR(pmisc_bar_addr, pf2vf_offset, msg); + + /* Wait in case remote func also attempting to get ownership */ + msleep(ADF_IOV_MSG_COLLISION_DETECT_DELAY); + + val = ADF_CSR_RD(pmisc_bar_addr, pf2vf_offset); + if ((val & local_in_use_mask) != local_in_use_pattern) { + dev_dbg(&GET_DEV(accel_dev), + "PF2VF CSR in use by remote - collision detected\n"); + ret = -EBUSY; + goto out; + } + + /* + * This function now owns the PV2VF CSR. The IN_USE_BY pattern must + * remain in the PF2VF CSR for all writes including ACK from remote + * until this local function relinquishes the CSR. Send the message + * by interrupting the remote. + */ + ADF_CSR_WR(pmisc_bar_addr, pf2vf_offset, msg | int_bit); + + /* Wait for confirmation from remote func it received the message */ + do { + msleep(ADF_IOV_MSG_ACK_DELAY); + val = ADF_CSR_RD(pmisc_bar_addr, pf2vf_offset); + } while ((val & int_bit) && (count++ < ADF_IOV_MSG_ACK_MAX_RETRY)); + + if (val & int_bit) { + dev_dbg(&GET_DEV(accel_dev), "ACK not received from remote\n"); + val &= ~int_bit; + ret = -EIO; + } + + /* Finished with PF2VF CSR; relinquish it and leave msg in CSR */ + ADF_CSR_WR(pmisc_bar_addr, pf2vf_offset, val & ~local_in_use_mask); +out: + mutex_unlock(lock); + return ret; +} + +/** + * adf_iov_putmsg() - send PF2VF message + * @accel_dev: Pointer to acceleration device. + * @msg: Message to send + * @vf_nr: VF number to which the message will be sent + * + * Function sends a messge from the PF to a VF + * + * Return: 0 on success, error code otherwise. + */ +int adf_iov_putmsg(struct adf_accel_dev *accel_dev, u32 msg, u8 vf_nr) +{ + u32 count = 0; + int ret; + + do { + ret = __adf_iov_putmsg(accel_dev, msg, vf_nr); + if (ret) + msleep(ADF_IOV_MSG_RETRY_DELAY); + } while (ret && (count++ < ADF_IOV_MSG_MAX_RETRIES)); + + return ret; +} +EXPORT_SYMBOL_GPL(adf_iov_putmsg); + +void adf_vf2pf_req_hndl(struct adf_accel_vf_info *vf_info) +{ + struct adf_accel_dev *accel_dev = vf_info->accel_dev; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + int bar_id = hw_data->get_misc_bar_id(hw_data); + struct adf_bar *pmisc = &GET_BARS(accel_dev)[bar_id]; + void __iomem *pmisc_addr = pmisc->virt_addr; + u32 msg, resp = 0, vf_nr = vf_info->vf_nr; + + /* Read message from the VF */ + msg = ADF_CSR_RD(pmisc_addr, hw_data->get_pf2vf_offset(vf_nr)); + + /* To ACK, clear the VF2PFINT bit */ + msg &= ~ADF_VF2PF_INT; + ADF_CSR_WR(pmisc_addr, hw_data->get_pf2vf_offset(vf_nr), msg); + + if (!(msg & ADF_VF2PF_MSGORIGIN_SYSTEM)) + /* Ignore legacy non-system (non-kernel) VF2PF messages */ + goto err; + + switch ((msg & ADF_VF2PF_MSGTYPE_MASK) >> ADF_VF2PF_MSGTYPE_SHIFT) { + case ADF_VF2PF_MSGTYPE_COMPAT_VER_REQ: + { + u8 vf_compat_ver = msg >> ADF_VF2PF_COMPAT_VER_REQ_SHIFT; + + resp = (ADF_PF2VF_MSGORIGIN_SYSTEM | + (ADF_PF2VF_MSGTYPE_VERSION_RESP << + ADF_PF2VF_MSGTYPE_SHIFT) | + (ADF_PFVF_COMPATIBILITY_VERSION << + ADF_PF2VF_VERSION_RESP_VERS_SHIFT)); + + dev_dbg(&GET_DEV(accel_dev), + "Compatibility Version Request from VF%d vers=%u\n", + vf_nr + 1, vf_compat_ver); + + if (vf_compat_ver < hw_data->min_iov_compat_ver) { + dev_err(&GET_DEV(accel_dev), + "VF (vers %d) incompatible with PF (vers %d)\n", + vf_compat_ver, ADF_PFVF_COMPATIBILITY_VERSION); + resp |= ADF_PF2VF_VF_INCOMPATIBLE << + ADF_PF2VF_VERSION_RESP_RESULT_SHIFT; + } else if (vf_compat_ver > ADF_PFVF_COMPATIBILITY_VERSION) { + dev_err(&GET_DEV(accel_dev), + "VF (vers %d) compat with PF (vers %d) unkn.\n", + vf_compat_ver, ADF_PFVF_COMPATIBILITY_VERSION); + resp |= ADF_PF2VF_VF_COMPAT_UNKNOWN << + ADF_PF2VF_VERSION_RESP_RESULT_SHIFT; + } else { + dev_dbg(&GET_DEV(accel_dev), + "VF (vers %d) compatible with PF (vers %d)\n", + vf_compat_ver, ADF_PFVF_COMPATIBILITY_VERSION); + resp |= ADF_PF2VF_VF_COMPATIBLE << + ADF_PF2VF_VERSION_RESP_RESULT_SHIFT; + } + } + break; + case ADF_VF2PF_MSGTYPE_VERSION_REQ: + dev_dbg(&GET_DEV(accel_dev), + "Legacy VersionRequest received from VF%d 0x%x\n", + vf_nr + 1, msg); + resp = (ADF_PF2VF_MSGORIGIN_SYSTEM | + (ADF_PF2VF_MSGTYPE_VERSION_RESP << + ADF_PF2VF_MSGTYPE_SHIFT) | + (ADF_PFVF_COMPATIBILITY_VERSION << + ADF_PF2VF_VERSION_RESP_VERS_SHIFT)); + resp |= ADF_PF2VF_VF_COMPATIBLE << + ADF_PF2VF_VERSION_RESP_RESULT_SHIFT; + /* Set legacy major and minor version num */ + resp |= 1 << ADF_PF2VF_MAJORVERSION_SHIFT | + 1 << ADF_PF2VF_MINORVERSION_SHIFT; + break; + case ADF_VF2PF_MSGTYPE_INIT: + { + dev_dbg(&GET_DEV(accel_dev), + "Init message received from VF%d 0x%x\n", + vf_nr + 1, msg); + vf_info->init = true; + } + break; + case ADF_VF2PF_MSGTYPE_SHUTDOWN: + { + dev_dbg(&GET_DEV(accel_dev), + "Shutdown message received from VF%d 0x%x\n", + vf_nr + 1, msg); + vf_info->init = false; + } + break; + default: + goto err; + } + + if (resp && adf_iov_putmsg(accel_dev, resp, vf_nr)) + dev_err(&GET_DEV(accel_dev), "Failed to send response to VF\n"); + + /* re-enable interrupt on PF from this VF */ + adf_enable_vf2pf_interrupts(accel_dev, (1 << vf_nr)); + return; +err: + dev_dbg(&GET_DEV(accel_dev), "Unknown message from VF%d (0x%x);\n", + vf_nr + 1, msg); +} + +void adf_pf2vf_notify_restarting(struct adf_accel_dev *accel_dev) +{ + struct adf_accel_vf_info *vf; + u32 msg = (ADF_PF2VF_MSGORIGIN_SYSTEM | + (ADF_PF2VF_MSGTYPE_RESTARTING << ADF_PF2VF_MSGTYPE_SHIFT)); + int i, num_vfs = pci_num_vf(accel_to_pci_dev(accel_dev)); + + for (i = 0, vf = accel_dev->pf.vf_info; i < num_vfs; i++, vf++) { + if (vf->init && adf_iov_putmsg(accel_dev, msg, i)) + dev_err(&GET_DEV(accel_dev), + "Failed to send restarting msg to VF%d\n", i); + } +} + +static int adf_vf2pf_request_version(struct adf_accel_dev *accel_dev) +{ + unsigned long timeout = msecs_to_jiffies(ADF_IOV_MSG_RESP_TIMEOUT); + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + u32 msg = 0; + int ret; + + msg = ADF_VF2PF_MSGORIGIN_SYSTEM; + msg |= ADF_VF2PF_MSGTYPE_COMPAT_VER_REQ << ADF_VF2PF_MSGTYPE_SHIFT; + msg |= ADF_PFVF_COMPATIBILITY_VERSION << ADF_VF2PF_COMPAT_VER_REQ_SHIFT; + BUILD_BUG_ON(ADF_PFVF_COMPATIBILITY_VERSION > 255); + + /* Send request from VF to PF */ + ret = adf_iov_putmsg(accel_dev, msg, 0); + if (ret) { + dev_err(&GET_DEV(accel_dev), + "Failed to send Compatibility Version Request.\n"); + return ret; + } + + /* Wait for response */ + if (!wait_for_completion_timeout(&accel_dev->vf.iov_msg_completion, + timeout)) { + dev_err(&GET_DEV(accel_dev), + "IOV request/response message timeout expired\n"); + return -EIO; + } + + /* Response from PF received, check compatibility */ + switch (accel_dev->vf.compatible) { + case ADF_PF2VF_VF_COMPATIBLE: + break; + case ADF_PF2VF_VF_COMPAT_UNKNOWN: + /* VF is newer than PF and decides whether it is compatible */ + if (accel_dev->vf.pf_version >= hw_data->min_iov_compat_ver) + break; + /* fall through */ + case ADF_PF2VF_VF_INCOMPATIBLE: + dev_err(&GET_DEV(accel_dev), + "PF (vers %d) and VF (vers %d) are not compatible\n", + accel_dev->vf.pf_version, + ADF_PFVF_COMPATIBILITY_VERSION); + return -EINVAL; + default: + dev_err(&GET_DEV(accel_dev), + "Invalid response from PF; assume not compatible\n"); + return -EINVAL; + } + return ret; +} + +/** + * adf_enable_vf2pf_comms() - Function enables communication from vf to pf + * + * @accel_dev: Pointer to acceleration device virtual function. + * + * Return: 0 on success, error code otherwise. + */ +int adf_enable_vf2pf_comms(struct adf_accel_dev *accel_dev) +{ + adf_enable_pf2vf_interrupts(accel_dev); + return adf_vf2pf_request_version(accel_dev); +} +EXPORT_SYMBOL_GPL(adf_enable_vf2pf_comms); diff --git a/kernel/drivers/crypto/qat/qat_common/adf_pf2vf_msg.h b/kernel/drivers/crypto/qat/qat_common/adf_pf2vf_msg.h new file mode 100644 index 000000000..5acd531a1 --- /dev/null +++ b/kernel/drivers/crypto/qat/qat_common/adf_pf2vf_msg.h @@ -0,0 +1,146 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2015 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2015 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef ADF_PF2VF_MSG_H +#define ADF_PF2VF_MSG_H + +/* + * PF<->VF Messaging + * The PF has an array of 32-bit PF2VF registers, one for each VF. The + * PF can access all these registers; each VF can access only the one + * register associated with that particular VF. + * + * The register functionally is split into two parts: + * The bottom half is for PF->VF messages. In particular when the first + * bit of this register (bit 0) gets set an interrupt will be triggered + * in the respective VF. + * The top half is for VF->PF messages. In particular when the first bit + * of this half of register (bit 16) gets set an interrupt will be triggered + * in the PF. + * + * The remaining bits within this register are available to encode messages. + * and implement a collision control mechanism to prevent concurrent use of + * the PF2VF register by both the PF and VF. + * + * 31 30 29 28 27 26 25 24 23 22 21 20 19 18 17 16 + * _______________________________________________ + * | | | | | | | | | | | | | | | | | + * +-----------------------------------------------+ + * \___________________________/ \_________/ ^ ^ + * ^ ^ | | + * | | | VF2PF Int + * | | Message Origin + * | Message Type + * Message-specific Data/Reserved + * + * 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0 + * _______________________________________________ + * | | | | | | | | | | | | | | | | | + * +-----------------------------------------------+ + * \___________________________/ \_________/ ^ ^ + * ^ ^ | | + * | | | PF2VF Int + * | | Message Origin + * | Message Type + * Message-specific Data/Reserved + * + * Message Origin (Should always be 1) + * A legacy out-of-tree QAT driver allowed for a set of messages not supported + * by this driver; these had a Msg Origin of 0 and are ignored by this driver. + * + * When a PF or VF attempts to send a message in the lower or upper 16 bits, + * respectively, the other 16 bits are written to first with a defined + * IN_USE_BY pattern as part of a collision control scheme (see adf_iov_putmsg). + */ + +#define ADF_PFVF_COMPATIBILITY_VERSION 0x1 /* PF<->VF compat */ + +/* PF->VF messages */ +#define ADF_PF2VF_INT BIT(0) +#define ADF_PF2VF_MSGORIGIN_SYSTEM BIT(1) +#define ADF_PF2VF_MSGTYPE_MASK 0x0000003C +#define ADF_PF2VF_MSGTYPE_SHIFT 2 +#define ADF_PF2VF_MSGTYPE_RESTARTING 0x01 +#define ADF_PF2VF_MSGTYPE_VERSION_RESP 0x02 +#define ADF_PF2VF_IN_USE_BY_PF 0x6AC20000 +#define ADF_PF2VF_IN_USE_BY_PF_MASK 0xFFFE0000 + +/* PF->VF Version Response */ +#define ADF_PF2VF_VERSION_RESP_VERS_MASK 0x00003FC0 +#define ADF_PF2VF_VERSION_RESP_VERS_SHIFT 6 +#define ADF_PF2VF_VERSION_RESP_RESULT_MASK 0x0000C000 +#define ADF_PF2VF_VERSION_RESP_RESULT_SHIFT 14 +#define ADF_PF2VF_MINORVERSION_SHIFT 6 +#define ADF_PF2VF_MAJORVERSION_SHIFT 10 +#define ADF_PF2VF_VF_COMPATIBLE 1 +#define ADF_PF2VF_VF_INCOMPATIBLE 2 +#define ADF_PF2VF_VF_COMPAT_UNKNOWN 3 + +/* VF->PF messages */ +#define ADF_VF2PF_IN_USE_BY_VF 0x00006AC2 +#define ADF_VF2PF_IN_USE_BY_VF_MASK 0x0000FFFE +#define ADF_VF2PF_INT BIT(16) +#define ADF_VF2PF_MSGORIGIN_SYSTEM BIT(17) +#define ADF_VF2PF_MSGTYPE_MASK 0x003C0000 +#define ADF_VF2PF_MSGTYPE_SHIFT 18 +#define ADF_VF2PF_MSGTYPE_INIT 0x3 +#define ADF_VF2PF_MSGTYPE_SHUTDOWN 0x4 +#define ADF_VF2PF_MSGTYPE_VERSION_REQ 0x5 +#define ADF_VF2PF_MSGTYPE_COMPAT_VER_REQ 0x6 + +/* VF->PF Compatible Version Request */ +#define ADF_VF2PF_COMPAT_VER_REQ_SHIFT 22 + +/* Collision detection */ +#define ADF_IOV_MSG_COLLISION_DETECT_DELAY 10 +#define ADF_IOV_MSG_ACK_DELAY 2 +#define ADF_IOV_MSG_ACK_MAX_RETRY 100 +#define ADF_IOV_MSG_RETRY_DELAY 5 +#define ADF_IOV_MSG_MAX_RETRIES 3 +#define ADF_IOV_MSG_RESP_TIMEOUT (ADF_IOV_MSG_ACK_DELAY * \ + ADF_IOV_MSG_ACK_MAX_RETRY + \ + ADF_IOV_MSG_COLLISION_DETECT_DELAY) +#endif /* ADF_IOV_MSG_H */ diff --git a/kernel/drivers/crypto/qat/qat_common/adf_sriov.c b/kernel/drivers/crypto/qat/qat_common/adf_sriov.c new file mode 100644 index 000000000..1117a8b58 --- /dev/null +++ b/kernel/drivers/crypto/qat/qat_common/adf_sriov.c @@ -0,0 +1,306 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2015 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2015 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#include +#include +#include +#include +#include "adf_common_drv.h" +#include "adf_cfg.h" +#include "adf_pf2vf_msg.h" + +static struct workqueue_struct *pf2vf_resp_wq; + +#define ME2FUNCTION_MAP_A_OFFSET (0x3A400 + 0x190) +#define ME2FUNCTION_MAP_A_NUM_REGS 96 + +#define ME2FUNCTION_MAP_B_OFFSET (0x3A400 + 0x310) +#define ME2FUNCTION_MAP_B_NUM_REGS 12 + +#define ME2FUNCTION_MAP_REG_SIZE 4 +#define ME2FUNCTION_MAP_VALID BIT(7) + +#define READ_CSR_ME2FUNCTION_MAP_A(pmisc_bar_addr, index) \ + ADF_CSR_RD(pmisc_bar_addr, ME2FUNCTION_MAP_A_OFFSET + \ + ME2FUNCTION_MAP_REG_SIZE * index) + +#define WRITE_CSR_ME2FUNCTION_MAP_A(pmisc_bar_addr, index, value) \ + ADF_CSR_WR(pmisc_bar_addr, ME2FUNCTION_MAP_A_OFFSET + \ + ME2FUNCTION_MAP_REG_SIZE * index, value) + +#define READ_CSR_ME2FUNCTION_MAP_B(pmisc_bar_addr, index) \ + ADF_CSR_RD(pmisc_bar_addr, ME2FUNCTION_MAP_B_OFFSET + \ + ME2FUNCTION_MAP_REG_SIZE * index) + +#define WRITE_CSR_ME2FUNCTION_MAP_B(pmisc_bar_addr, index, value) \ + ADF_CSR_WR(pmisc_bar_addr, ME2FUNCTION_MAP_B_OFFSET + \ + ME2FUNCTION_MAP_REG_SIZE * index, value) + +struct adf_pf2vf_resp { + struct work_struct pf2vf_resp_work; + struct adf_accel_vf_info *vf_info; +}; + +static void adf_iov_send_resp(struct work_struct *work) +{ + struct adf_pf2vf_resp *pf2vf_resp = + container_of(work, struct adf_pf2vf_resp, pf2vf_resp_work); + + adf_vf2pf_req_hndl(pf2vf_resp->vf_info); + kfree(pf2vf_resp); +} + +static void adf_vf2pf_bh_handler(void *data) +{ + struct adf_accel_vf_info *vf_info = (struct adf_accel_vf_info *)data; + struct adf_pf2vf_resp *pf2vf_resp; + + pf2vf_resp = kzalloc(sizeof(*pf2vf_resp), GFP_ATOMIC); + if (!pf2vf_resp) + return; + + pf2vf_resp->vf_info = vf_info; + INIT_WORK(&pf2vf_resp->pf2vf_resp_work, adf_iov_send_resp); + queue_work(pf2vf_resp_wq, &pf2vf_resp->pf2vf_resp_work); +} + +static int adf_enable_sriov(struct adf_accel_dev *accel_dev) +{ + struct pci_dev *pdev = accel_to_pci_dev(accel_dev); + int totalvfs = pci_sriov_get_totalvfs(pdev); + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_bar *pmisc = + &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)]; + void __iomem *pmisc_addr = pmisc->virt_addr; + struct adf_accel_vf_info *vf_info; + int i; + u32 reg; + + /* Workqueue for PF2VF responses */ + pf2vf_resp_wq = create_workqueue("qat_pf2vf_resp_wq"); + if (!pf2vf_resp_wq) + return -ENOMEM; + + for (i = 0, vf_info = accel_dev->pf.vf_info; i < totalvfs; + i++, vf_info++) { + /* This ptr will be populated when VFs will be created */ + vf_info->accel_dev = accel_dev; + vf_info->vf_nr = i; + + tasklet_init(&vf_info->vf2pf_bh_tasklet, + (void *)adf_vf2pf_bh_handler, + (unsigned long)vf_info); + mutex_init(&vf_info->pf2vf_lock); + ratelimit_state_init(&vf_info->vf2pf_ratelimit, + DEFAULT_RATELIMIT_INTERVAL, + DEFAULT_RATELIMIT_BURST); + } + + /* Set Valid bits in ME Thread to PCIe Function Mapping Group A */ + for (i = 0; i < ME2FUNCTION_MAP_A_NUM_REGS; i++) { + reg = READ_CSR_ME2FUNCTION_MAP_A(pmisc_addr, i); + reg |= ME2FUNCTION_MAP_VALID; + WRITE_CSR_ME2FUNCTION_MAP_A(pmisc_addr, i, reg); + } + + /* Set Valid bits in ME Thread to PCIe Function Mapping Group B */ + for (i = 0; i < ME2FUNCTION_MAP_B_NUM_REGS; i++) { + reg = READ_CSR_ME2FUNCTION_MAP_B(pmisc_addr, i); + reg |= ME2FUNCTION_MAP_VALID; + WRITE_CSR_ME2FUNCTION_MAP_B(pmisc_addr, i, reg); + } + + /* Enable VF to PF interrupts for all VFs */ + adf_enable_vf2pf_interrupts(accel_dev, GENMASK_ULL(totalvfs - 1, 0)); + + /* + * Due to the hardware design, when SR-IOV and the ring arbiter + * are enabled all the VFs supported in hardware must be enabled in + * order for all the hardware resources (i.e. bundles) to be usable. + * When SR-IOV is enabled, each of the VFs will own one bundle. + */ + return pci_enable_sriov(pdev, totalvfs); +} + +/** + * adf_disable_sriov() - Disable SRIOV for the device + * @pdev: Pointer to pci device. + * + * Function disables SRIOV for the pci device. + * + * Return: 0 on success, error code otherwise. + */ +void adf_disable_sriov(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_bar *pmisc = + &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)]; + void __iomem *pmisc_addr = pmisc->virt_addr; + int totalvfs = pci_sriov_get_totalvfs(accel_to_pci_dev(accel_dev)); + struct adf_accel_vf_info *vf; + u32 reg; + int i; + + if (!accel_dev->pf.vf_info) + return; + + adf_pf2vf_notify_restarting(accel_dev); + + pci_disable_sriov(accel_to_pci_dev(accel_dev)); + + /* Disable VF to PF interrupts */ + adf_disable_vf2pf_interrupts(accel_dev, 0xFFFFFFFF); + + /* Clear Valid bits in ME Thread to PCIe Function Mapping Group A */ + for (i = 0; i < ME2FUNCTION_MAP_A_NUM_REGS; i++) { + reg = READ_CSR_ME2FUNCTION_MAP_A(pmisc_addr, i); + reg &= ~ME2FUNCTION_MAP_VALID; + WRITE_CSR_ME2FUNCTION_MAP_A(pmisc_addr, i, reg); + } + + /* Clear Valid bits in ME Thread to PCIe Function Mapping Group B */ + for (i = 0; i < ME2FUNCTION_MAP_B_NUM_REGS; i++) { + reg = READ_CSR_ME2FUNCTION_MAP_B(pmisc_addr, i); + reg &= ~ME2FUNCTION_MAP_VALID; + WRITE_CSR_ME2FUNCTION_MAP_B(pmisc_addr, i, reg); + } + + for (i = 0, vf = accel_dev->pf.vf_info; i < totalvfs; i++, vf++) { + tasklet_disable(&vf->vf2pf_bh_tasklet); + tasklet_kill(&vf->vf2pf_bh_tasklet); + mutex_destroy(&vf->pf2vf_lock); + } + + kfree(accel_dev->pf.vf_info); + accel_dev->pf.vf_info = NULL; + + if (pf2vf_resp_wq) { + destroy_workqueue(pf2vf_resp_wq); + pf2vf_resp_wq = NULL; + } +} +EXPORT_SYMBOL_GPL(adf_disable_sriov); + +/** + * adf_sriov_configure() - Enable SRIOV for the device + * @pdev: Pointer to pci device. + * + * Function enables SRIOV for the pci device. + * + * Return: 0 on success, error code otherwise. + */ +int adf_sriov_configure(struct pci_dev *pdev, int numvfs) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + int totalvfs = pci_sriov_get_totalvfs(pdev); + unsigned long val; + int ret; + + if (!accel_dev) { + dev_err(&pdev->dev, "Failed to find accel_dev\n"); + return -EFAULT; + } + + if (!iommu_present(&pci_bus_type)) + dev_warn(&pdev->dev, "IOMMU should be enabled for SR-IOV to work correctly\n"); + + if (accel_dev->pf.vf_info) { + dev_info(&pdev->dev, "Already enabled for this device\n"); + return -EINVAL; + } + + if (adf_dev_started(accel_dev)) { + if (adf_devmgr_in_reset(accel_dev) || + adf_dev_in_use(accel_dev)) { + dev_err(&GET_DEV(accel_dev), "Device busy\n"); + return -EBUSY; + } + + if (adf_dev_stop(accel_dev)) { + dev_err(&GET_DEV(accel_dev), + "Failed to stop qat_dev%d\n", + accel_dev->accel_id); + return -EFAULT; + } + + adf_dev_shutdown(accel_dev); + } + + if (adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC)) + return -EFAULT; + val = 0; + if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + ADF_NUM_CY, (void *)&val, ADF_DEC)) + return -EFAULT; + + set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); + + /* Allocate memory for VF info structs */ + accel_dev->pf.vf_info = kcalloc(totalvfs, + sizeof(struct adf_accel_vf_info), + GFP_KERNEL); + if (!accel_dev->pf.vf_info) + return -ENOMEM; + + if (adf_dev_init(accel_dev)) { + dev_err(&GET_DEV(accel_dev), "Failed to init qat_dev%d\n", + accel_dev->accel_id); + return -EFAULT; + } + + if (adf_dev_start(accel_dev)) { + dev_err(&GET_DEV(accel_dev), "Failed to start qat_dev%d\n", + accel_dev->accel_id); + return -EFAULT; + } + + ret = adf_enable_sriov(accel_dev); + if (ret) + return ret; + + return numvfs; +} +EXPORT_SYMBOL_GPL(adf_sriov_configure); diff --git a/kernel/drivers/crypto/qat/qat_common/adf_transport.c b/kernel/drivers/crypto/qat/qat_common/adf_transport.c index ccec32748..3865ae8d9 100644 --- a/kernel/drivers/crypto/qat/qat_common/adf_transport.c +++ b/kernel/drivers/crypto/qat/qat_common/adf_transport.c @@ -264,6 +264,10 @@ int adf_create_ring(struct adf_accel_dev *accel_dev, const char *section, dev_err(&GET_DEV(accel_dev), "Can't get ring number\n"); return -EFAULT; } + if (ring_num >= ADF_ETR_MAX_RINGS_PER_BANK) { + dev_err(&GET_DEV(accel_dev), "Invalid ring number\n"); + return -EFAULT; + } bank = &transport_data->banks[bank_num]; if (adf_reserve_ring(bank, ring_num)) { @@ -285,7 +289,7 @@ int adf_create_ring(struct adf_accel_dev *accel_dev, const char *section, goto err; /* Enable HW arbitration for the given ring */ - accel_dev->hw_device->hw_arb_ring_enable(ring); + adf_update_ring_arb(ring); if (adf_ring_debugfs_add(ring, ring_name)) { dev_err(&GET_DEV(accel_dev), @@ -302,14 +306,13 @@ int adf_create_ring(struct adf_accel_dev *accel_dev, const char *section, err: adf_cleanup_ring(ring); adf_unreserve_ring(bank, ring_num); - accel_dev->hw_device->hw_arb_ring_disable(ring); + adf_update_ring_arb(ring); return ret; } void adf_remove_ring(struct adf_etr_ring_data *ring) { struct adf_etr_bank_data *bank = ring->bank; - struct adf_accel_dev *accel_dev = bank->accel_dev; /* Disable interrupts for the given ring */ adf_disable_ring_irq(bank, ring->ring_number); @@ -322,7 +325,7 @@ void adf_remove_ring(struct adf_etr_ring_data *ring) adf_ring_debugfs_rm(ring); adf_unreserve_ring(bank, ring->ring_number); /* Disable HW arbitration for the given ring */ - accel_dev->hw_device->hw_arb_ring_disable(ring); + adf_update_ring_arb(ring); adf_cleanup_ring(ring); } @@ -449,7 +452,7 @@ static int adf_init_bank(struct adf_accel_dev *accel_dev, err: for (i = 0; i < ADF_ETR_MAX_RINGS_PER_BANK; i++) { ring = &bank->rings[i]; - if (hw_data->tx_rings_mask & (1 << i) && ring->inflights) + if (hw_data->tx_rings_mask & (1 << i)) kfree(ring->inflights); } return -ENOMEM; @@ -463,7 +466,7 @@ err: * acceleration device accel_dev. * To be used by QAT device specific drivers. * - * Return: 0 on success, error code othewise. + * Return: 0 on success, error code otherwise. */ int adf_init_etr_data(struct adf_accel_dev *accel_dev) { diff --git a/kernel/drivers/crypto/qat/qat_common/adf_transport_access_macros.h b/kernel/drivers/crypto/qat/qat_common/adf_transport_access_macros.h index 160c9a36c..6ad7e4e1e 100644 --- a/kernel/drivers/crypto/qat/qat_common/adf_transport_access_macros.h +++ b/kernel/drivers/crypto/qat/qat_common/adf_transport_access_macros.h @@ -97,8 +97,9 @@ #define ADF_RING_SIZE_IN_BYTES_TO_SIZE(SIZE) ((1 << (SIZE - 1)) >> 7) /* Minimum ring bufer size for memory allocation */ -#define ADF_RING_SIZE_BYTES_MIN(SIZE) ((SIZE < ADF_RING_SIZE_4K) ? \ - ADF_RING_SIZE_4K : SIZE) +#define ADF_RING_SIZE_BYTES_MIN(SIZE) \ + ((SIZE < ADF_SIZE_TO_RING_SIZE_IN_BYTES(ADF_RING_SIZE_4K)) ? \ + ADF_SIZE_TO_RING_SIZE_IN_BYTES(ADF_RING_SIZE_4K) : SIZE) #define ADF_RING_SIZE_MODULO(SIZE) (SIZE + 0x6) #define ADF_SIZE_TO_POW(SIZE) ((((SIZE & 0x4) >> 1) | ((SIZE & 0x4) >> 2) | \ SIZE) & ~0x4) diff --git a/kernel/drivers/crypto/qat/qat_common/adf_transport_debug.c b/kernel/drivers/crypto/qat/qat_common/adf_transport_debug.c index e41986967..52340b9bb 100644 --- a/kernel/drivers/crypto/qat/qat_common/adf_transport_debug.c +++ b/kernel/drivers/crypto/qat/qat_common/adf_transport_debug.c @@ -86,9 +86,7 @@ static int adf_ring_show(struct seq_file *sfile, void *v) { struct adf_etr_ring_data *ring = sfile->private; struct adf_etr_bank_data *bank = ring->bank; - uint32_t *msg = v; void __iomem *csr = ring->bank->csr_addr; - int i, x; if (v == SEQ_START_TOKEN) { int head, tail, empty; @@ -113,18 +111,8 @@ static int adf_ring_show(struct seq_file *sfile, void *v) seq_puts(sfile, "----------- Ring data ------------\n"); return 0; } - seq_printf(sfile, "%p:", msg); - x = 0; - i = 0; - for (; i < (ADF_MSG_SIZE_TO_BYTES(ring->msg_size) >> 2); i++) { - seq_printf(sfile, " %08X", *(msg + i)); - if ((ADF_MSG_SIZE_TO_BYTES(ring->msg_size) >> 2) != i + 1 && - (++x == 8)) { - seq_printf(sfile, "\n%p:", msg + i + 1); - x = 0; - } - } - seq_puts(sfile, "\n"); + seq_hex_dump(sfile, "", DUMP_PREFIX_ADDRESS, 32, 4, + v, ADF_MSG_SIZE_TO_BYTES(ring->msg_size), false); return 0; } diff --git a/kernel/drivers/crypto/qat/qat_common/icp_qat_fw.h b/kernel/drivers/crypto/qat/qat_common/icp_qat_fw.h index f1e30e24a..46747f01b 100644 --- a/kernel/drivers/crypto/qat/qat_common/icp_qat_fw.h +++ b/kernel/drivers/crypto/qat/qat_common/icp_qat_fw.h @@ -249,6 +249,8 @@ struct icp_qat_fw_comn_resp { #define QAT_COMN_RESP_CRYPTO_STATUS_BITPOS 7 #define QAT_COMN_RESP_CRYPTO_STATUS_MASK 0x1 +#define QAT_COMN_RESP_PKE_STATUS_BITPOS 6 +#define QAT_COMN_RESP_PKE_STATUS_MASK 0x1 #define QAT_COMN_RESP_CMP_STATUS_BITPOS 5 #define QAT_COMN_RESP_CMP_STATUS_MASK 0x1 #define QAT_COMN_RESP_XLAT_STATUS_BITPOS 4 diff --git a/kernel/drivers/crypto/qat/qat_common/icp_qat_fw_pke.h b/kernel/drivers/crypto/qat/qat_common/icp_qat_fw_pke.h new file mode 100644 index 000000000..0d7a9b51c --- /dev/null +++ b/kernel/drivers/crypto/qat/qat_common/icp_qat_fw_pke.h @@ -0,0 +1,112 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef _ICP_QAT_FW_PKE_ +#define _ICP_QAT_FW_PKE_ + +#include "icp_qat_fw.h" + +struct icp_qat_fw_req_hdr_pke_cd_pars { + u64 content_desc_addr; + u32 content_desc_resrvd; + u32 func_id; +}; + +struct icp_qat_fw_req_pke_mid { + u64 opaque; + u64 src_data_addr; + u64 dest_data_addr; +}; + +struct icp_qat_fw_req_pke_hdr { + u8 resrvd1; + u8 resrvd2; + u8 service_type; + u8 hdr_flags; + u16 comn_req_flags; + u16 resrvd4; + struct icp_qat_fw_req_hdr_pke_cd_pars cd_pars; +}; + +struct icp_qat_fw_pke_request { + struct icp_qat_fw_req_pke_hdr pke_hdr; + struct icp_qat_fw_req_pke_mid pke_mid; + u8 output_param_count; + u8 input_param_count; + u16 resrvd1; + u32 resrvd2; + u64 next_req_adr; +}; + +struct icp_qat_fw_resp_pke_hdr { + u8 resrvd1; + u8 resrvd2; + u8 response_type; + u8 hdr_flags; + u16 comn_resp_flags; + u16 resrvd4; +}; + +struct icp_qat_fw_pke_resp { + struct icp_qat_fw_resp_pke_hdr pke_resp_hdr; + u64 opaque; + u64 src_data_addr; + u64 dest_data_addr; +}; + +#define ICP_QAT_FW_PKE_HDR_VALID_FLAG_BITPOS 7 +#define ICP_QAT_FW_PKE_HDR_VALID_FLAG_MASK 0x1 +#define ICP_QAT_FW_PKE_RESP_PKE_STAT_GET(status_word) \ + QAT_FIELD_GET(((status_word >> ICP_QAT_FW_COMN_ONE_BYTE_SHIFT) & \ + ICP_QAT_FW_COMN_SINGLE_BYTE_MASK), \ + QAT_COMN_RESP_PKE_STATUS_BITPOS, \ + QAT_COMN_RESP_PKE_STATUS_MASK) + +#define ICP_QAT_FW_PKE_HDR_VALID_FLAG_SET(hdr_t, val) \ + QAT_FIELD_SET((hdr_t.hdr_flags), (val), \ + ICP_QAT_FW_PKE_HDR_VALID_FLAG_BITPOS, \ + ICP_QAT_FW_PKE_HDR_VALID_FLAG_MASK) +#endif diff --git a/kernel/drivers/crypto/qat/qat_common/qat_algs.c b/kernel/drivers/crypto/qat/qat_common/qat_algs.c index 34139a889..59e4c3af1 100644 --- a/kernel/drivers/crypto/qat/qat_common/qat_algs.c +++ b/kernel/drivers/crypto/qat/qat_common/qat_algs.c @@ -47,13 +47,12 @@ #include #include #include -#include +#include #include #include #include #include #include -#include #include #include "adf_accel_devices.h" #include "adf_transport.h" @@ -63,13 +62,13 @@ #include "icp_qat_fw.h" #include "icp_qat_fw_la.h" -#define QAT_AES_HW_CONFIG_CBC_ENC(alg) \ - ICP_QAT_HW_CIPHER_CONFIG_BUILD(ICP_QAT_HW_CIPHER_CBC_MODE, alg, \ +#define QAT_AES_HW_CONFIG_ENC(alg, mode) \ + ICP_QAT_HW_CIPHER_CONFIG_BUILD(mode, alg, \ ICP_QAT_HW_CIPHER_NO_CONVERT, \ ICP_QAT_HW_CIPHER_ENCRYPT) -#define QAT_AES_HW_CONFIG_CBC_DEC(alg) \ - ICP_QAT_HW_CIPHER_CONFIG_BUILD(ICP_QAT_HW_CIPHER_CBC_MODE, alg, \ +#define QAT_AES_HW_CONFIG_DEC(alg, mode) \ + ICP_QAT_HW_CIPHER_CONFIG_BUILD(mode, alg, \ ICP_QAT_HW_CIPHER_KEY_CONVERT, \ ICP_QAT_HW_CIPHER_DECRYPT) @@ -113,9 +112,6 @@ struct qat_alg_aead_ctx { struct crypto_shash *hash_tfm; enum icp_qat_hw_auth_algo qat_hash_alg; struct qat_crypto_instance *inst; - struct crypto_tfm *tfm; - uint8_t salt[AES_BLOCK_SIZE]; - spinlock_t lock; /* protects qat_alg_aead_ctx struct */ }; struct qat_alg_ablkcipher_ctx { @@ -130,11 +126,6 @@ struct qat_alg_ablkcipher_ctx { spinlock_t lock; /* protects qat_alg_ablkcipher_ctx struct */ }; -static int get_current_node(void) -{ - return cpu_data(current_thread_info()->cpu).phys_proc_id; -} - static int qat_get_inter_state_size(enum icp_qat_hw_auth_algo qat_hash_alg) { switch (qat_hash_alg) { @@ -278,12 +269,13 @@ static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header) ICP_QAT_FW_LA_NO_UPDATE_STATE); } -static int qat_alg_aead_init_enc_session(struct qat_alg_aead_ctx *ctx, +static int qat_alg_aead_init_enc_session(struct crypto_aead *aead_tfm, int alg, - struct crypto_authenc_keys *keys) + struct crypto_authenc_keys *keys, + int mode) { - struct crypto_aead *aead_tfm = __crypto_aead_cast(ctx->tfm); - unsigned int digestsize = crypto_aead_crt(aead_tfm)->authsize; + struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(aead_tfm); + unsigned int digestsize = crypto_aead_authsize(aead_tfm); struct qat_enc *enc_ctx = &ctx->enc_cd->qat_enc_cd; struct icp_qat_hw_cipher_algo_blk *cipher = &enc_ctx->cipher; struct icp_qat_hw_auth_algo_blk *hash = @@ -297,7 +289,7 @@ static int qat_alg_aead_init_enc_session(struct qat_alg_aead_ctx *ctx, struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr; /* CD setup */ - cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_CBC_ENC(alg); + cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_ENC(alg, mode); memcpy(cipher->aes.key, keys->enckey, keys->enckeylen); hash->sha.inner_setup.auth_config.config = ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1, @@ -358,12 +350,13 @@ static int qat_alg_aead_init_enc_session(struct qat_alg_aead_ctx *ctx, return 0; } -static int qat_alg_aead_init_dec_session(struct qat_alg_aead_ctx *ctx, +static int qat_alg_aead_init_dec_session(struct crypto_aead *aead_tfm, int alg, - struct crypto_authenc_keys *keys) + struct crypto_authenc_keys *keys, + int mode) { - struct crypto_aead *aead_tfm = __crypto_aead_cast(ctx->tfm); - unsigned int digestsize = crypto_aead_crt(aead_tfm)->authsize; + struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(aead_tfm); + unsigned int digestsize = crypto_aead_authsize(aead_tfm); struct qat_dec *dec_ctx = &ctx->dec_cd->qat_dec_cd; struct icp_qat_hw_auth_algo_blk *hash = &dec_ctx->hash; struct icp_qat_hw_cipher_algo_blk *cipher = @@ -382,7 +375,7 @@ static int qat_alg_aead_init_dec_session(struct qat_alg_aead_ctx *ctx, sizeof(struct icp_qat_fw_la_cipher_req_params)); /* CD setup */ - cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_CBC_DEC(alg); + cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_DEC(alg, mode); memcpy(cipher->aes.key, keys->enckey, keys->enckeylen); hash->sha.inner_setup.auth_config.config = ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1, @@ -473,7 +466,7 @@ static void qat_alg_ablkcipher_init_com(struct qat_alg_ablkcipher_ctx *ctx, static void qat_alg_ablkcipher_init_enc(struct qat_alg_ablkcipher_ctx *ctx, int alg, const uint8_t *key, - unsigned int keylen) + unsigned int keylen, int mode) { struct icp_qat_hw_cipher_algo_blk *enc_cd = ctx->enc_cd; struct icp_qat_fw_la_bulk_req *req = &ctx->enc_fw_req; @@ -481,12 +474,12 @@ static void qat_alg_ablkcipher_init_enc(struct qat_alg_ablkcipher_ctx *ctx, qat_alg_ablkcipher_init_com(ctx, req, enc_cd, key, keylen); cd_pars->u.s.content_desc_addr = ctx->enc_cd_paddr; - enc_cd->aes.cipher_config.val = QAT_AES_HW_CONFIG_CBC_ENC(alg); + enc_cd->aes.cipher_config.val = QAT_AES_HW_CONFIG_ENC(alg, mode); } static void qat_alg_ablkcipher_init_dec(struct qat_alg_ablkcipher_ctx *ctx, int alg, const uint8_t *key, - unsigned int keylen) + unsigned int keylen, int mode) { struct icp_qat_hw_cipher_algo_blk *dec_cd = ctx->dec_cd; struct icp_qat_fw_la_bulk_req *req = &ctx->dec_fw_req; @@ -494,51 +487,67 @@ static void qat_alg_ablkcipher_init_dec(struct qat_alg_ablkcipher_ctx *ctx, qat_alg_ablkcipher_init_com(ctx, req, dec_cd, key, keylen); cd_pars->u.s.content_desc_addr = ctx->dec_cd_paddr; - dec_cd->aes.cipher_config.val = QAT_AES_HW_CONFIG_CBC_DEC(alg); + + if (mode != ICP_QAT_HW_CIPHER_CTR_MODE) + dec_cd->aes.cipher_config.val = + QAT_AES_HW_CONFIG_DEC(alg, mode); + else + dec_cd->aes.cipher_config.val = + QAT_AES_HW_CONFIG_ENC(alg, mode); } -static int qat_alg_validate_key(int key_len, int *alg) +static int qat_alg_validate_key(int key_len, int *alg, int mode) { - switch (key_len) { - case AES_KEYSIZE_128: - *alg = ICP_QAT_HW_CIPHER_ALGO_AES128; - break; - case AES_KEYSIZE_192: - *alg = ICP_QAT_HW_CIPHER_ALGO_AES192; - break; - case AES_KEYSIZE_256: - *alg = ICP_QAT_HW_CIPHER_ALGO_AES256; - break; - default: - return -EINVAL; + if (mode != ICP_QAT_HW_CIPHER_XTS_MODE) { + switch (key_len) { + case AES_KEYSIZE_128: + *alg = ICP_QAT_HW_CIPHER_ALGO_AES128; + break; + case AES_KEYSIZE_192: + *alg = ICP_QAT_HW_CIPHER_ALGO_AES192; + break; + case AES_KEYSIZE_256: + *alg = ICP_QAT_HW_CIPHER_ALGO_AES256; + break; + default: + return -EINVAL; + } + } else { + switch (key_len) { + case AES_KEYSIZE_128 << 1: + *alg = ICP_QAT_HW_CIPHER_ALGO_AES128; + break; + case AES_KEYSIZE_256 << 1: + *alg = ICP_QAT_HW_CIPHER_ALGO_AES256; + break; + default: + return -EINVAL; + } } return 0; } -static int qat_alg_aead_init_sessions(struct qat_alg_aead_ctx *ctx, - const uint8_t *key, unsigned int keylen) +static int qat_alg_aead_init_sessions(struct crypto_aead *tfm, const u8 *key, + unsigned int keylen, int mode) { struct crypto_authenc_keys keys; int alg; - if (crypto_rng_get_bytes(crypto_default_rng, ctx->salt, AES_BLOCK_SIZE)) - return -EFAULT; - if (crypto_authenc_extractkeys(&keys, key, keylen)) goto bad_key; - if (qat_alg_validate_key(keys.enckeylen, &alg)) + if (qat_alg_validate_key(keys.enckeylen, &alg, mode)) goto bad_key; - if (qat_alg_aead_init_enc_session(ctx, alg, &keys)) + if (qat_alg_aead_init_enc_session(tfm, alg, &keys, mode)) goto error; - if (qat_alg_aead_init_dec_session(ctx, alg, &keys)) + if (qat_alg_aead_init_dec_session(tfm, alg, &keys, mode)) goto error; return 0; bad_key: - crypto_tfm_set_flags(ctx->tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); return -EINVAL; error: return -EFAULT; @@ -546,15 +555,16 @@ error: static int qat_alg_ablkcipher_init_sessions(struct qat_alg_ablkcipher_ctx *ctx, const uint8_t *key, - unsigned int keylen) + unsigned int keylen, + int mode) { int alg; - if (qat_alg_validate_key(keylen, &alg)) + if (qat_alg_validate_key(keylen, &alg, mode)) goto bad_key; - qat_alg_ablkcipher_init_enc(ctx, alg, key, keylen); - qat_alg_ablkcipher_init_dec(ctx, alg, key, keylen); + qat_alg_ablkcipher_init_enc(ctx, alg, key, keylen, mode); + qat_alg_ablkcipher_init_dec(ctx, alg, key, keylen, mode); return 0; bad_key: crypto_tfm_set_flags(ctx->tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); @@ -567,7 +577,6 @@ static int qat_alg_aead_setkey(struct crypto_aead *tfm, const uint8_t *key, struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm); struct device *dev; - spin_lock(&ctx->lock); if (ctx->enc_cd) { /* rekeying */ dev = &GET_DEV(ctx->inst->accel_dev); @@ -581,7 +590,6 @@ static int qat_alg_aead_setkey(struct crypto_aead *tfm, const uint8_t *key, struct qat_crypto_instance *inst = qat_crypto_get_instance_node(node); if (!inst) { - spin_unlock(&ctx->lock); return -EINVAL; } @@ -591,19 +599,17 @@ static int qat_alg_aead_setkey(struct crypto_aead *tfm, const uint8_t *key, &ctx->enc_cd_paddr, GFP_ATOMIC); if (!ctx->enc_cd) { - spin_unlock(&ctx->lock); return -ENOMEM; } ctx->dec_cd = dma_zalloc_coherent(dev, sizeof(*ctx->dec_cd), &ctx->dec_cd_paddr, GFP_ATOMIC); if (!ctx->dec_cd) { - spin_unlock(&ctx->lock); goto out_free_enc; } } - spin_unlock(&ctx->lock); - if (qat_alg_aead_init_sessions(ctx, key, keylen)) + if (qat_alg_aead_init_sessions(tfm, key, keylen, + ICP_QAT_HW_CIPHER_CBC_MODE)) goto out_free_all; return 0; @@ -654,22 +660,20 @@ static void qat_alg_free_bufl(struct qat_crypto_instance *inst, } static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst, - struct scatterlist *assoc, struct scatterlist *sgl, - struct scatterlist *sglout, uint8_t *iv, - uint8_t ivlen, + struct scatterlist *sglout, struct qat_crypto_request *qat_req) { struct device *dev = &GET_DEV(inst->accel_dev); - int i, bufs = 0, sg_nctr = 0; - int n = sg_nents(sgl), assoc_n = sg_nents(assoc); + int i, sg_nctr = 0; + int n = sg_nents(sgl); struct qat_alg_buf_list *bufl; struct qat_alg_buf_list *buflout = NULL; dma_addr_t blp; dma_addr_t bloutp = 0; struct scatterlist *sg; size_t sz_out, sz = sizeof(struct qat_alg_buf_list) + - ((1 + n + assoc_n) * sizeof(struct qat_alg_buf)); + ((1 + n) * sizeof(struct qat_alg_buf)); if (unlikely(!n)) return -EINVAL; @@ -683,29 +687,8 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst, if (unlikely(dma_mapping_error(dev, blp))) goto err; - for_each_sg(assoc, sg, assoc_n, i) { - if (!sg->length) - continue; - bufl->bufers[bufs].addr = dma_map_single(dev, - sg_virt(sg), - sg->length, - DMA_BIDIRECTIONAL); - bufl->bufers[bufs].len = sg->length; - if (unlikely(dma_mapping_error(dev, bufl->bufers[bufs].addr))) - goto err; - bufs++; - } - if (ivlen) { - bufl->bufers[bufs].addr = dma_map_single(dev, iv, ivlen, - DMA_BIDIRECTIONAL); - bufl->bufers[bufs].len = ivlen; - if (unlikely(dma_mapping_error(dev, bufl->bufers[bufs].addr))) - goto err; - bufs++; - } - for_each_sg(sgl, sg, n, i) { - int y = sg_nctr + bufs; + int y = sg_nctr; if (!sg->length) continue; @@ -718,7 +701,7 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst, goto err; sg_nctr++; } - bufl->num_bufs = sg_nctr + bufs; + bufl->num_bufs = sg_nctr; qat_req->buf.bl = bufl; qat_req->buf.blp = blp; qat_req->buf.sz = sz; @@ -728,7 +711,7 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst, n = sg_nents(sglout); sz_out = sizeof(struct qat_alg_buf_list) + - ((1 + n + assoc_n) * sizeof(struct qat_alg_buf)); + ((1 + n) * sizeof(struct qat_alg_buf)); sg_nctr = 0; buflout = kzalloc_node(sz_out, GFP_ATOMIC, dev_to_node(&GET_DEV(inst->accel_dev))); @@ -738,14 +721,8 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst, if (unlikely(dma_mapping_error(dev, bloutp))) goto err; bufers = buflout->bufers; - /* For out of place operation dma map only data and - * reuse assoc mapping and iv */ - for (i = 0; i < bufs; i++) { - bufers[i].len = bufl->bufers[i].len; - bufers[i].addr = bufl->bufers[i].addr; - } for_each_sg(sglout, sg, n, i) { - int y = sg_nctr + bufs; + int y = sg_nctr; if (!sg->length) continue; @@ -758,7 +735,7 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst, bufers[y].len = sg->length; sg_nctr++; } - buflout->num_bufs = sg_nctr + bufs; + buflout->num_bufs = sg_nctr; buflout->num_mapped_bufs = sg_nctr; qat_req->buf.blout = buflout; qat_req->buf.bloutp = bloutp; @@ -772,7 +749,7 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst, err: dev_err(dev, "Failed to map buf for dma\n"); sg_nctr = 0; - for (i = 0; i < n + bufs; i++) + for (i = 0; i < n; i++) if (!dma_mapping_error(dev, bufl->bufers[i].addr)) dma_unmap_single(dev, bufl->bufers[i].addr, bufl->bufers[i].len, @@ -783,7 +760,7 @@ err: kfree(bufl); if (sgl != sglout && buflout) { n = sg_nents(sglout); - for (i = bufs; i < n + bufs; i++) + for (i = 0; i < n; i++) if (!dma_mapping_error(dev, buflout->bufers[i].addr)) dma_unmap_single(dev, buflout->bufers[i].addr, buflout->bufers[i].len, @@ -843,11 +820,10 @@ static int qat_alg_aead_dec(struct aead_request *areq) struct icp_qat_fw_la_cipher_req_params *cipher_param; struct icp_qat_fw_la_auth_req_params *auth_param; struct icp_qat_fw_la_bulk_req *msg; - int digst_size = crypto_aead_crt(aead_tfm)->authsize; + int digst_size = crypto_aead_authsize(aead_tfm); int ret, ctr = 0; - ret = qat_alg_sgl_to_bufl(ctx->inst, areq->assoc, areq->src, areq->dst, - areq->iv, AES_BLOCK_SIZE, qat_req); + ret = qat_alg_sgl_to_bufl(ctx->inst, areq->src, areq->dst, qat_req); if (unlikely(ret)) return ret; @@ -861,12 +837,11 @@ static int qat_alg_aead_dec(struct aead_request *areq) qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp; cipher_param = (void *)&qat_req->req.serv_specif_rqpars; cipher_param->cipher_length = areq->cryptlen - digst_size; - cipher_param->cipher_offset = areq->assoclen + AES_BLOCK_SIZE; + cipher_param->cipher_offset = areq->assoclen; memcpy(cipher_param->u.cipher_IV_array, areq->iv, AES_BLOCK_SIZE); auth_param = (void *)((uint8_t *)cipher_param + sizeof(*cipher_param)); auth_param->auth_off = 0; - auth_param->auth_len = areq->assoclen + - cipher_param->cipher_length + AES_BLOCK_SIZE; + auth_param->auth_len = areq->assoclen + cipher_param->cipher_length; do { ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg); } while (ret == -EAGAIN && ctr++ < 10); @@ -878,8 +853,7 @@ static int qat_alg_aead_dec(struct aead_request *areq) return -EINPROGRESS; } -static int qat_alg_aead_enc_internal(struct aead_request *areq, uint8_t *iv, - int enc_iv) +static int qat_alg_aead_enc(struct aead_request *areq) { struct crypto_aead *aead_tfm = crypto_aead_reqtfm(areq); struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm); @@ -888,10 +862,10 @@ static int qat_alg_aead_enc_internal(struct aead_request *areq, uint8_t *iv, struct icp_qat_fw_la_cipher_req_params *cipher_param; struct icp_qat_fw_la_auth_req_params *auth_param; struct icp_qat_fw_la_bulk_req *msg; + uint8_t *iv = areq->iv; int ret, ctr = 0; - ret = qat_alg_sgl_to_bufl(ctx->inst, areq->assoc, areq->src, areq->dst, - iv, AES_BLOCK_SIZE, qat_req); + ret = qat_alg_sgl_to_bufl(ctx->inst, areq->src, areq->dst, qat_req); if (unlikely(ret)) return ret; @@ -906,16 +880,12 @@ static int qat_alg_aead_enc_internal(struct aead_request *areq, uint8_t *iv, cipher_param = (void *)&qat_req->req.serv_specif_rqpars; auth_param = (void *)((uint8_t *)cipher_param + sizeof(*cipher_param)); - if (enc_iv) { - cipher_param->cipher_length = areq->cryptlen + AES_BLOCK_SIZE; - cipher_param->cipher_offset = areq->assoclen; - } else { - memcpy(cipher_param->u.cipher_IV_array, iv, AES_BLOCK_SIZE); - cipher_param->cipher_length = areq->cryptlen; - cipher_param->cipher_offset = areq->assoclen + AES_BLOCK_SIZE; - } + memcpy(cipher_param->u.cipher_IV_array, iv, AES_BLOCK_SIZE); + cipher_param->cipher_length = areq->cryptlen; + cipher_param->cipher_offset = areq->assoclen; + auth_param->auth_off = 0; - auth_param->auth_len = areq->assoclen + areq->cryptlen + AES_BLOCK_SIZE; + auth_param->auth_len = areq->assoclen + areq->cryptlen; do { ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg); @@ -928,28 +898,9 @@ static int qat_alg_aead_enc_internal(struct aead_request *areq, uint8_t *iv, return -EINPROGRESS; } -static int qat_alg_aead_enc(struct aead_request *areq) -{ - return qat_alg_aead_enc_internal(areq, areq->iv, 0); -} - -static int qat_alg_aead_genivenc(struct aead_givcrypt_request *req) -{ - struct crypto_aead *aead_tfm = crypto_aead_reqtfm(&req->areq); - struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm); - struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm); - __be64 seq; - - memcpy(req->giv, ctx->salt, AES_BLOCK_SIZE); - seq = cpu_to_be64(req->seq); - memcpy(req->giv + AES_BLOCK_SIZE - sizeof(uint64_t), - &seq, sizeof(uint64_t)); - return qat_alg_aead_enc_internal(&req->areq, req->giv, 1); -} - static int qat_alg_ablkcipher_setkey(struct crypto_ablkcipher *tfm, - const uint8_t *key, - unsigned int keylen) + const u8 *key, unsigned int keylen, + int mode) { struct qat_alg_ablkcipher_ctx *ctx = crypto_ablkcipher_ctx(tfm); struct device *dev; @@ -990,7 +941,7 @@ static int qat_alg_ablkcipher_setkey(struct crypto_ablkcipher *tfm, } } spin_unlock(&ctx->lock); - if (qat_alg_ablkcipher_init_sessions(ctx, key, keylen)) + if (qat_alg_ablkcipher_init_sessions(ctx, key, keylen, mode)) goto out_free_all; return 0; @@ -1008,6 +959,27 @@ out_free_enc: return -ENOMEM; } +static int qat_alg_ablkcipher_cbc_setkey(struct crypto_ablkcipher *tfm, + const u8 *key, unsigned int keylen) +{ + return qat_alg_ablkcipher_setkey(tfm, key, keylen, + ICP_QAT_HW_CIPHER_CBC_MODE); +} + +static int qat_alg_ablkcipher_ctr_setkey(struct crypto_ablkcipher *tfm, + const u8 *key, unsigned int keylen) +{ + return qat_alg_ablkcipher_setkey(tfm, key, keylen, + ICP_QAT_HW_CIPHER_CTR_MODE); +} + +static int qat_alg_ablkcipher_xts_setkey(struct crypto_ablkcipher *tfm, + const u8 *key, unsigned int keylen) +{ + return qat_alg_ablkcipher_setkey(tfm, key, keylen, + ICP_QAT_HW_CIPHER_XTS_MODE); +} + static int qat_alg_ablkcipher_encrypt(struct ablkcipher_request *req) { struct crypto_ablkcipher *atfm = crypto_ablkcipher_reqtfm(req); @@ -1018,8 +990,7 @@ static int qat_alg_ablkcipher_encrypt(struct ablkcipher_request *req) struct icp_qat_fw_la_bulk_req *msg; int ret, ctr = 0; - ret = qat_alg_sgl_to_bufl(ctx->inst, NULL, req->src, req->dst, - NULL, 0, qat_req); + ret = qat_alg_sgl_to_bufl(ctx->inst, req->src, req->dst, qat_req); if (unlikely(ret)) return ret; @@ -1056,8 +1027,7 @@ static int qat_alg_ablkcipher_decrypt(struct ablkcipher_request *req) struct icp_qat_fw_la_bulk_req *msg; int ret, ctr = 0; - ret = qat_alg_sgl_to_bufl(ctx->inst, NULL, req->src, req->dst, - NULL, 0, qat_req); + ret = qat_alg_sgl_to_bufl(ctx->inst, req->src, req->dst, qat_req); if (unlikely(ret)) return ret; @@ -1084,46 +1054,43 @@ static int qat_alg_ablkcipher_decrypt(struct ablkcipher_request *req) return -EINPROGRESS; } -static int qat_alg_aead_init(struct crypto_tfm *tfm, +static int qat_alg_aead_init(struct crypto_aead *tfm, enum icp_qat_hw_auth_algo hash, const char *hash_name) { - struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm); + struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm); ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0); if (IS_ERR(ctx->hash_tfm)) - return -EFAULT; - spin_lock_init(&ctx->lock); + return PTR_ERR(ctx->hash_tfm); ctx->qat_hash_alg = hash; - tfm->crt_aead.reqsize = sizeof(struct aead_request) + - sizeof(struct qat_crypto_request); - ctx->tfm = tfm; + crypto_aead_set_reqsize(tfm, sizeof(struct aead_request) + + sizeof(struct qat_crypto_request)); return 0; } -static int qat_alg_aead_sha1_init(struct crypto_tfm *tfm) +static int qat_alg_aead_sha1_init(struct crypto_aead *tfm) { return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA1, "sha1"); } -static int qat_alg_aead_sha256_init(struct crypto_tfm *tfm) +static int qat_alg_aead_sha256_init(struct crypto_aead *tfm) { return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA256, "sha256"); } -static int qat_alg_aead_sha512_init(struct crypto_tfm *tfm) +static int qat_alg_aead_sha512_init(struct crypto_aead *tfm) { return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA512, "sha512"); } -static void qat_alg_aead_exit(struct crypto_tfm *tfm) +static void qat_alg_aead_exit(struct crypto_aead *tfm) { - struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm); + struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm); struct qat_crypto_instance *inst = ctx->inst; struct device *dev; - if (!IS_ERR(ctx->hash_tfm)) - crypto_free_shash(ctx->hash_tfm); + crypto_free_shash(ctx->hash_tfm); if (!inst) return; @@ -1180,75 +1147,107 @@ static void qat_alg_ablkcipher_exit(struct crypto_tfm *tfm) qat_crypto_put_instance(inst); } -static struct crypto_alg qat_algs[] = { { - .cra_name = "authenc(hmac(sha1),cbc(aes))", - .cra_driver_name = "qat_aes_cbc_hmac_sha1", - .cra_priority = 4001, - .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct qat_alg_aead_ctx), - .cra_alignmask = 0, - .cra_type = &crypto_aead_type, - .cra_module = THIS_MODULE, - .cra_init = qat_alg_aead_sha1_init, - .cra_exit = qat_alg_aead_exit, - .cra_u = { - .aead = { - .setkey = qat_alg_aead_setkey, - .decrypt = qat_alg_aead_dec, - .encrypt = qat_alg_aead_enc, - .givencrypt = qat_alg_aead_genivenc, - .ivsize = AES_BLOCK_SIZE, - .maxauthsize = SHA1_DIGEST_SIZE, - }, + +static struct aead_alg qat_aeads[] = { { + .base = { + .cra_name = "authenc(hmac(sha1),cbc(aes))", + .cra_driver_name = "qat_aes_cbc_hmac_sha1", + .cra_priority = 4001, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct qat_alg_aead_ctx), + .cra_module = THIS_MODULE, + }, + .init = qat_alg_aead_sha1_init, + .exit = qat_alg_aead_exit, + .setkey = qat_alg_aead_setkey, + .decrypt = qat_alg_aead_dec, + .encrypt = qat_alg_aead_enc, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, +}, { + .base = { + .cra_name = "authenc(hmac(sha256),cbc(aes))", + .cra_driver_name = "qat_aes_cbc_hmac_sha256", + .cra_priority = 4001, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct qat_alg_aead_ctx), + .cra_module = THIS_MODULE, }, + .init = qat_alg_aead_sha256_init, + .exit = qat_alg_aead_exit, + .setkey = qat_alg_aead_setkey, + .decrypt = qat_alg_aead_dec, + .encrypt = qat_alg_aead_enc, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, }, { - .cra_name = "authenc(hmac(sha256),cbc(aes))", - .cra_driver_name = "qat_aes_cbc_hmac_sha256", + .base = { + .cra_name = "authenc(hmac(sha512),cbc(aes))", + .cra_driver_name = "qat_aes_cbc_hmac_sha512", + .cra_priority = 4001, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct qat_alg_aead_ctx), + .cra_module = THIS_MODULE, + }, + .init = qat_alg_aead_sha512_init, + .exit = qat_alg_aead_exit, + .setkey = qat_alg_aead_setkey, + .decrypt = qat_alg_aead_dec, + .encrypt = qat_alg_aead_enc, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, +} }; + +static struct crypto_alg qat_algs[] = { { + .cra_name = "cbc(aes)", + .cra_driver_name = "qat_aes_cbc", .cra_priority = 4001, - .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct qat_alg_aead_ctx), + .cra_ctxsize = sizeof(struct qat_alg_ablkcipher_ctx), .cra_alignmask = 0, - .cra_type = &crypto_aead_type, + .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, - .cra_init = qat_alg_aead_sha256_init, - .cra_exit = qat_alg_aead_exit, + .cra_init = qat_alg_ablkcipher_init, + .cra_exit = qat_alg_ablkcipher_exit, .cra_u = { - .aead = { - .setkey = qat_alg_aead_setkey, - .decrypt = qat_alg_aead_dec, - .encrypt = qat_alg_aead_enc, - .givencrypt = qat_alg_aead_genivenc, + .ablkcipher = { + .setkey = qat_alg_ablkcipher_cbc_setkey, + .decrypt = qat_alg_ablkcipher_decrypt, + .encrypt = qat_alg_ablkcipher_encrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, .ivsize = AES_BLOCK_SIZE, - .maxauthsize = SHA256_DIGEST_SIZE, }, }, }, { - .cra_name = "authenc(hmac(sha512),cbc(aes))", - .cra_driver_name = "qat_aes_cbc_hmac_sha512", + .cra_name = "ctr(aes)", + .cra_driver_name = "qat_aes_ctr", .cra_priority = 4001, - .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct qat_alg_aead_ctx), + .cra_ctxsize = sizeof(struct qat_alg_ablkcipher_ctx), .cra_alignmask = 0, - .cra_type = &crypto_aead_type, + .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, - .cra_init = qat_alg_aead_sha512_init, - .cra_exit = qat_alg_aead_exit, + .cra_init = qat_alg_ablkcipher_init, + .cra_exit = qat_alg_ablkcipher_exit, .cra_u = { - .aead = { - .setkey = qat_alg_aead_setkey, - .decrypt = qat_alg_aead_dec, - .encrypt = qat_alg_aead_enc, - .givencrypt = qat_alg_aead_genivenc, + .ablkcipher = { + .setkey = qat_alg_ablkcipher_ctr_setkey, + .decrypt = qat_alg_ablkcipher_decrypt, + .encrypt = qat_alg_ablkcipher_encrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, .ivsize = AES_BLOCK_SIZE, - .maxauthsize = SHA512_DIGEST_SIZE, }, }, }, { - .cra_name = "cbc(aes)", - .cra_driver_name = "qat_aes_cbc", + .cra_name = "xts(aes)", + .cra_driver_name = "qat_aes_xts", .cra_priority = 4001, .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = AES_BLOCK_SIZE, @@ -1260,7 +1259,7 @@ static struct crypto_alg qat_algs[] = { { .cra_exit = qat_alg_ablkcipher_exit, .cra_u = { .ablkcipher = { - .setkey = qat_alg_ablkcipher_setkey, + .setkey = qat_alg_ablkcipher_xts_setkey, .decrypt = qat_alg_ablkcipher_decrypt, .encrypt = qat_alg_ablkcipher_encrypt, .min_keysize = AES_MIN_KEY_SIZE, @@ -1272,42 +1271,44 @@ static struct crypto_alg qat_algs[] = { { int qat_algs_register(void) { - int ret = 0; + int ret = 0, i; mutex_lock(&algs_lock); - if (++active_devs == 1) { - int i; + if (++active_devs != 1) + goto unlock; - for (i = 0; i < ARRAY_SIZE(qat_algs); i++) - qat_algs[i].cra_flags = - (qat_algs[i].cra_type == &crypto_aead_type) ? - CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC : - CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC; + for (i = 0; i < ARRAY_SIZE(qat_algs); i++) + qat_algs[i].cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC; - ret = crypto_register_algs(qat_algs, ARRAY_SIZE(qat_algs)); - } - mutex_unlock(&algs_lock); - return ret; -} + ret = crypto_register_algs(qat_algs, ARRAY_SIZE(qat_algs)); + if (ret) + goto unlock; -int qat_algs_unregister(void) -{ - int ret = 0; + for (i = 0; i < ARRAY_SIZE(qat_aeads); i++) + qat_aeads[i].base.cra_flags = CRYPTO_ALG_ASYNC; - mutex_lock(&algs_lock); - if (--active_devs == 0) - ret = crypto_unregister_algs(qat_algs, ARRAY_SIZE(qat_algs)); + ret = crypto_register_aeads(qat_aeads, ARRAY_SIZE(qat_aeads)); + if (ret) + goto unreg_algs; + +unlock: mutex_unlock(&algs_lock); return ret; -} -int qat_algs_init(void) -{ - crypto_get_default_rng(); - return 0; +unreg_algs: + crypto_unregister_algs(qat_algs, ARRAY_SIZE(qat_algs)); + goto unlock; } -void qat_algs_exit(void) +void qat_algs_unregister(void) { - crypto_put_default_rng(); + mutex_lock(&algs_lock); + if (--active_devs != 0) + goto unlock; + + crypto_unregister_aeads(qat_aeads, ARRAY_SIZE(qat_aeads)); + crypto_unregister_algs(qat_algs, ARRAY_SIZE(qat_algs)); + +unlock: + mutex_unlock(&algs_lock); } diff --git a/kernel/drivers/crypto/qat/qat_common/qat_asym_algs.c b/kernel/drivers/crypto/qat/qat_common/qat_asym_algs.c new file mode 100644 index 000000000..51c594fda --- /dev/null +++ b/kernel/drivers/crypto/qat/qat_common/qat_asym_algs.c @@ -0,0 +1,755 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#include +#include +#include +#include +#include +#include +#include +#include "qat_rsapubkey-asn1.h" +#include "qat_rsaprivkey-asn1.h" +#include "icp_qat_fw_pke.h" +#include "adf_accel_devices.h" +#include "adf_transport.h" +#include "adf_common_drv.h" +#include "qat_crypto.h" + +static DEFINE_MUTEX(algs_lock); +static unsigned int active_devs; + +struct qat_rsa_input_params { + union { + struct { + dma_addr_t m; + dma_addr_t e; + dma_addr_t n; + } enc; + struct { + dma_addr_t c; + dma_addr_t d; + dma_addr_t n; + } dec; + u64 in_tab[8]; + }; +} __packed __aligned(64); + +struct qat_rsa_output_params { + union { + struct { + dma_addr_t c; + } enc; + struct { + dma_addr_t m; + } dec; + u64 out_tab[8]; + }; +} __packed __aligned(64); + +struct qat_rsa_ctx { + char *n; + char *e; + char *d; + dma_addr_t dma_n; + dma_addr_t dma_e; + dma_addr_t dma_d; + unsigned int key_sz; + struct qat_crypto_instance *inst; +} __packed __aligned(64); + +struct qat_rsa_request { + struct qat_rsa_input_params in; + struct qat_rsa_output_params out; + dma_addr_t phy_in; + dma_addr_t phy_out; + char *src_align; + char *dst_align; + struct icp_qat_fw_pke_request req; + struct qat_rsa_ctx *ctx; + int err; +} __aligned(64); + +static void qat_rsa_cb(struct icp_qat_fw_pke_resp *resp) +{ + struct akcipher_request *areq = (void *)(__force long)resp->opaque; + struct qat_rsa_request *req = PTR_ALIGN(akcipher_request_ctx(areq), 64); + struct device *dev = &GET_DEV(req->ctx->inst->accel_dev); + int err = ICP_QAT_FW_PKE_RESP_PKE_STAT_GET( + resp->pke_resp_hdr.comn_resp_flags); + + err = (err == ICP_QAT_FW_COMN_STATUS_FLAG_OK) ? 0 : -EINVAL; + + if (req->src_align) + dma_free_coherent(dev, req->ctx->key_sz, req->src_align, + req->in.enc.m); + else + dma_unmap_single(dev, req->in.enc.m, req->ctx->key_sz, + DMA_TO_DEVICE); + + areq->dst_len = req->ctx->key_sz; + if (req->dst_align) { + char *ptr = req->dst_align; + + while (!(*ptr) && areq->dst_len) { + areq->dst_len--; + ptr++; + } + + if (areq->dst_len != req->ctx->key_sz) + memmove(req->dst_align, ptr, areq->dst_len); + + scatterwalk_map_and_copy(req->dst_align, areq->dst, 0, + areq->dst_len, 1); + + dma_free_coherent(dev, req->ctx->key_sz, req->dst_align, + req->out.enc.c); + } else { + char *ptr = sg_virt(areq->dst); + + while (!(*ptr) && areq->dst_len) { + areq->dst_len--; + ptr++; + } + + if (sg_virt(areq->dst) != ptr && areq->dst_len) + memmove(sg_virt(areq->dst), ptr, areq->dst_len); + + dma_unmap_single(dev, req->out.enc.c, req->ctx->key_sz, + DMA_FROM_DEVICE); + } + + dma_unmap_single(dev, req->phy_in, sizeof(struct qat_rsa_input_params), + DMA_TO_DEVICE); + dma_unmap_single(dev, req->phy_out, + sizeof(struct qat_rsa_output_params), + DMA_TO_DEVICE); + + akcipher_request_complete(areq, err); +} + +void qat_alg_asym_callback(void *_resp) +{ + struct icp_qat_fw_pke_resp *resp = _resp; + + qat_rsa_cb(resp); +} + +#define PKE_RSA_EP_512 0x1c161b21 +#define PKE_RSA_EP_1024 0x35111bf7 +#define PKE_RSA_EP_1536 0x4d111cdc +#define PKE_RSA_EP_2048 0x6e111dba +#define PKE_RSA_EP_3072 0x7d111ea3 +#define PKE_RSA_EP_4096 0xa5101f7e + +static unsigned long qat_rsa_enc_fn_id(unsigned int len) +{ + unsigned int bitslen = len << 3; + + switch (bitslen) { + case 512: + return PKE_RSA_EP_512; + case 1024: + return PKE_RSA_EP_1024; + case 1536: + return PKE_RSA_EP_1536; + case 2048: + return PKE_RSA_EP_2048; + case 3072: + return PKE_RSA_EP_3072; + case 4096: + return PKE_RSA_EP_4096; + default: + return 0; + }; +} + +#define PKE_RSA_DP1_512 0x1c161b3c +#define PKE_RSA_DP1_1024 0x35111c12 +#define PKE_RSA_DP1_1536 0x4d111cf7 +#define PKE_RSA_DP1_2048 0x6e111dda +#define PKE_RSA_DP1_3072 0x7d111ebe +#define PKE_RSA_DP1_4096 0xa5101f98 + +static unsigned long qat_rsa_dec_fn_id(unsigned int len) +{ + unsigned int bitslen = len << 3; + + switch (bitslen) { + case 512: + return PKE_RSA_DP1_512; + case 1024: + return PKE_RSA_DP1_1024; + case 1536: + return PKE_RSA_DP1_1536; + case 2048: + return PKE_RSA_DP1_2048; + case 3072: + return PKE_RSA_DP1_3072; + case 4096: + return PKE_RSA_DP1_4096; + default: + return 0; + }; +} + +static int qat_rsa_enc(struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct qat_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct qat_crypto_instance *inst = ctx->inst; + struct device *dev = &GET_DEV(inst->accel_dev); + struct qat_rsa_request *qat_req = + PTR_ALIGN(akcipher_request_ctx(req), 64); + struct icp_qat_fw_pke_request *msg = &qat_req->req; + int ret, ctr = 0; + + if (unlikely(!ctx->n || !ctx->e)) + return -EINVAL; + + if (req->dst_len < ctx->key_sz) { + req->dst_len = ctx->key_sz; + return -EOVERFLOW; + } + memset(msg, '\0', sizeof(*msg)); + ICP_QAT_FW_PKE_HDR_VALID_FLAG_SET(msg->pke_hdr, + ICP_QAT_FW_COMN_REQ_FLAG_SET); + msg->pke_hdr.cd_pars.func_id = qat_rsa_enc_fn_id(ctx->key_sz); + if (unlikely(!msg->pke_hdr.cd_pars.func_id)) + return -EINVAL; + + qat_req->ctx = ctx; + msg->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_PKE; + msg->pke_hdr.comn_req_flags = + ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_PTR_TYPE_FLAT, + QAT_COMN_CD_FLD_TYPE_64BIT_ADR); + + qat_req->in.enc.e = ctx->dma_e; + qat_req->in.enc.n = ctx->dma_n; + ret = -ENOMEM; + + /* + * src can be of any size in valid range, but HW expects it to be the + * same as modulo n so in case it is different we need to allocate a + * new buf and copy src data. + * In other case we just need to map the user provided buffer. + * Also need to make sure that it is in contiguous buffer. + */ + if (sg_is_last(req->src) && req->src_len == ctx->key_sz) { + qat_req->src_align = NULL; + qat_req->in.enc.m = dma_map_single(dev, sg_virt(req->src), + req->src_len, DMA_TO_DEVICE); + if (unlikely(dma_mapping_error(dev, qat_req->in.enc.m))) + return ret; + + } else { + int shift = ctx->key_sz - req->src_len; + + qat_req->src_align = dma_zalloc_coherent(dev, ctx->key_sz, + &qat_req->in.enc.m, + GFP_KERNEL); + if (unlikely(!qat_req->src_align)) + return ret; + + scatterwalk_map_and_copy(qat_req->src_align + shift, req->src, + 0, req->src_len, 0); + } + if (sg_is_last(req->dst) && req->dst_len == ctx->key_sz) { + qat_req->dst_align = NULL; + qat_req->out.enc.c = dma_map_single(dev, sg_virt(req->dst), + req->dst_len, + DMA_FROM_DEVICE); + + if (unlikely(dma_mapping_error(dev, qat_req->out.enc.c))) + goto unmap_src; + + } else { + qat_req->dst_align = dma_zalloc_coherent(dev, ctx->key_sz, + &qat_req->out.enc.c, + GFP_KERNEL); + if (unlikely(!qat_req->dst_align)) + goto unmap_src; + + } + qat_req->in.in_tab[3] = 0; + qat_req->out.out_tab[1] = 0; + qat_req->phy_in = dma_map_single(dev, &qat_req->in.enc.m, + sizeof(struct qat_rsa_input_params), + DMA_TO_DEVICE); + if (unlikely(dma_mapping_error(dev, qat_req->phy_in))) + goto unmap_dst; + + qat_req->phy_out = dma_map_single(dev, &qat_req->out.enc.c, + sizeof(struct qat_rsa_output_params), + DMA_TO_DEVICE); + if (unlikely(dma_mapping_error(dev, qat_req->phy_out))) + goto unmap_in_params; + + msg->pke_mid.src_data_addr = qat_req->phy_in; + msg->pke_mid.dest_data_addr = qat_req->phy_out; + msg->pke_mid.opaque = (uint64_t)(__force long)req; + msg->input_param_count = 3; + msg->output_param_count = 1; + do { + ret = adf_send_message(ctx->inst->pke_tx, (uint32_t *)msg); + } while (ret == -EBUSY && ctr++ < 100); + + if (!ret) + return -EINPROGRESS; +unmap_src: + if (qat_req->src_align) + dma_free_coherent(dev, ctx->key_sz, qat_req->src_align, + qat_req->in.enc.m); + else + if (!dma_mapping_error(dev, qat_req->in.enc.m)) + dma_unmap_single(dev, qat_req->in.enc.m, ctx->key_sz, + DMA_TO_DEVICE); +unmap_dst: + if (qat_req->dst_align) + dma_free_coherent(dev, ctx->key_sz, qat_req->dst_align, + qat_req->out.enc.c); + else + if (!dma_mapping_error(dev, qat_req->out.enc.c)) + dma_unmap_single(dev, qat_req->out.enc.c, ctx->key_sz, + DMA_FROM_DEVICE); +unmap_in_params: + if (!dma_mapping_error(dev, qat_req->phy_in)) + dma_unmap_single(dev, qat_req->phy_in, + sizeof(struct qat_rsa_input_params), + DMA_TO_DEVICE); + if (!dma_mapping_error(dev, qat_req->phy_out)) + dma_unmap_single(dev, qat_req->phy_out, + sizeof(struct qat_rsa_output_params), + DMA_TO_DEVICE); + return ret; +} + +static int qat_rsa_dec(struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct qat_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct qat_crypto_instance *inst = ctx->inst; + struct device *dev = &GET_DEV(inst->accel_dev); + struct qat_rsa_request *qat_req = + PTR_ALIGN(akcipher_request_ctx(req), 64); + struct icp_qat_fw_pke_request *msg = &qat_req->req; + int ret, ctr = 0; + + if (unlikely(!ctx->n || !ctx->d)) + return -EINVAL; + + if (req->dst_len < ctx->key_sz) { + req->dst_len = ctx->key_sz; + return -EOVERFLOW; + } + memset(msg, '\0', sizeof(*msg)); + ICP_QAT_FW_PKE_HDR_VALID_FLAG_SET(msg->pke_hdr, + ICP_QAT_FW_COMN_REQ_FLAG_SET); + msg->pke_hdr.cd_pars.func_id = qat_rsa_dec_fn_id(ctx->key_sz); + if (unlikely(!msg->pke_hdr.cd_pars.func_id)) + return -EINVAL; + + qat_req->ctx = ctx; + msg->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_PKE; + msg->pke_hdr.comn_req_flags = + ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_PTR_TYPE_FLAT, + QAT_COMN_CD_FLD_TYPE_64BIT_ADR); + + qat_req->in.dec.d = ctx->dma_d; + qat_req->in.dec.n = ctx->dma_n; + ret = -ENOMEM; + + /* + * src can be of any size in valid range, but HW expects it to be the + * same as modulo n so in case it is different we need to allocate a + * new buf and copy src data. + * In other case we just need to map the user provided buffer. + * Also need to make sure that it is in contiguous buffer. + */ + if (sg_is_last(req->src) && req->src_len == ctx->key_sz) { + qat_req->src_align = NULL; + qat_req->in.dec.c = dma_map_single(dev, sg_virt(req->src), + req->dst_len, DMA_TO_DEVICE); + if (unlikely(dma_mapping_error(dev, qat_req->in.dec.c))) + return ret; + + } else { + int shift = ctx->key_sz - req->src_len; + + qat_req->src_align = dma_zalloc_coherent(dev, ctx->key_sz, + &qat_req->in.dec.c, + GFP_KERNEL); + if (unlikely(!qat_req->src_align)) + return ret; + + scatterwalk_map_and_copy(qat_req->src_align + shift, req->src, + 0, req->src_len, 0); + } + if (sg_is_last(req->dst) && req->dst_len == ctx->key_sz) { + qat_req->dst_align = NULL; + qat_req->out.dec.m = dma_map_single(dev, sg_virt(req->dst), + req->dst_len, + DMA_FROM_DEVICE); + + if (unlikely(dma_mapping_error(dev, qat_req->out.dec.m))) + goto unmap_src; + + } else { + qat_req->dst_align = dma_zalloc_coherent(dev, ctx->key_sz, + &qat_req->out.dec.m, + GFP_KERNEL); + if (unlikely(!qat_req->dst_align)) + goto unmap_src; + + } + + qat_req->in.in_tab[3] = 0; + qat_req->out.out_tab[1] = 0; + qat_req->phy_in = dma_map_single(dev, &qat_req->in.dec.c, + sizeof(struct qat_rsa_input_params), + DMA_TO_DEVICE); + if (unlikely(dma_mapping_error(dev, qat_req->phy_in))) + goto unmap_dst; + + qat_req->phy_out = dma_map_single(dev, &qat_req->out.dec.m, + sizeof(struct qat_rsa_output_params), + DMA_TO_DEVICE); + if (unlikely(dma_mapping_error(dev, qat_req->phy_out))) + goto unmap_in_params; + + msg->pke_mid.src_data_addr = qat_req->phy_in; + msg->pke_mid.dest_data_addr = qat_req->phy_out; + msg->pke_mid.opaque = (uint64_t)(__force long)req; + msg->input_param_count = 3; + msg->output_param_count = 1; + do { + ret = adf_send_message(ctx->inst->pke_tx, (uint32_t *)msg); + } while (ret == -EBUSY && ctr++ < 100); + + if (!ret) + return -EINPROGRESS; +unmap_src: + if (qat_req->src_align) + dma_free_coherent(dev, ctx->key_sz, qat_req->src_align, + qat_req->in.dec.c); + else + if (!dma_mapping_error(dev, qat_req->in.dec.c)) + dma_unmap_single(dev, qat_req->in.dec.c, ctx->key_sz, + DMA_TO_DEVICE); +unmap_dst: + if (qat_req->dst_align) + dma_free_coherent(dev, ctx->key_sz, qat_req->dst_align, + qat_req->out.dec.m); + else + if (!dma_mapping_error(dev, qat_req->out.dec.m)) + dma_unmap_single(dev, qat_req->out.dec.m, ctx->key_sz, + DMA_FROM_DEVICE); +unmap_in_params: + if (!dma_mapping_error(dev, qat_req->phy_in)) + dma_unmap_single(dev, qat_req->phy_in, + sizeof(struct qat_rsa_input_params), + DMA_TO_DEVICE); + if (!dma_mapping_error(dev, qat_req->phy_out)) + dma_unmap_single(dev, qat_req->phy_out, + sizeof(struct qat_rsa_output_params), + DMA_TO_DEVICE); + return ret; +} + +int qat_rsa_get_n(void *context, size_t hdrlen, unsigned char tag, + const void *value, size_t vlen) +{ + struct qat_rsa_ctx *ctx = context; + struct qat_crypto_instance *inst = ctx->inst; + struct device *dev = &GET_DEV(inst->accel_dev); + const char *ptr = value; + int ret; + + while (!*ptr && vlen) { + ptr++; + vlen--; + } + + ctx->key_sz = vlen; + ret = -EINVAL; + /* In FIPS mode only allow key size 2K & 3K */ + if (fips_enabled && (ctx->key_sz != 256 && ctx->key_sz != 384)) { + pr_err("QAT: RSA: key size not allowed in FIPS mode\n"); + goto err; + } + /* invalid key size provided */ + if (!qat_rsa_enc_fn_id(ctx->key_sz)) + goto err; + + ret = -ENOMEM; + ctx->n = dma_zalloc_coherent(dev, ctx->key_sz, &ctx->dma_n, GFP_KERNEL); + if (!ctx->n) + goto err; + + memcpy(ctx->n, ptr, ctx->key_sz); + return 0; +err: + ctx->key_sz = 0; + ctx->n = NULL; + return ret; +} + +int qat_rsa_get_e(void *context, size_t hdrlen, unsigned char tag, + const void *value, size_t vlen) +{ + struct qat_rsa_ctx *ctx = context; + struct qat_crypto_instance *inst = ctx->inst; + struct device *dev = &GET_DEV(inst->accel_dev); + const char *ptr = value; + + while (!*ptr && vlen) { + ptr++; + vlen--; + } + + if (!ctx->key_sz || !vlen || vlen > ctx->key_sz) { + ctx->e = NULL; + return -EINVAL; + } + + ctx->e = dma_zalloc_coherent(dev, ctx->key_sz, &ctx->dma_e, GFP_KERNEL); + if (!ctx->e) { + ctx->e = NULL; + return -ENOMEM; + } + memcpy(ctx->e + (ctx->key_sz - vlen), ptr, vlen); + return 0; +} + +int qat_rsa_get_d(void *context, size_t hdrlen, unsigned char tag, + const void *value, size_t vlen) +{ + struct qat_rsa_ctx *ctx = context; + struct qat_crypto_instance *inst = ctx->inst; + struct device *dev = &GET_DEV(inst->accel_dev); + const char *ptr = value; + int ret; + + while (!*ptr && vlen) { + ptr++; + vlen--; + } + + ret = -EINVAL; + if (!ctx->key_sz || !vlen || vlen > ctx->key_sz) + goto err; + + /* In FIPS mode only allow key size 2K & 3K */ + if (fips_enabled && (vlen != 256 && vlen != 384)) { + pr_err("QAT: RSA: key size not allowed in FIPS mode\n"); + goto err; + } + + ret = -ENOMEM; + ctx->d = dma_zalloc_coherent(dev, ctx->key_sz, &ctx->dma_d, GFP_KERNEL); + if (!ctx->n) + goto err; + + memcpy(ctx->d + (ctx->key_sz - vlen), ptr, vlen); + return 0; +err: + ctx->d = NULL; + return ret; +} + +static int qat_rsa_setkey(struct crypto_akcipher *tfm, const void *key, + unsigned int keylen, bool private) +{ + struct qat_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct device *dev = &GET_DEV(ctx->inst->accel_dev); + int ret; + + /* Free the old key if any */ + if (ctx->n) + dma_free_coherent(dev, ctx->key_sz, ctx->n, ctx->dma_n); + if (ctx->e) + dma_free_coherent(dev, ctx->key_sz, ctx->e, ctx->dma_e); + if (ctx->d) { + memset(ctx->d, '\0', ctx->key_sz); + dma_free_coherent(dev, ctx->key_sz, ctx->d, ctx->dma_d); + } + + ctx->n = NULL; + ctx->e = NULL; + ctx->d = NULL; + + if (private) + ret = asn1_ber_decoder(&qat_rsaprivkey_decoder, ctx, key, + keylen); + else + ret = asn1_ber_decoder(&qat_rsapubkey_decoder, ctx, key, + keylen); + if (ret < 0) + goto free; + + if (!ctx->n || !ctx->e) { + /* invalid key provided */ + ret = -EINVAL; + goto free; + } + if (private && !ctx->d) { + /* invalid private key provided */ + ret = -EINVAL; + goto free; + } + + return 0; +free: + if (ctx->d) { + memset(ctx->d, '\0', ctx->key_sz); + dma_free_coherent(dev, ctx->key_sz, ctx->d, ctx->dma_d); + ctx->d = NULL; + } + if (ctx->e) { + dma_free_coherent(dev, ctx->key_sz, ctx->e, ctx->dma_e); + ctx->e = NULL; + } + if (ctx->n) { + dma_free_coherent(dev, ctx->key_sz, ctx->n, ctx->dma_n); + ctx->n = NULL; + ctx->key_sz = 0; + } + return ret; +} + +static int qat_rsa_setpubkey(struct crypto_akcipher *tfm, const void *key, + unsigned int keylen) +{ + return qat_rsa_setkey(tfm, key, keylen, false); +} + +static int qat_rsa_setprivkey(struct crypto_akcipher *tfm, const void *key, + unsigned int keylen) +{ + return qat_rsa_setkey(tfm, key, keylen, true); +} + +static int qat_rsa_max_size(struct crypto_akcipher *tfm) +{ + struct qat_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + + return (ctx->n) ? ctx->key_sz : -EINVAL; +} + +static int qat_rsa_init_tfm(struct crypto_akcipher *tfm) +{ + struct qat_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct qat_crypto_instance *inst = + qat_crypto_get_instance_node(get_current_node()); + + if (!inst) + return -EINVAL; + + ctx->key_sz = 0; + ctx->inst = inst; + return 0; +} + +static void qat_rsa_exit_tfm(struct crypto_akcipher *tfm) +{ + struct qat_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct device *dev = &GET_DEV(ctx->inst->accel_dev); + + if (ctx->n) + dma_free_coherent(dev, ctx->key_sz, ctx->n, ctx->dma_n); + if (ctx->e) + dma_free_coherent(dev, ctx->key_sz, ctx->e, ctx->dma_e); + if (ctx->d) { + memset(ctx->d, '\0', ctx->key_sz); + dma_free_coherent(dev, ctx->key_sz, ctx->d, ctx->dma_d); + } + qat_crypto_put_instance(ctx->inst); + ctx->n = NULL; + ctx->d = NULL; + ctx->d = NULL; +} + +static struct akcipher_alg rsa = { + .encrypt = qat_rsa_enc, + .decrypt = qat_rsa_dec, + .sign = qat_rsa_dec, + .verify = qat_rsa_enc, + .set_pub_key = qat_rsa_setpubkey, + .set_priv_key = qat_rsa_setprivkey, + .max_size = qat_rsa_max_size, + .init = qat_rsa_init_tfm, + .exit = qat_rsa_exit_tfm, + .reqsize = sizeof(struct qat_rsa_request) + 64, + .base = { + .cra_name = "rsa", + .cra_driver_name = "qat-rsa", + .cra_priority = 1000, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct qat_rsa_ctx), + }, +}; + +int qat_asym_algs_register(void) +{ + int ret = 0; + + mutex_lock(&algs_lock); + if (++active_devs == 1) { + rsa.base.cra_flags = 0; + ret = crypto_register_akcipher(&rsa); + } + mutex_unlock(&algs_lock); + return ret; +} + +void qat_asym_algs_unregister(void) +{ + mutex_lock(&algs_lock); + if (--active_devs == 0) + crypto_unregister_akcipher(&rsa); + mutex_unlock(&algs_lock); +} diff --git a/kernel/drivers/crypto/qat/qat_common/qat_crypto.c b/kernel/drivers/crypto/qat/qat_common/qat_crypto.c index 3bd705ca5..9cab15497 100644 --- a/kernel/drivers/crypto/qat/qat_common/qat_crypto.c +++ b/kernel/drivers/crypto/qat/qat_common/qat_crypto.c @@ -60,8 +60,8 @@ static struct service_hndl qat_crypto; void qat_crypto_put_instance(struct qat_crypto_instance *inst) { - if (atomic_sub_return(1, &inst->refctr) == 0) - adf_dev_put(inst->accel_dev); + atomic_dec(&inst->refctr); + adf_dev_put(inst->accel_dev); } static int qat_crypto_free_instances(struct adf_accel_dev *accel_dev) @@ -88,12 +88,6 @@ static int qat_crypto_free_instances(struct adf_accel_dev *accel_dev) if (inst->pke_rx) adf_remove_ring(inst->pke_rx); - if (inst->rnd_tx) - adf_remove_ring(inst->rnd_tx); - - if (inst->rnd_rx) - adf_remove_ring(inst->rnd_rx); - list_del(list_ptr); kfree(inst); } @@ -103,47 +97,66 @@ static int qat_crypto_free_instances(struct adf_accel_dev *accel_dev) struct qat_crypto_instance *qat_crypto_get_instance_node(int node) { struct adf_accel_dev *accel_dev = NULL; - struct qat_crypto_instance *inst_best = NULL; + struct qat_crypto_instance *inst = NULL; struct list_head *itr; unsigned long best = ~0; list_for_each(itr, adf_devmgr_get_head()) { - accel_dev = list_entry(itr, struct adf_accel_dev, list); - if ((node == dev_to_node(&GET_DEV(accel_dev)) || - dev_to_node(&GET_DEV(accel_dev)) < 0) && - adf_dev_started(accel_dev)) - break; - accel_dev = NULL; + struct adf_accel_dev *tmp_dev; + unsigned long ctr; + + tmp_dev = list_entry(itr, struct adf_accel_dev, list); + + if ((node == dev_to_node(&GET_DEV(tmp_dev)) || + dev_to_node(&GET_DEV(tmp_dev)) < 0) && + adf_dev_started(tmp_dev) && + !list_empty(&tmp_dev->crypto_list)) { + ctr = atomic_read(&tmp_dev->ref_count); + if (best > ctr) { + accel_dev = tmp_dev; + best = ctr; + } + } } - if (!accel_dev) { - pr_err("QAT: Could not find a device on node %d\n", node); - accel_dev = adf_devmgr_get_first(); + if (!accel_dev) + pr_info("QAT: Could not find a device on node %d\n", node); + + /* Get any started device */ + list_for_each(itr, adf_devmgr_get_head()) { + struct adf_accel_dev *tmp_dev; + + tmp_dev = list_entry(itr, struct adf_accel_dev, list); + + if (adf_dev_started(tmp_dev) && + !list_empty(&tmp_dev->crypto_list)) { + accel_dev = tmp_dev; + break; + } } - if (!accel_dev || !adf_dev_started(accel_dev)) + + if (!accel_dev) return NULL; + best = ~0; list_for_each(itr, &accel_dev->crypto_list) { - struct qat_crypto_instance *inst; - unsigned long cur; - - inst = list_entry(itr, struct qat_crypto_instance, list); - cur = atomic_read(&inst->refctr); - if (best > cur) { - inst_best = inst; - best = cur; + struct qat_crypto_instance *tmp_inst; + unsigned long ctr; + + tmp_inst = list_entry(itr, struct qat_crypto_instance, list); + ctr = atomic_read(&tmp_inst->refctr); + if (best > ctr) { + inst = tmp_inst; + best = ctr; } } - if (inst_best) { - if (atomic_add_return(1, &inst_best->refctr) == 1) { - if (adf_dev_get(accel_dev)) { - atomic_dec(&inst_best->refctr); - dev_err(&GET_DEV(accel_dev), - "Could not increment dev refctr\n"); - return NULL; - } + if (inst) { + if (adf_dev_get(accel_dev)) { + dev_err(&GET_DEV(accel_dev), "Could not increment dev refctr\n"); + return NULL; } + atomic_inc(&inst->refctr); } - return inst_best; + return inst; } static int qat_crypto_create_instances(struct adf_accel_dev *accel_dev) @@ -158,7 +171,6 @@ static int qat_crypto_create_instances(struct adf_accel_dev *accel_dev) INIT_LIST_HEAD(&accel_dev->crypto_list); strlcpy(key, ADF_NUM_CY, sizeof(key)); - if (adf_cfg_get_param_value(accel_dev, SEC, key, val)) return -EFAULT; @@ -187,7 +199,9 @@ static int qat_crypto_create_instances(struct adf_accel_dev *accel_dev) if (kstrtoul(val, 10, &num_msg_sym)) goto err; + num_msg_sym = num_msg_sym >> 1; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, i); if (adf_cfg_get_param_value(accel_dev, SEC, key, val)) goto err; @@ -202,11 +216,6 @@ static int qat_crypto_create_instances(struct adf_accel_dev *accel_dev) msg_size, key, NULL, 0, &inst->sym_tx)) goto err; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_RND_TX, i); - if (adf_create_ring(accel_dev, SEC, bank, num_msg_asym, - msg_size, key, NULL, 0, &inst->rnd_tx)) - goto err; - msg_size = msg_size >> 1; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, i); if (adf_create_ring(accel_dev, SEC, bank, num_msg_asym, @@ -220,15 +229,9 @@ static int qat_crypto_create_instances(struct adf_accel_dev *accel_dev) &inst->sym_rx)) goto err; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_RND_RX, i); - if (adf_create_ring(accel_dev, SEC, bank, num_msg_asym, - msg_size, key, qat_alg_callback, 0, - &inst->rnd_rx)) - goto err; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, i); if (adf_create_ring(accel_dev, SEC, bank, num_msg_asym, - msg_size, key, qat_alg_callback, 0, + msg_size, key, qat_alg_asym_callback, 0, &inst->pke_rx)) goto err; } diff --git a/kernel/drivers/crypto/qat/qat_common/qat_crypto.h b/kernel/drivers/crypto/qat/qat_common/qat_crypto.h index d503007b4..dc0273fe3 100644 --- a/kernel/drivers/crypto/qat/qat_common/qat_crypto.h +++ b/kernel/drivers/crypto/qat/qat_common/qat_crypto.h @@ -57,8 +57,6 @@ struct qat_crypto_instance { struct adf_etr_ring_data *sym_rx; struct adf_etr_ring_data *pke_tx; struct adf_etr_ring_data *pke_rx; - struct adf_etr_ring_data *rnd_tx; - struct adf_etr_ring_data *rnd_rx; struct adf_accel_dev *accel_dev; struct list_head list; unsigned long state; diff --git a/kernel/drivers/crypto/qat/qat_common/qat_hal.c b/kernel/drivers/crypto/qat/qat_common/qat_hal.c index 274ff7e9d..380e76180 100644 --- a/kernel/drivers/crypto/qat/qat_common/qat_hal.c +++ b/kernel/drivers/crypto/qat/qat_common/qat_hal.c @@ -671,7 +671,6 @@ static int qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle) #define ICP_DH895XCC_CAP_OFFSET (ICP_DH895XCC_AE_OFFSET + 0x10000) #define LOCAL_TO_XFER_REG_OFFSET 0x800 #define ICP_DH895XCC_EP_OFFSET 0x3a000 -#define ICP_DH895XCC_PMISC_BAR 1 int qat_hal_init(struct adf_accel_dev *accel_dev) { unsigned char ae; @@ -679,21 +678,24 @@ int qat_hal_init(struct adf_accel_dev *accel_dev) struct icp_qat_fw_loader_handle *handle; struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev; struct adf_hw_device_data *hw_data = accel_dev->hw_device; - struct adf_bar *bar = + struct adf_bar *misc_bar = &pci_info->pci_bars[hw_data->get_misc_bar_id(hw_data)]; + struct adf_bar *sram_bar = + &pci_info->pci_bars[hw_data->get_sram_bar_id(hw_data)]; handle = kzalloc(sizeof(*handle), GFP_KERNEL); if (!handle) return -ENOMEM; - handle->hal_cap_g_ctl_csr_addr_v = bar->virt_addr + + handle->hal_cap_g_ctl_csr_addr_v = misc_bar->virt_addr + ICP_DH895XCC_CAP_OFFSET; - handle->hal_cap_ae_xfer_csr_addr_v = bar->virt_addr + + handle->hal_cap_ae_xfer_csr_addr_v = misc_bar->virt_addr + ICP_DH895XCC_AE_OFFSET; - handle->hal_ep_csr_addr_v = bar->virt_addr + ICP_DH895XCC_EP_OFFSET; + handle->hal_ep_csr_addr_v = misc_bar->virt_addr + + ICP_DH895XCC_EP_OFFSET; handle->hal_cap_ae_local_csr_addr_v = handle->hal_cap_ae_xfer_csr_addr_v + LOCAL_TO_XFER_REG_OFFSET; - + handle->hal_sram_addr_v = sram_bar->virt_addr; handle->hal_handle = kzalloc(sizeof(*handle->hal_handle), GFP_KERNEL); if (!handle->hal_handle) goto out_hal_handle; @@ -1032,7 +1034,7 @@ static int qat_hal_concat_micro_code(uint64_t *micro_inst, unsigned int inst_num, unsigned int size, unsigned int addr, unsigned int *value) { - int i, val_indx; + int i; unsigned int cur_value; const uint64_t *inst_arr; int fixup_offset; @@ -1040,8 +1042,7 @@ static int qat_hal_concat_micro_code(uint64_t *micro_inst, int orig_num; orig_num = inst_num; - val_indx = 0; - cur_value = value[val_indx++]; + cur_value = value[0]; inst_arr = inst_4b; usize = ARRAY_SIZE(inst_4b); fixup_offset = inst_num; diff --git a/kernel/drivers/crypto/qat/qat_common/qat_rsaprivkey.asn1 b/kernel/drivers/crypto/qat/qat_common/qat_rsaprivkey.asn1 new file mode 100644 index 000000000..f0066adb7 --- /dev/null +++ b/kernel/drivers/crypto/qat/qat_common/qat_rsaprivkey.asn1 @@ -0,0 +1,11 @@ +RsaPrivKey ::= SEQUENCE { + version INTEGER, + n INTEGER ({ qat_rsa_get_n }), + e INTEGER ({ qat_rsa_get_e }), + d INTEGER ({ qat_rsa_get_d }), + prime1 INTEGER, + prime2 INTEGER, + exponent1 INTEGER, + exponent2 INTEGER, + coefficient INTEGER +} diff --git a/kernel/drivers/crypto/qat/qat_common/qat_rsapubkey.asn1 b/kernel/drivers/crypto/qat/qat_common/qat_rsapubkey.asn1 new file mode 100644 index 000000000..bd667b31a --- /dev/null +++ b/kernel/drivers/crypto/qat/qat_common/qat_rsapubkey.asn1 @@ -0,0 +1,4 @@ +RsaPubKey ::= SEQUENCE { + n INTEGER ({ qat_rsa_get_n }), + e INTEGER ({ qat_rsa_get_e }) +} diff --git a/kernel/drivers/crypto/qat/qat_common/qat_uclo.c b/kernel/drivers/crypto/qat/qat_common/qat_uclo.c index 1e27f9f7f..c48f181e8 100644 --- a/kernel/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/kernel/drivers/crypto/qat/qat_common/qat_uclo.c @@ -359,28 +359,7 @@ static int qat_uclo_init_umem_seg(struct icp_qat_fw_loader_handle *handle, static int qat_uclo_init_ae_memory(struct icp_qat_fw_loader_handle *handle, struct icp_qat_uof_initmem *init_mem) { - unsigned int i; - struct icp_qat_uof_memvar_attr *mem_val_attr; - - mem_val_attr = - (struct icp_qat_uof_memvar_attr *)((unsigned long)init_mem + - sizeof(struct icp_qat_uof_initmem)); - switch (init_mem->region) { - case ICP_QAT_UOF_SRAM_REGION: - if ((init_mem->addr + init_mem->num_in_bytes) > - ICP_DH895XCC_PESRAM_BAR_SIZE) { - pr_err("QAT: initmem on SRAM is out of range"); - return -EINVAL; - } - for (i = 0; i < init_mem->val_attr_num; i++) { - qat_uclo_wr_sram_by_words(handle, - init_mem->addr + - mem_val_attr->offset_in_byte, - &mem_val_attr->value, 4); - mem_val_attr++; - } - break; case ICP_QAT_UOF_LMEM_REGION: if (qat_uclo_init_lmem_seg(handle, init_mem)) return -EINVAL; @@ -990,6 +969,12 @@ out_err: return -EFAULT; } +void qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle, + void *addr_ptr, int mem_size) +{ + qat_uclo_wr_sram_by_words(handle, 0, addr_ptr, ALIGN(mem_size, 4)); +} + int qat_uclo_map_uof_obj(struct icp_qat_fw_loader_handle *handle, void *addr_ptr, int mem_size) { diff --git a/kernel/drivers/crypto/qat/qat_dh895xcc/Makefile b/kernel/drivers/crypto/qat/qat_dh895xcc/Makefile index 25171c557..8c79c5437 100644 --- a/kernel/drivers/crypto/qat/qat_dh895xcc/Makefile +++ b/kernel/drivers/crypto/qat/qat_dh895xcc/Makefile @@ -2,7 +2,4 @@ ccflags-y := -I$(src)/../qat_common obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCC) += qat_dh895xcc.o qat_dh895xcc-objs := adf_drv.o \ adf_isr.o \ - adf_dh895xcc_hw_data.o \ - adf_hw_arbiter.o \ - qat_admin.o \ - adf_admin.o + adf_dh895xcc_hw_data.o diff --git a/kernel/drivers/crypto/qat/qat_dh895xcc/adf_admin.c b/kernel/drivers/crypto/qat/qat_dh895xcc/adf_admin.c deleted file mode 100644 index e4666065c..000000000 --- a/kernel/drivers/crypto/qat/qat_dh895xcc/adf_admin.c +++ /dev/null @@ -1,145 +0,0 @@ -/* - This file is provided under a dual BSD/GPLv2 license. When using or - redistributing this file, you may do so under either license. - - GPL LICENSE SUMMARY - Copyright(c) 2014 Intel Corporation. - This program is free software; you can redistribute it and/or modify - it under the terms of version 2 of the GNU General Public License as - published by the Free Software Foundation. - - This program is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - - Contact Information: - qat-linux@intel.com - - BSD LICENSE - Copyright(c) 2014 Intel Corporation. - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in - the documentation and/or other materials provided with the - distribution. - * Neither the name of Intel Corporation nor the names of its - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ -#include -#include -#include -#include -#include -#include -#include -#include "adf_drv.h" -#include "adf_dh895xcc_hw_data.h" - -#define ADF_ADMINMSG_LEN 32 - -struct adf_admin_comms { - dma_addr_t phy_addr; - void *virt_addr; - void __iomem *mailbox_addr; - struct mutex lock; /* protects adf_admin_comms struct */ -}; - -int adf_put_admin_msg_sync(struct adf_accel_dev *accel_dev, - uint32_t ae, void *in, void *out) -{ - struct adf_admin_comms *admin = accel_dev->admin; - int offset = ae * ADF_ADMINMSG_LEN * 2; - void __iomem *mailbox = admin->mailbox_addr; - int mb_offset = ae * ADF_DH895XCC_MAILBOX_STRIDE; - int times, received; - - mutex_lock(&admin->lock); - - if (ADF_CSR_RD(mailbox, mb_offset) == 1) { - mutex_unlock(&admin->lock); - return -EAGAIN; - } - - memcpy(admin->virt_addr + offset, in, ADF_ADMINMSG_LEN); - ADF_CSR_WR(mailbox, mb_offset, 1); - received = 0; - for (times = 0; times < 50; times++) { - msleep(20); - if (ADF_CSR_RD(mailbox, mb_offset) == 0) { - received = 1; - break; - } - } - if (received) - memcpy(out, admin->virt_addr + offset + - ADF_ADMINMSG_LEN, ADF_ADMINMSG_LEN); - else - dev_err(&GET_DEV(accel_dev), - "Failed to send admin msg to accelerator\n"); - - mutex_unlock(&admin->lock); - return received ? 0 : -EFAULT; -} - -int adf_init_admin_comms(struct adf_accel_dev *accel_dev) -{ - struct adf_admin_comms *admin; - struct adf_bar *pmisc = &GET_BARS(accel_dev)[ADF_DH895XCC_PMISC_BAR]; - void __iomem *csr = pmisc->virt_addr; - void __iomem *mailbox = csr + ADF_DH895XCC_MAILBOX_BASE_OFFSET; - uint64_t reg_val; - - admin = kzalloc_node(sizeof(*accel_dev->admin), GFP_KERNEL, - dev_to_node(&GET_DEV(accel_dev))); - if (!admin) - return -ENOMEM; - admin->virt_addr = dma_zalloc_coherent(&GET_DEV(accel_dev), PAGE_SIZE, - &admin->phy_addr, GFP_KERNEL); - if (!admin->virt_addr) { - dev_err(&GET_DEV(accel_dev), "Failed to allocate dma buff\n"); - kfree(admin); - return -ENOMEM; - } - reg_val = (uint64_t)admin->phy_addr; - ADF_CSR_WR(csr, ADF_DH895XCC_ADMINMSGUR_OFFSET, reg_val >> 32); - ADF_CSR_WR(csr, ADF_DH895XCC_ADMINMSGLR_OFFSET, reg_val); - mutex_init(&admin->lock); - admin->mailbox_addr = mailbox; - accel_dev->admin = admin; - return 0; -} - -void adf_exit_admin_comms(struct adf_accel_dev *accel_dev) -{ - struct adf_admin_comms *admin = accel_dev->admin; - - if (!admin) - return; - - if (admin->virt_addr) - dma_free_coherent(&GET_DEV(accel_dev), PAGE_SIZE, - admin->virt_addr, admin->phy_addr); - - mutex_destroy(&admin->lock); - kfree(admin); - accel_dev->admin = NULL; -} diff --git a/kernel/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/kernel/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c index b1386922d..ff54257ec 100644 --- a/kernel/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/kernel/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -45,8 +45,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include +#include +#include #include "adf_dh895xcc_hw_data.h" -#include "adf_common_drv.h" #include "adf_drv.h" /* Worker thread to service arbiter mappings based on dev SKUs */ @@ -117,6 +118,11 @@ static uint32_t get_etr_bar_id(struct adf_hw_device_data *self) return ADF_DH895XCC_ETR_BAR; } +static uint32_t get_sram_bar_id(struct adf_hw_device_data *self) +{ + return ADF_DH895XCC_SRAM_BAR; +} + static enum dev_sku_info get_sku(struct adf_hw_device_data *self) { int sku = (self->fuses & ADF_DH895XCC_FUSECTL_SKU_MASK) @@ -156,6 +162,16 @@ void adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev, } } +static uint32_t get_pf2vf_offset(uint32_t i) +{ + return ADF_DH895XCC_PF2VF_OFFSET(i); +} + +static uint32_t get_vintmsk_offset(uint32_t i) +{ + return ADF_DH895XCC_VINTMSK_OFFSET(i); +} + static void adf_enable_error_correction(struct adf_accel_dev *accel_dev) { struct adf_hw_device_data *hw_device = accel_dev->hw_device; @@ -192,18 +208,23 @@ static void adf_enable_ints(struct adf_accel_dev *accel_dev) /* Enable bundle and misc interrupts */ ADF_CSR_WR(addr, ADF_DH895XCC_SMIAPF0_MASK_OFFSET, - ADF_DH895XCC_SMIA0_MASK); + accel_dev->pf.vf_info ? 0 : + GENMASK_ULL(GET_MAX_BANKS(accel_dev) - 1, 0)); ADF_CSR_WR(addr, ADF_DH895XCC_SMIAPF1_MASK_OFFSET, ADF_DH895XCC_SMIA1_MASK); } +static int adf_pf_enable_vf2pf_comms(struct adf_accel_dev *accel_dev) +{ + return 0; +} + void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) { hw_data->dev_class = &dh895xcc_class; hw_data->instance_id = dh895xcc_class.instances++; hw_data->num_banks = ADF_DH895XCC_ETR_MAX_BANKS; hw_data->num_accel = ADF_DH895XCC_MAX_ACCELERATORS; - hw_data->pci_dev_id = ADF_DH895XCC_PCI_DEVICE_ID; hw_data->num_logical_accel = 1; hw_data->num_engines = ADF_DH895XCC_MAX_ACCELENGINES; hw_data->tx_rx_gap = ADF_DH895XCC_RX_RINGS_OFFSET; @@ -211,21 +232,28 @@ void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) hw_data->alloc_irq = adf_isr_resource_alloc; hw_data->free_irq = adf_isr_resource_free; hw_data->enable_error_correction = adf_enable_error_correction; - hw_data->hw_arb_ring_enable = adf_update_ring_arb_enable; - hw_data->hw_arb_ring_disable = adf_update_ring_arb_enable; hw_data->get_accel_mask = get_accel_mask; hw_data->get_ae_mask = get_ae_mask; hw_data->get_num_accels = get_num_accels; hw_data->get_num_aes = get_num_aes; hw_data->get_etr_bar_id = get_etr_bar_id; hw_data->get_misc_bar_id = get_misc_bar_id; + hw_data->get_pf2vf_offset = get_pf2vf_offset; + hw_data->get_vintmsk_offset = get_vintmsk_offset; + hw_data->get_sram_bar_id = get_sram_bar_id; hw_data->get_sku = get_sku; hw_data->fw_name = ADF_DH895XCC_FW; + hw_data->fw_mmp_name = ADF_DH895XCC_MMP; hw_data->init_admin_comms = adf_init_admin_comms; hw_data->exit_admin_comms = adf_exit_admin_comms; + hw_data->disable_iov = adf_disable_sriov; + hw_data->send_admin_init = adf_send_admin_init; hw_data->init_arb = adf_init_arb; hw_data->exit_arb = adf_exit_arb; + hw_data->get_arb_mapping = adf_get_arbiter_mapping; hw_data->enable_ints = adf_enable_ints; + hw_data->enable_vf2pf_comms = adf_pf_enable_vf2pf_comms; + hw_data->min_iov_compat_ver = ADF_PFVF_COMPATIBILITY_VERSION; } void adf_clean_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) diff --git a/kernel/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h b/kernel/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h index 25269a9f2..88dffb297 100644 --- a/kernel/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h +++ b/kernel/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h @@ -48,6 +48,7 @@ #define ADF_DH895x_HW_DATA_H_ /* PCIe configuration space */ +#define ADF_DH895XCC_SRAM_BAR 0 #define ADF_DH895XCC_PMISC_BAR 1 #define ADF_DH895XCC_ETR_BAR 2 #define ADF_DH895XCC_RX_RINGS_OFFSET 8 @@ -79,10 +80,11 @@ #define ADF_DH895XCC_CERRSSMSH(i) (i * 0x4000 + 0x10) #define ADF_DH895XCC_ERRSSMSH_EN BIT(3) -/* Admin Messages Registers */ -#define ADF_DH895XCC_ADMINMSGUR_OFFSET (0x3A000 + 0x574) -#define ADF_DH895XCC_ADMINMSGLR_OFFSET (0x3A000 + 0x578) -#define ADF_DH895XCC_MAILBOX_BASE_OFFSET 0x20970 -#define ADF_DH895XCC_MAILBOX_STRIDE 0x1000 +#define ADF_DH895XCC_ERRSOU3 (0x3A000 + 0x00C) +#define ADF_DH895XCC_ERRSOU5 (0x3A000 + 0x0D8) +#define ADF_DH895XCC_PF2VF_OFFSET(i) (0x3A000 + 0x280 + ((i) * 0x04)) +#define ADF_DH895XCC_VINTMSK_OFFSET(i) (0x3A000 + 0x200 + ((i) * 0x04)) +/* FW names */ #define ADF_DH895XCC_FW "qat_895xcc.bin" +#define ADF_DH895XCC_MMP "qat_mmp.bin" #endif diff --git a/kernel/drivers/crypto/qat/qat_dh895xcc/adf_drv.c b/kernel/drivers/crypto/qat/qat_dh895xcc/adf_drv.c index 9decea277..f8dd14f23 100644 --- a/kernel/drivers/crypto/qat/qat_dh895xcc/adf_drv.c +++ b/kernel/drivers/crypto/qat/qat_dh895xcc/adf_drv.c @@ -82,16 +82,21 @@ static struct pci_driver adf_driver = { .id_table = adf_pci_tbl, .name = adf_driver_name, .probe = adf_probe, - .remove = adf_remove + .remove = adf_remove, + .sriov_configure = adf_sriov_configure, }; +static void adf_cleanup_pci_dev(struct adf_accel_dev *accel_dev) +{ + pci_release_regions(accel_dev->accel_pci_dev.pci_dev); + pci_disable_device(accel_dev->accel_pci_dev.pci_dev); +} + static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) { struct adf_accel_pci *accel_pci_dev = &accel_dev->accel_pci_dev; int i; - adf_dev_shutdown(accel_dev); - for (i = 0; i < ADF_PCI_MAX_BARS; i++) { struct adf_bar *bar = &accel_pci_dev->pci_bars[i]; @@ -100,7 +105,7 @@ static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) } if (accel_dev->hw_device) { - switch (accel_dev->hw_device->pci_dev_id) { + switch (accel_pci_dev->pci_dev->device) { case ADF_DH895XCC_PCI_DEVICE_ID: adf_clean_hw_data_dh895xcc(accel_dev->hw_device); break; @@ -108,13 +113,11 @@ static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) break; } kfree(accel_dev->hw_device); + accel_dev->hw_device = NULL; } adf_cfg_dev_remove(accel_dev); debugfs_remove(accel_dev->debugfs_dir); - adf_devmgr_rm_dev(accel_dev); - pci_release_regions(accel_pci_dev->pci_dev); - pci_disable_device(accel_pci_dev->pci_dev); - kfree(accel_dev); + adf_devmgr_rm_dev(accel_dev, NULL); } static int adf_dev_configure(struct adf_accel_dev *accel_dev) @@ -167,12 +170,6 @@ static int adf_dev_configure(struct adf_accel_dev *accel_dev) key, (void *)&val, ADF_DEC)) goto err; - val = 4; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_RND_TX, i); - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, (void *)&val, ADF_DEC)) - goto err; - val = 8; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, i); if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, @@ -185,12 +182,6 @@ static int adf_dev_configure(struct adf_accel_dev *accel_dev) key, (void *)&val, ADF_DEC)) goto err; - val = 12; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_RND_RX, i); - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, (void *)&val, ADF_DEC)) - goto err; - val = ADF_COALESCING_DEF_TIME; snprintf(key, sizeof(key), ADF_ETRMGR_COALESCE_TIMER_FORMAT, i); if (adf_cfg_add_key_value_param(accel_dev, "Accelerator0", @@ -217,7 +208,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) struct adf_hw_device_data *hw_data; char name[ADF_DEVICE_NAME_LENGTH]; unsigned int i, bar_nr; - int ret; + int ret, bar_mask; switch (ent->device) { case ADF_DH895XCC_PCI_DEVICE_ID: @@ -241,10 +232,12 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) return -ENOMEM; INIT_LIST_HEAD(&accel_dev->crypto_list); + accel_pci_dev = &accel_dev->accel_pci_dev; + accel_pci_dev->pci_dev = pdev; /* Add accel device to accel table. * This should be called before adf_cleanup_accel is called */ - if (adf_devmgr_add_dev(accel_dev)) { + if (adf_devmgr_add_dev(accel_dev, NULL)) { dev_err(&pdev->dev, "Failed to add new accelerator device.\n"); kfree(accel_dev); return -EFAULT; @@ -267,7 +260,6 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) default: return -ENODEV; } - accel_pci_dev = &accel_dev->accel_pci_dev; pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); pci_read_config_dword(pdev, ADF_DH895XCC_FUSECTL_OFFSET, &hw_data->fuses); @@ -276,7 +268,6 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) hw_data->accel_mask = hw_data->get_accel_mask(hw_data->fuses); hw_data->ae_mask = hw_data->get_ae_mask(hw_data->fuses); accel_pci_dev->sku = hw_data->get_sku(hw_data); - accel_pci_dev->pci_dev = pdev; /* If the device has no acceleration engines then ignore it. */ if (!hw_data->accel_mask || !hw_data->ae_mask || ((~hw_data->ae_mask) & 0x01)) { @@ -286,11 +277,14 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) } /* Create dev top level debugfs entry */ - snprintf(name, sizeof(name), "%s%s_dev%d", ADF_DEVICE_NAME_PREFIX, - hw_data->dev_class->name, hw_data->instance_id); + snprintf(name, sizeof(name), "%s%s_%02x:%02d.%02d", + ADF_DEVICE_NAME_PREFIX, hw_data->dev_class->name, + pdev->bus->number, PCI_SLOT(pdev->devfn), + PCI_FUNC(pdev->devfn)); + accel_dev->debugfs_dir = debugfs_create_dir(name, NULL); if (!accel_dev->debugfs_dir) { - dev_err(&pdev->dev, "Could not create debugfs dir\n"); + dev_err(&pdev->dev, "Could not create debugfs dir %s\n", name); ret = -EINVAL; goto out_err; } @@ -300,6 +294,8 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if (ret) goto out_err; + pcie_set_readrq(pdev, 1024); + /* enable PCI device */ if (pci_enable_device(pdev)) { ret = -EFAULT; @@ -311,7 +307,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if ((pci_set_dma_mask(pdev, DMA_BIT_MASK(32)))) { dev_err(&pdev->dev, "No usable DMA configuration\n"); ret = -EFAULT; - goto out_err; + goto out_err_disable; } else { pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); } @@ -322,7 +318,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if (pci_request_regions(pdev, adf_driver_name)) { ret = -EFAULT; - goto out_err; + goto out_err_disable; } /* Read accelerator capabilities mask */ @@ -330,19 +326,21 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) &hw_data->accel_capabilities_mask); /* Find and map all the device's BARS */ - for (i = 0; i < ADF_PCI_MAX_BARS; i++) { - struct adf_bar *bar = &accel_pci_dev->pci_bars[i]; + i = 0; + bar_mask = pci_select_bars(pdev, IORESOURCE_MEM); + for_each_set_bit(bar_nr, (const unsigned long *)&bar_mask, + ADF_PCI_MAX_BARS * 2) { + struct adf_bar *bar = &accel_pci_dev->pci_bars[i++]; - bar_nr = i * 2; bar->base_addr = pci_resource_start(pdev, bar_nr); if (!bar->base_addr) break; bar->size = pci_resource_len(pdev, bar_nr); bar->virt_addr = pci_iomap(accel_pci_dev->pci_dev, bar_nr, 0); if (!bar->virt_addr) { - dev_err(&pdev->dev, "Failed to map BAR %d\n", i); + dev_err(&pdev->dev, "Failed to map BAR %d\n", bar_nr); ret = -EFAULT; - goto out_err; + goto out_err_free_reg; } } pci_set_master(pdev); @@ -350,32 +348,40 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if (adf_enable_aer(accel_dev, &adf_driver)) { dev_err(&pdev->dev, "Failed to enable aer\n"); ret = -EFAULT; - goto out_err; + goto out_err_free_reg; } if (pci_save_state(pdev)) { dev_err(&pdev->dev, "Failed to save pci state\n"); ret = -ENOMEM; - goto out_err; + goto out_err_free_reg; } ret = adf_dev_configure(accel_dev); if (ret) - goto out_err; + goto out_err_free_reg; ret = adf_dev_init(accel_dev); if (ret) - goto out_err; + goto out_err_dev_shutdown; ret = adf_dev_start(accel_dev); - if (ret) { - adf_dev_stop(accel_dev); - goto out_err; - } + if (ret) + goto out_err_dev_stop; - return 0; + return ret; + +out_err_dev_stop: + adf_dev_stop(accel_dev); +out_err_dev_shutdown: + adf_dev_shutdown(accel_dev); +out_err_free_reg: + pci_release_regions(accel_pci_dev->pci_dev); +out_err_disable: + pci_disable_device(accel_pci_dev->pci_dev); out_err: adf_cleanup_accel(accel_dev); + kfree(accel_dev); return ret; } @@ -389,15 +395,17 @@ static void adf_remove(struct pci_dev *pdev) } if (adf_dev_stop(accel_dev)) dev_err(&GET_DEV(accel_dev), "Failed to stop QAT accel dev\n"); + + adf_dev_shutdown(accel_dev); adf_disable_aer(accel_dev); adf_cleanup_accel(accel_dev); + adf_cleanup_pci_dev(accel_dev); + kfree(accel_dev); } static int __init adfdrv_init(void) { request_module("intel_qat"); - if (qat_admin_register()) - return -EFAULT; if (pci_register_driver(&adf_driver)) { pr_err("QAT: Driver initialization failed\n"); @@ -409,7 +417,6 @@ static int __init adfdrv_init(void) static void __exit adfdrv_release(void) { pci_unregister_driver(&adf_driver); - qat_admin_unregister(); } module_init(adfdrv_init); @@ -417,5 +424,6 @@ module_exit(adfdrv_release); MODULE_LICENSE("Dual BSD/GPL"); MODULE_AUTHOR("Intel"); -MODULE_FIRMWARE("qat_895xcc.bin"); +MODULE_FIRMWARE(ADF_DH895XCC_FW); MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); +MODULE_VERSION(ADF_DRV_VERSION); diff --git a/kernel/drivers/crypto/qat/qat_dh895xcc/adf_drv.h b/kernel/drivers/crypto/qat/qat_dh895xcc/adf_drv.h index a2fbb6ce7..85ff245bd 100644 --- a/kernel/drivers/crypto/qat/qat_dh895xcc/adf_drv.h +++ b/kernel/drivers/crypto/qat/qat_dh895xcc/adf_drv.h @@ -53,15 +53,6 @@ void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data); void adf_clean_hw_data_dh895xcc(struct adf_hw_device_data *hw_data); int adf_isr_resource_alloc(struct adf_accel_dev *accel_dev); void adf_isr_resource_free(struct adf_accel_dev *accel_dev); -void adf_update_ring_arb_enable(struct adf_etr_ring_data *ring); void adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev, uint32_t const **arb_map_config); -int adf_init_admin_comms(struct adf_accel_dev *accel_dev); -void adf_exit_admin_comms(struct adf_accel_dev *accel_dev); -int adf_put_admin_msg_sync(struct adf_accel_dev *accel_dev, - uint32_t ae, void *in, void *out); -int qat_admin_register(void); -int qat_admin_unregister(void); -int adf_init_arb(struct adf_accel_dev *accel_dev); -void adf_exit_arb(struct adf_accel_dev *accel_dev); #endif diff --git a/kernel/drivers/crypto/qat/qat_dh895xcc/adf_hw_arbiter.c b/kernel/drivers/crypto/qat/qat_dh895xcc/adf_hw_arbiter.c deleted file mode 100644 index 1864bdb36..000000000 --- a/kernel/drivers/crypto/qat/qat_dh895xcc/adf_hw_arbiter.c +++ /dev/null @@ -1,159 +0,0 @@ -/* - This file is provided under a dual BSD/GPLv2 license. When using or - redistributing this file, you may do so under either license. - - GPL LICENSE SUMMARY - Copyright(c) 2014 Intel Corporation. - This program is free software; you can redistribute it and/or modify - it under the terms of version 2 of the GNU General Public License as - published by the Free Software Foundation. - - This program is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - - Contact Information: - qat-linux@intel.com - - BSD LICENSE - Copyright(c) 2014 Intel Corporation. - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in - the documentation and/or other materials provided with the - distribution. - * Neither the name of Intel Corporation nor the names of its - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ -#include -#include -#include "adf_drv.h" - -#define ADF_ARB_NUM 4 -#define ADF_ARB_REQ_RING_NUM 8 -#define ADF_ARB_REG_SIZE 0x4 -#define ADF_ARB_WTR_SIZE 0x20 -#define ADF_ARB_OFFSET 0x30000 -#define ADF_ARB_REG_SLOT 0x1000 -#define ADF_ARB_WTR_OFFSET 0x010 -#define ADF_ARB_RO_EN_OFFSET 0x090 -#define ADF_ARB_WQCFG_OFFSET 0x100 -#define ADF_ARB_WRK_2_SER_MAP_OFFSET 0x180 -#define ADF_ARB_WRK_2_SER_MAP 10 -#define ADF_ARB_RINGSRVARBEN_OFFSET 0x19C - -#define WRITE_CSR_ARB_RINGSRVARBEN(csr_addr, index, value) \ - ADF_CSR_WR(csr_addr, ADF_ARB_RINGSRVARBEN_OFFSET + \ - (ADF_ARB_REG_SLOT * index), value) - -#define WRITE_CSR_ARB_RESPORDERING(csr_addr, index, value) \ - ADF_CSR_WR(csr_addr, (ADF_ARB_OFFSET + \ - ADF_ARB_RO_EN_OFFSET) + (ADF_ARB_REG_SIZE * index), value) - -#define WRITE_CSR_ARB_WEIGHT(csr_addr, arb, index, value) \ - ADF_CSR_WR(csr_addr, (ADF_ARB_OFFSET + \ - ADF_ARB_WTR_OFFSET) + (ADF_ARB_WTR_SIZE * arb) + \ - (ADF_ARB_REG_SIZE * index), value) - -#define WRITE_CSR_ARB_SARCONFIG(csr_addr, index, value) \ - ADF_CSR_WR(csr_addr, ADF_ARB_OFFSET + \ - (ADF_ARB_REG_SIZE * index), value) - -#define WRITE_CSR_ARB_WRK_2_SER_MAP(csr_addr, index, value) \ - ADF_CSR_WR(csr_addr, (ADF_ARB_OFFSET + \ - ADF_ARB_WRK_2_SER_MAP_OFFSET) + \ - (ADF_ARB_REG_SIZE * index), value) - -#define WRITE_CSR_ARB_WQCFG(csr_addr, index, value) \ - ADF_CSR_WR(csr_addr, (ADF_ARB_OFFSET + \ - ADF_ARB_WQCFG_OFFSET) + (ADF_ARB_REG_SIZE * index), value) - -int adf_init_arb(struct adf_accel_dev *accel_dev) -{ - void __iomem *csr = accel_dev->transport->banks[0].csr_addr; - uint32_t arb_cfg = 0x1 << 31 | 0x4 << 4 | 0x1; - uint32_t arb, i; - const uint32_t *thd_2_arb_cfg; - - /* Service arb configured for 32 bytes responses and - * ring flow control check enabled. */ - for (arb = 0; arb < ADF_ARB_NUM; arb++) - WRITE_CSR_ARB_SARCONFIG(csr, arb, arb_cfg); - - /* Setup service weighting */ - for (arb = 0; arb < ADF_ARB_NUM; arb++) - for (i = 0; i < ADF_ARB_REQ_RING_NUM; i++) - WRITE_CSR_ARB_WEIGHT(csr, arb, i, 0xFFFFFFFF); - - /* Setup ring response ordering */ - for (i = 0; i < ADF_ARB_REQ_RING_NUM; i++) - WRITE_CSR_ARB_RESPORDERING(csr, i, 0xFFFFFFFF); - - /* Setup worker queue registers */ - for (i = 0; i < ADF_ARB_WRK_2_SER_MAP; i++) - WRITE_CSR_ARB_WQCFG(csr, i, i); - - /* Map worker threads to service arbiters */ - adf_get_arbiter_mapping(accel_dev, &thd_2_arb_cfg); - - if (!thd_2_arb_cfg) - return -EFAULT; - - for (i = 0; i < ADF_ARB_WRK_2_SER_MAP; i++) - WRITE_CSR_ARB_WRK_2_SER_MAP(csr, i, *(thd_2_arb_cfg + i)); - - return 0; -} - -void adf_update_ring_arb_enable(struct adf_etr_ring_data *ring) -{ - WRITE_CSR_ARB_RINGSRVARBEN(ring->bank->csr_addr, - ring->bank->bank_number, - ring->bank->ring_mask & 0xFF); -} - -void adf_exit_arb(struct adf_accel_dev *accel_dev) -{ - void __iomem *csr; - unsigned int i; - - if (!accel_dev->transport) - return; - - csr = accel_dev->transport->banks[0].csr_addr; - - /* Reset arbiter configuration */ - for (i = 0; i < ADF_ARB_NUM; i++) - WRITE_CSR_ARB_SARCONFIG(csr, i, 0); - - /* Shutdown work queue */ - for (i = 0; i < ADF_ARB_WRK_2_SER_MAP; i++) - WRITE_CSR_ARB_WQCFG(csr, i, 0); - - /* Unmap worker threads to service arbiters */ - for (i = 0; i < ADF_ARB_WRK_2_SER_MAP; i++) - WRITE_CSR_ARB_WRK_2_SER_MAP(csr, i, 0); - - /* Disable arbitration on all rings */ - for (i = 0; i < GET_MAX_BANKS(accel_dev); i++) - WRITE_CSR_ARB_RINGSRVARBEN(csr, i, 0); -} diff --git a/kernel/drivers/crypto/qat/qat_dh895xcc/adf_isr.c b/kernel/drivers/crypto/qat/qat_dh895xcc/adf_isr.c index 0d03c109c..5570f7879 100644 --- a/kernel/drivers/crypto/qat/qat_dh895xcc/adf_isr.c +++ b/kernel/drivers/crypto/qat/qat_dh895xcc/adf_isr.c @@ -59,21 +59,30 @@ #include #include #include "adf_drv.h" +#include "adf_dh895xcc_hw_data.h" static int adf_enable_msix(struct adf_accel_dev *accel_dev) { struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev; struct adf_hw_device_data *hw_data = accel_dev->hw_device; - uint32_t msix_num_entries = hw_data->num_banks + 1; - int i; - - for (i = 0; i < msix_num_entries; i++) - pci_dev_info->msix_entries.entries[i].entry = i; + u32 msix_num_entries = 1; + + /* If SR-IOV is disabled, add entries for each bank */ + if (!accel_dev->pf.vf_info) { + int i; + + msix_num_entries += hw_data->num_banks; + for (i = 0; i < msix_num_entries; i++) + pci_dev_info->msix_entries.entries[i].entry = i; + } else { + pci_dev_info->msix_entries.entries[0].entry = + hw_data->num_banks; + } if (pci_enable_msix_exact(pci_dev_info->pci_dev, pci_dev_info->msix_entries.entries, msix_num_entries)) { - dev_err(&GET_DEV(accel_dev), "Failed to enable MSIX IRQ\n"); + dev_err(&GET_DEV(accel_dev), "Failed to enable MSI-X IRQ(s)\n"); return -EFAULT; } return 0; @@ -97,9 +106,58 @@ static irqreturn_t adf_msix_isr_ae(int irq, void *dev_ptr) { struct adf_accel_dev *accel_dev = dev_ptr; - dev_info(&GET_DEV(accel_dev), "qat_dev%d spurious AE interrupt\n", - accel_dev->accel_id); - return IRQ_HANDLED; +#ifdef CONFIG_PCI_IOV + /* If SR-IOV is enabled (vf_info is non-NULL), check for VF->PF ints */ + if (accel_dev->pf.vf_info) { + void __iomem *pmisc_bar_addr = + (&GET_BARS(accel_dev)[ADF_DH895XCC_PMISC_BAR])->virt_addr; + u32 vf_mask; + + /* Get the interrupt sources triggered by VFs */ + vf_mask = ((ADF_CSR_RD(pmisc_bar_addr, ADF_DH895XCC_ERRSOU5) & + 0x0000FFFF) << 16) | + ((ADF_CSR_RD(pmisc_bar_addr, ADF_DH895XCC_ERRSOU3) & + 0x01FFFE00) >> 9); + + if (vf_mask) { + struct adf_accel_vf_info *vf_info; + bool irq_handled = false; + int i; + + /* Disable VF2PF interrupts for VFs with pending ints */ + adf_disable_vf2pf_interrupts(accel_dev, vf_mask); + + /* + * Schedule tasklets to handle VF2PF interrupt BHs + * unless the VF is malicious and is attempting to + * flood the host OS with VF2PF interrupts. + */ + for_each_set_bit(i, (const unsigned long *)&vf_mask, + (sizeof(vf_mask) * BITS_PER_BYTE)) { + vf_info = accel_dev->pf.vf_info + i; + + if (!__ratelimit(&vf_info->vf2pf_ratelimit)) { + dev_info(&GET_DEV(accel_dev), + "Too many ints from VF%d\n", + vf_info->vf_nr + 1); + continue; + } + + /* Tasklet will re-enable ints from this VF */ + tasklet_hi_schedule(&vf_info->vf2pf_bh_tasklet); + irq_handled = true; + } + + if (irq_handled) + return IRQ_HANDLED; + } + } +#endif /* CONFIG_PCI_IOV */ + + dev_dbg(&GET_DEV(accel_dev), "qat_dev%d spurious AE interrupt\n", + accel_dev->accel_id); + + return IRQ_NONE; } static int adf_request_irqs(struct adf_accel_dev *accel_dev) @@ -108,28 +166,32 @@ static int adf_request_irqs(struct adf_accel_dev *accel_dev) struct adf_hw_device_data *hw_data = accel_dev->hw_device; struct msix_entry *msixe = pci_dev_info->msix_entries.entries; struct adf_etr_data *etr_data = accel_dev->transport; - int ret, i; + int ret, i = 0; char *name; - /* Request msix irq for all banks */ - for (i = 0; i < hw_data->num_banks; i++) { - struct adf_etr_bank_data *bank = &etr_data->banks[i]; - unsigned int cpu, cpus = num_online_cpus(); - - name = *(pci_dev_info->msix_entries.names + i); - snprintf(name, ADF_MAX_MSIX_VECTOR_NAME, - "qat%d-bundle%d", accel_dev->accel_id, i); - ret = request_irq(msixe[i].vector, - adf_msix_isr_bundle, 0, name, bank); - if (ret) { - dev_err(&GET_DEV(accel_dev), - "failed to enable irq %d for %s\n", - msixe[i].vector, name); - return ret; + /* Request msix irq for all banks unless SR-IOV enabled */ + if (!accel_dev->pf.vf_info) { + for (i = 0; i < hw_data->num_banks; i++) { + struct adf_etr_bank_data *bank = &etr_data->banks[i]; + unsigned int cpu, cpus = num_online_cpus(); + + name = *(pci_dev_info->msix_entries.names + i); + snprintf(name, ADF_MAX_MSIX_VECTOR_NAME, + "qat%d-bundle%d", accel_dev->accel_id, i); + ret = request_irq(msixe[i].vector, + adf_msix_isr_bundle, 0, name, bank); + if (ret) { + dev_err(&GET_DEV(accel_dev), + "failed to enable irq %d for %s\n", + msixe[i].vector, name); + return ret; + } + + cpu = ((accel_dev->accel_id * hw_data->num_banks) + + i) % cpus; + irq_set_affinity_hint(msixe[i].vector, + get_cpu_mask(cpu)); } - - cpu = ((accel_dev->accel_id * hw_data->num_banks) + i) % cpus; - irq_set_affinity_hint(msixe[i].vector, get_cpu_mask(cpu)); } /* Request msix irq for AE */ @@ -152,11 +214,13 @@ static void adf_free_irqs(struct adf_accel_dev *accel_dev) struct adf_hw_device_data *hw_data = accel_dev->hw_device; struct msix_entry *msixe = pci_dev_info->msix_entries.entries; struct adf_etr_data *etr_data = accel_dev->transport; - int i; + int i = 0; - for (i = 0; i < hw_data->num_banks; i++) { - irq_set_affinity_hint(msixe[i].vector, NULL); - free_irq(msixe[i].vector, &etr_data->banks[i]); + if (pci_dev_info->msix_entries.num_entries > 1) { + for (i = 0; i < hw_data->num_banks; i++) { + irq_set_affinity_hint(msixe[i].vector, NULL); + free_irq(msixe[i].vector, &etr_data->banks[i]); + } } irq_set_affinity_hint(msixe[i].vector, NULL); free_irq(msixe[i].vector, accel_dev); @@ -168,7 +232,11 @@ static int adf_isr_alloc_msix_entry_table(struct adf_accel_dev *accel_dev) char **names; struct msix_entry *entries; struct adf_hw_device_data *hw_data = accel_dev->hw_device; - uint32_t msix_num_entries = hw_data->num_banks + 1; + u32 msix_num_entries = 1; + + /* If SR-IOV is disabled (vf_info is NULL), add entries for each bank */ + if (!accel_dev->pf.vf_info) + msix_num_entries += hw_data->num_banks; entries = kzalloc_node(msix_num_entries * sizeof(*entries), GFP_KERNEL, dev_to_node(&GET_DEV(accel_dev))); @@ -185,6 +253,7 @@ static int adf_isr_alloc_msix_entry_table(struct adf_accel_dev *accel_dev) if (!(*(names + i))) goto err; } + accel_dev->accel_pci_dev.msix_entries.num_entries = msix_num_entries; accel_dev->accel_pci_dev.msix_entries.entries = entries; accel_dev->accel_pci_dev.msix_entries.names = names; return 0; @@ -198,13 +267,11 @@ err: static void adf_isr_free_msix_entry_table(struct adf_accel_dev *accel_dev) { - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - uint32_t msix_num_entries = hw_data->num_banks + 1; char **names = accel_dev->accel_pci_dev.msix_entries.names; int i; kfree(accel_dev->accel_pci_dev.msix_entries.entries); - for (i = 0; i < msix_num_entries; i++) + for (i = 0; i < accel_dev->accel_pci_dev.msix_entries.num_entries; i++) kfree(*(names + i)); kfree(names); } diff --git a/kernel/drivers/crypto/qat/qat_dh895xcc/qat_admin.c b/kernel/drivers/crypto/qat/qat_dh895xcc/qat_admin.c deleted file mode 100644 index 55b7a8e48..000000000 --- a/kernel/drivers/crypto/qat/qat_dh895xcc/qat_admin.c +++ /dev/null @@ -1,107 +0,0 @@ -/* - This file is provided under a dual BSD/GPLv2 license. When using or - redistributing this file, you may do so under either license. - - GPL LICENSE SUMMARY - Copyright(c) 2014 Intel Corporation. - This program is free software; you can redistribute it and/or modify - it under the terms of version 2 of the GNU General Public License as - published by the Free Software Foundation. - - This program is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - - Contact Information: - qat-linux@intel.com - - BSD LICENSE - Copyright(c) 2014 Intel Corporation. - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in - the documentation and/or other materials provided with the - distribution. - * Neither the name of Intel Corporation nor the names of its - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ -#include -#include -#include -#include "adf_drv.h" - -static struct service_hndl qat_admin; - -static int qat_send_admin_cmd(struct adf_accel_dev *accel_dev, int cmd) -{ - struct adf_hw_device_data *hw_device = accel_dev->hw_device; - struct icp_qat_fw_init_admin_req req; - struct icp_qat_fw_init_admin_resp resp; - int i; - - memset(&req, 0, sizeof(struct icp_qat_fw_init_admin_req)); - req.init_admin_cmd_id = cmd; - for (i = 0; i < hw_device->get_num_aes(hw_device); i++) { - memset(&resp, 0, sizeof(struct icp_qat_fw_init_admin_resp)); - if (adf_put_admin_msg_sync(accel_dev, i, &req, &resp) || - resp.init_resp_hdr.status) - return -EFAULT; - } - return 0; -} - -static int qat_admin_start(struct adf_accel_dev *accel_dev) -{ - return qat_send_admin_cmd(accel_dev, ICP_QAT_FW_INIT_ME); -} - -static int qat_admin_event_handler(struct adf_accel_dev *accel_dev, - enum adf_event event) -{ - int ret; - - switch (event) { - case ADF_EVENT_START: - ret = qat_admin_start(accel_dev); - break; - case ADF_EVENT_STOP: - case ADF_EVENT_INIT: - case ADF_EVENT_SHUTDOWN: - default: - ret = 0; - } - return ret; -} - -int qat_admin_register(void) -{ - memset(&qat_admin, 0, sizeof(struct service_hndl)); - qat_admin.event_hld = qat_admin_event_handler; - qat_admin.name = "qat_admin"; - qat_admin.admin = 1; - return adf_service_register(&qat_admin); -} - -int qat_admin_unregister(void) -{ - return adf_service_unregister(&qat_admin); -} diff --git a/kernel/drivers/crypto/qat/qat_dh895xccvf/Makefile b/kernel/drivers/crypto/qat/qat_dh895xccvf/Makefile new file mode 100644 index 000000000..85399fcbb --- /dev/null +++ b/kernel/drivers/crypto/qat/qat_dh895xccvf/Makefile @@ -0,0 +1,5 @@ +ccflags-y := -I$(src)/../qat_common +obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCCVF) += qat_dh895xccvf.o +qat_dh895xccvf-objs := adf_drv.o \ + adf_isr.o \ + adf_dh895xccvf_hw_data.o diff --git a/kernel/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c b/kernel/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c new file mode 100644 index 000000000..a9a27eff4 --- /dev/null +++ b/kernel/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c @@ -0,0 +1,172 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2015 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2015 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#include +#include +#include +#include "adf_dh895xccvf_hw_data.h" +#include "adf_drv.h" + +static struct adf_hw_device_class dh895xcciov_class = { + .name = ADF_DH895XCCVF_DEVICE_NAME, + .type = DEV_DH895XCCVF, + .instances = 0 +}; + +static u32 get_accel_mask(u32 fuse) +{ + return ADF_DH895XCCIOV_ACCELERATORS_MASK; +} + +static u32 get_ae_mask(u32 fuse) +{ + return ADF_DH895XCCIOV_ACCELENGINES_MASK; +} + +static u32 get_num_accels(struct adf_hw_device_data *self) +{ + return ADF_DH895XCCIOV_MAX_ACCELERATORS; +} + +static u32 get_num_aes(struct adf_hw_device_data *self) +{ + return ADF_DH895XCCIOV_MAX_ACCELENGINES; +} + +static u32 get_misc_bar_id(struct adf_hw_device_data *self) +{ + return ADF_DH895XCCIOV_PMISC_BAR; +} + +static u32 get_etr_bar_id(struct adf_hw_device_data *self) +{ + return ADF_DH895XCCIOV_ETR_BAR; +} + +static enum dev_sku_info get_sku(struct adf_hw_device_data *self) +{ + return DEV_SKU_VF; +} + +static u32 get_pf2vf_offset(u32 i) +{ + return ADF_DH895XCCIOV_PF2VF_OFFSET; +} + +static u32 get_vintmsk_offset(u32 i) +{ + return ADF_DH895XCCIOV_VINTMSK_OFFSET; +} + +static int adf_vf_int_noop(struct adf_accel_dev *accel_dev) +{ + return 0; +} + +static void adf_vf_void_noop(struct adf_accel_dev *accel_dev) +{ +} + +static int adf_vf2pf_init(struct adf_accel_dev *accel_dev) +{ + u32 msg = (ADF_VF2PF_MSGORIGIN_SYSTEM | + (ADF_VF2PF_MSGTYPE_INIT << ADF_VF2PF_MSGTYPE_SHIFT)); + + if (adf_iov_putmsg(accel_dev, msg, 0)) { + dev_err(&GET_DEV(accel_dev), + "Failed to send Init event to PF\n"); + return -EFAULT; + } + return 0; +} + +static void adf_vf2pf_shutdown(struct adf_accel_dev *accel_dev) +{ + u32 msg = (ADF_VF2PF_MSGORIGIN_SYSTEM | + (ADF_VF2PF_MSGTYPE_SHUTDOWN << ADF_VF2PF_MSGTYPE_SHIFT)); + + if (adf_iov_putmsg(accel_dev, msg, 0)) + dev_err(&GET_DEV(accel_dev), + "Failed to send Shutdown event to PF\n"); +} + +void adf_init_hw_data_dh895xcciov(struct adf_hw_device_data *hw_data) +{ + hw_data->dev_class = &dh895xcciov_class; + hw_data->instance_id = dh895xcciov_class.instances++; + hw_data->num_banks = ADF_DH895XCCIOV_ETR_MAX_BANKS; + hw_data->num_accel = ADF_DH895XCCIOV_MAX_ACCELERATORS; + hw_data->num_logical_accel = 1; + hw_data->num_engines = ADF_DH895XCCIOV_MAX_ACCELENGINES; + hw_data->tx_rx_gap = ADF_DH895XCCIOV_RX_RINGS_OFFSET; + hw_data->tx_rings_mask = ADF_DH895XCCIOV_TX_RINGS_MASK; + hw_data->alloc_irq = adf_vf_isr_resource_alloc; + hw_data->free_irq = adf_vf_isr_resource_free; + hw_data->enable_error_correction = adf_vf_void_noop; + hw_data->init_admin_comms = adf_vf_int_noop; + hw_data->exit_admin_comms = adf_vf_void_noop; + hw_data->send_admin_init = adf_vf2pf_init; + hw_data->init_arb = adf_vf_int_noop; + hw_data->exit_arb = adf_vf_void_noop; + hw_data->disable_iov = adf_vf2pf_shutdown; + hw_data->get_accel_mask = get_accel_mask; + hw_data->get_ae_mask = get_ae_mask; + hw_data->get_num_accels = get_num_accels; + hw_data->get_num_aes = get_num_aes; + hw_data->get_etr_bar_id = get_etr_bar_id; + hw_data->get_misc_bar_id = get_misc_bar_id; + hw_data->get_pf2vf_offset = get_pf2vf_offset; + hw_data->get_vintmsk_offset = get_vintmsk_offset; + hw_data->get_sku = get_sku; + hw_data->enable_ints = adf_vf_void_noop; + hw_data->enable_vf2pf_comms = adf_enable_vf2pf_comms; + hw_data->min_iov_compat_ver = ADF_PFVF_COMPATIBILITY_VERSION; +} + +void adf_clean_hw_data_dh895xcciov(struct adf_hw_device_data *hw_data) +{ + hw_data->dev_class->instances--; +} diff --git a/kernel/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.h b/kernel/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.h new file mode 100644 index 000000000..8f6babfef --- /dev/null +++ b/kernel/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.h @@ -0,0 +1,68 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2015 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2015 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef ADF_DH895XVF_HW_DATA_H_ +#define ADF_DH895XVF_HW_DATA_H_ + +#define ADF_DH895XCCIOV_PMISC_BAR 1 +#define ADF_DH895XCCIOV_ACCELERATORS_MASK 0x1 +#define ADF_DH895XCCIOV_ACCELENGINES_MASK 0x1 +#define ADF_DH895XCCIOV_MAX_ACCELERATORS 1 +#define ADF_DH895XCCIOV_MAX_ACCELENGINES 1 +#define ADF_DH895XCCIOV_RX_RINGS_OFFSET 8 +#define ADF_DH895XCCIOV_TX_RINGS_MASK 0xFF +#define ADF_DH895XCCIOV_ETR_BAR 0 +#define ADF_DH895XCCIOV_ETR_MAX_BANKS 1 + +#define ADF_DH895XCCIOV_PF2VF_OFFSET 0x200 +#define ADF_DH895XCC_PF2VF_PF2VFINT BIT(0) + +#define ADF_DH895XCCIOV_VINTSOU_OFFSET 0x204 +#define ADF_DH895XCC_VINTSOU_BUN BIT(0) +#define ADF_DH895XCC_VINTSOU_PF2VF BIT(1) + +#define ADF_DH895XCCIOV_VINTMSK_OFFSET 0x208 +#endif diff --git a/kernel/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c b/kernel/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c new file mode 100644 index 000000000..789426f21 --- /dev/null +++ b/kernel/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c @@ -0,0 +1,393 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "adf_dh895xccvf_hw_data.h" +#include "adf_drv.h" + +static const char adf_driver_name[] = ADF_DH895XCCVF_DEVICE_NAME; + +#define ADF_SYSTEM_DEVICE(device_id) \ + {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)} + +static const struct pci_device_id adf_pci_tbl[] = { + ADF_SYSTEM_DEVICE(ADF_DH895XCCIOV_PCI_DEVICE_ID), + {0,} +}; +MODULE_DEVICE_TABLE(pci, adf_pci_tbl); + +static int adf_probe(struct pci_dev *dev, const struct pci_device_id *ent); +static void adf_remove(struct pci_dev *dev); + +static struct pci_driver adf_driver = { + .id_table = adf_pci_tbl, + .name = adf_driver_name, + .probe = adf_probe, + .remove = adf_remove, +}; + +static void adf_cleanup_pci_dev(struct adf_accel_dev *accel_dev) +{ + pci_release_regions(accel_dev->accel_pci_dev.pci_dev); + pci_disable_device(accel_dev->accel_pci_dev.pci_dev); +} + +static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) +{ + struct adf_accel_pci *accel_pci_dev = &accel_dev->accel_pci_dev; + struct adf_accel_dev *pf; + int i; + + for (i = 0; i < ADF_PCI_MAX_BARS; i++) { + struct adf_bar *bar = &accel_pci_dev->pci_bars[i]; + + if (bar->virt_addr) + pci_iounmap(accel_pci_dev->pci_dev, bar->virt_addr); + } + + if (accel_dev->hw_device) { + switch (accel_pci_dev->pci_dev->device) { + case ADF_DH895XCCIOV_PCI_DEVICE_ID: + adf_clean_hw_data_dh895xcciov(accel_dev->hw_device); + break; + default: + break; + } + kfree(accel_dev->hw_device); + accel_dev->hw_device = NULL; + } + adf_cfg_dev_remove(accel_dev); + debugfs_remove(accel_dev->debugfs_dir); + pf = adf_devmgr_pci_to_accel_dev(accel_pci_dev->pci_dev->physfn); + adf_devmgr_rm_dev(accel_dev, pf); +} + +static int adf_dev_configure(struct adf_accel_dev *accel_dev) +{ + char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; + unsigned long val, bank = 0; + + if (adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC)) + goto err; + if (adf_cfg_section_add(accel_dev, "Accelerator0")) + goto err; + + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_BANK_NUM, 0); + if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, key, + (void *)&bank, ADF_DEC)) + goto err; + + val = bank; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_ETRMGR_CORE_AFFINITY, 0); + if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, key, + (void *)&val, ADF_DEC)) + goto err; + + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, 0); + + val = 128; + if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, key, + (void *)&val, ADF_DEC)) + goto err; + + val = 512; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, 0); + if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, (void *)&val, ADF_DEC)) + goto err; + + val = 0; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, 0); + if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, (void *)&val, ADF_DEC)) + goto err; + + val = 2; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, 0); + if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, (void *)&val, ADF_DEC)) + goto err; + + val = 8; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, 0); + if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, (void *)&val, ADF_DEC)) + goto err; + + val = 10; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, 0); + if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, (void *)&val, ADF_DEC)) + goto err; + + val = ADF_COALESCING_DEF_TIME; + snprintf(key, sizeof(key), ADF_ETRMGR_COALESCE_TIMER_FORMAT, + (int)bank); + if (adf_cfg_add_key_value_param(accel_dev, "Accelerator0", + key, (void *)&val, ADF_DEC)) + goto err; + + val = 1; + if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + ADF_NUM_CY, (void *)&val, ADF_DEC)) + goto err; + + set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); + return 0; +err: + dev_err(&GET_DEV(accel_dev), "Failed to configure QAT accel dev\n"); + return -EINVAL; +} + +static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + struct adf_accel_dev *accel_dev; + struct adf_accel_dev *pf; + struct adf_accel_pci *accel_pci_dev; + struct adf_hw_device_data *hw_data; + char name[ADF_DEVICE_NAME_LENGTH]; + unsigned int i, bar_nr; + int ret, bar_mask; + + switch (ent->device) { + case ADF_DH895XCCIOV_PCI_DEVICE_ID: + break; + default: + dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device); + return -ENODEV; + } + + accel_dev = kzalloc_node(sizeof(*accel_dev), GFP_KERNEL, + dev_to_node(&pdev->dev)); + if (!accel_dev) + return -ENOMEM; + + accel_dev->is_vf = true; + pf = adf_devmgr_pci_to_accel_dev(pdev->physfn); + accel_pci_dev = &accel_dev->accel_pci_dev; + accel_pci_dev->pci_dev = pdev; + + /* Add accel device to accel table */ + if (adf_devmgr_add_dev(accel_dev, pf)) { + dev_err(&pdev->dev, "Failed to add new accelerator device.\n"); + kfree(accel_dev); + return -EFAULT; + } + INIT_LIST_HEAD(&accel_dev->crypto_list); + + accel_dev->owner = THIS_MODULE; + /* Allocate and configure device configuration structure */ + hw_data = kzalloc_node(sizeof(*hw_data), GFP_KERNEL, + dev_to_node(&pdev->dev)); + if (!hw_data) { + ret = -ENOMEM; + goto out_err; + } + accel_dev->hw_device = hw_data; + switch (ent->device) { + case ADF_DH895XCCIOV_PCI_DEVICE_ID: + adf_init_hw_data_dh895xcciov(accel_dev->hw_device); + break; + default: + ret = -ENODEV; + goto out_err; + } + + /* Get Accelerators and Accelerators Engines masks */ + hw_data->accel_mask = hw_data->get_accel_mask(hw_data->fuses); + hw_data->ae_mask = hw_data->get_ae_mask(hw_data->fuses); + accel_pci_dev->sku = hw_data->get_sku(hw_data); + + /* Create dev top level debugfs entry */ + snprintf(name, sizeof(name), "%s%s_%02x:%02d.%02d", + ADF_DEVICE_NAME_PREFIX, hw_data->dev_class->name, + pdev->bus->number, PCI_SLOT(pdev->devfn), + PCI_FUNC(pdev->devfn)); + + accel_dev->debugfs_dir = debugfs_create_dir(name, NULL); + if (!accel_dev->debugfs_dir) { + dev_err(&pdev->dev, "Could not create debugfs dir %s\n", name); + ret = -EINVAL; + goto out_err; + } + + /* Create device configuration table */ + ret = adf_cfg_dev_add(accel_dev); + if (ret) + goto out_err; + + /* enable PCI device */ + if (pci_enable_device(pdev)) { + ret = -EFAULT; + goto out_err; + } + + /* set dma identifier */ + if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) { + if ((pci_set_dma_mask(pdev, DMA_BIT_MASK(32)))) { + dev_err(&pdev->dev, "No usable DMA configuration\n"); + ret = -EFAULT; + goto out_err_disable; + } else { + pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); + } + + } else { + pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); + } + + if (pci_request_regions(pdev, adf_driver_name)) { + ret = -EFAULT; + goto out_err_disable; + } + + /* Find and map all the device's BARS */ + i = 0; + bar_mask = pci_select_bars(pdev, IORESOURCE_MEM); + for_each_set_bit(bar_nr, (const unsigned long *)&bar_mask, + ADF_PCI_MAX_BARS * 2) { + struct adf_bar *bar = &accel_pci_dev->pci_bars[i++]; + + bar->base_addr = pci_resource_start(pdev, bar_nr); + if (!bar->base_addr) + break; + bar->size = pci_resource_len(pdev, bar_nr); + bar->virt_addr = pci_iomap(accel_pci_dev->pci_dev, bar_nr, 0); + if (!bar->virt_addr) { + dev_err(&pdev->dev, "Failed to map BAR %d\n", bar_nr); + ret = -EFAULT; + goto out_err_free_reg; + } + } + pci_set_master(pdev); + /* Completion for VF2PF request/response message exchange */ + init_completion(&accel_dev->vf.iov_msg_completion); + + ret = adf_dev_configure(accel_dev); + if (ret) + goto out_err_free_reg; + + ret = adf_dev_init(accel_dev); + if (ret) + goto out_err_dev_shutdown; + + ret = adf_dev_start(accel_dev); + if (ret) + goto out_err_dev_stop; + + return ret; + +out_err_dev_stop: + adf_dev_stop(accel_dev); +out_err_dev_shutdown: + adf_dev_shutdown(accel_dev); +out_err_free_reg: + pci_release_regions(accel_pci_dev->pci_dev); +out_err_disable: + pci_disable_device(accel_pci_dev->pci_dev); +out_err: + adf_cleanup_accel(accel_dev); + kfree(accel_dev); + return ret; +} + +static void adf_remove(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + if (!accel_dev) { + pr_err("QAT: Driver removal failed\n"); + return; + } + if (adf_dev_stop(accel_dev)) + dev_err(&GET_DEV(accel_dev), "Failed to stop QAT accel dev\n"); + + adf_dev_shutdown(accel_dev); + adf_cleanup_accel(accel_dev); + adf_cleanup_pci_dev(accel_dev); + kfree(accel_dev); +} + +static int __init adfdrv_init(void) +{ + request_module("intel_qat"); + + if (pci_register_driver(&adf_driver)) { + pr_err("QAT: Driver initialization failed\n"); + return -EFAULT; + } + return 0; +} + +static void __exit adfdrv_release(void) +{ + pci_unregister_driver(&adf_driver); + adf_clean_vf_map(true); +} + +module_init(adfdrv_init); +module_exit(adfdrv_release); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Intel"); +MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); +MODULE_VERSION(ADF_DRV_VERSION); diff --git a/kernel/drivers/crypto/qat/qat_dh895xccvf/adf_drv.h b/kernel/drivers/crypto/qat/qat_dh895xccvf/adf_drv.h new file mode 100644 index 000000000..e270e4a63 --- /dev/null +++ b/kernel/drivers/crypto/qat/qat_dh895xccvf/adf_drv.h @@ -0,0 +1,57 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef ADF_DH895xVF_DRV_H_ +#define ADF_DH895xVF_DRV_H_ +#include +#include + +void adf_init_hw_data_dh895xcciov(struct adf_hw_device_data *hw_data); +void adf_clean_hw_data_dh895xcciov(struct adf_hw_device_data *hw_data); +int adf_vf_isr_resource_alloc(struct adf_accel_dev *accel_dev); +void adf_vf_isr_resource_free(struct adf_accel_dev *accel_dev); +void adf_update_ring_arb_enable(struct adf_etr_ring_data *ring); +#endif diff --git a/kernel/drivers/crypto/qat/qat_dh895xccvf/adf_isr.c b/kernel/drivers/crypto/qat/qat_dh895xccvf/adf_isr.c new file mode 100644 index 000000000..87c5d8adb --- /dev/null +++ b/kernel/drivers/crypto/qat/qat_dh895xccvf/adf_isr.c @@ -0,0 +1,258 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "adf_drv.h" +#include "adf_dh895xccvf_hw_data.h" + +static int adf_enable_msi(struct adf_accel_dev *accel_dev) +{ + struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev; + int stat = pci_enable_msi(pci_dev_info->pci_dev); + + if (stat) { + dev_err(&GET_DEV(accel_dev), + "Failed to enable MSI interrupts\n"); + return stat; + } + + accel_dev->vf.irq_name = kzalloc(ADF_MAX_MSIX_VECTOR_NAME, GFP_KERNEL); + if (!accel_dev->vf.irq_name) + return -ENOMEM; + + return stat; +} + +static void adf_disable_msi(struct adf_accel_dev *accel_dev) +{ + struct pci_dev *pdev = accel_to_pci_dev(accel_dev); + + kfree(accel_dev->vf.irq_name); + pci_disable_msi(pdev); +} + +static void adf_pf2vf_bh_handler(void *data) +{ + struct adf_accel_dev *accel_dev = data; + void __iomem *pmisc_bar_addr = + (&GET_BARS(accel_dev)[ADF_DH895XCCIOV_PMISC_BAR])->virt_addr; + u32 msg; + + /* Read the message from PF */ + msg = ADF_CSR_RD(pmisc_bar_addr, ADF_DH895XCCIOV_PF2VF_OFFSET); + + if (!(msg & ADF_PF2VF_MSGORIGIN_SYSTEM)) + /* Ignore legacy non-system (non-kernel) PF2VF messages */ + goto err; + + switch ((msg & ADF_PF2VF_MSGTYPE_MASK) >> ADF_PF2VF_MSGTYPE_SHIFT) { + case ADF_PF2VF_MSGTYPE_RESTARTING: + dev_dbg(&GET_DEV(accel_dev), + "Restarting msg received from PF 0x%x\n", msg); + adf_dev_stop(accel_dev); + break; + case ADF_PF2VF_MSGTYPE_VERSION_RESP: + dev_dbg(&GET_DEV(accel_dev), + "Version resp received from PF 0x%x\n", msg); + accel_dev->vf.pf_version = + (msg & ADF_PF2VF_VERSION_RESP_VERS_MASK) >> + ADF_PF2VF_VERSION_RESP_VERS_SHIFT; + accel_dev->vf.compatible = + (msg & ADF_PF2VF_VERSION_RESP_RESULT_MASK) >> + ADF_PF2VF_VERSION_RESP_RESULT_SHIFT; + complete(&accel_dev->vf.iov_msg_completion); + break; + default: + goto err; + } + + /* To ack, clear the PF2VFINT bit */ + msg &= ~ADF_DH895XCC_PF2VF_PF2VFINT; + ADF_CSR_WR(pmisc_bar_addr, ADF_DH895XCCIOV_PF2VF_OFFSET, msg); + + /* Re-enable PF2VF interrupts */ + adf_enable_pf2vf_interrupts(accel_dev); + return; +err: + dev_err(&GET_DEV(accel_dev), + "Unknown message from PF (0x%x); leaving PF2VF ints disabled\n", + msg); +} + +static int adf_setup_pf2vf_bh(struct adf_accel_dev *accel_dev) +{ + tasklet_init(&accel_dev->vf.pf2vf_bh_tasklet, + (void *)adf_pf2vf_bh_handler, (unsigned long)accel_dev); + + mutex_init(&accel_dev->vf.vf2pf_lock); + return 0; +} + +static void adf_cleanup_pf2vf_bh(struct adf_accel_dev *accel_dev) +{ + tasklet_disable(&accel_dev->vf.pf2vf_bh_tasklet); + tasklet_kill(&accel_dev->vf.pf2vf_bh_tasklet); + mutex_destroy(&accel_dev->vf.vf2pf_lock); +} + +static irqreturn_t adf_isr(int irq, void *privdata) +{ + struct adf_accel_dev *accel_dev = privdata; + void __iomem *pmisc_bar_addr = + (&GET_BARS(accel_dev)[ADF_DH895XCCIOV_PMISC_BAR])->virt_addr; + u32 v_int; + + /* Read VF INT source CSR to determine the source of VF interrupt */ + v_int = ADF_CSR_RD(pmisc_bar_addr, ADF_DH895XCCIOV_VINTSOU_OFFSET); + + /* Check for PF2VF interrupt */ + if (v_int & ADF_DH895XCC_VINTSOU_PF2VF) { + /* Disable PF to VF interrupt */ + adf_disable_pf2vf_interrupts(accel_dev); + + /* Schedule tasklet to handle interrupt BH */ + tasklet_hi_schedule(&accel_dev->vf.pf2vf_bh_tasklet); + return IRQ_HANDLED; + } + + /* Check bundle interrupt */ + if (v_int & ADF_DH895XCC_VINTSOU_BUN) { + struct adf_etr_data *etr_data = accel_dev->transport; + struct adf_etr_bank_data *bank = &etr_data->banks[0]; + + /* Disable Flag and Coalesce Ring Interrupts */ + WRITE_CSR_INT_FLAG_AND_COL(bank->csr_addr, bank->bank_number, + 0); + tasklet_hi_schedule(&bank->resp_handler); + return IRQ_HANDLED; + } + + return IRQ_NONE; +} + +static int adf_request_msi_irq(struct adf_accel_dev *accel_dev) +{ + struct pci_dev *pdev = accel_to_pci_dev(accel_dev); + unsigned int cpu; + int ret; + + snprintf(accel_dev->vf.irq_name, ADF_MAX_MSIX_VECTOR_NAME, + "qat_%02x:%02d.%02d", pdev->bus->number, PCI_SLOT(pdev->devfn), + PCI_FUNC(pdev->devfn)); + ret = request_irq(pdev->irq, adf_isr, 0, accel_dev->vf.irq_name, + (void *)accel_dev); + if (ret) { + dev_err(&GET_DEV(accel_dev), "failed to enable irq for %s\n", + accel_dev->vf.irq_name); + return ret; + } + cpu = accel_dev->accel_id % num_online_cpus(); + irq_set_affinity_hint(pdev->irq, get_cpu_mask(cpu)); + + return ret; +} + +static int adf_setup_bh(struct adf_accel_dev *accel_dev) +{ + struct adf_etr_data *priv_data = accel_dev->transport; + + tasklet_init(&priv_data->banks[0].resp_handler, adf_response_handler, + (unsigned long)priv_data->banks); + return 0; +} + +static void adf_cleanup_bh(struct adf_accel_dev *accel_dev) +{ + struct adf_etr_data *priv_data = accel_dev->transport; + + tasklet_disable(&priv_data->banks[0].resp_handler); + tasklet_kill(&priv_data->banks[0].resp_handler); +} + +void adf_vf_isr_resource_free(struct adf_accel_dev *accel_dev) +{ + struct pci_dev *pdev = accel_to_pci_dev(accel_dev); + + irq_set_affinity_hint(pdev->irq, NULL); + free_irq(pdev->irq, (void *)accel_dev); + adf_cleanup_bh(accel_dev); + adf_cleanup_pf2vf_bh(accel_dev); + adf_disable_msi(accel_dev); +} + +int adf_vf_isr_resource_alloc(struct adf_accel_dev *accel_dev) +{ + if (adf_enable_msi(accel_dev)) + goto err_out; + + if (adf_setup_pf2vf_bh(accel_dev)) + goto err_out; + + if (adf_setup_bh(accel_dev)) + goto err_out; + + if (adf_request_msi_irq(accel_dev)) + goto err_out; + + return 0; +err_out: + adf_vf_isr_resource_free(accel_dev); + return -EFAULT; +} diff --git a/kernel/drivers/crypto/qce/ablkcipher.c b/kernel/drivers/crypto/qce/ablkcipher.c index ad592de47..2c0d63d48 100644 --- a/kernel/drivers/crypto/qce/ablkcipher.c +++ b/kernel/drivers/crypto/qce/ablkcipher.c @@ -44,10 +44,8 @@ static void qce_ablkcipher_done(void *data) error); if (diff_dst) - qce_unmapsg(qce->dev, rctx->src_sg, rctx->src_nents, dir_src, - rctx->dst_chained); - qce_unmapsg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst, - rctx->dst_chained); + dma_unmap_sg(qce->dev, rctx->src_sg, rctx->src_nents, dir_src); + dma_unmap_sg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst); sg_free_table(&rctx->dst_tbl); @@ -80,15 +78,11 @@ qce_ablkcipher_async_req_handle(struct crypto_async_request *async_req) dir_src = diff_dst ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL; dir_dst = diff_dst ? DMA_FROM_DEVICE : DMA_BIDIRECTIONAL; - rctx->src_nents = qce_countsg(req->src, req->nbytes, - &rctx->src_chained); - if (diff_dst) { - rctx->dst_nents = qce_countsg(req->dst, req->nbytes, - &rctx->dst_chained); - } else { + rctx->src_nents = sg_nents_for_len(req->src, req->nbytes); + if (diff_dst) + rctx->dst_nents = sg_nents_for_len(req->dst, req->nbytes); + else rctx->dst_nents = rctx->src_nents; - rctx->dst_chained = rctx->src_chained; - } rctx->dst_nents += 1; @@ -116,14 +110,12 @@ qce_ablkcipher_async_req_handle(struct crypto_async_request *async_req) sg_mark_end(sg); rctx->dst_sg = rctx->dst_tbl.sgl; - ret = qce_mapsg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst, - rctx->dst_chained); + ret = dma_map_sg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst); if (ret < 0) goto error_free; if (diff_dst) { - ret = qce_mapsg(qce->dev, req->src, rctx->src_nents, dir_src, - rctx->src_chained); + ret = dma_map_sg(qce->dev, req->src, rctx->src_nents, dir_src); if (ret < 0) goto error_unmap_dst; rctx->src_sg = req->src; @@ -149,11 +141,9 @@ error_terminate: qce_dma_terminate_all(&qce->dma); error_unmap_src: if (diff_dst) - qce_unmapsg(qce->dev, req->src, rctx->src_nents, dir_src, - rctx->src_chained); + dma_unmap_sg(qce->dev, req->src, rctx->src_nents, dir_src); error_unmap_dst: - qce_unmapsg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst, - rctx->dst_chained); + dma_unmap_sg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst); error_free: sg_free_table(&rctx->dst_tbl); return ret; diff --git a/kernel/drivers/crypto/qce/cipher.h b/kernel/drivers/crypto/qce/cipher.h index d5757cfcd..5c6a5f863 100644 --- a/kernel/drivers/crypto/qce/cipher.h +++ b/kernel/drivers/crypto/qce/cipher.h @@ -32,8 +32,6 @@ struct qce_cipher_ctx { * @ivsize: IV size * @src_nents: source entries * @dst_nents: destination entries - * @src_chained: is source chained - * @dst_chained: is destination chained * @result_sg: scatterlist used for result buffer * @dst_tbl: destination sg table * @dst_sg: destination sg pointer table beginning @@ -47,8 +45,6 @@ struct qce_cipher_reqctx { unsigned int ivsize; int src_nents; int dst_nents; - bool src_chained; - bool dst_chained; struct scatterlist result_sg; struct sg_table dst_tbl; struct scatterlist *dst_sg; diff --git a/kernel/drivers/crypto/qce/dma.c b/kernel/drivers/crypto/qce/dma.c index 378cb7686..4797e795c 100644 --- a/kernel/drivers/crypto/qce/dma.c +++ b/kernel/drivers/crypto/qce/dma.c @@ -54,58 +54,6 @@ void qce_dma_release(struct qce_dma_data *dma) kfree(dma->result_buf); } -int qce_mapsg(struct device *dev, struct scatterlist *sg, int nents, - enum dma_data_direction dir, bool chained) -{ - int err; - - if (chained) { - while (sg) { - err = dma_map_sg(dev, sg, 1, dir); - if (!err) - return -EFAULT; - sg = sg_next(sg); - } - } else { - err = dma_map_sg(dev, sg, nents, dir); - if (!err) - return -EFAULT; - } - - return nents; -} - -void qce_unmapsg(struct device *dev, struct scatterlist *sg, int nents, - enum dma_data_direction dir, bool chained) -{ - if (chained) - while (sg) { - dma_unmap_sg(dev, sg, 1, dir); - sg = sg_next(sg); - } - else - dma_unmap_sg(dev, sg, nents, dir); -} - -int qce_countsg(struct scatterlist *sglist, int nbytes, bool *chained) -{ - struct scatterlist *sg = sglist; - int nents = 0; - - if (chained) - *chained = false; - - while (nbytes > 0 && sg) { - nents++; - nbytes -= sg->length; - if (!sg_is_last(sg) && (sg + 1)->length == 0 && chained) - *chained = true; - sg = sg_next(sg); - } - - return nents; -} - struct scatterlist * qce_sgtable_add(struct sg_table *sgt, struct scatterlist *new_sgl) { diff --git a/kernel/drivers/crypto/qce/dma.h b/kernel/drivers/crypto/qce/dma.h index 65bedb81d..130235d17 100644 --- a/kernel/drivers/crypto/qce/dma.h +++ b/kernel/drivers/crypto/qce/dma.h @@ -49,11 +49,6 @@ int qce_dma_prep_sgs(struct qce_dma_data *dma, struct scatterlist *sg_in, dma_async_tx_callback cb, void *cb_param); void qce_dma_issue_pending(struct qce_dma_data *dma); int qce_dma_terminate_all(struct qce_dma_data *dma); -int qce_countsg(struct scatterlist *sg_list, int nbytes, bool *chained); -void qce_unmapsg(struct device *dev, struct scatterlist *sg, int nents, - enum dma_data_direction dir, bool chained); -int qce_mapsg(struct device *dev, struct scatterlist *sg, int nents, - enum dma_data_direction dir, bool chained); struct scatterlist * qce_sgtable_add(struct sg_table *sgt, struct scatterlist *sg_add); diff --git a/kernel/drivers/crypto/qce/sha.c b/kernel/drivers/crypto/qce/sha.c index 5c5df1d17..0c9973ec8 100644 --- a/kernel/drivers/crypto/qce/sha.c +++ b/kernel/drivers/crypto/qce/sha.c @@ -51,9 +51,8 @@ static void qce_ahash_done(void *data) if (error) dev_dbg(qce->dev, "ahash dma termination error (%d)\n", error); - qce_unmapsg(qce->dev, req->src, rctx->src_nents, DMA_TO_DEVICE, - rctx->src_chained); - qce_unmapsg(qce->dev, &rctx->result_sg, 1, DMA_FROM_DEVICE, 0); + dma_unmap_sg(qce->dev, req->src, rctx->src_nents, DMA_TO_DEVICE); + dma_unmap_sg(qce->dev, &rctx->result_sg, 1, DMA_FROM_DEVICE); memcpy(rctx->digest, result->auth_iv, digestsize); if (req->result) @@ -92,16 +91,14 @@ static int qce_ahash_async_req_handle(struct crypto_async_request *async_req) rctx->authklen = AES_KEYSIZE_128; } - rctx->src_nents = qce_countsg(req->src, req->nbytes, - &rctx->src_chained); - ret = qce_mapsg(qce->dev, req->src, rctx->src_nents, DMA_TO_DEVICE, - rctx->src_chained); + rctx->src_nents = sg_nents_for_len(req->src, req->nbytes); + ret = dma_map_sg(qce->dev, req->src, rctx->src_nents, DMA_TO_DEVICE); if (ret < 0) return ret; sg_init_one(&rctx->result_sg, qce->dma.result_buf, QCE_RESULT_BUF_SZ); - ret = qce_mapsg(qce->dev, &rctx->result_sg, 1, DMA_FROM_DEVICE, 0); + ret = dma_map_sg(qce->dev, &rctx->result_sg, 1, DMA_FROM_DEVICE); if (ret < 0) goto error_unmap_src; @@ -121,10 +118,9 @@ static int qce_ahash_async_req_handle(struct crypto_async_request *async_req) error_terminate: qce_dma_terminate_all(&qce->dma); error_unmap_dst: - qce_unmapsg(qce->dev, &rctx->result_sg, 1, DMA_FROM_DEVICE, 0); + dma_unmap_sg(qce->dev, &rctx->result_sg, 1, DMA_FROM_DEVICE); error_unmap_src: - qce_unmapsg(qce->dev, req->src, rctx->src_nents, DMA_TO_DEVICE, - rctx->src_chained); + dma_unmap_sg(qce->dev, req->src, rctx->src_nents, DMA_TO_DEVICE); return ret; } @@ -296,7 +292,7 @@ static int qce_ahash_update(struct ahash_request *req) if (rctx->buflen) { sg_init_table(rctx->sg, 2); sg_set_buf(rctx->sg, rctx->tmpbuf, rctx->buflen); - scatterwalk_sg_chain(rctx->sg, 2, req->src); + sg_chain(rctx->sg, 2, req->src); req->src = rctx->sg; } diff --git a/kernel/drivers/crypto/qce/sha.h b/kernel/drivers/crypto/qce/sha.h index 286f0d539..236bb5e9a 100644 --- a/kernel/drivers/crypto/qce/sha.h +++ b/kernel/drivers/crypto/qce/sha.h @@ -36,7 +36,6 @@ struct qce_sha_ctx { * @flags: operation flags * @src_orig: original request sg list * @nbytes_orig: original request number of bytes - * @src_chained: is source scatterlist chained * @src_nents: source number of entries * @byte_count: byte count * @count: save count in states during update, import and export @@ -55,7 +54,6 @@ struct qce_sha_reqctx { unsigned long flags; struct scatterlist *src_orig; unsigned int nbytes_orig; - bool src_chained; int src_nents; __be32 byte_count[2]; u64 count; diff --git a/kernel/drivers/crypto/sahara.c b/kernel/drivers/crypto/sahara.c index 6be377f6b..f68c24a98 100644 --- a/kernel/drivers/crypto/sahara.c +++ b/kernel/drivers/crypto/sahara.c @@ -173,7 +173,6 @@ struct sahara_aes_reqctx { * @sg_in_idx: number of hw links * @in_sg: scatterlist for input data * @in_sg_chain: scatterlists for chained input data - * @in_sg_chained: specifies if chained scatterlists are used or not * @total: total number of bytes for transfer * @last: is this the last block * @first: is this the first block @@ -191,7 +190,6 @@ struct sahara_sha_reqctx { unsigned int sg_in_idx; struct scatterlist *in_sg; struct scatterlist in_sg_chain[2]; - bool in_sg_chained; size_t total; unsigned int last; unsigned int first; @@ -274,31 +272,7 @@ static u32 sahara_aes_data_link_hdr(struct sahara_dev *dev) SAHARA_HDR_CHA_SKHA | SAHARA_HDR_PARITY_BIT; } -static int sahara_sg_length(struct scatterlist *sg, - unsigned int total) -{ - int sg_nb; - unsigned int len; - struct scatterlist *sg_list; - - sg_nb = 0; - sg_list = sg; - - while (total) { - len = min(sg_list->length, total); - - sg_nb++; - total -= len; - - sg_list = sg_next(sg_list); - if (!sg_list) - total = 0; - } - - return sg_nb; -} - -static char *sahara_err_src[16] = { +static const char *sahara_err_src[16] = { "No error", "Header error", "Descriptor length error", @@ -317,14 +291,14 @@ static char *sahara_err_src[16] = { "DMA error" }; -static char *sahara_err_dmasize[4] = { +static const char *sahara_err_dmasize[4] = { "Byte transfer", "Half-word transfer", "Word transfer", "Reserved" }; -static char *sahara_err_dmasrc[8] = { +static const char *sahara_err_dmasrc[8] = { "No error", "AHB bus error", "Internal IP bus error", @@ -335,7 +309,7 @@ static char *sahara_err_dmasrc[8] = { "DMA HW error" }; -static char *sahara_cha_errsrc[12] = { +static const char *sahara_cha_errsrc[12] = { "Input buffer non-empty", "Illegal address", "Illegal mode", @@ -350,7 +324,7 @@ static char *sahara_cha_errsrc[12] = { "Reserved" }; -static char *sahara_cha_err[4] = { "No error", "SKHA", "MDHA", "RNG" }; +static const char *sahara_cha_err[4] = { "No error", "SKHA", "MDHA", "RNG" }; static void sahara_decode_error(struct sahara_dev *dev, unsigned int error) { @@ -380,7 +354,7 @@ static void sahara_decode_error(struct sahara_dev *dev, unsigned int error) dev_err(dev->device, "\n"); } -static char *sahara_state[4] = { "Idle", "Busy", "Error", "HW Fault" }; +static const char *sahara_state[4] = { "Idle", "Busy", "Error", "HW Fault" }; static void sahara_decode_status(struct sahara_dev *dev, unsigned int status) { @@ -502,8 +476,8 @@ static int sahara_hw_descriptor_create(struct sahara_dev *dev) idx++; } - dev->nb_in_sg = sahara_sg_length(dev->in_sg, dev->total); - dev->nb_out_sg = sahara_sg_length(dev->out_sg, dev->total); + dev->nb_in_sg = sg_nents_for_len(dev->in_sg, dev->total); + dev->nb_out_sg = sg_nents_for_len(dev->out_sg, dev->total); if ((dev->nb_in_sg + dev->nb_out_sg) > SAHARA_MAX_HW_LINK) { dev_err(dev->device, "not enough hw links (%d)\n", dev->nb_in_sg + dev->nb_out_sg); @@ -818,45 +792,26 @@ static int sahara_sha_hw_links_create(struct sahara_dev *dev, dev->in_sg = rctx->in_sg; - dev->nb_in_sg = sahara_sg_length(dev->in_sg, rctx->total); + dev->nb_in_sg = sg_nents_for_len(dev->in_sg, rctx->total); if ((dev->nb_in_sg) > SAHARA_MAX_HW_LINK) { dev_err(dev->device, "not enough hw links (%d)\n", dev->nb_in_sg + dev->nb_out_sg); return -EINVAL; } - if (rctx->in_sg_chained) { - i = start; - sg = dev->in_sg; - while (sg) { - ret = dma_map_sg(dev->device, sg, 1, - DMA_TO_DEVICE); - if (!ret) - return -EFAULT; - - dev->hw_link[i]->len = sg->length; - dev->hw_link[i]->p = sg->dma_address; + sg = dev->in_sg; + ret = dma_map_sg(dev->device, dev->in_sg, dev->nb_in_sg, DMA_TO_DEVICE); + if (!ret) + return -EFAULT; + + for (i = start; i < dev->nb_in_sg + start; i++) { + dev->hw_link[i]->len = sg->length; + dev->hw_link[i]->p = sg->dma_address; + if (i == (dev->nb_in_sg + start - 1)) { + dev->hw_link[i]->next = 0; + } else { dev->hw_link[i]->next = dev->hw_phys_link[i + 1]; sg = sg_next(sg); - i += 1; - } - dev->hw_link[i-1]->next = 0; - } else { - sg = dev->in_sg; - ret = dma_map_sg(dev->device, dev->in_sg, dev->nb_in_sg, - DMA_TO_DEVICE); - if (!ret) - return -EFAULT; - - for (i = start; i < dev->nb_in_sg + start; i++) { - dev->hw_link[i]->len = sg->length; - dev->hw_link[i]->p = sg->dma_address; - if (i == (dev->nb_in_sg + start - 1)) { - dev->hw_link[i]->next = 0; - } else { - dev->hw_link[i]->next = dev->hw_phys_link[i + 1]; - sg = sg_next(sg); - } } } @@ -999,12 +954,11 @@ static int sahara_sha_prepare_request(struct ahash_request *req) sg_init_table(rctx->in_sg_chain, 2); sg_set_buf(rctx->in_sg_chain, rctx->rembuf, rctx->buf_cnt); - scatterwalk_sg_chain(rctx->in_sg_chain, 2, req->src); + sg_chain(rctx->in_sg_chain, 2, req->src); rctx->total = req->nbytes + rctx->buf_cnt; rctx->in_sg = rctx->in_sg_chain; - rctx->in_sg_chained = true; req->src = rctx->in_sg_chain; /* only data from previous operation */ } else if (rctx->buf_cnt) { @@ -1015,13 +969,11 @@ static int sahara_sha_prepare_request(struct ahash_request *req) /* buf was copied into rembuf above */ sg_init_one(rctx->in_sg, rctx->rembuf, rctx->buf_cnt); rctx->total = rctx->buf_cnt; - rctx->in_sg_chained = false; /* no data from previous operation */ } else { rctx->in_sg = req->src; rctx->total = req->nbytes; req->src = rctx->in_sg; - rctx->in_sg_chained = false; } /* on next call, we only have the remaining data in the buffer */ @@ -1030,23 +982,6 @@ static int sahara_sha_prepare_request(struct ahash_request *req) return -EINPROGRESS; } -static void sahara_sha_unmap_sg(struct sahara_dev *dev, - struct sahara_sha_reqctx *rctx) -{ - struct scatterlist *sg; - - if (rctx->in_sg_chained) { - sg = dev->in_sg; - while (sg) { - dma_unmap_sg(dev->device, sg, 1, DMA_TO_DEVICE); - sg = sg_next(sg); - } - } else { - dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg, - DMA_TO_DEVICE); - } -} - static int sahara_sha_process(struct ahash_request *req) { struct sahara_dev *dev = dev_ptr; @@ -1086,7 +1021,8 @@ static int sahara_sha_process(struct ahash_request *req) } if (rctx->sg_in_idx) - sahara_sha_unmap_sg(dev, rctx); + dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg, + DMA_TO_DEVICE); memcpy(rctx->context, dev->context_base, rctx->context_size); @@ -1516,7 +1452,7 @@ static int sahara_probe(struct platform_device *pdev) } /* Allocate HW descriptors */ - dev->hw_desc[0] = dma_alloc_coherent(&pdev->dev, + dev->hw_desc[0] = dmam_alloc_coherent(&pdev->dev, SAHARA_MAX_HW_DESC * sizeof(struct sahara_hw_desc), &dev->hw_phys_desc[0], GFP_KERNEL); if (!dev->hw_desc[0]) { @@ -1528,34 +1464,31 @@ static int sahara_probe(struct platform_device *pdev) sizeof(struct sahara_hw_desc); /* Allocate space for iv and key */ - dev->key_base = dma_alloc_coherent(&pdev->dev, 2 * AES_KEYSIZE_128, + dev->key_base = dmam_alloc_coherent(&pdev->dev, 2 * AES_KEYSIZE_128, &dev->key_phys_base, GFP_KERNEL); if (!dev->key_base) { dev_err(&pdev->dev, "Could not allocate memory for key\n"); - err = -ENOMEM; - goto err_key; + return -ENOMEM; } dev->iv_base = dev->key_base + AES_KEYSIZE_128; dev->iv_phys_base = dev->key_phys_base + AES_KEYSIZE_128; /* Allocate space for context: largest digest + message length field */ - dev->context_base = dma_alloc_coherent(&pdev->dev, + dev->context_base = dmam_alloc_coherent(&pdev->dev, SHA256_DIGEST_SIZE + 4, &dev->context_phys_base, GFP_KERNEL); if (!dev->context_base) { dev_err(&pdev->dev, "Could not allocate memory for MDHA context\n"); - err = -ENOMEM; - goto err_key; + return -ENOMEM; } /* Allocate space for HW links */ - dev->hw_link[0] = dma_alloc_coherent(&pdev->dev, + dev->hw_link[0] = dmam_alloc_coherent(&pdev->dev, SAHARA_MAX_HW_LINK * sizeof(struct sahara_hw_link), &dev->hw_phys_link[0], GFP_KERNEL); if (!dev->hw_link[0]) { dev_err(&pdev->dev, "Could not allocate hw links\n"); - err = -ENOMEM; - goto err_link; + return -ENOMEM; } for (i = 1; i < SAHARA_MAX_HW_LINK; i++) { dev->hw_phys_link[i] = dev->hw_phys_link[i - 1] + @@ -1572,14 +1505,17 @@ static int sahara_probe(struct platform_device *pdev) dev->kthread = kthread_run(sahara_queue_manage, dev, "sahara_crypto"); if (IS_ERR(dev->kthread)) { - err = PTR_ERR(dev->kthread); - goto err_link; + return PTR_ERR(dev->kthread); } init_completion(&dev->dma_completion); - clk_prepare_enable(dev->clk_ipg); - clk_prepare_enable(dev->clk_ahb); + err = clk_prepare_enable(dev->clk_ipg); + if (err) + return err; + err = clk_prepare_enable(dev->clk_ahb); + if (err) + goto clk_ipg_disable; version = sahara_read(dev, SAHARA_REG_VERSION); if (of_device_is_compatible(pdev->dev.of_node, "fsl,imx27-sahara")) { @@ -1616,24 +1552,11 @@ static int sahara_probe(struct platform_device *pdev) return 0; err_algs: - dma_free_coherent(&pdev->dev, - SAHARA_MAX_HW_LINK * sizeof(struct sahara_hw_link), - dev->hw_link[0], dev->hw_phys_link[0]); - clk_disable_unprepare(dev->clk_ipg); - clk_disable_unprepare(dev->clk_ahb); kthread_stop(dev->kthread); dev_ptr = NULL; -err_link: - dma_free_coherent(&pdev->dev, - 2 * AES_KEYSIZE_128, - dev->key_base, dev->key_phys_base); - dma_free_coherent(&pdev->dev, - SHA256_DIGEST_SIZE, - dev->context_base, dev->context_phys_base); -err_key: - dma_free_coherent(&pdev->dev, - SAHARA_MAX_HW_DESC * sizeof(struct sahara_hw_desc), - dev->hw_desc[0], dev->hw_phys_desc[0]); + clk_disable_unprepare(dev->clk_ahb); +clk_ipg_disable: + clk_disable_unprepare(dev->clk_ipg); return err; } @@ -1642,16 +1565,6 @@ static int sahara_remove(struct platform_device *pdev) { struct sahara_dev *dev = platform_get_drvdata(pdev); - dma_free_coherent(&pdev->dev, - SAHARA_MAX_HW_LINK * sizeof(struct sahara_hw_link), - dev->hw_link[0], dev->hw_phys_link[0]); - dma_free_coherent(&pdev->dev, - 2 * AES_KEYSIZE_128, - dev->key_base, dev->key_phys_base); - dma_free_coherent(&pdev->dev, - SAHARA_MAX_HW_DESC * sizeof(struct sahara_hw_desc), - dev->hw_desc[0], dev->hw_phys_desc[0]); - kthread_stop(dev->kthread); sahara_unregister_algs(dev); diff --git a/kernel/drivers/crypto/sunxi-ss/Makefile b/kernel/drivers/crypto/sunxi-ss/Makefile new file mode 100644 index 000000000..8f4c7a273 --- /dev/null +++ b/kernel/drivers/crypto/sunxi-ss/Makefile @@ -0,0 +1,2 @@ +obj-$(CONFIG_CRYPTO_DEV_SUN4I_SS) += sun4i-ss.o +sun4i-ss-y += sun4i-ss-core.o sun4i-ss-hash.o sun4i-ss-cipher.o diff --git a/kernel/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c b/kernel/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c new file mode 100644 index 000000000..a19ee127e --- /dev/null +++ b/kernel/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c @@ -0,0 +1,542 @@ +/* + * sun4i-ss-cipher.c - hardware cryptographic accelerator for Allwinner A20 SoC + * + * Copyright (C) 2013-2015 Corentin LABBE + * + * This file add support for AES cipher with 128,192,256 bits + * keysize in CBC and ECB mode. + * Add support also for DES and 3DES in CBC and ECB mode. + * + * You could find the datasheet in Documentation/arm/sunxi/README + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + */ +#include "sun4i-ss.h" + +static int sun4i_ss_opti_poll(struct ablkcipher_request *areq) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); + struct sun4i_ss_ctx *ss = op->ss; + unsigned int ivsize = crypto_ablkcipher_ivsize(tfm); + struct sun4i_cipher_req_ctx *ctx = ablkcipher_request_ctx(areq); + u32 mode = ctx->mode; + /* when activating SS, the default FIFO space is SS_RX_DEFAULT(32) */ + u32 rx_cnt = SS_RX_DEFAULT; + u32 tx_cnt = 0; + u32 spaces; + u32 v; + int i, err = 0; + unsigned int ileft = areq->nbytes; + unsigned int oleft = areq->nbytes; + unsigned int todo; + struct sg_mapping_iter mi, mo; + unsigned int oi, oo; /* offset for in and out */ + + if (areq->nbytes == 0) + return 0; + + if (!areq->info) { + dev_err_ratelimited(ss->dev, "ERROR: Empty IV\n"); + return -EINVAL; + } + + if (!areq->src || !areq->dst) { + dev_err_ratelimited(ss->dev, "ERROR: Some SGs are NULL\n"); + return -EINVAL; + } + + spin_lock_bh(&ss->slock); + + for (i = 0; i < op->keylen; i += 4) + writel(*(op->key + i / 4), ss->base + SS_KEY0 + i); + + if (areq->info) { + for (i = 0; i < 4 && i < ivsize / 4; i++) { + v = *(u32 *)(areq->info + i * 4); + writel(v, ss->base + SS_IV0 + i * 4); + } + } + writel(mode, ss->base + SS_CTL); + + sg_miter_start(&mi, areq->src, sg_nents(areq->src), + SG_MITER_FROM_SG | SG_MITER_ATOMIC); + sg_miter_start(&mo, areq->dst, sg_nents(areq->dst), + SG_MITER_TO_SG | SG_MITER_ATOMIC); + sg_miter_next(&mi); + sg_miter_next(&mo); + if (!mi.addr || !mo.addr) { + dev_err_ratelimited(ss->dev, "ERROR: sg_miter return null\n"); + err = -EINVAL; + goto release_ss; + } + + ileft = areq->nbytes / 4; + oleft = areq->nbytes / 4; + oi = 0; + oo = 0; + do { + todo = min3(rx_cnt, ileft, (mi.length - oi) / 4); + if (todo > 0) { + ileft -= todo; + writesl(ss->base + SS_RXFIFO, mi.addr + oi, todo); + oi += todo * 4; + } + if (oi == mi.length) { + sg_miter_next(&mi); + oi = 0; + } + + spaces = readl(ss->base + SS_FCSR); + rx_cnt = SS_RXFIFO_SPACES(spaces); + tx_cnt = SS_TXFIFO_SPACES(spaces); + + todo = min3(tx_cnt, oleft, (mo.length - oo) / 4); + if (todo > 0) { + oleft -= todo; + readsl(ss->base + SS_TXFIFO, mo.addr + oo, todo); + oo += todo * 4; + } + if (oo == mo.length) { + sg_miter_next(&mo); + oo = 0; + } + } while (oleft > 0); + + if (areq->info) { + for (i = 0; i < 4 && i < ivsize / 4; i++) { + v = readl(ss->base + SS_IV0 + i * 4); + *(u32 *)(areq->info + i * 4) = v; + } + } + +release_ss: + sg_miter_stop(&mi); + sg_miter_stop(&mo); + writel(0, ss->base + SS_CTL); + spin_unlock_bh(&ss->slock); + return err; +} + +/* Generic function that support SG with size not multiple of 4 */ +static int sun4i_ss_cipher_poll(struct ablkcipher_request *areq) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); + struct sun4i_ss_ctx *ss = op->ss; + int no_chunk = 1; + struct scatterlist *in_sg = areq->src; + struct scatterlist *out_sg = areq->dst; + unsigned int ivsize = crypto_ablkcipher_ivsize(tfm); + struct sun4i_cipher_req_ctx *ctx = ablkcipher_request_ctx(areq); + u32 mode = ctx->mode; + /* when activating SS, the default FIFO space is SS_RX_DEFAULT(32) */ + u32 rx_cnt = SS_RX_DEFAULT; + u32 tx_cnt = 0; + u32 v; + u32 spaces; + int i, err = 0; + unsigned int ileft = areq->nbytes; + unsigned int oleft = areq->nbytes; + unsigned int todo; + struct sg_mapping_iter mi, mo; + unsigned int oi, oo; /* offset for in and out */ + char buf[4 * SS_RX_MAX];/* buffer for linearize SG src */ + char bufo[4 * SS_TX_MAX]; /* buffer for linearize SG dst */ + unsigned int ob = 0; /* offset in buf */ + unsigned int obo = 0; /* offset in bufo*/ + unsigned int obl = 0; /* length of data in bufo */ + + if (areq->nbytes == 0) + return 0; + + if (!areq->info) { + dev_err_ratelimited(ss->dev, "ERROR: Empty IV\n"); + return -EINVAL; + } + + if (!areq->src || !areq->dst) { + dev_err_ratelimited(ss->dev, "ERROR: Some SGs are NULL\n"); + return -EINVAL; + } + + /* + * if we have only SGs with size multiple of 4, + * we can use the SS optimized function + */ + while (in_sg && no_chunk == 1) { + if ((in_sg->length % 4) != 0) + no_chunk = 0; + in_sg = sg_next(in_sg); + } + while (out_sg && no_chunk == 1) { + if ((out_sg->length % 4) != 0) + no_chunk = 0; + out_sg = sg_next(out_sg); + } + + if (no_chunk == 1) + return sun4i_ss_opti_poll(areq); + + spin_lock_bh(&ss->slock); + + for (i = 0; i < op->keylen; i += 4) + writel(*(op->key + i / 4), ss->base + SS_KEY0 + i); + + if (areq->info) { + for (i = 0; i < 4 && i < ivsize / 4; i++) { + v = *(u32 *)(areq->info + i * 4); + writel(v, ss->base + SS_IV0 + i * 4); + } + } + writel(mode, ss->base + SS_CTL); + + sg_miter_start(&mi, areq->src, sg_nents(areq->src), + SG_MITER_FROM_SG | SG_MITER_ATOMIC); + sg_miter_start(&mo, areq->dst, sg_nents(areq->dst), + SG_MITER_TO_SG | SG_MITER_ATOMIC); + sg_miter_next(&mi); + sg_miter_next(&mo); + if (!mi.addr || !mo.addr) { + dev_err_ratelimited(ss->dev, "ERROR: sg_miter return null\n"); + err = -EINVAL; + goto release_ss; + } + ileft = areq->nbytes; + oleft = areq->nbytes; + oi = 0; + oo = 0; + + while (oleft > 0) { + if (ileft > 0) { + /* + * todo is the number of consecutive 4byte word that we + * can read from current SG + */ + todo = min3(rx_cnt, ileft / 4, (mi.length - oi) / 4); + if (todo > 0 && ob == 0) { + writesl(ss->base + SS_RXFIFO, mi.addr + oi, + todo); + ileft -= todo * 4; + oi += todo * 4; + } else { + /* + * not enough consecutive bytes, so we need to + * linearize in buf. todo is in bytes + * After that copy, if we have a multiple of 4 + * we need to be able to write all buf in one + * pass, so it is why we min() with rx_cnt + */ + todo = min3(rx_cnt * 4 - ob, ileft, + mi.length - oi); + memcpy(buf + ob, mi.addr + oi, todo); + ileft -= todo; + oi += todo; + ob += todo; + if (ob % 4 == 0) { + writesl(ss->base + SS_RXFIFO, buf, + ob / 4); + ob = 0; + } + } + if (oi == mi.length) { + sg_miter_next(&mi); + oi = 0; + } + } + + spaces = readl(ss->base + SS_FCSR); + rx_cnt = SS_RXFIFO_SPACES(spaces); + tx_cnt = SS_TXFIFO_SPACES(spaces); + dev_dbg(ss->dev, "%x %u/%u %u/%u cnt=%u %u/%u %u/%u cnt=%u %u %u\n", + mode, + oi, mi.length, ileft, areq->nbytes, rx_cnt, + oo, mo.length, oleft, areq->nbytes, tx_cnt, + todo, ob); + + if (tx_cnt == 0) + continue; + /* todo in 4bytes word */ + todo = min3(tx_cnt, oleft / 4, (mo.length - oo) / 4); + if (todo > 0) { + readsl(ss->base + SS_TXFIFO, mo.addr + oo, todo); + oleft -= todo * 4; + oo += todo * 4; + if (oo == mo.length) { + sg_miter_next(&mo); + oo = 0; + } + } else { + /* + * read obl bytes in bufo, we read at maximum for + * emptying the device + */ + readsl(ss->base + SS_TXFIFO, bufo, tx_cnt); + obl = tx_cnt * 4; + obo = 0; + do { + /* + * how many bytes we can copy ? + * no more than remaining SG size + * no more than remaining buffer + * no need to test against oleft + */ + todo = min(mo.length - oo, obl - obo); + memcpy(mo.addr + oo, bufo + obo, todo); + oleft -= todo; + obo += todo; + oo += todo; + if (oo == mo.length) { + sg_miter_next(&mo); + oo = 0; + } + } while (obo < obl); + /* bufo must be fully used here */ + } + } + if (areq->info) { + for (i = 0; i < 4 && i < ivsize / 4; i++) { + v = readl(ss->base + SS_IV0 + i * 4); + *(u32 *)(areq->info + i * 4) = v; + } + } + +release_ss: + sg_miter_stop(&mi); + sg_miter_stop(&mo); + writel(0, ss->base + SS_CTL); + spin_unlock_bh(&ss->slock); + + return err; +} + +/* CBC AES */ +int sun4i_ss_cbc_aes_encrypt(struct ablkcipher_request *areq) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); + + rctx->mode = SS_OP_AES | SS_CBC | SS_ENABLED | SS_ENCRYPTION | + op->keymode; + return sun4i_ss_cipher_poll(areq); +} + +int sun4i_ss_cbc_aes_decrypt(struct ablkcipher_request *areq) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); + + rctx->mode = SS_OP_AES | SS_CBC | SS_ENABLED | SS_DECRYPTION | + op->keymode; + return sun4i_ss_cipher_poll(areq); +} + +/* ECB AES */ +int sun4i_ss_ecb_aes_encrypt(struct ablkcipher_request *areq) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); + + rctx->mode = SS_OP_AES | SS_ECB | SS_ENABLED | SS_ENCRYPTION | + op->keymode; + return sun4i_ss_cipher_poll(areq); +} + +int sun4i_ss_ecb_aes_decrypt(struct ablkcipher_request *areq) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); + + rctx->mode = SS_OP_AES | SS_ECB | SS_ENABLED | SS_DECRYPTION | + op->keymode; + return sun4i_ss_cipher_poll(areq); +} + +/* CBC DES */ +int sun4i_ss_cbc_des_encrypt(struct ablkcipher_request *areq) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); + + rctx->mode = SS_OP_DES | SS_CBC | SS_ENABLED | SS_ENCRYPTION | + op->keymode; + return sun4i_ss_cipher_poll(areq); +} + +int sun4i_ss_cbc_des_decrypt(struct ablkcipher_request *areq) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); + + rctx->mode = SS_OP_DES | SS_CBC | SS_ENABLED | SS_DECRYPTION | + op->keymode; + return sun4i_ss_cipher_poll(areq); +} + +/* ECB DES */ +int sun4i_ss_ecb_des_encrypt(struct ablkcipher_request *areq) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); + + rctx->mode = SS_OP_DES | SS_ECB | SS_ENABLED | SS_ENCRYPTION | + op->keymode; + return sun4i_ss_cipher_poll(areq); +} + +int sun4i_ss_ecb_des_decrypt(struct ablkcipher_request *areq) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); + + rctx->mode = SS_OP_DES | SS_ECB | SS_ENABLED | SS_DECRYPTION | + op->keymode; + return sun4i_ss_cipher_poll(areq); +} + +/* CBC 3DES */ +int sun4i_ss_cbc_des3_encrypt(struct ablkcipher_request *areq) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); + + rctx->mode = SS_OP_3DES | SS_CBC | SS_ENABLED | SS_ENCRYPTION | + op->keymode; + return sun4i_ss_cipher_poll(areq); +} + +int sun4i_ss_cbc_des3_decrypt(struct ablkcipher_request *areq) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); + + rctx->mode = SS_OP_3DES | SS_CBC | SS_ENABLED | SS_DECRYPTION | + op->keymode; + return sun4i_ss_cipher_poll(areq); +} + +/* ECB 3DES */ +int sun4i_ss_ecb_des3_encrypt(struct ablkcipher_request *areq) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); + + rctx->mode = SS_OP_3DES | SS_ECB | SS_ENABLED | SS_ENCRYPTION | + op->keymode; + return sun4i_ss_cipher_poll(areq); +} + +int sun4i_ss_ecb_des3_decrypt(struct ablkcipher_request *areq) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *rctx = ablkcipher_request_ctx(areq); + + rctx->mode = SS_OP_3DES | SS_ECB | SS_ENABLED | SS_DECRYPTION | + op->keymode; + return sun4i_ss_cipher_poll(areq); +} + +int sun4i_ss_cipher_init(struct crypto_tfm *tfm) +{ + struct sun4i_tfm_ctx *op = crypto_tfm_ctx(tfm); + struct crypto_alg *alg = tfm->__crt_alg; + struct sun4i_ss_alg_template *algt; + + memset(op, 0, sizeof(struct sun4i_tfm_ctx)); + + algt = container_of(alg, struct sun4i_ss_alg_template, alg.crypto); + op->ss = algt->ss; + + tfm->crt_ablkcipher.reqsize = sizeof(struct sun4i_cipher_req_ctx); + + return 0; +} + +/* check and set the AES key, prepare the mode to be used */ +int sun4i_ss_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key, + unsigned int keylen) +{ + struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); + struct sun4i_ss_ctx *ss = op->ss; + + switch (keylen) { + case 128 / 8: + op->keymode = SS_AES_128BITS; + break; + case 192 / 8: + op->keymode = SS_AES_192BITS; + break; + case 256 / 8: + op->keymode = SS_AES_256BITS; + break; + default: + dev_err(ss->dev, "ERROR: Invalid keylen %u\n", keylen); + crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + op->keylen = keylen; + memcpy(op->key, key, keylen); + return 0; +} + +/* check and set the DES key, prepare the mode to be used */ +int sun4i_ss_des_setkey(struct crypto_ablkcipher *tfm, const u8 *key, + unsigned int keylen) +{ + struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); + struct sun4i_ss_ctx *ss = op->ss; + u32 flags; + u32 tmp[DES_EXPKEY_WORDS]; + int ret; + + if (unlikely(keylen != DES_KEY_SIZE)) { + dev_err(ss->dev, "Invalid keylen %u\n", keylen); + crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + + flags = crypto_ablkcipher_get_flags(tfm); + + ret = des_ekey(tmp, key); + if (unlikely(ret == 0) && (flags & CRYPTO_TFM_REQ_WEAK_KEY)) { + crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_WEAK_KEY); + dev_dbg(ss->dev, "Weak key %u\n", keylen); + return -EINVAL; + } + + op->keylen = keylen; + memcpy(op->key, key, keylen); + return 0; +} + +/* check and set the 3DES key, prepare the mode to be used */ +int sun4i_ss_des3_setkey(struct crypto_ablkcipher *tfm, const u8 *key, + unsigned int keylen) +{ + struct sun4i_tfm_ctx *op = crypto_ablkcipher_ctx(tfm); + struct sun4i_ss_ctx *ss = op->ss; + + if (unlikely(keylen != 3 * DES_KEY_SIZE)) { + dev_err(ss->dev, "Invalid keylen %u\n", keylen); + crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + op->keylen = keylen; + memcpy(op->key, key, keylen); + return 0; +} diff --git a/kernel/drivers/crypto/sunxi-ss/sun4i-ss-core.c b/kernel/drivers/crypto/sunxi-ss/sun4i-ss-core.c new file mode 100644 index 000000000..107cd2a41 --- /dev/null +++ b/kernel/drivers/crypto/sunxi-ss/sun4i-ss-core.c @@ -0,0 +1,427 @@ +/* + * sun4i-ss-core.c - hardware cryptographic accelerator for Allwinner A20 SoC + * + * Copyright (C) 2013-2015 Corentin LABBE + * + * Core file which registers crypto algorithms supported by the SS. + * + * You could find a link for the datasheet in Documentation/arm/sunxi/README + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sun4i-ss.h" + +static struct sun4i_ss_alg_template ss_algs[] = { +{ .type = CRYPTO_ALG_TYPE_AHASH, + .mode = SS_OP_MD5, + .alg.hash = { + .init = sun4i_hash_init, + .update = sun4i_hash_update, + .final = sun4i_hash_final, + .finup = sun4i_hash_finup, + .digest = sun4i_hash_digest, + .export = sun4i_hash_export_md5, + .import = sun4i_hash_import_md5, + .halg = { + .digestsize = MD5_DIGEST_SIZE, + .statesize = sizeof(struct md5_state), + .base = { + .cra_name = "md5", + .cra_driver_name = "md5-sun4i-ss", + .cra_priority = 300, + .cra_alignmask = 3, + .cra_flags = CRYPTO_ALG_TYPE_AHASH, + .cra_blocksize = MD5_HMAC_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sun4i_req_ctx), + .cra_module = THIS_MODULE, + .cra_type = &crypto_ahash_type, + .cra_init = sun4i_hash_crainit + } + } + } +}, +{ .type = CRYPTO_ALG_TYPE_AHASH, + .mode = SS_OP_SHA1, + .alg.hash = { + .init = sun4i_hash_init, + .update = sun4i_hash_update, + .final = sun4i_hash_final, + .finup = sun4i_hash_finup, + .digest = sun4i_hash_digest, + .export = sun4i_hash_export_sha1, + .import = sun4i_hash_import_sha1, + .halg = { + .digestsize = SHA1_DIGEST_SIZE, + .statesize = sizeof(struct sha1_state), + .base = { + .cra_name = "sha1", + .cra_driver_name = "sha1-sun4i-ss", + .cra_priority = 300, + .cra_alignmask = 3, + .cra_flags = CRYPTO_ALG_TYPE_AHASH, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sun4i_req_ctx), + .cra_module = THIS_MODULE, + .cra_type = &crypto_ahash_type, + .cra_init = sun4i_hash_crainit + } + } + } +}, +{ .type = CRYPTO_ALG_TYPE_ABLKCIPHER, + .alg.crypto = { + .cra_name = "cbc(aes)", + .cra_driver_name = "cbc-aes-sun4i-ss", + .cra_priority = 300, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER, + .cra_ctxsize = sizeof(struct sun4i_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_alignmask = 3, + .cra_type = &crypto_ablkcipher_type, + .cra_init = sun4i_ss_cipher_init, + .cra_ablkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = sun4i_ss_aes_setkey, + .encrypt = sun4i_ss_cbc_aes_encrypt, + .decrypt = sun4i_ss_cbc_aes_decrypt, + } + } +}, +{ .type = CRYPTO_ALG_TYPE_ABLKCIPHER, + .alg.crypto = { + .cra_name = "ecb(aes)", + .cra_driver_name = "ecb-aes-sun4i-ss", + .cra_priority = 300, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER, + .cra_ctxsize = sizeof(struct sun4i_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_alignmask = 3, + .cra_type = &crypto_ablkcipher_type, + .cra_init = sun4i_ss_cipher_init, + .cra_ablkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = sun4i_ss_aes_setkey, + .encrypt = sun4i_ss_ecb_aes_encrypt, + .decrypt = sun4i_ss_ecb_aes_decrypt, + } + } +}, +{ .type = CRYPTO_ALG_TYPE_ABLKCIPHER, + .alg.crypto = { + .cra_name = "cbc(des)", + .cra_driver_name = "cbc-des-sun4i-ss", + .cra_priority = 300, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER, + .cra_ctxsize = sizeof(struct sun4i_req_ctx), + .cra_module = THIS_MODULE, + .cra_alignmask = 3, + .cra_type = &crypto_ablkcipher_type, + .cra_init = sun4i_ss_cipher_init, + .cra_u.ablkcipher = { + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + .setkey = sun4i_ss_des_setkey, + .encrypt = sun4i_ss_cbc_des_encrypt, + .decrypt = sun4i_ss_cbc_des_decrypt, + } + } +}, +{ .type = CRYPTO_ALG_TYPE_ABLKCIPHER, + .alg.crypto = { + .cra_name = "ecb(des)", + .cra_driver_name = "ecb-des-sun4i-ss", + .cra_priority = 300, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER, + .cra_ctxsize = sizeof(struct sun4i_req_ctx), + .cra_module = THIS_MODULE, + .cra_alignmask = 3, + .cra_type = &crypto_ablkcipher_type, + .cra_init = sun4i_ss_cipher_init, + .cra_u.ablkcipher = { + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .setkey = sun4i_ss_des_setkey, + .encrypt = sun4i_ss_ecb_des_encrypt, + .decrypt = sun4i_ss_ecb_des_decrypt, + } + } +}, +{ .type = CRYPTO_ALG_TYPE_ABLKCIPHER, + .alg.crypto = { + .cra_name = "cbc(des3_ede)", + .cra_driver_name = "cbc-des3-sun4i-ss", + .cra_priority = 300, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER, + .cra_ctxsize = sizeof(struct sun4i_req_ctx), + .cra_module = THIS_MODULE, + .cra_alignmask = 3, + .cra_type = &crypto_ablkcipher_type, + .cra_init = sun4i_ss_cipher_init, + .cra_u.ablkcipher = { + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + .setkey = sun4i_ss_des3_setkey, + .encrypt = sun4i_ss_cbc_des3_encrypt, + .decrypt = sun4i_ss_cbc_des3_decrypt, + } + } +}, +{ .type = CRYPTO_ALG_TYPE_ABLKCIPHER, + .alg.crypto = { + .cra_name = "ecb(des3_ede)", + .cra_driver_name = "ecb-des3-sun4i-ss", + .cra_priority = 300, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER, + .cra_ctxsize = sizeof(struct sun4i_req_ctx), + .cra_module = THIS_MODULE, + .cra_alignmask = 3, + .cra_type = &crypto_ablkcipher_type, + .cra_init = sun4i_ss_cipher_init, + .cra_u.ablkcipher = { + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + .setkey = sun4i_ss_des3_setkey, + .encrypt = sun4i_ss_ecb_des3_encrypt, + .decrypt = sun4i_ss_ecb_des3_decrypt, + } + } +}, +}; + +static int sun4i_ss_probe(struct platform_device *pdev) +{ + struct resource *res; + u32 v; + int err, i; + unsigned long cr; + const unsigned long cr_ahb = 24 * 1000 * 1000; + const unsigned long cr_mod = 150 * 1000 * 1000; + struct sun4i_ss_ctx *ss; + + if (!pdev->dev.of_node) + return -ENODEV; + + ss = devm_kzalloc(&pdev->dev, sizeof(*ss), GFP_KERNEL); + if (!ss) + return -ENOMEM; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + ss->base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(ss->base)) { + dev_err(&pdev->dev, "Cannot request MMIO\n"); + return PTR_ERR(ss->base); + } + + ss->ssclk = devm_clk_get(&pdev->dev, "mod"); + if (IS_ERR(ss->ssclk)) { + err = PTR_ERR(ss->ssclk); + dev_err(&pdev->dev, "Cannot get SS clock err=%d\n", err); + return err; + } + dev_dbg(&pdev->dev, "clock ss acquired\n"); + + ss->busclk = devm_clk_get(&pdev->dev, "ahb"); + if (IS_ERR(ss->busclk)) { + err = PTR_ERR(ss->busclk); + dev_err(&pdev->dev, "Cannot get AHB SS clock err=%d\n", err); + return err; + } + dev_dbg(&pdev->dev, "clock ahb_ss acquired\n"); + + ss->reset = devm_reset_control_get_optional(&pdev->dev, "ahb"); + if (IS_ERR(ss->reset)) { + if (PTR_ERR(ss->reset) == -EPROBE_DEFER) + return PTR_ERR(ss->reset); + dev_info(&pdev->dev, "no reset control found\n"); + ss->reset = NULL; + } + + /* Enable both clocks */ + err = clk_prepare_enable(ss->busclk); + if (err != 0) { + dev_err(&pdev->dev, "Cannot prepare_enable busclk\n"); + return err; + } + err = clk_prepare_enable(ss->ssclk); + if (err != 0) { + dev_err(&pdev->dev, "Cannot prepare_enable ssclk\n"); + goto error_ssclk; + } + + /* + * Check that clock have the correct rates given in the datasheet + * Try to set the clock to the maximum allowed + */ + err = clk_set_rate(ss->ssclk, cr_mod); + if (err != 0) { + dev_err(&pdev->dev, "Cannot set clock rate to ssclk\n"); + goto error_clk; + } + + /* Deassert reset if we have a reset control */ + if (ss->reset) { + err = reset_control_deassert(ss->reset); + if (err) { + dev_err(&pdev->dev, "Cannot deassert reset control\n"); + goto error_clk; + } + } + + /* + * The only impact on clocks below requirement are bad performance, + * so do not print "errors" + * warn on Overclocked clocks + */ + cr = clk_get_rate(ss->busclk); + if (cr >= cr_ahb) + dev_dbg(&pdev->dev, "Clock bus %lu (%lu MHz) (must be >= %lu)\n", + cr, cr / 1000000, cr_ahb); + else + dev_warn(&pdev->dev, "Clock bus %lu (%lu MHz) (must be >= %lu)\n", + cr, cr / 1000000, cr_ahb); + + cr = clk_get_rate(ss->ssclk); + if (cr <= cr_mod) + if (cr < cr_mod) + dev_warn(&pdev->dev, "Clock ss %lu (%lu MHz) (must be <= %lu)\n", + cr, cr / 1000000, cr_mod); + else + dev_dbg(&pdev->dev, "Clock ss %lu (%lu MHz) (must be <= %lu)\n", + cr, cr / 1000000, cr_mod); + else + dev_warn(&pdev->dev, "Clock ss is at %lu (%lu MHz) (must be <= %lu)\n", + cr, cr / 1000000, cr_mod); + + /* + * Datasheet named it "Die Bonding ID" + * I expect to be a sort of Security System Revision number. + * Since the A80 seems to have an other version of SS + * this info could be useful + */ + writel(SS_ENABLED, ss->base + SS_CTL); + v = readl(ss->base + SS_CTL); + v >>= 16; + v &= 0x07; + dev_info(&pdev->dev, "Die ID %d\n", v); + writel(0, ss->base + SS_CTL); + + ss->dev = &pdev->dev; + + spin_lock_init(&ss->slock); + + for (i = 0; i < ARRAY_SIZE(ss_algs); i++) { + ss_algs[i].ss = ss; + switch (ss_algs[i].type) { + case CRYPTO_ALG_TYPE_ABLKCIPHER: + err = crypto_register_alg(&ss_algs[i].alg.crypto); + if (err != 0) { + dev_err(ss->dev, "Fail to register %s\n", + ss_algs[i].alg.crypto.cra_name); + goto error_alg; + } + break; + case CRYPTO_ALG_TYPE_AHASH: + err = crypto_register_ahash(&ss_algs[i].alg.hash); + if (err != 0) { + dev_err(ss->dev, "Fail to register %s\n", + ss_algs[i].alg.hash.halg.base.cra_name); + goto error_alg; + } + break; + } + } + platform_set_drvdata(pdev, ss); + return 0; +error_alg: + i--; + for (; i >= 0; i--) { + switch (ss_algs[i].type) { + case CRYPTO_ALG_TYPE_ABLKCIPHER: + crypto_unregister_alg(&ss_algs[i].alg.crypto); + break; + case CRYPTO_ALG_TYPE_AHASH: + crypto_unregister_ahash(&ss_algs[i].alg.hash); + break; + } + } + if (ss->reset) + reset_control_assert(ss->reset); +error_clk: + clk_disable_unprepare(ss->ssclk); +error_ssclk: + clk_disable_unprepare(ss->busclk); + return err; +} + +static int sun4i_ss_remove(struct platform_device *pdev) +{ + int i; + struct sun4i_ss_ctx *ss = platform_get_drvdata(pdev); + + for (i = 0; i < ARRAY_SIZE(ss_algs); i++) { + switch (ss_algs[i].type) { + case CRYPTO_ALG_TYPE_ABLKCIPHER: + crypto_unregister_alg(&ss_algs[i].alg.crypto); + break; + case CRYPTO_ALG_TYPE_AHASH: + crypto_unregister_ahash(&ss_algs[i].alg.hash); + break; + } + } + + writel(0, ss->base + SS_CTL); + if (ss->reset) + reset_control_assert(ss->reset); + clk_disable_unprepare(ss->busclk); + clk_disable_unprepare(ss->ssclk); + return 0; +} + +static const struct of_device_id a20ss_crypto_of_match_table[] = { + { .compatible = "allwinner,sun4i-a10-crypto" }, + {} +}; +MODULE_DEVICE_TABLE(of, a20ss_crypto_of_match_table); + +static struct platform_driver sun4i_ss_driver = { + .probe = sun4i_ss_probe, + .remove = sun4i_ss_remove, + .driver = { + .name = "sun4i-ss", + .of_match_table = a20ss_crypto_of_match_table, + }, +}; + +module_platform_driver(sun4i_ss_driver); + +MODULE_DESCRIPTION("Allwinner Security System cryptographic accelerator"); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Corentin LABBE "); diff --git a/kernel/drivers/crypto/sunxi-ss/sun4i-ss-hash.c b/kernel/drivers/crypto/sunxi-ss/sun4i-ss-hash.c new file mode 100644 index 000000000..ff8031498 --- /dev/null +++ b/kernel/drivers/crypto/sunxi-ss/sun4i-ss-hash.c @@ -0,0 +1,492 @@ +/* + * sun4i-ss-hash.c - hardware cryptographic accelerator for Allwinner A20 SoC + * + * Copyright (C) 2013-2015 Corentin LABBE + * + * This file add support for MD5 and SHA1. + * + * You could find the datasheet in Documentation/arm/sunxi/README + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + */ +#include "sun4i-ss.h" +#include + +/* This is a totally arbitrary value */ +#define SS_TIMEOUT 100 + +int sun4i_hash_crainit(struct crypto_tfm *tfm) +{ + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct sun4i_req_ctx)); + return 0; +} + +/* sun4i_hash_init: initialize request context */ +int sun4i_hash_init(struct ahash_request *areq) +{ + struct sun4i_req_ctx *op = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg); + struct sun4i_ss_alg_template *algt; + struct sun4i_ss_ctx *ss; + + memset(op, 0, sizeof(struct sun4i_req_ctx)); + + algt = container_of(alg, struct sun4i_ss_alg_template, alg.hash); + ss = algt->ss; + op->ss = algt->ss; + op->mode = algt->mode; + + return 0; +} + +int sun4i_hash_export_md5(struct ahash_request *areq, void *out) +{ + struct sun4i_req_ctx *op = ahash_request_ctx(areq); + struct md5_state *octx = out; + int i; + + octx->byte_count = op->byte_count + op->len; + + memcpy(octx->block, op->buf, op->len); + + if (op->byte_count > 0) { + for (i = 0; i < 4; i++) + octx->hash[i] = op->hash[i]; + } else { + octx->hash[0] = SHA1_H0; + octx->hash[1] = SHA1_H1; + octx->hash[2] = SHA1_H2; + octx->hash[3] = SHA1_H3; + } + + return 0; +} + +int sun4i_hash_import_md5(struct ahash_request *areq, const void *in) +{ + struct sun4i_req_ctx *op = ahash_request_ctx(areq); + const struct md5_state *ictx = in; + int i; + + sun4i_hash_init(areq); + + op->byte_count = ictx->byte_count & ~0x3F; + op->len = ictx->byte_count & 0x3F; + + memcpy(op->buf, ictx->block, op->len); + + for (i = 0; i < 4; i++) + op->hash[i] = ictx->hash[i]; + + return 0; +} + +int sun4i_hash_export_sha1(struct ahash_request *areq, void *out) +{ + struct sun4i_req_ctx *op = ahash_request_ctx(areq); + struct sha1_state *octx = out; + int i; + + octx->count = op->byte_count + op->len; + + memcpy(octx->buffer, op->buf, op->len); + + if (op->byte_count > 0) { + for (i = 0; i < 5; i++) + octx->state[i] = op->hash[i]; + } else { + octx->state[0] = SHA1_H0; + octx->state[1] = SHA1_H1; + octx->state[2] = SHA1_H2; + octx->state[3] = SHA1_H3; + octx->state[4] = SHA1_H4; + } + + return 0; +} + +int sun4i_hash_import_sha1(struct ahash_request *areq, const void *in) +{ + struct sun4i_req_ctx *op = ahash_request_ctx(areq); + const struct sha1_state *ictx = in; + int i; + + sun4i_hash_init(areq); + + op->byte_count = ictx->count & ~0x3F; + op->len = ictx->count & 0x3F; + + memcpy(op->buf, ictx->buffer, op->len); + + for (i = 0; i < 5; i++) + op->hash[i] = ictx->state[i]; + + return 0; +} + +/* + * sun4i_hash_update: update hash engine + * + * Could be used for both SHA1 and MD5 + * Write data by step of 32bits and put then in the SS. + * + * Since we cannot leave partial data and hash state in the engine, + * we need to get the hash state at the end of this function. + * We can get the hash state every 64 bytes + * + * So the first work is to get the number of bytes to write to SS modulo 64 + * The extra bytes will go to a temporary buffer op->buf storing op->len bytes + * + * So at the begin of update() + * if op->len + areq->nbytes < 64 + * => all data will be written to wait buffer (op->buf) and end=0 + * if not, write all data from op->buf to the device and position end to + * complete to 64bytes + * + * example 1: + * update1 60o => op->len=60 + * update2 60o => need one more word to have 64 bytes + * end=4 + * so write all data from op->buf and one word of SGs + * write remaining data in op->buf + * final state op->len=56 + */ +int sun4i_hash_update(struct ahash_request *areq) +{ + u32 v, ivmode = 0; + unsigned int i = 0; + /* + * i is the total bytes read from SGs, to be compared to areq->nbytes + * i is important because we cannot rely on SG length since the sum of + * SG->length could be greater than areq->nbytes + */ + + struct sun4i_req_ctx *op = ahash_request_ctx(areq); + struct sun4i_ss_ctx *ss = op->ss; + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + unsigned int in_i = 0; /* advancement in the current SG */ + unsigned int end; + /* + * end is the position when we need to stop writing to the device, + * to be compared to i + */ + int in_r, err = 0; + unsigned int todo; + u32 spaces, rx_cnt = SS_RX_DEFAULT; + size_t copied = 0; + struct sg_mapping_iter mi; + + dev_dbg(ss->dev, "%s %s bc=%llu len=%u mode=%x wl=%u h0=%0x", + __func__, crypto_tfm_alg_name(areq->base.tfm), + op->byte_count, areq->nbytes, op->mode, + op->len, op->hash[0]); + + if (areq->nbytes == 0) + return 0; + + /* protect against overflow */ + if (areq->nbytes > UINT_MAX - op->len) { + dev_err(ss->dev, "Cannot process too large request\n"); + return -EINVAL; + } + + if (op->len + areq->nbytes < 64) { + /* linearize data to op->buf */ + copied = sg_pcopy_to_buffer(areq->src, sg_nents(areq->src), + op->buf + op->len, areq->nbytes, 0); + op->len += copied; + return 0; + } + + end = ((areq->nbytes + op->len) / 64) * 64 - op->len; + + if (end > areq->nbytes || areq->nbytes - end > 63) { + dev_err(ss->dev, "ERROR: Bound error %u %u\n", + end, areq->nbytes); + return -EINVAL; + } + + spin_lock_bh(&ss->slock); + + /* + * if some data have been processed before, + * we need to restore the partial hash state + */ + if (op->byte_count > 0) { + ivmode = SS_IV_ARBITRARY; + for (i = 0; i < 5; i++) + writel(op->hash[i], ss->base + SS_IV0 + i * 4); + } + /* Enable the device */ + writel(op->mode | SS_ENABLED | ivmode, ss->base + SS_CTL); + + i = 0; + sg_miter_start(&mi, areq->src, sg_nents(areq->src), + SG_MITER_FROM_SG | SG_MITER_ATOMIC); + sg_miter_next(&mi); + in_i = 0; + + do { + /* + * we need to linearize in two case: + * - the buffer is already used + * - the SG does not have enough byte remaining ( < 4) + */ + if (op->len > 0 || (mi.length - in_i) < 4) { + /* + * if we have entered here we have two reason to stop + * - the buffer is full + * - reach the end + */ + while (op->len < 64 && i < end) { + /* how many bytes we can read from current SG */ + in_r = min3(mi.length - in_i, end - i, + 64 - op->len); + memcpy(op->buf + op->len, mi.addr + in_i, in_r); + op->len += in_r; + i += in_r; + in_i += in_r; + if (in_i == mi.length) { + sg_miter_next(&mi); + in_i = 0; + } + } + if (op->len > 3 && (op->len % 4) == 0) { + /* write buf to the device */ + writesl(ss->base + SS_RXFIFO, op->buf, + op->len / 4); + op->byte_count += op->len; + op->len = 0; + } + } + if (mi.length - in_i > 3 && i < end) { + /* how many bytes we can read from current SG */ + in_r = min3(mi.length - in_i, areq->nbytes - i, + ((mi.length - in_i) / 4) * 4); + /* how many bytes we can write in the device*/ + todo = min3((u32)(end - i) / 4, rx_cnt, (u32)in_r / 4); + writesl(ss->base + SS_RXFIFO, mi.addr + in_i, todo); + op->byte_count += todo * 4; + i += todo * 4; + in_i += todo * 4; + rx_cnt -= todo; + if (rx_cnt == 0) { + spaces = readl(ss->base + SS_FCSR); + rx_cnt = SS_RXFIFO_SPACES(spaces); + } + if (in_i == mi.length) { + sg_miter_next(&mi); + in_i = 0; + } + } + } while (i < end); + /* final linear */ + if ((areq->nbytes - i) < 64) { + while (i < areq->nbytes && in_i < mi.length && op->len < 64) { + /* how many bytes we can read from current SG */ + in_r = min3(mi.length - in_i, areq->nbytes - i, + 64 - op->len); + memcpy(op->buf + op->len, mi.addr + in_i, in_r); + op->len += in_r; + i += in_r; + in_i += in_r; + if (in_i == mi.length) { + sg_miter_next(&mi); + in_i = 0; + } + } + } + + sg_miter_stop(&mi); + + writel(op->mode | SS_ENABLED | SS_DATA_END, ss->base + SS_CTL); + i = 0; + do { + v = readl(ss->base + SS_CTL); + i++; + } while (i < SS_TIMEOUT && (v & SS_DATA_END) > 0); + if (i >= SS_TIMEOUT) { + dev_err_ratelimited(ss->dev, + "ERROR: hash end timeout %d>%d ctl=%x len=%u\n", + i, SS_TIMEOUT, v, areq->nbytes); + err = -EIO; + goto release_ss; + } + + /* get the partial hash only if something was written */ + for (i = 0; i < crypto_ahash_digestsize(tfm) / 4; i++) + op->hash[i] = readl(ss->base + SS_MD0 + i * 4); + +release_ss: + writel(0, ss->base + SS_CTL); + spin_unlock_bh(&ss->slock); + return err; +} + +/* + * sun4i_hash_final: finalize hashing operation + * + * If we have some remaining bytes, we write them. + * Then ask the SS for finalizing the hashing operation + * + * I do not check RX FIFO size in this function since the size is 32 + * after each enabling and this function neither write more than 32 words. + */ +int sun4i_hash_final(struct ahash_request *areq) +{ + u32 v, ivmode = 0; + unsigned int i; + unsigned int j = 0; + int zeros, err = 0; + unsigned int index, padlen; + __be64 bits; + struct sun4i_req_ctx *op = ahash_request_ctx(areq); + struct sun4i_ss_ctx *ss = op->ss; + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + u32 bf[32]; + u32 wb = 0; + unsigned int nwait, nbw = 0; + + dev_dbg(ss->dev, "%s: byte=%llu len=%u mode=%x wl=%u h=%x", + __func__, op->byte_count, areq->nbytes, op->mode, + op->len, op->hash[0]); + + spin_lock_bh(&ss->slock); + + /* + * if we have already written something, + * restore the partial hash state + */ + if (op->byte_count > 0) { + ivmode = SS_IV_ARBITRARY; + for (i = 0; i < crypto_ahash_digestsize(tfm) / 4; i++) + writel(op->hash[i], ss->base + SS_IV0 + i * 4); + } + writel(op->mode | SS_ENABLED | ivmode, ss->base + SS_CTL); + + /* write the remaining words of the wait buffer */ + if (op->len > 0) { + nwait = op->len / 4; + if (nwait > 0) { + writesl(ss->base + SS_RXFIFO, op->buf, nwait); + op->byte_count += 4 * nwait; + } + nbw = op->len - 4 * nwait; + wb = *(u32 *)(op->buf + nwait * 4); + wb &= (0xFFFFFFFF >> (4 - nbw) * 8); + } + + /* write the remaining bytes of the nbw buffer */ + if (nbw > 0) { + wb |= ((1 << 7) << (nbw * 8)); + bf[j++] = wb; + } else { + bf[j++] = 1 << 7; + } + + /* + * number of space to pad to obtain 64o minus 8(size) minus 4 (final 1) + * I take the operations from other MD5/SHA1 implementations + */ + + /* we have already send 4 more byte of which nbw data */ + if (op->mode == SS_OP_MD5) { + index = (op->byte_count + 4) & 0x3f; + op->byte_count += nbw; + if (index > 56) + zeros = (120 - index) / 4; + else + zeros = (56 - index) / 4; + } else { + op->byte_count += nbw; + index = op->byte_count & 0x3f; + padlen = (index < 56) ? (56 - index) : ((64 + 56) - index); + zeros = (padlen - 1) / 4; + } + + memset(bf + j, 0, 4 * zeros); + j += zeros; + + /* write the length of data */ + if (op->mode == SS_OP_SHA1) { + bits = cpu_to_be64(op->byte_count << 3); + bf[j++] = bits & 0xffffffff; + bf[j++] = (bits >> 32) & 0xffffffff; + } else { + bf[j++] = (op->byte_count << 3) & 0xffffffff; + bf[j++] = (op->byte_count >> 29) & 0xffffffff; + } + writesl(ss->base + SS_RXFIFO, bf, j); + + /* Tell the SS to stop the hashing */ + writel(op->mode | SS_ENABLED | SS_DATA_END, ss->base + SS_CTL); + + /* + * Wait for SS to finish the hash. + * The timeout could happen only in case of bad overcloking + * or driver bug. + */ + i = 0; + do { + v = readl(ss->base + SS_CTL); + i++; + } while (i < SS_TIMEOUT && (v & SS_DATA_END) > 0); + if (i >= SS_TIMEOUT) { + dev_err_ratelimited(ss->dev, + "ERROR: hash end timeout %d>%d ctl=%x len=%u\n", + i, SS_TIMEOUT, v, areq->nbytes); + err = -EIO; + goto release_ss; + } + + /* Get the hash from the device */ + if (op->mode == SS_OP_SHA1) { + for (i = 0; i < 5; i++) { + v = cpu_to_be32(readl(ss->base + SS_MD0 + i * 4)); + memcpy(areq->result + i * 4, &v, 4); + } + } else { + for (i = 0; i < 4; i++) { + v = readl(ss->base + SS_MD0 + i * 4); + memcpy(areq->result + i * 4, &v, 4); + } + } + +release_ss: + writel(0, ss->base + SS_CTL); + spin_unlock_bh(&ss->slock); + return err; +} + +/* sun4i_hash_finup: finalize hashing operation after an update */ +int sun4i_hash_finup(struct ahash_request *areq) +{ + int err; + + err = sun4i_hash_update(areq); + if (err != 0) + return err; + + return sun4i_hash_final(areq); +} + +/* combo of init/update/final functions */ +int sun4i_hash_digest(struct ahash_request *areq) +{ + int err; + + err = sun4i_hash_init(areq); + if (err != 0) + return err; + + err = sun4i_hash_update(areq); + if (err != 0) + return err; + + return sun4i_hash_final(areq); +} diff --git a/kernel/drivers/crypto/sunxi-ss/sun4i-ss.h b/kernel/drivers/crypto/sunxi-ss/sun4i-ss.h new file mode 100644 index 000000000..8e9c05f6e --- /dev/null +++ b/kernel/drivers/crypto/sunxi-ss/sun4i-ss.h @@ -0,0 +1,201 @@ +/* + * sun4i-ss.h - hardware cryptographic accelerator for Allwinner A20 SoC + * + * Copyright (C) 2013-2015 Corentin LABBE + * + * Support AES cipher with 128,192,256 bits keysize. + * Support MD5 and SHA1 hash algorithms. + * Support DES and 3DES + * + * You could find the datasheet in Documentation/arm/sunxi/README + * + * Licensed under the GPL-2. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SS_CTL 0x00 +#define SS_KEY0 0x04 +#define SS_KEY1 0x08 +#define SS_KEY2 0x0C +#define SS_KEY3 0x10 +#define SS_KEY4 0x14 +#define SS_KEY5 0x18 +#define SS_KEY6 0x1C +#define SS_KEY7 0x20 + +#define SS_IV0 0x24 +#define SS_IV1 0x28 +#define SS_IV2 0x2C +#define SS_IV3 0x30 + +#define SS_FCSR 0x44 + +#define SS_MD0 0x4C +#define SS_MD1 0x50 +#define SS_MD2 0x54 +#define SS_MD3 0x58 +#define SS_MD4 0x5C + +#define SS_RXFIFO 0x200 +#define SS_TXFIFO 0x204 + +/* SS_CTL configuration values */ + +/* PRNG generator mode - bit 15 */ +#define SS_PRNG_ONESHOT (0 << 15) +#define SS_PRNG_CONTINUE (1 << 15) + +/* IV mode for hash */ +#define SS_IV_ARBITRARY (1 << 14) + +/* SS operation mode - bits 12-13 */ +#define SS_ECB (0 << 12) +#define SS_CBC (1 << 12) +#define SS_CTS (3 << 12) + +/* Counter width for CNT mode - bits 10-11 */ +#define SS_CNT_16BITS (0 << 10) +#define SS_CNT_32BITS (1 << 10) +#define SS_CNT_64BITS (2 << 10) + +/* Key size for AES - bits 8-9 */ +#define SS_AES_128BITS (0 << 8) +#define SS_AES_192BITS (1 << 8) +#define SS_AES_256BITS (2 << 8) + +/* Operation direction - bit 7 */ +#define SS_ENCRYPTION (0 << 7) +#define SS_DECRYPTION (1 << 7) + +/* SS Method - bits 4-6 */ +#define SS_OP_AES (0 << 4) +#define SS_OP_DES (1 << 4) +#define SS_OP_3DES (2 << 4) +#define SS_OP_SHA1 (3 << 4) +#define SS_OP_MD5 (4 << 4) +#define SS_OP_PRNG (5 << 4) + +/* Data end bit - bit 2 */ +#define SS_DATA_END (1 << 2) + +/* PRNG start bit - bit 1 */ +#define SS_PRNG_START (1 << 1) + +/* SS Enable bit - bit 0 */ +#define SS_DISABLED (0 << 0) +#define SS_ENABLED (1 << 0) + +/* SS_FCSR configuration values */ +/* RX FIFO status - bit 30 */ +#define SS_RXFIFO_FREE (1 << 30) + +/* RX FIFO empty spaces - bits 24-29 */ +#define SS_RXFIFO_SPACES(val) (((val) >> 24) & 0x3f) + +/* TX FIFO status - bit 22 */ +#define SS_TXFIFO_AVAILABLE (1 << 22) + +/* TX FIFO available spaces - bits 16-21 */ +#define SS_TXFIFO_SPACES(val) (((val) >> 16) & 0x3f) + +#define SS_RX_MAX 32 +#define SS_RX_DEFAULT SS_RX_MAX +#define SS_TX_MAX 33 + +#define SS_RXFIFO_EMP_INT_PENDING (1 << 10) +#define SS_TXFIFO_AVA_INT_PENDING (1 << 8) +#define SS_RXFIFO_EMP_INT_ENABLE (1 << 2) +#define SS_TXFIFO_AVA_INT_ENABLE (1 << 0) + +struct sun4i_ss_ctx { + void __iomem *base; + int irq; + struct clk *busclk; + struct clk *ssclk; + struct reset_control *reset; + struct device *dev; + struct resource *res; + spinlock_t slock; /* control the use of the device */ +}; + +struct sun4i_ss_alg_template { + u32 type; + u32 mode; + union { + struct crypto_alg crypto; + struct ahash_alg hash; + } alg; + struct sun4i_ss_ctx *ss; +}; + +struct sun4i_tfm_ctx { + u32 key[AES_MAX_KEY_SIZE / 4];/* divided by sizeof(u32) */ + u32 keylen; + u32 keymode; + struct sun4i_ss_ctx *ss; +}; + +struct sun4i_cipher_req_ctx { + u32 mode; +}; + +struct sun4i_req_ctx { + u32 mode; + u64 byte_count; /* number of bytes "uploaded" to the device */ + u32 hash[5]; /* for storing SS_IVx register */ + char buf[64]; + unsigned int len; + struct sun4i_ss_ctx *ss; +}; + +int sun4i_hash_crainit(struct crypto_tfm *tfm); +int sun4i_hash_init(struct ahash_request *areq); +int sun4i_hash_update(struct ahash_request *areq); +int sun4i_hash_final(struct ahash_request *areq); +int sun4i_hash_finup(struct ahash_request *areq); +int sun4i_hash_digest(struct ahash_request *areq); +int sun4i_hash_export_md5(struct ahash_request *areq, void *out); +int sun4i_hash_import_md5(struct ahash_request *areq, const void *in); +int sun4i_hash_export_sha1(struct ahash_request *areq, void *out); +int sun4i_hash_import_sha1(struct ahash_request *areq, const void *in); + +int sun4i_ss_cbc_aes_encrypt(struct ablkcipher_request *areq); +int sun4i_ss_cbc_aes_decrypt(struct ablkcipher_request *areq); +int sun4i_ss_ecb_aes_encrypt(struct ablkcipher_request *areq); +int sun4i_ss_ecb_aes_decrypt(struct ablkcipher_request *areq); + +int sun4i_ss_cbc_des_encrypt(struct ablkcipher_request *areq); +int sun4i_ss_cbc_des_decrypt(struct ablkcipher_request *areq); +int sun4i_ss_ecb_des_encrypt(struct ablkcipher_request *areq); +int sun4i_ss_ecb_des_decrypt(struct ablkcipher_request *areq); + +int sun4i_ss_cbc_des3_encrypt(struct ablkcipher_request *areq); +int sun4i_ss_cbc_des3_decrypt(struct ablkcipher_request *areq); +int sun4i_ss_ecb_des3_encrypt(struct ablkcipher_request *areq); +int sun4i_ss_ecb_des3_decrypt(struct ablkcipher_request *areq); + +int sun4i_ss_cipher_init(struct crypto_tfm *tfm); +int sun4i_ss_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key, + unsigned int keylen); +int sun4i_ss_des_setkey(struct crypto_ablkcipher *tfm, const u8 *key, + unsigned int keylen); +int sun4i_ss_des3_setkey(struct crypto_ablkcipher *tfm, const u8 *key, + unsigned int keylen); diff --git a/kernel/drivers/crypto/talitos.c b/kernel/drivers/crypto/talitos.c index f062158d4..b6f9f42e2 100644 --- a/kernel/drivers/crypto/talitos.c +++ b/kernel/drivers/crypto/talitos.c @@ -46,7 +46,7 @@ #include #include #include -#include +#include #include #include #include @@ -55,49 +55,92 @@ #include "talitos.h" -static void to_talitos_ptr(struct talitos_ptr *talitos_ptr, dma_addr_t dma_addr) +static void to_talitos_ptr(struct talitos_ptr *ptr, dma_addr_t dma_addr, + bool is_sec1) { - talitos_ptr->ptr = cpu_to_be32(lower_32_bits(dma_addr)); - talitos_ptr->eptr = upper_32_bits(dma_addr); + ptr->ptr = cpu_to_be32(lower_32_bits(dma_addr)); + if (!is_sec1) + ptr->eptr = upper_32_bits(dma_addr); +} + +static void to_talitos_ptr_len(struct talitos_ptr *ptr, unsigned int len, + bool is_sec1) +{ + if (is_sec1) { + ptr->res = 0; + ptr->len1 = cpu_to_be16(len); + } else { + ptr->len = cpu_to_be16(len); + } +} + +static unsigned short from_talitos_ptr_len(struct talitos_ptr *ptr, + bool is_sec1) +{ + if (is_sec1) + return be16_to_cpu(ptr->len1); + else + return be16_to_cpu(ptr->len); +} + +static void to_talitos_ptr_extent_clear(struct talitos_ptr *ptr, bool is_sec1) +{ + if (!is_sec1) + ptr->j_extent = 0; } /* * map virtual single (contiguous) pointer to h/w descriptor pointer */ static void map_single_talitos_ptr(struct device *dev, - struct talitos_ptr *talitos_ptr, - unsigned short len, void *data, - unsigned char extent, + struct talitos_ptr *ptr, + unsigned int len, void *data, enum dma_data_direction dir) { dma_addr_t dma_addr = dma_map_single(dev, data, len, dir); + struct talitos_private *priv = dev_get_drvdata(dev); + bool is_sec1 = has_ftr_sec1(priv); - talitos_ptr->len = cpu_to_be16(len); - to_talitos_ptr(talitos_ptr, dma_addr); - talitos_ptr->j_extent = extent; + to_talitos_ptr_len(ptr, len, is_sec1); + to_talitos_ptr(ptr, dma_addr, is_sec1); + to_talitos_ptr_extent_clear(ptr, is_sec1); } /* * unmap bus single (contiguous) h/w descriptor pointer */ static void unmap_single_talitos_ptr(struct device *dev, - struct talitos_ptr *talitos_ptr, + struct talitos_ptr *ptr, enum dma_data_direction dir) { - dma_unmap_single(dev, be32_to_cpu(talitos_ptr->ptr), - be16_to_cpu(talitos_ptr->len), dir); + struct talitos_private *priv = dev_get_drvdata(dev); + bool is_sec1 = has_ftr_sec1(priv); + + dma_unmap_single(dev, be32_to_cpu(ptr->ptr), + from_talitos_ptr_len(ptr, is_sec1), dir); } static int reset_channel(struct device *dev, int ch) { struct talitos_private *priv = dev_get_drvdata(dev); unsigned int timeout = TALITOS_TIMEOUT; + bool is_sec1 = has_ftr_sec1(priv); - setbits32(priv->chan[ch].reg + TALITOS_CCCR, TALITOS_CCCR_RESET); + if (is_sec1) { + setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, + TALITOS1_CCCR_LO_RESET); - while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) & TALITOS_CCCR_RESET) - && --timeout) - cpu_relax(); + while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR_LO) & + TALITOS1_CCCR_LO_RESET) && --timeout) + cpu_relax(); + } else { + setbits32(priv->chan[ch].reg + TALITOS_CCCR, + TALITOS2_CCCR_RESET); + + while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) & + TALITOS2_CCCR_RESET) && --timeout) + cpu_relax(); + } if (timeout == 0) { dev_err(dev, "failed to reset channel %d\n", ch); @@ -120,11 +163,12 @@ static int reset_device(struct device *dev) { struct talitos_private *priv = dev_get_drvdata(dev); unsigned int timeout = TALITOS_TIMEOUT; - u32 mcr = TALITOS_MCR_SWR; + bool is_sec1 = has_ftr_sec1(priv); + u32 mcr = is_sec1 ? TALITOS1_MCR_SWR : TALITOS2_MCR_SWR; setbits32(priv->reg + TALITOS_MCR, mcr); - while ((in_be32(priv->reg + TALITOS_MCR) & TALITOS_MCR_SWR) + while ((in_be32(priv->reg + TALITOS_MCR) & mcr) && --timeout) cpu_relax(); @@ -148,6 +192,7 @@ static int init_device(struct device *dev) { struct talitos_private *priv = dev_get_drvdata(dev); int ch, err; + bool is_sec1 = has_ftr_sec1(priv); /* * Master reset @@ -171,12 +216,19 @@ static int init_device(struct device *dev) } /* enable channel done and error interrupts */ - setbits32(priv->reg + TALITOS_IMR, TALITOS_IMR_INIT); - setbits32(priv->reg + TALITOS_IMR_LO, TALITOS_IMR_LO_INIT); + if (is_sec1) { + clrbits32(priv->reg + TALITOS_IMR, TALITOS1_IMR_INIT); + clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT); + /* disable parity error check in DEU (erroneous? test vect.) */ + setbits32(priv->reg_deu + TALITOS_EUICR, TALITOS1_DEUICR_KPE); + } else { + setbits32(priv->reg + TALITOS_IMR, TALITOS2_IMR_INIT); + setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT); + } /* disable integrity check error interrupts (use writeback instead) */ if (priv->features & TALITOS_FTR_HW_AUTH_CHECK) - setbits32(priv->reg + TALITOS_MDEUICR_LO, + setbits32(priv->reg_mdeu + TALITOS_EUICR_LO, TALITOS_MDEUICR_LO_ICE); return 0; @@ -204,6 +256,7 @@ int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc, struct talitos_request *request; unsigned long flags; int head; + bool is_sec1 = has_ftr_sec1(priv); spin_lock_irqsave(&priv->chan[ch].head_lock, flags); @@ -217,8 +270,17 @@ int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc, request = &priv->chan[ch].fifo[head]; /* map descriptor and save caller data */ - request->dma_desc = dma_map_single(dev, desc, sizeof(*desc), - DMA_BIDIRECTIONAL); + if (is_sec1) { + desc->hdr1 = desc->hdr; + desc->next_desc = 0; + request->dma_desc = dma_map_single(dev, &desc->hdr1, + TALITOS_DESC_SIZE, + DMA_BIDIRECTIONAL); + } else { + request->dma_desc = dma_map_single(dev, desc, + TALITOS_DESC_SIZE, + DMA_BIDIRECTIONAL); + } request->callback = callback; request->context = context; @@ -250,16 +312,21 @@ static void flush_channel(struct device *dev, int ch, int error, int reset_ch) struct talitos_request *request, saved_req; unsigned long flags; int tail, status; + bool is_sec1 = has_ftr_sec1(priv); spin_lock_irqsave(&priv->chan[ch].tail_lock, flags); tail = priv->chan[ch].tail; while (priv->chan[ch].fifo[tail].desc) { + __be32 hdr; + request = &priv->chan[ch].fifo[tail]; /* descriptors with their done bits set don't get the error */ rmb(); - if ((request->desc->hdr & DESC_HDR_DONE) == DESC_HDR_DONE) + hdr = is_sec1 ? request->desc->hdr1 : request->desc->hdr; + + if ((hdr & DESC_HDR_DONE) == DESC_HDR_DONE) status = 0; else if (!error) @@ -268,7 +335,7 @@ static void flush_channel(struct device *dev, int ch, int error, int reset_ch) status = error; dma_unmap_single(dev, request->dma_desc, - sizeof(struct talitos_desc), + TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL); /* copy entries so we can call callback outside lock */ @@ -302,8 +369,37 @@ static void flush_channel(struct device *dev, int ch, int error, int reset_ch) /* * process completed requests for channels that have done status */ -#define DEF_TALITOS_DONE(name, ch_done_mask) \ -static void talitos_done_##name(unsigned long data) \ +#define DEF_TALITOS1_DONE(name, ch_done_mask) \ +static void talitos1_done_##name(unsigned long data) \ +{ \ + struct device *dev = (struct device *)data; \ + struct talitos_private *priv = dev_get_drvdata(dev); \ + unsigned long flags; \ + \ + if (ch_done_mask & 0x10000000) \ + flush_channel(dev, 0, 0, 0); \ + if (priv->num_channels == 1) \ + goto out; \ + if (ch_done_mask & 0x40000000) \ + flush_channel(dev, 1, 0, 0); \ + if (ch_done_mask & 0x00010000) \ + flush_channel(dev, 2, 0, 0); \ + if (ch_done_mask & 0x00040000) \ + flush_channel(dev, 3, 0, 0); \ + \ +out: \ + /* At this point, all completed channels have been processed */ \ + /* Unmask done interrupts for channels completed later on. */ \ + spin_lock_irqsave(&priv->reg_lock, flags); \ + clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \ + clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT); \ + spin_unlock_irqrestore(&priv->reg_lock, flags); \ +} + +DEF_TALITOS1_DONE(4ch, TALITOS1_ISR_4CHDONE) + +#define DEF_TALITOS2_DONE(name, ch_done_mask) \ +static void talitos2_done_##name(unsigned long data) \ { \ struct device *dev = (struct device *)data; \ struct talitos_private *priv = dev_get_drvdata(dev); \ @@ -325,12 +421,13 @@ out: \ /* Unmask done interrupts for channels completed later on. */ \ spin_lock_irqsave(&priv->reg_lock, flags); \ setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \ - setbits32(priv->reg + TALITOS_IMR_LO, TALITOS_IMR_LO_INIT); \ + setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT); \ spin_unlock_irqrestore(&priv->reg_lock, flags); \ } -DEF_TALITOS_DONE(4ch, TALITOS_ISR_4CHDONE) -DEF_TALITOS_DONE(ch0_2, TALITOS_ISR_CH_0_2_DONE) -DEF_TALITOS_DONE(ch1_3, TALITOS_ISR_CH_1_3_DONE) + +DEF_TALITOS2_DONE(4ch, TALITOS2_ISR_4CHDONE) +DEF_TALITOS2_DONE(ch0_2, TALITOS2_ISR_CH_0_2_DONE) +DEF_TALITOS2_DONE(ch1_3, TALITOS2_ISR_CH_1_3_DONE) /* * locate current (offending) descriptor @@ -377,44 +474,44 @@ static void report_eu_error(struct device *dev, int ch, u32 desc_hdr) switch (desc_hdr & DESC_HDR_SEL0_MASK) { case DESC_HDR_SEL0_AFEU: dev_err(dev, "AFEUISR 0x%08x_%08x\n", - in_be32(priv->reg + TALITOS_AFEUISR), - in_be32(priv->reg + TALITOS_AFEUISR_LO)); + in_be32(priv->reg_afeu + TALITOS_EUISR), + in_be32(priv->reg_afeu + TALITOS_EUISR_LO)); break; case DESC_HDR_SEL0_DEU: dev_err(dev, "DEUISR 0x%08x_%08x\n", - in_be32(priv->reg + TALITOS_DEUISR), - in_be32(priv->reg + TALITOS_DEUISR_LO)); + in_be32(priv->reg_deu + TALITOS_EUISR), + in_be32(priv->reg_deu + TALITOS_EUISR_LO)); break; case DESC_HDR_SEL0_MDEUA: case DESC_HDR_SEL0_MDEUB: dev_err(dev, "MDEUISR 0x%08x_%08x\n", - in_be32(priv->reg + TALITOS_MDEUISR), - in_be32(priv->reg + TALITOS_MDEUISR_LO)); + in_be32(priv->reg_mdeu + TALITOS_EUISR), + in_be32(priv->reg_mdeu + TALITOS_EUISR_LO)); break; case DESC_HDR_SEL0_RNG: dev_err(dev, "RNGUISR 0x%08x_%08x\n", - in_be32(priv->reg + TALITOS_RNGUISR), - in_be32(priv->reg + TALITOS_RNGUISR_LO)); + in_be32(priv->reg_rngu + TALITOS_ISR), + in_be32(priv->reg_rngu + TALITOS_ISR_LO)); break; case DESC_HDR_SEL0_PKEU: dev_err(dev, "PKEUISR 0x%08x_%08x\n", - in_be32(priv->reg + TALITOS_PKEUISR), - in_be32(priv->reg + TALITOS_PKEUISR_LO)); + in_be32(priv->reg_pkeu + TALITOS_EUISR), + in_be32(priv->reg_pkeu + TALITOS_EUISR_LO)); break; case DESC_HDR_SEL0_AESU: dev_err(dev, "AESUISR 0x%08x_%08x\n", - in_be32(priv->reg + TALITOS_AESUISR), - in_be32(priv->reg + TALITOS_AESUISR_LO)); + in_be32(priv->reg_aesu + TALITOS_EUISR), + in_be32(priv->reg_aesu + TALITOS_EUISR_LO)); break; case DESC_HDR_SEL0_CRCU: dev_err(dev, "CRCUISR 0x%08x_%08x\n", - in_be32(priv->reg + TALITOS_CRCUISR), - in_be32(priv->reg + TALITOS_CRCUISR_LO)); + in_be32(priv->reg_crcu + TALITOS_EUISR), + in_be32(priv->reg_crcu + TALITOS_EUISR_LO)); break; case DESC_HDR_SEL0_KEU: dev_err(dev, "KEUISR 0x%08x_%08x\n", - in_be32(priv->reg + TALITOS_KEUISR), - in_be32(priv->reg + TALITOS_KEUISR_LO)); + in_be32(priv->reg_pkeu + TALITOS_EUISR), + in_be32(priv->reg_pkeu + TALITOS_EUISR_LO)); break; } @@ -422,13 +519,13 @@ static void report_eu_error(struct device *dev, int ch, u32 desc_hdr) case DESC_HDR_SEL1_MDEUA: case DESC_HDR_SEL1_MDEUB: dev_err(dev, "MDEUISR 0x%08x_%08x\n", - in_be32(priv->reg + TALITOS_MDEUISR), - in_be32(priv->reg + TALITOS_MDEUISR_LO)); + in_be32(priv->reg_mdeu + TALITOS_EUISR), + in_be32(priv->reg_mdeu + TALITOS_EUISR_LO)); break; case DESC_HDR_SEL1_CRCU: dev_err(dev, "CRCUISR 0x%08x_%08x\n", - in_be32(priv->reg + TALITOS_CRCUISR), - in_be32(priv->reg + TALITOS_CRCUISR_LO)); + in_be32(priv->reg_crcu + TALITOS_EUISR), + in_be32(priv->reg_crcu + TALITOS_EUISR_LO)); break; } @@ -445,17 +542,24 @@ static void talitos_error(struct device *dev, u32 isr, u32 isr_lo) { struct talitos_private *priv = dev_get_drvdata(dev); unsigned int timeout = TALITOS_TIMEOUT; - int ch, error, reset_dev = 0, reset_ch = 0; - u32 v, v_lo; + int ch, error, reset_dev = 0; + u32 v_lo; + bool is_sec1 = has_ftr_sec1(priv); + int reset_ch = is_sec1 ? 1 : 0; /* only SEC2 supports continuation */ for (ch = 0; ch < priv->num_channels; ch++) { /* skip channels without errors */ - if (!(isr & (1 << (ch * 2 + 1)))) - continue; + if (is_sec1) { + /* bits 29, 31, 17, 19 */ + if (!(isr & (1 << (29 + (ch & 1) * 2 - (ch & 2) * 6)))) + continue; + } else { + if (!(isr & (1 << (ch * 2 + 1)))) + continue; + } error = -EINVAL; - v = in_be32(priv->chan[ch].reg + TALITOS_CCPSR); v_lo = in_be32(priv->chan[ch].reg + TALITOS_CCPSR_LO); if (v_lo & TALITOS_CCPSR_LO_DOF) { @@ -471,23 +575,28 @@ static void talitos_error(struct device *dev, u32 isr, u32 isr_lo) if (v_lo & TALITOS_CCPSR_LO_MDTE) dev_err(dev, "master data transfer error\n"); if (v_lo & TALITOS_CCPSR_LO_SGDLZ) - dev_err(dev, "s/g data length zero error\n"); + dev_err(dev, is_sec1 ? "pointeur not complete error\n" + : "s/g data length zero error\n"); if (v_lo & TALITOS_CCPSR_LO_FPZ) - dev_err(dev, "fetch pointer zero error\n"); + dev_err(dev, is_sec1 ? "parity error\n" + : "fetch pointer zero error\n"); if (v_lo & TALITOS_CCPSR_LO_IDH) dev_err(dev, "illegal descriptor header error\n"); if (v_lo & TALITOS_CCPSR_LO_IEU) - dev_err(dev, "invalid execution unit error\n"); + dev_err(dev, is_sec1 ? "static assignment error\n" + : "invalid exec unit error\n"); if (v_lo & TALITOS_CCPSR_LO_EU) report_eu_error(dev, ch, current_desc_hdr(dev, ch)); - if (v_lo & TALITOS_CCPSR_LO_GB) - dev_err(dev, "gather boundary error\n"); - if (v_lo & TALITOS_CCPSR_LO_GRL) - dev_err(dev, "gather return/length error\n"); - if (v_lo & TALITOS_CCPSR_LO_SB) - dev_err(dev, "scatter boundary error\n"); - if (v_lo & TALITOS_CCPSR_LO_SRL) - dev_err(dev, "scatter return/length error\n"); + if (!is_sec1) { + if (v_lo & TALITOS_CCPSR_LO_GB) + dev_err(dev, "gather boundary error\n"); + if (v_lo & TALITOS_CCPSR_LO_GRL) + dev_err(dev, "gather return/length error\n"); + if (v_lo & TALITOS_CCPSR_LO_SB) + dev_err(dev, "scatter boundary error\n"); + if (v_lo & TALITOS_CCPSR_LO_SRL) + dev_err(dev, "scatter return/length error\n"); + } flush_channel(dev, ch, error, reset_ch); @@ -495,10 +604,10 @@ static void talitos_error(struct device *dev, u32 isr, u32 isr_lo) reset_channel(dev, ch); } else { setbits32(priv->chan[ch].reg + TALITOS_CCCR, - TALITOS_CCCR_CONT); + TALITOS2_CCCR_CONT); setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 0); while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) & - TALITOS_CCCR_CONT) && --timeout) + TALITOS2_CCCR_CONT) && --timeout) cpu_relax(); if (timeout == 0) { dev_err(dev, "failed to restart channel %d\n", @@ -507,9 +616,14 @@ static void talitos_error(struct device *dev, u32 isr, u32 isr_lo) } } } - if (reset_dev || isr & ~TALITOS_ISR_4CHERR || isr_lo) { - dev_err(dev, "done overflow, internal time out, or rngu error: " - "ISR 0x%08x_%08x\n", isr, isr_lo); + if (reset_dev || (is_sec1 && isr & ~TALITOS1_ISR_4CHERR) || + (!is_sec1 && isr & ~TALITOS2_ISR_4CHERR) || isr_lo) { + if (is_sec1 && (isr_lo & TALITOS1_ISR_TEA_ERR)) + dev_err(dev, "TEA error: ISR 0x%08x_%08x\n", + isr, isr_lo); + else + dev_err(dev, "done overflow, internal time out, or " + "rngu error: ISR 0x%08x_%08x\n", isr, isr_lo); /* purge request queues */ for (ch = 0; ch < priv->num_channels; ch++) @@ -520,8 +634,43 @@ static void talitos_error(struct device *dev, u32 isr, u32 isr_lo) } } -#define DEF_TALITOS_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \ -static irqreturn_t talitos_interrupt_##name(int irq, void *data) \ +#define DEF_TALITOS1_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \ +static irqreturn_t talitos1_interrupt_##name(int irq, void *data) \ +{ \ + struct device *dev = data; \ + struct talitos_private *priv = dev_get_drvdata(dev); \ + u32 isr, isr_lo; \ + unsigned long flags; \ + \ + spin_lock_irqsave(&priv->reg_lock, flags); \ + isr = in_be32(priv->reg + TALITOS_ISR); \ + isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \ + /* Acknowledge interrupt */ \ + out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \ + out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \ + \ + if (unlikely(isr & ch_err_mask || isr_lo & TALITOS1_IMR_LO_INIT)) { \ + spin_unlock_irqrestore(&priv->reg_lock, flags); \ + talitos_error(dev, isr & ch_err_mask, isr_lo); \ + } \ + else { \ + if (likely(isr & ch_done_mask)) { \ + /* mask further done interrupts. */ \ + setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \ + /* done_task will unmask done interrupts at exit */ \ + tasklet_schedule(&priv->done_task[tlet]); \ + } \ + spin_unlock_irqrestore(&priv->reg_lock, flags); \ + } \ + \ + return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \ + IRQ_NONE; \ +} + +DEF_TALITOS1_INTERRUPT(4ch, TALITOS1_ISR_4CHDONE, TALITOS1_ISR_4CHERR, 0) + +#define DEF_TALITOS2_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \ +static irqreturn_t talitos2_interrupt_##name(int irq, void *data) \ { \ struct device *dev = data; \ struct talitos_private *priv = dev_get_drvdata(dev); \ @@ -552,9 +701,12 @@ static irqreturn_t talitos_interrupt_##name(int irq, void *data) \ return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \ IRQ_NONE; \ } -DEF_TALITOS_INTERRUPT(4ch, TALITOS_ISR_4CHDONE, TALITOS_ISR_4CHERR, 0) -DEF_TALITOS_INTERRUPT(ch0_2, TALITOS_ISR_CH_0_2_DONE, TALITOS_ISR_CH_0_2_ERR, 0) -DEF_TALITOS_INTERRUPT(ch1_3, TALITOS_ISR_CH_1_3_DONE, TALITOS_ISR_CH_1_3_ERR, 1) + +DEF_TALITOS2_INTERRUPT(4ch, TALITOS2_ISR_4CHDONE, TALITOS2_ISR_4CHERR, 0) +DEF_TALITOS2_INTERRUPT(ch0_2, TALITOS2_ISR_CH_0_2_DONE, TALITOS2_ISR_CH_0_2_ERR, + 0) +DEF_TALITOS2_INTERRUPT(ch1_3, TALITOS2_ISR_CH_1_3_DONE, TALITOS2_ISR_CH_1_3_ERR, + 1) /* * hwrng @@ -567,7 +719,7 @@ static int talitos_rng_data_present(struct hwrng *rng, int wait) int i; for (i = 0; i < 20; i++) { - ofl = in_be32(priv->reg + TALITOS_RNGUSR_LO) & + ofl = in_be32(priv->reg_rngu + TALITOS_EUSR_LO) & TALITOS_RNGUSR_LO_OFL; if (ofl || !wait) break; @@ -583,8 +735,8 @@ static int talitos_rng_data_read(struct hwrng *rng, u32 *data) struct talitos_private *priv = dev_get_drvdata(dev); /* rng fifo requires 64-bit accesses */ - *data = in_be32(priv->reg + TALITOS_RNGU_FIFO); - *data = in_be32(priv->reg + TALITOS_RNGU_FIFO_LO); + *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO); + *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO_LO); return sizeof(u32); } @@ -595,8 +747,9 @@ static int talitos_rng_init(struct hwrng *rng) struct talitos_private *priv = dev_get_drvdata(dev); unsigned int timeout = TALITOS_TIMEOUT; - setbits32(priv->reg + TALITOS_RNGURCR_LO, TALITOS_RNGURCR_LO_SR); - while (!(in_be32(priv->reg + TALITOS_RNGUSR_LO) & TALITOS_RNGUSR_LO_RD) + setbits32(priv->reg_rngu + TALITOS_EURCR_LO, TALITOS_RNGURCR_LO_SR); + while (!(in_be32(priv->reg_rngu + TALITOS_EUSR_LO) + & TALITOS_RNGUSR_LO_RD) && --timeout) cpu_relax(); if (timeout == 0) { @@ -605,7 +758,7 @@ static int talitos_rng_init(struct hwrng *rng) } /* start generating */ - setbits32(priv->reg + TALITOS_RNGUDSR_LO, 0); + setbits32(priv->reg_rngu + TALITOS_EUDSR_LO, 0); return 0; } @@ -613,6 +766,7 @@ static int talitos_rng_init(struct hwrng *rng) static int talitos_register_rng(struct device *dev) { struct talitos_private *priv = dev_get_drvdata(dev); + int err; priv->rng.name = dev_driver_string(dev), priv->rng.init = talitos_rng_init, @@ -620,14 +774,22 @@ static int talitos_register_rng(struct device *dev) priv->rng.data_read = talitos_rng_data_read, priv->rng.priv = (unsigned long)dev; - return hwrng_register(&priv->rng); + err = hwrng_register(&priv->rng); + if (!err) + priv->rng_registered = true; + + return err; } static void talitos_unregister_rng(struct device *dev) { struct talitos_private *priv = dev_get_drvdata(dev); + if (!priv->rng_registered) + return; + hwrng_unregister(&priv->rng); + priv->rng_registered = false; } /* @@ -646,7 +808,6 @@ struct talitos_ctx { unsigned int keylen; unsigned int enckeylen; unsigned int authkeylen; - unsigned int authsize; }; #define HASH_MAX_BLOCK_SIZE SHA512_BLOCK_SIZE @@ -661,21 +822,11 @@ struct talitos_ahash_req_ctx { unsigned int first; unsigned int last; unsigned int to_hash_later; - u64 nbuf; + unsigned int nbuf; struct scatterlist bufsl[2]; struct scatterlist *psrc; }; -static int aead_setauthsize(struct crypto_aead *authenc, - unsigned int authsize) -{ - struct talitos_ctx *ctx = crypto_aead_ctx(authenc); - - ctx->authsize = authsize; - - return 0; -} - static int aead_setkey(struct crypto_aead *authenc, const u8 *key, unsigned int keylen) { @@ -704,59 +855,34 @@ badkey: /* * talitos_edesc - s/w-extended descriptor - * @assoc_nents: number of segments in associated data scatterlist * @src_nents: number of segments in input scatterlist * @dst_nents: number of segments in output scatterlist - * @assoc_chained: whether assoc is chained or not - * @src_chained: whether src is chained or not - * @dst_chained: whether dst is chained or not + * @icv_ool: whether ICV is out-of-line * @iv_dma: dma address of iv for checking continuity and link table * @dma_len: length of dma mapped link_tbl space - * @dma_link_tbl: bus physical address of link_tbl + * @dma_link_tbl: bus physical address of link_tbl/buf * @desc: h/w descriptor - * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1) + * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1) (SEC2) + * @buf: input and output buffeur (if {src,dst}_nents > 1) (SEC1) * * if decrypting (with authcheck), or either one of src_nents or dst_nents * is greater than 1, an integrity check value is concatenated to the end * of link_tbl data */ struct talitos_edesc { - int assoc_nents; int src_nents; int dst_nents; - bool assoc_chained; - bool src_chained; - bool dst_chained; + bool icv_ool; dma_addr_t iv_dma; int dma_len; dma_addr_t dma_link_tbl; struct talitos_desc desc; - struct talitos_ptr link_tbl[0]; + union { + struct talitos_ptr link_tbl[0]; + u8 buf[0]; + }; }; -static int talitos_map_sg(struct device *dev, struct scatterlist *sg, - unsigned int nents, enum dma_data_direction dir, - bool chained) -{ - if (unlikely(chained)) - while (sg) { - dma_map_sg(dev, sg, 1, dir); - sg = sg_next(sg); - } - else - dma_map_sg(dev, sg, nents, dir); - return nents; -} - -static void talitos_unmap_sg_chain(struct device *dev, struct scatterlist *sg, - enum dma_data_direction dir) -{ - while (sg) { - dma_unmap_sg(dev, sg, 1, dir); - sg = sg_next(sg); - } -} - static void talitos_sg_unmap(struct device *dev, struct talitos_edesc *edesc, struct scatterlist *src, @@ -766,24 +892,13 @@ static void talitos_sg_unmap(struct device *dev, unsigned int dst_nents = edesc->dst_nents ? : 1; if (src != dst) { - if (edesc->src_chained) - talitos_unmap_sg_chain(dev, src, DMA_TO_DEVICE); - else - dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); + dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); if (dst) { - if (edesc->dst_chained) - talitos_unmap_sg_chain(dev, dst, - DMA_FROM_DEVICE); - else - dma_unmap_sg(dev, dst, dst_nents, - DMA_FROM_DEVICE); + dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE); } } else - if (edesc->src_chained) - talitos_unmap_sg_chain(dev, src, DMA_BIDIRECTIONAL); - else - dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); + dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); } static void ipsec_esp_unmap(struct device *dev, @@ -795,14 +910,6 @@ static void ipsec_esp_unmap(struct device *dev, unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE); unmap_single_talitos_ptr(dev, &edesc->desc.ptr[0], DMA_TO_DEVICE); - if (edesc->assoc_chained) - talitos_unmap_sg_chain(dev, areq->assoc, DMA_TO_DEVICE); - else if (areq->assoclen) - /* assoc_nents counts also for IV in non-contiguous cases */ - dma_unmap_sg(dev, areq->assoc, - edesc->assoc_nents ? edesc->assoc_nents - 1 : 1, - DMA_TO_DEVICE); - talitos_sg_unmap(dev, edesc, areq->src, areq->dst); if (edesc->dma_len) @@ -819,7 +926,7 @@ static void ipsec_esp_encrypt_done(struct device *dev, { struct aead_request *areq = context; struct crypto_aead *authenc = crypto_aead_reqtfm(areq); - struct talitos_ctx *ctx = crypto_aead_ctx(authenc); + unsigned int authsize = crypto_aead_authsize(authenc); struct talitos_edesc *edesc; struct scatterlist *sg; void *icvdata; @@ -829,13 +936,12 @@ static void ipsec_esp_encrypt_done(struct device *dev, ipsec_esp_unmap(dev, edesc, areq); /* copy the generated ICV to dst */ - if (edesc->dst_nents) { + if (edesc->icv_ool) { icvdata = &edesc->link_tbl[edesc->src_nents + - edesc->dst_nents + 2 + - edesc->assoc_nents]; + edesc->dst_nents + 2]; sg = sg_last(areq->dst, edesc->dst_nents); - memcpy((char *)sg_virt(sg) + sg->length - ctx->authsize, - icvdata, ctx->authsize); + memcpy((char *)sg_virt(sg) + sg->length - authsize, + icvdata, authsize); } kfree(edesc); @@ -849,10 +955,10 @@ static void ipsec_esp_decrypt_swauth_done(struct device *dev, { struct aead_request *req = context; struct crypto_aead *authenc = crypto_aead_reqtfm(req); - struct talitos_ctx *ctx = crypto_aead_ctx(authenc); + unsigned int authsize = crypto_aead_authsize(authenc); struct talitos_edesc *edesc; struct scatterlist *sg; - void *icvdata; + char *oicv, *icv; edesc = container_of(desc, struct talitos_edesc, desc); @@ -860,16 +966,18 @@ static void ipsec_esp_decrypt_swauth_done(struct device *dev, if (!err) { /* auth check */ - if (edesc->dma_len) - icvdata = &edesc->link_tbl[edesc->src_nents + - edesc->dst_nents + 2 + - edesc->assoc_nents]; - else - icvdata = &edesc->link_tbl[0]; - sg = sg_last(req->dst, edesc->dst_nents ? : 1); - err = memcmp(icvdata, (char *)sg_virt(sg) + sg->length - - ctx->authsize, ctx->authsize) ? -EBADMSG : 0; + icv = (char *)sg_virt(sg) + sg->length - authsize; + + if (edesc->dma_len) { + oicv = (char *)&edesc->link_tbl[edesc->src_nents + + edesc->dst_nents + 2]; + if (edesc->icv_ool) + icv = oicv + authsize; + } else + oicv = (char *)&edesc->link_tbl[0]; + + err = crypto_memneq(oicv, icv, authsize) ? -EBADMSG : 0; } kfree(edesc); @@ -902,103 +1010,107 @@ static void ipsec_esp_decrypt_hwauth_done(struct device *dev, * convert scatterlist to SEC h/w link table format * stop at cryptlen bytes */ -static int sg_to_link_tbl(struct scatterlist *sg, int sg_count, - int cryptlen, struct talitos_ptr *link_tbl_ptr) +static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count, + unsigned int offset, int cryptlen, + struct talitos_ptr *link_tbl_ptr) { int n_sg = sg_count; + int count = 0; - while (n_sg--) { - to_talitos_ptr(link_tbl_ptr, sg_dma_address(sg)); - link_tbl_ptr->len = cpu_to_be16(sg_dma_len(sg)); - link_tbl_ptr->j_extent = 0; - link_tbl_ptr++; - cryptlen -= sg_dma_len(sg); - sg = sg_next(sg); - } + while (cryptlen && sg && n_sg--) { + unsigned int len = sg_dma_len(sg); + + if (offset >= len) { + offset -= len; + goto next; + } + + len -= offset; - /* adjust (decrease) last one (or two) entry's len to cryptlen */ - link_tbl_ptr--; - while (be16_to_cpu(link_tbl_ptr->len) <= (-cryptlen)) { - /* Empty this entry, and move to previous one */ - cryptlen += be16_to_cpu(link_tbl_ptr->len); - link_tbl_ptr->len = 0; - sg_count--; - link_tbl_ptr--; + if (len > cryptlen) + len = cryptlen; + + to_talitos_ptr(link_tbl_ptr + count, + sg_dma_address(sg) + offset, 0); + link_tbl_ptr[count].len = cpu_to_be16(len); + link_tbl_ptr[count].j_extent = 0; + count++; + cryptlen -= len; + offset = 0; + +next: + sg = sg_next(sg); } - link_tbl_ptr->len = cpu_to_be16(be16_to_cpu(link_tbl_ptr->len) - + cryptlen); /* tag end of link table */ - link_tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN; + if (count > 0) + link_tbl_ptr[count - 1].j_extent = DESC_PTR_LNKTBL_RETURN; - return sg_count; + return count; +} + +static inline int sg_to_link_tbl(struct scatterlist *sg, int sg_count, + int cryptlen, + struct talitos_ptr *link_tbl_ptr) +{ + return sg_to_link_tbl_offset(sg, sg_count, 0, cryptlen, + link_tbl_ptr); } /* * fill in and submit ipsec_esp descriptor */ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, - u64 seq, void (*callback) (struct device *dev, - struct talitos_desc *desc, - void *context, int error)) + void (*callback)(struct device *dev, + struct talitos_desc *desc, + void *context, int error)) { struct crypto_aead *aead = crypto_aead_reqtfm(areq); + unsigned int authsize = crypto_aead_authsize(aead); struct talitos_ctx *ctx = crypto_aead_ctx(aead); struct device *dev = ctx->dev; struct talitos_desc *desc = &edesc->desc; unsigned int cryptlen = areq->cryptlen; - unsigned int authsize = ctx->authsize; unsigned int ivsize = crypto_aead_ivsize(aead); + int tbl_off = 0; int sg_count, ret; int sg_link_tbl_len; /* hmac key */ map_single_talitos_ptr(dev, &desc->ptr[0], ctx->authkeylen, &ctx->key, - 0, DMA_TO_DEVICE); + DMA_TO_DEVICE); + + sg_count = dma_map_sg(dev, areq->src, edesc->src_nents ?: 1, + (areq->src == areq->dst) ? DMA_BIDIRECTIONAL + : DMA_TO_DEVICE); /* hmac data */ - desc->ptr[1].len = cpu_to_be16(areq->assoclen + ivsize); - if (edesc->assoc_nents) { - int tbl_off = edesc->src_nents + edesc->dst_nents + 2; - struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off]; + desc->ptr[1].len = cpu_to_be16(areq->assoclen); + if (sg_count > 1 && + (ret = sg_to_link_tbl_offset(areq->src, sg_count, 0, + areq->assoclen, + &edesc->link_tbl[tbl_off])) > 1) { + tbl_off += ret; to_talitos_ptr(&desc->ptr[1], edesc->dma_link_tbl + tbl_off * - sizeof(struct talitos_ptr)); + sizeof(struct talitos_ptr), 0); desc->ptr[1].j_extent = DESC_PTR_LNKTBL_JUMP; - /* assoc_nents - 1 entries for assoc, 1 for IV */ - sg_count = sg_to_link_tbl(areq->assoc, edesc->assoc_nents - 1, - areq->assoclen, tbl_ptr); - - /* add IV to link table */ - tbl_ptr += sg_count - 1; - tbl_ptr->j_extent = 0; - tbl_ptr++; - to_talitos_ptr(tbl_ptr, edesc->iv_dma); - tbl_ptr->len = cpu_to_be16(ivsize); - tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN; - dma_sync_single_for_device(dev, edesc->dma_link_tbl, edesc->dma_len, DMA_BIDIRECTIONAL); } else { - if (areq->assoclen) - to_talitos_ptr(&desc->ptr[1], - sg_dma_address(areq->assoc)); - else - to_talitos_ptr(&desc->ptr[1], edesc->iv_dma); + to_talitos_ptr(&desc->ptr[1], sg_dma_address(areq->src), 0); desc->ptr[1].j_extent = 0; } /* cipher iv */ - to_talitos_ptr(&desc->ptr[2], edesc->iv_dma); + to_talitos_ptr(&desc->ptr[2], edesc->iv_dma, 0); desc->ptr[2].len = cpu_to_be16(ivsize); desc->ptr[2].j_extent = 0; - /* Sync needed for the aead_givencrypt case */ - dma_sync_single_for_device(dev, edesc->iv_dma, ivsize, DMA_TO_DEVICE); /* cipher key */ map_single_talitos_ptr(dev, &desc->ptr[3], ctx->enckeylen, - (char *)&ctx->key + ctx->authkeylen, 0, + (char *)&ctx->key + ctx->authkeylen, DMA_TO_DEVICE); /* @@ -1010,53 +1122,44 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, desc->ptr[4].len = cpu_to_be16(cryptlen); desc->ptr[4].j_extent = authsize; - sg_count = talitos_map_sg(dev, areq->src, edesc->src_nents ? : 1, - (areq->src == areq->dst) ? DMA_BIDIRECTIONAL - : DMA_TO_DEVICE, - edesc->src_chained); + sg_link_tbl_len = cryptlen; + if (edesc->desc.hdr & DESC_HDR_MODE1_MDEU_CICV) + sg_link_tbl_len += authsize; - if (sg_count == 1) { - to_talitos_ptr(&desc->ptr[4], sg_dma_address(areq->src)); - } else { - sg_link_tbl_len = cryptlen; - - if (edesc->desc.hdr & DESC_HDR_MODE1_MDEU_CICV) - sg_link_tbl_len = cryptlen + authsize; - - sg_count = sg_to_link_tbl(areq->src, sg_count, sg_link_tbl_len, - &edesc->link_tbl[0]); - if (sg_count > 1) { - desc->ptr[4].j_extent |= DESC_PTR_LNKTBL_JUMP; - to_talitos_ptr(&desc->ptr[4], edesc->dma_link_tbl); - dma_sync_single_for_device(dev, edesc->dma_link_tbl, - edesc->dma_len, - DMA_BIDIRECTIONAL); - } else { - /* Only one segment now, so no link tbl needed */ - to_talitos_ptr(&desc->ptr[4], - sg_dma_address(areq->src)); - } - } + if (sg_count > 1 && + (ret = sg_to_link_tbl_offset(areq->src, sg_count, areq->assoclen, + sg_link_tbl_len, + &edesc->link_tbl[tbl_off])) > 1) { + tbl_off += ret; + desc->ptr[4].j_extent |= DESC_PTR_LNKTBL_JUMP; + to_talitos_ptr(&desc->ptr[4], edesc->dma_link_tbl + + tbl_off * + sizeof(struct talitos_ptr), 0); + dma_sync_single_for_device(dev, edesc->dma_link_tbl, + edesc->dma_len, + DMA_BIDIRECTIONAL); + } else + to_talitos_ptr(&desc->ptr[4], sg_dma_address(areq->src), 0); /* cipher out */ desc->ptr[5].len = cpu_to_be16(cryptlen); desc->ptr[5].j_extent = authsize; if (areq->src != areq->dst) - sg_count = talitos_map_sg(dev, areq->dst, - edesc->dst_nents ? : 1, - DMA_FROM_DEVICE, edesc->dst_chained); + sg_count = dma_map_sg(dev, areq->dst, edesc->dst_nents ? : 1, + DMA_FROM_DEVICE); - if (sg_count == 1) { - to_talitos_ptr(&desc->ptr[5], sg_dma_address(areq->dst)); - } else { - int tbl_off = edesc->src_nents + 1; + edesc->icv_ool = false; + + if (sg_count > 1 && + (sg_count = sg_to_link_tbl_offset(areq->dst, sg_count, + areq->assoclen, cryptlen, + &edesc->link_tbl[tbl_off])) > + 1) { struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off]; to_talitos_ptr(&desc->ptr[5], edesc->dma_link_tbl + - tbl_off * sizeof(struct talitos_ptr)); - sg_count = sg_to_link_tbl(areq->dst, sg_count, cryptlen, - tbl_ptr); + tbl_off * sizeof(struct talitos_ptr), 0); /* Add an entry to the link table for ICV data */ tbl_ptr += sg_count - 1; @@ -1067,16 +1170,19 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, /* icv data follows link tables */ to_talitos_ptr(tbl_ptr, edesc->dma_link_tbl + - (tbl_off + edesc->dst_nents + 1 + - edesc->assoc_nents) * - sizeof(struct talitos_ptr)); + (edesc->src_nents + edesc->dst_nents + + 2) * sizeof(struct talitos_ptr) + + authsize, 0); desc->ptr[5].j_extent |= DESC_PTR_LNKTBL_JUMP; dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl, edesc->dma_len, DMA_BIDIRECTIONAL); - } + + edesc->icv_ool = true; + } else + to_talitos_ptr(&desc->ptr[5], sg_dma_address(areq->dst), 0); /* iv out */ - map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv, 0, + map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv, DMA_FROM_DEVICE); ret = talitos_submit(dev, ctx->ch, desc, callback, areq); @@ -1087,31 +1193,10 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, return ret; } -/* - * derive number of elements in scatterlist - */ -static int sg_count(struct scatterlist *sg_list, int nbytes, bool *chained) -{ - struct scatterlist *sg = sg_list; - int sg_nents = 0; - - *chained = false; - while (nbytes > 0) { - sg_nents++; - nbytes -= sg->length; - if (!sg_is_last(sg) && (sg + 1)->length == 0) - *chained = true; - sg = sg_next(sg); - } - - return sg_nents; -} - /* * allocate and map the extended descriptor */ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, - struct scatterlist *assoc, struct scatterlist *src, struct scatterlist *dst, u8 *iv, @@ -1124,13 +1209,15 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, bool encrypt) { struct talitos_edesc *edesc; - int assoc_nents = 0, src_nents, dst_nents, alloc_len, dma_len; - bool assoc_chained = false, src_chained = false, dst_chained = false; + int src_nents, dst_nents, alloc_len, dma_len; dma_addr_t iv_dma = 0; gfp_t flags = cryptoflags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; + struct talitos_private *priv = dev_get_drvdata(dev); + bool is_sec1 = has_ftr_sec1(priv); + int max_len = is_sec1 ? TALITOS1_MAX_DATA_LEN : TALITOS2_MAX_DATA_LEN; - if (cryptlen + authsize > TALITOS_MAX_DATA_LEN) { + if (cryptlen + authsize > max_len) { dev_err(dev, "length exceeds h/w max limit\n"); return ERR_PTR(-EINVAL); } @@ -1138,44 +1225,33 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, if (ivsize) iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE); - if (assoclen) { - /* - * Currently it is assumed that iv is provided whenever assoc - * is. - */ - BUG_ON(!iv); - - assoc_nents = sg_count(assoc, assoclen, &assoc_chained); - talitos_map_sg(dev, assoc, assoc_nents, DMA_TO_DEVICE, - assoc_chained); - assoc_nents = (assoc_nents == 1) ? 0 : assoc_nents; - - if (assoc_nents || sg_dma_address(assoc) + assoclen != iv_dma) - assoc_nents = assoc_nents ? assoc_nents + 1 : 2; - } - if (!dst || dst == src) { - src_nents = sg_count(src, cryptlen + authsize, &src_chained); + src_nents = sg_nents_for_len(src, + assoclen + cryptlen + authsize); src_nents = (src_nents == 1) ? 0 : src_nents; dst_nents = dst ? src_nents : 0; } else { /* dst && dst != src*/ - src_nents = sg_count(src, cryptlen + (encrypt ? 0 : authsize), - &src_chained); + src_nents = sg_nents_for_len(src, assoclen + cryptlen + + (encrypt ? 0 : authsize)); src_nents = (src_nents == 1) ? 0 : src_nents; - dst_nents = sg_count(dst, cryptlen + (encrypt ? authsize : 0), - &dst_chained); + dst_nents = sg_nents_for_len(dst, assoclen + cryptlen + + (encrypt ? authsize : 0)); dst_nents = (dst_nents == 1) ? 0 : dst_nents; } /* * allocate space for base edesc plus the link tables, - * allowing for two separate entries for ICV and generated ICV (+ 2), - * and the ICV data itself + * allowing for two separate entries for AD and generated ICV (+ 2), + * and space for two sets of ICVs (stashed and generated) */ alloc_len = sizeof(struct talitos_edesc); - if (assoc_nents || src_nents || dst_nents) { - dma_len = (src_nents + dst_nents + 2 + assoc_nents) * - sizeof(struct talitos_ptr) + authsize; + if (src_nents || dst_nents) { + if (is_sec1) + dma_len = (src_nents ? cryptlen : 0) + + (dst_nents ? cryptlen : 0); + else + dma_len = (src_nents + dst_nents + 2) * + sizeof(struct talitos_ptr) + authsize * 2; alloc_len += dma_len; } else { dma_len = 0; @@ -1184,13 +1260,6 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, edesc = kmalloc(alloc_len, GFP_DMA | flags); if (!edesc) { - if (assoc_chained) - talitos_unmap_sg_chain(dev, assoc, DMA_TO_DEVICE); - else if (assoclen) - dma_unmap_sg(dev, assoc, - assoc_nents ? assoc_nents - 1 : 1, - DMA_TO_DEVICE); - if (iv_dma) dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE); @@ -1198,12 +1267,8 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, return ERR_PTR(-ENOMEM); } - edesc->assoc_nents = assoc_nents; edesc->src_nents = src_nents; edesc->dst_nents = dst_nents; - edesc->assoc_chained = assoc_chained; - edesc->src_chained = src_chained; - edesc->dst_chained = dst_chained; edesc->iv_dma = iv_dma; edesc->dma_len = dma_len; if (dma_len) @@ -1218,12 +1283,13 @@ static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq, u8 *iv, int icv_stashing, bool encrypt) { struct crypto_aead *authenc = crypto_aead_reqtfm(areq); + unsigned int authsize = crypto_aead_authsize(authenc); struct talitos_ctx *ctx = crypto_aead_ctx(authenc); unsigned int ivsize = crypto_aead_ivsize(authenc); - return talitos_edesc_alloc(ctx->dev, areq->assoc, areq->src, areq->dst, + return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst, iv, areq->assoclen, areq->cryptlen, - ctx->authsize, ivsize, icv_stashing, + authsize, ivsize, icv_stashing, areq->base.flags, encrypt); } @@ -1241,14 +1307,14 @@ static int aead_encrypt(struct aead_request *req) /* set encrypt */ edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT; - return ipsec_esp(edesc, req, 0, ipsec_esp_encrypt_done); + return ipsec_esp(edesc, req, ipsec_esp_encrypt_done); } static int aead_decrypt(struct aead_request *req) { struct crypto_aead *authenc = crypto_aead_reqtfm(req); + unsigned int authsize = crypto_aead_authsize(authenc); struct talitos_ctx *ctx = crypto_aead_ctx(authenc); - unsigned int authsize = ctx->authsize; struct talitos_private *priv = dev_get_drvdata(ctx->dev); struct talitos_edesc *edesc; struct scatterlist *sg; @@ -1273,7 +1339,7 @@ static int aead_decrypt(struct aead_request *req) /* reset integrity check result bits */ edesc->desc.hdr_lo = 0; - return ipsec_esp(edesc, req, 0, ipsec_esp_decrypt_hwauth_done); + return ipsec_esp(edesc, req, ipsec_esp_decrypt_hwauth_done); } /* Have to check the ICV with software */ @@ -1281,40 +1347,16 @@ static int aead_decrypt(struct aead_request *req) /* stash incoming ICV for later cmp with ICV generated by the h/w */ if (edesc->dma_len) - icvdata = &edesc->link_tbl[edesc->src_nents + - edesc->dst_nents + 2 + - edesc->assoc_nents]; + icvdata = (char *)&edesc->link_tbl[edesc->src_nents + + edesc->dst_nents + 2]; else icvdata = &edesc->link_tbl[0]; sg = sg_last(req->src, edesc->src_nents ? : 1); - memcpy(icvdata, (char *)sg_virt(sg) + sg->length - ctx->authsize, - ctx->authsize); - - return ipsec_esp(edesc, req, 0, ipsec_esp_decrypt_swauth_done); -} - -static int aead_givencrypt(struct aead_givcrypt_request *req) -{ - struct aead_request *areq = &req->areq; - struct crypto_aead *authenc = crypto_aead_reqtfm(areq); - struct talitos_ctx *ctx = crypto_aead_ctx(authenc); - struct talitos_edesc *edesc; - - /* allocate extended descriptor */ - edesc = aead_edesc_alloc(areq, req->giv, 0, true); - if (IS_ERR(edesc)) - return PTR_ERR(edesc); - - /* set encrypt */ - edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT; - - memcpy(req->giv, ctx->iv, crypto_aead_ivsize(authenc)); - /* avoid consecutive packets going out with same IV */ - *(__be64 *)req->giv ^= cpu_to_be64(req->seq); + memcpy(icvdata, (char *)sg_virt(sg) + sg->length - authsize, authsize); - return ipsec_esp(edesc, areq, req->seq, ipsec_esp_encrypt_done); + return ipsec_esp(edesc, req, ipsec_esp_decrypt_swauth_done); } static int ablkcipher_setkey(struct crypto_ablkcipher *cipher, @@ -1328,16 +1370,43 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *cipher, return 0; } +static void unmap_sg_talitos_ptr(struct device *dev, struct scatterlist *src, + struct scatterlist *dst, unsigned int len, + struct talitos_edesc *edesc) +{ + struct talitos_private *priv = dev_get_drvdata(dev); + bool is_sec1 = has_ftr_sec1(priv); + + if (is_sec1) { + if (!edesc->src_nents) { + dma_unmap_sg(dev, src, 1, + dst != src ? DMA_TO_DEVICE + : DMA_BIDIRECTIONAL); + } + if (dst && edesc->dst_nents) { + dma_sync_single_for_device(dev, + edesc->dma_link_tbl + len, + len, DMA_FROM_DEVICE); + sg_copy_from_buffer(dst, edesc->dst_nents ? : 1, + edesc->buf + len, len); + } else if (dst && dst != src) { + dma_unmap_sg(dev, dst, 1, DMA_FROM_DEVICE); + } + } else { + talitos_sg_unmap(dev, edesc, src, dst); + } +} + static void common_nonsnoop_unmap(struct device *dev, struct talitos_edesc *edesc, struct ablkcipher_request *areq) { unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE); + + unmap_sg_talitos_ptr(dev, areq->src, areq->dst, areq->nbytes, edesc); unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE); unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], DMA_TO_DEVICE); - talitos_sg_unmap(dev, edesc, areq->src, areq->dst); - if (edesc->dma_len) dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len, DMA_BIDIRECTIONAL); @@ -1359,6 +1428,100 @@ static void ablkcipher_done(struct device *dev, areq->base.complete(&areq->base, err); } +int map_sg_in_talitos_ptr(struct device *dev, struct scatterlist *src, + unsigned int len, struct talitos_edesc *edesc, + enum dma_data_direction dir, struct talitos_ptr *ptr) +{ + int sg_count; + struct talitos_private *priv = dev_get_drvdata(dev); + bool is_sec1 = has_ftr_sec1(priv); + + to_talitos_ptr_len(ptr, len, is_sec1); + + if (is_sec1) { + sg_count = edesc->src_nents ? : 1; + + if (sg_count == 1) { + dma_map_sg(dev, src, 1, dir); + to_talitos_ptr(ptr, sg_dma_address(src), is_sec1); + } else { + sg_copy_to_buffer(src, sg_count, edesc->buf, len); + to_talitos_ptr(ptr, edesc->dma_link_tbl, is_sec1); + dma_sync_single_for_device(dev, edesc->dma_link_tbl, + len, DMA_TO_DEVICE); + } + } else { + to_talitos_ptr_extent_clear(ptr, is_sec1); + + sg_count = dma_map_sg(dev, src, edesc->src_nents ? : 1, dir); + + if (sg_count == 1) { + to_talitos_ptr(ptr, sg_dma_address(src), is_sec1); + } else { + sg_count = sg_to_link_tbl(src, sg_count, len, + &edesc->link_tbl[0]); + if (sg_count > 1) { + to_talitos_ptr(ptr, edesc->dma_link_tbl, 0); + ptr->j_extent |= DESC_PTR_LNKTBL_JUMP; + dma_sync_single_for_device(dev, + edesc->dma_link_tbl, + edesc->dma_len, + DMA_BIDIRECTIONAL); + } else { + /* Only one segment now, so no link tbl needed*/ + to_talitos_ptr(ptr, sg_dma_address(src), + is_sec1); + } + } + } + return sg_count; +} + +void map_sg_out_talitos_ptr(struct device *dev, struct scatterlist *dst, + unsigned int len, struct talitos_edesc *edesc, + enum dma_data_direction dir, + struct talitos_ptr *ptr, int sg_count) +{ + struct talitos_private *priv = dev_get_drvdata(dev); + bool is_sec1 = has_ftr_sec1(priv); + + if (dir != DMA_NONE) + sg_count = dma_map_sg(dev, dst, edesc->dst_nents ? : 1, dir); + + to_talitos_ptr_len(ptr, len, is_sec1); + + if (is_sec1) { + if (sg_count == 1) { + if (dir != DMA_NONE) + dma_map_sg(dev, dst, 1, dir); + to_talitos_ptr(ptr, sg_dma_address(dst), is_sec1); + } else { + to_talitos_ptr(ptr, edesc->dma_link_tbl + len, is_sec1); + dma_sync_single_for_device(dev, + edesc->dma_link_tbl + len, + len, DMA_FROM_DEVICE); + } + } else { + to_talitos_ptr_extent_clear(ptr, is_sec1); + + if (sg_count == 1) { + to_talitos_ptr(ptr, sg_dma_address(dst), is_sec1); + } else { + struct talitos_ptr *link_tbl_ptr = + &edesc->link_tbl[edesc->src_nents + 1]; + + to_talitos_ptr(ptr, edesc->dma_link_tbl + + (edesc->src_nents + 1) * + sizeof(struct talitos_ptr), 0); + ptr->j_extent |= DESC_PTR_LNKTBL_JUMP; + sg_to_link_tbl(dst, sg_count, len, link_tbl_ptr); + dma_sync_single_for_device(dev, edesc->dma_link_tbl, + edesc->dma_len, + DMA_BIDIRECTIONAL); + } + } +} + static int common_nonsnoop(struct talitos_edesc *edesc, struct ablkcipher_request *areq, void (*callback) (struct device *dev, @@ -1372,83 +1535,41 @@ static int common_nonsnoop(struct talitos_edesc *edesc, unsigned int cryptlen = areq->nbytes; unsigned int ivsize = crypto_ablkcipher_ivsize(cipher); int sg_count, ret; + struct talitos_private *priv = dev_get_drvdata(dev); + bool is_sec1 = has_ftr_sec1(priv); /* first DWORD empty */ - desc->ptr[0].len = 0; - to_talitos_ptr(&desc->ptr[0], 0); - desc->ptr[0].j_extent = 0; + desc->ptr[0] = zero_entry; /* cipher iv */ - to_talitos_ptr(&desc->ptr[1], edesc->iv_dma); - desc->ptr[1].len = cpu_to_be16(ivsize); - desc->ptr[1].j_extent = 0; + to_talitos_ptr(&desc->ptr[1], edesc->iv_dma, is_sec1); + to_talitos_ptr_len(&desc->ptr[1], ivsize, is_sec1); + to_talitos_ptr_extent_clear(&desc->ptr[1], is_sec1); /* cipher key */ map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen, - (char *)&ctx->key, 0, DMA_TO_DEVICE); + (char *)&ctx->key, DMA_TO_DEVICE); /* * cipher in */ - desc->ptr[3].len = cpu_to_be16(cryptlen); - desc->ptr[3].j_extent = 0; - - sg_count = talitos_map_sg(dev, areq->src, edesc->src_nents ? : 1, - (areq->src == areq->dst) ? DMA_BIDIRECTIONAL - : DMA_TO_DEVICE, - edesc->src_chained); - - if (sg_count == 1) { - to_talitos_ptr(&desc->ptr[3], sg_dma_address(areq->src)); - } else { - sg_count = sg_to_link_tbl(areq->src, sg_count, cryptlen, - &edesc->link_tbl[0]); - if (sg_count > 1) { - to_talitos_ptr(&desc->ptr[3], edesc->dma_link_tbl); - desc->ptr[3].j_extent |= DESC_PTR_LNKTBL_JUMP; - dma_sync_single_for_device(dev, edesc->dma_link_tbl, - edesc->dma_len, - DMA_BIDIRECTIONAL); - } else { - /* Only one segment now, so no link tbl needed */ - to_talitos_ptr(&desc->ptr[3], - sg_dma_address(areq->src)); - } - } + sg_count = map_sg_in_talitos_ptr(dev, areq->src, cryptlen, edesc, + (areq->src == areq->dst) ? + DMA_BIDIRECTIONAL : DMA_TO_DEVICE, + &desc->ptr[3]); /* cipher out */ - desc->ptr[4].len = cpu_to_be16(cryptlen); - desc->ptr[4].j_extent = 0; - - if (areq->src != areq->dst) - sg_count = talitos_map_sg(dev, areq->dst, - edesc->dst_nents ? : 1, - DMA_FROM_DEVICE, edesc->dst_chained); - - if (sg_count == 1) { - to_talitos_ptr(&desc->ptr[4], sg_dma_address(areq->dst)); - } else { - struct talitos_ptr *link_tbl_ptr = - &edesc->link_tbl[edesc->src_nents + 1]; - - to_talitos_ptr(&desc->ptr[4], edesc->dma_link_tbl + - (edesc->src_nents + 1) * - sizeof(struct talitos_ptr)); - desc->ptr[4].j_extent |= DESC_PTR_LNKTBL_JUMP; - sg_count = sg_to_link_tbl(areq->dst, sg_count, cryptlen, - link_tbl_ptr); - dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl, - edesc->dma_len, DMA_BIDIRECTIONAL); - } + map_sg_out_talitos_ptr(dev, areq->dst, cryptlen, edesc, + (areq->src == areq->dst) ? DMA_NONE + : DMA_FROM_DEVICE, + &desc->ptr[4], sg_count); /* iv out */ - map_single_talitos_ptr(dev, &desc->ptr[5], ivsize, ctx->iv, 0, + map_single_talitos_ptr(dev, &desc->ptr[5], ivsize, ctx->iv, DMA_FROM_DEVICE); /* last DWORD empty */ - desc->ptr[6].len = 0; - to_talitos_ptr(&desc->ptr[6], 0); - desc->ptr[6].j_extent = 0; + desc->ptr[6] = zero_entry; ret = talitos_submit(dev, ctx->ch, desc, callback, areq); if (ret != -EINPROGRESS) { @@ -1465,7 +1586,7 @@ static struct talitos_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request * struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher); unsigned int ivsize = crypto_ablkcipher_ivsize(cipher); - return talitos_edesc_alloc(ctx->dev, NULL, areq->src, areq->dst, + return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst, areq->info, 0, areq->nbytes, 0, ivsize, 0, areq->base.flags, encrypt); } @@ -1508,20 +1629,22 @@ static void common_nonsnoop_hash_unmap(struct device *dev, struct ahash_request *areq) { struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); + struct talitos_private *priv = dev_get_drvdata(dev); + bool is_sec1 = has_ftr_sec1(priv); unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE); + unmap_sg_talitos_ptr(dev, req_ctx->psrc, NULL, 0, edesc); + /* When using hashctx-in, must unmap it. */ - if (edesc->desc.ptr[1].len) + if (from_talitos_ptr_len(&edesc->desc.ptr[1], is_sec1)) unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], DMA_TO_DEVICE); - if (edesc->desc.ptr[2].len) + if (from_talitos_ptr_len(&edesc->desc.ptr[2], is_sec1)) unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE); - talitos_sg_unmap(dev, edesc, req_ctx->psrc, NULL); - if (edesc->dma_len) dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len, DMA_BIDIRECTIONAL); @@ -1549,6 +1672,27 @@ static void ahash_done(struct device *dev, areq->base.complete(&areq->base, err); } +/* + * SEC1 doesn't like hashing of 0 sized message, so we do the padding + * ourself and submit a padded block + */ +void talitos_handle_buggy_hash(struct talitos_ctx *ctx, + struct talitos_edesc *edesc, + struct talitos_ptr *ptr) +{ + static u8 padded_hash[64] = { + 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + }; + + pr_err_once("Bug in SEC1, padding ourself\n"); + edesc->desc.hdr &= ~DESC_HDR_MODE0_MDEU_PAD; + map_single_talitos_ptr(ctx->dev, ptr, sizeof(padded_hash), + (char *)padded_hash, DMA_TO_DEVICE); +} + static int common_nonsnoop_hash(struct talitos_edesc *edesc, struct ahash_request *areq, unsigned int length, void (*callback) (struct device *dev, @@ -1560,7 +1704,9 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); struct device *dev = ctx->dev; struct talitos_desc *desc = &edesc->desc; - int sg_count, ret; + int ret; + struct talitos_private *priv = dev_get_drvdata(dev); + bool is_sec1 = has_ftr_sec1(priv); /* first DWORD empty */ desc->ptr[0] = zero_entry; @@ -1569,7 +1715,7 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, if (!req_ctx->first || req_ctx->swinit) { map_single_talitos_ptr(dev, &desc->ptr[1], req_ctx->hw_context_size, - (char *)req_ctx->hw_context, 0, + (char *)req_ctx->hw_context, DMA_TO_DEVICE); req_ctx->swinit = 0; } else { @@ -1581,38 +1727,15 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, /* HMAC key */ if (ctx->keylen) map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen, - (char *)&ctx->key, 0, DMA_TO_DEVICE); + (char *)&ctx->key, DMA_TO_DEVICE); else desc->ptr[2] = zero_entry; /* * data in */ - desc->ptr[3].len = cpu_to_be16(length); - desc->ptr[3].j_extent = 0; - - sg_count = talitos_map_sg(dev, req_ctx->psrc, - edesc->src_nents ? : 1, - DMA_TO_DEVICE, edesc->src_chained); - - if (sg_count == 1) { - to_talitos_ptr(&desc->ptr[3], sg_dma_address(req_ctx->psrc)); - } else { - sg_count = sg_to_link_tbl(req_ctx->psrc, sg_count, length, - &edesc->link_tbl[0]); - if (sg_count > 1) { - desc->ptr[3].j_extent |= DESC_PTR_LNKTBL_JUMP; - to_talitos_ptr(&desc->ptr[3], edesc->dma_link_tbl); - dma_sync_single_for_device(ctx->dev, - edesc->dma_link_tbl, - edesc->dma_len, - DMA_BIDIRECTIONAL); - } else { - /* Only one segment now, so no link tbl needed */ - to_talitos_ptr(&desc->ptr[3], - sg_dma_address(req_ctx->psrc)); - } - } + map_sg_in_talitos_ptr(dev, req_ctx->psrc, length, edesc, + DMA_TO_DEVICE, &desc->ptr[3]); /* fifth DWORD empty */ desc->ptr[4] = zero_entry; @@ -1621,15 +1744,18 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, if (req_ctx->last) map_single_talitos_ptr(dev, &desc->ptr[5], crypto_ahash_digestsize(tfm), - areq->result, 0, DMA_FROM_DEVICE); + areq->result, DMA_FROM_DEVICE); else map_single_talitos_ptr(dev, &desc->ptr[5], req_ctx->hw_context_size, - req_ctx->hw_context, 0, DMA_FROM_DEVICE); + req_ctx->hw_context, DMA_FROM_DEVICE); /* last DWORD empty */ desc->ptr[6] = zero_entry; + if (is_sec1 && from_talitos_ptr_len(&desc->ptr[3], true) == 0) + talitos_handle_buggy_hash(ctx, edesc, &desc->ptr[3]); + ret = talitos_submit(dev, ctx->ch, desc, callback, areq); if (ret != -EINPROGRESS) { common_nonsnoop_hash_unmap(dev, edesc, areq); @@ -1645,7 +1771,7 @@ static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq, struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); - return talitos_edesc_alloc(ctx->dev, NULL, req_ctx->psrc, NULL, NULL, 0, + return talitos_edesc_alloc(ctx->dev, req_ctx->psrc, NULL, NULL, 0, nbytes, 0, 0, 0, areq->base.flags, false); } @@ -1704,12 +1830,11 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) unsigned int nbytes_to_hash; unsigned int to_hash_later; unsigned int nsg; - bool chained; if (!req_ctx->last && (nbytes + req_ctx->nbuf <= blocksize)) { /* Buffer up to one whole block */ sg_copy_to_buffer(areq->src, - sg_count(areq->src, nbytes, &chained), + sg_nents_for_len(areq->src, nbytes), req_ctx->buf + req_ctx->nbuf, nbytes); req_ctx->nbuf += nbytes; return 0; @@ -1736,13 +1861,13 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) sg_init_table(req_ctx->bufsl, nsg); sg_set_buf(req_ctx->bufsl, req_ctx->buf, req_ctx->nbuf); if (nsg > 1) - scatterwalk_sg_chain(req_ctx->bufsl, 2, areq->src); + sg_chain(req_ctx->bufsl, 2, areq->src); req_ctx->psrc = req_ctx->bufsl; } else req_ctx->psrc = areq->src; if (to_hash_later) { - int nents = sg_count(areq->src, nbytes, &chained); + int nents = sg_nents_for_len(areq->src, nbytes); sg_pcopy_to_buffer(areq->src, nents, req_ctx->bufnext, to_hash_later, @@ -1911,6 +2036,7 @@ struct talitos_alg_template { union { struct crypto_alg crypto; struct ahash_alg hash; + struct aead_alg aead; } alg; __be32 desc_hdr_template; }; @@ -1918,15 +2044,16 @@ struct talitos_alg_template { static struct talitos_alg_template driver_algs[] = { /* AEAD algorithms. These use a single-pass ipsec_esp descriptor */ { .type = CRYPTO_ALG_TYPE_AEAD, - .alg.crypto = { - .cra_name = "authenc(hmac(sha1),cbc(aes))", - .cra_driver_name = "authenc-hmac-sha1-cbc-aes-talitos", - .cra_blocksize = AES_BLOCK_SIZE, - .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, - .cra_aead = { - .ivsize = AES_BLOCK_SIZE, - .maxauthsize = SHA1_DIGEST_SIZE, - } + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha1),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha1-" + "cbc-aes-talitos", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC, + }, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, }, .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | DESC_HDR_SEL0_AESU | @@ -1937,15 +2064,17 @@ static struct talitos_alg_template driver_algs[] = { DESC_HDR_MODE1_MDEU_SHA1_HMAC, }, { .type = CRYPTO_ALG_TYPE_AEAD, - .alg.crypto = { - .cra_name = "authenc(hmac(sha1),cbc(des3_ede))", - .cra_driver_name = "authenc-hmac-sha1-cbc-3des-talitos", - .cra_blocksize = DES3_EDE_BLOCK_SIZE, - .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, - .cra_aead = { - .ivsize = DES3_EDE_BLOCK_SIZE, - .maxauthsize = SHA1_DIGEST_SIZE, - } + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha1)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha1-" + "cbc-3des-talitos", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC, + }, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, }, .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | DESC_HDR_SEL0_DEU | @@ -1957,15 +2086,16 @@ static struct talitos_alg_template driver_algs[] = { DESC_HDR_MODE1_MDEU_SHA1_HMAC, }, { .type = CRYPTO_ALG_TYPE_AEAD, - .alg.crypto = { - .cra_name = "authenc(hmac(sha224),cbc(aes))", - .cra_driver_name = "authenc-hmac-sha224-cbc-aes-talitos", - .cra_blocksize = AES_BLOCK_SIZE, - .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, - .cra_aead = { - .ivsize = AES_BLOCK_SIZE, - .maxauthsize = SHA224_DIGEST_SIZE, - } + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha224),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha224-" + "cbc-aes-talitos", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC, + }, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, }, .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | DESC_HDR_SEL0_AESU | @@ -1976,15 +2106,17 @@ static struct talitos_alg_template driver_algs[] = { DESC_HDR_MODE1_MDEU_SHA224_HMAC, }, { .type = CRYPTO_ALG_TYPE_AEAD, - .alg.crypto = { - .cra_name = "authenc(hmac(sha224),cbc(des3_ede))", - .cra_driver_name = "authenc-hmac-sha224-cbc-3des-talitos", - .cra_blocksize = DES3_EDE_BLOCK_SIZE, - .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, - .cra_aead = { - .ivsize = DES3_EDE_BLOCK_SIZE, - .maxauthsize = SHA224_DIGEST_SIZE, - } + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha224)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha224-" + "cbc-3des-talitos", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC, + }, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, }, .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | DESC_HDR_SEL0_DEU | @@ -1996,15 +2128,16 @@ static struct talitos_alg_template driver_algs[] = { DESC_HDR_MODE1_MDEU_SHA224_HMAC, }, { .type = CRYPTO_ALG_TYPE_AEAD, - .alg.crypto = { - .cra_name = "authenc(hmac(sha256),cbc(aes))", - .cra_driver_name = "authenc-hmac-sha256-cbc-aes-talitos", - .cra_blocksize = AES_BLOCK_SIZE, - .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, - .cra_aead = { - .ivsize = AES_BLOCK_SIZE, - .maxauthsize = SHA256_DIGEST_SIZE, - } + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha256),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha256-" + "cbc-aes-talitos", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC, + }, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, }, .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | DESC_HDR_SEL0_AESU | @@ -2015,15 +2148,17 @@ static struct talitos_alg_template driver_algs[] = { DESC_HDR_MODE1_MDEU_SHA256_HMAC, }, { .type = CRYPTO_ALG_TYPE_AEAD, - .alg.crypto = { - .cra_name = "authenc(hmac(sha256),cbc(des3_ede))", - .cra_driver_name = "authenc-hmac-sha256-cbc-3des-talitos", - .cra_blocksize = DES3_EDE_BLOCK_SIZE, - .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, - .cra_aead = { - .ivsize = DES3_EDE_BLOCK_SIZE, - .maxauthsize = SHA256_DIGEST_SIZE, - } + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha256)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha256-" + "cbc-3des-talitos", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC, + }, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, }, .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | DESC_HDR_SEL0_DEU | @@ -2035,15 +2170,16 @@ static struct talitos_alg_template driver_algs[] = { DESC_HDR_MODE1_MDEU_SHA256_HMAC, }, { .type = CRYPTO_ALG_TYPE_AEAD, - .alg.crypto = { - .cra_name = "authenc(hmac(sha384),cbc(aes))", - .cra_driver_name = "authenc-hmac-sha384-cbc-aes-talitos", - .cra_blocksize = AES_BLOCK_SIZE, - .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, - .cra_aead = { - .ivsize = AES_BLOCK_SIZE, - .maxauthsize = SHA384_DIGEST_SIZE, - } + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha384),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha384-" + "cbc-aes-talitos", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC, + }, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, }, .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | DESC_HDR_SEL0_AESU | @@ -2054,15 +2190,17 @@ static struct talitos_alg_template driver_algs[] = { DESC_HDR_MODE1_MDEUB_SHA384_HMAC, }, { .type = CRYPTO_ALG_TYPE_AEAD, - .alg.crypto = { - .cra_name = "authenc(hmac(sha384),cbc(des3_ede))", - .cra_driver_name = "authenc-hmac-sha384-cbc-3des-talitos", - .cra_blocksize = DES3_EDE_BLOCK_SIZE, - .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, - .cra_aead = { - .ivsize = DES3_EDE_BLOCK_SIZE, - .maxauthsize = SHA384_DIGEST_SIZE, - } + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha384)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha384-" + "cbc-3des-talitos", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC, + }, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, }, .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | DESC_HDR_SEL0_DEU | @@ -2074,15 +2212,16 @@ static struct talitos_alg_template driver_algs[] = { DESC_HDR_MODE1_MDEUB_SHA384_HMAC, }, { .type = CRYPTO_ALG_TYPE_AEAD, - .alg.crypto = { - .cra_name = "authenc(hmac(sha512),cbc(aes))", - .cra_driver_name = "authenc-hmac-sha512-cbc-aes-talitos", - .cra_blocksize = AES_BLOCK_SIZE, - .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, - .cra_aead = { - .ivsize = AES_BLOCK_SIZE, - .maxauthsize = SHA512_DIGEST_SIZE, - } + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha512),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha512-" + "cbc-aes-talitos", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC, + }, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, }, .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | DESC_HDR_SEL0_AESU | @@ -2093,15 +2232,17 @@ static struct talitos_alg_template driver_algs[] = { DESC_HDR_MODE1_MDEUB_SHA512_HMAC, }, { .type = CRYPTO_ALG_TYPE_AEAD, - .alg.crypto = { - .cra_name = "authenc(hmac(sha512),cbc(des3_ede))", - .cra_driver_name = "authenc-hmac-sha512-cbc-3des-talitos", - .cra_blocksize = DES3_EDE_BLOCK_SIZE, - .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, - .cra_aead = { - .ivsize = DES3_EDE_BLOCK_SIZE, - .maxauthsize = SHA512_DIGEST_SIZE, - } + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(sha512)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha512-" + "cbc-3des-talitos", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC, + }, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, }, .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | DESC_HDR_SEL0_DEU | @@ -2113,15 +2254,16 @@ static struct talitos_alg_template driver_algs[] = { DESC_HDR_MODE1_MDEUB_SHA512_HMAC, }, { .type = CRYPTO_ALG_TYPE_AEAD, - .alg.crypto = { - .cra_name = "authenc(hmac(md5),cbc(aes))", - .cra_driver_name = "authenc-hmac-md5-cbc-aes-talitos", - .cra_blocksize = AES_BLOCK_SIZE, - .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, - .cra_aead = { - .ivsize = AES_BLOCK_SIZE, - .maxauthsize = MD5_DIGEST_SIZE, - } + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(md5),cbc(aes))", + .cra_driver_name = "authenc-hmac-md5-" + "cbc-aes-talitos", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC, + }, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, }, .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | DESC_HDR_SEL0_AESU | @@ -2132,15 +2274,16 @@ static struct talitos_alg_template driver_algs[] = { DESC_HDR_MODE1_MDEU_MD5_HMAC, }, { .type = CRYPTO_ALG_TYPE_AEAD, - .alg.crypto = { - .cra_name = "authenc(hmac(md5),cbc(des3_ede))", - .cra_driver_name = "authenc-hmac-md5-cbc-3des-talitos", - .cra_blocksize = DES3_EDE_BLOCK_SIZE, - .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, - .cra_aead = { - .ivsize = DES3_EDE_BLOCK_SIZE, - .maxauthsize = MD5_DIGEST_SIZE, - } + .alg.aead = { + .base = { + .cra_name = "authenc(hmac(md5),cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-md5-" + "cbc-3des-talitos", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC, + }, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, }, .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | DESC_HDR_SEL0_DEU | @@ -2408,15 +2551,9 @@ static int talitos_cra_init(struct crypto_tfm *tfm) return 0; } -static int talitos_cra_init_aead(struct crypto_tfm *tfm) +static int talitos_cra_init_aead(struct crypto_aead *tfm) { - struct talitos_ctx *ctx = crypto_tfm_ctx(tfm); - - talitos_cra_init(tfm); - - /* random first IV */ - get_random_bytes(ctx->iv, TALITOS_MAX_IV_LENGTH); - + talitos_cra_init(crypto_aead_tfm(tfm)); return 0; } @@ -2463,9 +2600,9 @@ static int talitos_remove(struct platform_device *ofdev) list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) { switch (t_alg->algt.type) { case CRYPTO_ALG_TYPE_ABLKCIPHER: - case CRYPTO_ALG_TYPE_AEAD: - crypto_unregister_alg(&t_alg->algt.alg.crypto); break; + case CRYPTO_ALG_TYPE_AEAD: + crypto_unregister_aead(&t_alg->algt.alg.aead); case CRYPTO_ALG_TYPE_AHASH: crypto_unregister_ahash(&t_alg->algt.alg.hash); break; @@ -2477,7 +2614,7 @@ static int talitos_remove(struct platform_device *ofdev) if (hw_supports(dev, DESC_HDR_SEL0_RNG)) talitos_unregister_rng(dev); - for (i = 0; i < priv->num_channels; i++) + for (i = 0; priv->chan && i < priv->num_channels; i++) kfree(priv->chan[i].fifo); kfree(priv->chan); @@ -2524,15 +2661,11 @@ static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev, alg->cra_ablkcipher.geniv = "eseqiv"; break; case CRYPTO_ALG_TYPE_AEAD: - alg = &t_alg->algt.alg.crypto; - alg->cra_init = talitos_cra_init_aead; - alg->cra_type = &crypto_aead_type; - alg->cra_aead.setkey = aead_setkey; - alg->cra_aead.setauthsize = aead_setauthsize; - alg->cra_aead.encrypt = aead_encrypt; - alg->cra_aead.decrypt = aead_decrypt; - alg->cra_aead.givencrypt = aead_givencrypt; - alg->cra_aead.geniv = ""; + alg = &t_alg->algt.alg.aead.base; + t_alg->algt.alg.aead.init = talitos_cra_init_aead; + t_alg->algt.alg.aead.setkey = aead_setkey; + t_alg->algt.alg.aead.encrypt = aead_encrypt; + t_alg->algt.alg.aead.decrypt = aead_decrypt; break; case CRYPTO_ALG_TYPE_AHASH: alg = &t_alg->algt.alg.hash.halg.base; @@ -2583,29 +2716,35 @@ static int talitos_probe_irq(struct platform_device *ofdev) struct device_node *np = ofdev->dev.of_node; struct talitos_private *priv = dev_get_drvdata(dev); int err; + bool is_sec1 = has_ftr_sec1(priv); priv->irq[0] = irq_of_parse_and_map(np, 0); if (!priv->irq[0]) { dev_err(dev, "failed to map irq\n"); return -EINVAL; } + if (is_sec1) { + err = request_irq(priv->irq[0], talitos1_interrupt_4ch, 0, + dev_driver_string(dev), dev); + goto primary_out; + } priv->irq[1] = irq_of_parse_and_map(np, 1); /* get the primary irq line */ if (!priv->irq[1]) { - err = request_irq(priv->irq[0], talitos_interrupt_4ch, 0, + err = request_irq(priv->irq[0], talitos2_interrupt_4ch, 0, dev_driver_string(dev), dev); goto primary_out; } - err = request_irq(priv->irq[0], talitos_interrupt_ch0_2, 0, + err = request_irq(priv->irq[0], talitos2_interrupt_ch0_2, 0, dev_driver_string(dev), dev); if (err) goto primary_out; /* get the secondary irq line */ - err = request_irq(priv->irq[1], talitos_interrupt_ch1_3, 0, + err = request_irq(priv->irq[1], talitos2_interrupt_ch1_3, 0, dev_driver_string(dev), dev); if (err) { dev_err(dev, "failed to request secondary irq\n"); @@ -2632,6 +2771,7 @@ static int talitos_probe(struct platform_device *ofdev) struct talitos_private *priv; const unsigned int *prop; int i, err; + int stride; priv = kzalloc(sizeof(struct talitos_private), GFP_KERNEL); if (!priv) @@ -2645,20 +2785,6 @@ static int talitos_probe(struct platform_device *ofdev) spin_lock_init(&priv->reg_lock); - err = talitos_probe_irq(ofdev); - if (err) - goto err_out; - - if (!priv->irq[1]) { - tasklet_init(&priv->done_task[0], talitos_done_4ch, - (unsigned long)dev); - } else { - tasklet_init(&priv->done_task[0], talitos_done_ch0_2, - (unsigned long)dev); - tasklet_init(&priv->done_task[1], talitos_done_ch1_3, - (unsigned long)dev); - } - priv->reg = of_iomap(np, 0); if (!priv->reg) { dev_err(dev, "failed to of_iomap\n"); @@ -2698,6 +2824,53 @@ static int talitos_probe(struct platform_device *ofdev) TALITOS_FTR_SHA224_HWINIT | TALITOS_FTR_HMAC_OK; + if (of_device_is_compatible(np, "fsl,sec1.0")) + priv->features |= TALITOS_FTR_SEC1; + + if (of_device_is_compatible(np, "fsl,sec1.2")) { + priv->reg_deu = priv->reg + TALITOS12_DEU; + priv->reg_aesu = priv->reg + TALITOS12_AESU; + priv->reg_mdeu = priv->reg + TALITOS12_MDEU; + stride = TALITOS1_CH_STRIDE; + } else if (of_device_is_compatible(np, "fsl,sec1.0")) { + priv->reg_deu = priv->reg + TALITOS10_DEU; + priv->reg_aesu = priv->reg + TALITOS10_AESU; + priv->reg_mdeu = priv->reg + TALITOS10_MDEU; + priv->reg_afeu = priv->reg + TALITOS10_AFEU; + priv->reg_rngu = priv->reg + TALITOS10_RNGU; + priv->reg_pkeu = priv->reg + TALITOS10_PKEU; + stride = TALITOS1_CH_STRIDE; + } else { + priv->reg_deu = priv->reg + TALITOS2_DEU; + priv->reg_aesu = priv->reg + TALITOS2_AESU; + priv->reg_mdeu = priv->reg + TALITOS2_MDEU; + priv->reg_afeu = priv->reg + TALITOS2_AFEU; + priv->reg_rngu = priv->reg + TALITOS2_RNGU; + priv->reg_pkeu = priv->reg + TALITOS2_PKEU; + priv->reg_keu = priv->reg + TALITOS2_KEU; + priv->reg_crcu = priv->reg + TALITOS2_CRCU; + stride = TALITOS2_CH_STRIDE; + } + + err = talitos_probe_irq(ofdev); + if (err) + goto err_out; + + if (of_device_is_compatible(np, "fsl,sec1.0")) { + tasklet_init(&priv->done_task[0], talitos1_done_4ch, + (unsigned long)dev); + } else { + if (!priv->irq[1]) { + tasklet_init(&priv->done_task[0], talitos2_done_4ch, + (unsigned long)dev); + } else { + tasklet_init(&priv->done_task[0], talitos2_done_ch0_2, + (unsigned long)dev); + tasklet_init(&priv->done_task[1], talitos2_done_ch1_3, + (unsigned long)dev); + } + } + priv->chan = kzalloc(sizeof(struct talitos_channel) * priv->num_channels, GFP_KERNEL); if (!priv->chan) { @@ -2709,7 +2882,7 @@ static int talitos_probe(struct platform_device *ofdev) priv->fifo_len = roundup_pow_of_two(priv->chfifo_len); for (i = 0; i < priv->num_channels; i++) { - priv->chan[i].reg = priv->reg + TALITOS_CH_STRIDE * (i + 1); + priv->chan[i].reg = priv->reg + stride * (i + 1); if (!priv->irq[1] || !(i & 1)) priv->chan[i].reg += TALITOS_CH_BASE_OFFSET; @@ -2751,7 +2924,7 @@ static int talitos_probe(struct platform_device *ofdev) for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { if (hw_supports(dev, driver_algs[i].desc_hdr_template)) { struct talitos_crypto_alg *t_alg; - char *name = NULL; + struct crypto_alg *alg = NULL; t_alg = talitos_alg_alloc(dev, &driver_algs[i]); if (IS_ERR(t_alg)) { @@ -2763,21 +2936,26 @@ static int talitos_probe(struct platform_device *ofdev) switch (t_alg->algt.type) { case CRYPTO_ALG_TYPE_ABLKCIPHER: - case CRYPTO_ALG_TYPE_AEAD: err = crypto_register_alg( &t_alg->algt.alg.crypto); - name = t_alg->algt.alg.crypto.cra_driver_name; + alg = &t_alg->algt.alg.crypto; break; + + case CRYPTO_ALG_TYPE_AEAD: + err = crypto_register_aead( + &t_alg->algt.alg.aead); + alg = &t_alg->algt.alg.aead.base; + break; + case CRYPTO_ALG_TYPE_AHASH: err = crypto_register_ahash( &t_alg->algt.alg.hash); - name = - t_alg->algt.alg.hash.halg.base.cra_driver_name; + alg = &t_alg->algt.alg.hash.halg.base; break; } if (err) { dev_err(dev, "%s alg registration failed\n", - name); + alg->cra_driver_name); kfree(t_alg); } else list_add_tail(&t_alg->entry, &priv->alg_list); @@ -2796,9 +2974,16 @@ err_out: } static const struct of_device_id talitos_match[] = { +#ifdef CONFIG_CRYPTO_DEV_TALITOS1 + { + .compatible = "fsl,sec1.0", + }, +#endif +#ifdef CONFIG_CRYPTO_DEV_TALITOS2 { .compatible = "fsl,sec2.0", }, +#endif {}, }; MODULE_DEVICE_TABLE(of, talitos_match); diff --git a/kernel/drivers/crypto/talitos.h b/kernel/drivers/crypto/talitos.h index 61a14054a..0090f3211 100644 --- a/kernel/drivers/crypto/talitos.h +++ b/kernel/drivers/crypto/talitos.h @@ -29,7 +29,8 @@ */ #define TALITOS_TIMEOUT 100000 -#define TALITOS_MAX_DATA_LEN 65535 +#define TALITOS1_MAX_DATA_LEN 32768 +#define TALITOS2_MAX_DATA_LEN 65535 #define DESC_TYPE(desc_hdr) ((be32_to_cpu(desc_hdr) >> 3) & 0x1f) #define PRIMARY_EU(desc_hdr) ((be32_to_cpu(desc_hdr) >> 28) & 0xf) @@ -37,26 +38,35 @@ /* descriptor pointer entry */ struct talitos_ptr { - __be16 len; /* length */ - u8 j_extent; /* jump to sg link table and/or extent */ - u8 eptr; /* extended address */ + union { + struct { /* SEC2 format */ + __be16 len; /* length */ + u8 j_extent; /* jump to sg link table and/or extent*/ + u8 eptr; /* extended address */ + }; + struct { /* SEC1 format */ + __be16 res; + __be16 len1; /* length */ + }; + }; __be32 ptr; /* address */ }; -static const struct talitos_ptr zero_entry = { - .len = 0, - .j_extent = 0, - .eptr = 0, - .ptr = 0 -}; +static const struct talitos_ptr zero_entry; /* descriptor */ struct talitos_desc { __be32 hdr; /* header high bits */ - __be32 hdr_lo; /* header low bits */ + union { + __be32 hdr_lo; /* header low bits */ + __be32 hdr1; /* header for SEC1 */ + }; struct talitos_ptr ptr[7]; /* ptr/len pair array */ + __be32 next_desc; /* next descriptor (SEC1) */ }; +#define TALITOS_DESC_SIZE (sizeof(struct talitos_desc) - sizeof(__be32)) + /** * talitos_request - descriptor submission request * @desc: descriptor pointer (kernel virtual) @@ -97,6 +107,14 @@ struct talitos_private { struct device *dev; struct platform_device *ofdev; void __iomem *reg; + void __iomem *reg_deu; + void __iomem *reg_aesu; + void __iomem *reg_mdeu; + void __iomem *reg_afeu; + void __iomem *reg_rngu; + void __iomem *reg_pkeu; + void __iomem *reg_keu; + void __iomem *reg_crcu; int irq[2]; /* SEC global registers lock */ @@ -131,6 +149,7 @@ struct talitos_private { /* hwrng device */ struct hwrng rng; + bool rng_registered; }; extern int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc, @@ -144,49 +163,80 @@ extern int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc, #define TALITOS_FTR_HW_AUTH_CHECK 0x00000002 #define TALITOS_FTR_SHA224_HWINIT 0x00000004 #define TALITOS_FTR_HMAC_OK 0x00000008 +#define TALITOS_FTR_SEC1 0x00000010 + +/* + * If both CONFIG_CRYPTO_DEV_TALITOS1 and CONFIG_CRYPTO_DEV_TALITOS2 are + * defined, we check the features which are set according to the device tree. + * Otherwise, we answer true or false directly + */ +static inline bool has_ftr_sec1(struct talitos_private *priv) +{ +#if defined(CONFIG_CRYPTO_DEV_TALITOS1) && defined(CONFIG_CRYPTO_DEV_TALITOS2) + return priv->features & TALITOS_FTR_SEC1 ? true : false; +#elif defined(CONFIG_CRYPTO_DEV_TALITOS1) + return true; +#else + return false; +#endif +} /* * TALITOS_xxx_LO addresses point to the low data bits (32-63) of the register */ +#define ISR1_FORMAT(x) (((x) << 28) | ((x) << 16)) +#define ISR2_FORMAT(x) (((x) << 4) | (x)) + /* global register offset addresses */ #define TALITOS_MCR 0x1030 /* master control register */ #define TALITOS_MCR_RCA0 (1 << 15) /* remap channel 0 */ #define TALITOS_MCR_RCA1 (1 << 14) /* remap channel 1 */ #define TALITOS_MCR_RCA2 (1 << 13) /* remap channel 2 */ #define TALITOS_MCR_RCA3 (1 << 12) /* remap channel 3 */ -#define TALITOS_MCR_SWR 0x1 /* s/w reset */ +#define TALITOS1_MCR_SWR 0x1000000 /* s/w reset */ +#define TALITOS2_MCR_SWR 0x1 /* s/w reset */ #define TALITOS_MCR_LO 0x1034 #define TALITOS_IMR 0x1008 /* interrupt mask register */ -#define TALITOS_IMR_INIT 0x100ff /* enable channel IRQs */ -#define TALITOS_IMR_DONE 0x00055 /* done IRQs */ +/* enable channel IRQs */ +#define TALITOS1_IMR_INIT ISR1_FORMAT(0xf) +#define TALITOS1_IMR_DONE ISR1_FORMAT(0x5) /* done IRQs */ +/* enable channel IRQs */ +#define TALITOS2_IMR_INIT (ISR2_FORMAT(0xf) | 0x10000) +#define TALITOS2_IMR_DONE ISR1_FORMAT(0x5) /* done IRQs */ #define TALITOS_IMR_LO 0x100C -#define TALITOS_IMR_LO_INIT 0x20000 /* allow RNGU error IRQs */ +#define TALITOS1_IMR_LO_INIT 0x2000000 /* allow RNGU error IRQs */ +#define TALITOS2_IMR_LO_INIT 0x20000 /* allow RNGU error IRQs */ #define TALITOS_ISR 0x1010 /* interrupt status register */ -#define TALITOS_ISR_4CHERR 0xaa /* 4 channel errors mask */ -#define TALITOS_ISR_4CHDONE 0x55 /* 4 channel done mask */ -#define TALITOS_ISR_CH_0_2_ERR 0x22 /* channels 0, 2 errors mask */ -#define TALITOS_ISR_CH_0_2_DONE 0x11 /* channels 0, 2 done mask */ -#define TALITOS_ISR_CH_1_3_ERR 0x88 /* channels 1, 3 errors mask */ -#define TALITOS_ISR_CH_1_3_DONE 0x44 /* channels 1, 3 done mask */ +#define TALITOS1_ISR_4CHERR ISR1_FORMAT(0xa) /* 4 ch errors mask */ +#define TALITOS1_ISR_4CHDONE ISR1_FORMAT(0x5) /* 4 ch done mask */ +#define TALITOS1_ISR_TEA_ERR 0x00000040 +#define TALITOS2_ISR_4CHERR ISR2_FORMAT(0xa) /* 4 ch errors mask */ +#define TALITOS2_ISR_4CHDONE ISR2_FORMAT(0x5) /* 4 ch done mask */ +#define TALITOS2_ISR_CH_0_2_ERR ISR2_FORMAT(0x2) /* ch 0, 2 err mask */ +#define TALITOS2_ISR_CH_0_2_DONE ISR2_FORMAT(0x1) /* ch 0, 2 done mask */ +#define TALITOS2_ISR_CH_1_3_ERR ISR2_FORMAT(0x8) /* ch 1, 3 err mask */ +#define TALITOS2_ISR_CH_1_3_DONE ISR2_FORMAT(0x4) /* ch 1, 3 done mask */ #define TALITOS_ISR_LO 0x1014 #define TALITOS_ICR 0x1018 /* interrupt clear register */ #define TALITOS_ICR_LO 0x101C /* channel register address stride */ #define TALITOS_CH_BASE_OFFSET 0x1000 /* default channel map base */ -#define TALITOS_CH_STRIDE 0x100 +#define TALITOS1_CH_STRIDE 0x1000 +#define TALITOS2_CH_STRIDE 0x100 /* channel configuration register */ #define TALITOS_CCCR 0x8 -#define TALITOS_CCCR_CONT 0x2 /* channel continue */ -#define TALITOS_CCCR_RESET 0x1 /* channel reset */ +#define TALITOS2_CCCR_CONT 0x2 /* channel continue on SEC2 */ +#define TALITOS2_CCCR_RESET 0x1 /* channel reset on SEC2 */ #define TALITOS_CCCR_LO 0xc #define TALITOS_CCCR_LO_IWSE 0x80 /* chan. ICCR writeback enab. */ #define TALITOS_CCCR_LO_EAE 0x20 /* extended address enable */ #define TALITOS_CCCR_LO_CDWE 0x10 /* chan. done writeback enab. */ #define TALITOS_CCCR_LO_NT 0x4 /* notification type */ #define TALITOS_CCCR_LO_CDIE 0x2 /* channel done IRQ enable */ +#define TALITOS1_CCCR_LO_RESET 0x1 /* channel reset on SEC1 */ /* CCPSR: channel pointer status register */ #define TALITOS_CCPSR 0x10 @@ -224,37 +274,48 @@ extern int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc, #define TALITOS_SCATTER 0xe0 #define TALITOS_SCATTER_LO 0xe4 +/* execution unit registers base */ +#define TALITOS2_DEU 0x2000 +#define TALITOS2_AESU 0x4000 +#define TALITOS2_MDEU 0x6000 +#define TALITOS2_AFEU 0x8000 +#define TALITOS2_RNGU 0xa000 +#define TALITOS2_PKEU 0xc000 +#define TALITOS2_KEU 0xe000 +#define TALITOS2_CRCU 0xf000 + +#define TALITOS12_AESU 0x4000 +#define TALITOS12_DEU 0x5000 +#define TALITOS12_MDEU 0x6000 + +#define TALITOS10_AFEU 0x8000 +#define TALITOS10_DEU 0xa000 +#define TALITOS10_MDEU 0xc000 +#define TALITOS10_RNGU 0xe000 +#define TALITOS10_PKEU 0x10000 +#define TALITOS10_AESU 0x12000 + /* execution unit interrupt status registers */ -#define TALITOS_DEUISR 0x2030 /* DES unit */ -#define TALITOS_DEUISR_LO 0x2034 -#define TALITOS_AESUISR 0x4030 /* AES unit */ -#define TALITOS_AESUISR_LO 0x4034 -#define TALITOS_MDEUISR 0x6030 /* message digest unit */ -#define TALITOS_MDEUISR_LO 0x6034 -#define TALITOS_MDEUICR 0x6038 /* interrupt control */ -#define TALITOS_MDEUICR_LO 0x603c +#define TALITOS_EUDSR 0x10 /* data size */ +#define TALITOS_EUDSR_LO 0x14 +#define TALITOS_EURCR 0x18 /* reset control*/ +#define TALITOS_EURCR_LO 0x1c +#define TALITOS_EUSR 0x28 /* rng status */ +#define TALITOS_EUSR_LO 0x2c +#define TALITOS_EUISR 0x30 +#define TALITOS_EUISR_LO 0x34 +#define TALITOS_EUICR 0x38 /* int. control */ +#define TALITOS_EUICR_LO 0x3c +#define TALITOS_EU_FIFO 0x800 /* output FIFO */ +#define TALITOS_EU_FIFO_LO 0x804 /* output FIFO */ +/* DES unit */ +#define TALITOS1_DEUICR_KPE 0x00200000 /* Key Parity Error */ +/* message digest unit */ #define TALITOS_MDEUICR_LO_ICE 0x4000 /* integrity check IRQ enable */ -#define TALITOS_AFEUISR 0x8030 /* arc4 unit */ -#define TALITOS_AFEUISR_LO 0x8034 -#define TALITOS_RNGUISR 0xa030 /* random number unit */ -#define TALITOS_RNGUISR_LO 0xa034 -#define TALITOS_RNGUSR 0xa028 /* rng status */ -#define TALITOS_RNGUSR_LO 0xa02c +/* random number unit */ #define TALITOS_RNGUSR_LO_RD 0x1 /* reset done */ #define TALITOS_RNGUSR_LO_OFL 0xff0000/* output FIFO length */ -#define TALITOS_RNGUDSR 0xa010 /* data size */ -#define TALITOS_RNGUDSR_LO 0xa014 -#define TALITOS_RNGU_FIFO 0xa800 /* output FIFO */ -#define TALITOS_RNGU_FIFO_LO 0xa804 /* output FIFO */ -#define TALITOS_RNGURCR 0xa018 /* reset control */ -#define TALITOS_RNGURCR_LO 0xa01c #define TALITOS_RNGURCR_LO_SR 0x1 /* software reset */ -#define TALITOS_PKEUISR 0xc030 /* public key unit */ -#define TALITOS_PKEUISR_LO 0xc034 -#define TALITOS_KEUISR 0xe030 /* kasumi unit */ -#define TALITOS_KEUISR_LO 0xe034 -#define TALITOS_CRCUISR 0xf030 /* cyclic redundancy check unit*/ -#define TALITOS_CRCUISR_LO 0xf034 #define TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256 0x28 #define TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512 0x48 diff --git a/kernel/drivers/crypto/ux500/Kconfig b/kernel/drivers/crypto/ux500/Kconfig index b35e5c4b0..30796441b 100644 --- a/kernel/drivers/crypto/ux500/Kconfig +++ b/kernel/drivers/crypto/ux500/Kconfig @@ -7,6 +7,8 @@ config CRYPTO_DEV_UX500_CRYP tristate "UX500 crypto driver for CRYP block" depends on CRYPTO_DEV_UX500 + select CRYPTO_ALGAPI + select CRYPTO_BLKCIPHER select CRYPTO_DES help This selects the crypto driver for the UX500_CRYP hardware. It supports @@ -16,7 +18,6 @@ config CRYPTO_DEV_UX500_HASH tristate "UX500 crypto driver for HASH block" depends on CRYPTO_DEV_UX500 select CRYPTO_HASH - select CRYPTO_HMAC help This selects the hash driver for the UX500_HASH hardware. Depends on UX500/STM DMA if running in DMA mode. @@ -24,7 +25,6 @@ config CRYPTO_DEV_UX500_HASH config CRYPTO_DEV_UX500_DEBUG bool "Activate ux500 platform debug-mode for crypto and hash block" depends on CRYPTO_DEV_UX500_CRYP || CRYPTO_DEV_UX500_HASH - default n help Say Y if you want to add debug prints to ux500_hash and ux500_cryp devices. diff --git a/kernel/drivers/crypto/ux500/cryp/cryp_core.c b/kernel/drivers/crypto/ux500/cryp/cryp_core.c index fded0a5cf..4c243c1ff 100644 --- a/kernel/drivers/crypto/ux500/cryp/cryp_core.c +++ b/kernel/drivers/crypto/ux500/cryp/cryp_core.c @@ -1414,7 +1414,7 @@ static int ux500_cryp_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; dev_dbg(dev, "[%s]", __func__); - device_data = kzalloc(sizeof(struct cryp_device_data), GFP_ATOMIC); + device_data = devm_kzalloc(dev, sizeof(*device_data), GFP_ATOMIC); if (!device_data) { dev_err(dev, "[%s]: kzalloc() failed!", __func__); ret = -ENOMEM; @@ -1435,23 +1435,15 @@ static int ux500_cryp_probe(struct platform_device *pdev) dev_err(dev, "[%s]: platform_get_resource() failed", __func__); ret = -ENODEV; - goto out_kfree; - } - - res = request_mem_region(res->start, resource_size(res), pdev->name); - if (res == NULL) { - dev_err(dev, "[%s]: request_mem_region() failed", - __func__); - ret = -EBUSY; - goto out_kfree; + goto out; } device_data->phybase = res->start; - device_data->base = ioremap(res->start, resource_size(res)); + device_data->base = devm_ioremap_resource(dev, res); if (!device_data->base) { dev_err(dev, "[%s]: ioremap failed!", __func__); ret = -ENOMEM; - goto out_free_mem; + goto out; } spin_lock_init(&device_data->ctx_lock); @@ -1463,11 +1455,11 @@ static int ux500_cryp_probe(struct platform_device *pdev) dev_err(dev, "[%s]: could not get cryp regulator", __func__); ret = PTR_ERR(device_data->pwr_regulator); device_data->pwr_regulator = NULL; - goto out_unmap; + goto out; } /* Enable the clk for CRYP hardware block */ - device_data->clk = clk_get(&pdev->dev, NULL); + device_data->clk = devm_clk_get(&pdev->dev, NULL); if (IS_ERR(device_data->clk)) { dev_err(dev, "[%s]: clk_get() failed!", __func__); ret = PTR_ERR(device_data->clk); @@ -1477,7 +1469,7 @@ static int ux500_cryp_probe(struct platform_device *pdev) ret = clk_prepare(device_data->clk); if (ret) { dev_err(dev, "[%s]: clk_prepare() failed!", __func__); - goto out_clk; + goto out_regulator; } /* Enable device power (and clock) */ @@ -1510,11 +1502,8 @@ static int ux500_cryp_probe(struct platform_device *pdev) goto out_power; } - ret = request_irq(res_irq->start, - cryp_interrupt_handler, - 0, - "cryp1", - device_data); + ret = devm_request_irq(&pdev->dev, res_irq->start, + cryp_interrupt_handler, 0, "cryp1", device_data); if (ret) { dev_err(dev, "[%s]: Unable to request IRQ", __func__); goto out_power; @@ -1550,28 +1539,15 @@ out_power: out_clk_unprepare: clk_unprepare(device_data->clk); -out_clk: - clk_put(device_data->clk); - out_regulator: regulator_put(device_data->pwr_regulator); -out_unmap: - iounmap(device_data->base); - -out_free_mem: - release_mem_region(res->start, resource_size(res)); - -out_kfree: - kfree(device_data); out: return ret; } static int ux500_cryp_remove(struct platform_device *pdev) { - struct resource *res = NULL; - struct resource *res_irq = NULL; struct cryp_device_data *device_data; dev_dbg(&pdev->dev, "[%s]", __func__); @@ -1607,37 +1583,18 @@ static int ux500_cryp_remove(struct platform_device *pdev) if (list_empty(&driver_data.device_list.k_list)) cryp_algs_unregister_all(); - res_irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0); - if (!res_irq) - dev_err(&pdev->dev, "[%s]: IORESOURCE_IRQ, unavailable", - __func__); - else { - disable_irq(res_irq->start); - free_irq(res_irq->start, device_data); - } - if (cryp_disable_power(&pdev->dev, device_data, false)) dev_err(&pdev->dev, "[%s]: cryp_disable_power() failed", __func__); clk_unprepare(device_data->clk); - clk_put(device_data->clk); regulator_put(device_data->pwr_regulator); - iounmap(device_data->base); - - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (res) - release_mem_region(res->start, resource_size(res)); - - kfree(device_data); - return 0; } static void ux500_cryp_shutdown(struct platform_device *pdev) { - struct resource *res_irq = NULL; struct cryp_device_data *device_data; dev_dbg(&pdev->dev, "[%s]", __func__); @@ -1673,15 +1630,6 @@ static void ux500_cryp_shutdown(struct platform_device *pdev) if (list_empty(&driver_data.device_list.k_list)) cryp_algs_unregister_all(); - res_irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0); - if (!res_irq) - dev_err(&pdev->dev, "[%s]: IORESOURCE_IRQ, unavailable", - __func__); - else { - disable_irq(res_irq->start); - free_irq(res_irq->start, device_data); - } - if (cryp_disable_power(&pdev->dev, device_data, false)) dev_err(&pdev->dev, "[%s]: cryp_disable_power() failed", __func__); @@ -1777,6 +1725,7 @@ static const struct of_device_id ux500_cryp_match[] = { { .compatible = "stericsson,ux500-cryp" }, { }, }; +MODULE_DEVICE_TABLE(of, ux500_cryp_match); static struct platform_driver cryp_driver = { .probe = ux500_cryp_probe, diff --git a/kernel/drivers/crypto/ux500/hash/hash_core.c b/kernel/drivers/crypto/ux500/hash/hash_core.c index 5f5f36062..f47d11204 100644 --- a/kernel/drivers/crypto/ux500/hash/hash_core.c +++ b/kernel/drivers/crypto/ux500/hash/hash_core.c @@ -1657,7 +1657,7 @@ static int ux500_hash_probe(struct platform_device *pdev) struct hash_device_data *device_data; struct device *dev = &pdev->dev; - device_data = kzalloc(sizeof(*device_data), GFP_ATOMIC); + device_data = devm_kzalloc(dev, sizeof(*device_data), GFP_ATOMIC); if (!device_data) { ret = -ENOMEM; goto out; @@ -1670,22 +1670,15 @@ static int ux500_hash_probe(struct platform_device *pdev) if (!res) { dev_dbg(dev, "%s: platform_get_resource() failed!\n", __func__); ret = -ENODEV; - goto out_kfree; - } - - res = request_mem_region(res->start, resource_size(res), pdev->name); - if (res == NULL) { - dev_dbg(dev, "%s: request_mem_region() failed!\n", __func__); - ret = -EBUSY; - goto out_kfree; + goto out; } device_data->phybase = res->start; - device_data->base = ioremap(res->start, resource_size(res)); + device_data->base = devm_ioremap_resource(dev, res); if (!device_data->base) { dev_err(dev, "%s: ioremap() failed!\n", __func__); ret = -ENOMEM; - goto out_free_mem; + goto out; } spin_lock_init(&device_data->ctx_lock); spin_lock_init(&device_data->power_state_lock); @@ -1696,11 +1689,11 @@ static int ux500_hash_probe(struct platform_device *pdev) dev_err(dev, "%s: regulator_get() failed!\n", __func__); ret = PTR_ERR(device_data->regulator); device_data->regulator = NULL; - goto out_unmap; + goto out; } /* Enable the clock for HASH1 hardware block */ - device_data->clk = clk_get(dev, NULL); + device_data->clk = devm_clk_get(dev, NULL); if (IS_ERR(device_data->clk)) { dev_err(dev, "%s: clk_get() failed!\n", __func__); ret = PTR_ERR(device_data->clk); @@ -1710,7 +1703,7 @@ static int ux500_hash_probe(struct platform_device *pdev) ret = clk_prepare(device_data->clk); if (ret) { dev_err(dev, "%s: clk_prepare() failed!\n", __func__); - goto out_clk; + goto out_regulator; } /* Enable device power (and clock) */ @@ -1752,20 +1745,9 @@ out_power: out_clk_unprepare: clk_unprepare(device_data->clk); -out_clk: - clk_put(device_data->clk); - out_regulator: regulator_put(device_data->regulator); -out_unmap: - iounmap(device_data->base); - -out_free_mem: - release_mem_region(res->start, resource_size(res)); - -out_kfree: - kfree(device_data); out: return ret; } @@ -1776,7 +1758,6 @@ out: */ static int ux500_hash_remove(struct platform_device *pdev) { - struct resource *res; struct hash_device_data *device_data; struct device *dev = &pdev->dev; @@ -1816,17 +1797,8 @@ static int ux500_hash_remove(struct platform_device *pdev) __func__); clk_unprepare(device_data->clk); - clk_put(device_data->clk); regulator_put(device_data->regulator); - iounmap(device_data->base); - - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (res) - release_mem_region(res->start, resource_size(res)); - - kfree(device_data); - return 0; } @@ -1836,7 +1808,6 @@ static int ux500_hash_remove(struct platform_device *pdev) */ static void ux500_hash_shutdown(struct platform_device *pdev) { - struct resource *res = NULL; struct hash_device_data *device_data; device_data = platform_get_drvdata(pdev); @@ -1870,12 +1841,6 @@ static void ux500_hash_shutdown(struct platform_device *pdev) if (list_empty(&driver_data.device_list.k_list)) ahash_algs_unregister_all(device_data); - iounmap(device_data->base); - - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (res) - release_mem_region(res->start, resource_size(res)); - if (hash_disable_power(device_data, false)) dev_err(&pdev->dev, "%s: hash_disable_power() failed\n", __func__); @@ -1958,6 +1923,7 @@ static const struct of_device_id ux500_hash_match[] = { { .compatible = "stericsson,ux500-hash" }, { }, }; +MODULE_DEVICE_TABLE(of, ux500_hash_match); static struct platform_driver hash_driver = { .probe = ux500_hash_probe, diff --git a/kernel/drivers/crypto/vmx/Kconfig b/kernel/drivers/crypto/vmx/Kconfig index 771babf16..89d8208d9 100644 --- a/kernel/drivers/crypto/vmx/Kconfig +++ b/kernel/drivers/crypto/vmx/Kconfig @@ -1,6 +1,6 @@ config CRYPTO_DEV_VMX_ENCRYPT tristate "Encryption acceleration support on P8 CPU" - depends on PPC64 && CRYPTO_DEV_VMX + depends on CRYPTO_DEV_VMX default y help Support for VMX cryptographic acceleration instructions on Power8 CPU. diff --git a/kernel/drivers/crypto/vmx/Makefile b/kernel/drivers/crypto/vmx/Makefile index c699c6e6c..d28ab96a2 100644 --- a/kernel/drivers/crypto/vmx/Makefile +++ b/kernel/drivers/crypto/vmx/Makefile @@ -4,7 +4,7 @@ vmx-crypto-objs := vmx.o aesp8-ppc.o ghashp8-ppc.o aes.o aes_cbc.o aes_ctr.o gha ifeq ($(CONFIG_CPU_LITTLE_ENDIAN),y) TARGET := linux-ppc64le else -TARGET := linux-pcc64 +TARGET := linux-ppc64 endif quiet_cmd_perl = PERL $@ diff --git a/kernel/drivers/crypto/vmx/aes.c b/kernel/drivers/crypto/vmx/aes.c index 41f93334c..263af709e 100644 --- a/kernel/drivers/crypto/vmx/aes.c +++ b/kernel/drivers/crypto/vmx/aes.c @@ -30,113 +30,121 @@ #include "aesp8-ppc.h" struct p8_aes_ctx { - struct crypto_cipher *fallback; - struct aes_key enc_key; - struct aes_key dec_key; + struct crypto_cipher *fallback; + struct aes_key enc_key; + struct aes_key dec_key; }; static int p8_aes_init(struct crypto_tfm *tfm) { - const char *alg; - struct crypto_cipher *fallback; - struct p8_aes_ctx *ctx = crypto_tfm_ctx(tfm); - - if (!(alg = crypto_tfm_alg_name(tfm))) { - printk(KERN_ERR "Failed to get algorithm name.\n"); - return -ENOENT; - } - - fallback = crypto_alloc_cipher(alg, 0 ,CRYPTO_ALG_NEED_FALLBACK); - if (IS_ERR(fallback)) { - printk(KERN_ERR "Failed to allocate transformation for '%s': %ld\n", - alg, PTR_ERR(fallback)); - return PTR_ERR(fallback); - } - printk(KERN_INFO "Using '%s' as fallback implementation.\n", - crypto_tfm_alg_driver_name((struct crypto_tfm *) fallback)); - - crypto_cipher_set_flags(fallback, - crypto_cipher_get_flags((struct crypto_cipher *) tfm)); - ctx->fallback = fallback; - - return 0; + const char *alg; + struct crypto_cipher *fallback; + struct p8_aes_ctx *ctx = crypto_tfm_ctx(tfm); + + if (!(alg = crypto_tfm_alg_name(tfm))) { + printk(KERN_ERR "Failed to get algorithm name.\n"); + return -ENOENT; + } + + fallback = crypto_alloc_cipher(alg, 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(fallback)) { + printk(KERN_ERR + "Failed to allocate transformation for '%s': %ld\n", + alg, PTR_ERR(fallback)); + return PTR_ERR(fallback); + } + printk(KERN_INFO "Using '%s' as fallback implementation.\n", + crypto_tfm_alg_driver_name((struct crypto_tfm *) fallback)); + + crypto_cipher_set_flags(fallback, + crypto_cipher_get_flags((struct + crypto_cipher *) + tfm)); + ctx->fallback = fallback; + + return 0; } static void p8_aes_exit(struct crypto_tfm *tfm) { - struct p8_aes_ctx *ctx = crypto_tfm_ctx(tfm); + struct p8_aes_ctx *ctx = crypto_tfm_ctx(tfm); - if (ctx->fallback) { - crypto_free_cipher(ctx->fallback); - ctx->fallback = NULL; - } + if (ctx->fallback) { + crypto_free_cipher(ctx->fallback); + ctx->fallback = NULL; + } } static int p8_aes_setkey(struct crypto_tfm *tfm, const u8 *key, - unsigned int keylen) + unsigned int keylen) { - int ret; - struct p8_aes_ctx *ctx = crypto_tfm_ctx(tfm); - - pagefault_disable(); - enable_kernel_altivec(); - enable_kernel_vsx(); - ret = aes_p8_set_encrypt_key(key, keylen * 8, &ctx->enc_key); - ret += aes_p8_set_decrypt_key(key, keylen * 8, &ctx->dec_key); - pagefault_enable(); - - ret += crypto_cipher_setkey(ctx->fallback, key, keylen); - return ret; + int ret; + struct p8_aes_ctx *ctx = crypto_tfm_ctx(tfm); + + preempt_disable(); + pagefault_disable(); + enable_kernel_altivec(); + enable_kernel_vsx(); + ret = aes_p8_set_encrypt_key(key, keylen * 8, &ctx->enc_key); + ret += aes_p8_set_decrypt_key(key, keylen * 8, &ctx->dec_key); + pagefault_enable(); + preempt_enable(); + + ret += crypto_cipher_setkey(ctx->fallback, key, keylen); + return ret; } static void p8_aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) { - struct p8_aes_ctx *ctx = crypto_tfm_ctx(tfm); - - if (in_interrupt()) { - crypto_cipher_encrypt_one(ctx->fallback, dst, src); - } else { - pagefault_disable(); - enable_kernel_altivec(); - enable_kernel_vsx(); - aes_p8_encrypt(src, dst, &ctx->enc_key); - pagefault_enable(); - } + struct p8_aes_ctx *ctx = crypto_tfm_ctx(tfm); + + if (in_interrupt()) { + crypto_cipher_encrypt_one(ctx->fallback, dst, src); + } else { + preempt_disable(); + pagefault_disable(); + enable_kernel_altivec(); + enable_kernel_vsx(); + aes_p8_encrypt(src, dst, &ctx->enc_key); + pagefault_enable(); + preempt_enable(); + } } static void p8_aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) { - struct p8_aes_ctx *ctx = crypto_tfm_ctx(tfm); - - if (in_interrupt()) { - crypto_cipher_decrypt_one(ctx->fallback, dst, src); - } else { - pagefault_disable(); - enable_kernel_altivec(); - enable_kernel_vsx(); - aes_p8_decrypt(src, dst, &ctx->dec_key); - pagefault_enable(); - } + struct p8_aes_ctx *ctx = crypto_tfm_ctx(tfm); + + if (in_interrupt()) { + crypto_cipher_decrypt_one(ctx->fallback, dst, src); + } else { + preempt_disable(); + pagefault_disable(); + enable_kernel_altivec(); + enable_kernel_vsx(); + aes_p8_decrypt(src, dst, &ctx->dec_key); + pagefault_enable(); + preempt_enable(); + } } struct crypto_alg p8_aes_alg = { - .cra_name = "aes", - .cra_driver_name = "p8_aes", - .cra_module = THIS_MODULE, - .cra_priority = 1000, - .cra_type = NULL, - .cra_flags = CRYPTO_ALG_TYPE_CIPHER | CRYPTO_ALG_NEED_FALLBACK, - .cra_alignmask = 0, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct p8_aes_ctx), - .cra_init = p8_aes_init, - .cra_exit = p8_aes_exit, - .cra_cipher = { - .cia_min_keysize = AES_MIN_KEY_SIZE, - .cia_max_keysize = AES_MAX_KEY_SIZE, - .cia_setkey = p8_aes_setkey, - .cia_encrypt = p8_aes_encrypt, - .cia_decrypt = p8_aes_decrypt, - }, + .cra_name = "aes", + .cra_driver_name = "p8_aes", + .cra_module = THIS_MODULE, + .cra_priority = 1000, + .cra_type = NULL, + .cra_flags = CRYPTO_ALG_TYPE_CIPHER | CRYPTO_ALG_NEED_FALLBACK, + .cra_alignmask = 0, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct p8_aes_ctx), + .cra_init = p8_aes_init, + .cra_exit = p8_aes_exit, + .cra_cipher = { + .cia_min_keysize = AES_MIN_KEY_SIZE, + .cia_max_keysize = AES_MAX_KEY_SIZE, + .cia_setkey = p8_aes_setkey, + .cia_encrypt = p8_aes_encrypt, + .cia_decrypt = p8_aes_decrypt, + }, }; - diff --git a/kernel/drivers/crypto/vmx/aes_cbc.c b/kernel/drivers/crypto/vmx/aes_cbc.c index c8e7f653e..0b8fe2ec5 100644 --- a/kernel/drivers/crypto/vmx/aes_cbc.c +++ b/kernel/drivers/crypto/vmx/aes_cbc.c @@ -31,157 +31,171 @@ #include "aesp8-ppc.h" struct p8_aes_cbc_ctx { - struct crypto_blkcipher *fallback; - struct aes_key enc_key; - struct aes_key dec_key; + struct crypto_blkcipher *fallback; + struct aes_key enc_key; + struct aes_key dec_key; }; static int p8_aes_cbc_init(struct crypto_tfm *tfm) { - const char *alg; - struct crypto_blkcipher *fallback; - struct p8_aes_cbc_ctx *ctx = crypto_tfm_ctx(tfm); - - if (!(alg = crypto_tfm_alg_name(tfm))) { - printk(KERN_ERR "Failed to get algorithm name.\n"); - return -ENOENT; - } - - fallback = crypto_alloc_blkcipher(alg, 0 ,CRYPTO_ALG_NEED_FALLBACK); - if (IS_ERR(fallback)) { - printk(KERN_ERR "Failed to allocate transformation for '%s': %ld\n", - alg, PTR_ERR(fallback)); - return PTR_ERR(fallback); - } - printk(KERN_INFO "Using '%s' as fallback implementation.\n", - crypto_tfm_alg_driver_name((struct crypto_tfm *) fallback)); - - crypto_blkcipher_set_flags(fallback, - crypto_blkcipher_get_flags((struct crypto_blkcipher *) tfm)); - ctx->fallback = fallback; - - return 0; + const char *alg; + struct crypto_blkcipher *fallback; + struct p8_aes_cbc_ctx *ctx = crypto_tfm_ctx(tfm); + + if (!(alg = crypto_tfm_alg_name(tfm))) { + printk(KERN_ERR "Failed to get algorithm name.\n"); + return -ENOENT; + } + + fallback = + crypto_alloc_blkcipher(alg, 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(fallback)) { + printk(KERN_ERR + "Failed to allocate transformation for '%s': %ld\n", + alg, PTR_ERR(fallback)); + return PTR_ERR(fallback); + } + printk(KERN_INFO "Using '%s' as fallback implementation.\n", + crypto_tfm_alg_driver_name((struct crypto_tfm *) fallback)); + + crypto_blkcipher_set_flags( + fallback, + crypto_blkcipher_get_flags((struct crypto_blkcipher *)tfm)); + ctx->fallback = fallback; + + return 0; } static void p8_aes_cbc_exit(struct crypto_tfm *tfm) { - struct p8_aes_cbc_ctx *ctx = crypto_tfm_ctx(tfm); + struct p8_aes_cbc_ctx *ctx = crypto_tfm_ctx(tfm); - if (ctx->fallback) { - crypto_free_blkcipher(ctx->fallback); - ctx->fallback = NULL; - } + if (ctx->fallback) { + crypto_free_blkcipher(ctx->fallback); + ctx->fallback = NULL; + } } static int p8_aes_cbc_setkey(struct crypto_tfm *tfm, const u8 *key, - unsigned int keylen) + unsigned int keylen) { - int ret; - struct p8_aes_cbc_ctx *ctx = crypto_tfm_ctx(tfm); - - pagefault_disable(); - enable_kernel_altivec(); - enable_kernel_vsx(); - ret = aes_p8_set_encrypt_key(key, keylen * 8, &ctx->enc_key); - ret += aes_p8_set_decrypt_key(key, keylen * 8, &ctx->dec_key); - pagefault_enable(); - - ret += crypto_blkcipher_setkey(ctx->fallback, key, keylen); - return ret; + int ret; + struct p8_aes_cbc_ctx *ctx = crypto_tfm_ctx(tfm); + + preempt_disable(); + pagefault_disable(); + enable_kernel_altivec(); + enable_kernel_vsx(); + ret = aes_p8_set_encrypt_key(key, keylen * 8, &ctx->enc_key); + ret += aes_p8_set_decrypt_key(key, keylen * 8, &ctx->dec_key); + pagefault_enable(); + preempt_enable(); + + ret += crypto_blkcipher_setkey(ctx->fallback, key, keylen); + return ret; } static int p8_aes_cbc_encrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, struct scatterlist *src, - unsigned int nbytes) + struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) { - int ret; - struct blkcipher_walk walk; - struct p8_aes_cbc_ctx *ctx = crypto_tfm_ctx( - crypto_blkcipher_tfm(desc->tfm)); - struct blkcipher_desc fallback_desc = { - .tfm = ctx->fallback, - .info = desc->info, - .flags = desc->flags - }; - - if (in_interrupt()) { - ret = crypto_blkcipher_encrypt(&fallback_desc, dst, src, nbytes); - } else { - pagefault_disable(); - enable_kernel_altivec(); - enable_kernel_vsx(); - - blkcipher_walk_init(&walk, dst, src, nbytes); - ret = blkcipher_walk_virt(desc, &walk); - while ((nbytes = walk.nbytes)) { - aes_p8_cbc_encrypt(walk.src.virt.addr, walk.dst.virt.addr, - nbytes & AES_BLOCK_MASK, &ctx->enc_key, walk.iv, 1); + int ret; + struct blkcipher_walk walk; + struct p8_aes_cbc_ctx *ctx = + crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm)); + struct blkcipher_desc fallback_desc = { + .tfm = ctx->fallback, + .info = desc->info, + .flags = desc->flags + }; + + if (in_interrupt()) { + ret = crypto_blkcipher_encrypt(&fallback_desc, dst, src, + nbytes); + } else { + preempt_disable(); + pagefault_disable(); + enable_kernel_altivec(); + enable_kernel_vsx(); + + blkcipher_walk_init(&walk, dst, src, nbytes); + ret = blkcipher_walk_virt(desc, &walk); + while ((nbytes = walk.nbytes)) { + aes_p8_cbc_encrypt(walk.src.virt.addr, + walk.dst.virt.addr, + nbytes & AES_BLOCK_MASK, + &ctx->enc_key, walk.iv, 1); nbytes &= AES_BLOCK_SIZE - 1; ret = blkcipher_walk_done(desc, &walk, nbytes); - } + } - pagefault_enable(); - } + pagefault_enable(); + preempt_enable(); + } - return ret; + return ret; } static int p8_aes_cbc_decrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, struct scatterlist *src, - unsigned int nbytes) + struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) { - int ret; - struct blkcipher_walk walk; - struct p8_aes_cbc_ctx *ctx = crypto_tfm_ctx( - crypto_blkcipher_tfm(desc->tfm)); - struct blkcipher_desc fallback_desc = { - .tfm = ctx->fallback, - .info = desc->info, - .flags = desc->flags - }; - - if (in_interrupt()) { - ret = crypto_blkcipher_decrypt(&fallback_desc, dst, src, nbytes); - } else { - pagefault_disable(); - enable_kernel_altivec(); - enable_kernel_vsx(); - - blkcipher_walk_init(&walk, dst, src, nbytes); - ret = blkcipher_walk_virt(desc, &walk); - while ((nbytes = walk.nbytes)) { - aes_p8_cbc_encrypt(walk.src.virt.addr, walk.dst.virt.addr, - nbytes & AES_BLOCK_MASK, &ctx->dec_key, walk.iv, 0); + int ret; + struct blkcipher_walk walk; + struct p8_aes_cbc_ctx *ctx = + crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm)); + struct blkcipher_desc fallback_desc = { + .tfm = ctx->fallback, + .info = desc->info, + .flags = desc->flags + }; + + if (in_interrupt()) { + ret = crypto_blkcipher_decrypt(&fallback_desc, dst, src, + nbytes); + } else { + preempt_disable(); + pagefault_disable(); + enable_kernel_altivec(); + enable_kernel_vsx(); + + blkcipher_walk_init(&walk, dst, src, nbytes); + ret = blkcipher_walk_virt(desc, &walk); + while ((nbytes = walk.nbytes)) { + aes_p8_cbc_encrypt(walk.src.virt.addr, + walk.dst.virt.addr, + nbytes & AES_BLOCK_MASK, + &ctx->dec_key, walk.iv, 0); nbytes &= AES_BLOCK_SIZE - 1; ret = blkcipher_walk_done(desc, &walk, nbytes); } - pagefault_enable(); - } + pagefault_enable(); + preempt_enable(); + } - return ret; + return ret; } struct crypto_alg p8_aes_cbc_alg = { - .cra_name = "cbc(aes)", - .cra_driver_name = "p8_aes_cbc", - .cra_module = THIS_MODULE, - .cra_priority = 1000, - .cra_type = &crypto_blkcipher_type, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | CRYPTO_ALG_NEED_FALLBACK, - .cra_alignmask = 0, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct p8_aes_cbc_ctx), - .cra_init = p8_aes_cbc_init, - .cra_exit = p8_aes_cbc_exit, - .cra_blkcipher = { - .ivsize = 0, - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .setkey = p8_aes_cbc_setkey, - .encrypt = p8_aes_cbc_encrypt, - .decrypt = p8_aes_cbc_decrypt, - }, + .cra_name = "cbc(aes)", + .cra_driver_name = "p8_aes_cbc", + .cra_module = THIS_MODULE, + .cra_priority = 1000, + .cra_type = &crypto_blkcipher_type, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | CRYPTO_ALG_NEED_FALLBACK, + .cra_alignmask = 0, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct p8_aes_cbc_ctx), + .cra_init = p8_aes_cbc_init, + .cra_exit = p8_aes_cbc_exit, + .cra_blkcipher = { + .ivsize = 0, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = p8_aes_cbc_setkey, + .encrypt = p8_aes_cbc_encrypt, + .decrypt = p8_aes_cbc_decrypt, + }, }; - diff --git a/kernel/drivers/crypto/vmx/aes_ctr.c b/kernel/drivers/crypto/vmx/aes_ctr.c index 266e708d6..ee1306cd8 100644 --- a/kernel/drivers/crypto/vmx/aes_ctr.c +++ b/kernel/drivers/crypto/vmx/aes_ctr.c @@ -30,141 +30,156 @@ #include "aesp8-ppc.h" struct p8_aes_ctr_ctx { - struct crypto_blkcipher *fallback; - struct aes_key enc_key; + struct crypto_blkcipher *fallback; + struct aes_key enc_key; }; static int p8_aes_ctr_init(struct crypto_tfm *tfm) { - const char *alg; - struct crypto_blkcipher *fallback; - struct p8_aes_ctr_ctx *ctx = crypto_tfm_ctx(tfm); - - if (!(alg = crypto_tfm_alg_name(tfm))) { - printk(KERN_ERR "Failed to get algorithm name.\n"); - return -ENOENT; - } - - fallback = crypto_alloc_blkcipher(alg, 0 ,CRYPTO_ALG_NEED_FALLBACK); - if (IS_ERR(fallback)) { - printk(KERN_ERR "Failed to allocate transformation for '%s': %ld\n", - alg, PTR_ERR(fallback)); - return PTR_ERR(fallback); - } - printk(KERN_INFO "Using '%s' as fallback implementation.\n", - crypto_tfm_alg_driver_name((struct crypto_tfm *) fallback)); - - crypto_blkcipher_set_flags(fallback, - crypto_blkcipher_get_flags((struct crypto_blkcipher *) tfm)); - ctx->fallback = fallback; - - return 0; + const char *alg; + struct crypto_blkcipher *fallback; + struct p8_aes_ctr_ctx *ctx = crypto_tfm_ctx(tfm); + + if (!(alg = crypto_tfm_alg_name(tfm))) { + printk(KERN_ERR "Failed to get algorithm name.\n"); + return -ENOENT; + } + + fallback = + crypto_alloc_blkcipher(alg, 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(fallback)) { + printk(KERN_ERR + "Failed to allocate transformation for '%s': %ld\n", + alg, PTR_ERR(fallback)); + return PTR_ERR(fallback); + } + printk(KERN_INFO "Using '%s' as fallback implementation.\n", + crypto_tfm_alg_driver_name((struct crypto_tfm *) fallback)); + + crypto_blkcipher_set_flags( + fallback, + crypto_blkcipher_get_flags((struct crypto_blkcipher *)tfm)); + ctx->fallback = fallback; + + return 0; } static void p8_aes_ctr_exit(struct crypto_tfm *tfm) { - struct p8_aes_ctr_ctx *ctx = crypto_tfm_ctx(tfm); + struct p8_aes_ctr_ctx *ctx = crypto_tfm_ctx(tfm); - if (ctx->fallback) { - crypto_free_blkcipher(ctx->fallback); - ctx->fallback = NULL; - } + if (ctx->fallback) { + crypto_free_blkcipher(ctx->fallback); + ctx->fallback = NULL; + } } static int p8_aes_ctr_setkey(struct crypto_tfm *tfm, const u8 *key, - unsigned int keylen) + unsigned int keylen) { - int ret; - struct p8_aes_ctr_ctx *ctx = crypto_tfm_ctx(tfm); + int ret; + struct p8_aes_ctr_ctx *ctx = crypto_tfm_ctx(tfm); - pagefault_disable(); - enable_kernel_altivec(); - enable_kernel_vsx(); - ret = aes_p8_set_encrypt_key(key, keylen * 8, &ctx->enc_key); - pagefault_enable(); + pagefault_disable(); + enable_kernel_altivec(); + enable_kernel_vsx(); + ret = aes_p8_set_encrypt_key(key, keylen * 8, &ctx->enc_key); + pagefault_enable(); - ret += crypto_blkcipher_setkey(ctx->fallback, key, keylen); - return ret; + ret += crypto_blkcipher_setkey(ctx->fallback, key, keylen); + return ret; } static void p8_aes_ctr_final(struct p8_aes_ctr_ctx *ctx, - struct blkcipher_walk *walk) + struct blkcipher_walk *walk) { - u8 *ctrblk = walk->iv; - u8 keystream[AES_BLOCK_SIZE]; - u8 *src = walk->src.virt.addr; - u8 *dst = walk->dst.virt.addr; - unsigned int nbytes = walk->nbytes; - - pagefault_disable(); - enable_kernel_altivec(); - enable_kernel_vsx(); - aes_p8_encrypt(ctrblk, keystream, &ctx->enc_key); - pagefault_enable(); - - crypto_xor(keystream, src, nbytes); - memcpy(dst, keystream, nbytes); - crypto_inc(ctrblk, AES_BLOCK_SIZE); + u8 *ctrblk = walk->iv; + u8 keystream[AES_BLOCK_SIZE]; + u8 *src = walk->src.virt.addr; + u8 *dst = walk->dst.virt.addr; + unsigned int nbytes = walk->nbytes; + + pagefault_disable(); + enable_kernel_altivec(); + enable_kernel_vsx(); + aes_p8_encrypt(ctrblk, keystream, &ctx->enc_key); + pagefault_enable(); + + crypto_xor(keystream, src, nbytes); + memcpy(dst, keystream, nbytes); + crypto_inc(ctrblk, AES_BLOCK_SIZE); } static int p8_aes_ctr_crypt(struct blkcipher_desc *desc, - struct scatterlist *dst, struct scatterlist *src, - unsigned int nbytes) + struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) { - int ret; - struct blkcipher_walk walk; - struct p8_aes_ctr_ctx *ctx = crypto_tfm_ctx( - crypto_blkcipher_tfm(desc->tfm)); - struct blkcipher_desc fallback_desc = { - .tfm = ctx->fallback, - .info = desc->info, - .flags = desc->flags - }; - - if (in_interrupt()) { - ret = crypto_blkcipher_encrypt(&fallback_desc, dst, src, nbytes); - } else { - blkcipher_walk_init(&walk, dst, src, nbytes); - ret = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE); - while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) { - pagefault_disable(); - enable_kernel_altivec(); - enable_kernel_vsx(); - aes_p8_ctr32_encrypt_blocks(walk.src.virt.addr, walk.dst.virt.addr, - (nbytes & AES_BLOCK_MASK)/AES_BLOCK_SIZE, &ctx->enc_key, walk.iv); - pagefault_enable(); - - crypto_inc(walk.iv, AES_BLOCK_SIZE); - nbytes &= AES_BLOCK_SIZE - 1; - ret = blkcipher_walk_done(desc, &walk, nbytes); - } - if (walk.nbytes) { - p8_aes_ctr_final(ctx, &walk); - ret = blkcipher_walk_done(desc, &walk, 0); - } - } - - return ret; + int ret; + u64 inc; + struct blkcipher_walk walk; + struct p8_aes_ctr_ctx *ctx = + crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm)); + struct blkcipher_desc fallback_desc = { + .tfm = ctx->fallback, + .info = desc->info, + .flags = desc->flags + }; + + if (in_interrupt()) { + ret = crypto_blkcipher_encrypt(&fallback_desc, dst, src, + nbytes); + } else { + blkcipher_walk_init(&walk, dst, src, nbytes); + ret = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE); + while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) { + pagefault_disable(); + enable_kernel_altivec(); + enable_kernel_vsx(); + aes_p8_ctr32_encrypt_blocks(walk.src.virt.addr, + walk.dst.virt.addr, + (nbytes & + AES_BLOCK_MASK) / + AES_BLOCK_SIZE, + &ctx->enc_key, + walk.iv); + pagefault_enable(); + + /* We need to update IV mostly for last bytes/round */ + inc = (nbytes & AES_BLOCK_MASK) / AES_BLOCK_SIZE; + if (inc > 0) + while (inc--) + crypto_inc(walk.iv, AES_BLOCK_SIZE); + + nbytes &= AES_BLOCK_SIZE - 1; + ret = blkcipher_walk_done(desc, &walk, nbytes); + } + if (walk.nbytes) { + p8_aes_ctr_final(ctx, &walk); + ret = blkcipher_walk_done(desc, &walk, 0); + } + } + + return ret; } struct crypto_alg p8_aes_ctr_alg = { - .cra_name = "ctr(aes)", - .cra_driver_name = "p8_aes_ctr", - .cra_module = THIS_MODULE, - .cra_priority = 1000, - .cra_type = &crypto_blkcipher_type, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | CRYPTO_ALG_NEED_FALLBACK, - .cra_alignmask = 0, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct p8_aes_ctr_ctx), - .cra_init = p8_aes_ctr_init, - .cra_exit = p8_aes_ctr_exit, - .cra_blkcipher = { - .ivsize = 0, - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .setkey = p8_aes_ctr_setkey, - .encrypt = p8_aes_ctr_crypt, - .decrypt = p8_aes_ctr_crypt, - }, + .cra_name = "ctr(aes)", + .cra_driver_name = "p8_aes_ctr", + .cra_module = THIS_MODULE, + .cra_priority = 1000, + .cra_type = &crypto_blkcipher_type, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | CRYPTO_ALG_NEED_FALLBACK, + .cra_alignmask = 0, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct p8_aes_ctr_ctx), + .cra_init = p8_aes_ctr_init, + .cra_exit = p8_aes_ctr_exit, + .cra_blkcipher = { + .ivsize = 0, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = p8_aes_ctr_setkey, + .encrypt = p8_aes_ctr_crypt, + .decrypt = p8_aes_ctr_crypt, + }, }; diff --git a/kernel/drivers/crypto/vmx/aesp8-ppc.h b/kernel/drivers/crypto/vmx/aesp8-ppc.h index e963945a8..4cd34ee54 100644 --- a/kernel/drivers/crypto/vmx/aesp8-ppc.h +++ b/kernel/drivers/crypto/vmx/aesp8-ppc.h @@ -4,17 +4,18 @@ #define AES_BLOCK_MASK (~(AES_BLOCK_SIZE-1)) struct aes_key { - u8 key[AES_MAX_KEYLENGTH]; - int rounds; + u8 key[AES_MAX_KEYLENGTH]; + int rounds; }; int aes_p8_set_encrypt_key(const u8 *userKey, const int bits, - struct aes_key *key); + struct aes_key *key); int aes_p8_set_decrypt_key(const u8 *userKey, const int bits, - struct aes_key *key); + struct aes_key *key); void aes_p8_encrypt(const u8 *in, u8 *out, const struct aes_key *key); -void aes_p8_decrypt(const u8 *in, u8 *out,const struct aes_key *key); +void aes_p8_decrypt(const u8 *in, u8 *out, const struct aes_key *key); void aes_p8_cbc_encrypt(const u8 *in, u8 *out, size_t len, - const struct aes_key *key, u8 *iv, const int enc); + const struct aes_key *key, u8 *iv, const int enc); void aes_p8_ctr32_encrypt_blocks(const u8 *in, u8 *out, - size_t len, const struct aes_key *key, const u8 *iv); + size_t len, const struct aes_key *key, + const u8 *iv); diff --git a/kernel/drivers/crypto/vmx/aesp8-ppc.pl b/kernel/drivers/crypto/vmx/aesp8-ppc.pl index 6c5c20c61..228053921 100644 --- a/kernel/drivers/crypto/vmx/aesp8-ppc.pl +++ b/kernel/drivers/crypto/vmx/aesp8-ppc.pl @@ -1437,28 +1437,28 @@ Load_ctr32_enc_key: ?vperm v31,v31,$out0,$keyperm lvx v25,$x10,$key_ # pre-load round[2] - vadduwm $two,$one,$one + vadduqm $two,$one,$one subi $inp,$inp,15 # undo "caller" $SHL $len,$len,4 - vadduwm $out1,$ivec,$one # counter values ... - vadduwm $out2,$ivec,$two + vadduqm $out1,$ivec,$one # counter values ... + vadduqm $out2,$ivec,$two vxor $out0,$ivec,$rndkey0 # ... xored with rndkey[0] le?li $idx,8 - vadduwm $out3,$out1,$two + vadduqm $out3,$out1,$two vxor $out1,$out1,$rndkey0 le?lvsl $inpperm,0,$idx - vadduwm $out4,$out2,$two + vadduqm $out4,$out2,$two vxor $out2,$out2,$rndkey0 le?vspltisb $tmp,0x0f - vadduwm $out5,$out3,$two + vadduqm $out5,$out3,$two vxor $out3,$out3,$rndkey0 le?vxor $inpperm,$inpperm,$tmp # transform for lvx_u/stvx_u - vadduwm $out6,$out4,$two + vadduqm $out6,$out4,$two vxor $out4,$out4,$rndkey0 - vadduwm $out7,$out5,$two + vadduqm $out7,$out5,$two vxor $out5,$out5,$rndkey0 - vadduwm $ivec,$out6,$two # next counter value + vadduqm $ivec,$out6,$two # next counter value vxor $out6,$out6,$rndkey0 vxor $out7,$out7,$rndkey0 @@ -1594,27 +1594,27 @@ Loop_ctr32_enc8x_middle: vcipherlast $in0,$out0,$in0 vcipherlast $in1,$out1,$in1 - vadduwm $out1,$ivec,$one # counter values ... + vadduqm $out1,$ivec,$one # counter values ... vcipherlast $in2,$out2,$in2 - vadduwm $out2,$ivec,$two + vadduqm $out2,$ivec,$two vxor $out0,$ivec,$rndkey0 # ... xored with rndkey[0] vcipherlast $in3,$out3,$in3 - vadduwm $out3,$out1,$two + vadduqm $out3,$out1,$two vxor $out1,$out1,$rndkey0 vcipherlast $in4,$out4,$in4 - vadduwm $out4,$out2,$two + vadduqm $out4,$out2,$two vxor $out2,$out2,$rndkey0 vcipherlast $in5,$out5,$in5 - vadduwm $out5,$out3,$two + vadduqm $out5,$out3,$two vxor $out3,$out3,$rndkey0 vcipherlast $in6,$out6,$in6 - vadduwm $out6,$out4,$two + vadduqm $out6,$out4,$two vxor $out4,$out4,$rndkey0 vcipherlast $in7,$out7,$in7 - vadduwm $out7,$out5,$two + vadduqm $out7,$out5,$two vxor $out5,$out5,$rndkey0 le?vperm $in0,$in0,$in0,$inpperm - vadduwm $ivec,$out6,$two # next counter value + vadduqm $ivec,$out6,$two # next counter value vxor $out6,$out6,$rndkey0 le?vperm $in1,$in1,$in1,$inpperm vxor $out7,$out7,$rndkey0 diff --git a/kernel/drivers/crypto/vmx/ghash.c b/kernel/drivers/crypto/vmx/ghash.c index 917b3f09e..2183a2e77 100644 --- a/kernel/drivers/crypto/vmx/ghash.c +++ b/kernel/drivers/crypto/vmx/ghash.c @@ -39,180 +39,192 @@ void gcm_init_p8(u128 htable[16], const u64 Xi[2]); void gcm_gmult_p8(u64 Xi[2], const u128 htable[16]); void gcm_ghash_p8(u64 Xi[2], const u128 htable[16], - const u8 *in,size_t len); + const u8 *in, size_t len); struct p8_ghash_ctx { - u128 htable[16]; - struct crypto_shash *fallback; + u128 htable[16]; + struct crypto_shash *fallback; }; struct p8_ghash_desc_ctx { - u64 shash[2]; - u8 buffer[GHASH_DIGEST_SIZE]; - int bytes; - struct shash_desc fallback_desc; + u64 shash[2]; + u8 buffer[GHASH_DIGEST_SIZE]; + int bytes; + struct shash_desc fallback_desc; }; static int p8_ghash_init_tfm(struct crypto_tfm *tfm) { - const char *alg; - struct crypto_shash *fallback; - struct crypto_shash *shash_tfm = __crypto_shash_cast(tfm); - struct p8_ghash_ctx *ctx = crypto_tfm_ctx(tfm); - - if (!(alg = crypto_tfm_alg_name(tfm))) { - printk(KERN_ERR "Failed to get algorithm name.\n"); - return -ENOENT; - } - - fallback = crypto_alloc_shash(alg, 0 ,CRYPTO_ALG_NEED_FALLBACK); - if (IS_ERR(fallback)) { - printk(KERN_ERR "Failed to allocate transformation for '%s': %ld\n", - alg, PTR_ERR(fallback)); - return PTR_ERR(fallback); - } - printk(KERN_INFO "Using '%s' as fallback implementation.\n", - crypto_tfm_alg_driver_name(crypto_shash_tfm(fallback))); - - crypto_shash_set_flags(fallback, - crypto_shash_get_flags((struct crypto_shash *) tfm)); - ctx->fallback = fallback; - - shash_tfm->descsize = sizeof(struct p8_ghash_desc_ctx) - + crypto_shash_descsize(fallback); - - return 0; + const char *alg; + struct crypto_shash *fallback; + struct crypto_shash *shash_tfm = __crypto_shash_cast(tfm); + struct p8_ghash_ctx *ctx = crypto_tfm_ctx(tfm); + + if (!(alg = crypto_tfm_alg_name(tfm))) { + printk(KERN_ERR "Failed to get algorithm name.\n"); + return -ENOENT; + } + + fallback = crypto_alloc_shash(alg, 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(fallback)) { + printk(KERN_ERR + "Failed to allocate transformation for '%s': %ld\n", + alg, PTR_ERR(fallback)); + return PTR_ERR(fallback); + } + printk(KERN_INFO "Using '%s' as fallback implementation.\n", + crypto_tfm_alg_driver_name(crypto_shash_tfm(fallback))); + + crypto_shash_set_flags(fallback, + crypto_shash_get_flags((struct crypto_shash + *) tfm)); + ctx->fallback = fallback; + + shash_tfm->descsize = sizeof(struct p8_ghash_desc_ctx) + + crypto_shash_descsize(fallback); + + return 0; } static void p8_ghash_exit_tfm(struct crypto_tfm *tfm) { - struct p8_ghash_ctx *ctx = crypto_tfm_ctx(tfm); + struct p8_ghash_ctx *ctx = crypto_tfm_ctx(tfm); - if (ctx->fallback) { - crypto_free_shash(ctx->fallback); - ctx->fallback = NULL; - } + if (ctx->fallback) { + crypto_free_shash(ctx->fallback); + ctx->fallback = NULL; + } } static int p8_ghash_init(struct shash_desc *desc) { - struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(desc->tfm)); - struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - dctx->bytes = 0; - memset(dctx->shash, 0, GHASH_DIGEST_SIZE); - dctx->fallback_desc.tfm = ctx->fallback; - dctx->fallback_desc.flags = desc->flags; - return crypto_shash_init(&dctx->fallback_desc); + struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(desc->tfm)); + struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc); + + dctx->bytes = 0; + memset(dctx->shash, 0, GHASH_DIGEST_SIZE); + dctx->fallback_desc.tfm = ctx->fallback; + dctx->fallback_desc.flags = desc->flags; + return crypto_shash_init(&dctx->fallback_desc); } static int p8_ghash_setkey(struct crypto_shash *tfm, const u8 *key, - unsigned int keylen) + unsigned int keylen) { - struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(tfm)); - - if (keylen != GHASH_KEY_LEN) - return -EINVAL; - - pagefault_disable(); - enable_kernel_altivec(); - enable_kernel_vsx(); - enable_kernel_fp(); - gcm_init_p8(ctx->htable, (const u64 *) key); - pagefault_enable(); - return crypto_shash_setkey(ctx->fallback, key, keylen); + struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(tfm)); + + if (keylen != GHASH_KEY_LEN) + return -EINVAL; + + preempt_disable(); + pagefault_disable(); + enable_kernel_altivec(); + enable_kernel_vsx(); + enable_kernel_fp(); + gcm_init_p8(ctx->htable, (const u64 *) key); + pagefault_enable(); + preempt_enable(); + return crypto_shash_setkey(ctx->fallback, key, keylen); } static int p8_ghash_update(struct shash_desc *desc, - const u8 *src, unsigned int srclen) + const u8 *src, unsigned int srclen) { - unsigned int len; - struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(desc->tfm)); - struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - if (IN_INTERRUPT) { - return crypto_shash_update(&dctx->fallback_desc, src, srclen); - } else { - if (dctx->bytes) { - if (dctx->bytes + srclen < GHASH_DIGEST_SIZE) { - memcpy(dctx->buffer + dctx->bytes, src, srclen); - dctx->bytes += srclen; - return 0; - } - memcpy(dctx->buffer + dctx->bytes, src, - GHASH_DIGEST_SIZE - dctx->bytes); - pagefault_disable(); - enable_kernel_altivec(); - enable_kernel_vsx(); - enable_kernel_fp(); - gcm_ghash_p8(dctx->shash, ctx->htable, dctx->buffer, - GHASH_DIGEST_SIZE); - pagefault_enable(); - src += GHASH_DIGEST_SIZE - dctx->bytes; - srclen -= GHASH_DIGEST_SIZE - dctx->bytes; - dctx->bytes = 0; - } - len = srclen & ~(GHASH_DIGEST_SIZE - 1); - if (len) { - pagefault_disable(); - enable_kernel_altivec(); - enable_kernel_vsx(); - enable_kernel_fp(); - gcm_ghash_p8(dctx->shash, ctx->htable, src, len); - pagefault_enable(); - src += len; - srclen -= len; - } - if (srclen) { - memcpy(dctx->buffer, src, srclen); - dctx->bytes = srclen; - } - return 0; - } + unsigned int len; + struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(desc->tfm)); + struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc); + + if (IN_INTERRUPT) { + return crypto_shash_update(&dctx->fallback_desc, src, + srclen); + } else { + if (dctx->bytes) { + if (dctx->bytes + srclen < GHASH_DIGEST_SIZE) { + memcpy(dctx->buffer + dctx->bytes, src, + srclen); + dctx->bytes += srclen; + return 0; + } + memcpy(dctx->buffer + dctx->bytes, src, + GHASH_DIGEST_SIZE - dctx->bytes); + preempt_disable(); + pagefault_disable(); + enable_kernel_altivec(); + enable_kernel_vsx(); + enable_kernel_fp(); + gcm_ghash_p8(dctx->shash, ctx->htable, + dctx->buffer, GHASH_DIGEST_SIZE); + pagefault_enable(); + preempt_enable(); + src += GHASH_DIGEST_SIZE - dctx->bytes; + srclen -= GHASH_DIGEST_SIZE - dctx->bytes; + dctx->bytes = 0; + } + len = srclen & ~(GHASH_DIGEST_SIZE - 1); + if (len) { + preempt_disable(); + pagefault_disable(); + enable_kernel_altivec(); + enable_kernel_vsx(); + enable_kernel_fp(); + gcm_ghash_p8(dctx->shash, ctx->htable, src, len); + pagefault_enable(); + preempt_enable(); + src += len; + srclen -= len; + } + if (srclen) { + memcpy(dctx->buffer, src, srclen); + dctx->bytes = srclen; + } + return 0; + } } static int p8_ghash_final(struct shash_desc *desc, u8 *out) { - int i; - struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(desc->tfm)); - struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - if (IN_INTERRUPT) { - return crypto_shash_final(&dctx->fallback_desc, out); - } else { - if (dctx->bytes) { - for (i = dctx->bytes; i < GHASH_DIGEST_SIZE; i++) - dctx->buffer[i] = 0; - pagefault_disable(); - enable_kernel_altivec(); - enable_kernel_vsx(); - enable_kernel_fp(); - gcm_ghash_p8(dctx->shash, ctx->htable, dctx->buffer, - GHASH_DIGEST_SIZE); - pagefault_enable(); - dctx->bytes = 0; - } - memcpy(out, dctx->shash, GHASH_DIGEST_SIZE); - return 0; - } + int i; + struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(desc->tfm)); + struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc); + + if (IN_INTERRUPT) { + return crypto_shash_final(&dctx->fallback_desc, out); + } else { + if (dctx->bytes) { + for (i = dctx->bytes; i < GHASH_DIGEST_SIZE; i++) + dctx->buffer[i] = 0; + preempt_disable(); + pagefault_disable(); + enable_kernel_altivec(); + enable_kernel_vsx(); + enable_kernel_fp(); + gcm_ghash_p8(dctx->shash, ctx->htable, + dctx->buffer, GHASH_DIGEST_SIZE); + pagefault_enable(); + preempt_enable(); + dctx->bytes = 0; + } + memcpy(out, dctx->shash, GHASH_DIGEST_SIZE); + return 0; + } } struct shash_alg p8_ghash_alg = { - .digestsize = GHASH_DIGEST_SIZE, - .init = p8_ghash_init, - .update = p8_ghash_update, - .final = p8_ghash_final, - .setkey = p8_ghash_setkey, - .descsize = sizeof(struct p8_ghash_desc_ctx), - .base = { - .cra_name = "ghash", - .cra_driver_name = "p8_ghash", - .cra_priority = 1000, - .cra_flags = CRYPTO_ALG_TYPE_SHASH | CRYPTO_ALG_NEED_FALLBACK, - .cra_blocksize = GHASH_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct p8_ghash_ctx), - .cra_module = THIS_MODULE, - .cra_init = p8_ghash_init_tfm, - .cra_exit = p8_ghash_exit_tfm, - }, + .digestsize = GHASH_DIGEST_SIZE, + .init = p8_ghash_init, + .update = p8_ghash_update, + .final = p8_ghash_final, + .setkey = p8_ghash_setkey, + .descsize = sizeof(struct p8_ghash_desc_ctx), + .base = { + .cra_name = "ghash", + .cra_driver_name = "p8_ghash", + .cra_priority = 1000, + .cra_flags = CRYPTO_ALG_TYPE_SHASH | CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = GHASH_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct p8_ghash_ctx), + .cra_module = THIS_MODULE, + .cra_init = p8_ghash_init_tfm, + .cra_exit = p8_ghash_exit_tfm, + }, }; diff --git a/kernel/drivers/crypto/vmx/ppc-xlate.pl b/kernel/drivers/crypto/vmx/ppc-xlate.pl index a59188494..b9997335f 100644 --- a/kernel/drivers/crypto/vmx/ppc-xlate.pl +++ b/kernel/drivers/crypto/vmx/ppc-xlate.pl @@ -169,6 +169,7 @@ my $vpmsumd = sub { vcrypto_op(@_, 1224); }; my $vpmsubh = sub { vcrypto_op(@_, 1096); }; my $vpmsumw = sub { vcrypto_op(@_, 1160); }; my $vaddudm = sub { vcrypto_op(@_, 192); }; +my $vadduqm = sub { vcrypto_op(@_, 256); }; my $mtsle = sub { my ($f, $arg) = @_; diff --git a/kernel/drivers/crypto/vmx/vmx.c b/kernel/drivers/crypto/vmx/vmx.c index 44d8d5cfe..e163d5770 100644 --- a/kernel/drivers/crypto/vmx/vmx.c +++ b/kernel/drivers/crypto/vmx/vmx.c @@ -32,57 +32,57 @@ extern struct crypto_alg p8_aes_alg; extern struct crypto_alg p8_aes_cbc_alg; extern struct crypto_alg p8_aes_ctr_alg; static struct crypto_alg *algs[] = { - &p8_aes_alg, - &p8_aes_cbc_alg, - &p8_aes_ctr_alg, - NULL, + &p8_aes_alg, + &p8_aes_cbc_alg, + &p8_aes_ctr_alg, + NULL, }; int __init p8_init(void) { - int ret = 0; - struct crypto_alg **alg_it; + int ret = 0; + struct crypto_alg **alg_it; - if (!(cur_cpu_spec->cpu_user_features2 & PPC_FEATURE2_VEC_CRYPTO)) - return -ENODEV; + if (!(cur_cpu_spec->cpu_user_features2 & PPC_FEATURE2_VEC_CRYPTO)) + return -ENODEV; - for (alg_it = algs; *alg_it; alg_it++) { - ret = crypto_register_alg(*alg_it); - printk(KERN_INFO "crypto_register_alg '%s' = %d\n", - (*alg_it)->cra_name, ret); - if (ret) { - for (alg_it--; alg_it >= algs; alg_it--) - crypto_unregister_alg(*alg_it); - break; - } - } - if (ret) - return ret; + for (alg_it = algs; *alg_it; alg_it++) { + ret = crypto_register_alg(*alg_it); + printk(KERN_INFO "crypto_register_alg '%s' = %d\n", + (*alg_it)->cra_name, ret); + if (ret) { + for (alg_it--; alg_it >= algs; alg_it--) + crypto_unregister_alg(*alg_it); + break; + } + } + if (ret) + return ret; - ret = crypto_register_shash(&p8_ghash_alg); - if (ret) { - for (alg_it = algs; *alg_it; alg_it++) - crypto_unregister_alg(*alg_it); - } - return ret; + ret = crypto_register_shash(&p8_ghash_alg); + if (ret) { + for (alg_it = algs; *alg_it; alg_it++) + crypto_unregister_alg(*alg_it); + } + return ret; } void __exit p8_exit(void) { - struct crypto_alg **alg_it; + struct crypto_alg **alg_it; - for (alg_it = algs; *alg_it; alg_it++) { - printk(KERN_INFO "Removing '%s'\n", (*alg_it)->cra_name); - crypto_unregister_alg(*alg_it); - } - crypto_unregister_shash(&p8_ghash_alg); + for (alg_it = algs; *alg_it; alg_it++) { + printk(KERN_INFO "Removing '%s'\n", (*alg_it)->cra_name); + crypto_unregister_alg(*alg_it); + } + crypto_unregister_shash(&p8_ghash_alg); } module_init(p8_init); module_exit(p8_exit); MODULE_AUTHOR("Marcelo Cerri"); -MODULE_DESCRIPTION("IBM VMX cryptogaphic acceleration instructions support on Power 8"); +MODULE_DESCRIPTION("IBM VMX cryptographic acceleration instructions " + "support on Power 8"); MODULE_LICENSE("GPL"); MODULE_VERSION("1.0.0"); - -- cgit 1.2.3-korg