diff options
Diffstat (limited to 'drivers')
207 files changed, 32427 insertions, 1163 deletions
diff --git a/drivers/ata/Kconfig b/drivers/ata/Kconfig index 7671dbac6015..e65d400efd44 100644 --- a/drivers/ata/Kconfig +++ b/drivers/ata/Kconfig @@ -141,6 +141,15 @@ config AHCI_SUNXI If unsure, say N. +config AHCI_TEGRA + tristate "NVIDIA Tegra124 AHCI SATA support" + depends on ARCH_TEGRA + help + This option enables support for the NVIDIA Tegra124 SoC's + onboard AHCI SATA. + + If unsure, say N. + config AHCI_XGENE tristate "APM X-Gene 6.0Gbps AHCI SATA host controller support" depends on PHY_XGENE diff --git a/drivers/ata/Makefile b/drivers/ata/Makefile index 5a02aeecef5b..ae41107afc1f 100644 --- a/drivers/ata/Makefile +++ b/drivers/ata/Makefile @@ -15,6 +15,7 @@ obj-$(CONFIG_AHCI_IMX) += ahci_imx.o libahci.o libahci_platform.o obj-$(CONFIG_AHCI_MVEBU) += ahci_mvebu.o libahci.o libahci_platform.o obj-$(CONFIG_AHCI_SUNXI) += ahci_sunxi.o libahci.o libahci_platform.o obj-$(CONFIG_AHCI_ST) += ahci_st.o libahci.o libahci_platform.o +obj-$(CONFIG_AHCI_TEGRA) += ahci_tegra.o libahci.o libahci_platform.o obj-$(CONFIG_AHCI_XGENE) += ahci_xgene.o libahci.o libahci_platform.o # SFF w/ custom DMA diff --git a/drivers/ata/acard-ahci.c b/drivers/ata/acard-ahci.c index 0cd7c7a39e5b..25d0ac32e721 100644 --- a/drivers/ata/acard-ahci.c +++ b/drivers/ata/acard-ahci.c @@ -441,7 +441,7 @@ static int acard_ahci_init_one(struct pci_dev *pdev, const struct pci_device_id hpriv->mmio = pcim_iomap_table(pdev)[AHCI_PCI_BAR]; /* save initial config */ - ahci_save_initial_config(&pdev->dev, hpriv, 0, 0); + ahci_save_initial_config(&pdev->dev, hpriv); /* prepare host */ if (hpriv->cap & HOST_CAP_NCQ) diff --git a/drivers/ata/ahci.c b/drivers/ata/ahci.c index 4cd52a4541a9..a29f8012fb08 100644 --- a/drivers/ata/ahci.c +++ b/drivers/ata/ahci.c @@ -526,8 +526,7 @@ static void ahci_pci_save_initial_config(struct pci_dev *pdev, "Disabling your PATA port. Use the boot option 'ahci.marvell_enable=0' to avoid this.\n"); } - ahci_save_initial_config(&pdev->dev, hpriv, force_port_map, - mask_port_map); + ahci_save_initial_config(&pdev->dev, hpriv); } static int ahci_pci_reset_controller(struct ata_host *host) diff --git a/drivers/ata/ahci.h b/drivers/ata/ahci.h index 5513296e5e2e..59ae0ee00149 100644 --- a/drivers/ata/ahci.h +++ b/drivers/ata/ahci.h @@ -53,7 +53,7 @@ enum { AHCI_MAX_PORTS = 32, - AHCI_MAX_CLKS = 3, + AHCI_MAX_CLKS = 4, AHCI_MAX_SG = 168, /* hardware max is 64K */ AHCI_DMA_BOUNDARY = 0xffffffff, AHCI_MAX_CMDS = 32, @@ -316,8 +316,12 @@ struct ahci_port_priv { }; struct ahci_host_priv { - void __iomem * mmio; /* bus-independent mem map */ + /* Input fields */ unsigned int flags; /* AHCI_HFLAG_* */ + u32 force_port_map; /* force port map */ + u32 mask_port_map; /* mask out particular bits */ + + void __iomem * mmio; /* bus-independent mem map */ u32 cap; /* cap to use */ u32 cap2; /* cap2 to use */ u32 port_map; /* port map to use */ @@ -330,7 +334,12 @@ struct ahci_host_priv { bool got_runtime_pm; /* Did we do pm_runtime_get? */ struct clk *clks[AHCI_MAX_CLKS]; /* Optional */ struct regulator *target_pwr; /* Optional */ - struct phy *phy; /* If platform uses phy */ + /* + * If platform uses PHYs. There is a 1:1 relation between the port number and + * the PHY position in this array. + */ + struct phy **phys; + unsigned nports; /* Number of ports */ void *plat_data; /* Other platform data */ /* * Optional ahci_start_engine override, if not set this gets set to the @@ -361,9 +370,7 @@ unsigned int ahci_dev_classify(struct ata_port *ap); void ahci_fill_cmd_slot(struct ahci_port_priv *pp, unsigned int tag, u32 opts); void ahci_save_initial_config(struct device *dev, - struct ahci_host_priv *hpriv, - unsigned int force_port_map, - unsigned int mask_port_map); + struct ahci_host_priv *hpriv); void ahci_init_controller(struct ata_host *host); int ahci_reset_controller(struct ata_host *host); diff --git a/drivers/ata/ahci_da850.c b/drivers/ata/ahci_da850.c index 2b77d53bccf8..ad1e71ec10cf 100644 --- a/drivers/ata/ahci_da850.c +++ b/drivers/ata/ahci_da850.c @@ -85,8 +85,7 @@ static int ahci_da850_probe(struct platform_device *pdev) da850_sata_init(dev, pwrdn_reg, hpriv->mmio); - rc = ahci_platform_init_host(pdev, hpriv, &ahci_da850_port_info, - 0, 0, 0); + rc = ahci_platform_init_host(pdev, hpriv, &ahci_da850_port_info); if (rc) goto disable_resources; diff --git a/drivers/ata/ahci_imx.c b/drivers/ata/ahci_imx.c index cac4360f272a..f3970b4ed889 100644 --- a/drivers/ata/ahci_imx.c +++ b/drivers/ata/ahci_imx.c @@ -64,6 +64,7 @@ struct imx_ahci_priv { struct regmap *gpr; bool no_device; bool first_time; + u32 phy_params; }; static int ahci_imx_hotplug; @@ -248,14 +249,7 @@ static int imx_sata_enable(struct ahci_host_priv *hpriv) IMX6Q_GPR13_SATA_TX_LVL_MASK | IMX6Q_GPR13_SATA_MPLL_CLK_EN | IMX6Q_GPR13_SATA_TX_EDGE_RATE, - IMX6Q_GPR13_SATA_RX_EQ_VAL_3_0_DB | - IMX6Q_GPR13_SATA_RX_LOS_LVL_SATA2M | - IMX6Q_GPR13_SATA_RX_DPLL_MODE_2P_4F | - IMX6Q_GPR13_SATA_SPD_MODE_3P0G | - IMX6Q_GPR13_SATA_MPLL_SS_EN | - IMX6Q_GPR13_SATA_TX_ATTEN_9_16 | - IMX6Q_GPR13_SATA_TX_BOOST_3_33_DB | - IMX6Q_GPR13_SATA_TX_LVL_1_025_V); + imxpriv->phy_params); regmap_update_bits(imxpriv->gpr, IOMUXC_GPR13, IMX6Q_GPR13_SATA_MPLL_CLK_EN, IMX6Q_GPR13_SATA_MPLL_CLK_EN); @@ -265,7 +259,7 @@ static int imx_sata_enable(struct ahci_host_priv *hpriv) ret = imx_sata_phy_reset(hpriv); if (ret) { dev_err(dev, "failed to reset phy: %d\n", ret); - goto disable_regulator; + goto disable_clk; } } @@ -273,6 +267,8 @@ static int imx_sata_enable(struct ahci_host_priv *hpriv) return 0; +disable_clk: + clk_disable_unprepare(imxpriv->sata_ref_clk); disable_regulator: if (hpriv->target_pwr) regulator_disable(hpriv->target_pwr); @@ -369,6 +365,165 @@ static const struct of_device_id imx_ahci_of_match[] = { }; MODULE_DEVICE_TABLE(of, imx_ahci_of_match); +struct reg_value { + u32 of_value; + u32 reg_value; +}; + +struct reg_property { + const char *name; + const struct reg_value *values; + size_t num_values; + u32 def_value; + u32 set_value; +}; + +static const struct reg_value gpr13_tx_level[] = { + { 937, IMX6Q_GPR13_SATA_TX_LVL_0_937_V }, + { 947, IMX6Q_GPR13_SATA_TX_LVL_0_947_V }, + { 957, IMX6Q_GPR13_SATA_TX_LVL_0_957_V }, + { 966, IMX6Q_GPR13_SATA_TX_LVL_0_966_V }, + { 976, IMX6Q_GPR13_SATA_TX_LVL_0_976_V }, + { 986, IMX6Q_GPR13_SATA_TX_LVL_0_986_V }, + { 996, IMX6Q_GPR13_SATA_TX_LVL_0_996_V }, + { 1005, IMX6Q_GPR13_SATA_TX_LVL_1_005_V }, + { 1015, IMX6Q_GPR13_SATA_TX_LVL_1_015_V }, + { 1025, IMX6Q_GPR13_SATA_TX_LVL_1_025_V }, + { 1035, IMX6Q_GPR13_SATA_TX_LVL_1_035_V }, + { 1045, IMX6Q_GPR13_SATA_TX_LVL_1_045_V }, + { 1054, IMX6Q_GPR13_SATA_TX_LVL_1_054_V }, + { 1064, IMX6Q_GPR13_SATA_TX_LVL_1_064_V }, + { 1074, IMX6Q_GPR13_SATA_TX_LVL_1_074_V }, + { 1084, IMX6Q_GPR13_SATA_TX_LVL_1_084_V }, + { 1094, IMX6Q_GPR13_SATA_TX_LVL_1_094_V }, + { 1104, IMX6Q_GPR13_SATA_TX_LVL_1_104_V }, + { 1113, IMX6Q_GPR13_SATA_TX_LVL_1_113_V }, + { 1123, IMX6Q_GPR13_SATA_TX_LVL_1_123_V }, + { 1133, IMX6Q_GPR13_SATA_TX_LVL_1_133_V }, + { 1143, IMX6Q_GPR13_SATA_TX_LVL_1_143_V }, + { 1152, IMX6Q_GPR13_SATA_TX_LVL_1_152_V }, + { 1162, IMX6Q_GPR13_SATA_TX_LVL_1_162_V }, + { 1172, IMX6Q_GPR13_SATA_TX_LVL_1_172_V }, + { 1182, IMX6Q_GPR13_SATA_TX_LVL_1_182_V }, + { 1191, IMX6Q_GPR13_SATA_TX_LVL_1_191_V }, + { 1201, IMX6Q_GPR13_SATA_TX_LVL_1_201_V }, + { 1211, IMX6Q_GPR13_SATA_TX_LVL_1_211_V }, + { 1221, IMX6Q_GPR13_SATA_TX_LVL_1_221_V }, + { 1230, IMX6Q_GPR13_SATA_TX_LVL_1_230_V }, + { 1240, IMX6Q_GPR13_SATA_TX_LVL_1_240_V } +}; + +static const struct reg_value gpr13_tx_boost[] = { + { 0, IMX6Q_GPR13_SATA_TX_BOOST_0_00_DB }, + { 370, IMX6Q_GPR13_SATA_TX_BOOST_0_37_DB }, + { 740, IMX6Q_GPR13_SATA_TX_BOOST_0_74_DB }, + { 1110, IMX6Q_GPR13_SATA_TX_BOOST_1_11_DB }, + { 1480, IMX6Q_GPR13_SATA_TX_BOOST_1_48_DB }, + { 1850, IMX6Q_GPR13_SATA_TX_BOOST_1_85_DB }, + { 2220, IMX6Q_GPR13_SATA_TX_BOOST_2_22_DB }, + { 2590, IMX6Q_GPR13_SATA_TX_BOOST_2_59_DB }, + { 2960, IMX6Q_GPR13_SATA_TX_BOOST_2_96_DB }, + { 3330, IMX6Q_GPR13_SATA_TX_BOOST_3_33_DB }, + { 3700, IMX6Q_GPR13_SATA_TX_BOOST_3_70_DB }, + { 4070, IMX6Q_GPR13_SATA_TX_BOOST_4_07_DB }, + { 4440, IMX6Q_GPR13_SATA_TX_BOOST_4_44_DB }, + { 4810, IMX6Q_GPR13_SATA_TX_BOOST_4_81_DB }, + { 5280, IMX6Q_GPR13_SATA_TX_BOOST_5_28_DB }, + { 5750, IMX6Q_GPR13_SATA_TX_BOOST_5_75_DB } +}; + +static const struct reg_value gpr13_tx_atten[] = { + { 8, IMX6Q_GPR13_SATA_TX_ATTEN_8_16 }, + { 9, IMX6Q_GPR13_SATA_TX_ATTEN_9_16 }, + { 10, IMX6Q_GPR13_SATA_TX_ATTEN_10_16 }, + { 12, IMX6Q_GPR13_SATA_TX_ATTEN_12_16 }, + { 14, IMX6Q_GPR13_SATA_TX_ATTEN_14_16 }, + { 16, IMX6Q_GPR13_SATA_TX_ATTEN_16_16 }, +}; + +static const struct reg_value gpr13_rx_eq[] = { + { 500, IMX6Q_GPR13_SATA_RX_EQ_VAL_0_5_DB }, + { 1000, IMX6Q_GPR13_SATA_RX_EQ_VAL_1_0_DB }, + { 1500, IMX6Q_GPR13_SATA_RX_EQ_VAL_1_5_DB }, + { 2000, IMX6Q_GPR13_SATA_RX_EQ_VAL_2_0_DB }, + { 2500, IMX6Q_GPR13_SATA_RX_EQ_VAL_2_5_DB }, + { 3000, IMX6Q_GPR13_SATA_RX_EQ_VAL_3_0_DB }, + { 3500, IMX6Q_GPR13_SATA_RX_EQ_VAL_3_5_DB }, + { 4000, IMX6Q_GPR13_SATA_RX_EQ_VAL_4_0_DB }, +}; + +static const struct reg_property gpr13_props[] = { + { + .name = "fsl,transmit-level-mV", + .values = gpr13_tx_level, + .num_values = ARRAY_SIZE(gpr13_tx_level), + .def_value = IMX6Q_GPR13_SATA_TX_LVL_1_025_V, + }, { + .name = "fsl,transmit-boost-mdB", + .values = gpr13_tx_boost, + .num_values = ARRAY_SIZE(gpr13_tx_boost), + .def_value = IMX6Q_GPR13_SATA_TX_BOOST_3_33_DB, + }, { + .name = "fsl,transmit-atten-16ths", + .values = gpr13_tx_atten, + .num_values = ARRAY_SIZE(gpr13_tx_atten), + .def_value = IMX6Q_GPR13_SATA_TX_ATTEN_9_16, + }, { + .name = "fsl,receive-eq-mdB", + .values = gpr13_rx_eq, + .num_values = ARRAY_SIZE(gpr13_rx_eq), + .def_value = IMX6Q_GPR13_SATA_RX_EQ_VAL_3_0_DB, + }, { + .name = "fsl,no-spread-spectrum", + .def_value = IMX6Q_GPR13_SATA_MPLL_SS_EN, + .set_value = 0, + }, +}; + +static u32 imx_ahci_parse_props(struct device *dev, + const struct reg_property *prop, size_t num) +{ + struct device_node *np = dev->of_node; + u32 reg_value = 0; + int i, j; + + for (i = 0; i < num; i++, prop++) { + u32 of_val; + + if (prop->num_values == 0) { + if (of_property_read_bool(np, prop->name)) + reg_value |= prop->set_value; + else + reg_value |= prop->def_value; + continue; + } + + if (of_property_read_u32(np, prop->name, &of_val)) { + dev_info(dev, "%s not specified, using %08x\n", + prop->name, prop->def_value); + reg_value |= prop->def_value; + continue; + } + + for (j = 0; j < prop->num_values; j++) { + if (prop->values[j].of_value == of_val) { + dev_info(dev, "%s value %u, using %08x\n", + prop->name, of_val, prop->values[j].reg_value); + reg_value |= prop->values[j].reg_value; + break; + } + } + + if (j == prop->num_values) { + dev_err(dev, "DT property %s is not a valid value\n", + prop->name); + reg_value |= prop->def_value; + } + } + + return reg_value; +} + static int imx_ahci_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -410,6 +565,8 @@ static int imx_ahci_probe(struct platform_device *pdev) } if (imxpriv->type == AHCI_IMX6Q) { + u32 reg_value; + imxpriv->gpr = syscon_regmap_lookup_by_compatible( "fsl,imx6q-iomuxc-gpr"); if (IS_ERR(imxpriv->gpr)) { @@ -417,6 +574,15 @@ static int imx_ahci_probe(struct platform_device *pdev) "failed to find fsl,imx6q-iomux-gpr regmap\n"); return PTR_ERR(imxpriv->gpr); } + + reg_value = imx_ahci_parse_props(dev, gpr13_props, + ARRAY_SIZE(gpr13_props)); + + imxpriv->phy_params = + IMX6Q_GPR13_SATA_RX_LOS_LVL_SATA2M | + IMX6Q_GPR13_SATA_RX_DPLL_MODE_2P_4F | + IMX6Q_GPR13_SATA_SPD_MODE_3P0G | + reg_value; } hpriv = ahci_platform_get_resources(pdev); @@ -454,8 +620,7 @@ static int imx_ahci_probe(struct platform_device *pdev) reg_val = clk_get_rate(imxpriv->ahb_clk) / 1000; writel(reg_val, hpriv->mmio + IMX_TIMER1MS); - ret = ahci_platform_init_host(pdev, hpriv, &ahci_imx_port_info, - 0, 0, 0); + ret = ahci_platform_init_host(pdev, hpriv, &ahci_imx_port_info); if (ret) goto disable_sata; diff --git a/drivers/ata/ahci_mvebu.c b/drivers/ata/ahci_mvebu.c index fd3dfd733b84..68672d2692ee 100644 --- a/drivers/ata/ahci_mvebu.c +++ b/drivers/ata/ahci_mvebu.c @@ -88,8 +88,7 @@ static int ahci_mvebu_probe(struct platform_device *pdev) ahci_mvebu_mbus_config(hpriv, dram); ahci_mvebu_regret_option(hpriv); - rc = ahci_platform_init_host(pdev, hpriv, &ahci_mvebu_port_info, - 0, 0, 0); + rc = ahci_platform_init_host(pdev, hpriv, &ahci_mvebu_port_info); if (rc) goto disable_resources; diff --git a/drivers/ata/ahci_platform.c b/drivers/ata/ahci_platform.c index b10d81ddb528..f61ddb9146d6 100644 --- a/drivers/ata/ahci_platform.c +++ b/drivers/ata/ahci_platform.c @@ -34,7 +34,6 @@ static int ahci_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; struct ahci_platform_data *pdata = dev_get_platdata(dev); struct ahci_host_priv *hpriv; - unsigned long hflags = 0; int rc; hpriv = ahci_platform_get_resources(pdev); @@ -58,10 +57,9 @@ static int ahci_probe(struct platform_device *pdev) } if (of_device_is_compatible(dev->of_node, "hisilicon,hisi-ahci")) - hflags |= AHCI_HFLAG_NO_FBS | AHCI_HFLAG_NO_NCQ; + hpriv->flags |= AHCI_HFLAG_NO_FBS | AHCI_HFLAG_NO_NCQ; - rc = ahci_platform_init_host(pdev, hpriv, &ahci_port_info, - hflags, 0, 0); + rc = ahci_platform_init_host(pdev, hpriv, &ahci_port_info); if (rc) goto pdata_exit; @@ -78,6 +76,8 @@ static SIMPLE_DEV_PM_OPS(ahci_pm_ops, ahci_platform_suspend, ahci_platform_resume); static const struct of_device_id ahci_of_match[] = { + { .compatible = "generic-ahci", }, + /* Keep the following compatibles for device tree compatibility */ { .compatible = "snps,spear-ahci", }, { .compatible = "snps,exynos5440-ahci", }, { .compatible = "ibm,476gtr-ahci", }, diff --git a/drivers/ata/ahci_st.c b/drivers/ata/ahci_st.c index 2595598df9ce..835d6eea84fd 100644 --- a/drivers/ata/ahci_st.c +++ b/drivers/ata/ahci_st.c @@ -166,7 +166,7 @@ static int st_ahci_probe(struct platform_device *pdev) if (err) return err; - err = ahci_platform_init_host(pdev, hpriv, &st_ahci_port_info, 0, 0, 0); + err = ahci_platform_init_host(pdev, hpriv, &st_ahci_port_info); if (err) { ahci_platform_disable_resources(hpriv); return err; @@ -221,7 +221,7 @@ static int st_ahci_resume(struct device *dev) static SIMPLE_DEV_PM_OPS(st_ahci_pm_ops, st_ahci_suspend, st_ahci_resume); -static struct of_device_id st_ahci_match[] = { +static const struct of_device_id st_ahci_match[] = { { .compatible = "st,ahci", }, {}, }; diff --git a/drivers/ata/ahci_sunxi.c b/drivers/ata/ahci_sunxi.c index 02002f125bd4..e44d675a30ec 100644 --- a/drivers/ata/ahci_sunxi.c +++ b/drivers/ata/ahci_sunxi.c @@ -167,7 +167,6 @@ static int ahci_sunxi_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct ahci_host_priv *hpriv; - unsigned long hflags; int rc; hpriv = ahci_platform_get_resources(pdev); @@ -184,11 +183,10 @@ static int ahci_sunxi_probe(struct platform_device *pdev) if (rc) goto disable_resources; - hflags = AHCI_HFLAG_32BIT_ONLY | AHCI_HFLAG_NO_MSI | - AHCI_HFLAG_NO_PMP | AHCI_HFLAG_YES_NCQ; + hpriv->flags = AHCI_HFLAG_32BIT_ONLY | AHCI_HFLAG_NO_MSI | + AHCI_HFLAG_NO_PMP | AHCI_HFLAG_YES_NCQ; - rc = ahci_platform_init_host(pdev, hpriv, &ahci_sunxi_port_info, - hflags, 0, 0); + rc = ahci_platform_init_host(pdev, hpriv, &ahci_sunxi_port_info); if (rc) goto disable_resources; diff --git a/drivers/ata/ahci_tegra.c b/drivers/ata/ahci_tegra.c new file mode 100644 index 000000000000..fc3df47fca35 --- /dev/null +++ b/drivers/ata/ahci_tegra.c @@ -0,0 +1,376 @@ +/* + * drivers/ata/ahci_tegra.c + * + * Copyright (c) 2014, NVIDIA CORPORATION. All rights reserved. + * + * Author: + * Mikko Perttunen <mperttunen@nvidia.com> + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#include <linux/ahci_platform.h> +#include <linux/reset.h> +#include <linux/errno.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/of_device.h> +#include <linux/platform_device.h> +#include <linux/tegra-powergate.h> +#include <linux/regulator/consumer.h> +#include "ahci.h" + +#define SATA_CONFIGURATION_0 0x180 +#define SATA_CONFIGURATION_EN_FPCI BIT(0) + +#define SCFG_OFFSET 0x1000 + +#define T_SATA0_CFG_1 0x04 +#define T_SATA0_CFG_1_IO_SPACE BIT(0) +#define T_SATA0_CFG_1_MEMORY_SPACE BIT(1) +#define T_SATA0_CFG_1_BUS_MASTER BIT(2) +#define T_SATA0_CFG_1_SERR BIT(8) + +#define T_SATA0_CFG_9 0x24 +#define T_SATA0_CFG_9_BASE_ADDRESS_SHIFT 13 + +#define SATA_FPCI_BAR5 0x94 +#define SATA_FPCI_BAR5_START_SHIFT 4 + +#define SATA_INTR_MASK 0x188 +#define SATA_INTR_MASK_IP_INT_MASK BIT(16) + +#define T_SATA0_AHCI_HBA_CAP_BKDR 0x300 + +#define T_SATA0_BKDOOR_CC 0x4a4 + +#define T_SATA0_CFG_SATA 0x54c +#define T_SATA0_CFG_SATA_BACKDOOR_PROG_IF_EN BIT(12) + +#define T_SATA0_CFG_MISC 0x550 + +#define T_SATA0_INDEX 0x680 + +#define T_SATA0_CHX_PHY_CTRL1_GEN1 0x690 +#define T_SATA0_CHX_PHY_CTRL1_GEN1_TX_AMP_MASK 0xff +#define T_SATA0_CHX_PHY_CTRL1_GEN1_TX_AMP_SHIFT 0 +#define T_SATA0_CHX_PHY_CTRL1_GEN1_TX_PEAK_MASK (0xff << 8) +#define T_SATA0_CHX_PHY_CTRL1_GEN1_TX_PEAK_SHIFT 8 + +#define T_SATA0_CHX_PHY_CTRL1_GEN2 0x694 +#define T_SATA0_CHX_PHY_CTRL1_GEN2_TX_AMP_MASK 0xff +#define T_SATA0_CHX_PHY_CTRL1_GEN2_TX_AMP_SHIFT 0 +#define T_SATA0_CHX_PHY_CTRL1_GEN2_TX_PEAK_MASK (0xff << 12) +#define T_SATA0_CHX_PHY_CTRL1_GEN2_TX_PEAK_SHIFT 12 + +#define T_SATA0_CHX_PHY_CTRL2 0x69c +#define T_SATA0_CHX_PHY_CTRL2_CDR_CNTL_GEN1 0x23 + +#define T_SATA0_CHX_PHY_CTRL11 0x6d0 +#define T_SATA0_CHX_PHY_CTRL11_GEN2_RX_EQ (0x2800 << 16) + +#define FUSE_SATA_CALIB 0x124 +#define FUSE_SATA_CALIB_MASK 0x3 + +struct sata_pad_calibration { + u8 gen1_tx_amp; + u8 gen1_tx_peak; + u8 gen2_tx_amp; + u8 gen2_tx_peak; +}; + +static const struct sata_pad_calibration tegra124_pad_calibration[] = { + {0x18, 0x04, 0x18, 0x0a}, + {0x0e, 0x04, 0x14, 0x0a}, + {0x0e, 0x07, 0x1a, 0x0e}, + {0x14, 0x0e, 0x1a, 0x0e}, +}; + +struct tegra_ahci_priv { + struct platform_device *pdev; + void __iomem *sata_regs; + struct reset_control *sata_rst; + struct reset_control *sata_oob_rst; + struct reset_control *sata_cold_rst; + /* Needs special handling, cannot use ahci_platform */ + struct clk *sata_clk; + struct regulator_bulk_data supplies[5]; +}; + +static int tegra_ahci_power_on(struct ahci_host_priv *hpriv) +{ + struct tegra_ahci_priv *tegra = hpriv->plat_data; + int ret; + + ret = regulator_bulk_enable(ARRAY_SIZE(tegra->supplies), + tegra->supplies); + if (ret) + return ret; + + ret = tegra_powergate_sequence_power_up(TEGRA_POWERGATE_SATA, + tegra->sata_clk, + tegra->sata_rst); + if (ret) + goto disable_regulators; + + reset_control_assert(tegra->sata_oob_rst); + reset_control_assert(tegra->sata_cold_rst); + + ret = ahci_platform_enable_resources(hpriv); + if (ret) + goto disable_power; + + reset_control_deassert(tegra->sata_cold_rst); + reset_control_deassert(tegra->sata_oob_rst); + + return 0; + +disable_power: + clk_disable_unprepare(tegra->sata_clk); + + tegra_powergate_power_off(TEGRA_POWERGATE_SATA); + +disable_regulators: + regulator_bulk_disable(ARRAY_SIZE(tegra->supplies), tegra->supplies); + + return ret; +} + +static void tegra_ahci_power_off(struct ahci_host_priv *hpriv) +{ + struct tegra_ahci_priv *tegra = hpriv->plat_data; + + ahci_platform_disable_resources(hpriv); + + reset_control_assert(tegra->sata_rst); + reset_control_assert(tegra->sata_oob_rst); + reset_control_assert(tegra->sata_cold_rst); + + clk_disable_unprepare(tegra->sata_clk); + tegra_powergate_power_off(TEGRA_POWERGATE_SATA); + + regulator_bulk_disable(ARRAY_SIZE(tegra->supplies), tegra->supplies); +} + +static int tegra_ahci_controller_init(struct ahci_host_priv *hpriv) +{ + struct tegra_ahci_priv *tegra = hpriv->plat_data; + int ret; + unsigned int val; + struct sata_pad_calibration calib; + + ret = tegra_ahci_power_on(hpriv); + if (ret) { + dev_err(&tegra->pdev->dev, + "failed to power on AHCI controller: %d\n", ret); + return ret; + } + + val = readl(tegra->sata_regs + SATA_CONFIGURATION_0); + val |= SATA_CONFIGURATION_EN_FPCI; + writel(val, tegra->sata_regs + SATA_CONFIGURATION_0); + + /* Pad calibration */ + + /* FIXME Always use calibration 0. Change this to read the calibration + * fuse once the fuse driver has landed. */ + val = 0; + + calib = tegra124_pad_calibration[val & FUSE_SATA_CALIB_MASK]; + + writel(BIT(0), tegra->sata_regs + SCFG_OFFSET + T_SATA0_INDEX); + + val = readl(tegra->sata_regs + + SCFG_OFFSET + T_SATA0_CHX_PHY_CTRL1_GEN1); + val &= ~T_SATA0_CHX_PHY_CTRL1_GEN1_TX_AMP_MASK; + val &= ~T_SATA0_CHX_PHY_CTRL1_GEN1_TX_PEAK_MASK; + val |= calib.gen1_tx_amp << + T_SATA0_CHX_PHY_CTRL1_GEN1_TX_AMP_SHIFT; + val |= calib.gen1_tx_peak << + T_SATA0_CHX_PHY_CTRL1_GEN1_TX_PEAK_SHIFT; + writel(val, tegra->sata_regs + SCFG_OFFSET + + T_SATA0_CHX_PHY_CTRL1_GEN1); + + val = readl(tegra->sata_regs + + SCFG_OFFSET + T_SATA0_CHX_PHY_CTRL1_GEN2); + val &= ~T_SATA0_CHX_PHY_CTRL1_GEN2_TX_AMP_MASK; + val &= ~T_SATA0_CHX_PHY_CTRL1_GEN2_TX_PEAK_MASK; + val |= calib.gen2_tx_amp << + T_SATA0_CHX_PHY_CTRL1_GEN1_TX_AMP_SHIFT; + val |= calib.gen2_tx_peak << + T_SATA0_CHX_PHY_CTRL1_GEN1_TX_PEAK_SHIFT; + writel(val, tegra->sata_regs + SCFG_OFFSET + + T_SATA0_CHX_PHY_CTRL1_GEN2); + + writel(T_SATA0_CHX_PHY_CTRL11_GEN2_RX_EQ, + tegra->sata_regs + SCFG_OFFSET + T_SATA0_CHX_PHY_CTRL11); + writel(T_SATA0_CHX_PHY_CTRL2_CDR_CNTL_GEN1, + tegra->sata_regs + SCFG_OFFSET + T_SATA0_CHX_PHY_CTRL2); + + writel(0, tegra->sata_regs + SCFG_OFFSET + T_SATA0_INDEX); + + /* Program controller device ID */ + + val = readl(tegra->sata_regs + SCFG_OFFSET + T_SATA0_CFG_SATA); + val |= T_SATA0_CFG_SATA_BACKDOOR_PROG_IF_EN; + writel(val, tegra->sata_regs + SCFG_OFFSET + T_SATA0_CFG_SATA); + + writel(0x01060100, tegra->sata_regs + SCFG_OFFSET + T_SATA0_BKDOOR_CC); + + val = readl(tegra->sata_regs + SCFG_OFFSET + T_SATA0_CFG_SATA); + val &= ~T_SATA0_CFG_SATA_BACKDOOR_PROG_IF_EN; + writel(val, tegra->sata_regs + SCFG_OFFSET + T_SATA0_CFG_SATA); + + /* Enable IO & memory access, bus master mode */ + + val = readl(tegra->sata_regs + SCFG_OFFSET + T_SATA0_CFG_1); + val |= T_SATA0_CFG_1_IO_SPACE | T_SATA0_CFG_1_MEMORY_SPACE | + T_SATA0_CFG_1_BUS_MASTER | T_SATA0_CFG_1_SERR; + writel(val, tegra->sata_regs + SCFG_OFFSET + T_SATA0_CFG_1); + + /* Program SATA MMIO */ + + writel(0x10000 << SATA_FPCI_BAR5_START_SHIFT, + tegra->sata_regs + SATA_FPCI_BAR5); + + writel(0x08000 << T_SATA0_CFG_9_BASE_ADDRESS_SHIFT, + tegra->sata_regs + SCFG_OFFSET + T_SATA0_CFG_9); + + /* Unmask SATA interrupts */ + + val = readl(tegra->sata_regs + SATA_INTR_MASK); + val |= SATA_INTR_MASK_IP_INT_MASK; + writel(val, tegra->sata_regs + SATA_INTR_MASK); + + return 0; +} + +static void tegra_ahci_controller_deinit(struct ahci_host_priv *hpriv) +{ + tegra_ahci_power_off(hpriv); +} + +static void tegra_ahci_host_stop(struct ata_host *host) +{ + struct ahci_host_priv *hpriv = host->private_data; + + tegra_ahci_controller_deinit(hpriv); +} + +static struct ata_port_operations ahci_tegra_port_ops = { + .inherits = &ahci_ops, + .host_stop = tegra_ahci_host_stop, +}; + +static const struct ata_port_info ahci_tegra_port_info = { + .flags = AHCI_FLAG_COMMON, + .pio_mask = ATA_PIO4, + .udma_mask = ATA_UDMA6, + .port_ops = &ahci_tegra_port_ops, +}; + +static const struct of_device_id tegra_ahci_of_match[] = { + { .compatible = "nvidia,tegra124-ahci" }, + {} +}; +MODULE_DEVICE_TABLE(of, tegra_ahci_of_match); + +static int tegra_ahci_probe(struct platform_device *pdev) +{ + struct ahci_host_priv *hpriv; + struct tegra_ahci_priv *tegra; + struct resource *res; + int ret; + + hpriv = ahci_platform_get_resources(pdev); + if (IS_ERR(hpriv)) + return PTR_ERR(hpriv); + + tegra = devm_kzalloc(&pdev->dev, sizeof(*tegra), GFP_KERNEL); + if (!tegra) + return -ENOMEM; + + hpriv->plat_data = tegra; + + tegra->pdev = pdev; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 1); + tegra->sata_regs = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(tegra->sata_regs)) + return PTR_ERR(tegra->sata_regs); + + tegra->sata_rst = devm_reset_control_get(&pdev->dev, "sata"); + if (IS_ERR(tegra->sata_rst)) { + dev_err(&pdev->dev, "Failed to get sata reset\n"); + return PTR_ERR(tegra->sata_rst); + } + + tegra->sata_oob_rst = devm_reset_control_get(&pdev->dev, "sata-oob"); + if (IS_ERR(tegra->sata_oob_rst)) { + dev_err(&pdev->dev, "Failed to get sata-oob reset\n"); + return PTR_ERR(tegra->sata_oob_rst); + } + + tegra->sata_cold_rst = devm_reset_control_get(&pdev->dev, "sata-cold"); + if (IS_ERR(tegra->sata_cold_rst)) { + dev_err(&pdev->dev, "Failed to get sata-cold reset\n"); + return PTR_ERR(tegra->sata_cold_rst); + } + + tegra->sata_clk = devm_clk_get(&pdev->dev, "sata"); + if (IS_ERR(tegra->sata_clk)) { + dev_err(&pdev->dev, "Failed to get sata clock\n"); + return PTR_ERR(tegra->sata_clk); + } + + tegra->supplies[0].supply = "avdd"; + tegra->supplies[1].supply = "hvdd"; + tegra->supplies[2].supply = "vddio"; + tegra->supplies[3].supply = "target-5v"; + tegra->supplies[4].supply = "target-12v"; + + ret = devm_regulator_bulk_get(&pdev->dev, ARRAY_SIZE(tegra->supplies), + tegra->supplies); + if (ret) { + dev_err(&pdev->dev, "Failed to get regulators\n"); + return ret; + } + + ret = tegra_ahci_controller_init(hpriv); + if (ret) + return ret; + + ret = ahci_platform_init_host(pdev, hpriv, &ahci_tegra_port_info); + if (ret) + goto deinit_controller; + + return 0; + +deinit_controller: + tegra_ahci_controller_deinit(hpriv); + + return ret; +}; + +static struct platform_driver tegra_ahci_driver = { + .probe = tegra_ahci_probe, + .remove = ata_platform_remove_one, + .driver = { + .name = "tegra-ahci", + .of_match_table = tegra_ahci_of_match, + }, + /* LP0 suspend support not implemented */ +}; +module_platform_driver(tegra_ahci_driver); + +MODULE_AUTHOR("Mikko Perttunen <mperttunen@nvidia.com>"); +MODULE_DESCRIPTION("Tegra124 AHCI SATA driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/ata/ahci_xgene.c b/drivers/ata/ahci_xgene.c index ee3a3659bd9e..bc281115490b 100644 --- a/drivers/ata/ahci_xgene.c +++ b/drivers/ata/ahci_xgene.c @@ -67,6 +67,9 @@ #define PORTAXICFG 0x000000bc #define PORTAXICFG_OUTTRANS_SET(dst, src) \ (((dst) & ~0x00f00000) | (((u32)(src) << 0x14) & 0x00f00000)) +#define PORTRANSCFG 0x000000c8 +#define PORTRANSCFG_RXWM_SET(dst, src) \ + (((dst) & ~0x0000007f) | (((u32)(src)) & 0x0000007f)) /* SATA host controller AXI CSR */ #define INT_SLV_TMOMASK 0x00000010 @@ -193,11 +196,11 @@ static void xgene_ahci_set_phy_cfg(struct xgene_ahci_context *ctx, int channel) /* Disable fix rate */ writel(0x0001fffe, mmio + PORTPHY1CFG); readl(mmio + PORTPHY1CFG); /* Force a barrier */ - writel(0x5018461c, mmio + PORTPHY2CFG); + writel(0x28183219, mmio + PORTPHY2CFG); readl(mmio + PORTPHY2CFG); /* Force a barrier */ - writel(0x1c081907, mmio + PORTPHY3CFG); + writel(0x13081008, mmio + PORTPHY3CFG); readl(mmio + PORTPHY3CFG); /* Force a barrier */ - writel(0x1c080815, mmio + PORTPHY4CFG); + writel(0x00480815, mmio + PORTPHY4CFG); readl(mmio + PORTPHY4CFG); /* Force a barrier */ /* Set window negotiation */ val = readl(mmio + PORTPHY5CFG); @@ -209,6 +212,10 @@ static void xgene_ahci_set_phy_cfg(struct xgene_ahci_context *ctx, int channel) val = PORTAXICFG_OUTTRANS_SET(val, 0xe); /* Set outstanding */ writel(val, mmio + PORTAXICFG); readl(mmio + PORTAXICFG); /* Force a barrier */ + /* Set the watermark threshold of the receive FIFO */ + val = readl(mmio + PORTRANSCFG); + val = PORTRANSCFG_RXWM_SET(val, 0x30); + writel(val, mmio + PORTRANSCFG); } /** @@ -415,7 +422,6 @@ static int xgene_ahci_probe(struct platform_device *pdev) struct ahci_host_priv *hpriv; struct xgene_ahci_context *ctx; struct resource *res; - unsigned long hflags; int rc; hpriv = ahci_platform_get_resources(pdev); @@ -474,20 +480,9 @@ static int xgene_ahci_probe(struct platform_device *pdev) /* Configure the host controller */ xgene_ahci_hw_init(hpriv); - /* - * Setup DMA mask. This is preliminary until the DMA range is sorted - * out. - */ - rc = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); - if (rc) { - dev_err(dev, "Unable to set dma mask\n"); - goto disable_resources; - } - - hflags = AHCI_HFLAG_NO_PMP | AHCI_HFLAG_YES_NCQ; + hpriv->flags = AHCI_HFLAG_NO_PMP | AHCI_HFLAG_YES_NCQ; - rc = ahci_platform_init_host(pdev, hpriv, &xgene_ahci_port_info, - hflags, 0, 0); + rc = ahci_platform_init_host(pdev, hpriv, &xgene_ahci_port_info); if (rc) goto disable_resources; diff --git a/drivers/ata/libahci.c b/drivers/ata/libahci.c index d72ce0470309..b784e9de426a 100644 --- a/drivers/ata/libahci.c +++ b/drivers/ata/libahci.c @@ -382,8 +382,6 @@ static ssize_t ahci_show_em_supported(struct device *dev, * ahci_save_initial_config - Save and fixup initial config values * @dev: target AHCI device * @hpriv: host private area to store config values - * @force_port_map: force port map to a specified value - * @mask_port_map: mask out particular bits from port map * * Some registers containing configuration info might be setup by * BIOS and might be cleared on reset. This function saves the @@ -398,10 +396,7 @@ static ssize_t ahci_show_em_supported(struct device *dev, * LOCKING: * None. */ -void ahci_save_initial_config(struct device *dev, - struct ahci_host_priv *hpriv, - unsigned int force_port_map, - unsigned int mask_port_map) +void ahci_save_initial_config(struct device *dev, struct ahci_host_priv *hpriv) { void __iomem *mmio = hpriv->mmio; u32 cap, cap2, vers, port_map; @@ -468,17 +463,17 @@ void ahci_save_initial_config(struct device *dev, cap &= ~HOST_CAP_FBS; } - if (force_port_map && port_map != force_port_map) { + if (hpriv->force_port_map && port_map != hpriv->force_port_map) { dev_info(dev, "forcing port_map 0x%x -> 0x%x\n", - port_map, force_port_map); - port_map = force_port_map; + port_map, hpriv->force_port_map); + port_map = hpriv->force_port_map; } - if (mask_port_map) { + if (hpriv->mask_port_map) { dev_warn(dev, "masking port_map 0x%x -> 0x%x\n", port_map, - port_map & mask_port_map); - port_map &= mask_port_map; + port_map & hpriv->mask_port_map); + port_map &= hpriv->mask_port_map; } /* cross check port_map and cap.n_ports */ diff --git a/drivers/ata/libahci_platform.c b/drivers/ata/libahci_platform.c index b0077589f065..5b92c290e6c6 100644 --- a/drivers/ata/libahci_platform.c +++ b/drivers/ata/libahci_platform.c @@ -39,6 +39,67 @@ static struct scsi_host_template ahci_platform_sht = { }; /** + * ahci_platform_enable_phys - Enable PHYs + * @hpriv: host private area to store config values + * + * This function enables all the PHYs found in hpriv->phys, if any. + * If a PHY fails to be enabled, it disables all the PHYs already + * enabled in reverse order and returns an error. + * + * RETURNS: + * 0 on success otherwise a negative error code + */ +int ahci_platform_enable_phys(struct ahci_host_priv *hpriv) +{ + int rc, i; + + for (i = 0; i < hpriv->nports; i++) { + if (!hpriv->phys[i]) + continue; + + rc = phy_init(hpriv->phys[i]); + if (rc) + goto disable_phys; + + rc = phy_power_on(hpriv->phys[i]); + if (rc) { + phy_exit(hpriv->phys[i]); + goto disable_phys; + } + } + + return 0; + +disable_phys: + while (--i >= 0) { + phy_power_off(hpriv->phys[i]); + phy_exit(hpriv->phys[i]); + } + return rc; +} +EXPORT_SYMBOL_GPL(ahci_platform_enable_phys); + +/** + * ahci_platform_disable_phys - Disable PHYs + * @hpriv: host private area to store config values + * + * This function disables all PHYs found in hpriv->phys. + */ +void ahci_platform_disable_phys(struct ahci_host_priv *hpriv) +{ + int i; + + for (i = 0; i < hpriv->nports; i++) { + if (!hpriv->phys[i]) + continue; + + phy_power_off(hpriv->phys[i]); + phy_exit(hpriv->phys[i]); + } +} +EXPORT_SYMBOL_GPL(ahci_platform_disable_phys); + +/** * ahci_platform_enable_clks - Enable platform clocks * @hpriv: host private area to store config values * @@ -92,7 +153,7 @@ EXPORT_SYMBOL_GPL(ahci_platform_disable_clks); * following order: * 1) Regulator * 2) Clocks (through ahci_platform_enable_clks) - * 3) Phy + * 3) Phys * * If resource enabling fails at any point the previous enabled resources * are disabled in reverse order. @@ -114,17 +175,9 @@ int ahci_platform_enable_resources(struct ahci_host_priv *hpriv) if (rc) goto disable_regulator; - if (hpriv->phy) { - rc = phy_init(hpriv->phy); - if (rc) - goto disable_clks; - - rc = phy_power_on(hpriv->phy); - if (rc) { - phy_exit(hpriv->phy); - goto disable_clks; - } - } + rc = ahci_platform_enable_phys(hpriv); + if (rc) + goto disable_clks; return 0; @@ -144,16 +197,13 @@ EXPORT_SYMBOL_GPL(ahci_platform_enable_resources); * * This function disables all ahci_platform managed resources in the * following order: - * 1) Phy + * 1) Phys * 2) Clocks (through ahci_platform_disable_clks) * 3) Regulator */ void ahci_platform_disable_resources(struct ahci_host_priv *hpriv) { - if (hpriv->phy) { - phy_power_off(hpriv->phy); - phy_exit(hpriv->phy); - } + ahci_platform_disable_phys(hpriv); ahci_platform_disable_clks(hpriv); @@ -187,7 +237,7 @@ static void ahci_platform_put_resources(struct device *dev, void *res) * 2) regulator for controlling the targets power (optional) * 3) 0 - AHCI_MAX_CLKS clocks, as specified in the devs devicetree node, * or for non devicetree enabled platforms a single clock - * 4) phy (optional) + * 4) phys (optional) * * RETURNS: * The allocated ahci_host_priv on success, otherwise an ERR_PTR value @@ -197,7 +247,9 @@ struct ahci_host_priv *ahci_platform_get_resources(struct platform_device *pdev) struct device *dev = &pdev->dev; struct ahci_host_priv *hpriv; struct clk *clk; - int i, rc = -ENOMEM; + struct device_node *child; + int i, enabled_ports = 0, rc = -ENOMEM; + u32 mask_port_map = 0; if (!devres_open_group(dev, NULL, GFP_KERNEL)) return ERR_PTR(-ENOMEM); @@ -246,28 +298,89 @@ struct ahci_host_priv *ahci_platform_get_resources(struct platform_device *pdev) hpriv->clks[i] = clk; } - hpriv->phy = devm_phy_get(dev, "sata-phy"); - if (IS_ERR(hpriv->phy)) { - rc = PTR_ERR(hpriv->phy); - switch (rc) { - case -ENOSYS: - /* No PHY support. Check if PHY is required. */ - if (of_find_property(dev->of_node, "phys", NULL)) { - dev_err(dev, "couldn't get sata-phy: ENOSYS\n"); + hpriv->nports = of_get_child_count(dev->of_node); + + if (hpriv->nports) { + hpriv->phys = devm_kzalloc(dev, + hpriv->nports * sizeof(*hpriv->phys), + GFP_KERNEL); + if (!hpriv->phys) { + rc = -ENOMEM; + goto err_out; + } + + for_each_child_of_node(dev->of_node, child) { + u32 port; + + if (!of_device_is_available(child)) + continue; + + if (of_property_read_u32(child, "reg", &port)) { + rc = -EINVAL; goto err_out; } - case -ENODEV: - /* continue normally */ - hpriv->phy = NULL; - break; - case -EPROBE_DEFER: - goto err_out; + if (port >= hpriv->nports) { + dev_warn(dev, "invalid port number %d\n", port); + continue; + } + + mask_port_map |= BIT(port); - default: - dev_err(dev, "couldn't get sata-phy\n"); + hpriv->phys[port] = devm_of_phy_get(dev, child, NULL); + if (IS_ERR(hpriv->phys[port])) { + rc = PTR_ERR(hpriv->phys[port]); + dev_err(dev, + "couldn't get PHY in node %s: %d\n", + child->name, rc); + goto err_out; + } + + enabled_ports++; + } + if (!enabled_ports) { + dev_warn(dev, "No port enabled\n"); + rc = -ENODEV; goto err_out; } + + if (!hpriv->mask_port_map) + hpriv->mask_port_map = mask_port_map; + } else { + /* + * If no sub-node was found, keep this for device tree + * compatibility + */ + struct phy *phy = devm_phy_get(dev, "sata-phy"); + if (!IS_ERR(phy)) { + hpriv->phys = devm_kzalloc(dev, sizeof(*hpriv->phys), + GFP_KERNEL); + if (!hpriv->phys) { + rc = -ENOMEM; + goto err_out; + } + + hpriv->phys[0] = phy; + hpriv->nports = 1; + } else { + rc = PTR_ERR(phy); + switch (rc) { + case -ENOSYS: + /* No PHY support. Check if PHY is required. */ + if (of_find_property(dev->of_node, "phys", NULL)) { + dev_err(dev, "couldn't get sata-phy: ENOSYS\n"); + goto err_out; + } + case -ENODEV: + /* continue normally */ + hpriv->phys = NULL; + break; + + default: + goto err_out; + + } + } } pm_runtime_enable(dev); @@ -288,12 +401,9 @@ EXPORT_SYMBOL_GPL(ahci_platform_get_resources); * @pdev: platform device pointer for the host * @hpriv: ahci-host private data for the host * @pi_template: template for the ata_port_info to use - * @host_flags: ahci host flags used in ahci_host_priv - * @force_port_map: param passed to ahci_save_initial_config - * @mask_port_map: param passed to ahci_save_initial_config * * This function does all the usual steps needed to bring up an - * ahci-platform host, note any necessary resources (ie clks, phy, etc.) + * ahci-platform host, note any necessary resources (ie clks, phys, etc.) * must be initialized / enabled before calling this. * * RETURNS: @@ -301,10 +411,7 @@ EXPORT_SYMBOL_GPL(ahci_platform_get_resources); */ int ahci_platform_init_host(struct platform_device *pdev, struct ahci_host_priv *hpriv, - const struct ata_port_info *pi_template, - unsigned long host_flags, - unsigned int force_port_map, - unsigned int mask_port_map) + const struct ata_port_info *pi_template) { struct device *dev = &pdev->dev; struct ata_port_info pi = *pi_template; @@ -319,10 +426,9 @@ int ahci_platform_init_host(struct platform_device *pdev, } /* prepare host */ - pi.private_data = (void *)host_flags; - hpriv->flags |= host_flags; + pi.private_data = (void *)(unsigned long)hpriv->flags; - ahci_save_initial_config(dev, hpriv, force_port_map, mask_port_map); + ahci_save_initial_config(dev, hpriv); if (hpriv->cap & HOST_CAP_NCQ) pi.flags |= ATA_FLAG_NCQ; @@ -369,6 +475,19 @@ int ahci_platform_init_host(struct platform_device *pdev, ap->ops = &ata_dummy_port_ops; } + if (hpriv->cap & HOST_CAP_64) { + rc = dma_coerce_mask_and_coherent(dev, DMA_BIT_MASK(64)); + if (rc) { + rc = dma_coerce_mask_and_coherent(dev, + DMA_BIT_MASK(32)); + if (rc) { + dev_err(dev, "Failed to enable 64-bit DMA.\n"); + return rc; + } + dev_warn(dev, "Enable 32-bit DMA instead of 64-bit.\n"); + } + } + rc = ahci_reset_controller(host); if (rc) return rc; @@ -399,7 +518,7 @@ static void ahci_host_stop(struct ata_host *host) * @dev: device pointer for the host * * This function does all the usual steps needed to suspend an - * ahci-platform host, note any necessary resources (ie clks, phy, etc.) + * ahci-platform host, note any necessary resources (ie clks, phys, etc.) * must be disabled after calling this. * * RETURNS: @@ -436,7 +555,7 @@ EXPORT_SYMBOL_GPL(ahci_platform_suspend_host); * @dev: device pointer for the host * * This function does all the usual steps needed to resume an ahci-platform - * host, note any necessary resources (ie clks, phy, etc.) must be + * host, note any necessary resources (ie clks, phys, etc.) must be * initialized / enabled before calling this. * * RETURNS: diff --git a/drivers/ata/pata_samsung_cf.c b/drivers/ata/pata_samsung_cf.c index fb528831fb92..2578fc16960a 100644 --- a/drivers/ata/pata_samsung_cf.c +++ b/drivers/ata/pata_samsung_cf.c @@ -54,7 +54,6 @@ enum s3c_cpu_type { TYPE_S3C64XX, - TYPE_S5PC100, TYPE_S5PV210, }; @@ -476,10 +475,6 @@ static void pata_s3c_hwinit(struct s3c_ide_info *info, writel(0x1b, info->ide_addr + S3C_ATA_IRQ_MSK); break; - case TYPE_S5PC100: - pata_s3c_cfg_mode(info->sfr_addr); - /* FALLTHROUGH */ - case TYPE_S5PV210: /* Configure as little endian */ pata_s3c_set_endian(info->ide_addr, 0); @@ -549,11 +544,6 @@ static int __init pata_s3c_probe(struct platform_device *pdev) info->sfr_addr = info->ide_addr + 0x1800; info->ide_addr += 0x1900; info->fifo_status_reg = 0x94; - } else if (cpu_type == TYPE_S5PC100) { - ap->ops = &pata_s5p_port_ops; - info->sfr_addr = info->ide_addr + 0x1800; - info->ide_addr += 0x1900; - info->fifo_status_reg = 0x84; } else { ap->ops = &pata_s5p_port_ops; info->fifo_status_reg = 0x84; @@ -653,9 +643,6 @@ static struct platform_device_id pata_s3c_driver_ids[] = { .name = "s3c64xx-pata", .driver_data = TYPE_S3C64XX, }, { - .name = "s5pc100-pata", - .driver_data = TYPE_S5PC100, - }, { .name = "s5pv210-pata", .driver_data = TYPE_S5PV210, }, diff --git a/drivers/ata/sata_fsl.c b/drivers/ata/sata_fsl.c index 616a6d2ac20c..07bc7e4dbd04 100644 --- a/drivers/ata/sata_fsl.c +++ b/drivers/ata/sata_fsl.c @@ -734,13 +734,12 @@ static int sata_fsl_port_start(struct ata_port *ap) if (!pp) return -ENOMEM; - mem = dma_alloc_coherent(dev, SATA_FSL_PORT_PRIV_DMA_SZ, &mem_dma, - GFP_KERNEL); + mem = dma_zalloc_coherent(dev, SATA_FSL_PORT_PRIV_DMA_SZ, &mem_dma, + GFP_KERNEL); if (!mem) { kfree(pp); return -ENOMEM; } - memset(mem, 0, SATA_FSL_PORT_PRIV_DMA_SZ); pp->cmdslot = mem; pp->cmdslot_paddr = mem_dma; diff --git a/drivers/ata/sata_highbank.c b/drivers/ata/sata_highbank.c index 65965cf5af06..da3bc2709c63 100644 --- a/drivers/ata/sata_highbank.c +++ b/drivers/ata/sata_highbank.c @@ -512,7 +512,7 @@ static int ahci_highbank_probe(struct platform_device *pdev) return rc; - ahci_save_initial_config(dev, hpriv, 0, 0); + ahci_save_initial_config(dev, hpriv); /* prepare host */ if (hpriv->cap & HOST_CAP_NCQ) diff --git a/drivers/ata/sata_sil24.c b/drivers/ata/sata_sil24.c index 0534890f118a..d81b20ddb527 100644 --- a/drivers/ata/sata_sil24.c +++ b/drivers/ata/sata_sil24.c @@ -1154,8 +1154,8 @@ static irqreturn_t sil24_interrupt(int irq, void *dev_instance) status = readl(host_base + HOST_IRQ_STAT); if (status == 0xffffffff) { - printk(KERN_ERR DRV_NAME ": IRQ status == 0xffffffff, " - "PCI fault or device removal?\n"); + dev_err(host->dev, "IRQ status == 0xffffffff, " + "PCI fault or device removal?\n"); goto out; } diff --git a/drivers/base/platform.c b/drivers/base/platform.c index eee48c49f5de..00f2208949d1 100644 --- a/drivers/base/platform.c +++ b/drivers/base/platform.c @@ -23,6 +23,7 @@ #include <linux/pm_runtime.h> #include <linux/idr.h> #include <linux/acpi.h> +#include <linux/clk/clk-conf.h> #include "base.h" #include "power/power.h" @@ -499,6 +500,10 @@ static int platform_drv_probe(struct device *_dev) struct platform_device *dev = to_platform_device(_dev); int ret; + ret = of_clk_set_defaults(_dev->of_node, false); + if (ret < 0) + return ret; + acpi_dev_pm_attach(_dev, true); ret = drv->probe(dev); diff --git a/drivers/clk/Kconfig b/drivers/clk/Kconfig index 9f9c5ae5359b..cfd3af7b2cbd 100644 --- a/drivers/clk/Kconfig +++ b/drivers/clk/Kconfig @@ -102,6 +102,13 @@ config COMMON_CLK_KEYSTONE Supports clock drivers for Keystone based SOCs. These SOCs have local a power sleep control module that gate the clock to the IPs and PLLs. +config COMMON_CLK_PALMAS + tristate "Clock driver for TI Palmas devices" + depends on MFD_PALMAS + ---help--- + This driver supports TI Palmas devices 32KHz output KG and KG_AUDIO + using common clock framework. + source "drivers/clk/qcom/Kconfig" endmenu diff --git a/drivers/clk/Makefile b/drivers/clk/Makefile index 567f10259029..f537a0b1f798 100644 --- a/drivers/clk/Makefile +++ b/drivers/clk/Makefile @@ -9,12 +9,16 @@ obj-$(CONFIG_COMMON_CLK) += clk-gate.o obj-$(CONFIG_COMMON_CLK) += clk-mux.o obj-$(CONFIG_COMMON_CLK) += clk-composite.o obj-$(CONFIG_COMMON_CLK) += clk-fractional-divider.o +ifeq ($(CONFIG_OF), y) +obj-$(CONFIG_COMMON_CLK) += clk-conf.o +endif # hardware specific clock types # please keep this section sorted lexicographically by file/directory path name obj-$(CONFIG_COMMON_CLK_AXI_CLKGEN) += clk-axi-clkgen.o obj-$(CONFIG_ARCH_AXXIA) += clk-axm5516.o obj-$(CONFIG_ARCH_BCM2835) += clk-bcm2835.o +obj-$(CONFIG_ARCH_CLPS711X) += clk-clps711x.o obj-$(CONFIG_ARCH_EFM32) += clk-efm32gg.o obj-$(CONFIG_ARCH_HIGHBANK) += clk-highbank.o obj-$(CONFIG_MACH_LOONGSON1) += clk-ls1x.o @@ -22,6 +26,7 @@ obj-$(CONFIG_COMMON_CLK_MAX77686) += clk-max77686.o obj-$(CONFIG_ARCH_MOXART) += clk-moxart.o obj-$(CONFIG_ARCH_NOMADIK) += clk-nomadik.o obj-$(CONFIG_ARCH_NSPIRE) += clk-nspire.o +obj-$(CONFIG_COMMON_CLK_PALMAS) += clk-palmas.o obj-$(CONFIG_CLK_PPC_CORENET) += clk-ppc-corenet.o obj-$(CONFIG_COMMON_CLK_S2MPS11) += clk-s2mps11.o obj-$(CONFIG_COMMON_CLK_SI5351) += clk-si5351.o diff --git a/drivers/clk/at91/clk-main.c b/drivers/clk/at91/clk-main.c index 733306131b99..59fa3cc96c9e 100644 --- a/drivers/clk/at91/clk-main.c +++ b/drivers/clk/at91/clk-main.c @@ -388,6 +388,7 @@ static unsigned long clk_main_recalc_rate(struct at91_pmc *pmc, if (parent_rate) return parent_rate; + pr_warn("Main crystal frequency not set, using approximate value\n"); tmp = pmc_read(pmc, AT91_CKGR_MCFR); if (!(tmp & AT91_PMC_MAINRDY)) return 0; diff --git a/drivers/clk/clk-clps711x.c b/drivers/clk/clk-clps711x.c new file mode 100644 index 000000000000..715eec1a9902 --- /dev/null +++ b/drivers/clk/clk-clps711x.c @@ -0,0 +1,192 @@ +/* + * Cirrus Logic CLPS711X CLK driver + * + * Copyright (C) 2014 Alexander Shiyan <shc_work@mail.ru> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + */ + +#include <linux/clk.h> +#include <linux/clk-provider.h> +#include <linux/clkdev.h> +#include <linux/io.h> +#include <linux/ioport.h> +#include <linux/of_address.h> +#include <linux/slab.h> +#include <linux/mfd/syscon/clps711x.h> + +#include <dt-bindings/clock/clps711x-clock.h> + +#define CLPS711X_SYSCON1 (0x0100) +#define CLPS711X_SYSCON2 (0x1100) +#define CLPS711X_SYSFLG2 (CLPS711X_SYSCON2 + SYSFLG_OFFSET) +#define CLPS711X_PLLR (0xa5a8) + +#define CLPS711X_EXT_FREQ (13000000) +#define CLPS711X_OSC_FREQ (3686400) + +static const struct clk_div_table spi_div_table[] = { + { .val = 0, .div = 32, }, + { .val = 1, .div = 8, }, + { .val = 2, .div = 2, }, + { .val = 3, .div = 1, }, +}; + +static const struct clk_div_table timer_div_table[] = { + { .val = 0, .div = 256, }, + { .val = 1, .div = 1, }, +}; + +struct clps711x_clk { + struct clk_onecell_data clk_data; + spinlock_t lock; + struct clk *clks[CLPS711X_CLK_MAX]; +}; + +static struct clps711x_clk * __init _clps711x_clk_init(void __iomem *base, + u32 fref) +{ + u32 tmp, f_cpu, f_pll, f_bus, f_tim, f_pwm, f_spi; + struct clps711x_clk *clps711x_clk; + unsigned i; + + if (!base) + return ERR_PTR(-ENOMEM); + + clps711x_clk = kzalloc(sizeof(*clps711x_clk), GFP_KERNEL); + if (!clps711x_clk) + return ERR_PTR(-ENOMEM); + + spin_lock_init(&clps711x_clk->lock); + + /* Read PLL multiplier value and sanity check */ + tmp = readl(base + CLPS711X_PLLR) >> 24; + if (((tmp >= 10) && (tmp <= 50)) || !fref) + f_pll = DIV_ROUND_UP(CLPS711X_OSC_FREQ * tmp, 2); + else + f_pll = fref; + + tmp = readl(base + CLPS711X_SYSFLG2); + if (tmp & SYSFLG2_CKMODE) { + f_cpu = CLPS711X_EXT_FREQ; + f_bus = CLPS711X_EXT_FREQ; + f_spi = DIV_ROUND_CLOSEST(CLPS711X_EXT_FREQ, 96); + f_pll = 0; + f_pwm = DIV_ROUND_CLOSEST(CLPS711X_EXT_FREQ, 128); + } else { + f_cpu = f_pll; + if (f_cpu > 36864000) + f_bus = DIV_ROUND_UP(f_cpu, 2); + else + f_bus = 36864000 / 2; + f_spi = DIV_ROUND_CLOSEST(f_cpu, 576); + f_pwm = DIV_ROUND_CLOSEST(f_cpu, 768); + } + + if (tmp & SYSFLG2_CKMODE) { + if (readl(base + CLPS711X_SYSCON2) & SYSCON2_OSTB) + f_tim = DIV_ROUND_CLOSEST(CLPS711X_EXT_FREQ, 26); + else + f_tim = DIV_ROUND_CLOSEST(CLPS711X_EXT_FREQ, 24); + } else + f_tim = DIV_ROUND_CLOSEST(f_cpu, 144); + + tmp = readl(base + CLPS711X_SYSCON1); + /* Timer1 in free running mode. + * Counter will wrap around to 0xffff when it underflows + * and will continue to count down. + */ + tmp &= ~(SYSCON1_TC1M | SYSCON1_TC1S); + /* Timer2 in prescale mode. + * Value writen is automatically re-loaded when + * the counter underflows. + */ + tmp |= SYSCON1_TC2M | SYSCON1_TC2S; + writel(tmp, base + CLPS711X_SYSCON1); + + clps711x_clk->clks[CLPS711X_CLK_DUMMY] = + clk_register_fixed_rate(NULL, "dummy", NULL, CLK_IS_ROOT, 0); + clps711x_clk->clks[CLPS711X_CLK_CPU] = + clk_register_fixed_rate(NULL, "cpu", NULL, CLK_IS_ROOT, f_cpu); + clps711x_clk->clks[CLPS711X_CLK_BUS] = + clk_register_fixed_rate(NULL, "bus", NULL, CLK_IS_ROOT, f_bus); + clps711x_clk->clks[CLPS711X_CLK_PLL] = + clk_register_fixed_rate(NULL, "pll", NULL, CLK_IS_ROOT, f_pll); + clps711x_clk->clks[CLPS711X_CLK_TIMERREF] = + clk_register_fixed_rate(NULL, "timer_ref", NULL, CLK_IS_ROOT, + f_tim); + clps711x_clk->clks[CLPS711X_CLK_TIMER1] = + clk_register_divider_table(NULL, "timer1", "timer_ref", 0, + base + CLPS711X_SYSCON1, 5, 1, 0, + timer_div_table, &clps711x_clk->lock); + clps711x_clk->clks[CLPS711X_CLK_TIMER2] = + clk_register_divider_table(NULL, "timer2", "timer_ref", 0, + base + CLPS711X_SYSCON1, 7, 1, 0, + timer_div_table, &clps711x_clk->lock); + clps711x_clk->clks[CLPS711X_CLK_PWM] = + clk_register_fixed_rate(NULL, "pwm", NULL, CLK_IS_ROOT, f_pwm); + clps711x_clk->clks[CLPS711X_CLK_SPIREF] = + clk_register_fixed_rate(NULL, "spi_ref", NULL, CLK_IS_ROOT, + f_spi); + clps711x_clk->clks[CLPS711X_CLK_SPI] = + clk_register_divider_table(NULL, "spi", "spi_ref", 0, + base + CLPS711X_SYSCON1, 16, 2, 0, + spi_div_table, &clps711x_clk->lock); + clps711x_clk->clks[CLPS711X_CLK_UART] = + clk_register_fixed_factor(NULL, "uart", "bus", 0, 1, 10); + clps711x_clk->clks[CLPS711X_CLK_TICK] = + clk_register_fixed_rate(NULL, "tick", NULL, CLK_IS_ROOT, 64); + + for (i = 0; i < CLPS711X_CLK_MAX; i++) + if (IS_ERR(clps711x_clk->clks[i])) + pr_err("clk %i: register failed with %ld\n", + i, PTR_ERR(clps711x_clk->clks[i])); + + return clps711x_clk; +} + +void __init clps711x_clk_init(void __iomem *base) +{ + struct clps711x_clk *clps711x_clk; + + clps711x_clk = _clps711x_clk_init(base, 73728000); + + BUG_ON(IS_ERR(clps711x_clk)); + + /* Clocksource */ + clk_register_clkdev(clps711x_clk->clks[CLPS711X_CLK_TIMER1], + NULL, "clps711x-timer.0"); + clk_register_clkdev(clps711x_clk->clks[CLPS711X_CLK_TIMER2], + NULL, "clps711x-timer.1"); + + /* Drivers */ + clk_register_clkdev(clps711x_clk->clks[CLPS711X_CLK_PWM], + NULL, "clps711x-pwm"); + clk_register_clkdev(clps711x_clk->clks[CLPS711X_CLK_UART], + NULL, "clps711x-uart.0"); + clk_register_clkdev(clps711x_clk->clks[CLPS711X_CLK_UART], + NULL, "clps711x-uart.1"); +} + +#ifdef CONFIG_OF +static void __init clps711x_clk_init_dt(struct device_node *np) +{ + void __iomem *base = of_iomap(np, 0); + struct clps711x_clk *clps711x_clk; + u32 fref = 0; + + WARN_ON(of_property_read_u32(np, "startup-frequency", &fref)); + + clps711x_clk = _clps711x_clk_init(base, fref); + BUG_ON(IS_ERR(clps711x_clk)); + + clps711x_clk->clk_data.clks = clps711x_clk->clks; + clps711x_clk->clk_data.clk_num = CLPS711X_CLK_MAX; + of_clk_add_provider(np, of_clk_src_onecell_get, + &clps711x_clk->clk_data); +} +CLK_OF_DECLARE(clps711x, "cirrus,clps711x-clk", clps711x_clk_init_dt); +#endif diff --git a/drivers/clk/clk-composite.c b/drivers/clk/clk-composite.c index 57a078e06efe..b9355daf8065 100644 --- a/drivers/clk/clk-composite.c +++ b/drivers/clk/clk-composite.c @@ -64,11 +64,56 @@ static long clk_composite_determine_rate(struct clk_hw *hw, unsigned long rate, const struct clk_ops *mux_ops = composite->mux_ops; struct clk_hw *rate_hw = composite->rate_hw; struct clk_hw *mux_hw = composite->mux_hw; + struct clk *parent; + unsigned long parent_rate; + long tmp_rate, best_rate = 0; + unsigned long rate_diff; + unsigned long best_rate_diff = ULONG_MAX; + int i; if (rate_hw && rate_ops && rate_ops->determine_rate) { rate_hw->clk = hw->clk; return rate_ops->determine_rate(rate_hw, rate, best_parent_rate, best_parent_p); + } else if (rate_hw && rate_ops && rate_ops->round_rate && + mux_hw && mux_ops && mux_ops->set_parent) { + *best_parent_p = NULL; + + if (__clk_get_flags(hw->clk) & CLK_SET_RATE_NO_REPARENT) { + *best_parent_p = clk_get_parent(mux_hw->clk); + *best_parent_rate = __clk_get_rate(*best_parent_p); + + return rate_ops->round_rate(rate_hw, rate, + best_parent_rate); + } + + for (i = 0; i < __clk_get_num_parents(mux_hw->clk); i++) { + parent = clk_get_parent_by_index(mux_hw->clk, i); + if (!parent) + continue; + + parent_rate = __clk_get_rate(parent); + + tmp_rate = rate_ops->round_rate(rate_hw, rate, + &parent_rate); + if (tmp_rate < 0) + continue; + + rate_diff = abs(rate - tmp_rate); + + if (!rate_diff || !*best_parent_p + || best_rate_diff > rate_diff) { + *best_parent_p = parent; + *best_parent_rate = parent_rate; + best_rate_diff = rate_diff; + best_rate = tmp_rate; + } + + if (!rate_diff) + return rate; + } + + return best_rate; } else if (mux_hw && mux_ops && mux_ops->determine_rate) { mux_hw->clk = hw->clk; return mux_ops->determine_rate(mux_hw, rate, best_parent_rate, @@ -162,7 +207,7 @@ struct clk *clk_register_composite(struct device *dev, const char *name, clk_composite_ops = &composite->ops; if (mux_hw && mux_ops) { - if (!mux_ops->get_parent || !mux_ops->set_parent) { + if (!mux_ops->get_parent) { clk = ERR_PTR(-EINVAL); goto err; } @@ -170,7 +215,8 @@ struct clk *clk_register_composite(struct device *dev, const char *name, composite->mux_hw = mux_hw; composite->mux_ops = mux_ops; clk_composite_ops->get_parent = clk_composite_get_parent; - clk_composite_ops->set_parent = clk_composite_set_parent; + if (mux_ops->set_parent) + clk_composite_ops->set_parent = clk_composite_set_parent; if (mux_ops->determine_rate) clk_composite_ops->determine_rate = clk_composite_determine_rate; } @@ -180,24 +226,27 @@ struct clk *clk_register_composite(struct device *dev, const char *name, clk = ERR_PTR(-EINVAL); goto err; } + clk_composite_ops->recalc_rate = clk_composite_recalc_rate; - /* .round_rate is a prerequisite for .set_rate */ - if (rate_ops->round_rate) { - clk_composite_ops->round_rate = clk_composite_round_rate; - if (rate_ops->set_rate) { - clk_composite_ops->set_rate = clk_composite_set_rate; - } - } else { - WARN(rate_ops->set_rate, - "%s: missing round_rate op is required\n", - __func__); + if (rate_ops->determine_rate) + clk_composite_ops->determine_rate = + clk_composite_determine_rate; + else if (rate_ops->round_rate) + clk_composite_ops->round_rate = + clk_composite_round_rate; + + /* .set_rate requires either .round_rate or .determine_rate */ + if (rate_ops->set_rate) { + if (rate_ops->determine_rate || rate_ops->round_rate) + clk_composite_ops->set_rate = + clk_composite_set_rate; + else + WARN(1, "%s: missing round_rate op is required\n", + __func__); } composite->rate_hw = rate_hw; composite->rate_ops = rate_ops; - clk_composite_ops->recalc_rate = clk_composite_recalc_rate; - if (rate_ops->determine_rate) - clk_composite_ops->determine_rate = clk_composite_determine_rate; } if (gate_hw && gate_ops) { diff --git a/drivers/clk/clk-conf.c b/drivers/clk/clk-conf.c new file mode 100644 index 000000000000..aad4796aa3ed --- /dev/null +++ b/drivers/clk/clk-conf.c @@ -0,0 +1,144 @@ +/* + * Copyright (C) 2014 Samsung Electronics Co., Ltd. + * Sylwester Nawrocki <s.nawrocki@samsung.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/clk.h> +#include <linux/clk-provider.h> +#include <linux/clk/clk-conf.h> +#include <linux/device.h> +#include <linux/of.h> +#include <linux/printk.h> +#include "clk.h" + +static int __set_clk_parents(struct device_node *node, bool clk_supplier) +{ + struct of_phandle_args clkspec; + int index, rc, num_parents; + struct clk *clk, *pclk; + + num_parents = of_count_phandle_with_args(node, "assigned-clock-parents", + "#clock-cells"); + if (num_parents == -EINVAL) + pr_err("clk: invalid value of clock-parents property at %s\n", + node->full_name); + + for (index = 0; index < num_parents; index++) { + rc = of_parse_phandle_with_args(node, "assigned-clock-parents", + "#clock-cells", index, &clkspec); + if (rc < 0) { + /* skip empty (null) phandles */ + if (rc == -ENOENT) + continue; + else + return rc; + } + if (clkspec.np == node && !clk_supplier) + return 0; + pclk = of_clk_get_by_clkspec(&clkspec); + if (IS_ERR(pclk)) { + pr_warn("clk: couldn't get parent clock %d for %s\n", + index, node->full_name); + return PTR_ERR(pclk); + } + + rc = of_parse_phandle_with_args(node, "assigned-clocks", + "#clock-cells", index, &clkspec); + if (rc < 0) + goto err; + if (clkspec.np == node && !clk_supplier) { + rc = 0; + goto err; + } + clk = of_clk_get_by_clkspec(&clkspec); + if (IS_ERR(clk)) { + pr_warn("clk: couldn't get parent clock %d for %s\n", + index, node->full_name); + rc = PTR_ERR(clk); + goto err; + } + + rc = clk_set_parent(clk, pclk); + if (rc < 0) + pr_err("clk: failed to reparent %s to %s: %d\n", + __clk_get_name(clk), __clk_get_name(pclk), rc); + clk_put(clk); + clk_put(pclk); + } + return 0; +err: + clk_put(pclk); + return rc; +} + +static int __set_clk_rates(struct device_node *node, bool clk_supplier) +{ + struct of_phandle_args clkspec; + struct property *prop; + const __be32 *cur; + int rc, index = 0; + struct clk *clk; + u32 rate; + + of_property_for_each_u32(node, "assigned-clock-rates", prop, cur, rate) { + if (rate) { + rc = of_parse_phandle_with_args(node, "assigned-clocks", + "#clock-cells", index, &clkspec); + if (rc < 0) { + /* skip empty (null) phandles */ + if (rc == -ENOENT) + continue; + else + return rc; + } + if (clkspec.np == node && !clk_supplier) + return 0; + + clk = of_clk_get_by_clkspec(&clkspec); + if (IS_ERR(clk)) { + pr_warn("clk: couldn't get clock %d for %s\n", + index, node->full_name); + return PTR_ERR(clk); + } + + rc = clk_set_rate(clk, rate); + if (rc < 0) + pr_err("clk: couldn't set %s clock rate: %d\n", + __clk_get_name(clk), rc); + clk_put(clk); + } + index++; + } + return 0; +} + +/** + * of_clk_set_defaults() - parse and set assigned clocks configuration + * @node: device node to apply clock settings for + * @clk_supplier: true if clocks supplied by @node should also be considered + * + * This function parses 'assigned-{clocks/clock-parents/clock-rates}' properties + * and sets any specified clock parents and rates. The @clk_supplier argument + * should be set to true if @node may be also a clock supplier of any clock + * listed in its 'assigned-clocks' or 'assigned-clock-parents' properties. + * If @clk_supplier is false the function exits returnning 0 as soon as it + * determines the @node is also a supplier of any of the clocks. + */ +int of_clk_set_defaults(struct device_node *node, bool clk_supplier) +{ + int rc; + + if (!node) + return 0; + + rc = __set_clk_parents(node, clk_supplier); + if (rc < 0) + return rc; + + return __set_clk_rates(node, clk_supplier); +} +EXPORT_SYMBOL_GPL(of_clk_set_defaults); diff --git a/drivers/clk/clk-palmas.c b/drivers/clk/clk-palmas.c new file mode 100644 index 000000000000..781630e1372b --- /dev/null +++ b/drivers/clk/clk-palmas.c @@ -0,0 +1,307 @@ +/* + * Clock driver for Palmas device. + * + * Copyright (c) 2013, NVIDIA Corporation. + * Copyright (c) 2013-2014 Texas Instruments, Inc. + * + * Author: Laxman Dewangan <ldewangan@nvidia.com> + * Peter Ujfalusi <peter.ujfalusi@ti.com> + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation version 2. + * + * This program is distributed "as is" WITHOUT ANY WARRANTY of any kind, + * whether express or implied; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ + +#include <linux/clk.h> +#include <linux/clkdev.h> +#include <linux/clk-provider.h> +#include <linux/mfd/palmas.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_device.h> +#include <linux/platform_device.h> +#include <linux/slab.h> + +#define PALMAS_CLOCK_DT_EXT_CONTROL_ENABLE1 1 +#define PALMAS_CLOCK_DT_EXT_CONTROL_ENABLE2 2 +#define PALMAS_CLOCK_DT_EXT_CONTROL_NSLEEP 3 + +struct palmas_clk32k_desc { + const char *clk_name; + unsigned int control_reg; + unsigned int enable_mask; + unsigned int sleep_mask; + unsigned int sleep_reqstr_id; + int delay; +}; + +struct palmas_clock_info { + struct device *dev; + struct clk *clk; + struct clk_hw hw; + struct palmas *palmas; + struct palmas_clk32k_desc *clk_desc; + int ext_control_pin; +}; + +static inline struct palmas_clock_info *to_palmas_clks_info(struct clk_hw *hw) +{ + return container_of(hw, struct palmas_clock_info, hw); +} + +static unsigned long palmas_clks_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + return 32768; +} + +static int palmas_clks_prepare(struct clk_hw *hw) +{ + struct palmas_clock_info *cinfo = to_palmas_clks_info(hw); + int ret; + + ret = palmas_update_bits(cinfo->palmas, PALMAS_RESOURCE_BASE, + cinfo->clk_desc->control_reg, + cinfo->clk_desc->enable_mask, + cinfo->clk_desc->enable_mask); + if (ret < 0) + dev_err(cinfo->dev, "Reg 0x%02x update failed, %d\n", + cinfo->clk_desc->control_reg, ret); + else if (cinfo->clk_desc->delay) + udelay(cinfo->clk_desc->delay); + + return ret; +} + +static void palmas_clks_unprepare(struct clk_hw *hw) +{ + struct palmas_clock_info *cinfo = to_palmas_clks_info(hw); + int ret; + + /* + * Clock can be disabled through external pin if it is externally + * controlled. + */ + if (cinfo->ext_control_pin) + return; + + ret = palmas_update_bits(cinfo->palmas, PALMAS_RESOURCE_BASE, + cinfo->clk_desc->control_reg, + cinfo->clk_desc->enable_mask, 0); + if (ret < 0) + dev_err(cinfo->dev, "Reg 0x%02x update failed, %d\n", + cinfo->clk_desc->control_reg, ret); +} + +static int palmas_clks_is_prepared(struct clk_hw *hw) +{ + struct palmas_clock_info *cinfo = to_palmas_clks_info(hw); + int ret; + u32 val; + + if (cinfo->ext_control_pin) + return 1; + + ret = palmas_read(cinfo->palmas, PALMAS_RESOURCE_BASE, + cinfo->clk_desc->control_reg, &val); + if (ret < 0) { + dev_err(cinfo->dev, "Reg 0x%02x read failed, %d\n", + cinfo->clk_desc->control_reg, ret); + return ret; + } + return !!(val & cinfo->clk_desc->enable_mask); +} + +static struct clk_ops palmas_clks_ops = { + .prepare = palmas_clks_prepare, + .unprepare = palmas_clks_unprepare, + .is_prepared = palmas_clks_is_prepared, + .recalc_rate = palmas_clks_recalc_rate, +}; + +struct palmas_clks_of_match_data { + struct clk_init_data init; + struct palmas_clk32k_desc desc; +}; + +static struct palmas_clks_of_match_data palmas_of_clk32kg = { + .init = { + .name = "clk32kg", + .ops = &palmas_clks_ops, + .flags = CLK_IS_ROOT | CLK_IGNORE_UNUSED, + }, + .desc = { + .clk_name = "clk32kg", + .control_reg = PALMAS_CLK32KG_CTRL, + .enable_mask = PALMAS_CLK32KG_CTRL_MODE_ACTIVE, + .sleep_mask = PALMAS_CLK32KG_CTRL_MODE_SLEEP, + .sleep_reqstr_id = PALMAS_EXTERNAL_REQSTR_ID_CLK32KG, + .delay = 200, + }, +}; + +static struct palmas_clks_of_match_data palmas_of_clk32kgaudio = { + .init = { + .name = "clk32kgaudio", + .ops = &palmas_clks_ops, + .flags = CLK_IS_ROOT | CLK_IGNORE_UNUSED, + }, + .desc = { + .clk_name = "clk32kgaudio", + .control_reg = PALMAS_CLK32KGAUDIO_CTRL, + .enable_mask = PALMAS_CLK32KG_CTRL_MODE_ACTIVE, + .sleep_mask = PALMAS_CLK32KG_CTRL_MODE_SLEEP, + .sleep_reqstr_id = PALMAS_EXTERNAL_REQSTR_ID_CLK32KGAUDIO, + .delay = 200, + }, +}; + +static struct of_device_id palmas_clks_of_match[] = { + { + .compatible = "ti,palmas-clk32kg", + .data = &palmas_of_clk32kg, + }, + { + .compatible = "ti,palmas-clk32kgaudio", + .data = &palmas_of_clk32kgaudio, + }, + { }, +}; +MODULE_DEVICE_TABLE(of, palmas_clks_of_match); + +static void palmas_clks_get_clk_data(struct platform_device *pdev, + struct palmas_clock_info *cinfo) +{ + struct device_node *node = pdev->dev.of_node; + unsigned int prop; + int ret; + + ret = of_property_read_u32(node, "ti,external-sleep-control", + &prop); + if (ret) + return; + + switch (prop) { + case PALMAS_CLOCK_DT_EXT_CONTROL_ENABLE1: + prop = PALMAS_EXT_CONTROL_ENABLE1; + break; + case PALMAS_CLOCK_DT_EXT_CONTROL_ENABLE2: + prop = PALMAS_EXT_CONTROL_ENABLE2; + break; + case PALMAS_CLOCK_DT_EXT_CONTROL_NSLEEP: + prop = PALMAS_EXT_CONTROL_NSLEEP; + break; + default: + dev_warn(&pdev->dev, "%s: Invalid ext control option: %u\n", + node->name, prop); + prop = 0; + break; + } + cinfo->ext_control_pin = prop; +} + +static int palmas_clks_init_configure(struct palmas_clock_info *cinfo) +{ + int ret; + + ret = palmas_update_bits(cinfo->palmas, PALMAS_RESOURCE_BASE, + cinfo->clk_desc->control_reg, + cinfo->clk_desc->sleep_mask, 0); + if (ret < 0) { + dev_err(cinfo->dev, "Reg 0x%02x update failed, %d\n", + cinfo->clk_desc->control_reg, ret); + return ret; + } + + if (cinfo->ext_control_pin) { + ret = clk_prepare(cinfo->clk); + if (ret < 0) { + dev_err(cinfo->dev, "Clock prep failed, %d\n", ret); + return ret; + } + + ret = palmas_ext_control_req_config(cinfo->palmas, + cinfo->clk_desc->sleep_reqstr_id, + cinfo->ext_control_pin, true); + if (ret < 0) { + dev_err(cinfo->dev, "Ext config for %s failed, %d\n", + cinfo->clk_desc->clk_name, ret); + return ret; + } + } + + return ret; +} +static int palmas_clks_probe(struct platform_device *pdev) +{ + struct palmas *palmas = dev_get_drvdata(pdev->dev.parent); + struct device_node *node = pdev->dev.of_node; + struct palmas_clks_of_match_data *match_data; + const struct of_device_id *match; + struct palmas_clock_info *cinfo; + struct clk *clk; + int ret; + + match = of_match_device(palmas_clks_of_match, &pdev->dev); + match_data = (struct palmas_clks_of_match_data *)match->data; + + cinfo = devm_kzalloc(&pdev->dev, sizeof(*cinfo), GFP_KERNEL); + if (!cinfo) + return -ENOMEM; + + palmas_clks_get_clk_data(pdev, cinfo); + platform_set_drvdata(pdev, cinfo); + + cinfo->dev = &pdev->dev; + cinfo->palmas = palmas; + + cinfo->clk_desc = &match_data->desc; + cinfo->hw.init = &match_data->init; + clk = devm_clk_register(&pdev->dev, &cinfo->hw); + if (IS_ERR(clk)) { + ret = PTR_ERR(clk); + dev_err(&pdev->dev, "Fail to register clock %s, %d\n", + match_data->desc.clk_name, ret); + return ret; + } + + cinfo->clk = clk; + ret = palmas_clks_init_configure(cinfo); + if (ret < 0) { + dev_err(&pdev->dev, "Clock config failed, %d\n", ret); + return ret; + } + + ret = of_clk_add_provider(node, of_clk_src_simple_get, cinfo->clk); + if (ret < 0) + dev_err(&pdev->dev, "Fail to add clock driver, %d\n", ret); + return ret; +} + +static int palmas_clks_remove(struct platform_device *pdev) +{ + of_clk_del_provider(pdev->dev.of_node); + return 0; +} + +static struct platform_driver palmas_clks_driver = { + .driver = { + .name = "palmas-clk", + .owner = THIS_MODULE, + .of_match_table = palmas_clks_of_match, + }, + .probe = palmas_clks_probe, + .remove = palmas_clks_remove, +}; + +module_platform_driver(palmas_clks_driver); + +MODULE_DESCRIPTION("Clock driver for Palmas Series Devices"); +MODULE_ALIAS("platform:palmas-clk"); +MODULE_AUTHOR("Peter Ujfalusi <peter.ujfalusi@ti.com>"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/clk/clk-ppc-corenet.c b/drivers/clk/clk-ppc-corenet.c index 8b284be4efa4..8e58edfeeb37 100644 --- a/drivers/clk/clk-ppc-corenet.c +++ b/drivers/clk/clk-ppc-corenet.c @@ -291,7 +291,7 @@ static const struct of_device_id ppc_clk_ids[] __initconst = { {} }; -static struct platform_driver ppc_corenet_clk_driver = { +static struct platform_driver ppc_corenet_clk_driver __initdata = { .driver = { .name = "ppc_corenet_clock", .owner = THIS_MODULE, diff --git a/drivers/clk/clk-s2mps11.c b/drivers/clk/clk-s2mps11.c index 3757e9e72d37..b7797fb12e12 100644 --- a/drivers/clk/clk-s2mps11.c +++ b/drivers/clk/clk-s2mps11.c @@ -46,7 +46,6 @@ struct s2mps11_clk { struct clk *clk; struct clk_lookup *lookup; u32 mask; - bool enabled; unsigned int reg; }; @@ -63,8 +62,6 @@ static int s2mps11_clk_prepare(struct clk_hw *hw) ret = regmap_update_bits(s2mps11->iodev->regmap_pmic, s2mps11->reg, s2mps11->mask, s2mps11->mask); - if (!ret) - s2mps11->enabled = true; return ret; } @@ -76,32 +73,32 @@ static void s2mps11_clk_unprepare(struct clk_hw *hw) ret = regmap_update_bits(s2mps11->iodev->regmap_pmic, s2mps11->reg, s2mps11->mask, ~s2mps11->mask); - - if (!ret) - s2mps11->enabled = false; } -static int s2mps11_clk_is_enabled(struct clk_hw *hw) +static int s2mps11_clk_is_prepared(struct clk_hw *hw) { + int ret; + u32 val; struct s2mps11_clk *s2mps11 = to_s2mps11_clk(hw); - return s2mps11->enabled; + ret = regmap_read(s2mps11->iodev->regmap_pmic, + s2mps11->reg, &val); + if (ret < 0) + return -EINVAL; + + return val & s2mps11->mask; } static unsigned long s2mps11_clk_recalc_rate(struct clk_hw *hw, unsigned long parent_rate) { - struct s2mps11_clk *s2mps11 = to_s2mps11_clk(hw); - if (s2mps11->enabled) - return 32768; - else - return 0; + return 32768; } static struct clk_ops s2mps11_clk_ops = { .prepare = s2mps11_clk_prepare, .unprepare = s2mps11_clk_unprepare, - .is_enabled = s2mps11_clk_is_enabled, + .is_prepared = s2mps11_clk_is_prepared, .recalc_rate = s2mps11_clk_recalc_rate, }; @@ -169,7 +166,6 @@ static int s2mps11_clk_probe(struct platform_device *pdev) unsigned int s2mps11_reg; struct clk_init_data *clks_init; int i, ret = 0; - u32 val; s2mps11_clks = devm_kzalloc(&pdev->dev, sizeof(*s2mps11_clk) * S2MPS11_CLKS_NUM, GFP_KERNEL); @@ -214,13 +210,6 @@ static int s2mps11_clk_probe(struct platform_device *pdev) s2mps11_clk->mask = 1 << i; s2mps11_clk->reg = s2mps11_reg; - ret = regmap_read(s2mps11_clk->iodev->regmap_pmic, - s2mps11_clk->reg, &val); - if (ret < 0) - goto err_reg; - - s2mps11_clk->enabled = val & s2mps11_clk->mask; - s2mps11_clk->clk = devm_clk_register(&pdev->dev, &s2mps11_clk->hw); if (IS_ERR(s2mps11_clk->clk)) { diff --git a/drivers/clk/clk.c b/drivers/clk/clk.c index 8b73edef151d..b76fa69b44cb 100644 --- a/drivers/clk/clk.c +++ b/drivers/clk/clk.c @@ -10,6 +10,7 @@ */ #include <linux/clk-private.h> +#include <linux/clk/clk-conf.h> #include <linux/module.h> #include <linux/mutex.h> #include <linux/spinlock.h> @@ -98,9 +99,19 @@ static void clk_enable_unlock(unsigned long flags) #include <linux/debugfs.h> static struct dentry *rootdir; -static struct dentry *orphandir; static int inited = 0; +static struct hlist_head *all_lists[] = { + &clk_root_list, + &clk_orphan_list, + NULL, +}; + +static struct hlist_head *orphan_list[] = { + &clk_orphan_list, + NULL, +}; + static void clk_summary_show_one(struct seq_file *s, struct clk *c, int level) { if (!c) @@ -130,17 +141,16 @@ static void clk_summary_show_subtree(struct seq_file *s, struct clk *c, static int clk_summary_show(struct seq_file *s, void *data) { struct clk *c; + struct hlist_head **lists = (struct hlist_head **)s->private; seq_puts(s, " clock enable_cnt prepare_cnt rate accuracy\n"); seq_puts(s, "--------------------------------------------------------------------------------\n"); clk_prepare_lock(); - hlist_for_each_entry(c, &clk_root_list, child_node) - clk_summary_show_subtree(s, c, 0); - - hlist_for_each_entry(c, &clk_orphan_list, child_node) - clk_summary_show_subtree(s, c, 0); + for (; *lists; lists++) + hlist_for_each_entry(c, *lists, child_node) + clk_summary_show_subtree(s, c, 0); clk_prepare_unlock(); @@ -193,21 +203,19 @@ static int clk_dump(struct seq_file *s, void *data) { struct clk *c; bool first_node = true; + struct hlist_head **lists = (struct hlist_head **)s->private; seq_printf(s, "{"); clk_prepare_lock(); - hlist_for_each_entry(c, &clk_root_list, child_node) { - if (!first_node) - seq_printf(s, ","); - first_node = false; - clk_dump_subtree(s, c, 0); - } - - hlist_for_each_entry(c, &clk_orphan_list, child_node) { - seq_printf(s, ","); - clk_dump_subtree(s, c, 0); + for (; *lists; lists++) { + hlist_for_each_entry(c, *lists, child_node) { + if (!first_node) + seq_puts(s, ","); + first_node = false; + clk_dump_subtree(s, c, 0); + } } clk_prepare_unlock(); @@ -276,9 +284,11 @@ static int clk_debug_create_one(struct clk *clk, struct dentry *pdentry) if (!d) goto err_out; - if (clk->ops->debug_init) - if (clk->ops->debug_init(clk->hw, clk->dentry)) + if (clk->ops->debug_init) { + ret = clk->ops->debug_init(clk->hw, clk->dentry); + if (ret) goto err_out; + } ret = 0; goto out; @@ -305,7 +315,7 @@ static int clk_debug_create_subtree(struct clk *clk, struct dentry *pdentry) goto out; hlist_for_each_entry(child, &clk->children, child_node) - clk_debug_create_subtree(child, clk->dentry); + clk_debug_create_subtree(child, pdentry); ret = 0; out: @@ -325,31 +335,12 @@ out: */ static int clk_debug_register(struct clk *clk) { - struct clk *parent; - struct dentry *pdentry; int ret = 0; if (!inited) goto out; - parent = clk->parent; - - /* - * Check to see if a clk is a root clk. Also check that it is - * safe to add this clk to debugfs - */ - if (!parent) - if (clk->flags & CLK_IS_ROOT) - pdentry = rootdir; - else - pdentry = orphandir; - else - if (parent->dentry) - pdentry = parent->dentry; - else - goto out; - - ret = clk_debug_create_subtree(clk, pdentry); + ret = clk_debug_create_subtree(clk, rootdir); out: return ret; @@ -370,38 +361,17 @@ static void clk_debug_unregister(struct clk *clk) debugfs_remove_recursive(clk->dentry); } -/** - * clk_debug_reparent - reparent clk node in the debugfs clk tree - * @clk: the clk being reparented - * @new_parent: the new clk parent, may be NULL - * - * Rename clk entry in the debugfs clk tree if debugfs has been - * initialized. Otherwise it bails out early since the debugfs clk tree - * will be created lazily by clk_debug_init as part of a late_initcall. - * - * Caller must hold prepare_lock. - */ -static void clk_debug_reparent(struct clk *clk, struct clk *new_parent) +struct dentry *clk_debugfs_add_file(struct clk *clk, char *name, umode_t mode, + void *data, const struct file_operations *fops) { - struct dentry *d; - struct dentry *new_parent_d; - - if (!inited) - return; + struct dentry *d = NULL; - if (new_parent) - new_parent_d = new_parent->dentry; - else - new_parent_d = orphandir; + if (clk->dentry) + d = debugfs_create_file(name, mode, clk->dentry, data, fops); - d = debugfs_rename(clk->dentry->d_parent, clk->dentry, - new_parent_d, clk->name); - if (d) - clk->dentry = d; - else - pr_debug("%s: failed to rename debugfs entry for %s\n", - __func__, clk->name); + return d; } +EXPORT_SYMBOL_GPL(clk_debugfs_add_file); /** * clk_debug_init - lazily create the debugfs clk tree visualization @@ -425,19 +395,24 @@ static int __init clk_debug_init(void) if (!rootdir) return -ENOMEM; - d = debugfs_create_file("clk_summary", S_IRUGO, rootdir, NULL, + d = debugfs_create_file("clk_summary", S_IRUGO, rootdir, &all_lists, &clk_summary_fops); if (!d) return -ENOMEM; - d = debugfs_create_file("clk_dump", S_IRUGO, rootdir, NULL, + d = debugfs_create_file("clk_dump", S_IRUGO, rootdir, &all_lists, &clk_dump_fops); if (!d) return -ENOMEM; - orphandir = debugfs_create_dir("orphans", rootdir); + d = debugfs_create_file("clk_orphan_summary", S_IRUGO, rootdir, + &orphan_list, &clk_summary_fops); + if (!d) + return -ENOMEM; - if (!orphandir) + d = debugfs_create_file("clk_orphan_dump", S_IRUGO, rootdir, + &orphan_list, &clk_dump_fops); + if (!d) return -ENOMEM; clk_prepare_lock(); @@ -446,7 +421,7 @@ static int __init clk_debug_init(void) clk_debug_create_subtree(clk, rootdir); hlist_for_each_entry(clk, &clk_orphan_list, child_node) - clk_debug_create_subtree(clk, orphandir); + clk_debug_create_subtree(clk, rootdir); inited = 1; @@ -1284,9 +1259,6 @@ static void __clk_set_parent_after(struct clk *clk, struct clk *parent, clk_disable(old_parent); __clk_unprepare(old_parent); } - - /* update debugfs with new clk tree topology */ - clk_debug_reparent(clk, parent); } static int __clk_set_parent(struct clk *clk, struct clk *parent, u8 p_index) @@ -1683,7 +1655,6 @@ out: void __clk_reparent(struct clk *clk, struct clk *new_parent) { clk_reparent(clk, new_parent); - clk_debug_reparent(clk, new_parent); __clk_recalc_accuracies(clk); __clk_recalc_rates(clk, POST_RATE_CHANGE); } @@ -2414,6 +2385,7 @@ int of_clk_add_provider(struct device_node *np, void *data) { struct of_clk_provider *cp; + int ret; cp = kzalloc(sizeof(struct of_clk_provider), GFP_KERNEL); if (!cp) @@ -2428,7 +2400,11 @@ int of_clk_add_provider(struct device_node *np, mutex_unlock(&of_clk_mutex); pr_debug("Added clock from %s\n", np->full_name); - return 0; + ret = of_clk_set_defaults(np, true); + if (ret < 0) + of_clk_del_provider(np); + + return ret; } EXPORT_SYMBOL_GPL(of_clk_add_provider); @@ -2605,7 +2581,10 @@ void __init of_clk_init(const struct of_device_id *matches) list_for_each_entry_safe(clk_provider, next, &clk_provider_list, node) { if (force || parent_ready(clk_provider->np)) { + clk_provider->clk_init_cb(clk_provider->np); + of_clk_set_defaults(clk_provider->np, true); + list_del(&clk_provider->node); kfree(clk_provider); is_init_done = true; @@ -2620,7 +2599,6 @@ void __init of_clk_init(const struct of_device_id *matches) */ if (!is_init_done) force = true; - } } #endif diff --git a/drivers/clk/clkdev.c b/drivers/clk/clkdev.c index f890b901c6bc..da4bda8b7fc7 100644 --- a/drivers/clk/clkdev.c +++ b/drivers/clk/clkdev.c @@ -101,8 +101,9 @@ struct clk *of_clk_get_by_name(struct device_node *np, const char *name) if (!IS_ERR(clk)) break; else if (name && index >= 0) { - pr_err("ERROR: could not get clock %s:%s(%i)\n", - np->full_name, name ? name : "", index); + if (PTR_ERR(clk) != -EPROBE_DEFER) + pr_err("ERROR: could not get clock %s:%s(%i)\n", + np->full_name, name ? name : "", index); return clk; } diff --git a/drivers/clk/qcom/Kconfig b/drivers/clk/qcom/Kconfig index 7f696b7d4422..1107351ed346 100644 --- a/drivers/clk/qcom/Kconfig +++ b/drivers/clk/qcom/Kconfig @@ -4,6 +4,31 @@ config COMMON_CLK_QCOM select REGMAP_MMIO select RESET_CONTROLLER +config APQ_GCC_8084 + tristate "APQ8084 Global Clock Controller" + depends on COMMON_CLK_QCOM + help + Support for the global clock controller on apq8084 devices. + Say Y if you want to use peripheral devices such as UART, SPI, + i2c, USB, SD/eMMC, SATA, PCIe, etc. + +config APQ_MMCC_8084 + tristate "APQ8084 Multimedia Clock Controller" + select APQ_GCC_8084 + depends on COMMON_CLK_QCOM + help + Support for the multimedia clock controller on apq8084 devices. + Say Y if you want to support multimedia devices such as display, + graphics, video encode/decode, camera, etc. + +config IPQ_GCC_806X + tristate "IPQ806x Global Clock Controller" + depends on COMMON_CLK_QCOM + help + Support for the global clock controller on ipq806x devices. + Say Y if you want to use peripheral devices such as UART, SPI, + i2c, USB, SD/eMMC, etc. + config MSM_GCC_8660 tristate "MSM8660 Global Clock Controller" depends on COMMON_CLK_QCOM diff --git a/drivers/clk/qcom/Makefile b/drivers/clk/qcom/Makefile index 689e05bf4f95..783cfb24faa4 100644 --- a/drivers/clk/qcom/Makefile +++ b/drivers/clk/qcom/Makefile @@ -8,6 +8,9 @@ clk-qcom-y += clk-rcg2.o clk-qcom-y += clk-branch.o clk-qcom-y += reset.o +obj-$(CONFIG_APQ_GCC_8084) += gcc-apq8084.o +obj-$(CONFIG_APQ_MMCC_8084) += mmcc-apq8084.o +obj-$(CONFIG_IPQ_GCC_806X) += gcc-ipq806x.o obj-$(CONFIG_MSM_GCC_8660) += gcc-msm8660.o obj-$(CONFIG_MSM_GCC_8960) += gcc-msm8960.o obj-$(CONFIG_MSM_GCC_8974) += gcc-msm8974.o diff --git a/drivers/clk/qcom/clk-pll.c b/drivers/clk/qcom/clk-pll.c index 0f927c538613..9db03d3b1657 100644 --- a/drivers/clk/qcom/clk-pll.c +++ b/drivers/clk/qcom/clk-pll.c @@ -166,7 +166,7 @@ const struct clk_ops clk_pll_vote_ops = { EXPORT_SYMBOL_GPL(clk_pll_vote_ops); static void -clk_pll_set_fsm_mode(struct clk_pll *pll, struct regmap *regmap) +clk_pll_set_fsm_mode(struct clk_pll *pll, struct regmap *regmap, u8 lock_count) { u32 val; u32 mask; @@ -175,7 +175,7 @@ clk_pll_set_fsm_mode(struct clk_pll *pll, struct regmap *regmap) regmap_update_bits(regmap, pll->mode_reg, PLL_VOTE_FSM_RESET, 0); /* Program bias count and lock count */ - val = 1 << PLL_BIAS_COUNT_SHIFT; + val = 1 << PLL_BIAS_COUNT_SHIFT | lock_count << PLL_LOCK_COUNT_SHIFT; mask = PLL_BIAS_COUNT_MASK << PLL_BIAS_COUNT_SHIFT; mask |= PLL_LOCK_COUNT_MASK << PLL_LOCK_COUNT_SHIFT; regmap_update_bits(regmap, pll->mode_reg, mask, val); @@ -212,11 +212,20 @@ static void clk_pll_configure(struct clk_pll *pll, struct regmap *regmap, regmap_update_bits(regmap, pll->config_reg, mask, val); } +void clk_pll_configure_sr(struct clk_pll *pll, struct regmap *regmap, + const struct pll_config *config, bool fsm_mode) +{ + clk_pll_configure(pll, regmap, config); + if (fsm_mode) + clk_pll_set_fsm_mode(pll, regmap, 8); +} +EXPORT_SYMBOL_GPL(clk_pll_configure_sr); + void clk_pll_configure_sr_hpm_lp(struct clk_pll *pll, struct regmap *regmap, const struct pll_config *config, bool fsm_mode) { clk_pll_configure(pll, regmap, config); if (fsm_mode) - clk_pll_set_fsm_mode(pll, regmap); + clk_pll_set_fsm_mode(pll, regmap, 0); } EXPORT_SYMBOL_GPL(clk_pll_configure_sr_hpm_lp); diff --git a/drivers/clk/qcom/clk-pll.h b/drivers/clk/qcom/clk-pll.h index 0775a99ca768..3003e9962472 100644 --- a/drivers/clk/qcom/clk-pll.h +++ b/drivers/clk/qcom/clk-pll.h @@ -60,6 +60,8 @@ struct pll_config { u32 aux_output_mask; }; +void clk_pll_configure_sr(struct clk_pll *pll, struct regmap *regmap, + const struct pll_config *config, bool fsm_mode); void clk_pll_configure_sr_hpm_lp(struct clk_pll *pll, struct regmap *regmap, const struct pll_config *config, bool fsm_mode); diff --git a/drivers/clk/qcom/clk-rcg.c b/drivers/clk/qcom/clk-rcg.c index abfc2b675aea..b638c5846dbf 100644 --- a/drivers/clk/qcom/clk-rcg.c +++ b/drivers/clk/qcom/clk-rcg.c @@ -417,20 +417,25 @@ static long clk_dyn_rcg_determine_rate(struct clk_hw *hw, unsigned long rate, return _freq_tbl_determine_rate(hw, rcg->freq_tbl, rate, p_rate, p); } -static int clk_rcg_set_rate(struct clk_hw *hw, unsigned long rate, - unsigned long parent_rate) +static long clk_rcg_bypass_determine_rate(struct clk_hw *hw, unsigned long rate, + unsigned long *p_rate, struct clk **p) { struct clk_rcg *rcg = to_clk_rcg(hw); - const struct freq_tbl *f; + const struct freq_tbl *f = rcg->freq_tbl; + + *p = clk_get_parent_by_index(hw->clk, f->src); + *p_rate = __clk_round_rate(*p, rate); + + return *p_rate; +} + +static int __clk_rcg_set_rate(struct clk_rcg *rcg, const struct freq_tbl *f) +{ u32 ns, md, ctl; struct mn *mn = &rcg->mn; u32 mask = 0; unsigned int reset_reg; - f = find_freq(rcg->freq_tbl, rate); - if (!f) - return -EINVAL; - if (rcg->mn.reset_in_cc) reset_reg = rcg->clkr.enable_reg; else @@ -466,6 +471,27 @@ static int clk_rcg_set_rate(struct clk_hw *hw, unsigned long rate, return 0; } +static int clk_rcg_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct clk_rcg *rcg = to_clk_rcg(hw); + const struct freq_tbl *f; + + f = find_freq(rcg->freq_tbl, rate); + if (!f) + return -EINVAL; + + return __clk_rcg_set_rate(rcg, f); +} + +static int clk_rcg_bypass_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct clk_rcg *rcg = to_clk_rcg(hw); + + return __clk_rcg_set_rate(rcg, rcg->freq_tbl); +} + static int __clk_dyn_rcg_set_rate(struct clk_hw *hw, unsigned long rate) { struct clk_dyn_rcg *rcg = to_clk_dyn_rcg(hw); @@ -503,6 +529,17 @@ const struct clk_ops clk_rcg_ops = { }; EXPORT_SYMBOL_GPL(clk_rcg_ops); +const struct clk_ops clk_rcg_bypass_ops = { + .enable = clk_enable_regmap, + .disable = clk_disable_regmap, + .get_parent = clk_rcg_get_parent, + .set_parent = clk_rcg_set_parent, + .recalc_rate = clk_rcg_recalc_rate, + .determine_rate = clk_rcg_bypass_determine_rate, + .set_rate = clk_rcg_bypass_set_rate, +}; +EXPORT_SYMBOL_GPL(clk_rcg_bypass_ops); + const struct clk_ops clk_dyn_rcg_ops = { .enable = clk_enable_regmap, .is_enabled = clk_is_enabled_regmap, diff --git a/drivers/clk/qcom/clk-rcg.h b/drivers/clk/qcom/clk-rcg.h index b9ec11dfd1b4..ba0523cefd2e 100644 --- a/drivers/clk/qcom/clk-rcg.h +++ b/drivers/clk/qcom/clk-rcg.h @@ -95,6 +95,7 @@ struct clk_rcg { }; extern const struct clk_ops clk_rcg_ops; +extern const struct clk_ops clk_rcg_bypass_ops; #define to_clk_rcg(_hw) container_of(to_clk_regmap(_hw), struct clk_rcg, clkr) diff --git a/drivers/clk/qcom/common.c b/drivers/clk/qcom/common.c index 9b5a1cfc6b91..eeb3eea01f4c 100644 --- a/drivers/clk/qcom/common.c +++ b/drivers/clk/qcom/common.c @@ -27,30 +27,35 @@ struct qcom_cc { struct clk *clks[]; }; -int qcom_cc_probe(struct platform_device *pdev, const struct qcom_cc_desc *desc) +struct regmap * +qcom_cc_map(struct platform_device *pdev, const struct qcom_cc_desc *desc) { void __iomem *base; struct resource *res; + struct device *dev = &pdev->dev; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + base = devm_ioremap_resource(dev, res); + if (IS_ERR(base)) + return ERR_CAST(base); + + return devm_regmap_init_mmio(dev, base, desc->config); +} +EXPORT_SYMBOL_GPL(qcom_cc_map); + +int qcom_cc_really_probe(struct platform_device *pdev, + const struct qcom_cc_desc *desc, struct regmap *regmap) +{ int i, ret; struct device *dev = &pdev->dev; struct clk *clk; struct clk_onecell_data *data; struct clk **clks; - struct regmap *regmap; struct qcom_reset_controller *reset; struct qcom_cc *cc; size_t num_clks = desc->num_clks; struct clk_regmap **rclks = desc->clks; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - base = devm_ioremap_resource(dev, res); - if (IS_ERR(base)) - return PTR_ERR(base); - - regmap = devm_regmap_init_mmio(dev, base, desc->config); - if (IS_ERR(regmap)) - return PTR_ERR(regmap); - cc = devm_kzalloc(dev, sizeof(*cc) + sizeof(*clks) * num_clks, GFP_KERNEL); if (!cc) @@ -91,6 +96,18 @@ int qcom_cc_probe(struct platform_device *pdev, const struct qcom_cc_desc *desc) return ret; } +EXPORT_SYMBOL_GPL(qcom_cc_really_probe); + +int qcom_cc_probe(struct platform_device *pdev, const struct qcom_cc_desc *desc) +{ + struct regmap *regmap; + + regmap = qcom_cc_map(pdev, desc); + if (IS_ERR(regmap)) + return PTR_ERR(regmap); + + return qcom_cc_really_probe(pdev, desc, regmap); +} EXPORT_SYMBOL_GPL(qcom_cc_probe); void qcom_cc_remove(struct platform_device *pdev) diff --git a/drivers/clk/qcom/common.h b/drivers/clk/qcom/common.h index 2c3cfc860348..2765e9d3da97 100644 --- a/drivers/clk/qcom/common.h +++ b/drivers/clk/qcom/common.h @@ -17,6 +17,7 @@ struct platform_device; struct regmap_config; struct clk_regmap; struct qcom_reset_map; +struct regmap; struct qcom_cc_desc { const struct regmap_config *config; @@ -26,6 +27,11 @@ struct qcom_cc_desc { size_t num_resets; }; +extern struct regmap *qcom_cc_map(struct platform_device *pdev, + const struct qcom_cc_desc *desc); +extern int qcom_cc_really_probe(struct platform_device *pdev, + const struct qcom_cc_desc *desc, + struct regmap *regmap); extern int qcom_cc_probe(struct platform_device *pdev, const struct qcom_cc_desc *desc); diff --git a/drivers/clk/qcom/gcc-apq8084.c b/drivers/clk/qcom/gcc-apq8084.c new file mode 100644 index 000000000000..ee52eb1c838a --- /dev/null +++ b/drivers/clk/qcom/gcc-apq8084.c @@ -0,0 +1,3611 @@ +/* + * Copyright (c) 2014, The Linux Foundation. All rights reserved. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include <linux/kernel.h> +#include <linux/bitops.h> +#include <linux/err.h> +#include <linux/platform_device.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_device.h> +#include <linux/clk-provider.h> +#include <linux/regmap.h> +#include <linux/reset-controller.h> + +#include <dt-bindings/clock/qcom,gcc-apq8084.h> +#include <dt-bindings/reset/qcom,gcc-apq8084.h> + +#include "common.h" +#include "clk-regmap.h" +#include "clk-pll.h" +#include "clk-rcg.h" +#include "clk-branch.h" +#include "reset.h" + +#define P_XO 0 +#define P_GPLL0 1 +#define P_GPLL1 1 +#define P_GPLL4 2 +#define P_PCIE_0_1_PIPE_CLK 1 +#define P_SATA_ASIC0_CLK 1 +#define P_SATA_RX_CLK 1 +#define P_SLEEP_CLK 1 + +static const u8 gcc_xo_gpll0_map[] = { + [P_XO] = 0, + [P_GPLL0] = 1, +}; + +static const char *gcc_xo_gpll0[] = { + "xo", + "gpll0_vote", +}; + +static const u8 gcc_xo_gpll0_gpll4_map[] = { + [P_XO] = 0, + [P_GPLL0] = 1, + [P_GPLL4] = 5, +}; + +static const char *gcc_xo_gpll0_gpll4[] = { + "xo", + "gpll0_vote", + "gpll4_vote", +}; + +static const u8 gcc_xo_sata_asic0_map[] = { + [P_XO] = 0, + [P_SATA_ASIC0_CLK] = 2, +}; + +static const char *gcc_xo_sata_asic0[] = { + "xo", + "sata_asic0_clk", +}; + +static const u8 gcc_xo_sata_rx_map[] = { + [P_XO] = 0, + [P_SATA_RX_CLK] = 2, +}; + +static const char *gcc_xo_sata_rx[] = { + "xo", + "sata_rx_clk", +}; + +static const u8 gcc_xo_pcie_map[] = { + [P_XO] = 0, + [P_PCIE_0_1_PIPE_CLK] = 2, +}; + +static const char *gcc_xo_pcie[] = { + "xo", + "pcie_pipe", +}; + +static const u8 gcc_xo_pcie_sleep_map[] = { + [P_XO] = 0, + [P_SLEEP_CLK] = 6, +}; + +static const char *gcc_xo_pcie_sleep[] = { + "xo", + "sleep_clk_src", +}; + +#define F(f, s, h, m, n) { (f), (s), (2 * (h) - 1), (m), (n) } + +static struct clk_pll gpll0 = { + .l_reg = 0x0004, + .m_reg = 0x0008, + .n_reg = 0x000c, + .config_reg = 0x0014, + .mode_reg = 0x0000, + .status_reg = 0x001c, + .status_bit = 17, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gpll0", + .parent_names = (const char *[]){ "xo" }, + .num_parents = 1, + .ops = &clk_pll_ops, + }, +}; + +static struct clk_regmap gpll0_vote = { + .enable_reg = 0x1480, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gpll0_vote", + .parent_names = (const char *[]){ "gpll0" }, + .num_parents = 1, + .ops = &clk_pll_vote_ops, + }, +}; + +static struct clk_rcg2 config_noc_clk_src = { + .cmd_rcgr = 0x0150, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "config_noc_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 periph_noc_clk_src = { + .cmd_rcgr = 0x0190, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "periph_noc_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 system_noc_clk_src = { + .cmd_rcgr = 0x0120, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "system_noc_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_pll gpll1 = { + .l_reg = 0x0044, + .m_reg = 0x0048, + .n_reg = 0x004c, + .config_reg = 0x0054, + .mode_reg = 0x0040, + .status_reg = 0x005c, + .status_bit = 17, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gpll1", + .parent_names = (const char *[]){ "xo" }, + .num_parents = 1, + .ops = &clk_pll_ops, + }, +}; + +static struct clk_regmap gpll1_vote = { + .enable_reg = 0x1480, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gpll1_vote", + .parent_names = (const char *[]){ "gpll1" }, + .num_parents = 1, + .ops = &clk_pll_vote_ops, + }, +}; + +static struct clk_pll gpll4 = { + .l_reg = 0x1dc4, + .m_reg = 0x1dc8, + .n_reg = 0x1dcc, + .config_reg = 0x1dd4, + .mode_reg = 0x1dc0, + .status_reg = 0x1ddc, + .status_bit = 17, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gpll4", + .parent_names = (const char *[]){ "xo" }, + .num_parents = 1, + .ops = &clk_pll_ops, + }, +}; + +static struct clk_regmap gpll4_vote = { + .enable_reg = 0x1480, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "gpll4_vote", + .parent_names = (const char *[]){ "gpll4" }, + .num_parents = 1, + .ops = &clk_pll_vote_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_ufs_axi_clk[] = { + F(100000000, P_GPLL0, 6, 0, 0), + F(200000000, P_GPLL0, 3, 0, 0), + F(240000000, P_GPLL0, 2.5, 0, 0), + { } +}; + +static struct clk_rcg2 ufs_axi_clk_src = { + .cmd_rcgr = 0x1d64, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_ufs_axi_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "ufs_axi_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_usb30_master_clk[] = { + F(125000000, P_GPLL0, 1, 5, 24), + { } +}; + +static struct clk_rcg2 usb30_master_clk_src = { + .cmd_rcgr = 0x03d4, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_usb30_master_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "usb30_master_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_usb30_sec_master_clk[] = { + F(125000000, P_GPLL0, 1, 5, 24), + { } +}; + +static struct clk_rcg2 usb30_sec_master_clk_src = { + .cmd_rcgr = 0x1bd4, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_usb30_sec_master_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "usb30_sec_master_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_branch gcc_usb30_sec_mock_utmi_clk = { + .halt_reg = 0x1bd0, + .clkr = { + .enable_reg = 0x1bd0, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_sec_mock_utmi_clk", + .parent_names = (const char *[]){ + "usb30_sec_mock_utmi_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_sec_sleep_clk = { + .halt_reg = 0x1bcc, + .clkr = { + .enable_reg = 0x1bcc, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_sec_sleep_clk", + .parent_names = (const char *[]){ + "sleep_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static const struct freq_tbl ftbl_gcc_blsp1_2_qup1_6_i2c_apps_clk[] = { + F(19200000, P_XO, 1, 0, 0), + F(50000000, P_GPLL0, 12, 0, 0), + { } +}; + +static struct clk_rcg2 blsp1_qup1_i2c_apps_clk_src = { + .cmd_rcgr = 0x0660, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_blsp1_2_qup1_6_i2c_apps_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp1_qup1_i2c_apps_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_blsp1_2_qup1_6_spi_apps_clk[] = { + F(960000, P_XO, 10, 1, 2), + F(4800000, P_XO, 4, 0, 0), + F(9600000, P_XO, 2, 0, 0), + F(15000000, P_GPLL0, 10, 1, 4), + F(19200000, P_XO, 1, 0, 0), + F(25000000, P_GPLL0, 12, 1, 2), + F(50000000, P_GPLL0, 12, 0, 0), + { } +}; + +static struct clk_rcg2 blsp1_qup1_spi_apps_clk_src = { + .cmd_rcgr = 0x064c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_blsp1_2_qup1_6_spi_apps_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp1_qup1_spi_apps_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp1_qup2_i2c_apps_clk_src = { + .cmd_rcgr = 0x06e0, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_blsp1_2_qup1_6_i2c_apps_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp1_qup2_i2c_apps_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp1_qup2_spi_apps_clk_src = { + .cmd_rcgr = 0x06cc, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_blsp1_2_qup1_6_spi_apps_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp1_qup2_spi_apps_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp1_qup3_i2c_apps_clk_src = { + .cmd_rcgr = 0x0760, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_blsp1_2_qup1_6_i2c_apps_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp1_qup3_i2c_apps_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp1_qup3_spi_apps_clk_src = { + .cmd_rcgr = 0x074c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_blsp1_2_qup1_6_spi_apps_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp1_qup3_spi_apps_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp1_qup4_i2c_apps_clk_src = { + .cmd_rcgr = 0x07e0, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_blsp1_2_qup1_6_i2c_apps_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp1_qup4_i2c_apps_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp1_qup4_spi_apps_clk_src = { + .cmd_rcgr = 0x07cc, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_blsp1_2_qup1_6_spi_apps_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp1_qup4_spi_apps_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp1_qup5_i2c_apps_clk_src = { + .cmd_rcgr = 0x0860, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_blsp1_2_qup1_6_i2c_apps_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp1_qup5_i2c_apps_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp1_qup5_spi_apps_clk_src = { + .cmd_rcgr = 0x084c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_blsp1_2_qup1_6_spi_apps_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp1_qup5_spi_apps_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp1_qup6_i2c_apps_clk_src = { + .cmd_rcgr = 0x08e0, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_blsp1_2_qup1_6_i2c_apps_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp1_qup6_i2c_apps_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp1_qup6_spi_apps_clk_src = { + .cmd_rcgr = 0x08cc, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_blsp1_2_qup1_6_spi_apps_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp1_qup6_spi_apps_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_blsp1_2_uart1_6_apps_clk[] = { + F(3686400, P_GPLL0, 1, 96, 15625), + F(7372800, P_GPLL0, 1, 192, 15625), + F(14745600, P_GPLL0, 1, 384, 15625), + F(16000000, P_GPLL0, 5, 2, 15), + F(19200000, P_XO, 1, 0, 0), + F(24000000, P_GPLL0, 5, 1, 5), + F(32000000, P_GPLL0, 1, 4, 75), + F(40000000, P_GPLL0, 15, 0, 0), + F(46400000, P_GPLL0, 1, 29, 375), + F(48000000, P_GPLL0, 12.5, 0, 0), + F(51200000, P_GPLL0, 1, 32, 375), + F(56000000, P_GPLL0, 1, 7, 75), + F(58982400, P_GPLL0, 1, 1536, 15625), + F(60000000, P_GPLL0, 10, 0, 0), + F(63160000, P_GPLL0, 9.5, 0, 0), + { } +}; + +static struct clk_rcg2 blsp1_uart1_apps_clk_src = { + .cmd_rcgr = 0x068c, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_blsp1_2_uart1_6_apps_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp1_uart1_apps_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp1_uart2_apps_clk_src = { + .cmd_rcgr = 0x070c, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_blsp1_2_uart1_6_apps_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp1_uart2_apps_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp1_uart3_apps_clk_src = { + .cmd_rcgr = 0x078c, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_blsp1_2_uart1_6_apps_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp1_uart3_apps_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp1_uart4_apps_clk_src = { + .cmd_rcgr = 0x080c, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_blsp1_2_uart1_6_apps_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp1_uart4_apps_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp1_uart5_apps_clk_src = { + .cmd_rcgr = 0x088c, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_blsp1_2_uart1_6_apps_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp1_uart5_apps_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp1_uart6_apps_clk_src = { + .cmd_rcgr = 0x090c, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_blsp1_2_uart1_6_apps_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp1_uart6_apps_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp2_qup1_i2c_apps_clk_src = { + .cmd_rcgr = 0x09a0, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_blsp1_2_qup1_6_i2c_apps_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp2_qup1_i2c_apps_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp2_qup1_spi_apps_clk_src = { + .cmd_rcgr = 0x098c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_blsp1_2_qup1_6_spi_apps_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp2_qup1_spi_apps_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp2_qup2_i2c_apps_clk_src = { + .cmd_rcgr = 0x0a20, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_blsp1_2_qup1_6_i2c_apps_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp2_qup2_i2c_apps_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp2_qup2_spi_apps_clk_src = { + .cmd_rcgr = 0x0a0c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_blsp1_2_qup1_6_spi_apps_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp2_qup2_spi_apps_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp2_qup3_i2c_apps_clk_src = { + .cmd_rcgr = 0x0aa0, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_blsp1_2_qup1_6_i2c_apps_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp2_qup3_i2c_apps_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp2_qup3_spi_apps_clk_src = { + .cmd_rcgr = 0x0a8c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_blsp1_2_qup1_6_spi_apps_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp2_qup3_spi_apps_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp2_qup4_i2c_apps_clk_src = { + .cmd_rcgr = 0x0b20, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_blsp1_2_qup1_6_i2c_apps_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp2_qup4_i2c_apps_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp2_qup4_spi_apps_clk_src = { + .cmd_rcgr = 0x0b0c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_blsp1_2_qup1_6_spi_apps_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp2_qup4_spi_apps_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp2_qup5_i2c_apps_clk_src = { + .cmd_rcgr = 0x0ba0, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_blsp1_2_qup1_6_i2c_apps_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp2_qup5_i2c_apps_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp2_qup5_spi_apps_clk_src = { + .cmd_rcgr = 0x0b8c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_blsp1_2_qup1_6_spi_apps_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp2_qup5_spi_apps_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp2_qup6_i2c_apps_clk_src = { + .cmd_rcgr = 0x0c20, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_blsp1_2_qup1_6_i2c_apps_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp2_qup6_i2c_apps_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp2_qup6_spi_apps_clk_src = { + .cmd_rcgr = 0x0c0c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_blsp1_2_qup1_6_spi_apps_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp2_qup6_spi_apps_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp2_uart1_apps_clk_src = { + .cmd_rcgr = 0x09cc, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_blsp1_2_uart1_6_apps_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp2_uart1_apps_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp2_uart2_apps_clk_src = { + .cmd_rcgr = 0x0a4c, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_blsp1_2_uart1_6_apps_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp2_uart2_apps_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp2_uart3_apps_clk_src = { + .cmd_rcgr = 0x0acc, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_blsp1_2_uart1_6_apps_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp2_uart3_apps_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp2_uart4_apps_clk_src = { + .cmd_rcgr = 0x0b4c, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_blsp1_2_uart1_6_apps_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp2_uart4_apps_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp2_uart5_apps_clk_src = { + .cmd_rcgr = 0x0bcc, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_blsp1_2_uart1_6_apps_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp2_uart5_apps_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 blsp2_uart6_apps_clk_src = { + .cmd_rcgr = 0x0c4c, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_blsp1_2_uart1_6_apps_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "blsp2_uart6_apps_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_ce1_clk[] = { + F(50000000, P_GPLL0, 12, 0, 0), + F(85710000, P_GPLL0, 7, 0, 0), + F(100000000, P_GPLL0, 6, 0, 0), + F(171430000, P_GPLL0, 3.5, 0, 0), + { } +}; + +static struct clk_rcg2 ce1_clk_src = { + .cmd_rcgr = 0x1050, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_ce1_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "ce1_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_ce2_clk[] = { + F(50000000, P_GPLL0, 12, 0, 0), + F(85710000, P_GPLL0, 7, 0, 0), + F(100000000, P_GPLL0, 6, 0, 0), + F(171430000, P_GPLL0, 3.5, 0, 0), + { } +}; + +static struct clk_rcg2 ce2_clk_src = { + .cmd_rcgr = 0x1090, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_ce2_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "ce2_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_ce3_clk[] = { + F(50000000, P_GPLL0, 12, 0, 0), + F(85710000, P_GPLL0, 7, 0, 0), + F(100000000, P_GPLL0, 6, 0, 0), + F(171430000, P_GPLL0, 3.5, 0, 0), + { } +}; + +static struct clk_rcg2 ce3_clk_src = { + .cmd_rcgr = 0x1d10, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_ce3_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "ce3_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_gp_clk[] = { + F(19200000, P_XO, 1, 0, 0), + F(100000000, P_GPLL0, 6, 0, 0), + F(200000000, P_GPLL0, 3, 0, 0), + { } +}; + +static struct clk_rcg2 gp1_clk_src = { + .cmd_rcgr = 0x1904, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_gp_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gp1_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gp2_clk_src = { + .cmd_rcgr = 0x1944, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_gp_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gp2_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gp3_clk_src = { + .cmd_rcgr = 0x1984, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_gp_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gp3_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_pcie_0_1_aux_clk[] = { + F(1010000, P_XO, 1, 1, 19), + { } +}; + +static struct clk_rcg2 pcie_0_aux_clk_src = { + .cmd_rcgr = 0x1b2c, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_xo_pcie_sleep_map, + .freq_tbl = ftbl_gcc_pcie_0_1_aux_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "pcie_0_aux_clk_src", + .parent_names = gcc_xo_pcie_sleep, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 pcie_1_aux_clk_src = { + .cmd_rcgr = 0x1bac, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_xo_pcie_sleep_map, + .freq_tbl = ftbl_gcc_pcie_0_1_aux_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "pcie_1_aux_clk_src", + .parent_names = gcc_xo_pcie_sleep, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_pcie_0_1_pipe_clk[] = { + F(125000000, P_PCIE_0_1_PIPE_CLK, 1, 0, 0), + F(250000000, P_PCIE_0_1_PIPE_CLK, 1, 0, 0), + { } +}; + +static struct clk_rcg2 pcie_0_pipe_clk_src = { + .cmd_rcgr = 0x1b18, + .hid_width = 5, + .parent_map = gcc_xo_pcie_map, + .freq_tbl = ftbl_gcc_pcie_0_1_pipe_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "pcie_0_pipe_clk_src", + .parent_names = gcc_xo_pcie, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 pcie_1_pipe_clk_src = { + .cmd_rcgr = 0x1b98, + .hid_width = 5, + .parent_map = gcc_xo_pcie_map, + .freq_tbl = ftbl_gcc_pcie_0_1_pipe_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "pcie_1_pipe_clk_src", + .parent_names = gcc_xo_pcie, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_pdm2_clk[] = { + F(60000000, P_GPLL0, 10, 0, 0), + { } +}; + +static struct clk_rcg2 pdm2_clk_src = { + .cmd_rcgr = 0x0cd0, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_pdm2_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "pdm2_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_sata_asic0_clk[] = { + F(75000000, P_SATA_ASIC0_CLK, 1, 0, 0), + F(150000000, P_SATA_ASIC0_CLK, 1, 0, 0), + F(300000000, P_SATA_ASIC0_CLK, 1, 0, 0), + { } +}; + +static struct clk_rcg2 sata_asic0_clk_src = { + .cmd_rcgr = 0x1c94, + .hid_width = 5, + .parent_map = gcc_xo_sata_asic0_map, + .freq_tbl = ftbl_gcc_sata_asic0_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "sata_asic0_clk_src", + .parent_names = gcc_xo_sata_asic0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_sata_pmalive_clk[] = { + F(19200000, P_XO, 1, 0, 0), + F(50000000, P_GPLL0, 12, 0, 0), + F(100000000, P_GPLL0, 6, 0, 0), + { } +}; + +static struct clk_rcg2 sata_pmalive_clk_src = { + .cmd_rcgr = 0x1c80, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_sata_pmalive_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "sata_pmalive_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_sata_rx_clk[] = { + F(75000000, P_SATA_RX_CLK, 1, 0, 0), + F(150000000, P_SATA_RX_CLK, 1, 0, 0), + F(300000000, P_SATA_RX_CLK, 1, 0, 0), + { } +}; + +static struct clk_rcg2 sata_rx_clk_src = { + .cmd_rcgr = 0x1ca8, + .hid_width = 5, + .parent_map = gcc_xo_sata_rx_map, + .freq_tbl = ftbl_gcc_sata_rx_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "sata_rx_clk_src", + .parent_names = gcc_xo_sata_rx, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_sata_rx_oob_clk[] = { + F(100000000, P_GPLL0, 6, 0, 0), + { } +}; + +static struct clk_rcg2 sata_rx_oob_clk_src = { + .cmd_rcgr = 0x1c5c, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_sata_rx_oob_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "sata_rx_oob_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_sdcc1_4_apps_clk[] = { + F(144000, P_XO, 16, 3, 25), + F(400000, P_XO, 12, 1, 4), + F(20000000, P_GPLL0, 15, 1, 2), + F(25000000, P_GPLL0, 12, 1, 2), + F(50000000, P_GPLL0, 12, 0, 0), + F(100000000, P_GPLL0, 6, 0, 0), + F(192000000, P_GPLL4, 4, 0, 0), + F(200000000, P_GPLL0, 3, 0, 0), + F(384000000, P_GPLL4, 2, 0, 0), + { } +}; + +static struct clk_rcg2 sdcc1_apps_clk_src = { + .cmd_rcgr = 0x04d0, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_gpll4_map, + .freq_tbl = ftbl_gcc_sdcc1_4_apps_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "sdcc1_apps_clk_src", + .parent_names = gcc_xo_gpll0_gpll4, + .num_parents = 3, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 sdcc2_apps_clk_src = { + .cmd_rcgr = 0x0510, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_sdcc1_4_apps_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "sdcc2_apps_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 sdcc3_apps_clk_src = { + .cmd_rcgr = 0x0550, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_sdcc1_4_apps_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "sdcc3_apps_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 sdcc4_apps_clk_src = { + .cmd_rcgr = 0x0590, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_sdcc1_4_apps_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "sdcc4_apps_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_tsif_ref_clk[] = { + F(105000, P_XO, 2, 1, 91), + { } +}; + +static struct clk_rcg2 tsif_ref_clk_src = { + .cmd_rcgr = 0x0d90, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_tsif_ref_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "tsif_ref_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_usb30_mock_utmi_clk[] = { + F(60000000, P_GPLL0, 10, 0, 0), + { } +}; + +static struct clk_rcg2 usb30_mock_utmi_clk_src = { + .cmd_rcgr = 0x03e8, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_usb30_mock_utmi_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "usb30_mock_utmi_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_usb30_sec_mock_utmi_clk[] = { + F(125000000, P_GPLL0, 1, 5, 24), + { } +}; + +static struct clk_rcg2 usb30_sec_mock_utmi_clk_src = { + .cmd_rcgr = 0x1be8, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_usb30_sec_mock_utmi_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "usb30_sec_mock_utmi_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_usb_hs_system_clk[] = { + F(75000000, P_GPLL0, 8, 0, 0), + { } +}; + +static struct clk_rcg2 usb_hs_system_clk_src = { + .cmd_rcgr = 0x0490, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_usb_hs_system_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "usb_hs_system_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_usb_hsic_clk[] = { + F(480000000, P_GPLL1, 1, 0, 0), + { } +}; + +static u8 usb_hsic_clk_src_map[] = { + [P_XO] = 0, + [P_GPLL1] = 4, +}; + +static struct clk_rcg2 usb_hsic_clk_src = { + .cmd_rcgr = 0x0440, + .hid_width = 5, + .parent_map = usb_hsic_clk_src_map, + .freq_tbl = ftbl_gcc_usb_hsic_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "usb_hsic_clk_src", + .parent_names = (const char *[]){ + "xo", + "gpll1_vote", + }, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_usb_hsic_ahb_clk_src[] = { + F(60000000, P_GPLL1, 8, 0, 0), + { } +}; + +static struct clk_rcg2 usb_hsic_ahb_clk_src = { + .cmd_rcgr = 0x046c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = usb_hsic_clk_src_map, + .freq_tbl = ftbl_gcc_usb_hsic_ahb_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "usb_hsic_ahb_clk_src", + .parent_names = (const char *[]){ + "xo", + "gpll1_vote", + }, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_usb_hsic_io_cal_clk[] = { + F(9600000, P_XO, 2, 0, 0), + { } +}; + +static struct clk_rcg2 usb_hsic_io_cal_clk_src = { + .cmd_rcgr = 0x0458, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_usb_hsic_io_cal_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "usb_hsic_io_cal_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 1, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_branch gcc_usb_hsic_mock_utmi_clk = { + .halt_reg = 0x1f14, + .clkr = { + .enable_reg = 0x1f14, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb_hsic_mock_utmi_clk", + .parent_names = (const char *[]){ + "usb_hsic_mock_utmi_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static const struct freq_tbl ftbl_gcc_usb_hsic_mock_utmi_clk[] = { + F(60000000, P_GPLL0, 10, 0, 0), + { } +}; + +static struct clk_rcg2 usb_hsic_mock_utmi_clk_src = { + .cmd_rcgr = 0x1f00, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_usb_hsic_mock_utmi_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "usb_hsic_mock_utmi_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 1, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_usb_hsic_system_clk[] = { + F(75000000, P_GPLL0, 8, 0, 0), + { } +}; + +static struct clk_rcg2 usb_hsic_system_clk_src = { + .cmd_rcgr = 0x041c, + .hid_width = 5, + .parent_map = gcc_xo_gpll0_map, + .freq_tbl = ftbl_gcc_usb_hsic_system_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "usb_hsic_system_clk_src", + .parent_names = gcc_xo_gpll0, + .num_parents = 2, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_branch gcc_bam_dma_ahb_clk = { + .halt_reg = 0x0d44, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x1484, + .enable_mask = BIT(12), + .hw.init = &(struct clk_init_data){ + .name = "gcc_bam_dma_ahb_clk", + .parent_names = (const char *[]){ + "periph_noc_clk_src", + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_ahb_clk = { + .halt_reg = 0x05c4, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x1484, + .enable_mask = BIT(17), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_ahb_clk", + .parent_names = (const char *[]){ + "periph_noc_clk_src", + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_qup1_i2c_apps_clk = { + .halt_reg = 0x0648, + .clkr = { + .enable_reg = 0x0648, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_qup1_i2c_apps_clk", + .parent_names = (const char *[]){ + "blsp1_qup1_i2c_apps_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_qup1_spi_apps_clk = { + .halt_reg = 0x0644, + .clkr = { + .enable_reg = 0x0644, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_qup1_spi_apps_clk", + .parent_names = (const char *[]){ + "blsp1_qup1_spi_apps_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_qup2_i2c_apps_clk = { + .halt_reg = 0x06c8, + .clkr = { + .enable_reg = 0x06c8, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_qup2_i2c_apps_clk", + .parent_names = (const char *[]){ + "blsp1_qup2_i2c_apps_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_qup2_spi_apps_clk = { + .halt_reg = 0x06c4, + .clkr = { + .enable_reg = 0x06c4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_qup2_spi_apps_clk", + .parent_names = (const char *[]){ + "blsp1_qup2_spi_apps_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_qup3_i2c_apps_clk = { + .halt_reg = 0x0748, + .clkr = { + .enable_reg = 0x0748, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_qup3_i2c_apps_clk", + .parent_names = (const char *[]){ + "blsp1_qup3_i2c_apps_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_qup3_spi_apps_clk = { + .halt_reg = 0x0744, + .clkr = { + .enable_reg = 0x0744, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_qup3_spi_apps_clk", + .parent_names = (const char *[]){ + "blsp1_qup3_spi_apps_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_qup4_i2c_apps_clk = { + .halt_reg = 0x07c8, + .clkr = { + .enable_reg = 0x07c8, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_qup4_i2c_apps_clk", + .parent_names = (const char *[]){ + "blsp1_qup4_i2c_apps_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_qup4_spi_apps_clk = { + .halt_reg = 0x07c4, + .clkr = { + .enable_reg = 0x07c4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_qup4_spi_apps_clk", + .parent_names = (const char *[]){ + "blsp1_qup4_spi_apps_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_qup5_i2c_apps_clk = { + .halt_reg = 0x0848, + .clkr = { + .enable_reg = 0x0848, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_qup5_i2c_apps_clk", + .parent_names = (const char *[]){ + "blsp1_qup5_i2c_apps_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_qup5_spi_apps_clk = { + .halt_reg = 0x0844, + .clkr = { + .enable_reg = 0x0844, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_qup5_spi_apps_clk", + .parent_names = (const char *[]){ + "blsp1_qup5_spi_apps_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_qup6_i2c_apps_clk = { + .halt_reg = 0x08c8, + .clkr = { + .enable_reg = 0x08c8, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_qup6_i2c_apps_clk", + .parent_names = (const char *[]){ + "blsp1_qup6_i2c_apps_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_qup6_spi_apps_clk = { + .halt_reg = 0x08c4, + .clkr = { + .enable_reg = 0x08c4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_qup6_spi_apps_clk", + .parent_names = (const char *[]){ + "blsp1_qup6_spi_apps_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_uart1_apps_clk = { + .halt_reg = 0x0684, + .clkr = { + .enable_reg = 0x0684, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_uart1_apps_clk", + .parent_names = (const char *[]){ + "blsp1_uart1_apps_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_uart2_apps_clk = { + .halt_reg = 0x0704, + .clkr = { + .enable_reg = 0x0704, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_uart2_apps_clk", + .parent_names = (const char *[]){ + "blsp1_uart2_apps_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_uart3_apps_clk = { + .halt_reg = 0x0784, + .clkr = { + .enable_reg = 0x0784, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_uart3_apps_clk", + .parent_names = (const char *[]){ + "blsp1_uart3_apps_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_uart4_apps_clk = { + .halt_reg = 0x0804, + .clkr = { + .enable_reg = 0x0804, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_uart4_apps_clk", + .parent_names = (const char *[]){ + "blsp1_uart4_apps_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_uart5_apps_clk = { + .halt_reg = 0x0884, + .clkr = { + .enable_reg = 0x0884, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_uart5_apps_clk", + .parent_names = (const char *[]){ + "blsp1_uart5_apps_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp1_uart6_apps_clk = { + .halt_reg = 0x0904, + .clkr = { + .enable_reg = 0x0904, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp1_uart6_apps_clk", + .parent_names = (const char *[]){ + "blsp1_uart6_apps_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp2_ahb_clk = { + .halt_reg = 0x0944, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x1484, + .enable_mask = BIT(15), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp2_ahb_clk", + .parent_names = (const char *[]){ + "periph_noc_clk_src", + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp2_qup1_i2c_apps_clk = { + .halt_reg = 0x0988, + .clkr = { + .enable_reg = 0x0988, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp2_qup1_i2c_apps_clk", + .parent_names = (const char *[]){ + "blsp2_qup1_i2c_apps_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp2_qup1_spi_apps_clk = { + .halt_reg = 0x0984, + .clkr = { + .enable_reg = 0x0984, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp2_qup1_spi_apps_clk", + .parent_names = (const char *[]){ + "blsp2_qup1_spi_apps_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp2_qup2_i2c_apps_clk = { + .halt_reg = 0x0a08, + .clkr = { + .enable_reg = 0x0a08, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp2_qup2_i2c_apps_clk", + .parent_names = (const char *[]){ + "blsp2_qup2_i2c_apps_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp2_qup2_spi_apps_clk = { + .halt_reg = 0x0a04, + .clkr = { + .enable_reg = 0x0a04, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp2_qup2_spi_apps_clk", + .parent_names = (const char *[]){ + "blsp2_qup2_spi_apps_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp2_qup3_i2c_apps_clk = { + .halt_reg = 0x0a88, + .clkr = { + .enable_reg = 0x0a88, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp2_qup3_i2c_apps_clk", + .parent_names = (const char *[]){ + "blsp2_qup3_i2c_apps_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp2_qup3_spi_apps_clk = { + .halt_reg = 0x0a84, + .clkr = { + .enable_reg = 0x0a84, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp2_qup3_spi_apps_clk", + .parent_names = (const char *[]){ + "blsp2_qup3_spi_apps_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp2_qup4_i2c_apps_clk = { + .halt_reg = 0x0b08, + .clkr = { + .enable_reg = 0x0b08, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp2_qup4_i2c_apps_clk", + .parent_names = (const char *[]){ + "blsp2_qup4_i2c_apps_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp2_qup4_spi_apps_clk = { + .halt_reg = 0x0b04, + .clkr = { + .enable_reg = 0x0b04, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp2_qup4_spi_apps_clk", + .parent_names = (const char *[]){ + "blsp2_qup4_spi_apps_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp2_qup5_i2c_apps_clk = { + .halt_reg = 0x0b88, + .clkr = { + .enable_reg = 0x0b88, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp2_qup5_i2c_apps_clk", + .parent_names = (const char *[]){ + "blsp2_qup5_i2c_apps_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp2_qup5_spi_apps_clk = { + .halt_reg = 0x0b84, + .clkr = { + .enable_reg = 0x0b84, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp2_qup5_spi_apps_clk", + .parent_names = (const char *[]){ + "blsp2_qup5_spi_apps_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp2_qup6_i2c_apps_clk = { + .halt_reg = 0x0c08, + .clkr = { + .enable_reg = 0x0c08, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp2_qup6_i2c_apps_clk", + .parent_names = (const char *[]){ + "blsp2_qup6_i2c_apps_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp2_qup6_spi_apps_clk = { + .halt_reg = 0x0c04, + .clkr = { + .enable_reg = 0x0c04, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp2_qup6_spi_apps_clk", + .parent_names = (const char *[]){ + "blsp2_qup6_spi_apps_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp2_uart1_apps_clk = { + .halt_reg = 0x09c4, + .clkr = { + .enable_reg = 0x09c4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp2_uart1_apps_clk", + .parent_names = (const char *[]){ + "blsp2_uart1_apps_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp2_uart2_apps_clk = { + .halt_reg = 0x0a44, + .clkr = { + .enable_reg = 0x0a44, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp2_uart2_apps_clk", + .parent_names = (const char *[]){ + "blsp2_uart2_apps_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp2_uart3_apps_clk = { + .halt_reg = 0x0ac4, + .clkr = { + .enable_reg = 0x0ac4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp2_uart3_apps_clk", + .parent_names = (const char *[]){ + "blsp2_uart3_apps_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp2_uart4_apps_clk = { + .halt_reg = 0x0b44, + .clkr = { + .enable_reg = 0x0b44, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp2_uart4_apps_clk", + .parent_names = (const char *[]){ + "blsp2_uart4_apps_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp2_uart5_apps_clk = { + .halt_reg = 0x0bc4, + .clkr = { + .enable_reg = 0x0bc4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp2_uart5_apps_clk", + .parent_names = (const char *[]){ + "blsp2_uart5_apps_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_blsp2_uart6_apps_clk = { + .halt_reg = 0x0c44, + .clkr = { + .enable_reg = 0x0c44, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_blsp2_uart6_apps_clk", + .parent_names = (const char *[]){ + "blsp2_uart6_apps_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_boot_rom_ahb_clk = { + .halt_reg = 0x0e04, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x1484, + .enable_mask = BIT(10), + .hw.init = &(struct clk_init_data){ + .name = "gcc_boot_rom_ahb_clk", + .parent_names = (const char *[]){ + "config_noc_clk_src", + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ce1_ahb_clk = { + .halt_reg = 0x104c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x1484, + .enable_mask = BIT(3), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ce1_ahb_clk", + .parent_names = (const char *[]){ + "config_noc_clk_src", + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ce1_axi_clk = { + .halt_reg = 0x1048, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x1484, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ce1_axi_clk", + .parent_names = (const char *[]){ + "system_noc_clk_src", + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ce1_clk = { + .halt_reg = 0x1050, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x1484, + .enable_mask = BIT(5), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ce1_clk", + .parent_names = (const char *[]){ + "ce1_clk_src", + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ce2_ahb_clk = { + .halt_reg = 0x108c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x1484, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ce2_ahb_clk", + .parent_names = (const char *[]){ + "config_noc_clk_src", + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ce2_axi_clk = { + .halt_reg = 0x1088, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x1484, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ce2_axi_clk", + .parent_names = (const char *[]){ + "system_noc_clk_src", + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ce2_clk = { + .halt_reg = 0x1090, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x1484, + .enable_mask = BIT(2), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ce2_clk", + .parent_names = (const char *[]){ + "ce2_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ce3_ahb_clk = { + .halt_reg = 0x1d0c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x1d0c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ce3_ahb_clk", + .parent_names = (const char *[]){ + "config_noc_clk_src", + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ce3_axi_clk = { + .halt_reg = 0x1088, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x1d08, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ce3_axi_clk", + .parent_names = (const char *[]){ + "system_noc_clk_src", + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ce3_clk = { + .halt_reg = 0x1090, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x1d04, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ce3_clk", + .parent_names = (const char *[]){ + "ce3_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gp1_clk = { + .halt_reg = 0x1900, + .clkr = { + .enable_reg = 0x1900, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gp1_clk", + .parent_names = (const char *[]){ + "gp1_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gp2_clk = { + .halt_reg = 0x1940, + .clkr = { + .enable_reg = 0x1940, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gp2_clk", + .parent_names = (const char *[]){ + "gp2_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gp3_clk = { + .halt_reg = 0x1980, + .clkr = { + .enable_reg = 0x1980, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gp3_clk", + .parent_names = (const char *[]){ + "gp3_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ocmem_noc_cfg_ahb_clk = { + .halt_reg = 0x0248, + .clkr = { + .enable_reg = 0x0248, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ocmem_noc_cfg_ahb_clk", + .parent_names = (const char *[]){ + "config_noc_clk_src", + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_aux_clk = { + .halt_reg = 0x1b10, + .clkr = { + .enable_reg = 0x1b10, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_aux_clk", + .parent_names = (const char *[]){ + "pcie_0_aux_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_cfg_ahb_clk = { + .halt_reg = 0x1b0c, + .clkr = { + .enable_reg = 0x1b0c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_cfg_ahb_clk", + .parent_names = (const char *[]){ + "config_noc_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_mstr_axi_clk = { + .halt_reg = 0x1b08, + .clkr = { + .enable_reg = 0x1b08, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_mstr_axi_clk", + .parent_names = (const char *[]){ + "config_noc_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_pipe_clk = { + .halt_reg = 0x1b14, + .clkr = { + .enable_reg = 0x1b14, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_pipe_clk", + .parent_names = (const char *[]){ + "pcie_0_pipe_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_slv_axi_clk = { + .halt_reg = 0x1b04, + .clkr = { + .enable_reg = 0x1b04, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_slv_axi_clk", + .parent_names = (const char *[]){ + "config_noc_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_aux_clk = { + .halt_reg = 0x1b90, + .clkr = { + .enable_reg = 0x1b90, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_aux_clk", + .parent_names = (const char *[]){ + "pcie_1_aux_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_cfg_ahb_clk = { + .halt_reg = 0x1b8c, + .clkr = { + .enable_reg = 0x1b8c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_cfg_ahb_clk", + .parent_names = (const char *[]){ + "config_noc_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_mstr_axi_clk = { + .halt_reg = 0x1b88, + .clkr = { + .enable_reg = 0x1b88, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_mstr_axi_clk", + .parent_names = (const char *[]){ + "config_noc_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_pipe_clk = { + .halt_reg = 0x1b94, + .clkr = { + .enable_reg = 0x1b94, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_pipe_clk", + .parent_names = (const char *[]){ + "pcie_1_pipe_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_slv_axi_clk = { + .halt_reg = 0x1b84, + .clkr = { + .enable_reg = 0x1b84, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_slv_axi_clk", + .parent_names = (const char *[]){ + "config_noc_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pdm2_clk = { + .halt_reg = 0x0ccc, + .clkr = { + .enable_reg = 0x0ccc, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pdm2_clk", + .parent_names = (const char *[]){ + "pdm2_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pdm_ahb_clk = { + .halt_reg = 0x0cc4, + .clkr = { + .enable_reg = 0x0cc4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pdm_ahb_clk", + .parent_names = (const char *[]){ + "periph_noc_clk_src", + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_periph_noc_usb_hsic_ahb_clk = { + .halt_reg = 0x01a4, + .clkr = { + .enable_reg = 0x01a4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_periph_noc_usb_hsic_ahb_clk", + .parent_names = (const char *[]){ + "usb_hsic_ahb_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_prng_ahb_clk = { + .halt_reg = 0x0d04, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x1484, + .enable_mask = BIT(13), + .hw.init = &(struct clk_init_data){ + .name = "gcc_prng_ahb_clk", + .parent_names = (const char *[]){ + "periph_noc_clk_src", + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sata_asic0_clk = { + .halt_reg = 0x1c54, + .clkr = { + .enable_reg = 0x1c54, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sata_asic0_clk", + .parent_names = (const char *[]){ + "sata_asic0_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sata_axi_clk = { + .halt_reg = 0x1c44, + .clkr = { + .enable_reg = 0x1c44, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sata_axi_clk", + .parent_names = (const char *[]){ + "config_noc_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sata_cfg_ahb_clk = { + .halt_reg = 0x1c48, + .clkr = { + .enable_reg = 0x1c48, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sata_cfg_ahb_clk", + .parent_names = (const char *[]){ + "config_noc_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sata_pmalive_clk = { + .halt_reg = 0x1c50, + .clkr = { + .enable_reg = 0x1c50, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sata_pmalive_clk", + .parent_names = (const char *[]){ + "sata_pmalive_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sata_rx_clk = { + .halt_reg = 0x1c58, + .clkr = { + .enable_reg = 0x1c58, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sata_rx_clk", + .parent_names = (const char *[]){ + "sata_rx_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sata_rx_oob_clk = { + .halt_reg = 0x1c4c, + .clkr = { + .enable_reg = 0x1c4c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sata_rx_oob_clk", + .parent_names = (const char *[]){ + "sata_rx_oob_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc1_ahb_clk = { + .halt_reg = 0x04c8, + .clkr = { + .enable_reg = 0x04c8, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc1_ahb_clk", + .parent_names = (const char *[]){ + "periph_noc_clk_src", + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc1_apps_clk = { + .halt_reg = 0x04c4, + .clkr = { + .enable_reg = 0x04c4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc1_apps_clk", + .parent_names = (const char *[]){ + "sdcc1_apps_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc1_cdccal_ff_clk = { + .halt_reg = 0x04e8, + .clkr = { + .enable_reg = 0x04e8, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc1_cdccal_ff_clk", + .parent_names = (const char *[]){ + "xo" + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc1_cdccal_sleep_clk = { + .halt_reg = 0x04e4, + .clkr = { + .enable_reg = 0x04e4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc1_cdccal_sleep_clk", + .parent_names = (const char *[]){ + "sleep_clk_src" + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc2_ahb_clk = { + .halt_reg = 0x0508, + .clkr = { + .enable_reg = 0x0508, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc2_ahb_clk", + .parent_names = (const char *[]){ + "periph_noc_clk_src", + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc2_apps_clk = { + .halt_reg = 0x0504, + .clkr = { + .enable_reg = 0x0504, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc2_apps_clk", + .parent_names = (const char *[]){ + "sdcc2_apps_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc3_ahb_clk = { + .halt_reg = 0x0548, + .clkr = { + .enable_reg = 0x0548, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc3_ahb_clk", + .parent_names = (const char *[]){ + "periph_noc_clk_src", + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc3_apps_clk = { + .halt_reg = 0x0544, + .clkr = { + .enable_reg = 0x0544, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc3_apps_clk", + .parent_names = (const char *[]){ + "sdcc3_apps_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc4_ahb_clk = { + .halt_reg = 0x0588, + .clkr = { + .enable_reg = 0x0588, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc4_ahb_clk", + .parent_names = (const char *[]){ + "periph_noc_clk_src", + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc4_apps_clk = { + .halt_reg = 0x0584, + .clkr = { + .enable_reg = 0x0584, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc4_apps_clk", + .parent_names = (const char *[]){ + "sdcc4_apps_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sys_noc_ufs_axi_clk = { + .halt_reg = 0x013c, + .clkr = { + .enable_reg = 0x013c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sys_noc_ufs_axi_clk", + .parent_names = (const char *[]){ + "ufs_axi_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sys_noc_usb3_axi_clk = { + .halt_reg = 0x0108, + .clkr = { + .enable_reg = 0x0108, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sys_noc_usb3_axi_clk", + .parent_names = (const char *[]){ + "usb30_master_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sys_noc_usb3_sec_axi_clk = { + .halt_reg = 0x0138, + .clkr = { + .enable_reg = 0x0138, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sys_noc_usb3_sec_axi_clk", + .parent_names = (const char *[]){ + "usb30_sec_master_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_tsif_ahb_clk = { + .halt_reg = 0x0d84, + .clkr = { + .enable_reg = 0x0d84, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_tsif_ahb_clk", + .parent_names = (const char *[]){ + "periph_noc_clk_src", + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_tsif_inactivity_timers_clk = { + .halt_reg = 0x0d8c, + .clkr = { + .enable_reg = 0x0d8c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_tsif_inactivity_timers_clk", + .parent_names = (const char *[]){ + "sleep_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_tsif_ref_clk = { + .halt_reg = 0x0d88, + .clkr = { + .enable_reg = 0x0d88, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_tsif_ref_clk", + .parent_names = (const char *[]){ + "tsif_ref_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_ahb_clk = { + .halt_reg = 0x1d48, + .clkr = { + .enable_reg = 0x1d48, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_ahb_clk", + .parent_names = (const char *[]){ + "config_noc_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_axi_clk = { + .halt_reg = 0x1d44, + .clkr = { + .enable_reg = 0x1d44, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_axi_clk", + .parent_names = (const char *[]){ + "ufs_axi_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_rx_cfg_clk = { + .halt_reg = 0x1d50, + .clkr = { + .enable_reg = 0x1d50, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_rx_cfg_clk", + .parent_names = (const char *[]){ + "ufs_axi_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_rx_symbol_0_clk = { + .halt_reg = 0x1d5c, + .clkr = { + .enable_reg = 0x1d5c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_rx_symbol_0_clk", + .parent_names = (const char *[]){ + "ufs_rx_symbol_0_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_rx_symbol_1_clk = { + .halt_reg = 0x1d60, + .clkr = { + .enable_reg = 0x1d60, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_rx_symbol_1_clk", + .parent_names = (const char *[]){ + "ufs_rx_symbol_1_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_tx_cfg_clk = { + .halt_reg = 0x1d4c, + .clkr = { + .enable_reg = 0x1d4c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_tx_cfg_clk", + .parent_names = (const char *[]){ + "ufs_axi_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_tx_symbol_0_clk = { + .halt_reg = 0x1d54, + .clkr = { + .enable_reg = 0x1d54, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_tx_symbol_0_clk", + .parent_names = (const char *[]){ + "ufs_tx_symbol_0_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_tx_symbol_1_clk = { + .halt_reg = 0x1d58, + .clkr = { + .enable_reg = 0x1d58, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_tx_symbol_1_clk", + .parent_names = (const char *[]){ + "ufs_tx_symbol_1_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb2a_phy_sleep_clk = { + .halt_reg = 0x04ac, + .clkr = { + .enable_reg = 0x04ac, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb2a_phy_sleep_clk", + .parent_names = (const char *[]){ + "sleep_clk_src", + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb2b_phy_sleep_clk = { + .halt_reg = 0x04b4, + .clkr = { + .enable_reg = 0x04b4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb2b_phy_sleep_clk", + .parent_names = (const char *[]){ + "sleep_clk_src", + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_master_clk = { + .halt_reg = 0x03c8, + .clkr = { + .enable_reg = 0x03c8, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_master_clk", + .parent_names = (const char *[]){ + "usb30_master_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_sec_master_clk = { + .halt_reg = 0x1bc8, + .clkr = { + .enable_reg = 0x1bc8, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_sec_master_clk", + .parent_names = (const char *[]){ + "usb30_sec_master_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_mock_utmi_clk = { + .halt_reg = 0x03d0, + .clkr = { + .enable_reg = 0x03d0, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_mock_utmi_clk", + .parent_names = (const char *[]){ + "usb30_mock_utmi_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_sleep_clk = { + .halt_reg = 0x03cc, + .clkr = { + .enable_reg = 0x03cc, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_sleep_clk", + .parent_names = (const char *[]){ + "sleep_clk_src", + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb_hs_ahb_clk = { + .halt_reg = 0x0488, + .clkr = { + .enable_reg = 0x0488, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb_hs_ahb_clk", + .parent_names = (const char *[]){ + "periph_noc_clk_src", + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb_hs_inactivity_timers_clk = { + .halt_reg = 0x048c, + .clkr = { + .enable_reg = 0x048c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb_hs_inactivity_timers_clk", + .parent_names = (const char *[]){ + "sleep_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb_hs_system_clk = { + .halt_reg = 0x0484, + .clkr = { + .enable_reg = 0x0484, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb_hs_system_clk", + .parent_names = (const char *[]){ + "usb_hs_system_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb_hsic_ahb_clk = { + .halt_reg = 0x0408, + .clkr = { + .enable_reg = 0x0408, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb_hsic_ahb_clk", + .parent_names = (const char *[]){ + "periph_noc_clk_src", + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb_hsic_clk = { + .halt_reg = 0x0410, + .clkr = { + .enable_reg = 0x0410, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb_hsic_clk", + .parent_names = (const char *[]){ + "usb_hsic_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb_hsic_io_cal_clk = { + .halt_reg = 0x0414, + .clkr = { + .enable_reg = 0x0414, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb_hsic_io_cal_clk", + .parent_names = (const char *[]){ + "usb_hsic_io_cal_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb_hsic_io_cal_sleep_clk = { + .halt_reg = 0x0418, + .clkr = { + .enable_reg = 0x0418, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb_hsic_io_cal_sleep_clk", + .parent_names = (const char *[]){ + "sleep_clk_src", + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb_hsic_system_clk = { + .halt_reg = 0x040c, + .clkr = { + .enable_reg = 0x040c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb_hsic_system_clk", + .parent_names = (const char *[]){ + "usb_hsic_system_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_regmap *gcc_apq8084_clocks[] = { + [GPLL0] = &gpll0.clkr, + [GPLL0_VOTE] = &gpll0_vote, + [GPLL1] = &gpll1.clkr, + [GPLL1_VOTE] = &gpll1_vote, + [GPLL4] = &gpll4.clkr, + [GPLL4_VOTE] = &gpll4_vote, + [CONFIG_NOC_CLK_SRC] = &config_noc_clk_src.clkr, + [PERIPH_NOC_CLK_SRC] = &periph_noc_clk_src.clkr, + [SYSTEM_NOC_CLK_SRC] = &system_noc_clk_src.clkr, + [UFS_AXI_CLK_SRC] = &ufs_axi_clk_src.clkr, + [USB30_MASTER_CLK_SRC] = &usb30_master_clk_src.clkr, + [USB30_SEC_MASTER_CLK_SRC] = &usb30_sec_master_clk_src.clkr, + [USB_HSIC_AHB_CLK_SRC] = &usb_hsic_ahb_clk_src.clkr, + [BLSP1_QUP1_I2C_APPS_CLK_SRC] = &blsp1_qup1_i2c_apps_clk_src.clkr, + [BLSP1_QUP1_SPI_APPS_CLK_SRC] = &blsp1_qup1_spi_apps_clk_src.clkr, + [BLSP1_QUP2_I2C_APPS_CLK_SRC] = &blsp1_qup2_i2c_apps_clk_src.clkr, + [BLSP1_QUP2_SPI_APPS_CLK_SRC] = &blsp1_qup2_spi_apps_clk_src.clkr, + [BLSP1_QUP3_I2C_APPS_CLK_SRC] = &blsp1_qup3_i2c_apps_clk_src.clkr, + [BLSP1_QUP3_SPI_APPS_CLK_SRC] = &blsp1_qup3_spi_apps_clk_src.clkr, + [BLSP1_QUP4_I2C_APPS_CLK_SRC] = &blsp1_qup4_i2c_apps_clk_src.clkr, + [BLSP1_QUP4_SPI_APPS_CLK_SRC] = &blsp1_qup4_spi_apps_clk_src.clkr, + [BLSP1_QUP5_I2C_APPS_CLK_SRC] = &blsp1_qup5_i2c_apps_clk_src.clkr, + [BLSP1_QUP5_SPI_APPS_CLK_SRC] = &blsp1_qup5_spi_apps_clk_src.clkr, + [BLSP1_QUP6_I2C_APPS_CLK_SRC] = &blsp1_qup6_i2c_apps_clk_src.clkr, + [BLSP1_QUP6_SPI_APPS_CLK_SRC] = &blsp1_qup6_spi_apps_clk_src.clkr, + [BLSP1_UART1_APPS_CLK_SRC] = &blsp1_uart1_apps_clk_src.clkr, + [BLSP1_UART2_APPS_CLK_SRC] = &blsp1_uart2_apps_clk_src.clkr, + [BLSP1_UART3_APPS_CLK_SRC] = &blsp1_uart3_apps_clk_src.clkr, + [BLSP1_UART4_APPS_CLK_SRC] = &blsp1_uart4_apps_clk_src.clkr, + [BLSP1_UART5_APPS_CLK_SRC] = &blsp1_uart5_apps_clk_src.clkr, + [BLSP1_UART6_APPS_CLK_SRC] = &blsp1_uart6_apps_clk_src.clkr, + [BLSP2_QUP1_I2C_APPS_CLK_SRC] = &blsp2_qup1_i2c_apps_clk_src.clkr, + [BLSP2_QUP1_SPI_APPS_CLK_SRC] = &blsp2_qup1_spi_apps_clk_src.clkr, + [BLSP2_QUP2_I2C_APPS_CLK_SRC] = &blsp2_qup2_i2c_apps_clk_src.clkr, + [BLSP2_QUP2_SPI_APPS_CLK_SRC] = &blsp2_qup2_spi_apps_clk_src.clkr, + [BLSP2_QUP3_I2C_APPS_CLK_SRC] = &blsp2_qup3_i2c_apps_clk_src.clkr, + [BLSP2_QUP3_SPI_APPS_CLK_SRC] = &blsp2_qup3_spi_apps_clk_src.clkr, + [BLSP2_QUP4_I2C_APPS_CLK_SRC] = &blsp2_qup4_i2c_apps_clk_src.clkr, + [BLSP2_QUP4_SPI_APPS_CLK_SRC] = &blsp2_qup4_spi_apps_clk_src.clkr, + [BLSP2_QUP5_I2C_APPS_CLK_SRC] = &blsp2_qup5_i2c_apps_clk_src.clkr, + [BLSP2_QUP5_SPI_APPS_CLK_SRC] = &blsp2_qup5_spi_apps_clk_src.clkr, + [BLSP2_QUP6_I2C_APPS_CLK_SRC] = &blsp2_qup6_i2c_apps_clk_src.clkr, + [BLSP2_QUP6_SPI_APPS_CLK_SRC] = &blsp2_qup6_spi_apps_clk_src.clkr, + [BLSP2_UART1_APPS_CLK_SRC] = &blsp2_uart1_apps_clk_src.clkr, + [BLSP2_UART2_APPS_CLK_SRC] = &blsp2_uart2_apps_clk_src.clkr, + [BLSP2_UART3_APPS_CLK_SRC] = &blsp2_uart3_apps_clk_src.clkr, + [BLSP2_UART4_APPS_CLK_SRC] = &blsp2_uart4_apps_clk_src.clkr, + [BLSP2_UART5_APPS_CLK_SRC] = &blsp2_uart5_apps_clk_src.clkr, + [BLSP2_UART6_APPS_CLK_SRC] = &blsp2_uart6_apps_clk_src.clkr, + [CE1_CLK_SRC] = &ce1_clk_src.clkr, + [CE2_CLK_SRC] = &ce2_clk_src.clkr, + [CE3_CLK_SRC] = &ce3_clk_src.clkr, + [GP1_CLK_SRC] = &gp1_clk_src.clkr, + [GP2_CLK_SRC] = &gp2_clk_src.clkr, + [GP3_CLK_SRC] = &gp3_clk_src.clkr, + [PCIE_0_AUX_CLK_SRC] = &pcie_0_aux_clk_src.clkr, + [PCIE_0_PIPE_CLK_SRC] = &pcie_0_pipe_clk_src.clkr, + [PCIE_1_AUX_CLK_SRC] = &pcie_1_aux_clk_src.clkr, + [PCIE_1_PIPE_CLK_SRC] = &pcie_1_pipe_clk_src.clkr, + [PDM2_CLK_SRC] = &pdm2_clk_src.clkr, + [SATA_ASIC0_CLK_SRC] = &sata_asic0_clk_src.clkr, + [SATA_PMALIVE_CLK_SRC] = &sata_pmalive_clk_src.clkr, + [SATA_RX_CLK_SRC] = &sata_rx_clk_src.clkr, + [SATA_RX_OOB_CLK_SRC] = &sata_rx_oob_clk_src.clkr, + [SDCC1_APPS_CLK_SRC] = &sdcc1_apps_clk_src.clkr, + [SDCC2_APPS_CLK_SRC] = &sdcc2_apps_clk_src.clkr, + [SDCC3_APPS_CLK_SRC] = &sdcc3_apps_clk_src.clkr, + [SDCC4_APPS_CLK_SRC] = &sdcc4_apps_clk_src.clkr, + [TSIF_REF_CLK_SRC] = &tsif_ref_clk_src.clkr, + [USB30_MOCK_UTMI_CLK_SRC] = &usb30_mock_utmi_clk_src.clkr, + [USB30_SEC_MOCK_UTMI_CLK_SRC] = &usb30_sec_mock_utmi_clk_src.clkr, + [USB_HS_SYSTEM_CLK_SRC] = &usb_hs_system_clk_src.clkr, + [USB_HSIC_CLK_SRC] = &usb_hsic_clk_src.clkr, + [USB_HSIC_IO_CAL_CLK_SRC] = &usb_hsic_io_cal_clk_src.clkr, + [USB_HSIC_MOCK_UTMI_CLK_SRC] = &usb_hsic_mock_utmi_clk_src.clkr, + [USB_HSIC_SYSTEM_CLK_SRC] = &usb_hsic_system_clk_src.clkr, + [GCC_BAM_DMA_AHB_CLK] = &gcc_bam_dma_ahb_clk.clkr, + [GCC_BLSP1_AHB_CLK] = &gcc_blsp1_ahb_clk.clkr, + [GCC_BLSP1_QUP1_I2C_APPS_CLK] = &gcc_blsp1_qup1_i2c_apps_clk.clkr, + [GCC_BLSP1_QUP1_SPI_APPS_CLK] = &gcc_blsp1_qup1_spi_apps_clk.clkr, + [GCC_BLSP1_QUP2_I2C_APPS_CLK] = &gcc_blsp1_qup2_i2c_apps_clk.clkr, + [GCC_BLSP1_QUP2_SPI_APPS_CLK] = &gcc_blsp1_qup2_spi_apps_clk.clkr, + [GCC_BLSP1_QUP3_I2C_APPS_CLK] = &gcc_blsp1_qup3_i2c_apps_clk.clkr, + [GCC_BLSP1_QUP3_SPI_APPS_CLK] = &gcc_blsp1_qup3_spi_apps_clk.clkr, + [GCC_BLSP1_QUP4_I2C_APPS_CLK] = &gcc_blsp1_qup4_i2c_apps_clk.clkr, + [GCC_BLSP1_QUP4_SPI_APPS_CLK] = &gcc_blsp1_qup4_spi_apps_clk.clkr, + [GCC_BLSP1_QUP5_I2C_APPS_CLK] = &gcc_blsp1_qup5_i2c_apps_clk.clkr, + [GCC_BLSP1_QUP5_SPI_APPS_CLK] = &gcc_blsp1_qup5_spi_apps_clk.clkr, + [GCC_BLSP1_QUP6_I2C_APPS_CLK] = &gcc_blsp1_qup6_i2c_apps_clk.clkr, + [GCC_BLSP1_QUP6_SPI_APPS_CLK] = &gcc_blsp1_qup6_spi_apps_clk.clkr, + [GCC_BLSP1_UART1_APPS_CLK] = &gcc_blsp1_uart1_apps_clk.clkr, + [GCC_BLSP1_UART2_APPS_CLK] = &gcc_blsp1_uart2_apps_clk.clkr, + [GCC_BLSP1_UART3_APPS_CLK] = &gcc_blsp1_uart3_apps_clk.clkr, + [GCC_BLSP1_UART4_APPS_CLK] = &gcc_blsp1_uart4_apps_clk.clkr, + [GCC_BLSP1_UART5_APPS_CLK] = &gcc_blsp1_uart5_apps_clk.clkr, + [GCC_BLSP1_UART6_APPS_CLK] = &gcc_blsp1_uart6_apps_clk.clkr, + [GCC_BLSP2_AHB_CLK] = &gcc_blsp2_ahb_clk.clkr, + [GCC_BLSP2_QUP1_I2C_APPS_CLK] = &gcc_blsp2_qup1_i2c_apps_clk.clkr, + [GCC_BLSP2_QUP1_SPI_APPS_CLK] = &gcc_blsp2_qup1_spi_apps_clk.clkr, + [GCC_BLSP2_QUP2_I2C_APPS_CLK] = &gcc_blsp2_qup2_i2c_apps_clk.clkr, + [GCC_BLSP2_QUP2_SPI_APPS_CLK] = &gcc_blsp2_qup2_spi_apps_clk.clkr, + [GCC_BLSP2_QUP3_I2C_APPS_CLK] = &gcc_blsp2_qup3_i2c_apps_clk.clkr, + [GCC_BLSP2_QUP3_SPI_APPS_CLK] = &gcc_blsp2_qup3_spi_apps_clk.clkr, + [GCC_BLSP2_QUP4_I2C_APPS_CLK] = &gcc_blsp2_qup4_i2c_apps_clk.clkr, + [GCC_BLSP2_QUP4_SPI_APPS_CLK] = &gcc_blsp2_qup4_spi_apps_clk.clkr, + [GCC_BLSP2_QUP5_I2C_APPS_CLK] = &gcc_blsp2_qup5_i2c_apps_clk.clkr, + [GCC_BLSP2_QUP5_SPI_APPS_CLK] = &gcc_blsp2_qup5_spi_apps_clk.clkr, + [GCC_BLSP2_QUP6_I2C_APPS_CLK] = &gcc_blsp2_qup6_i2c_apps_clk.clkr, + [GCC_BLSP2_QUP6_SPI_APPS_CLK] = &gcc_blsp2_qup6_spi_apps_clk.clkr, + [GCC_BLSP2_UART1_APPS_CLK] = &gcc_blsp2_uart1_apps_clk.clkr, + [GCC_BLSP2_UART2_APPS_CLK] = &gcc_blsp2_uart2_apps_clk.clkr, + [GCC_BLSP2_UART3_APPS_CLK] = &gcc_blsp2_uart3_apps_clk.clkr, + [GCC_BLSP2_UART4_APPS_CLK] = &gcc_blsp2_uart4_apps_clk.clkr, + [GCC_BLSP2_UART5_APPS_CLK] = &gcc_blsp2_uart5_apps_clk.clkr, + [GCC_BLSP2_UART6_APPS_CLK] = &gcc_blsp2_uart6_apps_clk.clkr, + [GCC_BOOT_ROM_AHB_CLK] = &gcc_boot_rom_ahb_clk.clkr, + [GCC_CE1_AHB_CLK] = &gcc_ce1_ahb_clk.clkr, + [GCC_CE1_AXI_CLK] = &gcc_ce1_axi_clk.clkr, + [GCC_CE1_CLK] = &gcc_ce1_clk.clkr, + [GCC_CE2_AHB_CLK] = &gcc_ce2_ahb_clk.clkr, + [GCC_CE2_AXI_CLK] = &gcc_ce2_axi_clk.clkr, + [GCC_CE2_CLK] = &gcc_ce2_clk.clkr, + [GCC_CE3_AHB_CLK] = &gcc_ce3_ahb_clk.clkr, + [GCC_CE3_AXI_CLK] = &gcc_ce3_axi_clk.clkr, + [GCC_CE3_CLK] = &gcc_ce3_clk.clkr, + [GCC_GP1_CLK] = &gcc_gp1_clk.clkr, + [GCC_GP2_CLK] = &gcc_gp2_clk.clkr, + [GCC_GP3_CLK] = &gcc_gp3_clk.clkr, + [GCC_OCMEM_NOC_CFG_AHB_CLK] = &gcc_ocmem_noc_cfg_ahb_clk.clkr, + [GCC_PCIE_0_AUX_CLK] = &gcc_pcie_0_aux_clk.clkr, + [GCC_PCIE_0_CFG_AHB_CLK] = &gcc_pcie_0_cfg_ahb_clk.clkr, + [GCC_PCIE_0_MSTR_AXI_CLK] = &gcc_pcie_0_mstr_axi_clk.clkr, + [GCC_PCIE_0_PIPE_CLK] = &gcc_pcie_0_pipe_clk.clkr, + [GCC_PCIE_0_SLV_AXI_CLK] = &gcc_pcie_0_slv_axi_clk.clkr, + [GCC_PCIE_1_AUX_CLK] = &gcc_pcie_1_aux_clk.clkr, + [GCC_PCIE_1_CFG_AHB_CLK] = &gcc_pcie_1_cfg_ahb_clk.clkr, + [GCC_PCIE_1_MSTR_AXI_CLK] = &gcc_pcie_1_mstr_axi_clk.clkr, + [GCC_PCIE_1_PIPE_CLK] = &gcc_pcie_1_pipe_clk.clkr, + [GCC_PCIE_1_SLV_AXI_CLK] = &gcc_pcie_1_slv_axi_clk.clkr, + [GCC_PDM2_CLK] = &gcc_pdm2_clk.clkr, + [GCC_PDM_AHB_CLK] = &gcc_pdm_ahb_clk.clkr, + [GCC_PERIPH_NOC_USB_HSIC_AHB_CLK] = &gcc_periph_noc_usb_hsic_ahb_clk.clkr, + [GCC_PRNG_AHB_CLK] = &gcc_prng_ahb_clk.clkr, + [GCC_SATA_ASIC0_CLK] = &gcc_sata_asic0_clk.clkr, + [GCC_SATA_AXI_CLK] = &gcc_sata_axi_clk.clkr, + [GCC_SATA_CFG_AHB_CLK] = &gcc_sata_cfg_ahb_clk.clkr, + [GCC_SATA_PMALIVE_CLK] = &gcc_sata_pmalive_clk.clkr, + [GCC_SATA_RX_CLK] = &gcc_sata_rx_clk.clkr, + [GCC_SATA_RX_OOB_CLK] = &gcc_sata_rx_oob_clk.clkr, + [GCC_SDCC1_AHB_CLK] = &gcc_sdcc1_ahb_clk.clkr, + [GCC_SDCC1_APPS_CLK] = &gcc_sdcc1_apps_clk.clkr, + [GCC_SDCC1_CDCCAL_FF_CLK] = &gcc_sdcc1_cdccal_ff_clk.clkr, + [GCC_SDCC1_CDCCAL_SLEEP_CLK] = &gcc_sdcc1_cdccal_sleep_clk.clkr, + [GCC_SDCC2_AHB_CLK] = &gcc_sdcc2_ahb_clk.clkr, + [GCC_SDCC2_APPS_CLK] = &gcc_sdcc2_apps_clk.clkr, + [GCC_SDCC3_AHB_CLK] = &gcc_sdcc3_ahb_clk.clkr, + [GCC_SDCC3_APPS_CLK] = &gcc_sdcc3_apps_clk.clkr, + [GCC_SDCC4_AHB_CLK] = &gcc_sdcc4_ahb_clk.clkr, + [GCC_SDCC4_APPS_CLK] = &gcc_sdcc4_apps_clk.clkr, + [GCC_SYS_NOC_UFS_AXI_CLK] = &gcc_sys_noc_ufs_axi_clk.clkr, + [GCC_SYS_NOC_USB3_AXI_CLK] = &gcc_sys_noc_usb3_axi_clk.clkr, + [GCC_SYS_NOC_USB3_SEC_AXI_CLK] = &gcc_sys_noc_usb3_sec_axi_clk.clkr, + [GCC_TSIF_AHB_CLK] = &gcc_tsif_ahb_clk.clkr, + [GCC_TSIF_INACTIVITY_TIMERS_CLK] = &gcc_tsif_inactivity_timers_clk.clkr, + [GCC_TSIF_REF_CLK] = &gcc_tsif_ref_clk.clkr, + [GCC_UFS_AHB_CLK] = &gcc_ufs_ahb_clk.clkr, + [GCC_UFS_AXI_CLK] = &gcc_ufs_axi_clk.clkr, + [GCC_UFS_RX_CFG_CLK] = &gcc_ufs_rx_cfg_clk.clkr, + [GCC_UFS_RX_SYMBOL_0_CLK] = &gcc_ufs_rx_symbol_0_clk.clkr, + [GCC_UFS_RX_SYMBOL_1_CLK] = &gcc_ufs_rx_symbol_1_clk.clkr, + [GCC_UFS_TX_CFG_CLK] = &gcc_ufs_tx_cfg_clk.clkr, + [GCC_UFS_TX_SYMBOL_0_CLK] = &gcc_ufs_tx_symbol_0_clk.clkr, + [GCC_UFS_TX_SYMBOL_1_CLK] = &gcc_ufs_tx_symbol_1_clk.clkr, + [GCC_USB2A_PHY_SLEEP_CLK] = &gcc_usb2a_phy_sleep_clk.clkr, + [GCC_USB2B_PHY_SLEEP_CLK] = &gcc_usb2b_phy_sleep_clk.clkr, + [GCC_USB30_MASTER_CLK] = &gcc_usb30_master_clk.clkr, + [GCC_USB30_MOCK_UTMI_CLK] = &gcc_usb30_mock_utmi_clk.clkr, + [GCC_USB30_SLEEP_CLK] = &gcc_usb30_sleep_clk.clkr, + [GCC_USB30_SEC_MASTER_CLK] = &gcc_usb30_sec_master_clk.clkr, + [GCC_USB30_SEC_MOCK_UTMI_CLK] = &gcc_usb30_sec_mock_utmi_clk.clkr, + [GCC_USB30_SEC_SLEEP_CLK] = &gcc_usb30_sec_sleep_clk.clkr, + [GCC_USB_HS_AHB_CLK] = &gcc_usb_hs_ahb_clk.clkr, + [GCC_USB_HS_INACTIVITY_TIMERS_CLK] = &gcc_usb_hs_inactivity_timers_clk.clkr, + [GCC_USB_HS_SYSTEM_CLK] = &gcc_usb_hs_system_clk.clkr, + [GCC_USB_HSIC_AHB_CLK] = &gcc_usb_hsic_ahb_clk.clkr, + [GCC_USB_HSIC_CLK] = &gcc_usb_hsic_clk.clkr, + [GCC_USB_HSIC_IO_CAL_CLK] = &gcc_usb_hsic_io_cal_clk.clkr, + [GCC_USB_HSIC_IO_CAL_SLEEP_CLK] = &gcc_usb_hsic_io_cal_sleep_clk.clkr, + [GCC_USB_HSIC_MOCK_UTMI_CLK] = &gcc_usb_hsic_mock_utmi_clk.clkr, + [GCC_USB_HSIC_SYSTEM_CLK] = &gcc_usb_hsic_system_clk.clkr, +}; + +static const struct qcom_reset_map gcc_apq8084_resets[] = { + [GCC_SYSTEM_NOC_BCR] = { 0x0100 }, + [GCC_CONFIG_NOC_BCR] = { 0x0140 }, + [GCC_PERIPH_NOC_BCR] = { 0x0180 }, + [GCC_IMEM_BCR] = { 0x0200 }, + [GCC_MMSS_BCR] = { 0x0240 }, + [GCC_QDSS_BCR] = { 0x0300 }, + [GCC_USB_30_BCR] = { 0x03c0 }, + [GCC_USB3_PHY_BCR] = { 0x03fc }, + [GCC_USB_HS_HSIC_BCR] = { 0x0400 }, + [GCC_USB_HS_BCR] = { 0x0480 }, + [GCC_USB2A_PHY_BCR] = { 0x04a8 }, + [GCC_USB2B_PHY_BCR] = { 0x04b0 }, + [GCC_SDCC1_BCR] = { 0x04c0 }, + [GCC_SDCC2_BCR] = { 0x0500 }, + [GCC_SDCC3_BCR] = { 0x0540 }, + [GCC_SDCC4_BCR] = { 0x0580 }, + [GCC_BLSP1_BCR] = { 0x05c0 }, + [GCC_BLSP1_QUP1_BCR] = { 0x0640 }, + [GCC_BLSP1_UART1_BCR] = { 0x0680 }, + [GCC_BLSP1_QUP2_BCR] = { 0x06c0 }, + [GCC_BLSP1_UART2_BCR] = { 0x0700 }, + [GCC_BLSP1_QUP3_BCR] = { 0x0740 }, + [GCC_BLSP1_UART3_BCR] = { 0x0780 }, + [GCC_BLSP1_QUP4_BCR] = { 0x07c0 }, + [GCC_BLSP1_UART4_BCR] = { 0x0800 }, + [GCC_BLSP1_QUP5_BCR] = { 0x0840 }, + [GCC_BLSP1_UART5_BCR] = { 0x0880 }, + [GCC_BLSP1_QUP6_BCR] = { 0x08c0 }, + [GCC_BLSP1_UART6_BCR] = { 0x0900 }, + [GCC_BLSP2_BCR] = { 0x0940 }, + [GCC_BLSP2_QUP1_BCR] = { 0x0980 }, + [GCC_BLSP2_UART1_BCR] = { 0x09c0 }, + [GCC_BLSP2_QUP2_BCR] = { 0x0a00 }, + [GCC_BLSP2_UART2_BCR] = { 0x0a40 }, + [GCC_BLSP2_QUP3_BCR] = { 0x0a80 }, + [GCC_BLSP2_UART3_BCR] = { 0x0ac0 }, + [GCC_BLSP2_QUP4_BCR] = { 0x0b00 }, + [GCC_BLSP2_UART4_BCR] = { 0x0b40 }, + [GCC_BLSP2_QUP5_BCR] = { 0x0b80 }, + [GCC_BLSP2_UART5_BCR] = { 0x0bc0 }, + [GCC_BLSP2_QUP6_BCR] = { 0x0c00 }, + [GCC_BLSP2_UART6_BCR] = { 0x0c40 }, + [GCC_PDM_BCR] = { 0x0cc0 }, + [GCC_PRNG_BCR] = { 0x0d00 }, + [GCC_BAM_DMA_BCR] = { 0x0d40 }, + [GCC_TSIF_BCR] = { 0x0d80 }, + [GCC_TCSR_BCR] = { 0x0dc0 }, + [GCC_BOOT_ROM_BCR] = { 0x0e00 }, + [GCC_MSG_RAM_BCR] = { 0x0e40 }, + [GCC_TLMM_BCR] = { 0x0e80 }, + [GCC_MPM_BCR] = { 0x0ec0 }, + [GCC_MPM_AHB_RESET] = { 0x0ec4, 1 }, + [GCC_MPM_NON_AHB_RESET] = { 0x0ec4, 2 }, + [GCC_SEC_CTRL_BCR] = { 0x0f40 }, + [GCC_SPMI_BCR] = { 0x0fc0 }, + [GCC_SPDM_BCR] = { 0x1000 }, + [GCC_CE1_BCR] = { 0x1040 }, + [GCC_CE2_BCR] = { 0x1080 }, + [GCC_BIMC_BCR] = { 0x1100 }, + [GCC_SNOC_BUS_TIMEOUT0_BCR] = { 0x1240 }, + [GCC_SNOC_BUS_TIMEOUT2_BCR] = { 0x1248 }, + [GCC_PNOC_BUS_TIMEOUT0_BCR] = { 0x1280 }, + [GCC_PNOC_BUS_TIMEOUT1_BCR] = { 0x1288 }, + [GCC_PNOC_BUS_TIMEOUT2_BCR] = { 0x1290 }, + [GCC_PNOC_BUS_TIMEOUT3_BCR] = { 0x1298 }, + [GCC_PNOC_BUS_TIMEOUT4_BCR] = { 0x12a0 }, + [GCC_CNOC_BUS_TIMEOUT0_BCR] = { 0x12c0 }, + [GCC_CNOC_BUS_TIMEOUT1_BCR] = { 0x12c8 }, + [GCC_CNOC_BUS_TIMEOUT2_BCR] = { 0x12d0 }, + [GCC_CNOC_BUS_TIMEOUT3_BCR] = { 0x12d8 }, + [GCC_CNOC_BUS_TIMEOUT4_BCR] = { 0x12e0 }, + [GCC_CNOC_BUS_TIMEOUT5_BCR] = { 0x12e8 }, + [GCC_CNOC_BUS_TIMEOUT6_BCR] = { 0x12f0 }, + [GCC_DEHR_BCR] = { 0x1300 }, + [GCC_RBCPR_BCR] = { 0x1380 }, + [GCC_MSS_RESTART] = { 0x1680 }, + [GCC_LPASS_RESTART] = { 0x16c0 }, + [GCC_WCSS_RESTART] = { 0x1700 }, + [GCC_VENUS_RESTART] = { 0x1740 }, + [GCC_COPSS_SMMU_BCR] = { 0x1a40 }, + [GCC_SPSS_BCR] = { 0x1a80 }, + [GCC_PCIE_0_BCR] = { 0x1ac0 }, + [GCC_PCIE_0_PHY_BCR] = { 0x1b00 }, + [GCC_PCIE_1_BCR] = { 0x1b40 }, + [GCC_PCIE_1_PHY_BCR] = { 0x1b80 }, + [GCC_USB_30_SEC_BCR] = { 0x1bc0 }, + [GCC_USB3_SEC_PHY_BCR] = { 0x1bfc }, + [GCC_SATA_BCR] = { 0x1c40 }, + [GCC_CE3_BCR] = { 0x1d00 }, + [GCC_UFS_BCR] = { 0x1d40 }, + [GCC_USB30_PHY_COM_BCR] = { 0x1e80 }, +}; + +static const struct regmap_config gcc_apq8084_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0x1fc0, + .fast_io = true, +}; + +static const struct qcom_cc_desc gcc_apq8084_desc = { + .config = &gcc_apq8084_regmap_config, + .clks = gcc_apq8084_clocks, + .num_clks = ARRAY_SIZE(gcc_apq8084_clocks), + .resets = gcc_apq8084_resets, + .num_resets = ARRAY_SIZE(gcc_apq8084_resets), +}; + +static const struct of_device_id gcc_apq8084_match_table[] = { + { .compatible = "qcom,gcc-apq8084" }, + { } +}; +MODULE_DEVICE_TABLE(of, gcc_apq8084_match_table); + +static int gcc_apq8084_probe(struct platform_device *pdev) +{ + struct clk *clk; + struct device *dev = &pdev->dev; + + /* Temporary until RPM clocks supported */ + clk = clk_register_fixed_rate(dev, "xo", NULL, CLK_IS_ROOT, 19200000); + if (IS_ERR(clk)) + return PTR_ERR(clk); + + clk = clk_register_fixed_rate(dev, "sleep_clk_src", NULL, + CLK_IS_ROOT, 32768); + if (IS_ERR(clk)) + return PTR_ERR(clk); + + return qcom_cc_probe(pdev, &gcc_apq8084_desc); +} + +static int gcc_apq8084_remove(struct platform_device *pdev) +{ + qcom_cc_remove(pdev); + return 0; +} + +static struct platform_driver gcc_apq8084_driver = { + .probe = gcc_apq8084_probe, + .remove = gcc_apq8084_remove, + .driver = { + .name = "gcc-apq8084", + .owner = THIS_MODULE, + .of_match_table = gcc_apq8084_match_table, + }, +}; + +static int __init gcc_apq8084_init(void) +{ + return platform_driver_register(&gcc_apq8084_driver); +} +core_initcall(gcc_apq8084_init); + +static void __exit gcc_apq8084_exit(void) +{ + platform_driver_unregister(&gcc_apq8084_driver); +} +module_exit(gcc_apq8084_exit); + +MODULE_DESCRIPTION("QCOM GCC APQ8084 Driver"); +MODULE_LICENSE("GPL v2"); +MODULE_ALIAS("platform:gcc-apq8084"); diff --git a/drivers/clk/qcom/gcc-ipq806x.c b/drivers/clk/qcom/gcc-ipq806x.c new file mode 100644 index 000000000000..4032e510d9aa --- /dev/null +++ b/drivers/clk/qcom/gcc-ipq806x.c @@ -0,0 +1,2424 @@ +/* + * Copyright (c) 2014, The Linux Foundation. All rights reserved. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include <linux/kernel.h> +#include <linux/bitops.h> +#include <linux/err.h> +#include <linux/platform_device.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_device.h> +#include <linux/clk-provider.h> +#include <linux/regmap.h> +#include <linux/reset-controller.h> + +#include <dt-bindings/clock/qcom,gcc-ipq806x.h> +#include <dt-bindings/reset/qcom,gcc-ipq806x.h> + +#include "common.h" +#include "clk-regmap.h" +#include "clk-pll.h" +#include "clk-rcg.h" +#include "clk-branch.h" +#include "reset.h" + +static struct clk_pll pll3 = { + .l_reg = 0x3164, + .m_reg = 0x3168, + .n_reg = 0x316c, + .config_reg = 0x3174, + .mode_reg = 0x3160, + .status_reg = 0x3178, + .status_bit = 16, + .clkr.hw.init = &(struct clk_init_data){ + .name = "pll3", + .parent_names = (const char *[]){ "pxo" }, + .num_parents = 1, + .ops = &clk_pll_ops, + }, +}; + +static struct clk_pll pll8 = { + .l_reg = 0x3144, + .m_reg = 0x3148, + .n_reg = 0x314c, + .config_reg = 0x3154, + .mode_reg = 0x3140, + .status_reg = 0x3158, + .status_bit = 16, + .clkr.hw.init = &(struct clk_init_data){ + .name = "pll8", + .parent_names = (const char *[]){ "pxo" }, + .num_parents = 1, + .ops = &clk_pll_ops, + }, +}; + +static struct clk_regmap pll8_vote = { + .enable_reg = 0x34c0, + .enable_mask = BIT(8), + .hw.init = &(struct clk_init_data){ + .name = "pll8_vote", + .parent_names = (const char *[]){ "pll8" }, + .num_parents = 1, + .ops = &clk_pll_vote_ops, + }, +}; + +static struct clk_pll pll14 = { + .l_reg = 0x31c4, + .m_reg = 0x31c8, + .n_reg = 0x31cc, + .config_reg = 0x31d4, + .mode_reg = 0x31c0, + .status_reg = 0x31d8, + .status_bit = 16, + .clkr.hw.init = &(struct clk_init_data){ + .name = "pll14", + .parent_names = (const char *[]){ "pxo" }, + .num_parents = 1, + .ops = &clk_pll_ops, + }, +}; + +static struct clk_regmap pll14_vote = { + .enable_reg = 0x34c0, + .enable_mask = BIT(14), + .hw.init = &(struct clk_init_data){ + .name = "pll14_vote", + .parent_names = (const char *[]){ "pll14" }, + .num_parents = 1, + .ops = &clk_pll_vote_ops, + }, +}; + +#define P_PXO 0 +#define P_PLL8 1 +#define P_PLL3 1 +#define P_PLL0 2 +#define P_CXO 2 + +static const u8 gcc_pxo_pll8_map[] = { + [P_PXO] = 0, + [P_PLL8] = 3, +}; + +static const char *gcc_pxo_pll8[] = { + "pxo", + "pll8_vote", +}; + +static const u8 gcc_pxo_pll8_cxo_map[] = { + [P_PXO] = 0, + [P_PLL8] = 3, + [P_CXO] = 5, +}; + +static const char *gcc_pxo_pll8_cxo[] = { + "pxo", + "pll8_vote", + "cxo", +}; + +static const u8 gcc_pxo_pll3_map[] = { + [P_PXO] = 0, + [P_PLL3] = 1, +}; + +static const u8 gcc_pxo_pll3_sata_map[] = { + [P_PXO] = 0, + [P_PLL3] = 6, +}; + +static const char *gcc_pxo_pll3[] = { + "pxo", + "pll3", +}; + +static const u8 gcc_pxo_pll8_pll0[] = { + [P_PXO] = 0, + [P_PLL8] = 3, + [P_PLL0] = 2, +}; + +static const char *gcc_pxo_pll8_pll0_map[] = { + "pxo", + "pll8_vote", + "pll0", +}; + +static struct freq_tbl clk_tbl_gsbi_uart[] = { + { 1843200, P_PLL8, 2, 6, 625 }, + { 3686400, P_PLL8, 2, 12, 625 }, + { 7372800, P_PLL8, 2, 24, 625 }, + { 14745600, P_PLL8, 2, 48, 625 }, + { 16000000, P_PLL8, 4, 1, 6 }, + { 24000000, P_PLL8, 4, 1, 4 }, + { 32000000, P_PLL8, 4, 1, 3 }, + { 40000000, P_PLL8, 1, 5, 48 }, + { 46400000, P_PLL8, 1, 29, 240 }, + { 48000000, P_PLL8, 4, 1, 2 }, + { 51200000, P_PLL8, 1, 2, 15 }, + { 56000000, P_PLL8, 1, 7, 48 }, + { 58982400, P_PLL8, 1, 96, 625 }, + { 64000000, P_PLL8, 2, 1, 3 }, + { } +}; + +static struct clk_rcg gsbi1_uart_src = { + .ns_reg = 0x29d4, + .md_reg = 0x29d0, + .mn = { + .mnctr_en_bit = 8, + .mnctr_reset_bit = 7, + .mnctr_mode_shift = 5, + .n_val_shift = 16, + .m_val_shift = 16, + .width = 16, + }, + .p = { + .pre_div_shift = 3, + .pre_div_width = 2, + }, + .s = { + .src_sel_shift = 0, + .parent_map = gcc_pxo_pll8_map, + }, + .freq_tbl = clk_tbl_gsbi_uart, + .clkr = { + .enable_reg = 0x29d4, + .enable_mask = BIT(11), + .hw.init = &(struct clk_init_data){ + .name = "gsbi1_uart_src", + .parent_names = gcc_pxo_pll8, + .num_parents = 2, + .ops = &clk_rcg_ops, + .flags = CLK_SET_PARENT_GATE, + }, + }, +}; + +static struct clk_branch gsbi1_uart_clk = { + .halt_reg = 0x2fcc, + .halt_bit = 12, + .clkr = { + .enable_reg = 0x29d4, + .enable_mask = BIT(9), + .hw.init = &(struct clk_init_data){ + .name = "gsbi1_uart_clk", + .parent_names = (const char *[]){ + "gsbi1_uart_src", + }, + .num_parents = 1, + .ops = &clk_branch_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_rcg gsbi2_uart_src = { + .ns_reg = 0x29f4, + .md_reg = 0x29f0, + .mn = { + .mnctr_en_bit = 8, + .mnctr_reset_bit = 7, + .mnctr_mode_shift = 5, + .n_val_shift = 16, + .m_val_shift = 16, + .width = 16, + }, + .p = { + .pre_div_shift = 3, + .pre_div_width = 2, + }, + .s = { + .src_sel_shift = 0, + .parent_map = gcc_pxo_pll8_map, + }, + .freq_tbl = clk_tbl_gsbi_uart, + .clkr = { + .enable_reg = 0x29f4, + .enable_mask = BIT(11), + .hw.init = &(struct clk_init_data){ + .name = "gsbi2_uart_src", + .parent_names = gcc_pxo_pll8, + .num_parents = 2, + .ops = &clk_rcg_ops, + .flags = CLK_SET_PARENT_GATE, + }, + }, +}; + +static struct clk_branch gsbi2_uart_clk = { + .halt_reg = 0x2fcc, + .halt_bit = 8, + .clkr = { + .enable_reg = 0x29f4, + .enable_mask = BIT(9), + .hw.init = &(struct clk_init_data){ + .name = "gsbi2_uart_clk", + .parent_names = (const char *[]){ + "gsbi2_uart_src", + }, + .num_parents = 1, + .ops = &clk_branch_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_rcg gsbi4_uart_src = { + .ns_reg = 0x2a34, + .md_reg = 0x2a30, + .mn = { + .mnctr_en_bit = 8, + .mnctr_reset_bit = 7, + .mnctr_mode_shift = 5, + .n_val_shift = 16, + .m_val_shift = 16, + .width = 16, + }, + .p = { + .pre_div_shift = 3, + .pre_div_width = 2, + }, + .s = { + .src_sel_shift = 0, + .parent_map = gcc_pxo_pll8_map, + }, + .freq_tbl = clk_tbl_gsbi_uart, + .clkr = { + .enable_reg = 0x2a34, + .enable_mask = BIT(11), + .hw.init = &(struct clk_init_data){ + .name = "gsbi4_uart_src", + .parent_names = gcc_pxo_pll8, + .num_parents = 2, + .ops = &clk_rcg_ops, + .flags = CLK_SET_PARENT_GATE, + }, + }, +}; + +static struct clk_branch gsbi4_uart_clk = { + .halt_reg = 0x2fd0, + .halt_bit = 26, + .clkr = { + .enable_reg = 0x2a34, + .enable_mask = BIT(9), + .hw.init = &(struct clk_init_data){ + .name = "gsbi4_uart_clk", + .parent_names = (const char *[]){ + "gsbi4_uart_src", + }, + .num_parents = 1, + .ops = &clk_branch_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_rcg gsbi5_uart_src = { + .ns_reg = 0x2a54, + .md_reg = 0x2a50, + .mn = { + .mnctr_en_bit = 8, + .mnctr_reset_bit = 7, + .mnctr_mode_shift = 5, + .n_val_shift = 16, + .m_val_shift = 16, + .width = 16, + }, + .p = { + .pre_div_shift = 3, + .pre_div_width = 2, + }, + .s = { + .src_sel_shift = 0, + .parent_map = gcc_pxo_pll8_map, + }, + .freq_tbl = clk_tbl_gsbi_uart, + .clkr = { + .enable_reg = 0x2a54, + .enable_mask = BIT(11), + .hw.init = &(struct clk_init_data){ + .name = "gsbi5_uart_src", + .parent_names = gcc_pxo_pll8, + .num_parents = 2, + .ops = &clk_rcg_ops, + .flags = CLK_SET_PARENT_GATE, + }, + }, +}; + +static struct clk_branch gsbi5_uart_clk = { + .halt_reg = 0x2fd0, + .halt_bit = 22, + .clkr = { + .enable_reg = 0x2a54, + .enable_mask = BIT(9), + .hw.init = &(struct clk_init_data){ + .name = "gsbi5_uart_clk", + .parent_names = (const char *[]){ + "gsbi5_uart_src", + }, + .num_parents = 1, + .ops = &clk_branch_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_rcg gsbi6_uart_src = { + .ns_reg = 0x2a74, + .md_reg = 0x2a70, + .mn = { + .mnctr_en_bit = 8, + .mnctr_reset_bit = 7, + .mnctr_mode_shift = 5, + .n_val_shift = 16, + .m_val_shift = 16, + .width = 16, + }, + .p = { + .pre_div_shift = 3, + .pre_div_width = 2, + }, + .s = { + .src_sel_shift = 0, + .parent_map = gcc_pxo_pll8_map, + }, + .freq_tbl = clk_tbl_gsbi_uart, + .clkr = { + .enable_reg = 0x2a74, + .enable_mask = BIT(11), + .hw.init = &(struct clk_init_data){ + .name = "gsbi6_uart_src", + .parent_names = gcc_pxo_pll8, + .num_parents = 2, + .ops = &clk_rcg_ops, + .flags = CLK_SET_PARENT_GATE, + }, + }, +}; + +static struct clk_branch gsbi6_uart_clk = { + .halt_reg = 0x2fd0, + .halt_bit = 18, + .clkr = { + .enable_reg = 0x2a74, + .enable_mask = BIT(9), + .hw.init = &(struct clk_init_data){ + .name = "gsbi6_uart_clk", + .parent_names = (const char *[]){ + "gsbi6_uart_src", + }, + .num_parents = 1, + .ops = &clk_branch_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_rcg gsbi7_uart_src = { + .ns_reg = 0x2a94, + .md_reg = 0x2a90, + .mn = { + .mnctr_en_bit = 8, + .mnctr_reset_bit = 7, + .mnctr_mode_shift = 5, + .n_val_shift = 16, + .m_val_shift = 16, + .width = 16, + }, + .p = { + .pre_div_shift = 3, + .pre_div_width = 2, + }, + .s = { + .src_sel_shift = 0, + .parent_map = gcc_pxo_pll8_map, + }, + .freq_tbl = clk_tbl_gsbi_uart, + .clkr = { + .enable_reg = 0x2a94, + .enable_mask = BIT(11), + .hw.init = &(struct clk_init_data){ + .name = "gsbi7_uart_src", + .parent_names = gcc_pxo_pll8, + .num_parents = 2, + .ops = &clk_rcg_ops, + .flags = CLK_SET_PARENT_GATE, + }, + }, +}; + +static struct clk_branch gsbi7_uart_clk = { + .halt_reg = 0x2fd0, + .halt_bit = 14, + .clkr = { + .enable_reg = 0x2a94, + .enable_mask = BIT(9), + .hw.init = &(struct clk_init_data){ + .name = "gsbi7_uart_clk", + .parent_names = (const char *[]){ + "gsbi7_uart_src", + }, + .num_parents = 1, + .ops = &clk_branch_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct freq_tbl clk_tbl_gsbi_qup[] = { + { 1100000, P_PXO, 1, 2, 49 }, + { 5400000, P_PXO, 1, 1, 5 }, + { 10800000, P_PXO, 1, 2, 5 }, + { 15060000, P_PLL8, 1, 2, 51 }, + { 24000000, P_PLL8, 4, 1, 4 }, + { 25600000, P_PLL8, 1, 1, 15 }, + { 27000000, P_PXO, 1, 0, 0 }, + { 48000000, P_PLL8, 4, 1, 2 }, + { 51200000, P_PLL8, 1, 2, 15 }, + { } +}; + +static struct clk_rcg gsbi1_qup_src = { + .ns_reg = 0x29cc, + .md_reg = 0x29c8, + .mn = { + .mnctr_en_bit = 8, + .mnctr_reset_bit = 7, + .mnctr_mode_shift = 5, + .n_val_shift = 16, + .m_val_shift = 16, + .width = 8, + }, + .p = { + .pre_div_shift = 3, + .pre_div_width = 2, + }, + .s = { + .src_sel_shift = 0, + .parent_map = gcc_pxo_pll8_map, + }, + .freq_tbl = clk_tbl_gsbi_qup, + .clkr = { + .enable_reg = 0x29cc, + .enable_mask = BIT(11), + .hw.init = &(struct clk_init_data){ + .name = "gsbi1_qup_src", + .parent_names = gcc_pxo_pll8, + .num_parents = 2, + .ops = &clk_rcg_ops, + .flags = CLK_SET_PARENT_GATE, + }, + }, +}; + +static struct clk_branch gsbi1_qup_clk = { + .halt_reg = 0x2fcc, + .halt_bit = 11, + .clkr = { + .enable_reg = 0x29cc, + .enable_mask = BIT(9), + .hw.init = &(struct clk_init_data){ + .name = "gsbi1_qup_clk", + .parent_names = (const char *[]){ "gsbi1_qup_src" }, + .num_parents = 1, + .ops = &clk_branch_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_rcg gsbi2_qup_src = { + .ns_reg = 0x29ec, + .md_reg = 0x29e8, + .mn = { + .mnctr_en_bit = 8, + .mnctr_reset_bit = 7, + .mnctr_mode_shift = 5, + .n_val_shift = 16, + .m_val_shift = 16, + .width = 8, + }, + .p = { + .pre_div_shift = 3, + .pre_div_width = 2, + }, + .s = { + .src_sel_shift = 0, + .parent_map = gcc_pxo_pll8_map, + }, + .freq_tbl = clk_tbl_gsbi_qup, + .clkr = { + .enable_reg = 0x29ec, + .enable_mask = BIT(11), + .hw.init = &(struct clk_init_data){ + .name = "gsbi2_qup_src", + .parent_names = gcc_pxo_pll8, + .num_parents = 2, + .ops = &clk_rcg_ops, + .flags = CLK_SET_PARENT_GATE, + }, + }, +}; + +static struct clk_branch gsbi2_qup_clk = { + .halt_reg = 0x2fcc, + .halt_bit = 6, + .clkr = { + .enable_reg = 0x29ec, + .enable_mask = BIT(9), + .hw.init = &(struct clk_init_data){ + .name = "gsbi2_qup_clk", + .parent_names = (const char *[]){ "gsbi2_qup_src" }, + .num_parents = 1, + .ops = &clk_branch_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_rcg gsbi4_qup_src = { + .ns_reg = 0x2a2c, + .md_reg = 0x2a28, + .mn = { + .mnctr_en_bit = 8, + .mnctr_reset_bit = 7, + .mnctr_mode_shift = 5, + .n_val_shift = 16, + .m_val_shift = 16, + .width = 8, + }, + .p = { + .pre_div_shift = 3, + .pre_div_width = 2, + }, + .s = { + .src_sel_shift = 0, + .parent_map = gcc_pxo_pll8_map, + }, + .freq_tbl = clk_tbl_gsbi_qup, + .clkr = { + .enable_reg = 0x2a2c, + .enable_mask = BIT(11), + .hw.init = &(struct clk_init_data){ + .name = "gsbi4_qup_src", + .parent_names = gcc_pxo_pll8, + .num_parents = 2, + .ops = &clk_rcg_ops, + .flags = CLK_SET_PARENT_GATE, + }, + }, +}; + +static struct clk_branch gsbi4_qup_clk = { + .halt_reg = 0x2fd0, + .halt_bit = 24, + .clkr = { + .enable_reg = 0x2a2c, + .enable_mask = BIT(9), + .hw.init = &(struct clk_init_data){ + .name = "gsbi4_qup_clk", + .parent_names = (const char *[]){ "gsbi4_qup_src" }, + .num_parents = 1, + .ops = &clk_branch_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_rcg gsbi5_qup_src = { + .ns_reg = 0x2a4c, + .md_reg = 0x2a48, + .mn = { + .mnctr_en_bit = 8, + .mnctr_reset_bit = 7, + .mnctr_mode_shift = 5, + .n_val_shift = 16, + .m_val_shift = 16, + .width = 8, + }, + .p = { + .pre_div_shift = 3, + .pre_div_width = 2, + }, + .s = { + .src_sel_shift = 0, + .parent_map = gcc_pxo_pll8_map, + }, + .freq_tbl = clk_tbl_gsbi_qup, + .clkr = { + .enable_reg = 0x2a4c, + .enable_mask = BIT(11), + .hw.init = &(struct clk_init_data){ + .name = "gsbi5_qup_src", + .parent_names = gcc_pxo_pll8, + .num_parents = 2, + .ops = &clk_rcg_ops, + .flags = CLK_SET_PARENT_GATE, + }, + }, +}; + +static struct clk_branch gsbi5_qup_clk = { + .halt_reg = 0x2fd0, + .halt_bit = 20, + .clkr = { + .enable_reg = 0x2a4c, + .enable_mask = BIT(9), + .hw.init = &(struct clk_init_data){ + .name = "gsbi5_qup_clk", + .parent_names = (const char *[]){ "gsbi5_qup_src" }, + .num_parents = 1, + .ops = &clk_branch_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_rcg gsbi6_qup_src = { + .ns_reg = 0x2a6c, + .md_reg = 0x2a68, + .mn = { + .mnctr_en_bit = 8, + .mnctr_reset_bit = 7, + .mnctr_mode_shift = 5, + .n_val_shift = 16, + .m_val_shift = 16, + .width = 8, + }, + .p = { + .pre_div_shift = 3, + .pre_div_width = 2, + }, + .s = { + .src_sel_shift = 0, + .parent_map = gcc_pxo_pll8_map, + }, + .freq_tbl = clk_tbl_gsbi_qup, + .clkr = { + .enable_reg = 0x2a6c, + .enable_mask = BIT(11), + .hw.init = &(struct clk_init_data){ + .name = "gsbi6_qup_src", + .parent_names = gcc_pxo_pll8, + .num_parents = 2, + .ops = &clk_rcg_ops, + .flags = CLK_SET_PARENT_GATE, + }, + }, +}; + +static struct clk_branch gsbi6_qup_clk = { + .halt_reg = 0x2fd0, + .halt_bit = 16, + .clkr = { + .enable_reg = 0x2a6c, + .enable_mask = BIT(9), + .hw.init = &(struct clk_init_data){ + .name = "gsbi6_qup_clk", + .parent_names = (const char *[]){ "gsbi6_qup_src" }, + .num_parents = 1, + .ops = &clk_branch_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_rcg gsbi7_qup_src = { + .ns_reg = 0x2a8c, + .md_reg = 0x2a88, + .mn = { + .mnctr_en_bit = 8, + .mnctr_reset_bit = 7, + .mnctr_mode_shift = 5, + .n_val_shift = 16, + .m_val_shift = 16, + .width = 8, + }, + .p = { + .pre_div_shift = 3, + .pre_div_width = 2, + }, + .s = { + .src_sel_shift = 0, + .parent_map = gcc_pxo_pll8_map, + }, + .freq_tbl = clk_tbl_gsbi_qup, + .clkr = { + .enable_reg = 0x2a8c, + .enable_mask = BIT(11), + .hw.init = &(struct clk_init_data){ + .name = "gsbi7_qup_src", + .parent_names = gcc_pxo_pll8, + .num_parents = 2, + .ops = &clk_rcg_ops, + .flags = CLK_SET_PARENT_GATE, + }, + }, +}; + +static struct clk_branch gsbi7_qup_clk = { + .halt_reg = 0x2fd0, + .halt_bit = 12, + .clkr = { + .enable_reg = 0x2a8c, + .enable_mask = BIT(9), + .hw.init = &(struct clk_init_data){ + .name = "gsbi7_qup_clk", + .parent_names = (const char *[]){ "gsbi7_qup_src" }, + .num_parents = 1, + .ops = &clk_branch_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch gsbi1_h_clk = { + .hwcg_reg = 0x29c0, + .hwcg_bit = 6, + .halt_reg = 0x2fcc, + .halt_bit = 13, + .clkr = { + .enable_reg = 0x29c0, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "gsbi1_h_clk", + .ops = &clk_branch_ops, + .flags = CLK_IS_ROOT, + }, + }, +}; + +static struct clk_branch gsbi2_h_clk = { + .hwcg_reg = 0x29e0, + .hwcg_bit = 6, + .halt_reg = 0x2fcc, + .halt_bit = 9, + .clkr = { + .enable_reg = 0x29e0, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "gsbi2_h_clk", + .ops = &clk_branch_ops, + .flags = CLK_IS_ROOT, + }, + }, +}; + +static struct clk_branch gsbi4_h_clk = { + .hwcg_reg = 0x2a20, + .hwcg_bit = 6, + .halt_reg = 0x2fd0, + .halt_bit = 27, + .clkr = { + .enable_reg = 0x2a20, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "gsbi4_h_clk", + .ops = &clk_branch_ops, + .flags = CLK_IS_ROOT, + }, + }, +}; + +static struct clk_branch gsbi5_h_clk = { + .hwcg_reg = 0x2a40, + .hwcg_bit = 6, + .halt_reg = 0x2fd0, + .halt_bit = 23, + .clkr = { + .enable_reg = 0x2a40, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "gsbi5_h_clk", + .ops = &clk_branch_ops, + .flags = CLK_IS_ROOT, + }, + }, +}; + +static struct clk_branch gsbi6_h_clk = { + .hwcg_reg = 0x2a60, + .hwcg_bit = 6, + .halt_reg = 0x2fd0, + .halt_bit = 19, + .clkr = { + .enable_reg = 0x2a60, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "gsbi6_h_clk", + .ops = &clk_branch_ops, + .flags = CLK_IS_ROOT, + }, + }, +}; + +static struct clk_branch gsbi7_h_clk = { + .hwcg_reg = 0x2a80, + .hwcg_bit = 6, + .halt_reg = 0x2fd0, + .halt_bit = 15, + .clkr = { + .enable_reg = 0x2a80, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "gsbi7_h_clk", + .ops = &clk_branch_ops, + .flags = CLK_IS_ROOT, + }, + }, +}; + +static const struct freq_tbl clk_tbl_gp[] = { + { 12500000, P_PXO, 2, 0, 0 }, + { 25000000, P_PXO, 1, 0, 0 }, + { 64000000, P_PLL8, 2, 1, 3 }, + { 76800000, P_PLL8, 1, 1, 5 }, + { 96000000, P_PLL8, 4, 0, 0 }, + { 128000000, P_PLL8, 3, 0, 0 }, + { 192000000, P_PLL8, 2, 0, 0 }, + { } +}; + +static struct clk_rcg gp0_src = { + .ns_reg = 0x2d24, + .md_reg = 0x2d00, + .mn = { + .mnctr_en_bit = 8, + .mnctr_reset_bit = 7, + .mnctr_mode_shift = 5, + .n_val_shift = 16, + .m_val_shift = 16, + .width = 8, + }, + .p = { + .pre_div_shift = 3, + .pre_div_width = 2, + }, + .s = { + .src_sel_shift = 0, + .parent_map = gcc_pxo_pll8_cxo_map, + }, + .freq_tbl = clk_tbl_gp, + .clkr = { + .enable_reg = 0x2d24, + .enable_mask = BIT(11), + .hw.init = &(struct clk_init_data){ + .name = "gp0_src", + .parent_names = gcc_pxo_pll8_cxo, + .num_parents = 3, + .ops = &clk_rcg_ops, + .flags = CLK_SET_PARENT_GATE, + }, + } +}; + +static struct clk_branch gp0_clk = { + .halt_reg = 0x2fd8, + .halt_bit = 7, + .clkr = { + .enable_reg = 0x2d24, + .enable_mask = BIT(9), + .hw.init = &(struct clk_init_data){ + .name = "gp0_clk", + .parent_names = (const char *[]){ "gp0_src" }, + .num_parents = 1, + .ops = &clk_branch_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_rcg gp1_src = { + .ns_reg = 0x2d44, + .md_reg = 0x2d40, + .mn = { + .mnctr_en_bit = 8, + .mnctr_reset_bit = 7, + .mnctr_mode_shift = 5, + .n_val_shift = 16, + .m_val_shift = 16, + .width = 8, + }, + .p = { + .pre_div_shift = 3, + .pre_div_width = 2, + }, + .s = { + .src_sel_shift = 0, + .parent_map = gcc_pxo_pll8_cxo_map, + }, + .freq_tbl = clk_tbl_gp, + .clkr = { + .enable_reg = 0x2d44, + .enable_mask = BIT(11), + .hw.init = &(struct clk_init_data){ + .name = "gp1_src", + .parent_names = gcc_pxo_pll8_cxo, + .num_parents = 3, + .ops = &clk_rcg_ops, + .flags = CLK_SET_RATE_GATE, + }, + } +}; + +static struct clk_branch gp1_clk = { + .halt_reg = 0x2fd8, + .halt_bit = 6, + .clkr = { + .enable_reg = 0x2d44, + .enable_mask = BIT(9), + .hw.init = &(struct clk_init_data){ + .name = "gp1_clk", + .parent_names = (const char *[]){ "gp1_src" }, + .num_parents = 1, + .ops = &clk_branch_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_rcg gp2_src = { + .ns_reg = 0x2d64, + .md_reg = 0x2d60, + .mn = { + .mnctr_en_bit = 8, + .mnctr_reset_bit = 7, + .mnctr_mode_shift = 5, + .n_val_shift = 16, + .m_val_shift = 16, + .width = 8, + }, + .p = { + .pre_div_shift = 3, + .pre_div_width = 2, + }, + .s = { + .src_sel_shift = 0, + .parent_map = gcc_pxo_pll8_cxo_map, + }, + .freq_tbl = clk_tbl_gp, + .clkr = { + .enable_reg = 0x2d64, + .enable_mask = BIT(11), + .hw.init = &(struct clk_init_data){ + .name = "gp2_src", + .parent_names = gcc_pxo_pll8_cxo, + .num_parents = 3, + .ops = &clk_rcg_ops, + .flags = CLK_SET_RATE_GATE, + }, + } +}; + +static struct clk_branch gp2_clk = { + .halt_reg = 0x2fd8, + .halt_bit = 5, + .clkr = { + .enable_reg = 0x2d64, + .enable_mask = BIT(9), + .hw.init = &(struct clk_init_data){ + .name = "gp2_clk", + .parent_names = (const char *[]){ "gp2_src" }, + .num_parents = 1, + .ops = &clk_branch_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch pmem_clk = { + .hwcg_reg = 0x25a0, + .hwcg_bit = 6, + .halt_reg = 0x2fc8, + .halt_bit = 20, + .clkr = { + .enable_reg = 0x25a0, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "pmem_clk", + .ops = &clk_branch_ops, + .flags = CLK_IS_ROOT, + }, + }, +}; + +static struct clk_rcg prng_src = { + .ns_reg = 0x2e80, + .p = { + .pre_div_shift = 3, + .pre_div_width = 4, + }, + .s = { + .src_sel_shift = 0, + .parent_map = gcc_pxo_pll8_map, + }, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "prng_src", + .parent_names = gcc_pxo_pll8, + .num_parents = 2, + .ops = &clk_rcg_ops, + }, + }, +}; + +static struct clk_branch prng_clk = { + .halt_reg = 0x2fd8, + .halt_check = BRANCH_HALT_VOTED, + .halt_bit = 10, + .clkr = { + .enable_reg = 0x3080, + .enable_mask = BIT(10), + .hw.init = &(struct clk_init_data){ + .name = "prng_clk", + .parent_names = (const char *[]){ "prng_src" }, + .num_parents = 1, + .ops = &clk_branch_ops, + }, + }, +}; + +static const struct freq_tbl clk_tbl_sdc[] = { + { 144000, P_PXO, 5, 18,625 }, + { 400000, P_PLL8, 4, 1, 240 }, + { 16000000, P_PLL8, 4, 1, 6 }, + { 17070000, P_PLL8, 1, 2, 45 }, + { 20210000, P_PLL8, 1, 1, 19 }, + { 24000000, P_PLL8, 4, 1, 4 }, + { 48000000, P_PLL8, 4, 1, 2 }, + { 64000000, P_PLL8, 3, 1, 2 }, + { 96000000, P_PLL8, 4, 0, 0 }, + { 192000000, P_PLL8, 2, 0, 0 }, + { } +}; + +static struct clk_rcg sdc1_src = { + .ns_reg = 0x282c, + .md_reg = 0x2828, + .mn = { + .mnctr_en_bit = 8, + .mnctr_reset_bit = 7, + .mnctr_mode_shift = 5, + .n_val_shift = 16, + .m_val_shift = 16, + .width = 8, + }, + .p = { + .pre_div_shift = 3, + .pre_div_width = 2, + }, + .s = { + .src_sel_shift = 0, + .parent_map = gcc_pxo_pll8_map, + }, + .freq_tbl = clk_tbl_sdc, + .clkr = { + .enable_reg = 0x282c, + .enable_mask = BIT(11), + .hw.init = &(struct clk_init_data){ + .name = "sdc1_src", + .parent_names = gcc_pxo_pll8, + .num_parents = 2, + .ops = &clk_rcg_ops, + .flags = CLK_SET_RATE_GATE, + }, + } +}; + +static struct clk_branch sdc1_clk = { + .halt_reg = 0x2fc8, + .halt_bit = 6, + .clkr = { + .enable_reg = 0x282c, + .enable_mask = BIT(9), + .hw.init = &(struct clk_init_data){ + .name = "sdc1_clk", + .parent_names = (const char *[]){ "sdc1_src" }, + .num_parents = 1, + .ops = &clk_branch_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_rcg sdc3_src = { + .ns_reg = 0x286c, + .md_reg = 0x2868, + .mn = { + .mnctr_en_bit = 8, + .mnctr_reset_bit = 7, + .mnctr_mode_shift = 5, + .n_val_shift = 16, + .m_val_shift = 16, + .width = 8, + }, + .p = { + .pre_div_shift = 3, + .pre_div_width = 2, + }, + .s = { + .src_sel_shift = 0, + .parent_map = gcc_pxo_pll8_map, + }, + .freq_tbl = clk_tbl_sdc, + .clkr = { + .enable_reg = 0x286c, + .enable_mask = BIT(11), + .hw.init = &(struct clk_init_data){ + .name = "sdc3_src", + .parent_names = gcc_pxo_pll8, + .num_parents = 2, + .ops = &clk_rcg_ops, + .flags = CLK_SET_RATE_GATE, + }, + } +}; + +static struct clk_branch sdc3_clk = { + .halt_reg = 0x2fc8, + .halt_bit = 4, + .clkr = { + .enable_reg = 0x286c, + .enable_mask = BIT(9), + .hw.init = &(struct clk_init_data){ + .name = "sdc3_clk", + .parent_names = (const char *[]){ "sdc3_src" }, + .num_parents = 1, + .ops = &clk_branch_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch sdc1_h_clk = { + .hwcg_reg = 0x2820, + .hwcg_bit = 6, + .halt_reg = 0x2fc8, + .halt_bit = 11, + .clkr = { + .enable_reg = 0x2820, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "sdc1_h_clk", + .ops = &clk_branch_ops, + .flags = CLK_IS_ROOT, + }, + }, +}; + +static struct clk_branch sdc3_h_clk = { + .hwcg_reg = 0x2860, + .hwcg_bit = 6, + .halt_reg = 0x2fc8, + .halt_bit = 9, + .clkr = { + .enable_reg = 0x2860, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "sdc3_h_clk", + .ops = &clk_branch_ops, + .flags = CLK_IS_ROOT, + }, + }, +}; + +static const struct freq_tbl clk_tbl_tsif_ref[] = { + { 105000, P_PXO, 1, 1, 256 }, + { } +}; + +static struct clk_rcg tsif_ref_src = { + .ns_reg = 0x2710, + .md_reg = 0x270c, + .mn = { + .mnctr_en_bit = 8, + .mnctr_reset_bit = 7, + .mnctr_mode_shift = 5, + .n_val_shift = 16, + .m_val_shift = 16, + .width = 16, + }, + .p = { + .pre_div_shift = 3, + .pre_div_width = 2, + }, + .s = { + .src_sel_shift = 0, + .parent_map = gcc_pxo_pll8_map, + }, + .freq_tbl = clk_tbl_tsif_ref, + .clkr = { + .enable_reg = 0x2710, + .enable_mask = BIT(11), + .hw.init = &(struct clk_init_data){ + .name = "tsif_ref_src", + .parent_names = gcc_pxo_pll8, + .num_parents = 2, + .ops = &clk_rcg_ops, + .flags = CLK_SET_RATE_GATE, + }, + } +}; + +static struct clk_branch tsif_ref_clk = { + .halt_reg = 0x2fd4, + .halt_bit = 5, + .clkr = { + .enable_reg = 0x2710, + .enable_mask = BIT(9), + .hw.init = &(struct clk_init_data){ + .name = "tsif_ref_clk", + .parent_names = (const char *[]){ "tsif_ref_src" }, + .num_parents = 1, + .ops = &clk_branch_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch tsif_h_clk = { + .hwcg_reg = 0x2700, + .hwcg_bit = 6, + .halt_reg = 0x2fd4, + .halt_bit = 7, + .clkr = { + .enable_reg = 0x2700, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "tsif_h_clk", + .ops = &clk_branch_ops, + .flags = CLK_IS_ROOT, + }, + }, +}; + +static struct clk_branch dma_bam_h_clk = { + .hwcg_reg = 0x25c0, + .hwcg_bit = 6, + .halt_reg = 0x2fc8, + .halt_bit = 12, + .clkr = { + .enable_reg = 0x25c0, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "dma_bam_h_clk", + .ops = &clk_branch_ops, + .flags = CLK_IS_ROOT, + }, + }, +}; + +static struct clk_branch adm0_clk = { + .halt_reg = 0x2fdc, + .halt_check = BRANCH_HALT_VOTED, + .halt_bit = 12, + .clkr = { + .enable_reg = 0x3080, + .enable_mask = BIT(2), + .hw.init = &(struct clk_init_data){ + .name = "adm0_clk", + .ops = &clk_branch_ops, + .flags = CLK_IS_ROOT, + }, + }, +}; + +static struct clk_branch adm0_pbus_clk = { + .hwcg_reg = 0x2208, + .hwcg_bit = 6, + .halt_reg = 0x2fdc, + .halt_check = BRANCH_HALT_VOTED, + .halt_bit = 11, + .clkr = { + .enable_reg = 0x3080, + .enable_mask = BIT(3), + .hw.init = &(struct clk_init_data){ + .name = "adm0_pbus_clk", + .ops = &clk_branch_ops, + .flags = CLK_IS_ROOT, + }, + }, +}; + +static struct clk_branch pmic_arb0_h_clk = { + .halt_reg = 0x2fd8, + .halt_check = BRANCH_HALT_VOTED, + .halt_bit = 22, + .clkr = { + .enable_reg = 0x3080, + .enable_mask = BIT(8), + .hw.init = &(struct clk_init_data){ + .name = "pmic_arb0_h_clk", + .ops = &clk_branch_ops, + .flags = CLK_IS_ROOT, + }, + }, +}; + +static struct clk_branch pmic_arb1_h_clk = { + .halt_reg = 0x2fd8, + .halt_check = BRANCH_HALT_VOTED, + .halt_bit = 21, + .clkr = { + .enable_reg = 0x3080, + .enable_mask = BIT(9), + .hw.init = &(struct clk_init_data){ + .name = "pmic_arb1_h_clk", + .ops = &clk_branch_ops, + .flags = CLK_IS_ROOT, + }, + }, +}; + +static struct clk_branch pmic_ssbi2_clk = { + .halt_reg = 0x2fd8, + .halt_check = BRANCH_HALT_VOTED, + .halt_bit = 23, + .clkr = { + .enable_reg = 0x3080, + .enable_mask = BIT(7), + .hw.init = &(struct clk_init_data){ + .name = "pmic_ssbi2_clk", + .ops = &clk_branch_ops, + .flags = CLK_IS_ROOT, + }, + }, +}; + +static struct clk_branch rpm_msg_ram_h_clk = { + .hwcg_reg = 0x27e0, + .hwcg_bit = 6, + .halt_reg = 0x2fd8, + .halt_check = BRANCH_HALT_VOTED, + .halt_bit = 12, + .clkr = { + .enable_reg = 0x3080, + .enable_mask = BIT(6), + .hw.init = &(struct clk_init_data){ + .name = "rpm_msg_ram_h_clk", + .ops = &clk_branch_ops, + .flags = CLK_IS_ROOT, + }, + }, +}; + +static const struct freq_tbl clk_tbl_pcie_ref[] = { + { 100000000, P_PLL3, 12, 0, 0 }, + { } +}; + +static struct clk_rcg pcie_ref_src = { + .ns_reg = 0x3860, + .p = { + .pre_div_shift = 3, + .pre_div_width = 4, + }, + .s = { + .src_sel_shift = 0, + .parent_map = gcc_pxo_pll3_map, + }, + .freq_tbl = clk_tbl_pcie_ref, + .clkr = { + .enable_reg = 0x3860, + .enable_mask = BIT(11), + .hw.init = &(struct clk_init_data){ + .name = "pcie_ref_src", + .parent_names = gcc_pxo_pll3, + .num_parents = 2, + .ops = &clk_rcg_ops, + .flags = CLK_SET_RATE_GATE, + }, + }, +}; + +static struct clk_branch pcie_ref_src_clk = { + .halt_reg = 0x2fdc, + .halt_bit = 30, + .clkr = { + .enable_reg = 0x3860, + .enable_mask = BIT(9), + .hw.init = &(struct clk_init_data){ + .name = "pcie_ref_src_clk", + .parent_names = (const char *[]){ "pcie_ref_src" }, + .num_parents = 1, + .ops = &clk_branch_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch pcie_a_clk = { + .halt_reg = 0x2fc0, + .halt_bit = 13, + .clkr = { + .enable_reg = 0x22c0, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "pcie_a_clk", + .ops = &clk_branch_ops, + .flags = CLK_IS_ROOT, + }, + }, +}; + +static struct clk_branch pcie_aux_clk = { + .halt_reg = 0x2fdc, + .halt_bit = 31, + .clkr = { + .enable_reg = 0x22c8, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "pcie_aux_clk", + .ops = &clk_branch_ops, + .flags = CLK_IS_ROOT, + }, + }, +}; + +static struct clk_branch pcie_h_clk = { + .halt_reg = 0x2fd4, + .halt_bit = 8, + .clkr = { + .enable_reg = 0x22cc, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "pcie_h_clk", + .ops = &clk_branch_ops, + .flags = CLK_IS_ROOT, + }, + }, +}; + +static struct clk_branch pcie_phy_clk = { + .halt_reg = 0x2fdc, + .halt_bit = 29, + .clkr = { + .enable_reg = 0x22d0, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "pcie_phy_clk", + .ops = &clk_branch_ops, + .flags = CLK_IS_ROOT, + }, + }, +}; + +static struct clk_rcg pcie1_ref_src = { + .ns_reg = 0x3aa0, + .p = { + .pre_div_shift = 3, + .pre_div_width = 4, + }, + .s = { + .src_sel_shift = 0, + .parent_map = gcc_pxo_pll3_map, + }, + .freq_tbl = clk_tbl_pcie_ref, + .clkr = { + .enable_reg = 0x3aa0, + .enable_mask = BIT(11), + .hw.init = &(struct clk_init_data){ + .name = "pcie1_ref_src", + .parent_names = gcc_pxo_pll3, + .num_parents = 2, + .ops = &clk_rcg_ops, + .flags = CLK_SET_RATE_GATE, + }, + }, +}; + +static struct clk_branch pcie1_ref_src_clk = { + .halt_reg = 0x2fdc, + .halt_bit = 27, + .clkr = { + .enable_reg = 0x3aa0, + .enable_mask = BIT(9), + .hw.init = &(struct clk_init_data){ + .name = "pcie1_ref_src_clk", + .parent_names = (const char *[]){ "pcie1_ref_src" }, + .num_parents = 1, + .ops = &clk_branch_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch pcie1_a_clk = { + .halt_reg = 0x2fc0, + .halt_bit = 10, + .clkr = { + .enable_reg = 0x3a80, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "pcie1_a_clk", + .ops = &clk_branch_ops, + .flags = CLK_IS_ROOT, + }, + }, +}; + +static struct clk_branch pcie1_aux_clk = { + .halt_reg = 0x2fdc, + .halt_bit = 28, + .clkr = { + .enable_reg = 0x3a88, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "pcie1_aux_clk", + .ops = &clk_branch_ops, + .flags = CLK_IS_ROOT, + }, + }, +}; + +static struct clk_branch pcie1_h_clk = { + .halt_reg = 0x2fd4, + .halt_bit = 9, + .clkr = { + .enable_reg = 0x3a8c, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "pcie1_h_clk", + .ops = &clk_branch_ops, + .flags = CLK_IS_ROOT, + }, + }, +}; + +static struct clk_branch pcie1_phy_clk = { + .halt_reg = 0x2fdc, + .halt_bit = 26, + .clkr = { + .enable_reg = 0x3a90, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "pcie1_phy_clk", + .ops = &clk_branch_ops, + .flags = CLK_IS_ROOT, + }, + }, +}; + +static struct clk_rcg pcie2_ref_src = { + .ns_reg = 0x3ae0, + .p = { + .pre_div_shift = 3, + .pre_div_width = 4, + }, + .s = { + .src_sel_shift = 0, + .parent_map = gcc_pxo_pll3_map, + }, + .freq_tbl = clk_tbl_pcie_ref, + .clkr = { + .enable_reg = 0x3ae0, + .enable_mask = BIT(11), + .hw.init = &(struct clk_init_data){ + .name = "pcie2_ref_src", + .parent_names = gcc_pxo_pll3, + .num_parents = 2, + .ops = &clk_rcg_ops, + .flags = CLK_SET_RATE_GATE, + }, + }, +}; + +static struct clk_branch pcie2_ref_src_clk = { + .halt_reg = 0x2fdc, + .halt_bit = 24, + .clkr = { + .enable_reg = 0x3ae0, + .enable_mask = BIT(9), + .hw.init = &(struct clk_init_data){ + .name = "pcie2_ref_src_clk", + .parent_names = (const char *[]){ "pcie2_ref_src" }, + .num_parents = 1, + .ops = &clk_branch_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch pcie2_a_clk = { + .halt_reg = 0x2fc0, + .halt_bit = 9, + .clkr = { + .enable_reg = 0x3ac0, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "pcie2_a_clk", + .ops = &clk_branch_ops, + .flags = CLK_IS_ROOT, + }, + }, +}; + +static struct clk_branch pcie2_aux_clk = { + .halt_reg = 0x2fdc, + .halt_bit = 25, + .clkr = { + .enable_reg = 0x3ac8, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "pcie2_aux_clk", + .ops = &clk_branch_ops, + .flags = CLK_IS_ROOT, + }, + }, +}; + +static struct clk_branch pcie2_h_clk = { + .halt_reg = 0x2fd4, + .halt_bit = 10, + .clkr = { + .enable_reg = 0x3acc, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "pcie2_h_clk", + .ops = &clk_branch_ops, + .flags = CLK_IS_ROOT, + }, + }, +}; + +static struct clk_branch pcie2_phy_clk = { + .halt_reg = 0x2fdc, + .halt_bit = 23, + .clkr = { + .enable_reg = 0x3ad0, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "pcie2_phy_clk", + .ops = &clk_branch_ops, + .flags = CLK_IS_ROOT, + }, + }, +}; + +static const struct freq_tbl clk_tbl_sata_ref[] = { + { 100000000, P_PLL3, 12, 0, 0 }, + { } +}; + +static struct clk_rcg sata_ref_src = { + .ns_reg = 0x2c08, + .p = { + .pre_div_shift = 3, + .pre_div_width = 4, + }, + .s = { + .src_sel_shift = 0, + .parent_map = gcc_pxo_pll3_sata_map, + }, + .freq_tbl = clk_tbl_sata_ref, + .clkr = { + .enable_reg = 0x2c08, + .enable_mask = BIT(7), + .hw.init = &(struct clk_init_data){ + .name = "sata_ref_src", + .parent_names = gcc_pxo_pll3, + .num_parents = 2, + .ops = &clk_rcg_ops, + .flags = CLK_SET_RATE_GATE, + }, + }, +}; + +static struct clk_branch sata_rxoob_clk = { + .halt_reg = 0x2fdc, + .halt_bit = 20, + .clkr = { + .enable_reg = 0x2c0c, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "sata_rxoob_clk", + .parent_names = (const char *[]){ "sata_ref_src" }, + .num_parents = 1, + .ops = &clk_branch_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch sata_pmalive_clk = { + .halt_reg = 0x2fdc, + .halt_bit = 19, + .clkr = { + .enable_reg = 0x2c10, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "sata_pmalive_clk", + .parent_names = (const char *[]){ "sata_ref_src" }, + .num_parents = 1, + .ops = &clk_branch_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch sata_phy_ref_clk = { + .halt_reg = 0x2fdc, + .halt_bit = 18, + .clkr = { + .enable_reg = 0x2c14, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "sata_phy_ref_clk", + .parent_names = (const char *[]){ "pxo" }, + .num_parents = 1, + .ops = &clk_branch_ops, + }, + }, +}; + +static struct clk_branch sata_a_clk = { + .halt_reg = 0x2fc0, + .halt_bit = 12, + .clkr = { + .enable_reg = 0x2c20, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "sata_a_clk", + .ops = &clk_branch_ops, + .flags = CLK_IS_ROOT, + }, + }, +}; + +static struct clk_branch sata_h_clk = { + .halt_reg = 0x2fdc, + .halt_bit = 21, + .clkr = { + .enable_reg = 0x2c00, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "sata_h_clk", + .ops = &clk_branch_ops, + .flags = CLK_IS_ROOT, + }, + }, +}; + +static struct clk_branch sfab_sata_s_h_clk = { + .halt_reg = 0x2fc4, + .halt_bit = 14, + .clkr = { + .enable_reg = 0x2480, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "sfab_sata_s_h_clk", + .ops = &clk_branch_ops, + .flags = CLK_IS_ROOT, + }, + }, +}; + +static struct clk_branch sata_phy_cfg_clk = { + .halt_reg = 0x2fcc, + .halt_bit = 14, + .clkr = { + .enable_reg = 0x2c40, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "sata_phy_cfg_clk", + .ops = &clk_branch_ops, + .flags = CLK_IS_ROOT, + }, + }, +}; + +static const struct freq_tbl clk_tbl_usb30_master[] = { + { 125000000, P_PLL0, 1, 5, 32 }, + { } +}; + +static struct clk_rcg usb30_master_clk_src = { + .ns_reg = 0x3b2c, + .md_reg = 0x3b28, + .mn = { + .mnctr_en_bit = 8, + .mnctr_reset_bit = 7, + .mnctr_mode_shift = 5, + .n_val_shift = 16, + .m_val_shift = 16, + .width = 8, + }, + .p = { + .pre_div_shift = 3, + .pre_div_width = 2, + }, + .s = { + .src_sel_shift = 0, + .parent_map = gcc_pxo_pll8_pll0, + }, + .freq_tbl = clk_tbl_usb30_master, + .clkr = { + .enable_reg = 0x3b2c, + .enable_mask = BIT(11), + .hw.init = &(struct clk_init_data){ + .name = "usb30_master_ref_src", + .parent_names = gcc_pxo_pll8_pll0_map, + .num_parents = 3, + .ops = &clk_rcg_ops, + .flags = CLK_SET_RATE_GATE, + }, + }, +}; + +static struct clk_branch usb30_0_branch_clk = { + .halt_reg = 0x2fc4, + .halt_bit = 22, + .clkr = { + .enable_reg = 0x3b24, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "usb30_0_branch_clk", + .parent_names = (const char *[]){ "usb30_master_ref_src", }, + .num_parents = 1, + .ops = &clk_branch_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch usb30_1_branch_clk = { + .halt_reg = 0x2fc4, + .halt_bit = 17, + .clkr = { + .enable_reg = 0x3b34, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "usb30_1_branch_clk", + .parent_names = (const char *[]){ "usb30_master_ref_src", }, + .num_parents = 1, + .ops = &clk_branch_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static const struct freq_tbl clk_tbl_usb30_utmi[] = { + { 60000000, P_PLL8, 1, 5, 32 }, + { } +}; + +static struct clk_rcg usb30_utmi_clk = { + .ns_reg = 0x3b44, + .md_reg = 0x3b40, + .mn = { + .mnctr_en_bit = 8, + .mnctr_reset_bit = 7, + .mnctr_mode_shift = 5, + .n_val_shift = 16, + .m_val_shift = 16, + .width = 8, + }, + .p = { + .pre_div_shift = 3, + .pre_div_width = 2, + }, + .s = { + .src_sel_shift = 0, + .parent_map = gcc_pxo_pll8_pll0, + }, + .freq_tbl = clk_tbl_usb30_utmi, + .clkr = { + .enable_reg = 0x3b44, + .enable_mask = BIT(11), + .hw.init = &(struct clk_init_data){ + .name = "usb30_utmi_clk", + .parent_names = gcc_pxo_pll8_pll0_map, + .num_parents = 3, + .ops = &clk_rcg_ops, + .flags = CLK_SET_RATE_GATE, + }, + }, +}; + +static struct clk_branch usb30_0_utmi_clk_ctl = { + .halt_reg = 0x2fc4, + .halt_bit = 21, + .clkr = { + .enable_reg = 0x3b48, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "usb30_0_utmi_clk_ctl", + .parent_names = (const char *[]){ "usb30_utmi_clk", }, + .num_parents = 1, + .ops = &clk_branch_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch usb30_1_utmi_clk_ctl = { + .halt_reg = 0x2fc4, + .halt_bit = 15, + .clkr = { + .enable_reg = 0x3b4c, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "usb30_1_utmi_clk_ctl", + .parent_names = (const char *[]){ "usb30_utmi_clk", }, + .num_parents = 1, + .ops = &clk_branch_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static const struct freq_tbl clk_tbl_usb[] = { + { 60000000, P_PLL8, 1, 5, 32 }, + { } +}; + +static struct clk_rcg usb_hs1_xcvr_clk_src = { + .ns_reg = 0x290C, + .md_reg = 0x2908, + .mn = { + .mnctr_en_bit = 8, + .mnctr_reset_bit = 7, + .mnctr_mode_shift = 5, + .n_val_shift = 16, + .m_val_shift = 16, + .width = 8, + }, + .p = { + .pre_div_shift = 3, + .pre_div_width = 2, + }, + .s = { + .src_sel_shift = 0, + .parent_map = gcc_pxo_pll8_pll0, + }, + .freq_tbl = clk_tbl_usb, + .clkr = { + .enable_reg = 0x2968, + .enable_mask = BIT(11), + .hw.init = &(struct clk_init_data){ + .name = "usb_hs1_xcvr_src", + .parent_names = gcc_pxo_pll8_pll0_map, + .num_parents = 3, + .ops = &clk_rcg_ops, + .flags = CLK_SET_RATE_GATE, + }, + }, +}; + +static struct clk_branch usb_hs1_xcvr_clk = { + .halt_reg = 0x2fcc, + .halt_bit = 17, + .clkr = { + .enable_reg = 0x290c, + .enable_mask = BIT(9), + .hw.init = &(struct clk_init_data){ + .name = "usb_hs1_xcvr_clk", + .parent_names = (const char *[]){ "usb_hs1_xcvr_src" }, + .num_parents = 1, + .ops = &clk_branch_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch usb_hs1_h_clk = { + .hwcg_reg = 0x2900, + .hwcg_bit = 6, + .halt_reg = 0x2fc8, + .halt_bit = 1, + .clkr = { + .enable_reg = 0x2900, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "usb_hs1_h_clk", + .ops = &clk_branch_ops, + .flags = CLK_IS_ROOT, + }, + }, +}; + +static struct clk_rcg usb_fs1_xcvr_clk_src = { + .ns_reg = 0x2968, + .md_reg = 0x2964, + .mn = { + .mnctr_en_bit = 8, + .mnctr_reset_bit = 7, + .mnctr_mode_shift = 5, + .n_val_shift = 16, + .m_val_shift = 16, + .width = 8, + }, + .p = { + .pre_div_shift = 3, + .pre_div_width = 2, + }, + .s = { + .src_sel_shift = 0, + .parent_map = gcc_pxo_pll8_pll0, + }, + .freq_tbl = clk_tbl_usb, + .clkr = { + .enable_reg = 0x2968, + .enable_mask = BIT(11), + .hw.init = &(struct clk_init_data){ + .name = "usb_fs1_xcvr_src", + .parent_names = gcc_pxo_pll8_pll0_map, + .num_parents = 3, + .ops = &clk_rcg_ops, + .flags = CLK_SET_RATE_GATE, + }, + }, +}; + +static struct clk_branch usb_fs1_xcvr_clk = { + .halt_reg = 0x2fcc, + .halt_bit = 17, + .clkr = { + .enable_reg = 0x2968, + .enable_mask = BIT(9), + .hw.init = &(struct clk_init_data){ + .name = "usb_fs1_xcvr_clk", + .parent_names = (const char *[]){ "usb_fs1_xcvr_src", }, + .num_parents = 1, + .ops = &clk_branch_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch usb_fs1_sys_clk = { + .halt_reg = 0x2fcc, + .halt_bit = 18, + .clkr = { + .enable_reg = 0x296c, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "usb_fs1_sys_clk", + .parent_names = (const char *[]){ "usb_fs1_xcvr_src", }, + .num_parents = 1, + .ops = &clk_branch_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch usb_fs1_h_clk = { + .halt_reg = 0x2fcc, + .halt_bit = 19, + .clkr = { + .enable_reg = 0x2960, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "usb_fs1_h_clk", + .ops = &clk_branch_ops, + .flags = CLK_IS_ROOT, + }, + }, +}; + +static struct clk_regmap *gcc_ipq806x_clks[] = { + [PLL3] = &pll3.clkr, + [PLL8] = &pll8.clkr, + [PLL8_VOTE] = &pll8_vote, + [PLL14] = &pll14.clkr, + [PLL14_VOTE] = &pll14_vote, + [GSBI1_UART_SRC] = &gsbi1_uart_src.clkr, + [GSBI1_UART_CLK] = &gsbi1_uart_clk.clkr, + [GSBI2_UART_SRC] = &gsbi2_uart_src.clkr, + [GSBI2_UART_CLK] = &gsbi2_uart_clk.clkr, + [GSBI4_UART_SRC] = &gsbi4_uart_src.clkr, + [GSBI4_UART_CLK] = &gsbi4_uart_clk.clkr, + [GSBI5_UART_SRC] = &gsbi5_uart_src.clkr, + [GSBI5_UART_CLK] = &gsbi5_uart_clk.clkr, + [GSBI6_UART_SRC] = &gsbi6_uart_src.clkr, + [GSBI6_UART_CLK] = &gsbi6_uart_clk.clkr, + [GSBI7_UART_SRC] = &gsbi7_uart_src.clkr, + [GSBI7_UART_CLK] = &gsbi7_uart_clk.clkr, + [GSBI1_QUP_SRC] = &gsbi1_qup_src.clkr, + [GSBI1_QUP_CLK] = &gsbi1_qup_clk.clkr, + [GSBI2_QUP_SRC] = &gsbi2_qup_src.clkr, + [GSBI2_QUP_CLK] = &gsbi2_qup_clk.clkr, + [GSBI4_QUP_SRC] = &gsbi4_qup_src.clkr, + [GSBI4_QUP_CLK] = &gsbi4_qup_clk.clkr, + [GSBI5_QUP_SRC] = &gsbi5_qup_src.clkr, + [GSBI5_QUP_CLK] = &gsbi5_qup_clk.clkr, + [GSBI6_QUP_SRC] = &gsbi6_qup_src.clkr, + [GSBI6_QUP_CLK] = &gsbi6_qup_clk.clkr, + [GSBI7_QUP_SRC] = &gsbi7_qup_src.clkr, + [GSBI7_QUP_CLK] = &gsbi7_qup_clk.clkr, + [GP0_SRC] = &gp0_src.clkr, + [GP0_CLK] = &gp0_clk.clkr, + [GP1_SRC] = &gp1_src.clkr, + [GP1_CLK] = &gp1_clk.clkr, + [GP2_SRC] = &gp2_src.clkr, + [GP2_CLK] = &gp2_clk.clkr, + [PMEM_A_CLK] = &pmem_clk.clkr, + [PRNG_SRC] = &prng_src.clkr, + [PRNG_CLK] = &prng_clk.clkr, + [SDC1_SRC] = &sdc1_src.clkr, + [SDC1_CLK] = &sdc1_clk.clkr, + [SDC3_SRC] = &sdc3_src.clkr, + [SDC3_CLK] = &sdc3_clk.clkr, + [TSIF_REF_SRC] = &tsif_ref_src.clkr, + [TSIF_REF_CLK] = &tsif_ref_clk.clkr, + [DMA_BAM_H_CLK] = &dma_bam_h_clk.clkr, + [GSBI1_H_CLK] = &gsbi1_h_clk.clkr, + [GSBI2_H_CLK] = &gsbi2_h_clk.clkr, + [GSBI4_H_CLK] = &gsbi4_h_clk.clkr, + [GSBI5_H_CLK] = &gsbi5_h_clk.clkr, + [GSBI6_H_CLK] = &gsbi6_h_clk.clkr, + [GSBI7_H_CLK] = &gsbi7_h_clk.clkr, + [TSIF_H_CLK] = &tsif_h_clk.clkr, + [SDC1_H_CLK] = &sdc1_h_clk.clkr, + [SDC3_H_CLK] = &sdc3_h_clk.clkr, + [ADM0_CLK] = &adm0_clk.clkr, + [ADM0_PBUS_CLK] = &adm0_pbus_clk.clkr, + [PCIE_A_CLK] = &pcie_a_clk.clkr, + [PCIE_AUX_CLK] = &pcie_aux_clk.clkr, + [PCIE_H_CLK] = &pcie_h_clk.clkr, + [PCIE_PHY_CLK] = &pcie_phy_clk.clkr, + [SFAB_SATA_S_H_CLK] = &sfab_sata_s_h_clk.clkr, + [PMIC_ARB0_H_CLK] = &pmic_arb0_h_clk.clkr, + [PMIC_ARB1_H_CLK] = &pmic_arb1_h_clk.clkr, + [PMIC_SSBI2_CLK] = &pmic_ssbi2_clk.clkr, + [RPM_MSG_RAM_H_CLK] = &rpm_msg_ram_h_clk.clkr, + [SATA_H_CLK] = &sata_h_clk.clkr, + [SATA_CLK_SRC] = &sata_ref_src.clkr, + [SATA_RXOOB_CLK] = &sata_rxoob_clk.clkr, + [SATA_PMALIVE_CLK] = &sata_pmalive_clk.clkr, + [SATA_PHY_REF_CLK] = &sata_phy_ref_clk.clkr, + [SATA_A_CLK] = &sata_a_clk.clkr, + [SATA_PHY_CFG_CLK] = &sata_phy_cfg_clk.clkr, + [PCIE_ALT_REF_SRC] = &pcie_ref_src.clkr, + [PCIE_ALT_REF_CLK] = &pcie_ref_src_clk.clkr, + [PCIE_1_A_CLK] = &pcie1_a_clk.clkr, + [PCIE_1_AUX_CLK] = &pcie1_aux_clk.clkr, + [PCIE_1_H_CLK] = &pcie1_h_clk.clkr, + [PCIE_1_PHY_CLK] = &pcie1_phy_clk.clkr, + [PCIE_1_ALT_REF_SRC] = &pcie1_ref_src.clkr, + [PCIE_1_ALT_REF_CLK] = &pcie1_ref_src_clk.clkr, + [PCIE_2_A_CLK] = &pcie2_a_clk.clkr, + [PCIE_2_AUX_CLK] = &pcie2_aux_clk.clkr, + [PCIE_2_H_CLK] = &pcie2_h_clk.clkr, + [PCIE_2_PHY_CLK] = &pcie2_phy_clk.clkr, + [PCIE_2_ALT_REF_SRC] = &pcie2_ref_src.clkr, + [PCIE_2_ALT_REF_CLK] = &pcie2_ref_src_clk.clkr, + [USB30_MASTER_SRC] = &usb30_master_clk_src.clkr, + [USB30_0_MASTER_CLK] = &usb30_0_branch_clk.clkr, + [USB30_1_MASTER_CLK] = &usb30_1_branch_clk.clkr, + [USB30_UTMI_SRC] = &usb30_utmi_clk.clkr, + [USB30_0_UTMI_CLK] = &usb30_0_utmi_clk_ctl.clkr, + [USB30_1_UTMI_CLK] = &usb30_1_utmi_clk_ctl.clkr, + [USB_HS1_H_CLK] = &usb_hs1_h_clk.clkr, + [USB_HS1_XCVR_SRC] = &usb_hs1_xcvr_clk_src.clkr, + [USB_HS1_XCVR_CLK] = &usb_hs1_xcvr_clk.clkr, + [USB_FS1_H_CLK] = &usb_fs1_h_clk.clkr, + [USB_FS1_XCVR_SRC] = &usb_fs1_xcvr_clk_src.clkr, + [USB_FS1_XCVR_CLK] = &usb_fs1_xcvr_clk.clkr, + [USB_FS1_SYSTEM_CLK] = &usb_fs1_sys_clk.clkr, +}; + +static const struct qcom_reset_map gcc_ipq806x_resets[] = { + [QDSS_STM_RESET] = { 0x2060, 6 }, + [AFAB_SMPSS_S_RESET] = { 0x20b8, 2 }, + [AFAB_SMPSS_M1_RESET] = { 0x20b8, 1 }, + [AFAB_SMPSS_M0_RESET] = { 0x20b8, 0 }, + [AFAB_EBI1_CH0_RESET] = { 0x20c0, 7 }, + [AFAB_EBI1_CH1_RESET] = { 0x20c4, 7 }, + [SFAB_ADM0_M0_RESET] = { 0x21e0, 7 }, + [SFAB_ADM0_M1_RESET] = { 0x21e4, 7 }, + [SFAB_ADM0_M2_RESET] = { 0x21e8, 7 }, + [ADM0_C2_RESET] = { 0x220c, 4 }, + [ADM0_C1_RESET] = { 0x220c, 3 }, + [ADM0_C0_RESET] = { 0x220c, 2 }, + [ADM0_PBUS_RESET] = { 0x220c, 1 }, + [ADM0_RESET] = { 0x220c, 0 }, + [QDSS_CLKS_SW_RESET] = { 0x2260, 5 }, + [QDSS_POR_RESET] = { 0x2260, 4 }, + [QDSS_TSCTR_RESET] = { 0x2260, 3 }, + [QDSS_HRESET_RESET] = { 0x2260, 2 }, + [QDSS_AXI_RESET] = { 0x2260, 1 }, + [QDSS_DBG_RESET] = { 0x2260, 0 }, + [SFAB_PCIE_M_RESET] = { 0x22d8, 1 }, + [SFAB_PCIE_S_RESET] = { 0x22d8, 0 }, + [PCIE_EXT_RESET] = { 0x22dc, 6 }, + [PCIE_PHY_RESET] = { 0x22dc, 5 }, + [PCIE_PCI_RESET] = { 0x22dc, 4 }, + [PCIE_POR_RESET] = { 0x22dc, 3 }, + [PCIE_HCLK_RESET] = { 0x22dc, 2 }, + [PCIE_ACLK_RESET] = { 0x22dc, 0 }, + [SFAB_LPASS_RESET] = { 0x23a0, 7 }, + [SFAB_AFAB_M_RESET] = { 0x23e0, 7 }, + [AFAB_SFAB_M0_RESET] = { 0x2420, 7 }, + [AFAB_SFAB_M1_RESET] = { 0x2424, 7 }, + [SFAB_SATA_S_RESET] = { 0x2480, 7 }, + [SFAB_DFAB_M_RESET] = { 0x2500, 7 }, + [DFAB_SFAB_M_RESET] = { 0x2520, 7 }, + [DFAB_SWAY0_RESET] = { 0x2540, 7 }, + [DFAB_SWAY1_RESET] = { 0x2544, 7 }, + [DFAB_ARB0_RESET] = { 0x2560, 7 }, + [DFAB_ARB1_RESET] = { 0x2564, 7 }, + [PPSS_PROC_RESET] = { 0x2594, 1 }, + [PPSS_RESET] = { 0x2594, 0 }, + [DMA_BAM_RESET] = { 0x25c0, 7 }, + [SPS_TIC_H_RESET] = { 0x2600, 7 }, + [SFAB_CFPB_M_RESET] = { 0x2680, 7 }, + [SFAB_CFPB_S_RESET] = { 0x26c0, 7 }, + [TSIF_H_RESET] = { 0x2700, 7 }, + [CE1_H_RESET] = { 0x2720, 7 }, + [CE1_CORE_RESET] = { 0x2724, 7 }, + [CE1_SLEEP_RESET] = { 0x2728, 7 }, + [CE2_H_RESET] = { 0x2740, 7 }, + [CE2_CORE_RESET] = { 0x2744, 7 }, + [SFAB_SFPB_M_RESET] = { 0x2780, 7 }, + [SFAB_SFPB_S_RESET] = { 0x27a0, 7 }, + [RPM_PROC_RESET] = { 0x27c0, 7 }, + [PMIC_SSBI2_RESET] = { 0x280c, 12 }, + [SDC1_RESET] = { 0x2830, 0 }, + [SDC2_RESET] = { 0x2850, 0 }, + [SDC3_RESET] = { 0x2870, 0 }, + [SDC4_RESET] = { 0x2890, 0 }, + [USB_HS1_RESET] = { 0x2910, 0 }, + [USB_HSIC_RESET] = { 0x2934, 0 }, + [USB_FS1_XCVR_RESET] = { 0x2974, 1 }, + [USB_FS1_RESET] = { 0x2974, 0 }, + [GSBI1_RESET] = { 0x29dc, 0 }, + [GSBI2_RESET] = { 0x29fc, 0 }, + [GSBI3_RESET] = { 0x2a1c, 0 }, + [GSBI4_RESET] = { 0x2a3c, 0 }, + [GSBI5_RESET] = { 0x2a5c, 0 }, + [GSBI6_RESET] = { 0x2a7c, 0 }, + [GSBI7_RESET] = { 0x2a9c, 0 }, + [SPDM_RESET] = { 0x2b6c, 0 }, + [SEC_CTRL_RESET] = { 0x2b80, 7 }, + [TLMM_H_RESET] = { 0x2ba0, 7 }, + [SFAB_SATA_M_RESET] = { 0x2c18, 0 }, + [SATA_RESET] = { 0x2c1c, 0 }, + [TSSC_RESET] = { 0x2ca0, 7 }, + [PDM_RESET] = { 0x2cc0, 12 }, + [MPM_H_RESET] = { 0x2da0, 7 }, + [MPM_RESET] = { 0x2da4, 0 }, + [SFAB_SMPSS_S_RESET] = { 0x2e00, 7 }, + [PRNG_RESET] = { 0x2e80, 12 }, + [SFAB_CE3_M_RESET] = { 0x36c8, 1 }, + [SFAB_CE3_S_RESET] = { 0x36c8, 0 }, + [CE3_SLEEP_RESET] = { 0x36d0, 7 }, + [PCIE_1_M_RESET] = { 0x3a98, 1 }, + [PCIE_1_S_RESET] = { 0x3a98, 0 }, + [PCIE_1_EXT_RESET] = { 0x3a9c, 6 }, + [PCIE_1_PHY_RESET] = { 0x3a9c, 5 }, + [PCIE_1_PCI_RESET] = { 0x3a9c, 4 }, + [PCIE_1_POR_RESET] = { 0x3a9c, 3 }, + [PCIE_1_HCLK_RESET] = { 0x3a9c, 2 }, + [PCIE_1_ACLK_RESET] = { 0x3a9c, 0 }, + [PCIE_2_M_RESET] = { 0x3ad8, 1 }, + [PCIE_2_S_RESET] = { 0x3ad8, 0 }, + [PCIE_2_EXT_RESET] = { 0x3adc, 6 }, + [PCIE_2_PHY_RESET] = { 0x3adc, 5 }, + [PCIE_2_PCI_RESET] = { 0x3adc, 4 }, + [PCIE_2_POR_RESET] = { 0x3adc, 3 }, + [PCIE_2_HCLK_RESET] = { 0x3adc, 2 }, + [PCIE_2_ACLK_RESET] = { 0x3adc, 0 }, + [SFAB_USB30_S_RESET] = { 0x3b54, 1 }, + [SFAB_USB30_M_RESET] = { 0x3b54, 0 }, + [USB30_0_PORT2_HS_PHY_RESET] = { 0x3b50, 5 }, + [USB30_0_MASTER_RESET] = { 0x3b50, 4 }, + [USB30_0_SLEEP_RESET] = { 0x3b50, 3 }, + [USB30_0_UTMI_PHY_RESET] = { 0x3b50, 2 }, + [USB30_0_POWERON_RESET] = { 0x3b50, 1 }, + [USB30_0_PHY_RESET] = { 0x3b50, 0 }, + [USB30_1_MASTER_RESET] = { 0x3b58, 4 }, + [USB30_1_SLEEP_RESET] = { 0x3b58, 3 }, + [USB30_1_UTMI_PHY_RESET] = { 0x3b58, 2 }, + [USB30_1_POWERON_RESET] = { 0x3b58, 1 }, + [USB30_1_PHY_RESET] = { 0x3b58, 0 }, + [NSSFB0_RESET] = { 0x3b60, 6 }, + [NSSFB1_RESET] = { 0x3b60, 7 }, +}; + +static const struct regmap_config gcc_ipq806x_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0x3e40, + .fast_io = true, +}; + +static const struct qcom_cc_desc gcc_ipq806x_desc = { + .config = &gcc_ipq806x_regmap_config, + .clks = gcc_ipq806x_clks, + .num_clks = ARRAY_SIZE(gcc_ipq806x_clks), + .resets = gcc_ipq806x_resets, + .num_resets = ARRAY_SIZE(gcc_ipq806x_resets), +}; + +static const struct of_device_id gcc_ipq806x_match_table[] = { + { .compatible = "qcom,gcc-ipq8064" }, + { } +}; +MODULE_DEVICE_TABLE(of, gcc_ipq806x_match_table); + +static int gcc_ipq806x_probe(struct platform_device *pdev) +{ + struct clk *clk; + struct device *dev = &pdev->dev; + + /* Temporary until RPM clocks supported */ + clk = clk_register_fixed_rate(dev, "cxo", NULL, CLK_IS_ROOT, 25000000); + if (IS_ERR(clk)) + return PTR_ERR(clk); + + clk = clk_register_fixed_rate(dev, "pxo", NULL, CLK_IS_ROOT, 25000000); + if (IS_ERR(clk)) + return PTR_ERR(clk); + + return qcom_cc_probe(pdev, &gcc_ipq806x_desc); +} + +static int gcc_ipq806x_remove(struct platform_device *pdev) +{ + qcom_cc_remove(pdev); + return 0; +} + +static struct platform_driver gcc_ipq806x_driver = { + .probe = gcc_ipq806x_probe, + .remove = gcc_ipq806x_remove, + .driver = { + .name = "gcc-ipq806x", + .owner = THIS_MODULE, + .of_match_table = gcc_ipq806x_match_table, + }, +}; + +static int __init gcc_ipq806x_init(void) +{ + return platform_driver_register(&gcc_ipq806x_driver); +} +core_initcall(gcc_ipq806x_init); + +static void __exit gcc_ipq806x_exit(void) +{ + platform_driver_unregister(&gcc_ipq806x_driver); +} +module_exit(gcc_ipq806x_exit); + +MODULE_DESCRIPTION("QCOM GCC IPQ806x Driver"); +MODULE_LICENSE("GPL v2"); +MODULE_ALIAS("platform:gcc-ipq806x"); diff --git a/drivers/clk/qcom/gcc-msm8960.c b/drivers/clk/qcom/gcc-msm8960.c index f4ffd91901f8..007534f7a2d7 100644 --- a/drivers/clk/qcom/gcc-msm8960.c +++ b/drivers/clk/qcom/gcc-msm8960.c @@ -104,6 +104,7 @@ static struct clk_regmap pll14_vote = { #define P_PXO 0 #define P_PLL8 1 +#define P_PLL3 2 #define P_CXO 2 static const u8 gcc_pxo_pll8_map[] = { @@ -128,6 +129,18 @@ static const char *gcc_pxo_pll8_cxo[] = { "cxo", }; +static const u8 gcc_pxo_pll8_pll3_map[] = { + [P_PXO] = 0, + [P_PLL8] = 3, + [P_PLL3] = 6, +}; + +static const char *gcc_pxo_pll8_pll3[] = { + "pxo", + "pll8_vote", + "pll3", +}; + static struct freq_tbl clk_tbl_gsbi_uart[] = { { 1843200, P_PLL8, 2, 6, 625 }, { 3686400, P_PLL8, 2, 12, 625 }, @@ -1928,6 +1941,104 @@ static struct clk_branch usb_hs1_xcvr_clk = { }, }; +static struct clk_rcg usb_hs3_xcvr_src = { + .ns_reg = 0x370c, + .md_reg = 0x3708, + .mn = { + .mnctr_en_bit = 8, + .mnctr_reset_bit = 7, + .mnctr_mode_shift = 5, + .n_val_shift = 16, + .m_val_shift = 16, + .width = 8, + }, + .p = { + .pre_div_shift = 3, + .pre_div_width = 2, + }, + .s = { + .src_sel_shift = 0, + .parent_map = gcc_pxo_pll8_map, + }, + .freq_tbl = clk_tbl_usb, + .clkr = { + .enable_reg = 0x370c, + .enable_mask = BIT(11), + .hw.init = &(struct clk_init_data){ + .name = "usb_hs3_xcvr_src", + .parent_names = gcc_pxo_pll8, + .num_parents = 2, + .ops = &clk_rcg_ops, + .flags = CLK_SET_RATE_GATE, + }, + } +}; + +static struct clk_branch usb_hs3_xcvr_clk = { + .halt_reg = 0x2fc8, + .halt_bit = 30, + .clkr = { + .enable_reg = 0x370c, + .enable_mask = BIT(9), + .hw.init = &(struct clk_init_data){ + .name = "usb_hs3_xcvr_clk", + .parent_names = (const char *[]){ "usb_hs3_xcvr_src" }, + .num_parents = 1, + .ops = &clk_branch_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_rcg usb_hs4_xcvr_src = { + .ns_reg = 0x372c, + .md_reg = 0x3728, + .mn = { + .mnctr_en_bit = 8, + .mnctr_reset_bit = 7, + .mnctr_mode_shift = 5, + .n_val_shift = 16, + .m_val_shift = 16, + .width = 8, + }, + .p = { + .pre_div_shift = 3, + .pre_div_width = 2, + }, + .s = { + .src_sel_shift = 0, + .parent_map = gcc_pxo_pll8_map, + }, + .freq_tbl = clk_tbl_usb, + .clkr = { + .enable_reg = 0x372c, + .enable_mask = BIT(11), + .hw.init = &(struct clk_init_data){ + .name = "usb_hs4_xcvr_src", + .parent_names = gcc_pxo_pll8, + .num_parents = 2, + .ops = &clk_rcg_ops, + .flags = CLK_SET_RATE_GATE, + }, + } +}; + +static struct clk_branch usb_hs4_xcvr_clk = { + .halt_reg = 0x2fc8, + .halt_bit = 2, + .clkr = { + .enable_reg = 0x372c, + .enable_mask = BIT(9), + .hw.init = &(struct clk_init_data){ + .name = "usb_hs4_xcvr_clk", + .parent_names = (const char *[]){ "usb_hs4_xcvr_src" }, + .num_parents = 1, + .ops = &clk_branch_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + static struct clk_rcg usb_hsic_xcvr_fs_src = { .ns_reg = 0x2928, .md_reg = 0x2924, @@ -2456,6 +2567,34 @@ static struct clk_branch usb_hs1_h_clk = { }, }; +static struct clk_branch usb_hs3_h_clk = { + .halt_reg = 0x2fc8, + .halt_bit = 31, + .clkr = { + .enable_reg = 0x3700, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "usb_hs3_h_clk", + .ops = &clk_branch_ops, + .flags = CLK_IS_ROOT, + }, + }, +}; + +static struct clk_branch usb_hs4_h_clk = { + .halt_reg = 0x2fc8, + .halt_bit = 7, + .clkr = { + .enable_reg = 0x3720, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "usb_hs4_h_clk", + .ops = &clk_branch_ops, + .flags = CLK_IS_ROOT, + }, + }, +}; + static struct clk_branch usb_hsic_h_clk = { .halt_reg = 0x2fcc, .halt_bit = 28, @@ -2582,6 +2721,244 @@ static struct clk_branch adm0_pbus_clk = { }, }; +static struct freq_tbl clk_tbl_ce3[] = { + { 48000000, P_PLL8, 8 }, + { 100000000, P_PLL3, 12 }, + { 120000000, P_PLL3, 10 }, + { } +}; + +static struct clk_rcg ce3_src = { + .ns_reg = 0x36c0, + .p = { + .pre_div_shift = 3, + .pre_div_width = 4, + }, + .s = { + .src_sel_shift = 0, + .parent_map = gcc_pxo_pll8_pll3_map, + }, + .freq_tbl = clk_tbl_ce3, + .clkr = { + .enable_reg = 0x2c08, + .enable_mask = BIT(7), + .hw.init = &(struct clk_init_data){ + .name = "ce3_src", + .parent_names = gcc_pxo_pll8_pll3, + .num_parents = 3, + .ops = &clk_rcg_ops, + .flags = CLK_SET_RATE_GATE, + }, + }, +}; + +static struct clk_branch ce3_core_clk = { + .halt_reg = 0x2fdc, + .halt_bit = 5, + .clkr = { + .enable_reg = 0x36c4, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "ce3_core_clk", + .parent_names = (const char *[]){ "ce3_src" }, + .num_parents = 1, + .ops = &clk_branch_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch ce3_h_clk = { + .halt_reg = 0x2fc4, + .halt_bit = 16, + .clkr = { + .enable_reg = 0x36c4, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "ce3_h_clk", + .parent_names = (const char *[]){ "ce3_src" }, + .num_parents = 1, + .ops = &clk_branch_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static const struct freq_tbl clk_tbl_sata_ref[] = { + { 48000000, P_PLL8, 8, 0, 0 }, + { 100000000, P_PLL3, 12, 0, 0 }, + { } +}; + +static struct clk_rcg sata_clk_src = { + .ns_reg = 0x2c08, + .p = { + .pre_div_shift = 3, + .pre_div_width = 4, + }, + .s = { + .src_sel_shift = 0, + .parent_map = gcc_pxo_pll8_pll3_map, + }, + .freq_tbl = clk_tbl_sata_ref, + .clkr = { + .enable_reg = 0x2c08, + .enable_mask = BIT(7), + .hw.init = &(struct clk_init_data){ + .name = "sata_clk_src", + .parent_names = gcc_pxo_pll8_pll3, + .num_parents = 3, + .ops = &clk_rcg_ops, + .flags = CLK_SET_RATE_GATE, + }, + }, +}; + +static struct clk_branch sata_rxoob_clk = { + .halt_reg = 0x2fdc, + .halt_bit = 26, + .clkr = { + .enable_reg = 0x2c0c, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "sata_rxoob_clk", + .parent_names = (const char *[]){ "sata_clk_src" }, + .num_parents = 1, + .ops = &clk_branch_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch sata_pmalive_clk = { + .halt_reg = 0x2fdc, + .halt_bit = 25, + .clkr = { + .enable_reg = 0x2c10, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "sata_pmalive_clk", + .parent_names = (const char *[]){ "sata_clk_src" }, + .num_parents = 1, + .ops = &clk_branch_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch sata_phy_ref_clk = { + .halt_reg = 0x2fdc, + .halt_bit = 24, + .clkr = { + .enable_reg = 0x2c14, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "sata_phy_ref_clk", + .parent_names = (const char *[]){ "pxo" }, + .num_parents = 1, + .ops = &clk_branch_ops, + }, + }, +}; + +static struct clk_branch sata_a_clk = { + .halt_reg = 0x2fc0, + .halt_bit = 12, + .clkr = { + .enable_reg = 0x2c20, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "sata_a_clk", + .ops = &clk_branch_ops, + .flags = CLK_IS_ROOT, + }, + }, +}; + +static struct clk_branch sata_h_clk = { + .halt_reg = 0x2fdc, + .halt_bit = 27, + .clkr = { + .enable_reg = 0x2c00, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "sata_h_clk", + .ops = &clk_branch_ops, + .flags = CLK_IS_ROOT, + }, + }, +}; + +static struct clk_branch sfab_sata_s_h_clk = { + .halt_reg = 0x2fc4, + .halt_bit = 14, + .clkr = { + .enable_reg = 0x2480, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "sfab_sata_s_h_clk", + .ops = &clk_branch_ops, + .flags = CLK_IS_ROOT, + }, + }, +}; + +static struct clk_branch sata_phy_cfg_clk = { + .halt_reg = 0x2fcc, + .halt_bit = 12, + .clkr = { + .enable_reg = 0x2c40, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "sata_phy_cfg_clk", + .ops = &clk_branch_ops, + .flags = CLK_IS_ROOT, + }, + }, +}; + +static struct clk_branch pcie_phy_ref_clk = { + .halt_reg = 0x2fdc, + .halt_bit = 29, + .clkr = { + .enable_reg = 0x22d0, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "pcie_phy_ref_clk", + .ops = &clk_branch_ops, + .flags = CLK_IS_ROOT, + }, + }, +}; + +static struct clk_branch pcie_h_clk = { + .halt_reg = 0x2fd4, + .halt_bit = 8, + .clkr = { + .enable_reg = 0x22cc, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "pcie_h_clk", + .ops = &clk_branch_ops, + .flags = CLK_IS_ROOT, + }, + }, +}; + +static struct clk_branch pcie_a_clk = { + .halt_reg = 0x2fc0, + .halt_bit = 13, + .clkr = { + .enable_reg = 0x22c0, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "pcie_a_clk", + .ops = &clk_branch_ops, + .flags = CLK_IS_ROOT, + }, + }, +}; + static struct clk_branch pmic_arb0_h_clk = { .halt_reg = 0x2fd8, .halt_check = BRANCH_HALT_VOTED, @@ -2869,13 +3246,205 @@ static const struct qcom_reset_map gcc_msm8960_resets[] = { }; static struct clk_regmap *gcc_apq8064_clks[] = { + [PLL3] = &pll3.clkr, [PLL8] = &pll8.clkr, [PLL8_VOTE] = &pll8_vote, + [PLL14] = &pll14.clkr, + [PLL14_VOTE] = &pll14_vote, + [GSBI1_UART_SRC] = &gsbi1_uart_src.clkr, + [GSBI1_UART_CLK] = &gsbi1_uart_clk.clkr, + [GSBI2_UART_SRC] = &gsbi2_uart_src.clkr, + [GSBI2_UART_CLK] = &gsbi2_uart_clk.clkr, + [GSBI3_UART_SRC] = &gsbi3_uart_src.clkr, + [GSBI3_UART_CLK] = &gsbi3_uart_clk.clkr, + [GSBI4_UART_SRC] = &gsbi4_uart_src.clkr, + [GSBI4_UART_CLK] = &gsbi4_uart_clk.clkr, + [GSBI5_UART_SRC] = &gsbi5_uart_src.clkr, + [GSBI5_UART_CLK] = &gsbi5_uart_clk.clkr, + [GSBI6_UART_SRC] = &gsbi6_uart_src.clkr, + [GSBI6_UART_CLK] = &gsbi6_uart_clk.clkr, [GSBI7_UART_SRC] = &gsbi7_uart_src.clkr, [GSBI7_UART_CLK] = &gsbi7_uart_clk.clkr, + [GSBI1_QUP_SRC] = &gsbi1_qup_src.clkr, + [GSBI1_QUP_CLK] = &gsbi1_qup_clk.clkr, + [GSBI2_QUP_SRC] = &gsbi2_qup_src.clkr, + [GSBI2_QUP_CLK] = &gsbi2_qup_clk.clkr, + [GSBI3_QUP_SRC] = &gsbi3_qup_src.clkr, + [GSBI3_QUP_CLK] = &gsbi3_qup_clk.clkr, + [GSBI4_QUP_SRC] = &gsbi4_qup_src.clkr, + [GSBI4_QUP_CLK] = &gsbi4_qup_clk.clkr, + [GSBI5_QUP_SRC] = &gsbi5_qup_src.clkr, + [GSBI5_QUP_CLK] = &gsbi5_qup_clk.clkr, + [GSBI6_QUP_SRC] = &gsbi6_qup_src.clkr, + [GSBI6_QUP_CLK] = &gsbi6_qup_clk.clkr, [GSBI7_QUP_SRC] = &gsbi7_qup_src.clkr, [GSBI7_QUP_CLK] = &gsbi7_qup_clk.clkr, + [GP0_SRC] = &gp0_src.clkr, + [GP0_CLK] = &gp0_clk.clkr, + [GP1_SRC] = &gp1_src.clkr, + [GP1_CLK] = &gp1_clk.clkr, + [GP2_SRC] = &gp2_src.clkr, + [GP2_CLK] = &gp2_clk.clkr, + [PMEM_A_CLK] = &pmem_clk.clkr, + [PRNG_SRC] = &prng_src.clkr, + [PRNG_CLK] = &prng_clk.clkr, + [SDC1_SRC] = &sdc1_src.clkr, + [SDC1_CLK] = &sdc1_clk.clkr, + [SDC2_SRC] = &sdc2_src.clkr, + [SDC2_CLK] = &sdc2_clk.clkr, + [SDC3_SRC] = &sdc3_src.clkr, + [SDC3_CLK] = &sdc3_clk.clkr, + [SDC4_SRC] = &sdc4_src.clkr, + [SDC4_CLK] = &sdc4_clk.clkr, + [TSIF_REF_SRC] = &tsif_ref_src.clkr, + [TSIF_REF_CLK] = &tsif_ref_clk.clkr, + [USB_HS1_XCVR_SRC] = &usb_hs1_xcvr_src.clkr, + [USB_HS1_XCVR_CLK] = &usb_hs1_xcvr_clk.clkr, + [USB_HS3_XCVR_SRC] = &usb_hs3_xcvr_src.clkr, + [USB_HS3_XCVR_CLK] = &usb_hs3_xcvr_clk.clkr, + [USB_HS4_XCVR_SRC] = &usb_hs4_xcvr_src.clkr, + [USB_HS4_XCVR_CLK] = &usb_hs4_xcvr_clk.clkr, + [USB_HSIC_XCVR_FS_SRC] = &usb_hsic_xcvr_fs_src.clkr, + [USB_HSIC_XCVR_FS_CLK] = &usb_hsic_xcvr_fs_clk.clkr, + [USB_HSIC_SYSTEM_CLK] = &usb_hsic_system_clk.clkr, + [USB_HSIC_HSIC_CLK] = &usb_hsic_hsic_clk.clkr, + [USB_HSIC_HSIO_CAL_CLK] = &usb_hsic_hsio_cal_clk.clkr, + [USB_FS1_XCVR_FS_SRC] = &usb_fs1_xcvr_fs_src.clkr, + [USB_FS1_XCVR_FS_CLK] = &usb_fs1_xcvr_fs_clk.clkr, + [USB_FS1_SYSTEM_CLK] = &usb_fs1_system_clk.clkr, + [SATA_H_CLK] = &sata_h_clk.clkr, + [SATA_CLK_SRC] = &sata_clk_src.clkr, + [SATA_RXOOB_CLK] = &sata_rxoob_clk.clkr, + [SATA_PMALIVE_CLK] = &sata_pmalive_clk.clkr, + [SATA_PHY_REF_CLK] = &sata_phy_ref_clk.clkr, + [SATA_PHY_CFG_CLK] = &sata_phy_cfg_clk.clkr, + [SATA_A_CLK] = &sata_a_clk.clkr, + [SFAB_SATA_S_H_CLK] = &sfab_sata_s_h_clk.clkr, + [CE3_SRC] = &ce3_src.clkr, + [CE3_CORE_CLK] = &ce3_core_clk.clkr, + [CE3_H_CLK] = &ce3_h_clk.clkr, + [DMA_BAM_H_CLK] = &dma_bam_h_clk.clkr, + [GSBI1_H_CLK] = &gsbi1_h_clk.clkr, + [GSBI2_H_CLK] = &gsbi2_h_clk.clkr, + [GSBI3_H_CLK] = &gsbi3_h_clk.clkr, + [GSBI4_H_CLK] = &gsbi4_h_clk.clkr, + [GSBI5_H_CLK] = &gsbi5_h_clk.clkr, + [GSBI6_H_CLK] = &gsbi6_h_clk.clkr, [GSBI7_H_CLK] = &gsbi7_h_clk.clkr, + [TSIF_H_CLK] = &tsif_h_clk.clkr, + [USB_FS1_H_CLK] = &usb_fs1_h_clk.clkr, + [USB_HS1_H_CLK] = &usb_hs1_h_clk.clkr, + [USB_HSIC_H_CLK] = &usb_hsic_h_clk.clkr, + [USB_HS3_H_CLK] = &usb_hs3_h_clk.clkr, + [USB_HS4_H_CLK] = &usb_hs4_h_clk.clkr, + [SDC1_H_CLK] = &sdc1_h_clk.clkr, + [SDC2_H_CLK] = &sdc2_h_clk.clkr, + [SDC3_H_CLK] = &sdc3_h_clk.clkr, + [SDC4_H_CLK] = &sdc4_h_clk.clkr, + [ADM0_CLK] = &adm0_clk.clkr, + [ADM0_PBUS_CLK] = &adm0_pbus_clk.clkr, + [PCIE_A_CLK] = &pcie_a_clk.clkr, + [PCIE_PHY_REF_CLK] = &pcie_phy_ref_clk.clkr, + [PCIE_H_CLK] = &pcie_h_clk.clkr, + [PMIC_ARB0_H_CLK] = &pmic_arb0_h_clk.clkr, + [PMIC_ARB1_H_CLK] = &pmic_arb1_h_clk.clkr, + [PMIC_SSBI2_CLK] = &pmic_ssbi2_clk.clkr, + [RPM_MSG_RAM_H_CLK] = &rpm_msg_ram_h_clk.clkr, +}; + +static const struct qcom_reset_map gcc_apq8064_resets[] = { + [QDSS_STM_RESET] = { 0x2060, 6 }, + [AFAB_SMPSS_S_RESET] = { 0x20b8, 2 }, + [AFAB_SMPSS_M1_RESET] = { 0x20b8, 1 }, + [AFAB_SMPSS_M0_RESET] = { 0x20b8 }, + [AFAB_EBI1_CH0_RESET] = { 0x20c0, 7 }, + [AFAB_EBI1_CH1_RESET] = { 0x20c4, 7}, + [SFAB_ADM0_M0_RESET] = { 0x21e0, 7 }, + [SFAB_ADM0_M1_RESET] = { 0x21e4, 7 }, + [SFAB_ADM0_M2_RESET] = { 0x21e8, 7 }, + [ADM0_C2_RESET] = { 0x220c, 4}, + [ADM0_C1_RESET] = { 0x220c, 3}, + [ADM0_C0_RESET] = { 0x220c, 2}, + [ADM0_PBUS_RESET] = { 0x220c, 1 }, + [ADM0_RESET] = { 0x220c }, + [QDSS_CLKS_SW_RESET] = { 0x2260, 5 }, + [QDSS_POR_RESET] = { 0x2260, 4 }, + [QDSS_TSCTR_RESET] = { 0x2260, 3 }, + [QDSS_HRESET_RESET] = { 0x2260, 2 }, + [QDSS_AXI_RESET] = { 0x2260, 1 }, + [QDSS_DBG_RESET] = { 0x2260 }, + [SFAB_PCIE_M_RESET] = { 0x22d8, 1 }, + [SFAB_PCIE_S_RESET] = { 0x22d8 }, + [PCIE_EXT_PCI_RESET] = { 0x22dc, 6 }, + [PCIE_PHY_RESET] = { 0x22dc, 5 }, + [PCIE_PCI_RESET] = { 0x22dc, 4 }, + [PCIE_POR_RESET] = { 0x22dc, 3 }, + [PCIE_HCLK_RESET] = { 0x22dc, 2 }, + [PCIE_ACLK_RESET] = { 0x22dc }, + [SFAB_USB3_M_RESET] = { 0x2360, 7 }, + [SFAB_RIVA_M_RESET] = { 0x2380, 7 }, + [SFAB_LPASS_RESET] = { 0x23a0, 7 }, + [SFAB_AFAB_M_RESET] = { 0x23e0, 7 }, + [AFAB_SFAB_M0_RESET] = { 0x2420, 7 }, + [AFAB_SFAB_M1_RESET] = { 0x2424, 7 }, + [SFAB_SATA_S_RESET] = { 0x2480, 7 }, + [SFAB_DFAB_M_RESET] = { 0x2500, 7 }, + [DFAB_SFAB_M_RESET] = { 0x2520, 7 }, + [DFAB_SWAY0_RESET] = { 0x2540, 7 }, + [DFAB_SWAY1_RESET] = { 0x2544, 7 }, + [DFAB_ARB0_RESET] = { 0x2560, 7 }, + [DFAB_ARB1_RESET] = { 0x2564, 7 }, + [PPSS_PROC_RESET] = { 0x2594, 1 }, + [PPSS_RESET] = { 0x2594}, + [DMA_BAM_RESET] = { 0x25c0, 7 }, + [SPS_TIC_H_RESET] = { 0x2600, 7 }, + [SFAB_CFPB_M_RESET] = { 0x2680, 7 }, + [SFAB_CFPB_S_RESET] = { 0x26c0, 7 }, + [TSIF_H_RESET] = { 0x2700, 7 }, + [CE1_H_RESET] = { 0x2720, 7 }, + [CE1_CORE_RESET] = { 0x2724, 7 }, + [CE1_SLEEP_RESET] = { 0x2728, 7 }, + [CE2_H_RESET] = { 0x2740, 7 }, + [CE2_CORE_RESET] = { 0x2744, 7 }, + [SFAB_SFPB_M_RESET] = { 0x2780, 7 }, + [SFAB_SFPB_S_RESET] = { 0x27a0, 7 }, + [RPM_PROC_RESET] = { 0x27c0, 7 }, + [PMIC_SSBI2_RESET] = { 0x280c, 12 }, + [SDC1_RESET] = { 0x2830 }, + [SDC2_RESET] = { 0x2850 }, + [SDC3_RESET] = { 0x2870 }, + [SDC4_RESET] = { 0x2890 }, + [USB_HS1_RESET] = { 0x2910 }, + [USB_HSIC_RESET] = { 0x2934 }, + [USB_FS1_XCVR_RESET] = { 0x2974, 1 }, + [USB_FS1_RESET] = { 0x2974 }, + [GSBI1_RESET] = { 0x29dc }, + [GSBI2_RESET] = { 0x29fc }, + [GSBI3_RESET] = { 0x2a1c }, + [GSBI4_RESET] = { 0x2a3c }, + [GSBI5_RESET] = { 0x2a5c }, + [GSBI6_RESET] = { 0x2a7c }, + [GSBI7_RESET] = { 0x2a9c }, + [SPDM_RESET] = { 0x2b6c }, + [TLMM_H_RESET] = { 0x2ba0, 7 }, + [SATA_SFAB_M_RESET] = { 0x2c18 }, + [SATA_RESET] = { 0x2c1c }, + [GSS_SLP_RESET] = { 0x2c60, 7 }, + [GSS_RESET] = { 0x2c64 }, + [TSSC_RESET] = { 0x2ca0, 7 }, + [PDM_RESET] = { 0x2cc0, 12 }, + [MPM_H_RESET] = { 0x2da0, 7 }, + [MPM_RESET] = { 0x2da4 }, + [SFAB_SMPSS_S_RESET] = { 0x2e00, 7 }, + [PRNG_RESET] = { 0x2e80, 12 }, + [RIVA_RESET] = { 0x35e0 }, + [CE3_H_RESET] = { 0x36c4, 7 }, + [SFAB_CE3_M_RESET] = { 0x36c8, 1 }, + [SFAB_CE3_S_RESET] = { 0x36c8 }, + [CE3_RESET] = { 0x36cc, 7 }, + [CE3_SLEEP_RESET] = { 0x36d0, 7 }, + [USB_HS3_RESET] = { 0x3710 }, + [USB_HS4_RESET] = { 0x3730 }, }; static const struct regmap_config gcc_msm8960_regmap_config = { @@ -2886,6 +3455,14 @@ static const struct regmap_config gcc_msm8960_regmap_config = { .fast_io = true, }; +static const struct regmap_config gcc_apq8064_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0x3880, + .fast_io = true, +}; + static const struct qcom_cc_desc gcc_msm8960_desc = { .config = &gcc_msm8960_regmap_config, .clks = gcc_msm8960_clks, @@ -2895,11 +3472,11 @@ static const struct qcom_cc_desc gcc_msm8960_desc = { }; static const struct qcom_cc_desc gcc_apq8064_desc = { - .config = &gcc_msm8960_regmap_config, + .config = &gcc_apq8064_regmap_config, .clks = gcc_apq8064_clks, .num_clks = ARRAY_SIZE(gcc_apq8064_clks), - .resets = gcc_msm8960_resets, - .num_resets = ARRAY_SIZE(gcc_msm8960_resets), + .resets = gcc_apq8064_resets, + .num_resets = ARRAY_SIZE(gcc_apq8064_resets), }; static const struct of_device_id gcc_msm8960_match_table[] = { diff --git a/drivers/clk/qcom/mmcc-apq8084.c b/drivers/clk/qcom/mmcc-apq8084.c new file mode 100644 index 000000000000..751eea376a2b --- /dev/null +++ b/drivers/clk/qcom/mmcc-apq8084.c @@ -0,0 +1,3352 @@ +/* + * Copyright (c) 2014, The Linux Foundation. All rights reserved. + * + * This software is licensed under the terms of the GNU General Public + * License version 2, as published by the Free Software Foundation, and + * may be copied, distributed, and modified under those terms. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include <linux/kernel.h> +#include <linux/platform_device.h> +#include <linux/module.h> +#include <linux/regmap.h> +#include <linux/reset-controller.h> + +#include <dt-bindings/clock/qcom,mmcc-apq8084.h> +#include <dt-bindings/reset/qcom,mmcc-apq8084.h> + +#include "common.h" +#include "clk-regmap.h" +#include "clk-pll.h" +#include "clk-rcg.h" +#include "clk-branch.h" +#include "reset.h" + +#define P_XO 0 +#define P_MMPLL0 1 +#define P_EDPLINK 1 +#define P_MMPLL1 2 +#define P_HDMIPLL 2 +#define P_GPLL0 3 +#define P_EDPVCO 3 +#define P_MMPLL4 4 +#define P_DSI0PLL 4 +#define P_DSI0PLL_BYTE 4 +#define P_MMPLL2 4 +#define P_MMPLL3 4 +#define P_GPLL1 5 +#define P_DSI1PLL 5 +#define P_DSI1PLL_BYTE 5 +#define P_MMSLEEP 6 + +static const u8 mmcc_xo_mmpll0_mmpll1_gpll0_map[] = { + [P_XO] = 0, + [P_MMPLL0] = 1, + [P_MMPLL1] = 2, + [P_GPLL0] = 5, +}; + +static const char *mmcc_xo_mmpll0_mmpll1_gpll0[] = { + "xo", + "mmpll0_vote", + "mmpll1_vote", + "mmss_gpll0_vote", +}; + +static const u8 mmcc_xo_mmpll0_dsi_hdmi_gpll0_map[] = { + [P_XO] = 0, + [P_MMPLL0] = 1, + [P_HDMIPLL] = 4, + [P_GPLL0] = 5, + [P_DSI0PLL] = 2, + [P_DSI1PLL] = 3, +}; + +static const char *mmcc_xo_mmpll0_dsi_hdmi_gpll0[] = { + "xo", + "mmpll0_vote", + "hdmipll", + "mmss_gpll0_vote", + "dsi0pll", + "dsi1pll", +}; + +static const u8 mmcc_xo_mmpll0_1_2_gpll0_map[] = { + [P_XO] = 0, + [P_MMPLL0] = 1, + [P_MMPLL1] = 2, + [P_GPLL0] = 5, + [P_MMPLL2] = 3, +}; + +static const char *mmcc_xo_mmpll0_1_2_gpll0[] = { + "xo", + "mmpll0_vote", + "mmpll1_vote", + "mmss_gpll0_vote", + "mmpll2", +}; + +static const u8 mmcc_xo_mmpll0_1_3_gpll0_map[] = { + [P_XO] = 0, + [P_MMPLL0] = 1, + [P_MMPLL1] = 2, + [P_GPLL0] = 5, + [P_MMPLL3] = 3, +}; + +static const char *mmcc_xo_mmpll0_1_3_gpll0[] = { + "xo", + "mmpll0_vote", + "mmpll1_vote", + "mmss_gpll0_vote", + "mmpll3", +}; + +static const u8 mmcc_xo_dsi_hdmi_edp_map[] = { + [P_XO] = 0, + [P_EDPLINK] = 4, + [P_HDMIPLL] = 3, + [P_EDPVCO] = 5, + [P_DSI0PLL] = 1, + [P_DSI1PLL] = 2, +}; + +static const char *mmcc_xo_dsi_hdmi_edp[] = { + "xo", + "edp_link_clk", + "hdmipll", + "edp_vco_div", + "dsi0pll", + "dsi1pll", +}; + +static const u8 mmcc_xo_dsi_hdmi_edp_gpll0_map[] = { + [P_XO] = 0, + [P_EDPLINK] = 4, + [P_HDMIPLL] = 3, + [P_GPLL0] = 5, + [P_DSI0PLL] = 1, + [P_DSI1PLL] = 2, +}; + +static const char *mmcc_xo_dsi_hdmi_edp_gpll0[] = { + "xo", + "edp_link_clk", + "hdmipll", + "gpll0_vote", + "dsi0pll", + "dsi1pll", +}; + +static const u8 mmcc_xo_dsibyte_hdmi_edp_gpll0_map[] = { + [P_XO] = 0, + [P_EDPLINK] = 4, + [P_HDMIPLL] = 3, + [P_GPLL0] = 5, + [P_DSI0PLL_BYTE] = 1, + [P_DSI1PLL_BYTE] = 2, +}; + +static const char *mmcc_xo_dsibyte_hdmi_edp_gpll0[] = { + "xo", + "edp_link_clk", + "hdmipll", + "gpll0_vote", + "dsi0pllbyte", + "dsi1pllbyte", +}; + +static const u8 mmcc_xo_mmpll0_1_4_gpll0_map[] = { + [P_XO] = 0, + [P_MMPLL0] = 1, + [P_MMPLL1] = 2, + [P_GPLL0] = 5, + [P_MMPLL4] = 3, +}; + +static const char *mmcc_xo_mmpll0_1_4_gpll0[] = { + "xo", + "mmpll0", + "mmpll1", + "mmpll4", + "gpll0", +}; + +static const u8 mmcc_xo_mmpll0_1_4_gpll1_0_map[] = { + [P_XO] = 0, + [P_MMPLL0] = 1, + [P_MMPLL1] = 2, + [P_MMPLL4] = 3, + [P_GPLL0] = 5, + [P_GPLL1] = 4, +}; + +static const char *mmcc_xo_mmpll0_1_4_gpll1_0[] = { + "xo", + "mmpll0", + "mmpll1", + "mmpll4", + "gpll1", + "gpll0", +}; + +static const u8 mmcc_xo_mmpll0_1_4_gpll1_0_sleep_map[] = { + [P_XO] = 0, + [P_MMPLL0] = 1, + [P_MMPLL1] = 2, + [P_MMPLL4] = 3, + [P_GPLL0] = 5, + [P_GPLL1] = 4, + [P_MMSLEEP] = 6, +}; + +static const char *mmcc_xo_mmpll0_1_4_gpll1_0_sleep[] = { + "xo", + "mmpll0", + "mmpll1", + "mmpll4", + "gpll1", + "gpll0", + "sleep_clk_src", +}; + +#define F(f, s, h, m, n) { (f), (s), (2 * (h) - 1), (m), (n) } + +static struct clk_pll mmpll0 = { + .l_reg = 0x0004, + .m_reg = 0x0008, + .n_reg = 0x000c, + .config_reg = 0x0014, + .mode_reg = 0x0000, + .status_reg = 0x001c, + .status_bit = 17, + .clkr.hw.init = &(struct clk_init_data){ + .name = "mmpll0", + .parent_names = (const char *[]){ "xo" }, + .num_parents = 1, + .ops = &clk_pll_ops, + }, +}; + +static struct clk_regmap mmpll0_vote = { + .enable_reg = 0x0100, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mmpll0_vote", + .parent_names = (const char *[]){ "mmpll0" }, + .num_parents = 1, + .ops = &clk_pll_vote_ops, + }, +}; + +static struct clk_pll mmpll1 = { + .l_reg = 0x0044, + .m_reg = 0x0048, + .n_reg = 0x004c, + .config_reg = 0x0050, + .mode_reg = 0x0040, + .status_reg = 0x005c, + .status_bit = 17, + .clkr.hw.init = &(struct clk_init_data){ + .name = "mmpll1", + .parent_names = (const char *[]){ "xo" }, + .num_parents = 1, + .ops = &clk_pll_ops, + }, +}; + +static struct clk_regmap mmpll1_vote = { + .enable_reg = 0x0100, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "mmpll1_vote", + .parent_names = (const char *[]){ "mmpll1" }, + .num_parents = 1, + .ops = &clk_pll_vote_ops, + }, +}; + +static struct clk_pll mmpll2 = { + .l_reg = 0x4104, + .m_reg = 0x4108, + .n_reg = 0x410c, + .config_reg = 0x4110, + .mode_reg = 0x4100, + .status_reg = 0x411c, + .clkr.hw.init = &(struct clk_init_data){ + .name = "mmpll2", + .parent_names = (const char *[]){ "xo" }, + .num_parents = 1, + .ops = &clk_pll_ops, + }, +}; + +static struct clk_pll mmpll3 = { + .l_reg = 0x0084, + .m_reg = 0x0088, + .n_reg = 0x008c, + .config_reg = 0x0090, + .mode_reg = 0x0080, + .status_reg = 0x009c, + .status_bit = 17, + .clkr.hw.init = &(struct clk_init_data){ + .name = "mmpll3", + .parent_names = (const char *[]){ "xo" }, + .num_parents = 1, + .ops = &clk_pll_ops, + }, +}; + +static struct clk_pll mmpll4 = { + .l_reg = 0x00a4, + .m_reg = 0x00a8, + .n_reg = 0x00ac, + .config_reg = 0x00b0, + .mode_reg = 0x0080, + .status_reg = 0x00bc, + .clkr.hw.init = &(struct clk_init_data){ + .name = "mmpll4", + .parent_names = (const char *[]){ "xo" }, + .num_parents = 1, + .ops = &clk_pll_ops, + }, +}; + +static struct clk_rcg2 mmss_ahb_clk_src = { + .cmd_rcgr = 0x5000, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_mmpll1_gpll0_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "mmss_ahb_clk_src", + .parent_names = mmcc_xo_mmpll0_mmpll1_gpll0, + .num_parents = 4, + .ops = &clk_rcg2_ops, + }, +}; + +static struct freq_tbl ftbl_mmss_axi_clk[] = { + F(19200000, P_XO, 1, 0, 0), + F(37500000, P_GPLL0, 16, 0, 0), + F(50000000, P_GPLL0, 12, 0, 0), + F(75000000, P_GPLL0, 8, 0, 0), + F(100000000, P_GPLL0, 6, 0, 0), + F(150000000, P_GPLL0, 4, 0, 0), + F(333430000, P_MMPLL1, 3.5, 0, 0), + F(400000000, P_MMPLL0, 2, 0, 0), + F(466800000, P_MMPLL1, 2.5, 0, 0), +}; + +static struct clk_rcg2 mmss_axi_clk_src = { + .cmd_rcgr = 0x5040, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_mmpll1_gpll0_map, + .freq_tbl = ftbl_mmss_axi_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "mmss_axi_clk_src", + .parent_names = mmcc_xo_mmpll0_mmpll1_gpll0, + .num_parents = 4, + .ops = &clk_rcg2_ops, + }, +}; + +static struct freq_tbl ftbl_ocmemnoc_clk[] = { + F(19200000, P_XO, 1, 0, 0), + F(37500000, P_GPLL0, 16, 0, 0), + F(50000000, P_GPLL0, 12, 0, 0), + F(75000000, P_GPLL0, 8, 0, 0), + F(109090000, P_GPLL0, 5.5, 0, 0), + F(150000000, P_GPLL0, 4, 0, 0), + F(228570000, P_MMPLL0, 3.5, 0, 0), + F(320000000, P_MMPLL0, 2.5, 0, 0), +}; + +static struct clk_rcg2 ocmemnoc_clk_src = { + .cmd_rcgr = 0x5090, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_mmpll1_gpll0_map, + .freq_tbl = ftbl_ocmemnoc_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "ocmemnoc_clk_src", + .parent_names = mmcc_xo_mmpll0_mmpll1_gpll0, + .num_parents = 4, + .ops = &clk_rcg2_ops, + }, +}; + +static struct freq_tbl ftbl_camss_csi0_3_clk[] = { + F(100000000, P_GPLL0, 6, 0, 0), + F(200000000, P_MMPLL0, 4, 0, 0), + { } +}; + +static struct clk_rcg2 csi0_clk_src = { + .cmd_rcgr = 0x3090, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_1_4_gpll0_map, + .freq_tbl = ftbl_camss_csi0_3_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "csi0_clk_src", + .parent_names = mmcc_xo_mmpll0_1_4_gpll0, + .num_parents = 5, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 csi1_clk_src = { + .cmd_rcgr = 0x3100, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_1_4_gpll0_map, + .freq_tbl = ftbl_camss_csi0_3_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "csi1_clk_src", + .parent_names = mmcc_xo_mmpll0_1_4_gpll0, + .num_parents = 5, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 csi2_clk_src = { + .cmd_rcgr = 0x3160, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_1_4_gpll0_map, + .freq_tbl = ftbl_camss_csi0_3_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "csi2_clk_src", + .parent_names = mmcc_xo_mmpll0_1_4_gpll0, + .num_parents = 5, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 csi3_clk_src = { + .cmd_rcgr = 0x31c0, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_1_4_gpll0_map, + .freq_tbl = ftbl_camss_csi0_3_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "csi3_clk_src", + .parent_names = mmcc_xo_mmpll0_1_4_gpll0, + .num_parents = 5, + .ops = &clk_rcg2_ops, + }, +}; + +static struct freq_tbl ftbl_camss_vfe_vfe0_1_clk[] = { + F(37500000, P_GPLL0, 16, 0, 0), + F(50000000, P_GPLL0, 12, 0, 0), + F(60000000, P_GPLL0, 10, 0, 0), + F(80000000, P_GPLL0, 7.5, 0, 0), + F(100000000, P_GPLL0, 6, 0, 0), + F(109090000, P_GPLL0, 5.5, 0, 0), + F(133330000, P_GPLL0, 4.5, 0, 0), + F(200000000, P_GPLL0, 3, 0, 0), + F(228570000, P_MMPLL0, 3.5, 0, 0), + F(266670000, P_MMPLL0, 3, 0, 0), + F(320000000, P_MMPLL0, 2.5, 0, 0), + F(465000000, P_MMPLL4, 2, 0, 0), + F(600000000, P_GPLL0, 1, 0, 0), + { } +}; + +static struct clk_rcg2 vfe0_clk_src = { + .cmd_rcgr = 0x3600, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_1_4_gpll0_map, + .freq_tbl = ftbl_camss_vfe_vfe0_1_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "vfe0_clk_src", + .parent_names = mmcc_xo_mmpll0_1_4_gpll0, + .num_parents = 5, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 vfe1_clk_src = { + .cmd_rcgr = 0x3620, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_1_4_gpll0_map, + .freq_tbl = ftbl_camss_vfe_vfe0_1_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "vfe1_clk_src", + .parent_names = mmcc_xo_mmpll0_1_4_gpll0, + .num_parents = 5, + .ops = &clk_rcg2_ops, + }, +}; + +static struct freq_tbl ftbl_mdss_mdp_clk[] = { + F(37500000, P_GPLL0, 16, 0, 0), + F(60000000, P_GPLL0, 10, 0, 0), + F(75000000, P_GPLL0, 8, 0, 0), + F(85710000, P_GPLL0, 7, 0, 0), + F(100000000, P_GPLL0, 6, 0, 0), + F(150000000, P_GPLL0, 4, 0, 0), + F(160000000, P_MMPLL0, 5, 0, 0), + F(200000000, P_MMPLL0, 4, 0, 0), + F(228570000, P_MMPLL0, 3.5, 0, 0), + F(300000000, P_GPLL0, 2, 0, 0), + F(320000000, P_MMPLL0, 2.5, 0, 0), + { } +}; + +static struct clk_rcg2 mdp_clk_src = { + .cmd_rcgr = 0x2040, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_dsi_hdmi_gpll0_map, + .freq_tbl = ftbl_mdss_mdp_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "mdp_clk_src", + .parent_names = mmcc_xo_mmpll0_dsi_hdmi_gpll0, + .num_parents = 6, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gfx3d_clk_src = { + .cmd_rcgr = 0x4000, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_1_2_gpll0_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gfx3d_clk_src", + .parent_names = mmcc_xo_mmpll0_1_2_gpll0, + .num_parents = 5, + .ops = &clk_rcg2_ops, + }, +}; + +static struct freq_tbl ftbl_camss_jpeg_jpeg0_2_clk[] = { + F(75000000, P_GPLL0, 8, 0, 0), + F(133330000, P_GPLL0, 4.5, 0, 0), + F(200000000, P_GPLL0, 3, 0, 0), + F(228570000, P_MMPLL0, 3.5, 0, 0), + F(266670000, P_MMPLL0, 3, 0, 0), + F(320000000, P_MMPLL0, 2.5, 0, 0), + { } +}; + +static struct clk_rcg2 jpeg0_clk_src = { + .cmd_rcgr = 0x3500, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_1_4_gpll0_map, + .freq_tbl = ftbl_camss_jpeg_jpeg0_2_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "jpeg0_clk_src", + .parent_names = mmcc_xo_mmpll0_1_4_gpll0, + .num_parents = 5, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 jpeg1_clk_src = { + .cmd_rcgr = 0x3520, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_1_4_gpll0_map, + .freq_tbl = ftbl_camss_jpeg_jpeg0_2_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "jpeg1_clk_src", + .parent_names = mmcc_xo_mmpll0_1_4_gpll0, + .num_parents = 5, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 jpeg2_clk_src = { + .cmd_rcgr = 0x3540, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_1_4_gpll0_map, + .freq_tbl = ftbl_camss_jpeg_jpeg0_2_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "jpeg2_clk_src", + .parent_names = mmcc_xo_mmpll0_1_4_gpll0, + .num_parents = 5, + .ops = &clk_rcg2_ops, + }, +}; + +static struct freq_tbl pixel_freq_tbl[] = { + { .src = P_DSI0PLL }, + { } +}; + +static struct clk_rcg2 pclk0_clk_src = { + .cmd_rcgr = 0x2000, + .mnd_width = 8, + .hid_width = 5, + .parent_map = mmcc_xo_dsi_hdmi_edp_gpll0_map, + .freq_tbl = pixel_freq_tbl, + .clkr.hw.init = &(struct clk_init_data){ + .name = "pclk0_clk_src", + .parent_names = mmcc_xo_dsi_hdmi_edp_gpll0, + .num_parents = 6, + .ops = &clk_pixel_ops, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_rcg2 pclk1_clk_src = { + .cmd_rcgr = 0x2020, + .mnd_width = 8, + .hid_width = 5, + .parent_map = mmcc_xo_dsi_hdmi_edp_gpll0_map, + .freq_tbl = pixel_freq_tbl, + .clkr.hw.init = &(struct clk_init_data){ + .name = "pclk1_clk_src", + .parent_names = mmcc_xo_dsi_hdmi_edp_gpll0, + .num_parents = 6, + .ops = &clk_pixel_ops, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct freq_tbl ftbl_venus0_vcodec0_clk[] = { + F(50000000, P_GPLL0, 12, 0, 0), + F(100000000, P_GPLL0, 6, 0, 0), + F(133330000, P_GPLL0, 4.5, 0, 0), + F(200000000, P_MMPLL0, 4, 0, 0), + F(266670000, P_MMPLL0, 3, 0, 0), + F(465000000, P_MMPLL3, 2, 0, 0), + { } +}; + +static struct clk_rcg2 vcodec0_clk_src = { + .cmd_rcgr = 0x1000, + .mnd_width = 8, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_1_3_gpll0_map, + .freq_tbl = ftbl_venus0_vcodec0_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "vcodec0_clk_src", + .parent_names = mmcc_xo_mmpll0_1_3_gpll0, + .num_parents = 5, + .ops = &clk_rcg2_ops, + }, +}; + +static struct freq_tbl ftbl_avsync_vp_clk[] = { + F(150000000, P_GPLL0, 4, 0, 0), + F(320000000, P_MMPLL0, 2.5, 0, 0), + { } +}; + +static struct clk_rcg2 vp_clk_src = { + .cmd_rcgr = 0x2430, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_mmpll1_gpll0_map, + .freq_tbl = ftbl_avsync_vp_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "vp_clk_src", + .parent_names = mmcc_xo_mmpll0_mmpll1_gpll0, + .num_parents = 4, + .ops = &clk_rcg2_ops, + }, +}; + +static struct freq_tbl ftbl_camss_cci_cci_clk[] = { + F(19200000, P_XO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 cci_clk_src = { + .cmd_rcgr = 0x3300, + .mnd_width = 8, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_1_4_gpll1_0_map, + .freq_tbl = ftbl_camss_cci_cci_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "cci_clk_src", + .parent_names = mmcc_xo_mmpll0_1_4_gpll1_0, + .num_parents = 6, + .ops = &clk_rcg2_ops, + }, +}; + +static struct freq_tbl ftbl_camss_gp0_1_clk[] = { + F(10000, P_XO, 16, 1, 120), + F(24000, P_XO, 16, 1, 50), + F(6000000, P_GPLL0, 10, 1, 10), + F(12000000, P_GPLL0, 10, 1, 5), + F(13000000, P_GPLL0, 4, 13, 150), + F(24000000, P_GPLL0, 5, 1, 5), + { } +}; + +static struct clk_rcg2 camss_gp0_clk_src = { + .cmd_rcgr = 0x3420, + .mnd_width = 8, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_1_4_gpll1_0_sleep_map, + .freq_tbl = ftbl_camss_gp0_1_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "camss_gp0_clk_src", + .parent_names = mmcc_xo_mmpll0_1_4_gpll1_0_sleep, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 camss_gp1_clk_src = { + .cmd_rcgr = 0x3450, + .mnd_width = 8, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_1_4_gpll1_0_sleep_map, + .freq_tbl = ftbl_camss_gp0_1_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "camss_gp1_clk_src", + .parent_names = mmcc_xo_mmpll0_1_4_gpll1_0_sleep, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static struct freq_tbl ftbl_camss_mclk0_3_clk[] = { + F(4800000, P_XO, 4, 0, 0), + F(6000000, P_GPLL0, 10, 1, 10), + F(8000000, P_GPLL0, 15, 1, 5), + F(9600000, P_XO, 2, 0, 0), + F(16000000, P_MMPLL0, 10, 1, 5), + F(19200000, P_XO, 1, 0, 0), + F(24000000, P_GPLL0, 5, 1, 5), + F(32000000, P_MMPLL0, 5, 1, 5), + F(48000000, P_GPLL0, 12.5, 0, 0), + F(64000000, P_MMPLL0, 12.5, 0, 0), + { } +}; + +static struct clk_rcg2 mclk0_clk_src = { + .cmd_rcgr = 0x3360, + .mnd_width = 8, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_1_4_gpll1_0_map, + .freq_tbl = ftbl_camss_mclk0_3_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "mclk0_clk_src", + .parent_names = mmcc_xo_mmpll0_1_4_gpll1_0, + .num_parents = 6, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 mclk1_clk_src = { + .cmd_rcgr = 0x3390, + .mnd_width = 8, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_1_4_gpll1_0_map, + .freq_tbl = ftbl_camss_mclk0_3_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "mclk1_clk_src", + .parent_names = mmcc_xo_mmpll0_1_4_gpll1_0, + .num_parents = 6, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 mclk2_clk_src = { + .cmd_rcgr = 0x33c0, + .mnd_width = 8, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_1_4_gpll1_0_map, + .freq_tbl = ftbl_camss_mclk0_3_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "mclk2_clk_src", + .parent_names = mmcc_xo_mmpll0_1_4_gpll1_0, + .num_parents = 6, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 mclk3_clk_src = { + .cmd_rcgr = 0x33f0, + .mnd_width = 8, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_1_4_gpll1_0_map, + .freq_tbl = ftbl_camss_mclk0_3_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "mclk3_clk_src", + .parent_names = mmcc_xo_mmpll0_1_4_gpll1_0, + .num_parents = 6, + .ops = &clk_rcg2_ops, + }, +}; + +static struct freq_tbl ftbl_camss_phy0_2_csi0_2phytimer_clk[] = { + F(100000000, P_GPLL0, 6, 0, 0), + F(200000000, P_MMPLL0, 4, 0, 0), + { } +}; + +static struct clk_rcg2 csi0phytimer_clk_src = { + .cmd_rcgr = 0x3000, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_1_4_gpll0_map, + .freq_tbl = ftbl_camss_phy0_2_csi0_2phytimer_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "csi0phytimer_clk_src", + .parent_names = mmcc_xo_mmpll0_1_4_gpll0, + .num_parents = 5, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 csi1phytimer_clk_src = { + .cmd_rcgr = 0x3030, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_1_4_gpll0_map, + .freq_tbl = ftbl_camss_phy0_2_csi0_2phytimer_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "csi1phytimer_clk_src", + .parent_names = mmcc_xo_mmpll0_mmpll1_gpll0, + .num_parents = 5, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 csi2phytimer_clk_src = { + .cmd_rcgr = 0x3060, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_1_4_gpll0_map, + .freq_tbl = ftbl_camss_phy0_2_csi0_2phytimer_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "csi2phytimer_clk_src", + .parent_names = mmcc_xo_mmpll0_1_4_gpll0, + .num_parents = 5, + .ops = &clk_rcg2_ops, + }, +}; + +static struct freq_tbl ftbl_camss_vfe_cpp_clk[] = { + F(133330000, P_GPLL0, 4.5, 0, 0), + F(266670000, P_MMPLL0, 3, 0, 0), + F(320000000, P_MMPLL0, 2.5, 0, 0), + F(372000000, P_MMPLL4, 2.5, 0, 0), + F(465000000, P_MMPLL4, 2, 0, 0), + F(600000000, P_GPLL0, 1, 0, 0), + { } +}; + +static struct clk_rcg2 cpp_clk_src = { + .cmd_rcgr = 0x3640, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_1_4_gpll0_map, + .freq_tbl = ftbl_camss_vfe_cpp_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "cpp_clk_src", + .parent_names = mmcc_xo_mmpll0_1_4_gpll0, + .num_parents = 5, + .ops = &clk_rcg2_ops, + }, +}; + +static struct freq_tbl byte_freq_tbl[] = { + { .src = P_DSI0PLL_BYTE }, + { } +}; + +static struct clk_rcg2 byte0_clk_src = { + .cmd_rcgr = 0x2120, + .hid_width = 5, + .parent_map = mmcc_xo_dsibyte_hdmi_edp_gpll0_map, + .freq_tbl = byte_freq_tbl, + .clkr.hw.init = &(struct clk_init_data){ + .name = "byte0_clk_src", + .parent_names = mmcc_xo_dsibyte_hdmi_edp_gpll0, + .num_parents = 6, + .ops = &clk_byte_ops, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_rcg2 byte1_clk_src = { + .cmd_rcgr = 0x2140, + .hid_width = 5, + .parent_map = mmcc_xo_dsibyte_hdmi_edp_gpll0_map, + .freq_tbl = byte_freq_tbl, + .clkr.hw.init = &(struct clk_init_data){ + .name = "byte1_clk_src", + .parent_names = mmcc_xo_dsibyte_hdmi_edp_gpll0, + .num_parents = 6, + .ops = &clk_byte_ops, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct freq_tbl ftbl_mdss_edpaux_clk[] = { + F(19200000, P_XO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 edpaux_clk_src = { + .cmd_rcgr = 0x20e0, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_mmpll1_gpll0_map, + .freq_tbl = ftbl_mdss_edpaux_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "edpaux_clk_src", + .parent_names = mmcc_xo_mmpll0_mmpll1_gpll0, + .num_parents = 4, + .ops = &clk_rcg2_ops, + }, +}; + +static struct freq_tbl ftbl_mdss_edplink_clk[] = { + F(135000000, P_EDPLINK, 2, 0, 0), + F(270000000, P_EDPLINK, 11, 0, 0), + { } +}; + +static struct clk_rcg2 edplink_clk_src = { + .cmd_rcgr = 0x20c0, + .hid_width = 5, + .parent_map = mmcc_xo_dsi_hdmi_edp_gpll0_map, + .freq_tbl = ftbl_mdss_edplink_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "edplink_clk_src", + .parent_names = mmcc_xo_dsi_hdmi_edp_gpll0, + .num_parents = 6, + .ops = &clk_rcg2_ops, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct freq_tbl edp_pixel_freq_tbl[] = { + { .src = P_EDPVCO }, + { } +}; + +static struct clk_rcg2 edppixel_clk_src = { + .cmd_rcgr = 0x20a0, + .mnd_width = 8, + .hid_width = 5, + .parent_map = mmcc_xo_dsi_hdmi_edp_map, + .freq_tbl = edp_pixel_freq_tbl, + .clkr.hw.init = &(struct clk_init_data){ + .name = "edppixel_clk_src", + .parent_names = mmcc_xo_dsi_hdmi_edp, + .num_parents = 6, + .ops = &clk_edp_pixel_ops, + }, +}; + +static struct freq_tbl ftbl_mdss_esc0_1_clk[] = { + F(19200000, P_XO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 esc0_clk_src = { + .cmd_rcgr = 0x2160, + .hid_width = 5, + .parent_map = mmcc_xo_dsibyte_hdmi_edp_gpll0_map, + .freq_tbl = ftbl_mdss_esc0_1_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "esc0_clk_src", + .parent_names = mmcc_xo_dsibyte_hdmi_edp_gpll0, + .num_parents = 6, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 esc1_clk_src = { + .cmd_rcgr = 0x2180, + .hid_width = 5, + .parent_map = mmcc_xo_dsibyte_hdmi_edp_gpll0_map, + .freq_tbl = ftbl_mdss_esc0_1_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "esc1_clk_src", + .parent_names = mmcc_xo_dsibyte_hdmi_edp_gpll0, + .num_parents = 6, + .ops = &clk_rcg2_ops, + }, +}; + +static struct freq_tbl extpclk_freq_tbl[] = { + { .src = P_HDMIPLL }, + { } +}; + +static struct clk_rcg2 extpclk_clk_src = { + .cmd_rcgr = 0x2060, + .hid_width = 5, + .parent_map = mmcc_xo_dsi_hdmi_edp_gpll0_map, + .freq_tbl = extpclk_freq_tbl, + .clkr.hw.init = &(struct clk_init_data){ + .name = "extpclk_clk_src", + .parent_names = mmcc_xo_dsi_hdmi_edp_gpll0, + .num_parents = 6, + .ops = &clk_byte_ops, + .flags = CLK_SET_RATE_PARENT, + }, +}; + +static struct freq_tbl ftbl_mdss_hdmi_clk[] = { + F(19200000, P_XO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 hdmi_clk_src = { + .cmd_rcgr = 0x2100, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_mmpll1_gpll0_map, + .freq_tbl = ftbl_mdss_hdmi_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "hdmi_clk_src", + .parent_names = mmcc_xo_mmpll0_mmpll1_gpll0, + .num_parents = 4, + .ops = &clk_rcg2_ops, + }, +}; + +static struct freq_tbl ftbl_mdss_vsync_clk[] = { + F(19200000, P_XO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 vsync_clk_src = { + .cmd_rcgr = 0x2080, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_mmpll1_gpll0_map, + .freq_tbl = ftbl_mdss_vsync_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "vsync_clk_src", + .parent_names = mmcc_xo_mmpll0_mmpll1_gpll0, + .num_parents = 4, + .ops = &clk_rcg2_ops, + }, +}; + +static struct freq_tbl ftbl_mmss_rbcpr_clk[] = { + F(50000000, P_GPLL0, 12, 0, 0), + { } +}; + +static struct clk_rcg2 rbcpr_clk_src = { + .cmd_rcgr = 0x4060, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_mmpll1_gpll0_map, + .freq_tbl = ftbl_mmss_rbcpr_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "rbcpr_clk_src", + .parent_names = mmcc_xo_mmpll0_mmpll1_gpll0, + .num_parents = 4, + .ops = &clk_rcg2_ops, + }, +}; + +static struct freq_tbl ftbl_oxili_rbbmtimer_clk[] = { + F(19200000, P_XO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 rbbmtimer_clk_src = { + .cmd_rcgr = 0x4090, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_mmpll1_gpll0_map, + .freq_tbl = ftbl_oxili_rbbmtimer_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "rbbmtimer_clk_src", + .parent_names = mmcc_xo_mmpll0_mmpll1_gpll0, + .num_parents = 4, + .ops = &clk_rcg2_ops, + }, +}; + +static struct freq_tbl ftbl_vpu_maple_clk[] = { + F(50000000, P_GPLL0, 12, 0, 0), + F(100000000, P_GPLL0, 6, 0, 0), + F(133330000, P_GPLL0, 4.5, 0, 0), + F(200000000, P_MMPLL0, 4, 0, 0), + F(266670000, P_MMPLL0, 3, 0, 0), + F(465000000, P_MMPLL3, 2, 0, 0), + { } +}; + +static struct clk_rcg2 maple_clk_src = { + .cmd_rcgr = 0x1320, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_mmpll1_gpll0_map, + .freq_tbl = ftbl_vpu_maple_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "maple_clk_src", + .parent_names = mmcc_xo_mmpll0_mmpll1_gpll0, + .num_parents = 4, + .ops = &clk_rcg2_ops, + }, +}; + +static struct freq_tbl ftbl_vpu_vdp_clk[] = { + F(50000000, P_GPLL0, 12, 0, 0), + F(100000000, P_GPLL0, 6, 0, 0), + F(200000000, P_MMPLL0, 4, 0, 0), + F(320000000, P_MMPLL0, 2.5, 0, 0), + F(400000000, P_MMPLL0, 2, 0, 0), + { } +}; + +static struct clk_rcg2 vdp_clk_src = { + .cmd_rcgr = 0x1300, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_mmpll1_gpll0_map, + .freq_tbl = ftbl_vpu_vdp_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "vdp_clk_src", + .parent_names = mmcc_xo_mmpll0_mmpll1_gpll0, + .num_parents = 4, + .ops = &clk_rcg2_ops, + }, +}; + +static struct freq_tbl ftbl_vpu_bus_clk[] = { + F(40000000, P_GPLL0, 15, 0, 0), + F(80000000, P_MMPLL0, 10, 0, 0), + { } +}; + +static struct clk_rcg2 vpu_bus_clk_src = { + .cmd_rcgr = 0x1340, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_mmpll1_gpll0_map, + .freq_tbl = ftbl_vpu_bus_clk, + .clkr.hw.init = &(struct clk_init_data){ + .name = "vpu_bus_clk_src", + .parent_names = mmcc_xo_mmpll0_mmpll1_gpll0, + .num_parents = 4, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_branch mmss_cxo_clk = { + .halt_reg = 0x5104, + .clkr = { + .enable_reg = 0x5104, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mmss_cxo_clk", + .parent_names = (const char *[]){ "xo" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mmss_sleepclk_clk = { + .halt_reg = 0x5100, + .clkr = { + .enable_reg = 0x5100, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mmss_sleepclk_clk", + .parent_names = (const char *[]){ + "sleep_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch avsync_ahb_clk = { + .halt_reg = 0x2414, + .clkr = { + .enable_reg = 0x2414, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "avsync_ahb_clk", + .parent_names = (const char *[]){ + "mmss_ahb_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch avsync_edppixel_clk = { + .halt_reg = 0x2418, + .clkr = { + .enable_reg = 0x2418, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "avsync_edppixel_clk", + .parent_names = (const char *[]){ + "edppixel_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch avsync_extpclk_clk = { + .halt_reg = 0x2410, + .clkr = { + .enable_reg = 0x2410, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "avsync_extpclk_clk", + .parent_names = (const char *[]){ + "extpclk_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch avsync_pclk0_clk = { + .halt_reg = 0x241c, + .clkr = { + .enable_reg = 0x241c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "avsync_pclk0_clk", + .parent_names = (const char *[]){ + "pclk0_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch avsync_pclk1_clk = { + .halt_reg = 0x2420, + .clkr = { + .enable_reg = 0x2420, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "avsync_pclk1_clk", + .parent_names = (const char *[]){ + "pclk1_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch avsync_vp_clk = { + .halt_reg = 0x2404, + .clkr = { + .enable_reg = 0x2404, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "avsync_vp_clk", + .parent_names = (const char *[]){ + "vp_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_ahb_clk = { + .halt_reg = 0x348c, + .clkr = { + .enable_reg = 0x348c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_ahb_clk", + .parent_names = (const char *[]){ + "mmss_ahb_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_cci_cci_ahb_clk = { + .halt_reg = 0x3348, + .clkr = { + .enable_reg = 0x3348, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_cci_cci_ahb_clk", + .parent_names = (const char *[]){ + "mmss_ahb_clk_src", + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_cci_cci_clk = { + .halt_reg = 0x3344, + .clkr = { + .enable_reg = 0x3344, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_cci_cci_clk", + .parent_names = (const char *[]){ + "cci_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi0_ahb_clk = { + .halt_reg = 0x30bc, + .clkr = { + .enable_reg = 0x30bc, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi0_ahb_clk", + .parent_names = (const char *[]){ + "mmss_ahb_clk_src", + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi0_clk = { + .halt_reg = 0x30b4, + .clkr = { + .enable_reg = 0x30b4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi0_clk", + .parent_names = (const char *[]){ + "csi0_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi0phy_clk = { + .halt_reg = 0x30c4, + .clkr = { + .enable_reg = 0x30c4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi0phy_clk", + .parent_names = (const char *[]){ + "csi0_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi0pix_clk = { + .halt_reg = 0x30e4, + .clkr = { + .enable_reg = 0x30e4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi0pix_clk", + .parent_names = (const char *[]){ + "csi0_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi0rdi_clk = { + .halt_reg = 0x30d4, + .clkr = { + .enable_reg = 0x30d4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi0rdi_clk", + .parent_names = (const char *[]){ + "csi0_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi1_ahb_clk = { + .halt_reg = 0x3128, + .clkr = { + .enable_reg = 0x3128, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi1_ahb_clk", + .parent_names = (const char *[]){ + "mmss_ahb_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi1_clk = { + .halt_reg = 0x3124, + .clkr = { + .enable_reg = 0x3124, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi1_clk", + .parent_names = (const char *[]){ + "csi1_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi1phy_clk = { + .halt_reg = 0x3134, + .clkr = { + .enable_reg = 0x3134, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi1phy_clk", + .parent_names = (const char *[]){ + "csi1_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi1pix_clk = { + .halt_reg = 0x3154, + .clkr = { + .enable_reg = 0x3154, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi1pix_clk", + .parent_names = (const char *[]){ + "csi1_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi1rdi_clk = { + .halt_reg = 0x3144, + .clkr = { + .enable_reg = 0x3144, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi1rdi_clk", + .parent_names = (const char *[]){ + "csi1_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi2_ahb_clk = { + .halt_reg = 0x3188, + .clkr = { + .enable_reg = 0x3188, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi2_ahb_clk", + .parent_names = (const char *[]){ + "mmss_ahb_clk_src", + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi2_clk = { + .halt_reg = 0x3184, + .clkr = { + .enable_reg = 0x3184, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi2_clk", + .parent_names = (const char *[]){ + "csi2_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi2phy_clk = { + .halt_reg = 0x3194, + .clkr = { + .enable_reg = 0x3194, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi2phy_clk", + .parent_names = (const char *[]){ + "csi2_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi2pix_clk = { + .halt_reg = 0x31b4, + .clkr = { + .enable_reg = 0x31b4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi2pix_clk", + .parent_names = (const char *[]){ + "csi2_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi2rdi_clk = { + .halt_reg = 0x31a4, + .clkr = { + .enable_reg = 0x31a4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi2rdi_clk", + .parent_names = (const char *[]){ + "csi2_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi3_ahb_clk = { + .halt_reg = 0x31e8, + .clkr = { + .enable_reg = 0x31e8, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi3_ahb_clk", + .parent_names = (const char *[]){ + "mmss_ahb_clk_src", + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi3_clk = { + .halt_reg = 0x31e4, + .clkr = { + .enable_reg = 0x31e4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi3_clk", + .parent_names = (const char *[]){ + "csi3_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi3phy_clk = { + .halt_reg = 0x31f4, + .clkr = { + .enable_reg = 0x31f4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi3phy_clk", + .parent_names = (const char *[]){ + "csi3_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi3pix_clk = { + .halt_reg = 0x3214, + .clkr = { + .enable_reg = 0x3214, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi3pix_clk", + .parent_names = (const char *[]){ + "csi3_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi3rdi_clk = { + .halt_reg = 0x3204, + .clkr = { + .enable_reg = 0x3204, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi3rdi_clk", + .parent_names = (const char *[]){ + "csi3_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi_vfe0_clk = { + .halt_reg = 0x3704, + .clkr = { + .enable_reg = 0x3704, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi_vfe0_clk", + .parent_names = (const char *[]){ + "vfe0_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi_vfe1_clk = { + .halt_reg = 0x3714, + .clkr = { + .enable_reg = 0x3714, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi_vfe1_clk", + .parent_names = (const char *[]){ + "vfe1_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_gp0_clk = { + .halt_reg = 0x3444, + .clkr = { + .enable_reg = 0x3444, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_gp0_clk", + .parent_names = (const char *[]){ + "camss_gp0_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_gp1_clk = { + .halt_reg = 0x3474, + .clkr = { + .enable_reg = 0x3474, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_gp1_clk", + .parent_names = (const char *[]){ + "camss_gp1_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_ispif_ahb_clk = { + .halt_reg = 0x3224, + .clkr = { + .enable_reg = 0x3224, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_ispif_ahb_clk", + .parent_names = (const char *[]){ + "mmss_ahb_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_jpeg_jpeg0_clk = { + .halt_reg = 0x35a8, + .clkr = { + .enable_reg = 0x35a8, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_jpeg_jpeg0_clk", + .parent_names = (const char *[]){ + "jpeg0_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_jpeg_jpeg1_clk = { + .halt_reg = 0x35ac, + .clkr = { + .enable_reg = 0x35ac, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_jpeg_jpeg1_clk", + .parent_names = (const char *[]){ + "jpeg1_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_jpeg_jpeg2_clk = { + .halt_reg = 0x35b0, + .clkr = { + .enable_reg = 0x35b0, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_jpeg_jpeg2_clk", + .parent_names = (const char *[]){ + "jpeg2_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_jpeg_jpeg_ahb_clk = { + .halt_reg = 0x35b4, + .clkr = { + .enable_reg = 0x35b4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_jpeg_jpeg_ahb_clk", + .parent_names = (const char *[]){ + "mmss_ahb_clk_src", + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_jpeg_jpeg_axi_clk = { + .halt_reg = 0x35b8, + .clkr = { + .enable_reg = 0x35b8, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_jpeg_jpeg_axi_clk", + .parent_names = (const char *[]){ + "mmss_axi_clk_src", + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_mclk0_clk = { + .halt_reg = 0x3384, + .clkr = { + .enable_reg = 0x3384, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_mclk0_clk", + .parent_names = (const char *[]){ + "mclk0_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_mclk1_clk = { + .halt_reg = 0x33b4, + .clkr = { + .enable_reg = 0x33b4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_mclk1_clk", + .parent_names = (const char *[]){ + "mclk1_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_mclk2_clk = { + .halt_reg = 0x33e4, + .clkr = { + .enable_reg = 0x33e4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_mclk2_clk", + .parent_names = (const char *[]){ + "mclk2_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_mclk3_clk = { + .halt_reg = 0x3414, + .clkr = { + .enable_reg = 0x3414, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_mclk3_clk", + .parent_names = (const char *[]){ + "mclk3_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_micro_ahb_clk = { + .halt_reg = 0x3494, + .clkr = { + .enable_reg = 0x3494, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_micro_ahb_clk", + .parent_names = (const char *[]){ + "mmss_ahb_clk_src", + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_phy0_csi0phytimer_clk = { + .halt_reg = 0x3024, + .clkr = { + .enable_reg = 0x3024, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_phy0_csi0phytimer_clk", + .parent_names = (const char *[]){ + "csi0phytimer_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_phy1_csi1phytimer_clk = { + .halt_reg = 0x3054, + .clkr = { + .enable_reg = 0x3054, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_phy1_csi1phytimer_clk", + .parent_names = (const char *[]){ + "csi1phytimer_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_phy2_csi2phytimer_clk = { + .halt_reg = 0x3084, + .clkr = { + .enable_reg = 0x3084, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_phy2_csi2phytimer_clk", + .parent_names = (const char *[]){ + "csi2phytimer_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_top_ahb_clk = { + .halt_reg = 0x3484, + .clkr = { + .enable_reg = 0x3484, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_top_ahb_clk", + .parent_names = (const char *[]){ + "mmss_ahb_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_vfe_cpp_ahb_clk = { + .halt_reg = 0x36b4, + .clkr = { + .enable_reg = 0x36b4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_vfe_cpp_ahb_clk", + .parent_names = (const char *[]){ + "mmss_ahb_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_vfe_cpp_clk = { + .halt_reg = 0x36b0, + .clkr = { + .enable_reg = 0x36b0, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_vfe_cpp_clk", + .parent_names = (const char *[]){ + "cpp_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_vfe_vfe0_clk = { + .halt_reg = 0x36a8, + .clkr = { + .enable_reg = 0x36a8, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_vfe_vfe0_clk", + .parent_names = (const char *[]){ + "vfe0_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_vfe_vfe1_clk = { + .halt_reg = 0x36ac, + .clkr = { + .enable_reg = 0x36ac, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_vfe_vfe1_clk", + .parent_names = (const char *[]){ + "vfe1_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_vfe_vfe_ahb_clk = { + .halt_reg = 0x36b8, + .clkr = { + .enable_reg = 0x36b8, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_vfe_vfe_ahb_clk", + .parent_names = (const char *[]){ + "mmss_ahb_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_vfe_vfe_axi_clk = { + .halt_reg = 0x36bc, + .clkr = { + .enable_reg = 0x36bc, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_vfe_vfe_axi_clk", + .parent_names = (const char *[]){ + "mmss_axi_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_ahb_clk = { + .halt_reg = 0x2308, + .clkr = { + .enable_reg = 0x2308, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_ahb_clk", + .parent_names = (const char *[]){ + "mmss_ahb_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_axi_clk = { + .halt_reg = 0x2310, + .clkr = { + .enable_reg = 0x2310, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_axi_clk", + .parent_names = (const char *[]){ + "mmss_axi_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_byte0_clk = { + .halt_reg = 0x233c, + .clkr = { + .enable_reg = 0x233c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_byte0_clk", + .parent_names = (const char *[]){ + "byte0_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_byte1_clk = { + .halt_reg = 0x2340, + .clkr = { + .enable_reg = 0x2340, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_byte1_clk", + .parent_names = (const char *[]){ + "byte1_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_edpaux_clk = { + .halt_reg = 0x2334, + .clkr = { + .enable_reg = 0x2334, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_edpaux_clk", + .parent_names = (const char *[]){ + "edpaux_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_edplink_clk = { + .halt_reg = 0x2330, + .clkr = { + .enable_reg = 0x2330, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_edplink_clk", + .parent_names = (const char *[]){ + "edplink_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_edppixel_clk = { + .halt_reg = 0x232c, + .clkr = { + .enable_reg = 0x232c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_edppixel_clk", + .parent_names = (const char *[]){ + "edppixel_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_esc0_clk = { + .halt_reg = 0x2344, + .clkr = { + .enable_reg = 0x2344, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_esc0_clk", + .parent_names = (const char *[]){ + "esc0_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_esc1_clk = { + .halt_reg = 0x2348, + .clkr = { + .enable_reg = 0x2348, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_esc1_clk", + .parent_names = (const char *[]){ + "esc1_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_extpclk_clk = { + .halt_reg = 0x2324, + .clkr = { + .enable_reg = 0x2324, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_extpclk_clk", + .parent_names = (const char *[]){ + "extpclk_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_hdmi_ahb_clk = { + .halt_reg = 0x230c, + .clkr = { + .enable_reg = 0x230c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_hdmi_ahb_clk", + .parent_names = (const char *[]){ + "mmss_ahb_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_hdmi_clk = { + .halt_reg = 0x2338, + .clkr = { + .enable_reg = 0x2338, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_hdmi_clk", + .parent_names = (const char *[]){ + "hdmi_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_mdp_clk = { + .halt_reg = 0x231c, + .clkr = { + .enable_reg = 0x231c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_mdp_clk", + .parent_names = (const char *[]){ + "mdp_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_mdp_lut_clk = { + .halt_reg = 0x2320, + .clkr = { + .enable_reg = 0x2320, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_mdp_lut_clk", + .parent_names = (const char *[]){ + "mdp_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_pclk0_clk = { + .halt_reg = 0x2314, + .clkr = { + .enable_reg = 0x2314, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_pclk0_clk", + .parent_names = (const char *[]){ + "pclk0_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_pclk1_clk = { + .halt_reg = 0x2318, + .clkr = { + .enable_reg = 0x2318, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_pclk1_clk", + .parent_names = (const char *[]){ + "pclk1_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_vsync_clk = { + .halt_reg = 0x2328, + .clkr = { + .enable_reg = 0x2328, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_vsync_clk", + .parent_names = (const char *[]){ + "vsync_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mmss_rbcpr_ahb_clk = { + .halt_reg = 0x4088, + .clkr = { + .enable_reg = 0x4088, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mmss_rbcpr_ahb_clk", + .parent_names = (const char *[]){ + "mmss_ahb_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mmss_rbcpr_clk = { + .halt_reg = 0x4084, + .clkr = { + .enable_reg = 0x4084, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mmss_rbcpr_clk", + .parent_names = (const char *[]){ + "rbcpr_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mmss_spdm_ahb_clk = { + .halt_reg = 0x0230, + .clkr = { + .enable_reg = 0x0230, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mmss_spdm_ahb_clk", + .parent_names = (const char *[]){ + "mmss_spdm_ahb_div_clk", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mmss_spdm_axi_clk = { + .halt_reg = 0x0210, + .clkr = { + .enable_reg = 0x0210, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mmss_spdm_axi_clk", + .parent_names = (const char *[]){ + "mmss_spdm_axi_div_clk", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mmss_spdm_csi0_clk = { + .halt_reg = 0x023c, + .clkr = { + .enable_reg = 0x023c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mmss_spdm_csi0_clk", + .parent_names = (const char *[]){ + "mmss_spdm_csi0_div_clk", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mmss_spdm_gfx3d_clk = { + .halt_reg = 0x022c, + .clkr = { + .enable_reg = 0x022c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mmss_spdm_gfx3d_clk", + .parent_names = (const char *[]){ + "mmss_spdm_gfx3d_div_clk", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mmss_spdm_jpeg0_clk = { + .halt_reg = 0x0204, + .clkr = { + .enable_reg = 0x0204, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mmss_spdm_jpeg0_clk", + .parent_names = (const char *[]){ + "mmss_spdm_jpeg0_div_clk", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mmss_spdm_jpeg1_clk = { + .halt_reg = 0x0208, + .clkr = { + .enable_reg = 0x0208, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mmss_spdm_jpeg1_clk", + .parent_names = (const char *[]){ + "mmss_spdm_jpeg1_div_clk", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mmss_spdm_jpeg2_clk = { + .halt_reg = 0x0224, + .clkr = { + .enable_reg = 0x0224, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mmss_spdm_jpeg2_clk", + .parent_names = (const char *[]){ + "mmss_spdm_jpeg2_div_clk", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mmss_spdm_mdp_clk = { + .halt_reg = 0x020c, + .clkr = { + .enable_reg = 0x020c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mmss_spdm_mdp_clk", + .parent_names = (const char *[]){ + "mmss_spdm_mdp_div_clk", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mmss_spdm_pclk0_clk = { + .halt_reg = 0x0234, + .clkr = { + .enable_reg = 0x0234, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mmss_spdm_pclk0_clk", + .parent_names = (const char *[]){ + "mmss_spdm_pclk0_div_clk", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mmss_spdm_pclk1_clk = { + .halt_reg = 0x0228, + .clkr = { + .enable_reg = 0x0228, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mmss_spdm_pclk1_clk", + .parent_names = (const char *[]){ + "mmss_spdm_pclk1_div_clk", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mmss_spdm_vcodec0_clk = { + .halt_reg = 0x0214, + .clkr = { + .enable_reg = 0x0214, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mmss_spdm_vcodec0_clk", + .parent_names = (const char *[]){ + "mmss_spdm_vcodec0_div_clk", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mmss_spdm_vfe0_clk = { + .halt_reg = 0x0218, + .clkr = { + .enable_reg = 0x0218, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mmss_spdm_vfe0_clk", + .parent_names = (const char *[]){ + "mmss_spdm_vfe0_div_clk", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mmss_spdm_vfe1_clk = { + .halt_reg = 0x021c, + .clkr = { + .enable_reg = 0x021c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mmss_spdm_vfe1_clk", + .parent_names = (const char *[]){ + "mmss_spdm_vfe1_div_clk", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mmss_spdm_rm_axi_clk = { + .halt_reg = 0x0304, + .clkr = { + .enable_reg = 0x0304, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mmss_spdm_rm_axi_clk", + .parent_names = (const char *[]){ + "mmss_axi_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mmss_spdm_rm_ocmemnoc_clk = { + .halt_reg = 0x0308, + .clkr = { + .enable_reg = 0x0308, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mmss_spdm_rm_ocmemnoc_clk", + .parent_names = (const char *[]){ + "ocmemnoc_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + + +static struct clk_branch mmss_misc_ahb_clk = { + .halt_reg = 0x502c, + .clkr = { + .enable_reg = 0x502c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mmss_misc_ahb_clk", + .parent_names = (const char *[]){ + "mmss_ahb_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mmss_mmssnoc_ahb_clk = { + .halt_reg = 0x5024, + .clkr = { + .enable_reg = 0x5024, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mmss_mmssnoc_ahb_clk", + .parent_names = (const char *[]){ + "mmss_ahb_clk_src", + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, + }, + }, +}; + +static struct clk_branch mmss_mmssnoc_bto_ahb_clk = { + .halt_reg = 0x5028, + .clkr = { + .enable_reg = 0x5028, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mmss_mmssnoc_bto_ahb_clk", + .parent_names = (const char *[]){ + "mmss_ahb_clk_src", + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, + }, + }, +}; + +static struct clk_branch mmss_mmssnoc_axi_clk = { + .halt_reg = 0x506c, + .clkr = { + .enable_reg = 0x506c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mmss_mmssnoc_axi_clk", + .parent_names = (const char *[]){ + "mmss_axi_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mmss_s0_axi_clk = { + .halt_reg = 0x5064, + .clkr = { + .enable_reg = 0x5064, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mmss_s0_axi_clk", + .parent_names = (const char *[]){ + "mmss_axi_clk_src", + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, + }, + }, +}; + +static struct clk_branch ocmemcx_ahb_clk = { + .halt_reg = 0x405c, + .clkr = { + .enable_reg = 0x405c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "ocmemcx_ahb_clk", + .parent_names = (const char *[]){ + "mmss_ahb_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch ocmemcx_ocmemnoc_clk = { + .halt_reg = 0x4058, + .clkr = { + .enable_reg = 0x4058, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "ocmemcx_ocmemnoc_clk", + .parent_names = (const char *[]){ + "ocmemnoc_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch oxili_ocmemgx_clk = { + .halt_reg = 0x402c, + .clkr = { + .enable_reg = 0x402c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "oxili_ocmemgx_clk", + .parent_names = (const char *[]){ + "gfx3d_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch oxili_gfx3d_clk = { + .halt_reg = 0x4028, + .clkr = { + .enable_reg = 0x4028, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "oxili_gfx3d_clk", + .parent_names = (const char *[]){ + "gfx3d_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch oxili_rbbmtimer_clk = { + .halt_reg = 0x40b0, + .clkr = { + .enable_reg = 0x40b0, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "oxili_rbbmtimer_clk", + .parent_names = (const char *[]){ + "rbbmtimer_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch oxilicx_ahb_clk = { + .halt_reg = 0x403c, + .clkr = { + .enable_reg = 0x403c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "oxilicx_ahb_clk", + .parent_names = (const char *[]){ + "mmss_ahb_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch venus0_ahb_clk = { + .halt_reg = 0x1030, + .clkr = { + .enable_reg = 0x1030, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "venus0_ahb_clk", + .parent_names = (const char *[]){ + "mmss_ahb_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch venus0_axi_clk = { + .halt_reg = 0x1034, + .clkr = { + .enable_reg = 0x1034, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "venus0_axi_clk", + .parent_names = (const char *[]){ + "mmss_axi_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch venus0_core0_vcodec_clk = { + .halt_reg = 0x1048, + .clkr = { + .enable_reg = 0x1048, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "venus0_core0_vcodec_clk", + .parent_names = (const char *[]){ + "vcodec0_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch venus0_core1_vcodec_clk = { + .halt_reg = 0x104c, + .clkr = { + .enable_reg = 0x104c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "venus0_core1_vcodec_clk", + .parent_names = (const char *[]){ + "vcodec0_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch venus0_ocmemnoc_clk = { + .halt_reg = 0x1038, + .clkr = { + .enable_reg = 0x1038, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "venus0_ocmemnoc_clk", + .parent_names = (const char *[]){ + "ocmemnoc_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch venus0_vcodec0_clk = { + .halt_reg = 0x1028, + .clkr = { + .enable_reg = 0x1028, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "venus0_vcodec0_clk", + .parent_names = (const char *[]){ + "vcodec0_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch vpu_ahb_clk = { + .halt_reg = 0x1430, + .clkr = { + .enable_reg = 0x1430, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "vpu_ahb_clk", + .parent_names = (const char *[]){ + "mmss_ahb_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch vpu_axi_clk = { + .halt_reg = 0x143c, + .clkr = { + .enable_reg = 0x143c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "vpu_axi_clk", + .parent_names = (const char *[]){ + "mmss_axi_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch vpu_bus_clk = { + .halt_reg = 0x1440, + .clkr = { + .enable_reg = 0x1440, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "vpu_bus_clk", + .parent_names = (const char *[]){ + "vpu_bus_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch vpu_cxo_clk = { + .halt_reg = 0x1434, + .clkr = { + .enable_reg = 0x1434, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "vpu_cxo_clk", + .parent_names = (const char *[]){ "xo" }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch vpu_maple_clk = { + .halt_reg = 0x142c, + .clkr = { + .enable_reg = 0x142c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "vpu_maple_clk", + .parent_names = (const char *[]){ + "maple_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch vpu_sleep_clk = { + .halt_reg = 0x1438, + .clkr = { + .enable_reg = 0x1438, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "vpu_sleep_clk", + .parent_names = (const char *[]){ + "sleep_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch vpu_vdp_clk = { + .halt_reg = 0x1428, + .clkr = { + .enable_reg = 0x1428, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "vpu_vdp_clk", + .parent_names = (const char *[]){ + "vdp_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static const struct pll_config mmpll1_config = { + .l = 60, + .m = 25, + .n = 32, + .vco_val = 0x0, + .vco_mask = 0x3 << 20, + .pre_div_val = 0x0, + .pre_div_mask = 0x7 << 12, + .post_div_val = 0x0, + .post_div_mask = 0x3 << 8, + .mn_ena_mask = BIT(24), + .main_output_mask = BIT(0), +}; + +static const struct pll_config mmpll3_config = { + .l = 48, + .m = 7, + .n = 16, + .vco_val = 0x0, + .vco_mask = 0x3 << 20, + .pre_div_val = 0x0, + .pre_div_mask = 0x7 << 12, + .post_div_val = 0x0, + .post_div_mask = 0x3 << 8, + .mn_ena_mask = BIT(24), + .main_output_mask = BIT(0), + .aux_output_mask = BIT(1), +}; + +static struct clk_regmap *mmcc_apq8084_clocks[] = { + [MMSS_AHB_CLK_SRC] = &mmss_ahb_clk_src.clkr, + [MMSS_AXI_CLK_SRC] = &mmss_axi_clk_src.clkr, + [MMPLL0] = &mmpll0.clkr, + [MMPLL0_VOTE] = &mmpll0_vote, + [MMPLL1] = &mmpll1.clkr, + [MMPLL1_VOTE] = &mmpll1_vote, + [MMPLL2] = &mmpll2.clkr, + [MMPLL3] = &mmpll3.clkr, + [MMPLL4] = &mmpll4.clkr, + [CSI0_CLK_SRC] = &csi0_clk_src.clkr, + [CSI1_CLK_SRC] = &csi1_clk_src.clkr, + [CSI2_CLK_SRC] = &csi2_clk_src.clkr, + [CSI3_CLK_SRC] = &csi3_clk_src.clkr, + [VCODEC0_CLK_SRC] = &vcodec0_clk_src.clkr, + [VFE0_CLK_SRC] = &vfe0_clk_src.clkr, + [VFE1_CLK_SRC] = &vfe1_clk_src.clkr, + [MDP_CLK_SRC] = &mdp_clk_src.clkr, + [PCLK0_CLK_SRC] = &pclk0_clk_src.clkr, + [PCLK1_CLK_SRC] = &pclk1_clk_src.clkr, + [OCMEMNOC_CLK_SRC] = &ocmemnoc_clk_src.clkr, + [GFX3D_CLK_SRC] = &gfx3d_clk_src.clkr, + [JPEG0_CLK_SRC] = &jpeg0_clk_src.clkr, + [JPEG1_CLK_SRC] = &jpeg1_clk_src.clkr, + [JPEG2_CLK_SRC] = &jpeg2_clk_src.clkr, + [EDPPIXEL_CLK_SRC] = &edppixel_clk_src.clkr, + [EXTPCLK_CLK_SRC] = &extpclk_clk_src.clkr, + [VP_CLK_SRC] = &vp_clk_src.clkr, + [CCI_CLK_SRC] = &cci_clk_src.clkr, + [CAMSS_GP0_CLK_SRC] = &camss_gp0_clk_src.clkr, + [CAMSS_GP1_CLK_SRC] = &camss_gp1_clk_src.clkr, + [MCLK0_CLK_SRC] = &mclk0_clk_src.clkr, + [MCLK1_CLK_SRC] = &mclk1_clk_src.clkr, + [MCLK2_CLK_SRC] = &mclk2_clk_src.clkr, + [MCLK3_CLK_SRC] = &mclk3_clk_src.clkr, + [CSI0PHYTIMER_CLK_SRC] = &csi0phytimer_clk_src.clkr, + [CSI1PHYTIMER_CLK_SRC] = &csi1phytimer_clk_src.clkr, + [CSI2PHYTIMER_CLK_SRC] = &csi2phytimer_clk_src.clkr, + [CPP_CLK_SRC] = &cpp_clk_src.clkr, + [BYTE0_CLK_SRC] = &byte0_clk_src.clkr, + [BYTE1_CLK_SRC] = &byte1_clk_src.clkr, + [EDPAUX_CLK_SRC] = &edpaux_clk_src.clkr, + [EDPLINK_CLK_SRC] = &edplink_clk_src.clkr, + [ESC0_CLK_SRC] = &esc0_clk_src.clkr, + [ESC1_CLK_SRC] = &esc1_clk_src.clkr, + [HDMI_CLK_SRC] = &hdmi_clk_src.clkr, + [VSYNC_CLK_SRC] = &vsync_clk_src.clkr, + [RBCPR_CLK_SRC] = &rbcpr_clk_src.clkr, + [RBBMTIMER_CLK_SRC] = &rbbmtimer_clk_src.clkr, + [MAPLE_CLK_SRC] = &maple_clk_src.clkr, + [VDP_CLK_SRC] = &vdp_clk_src.clkr, + [VPU_BUS_CLK_SRC] = &vpu_bus_clk_src.clkr, + [MMSS_CXO_CLK] = &mmss_cxo_clk.clkr, + [MMSS_SLEEPCLK_CLK] = &mmss_sleepclk_clk.clkr, + [AVSYNC_AHB_CLK] = &avsync_ahb_clk.clkr, + [AVSYNC_EDPPIXEL_CLK] = &avsync_edppixel_clk.clkr, + [AVSYNC_EXTPCLK_CLK] = &avsync_extpclk_clk.clkr, + [AVSYNC_PCLK0_CLK] = &avsync_pclk0_clk.clkr, + [AVSYNC_PCLK1_CLK] = &avsync_pclk1_clk.clkr, + [AVSYNC_VP_CLK] = &avsync_vp_clk.clkr, + [CAMSS_AHB_CLK] = &camss_ahb_clk.clkr, + [CAMSS_CCI_CCI_AHB_CLK] = &camss_cci_cci_ahb_clk.clkr, + [CAMSS_CCI_CCI_CLK] = &camss_cci_cci_clk.clkr, + [CAMSS_CSI0_AHB_CLK] = &camss_csi0_ahb_clk.clkr, + [CAMSS_CSI0_CLK] = &camss_csi0_clk.clkr, + [CAMSS_CSI0PHY_CLK] = &camss_csi0phy_clk.clkr, + [CAMSS_CSI0PIX_CLK] = &camss_csi0pix_clk.clkr, + [CAMSS_CSI0RDI_CLK] = &camss_csi0rdi_clk.clkr, + [CAMSS_CSI1_AHB_CLK] = &camss_csi1_ahb_clk.clkr, + [CAMSS_CSI1_CLK] = &camss_csi1_clk.clkr, + [CAMSS_CSI1PHY_CLK] = &camss_csi1phy_clk.clkr, + [CAMSS_CSI1PIX_CLK] = &camss_csi1pix_clk.clkr, + [CAMSS_CSI1RDI_CLK] = &camss_csi1rdi_clk.clkr, + [CAMSS_CSI2_AHB_CLK] = &camss_csi2_ahb_clk.clkr, + [CAMSS_CSI2_CLK] = &camss_csi2_clk.clkr, + [CAMSS_CSI2PHY_CLK] = &camss_csi2phy_clk.clkr, + [CAMSS_CSI2PIX_CLK] = &camss_csi2pix_clk.clkr, + [CAMSS_CSI2RDI_CLK] = &camss_csi2rdi_clk.clkr, + [CAMSS_CSI3_AHB_CLK] = &camss_csi3_ahb_clk.clkr, + [CAMSS_CSI3_CLK] = &camss_csi3_clk.clkr, + [CAMSS_CSI3PHY_CLK] = &camss_csi3phy_clk.clkr, + [CAMSS_CSI3PIX_CLK] = &camss_csi3pix_clk.clkr, + [CAMSS_CSI3RDI_CLK] = &camss_csi3rdi_clk.clkr, + [CAMSS_CSI_VFE0_CLK] = &camss_csi_vfe0_clk.clkr, + [CAMSS_CSI_VFE1_CLK] = &camss_csi_vfe1_clk.clkr, + [CAMSS_GP0_CLK] = &camss_gp0_clk.clkr, + [CAMSS_GP1_CLK] = &camss_gp1_clk.clkr, + [CAMSS_ISPIF_AHB_CLK] = &camss_ispif_ahb_clk.clkr, + [CAMSS_JPEG_JPEG0_CLK] = &camss_jpeg_jpeg0_clk.clkr, + [CAMSS_JPEG_JPEG1_CLK] = &camss_jpeg_jpeg1_clk.clkr, + [CAMSS_JPEG_JPEG2_CLK] = &camss_jpeg_jpeg2_clk.clkr, + [CAMSS_JPEG_JPEG_AHB_CLK] = &camss_jpeg_jpeg_ahb_clk.clkr, + [CAMSS_JPEG_JPEG_AXI_CLK] = &camss_jpeg_jpeg_axi_clk.clkr, + [CAMSS_MCLK0_CLK] = &camss_mclk0_clk.clkr, + [CAMSS_MCLK1_CLK] = &camss_mclk1_clk.clkr, + [CAMSS_MCLK2_CLK] = &camss_mclk2_clk.clkr, + [CAMSS_MCLK3_CLK] = &camss_mclk3_clk.clkr, + [CAMSS_MICRO_AHB_CLK] = &camss_micro_ahb_clk.clkr, + [CAMSS_PHY0_CSI0PHYTIMER_CLK] = &camss_phy0_csi0phytimer_clk.clkr, + [CAMSS_PHY1_CSI1PHYTIMER_CLK] = &camss_phy1_csi1phytimer_clk.clkr, + [CAMSS_PHY2_CSI2PHYTIMER_CLK] = &camss_phy2_csi2phytimer_clk.clkr, + [CAMSS_TOP_AHB_CLK] = &camss_top_ahb_clk.clkr, + [CAMSS_VFE_CPP_AHB_CLK] = &camss_vfe_cpp_ahb_clk.clkr, + [CAMSS_VFE_CPP_CLK] = &camss_vfe_cpp_clk.clkr, + [CAMSS_VFE_VFE0_CLK] = &camss_vfe_vfe0_clk.clkr, + [CAMSS_VFE_VFE1_CLK] = &camss_vfe_vfe1_clk.clkr, + [CAMSS_VFE_VFE_AHB_CLK] = &camss_vfe_vfe_ahb_clk.clkr, + [CAMSS_VFE_VFE_AXI_CLK] = &camss_vfe_vfe_axi_clk.clkr, + [MDSS_AHB_CLK] = &mdss_ahb_clk.clkr, + [MDSS_AXI_CLK] = &mdss_axi_clk.clkr, + [MDSS_BYTE0_CLK] = &mdss_byte0_clk.clkr, + [MDSS_BYTE1_CLK] = &mdss_byte1_clk.clkr, + [MDSS_EDPAUX_CLK] = &mdss_edpaux_clk.clkr, + [MDSS_EDPLINK_CLK] = &mdss_edplink_clk.clkr, + [MDSS_EDPPIXEL_CLK] = &mdss_edppixel_clk.clkr, + [MDSS_ESC0_CLK] = &mdss_esc0_clk.clkr, + [MDSS_ESC1_CLK] = &mdss_esc1_clk.clkr, + [MDSS_EXTPCLK_CLK] = &mdss_extpclk_clk.clkr, + [MDSS_HDMI_AHB_CLK] = &mdss_hdmi_ahb_clk.clkr, + [MDSS_HDMI_CLK] = &mdss_hdmi_clk.clkr, + [MDSS_MDP_CLK] = &mdss_mdp_clk.clkr, + [MDSS_MDP_LUT_CLK] = &mdss_mdp_lut_clk.clkr, + [MDSS_PCLK0_CLK] = &mdss_pclk0_clk.clkr, + [MDSS_PCLK1_CLK] = &mdss_pclk1_clk.clkr, + [MDSS_VSYNC_CLK] = &mdss_vsync_clk.clkr, + [MMSS_RBCPR_AHB_CLK] = &mmss_rbcpr_ahb_clk.clkr, + [MMSS_RBCPR_CLK] = &mmss_rbcpr_clk.clkr, + [MMSS_SPDM_AHB_CLK] = &mmss_spdm_ahb_clk.clkr, + [MMSS_SPDM_AXI_CLK] = &mmss_spdm_axi_clk.clkr, + [MMSS_SPDM_CSI0_CLK] = &mmss_spdm_csi0_clk.clkr, + [MMSS_SPDM_GFX3D_CLK] = &mmss_spdm_gfx3d_clk.clkr, + [MMSS_SPDM_JPEG0_CLK] = &mmss_spdm_jpeg0_clk.clkr, + [MMSS_SPDM_JPEG1_CLK] = &mmss_spdm_jpeg1_clk.clkr, + [MMSS_SPDM_JPEG2_CLK] = &mmss_spdm_jpeg2_clk.clkr, + [MMSS_SPDM_MDP_CLK] = &mmss_spdm_mdp_clk.clkr, + [MMSS_SPDM_PCLK0_CLK] = &mmss_spdm_pclk0_clk.clkr, + [MMSS_SPDM_PCLK1_CLK] = &mmss_spdm_pclk1_clk.clkr, + [MMSS_SPDM_VCODEC0_CLK] = &mmss_spdm_vcodec0_clk.clkr, + [MMSS_SPDM_VFE0_CLK] = &mmss_spdm_vfe0_clk.clkr, + [MMSS_SPDM_VFE1_CLK] = &mmss_spdm_vfe1_clk.clkr, + [MMSS_SPDM_RM_AXI_CLK] = &mmss_spdm_rm_axi_clk.clkr, + [MMSS_SPDM_RM_OCMEMNOC_CLK] = &mmss_spdm_rm_ocmemnoc_clk.clkr, + [MMSS_MISC_AHB_CLK] = &mmss_misc_ahb_clk.clkr, + [MMSS_MMSSNOC_AHB_CLK] = &mmss_mmssnoc_ahb_clk.clkr, + [MMSS_MMSSNOC_BTO_AHB_CLK] = &mmss_mmssnoc_bto_ahb_clk.clkr, + [MMSS_MMSSNOC_AXI_CLK] = &mmss_mmssnoc_axi_clk.clkr, + [MMSS_S0_AXI_CLK] = &mmss_s0_axi_clk.clkr, + [OCMEMCX_AHB_CLK] = &ocmemcx_ahb_clk.clkr, + [OCMEMCX_OCMEMNOC_CLK] = &ocmemcx_ocmemnoc_clk.clkr, + [OXILI_OCMEMGX_CLK] = &oxili_ocmemgx_clk.clkr, + [OXILI_GFX3D_CLK] = &oxili_gfx3d_clk.clkr, + [OXILI_RBBMTIMER_CLK] = &oxili_rbbmtimer_clk.clkr, + [OXILICX_AHB_CLK] = &oxilicx_ahb_clk.clkr, + [VENUS0_AHB_CLK] = &venus0_ahb_clk.clkr, + [VENUS0_AXI_CLK] = &venus0_axi_clk.clkr, + [VENUS0_CORE0_VCODEC_CLK] = &venus0_core0_vcodec_clk.clkr, + [VENUS0_CORE1_VCODEC_CLK] = &venus0_core1_vcodec_clk.clkr, + [VENUS0_OCMEMNOC_CLK] = &venus0_ocmemnoc_clk.clkr, + [VENUS0_VCODEC0_CLK] = &venus0_vcodec0_clk.clkr, + [VPU_AHB_CLK] = &vpu_ahb_clk.clkr, + [VPU_AXI_CLK] = &vpu_axi_clk.clkr, + [VPU_BUS_CLK] = &vpu_bus_clk.clkr, + [VPU_CXO_CLK] = &vpu_cxo_clk.clkr, + [VPU_MAPLE_CLK] = &vpu_maple_clk.clkr, + [VPU_SLEEP_CLK] = &vpu_sleep_clk.clkr, + [VPU_VDP_CLK] = &vpu_vdp_clk.clkr, +}; + +static const struct qcom_reset_map mmcc_apq8084_resets[] = { + [MMSS_SPDM_RESET] = { 0x0200 }, + [MMSS_SPDM_RM_RESET] = { 0x0300 }, + [VENUS0_RESET] = { 0x1020 }, + [VPU_RESET] = { 0x1400 }, + [MDSS_RESET] = { 0x2300 }, + [AVSYNC_RESET] = { 0x2400 }, + [CAMSS_PHY0_RESET] = { 0x3020 }, + [CAMSS_PHY1_RESET] = { 0x3050 }, + [CAMSS_PHY2_RESET] = { 0x3080 }, + [CAMSS_CSI0_RESET] = { 0x30b0 }, + [CAMSS_CSI0PHY_RESET] = { 0x30c0 }, + [CAMSS_CSI0RDI_RESET] = { 0x30d0 }, + [CAMSS_CSI0PIX_RESET] = { 0x30e0 }, + [CAMSS_CSI1_RESET] = { 0x3120 }, + [CAMSS_CSI1PHY_RESET] = { 0x3130 }, + [CAMSS_CSI1RDI_RESET] = { 0x3140 }, + [CAMSS_CSI1PIX_RESET] = { 0x3150 }, + [CAMSS_CSI2_RESET] = { 0x3180 }, + [CAMSS_CSI2PHY_RESET] = { 0x3190 }, + [CAMSS_CSI2RDI_RESET] = { 0x31a0 }, + [CAMSS_CSI2PIX_RESET] = { 0x31b0 }, + [CAMSS_CSI3_RESET] = { 0x31e0 }, + [CAMSS_CSI3PHY_RESET] = { 0x31f0 }, + [CAMSS_CSI3RDI_RESET] = { 0x3200 }, + [CAMSS_CSI3PIX_RESET] = { 0x3210 }, + [CAMSS_ISPIF_RESET] = { 0x3220 }, + [CAMSS_CCI_RESET] = { 0x3340 }, + [CAMSS_MCLK0_RESET] = { 0x3380 }, + [CAMSS_MCLK1_RESET] = { 0x33b0 }, + [CAMSS_MCLK2_RESET] = { 0x33e0 }, + [CAMSS_MCLK3_RESET] = { 0x3410 }, + [CAMSS_GP0_RESET] = { 0x3440 }, + [CAMSS_GP1_RESET] = { 0x3470 }, + [CAMSS_TOP_RESET] = { 0x3480 }, + [CAMSS_AHB_RESET] = { 0x3488 }, + [CAMSS_MICRO_RESET] = { 0x3490 }, + [CAMSS_JPEG_RESET] = { 0x35a0 }, + [CAMSS_VFE_RESET] = { 0x36a0 }, + [CAMSS_CSI_VFE0_RESET] = { 0x3700 }, + [CAMSS_CSI_VFE1_RESET] = { 0x3710 }, + [OXILI_RESET] = { 0x4020 }, + [OXILICX_RESET] = { 0x4030 }, + [OCMEMCX_RESET] = { 0x4050 }, + [MMSS_RBCRP_RESET] = { 0x4080 }, + [MMSSNOCAHB_RESET] = { 0x5020 }, + [MMSSNOCAXI_RESET] = { 0x5060 }, +}; + +static const struct regmap_config mmcc_apq8084_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0x5104, + .fast_io = true, +}; + +static const struct qcom_cc_desc mmcc_apq8084_desc = { + .config = &mmcc_apq8084_regmap_config, + .clks = mmcc_apq8084_clocks, + .num_clks = ARRAY_SIZE(mmcc_apq8084_clocks), + .resets = mmcc_apq8084_resets, + .num_resets = ARRAY_SIZE(mmcc_apq8084_resets), +}; + +static const struct of_device_id mmcc_apq8084_match_table[] = { + { .compatible = "qcom,mmcc-apq8084" }, + { } +}; +MODULE_DEVICE_TABLE(of, mmcc_apq8084_match_table); + +static int mmcc_apq8084_probe(struct platform_device *pdev) +{ + int ret; + struct regmap *regmap; + + ret = qcom_cc_probe(pdev, &mmcc_apq8084_desc); + if (ret) + return ret; + + regmap = dev_get_regmap(&pdev->dev, NULL); + clk_pll_configure_sr_hpm_lp(&mmpll1, regmap, &mmpll1_config, true); + clk_pll_configure_sr_hpm_lp(&mmpll3, regmap, &mmpll3_config, false); + + return 0; +} + +static int mmcc_apq8084_remove(struct platform_device *pdev) +{ + qcom_cc_remove(pdev); + return 0; +} + +static struct platform_driver mmcc_apq8084_driver = { + .probe = mmcc_apq8084_probe, + .remove = mmcc_apq8084_remove, + .driver = { + .name = "mmcc-apq8084", + .owner = THIS_MODULE, + .of_match_table = mmcc_apq8084_match_table, + }, +}; +module_platform_driver(mmcc_apq8084_driver); + +MODULE_DESCRIPTION("QCOM MMCC APQ8084 Driver"); +MODULE_LICENSE("GPL v2"); +MODULE_ALIAS("platform:mmcc-apq8084"); diff --git a/drivers/clk/qcom/mmcc-msm8960.c b/drivers/clk/qcom/mmcc-msm8960.c index 4c449b3170f6..2e80a219b8ea 100644 --- a/drivers/clk/qcom/mmcc-msm8960.c +++ b/drivers/clk/qcom/mmcc-msm8960.c @@ -37,6 +37,9 @@ #define P_PLL8 1 #define P_PLL2 2 #define P_PLL3 3 +#define P_PLL15 3 + +#define F_MN(f, s, _m, _n) { .freq = f, .src = s, .m = _m, .n = _n } static u8 mmcc_pxo_pll8_pll2_map[] = { [P_PXO] = 0, @@ -57,10 +60,24 @@ static u8 mmcc_pxo_pll8_pll2_pll3_map[] = { [P_PLL3] = 3, }; -static const char *mmcc_pxo_pll8_pll2_pll3[] = { +static const char *mmcc_pxo_pll8_pll2_pll15[] = { "pxo", + "pll8_vote", "pll2", + "pll15", +}; + +static u8 mmcc_pxo_pll8_pll2_pll15_map[] = { + [P_PXO] = 0, + [P_PLL8] = 2, + [P_PLL2] = 1, + [P_PLL15] = 3, +}; + +static const char *mmcc_pxo_pll8_pll2_pll3[] = { + "pxo", "pll8_vote", + "pll2", "pll3", }; @@ -80,6 +97,36 @@ static struct clk_pll pll2 = { }, }; +static struct clk_pll pll15 = { + .l_reg = 0x33c, + .m_reg = 0x340, + .n_reg = 0x344, + .config_reg = 0x348, + .mode_reg = 0x338, + .status_reg = 0x350, + .status_bit = 16, + .clkr.hw.init = &(struct clk_init_data){ + .name = "pll15", + .parent_names = (const char *[]){ "pxo" }, + .num_parents = 1, + .ops = &clk_pll_ops, + }, +}; + +static const struct pll_config pll15_config = { + .l = 33, + .m = 1, + .n = 3, + .vco_val = 0x2 << 16, + .vco_mask = 0x3 << 16, + .pre_div_val = 0x0, + .pre_div_mask = BIT(19), + .post_div_val = 0x0, + .post_div_mask = 0x3 << 20, + .mn_ena_mask = BIT(22), + .main_output_mask = BIT(23), +}; + static struct freq_tbl clk_tbl_cam[] = { { 6000000, P_PLL8, 4, 1, 16 }, { 8000000, P_PLL8, 4, 1, 12 }, @@ -710,18 +757,18 @@ static struct clk_branch csiphy2_timer_clk = { }; static struct freq_tbl clk_tbl_gfx2d[] = { - { 27000000, P_PXO, 1, 0 }, - { 48000000, P_PLL8, 1, 8 }, - { 54857000, P_PLL8, 1, 7 }, - { 64000000, P_PLL8, 1, 6 }, - { 76800000, P_PLL8, 1, 5 }, - { 96000000, P_PLL8, 1, 4 }, - { 128000000, P_PLL8, 1, 3 }, - { 145455000, P_PLL2, 2, 11 }, - { 160000000, P_PLL2, 1, 5 }, - { 177778000, P_PLL2, 2, 9 }, - { 200000000, P_PLL2, 1, 4 }, - { 228571000, P_PLL2, 2, 7 }, + F_MN( 27000000, P_PXO, 1, 0), + F_MN( 48000000, P_PLL8, 1, 8), + F_MN( 54857000, P_PLL8, 1, 7), + F_MN( 64000000, P_PLL8, 1, 6), + F_MN( 76800000, P_PLL8, 1, 5), + F_MN( 96000000, P_PLL8, 1, 4), + F_MN(128000000, P_PLL8, 1, 3), + F_MN(145455000, P_PLL2, 2, 11), + F_MN(160000000, P_PLL2, 1, 5), + F_MN(177778000, P_PLL2, 2, 9), + F_MN(200000000, P_PLL2, 1, 4), + F_MN(228571000, P_PLL2, 2, 7), { } }; @@ -842,22 +889,43 @@ static struct clk_branch gfx2d1_clk = { }; static struct freq_tbl clk_tbl_gfx3d[] = { - { 27000000, P_PXO, 1, 0 }, - { 48000000, P_PLL8, 1, 8 }, - { 54857000, P_PLL8, 1, 7 }, - { 64000000, P_PLL8, 1, 6 }, - { 76800000, P_PLL8, 1, 5 }, - { 96000000, P_PLL8, 1, 4 }, - { 128000000, P_PLL8, 1, 3 }, - { 145455000, P_PLL2, 2, 11 }, - { 160000000, P_PLL2, 1, 5 }, - { 177778000, P_PLL2, 2, 9 }, - { 200000000, P_PLL2, 1, 4 }, - { 228571000, P_PLL2, 2, 7 }, - { 266667000, P_PLL2, 1, 3 }, - { 300000000, P_PLL3, 1, 4 }, - { 320000000, P_PLL2, 2, 5 }, - { 400000000, P_PLL2, 1, 2 }, + F_MN( 27000000, P_PXO, 1, 0), + F_MN( 48000000, P_PLL8, 1, 8), + F_MN( 54857000, P_PLL8, 1, 7), + F_MN( 64000000, P_PLL8, 1, 6), + F_MN( 76800000, P_PLL8, 1, 5), + F_MN( 96000000, P_PLL8, 1, 4), + F_MN(128000000, P_PLL8, 1, 3), + F_MN(145455000, P_PLL2, 2, 11), + F_MN(160000000, P_PLL2, 1, 5), + F_MN(177778000, P_PLL2, 2, 9), + F_MN(200000000, P_PLL2, 1, 4), + F_MN(228571000, P_PLL2, 2, 7), + F_MN(266667000, P_PLL2, 1, 3), + F_MN(300000000, P_PLL3, 1, 4), + F_MN(320000000, P_PLL2, 2, 5), + F_MN(400000000, P_PLL2, 1, 2), + { } +}; + +static struct freq_tbl clk_tbl_gfx3d_8064[] = { + F_MN( 27000000, P_PXO, 0, 0), + F_MN( 48000000, P_PLL8, 1, 8), + F_MN( 54857000, P_PLL8, 1, 7), + F_MN( 64000000, P_PLL8, 1, 6), + F_MN( 76800000, P_PLL8, 1, 5), + F_MN( 96000000, P_PLL8, 1, 4), + F_MN(128000000, P_PLL8, 1, 3), + F_MN(145455000, P_PLL2, 2, 11), + F_MN(160000000, P_PLL2, 1, 5), + F_MN(177778000, P_PLL2, 2, 9), + F_MN(192000000, P_PLL8, 1, 2), + F_MN(200000000, P_PLL2, 1, 4), + F_MN(228571000, P_PLL2, 2, 7), + F_MN(266667000, P_PLL2, 1, 3), + F_MN(320000000, P_PLL2, 2, 5), + F_MN(400000000, P_PLL2, 1, 2), + F_MN(450000000, P_PLL15, 1, 2), { } }; @@ -897,12 +965,19 @@ static struct clk_dyn_rcg gfx3d_src = { .hw.init = &(struct clk_init_data){ .name = "gfx3d_src", .parent_names = mmcc_pxo_pll8_pll2_pll3, - .num_parents = 3, + .num_parents = 4, .ops = &clk_dyn_rcg_ops, }, }, }; +static const struct clk_init_data gfx3d_8064_init = { + .name = "gfx3d_src", + .parent_names = mmcc_pxo_pll8_pll2_pll15, + .num_parents = 4, + .ops = &clk_dyn_rcg_ops, +}; + static struct clk_branch gfx3d_clk = { .halt_reg = 0x01c8, .halt_bit = 4, @@ -919,6 +994,91 @@ static struct clk_branch gfx3d_clk = { }, }; +static struct freq_tbl clk_tbl_vcap[] = { + F_MN( 27000000, P_PXO, 0, 0), + F_MN( 54860000, P_PLL8, 1, 7), + F_MN( 64000000, P_PLL8, 1, 6), + F_MN( 76800000, P_PLL8, 1, 5), + F_MN(128000000, P_PLL8, 1, 3), + F_MN(160000000, P_PLL2, 1, 5), + F_MN(200000000, P_PLL2, 1, 4), + { } +}; + +static struct clk_dyn_rcg vcap_src = { + .ns_reg = 0x021c, + .md_reg[0] = 0x01ec, + .md_reg[1] = 0x0218, + .mn[0] = { + .mnctr_en_bit = 8, + .mnctr_reset_bit = 23, + .mnctr_mode_shift = 9, + .n_val_shift = 18, + .m_val_shift = 4, + .width = 4, + }, + .mn[1] = { + .mnctr_en_bit = 5, + .mnctr_reset_bit = 22, + .mnctr_mode_shift = 6, + .n_val_shift = 14, + .m_val_shift = 4, + .width = 4, + }, + .s[0] = { + .src_sel_shift = 3, + .parent_map = mmcc_pxo_pll8_pll2_map, + }, + .s[1] = { + .src_sel_shift = 0, + .parent_map = mmcc_pxo_pll8_pll2_map, + }, + .mux_sel_bit = 11, + .freq_tbl = clk_tbl_vcap, + .clkr = { + .enable_reg = 0x0178, + .enable_mask = BIT(2), + .hw.init = &(struct clk_init_data){ + .name = "vcap_src", + .parent_names = mmcc_pxo_pll8_pll2, + .num_parents = 3, + .ops = &clk_dyn_rcg_ops, + }, + }, +}; + +static struct clk_branch vcap_clk = { + .halt_reg = 0x0240, + .halt_bit = 15, + .clkr = { + .enable_reg = 0x0178, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "vcap_clk", + .parent_names = (const char *[]){ "vcap_src" }, + .num_parents = 1, + .ops = &clk_branch_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch vcap_npl_clk = { + .halt_reg = 0x0240, + .halt_bit = 25, + .clkr = { + .enable_reg = 0x0178, + .enable_mask = BIT(13), + .hw.init = &(struct clk_init_data){ + .name = "vcap_npl_clk", + .parent_names = (const char *[]){ "vcap_src" }, + .num_parents = 1, + .ops = &clk_branch_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + static struct freq_tbl clk_tbl_ijpeg[] = { { 27000000, P_PXO, 1, 0, 0 }, { 36570000, P_PLL8, 1, 2, 21 }, @@ -995,7 +1155,7 @@ static struct clk_rcg jpegd_src = { .ns_reg = 0x00ac, .p = { .pre_div_shift = 12, - .pre_div_width = 2, + .pre_div_width = 4, }, .s = { .src_sel_shift = 0, @@ -1115,7 +1275,7 @@ static struct clk_branch mdp_lut_clk = { .enable_reg = 0x016c, .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ - .parent_names = (const char *[]){ "mdp_clk" }, + .parent_names = (const char *[]){ "mdp_src" }, .num_parents = 1, .name = "mdp_lut_clk", .ops = &clk_branch_ops, @@ -1218,12 +1378,7 @@ static const char *mmcc_pxo_hdmi[] = { }; static struct freq_tbl clk_tbl_tv[] = { - { 25200000, P_HDMI_PLL, 1, 0, 0 }, - { 27000000, P_HDMI_PLL, 1, 0, 0 }, - { 27030000, P_HDMI_PLL, 1, 0, 0 }, - { 74250000, P_HDMI_PLL, 1, 0, 0 }, - { 108000000, P_HDMI_PLL, 1, 0, 0 }, - { 148500000, P_HDMI_PLL, 1, 0, 0 }, + { .src = P_HDMI_PLL, .pre_div = 1 }, { } }; @@ -1254,7 +1409,7 @@ static struct clk_rcg tv_src = { .name = "tv_src", .parent_names = mmcc_pxo_hdmi, .num_parents = 2, - .ops = &clk_rcg_ops, + .ops = &clk_rcg_bypass_ops, .flags = CLK_SET_RATE_PARENT, }, }, @@ -1326,6 +1481,38 @@ static struct clk_branch hdmi_tv_clk = { }, }; +static struct clk_branch rgb_tv_clk = { + .halt_reg = 0x0240, + .halt_bit = 27, + .clkr = { + .enable_reg = 0x0124, + .enable_mask = BIT(14), + .hw.init = &(struct clk_init_data){ + .parent_names = tv_src_name, + .num_parents = 1, + .name = "rgb_tv_clk", + .ops = &clk_branch_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static struct clk_branch npl_tv_clk = { + .halt_reg = 0x0240, + .halt_bit = 26, + .clkr = { + .enable_reg = 0x0124, + .enable_mask = BIT(16), + .hw.init = &(struct clk_init_data){ + .parent_names = tv_src_name, + .num_parents = 1, + .name = "npl_tv_clk", + .ops = &clk_branch_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + static struct clk_branch hdmi_app_clk = { .halt_reg = 0x01cc, .halt_bit = 25, @@ -1342,15 +1529,15 @@ static struct clk_branch hdmi_app_clk = { }; static struct freq_tbl clk_tbl_vcodec[] = { - { 27000000, P_PXO, 1, 0 }, - { 32000000, P_PLL8, 1, 12 }, - { 48000000, P_PLL8, 1, 8 }, - { 54860000, P_PLL8, 1, 7 }, - { 96000000, P_PLL8, 1, 4 }, - { 133330000, P_PLL2, 1, 6 }, - { 200000000, P_PLL2, 1, 4 }, - { 228570000, P_PLL2, 2, 7 }, - { 266670000, P_PLL2, 1, 3 }, + F_MN( 27000000, P_PXO, 1, 0), + F_MN( 32000000, P_PLL8, 1, 12), + F_MN( 48000000, P_PLL8, 1, 8), + F_MN( 54860000, P_PLL8, 1, 7), + F_MN( 96000000, P_PLL8, 1, 4), + F_MN(133330000, P_PLL2, 1, 6), + F_MN(200000000, P_PLL2, 1, 4), + F_MN(228570000, P_PLL2, 2, 7), + F_MN(266670000, P_PLL2, 1, 3), { } }; @@ -1701,6 +1888,22 @@ static struct clk_branch rot_axi_clk = { }, }; +static struct clk_branch vcap_axi_clk = { + .halt_reg = 0x0240, + .halt_bit = 20, + .hwcg_reg = 0x0244, + .hwcg_bit = 11, + .clkr = { + .enable_reg = 0x0244, + .enable_mask = BIT(12), + .hw.init = &(struct clk_init_data){ + .name = "vcap_axi_clk", + .ops = &clk_branch_ops, + .flags = CLK_IS_ROOT, + }, + }, +}; + static struct clk_branch vpe_axi_clk = { .hwcg_reg = 0x0020, .hwcg_bit = 27, @@ -2003,6 +2206,20 @@ static struct clk_branch tv_enc_ahb_clk = { }, }; +static struct clk_branch vcap_ahb_clk = { + .halt_reg = 0x0240, + .halt_bit = 23, + .clkr = { + .enable_reg = 0x0248, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "vcap_ahb_clk", + .ops = &clk_branch_ops, + .flags = CLK_IS_ROOT, + }, + }, +}; + static struct clk_branch vcodec_ahb_clk = { .hwcg_reg = 0x0038, .hwcg_bit = 26, @@ -2215,6 +2432,175 @@ static const struct qcom_reset_map mmcc_msm8960_resets[] = { [CSI_RDI2_RESET] = { 0x0214 }, }; +static struct clk_regmap *mmcc_apq8064_clks[] = { + [AMP_AHB_CLK] = &_ahb_clk.clkr, + [DSI2_S_AHB_CLK] = &dsi2_s_ahb_clk.clkr, + [JPEGD_AHB_CLK] = &jpegd_ahb_clk.clkr, + [DSI_S_AHB_CLK] = &dsi_s_ahb_clk.clkr, + [DSI2_M_AHB_CLK] = &dsi2_m_ahb_clk.clkr, + [VPE_AHB_CLK] = &vpe_ahb_clk.clkr, + [SMMU_AHB_CLK] = &smmu_ahb_clk.clkr, + [HDMI_M_AHB_CLK] = &hdmi_m_ahb_clk.clkr, + [VFE_AHB_CLK] = &vfe_ahb_clk.clkr, + [ROT_AHB_CLK] = &rot_ahb_clk.clkr, + [VCODEC_AHB_CLK] = &vcodec_ahb_clk.clkr, + [MDP_AHB_CLK] = &mdp_ahb_clk.clkr, + [DSI_M_AHB_CLK] = &dsi_m_ahb_clk.clkr, + [CSI_AHB_CLK] = &csi_ahb_clk.clkr, + [MMSS_IMEM_AHB_CLK] = &mmss_imem_ahb_clk.clkr, + [IJPEG_AHB_CLK] = &ijpeg_ahb_clk.clkr, + [HDMI_S_AHB_CLK] = &hdmi_s_ahb_clk.clkr, + [GFX3D_AHB_CLK] = &gfx3d_ahb_clk.clkr, + [JPEGD_AXI_CLK] = &jpegd_axi_clk.clkr, + [GMEM_AXI_CLK] = &gmem_axi_clk.clkr, + [MDP_AXI_CLK] = &mdp_axi_clk.clkr, + [MMSS_IMEM_AXI_CLK] = &mmss_imem_axi_clk.clkr, + [IJPEG_AXI_CLK] = &ijpeg_axi_clk.clkr, + [GFX3D_AXI_CLK] = &gfx3d_axi_clk.clkr, + [VCODEC_AXI_CLK] = &vcodec_axi_clk.clkr, + [VFE_AXI_CLK] = &vfe_axi_clk.clkr, + [VPE_AXI_CLK] = &vpe_axi_clk.clkr, + [ROT_AXI_CLK] = &rot_axi_clk.clkr, + [VCODEC_AXI_A_CLK] = &vcodec_axi_a_clk.clkr, + [VCODEC_AXI_B_CLK] = &vcodec_axi_b_clk.clkr, + [CSI0_SRC] = &csi0_src.clkr, + [CSI0_CLK] = &csi0_clk.clkr, + [CSI0_PHY_CLK] = &csi0_phy_clk.clkr, + [CSI1_SRC] = &csi1_src.clkr, + [CSI1_CLK] = &csi1_clk.clkr, + [CSI1_PHY_CLK] = &csi1_phy_clk.clkr, + [CSI2_SRC] = &csi2_src.clkr, + [CSI2_CLK] = &csi2_clk.clkr, + [CSI2_PHY_CLK] = &csi2_phy_clk.clkr, + [CSI_PIX_CLK] = &csi_pix_clk.clkr, + [CSI_RDI_CLK] = &csi_rdi_clk.clkr, + [MDP_VSYNC_CLK] = &mdp_vsync_clk.clkr, + [HDMI_APP_CLK] = &hdmi_app_clk.clkr, + [CSI_PIX1_CLK] = &csi_pix1_clk.clkr, + [CSI_RDI2_CLK] = &csi_rdi2_clk.clkr, + [CSI_RDI1_CLK] = &csi_rdi1_clk.clkr, + [GFX3D_SRC] = &gfx3d_src.clkr, + [GFX3D_CLK] = &gfx3d_clk.clkr, + [IJPEG_SRC] = &ijpeg_src.clkr, + [IJPEG_CLK] = &ijpeg_clk.clkr, + [JPEGD_SRC] = &jpegd_src.clkr, + [JPEGD_CLK] = &jpegd_clk.clkr, + [MDP_SRC] = &mdp_src.clkr, + [MDP_CLK] = &mdp_clk.clkr, + [MDP_LUT_CLK] = &mdp_lut_clk.clkr, + [ROT_SRC] = &rot_src.clkr, + [ROT_CLK] = &rot_clk.clkr, + [TV_DAC_CLK] = &tv_dac_clk.clkr, + [HDMI_TV_CLK] = &hdmi_tv_clk.clkr, + [MDP_TV_CLK] = &mdp_tv_clk.clkr, + [TV_SRC] = &tv_src.clkr, + [VCODEC_SRC] = &vcodec_src.clkr, + [VCODEC_CLK] = &vcodec_clk.clkr, + [VFE_SRC] = &vfe_src.clkr, + [VFE_CLK] = &vfe_clk.clkr, + [VFE_CSI_CLK] = &vfe_csi_clk.clkr, + [VPE_SRC] = &vpe_src.clkr, + [VPE_CLK] = &vpe_clk.clkr, + [CAMCLK0_SRC] = &camclk0_src.clkr, + [CAMCLK0_CLK] = &camclk0_clk.clkr, + [CAMCLK1_SRC] = &camclk1_src.clkr, + [CAMCLK1_CLK] = &camclk1_clk.clkr, + [CAMCLK2_SRC] = &camclk2_src.clkr, + [CAMCLK2_CLK] = &camclk2_clk.clkr, + [CSIPHYTIMER_SRC] = &csiphytimer_src.clkr, + [CSIPHY2_TIMER_CLK] = &csiphy2_timer_clk.clkr, + [CSIPHY1_TIMER_CLK] = &csiphy1_timer_clk.clkr, + [CSIPHY0_TIMER_CLK] = &csiphy0_timer_clk.clkr, + [PLL2] = &pll2.clkr, + [RGB_TV_CLK] = &rgb_tv_clk.clkr, + [NPL_TV_CLK] = &npl_tv_clk.clkr, + [VCAP_AHB_CLK] = &vcap_ahb_clk.clkr, + [VCAP_AXI_CLK] = &vcap_axi_clk.clkr, + [VCAP_SRC] = &vcap_src.clkr, + [VCAP_CLK] = &vcap_clk.clkr, + [VCAP_NPL_CLK] = &vcap_npl_clk.clkr, + [PLL15] = &pll15.clkr, +}; + +static const struct qcom_reset_map mmcc_apq8064_resets[] = { + [GFX3D_AXI_RESET] = { 0x0208, 17 }, + [VCAP_AXI_RESET] = { 0x0208, 16 }, + [VPE_AXI_RESET] = { 0x0208, 15 }, + [IJPEG_AXI_RESET] = { 0x0208, 14 }, + [MPD_AXI_RESET] = { 0x0208, 13 }, + [VFE_AXI_RESET] = { 0x0208, 9 }, + [SP_AXI_RESET] = { 0x0208, 8 }, + [VCODEC_AXI_RESET] = { 0x0208, 7 }, + [ROT_AXI_RESET] = { 0x0208, 6 }, + [VCODEC_AXI_A_RESET] = { 0x0208, 5 }, + [VCODEC_AXI_B_RESET] = { 0x0208, 4 }, + [FAB_S3_AXI_RESET] = { 0x0208, 3 }, + [FAB_S2_AXI_RESET] = { 0x0208, 2 }, + [FAB_S1_AXI_RESET] = { 0x0208, 1 }, + [FAB_S0_AXI_RESET] = { 0x0208 }, + [SMMU_GFX3D_ABH_RESET] = { 0x020c, 31 }, + [SMMU_VPE_AHB_RESET] = { 0x020c, 30 }, + [SMMU_VFE_AHB_RESET] = { 0x020c, 29 }, + [SMMU_ROT_AHB_RESET] = { 0x020c, 28 }, + [SMMU_VCODEC_B_AHB_RESET] = { 0x020c, 27 }, + [SMMU_VCODEC_A_AHB_RESET] = { 0x020c, 26 }, + [SMMU_MDP1_AHB_RESET] = { 0x020c, 25 }, + [SMMU_MDP0_AHB_RESET] = { 0x020c, 24 }, + [SMMU_JPEGD_AHB_RESET] = { 0x020c, 23 }, + [SMMU_IJPEG_AHB_RESET] = { 0x020c, 22 }, + [APU_AHB_RESET] = { 0x020c, 18 }, + [CSI_AHB_RESET] = { 0x020c, 17 }, + [TV_ENC_AHB_RESET] = { 0x020c, 15 }, + [VPE_AHB_RESET] = { 0x020c, 14 }, + [FABRIC_AHB_RESET] = { 0x020c, 13 }, + [GFX3D_AHB_RESET] = { 0x020c, 10 }, + [HDMI_AHB_RESET] = { 0x020c, 9 }, + [MSSS_IMEM_AHB_RESET] = { 0x020c, 8 }, + [IJPEG_AHB_RESET] = { 0x020c, 7 }, + [DSI_M_AHB_RESET] = { 0x020c, 6 }, + [DSI_S_AHB_RESET] = { 0x020c, 5 }, + [JPEGD_AHB_RESET] = { 0x020c, 4 }, + [MDP_AHB_RESET] = { 0x020c, 3 }, + [ROT_AHB_RESET] = { 0x020c, 2 }, + [VCODEC_AHB_RESET] = { 0x020c, 1 }, + [VFE_AHB_RESET] = { 0x020c, 0 }, + [SMMU_VCAP_AHB_RESET] = { 0x0200, 3 }, + [VCAP_AHB_RESET] = { 0x0200, 2 }, + [DSI2_M_AHB_RESET] = { 0x0200, 1 }, + [DSI2_S_AHB_RESET] = { 0x0200, 0 }, + [CSIPHY2_RESET] = { 0x0210, 31 }, + [CSI_PIX1_RESET] = { 0x0210, 30 }, + [CSIPHY0_RESET] = { 0x0210, 29 }, + [CSIPHY1_RESET] = { 0x0210, 28 }, + [CSI_RDI_RESET] = { 0x0210, 27 }, + [CSI_PIX_RESET] = { 0x0210, 26 }, + [DSI2_RESET] = { 0x0210, 25 }, + [VFE_CSI_RESET] = { 0x0210, 24 }, + [MDP_RESET] = { 0x0210, 21 }, + [AMP_RESET] = { 0x0210, 20 }, + [JPEGD_RESET] = { 0x0210, 19 }, + [CSI1_RESET] = { 0x0210, 18 }, + [VPE_RESET] = { 0x0210, 17 }, + [MMSS_FABRIC_RESET] = { 0x0210, 16 }, + [VFE_RESET] = { 0x0210, 15 }, + [GFX3D_RESET] = { 0x0210, 12 }, + [HDMI_RESET] = { 0x0210, 11 }, + [MMSS_IMEM_RESET] = { 0x0210, 10 }, + [IJPEG_RESET] = { 0x0210, 9 }, + [CSI0_RESET] = { 0x0210, 8 }, + [DSI_RESET] = { 0x0210, 7 }, + [VCODEC_RESET] = { 0x0210, 6 }, + [MDP_TV_RESET] = { 0x0210, 4 }, + [MDP_VSYNC_RESET] = { 0x0210, 3 }, + [ROT_RESET] = { 0x0210, 2 }, + [TV_HDMI_RESET] = { 0x0210, 1 }, + [VCAP_NPL_RESET] = { 0x0214, 4 }, + [VCAP_RESET] = { 0x0214, 3 }, + [CSI2_RESET] = { 0x0214, 2 }, + [CSI_RDI1_RESET] = { 0x0214, 1 }, + [CSI_RDI2_RESET] = { 0x0214 }, +}; + static const struct regmap_config mmcc_msm8960_regmap_config = { .reg_bits = 32, .reg_stride = 4, @@ -2223,6 +2609,14 @@ static const struct regmap_config mmcc_msm8960_regmap_config = { .fast_io = true, }; +static const struct regmap_config mmcc_apq8064_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0x350, + .fast_io = true, +}; + static const struct qcom_cc_desc mmcc_msm8960_desc = { .config = &mmcc_msm8960_regmap_config, .clks = mmcc_msm8960_clks, @@ -2231,15 +2625,47 @@ static const struct qcom_cc_desc mmcc_msm8960_desc = { .num_resets = ARRAY_SIZE(mmcc_msm8960_resets), }; +static const struct qcom_cc_desc mmcc_apq8064_desc = { + .config = &mmcc_apq8064_regmap_config, + .clks = mmcc_apq8064_clks, + .num_clks = ARRAY_SIZE(mmcc_apq8064_clks), + .resets = mmcc_apq8064_resets, + .num_resets = ARRAY_SIZE(mmcc_apq8064_resets), +}; + static const struct of_device_id mmcc_msm8960_match_table[] = { - { .compatible = "qcom,mmcc-msm8960" }, + { .compatible = "qcom,mmcc-msm8960", .data = &mmcc_msm8960_desc }, + { .compatible = "qcom,mmcc-apq8064", .data = &mmcc_apq8064_desc }, { } }; MODULE_DEVICE_TABLE(of, mmcc_msm8960_match_table); static int mmcc_msm8960_probe(struct platform_device *pdev) { - return qcom_cc_probe(pdev, &mmcc_msm8960_desc); + const struct of_device_id *match; + struct regmap *regmap; + bool is_8064; + struct device *dev = &pdev->dev; + + match = of_match_device(mmcc_msm8960_match_table, dev); + if (!match) + return -EINVAL; + + is_8064 = of_device_is_compatible(dev->of_node, "qcom,mmcc-apq8064"); + if (is_8064) { + gfx3d_src.freq_tbl = clk_tbl_gfx3d_8064; + gfx3d_src.clkr.hw.init = &gfx3d_8064_init; + gfx3d_src.s[0].parent_map = mmcc_pxo_pll8_pll2_pll15_map; + gfx3d_src.s[1].parent_map = mmcc_pxo_pll8_pll2_pll15_map; + } + + regmap = qcom_cc_map(pdev, match->data); + if (IS_ERR(regmap)) + return PTR_ERR(regmap); + + clk_pll_configure_sr(&pll15, regmap, &pll15_config, false); + + return qcom_cc_really_probe(pdev, match->data, regmap); } static int mmcc_msm8960_remove(struct platform_device *pdev) diff --git a/drivers/clk/qcom/mmcc-msm8974.c b/drivers/clk/qcom/mmcc-msm8974.c index c65b90515872..bc8f519c47aa 100644 --- a/drivers/clk/qcom/mmcc-msm8974.c +++ b/drivers/clk/qcom/mmcc-msm8974.c @@ -2547,18 +2547,16 @@ MODULE_DEVICE_TABLE(of, mmcc_msm8974_match_table); static int mmcc_msm8974_probe(struct platform_device *pdev) { - int ret; struct regmap *regmap; - ret = qcom_cc_probe(pdev, &mmcc_msm8974_desc); - if (ret) - return ret; + regmap = qcom_cc_map(pdev, &mmcc_msm8974_desc); + if (IS_ERR(regmap)) + return PTR_ERR(regmap); - regmap = dev_get_regmap(&pdev->dev, NULL); clk_pll_configure_sr_hpm_lp(&mmpll1, regmap, &mmpll1_config, true); clk_pll_configure_sr_hpm_lp(&mmpll3, regmap, &mmpll3_config, false); - return 0; + return qcom_cc_really_probe(pdev, &mmcc_msm8974_desc, regmap); } static int mmcc_msm8974_remove(struct platform_device *pdev) diff --git a/drivers/clk/rockchip/Makefile b/drivers/clk/rockchip/Makefile index 8d3aefad2e73..ee6b077381e1 100644 --- a/drivers/clk/rockchip/Makefile +++ b/drivers/clk/rockchip/Makefile @@ -3,3 +3,9 @@ # obj-y += clk-rockchip.o +obj-y += clk.o +obj-y += clk-pll.o +obj-$(CONFIG_RESET_CONTROLLER) += softrst.o + +obj-y += clk-rk3188.o +obj-y += clk-rk3288.o diff --git a/drivers/clk/rockchip/clk-pll.c b/drivers/clk/rockchip/clk-pll.c new file mode 100644 index 000000000000..f2a1c7abf4d9 --- /dev/null +++ b/drivers/clk/rockchip/clk-pll.c @@ -0,0 +1,431 @@ +/* + * Copyright (c) 2014 MundoReader S.L. + * Author: Heiko Stuebner <heiko@sntech.de> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include <asm/div64.h> +#include <linux/slab.h> +#include <linux/io.h> +#include <linux/delay.h> +#include <linux/clk.h> +#include <linux/clk-provider.h> +#include <linux/regmap.h> +#include "clk.h" + +#define PLL_MODE_MASK 0x3 +#define PLL_MODE_SLOW 0x0 +#define PLL_MODE_NORM 0x1 +#define PLL_MODE_DEEP 0x2 + +struct rockchip_clk_pll { + struct clk_hw hw; + + struct clk_mux pll_mux; + const struct clk_ops *pll_mux_ops; + + struct notifier_block clk_nb; + bool rate_change_remuxed; + + void __iomem *reg_base; + int lock_offset; + unsigned int lock_shift; + enum rockchip_pll_type type; + const struct rockchip_pll_rate_table *rate_table; + unsigned int rate_count; + spinlock_t *lock; +}; + +#define to_rockchip_clk_pll(_hw) container_of(_hw, struct rockchip_clk_pll, hw) +#define to_rockchip_clk_pll_nb(nb) \ + container_of(nb, struct rockchip_clk_pll, clk_nb) + +static const struct rockchip_pll_rate_table *rockchip_get_pll_settings( + struct rockchip_clk_pll *pll, unsigned long rate) +{ + const struct rockchip_pll_rate_table *rate_table = pll->rate_table; + int i; + + for (i = 0; i < pll->rate_count; i++) { + if (rate == rate_table[i].rate) + return &rate_table[i]; + } + + return NULL; +} + +static long rockchip_pll_round_rate(struct clk_hw *hw, + unsigned long drate, unsigned long *prate) +{ + struct rockchip_clk_pll *pll = to_rockchip_clk_pll(hw); + const struct rockchip_pll_rate_table *rate_table = pll->rate_table; + int i; + + /* Assumming rate_table is in descending order */ + for (i = 0; i < pll->rate_count; i++) { + if (drate >= rate_table[i].rate) + return rate_table[i].rate; + } + + /* return minimum supported value */ + return rate_table[i - 1].rate; +} + +/* + * Wait for the pll to reach the locked state. + * The calling set_rate function is responsible for making sure the + * grf regmap is available. + */ +static int rockchip_pll_wait_lock(struct rockchip_clk_pll *pll) +{ + struct regmap *grf = rockchip_clk_get_grf(); + unsigned int val; + int delay = 24000000, ret; + + while (delay > 0) { + ret = regmap_read(grf, pll->lock_offset, &val); + if (ret) { + pr_err("%s: failed to read pll lock status: %d\n", + __func__, ret); + return ret; + } + + if (val & BIT(pll->lock_shift)) + return 0; + delay--; + } + + pr_err("%s: timeout waiting for pll to lock\n", __func__); + return -ETIMEDOUT; +} + +/** + * Set pll mux when changing the pll rate. + * This makes sure to move the pll mux away from the actual pll before + * changing its rate and back to the original parent after the change. + */ +static int rockchip_pll_notifier_cb(struct notifier_block *nb, + unsigned long event, void *data) +{ + struct rockchip_clk_pll *pll = to_rockchip_clk_pll_nb(nb); + struct clk_mux *pll_mux = &pll->pll_mux; + const struct clk_ops *pll_mux_ops = pll->pll_mux_ops; + int cur_parent; + + switch (event) { + case PRE_RATE_CHANGE: + cur_parent = pll_mux_ops->get_parent(&pll_mux->hw); + if (cur_parent == PLL_MODE_NORM) { + pll_mux_ops->set_parent(&pll_mux->hw, PLL_MODE_SLOW); + pll->rate_change_remuxed = 1; + } + break; + case POST_RATE_CHANGE: + if (pll->rate_change_remuxed) { + pll_mux_ops->set_parent(&pll_mux->hw, PLL_MODE_NORM); + pll->rate_change_remuxed = 0; + } + break; + } + + return NOTIFY_OK; +} + +/** + * PLL used in RK3066, RK3188 and RK3288 + */ + +#define RK3066_PLL_RESET_DELAY(nr) ((nr * 500) / 24 + 1) + +#define RK3066_PLLCON(i) (i * 0x4) +#define RK3066_PLLCON0_OD_MASK 0xf +#define RK3066_PLLCON0_OD_SHIFT 0 +#define RK3066_PLLCON0_NR_MASK 0x3f +#define RK3066_PLLCON0_NR_SHIFT 8 +#define RK3066_PLLCON1_NF_MASK 0x1fff +#define RK3066_PLLCON1_NF_SHIFT 0 +#define RK3066_PLLCON2_BWADJ_MASK 0xfff +#define RK3066_PLLCON2_BWADJ_SHIFT 0 +#define RK3066_PLLCON3_RESET (1 << 5) +#define RK3066_PLLCON3_PWRDOWN (1 << 1) +#define RK3066_PLLCON3_BYPASS (1 << 0) + +static unsigned long rockchip_rk3066_pll_recalc_rate(struct clk_hw *hw, + unsigned long prate) +{ + struct rockchip_clk_pll *pll = to_rockchip_clk_pll(hw); + u64 nf, nr, no, rate64 = prate; + u32 pllcon; + + pllcon = readl_relaxed(pll->reg_base + RK3066_PLLCON(3)); + if (pllcon & RK3066_PLLCON3_BYPASS) { + pr_debug("%s: pll %s is bypassed\n", __func__, + __clk_get_name(hw->clk)); + return prate; + } + + pllcon = readl_relaxed(pll->reg_base + RK3066_PLLCON(1)); + nf = (pllcon >> RK3066_PLLCON1_NF_SHIFT) & RK3066_PLLCON1_NF_MASK; + + pllcon = readl_relaxed(pll->reg_base + RK3066_PLLCON(0)); + nr = (pllcon >> RK3066_PLLCON0_NR_SHIFT) & RK3066_PLLCON0_NR_MASK; + no = (pllcon >> RK3066_PLLCON0_OD_SHIFT) & RK3066_PLLCON0_OD_MASK; + + rate64 *= (nf + 1); + do_div(rate64, nr + 1); + do_div(rate64, no + 1); + + return (unsigned long)rate64; +} + +static int rockchip_rk3066_pll_set_rate(struct clk_hw *hw, unsigned long drate, + unsigned long prate) +{ + struct rockchip_clk_pll *pll = to_rockchip_clk_pll(hw); + const struct rockchip_pll_rate_table *rate; + unsigned long old_rate = rockchip_rk3066_pll_recalc_rate(hw, prate); + struct regmap *grf = rockchip_clk_get_grf(); + int ret; + + if (IS_ERR(grf)) { + pr_debug("%s: grf regmap not available, aborting rate change\n", + __func__); + return PTR_ERR(grf); + } + + pr_debug("%s: changing %s from %lu to %lu with a parent rate of %lu\n", + __func__, __clk_get_name(hw->clk), old_rate, drate, prate); + + /* Get required rate settings from table */ + rate = rockchip_get_pll_settings(pll, drate); + if (!rate) { + pr_err("%s: Invalid rate : %lu for pll clk %s\n", __func__, + drate, __clk_get_name(hw->clk)); + return -EINVAL; + } + + pr_debug("%s: rate settings for %lu (nr, no, nf): (%d, %d, %d)\n", + __func__, rate->rate, rate->nr, rate->no, rate->nf); + + /* enter reset mode */ + writel(HIWORD_UPDATE(RK3066_PLLCON3_RESET, RK3066_PLLCON3_RESET, 0), + pll->reg_base + RK3066_PLLCON(3)); + + /* update pll values */ + writel(HIWORD_UPDATE(rate->nr - 1, RK3066_PLLCON0_NR_MASK, + RK3066_PLLCON0_NR_SHIFT) | + HIWORD_UPDATE(rate->no - 1, RK3066_PLLCON0_OD_MASK, + RK3066_PLLCON0_OD_SHIFT), + pll->reg_base + RK3066_PLLCON(0)); + + writel_relaxed(HIWORD_UPDATE(rate->nf - 1, RK3066_PLLCON1_NF_MASK, + RK3066_PLLCON1_NF_SHIFT), + pll->reg_base + RK3066_PLLCON(1)); + writel_relaxed(HIWORD_UPDATE(rate->bwadj, RK3066_PLLCON2_BWADJ_MASK, + RK3066_PLLCON2_BWADJ_SHIFT), + pll->reg_base + RK3066_PLLCON(2)); + + /* leave reset and wait the reset_delay */ + writel(HIWORD_UPDATE(0, RK3066_PLLCON3_RESET, 0), + pll->reg_base + RK3066_PLLCON(3)); + udelay(RK3066_PLL_RESET_DELAY(rate->nr)); + + /* wait for the pll to lock */ + ret = rockchip_pll_wait_lock(pll); + if (ret) { + pr_warn("%s: pll did not lock, trying to restore old rate %lu\n", + __func__, old_rate); + rockchip_rk3066_pll_set_rate(hw, old_rate, prate); + } + + return ret; +} + +static int rockchip_rk3066_pll_enable(struct clk_hw *hw) +{ + struct rockchip_clk_pll *pll = to_rockchip_clk_pll(hw); + + writel(HIWORD_UPDATE(0, RK3066_PLLCON3_PWRDOWN, 0), + pll->reg_base + RK3066_PLLCON(3)); + + return 0; +} + +static void rockchip_rk3066_pll_disable(struct clk_hw *hw) +{ + struct rockchip_clk_pll *pll = to_rockchip_clk_pll(hw); + + writel(HIWORD_UPDATE(RK3066_PLLCON3_PWRDOWN, + RK3066_PLLCON3_PWRDOWN, 0), + pll->reg_base + RK3066_PLLCON(3)); +} + +static int rockchip_rk3066_pll_is_enabled(struct clk_hw *hw) +{ + struct rockchip_clk_pll *pll = to_rockchip_clk_pll(hw); + u32 pllcon = readl(pll->reg_base + RK3066_PLLCON(3)); + + return !(pllcon & RK3066_PLLCON3_PWRDOWN); +} + +static const struct clk_ops rockchip_rk3066_pll_clk_norate_ops = { + .recalc_rate = rockchip_rk3066_pll_recalc_rate, + .enable = rockchip_rk3066_pll_enable, + .disable = rockchip_rk3066_pll_disable, + .is_enabled = rockchip_rk3066_pll_is_enabled, +}; + +static const struct clk_ops rockchip_rk3066_pll_clk_ops = { + .recalc_rate = rockchip_rk3066_pll_recalc_rate, + .round_rate = rockchip_pll_round_rate, + .set_rate = rockchip_rk3066_pll_set_rate, + .enable = rockchip_rk3066_pll_enable, + .disable = rockchip_rk3066_pll_disable, + .is_enabled = rockchip_rk3066_pll_is_enabled, +}; + +/* + * Common registering of pll clocks + */ + +struct clk *rockchip_clk_register_pll(enum rockchip_pll_type pll_type, + const char *name, const char **parent_names, u8 num_parents, + void __iomem *base, int con_offset, int grf_lock_offset, + int lock_shift, int mode_offset, int mode_shift, + struct rockchip_pll_rate_table *rate_table, + spinlock_t *lock) +{ + const char *pll_parents[3]; + struct clk_init_data init; + struct rockchip_clk_pll *pll; + struct clk_mux *pll_mux; + struct clk *pll_clk, *mux_clk; + char pll_name[20]; + int ret; + + if (num_parents != 2) { + pr_err("%s: needs two parent clocks\n", __func__); + return ERR_PTR(-EINVAL); + } + + /* name the actual pll */ + snprintf(pll_name, sizeof(pll_name), "pll_%s", name); + + pll = kzalloc(sizeof(*pll), GFP_KERNEL); + if (!pll) + return ERR_PTR(-ENOMEM); + + init.name = pll_name; + + /* keep all plls untouched for now */ + init.flags = CLK_IGNORE_UNUSED; + + init.parent_names = &parent_names[0]; + init.num_parents = 1; + + if (rate_table) { + int len; + + /* find count of rates in rate_table */ + for (len = 0; rate_table[len].rate != 0; ) + len++; + + pll->rate_count = len; + pll->rate_table = kmemdup(rate_table, + pll->rate_count * + sizeof(struct rockchip_pll_rate_table), + GFP_KERNEL); + WARN(!pll->rate_table, + "%s: could not allocate rate table for %s\n", + __func__, name); + } + + switch (pll_type) { + case pll_rk3066: + if (!pll->rate_table) + init.ops = &rockchip_rk3066_pll_clk_norate_ops; + else + init.ops = &rockchip_rk3066_pll_clk_ops; + break; + default: + pr_warn("%s: Unknown pll type for pll clk %s\n", + __func__, name); + } + + pll->hw.init = &init; + pll->type = pll_type; + pll->reg_base = base + con_offset; + pll->lock_offset = grf_lock_offset; + pll->lock_shift = lock_shift; + pll->lock = lock; + pll->clk_nb.notifier_call = rockchip_pll_notifier_cb; + + pll_clk = clk_register(NULL, &pll->hw); + if (IS_ERR(pll_clk)) { + pr_err("%s: failed to register pll clock %s : %ld\n", + __func__, name, PTR_ERR(pll_clk)); + mux_clk = pll_clk; + goto err_pll; + } + + ret = clk_notifier_register(pll_clk, &pll->clk_nb); + if (ret) { + pr_err("%s: failed to register clock notifier for %s : %d\n", + __func__, name, ret); + mux_clk = ERR_PTR(ret); + goto err_pll_notifier; + } + + /* create the mux on top of the real pll */ + pll->pll_mux_ops = &clk_mux_ops; + pll_mux = &pll->pll_mux; + + /* the actual muxing is xin24m, pll-output, xin32k */ + pll_parents[0] = parent_names[0]; + pll_parents[1] = pll_name; + pll_parents[2] = parent_names[1]; + + init.name = name; + init.flags = CLK_SET_RATE_PARENT; + init.ops = pll->pll_mux_ops; + init.parent_names = pll_parents; + init.num_parents = ARRAY_SIZE(pll_parents); + + pll_mux->reg = base + mode_offset; + pll_mux->shift = mode_shift; + pll_mux->mask = PLL_MODE_MASK; + pll_mux->flags = 0; + pll_mux->lock = lock; + pll_mux->hw.init = &init; + + if (pll_type == pll_rk3066) + pll_mux->flags |= CLK_MUX_HIWORD_MASK; + + mux_clk = clk_register(NULL, &pll_mux->hw); + if (IS_ERR(mux_clk)) + goto err_mux; + + return mux_clk; + +err_mux: + ret = clk_notifier_unregister(pll_clk, &pll->clk_nb); + if (ret) { + pr_err("%s: could not unregister clock notifier in error path : %d\n", + __func__, ret); + return mux_clk; + } +err_pll_notifier: + clk_unregister(pll_clk); +err_pll: + kfree(pll); + return mux_clk; +} diff --git a/drivers/clk/rockchip/clk-rk3188.c b/drivers/clk/rockchip/clk-rk3188.c new file mode 100644 index 000000000000..a83a6d8d0fb6 --- /dev/null +++ b/drivers/clk/rockchip/clk-rk3188.c @@ -0,0 +1,672 @@ +/* + * Copyright (c) 2014 MundoReader S.L. + * Author: Heiko Stuebner <heiko@sntech.de> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include <linux/clk-provider.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include <dt-bindings/clock/rk3188-cru-common.h> +#include "clk.h" + +#define RK3188_GRF_SOC_STATUS 0xac + +enum rk3188_plls { + apll, cpll, dpll, gpll, +}; + +struct rockchip_pll_rate_table rk3188_pll_rates[] = { + RK3066_PLL_RATE(2208000000, 1, 92, 1), + RK3066_PLL_RATE(2184000000, 1, 91, 1), + RK3066_PLL_RATE(2160000000, 1, 90, 1), + RK3066_PLL_RATE(2136000000, 1, 89, 1), + RK3066_PLL_RATE(2112000000, 1, 88, 1), + RK3066_PLL_RATE(2088000000, 1, 87, 1), + RK3066_PLL_RATE(2064000000, 1, 86, 1), + RK3066_PLL_RATE(2040000000, 1, 85, 1), + RK3066_PLL_RATE(2016000000, 1, 84, 1), + RK3066_PLL_RATE(1992000000, 1, 83, 1), + RK3066_PLL_RATE(1968000000, 1, 82, 1), + RK3066_PLL_RATE(1944000000, 1, 81, 1), + RK3066_PLL_RATE(1920000000, 1, 80, 1), + RK3066_PLL_RATE(1896000000, 1, 79, 1), + RK3066_PLL_RATE(1872000000, 1, 78, 1), + RK3066_PLL_RATE(1848000000, 1, 77, 1), + RK3066_PLL_RATE(1824000000, 1, 76, 1), + RK3066_PLL_RATE(1800000000, 1, 75, 1), + RK3066_PLL_RATE(1776000000, 1, 74, 1), + RK3066_PLL_RATE(1752000000, 1, 73, 1), + RK3066_PLL_RATE(1728000000, 1, 72, 1), + RK3066_PLL_RATE(1704000000, 1, 71, 1), + RK3066_PLL_RATE(1680000000, 1, 70, 1), + RK3066_PLL_RATE(1656000000, 1, 69, 1), + RK3066_PLL_RATE(1632000000, 1, 68, 1), + RK3066_PLL_RATE(1608000000, 1, 67, 1), + RK3066_PLL_RATE(1560000000, 1, 65, 1), + RK3066_PLL_RATE(1512000000, 1, 63, 1), + RK3066_PLL_RATE(1488000000, 1, 62, 1), + RK3066_PLL_RATE(1464000000, 1, 61, 1), + RK3066_PLL_RATE(1440000000, 1, 60, 1), + RK3066_PLL_RATE(1416000000, 1, 59, 1), + RK3066_PLL_RATE(1392000000, 1, 58, 1), + RK3066_PLL_RATE(1368000000, 1, 57, 1), + RK3066_PLL_RATE(1344000000, 1, 56, 1), + RK3066_PLL_RATE(1320000000, 1, 55, 1), + RK3066_PLL_RATE(1296000000, 1, 54, 1), + RK3066_PLL_RATE(1272000000, 1, 53, 1), + RK3066_PLL_RATE(1248000000, 1, 52, 1), + RK3066_PLL_RATE(1224000000, 1, 51, 1), + RK3066_PLL_RATE(1200000000, 1, 50, 1), + RK3066_PLL_RATE(1188000000, 2, 99, 1), + RK3066_PLL_RATE(1176000000, 1, 49, 1), + RK3066_PLL_RATE(1128000000, 1, 47, 1), + RK3066_PLL_RATE(1104000000, 1, 46, 1), + RK3066_PLL_RATE(1008000000, 1, 84, 2), + RK3066_PLL_RATE( 912000000, 1, 76, 2), + RK3066_PLL_RATE( 891000000, 8, 594, 2), + RK3066_PLL_RATE( 888000000, 1, 74, 2), + RK3066_PLL_RATE( 816000000, 1, 68, 2), + RK3066_PLL_RATE( 798000000, 2, 133, 2), + RK3066_PLL_RATE( 792000000, 1, 66, 2), + RK3066_PLL_RATE( 768000000, 1, 64, 2), + RK3066_PLL_RATE( 742500000, 8, 495, 2), + RK3066_PLL_RATE( 696000000, 1, 58, 2), + RK3066_PLL_RATE( 600000000, 1, 50, 2), + RK3066_PLL_RATE( 594000000, 2, 198, 4), + RK3066_PLL_RATE( 552000000, 1, 46, 2), + RK3066_PLL_RATE( 504000000, 1, 84, 4), + RK3066_PLL_RATE( 456000000, 1, 76, 4), + RK3066_PLL_RATE( 408000000, 1, 68, 4), + RK3066_PLL_RATE( 384000000, 2, 128, 4), + RK3066_PLL_RATE( 360000000, 1, 60, 4), + RK3066_PLL_RATE( 312000000, 1, 52, 4), + RK3066_PLL_RATE( 300000000, 1, 50, 4), + RK3066_PLL_RATE( 297000000, 2, 198, 8), + RK3066_PLL_RATE( 252000000, 1, 84, 8), + RK3066_PLL_RATE( 216000000, 1, 72, 8), + RK3066_PLL_RATE( 148500000, 2, 99, 8), + RK3066_PLL_RATE( 126000000, 1, 84, 16), + RK3066_PLL_RATE( 48000000, 1, 64, 32), + { /* sentinel */ }, +}; + +PNAME(mux_pll_p) = { "xin24m", "xin32k" }; +PNAME(mux_armclk_p) = { "apll", "gpll_armclk" }; +PNAME(mux_ddrphy_p) = { "dpll", "gpll_ddr" }; +PNAME(mux_pll_src_gpll_cpll_p) = { "gpll", "cpll" }; +PNAME(mux_pll_src_cpll_gpll_p) = { "cpll", "gpll" }; +PNAME(mux_aclk_cpu_p) = { "apll", "gpll" }; +PNAME(mux_sclk_cif0_p) = { "cif0_pre", "xin24m" }; +PNAME(mux_sclk_i2s0_p) = { "i2s0_pre", "i2s0_frac", "xin12m" }; +PNAME(mux_sclk_spdif_p) = { "spdif_src", "spdif_frac", "xin12m" }; +PNAME(mux_sclk_uart0_p) = { "uart0_pre", "uart0_frac", "xin24m" }; +PNAME(mux_sclk_uart1_p) = { "uart1_pre", "uart1_frac", "xin24m" }; +PNAME(mux_sclk_uart2_p) = { "uart2_pre", "uart2_frac", "xin24m" }; +PNAME(mux_sclk_uart3_p) = { "uart3_pre", "uart3_frac", "xin24m" }; +PNAME(mux_sclk_hsadc_p) = { "hsadc_src", "hsadc_frac", "ext_hsadc" }; +PNAME(mux_mac_p) = { "gpll", "dpll" }; +PNAME(mux_sclk_macref_p) = { "mac_src", "ext_rmii" }; + +static struct rockchip_pll_clock rk3188_pll_clks[] __initdata = { + [apll] = PLL(pll_rk3066, PLL_APLL, "apll", mux_pll_p, 0, RK2928_PLL_CON(0), + RK2928_MODE_CON, 0, 6, rk3188_pll_rates), + [dpll] = PLL(pll_rk3066, PLL_DPLL, "dpll", mux_pll_p, 0, RK2928_PLL_CON(4), + RK2928_MODE_CON, 4, 5, NULL), + [cpll] = PLL(pll_rk3066, PLL_CPLL, "cpll", mux_pll_p, 0, RK2928_PLL_CON(8), + RK2928_MODE_CON, 8, 7, rk3188_pll_rates), + [gpll] = PLL(pll_rk3066, PLL_GPLL, "gpll", mux_pll_p, 0, RK2928_PLL_CON(12), + RK2928_MODE_CON, 12, 8, rk3188_pll_rates), +}; + +#define MFLAGS CLK_MUX_HIWORD_MASK +#define DFLAGS CLK_DIVIDER_HIWORD_MASK +#define GFLAGS (CLK_GATE_HIWORD_MASK | CLK_GATE_SET_TO_DISABLE) + +/* 2 ^ (val + 1) */ +static struct clk_div_table div_core_peri_t[] = { + { .val = 0, .div = 2 }, + { .val = 1, .div = 4 }, + { .val = 2, .div = 8 }, + { .val = 3, .div = 16 }, + { /* sentinel */ }, +}; + +static struct rockchip_clk_branch common_clk_branches[] __initdata = { + /* + * Clock-Architecture Diagram 2 + */ + + GATE(0, "gpll_armclk", "gpll", 0, RK2928_CLKGATE_CON(0), 1, GFLAGS), + + /* these two are set by the cpuclk and should not be changed */ + COMPOSITE_NOMUX_DIVTBL(CORE_PERI, "core_peri", "armclk", 0, + RK2928_CLKSEL_CON(0), 6, 2, DFLAGS | CLK_DIVIDER_READ_ONLY, + div_core_peri_t, RK2928_CLKGATE_CON(0), 0, GFLAGS), + + COMPOSITE(0, "aclk_vepu", mux_pll_src_cpll_gpll_p, 0, + RK2928_CLKSEL_CON(32), 7, 1, MFLAGS, 0, 5, DFLAGS, + RK2928_CLKGATE_CON(3), 9, GFLAGS), + GATE(0, "hclk_vepu", "aclk_vepu", 0, + RK2928_CLKGATE_CON(3), 10, GFLAGS), + COMPOSITE(0, "aclk_vdpu", mux_pll_src_cpll_gpll_p, 0, + RK2928_CLKSEL_CON(32), 15, 1, MFLAGS, 8, 5, DFLAGS, + RK2928_CLKGATE_CON(3), 11, GFLAGS), + GATE(0, "hclk_vdpu", "aclk_vdpu", 0, + RK2928_CLKGATE_CON(3), 12, GFLAGS), + + GATE(0, "gpll_ddr", "gpll", 0, + RK2928_CLKGATE_CON(1), 7, GFLAGS), + COMPOSITE(0, "ddrphy", mux_ddrphy_p, 0, + RK2928_CLKSEL_CON(26), 8, 1, MFLAGS, 0, 2, DFLAGS | CLK_DIVIDER_POWER_OF_TWO, + RK2928_CLKGATE_CON(0), 2, GFLAGS), + + GATE(0, "aclk_cpu", "aclk_cpu_pre", 0, + RK2928_CLKGATE_CON(0), 3, GFLAGS), + + DIV(0, "pclk_cpu_pre", "aclk_cpu_pre", 0, + RK2928_CLKSEL_CON(1), 12, 2, DFLAGS | CLK_DIVIDER_POWER_OF_TWO), + GATE(0, "atclk_cpu", "pclk_cpu_pre", 0, + RK2928_CLKGATE_CON(0), 6, GFLAGS), + GATE(0, "pclk_cpu", "pclk_cpu_pre", 0, + RK2928_CLKGATE_CON(0), 5, GFLAGS), + DIV(0, "hclk_cpu_pre", "aclk_cpu_pre", 0, + RK2928_CLKSEL_CON(1), 8, 2, DFLAGS | CLK_DIVIDER_POWER_OF_TWO), + COMPOSITE_NOMUX(0, "hclk_ahb2apb", "hclk_cpu_pre", 0, + RK2928_CLKSEL_CON(1), 14, 2, DFLAGS | CLK_DIVIDER_POWER_OF_TWO, + RK2928_CLKGATE_CON(4), 9, GFLAGS), + GATE(0, "hclk_cpu", "hclk_cpu_pre", 0, + RK2928_CLKGATE_CON(0), 4, GFLAGS), + + COMPOSITE(0, "aclk_lcdc0_pre", mux_pll_src_cpll_gpll_p, 0, + RK2928_CLKSEL_CON(31), 7, 1, MFLAGS, 0, 5, DFLAGS, + RK2928_CLKGATE_CON(3), 0, GFLAGS), + COMPOSITE(0, "aclk_lcdc1_pre", mux_pll_src_cpll_gpll_p, 0, + RK2928_CLKSEL_CON(31), 15, 1, MFLAGS, 8, 5, DFLAGS, + RK2928_CLKGATE_CON(1), 4, GFLAGS), + + GATE(0, "aclk_peri", "aclk_peri_pre", 0, + RK2928_CLKGATE_CON(2), 1, GFLAGS), + COMPOSITE_NOMUX(0, "hclk_peri", "aclk_peri_pre", 0, + RK2928_CLKSEL_CON(10), 8, 2, DFLAGS | CLK_DIVIDER_POWER_OF_TWO, + RK2928_CLKGATE_CON(2), 2, GFLAGS), + COMPOSITE_NOMUX(0, "pclk_peri", "aclk_peri_pre", 0, + RK2928_CLKSEL_CON(10), 12, 2, DFLAGS | CLK_DIVIDER_POWER_OF_TWO, + RK2928_CLKGATE_CON(2), 3, GFLAGS), + + MUX(0, "cif_src", mux_pll_src_cpll_gpll_p, 0, + RK2928_CLKSEL_CON(29), 0, 1, MFLAGS), + COMPOSITE_NOMUX(0, "cif0_pre", "cif_src", 0, + RK2928_CLKSEL_CON(29), 1, 5, DFLAGS, + RK2928_CLKGATE_CON(3), 7, GFLAGS), + MUX(SCLK_CIF0, "sclk_cif0", mux_sclk_cif0_p, 0, + RK2928_CLKSEL_CON(29), 7, 1, MFLAGS), + + GATE(0, "pclkin_cif0", "ext_cif0", 0, + RK2928_CLKGATE_CON(3), 3, GFLAGS), + + /* + * the 480m are generated inside the usb block from these clocks, + * but they are also a source for the hsicphy clock. + */ + GATE(SCLK_OTGPHY0, "sclk_otgphy0", "usb480m", 0, + RK2928_CLKGATE_CON(1), 5, GFLAGS), + GATE(SCLK_OTGPHY1, "sclk_otgphy1", "usb480m", 0, + RK2928_CLKGATE_CON(1), 6, GFLAGS), + + COMPOSITE(0, "mac_src", mux_mac_p, 0, + RK2928_CLKSEL_CON(21), 0, 1, MFLAGS, 8, 5, DFLAGS, + RK2928_CLKGATE_CON(2), 5, GFLAGS), + MUX(SCLK_MAC, "sclk_macref", mux_sclk_macref_p, CLK_SET_RATE_PARENT, + RK2928_CLKSEL_CON(21), 4, 1, MFLAGS), + GATE(0, "sclk_mac_lbtest", "sclk_macref", + RK2928_CLKGATE_CON(2), 12, 0, GFLAGS), + + COMPOSITE(0, "hsadc_src", mux_pll_src_gpll_cpll_p, 0, + RK2928_CLKSEL_CON(22), 0, 1, MFLAGS, 8, 8, DFLAGS, + RK2928_CLKGATE_CON(2), 6, GFLAGS), + COMPOSITE_FRAC(0, "hsadc_frac", "hsadc_src", + RK2928_CLKSEL_CON(23), 0, + RK2928_CLKGATE_CON(2), 7, 0, GFLAGS), + MUX(SCLK_HSADC, "sclk_hsadc", mux_sclk_hsadc_p, 0, + RK2928_CLKSEL_CON(22), 4, 2, MFLAGS), + + COMPOSITE_NOMUX(SCLK_SARADC, "sclk_saradc", "xin24m", 0, + RK2928_CLKSEL_CON(24), 8, 8, DFLAGS, + RK2928_CLKGATE_CON(2), 8, GFLAGS), + + /* + * Clock-Architecture Diagram 4 + */ + + GATE(SCLK_SMC, "sclk_smc", "hclk_peri", + RK2928_CLKGATE_CON(2), 4, 0, GFLAGS), + + COMPOSITE_NOMUX(SCLK_SPI0, "sclk_spi0", "pclk_peri", 0, + RK2928_CLKSEL_CON(25), 0, 7, DFLAGS, + RK2928_CLKGATE_CON(2), 9, GFLAGS), + COMPOSITE_NOMUX(SCLK_SPI1, "sclk_spi1", "pclk_peri", 0, + RK2928_CLKSEL_CON(25), 8, 7, DFLAGS, + RK2928_CLKGATE_CON(2), 10, GFLAGS), + + COMPOSITE_NOMUX(SCLK_SDMMC, "sclk_sdmmc", "hclk_peri", 0, + RK2928_CLKSEL_CON(11), 0, 6, DFLAGS, + RK2928_CLKGATE_CON(2), 11, GFLAGS), + COMPOSITE_NOMUX(SCLK_SDIO, "sclk_sdio", "hclk_peri", 0, + RK2928_CLKSEL_CON(12), 0, 6, DFLAGS, + RK2928_CLKGATE_CON(2), 13, GFLAGS), + COMPOSITE_NOMUX(SCLK_EMMC, "sclk_emmc", "hclk_peri", 0, + RK2928_CLKSEL_CON(12), 8, 6, DFLAGS, + RK2928_CLKGATE_CON(2), 14, GFLAGS), + + MUX(0, "uart_src", mux_pll_src_gpll_cpll_p, 0, + RK2928_CLKSEL_CON(12), 15, 1, MFLAGS), + COMPOSITE_NOMUX(0, "uart0_pre", "uart_src", 0, + RK2928_CLKSEL_CON(13), 0, 7, DFLAGS, + RK2928_CLKGATE_CON(1), 8, GFLAGS), + COMPOSITE_FRAC(0, "uart0_frac", "uart0_pre", 0, + RK2928_CLKSEL_CON(17), 0, + RK2928_CLKGATE_CON(1), 9, GFLAGS), + MUX(SCLK_UART0, "sclk_uart0", mux_sclk_uart0_p, 0, + RK2928_CLKSEL_CON(13), 8, 2, MFLAGS), + COMPOSITE_NOMUX(0, "uart1_pre", "uart_src", 0, + RK2928_CLKSEL_CON(14), 0, 7, DFLAGS, + RK2928_CLKGATE_CON(1), 10, GFLAGS), + COMPOSITE_FRAC(0, "uart1_frac", "uart1_pre", 0, + RK2928_CLKSEL_CON(18), 0, + RK2928_CLKGATE_CON(1), 11, GFLAGS), + MUX(SCLK_UART1, "sclk_uart1", mux_sclk_uart1_p, 0, + RK2928_CLKSEL_CON(14), 8, 2, MFLAGS), + COMPOSITE_NOMUX(0, "uart2_pre", "uart_src", 0, + RK2928_CLKSEL_CON(15), 0, 7, DFLAGS, + RK2928_CLKGATE_CON(1), 12, GFLAGS), + COMPOSITE_FRAC(0, "uart2_frac", "uart2_pre", 0, + RK2928_CLKSEL_CON(19), 0, + RK2928_CLKGATE_CON(1), 13, GFLAGS), + MUX(SCLK_UART2, "sclk_uart2", mux_sclk_uart2_p, 0, + RK2928_CLKSEL_CON(15), 8, 2, MFLAGS), + COMPOSITE_NOMUX(0, "uart3_pre", "uart_src", 0, + RK2928_CLKSEL_CON(16), 0, 7, DFLAGS, + RK2928_CLKGATE_CON(1), 14, GFLAGS), + COMPOSITE_FRAC(0, "uart3_frac", "uart3_pre", 0, + RK2928_CLKSEL_CON(20), 0, + RK2928_CLKGATE_CON(1), 15, GFLAGS), + MUX(SCLK_UART3, "sclk_uart3", mux_sclk_uart3_p, 0, + RK2928_CLKSEL_CON(16), 8, 2, MFLAGS), + + GATE(SCLK_JTAG, "jtag", "ext_jtag", 0, RK2928_CLKGATE_CON(1), 3, GFLAGS), + + GATE(SCLK_TIMER0, "timer0", "xin24m", 0, RK2928_CLKGATE_CON(1), 0, GFLAGS), + GATE(SCLK_TIMER1, "timer1", "xin24m", 0, RK2928_CLKGATE_CON(1), 1, GFLAGS), + + /* clk_core_pre gates */ + GATE(0, "core_dbg", "armclk", 0, RK2928_CLKGATE_CON(9), 0, GFLAGS), + + /* aclk_cpu gates */ + GATE(ACLK_DMA1, "aclk_dma1", "aclk_cpu", 0, RK2928_CLKGATE_CON(5), 0, GFLAGS), + GATE(0, "aclk_intmem", "aclk_cpu", 0, RK2928_CLKGATE_CON(4), 12, GFLAGS), + GATE(0, "aclk_strc_sys", "aclk_cpu", 0, RK2928_CLKGATE_CON(4), 10, GFLAGS), + + /* hclk_cpu gates */ + GATE(HCLK_ROM, "hclk_rom", "hclk_cpu", 0, RK2928_CLKGATE_CON(5), 6, GFLAGS), + GATE(HCLK_I2S0, "hclk_i2s0", "hclk_cpu", 0, RK2928_CLKGATE_CON(7), 2, GFLAGS), + GATE(HCLK_SPDIF, "hclk_spdif", "hclk_cpu", 0, RK2928_CLKGATE_CON(7), 1, GFLAGS), + GATE(0, "hclk_cpubus", "hclk_cpu", 0, RK2928_CLKGATE_CON(4), 8, GFLAGS), + /* hclk_ahb2apb is part of a clk branch */ + GATE(0, "hclk_vio_bus", "hclk_cpu", 0, RK2928_CLKGATE_CON(6), 12, GFLAGS), + GATE(HCLK_LCDC0, "hclk_lcdc0", "hclk_cpu", 0, RK2928_CLKGATE_CON(6), 1, GFLAGS), + GATE(HCLK_LCDC1, "hclk_lcdc1", "aclk_cpu", 0, RK2928_CLKGATE_CON(6), 2, GFLAGS), + GATE(HCLK_CIF0, "hclk_cif0", "hclk_cpu", 0, RK2928_CLKGATE_CON(6), 4, GFLAGS), + GATE(HCLK_IPP, "hclk_ipp", "hclk_cpu", 0, RK2928_CLKGATE_CON(6), 9, GFLAGS), + GATE(HCLK_RGA, "hclk_rga", "hclk_cpu", 0, RK2928_CLKGATE_CON(6), 10, GFLAGS), + + /* hclk_peri gates */ + GATE(0, "hclk_peri_axi_matrix", "hclk_peri", 0, RK2928_CLKGATE_CON(4), 0, GFLAGS), + GATE(0, "hclk_peri_ahb_arbi", "hclk_peri", 0, RK2928_CLKGATE_CON(4), 6, GFLAGS), + GATE(0, "hclk_emem_peri", "hclk_peri", 0, RK2928_CLKGATE_CON(4), 7, GFLAGS), + GATE(HCLK_EMAC, "hclk_emac", "hclk_peri", 0, RK2928_CLKGATE_CON(7), 0, GFLAGS), + GATE(HCLK_NANDC0, "hclk_nandc0", "hclk_peri", 0, RK2928_CLKGATE_CON(5), 9, GFLAGS), + GATE(0, "hclk_usb_peri", "hclk_peri", 0, RK2928_CLKGATE_CON(4), 5, GFLAGS), + GATE(HCLK_OTG0, "hclk_usbotg0", "hclk_peri", 0, RK2928_CLKGATE_CON(5), 13, GFLAGS), + GATE(HCLK_HSADC, "hclk_hsadc", "hclk_peri", 0, RK2928_CLKGATE_CON(7), 5, GFLAGS), + GATE(HCLK_PIDF, "hclk_pidfilter", "hclk_peri", 0, RK2928_CLKGATE_CON(7), 6, GFLAGS), + GATE(HCLK_SDMMC, "hclk_sdmmc", "hclk_peri", 0, RK2928_CLKGATE_CON(5), 10, GFLAGS), + GATE(HCLK_SDIO, "hclk_sdio", "hclk_peri", 0, RK2928_CLKGATE_CON(5), 11, GFLAGS), + GATE(HCLK_EMMC, "hclk_emmc", "hclk_peri", 0, RK2928_CLKGATE_CON(5), 12, GFLAGS), + + /* aclk_lcdc0_pre gates */ + GATE(0, "aclk_vio0", "aclk_lcdc0_pre", 0, RK2928_CLKGATE_CON(6), 13, GFLAGS), + GATE(ACLK_LCDC0, "aclk_lcdc0", "aclk_vio0", 0, RK2928_CLKGATE_CON(6), 0, GFLAGS), + GATE(ACLK_CIF0, "aclk_cif0", "aclk_vio0", 0, RK2928_CLKGATE_CON(6), 5, GFLAGS), + GATE(ACLK_IPP, "aclk_ipp", "aclk_vio0", 0, RK2928_CLKGATE_CON(6), 8, GFLAGS), + + /* aclk_lcdc1_pre gates */ + GATE(0, "aclk_vio1", "aclk_lcdc1_pre", 0, RK2928_CLKGATE_CON(9), 5, GFLAGS), + GATE(ACLK_LCDC1, "aclk_lcdc1", "aclk_vio1", 0, RK2928_CLKGATE_CON(6), 3, GFLAGS), + GATE(ACLK_RGA, "aclk_rga", "aclk_vio1", 0, RK2928_CLKGATE_CON(6), 11, GFLAGS), + + /* atclk_cpu gates */ + GATE(0, "atclk", "atclk_cpu", 0, RK2928_CLKGATE_CON(9), 3, GFLAGS), + GATE(0, "trace", "atclk_cpu", 0, RK2928_CLKGATE_CON(9), 2, GFLAGS), + + /* pclk_cpu gates */ + GATE(PCLK_PWM01, "pclk_pwm01", "pclk_cpu", 0, RK2928_CLKGATE_CON(7), 10, GFLAGS), + GATE(PCLK_TIMER0, "pclk_timer0", "pclk_cpu", 0, RK2928_CLKGATE_CON(7), 7, GFLAGS), + GATE(PCLK_I2C0, "pclk_i2c0", "pclk_cpu", 0, RK2928_CLKGATE_CON(8), 4, GFLAGS), + GATE(PCLK_I2C1, "pclk_i2c1", "pclk_cpu", 0, RK2928_CLKGATE_CON(8), 5, GFLAGS), + GATE(PCLK_GPIO0, "pclk_gpio0", "pclk_cpu", 0, RK2928_CLKGATE_CON(8), 9, GFLAGS), + GATE(PCLK_GPIO1, "pclk_gpio1", "pclk_cpu", 0, RK2928_CLKGATE_CON(8), 10, GFLAGS), + GATE(PCLK_GPIO2, "pclk_gpio2", "pclk_cpu", 0, RK2928_CLKGATE_CON(8), 11, GFLAGS), + GATE(PCLK_EFUSE, "pclk_efuse", "pclk_cpu", 0, RK2928_CLKGATE_CON(5), 2, GFLAGS), + GATE(PCLK_TZPC, "pclk_tzpc", "pclk_cpu", 0, RK2928_CLKGATE_CON(5), 3, GFLAGS), + GATE(0, "pclk_ddrupctl", "pclk_cpu", 0, RK2928_CLKGATE_CON(5), 7, GFLAGS), + GATE(0, "pclk_ddrpubl", "pclk_cpu", 0, RK2928_CLKGATE_CON(9), 6, GFLAGS), + GATE(0, "pclk_dbg", "pclk_cpu", 0, RK2928_CLKGATE_CON(9), 1, GFLAGS), + GATE(PCLK_GRF, "pclk_grf", "pclk_cpu", 0, RK2928_CLKGATE_CON(5), 4, GFLAGS), + GATE(PCLK_PMU, "pclk_pmu", "pclk_cpu", 0, RK2928_CLKGATE_CON(5), 5, GFLAGS), + + /* aclk_peri */ + GATE(ACLK_DMA2, "aclk_dma2", "aclk_peri", 0, RK2928_CLKGATE_CON(5), 1, GFLAGS), + GATE(ACLK_SMC, "aclk_smc", "aclk_peri", 0, RK2928_CLKGATE_CON(5), 8, GFLAGS), + GATE(0, "aclk_peri_niu", "aclk_peri", 0, RK2928_CLKGATE_CON(4), 4, GFLAGS), + GATE(0, "aclk_cpu_peri", "aclk_peri", 0, RK2928_CLKGATE_CON(4), 2, GFLAGS), + GATE(0, "aclk_peri_axi_matrix", "aclk_peri", 0, RK2928_CLKGATE_CON(4), 3, GFLAGS), + + /* pclk_peri gates */ + GATE(0, "pclk_peri_axi_matrix", "pclk_peri", 0, RK2928_CLKGATE_CON(4), 1, GFLAGS), + GATE(PCLK_PWM23, "pclk_pwm23", "pclk_peri", 0, RK2928_CLKGATE_CON(7), 11, GFLAGS), + GATE(PCLK_WDT, "pclk_wdt", "pclk_peri", 0, RK2928_CLKGATE_CON(7), 15, GFLAGS), + GATE(PCLK_SPI0, "pclk_spi0", "pclk_peri", 0, RK2928_CLKGATE_CON(7), 12, GFLAGS), + GATE(PCLK_SPI1, "pclk_spi1", "pclk_peri", 0, RK2928_CLKGATE_CON(7), 13, GFLAGS), + GATE(PCLK_UART2, "pclk_uart2", "pclk_peri", 0, RK2928_CLKGATE_CON(8), 2, GFLAGS), + GATE(PCLK_UART3, "pclk_uart3", "pclk_peri", 0, RK2928_CLKGATE_CON(8), 3, GFLAGS), + GATE(PCLK_I2C2, "pclk_i2c2", "pclk_peri", 0, RK2928_CLKGATE_CON(8), 6, GFLAGS), + GATE(PCLK_I2C3, "pclk_i2c3", "pclk_peri", 0, RK2928_CLKGATE_CON(8), 7, GFLAGS), + GATE(PCLK_I2C4, "pclk_i2c4", "pclk_peri", 0, RK2928_CLKGATE_CON(8), 8, GFLAGS), + GATE(PCLK_GPIO3, "pclk_gpio3", "pclk_peri", 0, RK2928_CLKGATE_CON(8), 12, GFLAGS), + GATE(PCLK_SARADC, "pclk_saradc", "pclk_peri", 0, RK2928_CLKGATE_CON(7), 14, GFLAGS), +}; + +PNAME(mux_rk3066_lcdc0_p) = { "dclk_lcdc0_src", "xin27m" }; +PNAME(mux_rk3066_lcdc1_p) = { "dclk_lcdc1_src", "xin27m" }; +PNAME(mux_sclk_cif1_p) = { "cif1_pre", "xin24m" }; +PNAME(mux_sclk_i2s1_p) = { "i2s1_pre", "i2s1_frac", "xin12m" }; +PNAME(mux_sclk_i2s2_p) = { "i2s2_pre", "i2s2_frac", "xin12m" }; + +static struct clk_div_table div_aclk_cpu_t[] = { + { .val = 0, .div = 1 }, + { .val = 1, .div = 2 }, + { .val = 2, .div = 3 }, + { .val = 3, .div = 4 }, + { .val = 4, .div = 8 }, + { /* sentinel */ }, +}; + +static struct rockchip_clk_branch rk3066a_clk_branches[] __initdata = { + COMPOSITE_NOGATE(0, "armclk", mux_armclk_p, 0, + RK2928_CLKSEL_CON(0), 8, 1, MFLAGS, 0, 5, DFLAGS), + DIVTBL(0, "aclk_cpu_pre", "armclk", 0, + RK2928_CLKSEL_CON(1), 0, 3, DFLAGS, div_aclk_cpu_t), + + GATE(CORE_L2C, "core_l2c", "aclk_cpu", 0, + RK2928_CLKGATE_CON(9), 4, GFLAGS), + + COMPOSITE(0, "aclk_peri_pre", mux_pll_src_gpll_cpll_p, 0, + RK2928_CLKSEL_CON(10), 15, 1, MFLAGS, 0, 5, DFLAGS, + RK2928_CLKGATE_CON(2), 0, GFLAGS), + + COMPOSITE(0, "dclk_lcdc0_src", mux_pll_src_cpll_gpll_p, 0, + RK2928_CLKSEL_CON(27), 0, 1, MFLAGS, 8, 8, DFLAGS, + RK2928_CLKGATE_CON(3), 1, GFLAGS), + MUX(DCLK_LCDC0, "dclk_lcdc0", mux_rk3066_lcdc0_p, 0, + RK2928_CLKSEL_CON(27), 4, 1, MFLAGS), + COMPOSITE(0, "dclk_lcdc1_src", mux_pll_src_cpll_gpll_p, 0, + RK2928_CLKSEL_CON(28), 0, 1, MFLAGS, 8, 8, DFLAGS, + RK2928_CLKGATE_CON(3), 2, GFLAGS), + MUX(DCLK_LCDC1, "dclk_lcdc1", mux_rk3066_lcdc1_p, 0, + RK2928_CLKSEL_CON(28), 4, 1, MFLAGS), + + COMPOSITE_NOMUX(0, "cif1_pre", "cif_src", 0, + RK2928_CLKSEL_CON(29), 8, 5, DFLAGS, + RK2928_CLKGATE_CON(3), 8, GFLAGS), + MUX(SCLK_CIF1, "sclk_cif1", mux_sclk_cif1_p, 0, + RK2928_CLKSEL_CON(29), 15, 1, MFLAGS), + + GATE(0, "pclkin_cif1", "ext_cif1", 0, + RK2928_CLKGATE_CON(3), 4, GFLAGS), + + COMPOSITE(0, "aclk_gpu_src", mux_pll_src_cpll_gpll_p, 0, + RK2928_CLKSEL_CON(33), 8, 1, MFLAGS, 0, 5, DFLAGS, + RK2928_CLKGATE_CON(3), 13, GFLAGS), + GATE(ACLK_GPU, "aclk_gpu", "aclk_gpu_src", 0, + RK2928_CLKGATE_CON(5), 15, GFLAGS), + + GATE(SCLK_TIMER2, "timer2", "xin24m", 0, + RK2928_CLKGATE_CON(3), 2, GFLAGS), + + COMPOSITE_NOMUX(0, "sclk_tsadc", "xin24m", 0, + RK2928_CLKSEL_CON(34), 0, 16, DFLAGS, + RK2928_CLKGATE_CON(2), 15, GFLAGS), + + MUX(0, "i2s_src", mux_pll_src_gpll_cpll_p, 0, + RK2928_CLKSEL_CON(2), 15, 1, MFLAGS), + COMPOSITE_NOMUX(0, "i2s0_pre", "i2s_src", 0, + RK2928_CLKSEL_CON(2), 0, 7, DFLAGS, + RK2928_CLKGATE_CON(0), 7, GFLAGS), + COMPOSITE_FRAC(0, "i2s0_frac", "i2s0_pre", 0, + RK2928_CLKSEL_CON(6), 0, + RK2928_CLKGATE_CON(0), 8, GFLAGS), + MUX(SCLK_I2S0, "sclk_i2s0", mux_sclk_i2s0_p, 0, + RK2928_CLKSEL_CON(2), 8, 2, MFLAGS), + COMPOSITE_NOMUX(0, "i2s1_pre", "i2s_src", 0, + RK2928_CLKSEL_CON(3), 0, 7, DFLAGS, + RK2928_CLKGATE_CON(0), 9, GFLAGS), + COMPOSITE_FRAC(0, "i2s1_frac", "i2s1_pre", 0, + RK2928_CLKSEL_CON(7), 0, + RK2928_CLKGATE_CON(0), 10, GFLAGS), + MUX(SCLK_I2S1, "sclk_i2s1", mux_sclk_i2s1_p, 0, + RK2928_CLKSEL_CON(3), 8, 2, MFLAGS), + COMPOSITE_NOMUX(0, "i2s2_pre", "i2s_src", 0, + RK2928_CLKSEL_CON(4), 0, 7, DFLAGS, + RK2928_CLKGATE_CON(0), 11, GFLAGS), + COMPOSITE_FRAC(0, "i2s2_frac", "i2s2_pre", 0, + RK2928_CLKSEL_CON(8), 0, + RK2928_CLKGATE_CON(0), 12, GFLAGS), + MUX(SCLK_I2S2, "sclk_i2s2", mux_sclk_i2s2_p, 0, + RK2928_CLKSEL_CON(4), 8, 2, MFLAGS), + COMPOSITE_NOMUX(0, "spdif_pre", "i2s_src", 0, + RK2928_CLKSEL_CON(5), 0, 7, DFLAGS, + RK2928_CLKGATE_CON(0), 13, GFLAGS), + COMPOSITE_FRAC(0, "spdif_frac", "spdif_pll", 0, + RK2928_CLKSEL_CON(9), 0, + RK2928_CLKGATE_CON(0), 14, GFLAGS), + MUX(SCLK_SPDIF, "sclk_spdif", mux_sclk_spdif_p, 0, + RK2928_CLKSEL_CON(5), 8, 2, MFLAGS), + + GATE(HCLK_I2S1, "hclk_i2s1", "hclk_cpu", 0, RK2928_CLKGATE_CON(7), 3, GFLAGS), + GATE(HCLK_I2S2, "hclk_i2s2", "hclk_cpu", 0, RK2928_CLKGATE_CON(7), 4, GFLAGS), + GATE(0, "hclk_cif1", "hclk_cpu", 0, RK2928_CLKGATE_CON(6), 6, GFLAGS), + GATE(0, "hclk_hdmi", "hclk_cpu", 0, RK2928_CLKGATE_CON(4), 14, GFLAGS), + + GATE(HCLK_OTG1, "hclk_usbotg1", "hclk_peri", 0, RK2928_CLKGATE_CON(5), 14, GFLAGS), + + GATE(0, "aclk_cif1", "aclk_vio1", 0, RK2928_CLKGATE_CON(6), 7, GFLAGS), + + GATE(PCLK_TIMER1, "pclk_timer1", "pclk_cpu", 0, RK2928_CLKGATE_CON(7), 8, GFLAGS), + GATE(PCLK_TIMER2, "pclk_timer2", "pclk_cpu", 0, RK2928_CLKGATE_CON(7), 9, GFLAGS), + GATE(PCLK_GPIO6, "pclk_gpio6", "pclk_cpu", 0, RK2928_CLKGATE_CON(8), 15, GFLAGS), + GATE(PCLK_UART0, "pclk_uart0", "pclk_cpu", 0, RK2928_CLKGATE_CON(8), 0, GFLAGS), + GATE(PCLK_UART1, "pclk_uart1", "pclk_cpu", 0, RK2928_CLKGATE_CON(8), 1, GFLAGS), + + GATE(PCLK_GPIO4, "pclk_gpio4", "pclk_peri", 0, RK2928_CLKGATE_CON(8), 13, GFLAGS), + GATE(PCLK_TSADC, "pclk_tsadc", "pclk_peri", 0, RK2928_CLKGATE_CON(4), 13, GFLAGS), +}; + +static struct clk_div_table div_rk3188_aclk_core_t[] = { + { .val = 0, .div = 1 }, + { .val = 1, .div = 2 }, + { .val = 2, .div = 3 }, + { .val = 3, .div = 4 }, + { .val = 4, .div = 8 }, + { /* sentinel */ }, +}; + +PNAME(mux_hsicphy_p) = { "sclk_otgphy0", "sclk_otgphy1", + "gpll", "cpll" }; + +static struct rockchip_clk_branch rk3188_clk_branches[] __initdata = { + COMPOSITE_NOGATE(0, "armclk", mux_armclk_p, 0, + RK2928_CLKSEL_CON(0), 8, 1, MFLAGS, 9, 5, DFLAGS), + COMPOSITE_NOMUX_DIVTBL(0, "aclk_core", "armclk", 0, + RK2928_CLKSEL_CON(1), 3, 3, DFLAGS | CLK_DIVIDER_READ_ONLY, + div_rk3188_aclk_core_t, RK2928_CLKGATE_CON(0), 7, GFLAGS), + + /* do not source aclk_cpu_pre from the apll, to keep complexity down */ + COMPOSITE_NOGATE(0, "aclk_cpu_pre", mux_aclk_cpu_p, CLK_SET_RATE_NO_REPARENT, + RK2928_CLKSEL_CON(0), 5, 1, MFLAGS, 0, 5, DFLAGS), + + GATE(CORE_L2C, "core_l2c", "armclk", 0, + RK2928_CLKGATE_CON(9), 4, GFLAGS), + + COMPOSITE(0, "aclk_peri_pre", mux_pll_src_cpll_gpll_p, 0, + RK2928_CLKSEL_CON(10), 15, 1, MFLAGS, 0, 5, DFLAGS, + RK2928_CLKGATE_CON(2), 0, GFLAGS), + + COMPOSITE(DCLK_LCDC0, "dclk_lcdc0", mux_pll_src_cpll_gpll_p, 0, + RK2928_CLKSEL_CON(27), 0, 1, MFLAGS, 8, 8, DFLAGS, + RK2928_CLKGATE_CON(3), 1, GFLAGS), + COMPOSITE(DCLK_LCDC1, "dclk_lcdc1", mux_pll_src_cpll_gpll_p, 0, + RK2928_CLKSEL_CON(28), 0, 1, MFLAGS, 8, 8, DFLAGS, + RK2928_CLKGATE_CON(3), 2, GFLAGS), + + COMPOSITE(0, "aclk_gpu_src", mux_pll_src_cpll_gpll_p, 0, + RK2928_CLKSEL_CON(34), 7, 1, MFLAGS, 0, 5, DFLAGS, + RK2928_CLKGATE_CON(3), 15, GFLAGS), + GATE(ACLK_GPU, "aclk_gpu", "aclk_gpu_src", 0, + RK2928_CLKGATE_CON(9), 7, GFLAGS), + + GATE(SCLK_TIMER2, "timer2", "xin24m", 0, RK2928_CLKGATE_CON(3), 4, GFLAGS), + GATE(SCLK_TIMER3, "timer3", "xin24m", 0, RK2928_CLKGATE_CON(1), 2, GFLAGS), + GATE(SCLK_TIMER4, "timer4", "xin24m", 0, RK2928_CLKGATE_CON(3), 5, GFLAGS), + GATE(SCLK_TIMER5, "timer5", "xin24m", 0, RK2928_CLKGATE_CON(3), 8, GFLAGS), + GATE(SCLK_TIMER6, "timer6", "xin24m", 0, RK2928_CLKGATE_CON(3), 14, GFLAGS), + + COMPOSITE_NODIV(0, "sclk_hsicphy_480m", mux_hsicphy_p, 0, + RK2928_CLKSEL_CON(30), 0, 2, DFLAGS, + RK2928_CLKGATE_CON(3), 6, GFLAGS), + DIV(0, "sclk_hsicphy_12m", "sclk_hsicphy_480m", 0, + RK2928_CLKGATE_CON(11), 8, 6, DFLAGS), + + MUX(0, "i2s_src", mux_pll_src_gpll_cpll_p, 0, + RK2928_CLKSEL_CON(2), 15, 1, MFLAGS), + COMPOSITE_NOMUX(0, "i2s0_pre", "i2s_src", 0, + RK2928_CLKSEL_CON(3), 0, 7, DFLAGS, + RK2928_CLKGATE_CON(0), 9, GFLAGS), + COMPOSITE_FRAC(0, "i2s0_frac", "i2s0_pre", 0, + RK2928_CLKSEL_CON(7), 0, + RK2928_CLKGATE_CON(0), 10, GFLAGS), + MUX(SCLK_I2S0, "sclk_i2s0", mux_sclk_i2s0_p, 0, + RK2928_CLKSEL_CON(3), 8, 2, MFLAGS), + COMPOSITE_NOMUX(0, "spdif_pre", "i2s_src", 0, + RK2928_CLKSEL_CON(5), 0, 7, DFLAGS, + RK2928_CLKGATE_CON(13), 13, GFLAGS), + COMPOSITE_FRAC(0, "spdif_frac", "spdif_pll", 0, + RK2928_CLKSEL_CON(9), 0, + RK2928_CLKGATE_CON(0), 14, GFLAGS), + MUX(SCLK_SPDIF, "sclk_spdif", mux_sclk_spdif_p, 0, + RK2928_CLKSEL_CON(5), 8, 2, MFLAGS), + + GATE(0, "hclk_imem0", "hclk_cpu", 0, RK2928_CLKGATE_CON(4), 14, GFLAGS), + GATE(0, "hclk_imem1", "hclk_cpu", 0, RK2928_CLKGATE_CON(4), 15, GFLAGS), + + GATE(HCLK_OTG1, "hclk_usbotg1", "hclk_peri", 0, RK2928_CLKGATE_CON(7), 3, GFLAGS), + GATE(HCLK_HSIC, "hclk_hsic", "hclk_peri", 0, RK2928_CLKGATE_CON(7), 4, GFLAGS), + + GATE(PCLK_TIMER3, "pclk_timer3", "pclk_cpu", 0, RK2928_CLKGATE_CON(7), 9, GFLAGS), + + GATE(PCLK_UART0, "pclk_uart0", "hclk_ahb2apb", 0, RK2928_CLKGATE_CON(8), 0, GFLAGS), + GATE(PCLK_UART1, "pclk_uart1", "hclk_ahb2apb", 0, RK2928_CLKGATE_CON(8), 1, GFLAGS), + + GATE(ACLK_GPS, "aclk_gps", "aclk_peri", 0, RK2928_CLKGATE_CON(8), 13, GFLAGS), +}; + +static void __init rk3188_common_clk_init(struct device_node *np) +{ + void __iomem *reg_base; + struct clk *clk; + + reg_base = of_iomap(np, 0); + if (!reg_base) { + pr_err("%s: could not map cru region\n", __func__); + return; + } + + rockchip_clk_init(np, reg_base, CLK_NR_CLKS); + + /* xin12m is created by an cru-internal divider */ + clk = clk_register_fixed_factor(NULL, "xin12m", "xin24m", 0, 1, 2); + if (IS_ERR(clk)) + pr_warn("%s: could not register clock xin12m: %ld\n", + __func__, PTR_ERR(clk)); + + clk = clk_register_fixed_factor(NULL, "usb480m", "xin24m", 0, 20, 1); + if (IS_ERR(clk)) + pr_warn("%s: could not register clock usb480m: %ld\n", + __func__, PTR_ERR(clk)); + + rockchip_clk_register_plls(rk3188_pll_clks, + ARRAY_SIZE(rk3188_pll_clks), + RK3188_GRF_SOC_STATUS); + rockchip_clk_register_branches(common_clk_branches, + ARRAY_SIZE(common_clk_branches)); + + rockchip_register_softrst(np, 9, reg_base + RK2928_SOFTRST_CON(0), + ROCKCHIP_SOFTRST_HIWORD_MASK); +} + +static void __init rk3066a_clk_init(struct device_node *np) +{ + rk3188_common_clk_init(np); + rockchip_clk_register_branches(rk3066a_clk_branches, + ARRAY_SIZE(rk3066a_clk_branches)); +} +CLK_OF_DECLARE(rk3066a_cru, "rockchip,rk3066a-cru", rk3066a_clk_init); + +static void __init rk3188a_clk_init(struct device_node *np) +{ + rk3188_common_clk_init(np); + rockchip_clk_register_branches(rk3188_clk_branches, + ARRAY_SIZE(rk3188_clk_branches)); +} +CLK_OF_DECLARE(rk3188a_cru, "rockchip,rk3188a-cru", rk3188a_clk_init); + +static void __init rk3188_clk_init(struct device_node *np) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(rk3188_pll_clks); i++) { + struct rockchip_pll_clock *pll = &rk3188_pll_clks[i]; + struct rockchip_pll_rate_table *rate; + + if (!pll->rate_table) + continue; + + rate = pll->rate_table; + while (rate->rate > 0) { + rate->bwadj = 0; + rate++; + } + } + + rk3188a_clk_init(np); +} +CLK_OF_DECLARE(rk3188_cru, "rockchip,rk3188-cru", rk3188_clk_init); diff --git a/drivers/clk/rockchip/clk-rk3288.c b/drivers/clk/rockchip/clk-rk3288.c new file mode 100644 index 000000000000..0d8c6c59a75e --- /dev/null +++ b/drivers/clk/rockchip/clk-rk3288.c @@ -0,0 +1,717 @@ +/* + * Copyright (c) 2014 MundoReader S.L. + * Author: Heiko Stuebner <heiko@sntech.de> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include <linux/clk-provider.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include <dt-bindings/clock/rk3288-cru.h> +#include "clk.h" + +#define RK3288_GRF_SOC_CON(x) (0x244 + x * 4) +#define RK3288_GRF_SOC_STATUS 0x280 + +enum rk3288_plls { + apll, dpll, cpll, gpll, npll, +}; + +struct rockchip_pll_rate_table rk3288_pll_rates[] = { + RK3066_PLL_RATE(2208000000, 1, 92, 1), + RK3066_PLL_RATE(2184000000, 1, 91, 1), + RK3066_PLL_RATE(2160000000, 1, 90, 1), + RK3066_PLL_RATE(2136000000, 1, 89, 1), + RK3066_PLL_RATE(2112000000, 1, 88, 1), + RK3066_PLL_RATE(2088000000, 1, 87, 1), + RK3066_PLL_RATE(2064000000, 1, 86, 1), + RK3066_PLL_RATE(2040000000, 1, 85, 1), + RK3066_PLL_RATE(2016000000, 1, 84, 1), + RK3066_PLL_RATE(1992000000, 1, 83, 1), + RK3066_PLL_RATE(1968000000, 1, 82, 1), + RK3066_PLL_RATE(1944000000, 1, 81, 1), + RK3066_PLL_RATE(1920000000, 1, 80, 1), + RK3066_PLL_RATE(1896000000, 1, 79, 1), + RK3066_PLL_RATE(1872000000, 1, 78, 1), + RK3066_PLL_RATE(1848000000, 1, 77, 1), + RK3066_PLL_RATE(1824000000, 1, 76, 1), + RK3066_PLL_RATE(1800000000, 1, 75, 1), + RK3066_PLL_RATE(1776000000, 1, 74, 1), + RK3066_PLL_RATE(1752000000, 1, 73, 1), + RK3066_PLL_RATE(1728000000, 1, 72, 1), + RK3066_PLL_RATE(1704000000, 1, 71, 1), + RK3066_PLL_RATE(1680000000, 1, 70, 1), + RK3066_PLL_RATE(1656000000, 1, 69, 1), + RK3066_PLL_RATE(1632000000, 1, 68, 1), + RK3066_PLL_RATE(1608000000, 1, 67, 1), + RK3066_PLL_RATE(1560000000, 1, 65, 1), + RK3066_PLL_RATE(1512000000, 1, 63, 1), + RK3066_PLL_RATE(1488000000, 1, 62, 1), + RK3066_PLL_RATE(1464000000, 1, 61, 1), + RK3066_PLL_RATE(1440000000, 1, 60, 1), + RK3066_PLL_RATE(1416000000, 1, 59, 1), + RK3066_PLL_RATE(1392000000, 1, 58, 1), + RK3066_PLL_RATE(1368000000, 1, 57, 1), + RK3066_PLL_RATE(1344000000, 1, 56, 1), + RK3066_PLL_RATE(1320000000, 1, 55, 1), + RK3066_PLL_RATE(1296000000, 1, 54, 1), + RK3066_PLL_RATE(1272000000, 1, 53, 1), + RK3066_PLL_RATE(1248000000, 1, 52, 1), + RK3066_PLL_RATE(1224000000, 1, 51, 1), + RK3066_PLL_RATE(1200000000, 1, 50, 1), + RK3066_PLL_RATE(1188000000, 2, 99, 1), + RK3066_PLL_RATE(1176000000, 1, 49, 1), + RK3066_PLL_RATE(1128000000, 1, 47, 1), + RK3066_PLL_RATE(1104000000, 1, 46, 1), + RK3066_PLL_RATE(1008000000, 1, 84, 2), + RK3066_PLL_RATE( 912000000, 1, 76, 2), + RK3066_PLL_RATE( 891000000, 8, 594, 2), + RK3066_PLL_RATE( 888000000, 1, 74, 2), + RK3066_PLL_RATE( 816000000, 1, 68, 2), + RK3066_PLL_RATE( 798000000, 2, 133, 2), + RK3066_PLL_RATE( 792000000, 1, 66, 2), + RK3066_PLL_RATE( 768000000, 1, 64, 2), + RK3066_PLL_RATE( 742500000, 8, 495, 2), + RK3066_PLL_RATE( 696000000, 1, 58, 2), + RK3066_PLL_RATE( 600000000, 1, 50, 2), + RK3066_PLL_RATE( 594000000, 2, 198, 4), + RK3066_PLL_RATE( 552000000, 1, 46, 2), + RK3066_PLL_RATE( 504000000, 1, 84, 4), + RK3066_PLL_RATE( 456000000, 1, 76, 4), + RK3066_PLL_RATE( 408000000, 1, 68, 4), + RK3066_PLL_RATE( 384000000, 2, 128, 4), + RK3066_PLL_RATE( 360000000, 1, 60, 4), + RK3066_PLL_RATE( 312000000, 1, 52, 4), + RK3066_PLL_RATE( 300000000, 1, 50, 4), + RK3066_PLL_RATE( 297000000, 2, 198, 8), + RK3066_PLL_RATE( 252000000, 1, 84, 8), + RK3066_PLL_RATE( 216000000, 1, 72, 8), + RK3066_PLL_RATE( 148500000, 2, 99, 8), + RK3066_PLL_RATE( 126000000, 1, 84, 16), + RK3066_PLL_RATE( 48000000, 1, 64, 32), + { /* sentinel */ }, +}; + +PNAME(mux_pll_p) = { "xin24m", "xin32k" }; +PNAME(mux_armclk_p) = { "apll_core", "gpll_core" }; +PNAME(mux_ddrphy_p) = { "dpll_ddr", "gpll_ddr" }; +PNAME(mux_aclk_cpu_src_p) = { "cpll_aclk_cpu", "gpll_aclk_cpu" }; + +PNAME(mux_pll_src_cpll_gpll_p) = { "cpll", "gpll" }; +PNAME(mux_pll_src_npll_cpll_gpll_p) = { "npll", "cpll", "gpll" }; +PNAME(mux_pll_src_cpll_gpll_npll_p) = { "cpll", "gpll", "npll" }; +PNAME(mux_pll_src_cpll_gpll_usb480m_p) = { "cpll", "gpll", "usb480m" }; + +PNAME(mux_mmc_src_p) = { "cpll", "gpll", "xin24m", "xin24m" }; +PNAME(mux_i2s_pre_p) = { "i2s_src", "i2s_frac", "ext_i2s", "xin12m" }; +PNAME(mux_i2s_clkout_p) = { "i2s_pre", "xin12m" }; +PNAME(mux_spdif_p) = { "spdif_pre", "spdif_frac", "xin12m" }; +PNAME(mux_spdif_8ch_p) = { "spdif_8ch_pre", "spdif_8ch_frac", "xin12m" }; +PNAME(mux_uart0_pll_p) = { "cpll", "gpll", "usbphy_480m_src", "npll" }; +PNAME(mux_uart0_p) = { "uart0_src", "uart0_frac", "xin24m" }; +PNAME(mux_uart1_p) = { "uart1_src", "uart1_frac", "xin24m" }; +PNAME(mux_uart2_p) = { "uart2_src", "uart2_frac", "xin24m" }; +PNAME(mux_uart3_p) = { "uart3_src", "uart3_frac", "xin24m" }; +PNAME(mux_uart4_p) = { "uart4_src", "uart4_frac", "xin24m" }; +PNAME(mux_cif_out_p) = { "cif_src", "xin24m" }; +PNAME(mux_macref_p) = { "mac_src", "ext_gmac" }; +PNAME(mux_hsadcout_p) = { "hsadc_src", "ext_hsadc" }; +PNAME(mux_edp_24m_p) = { "ext_edp_24m", "xin24m" }; +PNAME(mux_tspout_p) = { "cpll", "gpll", "npll", "xin27m" }; + +PNAME(mux_usbphy480m_p) = { "sclk_otgphy0", "sclk_otgphy1", + "sclk_otgphy2" }; +PNAME(mux_hsicphy480m_p) = { "cpll", "gpll", "usbphy480m_src" }; +PNAME(mux_hsicphy12m_p) = { "hsicphy12m_xin12m", "hsicphy12m_usbphy" }; + +static struct rockchip_pll_clock rk3288_pll_clks[] __initdata = { + [apll] = PLL(pll_rk3066, PLL_APLL, "apll", mux_pll_p, 0, RK3288_PLL_CON(0), + RK3288_MODE_CON, 0, 6, rk3288_pll_rates), + [dpll] = PLL(pll_rk3066, PLL_DPLL, "dpll", mux_pll_p, 0, RK3288_PLL_CON(4), + RK3288_MODE_CON, 4, 5, NULL), + [cpll] = PLL(pll_rk3066, PLL_CPLL, "cpll", mux_pll_p, 0, RK3288_PLL_CON(8), + RK3288_MODE_CON, 8, 7, rk3288_pll_rates), + [gpll] = PLL(pll_rk3066, PLL_GPLL, "gpll", mux_pll_p, 0, RK3288_PLL_CON(12), + RK3288_MODE_CON, 12, 8, rk3288_pll_rates), + [npll] = PLL(pll_rk3066, PLL_NPLL, "npll", mux_pll_p, 0, RK3288_PLL_CON(16), + RK3288_MODE_CON, 14, 9, NULL), +}; + +static struct clk_div_table div_hclk_cpu_t[] = { + { .val = 0, .div = 1 }, + { .val = 1, .div = 2 }, + { .val = 3, .div = 4 }, + { /* sentinel */}, +}; + +#define MFLAGS CLK_MUX_HIWORD_MASK +#define DFLAGS CLK_DIVIDER_HIWORD_MASK +#define GFLAGS (CLK_GATE_HIWORD_MASK | CLK_GATE_SET_TO_DISABLE) + +static struct rockchip_clk_branch rk3288_clk_branches[] __initdata = { + /* + * Clock-Architecture Diagram 1 + */ + + GATE(0, "apll_core", "apll", 0, + RK3288_CLKGATE_CON(0), 1, GFLAGS), + GATE(0, "gpll_core", "gpll", 0, + RK3288_CLKGATE_CON(0), 2, GFLAGS), + COMPOSITE_NOGATE(0, "armclk", mux_armclk_p, 0, + RK3288_CLKSEL_CON(0), 15, 1, MFLAGS, 8, 5, DFLAGS), + + COMPOSITE_NOMUX(0, "armcore0", "armclk", 0, + RK3288_CLKSEL_CON(36), 0, 3, DFLAGS, + RK3288_CLKGATE_CON(12), 0, GFLAGS), + COMPOSITE_NOMUX(0, "armcore1", "armclk", 0, + RK3288_CLKSEL_CON(36), 4, 3, DFLAGS, + RK3288_CLKGATE_CON(12), 1, GFLAGS), + COMPOSITE_NOMUX(0, "armcore2", "armclk", 0, + RK3288_CLKSEL_CON(36), 8, 3, DFLAGS, + RK3288_CLKGATE_CON(12), 2, GFLAGS), + COMPOSITE_NOMUX(0, "armcore3", "armclk", 0, + RK3288_CLKSEL_CON(36), 12, 3, DFLAGS, + RK3288_CLKGATE_CON(12), 3, GFLAGS), + COMPOSITE_NOMUX(0, "l2ram", "armclk", 0, + RK3288_CLKSEL_CON(37), 0, 3, DFLAGS, + RK3288_CLKGATE_CON(12), 4, GFLAGS), + COMPOSITE_NOMUX(0, "aclk_core_m0", "armclk", 0, + RK3288_CLKSEL_CON(0), 0, 4, DFLAGS, + RK3288_CLKGATE_CON(12), 5, GFLAGS), + COMPOSITE_NOMUX(0, "aclk_core_mp", "armclk", 0, + RK3288_CLKSEL_CON(0), 4, 4, DFLAGS, + RK3288_CLKGATE_CON(12), 6, GFLAGS), + COMPOSITE_NOMUX(0, "atclk", "armclk", 0, + RK3288_CLKSEL_CON(37), 4, 5, DFLAGS, + RK3288_CLKGATE_CON(12), 7, GFLAGS), + COMPOSITE_NOMUX(0, "pclk_dbg_pre", "armclk", 0, + RK3288_CLKSEL_CON(37), 9, 5, DFLAGS, + RK3288_CLKGATE_CON(12), 8, GFLAGS), + GATE(0, "pclk_dbg", "pclk_dbg_pre", 0, + RK3288_CLKGATE_CON(12), 9, GFLAGS), + GATE(0, "cs_dbg", "pclk_dbg_pre", 0, + RK3288_CLKGATE_CON(12), 10, GFLAGS), + GATE(0, "pclk_core_niu", "pclk_dbg_pre", 0, + RK3288_CLKGATE_CON(12), 11, GFLAGS), + + GATE(0, "dpll_ddr", "dpll", 0, + RK3288_CLKGATE_CON(0), 8, GFLAGS), + GATE(0, "gpll_ddr", "gpll", 0, + RK3288_CLKGATE_CON(0), 9, GFLAGS), + COMPOSITE_NOGATE(0, "ddrphy", mux_ddrphy_p, 0, + RK3288_CLKSEL_CON(26), 2, 1, MFLAGS, 0, 2, + DFLAGS | CLK_DIVIDER_POWER_OF_TWO), + + GATE(0, "gpll_aclk_cpu", "gpll", 0, + RK3288_CLKGATE_CON(0), 10, GFLAGS), + GATE(0, "cpll_aclk_cpu", "cpll", 0, + RK3288_CLKGATE_CON(0), 11, GFLAGS), + COMPOSITE_NOGATE(0, "aclk_cpu_src", mux_aclk_cpu_src_p, 0, + RK3288_CLKSEL_CON(1), 15, 1, MFLAGS, 3, 5, DFLAGS), + DIV(0, "aclk_cpu_pre", "aclk_cpu_src", 0, + RK3288_CLKSEL_CON(1), 0, 3, DFLAGS), + GATE(0, "aclk_cpu", "aclk_cpu_pre", 0, + RK3288_CLKGATE_CON(0), 3, GFLAGS), + COMPOSITE_NOMUX(0, "pclk_cpu", "aclk_cpu_pre", 0, + RK3288_CLKSEL_CON(1), 12, 3, DFLAGS, + RK3288_CLKGATE_CON(0), 5, GFLAGS), + COMPOSITE_NOMUX_DIVTBL(0, "hclk_cpu", "aclk_cpu_pre", 0, + RK3288_CLKSEL_CON(1), 8, 2, DFLAGS, div_hclk_cpu_t, + RK3288_CLKGATE_CON(0), 4, GFLAGS), + GATE(0, "c2c_host", "aclk_cpu_src", 0, + RK3288_CLKGATE_CON(13), 8, GFLAGS), + COMPOSITE_NOMUX(0, "crypto", "aclk_cpu_pre", 0, + RK3288_CLKSEL_CON(26), 6, 2, DFLAGS, + RK3288_CLKGATE_CON(5), 4, GFLAGS), + GATE(0, "aclk_bus_2pmu", "aclk_cpu_pre", 0, + RK3288_CLKGATE_CON(0), 7, GFLAGS), + + COMPOSITE(0, "i2s_src", mux_pll_src_cpll_gpll_p, 0, + RK3288_CLKSEL_CON(4), 15, 1, MFLAGS, 0, 7, DFLAGS, + RK3288_CLKGATE_CON(4), 1, GFLAGS), + COMPOSITE_FRAC(0, "i2s_frac", "i2s_src", 0, + RK3288_CLKSEL_CON(8), 0, + RK3288_CLKGATE_CON(4), 2, GFLAGS), + MUX(0, "i2s_pre", mux_i2s_pre_p, 0, + RK3288_CLKSEL_CON(4), 8, 2, MFLAGS), + COMPOSITE_NODIV(0, "i2s0_clkout", mux_i2s_clkout_p, 0, + RK3288_CLKSEL_CON(4), 12, 1, MFLAGS, + RK3288_CLKGATE_CON(4), 0, GFLAGS), + GATE(SCLK_I2S0, "sclk_i2s0", "i2s_pre", 0, + RK3288_CLKGATE_CON(4), 3, GFLAGS), + + MUX(0, "spdif_src", mux_pll_src_cpll_gpll_p, 0, + RK3288_CLKSEL_CON(5), 15, 1, MFLAGS), + COMPOSITE_NOMUX(0, "spdif_pre", "spdif_src", 0, + RK3288_CLKSEL_CON(5), 0, 7, DFLAGS, + RK3288_CLKGATE_CON(4), 4, GFLAGS), + COMPOSITE_FRAC(0, "spdif_frac", "spdif_src", 0, + RK3288_CLKSEL_CON(9), 0, + RK3288_CLKGATE_CON(4), 5, GFLAGS), + COMPOSITE_NODIV(SCLK_SPDIF, "sclk_spdif", mux_spdif_p, 0, + RK3288_CLKSEL_CON(5), 8, 2, MFLAGS, + RK3288_CLKGATE_CON(4), 6, GFLAGS), + COMPOSITE_NOMUX(0, "spdif_8ch_pre", "spdif_src", 0, + RK3288_CLKSEL_CON(40), 0, 7, DFLAGS, + RK3288_CLKGATE_CON(4), 7, GFLAGS), + COMPOSITE_FRAC(0, "spdif_8ch_frac", "spdif_8ch_src", 0, + RK3288_CLKSEL_CON(41), 0, + RK3288_CLKGATE_CON(4), 8, GFLAGS), + COMPOSITE_NODIV(SCLK_SPDIF8CH, "sclk_spdif_8ch", mux_spdif_8ch_p, 0, + RK3288_CLKSEL_CON(40), 8, 2, MFLAGS, + RK3288_CLKGATE_CON(4), 9, GFLAGS), + + GATE(0, "sclk_acc_efuse", "xin24m", 0, + RK3288_CLKGATE_CON(0), 12, GFLAGS), + + GATE(SCLK_TIMER0, "sclk_timer0", "xin24m", 0, + RK3288_CLKGATE_CON(1), 0, GFLAGS), + GATE(SCLK_TIMER1, "sclk_timer1", "xin24m", 0, + RK3288_CLKGATE_CON(1), 1, GFLAGS), + GATE(SCLK_TIMER2, "sclk_timer2", "xin24m", 0, + RK3288_CLKGATE_CON(1), 2, GFLAGS), + GATE(SCLK_TIMER3, "sclk_timer3", "xin24m", 0, + RK3288_CLKGATE_CON(1), 3, GFLAGS), + GATE(SCLK_TIMER4, "sclk_timer4", "xin24m", 0, + RK3288_CLKGATE_CON(1), 4, GFLAGS), + GATE(SCLK_TIMER5, "sclk_timer5", "xin24m", 0, + RK3288_CLKGATE_CON(1), 5, GFLAGS), + + /* + * Clock-Architecture Diagram 2 + */ + + COMPOSITE(0, "aclk_vepu", mux_pll_src_cpll_gpll_usb480m_p, 0, + RK3288_CLKSEL_CON(32), 6, 2, MFLAGS, 0, 5, DFLAGS, + RK3288_CLKGATE_CON(3), 9, GFLAGS), + COMPOSITE(0, "aclk_vdpu", mux_pll_src_cpll_gpll_usb480m_p, 0, + RK3288_CLKSEL_CON(32), 14, 2, MFLAGS, 8, 5, DFLAGS, + RK3288_CLKGATE_CON(3), 11, GFLAGS), + + COMPOSITE(0, "aclk_vio0", mux_pll_src_cpll_gpll_usb480m_p, 0, + RK3288_CLKSEL_CON(31), 6, 2, MFLAGS, 0, 5, DFLAGS, + RK3288_CLKGATE_CON(3), 0, GFLAGS), + DIV(0, "hclk_vio", "aclk_vio0", 0, + RK3288_CLKSEL_CON(28), 8, 5, DFLAGS), + COMPOSITE(0, "aclk_vio1", mux_pll_src_cpll_gpll_usb480m_p, 0, + RK3288_CLKSEL_CON(31), 14, 2, MFLAGS, 8, 5, DFLAGS, + RK3288_CLKGATE_CON(3), 2, GFLAGS), + + COMPOSITE(0, "aclk_rga_pre", mux_pll_src_cpll_gpll_usb480m_p, 0, + RK3288_CLKSEL_CON(30), 6, 2, MFLAGS, 0, 5, DFLAGS, + RK3288_CLKGATE_CON(3), 5, GFLAGS), + COMPOSITE(0, "sclk_rga", mux_pll_src_cpll_gpll_usb480m_p, 0, + RK3288_CLKSEL_CON(30), 14, 2, MFLAGS, 8, 5, DFLAGS, + RK3288_CLKGATE_CON(3), 4, GFLAGS), + + COMPOSITE(DCLK_VOP0, "dclk_vop0", mux_pll_src_cpll_gpll_npll_p, 0, + RK3288_CLKSEL_CON(27), 0, 2, MFLAGS, 8, 8, DFLAGS, + RK3288_CLKGATE_CON(3), 1, GFLAGS), + COMPOSITE(DCLK_VOP1, "dclk_vop1", mux_pll_src_cpll_gpll_npll_p, 0, + RK3288_CLKSEL_CON(29), 6, 2, MFLAGS, 8, 8, DFLAGS, + RK3288_CLKGATE_CON(3), 3, GFLAGS), + + COMPOSITE_NODIV(0, "sclk_edp_24m", mux_edp_24m_p, 0, + RK3288_CLKSEL_CON(28), 15, 1, MFLAGS, + RK3288_CLKGATE_CON(3), 12, GFLAGS), + COMPOSITE(0, "sclk_edp", mux_pll_src_cpll_gpll_npll_p, 0, + RK3288_CLKSEL_CON(28), 6, 2, MFLAGS, 0, 6, DFLAGS, + RK3288_CLKGATE_CON(3), 13, GFLAGS), + + COMPOSITE(0, "sclk_isp", mux_pll_src_cpll_gpll_npll_p, 0, + RK3288_CLKSEL_CON(6), 6, 2, MFLAGS, 0, 6, DFLAGS, + RK3288_CLKGATE_CON(3), 14, GFLAGS), + COMPOSITE(0, "sclk_isp_jpe", mux_pll_src_cpll_gpll_npll_p, 0, + RK3288_CLKSEL_CON(6), 14, 2, MFLAGS, 8, 6, DFLAGS, + RK3288_CLKGATE_CON(3), 15, GFLAGS), + + GATE(0, "sclk_hdmi_hdcp", "xin24m", 0, + RK3288_CLKGATE_CON(5), 12, GFLAGS), + GATE(0, "sclk_hdmi_cec", "xin32k", 0, + RK3288_CLKGATE_CON(5), 11, GFLAGS), + + COMPOSITE(0, "aclk_hevc", mux_pll_src_cpll_gpll_npll_p, 0, + RK3288_CLKSEL_CON(39), 14, 2, MFLAGS, 8, 5, DFLAGS, + RK3288_CLKGATE_CON(13), 13, GFLAGS), + DIV(0, "hclk_hevc", "aclk_hevc", 0, + RK3288_CLKSEL_CON(40), 12, 2, DFLAGS), + + COMPOSITE(0, "sclk_hevc_cabac", mux_pll_src_cpll_gpll_npll_p, 0, + RK3288_CLKSEL_CON(42), 6, 2, MFLAGS, 0, 5, DFLAGS, + RK3288_CLKGATE_CON(13), 14, GFLAGS), + COMPOSITE(0, "sclk_hevc_core", mux_pll_src_cpll_gpll_npll_p, 0, + RK3288_CLKSEL_CON(42), 14, 2, MFLAGS, 8, 5, DFLAGS, + RK3288_CLKGATE_CON(13), 15, GFLAGS), + + COMPOSITE_NODIV(0, "vip_src", mux_pll_src_cpll_gpll_p, 0, + RK3288_CLKSEL_CON(26), 8, 1, MFLAGS, + RK3288_CLKGATE_CON(3), 7, GFLAGS), + COMPOSITE_NOGATE(0, "sclk_vip_out", mux_cif_out_p, 0, + RK3288_CLKSEL_CON(26), 15, 1, MFLAGS, 9, 5, DFLAGS), + + DIV(0, "pclk_pd_alive", "gpll", 0, + RK3288_CLKSEL_CON(33), 8, 5, DFLAGS), + COMPOSITE_NOMUX(0, "pclk_pd_pmu", "gpll", 0, + RK3288_CLKSEL_CON(33), 0, 5, DFLAGS, + RK3288_CLKGATE_CON(5), 8, GFLAGS), + + COMPOSITE(SCLK_GPU, "sclk_gpu", mux_pll_src_cpll_gpll_usb480m_p, 0, + RK3288_CLKSEL_CON(34), 6, 2, MFLAGS, 0, 5, DFLAGS, + RK3288_CLKGATE_CON(5), 7, GFLAGS), + + COMPOSITE(0, "aclk_peri_src", mux_pll_src_cpll_gpll_p, 0, + RK3288_CLKSEL_CON(10), 15, 1, MFLAGS, 0, 5, DFLAGS, + RK3288_CLKGATE_CON(2), 0, GFLAGS), + COMPOSITE_NOMUX(0, "pclk_peri", "aclk_peri_src", 0, + RK3288_CLKSEL_CON(10), 12, 2, DFLAGS | CLK_DIVIDER_POWER_OF_TWO, + RK3288_CLKGATE_CON(2), 3, GFLAGS), + COMPOSITE_NOMUX(0, "hclk_peri", "aclk_peri_src", 0, + RK3288_CLKSEL_CON(10), 8, 2, DFLAGS | CLK_DIVIDER_POWER_OF_TWO, + RK3288_CLKGATE_CON(2), 2, GFLAGS), + GATE(0, "aclk_peri", "aclk_peri_src", 0, + RK3288_CLKGATE_CON(2), 1, GFLAGS), + + /* + * Clock-Architecture Diagram 3 + */ + + COMPOSITE(SCLK_SPI0, "sclk_spi0", mux_pll_src_cpll_gpll_p, 0, + RK3288_CLKSEL_CON(25), 7, 1, MFLAGS, 0, 7, DFLAGS, + RK3288_CLKGATE_CON(2), 9, GFLAGS), + COMPOSITE(SCLK_SPI1, "sclk_spi1", mux_pll_src_cpll_gpll_p, 0, + RK3288_CLKSEL_CON(25), 15, 1, MFLAGS, 8, 7, DFLAGS, + RK3288_CLKGATE_CON(2), 10, GFLAGS), + COMPOSITE(SCLK_SPI2, "sclk_spi2", mux_pll_src_cpll_gpll_p, 0, + RK3288_CLKSEL_CON(39), 7, 1, MFLAGS, 0, 7, DFLAGS, + RK3288_CLKGATE_CON(2), 11, GFLAGS), + + COMPOSITE(SCLK_SDMMC, "sclk_sdmmc", mux_mmc_src_p, 0, + RK3288_CLKSEL_CON(11), 6, 2, MFLAGS, 0, 6, DFLAGS, + RK3288_CLKGATE_CON(13), 0, GFLAGS), + COMPOSITE(SCLK_SDIO0, "sclk_sdio0", mux_mmc_src_p, 0, + RK3288_CLKSEL_CON(12), 6, 2, MFLAGS, 0, 6, DFLAGS, + RK3288_CLKGATE_CON(13), 1, GFLAGS), + COMPOSITE(SCLK_SDIO1, "sclk_sdio1", mux_mmc_src_p, 0, + RK3288_CLKSEL_CON(34), 14, 2, MFLAGS, 8, 6, DFLAGS, + RK3288_CLKGATE_CON(13), 2, GFLAGS), + COMPOSITE(SCLK_EMMC, "sclk_emmc", mux_mmc_src_p, 0, + RK3288_CLKSEL_CON(12), 14, 2, MFLAGS, 8, 6, DFLAGS, + RK3288_CLKGATE_CON(13), 3, GFLAGS), + + COMPOSITE(0, "sclk_tspout", mux_tspout_p, 0, + RK3288_CLKSEL_CON(35), 14, 2, MFLAGS, 8, 5, DFLAGS, + RK3288_CLKGATE_CON(4), 11, GFLAGS), + COMPOSITE(0, "sclk_tsp", mux_pll_src_cpll_gpll_npll_p, 0, + RK3288_CLKSEL_CON(35), 6, 2, MFLAGS, 0, 5, DFLAGS, + RK3288_CLKGATE_CON(4), 10, GFLAGS), + + GATE(SCLK_OTGPHY0, "sclk_otgphy0", "usb480m", 0, + RK3288_CLKGATE_CON(13), 4, GFLAGS), + GATE(SCLK_OTGPHY1, "sclk_otgphy1", "usb480m", 0, + RK3288_CLKGATE_CON(13), 5, GFLAGS), + GATE(SCLK_OTGPHY2, "sclk_otgphy2", "usb480m", 0, + RK3288_CLKGATE_CON(13), 6, GFLAGS), + GATE(SCLK_OTG_ADP, "sclk_otg_adp", "xin32k", 0, + RK3288_CLKGATE_CON(13), 7, GFLAGS), + + COMPOSITE_NOMUX(SCLK_TSADC, "sclk_tsadc", "xin32k", 0, + RK3288_CLKSEL_CON(2), 0, 6, DFLAGS, + RK3288_CLKGATE_CON(2), 7, GFLAGS), + + COMPOSITE_NOMUX(SCLK_SARADC, "sclk_saradc", "xin24m", 0, + RK3288_CLKSEL_CON(24), 8, 8, DFLAGS, + RK3288_CLKGATE_CON(2), 8, GFLAGS), + + GATE(SCLK_PS2C, "sclk_ps2c", "xin24m", 0, + RK3288_CLKGATE_CON(5), 13, GFLAGS), + + COMPOSITE(SCLK_NANDC0, "sclk_nandc0", mux_pll_src_cpll_gpll_p, 0, + RK3288_CLKSEL_CON(38), 7, 1, MFLAGS, 0, 5, DFLAGS, + RK3288_CLKGATE_CON(5), 5, GFLAGS), + COMPOSITE(SCLK_NANDC1, "sclk_nandc1", mux_pll_src_cpll_gpll_p, 0, + RK3288_CLKSEL_CON(38), 15, 1, MFLAGS, 8, 5, DFLAGS, + RK3288_CLKGATE_CON(5), 6, GFLAGS), + + COMPOSITE(0, "uart0_src", mux_uart0_pll_p, 0, + RK3288_CLKSEL_CON(13), 13, 2, MFLAGS, 0, 7, DFLAGS, + RK3288_CLKGATE_CON(1), 8, GFLAGS), + COMPOSITE_FRAC(0, "uart0_frac", "uart0_src", 0, + RK3288_CLKSEL_CON(17), 0, + RK3288_CLKGATE_CON(1), 9, GFLAGS), + MUX(SCLK_UART0, "sclk_uart0", mux_uart0_p, 0, + RK3288_CLKSEL_CON(13), 8, 2, MFLAGS), + MUX(0, "uart_src", mux_pll_src_cpll_gpll_p, 0, + RK3288_CLKSEL_CON(13), 15, 1, MFLAGS), + COMPOSITE_NOMUX(0, "uart1_src", "uart_src", 0, + RK3288_CLKSEL_CON(14), 0, 7, DFLAGS, + RK3288_CLKGATE_CON(1), 10, GFLAGS), + COMPOSITE_FRAC(0, "uart1_frac", "uart1_src", 0, + RK3288_CLKSEL_CON(18), 0, + RK3288_CLKGATE_CON(1), 11, GFLAGS), + MUX(SCLK_UART1, "sclk_uart1", mux_uart1_p, 0, + RK3288_CLKSEL_CON(14), 8, 2, MFLAGS), + COMPOSITE_NOMUX(0, "uart2_src", "uart_src", 0, + RK3288_CLKSEL_CON(15), 0, 7, DFLAGS, + RK3288_CLKGATE_CON(1), 12, GFLAGS), + COMPOSITE_FRAC(0, "uart2_frac", "uart2_src", 0, + RK3288_CLKSEL_CON(19), 0, + RK3288_CLKGATE_CON(1), 13, GFLAGS), + MUX(SCLK_UART2, "sclk_uart2", mux_uart2_p, 0, + RK3288_CLKSEL_CON(15), 8, 2, MFLAGS), + COMPOSITE_NOMUX(0, "uart3_src", "uart_src", 0, + RK3288_CLKSEL_CON(16), 0, 7, DFLAGS, + RK3288_CLKGATE_CON(1), 14, GFLAGS), + COMPOSITE_FRAC(0, "uart3_frac", "uart3_src", 0, + RK3288_CLKSEL_CON(20), 0, + RK3288_CLKGATE_CON(1), 15, GFLAGS), + MUX(SCLK_UART3, "sclk_uart3", mux_uart3_p, 0, + RK3288_CLKSEL_CON(16), 8, 2, MFLAGS), + COMPOSITE_NOMUX(0, "uart4_src", "uart_src", 0, + RK3288_CLKSEL_CON(3), 0, 7, DFLAGS, + RK3288_CLKGATE_CON(2), 12, GFLAGS), + COMPOSITE_FRAC(0, "uart4_frac", "uart4_src", 0, + RK3288_CLKSEL_CON(7), 0, + RK3288_CLKGATE_CON(2), 13, GFLAGS), + MUX(SCLK_UART4, "sclk_uart4", mux_uart4_p, 0, + RK3288_CLKSEL_CON(3), 8, 2, MFLAGS), + + COMPOSITE(0, "mac_src", mux_pll_src_npll_cpll_gpll_p, 0, + RK3288_CLKSEL_CON(21), 0, 2, MFLAGS, 8, 5, DFLAGS, + RK3288_CLKGATE_CON(2), 5, GFLAGS), + MUX(0, "macref", mux_macref_p, 0, + RK3288_CLKSEL_CON(21), 4, 1, MFLAGS), + GATE(0, "sclk_macref_out", "macref", 0, + RK3288_CLKGATE_CON(5), 3, GFLAGS), + GATE(SCLK_MACREF, "sclk_macref", "macref", 0, + RK3288_CLKGATE_CON(5), 2, GFLAGS), + GATE(SCLK_MAC_RX, "sclk_mac_rx", "macref", 0, + RK3288_CLKGATE_CON(5), 0, GFLAGS), + GATE(SCLK_MAC_TX, "sclk_mac_tx", "macref", 0, + RK3288_CLKGATE_CON(5), 1, GFLAGS), + + COMPOSITE(0, "hsadc_src", mux_pll_src_cpll_gpll_p, 0, + RK3288_CLKSEL_CON(22), 0, 1, MFLAGS, 8, 8, DFLAGS, + RK3288_CLKGATE_CON(2), 6, GFLAGS), + MUX(SCLK_HSADC, "sclk_hsadc_out", mux_hsadcout_p, 0, + RK3288_CLKSEL_CON(22), 4, 1, MFLAGS), + + GATE(0, "jtag", "ext_jtag", 0, + RK3288_CLKGATE_CON(4), 14, GFLAGS), + + COMPOSITE_NODIV(0, "usbphy480m_src", mux_usbphy480m_p, 0, + RK3288_CLKSEL_CON(13), 11, 2, MFLAGS, + RK3288_CLKGATE_CON(5), 15, GFLAGS), + COMPOSITE_NODIV(SCLK_HSICPHY480M, "sclk_hsicphy480m", mux_hsicphy480m_p, 0, + RK3288_CLKSEL_CON(29), 0, 2, MFLAGS, + RK3288_CLKGATE_CON(3), 6, GFLAGS), + GATE(0, "hsicphy12m_xin12m", "xin12m", 0, + RK3288_CLKGATE_CON(13), 9, GFLAGS), + DIV(0, "hsicphy12m_usbphy", "sclk_hsicphy480m", 0, + RK3288_CLKSEL_CON(11), 8, 6, DFLAGS), + MUX(SCLK_HSICPHY12M, "sclk_hsicphy12m", mux_hsicphy12m_p, 0, + RK3288_CLKSEL_CON(22), 4, 1, MFLAGS), + + /* + * Clock-Architecture Diagram 4 + */ + + /* aclk_cpu gates */ + GATE(0, "sclk_intmem0", "aclk_cpu", 0, RK3288_CLKGATE_CON(10), 5, GFLAGS), + GATE(0, "sclk_intmem1", "aclk_cpu", 0, RK3288_CLKGATE_CON(10), 6, GFLAGS), + GATE(0, "sclk_intmem2", "aclk_cpu", 0, RK3288_CLKGATE_CON(10), 7, GFLAGS), + GATE(ACLK_DMAC1, "aclk_dmac1", "aclk_cpu", 0, RK3288_CLKGATE_CON(10), 12, GFLAGS), + GATE(0, "aclk_strc_sys", "aclk_cpu", 0, RK3288_CLKGATE_CON(10), 13, GFLAGS), + GATE(0, "aclk_intmem", "aclk_cpu", 0, RK3288_CLKGATE_CON(10), 4, GFLAGS), + GATE(ACLK_CRYPTO, "aclk_crypto", "aclk_cpu", 0, RK3288_CLKGATE_CON(11), 6, GFLAGS), + GATE(0, "aclk_ccp", "aclk_cpu", 0, RK3288_CLKGATE_CON(11), 8, GFLAGS), + + /* hclk_cpu gates */ + GATE(HCLK_CRYPTO, "hclk_crypto", "hclk_cpu", 0, RK3288_CLKGATE_CON(11), 7, GFLAGS), + GATE(HCLK_I2S0, "hclk_i2s0", "hclk_cpu", 0, RK3288_CLKGATE_CON(10), 8, GFLAGS), + GATE(HCLK_ROM, "hclk_rom", "hclk_cpu", 0, RK3288_CLKGATE_CON(10), 9, GFLAGS), + GATE(HCLK_SPDIF, "hclk_spdif", "hclk_cpu", 0, RK3288_CLKGATE_CON(10), 10, GFLAGS), + GATE(HCLK_SPDIF8CH, "hclk_spdif_8ch", "hclk_cpu", 0, RK3288_CLKGATE_CON(10), 11, GFLAGS), + + /* pclk_cpu gates */ + GATE(PCLK_PWM, "pclk_pwm", "pclk_cpu", 0, RK3288_CLKGATE_CON(10), 0, GFLAGS), + GATE(PCLK_TIMER, "pclk_timer", "pclk_cpu", 0, RK3288_CLKGATE_CON(10), 1, GFLAGS), + GATE(PCLK_I2C0, "pclk_i2c0", "pclk_cpu", 0, RK3288_CLKGATE_CON(10), 2, GFLAGS), + GATE(PCLK_I2C1, "pclk_i2c1", "pclk_cpu", 0, RK3288_CLKGATE_CON(10), 3, GFLAGS), + GATE(0, "pclk_ddrupctl0", "pclk_cpu", 0, RK3288_CLKGATE_CON(10), 14, GFLAGS), + GATE(0, "pclk_publ0", "pclk_cpu", 0, RK3288_CLKGATE_CON(10), 15, GFLAGS), + GATE(0, "pclk_ddrupctl1", "pclk_cpu", 0, RK3288_CLKGATE_CON(11), 0, GFLAGS), + GATE(0, "pclk_publ1", "pclk_cpu", 0, RK3288_CLKGATE_CON(11), 1, GFLAGS), + GATE(0, "pclk_efuse_1024", "pclk_cpu", 0, RK3288_CLKGATE_CON(11), 2, GFLAGS), + GATE(PCLK_TZPC, "pclk_tzpc", "pclk_cpu", 0, RK3288_CLKGATE_CON(11), 3, GFLAGS), + GATE(PCLK_UART2, "pclk_uart2", "pclk_cpu", 0, RK3288_CLKGATE_CON(11), 9, GFLAGS), + GATE(0, "pclk_efuse_256", "pclk_cpu", 0, RK3288_CLKGATE_CON(11), 10, GFLAGS), + GATE(PCLK_RKPWM, "pclk_rkpwm", "pclk_cpu", 0, RK3288_CLKGATE_CON(11), 11, GFLAGS), + + /* ddrctrl [DDR Controller PHY clock] gates */ + GATE(0, "nclk_ddrupctl0", "ddrphy", 0, RK3288_CLKGATE_CON(11), 4, GFLAGS), + GATE(0, "nclk_ddrupctl1", "ddrphy", 0, RK3288_CLKGATE_CON(11), 5, GFLAGS), + + /* ddrphy gates */ + GATE(0, "sclk_ddrphy0", "ddrphy", 0, RK3288_CLKGATE_CON(4), 12, GFLAGS), + GATE(0, "sclk_ddrphy1", "ddrphy", 0, RK3288_CLKGATE_CON(4), 13, GFLAGS), + + /* aclk_peri gates */ + GATE(0, "aclk_peri_axi_matrix", "aclk_peri", 0, RK3288_CLKGATE_CON(6), 2, GFLAGS), + GATE(ACLK_DMAC2, "aclk_dmac2", "aclk_peri", 0, RK3288_CLKGATE_CON(6), 3, GFLAGS), + GATE(0, "aclk_peri_niu", "aclk_peri", 0, RK3288_CLKGATE_CON(7), 11, GFLAGS), + GATE(ACLK_MMU, "aclk_mmu", "aclk_peri", 0, RK3288_CLKGATE_CON(8), 12, GFLAGS), + GATE(ACLK_GMAC, "aclk_gmac", "aclk_peri", 0, RK3288_CLKGATE_CON(8), 0, GFLAGS), + GATE(HCLK_GPS, "hclk_gps", "aclk_peri", 0, RK3288_CLKGATE_CON(8), 2, GFLAGS), + + /* hclk_peri gates */ + GATE(0, "hclk_peri_matrix", "hclk_peri", 0, RK3288_CLKGATE_CON(6), 0, GFLAGS), + GATE(HCLK_OTG0, "hclk_otg0", "hclk_peri", 0, RK3288_CLKGATE_CON(7), 4, GFLAGS), + GATE(HCLK_USBHOST0, "hclk_host0", "hclk_peri", 0, RK3288_CLKGATE_CON(7), 6, GFLAGS), + GATE(HCLK_USBHOST1, "hclk_host1", "hclk_peri", 0, RK3288_CLKGATE_CON(7), 7, GFLAGS), + GATE(HCLK_HSIC, "hclk_hsic", "hclk_peri", 0, RK3288_CLKGATE_CON(7), 8, GFLAGS), + GATE(0, "hclk_usb_peri", "hclk_peri", 0, RK3288_CLKGATE_CON(7), 9, GFLAGS), + GATE(0, "hclk_peri_ahb_arbi", "hclk_peri", 0, RK3288_CLKGATE_CON(7), 10, GFLAGS), + GATE(0, "hclk_emem", "hclk_peri", 0, RK3288_CLKGATE_CON(7), 12, GFLAGS), + GATE(0, "hclk_mem", "hclk_peri", 0, RK3288_CLKGATE_CON(7), 13, GFLAGS), + GATE(HCLK_NANDC0, "hclk_nandc0", "hclk_peri", 0, RK3288_CLKGATE_CON(7), 14, GFLAGS), + GATE(HCLK_NANDC1, "hclk_nandc1", "hclk_peri", 0, RK3288_CLKGATE_CON(7), 15, GFLAGS), + GATE(HCLK_TSP, "hclk_tsp", "hclk_peri", 0, RK3288_CLKGATE_CON(8), 8, GFLAGS), + GATE(HCLK_SDMMC, "hclk_sdmmc", "hclk_peri", 0, RK3288_CLKGATE_CON(8), 3, GFLAGS), + GATE(HCLK_SDIO0, "hclk_sdio0", "hclk_peri", 0, RK3288_CLKGATE_CON(8), 4, GFLAGS), + GATE(HCLK_SDIO1, "hclk_sdio1", "hclk_peri", 0, RK3288_CLKGATE_CON(8), 5, GFLAGS), + GATE(HCLK_EMMC, "hclk_emmc", "hclk_peri", 0, RK3288_CLKGATE_CON(8), 6, GFLAGS), + GATE(HCLK_HSADC, "hclk_hsadc", "hclk_peri", 0, RK3288_CLKGATE_CON(8), 7, GFLAGS), + GATE(0, "pmu_hclk_otg0", "hclk_peri", 0, RK3288_CLKGATE_CON(7), 5, GFLAGS), + + /* pclk_peri gates */ + GATE(0, "pclk_peri_matrix", "pclk_peri", 0, RK3288_CLKGATE_CON(6), 1, GFLAGS), + GATE(PCLK_SPI0, "pclk_spi0", "pclk_peri", 0, RK3288_CLKGATE_CON(6), 4, GFLAGS), + GATE(PCLK_SPI1, "pclk_spi1", "pclk_peri", 0, RK3288_CLKGATE_CON(6), 5, GFLAGS), + GATE(PCLK_SPI2, "pclk_spi2", "pclk_peri", 0, RK3288_CLKGATE_CON(6), 6, GFLAGS), + GATE(PCLK_PS2C, "pclk_ps2c", "pclk_peri", 0, RK3288_CLKGATE_CON(6), 7, GFLAGS), + GATE(PCLK_UART0, "pclk_uart0", "pclk_peri", 0, RK3288_CLKGATE_CON(6), 8, GFLAGS), + GATE(PCLK_UART1, "pclk_uart1", "pclk_peri", 0, RK3288_CLKGATE_CON(6), 9, GFLAGS), + GATE(PCLK_I2C4, "pclk_i2c4", "pclk_peri", 0, RK3288_CLKGATE_CON(6), 15, GFLAGS), + GATE(PCLK_UART3, "pclk_uart3", "pclk_peri", 0, RK3288_CLKGATE_CON(6), 11, GFLAGS), + GATE(PCLK_UART4, "pclk_uart4", "pclk_peri", 0, RK3288_CLKGATE_CON(6), 12, GFLAGS), + GATE(PCLK_I2C2, "pclk_i2c2", "pclk_peri", 0, RK3288_CLKGATE_CON(6), 13, GFLAGS), + GATE(PCLK_I2C3, "pclk_i2c3", "pclk_peri", 0, RK3288_CLKGATE_CON(6), 14, GFLAGS), + GATE(PCLK_SARADC, "pclk_saradc", "pclk_peri", 0, RK3288_CLKGATE_CON(7), 1, GFLAGS), + GATE(PCLK_TSADC, "pclk_tsadc", "pclk_peri", 0, RK3288_CLKGATE_CON(7), 2, GFLAGS), + GATE(PCLK_SIM, "pclk_sim", "pclk_peri", 0, RK3288_CLKGATE_CON(7), 3, GFLAGS), + GATE(PCLK_I2C5, "pclk_i2c5", "pclk_peri", 0, RK3288_CLKGATE_CON(7), 0, GFLAGS), + GATE(PCLK_GMAC, "pclk_gmac", "pclk_peri", 0, RK3288_CLKGATE_CON(8), 1, GFLAGS), + + GATE(SCLK_LCDC_PWM0, "sclk_lcdc_pwm0", "xin24m", 0, RK3288_CLKGATE_CON(13), 10, GFLAGS), + GATE(SCLK_LCDC_PWM1, "sclk_lcdc_pwm1", "xin24m", 0, RK3288_CLKGATE_CON(13), 11, GFLAGS), + GATE(0, "sclk_pvtm_core", "xin24m", 0, RK3288_CLKGATE_CON(5), 9, GFLAGS), + GATE(0, "sclk_pvtm_gpu", "xin24m", 0, RK3288_CLKGATE_CON(5), 10, GFLAGS), + GATE(0, "sclk_mipidsi_24m", "xin24m", 0, RK3288_CLKGATE_CON(5), 15, GFLAGS), + + /* sclk_gpu gates */ + GATE(ACLK_GPU, "aclk_gpu", "sclk_gpu", 0, RK3288_CLKGATE_CON(18), 0, GFLAGS), + + /* pclk_pd_alive gates */ + GATE(PCLK_GPIO8, "pclk_gpio8", "pclk_pd_alive", 0, RK3288_CLKGATE_CON(14), 8, GFLAGS), + GATE(PCLK_GPIO7, "pclk_gpio7", "pclk_pd_alive", 0, RK3288_CLKGATE_CON(14), 7, GFLAGS), + GATE(PCLK_GPIO1, "pclk_gpio1", "pclk_pd_alive", 0, RK3288_CLKGATE_CON(14), 1, GFLAGS), + GATE(PCLK_GPIO2, "pclk_gpio2", "pclk_pd_alive", 0, RK3288_CLKGATE_CON(14), 2, GFLAGS), + GATE(PCLK_GPIO3, "pclk_gpio3", "pclk_pd_alive", 0, RK3288_CLKGATE_CON(14), 3, GFLAGS), + GATE(PCLK_GPIO4, "pclk_gpio4", "pclk_pd_alive", 0, RK3288_CLKGATE_CON(14), 4, GFLAGS), + GATE(PCLK_GPIO5, "pclk_gpio5", "pclk_pd_alive", 0, RK3288_CLKGATE_CON(14), 5, GFLAGS), + GATE(PCLK_GPIO6, "pclk_gpio6", "pclk_pd_alive", 0, RK3288_CLKGATE_CON(14), 6, GFLAGS), + GATE(PCLK_GRF, "pclk_grf", "pclk_pd_alive", 0, RK3288_CLKGATE_CON(14), 11, GFLAGS), + GATE(0, "pclk_alive_niu", "pclk_pd_alive", 0, RK3288_CLKGATE_CON(14), 12, GFLAGS), + + /* pclk_pd_pmu gates */ + GATE(PCLK_PMU, "pclk_pmu", "pclk_pd_pmu", 0, RK3288_CLKGATE_CON(17), 0, GFLAGS), + GATE(0, "pclk_intmem1", "pclk_pd_pmu", 0, RK3288_CLKGATE_CON(17), 1, GFLAGS), + GATE(0, "pclk_pmu_niu", "pclk_pd_pmu", 0, RK3288_CLKGATE_CON(17), 2, GFLAGS), + GATE(PCLK_SGRF, "pclk_sgrf", "pclk_pd_pmu", 0, RK3288_CLKGATE_CON(17), 3, GFLAGS), + GATE(PCLK_GPIO0, "pclk_gpio0", "pclk_pd_pmu", 0, RK3288_CLKGATE_CON(17), 4, GFLAGS), + + /* hclk_vio gates */ + GATE(HCLK_RGA, "hclk_rga", "hclk_vio", 0, RK3288_CLKGATE_CON(15), 1, GFLAGS), + GATE(HCLK_VOP0, "hclk_vop0", "hclk_vio", 0, RK3288_CLKGATE_CON(15), 6, GFLAGS), + GATE(HCLK_VOP1, "hclk_vop1", "hclk_vio", 0, RK3288_CLKGATE_CON(15), 8, GFLAGS), + GATE(0, "hclk_vio_ahb_arbi", "hclk_vio", 0, RK3288_CLKGATE_CON(15), 9, GFLAGS), + GATE(0, "hclk_vio_niu", "hclk_vio", 0, RK3288_CLKGATE_CON(15), 10, GFLAGS), + GATE(0, "hclk_vip", "hclk_vio", 0, RK3288_CLKGATE_CON(15), 15, GFLAGS), + GATE(HCLK_IEP, "hclk_iep", "hclk_vio", 0, RK3288_CLKGATE_CON(15), 3, GFLAGS), + GATE(HCLK_ISP, "hclk_isp", "hclk_vio", 0, RK3288_CLKGATE_CON(16), 1, GFLAGS), + GATE(0, "hclk_vio2_h2p", "hclk_vio", 0, RK3288_CLKGATE_CON(16), 10, GFLAGS), + GATE(0, "pclk_mipi_dsi0", "hclk_vio", 0, RK3288_CLKGATE_CON(16), 4, GFLAGS), + GATE(0, "pclk_mipi_dsi1", "hclk_vio", 0, RK3288_CLKGATE_CON(16), 5, GFLAGS), + GATE(0, "pclk_mipi_csi", "hclk_vio", 0, RK3288_CLKGATE_CON(16), 6, GFLAGS), + GATE(0, "pclk_lvds_phy", "hclk_vio", 0, RK3288_CLKGATE_CON(16), 7, GFLAGS), + GATE(0, "pclk_edp_ctrl", "hclk_vio", 0, RK3288_CLKGATE_CON(16), 8, GFLAGS), + GATE(0, "pclk_hdmi_ctrl", "hclk_vio", 0, RK3288_CLKGATE_CON(16), 9, GFLAGS), + GATE(0, "pclk_vio2_h2p", "hclk_vio", 0, RK3288_CLKGATE_CON(16), 11, GFLAGS), + + /* aclk_vio0 gates */ + GATE(ACLK_VOP0, "aclk_vop0", "aclk_vio0", 0, RK3288_CLKGATE_CON(15), 5, GFLAGS), + GATE(0, "aclk_iep", "aclk_vio0", 0, RK3288_CLKGATE_CON(15), 2, GFLAGS), + GATE(0, "aclk_vio0_niu", "aclk_vio0", 0, RK3288_CLKGATE_CON(15), 11, GFLAGS), + GATE(0, "aclk_vip", "aclk_vio0", 0, RK3288_CLKGATE_CON(15), 14, GFLAGS), + + /* aclk_vio1 gates */ + GATE(ACLK_VOP1, "aclk_vop1", "aclk_vio1", 0, RK3288_CLKGATE_CON(15), 7, GFLAGS), + GATE(0, "aclk_isp", "aclk_vio1", 0, RK3288_CLKGATE_CON(16), 2, GFLAGS), + GATE(0, "aclk_vio1_niu", "aclk_vio1", 0, RK3288_CLKGATE_CON(15), 12, GFLAGS), + + /* aclk_rga_pre gates */ + GATE(ACLK_RGA, "aclk_rga", "aclk_rga_pre", 0, RK3288_CLKGATE_CON(15), 0, GFLAGS), + GATE(0, "aclk_rga_niu", "aclk_rga_pre", 0, RK3288_CLKGATE_CON(15), 13, GFLAGS), + + /* + * Other ungrouped clocks. + */ + + GATE(0, "pclk_vip_in", "ext_vip", 0, RK3288_CLKGATE_CON(16), 0, GFLAGS), + GATE(0, "pclk_isp_in", "ext_isp", 0, RK3288_CLKGATE_CON(16), 3, GFLAGS), +}; + +static void __init rk3288_clk_init(struct device_node *np) +{ + void __iomem *reg_base; + struct clk *clk; + + reg_base = of_iomap(np, 0); + if (!reg_base) { + pr_err("%s: could not map cru region\n", __func__); + return; + } + + rockchip_clk_init(np, reg_base, CLK_NR_CLKS); + + /* xin12m is created by an cru-internal divider */ + clk = clk_register_fixed_factor(NULL, "xin12m", "xin24m", 0, 1, 2); + if (IS_ERR(clk)) + pr_warn("%s: could not register clock xin12m: %ld\n", + __func__, PTR_ERR(clk)); + + + clk = clk_register_fixed_factor(NULL, "usb480m", "xin24m", 0, 20, 1); + if (IS_ERR(clk)) + pr_warn("%s: could not register clock usb480m: %ld\n", + __func__, PTR_ERR(clk)); + + rockchip_clk_register_plls(rk3288_pll_clks, + ARRAY_SIZE(rk3288_pll_clks), + RK3288_GRF_SOC_STATUS); + rockchip_clk_register_branches(rk3288_clk_branches, + ARRAY_SIZE(rk3288_clk_branches)); + + rockchip_register_softrst(np, 9, reg_base + RK3288_SOFTRST_CON(0), + ROCKCHIP_SOFTRST_HIWORD_MASK); +} +CLK_OF_DECLARE(rk3288_cru, "rockchip,rk3288-cru", rk3288_clk_init); diff --git a/drivers/clk/rockchip/clk.c b/drivers/clk/rockchip/clk.c new file mode 100644 index 000000000000..278cf9dd1e23 --- /dev/null +++ b/drivers/clk/rockchip/clk.c @@ -0,0 +1,244 @@ +/* + * Copyright (c) 2014 MundoReader S.L. + * Author: Heiko Stuebner <heiko@sntech.de> + * + * based on + * + * samsung/clk.c + * Copyright (c) 2013 Samsung Electronics Co., Ltd. + * Copyright (c) 2013 Linaro Ltd. + * Author: Thomas Abraham <thomas.ab@samsung.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include <linux/slab.h> +#include <linux/clk.h> +#include <linux/clk-provider.h> +#include <linux/mfd/syscon.h> +#include <linux/regmap.h> +#include "clk.h" + +/** + * Register a clock branch. + * Most clock branches have a form like + * + * src1 --|--\ + * |M |--[GATE]-[DIV]- + * src2 --|--/ + * + * sometimes without one of those components. + */ +struct clk *rockchip_clk_register_branch(const char *name, + const char **parent_names, u8 num_parents, void __iomem *base, + int muxdiv_offset, u8 mux_shift, u8 mux_width, u8 mux_flags, + u8 div_shift, u8 div_width, u8 div_flags, + struct clk_div_table *div_table, int gate_offset, + u8 gate_shift, u8 gate_flags, unsigned long flags, + spinlock_t *lock) +{ + struct clk *clk; + struct clk_mux *mux = NULL; + struct clk_gate *gate = NULL; + struct clk_divider *div = NULL; + const struct clk_ops *mux_ops = NULL, *div_ops = NULL, + *gate_ops = NULL; + + if (num_parents > 1) { + mux = kzalloc(sizeof(*mux), GFP_KERNEL); + if (!mux) + return ERR_PTR(-ENOMEM); + + mux->reg = base + muxdiv_offset; + mux->shift = mux_shift; + mux->mask = BIT(mux_width) - 1; + mux->flags = mux_flags; + mux->lock = lock; + mux_ops = (mux_flags & CLK_MUX_READ_ONLY) ? &clk_mux_ro_ops + : &clk_mux_ops; + } + + if (gate_offset >= 0) { + gate = kzalloc(sizeof(*gate), GFP_KERNEL); + if (!gate) + return ERR_PTR(-ENOMEM); + + gate->flags = gate_flags; + gate->reg = base + gate_offset; + gate->bit_idx = gate_shift; + gate->lock = lock; + gate_ops = &clk_gate_ops; + } + + if (div_width > 0) { + div = kzalloc(sizeof(*div), GFP_KERNEL); + if (!div) + return ERR_PTR(-ENOMEM); + + div->flags = div_flags; + div->reg = base + muxdiv_offset; + div->shift = div_shift; + div->width = div_width; + div->lock = lock; + div->table = div_table; + div_ops = (div_flags & CLK_DIVIDER_READ_ONLY) + ? &clk_divider_ro_ops + : &clk_divider_ops; + } + + clk = clk_register_composite(NULL, name, parent_names, num_parents, + mux ? &mux->hw : NULL, mux_ops, + div ? &div->hw : NULL, div_ops, + gate ? &gate->hw : NULL, gate_ops, + flags); + + return clk; +} + +static DEFINE_SPINLOCK(clk_lock); +static struct clk **clk_table; +static void __iomem *reg_base; +static struct clk_onecell_data clk_data; +static struct device_node *cru_node; +static struct regmap *grf; + +void __init rockchip_clk_init(struct device_node *np, void __iomem *base, + unsigned long nr_clks) +{ + reg_base = base; + cru_node = np; + grf = ERR_PTR(-EPROBE_DEFER); + + clk_table = kcalloc(nr_clks, sizeof(struct clk *), GFP_KERNEL); + if (!clk_table) + pr_err("%s: could not allocate clock lookup table\n", __func__); + + clk_data.clks = clk_table; + clk_data.clk_num = nr_clks; + of_clk_add_provider(np, of_clk_src_onecell_get, &clk_data); +} + +struct regmap *rockchip_clk_get_grf(void) +{ + if (IS_ERR(grf)) + grf = syscon_regmap_lookup_by_phandle(cru_node, "rockchip,grf"); + return grf; +} + +void rockchip_clk_add_lookup(struct clk *clk, unsigned int id) +{ + if (clk_table && id) + clk_table[id] = clk; +} + +void __init rockchip_clk_register_plls(struct rockchip_pll_clock *list, + unsigned int nr_pll, int grf_lock_offset) +{ + struct clk *clk; + int idx; + + for (idx = 0; idx < nr_pll; idx++, list++) { + clk = rockchip_clk_register_pll(list->type, list->name, + list->parent_names, list->num_parents, + reg_base, list->con_offset, grf_lock_offset, + list->lock_shift, list->mode_offset, + list->mode_shift, list->rate_table, &clk_lock); + if (IS_ERR(clk)) { + pr_err("%s: failed to register clock %s\n", __func__, + list->name); + continue; + } + + rockchip_clk_add_lookup(clk, list->id); + } +} + +void __init rockchip_clk_register_branches( + struct rockchip_clk_branch *list, + unsigned int nr_clk) +{ + struct clk *clk = NULL; + unsigned int idx; + unsigned long flags; + + for (idx = 0; idx < nr_clk; idx++, list++) { + flags = list->flags; + + /* catch simple muxes */ + switch (list->branch_type) { + case branch_mux: + clk = clk_register_mux(NULL, list->name, + list->parent_names, list->num_parents, + flags, reg_base + list->muxdiv_offset, + list->mux_shift, list->mux_width, + list->mux_flags, &clk_lock); + break; + case branch_divider: + if (list->div_table) + clk = clk_register_divider_table(NULL, + list->name, list->parent_names[0], + flags, reg_base + list->muxdiv_offset, + list->div_shift, list->div_width, + list->div_flags, list->div_table, + &clk_lock); + else + clk = clk_register_divider(NULL, list->name, + list->parent_names[0], flags, + reg_base + list->muxdiv_offset, + list->div_shift, list->div_width, + list->div_flags, &clk_lock); + break; + case branch_fraction_divider: + /* unimplemented */ + continue; + break; + case branch_gate: + flags |= CLK_SET_RATE_PARENT; + + /* keep all gates untouched for now */ + flags |= CLK_IGNORE_UNUSED; + + clk = clk_register_gate(NULL, list->name, + list->parent_names[0], flags, + reg_base + list->gate_offset, + list->gate_shift, list->gate_flags, &clk_lock); + break; + case branch_composite: + /* keep all gates untouched for now */ + flags |= CLK_IGNORE_UNUSED; + + clk = rockchip_clk_register_branch(list->name, + list->parent_names, list->num_parents, + reg_base, list->muxdiv_offset, list->mux_shift, + list->mux_width, list->mux_flags, + list->div_shift, list->div_width, + list->div_flags, list->div_table, + list->gate_offset, list->gate_shift, + list->gate_flags, flags, &clk_lock); + break; + } + + /* none of the cases above matched */ + if (!clk) { + pr_err("%s: unknown clock type %d\n", + __func__, list->branch_type); + continue; + } + + if (IS_ERR(clk)) { + pr_err("%s: failed to register clock %s: %ld\n", + __func__, list->name, PTR_ERR(clk)); + continue; + } + + rockchip_clk_add_lookup(clk, list->id); + } +} diff --git a/drivers/clk/rockchip/clk.h b/drivers/clk/rockchip/clk.h new file mode 100644 index 000000000000..887cbdeca2aa --- /dev/null +++ b/drivers/clk/rockchip/clk.h @@ -0,0 +1,347 @@ +/* + * Copyright (c) 2014 MundoReader S.L. + * Author: Heiko Stuebner <heiko@sntech.de> + * + * based on + * + * samsung/clk.h + * Copyright (c) 2013 Samsung Electronics Co., Ltd. + * Copyright (c) 2013 Linaro Ltd. + * Author: Thomas Abraham <thomas.ab@samsung.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef CLK_ROCKCHIP_CLK_H +#define CLK_ROCKCHIP_CLK_H + +#include <linux/io.h> +#include <linux/clk.h> +#include <linux/clk-provider.h> + +#define HIWORD_UPDATE(val, mask, shift) \ + ((val) << (shift) | (mask) << ((shift) + 16)) + +/* register positions shared by RK2928, RK3066 and RK3188 */ +#define RK2928_PLL_CON(x) (x * 0x4) +#define RK2928_MODE_CON 0x40 +#define RK2928_CLKSEL_CON(x) (x * 0x4 + 0x44) +#define RK2928_CLKGATE_CON(x) (x * 0x4 + 0xd0) +#define RK2928_GLB_SRST_FST 0x100 +#define RK2928_GLB_SRST_SND 0x104 +#define RK2928_SOFTRST_CON(x) (x * 0x4 + 0x110) +#define RK2928_MISC_CON 0x134 + +#define RK3288_PLL_CON(x) RK2928_PLL_CON(x) +#define RK3288_MODE_CON 0x50 +#define RK3288_CLKSEL_CON(x) (x * 0x4 + 0x60) +#define RK3288_CLKGATE_CON(x) (x * 0x4 + 0x160) +#define RK3288_GLB_SRST_FST 0x1b0 +#define RK3288_GLB_SRST_SND 0x1b4 +#define RK3288_SOFTRST_CON(x) (x * 0x4 + 0x1b8) +#define RK3288_MISC_CON 0x1e8 + +enum rockchip_pll_type { + pll_rk3066, +}; + +#define RK3066_PLL_RATE(_rate, _nr, _nf, _no) \ +{ \ + .rate = _rate##U, \ + .nr = _nr, \ + .nf = _nf, \ + .no = _no, \ + .bwadj = (_nf >> 1), \ +} + +struct rockchip_pll_rate_table { + unsigned long rate; + unsigned int nr; + unsigned int nf; + unsigned int no; + unsigned int bwadj; +}; + +/** + * struct rockchip_pll_clock: information about pll clock + * @id: platform specific id of the clock. + * @name: name of this pll clock. + * @parent_name: name of the parent clock. + * @flags: optional flags for basic clock. + * @con_offset: offset of the register for configuring the PLL. + * @mode_offset: offset of the register for configuring the PLL-mode. + * @mode_shift: offset inside the mode-register for the mode of this pll. + * @lock_shift: offset inside the lock register for the lock status. + * @type: Type of PLL to be registered. + * @rate_table: Table of usable pll rates + */ +struct rockchip_pll_clock { + unsigned int id; + const char *name; + const char **parent_names; + u8 num_parents; + unsigned long flags; + int con_offset; + int mode_offset; + int mode_shift; + int lock_shift; + enum rockchip_pll_type type; + struct rockchip_pll_rate_table *rate_table; +}; + +#define PLL(_type, _id, _name, _pnames, _flags, _con, _mode, _mshift, \ + _lshift, _rtable) \ + { \ + .id = _id, \ + .type = _type, \ + .name = _name, \ + .parent_names = _pnames, \ + .num_parents = ARRAY_SIZE(_pnames), \ + .flags = CLK_GET_RATE_NOCACHE | _flags, \ + .con_offset = _con, \ + .mode_offset = _mode, \ + .mode_shift = _mshift, \ + .lock_shift = _lshift, \ + .rate_table = _rtable, \ + } + +struct clk *rockchip_clk_register_pll(enum rockchip_pll_type pll_type, + const char *name, const char **parent_names, u8 num_parents, + void __iomem *base, int con_offset, int grf_lock_offset, + int lock_shift, int reg_mode, int mode_shift, + struct rockchip_pll_rate_table *rate_table, + spinlock_t *lock); + +#define PNAME(x) static const char *x[] __initconst + +enum rockchip_clk_branch_type { + branch_composite, + branch_mux, + branch_divider, + branch_fraction_divider, + branch_gate, +}; + +struct rockchip_clk_branch { + unsigned int id; + enum rockchip_clk_branch_type branch_type; + const char *name; + const char **parent_names; + u8 num_parents; + unsigned long flags; + int muxdiv_offset; + u8 mux_shift; + u8 mux_width; + u8 mux_flags; + u8 div_shift; + u8 div_width; + u8 div_flags; + struct clk_div_table *div_table; + int gate_offset; + u8 gate_shift; + u8 gate_flags; +}; + +#define COMPOSITE(_id, cname, pnames, f, mo, ms, mw, mf, ds, dw,\ + df, go, gs, gf) \ + { \ + .id = _id, \ + .branch_type = branch_composite, \ + .name = cname, \ + .parent_names = pnames, \ + .num_parents = ARRAY_SIZE(pnames), \ + .flags = f, \ + .muxdiv_offset = mo, \ + .mux_shift = ms, \ + .mux_width = mw, \ + .mux_flags = mf, \ + .div_shift = ds, \ + .div_width = dw, \ + .div_flags = df, \ + .gate_offset = go, \ + .gate_shift = gs, \ + .gate_flags = gf, \ + } + +#define COMPOSITE_NOMUX(_id, cname, pname, f, mo, ds, dw, df, \ + go, gs, gf) \ + { \ + .id = _id, \ + .branch_type = branch_composite, \ + .name = cname, \ + .parent_names = (const char *[]){ pname }, \ + .num_parents = 1, \ + .flags = f, \ + .muxdiv_offset = mo, \ + .div_shift = ds, \ + .div_width = dw, \ + .div_flags = df, \ + .gate_offset = go, \ + .gate_shift = gs, \ + .gate_flags = gf, \ + } + +#define COMPOSITE_NOMUX_DIVTBL(_id, cname, pname, f, mo, ds, dw,\ + df, dt, go, gs, gf) \ + { \ + .id = _id, \ + .branch_type = branch_composite, \ + .name = cname, \ + .parent_names = (const char *[]){ pname }, \ + .num_parents = 1, \ + .flags = f, \ + .muxdiv_offset = mo, \ + .div_shift = ds, \ + .div_width = dw, \ + .div_flags = df, \ + .div_table = dt, \ + .gate_offset = go, \ + .gate_shift = gs, \ + .gate_flags = gf, \ + } + +#define COMPOSITE_NODIV(_id, cname, pnames, f, mo, ms, mw, mf, \ + go, gs, gf) \ + { \ + .id = _id, \ + .branch_type = branch_composite, \ + .name = cname, \ + .parent_names = pnames, \ + .num_parents = ARRAY_SIZE(pnames), \ + .flags = f, \ + .muxdiv_offset = mo, \ + .mux_shift = ms, \ + .mux_width = mw, \ + .mux_flags = mf, \ + .gate_offset = go, \ + .gate_shift = gs, \ + .gate_flags = gf, \ + } + +#define COMPOSITE_NOGATE(_id, cname, pnames, f, mo, ms, mw, mf, \ + ds, dw, df) \ + { \ + .id = _id, \ + .branch_type = branch_composite, \ + .name = cname, \ + .parent_names = pnames, \ + .num_parents = ARRAY_SIZE(pnames), \ + .flags = f, \ + .muxdiv_offset = mo, \ + .mux_shift = ms, \ + .mux_width = mw, \ + .mux_flags = mf, \ + .div_shift = ds, \ + .div_width = dw, \ + .div_flags = df, \ + .gate_offset = -1, \ + } + +#define COMPOSITE_FRAC(_id, cname, pname, f, mo, df, go, gs, gf)\ + { \ + .id = _id, \ + .branch_type = branch_fraction_divider, \ + .name = cname, \ + .parent_names = (const char *[]){ pname }, \ + .num_parents = 1, \ + .flags = f, \ + .muxdiv_offset = mo, \ + .div_shift = 16, \ + .div_width = 16, \ + .div_flags = df, \ + .gate_offset = go, \ + .gate_shift = gs, \ + .gate_flags = gf, \ + } + +#define MUX(_id, cname, pnames, f, o, s, w, mf) \ + { \ + .id = _id, \ + .branch_type = branch_mux, \ + .name = cname, \ + .parent_names = pnames, \ + .num_parents = ARRAY_SIZE(pnames), \ + .flags = f, \ + .muxdiv_offset = o, \ + .mux_shift = s, \ + .mux_width = w, \ + .mux_flags = mf, \ + .gate_offset = -1, \ + } + +#define DIV(_id, cname, pname, f, o, s, w, df) \ + { \ + .id = _id, \ + .branch_type = branch_divider, \ + .name = cname, \ + .parent_names = (const char *[]){ pname }, \ + .num_parents = 1, \ + .flags = f, \ + .muxdiv_offset = o, \ + .div_shift = s, \ + .div_width = w, \ + .div_flags = df, \ + .gate_offset = -1, \ + } + +#define DIVTBL(_id, cname, pname, f, o, s, w, df, dt) \ + { \ + .id = _id, \ + .branch_type = branch_divider, \ + .name = cname, \ + .parent_names = (const char *[]){ pname }, \ + .num_parents = 1, \ + .flags = f, \ + .muxdiv_offset = o, \ + .div_shift = s, \ + .div_width = w, \ + .div_flags = df, \ + .div_table = dt, \ + } + +#define GATE(_id, cname, pname, f, o, b, gf) \ + { \ + .id = _id, \ + .branch_type = branch_gate, \ + .name = cname, \ + .parent_names = (const char *[]){ pname }, \ + .num_parents = 1, \ + .flags = f, \ + .gate_offset = o, \ + .gate_shift = b, \ + .gate_flags = gf, \ + } + + +void rockchip_clk_init(struct device_node *np, void __iomem *base, + unsigned long nr_clks); +struct regmap *rockchip_clk_get_grf(void); +void rockchip_clk_add_lookup(struct clk *clk, unsigned int id); +void rockchip_clk_register_branches(struct rockchip_clk_branch *clk_list, + unsigned int nr_clk); +void rockchip_clk_register_plls(struct rockchip_pll_clock *pll_list, + unsigned int nr_pll, int grf_lock_offset); + +#define ROCKCHIP_SOFTRST_HIWORD_MASK BIT(0) + +#ifdef CONFIG_RESET_CONTROLLER +void rockchip_register_softrst(struct device_node *np, + unsigned int num_regs, + void __iomem *base, u8 flags); +#else +static inline void rockchip_register_softrst(struct device_node *np, + unsigned int num_regs, + void __iomem *base, u8 flags) +{ +} +#endif + +#endif diff --git a/drivers/clk/rockchip/softrst.c b/drivers/clk/rockchip/softrst.c new file mode 100644 index 000000000000..552f7bb15bc5 --- /dev/null +++ b/drivers/clk/rockchip/softrst.c @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2014 MundoReader S.L. + * Author: Heiko Stuebner <heiko@sntech.de> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include <linux/slab.h> +#include <linux/io.h> +#include <linux/reset-controller.h> +#include <linux/spinlock.h> +#include "clk.h" + +struct rockchip_softrst { + struct reset_controller_dev rcdev; + void __iomem *reg_base; + int num_regs; + int num_per_reg; + u8 flags; + spinlock_t lock; +}; + +static int rockchip_softrst_assert(struct reset_controller_dev *rcdev, + unsigned long id) +{ + struct rockchip_softrst *softrst = container_of(rcdev, + struct rockchip_softrst, + rcdev); + int bank = id / softrst->num_per_reg; + int offset = id % softrst->num_per_reg; + + if (softrst->flags & ROCKCHIP_SOFTRST_HIWORD_MASK) { + writel(BIT(offset) | (BIT(offset) << 16), + softrst->reg_base + (bank * 4)); + } else { + unsigned long flags; + u32 reg; + + spin_lock_irqsave(&softrst->lock, flags); + + reg = readl(softrst->reg_base + (bank * 4)); + writel(reg | BIT(offset), softrst->reg_base + (bank * 4)); + + spin_unlock_irqrestore(&softrst->lock, flags); + } + + return 0; +} + +static int rockchip_softrst_deassert(struct reset_controller_dev *rcdev, + unsigned long id) +{ + struct rockchip_softrst *softrst = container_of(rcdev, + struct rockchip_softrst, + rcdev); + int bank = id / softrst->num_per_reg; + int offset = id % softrst->num_per_reg; + + if (softrst->flags & ROCKCHIP_SOFTRST_HIWORD_MASK) { + writel((BIT(offset) << 16), softrst->reg_base + (bank * 4)); + } else { + unsigned long flags; + u32 reg; + + spin_lock_irqsave(&softrst->lock, flags); + + reg = readl(softrst->reg_base + (bank * 4)); + writel(reg & ~BIT(offset), softrst->reg_base + (bank * 4)); + + spin_unlock_irqrestore(&softrst->lock, flags); + } + + return 0; +} + +static struct reset_control_ops rockchip_softrst_ops = { + .assert = rockchip_softrst_assert, + .deassert = rockchip_softrst_deassert, +}; + +void __init rockchip_register_softrst(struct device_node *np, + unsigned int num_regs, + void __iomem *base, u8 flags) +{ + struct rockchip_softrst *softrst; + int ret; + + softrst = kzalloc(sizeof(*softrst), GFP_KERNEL); + if (!softrst) + return; + + spin_lock_init(&softrst->lock); + + softrst->reg_base = base; + softrst->flags = flags; + softrst->num_regs = num_regs; + softrst->num_per_reg = (flags & ROCKCHIP_SOFTRST_HIWORD_MASK) ? 16 + : 32; + + softrst->rcdev.owner = THIS_MODULE; + softrst->rcdev.nr_resets = num_regs * softrst->num_per_reg; + softrst->rcdev.ops = &rockchip_softrst_ops; + softrst->rcdev.of_node = np; + ret = reset_controller_register(&softrst->rcdev); + if (ret) { + pr_err("%s: could not register reset controller, %d\n", + __func__, ret); + kfree(softrst); + } +}; diff --git a/drivers/clk/samsung/Makefile b/drivers/clk/samsung/Makefile index 69e81773164e..2949a556af8f 100644 --- a/drivers/clk/samsung/Makefile +++ b/drivers/clk/samsung/Makefile @@ -11,6 +11,7 @@ obj-$(CONFIG_SOC_EXYNOS5410) += clk-exynos5410.o obj-$(CONFIG_SOC_EXYNOS5420) += clk-exynos5420.o obj-$(CONFIG_SOC_EXYNOS5440) += clk-exynos5440.o obj-$(CONFIG_ARCH_EXYNOS) += clk-exynos-audss.o +obj-$(CONFIG_ARCH_EXYNOS) += clk-exynos-clkout.o obj-$(CONFIG_S3C2410_COMMON_CLK)+= clk-s3c2410.o obj-$(CONFIG_S3C2410_COMMON_DCLK)+= clk-s3c2410-dclk.o obj-$(CONFIG_S3C2412_COMMON_CLK)+= clk-s3c2412.o diff --git a/drivers/clk/samsung/clk-exynos-clkout.c b/drivers/clk/samsung/clk-exynos-clkout.c new file mode 100644 index 000000000000..3a7cb2506731 --- /dev/null +++ b/drivers/clk/samsung/clk-exynos-clkout.c @@ -0,0 +1,153 @@ +/* + * Copyright (c) 2014 Samsung Electronics Co., Ltd. + * Author: Tomasz Figa <t.figa@samsung.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * Clock driver for Exynos clock output + */ + +#include <linux/clk.h> +#include <linux/clkdev.h> +#include <linux/clk-provider.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include <linux/syscore_ops.h> + +#define EXYNOS_CLKOUT_NR_CLKS 1 +#define EXYNOS_CLKOUT_PARENTS 32 + +#define EXYNOS_PMU_DEBUG_REG 0xa00 +#define EXYNOS_CLKOUT_DISABLE_SHIFT 0 +#define EXYNOS_CLKOUT_MUX_SHIFT 8 +#define EXYNOS4_CLKOUT_MUX_MASK 0xf +#define EXYNOS5_CLKOUT_MUX_MASK 0x1f + +struct exynos_clkout { + struct clk_gate gate; + struct clk_mux mux; + spinlock_t slock; + struct clk_onecell_data data; + struct clk *clk_table[EXYNOS_CLKOUT_NR_CLKS]; + void __iomem *reg; + u32 pmu_debug_save; +}; + +static struct exynos_clkout *clkout; + +static int exynos_clkout_suspend(void) +{ + clkout->pmu_debug_save = readl(clkout->reg + EXYNOS_PMU_DEBUG_REG); + + return 0; +} + +static void exynos_clkout_resume(void) +{ + writel(clkout->pmu_debug_save, clkout->reg + EXYNOS_PMU_DEBUG_REG); +} + +static struct syscore_ops exynos_clkout_syscore_ops = { + .suspend = exynos_clkout_suspend, + .resume = exynos_clkout_resume, +}; + +static void __init exynos_clkout_init(struct device_node *node, u32 mux_mask) +{ + const char *parent_names[EXYNOS_CLKOUT_PARENTS]; + struct clk *parents[EXYNOS_CLKOUT_PARENTS]; + int parent_count; + int ret; + int i; + + clkout = kzalloc(sizeof(*clkout), GFP_KERNEL); + if (!clkout) + return; + + spin_lock_init(&clkout->slock); + + parent_count = 0; + for (i = 0; i < EXYNOS_CLKOUT_PARENTS; ++i) { + char name[] = "clkoutXX"; + + snprintf(name, sizeof(name), "clkout%d", i); + parents[i] = of_clk_get_by_name(node, name); + if (IS_ERR(parents[i])) { + parent_names[i] = "none"; + continue; + } + + parent_names[i] = __clk_get_name(parents[i]); + parent_count = i + 1; + } + + if (!parent_count) + goto free_clkout; + + clkout->reg = of_iomap(node, 0); + if (!clkout->reg) + goto clks_put; + + clkout->gate.reg = clkout->reg + EXYNOS_PMU_DEBUG_REG; + clkout->gate.bit_idx = EXYNOS_CLKOUT_DISABLE_SHIFT; + clkout->gate.flags = CLK_GATE_SET_TO_DISABLE; + clkout->gate.lock = &clkout->slock; + + clkout->mux.reg = clkout->reg + EXYNOS_PMU_DEBUG_REG; + clkout->mux.mask = mux_mask; + clkout->mux.shift = EXYNOS_CLKOUT_MUX_SHIFT; + clkout->mux.lock = &clkout->slock; + + clkout->clk_table[0] = clk_register_composite(NULL, "clkout", + parent_names, parent_count, &clkout->mux.hw, + &clk_mux_ops, NULL, NULL, &clkout->gate.hw, + &clk_gate_ops, CLK_SET_RATE_PARENT + | CLK_SET_RATE_NO_REPARENT); + if (IS_ERR(clkout->clk_table[0])) + goto err_unmap; + + clkout->data.clks = clkout->clk_table; + clkout->data.clk_num = EXYNOS_CLKOUT_NR_CLKS; + ret = of_clk_add_provider(node, of_clk_src_onecell_get, &clkout->data); + if (ret) + goto err_clk_unreg; + + register_syscore_ops(&exynos_clkout_syscore_ops); + + return; + +err_clk_unreg: + clk_unregister(clkout->clk_table[0]); +err_unmap: + iounmap(clkout->reg); +clks_put: + for (i = 0; i < EXYNOS_CLKOUT_PARENTS; ++i) + if (!IS_ERR(parents[i])) + clk_put(parents[i]); +free_clkout: + kfree(clkout); + + pr_err("%s: failed to register clkout clock\n", __func__); +} + +static void __init exynos4_clkout_init(struct device_node *node) +{ + exynos_clkout_init(node, EXYNOS4_CLKOUT_MUX_MASK); +} +CLK_OF_DECLARE(exynos4210_clkout, "samsung,exynos4210-pmu", + exynos4_clkout_init); +CLK_OF_DECLARE(exynos4212_clkout, "samsung,exynos4212-pmu", + exynos4_clkout_init); +CLK_OF_DECLARE(exynos4412_clkout, "samsung,exynos4412-pmu", + exynos4_clkout_init); + +static void __init exynos5_clkout_init(struct device_node *node) +{ + exynos_clkout_init(node, EXYNOS5_CLKOUT_MUX_MASK); +} +CLK_OF_DECLARE(exynos5250_clkout, "samsung,exynos5250-pmu", + exynos5_clkout_init); +CLK_OF_DECLARE(exynos5420_clkout, "samsung,exynos5420-pmu", + exynos5_clkout_init); diff --git a/drivers/clk/samsung/clk-exynos3250.c b/drivers/clk/samsung/clk-exynos3250.c index 7a17bd40d1dd..dc85f8e7a2d7 100644 --- a/drivers/clk/samsung/clk-exynos3250.c +++ b/drivers/clk/samsung/clk-exynos3250.c @@ -87,6 +87,22 @@ #define SRC_CPU 0x14200 #define DIV_CPU0 0x14500 #define DIV_CPU1 0x14504 +#define PWR_CTRL1 0x15020 +#define PWR_CTRL2 0x15024 + +/* Below definitions are used for PWR_CTRL settings */ +#define PWR_CTRL1_CORE2_DOWN_RATIO(x) (((x) & 0x7) << 28) +#define PWR_CTRL1_CORE1_DOWN_RATIO(x) (((x) & 0x7) << 16) +#define PWR_CTRL1_DIV2_DOWN_EN (1 << 9) +#define PWR_CTRL1_DIV1_DOWN_EN (1 << 8) +#define PWR_CTRL1_USE_CORE3_WFE (1 << 7) +#define PWR_CTRL1_USE_CORE2_WFE (1 << 6) +#define PWR_CTRL1_USE_CORE1_WFE (1 << 5) +#define PWR_CTRL1_USE_CORE0_WFE (1 << 4) +#define PWR_CTRL1_USE_CORE3_WFI (1 << 3) +#define PWR_CTRL1_USE_CORE2_WFI (1 << 2) +#define PWR_CTRL1_USE_CORE1_WFI (1 << 1) +#define PWR_CTRL1_USE_CORE0_WFI (1 << 0) /* list of PLLs to be registered */ enum exynos3250_plls { @@ -168,6 +184,8 @@ static unsigned long exynos3250_cmu_clk_regs[] __initdata = { SRC_CPU, DIV_CPU0, DIV_CPU1, + PWR_CTRL1, + PWR_CTRL2, }; static int exynos3250_clk_suspend(void) @@ -748,6 +766,27 @@ static struct samsung_pll_clock exynos3250_plls[nr_plls] __initdata = { UPLL_LOCK, UPLL_CON0, NULL), }; +static void __init exynos3_core_down_clock(void) +{ + unsigned int tmp; + + /* + * Enable arm clock down (in idle) and set arm divider + * ratios in WFI/WFE state. + */ + tmp = (PWR_CTRL1_CORE2_DOWN_RATIO(7) | PWR_CTRL1_CORE1_DOWN_RATIO(7) | + PWR_CTRL1_DIV2_DOWN_EN | PWR_CTRL1_DIV1_DOWN_EN | + PWR_CTRL1_USE_CORE1_WFE | PWR_CTRL1_USE_CORE0_WFE | + PWR_CTRL1_USE_CORE1_WFI | PWR_CTRL1_USE_CORE0_WFI); + __raw_writel(tmp, reg_base + PWR_CTRL1); + + /* + * Disable the clock up feature on Exynos4x12, in case it was + * enabled by bootloader. + */ + __raw_writel(0x0, reg_base + PWR_CTRL2); +} + static void __init exynos3250_cmu_init(struct device_node *np) { struct samsung_clk_provider *ctx; @@ -775,6 +814,10 @@ static void __init exynos3250_cmu_init(struct device_node *np) samsung_clk_register_div(ctx, div_clks, ARRAY_SIZE(div_clks)); samsung_clk_register_gate(ctx, gate_clks, ARRAY_SIZE(gate_clks)); + exynos3_core_down_clock(); + exynos3250_clk_sleep_init(); + + samsung_clk_of_add_provider(np, ctx); } CLK_OF_DECLARE(exynos3250_cmu, "samsung,exynos3250-cmu", exynos3250_cmu_init); diff --git a/drivers/clk/samsung/clk-exynos4.c b/drivers/clk/samsung/clk-exynos4.c index 7f4a473a7ad7..ac163d7f5bc3 100644 --- a/drivers/clk/samsung/clk-exynos4.c +++ b/drivers/clk/samsung/clk-exynos4.c @@ -25,10 +25,12 @@ #define DIV_LEFTBUS 0x4500 #define GATE_IP_LEFTBUS 0x4800 #define E4X12_GATE_IP_IMAGE 0x4930 +#define CLKOUT_CMU_LEFTBUS 0x4a00 #define SRC_RIGHTBUS 0x8200 #define DIV_RIGHTBUS 0x8500 #define GATE_IP_RIGHTBUS 0x8800 #define E4X12_GATE_IP_PERIR 0x8960 +#define CLKOUT_CMU_RIGHTBUS 0x8a00 #define EPLL_LOCK 0xc010 #define VPLL_LOCK 0xc020 #define EPLL_CON0 0xc110 @@ -98,6 +100,7 @@ #define GATE_IP_PERIL 0xc950 #define E4210_GATE_IP_PERIR 0xc960 #define GATE_BLOCK 0xc970 +#define CLKOUT_CMU_TOP 0xca00 #define E4X12_MPLL_LOCK 0x10008 #define E4X12_MPLL_CON0 0x10108 #define SRC_DMC 0x10200 @@ -105,6 +108,7 @@ #define DIV_DMC0 0x10500 #define DIV_DMC1 0x10504 #define GATE_IP_DMC 0x10900 +#define CLKOUT_CMU_DMC 0x10a00 #define APLL_LOCK 0x14000 #define E4210_MPLL_LOCK 0x14008 #define APLL_CON0 0x14100 @@ -114,11 +118,28 @@ #define DIV_CPU1 0x14504 #define GATE_SCLK_CPU 0x14800 #define GATE_IP_CPU 0x14900 +#define CLKOUT_CMU_CPU 0x14a00 +#define PWR_CTRL1 0x15020 +#define E4X12_PWR_CTRL2 0x15024 #define E4X12_DIV_ISP0 0x18300 #define E4X12_DIV_ISP1 0x18304 #define E4X12_GATE_ISP0 0x18800 #define E4X12_GATE_ISP1 0x18804 +/* Below definitions are used for PWR_CTRL settings */ +#define PWR_CTRL1_CORE2_DOWN_RATIO(x) (((x) & 0x7) << 28) +#define PWR_CTRL1_CORE1_DOWN_RATIO(x) (((x) & 0x7) << 16) +#define PWR_CTRL1_DIV2_DOWN_EN (1 << 9) +#define PWR_CTRL1_DIV1_DOWN_EN (1 << 8) +#define PWR_CTRL1_USE_CORE3_WFE (1 << 7) +#define PWR_CTRL1_USE_CORE2_WFE (1 << 6) +#define PWR_CTRL1_USE_CORE1_WFE (1 << 5) +#define PWR_CTRL1_USE_CORE0_WFE (1 << 4) +#define PWR_CTRL1_USE_CORE3_WFI (1 << 3) +#define PWR_CTRL1_USE_CORE2_WFI (1 << 2) +#define PWR_CTRL1_USE_CORE1_WFI (1 << 1) +#define PWR_CTRL1_USE_CORE0_WFI (1 << 0) + /* the exynos4 soc type */ enum exynos4_soc { EXYNOS4210, @@ -155,6 +176,7 @@ static unsigned long exynos4210_clk_save[] __initdata = { E4210_GATE_IP_LCD1, E4210_GATE_IP_PERIR, E4210_MPLL_CON0, + PWR_CTRL1, }; static unsigned long exynos4x12_clk_save[] __initdata = { @@ -164,6 +186,8 @@ static unsigned long exynos4x12_clk_save[] __initdata = { E4X12_DIV_ISP, E4X12_DIV_CAM1, E4X12_MPLL_CON0, + PWR_CTRL1, + E4X12_PWR_CTRL2, }; static unsigned long exynos4_clk_pll_regs[] __initdata = { @@ -242,6 +266,11 @@ static unsigned long exynos4_clk_regs[] __initdata = { DIV_CPU1, GATE_SCLK_CPU, GATE_IP_CPU, + CLKOUT_CMU_LEFTBUS, + CLKOUT_CMU_RIGHTBUS, + CLKOUT_CMU_TOP, + CLKOUT_CMU_DMC, + CLKOUT_CMU_CPU, }; static const struct samsung_clk_reg_dump src_mask_suspend[] = { @@ -397,10 +426,32 @@ PNAME(mout_audio2_p4210) = { "cdclk2", "none", "sclk_hdmi24m", "sclk_epll", "sclk_vpll", }; PNAME(mout_mixer_p4210) = { "sclk_dac", "sclk_hdmi", }; PNAME(mout_dac_p4210) = { "sclk_vpll", "sclk_hdmiphy", }; +PNAME(mout_pwi_p4210) = { "xxti", "xusbxti", "sclk_hdmi24m", "sclk_usbphy0", + "sclk_usbphy1", "sclk_hdmiphy", "none", + "sclk_epll", "sclk_vpll" }; +PNAME(clkout_left_p4210) = { "sclk_mpll_div_2", "sclk_apll_div_2", + "div_gdl", "div_gpl" }; +PNAME(clkout_right_p4210) = { "sclk_mpll_div_2", "sclk_apll_div_2", + "div_gdr", "div_gpr" }; +PNAME(clkout_top_p4210) = { "fout_epll", "fout_vpll", "sclk_hdmi24m", + "sclk_usbphy0", "sclk_usbphy1", "sclk_hdmiphy", + "cdclk0", "cdclk1", "cdclk2", "spdif_extclk", + "aclk160", "aclk133", "aclk200", "aclk100", + "sclk_mfc", "sclk_g3d", "sclk_g2d", + "cam_a_pclk", "cam_b_pclk", "s_rxbyteclkhs0_2l", + "s_rxbyteclkhs0_4l" }; +PNAME(clkout_dmc_p4210) = { "div_dmcd", "div_dmcp", "div_acp_pclk", "div_dmc", + "div_dphy", "none", "div_pwi" }; +PNAME(clkout_cpu_p4210) = { "fout_apll_div_2", "none", "fout_mpll_div_2", + "none", "arm_clk_div_2", "div_corem0", + "div_corem1", "div_corem0", "div_atb", + "div_periph", "div_pclk_dbg", "div_hpm" }; /* Exynos 4x12-specific parent groups */ PNAME(mout_mpll_user_p4x12) = { "fin_pll", "sclk_mpll", }; PNAME(mout_core_p4x12) = { "mout_apll", "mout_mpll_user_c", }; +PNAME(mout_gdl_p4x12) = { "mout_mpll_user_l", "sclk_apll", }; +PNAME(mout_gdr_p4x12) = { "mout_mpll_user_r", "sclk_apll", }; PNAME(sclk_ampll_p4x12) = { "mout_mpll_user_t", "sclk_apll", }; PNAME(group1_p4x12) = { "xxti", "xusbxti", "sclk_hdmi24m", "sclk_usbphy0", "none", "sclk_hdmiphy", "mout_mpll_user_t", @@ -418,6 +469,32 @@ PNAME(aclk_p4412) = { "mout_mpll_user_t", "sclk_apll", }; PNAME(mout_user_aclk400_mcuisp_p4x12) = {"fin_pll", "div_aclk400_mcuisp", }; PNAME(mout_user_aclk200_p4x12) = {"fin_pll", "div_aclk200", }; PNAME(mout_user_aclk266_gps_p4x12) = {"fin_pll", "div_aclk266_gps", }; +PNAME(mout_pwi_p4x12) = { "xxti", "xusbxti", "sclk_hdmi24m", "sclk_usbphy0", + "none", "sclk_hdmiphy", "sclk_mpll", + "sclk_epll", "sclk_vpll" }; +PNAME(clkout_left_p4x12) = { "sclk_mpll_user_l_div_2", "sclk_apll_div_2", + "div_gdl", "div_gpl" }; +PNAME(clkout_right_p4x12) = { "sclk_mpll_user_r_div_2", "sclk_apll_div_2", + "div_gdr", "div_gpr" }; +PNAME(clkout_top_p4x12) = { "fout_epll", "fout_vpll", "sclk_hdmi24m", + "sclk_usbphy0", "none", "sclk_hdmiphy", + "cdclk0", "cdclk1", "cdclk2", "spdif_extclk", + "aclk160", "aclk133", "aclk200", "aclk100", + "sclk_mfc", "sclk_g3d", "aclk400_mcuisp", + "cam_a_pclk", "cam_b_pclk", "s_rxbyteclkhs0_2l", + "s_rxbyteclkhs0_4l", "rx_half_byte_clk_csis0", + "rx_half_byte_clk_csis1", "div_jpeg", + "sclk_pwm_isp", "sclk_spi0_isp", + "sclk_spi1_isp", "sclk_uart_isp", + "sclk_mipihsi", "sclk_hdmi", "sclk_fimd0", + "sclk_pcm0" }; +PNAME(clkout_dmc_p4x12) = { "div_dmcd", "div_dmcp", "aclk_acp", "div_acp_pclk", + "div_dmc", "div_dphy", "fout_mpll_div_2", + "div_pwi", "none", "div_c2c", "div_c2c_aclk" }; +PNAME(clkout_cpu_p4x12) = { "fout_apll_div_2", "none", "none", "none", + "arm_clk_div_2", "div_corem0", "div_corem1", + "div_cores", "div_atb", "div_periph", + "div_pclk_dbg", "div_hpm" }; /* fixed rate clocks generated outside the soc */ static struct samsung_fixed_rate_clock exynos4_fixed_rate_ext_clks[] __initdata = { @@ -436,6 +513,24 @@ static struct samsung_fixed_rate_clock exynos4210_fixed_rate_clks[] __initdata = FRATE(0, "sclk_usbphy1", NULL, CLK_IS_ROOT, 48000000), }; +static struct samsung_fixed_factor_clock exynos4_fixed_factor_clks[] __initdata = { + FFACTOR(0, "sclk_apll_div_2", "sclk_apll", 1, 2, 0), + FFACTOR(0, "fout_mpll_div_2", "fout_mpll", 1, 2, 0), + FFACTOR(0, "fout_apll_div_2", "fout_apll", 1, 2, 0), + FFACTOR(0, "arm_clk_div_2", "arm_clk", 1, 2, 0), +}; + +static struct samsung_fixed_factor_clock exynos4210_fixed_factor_clks[] __initdata = { + FFACTOR(0, "sclk_mpll_div_2", "sclk_mpll", 1, 2, 0), +}; + +static struct samsung_fixed_factor_clock exynos4x12_fixed_factor_clks[] __initdata = { + FFACTOR(0, "sclk_mpll_user_l_div_2", "mout_mpll_user_l", 1, 2, 0), + FFACTOR(0, "sclk_mpll_user_r_div_2", "mout_mpll_user_r", 1, 2, 0), + FFACTOR(0, "sclk_mpll_user_t_div_2", "mout_mpll_user_t", 1, 2, 0), + FFACTOR(0, "sclk_mpll_user_c_div_2", "mout_mpll_user_c", 1, 2, 0), +}; + /* list of mux clocks supported in all exynos4 soc's */ static struct samsung_mux_clock exynos4_mux_clks[] __initdata = { MUX_FA(CLK_MOUT_APLL, "mout_apll", mout_apll_p, SRC_CPU, 0, 1, @@ -451,6 +546,9 @@ static struct samsung_mux_clock exynos4_mux_clks[] __initdata = { MUX(0, "mout_onenand1", mout_onenand1_p, SRC_TOP0, 0, 1), MUX(CLK_SCLK_EPLL, "sclk_epll", mout_epll_p, SRC_TOP0, 4, 1), MUX(0, "mout_onenand", mout_onenand_p, SRC_TOP0, 28, 1), + + MUX(0, "mout_dmc_bus", sclk_ampll_p4210, SRC_DMC, 4, 1), + MUX(0, "mout_dphy", sclk_ampll_p4210, SRC_DMC, 8, 1), }; /* list of mux clocks supported in exynos4210 soc */ @@ -459,6 +557,14 @@ static struct samsung_mux_clock exynos4210_mux_early[] __initdata = { }; static struct samsung_mux_clock exynos4210_mux_clks[] __initdata = { + MUX(0, "mout_gdl", sclk_ampll_p4210, SRC_LEFTBUS, 0, 1), + MUX(0, "mout_clkout_leftbus", clkout_left_p4210, + CLKOUT_CMU_LEFTBUS, 0, 5), + + MUX(0, "mout_gdr", sclk_ampll_p4210, SRC_RIGHTBUS, 0, 1), + MUX(0, "mout_clkout_rightbus", clkout_right_p4210, + CLKOUT_CMU_RIGHTBUS, 0, 5), + MUX(0, "mout_aclk200", sclk_ampll_p4210, SRC_TOP0, 12, 1), MUX(0, "mout_aclk100", sclk_ampll_p4210, SRC_TOP0, 16, 1), MUX(0, "mout_aclk160", sclk_ampll_p4210, SRC_TOP0, 20, 1), @@ -472,6 +578,7 @@ static struct samsung_mux_clock exynos4210_mux_clks[] __initdata = { MUX(0, "mout_mipi1", group1_p4210, E4210_SRC_LCD1, 12, 4), MUX(CLK_SCLK_MPLL, "sclk_mpll", mout_mpll_p, SRC_CPU, 8, 1), MUX(CLK_MOUT_CORE, "mout_core", mout_core_p4210, SRC_CPU, 16, 1), + MUX(0, "mout_hpm", mout_core_p4210, SRC_CPU, 20, 1), MUX(CLK_SCLK_VPLL, "sclk_vpll", sclk_vpll_p4210, SRC_TOP0, 8, 1), MUX(CLK_MOUT_FIMC0, "mout_fimc0", group1_p4210, SRC_CAM, 0, 4), MUX(CLK_MOUT_FIMC1, "mout_fimc1", group1_p4210, SRC_CAM, 4, 4), @@ -503,12 +610,30 @@ static struct samsung_mux_clock exynos4210_mux_clks[] __initdata = { MUX(0, "mout_spi0", group1_p4210, SRC_PERIL1, 16, 4), MUX(0, "mout_spi1", group1_p4210, SRC_PERIL1, 20, 4), MUX(0, "mout_spi2", group1_p4210, SRC_PERIL1, 24, 4), + MUX(0, "mout_clkout_top", clkout_top_p4210, CLKOUT_CMU_TOP, 0, 5), + + MUX(0, "mout_pwi", mout_pwi_p4210, SRC_DMC, 16, 4), + MUX(0, "mout_clkout_dmc", clkout_dmc_p4210, CLKOUT_CMU_DMC, 0, 5), + + MUX(0, "mout_clkout_cpu", clkout_cpu_p4210, CLKOUT_CMU_CPU, 0, 5), }; /* list of mux clocks supported in exynos4x12 soc */ static struct samsung_mux_clock exynos4x12_mux_clks[] __initdata = { + MUX(0, "mout_mpll_user_l", mout_mpll_p, SRC_LEFTBUS, 4, 1), + MUX(0, "mout_gdl", mout_gdl_p4x12, SRC_LEFTBUS, 0, 1), + MUX(0, "mout_clkout_leftbus", clkout_left_p4x12, + CLKOUT_CMU_LEFTBUS, 0, 5), + + MUX(0, "mout_mpll_user_r", mout_mpll_p, SRC_RIGHTBUS, 4, 1), + MUX(0, "mout_gdr", mout_gdr_p4x12, SRC_RIGHTBUS, 0, 1), + MUX(0, "mout_clkout_rightbus", clkout_right_p4x12, + CLKOUT_CMU_RIGHTBUS, 0, 5), + MUX(CLK_MOUT_MPLL_USER_C, "mout_mpll_user_c", mout_mpll_user_p4x12, SRC_CPU, 24, 1), + MUX(0, "mout_clkout_cpu", clkout_cpu_p4x12, CLKOUT_CMU_CPU, 0, 5), + MUX(0, "mout_aclk266_gps", aclk_p4412, SRC_TOP1, 4, 1), MUX(0, "mout_aclk400_mcuisp", aclk_p4412, SRC_TOP1, 8, 1), MUX(CLK_MOUT_MPLL_USER_T, "mout_mpll_user_t", mout_mpll_user_p4x12, @@ -531,6 +656,7 @@ static struct samsung_mux_clock exynos4x12_mux_clks[] __initdata = { MUX(CLK_SCLK_MPLL, "sclk_mpll", mout_mpll_p, SRC_DMC, 12, 1), MUX(CLK_SCLK_VPLL, "sclk_vpll", mout_vpll_p, SRC_TOP0, 8, 1), MUX(CLK_MOUT_CORE, "mout_core", mout_core_p4x12, SRC_CPU, 16, 1), + MUX(0, "mout_hpm", mout_core_p4x12, SRC_CPU, 20, 1), MUX(CLK_MOUT_FIMC0, "mout_fimc0", group1_p4x12, SRC_CAM, 0, 4), MUX(CLK_MOUT_FIMC1, "mout_fimc1", group1_p4x12, SRC_CAM, 4, 4), MUX(CLK_MOUT_FIMC2, "mout_fimc2", group1_p4x12, SRC_CAM, 8, 4), @@ -565,15 +691,39 @@ static struct samsung_mux_clock exynos4x12_mux_clks[] __initdata = { MUX(0, "mout_spi0_isp", group1_p4x12, E4X12_SRC_ISP, 4, 4), MUX(0, "mout_spi1_isp", group1_p4x12, E4X12_SRC_ISP, 8, 4), MUX(0, "mout_uart_isp", group1_p4x12, E4X12_SRC_ISP, 12, 4), + MUX(0, "mout_clkout_top", clkout_top_p4x12, CLKOUT_CMU_TOP, 0, 5), + + MUX(0, "mout_c2c", sclk_ampll_p4210, SRC_DMC, 0, 1), + MUX(0, "mout_pwi", mout_pwi_p4x12, SRC_DMC, 16, 4), MUX(0, "mout_g2d0", sclk_ampll_p4210, SRC_DMC, 20, 1), MUX(0, "mout_g2d1", sclk_evpll_p, SRC_DMC, 24, 1), MUX(0, "mout_g2d", mout_g2d_p, SRC_DMC, 28, 1), + MUX(0, "mout_clkout_dmc", clkout_dmc_p4x12, CLKOUT_CMU_DMC, 0, 5), }; /* list of divider clocks supported in all exynos4 soc's */ static struct samsung_div_clock exynos4_div_clks[] __initdata = { + DIV(0, "div_gdl", "mout_gdl", DIV_LEFTBUS, 0, 3), + DIV(0, "div_gpl", "div_gdl", DIV_LEFTBUS, 4, 3), + DIV(0, "div_clkout_leftbus", "mout_clkout_leftbus", + CLKOUT_CMU_LEFTBUS, 8, 6), + + DIV(0, "div_gdr", "mout_gdr", DIV_RIGHTBUS, 0, 3), + DIV(0, "div_gpr", "div_gdr", DIV_RIGHTBUS, 4, 3), + DIV(0, "div_clkout_rightbus", "mout_clkout_rightbus", + CLKOUT_CMU_RIGHTBUS, 8, 6), + DIV(0, "div_core", "mout_core", DIV_CPU0, 0, 3), + DIV(0, "div_corem0", "div_core2", DIV_CPU0, 4, 3), + DIV(0, "div_corem1", "div_core2", DIV_CPU0, 8, 3), + DIV(0, "div_periph", "div_core2", DIV_CPU0, 12, 3), + DIV(0, "div_atb", "mout_core", DIV_CPU0, 16, 3), + DIV(0, "div_pclk_dbg", "div_atb", DIV_CPU0, 20, 3), DIV(0, "div_core2", "div_core", DIV_CPU0, 28, 3), + DIV(0, "div_copy", "mout_hpm", DIV_CPU1, 0, 3), + DIV(0, "div_hpm", "div_copy", DIV_CPU1, 4, 3), + DIV(0, "div_clkout_cpu", "mout_clkout_cpu", CLKOUT_CMU_CPU, 8, 6), + DIV(0, "div_fimc0", "mout_fimc0", DIV_CAM, 0, 4), DIV(0, "div_fimc1", "mout_fimc1", DIV_CAM, 4, 4), DIV(0, "div_fimc2", "mout_fimc2", DIV_CAM, 8, 4), @@ -631,6 +781,16 @@ static struct samsung_div_clock exynos4_div_clks[] __initdata = { CLK_SET_RATE_PARENT, 0), DIV_F(0, "div_mmc_pre3", "div_mmc3", DIV_FSYS2, 24, 8, CLK_SET_RATE_PARENT, 0), + DIV(0, "div_clkout_top", "mout_clkout_top", CLKOUT_CMU_TOP, 8, 6), + + DIV(0, "div_acp", "mout_dmc_bus", DIV_DMC0, 0, 3), + DIV(0, "div_acp_pclk", "div_acp", DIV_DMC0, 4, 3), + DIV(0, "div_dphy", "mout_dphy", DIV_DMC0, 8, 3), + DIV(0, "div_dmc", "mout_dmc_bus", DIV_DMC0, 12, 3), + DIV(0, "div_dmcd", "div_dmc", DIV_DMC0, 16, 3), + DIV(0, "div_dmcp", "div_dmcd", DIV_DMC0, 20, 3), + DIV(0, "div_pwi", "mout_pwi", DIV_DMC1, 8, 4), + DIV(0, "div_clkout_dmc", "mout_clkout_dmc", CLKOUT_CMU_DMC, 8, 6), }; /* list of divider clocks supported in exynos4210 soc */ @@ -671,6 +831,8 @@ static struct samsung_div_clock exynos4x12_div_clks[] __initdata = { DIV_F(CLK_DIV_MCUISP1, "div_mcuisp1", "div_mcuisp0", E4X12_DIV_ISP1, 8, 3, CLK_GET_RATE_NOCACHE, 0), DIV(CLK_SCLK_FIMG2D, "sclk_fimg2d", "mout_g2d", DIV_DMC1, 0, 4), + DIV(0, "div_c2c", "mout_c2c", DIV_DMC1, 4, 3), + DIV(0, "div_c2c_aclk", "div_c2c", DIV_DMC1, 12, 3), }; /* list of gate clocks supported in all exynos4 soc's */ @@ -680,6 +842,8 @@ static struct samsung_gate_clock exynos4_gate_clks[] __initdata = { * the device name and clock alias names specified below for some * of the clocks can be removed. */ + GATE(CLK_PPMULEFT, "ppmuleft", "aclk200", GATE_IP_LEFTBUS, 1, 0, 0), + GATE(CLK_PPMURIGHT, "ppmuright", "aclk200", GATE_IP_RIGHTBUS, 1, 0, 0), GATE(CLK_SCLK_HDMI, "sclk_hdmi", "mout_hdmi", SRC_MASK_TV, 0, 0, 0), GATE(CLK_SCLK_SPDIF, "sclk_spdif", "mout_spdif", SRC_MASK_PERIL1, 8, 0, 0), @@ -695,11 +859,13 @@ static struct samsung_gate_clock exynos4_gate_clks[] __initdata = { GATE(CLK_SROMC, "sromc", "aclk133", GATE_IP_FSYS, 11, 0, 0), GATE(CLK_SCLK_G3D, "sclk_g3d", "div_g3d", GATE_IP_G3D, 0, CLK_SET_RATE_PARENT, 0), + GATE(CLK_PPMUG3D, "ppmug3d", "aclk200", GATE_IP_G3D, 1, 0, 0), GATE(CLK_USB_DEVICE, "usb_device", "aclk133", GATE_IP_FSYS, 13, 0, 0), GATE(CLK_ONENAND, "onenand", "aclk133", GATE_IP_FSYS, 15, 0, 0), GATE(CLK_NFCON, "nfcon", "aclk133", GATE_IP_FSYS, 16, 0, 0), GATE(CLK_GPS, "gps", "aclk133", GATE_IP_GPS, 0, 0, 0), GATE(CLK_SMMU_GPS, "smmu_gps", "aclk133", GATE_IP_GPS, 1, 0, 0), + GATE(CLK_PPMUGPS, "ppmugps", "aclk200", GATE_IP_GPS, 2, 0, 0), GATE(CLK_SLIMBUS, "slimbus", "aclk100", GATE_IP_PERIL, 25, 0, 0), GATE(CLK_SCLK_CAM0, "sclk_cam0", "div_cam0", GATE_SCLK_CAM, 4, CLK_SET_RATE_PARENT, 0), @@ -781,19 +947,24 @@ static struct samsung_gate_clock exynos4_gate_clks[] __initdata = { 0, 0), GATE(CLK_SMMU_JPEG, "smmu_jpeg", "aclk160", GATE_IP_CAM, 11, 0, 0), + GATE(CLK_PPMUCAMIF, "ppmucamif", "aclk160", GATE_IP_CAM, 16, 0, 0), GATE(CLK_PIXELASYNCM0, "pxl_async0", "aclk160", GATE_IP_CAM, 17, 0, 0), GATE(CLK_PIXELASYNCM1, "pxl_async1", "aclk160", GATE_IP_CAM, 18, 0, 0), GATE(CLK_SMMU_TV, "smmu_tv", "aclk160", GATE_IP_TV, 4, 0, 0), + GATE(CLK_PPMUTV, "ppmutv", "aclk160", GATE_IP_TV, 5, 0, 0), GATE(CLK_MFC, "mfc", "aclk100", GATE_IP_MFC, 0, 0, 0), GATE(CLK_SMMU_MFCL, "smmu_mfcl", "aclk100", GATE_IP_MFC, 1, 0, 0), GATE(CLK_SMMU_MFCR, "smmu_mfcr", "aclk100", GATE_IP_MFC, 2, 0, 0), + GATE(CLK_PPMUMFC_L, "ppmumfc_l", "aclk100", GATE_IP_MFC, 3, 0, 0), + GATE(CLK_PPMUMFC_R, "ppmumfc_r", "aclk100", GATE_IP_MFC, 4, 0, 0), GATE(CLK_FIMD0, "fimd0", "aclk160", GATE_IP_LCD0, 0, 0, 0), GATE(CLK_SMMU_FIMD0, "smmu_fimd0", "aclk160", GATE_IP_LCD0, 4, 0, 0), + GATE(CLK_PPMULCD0, "ppmulcd0", "aclk160", GATE_IP_LCD0, 5, 0, 0), GATE(CLK_PDMA0, "pdma0", "aclk133", GATE_IP_FSYS, 0, 0, 0), GATE(CLK_PDMA1, "pdma1", "aclk133", GATE_IP_FSYS, 1, @@ -806,6 +977,7 @@ static struct samsung_gate_clock exynos4_gate_clks[] __initdata = { 0, 0), GATE(CLK_SDMMC3, "sdmmc3", "aclk133", GATE_IP_FSYS, 8, 0, 0), + GATE(CLK_PPMUFILE, "ppmufile", "aclk133", GATE_IP_FSYS, 17, 0, 0), GATE(CLK_UART0, "uart0", "aclk100", GATE_IP_PERIL, 0, 0, 0), GATE(CLK_UART1, "uart1", "aclk100", GATE_IP_PERIL, 1, @@ -852,6 +1024,21 @@ static struct samsung_gate_clock exynos4_gate_clks[] __initdata = { 0, 0), GATE(CLK_AC97, "ac97", "aclk100", GATE_IP_PERIL, 27, 0, 0), + GATE(CLK_PPMUDMC0, "ppmudmc0", "aclk133", GATE_IP_DMC, 8, 0, 0), + GATE(CLK_PPMUDMC1, "ppmudmc1", "aclk133", GATE_IP_DMC, 9, 0, 0), + GATE(CLK_PPMUCPU, "ppmucpu", "aclk133", GATE_IP_DMC, 10, 0, 0), + GATE(CLK_PPMUACP, "ppmuacp", "aclk133", GATE_IP_DMC, 16, 0, 0), + + GATE(CLK_OUT_LEFTBUS, "clkout_leftbus", "div_clkout_leftbus", + CLKOUT_CMU_LEFTBUS, 16, CLK_SET_RATE_PARENT, 0), + GATE(CLK_OUT_RIGHTBUS, "clkout_rightbus", "div_clkout_rightbus", + CLKOUT_CMU_RIGHTBUS, 16, CLK_SET_RATE_PARENT, 0), + GATE(CLK_OUT_TOP, "clkout_top", "div_clkout_top", + CLKOUT_CMU_TOP, 16, CLK_SET_RATE_PARENT, 0), + GATE(CLK_OUT_DMC, "clkout_dmc", "div_clkout_dmc", + CLKOUT_CMU_DMC, 16, CLK_SET_RATE_PARENT, 0), + GATE(CLK_OUT_CPU, "clkout_cpu", "div_clkout_cpu", + CLKOUT_CMU_CPU, 16, CLK_SET_RATE_PARENT, 0), }; /* list of gate clocks supported in exynos4210 soc */ @@ -863,6 +1050,9 @@ static struct samsung_gate_clock exynos4210_gate_clks[] __initdata = { GATE(CLK_SMMU_G2D, "smmu_g2d", "aclk200", E4210_GATE_IP_IMAGE, 3, 0, 0), GATE(CLK_SMMU_MDMA, "smmu_mdma", "aclk200", E4210_GATE_IP_IMAGE, 5, 0, 0), + GATE(CLK_PPMUIMAGE, "ppmuimage", "aclk200", E4210_GATE_IP_IMAGE, 9, 0, + 0), + GATE(CLK_PPMULCD1, "ppmulcd1", "aclk160", E4210_GATE_IP_LCD1, 5, 0, 0), GATE(CLK_PCIE_PHY, "pcie_phy", "aclk133", GATE_IP_FSYS, 2, 0, 0), GATE(CLK_SATA_PHY, "sata_phy", "aclk133", GATE_IP_FSYS, 3, 0, 0), GATE(CLK_SATA, "sata", "aclk133", GATE_IP_FSYS, 10, 0, 0), @@ -906,6 +1096,8 @@ static struct samsung_gate_clock exynos4x12_gate_clks[] __initdata = { GATE(CLK_MDMA, "mdma", "aclk200", E4X12_GATE_IP_IMAGE, 2, 0, 0), GATE(CLK_SMMU_MDMA, "smmu_mdma", "aclk200", E4X12_GATE_IP_IMAGE, 5, 0, 0), + GATE(CLK_PPMUIMAGE, "ppmuimage", "aclk200", E4X12_GATE_IP_IMAGE, 9, 0, + 0), GATE(CLK_MIPI_HSI, "mipi_hsi", "aclk133", GATE_IP_FSYS, 10, 0, 0), GATE(CLK_CHIPID, "chipid", "aclk100", E4X12_GATE_IP_PERIR, 0, 0, 0), GATE(CLK_SYSREG, "sysreg", "aclk100", E4X12_GATE_IP_PERIR, 1, @@ -1062,7 +1254,7 @@ static void __init exynos4_clk_register_finpll(struct samsung_clk_provider *ctx) } -static struct of_device_id ext_clk_match[] __initdata = { +static const struct of_device_id ext_clk_match[] __initconst = { { .compatible = "samsung,clock-xxti", .data = (void *)0, }, { .compatible = "samsung,clock-xusbxti", .data = (void *)1, }, {}, @@ -1164,6 +1356,32 @@ static struct samsung_pll_clock exynos4x12_plls[nr_plls] __initdata = { VPLL_LOCK, VPLL_CON0, NULL), }; +static void __init exynos4_core_down_clock(enum exynos4_soc soc) +{ + unsigned int tmp; + + /* + * Enable arm clock down (in idle) and set arm divider + * ratios in WFI/WFE state. + */ + tmp = (PWR_CTRL1_CORE2_DOWN_RATIO(7) | PWR_CTRL1_CORE1_DOWN_RATIO(7) | + PWR_CTRL1_DIV2_DOWN_EN | PWR_CTRL1_DIV1_DOWN_EN | + PWR_CTRL1_USE_CORE1_WFE | PWR_CTRL1_USE_CORE0_WFE | + PWR_CTRL1_USE_CORE1_WFI | PWR_CTRL1_USE_CORE0_WFI); + /* On Exynos4412 enable it also on core 2 and 3 */ + if (num_possible_cpus() == 4) + tmp |= PWR_CTRL1_USE_CORE3_WFE | PWR_CTRL1_USE_CORE2_WFE | + PWR_CTRL1_USE_CORE3_WFI | PWR_CTRL1_USE_CORE2_WFI; + __raw_writel(tmp, reg_base + PWR_CTRL1); + + /* + * Disable the clock up feature on Exynos4x12, in case it was + * enabled by bootloader. + */ + if (exynos4_soc == EXYNOS4X12) + __raw_writel(0x0, reg_base + E4X12_PWR_CTRL2); +} + /* register exynos4 clocks */ static void __init exynos4_clk_init(struct device_node *np, enum exynos4_soc soc) @@ -1224,6 +1442,8 @@ static void __init exynos4_clk_init(struct device_node *np, ARRAY_SIZE(exynos4_div_clks)); samsung_clk_register_gate(ctx, exynos4_gate_clks, ARRAY_SIZE(exynos4_gate_clks)); + samsung_clk_register_fixed_factor(ctx, exynos4_fixed_factor_clks, + ARRAY_SIZE(exynos4_fixed_factor_clks)); if (exynos4_soc == EXYNOS4210) { samsung_clk_register_fixed_rate(ctx, exynos4210_fixed_rate_clks, @@ -1236,6 +1456,9 @@ static void __init exynos4_clk_init(struct device_node *np, ARRAY_SIZE(exynos4210_gate_clks)); samsung_clk_register_alias(ctx, exynos4210_aliases, ARRAY_SIZE(exynos4210_aliases)); + samsung_clk_register_fixed_factor(ctx, + exynos4210_fixed_factor_clks, + ARRAY_SIZE(exynos4210_fixed_factor_clks)); } else { samsung_clk_register_mux(ctx, exynos4x12_mux_clks, ARRAY_SIZE(exynos4x12_mux_clks)); @@ -1245,13 +1468,19 @@ static void __init exynos4_clk_init(struct device_node *np, ARRAY_SIZE(exynos4x12_gate_clks)); samsung_clk_register_alias(ctx, exynos4x12_aliases, ARRAY_SIZE(exynos4x12_aliases)); + samsung_clk_register_fixed_factor(ctx, + exynos4x12_fixed_factor_clks, + ARRAY_SIZE(exynos4x12_fixed_factor_clks)); } samsung_clk_register_alias(ctx, exynos4_aliases, ARRAY_SIZE(exynos4_aliases)); + exynos4_core_down_clock(soc); exynos4_clk_sleep_init(); + samsung_clk_of_add_provider(np, ctx); + pr_info("%s clocks: sclk_apll = %ld, sclk_mpll = %ld\n" "\tsclk_epll = %ld, sclk_vpll = %ld, arm_clk = %ld\n", exynos4_soc == EXYNOS4210 ? "Exynos4210" : "Exynos4x12", diff --git a/drivers/clk/samsung/clk-exynos5250.c b/drivers/clk/samsung/clk-exynos5250.c index 184f64293b26..70ec3d2608a1 100644 --- a/drivers/clk/samsung/clk-exynos5250.c +++ b/drivers/clk/samsung/clk-exynos5250.c @@ -748,7 +748,7 @@ static struct samsung_pll_clock exynos5250_plls[nr_plls] __initdata = { VPLL_LOCK, VPLL_CON0, NULL), }; -static struct of_device_id ext_clk_match[] __initdata = { +static const struct of_device_id ext_clk_match[] __initconst = { { .compatible = "samsung,clock-xxti", .data = (void *)0, }, { }, }; @@ -820,6 +820,8 @@ static void __init exynos5250_clk_init(struct device_node *np) exynos5250_clk_sleep_init(); + samsung_clk_of_add_provider(np, ctx); + pr_info("Exynos5250: clock setup completed, armclk=%ld\n", _get_rate("div_arm2")); } diff --git a/drivers/clk/samsung/clk-exynos5260.c b/drivers/clk/samsung/clk-exynos5260.c index 64596ba58df1..ce3de97e5f11 100644 --- a/drivers/clk/samsung/clk-exynos5260.c +++ b/drivers/clk/samsung/clk-exynos5260.c @@ -206,6 +206,8 @@ void __init exynos5260_cmu_register_one(struct device_node *np, if (cmu->clk_regs) exynos5260_clk_sleep_init(reg_base, cmu->clk_regs, cmu->nr_clk_regs); + + samsung_clk_of_add_provider(np, ctx); } diff --git a/drivers/clk/samsung/clk-exynos5410.c b/drivers/clk/samsung/clk-exynos5410.c index c9505ab9ee70..231475bc2b99 100644 --- a/drivers/clk/samsung/clk-exynos5410.c +++ b/drivers/clk/samsung/clk-exynos5410.c @@ -204,6 +204,8 @@ static void __init exynos5410_clk_init(struct device_node *np) samsung_clk_register_gate(ctx, exynos5410_gate_clks, ARRAY_SIZE(exynos5410_gate_clks)); + samsung_clk_of_add_provider(np, ctx); + pr_debug("Exynos5410: clock setup completed.\n"); } CLK_OF_DECLARE(exynos5410_clk, "samsung,exynos5410-clock", exynos5410_clk_init); diff --git a/drivers/clk/samsung/clk-exynos5420.c b/drivers/clk/samsung/clk-exynos5420.c index a4e6cc782e5c..848d602efc06 100644 --- a/drivers/clk/samsung/clk-exynos5420.c +++ b/drivers/clk/samsung/clk-exynos5420.c @@ -28,6 +28,7 @@ #define GATE_BUS_CPU 0x700 #define GATE_SCLK_CPU 0x800 #define CLKOUT_CMU_CPU 0xa00 +#define SRC_MASK_CPERI 0x4300 #define GATE_IP_G2D 0x8800 #define CPLL_LOCK 0x10020 #define DPLL_LOCK 0x10030 @@ -70,6 +71,8 @@ #define SRC_TOP11 0x10284 #define SRC_TOP12 0x10288 #define SRC_TOP13 0x1028c /* 5800 specific */ +#define SRC_MASK_TOP0 0x10300 +#define SRC_MASK_TOP1 0x10304 #define SRC_MASK_TOP2 0x10308 #define SRC_MASK_TOP7 0x1031c #define SRC_MASK_DISP10 0x1032c @@ -77,6 +80,7 @@ #define SRC_MASK_FSYS 0x10340 #define SRC_MASK_PERIC0 0x10350 #define SRC_MASK_PERIC1 0x10354 +#define SRC_MASK_ISP 0x10370 #define DIV_TOP0 0x10500 #define DIV_TOP1 0x10504 #define DIV_TOP2 0x10508 @@ -98,6 +102,7 @@ #define DIV2_RATIO0 0x10590 #define DIV4_RATIO 0x105a0 #define GATE_BUS_TOP 0x10700 +#define GATE_BUS_DISP1 0x10728 #define GATE_BUS_GEN 0x1073c #define GATE_BUS_FSYS0 0x10740 #define GATE_BUS_FSYS2 0x10748 @@ -190,6 +195,10 @@ static unsigned long exynos5x_clk_regs[] __initdata = { SRC_MASK_FSYS, SRC_MASK_PERIC0, SRC_MASK_PERIC1, + SRC_MASK_TOP0, + SRC_MASK_TOP1, + SRC_MASK_MAU, + SRC_MASK_ISP, SRC_ISP, DIV_TOP0, DIV_TOP1, @@ -208,6 +217,7 @@ static unsigned long exynos5x_clk_regs[] __initdata = { SCLK_DIV_ISP1, DIV2_RATIO0, DIV4_RATIO, + GATE_BUS_DISP1, GATE_BUS_TOP, GATE_BUS_GEN, GATE_BUS_FSYS0, @@ -249,6 +259,22 @@ static unsigned long exynos5800_clk_regs[] __initdata = { GATE_IP_CAM, }; +static const struct samsung_clk_reg_dump exynos5420_set_clksrc[] = { + { .offset = SRC_MASK_CPERI, .value = 0xffffffff, }, + { .offset = SRC_MASK_TOP0, .value = 0x11111111, }, + { .offset = SRC_MASK_TOP1, .value = 0x11101111, }, + { .offset = SRC_MASK_TOP2, .value = 0x11111110, }, + { .offset = SRC_MASK_TOP7, .value = 0x00111100, }, + { .offset = SRC_MASK_DISP10, .value = 0x11111110, }, + { .offset = SRC_MASK_MAU, .value = 0x10000000, }, + { .offset = SRC_MASK_FSYS, .value = 0x11111110, }, + { .offset = SRC_MASK_PERIC0, .value = 0x11111110, }, + { .offset = SRC_MASK_PERIC1, .value = 0x11111100, }, + { .offset = SRC_MASK_ISP, .value = 0x11111000, }, + { .offset = GATE_BUS_DISP1, .value = 0xffffffff, }, + { .offset = GATE_IP_PERIC, .value = 0xffffffff, }, +}; + static int exynos5420_clk_suspend(void) { samsung_clk_save(reg_base, exynos5x_save, @@ -258,6 +284,9 @@ static int exynos5420_clk_suspend(void) samsung_clk_save(reg_base, exynos5800_save, ARRAY_SIZE(exynos5800_clk_regs)); + samsung_clk_restore(reg_base, exynos5420_set_clksrc, + ARRAY_SIZE(exynos5420_set_clksrc)); + return 0; } @@ -1169,6 +1198,28 @@ static struct samsung_gate_clock exynos5x_gate_clks[] __initdata = { GATE(CLK_G3D, "g3d", "mout_user_aclk_g3d", GATE_IP_G3D, 9, 0, 0), }; +static const struct samsung_pll_rate_table exynos5420_pll2550x_24mhz_tbl[] = { + PLL_35XX_RATE(2000000000, 250, 3, 0), + PLL_35XX_RATE(1900000000, 475, 6, 0), + PLL_35XX_RATE(1800000000, 225, 3, 0), + PLL_35XX_RATE(1700000000, 425, 6, 0), + PLL_35XX_RATE(1600000000, 200, 3, 0), + PLL_35XX_RATE(1500000000, 250, 4, 0), + PLL_35XX_RATE(1400000000, 175, 3, 0), + PLL_35XX_RATE(1300000000, 325, 6, 0), + PLL_35XX_RATE(1200000000, 200, 2, 1), + PLL_35XX_RATE(1100000000, 275, 3, 1), + PLL_35XX_RATE(1000000000, 250, 3, 1), + PLL_35XX_RATE(900000000, 150, 2, 1), + PLL_35XX_RATE(800000000, 200, 3, 1), + PLL_35XX_RATE(700000000, 175, 3, 1), + PLL_35XX_RATE(600000000, 200, 2, 2), + PLL_35XX_RATE(500000000, 250, 3, 2), + PLL_35XX_RATE(400000000, 200, 3, 2), + PLL_35XX_RATE(300000000, 200, 2, 3), + PLL_35XX_RATE(200000000, 200, 3, 3), +}; + static struct samsung_pll_clock exynos5x_plls[nr_plls] __initdata = { [apll] = PLL(pll_2550, CLK_FOUT_APLL, "fout_apll", "fin_pll", APLL_LOCK, APLL_CON0, NULL), @@ -1194,7 +1245,7 @@ static struct samsung_pll_clock exynos5x_plls[nr_plls] __initdata = { KPLL_CON0, NULL), }; -static struct of_device_id ext_clk_match[] __initdata = { +static const struct of_device_id ext_clk_match[] __initconst = { { .compatible = "samsung,exynos5420-oscclk", .data = (void *)0, }, { }, }; @@ -1222,6 +1273,12 @@ static void __init exynos5x_clk_init(struct device_node *np, samsung_clk_of_register_fixed_ext(ctx, exynos5x_fixed_rate_ext_clks, ARRAY_SIZE(exynos5x_fixed_rate_ext_clks), ext_clk_match); + + if (_get_rate("fin_pll") == 24 * MHZ) { + exynos5x_plls[apll].rate_table = exynos5420_pll2550x_24mhz_tbl; + exynos5x_plls[kpll].rate_table = exynos5420_pll2550x_24mhz_tbl; + } + samsung_clk_register_pll(ctx, exynos5x_plls, ARRAY_SIZE(exynos5x_plls), reg_base); samsung_clk_register_fixed_rate(ctx, exynos5x_fixed_rate_clks, @@ -1253,6 +1310,8 @@ static void __init exynos5x_clk_init(struct device_node *np, } exynos5420_clk_sleep_init(); + + samsung_clk_of_add_provider(np, ctx); } static void __init exynos5420_clk_init(struct device_node *np) diff --git a/drivers/clk/samsung/clk-exynos5440.c b/drivers/clk/samsung/clk-exynos5440.c index 647f1440aa6a..00d1d00a41de 100644 --- a/drivers/clk/samsung/clk-exynos5440.c +++ b/drivers/clk/samsung/clk-exynos5440.c @@ -84,7 +84,7 @@ static struct samsung_gate_clock exynos5440_gate_clks[] __initdata = { GATE(CLK_CS250_O, "cs250_o", "cs250", CLKEN_OV_VAL, 19, 0, 0), }; -static struct of_device_id ext_clk_match[] __initdata = { +static const struct of_device_id ext_clk_match[] __initconst = { { .compatible = "samsung,clock-xtal", .data = (void *)0, }, {}, }; @@ -123,6 +123,8 @@ static void __init exynos5440_clk_init(struct device_node *np) samsung_clk_register_gate(ctx, exynos5440_gate_clks, ARRAY_SIZE(exynos5440_gate_clks)); + samsung_clk_of_add_provider(np, ctx); + pr_info("Exynos5440: arm_clk = %ldHz\n", _get_rate("arm_clk")); pr_info("exynos5440 clock initialization complete\n"); } diff --git a/drivers/clk/samsung/clk-s3c2410.c b/drivers/clk/samsung/clk-s3c2410.c index 140f4733c02e..5d2f03461bc5 100644 --- a/drivers/clk/samsung/clk-s3c2410.c +++ b/drivers/clk/samsung/clk-s3c2410.c @@ -466,6 +466,8 @@ void __init s3c2410_common_clk_init(struct device_node *np, unsigned long xti_f, } s3c2410_clk_sleep_init(); + + samsung_clk_of_add_provider(np, ctx); } static void __init s3c2410_clk_init(struct device_node *np) diff --git a/drivers/clk/samsung/clk-s3c2412.c b/drivers/clk/samsung/clk-s3c2412.c index 23e4313f625e..34af09f6a155 100644 --- a/drivers/clk/samsung/clk-s3c2412.c +++ b/drivers/clk/samsung/clk-s3c2412.c @@ -265,6 +265,8 @@ void __init s3c2412_common_clk_init(struct device_node *np, unsigned long xti_f, ARRAY_SIZE(s3c2412_aliases)); s3c2412_clk_sleep_init(); + + samsung_clk_of_add_provider(np, ctx); } static void __init s3c2412_clk_init(struct device_node *np) diff --git a/drivers/clk/samsung/clk-s3c2443.c b/drivers/clk/samsung/clk-s3c2443.c index c4bbdabebaa4..c92f853fca9f 100644 --- a/drivers/clk/samsung/clk-s3c2443.c +++ b/drivers/clk/samsung/clk-s3c2443.c @@ -445,6 +445,8 @@ void __init s3c2443_common_clk_init(struct device_node *np, unsigned long xti_f, } s3c2443_clk_sleep_init(); + + samsung_clk_of_add_provider(np, ctx); } static void __init s3c2416_clk_init(struct device_node *np) diff --git a/drivers/clk/samsung/clk-s3c64xx.c b/drivers/clk/samsung/clk-s3c64xx.c index 8889ff1c10fc..0f590e5550cb 100644 --- a/drivers/clk/samsung/clk-s3c64xx.c +++ b/drivers/clk/samsung/clk-s3c64xx.c @@ -518,6 +518,8 @@ void __init s3c64xx_clk_init(struct device_node *np, unsigned long xtal_f, ARRAY_SIZE(s3c64xx_clock_aliases)); s3c64xx_clk_sleep_init(); + samsung_clk_of_add_provider(np, ctx); + pr_info("%s clocks: apll = %lu, mpll = %lu\n" "\tepll = %lu, arm_clk = %lu\n", is_s3c6400 ? "S3C6400" : "S3C6410", diff --git a/drivers/clk/samsung/clk.c b/drivers/clk/samsung/clk.c index 49629c71c9e7..deab84d9f37d 100644 --- a/drivers/clk/samsung/clk.c +++ b/drivers/clk/samsung/clk.c @@ -53,7 +53,6 @@ struct samsung_clk_provider *__init samsung_clk_init(struct device_node *np, { struct samsung_clk_provider *ctx; struct clk **clk_table; - int ret; int i; ctx = kzalloc(sizeof(struct samsung_clk_provider), GFP_KERNEL); @@ -72,17 +71,19 @@ struct samsung_clk_provider *__init samsung_clk_init(struct device_node *np, ctx->clk_data.clk_num = nr_clks; spin_lock_init(&ctx->lock); - if (!np) - return ctx; - - ret = of_clk_add_provider(np, of_clk_src_onecell_get, - &ctx->clk_data); - if (ret) - panic("could not register clock provide\n"); - return ctx; } +void __init samsung_clk_of_add_provider(struct device_node *np, + struct samsung_clk_provider *ctx) +{ + if (np) { + if (of_clk_add_provider(np, of_clk_src_onecell_get, + &ctx->clk_data)) + panic("could not register clk provider\n"); + } +} + /* add a clock instance to the clock lookup table used for dt based lookup */ void samsung_clk_add_lookup(struct samsung_clk_provider *ctx, struct clk *clk, unsigned int id) @@ -284,7 +285,7 @@ void __init samsung_clk_register_gate(struct samsung_clk_provider *ctx, void __init samsung_clk_of_register_fixed_ext(struct samsung_clk_provider *ctx, struct samsung_fixed_rate_clock *fixed_rate_clk, unsigned int nr_fixed_rate_clk, - struct of_device_id *clk_matches) + const struct of_device_id *clk_matches) { const struct of_device_id *match; struct device_node *clk_np; diff --git a/drivers/clk/samsung/clk.h b/drivers/clk/samsung/clk.h index 9693b80d924f..66ab36b5cef1 100644 --- a/drivers/clk/samsung/clk.h +++ b/drivers/clk/samsung/clk.h @@ -327,11 +327,13 @@ struct samsung_pll_clock { extern struct samsung_clk_provider *__init samsung_clk_init( struct device_node *np, void __iomem *base, unsigned long nr_clks); +extern void __init samsung_clk_of_add_provider(struct device_node *np, + struct samsung_clk_provider *ctx); extern void __init samsung_clk_of_register_fixed_ext( struct samsung_clk_provider *ctx, struct samsung_fixed_rate_clock *fixed_rate_clk, unsigned int nr_fixed_rate_clk, - struct of_device_id *clk_matches); + const struct of_device_id *clk_matches); extern void samsung_clk_add_lookup(struct samsung_clk_provider *ctx, struct clk *clk, unsigned int id); diff --git a/drivers/clk/spear/spear1310_clock.c b/drivers/clk/spear/spear1310_clock.c index 65894f7687ed..4daa5977793a 100644 --- a/drivers/clk/spear/spear1310_clock.c +++ b/drivers/clk/spear/spear1310_clock.c @@ -742,19 +742,19 @@ void __init spear1310_clk_init(void __iomem *misc_base, void __iomem *ras_base) clk = clk_register_gate(NULL, "pcie_sata_0_clk", "ahb_clk", 0, SPEAR1310_PERIP1_CLK_ENB, SPEAR1310_PCIE_SATA_0_CLK_ENB, 0, &_lock); - clk_register_clkdev(clk, NULL, "dw_pcie.0"); + clk_register_clkdev(clk, NULL, "b1000000.pcie"); clk_register_clkdev(clk, NULL, "b1000000.ahci"); clk = clk_register_gate(NULL, "pcie_sata_1_clk", "ahb_clk", 0, SPEAR1310_PERIP1_CLK_ENB, SPEAR1310_PCIE_SATA_1_CLK_ENB, 0, &_lock); - clk_register_clkdev(clk, NULL, "dw_pcie.1"); + clk_register_clkdev(clk, NULL, "b1800000.pcie"); clk_register_clkdev(clk, NULL, "b1800000.ahci"); clk = clk_register_gate(NULL, "pcie_sata_2_clk", "ahb_clk", 0, SPEAR1310_PERIP1_CLK_ENB, SPEAR1310_PCIE_SATA_2_CLK_ENB, 0, &_lock); - clk_register_clkdev(clk, NULL, "dw_pcie.2"); + clk_register_clkdev(clk, NULL, "b4000000.pcie"); clk_register_clkdev(clk, NULL, "b4000000.ahci"); clk = clk_register_gate(NULL, "sysram0_clk", "ahb_clk", 0, diff --git a/drivers/clk/spear/spear1340_clock.c b/drivers/clk/spear/spear1340_clock.c index fe835c1845fe..5a5c6648308d 100644 --- a/drivers/clk/spear/spear1340_clock.c +++ b/drivers/clk/spear/spear1340_clock.c @@ -839,7 +839,7 @@ void __init spear1340_clk_init(void __iomem *misc_base) clk = clk_register_gate(NULL, "pcie_sata_clk", "ahb_clk", 0, SPEAR1340_PERIP1_CLK_ENB, SPEAR1340_PCIE_SATA_CLK_ENB, 0, &_lock); - clk_register_clkdev(clk, NULL, "dw_pcie"); + clk_register_clkdev(clk, NULL, "b1000000.pcie"); clk_register_clkdev(clk, NULL, "b1000000.ahci"); clk = clk_register_gate(NULL, "sysram0_clk", "ahb_clk", 0, diff --git a/drivers/clk/st/Makefile b/drivers/clk/st/Makefile index c7455ffdbdf7..ede7b2f13092 100644 --- a/drivers/clk/st/Makefile +++ b/drivers/clk/st/Makefile @@ -1 +1 @@ -obj-y += clkgen-mux.o clkgen-pll.o clkgen-fsyn.o +obj-y += clkgen-mux.o clkgen-pll.o clkgen-fsyn.o clk-flexgen.o diff --git a/drivers/clk/st/clk-flexgen.c b/drivers/clk/st/clk-flexgen.c new file mode 100644 index 000000000000..2282cef9f2ff --- /dev/null +++ b/drivers/clk/st/clk-flexgen.c @@ -0,0 +1,331 @@ +/* + * clk-flexgen.c + * + * Copyright (C) ST-Microelectronics SA 2013 + * Author: Maxime Coquelin <maxime.coquelin@st.com> for ST-Microelectronics. + * License terms: GNU General Public License (GPL), version 2 */ + +#include <linux/clk-provider.h> +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/io.h> +#include <linux/err.h> +#include <linux/string.h> +#include <linux/of.h> +#include <linux/of_address.h> + +struct flexgen { + struct clk_hw hw; + + /* Crossbar */ + struct clk_mux mux; + /* Pre-divisor's gate */ + struct clk_gate pgate; + /* Pre-divisor */ + struct clk_divider pdiv; + /* Final divisor's gate */ + struct clk_gate fgate; + /* Final divisor */ + struct clk_divider fdiv; +}; + +#define to_flexgen(_hw) container_of(_hw, struct flexgen, hw) + +static int flexgen_enable(struct clk_hw *hw) +{ + struct flexgen *flexgen = to_flexgen(hw); + struct clk_hw *pgate_hw = &flexgen->pgate.hw; + struct clk_hw *fgate_hw = &flexgen->fgate.hw; + + pgate_hw->clk = hw->clk; + fgate_hw->clk = hw->clk; + + clk_gate_ops.enable(pgate_hw); + + clk_gate_ops.enable(fgate_hw); + + pr_debug("%s: flexgen output enabled\n", __clk_get_name(hw->clk)); + return 0; +} + +static void flexgen_disable(struct clk_hw *hw) +{ + struct flexgen *flexgen = to_flexgen(hw); + struct clk_hw *fgate_hw = &flexgen->fgate.hw; + + /* disable only the final gate */ + fgate_hw->clk = hw->clk; + + clk_gate_ops.disable(fgate_hw); + + pr_debug("%s: flexgen output disabled\n", __clk_get_name(hw->clk)); +} + +static int flexgen_is_enabled(struct clk_hw *hw) +{ + struct flexgen *flexgen = to_flexgen(hw); + struct clk_hw *fgate_hw = &flexgen->fgate.hw; + + fgate_hw->clk = hw->clk; + + if (!clk_gate_ops.is_enabled(fgate_hw)) + return 0; + + return 1; +} + +static u8 flexgen_get_parent(struct clk_hw *hw) +{ + struct flexgen *flexgen = to_flexgen(hw); + struct clk_hw *mux_hw = &flexgen->mux.hw; + + mux_hw->clk = hw->clk; + + return clk_mux_ops.get_parent(mux_hw); +} + +static int flexgen_set_parent(struct clk_hw *hw, u8 index) +{ + struct flexgen *flexgen = to_flexgen(hw); + struct clk_hw *mux_hw = &flexgen->mux.hw; + + mux_hw->clk = hw->clk; + + return clk_mux_ops.set_parent(mux_hw, index); +} + +static inline unsigned long +clk_best_div(unsigned long parent_rate, unsigned long rate) +{ + return parent_rate / rate + ((rate > (2*(parent_rate % rate))) ? 0 : 1); +} + +static long flexgen_round_rate(struct clk_hw *hw, unsigned long rate, + unsigned long *prate) +{ + unsigned long div; + + /* Round div according to exact prate and wished rate */ + div = clk_best_div(*prate, rate); + + if (__clk_get_flags(hw->clk) & CLK_SET_RATE_PARENT) { + *prate = rate * div; + return rate; + } + + return *prate / div; +} + +unsigned long flexgen_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct flexgen *flexgen = to_flexgen(hw); + struct clk_hw *pdiv_hw = &flexgen->pdiv.hw; + struct clk_hw *fdiv_hw = &flexgen->fdiv.hw; + unsigned long mid_rate; + + pdiv_hw->clk = hw->clk; + fdiv_hw->clk = hw->clk; + + mid_rate = clk_divider_ops.recalc_rate(pdiv_hw, parent_rate); + + return clk_divider_ops.recalc_rate(fdiv_hw, mid_rate); +} + +static int flexgen_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct flexgen *flexgen = to_flexgen(hw); + struct clk_hw *pdiv_hw = &flexgen->pdiv.hw; + struct clk_hw *fdiv_hw = &flexgen->fdiv.hw; + unsigned long primary_div = 0; + int ret = 0; + + pdiv_hw->clk = hw->clk; + fdiv_hw->clk = hw->clk; + + primary_div = clk_best_div(parent_rate, rate); + + clk_divider_ops.set_rate(fdiv_hw, parent_rate, parent_rate); + ret = clk_divider_ops.set_rate(pdiv_hw, rate, rate * primary_div); + + return ret; +} + +static const struct clk_ops flexgen_ops = { + .enable = flexgen_enable, + .disable = flexgen_disable, + .is_enabled = flexgen_is_enabled, + .get_parent = flexgen_get_parent, + .set_parent = flexgen_set_parent, + .round_rate = flexgen_round_rate, + .recalc_rate = flexgen_recalc_rate, + .set_rate = flexgen_set_rate, +}; + +struct clk *clk_register_flexgen(const char *name, + const char **parent_names, u8 num_parents, + void __iomem *reg, spinlock_t *lock, u32 idx, + unsigned long flexgen_flags) { + struct flexgen *fgxbar; + struct clk *clk; + struct clk_init_data init; + u32 xbar_shift; + void __iomem *xbar_reg, *fdiv_reg; + + fgxbar = kzalloc(sizeof(struct flexgen), GFP_KERNEL); + if (!fgxbar) + return ERR_PTR(-ENOMEM); + + init.name = name; + init.ops = &flexgen_ops; + init.flags = CLK_IS_BASIC | flexgen_flags; + init.parent_names = parent_names; + init.num_parents = num_parents; + + xbar_reg = reg + 0x18 + (idx & ~0x3); + xbar_shift = (idx % 4) * 0x8; + fdiv_reg = reg + 0x164 + idx * 4; + + /* Crossbar element config */ + fgxbar->mux.lock = lock; + fgxbar->mux.mask = BIT(6) - 1; + fgxbar->mux.reg = xbar_reg; + fgxbar->mux.shift = xbar_shift; + fgxbar->mux.table = NULL; + + + /* Pre-divider's gate config (in xbar register)*/ + fgxbar->pgate.lock = lock; + fgxbar->pgate.reg = xbar_reg; + fgxbar->pgate.bit_idx = xbar_shift + 6; + + /* Pre-divider config */ + fgxbar->pdiv.lock = lock; + fgxbar->pdiv.reg = reg + 0x58 + idx * 4; + fgxbar->pdiv.width = 10; + + /* Final divider's gate config */ + fgxbar->fgate.lock = lock; + fgxbar->fgate.reg = fdiv_reg; + fgxbar->fgate.bit_idx = 6; + + /* Final divider config */ + fgxbar->fdiv.lock = lock; + fgxbar->fdiv.reg = fdiv_reg; + fgxbar->fdiv.width = 6; + + fgxbar->hw.init = &init; + + clk = clk_register(NULL, &fgxbar->hw); + if (IS_ERR(clk)) + kfree(fgxbar); + else + pr_debug("%s: parent %s rate %u\n", + __clk_get_name(clk), + __clk_get_name(clk_get_parent(clk)), + (unsigned int)clk_get_rate(clk)); + return clk; +} + +static const char ** __init flexgen_get_parents(struct device_node *np, + int *num_parents) +{ + const char **parents; + int nparents, i; + + nparents = of_count_phandle_with_args(np, "clocks", "#clock-cells"); + if (WARN_ON(nparents <= 0)) + return NULL; + + parents = kcalloc(nparents, sizeof(const char *), GFP_KERNEL); + if (!parents) + return NULL; + + for (i = 0; i < nparents; i++) + parents[i] = of_clk_get_parent_name(np, i); + + *num_parents = nparents; + return parents; +} + +void __init st_of_flexgen_setup(struct device_node *np) +{ + struct device_node *pnode; + void __iomem *reg; + struct clk_onecell_data *clk_data; + const char **parents; + int num_parents, i; + spinlock_t *rlock = NULL; + unsigned long flex_flags = 0; + + pnode = of_get_parent(np); + if (!pnode) + return; + + reg = of_iomap(pnode, 0); + if (!reg) + return; + + parents = flexgen_get_parents(np, &num_parents); + if (!parents) + return; + + clk_data = kzalloc(sizeof(*clk_data), GFP_KERNEL); + if (!clk_data) + goto err; + + clk_data->clk_num = of_property_count_strings(np , + "clock-output-names"); + if (clk_data->clk_num <= 0) { + pr_err("%s: Failed to get number of output clocks (%d)", + __func__, clk_data->clk_num); + goto err; + } + + clk_data->clks = kcalloc(clk_data->clk_num, sizeof(struct clk *), + GFP_KERNEL); + if (!clk_data->clks) + goto err; + + rlock = kzalloc(sizeof(spinlock_t), GFP_KERNEL); + if (!rlock) + goto err; + + for (i = 0; i < clk_data->clk_num; i++) { + struct clk *clk; + const char *clk_name; + + if (of_property_read_string_index(np, "clock-output-names", + i, &clk_name)) { + break; + } + + /* + * If we read an empty clock name then the output is unused + */ + if (*clk_name == '\0') + continue; + + clk = clk_register_flexgen(clk_name, parents, num_parents, + reg, rlock, i, flex_flags); + + if (IS_ERR(clk)) + goto err; + + clk_data->clks[i] = clk; + } + + kfree(parents); + of_clk_add_provider(np, of_clk_src_onecell_get, clk_data); + + return; + +err: + if (clk_data) + kfree(clk_data->clks); + kfree(clk_data); + kfree(parents); + kfree(rlock); +} +CLK_OF_DECLARE(flexgen, "st,flexgen", st_of_flexgen_setup); diff --git a/drivers/clk/st/clkgen-fsyn.c b/drivers/clk/st/clkgen-fsyn.c index 4f53ee0778d9..af94ed82cfcb 100644 --- a/drivers/clk/st/clkgen-fsyn.c +++ b/drivers/clk/st/clkgen-fsyn.c @@ -41,7 +41,7 @@ struct stm_fs { unsigned long nsdiv; }; -static struct stm_fs fs216c65_rtbl[] = { +static const struct stm_fs fs216c65_rtbl[] = { { .mdiv = 0x1f, .pe = 0x0, .sdiv = 0x7, .nsdiv = 0 }, /* 312.5 Khz */ { .mdiv = 0x17, .pe = 0x25ed, .sdiv = 0x1, .nsdiv = 0 }, /* 27 MHz */ { .mdiv = 0x1a, .pe = 0x7b36, .sdiv = 0x2, .nsdiv = 1 }, /* 36.87 MHz */ @@ -49,31 +49,86 @@ static struct stm_fs fs216c65_rtbl[] = { { .mdiv = 0x11, .pe = 0x1c72, .sdiv = 0x1, .nsdiv = 1 }, /* 108 MHz */ }; -static struct stm_fs fs432c65_rtbl[] = { - { .mdiv = 0x1f, .pe = 0x0, .sdiv = 0x7, .nsdiv = 0 }, /* 625 Khz */ - { .mdiv = 0x11, .pe = 0x1c72, .sdiv = 0x2, .nsdiv = 1 }, /* 108 MHz */ - { .mdiv = 0x19, .pe = 0x121a, .sdiv = 0x0, .nsdiv = 1 }, /* 297 MHz */ +static const struct stm_fs fs432c65_rtbl[] = { + { .mdiv = 0x1f, .pe = 0x0, .sdiv = 0x7, .nsdiv = 0 }, /* 625 Khz */ + { .mdiv = 0x13, .pe = 0x777c, .sdiv = 0x4, .nsdiv = 1 }, /* 25.175 MHz */ + { .mdiv = 0x19, .pe = 0x4d35, .sdiv = 0x2, .nsdiv = 0 }, /* 25.200 MHz */ + { .mdiv = 0x11, .pe = 0x1c72, .sdiv = 0x4, .nsdiv = 1 }, /* 27.000 MHz */ + { .mdiv = 0x17, .pe = 0x28f5, .sdiv = 0x2, .nsdiv = 0 }, /* 27.027 MHz */ + { .mdiv = 0x16, .pe = 0x3359, .sdiv = 0x2, .nsdiv = 0 }, /* 28.320 MHz */ + { .mdiv = 0x1f, .pe = 0x2083, .sdiv = 0x3, .nsdiv = 1 }, /* 30.240 MHz */ + { .mdiv = 0x1e, .pe = 0x430d, .sdiv = 0x3, .nsdiv = 1 }, /* 31.500 MHz */ + { .mdiv = 0x17, .pe = 0x0, .sdiv = 0x3, .nsdiv = 1 }, /* 40.000 MHz */ + { .mdiv = 0x19, .pe = 0x121a, .sdiv = 0x1, .nsdiv = 0 }, /* 49.500 MHz */ + { .mdiv = 0x13, .pe = 0x6667, .sdiv = 0x3, .nsdiv = 1 }, /* 50.000 MHz */ + { .mdiv = 0x10, .pe = 0x1ee6, .sdiv = 0x3, .nsdiv = 1 }, /* 57.284 MHz */ + { .mdiv = 0x1d, .pe = 0x3b14, .sdiv = 0x2, .nsdiv = 1 }, /* 65.000 MHz */ + { .mdiv = 0x12, .pe = 0x7c65, .sdiv = 0x1, .nsdiv = 0 }, /* 71.000 MHz */ + { .mdiv = 0x19, .pe = 0xecd, .sdiv = 0x2, .nsdiv = 1 }, /* 74.176 MHz */ + { .mdiv = 0x19, .pe = 0x121a, .sdiv = 0x2, .nsdiv = 1 }, /* 74.250 MHz */ + { .mdiv = 0x19, .pe = 0x3334, .sdiv = 0x2, .nsdiv = 1 }, /* 75.000 MHz */ + { .mdiv = 0x18, .pe = 0x5138, .sdiv = 0x2, .nsdiv = 1 }, /* 78.800 MHz */ + { .mdiv = 0x1d, .pe = 0x77d, .sdiv = 0x0, .nsdiv = 0 }, /* 85.500 MHz */ + { .mdiv = 0x1c, .pe = 0x13d5, .sdiv = 0x0, .nsdiv = 0 }, /* 88.750 MHz */ + { .mdiv = 0x11, .pe = 0x1c72, .sdiv = 0x2, .nsdiv = 1 }, /* 108.000 MHz */ + { .mdiv = 0x17, .pe = 0x28f5, .sdiv = 0x0, .nsdiv = 0 }, /* 108.108 MHz */ + { .mdiv = 0x10, .pe = 0x6e26, .sdiv = 0x2, .nsdiv = 1 }, /* 118.963 MHz */ + { .mdiv = 0x15, .pe = 0x3e63, .sdiv = 0x0, .nsdiv = 0 }, /* 119.000 MHz */ + { .mdiv = 0x1c, .pe = 0x471d, .sdiv = 0x1, .nsdiv = 1 }, /* 135.000 MHz */ + { .mdiv = 0x19, .pe = 0xecd, .sdiv = 0x1, .nsdiv = 1 }, /* 148.352 MHz */ + { .mdiv = 0x19, .pe = 0x121a, .sdiv = 0x1, .nsdiv = 1 }, /* 148.500 MHz */ + { .mdiv = 0x19, .pe = 0x121a, .sdiv = 0x0, .nsdiv = 1 }, /* 297 MHz */ }; -static struct stm_fs fs660c32_rtbl[] = { - { .mdiv = 0x01, .pe = 0x2aaa, .sdiv = 0x8, .nsdiv = 0 }, /* 600 KHz */ - { .mdiv = 0x02, .pe = 0x3d33, .sdiv = 0x0, .nsdiv = 0 }, /* 148.5 Mhz */ - { .mdiv = 0x13, .pe = 0x5bcc, .sdiv = 0x0, .nsdiv = 1 }, /* 297 Mhz */ - { .mdiv = 0x0e, .pe = 0x1025, .sdiv = 0x0, .nsdiv = 1 }, /* 333 Mhz */ - { .mdiv = 0x0b, .pe = 0x715f, .sdiv = 0x0, .nsdiv = 1 }, /* 350 Mhz */ +static const struct stm_fs fs660c32_rtbl[] = { + { .mdiv = 0x14, .pe = 0x376b, .sdiv = 0x4, .nsdiv = 1 }, /* 25.175 MHz */ + { .mdiv = 0x14, .pe = 0x30c3, .sdiv = 0x4, .nsdiv = 1 }, /* 25.200 MHz */ + { .mdiv = 0x10, .pe = 0x71c7, .sdiv = 0x4, .nsdiv = 1 }, /* 27.000 MHz */ + { .mdiv = 0x00, .pe = 0x47af, .sdiv = 0x3, .nsdiv = 0 }, /* 27.027 MHz */ + { .mdiv = 0x0e, .pe = 0x4e1a, .sdiv = 0x4, .nsdiv = 1 }, /* 28.320 MHz */ + { .mdiv = 0x0b, .pe = 0x534d, .sdiv = 0x4, .nsdiv = 1 }, /* 30.240 MHz */ + { .mdiv = 0x17, .pe = 0x6fbf, .sdiv = 0x2, .nsdiv = 0 }, /* 31.500 MHz */ + { .mdiv = 0x01, .pe = 0x0, .sdiv = 0x4, .nsdiv = 1 }, /* 40.000 MHz */ + { .mdiv = 0x15, .pe = 0x2aab, .sdiv = 0x3, .nsdiv = 1 }, /* 49.500 MHz */ + { .mdiv = 0x14, .pe = 0x6666, .sdiv = 0x3, .nsdiv = 1 }, /* 50.000 MHz */ + { .mdiv = 0x1d, .pe = 0x395f, .sdiv = 0x1, .nsdiv = 0 }, /* 57.284 MHz */ + { .mdiv = 0x08, .pe = 0x4ec5, .sdiv = 0x3, .nsdiv = 1 }, /* 65.000 MHz */ + { .mdiv = 0x05, .pe = 0x1770, .sdiv = 0x3, .nsdiv = 1 }, /* 71.000 MHz */ + { .mdiv = 0x03, .pe = 0x4ba7, .sdiv = 0x3, .nsdiv = 1 }, /* 74.176 MHz */ + { .mdiv = 0x0f, .pe = 0x3426, .sdiv = 0x1, .nsdiv = 0 }, /* 74.250 MHz */ + { .mdiv = 0x0e, .pe = 0x7777, .sdiv = 0x1, .nsdiv = 0 }, /* 75.000 MHz */ + { .mdiv = 0x01, .pe = 0x4053, .sdiv = 0x3, .nsdiv = 1 }, /* 78.800 MHz */ + { .mdiv = 0x09, .pe = 0x15b5, .sdiv = 0x1, .nsdiv = 0 }, /* 85.500 MHz */ + { .mdiv = 0x1b, .pe = 0x3f19, .sdiv = 0x2, .nsdiv = 1 }, /* 88.750 MHz */ + { .mdiv = 0x10, .pe = 0x71c7, .sdiv = 0x2, .nsdiv = 1 }, /* 108.000 MHz */ + { .mdiv = 0x00, .pe = 0x47af, .sdiv = 0x1, .nsdiv = 0 }, /* 108.108 MHz */ + { .mdiv = 0x0c, .pe = 0x3118, .sdiv = 0x2, .nsdiv = 1 }, /* 118.963 MHz */ + { .mdiv = 0x0c, .pe = 0x2f54, .sdiv = 0x2, .nsdiv = 1 }, /* 119.000 MHz */ + { .mdiv = 0x07, .pe = 0xe39, .sdiv = 0x2, .nsdiv = 1 }, /* 135.000 MHz */ + { .mdiv = 0x03, .pe = 0x4ba7, .sdiv = 0x2, .nsdiv = 1 }, /* 148.352 MHz */ + { .mdiv = 0x0f, .pe = 0x3426, .sdiv = 0x0, .nsdiv = 0 }, /* 148.500 MHz */ + { .mdiv = 0x03, .pe = 0x4ba7, .sdiv = 0x1, .nsdiv = 1 }, /* 296.704 MHz */ + { .mdiv = 0x03, .pe = 0x471c, .sdiv = 0x1, .nsdiv = 1 }, /* 297.000 MHz */ + { .mdiv = 0x00, .pe = 0x295f, .sdiv = 0x1, .nsdiv = 1 }, /* 326.700 MHz */ + { .mdiv = 0x1f, .pe = 0x3633, .sdiv = 0x0, .nsdiv = 1 }, /* 333.000 MHz */ + { .mdiv = 0x1c, .pe = 0x0, .sdiv = 0x0, .nsdiv = 1 }, /* 352.000 Mhz */ }; struct clkgen_quadfs_data { bool reset_present; bool bwfilter_present; bool lockstatus_present; + bool powerup_polarity; + bool standby_polarity; bool nsdiv_present; + bool nrst_present; struct clkgen_field ndiv; struct clkgen_field ref_bw; struct clkgen_field nreset; struct clkgen_field npda; struct clkgen_field lock_status; + struct clkgen_field nrst[QUADFS_MAX_CHAN]; struct clkgen_field nsb[QUADFS_MAX_CHAN]; struct clkgen_field en[QUADFS_MAX_CHAN]; struct clkgen_field mdiv[QUADFS_MAX_CHAN]; @@ -82,9 +137,9 @@ struct clkgen_quadfs_data { struct clkgen_field nsdiv[QUADFS_MAX_CHAN]; const struct clk_ops *pll_ops; - struct stm_fs *rtbl; + const struct stm_fs *rtbl; u8 rtbl_cnt; - int (*get_rate)(unsigned long , struct stm_fs *, + int (*get_rate)(unsigned long , const struct stm_fs *, unsigned long *); }; @@ -94,11 +149,11 @@ static const struct clk_ops st_quadfs_fs216c65_ops; static const struct clk_ops st_quadfs_fs432c65_ops; static const struct clk_ops st_quadfs_fs660c32_ops; -static int clk_fs216c65_get_rate(unsigned long, struct stm_fs *, +static int clk_fs216c65_get_rate(unsigned long, const struct stm_fs *, unsigned long *); -static int clk_fs432c65_get_rate(unsigned long, struct stm_fs *, +static int clk_fs432c65_get_rate(unsigned long, const struct stm_fs *, unsigned long *); -static int clk_fs660c32_dig_get_rate(unsigned long, struct stm_fs *, +static int clk_fs660c32_dig_get_rate(unsigned long, const struct stm_fs *, unsigned long *); /* * Values for all of the standalone instances of this clock @@ -106,7 +161,7 @@ static int clk_fs660c32_dig_get_rate(unsigned long, struct stm_fs *, * that the individual channel standby control bits (nsb) are in the * first register along with the PLL control bits. */ -static struct clkgen_quadfs_data st_fs216c65_416 = { +static const struct clkgen_quadfs_data st_fs216c65_416 = { /* 416 specific */ .npda = CLKGEN_FIELD(0x0, 0x1, 14), .nsb = { CLKGEN_FIELD(0x0, 0x1, 10), @@ -143,7 +198,7 @@ static struct clkgen_quadfs_data st_fs216c65_416 = { .get_rate = clk_fs216c65_get_rate, }; -static struct clkgen_quadfs_data st_fs432c65_416 = { +static const struct clkgen_quadfs_data st_fs432c65_416 = { .npda = CLKGEN_FIELD(0x0, 0x1, 14), .nsb = { CLKGEN_FIELD(0x0, 0x1, 10), CLKGEN_FIELD(0x0, 0x1, 11), @@ -179,7 +234,7 @@ static struct clkgen_quadfs_data st_fs432c65_416 = { .get_rate = clk_fs432c65_get_rate, }; -static struct clkgen_quadfs_data st_fs660c32_E_416 = { +static const struct clkgen_quadfs_data st_fs660c32_E_416 = { .npda = CLKGEN_FIELD(0x0, 0x1, 14), .nsb = { CLKGEN_FIELD(0x0, 0x1, 10), CLKGEN_FIELD(0x0, 0x1, 11), @@ -215,7 +270,7 @@ static struct clkgen_quadfs_data st_fs660c32_E_416 = { .get_rate = clk_fs660c32_dig_get_rate, }; -static struct clkgen_quadfs_data st_fs660c32_F_416 = { +static const struct clkgen_quadfs_data st_fs660c32_F_416 = { .npda = CLKGEN_FIELD(0x0, 0x1, 14), .nsb = { CLKGEN_FIELD(0x0, 0x1, 10), CLKGEN_FIELD(0x0, 0x1, 11), @@ -251,6 +306,91 @@ static struct clkgen_quadfs_data st_fs660c32_F_416 = { .get_rate = clk_fs660c32_dig_get_rate, }; +static const struct clkgen_quadfs_data st_fs660c32_C_407 = { + .nrst_present = true, + .nrst = { CLKGEN_FIELD(0x2f0, 0x1, 0), + CLKGEN_FIELD(0x2f0, 0x1, 1), + CLKGEN_FIELD(0x2f0, 0x1, 2), + CLKGEN_FIELD(0x2f0, 0x1, 3) }, + .npda = CLKGEN_FIELD(0x2f0, 0x1, 12), + .nsb = { CLKGEN_FIELD(0x2f0, 0x1, 8), + CLKGEN_FIELD(0x2f0, 0x1, 9), + CLKGEN_FIELD(0x2f0, 0x1, 10), + CLKGEN_FIELD(0x2f0, 0x1, 11) }, + .nsdiv_present = true, + .nsdiv = { CLKGEN_FIELD(0x304, 0x1, 24), + CLKGEN_FIELD(0x308, 0x1, 24), + CLKGEN_FIELD(0x30c, 0x1, 24), + CLKGEN_FIELD(0x310, 0x1, 24) }, + .mdiv = { CLKGEN_FIELD(0x304, 0x1f, 15), + CLKGEN_FIELD(0x308, 0x1f, 15), + CLKGEN_FIELD(0x30c, 0x1f, 15), + CLKGEN_FIELD(0x310, 0x1f, 15) }, + .en = { CLKGEN_FIELD(0x2fc, 0x1, 0), + CLKGEN_FIELD(0x2fc, 0x1, 1), + CLKGEN_FIELD(0x2fc, 0x1, 2), + CLKGEN_FIELD(0x2fc, 0x1, 3) }, + .ndiv = CLKGEN_FIELD(0x2f4, 0x7, 16), + .pe = { CLKGEN_FIELD(0x304, 0x7fff, 0), + CLKGEN_FIELD(0x308, 0x7fff, 0), + CLKGEN_FIELD(0x30c, 0x7fff, 0), + CLKGEN_FIELD(0x310, 0x7fff, 0) }, + .sdiv = { CLKGEN_FIELD(0x304, 0xf, 20), + CLKGEN_FIELD(0x308, 0xf, 20), + CLKGEN_FIELD(0x30c, 0xf, 20), + CLKGEN_FIELD(0x310, 0xf, 20) }, + .lockstatus_present = true, + .lock_status = CLKGEN_FIELD(0x2A0, 0x1, 24), + .powerup_polarity = 1, + .standby_polarity = 1, + .pll_ops = &st_quadfs_pll_c32_ops, + .rtbl = fs660c32_rtbl, + .rtbl_cnt = ARRAY_SIZE(fs660c32_rtbl), + .get_rate = clk_fs660c32_dig_get_rate, +}; + +static const struct clkgen_quadfs_data st_fs660c32_D_407 = { + .nrst_present = true, + .nrst = { CLKGEN_FIELD(0x2a0, 0x1, 0), + CLKGEN_FIELD(0x2a0, 0x1, 1), + CLKGEN_FIELD(0x2a0, 0x1, 2), + CLKGEN_FIELD(0x2a0, 0x1, 3) }, + .ndiv = CLKGEN_FIELD(0x2a4, 0x7, 16), + .pe = { CLKGEN_FIELD(0x2b4, 0x7fff, 0), + CLKGEN_FIELD(0x2b8, 0x7fff, 0), + CLKGEN_FIELD(0x2bc, 0x7fff, 0), + CLKGEN_FIELD(0x2c0, 0x7fff, 0) }, + .sdiv = { CLKGEN_FIELD(0x2b4, 0xf, 20), + CLKGEN_FIELD(0x2b8, 0xf, 20), + CLKGEN_FIELD(0x2bc, 0xf, 20), + CLKGEN_FIELD(0x2c0, 0xf, 20) }, + .npda = CLKGEN_FIELD(0x2a0, 0x1, 12), + .nsb = { CLKGEN_FIELD(0x2a0, 0x1, 8), + CLKGEN_FIELD(0x2a0, 0x1, 9), + CLKGEN_FIELD(0x2a0, 0x1, 10), + CLKGEN_FIELD(0x2a0, 0x1, 11) }, + .nsdiv_present = true, + .nsdiv = { CLKGEN_FIELD(0x2b4, 0x1, 24), + CLKGEN_FIELD(0x2b8, 0x1, 24), + CLKGEN_FIELD(0x2bc, 0x1, 24), + CLKGEN_FIELD(0x2c0, 0x1, 24) }, + .mdiv = { CLKGEN_FIELD(0x2b4, 0x1f, 15), + CLKGEN_FIELD(0x2b8, 0x1f, 15), + CLKGEN_FIELD(0x2bc, 0x1f, 15), + CLKGEN_FIELD(0x2c0, 0x1f, 15) }, + .en = { CLKGEN_FIELD(0x2ac, 0x1, 0), + CLKGEN_FIELD(0x2ac, 0x1, 1), + CLKGEN_FIELD(0x2ac, 0x1, 2), + CLKGEN_FIELD(0x2ac, 0x1, 3) }, + .lockstatus_present = true, + .lock_status = CLKGEN_FIELD(0x2A0, 0x1, 24), + .powerup_polarity = 1, + .standby_polarity = 1, + .pll_ops = &st_quadfs_pll_c32_ops, + .rtbl = fs660c32_rtbl, + .rtbl_cnt = ARRAY_SIZE(fs660c32_rtbl), + .get_rate = clk_fs660c32_dig_get_rate,}; + /** * DOC: A Frequency Synthesizer that multiples its input clock by a fixed factor * @@ -308,7 +448,7 @@ static int quadfs_pll_enable(struct clk_hw *hw) /* * Power up the PLL */ - CLKGEN_WRITE(pll, npda, 1); + CLKGEN_WRITE(pll, npda, !pll->data->powerup_polarity); if (pll->lock) spin_unlock_irqrestore(pll->lock, flags); @@ -335,7 +475,7 @@ static void quadfs_pll_disable(struct clk_hw *hw) * Powerdown the PLL and then put block into soft reset if we have * reset control. */ - CLKGEN_WRITE(pll, npda, 0); + CLKGEN_WRITE(pll, npda, pll->data->powerup_polarity); if (pll->data->reset_present) CLKGEN_WRITE(pll, nreset, 0); @@ -611,7 +751,10 @@ static int quadfs_fsynth_enable(struct clk_hw *hw) if (fs->lock) spin_lock_irqsave(fs->lock, flags); - CLKGEN_WRITE(fs, nsb[fs->chan], 1); + CLKGEN_WRITE(fs, nsb[fs->chan], !fs->data->standby_polarity); + + if (fs->data->nrst_present) + CLKGEN_WRITE(fs, nrst[fs->chan], 0); if (fs->lock) spin_unlock_irqrestore(fs->lock, flags); @@ -631,7 +774,7 @@ static void quadfs_fsynth_disable(struct clk_hw *hw) if (fs->lock) spin_lock_irqsave(fs->lock, flags); - CLKGEN_WRITE(fs, nsb[fs->chan], 0); + CLKGEN_WRITE(fs, nsb[fs->chan], !fs->data->standby_polarity); if (fs->lock) spin_unlock_irqrestore(fs->lock, flags); @@ -645,12 +788,12 @@ static int quadfs_fsynth_is_enabled(struct clk_hw *hw) pr_debug("%s: %s enable bit = 0x%x\n", __func__, __clk_get_name(hw->clk), nsb); - return !!nsb; + return fs->data->standby_polarity ? !nsb : !!nsb; } #define P15 (uint64_t)(1 << 15) -static int clk_fs216c65_get_rate(unsigned long input, struct stm_fs *fs, +static int clk_fs216c65_get_rate(unsigned long input, const struct stm_fs *fs, unsigned long *rate) { uint64_t res; @@ -670,7 +813,7 @@ static int clk_fs216c65_get_rate(unsigned long input, struct stm_fs *fs, return 0; } -static int clk_fs432c65_get_rate(unsigned long input, struct stm_fs *fs, +static int clk_fs432c65_get_rate(unsigned long input, const struct stm_fs *fs, unsigned long *rate) { uint64_t res; @@ -693,7 +836,7 @@ static int clk_fs432c65_get_rate(unsigned long input, struct stm_fs *fs, #define P20 (uint64_t)(1 << 20) static int clk_fs660c32_dig_get_rate(unsigned long input, - struct stm_fs *fs, unsigned long *rate) + const struct stm_fs *fs, unsigned long *rate) { unsigned long s = (1 << fs->sdiv); unsigned long ns; @@ -749,7 +892,7 @@ static long quadfs_find_best_rate(struct clk_hw *hw, unsigned long drate, { struct st_clk_quadfs_fsynth *fs = to_quadfs_fsynth(hw); int (*clk_fs_get_rate)(unsigned long , - struct stm_fs *, unsigned long *); + const struct stm_fs *, unsigned long *); struct stm_fs prev_params; unsigned long prev_rate, rate = 0; unsigned long diff_rate, prev_diff_rate = ~0; @@ -793,7 +936,7 @@ static unsigned long quadfs_recalc_rate(struct clk_hw *hw, unsigned long rate = 0; struct stm_fs params; int (*clk_fs_get_rate)(unsigned long , - struct stm_fs *, unsigned long *); + const struct stm_fs *, unsigned long *); clk_fs_get_rate = fs->data->get_rate; @@ -917,19 +1060,31 @@ static struct clk * __init st_clk_register_quadfs_fsynth( static struct of_device_id quadfs_of_match[] = { { .compatible = "st,stih416-quadfs216", - .data = (void *)&st_fs216c65_416 + .data = &st_fs216c65_416 }, { .compatible = "st,stih416-quadfs432", - .data = (void *)&st_fs432c65_416 + .data = &st_fs432c65_416 }, { .compatible = "st,stih416-quadfs660-E", - .data = (void *)&st_fs660c32_E_416 + .data = &st_fs660c32_E_416 }, { .compatible = "st,stih416-quadfs660-F", - .data = (void *)&st_fs660c32_F_416 + .data = &st_fs660c32_F_416 + }, + { + .compatible = "st,stih407-quadfs660-C", + .data = &st_fs660c32_C_407 + }, + { + .compatible = "st,stih407-quadfs660-D", + .data = &st_fs660c32_D_407 + }, + { + .compatible = "st,stih407-quadfs660-D", + .data = (void *)&st_fs660c32_D_407 }, {} }; diff --git a/drivers/clk/st/clkgen-mux.c b/drivers/clk/st/clkgen-mux.c index a329906d1e81..79dc40b5cc68 100644 --- a/drivers/clk/st/clkgen-mux.c +++ b/drivers/clk/st/clkgen-mux.c @@ -580,6 +580,11 @@ static struct clkgen_mux_data stih416_a9_mux_data = { .shift = 0, .width = 2, }; +static struct clkgen_mux_data stih407_a9_mux_data = { + .offset = 0x1a4, + .shift = 1, + .width = 2, +}; static struct of_device_id mux_of_match[] = { { @@ -610,6 +615,10 @@ static struct of_device_id mux_of_match[] = { .compatible = "st,stih416-clkgen-a9-mux", .data = &stih416_a9_mux_data, }, + { + .compatible = "st,stih407-clkgen-a9-mux", + .data = &stih407_a9_mux_data, + }, {} }; @@ -765,7 +774,8 @@ void __init st_of_clkgen_vcc_setup(struct device_node *np) div->reg = reg + VCC_DIV_OFFSET; div->shift = 2 * i; div->width = 2; - div->flags = CLK_DIVIDER_POWER_OF_TWO; + div->flags = CLK_DIVIDER_POWER_OF_TWO | + CLK_DIVIDER_ROUND_CLOSEST; mux->reg = reg + VCC_MUX_OFFSET; mux->shift = 2 * i; diff --git a/drivers/clk/st/clkgen-pll.c b/drivers/clk/st/clkgen-pll.c index d8b9b1a2aeda..29769d79e306 100644 --- a/drivers/clk/st/clkgen-pll.c +++ b/drivers/clk/st/clkgen-pll.c @@ -59,7 +59,7 @@ static const struct clk_ops st_pll800c65_ops; static const struct clk_ops stm_pll3200c32_ops; static const struct clk_ops st_pll1200c32_ops; -static struct clkgen_pll_data st_pll1600c65_ax = { +static const struct clkgen_pll_data st_pll1600c65_ax = { .pdn_status = CLKGEN_FIELD(0x0, 0x1, 19), .locked_status = CLKGEN_FIELD(0x0, 0x1, 31), .mdiv = CLKGEN_FIELD(0x0, C65_MDIV_PLL1600_MASK, 0), @@ -67,7 +67,7 @@ static struct clkgen_pll_data st_pll1600c65_ax = { .ops = &st_pll1600c65_ops }; -static struct clkgen_pll_data st_pll800c65_ax = { +static const struct clkgen_pll_data st_pll800c65_ax = { .pdn_status = CLKGEN_FIELD(0x0, 0x1, 19), .locked_status = CLKGEN_FIELD(0x0, 0x1, 31), .mdiv = CLKGEN_FIELD(0x0, C65_MDIV_PLL800_MASK, 0), @@ -76,7 +76,7 @@ static struct clkgen_pll_data st_pll800c65_ax = { .ops = &st_pll800c65_ops }; -static struct clkgen_pll_data st_pll3200c32_a1x_0 = { +static const struct clkgen_pll_data st_pll3200c32_a1x_0 = { .pdn_status = CLKGEN_FIELD(0x0, 0x1, 31), .locked_status = CLKGEN_FIELD(0x4, 0x1, 31), .ndiv = CLKGEN_FIELD(0x0, C32_NDIV_MASK, 0x0), @@ -93,7 +93,7 @@ static struct clkgen_pll_data st_pll3200c32_a1x_0 = { .ops = &stm_pll3200c32_ops, }; -static struct clkgen_pll_data st_pll3200c32_a1x_1 = { +static const struct clkgen_pll_data st_pll3200c32_a1x_1 = { .pdn_status = CLKGEN_FIELD(0xC, 0x1, 31), .locked_status = CLKGEN_FIELD(0x10, 0x1, 31), .ndiv = CLKGEN_FIELD(0xC, C32_NDIV_MASK, 0x0), @@ -111,7 +111,7 @@ static struct clkgen_pll_data st_pll3200c32_a1x_1 = { }; /* 415 specific */ -static struct clkgen_pll_data st_pll3200c32_a9_415 = { +static const struct clkgen_pll_data st_pll3200c32_a9_415 = { .pdn_status = CLKGEN_FIELD(0x0, 0x1, 0), .locked_status = CLKGEN_FIELD(0x6C, 0x1, 0), .ndiv = CLKGEN_FIELD(0x0, C32_NDIV_MASK, 9), @@ -122,7 +122,7 @@ static struct clkgen_pll_data st_pll3200c32_a9_415 = { .ops = &stm_pll3200c32_ops, }; -static struct clkgen_pll_data st_pll3200c32_ddr_415 = { +static const struct clkgen_pll_data st_pll3200c32_ddr_415 = { .pdn_status = CLKGEN_FIELD(0x0, 0x1, 0), .locked_status = CLKGEN_FIELD(0x100, 0x1, 0), .ndiv = CLKGEN_FIELD(0x8, C32_NDIV_MASK, 0), @@ -135,7 +135,7 @@ static struct clkgen_pll_data st_pll3200c32_ddr_415 = { .ops = &stm_pll3200c32_ops, }; -static struct clkgen_pll_data st_pll1200c32_gpu_415 = { +static const struct clkgen_pll_data st_pll1200c32_gpu_415 = { .pdn_status = CLKGEN_FIELD(0x144, 0x1, 3), .locked_status = CLKGEN_FIELD(0x168, 0x1, 0), .ldf = CLKGEN_FIELD(0x0, C32_LDF_MASK, 3), @@ -146,7 +146,7 @@ static struct clkgen_pll_data st_pll1200c32_gpu_415 = { }; /* 416 specific */ -static struct clkgen_pll_data st_pll3200c32_a9_416 = { +static const struct clkgen_pll_data st_pll3200c32_a9_416 = { .pdn_status = CLKGEN_FIELD(0x0, 0x1, 0), .locked_status = CLKGEN_FIELD(0x6C, 0x1, 0), .ndiv = CLKGEN_FIELD(0x8, C32_NDIV_MASK, 0), @@ -157,7 +157,7 @@ static struct clkgen_pll_data st_pll3200c32_a9_416 = { .ops = &stm_pll3200c32_ops, }; -static struct clkgen_pll_data st_pll3200c32_ddr_416 = { +static const struct clkgen_pll_data st_pll3200c32_ddr_416 = { .pdn_status = CLKGEN_FIELD(0x0, 0x1, 0), .locked_status = CLKGEN_FIELD(0x10C, 0x1, 0), .ndiv = CLKGEN_FIELD(0x8, C32_NDIV_MASK, 0), @@ -170,7 +170,7 @@ static struct clkgen_pll_data st_pll3200c32_ddr_416 = { .ops = &stm_pll3200c32_ops, }; -static struct clkgen_pll_data st_pll1200c32_gpu_416 = { +static const struct clkgen_pll_data st_pll1200c32_gpu_416 = { .pdn_status = CLKGEN_FIELD(0x8E4, 0x1, 3), .locked_status = CLKGEN_FIELD(0x90C, 0x1, 0), .ldf = CLKGEN_FIELD(0x0, C32_LDF_MASK, 3), @@ -180,6 +180,54 @@ static struct clkgen_pll_data st_pll1200c32_gpu_416 = { .ops = &st_pll1200c32_ops, }; +static const struct clkgen_pll_data st_pll3200c32_407_a0 = { + /* 407 A0 */ + .pdn_status = CLKGEN_FIELD(0x2a0, 0x1, 8), + .locked_status = CLKGEN_FIELD(0x2a0, 0x1, 24), + .ndiv = CLKGEN_FIELD(0x2a4, C32_NDIV_MASK, 16), + .idf = CLKGEN_FIELD(0x2a4, C32_IDF_MASK, 0x0), + .num_odfs = 1, + .odf = { CLKGEN_FIELD(0x2b4, C32_ODF_MASK, 0) }, + .odf_gate = { CLKGEN_FIELD(0x2b4, 0x1, 6) }, + .ops = &stm_pll3200c32_ops, +}; + +static const struct clkgen_pll_data st_pll3200c32_407_c0_0 = { + /* 407 C0 PLL0 */ + .pdn_status = CLKGEN_FIELD(0x2a0, 0x1, 8), + .locked_status = CLKGEN_FIELD(0x2a0, 0x1, 24), + .ndiv = CLKGEN_FIELD(0x2a4, C32_NDIV_MASK, 16), + .idf = CLKGEN_FIELD(0x2a4, C32_IDF_MASK, 0x0), + .num_odfs = 1, + .odf = { CLKGEN_FIELD(0x2b4, C32_ODF_MASK, 0) }, + .odf_gate = { CLKGEN_FIELD(0x2b4, 0x1, 6) }, + .ops = &stm_pll3200c32_ops, +}; + +static const struct clkgen_pll_data st_pll3200c32_407_c0_1 = { + /* 407 C0 PLL1 */ + .pdn_status = CLKGEN_FIELD(0x2c8, 0x1, 8), + .locked_status = CLKGEN_FIELD(0x2c8, 0x1, 24), + .ndiv = CLKGEN_FIELD(0x2cc, C32_NDIV_MASK, 16), + .idf = CLKGEN_FIELD(0x2cc, C32_IDF_MASK, 0x0), + .num_odfs = 1, + .odf = { CLKGEN_FIELD(0x2dc, C32_ODF_MASK, 0) }, + .odf_gate = { CLKGEN_FIELD(0x2dc, 0x1, 6) }, + .ops = &stm_pll3200c32_ops, +}; + +static const struct clkgen_pll_data st_pll3200c32_407_a9 = { + /* 407 A9 */ + .pdn_status = CLKGEN_FIELD(0x1a8, 0x1, 0), + .locked_status = CLKGEN_FIELD(0x87c, 0x1, 0), + .ndiv = CLKGEN_FIELD(0x1b0, C32_NDIV_MASK, 0), + .idf = CLKGEN_FIELD(0x1a8, C32_IDF_MASK, 25), + .num_odfs = 1, + .odf = { CLKGEN_FIELD(0x1b0, C32_ODF_MASK, 8) }, + .odf_gate = { CLKGEN_FIELD(0x1ac, 0x1, 28) }, + .ops = &stm_pll3200c32_ops, +}; + /** * DOC: Clock Generated by PLL, rate set and enabled by bootloader * @@ -450,9 +498,8 @@ static void __init clkgena_c65_pll_setup(struct device_node *np) * PLL0 HS (high speed) output */ clk_data->clks[0] = clkgen_pll_register(parent_name, - &st_pll1600c65_ax, - reg + CLKGENAx_PLL0_OFFSET, - clk_name); + (struct clkgen_pll_data *) &st_pll1600c65_ax, + reg + CLKGENAx_PLL0_OFFSET, clk_name); if (IS_ERR(clk_data->clks[0])) goto err; @@ -480,9 +527,8 @@ static void __init clkgena_c65_pll_setup(struct device_node *np) * PLL1 output */ clk_data->clks[2] = clkgen_pll_register(parent_name, - &st_pll800c65_ax, - reg + CLKGENAx_PLL1_OFFSET, - clk_name); + (struct clkgen_pll_data *) &st_pll800c65_ax, + reg + CLKGENAx_PLL1_OFFSET, clk_name); if (IS_ERR(clk_data->clks[2])) goto err; @@ -572,6 +618,22 @@ static struct of_device_id c32_pll_of_match[] = { .compatible = "st,stih416-plls-c32-ddr", .data = &st_pll3200c32_ddr_416, }, + { + .compatible = "st,stih407-plls-c32-a0", + .data = &st_pll3200c32_407_a0, + }, + { + .compatible = "st,stih407-plls-c32-c0_0", + .data = &st_pll3200c32_407_c0_0, + }, + { + .compatible = "st,stih407-plls-c32-c0_1", + .data = &st_pll3200c32_407_c0_1, + }, + { + .compatible = "st,stih407-plls-c32-a9", + .data = &st_pll3200c32_407_a9, + }, {} }; diff --git a/drivers/clk/sunxi/Makefile b/drivers/clk/sunxi/Makefile index 762fd64dbd1f..6850cba35871 100644 --- a/drivers/clk/sunxi/Makefile +++ b/drivers/clk/sunxi/Makefile @@ -6,4 +6,6 @@ obj-y += clk-sunxi.o clk-factors.o obj-y += clk-a10-hosc.o obj-y += clk-a20-gmac.o -obj-$(CONFIG_MFD_SUN6I_PRCM) += clk-sun6i-ar100.o clk-sun6i-apb0.o clk-sun6i-apb0-gates.o +obj-$(CONFIG_MFD_SUN6I_PRCM) += \ + clk-sun6i-ar100.o clk-sun6i-apb0.o clk-sun6i-apb0-gates.o \ + clk-sun8i-apb0.o diff --git a/drivers/clk/sunxi/clk-a20-gmac.c b/drivers/clk/sunxi/clk-a20-gmac.c index 633ddc4389ef..5296fd6dd7b3 100644 --- a/drivers/clk/sunxi/clk-a20-gmac.c +++ b/drivers/clk/sunxi/clk-a20-gmac.c @@ -60,7 +60,7 @@ static void __init sun7i_a20_gmac_clk_setup(struct device_node *node) struct clk_gate *gate; const char *clk_name = node->name; const char *parents[SUN7I_A20_GMAC_PARENTS]; - void *reg; + void __iomem *reg; if (of_property_read_string(node, "clock-output-names", &clk_name)) return; diff --git a/drivers/clk/sunxi/clk-factors.c b/drivers/clk/sunxi/clk-factors.c index 3806d97e529b..2057c8ac648f 100644 --- a/drivers/clk/sunxi/clk-factors.c +++ b/drivers/clk/sunxi/clk-factors.c @@ -62,7 +62,7 @@ static unsigned long clk_factors_recalc_rate(struct clk_hw *hw, p = FACTOR_GET(config->pshift, config->pwidth, reg); /* Calculate the rate */ - rate = (parent_rate * n * (k + 1) >> p) / (m + 1); + rate = (parent_rate * (n + config->n_start) * (k + 1) >> p) / (m + 1); return rate; } diff --git a/drivers/clk/sunxi/clk-factors.h b/drivers/clk/sunxi/clk-factors.h index 02e1a43ebac7..d2d0efa39379 100644 --- a/drivers/clk/sunxi/clk-factors.h +++ b/drivers/clk/sunxi/clk-factors.h @@ -15,6 +15,7 @@ struct clk_factors_config { u8 mwidth; u8 pshift; u8 pwidth; + u8 n_start; }; struct clk_factors { diff --git a/drivers/clk/sunxi/clk-sun6i-apb0-gates.c b/drivers/clk/sunxi/clk-sun6i-apb0-gates.c index 670f90d629d7..e10d0521ec76 100644 --- a/drivers/clk/sunxi/clk-sun6i-apb0-gates.c +++ b/drivers/clk/sunxi/clk-sun6i-apb0-gates.c @@ -9,23 +9,53 @@ */ #include <linux/clk-provider.h> +#include <linux/clkdev.h> #include <linux/module.h> #include <linux/of.h> +#include <linux/of_device.h> #include <linux/platform_device.h> #define SUN6I_APB0_GATES_MAX_SIZE 32 +struct gates_data { + DECLARE_BITMAP(mask, SUN6I_APB0_GATES_MAX_SIZE); +}; + +static const struct gates_data sun6i_a31_apb0_gates __initconst = { + .mask = {0x7F}, +}; + +static const struct gates_data sun8i_a23_apb0_gates __initconst = { + .mask = {0x5D}, +}; + +static const struct of_device_id sun6i_a31_apb0_gates_clk_dt_ids[] = { + { .compatible = "allwinner,sun6i-a31-apb0-gates-clk", .data = &sun6i_a31_apb0_gates }, + { .compatible = "allwinner,sun8i-a23-apb0-gates-clk", .data = &sun8i_a23_apb0_gates }, + { /* sentinel */ } +}; + static int sun6i_a31_apb0_gates_clk_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; struct clk_onecell_data *clk_data; + const struct of_device_id *device; + const struct gates_data *data; const char *clk_parent; const char *clk_name; struct resource *r; void __iomem *reg; - int gate_id; int ngates; int i; + int j = 0; + + if (!np) + return -ENODEV; + + device = of_match_device(sun6i_a31_apb0_gates_clk_dt_ids, &pdev->dev); + if (!device) + return -ENODEV; + data = device->data; r = platform_get_resource(pdev, IORESOURCE_MEM, 0); reg = devm_ioremap_resource(&pdev->dev, r); @@ -36,54 +66,36 @@ static int sun6i_a31_apb0_gates_clk_probe(struct platform_device *pdev) if (!clk_parent) return -EINVAL; - ngates = of_property_count_strings(np, "clock-output-names"); - if (ngates < 0) - return ngates; - - if (!ngates || ngates > SUN6I_APB0_GATES_MAX_SIZE) - return -EINVAL; - clk_data = devm_kzalloc(&pdev->dev, sizeof(struct clk_onecell_data), GFP_KERNEL); if (!clk_data) return -ENOMEM; - clk_data->clks = devm_kzalloc(&pdev->dev, - SUN6I_APB0_GATES_MAX_SIZE * - sizeof(struct clk *), - GFP_KERNEL); + /* Worst-case size approximation and memory allocation */ + ngates = find_last_bit(data->mask, SUN6I_APB0_GATES_MAX_SIZE); + clk_data->clks = devm_kcalloc(&pdev->dev, (ngates + 1), + sizeof(struct clk *), GFP_KERNEL); if (!clk_data->clks) return -ENOMEM; - for (i = 0; i < ngates; i++) { + for_each_set_bit(i, data->mask, SUN6I_APB0_GATES_MAX_SIZE) { of_property_read_string_index(np, "clock-output-names", - i, &clk_name); + j, &clk_name); - gate_id = i; - of_property_read_u32_index(np, "clock-indices", i, &gate_id); + clk_data->clks[i] = clk_register_gate(&pdev->dev, clk_name, + clk_parent, 0, reg, i, + 0, NULL); + WARN_ON(IS_ERR(clk_data->clks[i])); + clk_register_clkdev(clk_data->clks[i], clk_name, NULL); - WARN_ON(gate_id >= SUN6I_APB0_GATES_MAX_SIZE); - if (gate_id >= SUN6I_APB0_GATES_MAX_SIZE) - continue; - - clk_data->clks[gate_id] = clk_register_gate(&pdev->dev, - clk_name, - clk_parent, 0, - reg, gate_id, - 0, NULL); - WARN_ON(IS_ERR(clk_data->clks[gate_id])); + j++; } - clk_data->clk_num = ngates; + clk_data->clk_num = ngates + 1; return of_clk_add_provider(np, of_clk_src_onecell_get, clk_data); } -const struct of_device_id sun6i_a31_apb0_gates_clk_dt_ids[] = { - { .compatible = "allwinner,sun6i-a31-apb0-gates-clk" }, - { /* sentinel */ } -}; - static struct platform_driver sun6i_a31_apb0_gates_clk_driver = { .driver = { .name = "sun6i-a31-apb0-gates-clk", diff --git a/drivers/clk/sunxi/clk-sun6i-apb0.c b/drivers/clk/sunxi/clk-sun6i-apb0.c index 11f17c34c2ae..1fa23371c8c6 100644 --- a/drivers/clk/sunxi/clk-sun6i-apb0.c +++ b/drivers/clk/sunxi/clk-sun6i-apb0.c @@ -57,7 +57,7 @@ static int sun6i_a31_apb0_clk_probe(struct platform_device *pdev) return of_clk_add_provider(np, of_clk_src_simple_get, clk); } -const struct of_device_id sun6i_a31_apb0_clk_dt_ids[] = { +static const struct of_device_id sun6i_a31_apb0_clk_dt_ids[] = { { .compatible = "allwinner,sun6i-a31-apb0-clk" }, { /* sentinel */ } }; diff --git a/drivers/clk/sunxi/clk-sun6i-ar100.c b/drivers/clk/sunxi/clk-sun6i-ar100.c index f73cc051f0dd..eca8ca025b6a 100644 --- a/drivers/clk/sunxi/clk-sun6i-ar100.c +++ b/drivers/clk/sunxi/clk-sun6i-ar100.c @@ -160,7 +160,7 @@ static int ar100_set_rate(struct clk_hw *hw, unsigned long rate, return 0; } -struct clk_ops ar100_ops = { +static struct clk_ops ar100_ops = { .recalc_rate = ar100_recalc_rate, .determine_rate = ar100_determine_rate, .set_parent = ar100_set_parent, @@ -213,7 +213,7 @@ static int sun6i_a31_ar100_clk_probe(struct platform_device *pdev) return of_clk_add_provider(np, of_clk_src_simple_get, clk); } -const struct of_device_id sun6i_a31_ar100_clk_dt_ids[] = { +static const struct of_device_id sun6i_a31_ar100_clk_dt_ids[] = { { .compatible = "allwinner,sun6i-a31-ar100-clk" }, { /* sentinel */ } }; diff --git a/drivers/clk/sunxi/clk-sun8i-apb0.c b/drivers/clk/sunxi/clk-sun8i-apb0.c new file mode 100644 index 000000000000..1f5ba9b4b8cd --- /dev/null +++ b/drivers/clk/sunxi/clk-sun8i-apb0.c @@ -0,0 +1,68 @@ +/* + * Copyright (C) 2014 Chen-Yu Tsai + * Author: Chen-Yu Tsai <wens@csie.org> + * + * Allwinner A23 APB0 clock driver + * + * License Terms: GNU General Public License v2 + * + * Based on clk-sun6i-apb0.c + * Allwinner A31 APB0 clock driver + * + * Copyright (C) 2014 Free Electrons + * Author: Boris BREZILLON <boris.brezillon@free-electrons.com> + * + */ + +#include <linux/clk-provider.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/platform_device.h> + +static int sun8i_a23_apb0_clk_probe(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + const char *clk_name = np->name; + const char *clk_parent; + struct resource *r; + void __iomem *reg; + struct clk *clk; + + r = platform_get_resource(pdev, IORESOURCE_MEM, 0); + reg = devm_ioremap_resource(&pdev->dev, r); + if (IS_ERR(reg)) + return PTR_ERR(reg); + + clk_parent = of_clk_get_parent_name(np, 0); + if (!clk_parent) + return -EINVAL; + + of_property_read_string(np, "clock-output-names", &clk_name); + + /* The A23 APB0 clock is a standard 2 bit wide divider clock */ + clk = clk_register_divider(&pdev->dev, clk_name, clk_parent, 0, reg, + 0, 2, CLK_DIVIDER_POWER_OF_TWO, NULL); + if (IS_ERR(clk)) + return PTR_ERR(clk); + + return of_clk_add_provider(np, of_clk_src_simple_get, clk); +} + +static const struct of_device_id sun8i_a23_apb0_clk_dt_ids[] = { + { .compatible = "allwinner,sun8i-a23-apb0-clk" }, + { /* sentinel */ } +}; + +static struct platform_driver sun8i_a23_apb0_clk_driver = { + .driver = { + .name = "sun8i-a23-apb0-clk", + .owner = THIS_MODULE, + .of_match_table = sun8i_a23_apb0_clk_dt_ids, + }, + .probe = sun8i_a23_apb0_clk_probe, +}; +module_platform_driver(sun8i_a23_apb0_clk_driver); + +MODULE_AUTHOR("Chen-Yu Tsai <wens@csie.org>"); +MODULE_DESCRIPTION("Allwinner A23 APB0 clock Driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/clk/sunxi/clk-sunxi.c b/drivers/clk/sunxi/clk-sunxi.c index fb2ce8440f0e..b654b7b1d137 100644 --- a/drivers/clk/sunxi/clk-sunxi.c +++ b/drivers/clk/sunxi/clk-sunxi.c @@ -164,6 +164,54 @@ static void sun6i_a31_get_pll1_factors(u32 *freq, u32 parent_rate, } /** + * sun8i_a23_get_pll1_factors() - calculates n, k, m, p factors for PLL1 + * PLL1 rate is calculated as follows + * rate = (parent_rate * (n + 1) * (k + 1) >> p) / (m + 1); + * parent_rate is always 24Mhz + */ + +static void sun8i_a23_get_pll1_factors(u32 *freq, u32 parent_rate, + u8 *n, u8 *k, u8 *m, u8 *p) +{ + u8 div; + + /* Normalize value to a 6M multiple */ + div = *freq / 6000000; + *freq = 6000000 * div; + + /* we were called to round the frequency, we can now return */ + if (n == NULL) + return; + + /* m is always zero for pll1 */ + *m = 0; + + /* k is 1 only on these cases */ + if (*freq >= 768000000 || *freq == 42000000 || *freq == 54000000) + *k = 1; + else + *k = 0; + + /* p will be 2 for divs under 20 and odd divs under 32 */ + if (div < 20 || (div < 32 && (div & 1))) + *p = 2; + + /* p will be 1 for even divs under 32, divs under 40 and odd pairs + * of divs between 40-62 */ + else if (div < 40 || (div < 64 && (div & 2))) + *p = 1; + + /* any other entries have p = 0 */ + else + *p = 0; + + /* calculate a suitable n based on k and p */ + div <<= *p; + div /= (*k + 1); + *n = div / 4 - 1; +} + +/** * sun4i_get_pll5_factors() - calculates n, k factors for PLL5 * PLL5 rate is calculated as follows * rate = parent_rate * n * (k + 1) @@ -422,6 +470,18 @@ static struct clk_factors_config sun6i_a31_pll1_config = { .mwidth = 2, }; +static struct clk_factors_config sun8i_a23_pll1_config = { + .nshift = 8, + .nwidth = 5, + .kshift = 4, + .kwidth = 2, + .mshift = 0, + .mwidth = 2, + .pshift = 16, + .pwidth = 2, + .n_start = 1, +}; + static struct clk_factors_config sun4i_pll5_config = { .nshift = 8, .nwidth = 5, @@ -471,6 +531,12 @@ static const struct factors_data sun6i_a31_pll1_data __initconst = { .getter = sun6i_a31_get_pll1_factors, }; +static const struct factors_data sun8i_a23_pll1_data __initconst = { + .enable = 31, + .table = &sun8i_a23_pll1_config, + .getter = sun8i_a23_get_pll1_factors, +}; + static const struct factors_data sun7i_a20_pll4_data __initconst = { .enable = 31, .table = &sun4i_pll5_config, @@ -527,7 +593,7 @@ static struct clk * __init sunxi_factors_clk_setup(struct device_node *node, struct clk_hw *mux_hw = NULL; const char *clk_name = node->name; const char *parents[SUNXI_MAX_PARENTS]; - void *reg; + void __iomem *reg; int i = 0; reg = of_iomap(node, 0); @@ -632,7 +698,7 @@ static void __init sunxi_mux_clk_setup(struct device_node *node, struct clk *clk; const char *clk_name = node->name; const char *parents[SUNXI_MAX_PARENTS]; - void *reg; + void __iomem *reg; int i = 0; reg = of_iomap(node, 0); @@ -664,6 +730,7 @@ struct div_data { u8 shift; u8 pow; u8 width; + const struct clk_div_table *table; }; static const struct div_data sun4i_axi_data __initconst = { @@ -672,6 +739,23 @@ static const struct div_data sun4i_axi_data __initconst = { .width = 2, }; +static const struct clk_div_table sun8i_a23_axi_table[] __initconst = { + { .val = 0, .div = 1 }, + { .val = 1, .div = 2 }, + { .val = 2, .div = 3 }, + { .val = 3, .div = 4 }, + { .val = 4, .div = 4 }, + { .val = 5, .div = 4 }, + { .val = 6, .div = 4 }, + { .val = 7, .div = 4 }, + { } /* sentinel */ +}; + +static const struct div_data sun8i_a23_axi_data __initconst = { + .width = 3, + .table = sun8i_a23_axi_table, +}; + static const struct div_data sun4i_ahb_data __initconst = { .shift = 4, .pow = 1, @@ -696,7 +780,7 @@ static void __init sunxi_divider_clk_setup(struct device_node *node, struct clk *clk; const char *clk_name = node->name; const char *clk_parent; - void *reg; + void __iomem *reg; reg = of_iomap(node, 0); @@ -704,10 +788,10 @@ static void __init sunxi_divider_clk_setup(struct device_node *node, of_property_read_string(node, "clock-output-names", &clk_name); - clk = clk_register_divider(NULL, clk_name, clk_parent, 0, - reg, data->shift, data->width, - data->pow ? CLK_DIVIDER_POWER_OF_TWO : 0, - &clk_lock); + clk = clk_register_divider_table(NULL, clk_name, clk_parent, 0, + reg, data->shift, data->width, + data->pow ? CLK_DIVIDER_POWER_OF_TWO : 0, + data->table, &clk_lock); if (clk) { of_clk_add_provider(node, of_clk_src_simple_get, clk); clk_register_clkdev(clk, clk_name, NULL); @@ -804,6 +888,10 @@ static const struct gates_data sun7i_a20_ahb_gates_data __initconst = { .mask = { 0x12f77fff, 0x16ff3f }, }; +static const struct gates_data sun8i_a23_ahb1_gates_data __initconst = { + .mask = {0x25386742, 0x2505111}, +}; + static const struct gates_data sun4i_apb0_gates_data __initconst = { .mask = {0x4EF}, }; @@ -836,6 +924,10 @@ static const struct gates_data sun6i_a31_apb1_gates_data __initconst = { .mask = {0x3031}, }; +static const struct gates_data sun8i_a23_apb1_gates_data __initconst = { + .mask = {0x3021}, +}; + static const struct gates_data sun6i_a31_apb2_gates_data __initconst = { .mask = {0x3F000F}, }; @@ -844,6 +936,10 @@ static const struct gates_data sun7i_a20_apb1_gates_data __initconst = { .mask = { 0xff80ff }, }; +static const struct gates_data sun8i_a23_apb2_gates_data __initconst = { + .mask = {0x1F0007}, +}; + static const struct gates_data sun4i_a10_usb_gates_data __initconst = { .mask = {0x1C0}, .reset_mask = 0x07, @@ -866,11 +962,10 @@ static void __init sunxi_gates_clk_setup(struct device_node *node, struct gates_reset_data *reset_data; const char *clk_parent; const char *clk_name; - void *reg; + void __iomem *reg; int qty; int i = 0; int j = 0; - int ignore; reg = of_iomap(node, 0); @@ -891,14 +986,12 @@ static void __init sunxi_gates_clk_setup(struct device_node *node, of_property_read_string_index(node, "clock-output-names", j, &clk_name); - /* No driver claims this clock, but it should remain gated */ - ignore = !strcmp("ahb_sdram", clk_name) ? CLK_IGNORE_UNUSED : 0; - clk_data->clks[i] = clk_register_gate(NULL, clk_name, - clk_parent, ignore, + clk_parent, 0, reg + 4 * (i/32), i % 32, 0, &clk_lock); WARN_ON(IS_ERR(clk_data->clks[i])); + clk_register_clkdev(clk_data->clks[i], clk_name, NULL); j++; } @@ -991,7 +1084,7 @@ static void __init sunxi_divs_clk_setup(struct device_node *node, struct clk_gate *gate = NULL; struct clk_fixed_factor *fix_factor; struct clk_divider *divider; - void *reg; + void __iomem *reg; int i = 0; int flags, clkflags; @@ -1102,6 +1195,7 @@ free_clkdata: static const struct of_device_id clk_factors_match[] __initconst = { {.compatible = "allwinner,sun4i-a10-pll1-clk", .data = &sun4i_pll1_data,}, {.compatible = "allwinner,sun6i-a31-pll1-clk", .data = &sun6i_a31_pll1_data,}, + {.compatible = "allwinner,sun8i-a23-pll1-clk", .data = &sun8i_a23_pll1_data,}, {.compatible = "allwinner,sun7i-a20-pll4-clk", .data = &sun7i_a20_pll4_data,}, {.compatible = "allwinner,sun6i-a31-pll6-clk", .data = &sun6i_a31_pll6_data,}, {.compatible = "allwinner,sun4i-a10-apb1-clk", .data = &sun4i_apb1_data,}, @@ -1113,6 +1207,7 @@ static const struct of_device_id clk_factors_match[] __initconst = { /* Matches for divider clocks */ static const struct of_device_id clk_div_match[] __initconst = { {.compatible = "allwinner,sun4i-a10-axi-clk", .data = &sun4i_axi_data,}, + {.compatible = "allwinner,sun8i-a23-axi-clk", .data = &sun8i_a23_axi_data,}, {.compatible = "allwinner,sun4i-a10-ahb-clk", .data = &sun4i_ahb_data,}, {.compatible = "allwinner,sun4i-a10-apb0-clk", .data = &sun4i_apb0_data,}, {.compatible = "allwinner,sun6i-a31-apb2-div-clk", .data = &sun6i_a31_apb2_div_data,}, @@ -1142,6 +1237,7 @@ static const struct of_device_id clk_gates_match[] __initconst = { {.compatible = "allwinner,sun5i-a13-ahb-gates-clk", .data = &sun5i_a13_ahb_gates_data,}, {.compatible = "allwinner,sun6i-a31-ahb1-gates-clk", .data = &sun6i_a31_ahb1_gates_data,}, {.compatible = "allwinner,sun7i-a20-ahb-gates-clk", .data = &sun7i_a20_ahb_gates_data,}, + {.compatible = "allwinner,sun8i-a23-ahb1-gates-clk", .data = &sun8i_a23_ahb1_gates_data,}, {.compatible = "allwinner,sun4i-a10-apb0-gates-clk", .data = &sun4i_apb0_gates_data,}, {.compatible = "allwinner,sun5i-a10s-apb0-gates-clk", .data = &sun5i_a10s_apb0_gates_data,}, {.compatible = "allwinner,sun5i-a13-apb0-gates-clk", .data = &sun5i_a13_apb0_gates_data,}, @@ -1151,7 +1247,9 @@ static const struct of_device_id clk_gates_match[] __initconst = { {.compatible = "allwinner,sun5i-a13-apb1-gates-clk", .data = &sun5i_a13_apb1_gates_data,}, {.compatible = "allwinner,sun6i-a31-apb1-gates-clk", .data = &sun6i_a31_apb1_gates_data,}, {.compatible = "allwinner,sun7i-a20-apb1-gates-clk", .data = &sun7i_a20_apb1_gates_data,}, + {.compatible = "allwinner,sun8i-a23-apb1-gates-clk", .data = &sun8i_a23_apb1_gates_data,}, {.compatible = "allwinner,sun6i-a31-apb2-gates-clk", .data = &sun6i_a31_apb2_gates_data,}, + {.compatible = "allwinner,sun8i-a23-apb2-gates-clk", .data = &sun8i_a23_apb2_gates_data,}, {.compatible = "allwinner,sun4i-a10-usb-clk", .data = &sun4i_a10_usb_gates_data,}, {.compatible = "allwinner,sun5i-a13-usb-clk", .data = &sun5i_a13_usb_gates_data,}, {.compatible = "allwinner,sun6i-a31-usb-clk", .data = &sun6i_a31_usb_gates_data,}, @@ -1202,6 +1300,7 @@ static void __init sunxi_init_clocks(const char *clocks[], int nclocks) static const char *sun4i_a10_critical_clocks[] __initdata = { "pll5_ddr", + "ahb_sdram", }; static void __init sun4i_a10_init_clocks(struct device_node *node) @@ -1214,6 +1313,7 @@ CLK_OF_DECLARE(sun4i_a10_clk_init, "allwinner,sun4i-a10", sun4i_a10_init_clocks) static const char *sun5i_critical_clocks[] __initdata = { "mbus", "pll5_ddr", + "ahb_sdram", }; static void __init sun5i_init_clocks(struct device_node *node) @@ -1236,3 +1336,4 @@ static void __init sun6i_init_clocks(struct device_node *node) ARRAY_SIZE(sun6i_critical_clocks)); } CLK_OF_DECLARE(sun6i_a31_clk_init, "allwinner,sun6i-a31", sun6i_init_clocks); +CLK_OF_DECLARE(sun8i_a23_clk_init, "allwinner,sun8i-a23", sun6i_init_clocks); diff --git a/drivers/clk/tegra/clk-pll.c b/drivers/clk/tegra/clk-pll.c index 637b62ccc91e..c7c6d8fb32fb 100644 --- a/drivers/clk/tegra/clk-pll.c +++ b/drivers/clk/tegra/clk-pll.c @@ -110,6 +110,12 @@ #define XUSBIO_PLL_CFG0_SEQ_ENABLE BIT(24) #define XUSBIO_PLL_CFG0_SEQ_START_STATE BIT(25) +#define SATA_PLL_CFG0 0x490 +#define SATA_PLL_CFG0_PADPLL_RESET_SWCTL BIT(0) +#define SATA_PLL_CFG0_PADPLL_USE_LOCKDET BIT(2) +#define SATA_PLL_CFG0_SEQ_ENABLE BIT(24) +#define SATA_PLL_CFG0_SEQ_START_STATE BIT(25) + #define PLLE_MISC_PLLE_PTS BIT(8) #define PLLE_MISC_IDDQ_SW_VALUE BIT(13) #define PLLE_MISC_IDDQ_SW_CTRL BIT(14) @@ -1361,6 +1367,19 @@ static int clk_plle_tegra114_enable(struct clk_hw *hw) val |= XUSBIO_PLL_CFG0_SEQ_ENABLE; pll_writel(val, XUSBIO_PLL_CFG0, pll); + /* Enable hw control of SATA pll */ + val = pll_readl(SATA_PLL_CFG0, pll); + val &= ~SATA_PLL_CFG0_PADPLL_RESET_SWCTL; + val |= SATA_PLL_CFG0_PADPLL_USE_LOCKDET; + val |= SATA_PLL_CFG0_SEQ_START_STATE; + pll_writel(val, SATA_PLL_CFG0, pll); + + udelay(1); + + val = pll_readl(SATA_PLL_CFG0, pll); + val |= SATA_PLL_CFG0_SEQ_ENABLE; + pll_writel(val, SATA_PLL_CFG0, pll); + out: if (pll->lock) spin_unlock_irqrestore(pll->lock, flags); diff --git a/drivers/clk/tegra/clk-tegra-periph.c b/drivers/clk/tegra/clk-tegra-periph.c index adf6b814b5bc..37f32c49674e 100644 --- a/drivers/clk/tegra/clk-tegra-periph.c +++ b/drivers/clk/tegra/clk-tegra-periph.c @@ -469,7 +469,7 @@ static struct tegra_periph_init_data periph_clks[] = { MUX("sata", mux_pllp_pllc_pllm_clkm, CLK_SOURCE_SATA, 124, TEGRA_PERIPH_ON_APB, tegra_clk_sata), MUX("adx1", mux_plla_pllc_pllp_clkm, CLK_SOURCE_ADX1, 180, TEGRA_PERIPH_ON_APB, tegra_clk_adx1), MUX("amx1", mux_plla_pllc_pllp_clkm, CLK_SOURCE_AMX1, 185, TEGRA_PERIPH_ON_APB, tegra_clk_amx1), - MUX("vi_sensor2", mux_pllm_pllc2_c_c3_pllp_plla, CLK_SOURCE_VI_SENSOR2, 20, TEGRA_PERIPH_NO_RESET, tegra_clk_vi_sensor2), + MUX("vi_sensor2", mux_pllm_pllc2_c_c3_pllp_plla, CLK_SOURCE_VI_SENSOR2, 165, TEGRA_PERIPH_NO_RESET, tegra_clk_vi_sensor2), MUX8("sdmmc1", mux_pllp_pllc2_c_c3_pllm_clkm, CLK_SOURCE_SDMMC1, 14, 0, tegra_clk_sdmmc1_8), MUX8("sdmmc2", mux_pllp_pllc2_c_c3_pllm_clkm, CLK_SOURCE_SDMMC2, 9, 0, tegra_clk_sdmmc2_8), MUX8("sdmmc3", mux_pllp_pllc2_c_c3_pllm_clkm, CLK_SOURCE_SDMMC3, 69, 0, tegra_clk_sdmmc3_8), @@ -487,7 +487,7 @@ static struct tegra_periph_init_data periph_clks[] = { MUX8("extern2", mux_plla_clk32_pllp_clkm_plle, CLK_SOURCE_EXTERN2, 121, 0, tegra_clk_extern2), MUX8("extern3", mux_plla_clk32_pllp_clkm_plle, CLK_SOURCE_EXTERN3, 122, 0, tegra_clk_extern3), MUX8("soc_therm", mux_pllm_pllc_pllp_plla, CLK_SOURCE_SOC_THERM, 78, TEGRA_PERIPH_ON_APB, tegra_clk_soc_therm), - MUX8("vi_sensor", mux_pllm_pllc2_c_c3_pllp_plla, CLK_SOURCE_VI_SENSOR, 20, TEGRA_PERIPH_NO_RESET, tegra_clk_vi_sensor_8), + MUX8("vi_sensor", mux_pllm_pllc2_c_c3_pllp_plla, CLK_SOURCE_VI_SENSOR, 164, TEGRA_PERIPH_NO_RESET, tegra_clk_vi_sensor_8), MUX8("isp", mux_pllm_pllc_pllp_plla_clkm_pllc4, CLK_SOURCE_ISP, 23, TEGRA_PERIPH_ON_APB, tegra_clk_isp_8), MUX8("entropy", mux_pllp_clkm1, CLK_SOURCE_ENTROPY, 149, 0, tegra_clk_entropy), MUX8("hdmi_audio", mux_pllp3_pllc_clkm, CLK_SOURCE_HDMI_AUDIO, 176, TEGRA_PERIPH_NO_RESET, tegra_clk_hdmi_audio), diff --git a/drivers/clk/tegra/clk-tegra114.c b/drivers/clk/tegra/clk-tegra114.c index b9c8ba258ef0..f760f31d05c4 100644 --- a/drivers/clk/tegra/clk-tegra114.c +++ b/drivers/clk/tegra/clk-tegra114.c @@ -151,6 +151,13 @@ /* Tegra CPU clock and reset control regs */ #define CLK_RST_CONTROLLER_CPU_CMPLX_STATUS 0x470 +#define MUX8(_name, _parents, _offset, \ + _clk_num, _gate_flags, _clk_id) \ + TEGRA_INIT_DATA_TABLE(_name, NULL, NULL, _parents, _offset,\ + 29, MASK(3), 0, 0, 8, 1, TEGRA_DIVIDER_ROUND_UP,\ + _clk_num, _gate_flags, _clk_id, _parents##_idx, 0,\ + NULL) + #ifdef CONFIG_PM_SLEEP static struct cpu_clk_suspend_context { u32 clk_csite_src; @@ -777,7 +784,6 @@ static struct tegra_clk tegra114_clks[tegra_clk_max] __initdata = { [tegra_clk_spdif_in] = { .dt_id = TEGRA114_CLK_SPDIF_IN, .present = true }, [tegra_clk_spdif_out] = { .dt_id = TEGRA114_CLK_SPDIF_OUT, .present = true }, [tegra_clk_vi_8] = { .dt_id = TEGRA114_CLK_VI, .present = true }, - [tegra_clk_vi_sensor_8] = { .dt_id = TEGRA114_CLK_VI_SENSOR, .present = true }, [tegra_clk_fuse] = { .dt_id = TEGRA114_CLK_FUSE, .present = true }, [tegra_clk_fuse_burn] = { .dt_id = TEGRA114_CLK_FUSE_BURN, .present = true }, [tegra_clk_clk_32k] = { .dt_id = TEGRA114_CLK_CLK_32K, .present = true }, @@ -923,6 +929,13 @@ static struct tegra_devclk devclks[] __initdata = { { .dev_id = "timer", .dt_id = TEGRA114_CLK_TIMER }, }; +static const char *mux_pllm_pllc2_c_c3_pllp_plla[] = { + "pll_m", "pll_c2", "pll_c", "pll_c3", "pll_p", "pll_a_out0" +}; +static u32 mux_pllm_pllc2_c_c3_pllp_plla_idx[] = { + [0] = 0, [1] = 1, [2] = 2, [3] = 3, [4] = 4, [5] = 6, +}; + static struct clk **clks; static unsigned long osc_freq; @@ -1178,10 +1191,18 @@ static void __init tegra114_pll_init(void __iomem *clk_base, clks[TEGRA114_CLK_PLL_E_OUT0] = clk; } +#define CLK_SOURCE_VI_SENSOR 0x1a8 + +static struct tegra_periph_init_data tegra_periph_clk_list[] = { + MUX8("vi_sensor", mux_pllm_pllc2_c_c3_pllp_plla, CLK_SOURCE_VI_SENSOR, 20, TEGRA_PERIPH_NO_RESET, TEGRA114_CLK_VI_SENSOR), +}; + static __init void tegra114_periph_clk_init(void __iomem *clk_base, void __iomem *pmc_base) { struct clk *clk; + struct tegra_periph_init_data *data; + int i; /* xusb_ss_div2 */ clk = clk_register_fixed_factor(NULL, "xusb_ss_div2", "xusb_ss_src", 0, @@ -1209,6 +1230,14 @@ static __init void tegra114_periph_clk_init(void __iomem *clk_base, clk_base + CLK_SOURCE_EMC, 29, 3, 0, NULL); + for (i = 0; i < ARRAY_SIZE(tegra_periph_clk_list); i++) { + data = &tegra_periph_clk_list[i]; + clk = tegra_clk_register_periph(data->name, + data->p.parent_names, data->num_parents, + &data->periph, clk_base, data->offset, data->flags); + clks[data->clk_id] = clk; + } + tegra_periph_clk_init(clk_base, pmc_base, tegra114_clks, &pll_p_params); } diff --git a/drivers/clk/tegra/clk-tegra124.c b/drivers/clk/tegra/clk-tegra124.c index 80efe51fdcdf..9525c684d149 100644 --- a/drivers/clk/tegra/clk-tegra124.c +++ b/drivers/clk/tegra/clk-tegra124.c @@ -869,7 +869,7 @@ static struct tegra_clk tegra124_clks[tegra_clk_max] __initdata = { [tegra_clk_spdif_in] = { .dt_id = TEGRA124_CLK_SPDIF_IN, .present = true }, [tegra_clk_spdif_out] = { .dt_id = TEGRA124_CLK_SPDIF_OUT, .present = true }, [tegra_clk_vi_9] = { .dt_id = TEGRA124_CLK_VI, .present = true }, - [tegra_clk_vi_sensor] = { .dt_id = TEGRA124_CLK_VI_SENSOR, .present = true }, + [tegra_clk_vi_sensor_8] = { .dt_id = TEGRA124_CLK_VI_SENSOR, .present = true }, [tegra_clk_fuse] = { .dt_id = TEGRA124_CLK_FUSE, .present = true }, [tegra_clk_fuse_burn] = { .dt_id = TEGRA124_CLK_FUSE_BURN, .present = true }, [tegra_clk_clk_32k] = { .dt_id = TEGRA124_CLK_CLK_32K, .present = true }, @@ -1369,6 +1369,14 @@ static struct tegra_clk_init_table init_table[] __initdata = { {TEGRA124_CLK_XUSB_HS_SRC, TEGRA124_CLK_PLL_U_60M, 60000000, 0}, {TEGRA124_CLK_XUSB_FALCON_SRC, TEGRA124_CLK_PLL_RE_OUT, 224000000, 0}, {TEGRA124_CLK_XUSB_HOST_SRC, TEGRA124_CLK_PLL_RE_OUT, 112000000, 0}, + {TEGRA124_CLK_SATA, TEGRA124_CLK_PLL_P, 104000000, 0}, + {TEGRA124_CLK_SATA_OOB, TEGRA124_CLK_PLL_P, 204000000, 0}, + {TEGRA124_CLK_EMC, TEGRA124_CLK_CLK_MAX, 0, 1}, + {TEGRA124_CLK_CCLK_G, TEGRA124_CLK_CLK_MAX, 0, 1}, + {TEGRA124_CLK_MSELECT, TEGRA124_CLK_CLK_MAX, 0, 1}, + {TEGRA124_CLK_CSITE, TEGRA124_CLK_CLK_MAX, 0, 1}, + {TEGRA124_CLK_TSENSOR, TEGRA124_CLK_CLK_M, 400000, 0}, + {TEGRA124_CLK_SOC_THERM, TEGRA124_CLK_PLL_P, 51000000, 0}, /* This MUST be the last entry. */ {TEGRA124_CLK_CLK_MAX, TEGRA124_CLK_CLK_MAX, 0, 0}, }; diff --git a/drivers/clk/tegra/clk.c b/drivers/clk/tegra/clk.c index c0a7d7723510..bf452b62beb8 100644 --- a/drivers/clk/tegra/clk.c +++ b/drivers/clk/tegra/clk.c @@ -277,6 +277,12 @@ void __init tegra_register_devclks(struct tegra_devclk *dev_clks, int num) for (i = 0; i < num; i++, dev_clks++) clk_register_clkdev(clks[dev_clks->dt_id], dev_clks->con_id, dev_clks->dev_id); + + for (i = 0; i < clk_num; i++) { + if (!IS_ERR_OR_NULL(clks[i])) + clk_register_clkdev(clks[i], __clk_get_name(clks[i]), + "tegra-clk-debug"); + } } struct clk ** __init tegra_lookup_dt_id(int clk_id, diff --git a/drivers/clk/ti/clk-7xx.c b/drivers/clk/ti/clk-7xx.c index cb8e6f14e880..62ac8f6e480c 100644 --- a/drivers/clk/ti/clk-7xx.c +++ b/drivers/clk/ti/clk-7xx.c @@ -18,6 +18,7 @@ #define DRA7_DPLL_ABE_DEFFREQ 180633600 #define DRA7_DPLL_GMAC_DEFFREQ 1000000000 +#define DRA7_DPLL_USB_DEFFREQ 960000000 static struct ti_dt_clk dra7xx_clks[] = { @@ -332,5 +333,15 @@ int __init dra7xx_dt_clk_init(void) if (rc) pr_err("%s: failed to configure GMAC DPLL!\n", __func__); + dpll_ck = clk_get_sys(NULL, "dpll_usb_ck"); + rc = clk_set_rate(dpll_ck, DRA7_DPLL_USB_DEFFREQ); + if (rc) + pr_err("%s: failed to configure USB DPLL!\n", __func__); + + dpll_ck = clk_get_sys(NULL, "dpll_usb_m2_ck"); + rc = clk_set_rate(dpll_ck, DRA7_DPLL_USB_DEFFREQ/2); + if (rc) + pr_err("%s: failed to set USB_DPLL M2 OUT\n", __func__); + return rc; } diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 02f177aeb16c..2fb0fdfc87df 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -391,7 +391,7 @@ config CRYPTO_DEV_ATMEL_SHA config CRYPTO_DEV_CCP bool "Support for AMD Cryptographic Coprocessor" - depends on X86 && PCI + depends on (X86 && PCI) || ARM64 default n help The AMD Cryptographic Coprocessor provides hardware support @@ -418,4 +418,22 @@ config CRYPTO_DEV_MXS_DCP To compile this driver as a module, choose M here: the module will be called mxs-dcp. +source "drivers/crypto/qat/Kconfig" + +config CRYPTO_DEV_QCE + tristate "Qualcomm crypto engine accelerator" + depends on (ARCH_QCOM || COMPILE_TEST) && HAS_DMA && HAS_IOMEM + select CRYPTO_AES + select CRYPTO_DES + select CRYPTO_ECB + select CRYPTO_CBC + select CRYPTO_XTS + select CRYPTO_CTR + select CRYPTO_ALGAPI + select CRYPTO_BLKCIPHER + help + This driver supports Qualcomm crypto engine accelerator + hardware. To compile this driver as a module, choose M here. The + module will be called qcrypto. + endif # CRYPTO_HW diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile index 482f090d16d0..3924f93d5774 100644 --- a/drivers/crypto/Makefile +++ b/drivers/crypto/Makefile @@ -23,3 +23,5 @@ obj-$(CONFIG_CRYPTO_DEV_S5P) += s5p-sss.o obj-$(CONFIG_CRYPTO_DEV_SAHARA) += sahara.o obj-$(CONFIG_CRYPTO_DEV_TALITOS) += talitos.o obj-$(CONFIG_CRYPTO_DEV_UX500) += ux500/ +obj-$(CONFIG_CRYPTO_DEV_QAT) += qat/ +obj-$(CONFIG_CRYPTO_DEV_QCE) += qce/ diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index 37f9cc98ba17..e4c6c58fbb03 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -1292,7 +1292,7 @@ static struct platform_driver crypto4xx_driver = { .of_match_table = crypto4xx_match, }, .probe = crypto4xx_probe, - .remove = crypto4xx_remove, + .remove = __exit_p(crypto4xx_remove), }; module_platform_driver(crypto4xx_driver); diff --git a/drivers/crypto/atmel-sha.c b/drivers/crypto/atmel-sha.c index 0618be06b9fb..9a4f69eaa5e0 100644 --- a/drivers/crypto/atmel-sha.c +++ b/drivers/crypto/atmel-sha.c @@ -1353,7 +1353,6 @@ static struct crypto_platform_data *atmel_sha_of_init(struct platform_device *pd GFP_KERNEL); if (!pdata->dma_slave) { dev_err(&pdev->dev, "could not allocate memory for dma_slave\n"); - devm_kfree(&pdev->dev, pdata); return ERR_PTR(-ENOMEM); } @@ -1375,7 +1374,8 @@ static int atmel_sha_probe(struct platform_device *pdev) unsigned long sha_phys_size; int err; - sha_dd = kzalloc(sizeof(struct atmel_sha_dev), GFP_KERNEL); + sha_dd = devm_kzalloc(&pdev->dev, sizeof(struct atmel_sha_dev), + GFP_KERNEL); if (sha_dd == NULL) { dev_err(dev, "unable to alloc data struct.\n"); err = -ENOMEM; @@ -1490,8 +1490,6 @@ clk_err: free_irq(sha_dd->irq, sha_dd); res_err: tasklet_kill(&sha_dd->done_task); - kfree(sha_dd); - sha_dd = NULL; sha_dd_err: dev_err(dev, "initialization failed.\n"); @@ -1523,9 +1521,6 @@ static int atmel_sha_remove(struct platform_device *pdev) if (sha_dd->irq >= 0) free_irq(sha_dd->irq, sha_dd); - kfree(sha_dd); - sha_dd = NULL; - return 0; } diff --git a/drivers/crypto/atmel-tdes.c b/drivers/crypto/atmel-tdes.c index 6cde5b530c69..d3a9041938ea 100644 --- a/drivers/crypto/atmel-tdes.c +++ b/drivers/crypto/atmel-tdes.c @@ -1337,7 +1337,6 @@ static struct crypto_platform_data *atmel_tdes_of_init(struct platform_device *p GFP_KERNEL); if (!pdata->dma_slave) { dev_err(&pdev->dev, "could not allocate memory for dma_slave\n"); - devm_kfree(&pdev->dev, pdata); return ERR_PTR(-ENOMEM); } @@ -1359,7 +1358,7 @@ static int atmel_tdes_probe(struct platform_device *pdev) unsigned long tdes_phys_size; int err; - tdes_dd = kzalloc(sizeof(struct atmel_tdes_dev), GFP_KERNEL); + tdes_dd = devm_kmalloc(&pdev->dev, sizeof(*tdes_dd), GFP_KERNEL); if (tdes_dd == NULL) { dev_err(dev, "unable to alloc data struct.\n"); err = -ENOMEM; @@ -1483,8 +1482,6 @@ tdes_irq_err: res_err: tasklet_kill(&tdes_dd->done_task); tasklet_kill(&tdes_dd->queue_task); - kfree(tdes_dd); - tdes_dd = NULL; tdes_dd_err: dev_err(dev, "initialization failed.\n"); @@ -1519,9 +1516,6 @@ static int atmel_tdes_remove(struct platform_device *pdev) if (tdes_dd->irq >= 0) free_irq(tdes_dd->irq, tdes_dd); - kfree(tdes_dd); - tdes_dd = NULL; - return 0; } diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index c09ce1f040d3..a80ea853701d 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -97,6 +97,13 @@ static inline void append_dec_op1(u32 *desc, u32 type) { u32 *jump_cmd, *uncond_jump_cmd; + /* DK bit is valid only for AES */ + if ((type & OP_ALG_ALGSEL_MASK) != OP_ALG_ALGSEL_AES) { + append_operation(desc, type | OP_ALG_AS_INITFINAL | + OP_ALG_DECRYPT); + return; + } + jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD); append_operation(desc, type | OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT); @@ -786,7 +793,7 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, desc_bytes(desc), DMA_TO_DEVICE); - if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { + if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { dev_err(jrdev, "unable to map shared descriptor\n"); return -ENOMEM; } @@ -1313,8 +1320,13 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, DMA_FROM_DEVICE, dst_chained); } - /* Check if data are contiguous */ iv_dma = dma_map_single(jrdev, req->iv, ivsize, DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, iv_dma)) { + dev_err(jrdev, "unable to map IV\n"); + return ERR_PTR(-ENOMEM); + } + + /* Check if data are contiguous */ if (assoc_nents || sg_dma_address(req->assoc) + req->assoclen != iv_dma || src_nents || iv_dma + ivsize != sg_dma_address(req->src)) { @@ -1345,8 +1357,6 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, edesc->sec4_sg_bytes = sec4_sg_bytes; edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) + desc_bytes; - edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, - sec4_sg_bytes, DMA_TO_DEVICE); *all_contig_ptr = all_contig; sec4_sg_index = 0; @@ -1369,6 +1379,12 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, sg_to_sec4_sg_last(req->dst, dst_nents, edesc->sec4_sg + sec4_sg_index, 0); } + edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, + sec4_sg_bytes, DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { + dev_err(jrdev, "unable to map S/G table\n"); + return ERR_PTR(-ENOMEM); + } return edesc; } @@ -1494,8 +1510,13 @@ static struct aead_edesc *aead_giv_edesc_alloc(struct aead_givcrypt_request DMA_FROM_DEVICE, dst_chained); } - /* Check if data are contiguous */ iv_dma = dma_map_single(jrdev, greq->giv, ivsize, DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, iv_dma)) { + dev_err(jrdev, "unable to map IV\n"); + return ERR_PTR(-ENOMEM); + } + + /* Check if data are contiguous */ if (assoc_nents || sg_dma_address(req->assoc) + req->assoclen != iv_dma || src_nents || iv_dma + ivsize != sg_dma_address(req->src)) contig &= ~GIV_SRC_CONTIG; @@ -1534,8 +1555,6 @@ static struct aead_edesc *aead_giv_edesc_alloc(struct aead_givcrypt_request edesc->sec4_sg_bytes = sec4_sg_bytes; edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) + desc_bytes; - edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, - sec4_sg_bytes, DMA_TO_DEVICE); *contig_ptr = contig; sec4_sg_index = 0; @@ -1559,6 +1578,12 @@ static struct aead_edesc *aead_giv_edesc_alloc(struct aead_givcrypt_request sg_to_sec4_sg_last(req->dst, dst_nents, edesc->sec4_sg + sec4_sg_index, 0); } + edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, + sec4_sg_bytes, DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { + dev_err(jrdev, "unable to map S/G table\n"); + return ERR_PTR(-ENOMEM); + } return edesc; } @@ -1650,11 +1675,16 @@ static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request DMA_FROM_DEVICE, dst_chained); } + iv_dma = dma_map_single(jrdev, req->info, ivsize, DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, iv_dma)) { + dev_err(jrdev, "unable to map IV\n"); + return ERR_PTR(-ENOMEM); + } + /* * Check if iv can be contiguous with source and destination. * If so, include it. If not, create scatterlist. */ - iv_dma = dma_map_single(jrdev, req->info, ivsize, DMA_TO_DEVICE); if (!src_nents && iv_dma + ivsize == sg_dma_address(req->src)) iv_contig = true; else @@ -1693,6 +1723,11 @@ static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, sec4_sg_bytes, DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { + dev_err(jrdev, "unable to map S/G table\n"); + return ERR_PTR(-ENOMEM); + } + edesc->iv_dma = iv_dma; #ifdef DEBUG @@ -2441,8 +2476,37 @@ static struct caam_crypto_alg *caam_alg_alloc(struct caam_alg_template static int __init caam_algapi_init(void) { + struct device_node *dev_node; + struct platform_device *pdev; + struct device *ctrldev; + void *priv; int i = 0, err = 0; + dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); + if (!dev_node) { + dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); + if (!dev_node) + return -ENODEV; + } + + pdev = of_find_device_by_node(dev_node); + if (!pdev) { + of_node_put(dev_node); + return -ENODEV; + } + + ctrldev = &pdev->dev; + priv = dev_get_drvdata(ctrldev); + of_node_put(dev_node); + + /* + * If priv is NULL, it's probably because the caam driver wasn't + * properly initialized (e.g. RNG4 init failed). Thus, bail out here. + */ + if (!priv) + return -ENODEV; + + INIT_LIST_HEAD(&alg_list); /* register crypto algorithms the device supports */ diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c index 0d9284ef96a8..b464d03ebf40 100644 --- a/drivers/crypto/caam/caamhash.c +++ b/drivers/crypto/caam/caamhash.c @@ -137,13 +137,20 @@ struct caam_hash_state { /* Common job descriptor seq in/out ptr routines */ /* Map state->caam_ctx, and append seq_out_ptr command that points to it */ -static inline void map_seq_out_ptr_ctx(u32 *desc, struct device *jrdev, - struct caam_hash_state *state, - int ctx_len) +static inline int map_seq_out_ptr_ctx(u32 *desc, struct device *jrdev, + struct caam_hash_state *state, + int ctx_len) { state->ctx_dma = dma_map_single(jrdev, state->caam_ctx, ctx_len, DMA_FROM_DEVICE); + if (dma_mapping_error(jrdev, state->ctx_dma)) { + dev_err(jrdev, "unable to map ctx\n"); + return -ENOMEM; + } + append_seq_out_ptr(desc, state->ctx_dma, ctx_len, 0); + + return 0; } /* Map req->result, and append seq_out_ptr command that points to it */ @@ -201,14 +208,19 @@ try_buf_map_to_sec4_sg(struct device *jrdev, struct sec4_sg_entry *sec4_sg, } /* Map state->caam_ctx, and add it to link table */ -static inline void ctx_map_to_sec4_sg(u32 *desc, struct device *jrdev, - struct caam_hash_state *state, - int ctx_len, - struct sec4_sg_entry *sec4_sg, - u32 flag) +static inline int ctx_map_to_sec4_sg(u32 *desc, struct device *jrdev, + struct caam_hash_state *state, int ctx_len, + struct sec4_sg_entry *sec4_sg, u32 flag) { state->ctx_dma = dma_map_single(jrdev, state->caam_ctx, ctx_len, flag); + if (dma_mapping_error(jrdev, state->ctx_dma)) { + dev_err(jrdev, "unable to map ctx\n"); + return -ENOMEM; + } + dma_to_sec4_sg_one(sec4_sg, state->ctx_dma, ctx_len, 0); + + return 0; } /* Common shared descriptor commands */ @@ -487,11 +499,11 @@ static int hash_digest_key(struct caam_hash_ctx *ctx, const u8 *key_in, digestsize, 1); #endif } - *keylen = digestsize; - dma_unmap_single(jrdev, src_dma, *keylen, DMA_TO_DEVICE); dma_unmap_single(jrdev, dst_dma, digestsize, DMA_FROM_DEVICE); + *keylen = digestsize; + kfree(desc); return ret; @@ -706,7 +718,7 @@ static void ahash_done_ctx_src(struct device *jrdev, u32 *desc, u32 err, if (err) caam_jr_strstatus(jrdev, err); - ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE); + ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_TO_DEVICE); kfree(edesc); #ifdef DEBUG @@ -741,7 +753,7 @@ static void ahash_done_ctx_dst(struct device *jrdev, u32 *desc, u32 err, if (err) caam_jr_strstatus(jrdev, err); - ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE); + ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_FROM_DEVICE); kfree(edesc); #ifdef DEBUG @@ -808,12 +820,11 @@ static int ahash_update_ctx(struct ahash_request *req) edesc->sec4_sg_bytes = sec4_sg_bytes; edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) + DESC_JOB_IO_LEN; - edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, - sec4_sg_bytes, - DMA_TO_DEVICE); - ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len, - edesc->sec4_sg, DMA_BIDIRECTIONAL); + ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len, + edesc->sec4_sg, DMA_BIDIRECTIONAL); + if (ret) + return ret; state->buf_dma = try_buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, @@ -839,6 +850,14 @@ static int ahash_update_ctx(struct ahash_request *req) init_job_desc_shared(desc, ptr, sh_len, HDR_SHARE_DEFER | HDR_REVERSE); + edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, + sec4_sg_bytes, + DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { + dev_err(jrdev, "unable to map S/G table\n"); + return -ENOMEM; + } + append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len + to_hash, LDST_SGF); @@ -911,23 +930,34 @@ static int ahash_final_ctx(struct ahash_request *req) edesc->sec4_sg_bytes = sec4_sg_bytes; edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) + DESC_JOB_IO_LEN; - edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, - sec4_sg_bytes, DMA_TO_DEVICE); edesc->src_nents = 0; - ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len, edesc->sec4_sg, - DMA_TO_DEVICE); + ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len, + edesc->sec4_sg, DMA_TO_DEVICE); + if (ret) + return ret; state->buf_dma = try_buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, buf, state->buf_dma, buflen, last_buflen); (edesc->sec4_sg + sec4_sg_bytes - 1)->len |= SEC4_SG_LEN_FIN; + edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, + sec4_sg_bytes, DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { + dev_err(jrdev, "unable to map S/G table\n"); + return -ENOMEM; + } + append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len + buflen, LDST_SGF); edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result, digestsize); + if (dma_mapping_error(jrdev, edesc->dst_dma)) { + dev_err(jrdev, "unable to map dst\n"); + return -ENOMEM; + } #ifdef DEBUG print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", @@ -989,11 +1019,11 @@ static int ahash_finup_ctx(struct ahash_request *req) edesc->sec4_sg_bytes = sec4_sg_bytes; edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) + DESC_JOB_IO_LEN; - edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, - sec4_sg_bytes, DMA_TO_DEVICE); - ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len, edesc->sec4_sg, - DMA_TO_DEVICE); + ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len, + edesc->sec4_sg, DMA_TO_DEVICE); + if (ret) + return ret; state->buf_dma = try_buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, buf, state->buf_dma, buflen, @@ -1002,11 +1032,22 @@ static int ahash_finup_ctx(struct ahash_request *req) src_map_to_sec4_sg(jrdev, req->src, src_nents, edesc->sec4_sg + sec4_sg_src_index, chained); + edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, + sec4_sg_bytes, DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { + dev_err(jrdev, "unable to map S/G table\n"); + return -ENOMEM; + } + append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len + buflen + req->nbytes, LDST_SGF); edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result, digestsize); + if (dma_mapping_error(jrdev, edesc->dst_dma)) { + dev_err(jrdev, "unable to map dst\n"); + return -ENOMEM; + } #ifdef DEBUG print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", @@ -1056,8 +1097,7 @@ static int ahash_digest(struct ahash_request *req) } edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) + DESC_JOB_IO_LEN; - edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, - sec4_sg_bytes, DMA_TO_DEVICE); + edesc->sec4_sg_bytes = sec4_sg_bytes; edesc->src_nents = src_nents; edesc->chained = chained; @@ -1067,6 +1107,12 @@ static int ahash_digest(struct ahash_request *req) if (src_nents) { sg_to_sec4_sg_last(req->src, src_nents, edesc->sec4_sg, 0); + edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, + sec4_sg_bytes, DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { + dev_err(jrdev, "unable to map S/G table\n"); + return -ENOMEM; + } src_dma = edesc->sec4_sg_dma; options = LDST_SGF; } else { @@ -1077,6 +1123,10 @@ static int ahash_digest(struct ahash_request *req) edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result, digestsize); + if (dma_mapping_error(jrdev, edesc->dst_dma)) { + dev_err(jrdev, "unable to map dst\n"); + return -ENOMEM; + } #ifdef DEBUG print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", @@ -1125,11 +1175,19 @@ static int ahash_final_no_ctx(struct ahash_request *req) init_job_desc_shared(desc, ptr, sh_len, HDR_SHARE_DEFER | HDR_REVERSE); state->buf_dma = dma_map_single(jrdev, buf, buflen, DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, state->buf_dma)) { + dev_err(jrdev, "unable to map src\n"); + return -ENOMEM; + } append_seq_in_ptr(desc, state->buf_dma, buflen, 0); edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result, digestsize); + if (dma_mapping_error(jrdev, edesc->dst_dma)) { + dev_err(jrdev, "unable to map dst\n"); + return -ENOMEM; + } edesc->src_nents = 0; #ifdef DEBUG @@ -1197,9 +1255,7 @@ static int ahash_update_no_ctx(struct ahash_request *req) edesc->sec4_sg_bytes = sec4_sg_bytes; edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) + DESC_JOB_IO_LEN; - edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, - sec4_sg_bytes, - DMA_TO_DEVICE); + edesc->dst_dma = 0; state->buf_dma = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, buf, *buflen); @@ -1216,9 +1272,19 @@ static int ahash_update_no_ctx(struct ahash_request *req) init_job_desc_shared(desc, ptr, sh_len, HDR_SHARE_DEFER | HDR_REVERSE); + edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, + sec4_sg_bytes, + DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { + dev_err(jrdev, "unable to map S/G table\n"); + return -ENOMEM; + } + append_seq_in_ptr(desc, edesc->sec4_sg_dma, to_hash, LDST_SGF); - map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len); + ret = map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len); + if (ret) + return ret; #ifdef DEBUG print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", @@ -1297,8 +1363,6 @@ static int ahash_finup_no_ctx(struct ahash_request *req) edesc->sec4_sg_bytes = sec4_sg_bytes; edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) + DESC_JOB_IO_LEN; - edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, - sec4_sg_bytes, DMA_TO_DEVICE); state->buf_dma = try_buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, buf, state->buf_dma, buflen, @@ -1307,11 +1371,22 @@ static int ahash_finup_no_ctx(struct ahash_request *req) src_map_to_sec4_sg(jrdev, req->src, src_nents, edesc->sec4_sg + 1, chained); + edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, + sec4_sg_bytes, DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { + dev_err(jrdev, "unable to map S/G table\n"); + return -ENOMEM; + } + append_seq_in_ptr(desc, edesc->sec4_sg_dma, buflen + req->nbytes, LDST_SGF); edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result, digestsize); + if (dma_mapping_error(jrdev, edesc->dst_dma)) { + dev_err(jrdev, "unable to map dst\n"); + return -ENOMEM; + } #ifdef DEBUG print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", @@ -1380,13 +1455,19 @@ static int ahash_update_first(struct ahash_request *req) edesc->sec4_sg_bytes = sec4_sg_bytes; edesc->sec4_sg = (void *)edesc + sizeof(struct ahash_edesc) + DESC_JOB_IO_LEN; - edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, - sec4_sg_bytes, - DMA_TO_DEVICE); + edesc->dst_dma = 0; if (src_nents) { sg_to_sec4_sg_last(req->src, src_nents, edesc->sec4_sg, 0); + edesc->sec4_sg_dma = dma_map_single(jrdev, + edesc->sec4_sg, + sec4_sg_bytes, + DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { + dev_err(jrdev, "unable to map S/G table\n"); + return -ENOMEM; + } src_dma = edesc->sec4_sg_dma; options = LDST_SGF; } else { @@ -1404,7 +1485,9 @@ static int ahash_update_first(struct ahash_request *req) append_seq_in_ptr(desc, src_dma, to_hash, options); - map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len); + ret = map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len); + if (ret) + return ret; #ifdef DEBUG print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", @@ -1453,6 +1536,7 @@ static int ahash_init(struct ahash_request *req) state->final = ahash_final_no_ctx; state->current_buf = 0; + state->buf_dma = 0; return 0; } @@ -1787,8 +1871,36 @@ caam_hash_alloc(struct caam_hash_template *template, static int __init caam_algapi_hash_init(void) { + struct device_node *dev_node; + struct platform_device *pdev; + struct device *ctrldev; + void *priv; int i = 0, err = 0; + dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); + if (!dev_node) { + dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); + if (!dev_node) + return -ENODEV; + } + + pdev = of_find_device_by_node(dev_node); + if (!pdev) { + of_node_put(dev_node); + return -ENODEV; + } + + ctrldev = &pdev->dev; + priv = dev_get_drvdata(ctrldev); + of_node_put(dev_node); + + /* + * If priv is NULL, it's probably because the caam driver wasn't + * properly initialized (e.g. RNG4 init failed). Thus, bail out here. + */ + if (!priv) + return -ENODEV; + INIT_LIST_HEAD(&hash_list); /* register crypto algorithms the device supports */ diff --git a/drivers/crypto/caam/caamrng.c b/drivers/crypto/caam/caamrng.c index 8c07d3153f12..ae31e555793c 100644 --- a/drivers/crypto/caam/caamrng.c +++ b/drivers/crypto/caam/caamrng.c @@ -185,7 +185,7 @@ static int caam_read(struct hwrng *rng, void *data, size_t max, bool wait) max - copied_idx, false); } -static inline void rng_create_sh_desc(struct caam_rng_ctx *ctx) +static inline int rng_create_sh_desc(struct caam_rng_ctx *ctx) { struct device *jrdev = ctx->jrdev; u32 *desc = ctx->sh_desc; @@ -203,13 +203,18 @@ static inline void rng_create_sh_desc(struct caam_rng_ctx *ctx) ctx->sh_desc_dma = dma_map_single(jrdev, desc, desc_bytes(desc), DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, ctx->sh_desc_dma)) { + dev_err(jrdev, "unable to map shared descriptor\n"); + return -ENOMEM; + } #ifdef DEBUG print_hex_dump(KERN_ERR, "rng shdesc@: ", DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); #endif + return 0; } -static inline void rng_create_job_desc(struct caam_rng_ctx *ctx, int buf_id) +static inline int rng_create_job_desc(struct caam_rng_ctx *ctx, int buf_id) { struct device *jrdev = ctx->jrdev; struct buf_data *bd = &ctx->bufs[buf_id]; @@ -220,12 +225,17 @@ static inline void rng_create_job_desc(struct caam_rng_ctx *ctx, int buf_id) HDR_REVERSE); bd->addr = dma_map_single(jrdev, bd->buf, RN_BUF_SIZE, DMA_FROM_DEVICE); + if (dma_mapping_error(jrdev, bd->addr)) { + dev_err(jrdev, "unable to map dst\n"); + return -ENOMEM; + } append_seq_out_ptr_intlen(desc, bd->addr, RN_BUF_SIZE, 0); #ifdef DEBUG print_hex_dump(KERN_ERR, "rng job desc@: ", DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); #endif + return 0; } static void caam_cleanup(struct hwrng *rng) @@ -242,24 +252,44 @@ static void caam_cleanup(struct hwrng *rng) rng_unmap_ctx(rng_ctx); } -static void caam_init_buf(struct caam_rng_ctx *ctx, int buf_id) +static int caam_init_buf(struct caam_rng_ctx *ctx, int buf_id) { struct buf_data *bd = &ctx->bufs[buf_id]; + int err; + + err = rng_create_job_desc(ctx, buf_id); + if (err) + return err; - rng_create_job_desc(ctx, buf_id); atomic_set(&bd->empty, BUF_EMPTY); submit_job(ctx, buf_id == ctx->current_buf); wait_for_completion(&bd->filled); + + return 0; } -static void caam_init_rng(struct caam_rng_ctx *ctx, struct device *jrdev) +static int caam_init_rng(struct caam_rng_ctx *ctx, struct device *jrdev) { + int err; + ctx->jrdev = jrdev; - rng_create_sh_desc(ctx); + + err = rng_create_sh_desc(ctx); + if (err) + return err; + ctx->current_buf = 0; ctx->cur_buf_idx = 0; - caam_init_buf(ctx, 0); - caam_init_buf(ctx, 1); + + err = caam_init_buf(ctx, 0); + if (err) + return err; + + err = caam_init_buf(ctx, 1); + if (err) + return err; + + return 0; } static struct hwrng caam_rng = { @@ -278,6 +308,35 @@ static void __exit caam_rng_exit(void) static int __init caam_rng_init(void) { struct device *dev; + struct device_node *dev_node; + struct platform_device *pdev; + struct device *ctrldev; + void *priv; + int err; + + dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); + if (!dev_node) { + dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); + if (!dev_node) + return -ENODEV; + } + + pdev = of_find_device_by_node(dev_node); + if (!pdev) { + of_node_put(dev_node); + return -ENODEV; + } + + ctrldev = &pdev->dev; + priv = dev_get_drvdata(ctrldev); + of_node_put(dev_node); + + /* + * If priv is NULL, it's probably because the caam driver wasn't + * properly initialized (e.g. RNG4 init failed). Thus, bail out here. + */ + if (!priv) + return -ENODEV; dev = caam_jr_alloc(); if (IS_ERR(dev)) { @@ -287,7 +346,9 @@ static int __init caam_rng_init(void) rng_ctx = kmalloc(sizeof(struct caam_rng_ctx), GFP_DMA); if (!rng_ctx) return -ENOMEM; - caam_init_rng(rng_ctx, dev); + err = caam_init_rng(rng_ctx, dev); + if (err) + return err; dev_info(dev, "registering rng-caam\n"); return hwrng_register(&caam_rng); diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c index 1c38f86bf63a..3cade79ea41e 100644 --- a/drivers/crypto/caam/ctrl.c +++ b/drivers/crypto/caam/ctrl.c @@ -5,6 +5,7 @@ * Copyright 2008-2012 Freescale Semiconductor, Inc. */ +#include <linux/device.h> #include <linux/of_address.h> #include <linux/of_irq.h> @@ -87,6 +88,17 @@ static inline int run_descriptor_deco0(struct device *ctrldev, u32 *desc, /* Set the bit to request direct access to DECO0 */ topregs = (struct caam_full __iomem *)ctrlpriv->ctrl; + + if (ctrlpriv->virt_en == 1) { + setbits32(&topregs->ctrl.deco_rsr, DECORSR_JR0); + + while (!(rd_reg32(&topregs->ctrl.deco_rsr) & DECORSR_VALID) && + --timeout) + cpu_relax(); + + timeout = 100000; + } + setbits32(&topregs->ctrl.deco_rq, DECORR_RQD0ENABLE); while (!(rd_reg32(&topregs->ctrl.deco_rq) & DECORR_DEN0) && @@ -129,6 +141,9 @@ static inline int run_descriptor_deco0(struct device *ctrldev, u32 *desc, *status = rd_reg32(&topregs->deco.op_status_hi) & DECO_OP_STATUS_HI_ERR_MASK; + if (ctrlpriv->virt_en == 1) + clrbits32(&topregs->ctrl.deco_rsr, DECORSR_JR0); + /* Mark the DECO as free */ clrbits32(&topregs->ctrl.deco_rq, DECORR_RQD0ENABLE); @@ -295,9 +310,6 @@ static int caam_remove(struct platform_device *pdev) /* Unmap controller region */ iounmap(&topregs->ctrl); - kfree(ctrlpriv->jrpdev); - kfree(ctrlpriv); - return ret; } @@ -380,9 +392,11 @@ static int caam_probe(struct platform_device *pdev) #ifdef CONFIG_DEBUG_FS struct caam_perfmon *perfmon; #endif - u64 cha_vid; + u32 scfgr, comp_params; + u32 cha_vid_ls; - ctrlpriv = kzalloc(sizeof(struct caam_drv_private), GFP_KERNEL); + ctrlpriv = devm_kzalloc(&pdev->dev, sizeof(struct caam_drv_private), + GFP_KERNEL); if (!ctrlpriv) return -ENOMEM; @@ -413,13 +427,40 @@ static int caam_probe(struct platform_device *pdev) setbits32(&topregs->ctrl.mcr, MCFGR_WDENABLE | (sizeof(dma_addr_t) == sizeof(u64) ? MCFGR_LONG_PTR : 0)); + /* + * Read the Compile Time paramters and SCFGR to determine + * if Virtualization is enabled for this platform + */ + comp_params = rd_reg32(&topregs->ctrl.perfmon.comp_parms_ms); + scfgr = rd_reg32(&topregs->ctrl.scfgr); + + ctrlpriv->virt_en = 0; + if (comp_params & CTPR_MS_VIRT_EN_INCL) { + /* VIRT_EN_INCL = 1 & VIRT_EN_POR = 1 or + * VIRT_EN_INCL = 1 & VIRT_EN_POR = 0 & SCFGR_VIRT_EN = 1 + */ + if ((comp_params & CTPR_MS_VIRT_EN_POR) || + (!(comp_params & CTPR_MS_VIRT_EN_POR) && + (scfgr & SCFGR_VIRT_EN))) + ctrlpriv->virt_en = 1; + } else { + /* VIRT_EN_INCL = 0 && VIRT_EN_POR_VALUE = 1 */ + if (comp_params & CTPR_MS_VIRT_EN_POR) + ctrlpriv->virt_en = 1; + } + + if (ctrlpriv->virt_en == 1) + setbits32(&topregs->ctrl.jrstart, JRSTART_JR0_START | + JRSTART_JR1_START | JRSTART_JR2_START | + JRSTART_JR3_START); + if (sizeof(dma_addr_t) == sizeof(u64)) if (of_device_is_compatible(nprop, "fsl,sec-v5.0")) - dma_set_mask(dev, DMA_BIT_MASK(40)); + dma_set_mask_and_coherent(dev, DMA_BIT_MASK(40)); else - dma_set_mask(dev, DMA_BIT_MASK(36)); + dma_set_mask_and_coherent(dev, DMA_BIT_MASK(36)); else - dma_set_mask(dev, DMA_BIT_MASK(32)); + dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); /* * Detect and enable JobRs @@ -432,8 +473,9 @@ static int caam_probe(struct platform_device *pdev) of_device_is_compatible(np, "fsl,sec4.0-job-ring")) rspec++; - ctrlpriv->jrpdev = kzalloc(sizeof(struct platform_device *) * rspec, - GFP_KERNEL); + ctrlpriv->jrpdev = devm_kzalloc(&pdev->dev, + sizeof(struct platform_device *) * rspec, + GFP_KERNEL); if (ctrlpriv->jrpdev == NULL) { iounmap(&topregs->ctrl); return -ENOMEM; @@ -456,8 +498,9 @@ static int caam_probe(struct platform_device *pdev) } /* Check to see if QI present. If so, enable */ - ctrlpriv->qi_present = !!(rd_reg64(&topregs->ctrl.perfmon.comp_parms) & - CTPR_QI_MASK); + ctrlpriv->qi_present = + !!(rd_reg32(&topregs->ctrl.perfmon.comp_parms_ms) & + CTPR_MS_QI_MASK); if (ctrlpriv->qi_present) { ctrlpriv->qi = (struct caam_queue_if __force *)&topregs->qi; /* This is all that's required to physically enable QI */ @@ -471,13 +514,13 @@ static int caam_probe(struct platform_device *pdev) return -ENOMEM; } - cha_vid = rd_reg64(&topregs->ctrl.perfmon.cha_id); + cha_vid_ls = rd_reg32(&topregs->ctrl.perfmon.cha_id_ls); /* * If SEC has RNG version >= 4 and RNG state handle has not been * already instantiated, do RNG instantiation */ - if ((cha_vid & CHA_ID_RNG_MASK) >> CHA_ID_RNG_SHIFT >= 4) { + if ((cha_vid_ls & CHA_ID_LS_RNG_MASK) >> CHA_ID_LS_RNG_SHIFT >= 4) { ctrlpriv->rng4_sh_init = rd_reg32(&topregs->ctrl.r4tst[0].rdsta); /* @@ -531,7 +574,8 @@ static int caam_probe(struct platform_device *pdev) /* NOTE: RTIC detection ought to go here, around Si time */ - caam_id = rd_reg64(&topregs->ctrl.perfmon.caam_id); + caam_id = (u64)rd_reg32(&topregs->ctrl.perfmon.caam_id_ms) << 32 | + (u64)rd_reg32(&topregs->ctrl.perfmon.caam_id_ls); /* Report "alive" for developer to see */ dev_info(dev, "device ID = 0x%016llx (Era %d)\n", caam_id, @@ -547,7 +591,7 @@ static int caam_probe(struct platform_device *pdev) */ perfmon = (struct caam_perfmon __force *)&ctrl->perfmon; - ctrlpriv->dfs_root = debugfs_create_dir("caam", NULL); + ctrlpriv->dfs_root = debugfs_create_dir(dev_name(dev), NULL); ctrlpriv->ctl = debugfs_create_dir("ctl", ctrlpriv->dfs_root); /* Controller-level - performance monitor counters */ diff --git a/drivers/crypto/caam/desc.h b/drivers/crypto/caam/desc.h index 7e4500f18df6..d397ff9d56fd 100644 --- a/drivers/crypto/caam/desc.h +++ b/drivers/crypto/caam/desc.h @@ -321,7 +321,6 @@ struct sec4_sg_entry { /* Continue - Not the last FIFO store to come */ #define FIFOST_CONT_SHIFT 23 #define FIFOST_CONT_MASK (1 << FIFOST_CONT_SHIFT) -#define FIFOST_CONT_MASK (1 << FIFOST_CONT_SHIFT) /* * Extended Length - use 32-bit extended length that diff --git a/drivers/crypto/caam/intern.h b/drivers/crypto/caam/intern.h index 6d85fcc5bd0a..97363db4e56e 100644 --- a/drivers/crypto/caam/intern.h +++ b/drivers/crypto/caam/intern.h @@ -82,6 +82,7 @@ struct caam_drv_private { u8 total_jobrs; /* Total Job Rings in device */ u8 qi_present; /* Nonzero if QI present in device */ int secvio_irq; /* Security violation interrupt number */ + int virt_en; /* Virtualization enabled in CAAM */ #define RNG4_MAX_HANDLES 2 /* RNG4 block */ diff --git a/drivers/crypto/caam/jr.c b/drivers/crypto/caam/jr.c index b512a4ba7569..4d18e27ffa9e 100644 --- a/drivers/crypto/caam/jr.c +++ b/drivers/crypto/caam/jr.c @@ -476,11 +476,11 @@ static int caam_jr_probe(struct platform_device *pdev) if (sizeof(dma_addr_t) == sizeof(u64)) if (of_device_is_compatible(nprop, "fsl,sec-v5.0-job-ring")) - dma_set_mask(jrdev, DMA_BIT_MASK(40)); + dma_set_mask_and_coherent(jrdev, DMA_BIT_MASK(40)); else - dma_set_mask(jrdev, DMA_BIT_MASK(36)); + dma_set_mask_and_coherent(jrdev, DMA_BIT_MASK(36)); else - dma_set_mask(jrdev, DMA_BIT_MASK(32)); + dma_set_mask_and_coherent(jrdev, DMA_BIT_MASK(32)); /* Identify the interrupt */ jrpriv->irq = irq_of_parse_and_map(nprop, 0); diff --git a/drivers/crypto/caam/regs.h b/drivers/crypto/caam/regs.h index cbde8b95a6f8..f48e344ffc39 100644 --- a/drivers/crypto/caam/regs.h +++ b/drivers/crypto/caam/regs.h @@ -84,6 +84,7 @@ #endif #ifndef CONFIG_64BIT +#ifdef __BIG_ENDIAN static inline void wr_reg64(u64 __iomem *reg, u64 data) { wr_reg32((u32 __iomem *)reg, (data & 0xffffffff00000000ull) >> 32); @@ -95,6 +96,21 @@ static inline u64 rd_reg64(u64 __iomem *reg) return (((u64)rd_reg32((u32 __iomem *)reg)) << 32) | ((u64)rd_reg32((u32 __iomem *)reg + 1)); } +#else +#ifdef __LITTLE_ENDIAN +static inline void wr_reg64(u64 __iomem *reg, u64 data) +{ + wr_reg32((u32 __iomem *)reg + 1, (data & 0xffffffff00000000ull) >> 32); + wr_reg32((u32 __iomem *)reg, data & 0x00000000ffffffffull); +} + +static inline u64 rd_reg64(u64 __iomem *reg) +{ + return (((u64)rd_reg32((u32 __iomem *)reg + 1)) << 32) | + ((u64)rd_reg32((u32 __iomem *)reg)); +} +#endif +#endif #endif /* @@ -114,45 +130,45 @@ struct jr_outentry { */ /* Number of DECOs */ -#define CHA_NUM_DECONUM_SHIFT 56 -#define CHA_NUM_DECONUM_MASK (0xfull << CHA_NUM_DECONUM_SHIFT) +#define CHA_NUM_MS_DECONUM_SHIFT 24 +#define CHA_NUM_MS_DECONUM_MASK (0xfull << CHA_NUM_MS_DECONUM_SHIFT) /* CHA Version IDs */ -#define CHA_ID_AES_SHIFT 0 -#define CHA_ID_AES_MASK (0xfull << CHA_ID_AES_SHIFT) +#define CHA_ID_LS_AES_SHIFT 0 +#define CHA_ID_LS_AES_MASK (0xfull << CHA_ID_LS_AES_SHIFT) -#define CHA_ID_DES_SHIFT 4 -#define CHA_ID_DES_MASK (0xfull << CHA_ID_DES_SHIFT) +#define CHA_ID_LS_DES_SHIFT 4 +#define CHA_ID_LS_DES_MASK (0xfull << CHA_ID_LS_DES_SHIFT) -#define CHA_ID_ARC4_SHIFT 8 -#define CHA_ID_ARC4_MASK (0xfull << CHA_ID_ARC4_SHIFT) +#define CHA_ID_LS_ARC4_SHIFT 8 +#define CHA_ID_LS_ARC4_MASK (0xfull << CHA_ID_LS_ARC4_SHIFT) -#define CHA_ID_MD_SHIFT 12 -#define CHA_ID_MD_MASK (0xfull << CHA_ID_MD_SHIFT) +#define CHA_ID_LS_MD_SHIFT 12 +#define CHA_ID_LS_MD_MASK (0xfull << CHA_ID_LS_MD_SHIFT) -#define CHA_ID_RNG_SHIFT 16 -#define CHA_ID_RNG_MASK (0xfull << CHA_ID_RNG_SHIFT) +#define CHA_ID_LS_RNG_SHIFT 16 +#define CHA_ID_LS_RNG_MASK (0xfull << CHA_ID_LS_RNG_SHIFT) -#define CHA_ID_SNW8_SHIFT 20 -#define CHA_ID_SNW8_MASK (0xfull << CHA_ID_SNW8_SHIFT) +#define CHA_ID_LS_SNW8_SHIFT 20 +#define CHA_ID_LS_SNW8_MASK (0xfull << CHA_ID_LS_SNW8_SHIFT) -#define CHA_ID_KAS_SHIFT 24 -#define CHA_ID_KAS_MASK (0xfull << CHA_ID_KAS_SHIFT) +#define CHA_ID_LS_KAS_SHIFT 24 +#define CHA_ID_LS_KAS_MASK (0xfull << CHA_ID_LS_KAS_SHIFT) -#define CHA_ID_PK_SHIFT 28 -#define CHA_ID_PK_MASK (0xfull << CHA_ID_PK_SHIFT) +#define CHA_ID_LS_PK_SHIFT 28 +#define CHA_ID_LS_PK_MASK (0xfull << CHA_ID_LS_PK_SHIFT) -#define CHA_ID_CRC_SHIFT 32 -#define CHA_ID_CRC_MASK (0xfull << CHA_ID_CRC_SHIFT) +#define CHA_ID_MS_CRC_SHIFT 0 +#define CHA_ID_MS_CRC_MASK (0xfull << CHA_ID_MS_CRC_SHIFT) -#define CHA_ID_SNW9_SHIFT 36 -#define CHA_ID_SNW9_MASK (0xfull << CHA_ID_SNW9_SHIFT) +#define CHA_ID_MS_SNW9_SHIFT 4 +#define CHA_ID_MS_SNW9_MASK (0xfull << CHA_ID_MS_SNW9_SHIFT) -#define CHA_ID_DECO_SHIFT 56 -#define CHA_ID_DECO_MASK (0xfull << CHA_ID_DECO_SHIFT) +#define CHA_ID_MS_DECO_SHIFT 24 +#define CHA_ID_MS_DECO_MASK (0xfull << CHA_ID_MS_DECO_SHIFT) -#define CHA_ID_JR_SHIFT 60 -#define CHA_ID_JR_MASK (0xfull << CHA_ID_JR_SHIFT) +#define CHA_ID_MS_JR_SHIFT 28 +#define CHA_ID_MS_JR_MASK (0xfull << CHA_ID_MS_JR_SHIFT) struct sec_vid { u16 ip_id; @@ -172,10 +188,14 @@ struct caam_perfmon { u64 rsvd[13]; /* CAAM Hardware Instantiation Parameters fa0-fbf */ - u64 cha_rev; /* CRNR - CHA Revision Number */ -#define CTPR_QI_SHIFT 57 -#define CTPR_QI_MASK (0x1ull << CTPR_QI_SHIFT) - u64 comp_parms; /* CTPR - Compile Parameters Register */ + u32 cha_rev_ms; /* CRNR - CHA Rev No. Most significant half*/ + u32 cha_rev_ls; /* CRNR - CHA Rev No. Least significant half*/ +#define CTPR_MS_QI_SHIFT 25 +#define CTPR_MS_QI_MASK (0x1ull << CTPR_MS_QI_SHIFT) +#define CTPR_MS_VIRT_EN_INCL 0x00000001 +#define CTPR_MS_VIRT_EN_POR 0x00000002 + u32 comp_parms_ms; /* CTPR - Compile Parameters Register */ + u32 comp_parms_ls; /* CTPR - Compile Parameters Register */ u64 rsvd1[2]; /* CAAM Global Status fc0-fdf */ @@ -189,9 +209,12 @@ struct caam_perfmon { /* Component Instantiation Parameters fe0-fff */ u32 rtic_id; /* RVID - RTIC Version ID */ u32 ccb_id; /* CCBVID - CCB Version ID */ - u64 cha_id; /* CHAVID - CHA Version ID */ - u64 cha_num; /* CHANUM - CHA Number */ - u64 caam_id; /* CAAMVID - CAAM Version ID */ + u32 cha_id_ms; /* CHAVID - CHA Version ID Most Significant*/ + u32 cha_id_ls; /* CHAVID - CHA Version ID Least Significant*/ + u32 cha_num_ms; /* CHANUM - CHA Number Most Significant */ + u32 cha_num_ls; /* CHANUM - CHA Number Least Significant*/ + u32 caam_id_ms; /* CAAMVID - CAAM Version ID MS */ + u32 caam_id_ls; /* CAAMVID - CAAM Version ID LS */ }; /* LIODN programming for DMA configuration */ @@ -304,9 +327,12 @@ struct caam_ctrl { /* Bus Access Configuration Section 010-11f */ /* Read/Writable */ struct masterid jr_mid[4]; /* JRxLIODNR - JobR LIODN setup */ - u32 rsvd3[12]; + u32 rsvd3[11]; + u32 jrstart; /* JRSTART - Job Ring Start Register */ struct masterid rtic_mid[4]; /* RTICxLIODNR - RTIC LIODN setup */ - u32 rsvd4[7]; + u32 rsvd4[5]; + u32 deco_rsr; /* DECORSR - Deco Request Source */ + u32 rsvd11; u32 deco_rq; /* DECORR - DECO Request */ struct partid deco_mid[5]; /* DECOxLIODNR - 1 per DECO */ u32 rsvd5[22]; @@ -347,7 +373,10 @@ struct caam_ctrl { #define MCFGR_DMA_RESET 0x10000000 #define MCFGR_LONG_PTR 0x00010000 /* Use >32-bit desc addressing */ #define SCFGR_RDBENABLE 0x00000400 +#define SCFGR_VIRT_EN 0x00008000 #define DECORR_RQD0ENABLE 0x00000001 /* Enable DECO0 for direct access */ +#define DECORSR_JR0 0x00000001 /* JR to supply TZ, SDID, ICID */ +#define DECORSR_VALID 0x80000000 #define DECORR_DEN0 0x00010000 /* DECO0 available for access*/ /* AXI read cache control */ @@ -365,6 +394,12 @@ struct caam_ctrl { #define MCFGR_AXIPRI 0x00000008 /* Assert AXI priority sideband */ #define MCFGR_BURST_64 0x00000001 /* Max burst size */ +/* JRSTART register offsets */ +#define JRSTART_JR0_START 0x00000001 /* Start Job ring 0 */ +#define JRSTART_JR1_START 0x00000002 /* Start Job ring 1 */ +#define JRSTART_JR2_START 0x00000004 /* Start Job ring 2 */ +#define JRSTART_JR3_START 0x00000008 /* Start Job ring 3 */ + /* * caam_job_ring - direct job ring setup * 1-4 possible per instantiation, base + 1000/2000/3000/4000 diff --git a/drivers/crypto/ccp/Makefile b/drivers/crypto/ccp/Makefile index d3505a018720..7f592d8d07bb 100644 --- a/drivers/crypto/ccp/Makefile +++ b/drivers/crypto/ccp/Makefile @@ -1,6 +1,11 @@ obj-$(CONFIG_CRYPTO_DEV_CCP_DD) += ccp.o ccp-objs := ccp-dev.o ccp-ops.o +ifdef CONFIG_X86 ccp-objs += ccp-pci.o +endif +ifdef CONFIG_ARM64 +ccp-objs += ccp-platform.o +endif obj-$(CONFIG_CRYPTO_DEV_CCP_CRYPTO) += ccp-crypto.o ccp-crypto-objs := ccp-crypto-main.o \ diff --git a/drivers/crypto/ccp/ccp-dev.c b/drivers/crypto/ccp/ccp-dev.c index 2c7816149b01..a7d110652a74 100644 --- a/drivers/crypto/ccp/ccp-dev.c +++ b/drivers/crypto/ccp/ccp-dev.c @@ -20,7 +20,9 @@ #include <linux/delay.h> #include <linux/hw_random.h> #include <linux/cpu.h> +#ifdef CONFIG_X86 #include <asm/cpu_device_id.h> +#endif #include <linux/ccp.h> #include "ccp-dev.h" @@ -360,6 +362,12 @@ int ccp_init(struct ccp_device *ccp) /* Build queue interrupt mask (two interrupts per queue) */ qim |= cmd_q->int_ok | cmd_q->int_err; +#ifdef CONFIG_ARM64 + /* For arm64 set the recommended queue cache settings */ + iowrite32(ccp->axcache, ccp->io_regs + CMD_Q_CACHE_BASE + + (CMD_Q_CACHE_INC * i)); +#endif + dev_dbg(dev, "queue #%u available\n", i); } if (ccp->cmd_q_count == 0) { @@ -558,12 +566,15 @@ bool ccp_queues_suspended(struct ccp_device *ccp) } #endif +#ifdef CONFIG_X86 static const struct x86_cpu_id ccp_support[] = { { X86_VENDOR_AMD, 22, }, }; +#endif static int __init ccp_mod_init(void) { +#ifdef CONFIG_X86 struct cpuinfo_x86 *cpuinfo = &boot_cpu_data; int ret; @@ -589,12 +600,30 @@ static int __init ccp_mod_init(void) break; } +#endif + +#ifdef CONFIG_ARM64 + int ret; + + ret = ccp_platform_init(); + if (ret) + return ret; + + /* Don't leave the driver loaded if init failed */ + if (!ccp_get_device()) { + ccp_platform_exit(); + return -ENODEV; + } + + return 0; +#endif return -ENODEV; } static void __exit ccp_mod_exit(void) { +#ifdef CONFIG_X86 struct cpuinfo_x86 *cpuinfo = &boot_cpu_data; switch (cpuinfo->x86) { @@ -602,6 +631,11 @@ static void __exit ccp_mod_exit(void) ccp_pci_exit(); break; } +#endif + +#ifdef CONFIG_ARM64 + ccp_platform_exit(); +#endif } module_init(ccp_mod_init); diff --git a/drivers/crypto/ccp/ccp-dev.h b/drivers/crypto/ccp/ccp-dev.h index 7ec536e702ec..62ff35a6b9ec 100644 --- a/drivers/crypto/ccp/ccp-dev.h +++ b/drivers/crypto/ccp/ccp-dev.h @@ -23,8 +23,6 @@ #include <linux/hw_random.h> -#define IO_OFFSET 0x20000 - #define MAX_DMAPOOL_NAME_LEN 32 #define MAX_HW_QUEUES 5 @@ -32,6 +30,9 @@ #define TRNG_RETRIES 10 +#define CACHE_NONE 0x00 +#define CACHE_WB_NO_ALLOC 0xb7 + /****** Register Mappings ******/ #define Q_MASK_REG 0x000 @@ -50,7 +51,7 @@ #define CMD_Q_INT_STATUS_BASE 0x214 #define CMD_Q_STATUS_INCR 0x20 -#define CMD_Q_CACHE 0x228 +#define CMD_Q_CACHE_BASE 0x228 #define CMD_Q_CACHE_INC 0x20 #define CMD_Q_ERROR(__qs) ((__qs) & 0x0000003f); @@ -194,6 +195,7 @@ struct ccp_device { void *dev_specific; int (*get_irq)(struct ccp_device *ccp); void (*free_irq)(struct ccp_device *ccp); + unsigned int irq; /* * I/O area used for device communication. The register mapping @@ -254,12 +256,18 @@ struct ccp_device { /* Suspend support */ unsigned int suspending; wait_queue_head_t suspend_queue; + + /* DMA caching attribute support */ + unsigned int axcache; }; int ccp_pci_init(void); void ccp_pci_exit(void); +int ccp_platform_init(void); +void ccp_platform_exit(void); + struct ccp_device *ccp_alloc_struct(struct device *dev); int ccp_init(struct ccp_device *ccp); void ccp_destroy(struct ccp_device *ccp); diff --git a/drivers/crypto/ccp/ccp-ops.c b/drivers/crypto/ccp/ccp-ops.c index 9ae006d69df4..8729364261d7 100644 --- a/drivers/crypto/ccp/ccp-ops.c +++ b/drivers/crypto/ccp/ccp-ops.c @@ -1606,7 +1606,7 @@ static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) goto e_ksb; ccp_reverse_set_dm_area(&exp, rsa->exp, rsa->exp_len, CCP_KSB_BYTES, - true); + false); ret = ccp_copy_to_ksb(cmd_q, &exp, op.jobid, op.ksb_key, CCP_PASSTHRU_BYTESWAP_NOOP); if (ret) { @@ -1623,10 +1623,10 @@ static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) goto e_exp; ccp_reverse_set_dm_area(&src, rsa->mod, rsa->mod_len, CCP_KSB_BYTES, - true); + false); src.address += o_len; /* Adjust the address for the copy operation */ ccp_reverse_set_dm_area(&src, rsa->src, rsa->src_len, CCP_KSB_BYTES, - true); + false); src.address -= o_len; /* Reset the address to original value */ /* Prepare the output area for the operation */ @@ -1841,20 +1841,20 @@ static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) /* Copy the ECC modulus */ ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len, - CCP_ECC_OPERAND_SIZE, true); + CCP_ECC_OPERAND_SIZE, false); src.address += CCP_ECC_OPERAND_SIZE; /* Copy the first operand */ ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_1, ecc->u.mm.operand_1_len, - CCP_ECC_OPERAND_SIZE, true); + CCP_ECC_OPERAND_SIZE, false); src.address += CCP_ECC_OPERAND_SIZE; if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) { /* Copy the second operand */ ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_2, ecc->u.mm.operand_2_len, - CCP_ECC_OPERAND_SIZE, true); + CCP_ECC_OPERAND_SIZE, false); src.address += CCP_ECC_OPERAND_SIZE; } @@ -1960,17 +1960,17 @@ static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) /* Copy the ECC modulus */ ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len, - CCP_ECC_OPERAND_SIZE, true); + CCP_ECC_OPERAND_SIZE, false); src.address += CCP_ECC_OPERAND_SIZE; /* Copy the first point X and Y coordinate */ ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.x, ecc->u.pm.point_1.x_len, - CCP_ECC_OPERAND_SIZE, true); + CCP_ECC_OPERAND_SIZE, false); src.address += CCP_ECC_OPERAND_SIZE; ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.y, ecc->u.pm.point_1.y_len, - CCP_ECC_OPERAND_SIZE, true); + CCP_ECC_OPERAND_SIZE, false); src.address += CCP_ECC_OPERAND_SIZE; /* Set the first point Z coordianate to 1 */ @@ -1981,11 +1981,11 @@ static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) /* Copy the second point X and Y coordinate */ ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.x, ecc->u.pm.point_2.x_len, - CCP_ECC_OPERAND_SIZE, true); + CCP_ECC_OPERAND_SIZE, false); src.address += CCP_ECC_OPERAND_SIZE; ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.y, ecc->u.pm.point_2.y_len, - CCP_ECC_OPERAND_SIZE, true); + CCP_ECC_OPERAND_SIZE, false); src.address += CCP_ECC_OPERAND_SIZE; /* Set the second point Z coordianate to 1 */ @@ -1995,14 +1995,14 @@ static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) /* Copy the Domain "a" parameter */ ccp_reverse_set_dm_area(&src, ecc->u.pm.domain_a, ecc->u.pm.domain_a_len, - CCP_ECC_OPERAND_SIZE, true); + CCP_ECC_OPERAND_SIZE, false); src.address += CCP_ECC_OPERAND_SIZE; if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) { /* Copy the scalar value */ ccp_reverse_set_dm_area(&src, ecc->u.pm.scalar, ecc->u.pm.scalar_len, - CCP_ECC_OPERAND_SIZE, true); + CCP_ECC_OPERAND_SIZE, false); src.address += CCP_ECC_OPERAND_SIZE; } } diff --git a/drivers/crypto/ccp/ccp-pci.c b/drivers/crypto/ccp/ccp-pci.c index 0d746236df5e..180cc87b4dbb 100644 --- a/drivers/crypto/ccp/ccp-pci.c +++ b/drivers/crypto/ccp/ccp-pci.c @@ -12,8 +12,10 @@ #include <linux/module.h> #include <linux/kernel.h> +#include <linux/device.h> #include <linux/pci.h> #include <linux/pci_ids.h> +#include <linux/dma-mapping.h> #include <linux/kthread.h> #include <linux/sched.h> #include <linux/interrupt.h> @@ -24,6 +26,8 @@ #include "ccp-dev.h" #define IO_BAR 2 +#define IO_OFFSET 0x20000 + #define MSIX_VECTORS 2 struct ccp_msix { @@ -89,7 +93,8 @@ static int ccp_get_msi_irq(struct ccp_device *ccp) if (ret) return ret; - ret = request_irq(pdev->irq, ccp_irq_handler, 0, "ccp", dev); + ccp->irq = pdev->irq; + ret = request_irq(ccp->irq, ccp_irq_handler, 0, "ccp", dev); if (ret) { dev_notice(dev, "unable to allocate MSI IRQ (%d)\n", ret); goto e_msi; @@ -136,7 +141,7 @@ static void ccp_free_irqs(struct ccp_device *ccp) dev); pci_disable_msix(pdev); } else { - free_irq(pdev->irq, dev); + free_irq(ccp->irq, dev); pci_disable_msi(pdev); } } @@ -147,21 +152,12 @@ static int ccp_find_mmio_area(struct ccp_device *ccp) struct pci_dev *pdev = container_of(dev, struct pci_dev, dev); resource_size_t io_len; unsigned long io_flags; - int bar; io_flags = pci_resource_flags(pdev, IO_BAR); io_len = pci_resource_len(pdev, IO_BAR); if ((io_flags & IORESOURCE_MEM) && (io_len >= (IO_OFFSET + 0x800))) return IO_BAR; - for (bar = 0; bar < PCI_STD_RESOURCE_END; bar++) { - io_flags = pci_resource_flags(pdev, bar); - io_len = pci_resource_len(pdev, bar); - if ((io_flags & IORESOURCE_MEM) && - (io_len >= (IO_OFFSET + 0x800))) - return bar; - } - return -EIO; } @@ -214,20 +210,13 @@ static int ccp_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) } ccp->io_regs = ccp->io_map + IO_OFFSET; - ret = dma_set_mask(dev, DMA_BIT_MASK(48)); - if (ret == 0) { - ret = dma_set_coherent_mask(dev, DMA_BIT_MASK(48)); + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48)); + if (ret) { + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); if (ret) { - dev_err(dev, - "pci_set_consistent_dma_mask failed (%d)\n", + dev_err(dev, "dma_set_mask_and_coherent failed (%d)\n", ret); - goto e_bar0; - } - } else { - ret = dma_set_mask(dev, DMA_BIT_MASK(32)); - if (ret) { - dev_err(dev, "pci_set_dma_mask failed (%d)\n", ret); - goto e_bar0; + goto e_iomap; } } @@ -235,13 +224,13 @@ static int ccp_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) ret = ccp_init(ccp); if (ret) - goto e_bar0; + goto e_iomap; dev_notice(dev, "enabled\n"); return 0; -e_bar0: +e_iomap: pci_iounmap(pdev, ccp->io_map); e_device: diff --git a/drivers/crypto/ccp/ccp-platform.c b/drivers/crypto/ccp/ccp-platform.c new file mode 100644 index 000000000000..b0a2806908f1 --- /dev/null +++ b/drivers/crypto/ccp/ccp-platform.c @@ -0,0 +1,230 @@ +/* + * AMD Cryptographic Coprocessor (CCP) driver + * + * Copyright (C) 2014 Advanced Micro Devices, Inc. + * + * Author: Tom Lendacky <thomas.lendacky@amd.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/device.h> +#include <linux/platform_device.h> +#include <linux/ioport.h> +#include <linux/dma-mapping.h> +#include <linux/kthread.h> +#include <linux/sched.h> +#include <linux/interrupt.h> +#include <linux/spinlock.h> +#include <linux/delay.h> +#include <linux/ccp.h> +#include <linux/of.h> + +#include "ccp-dev.h" + + +static int ccp_get_irq(struct ccp_device *ccp) +{ + struct device *dev = ccp->dev; + struct platform_device *pdev = container_of(dev, + struct platform_device, dev); + int ret; + + ret = platform_get_irq(pdev, 0); + if (ret < 0) + return ret; + + ccp->irq = ret; + ret = request_irq(ccp->irq, ccp_irq_handler, 0, "ccp", dev); + if (ret) { + dev_notice(dev, "unable to allocate IRQ (%d)\n", ret); + return ret; + } + + return 0; +} + +static int ccp_get_irqs(struct ccp_device *ccp) +{ + struct device *dev = ccp->dev; + int ret; + + ret = ccp_get_irq(ccp); + if (!ret) + return 0; + + /* Couldn't get an interrupt */ + dev_notice(dev, "could not enable interrupts (%d)\n", ret); + + return ret; +} + +static void ccp_free_irqs(struct ccp_device *ccp) +{ + struct device *dev = ccp->dev; + + free_irq(ccp->irq, dev); +} + +static struct resource *ccp_find_mmio_area(struct ccp_device *ccp) +{ + struct device *dev = ccp->dev; + struct platform_device *pdev = container_of(dev, + struct platform_device, dev); + struct resource *ior; + + ior = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (ior && (resource_size(ior) >= 0x800)) + return ior; + + return NULL; +} + +static int ccp_platform_probe(struct platform_device *pdev) +{ + struct ccp_device *ccp; + struct device *dev = &pdev->dev; + struct resource *ior; + int ret; + + ret = -ENOMEM; + ccp = ccp_alloc_struct(dev); + if (!ccp) + goto e_err; + + ccp->dev_specific = NULL; + ccp->get_irq = ccp_get_irqs; + ccp->free_irq = ccp_free_irqs; + + ior = ccp_find_mmio_area(ccp); + ccp->io_map = devm_ioremap_resource(dev, ior); + if (IS_ERR(ccp->io_map)) { + ret = PTR_ERR(ccp->io_map); + goto e_free; + } + ccp->io_regs = ccp->io_map; + + if (!dev->dma_mask) + dev->dma_mask = &dev->coherent_dma_mask; + *(dev->dma_mask) = DMA_BIT_MASK(48); + dev->coherent_dma_mask = DMA_BIT_MASK(48); + + if (of_property_read_bool(dev->of_node, "dma-coherent")) + ccp->axcache = CACHE_WB_NO_ALLOC; + else + ccp->axcache = CACHE_NONE; + + dev_set_drvdata(dev, ccp); + + ret = ccp_init(ccp); + if (ret) + goto e_free; + + dev_notice(dev, "enabled\n"); + + return 0; + +e_free: + kfree(ccp); + +e_err: + dev_notice(dev, "initialization failed\n"); + return ret; +} + +static int ccp_platform_remove(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct ccp_device *ccp = dev_get_drvdata(dev); + + ccp_destroy(ccp); + + kfree(ccp); + + dev_notice(dev, "disabled\n"); + + return 0; +} + +#ifdef CONFIG_PM +static int ccp_platform_suspend(struct platform_device *pdev, + pm_message_t state) +{ + struct device *dev = &pdev->dev; + struct ccp_device *ccp = dev_get_drvdata(dev); + unsigned long flags; + unsigned int i; + + spin_lock_irqsave(&ccp->cmd_lock, flags); + + ccp->suspending = 1; + + /* Wake all the queue kthreads to prepare for suspend */ + for (i = 0; i < ccp->cmd_q_count; i++) + wake_up_process(ccp->cmd_q[i].kthread); + + spin_unlock_irqrestore(&ccp->cmd_lock, flags); + + /* Wait for all queue kthreads to say they're done */ + while (!ccp_queues_suspended(ccp)) + wait_event_interruptible(ccp->suspend_queue, + ccp_queues_suspended(ccp)); + + return 0; +} + +static int ccp_platform_resume(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct ccp_device *ccp = dev_get_drvdata(dev); + unsigned long flags; + unsigned int i; + + spin_lock_irqsave(&ccp->cmd_lock, flags); + + ccp->suspending = 0; + + /* Wake up all the kthreads */ + for (i = 0; i < ccp->cmd_q_count; i++) { + ccp->cmd_q[i].suspended = 0; + wake_up_process(ccp->cmd_q[i].kthread); + } + + spin_unlock_irqrestore(&ccp->cmd_lock, flags); + + return 0; +} +#endif + +static const struct of_device_id ccp_platform_ids[] = { + { .compatible = "amd,ccp-seattle-v1a" }, + { }, +}; + +static struct platform_driver ccp_platform_driver = { + .driver = { + .name = "AMD Cryptographic Coprocessor", + .owner = THIS_MODULE, + .of_match_table = ccp_platform_ids, + }, + .probe = ccp_platform_probe, + .remove = ccp_platform_remove, +#ifdef CONFIG_PM + .suspend = ccp_platform_suspend, + .resume = ccp_platform_resume, +#endif +}; + +int ccp_platform_init(void) +{ + return platform_driver_register(&ccp_platform_driver); +} + +void ccp_platform_exit(void) +{ + platform_driver_unregister(&ccp_platform_driver); +} diff --git a/drivers/crypto/nx/nx-842.c b/drivers/crypto/nx/nx-842.c index 502edf0a2933..544f6d327ede 100644 --- a/drivers/crypto/nx/nx-842.c +++ b/drivers/crypto/nx/nx-842.c @@ -1247,7 +1247,7 @@ static struct vio_device_id nx842_driver_ids[] = { static struct vio_driver nx842_driver = { .name = MODULE_NAME, .probe = nx842_probe, - .remove = nx842_remove, + .remove = __exit_p(nx842_remove), .get_desired_dma = nx842_get_desired_dma, .id_table = nx842_driver_ids, }; diff --git a/drivers/crypto/qat/Kconfig b/drivers/crypto/qat/Kconfig new file mode 100644 index 000000000000..49bede2a9f77 --- /dev/null +++ b/drivers/crypto/qat/Kconfig @@ -0,0 +1,23 @@ +config CRYPTO_DEV_QAT + tristate + select CRYPTO_AEAD + select CRYPTO_AUTHENC + select CRYPTO_ALGAPI + select CRYPTO_AES + select CRYPTO_CBC + select CRYPTO_SHA1 + select CRYPTO_SHA256 + select CRYPTO_SHA512 + select FW_LOADER + +config CRYPTO_DEV_QAT_DH895xCC + tristate "Support for Intel(R) DH895xCC" + depends on X86 && PCI + default n + select CRYPTO_DEV_QAT + help + Support for Intel(R) DH895xcc with Intel(R) QuickAssist Technology + for accelerating crypto and compression workloads. + + To compile this as a module, choose M here: the module + will be called qat_dh895xcc. diff --git a/drivers/crypto/qat/Makefile b/drivers/crypto/qat/Makefile new file mode 100644 index 000000000000..d11481be225e --- /dev/null +++ b/drivers/crypto/qat/Makefile @@ -0,0 +1,2 @@ +obj-$(CONFIG_CRYPTO_DEV_QAT) += qat_common/ +obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCC) += qat_dh895xcc/ diff --git a/drivers/crypto/qat/qat_common/Makefile b/drivers/crypto/qat/qat_common/Makefile new file mode 100644 index 000000000000..e0424dc382fe --- /dev/null +++ b/drivers/crypto/qat/qat_common/Makefile @@ -0,0 +1,14 @@ +obj-$(CONFIG_CRYPTO_DEV_QAT) += intel_qat.o +intel_qat-objs := adf_cfg.o \ + adf_ctl_drv.o \ + adf_dev_mgr.o \ + adf_init.o \ + adf_accel_engine.o \ + adf_aer.o \ + adf_transport.o \ + qat_crypto.o \ + qat_algs.o \ + qat_uclo.o \ + qat_hal.o + +intel_qat-$(CONFIG_DEBUG_FS) += adf_transport_debug.o diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h new file mode 100644 index 000000000000..9282381b03ce --- /dev/null +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -0,0 +1,205 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef ADF_ACCEL_DEVICES_H_ +#define ADF_ACCEL_DEVICES_H_ +#include <linux/module.h> +#include <linux/atomic.h> +#include <linux/list.h> +#include <linux/proc_fs.h> +#include <linux/io.h> +#include "adf_cfg_common.h" + +#define PCI_VENDOR_ID_INTEL 0x8086 +#define ADF_DH895XCC_DEVICE_NAME "dh895xcc" +#define ADF_DH895XCC_PCI_DEVICE_ID 0x435 +#define ADF_DH895XCC_PMISC_BAR 1 +#define ADF_DH895XCC_ETR_BAR 2 +#define ADF_PCI_MAX_BARS 3 +#define ADF_DEVICE_NAME_LENGTH 32 +#define ADF_ETR_MAX_RINGS_PER_BANK 16 +#define ADF_MAX_MSIX_VECTOR_NAME 16 +#define ADF_DEVICE_NAME_PREFIX "qat_" + +enum adf_accel_capabilities { + ADF_ACCEL_CAPABILITIES_NULL = 0, + ADF_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC = 1, + ADF_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC = 2, + ADF_ACCEL_CAPABILITIES_CIPHER = 4, + ADF_ACCEL_CAPABILITIES_AUTHENTICATION = 8, + ADF_ACCEL_CAPABILITIES_COMPRESSION = 32, + ADF_ACCEL_CAPABILITIES_LZS_COMPRESSION = 64, + ADF_ACCEL_CAPABILITIES_RANDOM_NUMBER = 128 +}; + +struct adf_bar { + resource_size_t base_addr; + void __iomem *virt_addr; + resource_size_t size; +} __packed; + +struct adf_accel_msix { + struct msix_entry *entries; + char **names; +} __packed; + +struct adf_accel_pci { + struct pci_dev *pci_dev; + struct adf_accel_msix msix_entries; + struct adf_bar pci_bars[ADF_PCI_MAX_BARS]; + uint8_t revid; + uint8_t sku; +} __packed; + +enum dev_state { + DEV_DOWN = 0, + DEV_UP +}; + +enum dev_sku_info { + DEV_SKU_1 = 0, + DEV_SKU_2, + DEV_SKU_3, + DEV_SKU_4, + DEV_SKU_UNKNOWN, +}; + +static inline const char *get_sku_info(enum dev_sku_info info) +{ + switch (info) { + case DEV_SKU_1: + return "SKU1"; + case DEV_SKU_2: + return "SKU2"; + case DEV_SKU_3: + return "SKU3"; + case DEV_SKU_4: + return "SKU4"; + case DEV_SKU_UNKNOWN: + default: + break; + } + return "Unknown SKU"; +} + +struct adf_hw_device_class { + const char *name; + const enum adf_device_type type; + uint32_t instances; +} __packed; + +struct adf_cfg_device_data; +struct adf_accel_dev; +struct adf_etr_data; +struct adf_etr_ring_data; + +struct adf_hw_device_data { + struct adf_hw_device_class *dev_class; + uint32_t (*get_accel_mask)(uint32_t fuse); + uint32_t (*get_ae_mask)(uint32_t fuse); + uint32_t (*get_misc_bar_id)(struct adf_hw_device_data *self); + uint32_t (*get_etr_bar_id)(struct adf_hw_device_data *self); + uint32_t (*get_num_aes)(struct adf_hw_device_data *self); + uint32_t (*get_num_accels)(struct adf_hw_device_data *self); + enum dev_sku_info (*get_sku)(struct adf_hw_device_data *self); + void (*hw_arb_ring_enable)(struct adf_etr_ring_data *ring); + void (*hw_arb_ring_disable)(struct adf_etr_ring_data *ring); + int (*alloc_irq)(struct adf_accel_dev *accel_dev); + void (*free_irq)(struct adf_accel_dev *accel_dev); + void (*enable_error_correction)(struct adf_accel_dev *accel_dev); + const char *fw_name; + uint32_t pci_dev_id; + uint32_t fuses; + uint32_t accel_capabilities_mask; + uint16_t accel_mask; + uint16_t ae_mask; + uint16_t tx_rings_mask; + uint8_t tx_rx_gap; + uint8_t instance_id; + uint8_t num_banks; + uint8_t num_accel; + uint8_t num_logical_accel; + uint8_t num_engines; +} __packed; + +/* CSR write macro */ +#define ADF_CSR_WR(csr_base, csr_offset, val) \ + __raw_writel(val, csr_base + csr_offset) + +/* CSR read macro */ +#define ADF_CSR_RD(csr_base, csr_offset) __raw_readl(csr_base + csr_offset) + +#define GET_DEV(accel_dev) ((accel_dev)->accel_pci_dev.pci_dev->dev) +#define GET_BARS(accel_dev) ((accel_dev)->accel_pci_dev.pci_bars) +#define GET_HW_DATA(accel_dev) (accel_dev->hw_device) +#define GET_MAX_BANKS(accel_dev) (GET_HW_DATA(accel_dev)->num_banks) +#define GET_MAX_ACCELENGINES(accel_dev) (GET_HW_DATA(accel_dev)->num_engines) +#define accel_to_pci_dev(accel_ptr) accel_ptr->accel_pci_dev.pci_dev + +struct adf_admin_comms; +struct icp_qat_fw_loader_handle; +struct adf_fw_loader_data { + struct icp_qat_fw_loader_handle *fw_loader; + const struct firmware *uof_fw; +}; + +struct adf_accel_dev { + struct adf_etr_data *transport; + struct adf_hw_device_data *hw_device; + struct adf_cfg_device_data *cfg; + struct adf_fw_loader_data *fw_loader; + struct adf_admin_comms *admin; + struct list_head crypto_list; + unsigned long status; + atomic_t ref_count; + struct dentry *debugfs_dir; + struct list_head list; + struct module *owner; + uint8_t accel_id; + uint8_t numa_node; + struct adf_accel_pci accel_pci_dev; +} __packed; +#endif diff --git a/drivers/crypto/qat/qat_common/adf_accel_engine.c b/drivers/crypto/qat/qat_common/adf_accel_engine.c new file mode 100644 index 000000000000..c77453b900a3 --- /dev/null +++ b/drivers/crypto/qat/qat_common/adf_accel_engine.c @@ -0,0 +1,168 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#include <linux/firmware.h> +#include <linux/pci.h> +#include "adf_cfg.h" +#include "adf_accel_devices.h" +#include "adf_common_drv.h" +#include "icp_qat_uclo.h" + +int adf_ae_fw_load(struct adf_accel_dev *accel_dev) +{ + struct adf_fw_loader_data *loader_data = accel_dev->fw_loader; + struct adf_hw_device_data *hw_device = accel_dev->hw_device; + void *uof_addr; + uint32_t uof_size; + + if (request_firmware(&loader_data->uof_fw, hw_device->fw_name, + &accel_dev->accel_pci_dev.pci_dev->dev)) { + pr_err("QAT: Failed to load firmware %s\n", hw_device->fw_name); + return -EFAULT; + } + + uof_size = loader_data->uof_fw->size; + uof_addr = (void *)loader_data->uof_fw->data; + if (qat_uclo_map_uof_obj(loader_data->fw_loader, uof_addr, uof_size)) { + pr_err("QAT: Failed to map UOF\n"); + goto out_err; + } + if (qat_uclo_wr_all_uimage(loader_data->fw_loader)) { + pr_err("QAT: Failed to map UOF\n"); + goto out_err; + } + return 0; + +out_err: + release_firmware(loader_data->uof_fw); + return -EFAULT; +} + +int adf_ae_fw_release(struct adf_accel_dev *accel_dev) +{ + struct adf_fw_loader_data *loader_data = accel_dev->fw_loader; + + release_firmware(loader_data->uof_fw); + qat_uclo_del_uof_obj(loader_data->fw_loader); + qat_hal_deinit(loader_data->fw_loader); + loader_data->fw_loader = NULL; + return 0; +} + +int adf_ae_start(struct adf_accel_dev *accel_dev) +{ + struct adf_fw_loader_data *loader_data = accel_dev->fw_loader; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + uint32_t ae_ctr, ae, max_aes = GET_MAX_ACCELENGINES(accel_dev); + + for (ae = 0, ae_ctr = 0; ae < max_aes; ae++) { + if (hw_data->ae_mask & (1 << ae)) { + qat_hal_start(loader_data->fw_loader, ae, 0xFF); + ae_ctr++; + } + } + pr_info("QAT: qat_dev%d started %d acceleration engines\n", + accel_dev->accel_id, ae_ctr); + return 0; +} + +int adf_ae_stop(struct adf_accel_dev *accel_dev) +{ + struct adf_fw_loader_data *loader_data = accel_dev->fw_loader; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + uint32_t ae_ctr, ae, max_aes = GET_MAX_ACCELENGINES(accel_dev); + + for (ae = 0, ae_ctr = 0; ae < max_aes; ae++) { + if (hw_data->ae_mask & (1 << ae)) { + qat_hal_stop(loader_data->fw_loader, ae, 0xFF); + ae_ctr++; + } + } + pr_info("QAT: qat_dev%d stopped %d acceleration engines\n", + accel_dev->accel_id, ae_ctr); + return 0; +} + +static int adf_ae_reset(struct adf_accel_dev *accel_dev, int ae) +{ + struct adf_fw_loader_data *loader_data = accel_dev->fw_loader; + + qat_hal_reset(loader_data->fw_loader); + if (qat_hal_clr_reset(loader_data->fw_loader)) + return -EFAULT; + + return 0; +} + +int adf_ae_init(struct adf_accel_dev *accel_dev) +{ + struct adf_fw_loader_data *loader_data; + + loader_data = kzalloc(sizeof(*loader_data), GFP_KERNEL); + if (!loader_data) + return -ENOMEM; + + accel_dev->fw_loader = loader_data; + if (qat_hal_init(accel_dev)) { + pr_err("QAT: Failed to init the AEs\n"); + kfree(loader_data); + return -EFAULT; + } + if (adf_ae_reset(accel_dev, 0)) { + pr_err("QAT: Failed to reset the AEs\n"); + qat_hal_deinit(loader_data->fw_loader); + kfree(loader_data); + return -EFAULT; + } + return 0; +} + +int adf_ae_shutdown(struct adf_accel_dev *accel_dev) +{ + kfree(accel_dev->fw_loader); + accel_dev->fw_loader = NULL; + return 0; +} diff --git a/drivers/crypto/qat/qat_common/adf_aer.c b/drivers/crypto/qat/qat_common/adf_aer.c new file mode 100644 index 000000000000..c29d4c3926bf --- /dev/null +++ b/drivers/crypto/qat/qat_common/adf_aer.c @@ -0,0 +1,259 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#include <linux/kernel.h> +#include <linux/pci.h> +#include <linux/aer.h> +#include <linux/completion.h> +#include <linux/workqueue.h> +#include <linux/delay.h> +#include "adf_accel_devices.h" +#include "adf_common_drv.h" + +static struct workqueue_struct *device_reset_wq; + +static pci_ers_result_t adf_error_detected(struct pci_dev *pdev, + pci_channel_state_t state) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + pr_info("QAT: Acceleration driver hardware error detected.\n"); + if (!accel_dev) { + pr_err("QAT: Can't find acceleration device\n"); + return PCI_ERS_RESULT_DISCONNECT; + } + + if (state == pci_channel_io_perm_failure) { + pr_err("QAT: Can't recover from device error\n"); + return PCI_ERS_RESULT_DISCONNECT; + } + + return PCI_ERS_RESULT_NEED_RESET; +} + +/* reset dev data */ +struct adf_reset_dev_data { + int mode; + struct adf_accel_dev *accel_dev; + struct completion compl; + struct work_struct reset_work; +}; + +#define PPDSTAT_OFFSET 0x7E +static void adf_dev_restore(struct adf_accel_dev *accel_dev) +{ + struct pci_dev *pdev = accel_to_pci_dev(accel_dev); + struct pci_dev *parent = pdev->bus->self; + uint16_t ppdstat = 0, bridge_ctl = 0; + int pending = 0; + + pr_info("QAT: Reseting device qat_dev%d\n", accel_dev->accel_id); + pci_read_config_word(pdev, PPDSTAT_OFFSET, &ppdstat); + pending = ppdstat & PCI_EXP_DEVSTA_TRPND; + if (pending) { + int ctr = 0; + + do { + msleep(100); + pci_read_config_word(pdev, PPDSTAT_OFFSET, &ppdstat); + pending = ppdstat & PCI_EXP_DEVSTA_TRPND; + } while (pending && ctr++ < 10); + } + + if (pending) + pr_info("QAT: Transaction still in progress. Proceeding\n"); + + pci_read_config_word(parent, PCI_BRIDGE_CONTROL, &bridge_ctl); + bridge_ctl |= PCI_BRIDGE_CTL_BUS_RESET; + pci_write_config_word(parent, PCI_BRIDGE_CONTROL, bridge_ctl); + msleep(100); + bridge_ctl &= ~PCI_BRIDGE_CTL_BUS_RESET; + pci_write_config_word(parent, PCI_BRIDGE_CONTROL, bridge_ctl); + msleep(100); + pci_restore_state(pdev); + pci_save_state(pdev); +} + +static void adf_device_reset_worker(struct work_struct *work) +{ + struct adf_reset_dev_data *reset_data = + container_of(work, struct adf_reset_dev_data, reset_work); + struct adf_accel_dev *accel_dev = reset_data->accel_dev; + + adf_dev_restarting_notify(accel_dev); + adf_dev_stop(accel_dev); + adf_dev_restore(accel_dev); + if (adf_dev_start(accel_dev)) { + /* The device hanged and we can't restart it so stop here */ + dev_err(&GET_DEV(accel_dev), "Restart device failed\n"); + kfree(reset_data); + WARN(1, "QAT: device restart failed. Device is unusable\n"); + return; + } + adf_dev_restarted_notify(accel_dev); + clear_bit(ADF_STATUS_RESTARTING, &accel_dev->status); + + /* The dev is back alive. Notify the caller if in sync mode */ + if (reset_data->mode == ADF_DEV_RESET_SYNC) + complete(&reset_data->compl); + else + kfree(reset_data); +} + +static int adf_dev_aer_schedule_reset(struct adf_accel_dev *accel_dev, + enum adf_dev_reset_mode mode) +{ + struct adf_reset_dev_data *reset_data; + + if (adf_dev_started(accel_dev) && + !test_bit(ADF_STATUS_RESTARTING, &accel_dev->status)) + return 0; + + set_bit(ADF_STATUS_RESTARTING, &accel_dev->status); + reset_data = kzalloc(sizeof(*reset_data), GFP_ATOMIC); + if (!reset_data) + return -ENOMEM; + reset_data->accel_dev = accel_dev; + init_completion(&reset_data->compl); + reset_data->mode = mode; + INIT_WORK(&reset_data->reset_work, adf_device_reset_worker); + queue_work(device_reset_wq, &reset_data->reset_work); + + /* If in sync mode wait for the result */ + if (mode == ADF_DEV_RESET_SYNC) { + int ret = 0; + /* Maximum device reset time is 10 seconds */ + unsigned long wait_jiffies = msecs_to_jiffies(10000); + unsigned long timeout = wait_for_completion_timeout( + &reset_data->compl, wait_jiffies); + if (!timeout) { + pr_err("QAT: Reset device timeout expired\n"); + ret = -EFAULT; + } + kfree(reset_data); + return ret; + } + return 0; +} + +static pci_ers_result_t adf_slot_reset(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + if (!accel_dev) { + pr_err("QAT: Can't find acceleration device\n"); + return PCI_ERS_RESULT_DISCONNECT; + } + pci_cleanup_aer_uncorrect_error_status(pdev); + if (adf_dev_aer_schedule_reset(accel_dev, ADF_DEV_RESET_SYNC)) + return PCI_ERS_RESULT_DISCONNECT; + + return PCI_ERS_RESULT_RECOVERED; +} + +static void adf_resume(struct pci_dev *pdev) +{ + pr_info("QAT: Acceleration driver reset completed\n"); + pr_info("QAT: Device is up and runnig\n"); +} + +static struct pci_error_handlers adf_err_handler = { + .error_detected = adf_error_detected, + .slot_reset = adf_slot_reset, + .resume = adf_resume, +}; + +/** + * adf_enable_aer() - Enable Advance Error Reporting for acceleration device + * @accel_dev: Pointer to acceleration device. + * @adf: PCI device driver owning the given acceleration device. + * + * Function enables PCI Advance Error Reporting for the + * QAT acceleration device accel_dev. + * To be used by QAT device specific drivers. + * + * Return: 0 on success, error code othewise. + */ +int adf_enable_aer(struct adf_accel_dev *accel_dev, struct pci_driver *adf) +{ + struct pci_dev *pdev = accel_to_pci_dev(accel_dev); + + adf->err_handler = &adf_err_handler; + pci_enable_pcie_error_reporting(pdev); + return 0; +} +EXPORT_SYMBOL_GPL(adf_enable_aer); + +/** + * adf_disable_aer() - Enable Advance Error Reporting for acceleration device + * @accel_dev: Pointer to acceleration device. + * + * Function disables PCI Advance Error Reporting for the + * QAT acceleration device accel_dev. + * To be used by QAT device specific drivers. + * + * Return: void + */ +void adf_disable_aer(struct adf_accel_dev *accel_dev) +{ + struct pci_dev *pdev = accel_to_pci_dev(accel_dev); + + pci_disable_pcie_error_reporting(pdev); +} +EXPORT_SYMBOL_GPL(adf_disable_aer); + +int adf_init_aer(void) +{ + device_reset_wq = create_workqueue("qat_device_reset_wq"); + return (device_reset_wq == NULL) ? -EFAULT : 0; +} + +void adf_exit_aer(void) +{ + if (device_reset_wq) + destroy_workqueue(device_reset_wq); + device_reset_wq = NULL; +} diff --git a/drivers/crypto/qat/qat_common/adf_cfg.c b/drivers/crypto/qat/qat_common/adf_cfg.c new file mode 100644 index 000000000000..aba7f1d043fb --- /dev/null +++ b/drivers/crypto/qat/qat_common/adf_cfg.c @@ -0,0 +1,361 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#include <linux/mutex.h> +#include <linux/slab.h> +#include <linux/list.h> +#include <linux/seq_file.h> +#include "adf_accel_devices.h" +#include "adf_cfg.h" + +static DEFINE_MUTEX(qat_cfg_read_lock); + +static void *qat_dev_cfg_start(struct seq_file *sfile, loff_t *pos) +{ + struct adf_cfg_device_data *dev_cfg = sfile->private; + + mutex_lock(&qat_cfg_read_lock); + return seq_list_start(&dev_cfg->sec_list, *pos); +} + +static int qat_dev_cfg_show(struct seq_file *sfile, void *v) +{ + struct list_head *list; + struct adf_cfg_section *sec = + list_entry(v, struct adf_cfg_section, list); + + seq_printf(sfile, "[%s]\n", sec->name); + list_for_each(list, &sec->param_head) { + struct adf_cfg_key_val *ptr = + list_entry(list, struct adf_cfg_key_val, list); + seq_printf(sfile, "%s = %s\n", ptr->key, ptr->val); + } + return 0; +} + +static void *qat_dev_cfg_next(struct seq_file *sfile, void *v, loff_t *pos) +{ + struct adf_cfg_device_data *dev_cfg = sfile->private; + + return seq_list_next(v, &dev_cfg->sec_list, pos); +} + +static void qat_dev_cfg_stop(struct seq_file *sfile, void *v) +{ + mutex_unlock(&qat_cfg_read_lock); +} + +static const struct seq_operations qat_dev_cfg_sops = { + .start = qat_dev_cfg_start, + .next = qat_dev_cfg_next, + .stop = qat_dev_cfg_stop, + .show = qat_dev_cfg_show +}; + +static int qat_dev_cfg_open(struct inode *inode, struct file *file) +{ + int ret = seq_open(file, &qat_dev_cfg_sops); + + if (!ret) { + struct seq_file *seq_f = file->private_data; + + seq_f->private = inode->i_private; + } + return ret; +} + +static const struct file_operations qat_dev_cfg_fops = { + .open = qat_dev_cfg_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release +}; + +/** + * adf_cfg_dev_add() - Create an acceleration device configuration table. + * @accel_dev: Pointer to acceleration device. + * + * Function creates a configuration table for the given acceleration device. + * The table stores device specific config values. + * To be used by QAT device specific drivers. + * + * Return: 0 on success, error code othewise. + */ +int adf_cfg_dev_add(struct adf_accel_dev *accel_dev) +{ + struct adf_cfg_device_data *dev_cfg_data; + + dev_cfg_data = kzalloc(sizeof(*dev_cfg_data), GFP_KERNEL); + if (!dev_cfg_data) + return -ENOMEM; + INIT_LIST_HEAD(&dev_cfg_data->sec_list); + init_rwsem(&dev_cfg_data->lock); + accel_dev->cfg = dev_cfg_data; + + /* accel_dev->debugfs_dir should always be non-NULL here */ + dev_cfg_data->debug = debugfs_create_file("dev_cfg", S_IRUSR, + accel_dev->debugfs_dir, + dev_cfg_data, + &qat_dev_cfg_fops); + if (!dev_cfg_data->debug) { + pr_err("QAT: Failed to create qat cfg debugfs entry.\n"); + kfree(dev_cfg_data); + accel_dev->cfg = NULL; + return -EFAULT; + } + return 0; +} +EXPORT_SYMBOL_GPL(adf_cfg_dev_add); + +static void adf_cfg_section_del_all(struct list_head *head); + +void adf_cfg_del_all(struct adf_accel_dev *accel_dev) +{ + struct adf_cfg_device_data *dev_cfg_data = accel_dev->cfg; + + down_write(&dev_cfg_data->lock); + adf_cfg_section_del_all(&dev_cfg_data->sec_list); + up_write(&dev_cfg_data->lock); +} + +/** + * adf_cfg_dev_remove() - Clears acceleration device configuration table. + * @accel_dev: Pointer to acceleration device. + * + * Function removes configuration table from the given acceleration device + * and frees all allocated memory. + * To be used by QAT device specific drivers. + * + * Return: void + */ +void adf_cfg_dev_remove(struct adf_accel_dev *accel_dev) +{ + struct adf_cfg_device_data *dev_cfg_data = accel_dev->cfg; + + down_write(&dev_cfg_data->lock); + adf_cfg_section_del_all(&dev_cfg_data->sec_list); + up_write(&dev_cfg_data->lock); + debugfs_remove(dev_cfg_data->debug); + kfree(dev_cfg_data); + accel_dev->cfg = NULL; +} +EXPORT_SYMBOL_GPL(adf_cfg_dev_remove); + +static void adf_cfg_keyval_add(struct adf_cfg_key_val *new, + struct adf_cfg_section *sec) +{ + list_add_tail(&new->list, &sec->param_head); +} + +static void adf_cfg_keyval_del_all(struct list_head *head) +{ + struct list_head *list_ptr, *tmp; + + list_for_each_prev_safe(list_ptr, tmp, head) { + struct adf_cfg_key_val *ptr = + list_entry(list_ptr, struct adf_cfg_key_val, list); + list_del(list_ptr); + kfree(ptr); + } +} + +static void adf_cfg_section_del_all(struct list_head *head) +{ + struct adf_cfg_section *ptr; + struct list_head *list, *tmp; + + list_for_each_prev_safe(list, tmp, head) { + ptr = list_entry(list, struct adf_cfg_section, list); + adf_cfg_keyval_del_all(&ptr->param_head); + list_del(list); + kfree(ptr); + } +} + +static struct adf_cfg_key_val *adf_cfg_key_value_find(struct adf_cfg_section *s, + const char *key) +{ + struct list_head *list; + + list_for_each(list, &s->param_head) { + struct adf_cfg_key_val *ptr = + list_entry(list, struct adf_cfg_key_val, list); + if (!strcmp(ptr->key, key)) + return ptr; + } + return NULL; +} + +static struct adf_cfg_section *adf_cfg_sec_find(struct adf_accel_dev *accel_dev, + const char *sec_name) +{ + struct adf_cfg_device_data *cfg = accel_dev->cfg; + struct list_head *list; + + list_for_each(list, &cfg->sec_list) { + struct adf_cfg_section *ptr = + list_entry(list, struct adf_cfg_section, list); + if (!strcmp(ptr->name, sec_name)) + return ptr; + } + return NULL; +} + +static int adf_cfg_key_val_get(struct adf_accel_dev *accel_dev, + const char *sec_name, + const char *key_name, + char *val) +{ + struct adf_cfg_section *sec = adf_cfg_sec_find(accel_dev, sec_name); + struct adf_cfg_key_val *keyval = NULL; + + if (sec) + keyval = adf_cfg_key_value_find(sec, key_name); + if (keyval) { + memcpy(val, keyval->val, ADF_CFG_MAX_VAL_LEN_IN_BYTES); + return 0; + } + return -1; +} + +/** + * adf_cfg_add_key_value_param() - Add key-value config entry to config table. + * @accel_dev: Pointer to acceleration device. + * @section_name: Name of the section where the param will be added + * @key: The key string + * @val: Value pain for the given @key + * @type: Type - string, int or address + * + * Function adds configuration key - value entry in the appropriate section + * in the given acceleration device + * To be used by QAT device specific drivers. + * + * Return: 0 on success, error code othewise. + */ +int adf_cfg_add_key_value_param(struct adf_accel_dev *accel_dev, + const char *section_name, + const char *key, const void *val, + enum adf_cfg_val_type type) +{ + struct adf_cfg_device_data *cfg = accel_dev->cfg; + struct adf_cfg_key_val *key_val; + struct adf_cfg_section *section = adf_cfg_sec_find(accel_dev, + section_name); + if (!section) + return -EFAULT; + + key_val = kzalloc(sizeof(*key_val), GFP_KERNEL); + if (!key_val) + return -ENOMEM; + + INIT_LIST_HEAD(&key_val->list); + strlcpy(key_val->key, key, sizeof(key_val->key)); + + if (type == ADF_DEC) { + snprintf(key_val->val, ADF_CFG_MAX_VAL_LEN_IN_BYTES, + "%ld", (*((long *)val))); + } else if (type == ADF_STR) { + strlcpy(key_val->val, (char *)val, sizeof(key_val->val)); + } else if (type == ADF_HEX) { + snprintf(key_val->val, ADF_CFG_MAX_VAL_LEN_IN_BYTES, + "0x%lx", (unsigned long)val); + } else { + pr_err("QAT: Unknown type given.\n"); + kfree(key_val); + return -1; + } + key_val->type = type; + down_write(&cfg->lock); + adf_cfg_keyval_add(key_val, section); + up_write(&cfg->lock); + return 0; +} +EXPORT_SYMBOL_GPL(adf_cfg_add_key_value_param); + +/** + * adf_cfg_section_add() - Add config section entry to config table. + * @accel_dev: Pointer to acceleration device. + * @name: Name of the section + * + * Function adds configuration section where key - value entries + * will be stored. + * To be used by QAT device specific drivers. + * + * Return: 0 on success, error code othewise. + */ +int adf_cfg_section_add(struct adf_accel_dev *accel_dev, const char *name) +{ + struct adf_cfg_device_data *cfg = accel_dev->cfg; + struct adf_cfg_section *sec = adf_cfg_sec_find(accel_dev, name); + + if (sec) + return 0; + + sec = kzalloc(sizeof(*sec), GFP_KERNEL); + if (!sec) + return -ENOMEM; + + strlcpy(sec->name, name, sizeof(sec->name)); + INIT_LIST_HEAD(&sec->param_head); + down_write(&cfg->lock); + list_add_tail(&sec->list, &cfg->sec_list); + up_write(&cfg->lock); + return 0; +} +EXPORT_SYMBOL_GPL(adf_cfg_section_add); + +int adf_cfg_get_param_value(struct adf_accel_dev *accel_dev, + const char *section, const char *name, + char *value) +{ + struct adf_cfg_device_data *cfg = accel_dev->cfg; + int ret; + + down_read(&cfg->lock); + ret = adf_cfg_key_val_get(accel_dev, section, name, value); + up_read(&cfg->lock); + return ret; +} diff --git a/drivers/crypto/qat/qat_common/adf_cfg.h b/drivers/crypto/qat/qat_common/adf_cfg.h new file mode 100644 index 000000000000..6a9c6f6b5ec9 --- /dev/null +++ b/drivers/crypto/qat/qat_common/adf_cfg.h @@ -0,0 +1,87 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef ADF_CFG_H_ +#define ADF_CFG_H_ + +#include <linux/list.h> +#include <linux/rwsem.h> +#include <linux/debugfs.h> +#include "adf_accel_devices.h" +#include "adf_cfg_common.h" +#include "adf_cfg_strings.h" + +struct adf_cfg_key_val { + char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; + char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; + enum adf_cfg_val_type type; + struct list_head list; +}; + +struct adf_cfg_section { + char name[ADF_CFG_MAX_SECTION_LEN_IN_BYTES]; + struct list_head list; + struct list_head param_head; +}; + +struct adf_cfg_device_data { + struct list_head sec_list; + struct dentry *debug; + struct rw_semaphore lock; +}; + +int adf_cfg_dev_add(struct adf_accel_dev *accel_dev); +void adf_cfg_dev_remove(struct adf_accel_dev *accel_dev); +int adf_cfg_section_add(struct adf_accel_dev *accel_dev, const char *name); +void adf_cfg_del_all(struct adf_accel_dev *accel_dev); +int adf_cfg_add_key_value_param(struct adf_accel_dev *accel_dev, + const char *section_name, + const char *key, const void *val, + enum adf_cfg_val_type type); +int adf_cfg_get_param_value(struct adf_accel_dev *accel_dev, + const char *section, const char *name, char *value); + +#endif diff --git a/drivers/crypto/qat/qat_common/adf_cfg_common.h b/drivers/crypto/qat/qat_common/adf_cfg_common.h new file mode 100644 index 000000000000..88b82187ac35 --- /dev/null +++ b/drivers/crypto/qat/qat_common/adf_cfg_common.h @@ -0,0 +1,100 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef ADF_CFG_COMMON_H_ +#define ADF_CFG_COMMON_H_ + +#include <linux/types.h> +#include <linux/ioctl.h> + +#define ADF_CFG_MAX_STR_LEN 64 +#define ADF_CFG_MAX_KEY_LEN_IN_BYTES ADF_CFG_MAX_STR_LEN +#define ADF_CFG_MAX_VAL_LEN_IN_BYTES ADF_CFG_MAX_STR_LEN +#define ADF_CFG_MAX_SECTION_LEN_IN_BYTES ADF_CFG_MAX_STR_LEN +#define ADF_CFG_BASE_DEC 10 +#define ADF_CFG_BASE_HEX 16 +#define ADF_CFG_ALL_DEVICES 0xFE +#define ADF_CFG_NO_DEVICE 0xFF +#define ADF_CFG_AFFINITY_WHATEVER 0xFF +#define MAX_DEVICE_NAME_SIZE 32 +#define ADF_MAX_DEVICES 32 + +enum adf_cfg_val_type { + ADF_DEC, + ADF_HEX, + ADF_STR +}; + +enum adf_device_type { + DEV_UNKNOWN = 0, + DEV_DH895XCC, +}; + +struct adf_dev_status_info { + enum adf_device_type type; + uint8_t accel_id; + uint8_t instance_id; + uint8_t num_ae; + uint8_t num_accel; + uint8_t num_logical_accel; + uint8_t banks_per_accel; + uint8_t state; + uint8_t bus; + uint8_t dev; + uint8_t fun; + char name[MAX_DEVICE_NAME_SIZE]; +}; + +#define ADF_CTL_IOC_MAGIC 'a' +#define IOCTL_CONFIG_SYS_RESOURCE_PARAMETERS _IOW(ADF_CTL_IOC_MAGIC, 0, \ + struct adf_user_cfg_ctl_data) +#define IOCTL_STOP_ACCEL_DEV _IOW(ADF_CTL_IOC_MAGIC, 1, \ + struct adf_user_cfg_ctl_data) +#define IOCTL_START_ACCEL_DEV _IOW(ADF_CTL_IOC_MAGIC, 2, \ + struct adf_user_cfg_ctl_data) +#define IOCTL_STATUS_ACCEL_DEV _IOW(ADF_CTL_IOC_MAGIC, 3, uint32_t) +#define IOCTL_GET_NUM_DEVICES _IOW(ADF_CTL_IOC_MAGIC, 4, int32_t) +#endif diff --git a/drivers/crypto/qat/qat_common/adf_cfg_strings.h b/drivers/crypto/qat/qat_common/adf_cfg_strings.h new file mode 100644 index 000000000000..c7ac758ebc90 --- /dev/null +++ b/drivers/crypto/qat/qat_common/adf_cfg_strings.h @@ -0,0 +1,83 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef ADF_CFG_STRINGS_H_ +#define ADF_CFG_STRINGS_H_ + +#define ADF_GENERAL_SEC "GENERAL" +#define ADF_KERNEL_SEC "KERNEL" +#define ADF_ACCEL_SEC "Accelerator" +#define ADF_NUM_CY "NumberCyInstances" +#define ADF_NUM_DC "NumberDcInstances" +#define ADF_RING_SYM_SIZE "NumConcurrentSymRequests" +#define ADF_RING_ASYM_SIZE "NumConcurrentAsymRequests" +#define ADF_RING_DC_SIZE "NumConcurrentRequests" +#define ADF_RING_ASYM_TX "RingAsymTx" +#define ADF_RING_SYM_TX "RingSymTx" +#define ADF_RING_RND_TX "RingNrbgTx" +#define ADF_RING_ASYM_RX "RingAsymRx" +#define ADF_RING_SYM_RX "RinSymRx" +#define ADF_RING_RND_RX "RingNrbgRx" +#define ADF_RING_DC_TX "RingTx" +#define ADF_RING_DC_RX "RingRx" +#define ADF_ETRMGR_BANK "Bank" +#define ADF_RING_BANK_NUM "BankNumber" +#define ADF_CY "Cy" +#define ADF_DC "Dc" +#define ADF_ETRMGR_COALESCING_ENABLED "InterruptCoalescingEnabled" +#define ADF_ETRMGR_COALESCING_ENABLED_FORMAT \ + ADF_ETRMGR_BANK"%d"ADF_ETRMGR_COALESCING_ENABLED +#define ADF_ETRMGR_COALESCE_TIMER "InterruptCoalescingTimerNs" +#define ADF_ETRMGR_COALESCE_TIMER_FORMAT \ + ADF_ETRMGR_BANK"%d"ADF_ETRMGR_COALESCE_TIMER +#define ADF_ETRMGR_COALESCING_MSG_ENABLED "InterruptCoalescingNumResponses" +#define ADF_ETRMGR_COALESCING_MSG_ENABLED_FORMAT \ + ADF_ETRMGR_BANK"%d"ADF_ETRMGR_COALESCING_MSG_ENABLED +#define ADF_ETRMGR_CORE_AFFINITY "CoreAffinity" +#define ADF_ETRMGR_CORE_AFFINITY_FORMAT \ + ADF_ETRMGR_BANK"%d"ADF_ETRMGR_CORE_AFFINITY +#define ADF_ACCEL_STR "Accelerator%d" +#endif diff --git a/drivers/crypto/qat/qat_common/adf_cfg_user.h b/drivers/crypto/qat/qat_common/adf_cfg_user.h new file mode 100644 index 000000000000..0c38a155a865 --- /dev/null +++ b/drivers/crypto/qat/qat_common/adf_cfg_user.h @@ -0,0 +1,94 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef ADF_CFG_USER_H_ +#define ADF_CFG_USER_H_ + +#include "adf_cfg_common.h" +#include "adf_cfg_strings.h" + +struct adf_user_cfg_key_val { + char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; + char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; + union { + char *user_val_ptr; + uint64_t padding1; + }; + union { + struct adf_user_cfg_key_val *prev; + uint64_t padding2; + }; + union { + struct adf_user_cfg_key_val *next; + uint64_t padding3; + }; + enum adf_cfg_val_type type; +}; + +struct adf_user_cfg_section { + char name[ADF_CFG_MAX_SECTION_LEN_IN_BYTES]; + union { + struct adf_user_cfg_key_val *params; + uint64_t padding1; + }; + union { + struct adf_user_cfg_section *prev; + uint64_t padding2; + }; + union { + struct adf_user_cfg_section *next; + uint64_t padding3; + }; +}; + +struct adf_user_cfg_ctl_data { + union { + struct adf_user_cfg_section *config_section; + uint64_t padding; + }; + uint8_t device_id; +}; +#endif diff --git a/drivers/crypto/qat/qat_common/adf_common_drv.h b/drivers/crypto/qat/qat_common/adf_common_drv.h new file mode 100644 index 000000000000..5e8f9d431e5d --- /dev/null +++ b/drivers/crypto/qat/qat_common/adf_common_drv.h @@ -0,0 +1,192 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef ADF_DRV_H +#define ADF_DRV_H + +#include <linux/list.h> +#include <linux/pci.h> +#include "adf_accel_devices.h" +#include "icp_qat_fw_loader_handle.h" +#include "icp_qat_hal.h" + +#define ADF_STATUS_RESTARTING 0 +#define ADF_STATUS_STARTING 1 +#define ADF_STATUS_CONFIGURED 2 +#define ADF_STATUS_STARTED 3 +#define ADF_STATUS_AE_INITIALISED 4 +#define ADF_STATUS_AE_UCODE_LOADED 5 +#define ADF_STATUS_AE_STARTED 6 +#define ADF_STATUS_ORPHAN_TH_RUNNING 7 +#define ADF_STATUS_IRQ_ALLOCATED 8 + +enum adf_dev_reset_mode { + ADF_DEV_RESET_ASYNC = 0, + ADF_DEV_RESET_SYNC +}; + +enum adf_event { + ADF_EVENT_INIT = 0, + ADF_EVENT_START, + ADF_EVENT_STOP, + ADF_EVENT_SHUTDOWN, + ADF_EVENT_RESTARTING, + ADF_EVENT_RESTARTED, +}; + +struct service_hndl { + int (*event_hld)(struct adf_accel_dev *accel_dev, + enum adf_event event); + unsigned long init_status; + unsigned long start_status; + char *name; + struct list_head list; + int admin; +}; + +int adf_service_register(struct service_hndl *service); +int adf_service_unregister(struct service_hndl *service); + +int adf_dev_init(struct adf_accel_dev *accel_dev); +int adf_dev_start(struct adf_accel_dev *accel_dev); +int adf_dev_stop(struct adf_accel_dev *accel_dev); +int adf_dev_shutdown(struct adf_accel_dev *accel_dev); + +int adf_ctl_dev_register(void); +void adf_ctl_dev_unregister(void); +int adf_processes_dev_register(void); +void adf_processes_dev_unregister(void); + +int adf_devmgr_add_dev(struct adf_accel_dev *accel_dev); +void adf_devmgr_rm_dev(struct adf_accel_dev *accel_dev); +struct list_head *adf_devmgr_get_head(void); +struct adf_accel_dev *adf_devmgr_get_dev_by_id(uint32_t id); +struct adf_accel_dev *adf_devmgr_get_first(void); +struct adf_accel_dev *adf_devmgr_pci_to_accel_dev(struct pci_dev *pci_dev); +int adf_devmgr_verify_id(uint32_t id); +void adf_devmgr_get_num_dev(uint32_t *num); +int adf_devmgr_in_reset(struct adf_accel_dev *accel_dev); +int adf_dev_started(struct adf_accel_dev *accel_dev); +int adf_dev_restarting_notify(struct adf_accel_dev *accel_dev); +int adf_dev_restarted_notify(struct adf_accel_dev *accel_dev); +int adf_ae_init(struct adf_accel_dev *accel_dev); +int adf_ae_shutdown(struct adf_accel_dev *accel_dev); +int adf_ae_fw_load(struct adf_accel_dev *accel_dev); +int adf_ae_fw_release(struct adf_accel_dev *accel_dev); +int adf_ae_start(struct adf_accel_dev *accel_dev); +int adf_ae_stop(struct adf_accel_dev *accel_dev); + +int adf_enable_aer(struct adf_accel_dev *accel_dev, struct pci_driver *adf); +void adf_disable_aer(struct adf_accel_dev *accel_dev); +int adf_init_aer(void); +void adf_exit_aer(void); + +int adf_dev_get(struct adf_accel_dev *accel_dev); +void adf_dev_put(struct adf_accel_dev *accel_dev); +int adf_dev_in_use(struct adf_accel_dev *accel_dev); +int adf_init_etr_data(struct adf_accel_dev *accel_dev); +void adf_cleanup_etr_data(struct adf_accel_dev *accel_dev); +int qat_crypto_register(void); +int qat_crypto_unregister(void); +struct qat_crypto_instance *qat_crypto_get_instance_node(int node); +void qat_crypto_put_instance(struct qat_crypto_instance *inst); +void qat_alg_callback(void *resp); +int qat_algs_init(void); +void qat_algs_exit(void); +int qat_algs_register(void); +int qat_algs_unregister(void); + +int qat_hal_init(struct adf_accel_dev *accel_dev); +void qat_hal_deinit(struct icp_qat_fw_loader_handle *handle); +void qat_hal_start(struct icp_qat_fw_loader_handle *handle, unsigned char ae, + unsigned int ctx_mask); +void qat_hal_stop(struct icp_qat_fw_loader_handle *handle, unsigned char ae, + unsigned int ctx_mask); +void qat_hal_reset(struct icp_qat_fw_loader_handle *handle); +int qat_hal_clr_reset(struct icp_qat_fw_loader_handle *handle); +void qat_hal_set_live_ctx(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned int ctx_mask); +int qat_hal_set_ae_lm_mode(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, enum icp_qat_uof_regtype lm_type, + unsigned char mode); +int qat_hal_set_ae_ctx_mode(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned char mode); +int qat_hal_set_ae_nn_mode(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned char mode); +void qat_hal_set_pc(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned int ctx_mask, unsigned int upc); +void qat_hal_wr_uwords(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned int uaddr, + unsigned int words_num, uint64_t *uword); +void qat_hal_wr_umem(struct icp_qat_fw_loader_handle *handle, unsigned char ae, + unsigned int uword_addr, unsigned int words_num, + unsigned int *data); +int qat_hal_get_ins_num(void); +int qat_hal_batch_wr_lm(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, + struct icp_qat_uof_batch_init *lm_init_header); +int qat_hal_init_gpr(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned char ctx_mask, + enum icp_qat_uof_regtype reg_type, + unsigned short reg_num, unsigned int regdata); +int qat_hal_init_wr_xfer(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned char ctx_mask, + enum icp_qat_uof_regtype reg_type, + unsigned short reg_num, unsigned int regdata); +int qat_hal_init_rd_xfer(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned char ctx_mask, + enum icp_qat_uof_regtype reg_type, + unsigned short reg_num, unsigned int regdata); +int qat_hal_init_nn(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned char ctx_mask, + unsigned short reg_num, unsigned int regdata); +int qat_hal_wr_lm(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned short lm_addr, unsigned int value); +int qat_uclo_wr_all_uimage(struct icp_qat_fw_loader_handle *handle); +void qat_uclo_del_uof_obj(struct icp_qat_fw_loader_handle *handle); +int qat_uclo_map_uof_obj(struct icp_qat_fw_loader_handle *handle, + void *addr_ptr, int mem_size); +#endif diff --git a/drivers/crypto/qat/qat_common/adf_ctl_drv.c b/drivers/crypto/qat/qat_common/adf_ctl_drv.c new file mode 100644 index 000000000000..d97069b8a8e4 --- /dev/null +++ b/drivers/crypto/qat/qat_common/adf_ctl_drv.c @@ -0,0 +1,490 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#include <linux/module.h> +#include <linux/mutex.h> +#include <linux/slab.h> +#include <linux/fs.h> +#include <linux/bitops.h> +#include <linux/pci.h> +#include <linux/cdev.h> +#include <linux/uaccess.h> + +#include "adf_accel_devices.h" +#include "adf_common_drv.h" +#include "adf_cfg.h" +#include "adf_cfg_common.h" +#include "adf_cfg_user.h" + +#define DEVICE_NAME "qat_adf_ctl" + +static DEFINE_MUTEX(adf_ctl_lock); +static long adf_ctl_ioctl(struct file *fp, unsigned int cmd, unsigned long arg); + +static const struct file_operations adf_ctl_ops = { + .owner = THIS_MODULE, + .unlocked_ioctl = adf_ctl_ioctl, + .compat_ioctl = adf_ctl_ioctl, +}; + +struct adf_ctl_drv_info { + unsigned int major; + struct cdev drv_cdev; + struct class *drv_class; +}; + +static struct adf_ctl_drv_info adt_ctl_drv; + +static void adf_chr_drv_destroy(void) +{ + device_destroy(adt_ctl_drv.drv_class, MKDEV(adt_ctl_drv.major, 0)); + cdev_del(&adt_ctl_drv.drv_cdev); + class_destroy(adt_ctl_drv.drv_class); + unregister_chrdev_region(MKDEV(adt_ctl_drv.major, 0), 1); +} + +static int adf_chr_drv_create(void) +{ + dev_t dev_id; + struct device *drv_device; + + if (alloc_chrdev_region(&dev_id, 0, 1, DEVICE_NAME)) { + pr_err("QAT: unable to allocate chrdev region\n"); + return -EFAULT; + } + + adt_ctl_drv.drv_class = class_create(THIS_MODULE, DEVICE_NAME); + if (IS_ERR(adt_ctl_drv.drv_class)) { + pr_err("QAT: class_create failed for adf_ctl\n"); + goto err_chrdev_unreg; + } + adt_ctl_drv.major = MAJOR(dev_id); + cdev_init(&adt_ctl_drv.drv_cdev, &adf_ctl_ops); + if (cdev_add(&adt_ctl_drv.drv_cdev, dev_id, 1)) { + pr_err("QAT: cdev add failed\n"); + goto err_class_destr; + } + + drv_device = device_create(adt_ctl_drv.drv_class, NULL, + MKDEV(adt_ctl_drv.major, 0), + NULL, DEVICE_NAME); + if (!drv_device) { + pr_err("QAT: failed to create device\n"); + goto err_cdev_del; + } + return 0; +err_cdev_del: + cdev_del(&adt_ctl_drv.drv_cdev); +err_class_destr: + class_destroy(adt_ctl_drv.drv_class); +err_chrdev_unreg: + unregister_chrdev_region(dev_id, 1); + return -EFAULT; +} + +static int adf_ctl_alloc_resources(struct adf_user_cfg_ctl_data **ctl_data, + unsigned long arg) +{ + struct adf_user_cfg_ctl_data *cfg_data; + + cfg_data = kzalloc(sizeof(*cfg_data), GFP_KERNEL); + if (!cfg_data) + return -ENOMEM; + + /* Initialize device id to NO DEVICE as 0 is a valid device id */ + cfg_data->device_id = ADF_CFG_NO_DEVICE; + + if (copy_from_user(cfg_data, (void __user *)arg, sizeof(*cfg_data))) { + pr_err("QAT: failed to copy from user cfg_data.\n"); + kfree(cfg_data); + return -EIO; + } + + *ctl_data = cfg_data; + return 0; +} + +static int adf_add_key_value_data(struct adf_accel_dev *accel_dev, + const char *section, + const struct adf_user_cfg_key_val *key_val) +{ + if (key_val->type == ADF_HEX) { + long *ptr = (long *)key_val->val; + long val = *ptr; + + if (adf_cfg_add_key_value_param(accel_dev, section, + key_val->key, (void *)val, + key_val->type)) { + pr_err("QAT: failed to add keyvalue.\n"); + return -EFAULT; + } + } else { + if (adf_cfg_add_key_value_param(accel_dev, section, + key_val->key, key_val->val, + key_val->type)) { + pr_err("QAT: failed to add keyvalue.\n"); + return -EFAULT; + } + } + return 0; +} + +static int adf_copy_key_value_data(struct adf_accel_dev *accel_dev, + struct adf_user_cfg_ctl_data *ctl_data) +{ + struct adf_user_cfg_key_val key_val; + struct adf_user_cfg_key_val *params_head; + struct adf_user_cfg_section section, *section_head; + + section_head = ctl_data->config_section; + + while (section_head) { + if (copy_from_user(§ion, (void __user *)section_head, + sizeof(*section_head))) { + pr_err("QAT: failed to copy section info\n"); + goto out_err; + } + + if (adf_cfg_section_add(accel_dev, section.name)) { + pr_err("QAT: failed to add section.\n"); + goto out_err; + } + + params_head = section_head->params; + + while (params_head) { + if (copy_from_user(&key_val, (void __user *)params_head, + sizeof(key_val))) { + pr_err("QAT: Failed to copy keyvalue.\n"); + goto out_err; + } + if (adf_add_key_value_data(accel_dev, section.name, + &key_val)) { + goto out_err; + } + params_head = key_val.next; + } + section_head = section.next; + } + return 0; +out_err: + adf_cfg_del_all(accel_dev); + return -EFAULT; +} + +static int adf_ctl_ioctl_dev_config(struct file *fp, unsigned int cmd, + unsigned long arg) +{ + int ret; + struct adf_user_cfg_ctl_data *ctl_data; + struct adf_accel_dev *accel_dev; + + ret = adf_ctl_alloc_resources(&ctl_data, arg); + if (ret) + return ret; + + accel_dev = adf_devmgr_get_dev_by_id(ctl_data->device_id); + if (!accel_dev) { + ret = -EFAULT; + goto out; + } + + if (adf_dev_started(accel_dev)) { + ret = -EFAULT; + goto out; + } + + if (adf_copy_key_value_data(accel_dev, ctl_data)) { + ret = -EFAULT; + goto out; + } + set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); +out: + kfree(ctl_data); + return ret; +} + +static int adf_ctl_is_device_in_use(int id) +{ + struct list_head *itr, *head = adf_devmgr_get_head(); + + list_for_each(itr, head) { + struct adf_accel_dev *dev = + list_entry(itr, struct adf_accel_dev, list); + + if (id == dev->accel_id || id == ADF_CFG_ALL_DEVICES) { + if (adf_devmgr_in_reset(dev) || adf_dev_in_use(dev)) { + pr_info("QAT: device qat_dev%d is busy\n", + dev->accel_id); + return -EBUSY; + } + } + } + return 0; +} + +static int adf_ctl_stop_devices(uint32_t id) +{ + struct list_head *itr, *head = adf_devmgr_get_head(); + int ret = 0; + + list_for_each(itr, head) { + struct adf_accel_dev *accel_dev = + list_entry(itr, struct adf_accel_dev, list); + if (id == accel_dev->accel_id || id == ADF_CFG_ALL_DEVICES) { + if (!adf_dev_started(accel_dev)) + continue; + + if (adf_dev_stop(accel_dev)) { + pr_err("QAT: Failed to stop qat_dev%d\n", id); + ret = -EFAULT; + } + } + } + return ret; +} + +static int adf_ctl_ioctl_dev_stop(struct file *fp, unsigned int cmd, + unsigned long arg) +{ + int ret; + struct adf_user_cfg_ctl_data *ctl_data; + + ret = adf_ctl_alloc_resources(&ctl_data, arg); + if (ret) + return ret; + + if (adf_devmgr_verify_id(ctl_data->device_id)) { + pr_err("QAT: Device %d not found\n", ctl_data->device_id); + ret = -ENODEV; + goto out; + } + + ret = adf_ctl_is_device_in_use(ctl_data->device_id); + if (ret) + goto out; + + if (ctl_data->device_id == ADF_CFG_ALL_DEVICES) + pr_info("QAT: Stopping all acceleration devices.\n"); + else + pr_info("QAT: Stopping acceleration device qat_dev%d.\n", + ctl_data->device_id); + + ret = adf_ctl_stop_devices(ctl_data->device_id); + if (ret) + pr_err("QAT: failed to stop device.\n"); +out: + kfree(ctl_data); + return ret; +} + +static int adf_ctl_ioctl_dev_start(struct file *fp, unsigned int cmd, + unsigned long arg) +{ + int ret; + struct adf_user_cfg_ctl_data *ctl_data; + struct adf_accel_dev *accel_dev; + + ret = adf_ctl_alloc_resources(&ctl_data, arg); + if (ret) + return ret; + + accel_dev = adf_devmgr_get_dev_by_id(ctl_data->device_id); + if (!accel_dev) { + pr_err("QAT: Device %d not found\n", ctl_data->device_id); + ret = -ENODEV; + goto out; + } + + if (!adf_dev_started(accel_dev)) { + pr_info("QAT: Starting acceleration device qat_dev%d.\n", + ctl_data->device_id); + ret = adf_dev_start(accel_dev); + } else { + pr_info("QAT: Acceleration device qat_dev%d already started.\n", + ctl_data->device_id); + } + if (ret) { + pr_err("QAT: Failed to start qat_dev%d\n", ctl_data->device_id); + adf_dev_stop(accel_dev); + } +out: + kfree(ctl_data); + return ret; +} + +static int adf_ctl_ioctl_get_num_devices(struct file *fp, unsigned int cmd, + unsigned long arg) +{ + uint32_t num_devices = 0; + + adf_devmgr_get_num_dev(&num_devices); + if (copy_to_user((void __user *)arg, &num_devices, sizeof(num_devices))) + return -EFAULT; + + return 0; +} + +static int adf_ctl_ioctl_get_status(struct file *fp, unsigned int cmd, + unsigned long arg) +{ + struct adf_hw_device_data *hw_data; + struct adf_dev_status_info dev_info; + struct adf_accel_dev *accel_dev; + + if (copy_from_user(&dev_info, (void __user *)arg, + sizeof(struct adf_dev_status_info))) { + pr_err("QAT: failed to copy from user.\n"); + return -EFAULT; + } + + accel_dev = adf_devmgr_get_dev_by_id(dev_info.accel_id); + if (!accel_dev) { + pr_err("QAT: Device %d not found\n", dev_info.accel_id); + return -ENODEV; + } + hw_data = accel_dev->hw_device; + dev_info.state = adf_dev_started(accel_dev) ? DEV_UP : DEV_DOWN; + dev_info.num_ae = hw_data->get_num_aes(hw_data); + dev_info.num_accel = hw_data->get_num_accels(hw_data); + dev_info.num_logical_accel = hw_data->num_logical_accel; + dev_info.banks_per_accel = hw_data->num_banks + / hw_data->num_logical_accel; + strlcpy(dev_info.name, hw_data->dev_class->name, sizeof(dev_info.name)); + dev_info.instance_id = hw_data->instance_id; + dev_info.type = hw_data->dev_class->type; + dev_info.bus = accel_to_pci_dev(accel_dev)->bus->number; + dev_info.dev = PCI_SLOT(accel_to_pci_dev(accel_dev)->devfn); + dev_info.fun = PCI_FUNC(accel_to_pci_dev(accel_dev)->devfn); + + if (copy_to_user((void __user *)arg, &dev_info, + sizeof(struct adf_dev_status_info))) { + pr_err("QAT: failed to copy status.\n"); + return -EFAULT; + } + return 0; +} + +static long adf_ctl_ioctl(struct file *fp, unsigned int cmd, unsigned long arg) +{ + int ret; + + if (mutex_lock_interruptible(&adf_ctl_lock)) + return -EFAULT; + + switch (cmd) { + case IOCTL_CONFIG_SYS_RESOURCE_PARAMETERS: + ret = adf_ctl_ioctl_dev_config(fp, cmd, arg); + break; + + case IOCTL_STOP_ACCEL_DEV: + ret = adf_ctl_ioctl_dev_stop(fp, cmd, arg); + break; + + case IOCTL_START_ACCEL_DEV: + ret = adf_ctl_ioctl_dev_start(fp, cmd, arg); + break; + + case IOCTL_GET_NUM_DEVICES: + ret = adf_ctl_ioctl_get_num_devices(fp, cmd, arg); + break; + + case IOCTL_STATUS_ACCEL_DEV: + ret = adf_ctl_ioctl_get_status(fp, cmd, arg); + break; + default: + pr_err("QAT: Invalid ioclt\n"); + ret = -EFAULT; + break; + } + mutex_unlock(&adf_ctl_lock); + return ret; +} + +static int __init adf_register_ctl_device_driver(void) +{ + mutex_init(&adf_ctl_lock); + + if (qat_algs_init()) + goto err_algs_init; + + if (adf_chr_drv_create()) + goto err_chr_dev; + + if (adf_init_aer()) + goto err_aer; + + if (qat_crypto_register()) + goto err_crypto_register; + + return 0; + +err_crypto_register: + adf_exit_aer(); +err_aer: + adf_chr_drv_destroy(); +err_chr_dev: + qat_algs_exit(); +err_algs_init: + mutex_destroy(&adf_ctl_lock); + return -EFAULT; +} + +static void __exit adf_unregister_ctl_device_driver(void) +{ + adf_chr_drv_destroy(); + adf_exit_aer(); + qat_crypto_unregister(); + qat_algs_exit(); + mutex_destroy(&adf_ctl_lock); +} + +module_init(adf_register_ctl_device_driver); +module_exit(adf_unregister_ctl_device_driver); +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Intel"); +MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); +MODULE_ALIAS("intel_qat"); diff --git a/drivers/crypto/qat/qat_common/adf_dev_mgr.c b/drivers/crypto/qat/qat_common/adf_dev_mgr.c new file mode 100644 index 000000000000..ae71555c0868 --- /dev/null +++ b/drivers/crypto/qat/qat_common/adf_dev_mgr.c @@ -0,0 +1,215 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#include <linux/mutex.h> +#include <linux/list.h> +#include "adf_cfg.h" +#include "adf_common_drv.h" + +static LIST_HEAD(accel_table); +static DEFINE_MUTEX(table_lock); +static uint32_t num_devices; + +/** + * adf_devmgr_add_dev() - Add accel_dev to the acceleration framework + * @accel_dev: Pointer to acceleration device. + * + * Function adds acceleration device to the acceleration framework. + * To be used by QAT device specific drivers. + * + * Return: 0 on success, error code othewise. + */ +int adf_devmgr_add_dev(struct adf_accel_dev *accel_dev) +{ + struct list_head *itr; + + if (num_devices == ADF_MAX_DEVICES) { + pr_err("QAT: Only support up to %d devices\n", ADF_MAX_DEVICES); + return -EFAULT; + } + + mutex_lock(&table_lock); + list_for_each(itr, &accel_table) { + struct adf_accel_dev *ptr = + list_entry(itr, struct adf_accel_dev, list); + + if (ptr == accel_dev) { + mutex_unlock(&table_lock); + return -EEXIST; + } + } + atomic_set(&accel_dev->ref_count, 0); + list_add_tail(&accel_dev->list, &accel_table); + accel_dev->accel_id = num_devices++; + mutex_unlock(&table_lock); + return 0; +} +EXPORT_SYMBOL_GPL(adf_devmgr_add_dev); + +struct list_head *adf_devmgr_get_head(void) +{ + return &accel_table; +} + +/** + * adf_devmgr_rm_dev() - Remove accel_dev from the acceleration framework. + * @accel_dev: Pointer to acceleration device. + * + * Function removes acceleration device from the acceleration framework. + * To be used by QAT device specific drivers. + * + * Return: void + */ +void adf_devmgr_rm_dev(struct adf_accel_dev *accel_dev) +{ + mutex_lock(&table_lock); + list_del(&accel_dev->list); + num_devices--; + mutex_unlock(&table_lock); +} +EXPORT_SYMBOL_GPL(adf_devmgr_rm_dev); + +struct adf_accel_dev *adf_devmgr_get_first(void) +{ + struct adf_accel_dev *dev = NULL; + + if (!list_empty(&accel_table)) + dev = list_first_entry(&accel_table, struct adf_accel_dev, + list); + return dev; +} + +/** + * adf_devmgr_pci_to_accel_dev() - Get accel_dev associated with the pci_dev. + * @accel_dev: Pointer to pci device. + * + * Function returns acceleration device associated with the given pci device. + * To be used by QAT device specific drivers. + * + * Return: pinter to accel_dev or NULL if not found. + */ +struct adf_accel_dev *adf_devmgr_pci_to_accel_dev(struct pci_dev *pci_dev) +{ + struct list_head *itr; + + list_for_each(itr, &accel_table) { + struct adf_accel_dev *ptr = + list_entry(itr, struct adf_accel_dev, list); + + if (ptr->accel_pci_dev.pci_dev == pci_dev) { + mutex_unlock(&table_lock); + return ptr; + } + } + return NULL; +} +EXPORT_SYMBOL_GPL(adf_devmgr_pci_to_accel_dev); + +struct adf_accel_dev *adf_devmgr_get_dev_by_id(uint32_t id) +{ + struct list_head *itr; + + list_for_each(itr, &accel_table) { + struct adf_accel_dev *ptr = + list_entry(itr, struct adf_accel_dev, list); + + if (ptr->accel_id == id) { + mutex_unlock(&table_lock); + return ptr; + } + } + return NULL; +} + +int adf_devmgr_verify_id(uint32_t id) +{ + if (id == ADF_CFG_ALL_DEVICES) + return 0; + + if (adf_devmgr_get_dev_by_id(id)) + return 0; + + return -ENODEV; +} + +void adf_devmgr_get_num_dev(uint32_t *num) +{ + struct list_head *itr; + + *num = 0; + list_for_each(itr, &accel_table) { + (*num)++; + } +} + +int adf_dev_in_use(struct adf_accel_dev *accel_dev) +{ + return atomic_read(&accel_dev->ref_count) != 0; +} + +int adf_dev_get(struct adf_accel_dev *accel_dev) +{ + if (atomic_add_return(1, &accel_dev->ref_count) == 1) + if (!try_module_get(accel_dev->owner)) + return -EFAULT; + return 0; +} + +void adf_dev_put(struct adf_accel_dev *accel_dev) +{ + if (atomic_sub_return(1, &accel_dev->ref_count) == 0) + module_put(accel_dev->owner); +} + +int adf_devmgr_in_reset(struct adf_accel_dev *accel_dev) +{ + return test_bit(ADF_STATUS_RESTARTING, &accel_dev->status); +} + +int adf_dev_started(struct adf_accel_dev *accel_dev) +{ + return test_bit(ADF_STATUS_STARTED, &accel_dev->status); +} diff --git a/drivers/crypto/qat/qat_common/adf_init.c b/drivers/crypto/qat/qat_common/adf_init.c new file mode 100644 index 000000000000..5c0e47a00a87 --- /dev/null +++ b/drivers/crypto/qat/qat_common/adf_init.c @@ -0,0 +1,388 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#include <linux/mutex.h> +#include <linux/list.h> +#include <linux/bitops.h> +#include <linux/delay.h> +#include "adf_accel_devices.h" +#include "adf_cfg.h" +#include "adf_common_drv.h" + +static LIST_HEAD(service_table); +static DEFINE_MUTEX(service_lock); + +static void adf_service_add(struct service_hndl *service) +{ + mutex_lock(&service_lock); + list_add(&service->list, &service_table); + mutex_unlock(&service_lock); +} + +/** + * adf_service_register() - Register acceleration service in the accel framework + * @service: Pointer to the service + * + * Function adds the acceleration service to the acceleration framework. + * To be used by QAT device specific drivers. + * + * Return: 0 on success, error code othewise. + */ +int adf_service_register(struct service_hndl *service) +{ + service->init_status = 0; + service->start_status = 0; + adf_service_add(service); + return 0; +} +EXPORT_SYMBOL_GPL(adf_service_register); + +static void adf_service_remove(struct service_hndl *service) +{ + mutex_lock(&service_lock); + list_del(&service->list); + mutex_unlock(&service_lock); +} + +/** + * adf_service_unregister() - Unregister acceleration service from the framework + * @service: Pointer to the service + * + * Function remove the acceleration service from the acceleration framework. + * To be used by QAT device specific drivers. + * + * Return: 0 on success, error code othewise. + */ +int adf_service_unregister(struct service_hndl *service) +{ + if (service->init_status || service->start_status) { + pr_err("QAT: Could not remove active service\n"); + return -EFAULT; + } + adf_service_remove(service); + return 0; +} +EXPORT_SYMBOL_GPL(adf_service_unregister); + +/** + * adf_dev_start() - Start acceleration service for the given accel device + * @accel_dev: Pointer to acceleration device. + * + * Function notifies all the registered services that the acceleration device + * is ready to be used. + * To be used by QAT device specific drivers. + * + * Return: 0 on success, error code othewise. + */ +int adf_dev_start(struct adf_accel_dev *accel_dev) +{ + struct service_hndl *service; + struct list_head *list_itr; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + + if (!test_bit(ADF_STATUS_CONFIGURED, &accel_dev->status)) { + pr_info("QAT: Device not configured\n"); + return -EFAULT; + } + set_bit(ADF_STATUS_STARTING, &accel_dev->status); + + if (adf_ae_init(accel_dev)) { + pr_err("QAT: Failed to initialise Acceleration Engine\n"); + return -EFAULT; + } + set_bit(ADF_STATUS_AE_INITIALISED, &accel_dev->status); + + if (adf_ae_fw_load(accel_dev)) { + pr_err("QAT: Failed to load acceleration FW\n"); + adf_ae_fw_release(accel_dev); + return -EFAULT; + } + set_bit(ADF_STATUS_AE_UCODE_LOADED, &accel_dev->status); + + if (hw_data->alloc_irq(accel_dev)) { + pr_err("QAT: Failed to allocate interrupts\n"); + return -EFAULT; + } + set_bit(ADF_STATUS_IRQ_ALLOCATED, &accel_dev->status); + + /* + * Subservice initialisation is divided into two stages: init and start. + * This is to facilitate any ordering dependencies between services + * prior to starting any of the accelerators. + */ + list_for_each(list_itr, &service_table) { + service = list_entry(list_itr, struct service_hndl, list); + if (!service->admin) + continue; + if (service->event_hld(accel_dev, ADF_EVENT_INIT)) { + pr_err("QAT: Failed to initialise service %s\n", + service->name); + return -EFAULT; + } + set_bit(accel_dev->accel_id, &service->init_status); + } + list_for_each(list_itr, &service_table) { + service = list_entry(list_itr, struct service_hndl, list); + if (service->admin) + continue; + if (service->event_hld(accel_dev, ADF_EVENT_INIT)) { + pr_err("QAT: Failed to initialise service %s\n", + service->name); + return -EFAULT; + } + set_bit(accel_dev->accel_id, &service->init_status); + } + + hw_data->enable_error_correction(accel_dev); + + if (adf_ae_start(accel_dev)) { + pr_err("QAT: AE Start Failed\n"); + return -EFAULT; + } + set_bit(ADF_STATUS_AE_STARTED, &accel_dev->status); + + list_for_each(list_itr, &service_table) { + service = list_entry(list_itr, struct service_hndl, list); + if (!service->admin) + continue; + if (service->event_hld(accel_dev, ADF_EVENT_START)) { + pr_err("QAT: Failed to start service %s\n", + service->name); + return -EFAULT; + } + set_bit(accel_dev->accel_id, &service->start_status); + } + list_for_each(list_itr, &service_table) { + service = list_entry(list_itr, struct service_hndl, list); + if (service->admin) + continue; + if (service->event_hld(accel_dev, ADF_EVENT_START)) { + pr_err("QAT: Failed to start service %s\n", + service->name); + return -EFAULT; + } + set_bit(accel_dev->accel_id, &service->start_status); + } + + clear_bit(ADF_STATUS_STARTING, &accel_dev->status); + set_bit(ADF_STATUS_STARTED, &accel_dev->status); + + if (qat_algs_register()) { + pr_err("QAT: Failed to register crypto algs\n"); + set_bit(ADF_STATUS_STARTING, &accel_dev->status); + clear_bit(ADF_STATUS_STARTED, &accel_dev->status); + return -EFAULT; + } + return 0; +} +EXPORT_SYMBOL_GPL(adf_dev_start); + +/** + * adf_dev_stop() - Stop acceleration service for the given accel device + * @accel_dev: Pointer to acceleration device. + * + * Function notifies all the registered services that the acceleration device + * is shuting down. + * To be used by QAT device specific drivers. + * + * Return: 0 on success, error code othewise. + */ +int adf_dev_stop(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct service_hndl *service; + struct list_head *list_itr; + int ret, wait = 0; + + if (!adf_dev_started(accel_dev) && + !test_bit(ADF_STATUS_STARTING, &accel_dev->status)) { + return 0; + } + clear_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); + clear_bit(ADF_STATUS_STARTING, &accel_dev->status); + clear_bit(ADF_STATUS_STARTED, &accel_dev->status); + + if (qat_algs_unregister()) + pr_err("QAT: Failed to unregister crypto algs\n"); + + list_for_each(list_itr, &service_table) { + service = list_entry(list_itr, struct service_hndl, list); + if (service->admin) + continue; + if (!test_bit(accel_dev->accel_id, &service->start_status)) + continue; + ret = service->event_hld(accel_dev, ADF_EVENT_STOP); + if (!ret) { + clear_bit(accel_dev->accel_id, &service->start_status); + } else if (ret == -EAGAIN) { + wait = 1; + clear_bit(accel_dev->accel_id, &service->start_status); + } + } + list_for_each(list_itr, &service_table) { + service = list_entry(list_itr, struct service_hndl, list); + if (!service->admin) + continue; + if (!test_bit(accel_dev->accel_id, &service->start_status)) + continue; + if (service->event_hld(accel_dev, ADF_EVENT_STOP)) + pr_err("QAT: Failed to shutdown service %s\n", + service->name); + else + clear_bit(accel_dev->accel_id, &service->start_status); + } + + if (wait) + msleep(100); + + if (adf_dev_started(accel_dev)) { + if (adf_ae_stop(accel_dev)) + pr_err("QAT: failed to stop AE\n"); + else + clear_bit(ADF_STATUS_AE_STARTED, &accel_dev->status); + } + + if (test_bit(ADF_STATUS_AE_UCODE_LOADED, &accel_dev->status)) { + if (adf_ae_fw_release(accel_dev)) + pr_err("QAT: Failed to release the ucode\n"); + else + clear_bit(ADF_STATUS_AE_UCODE_LOADED, + &accel_dev->status); + } + + if (test_bit(ADF_STATUS_AE_INITIALISED, &accel_dev->status)) { + if (adf_ae_shutdown(accel_dev)) + pr_err("QAT: Failed to shutdown Accel Engine\n"); + else + clear_bit(ADF_STATUS_AE_INITIALISED, + &accel_dev->status); + } + + list_for_each(list_itr, &service_table) { + service = list_entry(list_itr, struct service_hndl, list); + if (service->admin) + continue; + if (!test_bit(accel_dev->accel_id, &service->init_status)) + continue; + if (service->event_hld(accel_dev, ADF_EVENT_SHUTDOWN)) + pr_err("QAT: Failed to shutdown service %s\n", + service->name); + else + clear_bit(accel_dev->accel_id, &service->init_status); + } + list_for_each(list_itr, &service_table) { + service = list_entry(list_itr, struct service_hndl, list); + if (!service->admin) + continue; + if (!test_bit(accel_dev->accel_id, &service->init_status)) + continue; + if (service->event_hld(accel_dev, ADF_EVENT_SHUTDOWN)) + pr_err("QAT: Failed to shutdown service %s\n", + service->name); + else + clear_bit(accel_dev->accel_id, &service->init_status); + } + + if (test_bit(ADF_STATUS_IRQ_ALLOCATED, &accel_dev->status)) { + hw_data->free_irq(accel_dev); + clear_bit(ADF_STATUS_IRQ_ALLOCATED, &accel_dev->status); + } + + /* Delete configuration only if not restarting */ + if (!test_bit(ADF_STATUS_RESTARTING, &accel_dev->status)) + adf_cfg_del_all(accel_dev); + + return 0; +} +EXPORT_SYMBOL_GPL(adf_dev_stop); + +int adf_dev_restarting_notify(struct adf_accel_dev *accel_dev) +{ + struct service_hndl *service; + struct list_head *list_itr; + + list_for_each(list_itr, &service_table) { + service = list_entry(list_itr, struct service_hndl, list); + if (service->admin) + continue; + if (service->event_hld(accel_dev, ADF_EVENT_RESTARTING)) + pr_err("QAT: Failed to restart service %s.\n", + service->name); + } + list_for_each(list_itr, &service_table) { + service = list_entry(list_itr, struct service_hndl, list); + if (!service->admin) + continue; + if (service->event_hld(accel_dev, ADF_EVENT_RESTARTING)) + pr_err("QAT: Failed to restart service %s.\n", + service->name); + } + return 0; +} + +int adf_dev_restarted_notify(struct adf_accel_dev *accel_dev) +{ + struct service_hndl *service; + struct list_head *list_itr; + + list_for_each(list_itr, &service_table) { + service = list_entry(list_itr, struct service_hndl, list); + if (service->admin) + continue; + if (service->event_hld(accel_dev, ADF_EVENT_RESTARTED)) + pr_err("QAT: Failed to restart service %s.\n", + service->name); + } + list_for_each(list_itr, &service_table) { + service = list_entry(list_itr, struct service_hndl, list); + if (!service->admin) + continue; + if (service->event_hld(accel_dev, ADF_EVENT_RESTARTED)) + pr_err("QAT: Failed to restart service %s.\n", + service->name); + } + return 0; +} diff --git a/drivers/crypto/qat/qat_common/adf_transport.c b/drivers/crypto/qat/qat_common/adf_transport.c new file mode 100644 index 000000000000..5f3fa45348b4 --- /dev/null +++ b/drivers/crypto/qat/qat_common/adf_transport.c @@ -0,0 +1,567 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#include <linux/delay.h> +#include "adf_accel_devices.h" +#include "adf_transport_internal.h" +#include "adf_transport_access_macros.h" +#include "adf_cfg.h" +#include "adf_common_drv.h" + +static inline uint32_t adf_modulo(uint32_t data, uint32_t shift) +{ + uint32_t div = data >> shift; + uint32_t mult = div << shift; + + return data - mult; +} + +static inline int adf_check_ring_alignment(uint64_t addr, uint64_t size) +{ + if (((size - 1) & addr) != 0) + return -EFAULT; + return 0; +} + +static int adf_verify_ring_size(uint32_t msg_size, uint32_t msg_num) +{ + int i = ADF_MIN_RING_SIZE; + + for (; i <= ADF_MAX_RING_SIZE; i++) + if ((msg_size * msg_num) == ADF_SIZE_TO_RING_SIZE_IN_BYTES(i)) + return i; + + return ADF_DEFAULT_RING_SIZE; +} + +static int adf_reserve_ring(struct adf_etr_bank_data *bank, uint32_t ring) +{ + spin_lock(&bank->lock); + if (bank->ring_mask & (1 << ring)) { + spin_unlock(&bank->lock); + return -EFAULT; + } + bank->ring_mask |= (1 << ring); + spin_unlock(&bank->lock); + return 0; +} + +static void adf_unreserve_ring(struct adf_etr_bank_data *bank, uint32_t ring) +{ + spin_lock(&bank->lock); + bank->ring_mask &= ~(1 << ring); + spin_unlock(&bank->lock); +} + +static void adf_enable_ring_irq(struct adf_etr_bank_data *bank, uint32_t ring) +{ + spin_lock_bh(&bank->lock); + bank->irq_mask |= (1 << ring); + spin_unlock_bh(&bank->lock); + WRITE_CSR_INT_COL_EN(bank->csr_addr, bank->bank_number, bank->irq_mask); + WRITE_CSR_INT_COL_CTL(bank->csr_addr, bank->bank_number, + bank->irq_coalesc_timer); +} + +static void adf_disable_ring_irq(struct adf_etr_bank_data *bank, uint32_t ring) +{ + spin_lock_bh(&bank->lock); + bank->irq_mask &= ~(1 << ring); + spin_unlock_bh(&bank->lock); + WRITE_CSR_INT_COL_EN(bank->csr_addr, bank->bank_number, bank->irq_mask); +} + +int adf_send_message(struct adf_etr_ring_data *ring, uint32_t *msg) +{ + if (atomic_add_return(1, ring->inflights) > + ADF_MAX_INFLIGHTS(ring->ring_size, ring->msg_size)) { + atomic_dec(ring->inflights); + return -EAGAIN; + } + spin_lock_bh(&ring->lock); + memcpy(ring->base_addr + ring->tail, msg, + ADF_MSG_SIZE_TO_BYTES(ring->msg_size)); + + ring->tail = adf_modulo(ring->tail + + ADF_MSG_SIZE_TO_BYTES(ring->msg_size), + ADF_RING_SIZE_MODULO(ring->ring_size)); + WRITE_CSR_RING_TAIL(ring->bank->csr_addr, ring->bank->bank_number, + ring->ring_number, ring->tail); + spin_unlock_bh(&ring->lock); + return 0; +} + +static int adf_handle_response(struct adf_etr_ring_data *ring) +{ + uint32_t msg_counter = 0; + uint32_t *msg = (uint32_t *)(ring->base_addr + ring->head); + + while (*msg != ADF_RING_EMPTY_SIG) { + ring->callback((uint32_t *)msg); + *msg = ADF_RING_EMPTY_SIG; + ring->head = adf_modulo(ring->head + + ADF_MSG_SIZE_TO_BYTES(ring->msg_size), + ADF_RING_SIZE_MODULO(ring->ring_size)); + msg_counter++; + msg = (uint32_t *)(ring->base_addr + ring->head); + } + if (msg_counter > 0) { + WRITE_CSR_RING_HEAD(ring->bank->csr_addr, + ring->bank->bank_number, + ring->ring_number, ring->head); + atomic_sub(msg_counter, ring->inflights); + } + return 0; +} + +static void adf_configure_tx_ring(struct adf_etr_ring_data *ring) +{ + uint32_t ring_config = BUILD_RING_CONFIG(ring->ring_size); + + WRITE_CSR_RING_CONFIG(ring->bank->csr_addr, ring->bank->bank_number, + ring->ring_number, ring_config); +} + +static void adf_configure_rx_ring(struct adf_etr_ring_data *ring) +{ + uint32_t ring_config = + BUILD_RESP_RING_CONFIG(ring->ring_size, + ADF_RING_NEAR_WATERMARK_512, + ADF_RING_NEAR_WATERMARK_0); + + WRITE_CSR_RING_CONFIG(ring->bank->csr_addr, ring->bank->bank_number, + ring->ring_number, ring_config); +} + +static int adf_init_ring(struct adf_etr_ring_data *ring) +{ + struct adf_etr_bank_data *bank = ring->bank; + struct adf_accel_dev *accel_dev = bank->accel_dev; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + uint64_t ring_base; + uint32_t ring_size_bytes = + ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size); + + ring_size_bytes = ADF_RING_SIZE_BYTES_MIN(ring_size_bytes); + ring->base_addr = dma_alloc_coherent(&GET_DEV(accel_dev), + ring_size_bytes, &ring->dma_addr, + GFP_KERNEL); + if (!ring->base_addr) + return -ENOMEM; + + memset(ring->base_addr, 0x7F, ring_size_bytes); + /* The base_addr has to be aligned to the size of the buffer */ + if (adf_check_ring_alignment(ring->dma_addr, ring_size_bytes)) { + pr_err("QAT: Ring address not aligned\n"); + dma_free_coherent(&GET_DEV(accel_dev), ring_size_bytes, + ring->base_addr, ring->dma_addr); + return -EFAULT; + } + + if (hw_data->tx_rings_mask & (1 << ring->ring_number)) + adf_configure_tx_ring(ring); + + else + adf_configure_rx_ring(ring); + + ring_base = BUILD_RING_BASE_ADDR(ring->dma_addr, ring->ring_size); + WRITE_CSR_RING_BASE(ring->bank->csr_addr, ring->bank->bank_number, + ring->ring_number, ring_base); + spin_lock_init(&ring->lock); + return 0; +} + +static void adf_cleanup_ring(struct adf_etr_ring_data *ring) +{ + uint32_t ring_size_bytes = + ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size); + ring_size_bytes = ADF_RING_SIZE_BYTES_MIN(ring_size_bytes); + + if (ring->base_addr) { + memset(ring->base_addr, 0x7F, ring_size_bytes); + dma_free_coherent(&GET_DEV(ring->bank->accel_dev), + ring_size_bytes, ring->base_addr, + ring->dma_addr); + } +} + +int adf_create_ring(struct adf_accel_dev *accel_dev, const char *section, + uint32_t bank_num, uint32_t num_msgs, + uint32_t msg_size, const char *ring_name, + adf_callback_fn callback, int poll_mode, + struct adf_etr_ring_data **ring_ptr) +{ + struct adf_etr_data *transport_data = accel_dev->transport; + struct adf_etr_bank_data *bank; + struct adf_etr_ring_data *ring; + char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; + uint32_t ring_num; + int ret; + + if (bank_num >= GET_MAX_BANKS(accel_dev)) { + pr_err("QAT: Invalid bank number\n"); + return -EFAULT; + } + if (msg_size > ADF_MSG_SIZE_TO_BYTES(ADF_MAX_MSG_SIZE)) { + pr_err("QAT: Invalid msg size\n"); + return -EFAULT; + } + if (ADF_MAX_INFLIGHTS(adf_verify_ring_size(msg_size, num_msgs), + ADF_BYTES_TO_MSG_SIZE(msg_size)) < 2) { + pr_err("QAT: Invalid ring size for given msg size\n"); + return -EFAULT; + } + if (adf_cfg_get_param_value(accel_dev, section, ring_name, val)) { + pr_err("QAT: Section %s, no such entry : %s\n", + section, ring_name); + return -EFAULT; + } + if (kstrtouint(val, 10, &ring_num)) { + pr_err("QAT: Can't get ring number\n"); + return -EFAULT; + } + + bank = &transport_data->banks[bank_num]; + if (adf_reserve_ring(bank, ring_num)) { + pr_err("QAT: Ring %d, %s already exists.\n", + ring_num, ring_name); + return -EFAULT; + } + ring = &bank->rings[ring_num]; + ring->ring_number = ring_num; + ring->bank = bank; + ring->callback = callback; + ring->msg_size = ADF_BYTES_TO_MSG_SIZE(msg_size); + ring->ring_size = adf_verify_ring_size(msg_size, num_msgs); + ring->head = 0; + ring->tail = 0; + atomic_set(ring->inflights, 0); + ret = adf_init_ring(ring); + if (ret) + goto err; + + /* Enable HW arbitration for the given ring */ + accel_dev->hw_device->hw_arb_ring_enable(ring); + + if (adf_ring_debugfs_add(ring, ring_name)) { + pr_err("QAT: Couldn't add ring debugfs entry\n"); + ret = -EFAULT; + goto err; + } + + /* Enable interrupts if needed */ + if (callback && (!poll_mode)) + adf_enable_ring_irq(bank, ring->ring_number); + *ring_ptr = ring; + return 0; +err: + adf_cleanup_ring(ring); + adf_unreserve_ring(bank, ring_num); + accel_dev->hw_device->hw_arb_ring_disable(ring); + return ret; +} + +void adf_remove_ring(struct adf_etr_ring_data *ring) +{ + struct adf_etr_bank_data *bank = ring->bank; + struct adf_accel_dev *accel_dev = bank->accel_dev; + + /* Disable interrupts for the given ring */ + adf_disable_ring_irq(bank, ring->ring_number); + + /* Clear PCI config space */ + WRITE_CSR_RING_CONFIG(bank->csr_addr, bank->bank_number, + ring->ring_number, 0); + WRITE_CSR_RING_BASE(bank->csr_addr, bank->bank_number, + ring->ring_number, 0); + adf_ring_debugfs_rm(ring); + adf_unreserve_ring(bank, ring->ring_number); + /* Disable HW arbitration for the given ring */ + accel_dev->hw_device->hw_arb_ring_disable(ring); + adf_cleanup_ring(ring); +} + +static void adf_ring_response_handler(struct adf_etr_bank_data *bank) +{ + uint32_t empty_rings, i; + + empty_rings = READ_CSR_E_STAT(bank->csr_addr, bank->bank_number); + empty_rings = ~empty_rings & bank->irq_mask; + + for (i = 0; i < ADF_ETR_MAX_RINGS_PER_BANK; ++i) { + if (empty_rings & (1 << i)) + adf_handle_response(&bank->rings[i]); + } +} + +/** + * adf_response_handler() - Bottom half handler response handler + * @bank_addr: Address of a ring bank for with the BH was scheduled. + * + * Function is the bottom half handler for the response from acceleration + * device. There is one handler for every ring bank. Function checks all + * communication rings in the bank. + * To be used by QAT device specific drivers. + * + * Return: void + */ +void adf_response_handler(unsigned long bank_addr) +{ + struct adf_etr_bank_data *bank = (void *)bank_addr; + + /* Handle all the responses nad reenable IRQs */ + adf_ring_response_handler(bank); + WRITE_CSR_INT_FLAG_AND_COL(bank->csr_addr, bank->bank_number, + bank->irq_mask); +} +EXPORT_SYMBOL_GPL(adf_response_handler); + +static inline int adf_get_cfg_int(struct adf_accel_dev *accel_dev, + const char *section, const char *format, + uint32_t key, uint32_t *value) +{ + char key_buf[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; + char val_buf[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; + + snprintf(key_buf, ADF_CFG_MAX_KEY_LEN_IN_BYTES, format, key); + + if (adf_cfg_get_param_value(accel_dev, section, key_buf, val_buf)) + return -EFAULT; + + if (kstrtouint(val_buf, 10, value)) + return -EFAULT; + return 0; +} + +static void adf_enable_coalesc(struct adf_etr_bank_data *bank, + const char *section, uint32_t bank_num_in_accel) +{ + if (adf_get_cfg_int(bank->accel_dev, section, + ADF_ETRMGR_COALESCE_TIMER_FORMAT, + bank_num_in_accel, &bank->irq_coalesc_timer)) + bank->irq_coalesc_timer = ADF_COALESCING_DEF_TIME; + + if (ADF_COALESCING_MAX_TIME < bank->irq_coalesc_timer || + ADF_COALESCING_MIN_TIME > bank->irq_coalesc_timer) + bank->irq_coalesc_timer = ADF_COALESCING_DEF_TIME; +} + +static int adf_init_bank(struct adf_accel_dev *accel_dev, + struct adf_etr_bank_data *bank, + uint32_t bank_num, void __iomem *csr_addr) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_etr_ring_data *ring; + struct adf_etr_ring_data *tx_ring; + uint32_t i, coalesc_enabled; + + memset(bank, 0, sizeof(*bank)); + bank->bank_number = bank_num; + bank->csr_addr = csr_addr; + bank->accel_dev = accel_dev; + spin_lock_init(&bank->lock); + + /* Enable IRQ coalescing always. This will allow to use + * the optimised flag and coalesc register. + * If it is disabled in the config file just use min time value */ + if (adf_get_cfg_int(accel_dev, "Accelerator0", + ADF_ETRMGR_COALESCING_ENABLED_FORMAT, + bank_num, &coalesc_enabled) && coalesc_enabled) + adf_enable_coalesc(bank, "Accelerator0", bank_num); + else + bank->irq_coalesc_timer = ADF_COALESCING_MIN_TIME; + + for (i = 0; i < ADF_ETR_MAX_RINGS_PER_BANK; i++) { + WRITE_CSR_RING_CONFIG(csr_addr, bank_num, i, 0); + WRITE_CSR_RING_BASE(csr_addr, bank_num, i, 0); + ring = &bank->rings[i]; + if (hw_data->tx_rings_mask & (1 << i)) { + ring->inflights = kzalloc_node(sizeof(atomic_t), + GFP_KERNEL, + accel_dev->numa_node); + if (!ring->inflights) + goto err; + } else { + if (i < hw_data->tx_rx_gap) { + pr_err("QAT: Invalid tx rings mask config\n"); + goto err; + } + tx_ring = &bank->rings[i - hw_data->tx_rx_gap]; + ring->inflights = tx_ring->inflights; + } + } + if (adf_bank_debugfs_add(bank)) { + pr_err("QAT: Failed to add bank debugfs entry\n"); + goto err; + } + + WRITE_CSR_INT_SRCSEL(csr_addr, bank_num); + return 0; +err: + for (i = 0; i < ADF_ETR_MAX_RINGS_PER_BANK; i++) { + ring = &bank->rings[i]; + if (hw_data->tx_rings_mask & (1 << i) && ring->inflights) + kfree(ring->inflights); + } + return -ENOMEM; +} + +/** + * adf_init_etr_data() - Initialize transport rings for acceleration device + * @accel_dev: Pointer to acceleration device. + * + * Function is the initializes the communications channels (rings) to the + * acceleration device accel_dev. + * To be used by QAT device specific drivers. + * + * Return: 0 on success, error code othewise. + */ +int adf_init_etr_data(struct adf_accel_dev *accel_dev) +{ + struct adf_etr_data *etr_data; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + void __iomem *csr_addr; + uint32_t size; + uint32_t num_banks = 0; + int i, ret; + + etr_data = kzalloc_node(sizeof(*etr_data), GFP_KERNEL, + accel_dev->numa_node); + if (!etr_data) + return -ENOMEM; + + num_banks = GET_MAX_BANKS(accel_dev); + size = num_banks * sizeof(struct adf_etr_bank_data); + etr_data->banks = kzalloc_node(size, GFP_KERNEL, accel_dev->numa_node); + if (!etr_data->banks) { + ret = -ENOMEM; + goto err_bank; + } + + accel_dev->transport = etr_data; + i = hw_data->get_etr_bar_id(hw_data); + csr_addr = accel_dev->accel_pci_dev.pci_bars[i].virt_addr; + + /* accel_dev->debugfs_dir should always be non-NULL here */ + etr_data->debug = debugfs_create_dir("transport", + accel_dev->debugfs_dir); + if (!etr_data->debug) { + pr_err("QAT: Unable to create transport debugfs entry\n"); + ret = -ENOENT; + goto err_bank_debug; + } + + for (i = 0; i < num_banks; i++) { + ret = adf_init_bank(accel_dev, &etr_data->banks[i], i, + csr_addr); + if (ret) + goto err_bank_all; + } + + return 0; + +err_bank_all: + debugfs_remove(etr_data->debug); +err_bank_debug: + kfree(etr_data->banks); +err_bank: + kfree(etr_data); + accel_dev->transport = NULL; + return ret; +} +EXPORT_SYMBOL_GPL(adf_init_etr_data); + +static void cleanup_bank(struct adf_etr_bank_data *bank) +{ + uint32_t i; + + for (i = 0; i < ADF_ETR_MAX_RINGS_PER_BANK; i++) { + struct adf_accel_dev *accel_dev = bank->accel_dev; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_etr_ring_data *ring = &bank->rings[i]; + + if (bank->ring_mask & (1 << i)) + adf_cleanup_ring(ring); + + if (hw_data->tx_rings_mask & (1 << i)) + kfree(ring->inflights); + } + adf_bank_debugfs_rm(bank); + memset(bank, 0, sizeof(*bank)); +} + +static void adf_cleanup_etr_handles(struct adf_accel_dev *accel_dev) +{ + struct adf_etr_data *etr_data = accel_dev->transport; + uint32_t i, num_banks = GET_MAX_BANKS(accel_dev); + + for (i = 0; i < num_banks; i++) + cleanup_bank(&etr_data->banks[i]); +} + +/** + * adf_cleanup_etr_data() - Clear transport rings for acceleration device + * @accel_dev: Pointer to acceleration device. + * + * Function is the clears the communications channels (rings) of the + * acceleration device accel_dev. + * To be used by QAT device specific drivers. + * + * Return: void + */ +void adf_cleanup_etr_data(struct adf_accel_dev *accel_dev) +{ + struct adf_etr_data *etr_data = accel_dev->transport; + + if (etr_data) { + adf_cleanup_etr_handles(accel_dev); + debugfs_remove(etr_data->debug); + kfree(etr_data->banks); + kfree(etr_data); + accel_dev->transport = NULL; + } +} +EXPORT_SYMBOL_GPL(adf_cleanup_etr_data); diff --git a/drivers/crypto/qat/qat_common/adf_transport.h b/drivers/crypto/qat/qat_common/adf_transport.h new file mode 100644 index 000000000000..386485bd9c95 --- /dev/null +++ b/drivers/crypto/qat/qat_common/adf_transport.h @@ -0,0 +1,63 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef ADF_TRANSPORT_H +#define ADF_TRANSPORT_H + +#include "adf_accel_devices.h" + +struct adf_etr_ring_data; + +typedef void (*adf_callback_fn)(void *resp_msg); + +int adf_create_ring(struct adf_accel_dev *accel_dev, const char *section, + uint32_t bank_num, uint32_t num_mgs, uint32_t msg_size, + const char *ring_name, adf_callback_fn callback, + int poll_mode, struct adf_etr_ring_data **ring_ptr); + +int adf_send_message(struct adf_etr_ring_data *ring, uint32_t *msg); +void adf_remove_ring(struct adf_etr_ring_data *ring); +#endif diff --git a/drivers/crypto/qat/qat_common/adf_transport_access_macros.h b/drivers/crypto/qat/qat_common/adf_transport_access_macros.h new file mode 100644 index 000000000000..91d88d676580 --- /dev/null +++ b/drivers/crypto/qat/qat_common/adf_transport_access_macros.h @@ -0,0 +1,160 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef ADF_TRANSPORT_ACCESS_MACROS_H +#define ADF_TRANSPORT_ACCESS_MACROS_H + +#include "adf_accel_devices.h" +#define ADF_BANK_INT_SRC_SEL_MASK_0 0x4444444CUL +#define ADF_BANK_INT_SRC_SEL_MASK_X 0x44444444UL +#define ADF_RING_CSR_RING_CONFIG 0x000 +#define ADF_RING_CSR_RING_LBASE 0x040 +#define ADF_RING_CSR_RING_UBASE 0x080 +#define ADF_RING_CSR_RING_HEAD 0x0C0 +#define ADF_RING_CSR_RING_TAIL 0x100 +#define ADF_RING_CSR_E_STAT 0x14C +#define ADF_RING_CSR_INT_SRCSEL 0x174 +#define ADF_RING_CSR_INT_SRCSEL_2 0x178 +#define ADF_RING_CSR_INT_COL_EN 0x17C +#define ADF_RING_CSR_INT_COL_CTL 0x180 +#define ADF_RING_CSR_INT_FLAG_AND_COL 0x184 +#define ADF_RING_CSR_INT_COL_CTL_ENABLE 0x80000000 +#define ADF_RING_BUNDLE_SIZE 0x1000 +#define ADF_RING_CONFIG_NEAR_FULL_WM 0x0A +#define ADF_RING_CONFIG_NEAR_EMPTY_WM 0x05 +#define ADF_COALESCING_MIN_TIME 0x1FF +#define ADF_COALESCING_MAX_TIME 0xFFFFF +#define ADF_COALESCING_DEF_TIME 0x27FF +#define ADF_RING_NEAR_WATERMARK_512 0x08 +#define ADF_RING_NEAR_WATERMARK_0 0x00 +#define ADF_RING_EMPTY_SIG 0x7F7F7F7F + +/* Valid internal ring size values */ +#define ADF_RING_SIZE_128 0x01 +#define ADF_RING_SIZE_256 0x02 +#define ADF_RING_SIZE_512 0x03 +#define ADF_RING_SIZE_4K 0x06 +#define ADF_RING_SIZE_16K 0x08 +#define ADF_RING_SIZE_4M 0x10 +#define ADF_MIN_RING_SIZE ADF_RING_SIZE_128 +#define ADF_MAX_RING_SIZE ADF_RING_SIZE_4M +#define ADF_DEFAULT_RING_SIZE ADF_RING_SIZE_16K + +/* Valid internal msg size values internal */ +#define ADF_MSG_SIZE_32 0x01 +#define ADF_MSG_SIZE_64 0x02 +#define ADF_MSG_SIZE_128 0x04 +#define ADF_MIN_MSG_SIZE ADF_MSG_SIZE_32 +#define ADF_MAX_MSG_SIZE ADF_MSG_SIZE_128 + +/* Size to bytes conversion macros for ring and msg values */ +#define ADF_MSG_SIZE_TO_BYTES(SIZE) (SIZE << 5) +#define ADF_BYTES_TO_MSG_SIZE(SIZE) (SIZE >> 5) +#define ADF_SIZE_TO_RING_SIZE_IN_BYTES(SIZE) ((1 << (SIZE - 1)) << 7) +#define ADF_RING_SIZE_IN_BYTES_TO_SIZE(SIZE) ((1 << (SIZE - 1)) >> 7) + +/* Minimum ring bufer size for memory allocation */ +#define ADF_RING_SIZE_BYTES_MIN(SIZE) ((SIZE < ADF_RING_SIZE_4K) ? \ + ADF_RING_SIZE_4K : SIZE) +#define ADF_RING_SIZE_MODULO(SIZE) (SIZE + 0x6) +#define ADF_MAX_INFLIGHTS(RING_SIZE, MSG_SIZE) \ + ((((1 << (RING_SIZE - 1)) << 4) >> MSG_SIZE) - 1) +#define BUILD_RING_CONFIG(size) \ + ((ADF_RING_NEAR_WATERMARK_0 << ADF_RING_CONFIG_NEAR_FULL_WM) \ + | (ADF_RING_NEAR_WATERMARK_0 << ADF_RING_CONFIG_NEAR_EMPTY_WM) \ + | size) +#define BUILD_RESP_RING_CONFIG(size, watermark_nf, watermark_ne) \ + ((watermark_nf << ADF_RING_CONFIG_NEAR_FULL_WM) \ + | (watermark_ne << ADF_RING_CONFIG_NEAR_EMPTY_WM) \ + | size) +#define BUILD_RING_BASE_ADDR(addr, size) \ + ((addr >> 6) & (0xFFFFFFFFFFFFFFFFULL << size)) +#define READ_CSR_RING_HEAD(csr_base_addr, bank, ring) \ + ADF_CSR_RD(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ + ADF_RING_CSR_RING_HEAD + (ring << 2)) +#define READ_CSR_RING_TAIL(csr_base_addr, bank, ring) \ + ADF_CSR_RD(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ + ADF_RING_CSR_RING_TAIL + (ring << 2)) +#define READ_CSR_E_STAT(csr_base_addr, bank) \ + ADF_CSR_RD(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ + ADF_RING_CSR_E_STAT) +#define WRITE_CSR_RING_CONFIG(csr_base_addr, bank, ring, value) \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ + ADF_RING_CSR_RING_CONFIG + (ring << 2), value) +#define WRITE_CSR_RING_BASE(csr_base_addr, bank, ring, value) \ +do { \ + uint32_t l_base = 0, u_base = 0; \ + l_base = (uint32_t)(value & 0xFFFFFFFF); \ + u_base = (uint32_t)((value & 0xFFFFFFFF00000000ULL) >> 32); \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ + ADF_RING_CSR_RING_LBASE + (ring << 2), l_base); \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ + ADF_RING_CSR_RING_UBASE + (ring << 2), u_base); \ +} while (0) +#define WRITE_CSR_RING_HEAD(csr_base_addr, bank, ring, value) \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ + ADF_RING_CSR_RING_HEAD + (ring << 2), value) +#define WRITE_CSR_RING_TAIL(csr_base_addr, bank, ring, value) \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ + ADF_RING_CSR_RING_TAIL + (ring << 2), value) +#define WRITE_CSR_INT_SRCSEL(csr_base_addr, bank) \ +do { \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ + ADF_RING_CSR_INT_SRCSEL, ADF_BANK_INT_SRC_SEL_MASK_0); \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ + ADF_RING_CSR_INT_SRCSEL_2, ADF_BANK_INT_SRC_SEL_MASK_X); \ +} while (0) +#define WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value) \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ + ADF_RING_CSR_INT_COL_EN, value) +#define WRITE_CSR_INT_COL_CTL(csr_base_addr, bank, value) \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ + ADF_RING_CSR_INT_COL_CTL, \ + ADF_RING_CSR_INT_COL_CTL_ENABLE | value) +#define WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value) \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ + ADF_RING_CSR_INT_FLAG_AND_COL, value) +#endif diff --git a/drivers/crypto/qat/qat_common/adf_transport_debug.c b/drivers/crypto/qat/qat_common/adf_transport_debug.c new file mode 100644 index 000000000000..6b6974553514 --- /dev/null +++ b/drivers/crypto/qat/qat_common/adf_transport_debug.c @@ -0,0 +1,304 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#include <linux/mutex.h> +#include <linux/slab.h> +#include <linux/seq_file.h> +#include "adf_accel_devices.h" +#include "adf_transport_internal.h" +#include "adf_transport_access_macros.h" + +static DEFINE_MUTEX(ring_read_lock); +static DEFINE_MUTEX(bank_read_lock); + +static void *adf_ring_start(struct seq_file *sfile, loff_t *pos) +{ + struct adf_etr_ring_data *ring = sfile->private; + + mutex_lock(&ring_read_lock); + if (*pos == 0) + return SEQ_START_TOKEN; + + if (*pos >= (ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size) / + ADF_MSG_SIZE_TO_BYTES(ring->msg_size))) + return NULL; + + return ring->base_addr + + (ADF_MSG_SIZE_TO_BYTES(ring->msg_size) * (*pos)++); +} + +static void *adf_ring_next(struct seq_file *sfile, void *v, loff_t *pos) +{ + struct adf_etr_ring_data *ring = sfile->private; + + if (*pos >= (ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size) / + ADF_MSG_SIZE_TO_BYTES(ring->msg_size))) + return NULL; + + return ring->base_addr + + (ADF_MSG_SIZE_TO_BYTES(ring->msg_size) * (*pos)++); +} + +static int adf_ring_show(struct seq_file *sfile, void *v) +{ + struct adf_etr_ring_data *ring = sfile->private; + struct adf_etr_bank_data *bank = ring->bank; + uint32_t *msg = v; + void __iomem *csr = ring->bank->csr_addr; + int i, x; + + if (v == SEQ_START_TOKEN) { + int head, tail, empty; + + head = READ_CSR_RING_HEAD(csr, bank->bank_number, + ring->ring_number); + tail = READ_CSR_RING_TAIL(csr, bank->bank_number, + ring->ring_number); + empty = READ_CSR_E_STAT(csr, bank->bank_number); + + seq_puts(sfile, "------- Ring configuration -------\n"); + seq_printf(sfile, "ring num %d, bank num %d\n", + ring->ring_number, ring->bank->bank_number); + seq_printf(sfile, "head %x, tail %x, empty: %d\n", + head, tail, (empty & 1 << ring->ring_number) + >> ring->ring_number); + seq_printf(sfile, "ring size %d, msg size %d\n", + ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size), + ADF_MSG_SIZE_TO_BYTES(ring->msg_size)); + seq_puts(sfile, "----------- Ring data ------------\n"); + return 0; + } + seq_printf(sfile, "%p:", msg); + x = 0; + i = 0; + for (; i < (ADF_MSG_SIZE_TO_BYTES(ring->msg_size) >> 2); i++) { + seq_printf(sfile, " %08X", *(msg + i)); + if ((ADF_MSG_SIZE_TO_BYTES(ring->msg_size) >> 2) != i + 1 && + (++x == 8)) { + seq_printf(sfile, "\n%p:", msg + i + 1); + x = 0; + } + } + seq_puts(sfile, "\n"); + return 0; +} + +static void adf_ring_stop(struct seq_file *sfile, void *v) +{ + mutex_unlock(&ring_read_lock); +} + +static const struct seq_operations adf_ring_sops = { + .start = adf_ring_start, + .next = adf_ring_next, + .stop = adf_ring_stop, + .show = adf_ring_show +}; + +static int adf_ring_open(struct inode *inode, struct file *file) +{ + int ret = seq_open(file, &adf_ring_sops); + + if (!ret) { + struct seq_file *seq_f = file->private_data; + + seq_f->private = inode->i_private; + } + return ret; +} + +static const struct file_operations adf_ring_debug_fops = { + .open = adf_ring_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release +}; + +int adf_ring_debugfs_add(struct adf_etr_ring_data *ring, const char *name) +{ + struct adf_etr_ring_debug_entry *ring_debug; + char entry_name[8]; + + ring_debug = kzalloc(sizeof(*ring_debug), GFP_KERNEL); + if (!ring_debug) + return -ENOMEM; + + strlcpy(ring_debug->ring_name, name, sizeof(ring_debug->ring_name)); + snprintf(entry_name, sizeof(entry_name), "ring_%02d", + ring->ring_number); + + ring_debug->debug = debugfs_create_file(entry_name, S_IRUSR, + ring->bank->bank_debug_dir, + ring, &adf_ring_debug_fops); + if (!ring_debug->debug) { + pr_err("QAT: Failed to create ring debug entry.\n"); + kfree(ring_debug); + return -EFAULT; + } + ring->ring_debug = ring_debug; + return 0; +} + +void adf_ring_debugfs_rm(struct adf_etr_ring_data *ring) +{ + if (ring->ring_debug) { + debugfs_remove(ring->ring_debug->debug); + kfree(ring->ring_debug); + ring->ring_debug = NULL; + } +} + +static void *adf_bank_start(struct seq_file *sfile, loff_t *pos) +{ + mutex_lock(&bank_read_lock); + if (*pos == 0) + return SEQ_START_TOKEN; + + if (*pos >= ADF_ETR_MAX_RINGS_PER_BANK) + return NULL; + + return pos; +} + +static void *adf_bank_next(struct seq_file *sfile, void *v, loff_t *pos) +{ + if (++(*pos) >= ADF_ETR_MAX_RINGS_PER_BANK) + return NULL; + + return pos; +} + +static int adf_bank_show(struct seq_file *sfile, void *v) +{ + struct adf_etr_bank_data *bank = sfile->private; + + if (v == SEQ_START_TOKEN) { + seq_printf(sfile, "------- Bank %d configuration -------\n", + bank->bank_number); + } else { + int ring_id = *((int *)v) - 1; + struct adf_etr_ring_data *ring = &bank->rings[ring_id]; + void __iomem *csr = bank->csr_addr; + int head, tail, empty; + + if (!(bank->ring_mask & 1 << ring_id)) + return 0; + + head = READ_CSR_RING_HEAD(csr, bank->bank_number, + ring->ring_number); + tail = READ_CSR_RING_TAIL(csr, bank->bank_number, + ring->ring_number); + empty = READ_CSR_E_STAT(csr, bank->bank_number); + + seq_printf(sfile, + "ring num %02d, head %04x, tail %04x, empty: %d\n", + ring->ring_number, head, tail, + (empty & 1 << ring->ring_number) >> + ring->ring_number); + } + return 0; +} + +static void adf_bank_stop(struct seq_file *sfile, void *v) +{ + mutex_unlock(&bank_read_lock); +} + +static const struct seq_operations adf_bank_sops = { + .start = adf_bank_start, + .next = adf_bank_next, + .stop = adf_bank_stop, + .show = adf_bank_show +}; + +static int adf_bank_open(struct inode *inode, struct file *file) +{ + int ret = seq_open(file, &adf_bank_sops); + + if (!ret) { + struct seq_file *seq_f = file->private_data; + + seq_f->private = inode->i_private; + } + return ret; +} + +static const struct file_operations adf_bank_debug_fops = { + .open = adf_bank_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release +}; + +int adf_bank_debugfs_add(struct adf_etr_bank_data *bank) +{ + struct adf_accel_dev *accel_dev = bank->accel_dev; + struct dentry *parent = accel_dev->transport->debug; + char name[8]; + + snprintf(name, sizeof(name), "bank_%02d", bank->bank_number); + bank->bank_debug_dir = debugfs_create_dir(name, parent); + if (!bank->bank_debug_dir) { + pr_err("QAT: Failed to create bank debug dir.\n"); + return -EFAULT; + } + + bank->bank_debug_cfg = debugfs_create_file("config", S_IRUSR, + bank->bank_debug_dir, bank, + &adf_bank_debug_fops); + if (!bank->bank_debug_cfg) { + pr_err("QAT: Failed to create bank debug entry.\n"); + debugfs_remove(bank->bank_debug_dir); + return -EFAULT; + } + return 0; +} + +void adf_bank_debugfs_rm(struct adf_etr_bank_data *bank) +{ + debugfs_remove(bank->bank_debug_cfg); + debugfs_remove(bank->bank_debug_dir); +} diff --git a/drivers/crypto/qat/qat_common/adf_transport_internal.h b/drivers/crypto/qat/qat_common/adf_transport_internal.h new file mode 100644 index 000000000000..f854bac276b0 --- /dev/null +++ b/drivers/crypto/qat/qat_common/adf_transport_internal.h @@ -0,0 +1,118 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef ADF_TRANSPORT_INTRN_H +#define ADF_TRANSPORT_INTRN_H + +#include <linux/interrupt.h> +#include <linux/atomic.h> +#include <linux/spinlock_types.h> +#include "adf_transport.h" + +struct adf_etr_ring_debug_entry { + char ring_name[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; + struct dentry *debug; +}; + +struct adf_etr_ring_data { + void *base_addr; + atomic_t *inflights; + spinlock_t lock; /* protects ring data struct */ + adf_callback_fn callback; + struct adf_etr_bank_data *bank; + dma_addr_t dma_addr; + uint16_t head; + uint16_t tail; + uint8_t ring_number; + uint8_t ring_size; + uint8_t msg_size; + uint8_t reserved; + struct adf_etr_ring_debug_entry *ring_debug; +} __packed; + +struct adf_etr_bank_data { + struct adf_etr_ring_data rings[ADF_ETR_MAX_RINGS_PER_BANK]; + struct tasklet_struct resp_hanlder; + void __iomem *csr_addr; + struct adf_accel_dev *accel_dev; + uint32_t irq_coalesc_timer; + uint16_t ring_mask; + uint16_t irq_mask; + spinlock_t lock; /* protects bank data struct */ + struct dentry *bank_debug_dir; + struct dentry *bank_debug_cfg; + uint32_t bank_number; +} __packed; + +struct adf_etr_data { + struct adf_etr_bank_data *banks; + struct dentry *debug; +}; + +void adf_response_handler(unsigned long bank_addr); +#ifdef CONFIG_DEBUG_FS +#include <linux/debugfs.h> +int adf_bank_debugfs_add(struct adf_etr_bank_data *bank); +void adf_bank_debugfs_rm(struct adf_etr_bank_data *bank); +int adf_ring_debugfs_add(struct adf_etr_ring_data *ring, const char *name); +void adf_ring_debugfs_rm(struct adf_etr_ring_data *ring); +#else +static inline int adf_bank_debugfs_add(struct adf_etr_bank_data *bank) +{ + return 0; +} + +#define adf_bank_debugfs_rm(bank) do {} while (0) + +static inline int adf_ring_debugfs_add(struct adf_etr_ring_data *ring, + const char *name) +{ + return 0; +} + +#define adf_ring_debugfs_rm(ring) do {} while (0) +#endif +#endif diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw.h b/drivers/crypto/qat/qat_common/icp_qat_fw.h new file mode 100644 index 000000000000..f1e30e24a419 --- /dev/null +++ b/drivers/crypto/qat/qat_common/icp_qat_fw.h @@ -0,0 +1,316 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef _ICP_QAT_FW_H_ +#define _ICP_QAT_FW_H_ +#include <linux/types.h> +#include "icp_qat_hw.h" + +#define QAT_FIELD_SET(flags, val, bitpos, mask) \ +{ (flags) = (((flags) & (~((mask) << (bitpos)))) | \ + (((val) & (mask)) << (bitpos))) ; } + +#define QAT_FIELD_GET(flags, bitpos, mask) \ + (((flags) >> (bitpos)) & (mask)) + +#define ICP_QAT_FW_REQ_DEFAULT_SZ 128 +#define ICP_QAT_FW_RESP_DEFAULT_SZ 32 +#define ICP_QAT_FW_COMN_ONE_BYTE_SHIFT 8 +#define ICP_QAT_FW_COMN_SINGLE_BYTE_MASK 0xFF +#define ICP_QAT_FW_NUM_LONGWORDS_1 1 +#define ICP_QAT_FW_NUM_LONGWORDS_2 2 +#define ICP_QAT_FW_NUM_LONGWORDS_3 3 +#define ICP_QAT_FW_NUM_LONGWORDS_4 4 +#define ICP_QAT_FW_NUM_LONGWORDS_5 5 +#define ICP_QAT_FW_NUM_LONGWORDS_6 6 +#define ICP_QAT_FW_NUM_LONGWORDS_7 7 +#define ICP_QAT_FW_NUM_LONGWORDS_10 10 +#define ICP_QAT_FW_NUM_LONGWORDS_13 13 +#define ICP_QAT_FW_NULL_REQ_SERV_ID 1 + +enum icp_qat_fw_comn_resp_serv_id { + ICP_QAT_FW_COMN_RESP_SERV_NULL, + ICP_QAT_FW_COMN_RESP_SERV_CPM_FW, + ICP_QAT_FW_COMN_RESP_SERV_DELIMITER +}; + +enum icp_qat_fw_comn_request_id { + ICP_QAT_FW_COMN_REQ_NULL = 0, + ICP_QAT_FW_COMN_REQ_CPM_FW_PKE = 3, + ICP_QAT_FW_COMN_REQ_CPM_FW_LA = 4, + ICP_QAT_FW_COMN_REQ_CPM_FW_DMA = 7, + ICP_QAT_FW_COMN_REQ_CPM_FW_COMP = 9, + ICP_QAT_FW_COMN_REQ_DELIMITER +}; + +struct icp_qat_fw_comn_req_hdr_cd_pars { + union { + struct { + uint64_t content_desc_addr; + uint16_t content_desc_resrvd1; + uint8_t content_desc_params_sz; + uint8_t content_desc_hdr_resrvd2; + uint32_t content_desc_resrvd3; + } s; + struct { + uint32_t serv_specif_fields[4]; + } s1; + } u; +}; + +struct icp_qat_fw_comn_req_mid { + uint64_t opaque_data; + uint64_t src_data_addr; + uint64_t dest_data_addr; + uint32_t src_length; + uint32_t dst_length; +}; + +struct icp_qat_fw_comn_req_cd_ctrl { + uint32_t content_desc_ctrl_lw[ICP_QAT_FW_NUM_LONGWORDS_5]; +}; + +struct icp_qat_fw_comn_req_hdr { + uint8_t resrvd1; + uint8_t service_cmd_id; + uint8_t service_type; + uint8_t hdr_flags; + uint16_t serv_specif_flags; + uint16_t comn_req_flags; +}; + +struct icp_qat_fw_comn_req_rqpars { + uint32_t serv_specif_rqpars_lw[ICP_QAT_FW_NUM_LONGWORDS_13]; +}; + +struct icp_qat_fw_comn_req { + struct icp_qat_fw_comn_req_hdr comn_hdr; + struct icp_qat_fw_comn_req_hdr_cd_pars cd_pars; + struct icp_qat_fw_comn_req_mid comn_mid; + struct icp_qat_fw_comn_req_rqpars serv_specif_rqpars; + struct icp_qat_fw_comn_req_cd_ctrl cd_ctrl; +}; + +struct icp_qat_fw_comn_error { + uint8_t xlat_err_code; + uint8_t cmp_err_code; +}; + +struct icp_qat_fw_comn_resp_hdr { + uint8_t resrvd1; + uint8_t service_id; + uint8_t response_type; + uint8_t hdr_flags; + struct icp_qat_fw_comn_error comn_error; + uint8_t comn_status; + uint8_t cmd_id; +}; + +struct icp_qat_fw_comn_resp { + struct icp_qat_fw_comn_resp_hdr comn_hdr; + uint64_t opaque_data; + uint32_t resrvd[ICP_QAT_FW_NUM_LONGWORDS_4]; +}; + +#define ICP_QAT_FW_COMN_REQ_FLAG_SET 1 +#define ICP_QAT_FW_COMN_REQ_FLAG_CLR 0 +#define ICP_QAT_FW_COMN_VALID_FLAG_BITPOS 7 +#define ICP_QAT_FW_COMN_VALID_FLAG_MASK 0x1 +#define ICP_QAT_FW_COMN_HDR_RESRVD_FLD_MASK 0x7F + +#define ICP_QAT_FW_COMN_OV_SRV_TYPE_GET(icp_qat_fw_comn_req_hdr_t) \ + icp_qat_fw_comn_req_hdr_t.service_type + +#define ICP_QAT_FW_COMN_OV_SRV_TYPE_SET(icp_qat_fw_comn_req_hdr_t, val) \ + icp_qat_fw_comn_req_hdr_t.service_type = val + +#define ICP_QAT_FW_COMN_OV_SRV_CMD_ID_GET(icp_qat_fw_comn_req_hdr_t) \ + icp_qat_fw_comn_req_hdr_t.service_cmd_id + +#define ICP_QAT_FW_COMN_OV_SRV_CMD_ID_SET(icp_qat_fw_comn_req_hdr_t, val) \ + icp_qat_fw_comn_req_hdr_t.service_cmd_id = val + +#define ICP_QAT_FW_COMN_HDR_VALID_FLAG_GET(hdr_t) \ + ICP_QAT_FW_COMN_VALID_FLAG_GET(hdr_t.hdr_flags) + +#define ICP_QAT_FW_COMN_HDR_VALID_FLAG_SET(hdr_t, val) \ + ICP_QAT_FW_COMN_VALID_FLAG_SET(hdr_t, val) + +#define ICP_QAT_FW_COMN_VALID_FLAG_GET(hdr_flags) \ + QAT_FIELD_GET(hdr_flags, \ + ICP_QAT_FW_COMN_VALID_FLAG_BITPOS, \ + ICP_QAT_FW_COMN_VALID_FLAG_MASK) + +#define ICP_QAT_FW_COMN_HDR_RESRVD_FLD_GET(hdr_flags) \ + (hdr_flags & ICP_QAT_FW_COMN_HDR_RESRVD_FLD_MASK) + +#define ICP_QAT_FW_COMN_VALID_FLAG_SET(hdr_t, val) \ + QAT_FIELD_SET((hdr_t.hdr_flags), (val), \ + ICP_QAT_FW_COMN_VALID_FLAG_BITPOS, \ + ICP_QAT_FW_COMN_VALID_FLAG_MASK) + +#define ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(valid) \ + (((valid) & ICP_QAT_FW_COMN_VALID_FLAG_MASK) << \ + ICP_QAT_FW_COMN_VALID_FLAG_BITPOS) + +#define QAT_COMN_PTR_TYPE_BITPOS 0 +#define QAT_COMN_PTR_TYPE_MASK 0x1 +#define QAT_COMN_CD_FLD_TYPE_BITPOS 1 +#define QAT_COMN_CD_FLD_TYPE_MASK 0x1 +#define QAT_COMN_PTR_TYPE_FLAT 0x0 +#define QAT_COMN_PTR_TYPE_SGL 0x1 +#define QAT_COMN_CD_FLD_TYPE_64BIT_ADR 0x0 +#define QAT_COMN_CD_FLD_TYPE_16BYTE_DATA 0x1 + +#define ICP_QAT_FW_COMN_FLAGS_BUILD(cdt, ptr) \ + ((((cdt) & QAT_COMN_CD_FLD_TYPE_MASK) << QAT_COMN_CD_FLD_TYPE_BITPOS) \ + | (((ptr) & QAT_COMN_PTR_TYPE_MASK) << QAT_COMN_PTR_TYPE_BITPOS)) + +#define ICP_QAT_FW_COMN_PTR_TYPE_GET(flags) \ + QAT_FIELD_GET(flags, QAT_COMN_PTR_TYPE_BITPOS, QAT_COMN_PTR_TYPE_MASK) + +#define ICP_QAT_FW_COMN_CD_FLD_TYPE_GET(flags) \ + QAT_FIELD_GET(flags, QAT_COMN_CD_FLD_TYPE_BITPOS, \ + QAT_COMN_CD_FLD_TYPE_MASK) + +#define ICP_QAT_FW_COMN_PTR_TYPE_SET(flags, val) \ + QAT_FIELD_SET(flags, val, QAT_COMN_PTR_TYPE_BITPOS, \ + QAT_COMN_PTR_TYPE_MASK) + +#define ICP_QAT_FW_COMN_CD_FLD_TYPE_SET(flags, val) \ + QAT_FIELD_SET(flags, val, QAT_COMN_CD_FLD_TYPE_BITPOS, \ + QAT_COMN_CD_FLD_TYPE_MASK) + +#define ICP_QAT_FW_COMN_NEXT_ID_BITPOS 4 +#define ICP_QAT_FW_COMN_NEXT_ID_MASK 0xF0 +#define ICP_QAT_FW_COMN_CURR_ID_BITPOS 0 +#define ICP_QAT_FW_COMN_CURR_ID_MASK 0x0F + +#define ICP_QAT_FW_COMN_NEXT_ID_GET(cd_ctrl_hdr_t) \ + ((((cd_ctrl_hdr_t)->next_curr_id) & ICP_QAT_FW_COMN_NEXT_ID_MASK) \ + >> (ICP_QAT_FW_COMN_NEXT_ID_BITPOS)) + +#define ICP_QAT_FW_COMN_NEXT_ID_SET(cd_ctrl_hdr_t, val) \ + { ((cd_ctrl_hdr_t)->next_curr_id) = ((((cd_ctrl_hdr_t)->next_curr_id) \ + & ICP_QAT_FW_COMN_CURR_ID_MASK) | \ + ((val << ICP_QAT_FW_COMN_NEXT_ID_BITPOS) \ + & ICP_QAT_FW_COMN_NEXT_ID_MASK)); } + +#define ICP_QAT_FW_COMN_CURR_ID_GET(cd_ctrl_hdr_t) \ + (((cd_ctrl_hdr_t)->next_curr_id) & ICP_QAT_FW_COMN_CURR_ID_MASK) + +#define ICP_QAT_FW_COMN_CURR_ID_SET(cd_ctrl_hdr_t, val) \ + { ((cd_ctrl_hdr_t)->next_curr_id) = ((((cd_ctrl_hdr_t)->next_curr_id) \ + & ICP_QAT_FW_COMN_NEXT_ID_MASK) | \ + ((val) & ICP_QAT_FW_COMN_CURR_ID_MASK)); } + +#define QAT_COMN_RESP_CRYPTO_STATUS_BITPOS 7 +#define QAT_COMN_RESP_CRYPTO_STATUS_MASK 0x1 +#define QAT_COMN_RESP_CMP_STATUS_BITPOS 5 +#define QAT_COMN_RESP_CMP_STATUS_MASK 0x1 +#define QAT_COMN_RESP_XLAT_STATUS_BITPOS 4 +#define QAT_COMN_RESP_XLAT_STATUS_MASK 0x1 +#define QAT_COMN_RESP_CMP_END_OF_LAST_BLK_BITPOS 3 +#define QAT_COMN_RESP_CMP_END_OF_LAST_BLK_MASK 0x1 + +#define ICP_QAT_FW_COMN_RESP_STATUS_BUILD(crypto, comp, xlat, eolb) \ + ((((crypto) & QAT_COMN_RESP_CRYPTO_STATUS_MASK) << \ + QAT_COMN_RESP_CRYPTO_STATUS_BITPOS) | \ + (((comp) & QAT_COMN_RESP_CMP_STATUS_MASK) << \ + QAT_COMN_RESP_CMP_STATUS_BITPOS) | \ + (((xlat) & QAT_COMN_RESP_XLAT_STATUS_MASK) << \ + QAT_COMN_RESP_XLAT_STATUS_BITPOS) | \ + (((eolb) & QAT_COMN_RESP_CMP_END_OF_LAST_BLK_MASK) << \ + QAT_COMN_RESP_CMP_END_OF_LAST_BLK_BITPOS)) + +#define ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(status) \ + QAT_FIELD_GET(status, QAT_COMN_RESP_CRYPTO_STATUS_BITPOS, \ + QAT_COMN_RESP_CRYPTO_STATUS_MASK) + +#define ICP_QAT_FW_COMN_RESP_CMP_STAT_GET(status) \ + QAT_FIELD_GET(status, QAT_COMN_RESP_CMP_STATUS_BITPOS, \ + QAT_COMN_RESP_CMP_STATUS_MASK) + +#define ICP_QAT_FW_COMN_RESP_XLAT_STAT_GET(status) \ + QAT_FIELD_GET(status, QAT_COMN_RESP_XLAT_STATUS_BITPOS, \ + QAT_COMN_RESP_XLAT_STATUS_MASK) + +#define ICP_QAT_FW_COMN_RESP_CMP_END_OF_LAST_BLK_FLAG_GET(status) \ + QAT_FIELD_GET(status, QAT_COMN_RESP_CMP_END_OF_LAST_BLK_BITPOS, \ + QAT_COMN_RESP_CMP_END_OF_LAST_BLK_MASK) + +#define ICP_QAT_FW_COMN_STATUS_FLAG_OK 0 +#define ICP_QAT_FW_COMN_STATUS_FLAG_ERROR 1 +#define ICP_QAT_FW_COMN_STATUS_CMP_END_OF_LAST_BLK_FLAG_CLR 0 +#define ICP_QAT_FW_COMN_STATUS_CMP_END_OF_LAST_BLK_FLAG_SET 1 +#define ERR_CODE_NO_ERROR 0 +#define ERR_CODE_INVALID_BLOCK_TYPE -1 +#define ERR_CODE_NO_MATCH_ONES_COMP -2 +#define ERR_CODE_TOO_MANY_LEN_OR_DIS -3 +#define ERR_CODE_INCOMPLETE_LEN -4 +#define ERR_CODE_RPT_LEN_NO_FIRST_LEN -5 +#define ERR_CODE_RPT_GT_SPEC_LEN -6 +#define ERR_CODE_INV_LIT_LEN_CODE_LEN -7 +#define ERR_CODE_INV_DIS_CODE_LEN -8 +#define ERR_CODE_INV_LIT_LEN_DIS_IN_BLK -9 +#define ERR_CODE_DIS_TOO_FAR_BACK -10 +#define ERR_CODE_OVERFLOW_ERROR -11 +#define ERR_CODE_SOFT_ERROR -12 +#define ERR_CODE_FATAL_ERROR -13 +#define ERR_CODE_SSM_ERROR -14 +#define ERR_CODE_ENDPOINT_ERROR -15 + +enum icp_qat_fw_slice { + ICP_QAT_FW_SLICE_NULL = 0, + ICP_QAT_FW_SLICE_CIPHER = 1, + ICP_QAT_FW_SLICE_AUTH = 2, + ICP_QAT_FW_SLICE_DRAM_RD = 3, + ICP_QAT_FW_SLICE_DRAM_WR = 4, + ICP_QAT_FW_SLICE_COMP = 5, + ICP_QAT_FW_SLICE_XLAT = 6, + ICP_QAT_FW_SLICE_DELIMITER +}; +#endif diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_init_admin.h b/drivers/crypto/qat/qat_common/icp_qat_fw_init_admin.h new file mode 100644 index 000000000000..72a59faa9005 --- /dev/null +++ b/drivers/crypto/qat/qat_common/icp_qat_fw_init_admin.h @@ -0,0 +1,131 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef _ICP_QAT_FW_INIT_ADMIN_H_ +#define _ICP_QAT_FW_INIT_ADMIN_H_ + +#include "icp_qat_fw.h" + +enum icp_qat_fw_init_admin_cmd_id { + ICP_QAT_FW_INIT_ME = 0, + ICP_QAT_FW_TRNG_ENABLE = 1, + ICP_QAT_FW_TRNG_DISABLE = 2, + ICP_QAT_FW_CONSTANTS_CFG = 3, + ICP_QAT_FW_STATUS_GET = 4, + ICP_QAT_FW_COUNTERS_GET = 5, + ICP_QAT_FW_LOOPBACK = 6, + ICP_QAT_FW_HEARTBEAT_SYNC = 7, + ICP_QAT_FW_HEARTBEAT_GET = 8 +}; + +enum icp_qat_fw_init_admin_resp_status { + ICP_QAT_FW_INIT_RESP_STATUS_SUCCESS = 0, + ICP_QAT_FW_INIT_RESP_STATUS_FAIL +}; + +struct icp_qat_fw_init_admin_req { + uint16_t init_cfg_sz; + uint8_t resrvd1; + uint8_t init_admin_cmd_id; + uint32_t resrvd2; + uint64_t opaque_data; + uint64_t init_cfg_ptr; + uint64_t resrvd3; +}; + +struct icp_qat_fw_init_admin_resp_hdr { + uint8_t flags; + uint8_t resrvd1; + uint8_t status; + uint8_t init_admin_cmd_id; +}; + +struct icp_qat_fw_init_admin_resp_pars { + union { + uint32_t resrvd1[ICP_QAT_FW_NUM_LONGWORDS_4]; + struct { + uint32_t version_patch_num; + uint8_t context_id; + uint8_t ae_id; + uint16_t resrvd1; + uint64_t resrvd2; + } s1; + struct { + uint64_t req_rec_count; + uint64_t resp_sent_count; + } s2; + } u; +}; + +struct icp_qat_fw_init_admin_resp { + struct icp_qat_fw_init_admin_resp_hdr init_resp_hdr; + union { + uint32_t resrvd2; + struct { + uint16_t version_minor_num; + uint16_t version_major_num; + } s; + } u; + uint64_t opaque_data; + struct icp_qat_fw_init_admin_resp_pars init_resp_pars; +}; + +#define ICP_QAT_FW_COMN_HEARTBEAT_OK 0 +#define ICP_QAT_FW_COMN_HEARTBEAT_BLOCKED 1 +#define ICP_QAT_FW_COMN_HEARTBEAT_FLAG_BITPOS 0 +#define ICP_QAT_FW_COMN_HEARTBEAT_FLAG_MASK 0x1 +#define ICP_QAT_FW_COMN_STATUS_RESRVD_FLD_MASK 0xFE +#define ICP_QAT_FW_COMN_HEARTBEAT_HDR_FLAG_GET(hdr_t) \ + ICP_QAT_FW_COMN_HEARTBEAT_FLAG_GET(hdr_t.flags) + +#define ICP_QAT_FW_COMN_HEARTBEAT_HDR_FLAG_SET(hdr_t, val) \ + ICP_QAT_FW_COMN_HEARTBEAT_FLAG_SET(hdr_t, val) + +#define ICP_QAT_FW_COMN_HEARTBEAT_FLAG_GET(flags) \ + QAT_FIELD_GET(flags, \ + ICP_QAT_FW_COMN_HEARTBEAT_FLAG_BITPOS, \ + ICP_QAT_FW_COMN_HEARTBEAT_FLAG_MASK) +#endif diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_la.h b/drivers/crypto/qat/qat_common/icp_qat_fw_la.h new file mode 100644 index 000000000000..c8d26697e8ea --- /dev/null +++ b/drivers/crypto/qat/qat_common/icp_qat_fw_la.h @@ -0,0 +1,404 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef _ICP_QAT_FW_LA_H_ +#define _ICP_QAT_FW_LA_H_ +#include "icp_qat_fw.h" + +enum icp_qat_fw_la_cmd_id { + ICP_QAT_FW_LA_CMD_CIPHER = 0, + ICP_QAT_FW_LA_CMD_AUTH = 1, + ICP_QAT_FW_LA_CMD_CIPHER_HASH = 2, + ICP_QAT_FW_LA_CMD_HASH_CIPHER = 3, + ICP_QAT_FW_LA_CMD_TRNG_GET_RANDOM = 4, + ICP_QAT_FW_LA_CMD_TRNG_TEST = 5, + ICP_QAT_FW_LA_CMD_SSL3_KEY_DERIVE = 6, + ICP_QAT_FW_LA_CMD_TLS_V1_1_KEY_DERIVE = 7, + ICP_QAT_FW_LA_CMD_TLS_V1_2_KEY_DERIVE = 8, + ICP_QAT_FW_LA_CMD_MGF1 = 9, + ICP_QAT_FW_LA_CMD_AUTH_PRE_COMP = 10, + ICP_QAT_FW_LA_CMD_CIPHER_PRE_COMP = 11, + ICP_QAT_FW_LA_CMD_DELIMITER = 12 +}; + +#define ICP_QAT_FW_LA_ICV_VER_STATUS_PASS ICP_QAT_FW_COMN_STATUS_FLAG_OK +#define ICP_QAT_FW_LA_ICV_VER_STATUS_FAIL ICP_QAT_FW_COMN_STATUS_FLAG_ERROR +#define ICP_QAT_FW_LA_TRNG_STATUS_PASS ICP_QAT_FW_COMN_STATUS_FLAG_OK +#define ICP_QAT_FW_LA_TRNG_STATUS_FAIL ICP_QAT_FW_COMN_STATUS_FLAG_ERROR + +struct icp_qat_fw_la_bulk_req { + struct icp_qat_fw_comn_req_hdr comn_hdr; + struct icp_qat_fw_comn_req_hdr_cd_pars cd_pars; + struct icp_qat_fw_comn_req_mid comn_mid; + struct icp_qat_fw_comn_req_rqpars serv_specif_rqpars; + struct icp_qat_fw_comn_req_cd_ctrl cd_ctrl; +}; + +#define ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS 1 +#define ICP_QAT_FW_LA_GCM_IV_LEN_NOT_12_OCTETS 0 +#define QAT_FW_LA_ZUC_3G_PROTO_FLAG_BITPOS 12 +#define ICP_QAT_FW_LA_ZUC_3G_PROTO 1 +#define QAT_FW_LA_ZUC_3G_PROTO_FLAG_MASK 0x1 +#define QAT_LA_GCM_IV_LEN_FLAG_BITPOS 11 +#define QAT_LA_GCM_IV_LEN_FLAG_MASK 0x1 +#define ICP_QAT_FW_LA_DIGEST_IN_BUFFER 1 +#define ICP_QAT_FW_LA_NO_DIGEST_IN_BUFFER 0 +#define QAT_LA_DIGEST_IN_BUFFER_BITPOS 10 +#define QAT_LA_DIGEST_IN_BUFFER_MASK 0x1 +#define ICP_QAT_FW_LA_SNOW_3G_PROTO 4 +#define ICP_QAT_FW_LA_GCM_PROTO 2 +#define ICP_QAT_FW_LA_CCM_PROTO 1 +#define ICP_QAT_FW_LA_NO_PROTO 0 +#define QAT_LA_PROTO_BITPOS 7 +#define QAT_LA_PROTO_MASK 0x7 +#define ICP_QAT_FW_LA_CMP_AUTH_RES 1 +#define ICP_QAT_FW_LA_NO_CMP_AUTH_RES 0 +#define QAT_LA_CMP_AUTH_RES_BITPOS 6 +#define QAT_LA_CMP_AUTH_RES_MASK 0x1 +#define ICP_QAT_FW_LA_RET_AUTH_RES 1 +#define ICP_QAT_FW_LA_NO_RET_AUTH_RES 0 +#define QAT_LA_RET_AUTH_RES_BITPOS 5 +#define QAT_LA_RET_AUTH_RES_MASK 0x1 +#define ICP_QAT_FW_LA_UPDATE_STATE 1 +#define ICP_QAT_FW_LA_NO_UPDATE_STATE 0 +#define QAT_LA_UPDATE_STATE_BITPOS 4 +#define QAT_LA_UPDATE_STATE_MASK 0x1 +#define ICP_QAT_FW_CIPH_AUTH_CFG_OFFSET_IN_CD_SETUP 0 +#define ICP_QAT_FW_CIPH_AUTH_CFG_OFFSET_IN_SHRAM_CP 1 +#define QAT_LA_CIPH_AUTH_CFG_OFFSET_BITPOS 3 +#define QAT_LA_CIPH_AUTH_CFG_OFFSET_MASK 0x1 +#define ICP_QAT_FW_CIPH_IV_64BIT_PTR 0 +#define ICP_QAT_FW_CIPH_IV_16BYTE_DATA 1 +#define QAT_LA_CIPH_IV_FLD_BITPOS 2 +#define QAT_LA_CIPH_IV_FLD_MASK 0x1 +#define ICP_QAT_FW_LA_PARTIAL_NONE 0 +#define ICP_QAT_FW_LA_PARTIAL_START 1 +#define ICP_QAT_FW_LA_PARTIAL_MID 3 +#define ICP_QAT_FW_LA_PARTIAL_END 2 +#define QAT_LA_PARTIAL_BITPOS 0 +#define QAT_LA_PARTIAL_MASK 0x3 +#define ICP_QAT_FW_LA_FLAGS_BUILD(zuc_proto, gcm_iv_len, auth_rslt, proto, \ + cmp_auth, ret_auth, update_state, \ + ciph_iv, ciphcfg, partial) \ + (((zuc_proto & QAT_FW_LA_ZUC_3G_PROTO_FLAG_MASK) << \ + QAT_FW_LA_ZUC_3G_PROTO_FLAG_BITPOS) | \ + ((gcm_iv_len & QAT_LA_GCM_IV_LEN_FLAG_MASK) << \ + QAT_LA_GCM_IV_LEN_FLAG_BITPOS) | \ + ((auth_rslt & QAT_LA_DIGEST_IN_BUFFER_MASK) << \ + QAT_LA_DIGEST_IN_BUFFER_BITPOS) | \ + ((proto & QAT_LA_PROTO_MASK) << \ + QAT_LA_PROTO_BITPOS) | \ + ((cmp_auth & QAT_LA_CMP_AUTH_RES_MASK) << \ + QAT_LA_CMP_AUTH_RES_BITPOS) | \ + ((ret_auth & QAT_LA_RET_AUTH_RES_MASK) << \ + QAT_LA_RET_AUTH_RES_BITPOS) | \ + ((update_state & QAT_LA_UPDATE_STATE_MASK) << \ + QAT_LA_UPDATE_STATE_BITPOS) | \ + ((ciph_iv & QAT_LA_CIPH_IV_FLD_MASK) << \ + QAT_LA_CIPH_IV_FLD_BITPOS) | \ + ((ciphcfg & QAT_LA_CIPH_AUTH_CFG_OFFSET_MASK) << \ + QAT_LA_CIPH_AUTH_CFG_OFFSET_BITPOS) | \ + ((partial & QAT_LA_PARTIAL_MASK) << \ + QAT_LA_PARTIAL_BITPOS)) + +#define ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_GET(flags) \ + QAT_FIELD_GET(flags, QAT_LA_CIPH_IV_FLD_BITPOS, \ + QAT_LA_CIPH_IV_FLD_MASK) + +#define ICP_QAT_FW_LA_CIPH_AUTH_CFG_OFFSET_FLAG_GET(flags) \ + QAT_FIELD_GET(flags, QAT_LA_CIPH_AUTH_CFG_OFFSET_BITPOS, \ + QAT_LA_CIPH_AUTH_CFG_OFFSET_MASK) + +#define ICP_QAT_FW_LA_ZUC_3G_PROTO_FLAG_GET(flags) \ + QAT_FIELD_GET(flags, QAT_FW_LA_ZUC_3G_PROTO_FLAG_BITPOS, \ + QAT_FW_LA_ZUC_3G_PROTO_FLAG_MASK) + +#define ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_GET(flags) \ + QAT_FIELD_GET(flags, QAT_LA_GCM_IV_LEN_FLAG_BITPOS, \ + QAT_LA_GCM_IV_LEN_FLAG_MASK) + +#define ICP_QAT_FW_LA_PROTO_GET(flags) \ + QAT_FIELD_GET(flags, QAT_LA_PROTO_BITPOS, QAT_LA_PROTO_MASK) + +#define ICP_QAT_FW_LA_CMP_AUTH_GET(flags) \ + QAT_FIELD_GET(flags, QAT_LA_CMP_AUTH_RES_BITPOS, \ + QAT_LA_CMP_AUTH_RES_MASK) + +#define ICP_QAT_FW_LA_RET_AUTH_GET(flags) \ + QAT_FIELD_GET(flags, QAT_LA_RET_AUTH_RES_BITPOS, \ + QAT_LA_RET_AUTH_RES_MASK) + +#define ICP_QAT_FW_LA_DIGEST_IN_BUFFER_GET(flags) \ + QAT_FIELD_GET(flags, QAT_LA_DIGEST_IN_BUFFER_BITPOS, \ + QAT_LA_DIGEST_IN_BUFFER_MASK) + +#define ICP_QAT_FW_LA_UPDATE_STATE_GET(flags) \ + QAT_FIELD_GET(flags, QAT_LA_UPDATE_STATE_BITPOS, \ + QAT_LA_UPDATE_STATE_MASK) + +#define ICP_QAT_FW_LA_PARTIAL_GET(flags) \ + QAT_FIELD_GET(flags, QAT_LA_PARTIAL_BITPOS, \ + QAT_LA_PARTIAL_MASK) + +#define ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(flags, val) \ + QAT_FIELD_SET(flags, val, QAT_LA_CIPH_IV_FLD_BITPOS, \ + QAT_LA_CIPH_IV_FLD_MASK) + +#define ICP_QAT_FW_LA_CIPH_AUTH_CFG_OFFSET_FLAG_SET(flags, val) \ + QAT_FIELD_SET(flags, val, QAT_LA_CIPH_AUTH_CFG_OFFSET_BITPOS, \ + QAT_LA_CIPH_AUTH_CFG_OFFSET_MASK) + +#define ICP_QAT_FW_LA_ZUC_3G_PROTO_FLAG_SET(flags, val) \ + QAT_FIELD_SET(flags, val, QAT_FW_LA_ZUC_3G_PROTO_FLAG_BITPOS, \ + QAT_FW_LA_ZUC_3G_PROTO_FLAG_MASK) + +#define ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(flags, val) \ + QAT_FIELD_SET(flags, val, QAT_LA_GCM_IV_LEN_FLAG_BITPOS, \ + QAT_LA_GCM_IV_LEN_FLAG_MASK) + +#define ICP_QAT_FW_LA_PROTO_SET(flags, val) \ + QAT_FIELD_SET(flags, val, QAT_LA_PROTO_BITPOS, \ + QAT_LA_PROTO_MASK) + +#define ICP_QAT_FW_LA_CMP_AUTH_SET(flags, val) \ + QAT_FIELD_SET(flags, val, QAT_LA_CMP_AUTH_RES_BITPOS, \ + QAT_LA_CMP_AUTH_RES_MASK) + +#define ICP_QAT_FW_LA_RET_AUTH_SET(flags, val) \ + QAT_FIELD_SET(flags, val, QAT_LA_RET_AUTH_RES_BITPOS, \ + QAT_LA_RET_AUTH_RES_MASK) + +#define ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(flags, val) \ + QAT_FIELD_SET(flags, val, QAT_LA_DIGEST_IN_BUFFER_BITPOS, \ + QAT_LA_DIGEST_IN_BUFFER_MASK) + +#define ICP_QAT_FW_LA_UPDATE_STATE_SET(flags, val) \ + QAT_FIELD_SET(flags, val, QAT_LA_UPDATE_STATE_BITPOS, \ + QAT_LA_UPDATE_STATE_MASK) + +#define ICP_QAT_FW_LA_PARTIAL_SET(flags, val) \ + QAT_FIELD_SET(flags, val, QAT_LA_PARTIAL_BITPOS, \ + QAT_LA_PARTIAL_MASK) + +struct icp_qat_fw_cipher_req_hdr_cd_pars { + union { + struct { + uint64_t content_desc_addr; + uint16_t content_desc_resrvd1; + uint8_t content_desc_params_sz; + uint8_t content_desc_hdr_resrvd2; + uint32_t content_desc_resrvd3; + } s; + struct { + uint32_t cipher_key_array[ICP_QAT_FW_NUM_LONGWORDS_4]; + } s1; + } u; +}; + +struct icp_qat_fw_cipher_auth_req_hdr_cd_pars { + union { + struct { + uint64_t content_desc_addr; + uint16_t content_desc_resrvd1; + uint8_t content_desc_params_sz; + uint8_t content_desc_hdr_resrvd2; + uint32_t content_desc_resrvd3; + } s; + struct { + uint32_t cipher_key_array[ICP_QAT_FW_NUM_LONGWORDS_4]; + } sl; + } u; +}; + +struct icp_qat_fw_cipher_cd_ctrl_hdr { + uint8_t cipher_state_sz; + uint8_t cipher_key_sz; + uint8_t cipher_cfg_offset; + uint8_t next_curr_id; + uint8_t cipher_padding_sz; + uint8_t resrvd1; + uint16_t resrvd2; + uint32_t resrvd3[ICP_QAT_FW_NUM_LONGWORDS_3]; +}; + +struct icp_qat_fw_auth_cd_ctrl_hdr { + uint32_t resrvd1; + uint8_t resrvd2; + uint8_t hash_flags; + uint8_t hash_cfg_offset; + uint8_t next_curr_id; + uint8_t resrvd3; + uint8_t outer_prefix_sz; + uint8_t final_sz; + uint8_t inner_res_sz; + uint8_t resrvd4; + uint8_t inner_state1_sz; + uint8_t inner_state2_offset; + uint8_t inner_state2_sz; + uint8_t outer_config_offset; + uint8_t outer_state1_sz; + uint8_t outer_res_sz; + uint8_t outer_prefix_offset; +}; + +struct icp_qat_fw_cipher_auth_cd_ctrl_hdr { + uint8_t cipher_state_sz; + uint8_t cipher_key_sz; + uint8_t cipher_cfg_offset; + uint8_t next_curr_id_cipher; + uint8_t cipher_padding_sz; + uint8_t hash_flags; + uint8_t hash_cfg_offset; + uint8_t next_curr_id_auth; + uint8_t resrvd1; + uint8_t outer_prefix_sz; + uint8_t final_sz; + uint8_t inner_res_sz; + uint8_t resrvd2; + uint8_t inner_state1_sz; + uint8_t inner_state2_offset; + uint8_t inner_state2_sz; + uint8_t outer_config_offset; + uint8_t outer_state1_sz; + uint8_t outer_res_sz; + uint8_t outer_prefix_offset; +}; + +#define ICP_QAT_FW_AUTH_HDR_FLAG_DO_NESTED 1 +#define ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED 0 +#define ICP_QAT_FW_CCM_GCM_AAD_SZ_MAX 240 +#define ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET \ + (sizeof(struct icp_qat_fw_la_cipher_req_params_t)) +#define ICP_QAT_FW_CIPHER_REQUEST_PARAMETERS_OFFSET (0) + +struct icp_qat_fw_la_cipher_req_params { + uint32_t cipher_offset; + uint32_t cipher_length; + union { + uint32_t cipher_IV_array[ICP_QAT_FW_NUM_LONGWORDS_4]; + struct { + uint64_t cipher_IV_ptr; + uint64_t resrvd1; + } s; + } u; +}; + +struct icp_qat_fw_la_auth_req_params { + uint32_t auth_off; + uint32_t auth_len; + union { + uint64_t auth_partial_st_prefix; + uint64_t aad_adr; + } u1; + uint64_t auth_res_addr; + union { + uint8_t inner_prefix_sz; + uint8_t aad_sz; + } u2; + uint8_t resrvd1; + uint8_t hash_state_sz; + uint8_t auth_res_sz; +} __packed; + +struct icp_qat_fw_la_auth_req_params_resrvd_flds { + uint32_t resrvd[ICP_QAT_FW_NUM_LONGWORDS_6]; + union { + uint8_t inner_prefix_sz; + uint8_t aad_sz; + } u2; + uint8_t resrvd1; + uint16_t resrvd2; +}; + +struct icp_qat_fw_la_resp { + struct icp_qat_fw_comn_resp_hdr comn_resp; + uint64_t opaque_data; + uint32_t resrvd[ICP_QAT_FW_NUM_LONGWORDS_4]; +}; + +#define ICP_QAT_FW_CIPHER_NEXT_ID_GET(cd_ctrl_hdr_t) \ + ((((cd_ctrl_hdr_t)->next_curr_id_cipher) & \ + ICP_QAT_FW_COMN_NEXT_ID_MASK) >> (ICP_QAT_FW_COMN_NEXT_ID_BITPOS)) + +#define ICP_QAT_FW_CIPHER_NEXT_ID_SET(cd_ctrl_hdr_t, val) \ +{ (cd_ctrl_hdr_t)->next_curr_id_cipher = \ + ((((cd_ctrl_hdr_t)->next_curr_id_cipher) \ + & ICP_QAT_FW_COMN_CURR_ID_MASK) | \ + ((val << ICP_QAT_FW_COMN_NEXT_ID_BITPOS) \ + & ICP_QAT_FW_COMN_NEXT_ID_MASK)) } + +#define ICP_QAT_FW_CIPHER_CURR_ID_GET(cd_ctrl_hdr_t) \ + (((cd_ctrl_hdr_t)->next_curr_id_cipher) \ + & ICP_QAT_FW_COMN_CURR_ID_MASK) + +#define ICP_QAT_FW_CIPHER_CURR_ID_SET(cd_ctrl_hdr_t, val) \ +{ (cd_ctrl_hdr_t)->next_curr_id_cipher = \ + ((((cd_ctrl_hdr_t)->next_curr_id_cipher) \ + & ICP_QAT_FW_COMN_NEXT_ID_MASK) | \ + ((val) & ICP_QAT_FW_COMN_CURR_ID_MASK)) } + +#define ICP_QAT_FW_AUTH_NEXT_ID_GET(cd_ctrl_hdr_t) \ + ((((cd_ctrl_hdr_t)->next_curr_id_auth) & ICP_QAT_FW_COMN_NEXT_ID_MASK) \ + >> (ICP_QAT_FW_COMN_NEXT_ID_BITPOS)) + +#define ICP_QAT_FW_AUTH_NEXT_ID_SET(cd_ctrl_hdr_t, val) \ +{ (cd_ctrl_hdr_t)->next_curr_id_auth = \ + ((((cd_ctrl_hdr_t)->next_curr_id_auth) \ + & ICP_QAT_FW_COMN_CURR_ID_MASK) | \ + ((val << ICP_QAT_FW_COMN_NEXT_ID_BITPOS) \ + & ICP_QAT_FW_COMN_NEXT_ID_MASK)) } + +#define ICP_QAT_FW_AUTH_CURR_ID_GET(cd_ctrl_hdr_t) \ + (((cd_ctrl_hdr_t)->next_curr_id_auth) \ + & ICP_QAT_FW_COMN_CURR_ID_MASK) + +#define ICP_QAT_FW_AUTH_CURR_ID_SET(cd_ctrl_hdr_t, val) \ +{ (cd_ctrl_hdr_t)->next_curr_id_auth = \ + ((((cd_ctrl_hdr_t)->next_curr_id_auth) \ + & ICP_QAT_FW_COMN_NEXT_ID_MASK) | \ + ((val) & ICP_QAT_FW_COMN_CURR_ID_MASK)) } + +#endif diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h new file mode 100644 index 000000000000..5e1aa40c0404 --- /dev/null +++ b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h @@ -0,0 +1,78 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef __ICP_QAT_FW_LOADER_HANDLE_H__ +#define __ICP_QAT_FW_LOADER_HANDLE_H__ +#include "icp_qat_uclo.h" + +struct icp_qat_fw_loader_ae_data { + unsigned int state; + unsigned int ustore_size; + unsigned int free_addr; + unsigned int free_size; + unsigned int live_ctx_mask; +}; + +struct icp_qat_fw_loader_hal_handle { + struct icp_qat_fw_loader_ae_data aes[ICP_QAT_UCLO_MAX_AE]; + unsigned int ae_mask; + unsigned int slice_mask; + unsigned int revision_id; + unsigned int ae_max_num; + unsigned int upc_mask; + unsigned int max_ustore; +}; + +struct icp_qat_fw_loader_handle { + struct icp_qat_fw_loader_hal_handle *hal_handle; + void *obj_handle; + void __iomem *hal_sram_addr_v; + void __iomem *hal_cap_g_ctl_csr_addr_v; + void __iomem *hal_cap_ae_xfer_csr_addr_v; + void __iomem *hal_cap_ae_local_csr_addr_v; + void __iomem *hal_ep_csr_addr_v; +}; +#endif diff --git a/drivers/crypto/qat/qat_common/icp_qat_hal.h b/drivers/crypto/qat/qat_common/icp_qat_hal.h new file mode 100644 index 000000000000..85b6d241ea82 --- /dev/null +++ b/drivers/crypto/qat/qat_common/icp_qat_hal.h @@ -0,0 +1,125 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef __ICP_QAT_HAL_H +#define __ICP_QAT_HAL_H +#include "icp_qat_fw_loader_handle.h" + +enum hal_global_csr { + MISC_CONTROL = 0x04, + ICP_RESET = 0x0c, + ICP_GLOBAL_CLK_ENABLE = 0x50 +}; + +enum hal_ae_csr { + USTORE_ADDRESS = 0x000, + USTORE_DATA_LOWER = 0x004, + USTORE_DATA_UPPER = 0x008, + ALU_OUT = 0x010, + CTX_ARB_CNTL = 0x014, + CTX_ENABLES = 0x018, + CC_ENABLE = 0x01c, + CSR_CTX_POINTER = 0x020, + CTX_STS_INDIRECT = 0x040, + ACTIVE_CTX_STATUS = 0x044, + CTX_SIG_EVENTS_INDIRECT = 0x048, + CTX_SIG_EVENTS_ACTIVE = 0x04c, + CTX_WAKEUP_EVENTS_INDIRECT = 0x050, + LM_ADDR_0_INDIRECT = 0x060, + LM_ADDR_1_INDIRECT = 0x068, + INDIRECT_LM_ADDR_0_BYTE_INDEX = 0x0e0, + INDIRECT_LM_ADDR_1_BYTE_INDEX = 0x0e8, + FUTURE_COUNT_SIGNAL_INDIRECT = 0x078, + TIMESTAMP_LOW = 0x0c0, + TIMESTAMP_HIGH = 0x0c4, + PROFILE_COUNT = 0x144, + SIGNATURE_ENABLE = 0x150, + AE_MISC_CONTROL = 0x160, + LOCAL_CSR_STATUS = 0x180, +}; + +#define UA_ECS (0x1 << 31) +#define ACS_ABO_BITPOS 31 +#define ACS_ACNO 0x7 +#define CE_ENABLE_BITPOS 0x8 +#define CE_LMADDR_0_GLOBAL_BITPOS 16 +#define CE_LMADDR_1_GLOBAL_BITPOS 17 +#define CE_NN_MODE_BITPOS 20 +#define CE_REG_PAR_ERR_BITPOS 25 +#define CE_BREAKPOINT_BITPOS 27 +#define CE_CNTL_STORE_PARITY_ERROR_BITPOS 29 +#define CE_INUSE_CONTEXTS_BITPOS 31 +#define CE_NN_MODE (0x1 << CE_NN_MODE_BITPOS) +#define CE_INUSE_CONTEXTS (0x1 << CE_INUSE_CONTEXTS_BITPOS) +#define XCWE_VOLUNTARY (0x1) +#define LCS_STATUS (0x1) +#define MMC_SHARE_CS_BITPOS 2 +#define GLOBAL_CSR 0xA00 + +#define SET_CAP_CSR(handle, csr, val) \ + ADF_CSR_WR(handle->hal_cap_g_ctl_csr_addr_v, csr, val) +#define GET_CAP_CSR(handle, csr) \ + ADF_CSR_RD(handle->hal_cap_g_ctl_csr_addr_v, csr) +#define SET_GLB_CSR(handle, csr, val) SET_CAP_CSR(handle, csr + GLOBAL_CSR, val) +#define GET_GLB_CSR(handle, csr) GET_CAP_CSR(handle, GLOBAL_CSR + csr) +#define AE_CSR(handle, ae) \ + (handle->hal_cap_ae_local_csr_addr_v + \ + ((ae & handle->hal_handle->ae_mask) << 12)) +#define AE_CSR_ADDR(handle, ae, csr) (AE_CSR(handle, ae) + (0x3ff & csr)) +#define SET_AE_CSR(handle, ae, csr, val) \ + ADF_CSR_WR(AE_CSR_ADDR(handle, ae, csr), 0, val) +#define GET_AE_CSR(handle, ae, csr) ADF_CSR_RD(AE_CSR_ADDR(handle, ae, csr), 0) +#define AE_XFER(handle, ae) \ + (handle->hal_cap_ae_xfer_csr_addr_v + \ + ((ae & handle->hal_handle->ae_mask) << 12)) +#define AE_XFER_ADDR(handle, ae, reg) (AE_XFER(handle, ae) + \ + ((reg & 0xff) << 2)) +#define SET_AE_XFER(handle, ae, reg, val) \ + ADF_CSR_WR(AE_XFER_ADDR(handle, ae, reg), 0, val) +#define SRAM_WRITE(handle, addr, val) \ + ADF_CSR_WR(handle->hal_sram_addr_v, addr, val) +#define SRAM_READ(handle, addr) ADF_CSR_RD(handle->hal_sram_addr_v, addr) +#endif diff --git a/drivers/crypto/qat/qat_common/icp_qat_hw.h b/drivers/crypto/qat/qat_common/icp_qat_hw.h new file mode 100644 index 000000000000..5031f8c10d75 --- /dev/null +++ b/drivers/crypto/qat/qat_common/icp_qat_hw.h @@ -0,0 +1,305 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef _ICP_QAT_HW_H_ +#define _ICP_QAT_HW_H_ + +enum icp_qat_hw_ae_id { + ICP_QAT_HW_AE_0 = 0, + ICP_QAT_HW_AE_1 = 1, + ICP_QAT_HW_AE_2 = 2, + ICP_QAT_HW_AE_3 = 3, + ICP_QAT_HW_AE_4 = 4, + ICP_QAT_HW_AE_5 = 5, + ICP_QAT_HW_AE_6 = 6, + ICP_QAT_HW_AE_7 = 7, + ICP_QAT_HW_AE_8 = 8, + ICP_QAT_HW_AE_9 = 9, + ICP_QAT_HW_AE_10 = 10, + ICP_QAT_HW_AE_11 = 11, + ICP_QAT_HW_AE_DELIMITER = 12 +}; + +enum icp_qat_hw_qat_id { + ICP_QAT_HW_QAT_0 = 0, + ICP_QAT_HW_QAT_1 = 1, + ICP_QAT_HW_QAT_2 = 2, + ICP_QAT_HW_QAT_3 = 3, + ICP_QAT_HW_QAT_4 = 4, + ICP_QAT_HW_QAT_5 = 5, + ICP_QAT_HW_QAT_DELIMITER = 6 +}; + +enum icp_qat_hw_auth_algo { + ICP_QAT_HW_AUTH_ALGO_NULL = 0, + ICP_QAT_HW_AUTH_ALGO_SHA1 = 1, + ICP_QAT_HW_AUTH_ALGO_MD5 = 2, + ICP_QAT_HW_AUTH_ALGO_SHA224 = 3, + ICP_QAT_HW_AUTH_ALGO_SHA256 = 4, + ICP_QAT_HW_AUTH_ALGO_SHA384 = 5, + ICP_QAT_HW_AUTH_ALGO_SHA512 = 6, + ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC = 7, + ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC = 8, + ICP_QAT_HW_AUTH_ALGO_AES_F9 = 9, + ICP_QAT_HW_AUTH_ALGO_GALOIS_128 = 10, + ICP_QAT_HW_AUTH_ALGO_GALOIS_64 = 11, + ICP_QAT_HW_AUTH_ALGO_KASUMI_F9 = 12, + ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2 = 13, + ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3 = 14, + ICP_QAT_HW_AUTH_RESERVED_1 = 15, + ICP_QAT_HW_AUTH_RESERVED_2 = 16, + ICP_QAT_HW_AUTH_ALGO_SHA3_256 = 17, + ICP_QAT_HW_AUTH_RESERVED_3 = 18, + ICP_QAT_HW_AUTH_ALGO_SHA3_512 = 19, + ICP_QAT_HW_AUTH_ALGO_DELIMITER = 20 +}; + +enum icp_qat_hw_auth_mode { + ICP_QAT_HW_AUTH_MODE0 = 0, + ICP_QAT_HW_AUTH_MODE1 = 1, + ICP_QAT_HW_AUTH_MODE2 = 2, + ICP_QAT_HW_AUTH_MODE_DELIMITER = 3 +}; + +struct icp_qat_hw_auth_config { + uint32_t config; + uint32_t reserved; +}; + +#define QAT_AUTH_MODE_BITPOS 4 +#define QAT_AUTH_MODE_MASK 0xF +#define QAT_AUTH_ALGO_BITPOS 0 +#define QAT_AUTH_ALGO_MASK 0xF +#define QAT_AUTH_CMP_BITPOS 8 +#define QAT_AUTH_CMP_MASK 0x7F +#define QAT_AUTH_SHA3_PADDING_BITPOS 16 +#define QAT_AUTH_SHA3_PADDING_MASK 0x1 +#define QAT_AUTH_ALGO_SHA3_BITPOS 22 +#define QAT_AUTH_ALGO_SHA3_MASK 0x3 +#define ICP_QAT_HW_AUTH_CONFIG_BUILD(mode, algo, cmp_len) \ + (((mode & QAT_AUTH_MODE_MASK) << QAT_AUTH_MODE_BITPOS) | \ + ((algo & QAT_AUTH_ALGO_MASK) << QAT_AUTH_ALGO_BITPOS) | \ + (((algo >> 4) & QAT_AUTH_ALGO_SHA3_MASK) << \ + QAT_AUTH_ALGO_SHA3_BITPOS) | \ + (((((algo == ICP_QAT_HW_AUTH_ALGO_SHA3_256) || \ + (algo == ICP_QAT_HW_AUTH_ALGO_SHA3_512)) ? 1 : 0) \ + & QAT_AUTH_SHA3_PADDING_MASK) << QAT_AUTH_SHA3_PADDING_BITPOS) | \ + ((cmp_len & QAT_AUTH_CMP_MASK) << QAT_AUTH_CMP_BITPOS)) + +struct icp_qat_hw_auth_counter { + __be32 counter; + uint32_t reserved; +}; + +#define QAT_AUTH_COUNT_MASK 0xFFFFFFFF +#define QAT_AUTH_COUNT_BITPOS 0 +#define ICP_QAT_HW_AUTH_COUNT_BUILD(val) \ + (((val) & QAT_AUTH_COUNT_MASK) << QAT_AUTH_COUNT_BITPOS) + +struct icp_qat_hw_auth_setup { + struct icp_qat_hw_auth_config auth_config; + struct icp_qat_hw_auth_counter auth_counter; +}; + +#define QAT_HW_DEFAULT_ALIGNMENT 8 +#define QAT_HW_ROUND_UP(val, n) (((val) + ((n)-1)) & (~(n-1))) +#define ICP_QAT_HW_NULL_STATE1_SZ 32 +#define ICP_QAT_HW_MD5_STATE1_SZ 16 +#define ICP_QAT_HW_SHA1_STATE1_SZ 20 +#define ICP_QAT_HW_SHA224_STATE1_SZ 32 +#define ICP_QAT_HW_SHA256_STATE1_SZ 32 +#define ICP_QAT_HW_SHA3_256_STATE1_SZ 32 +#define ICP_QAT_HW_SHA384_STATE1_SZ 64 +#define ICP_QAT_HW_SHA512_STATE1_SZ 64 +#define ICP_QAT_HW_SHA3_512_STATE1_SZ 64 +#define ICP_QAT_HW_SHA3_224_STATE1_SZ 28 +#define ICP_QAT_HW_SHA3_384_STATE1_SZ 48 +#define ICP_QAT_HW_AES_XCBC_MAC_STATE1_SZ 16 +#define ICP_QAT_HW_AES_CBC_MAC_STATE1_SZ 16 +#define ICP_QAT_HW_AES_F9_STATE1_SZ 32 +#define ICP_QAT_HW_KASUMI_F9_STATE1_SZ 16 +#define ICP_QAT_HW_GALOIS_128_STATE1_SZ 16 +#define ICP_QAT_HW_SNOW_3G_UIA2_STATE1_SZ 8 +#define ICP_QAT_HW_ZUC_3G_EIA3_STATE1_SZ 8 +#define ICP_QAT_HW_NULL_STATE2_SZ 32 +#define ICP_QAT_HW_MD5_STATE2_SZ 16 +#define ICP_QAT_HW_SHA1_STATE2_SZ 20 +#define ICP_QAT_HW_SHA224_STATE2_SZ 32 +#define ICP_QAT_HW_SHA256_STATE2_SZ 32 +#define ICP_QAT_HW_SHA3_256_STATE2_SZ 0 +#define ICP_QAT_HW_SHA384_STATE2_SZ 64 +#define ICP_QAT_HW_SHA512_STATE2_SZ 64 +#define ICP_QAT_HW_SHA3_512_STATE2_SZ 0 +#define ICP_QAT_HW_SHA3_224_STATE2_SZ 0 +#define ICP_QAT_HW_SHA3_384_STATE2_SZ 0 +#define ICP_QAT_HW_AES_XCBC_MAC_KEY_SZ 16 +#define ICP_QAT_HW_AES_CBC_MAC_KEY_SZ 16 +#define ICP_QAT_HW_AES_CCM_CBC_E_CTR0_SZ 16 +#define ICP_QAT_HW_F9_IK_SZ 16 +#define ICP_QAT_HW_F9_FK_SZ 16 +#define ICP_QAT_HW_KASUMI_F9_STATE2_SZ (ICP_QAT_HW_F9_IK_SZ + \ + ICP_QAT_HW_F9_FK_SZ) +#define ICP_QAT_HW_AES_F9_STATE2_SZ ICP_QAT_HW_KASUMI_F9_STATE2_SZ +#define ICP_QAT_HW_SNOW_3G_UIA2_STATE2_SZ 24 +#define ICP_QAT_HW_ZUC_3G_EIA3_STATE2_SZ 32 +#define ICP_QAT_HW_GALOIS_H_SZ 16 +#define ICP_QAT_HW_GALOIS_LEN_A_SZ 8 +#define ICP_QAT_HW_GALOIS_E_CTR0_SZ 16 + +struct icp_qat_hw_auth_sha512 { + struct icp_qat_hw_auth_setup inner_setup; + uint8_t state1[ICP_QAT_HW_SHA512_STATE1_SZ]; + struct icp_qat_hw_auth_setup outer_setup; + uint8_t state2[ICP_QAT_HW_SHA512_STATE2_SZ]; +}; + +struct icp_qat_hw_auth_algo_blk { + struct icp_qat_hw_auth_sha512 sha; +}; + +#define ICP_QAT_HW_GALOIS_LEN_A_BITPOS 0 +#define ICP_QAT_HW_GALOIS_LEN_A_MASK 0xFFFFFFFF + +enum icp_qat_hw_cipher_algo { + ICP_QAT_HW_CIPHER_ALGO_NULL = 0, + ICP_QAT_HW_CIPHER_ALGO_DES = 1, + ICP_QAT_HW_CIPHER_ALGO_3DES = 2, + ICP_QAT_HW_CIPHER_ALGO_AES128 = 3, + ICP_QAT_HW_CIPHER_ALGO_AES192 = 4, + ICP_QAT_HW_CIPHER_ALGO_AES256 = 5, + ICP_QAT_HW_CIPHER_ALGO_ARC4 = 6, + ICP_QAT_HW_CIPHER_ALGO_KASUMI = 7, + ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2 = 8, + ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3 = 9, + ICP_QAT_HW_CIPHER_DELIMITER = 10 +}; + +enum icp_qat_hw_cipher_mode { + ICP_QAT_HW_CIPHER_ECB_MODE = 0, + ICP_QAT_HW_CIPHER_CBC_MODE = 1, + ICP_QAT_HW_CIPHER_CTR_MODE = 2, + ICP_QAT_HW_CIPHER_F8_MODE = 3, + ICP_QAT_HW_CIPHER_XTS_MODE = 6, + ICP_QAT_HW_CIPHER_MODE_DELIMITER = 7 +}; + +struct icp_qat_hw_cipher_config { + uint32_t val; + uint32_t reserved; +}; + +enum icp_qat_hw_cipher_dir { + ICP_QAT_HW_CIPHER_ENCRYPT = 0, + ICP_QAT_HW_CIPHER_DECRYPT = 1, +}; + +enum icp_qat_hw_cipher_convert { + ICP_QAT_HW_CIPHER_NO_CONVERT = 0, + ICP_QAT_HW_CIPHER_KEY_CONVERT = 1, +}; + +#define QAT_CIPHER_MODE_BITPOS 4 +#define QAT_CIPHER_MODE_MASK 0xF +#define QAT_CIPHER_ALGO_BITPOS 0 +#define QAT_CIPHER_ALGO_MASK 0xF +#define QAT_CIPHER_CONVERT_BITPOS 9 +#define QAT_CIPHER_CONVERT_MASK 0x1 +#define QAT_CIPHER_DIR_BITPOS 8 +#define QAT_CIPHER_DIR_MASK 0x1 +#define QAT_CIPHER_MODE_F8_KEY_SZ_MULT 2 +#define QAT_CIPHER_MODE_XTS_KEY_SZ_MULT 2 +#define ICP_QAT_HW_CIPHER_CONFIG_BUILD(mode, algo, convert, dir) \ + (((mode & QAT_CIPHER_MODE_MASK) << QAT_CIPHER_MODE_BITPOS) | \ + ((algo & QAT_CIPHER_ALGO_MASK) << QAT_CIPHER_ALGO_BITPOS) | \ + ((convert & QAT_CIPHER_CONVERT_MASK) << QAT_CIPHER_CONVERT_BITPOS) | \ + ((dir & QAT_CIPHER_DIR_MASK) << QAT_CIPHER_DIR_BITPOS)) +#define ICP_QAT_HW_DES_BLK_SZ 8 +#define ICP_QAT_HW_3DES_BLK_SZ 8 +#define ICP_QAT_HW_NULL_BLK_SZ 8 +#define ICP_QAT_HW_AES_BLK_SZ 16 +#define ICP_QAT_HW_KASUMI_BLK_SZ 8 +#define ICP_QAT_HW_SNOW_3G_BLK_SZ 8 +#define ICP_QAT_HW_ZUC_3G_BLK_SZ 8 +#define ICP_QAT_HW_NULL_KEY_SZ 256 +#define ICP_QAT_HW_DES_KEY_SZ 8 +#define ICP_QAT_HW_3DES_KEY_SZ 24 +#define ICP_QAT_HW_AES_128_KEY_SZ 16 +#define ICP_QAT_HW_AES_192_KEY_SZ 24 +#define ICP_QAT_HW_AES_256_KEY_SZ 32 +#define ICP_QAT_HW_AES_128_F8_KEY_SZ (ICP_QAT_HW_AES_128_KEY_SZ * \ + QAT_CIPHER_MODE_F8_KEY_SZ_MULT) +#define ICP_QAT_HW_AES_192_F8_KEY_SZ (ICP_QAT_HW_AES_192_KEY_SZ * \ + QAT_CIPHER_MODE_F8_KEY_SZ_MULT) +#define ICP_QAT_HW_AES_256_F8_KEY_SZ (ICP_QAT_HW_AES_256_KEY_SZ * \ + QAT_CIPHER_MODE_F8_KEY_SZ_MULT) +#define ICP_QAT_HW_AES_128_XTS_KEY_SZ (ICP_QAT_HW_AES_128_KEY_SZ * \ + QAT_CIPHER_MODE_XTS_KEY_SZ_MULT) +#define ICP_QAT_HW_AES_256_XTS_KEY_SZ (ICP_QAT_HW_AES_256_KEY_SZ * \ + QAT_CIPHER_MODE_XTS_KEY_SZ_MULT) +#define ICP_QAT_HW_KASUMI_KEY_SZ 16 +#define ICP_QAT_HW_KASUMI_F8_KEY_SZ (ICP_QAT_HW_KASUMI_KEY_SZ * \ + QAT_CIPHER_MODE_F8_KEY_SZ_MULT) +#define ICP_QAT_HW_AES_128_XTS_KEY_SZ (ICP_QAT_HW_AES_128_KEY_SZ * \ + QAT_CIPHER_MODE_XTS_KEY_SZ_MULT) +#define ICP_QAT_HW_AES_256_XTS_KEY_SZ (ICP_QAT_HW_AES_256_KEY_SZ * \ + QAT_CIPHER_MODE_XTS_KEY_SZ_MULT) +#define ICP_QAT_HW_ARC4_KEY_SZ 256 +#define ICP_QAT_HW_SNOW_3G_UEA2_KEY_SZ 16 +#define ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ 16 +#define ICP_QAT_HW_ZUC_3G_EEA3_KEY_SZ 16 +#define ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ 16 +#define ICP_QAT_HW_MODE_F8_NUM_REG_TO_CLEAR 2 +#define INIT_SHRAM_CONSTANTS_TABLE_SZ 1024 + +struct icp_qat_hw_cipher_aes256_f8 { + struct icp_qat_hw_cipher_config cipher_config; + uint8_t key[ICP_QAT_HW_AES_256_F8_KEY_SZ]; +}; + +struct icp_qat_hw_cipher_algo_blk { + struct icp_qat_hw_cipher_aes256_f8 aes; +}; +#endif diff --git a/drivers/crypto/qat/qat_common/icp_qat_uclo.h b/drivers/crypto/qat/qat_common/icp_qat_uclo.h new file mode 100644 index 000000000000..2132a8cbc4ec --- /dev/null +++ b/drivers/crypto/qat/qat_common/icp_qat_uclo.h @@ -0,0 +1,377 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef __ICP_QAT_UCLO_H__ +#define __ICP_QAT_UCLO_H__ + +#define ICP_QAT_AC_C_CPU_TYPE 0x00400000 +#define ICP_QAT_UCLO_MAX_AE 12 +#define ICP_QAT_UCLO_MAX_CTX 8 +#define ICP_QAT_UCLO_MAX_UIMAGE (ICP_QAT_UCLO_MAX_AE * ICP_QAT_UCLO_MAX_CTX) +#define ICP_QAT_UCLO_MAX_USTORE 0x4000 +#define ICP_QAT_UCLO_MAX_XFER_REG 128 +#define ICP_QAT_UCLO_MAX_GPR_REG 128 +#define ICP_QAT_UCLO_MAX_NN_REG 128 +#define ICP_QAT_UCLO_MAX_LMEM_REG 1024 +#define ICP_QAT_UCLO_AE_ALL_CTX 0xff +#define ICP_QAT_UOF_OBJID_LEN 8 +#define ICP_QAT_UOF_FID 0xc6c2 +#define ICP_QAT_UOF_MAJVER 0x4 +#define ICP_QAT_UOF_MINVER 0x11 +#define ICP_QAT_UOF_NN_MODE_NOTCARE 0xff +#define ICP_QAT_UOF_OBJS "UOF_OBJS" +#define ICP_QAT_UOF_STRT "UOF_STRT" +#define ICP_QAT_UOF_GTID "UOF_GTID" +#define ICP_QAT_UOF_IMAG "UOF_IMAG" +#define ICP_QAT_UOF_IMEM "UOF_IMEM" +#define ICP_QAT_UOF_MSEG "UOF_MSEG" +#define ICP_QAT_UOF_LOCAL_SCOPE 1 +#define ICP_QAT_UOF_INIT_EXPR 0 +#define ICP_QAT_UOF_INIT_REG 1 +#define ICP_QAT_UOF_INIT_REG_CTX 2 +#define ICP_QAT_UOF_INIT_EXPR_ENDIAN_SWAP 3 + +#define ICP_QAT_CTX_MODE(ae_mode) ((ae_mode) & 0xf) +#define ICP_QAT_NN_MODE(ae_mode) (((ae_mode) >> 0x4) & 0xf) +#define ICP_QAT_SHARED_USTORE_MODE(ae_mode) (((ae_mode) >> 0xb) & 0x1) +#define RELOADABLE_CTX_SHARED_MODE(ae_mode) (((ae_mode) >> 0xc) & 0x1) + +#define ICP_QAT_LOC_MEM0_MODE(ae_mode) (((ae_mode) >> 0x8) & 0x1) +#define ICP_QAT_LOC_MEM1_MODE(ae_mode) (((ae_mode) >> 0x9) & 0x1) + +enum icp_qat_uof_mem_region { + ICP_QAT_UOF_SRAM_REGION = 0x0, + ICP_QAT_UOF_LMEM_REGION = 0x3, + ICP_QAT_UOF_UMEM_REGION = 0x5 +}; + +enum icp_qat_uof_regtype { + ICP_NO_DEST, + ICP_GPA_REL, + ICP_GPA_ABS, + ICP_GPB_REL, + ICP_GPB_ABS, + ICP_SR_REL, + ICP_SR_RD_REL, + ICP_SR_WR_REL, + ICP_SR_ABS, + ICP_SR_RD_ABS, + ICP_SR_WR_ABS, + ICP_DR_REL, + ICP_DR_RD_REL, + ICP_DR_WR_REL, + ICP_DR_ABS, + ICP_DR_RD_ABS, + ICP_DR_WR_ABS, + ICP_LMEM, + ICP_LMEM0, + ICP_LMEM1, + ICP_NEIGH_REL, +}; + +struct icp_qat_uclo_page { + struct icp_qat_uclo_encap_page *encap_page; + struct icp_qat_uclo_region *region; + unsigned int flags; +}; + +struct icp_qat_uclo_region { + struct icp_qat_uclo_page *loaded; + struct icp_qat_uclo_page *page; +}; + +struct icp_qat_uclo_aeslice { + struct icp_qat_uclo_region *region; + struct icp_qat_uclo_page *page; + struct icp_qat_uclo_page *cur_page[ICP_QAT_UCLO_MAX_CTX]; + struct icp_qat_uclo_encapme *encap_image; + unsigned int ctx_mask_assigned; + unsigned int new_uaddr[ICP_QAT_UCLO_MAX_CTX]; +}; + +struct icp_qat_uclo_aedata { + unsigned int slice_num; + unsigned int eff_ustore_size; + struct icp_qat_uclo_aeslice ae_slices[ICP_QAT_UCLO_MAX_CTX]; +}; + +struct icp_qat_uof_encap_obj { + char *beg_uof; + struct icp_qat_uof_objhdr *obj_hdr; + struct icp_qat_uof_chunkhdr *chunk_hdr; + struct icp_qat_uof_varmem_seg *var_mem_seg; +}; + +struct icp_qat_uclo_encap_uwblock { + unsigned int start_addr; + unsigned int words_num; + uint64_t micro_words; +}; + +struct icp_qat_uclo_encap_page { + unsigned int def_page; + unsigned int page_region; + unsigned int beg_addr_v; + unsigned int beg_addr_p; + unsigned int micro_words_num; + unsigned int uwblock_num; + struct icp_qat_uclo_encap_uwblock *uwblock; +}; + +struct icp_qat_uclo_encapme { + struct icp_qat_uof_image *img_ptr; + struct icp_qat_uclo_encap_page *page; + unsigned int ae_reg_num; + struct icp_qat_uof_ae_reg *ae_reg; + unsigned int init_regsym_num; + struct icp_qat_uof_init_regsym *init_regsym; + unsigned int sbreak_num; + struct icp_qat_uof_sbreak *sbreak; + unsigned int uwords_num; +}; + +struct icp_qat_uclo_init_mem_table { + unsigned int entry_num; + struct icp_qat_uof_initmem *init_mem; +}; + +struct icp_qat_uclo_objhdr { + char *file_buff; + unsigned int checksum; + unsigned int size; +}; + +struct icp_qat_uof_strtable { + unsigned int table_len; + unsigned int reserved; + uint64_t strings; +}; + +struct icp_qat_uclo_objhandle { + unsigned int prod_type; + unsigned int prod_rev; + struct icp_qat_uclo_objhdr *obj_hdr; + struct icp_qat_uof_encap_obj encap_uof_obj; + struct icp_qat_uof_strtable str_table; + struct icp_qat_uclo_encapme ae_uimage[ICP_QAT_UCLO_MAX_UIMAGE]; + struct icp_qat_uclo_aedata ae_data[ICP_QAT_UCLO_MAX_AE]; + struct icp_qat_uclo_init_mem_table init_mem_tab; + struct icp_qat_uof_batch_init *lm_init_tab[ICP_QAT_UCLO_MAX_AE]; + struct icp_qat_uof_batch_init *umem_init_tab[ICP_QAT_UCLO_MAX_AE]; + int uimage_num; + int uword_in_bytes; + int global_inited; + unsigned int ae_num; + unsigned int ustore_phy_size; + void *obj_buf; + uint64_t *uword_buf; +}; + +struct icp_qat_uof_uword_block { + unsigned int start_addr; + unsigned int words_num; + unsigned int uword_offset; + unsigned int reserved; +}; + +struct icp_qat_uof_filehdr { + unsigned short file_id; + unsigned short reserved1; + char min_ver; + char maj_ver; + unsigned short reserved2; + unsigned short max_chunks; + unsigned short num_chunks; +}; + +struct icp_qat_uof_filechunkhdr { + char chunk_id[ICP_QAT_UOF_OBJID_LEN]; + unsigned int checksum; + unsigned int offset; + unsigned int size; +}; + +struct icp_qat_uof_objhdr { + unsigned int cpu_type; + unsigned short min_cpu_ver; + unsigned short max_cpu_ver; + short max_chunks; + short num_chunks; + unsigned int reserved1; + unsigned int reserved2; +}; + +struct icp_qat_uof_chunkhdr { + char chunk_id[ICP_QAT_UOF_OBJID_LEN]; + unsigned int offset; + unsigned int size; +}; + +struct icp_qat_uof_memvar_attr { + unsigned int offset_in_byte; + unsigned int value; +}; + +struct icp_qat_uof_initmem { + unsigned int sym_name; + char region; + char scope; + unsigned short reserved1; + unsigned int addr; + unsigned int num_in_bytes; + unsigned int val_attr_num; +}; + +struct icp_qat_uof_init_regsym { + unsigned int sym_name; + char init_type; + char value_type; + char reg_type; + unsigned char ctx; + unsigned int reg_addr; + unsigned int value; +}; + +struct icp_qat_uof_varmem_seg { + unsigned int sram_base; + unsigned int sram_size; + unsigned int sram_alignment; + unsigned int sdram_base; + unsigned int sdram_size; + unsigned int sdram_alignment; + unsigned int sdram1_base; + unsigned int sdram1_size; + unsigned int sdram1_alignment; + unsigned int scratch_base; + unsigned int scratch_size; + unsigned int scratch_alignment; +}; + +struct icp_qat_uof_gtid { + char tool_id[ICP_QAT_UOF_OBJID_LEN]; + int tool_ver; + unsigned int reserved1; + unsigned int reserved2; +}; + +struct icp_qat_uof_sbreak { + unsigned int page_num; + unsigned int virt_uaddr; + unsigned char sbreak_type; + unsigned char reg_type; + unsigned short reserved1; + unsigned int addr_offset; + unsigned int reg_addr; +}; + +struct icp_qat_uof_code_page { + unsigned int page_region; + unsigned int page_num; + unsigned char def_page; + unsigned char reserved2; + unsigned short reserved1; + unsigned int beg_addr_v; + unsigned int beg_addr_p; + unsigned int neigh_reg_tab_offset; + unsigned int uc_var_tab_offset; + unsigned int imp_var_tab_offset; + unsigned int imp_expr_tab_offset; + unsigned int code_area_offset; +}; + +struct icp_qat_uof_image { + unsigned int img_name; + unsigned int ae_assigned; + unsigned int ctx_assigned; + unsigned int cpu_type; + unsigned int entry_address; + unsigned int fill_pattern[2]; + unsigned int reloadable_size; + unsigned char sensitivity; + unsigned char reserved; + unsigned short ae_mode; + unsigned short max_ver; + unsigned short min_ver; + unsigned short image_attrib; + unsigned short reserved2; + unsigned short page_region_num; + unsigned short numpages; + unsigned int reg_tab_offset; + unsigned int init_reg_sym_tab; + unsigned int sbreak_tab; + unsigned int app_metadata; +}; + +struct icp_qat_uof_objtable { + unsigned int entry_num; +}; + +struct icp_qat_uof_ae_reg { + unsigned int name; + unsigned int vis_name; + unsigned short type; + unsigned short addr; + unsigned short access_mode; + unsigned char visible; + unsigned char reserved1; + unsigned short ref_count; + unsigned short reserved2; + unsigned int xo_id; +}; + +struct icp_qat_uof_code_area { + unsigned int micro_words_num; + unsigned int uword_block_tab; +}; + +struct icp_qat_uof_batch_init { + unsigned int ae; + unsigned int addr; + unsigned int *value; + unsigned int size; + struct icp_qat_uof_batch_init *next; +}; +#endif diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c new file mode 100644 index 000000000000..59df48872955 --- /dev/null +++ b/drivers/crypto/qat/qat_common/qat_algs.c @@ -0,0 +1,1038 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/crypto.h> +#include <crypto/aead.h> +#include <crypto/aes.h> +#include <crypto/sha.h> +#include <crypto/hash.h> +#include <crypto/algapi.h> +#include <crypto/authenc.h> +#include <crypto/rng.h> +#include <linux/dma-mapping.h> +#include "adf_accel_devices.h" +#include "adf_transport.h" +#include "adf_common_drv.h" +#include "qat_crypto.h" +#include "icp_qat_hw.h" +#include "icp_qat_fw.h" +#include "icp_qat_fw_la.h" + +#define QAT_AES_HW_CONFIG_ENC(alg) \ + ICP_QAT_HW_CIPHER_CONFIG_BUILD(ICP_QAT_HW_CIPHER_CBC_MODE, alg, \ + ICP_QAT_HW_CIPHER_NO_CONVERT, \ + ICP_QAT_HW_CIPHER_ENCRYPT) + +#define QAT_AES_HW_CONFIG_DEC(alg) \ + ICP_QAT_HW_CIPHER_CONFIG_BUILD(ICP_QAT_HW_CIPHER_CBC_MODE, alg, \ + ICP_QAT_HW_CIPHER_KEY_CONVERT, \ + ICP_QAT_HW_CIPHER_DECRYPT) + +static atomic_t active_dev; + +struct qat_alg_buf { + uint32_t len; + uint32_t resrvd; + uint64_t addr; +} __packed; + +struct qat_alg_buf_list { + uint64_t resrvd; + uint32_t num_bufs; + uint32_t num_mapped_bufs; + struct qat_alg_buf bufers[]; +} __packed __aligned(64); + +/* Common content descriptor */ +struct qat_alg_cd { + union { + struct qat_enc { /* Encrypt content desc */ + struct icp_qat_hw_cipher_algo_blk cipher; + struct icp_qat_hw_auth_algo_blk hash; + } qat_enc_cd; + struct qat_dec { /* Decrytp content desc */ + struct icp_qat_hw_auth_algo_blk hash; + struct icp_qat_hw_cipher_algo_blk cipher; + } qat_dec_cd; + }; +} __aligned(64); + +#define MAX_AUTH_STATE_SIZE sizeof(struct icp_qat_hw_auth_algo_blk) + +struct qat_auth_state { + uint8_t data[MAX_AUTH_STATE_SIZE]; +} __aligned(64); + +struct qat_alg_session_ctx { + struct qat_alg_cd *enc_cd; + dma_addr_t enc_cd_paddr; + struct qat_alg_cd *dec_cd; + dma_addr_t dec_cd_paddr; + struct qat_auth_state *auth_hw_state_enc; + dma_addr_t auth_state_enc_paddr; + struct qat_auth_state *auth_hw_state_dec; + dma_addr_t auth_state_dec_paddr; + struct icp_qat_fw_la_bulk_req enc_fw_req_tmpl; + struct icp_qat_fw_la_bulk_req dec_fw_req_tmpl; + struct qat_crypto_instance *inst; + struct crypto_tfm *tfm; + struct crypto_shash *hash_tfm; + enum icp_qat_hw_auth_algo qat_hash_alg; + uint8_t salt[AES_BLOCK_SIZE]; + spinlock_t lock; /* protects qat_alg_session_ctx struct */ +}; + +static int get_current_node(void) +{ + return cpu_data(current_thread_info()->cpu).phys_proc_id; +} + +static int qat_get_inter_state_size(enum icp_qat_hw_auth_algo qat_hash_alg) +{ + switch (qat_hash_alg) { + case ICP_QAT_HW_AUTH_ALGO_SHA1: + return ICP_QAT_HW_SHA1_STATE1_SZ; + case ICP_QAT_HW_AUTH_ALGO_SHA256: + return ICP_QAT_HW_SHA256_STATE1_SZ; + case ICP_QAT_HW_AUTH_ALGO_SHA512: + return ICP_QAT_HW_SHA512_STATE1_SZ; + default: + return -EFAULT; + }; + return -EFAULT; +} + +static int qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk *hash, + struct qat_alg_session_ctx *ctx, + const uint8_t *auth_key, + unsigned int auth_keylen, uint8_t *auth_state) +{ + struct { + struct shash_desc shash; + char ctx[crypto_shash_descsize(ctx->hash_tfm)]; + } desc; + struct sha1_state sha1; + struct sha256_state sha256; + struct sha512_state sha512; + int block_size = crypto_shash_blocksize(ctx->hash_tfm); + int digest_size = crypto_shash_digestsize(ctx->hash_tfm); + uint8_t *ipad = auth_state; + uint8_t *opad = ipad + block_size; + __be32 *hash_state_out; + __be64 *hash512_state_out; + int i, offset; + + desc.shash.tfm = ctx->hash_tfm; + desc.shash.flags = 0x0; + + if (auth_keylen > block_size) { + char buff[SHA512_BLOCK_SIZE]; + int ret = crypto_shash_digest(&desc.shash, auth_key, + auth_keylen, buff); + if (ret) + return ret; + + memcpy(ipad, buff, digest_size); + memcpy(opad, buff, digest_size); + memset(ipad + digest_size, 0, block_size - digest_size); + memset(opad + digest_size, 0, block_size - digest_size); + } else { + memcpy(ipad, auth_key, auth_keylen); + memcpy(opad, auth_key, auth_keylen); + memset(ipad + auth_keylen, 0, block_size - auth_keylen); + memset(opad + auth_keylen, 0, block_size - auth_keylen); + } + + for (i = 0; i < block_size; i++) { + char *ipad_ptr = ipad + i; + char *opad_ptr = opad + i; + *ipad_ptr ^= 0x36; + *opad_ptr ^= 0x5C; + } + + if (crypto_shash_init(&desc.shash)) + return -EFAULT; + + if (crypto_shash_update(&desc.shash, ipad, block_size)) + return -EFAULT; + + hash_state_out = (__be32 *)hash->sha.state1; + hash512_state_out = (__be64 *)hash_state_out; + + switch (ctx->qat_hash_alg) { + case ICP_QAT_HW_AUTH_ALGO_SHA1: + if (crypto_shash_export(&desc.shash, &sha1)) + return -EFAULT; + for (i = 0; i < digest_size >> 2; i++, hash_state_out++) + *hash_state_out = cpu_to_be32(*(sha1.state + i)); + break; + case ICP_QAT_HW_AUTH_ALGO_SHA256: + if (crypto_shash_export(&desc.shash, &sha256)) + return -EFAULT; + for (i = 0; i < digest_size >> 2; i++, hash_state_out++) + *hash_state_out = cpu_to_be32(*(sha256.state + i)); + break; + case ICP_QAT_HW_AUTH_ALGO_SHA512: + if (crypto_shash_export(&desc.shash, &sha512)) + return -EFAULT; + for (i = 0; i < digest_size >> 3; i++, hash512_state_out++) + *hash512_state_out = cpu_to_be64(*(sha512.state + i)); + break; + default: + return -EFAULT; + } + + if (crypto_shash_init(&desc.shash)) + return -EFAULT; + + if (crypto_shash_update(&desc.shash, opad, block_size)) + return -EFAULT; + + offset = round_up(qat_get_inter_state_size(ctx->qat_hash_alg), 8); + hash_state_out = (__be32 *)(hash->sha.state1 + offset); + hash512_state_out = (__be64 *)hash_state_out; + + switch (ctx->qat_hash_alg) { + case ICP_QAT_HW_AUTH_ALGO_SHA1: + if (crypto_shash_export(&desc.shash, &sha1)) + return -EFAULT; + for (i = 0; i < digest_size >> 2; i++, hash_state_out++) + *hash_state_out = cpu_to_be32(*(sha1.state + i)); + break; + case ICP_QAT_HW_AUTH_ALGO_SHA256: + if (crypto_shash_export(&desc.shash, &sha256)) + return -EFAULT; + for (i = 0; i < digest_size >> 2; i++, hash_state_out++) + *hash_state_out = cpu_to_be32(*(sha256.state + i)); + break; + case ICP_QAT_HW_AUTH_ALGO_SHA512: + if (crypto_shash_export(&desc.shash, &sha512)) + return -EFAULT; + for (i = 0; i < digest_size >> 3; i++, hash512_state_out++) + *hash512_state_out = cpu_to_be64(*(sha512.state + i)); + break; + default: + return -EFAULT; + } + return 0; +} + +static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header) +{ + header->hdr_flags = + ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(ICP_QAT_FW_COMN_REQ_FLAG_SET); + header->service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_LA; + header->comn_req_flags = + ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR, + QAT_COMN_PTR_TYPE_SGL); + ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_DIGEST_IN_BUFFER); + ICP_QAT_FW_LA_PARTIAL_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_PARTIAL_NONE); + ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(header->serv_specif_flags, + ICP_QAT_FW_CIPH_IV_16BYTE_DATA); + ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_NO_PROTO); + ICP_QAT_FW_LA_UPDATE_STATE_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_NO_UPDATE_STATE); +} + +static int qat_alg_init_enc_session(struct qat_alg_session_ctx *ctx, + int alg, struct crypto_authenc_keys *keys) +{ + struct crypto_aead *aead_tfm = __crypto_aead_cast(ctx->tfm); + unsigned int digestsize = crypto_aead_crt(aead_tfm)->authsize; + struct qat_enc *enc_ctx = &ctx->enc_cd->qat_enc_cd; + struct icp_qat_hw_cipher_algo_blk *cipher = &enc_ctx->cipher; + struct icp_qat_hw_auth_algo_blk *hash = + (struct icp_qat_hw_auth_algo_blk *)((char *)enc_ctx + + sizeof(struct icp_qat_hw_auth_setup) + keys->enckeylen); + struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->enc_fw_req_tmpl; + struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; + struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; + void *ptr = &req_tmpl->cd_ctrl; + struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = ptr; + struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr; + struct icp_qat_fw_la_auth_req_params *auth_param = + (struct icp_qat_fw_la_auth_req_params *) + ((char *)&req_tmpl->serv_specif_rqpars + + sizeof(struct icp_qat_fw_la_cipher_req_params)); + + /* CD setup */ + cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_ENC(alg); + memcpy(cipher->aes.key, keys->enckey, keys->enckeylen); + hash->sha.inner_setup.auth_config.config = + ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1, + ctx->qat_hash_alg, digestsize); + hash->sha.inner_setup.auth_counter.counter = + cpu_to_be32(crypto_shash_blocksize(ctx->hash_tfm)); + + if (qat_alg_do_precomputes(hash, ctx, keys->authkey, keys->authkeylen, + (uint8_t *)ctx->auth_hw_state_enc)) + return -EFAULT; + + /* Request setup */ + qat_alg_init_common_hdr(header); + header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER_HASH; + ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_RET_AUTH_RES); + ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_NO_CMP_AUTH_RES); + cd_pars->u.s.content_desc_addr = ctx->enc_cd_paddr; + cd_pars->u.s.content_desc_params_sz = sizeof(struct qat_alg_cd) >> 3; + + /* Cipher CD config setup */ + cipher_cd_ctrl->cipher_key_sz = keys->enckeylen >> 3; + cipher_cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3; + cipher_cd_ctrl->cipher_cfg_offset = 0; + ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER); + ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_AUTH); + /* Auth CD config setup */ + hash_cd_ctrl->hash_cfg_offset = ((char *)hash - (char *)cipher) >> 3; + hash_cd_ctrl->hash_flags = ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED; + hash_cd_ctrl->inner_res_sz = digestsize; + hash_cd_ctrl->final_sz = digestsize; + + switch (ctx->qat_hash_alg) { + case ICP_QAT_HW_AUTH_ALGO_SHA1: + hash_cd_ctrl->inner_state1_sz = + round_up(ICP_QAT_HW_SHA1_STATE1_SZ, 8); + hash_cd_ctrl->inner_state2_sz = + round_up(ICP_QAT_HW_SHA1_STATE2_SZ, 8); + break; + case ICP_QAT_HW_AUTH_ALGO_SHA256: + hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA256_STATE1_SZ; + hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA256_STATE2_SZ; + break; + case ICP_QAT_HW_AUTH_ALGO_SHA512: + hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA512_STATE1_SZ; + hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA512_STATE2_SZ; + break; + default: + break; + } + hash_cd_ctrl->inner_state2_offset = hash_cd_ctrl->hash_cfg_offset + + ((sizeof(struct icp_qat_hw_auth_setup) + + round_up(hash_cd_ctrl->inner_state1_sz, 8)) >> 3); + auth_param->u1.auth_partial_st_prefix = ctx->auth_state_enc_paddr + + sizeof(struct icp_qat_hw_auth_counter) + + round_up(hash_cd_ctrl->inner_state1_sz, 8); + ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_AUTH); + ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR); + return 0; +} + +static int qat_alg_init_dec_session(struct qat_alg_session_ctx *ctx, + int alg, struct crypto_authenc_keys *keys) +{ + struct crypto_aead *aead_tfm = __crypto_aead_cast(ctx->tfm); + unsigned int digestsize = crypto_aead_crt(aead_tfm)->authsize; + struct qat_dec *dec_ctx = &ctx->dec_cd->qat_dec_cd; + struct icp_qat_hw_auth_algo_blk *hash = &dec_ctx->hash; + struct icp_qat_hw_cipher_algo_blk *cipher = + (struct icp_qat_hw_cipher_algo_blk *)((char *)dec_ctx + + sizeof(struct icp_qat_hw_auth_setup) + + roundup(crypto_shash_digestsize(ctx->hash_tfm), 8) * 2); + struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->dec_fw_req_tmpl; + struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; + struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; + void *ptr = &req_tmpl->cd_ctrl; + struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = ptr; + struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr; + struct icp_qat_fw_la_auth_req_params *auth_param = + (struct icp_qat_fw_la_auth_req_params *) + ((char *)&req_tmpl->serv_specif_rqpars + + sizeof(struct icp_qat_fw_la_cipher_req_params)); + + /* CD setup */ + cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_DEC(alg); + memcpy(cipher->aes.key, keys->enckey, keys->enckeylen); + hash->sha.inner_setup.auth_config.config = + ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1, + ctx->qat_hash_alg, + digestsize); + hash->sha.inner_setup.auth_counter.counter = + cpu_to_be32(crypto_shash_blocksize(ctx->hash_tfm)); + + if (qat_alg_do_precomputes(hash, ctx, keys->authkey, keys->authkeylen, + (uint8_t *)ctx->auth_hw_state_dec)) + return -EFAULT; + + /* Request setup */ + qat_alg_init_common_hdr(header); + header->service_cmd_id = ICP_QAT_FW_LA_CMD_HASH_CIPHER; + ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_NO_RET_AUTH_RES); + ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_CMP_AUTH_RES); + cd_pars->u.s.content_desc_addr = ctx->dec_cd_paddr; + cd_pars->u.s.content_desc_params_sz = sizeof(struct qat_alg_cd) >> 3; + + /* Cipher CD config setup */ + cipher_cd_ctrl->cipher_key_sz = keys->enckeylen >> 3; + cipher_cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3; + cipher_cd_ctrl->cipher_cfg_offset = + (sizeof(struct icp_qat_hw_auth_setup) + + roundup(crypto_shash_digestsize(ctx->hash_tfm), 8) * 2) >> 3; + ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER); + ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR); + + /* Auth CD config setup */ + hash_cd_ctrl->hash_cfg_offset = 0; + hash_cd_ctrl->hash_flags = ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED; + hash_cd_ctrl->inner_res_sz = digestsize; + hash_cd_ctrl->final_sz = digestsize; + + switch (ctx->qat_hash_alg) { + case ICP_QAT_HW_AUTH_ALGO_SHA1: + hash_cd_ctrl->inner_state1_sz = + round_up(ICP_QAT_HW_SHA1_STATE1_SZ, 8); + hash_cd_ctrl->inner_state2_sz = + round_up(ICP_QAT_HW_SHA1_STATE2_SZ, 8); + break; + case ICP_QAT_HW_AUTH_ALGO_SHA256: + hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA256_STATE1_SZ; + hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA256_STATE2_SZ; + break; + case ICP_QAT_HW_AUTH_ALGO_SHA512: + hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA512_STATE1_SZ; + hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA512_STATE2_SZ; + break; + default: + break; + } + + hash_cd_ctrl->inner_state2_offset = hash_cd_ctrl->hash_cfg_offset + + ((sizeof(struct icp_qat_hw_auth_setup) + + round_up(hash_cd_ctrl->inner_state1_sz, 8)) >> 3); + auth_param->u1.auth_partial_st_prefix = ctx->auth_state_enc_paddr + + sizeof(struct icp_qat_hw_auth_counter) + + round_up(hash_cd_ctrl->inner_state1_sz, 8); + auth_param->auth_res_sz = digestsize; + ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_AUTH); + ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER); + return 0; +} + +static int qat_alg_init_sessions(struct qat_alg_session_ctx *ctx, + const uint8_t *key, unsigned int keylen) +{ + struct crypto_authenc_keys keys; + int alg; + + if (crypto_rng_get_bytes(crypto_default_rng, ctx->salt, AES_BLOCK_SIZE)) + return -EFAULT; + + if (crypto_authenc_extractkeys(&keys, key, keylen)) + goto bad_key; + + switch (keys.enckeylen) { + case AES_KEYSIZE_128: + alg = ICP_QAT_HW_CIPHER_ALGO_AES128; + break; + case AES_KEYSIZE_192: + alg = ICP_QAT_HW_CIPHER_ALGO_AES192; + break; + case AES_KEYSIZE_256: + alg = ICP_QAT_HW_CIPHER_ALGO_AES256; + break; + default: + goto bad_key; + break; + } + + if (qat_alg_init_enc_session(ctx, alg, &keys)) + goto error; + + if (qat_alg_init_dec_session(ctx, alg, &keys)) + goto error; + + return 0; +bad_key: + crypto_tfm_set_flags(ctx->tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; +error: + return -EFAULT; +} + +static int qat_alg_setkey(struct crypto_aead *tfm, const uint8_t *key, + unsigned int keylen) +{ + struct qat_alg_session_ctx *ctx = crypto_aead_ctx(tfm); + struct device *dev; + + spin_lock(&ctx->lock); + if (ctx->enc_cd) { + /* rekeying */ + dev = &GET_DEV(ctx->inst->accel_dev); + memset(ctx->enc_cd, 0, sizeof(struct qat_alg_cd)); + memset(ctx->dec_cd, 0, sizeof(struct qat_alg_cd)); + memset(ctx->auth_hw_state_enc, 0, + sizeof(struct qat_auth_state)); + memset(ctx->auth_hw_state_dec, 0, + sizeof(struct qat_auth_state)); + memset(&ctx->enc_fw_req_tmpl, 0, + sizeof(struct icp_qat_fw_la_bulk_req)); + memset(&ctx->dec_fw_req_tmpl, 0, + sizeof(struct icp_qat_fw_la_bulk_req)); + } else { + /* new key */ + int node = get_current_node(); + struct qat_crypto_instance *inst = + qat_crypto_get_instance_node(node); + if (!inst) { + spin_unlock(&ctx->lock); + return -EINVAL; + } + + dev = &GET_DEV(inst->accel_dev); + ctx->inst = inst; + ctx->enc_cd = dma_zalloc_coherent(dev, + sizeof(struct qat_alg_cd), + &ctx->enc_cd_paddr, + GFP_ATOMIC); + if (!ctx->enc_cd) { + spin_unlock(&ctx->lock); + return -ENOMEM; + } + ctx->dec_cd = dma_zalloc_coherent(dev, + sizeof(struct qat_alg_cd), + &ctx->dec_cd_paddr, + GFP_ATOMIC); + if (!ctx->dec_cd) { + spin_unlock(&ctx->lock); + goto out_free_enc; + } + ctx->auth_hw_state_enc = + dma_zalloc_coherent(dev, sizeof(struct qat_auth_state), + &ctx->auth_state_enc_paddr, + GFP_ATOMIC); + if (!ctx->auth_hw_state_enc) { + spin_unlock(&ctx->lock); + goto out_free_dec; + } + ctx->auth_hw_state_dec = + dma_zalloc_coherent(dev, sizeof(struct qat_auth_state), + &ctx->auth_state_dec_paddr, + GFP_ATOMIC); + if (!ctx->auth_hw_state_dec) { + spin_unlock(&ctx->lock); + goto out_free_auth_enc; + } + } + spin_unlock(&ctx->lock); + if (qat_alg_init_sessions(ctx, key, keylen)) + goto out_free_all; + + return 0; + +out_free_all: + dma_free_coherent(dev, sizeof(struct qat_auth_state), + ctx->auth_hw_state_dec, ctx->auth_state_dec_paddr); + ctx->auth_hw_state_dec = NULL; +out_free_auth_enc: + dma_free_coherent(dev, sizeof(struct qat_auth_state), + ctx->auth_hw_state_enc, ctx->auth_state_enc_paddr); + ctx->auth_hw_state_enc = NULL; +out_free_dec: + dma_free_coherent(dev, sizeof(struct qat_alg_cd), + ctx->dec_cd, ctx->dec_cd_paddr); + ctx->dec_cd = NULL; +out_free_enc: + dma_free_coherent(dev, sizeof(struct qat_alg_cd), + ctx->enc_cd, ctx->enc_cd_paddr); + ctx->enc_cd = NULL; + return -ENOMEM; +} + +static void qat_alg_free_bufl(struct qat_crypto_instance *inst, + struct qat_crypto_request *qat_req) +{ + struct device *dev = &GET_DEV(inst->accel_dev); + struct qat_alg_buf_list *bl = qat_req->buf.bl; + struct qat_alg_buf_list *blout = qat_req->buf.blout; + dma_addr_t blp = qat_req->buf.blp; + dma_addr_t blpout = qat_req->buf.bloutp; + size_t sz = qat_req->buf.sz; + int i, bufs = bl->num_bufs; + + for (i = 0; i < bl->num_bufs; i++) + dma_unmap_single(dev, bl->bufers[i].addr, + bl->bufers[i].len, DMA_BIDIRECTIONAL); + + dma_unmap_single(dev, blp, sz, DMA_TO_DEVICE); + kfree(bl); + if (blp != blpout) { + /* If out of place operation dma unmap only data */ + int bufless = bufs - blout->num_mapped_bufs; + + for (i = bufless; i < bufs; i++) { + dma_unmap_single(dev, blout->bufers[i].addr, + blout->bufers[i].len, + DMA_BIDIRECTIONAL); + } + dma_unmap_single(dev, blpout, sz, DMA_TO_DEVICE); + kfree(blout); + } +} + +static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst, + struct scatterlist *assoc, + struct scatterlist *sgl, + struct scatterlist *sglout, uint8_t *iv, + uint8_t ivlen, + struct qat_crypto_request *qat_req) +{ + struct device *dev = &GET_DEV(inst->accel_dev); + int i, bufs = 0, n = sg_nents(sgl), assoc_n = sg_nents(assoc); + struct qat_alg_buf_list *bufl; + struct qat_alg_buf_list *buflout = NULL; + dma_addr_t blp; + dma_addr_t bloutp = 0; + struct scatterlist *sg; + size_t sz = sizeof(struct qat_alg_buf_list) + + ((1 + n + assoc_n) * sizeof(struct qat_alg_buf)); + + if (unlikely(!n)) + return -EINVAL; + + bufl = kmalloc_node(sz, GFP_ATOMIC, inst->accel_dev->numa_node); + if (unlikely(!bufl)) + return -ENOMEM; + + blp = dma_map_single(dev, bufl, sz, DMA_TO_DEVICE); + if (unlikely(dma_mapping_error(dev, blp))) + goto err; + + for_each_sg(assoc, sg, assoc_n, i) { + bufl->bufers[bufs].addr = dma_map_single(dev, + sg_virt(sg), + sg->length, + DMA_BIDIRECTIONAL); + bufl->bufers[bufs].len = sg->length; + if (unlikely(dma_mapping_error(dev, bufl->bufers[bufs].addr))) + goto err; + bufs++; + } + bufl->bufers[bufs].addr = dma_map_single(dev, iv, ivlen, + DMA_BIDIRECTIONAL); + bufl->bufers[bufs].len = ivlen; + if (unlikely(dma_mapping_error(dev, bufl->bufers[bufs].addr))) + goto err; + bufs++; + + for_each_sg(sgl, sg, n, i) { + int y = i + bufs; + + bufl->bufers[y].addr = dma_map_single(dev, sg_virt(sg), + sg->length, + DMA_BIDIRECTIONAL); + bufl->bufers[y].len = sg->length; + if (unlikely(dma_mapping_error(dev, bufl->bufers[y].addr))) + goto err; + } + bufl->num_bufs = n + bufs; + qat_req->buf.bl = bufl; + qat_req->buf.blp = blp; + qat_req->buf.sz = sz; + /* Handle out of place operation */ + if (sgl != sglout) { + struct qat_alg_buf *bufers; + + buflout = kmalloc_node(sz, GFP_ATOMIC, + inst->accel_dev->numa_node); + if (unlikely(!buflout)) + goto err; + bloutp = dma_map_single(dev, buflout, sz, DMA_TO_DEVICE); + if (unlikely(dma_mapping_error(dev, bloutp))) + goto err; + bufers = buflout->bufers; + /* For out of place operation dma map only data and + * reuse assoc mapping and iv */ + for (i = 0; i < bufs; i++) { + bufers[i].len = bufl->bufers[i].len; + bufers[i].addr = bufl->bufers[i].addr; + } + for_each_sg(sglout, sg, n, i) { + int y = i + bufs; + + bufers[y].addr = dma_map_single(dev, sg_virt(sg), + sg->length, + DMA_BIDIRECTIONAL); + buflout->bufers[y].len = sg->length; + if (unlikely(dma_mapping_error(dev, bufers[y].addr))) + goto err; + } + buflout->num_bufs = n + bufs; + buflout->num_mapped_bufs = n; + qat_req->buf.blout = buflout; + qat_req->buf.bloutp = bloutp; + } else { + /* Otherwise set the src and dst to the same address */ + qat_req->buf.bloutp = qat_req->buf.blp; + } + return 0; +err: + dev_err(dev, "Failed to map buf for dma\n"); + for_each_sg(sgl, sg, n + bufs, i) { + if (!dma_mapping_error(dev, bufl->bufers[i].addr)) { + dma_unmap_single(dev, bufl->bufers[i].addr, + bufl->bufers[i].len, + DMA_BIDIRECTIONAL); + } + } + if (!dma_mapping_error(dev, blp)) + dma_unmap_single(dev, blp, sz, DMA_TO_DEVICE); + kfree(bufl); + if (sgl != sglout && buflout) { + for_each_sg(sglout, sg, n, i) { + int y = i + bufs; + + if (!dma_mapping_error(dev, buflout->bufers[y].addr)) + dma_unmap_single(dev, buflout->bufers[y].addr, + buflout->bufers[y].len, + DMA_BIDIRECTIONAL); + } + if (!dma_mapping_error(dev, bloutp)) + dma_unmap_single(dev, bloutp, sz, DMA_TO_DEVICE); + kfree(buflout); + } + return -ENOMEM; +} + +void qat_alg_callback(void *resp) +{ + struct icp_qat_fw_la_resp *qat_resp = resp; + struct qat_crypto_request *qat_req = + (void *)(__force long)qat_resp->opaque_data; + struct qat_alg_session_ctx *ctx = qat_req->ctx; + struct qat_crypto_instance *inst = ctx->inst; + struct aead_request *areq = qat_req->areq; + uint8_t stat_filed = qat_resp->comn_resp.comn_status; + int res = 0, qat_res = ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(stat_filed); + + qat_alg_free_bufl(inst, qat_req); + if (unlikely(qat_res != ICP_QAT_FW_COMN_STATUS_FLAG_OK)) + res = -EBADMSG; + areq->base.complete(&areq->base, res); +} + +static int qat_alg_dec(struct aead_request *areq) +{ + struct crypto_aead *aead_tfm = crypto_aead_reqtfm(areq); + struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm); + struct qat_alg_session_ctx *ctx = crypto_tfm_ctx(tfm); + struct qat_crypto_request *qat_req = aead_request_ctx(areq); + struct icp_qat_fw_la_cipher_req_params *cipher_param; + struct icp_qat_fw_la_auth_req_params *auth_param; + struct icp_qat_fw_la_bulk_req *msg; + int digst_size = crypto_aead_crt(aead_tfm)->authsize; + int ret, ctr = 0; + + ret = qat_alg_sgl_to_bufl(ctx->inst, areq->assoc, areq->src, areq->dst, + areq->iv, AES_BLOCK_SIZE, qat_req); + if (unlikely(ret)) + return ret; + + msg = &qat_req->req; + *msg = ctx->dec_fw_req_tmpl; + qat_req->ctx = ctx; + qat_req->areq = areq; + qat_req->req.comn_mid.opaque_data = (uint64_t)(__force long)qat_req; + qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp; + qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp; + cipher_param = (void *)&qat_req->req.serv_specif_rqpars; + cipher_param->cipher_length = areq->cryptlen - digst_size; + cipher_param->cipher_offset = areq->assoclen + AES_BLOCK_SIZE; + memcpy(cipher_param->u.cipher_IV_array, areq->iv, AES_BLOCK_SIZE); + auth_param = (void *)((uint8_t *)cipher_param + sizeof(*cipher_param)); + auth_param->auth_off = 0; + auth_param->auth_len = areq->assoclen + + cipher_param->cipher_length + AES_BLOCK_SIZE; + do { + ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg); + } while (ret == -EAGAIN && ctr++ < 10); + + if (ret == -EAGAIN) { + qat_alg_free_bufl(ctx->inst, qat_req); + return -EBUSY; + } + return -EINPROGRESS; +} + +static int qat_alg_enc_internal(struct aead_request *areq, uint8_t *iv, + int enc_iv) +{ + struct crypto_aead *aead_tfm = crypto_aead_reqtfm(areq); + struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm); + struct qat_alg_session_ctx *ctx = crypto_tfm_ctx(tfm); + struct qat_crypto_request *qat_req = aead_request_ctx(areq); + struct icp_qat_fw_la_cipher_req_params *cipher_param; + struct icp_qat_fw_la_auth_req_params *auth_param; + struct icp_qat_fw_la_bulk_req *msg; + int ret, ctr = 0; + + ret = qat_alg_sgl_to_bufl(ctx->inst, areq->assoc, areq->src, areq->dst, + iv, AES_BLOCK_SIZE, qat_req); + if (unlikely(ret)) + return ret; + + msg = &qat_req->req; + *msg = ctx->enc_fw_req_tmpl; + qat_req->ctx = ctx; + qat_req->areq = areq; + qat_req->req.comn_mid.opaque_data = (uint64_t)(__force long)qat_req; + qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp; + qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp; + cipher_param = (void *)&qat_req->req.serv_specif_rqpars; + auth_param = (void *)((uint8_t *)cipher_param + sizeof(*cipher_param)); + + if (enc_iv) { + cipher_param->cipher_length = areq->cryptlen + AES_BLOCK_SIZE; + cipher_param->cipher_offset = areq->assoclen; + } else { + memcpy(cipher_param->u.cipher_IV_array, iv, AES_BLOCK_SIZE); + cipher_param->cipher_length = areq->cryptlen; + cipher_param->cipher_offset = areq->assoclen + AES_BLOCK_SIZE; + } + auth_param->auth_off = 0; + auth_param->auth_len = areq->assoclen + areq->cryptlen + AES_BLOCK_SIZE; + + do { + ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg); + } while (ret == -EAGAIN && ctr++ < 10); + + if (ret == -EAGAIN) { + qat_alg_free_bufl(ctx->inst, qat_req); + return -EBUSY; + } + return -EINPROGRESS; +} + +static int qat_alg_enc(struct aead_request *areq) +{ + return qat_alg_enc_internal(areq, areq->iv, 0); +} + +static int qat_alg_genivenc(struct aead_givcrypt_request *req) +{ + struct crypto_aead *aead_tfm = crypto_aead_reqtfm(&req->areq); + struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm); + struct qat_alg_session_ctx *ctx = crypto_tfm_ctx(tfm); + __be64 seq; + + memcpy(req->giv, ctx->salt, AES_BLOCK_SIZE); + seq = cpu_to_be64(req->seq); + memcpy(req->giv + AES_BLOCK_SIZE - sizeof(uint64_t), + &seq, sizeof(uint64_t)); + return qat_alg_enc_internal(&req->areq, req->giv, 1); +} + +static int qat_alg_init(struct crypto_tfm *tfm, + enum icp_qat_hw_auth_algo hash, const char *hash_name) +{ + struct qat_alg_session_ctx *ctx = crypto_tfm_ctx(tfm); + + memset(ctx, '\0', sizeof(*ctx)); + ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0); + if (IS_ERR(ctx->hash_tfm)) + return -EFAULT; + spin_lock_init(&ctx->lock); + ctx->qat_hash_alg = hash; + tfm->crt_aead.reqsize = sizeof(struct aead_request) + + sizeof(struct qat_crypto_request); + ctx->tfm = tfm; + return 0; +} + +static int qat_alg_sha1_init(struct crypto_tfm *tfm) +{ + return qat_alg_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA1, "sha1"); +} + +static int qat_alg_sha256_init(struct crypto_tfm *tfm) +{ + return qat_alg_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA256, "sha256"); +} + +static int qat_alg_sha512_init(struct crypto_tfm *tfm) +{ + return qat_alg_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA512, "sha512"); +} + +static void qat_alg_exit(struct crypto_tfm *tfm) +{ + struct qat_alg_session_ctx *ctx = crypto_tfm_ctx(tfm); + struct qat_crypto_instance *inst = ctx->inst; + struct device *dev; + + if (!IS_ERR(ctx->hash_tfm)) + crypto_free_shash(ctx->hash_tfm); + + if (!inst) + return; + + dev = &GET_DEV(inst->accel_dev); + if (ctx->enc_cd) + dma_free_coherent(dev, sizeof(struct qat_alg_cd), + ctx->enc_cd, ctx->enc_cd_paddr); + if (ctx->dec_cd) + dma_free_coherent(dev, sizeof(struct qat_alg_cd), + ctx->dec_cd, ctx->dec_cd_paddr); + if (ctx->auth_hw_state_enc) + dma_free_coherent(dev, sizeof(struct qat_auth_state), + ctx->auth_hw_state_enc, + ctx->auth_state_enc_paddr); + + if (ctx->auth_hw_state_dec) + dma_free_coherent(dev, sizeof(struct qat_auth_state), + ctx->auth_hw_state_dec, + ctx->auth_state_dec_paddr); + + qat_crypto_put_instance(inst); +} + +static struct crypto_alg qat_algs[] = { { + .cra_name = "authenc(hmac(sha1),cbc(aes))", + .cra_driver_name = "qat_aes_cbc_hmac_sha1", + .cra_priority = 4001, + .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct qat_alg_session_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_aead_type, + .cra_module = THIS_MODULE, + .cra_init = qat_alg_sha1_init, + .cra_exit = qat_alg_exit, + .cra_u = { + .aead = { + .setkey = qat_alg_setkey, + .decrypt = qat_alg_dec, + .encrypt = qat_alg_enc, + .givencrypt = qat_alg_genivenc, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + }, +}, { + .cra_name = "authenc(hmac(sha256),cbc(aes))", + .cra_driver_name = "qat_aes_cbc_hmac_sha256", + .cra_priority = 4001, + .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct qat_alg_session_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_aead_type, + .cra_module = THIS_MODULE, + .cra_init = qat_alg_sha256_init, + .cra_exit = qat_alg_exit, + .cra_u = { + .aead = { + .setkey = qat_alg_setkey, + .decrypt = qat_alg_dec, + .encrypt = qat_alg_enc, + .givencrypt = qat_alg_genivenc, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + }, +}, { + .cra_name = "authenc(hmac(sha512),cbc(aes))", + .cra_driver_name = "qat_aes_cbc_hmac_sha512", + .cra_priority = 4001, + .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct qat_alg_session_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_aead_type, + .cra_module = THIS_MODULE, + .cra_init = qat_alg_sha512_init, + .cra_exit = qat_alg_exit, + .cra_u = { + .aead = { + .setkey = qat_alg_setkey, + .decrypt = qat_alg_dec, + .encrypt = qat_alg_enc, + .givencrypt = qat_alg_genivenc, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + }, +} }; + +int qat_algs_register(void) +{ + if (atomic_add_return(1, &active_dev) == 1) { + int i; + + for (i = 0; i < ARRAY_SIZE(qat_algs); i++) + qat_algs[i].cra_flags = CRYPTO_ALG_TYPE_AEAD | + CRYPTO_ALG_ASYNC; + return crypto_register_algs(qat_algs, ARRAY_SIZE(qat_algs)); + } + return 0; +} + +int qat_algs_unregister(void) +{ + if (atomic_sub_return(1, &active_dev) == 0) + return crypto_unregister_algs(qat_algs, ARRAY_SIZE(qat_algs)); + return 0; +} + +int qat_algs_init(void) +{ + atomic_set(&active_dev, 0); + crypto_get_default_rng(); + return 0; +} + +void qat_algs_exit(void) +{ + crypto_put_default_rng(); +} diff --git a/drivers/crypto/qat/qat_common/qat_crypto.c b/drivers/crypto/qat/qat_common/qat_crypto.c new file mode 100644 index 000000000000..0d59bcb50de1 --- /dev/null +++ b/drivers/crypto/qat/qat_common/qat_crypto.c @@ -0,0 +1,284 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#include <linux/module.h> +#include <linux/slab.h> +#include "adf_accel_devices.h" +#include "adf_common_drv.h" +#include "adf_transport.h" +#include "adf_cfg.h" +#include "adf_cfg_strings.h" +#include "qat_crypto.h" +#include "icp_qat_fw.h" + +#define SEC ADF_KERNEL_SEC + +static struct service_hndl qat_crypto; + +void qat_crypto_put_instance(struct qat_crypto_instance *inst) +{ + if (atomic_sub_return(1, &inst->refctr) == 0) + adf_dev_put(inst->accel_dev); +} + +static int qat_crypto_free_instances(struct adf_accel_dev *accel_dev) +{ + struct qat_crypto_instance *inst; + struct list_head *list_ptr, *tmp; + int i; + + list_for_each_safe(list_ptr, tmp, &accel_dev->crypto_list) { + inst = list_entry(list_ptr, struct qat_crypto_instance, list); + + for (i = 0; i < atomic_read(&inst->refctr); i++) + qat_crypto_put_instance(inst); + + if (inst->sym_tx) + adf_remove_ring(inst->sym_tx); + + if (inst->sym_rx) + adf_remove_ring(inst->sym_rx); + + if (inst->pke_tx) + adf_remove_ring(inst->pke_tx); + + if (inst->pke_rx) + adf_remove_ring(inst->pke_rx); + + if (inst->rnd_tx) + adf_remove_ring(inst->rnd_tx); + + if (inst->rnd_rx) + adf_remove_ring(inst->rnd_rx); + + list_del(list_ptr); + kfree(inst); + } + return 0; +} + +struct qat_crypto_instance *qat_crypto_get_instance_node(int node) +{ + struct adf_accel_dev *accel_dev = NULL; + struct qat_crypto_instance *inst_best = NULL; + struct list_head *itr; + unsigned long best = ~0; + + list_for_each(itr, adf_devmgr_get_head()) { + accel_dev = list_entry(itr, struct adf_accel_dev, list); + if (accel_dev->numa_node == node && adf_dev_started(accel_dev)) + break; + accel_dev = NULL; + } + if (!accel_dev) { + pr_err("QAT: Could not find device on give node\n"); + accel_dev = adf_devmgr_get_first(); + } + if (!accel_dev || !adf_dev_started(accel_dev)) + return NULL; + + list_for_each(itr, &accel_dev->crypto_list) { + struct qat_crypto_instance *inst; + unsigned long cur; + + inst = list_entry(itr, struct qat_crypto_instance, list); + cur = atomic_read(&inst->refctr); + if (best > cur) { + inst_best = inst; + best = cur; + } + } + if (inst_best) { + if (atomic_add_return(1, &inst_best->refctr) == 1) { + if (adf_dev_get(accel_dev)) { + atomic_dec(&inst_best->refctr); + pr_err("QAT: Could increment dev refctr\n"); + return NULL; + } + } + } + return inst_best; +} + +static int qat_crypto_create_instances(struct adf_accel_dev *accel_dev) +{ + int i; + unsigned long bank; + unsigned long num_inst, num_msg_sym, num_msg_asym; + int msg_size; + struct qat_crypto_instance *inst; + char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; + char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; + + INIT_LIST_HEAD(&accel_dev->crypto_list); + strlcpy(key, ADF_NUM_CY, sizeof(key)); + + if (adf_cfg_get_param_value(accel_dev, SEC, key, val)) + return -EFAULT; + + if (kstrtoul(val, 0, &num_inst)) + return -EFAULT; + + for (i = 0; i < num_inst; i++) { + inst = kzalloc_node(sizeof(*inst), GFP_KERNEL, + accel_dev->numa_node); + if (!inst) + goto err; + + list_add_tail(&inst->list, &accel_dev->crypto_list); + inst->id = i; + atomic_set(&inst->refctr, 0); + inst->accel_dev = accel_dev; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_BANK_NUM, i); + if (adf_cfg_get_param_value(accel_dev, SEC, key, val)) + goto err; + + if (kstrtoul(val, 10, &bank)) + goto err; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, i); + if (adf_cfg_get_param_value(accel_dev, SEC, key, val)) + goto err; + + if (kstrtoul(val, 10, &num_msg_sym)) + goto err; + num_msg_sym = num_msg_sym >> 1; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, i); + if (adf_cfg_get_param_value(accel_dev, SEC, key, val)) + goto err; + + if (kstrtoul(val, 10, &num_msg_asym)) + goto err; + num_msg_asym = num_msg_asym >> 1; + + msg_size = ICP_QAT_FW_REQ_DEFAULT_SZ; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, i); + if (adf_create_ring(accel_dev, SEC, bank, num_msg_sym, + msg_size, key, NULL, 0, &inst->sym_tx)) + goto err; + + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_RND_TX, i); + if (adf_create_ring(accel_dev, SEC, bank, num_msg_asym, + msg_size, key, NULL, 0, &inst->rnd_tx)) + goto err; + + msg_size = msg_size >> 1; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, i); + if (adf_create_ring(accel_dev, SEC, bank, num_msg_asym, + msg_size, key, NULL, 0, &inst->pke_tx)) + goto err; + + msg_size = ICP_QAT_FW_RESP_DEFAULT_SZ; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, i); + if (adf_create_ring(accel_dev, SEC, bank, num_msg_sym, + msg_size, key, qat_alg_callback, 0, + &inst->sym_rx)) + goto err; + + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_RND_RX, i); + if (adf_create_ring(accel_dev, SEC, bank, num_msg_asym, + msg_size, key, qat_alg_callback, 0, + &inst->rnd_rx)) + goto err; + + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, i); + if (adf_create_ring(accel_dev, SEC, bank, num_msg_asym, + msg_size, key, qat_alg_callback, 0, + &inst->pke_rx)) + goto err; + } + return 0; +err: + qat_crypto_free_instances(accel_dev); + return -ENOMEM; +} + +static int qat_crypto_init(struct adf_accel_dev *accel_dev) +{ + if (qat_crypto_create_instances(accel_dev)) + return -EFAULT; + + return 0; +} + +static int qat_crypto_shutdown(struct adf_accel_dev *accel_dev) +{ + return qat_crypto_free_instances(accel_dev); +} + +static int qat_crypto_event_handler(struct adf_accel_dev *accel_dev, + enum adf_event event) +{ + int ret; + + switch (event) { + case ADF_EVENT_INIT: + ret = qat_crypto_init(accel_dev); + break; + case ADF_EVENT_SHUTDOWN: + ret = qat_crypto_shutdown(accel_dev); + break; + case ADF_EVENT_RESTARTING: + case ADF_EVENT_RESTARTED: + case ADF_EVENT_START: + case ADF_EVENT_STOP: + default: + ret = 0; + } + return ret; +} + +int qat_crypto_register(void) +{ + memset(&qat_crypto, 0, sizeof(qat_crypto)); + qat_crypto.event_hld = qat_crypto_event_handler; + qat_crypto.name = "qat_crypto"; + return adf_service_register(&qat_crypto); +} + +int qat_crypto_unregister(void) +{ + return adf_service_unregister(&qat_crypto); +} diff --git a/drivers/crypto/qat/qat_common/qat_crypto.h b/drivers/crypto/qat/qat_common/qat_crypto.h new file mode 100644 index 000000000000..ab8468d11ddb --- /dev/null +++ b/drivers/crypto/qat/qat_common/qat_crypto.h @@ -0,0 +1,83 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef _QAT_CRYPTO_INSTANCE_H_ +#define _QAT_CRYPTO_INSTANCE_H_ + +#include <linux/list.h> +#include <linux/slab.h> +#include "adf_accel_devices.h" +#include "icp_qat_fw_la.h" + +struct qat_crypto_instance { + struct adf_etr_ring_data *sym_tx; + struct adf_etr_ring_data *sym_rx; + struct adf_etr_ring_data *pke_tx; + struct adf_etr_ring_data *pke_rx; + struct adf_etr_ring_data *rnd_tx; + struct adf_etr_ring_data *rnd_rx; + struct adf_accel_dev *accel_dev; + struct list_head list; + unsigned long state; + int id; + atomic_t refctr; +}; + +struct qat_crypto_request_buffs { + struct qat_alg_buf_list *bl; + dma_addr_t blp; + struct qat_alg_buf_list *blout; + dma_addr_t bloutp; + size_t sz; +}; + +struct qat_crypto_request { + struct icp_qat_fw_la_bulk_req req; + struct qat_alg_session_ctx *ctx; + struct aead_request *areq; + struct qat_crypto_request_buffs buf; +}; +#endif diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c new file mode 100644 index 000000000000..9b8a31521ff3 --- /dev/null +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -0,0 +1,1393 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#include <linux/slab.h> + +#include "adf_accel_devices.h" +#include "adf_common_drv.h" +#include "icp_qat_hal.h" +#include "icp_qat_uclo.h" + +#define BAD_REGADDR 0xffff +#define MAX_RETRY_TIMES 10000 +#define INIT_CTX_ARB_VALUE 0x0 +#define INIT_CTX_ENABLE_VALUE 0x0 +#define INIT_PC_VALUE 0x0 +#define INIT_WAKEUP_EVENTS_VALUE 0x1 +#define INIT_SIG_EVENTS_VALUE 0x1 +#define INIT_CCENABLE_VALUE 0x2000 +#define RST_CSR_QAT_LSB 20 +#define RST_CSR_AE_LSB 0 +#define MC_TIMESTAMP_ENABLE (0x1 << 7) + +#define IGNORE_W1C_MASK ((~(1 << CE_BREAKPOINT_BITPOS)) & \ + (~(1 << CE_CNTL_STORE_PARITY_ERROR_BITPOS)) & \ + (~(1 << CE_REG_PAR_ERR_BITPOS))) +#define INSERT_IMMED_GPRA_CONST(inst, const_val) \ + (inst = ((inst & 0xFFFF00C03FFull) | \ + ((((const_val) << 12) & 0x0FF00000ull) | \ + (((const_val) << 10) & 0x0003FC00ull)))) +#define INSERT_IMMED_GPRB_CONST(inst, const_val) \ + (inst = ((inst & 0xFFFF00FFF00ull) | \ + ((((const_val) << 12) & 0x0FF00000ull) | \ + (((const_val) << 0) & 0x000000FFull)))) + +#define AE(handle, ae) handle->hal_handle->aes[ae] + +static const uint64_t inst_4b[] = { + 0x0F0400C0000ull, 0x0F4400C0000ull, 0x0F040000300ull, 0x0F440000300ull, + 0x0FC066C0000ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull, + 0x0A021000000ull +}; + +static const uint64_t inst[] = { + 0x0F0000C0000ull, 0x0F000000380ull, 0x0D805000011ull, 0x0FC082C0300ull, + 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull, + 0x0A0643C0000ull, 0x0BAC0000301ull, 0x0D802000101ull, 0x0F0000C0001ull, + 0x0FC066C0001ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull, + 0x0F000400300ull, 0x0A0610C0000ull, 0x0BAC0000301ull, 0x0D804400101ull, + 0x0A0580C0000ull, 0x0A0581C0000ull, 0x0A0582C0000ull, 0x0A0583C0000ull, + 0x0A0584C0000ull, 0x0A0585C0000ull, 0x0A0586C0000ull, 0x0A0587C0000ull, + 0x0A0588C0000ull, 0x0A0589C0000ull, 0x0A058AC0000ull, 0x0A058BC0000ull, + 0x0A058CC0000ull, 0x0A058DC0000ull, 0x0A058EC0000ull, 0x0A058FC0000ull, + 0x0A05C0C0000ull, 0x0A05C1C0000ull, 0x0A05C2C0000ull, 0x0A05C3C0000ull, + 0x0A05C4C0000ull, 0x0A05C5C0000ull, 0x0A05C6C0000ull, 0x0A05C7C0000ull, + 0x0A05C8C0000ull, 0x0A05C9C0000ull, 0x0A05CAC0000ull, 0x0A05CBC0000ull, + 0x0A05CCC0000ull, 0x0A05CDC0000ull, 0x0A05CEC0000ull, 0x0A05CFC0000ull, + 0x0A0400C0000ull, 0x0B0400C0000ull, 0x0A0401C0000ull, 0x0B0401C0000ull, + 0x0A0402C0000ull, 0x0B0402C0000ull, 0x0A0403C0000ull, 0x0B0403C0000ull, + 0x0A0404C0000ull, 0x0B0404C0000ull, 0x0A0405C0000ull, 0x0B0405C0000ull, + 0x0A0406C0000ull, 0x0B0406C0000ull, 0x0A0407C0000ull, 0x0B0407C0000ull, + 0x0A0408C0000ull, 0x0B0408C0000ull, 0x0A0409C0000ull, 0x0B0409C0000ull, + 0x0A040AC0000ull, 0x0B040AC0000ull, 0x0A040BC0000ull, 0x0B040BC0000ull, + 0x0A040CC0000ull, 0x0B040CC0000ull, 0x0A040DC0000ull, 0x0B040DC0000ull, + 0x0A040EC0000ull, 0x0B040EC0000ull, 0x0A040FC0000ull, 0x0B040FC0000ull, + 0x0D81581C010ull, 0x0E000010000ull, 0x0E000010000ull, +}; + +void qat_hal_set_live_ctx(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned int ctx_mask) +{ + AE(handle, ae).live_ctx_mask = ctx_mask; +} + +#define CSR_RETRY_TIMES 500 +static int qat_hal_rd_ae_csr(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned int csr, + unsigned int *value) +{ + unsigned int iterations = CSR_RETRY_TIMES; + + do { + *value = GET_AE_CSR(handle, ae, csr); + if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS)) + return 0; + } while (iterations--); + + pr_err("QAT: Read CSR timeout\n"); + return -EFAULT; +} + +static int qat_hal_wr_ae_csr(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned int csr, + unsigned int value) +{ + unsigned int iterations = CSR_RETRY_TIMES; + + do { + SET_AE_CSR(handle, ae, csr, value); + if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS)) + return 0; + } while (iterations--); + + pr_err("QAT: Write CSR Timeout\n"); + return -EFAULT; +} + +static void qat_hal_get_wakeup_event(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned char ctx, + unsigned int *events) +{ + unsigned int cur_ctx; + + qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx); + qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx); + qat_hal_rd_ae_csr(handle, ae, CTX_WAKEUP_EVENTS_INDIRECT, events); + qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx); +} + +static int qat_hal_wait_cycles(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned int cycles, + int chk_inactive) +{ + unsigned int base_cnt = 0, cur_cnt = 0; + unsigned int csr = (1 << ACS_ABO_BITPOS); + int times = MAX_RETRY_TIMES; + int elapsed_cycles = 0; + + qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT, &base_cnt); + base_cnt &= 0xffff; + while ((int)cycles > elapsed_cycles && times--) { + if (chk_inactive) + qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &csr); + + qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT, &cur_cnt); + cur_cnt &= 0xffff; + elapsed_cycles = cur_cnt - base_cnt; + + if (elapsed_cycles < 0) + elapsed_cycles += 0x10000; + + /* ensure at least 8 time cycles elapsed in wait_cycles */ + if (elapsed_cycles >= 8 && !(csr & (1 << ACS_ABO_BITPOS))) + return 0; + } + if (!times) { + pr_err("QAT: wait_num_cycles time out\n"); + return -EFAULT; + } + return 0; +} + +#define CLR_BIT(wrd, bit) (wrd & ~(1 << bit)) +#define SET_BIT(wrd, bit) (wrd | 1 << bit) + +int qat_hal_set_ae_ctx_mode(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned char mode) +{ + unsigned int csr, new_csr; + + if ((mode != 4) && (mode != 8)) { + pr_err("QAT: bad ctx mode=%d\n", mode); + return -EINVAL; + } + + /* Sets the accelaration engine context mode to either four or eight */ + qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr); + csr = IGNORE_W1C_MASK & csr; + new_csr = (mode == 4) ? + SET_BIT(csr, CE_INUSE_CONTEXTS_BITPOS) : + CLR_BIT(csr, CE_INUSE_CONTEXTS_BITPOS); + qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr); + return 0; +} + +int qat_hal_set_ae_nn_mode(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned char mode) +{ + unsigned int csr, new_csr; + + qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr); + csr &= IGNORE_W1C_MASK; + + new_csr = (mode) ? + SET_BIT(csr, CE_NN_MODE_BITPOS) : + CLR_BIT(csr, CE_NN_MODE_BITPOS); + + if (new_csr != csr) + qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr); + + return 0; +} + +int qat_hal_set_ae_lm_mode(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, enum icp_qat_uof_regtype lm_type, + unsigned char mode) +{ + unsigned int csr, new_csr; + + qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr); + csr &= IGNORE_W1C_MASK; + switch (lm_type) { + case ICP_LMEM0: + new_csr = (mode) ? + SET_BIT(csr, CE_LMADDR_0_GLOBAL_BITPOS) : + CLR_BIT(csr, CE_LMADDR_0_GLOBAL_BITPOS); + break; + case ICP_LMEM1: + new_csr = (mode) ? + SET_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS) : + CLR_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS); + break; + default: + pr_err("QAT: lmType = 0x%x\n", lm_type); + return -EINVAL; + } + + if (new_csr != csr) + qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr); + return 0; +} + +static unsigned short qat_hal_get_reg_addr(unsigned int type, + unsigned short reg_num) +{ + unsigned short reg_addr; + + switch (type) { + case ICP_GPA_ABS: + case ICP_GPB_ABS: + reg_addr = 0x80 | (reg_num & 0x7f); + break; + case ICP_GPA_REL: + case ICP_GPB_REL: + reg_addr = reg_num & 0x1f; + break; + case ICP_SR_RD_REL: + case ICP_SR_WR_REL: + case ICP_SR_REL: + reg_addr = 0x180 | (reg_num & 0x1f); + break; + case ICP_SR_ABS: + reg_addr = 0x140 | ((reg_num & 0x3) << 1); + break; + case ICP_DR_RD_REL: + case ICP_DR_WR_REL: + case ICP_DR_REL: + reg_addr = 0x1c0 | (reg_num & 0x1f); + break; + case ICP_DR_ABS: + reg_addr = 0x100 | ((reg_num & 0x3) << 1); + break; + case ICP_NEIGH_REL: + reg_addr = 0x280 | (reg_num & 0x1f); + break; + case ICP_LMEM0: + reg_addr = 0x200; + break; + case ICP_LMEM1: + reg_addr = 0x220; + break; + case ICP_NO_DEST: + reg_addr = 0x300 | (reg_num & 0xff); + break; + default: + reg_addr = BAD_REGADDR; + break; + } + return reg_addr; +} + +void qat_hal_reset(struct icp_qat_fw_loader_handle *handle) +{ + unsigned int ae_reset_csr; + + ae_reset_csr = GET_GLB_CSR(handle, ICP_RESET); + ae_reset_csr |= handle->hal_handle->ae_mask << RST_CSR_AE_LSB; + ae_reset_csr |= handle->hal_handle->slice_mask << RST_CSR_QAT_LSB; + SET_GLB_CSR(handle, ICP_RESET, ae_reset_csr); +} + +static void qat_hal_wr_indr_csr(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned int ctx_mask, + unsigned int ae_csr, unsigned int csr_val) +{ + unsigned int ctx, cur_ctx; + + qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx); + + for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) { + if (!(ctx_mask & (1 << ctx))) + continue; + qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx); + qat_hal_wr_ae_csr(handle, ae, ae_csr, csr_val); + } + + qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx); +} + +static void qat_hal_rd_indr_csr(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned char ctx, + unsigned int ae_csr, unsigned int *csr_val) +{ + unsigned int cur_ctx; + + qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx); + qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx); + qat_hal_rd_ae_csr(handle, ae, ae_csr, csr_val); + qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx); +} + +static void qat_hal_put_sig_event(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned int ctx_mask, + unsigned int events) +{ + unsigned int ctx, cur_ctx; + + qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx); + for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) { + if (!(ctx_mask & (1 << ctx))) + continue; + qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx); + qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_INDIRECT, events); + } + qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx); +} + +static void qat_hal_put_wakeup_event(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned int ctx_mask, + unsigned int events) +{ + unsigned int ctx, cur_ctx; + + qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx); + for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) { + if (!(ctx_mask & (1 << ctx))) + continue; + qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx); + qat_hal_wr_ae_csr(handle, ae, CTX_WAKEUP_EVENTS_INDIRECT, + events); + } + qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx); +} + +static int qat_hal_check_ae_alive(struct icp_qat_fw_loader_handle *handle) +{ + unsigned int base_cnt, cur_cnt; + unsigned char ae; + unsigned int times = MAX_RETRY_TIMES; + + for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { + if (!(handle->hal_handle->ae_mask & (1 << ae))) + continue; + + qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT, + (unsigned int *)&base_cnt); + base_cnt &= 0xffff; + + do { + qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT, + (unsigned int *)&cur_cnt); + cur_cnt &= 0xffff; + } while (times-- && (cur_cnt == base_cnt)); + + if (!times) { + pr_err("QAT: AE%d is inactive!!\n", ae); + return -EFAULT; + } + } + + return 0; +} + +static void qat_hal_reset_timestamp(struct icp_qat_fw_loader_handle *handle) +{ + unsigned int misc_ctl; + unsigned char ae; + + /* stop the timestamp timers */ + misc_ctl = GET_GLB_CSR(handle, MISC_CONTROL); + if (misc_ctl & MC_TIMESTAMP_ENABLE) + SET_GLB_CSR(handle, MISC_CONTROL, misc_ctl & + (~MC_TIMESTAMP_ENABLE)); + + for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { + if (!(handle->hal_handle->ae_mask & (1 << ae))) + continue; + qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_LOW, 0); + qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_HIGH, 0); + } + /* start timestamp timers */ + SET_GLB_CSR(handle, MISC_CONTROL, misc_ctl | MC_TIMESTAMP_ENABLE); +} + +#define ESRAM_AUTO_TINIT (1<<2) +#define ESRAM_AUTO_TINIT_DONE (1<<3) +#define ESRAM_AUTO_INIT_USED_CYCLES (1640) +#define ESRAM_AUTO_INIT_CSR_OFFSET 0xC1C +static int qat_hal_init_esram(struct icp_qat_fw_loader_handle *handle) +{ + void __iomem *csr_addr = handle->hal_ep_csr_addr_v + + ESRAM_AUTO_INIT_CSR_OFFSET; + unsigned int csr_val, times = 30; + + csr_val = ADF_CSR_RD(csr_addr, 0); + if ((csr_val & ESRAM_AUTO_TINIT) && (csr_val & ESRAM_AUTO_TINIT_DONE)) + return 0; + + csr_val = ADF_CSR_RD(csr_addr, 0); + csr_val |= ESRAM_AUTO_TINIT; + ADF_CSR_WR(csr_addr, 0, csr_val); + + do { + qat_hal_wait_cycles(handle, 0, ESRAM_AUTO_INIT_USED_CYCLES, 0); + csr_val = ADF_CSR_RD(csr_addr, 0); + } while (!(csr_val & ESRAM_AUTO_TINIT_DONE) && times--); + if ((!times)) { + pr_err("QAT: Fail to init eSram!\n"); + return -EFAULT; + } + return 0; +} + +#define SHRAM_INIT_CYCLES 2060 +int qat_hal_clr_reset(struct icp_qat_fw_loader_handle *handle) +{ + unsigned int ae_reset_csr; + unsigned char ae; + unsigned int clk_csr; + unsigned int times = 100; + unsigned int csr; + + /* write to the reset csr */ + ae_reset_csr = GET_GLB_CSR(handle, ICP_RESET); + ae_reset_csr &= ~(handle->hal_handle->ae_mask << RST_CSR_AE_LSB); + ae_reset_csr &= ~(handle->hal_handle->slice_mask << RST_CSR_QAT_LSB); + do { + SET_GLB_CSR(handle, ICP_RESET, ae_reset_csr); + if (!(times--)) + goto out_err; + csr = GET_GLB_CSR(handle, ICP_RESET); + } while ((handle->hal_handle->ae_mask | + (handle->hal_handle->slice_mask << RST_CSR_QAT_LSB)) & csr); + /* enable clock */ + clk_csr = GET_GLB_CSR(handle, ICP_GLOBAL_CLK_ENABLE); + clk_csr |= handle->hal_handle->ae_mask << 0; + clk_csr |= handle->hal_handle->slice_mask << 20; + SET_GLB_CSR(handle, ICP_GLOBAL_CLK_ENABLE, clk_csr); + if (qat_hal_check_ae_alive(handle)) + goto out_err; + + /* Set undefined power-up/reset states to reasonable default values */ + for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { + if (!(handle->hal_handle->ae_mask & (1 << ae))) + continue; + qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, + INIT_CTX_ENABLE_VALUE); + qat_hal_wr_indr_csr(handle, ae, ICP_QAT_UCLO_AE_ALL_CTX, + CTX_STS_INDIRECT, + handle->hal_handle->upc_mask & + INIT_PC_VALUE); + qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, INIT_CTX_ARB_VALUE); + qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, INIT_CCENABLE_VALUE); + qat_hal_put_wakeup_event(handle, ae, + ICP_QAT_UCLO_AE_ALL_CTX, + INIT_WAKEUP_EVENTS_VALUE); + qat_hal_put_sig_event(handle, ae, + ICP_QAT_UCLO_AE_ALL_CTX, + INIT_SIG_EVENTS_VALUE); + } + if (qat_hal_init_esram(handle)) + goto out_err; + if (qat_hal_wait_cycles(handle, 0, SHRAM_INIT_CYCLES, 0)) + goto out_err; + qat_hal_reset_timestamp(handle); + + return 0; +out_err: + pr_err("QAT: failed to get device out of reset\n"); + return -EFAULT; +} + +static void qat_hal_disable_ctx(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned int ctx_mask) +{ + unsigned int ctx; + + qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx); + ctx &= IGNORE_W1C_MASK & + (~((ctx_mask & ICP_QAT_UCLO_AE_ALL_CTX) << CE_ENABLE_BITPOS)); + qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx); +} + +static uint64_t qat_hal_parity_64bit(uint64_t word) +{ + word ^= word >> 1; + word ^= word >> 2; + word ^= word >> 4; + word ^= word >> 8; + word ^= word >> 16; + word ^= word >> 32; + return word & 1; +} + +static uint64_t qat_hal_set_uword_ecc(uint64_t uword) +{ + uint64_t bit0_mask = 0xff800007fffULL, bit1_mask = 0x1f801ff801fULL, + bit2_mask = 0xe387e0781e1ULL, bit3_mask = 0x7cb8e388e22ULL, + bit4_mask = 0xaf5b2c93244ULL, bit5_mask = 0xf56d5525488ULL, + bit6_mask = 0xdaf69a46910ULL; + + /* clear the ecc bits */ + uword &= ~(0x7fULL << 0x2C); + uword |= qat_hal_parity_64bit(bit0_mask & uword) << 0x2C; + uword |= qat_hal_parity_64bit(bit1_mask & uword) << 0x2D; + uword |= qat_hal_parity_64bit(bit2_mask & uword) << 0x2E; + uword |= qat_hal_parity_64bit(bit3_mask & uword) << 0x2F; + uword |= qat_hal_parity_64bit(bit4_mask & uword) << 0x30; + uword |= qat_hal_parity_64bit(bit5_mask & uword) << 0x31; + uword |= qat_hal_parity_64bit(bit6_mask & uword) << 0x32; + return uword; +} + +void qat_hal_wr_uwords(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned int uaddr, + unsigned int words_num, uint64_t *uword) +{ + unsigned int ustore_addr; + unsigned int i; + + qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS, &ustore_addr); + uaddr |= UA_ECS; + qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr); + for (i = 0; i < words_num; i++) { + unsigned int uwrd_lo, uwrd_hi; + uint64_t tmp; + + tmp = qat_hal_set_uword_ecc(uword[i]); + uwrd_lo = (unsigned int)(tmp & 0xffffffff); + uwrd_hi = (unsigned int)(tmp >> 0x20); + qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo); + qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi); + } + qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr); +} + +static void qat_hal_enable_ctx(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned int ctx_mask) +{ + unsigned int ctx; + + qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx); + ctx &= IGNORE_W1C_MASK; + ctx_mask &= (ctx & CE_INUSE_CONTEXTS) ? 0x55 : 0xFF; + ctx |= (ctx_mask << CE_ENABLE_BITPOS); + qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx); +} + +static int qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle) +{ + unsigned char ae; + unsigned int ctx_mask = ICP_QAT_UCLO_AE_ALL_CTX; + int times = MAX_RETRY_TIMES; + unsigned int csr_val = 0; + unsigned short reg; + unsigned int savctx = 0; + int ret = 0; + + for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { + if (!(handle->hal_handle->ae_mask & (1 << ae))) + continue; + for (reg = 0; reg < ICP_QAT_UCLO_MAX_GPR_REG; reg++) { + qat_hal_init_rd_xfer(handle, ae, 0, ICP_SR_RD_ABS, + reg, 0); + qat_hal_init_rd_xfer(handle, ae, 0, ICP_DR_RD_ABS, + reg, 0); + } + qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &csr_val); + csr_val &= ~(1 << MMC_SHARE_CS_BITPOS); + qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, csr_val); + qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr_val); + csr_val &= IGNORE_W1C_MASK; + csr_val |= CE_NN_MODE; + qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, csr_val); + qat_hal_wr_uwords(handle, ae, 0, ARRAY_SIZE(inst), + (uint64_t *)inst); + qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT, + handle->hal_handle->upc_mask & + INIT_PC_VALUE); + qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &savctx); + qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, 0); + qat_hal_put_wakeup_event(handle, ae, ctx_mask, XCWE_VOLUNTARY); + qat_hal_wr_indr_csr(handle, ae, ctx_mask, + CTX_SIG_EVENTS_INDIRECT, 0); + qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0); + qat_hal_enable_ctx(handle, ae, ctx_mask); + } + for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { + if (!(handle->hal_handle->ae_mask & (1 << ae))) + continue; + /* wait for AE to finish */ + do { + ret = qat_hal_wait_cycles(handle, ae, 20, 1); + } while (ret && times--); + + if (!times) { + pr_err("QAT: clear GPR of AE %d failed", ae); + return -EINVAL; + } + qat_hal_disable_ctx(handle, ae, ctx_mask); + qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, + savctx & ACS_ACNO); + qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, + INIT_CTX_ENABLE_VALUE); + qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT, + handle->hal_handle->upc_mask & + INIT_PC_VALUE); + qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, INIT_CTX_ARB_VALUE); + qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, INIT_CCENABLE_VALUE); + qat_hal_put_wakeup_event(handle, ae, ctx_mask, + INIT_WAKEUP_EVENTS_VALUE); + qat_hal_put_sig_event(handle, ae, ctx_mask, + INIT_SIG_EVENTS_VALUE); + } + return 0; +} + +#define ICP_DH895XCC_AE_OFFSET 0x20000 +#define ICP_DH895XCC_CAP_OFFSET (ICP_DH895XCC_AE_OFFSET + 0x10000) +#define LOCAL_TO_XFER_REG_OFFSET 0x800 +#define ICP_DH895XCC_EP_OFFSET 0x3a000 +#define ICP_DH895XCC_PMISC_BAR 1 +int qat_hal_init(struct adf_accel_dev *accel_dev) +{ + unsigned char ae; + unsigned int max_en_ae_id = 0; + struct icp_qat_fw_loader_handle *handle; + struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_bar *bar = &pci_info->pci_bars[ADF_DH895XCC_PMISC_BAR]; + + handle = kzalloc(sizeof(*handle), GFP_KERNEL); + if (!handle) + return -ENOMEM; + + handle->hal_cap_g_ctl_csr_addr_v = bar->virt_addr + + ICP_DH895XCC_CAP_OFFSET; + handle->hal_cap_ae_xfer_csr_addr_v = bar->virt_addr + + ICP_DH895XCC_AE_OFFSET; + handle->hal_ep_csr_addr_v = bar->virt_addr + ICP_DH895XCC_EP_OFFSET; + handle->hal_cap_ae_local_csr_addr_v = + handle->hal_cap_ae_xfer_csr_addr_v + LOCAL_TO_XFER_REG_OFFSET; + + handle->hal_handle = kzalloc(sizeof(*handle->hal_handle), GFP_KERNEL); + if (!handle->hal_handle) + goto out_hal_handle; + handle->hal_handle->revision_id = accel_dev->accel_pci_dev.revid; + handle->hal_handle->ae_mask = hw_data->ae_mask; + handle->hal_handle->slice_mask = hw_data->accel_mask; + /* create AE objects */ + handle->hal_handle->upc_mask = 0x1ffff; + handle->hal_handle->max_ustore = 0x4000; + for (ae = 0; ae < ICP_QAT_UCLO_MAX_AE; ae++) { + if (!(hw_data->ae_mask & (1 << ae))) + continue; + handle->hal_handle->aes[ae].free_addr = 0; + handle->hal_handle->aes[ae].free_size = + handle->hal_handle->max_ustore; + handle->hal_handle->aes[ae].ustore_size = + handle->hal_handle->max_ustore; + handle->hal_handle->aes[ae].live_ctx_mask = + ICP_QAT_UCLO_AE_ALL_CTX; + max_en_ae_id = ae; + } + handle->hal_handle->ae_max_num = max_en_ae_id + 1; + /* take all AEs out of reset */ + if (qat_hal_clr_reset(handle)) { + pr_err("QAT: qat_hal_clr_reset error\n"); + goto out_err; + } + if (qat_hal_clear_gpr(handle)) + goto out_err; + /* Set SIGNATURE_ENABLE[0] to 0x1 in order to enable ALU_OUT csr */ + for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { + unsigned int csr_val = 0; + + if (!(hw_data->ae_mask & (1 << ae))) + continue; + qat_hal_rd_ae_csr(handle, ae, SIGNATURE_ENABLE, &csr_val); + csr_val |= 0x1; + qat_hal_wr_ae_csr(handle, ae, SIGNATURE_ENABLE, csr_val); + } + accel_dev->fw_loader->fw_loader = handle; + return 0; + +out_err: + kfree(handle->hal_handle); +out_hal_handle: + kfree(handle); + return -EFAULT; +} + +void qat_hal_deinit(struct icp_qat_fw_loader_handle *handle) +{ + if (!handle) + return; + kfree(handle->hal_handle); + kfree(handle); +} + +void qat_hal_start(struct icp_qat_fw_loader_handle *handle, unsigned char ae, + unsigned int ctx_mask) +{ + qat_hal_put_wakeup_event(handle, ae, (~ctx_mask) & + ICP_QAT_UCLO_AE_ALL_CTX, 0x10000); + qat_hal_enable_ctx(handle, ae, ctx_mask); +} + +void qat_hal_stop(struct icp_qat_fw_loader_handle *handle, unsigned char ae, + unsigned int ctx_mask) +{ + qat_hal_disable_ctx(handle, ae, ctx_mask); +} + +void qat_hal_set_pc(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned int ctx_mask, unsigned int upc) +{ + qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT, + handle->hal_handle->upc_mask & upc); +} + +static void qat_hal_get_uwords(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned int uaddr, + unsigned int words_num, uint64_t *uword) +{ + unsigned int i, uwrd_lo, uwrd_hi; + unsigned int ustore_addr, misc_control; + + qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &misc_control); + qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, + misc_control & 0xfffffffb); + qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS, &ustore_addr); + uaddr |= UA_ECS; + for (i = 0; i < words_num; i++) { + qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr); + uaddr++; + qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_LOWER, &uwrd_lo); + qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_UPPER, &uwrd_hi); + uword[i] = uwrd_hi; + uword[i] = (uword[i] << 0x20) | uwrd_lo; + } + qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, misc_control); + qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr); +} + +void qat_hal_wr_umem(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned int uaddr, + unsigned int words_num, unsigned int *data) +{ + unsigned int i, ustore_addr; + + qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS, &ustore_addr); + uaddr |= UA_ECS; + qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr); + for (i = 0; i < words_num; i++) { + unsigned int uwrd_lo, uwrd_hi, tmp; + + uwrd_lo = ((data[i] & 0xfff0000) << 4) | (0x3 << 18) | + ((data[i] & 0xff00) << 2) | + (0x3 << 8) | (data[i] & 0xff); + uwrd_hi = (0xf << 4) | ((data[i] & 0xf0000000) >> 28); + uwrd_hi |= (hweight32(data[i] & 0xffff) & 0x1) << 8; + tmp = ((data[i] >> 0x10) & 0xffff); + uwrd_hi |= (hweight32(tmp) & 0x1) << 9; + qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo); + qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi); + } + qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr); +} + +#define MAX_EXEC_INST 100 +static int qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned char ctx, + uint64_t *micro_inst, unsigned int inst_num, + int code_off, unsigned int max_cycle, + unsigned int *endpc) +{ + uint64_t savuwords[MAX_EXEC_INST]; + unsigned int ind_lm_addr0, ind_lm_addr1; + unsigned int ind_lm_addr_byte0, ind_lm_addr_byte1; + unsigned int ind_cnt_sig; + unsigned int ind_sig, act_sig; + unsigned int csr_val = 0, newcsr_val; + unsigned int savctx; + unsigned int savcc, wakeup_events, savpc; + unsigned int ctxarb_ctl, ctx_enables; + + if ((inst_num > handle->hal_handle->max_ustore) || !micro_inst) { + pr_err("QAT: invalid instruction num %d\n", inst_num); + return -EINVAL; + } + /* save current context */ + qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_0_INDIRECT, &ind_lm_addr0); + qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_1_INDIRECT, &ind_lm_addr1); + qat_hal_rd_indr_csr(handle, ae, ctx, INDIRECT_LM_ADDR_0_BYTE_INDEX, + &ind_lm_addr_byte0); + qat_hal_rd_indr_csr(handle, ae, ctx, INDIRECT_LM_ADDR_1_BYTE_INDEX, + &ind_lm_addr_byte1); + if (inst_num <= MAX_EXEC_INST) + qat_hal_get_uwords(handle, ae, 0, inst_num, savuwords); + qat_hal_get_wakeup_event(handle, ae, ctx, &wakeup_events); + qat_hal_rd_indr_csr(handle, ae, ctx, CTX_STS_INDIRECT, &savpc); + savpc = (savpc & handle->hal_handle->upc_mask) >> 0; + qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables); + ctx_enables &= IGNORE_W1C_MASK; + qat_hal_rd_ae_csr(handle, ae, CC_ENABLE, &savcc); + qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &savctx); + qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL, &ctxarb_ctl); + qat_hal_rd_indr_csr(handle, ae, ctx, FUTURE_COUNT_SIGNAL_INDIRECT, + &ind_cnt_sig); + qat_hal_rd_indr_csr(handle, ae, ctx, CTX_SIG_EVENTS_INDIRECT, &ind_sig); + qat_hal_rd_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, &act_sig); + /* execute micro codes */ + qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables); + qat_hal_wr_uwords(handle, ae, 0, inst_num, micro_inst); + qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_STS_INDIRECT, 0); + qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, ctx & ACS_ACNO); + if (code_off) + qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc & 0xffffdfff); + qat_hal_put_wakeup_event(handle, ae, (1 << ctx), XCWE_VOLUNTARY); + qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_SIG_EVENTS_INDIRECT, 0); + qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0); + qat_hal_enable_ctx(handle, ae, (1 << ctx)); + /* wait for micro codes to finish */ + if (qat_hal_wait_cycles(handle, ae, max_cycle, 1) != 0) + return -EFAULT; + if (endpc) { + unsigned int ctx_status; + + qat_hal_rd_indr_csr(handle, ae, ctx, CTX_STS_INDIRECT, + &ctx_status); + *endpc = ctx_status & handle->hal_handle->upc_mask; + } + /* retore to saved context */ + qat_hal_disable_ctx(handle, ae, (1 << ctx)); + if (inst_num <= MAX_EXEC_INST) + qat_hal_wr_uwords(handle, ae, 0, inst_num, savuwords); + qat_hal_put_wakeup_event(handle, ae, (1 << ctx), wakeup_events); + qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_STS_INDIRECT, + handle->hal_handle->upc_mask & savpc); + qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &csr_val); + newcsr_val = CLR_BIT(csr_val, MMC_SHARE_CS_BITPOS); + qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, newcsr_val); + qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc); + qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, savctx & ACS_ACNO); + qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, ctxarb_ctl); + qat_hal_wr_indr_csr(handle, ae, (1 << ctx), + LM_ADDR_0_INDIRECT, ind_lm_addr0); + qat_hal_wr_indr_csr(handle, ae, (1 << ctx), + LM_ADDR_1_INDIRECT, ind_lm_addr1); + qat_hal_wr_indr_csr(handle, ae, (1 << ctx), + INDIRECT_LM_ADDR_0_BYTE_INDEX, ind_lm_addr_byte0); + qat_hal_wr_indr_csr(handle, ae, (1 << ctx), + INDIRECT_LM_ADDR_1_BYTE_INDEX, ind_lm_addr_byte1); + qat_hal_wr_indr_csr(handle, ae, (1 << ctx), + FUTURE_COUNT_SIGNAL_INDIRECT, ind_cnt_sig); + qat_hal_wr_indr_csr(handle, ae, (1 << ctx), + CTX_SIG_EVENTS_INDIRECT, ind_sig); + qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, act_sig); + qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables); + + return 0; +} + +static int qat_hal_rd_rel_reg(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned char ctx, + enum icp_qat_uof_regtype reg_type, + unsigned short reg_num, unsigned int *data) +{ + unsigned int savctx, uaddr, uwrd_lo, uwrd_hi; + unsigned int ctxarb_cntl, ustore_addr, ctx_enables; + unsigned short reg_addr; + int status = 0; + uint64_t insts, savuword; + + reg_addr = qat_hal_get_reg_addr(reg_type, reg_num); + if (reg_addr == BAD_REGADDR) { + pr_err("QAT: bad regaddr=0x%x\n", reg_addr); + return -EINVAL; + } + switch (reg_type) { + case ICP_GPA_REL: + insts = 0xA070000000ull | (reg_addr & 0x3ff); + break; + default: + insts = (uint64_t)0xA030000000ull | ((reg_addr & 0x3ff) << 10); + break; + } + qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &savctx); + qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL, &ctxarb_cntl); + qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables); + ctx_enables &= IGNORE_W1C_MASK; + if (ctx != (savctx & ACS_ACNO)) + qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, + ctx & ACS_ACNO); + qat_hal_get_uwords(handle, ae, 0, 1, &savuword); + qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables); + qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS, &ustore_addr); + uaddr = UA_ECS; + qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr); + insts = qat_hal_set_uword_ecc(insts); + uwrd_lo = (unsigned int)(insts & 0xffffffff); + uwrd_hi = (unsigned int)(insts >> 0x20); + qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo); + qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi); + qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr); + /* delay for at least 8 cycles */ + qat_hal_wait_cycles(handle, ae, 0x8, 0); + /* + * read ALU output + * the instruction should have been executed + * prior to clearing the ECS in putUwords + */ + qat_hal_rd_ae_csr(handle, ae, ALU_OUT, data); + qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr); + qat_hal_wr_uwords(handle, ae, 0, 1, &savuword); + if (ctx != (savctx & ACS_ACNO)) + qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, + savctx & ACS_ACNO); + qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, ctxarb_cntl); + qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables); + + return status; +} + +static int qat_hal_wr_rel_reg(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned char ctx, + enum icp_qat_uof_regtype reg_type, + unsigned short reg_num, unsigned int data) +{ + unsigned short src_hiaddr, src_lowaddr, dest_addr, data16hi, data16lo; + uint64_t insts[] = { + 0x0F440000000ull, + 0x0F040000000ull, + 0x0F0000C0300ull, + 0x0E000010000ull + }; + const int num_inst = ARRAY_SIZE(insts), code_off = 1; + const int imm_w1 = 0, imm_w0 = 1; + + dest_addr = qat_hal_get_reg_addr(reg_type, reg_num); + if (dest_addr == BAD_REGADDR) { + pr_err("QAT: bad destAddr=0x%x\n", dest_addr); + return -EINVAL; + } + + data16lo = 0xffff & data; + data16hi = 0xffff & (data >> 0x10); + src_hiaddr = qat_hal_get_reg_addr(ICP_NO_DEST, (unsigned short) + (0xff & data16hi)); + src_lowaddr = qat_hal_get_reg_addr(ICP_NO_DEST, (unsigned short) + (0xff & data16lo)); + switch (reg_type) { + case ICP_GPA_REL: + insts[imm_w1] = insts[imm_w1] | ((data16hi >> 8) << 20) | + ((src_hiaddr & 0x3ff) << 10) | (dest_addr & 0x3ff); + insts[imm_w0] = insts[imm_w0] | ((data16lo >> 8) << 20) | + ((src_lowaddr & 0x3ff) << 10) | (dest_addr & 0x3ff); + break; + default: + insts[imm_w1] = insts[imm_w1] | ((data16hi >> 8) << 20) | + ((dest_addr & 0x3ff) << 10) | (src_hiaddr & 0x3ff); + + insts[imm_w0] = insts[imm_w0] | ((data16lo >> 8) << 20) | + ((dest_addr & 0x3ff) << 10) | (src_lowaddr & 0x3ff); + break; + } + + return qat_hal_exec_micro_inst(handle, ae, ctx, insts, num_inst, + code_off, num_inst * 0x5, NULL); +} + +int qat_hal_get_ins_num(void) +{ + return ARRAY_SIZE(inst_4b); +} + +static int qat_hal_concat_micro_code(uint64_t *micro_inst, + unsigned int inst_num, unsigned int size, + unsigned int addr, unsigned int *value) +{ + int i, val_indx; + unsigned int cur_value; + const uint64_t *inst_arr; + int fixup_offset; + int usize = 0; + int orig_num; + + orig_num = inst_num; + val_indx = 0; + cur_value = value[val_indx++]; + inst_arr = inst_4b; + usize = ARRAY_SIZE(inst_4b); + fixup_offset = inst_num; + for (i = 0; i < usize; i++) + micro_inst[inst_num++] = inst_arr[i]; + INSERT_IMMED_GPRA_CONST(micro_inst[fixup_offset], (addr)); + fixup_offset++; + INSERT_IMMED_GPRA_CONST(micro_inst[fixup_offset], 0); + fixup_offset++; + INSERT_IMMED_GPRB_CONST(micro_inst[fixup_offset], (cur_value >> 0)); + fixup_offset++; + INSERT_IMMED_GPRB_CONST(micro_inst[fixup_offset], (cur_value >> 0x10)); + + return inst_num - orig_num; +} + +static int qat_hal_exec_micro_init_lm(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned char ctx, + int *pfirst_exec, uint64_t *micro_inst, + unsigned int inst_num) +{ + int stat = 0; + unsigned int gpra0 = 0, gpra1 = 0, gpra2 = 0; + unsigned int gprb0 = 0, gprb1 = 0; + + if (*pfirst_exec) { + qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0, &gpra0); + qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x1, &gpra1); + qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x2, &gpra2); + qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0, &gprb0); + qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0x1, &gprb1); + *pfirst_exec = 0; + } + stat = qat_hal_exec_micro_inst(handle, ae, ctx, micro_inst, inst_num, 1, + inst_num * 0x5, NULL); + if (stat != 0) + return -EFAULT; + qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0, gpra0); + qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x1, gpra1); + qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x2, gpra2); + qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0, gprb0); + qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0x1, gprb1); + + return 0; +} + +int qat_hal_batch_wr_lm(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, + struct icp_qat_uof_batch_init *lm_init_header) +{ + struct icp_qat_uof_batch_init *plm_init; + uint64_t *micro_inst_arry; + int micro_inst_num; + int alloc_inst_size; + int first_exec = 1; + int stat = 0; + + plm_init = lm_init_header->next; + alloc_inst_size = lm_init_header->size; + if ((unsigned int)alloc_inst_size > handle->hal_handle->max_ustore) + alloc_inst_size = handle->hal_handle->max_ustore; + micro_inst_arry = kmalloc_array(alloc_inst_size, sizeof(uint64_t), + GFP_KERNEL); + if (!micro_inst_arry) + return -ENOMEM; + micro_inst_num = 0; + while (plm_init) { + unsigned int addr, *value, size; + + ae = plm_init->ae; + addr = plm_init->addr; + value = plm_init->value; + size = plm_init->size; + micro_inst_num += qat_hal_concat_micro_code(micro_inst_arry, + micro_inst_num, + size, addr, value); + plm_init = plm_init->next; + } + /* exec micro codes */ + if (micro_inst_arry && (micro_inst_num > 0)) { + micro_inst_arry[micro_inst_num++] = 0x0E000010000ull; + stat = qat_hal_exec_micro_init_lm(handle, ae, 0, &first_exec, + micro_inst_arry, + micro_inst_num); + } + kfree(micro_inst_arry); + return stat; +} + +static int qat_hal_put_rel_rd_xfer(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned char ctx, + enum icp_qat_uof_regtype reg_type, + unsigned short reg_num, unsigned int val) +{ + int status = 0; + unsigned int reg_addr; + unsigned int ctx_enables; + unsigned short mask; + unsigned short dr_offset = 0x10; + + status = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables); + if (CE_INUSE_CONTEXTS & ctx_enables) { + if (ctx & 0x1) { + pr_err("QAT: bad 4-ctx mode,ctx=0x%x\n", ctx); + return -EINVAL; + } + mask = 0x1f; + dr_offset = 0x20; + } else { + mask = 0x0f; + } + if (reg_num & ~mask) + return -EINVAL; + reg_addr = reg_num + (ctx << 0x5); + switch (reg_type) { + case ICP_SR_RD_REL: + case ICP_SR_REL: + SET_AE_XFER(handle, ae, reg_addr, val); + break; + case ICP_DR_RD_REL: + case ICP_DR_REL: + SET_AE_XFER(handle, ae, (reg_addr + dr_offset), val); + break; + default: + status = -EINVAL; + break; + } + return status; +} + +static int qat_hal_put_rel_wr_xfer(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned char ctx, + enum icp_qat_uof_regtype reg_type, + unsigned short reg_num, unsigned int data) +{ + unsigned int gprval, ctx_enables; + unsigned short src_hiaddr, src_lowaddr, gpr_addr, xfr_addr, data16hi, + data16low; + unsigned short reg_mask; + int status = 0; + uint64_t micro_inst[] = { + 0x0F440000000ull, + 0x0F040000000ull, + 0x0A000000000ull, + 0x0F0000C0300ull, + 0x0E000010000ull + }; + const int num_inst = ARRAY_SIZE(micro_inst), code_off = 1; + const unsigned short gprnum = 0, dly = num_inst * 0x5; + + qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables); + if (CE_INUSE_CONTEXTS & ctx_enables) { + if (ctx & 0x1) { + pr_err("QAT: 4-ctx mode,ctx=0x%x\n", ctx); + return -EINVAL; + } + reg_mask = (unsigned short)~0x1f; + } else { + reg_mask = (unsigned short)~0xf; + } + if (reg_num & reg_mask) + return -EINVAL; + xfr_addr = qat_hal_get_reg_addr(reg_type, reg_num); + if (xfr_addr == BAD_REGADDR) { + pr_err("QAT: bad xfrAddr=0x%x\n", xfr_addr); + return -EINVAL; + } + qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, &gprval); + gpr_addr = qat_hal_get_reg_addr(ICP_GPB_REL, gprnum); + data16low = 0xffff & data; + data16hi = 0xffff & (data >> 0x10); + src_hiaddr = qat_hal_get_reg_addr(ICP_NO_DEST, + (unsigned short)(0xff & data16hi)); + src_lowaddr = qat_hal_get_reg_addr(ICP_NO_DEST, + (unsigned short)(0xff & data16low)); + micro_inst[0] = micro_inst[0x0] | ((data16hi >> 8) << 20) | + ((gpr_addr & 0x3ff) << 10) | (src_hiaddr & 0x3ff); + micro_inst[1] = micro_inst[0x1] | ((data16low >> 8) << 20) | + ((gpr_addr & 0x3ff) << 10) | (src_lowaddr & 0x3ff); + micro_inst[0x2] = micro_inst[0x2] | + ((xfr_addr & 0x3ff) << 20) | ((gpr_addr & 0x3ff) << 10); + status = qat_hal_exec_micro_inst(handle, ae, ctx, micro_inst, num_inst, + code_off, dly, NULL); + qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, gprval); + return status; +} + +static int qat_hal_put_rel_nn(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned char ctx, + unsigned short nn, unsigned int val) +{ + unsigned int ctx_enables; + int stat = 0; + + qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables); + ctx_enables &= IGNORE_W1C_MASK; + qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables | CE_NN_MODE); + + stat = qat_hal_put_rel_wr_xfer(handle, ae, ctx, ICP_NEIGH_REL, nn, val); + qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables); + return stat; +} + +static int qat_hal_convert_abs_to_rel(struct icp_qat_fw_loader_handle + *handle, unsigned char ae, + unsigned short absreg_num, + unsigned short *relreg, + unsigned char *ctx) +{ + unsigned int ctx_enables; + + qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables); + if (ctx_enables & CE_INUSE_CONTEXTS) { + /* 4-ctx mode */ + *relreg = absreg_num & 0x1F; + *ctx = (absreg_num >> 0x4) & 0x6; + } else { + /* 8-ctx mode */ + *relreg = absreg_num & 0x0F; + *ctx = (absreg_num >> 0x4) & 0x7; + } + return 0; +} + +int qat_hal_init_gpr(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned char ctx_mask, + enum icp_qat_uof_regtype reg_type, + unsigned short reg_num, unsigned int regdata) +{ + int stat = 0; + unsigned short reg; + unsigned char ctx = 0; + enum icp_qat_uof_regtype type; + + if (reg_num >= ICP_QAT_UCLO_MAX_GPR_REG) + return -EINVAL; + + do { + if (ctx_mask == 0) { + qat_hal_convert_abs_to_rel(handle, ae, reg_num, ®, + &ctx); + type = reg_type - 1; + } else { + reg = reg_num; + type = reg_type; + if (!test_bit(ctx, (unsigned long *)&ctx_mask)) + continue; + } + stat = qat_hal_wr_rel_reg(handle, ae, ctx, type, reg, regdata); + if (stat) { + pr_err("QAT: write gpr fail\n"); + return -EINVAL; + } + } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX)); + + return 0; +} + +int qat_hal_init_wr_xfer(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned char ctx_mask, + enum icp_qat_uof_regtype reg_type, + unsigned short reg_num, unsigned int regdata) +{ + int stat = 0; + unsigned short reg; + unsigned char ctx = 0; + enum icp_qat_uof_regtype type; + + if (reg_num >= ICP_QAT_UCLO_MAX_XFER_REG) + return -EINVAL; + + do { + if (ctx_mask == 0) { + qat_hal_convert_abs_to_rel(handle, ae, reg_num, ®, + &ctx); + type = reg_type - 3; + } else { + reg = reg_num; + type = reg_type; + if (!test_bit(ctx, (unsigned long *)&ctx_mask)) + continue; + } + stat = qat_hal_put_rel_wr_xfer(handle, ae, ctx, type, reg, + regdata); + if (stat) { + pr_err("QAT: write wr xfer fail\n"); + return -EINVAL; + } + } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX)); + + return 0; +} + +int qat_hal_init_rd_xfer(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned char ctx_mask, + enum icp_qat_uof_regtype reg_type, + unsigned short reg_num, unsigned int regdata) +{ + int stat = 0; + unsigned short reg; + unsigned char ctx = 0; + enum icp_qat_uof_regtype type; + + if (reg_num >= ICP_QAT_UCLO_MAX_XFER_REG) + return -EINVAL; + + do { + if (ctx_mask == 0) { + qat_hal_convert_abs_to_rel(handle, ae, reg_num, ®, + &ctx); + type = reg_type - 3; + } else { + reg = reg_num; + type = reg_type; + if (!test_bit(ctx, (unsigned long *)&ctx_mask)) + continue; + } + stat = qat_hal_put_rel_rd_xfer(handle, ae, ctx, type, reg, + regdata); + if (stat) { + pr_err("QAT: write rd xfer fail\n"); + return -EINVAL; + } + } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX)); + + return 0; +} + +int qat_hal_init_nn(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned char ctx_mask, + unsigned short reg_num, unsigned int regdata) +{ + int stat = 0; + unsigned char ctx; + + if (ctx_mask == 0) + return -EINVAL; + + for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) { + if (!test_bit(ctx, (unsigned long *)&ctx_mask)) + continue; + stat = qat_hal_put_rel_nn(handle, ae, ctx, reg_num, regdata); + if (stat) { + pr_err("QAT: write neigh error\n"); + return -EINVAL; + } + } + + return 0; +} diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c new file mode 100644 index 000000000000..1e27f9f7fddf --- /dev/null +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -0,0 +1,1181 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#include <linux/slab.h> +#include <linux/ctype.h> +#include <linux/kernel.h> + +#include "adf_accel_devices.h" +#include "adf_common_drv.h" +#include "icp_qat_uclo.h" +#include "icp_qat_hal.h" +#include "icp_qat_fw_loader_handle.h" + +#define UWORD_CPYBUF_SIZE 1024 +#define INVLD_UWORD 0xffffffffffull +#define PID_MINOR_REV 0xf +#define PID_MAJOR_REV (0xf << 4) + +static int qat_uclo_init_ae_data(struct icp_qat_uclo_objhandle *obj_handle, + unsigned int ae, unsigned int image_num) +{ + struct icp_qat_uclo_aedata *ae_data; + struct icp_qat_uclo_encapme *encap_image; + struct icp_qat_uclo_page *page = NULL; + struct icp_qat_uclo_aeslice *ae_slice = NULL; + + ae_data = &obj_handle->ae_data[ae]; + encap_image = &obj_handle->ae_uimage[image_num]; + ae_slice = &ae_data->ae_slices[ae_data->slice_num]; + ae_slice->encap_image = encap_image; + + if (encap_image->img_ptr) { + ae_slice->ctx_mask_assigned = + encap_image->img_ptr->ctx_assigned; + ae_data->eff_ustore_size = obj_handle->ustore_phy_size; + } else { + ae_slice->ctx_mask_assigned = 0; + } + ae_slice->region = kzalloc(sizeof(*ae_slice->region), GFP_KERNEL); + if (!ae_slice->region) + return -ENOMEM; + ae_slice->page = kzalloc(sizeof(*ae_slice->page), GFP_KERNEL); + if (!ae_slice->page) + goto out_err; + page = ae_slice->page; + page->encap_page = encap_image->page; + ae_slice->page->region = ae_slice->region; + ae_data->slice_num++; + return 0; +out_err: + kfree(ae_slice->region); + ae_slice->region = NULL; + return -ENOMEM; +} + +static int qat_uclo_free_ae_data(struct icp_qat_uclo_aedata *ae_data) +{ + unsigned int i; + + if (!ae_data) { + pr_err("QAT: bad argument, ae_data is NULL\n "); + return -EINVAL; + } + + for (i = 0; i < ae_data->slice_num; i++) { + kfree(ae_data->ae_slices[i].region); + ae_data->ae_slices[i].region = NULL; + kfree(ae_data->ae_slices[i].page); + ae_data->ae_slices[i].page = NULL; + } + return 0; +} + +static char *qat_uclo_get_string(struct icp_qat_uof_strtable *str_table, + unsigned int str_offset) +{ + if ((!str_table->table_len) || (str_offset > str_table->table_len)) + return NULL; + return (char *)(((unsigned long)(str_table->strings)) + str_offset); +} + +static int qat_uclo_check_format(struct icp_qat_uof_filehdr *hdr) +{ + int maj = hdr->maj_ver & 0xff; + int min = hdr->min_ver & 0xff; + + if (hdr->file_id != ICP_QAT_UOF_FID) { + pr_err("QAT: Invalid header 0x%x\n", hdr->file_id); + return -EINVAL; + } + if (min != ICP_QAT_UOF_MINVER || maj != ICP_QAT_UOF_MAJVER) { + pr_err("QAT: bad UOF version, major 0x%x, minor 0x%x\n", + maj, min); + return -EINVAL; + } + return 0; +} + +static void qat_uclo_wr_sram_by_words(struct icp_qat_fw_loader_handle *handle, + unsigned int addr, unsigned int *val, + unsigned int num_in_bytes) +{ + unsigned int outval; + unsigned char *ptr = (unsigned char *)val; + + while (num_in_bytes) { + memcpy(&outval, ptr, 4); + SRAM_WRITE(handle, addr, outval); + num_in_bytes -= 4; + ptr += 4; + addr += 4; + } +} + +static void qat_uclo_wr_umem_by_words(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned int addr, + unsigned int *val, + unsigned int num_in_bytes) +{ + unsigned int outval; + unsigned char *ptr = (unsigned char *)val; + + addr >>= 0x2; /* convert to uword address */ + + while (num_in_bytes) { + memcpy(&outval, ptr, 4); + qat_hal_wr_umem(handle, ae, addr++, 1, &outval); + num_in_bytes -= 4; + ptr += 4; + } +} + +static void qat_uclo_batch_wr_umem(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, + struct icp_qat_uof_batch_init + *umem_init_header) +{ + struct icp_qat_uof_batch_init *umem_init; + + if (!umem_init_header) + return; + umem_init = umem_init_header->next; + while (umem_init) { + unsigned int addr, *value, size; + + ae = umem_init->ae; + addr = umem_init->addr; + value = umem_init->value; + size = umem_init->size; + qat_uclo_wr_umem_by_words(handle, ae, addr, value, size); + umem_init = umem_init->next; + } +} + +static void +qat_uclo_cleanup_batch_init_list(struct icp_qat_fw_loader_handle *handle, + struct icp_qat_uof_batch_init **base) +{ + struct icp_qat_uof_batch_init *umem_init; + + umem_init = *base; + while (umem_init) { + struct icp_qat_uof_batch_init *pre; + + pre = umem_init; + umem_init = umem_init->next; + kfree(pre); + } + *base = NULL; +} + +static int qat_uclo_parse_num(char *str, unsigned int *num) +{ + char buf[16] = {0}; + unsigned long ae = 0; + int i; + + strncpy(buf, str, 15); + for (i = 0; i < 16; i++) { + if (!isdigit(buf[i])) { + buf[i] = '\0'; + break; + } + } + if ((kstrtoul(buf, 10, &ae))) + return -EFAULT; + + *num = (unsigned int)ae; + return 0; +} + +static int qat_uclo_fetch_initmem_ae(struct icp_qat_fw_loader_handle *handle, + struct icp_qat_uof_initmem *init_mem, + unsigned int size_range, unsigned int *ae) +{ + struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + char *str; + + if ((init_mem->addr + init_mem->num_in_bytes) > (size_range << 0x2)) { + pr_err("QAT: initmem is out of range"); + return -EINVAL; + } + if (init_mem->scope != ICP_QAT_UOF_LOCAL_SCOPE) { + pr_err("QAT: Memory scope for init_mem error\n"); + return -EINVAL; + } + str = qat_uclo_get_string(&obj_handle->str_table, init_mem->sym_name); + if (!str) { + pr_err("QAT: AE name assigned in UOF init table is NULL\n"); + return -EINVAL; + } + if (qat_uclo_parse_num(str, ae)) { + pr_err("QAT: Parse num for AE number failed\n"); + return -EINVAL; + } + if (*ae >= ICP_QAT_UCLO_MAX_AE) { + pr_err("QAT: ae %d out of range\n", *ae); + return -EINVAL; + } + return 0; +} + +static int qat_uclo_create_batch_init_list(struct icp_qat_fw_loader_handle + *handle, struct icp_qat_uof_initmem + *init_mem, unsigned int ae, + struct icp_qat_uof_batch_init + **init_tab_base) +{ + struct icp_qat_uof_batch_init *init_header, *tail; + struct icp_qat_uof_batch_init *mem_init, *tail_old; + struct icp_qat_uof_memvar_attr *mem_val_attr; + unsigned int i, flag = 0; + + mem_val_attr = + (struct icp_qat_uof_memvar_attr *)((unsigned long)init_mem + + sizeof(struct icp_qat_uof_initmem)); + + init_header = *init_tab_base; + if (!init_header) { + init_header = kzalloc(sizeof(*init_header), GFP_KERNEL); + if (!init_header) + return -ENOMEM; + init_header->size = 1; + *init_tab_base = init_header; + flag = 1; + } + tail_old = init_header; + while (tail_old->next) + tail_old = tail_old->next; + tail = tail_old; + for (i = 0; i < init_mem->val_attr_num; i++) { + mem_init = kzalloc(sizeof(*mem_init), GFP_KERNEL); + if (!mem_init) + goto out_err; + mem_init->ae = ae; + mem_init->addr = init_mem->addr + mem_val_attr->offset_in_byte; + mem_init->value = &mem_val_attr->value; + mem_init->size = 4; + mem_init->next = NULL; + tail->next = mem_init; + tail = mem_init; + init_header->size += qat_hal_get_ins_num(); + mem_val_attr++; + } + return 0; +out_err: + while (tail_old) { + mem_init = tail_old->next; + kfree(tail_old); + tail_old = mem_init; + } + if (flag) + kfree(*init_tab_base); + return -ENOMEM; +} + +static int qat_uclo_init_lmem_seg(struct icp_qat_fw_loader_handle *handle, + struct icp_qat_uof_initmem *init_mem) +{ + struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + unsigned int ae; + + if (qat_uclo_fetch_initmem_ae(handle, init_mem, + ICP_QAT_UCLO_MAX_LMEM_REG, &ae)) + return -EINVAL; + if (qat_uclo_create_batch_init_list(handle, init_mem, ae, + &obj_handle->lm_init_tab[ae])) + return -EINVAL; + return 0; +} + +static int qat_uclo_init_umem_seg(struct icp_qat_fw_loader_handle *handle, + struct icp_qat_uof_initmem *init_mem) +{ + struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + unsigned int ae, ustore_size, uaddr, i; + + ustore_size = obj_handle->ustore_phy_size; + if (qat_uclo_fetch_initmem_ae(handle, init_mem, ustore_size, &ae)) + return -EINVAL; + if (qat_uclo_create_batch_init_list(handle, init_mem, ae, + &obj_handle->umem_init_tab[ae])) + return -EINVAL; + /* set the highest ustore address referenced */ + uaddr = (init_mem->addr + init_mem->num_in_bytes) >> 0x2; + for (i = 0; i < obj_handle->ae_data[ae].slice_num; i++) { + if (obj_handle->ae_data[ae].ae_slices[i]. + encap_image->uwords_num < uaddr) + obj_handle->ae_data[ae].ae_slices[i]. + encap_image->uwords_num = uaddr; + } + return 0; +} + +#define ICP_DH895XCC_PESRAM_BAR_SIZE 0x80000 +static int qat_uclo_init_ae_memory(struct icp_qat_fw_loader_handle *handle, + struct icp_qat_uof_initmem *init_mem) +{ + unsigned int i; + struct icp_qat_uof_memvar_attr *mem_val_attr; + + mem_val_attr = + (struct icp_qat_uof_memvar_attr *)((unsigned long)init_mem + + sizeof(struct icp_qat_uof_initmem)); + + switch (init_mem->region) { + case ICP_QAT_UOF_SRAM_REGION: + if ((init_mem->addr + init_mem->num_in_bytes) > + ICP_DH895XCC_PESRAM_BAR_SIZE) { + pr_err("QAT: initmem on SRAM is out of range"); + return -EINVAL; + } + for (i = 0; i < init_mem->val_attr_num; i++) { + qat_uclo_wr_sram_by_words(handle, + init_mem->addr + + mem_val_attr->offset_in_byte, + &mem_val_attr->value, 4); + mem_val_attr++; + } + break; + case ICP_QAT_UOF_LMEM_REGION: + if (qat_uclo_init_lmem_seg(handle, init_mem)) + return -EINVAL; + break; + case ICP_QAT_UOF_UMEM_REGION: + if (qat_uclo_init_umem_seg(handle, init_mem)) + return -EINVAL; + break; + default: + pr_err("QAT: initmem region error. region type=0x%x\n", + init_mem->region); + return -EINVAL; + } + return 0; +} + +static int qat_uclo_init_ustore(struct icp_qat_fw_loader_handle *handle, + struct icp_qat_uclo_encapme *image) +{ + unsigned int i; + struct icp_qat_uclo_encap_page *page; + struct icp_qat_uof_image *uof_image; + unsigned char ae; + unsigned int ustore_size; + unsigned int patt_pos; + struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + uint64_t *fill_data; + + uof_image = image->img_ptr; + fill_data = kcalloc(ICP_QAT_UCLO_MAX_USTORE, sizeof(uint64_t), + GFP_KERNEL); + if (!fill_data) + return -ENOMEM; + for (i = 0; i < ICP_QAT_UCLO_MAX_USTORE; i++) + memcpy(&fill_data[i], &uof_image->fill_pattern, + sizeof(uint64_t)); + page = image->page; + + for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { + if (!test_bit(ae, (unsigned long *)&uof_image->ae_assigned)) + continue; + ustore_size = obj_handle->ae_data[ae].eff_ustore_size; + patt_pos = page->beg_addr_p + page->micro_words_num; + + qat_hal_wr_uwords(handle, (unsigned char)ae, 0, + page->beg_addr_p, &fill_data[0]); + qat_hal_wr_uwords(handle, (unsigned char)ae, patt_pos, + ustore_size - patt_pos + 1, + &fill_data[page->beg_addr_p]); + } + kfree(fill_data); + return 0; +} + +static int qat_uclo_init_memory(struct icp_qat_fw_loader_handle *handle) +{ + int i, ae; + struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + struct icp_qat_uof_initmem *initmem = obj_handle->init_mem_tab.init_mem; + + for (i = 0; i < obj_handle->init_mem_tab.entry_num; i++) { + if (initmem->num_in_bytes) { + if (qat_uclo_init_ae_memory(handle, initmem)) + return -EINVAL; + } + initmem = (struct icp_qat_uof_initmem *)((unsigned long)( + (unsigned long)initmem + + sizeof(struct icp_qat_uof_initmem)) + + (sizeof(struct icp_qat_uof_memvar_attr) * + initmem->val_attr_num)); + } + for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { + if (qat_hal_batch_wr_lm(handle, ae, + obj_handle->lm_init_tab[ae])) { + pr_err("QAT: fail to batch init lmem for AE %d\n", ae); + return -EINVAL; + } + qat_uclo_cleanup_batch_init_list(handle, + &obj_handle->lm_init_tab[ae]); + qat_uclo_batch_wr_umem(handle, ae, + obj_handle->umem_init_tab[ae]); + qat_uclo_cleanup_batch_init_list(handle, + &obj_handle-> + umem_init_tab[ae]); + } + return 0; +} + +static void *qat_uclo_find_chunk(struct icp_qat_uof_objhdr *obj_hdr, + char *chunk_id, void *cur) +{ + int i; + struct icp_qat_uof_chunkhdr *chunk_hdr = + (struct icp_qat_uof_chunkhdr *) + ((unsigned long)obj_hdr + sizeof(struct icp_qat_uof_objhdr)); + + for (i = 0; i < obj_hdr->num_chunks; i++) { + if ((cur < (void *)&chunk_hdr[i]) && + !strncmp(chunk_hdr[i].chunk_id, chunk_id, + ICP_QAT_UOF_OBJID_LEN)) { + return &chunk_hdr[i]; + } + } + return NULL; +} + +static unsigned int qat_uclo_calc_checksum(unsigned int reg, int ch) +{ + int i; + unsigned int topbit = 1 << 0xF; + unsigned int inbyte = (unsigned int)((reg >> 0x18) ^ ch); + + reg ^= inbyte << 0x8; + for (i = 0; i < 0x8; i++) { + if (reg & topbit) + reg = (reg << 1) ^ 0x1021; + else + reg <<= 1; + } + return reg & 0xFFFF; +} + +static unsigned int qat_uclo_calc_str_checksum(char *ptr, int num) +{ + unsigned int chksum = 0; + + if (ptr) + while (num--) + chksum = qat_uclo_calc_checksum(chksum, *ptr++); + return chksum; +} + +static struct icp_qat_uclo_objhdr * +qat_uclo_map_chunk(char *buf, struct icp_qat_uof_filehdr *file_hdr, + char *chunk_id) +{ + struct icp_qat_uof_filechunkhdr *file_chunk; + struct icp_qat_uclo_objhdr *obj_hdr; + char *chunk; + int i; + + file_chunk = (struct icp_qat_uof_filechunkhdr *) + (buf + sizeof(struct icp_qat_uof_filehdr)); + for (i = 0; i < file_hdr->num_chunks; i++) { + if (!strncmp(file_chunk->chunk_id, chunk_id, + ICP_QAT_UOF_OBJID_LEN)) { + chunk = buf + file_chunk->offset; + if (file_chunk->checksum != qat_uclo_calc_str_checksum( + chunk, file_chunk->size)) + break; + obj_hdr = kzalloc(sizeof(*obj_hdr), GFP_KERNEL); + if (!obj_hdr) + break; + obj_hdr->file_buff = chunk; + obj_hdr->checksum = file_chunk->checksum; + obj_hdr->size = file_chunk->size; + return obj_hdr; + } + file_chunk++; + } + return NULL; +} + +static unsigned int +qat_uclo_check_image_compat(struct icp_qat_uof_encap_obj *encap_uof_obj, + struct icp_qat_uof_image *image) +{ + struct icp_qat_uof_objtable *uc_var_tab, *imp_var_tab, *imp_expr_tab; + struct icp_qat_uof_objtable *neigh_reg_tab; + struct icp_qat_uof_code_page *code_page; + + code_page = (struct icp_qat_uof_code_page *) + ((char *)image + sizeof(struct icp_qat_uof_image)); + uc_var_tab = (struct icp_qat_uof_objtable *)(encap_uof_obj->beg_uof + + code_page->uc_var_tab_offset); + imp_var_tab = (struct icp_qat_uof_objtable *)(encap_uof_obj->beg_uof + + code_page->imp_var_tab_offset); + imp_expr_tab = (struct icp_qat_uof_objtable *) + (encap_uof_obj->beg_uof + + code_page->imp_expr_tab_offset); + if (uc_var_tab->entry_num || imp_var_tab->entry_num || + imp_expr_tab->entry_num) { + pr_err("QAT: UOF can't contain imported variable to be parsed"); + return -EINVAL; + } + neigh_reg_tab = (struct icp_qat_uof_objtable *) + (encap_uof_obj->beg_uof + + code_page->neigh_reg_tab_offset); + if (neigh_reg_tab->entry_num) { + pr_err("QAT: UOF can't contain shared control store feature"); + return -EINVAL; + } + if (image->numpages > 1) { + pr_err("QAT: UOF can't contain multiple pages"); + return -EINVAL; + } + if (ICP_QAT_SHARED_USTORE_MODE(image->ae_mode)) { + pr_err("QAT: UOF can't use shared control store feature"); + return -EFAULT; + } + if (RELOADABLE_CTX_SHARED_MODE(image->ae_mode)) { + pr_err("QAT: UOF can't use reloadable feature"); + return -EFAULT; + } + return 0; +} + +static void qat_uclo_map_image_page(struct icp_qat_uof_encap_obj + *encap_uof_obj, + struct icp_qat_uof_image *img, + struct icp_qat_uclo_encap_page *page) +{ + struct icp_qat_uof_code_page *code_page; + struct icp_qat_uof_code_area *code_area; + struct icp_qat_uof_objtable *uword_block_tab; + struct icp_qat_uof_uword_block *uwblock; + int i; + + code_page = (struct icp_qat_uof_code_page *) + ((char *)img + sizeof(struct icp_qat_uof_image)); + page->def_page = code_page->def_page; + page->page_region = code_page->page_region; + page->beg_addr_v = code_page->beg_addr_v; + page->beg_addr_p = code_page->beg_addr_p; + code_area = (struct icp_qat_uof_code_area *)(encap_uof_obj->beg_uof + + code_page->code_area_offset); + page->micro_words_num = code_area->micro_words_num; + uword_block_tab = (struct icp_qat_uof_objtable *) + (encap_uof_obj->beg_uof + + code_area->uword_block_tab); + page->uwblock_num = uword_block_tab->entry_num; + uwblock = (struct icp_qat_uof_uword_block *)((char *)uword_block_tab + + sizeof(struct icp_qat_uof_objtable)); + page->uwblock = (struct icp_qat_uclo_encap_uwblock *)uwblock; + for (i = 0; i < uword_block_tab->entry_num; i++) + page->uwblock[i].micro_words = + (unsigned long)encap_uof_obj->beg_uof + uwblock[i].uword_offset; +} + +static int qat_uclo_map_uimage(struct icp_qat_uclo_objhandle *obj_handle, + struct icp_qat_uclo_encapme *ae_uimage, + int max_image) +{ + int i, j; + struct icp_qat_uof_chunkhdr *chunk_hdr = NULL; + struct icp_qat_uof_image *image; + struct icp_qat_uof_objtable *ae_regtab; + struct icp_qat_uof_objtable *init_reg_sym_tab; + struct icp_qat_uof_objtable *sbreak_tab; + struct icp_qat_uof_encap_obj *encap_uof_obj = + &obj_handle->encap_uof_obj; + + for (j = 0; j < max_image; j++) { + chunk_hdr = qat_uclo_find_chunk(encap_uof_obj->obj_hdr, + ICP_QAT_UOF_IMAG, chunk_hdr); + if (!chunk_hdr) + break; + image = (struct icp_qat_uof_image *)(encap_uof_obj->beg_uof + + chunk_hdr->offset); + ae_regtab = (struct icp_qat_uof_objtable *) + (image->reg_tab_offset + + obj_handle->obj_hdr->file_buff); + ae_uimage[j].ae_reg_num = ae_regtab->entry_num; + ae_uimage[j].ae_reg = (struct icp_qat_uof_ae_reg *) + (((char *)ae_regtab) + + sizeof(struct icp_qat_uof_objtable)); + init_reg_sym_tab = (struct icp_qat_uof_objtable *) + (image->init_reg_sym_tab + + obj_handle->obj_hdr->file_buff); + ae_uimage[j].init_regsym_num = init_reg_sym_tab->entry_num; + ae_uimage[j].init_regsym = (struct icp_qat_uof_init_regsym *) + (((char *)init_reg_sym_tab) + + sizeof(struct icp_qat_uof_objtable)); + sbreak_tab = (struct icp_qat_uof_objtable *) + (image->sbreak_tab + obj_handle->obj_hdr->file_buff); + ae_uimage[j].sbreak_num = sbreak_tab->entry_num; + ae_uimage[j].sbreak = (struct icp_qat_uof_sbreak *) + (((char *)sbreak_tab) + + sizeof(struct icp_qat_uof_objtable)); + ae_uimage[j].img_ptr = image; + if (qat_uclo_check_image_compat(encap_uof_obj, image)) + goto out_err; + ae_uimage[j].page = + kzalloc(sizeof(struct icp_qat_uclo_encap_page), + GFP_KERNEL); + if (!ae_uimage[j].page) + goto out_err; + qat_uclo_map_image_page(encap_uof_obj, image, + ae_uimage[j].page); + } + return j; +out_err: + for (i = 0; i < j; i++) + kfree(ae_uimage[i].page); + return 0; +} + +static int qat_uclo_map_ae(struct icp_qat_fw_loader_handle *handle, int max_ae) +{ + int i, ae; + int mflag = 0; + struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + + for (ae = 0; ae <= max_ae; ae++) { + if (!test_bit(ae, + (unsigned long *)&handle->hal_handle->ae_mask)) + continue; + for (i = 0; i < obj_handle->uimage_num; i++) { + if (!test_bit(ae, (unsigned long *) + &obj_handle->ae_uimage[i].img_ptr->ae_assigned)) + continue; + mflag = 1; + if (qat_uclo_init_ae_data(obj_handle, ae, i)) + return -EINVAL; + } + } + if (!mflag) { + pr_err("QAT: uimage uses AE not set"); + return -EINVAL; + } + return 0; +} + +static struct icp_qat_uof_strtable * +qat_uclo_map_str_table(struct icp_qat_uclo_objhdr *obj_hdr, + char *tab_name, struct icp_qat_uof_strtable *str_table) +{ + struct icp_qat_uof_chunkhdr *chunk_hdr; + + chunk_hdr = qat_uclo_find_chunk((struct icp_qat_uof_objhdr *) + obj_hdr->file_buff, tab_name, NULL); + if (chunk_hdr) { + int hdr_size; + + memcpy(&str_table->table_len, obj_hdr->file_buff + + chunk_hdr->offset, sizeof(str_table->table_len)); + hdr_size = (char *)&str_table->strings - (char *)str_table; + str_table->strings = (unsigned long)obj_hdr->file_buff + + chunk_hdr->offset + hdr_size; + return str_table; + } + return NULL; +} + +static void +qat_uclo_map_initmem_table(struct icp_qat_uof_encap_obj *encap_uof_obj, + struct icp_qat_uclo_init_mem_table *init_mem_tab) +{ + struct icp_qat_uof_chunkhdr *chunk_hdr; + + chunk_hdr = qat_uclo_find_chunk(encap_uof_obj->obj_hdr, + ICP_QAT_UOF_IMEM, NULL); + if (chunk_hdr) { + memmove(&init_mem_tab->entry_num, encap_uof_obj->beg_uof + + chunk_hdr->offset, sizeof(unsigned int)); + init_mem_tab->init_mem = (struct icp_qat_uof_initmem *) + (encap_uof_obj->beg_uof + chunk_hdr->offset + + sizeof(unsigned int)); + } +} + +static int qat_uclo_check_uof_compat(struct icp_qat_uclo_objhandle *obj_handle) +{ + unsigned int maj_ver, prod_type = obj_handle->prod_type; + + if (!(prod_type & obj_handle->encap_uof_obj.obj_hdr->cpu_type)) { + pr_err("QAT: UOF type 0x%x not match with cur platform 0x%x\n", + obj_handle->encap_uof_obj.obj_hdr->cpu_type, prod_type); + return -EINVAL; + } + maj_ver = obj_handle->prod_rev & 0xff; + if ((obj_handle->encap_uof_obj.obj_hdr->max_cpu_ver < maj_ver) || + (obj_handle->encap_uof_obj.obj_hdr->min_cpu_ver > maj_ver)) { + pr_err("QAT: UOF majVer 0x%x out of range\n", maj_ver); + return -EINVAL; + } + return 0; +} + +static int qat_uclo_init_reg(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned char ctx_mask, + enum icp_qat_uof_regtype reg_type, + unsigned short reg_addr, unsigned int value) +{ + switch (reg_type) { + case ICP_GPA_ABS: + case ICP_GPB_ABS: + ctx_mask = 0; + case ICP_GPA_REL: + case ICP_GPB_REL: + return qat_hal_init_gpr(handle, ae, ctx_mask, reg_type, + reg_addr, value); + case ICP_SR_ABS: + case ICP_DR_ABS: + case ICP_SR_RD_ABS: + case ICP_DR_RD_ABS: + ctx_mask = 0; + case ICP_SR_REL: + case ICP_DR_REL: + case ICP_SR_RD_REL: + case ICP_DR_RD_REL: + return qat_hal_init_rd_xfer(handle, ae, ctx_mask, reg_type, + reg_addr, value); + case ICP_SR_WR_ABS: + case ICP_DR_WR_ABS: + ctx_mask = 0; + case ICP_SR_WR_REL: + case ICP_DR_WR_REL: + return qat_hal_init_wr_xfer(handle, ae, ctx_mask, reg_type, + reg_addr, value); + case ICP_NEIGH_REL: + return qat_hal_init_nn(handle, ae, ctx_mask, reg_addr, value); + default: + pr_err("QAT: UOF uses not supported reg type 0x%x\n", reg_type); + return -EFAULT; + } + return 0; +} + +static int qat_uclo_init_reg_sym(struct icp_qat_fw_loader_handle *handle, + unsigned int ae, + struct icp_qat_uclo_encapme *encap_ae) +{ + unsigned int i; + unsigned char ctx_mask; + struct icp_qat_uof_init_regsym *init_regsym; + + if (ICP_QAT_CTX_MODE(encap_ae->img_ptr->ae_mode) == + ICP_QAT_UCLO_MAX_CTX) + ctx_mask = 0xff; + else + ctx_mask = 0x55; + + for (i = 0; i < encap_ae->init_regsym_num; i++) { + unsigned int exp_res; + + init_regsym = &encap_ae->init_regsym[i]; + exp_res = init_regsym->value; + switch (init_regsym->init_type) { + case ICP_QAT_UOF_INIT_REG: + qat_uclo_init_reg(handle, ae, ctx_mask, + (enum icp_qat_uof_regtype) + init_regsym->reg_type, + (unsigned short)init_regsym->reg_addr, + exp_res); + break; + case ICP_QAT_UOF_INIT_REG_CTX: + /* check if ctx is appropriate for the ctxMode */ + if (!((1 << init_regsym->ctx) & ctx_mask)) { + pr_err("QAT: invalid ctx num = 0x%x\n", + init_regsym->ctx); + return -EINVAL; + } + qat_uclo_init_reg(handle, ae, + (unsigned char) + (1 << init_regsym->ctx), + (enum icp_qat_uof_regtype) + init_regsym->reg_type, + (unsigned short)init_regsym->reg_addr, + exp_res); + break; + case ICP_QAT_UOF_INIT_EXPR: + pr_err("QAT: INIT_EXPR feature not supported\n"); + return -EINVAL; + case ICP_QAT_UOF_INIT_EXPR_ENDIAN_SWAP: + pr_err("QAT: INIT_EXPR_ENDIAN_SWAP feature not supported\n"); + return -EINVAL; + default: + break; + } + } + return 0; +} + +static int qat_uclo_init_globals(struct icp_qat_fw_loader_handle *handle) +{ + struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + unsigned int s, ae; + + if (obj_handle->global_inited) + return 0; + if (obj_handle->init_mem_tab.entry_num) { + if (qat_uclo_init_memory(handle)) { + pr_err("QAT: initialize memory failed\n"); + return -EINVAL; + } + } + for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { + for (s = 0; s < obj_handle->ae_data[ae].slice_num; s++) { + if (!obj_handle->ae_data[ae].ae_slices[s].encap_image) + continue; + if (qat_uclo_init_reg_sym(handle, ae, + obj_handle->ae_data[ae]. + ae_slices[s].encap_image)) + return -EINVAL; + } + } + obj_handle->global_inited = 1; + return 0; +} + +static int qat_uclo_set_ae_mode(struct icp_qat_fw_loader_handle *handle) +{ + unsigned char ae, nn_mode, s; + struct icp_qat_uof_image *uof_image; + struct icp_qat_uclo_aedata *ae_data; + struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + + for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { + if (!test_bit(ae, + (unsigned long *)&handle->hal_handle->ae_mask)) + continue; + ae_data = &obj_handle->ae_data[ae]; + for (s = 0; s < min_t(unsigned int, ae_data->slice_num, + ICP_QAT_UCLO_MAX_CTX); s++) { + if (!obj_handle->ae_data[ae].ae_slices[s].encap_image) + continue; + uof_image = ae_data->ae_slices[s].encap_image->img_ptr; + if (qat_hal_set_ae_ctx_mode(handle, ae, + (char)ICP_QAT_CTX_MODE + (uof_image->ae_mode))) { + pr_err("QAT: qat_hal_set_ae_ctx_mode error\n"); + return -EFAULT; + } + nn_mode = ICP_QAT_NN_MODE(uof_image->ae_mode); + if (qat_hal_set_ae_nn_mode(handle, ae, nn_mode)) { + pr_err("QAT: qat_hal_set_ae_nn_mode error\n"); + return -EFAULT; + } + if (qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM0, + (char)ICP_QAT_LOC_MEM0_MODE + (uof_image->ae_mode))) { + pr_err("QAT: qat_hal_set_ae_lm_mode LMEM0 error\n"); + return -EFAULT; + } + if (qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM1, + (char)ICP_QAT_LOC_MEM1_MODE + (uof_image->ae_mode))) { + pr_err("QAT: qat_hal_set_ae_lm_mode LMEM1 error\n"); + return -EFAULT; + } + } + } + return 0; +} + +static void qat_uclo_init_uword_num(struct icp_qat_fw_loader_handle *handle) +{ + struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + struct icp_qat_uclo_encapme *image; + int a; + + for (a = 0; a < obj_handle->uimage_num; a++) { + image = &obj_handle->ae_uimage[a]; + image->uwords_num = image->page->beg_addr_p + + image->page->micro_words_num; + } +} + +static int qat_uclo_parse_uof_obj(struct icp_qat_fw_loader_handle *handle) +{ + struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + unsigned int ae; + + obj_handle->uword_buf = kcalloc(UWORD_CPYBUF_SIZE, sizeof(uint64_t), + GFP_KERNEL); + if (!obj_handle->uword_buf) + return -ENOMEM; + obj_handle->encap_uof_obj.beg_uof = obj_handle->obj_hdr->file_buff; + obj_handle->encap_uof_obj.obj_hdr = (struct icp_qat_uof_objhdr *) + obj_handle->obj_hdr->file_buff; + obj_handle->uword_in_bytes = 6; + obj_handle->prod_type = ICP_QAT_AC_C_CPU_TYPE; + obj_handle->prod_rev = PID_MAJOR_REV | + (PID_MINOR_REV & handle->hal_handle->revision_id); + if (qat_uclo_check_uof_compat(obj_handle)) { + pr_err("QAT: UOF incompatible\n"); + return -EINVAL; + } + obj_handle->ustore_phy_size = ICP_QAT_UCLO_MAX_USTORE; + if (!obj_handle->obj_hdr->file_buff || + !qat_uclo_map_str_table(obj_handle->obj_hdr, ICP_QAT_UOF_STRT, + &obj_handle->str_table)) { + pr_err("QAT: UOF doesn't have effective images\n"); + goto out_err; + } + obj_handle->uimage_num = + qat_uclo_map_uimage(obj_handle, obj_handle->ae_uimage, + ICP_QAT_UCLO_MAX_AE * ICP_QAT_UCLO_MAX_CTX); + if (!obj_handle->uimage_num) + goto out_err; + if (qat_uclo_map_ae(handle, handle->hal_handle->ae_max_num)) { + pr_err("QAT: Bad object\n"); + goto out_check_uof_aemask_err; + } + qat_uclo_init_uword_num(handle); + qat_uclo_map_initmem_table(&obj_handle->encap_uof_obj, + &obj_handle->init_mem_tab); + if (qat_uclo_set_ae_mode(handle)) + goto out_check_uof_aemask_err; + return 0; +out_check_uof_aemask_err: + for (ae = 0; ae < obj_handle->uimage_num; ae++) + kfree(obj_handle->ae_uimage[ae].page); +out_err: + kfree(obj_handle->uword_buf); + return -EFAULT; +} + +int qat_uclo_map_uof_obj(struct icp_qat_fw_loader_handle *handle, + void *addr_ptr, int mem_size) +{ + struct icp_qat_uof_filehdr *filehdr; + struct icp_qat_uclo_objhandle *objhdl; + + BUILD_BUG_ON(ICP_QAT_UCLO_MAX_AE >= + (sizeof(handle->hal_handle->ae_mask) * 8)); + + if (!handle || !addr_ptr || mem_size < 24) + return -EINVAL; + objhdl = kzalloc(sizeof(*objhdl), GFP_KERNEL); + if (!objhdl) + return -ENOMEM; + objhdl->obj_buf = kmemdup(addr_ptr, mem_size, GFP_KERNEL); + if (!objhdl->obj_buf) + goto out_objbuf_err; + filehdr = (struct icp_qat_uof_filehdr *)objhdl->obj_buf; + if (qat_uclo_check_format(filehdr)) + goto out_objhdr_err; + objhdl->obj_hdr = qat_uclo_map_chunk((char *)objhdl->obj_buf, filehdr, + ICP_QAT_UOF_OBJS); + if (!objhdl->obj_hdr) { + pr_err("QAT: object file chunk is null\n"); + goto out_objhdr_err; + } + handle->obj_handle = objhdl; + if (qat_uclo_parse_uof_obj(handle)) + goto out_overlay_obj_err; + return 0; + +out_overlay_obj_err: + handle->obj_handle = NULL; + kfree(objhdl->obj_hdr); +out_objhdr_err: + kfree(objhdl->obj_buf); +out_objbuf_err: + kfree(objhdl); + return -ENOMEM; +} + +void qat_uclo_del_uof_obj(struct icp_qat_fw_loader_handle *handle) +{ + struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + unsigned int a; + + if (!obj_handle) + return; + + kfree(obj_handle->uword_buf); + for (a = 0; a < obj_handle->uimage_num; a++) + kfree(obj_handle->ae_uimage[a].page); + + for (a = 0; a < handle->hal_handle->ae_max_num; a++) + qat_uclo_free_ae_data(&obj_handle->ae_data[a]); + + kfree(obj_handle->obj_hdr); + kfree(obj_handle->obj_buf); + kfree(obj_handle); + handle->obj_handle = NULL; +} + +static void qat_uclo_fill_uwords(struct icp_qat_uclo_objhandle *obj_handle, + struct icp_qat_uclo_encap_page *encap_page, + uint64_t *uword, unsigned int addr_p, + unsigned int raddr, uint64_t fill) +{ + uint64_t uwrd = 0; + unsigned int i; + + if (!encap_page) { + *uword = fill; + return; + } + for (i = 0; i < encap_page->uwblock_num; i++) { + if (raddr >= encap_page->uwblock[i].start_addr && + raddr <= encap_page->uwblock[i].start_addr + + encap_page->uwblock[i].words_num - 1) { + raddr -= encap_page->uwblock[i].start_addr; + raddr *= obj_handle->uword_in_bytes; + memcpy(&uwrd, (void *)(((unsigned long) + encap_page->uwblock[i].micro_words) + raddr), + obj_handle->uword_in_bytes); + uwrd = uwrd & 0xbffffffffffull; + } + } + *uword = uwrd; + if (*uword == INVLD_UWORD) + *uword = fill; +} + +static void qat_uclo_wr_uimage_raw_page(struct icp_qat_fw_loader_handle *handle, + struct icp_qat_uclo_encap_page + *encap_page, unsigned int ae) +{ + unsigned int uw_physical_addr, uw_relative_addr, i, words_num, cpylen; + struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + uint64_t fill_pat; + + /* load the page starting at appropriate ustore address */ + /* get fill-pattern from an image -- they are all the same */ + memcpy(&fill_pat, obj_handle->ae_uimage[0].img_ptr->fill_pattern, + sizeof(uint64_t)); + uw_physical_addr = encap_page->beg_addr_p; + uw_relative_addr = 0; + words_num = encap_page->micro_words_num; + while (words_num) { + if (words_num < UWORD_CPYBUF_SIZE) + cpylen = words_num; + else + cpylen = UWORD_CPYBUF_SIZE; + + /* load the buffer */ + for (i = 0; i < cpylen; i++) + qat_uclo_fill_uwords(obj_handle, encap_page, + &obj_handle->uword_buf[i], + uw_physical_addr + i, + uw_relative_addr + i, fill_pat); + + /* copy the buffer to ustore */ + qat_hal_wr_uwords(handle, (unsigned char)ae, + uw_physical_addr, cpylen, + obj_handle->uword_buf); + + uw_physical_addr += cpylen; + uw_relative_addr += cpylen; + words_num -= cpylen; + } +} + +static void qat_uclo_wr_uimage_page(struct icp_qat_fw_loader_handle *handle, + struct icp_qat_uof_image *image) +{ + struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + unsigned int ctx_mask, s; + struct icp_qat_uclo_page *page; + unsigned char ae; + int ctx; + + if (ICP_QAT_CTX_MODE(image->ae_mode) == ICP_QAT_UCLO_MAX_CTX) + ctx_mask = 0xff; + else + ctx_mask = 0x55; + /* load the default page and set assigned CTX PC + * to the entrypoint address */ + for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { + if (!test_bit(ae, (unsigned long *)&image->ae_assigned)) + continue; + /* find the slice to which this image is assigned */ + for (s = 0; s < obj_handle->ae_data[ae].slice_num; s++) { + if (image->ctx_assigned & obj_handle->ae_data[ae]. + ae_slices[s].ctx_mask_assigned) + break; + } + if (s >= obj_handle->ae_data[ae].slice_num) + continue; + page = obj_handle->ae_data[ae].ae_slices[s].page; + if (!page->encap_page->def_page) + continue; + qat_uclo_wr_uimage_raw_page(handle, page->encap_page, ae); + + page = obj_handle->ae_data[ae].ae_slices[s].page; + for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) + obj_handle->ae_data[ae].ae_slices[s].cur_page[ctx] = + (ctx_mask & (1 << ctx)) ? page : NULL; + qat_hal_set_live_ctx(handle, (unsigned char)ae, + image->ctx_assigned); + qat_hal_set_pc(handle, (unsigned char)ae, image->ctx_assigned, + image->entry_address); + } +} + +int qat_uclo_wr_all_uimage(struct icp_qat_fw_loader_handle *handle) +{ + struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + unsigned int i; + + if (qat_uclo_init_globals(handle)) + return -EINVAL; + for (i = 0; i < obj_handle->uimage_num; i++) { + if (!obj_handle->ae_uimage[i].img_ptr) + return -EINVAL; + if (qat_uclo_init_ustore(handle, &obj_handle->ae_uimage[i])) + return -EINVAL; + qat_uclo_wr_uimage_page(handle, + obj_handle->ae_uimage[i].img_ptr); + } + return 0; +} diff --git a/drivers/crypto/qat/qat_dh895xcc/Makefile b/drivers/crypto/qat/qat_dh895xcc/Makefile new file mode 100644 index 000000000000..25171c557043 --- /dev/null +++ b/drivers/crypto/qat/qat_dh895xcc/Makefile @@ -0,0 +1,8 @@ +ccflags-y := -I$(src)/../qat_common +obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCC) += qat_dh895xcc.o +qat_dh895xcc-objs := adf_drv.o \ + adf_isr.o \ + adf_dh895xcc_hw_data.o \ + adf_hw_arbiter.o \ + qat_admin.o \ + adf_admin.o diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_admin.c b/drivers/crypto/qat/qat_dh895xcc/adf_admin.c new file mode 100644 index 000000000000..978d6c56639d --- /dev/null +++ b/drivers/crypto/qat/qat_dh895xcc/adf_admin.c @@ -0,0 +1,144 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#include <linux/types.h> +#include <linux/mutex.h> +#include <linux/slab.h> +#include <linux/delay.h> +#include <linux/pci.h> +#include <linux/dma-mapping.h> +#include <adf_accel_devices.h> +#include "adf_drv.h" +#include "adf_dh895xcc_hw_data.h" + +#define ADF_ADMINMSG_LEN 32 + +struct adf_admin_comms { + dma_addr_t phy_addr; + void *virt_addr; + void __iomem *mailbox_addr; + struct mutex lock; /* protects adf_admin_comms struct */ +}; + +int adf_put_admin_msg_sync(struct adf_accel_dev *accel_dev, + uint32_t ae, void *in, void *out) +{ + struct adf_admin_comms *admin = accel_dev->admin; + int offset = ae * ADF_ADMINMSG_LEN * 2; + void __iomem *mailbox = admin->mailbox_addr; + int mb_offset = ae * ADF_DH895XCC_MAILBOX_STRIDE; + int times, received; + + mutex_lock(&admin->lock); + + if (ADF_CSR_RD(mailbox, mb_offset) == 1) { + mutex_unlock(&admin->lock); + return -EAGAIN; + } + + memcpy(admin->virt_addr + offset, in, ADF_ADMINMSG_LEN); + ADF_CSR_WR(mailbox, mb_offset, 1); + received = 0; + for (times = 0; times < 50; times++) { + msleep(20); + if (ADF_CSR_RD(mailbox, mb_offset) == 0) { + received = 1; + break; + } + } + if (received) + memcpy(out, admin->virt_addr + offset + + ADF_ADMINMSG_LEN, ADF_ADMINMSG_LEN); + else + pr_err("QAT: Failed to send admin msg to accelerator\n"); + + mutex_unlock(&admin->lock); + return received ? 0 : -EFAULT; +} + +int adf_init_admin_comms(struct adf_accel_dev *accel_dev) +{ + struct adf_admin_comms *admin; + struct adf_bar *pmisc = &GET_BARS(accel_dev)[ADF_DH895XCC_PMISC_BAR]; + void __iomem *csr = pmisc->virt_addr; + void __iomem *mailbox = csr + ADF_DH895XCC_MAILBOX_BASE_OFFSET; + uint64_t reg_val; + + admin = kzalloc_node(sizeof(*accel_dev->admin), GFP_KERNEL, + accel_dev->numa_node); + if (!admin) + return -ENOMEM; + admin->virt_addr = dma_zalloc_coherent(&GET_DEV(accel_dev), PAGE_SIZE, + &admin->phy_addr, GFP_KERNEL); + if (!admin->virt_addr) { + dev_err(&GET_DEV(accel_dev), "Failed to allocate dma buff\n"); + kfree(admin); + return -ENOMEM; + } + reg_val = (uint64_t)admin->phy_addr; + ADF_CSR_WR(csr, ADF_DH895XCC_ADMINMSGUR_OFFSET, reg_val >> 32); + ADF_CSR_WR(csr, ADF_DH895XCC_ADMINMSGLR_OFFSET, reg_val); + mutex_init(&admin->lock); + admin->mailbox_addr = mailbox; + accel_dev->admin = admin; + return 0; +} + +void adf_exit_admin_comms(struct adf_accel_dev *accel_dev) +{ + struct adf_admin_comms *admin = accel_dev->admin; + + if (!admin) + return; + + if (admin->virt_addr) + dma_free_coherent(&GET_DEV(accel_dev), PAGE_SIZE, + admin->virt_addr, admin->phy_addr); + + mutex_destroy(&admin->lock); + kfree(admin); + accel_dev->admin = NULL; +} diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c new file mode 100644 index 000000000000..ef05825cc651 --- /dev/null +++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -0,0 +1,214 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#include <adf_accel_devices.h> +#include "adf_dh895xcc_hw_data.h" +#include "adf_drv.h" + +/* Worker thread to service arbiter mappings based on dev SKUs */ +static const uint32_t thrd_to_arb_map_sku4[] = { + 0x12222AAA, 0x11666666, 0x12222AAA, 0x11666666, + 0x12222AAA, 0x11222222, 0x12222AAA, 0x11222222, + 0x00000000, 0x00000000, 0x00000000, 0x00000000 +}; + +static const uint32_t thrd_to_arb_map_sku6[] = { + 0x12222AAA, 0x11666666, 0x12222AAA, 0x11666666, + 0x12222AAA, 0x11222222, 0x12222AAA, 0x11222222, + 0x12222AAA, 0x11222222, 0x12222AAA, 0x11222222 +}; + +static struct adf_hw_device_class dh895xcc_class = { + .name = ADF_DH895XCC_DEVICE_NAME, + .type = DEV_DH895XCC, + .instances = 0 +}; + +static uint32_t get_accel_mask(uint32_t fuse) +{ + return (~fuse) >> ADF_DH895XCC_ACCELERATORS_REG_OFFSET & + ADF_DH895XCC_ACCELERATORS_MASK; +} + +static uint32_t get_ae_mask(uint32_t fuse) +{ + return (~fuse) & ADF_DH895XCC_ACCELENGINES_MASK; +} + +static uint32_t get_num_accels(struct adf_hw_device_data *self) +{ + uint32_t i, ctr = 0; + + if (!self || !self->accel_mask) + return 0; + + for (i = 0; i < ADF_DH895XCC_MAX_ACCELERATORS; i++) { + if (self->accel_mask & (1 << i)) + ctr++; + } + return ctr; +} + +static uint32_t get_num_aes(struct adf_hw_device_data *self) +{ + uint32_t i, ctr = 0; + + if (!self || !self->ae_mask) + return 0; + + for (i = 0; i < ADF_DH895XCC_MAX_ACCELENGINES; i++) { + if (self->ae_mask & (1 << i)) + ctr++; + } + return ctr; +} + +static uint32_t get_misc_bar_id(struct adf_hw_device_data *self) +{ + return ADF_DH895XCC_PMISC_BAR; +} + +static uint32_t get_etr_bar_id(struct adf_hw_device_data *self) +{ + return ADF_DH895XCC_ETR_BAR; +} + +static enum dev_sku_info get_sku(struct adf_hw_device_data *self) +{ + int sku = (self->fuses & ADF_DH895XCC_FUSECTL_SKU_MASK) + >> ADF_DH895XCC_FUSECTL_SKU_SHIFT; + + switch (sku) { + case ADF_DH895XCC_FUSECTL_SKU_1: + return DEV_SKU_1; + case ADF_DH895XCC_FUSECTL_SKU_2: + return DEV_SKU_2; + case ADF_DH895XCC_FUSECTL_SKU_3: + return DEV_SKU_3; + case ADF_DH895XCC_FUSECTL_SKU_4: + return DEV_SKU_4; + default: + return DEV_SKU_UNKNOWN; + } + return DEV_SKU_UNKNOWN; +} + +void adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev, + uint32_t const **arb_map_config) +{ + switch (accel_dev->accel_pci_dev.sku) { + case DEV_SKU_1: + *arb_map_config = thrd_to_arb_map_sku4; + break; + + case DEV_SKU_2: + case DEV_SKU_4: + *arb_map_config = thrd_to_arb_map_sku6; + break; + default: + pr_err("QAT: The configuration doesn't match any SKU"); + *arb_map_config = NULL; + } +} + +static void adf_enable_error_correction(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_device = accel_dev->hw_device; + struct adf_bar *misc_bar = &GET_BARS(accel_dev)[ADF_DH895XCC_PMISC_BAR]; + void __iomem *csr = misc_bar->virt_addr; + unsigned int val, i; + + /* Enable Accel Engine error detection & correction */ + for (i = 0; i < hw_device->get_num_aes(hw_device); i++) { + val = ADF_CSR_RD(csr, ADF_DH895XCC_AE_CTX_ENABLES(i)); + val |= ADF_DH895XCC_ENABLE_AE_ECC_ERR; + ADF_CSR_WR(csr, ADF_DH895XCC_AE_CTX_ENABLES(i), val); + val = ADF_CSR_RD(csr, ADF_DH895XCC_AE_MISC_CONTROL(i)); + val |= ADF_DH895XCC_ENABLE_AE_ECC_PARITY_CORR; + ADF_CSR_WR(csr, ADF_DH895XCC_AE_MISC_CONTROL(i), val); + } + + /* Enable shared memory error detection & correction */ + for (i = 0; i < hw_device->get_num_accels(hw_device); i++) { + val = ADF_CSR_RD(csr, ADF_DH895XCC_UERRSSMSH(i)); + val |= ADF_DH895XCC_ERRSSMSH_EN; + ADF_CSR_WR(csr, ADF_DH895XCC_UERRSSMSH(i), val); + val = ADF_CSR_RD(csr, ADF_DH895XCC_CERRSSMSH(i)); + val |= ADF_DH895XCC_ERRSSMSH_EN; + ADF_CSR_WR(csr, ADF_DH895XCC_CERRSSMSH(i), val); + } +} + +void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) +{ + hw_data->dev_class = &dh895xcc_class; + hw_data->instance_id = dh895xcc_class.instances++; + hw_data->num_banks = ADF_DH895XCC_ETR_MAX_BANKS; + hw_data->num_accel = ADF_DH895XCC_MAX_ACCELERATORS; + hw_data->pci_dev_id = ADF_DH895XCC_PCI_DEVICE_ID; + hw_data->num_logical_accel = 1; + hw_data->num_engines = ADF_DH895XCC_MAX_ACCELENGINES; + hw_data->tx_rx_gap = ADF_DH895XCC_RX_RINGS_OFFSET; + hw_data->tx_rings_mask = ADF_DH895XCC_TX_RINGS_MASK; + hw_data->alloc_irq = adf_isr_resource_alloc; + hw_data->free_irq = adf_isr_resource_free; + hw_data->enable_error_correction = adf_enable_error_correction; + hw_data->hw_arb_ring_enable = adf_update_ring_arb_enable; + hw_data->hw_arb_ring_disable = adf_update_ring_arb_enable; + hw_data->get_accel_mask = get_accel_mask; + hw_data->get_ae_mask = get_ae_mask; + hw_data->get_num_accels = get_num_accels; + hw_data->get_num_aes = get_num_aes; + hw_data->get_etr_bar_id = get_etr_bar_id; + hw_data->get_misc_bar_id = get_misc_bar_id; + hw_data->get_sku = get_sku; + hw_data->fw_name = ADF_DH895XCC_FW; +} + +void adf_clean_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) +{ + hw_data->dev_class->instances--; +} diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h new file mode 100644 index 000000000000..b707f292b377 --- /dev/null +++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h @@ -0,0 +1,86 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef ADF_DH895x_HW_DATA_H_ +#define ADF_DH895x_HW_DATA_H_ + +/* PCIe configuration space */ +#define ADF_DH895XCC_RX_RINGS_OFFSET 8 +#define ADF_DH895XCC_TX_RINGS_MASK 0xFF +#define ADF_DH895XCC_FUSECTL_OFFSET 0x40 +#define ADF_DH895XCC_FUSECTL_SKU_MASK 0x300000 +#define ADF_DH895XCC_FUSECTL_SKU_SHIFT 20 +#define ADF_DH895XCC_FUSECTL_SKU_1 0x0 +#define ADF_DH895XCC_FUSECTL_SKU_2 0x1 +#define ADF_DH895XCC_FUSECTL_SKU_3 0x2 +#define ADF_DH895XCC_FUSECTL_SKU_4 0x3 +#define ADF_DH895XCC_MAX_ACCELERATORS 6 +#define ADF_DH895XCC_MAX_ACCELENGINES 12 +#define ADF_DH895XCC_ACCELERATORS_REG_OFFSET 13 +#define ADF_DH895XCC_ACCELERATORS_MASK 0x3F +#define ADF_DH895XCC_ACCELENGINES_MASK 0xFFF +#define ADF_DH895XCC_LEGFUSE_OFFSET 0x4C +#define ADF_DH895XCC_ETR_MAX_BANKS 32 +#define ADF_DH895XCC_SMIAPF0_MASK_OFFSET (0x3A000 + 0x28) +#define ADF_DH895XCC_SMIAPF1_MASK_OFFSET (0x3A000 + 0x30) +#define ADF_DH895XCC_SMIA0_MASK 0xFFFF +#define ADF_DH895XCC_SMIA1_MASK 0x1 +/* Error detection and correction */ +#define ADF_DH895XCC_AE_CTX_ENABLES(i) (i * 0x1000 + 0x20818) +#define ADF_DH895XCC_AE_MISC_CONTROL(i) (i * 0x1000 + 0x20960) +#define ADF_DH895XCC_ENABLE_AE_ECC_ERR (1 << 28) +#define ADF_DH895XCC_ENABLE_AE_ECC_PARITY_CORR (1 << 24 | 1 << 12) +#define ADF_DH895XCC_UERRSSMSH(i) (i * 0x4000 + 0x18) +#define ADF_DH895XCC_CERRSSMSH(i) (i * 0x4000 + 0x10) +#define ADF_DH895XCC_ERRSSMSH_EN (1 << 3) + +/* Admin Messages Registers */ +#define ADF_DH895XCC_ADMINMSGUR_OFFSET (0x3A000 + 0x574) +#define ADF_DH895XCC_ADMINMSGLR_OFFSET (0x3A000 + 0x578) +#define ADF_DH895XCC_MAILBOX_BASE_OFFSET 0x20970 +#define ADF_DH895XCC_MAILBOX_STRIDE 0x1000 +#define ADF_DH895XCC_FW "qat_895xcc.bin" +#endif diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_drv.c b/drivers/crypto/qat/qat_dh895xcc/adf_drv.c new file mode 100644 index 000000000000..0d0435a41be9 --- /dev/null +++ b/drivers/crypto/qat/qat_dh895xcc/adf_drv.c @@ -0,0 +1,449 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/init.h> +#include <linux/types.h> +#include <linux/fs.h> +#include <linux/slab.h> +#include <linux/errno.h> +#include <linux/device.h> +#include <linux/dma-mapping.h> +#include <linux/platform_device.h> +#include <linux/workqueue.h> +#include <linux/io.h> +#include <adf_accel_devices.h> +#include <adf_common_drv.h> +#include <adf_cfg.h> +#include <adf_transport_access_macros.h> +#include "adf_dh895xcc_hw_data.h" +#include "adf_drv.h" + +static const char adf_driver_name[] = ADF_DH895XCC_DEVICE_NAME; + +#define ADF_SYSTEM_DEVICE(device_id) \ + {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)} + +static const struct pci_device_id adf_pci_tbl[] = { + ADF_SYSTEM_DEVICE(ADF_DH895XCC_PCI_DEVICE_ID), + {0,} +}; +MODULE_DEVICE_TABLE(pci, adf_pci_tbl); + +static int adf_probe(struct pci_dev *dev, const struct pci_device_id *ent); +static void adf_remove(struct pci_dev *dev); + +static struct pci_driver adf_driver = { + .id_table = adf_pci_tbl, + .name = adf_driver_name, + .probe = adf_probe, + .remove = adf_remove +}; + +static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) +{ + struct adf_accel_pci *accel_pci_dev = &accel_dev->accel_pci_dev; + int i; + + adf_exit_admin_comms(accel_dev); + adf_exit_arb(accel_dev); + adf_cleanup_etr_data(accel_dev); + + for (i = 0; i < ADF_PCI_MAX_BARS; i++) { + struct adf_bar *bar = &accel_pci_dev->pci_bars[i]; + + if (bar->virt_addr) + pci_iounmap(accel_pci_dev->pci_dev, bar->virt_addr); + } + + if (accel_dev->hw_device) { + switch (accel_dev->hw_device->pci_dev_id) { + case ADF_DH895XCC_PCI_DEVICE_ID: + adf_clean_hw_data_dh895xcc(accel_dev->hw_device); + break; + default: + break; + } + kfree(accel_dev->hw_device); + } + adf_cfg_dev_remove(accel_dev); + debugfs_remove(accel_dev->debugfs_dir); + adf_devmgr_rm_dev(accel_dev); + pci_release_regions(accel_pci_dev->pci_dev); + pci_disable_device(accel_pci_dev->pci_dev); + kfree(accel_dev); +} + +static uint8_t adf_get_dev_node_id(struct pci_dev *pdev) +{ + unsigned int bus_per_cpu = 0; + struct cpuinfo_x86 *c = &cpu_data(num_online_cpus() - 1); + + if (!c->phys_proc_id) + return 0; + + bus_per_cpu = 256 / (c->phys_proc_id + 1); + + if (bus_per_cpu != 0) + return pdev->bus->number / bus_per_cpu; + return 0; +} + +static int qat_dev_start(struct adf_accel_dev *accel_dev) +{ + int cpus = num_online_cpus(); + int banks = GET_MAX_BANKS(accel_dev); + int instances = min(cpus, banks); + char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; + int i; + unsigned long val; + + if (adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC)) + goto err; + if (adf_cfg_section_add(accel_dev, "Accelerator0")) + goto err; + for (i = 0; i < instances; i++) { + val = i; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_BANK_NUM, i); + if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, (void *)&val, ADF_DEC)) + goto err; + + snprintf(key, sizeof(key), ADF_CY "%d" ADF_ETRMGR_CORE_AFFINITY, + i); + if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, (void *)&val, ADF_DEC)) + goto err; + + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, i); + val = 128; + if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, (void *)&val, ADF_DEC)) + goto err; + + val = 512; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, i); + if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, (void *)&val, ADF_DEC)) + goto err; + + val = 0; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, i); + if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, (void *)&val, ADF_DEC)) + goto err; + + val = 2; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, i); + if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, (void *)&val, ADF_DEC)) + goto err; + + val = 4; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_RND_TX, i); + if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, (void *)&val, ADF_DEC)) + goto err; + + val = 8; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, i); + if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, (void *)&val, ADF_DEC)) + goto err; + + val = 10; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, i); + if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, (void *)&val, ADF_DEC)) + goto err; + + val = 12; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_RND_RX, i); + if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, (void *)&val, ADF_DEC)) + goto err; + + val = ADF_COALESCING_DEF_TIME; + snprintf(key, sizeof(key), ADF_ETRMGR_COALESCE_TIMER_FORMAT, i); + if (adf_cfg_add_key_value_param(accel_dev, "Accelerator0", + key, (void *)&val, ADF_DEC)) + goto err; + } + + val = i; + if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + ADF_NUM_CY, (void *)&val, ADF_DEC)) + goto err; + + set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); + return adf_dev_start(accel_dev); +err: + dev_err(&GET_DEV(accel_dev), "Failed to start QAT accel dev\n"); + return -EINVAL; +} + +static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + struct adf_accel_dev *accel_dev; + struct adf_accel_pci *accel_pci_dev; + struct adf_hw_device_data *hw_data; + void __iomem *pmisc_bar_addr = NULL; + char name[ADF_DEVICE_NAME_LENGTH]; + unsigned int i, bar_nr; + uint8_t node; + int ret; + + switch (ent->device) { + case ADF_DH895XCC_PCI_DEVICE_ID: + break; + default: + dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device); + return -ENODEV; + } + + node = adf_get_dev_node_id(pdev); + accel_dev = kzalloc_node(sizeof(*accel_dev), GFP_KERNEL, node); + if (!accel_dev) + return -ENOMEM; + + accel_dev->numa_node = node; + INIT_LIST_HEAD(&accel_dev->crypto_list); + + /* Add accel device to accel table. + * This should be called before adf_cleanup_accel is called */ + if (adf_devmgr_add_dev(accel_dev)) { + dev_err(&pdev->dev, "Failed to add new accelerator device.\n"); + kfree(accel_dev); + return -EFAULT; + } + + accel_dev->owner = THIS_MODULE; + /* Allocate and configure device configuration structure */ + hw_data = kzalloc_node(sizeof(*hw_data), GFP_KERNEL, node); + if (!hw_data) { + ret = -ENOMEM; + goto out_err; + } + + accel_dev->hw_device = hw_data; + switch (ent->device) { + case ADF_DH895XCC_PCI_DEVICE_ID: + adf_init_hw_data_dh895xcc(accel_dev->hw_device); + break; + default: + return -ENODEV; + } + accel_pci_dev = &accel_dev->accel_pci_dev; + pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); + pci_read_config_dword(pdev, ADF_DH895XCC_FUSECTL_OFFSET, + &hw_data->fuses); + + /* Get Accelerators and Accelerators Engines masks */ + hw_data->accel_mask = hw_data->get_accel_mask(hw_data->fuses); + hw_data->ae_mask = hw_data->get_ae_mask(hw_data->fuses); + accel_pci_dev->sku = hw_data->get_sku(hw_data); + accel_pci_dev->pci_dev = pdev; + /* If the device has no acceleration engines then ignore it. */ + if (!hw_data->accel_mask || !hw_data->ae_mask || + ((~hw_data->ae_mask) & 0x01)) { + dev_err(&pdev->dev, "No acceleration units found"); + ret = -EFAULT; + goto out_err; + } + + /* Create dev top level debugfs entry */ + snprintf(name, sizeof(name), "%s%s_dev%d", ADF_DEVICE_NAME_PREFIX, + hw_data->dev_class->name, hw_data->instance_id); + accel_dev->debugfs_dir = debugfs_create_dir(name, NULL); + if (!accel_dev->debugfs_dir) { + dev_err(&pdev->dev, "Could not create debugfs dir\n"); + ret = -EINVAL; + goto out_err; + } + + /* Create device configuration table */ + ret = adf_cfg_dev_add(accel_dev); + if (ret) + goto out_err; + + /* enable PCI device */ + if (pci_enable_device(pdev)) { + ret = -EFAULT; + goto out_err; + } + + /* set dma identifier */ + if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) { + if ((pci_set_dma_mask(pdev, DMA_BIT_MASK(32)))) { + dev_err(&pdev->dev, "No usable DMA configuration\n"); + ret = -EFAULT; + goto out_err; + } else { + pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); + } + + } else { + pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); + } + + if (pci_request_regions(pdev, adf_driver_name)) { + ret = -EFAULT; + goto out_err; + } + + /* Read accelerator capabilities mask */ + pci_read_config_dword(pdev, ADF_DH895XCC_LEGFUSE_OFFSET, + &hw_data->accel_capabilities_mask); + + /* Find and map all the device's BARS */ + for (i = 0; i < ADF_PCI_MAX_BARS; i++) { + struct adf_bar *bar = &accel_pci_dev->pci_bars[i]; + + bar_nr = i * 2; + bar->base_addr = pci_resource_start(pdev, bar_nr); + if (!bar->base_addr) + break; + bar->size = pci_resource_len(pdev, bar_nr); + bar->virt_addr = pci_iomap(accel_pci_dev->pci_dev, bar_nr, 0); + if (!bar->virt_addr) { + dev_err(&pdev->dev, "Failed to map BAR %d\n", i); + ret = -EFAULT; + goto out_err; + } + if (i == ADF_DH895XCC_PMISC_BAR) + pmisc_bar_addr = bar->virt_addr; + } + pci_set_master(pdev); + + if (adf_enable_aer(accel_dev, &adf_driver)) { + dev_err(&pdev->dev, "Failed to enable aer\n"); + ret = -EFAULT; + goto out_err; + } + + if (adf_init_etr_data(accel_dev)) { + dev_err(&pdev->dev, "Failed initialize etr\n"); + ret = -EFAULT; + goto out_err; + } + + if (adf_init_admin_comms(accel_dev)) { + dev_err(&pdev->dev, "Failed initialize admin comms\n"); + ret = -EFAULT; + goto out_err; + } + + if (adf_init_arb(accel_dev)) { + dev_err(&pdev->dev, "Failed initialize hw arbiter\n"); + ret = -EFAULT; + goto out_err; + } + if (pci_save_state(pdev)) { + dev_err(&pdev->dev, "Failed to save pci state\n"); + ret = -ENOMEM; + goto out_err; + } + + /* Enable bundle and misc interrupts */ + ADF_CSR_WR(pmisc_bar_addr, ADF_DH895XCC_SMIAPF0_MASK_OFFSET, + ADF_DH895XCC_SMIA0_MASK); + ADF_CSR_WR(pmisc_bar_addr, ADF_DH895XCC_SMIAPF1_MASK_OFFSET, + ADF_DH895XCC_SMIA1_MASK); + + ret = qat_dev_start(accel_dev); + if (ret) { + adf_dev_stop(accel_dev); + goto out_err; + } + + return 0; +out_err: + adf_cleanup_accel(accel_dev); + return ret; +} + +static void __exit adf_remove(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + if (!accel_dev) { + pr_err("QAT: Driver removal failed\n"); + return; + } + if (adf_dev_stop(accel_dev)) + dev_err(&GET_DEV(accel_dev), "Failed to stop QAT accel dev\n"); + adf_disable_aer(accel_dev); + adf_cleanup_accel(accel_dev); +} + +static int __init adfdrv_init(void) +{ + request_module("intel_qat"); + if (qat_admin_register()) + return -EFAULT; + + if (pci_register_driver(&adf_driver)) { + pr_err("QAT: Driver initialization failed\n"); + return -EFAULT; + } + return 0; +} + +static void __exit adfdrv_release(void) +{ + pci_unregister_driver(&adf_driver); + qat_admin_unregister(); +} + +module_init(adfdrv_init); +module_exit(adfdrv_release); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Intel"); +MODULE_FIRMWARE("qat_895xcc.bin"); +MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_drv.h b/drivers/crypto/qat/qat_dh895xcc/adf_drv.h new file mode 100644 index 000000000000..a2fbb6ce75cd --- /dev/null +++ b/drivers/crypto/qat/qat_dh895xcc/adf_drv.h @@ -0,0 +1,67 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef ADF_DH895x_DRV_H_ +#define ADF_DH895x_DRV_H_ +#include <adf_accel_devices.h> +#include <adf_transport.h> + +void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data); +void adf_clean_hw_data_dh895xcc(struct adf_hw_device_data *hw_data); +int adf_isr_resource_alloc(struct adf_accel_dev *accel_dev); +void adf_isr_resource_free(struct adf_accel_dev *accel_dev); +void adf_update_ring_arb_enable(struct adf_etr_ring_data *ring); +void adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev, + uint32_t const **arb_map_config); +int adf_init_admin_comms(struct adf_accel_dev *accel_dev); +void adf_exit_admin_comms(struct adf_accel_dev *accel_dev); +int adf_put_admin_msg_sync(struct adf_accel_dev *accel_dev, + uint32_t ae, void *in, void *out); +int qat_admin_register(void); +int qat_admin_unregister(void); +int adf_init_arb(struct adf_accel_dev *accel_dev); +void adf_exit_arb(struct adf_accel_dev *accel_dev); +#endif diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_hw_arbiter.c b/drivers/crypto/qat/qat_dh895xcc/adf_hw_arbiter.c new file mode 100644 index 000000000000..1864bdb36f8f --- /dev/null +++ b/drivers/crypto/qat/qat_dh895xcc/adf_hw_arbiter.c @@ -0,0 +1,159 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#include <adf_accel_devices.h> +#include <adf_transport_internal.h> +#include "adf_drv.h" + +#define ADF_ARB_NUM 4 +#define ADF_ARB_REQ_RING_NUM 8 +#define ADF_ARB_REG_SIZE 0x4 +#define ADF_ARB_WTR_SIZE 0x20 +#define ADF_ARB_OFFSET 0x30000 +#define ADF_ARB_REG_SLOT 0x1000 +#define ADF_ARB_WTR_OFFSET 0x010 +#define ADF_ARB_RO_EN_OFFSET 0x090 +#define ADF_ARB_WQCFG_OFFSET 0x100 +#define ADF_ARB_WRK_2_SER_MAP_OFFSET 0x180 +#define ADF_ARB_WRK_2_SER_MAP 10 +#define ADF_ARB_RINGSRVARBEN_OFFSET 0x19C + +#define WRITE_CSR_ARB_RINGSRVARBEN(csr_addr, index, value) \ + ADF_CSR_WR(csr_addr, ADF_ARB_RINGSRVARBEN_OFFSET + \ + (ADF_ARB_REG_SLOT * index), value) + +#define WRITE_CSR_ARB_RESPORDERING(csr_addr, index, value) \ + ADF_CSR_WR(csr_addr, (ADF_ARB_OFFSET + \ + ADF_ARB_RO_EN_OFFSET) + (ADF_ARB_REG_SIZE * index), value) + +#define WRITE_CSR_ARB_WEIGHT(csr_addr, arb, index, value) \ + ADF_CSR_WR(csr_addr, (ADF_ARB_OFFSET + \ + ADF_ARB_WTR_OFFSET) + (ADF_ARB_WTR_SIZE * arb) + \ + (ADF_ARB_REG_SIZE * index), value) + +#define WRITE_CSR_ARB_SARCONFIG(csr_addr, index, value) \ + ADF_CSR_WR(csr_addr, ADF_ARB_OFFSET + \ + (ADF_ARB_REG_SIZE * index), value) + +#define WRITE_CSR_ARB_WRK_2_SER_MAP(csr_addr, index, value) \ + ADF_CSR_WR(csr_addr, (ADF_ARB_OFFSET + \ + ADF_ARB_WRK_2_SER_MAP_OFFSET) + \ + (ADF_ARB_REG_SIZE * index), value) + +#define WRITE_CSR_ARB_WQCFG(csr_addr, index, value) \ + ADF_CSR_WR(csr_addr, (ADF_ARB_OFFSET + \ + ADF_ARB_WQCFG_OFFSET) + (ADF_ARB_REG_SIZE * index), value) + +int adf_init_arb(struct adf_accel_dev *accel_dev) +{ + void __iomem *csr = accel_dev->transport->banks[0].csr_addr; + uint32_t arb_cfg = 0x1 << 31 | 0x4 << 4 | 0x1; + uint32_t arb, i; + const uint32_t *thd_2_arb_cfg; + + /* Service arb configured for 32 bytes responses and + * ring flow control check enabled. */ + for (arb = 0; arb < ADF_ARB_NUM; arb++) + WRITE_CSR_ARB_SARCONFIG(csr, arb, arb_cfg); + + /* Setup service weighting */ + for (arb = 0; arb < ADF_ARB_NUM; arb++) + for (i = 0; i < ADF_ARB_REQ_RING_NUM; i++) + WRITE_CSR_ARB_WEIGHT(csr, arb, i, 0xFFFFFFFF); + + /* Setup ring response ordering */ + for (i = 0; i < ADF_ARB_REQ_RING_NUM; i++) + WRITE_CSR_ARB_RESPORDERING(csr, i, 0xFFFFFFFF); + + /* Setup worker queue registers */ + for (i = 0; i < ADF_ARB_WRK_2_SER_MAP; i++) + WRITE_CSR_ARB_WQCFG(csr, i, i); + + /* Map worker threads to service arbiters */ + adf_get_arbiter_mapping(accel_dev, &thd_2_arb_cfg); + + if (!thd_2_arb_cfg) + return -EFAULT; + + for (i = 0; i < ADF_ARB_WRK_2_SER_MAP; i++) + WRITE_CSR_ARB_WRK_2_SER_MAP(csr, i, *(thd_2_arb_cfg + i)); + + return 0; +} + +void adf_update_ring_arb_enable(struct adf_etr_ring_data *ring) +{ + WRITE_CSR_ARB_RINGSRVARBEN(ring->bank->csr_addr, + ring->bank->bank_number, + ring->bank->ring_mask & 0xFF); +} + +void adf_exit_arb(struct adf_accel_dev *accel_dev) +{ + void __iomem *csr; + unsigned int i; + + if (!accel_dev->transport) + return; + + csr = accel_dev->transport->banks[0].csr_addr; + + /* Reset arbiter configuration */ + for (i = 0; i < ADF_ARB_NUM; i++) + WRITE_CSR_ARB_SARCONFIG(csr, i, 0); + + /* Shutdown work queue */ + for (i = 0; i < ADF_ARB_WRK_2_SER_MAP; i++) + WRITE_CSR_ARB_WQCFG(csr, i, 0); + + /* Unmap worker threads to service arbiters */ + for (i = 0; i < ADF_ARB_WRK_2_SER_MAP; i++) + WRITE_CSR_ARB_WRK_2_SER_MAP(csr, i, 0); + + /* Disable arbitration on all rings */ + for (i = 0; i < GET_MAX_BANKS(accel_dev); i++) + WRITE_CSR_ARB_RINGSRVARBEN(csr, i, 0); +} diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_isr.c b/drivers/crypto/qat/qat_dh895xcc/adf_isr.c new file mode 100644 index 000000000000..d4172dedf775 --- /dev/null +++ b/drivers/crypto/qat/qat_dh895xcc/adf_isr.c @@ -0,0 +1,266 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#include <linux/kernel.h> +#include <linux/init.h> +#include <linux/types.h> +#include <linux/pci.h> +#include <linux/slab.h> +#include <linux/errno.h> +#include <linux/interrupt.h> +#include <adf_accel_devices.h> +#include <adf_common_drv.h> +#include <adf_cfg.h> +#include <adf_cfg_strings.h> +#include <adf_cfg_common.h> +#include <adf_transport_access_macros.h> +#include <adf_transport_internal.h> +#include "adf_drv.h" + +static int adf_enable_msix(struct adf_accel_dev *accel_dev) +{ + struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + uint32_t msix_num_entries = hw_data->num_banks + 1; + int i; + + for (i = 0; i < msix_num_entries; i++) + pci_dev_info->msix_entries.entries[i].entry = i; + + if (pci_enable_msix(pci_dev_info->pci_dev, + pci_dev_info->msix_entries.entries, + msix_num_entries)) { + pr_err("QAT: Failed to enable MSIX IRQ\n"); + return -EFAULT; + } + return 0; +} + +static void adf_disable_msix(struct adf_accel_pci *pci_dev_info) +{ + pci_disable_msix(pci_dev_info->pci_dev); +} + +static irqreturn_t adf_msix_isr_bundle(int irq, void *bank_ptr) +{ + struct adf_etr_bank_data *bank = bank_ptr; + + WRITE_CSR_INT_FLAG_AND_COL(bank->csr_addr, bank->bank_number, 0); + tasklet_hi_schedule(&bank->resp_hanlder); + return IRQ_HANDLED; +} + +static irqreturn_t adf_msix_isr_ae(int irq, void *dev_ptr) +{ + struct adf_accel_dev *accel_dev = dev_ptr; + + pr_info("QAT: qat_dev%d spurious AE interrupt\n", accel_dev->accel_id); + return IRQ_HANDLED; +} + +static int adf_request_irqs(struct adf_accel_dev *accel_dev) +{ + struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct msix_entry *msixe = pci_dev_info->msix_entries.entries; + struct adf_etr_data *etr_data = accel_dev->transport; + int ret, i; + char *name; + + /* Request msix irq for all banks */ + for (i = 0; i < hw_data->num_banks; i++) { + struct adf_etr_bank_data *bank = &etr_data->banks[i]; + unsigned int cpu, cpus = num_online_cpus(); + + name = *(pci_dev_info->msix_entries.names + i); + snprintf(name, ADF_MAX_MSIX_VECTOR_NAME, + "qat%d-bundle%d", accel_dev->accel_id, i); + ret = request_irq(msixe[i].vector, + adf_msix_isr_bundle, 0, name, bank); + if (ret) { + pr_err("QAT: failed to enable irq %d for %s\n", + msixe[i].vector, name); + return ret; + } + + cpu = ((accel_dev->accel_id * hw_data->num_banks) + i) % cpus; + irq_set_affinity_hint(msixe[i].vector, get_cpu_mask(cpu)); + } + + /* Request msix irq for AE */ + name = *(pci_dev_info->msix_entries.names + i); + snprintf(name, ADF_MAX_MSIX_VECTOR_NAME, + "qat%d-ae-cluster", accel_dev->accel_id); + ret = request_irq(msixe[i].vector, adf_msix_isr_ae, 0, name, accel_dev); + if (ret) { + pr_err("QAT: failed to enable irq %d, for %s\n", + msixe[i].vector, name); + return ret; + } + return ret; +} + +static void adf_free_irqs(struct adf_accel_dev *accel_dev) +{ + struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct msix_entry *msixe = pci_dev_info->msix_entries.entries; + struct adf_etr_data *etr_data = accel_dev->transport; + int i; + + for (i = 0; i < hw_data->num_banks; i++) { + irq_set_affinity_hint(msixe[i].vector, NULL); + free_irq(msixe[i].vector, &etr_data->banks[i]); + } + irq_set_affinity_hint(msixe[i].vector, NULL); + free_irq(msixe[i].vector, accel_dev); +} + +static int adf_isr_alloc_msix_entry_table(struct adf_accel_dev *accel_dev) +{ + int i; + char **names; + struct msix_entry *entries; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + uint32_t msix_num_entries = hw_data->num_banks + 1; + + entries = kzalloc_node(msix_num_entries * sizeof(*entries), + GFP_KERNEL, accel_dev->numa_node); + if (!entries) + return -ENOMEM; + + names = kcalloc(msix_num_entries, sizeof(char *), GFP_KERNEL); + if (!names) { + kfree(entries); + return -ENOMEM; + } + for (i = 0; i < msix_num_entries; i++) { + *(names + i) = kzalloc(ADF_MAX_MSIX_VECTOR_NAME, GFP_KERNEL); + if (!(*(names + i))) + goto err; + } + accel_dev->accel_pci_dev.msix_entries.entries = entries; + accel_dev->accel_pci_dev.msix_entries.names = names; + return 0; +err: + for (i = 0; i < msix_num_entries; i++) { + if (*(names + i)) + kfree(*(names + i)); + } + kfree(entries); + kfree(names); + return -ENOMEM; +} + +static void adf_isr_free_msix_entry_table(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + uint32_t msix_num_entries = hw_data->num_banks + 1; + char **names = accel_dev->accel_pci_dev.msix_entries.names; + int i; + + kfree(accel_dev->accel_pci_dev.msix_entries.entries); + for (i = 0; i < msix_num_entries; i++) { + if (*(names + i)) + kfree(*(names + i)); + } + kfree(names); +} + +static int adf_setup_bh(struct adf_accel_dev *accel_dev) +{ + struct adf_etr_data *priv_data = accel_dev->transport; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + int i; + + for (i = 0; i < hw_data->num_banks; i++) + tasklet_init(&priv_data->banks[i].resp_hanlder, + adf_response_handler, + (unsigned long)&priv_data->banks[i]); + return 0; +} + +static void adf_cleanup_bh(struct adf_accel_dev *accel_dev) +{ + struct adf_etr_data *priv_data = accel_dev->transport; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + int i; + + for (i = 0; i < hw_data->num_banks; i++) { + tasklet_disable(&priv_data->banks[i].resp_hanlder); + tasklet_kill(&priv_data->banks[i].resp_hanlder); + } +} + +void adf_isr_resource_free(struct adf_accel_dev *accel_dev) +{ + adf_free_irqs(accel_dev); + adf_cleanup_bh(accel_dev); + adf_disable_msix(&accel_dev->accel_pci_dev); + adf_isr_free_msix_entry_table(accel_dev); +} + +int adf_isr_resource_alloc(struct adf_accel_dev *accel_dev) +{ + int ret; + + ret = adf_isr_alloc_msix_entry_table(accel_dev); + if (ret) + return ret; + if (adf_enable_msix(accel_dev)) + goto err_out; + + if (adf_setup_bh(accel_dev)) + goto err_out; + + if (adf_request_irqs(accel_dev)) + goto err_out; + + return 0; +err_out: + adf_isr_resource_free(accel_dev); + return -EFAULT; +} diff --git a/drivers/crypto/qat/qat_dh895xcc/qat_admin.c b/drivers/crypto/qat/qat_dh895xcc/qat_admin.c new file mode 100644 index 000000000000..55b7a8e48bad --- /dev/null +++ b/drivers/crypto/qat/qat_dh895xcc/qat_admin.c @@ -0,0 +1,107 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#include <icp_qat_fw_init_admin.h> +#include <adf_accel_devices.h> +#include <adf_common_drv.h> +#include "adf_drv.h" + +static struct service_hndl qat_admin; + +static int qat_send_admin_cmd(struct adf_accel_dev *accel_dev, int cmd) +{ + struct adf_hw_device_data *hw_device = accel_dev->hw_device; + struct icp_qat_fw_init_admin_req req; + struct icp_qat_fw_init_admin_resp resp; + int i; + + memset(&req, 0, sizeof(struct icp_qat_fw_init_admin_req)); + req.init_admin_cmd_id = cmd; + for (i = 0; i < hw_device->get_num_aes(hw_device); i++) { + memset(&resp, 0, sizeof(struct icp_qat_fw_init_admin_resp)); + if (adf_put_admin_msg_sync(accel_dev, i, &req, &resp) || + resp.init_resp_hdr.status) + return -EFAULT; + } + return 0; +} + +static int qat_admin_start(struct adf_accel_dev *accel_dev) +{ + return qat_send_admin_cmd(accel_dev, ICP_QAT_FW_INIT_ME); +} + +static int qat_admin_event_handler(struct adf_accel_dev *accel_dev, + enum adf_event event) +{ + int ret; + + switch (event) { + case ADF_EVENT_START: + ret = qat_admin_start(accel_dev); + break; + case ADF_EVENT_STOP: + case ADF_EVENT_INIT: + case ADF_EVENT_SHUTDOWN: + default: + ret = 0; + } + return ret; +} + +int qat_admin_register(void) +{ + memset(&qat_admin, 0, sizeof(struct service_hndl)); + qat_admin.event_hld = qat_admin_event_handler; + qat_admin.name = "qat_admin"; + qat_admin.admin = 1; + return adf_service_register(&qat_admin); +} + +int qat_admin_unregister(void) +{ + return adf_service_unregister(&qat_admin); +} diff --git a/drivers/crypto/qce/Makefile b/drivers/crypto/qce/Makefile new file mode 100644 index 000000000000..348dc3173afa --- /dev/null +++ b/drivers/crypto/qce/Makefile @@ -0,0 +1,6 @@ +obj-$(CONFIG_CRYPTO_DEV_QCE) += qcrypto.o +qcrypto-objs := core.o \ + common.o \ + dma.o \ + sha.o \ + ablkcipher.o diff --git a/drivers/crypto/qce/ablkcipher.c b/drivers/crypto/qce/ablkcipher.c new file mode 100644 index 000000000000..ad592de475a4 --- /dev/null +++ b/drivers/crypto/qce/ablkcipher.c @@ -0,0 +1,431 @@ +/* + * Copyright (c) 2010-2014, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include <linux/device.h> +#include <linux/interrupt.h> +#include <linux/types.h> +#include <crypto/aes.h> +#include <crypto/algapi.h> +#include <crypto/des.h> + +#include "cipher.h" + +static LIST_HEAD(ablkcipher_algs); + +static void qce_ablkcipher_done(void *data) +{ + struct crypto_async_request *async_req = data; + struct ablkcipher_request *req = ablkcipher_request_cast(async_req); + struct qce_cipher_reqctx *rctx = ablkcipher_request_ctx(req); + struct qce_alg_template *tmpl = to_cipher_tmpl(async_req->tfm); + struct qce_device *qce = tmpl->qce; + enum dma_data_direction dir_src, dir_dst; + u32 status; + int error; + bool diff_dst; + + diff_dst = (req->src != req->dst) ? true : false; + dir_src = diff_dst ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL; + dir_dst = diff_dst ? DMA_FROM_DEVICE : DMA_BIDIRECTIONAL; + + error = qce_dma_terminate_all(&qce->dma); + if (error) + dev_dbg(qce->dev, "ablkcipher dma termination error (%d)\n", + error); + + if (diff_dst) + qce_unmapsg(qce->dev, rctx->src_sg, rctx->src_nents, dir_src, + rctx->dst_chained); + qce_unmapsg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst, + rctx->dst_chained); + + sg_free_table(&rctx->dst_tbl); + + error = qce_check_status(qce, &status); + if (error < 0) + dev_dbg(qce->dev, "ablkcipher operation error (%x)\n", status); + + qce->async_req_done(tmpl->qce, error); +} + +static int +qce_ablkcipher_async_req_handle(struct crypto_async_request *async_req) +{ + struct ablkcipher_request *req = ablkcipher_request_cast(async_req); + struct qce_cipher_reqctx *rctx = ablkcipher_request_ctx(req); + struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); + struct qce_alg_template *tmpl = to_cipher_tmpl(async_req->tfm); + struct qce_device *qce = tmpl->qce; + enum dma_data_direction dir_src, dir_dst; + struct scatterlist *sg; + bool diff_dst; + gfp_t gfp; + int ret; + + rctx->iv = req->info; + rctx->ivsize = crypto_ablkcipher_ivsize(ablkcipher); + rctx->cryptlen = req->nbytes; + + diff_dst = (req->src != req->dst) ? true : false; + dir_src = diff_dst ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL; + dir_dst = diff_dst ? DMA_FROM_DEVICE : DMA_BIDIRECTIONAL; + + rctx->src_nents = qce_countsg(req->src, req->nbytes, + &rctx->src_chained); + if (diff_dst) { + rctx->dst_nents = qce_countsg(req->dst, req->nbytes, + &rctx->dst_chained); + } else { + rctx->dst_nents = rctx->src_nents; + rctx->dst_chained = rctx->src_chained; + } + + rctx->dst_nents += 1; + + gfp = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + + ret = sg_alloc_table(&rctx->dst_tbl, rctx->dst_nents, gfp); + if (ret) + return ret; + + sg_init_one(&rctx->result_sg, qce->dma.result_buf, QCE_RESULT_BUF_SZ); + + sg = qce_sgtable_add(&rctx->dst_tbl, req->dst); + if (IS_ERR(sg)) { + ret = PTR_ERR(sg); + goto error_free; + } + + sg = qce_sgtable_add(&rctx->dst_tbl, &rctx->result_sg); + if (IS_ERR(sg)) { + ret = PTR_ERR(sg); + goto error_free; + } + + sg_mark_end(sg); + rctx->dst_sg = rctx->dst_tbl.sgl; + + ret = qce_mapsg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst, + rctx->dst_chained); + if (ret < 0) + goto error_free; + + if (diff_dst) { + ret = qce_mapsg(qce->dev, req->src, rctx->src_nents, dir_src, + rctx->src_chained); + if (ret < 0) + goto error_unmap_dst; + rctx->src_sg = req->src; + } else { + rctx->src_sg = rctx->dst_sg; + } + + ret = qce_dma_prep_sgs(&qce->dma, rctx->src_sg, rctx->src_nents, + rctx->dst_sg, rctx->dst_nents, + qce_ablkcipher_done, async_req); + if (ret) + goto error_unmap_src; + + qce_dma_issue_pending(&qce->dma); + + ret = qce_start(async_req, tmpl->crypto_alg_type, req->nbytes, 0); + if (ret) + goto error_terminate; + + return 0; + +error_terminate: + qce_dma_terminate_all(&qce->dma); +error_unmap_src: + if (diff_dst) + qce_unmapsg(qce->dev, req->src, rctx->src_nents, dir_src, + rctx->src_chained); +error_unmap_dst: + qce_unmapsg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst, + rctx->dst_chained); +error_free: + sg_free_table(&rctx->dst_tbl); + return ret; +} + +static int qce_ablkcipher_setkey(struct crypto_ablkcipher *ablk, const u8 *key, + unsigned int keylen) +{ + struct crypto_tfm *tfm = crypto_ablkcipher_tfm(ablk); + struct qce_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + unsigned long flags = to_cipher_tmpl(tfm)->alg_flags; + int ret; + + if (!key || !keylen) + return -EINVAL; + + if (IS_AES(flags)) { + switch (keylen) { + case AES_KEYSIZE_128: + case AES_KEYSIZE_256: + break; + default: + goto fallback; + } + } else if (IS_DES(flags)) { + u32 tmp[DES_EXPKEY_WORDS]; + + ret = des_ekey(tmp, key); + if (!ret && crypto_ablkcipher_get_flags(ablk) & + CRYPTO_TFM_REQ_WEAK_KEY) + goto weakkey; + } + + ctx->enc_keylen = keylen; + memcpy(ctx->enc_key, key, keylen); + return 0; +fallback: + ret = crypto_ablkcipher_setkey(ctx->fallback, key, keylen); + if (!ret) + ctx->enc_keylen = keylen; + return ret; +weakkey: + crypto_ablkcipher_set_flags(ablk, CRYPTO_TFM_RES_WEAK_KEY); + return -EINVAL; +} + +static int qce_ablkcipher_crypt(struct ablkcipher_request *req, int encrypt) +{ + struct crypto_tfm *tfm = + crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req)); + struct qce_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + struct qce_cipher_reqctx *rctx = ablkcipher_request_ctx(req); + struct qce_alg_template *tmpl = to_cipher_tmpl(tfm); + int ret; + + rctx->flags = tmpl->alg_flags; + rctx->flags |= encrypt ? QCE_ENCRYPT : QCE_DECRYPT; + + if (IS_AES(rctx->flags) && ctx->enc_keylen != AES_KEYSIZE_128 && + ctx->enc_keylen != AES_KEYSIZE_256) { + ablkcipher_request_set_tfm(req, ctx->fallback); + ret = encrypt ? crypto_ablkcipher_encrypt(req) : + crypto_ablkcipher_decrypt(req); + ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm)); + return ret; + } + + return tmpl->qce->async_req_enqueue(tmpl->qce, &req->base); +} + +static int qce_ablkcipher_encrypt(struct ablkcipher_request *req) +{ + return qce_ablkcipher_crypt(req, 1); +} + +static int qce_ablkcipher_decrypt(struct ablkcipher_request *req) +{ + return qce_ablkcipher_crypt(req, 0); +} + +static int qce_ablkcipher_init(struct crypto_tfm *tfm) +{ + struct qce_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + memset(ctx, 0, sizeof(*ctx)); + tfm->crt_ablkcipher.reqsize = sizeof(struct qce_cipher_reqctx); + + ctx->fallback = crypto_alloc_ablkcipher(crypto_tfm_alg_name(tfm), + CRYPTO_ALG_TYPE_ABLKCIPHER, + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->fallback)) + return PTR_ERR(ctx->fallback); + + return 0; +} + +static void qce_ablkcipher_exit(struct crypto_tfm *tfm) +{ + struct qce_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + crypto_free_ablkcipher(ctx->fallback); +} + +struct qce_ablkcipher_def { + unsigned long flags; + const char *name; + const char *drv_name; + unsigned int blocksize; + unsigned int ivsize; + unsigned int min_keysize; + unsigned int max_keysize; +}; + +static const struct qce_ablkcipher_def ablkcipher_def[] = { + { + .flags = QCE_ALG_AES | QCE_MODE_ECB, + .name = "ecb(aes)", + .drv_name = "ecb-aes-qce", + .blocksize = AES_BLOCK_SIZE, + .ivsize = AES_BLOCK_SIZE, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + }, + { + .flags = QCE_ALG_AES | QCE_MODE_CBC, + .name = "cbc(aes)", + .drv_name = "cbc-aes-qce", + .blocksize = AES_BLOCK_SIZE, + .ivsize = AES_BLOCK_SIZE, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + }, + { + .flags = QCE_ALG_AES | QCE_MODE_CTR, + .name = "ctr(aes)", + .drv_name = "ctr-aes-qce", + .blocksize = AES_BLOCK_SIZE, + .ivsize = AES_BLOCK_SIZE, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + }, + { + .flags = QCE_ALG_AES | QCE_MODE_XTS, + .name = "xts(aes)", + .drv_name = "xts-aes-qce", + .blocksize = AES_BLOCK_SIZE, + .ivsize = AES_BLOCK_SIZE, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + }, + { + .flags = QCE_ALG_DES | QCE_MODE_ECB, + .name = "ecb(des)", + .drv_name = "ecb-des-qce", + .blocksize = DES_BLOCK_SIZE, + .ivsize = 0, + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + }, + { + .flags = QCE_ALG_DES | QCE_MODE_CBC, + .name = "cbc(des)", + .drv_name = "cbc-des-qce", + .blocksize = DES_BLOCK_SIZE, + .ivsize = DES_BLOCK_SIZE, + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + }, + { + .flags = QCE_ALG_3DES | QCE_MODE_ECB, + .name = "ecb(des3_ede)", + .drv_name = "ecb-3des-qce", + .blocksize = DES3_EDE_BLOCK_SIZE, + .ivsize = 0, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + }, + { + .flags = QCE_ALG_3DES | QCE_MODE_CBC, + .name = "cbc(des3_ede)", + .drv_name = "cbc-3des-qce", + .blocksize = DES3_EDE_BLOCK_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + }, +}; + +static int qce_ablkcipher_register_one(const struct qce_ablkcipher_def *def, + struct qce_device *qce) +{ + struct qce_alg_template *tmpl; + struct crypto_alg *alg; + int ret; + + tmpl = kzalloc(sizeof(*tmpl), GFP_KERNEL); + if (!tmpl) + return -ENOMEM; + + alg = &tmpl->alg.crypto; + + snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", def->name); + snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", + def->drv_name); + + alg->cra_blocksize = def->blocksize; + alg->cra_ablkcipher.ivsize = def->ivsize; + alg->cra_ablkcipher.min_keysize = def->min_keysize; + alg->cra_ablkcipher.max_keysize = def->max_keysize; + alg->cra_ablkcipher.setkey = qce_ablkcipher_setkey; + alg->cra_ablkcipher.encrypt = qce_ablkcipher_encrypt; + alg->cra_ablkcipher.decrypt = qce_ablkcipher_decrypt; + + alg->cra_priority = 300; + alg->cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK; + alg->cra_ctxsize = sizeof(struct qce_cipher_ctx); + alg->cra_alignmask = 0; + alg->cra_type = &crypto_ablkcipher_type; + alg->cra_module = THIS_MODULE; + alg->cra_init = qce_ablkcipher_init; + alg->cra_exit = qce_ablkcipher_exit; + INIT_LIST_HEAD(&alg->cra_list); + + INIT_LIST_HEAD(&tmpl->entry); + tmpl->crypto_alg_type = CRYPTO_ALG_TYPE_ABLKCIPHER; + tmpl->alg_flags = def->flags; + tmpl->qce = qce; + + ret = crypto_register_alg(alg); + if (ret) { + kfree(tmpl); + dev_err(qce->dev, "%s registration failed\n", alg->cra_name); + return ret; + } + + list_add_tail(&tmpl->entry, &ablkcipher_algs); + dev_dbg(qce->dev, "%s is registered\n", alg->cra_name); + return 0; +} + +static void qce_ablkcipher_unregister(struct qce_device *qce) +{ + struct qce_alg_template *tmpl, *n; + + list_for_each_entry_safe(tmpl, n, &ablkcipher_algs, entry) { + crypto_unregister_alg(&tmpl->alg.crypto); + list_del(&tmpl->entry); + kfree(tmpl); + } +} + +static int qce_ablkcipher_register(struct qce_device *qce) +{ + int ret, i; + + for (i = 0; i < ARRAY_SIZE(ablkcipher_def); i++) { + ret = qce_ablkcipher_register_one(&ablkcipher_def[i], qce); + if (ret) + goto err; + } + + return 0; +err: + qce_ablkcipher_unregister(qce); + return ret; +} + +const struct qce_algo_ops ablkcipher_ops = { + .type = CRYPTO_ALG_TYPE_ABLKCIPHER, + .register_algs = qce_ablkcipher_register, + .unregister_algs = qce_ablkcipher_unregister, + .async_req_handle = qce_ablkcipher_async_req_handle, +}; diff --git a/drivers/crypto/qce/cipher.h b/drivers/crypto/qce/cipher.h new file mode 100644 index 000000000000..d5757cfcda2d --- /dev/null +++ b/drivers/crypto/qce/cipher.h @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2010-2014, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef _CIPHER_H_ +#define _CIPHER_H_ + +#include "common.h" +#include "core.h" + +#define QCE_MAX_KEY_SIZE 64 + +struct qce_cipher_ctx { + u8 enc_key[QCE_MAX_KEY_SIZE]; + unsigned int enc_keylen; + struct crypto_ablkcipher *fallback; +}; + +/** + * struct qce_cipher_reqctx - holds private cipher objects per request + * @flags: operation flags + * @iv: pointer to the IV + * @ivsize: IV size + * @src_nents: source entries + * @dst_nents: destination entries + * @src_chained: is source chained + * @dst_chained: is destination chained + * @result_sg: scatterlist used for result buffer + * @dst_tbl: destination sg table + * @dst_sg: destination sg pointer table beginning + * @src_tbl: source sg table + * @src_sg: source sg pointer table beginning; + * @cryptlen: crypto length + */ +struct qce_cipher_reqctx { + unsigned long flags; + u8 *iv; + unsigned int ivsize; + int src_nents; + int dst_nents; + bool src_chained; + bool dst_chained; + struct scatterlist result_sg; + struct sg_table dst_tbl; + struct scatterlist *dst_sg; + struct sg_table src_tbl; + struct scatterlist *src_sg; + unsigned int cryptlen; +}; + +static inline struct qce_alg_template *to_cipher_tmpl(struct crypto_tfm *tfm) +{ + struct crypto_alg *alg = tfm->__crt_alg; + return container_of(alg, struct qce_alg_template, alg.crypto); +} + +extern const struct qce_algo_ops ablkcipher_ops; + +#endif /* _CIPHER_H_ */ diff --git a/drivers/crypto/qce/common.c b/drivers/crypto/qce/common.c new file mode 100644 index 000000000000..1fb5fde7fc03 --- /dev/null +++ b/drivers/crypto/qce/common.c @@ -0,0 +1,438 @@ +/* + * Copyright (c) 2012-2014, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include <linux/err.h> +#include <linux/interrupt.h> +#include <linux/types.h> +#include <crypto/scatterwalk.h> +#include <crypto/sha.h> + +#include "cipher.h" +#include "common.h" +#include "core.h" +#include "regs-v5.h" +#include "sha.h" + +#define QCE_SECTOR_SIZE 512 + +static inline u32 qce_read(struct qce_device *qce, u32 offset) +{ + return readl(qce->base + offset); +} + +static inline void qce_write(struct qce_device *qce, u32 offset, u32 val) +{ + writel(val, qce->base + offset); +} + +static inline void qce_write_array(struct qce_device *qce, u32 offset, + const u32 *val, unsigned int len) +{ + int i; + + for (i = 0; i < len; i++) + qce_write(qce, offset + i * sizeof(u32), val[i]); +} + +static inline void +qce_clear_array(struct qce_device *qce, u32 offset, unsigned int len) +{ + int i; + + for (i = 0; i < len; i++) + qce_write(qce, offset + i * sizeof(u32), 0); +} + +static u32 qce_encr_cfg(unsigned long flags, u32 aes_key_size) +{ + u32 cfg = 0; + + if (IS_AES(flags)) { + if (aes_key_size == AES_KEYSIZE_128) + cfg |= ENCR_KEY_SZ_AES128 << ENCR_KEY_SZ_SHIFT; + else if (aes_key_size == AES_KEYSIZE_256) + cfg |= ENCR_KEY_SZ_AES256 << ENCR_KEY_SZ_SHIFT; + } + + if (IS_AES(flags)) + cfg |= ENCR_ALG_AES << ENCR_ALG_SHIFT; + else if (IS_DES(flags) || IS_3DES(flags)) + cfg |= ENCR_ALG_DES << ENCR_ALG_SHIFT; + + if (IS_DES(flags)) + cfg |= ENCR_KEY_SZ_DES << ENCR_KEY_SZ_SHIFT; + + if (IS_3DES(flags)) + cfg |= ENCR_KEY_SZ_3DES << ENCR_KEY_SZ_SHIFT; + + switch (flags & QCE_MODE_MASK) { + case QCE_MODE_ECB: + cfg |= ENCR_MODE_ECB << ENCR_MODE_SHIFT; + break; + case QCE_MODE_CBC: + cfg |= ENCR_MODE_CBC << ENCR_MODE_SHIFT; + break; + case QCE_MODE_CTR: + cfg |= ENCR_MODE_CTR << ENCR_MODE_SHIFT; + break; + case QCE_MODE_XTS: + cfg |= ENCR_MODE_XTS << ENCR_MODE_SHIFT; + break; + case QCE_MODE_CCM: + cfg |= ENCR_MODE_CCM << ENCR_MODE_SHIFT; + cfg |= LAST_CCM_XFR << LAST_CCM_SHIFT; + break; + default: + return ~0; + } + + return cfg; +} + +static u32 qce_auth_cfg(unsigned long flags, u32 key_size) +{ + u32 cfg = 0; + + if (IS_AES(flags) && (IS_CCM(flags) || IS_CMAC(flags))) + cfg |= AUTH_ALG_AES << AUTH_ALG_SHIFT; + else + cfg |= AUTH_ALG_SHA << AUTH_ALG_SHIFT; + + if (IS_CCM(flags) || IS_CMAC(flags)) { + if (key_size == AES_KEYSIZE_128) + cfg |= AUTH_KEY_SZ_AES128 << AUTH_KEY_SIZE_SHIFT; + else if (key_size == AES_KEYSIZE_256) + cfg |= AUTH_KEY_SZ_AES256 << AUTH_KEY_SIZE_SHIFT; + } + + if (IS_SHA1(flags) || IS_SHA1_HMAC(flags)) + cfg |= AUTH_SIZE_SHA1 << AUTH_SIZE_SHIFT; + else if (IS_SHA256(flags) || IS_SHA256_HMAC(flags)) + cfg |= AUTH_SIZE_SHA256 << AUTH_SIZE_SHIFT; + else if (IS_CMAC(flags)) + cfg |= AUTH_SIZE_ENUM_16_BYTES << AUTH_SIZE_SHIFT; + + if (IS_SHA1(flags) || IS_SHA256(flags)) + cfg |= AUTH_MODE_HASH << AUTH_MODE_SHIFT; + else if (IS_SHA1_HMAC(flags) || IS_SHA256_HMAC(flags) || + IS_CBC(flags) || IS_CTR(flags)) + cfg |= AUTH_MODE_HMAC << AUTH_MODE_SHIFT; + else if (IS_AES(flags) && IS_CCM(flags)) + cfg |= AUTH_MODE_CCM << AUTH_MODE_SHIFT; + else if (IS_AES(flags) && IS_CMAC(flags)) + cfg |= AUTH_MODE_CMAC << AUTH_MODE_SHIFT; + + if (IS_SHA(flags) || IS_SHA_HMAC(flags)) + cfg |= AUTH_POS_BEFORE << AUTH_POS_SHIFT; + + if (IS_CCM(flags)) + cfg |= QCE_MAX_NONCE_WORDS << AUTH_NONCE_NUM_WORDS_SHIFT; + + if (IS_CBC(flags) || IS_CTR(flags) || IS_CCM(flags) || + IS_CMAC(flags)) + cfg |= BIT(AUTH_LAST_SHIFT) | BIT(AUTH_FIRST_SHIFT); + + return cfg; +} + +static u32 qce_config_reg(struct qce_device *qce, int little) +{ + u32 beats = (qce->burst_size >> 3) - 1; + u32 pipe_pair = qce->pipe_pair_id; + u32 config; + + config = (beats << REQ_SIZE_SHIFT) & REQ_SIZE_MASK; + config |= BIT(MASK_DOUT_INTR_SHIFT) | BIT(MASK_DIN_INTR_SHIFT) | + BIT(MASK_OP_DONE_INTR_SHIFT) | BIT(MASK_ERR_INTR_SHIFT); + config |= (pipe_pair << PIPE_SET_SELECT_SHIFT) & PIPE_SET_SELECT_MASK; + config &= ~HIGH_SPD_EN_N_SHIFT; + + if (little) + config |= BIT(LITTLE_ENDIAN_MODE_SHIFT); + + return config; +} + +void qce_cpu_to_be32p_array(__be32 *dst, const u8 *src, unsigned int len) +{ + __be32 *d = dst; + const u8 *s = src; + unsigned int n; + + n = len / sizeof(u32); + for (; n > 0; n--) { + *d = cpu_to_be32p((const __u32 *) s); + s += sizeof(__u32); + d++; + } +} + +static void qce_xts_swapiv(__be32 *dst, const u8 *src, unsigned int ivsize) +{ + u8 swap[QCE_AES_IV_LENGTH]; + u32 i, j; + + if (ivsize > QCE_AES_IV_LENGTH) + return; + + memset(swap, 0, QCE_AES_IV_LENGTH); + + for (i = (QCE_AES_IV_LENGTH - ivsize), j = ivsize - 1; + i < QCE_AES_IV_LENGTH; i++, j--) + swap[i] = src[j]; + + qce_cpu_to_be32p_array(dst, swap, QCE_AES_IV_LENGTH); +} + +static void qce_xtskey(struct qce_device *qce, const u8 *enckey, + unsigned int enckeylen, unsigned int cryptlen) +{ + u32 xtskey[QCE_MAX_CIPHER_KEY_SIZE / sizeof(u32)] = {0}; + unsigned int xtsklen = enckeylen / (2 * sizeof(u32)); + unsigned int xtsdusize; + + qce_cpu_to_be32p_array((__be32 *)xtskey, enckey + enckeylen / 2, + enckeylen / 2); + qce_write_array(qce, REG_ENCR_XTS_KEY0, xtskey, xtsklen); + + /* xts du size 512B */ + xtsdusize = min_t(u32, QCE_SECTOR_SIZE, cryptlen); + qce_write(qce, REG_ENCR_XTS_DU_SIZE, xtsdusize); +} + +static void qce_setup_config(struct qce_device *qce) +{ + u32 config; + + /* get big endianness */ + config = qce_config_reg(qce, 0); + + /* clear status */ + qce_write(qce, REG_STATUS, 0); + qce_write(qce, REG_CONFIG, config); +} + +static inline void qce_crypto_go(struct qce_device *qce) +{ + qce_write(qce, REG_GOPROC, BIT(GO_SHIFT) | BIT(RESULTS_DUMP_SHIFT)); +} + +static int qce_setup_regs_ahash(struct crypto_async_request *async_req, + u32 totallen, u32 offset) +{ + struct ahash_request *req = ahash_request_cast(async_req); + struct crypto_ahash *ahash = __crypto_ahash_cast(async_req->tfm); + struct qce_sha_reqctx *rctx = ahash_request_ctx(req); + struct qce_alg_template *tmpl = to_ahash_tmpl(async_req->tfm); + struct qce_device *qce = tmpl->qce; + unsigned int digestsize = crypto_ahash_digestsize(ahash); + unsigned int blocksize = crypto_tfm_alg_blocksize(async_req->tfm); + __be32 auth[SHA256_DIGEST_SIZE / sizeof(__be32)] = {0}; + __be32 mackey[QCE_SHA_HMAC_KEY_SIZE / sizeof(__be32)] = {0}; + u32 auth_cfg = 0, config; + unsigned int iv_words; + + /* if not the last, the size has to be on the block boundary */ + if (!rctx->last_blk && req->nbytes % blocksize) + return -EINVAL; + + qce_setup_config(qce); + + if (IS_CMAC(rctx->flags)) { + qce_write(qce, REG_AUTH_SEG_CFG, 0); + qce_write(qce, REG_ENCR_SEG_CFG, 0); + qce_write(qce, REG_ENCR_SEG_SIZE, 0); + qce_clear_array(qce, REG_AUTH_IV0, 16); + qce_clear_array(qce, REG_AUTH_KEY0, 16); + qce_clear_array(qce, REG_AUTH_BYTECNT0, 4); + + auth_cfg = qce_auth_cfg(rctx->flags, rctx->authklen); + } + + if (IS_SHA_HMAC(rctx->flags) || IS_CMAC(rctx->flags)) { + u32 authkey_words = rctx->authklen / sizeof(u32); + + qce_cpu_to_be32p_array(mackey, rctx->authkey, rctx->authklen); + qce_write_array(qce, REG_AUTH_KEY0, (u32 *)mackey, + authkey_words); + } + + if (IS_CMAC(rctx->flags)) + goto go_proc; + + if (rctx->first_blk) + memcpy(auth, rctx->digest, digestsize); + else + qce_cpu_to_be32p_array(auth, rctx->digest, digestsize); + + iv_words = (IS_SHA1(rctx->flags) || IS_SHA1_HMAC(rctx->flags)) ? 5 : 8; + qce_write_array(qce, REG_AUTH_IV0, (u32 *)auth, iv_words); + + if (rctx->first_blk) + qce_clear_array(qce, REG_AUTH_BYTECNT0, 4); + else + qce_write_array(qce, REG_AUTH_BYTECNT0, + (u32 *)rctx->byte_count, 2); + + auth_cfg = qce_auth_cfg(rctx->flags, 0); + + if (rctx->last_blk) + auth_cfg |= BIT(AUTH_LAST_SHIFT); + else + auth_cfg &= ~BIT(AUTH_LAST_SHIFT); + + if (rctx->first_blk) + auth_cfg |= BIT(AUTH_FIRST_SHIFT); + else + auth_cfg &= ~BIT(AUTH_FIRST_SHIFT); + +go_proc: + qce_write(qce, REG_AUTH_SEG_CFG, auth_cfg); + qce_write(qce, REG_AUTH_SEG_SIZE, req->nbytes); + qce_write(qce, REG_AUTH_SEG_START, 0); + qce_write(qce, REG_ENCR_SEG_CFG, 0); + qce_write(qce, REG_SEG_SIZE, req->nbytes); + + /* get little endianness */ + config = qce_config_reg(qce, 1); + qce_write(qce, REG_CONFIG, config); + + qce_crypto_go(qce); + + return 0; +} + +static int qce_setup_regs_ablkcipher(struct crypto_async_request *async_req, + u32 totallen, u32 offset) +{ + struct ablkcipher_request *req = ablkcipher_request_cast(async_req); + struct qce_cipher_reqctx *rctx = ablkcipher_request_ctx(req); + struct qce_cipher_ctx *ctx = crypto_tfm_ctx(async_req->tfm); + struct qce_alg_template *tmpl = to_cipher_tmpl(async_req->tfm); + struct qce_device *qce = tmpl->qce; + __be32 enckey[QCE_MAX_CIPHER_KEY_SIZE / sizeof(__be32)] = {0}; + __be32 enciv[QCE_MAX_IV_SIZE / sizeof(__be32)] = {0}; + unsigned int enckey_words, enciv_words; + unsigned int keylen; + u32 encr_cfg = 0, auth_cfg = 0, config; + unsigned int ivsize = rctx->ivsize; + unsigned long flags = rctx->flags; + + qce_setup_config(qce); + + if (IS_XTS(flags)) + keylen = ctx->enc_keylen / 2; + else + keylen = ctx->enc_keylen; + + qce_cpu_to_be32p_array(enckey, ctx->enc_key, keylen); + enckey_words = keylen / sizeof(u32); + + qce_write(qce, REG_AUTH_SEG_CFG, auth_cfg); + + encr_cfg = qce_encr_cfg(flags, keylen); + + if (IS_DES(flags)) { + enciv_words = 2; + enckey_words = 2; + } else if (IS_3DES(flags)) { + enciv_words = 2; + enckey_words = 6; + } else if (IS_AES(flags)) { + if (IS_XTS(flags)) + qce_xtskey(qce, ctx->enc_key, ctx->enc_keylen, + rctx->cryptlen); + enciv_words = 4; + } else { + return -EINVAL; + } + + qce_write_array(qce, REG_ENCR_KEY0, (u32 *)enckey, enckey_words); + + if (!IS_ECB(flags)) { + if (IS_XTS(flags)) + qce_xts_swapiv(enciv, rctx->iv, ivsize); + else + qce_cpu_to_be32p_array(enciv, rctx->iv, ivsize); + + qce_write_array(qce, REG_CNTR0_IV0, (u32 *)enciv, enciv_words); + } + + if (IS_ENCRYPT(flags)) + encr_cfg |= BIT(ENCODE_SHIFT); + + qce_write(qce, REG_ENCR_SEG_CFG, encr_cfg); + qce_write(qce, REG_ENCR_SEG_SIZE, rctx->cryptlen); + qce_write(qce, REG_ENCR_SEG_START, offset & 0xffff); + + if (IS_CTR(flags)) { + qce_write(qce, REG_CNTR_MASK, ~0); + qce_write(qce, REG_CNTR_MASK0, ~0); + qce_write(qce, REG_CNTR_MASK1, ~0); + qce_write(qce, REG_CNTR_MASK2, ~0); + } + + qce_write(qce, REG_SEG_SIZE, totallen); + + /* get little endianness */ + config = qce_config_reg(qce, 1); + qce_write(qce, REG_CONFIG, config); + + qce_crypto_go(qce); + + return 0; +} + +int qce_start(struct crypto_async_request *async_req, u32 type, u32 totallen, + u32 offset) +{ + switch (type) { + case CRYPTO_ALG_TYPE_ABLKCIPHER: + return qce_setup_regs_ablkcipher(async_req, totallen, offset); + case CRYPTO_ALG_TYPE_AHASH: + return qce_setup_regs_ahash(async_req, totallen, offset); + default: + return -EINVAL; + } +} + +#define STATUS_ERRORS \ + (BIT(SW_ERR_SHIFT) | BIT(AXI_ERR_SHIFT) | BIT(HSD_ERR_SHIFT)) + +int qce_check_status(struct qce_device *qce, u32 *status) +{ + int ret = 0; + + *status = qce_read(qce, REG_STATUS); + + /* + * Don't use result dump status. The operation may not be complete. + * Instead, use the status we just read from device. In case, we need to + * use result_status from result dump the result_status needs to be byte + * swapped, since we set the device to little endian. + */ + if (*status & STATUS_ERRORS || !(*status & BIT(OPERATION_DONE_SHIFT))) + ret = -ENXIO; + + return ret; +} + +void qce_get_version(struct qce_device *qce, u32 *major, u32 *minor, u32 *step) +{ + u32 val; + + val = qce_read(qce, REG_VERSION); + *major = (val & CORE_MAJOR_REV_MASK) >> CORE_MAJOR_REV_SHIFT; + *minor = (val & CORE_MINOR_REV_MASK) >> CORE_MINOR_REV_SHIFT; + *step = (val & CORE_STEP_REV_MASK) >> CORE_STEP_REV_SHIFT; +} diff --git a/drivers/crypto/qce/common.h b/drivers/crypto/qce/common.h new file mode 100644 index 000000000000..a4addd4f7d6c --- /dev/null +++ b/drivers/crypto/qce/common.h @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2010-2014, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef _COMMON_H_ +#define _COMMON_H_ + +#include <linux/crypto.h> +#include <linux/types.h> +#include <crypto/aes.h> +#include <crypto/hash.h> + +/* key size in bytes */ +#define QCE_SHA_HMAC_KEY_SIZE 64 +#define QCE_MAX_CIPHER_KEY_SIZE AES_KEYSIZE_256 + +/* IV length in bytes */ +#define QCE_AES_IV_LENGTH AES_BLOCK_SIZE +/* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */ +#define QCE_MAX_IV_SIZE AES_BLOCK_SIZE + +/* maximum nonce bytes */ +#define QCE_MAX_NONCE 16 +#define QCE_MAX_NONCE_WORDS (QCE_MAX_NONCE / sizeof(u32)) + +/* burst size alignment requirement */ +#define QCE_MAX_ALIGN_SIZE 64 + +/* cipher algorithms */ +#define QCE_ALG_DES BIT(0) +#define QCE_ALG_3DES BIT(1) +#define QCE_ALG_AES BIT(2) + +/* hash and hmac algorithms */ +#define QCE_HASH_SHA1 BIT(3) +#define QCE_HASH_SHA256 BIT(4) +#define QCE_HASH_SHA1_HMAC BIT(5) +#define QCE_HASH_SHA256_HMAC BIT(6) +#define QCE_HASH_AES_CMAC BIT(7) + +/* cipher modes */ +#define QCE_MODE_CBC BIT(8) +#define QCE_MODE_ECB BIT(9) +#define QCE_MODE_CTR BIT(10) +#define QCE_MODE_XTS BIT(11) +#define QCE_MODE_CCM BIT(12) +#define QCE_MODE_MASK GENMASK(12, 8) + +/* cipher encryption/decryption operations */ +#define QCE_ENCRYPT BIT(13) +#define QCE_DECRYPT BIT(14) + +#define IS_DES(flags) (flags & QCE_ALG_DES) +#define IS_3DES(flags) (flags & QCE_ALG_3DES) +#define IS_AES(flags) (flags & QCE_ALG_AES) + +#define IS_SHA1(flags) (flags & QCE_HASH_SHA1) +#define IS_SHA256(flags) (flags & QCE_HASH_SHA256) +#define IS_SHA1_HMAC(flags) (flags & QCE_HASH_SHA1_HMAC) +#define IS_SHA256_HMAC(flags) (flags & QCE_HASH_SHA256_HMAC) +#define IS_CMAC(flags) (flags & QCE_HASH_AES_CMAC) +#define IS_SHA(flags) (IS_SHA1(flags) || IS_SHA256(flags)) +#define IS_SHA_HMAC(flags) \ + (IS_SHA1_HMAC(flags) || IS_SHA256_HMAC(flags)) + +#define IS_CBC(mode) (mode & QCE_MODE_CBC) +#define IS_ECB(mode) (mode & QCE_MODE_ECB) +#define IS_CTR(mode) (mode & QCE_MODE_CTR) +#define IS_XTS(mode) (mode & QCE_MODE_XTS) +#define IS_CCM(mode) (mode & QCE_MODE_CCM) + +#define IS_ENCRYPT(dir) (dir & QCE_ENCRYPT) +#define IS_DECRYPT(dir) (dir & QCE_DECRYPT) + +struct qce_alg_template { + struct list_head entry; + u32 crypto_alg_type; + unsigned long alg_flags; + const u32 *std_iv; + union { + struct crypto_alg crypto; + struct ahash_alg ahash; + } alg; + struct qce_device *qce; +}; + +void qce_cpu_to_be32p_array(__be32 *dst, const u8 *src, unsigned int len); +int qce_check_status(struct qce_device *qce, u32 *status); +void qce_get_version(struct qce_device *qce, u32 *major, u32 *minor, u32 *step); +int qce_start(struct crypto_async_request *async_req, u32 type, u32 totallen, + u32 offset); + +#endif /* _COMMON_H_ */ diff --git a/drivers/crypto/qce/core.c b/drivers/crypto/qce/core.c new file mode 100644 index 000000000000..33ae3545dc48 --- /dev/null +++ b/drivers/crypto/qce/core.c @@ -0,0 +1,286 @@ +/* + * Copyright (c) 2010-2014, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include <linux/clk.h> +#include <linux/interrupt.h> +#include <linux/module.h> +#include <linux/platform_device.h> +#include <linux/spinlock.h> +#include <linux/types.h> +#include <crypto/algapi.h> +#include <crypto/internal/hash.h> +#include <crypto/sha.h> + +#include "core.h" +#include "cipher.h" +#include "sha.h" + +#define QCE_MAJOR_VERSION5 0x05 +#define QCE_QUEUE_LENGTH 1 + +static const struct qce_algo_ops *qce_ops[] = { + &ablkcipher_ops, + &ahash_ops, +}; + +static void qce_unregister_algs(struct qce_device *qce) +{ + const struct qce_algo_ops *ops; + int i; + + for (i = 0; i < ARRAY_SIZE(qce_ops); i++) { + ops = qce_ops[i]; + ops->unregister_algs(qce); + } +} + +static int qce_register_algs(struct qce_device *qce) +{ + const struct qce_algo_ops *ops; + int i, ret = -ENODEV; + + for (i = 0; i < ARRAY_SIZE(qce_ops); i++) { + ops = qce_ops[i]; + ret = ops->register_algs(qce); + if (ret) + break; + } + + return ret; +} + +static int qce_handle_request(struct crypto_async_request *async_req) +{ + int ret = -EINVAL, i; + const struct qce_algo_ops *ops; + u32 type = crypto_tfm_alg_type(async_req->tfm); + + for (i = 0; i < ARRAY_SIZE(qce_ops); i++) { + ops = qce_ops[i]; + if (type != ops->type) + continue; + ret = ops->async_req_handle(async_req); + break; + } + + return ret; +} + +static int qce_handle_queue(struct qce_device *qce, + struct crypto_async_request *req) +{ + struct crypto_async_request *async_req, *backlog; + unsigned long flags; + int ret = 0, err; + + spin_lock_irqsave(&qce->lock, flags); + + if (req) + ret = crypto_enqueue_request(&qce->queue, req); + + /* busy, do not dequeue request */ + if (qce->req) { + spin_unlock_irqrestore(&qce->lock, flags); + return ret; + } + + backlog = crypto_get_backlog(&qce->queue); + async_req = crypto_dequeue_request(&qce->queue); + if (async_req) + qce->req = async_req; + + spin_unlock_irqrestore(&qce->lock, flags); + + if (!async_req) + return ret; + + if (backlog) { + spin_lock_bh(&qce->lock); + backlog->complete(backlog, -EINPROGRESS); + spin_unlock_bh(&qce->lock); + } + + err = qce_handle_request(async_req); + if (err) { + qce->result = err; + tasklet_schedule(&qce->done_tasklet); + } + + return ret; +} + +static void qce_tasklet_req_done(unsigned long data) +{ + struct qce_device *qce = (struct qce_device *)data; + struct crypto_async_request *req; + unsigned long flags; + + spin_lock_irqsave(&qce->lock, flags); + req = qce->req; + qce->req = NULL; + spin_unlock_irqrestore(&qce->lock, flags); + + if (req) + req->complete(req, qce->result); + + qce_handle_queue(qce, NULL); +} + +static int qce_async_request_enqueue(struct qce_device *qce, + struct crypto_async_request *req) +{ + return qce_handle_queue(qce, req); +} + +static void qce_async_request_done(struct qce_device *qce, int ret) +{ + qce->result = ret; + tasklet_schedule(&qce->done_tasklet); +} + +static int qce_check_version(struct qce_device *qce) +{ + u32 major, minor, step; + + qce_get_version(qce, &major, &minor, &step); + + /* + * the driver does not support v5 with minor 0 because it has special + * alignment requirements. + */ + if (major != QCE_MAJOR_VERSION5 || minor == 0) + return -ENODEV; + + qce->burst_size = QCE_BAM_BURST_SIZE; + qce->pipe_pair_id = 1; + + dev_dbg(qce->dev, "Crypto device found, version %d.%d.%d\n", + major, minor, step); + + return 0; +} + +static int qce_crypto_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct qce_device *qce; + struct resource *res; + int ret; + + qce = devm_kzalloc(dev, sizeof(*qce), GFP_KERNEL); + if (!qce) + return -ENOMEM; + + qce->dev = dev; + platform_set_drvdata(pdev, qce); + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + qce->base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(qce->base)) + return PTR_ERR(qce->base); + + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); + if (ret < 0) + return ret; + + qce->core = devm_clk_get(qce->dev, "core"); + if (IS_ERR(qce->core)) + return PTR_ERR(qce->core); + + qce->iface = devm_clk_get(qce->dev, "iface"); + if (IS_ERR(qce->iface)) + return PTR_ERR(qce->iface); + + qce->bus = devm_clk_get(qce->dev, "bus"); + if (IS_ERR(qce->bus)) + return PTR_ERR(qce->bus); + + ret = clk_prepare_enable(qce->core); + if (ret) + return ret; + + ret = clk_prepare_enable(qce->iface); + if (ret) + goto err_clks_core; + + ret = clk_prepare_enable(qce->bus); + if (ret) + goto err_clks_iface; + + ret = qce_dma_request(qce->dev, &qce->dma); + if (ret) + goto err_clks; + + ret = qce_check_version(qce); + if (ret) + goto err_clks; + + spin_lock_init(&qce->lock); + tasklet_init(&qce->done_tasklet, qce_tasklet_req_done, + (unsigned long)qce); + crypto_init_queue(&qce->queue, QCE_QUEUE_LENGTH); + + qce->async_req_enqueue = qce_async_request_enqueue; + qce->async_req_done = qce_async_request_done; + + ret = qce_register_algs(qce); + if (ret) + goto err_dma; + + return 0; + +err_dma: + qce_dma_release(&qce->dma); +err_clks: + clk_disable_unprepare(qce->bus); +err_clks_iface: + clk_disable_unprepare(qce->iface); +err_clks_core: + clk_disable_unprepare(qce->core); + return ret; +} + +static int qce_crypto_remove(struct platform_device *pdev) +{ + struct qce_device *qce = platform_get_drvdata(pdev); + + tasklet_kill(&qce->done_tasklet); + qce_unregister_algs(qce); + qce_dma_release(&qce->dma); + clk_disable_unprepare(qce->bus); + clk_disable_unprepare(qce->iface); + clk_disable_unprepare(qce->core); + return 0; +} + +static const struct of_device_id qce_crypto_of_match[] = { + { .compatible = "qcom,crypto-v5.1", }, + {} +}; +MODULE_DEVICE_TABLE(of, qce_crypto_of_match); + +static struct platform_driver qce_crypto_driver = { + .probe = qce_crypto_probe, + .remove = qce_crypto_remove, + .driver = { + .owner = THIS_MODULE, + .name = KBUILD_MODNAME, + .of_match_table = qce_crypto_of_match, + }, +}; +module_platform_driver(qce_crypto_driver); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("Qualcomm crypto engine driver"); +MODULE_ALIAS("platform:" KBUILD_MODNAME); +MODULE_AUTHOR("The Linux Foundation"); diff --git a/drivers/crypto/qce/core.h b/drivers/crypto/qce/core.h new file mode 100644 index 000000000000..549965d4d91f --- /dev/null +++ b/drivers/crypto/qce/core.h @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2010-2014, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef _CORE_H_ +#define _CORE_H_ + +#include "dma.h" + +/** + * struct qce_device - crypto engine device structure + * @queue: crypto request queue + * @lock: the lock protects queue and req + * @done_tasklet: done tasklet object + * @req: current active request + * @result: result of current transform + * @base: virtual IO base + * @dev: pointer to device structure + * @core: core device clock + * @iface: interface clock + * @bus: bus clock + * @dma: pointer to dma data + * @burst_size: the crypto burst size + * @pipe_pair_id: which pipe pair id the device using + * @async_req_enqueue: invoked by every algorithm to enqueue a request + * @async_req_done: invoked by every algorithm to finish its request + */ +struct qce_device { + struct crypto_queue queue; + spinlock_t lock; + struct tasklet_struct done_tasklet; + struct crypto_async_request *req; + int result; + void __iomem *base; + struct device *dev; + struct clk *core, *iface, *bus; + struct qce_dma_data dma; + int burst_size; + unsigned int pipe_pair_id; + int (*async_req_enqueue)(struct qce_device *qce, + struct crypto_async_request *req); + void (*async_req_done)(struct qce_device *qce, int ret); +}; + +/** + * struct qce_algo_ops - algorithm operations per crypto type + * @type: should be CRYPTO_ALG_TYPE_XXX + * @register_algs: invoked by core to register the algorithms + * @unregister_algs: invoked by core to unregister the algorithms + * @async_req_handle: invoked by core to handle enqueued request + */ +struct qce_algo_ops { + u32 type; + int (*register_algs)(struct qce_device *qce); + void (*unregister_algs)(struct qce_device *qce); + int (*async_req_handle)(struct crypto_async_request *async_req); +}; + +#endif /* _CORE_H_ */ diff --git a/drivers/crypto/qce/dma.c b/drivers/crypto/qce/dma.c new file mode 100644 index 000000000000..0fb21e13f247 --- /dev/null +++ b/drivers/crypto/qce/dma.c @@ -0,0 +1,186 @@ +/* + * Copyright (c) 2012-2014, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include <linux/dmaengine.h> +#include <crypto/scatterwalk.h> + +#include "dma.h" + +int qce_dma_request(struct device *dev, struct qce_dma_data *dma) +{ + int ret; + + dma->txchan = dma_request_slave_channel_reason(dev, "tx"); + if (IS_ERR(dma->txchan)) + return PTR_ERR(dma->txchan); + + dma->rxchan = dma_request_slave_channel_reason(dev, "rx"); + if (IS_ERR(dma->rxchan)) { + ret = PTR_ERR(dma->rxchan); + goto error_rx; + } + + dma->result_buf = kmalloc(QCE_RESULT_BUF_SZ + QCE_IGNORE_BUF_SZ, + GFP_KERNEL); + if (!dma->result_buf) { + ret = -ENOMEM; + goto error_nomem; + } + + dma->ignore_buf = dma->result_buf + QCE_RESULT_BUF_SZ; + + return 0; +error_nomem: + dma_release_channel(dma->rxchan); +error_rx: + dma_release_channel(dma->txchan); + return ret; +} + +void qce_dma_release(struct qce_dma_data *dma) +{ + dma_release_channel(dma->txchan); + dma_release_channel(dma->rxchan); + kfree(dma->result_buf); +} + +int qce_mapsg(struct device *dev, struct scatterlist *sg, int nents, + enum dma_data_direction dir, bool chained) +{ + int err; + + if (chained) { + while (sg) { + err = dma_map_sg(dev, sg, 1, dir); + if (!err) + return -EFAULT; + sg = scatterwalk_sg_next(sg); + } + } else { + err = dma_map_sg(dev, sg, nents, dir); + if (!err) + return -EFAULT; + } + + return nents; +} + +void qce_unmapsg(struct device *dev, struct scatterlist *sg, int nents, + enum dma_data_direction dir, bool chained) +{ + if (chained) + while (sg) { + dma_unmap_sg(dev, sg, 1, dir); + sg = scatterwalk_sg_next(sg); + } + else + dma_unmap_sg(dev, sg, nents, dir); +} + +int qce_countsg(struct scatterlist *sglist, int nbytes, bool *chained) +{ + struct scatterlist *sg = sglist; + int nents = 0; + + if (chained) + *chained = false; + + while (nbytes > 0 && sg) { + nents++; + nbytes -= sg->length; + if (!sg_is_last(sg) && (sg + 1)->length == 0 && chained) + *chained = true; + sg = scatterwalk_sg_next(sg); + } + + return nents; +} + +struct scatterlist * +qce_sgtable_add(struct sg_table *sgt, struct scatterlist *new_sgl) +{ + struct scatterlist *sg = sgt->sgl, *sg_last = NULL; + + while (sg) { + if (!sg_page(sg)) + break; + sg = sg_next(sg); + } + + if (!sg) + return ERR_PTR(-EINVAL); + + while (new_sgl && sg) { + sg_set_page(sg, sg_page(new_sgl), new_sgl->length, + new_sgl->offset); + sg_last = sg; + sg = sg_next(sg); + new_sgl = sg_next(new_sgl); + } + + return sg_last; +} + +static int qce_dma_prep_sg(struct dma_chan *chan, struct scatterlist *sg, + int nents, unsigned long flags, + enum dma_transfer_direction dir, + dma_async_tx_callback cb, void *cb_param) +{ + struct dma_async_tx_descriptor *desc; + dma_cookie_t cookie; + + if (!sg || !nents) + return -EINVAL; + + desc = dmaengine_prep_slave_sg(chan, sg, nents, dir, flags); + if (!desc) + return -EINVAL; + + desc->callback = cb; + desc->callback_param = cb_param; + cookie = dmaengine_submit(desc); + + return dma_submit_error(cookie); +} + +int qce_dma_prep_sgs(struct qce_dma_data *dma, struct scatterlist *rx_sg, + int rx_nents, struct scatterlist *tx_sg, int tx_nents, + dma_async_tx_callback cb, void *cb_param) +{ + struct dma_chan *rxchan = dma->rxchan; + struct dma_chan *txchan = dma->txchan; + unsigned long flags = DMA_PREP_INTERRUPT | DMA_CTRL_ACK; + int ret; + + ret = qce_dma_prep_sg(rxchan, rx_sg, rx_nents, flags, DMA_MEM_TO_DEV, + NULL, NULL); + if (ret) + return ret; + + return qce_dma_prep_sg(txchan, tx_sg, tx_nents, flags, DMA_DEV_TO_MEM, + cb, cb_param); +} + +void qce_dma_issue_pending(struct qce_dma_data *dma) +{ + dma_async_issue_pending(dma->rxchan); + dma_async_issue_pending(dma->txchan); +} + +int qce_dma_terminate_all(struct qce_dma_data *dma) +{ + int ret; + + ret = dmaengine_terminate_all(dma->rxchan); + return ret ?: dmaengine_terminate_all(dma->txchan); +} diff --git a/drivers/crypto/qce/dma.h b/drivers/crypto/qce/dma.h new file mode 100644 index 000000000000..805e378d59e9 --- /dev/null +++ b/drivers/crypto/qce/dma.h @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2011-2014, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef _DMA_H_ +#define _DMA_H_ + +/* maximum data transfer block size between BAM and CE */ +#define QCE_BAM_BURST_SIZE 64 + +#define QCE_AUTHIV_REGS_CNT 16 +#define QCE_AUTH_BYTECOUNT_REGS_CNT 4 +#define QCE_CNTRIV_REGS_CNT 4 + +struct qce_result_dump { + u32 auth_iv[QCE_AUTHIV_REGS_CNT]; + u32 auth_byte_count[QCE_AUTH_BYTECOUNT_REGS_CNT]; + u32 encr_cntr_iv[QCE_CNTRIV_REGS_CNT]; + u32 status; + u32 status2; +}; + +#define QCE_IGNORE_BUF_SZ (2 * QCE_BAM_BURST_SIZE) +#define QCE_RESULT_BUF_SZ \ + ALIGN(sizeof(struct qce_result_dump), QCE_BAM_BURST_SIZE) + +struct qce_dma_data { + struct dma_chan *txchan; + struct dma_chan *rxchan; + struct qce_result_dump *result_buf; + void *ignore_buf; +}; + +int qce_dma_request(struct device *dev, struct qce_dma_data *dma); +void qce_dma_release(struct qce_dma_data *dma); +int qce_dma_prep_sgs(struct qce_dma_data *dma, struct scatterlist *sg_in, + int in_ents, struct scatterlist *sg_out, int out_ents, + dma_async_tx_callback cb, void *cb_param); +void qce_dma_issue_pending(struct qce_dma_data *dma); +int qce_dma_terminate_all(struct qce_dma_data *dma); +int qce_countsg(struct scatterlist *sg_list, int nbytes, bool *chained); +void qce_unmapsg(struct device *dev, struct scatterlist *sg, int nents, + enum dma_data_direction dir, bool chained); +int qce_mapsg(struct device *dev, struct scatterlist *sg, int nents, + enum dma_data_direction dir, bool chained); +struct scatterlist * +qce_sgtable_add(struct sg_table *sgt, struct scatterlist *sg_add); + +#endif /* _DMA_H_ */ diff --git a/drivers/crypto/qce/regs-v5.h b/drivers/crypto/qce/regs-v5.h new file mode 100644 index 000000000000..f0e19e35664a --- /dev/null +++ b/drivers/crypto/qce/regs-v5.h @@ -0,0 +1,334 @@ +/* + * Copyright (c) 2012-2014, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef _REGS_V5_H_ +#define _REGS_V5_H_ + +#include <linux/bitops.h> + +#define REG_VERSION 0x000 +#define REG_STATUS 0x100 +#define REG_STATUS2 0x104 +#define REG_ENGINES_AVAIL 0x108 +#define REG_FIFO_SIZES 0x10c +#define REG_SEG_SIZE 0x110 +#define REG_GOPROC 0x120 +#define REG_ENCR_SEG_CFG 0x200 +#define REG_ENCR_SEG_SIZE 0x204 +#define REG_ENCR_SEG_START 0x208 +#define REG_CNTR0_IV0 0x20c +#define REG_CNTR1_IV1 0x210 +#define REG_CNTR2_IV2 0x214 +#define REG_CNTR3_IV3 0x218 +#define REG_CNTR_MASK 0x21C +#define REG_ENCR_CCM_INT_CNTR0 0x220 +#define REG_ENCR_CCM_INT_CNTR1 0x224 +#define REG_ENCR_CCM_INT_CNTR2 0x228 +#define REG_ENCR_CCM_INT_CNTR3 0x22c +#define REG_ENCR_XTS_DU_SIZE 0x230 +#define REG_CNTR_MASK2 0x234 +#define REG_CNTR_MASK1 0x238 +#define REG_CNTR_MASK0 0x23c +#define REG_AUTH_SEG_CFG 0x300 +#define REG_AUTH_SEG_SIZE 0x304 +#define REG_AUTH_SEG_START 0x308 +#define REG_AUTH_IV0 0x310 +#define REG_AUTH_IV1 0x314 +#define REG_AUTH_IV2 0x318 +#define REG_AUTH_IV3 0x31c +#define REG_AUTH_IV4 0x320 +#define REG_AUTH_IV5 0x324 +#define REG_AUTH_IV6 0x328 +#define REG_AUTH_IV7 0x32c +#define REG_AUTH_IV8 0x330 +#define REG_AUTH_IV9 0x334 +#define REG_AUTH_IV10 0x338 +#define REG_AUTH_IV11 0x33c +#define REG_AUTH_IV12 0x340 +#define REG_AUTH_IV13 0x344 +#define REG_AUTH_IV14 0x348 +#define REG_AUTH_IV15 0x34c +#define REG_AUTH_INFO_NONCE0 0x350 +#define REG_AUTH_INFO_NONCE1 0x354 +#define REG_AUTH_INFO_NONCE2 0x358 +#define REG_AUTH_INFO_NONCE3 0x35c +#define REG_AUTH_BYTECNT0 0x390 +#define REG_AUTH_BYTECNT1 0x394 +#define REG_AUTH_BYTECNT2 0x398 +#define REG_AUTH_BYTECNT3 0x39c +#define REG_AUTH_EXP_MAC0 0x3a0 +#define REG_AUTH_EXP_MAC1 0x3a4 +#define REG_AUTH_EXP_MAC2 0x3a8 +#define REG_AUTH_EXP_MAC3 0x3ac +#define REG_AUTH_EXP_MAC4 0x3b0 +#define REG_AUTH_EXP_MAC5 0x3b4 +#define REG_AUTH_EXP_MAC6 0x3b8 +#define REG_AUTH_EXP_MAC7 0x3bc +#define REG_CONFIG 0x400 +#define REG_GOPROC_QC_KEY 0x1000 +#define REG_GOPROC_OEM_KEY 0x2000 +#define REG_ENCR_KEY0 0x3000 +#define REG_ENCR_KEY1 0x3004 +#define REG_ENCR_KEY2 0x3008 +#define REG_ENCR_KEY3 0x300c +#define REG_ENCR_KEY4 0x3010 +#define REG_ENCR_KEY5 0x3014 +#define REG_ENCR_KEY6 0x3018 +#define REG_ENCR_KEY7 0x301c +#define REG_ENCR_XTS_KEY0 0x3020 +#define REG_ENCR_XTS_KEY1 0x3024 +#define REG_ENCR_XTS_KEY2 0x3028 +#define REG_ENCR_XTS_KEY3 0x302c +#define REG_ENCR_XTS_KEY4 0x3030 +#define REG_ENCR_XTS_KEY5 0x3034 +#define REG_ENCR_XTS_KEY6 0x3038 +#define REG_ENCR_XTS_KEY7 0x303c +#define REG_AUTH_KEY0 0x3040 +#define REG_AUTH_KEY1 0x3044 +#define REG_AUTH_KEY2 0x3048 +#define REG_AUTH_KEY3 0x304c +#define REG_AUTH_KEY4 0x3050 +#define REG_AUTH_KEY5 0x3054 +#define REG_AUTH_KEY6 0x3058 +#define REG_AUTH_KEY7 0x305c +#define REG_AUTH_KEY8 0x3060 +#define REG_AUTH_KEY9 0x3064 +#define REG_AUTH_KEY10 0x3068 +#define REG_AUTH_KEY11 0x306c +#define REG_AUTH_KEY12 0x3070 +#define REG_AUTH_KEY13 0x3074 +#define REG_AUTH_KEY14 0x3078 +#define REG_AUTH_KEY15 0x307c + +/* Register bits - REG_VERSION */ +#define CORE_STEP_REV_SHIFT 0 +#define CORE_STEP_REV_MASK GENMASK(15, 0) +#define CORE_MINOR_REV_SHIFT 16 +#define CORE_MINOR_REV_MASK GENMASK(23, 16) +#define CORE_MAJOR_REV_SHIFT 24 +#define CORE_MAJOR_REV_MASK GENMASK(31, 24) + +/* Register bits - REG_STATUS */ +#define MAC_FAILED_SHIFT 31 +#define DOUT_SIZE_AVAIL_SHIFT 26 +#define DOUT_SIZE_AVAIL_MASK GENMASK(30, 26) +#define DIN_SIZE_AVAIL_SHIFT 21 +#define DIN_SIZE_AVAIL_MASK GENMASK(25, 21) +#define HSD_ERR_SHIFT 20 +#define ACCESS_VIOL_SHIFT 19 +#define PIPE_ACTIVE_ERR_SHIFT 18 +#define CFG_CHNG_ERR_SHIFT 17 +#define DOUT_ERR_SHIFT 16 +#define DIN_ERR_SHIFT 15 +#define AXI_ERR_SHIFT 14 +#define CRYPTO_STATE_SHIFT 10 +#define CRYPTO_STATE_MASK GENMASK(13, 10) +#define ENCR_BUSY_SHIFT 9 +#define AUTH_BUSY_SHIFT 8 +#define DOUT_INTR_SHIFT 7 +#define DIN_INTR_SHIFT 6 +#define OP_DONE_INTR_SHIFT 5 +#define ERR_INTR_SHIFT 4 +#define DOUT_RDY_SHIFT 3 +#define DIN_RDY_SHIFT 2 +#define OPERATION_DONE_SHIFT 1 +#define SW_ERR_SHIFT 0 + +/* Register bits - REG_STATUS2 */ +#define AXI_EXTRA_SHIFT 1 +#define LOCKED_SHIFT 2 + +/* Register bits - REG_CONFIG */ +#define REQ_SIZE_SHIFT 17 +#define REQ_SIZE_MASK GENMASK(20, 17) +#define REQ_SIZE_ENUM_1_BEAT 0 +#define REQ_SIZE_ENUM_2_BEAT 1 +#define REQ_SIZE_ENUM_3_BEAT 2 +#define REQ_SIZE_ENUM_4_BEAT 3 +#define REQ_SIZE_ENUM_5_BEAT 4 +#define REQ_SIZE_ENUM_6_BEAT 5 +#define REQ_SIZE_ENUM_7_BEAT 6 +#define REQ_SIZE_ENUM_8_BEAT 7 +#define REQ_SIZE_ENUM_9_BEAT 8 +#define REQ_SIZE_ENUM_10_BEAT 9 +#define REQ_SIZE_ENUM_11_BEAT 10 +#define REQ_SIZE_ENUM_12_BEAT 11 +#define REQ_SIZE_ENUM_13_BEAT 12 +#define REQ_SIZE_ENUM_14_BEAT 13 +#define REQ_SIZE_ENUM_15_BEAT 14 +#define REQ_SIZE_ENUM_16_BEAT 15 + +#define MAX_QUEUED_REQ_SHIFT 14 +#define MAX_QUEUED_REQ_MASK GENMASK(24, 16) +#define ENUM_1_QUEUED_REQS 0 +#define ENUM_2_QUEUED_REQS 1 +#define ENUM_3_QUEUED_REQS 2 + +#define IRQ_ENABLES_SHIFT 10 +#define IRQ_ENABLES_MASK GENMASK(13, 10) + +#define LITTLE_ENDIAN_MODE_SHIFT 9 +#define PIPE_SET_SELECT_SHIFT 5 +#define PIPE_SET_SELECT_MASK GENMASK(8, 5) + +#define HIGH_SPD_EN_N_SHIFT 4 +#define MASK_DOUT_INTR_SHIFT 3 +#define MASK_DIN_INTR_SHIFT 2 +#define MASK_OP_DONE_INTR_SHIFT 1 +#define MASK_ERR_INTR_SHIFT 0 + +/* Register bits - REG_AUTH_SEG_CFG */ +#define COMP_EXP_MAC_SHIFT 24 +#define COMP_EXP_MAC_DISABLED 0 +#define COMP_EXP_MAC_ENABLED 1 + +#define F9_DIRECTION_SHIFT 23 +#define F9_DIRECTION_UPLINK 0 +#define F9_DIRECTION_DOWNLINK 1 + +#define AUTH_NONCE_NUM_WORDS_SHIFT 20 +#define AUTH_NONCE_NUM_WORDS_MASK GENMASK(22, 20) + +#define USE_PIPE_KEY_AUTH_SHIFT 19 +#define USE_HW_KEY_AUTH_SHIFT 18 +#define AUTH_FIRST_SHIFT 17 +#define AUTH_LAST_SHIFT 16 + +#define AUTH_POS_SHIFT 14 +#define AUTH_POS_MASK GENMASK(15, 14) +#define AUTH_POS_BEFORE 0 +#define AUTH_POS_AFTER 1 + +#define AUTH_SIZE_SHIFT 9 +#define AUTH_SIZE_MASK GENMASK(13, 9) +#define AUTH_SIZE_SHA1 0 +#define AUTH_SIZE_SHA256 1 +#define AUTH_SIZE_ENUM_1_BYTES 0 +#define AUTH_SIZE_ENUM_2_BYTES 1 +#define AUTH_SIZE_ENUM_3_BYTES 2 +#define AUTH_SIZE_ENUM_4_BYTES 3 +#define AUTH_SIZE_ENUM_5_BYTES 4 +#define AUTH_SIZE_ENUM_6_BYTES 5 +#define AUTH_SIZE_ENUM_7_BYTES 6 +#define AUTH_SIZE_ENUM_8_BYTES 7 +#define AUTH_SIZE_ENUM_9_BYTES 8 +#define AUTH_SIZE_ENUM_10_BYTES 9 +#define AUTH_SIZE_ENUM_11_BYTES 10 +#define AUTH_SIZE_ENUM_12_BYTES 11 +#define AUTH_SIZE_ENUM_13_BYTES 12 +#define AUTH_SIZE_ENUM_14_BYTES 13 +#define AUTH_SIZE_ENUM_15_BYTES 14 +#define AUTH_SIZE_ENUM_16_BYTES 15 + +#define AUTH_MODE_SHIFT 6 +#define AUTH_MODE_MASK GENMASK(8, 6) +#define AUTH_MODE_HASH 0 +#define AUTH_MODE_HMAC 1 +#define AUTH_MODE_CCM 0 +#define AUTH_MODE_CMAC 1 + +#define AUTH_KEY_SIZE_SHIFT 3 +#define AUTH_KEY_SIZE_MASK GENMASK(5, 3) +#define AUTH_KEY_SZ_AES128 0 +#define AUTH_KEY_SZ_AES256 2 + +#define AUTH_ALG_SHIFT 0 +#define AUTH_ALG_MASK GENMASK(2, 0) +#define AUTH_ALG_NONE 0 +#define AUTH_ALG_SHA 1 +#define AUTH_ALG_AES 2 +#define AUTH_ALG_KASUMI 3 +#define AUTH_ALG_SNOW3G 4 +#define AUTH_ALG_ZUC 5 + +/* Register bits - REG_ENCR_XTS_DU_SIZE */ +#define ENCR_XTS_DU_SIZE_SHIFT 0 +#define ENCR_XTS_DU_SIZE_MASK GENMASK(19, 0) + +/* Register bits - REG_ENCR_SEG_CFG */ +#define F8_KEYSTREAM_ENABLE_SHIFT 17 +#define F8_KEYSTREAM_DISABLED 0 +#define F8_KEYSTREAM_ENABLED 1 + +#define F8_DIRECTION_SHIFT 16 +#define F8_DIRECTION_UPLINK 0 +#define F8_DIRECTION_DOWNLINK 1 + +#define USE_PIPE_KEY_ENCR_SHIFT 15 +#define USE_PIPE_KEY_ENCR_ENABLED 1 +#define USE_KEY_REGISTERS 0 + +#define USE_HW_KEY_ENCR_SHIFT 14 +#define USE_KEY_REG 0 +#define USE_HW_KEY 1 + +#define LAST_CCM_SHIFT 13 +#define LAST_CCM_XFR 1 +#define INTERM_CCM_XFR 0 + +#define CNTR_ALG_SHIFT 11 +#define CNTR_ALG_MASK GENMASK(12, 11) +#define CNTR_ALG_NIST 0 + +#define ENCODE_SHIFT 10 + +#define ENCR_MODE_SHIFT 6 +#define ENCR_MODE_MASK GENMASK(9, 6) +#define ENCR_MODE_ECB 0 +#define ENCR_MODE_CBC 1 +#define ENCR_MODE_CTR 2 +#define ENCR_MODE_XTS 3 +#define ENCR_MODE_CCM 4 + +#define ENCR_KEY_SZ_SHIFT 3 +#define ENCR_KEY_SZ_MASK GENMASK(5, 3) +#define ENCR_KEY_SZ_DES 0 +#define ENCR_KEY_SZ_3DES 1 +#define ENCR_KEY_SZ_AES128 0 +#define ENCR_KEY_SZ_AES256 2 + +#define ENCR_ALG_SHIFT 0 +#define ENCR_ALG_MASK GENMASK(2, 0) +#define ENCR_ALG_NONE 0 +#define ENCR_ALG_DES 1 +#define ENCR_ALG_AES 2 +#define ENCR_ALG_KASUMI 4 +#define ENCR_ALG_SNOW_3G 5 +#define ENCR_ALG_ZUC 6 + +/* Register bits - REG_GOPROC */ +#define GO_SHIFT 0 +#define CLR_CNTXT_SHIFT 1 +#define RESULTS_DUMP_SHIFT 2 + +/* Register bits - REG_ENGINES_AVAIL */ +#define ENCR_AES_SEL_SHIFT 0 +#define DES_SEL_SHIFT 1 +#define ENCR_SNOW3G_SEL_SHIFT 2 +#define ENCR_KASUMI_SEL_SHIFT 3 +#define SHA_SEL_SHIFT 4 +#define SHA512_SEL_SHIFT 5 +#define AUTH_AES_SEL_SHIFT 6 +#define AUTH_SNOW3G_SEL_SHIFT 7 +#define AUTH_KASUMI_SEL_SHIFT 8 +#define BAM_PIPE_SETS_SHIFT 9 +#define BAM_PIPE_SETS_MASK GENMASK(12, 9) +#define AXI_WR_BEATS_SHIFT 13 +#define AXI_WR_BEATS_MASK GENMASK(18, 13) +#define AXI_RD_BEATS_SHIFT 19 +#define AXI_RD_BEATS_MASK GENMASK(24, 19) +#define ENCR_ZUC_SEL_SHIFT 26 +#define AUTH_ZUC_SEL_SHIFT 27 +#define ZUC_ENABLE_SHIFT 28 + +#endif /* _REGS_V5_H_ */ diff --git a/drivers/crypto/qce/sha.c b/drivers/crypto/qce/sha.c new file mode 100644 index 000000000000..f3385934eed2 --- /dev/null +++ b/drivers/crypto/qce/sha.c @@ -0,0 +1,588 @@ +/* + * Copyright (c) 2010-2014, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include <linux/device.h> +#include <linux/interrupt.h> +#include <crypto/internal/hash.h> + +#include "common.h" +#include "core.h" +#include "sha.h" + +/* crypto hw padding constant for first operation */ +#define SHA_PADDING 64 +#define SHA_PADDING_MASK (SHA_PADDING - 1) + +static LIST_HEAD(ahash_algs); + +static const u32 std_iv_sha1[SHA256_DIGEST_SIZE / sizeof(u32)] = { + SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4, 0, 0, 0 +}; + +static const u32 std_iv_sha256[SHA256_DIGEST_SIZE / sizeof(u32)] = { + SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3, + SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7 +}; + +static void qce_ahash_done(void *data) +{ + struct crypto_async_request *async_req = data; + struct ahash_request *req = ahash_request_cast(async_req); + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct qce_sha_reqctx *rctx = ahash_request_ctx(req); + struct qce_alg_template *tmpl = to_ahash_tmpl(async_req->tfm); + struct qce_device *qce = tmpl->qce; + struct qce_result_dump *result = qce->dma.result_buf; + unsigned int digestsize = crypto_ahash_digestsize(ahash); + int error; + u32 status; + + error = qce_dma_terminate_all(&qce->dma); + if (error) + dev_dbg(qce->dev, "ahash dma termination error (%d)\n", error); + + qce_unmapsg(qce->dev, req->src, rctx->src_nents, DMA_TO_DEVICE, + rctx->src_chained); + qce_unmapsg(qce->dev, &rctx->result_sg, 1, DMA_FROM_DEVICE, 0); + + memcpy(rctx->digest, result->auth_iv, digestsize); + if (req->result) + memcpy(req->result, result->auth_iv, digestsize); + + rctx->byte_count[0] = cpu_to_be32(result->auth_byte_count[0]); + rctx->byte_count[1] = cpu_to_be32(result->auth_byte_count[1]); + + error = qce_check_status(qce, &status); + if (error < 0) + dev_dbg(qce->dev, "ahash operation error (%x)\n", status); + + req->src = rctx->src_orig; + req->nbytes = rctx->nbytes_orig; + rctx->last_blk = false; + rctx->first_blk = false; + + qce->async_req_done(tmpl->qce, error); +} + +static int qce_ahash_async_req_handle(struct crypto_async_request *async_req) +{ + struct ahash_request *req = ahash_request_cast(async_req); + struct qce_sha_reqctx *rctx = ahash_request_ctx(req); + struct qce_sha_ctx *ctx = crypto_tfm_ctx(async_req->tfm); + struct qce_alg_template *tmpl = to_ahash_tmpl(async_req->tfm); + struct qce_device *qce = tmpl->qce; + unsigned long flags = rctx->flags; + int ret; + + if (IS_SHA_HMAC(flags)) { + rctx->authkey = ctx->authkey; + rctx->authklen = QCE_SHA_HMAC_KEY_SIZE; + } else if (IS_CMAC(flags)) { + rctx->authkey = ctx->authkey; + rctx->authklen = AES_KEYSIZE_128; + } + + rctx->src_nents = qce_countsg(req->src, req->nbytes, + &rctx->src_chained); + ret = qce_mapsg(qce->dev, req->src, rctx->src_nents, DMA_TO_DEVICE, + rctx->src_chained); + if (ret < 0) + return ret; + + sg_init_one(&rctx->result_sg, qce->dma.result_buf, QCE_RESULT_BUF_SZ); + + ret = qce_mapsg(qce->dev, &rctx->result_sg, 1, DMA_FROM_DEVICE, 0); + if (ret < 0) + goto error_unmap_src; + + ret = qce_dma_prep_sgs(&qce->dma, req->src, rctx->src_nents, + &rctx->result_sg, 1, qce_ahash_done, async_req); + if (ret) + goto error_unmap_dst; + + qce_dma_issue_pending(&qce->dma); + + ret = qce_start(async_req, tmpl->crypto_alg_type, 0, 0); + if (ret) + goto error_terminate; + + return 0; + +error_terminate: + qce_dma_terminate_all(&qce->dma); +error_unmap_dst: + qce_unmapsg(qce->dev, &rctx->result_sg, 1, DMA_FROM_DEVICE, 0); +error_unmap_src: + qce_unmapsg(qce->dev, req->src, rctx->src_nents, DMA_TO_DEVICE, + rctx->src_chained); + return ret; +} + +static int qce_ahash_init(struct ahash_request *req) +{ + struct qce_sha_reqctx *rctx = ahash_request_ctx(req); + struct qce_alg_template *tmpl = to_ahash_tmpl(req->base.tfm); + const u32 *std_iv = tmpl->std_iv; + + memset(rctx, 0, sizeof(*rctx)); + rctx->first_blk = true; + rctx->last_blk = false; + rctx->flags = tmpl->alg_flags; + memcpy(rctx->digest, std_iv, sizeof(rctx->digest)); + + return 0; +} + +static int qce_ahash_export(struct ahash_request *req, void *out) +{ + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct qce_sha_reqctx *rctx = ahash_request_ctx(req); + unsigned long flags = rctx->flags; + unsigned int digestsize = crypto_ahash_digestsize(ahash); + unsigned int blocksize = + crypto_tfm_alg_blocksize(crypto_ahash_tfm(ahash)); + + if (IS_SHA1(flags) || IS_SHA1_HMAC(flags)) { + struct sha1_state *out_state = out; + + out_state->count = rctx->count; + qce_cpu_to_be32p_array((__be32 *)out_state->state, + rctx->digest, digestsize); + memcpy(out_state->buffer, rctx->buf, blocksize); + } else if (IS_SHA256(flags) || IS_SHA256_HMAC(flags)) { + struct sha256_state *out_state = out; + + out_state->count = rctx->count; + qce_cpu_to_be32p_array((__be32 *)out_state->state, + rctx->digest, digestsize); + memcpy(out_state->buf, rctx->buf, blocksize); + } else { + return -EINVAL; + } + + return 0; +} + +static int qce_import_common(struct ahash_request *req, u64 in_count, + const u32 *state, const u8 *buffer, bool hmac) +{ + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct qce_sha_reqctx *rctx = ahash_request_ctx(req); + unsigned int digestsize = crypto_ahash_digestsize(ahash); + unsigned int blocksize; + u64 count = in_count; + + blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(ahash)); + rctx->count = in_count; + memcpy(rctx->buf, buffer, blocksize); + + if (in_count <= blocksize) { + rctx->first_blk = 1; + } else { + rctx->first_blk = 0; + /* + * For HMAC, there is a hardware padding done when first block + * is set. Therefore the byte_count must be incremened by 64 + * after the first block operation. + */ + if (hmac) + count += SHA_PADDING; + } + + rctx->byte_count[0] = (__force __be32)(count & ~SHA_PADDING_MASK); + rctx->byte_count[1] = (__force __be32)(count >> 32); + qce_cpu_to_be32p_array((__be32 *)rctx->digest, (const u8 *)state, + digestsize); + rctx->buflen = (unsigned int)(in_count & (blocksize - 1)); + + return 0; +} + +static int qce_ahash_import(struct ahash_request *req, const void *in) +{ + struct qce_sha_reqctx *rctx = ahash_request_ctx(req); + unsigned long flags = rctx->flags; + bool hmac = IS_SHA_HMAC(flags); + int ret = -EINVAL; + + if (IS_SHA1(flags) || IS_SHA1_HMAC(flags)) { + const struct sha1_state *state = in; + + ret = qce_import_common(req, state->count, state->state, + state->buffer, hmac); + } else if (IS_SHA256(flags) || IS_SHA256_HMAC(flags)) { + const struct sha256_state *state = in; + + ret = qce_import_common(req, state->count, state->state, + state->buf, hmac); + } + + return ret; +} + +static int qce_ahash_update(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct qce_sha_reqctx *rctx = ahash_request_ctx(req); + struct qce_alg_template *tmpl = to_ahash_tmpl(req->base.tfm); + struct qce_device *qce = tmpl->qce; + struct scatterlist *sg_last, *sg; + unsigned int total, len; + unsigned int hash_later; + unsigned int nbytes; + unsigned int blocksize; + + blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); + rctx->count += req->nbytes; + + /* check for buffer from previous updates and append it */ + total = req->nbytes + rctx->buflen; + + if (total <= blocksize) { + scatterwalk_map_and_copy(rctx->buf + rctx->buflen, req->src, + 0, req->nbytes, 0); + rctx->buflen += req->nbytes; + return 0; + } + + /* save the original req structure fields */ + rctx->src_orig = req->src; + rctx->nbytes_orig = req->nbytes; + + /* + * if we have data from previous update copy them on buffer. The old + * data will be combined with current request bytes. + */ + if (rctx->buflen) + memcpy(rctx->tmpbuf, rctx->buf, rctx->buflen); + + /* calculate how many bytes will be hashed later */ + hash_later = total % blocksize; + if (hash_later) { + unsigned int src_offset = req->nbytes - hash_later; + scatterwalk_map_and_copy(rctx->buf, req->src, src_offset, + hash_later, 0); + } + + /* here nbytes is multiple of blocksize */ + nbytes = total - hash_later; + + len = rctx->buflen; + sg = sg_last = req->src; + + while (len < nbytes && sg) { + if (len + sg_dma_len(sg) > nbytes) + break; + len += sg_dma_len(sg); + sg_last = sg; + sg = scatterwalk_sg_next(sg); + } + + if (!sg_last) + return -EINVAL; + + sg_mark_end(sg_last); + + if (rctx->buflen) { + sg_init_table(rctx->sg, 2); + sg_set_buf(rctx->sg, rctx->tmpbuf, rctx->buflen); + scatterwalk_sg_chain(rctx->sg, 2, req->src); + req->src = rctx->sg; + } + + req->nbytes = nbytes; + rctx->buflen = hash_later; + + return qce->async_req_enqueue(tmpl->qce, &req->base); +} + +static int qce_ahash_final(struct ahash_request *req) +{ + struct qce_sha_reqctx *rctx = ahash_request_ctx(req); + struct qce_alg_template *tmpl = to_ahash_tmpl(req->base.tfm); + struct qce_device *qce = tmpl->qce; + + if (!rctx->buflen) + return 0; + + rctx->last_blk = true; + + rctx->src_orig = req->src; + rctx->nbytes_orig = req->nbytes; + + memcpy(rctx->tmpbuf, rctx->buf, rctx->buflen); + sg_init_one(rctx->sg, rctx->tmpbuf, rctx->buflen); + + req->src = rctx->sg; + req->nbytes = rctx->buflen; + + return qce->async_req_enqueue(tmpl->qce, &req->base); +} + +static int qce_ahash_digest(struct ahash_request *req) +{ + struct qce_sha_reqctx *rctx = ahash_request_ctx(req); + struct qce_alg_template *tmpl = to_ahash_tmpl(req->base.tfm); + struct qce_device *qce = tmpl->qce; + int ret; + + ret = qce_ahash_init(req); + if (ret) + return ret; + + rctx->src_orig = req->src; + rctx->nbytes_orig = req->nbytes; + rctx->first_blk = true; + rctx->last_blk = true; + + return qce->async_req_enqueue(tmpl->qce, &req->base); +} + +struct qce_ahash_result { + struct completion completion; + int error; +}; + +static void qce_digest_complete(struct crypto_async_request *req, int error) +{ + struct qce_ahash_result *result = req->data; + + if (error == -EINPROGRESS) + return; + + result->error = error; + complete(&result->completion); +} + +static int qce_ahash_hmac_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen) +{ + unsigned int digestsize = crypto_ahash_digestsize(tfm); + struct qce_sha_ctx *ctx = crypto_tfm_ctx(&tfm->base); + struct qce_ahash_result result; + struct ahash_request *req; + struct scatterlist sg; + unsigned int blocksize; + struct crypto_ahash *ahash_tfm; + u8 *buf; + int ret; + const char *alg_name; + + blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); + memset(ctx->authkey, 0, sizeof(ctx->authkey)); + + if (keylen <= blocksize) { + memcpy(ctx->authkey, key, keylen); + return 0; + } + + if (digestsize == SHA1_DIGEST_SIZE) + alg_name = "sha1-qce"; + else if (digestsize == SHA256_DIGEST_SIZE) + alg_name = "sha256-qce"; + else + return -EINVAL; + + ahash_tfm = crypto_alloc_ahash(alg_name, CRYPTO_ALG_TYPE_AHASH, + CRYPTO_ALG_TYPE_AHASH_MASK); + if (IS_ERR(ahash_tfm)) + return PTR_ERR(ahash_tfm); + + req = ahash_request_alloc(ahash_tfm, GFP_KERNEL); + if (!req) { + ret = -ENOMEM; + goto err_free_ahash; + } + + init_completion(&result.completion); + ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + qce_digest_complete, &result); + crypto_ahash_clear_flags(ahash_tfm, ~0); + + buf = kzalloc(keylen + QCE_MAX_ALIGN_SIZE, GFP_KERNEL); + if (!buf) { + ret = -ENOMEM; + goto err_free_req; + } + + memcpy(buf, key, keylen); + sg_init_one(&sg, buf, keylen); + ahash_request_set_crypt(req, &sg, ctx->authkey, keylen); + + ret = crypto_ahash_digest(req); + if (ret == -EINPROGRESS || ret == -EBUSY) { + ret = wait_for_completion_interruptible(&result.completion); + if (!ret) + ret = result.error; + } + + if (ret) + crypto_ahash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + + kfree(buf); +err_free_req: + ahash_request_free(req); +err_free_ahash: + crypto_free_ahash(ahash_tfm); + return ret; +} + +static int qce_ahash_cra_init(struct crypto_tfm *tfm) +{ + struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); + struct qce_sha_ctx *ctx = crypto_tfm_ctx(tfm); + + crypto_ahash_set_reqsize(ahash, sizeof(struct qce_sha_reqctx)); + memset(ctx, 0, sizeof(*ctx)); + return 0; +} + +struct qce_ahash_def { + unsigned long flags; + const char *name; + const char *drv_name; + unsigned int digestsize; + unsigned int blocksize; + unsigned int statesize; + const u32 *std_iv; +}; + +static const struct qce_ahash_def ahash_def[] = { + { + .flags = QCE_HASH_SHA1, + .name = "sha1", + .drv_name = "sha1-qce", + .digestsize = SHA1_DIGEST_SIZE, + .blocksize = SHA1_BLOCK_SIZE, + .statesize = sizeof(struct sha1_state), + .std_iv = std_iv_sha1, + }, + { + .flags = QCE_HASH_SHA256, + .name = "sha256", + .drv_name = "sha256-qce", + .digestsize = SHA256_DIGEST_SIZE, + .blocksize = SHA256_BLOCK_SIZE, + .statesize = sizeof(struct sha256_state), + .std_iv = std_iv_sha256, + }, + { + .flags = QCE_HASH_SHA1_HMAC, + .name = "hmac(sha1)", + .drv_name = "hmac-sha1-qce", + .digestsize = SHA1_DIGEST_SIZE, + .blocksize = SHA1_BLOCK_SIZE, + .statesize = sizeof(struct sha1_state), + .std_iv = std_iv_sha1, + }, + { + .flags = QCE_HASH_SHA256_HMAC, + .name = "hmac(sha256)", + .drv_name = "hmac-sha256-qce", + .digestsize = SHA256_DIGEST_SIZE, + .blocksize = SHA256_BLOCK_SIZE, + .statesize = sizeof(struct sha256_state), + .std_iv = std_iv_sha256, + }, +}; + +static int qce_ahash_register_one(const struct qce_ahash_def *def, + struct qce_device *qce) +{ + struct qce_alg_template *tmpl; + struct ahash_alg *alg; + struct crypto_alg *base; + int ret; + + tmpl = kzalloc(sizeof(*tmpl), GFP_KERNEL); + if (!tmpl) + return -ENOMEM; + + tmpl->std_iv = def->std_iv; + + alg = &tmpl->alg.ahash; + alg->init = qce_ahash_init; + alg->update = qce_ahash_update; + alg->final = qce_ahash_final; + alg->digest = qce_ahash_digest; + alg->export = qce_ahash_export; + alg->import = qce_ahash_import; + if (IS_SHA_HMAC(def->flags)) + alg->setkey = qce_ahash_hmac_setkey; + alg->halg.digestsize = def->digestsize; + alg->halg.statesize = def->statesize; + + base = &alg->halg.base; + base->cra_blocksize = def->blocksize; + base->cra_priority = 300; + base->cra_flags = CRYPTO_ALG_ASYNC; + base->cra_ctxsize = sizeof(struct qce_sha_ctx); + base->cra_alignmask = 0; + base->cra_module = THIS_MODULE; + base->cra_init = qce_ahash_cra_init; + INIT_LIST_HEAD(&base->cra_list); + + snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "%s", def->name); + snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", + def->drv_name); + + INIT_LIST_HEAD(&tmpl->entry); + tmpl->crypto_alg_type = CRYPTO_ALG_TYPE_AHASH; + tmpl->alg_flags = def->flags; + tmpl->qce = qce; + + ret = crypto_register_ahash(alg); + if (ret) { + kfree(tmpl); + dev_err(qce->dev, "%s registration failed\n", base->cra_name); + return ret; + } + + list_add_tail(&tmpl->entry, &ahash_algs); + dev_dbg(qce->dev, "%s is registered\n", base->cra_name); + return 0; +} + +static void qce_ahash_unregister(struct qce_device *qce) +{ + struct qce_alg_template *tmpl, *n; + + list_for_each_entry_safe(tmpl, n, &ahash_algs, entry) { + crypto_unregister_ahash(&tmpl->alg.ahash); + list_del(&tmpl->entry); + kfree(tmpl); + } +} + +static int qce_ahash_register(struct qce_device *qce) +{ + int ret, i; + + for (i = 0; i < ARRAY_SIZE(ahash_def); i++) { + ret = qce_ahash_register_one(&ahash_def[i], qce); + if (ret) + goto err; + } + + return 0; +err: + qce_ahash_unregister(qce); + return ret; +} + +const struct qce_algo_ops ahash_ops = { + .type = CRYPTO_ALG_TYPE_AHASH, + .register_algs = qce_ahash_register, + .unregister_algs = qce_ahash_unregister, + .async_req_handle = qce_ahash_async_req_handle, +}; diff --git a/drivers/crypto/qce/sha.h b/drivers/crypto/qce/sha.h new file mode 100644 index 000000000000..286f0d5397f3 --- /dev/null +++ b/drivers/crypto/qce/sha.h @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2010-2014, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef _SHA_H_ +#define _SHA_H_ + +#include <crypto/scatterwalk.h> +#include <crypto/sha.h> + +#include "common.h" +#include "core.h" + +#define QCE_SHA_MAX_BLOCKSIZE SHA256_BLOCK_SIZE +#define QCE_SHA_MAX_DIGESTSIZE SHA256_DIGEST_SIZE + +struct qce_sha_ctx { + u8 authkey[QCE_SHA_MAX_BLOCKSIZE]; +}; + +/** + * struct qce_sha_reqctx - holds private ahash objects per request + * @buf: used during update, import and export + * @tmpbuf: buffer for internal use + * @digest: calculated digest buffer + * @buflen: length of the buffer + * @flags: operation flags + * @src_orig: original request sg list + * @nbytes_orig: original request number of bytes + * @src_chained: is source scatterlist chained + * @src_nents: source number of entries + * @byte_count: byte count + * @count: save count in states during update, import and export + * @first_blk: is it the first block + * @last_blk: is it the last block + * @sg: used to chain sg lists + * @authkey: pointer to auth key in sha ctx + * @authklen: auth key length + * @result_sg: scatterlist used for result buffer + */ +struct qce_sha_reqctx { + u8 buf[QCE_SHA_MAX_BLOCKSIZE]; + u8 tmpbuf[QCE_SHA_MAX_BLOCKSIZE]; + u8 digest[QCE_SHA_MAX_DIGESTSIZE]; + unsigned int buflen; + unsigned long flags; + struct scatterlist *src_orig; + unsigned int nbytes_orig; + bool src_chained; + int src_nents; + __be32 byte_count[2]; + u64 count; + bool first_blk; + bool last_blk; + struct scatterlist sg[2]; + u8 *authkey; + unsigned int authklen; + struct scatterlist result_sg; +}; + +static inline struct qce_alg_template *to_ahash_tmpl(struct crypto_tfm *tfm) +{ + struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); + struct ahash_alg *alg = container_of(crypto_hash_alg_common(ahash), + struct ahash_alg, halg); + + return container_of(alg, struct qce_alg_template, alg.ahash); +} + +extern const struct qce_algo_ops ahash_ops; + +#endif /* _SHA_H_ */ diff --git a/drivers/crypto/ux500/cryp/cryp_core.c b/drivers/crypto/ux500/cryp/cryp_core.c index a999f537228f..92105f3dc8e0 100644 --- a/drivers/crypto/ux500/cryp/cryp_core.c +++ b/drivers/crypto/ux500/cryp/cryp_core.c @@ -190,7 +190,7 @@ static void add_session_id(struct cryp_ctx *ctx) static irqreturn_t cryp_interrupt_handler(int irq, void *param) { struct cryp_ctx *ctx; - int i; + int count; struct cryp_device_data *device_data; if (param == NULL) { @@ -215,12 +215,11 @@ static irqreturn_t cryp_interrupt_handler(int irq, void *param) if (cryp_pending_irq_src(device_data, CRYP_IRQ_SRC_OUTPUT_FIFO)) { if (ctx->outlen / ctx->blocksize > 0) { - for (i = 0; i < ctx->blocksize / 4; i++) { - *(ctx->outdata) = readl_relaxed( - &device_data->base->dout); - ctx->outdata += 4; - ctx->outlen -= 4; - } + count = ctx->blocksize / 4; + + readsl(&device_data->base->dout, ctx->outdata, count); + ctx->outdata += count; + ctx->outlen -= count; if (ctx->outlen == 0) { cryp_disable_irq_src(device_data, @@ -230,12 +229,12 @@ static irqreturn_t cryp_interrupt_handler(int irq, void *param) } else if (cryp_pending_irq_src(device_data, CRYP_IRQ_SRC_INPUT_FIFO)) { if (ctx->datalen / ctx->blocksize > 0) { - for (i = 0 ; i < ctx->blocksize / 4; i++) { - writel_relaxed(ctx->indata, - &device_data->base->din); - ctx->indata += 4; - ctx->datalen -= 4; - } + count = ctx->blocksize / 4; + + writesl(&device_data->base->din, ctx->indata, count); + + ctx->indata += count; + ctx->datalen -= count; if (ctx->datalen == 0) cryp_disable_irq_src(device_data, diff --git a/drivers/edac/Kconfig b/drivers/edac/Kconfig index 878f09005fad..e339c6b91425 100644 --- a/drivers/edac/Kconfig +++ b/drivers/edac/Kconfig @@ -186,6 +186,13 @@ config EDAC_I3200 Support for error detection and correction on the Intel 3200 and 3210 server chipsets. +config EDAC_IE31200 + tristate "Intel e312xx" + depends on EDAC_MM_EDAC && PCI && X86 + help + Support for error detection and correction on the Intel + E3-1200 based DRAM controllers. + config EDAC_X38 tristate "Intel X38" depends on EDAC_MM_EDAC && PCI && X86 diff --git a/drivers/edac/Makefile b/drivers/edac/Makefile index 4154ed6a02c6..c479a24d8f77 100644 --- a/drivers/edac/Makefile +++ b/drivers/edac/Makefile @@ -37,6 +37,7 @@ obj-$(CONFIG_EDAC_I82875P) += i82875p_edac.o obj-$(CONFIG_EDAC_I82975X) += i82975x_edac.o obj-$(CONFIG_EDAC_I3000) += i3000_edac.o obj-$(CONFIG_EDAC_I3200) += i3200_edac.o +obj-$(CONFIG_EDAC_IE31200) += ie31200_edac.o obj-$(CONFIG_EDAC_X38) += x38_edac.o obj-$(CONFIG_EDAC_I82860) += i82860_edac.o obj-$(CONFIG_EDAC_R82600) += r82600_edac.o diff --git a/drivers/edac/edac_module.c b/drivers/edac/edac_module.c index a66941fea5a4..e6d1691dfa45 100644 --- a/drivers/edac/edac_module.c +++ b/drivers/edac/edac_module.c @@ -28,7 +28,7 @@ static int edac_set_debug_level(const char *buf, struct kernel_param *kp) if (ret) return ret; - if (val < 0 || val > 4) + if (val > 4) return -EINVAL; return param_set_int(buf, kp); diff --git a/drivers/edac/ie31200_edac.c b/drivers/edac/ie31200_edac.c new file mode 100644 index 000000000000..a981dc6fd88e --- /dev/null +++ b/drivers/edac/ie31200_edac.c @@ -0,0 +1,536 @@ +/* + * Intel E3-1200 + * Copyright (C) 2014 Jason Baron <jbaron@akamai.com> + * + * Support for the E3-1200 processor family. Heavily based on previous + * Intel EDAC drivers. + * + * Since the DRAM controller is on the cpu chip, we can use its PCI device + * id to identify these processors. + * + * PCI DRAM controller device ids (Taken from The PCI ID Repository - http://pci-ids.ucw.cz/) + * + * 0108: Xeon E3-1200 Processor Family DRAM Controller + * 010c: Xeon E3-1200/2nd Generation Core Processor Family DRAM Controller + * 0150: Xeon E3-1200 v2/3rd Gen Core processor DRAM Controller + * 0158: Xeon E3-1200 v2/Ivy Bridge DRAM Controller + * 015c: Xeon E3-1200 v2/3rd Gen Core processor DRAM Controller + * 0c04: Xeon E3-1200 v3/4th Gen Core Processor DRAM Controller + * 0c08: Xeon E3-1200 v3 Processor DRAM Controller + * + * Based on Intel specification: + * http://www.intel.com/content/dam/www/public/us/en/documents/datasheets/xeon-e3-1200v3-vol-2-datasheet.pdf + * http://www.intel.com/content/www/us/en/processors/xeon/xeon-e3-1200-family-vol-2-datasheet.html + * + * According to the above datasheet (p.16): + * " + * 6. Software must not access B0/D0/F0 32-bit memory-mapped registers with + * requests that cross a DW boundary. + * " + * + * Thus, we make use of the explicit: lo_hi_readq(), which breaks the readq into + * 2 readl() calls. This restriction may be lifted in subsequent chip releases, + * but lo_hi_readq() ensures that we are safe across all e3-1200 processors. + */ + +#include <linux/module.h> +#include <linux/init.h> +#include <linux/pci.h> +#include <linux/pci_ids.h> +#include <linux/edac.h> + +#include <asm-generic/io-64-nonatomic-lo-hi.h> +#include "edac_core.h" + +#define IE31200_REVISION "1.0" +#define EDAC_MOD_STR "ie31200_edac" + +#define ie31200_printk(level, fmt, arg...) \ + edac_printk(level, "ie31200", fmt, ##arg) + +#define PCI_DEVICE_ID_INTEL_IE31200_HB_1 0x0108 +#define PCI_DEVICE_ID_INTEL_IE31200_HB_2 0x010c +#define PCI_DEVICE_ID_INTEL_IE31200_HB_3 0x0150 +#define PCI_DEVICE_ID_INTEL_IE31200_HB_4 0x0158 +#define PCI_DEVICE_ID_INTEL_IE31200_HB_5 0x015c +#define PCI_DEVICE_ID_INTEL_IE31200_HB_6 0x0c04 +#define PCI_DEVICE_ID_INTEL_IE31200_HB_7 0x0c08 + +#define IE31200_DIMMS 4 +#define IE31200_RANKS 8 +#define IE31200_RANKS_PER_CHANNEL 4 +#define IE31200_DIMMS_PER_CHANNEL 2 +#define IE31200_CHANNELS 2 + +/* Intel IE31200 register addresses - device 0 function 0 - DRAM Controller */ +#define IE31200_MCHBAR_LOW 0x48 +#define IE31200_MCHBAR_HIGH 0x4c +#define IE31200_MCHBAR_MASK GENMASK_ULL(38, 15) +#define IE31200_MMR_WINDOW_SIZE BIT(15) + +/* + * Error Status Register (16b) + * + * 15 reserved + * 14 Isochronous TBWRR Run Behind FIFO Full + * (ITCV) + * 13 Isochronous TBWRR Run Behind FIFO Put + * (ITSTV) + * 12 reserved + * 11 MCH Thermal Sensor Event + * for SMI/SCI/SERR (GTSE) + * 10 reserved + * 9 LOCK to non-DRAM Memory Flag (LCKF) + * 8 reserved + * 7 DRAM Throttle Flag (DTF) + * 6:2 reserved + * 1 Multi-bit DRAM ECC Error Flag (DMERR) + * 0 Single-bit DRAM ECC Error Flag (DSERR) + */ +#define IE31200_ERRSTS 0xc8 +#define IE31200_ERRSTS_UE BIT(1) +#define IE31200_ERRSTS_CE BIT(0) +#define IE31200_ERRSTS_BITS (IE31200_ERRSTS_UE | IE31200_ERRSTS_CE) + +/* + * Channel 0 ECC Error Log (64b) + * + * 63:48 Error Column Address (ERRCOL) + * 47:32 Error Row Address (ERRROW) + * 31:29 Error Bank Address (ERRBANK) + * 28:27 Error Rank Address (ERRRANK) + * 26:24 reserved + * 23:16 Error Syndrome (ERRSYND) + * 15: 2 reserved + * 1 Multiple Bit Error Status (MERRSTS) + * 0 Correctable Error Status (CERRSTS) + */ +#define IE31200_C0ECCERRLOG 0x40c8 +#define IE31200_C1ECCERRLOG 0x44c8 +#define IE31200_ECCERRLOG_CE BIT(0) +#define IE31200_ECCERRLOG_UE BIT(1) +#define IE31200_ECCERRLOG_RANK_BITS GENMASK_ULL(28, 27) +#define IE31200_ECCERRLOG_RANK_SHIFT 27 +#define IE31200_ECCERRLOG_SYNDROME_BITS GENMASK_ULL(23, 16) +#define IE31200_ECCERRLOG_SYNDROME_SHIFT 16 + +#define IE31200_ECCERRLOG_SYNDROME(log) \ + ((log & IE31200_ECCERRLOG_SYNDROME_BITS) >> \ + IE31200_ECCERRLOG_SYNDROME_SHIFT) + +#define IE31200_CAPID0 0xe4 +#define IE31200_CAPID0_PDCD BIT(4) +#define IE31200_CAPID0_DDPCD BIT(6) +#define IE31200_CAPID0_ECC BIT(1) + +#define IE31200_MAD_DIMM_0_OFFSET 0x5004 +#define IE31200_MAD_DIMM_SIZE GENMASK_ULL(7, 0) +#define IE31200_MAD_DIMM_A_RANK BIT(17) +#define IE31200_MAD_DIMM_A_WIDTH BIT(19) + +#define IE31200_PAGES(n) (n << (28 - PAGE_SHIFT)) + +static int nr_channels; + +struct ie31200_priv { + void __iomem *window; +}; + +enum ie31200_chips { + IE31200 = 0, +}; + +struct ie31200_dev_info { + const char *ctl_name; +}; + +struct ie31200_error_info { + u16 errsts; + u16 errsts2; + u64 eccerrlog[IE31200_CHANNELS]; +}; + +static const struct ie31200_dev_info ie31200_devs[] = { + [IE31200] = { + .ctl_name = "IE31200" + }, +}; + +struct dimm_data { + u8 size; /* in 256MB multiples */ + u8 dual_rank : 1, + x16_width : 1; /* 0 means x8 width */ +}; + +static int how_many_channels(struct pci_dev *pdev) +{ + int n_channels; + unsigned char capid0_2b; /* 2nd byte of CAPID0 */ + + pci_read_config_byte(pdev, IE31200_CAPID0 + 1, &capid0_2b); + + /* check PDCD: Dual Channel Disable */ + if (capid0_2b & IE31200_CAPID0_PDCD) { + edac_dbg(0, "In single channel mode\n"); + n_channels = 1; + } else { + edac_dbg(0, "In dual channel mode\n"); + n_channels = 2; + } + + /* check DDPCD - check if both channels are filled */ + if (capid0_2b & IE31200_CAPID0_DDPCD) + edac_dbg(0, "2 DIMMS per channel disabled\n"); + else + edac_dbg(0, "2 DIMMS per channel enabled\n"); + + return n_channels; +} + +static bool ecc_capable(struct pci_dev *pdev) +{ + unsigned char capid0_4b; /* 4th byte of CAPID0 */ + + pci_read_config_byte(pdev, IE31200_CAPID0 + 3, &capid0_4b); + if (capid0_4b & IE31200_CAPID0_ECC) + return false; + return true; +} + +static int eccerrlog_row(int channel, u64 log) +{ + int rank = ((log & IE31200_ECCERRLOG_RANK_BITS) >> + IE31200_ECCERRLOG_RANK_SHIFT); + return rank | (channel * IE31200_RANKS_PER_CHANNEL); +} + +static void ie31200_clear_error_info(struct mem_ctl_info *mci) +{ + /* + * Clear any error bits. + * (Yes, we really clear bits by writing 1 to them.) + */ + pci_write_bits16(to_pci_dev(mci->pdev), IE31200_ERRSTS, + IE31200_ERRSTS_BITS, IE31200_ERRSTS_BITS); +} + +static void ie31200_get_and_clear_error_info(struct mem_ctl_info *mci, + struct ie31200_error_info *info) +{ + struct pci_dev *pdev; + struct ie31200_priv *priv = mci->pvt_info; + void __iomem *window = priv->window; + + pdev = to_pci_dev(mci->pdev); + + /* + * This is a mess because there is no atomic way to read all the + * registers at once and the registers can transition from CE being + * overwritten by UE. + */ + pci_read_config_word(pdev, IE31200_ERRSTS, &info->errsts); + if (!(info->errsts & IE31200_ERRSTS_BITS)) + return; + + info->eccerrlog[0] = lo_hi_readq(window + IE31200_C0ECCERRLOG); + if (nr_channels == 2) + info->eccerrlog[1] = lo_hi_readq(window + IE31200_C1ECCERRLOG); + + pci_read_config_word(pdev, IE31200_ERRSTS, &info->errsts2); + + /* + * If the error is the same for both reads then the first set + * of reads is valid. If there is a change then there is a CE + * with no info and the second set of reads is valid and + * should be UE info. + */ + if ((info->errsts ^ info->errsts2) & IE31200_ERRSTS_BITS) { + info->eccerrlog[0] = lo_hi_readq(window + IE31200_C0ECCERRLOG); + if (nr_channels == 2) + info->eccerrlog[1] = + lo_hi_readq(window + IE31200_C1ECCERRLOG); + } + + ie31200_clear_error_info(mci); +} + +static void ie31200_process_error_info(struct mem_ctl_info *mci, + struct ie31200_error_info *info) +{ + int channel; + u64 log; + + if (!(info->errsts & IE31200_ERRSTS_BITS)) + return; + + if ((info->errsts ^ info->errsts2) & IE31200_ERRSTS_BITS) { + edac_mc_handle_error(HW_EVENT_ERR_UNCORRECTED, mci, 1, 0, 0, 0, + -1, -1, -1, "UE overwrote CE", ""); + info->errsts = info->errsts2; + } + + for (channel = 0; channel < nr_channels; channel++) { + log = info->eccerrlog[channel]; + if (log & IE31200_ECCERRLOG_UE) { + edac_mc_handle_error(HW_EVENT_ERR_UNCORRECTED, mci, 1, + 0, 0, 0, + eccerrlog_row(channel, log), + channel, -1, + "ie31200 UE", ""); + } else if (log & IE31200_ECCERRLOG_CE) { + edac_mc_handle_error(HW_EVENT_ERR_CORRECTED, mci, 1, + 0, 0, + IE31200_ECCERRLOG_SYNDROME(log), + eccerrlog_row(channel, log), + channel, -1, + "ie31200 CE", ""); + } + } +} + +static void ie31200_check(struct mem_ctl_info *mci) +{ + struct ie31200_error_info info; + + edac_dbg(1, "MC%d\n", mci->mc_idx); + ie31200_get_and_clear_error_info(mci, &info); + ie31200_process_error_info(mci, &info); +} + +static void __iomem *ie31200_map_mchbar(struct pci_dev *pdev) +{ + union { + u64 mchbar; + struct { + u32 mchbar_low; + u32 mchbar_high; + }; + } u; + void __iomem *window; + + pci_read_config_dword(pdev, IE31200_MCHBAR_LOW, &u.mchbar_low); + pci_read_config_dword(pdev, IE31200_MCHBAR_HIGH, &u.mchbar_high); + u.mchbar &= IE31200_MCHBAR_MASK; + + if (u.mchbar != (resource_size_t)u.mchbar) { + ie31200_printk(KERN_ERR, "mmio space beyond accessible range (0x%llx)\n", + (unsigned long long)u.mchbar); + return NULL; + } + + window = ioremap_nocache(u.mchbar, IE31200_MMR_WINDOW_SIZE); + if (!window) + ie31200_printk(KERN_ERR, "Cannot map mmio space at 0x%llx\n", + (unsigned long long)u.mchbar); + + return window; +} + +static int ie31200_probe1(struct pci_dev *pdev, int dev_idx) +{ + int i, j, ret; + struct mem_ctl_info *mci = NULL; + struct edac_mc_layer layers[2]; + struct dimm_data dimm_info[IE31200_CHANNELS][IE31200_DIMMS_PER_CHANNEL]; + void __iomem *window; + struct ie31200_priv *priv; + u32 addr_decode; + + edac_dbg(0, "MC:\n"); + + if (!ecc_capable(pdev)) { + ie31200_printk(KERN_INFO, "No ECC support\n"); + return -ENODEV; + } + + nr_channels = how_many_channels(pdev); + layers[0].type = EDAC_MC_LAYER_CHIP_SELECT; + layers[0].size = IE31200_DIMMS; + layers[0].is_virt_csrow = true; + layers[1].type = EDAC_MC_LAYER_CHANNEL; + layers[1].size = nr_channels; + layers[1].is_virt_csrow = false; + mci = edac_mc_alloc(0, ARRAY_SIZE(layers), layers, + sizeof(struct ie31200_priv)); + if (!mci) + return -ENOMEM; + + window = ie31200_map_mchbar(pdev); + if (!window) { + ret = -ENODEV; + goto fail_free; + } + + edac_dbg(3, "MC: init mci\n"); + mci->pdev = &pdev->dev; + mci->mtype_cap = MEM_FLAG_DDR3; + mci->edac_ctl_cap = EDAC_FLAG_SECDED; + mci->edac_cap = EDAC_FLAG_SECDED; + mci->mod_name = EDAC_MOD_STR; + mci->mod_ver = IE31200_REVISION; + mci->ctl_name = ie31200_devs[dev_idx].ctl_name; + mci->dev_name = pci_name(pdev); + mci->edac_check = ie31200_check; + mci->ctl_page_to_phys = NULL; + priv = mci->pvt_info; + priv->window = window; + + /* populate DIMM info */ + for (i = 0; i < IE31200_CHANNELS; i++) { + addr_decode = readl(window + IE31200_MAD_DIMM_0_OFFSET + + (i * 4)); + edac_dbg(0, "addr_decode: 0x%x\n", addr_decode); + for (j = 0; j < IE31200_DIMMS_PER_CHANNEL; j++) { + dimm_info[i][j].size = (addr_decode >> (j * 8)) & + IE31200_MAD_DIMM_SIZE; + dimm_info[i][j].dual_rank = (addr_decode & + (IE31200_MAD_DIMM_A_RANK << j)) ? 1 : 0; + dimm_info[i][j].x16_width = (addr_decode & + (IE31200_MAD_DIMM_A_WIDTH << j)) ? 1 : 0; + edac_dbg(0, "size: 0x%x, rank: %d, width: %d\n", + dimm_info[i][j].size, + dimm_info[i][j].dual_rank, + dimm_info[i][j].x16_width); + } + } + + /* + * The dram rank boundary (DRB) reg values are boundary addresses + * for each DRAM rank with a granularity of 64MB. DRB regs are + * cumulative; the last one will contain the total memory + * contained in all ranks. + */ + for (i = 0; i < IE31200_DIMMS_PER_CHANNEL; i++) { + for (j = 0; j < IE31200_CHANNELS; j++) { + struct dimm_info *dimm; + unsigned long nr_pages; + + nr_pages = IE31200_PAGES(dimm_info[j][i].size); + if (nr_pages == 0) + continue; + + if (dimm_info[j][i].dual_rank) { + nr_pages = nr_pages / 2; + dimm = EDAC_DIMM_PTR(mci->layers, mci->dimms, + mci->n_layers, (i * 2) + 1, + j, 0); + dimm->nr_pages = nr_pages; + edac_dbg(0, "set nr pages: 0x%lx\n", nr_pages); + dimm->grain = 8; /* just a guess */ + dimm->mtype = MEM_DDR3; + dimm->dtype = DEV_UNKNOWN; + dimm->edac_mode = EDAC_UNKNOWN; + } + dimm = EDAC_DIMM_PTR(mci->layers, mci->dimms, + mci->n_layers, i * 2, j, 0); + dimm->nr_pages = nr_pages; + edac_dbg(0, "set nr pages: 0x%lx\n", nr_pages); + dimm->grain = 8; /* same guess */ + dimm->mtype = MEM_DDR3; + dimm->dtype = DEV_UNKNOWN; + dimm->edac_mode = EDAC_UNKNOWN; + } + } + + ie31200_clear_error_info(mci); + + if (edac_mc_add_mc(mci)) { + edac_dbg(3, "MC: failed edac_mc_add_mc()\n"); + ret = -ENODEV; + goto fail_unmap; + } + + /* get this far and it's successful */ + edac_dbg(3, "MC: success\n"); + return 0; + +fail_unmap: + iounmap(window); + +fail_free: + edac_mc_free(mci); + + return ret; +} + +static int ie31200_init_one(struct pci_dev *pdev, + const struct pci_device_id *ent) +{ + edac_dbg(0, "MC:\n"); + + if (pci_enable_device(pdev) < 0) + return -EIO; + + return ie31200_probe1(pdev, ent->driver_data); +} + +static void ie31200_remove_one(struct pci_dev *pdev) +{ + struct mem_ctl_info *mci; + struct ie31200_priv *priv; + + edac_dbg(0, "\n"); + mci = edac_mc_del_mc(&pdev->dev); + if (!mci) + return; + priv = mci->pvt_info; + iounmap(priv->window); + edac_mc_free(mci); +} + +static const struct pci_device_id ie31200_pci_tbl[] = { + { + PCI_VEND_DEV(INTEL, IE31200_HB_1), PCI_ANY_ID, PCI_ANY_ID, 0, 0, + IE31200}, + { + PCI_VEND_DEV(INTEL, IE31200_HB_2), PCI_ANY_ID, PCI_ANY_ID, 0, 0, + IE31200}, + { + PCI_VEND_DEV(INTEL, IE31200_HB_3), PCI_ANY_ID, PCI_ANY_ID, 0, 0, + IE31200}, + { + PCI_VEND_DEV(INTEL, IE31200_HB_4), PCI_ANY_ID, PCI_ANY_ID, 0, 0, + IE31200}, + { + PCI_VEND_DEV(INTEL, IE31200_HB_5), PCI_ANY_ID, PCI_ANY_ID, 0, 0, + IE31200}, + { + PCI_VEND_DEV(INTEL, IE31200_HB_6), PCI_ANY_ID, PCI_ANY_ID, 0, 0, + IE31200}, + { + PCI_VEND_DEV(INTEL, IE31200_HB_7), PCI_ANY_ID, PCI_ANY_ID, 0, 0, + IE31200}, + { + 0, + } /* 0 terminated list. */ +}; +MODULE_DEVICE_TABLE(pci, ie31200_pci_tbl); + +static struct pci_driver ie31200_driver = { + .name = EDAC_MOD_STR, + .probe = ie31200_init_one, + .remove = ie31200_remove_one, + .id_table = ie31200_pci_tbl, +}; + +static int __init ie31200_init(void) +{ + edac_dbg(3, "MC:\n"); + /* Ensure that the OPSTATE is set correctly for POLL or NMI */ + opstate_init(); + + return pci_register_driver(&ie31200_driver); +} + +static void __exit ie31200_exit(void) +{ + edac_dbg(3, "MC:\n"); + pci_unregister_driver(&ie31200_driver); +} + +module_init(ie31200_init); +module_exit(ie31200_exit); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Jason Baron <jbaron@akamai.com>"); +MODULE_DESCRIPTION("MC support for Intel Processor E31200 memory hub controllers"); diff --git a/drivers/edac/mce_amd.c b/drivers/edac/mce_amd.c index 5f43620d580a..f78c1c54dbd5 100644 --- a/drivers/edac/mce_amd.c +++ b/drivers/edac/mce_amd.c @@ -78,7 +78,8 @@ static const char * const f15h_mc1_mce_desc[] = { "uop queue", "insn buffer", "predecode buffer", - "fetch address FIFO" + "fetch address FIFO", + "dispatch uop queue" }; static const char * const f15h_mc2_mce_desc[] = { @@ -267,6 +268,12 @@ static bool f15h_mc0_mce(u16 ec, u8 xec) pr_cont("System Read Data Error.\n"); else pr_cont(" Internal error condition type %d.\n", xec); + } else if (INT_ERROR(ec)) { + if (xec <= 0x1f) + pr_cont("Hardware Assert.\n"); + else + ret = false; + } else ret = false; @@ -373,7 +380,7 @@ static bool f15h_mc1_mce(u16 ec, u8 xec) pr_cont("%s.\n", f15h_mc1_mce_desc[xec-4]); break; - case 0x11 ... 0x14: + case 0x11 ... 0x15: pr_cont("Decoder %s parity error.\n", f15h_mc1_mce_desc[xec-4]); break; @@ -397,10 +404,20 @@ static void decode_mc1_mce(struct mce *m) bool k8 = (boot_cpu_data.x86 == 0xf && (m->status & BIT_64(58))); pr_cont("during %s.\n", (k8 ? "system linefill" : "NB data read")); + } else if (INT_ERROR(ec)) { + if (xec <= 0x3f) + pr_cont("Hardware Assert.\n"); + else + goto wrong_mc1_mce; } else if (fam_ops->mc1_mce(ec, xec)) ; else - pr_emerg(HW_ERR "Corrupted MC1 MCE info?\n"); + goto wrong_mc1_mce; + + return; + +wrong_mc1_mce: + pr_emerg(HW_ERR "Corrupted MC1 MCE info?\n"); } static bool k8_mc2_mce(u16 ec, u8 xec) @@ -468,6 +485,11 @@ static bool f15h_mc2_mce(u16 ec, u8 xec) default: ret = false; } + } else if (INT_ERROR(ec)) { + if (xec <= 0x3f) + pr_cont("Hardware Assert.\n"); + else + ret = false; } return ret; @@ -615,6 +637,7 @@ static void decode_mc4_mce(struct mce *m) static void decode_mc5_mce(struct mce *m) { struct cpuinfo_x86 *c = &boot_cpu_data; + u16 ec = EC(m->status); u8 xec = XEC(m->status, xec_mask); if (c->x86 == 0xf || c->x86 == 0x11) @@ -622,6 +645,14 @@ static void decode_mc5_mce(struct mce *m) pr_emerg(HW_ERR "MC5 Error: "); + if (INT_ERROR(ec)) { + if (xec <= 0x1f) { + pr_cont("Hardware Assert.\n"); + return; + } else + goto wrong_mc5_mce; + } + if (xec == 0x0 || xec == 0xc) pr_cont("%s.\n", mc5_mce_desc[xec]); else if (xec <= 0xd) @@ -642,6 +673,10 @@ static void decode_mc6_mce(struct mce *m) pr_emerg(HW_ERR "MC6 Error: "); switch (xec) { + case 0x0: + pr_cont("Hardware Assertion"); + break; + case 0x1: pr_cont("Free List"); break; @@ -857,7 +892,8 @@ static int __init mce_amd_init(void) break; case 0x15: - xec_mask = 0x1f; + xec_mask = c->x86_model == 0x60 ? 0x3f : 0x1f; + fam_ops->mc0_mce = f15h_mc0_mce; fam_ops->mc1_mce = f15h_mc1_mce; fam_ops->mc2_mce = f15h_mc2_mce; diff --git a/drivers/edac/x38_edac.c b/drivers/edac/x38_edac.c index 4891b450830b..e644b52c287c 100644 --- a/drivers/edac/x38_edac.c +++ b/drivers/edac/x38_edac.c @@ -14,6 +14,8 @@ #include <linux/pci.h> #include <linux/pci_ids.h> #include <linux/edac.h> + +#include <asm-generic/io-64-nonatomic-lo-hi.h> #include "edac_core.h" #define X38_REVISION "1.1" @@ -161,11 +163,6 @@ static void x38_clear_error_info(struct mem_ctl_info *mci) X38_ERRSTS_BITS); } -static u64 x38_readq(const void __iomem *addr) -{ - return readl(addr) | (((u64)readl(addr + 4)) << 32); -} - static void x38_get_and_clear_error_info(struct mem_ctl_info *mci, struct x38_error_info *info) { @@ -183,9 +180,9 @@ static void x38_get_and_clear_error_info(struct mem_ctl_info *mci, if (!(info->errsts & X38_ERRSTS_BITS)) return; - info->eccerrlog[0] = x38_readq(window + X38_C0ECCERRLOG); + info->eccerrlog[0] = lo_hi_readq(window + X38_C0ECCERRLOG); if (x38_channel_num == 2) - info->eccerrlog[1] = x38_readq(window + X38_C1ECCERRLOG); + info->eccerrlog[1] = lo_hi_readq(window + X38_C1ECCERRLOG); pci_read_config_word(pdev, X38_ERRSTS, &info->errsts2); @@ -196,10 +193,10 @@ static void x38_get_and_clear_error_info(struct mem_ctl_info *mci, * should be UE info. */ if ((info->errsts ^ info->errsts2) & X38_ERRSTS_BITS) { - info->eccerrlog[0] = x38_readq(window + X38_C0ECCERRLOG); + info->eccerrlog[0] = lo_hi_readq(window + X38_C0ECCERRLOG); if (x38_channel_num == 2) info->eccerrlog[1] = - x38_readq(window + X38_C1ECCERRLOG); + lo_hi_readq(window + X38_C1ECCERRLOG); } x38_clear_error_info(mci); diff --git a/drivers/i2c/i2c-core.c b/drivers/i2c/i2c-core.c index 7c7f4b856bad..66aa83b99383 100644 --- a/drivers/i2c/i2c-core.c +++ b/drivers/i2c/i2c-core.c @@ -42,6 +42,7 @@ #include <linux/of.h> #include <linux/of_device.h> #include <linux/of_irq.h> +#include <linux/clk/clk-conf.h> #include <linux/completion.h> #include <linux/hardirq.h> #include <linux/irqflags.h> @@ -274,6 +275,10 @@ static int i2c_device_probe(struct device *dev) client->flags & I2C_CLIENT_WAKE); dev_dbg(dev, "probe\n"); + status = of_clk_set_defaults(dev->of_node, false); + if (status < 0) + return status; + acpi_dev_pm_attach(&client->dev, true); status = driver->probe(client, i2c_match_id(driver->id_table, client)); if (status) diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig index bbb746e35500..7f0c2a30267b 100644 --- a/drivers/irqchip/Kconfig +++ b/drivers/irqchip/Kconfig @@ -10,6 +10,11 @@ config ARM_GIC config GIC_NON_BANKED bool +config ARM_GIC_V3 + bool + select IRQ_DOMAIN + select MULTI_IRQ_HANDLER + config ARM_NVIC bool select IRQ_DOMAIN diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile index 62a13e5ef98f..c57e642700d4 100644 --- a/drivers/irqchip/Makefile +++ b/drivers/irqchip/Makefile @@ -15,7 +15,8 @@ obj-$(CONFIG_ORION_IRQCHIP) += irq-orion.o obj-$(CONFIG_ARCH_SUNXI) += irq-sun4i.o obj-$(CONFIG_ARCH_SUNXI) += irq-sunxi-nmi.o obj-$(CONFIG_ARCH_SPEAR3XX) += spear-shirq.o -obj-$(CONFIG_ARM_GIC) += irq-gic.o +obj-$(CONFIG_ARM_GIC) += irq-gic.o irq-gic-common.o +obj-$(CONFIG_ARM_GIC_V3) += irq-gic-v3.o irq-gic-common.o obj-$(CONFIG_ARM_NVIC) += irq-nvic.o obj-$(CONFIG_ARM_VIC) += irq-vic.o obj-$(CONFIG_IMGPDC_IRQ) += irq-imgpdc.o diff --git a/drivers/irqchip/irq-gic-common.c b/drivers/irqchip/irq-gic-common.c new file mode 100644 index 000000000000..60ac704d2090 --- /dev/null +++ b/drivers/irqchip/irq-gic-common.c @@ -0,0 +1,115 @@ +/* + * Copyright (C) 2002 ARM Limited, All Rights Reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/irq.h> +#include <linux/irqchip/arm-gic.h> + +#include "irq-gic-common.h" + +void gic_configure_irq(unsigned int irq, unsigned int type, + void __iomem *base, void (*sync_access)(void)) +{ + u32 enablemask = 1 << (irq % 32); + u32 enableoff = (irq / 32) * 4; + u32 confmask = 0x2 << ((irq % 16) * 2); + u32 confoff = (irq / 16) * 4; + bool enabled = false; + u32 val; + + /* + * Read current configuration register, and insert the config + * for "irq", depending on "type". + */ + val = readl_relaxed(base + GIC_DIST_CONFIG + confoff); + if (type == IRQ_TYPE_LEVEL_HIGH) + val &= ~confmask; + else if (type == IRQ_TYPE_EDGE_RISING) + val |= confmask; + + /* + * As recommended by the spec, disable the interrupt before changing + * the configuration + */ + if (readl_relaxed(base + GIC_DIST_ENABLE_SET + enableoff) & enablemask) { + writel_relaxed(enablemask, base + GIC_DIST_ENABLE_CLEAR + enableoff); + if (sync_access) + sync_access(); + enabled = true; + } + + /* + * Write back the new configuration, and possibly re-enable + * the interrupt. + */ + writel_relaxed(val, base + GIC_DIST_CONFIG + confoff); + + if (enabled) + writel_relaxed(enablemask, base + GIC_DIST_ENABLE_SET + enableoff); + + if (sync_access) + sync_access(); +} + +void __init gic_dist_config(void __iomem *base, int gic_irqs, + void (*sync_access)(void)) +{ + unsigned int i; + + /* + * Set all global interrupts to be level triggered, active low. + */ + for (i = 32; i < gic_irqs; i += 16) + writel_relaxed(0, base + GIC_DIST_CONFIG + i / 4); + + /* + * Set priority on all global interrupts. + */ + for (i = 32; i < gic_irqs; i += 4) + writel_relaxed(0xa0a0a0a0, base + GIC_DIST_PRI + i); + + /* + * Disable all interrupts. Leave the PPI and SGIs alone + * as they are enabled by redistributor registers. + */ + for (i = 32; i < gic_irqs; i += 32) + writel_relaxed(0xffffffff, base + GIC_DIST_ENABLE_CLEAR + i / 8); + + if (sync_access) + sync_access(); +} + +void gic_cpu_config(void __iomem *base, void (*sync_access)(void)) +{ + int i; + + /* + * Deal with the banked PPI and SGI interrupts - disable all + * PPI interrupts, ensure all SGI interrupts are enabled. + */ + writel_relaxed(0xffff0000, base + GIC_DIST_ENABLE_CLEAR); + writel_relaxed(0x0000ffff, base + GIC_DIST_ENABLE_SET); + + /* + * Set priority on PPI and SGI interrupts + */ + for (i = 0; i < 32; i += 4) + writel_relaxed(0xa0a0a0a0, base + GIC_DIST_PRI + i * 4 / 4); + + if (sync_access) + sync_access(); +} diff --git a/drivers/irqchip/irq-gic-common.h b/drivers/irqchip/irq-gic-common.h new file mode 100644 index 000000000000..b41f02481c3a --- /dev/null +++ b/drivers/irqchip/irq-gic-common.h @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2002 ARM Limited, All Rights Reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#ifndef _IRQ_GIC_COMMON_H +#define _IRQ_GIC_COMMON_H + +#include <linux/of.h> +#include <linux/irqdomain.h> + +void gic_configure_irq(unsigned int irq, unsigned int type, + void __iomem *base, void (*sync_access)(void)); +void gic_dist_config(void __iomem *base, int gic_irqs, + void (*sync_access)(void)); +void gic_cpu_config(void __iomem *base, void (*sync_access)(void)); + +#endif /* _IRQ_GIC_COMMON_H */ diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c new file mode 100644 index 000000000000..57eaa5a0b1e3 --- /dev/null +++ b/drivers/irqchip/irq-gic-v3.c @@ -0,0 +1,692 @@ +/* + * Copyright (C) 2013, 2014 ARM Limited, All Rights Reserved. + * Author: Marc Zyngier <marc.zyngier@arm.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include <linux/cpu.h> +#include <linux/delay.h> +#include <linux/interrupt.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include <linux/of_irq.h> +#include <linux/percpu.h> +#include <linux/slab.h> + +#include <linux/irqchip/arm-gic-v3.h> + +#include <asm/cputype.h> +#include <asm/exception.h> +#include <asm/smp_plat.h> + +#include "irq-gic-common.h" +#include "irqchip.h" + +struct gic_chip_data { + void __iomem *dist_base; + void __iomem **redist_base; + void __percpu __iomem **rdist; + struct irq_domain *domain; + u64 redist_stride; + u32 redist_regions; + unsigned int irq_nr; +}; + +static struct gic_chip_data gic_data __read_mostly; + +#define gic_data_rdist() (this_cpu_ptr(gic_data.rdist)) +#define gic_data_rdist_rd_base() (*gic_data_rdist()) +#define gic_data_rdist_sgi_base() (gic_data_rdist_rd_base() + SZ_64K) + +/* Our default, arbitrary priority value. Linux only uses one anyway. */ +#define DEFAULT_PMR_VALUE 0xf0 + +static inline unsigned int gic_irq(struct irq_data *d) +{ + return d->hwirq; +} + +static inline int gic_irq_in_rdist(struct irq_data *d) +{ + return gic_irq(d) < 32; +} + +static inline void __iomem *gic_dist_base(struct irq_data *d) +{ + if (gic_irq_in_rdist(d)) /* SGI+PPI -> SGI_base for this CPU */ + return gic_data_rdist_sgi_base(); + + if (d->hwirq <= 1023) /* SPI -> dist_base */ + return gic_data.dist_base; + + if (d->hwirq >= 8192) + BUG(); /* LPI Detected!!! */ + + return NULL; +} + +static void gic_do_wait_for_rwp(void __iomem *base) +{ + u32 count = 1000000; /* 1s! */ + + while (readl_relaxed(base + GICD_CTLR) & GICD_CTLR_RWP) { + count--; + if (!count) { + pr_err_ratelimited("RWP timeout, gone fishing\n"); + return; + } + cpu_relax(); + udelay(1); + }; +} + +/* Wait for completion of a distributor change */ +static void gic_dist_wait_for_rwp(void) +{ + gic_do_wait_for_rwp(gic_data.dist_base); +} + +/* Wait for completion of a redistributor change */ +static void gic_redist_wait_for_rwp(void) +{ + gic_do_wait_for_rwp(gic_data_rdist_rd_base()); +} + +/* Low level accessors */ +static u64 gic_read_iar(void) +{ + u64 irqstat; + + asm volatile("mrs_s %0, " __stringify(ICC_IAR1_EL1) : "=r" (irqstat)); + return irqstat; +} + +static void gic_write_pmr(u64 val) +{ + asm volatile("msr_s " __stringify(ICC_PMR_EL1) ", %0" : : "r" (val)); +} + +static void gic_write_ctlr(u64 val) +{ + asm volatile("msr_s " __stringify(ICC_CTLR_EL1) ", %0" : : "r" (val)); + isb(); +} + +static void gic_write_grpen1(u64 val) +{ + asm volatile("msr_s " __stringify(ICC_GRPEN1_EL1) ", %0" : : "r" (val)); + isb(); +} + +static void gic_write_sgi1r(u64 val) +{ + asm volatile("msr_s " __stringify(ICC_SGI1R_EL1) ", %0" : : "r" (val)); +} + +static void gic_enable_sre(void) +{ + u64 val; + + asm volatile("mrs_s %0, " __stringify(ICC_SRE_EL1) : "=r" (val)); + val |= ICC_SRE_EL1_SRE; + asm volatile("msr_s " __stringify(ICC_SRE_EL1) ", %0" : : "r" (val)); + isb(); + + /* + * Need to check that the SRE bit has actually been set. If + * not, it means that SRE is disabled at EL2. We're going to + * die painfully, and there is nothing we can do about it. + * + * Kindly inform the luser. + */ + asm volatile("mrs_s %0, " __stringify(ICC_SRE_EL1) : "=r" (val)); + if (!(val & ICC_SRE_EL1_SRE)) + pr_err("GIC: unable to set SRE (disabled at EL2), panic ahead\n"); +} + +static void gic_enable_redist(void) +{ + void __iomem *rbase; + u32 count = 1000000; /* 1s! */ + u32 val; + + rbase = gic_data_rdist_rd_base(); + + /* Wake up this CPU redistributor */ + val = readl_relaxed(rbase + GICR_WAKER); + val &= ~GICR_WAKER_ProcessorSleep; + writel_relaxed(val, rbase + GICR_WAKER); + + while (readl_relaxed(rbase + GICR_WAKER) & GICR_WAKER_ChildrenAsleep) { + count--; + if (!count) { + pr_err_ratelimited("redist didn't wake up...\n"); + return; + } + cpu_relax(); + udelay(1); + }; +} + +/* + * Routines to disable, enable, EOI and route interrupts + */ +static void gic_poke_irq(struct irq_data *d, u32 offset) +{ + u32 mask = 1 << (gic_irq(d) % 32); + void (*rwp_wait)(void); + void __iomem *base; + + if (gic_irq_in_rdist(d)) { + base = gic_data_rdist_sgi_base(); + rwp_wait = gic_redist_wait_for_rwp; + } else { + base = gic_data.dist_base; + rwp_wait = gic_dist_wait_for_rwp; + } + + writel_relaxed(mask, base + offset + (gic_irq(d) / 32) * 4); + rwp_wait(); +} + +static int gic_peek_irq(struct irq_data *d, u32 offset) +{ + u32 mask = 1 << (gic_irq(d) % 32); + void __iomem *base; + + if (gic_irq_in_rdist(d)) + base = gic_data_rdist_sgi_base(); + else + base = gic_data.dist_base; + + return !!(readl_relaxed(base + offset + (gic_irq(d) / 32) * 4) & mask); +} + +static void gic_mask_irq(struct irq_data *d) +{ + gic_poke_irq(d, GICD_ICENABLER); +} + +static void gic_unmask_irq(struct irq_data *d) +{ + gic_poke_irq(d, GICD_ISENABLER); +} + +static void gic_eoi_irq(struct irq_data *d) +{ + gic_write_eoir(gic_irq(d)); +} + +static int gic_set_type(struct irq_data *d, unsigned int type) +{ + unsigned int irq = gic_irq(d); + void (*rwp_wait)(void); + void __iomem *base; + + /* Interrupt configuration for SGIs can't be changed */ + if (irq < 16) + return -EINVAL; + + if (type != IRQ_TYPE_LEVEL_HIGH && type != IRQ_TYPE_EDGE_RISING) + return -EINVAL; + + if (gic_irq_in_rdist(d)) { + base = gic_data_rdist_sgi_base(); + rwp_wait = gic_redist_wait_for_rwp; + } else { + base = gic_data.dist_base; + rwp_wait = gic_dist_wait_for_rwp; + } + + gic_configure_irq(irq, type, base, rwp_wait); + + return 0; +} + +static u64 gic_mpidr_to_affinity(u64 mpidr) +{ + u64 aff; + + aff = (MPIDR_AFFINITY_LEVEL(mpidr, 3) << 32 | + MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 | + MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 | + MPIDR_AFFINITY_LEVEL(mpidr, 0)); + + return aff; +} + +static asmlinkage void __exception_irq_entry gic_handle_irq(struct pt_regs *regs) +{ + u64 irqnr; + + do { + irqnr = gic_read_iar(); + + if (likely(irqnr > 15 && irqnr < 1020)) { + u64 irq = irq_find_mapping(gic_data.domain, irqnr); + if (likely(irq)) { + handle_IRQ(irq, regs); + continue; + } + + WARN_ONCE(true, "Unexpected SPI received!\n"); + gic_write_eoir(irqnr); + } + if (irqnr < 16) { + gic_write_eoir(irqnr); +#ifdef CONFIG_SMP + handle_IPI(irqnr, regs); +#else + WARN_ONCE(true, "Unexpected SGI received!\n"); +#endif + continue; + } + } while (irqnr != ICC_IAR1_EL1_SPURIOUS); +} + +static void __init gic_dist_init(void) +{ + unsigned int i; + u64 affinity; + void __iomem *base = gic_data.dist_base; + + /* Disable the distributor */ + writel_relaxed(0, base + GICD_CTLR); + gic_dist_wait_for_rwp(); + + gic_dist_config(base, gic_data.irq_nr, gic_dist_wait_for_rwp); + + /* Enable distributor with ARE, Group1 */ + writel_relaxed(GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1A | GICD_CTLR_ENABLE_G1, + base + GICD_CTLR); + + /* + * Set all global interrupts to the boot CPU only. ARE must be + * enabled. + */ + affinity = gic_mpidr_to_affinity(cpu_logical_map(smp_processor_id())); + for (i = 32; i < gic_data.irq_nr; i++) + writeq_relaxed(affinity, base + GICD_IROUTER + i * 8); +} + +static int gic_populate_rdist(void) +{ + u64 mpidr = cpu_logical_map(smp_processor_id()); + u64 typer; + u32 aff; + int i; + + /* + * Convert affinity to a 32bit value that can be matched to + * GICR_TYPER bits [63:32]. + */ + aff = (MPIDR_AFFINITY_LEVEL(mpidr, 3) << 24 | + MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 | + MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 | + MPIDR_AFFINITY_LEVEL(mpidr, 0)); + + for (i = 0; i < gic_data.redist_regions; i++) { + void __iomem *ptr = gic_data.redist_base[i]; + u32 reg; + + reg = readl_relaxed(ptr + GICR_PIDR2) & GIC_PIDR2_ARCH_MASK; + if (reg != GIC_PIDR2_ARCH_GICv3 && + reg != GIC_PIDR2_ARCH_GICv4) { /* We're in trouble... */ + pr_warn("No redistributor present @%p\n", ptr); + break; + } + + do { + typer = readq_relaxed(ptr + GICR_TYPER); + if ((typer >> 32) == aff) { + gic_data_rdist_rd_base() = ptr; + pr_info("CPU%d: found redistributor %llx @%p\n", + smp_processor_id(), + (unsigned long long)mpidr, ptr); + return 0; + } + + if (gic_data.redist_stride) { + ptr += gic_data.redist_stride; + } else { + ptr += SZ_64K * 2; /* Skip RD_base + SGI_base */ + if (typer & GICR_TYPER_VLPIS) + ptr += SZ_64K * 2; /* Skip VLPI_base + reserved page */ + } + } while (!(typer & GICR_TYPER_LAST)); + } + + /* We couldn't even deal with ourselves... */ + WARN(true, "CPU%d: mpidr %llx has no re-distributor!\n", + smp_processor_id(), (unsigned long long)mpidr); + return -ENODEV; +} + +static void gic_cpu_init(void) +{ + void __iomem *rbase; + + /* Register ourselves with the rest of the world */ + if (gic_populate_rdist()) + return; + + gic_enable_redist(); + + rbase = gic_data_rdist_sgi_base(); + + gic_cpu_config(rbase, gic_redist_wait_for_rwp); + + /* Enable system registers */ + gic_enable_sre(); + + /* Set priority mask register */ + gic_write_pmr(DEFAULT_PMR_VALUE); + + /* EOI deactivates interrupt too (mode 0) */ + gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop_dir); + + /* ... and let's hit the road... */ + gic_write_grpen1(1); +} + +#ifdef CONFIG_SMP +static int gic_secondary_init(struct notifier_block *nfb, + unsigned long action, void *hcpu) +{ + if (action == CPU_STARTING || action == CPU_STARTING_FROZEN) + gic_cpu_init(); + return NOTIFY_OK; +} + +/* + * Notifier for enabling the GIC CPU interface. Set an arbitrarily high + * priority because the GIC needs to be up before the ARM generic timers. + */ +static struct notifier_block gic_cpu_notifier = { + .notifier_call = gic_secondary_init, + .priority = 100, +}; + +static u16 gic_compute_target_list(int *base_cpu, const struct cpumask *mask, + u64 cluster_id) +{ + int cpu = *base_cpu; + u64 mpidr = cpu_logical_map(cpu); + u16 tlist = 0; + + while (cpu < nr_cpu_ids) { + /* + * If we ever get a cluster of more than 16 CPUs, just + * scream and skip that CPU. + */ + if (WARN_ON((mpidr & 0xff) >= 16)) + goto out; + + tlist |= 1 << (mpidr & 0xf); + + cpu = cpumask_next(cpu, mask); + if (cpu == nr_cpu_ids) + goto out; + + mpidr = cpu_logical_map(cpu); + + if (cluster_id != (mpidr & ~0xffUL)) { + cpu--; + goto out; + } + } +out: + *base_cpu = cpu; + return tlist; +} + +static void gic_send_sgi(u64 cluster_id, u16 tlist, unsigned int irq) +{ + u64 val; + + val = (MPIDR_AFFINITY_LEVEL(cluster_id, 3) << 48 | + MPIDR_AFFINITY_LEVEL(cluster_id, 2) << 32 | + irq << 24 | + MPIDR_AFFINITY_LEVEL(cluster_id, 1) << 16 | + tlist); + + pr_debug("CPU%d: ICC_SGI1R_EL1 %llx\n", smp_processor_id(), val); + gic_write_sgi1r(val); +} + +static void gic_raise_softirq(const struct cpumask *mask, unsigned int irq) +{ + int cpu; + + if (WARN_ON(irq >= 16)) + return; + + /* + * Ensure that stores to Normal memory are visible to the + * other CPUs before issuing the IPI. + */ + smp_wmb(); + + for_each_cpu_mask(cpu, *mask) { + u64 cluster_id = cpu_logical_map(cpu) & ~0xffUL; + u16 tlist; + + tlist = gic_compute_target_list(&cpu, mask, cluster_id); + gic_send_sgi(cluster_id, tlist, irq); + } + + /* Force the above writes to ICC_SGI1R_EL1 to be executed */ + isb(); +} + +static void gic_smp_init(void) +{ + set_smp_cross_call(gic_raise_softirq); + register_cpu_notifier(&gic_cpu_notifier); +} + +static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val, + bool force) +{ + unsigned int cpu = cpumask_any_and(mask_val, cpu_online_mask); + void __iomem *reg; + int enabled; + u64 val; + + if (gic_irq_in_rdist(d)) + return -EINVAL; + + /* If interrupt was enabled, disable it first */ + enabled = gic_peek_irq(d, GICD_ISENABLER); + if (enabled) + gic_mask_irq(d); + + reg = gic_dist_base(d) + GICD_IROUTER + (gic_irq(d) * 8); + val = gic_mpidr_to_affinity(cpu_logical_map(cpu)); + + writeq_relaxed(val, reg); + + /* + * If the interrupt was enabled, enabled it again. Otherwise, + * just wait for the distributor to have digested our changes. + */ + if (enabled) + gic_unmask_irq(d); + else + gic_dist_wait_for_rwp(); + + return IRQ_SET_MASK_OK; +} +#else +#define gic_set_affinity NULL +#define gic_smp_init() do { } while(0) +#endif + +static struct irq_chip gic_chip = { + .name = "GICv3", + .irq_mask = gic_mask_irq, + .irq_unmask = gic_unmask_irq, + .irq_eoi = gic_eoi_irq, + .irq_set_type = gic_set_type, + .irq_set_affinity = gic_set_affinity, +}; + +static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq, + irq_hw_number_t hw) +{ + /* SGIs are private to the core kernel */ + if (hw < 16) + return -EPERM; + /* PPIs */ + if (hw < 32) { + irq_set_percpu_devid(irq); + irq_set_chip_and_handler(irq, &gic_chip, + handle_percpu_devid_irq); + set_irq_flags(irq, IRQF_VALID | IRQF_NOAUTOEN); + } + /* SPIs */ + if (hw >= 32 && hw < gic_data.irq_nr) { + irq_set_chip_and_handler(irq, &gic_chip, + handle_fasteoi_irq); + set_irq_flags(irq, IRQF_VALID | IRQF_PROBE); + } + irq_set_chip_data(irq, d->host_data); + return 0; +} + +static int gic_irq_domain_xlate(struct irq_domain *d, + struct device_node *controller, + const u32 *intspec, unsigned int intsize, + unsigned long *out_hwirq, unsigned int *out_type) +{ + if (d->of_node != controller) + return -EINVAL; + if (intsize < 3) + return -EINVAL; + + switch(intspec[0]) { + case 0: /* SPI */ + *out_hwirq = intspec[1] + 32; + break; + case 1: /* PPI */ + *out_hwirq = intspec[1] + 16; + break; + default: + return -EINVAL; + } + + *out_type = intspec[2] & IRQ_TYPE_SENSE_MASK; + return 0; +} + +static const struct irq_domain_ops gic_irq_domain_ops = { + .map = gic_irq_domain_map, + .xlate = gic_irq_domain_xlate, +}; + +static int __init gic_of_init(struct device_node *node, struct device_node *parent) +{ + void __iomem *dist_base; + void __iomem **redist_base; + u64 redist_stride; + u32 redist_regions; + u32 reg; + int gic_irqs; + int err; + int i; + + dist_base = of_iomap(node, 0); + if (!dist_base) { + pr_err("%s: unable to map gic dist registers\n", + node->full_name); + return -ENXIO; + } + + reg = readl_relaxed(dist_base + GICD_PIDR2) & GIC_PIDR2_ARCH_MASK; + if (reg != GIC_PIDR2_ARCH_GICv3 && reg != GIC_PIDR2_ARCH_GICv4) { + pr_err("%s: no distributor detected, giving up\n", + node->full_name); + err = -ENODEV; + goto out_unmap_dist; + } + + if (of_property_read_u32(node, "#redistributor-regions", &redist_regions)) + redist_regions = 1; + + redist_base = kzalloc(sizeof(*redist_base) * redist_regions, GFP_KERNEL); + if (!redist_base) { + err = -ENOMEM; + goto out_unmap_dist; + } + + for (i = 0; i < redist_regions; i++) { + redist_base[i] = of_iomap(node, 1 + i); + if (!redist_base[i]) { + pr_err("%s: couldn't map region %d\n", + node->full_name, i); + err = -ENODEV; + goto out_unmap_rdist; + } + } + + if (of_property_read_u64(node, "redistributor-stride", &redist_stride)) + redist_stride = 0; + + gic_data.dist_base = dist_base; + gic_data.redist_base = redist_base; + gic_data.redist_regions = redist_regions; + gic_data.redist_stride = redist_stride; + + /* + * Find out how many interrupts are supported. + * The GIC only supports up to 1020 interrupt sources (SGI+PPI+SPI) + */ + gic_irqs = readl_relaxed(gic_data.dist_base + GICD_TYPER) & 0x1f; + gic_irqs = (gic_irqs + 1) * 32; + if (gic_irqs > 1020) + gic_irqs = 1020; + gic_data.irq_nr = gic_irqs; + + gic_data.domain = irq_domain_add_tree(node, &gic_irq_domain_ops, + &gic_data); + gic_data.rdist = alloc_percpu(typeof(*gic_data.rdist)); + + if (WARN_ON(!gic_data.domain) || WARN_ON(!gic_data.rdist)) { + err = -ENOMEM; + goto out_free; + } + + set_handle_irq(gic_handle_irq); + + gic_smp_init(); + gic_dist_init(); + gic_cpu_init(); + + return 0; + +out_free: + if (gic_data.domain) + irq_domain_remove(gic_data.domain); + free_percpu(gic_data.rdist); +out_unmap_rdist: + for (i = 0; i < redist_regions; i++) + if (redist_base[i]) + iounmap(redist_base[i]); + kfree(redist_base); +out_unmap_dist: + iounmap(dist_base); + return err; +} + +IRQCHIP_DECLARE(gic_v3, "arm,gic-v3", gic_of_init); diff --git a/drivers/irqchip/irq-gic.c b/drivers/irqchip/irq-gic.c index 7c131cf7cc13..9c1f883fc5a3 100644 --- a/drivers/irqchip/irq-gic.c +++ b/drivers/irqchip/irq-gic.c @@ -47,6 +47,7 @@ #include <asm/exception.h> #include <asm/smp_plat.h> +#include "irq-gic-common.h" #include "irqchip.h" union gic_base { @@ -189,12 +190,6 @@ static int gic_set_type(struct irq_data *d, unsigned int type) { void __iomem *base = gic_dist_base(d); unsigned int gicirq = gic_irq(d); - u32 enablemask = 1 << (gicirq % 32); - u32 enableoff = (gicirq / 32) * 4; - u32 confmask = 0x2 << ((gicirq % 16) * 2); - u32 confoff = (gicirq / 16) * 4; - bool enabled = false; - u32 val; /* Interrupt configuration for SGIs can't be changed */ if (gicirq < 16) @@ -208,25 +203,7 @@ static int gic_set_type(struct irq_data *d, unsigned int type) if (gic_arch_extn.irq_set_type) gic_arch_extn.irq_set_type(d, type); - val = readl_relaxed(base + GIC_DIST_CONFIG + confoff); - if (type == IRQ_TYPE_LEVEL_HIGH) - val &= ~confmask; - else if (type == IRQ_TYPE_EDGE_RISING) - val |= confmask; - - /* - * As recommended by the spec, disable the interrupt before changing - * the configuration - */ - if (readl_relaxed(base + GIC_DIST_ENABLE_SET + enableoff) & enablemask) { - writel_relaxed(enablemask, base + GIC_DIST_ENABLE_CLEAR + enableoff); - enabled = true; - } - - writel_relaxed(val, base + GIC_DIST_CONFIG + confoff); - - if (enabled) - writel_relaxed(enablemask, base + GIC_DIST_ENABLE_SET + enableoff); + gic_configure_irq(gicirq, type, base, NULL); raw_spin_unlock(&irq_controller_lock); @@ -388,12 +365,6 @@ static void __init gic_dist_init(struct gic_chip_data *gic) writel_relaxed(0, base + GIC_DIST_CTRL); /* - * Set all global interrupts to be level triggered, active low. - */ - for (i = 32; i < gic_irqs; i += 16) - writel_relaxed(0, base + GIC_DIST_CONFIG + i * 4 / 16); - - /* * Set all global interrupts to this CPU only. */ cpumask = gic_get_cpumask(gic); @@ -402,18 +373,7 @@ static void __init gic_dist_init(struct gic_chip_data *gic) for (i = 32; i < gic_irqs; i += 4) writel_relaxed(cpumask, base + GIC_DIST_TARGET + i * 4 / 4); - /* - * Set priority on all global interrupts. - */ - for (i = 32; i < gic_irqs; i += 4) - writel_relaxed(0xa0a0a0a0, base + GIC_DIST_PRI + i * 4 / 4); - - /* - * Disable all interrupts. Leave the PPI and SGIs alone - * as these enables are banked registers. - */ - for (i = 32; i < gic_irqs; i += 32) - writel_relaxed(0xffffffff, base + GIC_DIST_ENABLE_CLEAR + i * 4 / 32); + gic_dist_config(base, gic_irqs, NULL); writel_relaxed(1, base + GIC_DIST_CTRL); } @@ -440,18 +400,7 @@ static void gic_cpu_init(struct gic_chip_data *gic) if (i != cpu) gic_cpu_map[i] &= ~cpu_mask; - /* - * Deal with the banked PPI and SGI interrupts - disable all - * PPI interrupts, ensure all SGI interrupts are enabled. - */ - writel_relaxed(0xffff0000, dist_base + GIC_DIST_ENABLE_CLEAR); - writel_relaxed(0x0000ffff, dist_base + GIC_DIST_ENABLE_SET); - - /* - * Set priority on PPI and SGI interrupts - */ - for (i = 0; i < 32; i += 4) - writel_relaxed(0xa0a0a0a0, dist_base + GIC_DIST_PRI + i * 4 / 4); + gic_cpu_config(dist_base, NULL); writel_relaxed(0xf0, base + GIC_CPU_PRIMASK); writel_relaxed(1, base + GIC_CPU_CTRL); diff --git a/drivers/md/dm-bufio.c b/drivers/md/dm-bufio.c index d724459860d9..ab472c557d18 100644 --- a/drivers/md/dm-bufio.c +++ b/drivers/md/dm-bufio.c @@ -615,16 +615,6 @@ static void write_endio(struct bio *bio, int error) } /* - * This function is called when wait_on_bit is actually waiting. - */ -static int do_io_schedule(void *word) -{ - io_schedule(); - - return 0; -} - -/* * Initiate a write on a dirty buffer, but don't wait for it. * * - If the buffer is not dirty, exit. @@ -640,8 +630,7 @@ static void __write_dirty_buffer(struct dm_buffer *b, return; clear_bit(B_DIRTY, &b->state); - wait_on_bit_lock(&b->state, B_WRITING, - do_io_schedule, TASK_UNINTERRUPTIBLE); + wait_on_bit_lock_io(&b->state, B_WRITING, TASK_UNINTERRUPTIBLE); if (!write_list) submit_io(b, WRITE, b->block, write_endio); @@ -675,9 +664,9 @@ static void __make_buffer_clean(struct dm_buffer *b) if (!b->state) /* fast case */ return; - wait_on_bit(&b->state, B_READING, do_io_schedule, TASK_UNINTERRUPTIBLE); + wait_on_bit_io(&b->state, B_READING, TASK_UNINTERRUPTIBLE); __write_dirty_buffer(b, NULL); - wait_on_bit(&b->state, B_WRITING, do_io_schedule, TASK_UNINTERRUPTIBLE); + wait_on_bit_io(&b->state, B_WRITING, TASK_UNINTERRUPTIBLE); } /* @@ -1030,7 +1019,7 @@ static void *new_read(struct dm_bufio_client *c, sector_t block, if (need_submit) submit_io(b, READ, b->block, read_endio); - wait_on_bit(&b->state, B_READING, do_io_schedule, TASK_UNINTERRUPTIBLE); + wait_on_bit_io(&b->state, B_READING, TASK_UNINTERRUPTIBLE); if (b->read_error) { int error = b->read_error; @@ -1209,15 +1198,13 @@ again: dropped_lock = 1; b->hold_count++; dm_bufio_unlock(c); - wait_on_bit(&b->state, B_WRITING, - do_io_schedule, - TASK_UNINTERRUPTIBLE); + wait_on_bit_io(&b->state, B_WRITING, + TASK_UNINTERRUPTIBLE); dm_bufio_lock(c); b->hold_count--; } else - wait_on_bit(&b->state, B_WRITING, - do_io_schedule, - TASK_UNINTERRUPTIBLE); + wait_on_bit_io(&b->state, B_WRITING, + TASK_UNINTERRUPTIBLE); } if (!test_bit(B_DIRTY, &b->state) && @@ -1321,15 +1308,15 @@ retry: __write_dirty_buffer(b, NULL); if (b->hold_count == 1) { - wait_on_bit(&b->state, B_WRITING, - do_io_schedule, TASK_UNINTERRUPTIBLE); + wait_on_bit_io(&b->state, B_WRITING, + TASK_UNINTERRUPTIBLE); set_bit(B_DIRTY, &b->state); __unlink_buffer(b); __link_buffer(b, new_block, LIST_DIRTY); } else { sector_t old_block; - wait_on_bit_lock(&b->state, B_WRITING, - do_io_schedule, TASK_UNINTERRUPTIBLE); + wait_on_bit_lock_io(&b->state, B_WRITING, + TASK_UNINTERRUPTIBLE); /* * Relink buffer to "new_block" so that write_callback * sees "new_block" as a block number. @@ -1341,8 +1328,8 @@ retry: __unlink_buffer(b); __link_buffer(b, new_block, b->list_mode); submit_io(b, WRITE, new_block, write_endio); - wait_on_bit(&b->state, B_WRITING, - do_io_schedule, TASK_UNINTERRUPTIBLE); + wait_on_bit_io(&b->state, B_WRITING, + TASK_UNINTERRUPTIBLE); __unlink_buffer(b); __link_buffer(b, old_block, b->list_mode); } diff --git a/drivers/md/dm-snap.c b/drivers/md/dm-snap.c index 5bd2290cfb1e..864b03f47727 100644 --- a/drivers/md/dm-snap.c +++ b/drivers/md/dm-snap.c @@ -1032,21 +1032,13 @@ static void start_merge(struct dm_snapshot *s) snapshot_merge_next_chunks(s); } -static int wait_schedule(void *ptr) -{ - schedule(); - - return 0; -} - /* * Stop the merging process and wait until it finishes. */ static void stop_merge(struct dm_snapshot *s) { set_bit(SHUTDOWN_MERGE, &s->state_bits); - wait_on_bit(&s->state_bits, RUNNING_MERGE, wait_schedule, - TASK_UNINTERRUPTIBLE); + wait_on_bit(&s->state_bits, RUNNING_MERGE, TASK_UNINTERRUPTIBLE); clear_bit(SHUTDOWN_MERGE, &s->state_bits); } diff --git a/drivers/media/usb/dvb-usb-v2/dvb_usb_core.c b/drivers/media/usb/dvb-usb-v2/dvb_usb_core.c index e35580618936..f296394bb7c5 100644 --- a/drivers/media/usb/dvb-usb-v2/dvb_usb_core.c +++ b/drivers/media/usb/dvb-usb-v2/dvb_usb_core.c @@ -253,13 +253,6 @@ static int dvb_usbv2_adapter_stream_exit(struct dvb_usb_adapter *adap) return usb_urb_exitv2(&adap->stream); } -static int wait_schedule(void *ptr) -{ - schedule(); - - return 0; -} - static int dvb_usb_start_feed(struct dvb_demux_feed *dvbdmxfeed) { struct dvb_usb_adapter *adap = dvbdmxfeed->demux->priv; @@ -273,8 +266,7 @@ static int dvb_usb_start_feed(struct dvb_demux_feed *dvbdmxfeed) dvbdmxfeed->pid, dvbdmxfeed->index); /* wait init is done */ - wait_on_bit(&adap->state_bits, ADAP_INIT, wait_schedule, - TASK_UNINTERRUPTIBLE); + wait_on_bit(&adap->state_bits, ADAP_INIT, TASK_UNINTERRUPTIBLE); if (adap->active_fe == -1) return -EINVAL; @@ -568,7 +560,7 @@ static int dvb_usb_fe_sleep(struct dvb_frontend *fe) if (!adap->suspend_resume_active) { set_bit(ADAP_SLEEP, &adap->state_bits); - wait_on_bit(&adap->state_bits, ADAP_STREAMING, wait_schedule, + wait_on_bit(&adap->state_bits, ADAP_STREAMING, TASK_UNINTERRUPTIBLE); } diff --git a/drivers/net/Makefile b/drivers/net/Makefile index 3fef8a81c0f6..331d9cfa6193 100644 --- a/drivers/net/Makefile +++ b/drivers/net/Makefile @@ -65,6 +65,7 @@ obj-$(CONFIG_USB_CATC) += usb/ obj-$(CONFIG_USB_KAWETH) += usb/ obj-$(CONFIG_USB_PEGASUS) += usb/ obj-$(CONFIG_USB_RTL8150) += usb/ +obj-$(CONFIG_USB_RTL8152) += usb/ obj-$(CONFIG_USB_HSO) += usb/ obj-$(CONFIG_USB_USBNET) += usb/ obj-$(CONFIG_USB_ZD1201) += usb/ diff --git a/drivers/net/bonding/bond_sysfs_slave.c b/drivers/net/bonding/bond_sysfs_slave.c index 198677f58ce0..5cd532ca1cfe 100644 --- a/drivers/net/bonding/bond_sysfs_slave.c +++ b/drivers/net/bonding/bond_sysfs_slave.c @@ -125,7 +125,7 @@ int bond_sysfs_slave_add(struct slave *slave) for (a = slave_attrs; *a; ++a) { err = sysfs_create_file(&slave->kobj, &((*a)->attr)); if (err) { - kobject_del(&slave->kobj); + kobject_put(&slave->kobj); return err; } } @@ -140,5 +140,5 @@ void bond_sysfs_slave_del(struct slave *slave) for (a = slave_attrs; *a; ++a) sysfs_remove_file(&slave->kobj, &((*a)->attr)); - kobject_del(&slave->kobj); + kobject_put(&slave->kobj); } diff --git a/drivers/net/ethernet/brocade/bna/bnad.c b/drivers/net/ethernet/brocade/bna/bnad.c index 3a77f9ead004..556aab75f490 100644 --- a/drivers/net/ethernet/brocade/bna/bnad.c +++ b/drivers/net/ethernet/brocade/bna/bnad.c @@ -600,9 +600,9 @@ bnad_cq_process(struct bnad *bnad, struct bna_ccb *ccb, int budget) prefetch(bnad->netdev); cq = ccb->sw_q; - cmpl = &cq[ccb->producer_index]; while (packets < budget) { + cmpl = &cq[ccb->producer_index]; if (!cmpl->valid) break; /* The 'valid' field is set by the adapter, only after writing diff --git a/drivers/net/ethernet/brocade/bna/bnad_ethtool.c b/drivers/net/ethernet/brocade/bna/bnad_ethtool.c index 882cad71ad62..d26adac6ab99 100644 --- a/drivers/net/ethernet/brocade/bna/bnad_ethtool.c +++ b/drivers/net/ethernet/brocade/bna/bnad_ethtool.c @@ -997,10 +997,8 @@ bnad_get_eeprom(struct net_device *netdev, struct ethtool_eeprom *eeprom, unsigned long flags = 0; int ret = 0; - /* Check if the flash read request is valid */ - if (eeprom->magic != (bnad->pcidev->vendor | - (bnad->pcidev->device << 16))) - return -EFAULT; + /* Fill the magic value */ + eeprom->magic = bnad->pcidev->vendor | (bnad->pcidev->device << 16); /* Query the flash partition based on the offset */ flash_part = bnad_get_flash_partition_by_offset(bnad, diff --git a/drivers/net/macvlan.c b/drivers/net/macvlan.c index 958df383068a..ef8a5c20236a 100644 --- a/drivers/net/macvlan.c +++ b/drivers/net/macvlan.c @@ -646,6 +646,7 @@ static int macvlan_init(struct net_device *dev) (lowerdev->state & MACVLAN_STATE_MASK); dev->features = lowerdev->features & MACVLAN_FEATURES; dev->features |= ALWAYS_ON_FEATURES; + dev->vlan_features = lowerdev->vlan_features & MACVLAN_FEATURES; dev->gso_max_size = lowerdev->gso_max_size; dev->iflink = lowerdev->ifindex; dev->hard_header_len = lowerdev->hard_header_len; diff --git a/drivers/net/usb/cdc_subset.c b/drivers/net/usb/cdc_subset.c index 91f0919fe278..6ea98cff2d3b 100644 --- a/drivers/net/usb/cdc_subset.c +++ b/drivers/net/usb/cdc_subset.c @@ -85,14 +85,28 @@ static int always_connected (struct usbnet *dev) * *-------------------------------------------------------------------------*/ +static void m5632_recover(struct usbnet *dev) +{ + struct usb_device *udev = dev->udev; + struct usb_interface *intf = dev->intf; + int r; + + r = usb_lock_device_for_reset(udev, intf); + if (r < 0) + return; + + usb_reset_device(udev); + usb_unlock_device(udev); +} + static const struct driver_info ali_m5632_info = { .description = "ALi M5632", .flags = FLAG_POINTTOPOINT, + .recover = m5632_recover, }; #endif - #ifdef CONFIG_USB_AN2720 #define HAVE_HARDWARE @@ -326,12 +340,23 @@ static const struct usb_device_id products [] = { MODULE_DEVICE_TABLE(usb, products); /*-------------------------------------------------------------------------*/ +static int dummy_prereset(struct usb_interface *intf) +{ + return 0; +} + +static int dummy_postreset(struct usb_interface *intf) +{ + return 0; +} static struct usb_driver cdc_subset_driver = { .name = "cdc_subset", .probe = usbnet_probe, .suspend = usbnet_suspend, .resume = usbnet_resume, + .pre_reset = dummy_prereset, + .post_reset = dummy_postreset, .disconnect = usbnet_disconnect, .id_table = products, .disable_hub_initiated_lpm = 1, diff --git a/drivers/net/usb/usbnet.c b/drivers/net/usb/usbnet.c index f9e96c427558..5173821a9575 100644 --- a/drivers/net/usb/usbnet.c +++ b/drivers/net/usb/usbnet.c @@ -1218,8 +1218,12 @@ void usbnet_tx_timeout (struct net_device *net) unlink_urbs (dev, &dev->txq); tasklet_schedule (&dev->bh); - - // FIXME: device recovery -- reset? + /* this needs to be handled individually because the generic layer + * doesn't know what is sufficient and could not restore private + * information if a remedy of an unconditional reset were used. + */ + if (dev->driver_info->recover) + (dev->driver_info->recover)(dev); } EXPORT_SYMBOL_GPL(usbnet_tx_timeout); diff --git a/drivers/net/xen-netfront.c b/drivers/net/xen-netfront.c index 055222bae6e4..28204bc4f369 100644 --- a/drivers/net/xen-netfront.c +++ b/drivers/net/xen-netfront.c @@ -1196,22 +1196,6 @@ static void xennet_release_rx_bufs(struct netfront_queue *queue) spin_unlock_bh(&queue->rx_lock); } -static void xennet_uninit(struct net_device *dev) -{ - struct netfront_info *np = netdev_priv(dev); - unsigned int num_queues = dev->real_num_tx_queues; - struct netfront_queue *queue; - unsigned int i; - - for (i = 0; i < num_queues; ++i) { - queue = &np->queues[i]; - xennet_release_tx_bufs(queue); - xennet_release_rx_bufs(queue); - gnttab_free_grant_references(queue->gref_tx_head); - gnttab_free_grant_references(queue->gref_rx_head); - } -} - static netdev_features_t xennet_fix_features(struct net_device *dev, netdev_features_t features) { @@ -1313,7 +1297,6 @@ static void xennet_poll_controller(struct net_device *dev) static const struct net_device_ops xennet_netdev_ops = { .ndo_open = xennet_open, - .ndo_uninit = xennet_uninit, .ndo_stop = xennet_close, .ndo_start_xmit = xennet_start_xmit, .ndo_change_mtu = xennet_change_mtu, @@ -1455,6 +1438,11 @@ static void xennet_disconnect_backend(struct netfront_info *info) napi_synchronize(&queue->napi); + xennet_release_tx_bufs(queue); + xennet_release_rx_bufs(queue); + gnttab_free_grant_references(queue->gref_tx_head); + gnttab_free_grant_references(queue->gref_rx_head); + /* End access and free the pages */ xennet_end_access(queue->tx_ring_ref, queue->tx.sring); xennet_end_access(queue->rx_ring_ref, queue->rx.sring); @@ -1827,8 +1815,8 @@ static int xennet_create_queues(struct netfront_info *info, ret = xennet_init_queue(queue); if (ret < 0) { - dev_warn(&info->netdev->dev, "only created %d queues\n", - num_queues); + dev_warn(&info->netdev->dev, + "only created %d queues\n", i); num_queues = i; break; } @@ -2001,7 +1989,7 @@ abort_transaction_no_dev_fatal: info->queues = NULL; rtnl_lock(); netif_set_real_num_tx_queues(info->netdev, 0); - rtnl_lock(); + rtnl_unlock(); out: return err; } @@ -2010,10 +1998,7 @@ static int xennet_connect(struct net_device *dev) { struct netfront_info *np = netdev_priv(dev); unsigned int num_queues = 0; - int i, requeue_idx, err; - struct sk_buff *skb; - grant_ref_t ref; - struct xen_netif_rx_request *req; + int err; unsigned int feature_rx_copy; unsigned int j = 0; struct netfront_queue *queue = NULL; @@ -2040,47 +2025,8 @@ static int xennet_connect(struct net_device *dev) netdev_update_features(dev); rtnl_unlock(); - /* By now, the queue structures have been set up */ - for (j = 0; j < num_queues; ++j) { - queue = &np->queues[j]; - - /* Step 1: Discard all pending TX packet fragments. */ - spin_lock_irq(&queue->tx_lock); - xennet_release_tx_bufs(queue); - spin_unlock_irq(&queue->tx_lock); - - /* Step 2: Rebuild the RX buffer freelist and the RX ring itself. */ - spin_lock_bh(&queue->rx_lock); - - for (requeue_idx = 0, i = 0; i < NET_RX_RING_SIZE; i++) { - skb_frag_t *frag; - const struct page *page; - if (!queue->rx_skbs[i]) - continue; - - skb = queue->rx_skbs[requeue_idx] = xennet_get_rx_skb(queue, i); - ref = queue->grant_rx_ref[requeue_idx] = xennet_get_rx_ref(queue, i); - req = RING_GET_REQUEST(&queue->rx, requeue_idx); - - frag = &skb_shinfo(skb)->frags[0]; - page = skb_frag_page(frag); - gnttab_grant_foreign_access_ref( - ref, queue->info->xbdev->otherend_id, - pfn_to_mfn(page_to_pfn(page)), - 0); - req->gref = ref; - req->id = requeue_idx; - - requeue_idx++; - } - - queue->rx.req_prod_pvt = requeue_idx; - - spin_unlock_bh(&queue->rx_lock); - } - /* - * Step 3: All public and private state should now be sane. Get + * All public and private state should now be sane. Get * ready to start sending and receiving packets and give the driver * domain a kick because we've probably just requeued some * packets. diff --git a/drivers/pci/host/pci-host-generic.c b/drivers/pci/host/pci-host-generic.c index 44fe6aa6a43f..3d2076f59911 100644 --- a/drivers/pci/host/pci-host-generic.c +++ b/drivers/pci/host/pci-host-generic.c @@ -385,4 +385,4 @@ module_platform_driver(gen_pci_driver); MODULE_DESCRIPTION("Generic PCI host driver"); MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>"); -MODULE_LICENSE("GPLv2"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/pci/host/pci-mvebu.c b/drivers/pci/host/pci-mvebu.c index ce23e0f076b6..a8c6f1a92e0f 100644 --- a/drivers/pci/host/pci-mvebu.c +++ b/drivers/pci/host/pci-mvebu.c @@ -1094,4 +1094,4 @@ module_platform_driver(mvebu_pcie_driver); MODULE_AUTHOR("Thomas Petazzoni <thomas.petazzoni@free-electrons.com>"); MODULE_DESCRIPTION("Marvell EBU PCIe driver"); -MODULE_LICENSE("GPLv2"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/pci/host/pci-tegra.c b/drivers/pci/host/pci-tegra.c index 083cf37ca047..c284e841e3ea 100644 --- a/drivers/pci/host/pci-tegra.c +++ b/drivers/pci/host/pci-tegra.c @@ -1716,4 +1716,4 @@ module_platform_driver(tegra_pcie_driver); MODULE_AUTHOR("Thierry Reding <treding@nvidia.com>"); MODULE_DESCRIPTION("NVIDIA Tegra PCIe driver"); -MODULE_LICENSE("GPLv2"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/pci/host/pcie-rcar.c b/drivers/pci/host/pcie-rcar.c index f7d3de32c9a0..4884ee5e07d4 100644 --- a/drivers/pci/host/pcie-rcar.c +++ b/drivers/pci/host/pcie-rcar.c @@ -105,7 +105,7 @@ #define PCIE_CONF_DEV(d) (((d) & 0x1f) << 19) #define PCIE_CONF_FUNC(f) (((f) & 0x7) << 16) -#define PCI_MAX_RESOURCES 4 +#define RCAR_PCI_MAX_RESOURCES 4 #define MAX_NR_INBOUND_MAPS 6 struct rcar_msi { @@ -127,7 +127,7 @@ static inline struct rcar_msi *to_rcar_msi(struct msi_chip *chip) struct rcar_pcie { struct device *dev; void __iomem *base; - struct resource res[PCI_MAX_RESOURCES]; + struct resource res[RCAR_PCI_MAX_RESOURCES]; struct resource busn; int root_bus_nr; struct clk *clk; @@ -140,36 +140,37 @@ static inline struct rcar_pcie *sys_to_pcie(struct pci_sys_data *sys) return sys->private_data; } -static void pci_write_reg(struct rcar_pcie *pcie, unsigned long val, - unsigned long reg) +static void rcar_pci_write_reg(struct rcar_pcie *pcie, unsigned long val, + unsigned long reg) { writel(val, pcie->base + reg); } -static unsigned long pci_read_reg(struct rcar_pcie *pcie, unsigned long reg) +static unsigned long rcar_pci_read_reg(struct rcar_pcie *pcie, + unsigned long reg) { return readl(pcie->base + reg); } enum { - PCI_ACCESS_READ, - PCI_ACCESS_WRITE, + RCAR_PCI_ACCESS_READ, + RCAR_PCI_ACCESS_WRITE, }; static void rcar_rmw32(struct rcar_pcie *pcie, int where, u32 mask, u32 data) { int shift = 8 * (where & 3); - u32 val = pci_read_reg(pcie, where & ~3); + u32 val = rcar_pci_read_reg(pcie, where & ~3); val &= ~(mask << shift); val |= data << shift; - pci_write_reg(pcie, val, where & ~3); + rcar_pci_write_reg(pcie, val, where & ~3); } static u32 rcar_read_conf(struct rcar_pcie *pcie, int where) { int shift = 8 * (where & 3); - u32 val = pci_read_reg(pcie, where & ~3); + u32 val = rcar_pci_read_reg(pcie, where & ~3); return val >> shift; } @@ -205,14 +206,14 @@ static int rcar_pcie_config_access(struct rcar_pcie *pcie, if (dev != 0) return PCIBIOS_DEVICE_NOT_FOUND; - if (access_type == PCI_ACCESS_READ) { - *data = pci_read_reg(pcie, PCICONF(index)); + if (access_type == RCAR_PCI_ACCESS_READ) { + *data = rcar_pci_read_reg(pcie, PCICONF(index)); } else { /* Keep an eye out for changes to the root bus number */ if (pci_is_root_bus(bus) && (reg == PCI_PRIMARY_BUS)) pcie->root_bus_nr = *data & 0xff; - pci_write_reg(pcie, *data, PCICONF(index)); + rcar_pci_write_reg(pcie, *data, PCICONF(index)); } return PCIBIOS_SUCCESSFUL; @@ -222,20 +223,20 @@ static int rcar_pcie_config_access(struct rcar_pcie *pcie, return PCIBIOS_DEVICE_NOT_FOUND; /* Clear errors */ - pci_write_reg(pcie, pci_read_reg(pcie, PCIEERRFR), PCIEERRFR); + rcar_pci_write_reg(pcie, rcar_pci_read_reg(pcie, PCIEERRFR), PCIEERRFR); /* Set the PIO address */ - pci_write_reg(pcie, PCIE_CONF_BUS(bus->number) | PCIE_CONF_DEV(dev) | - PCIE_CONF_FUNC(func) | reg, PCIECAR); + rcar_pci_write_reg(pcie, PCIE_CONF_BUS(bus->number) | + PCIE_CONF_DEV(dev) | PCIE_CONF_FUNC(func) | reg, PCIECAR); /* Enable the configuration access */ if (bus->parent->number == pcie->root_bus_nr) - pci_write_reg(pcie, CONFIG_SEND_ENABLE | TYPE0, PCIECCTLR); + rcar_pci_write_reg(pcie, CONFIG_SEND_ENABLE | TYPE0, PCIECCTLR); else - pci_write_reg(pcie, CONFIG_SEND_ENABLE | TYPE1, PCIECCTLR); + rcar_pci_write_reg(pcie, CONFIG_SEND_ENABLE | TYPE1, PCIECCTLR); /* Check for errors */ - if (pci_read_reg(pcie, PCIEERRFR) & UNSUPPORTED_REQUEST) + if (rcar_pci_read_reg(pcie, PCIEERRFR) & UNSUPPORTED_REQUEST) return PCIBIOS_DEVICE_NOT_FOUND; /* Check for master and target aborts */ @@ -243,13 +244,13 @@ static int rcar_pcie_config_access(struct rcar_pcie *pcie, (PCI_STATUS_REC_MASTER_ABORT | PCI_STATUS_REC_TARGET_ABORT)) return PCIBIOS_DEVICE_NOT_FOUND; - if (access_type == PCI_ACCESS_READ) - *data = pci_read_reg(pcie, PCIECDR); + if (access_type == RCAR_PCI_ACCESS_READ) + *data = rcar_pci_read_reg(pcie, PCIECDR); else - pci_write_reg(pcie, *data, PCIECDR); + rcar_pci_write_reg(pcie, *data, PCIECDR); /* Disable the configuration access */ - pci_write_reg(pcie, 0, PCIECCTLR); + rcar_pci_write_reg(pcie, 0, PCIECCTLR); return PCIBIOS_SUCCESSFUL; } @@ -260,12 +261,7 @@ static int rcar_pcie_read_conf(struct pci_bus *bus, unsigned int devfn, struct rcar_pcie *pcie = sys_to_pcie(bus->sysdata); int ret; - if ((size == 2) && (where & 1)) - return PCIBIOS_BAD_REGISTER_NUMBER; - else if ((size == 4) && (where & 3)) - return PCIBIOS_BAD_REGISTER_NUMBER; - - ret = rcar_pcie_config_access(pcie, PCI_ACCESS_READ, + ret = rcar_pcie_config_access(pcie, RCAR_PCI_ACCESS_READ, bus, devfn, where, val); if (ret != PCIBIOS_SUCCESSFUL) { *val = 0xffffffff; @@ -291,12 +287,7 @@ static int rcar_pcie_write_conf(struct pci_bus *bus, unsigned int devfn, int shift, ret; u32 data; - if ((size == 2) && (where & 1)) - return PCIBIOS_BAD_REGISTER_NUMBER; - else if ((size == 4) && (where & 3)) - return PCIBIOS_BAD_REGISTER_NUMBER; - - ret = rcar_pcie_config_access(pcie, PCI_ACCESS_READ, + ret = rcar_pcie_config_access(pcie, RCAR_PCI_ACCESS_READ, bus, devfn, where, &data); if (ret != PCIBIOS_SUCCESSFUL) return ret; @@ -315,7 +306,7 @@ static int rcar_pcie_write_conf(struct pci_bus *bus, unsigned int devfn, } else data = val; - ret = rcar_pcie_config_access(pcie, PCI_ACCESS_WRITE, + ret = rcar_pcie_config_access(pcie, RCAR_PCI_ACCESS_WRITE, bus, devfn, where, &data); return ret; @@ -326,14 +317,15 @@ static struct pci_ops rcar_pcie_ops = { .write = rcar_pcie_write_conf, }; -static void rcar_pcie_setup_window(int win, struct resource *res, - struct rcar_pcie *pcie) +static void rcar_pcie_setup_window(int win, struct rcar_pcie *pcie) { + struct resource *res = &pcie->res[win]; + /* Setup PCIe address space mappings for each resource */ resource_size_t size; u32 mask; - pci_write_reg(pcie, 0x00000000, PCIEPTCTLR(win)); + rcar_pci_write_reg(pcie, 0x00000000, PCIEPTCTLR(win)); /* * The PAMR mask is calculated in units of 128Bytes, which @@ -341,17 +333,17 @@ static void rcar_pcie_setup_window(int win, struct resource *res, */ size = resource_size(res); mask = (roundup_pow_of_two(size) / SZ_128) - 1; - pci_write_reg(pcie, mask << 7, PCIEPAMR(win)); + rcar_pci_write_reg(pcie, mask << 7, PCIEPAMR(win)); - pci_write_reg(pcie, upper_32_bits(res->start), PCIEPARH(win)); - pci_write_reg(pcie, lower_32_bits(res->start), PCIEPARL(win)); + rcar_pci_write_reg(pcie, upper_32_bits(res->start), PCIEPARH(win)); + rcar_pci_write_reg(pcie, lower_32_bits(res->start), PCIEPARL(win)); /* First resource is for IO */ mask = PAR_ENABLE; if (res->flags & IORESOURCE_IO) mask |= IO_SPACE; - pci_write_reg(pcie, mask, PCIEPTCTLR(win)); + rcar_pci_write_reg(pcie, mask, PCIEPTCTLR(win)); } static int rcar_pcie_setup(int nr, struct pci_sys_data *sys) @@ -363,13 +355,13 @@ static int rcar_pcie_setup(int nr, struct pci_sys_data *sys) pcie->root_bus_nr = -1; /* Setup PCI resources */ - for (i = 0; i < PCI_MAX_RESOURCES; i++) { + for (i = 0; i < RCAR_PCI_MAX_RESOURCES; i++) { res = &pcie->res[i]; if (!res->flags) continue; - rcar_pcie_setup_window(i, res, pcie); + rcar_pcie_setup_window(i, pcie); if (res->flags & IORESOURCE_IO) pci_ioremap_io(nr * SZ_64K, res->start); @@ -415,7 +407,7 @@ static int phy_wait_for_ack(struct rcar_pcie *pcie) unsigned int timeout = 100; while (timeout--) { - if (pci_read_reg(pcie, H1_PCIEPHYADRR) & PHY_ACK) + if (rcar_pci_read_reg(pcie, H1_PCIEPHYADRR) & PHY_ACK) return 0; udelay(100); @@ -438,15 +430,15 @@ static void phy_write_reg(struct rcar_pcie *pcie, ((addr & 0xff) << ADR_POS); /* Set write data */ - pci_write_reg(pcie, data, H1_PCIEPHYDOUTR); - pci_write_reg(pcie, phyaddr, H1_PCIEPHYADRR); + rcar_pci_write_reg(pcie, data, H1_PCIEPHYDOUTR); + rcar_pci_write_reg(pcie, phyaddr, H1_PCIEPHYADRR); /* Ignore errors as they will be dealt with if the data link is down */ phy_wait_for_ack(pcie); /* Clear command */ - pci_write_reg(pcie, 0, H1_PCIEPHYDOUTR); - pci_write_reg(pcie, 0, H1_PCIEPHYADRR); + rcar_pci_write_reg(pcie, 0, H1_PCIEPHYDOUTR); + rcar_pci_write_reg(pcie, 0, H1_PCIEPHYADRR); /* Ignore errors as they will be dealt with if the data link is down */ phy_wait_for_ack(pcie); @@ -457,7 +449,7 @@ static int rcar_pcie_wait_for_dl(struct rcar_pcie *pcie) unsigned int timeout = 10; while (timeout--) { - if ((pci_read_reg(pcie, PCIETSTR) & DATA_LINK_ACTIVE)) + if ((rcar_pci_read_reg(pcie, PCIETSTR) & DATA_LINK_ACTIVE)) return 0; msleep(5); @@ -471,17 +463,17 @@ static int rcar_pcie_hw_init(struct rcar_pcie *pcie) int err; /* Begin initialization */ - pci_write_reg(pcie, 0, PCIETCTLR); + rcar_pci_write_reg(pcie, 0, PCIETCTLR); /* Set mode */ - pci_write_reg(pcie, 1, PCIEMSR); + rcar_pci_write_reg(pcie, 1, PCIEMSR); /* * Initial header for port config space is type 1, set the device * class to match. Hardware takes care of propagating the IDSETR * settings, so there is no need to bother with a quirk. */ - pci_write_reg(pcie, PCI_CLASS_BRIDGE_PCI << 16, IDSETR1); + rcar_pci_write_reg(pcie, PCI_CLASS_BRIDGE_PCI << 16, IDSETR1); /* * Setup Secondary Bus Number & Subordinate Bus Number, even though @@ -491,33 +483,31 @@ static int rcar_pcie_hw_init(struct rcar_pcie *pcie) rcar_rmw32(pcie, RCONF(PCI_SUBORDINATE_BUS), 0xff, 1); /* Initialize default capabilities. */ - rcar_rmw32(pcie, REXPCAP(0), 0, PCI_CAP_ID_EXP); + rcar_rmw32(pcie, REXPCAP(0), 0xff, PCI_CAP_ID_EXP); rcar_rmw32(pcie, REXPCAP(PCI_EXP_FLAGS), PCI_EXP_FLAGS_TYPE, PCI_EXP_TYPE_ROOT_PORT << 4); rcar_rmw32(pcie, RCONF(PCI_HEADER_TYPE), 0x7f, PCI_HEADER_TYPE_BRIDGE); /* Enable data link layer active state reporting */ - rcar_rmw32(pcie, REXPCAP(PCI_EXP_LNKCAP), 0, PCI_EXP_LNKCAP_DLLLARC); + rcar_rmw32(pcie, REXPCAP(PCI_EXP_LNKCAP), PCI_EXP_LNKCAP_DLLLARC, + PCI_EXP_LNKCAP_DLLLARC); /* Write out the physical slot number = 0 */ rcar_rmw32(pcie, REXPCAP(PCI_EXP_SLTCAP), PCI_EXP_SLTCAP_PSN, 0); /* Set the completion timer timeout to the maximum 50ms. */ - rcar_rmw32(pcie, TLCTLR+1, 0x3f, 50); + rcar_rmw32(pcie, TLCTLR + 1, 0x3f, 50); /* Terminate list of capabilities (Next Capability Offset=0) */ - rcar_rmw32(pcie, RVCCAP(0), 0xfff0, 0); - - /* Enable MAC data scrambling. */ - rcar_rmw32(pcie, MACCTLR, SCRAMBLE_DISABLE, 0); + rcar_rmw32(pcie, RVCCAP(0), 0xfff00000, 0); /* Enable MSI */ if (IS_ENABLED(CONFIG_PCI_MSI)) - pci_write_reg(pcie, 0x101f0000, PCIEMSITXR); + rcar_pci_write_reg(pcie, 0x101f0000, PCIEMSITXR); /* Finish initialization - establish a PCI Express link */ - pci_write_reg(pcie, CFINIT, PCIETCTLR); + rcar_pci_write_reg(pcie, CFINIT, PCIETCTLR); /* This will timeout if we don't have a link. */ err = rcar_pcie_wait_for_dl(pcie); @@ -527,11 +517,6 @@ static int rcar_pcie_hw_init(struct rcar_pcie *pcie) /* Enable INTx interrupts */ rcar_rmw32(pcie, PCIEINTXR, 0, 0xF << 8); - /* Enable slave Bus Mastering */ - rcar_rmw32(pcie, RCONF(PCI_STATUS), PCI_STATUS_DEVSEL_MASK, - PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER | - PCI_STATUS_CAP_LIST | PCI_STATUS_DEVSEL_FAST); - wmb(); return 0; @@ -560,7 +545,7 @@ static int rcar_pcie_hw_init_h1(struct rcar_pcie *pcie) phy_write_reg(pcie, 0, 0x66, 0x1, 0x00008000); while (timeout--) { - if (pci_read_reg(pcie, H1_PCIEPHYSR)) + if (rcar_pci_read_reg(pcie, H1_PCIEPHYSR)) return rcar_pcie_hw_init(pcie); msleep(5); @@ -599,7 +584,7 @@ static irqreturn_t rcar_pcie_msi_irq(int irq, void *data) struct rcar_msi *msi = &pcie->msi; unsigned long reg; - reg = pci_read_reg(pcie, PCIEMSIFR); + reg = rcar_pci_read_reg(pcie, PCIEMSIFR); /* MSI & INTx share an interrupt - we only handle MSI here */ if (!reg) @@ -610,7 +595,7 @@ static irqreturn_t rcar_pcie_msi_irq(int irq, void *data) unsigned int irq; /* clear the interrupt */ - pci_write_reg(pcie, 1 << index, PCIEMSIFR); + rcar_pci_write_reg(pcie, 1 << index, PCIEMSIFR); irq = irq_find_mapping(msi->domain, index); if (irq) { @@ -624,7 +609,7 @@ static irqreturn_t rcar_pcie_msi_irq(int irq, void *data) } /* see if there's any more pending in this vector */ - reg = pci_read_reg(pcie, PCIEMSIFR); + reg = rcar_pci_read_reg(pcie, PCIEMSIFR); } return IRQ_HANDLED; @@ -651,8 +636,8 @@ static int rcar_msi_setup_irq(struct msi_chip *chip, struct pci_dev *pdev, irq_set_msi_desc(irq, desc); - msg.address_lo = pci_read_reg(pcie, PCIEMSIALR) & ~MSIFE; - msg.address_hi = pci_read_reg(pcie, PCIEMSIAUR); + msg.address_lo = rcar_pci_read_reg(pcie, PCIEMSIALR) & ~MSIFE; + msg.address_hi = rcar_pci_read_reg(pcie, PCIEMSIAUR); msg.data = hwirq; write_msi_msg(irq, &msg); @@ -729,11 +714,11 @@ static int rcar_pcie_enable_msi(struct rcar_pcie *pcie) msi->pages = __get_free_pages(GFP_KERNEL, 0); base = virt_to_phys((void *)msi->pages); - pci_write_reg(pcie, base | MSIFE, PCIEMSIALR); - pci_write_reg(pcie, 0, PCIEMSIAUR); + rcar_pci_write_reg(pcie, base | MSIFE, PCIEMSIALR); + rcar_pci_write_reg(pcie, 0, PCIEMSIAUR); /* enable all MSI interrupts */ - pci_write_reg(pcie, 0xffffffff, PCIEMSIIER); + rcar_pci_write_reg(pcie, 0xffffffff, PCIEMSIIER); return 0; @@ -826,6 +811,7 @@ static int rcar_pcie_inbound_ranges(struct rcar_pcie *pcie, if (cpu_addr > 0) { unsigned long nr_zeros = __ffs64(cpu_addr); u64 alignment = 1ULL << nr_zeros; + size = min(range->size, alignment); } else { size = range->size; @@ -841,13 +827,13 @@ static int rcar_pcie_inbound_ranges(struct rcar_pcie *pcie, * Set up 64-bit inbound regions as the range parser doesn't * distinguish between 32 and 64-bit types. */ - pci_write_reg(pcie, lower_32_bits(pci_addr), PCIEPRAR(idx)); - pci_write_reg(pcie, lower_32_bits(cpu_addr), PCIELAR(idx)); - pci_write_reg(pcie, lower_32_bits(mask) | flags, PCIELAMR(idx)); + rcar_pci_write_reg(pcie, lower_32_bits(pci_addr), PCIEPRAR(idx)); + rcar_pci_write_reg(pcie, lower_32_bits(cpu_addr), PCIELAR(idx)); + rcar_pci_write_reg(pcie, lower_32_bits(mask) | flags, PCIELAMR(idx)); - pci_write_reg(pcie, upper_32_bits(pci_addr), PCIEPRAR(idx+1)); - pci_write_reg(pcie, upper_32_bits(cpu_addr), PCIELAR(idx+1)); - pci_write_reg(pcie, 0, PCIELAMR(idx+1)); + rcar_pci_write_reg(pcie, upper_32_bits(pci_addr), PCIEPRAR(idx+1)); + rcar_pci_write_reg(pcie, upper_32_bits(cpu_addr), PCIELAR(idx+1)); + rcar_pci_write_reg(pcie, 0, PCIELAMR(idx + 1)); pci_addr += size; cpu_addr += size; @@ -952,7 +938,7 @@ static int rcar_pcie_probe(struct platform_device *pdev) of_pci_range_to_resource(&range, pdev->dev.of_node, &pcie->res[win++]); - if (win > PCI_MAX_RESOURCES) + if (win > RCAR_PCI_MAX_RESOURCES) break; } @@ -982,7 +968,7 @@ static int rcar_pcie_probe(struct platform_device *pdev) return 0; } - data = pci_read_reg(pcie, MACSR); + data = rcar_pci_read_reg(pcie, MACSR); dev_info(&pdev->dev, "PCIe x%d: link up\n", (data >> 20) & 0x3f); rcar_pcie_enable(pcie); @@ -1003,4 +989,4 @@ module_platform_driver(rcar_pcie_driver); MODULE_AUTHOR("Phil Edworthy <phil.edworthy@renesas.com>"); MODULE_DESCRIPTION("Renesas R-Car PCIe driver"); -MODULE_LICENSE("GPLv2"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/pci/hotplug/cpqphp_sysfs.c b/drivers/pci/hotplug/cpqphp_sysfs.c index 4a392c44e3d3..d81648f71425 100644 --- a/drivers/pci/hotplug/cpqphp_sysfs.c +++ b/drivers/pci/hotplug/cpqphp_sysfs.c @@ -216,8 +216,7 @@ void cpqhp_create_debugfs_files(struct controller *ctrl) void cpqhp_remove_debugfs_files(struct controller *ctrl) { - if (ctrl->dentry) - debugfs_remove(ctrl->dentry); + debugfs_remove(ctrl->dentry); ctrl->dentry = NULL; } diff --git a/drivers/pci/hotplug/pciehp.h b/drivers/pci/hotplug/pciehp.h index 8e9012dca450..9e5a9fbb93d7 100644 --- a/drivers/pci/hotplug/pciehp.h +++ b/drivers/pci/hotplug/pciehp.h @@ -92,9 +92,10 @@ struct controller { struct slot *slot; wait_queue_head_t queue; /* sleep & wake process */ u32 slot_cap; + u32 slot_ctrl; struct timer_list poll_timer; + unsigned long cmd_started; /* jiffies */ unsigned int cmd_busy:1; - unsigned int no_cmd_complete:1; unsigned int link_active_reporting:1; unsigned int notification_enabled:1; unsigned int power_fault_detected; diff --git a/drivers/pci/hotplug/pciehp_core.c b/drivers/pci/hotplug/pciehp_core.c index a2297db80813..07aa722bb12c 100644 --- a/drivers/pci/hotplug/pciehp_core.c +++ b/drivers/pci/hotplug/pciehp_core.c @@ -255,6 +255,13 @@ static int pciehp_probe(struct pcie_device *dev) else if (pciehp_acpi_slot_detection_check(dev->port)) goto err_out_none; + if (!dev->port->subordinate) { + /* Can happen if we run out of bus numbers during probe */ + dev_err(&dev->device, + "Hotplug bridge without secondary bus, ignoring\n"); + goto err_out_none; + } + ctrl = pcie_init(dev); if (!ctrl) { dev_err(&dev->device, "Controller initialization failed\n"); diff --git a/drivers/pci/hotplug/pciehp_hpc.c b/drivers/pci/hotplug/pciehp_hpc.c index 42914e04d110..9da84b8b27d8 100644 --- a/drivers/pci/hotplug/pciehp_hpc.c +++ b/drivers/pci/hotplug/pciehp_hpc.c @@ -104,11 +104,10 @@ static inline void pciehp_free_irq(struct controller *ctrl) free_irq(ctrl->pcie->irq, ctrl); } -static int pcie_poll_cmd(struct controller *ctrl) +static int pcie_poll_cmd(struct controller *ctrl, int timeout) { struct pci_dev *pdev = ctrl_dev(ctrl); u16 slot_status; - int timeout = 1000; pcie_capability_read_word(pdev, PCI_EXP_SLTSTA, &slot_status); if (slot_status & PCI_EXP_SLTSTA_CC) { @@ -129,18 +128,52 @@ static int pcie_poll_cmd(struct controller *ctrl) return 0; /* timeout */ } -static void pcie_wait_cmd(struct controller *ctrl, int poll) +static void pcie_wait_cmd(struct controller *ctrl) { unsigned int msecs = pciehp_poll_mode ? 2500 : 1000; - unsigned long timeout = msecs_to_jiffies(msecs); + unsigned long duration = msecs_to_jiffies(msecs); + unsigned long cmd_timeout = ctrl->cmd_started + duration; + unsigned long now, timeout; int rc; - if (poll) - rc = pcie_poll_cmd(ctrl); + /* + * If the controller does not generate notifications for command + * completions, we never need to wait between writes. + */ + if (NO_CMD_CMPL(ctrl)) + return; + + if (!ctrl->cmd_busy) + return; + + /* + * Even if the command has already timed out, we want to call + * pcie_poll_cmd() so it can clear PCI_EXP_SLTSTA_CC. + */ + now = jiffies; + if (time_before_eq(cmd_timeout, now)) + timeout = 1; else + timeout = cmd_timeout - now; + + if (ctrl->slot_ctrl & PCI_EXP_SLTCTL_HPIE && + ctrl->slot_ctrl & PCI_EXP_SLTCTL_CCIE) rc = wait_event_timeout(ctrl->queue, !ctrl->cmd_busy, timeout); + else + rc = pcie_poll_cmd(ctrl, timeout); + + /* + * Controllers with errata like Intel CF118 don't generate + * completion notifications unless the power/indicator/interlock + * control bits are changed. On such controllers, we'll emit this + * timeout message when we wait for completion of commands that + * don't change those bits, e.g., commands that merely enable + * interrupts. + */ if (!rc) - ctrl_dbg(ctrl, "Command not completed in 1000 msec\n"); + ctrl_info(ctrl, "Timeout on hotplug command %#010x (issued %u msec ago)\n", + ctrl->slot_ctrl, + jiffies_to_msecs(now - ctrl->cmd_started)); } /** @@ -152,34 +185,12 @@ static void pcie_wait_cmd(struct controller *ctrl, int poll) static void pcie_write_cmd(struct controller *ctrl, u16 cmd, u16 mask) { struct pci_dev *pdev = ctrl_dev(ctrl); - u16 slot_status; u16 slot_ctrl; mutex_lock(&ctrl->ctrl_lock); - pcie_capability_read_word(pdev, PCI_EXP_SLTSTA, &slot_status); - if (slot_status & PCI_EXP_SLTSTA_CC) { - pcie_capability_write_word(pdev, PCI_EXP_SLTSTA, - PCI_EXP_SLTSTA_CC); - if (!ctrl->no_cmd_complete) { - /* - * After 1 sec and CMD_COMPLETED still not set, just - * proceed forward to issue the next command according - * to spec. Just print out the error message. - */ - ctrl_dbg(ctrl, "CMD_COMPLETED not clear after 1 sec\n"); - } else if (!NO_CMD_CMPL(ctrl)) { - /* - * This controller seems to notify of command completed - * event even though it supports none of power - * controller, attention led, power led and EMI. - */ - ctrl_dbg(ctrl, "Unexpected CMD_COMPLETED. Need to wait for command completed event\n"); - ctrl->no_cmd_complete = 0; - } else { - ctrl_dbg(ctrl, "Unexpected CMD_COMPLETED. Maybe the controller is broken\n"); - } - } + /* Wait for any previous command that might still be in progress */ + pcie_wait_cmd(ctrl); pcie_capability_read_word(pdev, PCI_EXP_SLTCTL, &slot_ctrl); slot_ctrl &= ~mask; @@ -187,22 +198,9 @@ static void pcie_write_cmd(struct controller *ctrl, u16 cmd, u16 mask) ctrl->cmd_busy = 1; smp_mb(); pcie_capability_write_word(pdev, PCI_EXP_SLTCTL, slot_ctrl); + ctrl->cmd_started = jiffies; + ctrl->slot_ctrl = slot_ctrl; - /* - * Wait for command completion. - */ - if (!ctrl->no_cmd_complete) { - int poll = 0; - /* - * if hotplug interrupt is not enabled or command - * completed interrupt is not enabled, we need to poll - * command completed event. - */ - if (!(slot_ctrl & PCI_EXP_SLTCTL_HPIE) || - !(slot_ctrl & PCI_EXP_SLTCTL_CCIE)) - poll = 1; - pcie_wait_cmd(ctrl, poll); - } mutex_unlock(&ctrl->ctrl_lock); } @@ -773,15 +771,6 @@ struct controller *pcie_init(struct pcie_device *dev) mutex_init(&ctrl->ctrl_lock); init_waitqueue_head(&ctrl->queue); dbg_ctrl(ctrl); - /* - * Controller doesn't notify of command completion if the "No - * Command Completed Support" bit is set in Slot Capability - * register or the controller supports none of power - * controller, attention led, power led and EMI. - */ - if (NO_CMD_CMPL(ctrl) || - !(POWER_CTRL(ctrl) | ATTN_LED(ctrl) | PWR_LED(ctrl) | EMI(ctrl))) - ctrl->no_cmd_complete = 1; /* Check if Data Link Layer Link Active Reporting is implemented */ pcie_capability_read_dword(pdev, PCI_EXP_LNKCAP, &link_cap); @@ -794,7 +783,7 @@ struct controller *pcie_init(struct pcie_device *dev) pcie_capability_write_word(pdev, PCI_EXP_SLTSTA, PCI_EXP_SLTSTA_ABP | PCI_EXP_SLTSTA_PFD | PCI_EXP_SLTSTA_MRLSC | PCI_EXP_SLTSTA_PDC | - PCI_EXP_SLTSTA_CC); + PCI_EXP_SLTSTA_CC | PCI_EXP_SLTSTA_DLLSC); /* Disable software notification */ pcie_disable_notification(ctrl); diff --git a/drivers/pci/msi.c b/drivers/pci/msi.c index 13f3d3037272..5a40516444f3 100644 --- a/drivers/pci/msi.c +++ b/drivers/pci/msi.c @@ -149,15 +149,14 @@ static void msi_set_enable(struct pci_dev *dev, int enable) pci_write_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, control); } -static void msix_set_enable(struct pci_dev *dev, int enable) +static void msix_clear_and_set_ctrl(struct pci_dev *dev, u16 clear, u16 set) { - u16 control; + u16 ctrl; - pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &control); - control &= ~PCI_MSIX_FLAGS_ENABLE; - if (enable) - control |= PCI_MSIX_FLAGS_ENABLE; - pci_write_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, control); + pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &ctrl); + ctrl &= ~clear; + ctrl |= set; + pci_write_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, ctrl); } static inline __attribute_const__ u32 msi_mask(unsigned x) @@ -168,16 +167,6 @@ static inline __attribute_const__ u32 msi_mask(unsigned x) return (1 << (1 << x)) - 1; } -static inline __attribute_const__ u32 msi_capable_mask(u16 control) -{ - return msi_mask((control >> 1) & 7); -} - -static inline __attribute_const__ u32 msi_enabled_mask(u16 control) -{ - return msi_mask((control >> 4) & 7); -} - /* * PCI 2.3 does not specify mask bits for each MSI interrupt. Attempting to * mask all MSI interrupts by clearing the MSI enable bit does not work @@ -246,7 +235,7 @@ static void msi_set_mask_bit(struct irq_data *data, u32 flag) msix_mask_irq(desc, flag); readl(desc->mask_base); /* Flush write to device */ } else { - unsigned offset = data->irq - desc->dev->irq; + unsigned offset = data->irq - desc->irq; msi_mask_irq(desc, 1 << offset, flag << offset); } } @@ -460,7 +449,8 @@ static void __pci_restore_msi_state(struct pci_dev *dev) arch_restore_msi_irqs(dev); pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control); - msi_mask_irq(entry, msi_capable_mask(control), entry->masked); + msi_mask_irq(entry, msi_mask(entry->msi_attrib.multi_cap), + entry->masked); control &= ~PCI_MSI_FLAGS_QSIZE; control |= (entry->msi_attrib.multiple << 4) | PCI_MSI_FLAGS_ENABLE; pci_write_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, control); @@ -469,26 +459,22 @@ static void __pci_restore_msi_state(struct pci_dev *dev) static void __pci_restore_msix_state(struct pci_dev *dev) { struct msi_desc *entry; - u16 control; if (!dev->msix_enabled) return; BUG_ON(list_empty(&dev->msi_list)); - entry = list_first_entry(&dev->msi_list, struct msi_desc, list); - pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &control); /* route the table */ pci_intx_for_msi(dev, 0); - control |= PCI_MSIX_FLAGS_ENABLE | PCI_MSIX_FLAGS_MASKALL; - pci_write_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, control); + msix_clear_and_set_ctrl(dev, 0, + PCI_MSIX_FLAGS_ENABLE | PCI_MSIX_FLAGS_MASKALL); arch_restore_msi_irqs(dev); list_for_each_entry(entry, &dev->msi_list, list) { msix_mask_irq(entry, entry->masked); } - control &= ~PCI_MSIX_FLAGS_MASKALL; - pci_write_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, control); + msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL, 0); } void pci_restore_msi_state(struct pci_dev *dev) @@ -501,7 +487,6 @@ EXPORT_SYMBOL_GPL(pci_restore_msi_state); static ssize_t msi_mode_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct pci_dev *pdev = to_pci_dev(dev); struct msi_desc *entry; unsigned long irq; int retval; @@ -510,12 +495,11 @@ static ssize_t msi_mode_show(struct device *dev, struct device_attribute *attr, if (retval) return retval; - list_for_each_entry(entry, &pdev->msi_list, list) { - if (entry->irq == irq) { - return sprintf(buf, "%s\n", - entry->msi_attrib.is_msix ? "msix" : "msi"); - } - } + entry = irq_get_msi_desc(irq); + if (entry) + return sprintf(buf, "%s\n", + entry->msi_attrib.is_msix ? "msix" : "msi"); + return -ENODEV; } @@ -594,6 +578,38 @@ error_attrs: return ret; } +static struct msi_desc *msi_setup_entry(struct pci_dev *dev) +{ + u16 control; + struct msi_desc *entry; + + /* MSI Entry Initialization */ + entry = alloc_msi_entry(dev); + if (!entry) + return NULL; + + pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control); + + entry->msi_attrib.is_msix = 0; + entry->msi_attrib.is_64 = !!(control & PCI_MSI_FLAGS_64BIT); + entry->msi_attrib.entry_nr = 0; + entry->msi_attrib.maskbit = !!(control & PCI_MSI_FLAGS_MASKBIT); + entry->msi_attrib.default_irq = dev->irq; /* Save IOAPIC IRQ */ + entry->msi_attrib.pos = dev->msi_cap; + entry->msi_attrib.multi_cap = (control & PCI_MSI_FLAGS_QMASK) >> 1; + + if (control & PCI_MSI_FLAGS_64BIT) + entry->mask_pos = dev->msi_cap + PCI_MSI_MASK_64; + else + entry->mask_pos = dev->msi_cap + PCI_MSI_MASK_32; + + /* Save the initial mask status */ + if (entry->msi_attrib.maskbit) + pci_read_config_dword(dev, entry->mask_pos, &entry->masked); + + return entry; +} + /** * msi_capability_init - configure device's MSI capability structure * @dev: pointer to the pci_dev data structure of MSI device function @@ -609,32 +625,16 @@ static int msi_capability_init(struct pci_dev *dev, int nvec) { struct msi_desc *entry; int ret; - u16 control; unsigned mask; msi_set_enable(dev, 0); /* Disable MSI during set up */ - pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control); - /* MSI Entry Initialization */ - entry = alloc_msi_entry(dev); + entry = msi_setup_entry(dev); if (!entry) return -ENOMEM; - entry->msi_attrib.is_msix = 0; - entry->msi_attrib.is_64 = !!(control & PCI_MSI_FLAGS_64BIT); - entry->msi_attrib.entry_nr = 0; - entry->msi_attrib.maskbit = !!(control & PCI_MSI_FLAGS_MASKBIT); - entry->msi_attrib.default_irq = dev->irq; /* Save IOAPIC IRQ */ - entry->msi_attrib.pos = dev->msi_cap; - - if (control & PCI_MSI_FLAGS_64BIT) - entry->mask_pos = dev->msi_cap + PCI_MSI_MASK_64; - else - entry->mask_pos = dev->msi_cap + PCI_MSI_MASK_32; /* All MSIs are unmasked by default, Mask them all */ - if (entry->msi_attrib.maskbit) - pci_read_config_dword(dev, entry->mask_pos, &entry->masked); - mask = msi_capable_mask(control); + mask = msi_mask(entry->msi_attrib.multi_cap); msi_mask_irq(entry, mask, mask); list_add_tail(&entry->list, &dev->msi_list); @@ -743,12 +743,10 @@ static int msix_capability_init(struct pci_dev *dev, u16 control; void __iomem *base; - pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &control); - /* Ensure MSI-X is disabled while it is set up */ - control &= ~PCI_MSIX_FLAGS_ENABLE; - pci_write_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, control); + msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_ENABLE, 0); + pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &control); /* Request & Map MSI-X table region */ base = msix_map_region(dev, msix_table_size(control)); if (!base) @@ -767,8 +765,8 @@ static int msix_capability_init(struct pci_dev *dev, * MSI-X registers. We need to mask all the vectors to prevent * interrupts coming in before they're fully set up. */ - control |= PCI_MSIX_FLAGS_MASKALL | PCI_MSIX_FLAGS_ENABLE; - pci_write_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, control); + msix_clear_and_set_ctrl(dev, 0, + PCI_MSIX_FLAGS_MASKALL | PCI_MSIX_FLAGS_ENABLE); msix_program_entries(dev, entries); @@ -780,8 +778,7 @@ static int msix_capability_init(struct pci_dev *dev, pci_intx_for_msi(dev, 0); dev->msix_enabled = 1; - control &= ~PCI_MSIX_FLAGS_MASKALL; - pci_write_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, control); + msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL, 0); return 0; @@ -882,7 +879,6 @@ void pci_msi_shutdown(struct pci_dev *dev) { struct msi_desc *desc; u32 mask; - u16 ctrl; if (!pci_msi_enable || !dev || !dev->msi_enabled) return; @@ -895,8 +891,7 @@ void pci_msi_shutdown(struct pci_dev *dev) dev->msi_enabled = 0; /* Return the device with MSI unmasked as initial states */ - pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &ctrl); - mask = msi_capable_mask(ctrl); + mask = msi_mask(desc->msi_attrib.multi_cap); /* Keep cached state to be restored */ arch_msi_mask_irq(desc, mask, ~mask); @@ -1001,7 +996,7 @@ void pci_msix_shutdown(struct pci_dev *dev) arch_msix_mask_irq(entry, 1); } - msix_set_enable(dev, 0); + msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_ENABLE, 0); pci_intx_for_msi(dev, 1); dev->msix_enabled = 0; } @@ -1016,24 +1011,6 @@ void pci_disable_msix(struct pci_dev *dev) } EXPORT_SYMBOL(pci_disable_msix); -/** - * msi_remove_pci_irq_vectors - reclaim MSI(X) irqs to unused state - * @dev: pointer to the pci_dev data structure of MSI(X) device function - * - * Being called during hotplug remove, from which the device function - * is hot-removed. All previous assigned MSI/MSI-X irqs, if - * allocated for this device function, are reclaimed to unused state, - * which may be used later on. - **/ -void msi_remove_pci_irq_vectors(struct pci_dev *dev) -{ - if (!pci_msi_enable || !dev) - return; - - if (dev->msi_enabled || dev->msix_enabled) - free_msi_irqs(dev); -} - void pci_no_msi(void) { pci_msi_enable = 0; @@ -1065,7 +1042,7 @@ void pci_msi_init_pci_dev(struct pci_dev *dev) dev->msix_cap = pci_find_capability(dev, PCI_CAP_ID_MSIX); if (dev->msix_cap) - msix_set_enable(dev, 0); + msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_ENABLE, 0); } /** diff --git a/drivers/pci/pci-label.c b/drivers/pci/pci-label.c index a3fbe2012ea3..2ab1b47c7651 100644 --- a/drivers/pci/pci-label.c +++ b/drivers/pci/pci-label.c @@ -161,8 +161,8 @@ enum acpi_attr_enum { static void dsm_label_utf16s_to_utf8s(union acpi_object *obj, char *buf) { int len; - len = utf16s_to_utf8s((const wchar_t *)obj->string.pointer, - obj->string.length, + len = utf16s_to_utf8s((const wchar_t *)obj->buffer.pointer, + obj->buffer.length, UTF16_LITTLE_ENDIAN, buf, PAGE_SIZE); buf[len] = '\n'; @@ -187,16 +187,22 @@ static int dsm_get_label(struct device *dev, char *buf, tmp = obj->package.elements; if (obj->type == ACPI_TYPE_PACKAGE && obj->package.count == 2 && tmp[0].type == ACPI_TYPE_INTEGER && - tmp[1].type == ACPI_TYPE_STRING) { + (tmp[1].type == ACPI_TYPE_STRING || + tmp[1].type == ACPI_TYPE_BUFFER)) { /* * The second string element is optional even when * this _DSM is implemented; when not implemented, * this entry must return a null string. */ - if (attr == ACPI_ATTR_INDEX_SHOW) + if (attr == ACPI_ATTR_INDEX_SHOW) { scnprintf(buf, PAGE_SIZE, "%llu\n", tmp->integer.value); - else if (attr == ACPI_ATTR_LABEL_SHOW) - dsm_label_utf16s_to_utf8s(tmp + 1, buf); + } else if (attr == ACPI_ATTR_LABEL_SHOW) { + if (tmp[1].type == ACPI_TYPE_STRING) + scnprintf(buf, PAGE_SIZE, "%s\n", + tmp[1].string.pointer); + else if (tmp[1].type == ACPI_TYPE_BUFFER) + dsm_label_utf16s_to_utf8s(tmp + 1, buf); + } len = strlen(buf) > 0 ? strlen(buf) : -1; } diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index 1c8592b0e146..2c9ac70254e2 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -839,12 +839,6 @@ int pci_set_power_state(struct pci_dev *dev, pci_power_t state) if (!__pci_complete_power_transition(dev, state)) error = 0; - /* - * When aspm_policy is "powersave" this call ensures - * that ASPM is configured. - */ - if (!error && dev->bus->self) - pcie_aspm_powersave_config_link(dev->bus->self); return error; } @@ -1195,12 +1189,18 @@ int __weak pcibios_enable_device(struct pci_dev *dev, int bars) static int do_pci_enable_device(struct pci_dev *dev, int bars) { int err; + struct pci_dev *bridge; u16 cmd; u8 pin; err = pci_set_power_state(dev, PCI_D0); if (err < 0 && err != -EIO) return err; + + bridge = pci_upstream_bridge(dev); + if (bridge) + pcie_aspm_powersave_config_link(bridge); + err = pcibios_enable_device(dev, bars); if (err < 0) return err; @@ -3198,7 +3198,7 @@ static int pci_pm_reset(struct pci_dev *dev, int probe) return 0; } -void __weak pcibios_reset_secondary_bus(struct pci_dev *dev) +void pci_reset_secondary_bus(struct pci_dev *dev) { u16 ctrl; @@ -3224,6 +3224,11 @@ void __weak pcibios_reset_secondary_bus(struct pci_dev *dev) ssleep(1); } +void __weak pcibios_reset_secondary_bus(struct pci_dev *dev) +{ + pci_reset_secondary_bus(dev); +} + /** * pci_reset_bridge_secondary_bus - Reset the secondary bus on a PCI bridge. * @dev: Bridge device diff --git a/drivers/pci/pcie/portdrv_pci.c b/drivers/pci/pcie/portdrv_pci.c index 80887eaa0668..2ccc9b926ea7 100644 --- a/drivers/pci/pcie/portdrv_pci.c +++ b/drivers/pci/pcie/portdrv_pci.c @@ -203,10 +203,6 @@ static int pcie_portdrv_probe(struct pci_dev *dev, (pci_pcie_type(dev) != PCI_EXP_TYPE_DOWNSTREAM))) return -ENODEV; - if (!dev->irq && dev->pin) { - dev_warn(&dev->dev, "device [%04x:%04x] has invalid IRQ; check vendor BIOS\n", - dev->vendor, dev->device); - } status = pcie_port_device_register(dev); if (status) return status; diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c index d0f69269eb6c..ad566827b547 100644 --- a/drivers/pci/quirks.c +++ b/drivers/pci/quirks.c @@ -3405,6 +3405,8 @@ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_ASMEDIA, 0x1080, DECLARE_PCI_FIXUP_HEADER(0x10e3, 0x8113, quirk_use_pcie_bridge_dma_alias); /* ITE 8892, https://bugzilla.kernel.org/show_bug.cgi?id=73551 */ DECLARE_PCI_FIXUP_HEADER(0x1283, 0x8892, quirk_use_pcie_bridge_dma_alias); +/* Intel 82801, https://bugzilla.kernel.org/show_bug.cgi?id=44881#c49 */ +DECLARE_PCI_FIXUP_HEADER(0x8086, 0x244e, quirk_use_pcie_bridge_dma_alias); static struct pci_dev *pci_func_0_dma_source(struct pci_dev *dev) { diff --git a/drivers/pci/setup-bus.c b/drivers/pci/setup-bus.c index a5a63ecfb628..6373985ad3f7 100644 --- a/drivers/pci/setup-bus.c +++ b/drivers/pci/setup-bus.c @@ -925,7 +925,7 @@ static int pbus_size_mem(struct pci_bus *bus, unsigned long mask, { struct pci_dev *dev; resource_size_t min_align, align, size, size0, size1; - resource_size_t aligns[14]; /* Alignments from 1Mb to 8Gb */ + resource_size_t aligns[18]; /* Alignments from 1Mb to 128Gb */ int order, max_order; struct resource *b_res = find_free_bus_resource(bus, mask | IORESOURCE_PREFETCH, type); diff --git a/drivers/pci/setup-res.c b/drivers/pci/setup-res.c index caed1ce6facd..b7c3a5ea1fca 100644 --- a/drivers/pci/setup-res.c +++ b/drivers/pci/setup-res.c @@ -166,11 +166,10 @@ static int pci_revert_fw_address(struct resource *res, struct pci_dev *dev, { struct resource *root, *conflict; resource_size_t fw_addr, start, end; - int ret = 0; fw_addr = pcibios_retrieve_fw_addr(dev, resno); if (!fw_addr) - return 1; + return -ENOMEM; start = res->start; end = res->end; @@ -189,14 +188,13 @@ static int pci_revert_fw_address(struct resource *res, struct pci_dev *dev, resno, res); conflict = request_resource_conflict(root, res); if (conflict) { - dev_info(&dev->dev, - "BAR %d: %pR conflicts with %s %pR\n", resno, - res, conflict->name, conflict); + dev_info(&dev->dev, "BAR %d: %pR conflicts with %s %pR\n", + resno, res, conflict->name, conflict); res->start = start; res->end = end; - ret = 1; + return -EBUSY; } - return ret; + return 0; } static int __pci_assign_resource(struct pci_bus *bus, struct pci_dev *dev, @@ -250,10 +248,8 @@ static int __pci_assign_resource(struct pci_bus *bus, struct pci_dev *dev, static int _pci_assign_resource(struct pci_dev *dev, int resno, resource_size_t size, resource_size_t min_align) { - struct resource *res = dev->resource + resno; struct pci_bus *bus; int ret; - char *type; bus = dev->bus; while ((ret = __pci_assign_resource(bus, dev, resno, size, min_align))) { @@ -262,21 +258,6 @@ static int _pci_assign_resource(struct pci_dev *dev, int resno, bus = bus->parent; } - if (ret) { - if (res->flags & IORESOURCE_MEM) - if (res->flags & IORESOURCE_PREFETCH) - type = "mem pref"; - else - type = "mem"; - else if (res->flags & IORESOURCE_IO) - type = "io"; - else - type = "unknown"; - dev_info(&dev->dev, - "BAR %d: can't assign %s (size %#llx)\n", - resno, type, (unsigned long long) resource_size(res)); - } - return ret; } @@ -302,17 +283,24 @@ int pci_assign_resource(struct pci_dev *dev, int resno) * where firmware left it. That at least has a chance of * working, which is better than just leaving it disabled. */ - if (ret < 0) + if (ret < 0) { + dev_info(&dev->dev, "BAR %d: no space for %pR\n", resno, res); ret = pci_revert_fw_address(res, dev, resno, size); + } - if (!ret) { - res->flags &= ~IORESOURCE_UNSET; - res->flags &= ~IORESOURCE_STARTALIGN; - dev_info(&dev->dev, "BAR %d: assigned %pR\n", resno, res); - if (resno < PCI_BRIDGE_RESOURCES) - pci_update_resource(dev, resno); + if (ret < 0) { + dev_info(&dev->dev, "BAR %d: failed to assign %pR\n", resno, + res); + return ret; } - return ret; + + res->flags &= ~IORESOURCE_UNSET; + res->flags &= ~IORESOURCE_STARTALIGN; + dev_info(&dev->dev, "BAR %d: assigned %pR\n", resno, res); + if (resno < PCI_BRIDGE_RESOURCES) + pci_update_resource(dev, resno); + + return 0; } EXPORT_SYMBOL(pci_assign_resource); @@ -320,9 +308,11 @@ int pci_reassign_resource(struct pci_dev *dev, int resno, resource_size_t addsiz resource_size_t min_align) { struct resource *res = dev->resource + resno; + unsigned long flags; resource_size_t new_size; int ret; + flags = res->flags; res->flags |= IORESOURCE_UNSET; if (!res->parent) { dev_info(&dev->dev, "BAR %d: can't reassign an unassigned resource %pR\n", @@ -333,14 +323,21 @@ int pci_reassign_resource(struct pci_dev *dev, int resno, resource_size_t addsiz /* already aligned with min_align */ new_size = resource_size(res) + addsize; ret = _pci_assign_resource(dev, resno, new_size, min_align); - if (!ret) { - res->flags &= ~IORESOURCE_UNSET; - res->flags &= ~IORESOURCE_STARTALIGN; - dev_info(&dev->dev, "BAR %d: reassigned %pR\n", resno, res); - if (resno < PCI_BRIDGE_RESOURCES) - pci_update_resource(dev, resno); + if (ret) { + res->flags = flags; + dev_info(&dev->dev, "BAR %d: %pR (failed to expand by %#llx)\n", + resno, res, (unsigned long long) addsize); + return ret; } - return ret; + + res->flags &= ~IORESOURCE_UNSET; + res->flags &= ~IORESOURCE_STARTALIGN; + dev_info(&dev->dev, "BAR %d: reassigned %pR (expanded by %#llx)\n", + resno, res, (unsigned long long) addsize); + if (resno < PCI_BRIDGE_RESOURCES) + pci_update_resource(dev, resno); + + return 0; } int pci_enable_resources(struct pci_dev *dev, int mask) diff --git a/drivers/scsi/scsi_trace.c b/drivers/scsi/scsi_trace.c index 2bea4f0b684a..503594e5f76d 100644 --- a/drivers/scsi/scsi_trace.c +++ b/drivers/scsi/scsi_trace.c @@ -28,7 +28,7 @@ scsi_trace_misc(struct trace_seq *, unsigned char *, int); static const char * scsi_trace_rw6(struct trace_seq *p, unsigned char *cdb, int len) { - const char *ret = p->buffer + p->len; + const char *ret = trace_seq_buffer_ptr(p); sector_t lba = 0, txlen = 0; lba |= ((cdb[1] & 0x1F) << 16); @@ -46,7 +46,7 @@ scsi_trace_rw6(struct trace_seq *p, unsigned char *cdb, int len) static const char * scsi_trace_rw10(struct trace_seq *p, unsigned char *cdb, int len) { - const char *ret = p->buffer + p->len; + const char *ret = trace_seq_buffer_ptr(p); sector_t lba = 0, txlen = 0; lba |= (cdb[2] << 24); @@ -71,7 +71,7 @@ scsi_trace_rw10(struct trace_seq *p, unsigned char *cdb, int len) static const char * scsi_trace_rw12(struct trace_seq *p, unsigned char *cdb, int len) { - const char *ret = p->buffer + p->len; + const char *ret = trace_seq_buffer_ptr(p); sector_t lba = 0, txlen = 0; lba |= (cdb[2] << 24); @@ -94,7 +94,7 @@ scsi_trace_rw12(struct trace_seq *p, unsigned char *cdb, int len) static const char * scsi_trace_rw16(struct trace_seq *p, unsigned char *cdb, int len) { - const char *ret = p->buffer + p->len; + const char *ret = trace_seq_buffer_ptr(p); sector_t lba = 0, txlen = 0; lba |= ((u64)cdb[2] << 56); @@ -125,7 +125,7 @@ scsi_trace_rw16(struct trace_seq *p, unsigned char *cdb, int len) static const char * scsi_trace_rw32(struct trace_seq *p, unsigned char *cdb, int len) { - const char *ret = p->buffer + p->len, *cmd; + const char *ret = trace_seq_buffer_ptr(p), *cmd; sector_t lba = 0, txlen = 0; u32 ei_lbrt = 0; @@ -180,7 +180,7 @@ out: static const char * scsi_trace_unmap(struct trace_seq *p, unsigned char *cdb, int len) { - const char *ret = p->buffer + p->len; + const char *ret = trace_seq_buffer_ptr(p); unsigned int regions = cdb[7] << 8 | cdb[8]; trace_seq_printf(p, "regions=%u", (regions - 8) / 16); @@ -192,7 +192,7 @@ scsi_trace_unmap(struct trace_seq *p, unsigned char *cdb, int len) static const char * scsi_trace_service_action_in(struct trace_seq *p, unsigned char *cdb, int len) { - const char *ret = p->buffer + p->len, *cmd; + const char *ret = trace_seq_buffer_ptr(p), *cmd; sector_t lba = 0; u32 alloc_len = 0; @@ -247,7 +247,7 @@ scsi_trace_varlen(struct trace_seq *p, unsigned char *cdb, int len) static const char * scsi_trace_misc(struct trace_seq *p, unsigned char *cdb, int len) { - const char *ret = p->buffer + p->len; + const char *ret = trace_seq_buffer_ptr(p); trace_seq_printf(p, "-"); trace_seq_putc(p, 0); diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c index d4f9670b51bc..22aa41cace82 100644 --- a/drivers/spi/spi.c +++ b/drivers/spi/spi.c @@ -29,6 +29,7 @@ #include <linux/mutex.h> #include <linux/of_device.h> #include <linux/of_irq.h> +#include <linux/clk/clk-conf.h> #include <linux/slab.h> #include <linux/mod_devicetable.h> #include <linux/spi/spi.h> @@ -259,6 +260,10 @@ static int spi_drv_probe(struct device *dev) const struct spi_driver *sdrv = to_spi_driver(dev->driver); int ret; + ret = of_clk_set_defaults(dev->of_node, false); + if (ret) + return ret; + acpi_dev_pm_attach(dev, true); ret = sdrv->probe(to_spi_device(dev)); if (ret) diff --git a/drivers/target/target_core_tpg.c b/drivers/target/target_core_tpg.c index c036595b17cf..fddfae61222f 100644 --- a/drivers/target/target_core_tpg.c +++ b/drivers/target/target_core_tpg.c @@ -825,7 +825,7 @@ int core_tpg_add_lun( ret = core_dev_export(dev, tpg, lun); if (ret < 0) { - percpu_ref_cancel_init(&lun->lun_ref); + percpu_ref_exit(&lun->lun_ref); return ret; } @@ -880,5 +880,7 @@ int core_tpg_post_dellun( lun->lun_status = TRANSPORT_LUN_STATUS_FREE; spin_unlock(&tpg->tpg_lun_lock); + percpu_ref_exit(&lun->lun_ref); + return 0; } diff --git a/drivers/video/fbdev/efifb.c b/drivers/video/fbdev/efifb.c index ae9618ff6735..982f6abe6faf 100644 --- a/drivers/video/fbdev/efifb.c +++ b/drivers/video/fbdev/efifb.c @@ -19,8 +19,6 @@ static bool request_mem_succeeded = false; -static struct pci_dev *default_vga; - static struct fb_var_screeninfo efifb_defined = { .activate = FB_ACTIVATE_NOW, .height = -1, @@ -84,23 +82,10 @@ static struct fb_ops efifb_ops = { .fb_imageblit = cfb_imageblit, }; -struct pci_dev *vga_default_device(void) -{ - return default_vga; -} - -EXPORT_SYMBOL_GPL(vga_default_device); - -void vga_set_default_device(struct pci_dev *pdev) -{ - default_vga = pdev; -} - static int efifb_setup(char *options) { char *this_opt; int i; - struct pci_dev *dev = NULL; if (options && *options) { while ((this_opt = strsep(&options, ",")) != NULL) { @@ -126,30 +111,6 @@ static int efifb_setup(char *options) } } - for_each_pci_dev(dev) { - int i; - - if ((dev->class >> 8) != PCI_CLASS_DISPLAY_VGA) - continue; - - for (i=0; i < DEVICE_COUNT_RESOURCE; i++) { - resource_size_t start, end; - - if (!(pci_resource_flags(dev, i) & IORESOURCE_MEM)) - continue; - - start = pci_resource_start(dev, i); - end = pci_resource_end(dev, i); - - if (!start || !end) - continue; - - if (screen_info.lfb_base >= start && - (screen_info.lfb_base + screen_info.lfb_size) < end) - default_vga = dev; - } - } - return 0; } diff --git a/drivers/zorro/names.c b/drivers/zorro/names.c index 6f3fd9903ac3..83eedddbb794 100644 --- a/drivers/zorro/names.c +++ b/drivers/zorro/names.c @@ -46,13 +46,13 @@ struct zorro_manuf_info { #include "devlist.h" static struct zorro_manuf_info __initdata zorro_manuf_list[] = { -#define MANUF( manuf, name ) { 0x##manuf, sizeof(__prods_##manuf) / sizeof(struct zorro_prod_info), __manufstr_##manuf, __prods_##manuf }, +#define MANUF( manuf, name ) { 0x##manuf, ARRAY_SIZE(__prods_##manuf), __manufstr_##manuf, __prods_##manuf }, #define ENDMANUF() #define PRODUCT( manuf, prod, name ) #include "devlist.h" }; -#define MANUFS (sizeof(zorro_manuf_list)/sizeof(struct zorro_manuf_info)) +#define MANUFS ARRAY_SIZE(zorro_manuf_list) void __init zorro_name_device(struct zorro_dev *dev) { |