From d6a6af2c181400aade59417e698c7cd9bec8804e Mon Sep 17 00:00:00 2001 From: Ian Munsie Date: Mon, 8 Dec 2014 19:17:59 +1100 Subject: cxl: Disable AFU debug flag Upon inspection of the implementation specific registers, it was discovered that the high bit of the implementation specific RXCTL register was enabled, which enables the DEADB00F debug feature. The debug feature causes MMIO reads to a disabled AFU to respond with 0xDEADB00F instead of all Fs. In general this should not be visible as the kernel will only allow MMIO access to enabled AFUs, but there may be some circumstances where an AFU may become disabled while it is use. One such case would be an AFU designed to only be used in the dedicated process mode and to disable itself after it has completed it's work (however even in that case the effects of this debug flag would be limited as the userspace application must have completed any required MMIO accesses before the AFU disables itself with or without the flag). This patch removes the debug flag and replaces the magic value programmed into this register with a preprocessor define so it is clearer what the rest of this initialisation does. Signed-off-by: Ian Munsie Signed-off-by: Michael Ellerman --- drivers/misc/cxl/pci.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/misc/cxl/pci.c') diff --git a/drivers/misc/cxl/pci.c b/drivers/misc/cxl/pci.c index 0f2cc9f8b4db..2ccd0a91d486 100644 --- a/drivers/misc/cxl/pci.c +++ b/drivers/misc/cxl/pci.c @@ -348,7 +348,7 @@ static int init_implementation_afu_regs(struct cxl_afu *afu) cxl_p1n_write(afu, CXL_PSL_COALLOC_A, 0xFF000000FEFEFEFEULL); /* for debugging with trace arrays */ cxl_p1n_write(afu, CXL_PSL_SLICE_TRACE, 0x0000FFFF00000000ULL); - cxl_p1n_write(afu, CXL_PSL_RXCTL_A, 0xF000000000000000ULL); + cxl_p1n_write(afu, CXL_PSL_RXCTL_A, CXL_PSL_RXCTL_AFUHP_4S); return 0; } -- cgit v1.2.3 From 4beb5421babee1204757b877622830c6aa31be6d Mon Sep 17 00:00:00 2001 From: Ryan Grimm Date: Mon, 19 Jan 2015 11:52:48 -0600 Subject: cxl: Use image state defaults for reloading FPGA Select defaults such that a PERST causes flash image reload. Select which image based on what the card is set up to load. CXL_VSEC_PERST_LOADS_IMAGE selects whether PERST assertion causes flash image load. CXL_VSEC_PERST_SELECT_USER selects which image is loaded on the next PERST. cxl_update_image_control writes these bits into the VSEC. Signed-off-by: Ryan Grimm Acked-by: Ian Munsie Signed-off-by: Michael Ellerman --- drivers/misc/cxl/cxl.h | 1 + drivers/misc/cxl/pci.c | 42 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 41 insertions(+), 2 deletions(-) (limited to 'drivers/misc/cxl/pci.c') diff --git a/drivers/misc/cxl/cxl.h b/drivers/misc/cxl/cxl.h index 0df04380bfd6..518c4c6e6151 100644 --- a/drivers/misc/cxl/cxl.h +++ b/drivers/misc/cxl/cxl.h @@ -488,6 +488,7 @@ void cxl_release_one_irq(struct cxl *adapter, int hwirq); int cxl_alloc_irq_ranges(struct cxl_irq_ranges *irqs, struct cxl *adapter, unsigned int num); void cxl_release_irq_ranges(struct cxl_irq_ranges *irqs, struct cxl *adapter); int cxl_setup_irq(struct cxl *adapter, unsigned int hwirq, unsigned int virq); +int cxl_update_image_control(struct cxl *adapter); /* common == phyp + powernv */ struct cxl_process_element_common { diff --git a/drivers/misc/cxl/pci.c b/drivers/misc/cxl/pci.c index 2ccd0a91d486..014f4c928e4c 100644 --- a/drivers/misc/cxl/pci.c +++ b/drivers/misc/cxl/pci.c @@ -361,6 +361,41 @@ int cxl_setup_irq(struct cxl *adapter, unsigned int hwirq, return pnv_cxl_ioda_msi_setup(dev, hwirq, virq); } +int cxl_update_image_control(struct cxl *adapter) +{ + struct pci_dev *dev = to_pci_dev(adapter->dev.parent); + int rc; + int vsec; + u8 image_state; + + if (!(vsec = find_cxl_vsec(dev))) { + dev_err(&dev->dev, "ABORTING: CXL VSEC not found!\n"); + return -ENODEV; + } + + if ((rc = CXL_READ_VSEC_IMAGE_STATE(dev, vsec, &image_state))) { + dev_err(&dev->dev, "failed to read image state: %i\n", rc); + return rc; + } + + if (adapter->perst_loads_image) + image_state |= CXL_VSEC_PERST_LOADS_IMAGE; + else + image_state &= ~CXL_VSEC_PERST_LOADS_IMAGE; + + if (adapter->perst_select_user) + image_state |= CXL_VSEC_PERST_SELECT_USER; + else + image_state &= ~CXL_VSEC_PERST_SELECT_USER; + + if ((rc = CXL_WRITE_VSEC_IMAGE_STATE(dev, vsec, image_state))) { + dev_err(&dev->dev, "failed to update image control: %i\n", rc); + return rc; + } + + return 0; +} + int cxl_alloc_one_irq(struct cxl *adapter) { struct pci_dev *dev = to_pci_dev(adapter->dev.parent); @@ -770,8 +805,8 @@ static int cxl_read_vsec(struct cxl *adapter, struct pci_dev *dev) CXL_READ_VSEC_BASE_IMAGE(dev, vsec, &adapter->base_image); CXL_READ_VSEC_IMAGE_STATE(dev, vsec, &image_state); adapter->user_image_loaded = !!(image_state & CXL_VSEC_USER_IMAGE_LOADED); - adapter->perst_loads_image = !!(image_state & CXL_VSEC_PERST_LOADS_IMAGE); - adapter->perst_select_user = !!(image_state & CXL_VSEC_PERST_SELECT_USER); + adapter->perst_loads_image = true; + adapter->perst_select_user = !!(image_state & CXL_VSEC_USER_IMAGE_LOADED); CXL_READ_VSEC_NAFUS(dev, vsec, &adapter->slices); CXL_READ_VSEC_AFU_DESC_OFF(dev, vsec, &afu_desc_off); @@ -879,6 +914,9 @@ static struct cxl *cxl_init_adapter(struct pci_dev *dev) if ((rc = cxl_vsec_looks_ok(adapter, dev))) goto err2; + if ((rc = cxl_update_image_control(adapter))) + goto err2; + if ((rc = cxl_map_adapter_regs(adapter, dev))) goto err2; -- cgit v1.2.3 From 1212aa1c8c9ca34642f7737e1edaa96c9ce3d7dd Mon Sep 17 00:00:00 2001 From: Ryan Grimm Date: Mon, 19 Jan 2015 11:52:50 -0600 Subject: cxl: Enable CAPP recovery Turning snoops on is the last step in CAPP recovery. Sapphire is expected to have reinitialized the PHB and done the previous recovery steps. Add mode argument to opal call to do this. Driver can turn snoops off although it does not currently. Signed-off-by: Ryan Grimm Acked-by: Ian Munsie Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/opal.h | 8 ++++++++ arch/powerpc/include/asm/pnv-pci.h | 2 +- arch/powerpc/platforms/powernv/pci-ioda.c | 6 +++--- drivers/misc/cxl/pci.c | 8 +++++++- 4 files changed, 19 insertions(+), 5 deletions(-) (limited to 'drivers/misc/cxl/pci.c') diff --git a/arch/powerpc/include/asm/opal.h b/arch/powerpc/include/asm/opal.h index eb95b675109b..2baf8a5925ca 100644 --- a/arch/powerpc/include/asm/opal.h +++ b/arch/powerpc/include/asm/opal.h @@ -595,6 +595,14 @@ enum { OPAL_PHB3_NUM_PEST_REGS = 256 }; +/* CAPI modes for PHB */ +enum { + OPAL_PHB_CAPI_MODE_PCIE = 0, + OPAL_PHB_CAPI_MODE_CAPI = 1, + OPAL_PHB_CAPI_MODE_SNOOP_OFF = 2, + OPAL_PHB_CAPI_MODE_SNOOP_ON = 3, +}; + struct OpalIoPhbErrorCommon { __be32 version; __be32 ioType; diff --git a/arch/powerpc/include/asm/pnv-pci.h b/arch/powerpc/include/asm/pnv-pci.h index f09a22fa1bd7..3c00d648336d 100644 --- a/arch/powerpc/include/asm/pnv-pci.h +++ b/arch/powerpc/include/asm/pnv-pci.h @@ -13,7 +13,7 @@ #include #include -int pnv_phb_to_cxl(struct pci_dev *dev); +int pnv_phb_to_cxl_mode(struct pci_dev *dev, uint64_t mode); int pnv_cxl_ioda_msi_setup(struct pci_dev *dev, unsigned int hwirq, unsigned int virq); int pnv_cxl_alloc_hwirqs(struct pci_dev *dev, int num); diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c index fac88ed8a915..5d52d6f274f8 100644 --- a/arch/powerpc/platforms/powernv/pci-ioda.c +++ b/arch/powerpc/platforms/powernv/pci-ioda.c @@ -1468,7 +1468,7 @@ struct device_node *pnv_pci_to_phb_node(struct pci_dev *dev) } EXPORT_SYMBOL(pnv_pci_to_phb_node); -int pnv_phb_to_cxl(struct pci_dev *dev) +int pnv_phb_to_cxl_mode(struct pci_dev *dev, uint64_t mode) { struct pci_controller *hose = pci_bus_to_host(dev->bus); struct pnv_phb *phb = hose->private_data; @@ -1481,13 +1481,13 @@ int pnv_phb_to_cxl(struct pci_dev *dev) pe_info(pe, "Switching PHB to CXL\n"); - rc = opal_pci_set_phb_cxl_mode(phb->opal_id, 1, pe->pe_number); + rc = opal_pci_set_phb_cxl_mode(phb->opal_id, mode, pe->pe_number); if (rc) dev_err(&dev->dev, "opal_pci_set_phb_cxl_mode failed: %i\n", rc); return rc; } -EXPORT_SYMBOL(pnv_phb_to_cxl); +EXPORT_SYMBOL(pnv_phb_to_cxl_mode); /* Find PHB for cxl dev and allocate MSI hwirqs? * Returns the absolute hardware IRQ number diff --git a/drivers/misc/cxl/pci.c b/drivers/misc/cxl/pci.c index 014f4c928e4c..a4a4e0217eed 100644 --- a/drivers/misc/cxl/pci.c +++ b/drivers/misc/cxl/pci.c @@ -926,9 +926,15 @@ static struct cxl *cxl_init_adapter(struct pci_dev *dev) if ((rc = init_implementation_adapter_regs(adapter, dev))) goto err3; - if ((rc = pnv_phb_to_cxl(dev))) + if ((rc = pnv_phb_to_cxl_mode(dev, OPAL_PHB_CAPI_MODE_CAPI))) goto err3; + /* If recovery happened, the last step is to turn on snooping. + * In the non-recovery case this has no effect */ + if ((rc = pnv_phb_to_cxl_mode(dev, OPAL_PHB_CAPI_MODE_SNOOP_ON))) { + goto err3; + } + if ((rc = cxl_register_psl_err_irq(adapter))) goto err3; -- cgit v1.2.3 From 62fa19d4b4fd781ad37c9155c6332f28a9e97a2c Mon Sep 17 00:00:00 2001 From: Ryan Grimm Date: Mon, 19 Jan 2015 11:52:51 -0600 Subject: cxl: Add ability to reset the card Adds reset to sysfs which will PERST the card. If load_image_on_perst is set to "user" or "factory", the PERST will cause that image to be loaded. load_image_on_perst is set to "user" for production. "none" could be used for debugging. The PSL trace arrays are preserved which then can be read through debugfs. PERST also triggers CAPP recovery. An HMI comes in, which is handled by EEH. EEH unbinds the driver, calls into Sapphire to reinitialize the PHB, then rebinds the driver. Signed-off-by: Ryan Grimm Acked-by: Ian Munsie Signed-off-by: Michael Ellerman --- Documentation/ABI/testing/sysfs-class-cxl | 7 ++++++ drivers/misc/cxl/cxl.h | 1 + drivers/misc/cxl/pci.c | 37 +++++++++++++++++++++++++++++++ drivers/misc/cxl/sysfs.c | 18 +++++++++++++++ 4 files changed, 63 insertions(+) (limited to 'drivers/misc/cxl/pci.c') diff --git a/Documentation/ABI/testing/sysfs-class-cxl b/Documentation/ABI/testing/sysfs-class-cxl index 5941ff38d4a3..9ea01068a16c 100644 --- a/Documentation/ABI/testing/sysfs-class-cxl +++ b/Documentation/ABI/testing/sysfs-class-cxl @@ -146,3 +146,10 @@ Description: read/write user or factory image to be loaded. Default is to reload on PERST whichever image the card has loaded. + +What: /sys/class/cxl//reset +Date: October 2014 +Contact: linuxppc-dev@lists.ozlabs.org +Description: write only + Writing 1 will issue a PERST to card which may cause the card + to reload the FPGA depending on load_image_on_perst. diff --git a/drivers/misc/cxl/cxl.h b/drivers/misc/cxl/cxl.h index 518c4c6e6151..6a6a487464c5 100644 --- a/drivers/misc/cxl/cxl.h +++ b/drivers/misc/cxl/cxl.h @@ -489,6 +489,7 @@ int cxl_alloc_irq_ranges(struct cxl_irq_ranges *irqs, struct cxl *adapter, unsig void cxl_release_irq_ranges(struct cxl_irq_ranges *irqs, struct cxl *adapter); int cxl_setup_irq(struct cxl *adapter, unsigned int hwirq, unsigned int virq); int cxl_update_image_control(struct cxl *adapter); +int cxl_reset(struct cxl *adapter); /* common == phyp + powernv */ struct cxl_process_element_common { diff --git a/drivers/misc/cxl/pci.c b/drivers/misc/cxl/pci.c index a4a4e0217eed..428ea8ba25fc 100644 --- a/drivers/misc/cxl/pci.c +++ b/drivers/misc/cxl/pci.c @@ -21,6 +21,7 @@ #include #include /* for struct pci_controller */ #include +#include #include "cxl.h" @@ -741,6 +742,42 @@ static void cxl_remove_afu(struct cxl_afu *afu) device_unregister(&afu->dev); } +int cxl_reset(struct cxl *adapter) +{ + struct pci_dev *dev = to_pci_dev(adapter->dev.parent); + int rc; + int i; + u32 val; + + dev_info(&dev->dev, "CXL reset\n"); + + for (i = 0; i < adapter->slices; i++) + cxl_remove_afu(adapter->afu[i]); + + /* pcie_warm_reset requests a fundamental pci reset which includes a + * PERST assert/deassert. PERST triggers a loading of the image + * if "user" or "factory" is selected in sysfs */ + if ((rc = pci_set_pcie_reset_state(dev, pcie_warm_reset))) { + dev_err(&dev->dev, "cxl: pcie_warm_reset failed\n"); + return rc; + } + + /* the PERST done above fences the PHB. So, reset depends on EEH + * to unbind the driver, tell Sapphire to reinit the PHB, and rebind + * the driver. Do an mmio read explictly to ensure EEH notices the + * fenced PHB. Retry for a few seconds before giving up. */ + i = 0; + while (((val = mmio_read32be(adapter->p1_mmio)) != 0xffffffff) && + (i < 5)) { + msleep(500); + i++; + } + + if (val != 0xffffffff) + dev_err(&dev->dev, "cxl: PERST failed to trigger EEH\n"); + + return rc; +} static int cxl_map_adapter_regs(struct cxl *adapter, struct pci_dev *dev) { diff --git a/drivers/misc/cxl/sysfs.c b/drivers/misc/cxl/sysfs.c index ed4ad461143c..adf1f6d84913 100644 --- a/drivers/misc/cxl/sysfs.c +++ b/drivers/misc/cxl/sysfs.c @@ -56,6 +56,23 @@ static ssize_t image_loaded_show(struct device *device, return scnprintf(buf, PAGE_SIZE, "factory\n"); } +static ssize_t reset_adapter_store(struct device *device, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct cxl *adapter = to_cxl_adapter(device); + int rc; + int val; + + rc = sscanf(buf, "%i", &val); + if ((rc != 1) || (val != 1)) + return -EINVAL; + + if ((rc = cxl_reset(adapter))) + return rc; + return count; +} + static ssize_t load_image_on_perst_show(struct device *device, struct device_attribute *attr, char *buf) @@ -100,6 +117,7 @@ static struct device_attribute adapter_attrs[] = { __ATTR_RO(base_image), __ATTR_RO(image_loaded), __ATTR_RW(load_image_on_perst), + __ATTR(reset, S_IWUSR, NULL, reset_adapter_store), }; -- cgit v1.2.3 From 6f963ec2d6bf2476a16799eece920acb2100ff1c Mon Sep 17 00:00:00 2001 From: Ryan Grimm Date: Wed, 28 Jan 2015 20:16:04 -0600 Subject: cxl: Fix device_node reference counting When unbinding and rebinding the driver on a system with a card in PHB0, this error condition is reached after a few attempts: ERROR: Bad of_node_put() on /pciex@3fffe40000000 CPU: 0 PID: 3040 Comm: bash Not tainted 3.18.0-rc3-12545-g3627ffe #152 Call Trace: [c000000721acb5c0] [c00000000086ef94] .dump_stack+0x84/0xb0 (unreliable) [c000000721acb640] [c00000000073a0a8] .of_node_release+0xd8/0xe0 [c000000721acb6d0] [c00000000044bc44] .kobject_release+0x74/0xe0 [c000000721acb760] [c0000000007394fc] .of_node_put+0x1c/0x30 [c000000721acb7d0] [c000000000545cd8] .cxl_probe+0x1a98/0x1d50 [c000000721acb900] [c0000000004845a0] .local_pci_probe+0x40/0xc0 [c000000721acb980] [c000000000484998] .pci_device_probe+0x128/0x170 [c000000721acba30] [c00000000052400c] .driver_probe_device+0xac/0x2a0 [c000000721acbad0] [c000000000522468] .bind_store+0x108/0x160 [c000000721acbb70] [c000000000521448] .drv_attr_store+0x38/0x60 [c000000721acbbe0] [c000000000293840] .sysfs_kf_write+0x60/0xa0 [c000000721acbc50] [c000000000292500] .kernfs_fop_write+0x140/0x1d0 [c000000721acbcf0] [c000000000208648] .vfs_write+0xd8/0x260 [c000000721acbd90] [c000000000208b18] .SyS_write+0x58/0x100 [c000000721acbe30] [c000000000009258] syscall_exit+0x0/0x98 We are missing a call to of_node_get(). pnv_pci_to_phb_node() should call of_node_get() otherwise np's reference count isn't incremented and it might go away. Rename pnv_pci_to_phb_node() to pnv_pci_get_phb_node() so it's clear it calls of_node_get(). Signed-off-by: Ryan Grimm Acked-by: Ian Munsie Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/pnv-pci.h | 2 +- arch/powerpc/platforms/powernv/pci-ioda.c | 6 +++--- drivers/misc/cxl/pci.c | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) (limited to 'drivers/misc/cxl/pci.c') diff --git a/arch/powerpc/include/asm/pnv-pci.h b/arch/powerpc/include/asm/pnv-pci.h index 3c00d648336d..f9b498292a5c 100644 --- a/arch/powerpc/include/asm/pnv-pci.h +++ b/arch/powerpc/include/asm/pnv-pci.h @@ -19,7 +19,7 @@ int pnv_cxl_ioda_msi_setup(struct pci_dev *dev, unsigned int hwirq, int pnv_cxl_alloc_hwirqs(struct pci_dev *dev, int num); void pnv_cxl_release_hwirqs(struct pci_dev *dev, int hwirq, int num); int pnv_cxl_get_irq_count(struct pci_dev *dev); -struct device_node *pnv_pci_to_phb_node(struct pci_dev *dev); +struct device_node *pnv_pci_get_phb_node(struct pci_dev *dev); #ifdef CONFIG_CXL_BASE int pnv_cxl_alloc_hwirq_ranges(struct cxl_irq_ranges *irqs, diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c index 85b473823fda..6c9ff2b95119 100644 --- a/arch/powerpc/platforms/powernv/pci-ioda.c +++ b/arch/powerpc/platforms/powernv/pci-ioda.c @@ -1487,13 +1487,13 @@ static void set_msi_irq_chip(struct pnv_phb *phb, unsigned int virq) #ifdef CONFIG_CXL_BASE -struct device_node *pnv_pci_to_phb_node(struct pci_dev *dev) +struct device_node *pnv_pci_get_phb_node(struct pci_dev *dev) { struct pci_controller *hose = pci_bus_to_host(dev->bus); - return hose->dn; + return of_node_get(hose->dn); } -EXPORT_SYMBOL(pnv_pci_to_phb_node); +EXPORT_SYMBOL(pnv_pci_get_phb_node); int pnv_phb_to_cxl_mode(struct pci_dev *dev, uint64_t mode) { diff --git a/drivers/misc/cxl/pci.c b/drivers/misc/cxl/pci.c index 428ea8ba25fc..cb250673b5c6 100644 --- a/drivers/misc/cxl/pci.c +++ b/drivers/misc/cxl/pci.c @@ -317,7 +317,7 @@ static int init_implementation_adapter_regs(struct cxl *adapter, struct pci_dev u64 psl_dsnctl; u64 chipid; - if (!(np = pnv_pci_to_phb_node(dev))) + if (!(np = pnv_pci_get_phb_node(dev))) return -ENODEV; while (np && !(prop = of_get_property(np, "ibm,chip-id", NULL))) -- cgit v1.2.3 From b087e6190ddcd9ae4e8ff2c788d2b32f193e946b Mon Sep 17 00:00:00 2001 From: Ian Munsie Date: Wed, 4 Feb 2015 19:09:01 +1100 Subject: cxl: Export optional AFU configuration record in sysfs An AFU may optionally contain one or more PCIe like configuration records, which can be used to identify the AFU. This patch adds support for exposing the raw config space and the vendor, device and class code under sysfs. These will appear in a subdirectory of the AFU device corresponding with the configuration record number, e.g. cat /sys/class/cxl/afu0.0/cr0/vendor 0x1014 cat /sys/class/cxl/afu0.0/cr0/device 0x4350 cat /sys/class/cxl/afu0.0/cr0/class 0x120000 hexdump -C /sys/class/cxl/afu0.0/cr0/config 00000000 14 10 50 43 00 00 00 00 06 00 00 12 00 00 00 00 |..PC............| 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| * 00000100 These files behave in much the same way as the equivalent files for PCI devices, with one exception being that the config file is currently read-only and restricted to the root user. It is not necessarily required to be this strict, but we currently do not have a compelling use-case to make it writable and/or world-readable, so I erred on the side of being restrictive. Signed-off-by: Ian Munsie Signed-off-by: Michael Ellerman --- Documentation/ABI/testing/sysfs-class-cxl | 37 ++++++ drivers/misc/cxl/cxl.h | 13 +++ drivers/misc/cxl/pci.c | 23 ++++ drivers/misc/cxl/sysfs.c | 179 ++++++++++++++++++++++++++++-- 4 files changed, 242 insertions(+), 10 deletions(-) (limited to 'drivers/misc/cxl/pci.c') diff --git a/Documentation/ABI/testing/sysfs-class-cxl b/Documentation/ABI/testing/sysfs-class-cxl index 9ea01068a16c..3680364b4048 100644 --- a/Documentation/ABI/testing/sysfs-class-cxl +++ b/Documentation/ABI/testing/sysfs-class-cxl @@ -81,6 +81,43 @@ Description: read only this this kernel supports. +AFU configuration records (eg. /sys/class/cxl/afu0.0/cr0): + +An AFU may optionally export one or more PCIe like configuration records, known +as AFU configuration records, which will show up here (if present). + +What: /sys/class/cxl//cr/vendor +Date: February 2015 +Contact: linuxppc-dev@lists.ozlabs.org +Description: read only + Hexadecimal value of the vendor ID found in this AFU + configuration record. + +What: /sys/class/cxl//cr/device +Date: February 2015 +Contact: linuxppc-dev@lists.ozlabs.org +Description: read only + Hexadecimal value of the device ID found in this AFU + configuration record. + +What: /sys/class/cxl//cr/vendor +Date: February 2015 +Contact: linuxppc-dev@lists.ozlabs.org +Description: read only + Hexadecimal value of the class code found in this AFU + configuration record. + +What: /sys/class/cxl//cr/config +Date: February 2015 +Contact: linuxppc-dev@lists.ozlabs.org +Description: read only + This binary file provides raw access to the AFU configuration + record. The format is expected to match the either the standard + or extended configuration space defined by the PCIe + specification. + + + Master contexts (eg. /sys/class/cxl/afu0.0m) What: /sys/class/cxl/m/mmio_size diff --git a/drivers/misc/cxl/cxl.h b/drivers/misc/cxl/cxl.h index 6a6a487464c5..a1cee4767ec6 100644 --- a/drivers/misc/cxl/cxl.h +++ b/drivers/misc/cxl/cxl.h @@ -382,6 +382,10 @@ struct cxl_afu { int slice; int modes_supported; int current_mode; + int crs_num; + u64 crs_len; + u64 crs_offset; + struct list_head crs; enum prefault_modes prefault_mode; bool psa; bool pp_psa; @@ -551,6 +555,15 @@ static inline void __iomem *_cxl_p2n_addr(struct cxl_afu *afu, cxl_p2n_reg_t reg #define cxl_p2n_read(afu, reg) \ in_be64(_cxl_p2n_addr(afu, reg)) + +#define cxl_afu_cr_read64(afu, cr, off) \ + in_le64((afu)->afu_desc_mmio + (afu)->crs_offset + ((cr) * (afu)->crs_len) + (off)) +#define cxl_afu_cr_read32(afu, cr, off) \ + in_le32((afu)->afu_desc_mmio + (afu)->crs_offset + ((cr) * (afu)->crs_len) + (off)) +u16 cxl_afu_cr_read16(struct cxl_afu *afu, int cr, u64 off); +u8 cxl_afu_cr_read8(struct cxl_afu *afu, int cr, u64 off); + + struct cxl_calls { void (*cxl_slbia)(struct mm_struct *mm); struct module *owner; diff --git a/drivers/misc/cxl/pci.c b/drivers/misc/cxl/pci.c index cb250673b5c6..2b2e1b80d759 100644 --- a/drivers/misc/cxl/pci.c +++ b/drivers/misc/cxl/pci.c @@ -114,6 +114,24 @@ #define AFUD_EB_LEN(val) EXTRACT_PPC_BITS(val, 8, 63) #define AFUD_READ_EB_OFF(afu) AFUD_READ(afu, 0x48) +u16 cxl_afu_cr_read16(struct cxl_afu *afu, int cr, u64 off) +{ + u64 aligned_off = off & ~0x3L; + u32 val; + + val = cxl_afu_cr_read32(afu, cr, aligned_off); + return (val >> ((off & 0x2) * 8)) & 0xffff; +} + +u8 cxl_afu_cr_read8(struct cxl_afu *afu, int cr, u64 off) +{ + u64 aligned_off = off & ~0x3L; + u32 val; + + val = cxl_afu_cr_read32(afu, cr, aligned_off); + return (val >> ((off & 0x3) * 8)) & 0xff; +} + static DEFINE_PCI_DEVICE_TABLE(cxl_pci_tbl) = { { PCI_DEVICE(PCI_VENDOR_ID_IBM, 0x0477), }, { PCI_DEVICE(PCI_VENDOR_ID_IBM, 0x044b), }, @@ -556,6 +574,7 @@ static int cxl_read_afu_descriptor(struct cxl_afu *afu) val = AFUD_READ_INFO(afu); afu->pp_irqs = AFUD_NUM_INTS_PER_PROC(val); afu->max_procs_virtualised = AFUD_NUM_PROCS(val); + afu->crs_num = AFUD_NUM_CRS(val); if (AFUD_AFU_DIRECTED(val)) afu->modes_supported |= CXL_MODE_DIRECTED; @@ -570,6 +589,10 @@ static int cxl_read_afu_descriptor(struct cxl_afu *afu) if ((afu->pp_psa = AFUD_PPPSA_PP(val))) afu->pp_offset = AFUD_READ_PPPSA_OFF(afu); + val = AFUD_READ_CR(afu); + afu->crs_len = AFUD_CR_LEN(val) * 256; + afu->crs_offset = AFUD_READ_CR_OFF(afu); + return 0; } diff --git a/drivers/misc/cxl/sysfs.c b/drivers/misc/cxl/sysfs.c index adf1f6d84913..d0c38c7bc0c4 100644 --- a/drivers/misc/cxl/sysfs.c +++ b/drivers/misc/cxl/sysfs.c @@ -10,6 +10,7 @@ #include #include #include +#include #include "cxl.h" @@ -367,8 +368,6 @@ static struct device_attribute afu_attrs[] = { __ATTR(reset, S_IWUSR, NULL, reset_store_afu), }; - - int cxl_sysfs_adapter_add(struct cxl *adapter) { int i, rc; @@ -391,31 +390,191 @@ void cxl_sysfs_adapter_remove(struct cxl *adapter) device_remove_file(&adapter->dev, &adapter_attrs[i]); } +struct afu_config_record { + struct kobject kobj; + struct bin_attribute config_attr; + struct list_head list; + int cr; + u16 device; + u16 vendor; + u32 class; +}; + +#define to_cr(obj) container_of(obj, struct afu_config_record, kobj) + +static ssize_t vendor_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct afu_config_record *cr = to_cr(kobj); + + return scnprintf(buf, PAGE_SIZE, "0x%.4x\n", cr->vendor); +} + +static ssize_t device_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct afu_config_record *cr = to_cr(kobj); + + return scnprintf(buf, PAGE_SIZE, "0x%.4x\n", cr->device); +} + +static ssize_t class_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct afu_config_record *cr = to_cr(kobj); + + return scnprintf(buf, PAGE_SIZE, "0x%.6x\n", cr->class); +} + +static ssize_t afu_read_config(struct file *filp, struct kobject *kobj, + struct bin_attribute *bin_attr, char *buf, + loff_t off, size_t count) +{ + struct afu_config_record *cr = to_cr(kobj); + struct cxl_afu *afu = to_cxl_afu(container_of(kobj->parent, struct device, kobj)); + + u64 i, j, val, size = afu->crs_len; + + if (off > size) + return 0; + if (off + count > size) + count = size - off; + + for (i = 0; i < count;) { + val = cxl_afu_cr_read64(afu, cr->cr, off & ~0x7); + for (j = off & 0x7; j < 8 && i < count; i++, j++, off++) + buf[i] = (val >> (j * 8)) & 0xff; + } + + return count; +} + +static struct kobj_attribute vendor_attribute = + __ATTR_RO(vendor); +static struct kobj_attribute device_attribute = + __ATTR_RO(device); +static struct kobj_attribute class_attribute = + __ATTR_RO(class); + +static struct attribute *afu_cr_attrs[] = { + &vendor_attribute.attr, + &device_attribute.attr, + &class_attribute.attr, + NULL, +}; + +static void release_afu_config_record(struct kobject *kobj) +{ + struct afu_config_record *cr = to_cr(kobj); + + kfree(cr); +} + +static struct kobj_type afu_config_record_type = { + .sysfs_ops = &kobj_sysfs_ops, + .release = release_afu_config_record, + .default_attrs = afu_cr_attrs, +}; + +static struct afu_config_record *cxl_sysfs_afu_new_cr(struct cxl_afu *afu, int cr_idx) +{ + struct afu_config_record *cr; + int rc; + + cr = kzalloc(sizeof(struct afu_config_record), GFP_KERNEL); + if (!cr) + return ERR_PTR(-ENOMEM); + + cr->cr = cr_idx; + cr->device = cxl_afu_cr_read16(afu, cr_idx, PCI_DEVICE_ID); + cr->vendor = cxl_afu_cr_read16(afu, cr_idx, PCI_VENDOR_ID); + cr->class = cxl_afu_cr_read32(afu, cr_idx, PCI_CLASS_REVISION) >> 8; + + /* + * Export raw AFU PCIe like config record. For now this is read only by + * root - we can expand that later to be readable by non-root and maybe + * even writable provided we have a good use-case. Once we suport + * exposing AFUs through a virtual PHB they will get that for free from + * Linux' PCI infrastructure, but until then it's not clear that we + * need it for anything since the main use case is just identifying + * AFUs, which can be done via the vendor, device and class attributes. + */ + sysfs_bin_attr_init(&cr->config_attr); + cr->config_attr.attr.name = "config"; + cr->config_attr.attr.mode = S_IRUSR; + cr->config_attr.size = afu->crs_len; + cr->config_attr.read = afu_read_config; + + rc = kobject_init_and_add(&cr->kobj, &afu_config_record_type, + &afu->dev.kobj, "cr%i", cr->cr); + if (rc) + goto err; + + rc = sysfs_create_bin_file(&cr->kobj, &cr->config_attr); + if (rc) + goto err1; + + rc = kobject_uevent(&cr->kobj, KOBJ_ADD); + if (rc) + goto err2; + + return cr; +err2: + sysfs_remove_bin_file(&cr->kobj, &cr->config_attr); +err1: + kobject_put(&cr->kobj); + return ERR_PTR(rc); +err: + kfree(cr); + return ERR_PTR(rc); +} + +void cxl_sysfs_afu_remove(struct cxl_afu *afu) +{ + struct afu_config_record *cr, *tmp; + int i; + + for (i = 0; i < ARRAY_SIZE(afu_attrs); i++) + device_remove_file(&afu->dev, &afu_attrs[i]); + + list_for_each_entry_safe(cr, tmp, &afu->crs, list) { + sysfs_remove_bin_file(&cr->kobj, &cr->config_attr); + kobject_put(&cr->kobj); + } +} + int cxl_sysfs_afu_add(struct cxl_afu *afu) { + struct afu_config_record *cr; int i, rc; + INIT_LIST_HEAD(&afu->crs); + for (i = 0; i < ARRAY_SIZE(afu_attrs); i++) { if ((rc = device_create_file(&afu->dev, &afu_attrs[i]))) goto err; } + for (i = 0; i < afu->crs_num; i++) { + cr = cxl_sysfs_afu_new_cr(afu, i); + if (IS_ERR(cr)) { + rc = PTR_ERR(cr); + goto err1; + } + list_add(&cr->list, &afu->crs); + } + return 0; +err1: + cxl_sysfs_afu_remove(afu); + return rc; err: for (i--; i >= 0; i--) device_remove_file(&afu->dev, &afu_attrs[i]); return rc; } -void cxl_sysfs_afu_remove(struct cxl_afu *afu) -{ - int i; - - for (i = 0; i < ARRAY_SIZE(afu_attrs); i++) - device_remove_file(&afu->dev, &afu_attrs[i]); -} - int cxl_sysfs_afu_m_add(struct cxl_afu *afu) { int i, rc; -- cgit v1.2.3 From 3d5be0392f52489b7572903d83f1f267f26e6b69 Mon Sep 17 00:00:00 2001 From: Ian Munsie Date: Wed, 4 Feb 2015 19:09:02 +1100 Subject: cxl: Fail AFU initialisation if an invalid configuration record is found If an AFU claims to have a configuration record but doesn't actually contain a vendor and device ID, fail the AFU initialisation. Right now this is just a way of politely letting AFU developers know that they need to fix their config space, but later on we may expose the AFUs as actual PCI devices in their own right and don't want to inadvertendly expose an AFU with a bad config space. Signed-off-by: Ian Munsie Signed-off-by: Michael Ellerman --- drivers/misc/cxl/pci.c | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'drivers/misc/cxl/pci.c') diff --git a/drivers/misc/cxl/pci.c b/drivers/misc/cxl/pci.c index 2b2e1b80d759..1ef01647265f 100644 --- a/drivers/misc/cxl/pci.c +++ b/drivers/misc/cxl/pci.c @@ -598,6 +598,8 @@ static int cxl_read_afu_descriptor(struct cxl_afu *afu) static int cxl_afu_descriptor_looks_ok(struct cxl_afu *afu) { + int i; + if (afu->psa && afu->adapter->ps_size < (afu->pp_offset + afu->pp_size*afu->max_procs_virtualised)) { dev_err(&afu->dev, "per-process PSA can't fit inside the PSA!\n"); @@ -607,6 +609,13 @@ static int cxl_afu_descriptor_looks_ok(struct cxl_afu *afu) if (afu->pp_psa && (afu->pp_size < PAGE_SIZE)) dev_warn(&afu->dev, "AFU uses < PAGE_SIZE per-process PSA!"); + for (i = 0; i < afu->crs_num; i++) { + if ((cxl_afu_cr_read32(afu, i, 0) == 0)) { + dev_err(&afu->dev, "ABORTING: AFU configuration record %i is invalid\n", i); + return -EINVAL; + } + } + return 0; } -- cgit v1.2.3