From ea8c64ace86647260ec4255f483e5844d62af2df Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 10 Jan 2018 16:21:13 +0100 Subject: dma-mapping: move swiotlb arch helpers to a new header phys_to_dma, dma_to_phys and dma_capable are helpers published by architecture code for use of swiotlb and xen-swiotlb only. Drivers are not supposed to use these directly, but use the DMA API instead. Move these to a new asm/dma-direct.h helper, included by a linux/dma-direct.h wrapper that provides the default linear mapping unless the architecture wants to override it. In the MIPS case the existing dma-coherent.h is reused for now as untangling it will take a bit of work. Signed-off-by: Christoph Hellwig Acked-by: Robin Murphy --- lib/swiotlb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/swiotlb.c b/lib/swiotlb.c index cea19aaf303c..6583f3512386 100644 --- a/lib/swiotlb.c +++ b/lib/swiotlb.c @@ -18,7 +18,7 @@ */ #include -#include +#include #include #include #include -- cgit v1.2.3 From 002e67454f61bb67d8071ac4d0cacb86a01d18e0 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 9 Jan 2018 16:30:23 +0100 Subject: dma-direct: rename dma_noop to dma_direct The trivial direct mapping implementation already does a virtual to physical translation which isn't strictly a noop, and will soon learn to do non-direct but linear physical to dma translations through the device offset and a few small tricks. Rename it to a better fitting name. Signed-off-by: Christoph Hellwig Reviewed-by: Vladimir Murzin --- MAINTAINERS | 2 +- arch/arm/Kconfig | 2 +- arch/arm/include/asm/dma-mapping.h | 2 +- arch/arm/mm/dma-mapping-nommu.c | 8 ++--- arch/m32r/Kconfig | 2 +- arch/riscv/Kconfig | 2 +- arch/s390/Kconfig | 2 +- include/asm-generic/dma-mapping.h | 2 +- include/linux/dma-mapping.h | 2 +- lib/Kconfig | 2 +- lib/Makefile | 2 +- lib/dma-direct.c | 63 +++++++++++++++++++++++++++++++++++ lib/dma-noop.c | 68 -------------------------------------- 13 files changed, 77 insertions(+), 82 deletions(-) create mode 100644 lib/dma-direct.c delete mode 100644 lib/dma-noop.c (limited to 'lib') diff --git a/MAINTAINERS b/MAINTAINERS index 234e642e7149..2d54e636d625 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -4334,7 +4334,7 @@ T: git git://git.infradead.org/users/hch/dma-mapping.git W: http://git.infradead.org/users/hch/dma-mapping.git S: Supported F: lib/dma-debug.c -F: lib/dma-noop.c +F: lib/dma-direct.c F: lib/dma-virt.c F: drivers/base/dma-mapping.c F: drivers/base/dma-coherent.c diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 00d889a37965..430a0aa710d6 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -25,7 +25,7 @@ config ARM select CLONE_BACKWARDS select CPU_PM if (SUSPEND || CPU_IDLE) select DCACHE_WORD_ACCESS if HAVE_EFFICIENT_UNALIGNED_ACCESS - select DMA_NOOP_OPS if !MMU + select DMA_DIRECT_OPS if !MMU select EDAC_SUPPORT select EDAC_ATOMIC_SCRUB select GENERIC_ALLOCATOR diff --git a/arch/arm/include/asm/dma-mapping.h b/arch/arm/include/asm/dma-mapping.h index e5d9020c9ee1..8436f6ade57d 100644 --- a/arch/arm/include/asm/dma-mapping.h +++ b/arch/arm/include/asm/dma-mapping.h @@ -18,7 +18,7 @@ extern const struct dma_map_ops arm_coherent_dma_ops; static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus) { - return IS_ENABLED(CONFIG_MMU) ? &arm_dma_ops : &dma_noop_ops; + return IS_ENABLED(CONFIG_MMU) ? &arm_dma_ops : &dma_direct_ops; } #ifdef __arch_page_to_dma diff --git a/arch/arm/mm/dma-mapping-nommu.c b/arch/arm/mm/dma-mapping-nommu.c index 6db5fc26d154..4d8042521e89 100644 --- a/arch/arm/mm/dma-mapping-nommu.c +++ b/arch/arm/mm/dma-mapping-nommu.c @@ -22,7 +22,7 @@ #include "dma.h" /* - * dma_noop_ops is used if + * dma_direct_ops is used if * - MMU/MPU is off * - cpu is v7m w/o cache support * - device is coherent @@ -39,7 +39,7 @@ static void *arm_nommu_dma_alloc(struct device *dev, size_t size, unsigned long attrs) { - const struct dma_map_ops *ops = &dma_noop_ops; + const struct dma_map_ops *ops = &dma_direct_ops; void *ret; /* @@ -70,7 +70,7 @@ static void arm_nommu_dma_free(struct device *dev, size_t size, void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs) { - const struct dma_map_ops *ops = &dma_noop_ops; + const struct dma_map_ops *ops = &dma_direct_ops; if (attrs & DMA_ATTR_NON_CONSISTENT) { ops->free(dev, size, cpu_addr, dma_addr, attrs); @@ -213,7 +213,7 @@ EXPORT_SYMBOL(arm_nommu_dma_ops); static const struct dma_map_ops *arm_nommu_get_dma_map_ops(bool coherent) { - return coherent ? &dma_noop_ops : &arm_nommu_dma_ops; + return coherent ? &dma_direct_ops : &arm_nommu_dma_ops; } void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, diff --git a/arch/m32r/Kconfig b/arch/m32r/Kconfig index 498398d915c1..dd84ee194579 100644 --- a/arch/m32r/Kconfig +++ b/arch/m32r/Kconfig @@ -19,7 +19,7 @@ config M32R select MODULES_USE_ELF_RELA select HAVE_DEBUG_STACKOVERFLOW select CPU_NO_EFFICIENT_FFS - select DMA_NOOP_OPS + select DMA_DIRECT_OPS select ARCH_NO_COHERENT_DMA_MMAP if !MMU config SBUS diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 2c6adf12713a..865e14f50c14 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -83,7 +83,7 @@ config PGTABLE_LEVELS config HAVE_KPROBES def_bool n -config DMA_NOOP_OPS +config DMA_DIRECT_OPS def_bool y menu "Platform type" diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 829c67986db7..9376637229c9 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -140,7 +140,7 @@ config S390 select HAVE_DEBUG_KMEMLEAK select HAVE_DMA_API_DEBUG select HAVE_DMA_CONTIGUOUS - select DMA_NOOP_OPS + select DMA_DIRECT_OPS select HAVE_DYNAMIC_FTRACE select HAVE_DYNAMIC_FTRACE_WITH_REGS select HAVE_EFFICIENT_UNALIGNED_ACCESS diff --git a/include/asm-generic/dma-mapping.h b/include/asm-generic/dma-mapping.h index 164031531d85..880a292d792f 100644 --- a/include/asm-generic/dma-mapping.h +++ b/include/asm-generic/dma-mapping.h @@ -4,7 +4,7 @@ static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus) { - return &dma_noop_ops; + return &dma_direct_ops; } #endif /* _ASM_GENERIC_DMA_MAPPING_H */ diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h index 46542ad9d709..34fe8463d10e 100644 --- a/include/linux/dma-mapping.h +++ b/include/linux/dma-mapping.h @@ -136,7 +136,7 @@ struct dma_map_ops { int is_phys; }; -extern const struct dma_map_ops dma_noop_ops; +extern const struct dma_map_ops dma_direct_ops; extern const struct dma_map_ops dma_virt_ops; #define DMA_BIT_MASK(n) (((n) == 64) ? ~0ULL : ((1ULL<<(n))-1)) diff --git a/lib/Kconfig b/lib/Kconfig index c5e84fbcb30b..9d3d649c9dc9 100644 --- a/lib/Kconfig +++ b/lib/Kconfig @@ -409,7 +409,7 @@ config HAS_DMA depends on !NO_DMA default y -config DMA_NOOP_OPS +config DMA_DIRECT_OPS bool depends on HAS_DMA && (!64BIT || ARCH_DMA_ADDR_T_64BIT) default n diff --git a/lib/Makefile b/lib/Makefile index d11c48ec8ffd..749851abe85a 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -28,7 +28,7 @@ lib-y := ctype.o string.o vsprintf.o cmdline.o \ lib-$(CONFIG_MMU) += ioremap.o lib-$(CONFIG_SMP) += cpumask.o -lib-$(CONFIG_DMA_NOOP_OPS) += dma-noop.o +lib-$(CONFIG_DMA_DIRECT_OPS) += dma-direct.o lib-$(CONFIG_DMA_VIRT_OPS) += dma-virt.o lib-y += kobject.o klist.o diff --git a/lib/dma-direct.c b/lib/dma-direct.c new file mode 100644 index 000000000000..0ec3262a3148 --- /dev/null +++ b/lib/dma-direct.c @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * lib/dma-noop.c + * + * DMA operations that map to physical addresses without flushing memory. + */ +#include +#include +#include +#include +#include + +static void *dma_direct_alloc(struct device *dev, size_t size, + dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) +{ + void *ret; + + ret = (void *)__get_free_pages(gfp, get_order(size)); + if (ret) + *dma_handle = virt_to_phys(ret) - PFN_PHYS(dev->dma_pfn_offset); + + return ret; +} + +static void dma_direct_free(struct device *dev, size_t size, void *cpu_addr, + dma_addr_t dma_addr, unsigned long attrs) +{ + free_pages((unsigned long)cpu_addr, get_order(size)); +} + +static dma_addr_t dma_direct_map_page(struct device *dev, struct page *page, + unsigned long offset, size_t size, enum dma_data_direction dir, + unsigned long attrs) +{ + return page_to_phys(page) + offset - PFN_PHYS(dev->dma_pfn_offset); +} + +static int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, + int nents, enum dma_data_direction dir, unsigned long attrs) +{ + int i; + struct scatterlist *sg; + + for_each_sg(sgl, sg, nents, i) { + dma_addr_t offset = PFN_PHYS(dev->dma_pfn_offset); + void *va; + + BUG_ON(!sg_page(sg)); + va = sg_virt(sg); + sg_dma_address(sg) = (dma_addr_t)virt_to_phys(va) - offset; + sg_dma_len(sg) = sg->length; + } + + return nents; +} + +const struct dma_map_ops dma_direct_ops = { + .alloc = dma_direct_alloc, + .free = dma_direct_free, + .map_page = dma_direct_map_page, + .map_sg = dma_direct_map_sg, +}; +EXPORT_SYMBOL(dma_direct_ops); diff --git a/lib/dma-noop.c b/lib/dma-noop.c deleted file mode 100644 index a10185b0c2d4..000000000000 --- a/lib/dma-noop.c +++ /dev/null @@ -1,68 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * lib/dma-noop.c - * - * DMA operations that map to physical addresses without flushing memory. - */ -#include -#include -#include -#include -#include - -static void *dma_noop_alloc(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t gfp, - unsigned long attrs) -{ - void *ret; - - ret = (void *)__get_free_pages(gfp, get_order(size)); - if (ret) - *dma_handle = virt_to_phys(ret) - PFN_PHYS(dev->dma_pfn_offset); - - return ret; -} - -static void dma_noop_free(struct device *dev, size_t size, - void *cpu_addr, dma_addr_t dma_addr, - unsigned long attrs) -{ - free_pages((unsigned long)cpu_addr, get_order(size)); -} - -static dma_addr_t dma_noop_map_page(struct device *dev, struct page *page, - unsigned long offset, size_t size, - enum dma_data_direction dir, - unsigned long attrs) -{ - return page_to_phys(page) + offset - PFN_PHYS(dev->dma_pfn_offset); -} - -static int dma_noop_map_sg(struct device *dev, struct scatterlist *sgl, int nents, - enum dma_data_direction dir, - unsigned long attrs) -{ - int i; - struct scatterlist *sg; - - for_each_sg(sgl, sg, nents, i) { - dma_addr_t offset = PFN_PHYS(dev->dma_pfn_offset); - void *va; - - BUG_ON(!sg_page(sg)); - va = sg_virt(sg); - sg_dma_address(sg) = (dma_addr_t)virt_to_phys(va) - offset; - sg_dma_len(sg) = sg->length; - } - - return nents; -} - -const struct dma_map_ops dma_noop_ops = { - .alloc = dma_noop_alloc, - .free = dma_noop_free, - .map_page = dma_noop_map_page, - .map_sg = dma_noop_map_sg, -}; - -EXPORT_SYMBOL(dma_noop_ops); -- cgit v1.2.3 From 2e86a04780c6ebb64581013054f8260d040c2c9e Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 22 Dec 2017 11:29:51 +0100 Subject: dma-direct: use phys_to_dma This means it uses whatever linear remapping scheme that the architecture provides is used in the generic dma_direct ops. Signed-off-by: Christoph Hellwig Reviewed-by: Vladimir Murzin --- lib/dma-direct.c | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/lib/dma-direct.c b/lib/dma-direct.c index 0ec3262a3148..12ea9653781b 100644 --- a/lib/dma-direct.c +++ b/lib/dma-direct.c @@ -1,12 +1,11 @@ // SPDX-License-Identifier: GPL-2.0 /* - * lib/dma-noop.c - * - * DMA operations that map to physical addresses without flushing memory. + * DMA operations that map physical memory directly without using an IOMMU or + * flushing caches. */ #include #include -#include +#include #include #include @@ -17,7 +16,7 @@ static void *dma_direct_alloc(struct device *dev, size_t size, ret = (void *)__get_free_pages(gfp, get_order(size)); if (ret) - *dma_handle = virt_to_phys(ret) - PFN_PHYS(dev->dma_pfn_offset); + *dma_handle = phys_to_dma(dev, virt_to_phys(ret)); return ret; } @@ -32,7 +31,7 @@ static dma_addr_t dma_direct_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, unsigned long attrs) { - return page_to_phys(page) + offset - PFN_PHYS(dev->dma_pfn_offset); + return phys_to_dma(dev, page_to_phys(page)) + offset; } static int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, @@ -42,12 +41,9 @@ static int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, struct scatterlist *sg; for_each_sg(sgl, sg, nents, i) { - dma_addr_t offset = PFN_PHYS(dev->dma_pfn_offset); - void *va; - BUG_ON(!sg_page(sg)); - va = sg_virt(sg); - sg_dma_address(sg) = (dma_addr_t)virt_to_phys(va) - offset; + + sg_dma_address(sg) = phys_to_dma(dev, sg_phys(sg)); sg_dma_len(sg) = sg->length; } -- cgit v1.2.3 From 2797596992898034256a6c40168d357ccf0ba0b8 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 9 Jan 2018 16:30:47 +0100 Subject: dma-direct: add dma address sanity checks Roughly based on the x86 pci-nommu implementation. Signed-off-by: Christoph Hellwig --- lib/dma-direct.c | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/dma-direct.c b/lib/dma-direct.c index 12ea9653781b..32fd4d9e4c47 100644 --- a/lib/dma-direct.c +++ b/lib/dma-direct.c @@ -9,6 +9,23 @@ #include #include +#define DIRECT_MAPPING_ERROR 0 + +static bool +check_addr(struct device *dev, dma_addr_t dma_addr, size_t size, + const char *caller) +{ + if (unlikely(dev && !dma_capable(dev, dma_addr, size))) { + if (*dev->dma_mask >= DMA_BIT_MASK(32)) { + dev_err(dev, + "%s: overflow %pad+%zu of device mask %llx\n", + caller, &dma_addr, size, *dev->dma_mask); + } + return false; + } + return true; +} + static void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) { @@ -31,7 +48,11 @@ static dma_addr_t dma_direct_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, unsigned long attrs) { - return phys_to_dma(dev, page_to_phys(page)) + offset; + dma_addr_t dma_addr = phys_to_dma(dev, page_to_phys(page)) + offset; + + if (!check_addr(dev, dma_addr, size, __func__)) + return DIRECT_MAPPING_ERROR; + return dma_addr; } static int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, @@ -44,16 +65,24 @@ static int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, BUG_ON(!sg_page(sg)); sg_dma_address(sg) = phys_to_dma(dev, sg_phys(sg)); + if (!check_addr(dev, sg_dma_address(sg), sg->length, __func__)) + return 0; sg_dma_len(sg) = sg->length; } return nents; } +static int dma_direct_mapping_error(struct device *dev, dma_addr_t dma_addr) +{ + return dma_addr == DIRECT_MAPPING_ERROR; +} + const struct dma_map_ops dma_direct_ops = { .alloc = dma_direct_alloc, .free = dma_direct_free, .map_page = dma_direct_map_page, .map_sg = dma_direct_map_sg, + .mapping_error = dma_direct_mapping_error, }; EXPORT_SYMBOL(dma_direct_ops); -- cgit v1.2.3 From 080321d3b3139b3b3ec0da31a7887eebd50b5979 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 22 Dec 2017 11:51:44 +0100 Subject: dma-direct: add support for CMA allocation Try the CMA allocator for coherent allocations if supported. Roughly modelled after the x86 code. Signed-off-by: Christoph Hellwig --- lib/dma-direct.c | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/dma-direct.c b/lib/dma-direct.c index 32fd4d9e4c47..a9ae98be7af3 100644 --- a/lib/dma-direct.c +++ b/lib/dma-direct.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #define DIRECT_MAPPING_ERROR 0 @@ -29,19 +30,30 @@ check_addr(struct device *dev, dma_addr_t dma_addr, size_t size, static void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) { - void *ret; + unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; + int page_order = get_order(size); + struct page *page = NULL; - ret = (void *)__get_free_pages(gfp, get_order(size)); - if (ret) - *dma_handle = phys_to_dma(dev, virt_to_phys(ret)); + /* CMA can be used only in the context which permits sleeping */ + if (gfpflags_allow_blocking(gfp)) + page = dma_alloc_from_contiguous(dev, count, page_order, gfp); + if (!page) + page = alloc_pages(gfp, page_order); + if (!page) + return NULL; - return ret; + *dma_handle = phys_to_dma(dev, page_to_phys(page)); + memset(page_address(page), 0, size); + return page_address(page); } static void dma_direct_free(struct device *dev, size_t size, void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs) { - free_pages((unsigned long)cpu_addr, get_order(size)); + unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; + + if (!dma_release_from_contiguous(dev, virt_to_page(cpu_addr), count)) + free_pages((unsigned long)cpu_addr, get_order(size)); } static dma_addr_t dma_direct_map_page(struct device *dev, struct page *page, -- cgit v1.2.3 From 21f237e4d08511eb662b59d8ff1b64be68e0c2bc Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 22 Dec 2017 11:55:23 +0100 Subject: dma-direct: use node local allocations for coherent memory To preserve the x86 behavior. Signed-off-by: Christoph Hellwig Reviewed-by: Robin Murphy --- lib/dma-direct.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/dma-direct.c b/lib/dma-direct.c index a9ae98be7af3..f04a424f91fa 100644 --- a/lib/dma-direct.c +++ b/lib/dma-direct.c @@ -38,7 +38,7 @@ static void *dma_direct_alloc(struct device *dev, size_t size, if (gfpflags_allow_blocking(gfp)) page = dma_alloc_from_contiguous(dev, count, page_order, gfp); if (!page) - page = alloc_pages(gfp, page_order); + page = alloc_pages_node(dev_to_node(dev), gfp, page_order); if (!page) return NULL; -- cgit v1.2.3 From c61e9637340e1d0672c6f0c0b11aa81d7262c49d Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 9 Jan 2018 23:39:03 +0100 Subject: dma-direct: add support for allocation from ZONE_DMA and ZONE_DMA32 This allows to dip into zones for lower memory if they are available. If one of the zones is not available the corresponding GFP_* flag will evaluate to 0 so they won't change anything. We provide an arch tunable for those architectures that do not use GFP_DMA for the lowest 24-bits, given that there are a few. Roughly based on the x86 code. Signed-off-by: Christoph Hellwig --- lib/dma-direct.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'lib') diff --git a/lib/dma-direct.c b/lib/dma-direct.c index f04a424f91fa..8f76032ebc3c 100644 --- a/lib/dma-direct.c +++ b/lib/dma-direct.c @@ -12,6 +12,14 @@ #define DIRECT_MAPPING_ERROR 0 +/* + * Most architectures use ZONE_DMA for the first 16 Megabytes, but + * some use it for entirely different regions: + */ +#ifndef ARCH_ZONE_DMA_BITS +#define ARCH_ZONE_DMA_BITS 24 +#endif + static bool check_addr(struct device *dev, dma_addr_t dma_addr, size_t size, const char *caller) @@ -34,6 +42,12 @@ static void *dma_direct_alloc(struct device *dev, size_t size, int page_order = get_order(size); struct page *page = NULL; + /* GFP_DMA32 and GFP_DMA are no ops without the corresponding zones: */ + if (dev->coherent_dma_mask <= DMA_BIT_MASK(ARCH_ZONE_DMA_BITS)) + gfp |= GFP_DMA; + if (dev->coherent_dma_mask <= DMA_BIT_MASK(32) && !(gfp & GFP_DMA)) + gfp |= GFP_DMA32; + /* CMA can be used only in the context which permits sleeping */ if (gfpflags_allow_blocking(gfp)) page = dma_alloc_from_contiguous(dev, count, page_order, gfp); -- cgit v1.2.3 From 95f183916d4b0bc1943684948ecdd2469f1aa978 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 9 Jan 2018 23:40:57 +0100 Subject: dma-direct: retry allocations using GFP_DMA for small masks If an attempt to allocate memory succeeded, but isn't inside the supported DMA mask, retry the allocation with GFP_DMA set as a last resort. Based on the x86 code, but an off by one error in what is now dma_coherent_ok has been fixed vs the x86 code. Signed-off-by: Christoph Hellwig --- lib/dma-direct.c | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/dma-direct.c b/lib/dma-direct.c index 8f76032ebc3c..4e43c2bb7f5f 100644 --- a/lib/dma-direct.c +++ b/lib/dma-direct.c @@ -35,6 +35,11 @@ check_addr(struct device *dev, dma_addr_t dma_addr, size_t size, return true; } +static bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size) +{ + return phys_to_dma(dev, phys) + size - 1 <= dev->coherent_dma_mask; +} + static void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) { @@ -48,11 +53,29 @@ static void *dma_direct_alloc(struct device *dev, size_t size, if (dev->coherent_dma_mask <= DMA_BIT_MASK(32) && !(gfp & GFP_DMA)) gfp |= GFP_DMA32; +again: /* CMA can be used only in the context which permits sleeping */ - if (gfpflags_allow_blocking(gfp)) + if (gfpflags_allow_blocking(gfp)) { page = dma_alloc_from_contiguous(dev, count, page_order, gfp); + if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) { + dma_release_from_contiguous(dev, page, count); + page = NULL; + } + } if (!page) page = alloc_pages_node(dev_to_node(dev), gfp, page_order); + + if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) { + __free_pages(page, page_order); + page = NULL; + + if (dev->coherent_dma_mask < DMA_BIT_MASK(32) && + !(gfp & GFP_DMA)) { + gfp = (gfp & ~GFP_DMA32) | GFP_DMA; + goto again; + } + } + if (!page) return NULL; -- cgit v1.2.3 From 19dca8c0efa30e0a45e79f237060d0f307045752 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 23 Dec 2017 13:46:06 +0100 Subject: dma-direct: make dma_direct_{alloc,free} available to other implementations So that they don't need to indirect through the operation vector. Signed-off-by: Christoph Hellwig Reviewed-by: Vladimir Murzin --- arch/arm/mm/dma-mapping-nommu.c | 9 +++------ include/linux/dma-direct.h | 5 +++++ lib/dma-direct.c | 6 +++--- 3 files changed, 11 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/arch/arm/mm/dma-mapping-nommu.c b/arch/arm/mm/dma-mapping-nommu.c index 4d8042521e89..619f24a42d09 100644 --- a/arch/arm/mm/dma-mapping-nommu.c +++ b/arch/arm/mm/dma-mapping-nommu.c @@ -11,7 +11,7 @@ #include #include -#include +#include #include #include @@ -39,7 +39,6 @@ static void *arm_nommu_dma_alloc(struct device *dev, size_t size, unsigned long attrs) { - const struct dma_map_ops *ops = &dma_direct_ops; void *ret; /* @@ -48,7 +47,7 @@ static void *arm_nommu_dma_alloc(struct device *dev, size_t size, */ if (attrs & DMA_ATTR_NON_CONSISTENT) - return ops->alloc(dev, size, dma_handle, gfp, attrs); + return dma_direct_alloc(dev, size, dma_handle, gfp, attrs); ret = dma_alloc_from_global_coherent(size, dma_handle); @@ -70,10 +69,8 @@ static void arm_nommu_dma_free(struct device *dev, size_t size, void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs) { - const struct dma_map_ops *ops = &dma_direct_ops; - if (attrs & DMA_ATTR_NON_CONSISTENT) { - ops->free(dev, size, cpu_addr, dma_addr, attrs); + dma_direct_free(dev, size, cpu_addr, dma_addr, attrs); } else { int ret = dma_release_from_global_coherent(get_order(size), cpu_addr); diff --git a/include/linux/dma-direct.h b/include/linux/dma-direct.h index 10e924b7cba7..4788bf0bf683 100644 --- a/include/linux/dma-direct.h +++ b/include/linux/dma-direct.h @@ -38,4 +38,9 @@ static inline void dma_mark_clean(void *addr, size_t size) } #endif /* CONFIG_ARCH_HAS_DMA_MARK_CLEAN */ +void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, + gfp_t gfp, unsigned long attrs); +void dma_direct_free(struct device *dev, size_t size, void *cpu_addr, + dma_addr_t dma_addr, unsigned long attrs); + #endif /* _LINUX_DMA_DIRECT_H */ diff --git a/lib/dma-direct.c b/lib/dma-direct.c index 4e43c2bb7f5f..784a68dfdbe3 100644 --- a/lib/dma-direct.c +++ b/lib/dma-direct.c @@ -40,8 +40,8 @@ static bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size) return phys_to_dma(dev, phys) + size - 1 <= dev->coherent_dma_mask; } -static void *dma_direct_alloc(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) +void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, + gfp_t gfp, unsigned long attrs) { unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; int page_order = get_order(size); @@ -84,7 +84,7 @@ again: return page_address(page); } -static void dma_direct_free(struct device *dev, size_t size, void *cpu_addr, +void dma_direct_free(struct device *dev, size_t size, void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs) { unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; -- cgit v1.2.3 From 1a9777a8a01fb88659a3dda48080c95c34cab7d3 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sun, 24 Dec 2017 15:04:32 +0100 Subject: dma-direct: reject too small dma masks Signed-off-by: Christoph Hellwig Reviewed-by: Robin Murphy --- include/linux/dma-direct.h | 1 + lib/dma-direct.c | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+) (limited to 'lib') diff --git a/include/linux/dma-direct.h b/include/linux/dma-direct.h index 4788bf0bf683..bcdb1a3e4b1f 100644 --- a/include/linux/dma-direct.h +++ b/include/linux/dma-direct.h @@ -42,5 +42,6 @@ void *dma_direct_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs); void dma_direct_free(struct device *dev, size_t size, void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs); +int dma_direct_supported(struct device *dev, u64 mask); #endif /* _LINUX_DMA_DIRECT_H */ diff --git a/lib/dma-direct.c b/lib/dma-direct.c index 784a68dfdbe3..40b1f92f2214 100644 --- a/lib/dma-direct.c +++ b/lib/dma-direct.c @@ -122,6 +122,24 @@ static int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, return nents; } +int dma_direct_supported(struct device *dev, u64 mask) +{ +#ifdef CONFIG_ZONE_DMA + if (mask < DMA_BIT_MASK(ARCH_ZONE_DMA_BITS)) + return 0; +#else + /* + * Because 32-bit DMA masks are so common we expect every architecture + * to be able to satisfy them - either by not supporting more physical + * memory, or by providing a ZONE_DMA32. If neither is the case, the + * architecture needs to use an IOMMU instead of the direct mapping. + */ + if (mask < DMA_BIT_MASK(32)) + return 0; +#endif + return 1; +} + static int dma_direct_mapping_error(struct device *dev, dma_addr_t dma_addr) { return dma_addr == DIRECT_MAPPING_ERROR; @@ -132,6 +150,7 @@ const struct dma_map_ops dma_direct_ops = { .free = dma_direct_free, .map_page = dma_direct_map_page, .map_sg = dma_direct_map_sg, + .dma_supported = dma_direct_supported, .mapping_error = dma_direct_mapping_error, }; EXPORT_SYMBOL(dma_direct_ops); -- cgit v1.2.3 From d0bc0c2a31c95002d37c3cc511ffdcab851b3256 Mon Sep 17 00:00:00 2001 From: Christian König Date: Thu, 4 Jan 2018 14:24:19 +0100 Subject: swiotlb: suppress warning when __GFP_NOWARN is set MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TTM tries to allocate coherent memory in chunks of 2MB first to improve TLB efficiency and falls back to allocating 4K pages if that fails. Suppress the warning when the 2MB allocations fails since there is a valid fall back path. Signed-off-by: Christian König Reported-by: Mike Galbraith Acked-by: Konrad Rzeszutek Wilk Bug: https://bugs.freedesktop.org/show_bug.cgi?id=104082 CC: stable@vger.kernel.org Signed-off-by: Christoph Hellwig --- lib/swiotlb.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/swiotlb.c b/lib/swiotlb.c index 6583f3512386..125c1062119f 100644 --- a/lib/swiotlb.c +++ b/lib/swiotlb.c @@ -586,7 +586,7 @@ phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, not_found: spin_unlock_irqrestore(&io_tlb_lock, flags); - if (printk_ratelimit()) + if (!(attrs & DMA_ATTR_NO_WARN) && printk_ratelimit()) dev_warn(hwdev, "swiotlb buffer is full (sz: %zd bytes)\n", size); return SWIOTLB_MAP_ERROR; found: @@ -713,6 +713,7 @@ void * swiotlb_alloc_coherent(struct device *hwdev, size_t size, dma_addr_t *dma_handle, gfp_t flags) { + bool warn = !(flags & __GFP_NOWARN); dma_addr_t dev_addr; void *ret; int order = get_order(size); @@ -738,8 +739,8 @@ swiotlb_alloc_coherent(struct device *hwdev, size_t size, * GFP_DMA memory; fall back on map_single(), which * will grab memory from the lowest available address range. */ - phys_addr_t paddr = map_single(hwdev, 0, size, - DMA_FROM_DEVICE, 0); + phys_addr_t paddr = map_single(hwdev, 0, size, DMA_FROM_DEVICE, + warn ? 0 : DMA_ATTR_NO_WARN); if (paddr == SWIOTLB_MAP_ERROR) goto err_warn; @@ -769,9 +770,11 @@ swiotlb_alloc_coherent(struct device *hwdev, size_t size, return ret; err_warn: - pr_warn("swiotlb: coherent allocation failed for device %s size=%zu\n", - dev_name(hwdev), size); - dump_stack(); + if (warn && printk_ratelimit()) { + pr_warn("swiotlb: coherent allocation failed for device %s size=%zu\n", + dev_name(hwdev), size); + dump_stack(); + } return NULL; } -- cgit v1.2.3 From 7f2c8bbd321f18e4ccfd262748bd58fb7d4bb1db Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 23 Dec 2017 14:14:54 +0100 Subject: swiotlb: rename swiotlb_free to swiotlb_exit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christoph Hellwig Acked-by: Christian König Reviewed-by: Konrad Rzeszutek Wilk --- arch/powerpc/kernel/dma-swiotlb.c | 2 +- arch/x86/kernel/pci-swiotlb.c | 2 +- include/linux/swiotlb.h | 4 ++-- lib/swiotlb.c | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/arch/powerpc/kernel/dma-swiotlb.c b/arch/powerpc/kernel/dma-swiotlb.c index 506ac4fafac5..88f3963ca30f 100644 --- a/arch/powerpc/kernel/dma-swiotlb.c +++ b/arch/powerpc/kernel/dma-swiotlb.c @@ -121,7 +121,7 @@ static int __init check_swiotlb_enabled(void) if (ppc_swiotlb_enable) swiotlb_print_info(); else - swiotlb_free(); + swiotlb_exit(); return 0; } diff --git a/arch/x86/kernel/pci-swiotlb.c b/arch/x86/kernel/pci-swiotlb.c index 0d77603c2f50..0ee0f8f34251 100644 --- a/arch/x86/kernel/pci-swiotlb.c +++ b/arch/x86/kernel/pci-swiotlb.c @@ -120,7 +120,7 @@ void __init pci_swiotlb_late_init(void) { /* An IOMMU turned us off. */ if (!swiotlb) - swiotlb_free(); + swiotlb_exit(); else { printk(KERN_INFO "PCI-DMA: " "Using software bounce buffering for IO (SWIOTLB)\n"); diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h index 24ed817082ee..606375e35d87 100644 --- a/include/linux/swiotlb.h +++ b/include/linux/swiotlb.h @@ -115,10 +115,10 @@ extern int swiotlb_dma_supported(struct device *hwdev, u64 mask); #ifdef CONFIG_SWIOTLB -extern void __init swiotlb_free(void); +extern void __init swiotlb_exit(void); unsigned int swiotlb_max_segment(void); #else -static inline void swiotlb_free(void) { } +static inline void swiotlb_exit(void) { } static inline unsigned int swiotlb_max_segment(void) { return 0; } #endif diff --git a/lib/swiotlb.c b/lib/swiotlb.c index 125c1062119f..cf5311908fa9 100644 --- a/lib/swiotlb.c +++ b/lib/swiotlb.c @@ -417,7 +417,7 @@ cleanup2: return -ENOMEM; } -void __init swiotlb_free(void) +void __init swiotlb_exit(void) { if (!io_tlb_orig_addr) return; -- cgit v1.2.3 From 251533eb35b22f9e87a440b14c6f20e2626397b3 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 9 Jan 2018 16:44:16 +0100 Subject: swiotlb: add common swiotlb_map_ops MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently all architectures that want to use swiotlb have to implement their own dma_map_ops instances. Provide a generic one based on the x86 implementation which first calls into dma_direct to try a full blown direct mapping implementation (including e.g. CMA) before falling back allocating from the swiotlb buffer. Signed-off-by: Christoph Hellwig Acked-by: Christian König --- include/linux/swiotlb.h | 8 ++++++++ lib/swiotlb.c | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) (limited to 'lib') diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h index 606375e35d87..5b1f2a00491c 100644 --- a/include/linux/swiotlb.h +++ b/include/linux/swiotlb.h @@ -66,6 +66,12 @@ extern void swiotlb_tbl_sync_single(struct device *hwdev, enum dma_sync_target target); /* Accessory functions. */ + +void *swiotlb_alloc(struct device *hwdev, size_t size, dma_addr_t *dma_handle, + gfp_t flags, unsigned long attrs); +void swiotlb_free(struct device *dev, size_t size, void *vaddr, + dma_addr_t dma_addr, unsigned long attrs); + extern void *swiotlb_alloc_coherent(struct device *hwdev, size_t size, dma_addr_t *dma_handle, gfp_t flags); @@ -126,4 +132,6 @@ extern void swiotlb_print_info(void); extern int is_swiotlb_buffer(phys_addr_t paddr); extern void swiotlb_set_max_segment(unsigned int); +extern const struct dma_map_ops swiotlb_dma_ops; + #endif /* __LINUX_SWIOTLB_H */ diff --git a/lib/swiotlb.c b/lib/swiotlb.c index cf5311908fa9..0fae2f45c3c0 100644 --- a/lib/swiotlb.c +++ b/lib/swiotlb.c @@ -1087,3 +1087,46 @@ swiotlb_dma_supported(struct device *hwdev, u64 mask) return swiotlb_phys_to_dma(hwdev, io_tlb_end - 1) <= mask; } EXPORT_SYMBOL(swiotlb_dma_supported); + +#ifdef CONFIG_DMA_DIRECT_OPS +void *swiotlb_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, + gfp_t gfp, unsigned long attrs) +{ + void *vaddr; + + /* + * Don't print a warning when the first allocation attempt fails. + * swiotlb_alloc_coherent() will print a warning when the DMA memory + * allocation ultimately failed. + */ + gfp |= __GFP_NOWARN; + + vaddr = dma_direct_alloc(dev, size, dma_handle, gfp, attrs); + if (!vaddr) + vaddr = swiotlb_alloc_coherent(dev, size, dma_handle, gfp); + return vaddr; +} + +void swiotlb_free(struct device *dev, size_t size, void *vaddr, + dma_addr_t dma_addr, unsigned long attrs) +{ + if (is_swiotlb_buffer(dma_to_phys(dev, dma_addr))) + swiotlb_free_coherent(dev, size, vaddr, dma_addr); + else + dma_direct_free(dev, size, vaddr, dma_addr, attrs); +} + +const struct dma_map_ops swiotlb_dma_ops = { + .mapping_error = swiotlb_dma_mapping_error, + .alloc = swiotlb_alloc, + .free = swiotlb_free, + .sync_single_for_cpu = swiotlb_sync_single_for_cpu, + .sync_single_for_device = swiotlb_sync_single_for_device, + .sync_sg_for_cpu = swiotlb_sync_sg_for_cpu, + .sync_sg_for_device = swiotlb_sync_sg_for_device, + .map_sg = swiotlb_map_sg_attrs, + .unmap_sg = swiotlb_unmap_sg_attrs, + .map_page = swiotlb_map_page, + .unmap_page = swiotlb_unmap_page, +}; +#endif /* CONFIG_DMA_DIRECT_OPS */ -- cgit v1.2.3 From aaf796dc6e84e809e4c791b6517326b26312c972 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sun, 24 Dec 2017 15:07:34 +0100 Subject: swiotlb: wire up ->dma_supported in swiotlb_dma_ops MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To properly reject too small DMA masks based on the addressability of the bounce buffer. Signed-off-by: Christoph Hellwig Acked-by: Christian König --- lib/swiotlb.c | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/swiotlb.c b/lib/swiotlb.c index 0fae2f45c3c0..539fd1099ba9 100644 --- a/lib/swiotlb.c +++ b/lib/swiotlb.c @@ -1128,5 +1128,6 @@ const struct dma_map_ops swiotlb_dma_ops = { .unmap_sg = swiotlb_unmap_sg_attrs, .map_page = swiotlb_map_page, .unmap_page = swiotlb_unmap_page, + .dma_supported = swiotlb_dma_supported, }; #endif /* CONFIG_DMA_DIRECT_OPS */ -- cgit v1.2.3 From a25381aa3ae60a2e028c95f1318649b13cbbce30 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 23 Dec 2017 20:56:02 +0100 Subject: swiotlb: refactor coherent buffer freeing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Factor out a new swiotlb_free_buffer helper that checks if an address is allocated from the swiotlb bounce buffer, and if yes frees it. This allows to simplify the swiotlb_free implemenation that uses dma_direct_free to free the non-bounce buffer allocations. Signed-off-by: Christoph Hellwig Acked-by: Christian König --- lib/swiotlb.c | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/swiotlb.c b/lib/swiotlb.c index 539fd1099ba9..1a147f1354a1 100644 --- a/lib/swiotlb.c +++ b/lib/swiotlb.c @@ -780,22 +780,31 @@ err_warn: } EXPORT_SYMBOL(swiotlb_alloc_coherent); +static bool swiotlb_free_buffer(struct device *dev, size_t size, + dma_addr_t dma_addr) +{ + phys_addr_t phys_addr = dma_to_phys(dev, dma_addr); + + WARN_ON_ONCE(irqs_disabled()); + + if (!is_swiotlb_buffer(phys_addr)) + return false; + + /* + * DMA_TO_DEVICE to avoid memcpy in swiotlb_tbl_unmap_single. + * DMA_ATTR_SKIP_CPU_SYNC is optional. + */ + swiotlb_tbl_unmap_single(dev, phys_addr, size, DMA_TO_DEVICE, + DMA_ATTR_SKIP_CPU_SYNC); + return true; +} + void swiotlb_free_coherent(struct device *hwdev, size_t size, void *vaddr, dma_addr_t dev_addr) { - phys_addr_t paddr = dma_to_phys(hwdev, dev_addr); - - WARN_ON(irqs_disabled()); - if (!is_swiotlb_buffer(paddr)) + if (!swiotlb_free_buffer(hwdev, size, dev_addr)) free_pages((unsigned long)vaddr, get_order(size)); - else - /* - * DMA_TO_DEVICE to avoid memcpy in swiotlb_tbl_unmap_single. - * DMA_ATTR_SKIP_CPU_SYNC is optional. - */ - swiotlb_tbl_unmap_single(hwdev, paddr, size, DMA_TO_DEVICE, - DMA_ATTR_SKIP_CPU_SYNC); } EXPORT_SYMBOL(swiotlb_free_coherent); @@ -1110,9 +1119,7 @@ void *swiotlb_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, void swiotlb_free(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_addr, unsigned long attrs) { - if (is_swiotlb_buffer(dma_to_phys(dev, dma_addr))) - swiotlb_free_coherent(dev, size, vaddr, dma_addr); - else + if (!swiotlb_free_buffer(dev, size, dma_addr)) dma_direct_free(dev, size, vaddr, dma_addr, attrs); } -- cgit v1.2.3 From 0176adb004065d6815a8e67946752df4cd947c5b Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 9 Jan 2018 22:15:30 +0100 Subject: swiotlb: refactor coherent buffer allocation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Factor out a new swiotlb_alloc_buffer helper that allocates DMA coherent memory from the swiotlb bounce buffer. This allows to simplify the swiotlb_alloc implemenation that uses dma_direct_alloc to try to allocate a reachable buffer first. Signed-off-by: Christoph Hellwig Acked-by: Christian König --- lib/swiotlb.c | 122 +++++++++++++++++++++++++++++++--------------------------- 1 file changed, 65 insertions(+), 57 deletions(-) (limited to 'lib') diff --git a/lib/swiotlb.c b/lib/swiotlb.c index 1a147f1354a1..0039b7c5e690 100644 --- a/lib/swiotlb.c +++ b/lib/swiotlb.c @@ -709,75 +709,79 @@ void swiotlb_tbl_sync_single(struct device *hwdev, phys_addr_t tlb_addr, } EXPORT_SYMBOL_GPL(swiotlb_tbl_sync_single); -void * -swiotlb_alloc_coherent(struct device *hwdev, size_t size, - dma_addr_t *dma_handle, gfp_t flags) +static inline bool dma_coherent_ok(struct device *dev, dma_addr_t addr, + size_t size) { - bool warn = !(flags & __GFP_NOWARN); - dma_addr_t dev_addr; - void *ret; - int order = get_order(size); - u64 dma_mask = DMA_BIT_MASK(32); + u64 mask = DMA_BIT_MASK(32); - if (hwdev && hwdev->coherent_dma_mask) - dma_mask = hwdev->coherent_dma_mask; + if (dev && dev->coherent_dma_mask) + mask = dev->coherent_dma_mask; + return addr + size - 1 <= mask; +} - ret = (void *)__get_free_pages(flags, order); - if (ret) { - dev_addr = swiotlb_virt_to_bus(hwdev, ret); - if (dev_addr + size - 1 > dma_mask) { - /* - * The allocated memory isn't reachable by the device. - */ - free_pages((unsigned long) ret, order); - ret = NULL; - } - } - if (!ret) { - /* - * We are either out of memory or the device can't DMA to - * GFP_DMA memory; fall back on map_single(), which - * will grab memory from the lowest available address range. - */ - phys_addr_t paddr = map_single(hwdev, 0, size, DMA_FROM_DEVICE, - warn ? 0 : DMA_ATTR_NO_WARN); - if (paddr == SWIOTLB_MAP_ERROR) - goto err_warn; +static void * +swiotlb_alloc_buffer(struct device *dev, size_t size, dma_addr_t *dma_handle, + unsigned long attrs) +{ + phys_addr_t phys_addr; - ret = phys_to_virt(paddr); - dev_addr = swiotlb_phys_to_dma(hwdev, paddr); + if (swiotlb_force == SWIOTLB_NO_FORCE) + goto out_warn; - /* Confirm address can be DMA'd by device */ - if (dev_addr + size - 1 > dma_mask) { - printk("hwdev DMA mask = 0x%016Lx, dev_addr = 0x%016Lx\n", - (unsigned long long)dma_mask, - (unsigned long long)dev_addr); + phys_addr = swiotlb_tbl_map_single(dev, + swiotlb_phys_to_dma(dev, io_tlb_start), + 0, size, DMA_FROM_DEVICE, 0); + if (phys_addr == SWIOTLB_MAP_ERROR) + goto out_warn; - /* - * DMA_TO_DEVICE to avoid memcpy in unmap_single. - * The DMA_ATTR_SKIP_CPU_SYNC is optional. - */ - swiotlb_tbl_unmap_single(hwdev, paddr, - size, DMA_TO_DEVICE, - DMA_ATTR_SKIP_CPU_SYNC); - goto err_warn; - } - } + *dma_handle = swiotlb_phys_to_dma(dev, phys_addr); + if (dma_coherent_ok(dev, *dma_handle, size)) + goto out_unmap; - *dma_handle = dev_addr; - memset(ret, 0, size); + memset(phys_to_virt(phys_addr), 0, size); + return phys_to_virt(phys_addr); - return ret; +out_unmap: + dev_warn(dev, "hwdev DMA mask = 0x%016Lx, dev_addr = 0x%016Lx\n", + (unsigned long long)(dev ? dev->coherent_dma_mask : 0), + (unsigned long long)*dma_handle); -err_warn: - if (warn && printk_ratelimit()) { - pr_warn("swiotlb: coherent allocation failed for device %s size=%zu\n", - dev_name(hwdev), size); + /* + * DMA_TO_DEVICE to avoid memcpy in unmap_single. + * DMA_ATTR_SKIP_CPU_SYNC is optional. + */ + swiotlb_tbl_unmap_single(dev, phys_addr, size, DMA_TO_DEVICE, + DMA_ATTR_SKIP_CPU_SYNC); +out_warn: + if ((attrs & DMA_ATTR_NO_WARN) && printk_ratelimit()) { + dev_warn(dev, + "swiotlb: coherent allocation failed, size=%zu\n", + size); dump_stack(); } - return NULL; } + +void * +swiotlb_alloc_coherent(struct device *hwdev, size_t size, + dma_addr_t *dma_handle, gfp_t flags) +{ + int order = get_order(size); + unsigned long attrs = (flags & __GFP_NOWARN) ? DMA_ATTR_NO_WARN : 0; + void *ret; + + ret = (void *)__get_free_pages(flags, order); + if (ret) { + *dma_handle = swiotlb_virt_to_bus(hwdev, ret); + if (dma_coherent_ok(hwdev, *dma_handle, size)) { + memset(ret, 0, size); + return ret; + } + free_pages((unsigned long)ret, order); + } + + return swiotlb_alloc_buffer(hwdev, size, dma_handle, attrs); +} EXPORT_SYMBOL(swiotlb_alloc_coherent); static bool swiotlb_free_buffer(struct device *dev, size_t size, @@ -1103,6 +1107,10 @@ void *swiotlb_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, { void *vaddr; + /* temporary workaround: */ + if (gfp & __GFP_NOWARN) + attrs |= DMA_ATTR_NO_WARN; + /* * Don't print a warning when the first allocation attempt fails. * swiotlb_alloc_coherent() will print a warning when the DMA memory @@ -1112,7 +1120,7 @@ void *swiotlb_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, vaddr = dma_direct_alloc(dev, size, dma_handle, gfp, attrs); if (!vaddr) - vaddr = swiotlb_alloc_coherent(dev, size, dma_handle, gfp); + vaddr = swiotlb_alloc_buffer(dev, size, dma_handle, attrs); return vaddr; } -- cgit v1.2.3 From 4bd89ed39b2ab8dc4ac4b6c59b07d420b0213bec Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sun, 24 Dec 2017 14:25:05 +0100 Subject: swiotlb: remove various exports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All these symbols are only used by arch dma_ops implementations or xen-swiotlb. None of which can be modular. Signed-off-by: Christoph Hellwig Acked-by: Christian König --- lib/swiotlb.c | 13 ------------- 1 file changed, 13 deletions(-) (limited to 'lib') diff --git a/lib/swiotlb.c b/lib/swiotlb.c index 0039b7c5e690..c43ec2271469 100644 --- a/lib/swiotlb.c +++ b/lib/swiotlb.c @@ -605,7 +605,6 @@ found: return tlb_addr; } -EXPORT_SYMBOL_GPL(swiotlb_tbl_map_single); /* * Allocates bounce buffer and returns its kernel virtual address. @@ -675,7 +674,6 @@ void swiotlb_tbl_unmap_single(struct device *hwdev, phys_addr_t tlb_addr, } spin_unlock_irqrestore(&io_tlb_lock, flags); } -EXPORT_SYMBOL_GPL(swiotlb_tbl_unmap_single); void swiotlb_tbl_sync_single(struct device *hwdev, phys_addr_t tlb_addr, size_t size, enum dma_data_direction dir, @@ -707,7 +705,6 @@ void swiotlb_tbl_sync_single(struct device *hwdev, phys_addr_t tlb_addr, BUG(); } } -EXPORT_SYMBOL_GPL(swiotlb_tbl_sync_single); static inline bool dma_coherent_ok(struct device *dev, dma_addr_t addr, size_t size) @@ -884,7 +881,6 @@ dma_addr_t swiotlb_map_page(struct device *dev, struct page *page, return swiotlb_phys_to_dma(dev, io_tlb_overflow_buffer); } -EXPORT_SYMBOL_GPL(swiotlb_map_page); /* * Unmap a single streaming mode DMA translation. The dma_addr and size must @@ -925,7 +921,6 @@ void swiotlb_unmap_page(struct device *hwdev, dma_addr_t dev_addr, { unmap_single(hwdev, dev_addr, size, dir, attrs); } -EXPORT_SYMBOL_GPL(swiotlb_unmap_page); /* * Make physical memory consistent for a single streaming mode DMA translation @@ -963,7 +958,6 @@ swiotlb_sync_single_for_cpu(struct device *hwdev, dma_addr_t dev_addr, { swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_CPU); } -EXPORT_SYMBOL(swiotlb_sync_single_for_cpu); void swiotlb_sync_single_for_device(struct device *hwdev, dma_addr_t dev_addr, @@ -971,7 +965,6 @@ swiotlb_sync_single_for_device(struct device *hwdev, dma_addr_t dev_addr, { swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_DEVICE); } -EXPORT_SYMBOL(swiotlb_sync_single_for_device); /* * Map a set of buffers described by scatterlist in streaming mode for DMA. @@ -1023,7 +1016,6 @@ swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, int nelems, } return nelems; } -EXPORT_SYMBOL(swiotlb_map_sg_attrs); /* * Unmap a set of streaming mode DMA translations. Again, cpu read rules @@ -1043,7 +1035,6 @@ swiotlb_unmap_sg_attrs(struct device *hwdev, struct scatterlist *sgl, unmap_single(hwdev, sg->dma_address, sg_dma_len(sg), dir, attrs); } -EXPORT_SYMBOL(swiotlb_unmap_sg_attrs); /* * Make physical memory consistent for a set of streaming mode DMA translations @@ -1071,7 +1062,6 @@ swiotlb_sync_sg_for_cpu(struct device *hwdev, struct scatterlist *sg, { swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_CPU); } -EXPORT_SYMBOL(swiotlb_sync_sg_for_cpu); void swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg, @@ -1079,14 +1069,12 @@ swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg, { swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_DEVICE); } -EXPORT_SYMBOL(swiotlb_sync_sg_for_device); int swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t dma_addr) { return (dma_addr == swiotlb_phys_to_dma(hwdev, io_tlb_overflow_buffer)); } -EXPORT_SYMBOL(swiotlb_dma_mapping_error); /* * Return whether the given device DMA address mask can be supported @@ -1099,7 +1087,6 @@ swiotlb_dma_supported(struct device *hwdev, u64 mask) { return swiotlb_phys_to_dma(hwdev, io_tlb_end - 1) <= mask; } -EXPORT_SYMBOL(swiotlb_dma_supported); #ifdef CONFIG_DMA_DIRECT_OPS void *swiotlb_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, -- cgit v1.2.3