diff options
author | Stephen Rothwell <sfr@canb.auug.org.au> | 2018-11-21 12:25:37 +1100 |
---|---|---|
committer | Stephen Rothwell <sfr@canb.auug.org.au> | 2018-11-21 12:25:37 +1100 |
commit | a1755c5138783eef7a9bde486509eea9a1da024c (patch) | |
tree | 47bdf334b3c0e5c98a12f17463b7092884d6a9b7 /drivers | |
parent | 0aff06d2bfbf9cf0853af6fa81ad828d41f421ff (diff) | |
parent | 5f7b434df1b56e0e4999f1ddc985fa4e6444ead5 (diff) |
Merge remote-tracking branch 'iommu/next'
Diffstat (limited to 'drivers')
-rw-r--r-- | drivers/gpu/drm/i915/i915_gem_execbuffer.c | 2 | ||||
-rw-r--r-- | drivers/gpu/drm/i915/intel_display.c | 2 | ||||
-rw-r--r-- | drivers/gpu/drm/vmwgfx/vmwgfx_drv.c | 2 | ||||
-rw-r--r-- | drivers/iommu/amd_iommu.c | 204 | ||||
-rw-r--r-- | drivers/iommu/amd_iommu_init.c | 3 | ||||
-rw-r--r-- | drivers/iommu/amd_iommu_types.h | 1 | ||||
-rw-r--r-- | drivers/iommu/intel-svm.c | 2 | ||||
-rw-r--r-- | drivers/iommu/iommu.c | 41 | ||||
-rw-r--r-- | drivers/iommu/ipmmu-vmsa.c | 11 | ||||
-rw-r--r-- | drivers/iommu/mtk_iommu.c | 4 | ||||
-rw-r--r-- | drivers/iommu/mtk_iommu_v1.c | 4 | ||||
-rw-r--r-- | drivers/misc/mic/scif/scif_rma.c | 2 | ||||
-rw-r--r-- | drivers/misc/mic/scif/scif_rma.h | 2 | ||||
-rw-r--r-- | drivers/vfio/vfio_iommu_type1.c | 33 |
14 files changed, 180 insertions, 133 deletions
diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c index d4fac09095f8..04f3f7d18fdb 100644 --- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c +++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c @@ -26,7 +26,7 @@ * */ -#include <linux/dma_remapping.h> +#include <linux/intel-iommu.h> #include <linux/reservation.h> #include <linux/sync_file.h> #include <linux/uaccess.h> diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c index 132e978227fb..c051088154c6 100644 --- a/drivers/gpu/drm/i915/intel_display.c +++ b/drivers/gpu/drm/i915/intel_display.c @@ -46,7 +46,7 @@ #include <drm/drm_plane_helper.h> #include <drm/drm_rect.h> #include <drm/drm_atomic_uapi.h> -#include <linux/dma_remapping.h> +#include <linux/intel-iommu.h> #include <linux/reservation.h> /* Primary plane formats for gen <= 3 */ diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c index b9c078860a7c..5788ef6687dc 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c @@ -34,7 +34,7 @@ #include <drm/ttm/ttm_placement.h> #include <drm/ttm/ttm_bo_driver.h> #include <drm/ttm/ttm_module.h> -#include <linux/dma_remapping.h> +#include <linux/intel-iommu.h> #define VMWGFX_DRIVER_DESC "Linux drm driver for VMware graphics devices" #define VMWGFX_CHIP_SVGAII 0 diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 1167ff0416cf..71797b3d67e5 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -1317,6 +1317,101 @@ static void domain_flush_devices(struct protection_domain *domain) * ****************************************************************************/ +static void free_page_list(struct page *freelist) +{ + while (freelist != NULL) { + unsigned long p = (unsigned long)page_address(freelist); + freelist = freelist->freelist; + free_page(p); + } +} + +static struct page *free_pt_page(unsigned long pt, struct page *freelist) +{ + struct page *p = virt_to_page((void *)pt); + + p->freelist = freelist; + + return p; +} + +#define DEFINE_FREE_PT_FN(LVL, FN) \ +static struct page *free_pt_##LVL (unsigned long __pt, struct page *freelist) \ +{ \ + unsigned long p; \ + u64 *pt; \ + int i; \ + \ + pt = (u64 *)__pt; \ + \ + for (i = 0; i < 512; ++i) { \ + /* PTE present? */ \ + if (!IOMMU_PTE_PRESENT(pt[i])) \ + continue; \ + \ + /* Large PTE? */ \ + if (PM_PTE_LEVEL(pt[i]) == 0 || \ + PM_PTE_LEVEL(pt[i]) == 7) \ + continue; \ + \ + p = (unsigned long)IOMMU_PTE_PAGE(pt[i]); \ + freelist = FN(p, freelist); \ + } \ + \ + return free_pt_page((unsigned long)pt, freelist); \ +} + +DEFINE_FREE_PT_FN(l2, free_pt_page) +DEFINE_FREE_PT_FN(l3, free_pt_l2) +DEFINE_FREE_PT_FN(l4, free_pt_l3) +DEFINE_FREE_PT_FN(l5, free_pt_l4) +DEFINE_FREE_PT_FN(l6, free_pt_l5) + +static struct page *free_sub_pt(unsigned long root, int mode, + struct page *freelist) +{ + switch (mode) { + case PAGE_MODE_NONE: + case PAGE_MODE_7_LEVEL: + break; + case PAGE_MODE_1_LEVEL: + freelist = free_pt_page(root, freelist); + break; + case PAGE_MODE_2_LEVEL: + freelist = free_pt_l2(root, freelist); + break; + case PAGE_MODE_3_LEVEL: + freelist = free_pt_l3(root, freelist); + break; + case PAGE_MODE_4_LEVEL: + freelist = free_pt_l4(root, freelist); + break; + case PAGE_MODE_5_LEVEL: + freelist = free_pt_l5(root, freelist); + break; + case PAGE_MODE_6_LEVEL: + freelist = free_pt_l6(root, freelist); + break; + default: + BUG(); + } + + return freelist; +} + +static void free_pagetable(struct protection_domain *domain) +{ + unsigned long root = (unsigned long)domain->pt_root; + struct page *freelist = NULL; + + BUG_ON(domain->mode < PAGE_MODE_NONE || + domain->mode > PAGE_MODE_6_LEVEL); + + free_sub_pt(root, domain->mode, freelist); + + free_page_list(freelist); +} + /* * This function is used to add another level to an IO page table. Adding * another level increases the size of the address space by 9 bits to a size up @@ -1365,10 +1460,13 @@ static u64 *alloc_pte(struct protection_domain *domain, while (level > end_lvl) { u64 __pte, __npte; + int pte_level; - __pte = *pte; + __pte = *pte; + pte_level = PM_PTE_LEVEL(__pte); - if (!IOMMU_PTE_PRESENT(__pte)) { + if (!IOMMU_PTE_PRESENT(__pte) || + pte_level == PAGE_MODE_7_LEVEL) { page = (u64 *)get_zeroed_page(gfp); if (!page) return NULL; @@ -1376,19 +1474,21 @@ static u64 *alloc_pte(struct protection_domain *domain, __npte = PM_LEVEL_PDE(level, iommu_virt_to_phys(page)); /* pte could have been changed somewhere. */ - if (cmpxchg64(pte, __pte, __npte) != __pte) { + if (cmpxchg64(pte, __pte, __npte) != __pte) free_page((unsigned long)page); - continue; - } + else if (pte_level == PAGE_MODE_7_LEVEL) + domain->updated = true; + + continue; } /* No level skipping support yet */ - if (PM_PTE_LEVEL(*pte) != level) + if (pte_level != level) return NULL; level -= 1; - pte = IOMMU_PTE_PAGE(*pte); + pte = IOMMU_PTE_PAGE(__pte); if (pte_page && level == end_lvl) *pte_page = pte; @@ -1457,6 +1557,25 @@ static u64 *fetch_pte(struct protection_domain *domain, return pte; } +static struct page *free_clear_pte(u64 *pte, u64 pteval, struct page *freelist) +{ + unsigned long pt; + int mode; + + while (cmpxchg64(pte, pteval, 0) != pteval) { + pr_warn("AMD-Vi: IOMMU pte changed since we read it\n"); + pteval = *pte; + } + + if (!IOMMU_PTE_PRESENT(pteval)) + return freelist; + + pt = (unsigned long)IOMMU_PTE_PAGE(pteval); + mode = IOMMU_PTE_MODE(pteval); + + return free_sub_pt(pt, mode, freelist); +} + /* * Generic mapping functions. It maps a physical address into a DMA * address space. It allocates the page table pages if necessary. @@ -1471,6 +1590,7 @@ static int iommu_map_page(struct protection_domain *dom, int prot, gfp_t gfp) { + struct page *freelist = NULL; u64 __pte, *pte; int i, count; @@ -1487,8 +1607,10 @@ static int iommu_map_page(struct protection_domain *dom, return -ENOMEM; for (i = 0; i < count; ++i) - if (IOMMU_PTE_PRESENT(pte[i])) - return -EBUSY; + freelist = free_clear_pte(&pte[i], pte[i], freelist); + + if (freelist != NULL) + dom->updated = true; if (count > 1) { __pte = PAGE_SIZE_PTE(__sme_set(phys_addr), page_size); @@ -1506,6 +1628,9 @@ static int iommu_map_page(struct protection_domain *dom, update_domain(dom); + /* Everything flushed out, free pages now */ + free_page_list(freelist); + return 0; } @@ -1638,67 +1763,6 @@ static void domain_id_free(int id) spin_unlock(&pd_bitmap_lock); } -#define DEFINE_FREE_PT_FN(LVL, FN) \ -static void free_pt_##LVL (unsigned long __pt) \ -{ \ - unsigned long p; \ - u64 *pt; \ - int i; \ - \ - pt = (u64 *)__pt; \ - \ - for (i = 0; i < 512; ++i) { \ - /* PTE present? */ \ - if (!IOMMU_PTE_PRESENT(pt[i])) \ - continue; \ - \ - /* Large PTE? */ \ - if (PM_PTE_LEVEL(pt[i]) == 0 || \ - PM_PTE_LEVEL(pt[i]) == 7) \ - continue; \ - \ - p = (unsigned long)IOMMU_PTE_PAGE(pt[i]); \ - FN(p); \ - } \ - free_page((unsigned long)pt); \ -} - -DEFINE_FREE_PT_FN(l2, free_page) -DEFINE_FREE_PT_FN(l3, free_pt_l2) -DEFINE_FREE_PT_FN(l4, free_pt_l3) -DEFINE_FREE_PT_FN(l5, free_pt_l4) -DEFINE_FREE_PT_FN(l6, free_pt_l5) - -static void free_pagetable(struct protection_domain *domain) -{ - unsigned long root = (unsigned long)domain->pt_root; - - switch (domain->mode) { - case PAGE_MODE_NONE: - break; - case PAGE_MODE_1_LEVEL: - free_page(root); - break; - case PAGE_MODE_2_LEVEL: - free_pt_l2(root); - break; - case PAGE_MODE_3_LEVEL: - free_pt_l3(root); - break; - case PAGE_MODE_4_LEVEL: - free_pt_l4(root); - break; - case PAGE_MODE_5_LEVEL: - free_pt_l5(root); - break; - case PAGE_MODE_6_LEVEL: - free_pt_l6(root); - break; - default: - BUG(); - } -} - static void free_gcr3_tbl_level1(u64 *tbl) { u64 *ptr; diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c index bb2cd29e1658..d8f7000a466a 100644 --- a/drivers/iommu/amd_iommu_init.c +++ b/drivers/iommu/amd_iommu_init.c @@ -797,7 +797,8 @@ static int iommu_init_ga_log(struct amd_iommu *iommu) entry = iommu_virt_to_phys(iommu->ga_log) | GA_LOG_SIZE_512; memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_BASE_OFFSET, &entry, sizeof(entry)); - entry = (iommu_virt_to_phys(iommu->ga_log) & 0xFFFFFFFFFFFFFULL) & ~7ULL; + entry = (iommu_virt_to_phys(iommu->ga_log_tail) & + (BIT_ULL(52)-1)) & ~7ULL; memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_TAIL_OFFSET, &entry, sizeof(entry)); writel(0x00, iommu->mmio_base + MMIO_GA_HEAD_OFFSET); diff --git a/drivers/iommu/amd_iommu_types.h b/drivers/iommu/amd_iommu_types.h index e2b342e65a7b..eae0741f72dc 100644 --- a/drivers/iommu/amd_iommu_types.h +++ b/drivers/iommu/amd_iommu_types.h @@ -269,6 +269,7 @@ #define PAGE_MODE_4_LEVEL 0x04 #define PAGE_MODE_5_LEVEL 0x05 #define PAGE_MODE_6_LEVEL 0x06 +#define PAGE_MODE_7_LEVEL 0x07 #define PM_LEVEL_SHIFT(x) (12 + ((x) * 9)) #define PM_LEVEL_SIZE(x) (((x) < 6) ? \ diff --git a/drivers/iommu/intel-svm.c b/drivers/iommu/intel-svm.c index db301efe126d..887150907526 100644 --- a/drivers/iommu/intel-svm.c +++ b/drivers/iommu/intel-svm.c @@ -595,7 +595,7 @@ static irqreturn_t prq_event_thread(int irq, void *d) pr_err("%s: Page request without PASID: %08llx %08llx\n", iommu->name, ((unsigned long long *)req)[0], ((unsigned long long *)req)[1]); - goto bad_req; + goto no_pasid; } if (!svm || svm->pasid != req->pasid) { diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index edbdf5d6962c..f8ec49e0f6c6 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -1712,33 +1712,32 @@ EXPORT_SYMBOL_GPL(iommu_unmap_fast); size_t iommu_map_sg(struct iommu_domain *domain, unsigned long iova, struct scatterlist *sg, unsigned int nents, int prot) { - struct scatterlist *s; - size_t mapped = 0; - unsigned int i, min_pagesz; + size_t len = 0, mapped = 0; + phys_addr_t start; + unsigned int i = 0; int ret; - if (unlikely(domain->pgsize_bitmap == 0UL)) - return 0; - - min_pagesz = 1 << __ffs(domain->pgsize_bitmap); + while (i <= nents) { + phys_addr_t s_phys = sg_phys(sg); - for_each_sg(sg, s, nents, i) { - phys_addr_t phys = page_to_phys(sg_page(s)) + s->offset; + if (len && s_phys != start + len) { + ret = iommu_map(domain, iova + mapped, start, len, prot); + if (ret) + goto out_err; - /* - * We are mapping on IOMMU page boundaries, so offset within - * the page must be 0. However, the IOMMU may support pages - * smaller than PAGE_SIZE, so s->offset may still represent - * an offset of that boundary within the CPU page. - */ - if (!IS_ALIGNED(s->offset, min_pagesz)) - goto out_err; + mapped += len; + len = 0; + } - ret = iommu_map(domain, iova + mapped, phys, s->length, prot); - if (ret) - goto out_err; + if (len) { + len += sg->length; + } else { + len = sg->length; + start = s_phys; + } - mapped += s->length; + if (++i < nents) + sg = sg_next(sg); } return mapped; diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index b98a03189580..9e2655f1c1bf 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -498,6 +498,9 @@ static int ipmmu_domain_init_context(struct ipmmu_vmsa_domain *domain) static void ipmmu_domain_destroy_context(struct ipmmu_vmsa_domain *domain) { + if (!domain->mmu) + return; + /* * Disable the context. Flush the TLB as required when modifying the * context registers. @@ -758,10 +761,12 @@ static bool ipmmu_slave_whitelist(struct device *dev) } static const struct soc_device_attribute soc_rcar_gen3[] = { + { .soc_id = "r8a774a1", }, { .soc_id = "r8a7795", }, { .soc_id = "r8a7796", }, { .soc_id = "r8a77965", }, { .soc_id = "r8a77970", }, + { .soc_id = "r8a77990", }, { .soc_id = "r8a77995", }, { /* sentinel */ } }; @@ -938,6 +943,9 @@ static const struct of_device_id ipmmu_of_ids[] = { .compatible = "renesas,ipmmu-vmsa", .data = &ipmmu_features_default, }, { + .compatible = "renesas,ipmmu-r8a774a1", + .data = &ipmmu_features_rcar_gen3, + }, { .compatible = "renesas,ipmmu-r8a7795", .data = &ipmmu_features_rcar_gen3, }, { @@ -950,6 +958,9 @@ static const struct of_device_id ipmmu_of_ids[] = { .compatible = "renesas,ipmmu-r8a77970", .data = &ipmmu_features_rcar_gen3, }, { + .compatible = "renesas,ipmmu-r8a77990", + .data = &ipmmu_features_rcar_gen3, + }, { .compatible = "renesas,ipmmu-r8a77995", .data = &ipmmu_features_rcar_gen3, }, { diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 44bd5b9166bb..d69a9ffa58a4 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -113,7 +113,7 @@ struct mtk_iommu_domain { struct iommu_domain domain; }; -static struct iommu_ops mtk_iommu_ops; +static const struct iommu_ops mtk_iommu_ops; static LIST_HEAD(m4ulist); /* List all the M4U HWs */ @@ -488,7 +488,7 @@ static int mtk_iommu_of_xlate(struct device *dev, struct of_phandle_args *args) return iommu_fwspec_add_ids(dev, args->args, 1); } -static struct iommu_ops mtk_iommu_ops = { +static const struct iommu_ops mtk_iommu_ops = { .domain_alloc = mtk_iommu_domain_alloc, .domain_free = mtk_iommu_domain_free, .attach_dev = mtk_iommu_attach_device, diff --git a/drivers/iommu/mtk_iommu_v1.c b/drivers/iommu/mtk_iommu_v1.c index 0e780848f59b..27867b862d7a 100644 --- a/drivers/iommu/mtk_iommu_v1.c +++ b/drivers/iommu/mtk_iommu_v1.c @@ -362,7 +362,7 @@ static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain, return pa; } -static struct iommu_ops mtk_iommu_ops; +static const struct iommu_ops mtk_iommu_ops; /* * MTK generation one iommu HW only support one iommu domain, and all the client @@ -524,7 +524,7 @@ static int mtk_iommu_hw_init(const struct mtk_iommu_data *data) return 0; } -static struct iommu_ops mtk_iommu_ops = { +static const struct iommu_ops mtk_iommu_ops = { .domain_alloc = mtk_iommu_domain_alloc, .domain_free = mtk_iommu_domain_free, .attach_dev = mtk_iommu_attach_device, diff --git a/drivers/misc/mic/scif/scif_rma.c b/drivers/misc/mic/scif/scif_rma.c index c824329f7012..b441f6b0c743 100644 --- a/drivers/misc/mic/scif/scif_rma.c +++ b/drivers/misc/mic/scif/scif_rma.c @@ -15,7 +15,7 @@ * Intel SCIF driver. * */ -#include <linux/dma_remapping.h> +#include <linux/intel-iommu.h> #include <linux/pagemap.h> #include <linux/sched/mm.h> #include <linux/sched/signal.h> diff --git a/drivers/misc/mic/scif/scif_rma.h b/drivers/misc/mic/scif/scif_rma.h index fa6722279196..d90a06d4e93b 100644 --- a/drivers/misc/mic/scif/scif_rma.h +++ b/drivers/misc/mic/scif/scif_rma.h @@ -53,7 +53,7 @@ #ifndef SCIF_RMA_H #define SCIF_RMA_H -#include <linux/dma_remapping.h> +#include <linux/intel-iommu.h> #include <linux/mmu_notifier.h> #include "../bus/scif_bus.h" diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index d9fd3188615d..7651cfb14836 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -978,32 +978,6 @@ unlock: return ret; } -/* - * Turns out AMD IOMMU has a page table bug where it won't map large pages - * to a region that previously mapped smaller pages. This should be fixed - * soon, so this is just a temporary workaround to break mappings down into - * PAGE_SIZE. Better to map smaller pages than nothing. - */ -static int map_try_harder(struct vfio_domain *domain, dma_addr_t iova, - unsigned long pfn, long npage, int prot) -{ - long i; - int ret = 0; - - for (i = 0; i < npage; i++, pfn++, iova += PAGE_SIZE) { - ret = iommu_map(domain->domain, iova, - (phys_addr_t)pfn << PAGE_SHIFT, - PAGE_SIZE, prot | domain->prot); - if (ret) - break; - } - - for (; i < npage && i > 0; i--, iova -= PAGE_SIZE) - iommu_unmap(domain->domain, iova, PAGE_SIZE); - - return ret; -} - static int vfio_iommu_map(struct vfio_iommu *iommu, dma_addr_t iova, unsigned long pfn, long npage, int prot) { @@ -1013,11 +987,8 @@ static int vfio_iommu_map(struct vfio_iommu *iommu, dma_addr_t iova, list_for_each_entry(d, &iommu->domain_list, next) { ret = iommu_map(d->domain, iova, (phys_addr_t)pfn << PAGE_SHIFT, npage << PAGE_SHIFT, prot | d->prot); - if (ret) { - if (ret != -EBUSY || - map_try_harder(d, iova, pfn, npage, prot)) - goto unwind; - } + if (ret) + goto unwind; cond_resched(); } |