diff options
Diffstat (limited to 'drivers/gpu/drm/i915/i915_gem_gtt.c')
| -rw-r--r-- | drivers/gpu/drm/i915/i915_gem_gtt.c | 383 |
1 files changed, 183 insertions, 200 deletions
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c index 49e4f26b79d8..7cfafdc80b17 100644 --- a/drivers/gpu/drm/i915/i915_gem_gtt.c +++ b/drivers/gpu/drm/i915/i915_gem_gtt.c @@ -658,7 +658,7 @@ static int gen8_write_pdp(struct drm_i915_gem_request *req, unsigned entry, dma_addr_t addr) { - struct intel_engine_cs *ring = req->ring; + struct intel_engine_cs *engine = req->engine; int ret; BUG_ON(entry >= 4); @@ -667,13 +667,13 @@ static int gen8_write_pdp(struct drm_i915_gem_request *req, if (ret) return ret; - intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); - intel_ring_emit_reg(ring, GEN8_RING_PDP_UDW(ring, entry)); - intel_ring_emit(ring, upper_32_bits(addr)); - intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); - intel_ring_emit_reg(ring, GEN8_RING_PDP_LDW(ring, entry)); - intel_ring_emit(ring, lower_32_bits(addr)); - intel_ring_advance(ring); + intel_ring_emit(engine, MI_LOAD_REGISTER_IMM(1)); + intel_ring_emit_reg(engine, GEN8_RING_PDP_UDW(engine, entry)); + intel_ring_emit(engine, upper_32_bits(addr)); + intel_ring_emit(engine, MI_LOAD_REGISTER_IMM(1)); + intel_ring_emit_reg(engine, GEN8_RING_PDP_LDW(engine, entry)); + intel_ring_emit(engine, lower_32_bits(addr)); + intel_ring_advance(engine); return 0; } @@ -1637,7 +1637,7 @@ static void gen6_write_page_range(struct drm_i915_private *dev_priv, /* Make sure write is complete before other code can use this page * table. Also require for WC mapped PTEs */ - readl(dev_priv->gtt.gsm); + readl(dev_priv->ggtt.gsm); } static uint32_t get_pd_offset(struct i915_hw_ppgtt *ppgtt) @@ -1650,11 +1650,11 @@ static uint32_t get_pd_offset(struct i915_hw_ppgtt *ppgtt) static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt, struct drm_i915_gem_request *req) { - struct intel_engine_cs *ring = req->ring; + struct intel_engine_cs *engine = req->engine; int ret; /* NB: TLBs must be flushed and invalidated before a switch */ - ret = ring->flush(req, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS); + ret = engine->flush(req, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS); if (ret) return ret; @@ -1662,13 +1662,13 @@ static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt, if (ret) return ret; - intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2)); - intel_ring_emit_reg(ring, RING_PP_DIR_DCLV(ring)); - intel_ring_emit(ring, PP_DIR_DCLV_2G); - intel_ring_emit_reg(ring, RING_PP_DIR_BASE(ring)); - intel_ring_emit(ring, get_pd_offset(ppgtt)); - intel_ring_emit(ring, MI_NOOP); - intel_ring_advance(ring); + intel_ring_emit(engine, MI_LOAD_REGISTER_IMM(2)); + intel_ring_emit_reg(engine, RING_PP_DIR_DCLV(engine)); + intel_ring_emit(engine, PP_DIR_DCLV_2G); + intel_ring_emit_reg(engine, RING_PP_DIR_BASE(engine)); + intel_ring_emit(engine, get_pd_offset(ppgtt)); + intel_ring_emit(engine, MI_NOOP); + intel_ring_advance(engine); return 0; } @@ -1676,22 +1676,22 @@ static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt, static int vgpu_mm_switch(struct i915_hw_ppgtt *ppgtt, struct drm_i915_gem_request *req) { - struct intel_engine_cs *ring = req->ring; + struct intel_engine_cs *engine = req->engine; struct drm_i915_private *dev_priv = to_i915(ppgtt->base.dev); - I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G); - I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt)); + I915_WRITE(RING_PP_DIR_DCLV(engine), PP_DIR_DCLV_2G); + I915_WRITE(RING_PP_DIR_BASE(engine), get_pd_offset(ppgtt)); return 0; } static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt, struct drm_i915_gem_request *req) { - struct intel_engine_cs *ring = req->ring; + struct intel_engine_cs *engine = req->engine; int ret; /* NB: TLBs must be flushed and invalidated before a switch */ - ret = ring->flush(req, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS); + ret = engine->flush(req, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS); if (ret) return ret; @@ -1699,17 +1699,17 @@ static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt, if (ret) return ret; - intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2)); - intel_ring_emit_reg(ring, RING_PP_DIR_DCLV(ring)); - intel_ring_emit(ring, PP_DIR_DCLV_2G); - intel_ring_emit_reg(ring, RING_PP_DIR_BASE(ring)); - intel_ring_emit(ring, get_pd_offset(ppgtt)); - intel_ring_emit(ring, MI_NOOP); - intel_ring_advance(ring); + intel_ring_emit(engine, MI_LOAD_REGISTER_IMM(2)); + intel_ring_emit_reg(engine, RING_PP_DIR_DCLV(engine)); + intel_ring_emit(engine, PP_DIR_DCLV_2G); + intel_ring_emit_reg(engine, RING_PP_DIR_BASE(engine)); + intel_ring_emit(engine, get_pd_offset(ppgtt)); + intel_ring_emit(engine, MI_NOOP); + intel_ring_advance(engine); /* XXX: RCS is the only one to auto invalidate the TLBs? */ - if (ring->id != RCS) { - ret = ring->flush(req, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS); + if (engine->id != RCS) { + ret = engine->flush(req, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS); if (ret) return ret; } @@ -1720,15 +1720,15 @@ static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt, static int gen6_mm_switch(struct i915_hw_ppgtt *ppgtt, struct drm_i915_gem_request *req) { - struct intel_engine_cs *ring = req->ring; + struct intel_engine_cs *engine = req->engine; struct drm_device *dev = ppgtt->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; - I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G); - I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt)); + I915_WRITE(RING_PP_DIR_DCLV(engine), PP_DIR_DCLV_2G); + I915_WRITE(RING_PP_DIR_BASE(engine), get_pd_offset(ppgtt)); - POSTING_READ(RING_PP_DIR_DCLV(ring)); + POSTING_READ(RING_PP_DIR_DCLV(engine)); return 0; } @@ -1736,12 +1736,11 @@ static int gen6_mm_switch(struct i915_hw_ppgtt *ppgtt, static void gen8_ppgtt_enable(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring; - int j; + struct intel_engine_cs *engine; - for_each_ring(ring, dev_priv, j) { + for_each_engine(engine, dev_priv) { u32 four_level = USES_FULL_48BIT_PPGTT(dev) ? GEN8_GFX_PPGTT_48B : 0; - I915_WRITE(RING_MODE_GEN7(ring), + I915_WRITE(RING_MODE_GEN7(engine), _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE | four_level)); } } @@ -1749,9 +1748,8 @@ static void gen8_ppgtt_enable(struct drm_device *dev) static void gen7_ppgtt_enable(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring; + struct intel_engine_cs *engine; uint32_t ecochk, ecobits; - int i; ecobits = I915_READ(GAC_ECO_BITS); I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B); @@ -1765,9 +1763,9 @@ static void gen7_ppgtt_enable(struct drm_device *dev) } I915_WRITE(GAM_ECOCHK, ecochk); - for_each_ring(ring, dev_priv, i) { + for_each_engine(engine, dev_priv) { /* GFX_MODE is per-ring on gen7+ */ - I915_WRITE(RING_MODE_GEN7(ring), + I915_WRITE(RING_MODE_GEN7(engine), _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); } } @@ -1932,7 +1930,7 @@ static int gen6_alloc_va_range(struct i915_address_space *vm, /* Make sure write is complete before other code can use this page * table. Also require for WC mapped PTEs */ - readl(dev_priv->gtt.gsm); + readl(dev_priv->ggtt.gsm); mark_tlbs_dirty(ppgtt); return 0; @@ -2005,23 +2003,23 @@ static int gen6_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt) * allocator works in address space sizes, so it's multiplied by page * size. We allocate at the top of the GTT to avoid fragmentation. */ - BUG_ON(!drm_mm_initialized(&dev_priv->gtt.base.mm)); + BUG_ON(!drm_mm_initialized(&dev_priv->ggtt.base.mm)); ret = gen6_init_scratch(vm); if (ret) return ret; alloc: - ret = drm_mm_insert_node_in_range_generic(&dev_priv->gtt.base.mm, + ret = drm_mm_insert_node_in_range_generic(&dev_priv->ggtt.base.mm, &ppgtt->node, GEN6_PD_SIZE, GEN6_PD_ALIGN, 0, - 0, dev_priv->gtt.base.total, + 0, dev_priv->ggtt.base.total, DRM_MM_TOPDOWN); if (ret == -ENOSPC && !retried) { - ret = i915_gem_evict_something(dev, &dev_priv->gtt.base, + ret = i915_gem_evict_something(dev, &dev_priv->ggtt.base, GEN6_PD_SIZE, GEN6_PD_ALIGN, I915_CACHE_NONE, - 0, dev_priv->gtt.base.total, + 0, dev_priv->ggtt.base.total, 0); if (ret) goto err_out; @@ -2034,7 +2032,7 @@ alloc: goto err_out; - if (ppgtt->node.start < dev_priv->gtt.mappable_end) + if (ppgtt->node.start < dev_priv->ggtt.mappable_end) DRM_DEBUG("Forced to use aperture for PDEs\n"); return 0; @@ -2065,7 +2063,7 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) struct drm_i915_private *dev_priv = dev->dev_private; int ret; - ppgtt->base.pte_encode = dev_priv->gtt.base.pte_encode; + ppgtt->base.pte_encode = dev_priv->ggtt.base.pte_encode; if (IS_GEN6(dev)) { ppgtt->switch_mm = gen6_mm_switch; } else if (IS_HASWELL(dev)) { @@ -2095,7 +2093,7 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) ppgtt->pd.base.ggtt_offset = ppgtt->node.start / PAGE_SIZE * sizeof(gen6_pte_t); - ppgtt->pd_addr = (gen6_pte_t __iomem *)dev_priv->gtt.gsm + + ppgtt->pd_addr = (gen6_pte_t __iomem *)dev_priv->ggtt.gsm + ppgtt->pd.base.ggtt_offset / sizeof(gen6_pte_t); gen6_scratch_va_range(ppgtt, 0, ppgtt->base.total); @@ -2192,7 +2190,7 @@ int i915_ppgtt_init_hw(struct drm_device *dev) int i915_ppgtt_init_ring(struct drm_i915_gem_request *req) { - struct drm_i915_private *dev_priv = req->ring->dev->dev_private; + struct drm_i915_private *dev_priv = req->i915; struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; if (i915.enable_execlists) @@ -2265,7 +2263,7 @@ static bool do_idling(struct drm_i915_private *dev_priv) { bool ret = dev_priv->mm.interruptible; - if (unlikely(dev_priv->gtt.do_idle_maps)) { + if (unlikely(dev_priv->ggtt.do_idle_maps)) { dev_priv->mm.interruptible = false; if (i915_gpu_idle(dev_priv->dev)) { DRM_ERROR("Couldn't idle GPU\n"); @@ -2279,22 +2277,21 @@ static bool do_idling(struct drm_i915_private *dev_priv) static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible) { - if (unlikely(dev_priv->gtt.do_idle_maps)) + if (unlikely(dev_priv->ggtt.do_idle_maps)) dev_priv->mm.interruptible = interruptible; } void i915_check_and_clear_faults(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - struct intel_engine_cs *ring; - int i; + struct intel_engine_cs *engine; if (INTEL_INFO(dev)->gen < 6) return; - for_each_ring(ring, dev_priv, i) { + for_each_engine(engine, dev_priv) { u32 fault_reg; - fault_reg = I915_READ(RING_FAULT_REG(ring)); + fault_reg = I915_READ(RING_FAULT_REG(engine)); if (fault_reg & RING_FAULT_VALID) { DRM_DEBUG_DRIVER("Unexpected fault\n" "\tAddr: 0x%08lx\n" @@ -2305,11 +2302,11 @@ void i915_check_and_clear_faults(struct drm_device *dev) fault_reg & RING_FAULT_GTTSEL_MASK ? "GGTT" : "PPGTT", RING_FAULT_SRCID(fault_reg), RING_FAULT_FAULT_TYPE(fault_reg)); - I915_WRITE(RING_FAULT_REG(ring), + I915_WRITE(RING_FAULT_REG(engine), fault_reg & ~RING_FAULT_VALID); } } - POSTING_READ(RING_FAULT_REG(&dev_priv->ring[RCS])); + POSTING_READ(RING_FAULT_REG(&dev_priv->engine[RCS])); } static void i915_ggtt_flush(struct drm_i915_private *dev_priv) @@ -2334,9 +2331,9 @@ void i915_gem_suspend_gtt_mappings(struct drm_device *dev) i915_check_and_clear_faults(dev); - dev_priv->gtt.base.clear_range(&dev_priv->gtt.base, - dev_priv->gtt.base.start, - dev_priv->gtt.base.total, + dev_priv->ggtt.base.clear_range(&dev_priv->ggtt.base, + dev_priv->ggtt.base.start, + dev_priv->ggtt.base.total, true); i915_ggtt_flush(dev_priv); @@ -2370,7 +2367,7 @@ static void gen8_ggtt_insert_entries(struct i915_address_space *vm, struct drm_i915_private *dev_priv = vm->dev->dev_private; unsigned first_entry = start >> PAGE_SHIFT; gen8_pte_t __iomem *gtt_entries = - (gen8_pte_t __iomem *)dev_priv->gtt.gsm + first_entry; + (gen8_pte_t __iomem *)dev_priv->ggtt.gsm + first_entry; int i = 0; struct sg_page_iter sg_iter; dma_addr_t addr = 0; /* shut up gcc */ @@ -2447,7 +2444,7 @@ static void gen6_ggtt_insert_entries(struct i915_address_space *vm, struct drm_i915_private *dev_priv = vm->dev->dev_private; unsigned first_entry = start >> PAGE_SHIFT; gen6_pte_t __iomem *gtt_entries = - (gen6_pte_t __iomem *)dev_priv->gtt.gsm + first_entry; + (gen6_pte_t __iomem *)dev_priv->ggtt.gsm + first_entry; int i = 0; struct sg_page_iter sg_iter; dma_addr_t addr = 0; @@ -2491,8 +2488,8 @@ static void gen8_ggtt_clear_range(struct i915_address_space *vm, unsigned first_entry = start >> PAGE_SHIFT; unsigned num_entries = length >> PAGE_SHIFT; gen8_pte_t scratch_pte, __iomem *gtt_base = - (gen8_pte_t __iomem *) dev_priv->gtt.gsm + first_entry; - const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry; + (gen8_pte_t __iomem *) dev_priv->ggtt.gsm + first_entry; + const int max_entries = gtt_total_entries(dev_priv->ggtt) - first_entry; int i; int rpm_atomic_seq; @@ -2522,8 +2519,8 @@ static void gen6_ggtt_clear_range(struct i915_address_space *vm, unsigned first_entry = start >> PAGE_SHIFT; unsigned num_entries = length >> PAGE_SHIFT; gen6_pte_t scratch_pte, __iomem *gtt_base = - (gen6_pte_t __iomem *) dev_priv->gtt.gsm + first_entry; - const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry; + (gen6_pte_t __iomem *) dev_priv->ggtt.gsm + first_entry; + const int max_entries = gtt_total_entries(dev_priv->ggtt) - first_entry; int i; int rpm_atomic_seq; @@ -2613,32 +2610,31 @@ static int aliasing_gtt_bind_vma(struct i915_vma *vma, enum i915_cache_level cache_level, u32 flags) { - struct drm_device *dev = vma->vm->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_i915_gem_object *obj = vma->obj; - struct sg_table *pages = obj->pages; - u32 pte_flags = 0; + u32 pte_flags; int ret; ret = i915_get_ggtt_vma_pages(vma); if (ret) return ret; - pages = vma->ggtt_view.pages; /* Currently applicable only to VLV */ - if (obj->gt_ro) + pte_flags = 0; + if (vma->obj->gt_ro) pte_flags |= PTE_READ_ONLY; if (flags & GLOBAL_BIND) { - vma->vm->insert_entries(vma->vm, pages, + vma->vm->insert_entries(vma->vm, + vma->ggtt_view.pages, vma->node.start, cache_level, pte_flags); } if (flags & LOCAL_BIND) { - struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt; - appgtt->base.insert_entries(&appgtt->base, pages, + struct i915_hw_ppgtt *appgtt = + to_i915(vma->vm->dev)->mm.aliasing_ppgtt; + appgtt->base.insert_entries(&appgtt->base, + vma->ggtt_view.pages, vma->node.start, cache_level, pte_flags); } @@ -2718,7 +2714,7 @@ static int i915_gem_setup_global_gtt(struct drm_device *dev, * of the aperture. */ struct drm_i915_private *dev_priv = dev->dev_private; - struct i915_address_space *ggtt_vm = &dev_priv->gtt.base; + struct i915_address_space *ggtt_vm = &dev_priv->ggtt.base; struct drm_mm_node *entry; struct drm_i915_gem_object *obj; unsigned long hole_start, hole_end; @@ -2801,8 +2797,8 @@ static int i915_gem_setup_global_gtt(struct drm_device *dev, true); dev_priv->mm.aliasing_ppgtt = ppgtt; - WARN_ON(dev_priv->gtt.base.bind_vma != ggtt_bind_vma); - dev_priv->gtt.base.bind_vma = aliasing_gtt_bind_vma; + WARN_ON(dev_priv->ggtt.base.bind_vma != ggtt_bind_vma); + dev_priv->ggtt.base.bind_vma = aliasing_gtt_bind_vma; } return 0; @@ -2813,8 +2809,8 @@ void i915_gem_init_global_gtt(struct drm_device *dev) struct drm_i915_private *dev_priv = dev->dev_private; u64 gtt_size, mappable_size; - gtt_size = dev_priv->gtt.base.total; - mappable_size = dev_priv->gtt.mappable_end; + gtt_size = dev_priv->ggtt.base.total; + mappable_size = dev_priv->ggtt.mappable_end; i915_gem_setup_global_gtt(dev, 0, mappable_size, gtt_size); } @@ -2822,7 +2818,7 @@ void i915_gem_init_global_gtt(struct drm_device *dev) void i915_global_gtt_cleanup(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - struct i915_address_space *vm = &dev_priv->gtt.base; + struct i915_address_space *vm = &dev_priv->ggtt.base; if (dev_priv->mm.aliasing_ppgtt) { struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; @@ -2940,10 +2936,10 @@ static int ggtt_probe_common(struct drm_device *dev, * readback check when writing GTT PTE entries. */ if (IS_BROXTON(dev)) - dev_priv->gtt.gsm = ioremap_nocache(gtt_phys_addr, gtt_size); + dev_priv->ggtt.gsm = ioremap_nocache(gtt_phys_addr, gtt_size); else - dev_priv->gtt.gsm = ioremap_wc(gtt_phys_addr, gtt_size); - if (!dev_priv->gtt.gsm) { + dev_priv->ggtt.gsm = ioremap_wc(gtt_phys_addr, gtt_size); + if (!dev_priv->ggtt.gsm) { DRM_ERROR("Failed to map the gtt page table\n"); return -ENOMEM; } @@ -2952,11 +2948,11 @@ static int ggtt_probe_common(struct drm_device *dev, if (IS_ERR(scratch_page)) { DRM_ERROR("Scratch setup failed\n"); /* iounmap will also get called at remove, but meh */ - iounmap(dev_priv->gtt.gsm); + iounmap(dev_priv->ggtt.gsm); return PTR_ERR(scratch_page); } - dev_priv->gtt.base.scratch_page = scratch_page; + dev_priv->ggtt.base.scratch_page = scratch_page; return 0; } @@ -3034,20 +3030,16 @@ static void chv_setup_private_ppat(struct drm_i915_private *dev_priv) I915_WRITE(GEN8_PRIVATE_PAT_HI, pat >> 32); } -static int gen8_gmch_probe(struct drm_device *dev, - u64 *gtt_total, - size_t *stolen, - phys_addr_t *mappable_base, - u64 *mappable_end) +static int gen8_gmch_probe(struct i915_ggtt *ggtt) { + struct drm_device *dev = ggtt->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; - u64 gtt_size; u16 snb_gmch_ctl; int ret; /* TODO: We're not aware of mappable constraints on gen8 yet */ - *mappable_base = pci_resource_start(dev->pdev, 2); - *mappable_end = pci_resource_len(dev->pdev, 2); + ggtt->mappable_base = pci_resource_start(dev->pdev, 2); + ggtt->mappable_end = pci_resource_len(dev->pdev, 2); if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(39))) pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(39)); @@ -3055,56 +3047,51 @@ static int gen8_gmch_probe(struct drm_device *dev, pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl); if (INTEL_INFO(dev)->gen >= 9) { - *stolen = gen9_get_stolen_size(snb_gmch_ctl); - gtt_size = gen8_get_total_gtt_size(snb_gmch_ctl); + ggtt->stolen_size = gen9_get_stolen_size(snb_gmch_ctl); + ggtt->size = gen8_get_total_gtt_size(snb_gmch_ctl); } else if (IS_CHERRYVIEW(dev)) { - *stolen = chv_get_stolen_size(snb_gmch_ctl); - gtt_size = chv_get_total_gtt_size(snb_gmch_ctl); + ggtt->stolen_size = chv_get_stolen_size(snb_gmch_ctl); + ggtt->size = chv_get_total_gtt_size(snb_gmch_ctl); } else { - *stolen = gen8_get_stolen_size(snb_gmch_ctl); - gtt_size = gen8_get_total_gtt_size(snb_gmch_ctl); + ggtt->stolen_size = gen8_get_stolen_size(snb_gmch_ctl); + ggtt->size = gen8_get_total_gtt_size(snb_gmch_ctl); } - *gtt_total = (gtt_size / sizeof(gen8_pte_t)) << PAGE_SHIFT; + ggtt->base.total = (ggtt->size / sizeof(gen8_pte_t)) << PAGE_SHIFT; if (IS_CHERRYVIEW(dev) || IS_BROXTON(dev)) chv_setup_private_ppat(dev_priv); else bdw_setup_private_ppat(dev_priv); - ret = ggtt_probe_common(dev, gtt_size); - - dev_priv->gtt.base.clear_range = gen8_ggtt_clear_range; - dev_priv->gtt.base.insert_entries = gen8_ggtt_insert_entries; - dev_priv->gtt.base.bind_vma = ggtt_bind_vma; - dev_priv->gtt.base.unbind_vma = ggtt_unbind_vma; + ret = ggtt_probe_common(dev, ggtt->size); + ggtt->base.clear_range = gen8_ggtt_clear_range; if (IS_CHERRYVIEW(dev_priv)) - dev_priv->gtt.base.insert_entries = gen8_ggtt_insert_entries__BKL; + ggtt->base.insert_entries = gen8_ggtt_insert_entries__BKL; + else + ggtt->base.insert_entries = gen8_ggtt_insert_entries; + ggtt->base.bind_vma = ggtt_bind_vma; + ggtt->base.unbind_vma = ggtt_unbind_vma; + return ret; } -static int gen6_gmch_probe(struct drm_device *dev, - u64 *gtt_total, - size_t *stolen, - phys_addr_t *mappable_base, - u64 *mappable_end) +static int gen6_gmch_probe(struct i915_ggtt *ggtt) { - struct drm_i915_private *dev_priv = dev->dev_private; - unsigned int gtt_size; + struct drm_device *dev = ggtt->base.dev; u16 snb_gmch_ctl; int ret; - *mappable_base = pci_resource_start(dev->pdev, 2); - *mappable_end = pci_resource_len(dev->pdev, 2); + ggtt->mappable_base = pci_resource_start(dev->pdev, 2); + ggtt->mappable_end = pci_resource_len(dev->pdev, 2); /* 64/512MB is the current min/max we actually know of, but this is just * a coarse sanity check. */ - if ((*mappable_end < (64<<20) || (*mappable_end > (512<<20)))) { - DRM_ERROR("Unknown GMADR size (%llx)\n", - dev_priv->gtt.mappable_end); + if ((ggtt->mappable_end < (64<<20) || (ggtt->mappable_end > (512<<20)))) { + DRM_ERROR("Unknown GMADR size (%llx)\n", ggtt->mappable_end); return -ENXIO; } @@ -3112,36 +3099,31 @@ static int gen6_gmch_probe(struct drm_device *dev, pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(40)); pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl); - *stolen = gen6_get_stolen_size(snb_gmch_ctl); - - gtt_size = gen6_get_total_gtt_size(snb_gmch_ctl); - *gtt_total = (gtt_size / sizeof(gen6_pte_t)) << PAGE_SHIFT; + ggtt->stolen_size = gen6_get_stolen_size(snb_gmch_ctl); + ggtt->size = gen6_get_total_gtt_size(snb_gmch_ctl); + ggtt->base.total = (ggtt->size / sizeof(gen6_pte_t)) << PAGE_SHIFT; - ret = ggtt_probe_common(dev, gtt_size); + ret = ggtt_probe_common(dev, ggtt->size); - dev_priv->gtt.base.clear_range = gen6_ggtt_clear_range; - dev_priv->gtt.base.insert_entries = gen6_ggtt_insert_entries; - dev_priv->gtt.base.bind_vma = ggtt_bind_vma; - dev_priv->gtt.base.unbind_vma = ggtt_unbind_vma; + ggtt->base.clear_range = gen6_ggtt_clear_range; + ggtt->base.insert_entries = gen6_ggtt_insert_entries; + ggtt->base.bind_vma = ggtt_bind_vma; + ggtt->base.unbind_vma = ggtt_unbind_vma; return ret; } static void gen6_gmch_remove(struct i915_address_space *vm) { + struct i915_ggtt *ggtt = container_of(vm, struct i915_ggtt, base); - struct i915_gtt *gtt = container_of(vm, struct i915_gtt, base); - - iounmap(gtt->gsm); + iounmap(ggtt->gsm); free_scratch_page(vm->dev, vm->scratch_page); } -static int i915_gmch_probe(struct drm_device *dev, - u64 *gtt_total, - size_t *stolen, - phys_addr_t *mappable_base, - u64 *mappable_end) +static int i915_gmch_probe(struct i915_ggtt *ggtt) { + struct drm_device *dev = ggtt->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; int ret; @@ -3151,15 +3133,16 @@ static int i915_gmch_probe(struct drm_device *dev, return -EIO; } - intel_gtt_get(gtt_total, stolen, mappable_base, mappable_end); + intel_gtt_get(&ggtt->base.total, &ggtt->stolen_size, + &ggtt->mappable_base, &ggtt->mappable_end); - dev_priv->gtt.do_idle_maps = needs_idle_maps(dev_priv->dev); - dev_priv->gtt.base.insert_entries = i915_ggtt_insert_entries; - dev_priv->gtt.base.clear_range = i915_ggtt_clear_range; - dev_priv->gtt.base.bind_vma = ggtt_bind_vma; - dev_priv->gtt.base.unbind_vma = ggtt_unbind_vma; + ggtt->do_idle_maps = needs_idle_maps(dev_priv->dev); + ggtt->base.insert_entries = i915_ggtt_insert_entries; + ggtt->base.clear_range = i915_ggtt_clear_range; + ggtt->base.bind_vma = ggtt_bind_vma; + ggtt->base.unbind_vma = ggtt_unbind_vma; - if (unlikely(dev_priv->gtt.do_idle_maps)) + if (unlikely(ggtt->do_idle_maps)) DRM_INFO("applying Ironlake quirks for intel_iommu\n"); return 0; @@ -3173,38 +3156,45 @@ static void i915_gmch_remove(struct i915_address_space *vm) int i915_gem_gtt_init(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - struct i915_gtt *gtt = &dev_priv->gtt; + struct i915_ggtt *ggtt = &dev_priv->ggtt; int ret; if (INTEL_INFO(dev)->gen <= 5) { - gtt->gtt_probe = i915_gmch_probe; - gtt->base.cleanup = i915_gmch_remove; + ggtt->probe = i915_gmch_probe; + ggtt->base.cleanup = i915_gmch_remove; } else if (INTEL_INFO(dev)->gen < 8) { - gtt->gtt_probe = gen6_gmch_probe; - gtt->base.cleanup = gen6_gmch_remove; + ggtt->probe = gen6_gmch_probe; + ggtt->base.cleanup = gen6_gmch_remove; if (IS_HASWELL(dev) && dev_priv->ellc_size) - gtt->base.pte_encode = iris_pte_encode; + ggtt->base.pte_encode = iris_pte_encode; else if (IS_HASWELL(dev)) - gtt->base.pte_encode = hsw_pte_encode; + ggtt->base.pte_encode = hsw_pte_encode; else if (IS_VALLEYVIEW(dev)) - gtt->base.pte_encode = byt_pte_encode; + ggtt->base.pte_encode = byt_pte_encode; else if (INTEL_INFO(dev)->gen >= 7) - gtt->base.pte_encode = ivb_pte_encode; + ggtt->base.pte_encode = ivb_pte_encode; else - gtt->base.pte_encode = snb_pte_encode; + ggtt->base.pte_encode = snb_pte_encode; } else { - dev_priv->gtt.gtt_probe = gen8_gmch_probe; - dev_priv->gtt.base.cleanup = gen6_gmch_remove; + ggtt->probe = gen8_gmch_probe; + ggtt->base.cleanup = gen6_gmch_remove; } - gtt->base.dev = dev; - gtt->base.is_ggtt = true; + ggtt->base.dev = dev; + ggtt->base.is_ggtt = true; - ret = gtt->gtt_probe(dev, >t->base.total, >t->stolen_size, - >t->mappable_base, >t->mappable_end); + ret = ggtt->probe(ggtt); if (ret) return ret; + if ((ggtt->base.total - 1) >> 32) { + DRM_ERROR("We never expected a Global GTT with more than 32bits" + "of address space! Found %lldM!\n", + ggtt->base.total >> 20); + ggtt->base.total = 1ULL << 32; + ggtt->mappable_end = min(ggtt->mappable_end, ggtt->base.total); + } + /* * Initialise stolen early so that we may reserve preallocated * objects for the BIOS to KMS transition. @@ -3215,9 +3205,9 @@ int i915_gem_gtt_init(struct drm_device *dev) /* GMADR is the PCI mmio aperture into the global GTT. */ DRM_INFO("Memory usable by graphics device = %lluM\n", - gtt->base.total >> 20); - DRM_DEBUG_DRIVER("GMADR size = %lldM\n", gtt->mappable_end >> 20); - DRM_DEBUG_DRIVER("GTT stolen size = %zdM\n", gtt->stolen_size >> 20); + ggtt->base.total >> 20); + DRM_DEBUG_DRIVER("GMADR size = %lldM\n", ggtt->mappable_end >> 20); + DRM_DEBUG_DRIVER("GTT stolen size = %zdM\n", ggtt->stolen_size >> 20); #ifdef CONFIG_INTEL_IOMMU if (intel_iommu_gfx_mapped) DRM_INFO("VT-d active for gfx access\n"); @@ -3234,7 +3224,7 @@ int i915_gem_gtt_init(struct drm_device *dev) return 0; out_gtt_cleanup: - gtt->base.cleanup(&dev_priv->gtt.base); + ggtt->base.cleanup(&dev_priv->ggtt.base); return ret; } @@ -3250,13 +3240,13 @@ void i915_gem_restore_gtt_mappings(struct drm_device *dev) i915_check_and_clear_faults(dev); /* First fill our portion of the GTT with scratch pages */ - dev_priv->gtt.base.clear_range(&dev_priv->gtt.base, - dev_priv->gtt.base.start, - dev_priv->gtt.base.total, + dev_priv->ggtt.base.clear_range(&dev_priv->ggtt.base, + dev_priv->ggtt.base.start, + dev_priv->ggtt.base.total, true); /* Cache flush objects bound into GGTT and rebind them. */ - vm = &dev_priv->gtt.base; + vm = &dev_priv->ggtt.base; list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { flush = false; list_for_each_entry(vma, &obj->vma_list, obj_link) { @@ -3377,11 +3367,6 @@ rotate_pages(const dma_addr_t *in, unsigned int offset, unsigned int column, row; unsigned int src_idx; - if (!sg) { - st->nents = 0; - sg = st->sgl; - } - for (column = 0; column < width; column++) { src_idx = stride * (height - 1) + column; for (row = 0; row < height; row++) { @@ -3405,7 +3390,7 @@ static struct sg_table * intel_rotate_fb_obj_pages(struct intel_rotation_info *rot_info, struct drm_i915_gem_object *obj) { - unsigned int size_pages = rot_info->size >> PAGE_SHIFT; + unsigned int size_pages = rot_info->plane[0].width * rot_info->plane[0].height; unsigned int size_pages_uv; struct sg_page_iter sg_iter; unsigned long i; @@ -3423,7 +3408,7 @@ intel_rotate_fb_obj_pages(struct intel_rotation_info *rot_info, /* Account for UV plane with NV12. */ if (rot_info->pixel_format == DRM_FORMAT_NV12) - size_pages_uv = rot_info->size_uv >> PAGE_SHIFT; + size_pages_uv = rot_info->plane[1].width * rot_info->plane[1].height; else size_pages_uv = 0; @@ -3443,11 +3428,14 @@ intel_rotate_fb_obj_pages(struct intel_rotation_info *rot_info, i++; } + st->nents = 0; + sg = st->sgl; + /* Rotate the pages. */ sg = rotate_pages(page_addr_list, 0, - rot_info->width_pages, rot_info->height_pages, - rot_info->width_pages, - st, NULL); + rot_info->plane[0].width, rot_info->plane[0].height, + rot_info->plane[0].width, + st, sg); /* Append the UV plane if NV12. */ if (rot_info->pixel_format == DRM_FORMAT_NV12) { @@ -3459,18 +3447,15 @@ intel_rotate_fb_obj_pages(struct intel_rotation_info *rot_info, rot_info->uv_start_page = uv_start_page; - rotate_pages(page_addr_list, uv_start_page, - rot_info->width_pages_uv, - rot_info->height_pages_uv, - rot_info->width_pages_uv, - st, sg); + sg = rotate_pages(page_addr_list, rot_info->uv_start_page, + rot_info->plane[1].width, rot_info->plane[1].height, + rot_info->plane[1].width, + st, sg); } - DRM_DEBUG_KMS( - "Created rotated page mapping for object size %zu (pitch=%u, height=%u, pixel_format=0x%x, %ux%u tiles, %u pages (%u plane 0)).\n", - obj->base.size, rot_info->pitch, rot_info->height, - rot_info->pixel_format, rot_info->width_pages, - rot_info->height_pages, size_pages + size_pages_uv, + DRM_DEBUG_KMS("Created rotated page mapping for object size %zu (%ux%u tiles, %u pages (%u plane 0)).\n", + obj->base.size, rot_info->plane[0].width, + rot_info->plane[0].height, size_pages + size_pages_uv, size_pages); drm_free_large(page_addr_list); @@ -3482,11 +3467,9 @@ err_sg_alloc: err_st_alloc: drm_free_large(page_addr_list); - DRM_DEBUG_KMS( - "Failed to create rotated mapping for object size %zu! (%d) (pitch=%u, height=%u, pixel_format=0x%x, %ux%u tiles, %u pages (%u plane 0))\n", - obj->base.size, ret, rot_info->pitch, rot_info->height, - rot_info->pixel_format, rot_info->width_pages, - rot_info->height_pages, size_pages + size_pages_uv, + DRM_DEBUG_KMS("Failed to create rotated mapping for object size %zu! (%d) (%ux%u tiles, %u pages (%u plane 0))\n", + obj->base.size, ret, rot_info->plane[0].width, + rot_info->plane[0].height, size_pages + size_pages_uv, size_pages); return ERR_PTR(ret); } @@ -3634,7 +3617,7 @@ i915_ggtt_view_size(struct drm_i915_gem_object *obj, if (view->type == I915_GGTT_VIEW_NORMAL) { return obj->base.size; } else if (view->type == I915_GGTT_VIEW_ROTATED) { - return view->params.rotated.size; + return intel_rotation_info_size(&view->params.rotated) << PAGE_SHIFT; } else if (view->type == I915_GGTT_VIEW_PARTIAL) { return view->params.partial.size << PAGE_SHIFT; } else { |