diff options
Diffstat (limited to 'drivers/staging/psb/ttm/ttm_bo_vm.c')
-rw-r--r-- | drivers/staging/psb/ttm/ttm_bo_vm.c | 597 |
1 files changed, 597 insertions, 0 deletions
diff --git a/drivers/staging/psb/ttm/ttm_bo_vm.c b/drivers/staging/psb/ttm/ttm_bo_vm.c new file mode 100644 index 000000000000..ee1f9c6c3550 --- /dev/null +++ b/drivers/staging/psb/ttm/ttm_bo_vm.c @@ -0,0 +1,597 @@ +/************************************************************************** + * + * Copyright (c) 2006-2008 Tungsten Graphics, Inc., Cedar Park, TX., USA + * All Rights Reserved. + * Copyright (c) 2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> + */ + + +#include <linux/mm.h> +#include <linux/version.h> +#include <linux/rbtree.h> +#include <linux/list.h> +#include <asm/uaccess.h> +#include "ttm_bo_driver.h" +#include "ttm_placement_common.h" + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,25)) +#error "TTM doesn't build on kernel versions below 2.6.25." +#endif + +#define TTM_BO_VM_NUM_PREFAULT 16 + +static struct ttm_buffer_object *ttm_bo_vm_lookup_rb(struct ttm_bo_device *bdev, + unsigned long page_start, + unsigned long num_pages) +{ + struct rb_node *cur = bdev->addr_space_rb.rb_node; + unsigned long cur_offset; + struct ttm_buffer_object *bo; + struct ttm_buffer_object *best_bo = NULL; + + while (likely(cur != NULL)) { + bo = rb_entry(cur, struct ttm_buffer_object, vm_rb); + cur_offset = bo->vm_node->start; + if (page_start >= cur_offset) { + cur = cur->rb_right; + best_bo = bo; + if (page_start == cur_offset) + break; + } else + cur = cur->rb_left; + } + + if (unlikely(best_bo == NULL)) + return NULL; + + if (unlikely((best_bo->vm_node->start + best_bo->num_pages) < + (page_start + num_pages))) + return NULL; + + return best_bo; +} + +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,26)) +static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) +{ + struct ttm_buffer_object *bo = (struct ttm_buffer_object *) + vma->vm_private_data; + struct ttm_bo_device *bdev = bo->bdev; + unsigned long bus_base; + unsigned long bus_offset; + unsigned long bus_size; + unsigned long page_offset; + unsigned long page_last; + unsigned long pfn; + struct ttm_tt *ttm = NULL; + struct page *page; + int ret; + int i; + bool is_iomem; + unsigned long address = (unsigned long)vmf->virtual_address; + int retval = VM_FAULT_NOPAGE; + + ret = ttm_bo_reserve(bo, true, false, false, 0); + if (unlikely(ret != 0)) + return VM_FAULT_NOPAGE; + + mutex_lock(&bo->mutex); + + /* + * Wait for buffer data in transit, due to a pipelined + * move. + */ + + if (bo->priv_flags & TTM_BO_PRIV_FLAG_MOVING) { + ret = ttm_bo_wait(bo, false, true, false); + if (unlikely(ret != 0)) { + retval = (ret != -ERESTART) ? + VM_FAULT_SIGBUS : VM_FAULT_NOPAGE; + goto out_unlock; + } + } + + ret = ttm_bo_pci_offset(bdev, &bo->mem, &bus_base, &bus_offset, + &bus_size); + if (unlikely(ret != 0)) { + retval = VM_FAULT_SIGBUS; + goto out_unlock; + } + + is_iomem = (bus_size != 0); + + page_offset = ((address - vma->vm_start) >> PAGE_SHIFT) + + bo->vm_node->start - vma->vm_pgoff; + page_last = ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT) + + bo->vm_node->start - vma->vm_pgoff; + + if (unlikely(page_offset >= bo->num_pages)) { + retval = VM_FAULT_SIGBUS; + goto out_unlock; + } + + /* + * Strictly, we're not allowed to modify vma->vm_page_prot here, + * since the mmap_sem is only held in read mode. However, we + * modify only the caching bits of vma->vm_page_prot and + * consider those bits protected by + * the bo->mutex, as we should be the only writers. + * There shouldn't really be any readers of these bits except + * within vm_insert_mixed()? fork? + * + * TODO: Add a list of vmas to the bo, and change the + * vma->vm_page_prot when the object changes caching policy, with + * the correct locks held. + */ + + if (is_iomem) { + vma->vm_page_prot = ttm_io_prot(bo->mem.flags, + vma->vm_page_prot); + } else { + ttm = bo->ttm; + vma->vm_page_prot = (bo->mem.flags & TTM_PL_FLAG_CACHED) ? + vm_get_page_prot(vma->vm_flags) : + ttm_io_prot(bo->mem.flags, vma->vm_page_prot); + } + + /* + * Speculatively prefault a number of pages. Only error on + * first page. + */ + + for (i = 0; i < TTM_BO_VM_NUM_PREFAULT; ++i) { + + if (is_iomem) + pfn = ((bus_base + bus_offset) >> PAGE_SHIFT) + + page_offset; + else { + page = ttm_tt_get_page(ttm, page_offset); + if (unlikely(!page && i == 0)) { + retval = VM_FAULT_OOM; + goto out_unlock; + } else if (unlikely(!page)) { + break; + } + pfn = page_to_pfn(page); + } + +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,29)) + ret = vm_insert_mixed(vma, address, pfn); +#else + ret = vm_insert_pfn(vma, address, pfn); +#endif + /* + * Somebody beat us to this PTE or prefaulting to + * an already populated PTE, or prefaulting error. + */ + + if (unlikely((ret == -EBUSY) || (ret != 0 && i > 0))) + break; + else if (unlikely(ret != 0)) { + retval = + (ret == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS; + goto out_unlock; + + } + + address += PAGE_SIZE; + if (unlikely(++page_offset >= page_last)) + break; + } + + out_unlock: + mutex_unlock(&bo->mutex); + ttm_bo_unreserve(bo); + return retval; +} + +#else + +static unsigned long ttm_bo_vm_nopfn(struct vm_area_struct *vma, + unsigned long address) +{ + struct ttm_buffer_object *bo = (struct ttm_buffer_object *) + vma->vm_private_data; + struct ttm_bo_device *bdev = bo->bdev; + unsigned long bus_base; + unsigned long bus_offset; + unsigned long bus_size; + unsigned long page_offset; + unsigned long page_last; + unsigned long pfn; + struct ttm_tt *ttm = NULL; + struct page *page; + int ret; + int i; + bool is_iomem; + unsigned long retval = NOPFN_REFAULT; + + ret = ttm_bo_reserve(bo, true, false, false, 0); + if (unlikely(ret != 0)) + return NOPFN_REFAULT; + + mutex_lock(&bo->mutex); + + /* + * Wait for buffer data in transit, due to a pipelined + * move. + */ + + if (bo->priv_flags & TTM_BO_PRIV_FLAG_MOVING) { + ret = ttm_bo_wait(bo, false, true, false); + if (unlikely(ret != 0)) { + retval = (ret != -ERESTART) ? + NOPFN_SIGBUS : NOPFN_REFAULT; + goto out_unlock; + } + } + + ret = ttm_bo_pci_offset(bdev, &bo->mem, &bus_base, &bus_offset, + &bus_size); + if (unlikely(ret != 0)) { + printk(KERN_ERR "Attempted buffer object access " + "of unmappable object.\n"); + retval = NOPFN_SIGBUS; + goto out_unlock; + } + + is_iomem = (bus_size != 0); + + page_offset = ((address - vma->vm_start) >> PAGE_SHIFT) + + bo->vm_node->start - vma->vm_pgoff; + + page_last = ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT) + + bo->vm_node->start - vma->vm_pgoff; + + if (unlikely(page_offset >= bo->num_pages)) { + printk(KERN_ERR "Attempted buffer object access " + "outside object.\n"); + retval = NOPFN_SIGBUS; + goto out_unlock; + } + + /* + * Strictly, we're not allowed to modify vma->vm_page_prot here, + * since the mmap_sem is only held in read mode. However, we + * modify only the caching bits of vma->vm_page_prot and + * consider those bits protected by + * the bo->mutex, as we should be the only writers. + * There shouldn't really be any readers of these bits except + * within vm_insert_mixed()? fork? + * + * TODO: Add a list of vmas to the bo, and change the + * vma->vm_page_prot when the object changes caching policy, with + * the correct locks held. + */ + + if (is_iomem) { + vma->vm_page_prot = ttm_io_prot(bo->mem.flags, + vma->vm_page_prot); + } else { + ttm = bo->ttm; + vma->vm_page_prot = (bo->mem.flags & TTM_PL_FLAG_CACHED) ? + vm_get_page_prot(vma->vm_flags) : + ttm_io_prot(bo->mem.flags, vma->vm_page_prot); + } + + /* + * Speculatively prefault a number of pages. Only error on + * first page. + */ + + for (i = 0; i < TTM_BO_VM_NUM_PREFAULT; ++i) { + + if (is_iomem) + pfn = ((bus_base + bus_offset) >> PAGE_SHIFT) + + page_offset; + else { + page = ttm_tt_get_page(ttm, page_offset); + if (unlikely(!page && i == 0)) { + retval = NOPFN_OOM; + goto out_unlock; + } else if (unlikely(!page)) { + break; + } + pfn = page_to_pfn(page); + } + + ret = vm_insert_pfn(vma, address, pfn); + if (unlikely(ret == -EBUSY || (ret != 0 && i != 0))) + break; + + /* + * Somebody beat us to this PTE or prefaulting to + * an already populated PTE, or prefaulting error. + */ + + if (unlikely((ret == -EBUSY) || (ret != 0 && i > 0))) + break; + else if (unlikely(ret != 0)) { + retval = + (ret == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS; + goto out_unlock; + } + + address += PAGE_SIZE; + if (unlikely(++page_offset >= page_last)) + break; + } + + out_unlock: + mutex_unlock(&bo->mutex); + ttm_bo_unreserve(bo); + return retval; +} +#endif + +static void ttm_bo_vm_open(struct vm_area_struct *vma) +{ + struct ttm_buffer_object *bo = + (struct ttm_buffer_object *)vma->vm_private_data; + + (void)ttm_bo_reference(bo); +} + +static void ttm_bo_vm_close(struct vm_area_struct *vma) +{ + struct ttm_buffer_object *bo = + (struct ttm_buffer_object *)vma->vm_private_data; + + ttm_bo_unref(&bo); + vma->vm_private_data = NULL; +} + +static struct vm_operations_struct ttm_bo_vm_ops = { +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,26)) + .fault = ttm_bo_vm_fault, +#else + .nopfn = ttm_bo_vm_nopfn, +#endif + .open = ttm_bo_vm_open, + .close = ttm_bo_vm_close +}; + +int ttm_bo_mmap(struct file *filp, struct vm_area_struct *vma, + struct ttm_bo_device *bdev) +{ + struct ttm_bo_driver *driver; + struct ttm_buffer_object *bo; + int ret; + + read_lock(&bdev->vm_lock); + bo = ttm_bo_vm_lookup_rb(bdev, vma->vm_pgoff, + (vma->vm_end - vma->vm_start) >> PAGE_SHIFT); + if (likely(bo != NULL)) + ttm_bo_reference(bo); + read_unlock(&bdev->vm_lock); + + if (unlikely(bo == NULL)) { + printk(KERN_ERR "Could not find buffer object to map.\n"); + ret = -EINVAL; + goto out_unref; + } + + driver = bo->bdev->driver; + if (unlikely(!driver->verify_access)) { + ret = -EPERM; + goto out_unref; + } + ret = driver->verify_access(bo, filp); + if (unlikely(ret != 0)) + goto out_unref; + + vma->vm_ops = &ttm_bo_vm_ops; + + /* + * Note: We're transferring the bo reference to + * vma->vm_private_data here. + */ + + vma->vm_private_data = bo; +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,29)) + vma->vm_flags |= VM_RESERVED | VM_IO | VM_MIXEDMAP | VM_DONTEXPAND; +#else + vma->vm_flags |= VM_RESERVED | VM_IO | VM_PFNMAP | VM_DONTEXPAND; +#endif + return 0; + out_unref: + ttm_bo_unref(&bo); + return ret; +} + +int ttm_fbdev_mmap(struct vm_area_struct *vma, struct ttm_buffer_object *bo) +{ + if (vma->vm_pgoff != 0) + return -EACCES; + + vma->vm_ops = &ttm_bo_vm_ops; + vma->vm_private_data = ttm_bo_reference(bo); +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,29)) + vma->vm_flags |= VM_RESERVED | VM_IO | VM_MIXEDMAP | VM_DONTEXPAND; +#else + vma->vm_flags |= VM_RESERVED | VM_IO | VM_PFNMAP | VM_DONTEXPAND; +#endif + return 0; +} + +ssize_t ttm_bo_io(struct ttm_bo_device * bdev, struct file * filp, + const char __user * wbuf, char __user * rbuf, size_t count, + loff_t * f_pos, bool write) +{ + struct ttm_buffer_object *bo; + struct ttm_bo_driver *driver; + struct ttm_bo_kmap_obj map; + unsigned long dev_offset = (*f_pos >> PAGE_SHIFT); + unsigned long kmap_offset; + unsigned long kmap_end; + unsigned long kmap_num; + size_t io_size; + unsigned int page_offset; + char *virtual; + int ret; + bool no_wait = false; + bool dummy; + + driver = bo->bdev->driver; + read_lock(&bdev->vm_lock); + bo = ttm_bo_vm_lookup_rb(bdev, dev_offset, 1); + if (likely(bo != NULL)) + ttm_bo_reference(bo); + read_unlock(&bdev->vm_lock); + + if (unlikely(bo == NULL)) + return -EFAULT; + + if (unlikely(driver->verify_access)) + return -EPERM; + + ret = driver->verify_access(bo, filp); + if (unlikely(ret != 0)) + goto out_unref; + + kmap_offset = dev_offset - bo->vm_node->start; + if (unlikely(kmap_offset) >= bo->num_pages) { + ret = -EFBIG; + goto out_unref; + } + + page_offset = *f_pos & ~PAGE_MASK; + io_size = bo->num_pages - kmap_offset; + io_size = (io_size << PAGE_SHIFT) - page_offset; + if (count < io_size) + io_size = count; + + kmap_end = (*f_pos + count - 1) >> PAGE_SHIFT; + kmap_num = kmap_end - kmap_offset + 1; + + ret = ttm_bo_reserve(bo, true, no_wait, false, 0); + + switch (ret) { + case 0: + break; + case -ERESTART: + ret = -EINTR; + goto out_unref; + case -EBUSY: + ret = -EAGAIN; + goto out_unref; + default: + goto out_unref; + } + + ret = ttm_bo_kmap(bo, kmap_offset, kmap_num, &map); + if (unlikely(ret != 0)) + goto out_unref; + + virtual = ttm_kmap_obj_virtual(&map, &dummy); + virtual += page_offset; + + if (write) + ret = copy_from_user(virtual, wbuf, io_size); + else + ret = copy_to_user(rbuf, virtual, io_size); + + ttm_bo_kunmap(&map); + ttm_bo_unreserve(bo); + ttm_bo_unref(&bo); + + if (unlikely(ret != 0)) + return -EFBIG; + + *f_pos += io_size; + + return io_size; + out_unref: + ttm_bo_unref(&bo); + return ret; +} + +ssize_t ttm_bo_fbdev_io(struct ttm_buffer_object * bo, const char __user * wbuf, + char __user * rbuf, size_t count, loff_t * f_pos, + bool write) +{ + struct ttm_bo_kmap_obj map; + unsigned long kmap_offset; + unsigned long kmap_end; + unsigned long kmap_num; + size_t io_size; + unsigned int page_offset; + char *virtual; + int ret; + bool no_wait = false; + bool dummy; + + kmap_offset = (*f_pos >> PAGE_SHIFT); + if (unlikely(kmap_offset) >= bo->num_pages) + return -EFBIG; + + page_offset = *f_pos & ~PAGE_MASK; + io_size = bo->num_pages - kmap_offset; + io_size = (io_size << PAGE_SHIFT) - page_offset; + if (count < io_size) + io_size = count; + + kmap_end = (*f_pos + count - 1) >> PAGE_SHIFT; + kmap_num = kmap_end - kmap_offset + 1; + + ret = ttm_bo_reserve(bo, true, no_wait, false, 0); + + switch (ret) { + case 0: + break; + case -ERESTART: + return -EINTR; + case -EBUSY: + return -EAGAIN; + default: + return ret; + } + + ret = ttm_bo_kmap(bo, kmap_offset, kmap_num, &map); + if (unlikely(ret != 0)) + return ret; + + virtual = ttm_kmap_obj_virtual(&map, &dummy); + virtual += page_offset; + + if (write) + ret = copy_from_user(virtual, wbuf, io_size); + else + ret = copy_to_user(rbuf, virtual, io_size); + + ttm_bo_kunmap(&map); + ttm_bo_unreserve(bo); + ttm_bo_unref(&bo); + + if (unlikely(ret != 0)) + return ret; + + *f_pos += io_size; + + return io_size; +} |