svn commit: r247835 - in head/sys: dev/drm2 dev/drm2/ttm modules/drm2/drm2
Konstantin Belousov
kib at FreeBSD.org
Tue Mar 5 09:49:35 UTC 2013
Author: kib
Date: Tue Mar 5 09:49:34 2013
New Revision: 247835
URL: http://svnweb.freebsd.org/changeset/base/247835
Log:
Import the preliminary port of the TTM.
The early commit is done to facilitate the off-tree work on the
porting of the Radeon driver.
Sponsored by: The FreeBSD Foundation
Debugged and tested by: dumbbell
MFC after: 1 month
Added:
head/sys/dev/drm2/ttm/
head/sys/dev/drm2/ttm/ttm_agp_backend.c (contents, props changed)
head/sys/dev/drm2/ttm/ttm_bo.c (contents, props changed)
head/sys/dev/drm2/ttm/ttm_bo_api.h (contents, props changed)
head/sys/dev/drm2/ttm/ttm_bo_driver.h (contents, props changed)
head/sys/dev/drm2/ttm/ttm_bo_manager.c (contents, props changed)
head/sys/dev/drm2/ttm/ttm_bo_util.c (contents, props changed)
head/sys/dev/drm2/ttm/ttm_bo_vm.c (contents, props changed)
head/sys/dev/drm2/ttm/ttm_execbuf_util.c (contents, props changed)
head/sys/dev/drm2/ttm/ttm_execbuf_util.h (contents, props changed)
head/sys/dev/drm2/ttm/ttm_lock.c (contents, props changed)
head/sys/dev/drm2/ttm/ttm_lock.h (contents, props changed)
head/sys/dev/drm2/ttm/ttm_memory.c (contents, props changed)
head/sys/dev/drm2/ttm/ttm_memory.h (contents, props changed)
head/sys/dev/drm2/ttm/ttm_module.h (contents, props changed)
head/sys/dev/drm2/ttm/ttm_object.c (contents, props changed)
head/sys/dev/drm2/ttm/ttm_object.h (contents, props changed)
head/sys/dev/drm2/ttm/ttm_page_alloc.c (contents, props changed)
head/sys/dev/drm2/ttm/ttm_page_alloc.h (contents, props changed)
head/sys/dev/drm2/ttm/ttm_page_alloc_dma.c (contents, props changed)
head/sys/dev/drm2/ttm/ttm_placement.h (contents, props changed)
head/sys/dev/drm2/ttm/ttm_tt.c (contents, props changed)
Modified:
head/sys/dev/drm2/drmP.h
head/sys/dev/drm2/drm_drv.c
head/sys/dev/drm2/drm_gem.c
head/sys/modules/drm2/drm2/Makefile
Modified: head/sys/dev/drm2/drmP.h
==============================================================================
--- head/sys/dev/drm2/drmP.h Tue Mar 5 09:27:21 2013 (r247834)
+++ head/sys/dev/drm2/drmP.h Tue Mar 5 09:49:34 2013 (r247835)
@@ -906,6 +906,7 @@ struct drm_device {
struct drm_minor *control; /**< Control node for card */
struct drm_minor *primary; /**< render type primary screen head */
+ void *drm_ttm_bo;
struct unrhdr *drw_unrhdr;
/* RB tree of drawable infos */
RB_HEAD(drawable_tree, bsd_drm_drawable_info) drw_head;
@@ -1302,10 +1303,14 @@ void drm_gem_release(struct drm_device *
int drm_gem_create_mmap_offset(struct drm_gem_object *obj);
void drm_gem_free_mmap_offset(struct drm_gem_object *obj);
-int drm_gem_mmap_single(struct cdev *kdev, vm_ooffset_t *offset, vm_size_t size,
- struct vm_object **obj_res, int nprot);
+int drm_gem_mmap_single(struct drm_device *dev, vm_ooffset_t *offset,
+ vm_size_t size, struct vm_object **obj_res, int nprot);
void drm_gem_pager_dtr(void *obj);
+struct ttm_bo_device;
+int ttm_bo_mmap_single(struct ttm_bo_device *bdev, vm_ooffset_t *offset,
+ vm_size_t size, struct vm_object **obj_res, int nprot);
+
void drm_device_lock_mtx(struct drm_device *dev);
void drm_device_unlock_mtx(struct drm_device *dev);
int drm_device_sleep_mtx(struct drm_device *dev, void *chan, int flags,
Modified: head/sys/dev/drm2/drm_drv.c
==============================================================================
--- head/sys/dev/drm2/drm_drv.c Tue Mar 5 09:27:21 2013 (r247834)
+++ head/sys/dev/drm2/drm_drv.c Tue Mar 5 09:49:34 2013 (r247835)
@@ -58,6 +58,8 @@ static int drm_load(struct drm_device *d
static void drm_unload(struct drm_device *dev);
static drm_pci_id_list_t *drm_find_description(int vendor, int device,
drm_pci_id_list_t *idlist);
+static int drm_mmap_single(struct cdev *kdev, vm_ooffset_t *offset,
+ vm_size_t size, struct vm_object **obj_res, int nprot);
static int
drm_modevent(module_t mod, int type, void *data)
@@ -187,7 +189,7 @@ static struct cdevsw drm_cdevsw = {
.d_ioctl = drm_ioctl,
.d_poll = drm_poll,
.d_mmap = drm_mmap,
- .d_mmap_single = drm_gem_mmap_single,
+ .d_mmap_single = drm_mmap_single,
.d_name = "drm",
.d_flags = D_TRACKCLOSE
};
@@ -955,6 +957,23 @@ drm_add_busid_modesetting(struct drm_dev
return (0);
}
+static int
+drm_mmap_single(struct cdev *kdev, vm_ooffset_t *offset, vm_size_t size,
+ struct vm_object **obj_res, int nprot)
+{
+ struct drm_device *dev;
+
+ dev = drm_get_device_from_kdev(kdev);
+ if ((dev->driver->driver_features & DRIVER_GEM) != 0) {
+ return (drm_gem_mmap_single(dev, offset, size, obj_res, nprot));
+ } else if (dev->drm_ttm_bo != NULL) {
+ return (ttm_bo_mmap_single(dev->drm_ttm_bo, offset, size,
+ obj_res, nprot));
+ } else {
+ return (ENODEV);
+ }
+}
+
#if DRM_LINUX
#include <sys/sysproto.h>
Modified: head/sys/dev/drm2/drm_gem.c
==============================================================================
--- head/sys/dev/drm2/drm_gem.c Tue Mar 5 09:27:21 2013 (r247834)
+++ head/sys/dev/drm2/drm_gem.c Tue Mar 5 09:49:34 2013 (r247835)
@@ -441,16 +441,12 @@ drm_gem_free_mmap_offset(struct drm_gem_
}
int
-drm_gem_mmap_single(struct cdev *kdev, vm_ooffset_t *offset, vm_size_t size,
+drm_gem_mmap_single(struct drm_device *dev, vm_ooffset_t *offset, vm_size_t size,
struct vm_object **obj_res, int nprot)
{
- struct drm_device *dev;
struct drm_gem_object *gem_obj;
struct vm_object *vm_obj;
- dev = drm_get_device_from_kdev(kdev);
- if ((dev->driver->driver_features & DRIVER_GEM) == 0)
- return (ENODEV);
DRM_LOCK(dev);
gem_obj = drm_gem_object_from_offset(dev, *offset);
if (gem_obj == NULL) {
Added: head/sys/dev/drm2/ttm/ttm_agp_backend.c
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ head/sys/dev/drm2/ttm/ttm_agp_backend.c Tue Mar 5 09:49:34 2013 (r247835)
@@ -0,0 +1,145 @@
+/**************************************************************************
+ *
+ * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sub license, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
+ * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ * USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ **************************************************************************/
+/*
+ * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
+ * Keith Packard.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <dev/drm2/drmP.h>
+#include <dev/drm2/ttm/ttm_module.h>
+#include <dev/drm2/ttm/ttm_bo_driver.h>
+#include <dev/drm2/ttm/ttm_page_alloc.h>
+#ifdef TTM_HAS_AGP
+#include <dev/drm2/ttm/ttm_placement.h>
+
+struct ttm_agp_backend {
+ struct ttm_tt ttm;
+ struct agp_memory *mem;
+ device_t bridge;
+};
+
+MALLOC_DEFINE(M_TTM_AGP, "ttm_agp", "TTM AGP Backend");
+
+static int ttm_agp_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem)
+{
+ struct ttm_agp_backend *agp_be = container_of(ttm, struct ttm_agp_backend, ttm);
+ struct drm_mm_node *node = bo_mem->mm_node;
+ struct agp_memory *mem;
+ int ret, cached = (bo_mem->placement & TTM_PL_FLAG_CACHED);
+ unsigned i;
+
+ mem = agp_alloc_memory(agp_be->bridge, AGP_USER_MEMORY, ttm->num_pages);
+ if (unlikely(mem == NULL))
+ return -ENOMEM;
+
+ mem->page_count = 0;
+ for (i = 0; i < ttm->num_pages; i++) {
+ vm_page_t page = ttm->pages[i];
+
+ if (!page)
+ page = ttm->dummy_read_page;
+
+ mem->pages[mem->page_count++] = page;
+ }
+ agp_be->mem = mem;
+
+ mem->is_flushed = 1;
+ mem->type = (cached) ? AGP_USER_CACHED_MEMORY : AGP_USER_MEMORY;
+
+ ret = agp_bind_memory(mem, node->start);
+ if (ret)
+ pr_err("AGP Bind memory failed\n");
+
+ return ret;
+}
+
+static int ttm_agp_unbind(struct ttm_tt *ttm)
+{
+ struct ttm_agp_backend *agp_be = container_of(ttm, struct ttm_agp_backend, ttm);
+
+ if (agp_be->mem) {
+ if (agp_be->mem->is_bound)
+ return agp_unbind_memory(agp_be->mem);
+ agp_free_memory(agp_be->mem);
+ agp_be->mem = NULL;
+ }
+ return 0;
+}
+
+static void ttm_agp_destroy(struct ttm_tt *ttm)
+{
+ struct ttm_agp_backend *agp_be = container_of(ttm, struct ttm_agp_backend, ttm);
+
+ if (agp_be->mem)
+ ttm_agp_unbind(ttm);
+ ttm_tt_fini(ttm);
+ free(agp_be, M_TTM_AGP);
+}
+
+static struct ttm_backend_func ttm_agp_func = {
+ .bind = ttm_agp_bind,
+ .unbind = ttm_agp_unbind,
+ .destroy = ttm_agp_destroy,
+};
+
+struct ttm_tt *ttm_agp_tt_create(struct ttm_bo_device *bdev,
+ device_t bridge,
+ unsigned long size, uint32_t page_flags,
+ vm_page_t dummy_read_page)
+{
+ struct ttm_agp_backend *agp_be;
+
+ agp_be = malloc(sizeof(*agp_be), M_TTM_AGP, M_WAITOK | M_ZERO);
+
+ agp_be->mem = NULL;
+ agp_be->bridge = bridge;
+ agp_be->ttm.func = &ttm_agp_func;
+
+ if (ttm_tt_init(&agp_be->ttm, bdev, size, page_flags, dummy_read_page)) {
+ return NULL;
+ }
+
+ return &agp_be->ttm;
+}
+
+int ttm_agp_tt_populate(struct ttm_tt *ttm)
+{
+ if (ttm->state != tt_unpopulated)
+ return 0;
+
+ return ttm_pool_populate(ttm);
+}
+
+void ttm_agp_tt_unpopulate(struct ttm_tt *ttm)
+{
+ ttm_pool_unpopulate(ttm);
+}
+
+#endif
Added: head/sys/dev/drm2/ttm/ttm_bo.c
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ head/sys/dev/drm2/ttm/ttm_bo.c Tue Mar 5 09:49:34 2013 (r247835)
@@ -0,0 +1,1820 @@
+/**************************************************************************
+ *
+ * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sub license, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
+ * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ * USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ **************************************************************************/
+/*
+ * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <dev/drm2/drmP.h>
+#include <dev/drm2/ttm/ttm_module.h>
+#include <dev/drm2/ttm/ttm_bo_driver.h>
+#include <dev/drm2/ttm/ttm_placement.h>
+
+#define TTM_ASSERT_LOCKED(param)
+#define TTM_DEBUG(fmt, arg...)
+#define TTM_BO_HASH_ORDER 13
+
+static int ttm_bo_setup_vm(struct ttm_buffer_object *bo);
+static int ttm_bo_swapout(struct ttm_mem_shrink *shrink);
+static void ttm_bo_global_kobj_release(struct ttm_bo_global *glob);
+
+MALLOC_DEFINE(M_TTM_BO, "ttm_bo", "TTM Buffer Objects");
+
+static inline int ttm_mem_type_from_flags(uint32_t flags, uint32_t *mem_type)
+{
+ int i;
+
+ for (i = 0; i <= TTM_PL_PRIV5; i++)
+ if (flags & (1 << i)) {
+ *mem_type = i;
+ return 0;
+ }
+ return -EINVAL;
+}
+
+static void ttm_mem_type_debug(struct ttm_bo_device *bdev, int mem_type)
+{
+ struct ttm_mem_type_manager *man = &bdev->man[mem_type];
+
+ printf(" has_type: %d\n", man->has_type);
+ printf(" use_type: %d\n", man->use_type);
+ printf(" flags: 0x%08X\n", man->flags);
+ printf(" gpu_offset: 0x%08lX\n", man->gpu_offset);
+ printf(" size: %ju\n", (uintmax_t)man->size);
+ printf(" available_caching: 0x%08X\n", man->available_caching);
+ printf(" default_caching: 0x%08X\n", man->default_caching);
+ if (mem_type != TTM_PL_SYSTEM)
+ (*man->func->debug)(man, TTM_PFX);
+}
+
+static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo,
+ struct ttm_placement *placement)
+{
+ int i, ret, mem_type;
+
+ printf("No space for %p (%lu pages, %luK, %luM)\n",
+ bo, bo->mem.num_pages, bo->mem.size >> 10,
+ bo->mem.size >> 20);
+ for (i = 0; i < placement->num_placement; i++) {
+ ret = ttm_mem_type_from_flags(placement->placement[i],
+ &mem_type);
+ if (ret)
+ return;
+ printf(" placement[%d]=0x%08X (%d)\n",
+ i, placement->placement[i], mem_type);
+ ttm_mem_type_debug(bo->bdev, mem_type);
+ }
+}
+
+#if 0
+static ssize_t ttm_bo_global_show(struct ttm_bo_global *glob,
+ char *buffer)
+{
+
+ return snprintf(buffer, PAGE_SIZE, "%lu\n",
+ (unsigned long) atomic_read(&glob->bo_count));
+}
+#endif
+
+static inline uint32_t ttm_bo_type_flags(unsigned type)
+{
+ return 1 << (type);
+}
+
+static void ttm_bo_release_list(struct ttm_buffer_object *bo)
+{
+ struct ttm_bo_device *bdev = bo->bdev;
+ size_t acc_size = bo->acc_size;
+
+ MPASS(atomic_read(&bo->list_kref) == 0);
+ MPASS(atomic_read(&bo->kref) == 0);
+ MPASS(atomic_read(&bo->cpu_writers) == 0);
+ MPASS(bo->sync_obj == NULL);
+ MPASS(bo->mem.mm_node == NULL);
+ MPASS(list_empty(&bo->lru));
+ MPASS(list_empty(&bo->ddestroy));
+
+ if (bo->ttm)
+ ttm_tt_destroy(bo->ttm);
+ atomic_dec(&bo->glob->bo_count);
+ if (bo->destroy)
+ bo->destroy(bo);
+ else {
+ free(bo, M_TTM_BO);
+ }
+ ttm_mem_global_free(bdev->glob->mem_glob, acc_size);
+}
+
+int
+ttm_bo_wait_unreserved_locked(struct ttm_buffer_object *bo, bool interruptible)
+{
+ const char *wmsg;
+ int flags, ret;
+
+ ret = 0;
+ if (interruptible) {
+ flags = PCATCH;
+ wmsg = "ttbowi";
+ } else {
+ flags = 0;
+ wmsg = "ttbowu";
+ }
+ while (!ttm_bo_is_reserved(bo)) {
+ ret = -msleep(bo, &bo->glob->lru_lock, flags, wmsg, 0);
+ if (ret != 0)
+ break;
+ }
+ return (ret);
+}
+
+void ttm_bo_add_to_lru(struct ttm_buffer_object *bo)
+{
+ struct ttm_bo_device *bdev = bo->bdev;
+ struct ttm_mem_type_manager *man;
+
+ MPASS(ttm_bo_is_reserved(bo));
+
+ if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) {
+
+ MPASS(list_empty(&bo->lru));
+
+ man = &bdev->man[bo->mem.mem_type];
+ list_add_tail(&bo->lru, &man->lru);
+ refcount_acquire(&bo->list_kref);
+
+ if (bo->ttm != NULL) {
+ list_add_tail(&bo->swap, &bo->glob->swap_lru);
+ refcount_acquire(&bo->list_kref);
+ }
+ }
+}
+
+int ttm_bo_del_from_lru(struct ttm_buffer_object *bo)
+{
+ int put_count = 0;
+
+ if (!list_empty(&bo->swap)) {
+ list_del_init(&bo->swap);
+ ++put_count;
+ }
+ if (!list_empty(&bo->lru)) {
+ list_del_init(&bo->lru);
+ ++put_count;
+ }
+
+ /*
+ * TODO: Add a driver hook to delete from
+ * driver-specific LRU's here.
+ */
+
+ return put_count;
+}
+
+int ttm_bo_reserve_locked(struct ttm_buffer_object *bo,
+ bool interruptible,
+ bool no_wait, bool use_sequence, uint32_t sequence)
+{
+ int ret;
+
+ while (unlikely(atomic_read(&bo->reserved) != 0)) {
+ /**
+ * Deadlock avoidance for multi-bo reserving.
+ */
+ if (use_sequence && bo->seq_valid) {
+ /**
+ * We've already reserved this one.
+ */
+ if (unlikely(sequence == bo->val_seq))
+ return -EDEADLK;
+ /**
+ * Already reserved by a thread that will not back
+ * off for us. We need to back off.
+ */
+ if (unlikely(sequence - bo->val_seq < (1 << 31)))
+ return -EAGAIN;
+ }
+
+ if (no_wait)
+ return -EBUSY;
+
+ ret = ttm_bo_wait_unreserved_locked(bo, interruptible);
+ if (unlikely(ret))
+ return ret;
+ }
+
+ atomic_set(&bo->reserved, 1);
+ if (use_sequence) {
+ /**
+ * Wake up waiters that may need to recheck for deadlock,
+ * if we decreased the sequence number.
+ */
+ if (unlikely((bo->val_seq - sequence < (1 << 31))
+ || !bo->seq_valid))
+ wakeup(bo);
+
+ bo->val_seq = sequence;
+ bo->seq_valid = true;
+ } else {
+ bo->seq_valid = false;
+ }
+
+ return 0;
+}
+
+void ttm_bo_list_ref_sub(struct ttm_buffer_object *bo, int count,
+ bool never_free)
+{
+ u_int old;
+
+ old = atomic_fetchadd_int(&bo->list_kref, -count);
+ if (old <= count) {
+ if (never_free)
+ panic("ttm_bo_ref_buf");
+ ttm_bo_release_list(bo);
+ }
+}
+
+int ttm_bo_reserve(struct ttm_buffer_object *bo,
+ bool interruptible,
+ bool no_wait, bool use_sequence, uint32_t sequence)
+{
+ struct ttm_bo_global *glob = bo->glob;
+ int put_count = 0;
+ int ret;
+
+ mtx_lock(&glob->lru_lock);
+ ret = ttm_bo_reserve_locked(bo, interruptible, no_wait, use_sequence,
+ sequence);
+ if (likely(ret == 0))
+ put_count = ttm_bo_del_from_lru(bo);
+ mtx_unlock(&glob->lru_lock);
+
+ ttm_bo_list_ref_sub(bo, put_count, true);
+
+ return ret;
+}
+
+void ttm_bo_unreserve_locked(struct ttm_buffer_object *bo)
+{
+ ttm_bo_add_to_lru(bo);
+ atomic_set(&bo->reserved, 0);
+ wakeup(bo);
+}
+
+void ttm_bo_unreserve(struct ttm_buffer_object *bo)
+{
+ struct ttm_bo_global *glob = bo->glob;
+
+ mtx_lock(&glob->lru_lock);
+ ttm_bo_unreserve_locked(bo);
+ mtx_unlock(&glob->lru_lock);
+}
+
+/*
+ * Call bo->mutex locked.
+ */
+static int ttm_bo_add_ttm(struct ttm_buffer_object *bo, bool zero_alloc)
+{
+ struct ttm_bo_device *bdev = bo->bdev;
+ struct ttm_bo_global *glob = bo->glob;
+ int ret = 0;
+ uint32_t page_flags = 0;
+
+ TTM_ASSERT_LOCKED(&bo->mutex);
+ bo->ttm = NULL;
+
+ if (bdev->need_dma32)
+ page_flags |= TTM_PAGE_FLAG_DMA32;
+
+ switch (bo->type) {
+ case ttm_bo_type_device:
+ if (zero_alloc)
+ page_flags |= TTM_PAGE_FLAG_ZERO_ALLOC;
+ case ttm_bo_type_kernel:
+ bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT,
+ page_flags, glob->dummy_read_page);
+ if (unlikely(bo->ttm == NULL))
+ ret = -ENOMEM;
+ break;
+ case ttm_bo_type_sg:
+ bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT,
+ page_flags | TTM_PAGE_FLAG_SG,
+ glob->dummy_read_page);
+ if (unlikely(bo->ttm == NULL)) {
+ ret = -ENOMEM;
+ break;
+ }
+ bo->ttm->sg = bo->sg;
+ break;
+ default:
+ printf("[TTM] Illegal buffer object type\n");
+ ret = -EINVAL;
+ break;
+ }
+
+ return ret;
+}
+
+static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
+ struct ttm_mem_reg *mem,
+ bool evict, bool interruptible,
+ bool no_wait_gpu)
+{
+ struct ttm_bo_device *bdev = bo->bdev;
+ bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem);
+ bool new_is_pci = ttm_mem_reg_is_pci(bdev, mem);
+ struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type];
+ struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type];
+ int ret = 0;
+
+ if (old_is_pci || new_is_pci ||
+ ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0)) {
+ ret = ttm_mem_io_lock(old_man, true);
+ if (unlikely(ret != 0))
+ goto out_err;
+ ttm_bo_unmap_virtual_locked(bo);
+ ttm_mem_io_unlock(old_man);
+ }
+
+ /*
+ * Create and bind a ttm if required.
+ */
+
+ if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) {
+ if (bo->ttm == NULL) {
+ bool zero = !(old_man->flags & TTM_MEMTYPE_FLAG_FIXED);
+ ret = ttm_bo_add_ttm(bo, zero);
+ if (ret)
+ goto out_err;
+ }
+
+ ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement);
+ if (ret)
+ goto out_err;
+
+ if (mem->mem_type != TTM_PL_SYSTEM) {
+ ret = ttm_tt_bind(bo->ttm, mem);
+ if (ret)
+ goto out_err;
+ }
+
+ if (bo->mem.mem_type == TTM_PL_SYSTEM) {
+ if (bdev->driver->move_notify)
+ bdev->driver->move_notify(bo, mem);
+ bo->mem = *mem;
+ mem->mm_node = NULL;
+ goto moved;
+ }
+ }
+
+ if (bdev->driver->move_notify)
+ bdev->driver->move_notify(bo, mem);
+
+ if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) &&
+ !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED))
+ ret = ttm_bo_move_ttm(bo, evict, no_wait_gpu, mem);
+ else if (bdev->driver->move)
+ ret = bdev->driver->move(bo, evict, interruptible,
+ no_wait_gpu, mem);
+ else
+ ret = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, mem);
+
+ if (ret) {
+ if (bdev->driver->move_notify) {
+ struct ttm_mem_reg tmp_mem = *mem;
+ *mem = bo->mem;
+ bo->mem = tmp_mem;
+ bdev->driver->move_notify(bo, mem);
+ bo->mem = *mem;
+ }
+
+ goto out_err;
+ }
+
+moved:
+ if (bo->evicted) {
+ ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement);
+ if (ret)
+ printf("[TTM] Can not flush read caches\n");
+ bo->evicted = false;
+ }
+
+ if (bo->mem.mm_node) {
+ bo->offset = (bo->mem.start << PAGE_SHIFT) +
+ bdev->man[bo->mem.mem_type].gpu_offset;
+ bo->cur_placement = bo->mem.placement;
+ } else
+ bo->offset = 0;
+
+ return 0;
+
+out_err:
+ new_man = &bdev->man[bo->mem.mem_type];
+ if ((new_man->flags & TTM_MEMTYPE_FLAG_FIXED) && bo->ttm) {
+ ttm_tt_unbind(bo->ttm);
+ ttm_tt_destroy(bo->ttm);
+ bo->ttm = NULL;
+ }
+
+ return ret;
+}
+
+/**
+ * Call bo::reserved.
+ * Will release GPU memory type usage on destruction.
+ * This is the place to put in driver specific hooks to release
+ * driver private resources.
+ * Will release the bo::reserved lock.
+ */
+
+static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo)
+{
+ if (bo->bdev->driver->move_notify)
+ bo->bdev->driver->move_notify(bo, NULL);
+
+ if (bo->ttm) {
+ ttm_tt_unbind(bo->ttm);
+ ttm_tt_destroy(bo->ttm);
+ bo->ttm = NULL;
+ }
+ ttm_bo_mem_put(bo, &bo->mem);
+
+ atomic_set(&bo->reserved, 0);
+ wakeup(&bo);
+
+ /*
+ * Since the final reference to this bo may not be dropped by
+ * the current task we have to put a memory barrier here to make
+ * sure the changes done in this function are always visible.
+ *
+ * This function only needs protection against the final kref_put.
+ */
+ mb();
+}
+
+static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo)
+{
+ struct ttm_bo_device *bdev = bo->bdev;
+ struct ttm_bo_global *glob = bo->glob;
+ struct ttm_bo_driver *driver = bdev->driver;
+ void *sync_obj = NULL;
+ int put_count;
+ int ret;
+
+ mtx_lock(&glob->lru_lock);
+ ret = ttm_bo_reserve_locked(bo, false, true, false, 0);
+
+ mtx_lock(&bdev->fence_lock);
+ (void) ttm_bo_wait(bo, false, false, true);
+ if (!ret && !bo->sync_obj) {
+ mtx_unlock(&bdev->fence_lock);
+ put_count = ttm_bo_del_from_lru(bo);
+
+ mtx_unlock(&glob->lru_lock);
+ ttm_bo_cleanup_memtype_use(bo);
+
+ ttm_bo_list_ref_sub(bo, put_count, true);
+
+ return;
+ }
+ if (bo->sync_obj)
+ sync_obj = driver->sync_obj_ref(bo->sync_obj);
+ mtx_unlock(&bdev->fence_lock);
+
+ if (!ret) {
+ atomic_set(&bo->reserved, 0);
+ wakeup(bo);
+ }
+
+ refcount_acquire(&bo->list_kref);
+ list_add_tail(&bo->ddestroy, &bdev->ddestroy);
+ mtx_unlock(&glob->lru_lock);
+
+ if (sync_obj) {
+ driver->sync_obj_flush(sync_obj);
+ driver->sync_obj_unref(&sync_obj);
+ }
+ taskqueue_enqueue_timeout(taskqueue_thread, &bdev->wq,
+ ((hz / 100) < 1) ? 1 : hz / 100);
+}
+
+/**
+ * function ttm_bo_cleanup_refs_and_unlock
+ * If bo idle, remove from delayed- and lru lists, and unref.
+ * If not idle, do nothing.
+ *
+ * Must be called with lru_lock and reservation held, this function
+ * will drop both before returning.
+ *
+ * @interruptible Any sleeps should occur interruptibly.
+ * @no_wait_gpu Never wait for gpu. Return -EBUSY instead.
+ */
+
+static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo,
+ bool interruptible,
+ bool no_wait_gpu)
+{
+ struct ttm_bo_device *bdev = bo->bdev;
+ struct ttm_bo_driver *driver = bdev->driver;
+ struct ttm_bo_global *glob = bo->glob;
+ int put_count;
+ int ret;
+
+ mtx_lock(&bdev->fence_lock);
+ ret = ttm_bo_wait(bo, false, false, true);
+
+ if (ret && !no_wait_gpu) {
+ void *sync_obj;
+
+ /*
+ * Take a reference to the fence and unreserve,
+ * at this point the buffer should be dead, so
+ * no new sync objects can be attached.
+ */
+ sync_obj = driver->sync_obj_ref(bo->sync_obj);
+ mtx_unlock(&bdev->fence_lock);
+
+ atomic_set(&bo->reserved, 0);
+ wakeup(bo);
+ mtx_unlock(&glob->lru_lock);
+
+ ret = driver->sync_obj_wait(sync_obj, false, interruptible);
+ driver->sync_obj_unref(&sync_obj);
+ if (ret)
+ return ret;
+
+ /*
+ * remove sync_obj with ttm_bo_wait, the wait should be
+ * finished, and no new wait object should have been added.
+ */
+ mtx_lock(&bdev->fence_lock);
+ ret = ttm_bo_wait(bo, false, false, true);
+ mtx_unlock(&bdev->fence_lock);
+ if (ret)
+ return ret;
+
+ mtx_lock(&glob->lru_lock);
+ ret = ttm_bo_reserve_locked(bo, false, true, false, 0);
+
+ /*
+ * We raced, and lost, someone else holds the reservation now,
+ * and is probably busy in ttm_bo_cleanup_memtype_use.
+ *
+ * Even if it's not the case, because we finished waiting any
+ * delayed destruction would succeed, so just return success
+ * here.
+ */
+ if (ret) {
+ mtx_unlock(&glob->lru_lock);
+ return 0;
+ }
+ } else
+ mtx_unlock(&bdev->fence_lock);
+
+ if (ret || unlikely(list_empty(&bo->ddestroy))) {
+ atomic_set(&bo->reserved, 0);
+ wakeup(bo);
+ mtx_unlock(&glob->lru_lock);
+ return ret;
+ }
+
+ put_count = ttm_bo_del_from_lru(bo);
+ list_del_init(&bo->ddestroy);
+ ++put_count;
+
+ mtx_unlock(&glob->lru_lock);
+ ttm_bo_cleanup_memtype_use(bo);
+
+ ttm_bo_list_ref_sub(bo, put_count, true);
+
+ return 0;
+}
+
+/**
+ * Traverse the delayed list, and call ttm_bo_cleanup_refs on all
+ * encountered buffers.
+ */
+
+static int ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all)
+{
+ struct ttm_bo_global *glob = bdev->glob;
+ struct ttm_buffer_object *entry = NULL;
+ int ret = 0;
+
+ mtx_lock(&glob->lru_lock);
+ if (list_empty(&bdev->ddestroy))
+ goto out_unlock;
+
+ entry = list_first_entry(&bdev->ddestroy,
+ struct ttm_buffer_object, ddestroy);
+ refcount_acquire(&entry->list_kref);
+
+ for (;;) {
+ struct ttm_buffer_object *nentry = NULL;
+
+ if (entry->ddestroy.next != &bdev->ddestroy) {
+ nentry = list_first_entry(&entry->ddestroy,
+ struct ttm_buffer_object, ddestroy);
+ refcount_acquire(&nentry->list_kref);
+ }
+
+ ret = ttm_bo_reserve_locked(entry, false, !remove_all, false, 0);
+ if (!ret)
+ ret = ttm_bo_cleanup_refs_and_unlock(entry, false,
+ !remove_all);
+ else
+ mtx_unlock(&glob->lru_lock);
+
+ if (refcount_release(&entry->list_kref))
+ ttm_bo_release_list(entry);
+ entry = nentry;
+
+ if (ret || !entry)
+ goto out;
+
+ mtx_lock(&glob->lru_lock);
+ if (list_empty(&entry->ddestroy))
+ break;
+ }
+
+out_unlock:
+ mtx_unlock(&glob->lru_lock);
+out:
+ if (entry && refcount_release(&entry->list_kref))
+ ttm_bo_release_list(entry);
+ return ret;
+}
+
+static void ttm_bo_delayed_workqueue(void *arg, int pending __unused)
+{
+ struct ttm_bo_device *bdev = arg;
+
+ if (ttm_bo_delayed_delete(bdev, false)) {
+ taskqueue_enqueue_timeout(taskqueue_thread, &bdev->wq,
+ ((hz / 100) < 1) ? 1 : hz / 100);
+ }
+}
+
+static void ttm_bo_release(struct ttm_buffer_object *bo)
+{
+ struct ttm_bo_device *bdev = bo->bdev;
+ struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];
+
+ rw_wlock(&bdev->vm_lock);
+ if (likely(bo->vm_node != NULL)) {
+ RB_REMOVE(ttm_bo_device_buffer_objects,
+ &bdev->addr_space_rb, bo);
+ drm_mm_put_block(bo->vm_node);
+ bo->vm_node = NULL;
+ }
+ rw_wunlock(&bdev->vm_lock);
+ ttm_mem_io_lock(man, false);
+ ttm_mem_io_free_vm(bo);
+ ttm_mem_io_unlock(man);
+ ttm_bo_cleanup_refs_or_queue(bo);
+ if (refcount_release(&bo->list_kref))
+ ttm_bo_release_list(bo);
+}
+
+void ttm_bo_unref(struct ttm_buffer_object **p_bo)
+{
+ struct ttm_buffer_object *bo = *p_bo;
+
+ *p_bo = NULL;
+ if (refcount_release(&bo->kref))
+ ttm_bo_release(bo);
+}
+
+int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev)
+{
+ int pending;
+
+ taskqueue_cancel_timeout(taskqueue_thread, &bdev->wq, &pending);
+ if (pending)
+ taskqueue_drain_timeout(taskqueue_thread, &bdev->wq);
+ return (pending);
+}
+
+void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched)
+{
+ if (resched) {
+ taskqueue_enqueue_timeout(taskqueue_thread, &bdev->wq,
+ ((hz / 100) < 1) ? 1 : hz / 100);
+ }
+}
+
+static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible,
+ bool no_wait_gpu)
+{
+ struct ttm_bo_device *bdev = bo->bdev;
+ struct ttm_mem_reg evict_mem;
+ struct ttm_placement placement;
+ int ret = 0;
+
+ mtx_lock(&bdev->fence_lock);
+ ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu);
+ mtx_unlock(&bdev->fence_lock);
+
+ if (unlikely(ret != 0)) {
+ if (ret != -ERESTART) {
+ printf("[TTM] Failed to expire sync object before buffer eviction\n");
+ }
+ goto out;
+ }
+
+ MPASS(ttm_bo_is_reserved(bo));
+
+ evict_mem = bo->mem;
+ evict_mem.mm_node = NULL;
+ evict_mem.bus.io_reserved_vm = false;
+ evict_mem.bus.io_reserved_count = 0;
+
+ placement.fpfn = 0;
*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
More information about the svn-src-all
mailing list