svn commit: r247835 - in head/sys: dev/drm2 dev/drm2/ttm modules/drm2/drm2
Ronald Klop
ronald-freebsd8 at klop.yi.org
Tue Mar 5 15:45:09 UTC 2013
I know this is not done yet, but I appreciate the work very much.
Regards,
Ronald.
On Tue, 05 Mar 2013 10:49:34 +0100, Konstantin Belousov <kib at freebsd.org>
wrote:
> Author: kib
> Date: Tue Mar 5 09:49:34 2013
> New Revision: 247835
> URL: http://svnweb.freebsd.org/changeset/base/247835
>
> Log:
> Import the preliminary port of the TTM.
> The early commit is done to facilitate the off-tree work on the
> porting of the Radeon driver.
> Sponsored by: The FreeBSD Foundation
> Debugged and tested by: dumbbell
> MFC after: 1 month
>
> Added:
> head/sys/dev/drm2/ttm/
> head/sys/dev/drm2/ttm/ttm_agp_backend.c (contents, props changed)
> head/sys/dev/drm2/ttm/ttm_bo.c (contents, props changed)
> head/sys/dev/drm2/ttm/ttm_bo_api.h (contents, props changed)
> head/sys/dev/drm2/ttm/ttm_bo_driver.h (contents, props changed)
> head/sys/dev/drm2/ttm/ttm_bo_manager.c (contents, props changed)
> head/sys/dev/drm2/ttm/ttm_bo_util.c (contents, props changed)
> head/sys/dev/drm2/ttm/ttm_bo_vm.c (contents, props changed)
> head/sys/dev/drm2/ttm/ttm_execbuf_util.c (contents, props changed)
> head/sys/dev/drm2/ttm/ttm_execbuf_util.h (contents, props changed)
> head/sys/dev/drm2/ttm/ttm_lock.c (contents, props changed)
> head/sys/dev/drm2/ttm/ttm_lock.h (contents, props changed)
> head/sys/dev/drm2/ttm/ttm_memory.c (contents, props changed)
> head/sys/dev/drm2/ttm/ttm_memory.h (contents, props changed)
> head/sys/dev/drm2/ttm/ttm_module.h (contents, props changed)
> head/sys/dev/drm2/ttm/ttm_object.c (contents, props changed)
> head/sys/dev/drm2/ttm/ttm_object.h (contents, props changed)
> head/sys/dev/drm2/ttm/ttm_page_alloc.c (contents, props changed)
> head/sys/dev/drm2/ttm/ttm_page_alloc.h (contents, props changed)
> head/sys/dev/drm2/ttm/ttm_page_alloc_dma.c (contents, props changed)
> head/sys/dev/drm2/ttm/ttm_placement.h (contents, props changed)
> head/sys/dev/drm2/ttm/ttm_tt.c (contents, props changed)
> Modified:
> head/sys/dev/drm2/drmP.h
> head/sys/dev/drm2/drm_drv.c
> head/sys/dev/drm2/drm_gem.c
> head/sys/modules/drm2/drm2/Makefile
>
> Modified: head/sys/dev/drm2/drmP.h
> ==============================================================================
> --- head/sys/dev/drm2/drmP.h Tue Mar 5 09:27:21 2013 (r247834)
> +++ head/sys/dev/drm2/drmP.h Tue Mar 5 09:49:34 2013 (r247835)
> @@ -906,6 +906,7 @@ struct drm_device {
> struct drm_minor *control; /**< Control node for card */
> struct drm_minor *primary; /**< render type primary screen head */
> + void *drm_ttm_bo;
> struct unrhdr *drw_unrhdr;
> /* RB tree of drawable infos */
> RB_HEAD(drawable_tree, bsd_drm_drawable_info) drw_head;
> @@ -1302,10 +1303,14 @@ void drm_gem_release(struct drm_device *
> int drm_gem_create_mmap_offset(struct drm_gem_object *obj);
> void drm_gem_free_mmap_offset(struct drm_gem_object *obj);
> -int drm_gem_mmap_single(struct cdev *kdev, vm_ooffset_t *offset,
> vm_size_t size,
> - struct vm_object **obj_res, int nprot);
> +int drm_gem_mmap_single(struct drm_device *dev, vm_ooffset_t *offset,
> + vm_size_t size, struct vm_object **obj_res, int nprot);
> void drm_gem_pager_dtr(void *obj);
> +struct ttm_bo_device;
> +int ttm_bo_mmap_single(struct ttm_bo_device *bdev, vm_ooffset_t *offset,
> + vm_size_t size, struct vm_object **obj_res, int nprot);
> +
> void drm_device_lock_mtx(struct drm_device *dev);
> void drm_device_unlock_mtx(struct drm_device *dev);
> int drm_device_sleep_mtx(struct drm_device *dev, void *chan, int flags,
>
> Modified: head/sys/dev/drm2/drm_drv.c
> ==============================================================================
> --- head/sys/dev/drm2/drm_drv.c Tue Mar 5 09:27:21 2013 (r247834)
> +++ head/sys/dev/drm2/drm_drv.c Tue Mar 5 09:49:34 2013 (r247835)
> @@ -58,6 +58,8 @@ static int drm_load(struct drm_device *d
> static void drm_unload(struct drm_device *dev);
> static drm_pci_id_list_t *drm_find_description(int vendor, int device,
> drm_pci_id_list_t *idlist);
> +static int drm_mmap_single(struct cdev *kdev, vm_ooffset_t *offset,
> + vm_size_t size, struct vm_object **obj_res, int nprot);
> static int
> drm_modevent(module_t mod, int type, void *data)
> @@ -187,7 +189,7 @@ static struct cdevsw drm_cdevsw = {
> .d_ioctl = drm_ioctl,
> .d_poll = drm_poll,
> .d_mmap = drm_mmap,
> - .d_mmap_single = drm_gem_mmap_single,
> + .d_mmap_single = drm_mmap_single,
> .d_name = "drm",
> .d_flags = D_TRACKCLOSE
> };
> @@ -955,6 +957,23 @@ drm_add_busid_modesetting(struct drm_dev
> return (0);
> }
> +static int
> +drm_mmap_single(struct cdev *kdev, vm_ooffset_t *offset, vm_size_t size,
> + struct vm_object **obj_res, int nprot)
> +{
> + struct drm_device *dev;
> +
> + dev = drm_get_device_from_kdev(kdev);
> + if ((dev->driver->driver_features & DRIVER_GEM) != 0) {
> + return (drm_gem_mmap_single(dev, offset, size, obj_res, nprot));
> + } else if (dev->drm_ttm_bo != NULL) {
> + return (ttm_bo_mmap_single(dev->drm_ttm_bo, offset, size,
> + obj_res, nprot));
> + } else {
> + return (ENODEV);
> + }
> +}
> +
> #if DRM_LINUX
> #include <sys/sysproto.h>
>
> Modified: head/sys/dev/drm2/drm_gem.c
> ==============================================================================
> --- head/sys/dev/drm2/drm_gem.c Tue Mar 5 09:27:21 2013 (r247834)
> +++ head/sys/dev/drm2/drm_gem.c Tue Mar 5 09:49:34 2013 (r247835)
> @@ -441,16 +441,12 @@ drm_gem_free_mmap_offset(struct drm_gem_
> }
> int
> -drm_gem_mmap_single(struct cdev *kdev, vm_ooffset_t *offset, vm_size_t
> size,
> +drm_gem_mmap_single(struct drm_device *dev, vm_ooffset_t *offset,
> vm_size_t size,
> struct vm_object **obj_res, int nprot)
> {
> - struct drm_device *dev;
> struct drm_gem_object *gem_obj;
> struct vm_object *vm_obj;
> - dev = drm_get_device_from_kdev(kdev);
> - if ((dev->driver->driver_features & DRIVER_GEM) == 0)
> - return (ENODEV);
> DRM_LOCK(dev);
> gem_obj = drm_gem_object_from_offset(dev, *offset);
> if (gem_obj == NULL) {
>
> Added: head/sys/dev/drm2/ttm/ttm_agp_backend.c
> ==============================================================================
> --- /dev/null 00:00:00 1970 (empty, because file is newly added)
> +++ head/sys/dev/drm2/ttm/ttm_agp_backend.c Tue Mar 5 09:49:34
> 2013 (r247835)
> @@ -0,0 +1,145 @@
> +/**************************************************************************
> + *
> + * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
> + * All Rights Reserved.
> + *
> + * Permission is hereby granted, free of charge, to any person
> obtaining a
> + * copy of this software and associated documentation files (the
> + * "Software"), to deal in the Software without restriction, including
> + * without limitation the rights to use, copy, modify, merge, publish,
> + * distribute, sub license, and/or sell copies of the Software, and to
> + * permit persons to whom the Software is furnished to do so, subject to
> + * the following conditions:
> + *
> + * The above copyright notice and this permission notice (including the
> + * next paragraph) shall be included in all copies or substantial
> portions
> + * of the Software.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
> EXPRESS OR
> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
> MERCHANTABILITY,
> + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT
> SHALL
> + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR
> ANY CLAIM,
> + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
> + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE
> OR THE
> + * USE OR OTHER DEALINGS IN THE SOFTWARE.
> + *
> +
> **************************************************************************/
> +/*
> + * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
> + * Keith Packard.
> + */
> +
> +#include <sys/cdefs.h>
> +__FBSDID("$FreeBSD$");
> +
> +#include <dev/drm2/drmP.h>
> +#include <dev/drm2/ttm/ttm_module.h>
> +#include <dev/drm2/ttm/ttm_bo_driver.h>
> +#include <dev/drm2/ttm/ttm_page_alloc.h>
> +#ifdef TTM_HAS_AGP
> +#include <dev/drm2/ttm/ttm_placement.h>
> +
> +struct ttm_agp_backend {
> + struct ttm_tt ttm;
> + struct agp_memory *mem;
> + device_t bridge;
> +};
> +
> +MALLOC_DEFINE(M_TTM_AGP, "ttm_agp", "TTM AGP Backend");
> +
> +static int ttm_agp_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem)
> +{
> + struct ttm_agp_backend *agp_be = container_of(ttm, struct
> ttm_agp_backend, ttm);
> + struct drm_mm_node *node = bo_mem->mm_node;
> + struct agp_memory *mem;
> + int ret, cached = (bo_mem->placement & TTM_PL_FLAG_CACHED);
> + unsigned i;
> +
> + mem = agp_alloc_memory(agp_be->bridge, AGP_USER_MEMORY,
> ttm->num_pages);
> + if (unlikely(mem == NULL))
> + return -ENOMEM;
> +
> + mem->page_count = 0;
> + for (i = 0; i < ttm->num_pages; i++) {
> + vm_page_t page = ttm->pages[i];
> +
> + if (!page)
> + page = ttm->dummy_read_page;
> +
> + mem->pages[mem->page_count++] = page;
> + }
> + agp_be->mem = mem;
> +
> + mem->is_flushed = 1;
> + mem->type = (cached) ? AGP_USER_CACHED_MEMORY : AGP_USER_MEMORY;
> +
> + ret = agp_bind_memory(mem, node->start);
> + if (ret)
> + pr_err("AGP Bind memory failed\n");
> +
> + return ret;
> +}
> +
> +static int ttm_agp_unbind(struct ttm_tt *ttm)
> +{
> + struct ttm_agp_backend *agp_be = container_of(ttm, struct
> ttm_agp_backend, ttm);
> +
> + if (agp_be->mem) {
> + if (agp_be->mem->is_bound)
> + return agp_unbind_memory(agp_be->mem);
> + agp_free_memory(agp_be->mem);
> + agp_be->mem = NULL;
> + }
> + return 0;
> +}
> +
> +static void ttm_agp_destroy(struct ttm_tt *ttm)
> +{
> + struct ttm_agp_backend *agp_be = container_of(ttm, struct
> ttm_agp_backend, ttm);
> +
> + if (agp_be->mem)
> + ttm_agp_unbind(ttm);
> + ttm_tt_fini(ttm);
> + free(agp_be, M_TTM_AGP);
> +}
> +
> +static struct ttm_backend_func ttm_agp_func = {
> + .bind = ttm_agp_bind,
> + .unbind = ttm_agp_unbind,
> + .destroy = ttm_agp_destroy,
> +};
> +
> +struct ttm_tt *ttm_agp_tt_create(struct ttm_bo_device *bdev,
> + device_t bridge,
> + unsigned long size, uint32_t page_flags,
> + vm_page_t dummy_read_page)
> +{
> + struct ttm_agp_backend *agp_be;
> +
> + agp_be = malloc(sizeof(*agp_be), M_TTM_AGP, M_WAITOK | M_ZERO);
> +
> + agp_be->mem = NULL;
> + agp_be->bridge = bridge;
> + agp_be->ttm.func = &ttm_agp_func;
> +
> + if (ttm_tt_init(&agp_be->ttm, bdev, size, page_flags,
> dummy_read_page)) {
> + return NULL;
> + }
> +
> + return &agp_be->ttm;
> +}
> +
> +int ttm_agp_tt_populate(struct ttm_tt *ttm)
> +{
> + if (ttm->state != tt_unpopulated)
> + return 0;
> +
> + return ttm_pool_populate(ttm);
> +}
> +
> +void ttm_agp_tt_unpopulate(struct ttm_tt *ttm)
> +{
> + ttm_pool_unpopulate(ttm);
> +}
> +
> +#endif
>
> Added: head/sys/dev/drm2/ttm/ttm_bo.c
> ==============================================================================
> --- /dev/null 00:00:00 1970 (empty, because file is newly added)
> +++ head/sys/dev/drm2/ttm/ttm_bo.c Tue Mar 5 09:49:34 2013 (r247835)
> @@ -0,0 +1,1820 @@
> +/**************************************************************************
> + *
> + * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
> + * All Rights Reserved.
> + *
> + * Permission is hereby granted, free of charge, to any person
> obtaining a
> + * copy of this software and associated documentation files (the
> + * "Software"), to deal in the Software without restriction, including
> + * without limitation the rights to use, copy, modify, merge, publish,
> + * distribute, sub license, and/or sell copies of the Software, and to
> + * permit persons to whom the Software is furnished to do so, subject to
> + * the following conditions:
> + *
> + * The above copyright notice and this permission notice (including the
> + * next paragraph) shall be included in all copies or substantial
> portions
> + * of the Software.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
> EXPRESS OR
> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
> MERCHANTABILITY,
> + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT
> SHALL
> + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR
> ANY CLAIM,
> + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
> + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE
> OR THE
> + * USE OR OTHER DEALINGS IN THE SOFTWARE.
> + *
> +
> **************************************************************************/
> +/*
> + * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
> + */
> +
> +#include <sys/cdefs.h>
> +__FBSDID("$FreeBSD$");
> +
> +#include <dev/drm2/drmP.h>
> +#include <dev/drm2/ttm/ttm_module.h>
> +#include <dev/drm2/ttm/ttm_bo_driver.h>
> +#include <dev/drm2/ttm/ttm_placement.h>
> +
> +#define TTM_ASSERT_LOCKED(param)
> +#define TTM_DEBUG(fmt, arg...)
> +#define TTM_BO_HASH_ORDER 13
> +
> +static int ttm_bo_setup_vm(struct ttm_buffer_object *bo);
> +static int ttm_bo_swapout(struct ttm_mem_shrink *shrink);
> +static void ttm_bo_global_kobj_release(struct ttm_bo_global *glob);
> +
> +MALLOC_DEFINE(M_TTM_BO, "ttm_bo", "TTM Buffer Objects");
> +
> +static inline int ttm_mem_type_from_flags(uint32_t flags, uint32_t
> *mem_type)
> +{
> + int i;
> +
> + for (i = 0; i <= TTM_PL_PRIV5; i++)
> + if (flags & (1 << i)) {
> + *mem_type = i;
> + return 0;
> + }
> + return -EINVAL;
> +}
> +
> +static void ttm_mem_type_debug(struct ttm_bo_device *bdev, int mem_type)
> +{
> + struct ttm_mem_type_manager *man = &bdev->man[mem_type];
> +
> + printf(" has_type: %d\n", man->has_type);
> + printf(" use_type: %d\n", man->use_type);
> + printf(" flags: 0x%08X\n", man->flags);
> + printf(" gpu_offset: 0x%08lX\n", man->gpu_offset);
> + printf(" size: %ju\n", (uintmax_t)man->size);
> + printf(" available_caching: 0x%08X\n", man->available_caching);
> + printf(" default_caching: 0x%08X\n", man->default_caching);
> + if (mem_type != TTM_PL_SYSTEM)
> + (*man->func->debug)(man, TTM_PFX);
> +}
> +
> +static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo,
> + struct ttm_placement *placement)
> +{
> + int i, ret, mem_type;
> +
> + printf("No space for %p (%lu pages, %luK, %luM)\n",
> + bo, bo->mem.num_pages, bo->mem.size >> 10,
> + bo->mem.size >> 20);
> + for (i = 0; i < placement->num_placement; i++) {
> + ret = ttm_mem_type_from_flags(placement->placement[i],
> + &mem_type);
> + if (ret)
> + return;
> + printf(" placement[%d]=0x%08X (%d)\n",
> + i, placement->placement[i], mem_type);
> + ttm_mem_type_debug(bo->bdev, mem_type);
> + }
> +}
> +
> +#if 0
> +static ssize_t ttm_bo_global_show(struct ttm_bo_global *glob,
> + char *buffer)
> +{
> +
> + return snprintf(buffer, PAGE_SIZE, "%lu\n",
> + (unsigned long) atomic_read(&glob->bo_count));
> +}
> +#endif
> +
> +static inline uint32_t ttm_bo_type_flags(unsigned type)
> +{
> + return 1 << (type);
> +}
> +
> +static void ttm_bo_release_list(struct ttm_buffer_object *bo)
> +{
> + struct ttm_bo_device *bdev = bo->bdev;
> + size_t acc_size = bo->acc_size;
> +
> + MPASS(atomic_read(&bo->list_kref) == 0);
> + MPASS(atomic_read(&bo->kref) == 0);
> + MPASS(atomic_read(&bo->cpu_writers) == 0);
> + MPASS(bo->sync_obj == NULL);
> + MPASS(bo->mem.mm_node == NULL);
> + MPASS(list_empty(&bo->lru));
> + MPASS(list_empty(&bo->ddestroy));
> +
> + if (bo->ttm)
> + ttm_tt_destroy(bo->ttm);
> + atomic_dec(&bo->glob->bo_count);
> + if (bo->destroy)
> + bo->destroy(bo);
> + else {
> + free(bo, M_TTM_BO);
> + }
> + ttm_mem_global_free(bdev->glob->mem_glob, acc_size);
> +}
> +
> +int
> +ttm_bo_wait_unreserved_locked(struct ttm_buffer_object *bo, bool
> interruptible)
> +{
> + const char *wmsg;
> + int flags, ret;
> +
> + ret = 0;
> + if (interruptible) {
> + flags = PCATCH;
> + wmsg = "ttbowi";
> + } else {
> + flags = 0;
> + wmsg = "ttbowu";
> + }
> + while (!ttm_bo_is_reserved(bo)) {
> + ret = -msleep(bo, &bo->glob->lru_lock, flags, wmsg, 0);
> + if (ret != 0)
> + break;
> + }
> + return (ret);
> +}
> +
> +void ttm_bo_add_to_lru(struct ttm_buffer_object *bo)
> +{
> + struct ttm_bo_device *bdev = bo->bdev;
> + struct ttm_mem_type_manager *man;
> +
> + MPASS(ttm_bo_is_reserved(bo));
> +
> + if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) {
> +
> + MPASS(list_empty(&bo->lru));
> +
> + man = &bdev->man[bo->mem.mem_type];
> + list_add_tail(&bo->lru, &man->lru);
> + refcount_acquire(&bo->list_kref);
> +
> + if (bo->ttm != NULL) {
> + list_add_tail(&bo->swap, &bo->glob->swap_lru);
> + refcount_acquire(&bo->list_kref);
> + }
> + }
> +}
> +
> +int ttm_bo_del_from_lru(struct ttm_buffer_object *bo)
> +{
> + int put_count = 0;
> +
> + if (!list_empty(&bo->swap)) {
> + list_del_init(&bo->swap);
> + ++put_count;
> + }
> + if (!list_empty(&bo->lru)) {
> + list_del_init(&bo->lru);
> + ++put_count;
> + }
> +
> + /*
> + * TODO: Add a driver hook to delete from
> + * driver-specific LRU's here.
> + */
> +
> + return put_count;
> +}
> +
> +int ttm_bo_reserve_locked(struct ttm_buffer_object *bo,
> + bool interruptible,
> + bool no_wait, bool use_sequence, uint32_t sequence)
> +{
> + int ret;
> +
> + while (unlikely(atomic_read(&bo->reserved) != 0)) {
> + /**
> + * Deadlock avoidance for multi-bo reserving.
> + */
> + if (use_sequence && bo->seq_valid) {
> + /**
> + * We've already reserved this one.
> + */
> + if (unlikely(sequence == bo->val_seq))
> + return -EDEADLK;
> + /**
> + * Already reserved by a thread that will not back
> + * off for us. We need to back off.
> + */
> + if (unlikely(sequence - bo->val_seq < (1 << 31)))
> + return -EAGAIN;
> + }
> +
> + if (no_wait)
> + return -EBUSY;
> +
> + ret = ttm_bo_wait_unreserved_locked(bo, interruptible);
> + if (unlikely(ret))
> + return ret;
> + }
> +
> + atomic_set(&bo->reserved, 1);
> + if (use_sequence) {
> + /**
> + * Wake up waiters that may need to recheck for deadlock,
> + * if we decreased the sequence number.
> + */
> + if (unlikely((bo->val_seq - sequence < (1 << 31))
> + || !bo->seq_valid))
> + wakeup(bo);
> +
> + bo->val_seq = sequence;
> + bo->seq_valid = true;
> + } else {
> + bo->seq_valid = false;
> + }
> +
> + return 0;
> +}
> +
> +void ttm_bo_list_ref_sub(struct ttm_buffer_object *bo, int count,
> + bool never_free)
> +{
> + u_int old;
> +
> + old = atomic_fetchadd_int(&bo->list_kref, -count);
> + if (old <= count) {
> + if (never_free)
> + panic("ttm_bo_ref_buf");
> + ttm_bo_release_list(bo);
> + }
> +}
> +
> +int ttm_bo_reserve(struct ttm_buffer_object *bo,
> + bool interruptible,
> + bool no_wait, bool use_sequence, uint32_t sequence)
> +{
> + struct ttm_bo_global *glob = bo->glob;
> + int put_count = 0;
> + int ret;
> +
> + mtx_lock(&glob->lru_lock);
> + ret = ttm_bo_reserve_locked(bo, interruptible, no_wait, use_sequence,
> + sequence);
> + if (likely(ret == 0))
> + put_count = ttm_bo_del_from_lru(bo);
> + mtx_unlock(&glob->lru_lock);
> +
> + ttm_bo_list_ref_sub(bo, put_count, true);
> +
> + return ret;
> +}
> +
> +void ttm_bo_unreserve_locked(struct ttm_buffer_object *bo)
> +{
> + ttm_bo_add_to_lru(bo);
> + atomic_set(&bo->reserved, 0);
> + wakeup(bo);
> +}
> +
> +void ttm_bo_unreserve(struct ttm_buffer_object *bo)
> +{
> + struct ttm_bo_global *glob = bo->glob;
> +
> + mtx_lock(&glob->lru_lock);
> + ttm_bo_unreserve_locked(bo);
> + mtx_unlock(&glob->lru_lock);
> +}
> +
> +/*
> + * Call bo->mutex locked.
> + */
> +static int ttm_bo_add_ttm(struct ttm_buffer_object *bo, bool zero_alloc)
> +{
> + struct ttm_bo_device *bdev = bo->bdev;
> + struct ttm_bo_global *glob = bo->glob;
> + int ret = 0;
> + uint32_t page_flags = 0;
> +
> + TTM_ASSERT_LOCKED(&bo->mutex);
> + bo->ttm = NULL;
> +
> + if (bdev->need_dma32)
> + page_flags |= TTM_PAGE_FLAG_DMA32;
> +
> + switch (bo->type) {
> + case ttm_bo_type_device:
> + if (zero_alloc)
> + page_flags |= TTM_PAGE_FLAG_ZERO_ALLOC;
> + case ttm_bo_type_kernel:
> + bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages <<
> PAGE_SHIFT,
> + page_flags, glob->dummy_read_page);
> + if (unlikely(bo->ttm == NULL))
> + ret = -ENOMEM;
> + break;
> + case ttm_bo_type_sg:
> + bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages <<
> PAGE_SHIFT,
> + page_flags | TTM_PAGE_FLAG_SG,
> + glob->dummy_read_page);
> + if (unlikely(bo->ttm == NULL)) {
> + ret = -ENOMEM;
> + break;
> + }
> + bo->ttm->sg = bo->sg;
> + break;
> + default:
> + printf("[TTM] Illegal buffer object type\n");
> + ret = -EINVAL;
> + break;
> + }
> +
> + return ret;
> +}
> +
> +static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
> + struct ttm_mem_reg *mem,
> + bool evict, bool interruptible,
> + bool no_wait_gpu)
> +{
> + struct ttm_bo_device *bdev = bo->bdev;
> + bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem);
> + bool new_is_pci = ttm_mem_reg_is_pci(bdev, mem);
> + struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type];
> + struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type];
> + int ret = 0;
> +
> + if (old_is_pci || new_is_pci ||
> + ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0))
> {
> + ret = ttm_mem_io_lock(old_man, true);
> + if (unlikely(ret != 0))
> + goto out_err;
> + ttm_bo_unmap_virtual_locked(bo);
> + ttm_mem_io_unlock(old_man);
> + }
> +
> + /*
> + * Create and bind a ttm if required.
> + */
> +
> + if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) {
> + if (bo->ttm == NULL) {
> + bool zero = !(old_man->flags & TTM_MEMTYPE_FLAG_FIXED);
> + ret = ttm_bo_add_ttm(bo, zero);
> + if (ret)
> + goto out_err;
> + }
> +
> + ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement);
> + if (ret)
> + goto out_err;
> +
> + if (mem->mem_type != TTM_PL_SYSTEM) {
> + ret = ttm_tt_bind(bo->ttm, mem);
> + if (ret)
> + goto out_err;
> + }
> +
> + if (bo->mem.mem_type == TTM_PL_SYSTEM) {
> + if (bdev->driver->move_notify)
> + bdev->driver->move_notify(bo, mem);
> + bo->mem = *mem;
> + mem->mm_node = NULL;
> + goto moved;
> + }
> + }
> +
> + if (bdev->driver->move_notify)
> + bdev->driver->move_notify(bo, mem);
> +
> + if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) &&
> + !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED))
> + ret = ttm_bo_move_ttm(bo, evict, no_wait_gpu, mem);
> + else if (bdev->driver->move)
> + ret = bdev->driver->move(bo, evict, interruptible,
> + no_wait_gpu, mem);
> + else
> + ret = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, mem);
> +
> + if (ret) {
> + if (bdev->driver->move_notify) {
> + struct ttm_mem_reg tmp_mem = *mem;
> + *mem = bo->mem;
> + bo->mem = tmp_mem;
> + bdev->driver->move_notify(bo, mem);
> + bo->mem = *mem;
> + }
> +
> + goto out_err;
> + }
> +
> +moved:
> + if (bo->evicted) {
> + ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement);
> + if (ret)
> + printf("[TTM] Can not flush read caches\n");
> + bo->evicted = false;
> + }
> +
> + if (bo->mem.mm_node) {
> + bo->offset = (bo->mem.start << PAGE_SHIFT) +
> + bdev->man[bo->mem.mem_type].gpu_offset;
> + bo->cur_placement = bo->mem.placement;
> + } else
> + bo->offset = 0;
> +
> + return 0;
> +
> +out_err:
> + new_man = &bdev->man[bo->mem.mem_type];
> + if ((new_man->flags & TTM_MEMTYPE_FLAG_FIXED) && bo->ttm) {
> + ttm_tt_unbind(bo->ttm);
> + ttm_tt_destroy(bo->ttm);
> + bo->ttm = NULL;
> + }
> +
> + return ret;
> +}
> +
> +/**
> + * Call bo::reserved.
> + * Will release GPU memory type usage on destruction.
> + * This is the place to put in driver specific hooks to release
> + * driver private resources.
> + * Will release the bo::reserved lock.
> + */
> +
> +static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo)
> +{
> + if (bo->bdev->driver->move_notify)
> + bo->bdev->driver->move_notify(bo, NULL);
> +
> + if (bo->ttm) {
> + ttm_tt_unbind(bo->ttm);
> + ttm_tt_destroy(bo->ttm);
> + bo->ttm = NULL;
> + }
> + ttm_bo_mem_put(bo, &bo->mem);
> +
> + atomic_set(&bo->reserved, 0);
> + wakeup(&bo);
> +
> + /*
> + * Since the final reference to this bo may not be dropped by
> + * the current task we have to put a memory barrier here to make
> + * sure the changes done in this function are always visible.
> + *
> + * This function only needs protection against the final kref_put.
> + */
> + mb();
> +}
> +
> +static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo)
> +{
> + struct ttm_bo_device *bdev = bo->bdev;
> + struct ttm_bo_global *glob = bo->glob;
> + struct ttm_bo_driver *driver = bdev->driver;
> + void *sync_obj = NULL;
> + int put_count;
> + int ret;
> +
> + mtx_lock(&glob->lru_lock);
> + ret = ttm_bo_reserve_locked(bo, false, true, false, 0);
> +
> + mtx_lock(&bdev->fence_lock);
> + (void) ttm_bo_wait(bo, false, false, true);
> + if (!ret && !bo->sync_obj) {
> + mtx_unlock(&bdev->fence_lock);
> + put_count = ttm_bo_del_from_lru(bo);
> +
> + mtx_unlock(&glob->lru_lock);
> + ttm_bo_cleanup_memtype_use(bo);
> +
> + ttm_bo_list_ref_sub(bo, put_count, true);
> +
> + return;
> + }
> + if (bo->sync_obj)
> + sync_obj = driver->sync_obj_ref(bo->sync_obj);
> + mtx_unlock(&bdev->fence_lock);
> +
> + if (!ret) {
> + atomic_set(&bo->reserved, 0);
> + wakeup(bo);
> + }
> +
> + refcount_acquire(&bo->list_kref);
> + list_add_tail(&bo->ddestroy, &bdev->ddestroy);
> + mtx_unlock(&glob->lru_lock);
> +
> + if (sync_obj) {
> + driver->sync_obj_flush(sync_obj);
> + driver->sync_obj_unref(&sync_obj);
> + }
> + taskqueue_enqueue_timeout(taskqueue_thread, &bdev->wq,
> + ((hz / 100) < 1) ? 1 : hz / 100);
> +}
> +
> +/**
> + * function ttm_bo_cleanup_refs_and_unlock
> + * If bo idle, remove from delayed- and lru lists, and unref.
> + * If not idle, do nothing.
> + *
> + * Must be called with lru_lock and reservation held, this function
> + * will drop both before returning.
> + *
> + * @interruptible Any sleeps should occur interruptibly.
> + * @no_wait_gpu Never wait for gpu. Return -EBUSY instead.
> + */
> +
> +static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo,
> + bool interruptible,
> + bool no_wait_gpu)
> +{
> + struct ttm_bo_device *bdev = bo->bdev;
> + struct ttm_bo_driver *driver = bdev->driver;
> + struct ttm_bo_global *glob = bo->glob;
> + int put_count;
> + int ret;
> +
> + mtx_lock(&bdev->fence_lock);
> + ret = ttm_bo_wait(bo, false, false, true);
> +
> + if (ret && !no_wait_gpu) {
> + void *sync_obj;
> +
> + /*
> + * Take a reference to the fence and unreserve,
> + * at this point the buffer should be dead, so
> + * no new sync objects can be attached.
> + */
> + sync_obj = driver->sync_obj_ref(bo->sync_obj);
> + mtx_unlock(&bdev->fence_lock);
> +
> + atomic_set(&bo->reserved, 0);
> + wakeup(bo);
> + mtx_unlock(&glob->lru_lock);
> +
> + ret = driver->sync_obj_wait(sync_obj, false, interruptible);
> + driver->sync_obj_unref(&sync_obj);
> + if (ret)
> + return ret;
> +
> + /*
> + * remove sync_obj with ttm_bo_wait, the wait should be
> + * finished, and no new wait object should have been added.
> + */
> + mtx_lock(&bdev->fence_lock);
> + ret = ttm_bo_wait(bo, false, false, true);
> + mtx_unlock(&bdev->fence_lock);
> + if (ret)
> + return ret;
> +
> + mtx_lock(&glob->lru_lock);
> + ret = ttm_bo_reserve_locked(bo, false, true, false, 0);
> +
> + /*
> + * We raced, and lost, someone else holds the reservation now,
> + * and is probably busy in ttm_bo_cleanup_memtype_use.
> + *
> + * Even if it's not the case, because we finished waiting any
> + * delayed destruction would succeed, so just return success
> + * here.
> + */
> + if (ret) {
> + mtx_unlock(&glob->lru_lock);
> + return 0;
> + }
> + } else
> + mtx_unlock(&bdev->fence_lock);
> +
> + if (ret || unlikely(list_empty(&bo->ddestroy))) {
> + atomic_set(&bo->reserved, 0);
> + wakeup(bo);
> + mtx_unlock(&glob->lru_lock);
> + return ret;
> + }
> +
> + put_count = ttm_bo_del_from_lru(bo);
> + list_del_init(&bo->ddestroy);
> + ++put_count;
> +
> + mtx_unlock(&glob->lru_lock);
> + ttm_bo_cleanup_memtype_use(bo);
> +
> + ttm_bo_list_ref_sub(bo, put_count, true);
> +
> + return 0;
> +}
> +
> +/**
> + * Traverse the delayed list, and call ttm_bo_cleanup_refs on all
> + * encountered buffers.
> + */
> +
> +static int ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool
> remove_all)
> +{
> + struct ttm_bo_global *glob = bdev->glob;
> + struct ttm_buffer_object *entry = NULL;
> + int ret = 0;
> +
> + mtx_lock(&glob->lru_lock);
> + if (list_empty(&bdev->ddestroy))
> + goto out_unlock;
> +
> + entry = list_first_entry(&bdev->ddestroy,
> + struct ttm_buffer_object, ddestroy);
> + refcount_acquire(&entry->list_kref);
> +
> + for (;;) {
> + struct ttm_buffer_object *nentry = NULL;
> +
> + if (entry->ddestroy.next != &bdev->ddestroy) {
> + nentry = list_first_entry(&entry->ddestroy,
> + struct ttm_buffer_object, ddestroy);
> + refcount_acquire(&nentry->list_kref);
> + }
> +
> + ret = ttm_bo_reserve_locked(entry, false, !remove_all, false, 0);
> + if (!ret)
> + ret = ttm_bo_cleanup_refs_and_unlock(entry, false,
> + !remove_all);
> + else
> + mtx_unlock(&glob->lru_lock);
> +
> + if (refcount_release(&entry->list_kref))
> + ttm_bo_release_list(entry);
> + entry = nentry;
> +
> + if (ret || !entry)
> + goto out;
> +
> + mtx_lock(&glob->lru_lock);
> + if (list_empty(&entry->ddestroy))
> + break;
> + }
> +
> +out_unlock:
> + mtx_unlock(&glob->lru_lock);
> +out:
> + if (entry && refcount_release(&entry->list_kref))
> + ttm_bo_release_list(entry);
> + return ret;
> +}
> +
> +static void ttm_bo_delayed_workqueue(void *arg, int pending __unused)
> +{
> + struct ttm_bo_device *bdev = arg;
> +
> + if (ttm_bo_delayed_delete(bdev, false)) {
> + taskqueue_enqueue_timeout(taskqueue_thread, &bdev->wq,
> + ((hz / 100) < 1) ? 1 : hz / 100);
> + }
> +}
> +
> +static void ttm_bo_release(struct ttm_buffer_object *bo)
> +{
> + struct ttm_bo_device *bdev = bo->bdev;
> + struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];
> +
> + rw_wlock(&bdev->vm_lock);
> + if (likely(bo->vm_node != NULL)) {
> + RB_REMOVE(ttm_bo_device_buffer_objects,
> + &bdev->addr_space_rb, bo);
> + drm_mm_put_block(bo->vm_node);
> + bo->vm_node = NULL;
> + }
> + rw_wunlock(&bdev->vm_lock);
> + ttm_mem_io_lock(man, false);
> + ttm_mem_io_free_vm(bo);
> + ttm_mem_io_unlock(man);
> + ttm_bo_cleanup_refs_or_queue(bo);
> + if (refcount_release(&bo->list_kref))
> + ttm_bo_release_list(bo);
> +}
> +
> +void ttm_bo_unref(struct ttm_buffer_object **p_bo)
> +{
> + struct ttm_buffer_object *bo = *p_bo;
> +
> + *p_bo = NULL;
> + if (refcount_release(&bo->kref))
> + ttm_bo_release(bo);
> +}
> +
> +int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev)
> +{
> + int pending;
> +
> + taskqueue_cancel_timeout(taskqueue_thread, &bdev->wq, &pending);
> + if (pending)
> + taskqueue_drain_timeout(taskqueue_thread, &bdev->wq);
> + return (pending);
> +}
> +
> +void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int
> resched)
> +{
> + if (resched) {
> + taskqueue_enqueue_timeout(taskqueue_thread, &bdev->wq,
> + ((hz / 100) < 1) ? 1 : hz / 100);
> + }
> +}
> +
> +static int ttm_bo_evict(struct ttm_buffer_object *bo, bool
> interruptible,
> + bool no_wait_gpu)
> +{
> + struct ttm_bo_device *bdev = bo->bdev;
> + struct ttm_mem_reg evict_mem;
> + struct ttm_placement placement;
> + int ret = 0;
> +
> + mtx_lock(&bdev->fence_lock);
> + ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu);
> + mtx_unlock(&bdev->fence_lock);
> +
> + if (unlikely(ret != 0)) {
> + if (ret != -ERESTART) {
> + printf("[TTM] Failed to expire sync object before buffer
> eviction\n");
> + }
> + goto out;
> + }
> +
> + MPASS(ttm_bo_is_reserved(bo));
> +
> + evict_mem = bo->mem;
> + evict_mem.mm_node = NULL;
> + evict_mem.bus.io_reserved_vm = false;
> + evict_mem.bus.io_reserved_count = 0;
> +
> + placement.fpfn = 0;
>
> *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
> _______________________________________________
> svn-src-all at freebsd.org mailing list
> http://lists.freebsd.org/mailman/listinfo/svn-src-all
> To unsubscribe, send any mail to "svn-src-all-unsubscribe at freebsd.org"
More information about the svn-src-all
mailing list