kolibrios/drivers/video/drm/i915/sna/kgem.c

/*
 * Copyright (c) 2011 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 * Authors:
 *    Chris Wilson <chris@chris-wilson.co.uk>
 *
 */


#ifdef HAVE_CONFIG_H
#include "config.h"
#endif

#include <drmP.h>
#include <drm.h>
#include "i915_drm.h"
#include "i915_drv.h"
#include "intel_drv.h"

#include <linux/kernel.h>
#include "../bitmap.h"
#include "sna.h"
//#include "sna_reg.h"
//#include <time.h>
//#include <errno.h>

#define NO_CACHE 1

#define list_is_empty list_empty
#define list_init     INIT_LIST_HEAD

extern struct drm_device *main_device;

static struct kgem_bo *
search_linear_cache(struct kgem *kgem, unsigned int num_pages,
                    unsigned flags);

#define INT16_MAX              (32767)

#define PAGE_ALIGN(x) ALIGN(x, PAGE_SIZE)
#define NUM_PAGES(x) (((x) + PAGE_SIZE-1) / PAGE_SIZE)

#define MAX_GTT_VMA_CACHE 512
#define MAX_CPU_VMA_CACHE INT16_MAX
#define MAP_PRESERVE_TIME 10

#define CPU_MAP(ptr) ((void*)((uintptr_t)(ptr) & ~1))
#define MAKE_CPU_MAP(ptr) ((void*)((uintptr_t)(ptr) | 1))

struct kgem_partial_bo {
	struct kgem_bo base;
	void *mem;
	uint32_t used;
	uint32_t need_io : 1;
	uint32_t write : 2;
	uint32_t mmapped : 1;
};

static struct kgem_bo *__kgem_freed_bo;
static struct drm_i915_gem_exec_object2 _kgem_dummy_exec;

static inline int bytes(struct kgem_bo *bo)
{
	return kgem_bo_size(bo);
}

#define bucket(B) (B)->size.pages.bucket
#define num_pages(B) (B)->size.pages.count

static void kgem_sna_reset(struct kgem *kgem)
{
    struct sna *sna = container_of(kgem, struct sna, kgem);

    sna->render.reset(sna);
    sna->blt_state.fill_bo = 0;
}

static void kgem_sna_flush(struct kgem *kgem)
{
	struct sna *sna = container_of(kgem, struct sna, kgem);

	sna->render.flush(sna);

	if (sna->render.solid_cache.dirty)
		sna_render_flush_solid(sna);
}

static int __gem_write(int fd, uint32_t handle,
               int offset, int length,
               const void *src)
{
    DBG(("%s(handle=%x, offset=%d, len=%d)\n", __FUNCTION__,
         handle, offset, length));

    write_gem_object(handle, offset, length, src);
    return 0;
}


static int gem_write(int fd, uint32_t handle,
             int offset, int length,
             const void *src)
{
    u32 _offset;
    u32 _size;
    u8  *data_ptr;

    DBG(("%s(handle=%x, offset=%d, len=%d)\n", __FUNCTION__,
         handle, offset, length));

    /* align the transfer to cachelines; fortuitously this is safe! */
    if ((offset | length) & 63) {
        _offset = offset & ~63;
        _size = ALIGN(offset+length, 64) - _offset;
        data_ptr = (u8*)src + _offset - offset;
    } else {
        _offset = offset;
        _size = length;
        data_ptr = (u8*)src;
    }

    write_gem_object(handle, _offset, _size, data_ptr);
    return 0;
}

static void kgem_bo_retire(struct kgem *kgem, struct kgem_bo *bo)
{
    DBG(("%s: handle=%x, domain=%d\n",
	     __FUNCTION__, bo->handle, bo->domain));
	assert(!kgem_busy(kgem, bo->handle));

	if (bo->domain == DOMAIN_GPU)
		kgem_retire(kgem);

	if (bo->exec == NULL) {
        DBG(("%s: retiring bo handle=%x (needed flush? %d), rq? %d\n",
		     __FUNCTION__, bo->handle, bo->needs_flush, bo->rq != NULL));
		bo->rq = NULL;
		list_del(&bo->request);
		bo->needs_flush = bo->flush;
	}
}

Bool kgem_bo_write(struct kgem *kgem, struct kgem_bo *bo,
           const void *data, int length)
{
    assert(bo->refcnt);
    assert(!bo->purged);
    assert(!kgem_busy(kgem, bo->handle));

    assert(length <= bytes(bo));
    if (gem_write(kgem->fd, bo->handle, 0, length, data))
        return FALSE;

    DBG(("%s: flush=%d, domain=%d\n", __FUNCTION__, bo->flush, bo->domain));
    kgem_bo_retire(kgem, bo);
    bo->domain = DOMAIN_NONE;
    return TRUE;
}

static uint32_t gem_create(int fd, int num_pages)
{
    struct drm_i915_gem_object *obj;
    int       ret;

    /* Allocate the new object */
    obj = i915_gem_alloc_object(main_device,
                                PAGE_SIZE * num_pages);
    if (obj == NULL)
        goto err1;

    ret = i915_gem_object_pin(obj, 4096, true);
    if (ret)
        goto err2;

    return (uint32_t)obj;

err2:
    drm_gem_object_unreference(&obj->base);
err1:
    return 0;
}

static bool
kgem_bo_set_purgeable(struct kgem *kgem, struct kgem_bo *bo)
{
    return true;
}


static bool
kgem_bo_clear_purgeable(struct kgem *kgem, struct kgem_bo *bo)
{
    return true;
}

static void gem_close(int fd, uint32_t handle)
{
    destroy_gem_object(handle);
}


/*
constant inline static unsigned long __fls(unsigned long word)
{
    asm("bsr %1,%0"
        : "=r" (word)
        : "rm" (word));
    return word;
}
*/

constant inline static int cache_bucket(int num_pages)
{
    return __fls(num_pages);
}

static struct kgem_bo *__kgem_bo_init(struct kgem_bo *bo,
                      int handle, int num_pages)
{
    assert(num_pages);
    memset(bo, 0, sizeof(*bo));

    bo->refcnt = 1;
    bo->handle = handle;
    num_pages(bo) = num_pages;
    bucket(bo) = cache_bucket(num_pages);
    bo->reusable = true;
    bo->domain = DOMAIN_CPU;
    list_init(&bo->request);
    list_init(&bo->list);
    list_init(&bo->vma);

    return bo;
}

static struct kgem_bo *__kgem_bo_alloc(int handle, int num_pages)
{
    struct kgem_bo *bo;

    if (__kgem_freed_bo) {
        bo = __kgem_freed_bo;
        __kgem_freed_bo = *(struct kgem_bo **)bo;
    } else {
        bo = malloc(sizeof(*bo));
        if (bo == NULL)
            return NULL;
    }

    return __kgem_bo_init(bo, handle, num_pages);
}

static struct kgem_request _kgem_static_request;

static struct kgem_request *__kgem_request_alloc(void)
{
    struct kgem_request *rq;

    rq = malloc(sizeof(*rq));
    if (rq == NULL)
        rq = &_kgem_static_request;

    list_init(&rq->buffers);

    return rq;
}

static struct list_head *inactive(struct kgem *kgem, int num_pages)
{
    return &kgem->inactive[cache_bucket(num_pages)];
}

static struct list_head *active(struct kgem *kgem, int num_pages, int tiling)
{
    return &kgem->active[cache_bucket(num_pages)][tiling];
}



void kgem_init(struct kgem *kgem, int gen)
{
    struct drm_i915_gem_get_aperture aperture;
    struct drm_i915_gem_object     *obj;

    size_t totalram;
    unsigned int i, j;
    int ret;

    memset(kgem, 0, sizeof(*kgem));

    kgem->gen = gen;
    kgem->wedged  = 0;
//    kgem->wedged |= DBG_NO_HW;

    obj = i915_gem_alloc_object(main_device, 4096*4);
    if (obj == NULL)
        goto err2;

    ret = i915_gem_object_pin(obj, 4096, true);
    if (ret)
        goto err3;

    kgem->batch_ptr = drm_intel_bo_map(obj, true);
    kgem->batch = kgem->batch_ptr;
    kgem->batch_idx = 0;
    kgem->batch_obj = obj;

    kgem->max_batch_size = 1024; //ARRAY_SIZE(kgem->batch);

    kgem->half_cpu_cache_pages = (2048*1024) >> 13;

    list_init(&kgem->partial);
    list_init(&kgem->requests);
    list_init(&kgem->flushing);
    list_init(&kgem->large);
    for (i = 0; i < ARRAY_SIZE(kgem->inactive); i++)
        list_init(&kgem->inactive[i]);
    for (i = 0; i < ARRAY_SIZE(kgem->active); i++) {
        for (j = 0; j < ARRAY_SIZE(kgem->active[i]); j++)
            list_init(&kgem->active[i][j]);
    }
    for (i = 0; i < ARRAY_SIZE(kgem->vma); i++) {
        for (j = 0; j < ARRAY_SIZE(kgem->vma[i].inactive); j++)
            list_init(&kgem->vma[i].inactive[j]);
    }
    kgem->vma[MAP_GTT].count = -MAX_GTT_VMA_CACHE;
    kgem->vma[MAP_CPU].count = -MAX_CPU_VMA_CACHE;

    kgem->next_request = __kgem_request_alloc();

//#if defined(USE_VMAP) && defined(I915_PARAM_HAS_VMAP)
//    if (!DBG_NO_VMAP)
//        kgem->has_vmap = gem_param(kgem, I915_PARAM_HAS_VMAP) > 0;
//#endif
//    DBG(("%s: using vmap=%d\n", __FUNCTION__, kgem->has_vmap));

    if (gen < 40) {
//        if (!DBG_NO_RELAXED_FENCING) {
//            kgem->has_relaxed_fencing =
//                gem_param(kgem, I915_PARAM_HAS_RELAXED_FENCING) > 0;
//        }
    } else
        kgem->has_relaxed_fencing = 1;
    DBG(("%s: has relaxed fencing? %d\n", __FUNCTION__,
         kgem->has_relaxed_fencing));

    kgem->has_llc = (gen >= 60)?true:false;
    kgem->has_cpu_bo = kgem->has_llc;
    DBG(("%s: cpu bo enabled %d: llc? %d\n", __FUNCTION__,
         kgem->has_cpu_bo, kgem->has_llc));

    kgem->has_semaphores = false;
//    if (gen >= 60 && semaphores_enabled())
//        kgem->has_semaphores = true;
//    DBG(("%s: semaphores enabled? %d\n", __FUNCTION__,
//         kgem->has_semaphores));

    VG_CLEAR(aperture);
    aperture.aper_size = 64*1024*1024;
    i915_gem_get_aperture_ioctl(main_device, &aperture, NULL);
    kgem->aperture_total = aperture.aper_size;
    kgem->aperture_high = aperture.aper_size * 3/4;
    kgem->aperture_low = aperture.aper_size * 1/3;
    DBG(("%s: aperture low=%d [%d], high=%d [%d]\n", __FUNCTION__,
         kgem->aperture_low, kgem->aperture_low / (1024*1024),
         kgem->aperture_high, kgem->aperture_high / (1024*1024)));

    kgem->aperture_mappable = aperture.aper_size;
    DBG(("%s: aperture mappable=%d [%d MiB]\n", __FUNCTION__,
         kgem->aperture_mappable, kgem->aperture_mappable / (1024*1024)));

    kgem->partial_buffer_size = 64 * 1024;
    while (kgem->partial_buffer_size < kgem->aperture_mappable >> 10)
        kgem->partial_buffer_size *= 2;
    DBG(("%s: partial buffer size=%d [%d KiB]\n", __FUNCTION__,
         kgem->partial_buffer_size, kgem->partial_buffer_size / 1024));

    kgem->min_alignment = 4;
    if (gen < 60)
        /* XXX workaround an issue where we appear to fail to
         * disable dual-stream mode */
        kgem->min_alignment = 64;

    kgem->max_object_size = 2 * kgem->aperture_total / 3;
    kgem->max_cpu_size = kgem->max_object_size;
    kgem->max_gpu_size = kgem->max_object_size;
    if (!kgem->has_llc)
        kgem->max_gpu_size = MAX_CACHE_SIZE;
    if (gen < 40) {
        /* If we have to use fences for blitting, we have to make
         * sure we can fit them into the aperture.
         */
        kgem->max_gpu_size = kgem->aperture_mappable / 2;
        if (kgem->max_gpu_size > kgem->aperture_low)
            kgem->max_gpu_size = kgem->aperture_low;
    }
    if (kgem->max_gpu_size > kgem->max_cpu_size)
        kgem->max_gpu_size = kgem->max_cpu_size;

    kgem->max_upload_tile_size = kgem->aperture_mappable / 2;
    if (kgem->max_upload_tile_size > kgem->max_gpu_size / 2)
        kgem->max_upload_tile_size = kgem->max_gpu_size / 2;

    kgem->max_copy_tile_size = (MAX_CACHE_SIZE + 1)/2;
    if (kgem->max_copy_tile_size > kgem->max_gpu_size / 2)
        kgem->max_copy_tile_size = kgem->max_gpu_size / 2;

    totalram = 1024*1024; //total_ram_size();
    if (totalram == 0) {
        DBG(("%s: total ram size unknown, assuming maximum of total aperture\n",
             __FUNCTION__));
        totalram = kgem->aperture_total;
    }
    if (kgem->max_object_size > totalram / 2)
        kgem->max_object_size = totalram / 2;
    if (kgem->max_cpu_size > totalram / 2)
        kgem->max_cpu_size = totalram / 2;
    if (kgem->max_gpu_size > totalram / 4)
        kgem->max_gpu_size = totalram / 4;

    kgem->large_object_size = MAX_CACHE_SIZE;
    if (kgem->large_object_size > kgem->max_gpu_size)
        kgem->large_object_size = kgem->max_gpu_size;

    DBG(("%s: large object thresold=%d\n",
         __FUNCTION__, kgem->large_object_size));
    DBG(("%s: max object size (gpu=%d, cpu=%d, tile upload=%d, copy=%d)\n",
         __FUNCTION__,
         kgem->max_gpu_size, kgem->max_cpu_size,
         kgem->max_upload_tile_size, kgem->max_copy_tile_size));

    /* Convert the aperture thresholds to pages */
    kgem->aperture_low /= PAGE_SIZE;
    kgem->aperture_high /= PAGE_SIZE;

//    kgem->fence_max = gem_param(kgem, I915_PARAM_NUM_FENCES_AVAIL) - 2;
//    if ((int)kgem->fence_max < 0)
        kgem->fence_max = 5; /* minimum safe value for all hw */
    DBG(("%s: max fences=%d\n", __FUNCTION__, kgem->fence_max));
err3:
err2:
    return;
}

static struct drm_i915_gem_exec_object2 *
kgem_add_handle(struct kgem *kgem, struct kgem_bo *bo)
{
	struct drm_i915_gem_exec_object2 *exec;

	DBG(("%s: handle=%d, index=%d\n",
	     __FUNCTION__, bo->handle, kgem->nexec));

	assert(kgem->nexec < ARRAY_SIZE(kgem->exec));
	exec = memset(&kgem->exec[kgem->nexec++], 0, sizeof(*exec));
	exec->handle = bo->handle;
	exec->offset = bo->presumed_offset;

	kgem->aperture += num_pages(bo);

	return exec;
}

void _kgem_add_bo(struct kgem *kgem, struct kgem_bo *bo)
{
	bo->exec = kgem_add_handle(kgem, bo);
	bo->rq = kgem->next_request;

	list_move(&bo->request, &kgem->next_request->buffers);

	/* XXX is it worth working around gcc here? */
	kgem->flush |= bo->flush;
	kgem->sync |= bo->sync;
	kgem->scanout |= bo->scanout;
}

static uint32_t kgem_end_batch(struct kgem *kgem)
{
//	kgem->context_switch(kgem, KGEM_NONE);

	kgem->batch[kgem->nbatch++] = MI_BATCH_BUFFER_END;
	if (kgem->nbatch & 1)
		kgem->batch[kgem->nbatch++] = MI_NOOP;

	return kgem->nbatch;
}

static void kgem_fixup_self_relocs(struct kgem *kgem, struct kgem_bo *bo)
{
	int n;

    for (n = 0; n < kgem->nreloc; n++)
    {
        if (kgem->reloc[n].target_handle == 0)
        {
			kgem->reloc[n].target_handle = bo->handle;
			kgem->reloc[n].presumed_offset = bo->presumed_offset;
			kgem->batch[kgem->reloc[n].offset/sizeof(kgem->batch[0])] =
				kgem->reloc[n].delta + bo->presumed_offset;

            dbgprintf("fixup reloc %d pos %d handle %d delta %x \n",
                       n, kgem->reloc[n].offset/sizeof(kgem->batch[0]),
                       bo->handle, kgem->reloc[n].delta);
		}
	}
}

static void kgem_bo_binding_free(struct kgem *kgem, struct kgem_bo *bo)
{
	struct kgem_bo_binding *b;

	b = bo->binding.next;
	while (b) {
		struct kgem_bo_binding *next = b->next;
		free (b);
		b = next;
	}
}

static void kgem_bo_release_map(struct kgem *kgem, struct kgem_bo *bo)
{
	int type = IS_CPU_MAP(bo->map);

	DBG(("%s: releasing %s vma for handle=%d, count=%d\n",
	     __FUNCTION__, type ? "CPU" : "GTT",
	     bo->handle, kgem->vma[type].count));

	VG(if (type) VALGRIND_FREELIKE_BLOCK(CPU_MAP(bo->map), 0));
//	munmap(CPU_MAP(bo->map), bytes(bo));
	bo->map = NULL;

	if (!list_is_empty(&bo->vma)) {
		list_del(&bo->vma);
		kgem->vma[type].count--;
	}
}

static void kgem_bo_free(struct kgem *kgem, struct kgem_bo *bo)
{
    DBG(("%s: handle=%d\n", __FUNCTION__, bo->handle));
    assert(bo->refcnt == 0);
    assert(bo->exec == NULL);

    kgem_bo_binding_free(kgem, bo);

    if (bo->map)
        kgem_bo_release_map(kgem, bo);
    assert(list_is_empty(&bo->vma));

    list_del(&bo->list);
    list_del(&bo->request);
    gem_close(kgem->fd, bo->handle);

    if (!bo->io) {
        *(struct kgem_bo **)bo = __kgem_freed_bo;
        __kgem_freed_bo = bo;
    } else
        free(bo);
}

inline static void kgem_bo_move_to_inactive(struct kgem *kgem,
                        struct kgem_bo *bo)
{
    assert(!kgem_busy(kgem, bo->handle));
    assert(!bo->proxy);
    assert(!bo->io);
    assert(!bo->needs_flush);
    assert(bo->rq == NULL);
    assert(bo->domain != DOMAIN_GPU);

    if (bucket(bo) >= NUM_CACHE_BUCKETS) {
        kgem_bo_free(kgem, bo);
        return;
    }

    list_move(&bo->list, &kgem->inactive[bucket(bo)]);
    if (bo->map) {
        int type = IS_CPU_MAP(bo->map);
        if (bucket(bo) >= NUM_CACHE_BUCKETS ||
            (!type && !kgem_bo_is_mappable(kgem, bo))) {
            list_del(&bo->vma);
//            munmap(CPU_MAP(bo->map), bytes(bo));
            bo->map = NULL;
        }
        if (bo->map) {
            list_move(&bo->vma, &kgem->vma[type].inactive[bucket(bo)]);
            kgem->vma[type].count++;
        }
    }

    kgem->need_expire = true;
}

inline static void kgem_bo_remove_from_inactive(struct kgem *kgem,
                        struct kgem_bo *bo)
{
    list_del(&bo->list);
    assert(bo->rq == NULL);
    if (bo->map) {
        assert(!list_is_empty(&bo->vma));
        list_del(&bo->vma);
        kgem->vma[IS_CPU_MAP(bo->map)].count--;
    }
}

inline static void kgem_bo_remove_from_active(struct kgem *kgem,
                          struct kgem_bo *bo)
{
    list_del(&bo->list);
    if (bo->rq == &_kgem_static_request)
        list_del(&bo->request);
    assert(list_is_empty(&bo->vma));
}

static void __kgem_bo_destroy(struct kgem *kgem, struct kgem_bo *bo)
{
    DBG(("%s: handle=%d\n", __FUNCTION__, bo->handle));

    assert(list_is_empty(&bo->list));
    assert(bo->refcnt == 0);

    bo->binding.offset = 0;

    if (NO_CACHE)
        goto destroy;

    if (bo->io) {
        struct kgem_bo *base;

        base = malloc(sizeof(*base));
        if (base) {
            DBG(("%s: transferring io handle=%d to bo\n",
                 __FUNCTION__, bo->handle));
            /* transfer the handle to a minimum bo */
            memcpy(base, bo, sizeof (*base));
            base->reusable = true;
            base->io = false;
            list_init(&base->list);
            list_replace(&bo->request, &base->request);
            list_replace(&bo->vma, &base->vma);
            free(bo);
            bo = base;
        }
    }

    if (!bo->reusable) {
        DBG(("%s: handle=%d, not reusable\n",
             __FUNCTION__, bo->handle));
        goto destroy;
    }

    if (!kgem->has_llc && IS_CPU_MAP(bo->map) && bo->domain != DOMAIN_CPU)
        kgem_bo_release_map(kgem, bo);

    assert(list_is_empty(&bo->vma));
    assert(list_is_empty(&bo->list));
    assert(bo->vmap == false && bo->sync == false);
    assert(bo->io == false);

    bo->scanout = bo->flush = false;
    if (bo->rq) {
        struct list *cache;

        DBG(("%s: handle=%d -> active\n", __FUNCTION__, bo->handle));
        if (bucket(bo) < NUM_CACHE_BUCKETS)
            cache = &kgem->active[bucket(bo)][bo->tiling];
        else
            cache = &kgem->large;
        list_add(&bo->list, cache);
        return;
    }

    assert(bo->exec == NULL);
    assert(list_is_empty(&bo->request));
/*
    if (bo->needs_flush) {
        if ((bo->needs_flush = kgem_busy(kgem, bo->handle))) {
            struct list *cache;

            DBG(("%s: handle=%d -> flushing\n",
                 __FUNCTION__, bo->handle));

            list_add(&bo->request, &kgem->flushing);
            if (bucket(bo) < NUM_CACHE_BUCKETS)
                cache = &kgem->active[bucket(bo)][bo->tiling];
            else
                cache = &kgem->large;
            list_add(&bo->list, cache);
            bo->rq = &_kgem_static_request;
            return;
        }

        bo->domain = DOMAIN_NONE;
    }
*/
    if (!IS_CPU_MAP(bo->map)) {
        if (!kgem_bo_set_purgeable(kgem, bo))
            goto destroy;

        if (!kgem->has_llc && bo->domain == DOMAIN_CPU)
            goto destroy;

        DBG(("%s: handle=%d, purged\n",
             __FUNCTION__, bo->handle));
    }

    DBG(("%s: handle=%d -> inactive\n", __FUNCTION__, bo->handle));
    kgem_bo_move_to_inactive(kgem, bo);
    return;

destroy:
    if (!bo->exec)
        kgem_bo_free(kgem, bo);
}





bool kgem_retire(struct kgem *kgem)
{
    struct kgem_bo *bo, *next;
    bool retired = false;

    DBG(("%s\n", __FUNCTION__));

    list_for_each_entry_safe(bo, next, &kgem->flushing, request) {
        assert(bo->refcnt == 0);
        assert(bo->rq == &_kgem_static_request);
        assert(bo->exec == NULL);

//        if (kgem_busy(kgem, bo->handle))
//            break;

        DBG(("%s: moving %d from flush to inactive\n",
             __FUNCTION__, bo->handle));
        if (kgem_bo_set_purgeable(kgem, bo)) {
            bo->needs_flush = false;
            bo->domain = DOMAIN_NONE;
            bo->rq = NULL;
            list_del(&bo->request);
            kgem_bo_move_to_inactive(kgem, bo);
        } else
            kgem_bo_free(kgem, bo);

        retired = true;
    }

    while (!list_is_empty(&kgem->requests)) {
        struct kgem_request *rq;

        rq = list_first_entry(&kgem->requests,
                      struct kgem_request,
                      list);
//        if (kgem_busy(kgem, rq->bo->handle))
//            break;

        DBG(("%s: request %d complete\n",
             __FUNCTION__, rq->bo->handle));

        while (!list_is_empty(&rq->buffers)) {
            bo = list_first_entry(&rq->buffers,
                          struct kgem_bo,
                          request);

            assert(bo->rq == rq);
            assert(bo->exec == NULL);
            assert(bo->domain == DOMAIN_GPU);

            list_del(&bo->request);
            bo->rq = NULL;

//            if (bo->needs_flush)
//                bo->needs_flush = kgem_busy(kgem, bo->handle);
            if (!bo->needs_flush)
                bo->domain = DOMAIN_NONE;

            if (bo->refcnt)
                continue;

            if (!bo->reusable) {
                DBG(("%s: closing %d\n",
                     __FUNCTION__, bo->handle));
                kgem_bo_free(kgem, bo);
                continue;
            }

            if (bo->needs_flush) {
                DBG(("%s: moving %d to flushing\n",
                     __FUNCTION__, bo->handle));
                list_add(&bo->request, &kgem->flushing);
                bo->rq = &_kgem_static_request;
            } else if (kgem_bo_set_purgeable(kgem, bo)) {
                DBG(("%s: moving %d to inactive\n",
                     __FUNCTION__, bo->handle));
                kgem_bo_move_to_inactive(kgem, bo);
                retired = true;
            } else {
                DBG(("%s: closing %d\n",
                     __FUNCTION__, bo->handle));
                kgem_bo_free(kgem, bo);
            }
        }

        rq->bo->refcnt--;
        assert(rq->bo->refcnt == 0);
        assert(rq->bo->rq == NULL);
        assert(list_is_empty(&rq->bo->request));
        if (kgem_bo_set_purgeable(kgem, rq->bo)) {
            kgem_bo_move_to_inactive(kgem, rq->bo);
            retired = true;
        } else {
            DBG(("%s: closing %d\n",
                 __FUNCTION__, rq->bo->handle));
            kgem_bo_free(kgem, rq->bo);
        }

        list_del(&rq->list);
        free(rq);
    }

    kgem->need_retire = !list_is_empty(&kgem->requests);
    DBG(("%s -- need_retire=%d\n", __FUNCTION__, kgem->need_retire));

    kgem->retire(kgem);

    return retired;
}









































static int kgem_batch_write(struct kgem *kgem, uint32_t handle, uint32_t size)
{
	int ret;

	assert(!kgem_busy(kgem, handle));

	/* If there is no surface data, just upload the batch */
	if (kgem->surface == kgem->max_batch_size)
		return gem_write(kgem->fd, handle,
				 0, sizeof(uint32_t)*kgem->nbatch,
				 kgem->batch);

	/* Are the batch pages conjoint with the surface pages? */
	if (kgem->surface < kgem->nbatch + PAGE_SIZE/4) {
		assert(size == sizeof(kgem->batch));
		return gem_write(kgem->fd, handle,
				 0, sizeof(kgem->batch),
				 kgem->batch);
	}

	/* Disjoint surface/batch, upload separately */
	ret = gem_write(kgem->fd, handle,
			0, sizeof(uint32_t)*kgem->nbatch,
			kgem->batch);
	if (ret)
		return ret;

   assert(kgem->nbatch*sizeof(uint32_t) <=
	       sizeof(uint32_t)*kgem->surface - (sizeof(kgem->batch)-size));
   return __gem_write(kgem->fd, handle,
			sizeof(uint32_t)*kgem->surface - (sizeof(kgem->batch)-size),
			sizeof(kgem->batch) - sizeof(uint32_t)*kgem->surface,
           kgem->batch + kgem->surface);
}

void kgem_reset(struct kgem *kgem)
{
//    ENTER();

    kgem->nfence = 0;
    kgem->nexec = 0;
    kgem->nreloc = 0;
    kgem->aperture = 0;
    kgem->aperture_fenced = 0;
    kgem->nbatch = 0;
    kgem->surface = kgem->max_batch_size;
    kgem->mode = KGEM_NONE;
    kgem->flush = 0;
    kgem->scanout = 0;

    kgem->batch = kgem->batch_ptr+1024*kgem->batch_idx;

    kgem->next_request = __kgem_request_alloc();

    kgem_sna_reset(kgem);
//    dbgprintf("surface %x\n", kgem->surface);
//    LEAVE();
}

static int compact_batch_surface(struct kgem *kgem)
{
	int size, shrink, n;

	/* See if we can pack the contents into one or two pages */
	size = kgem->max_batch_size - kgem->surface + kgem->nbatch;
	if (size > 2048)
		return sizeof(kgem->batch);
	else if (size > 1024)
		size = 8192, shrink = 2*4096;
	else
		size = 4096, shrink = 3*4096;


	for (n = 0; n < kgem->nreloc; n++) {
		if (kgem->reloc[n].read_domains == I915_GEM_DOMAIN_INSTRUCTION &&
		    kgem->reloc[n].target_handle == 0)
			kgem->reloc[n].delta -= shrink;

		if (kgem->reloc[n].offset >= size)
			kgem->reloc[n].offset -= shrink;
	}

	return size;
}

int exec_batch(struct drm_device *dev, struct intel_ring_buffer *ring,
               batchbuffer_t *exec);

void _kgem_submit(struct kgem *kgem, batchbuffer_t *exb)
{
	struct kgem_request *rq;
	uint32_t batch_end;
	int size;

	assert(!DBG_NO_HW);

	assert(kgem->nbatch);
	assert(kgem->nbatch <= KGEM_BATCH_SIZE(kgem));
	assert(kgem->nbatch <= kgem->surface);

	batch_end = kgem_end_batch(kgem);
	kgem_sna_flush(kgem);

	DBG(("batch[%d/%d]: %d %d %d, nreloc=%d, nexec=%d, nfence=%d, aperture=%d\n",
	     kgem->mode, kgem->ring, batch_end, kgem->nbatch, kgem->surface,
	     kgem->nreloc, kgem->nexec, kgem->nfence, kgem->aperture));

	assert(kgem->nbatch <= kgem->max_batch_size);
	assert(kgem->nbatch <= kgem->surface);
	assert(kgem->nreloc <= ARRAY_SIZE(kgem->reloc));
	assert(kgem->nexec < ARRAY_SIZE(kgem->exec));
	assert(kgem->nfence <= kgem->fence_max);

//   kgem_finish_partials(kgem);

	rq = kgem->next_request;
//   if (kgem->surface != kgem->max_batch_size)
//       size = compact_batch_surface(kgem);
//   else
		size = kgem->nbatch * sizeof(kgem->batch[0]);
#if 0
    {
        int i;

        dbgprintf("\nDump batch\n\n");

        for(i=0; i < kgem->nbatch; i++)
        {
            dbgprintf("\t0x%08x,\t/* %d */\n",
                      kgem->batch[i], i);
        }
        dbgprintf("\ndone\n");
    };
#endif

    exb->batch = kgem->batch_obj;
    exb->exec_start = kgem->batch_obj->gtt_offset+kgem->batch_idx*4096;
    exb->exec_len   = sizeof(uint32_t)*kgem->nbatch;

    exec_batch(main_device, NULL, exb);

//   if (kgem->wedged)
//       kgem_cleanup(kgem);

    kgem->batch_idx++;
    kgem->batch_idx&= 3;

	kgem->flush_now = kgem->scanout;
	kgem_reset(kgem);

	assert(kgem->next_request != NULL);
}

static struct kgem_bo *
search_linear_cache(struct kgem *kgem, unsigned int num_pages, unsigned flags)
{
    struct kgem_bo *bo, *first = NULL;
    bool use_active = (flags & CREATE_INACTIVE) == 0;
    struct list_head *cache;

    if (num_pages >= MAX_CACHE_SIZE / PAGE_SIZE)
        return NULL;

    if (!use_active &&
        list_is_empty(inactive(kgem, num_pages)) &&
        !list_is_empty(active(kgem, num_pages, I915_TILING_NONE)) &&
        !kgem_retire(kgem))
        return NULL;

    if (!use_active && flags & (CREATE_CPU_MAP | CREATE_GTT_MAP)) {
        int for_cpu = !!(flags & CREATE_CPU_MAP);
        cache = &kgem->vma[for_cpu].inactive[cache_bucket(num_pages)];
        list_for_each_entry(bo, cache, vma) {
            assert(IS_CPU_MAP(bo->map) == for_cpu);
            assert(bucket(bo) == cache_bucket(num_pages));

            if (num_pages > num_pages(bo)) {
                DBG(("inactive too small: %d < %d\n",
                     num_pages(bo), num_pages));
                continue;
            }

            if (bo->purged && !kgem_bo_clear_purgeable(kgem, bo)) {
                kgem_bo_free(kgem, bo);
                break;
            }

//            if (I915_TILING_NONE != bo->tiling &&
//                gem_set_tiling(kgem->fd, bo->handle,
//                       I915_TILING_NONE, 0) != I915_TILING_NONE)
//                continue;

            kgem_bo_remove_from_inactive(kgem, bo);

            bo->tiling = I915_TILING_NONE;
            bo->pitch = 0;
            bo->delta = 0;
            DBG(("  %s: found handle=%d (num_pages=%d) in linear vma cache\n",
                 __FUNCTION__, bo->handle, num_pages(bo)));
            assert(use_active || bo->domain != DOMAIN_GPU);
            assert(!bo->needs_flush);
            //assert(!kgem_busy(kgem, bo->handle));
            return bo;
        }
    }

    cache = use_active ? active(kgem, num_pages, I915_TILING_NONE) : inactive(kgem, num_pages);
    list_for_each_entry(bo, cache, list) {
        assert(bo->refcnt == 0);
        assert(bo->reusable);
        assert(!!bo->rq == !!use_active);

        if (num_pages > num_pages(bo))
            continue;

        if (use_active && bo->tiling != I915_TILING_NONE)
            continue;

        if (bo->purged && !kgem_bo_clear_purgeable(kgem, bo)) {
            kgem_bo_free(kgem, bo);
            break;
        }
/*
        if (I915_TILING_NONE != bo->tiling) {
            if (use_active)
                continue;

            if (gem_set_tiling(kgem->fd, bo->handle,
                       I915_TILING_NONE, 0) != I915_TILING_NONE)
                continue;

            bo->tiling = I915_TILING_NONE;
        }
*/
        if (bo->map) {
            if (flags & (CREATE_CPU_MAP | CREATE_GTT_MAP)) {
                int for_cpu = !!(flags & CREATE_CPU_MAP);
                if (IS_CPU_MAP(bo->map) != for_cpu) {
                    if (first != NULL)
                        break;

                    first = bo;
                    continue;
                }
            } else {
                if (first != NULL)
                    break;

                first = bo;
                continue;
            }
        } else {
            if (flags & (CREATE_CPU_MAP | CREATE_GTT_MAP)) {
                if (first != NULL)
                    break;

                first = bo;
                continue;
            }
        }

        if (use_active)
            kgem_bo_remove_from_active(kgem, bo);
        else
            kgem_bo_remove_from_inactive(kgem, bo);

        assert(bo->tiling == I915_TILING_NONE);
        bo->pitch = 0;
        bo->delta = 0;
        DBG(("  %s: found handle=%d (num_pages=%d) in linear %s cache\n",
             __FUNCTION__, bo->handle, num_pages(bo),
             use_active ? "active" : "inactive"));
        assert(use_active || bo->domain != DOMAIN_GPU);
        assert(!bo->needs_flush || use_active);
        //assert(use_active || !kgem_busy(kgem, bo->handle));
        return bo;
    }

    if (first) {
        assert(first->tiling == I915_TILING_NONE);

        if (use_active)
            kgem_bo_remove_from_active(kgem, first);
        else
            kgem_bo_remove_from_inactive(kgem, first);

        first->pitch = 0;
        first->delta = 0;
        DBG(("  %s: found handle=%d (num_pages=%d) in linear %s cache\n",
             __FUNCTION__, first->handle, num_pages(first),
             use_active ? "active" : "inactive"));
        assert(use_active || first->domain != DOMAIN_GPU);
        assert(!first->needs_flush || use_active);
        //assert(use_active || !kgem_busy(kgem, first->handle));
        return first;
    }

    return NULL;
}






struct kgem_bo *kgem_create_linear(struct kgem *kgem, int size)
{
    struct kgem_bo *bo;
    uint32_t handle;

    DBG(("%s(%d)\n", __FUNCTION__, size));

    size = (size + PAGE_SIZE - 1) / PAGE_SIZE;
    bo = search_linear_cache(kgem, size, CREATE_INACTIVE);
    if (bo)
        return kgem_bo_reference(bo);

    handle = gem_create(kgem->fd, size);
    if (handle == 0)
        return NULL;

    DBG(("%s: new handle=%x\n", __FUNCTION__, handle));
    bo = __kgem_bo_alloc(handle, size);
    if (bo == NULL) {
        gem_close(kgem->fd, handle);
        return NULL;
    }
    struct drm_i915_gem_object *obj;
    obj = (void*)handle;

    bo->gaddr = obj->gtt_offset;
    return bo;
}






inline int kgem_bo_fenced_size(struct kgem *kgem, struct kgem_bo *bo)
{
	unsigned int size;

	assert(bo->tiling);
	assert(kgem->gen < 40);

	if (kgem->gen < 30)
		size = 512 * 1024;
	else
		size = 1024 * 1024;
	while (size < bytes(bo))
		size *= 2;

	return size;
}
















































void _kgem_bo_destroy(struct kgem *kgem, struct kgem_bo *bo)
{
//    if (bo->proxy) {
//        assert(bo->map == NULL);
//        if (bo->io && bo->exec == NULL)
//            _kgem_bo_delete_partial(kgem, bo);
//        kgem_bo_unref(kgem, bo->proxy);
//        kgem_bo_binding_free(kgem, bo);
//        _list_del(&bo->request);
//        free(bo);
//        return;
//    }

//    if (bo->vmap)
//        kgem_bo_sync__cpu(kgem, bo);

    __kgem_bo_destroy(kgem, bo);
}

void __kgem_flush(struct kgem *kgem, struct kgem_bo *bo)
{
	/* The kernel will emit a flush *and* update its own flushing lists. */
//	kgem_busy(kgem, bo->handle);
}

bool kgem_check_bo(struct kgem *kgem, ...)
{
    va_list ap;
    struct kgem_bo *bo;
    int num_exec = 0;
    int num_pages = 0;

    va_start(ap, kgem);
    while ((bo = va_arg(ap, struct kgem_bo *))) {
        if (bo->exec)
            continue;

        if (bo->proxy) {
            bo = bo->proxy;
            if (bo->exec)
                continue;
        }
        num_pages += num_pages(bo);
        num_exec++;
    }
    va_end(ap);

    if (!num_pages)
        return true;

    if (kgem->aperture > kgem->aperture_low)
        return false;

    if (num_pages + kgem->aperture > kgem->aperture_high)
        return false;

    if (kgem->nexec + num_exec >= KGEM_EXEC_SIZE(kgem))
        return false;

    return true;
}

/*
bool kgem_check_bo_fenced(struct kgem *kgem, ...)
{
	va_list ap;
	struct kgem_bo *bo;
	int num_fence = 0;
	int num_exec = 0;
	int num_pages = 0;
	int fenced_size = 0;

	va_start(ap, kgem);
	while ((bo = va_arg(ap, struct kgem_bo *))) {
		if (bo->proxy)
			bo = bo->proxy;
		if (bo->exec) {
			if (kgem->gen >= 40 || bo->tiling == I915_TILING_NONE)
				continue;

			if ((bo->exec->flags & EXEC_OBJECT_NEEDS_FENCE) == 0) {
				fenced_size += kgem_bo_fenced_size(kgem, bo);
				num_fence++;
			}

			continue;
		}

		num_pages += num_pages(bo);
		num_exec++;
		if (kgem->gen < 40 && bo->tiling) {
			fenced_size += kgem_bo_fenced_size(kgem, bo);
			num_fence++;
		}
	}
	va_end(ap);

	if (fenced_size + kgem->aperture_fenced > kgem->aperture_mappable)
		return false;

	if (kgem->nfence + num_fence > kgem->fence_max)
		return false;

	if (!num_pages)
		return true;

	if (kgem->aperture > kgem->aperture_low)
		return false;

	if (num_pages + kgem->aperture > kgem->aperture_high)
		return false;

	if (kgem->nexec + num_exec >= KGEM_EXEC_SIZE(kgem))
		return false;

	return true;
}
*/
#if 0
uint32_t kgem_add_reloc(struct kgem *kgem,
            uint32_t pos,
            struct kgem_bo *bo,
            uint32_t read_write_domain,
            uint32_t delta)
{
    int index;

	DBG(("%s: handle=%d, pos=%d, delta=%d, domains=%08x\n",
         __FUNCTION__, bo ? bo->handle : 0, pos, delta, read_write_domain));

    assert((read_write_domain & 0x7fff) == 0 || bo != NULL);

    index = kgem->nreloc++;
    assert(index < ARRAY_SIZE(kgem->reloc));
    kgem->reloc[index].offset = pos * sizeof(kgem->batch[0]);
    if (bo) {
        assert(bo->refcnt);
        assert(!bo->purged);

        delta += bo->delta;
        if (bo->proxy) {
            DBG(("%s: adding proxy for handle=%d\n",
                 __FUNCTION__, bo->handle));
            assert(bo->handle == bo->proxy->handle);
            /* need to release the cache upon batch submit */
            list_move(&bo->request, &kgem->next_request->buffers);
            bo->exec = &_kgem_dummy_exec;
            bo = bo->proxy;
        }

        assert(!bo->purged);

//       if (bo->exec == NULL)
//           _kgem_add_bo(kgem, bo);

//        if (kgem->gen < 40 && read_write_domain & KGEM_RELOC_FENCED) {
//            if (bo->tiling &&
//                (bo->exec->flags & EXEC_OBJECT_NEEDS_FENCE) == 0) {
//                assert(kgem->nfence < kgem->fence_max);
//                kgem->aperture_fenced +=
//                    kgem_bo_fenced_size(kgem, bo);
//                kgem->nfence++;
//            }
//            bo->exec->flags |= EXEC_OBJECT_NEEDS_FENCE;
//        }

        kgem->reloc[index].delta = delta;
        kgem->reloc[index].target_handle = bo->handle;
        kgem->reloc[index].presumed_offset = bo->presumed_offset;

        if (read_write_domain & 0x7fff) {
           DBG(("%s: marking handle=%d dirty\n",
                __FUNCTION__, bo->handle));
           bo->needs_flush = bo->dirty = true;
       }

        delta += bo->presumed_offset;
    } else {
        kgem->reloc[index].delta = delta;
        kgem->reloc[index].target_handle = 0;
        kgem->reloc[index].presumed_offset = 0;
    }
    kgem->reloc[index].read_domains = read_write_domain >> 16;
    kgem->reloc[index].write_domain = read_write_domain & 0x7fff;

    return delta;
}
#endif









void *kgem_bo_map(struct kgem *kgem, struct kgem_bo *bo)
{
    void *ptr;

    DBG(("%s: handle=%d, offset=%d, tiling=%d, map=%p, domain=%d\n", __FUNCTION__,
         bo->handle, bo->presumed_offset, bo->tiling, bo->map, bo->domain));

    assert(!bo->purged);
    assert(bo->exec == NULL);
    assert(list_is_empty(&bo->list));

//    if (bo->tiling == I915_TILING_NONE &&
//        (kgem->has_llc || bo->domain == bo->presumed_offset)) {
        DBG(("%s: converting request for GTT map into CPU map\n",
             __FUNCTION__));
        ptr = kgem_bo_map__cpu(kgem, bo);
//        kgem_bo_sync__cpu(kgem, bo);
        return ptr;
//    }

#if 0

    if (IS_CPU_MAP(bo->map))
        kgem_bo_release_map(kgem, bo);

    ptr = bo->map;
    if (ptr == NULL) {
        assert(bytes(bo) <= kgem->aperture_mappable / 4);

        kgem_trim_vma_cache(kgem, MAP_GTT, bucket(bo));

        ptr = gem_mmap(kgem->fd, bo->handle, bytes(bo),
                   PROT_READ | PROT_WRITE);
        if (ptr == NULL)
            return NULL;

        /* Cache this mapping to avoid the overhead of an
         * excruciatingly slow GTT pagefault. This is more an
         * issue with compositing managers which need to frequently
         * flush CPU damage to their GPU bo.
         */
        bo->map = ptr;
        DBG(("%s: caching GTT vma for %d\n", __FUNCTION__, bo->handle));
    }

    if (bo->domain != DOMAIN_GTT) {
        struct drm_i915_gem_set_domain set_domain;

        DBG(("%s: sync: needs_flush? %d, domain? %d\n", __FUNCTION__,
             bo->needs_flush, bo->domain));

        /* XXX use PROT_READ to avoid the write flush? */

        VG_CLEAR(set_domain);
        set_domain.handle = bo->handle;
        set_domain.read_domains = I915_GEM_DOMAIN_GTT;
        set_domain.write_domain = I915_GEM_DOMAIN_GTT;
        drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &set_domain);

        kgem_bo_retire(kgem, bo);
        bo->domain = DOMAIN_GTT;
    }
#endif

    return ptr;
}

void *kgem_bo_map__cpu(struct kgem *kgem, struct kgem_bo *bo)
{
//    struct drm_i915_gem_mmap mmap_arg;

    DBG(("%s(handle=%d, size=%d)\n", __FUNCTION__, bo->handle, bytes(bo)));
    assert(!bo->purged);
    assert(list_is_empty(&bo->list));

    if (IS_CPU_MAP(bo->map))
        return CPU_MAP(bo->map);

    struct drm_i915_gem_object *obj = (void*)bo->handle;
    u8    *dst;
    int    ret;

    if(obj->pin_count == 0)
    {
        ret = i915_gem_object_pin(obj, 4096, true);
        if (ret)
            return NULL;
    };

    dst = drm_intel_bo_map(obj, true);
    DBG(("%s: caching CPU vma for %d\n", __FUNCTION__, bo->handle));
    bo->map = MAKE_CPU_MAP(dst);
    return (void *)dst;


#if 0
    if (bo->map)
        kgem_bo_release_map(kgem, bo);

    kgem_trim_vma_cache(kgem, MAP_CPU, bucket(bo));

    VG_CLEAR(mmap_arg);
    mmap_arg.handle = bo->handle;
    mmap_arg.offset = 0;
    mmap_arg.size = bytes(bo);
    if (drmIoctl(kgem->fd, DRM_IOCTL_I915_GEM_MMAP, &mmap_arg)) {
        ErrorF("%s: failed to mmap %d, %d bytes, into CPU domain\n",
               __FUNCTION__, bo->handle, bytes(bo));
        return NULL;
    }

    VG(VALGRIND_MALLOCLIKE_BLOCK(mmap_arg.addr_ptr, bytes(bo), 0, 1));
#endif

}



















void kgem_clear_dirty(struct kgem *kgem)
{
    struct kgem_request *rq = kgem->next_request;
    struct kgem_bo *bo;

    list_for_each_entry(bo, &rq->buffers, request)
        bo->dirty = false;
}

struct kgem_bo *kgem_create_proxy(struct kgem_bo *target,
				  int offset, int length)
{
	struct kgem_bo *bo;

	DBG(("%s: target handle=%d, offset=%d, length=%d, io=%d\n",
	     __FUNCTION__, target->handle, offset, length, target->io));

	bo = __kgem_bo_alloc(target->handle, length);
	if (bo == NULL)
		return NULL;

	bo->reusable = false;
	bo->size.bytes = length;

	bo->io = target->io;
	bo->dirty = target->dirty;
	bo->tiling = target->tiling;
	bo->pitch = target->pitch;

	if (target->proxy) {
		offset += target->delta;
		target = target->proxy;
	}
	bo->proxy = kgem_bo_reference(target);
	bo->delta = offset;
    bo->gaddr = offset + target->gaddr;
	return bo;
}










uint32_t kgem_bo_get_binding(struct kgem_bo *bo, uint32_t format)
{
    struct kgem_bo_binding *b;

    for (b = &bo->binding; b && b->offset; b = b->next)
        if (format == b->format)
            return b->offset;

    return 0;
}

void kgem_bo_set_binding(struct kgem_bo *bo, uint32_t format, uint16_t offset)
{
    struct kgem_bo_binding *b;

    for (b = &bo->binding; b; b = b->next) {
        if (b->offset)
            continue;

        b->offset = offset;
        b->format = format;

        if (b->next)
            b->next->offset = 0;

        return;
    }

    b = malloc(sizeof(*b));
    if (b) {
        b->next = bo->binding.next;
        b->format = format;
        b->offset = offset;
        bo->binding.next = b;
    }
}


struct kgem_bo *create_bo(bitmap_t *bitmap)
{
  struct kgem_bo *bo;

  bo = __kgem_bo_alloc(bitmap->obj, 1024*768*4/4096);
  bo->gaddr  = bitmap->gaddr;
  bo->pitch  = bitmap->pitch;
  bo->tiling = 0;
  return bo;

};