src/gallium/drivers/nouveau/nouveau_mm.c

   1
   2 #include <inttypes.h>
   3
   4 #include "util/u_inlines.h"
   5 #include "util/u_memory.h"
   6 #include "util/u_double_list.h"
   7
   8 #include "nouveau_winsys.h"
   9 #include "nouveau_screen.h"
  10 #include "nouveau_mm.h"
  11
  12 #define MM_MIN_ORDER 7
  13 #define MM_MAX_ORDER 20
  14
  15 #define MM_NUM_BUCKETS (MM_MAX_ORDER - MM_MIN_ORDER + 1)
  16
  17 #define MM_MIN_SIZE (1 << MM_MIN_ORDER)
  18 #define MM_MAX_SIZE (1 << MM_MAX_ORDER)
  19
  20 struct mm_bucket {
  21    struct list_head free;
  22    struct list_head used;
  23    struct list_head full;
  24    int num_free;
  25 };
  26
  27 struct nouveau_mman {
  28    struct nouveau_device *dev;
  29    struct mm_bucket bucket[MM_NUM_BUCKETS];
  30    uint32_t domain;
  31    union nouveau_bo_config config;
  32    uint64_t allocated;
  33 };
  34
  35 struct mm_slab {
  36    struct list_head head;
  37    struct nouveau_bo *bo;
  38    struct nouveau_mman *cache;
  39    int order;
  40    int count;
  41    int free;
  42    uint32_t bits[0];
  43 };
  44
  45 static int
  46 mm_slab_alloc(struct mm_slab *slab)
  47 {
  48    int i, n, b;
  49
  50    if (slab->free == 0)
  51       return -1;
  52
  53    for (i = 0; i < (slab->count + 31) / 32; ++i) {
  54       b = ffs(slab->bits[i]) - 1;
  55       if (b >= 0) {
  56          n = i * 32 + b;
  57          assert(n < slab->count);
  58          slab->free--;
  59          slab->bits[i] &= ~(1 << b);
  60          return n;
  61       }
  62    }
  63    return -1;
  64 }
  65
  66 static INLINE void
  67 mm_slab_free(struct mm_slab *slab, int i)
  68 {
  69    assert(i < slab->count);
  70    slab->bits[i / 32] |= 1 << (i % 32);
  71    slab->free++;
  72    assert(slab->free <= slab->count);
  73 }
  74
  75 static INLINE int
  76 mm_get_order(uint32_t size)
  77 {
  78    int s = __builtin_clz(size) ^ 31;
  79
  80    if (size > (1 << s))
  81       s += 1;
  82    return s;
  83 }
  84
  85 static struct mm_bucket *
  86 mm_bucket_by_order(struct nouveau_mman *cache, int order)
  87 {
  88    if (order > MM_MAX_ORDER)
  89       return NULL;
  90    return &cache->bucket[MAX2(order, MM_MIN_ORDER) - MM_MIN_ORDER];
  91 }
  92
  93 static struct mm_bucket *
  94 mm_bucket_by_size(struct nouveau_mman *cache, unsigned size)
  95 {
  96    return mm_bucket_by_order(cache, mm_get_order(size));
  97 }
  98
  99 /* size of bo allocation for slab with chunks of (1 << chunk_order) bytes */
 100 static INLINE uint32_t
 101 mm_default_slab_size(unsigned chunk_order)
 102 {
 103    static const int8_t slab_order[MM_MAX_ORDER - MM_MIN_ORDER + 1] =
 104    {
 105       12, 12, 13, 14, 14, 17, 17, 17, 17, 19, 19, 20, 21, 22
 106    };
 107
 108    assert(chunk_order <= MM_MAX_ORDER && chunk_order >= MM_MIN_ORDER);
 109
 110    return 1 << slab_order[chunk_order - MM_MIN_ORDER];
 111 }
 112
 113 static int
 114 mm_slab_new(struct nouveau_mman *cache, int chunk_order)
 115 {
 116    struct mm_slab *slab;
 117    int words, ret;
 118    const uint32_t size = mm_default_slab_size(chunk_order);
 119
 120    words = ((size >> chunk_order) + 31) / 32;
 121    assert(words);
 122
 123    slab = MALLOC(sizeof(struct mm_slab) + words * 4);
 124    if (!slab)
 125       return PIPE_ERROR_OUT_OF_MEMORY;
 126
 127    memset(&slab->bits[0], ~0, words * 4);
 128
 129    slab->bo = NULL;
 130
 131    ret = nouveau_bo_new(cache->dev, cache->domain, 0, size, &cache->config,
 132                         &slab->bo);
 133    if (ret) {
 134       FREE(slab);
 135       return PIPE_ERROR_OUT_OF_MEMORY;
 136    }
 137
 138    LIST_INITHEAD(&slab->head);
 139
 140    slab->cache = cache;
 141    slab->order = chunk_order;
 142    slab->count = slab->free = size >> chunk_order;
 143
 144    LIST_ADD(&slab->head, &mm_bucket_by_order(cache, chunk_order)->free);
 145
 146    cache->allocated += size;
 147
 148    if (nouveau_mesa_debug)
 149       debug_printf("MM: new slab, total memory = %"PRIu64" KiB\n",
 150                    cache->allocated / 1024);
 151
 152    return PIPE_OK;
 153 }
 154
 155 /* @return token to identify slab or NULL if we just allocated a new bo */
 156 struct nouveau_mm_allocation *
 157 nouveau_mm_allocate(struct nouveau_mman *cache,
 158                     uint32_t size, struct nouveau_bo **bo, uint32_t *offset)
 159 {
 160    struct mm_bucket *bucket;
 161    struct mm_slab *slab;
 162    struct nouveau_mm_allocation *alloc;
 163    int ret;
 164
 165    bucket = mm_bucket_by_size(cache, size);
 166    if (!bucket) {
 167       ret = nouveau_bo_new(cache->dev, cache->domain, 0, size, &cache->config,
 168                            bo);
 169       if (ret)
 170          debug_printf("bo_new(%x, %x): %i\n",
 171                       size, cache->config.nv50.memtype, ret);
 172
 173       *offset = 0;
 174       return NULL;
 175    }
 176
 177    if (!LIST_IS_EMPTY(&bucket->used)) {
 178       slab = LIST_ENTRY(struct mm_slab, bucket->used.next, head);
 179    } else {
 180       if (LIST_IS_EMPTY(&bucket->free)) {
 181          mm_slab_new(cache, MAX2(mm_get_order(size), MM_MIN_ORDER));
 182       }
 183       slab = LIST_ENTRY(struct mm_slab, bucket->free.next, head);
 184
 185       LIST_DEL(&slab->head);
 186       LIST_ADD(&slab->head, &bucket->used);
 187    }
 188
 189    *offset = mm_slab_alloc(slab) << slab->order;
 190
 191    alloc = MALLOC_STRUCT(nouveau_mm_allocation);
 192    if (!alloc)
 193       return NULL;
 194
 195    nouveau_bo_ref(slab->bo, bo);
 196
 197    if (slab->free == 0) {
 198       LIST_DEL(&slab->head);
 199       LIST_ADD(&slab->head, &bucket->full);
 200    }
 201
 202    alloc->next = NULL;
 203    alloc->offset = *offset;
 204    alloc->priv = (void *)slab;
 205
 206    return alloc;
 207 }
 208
 209 void
 210 nouveau_mm_free(struct nouveau_mm_allocation *alloc)
 211 {
 212    struct mm_slab *slab = (struct mm_slab *)alloc->priv;
 213    struct mm_bucket *bucket = mm_bucket_by_order(slab->cache, slab->order);
 214
 215    mm_slab_free(slab, alloc->offset >> slab->order);
 216
 217    if (slab->free == slab->count) {
 218       LIST_DEL(&slab->head);
 219       LIST_ADDTAIL(&slab->head, &bucket->free);
 220    } else
 221    if (slab->free == 1) {
 222       LIST_DEL(&slab->head);
 223       LIST_ADDTAIL(&slab->head, &bucket->used);
 224    }
 225
 226    FREE(alloc);
 227 }
 228
 229 void
 230 nouveau_mm_free_work(void *data)
 231 {
 232    nouveau_mm_free(data);
 233 }
 234
 235 struct nouveau_mman *
 236 nouveau_mm_create(struct nouveau_device *dev, uint32_t domain,
 237                   union nouveau_bo_config *config)
 238 {
 239    struct nouveau_mman *cache = MALLOC_STRUCT(nouveau_mman);
 240    int i;
 241
 242    if (!cache)
 243       return NULL;
 244
 245    cache->dev = dev;
 246    cache->domain = domain;
 247    cache->config = *config;
 248    cache->allocated = 0;
 249
 250    for (i = 0; i < MM_NUM_BUCKETS; ++i) {
 251       LIST_INITHEAD(&cache->bucket[i].free);
 252       LIST_INITHEAD(&cache->bucket[i].used);
 253       LIST_INITHEAD(&cache->bucket[i].full);
 254    }
 255
 256    return cache;
 257 }
 258
 259 static INLINE void
 260 nouveau_mm_free_slabs(struct list_head *head)
 261 {
 262    struct mm_slab *slab, *next;
 263
 264    LIST_FOR_EACH_ENTRY_SAFE(slab, next, head, head) {
 265       LIST_DEL(&slab->head);
 266       nouveau_bo_ref(NULL, &slab->bo);
 267       FREE(slab);
 268    }
 269 }
 270
 271 void
 272 nouveau_mm_destroy(struct nouveau_mman *cache)
 273 {
 274    int i;
 275
 276    if (!cache)
 277       return;
 278
 279    for (i = 0; i < MM_NUM_BUCKETS; ++i) {
 280       if (!LIST_IS_EMPTY(&cache->bucket[i].used) ||
 281           !LIST_IS_EMPTY(&cache->bucket[i].full))
 282          debug_printf("WARNING: destroying GPU memory cache "
 283                       "with some buffers still in use\n");
 284
 285       nouveau_mm_free_slabs(&cache->bucket[i].free);
 286       nouveau_mm_free_slabs(&cache->bucket[i].used);
 287       nouveau_mm_free_slabs(&cache->bucket[i].full);
 288    }
 289
 290    FREE(cache);
 291 }
 292