src/gallium/drivers/nouveau/nouveau_mm.c

   1
   2 #include <inttypes.h>
   3
   4 #include "util/u_inlines.h"
   5 #include "util/u_memory.h"
   6 #include "util/list.h"
   7
   8 #include "nouveau_winsys.h"
   9 #include "nouveau_screen.h"
  10 #include "nouveau_mm.h"
  11
  12 /* TODO: Higher orders can waste a lot of space for npot size buffers, should
  13  * add an extra cache for such buffer objects.
  14  *
  15  * HACK: Max order == 21 to accommodate TF2's 1.5 MiB, frequently reallocated
  16  * vertex buffer (VM flush (?) decreases performance dramatically).
  17  */
  18
  19 #define MM_MIN_ORDER 7 /* >= 6 to not violate ARB_map_buffer_alignment */
  20 #define MM_MAX_ORDER 21
  21
  22 #define MM_NUM_BUCKETS (MM_MAX_ORDER - MM_MIN_ORDER + 1)
  23
  24 #define MM_MIN_SIZE (1 << MM_MIN_ORDER)
  25 #define MM_MAX_SIZE (1 << MM_MAX_ORDER)
  26
  27 struct mm_bucket {
  28    struct list_head free;
  29    struct list_head used;
  30    struct list_head full;
  31    int num_free;
  32 };
  33
  34 struct nouveau_mman {
  35    struct nouveau_device *dev;
  36    struct mm_bucket bucket[MM_NUM_BUCKETS];
  37    uint32_t domain;
  38    union nouveau_bo_config config;
  39    uint64_t allocated;
  40 };
  41
  42 struct mm_slab {
  43    struct list_head head;
  44    struct nouveau_bo *bo;
  45    struct nouveau_mman *cache;
  46    int order;
  47    int count;
  48    int free;
  49    uint32_t bits[0];
  50 };
  51
  52 static int
  53 mm_slab_alloc(struct mm_slab *slab)
  54 {
  55    int i, n, b;
  56
  57    if (slab->free == 0)
  58       return -1;
  59
  60    for (i = 0; i < (slab->count + 31) / 32; ++i) {
  61       b = ffs(slab->bits[i]) - 1;
  62       if (b >= 0) {
  63          n = i * 32 + b;
  64          assert(n < slab->count);
  65          slab->free--;
  66          slab->bits[i] &= ~(1 << b);
  67          return n;
  68       }
  69    }
  70    return -1;
  71 }
  72
  73 static inline void
  74 mm_slab_free(struct mm_slab *slab, int i)
  75 {
  76    assert(i < slab->count);
  77    slab->bits[i / 32] |= 1 << (i % 32);
  78    slab->free++;
  79    assert(slab->free <= slab->count);
  80 }
  81
  82 static inline int
  83 mm_get_order(uint32_t size)
  84 {
  85    int s = __builtin_clz(size) ^ 31;
  86
  87    if (size > (1 << s))
  88       s += 1;
  89    return s;
  90 }
  91
  92 static struct mm_bucket *
  93 mm_bucket_by_order(struct nouveau_mman *cache, int order)
  94 {
  95    if (order > MM_MAX_ORDER)
  96       return NULL;
  97    return &cache->bucket[MAX2(order, MM_MIN_ORDER) - MM_MIN_ORDER];
  98 }
  99
 100 static struct mm_bucket *
 101 mm_bucket_by_size(struct nouveau_mman *cache, unsigned size)
 102 {
 103    return mm_bucket_by_order(cache, mm_get_order(size));
 104 }
 105
 106 /* size of bo allocation for slab with chunks of (1 << chunk_order) bytes */
 107 static inline uint32_t
 108 mm_default_slab_size(unsigned chunk_order)
 109 {
 110    static const int8_t slab_order[MM_MAX_ORDER - MM_MIN_ORDER + 1] =
 111    {
 112       12, 12, 13, 14, 14, 17, 17, 17, 17, 19, 19, 20, 21, 22, 22
 113    };
 114
 115    assert(chunk_order <= MM_MAX_ORDER && chunk_order >= MM_MIN_ORDER);
 116
 117    return 1 << slab_order[chunk_order - MM_MIN_ORDER];
 118 }
 119
 120 static int
 121 mm_slab_new(struct nouveau_mman *cache, int chunk_order)
 122 {
 123    struct mm_slab *slab;
 124    int words, ret;
 125    const uint32_t size = mm_default_slab_size(chunk_order);
 126
 127    words = ((size >> chunk_order) + 31) / 32;
 128    assert(words);
 129
 130    slab = MALLOC(sizeof(struct mm_slab) + words * 4);
 131    if (!slab)
 132       return PIPE_ERROR_OUT_OF_MEMORY;
 133
 134    memset(&slab->bits[0], ~0, words * 4);
 135
 136    slab->bo = NULL;
 137
 138    ret = nouveau_bo_new(cache->dev, cache->domain, 0, size, &cache->config,
 139                         &slab->bo);
 140    if (ret) {
 141       FREE(slab);
 142       return PIPE_ERROR_OUT_OF_MEMORY;
 143    }
 144
 145    LIST_INITHEAD(&slab->head);
 146
 147    slab->cache = cache;
 148    slab->order = chunk_order;
 149    slab->count = slab->free = size >> chunk_order;
 150
 151    LIST_ADD(&slab->head, &mm_bucket_by_order(cache, chunk_order)->free);
 152
 153    cache->allocated += size;
 154
 155    if (nouveau_mesa_debug)
 156       debug_printf("MM: new slab, total memory = %"PRIu64" KiB\n",
 157                    cache->allocated / 1024);
 158
 159    return PIPE_OK;
 160 }
 161
 162 /* @return token to identify slab or NULL if we just allocated a new bo */
 163 struct nouveau_mm_allocation *
 164 nouveau_mm_allocate(struct nouveau_mman *cache,
 165                     uint32_t size, struct nouveau_bo **bo, uint32_t *offset)
 166 {
 167    struct mm_bucket *bucket;
 168    struct mm_slab *slab;
 169    struct nouveau_mm_allocation *alloc;
 170    int ret;
 171
 172    bucket = mm_bucket_by_size(cache, size);
 173    if (!bucket) {
 174       ret = nouveau_bo_new(cache->dev, cache->domain, 0, size, &cache->config,
 175                            bo);
 176       if (ret)
 177          debug_printf("bo_new(%x, %x): %i\n",
 178                       size, cache->config.nv50.memtype, ret);
 179
 180       *offset = 0;
 181       return NULL;
 182    }
 183
 184    if (!LIST_IS_EMPTY(&bucket->used)) {
 185       slab = LIST_ENTRY(struct mm_slab, bucket->used.next, head);
 186    } else {
 187       if (LIST_IS_EMPTY(&bucket->free)) {
 188          mm_slab_new(cache, MAX2(mm_get_order(size), MM_MIN_ORDER));
 189       }
 190       slab = LIST_ENTRY(struct mm_slab, bucket->free.next, head);
 191
 192       LIST_DEL(&slab->head);
 193       LIST_ADD(&slab->head, &bucket->used);
 194    }
 195
 196    *offset = mm_slab_alloc(slab) << slab->order;
 197
 198    alloc = MALLOC_STRUCT(nouveau_mm_allocation);
 199    if (!alloc)
 200       return NULL;
 201
 202    nouveau_bo_ref(slab->bo, bo);
 203
 204    if (slab->free == 0) {
 205       LIST_DEL(&slab->head);
 206       LIST_ADD(&slab->head, &bucket->full);
 207    }
 208
 209    alloc->next = NULL;
 210    alloc->offset = *offset;
 211    alloc->priv = (void *)slab;
 212
 213    return alloc;
 214 }
 215
 216 void
 217 nouveau_mm_free(struct nouveau_mm_allocation *alloc)
 218 {
 219    struct mm_slab *slab = (struct mm_slab *)alloc->priv;
 220    struct mm_bucket *bucket = mm_bucket_by_order(slab->cache, slab->order);
 221
 222    mm_slab_free(slab, alloc->offset >> slab->order);
 223
 224    if (slab->free == slab->count) {
 225       LIST_DEL(&slab->head);
 226       LIST_ADDTAIL(&slab->head, &bucket->free);
 227    } else
 228    if (slab->free == 1) {
 229       LIST_DEL(&slab->head);
 230       LIST_ADDTAIL(&slab->head, &bucket->used);
 231    }
 232
 233    FREE(alloc);
 234 }
 235
 236 void
 237 nouveau_mm_free_work(void *data)
 238 {
 239    nouveau_mm_free(data);
 240 }
 241
 242 struct nouveau_mman *
 243 nouveau_mm_create(struct nouveau_device *dev, uint32_t domain,
 244                   union nouveau_bo_config *config)
 245 {
 246    struct nouveau_mman *cache = MALLOC_STRUCT(nouveau_mman);
 247    int i;
 248
 249    if (!cache)
 250       return NULL;
 251
 252    cache->dev = dev;
 253    cache->domain = domain;
 254    cache->config = *config;
 255    cache->allocated = 0;
 256
 257    for (i = 0; i < MM_NUM_BUCKETS; ++i) {
 258       LIST_INITHEAD(&cache->bucket[i].free);
 259       LIST_INITHEAD(&cache->bucket[i].used);
 260       LIST_INITHEAD(&cache->bucket[i].full);
 261    }
 262
 263    return cache;
 264 }
 265
 266 static inline void
 267 nouveau_mm_free_slabs(struct list_head *head)
 268 {
 269    struct mm_slab *slab, *next;
 270
 271    LIST_FOR_EACH_ENTRY_SAFE(slab, next, head, head) {
 272       LIST_DEL(&slab->head);
 273       nouveau_bo_ref(NULL, &slab->bo);
 274       FREE(slab);
 275    }
 276 }
 277
 278 void
 279 nouveau_mm_destroy(struct nouveau_mman *cache)
 280 {
 281    int i;
 282
 283    if (!cache)
 284       return;
 285
 286    for (i = 0; i < MM_NUM_BUCKETS; ++i) {
 287       if (!LIST_IS_EMPTY(&cache->bucket[i].used) ||
 288           !LIST_IS_EMPTY(&cache->bucket[i].full))
 289          debug_printf("WARNING: destroying GPU memory cache "
 290                       "with some buffers still in use\n");
 291
 292       nouveau_mm_free_slabs(&cache->bucket[i].free);
 293       nouveau_mm_free_slabs(&cache->bucket[i].used);
 294       nouveau_mm_free_slabs(&cache->bucket[i].full);
 295    }
 296
 297    FREE(cache);
 298 }