4 #include "util/u_inlines.h"
5 #include "util/u_memory.h"
8 #include "nouveau_winsys.h"
9 #include "nouveau_screen.h"
10 #include "nouveau_mm.h"
12 /* TODO: Higher orders can waste a lot of space for npot size buffers, should
13 * add an extra cache for such buffer objects.
15 * HACK: Max order == 21 to accommodate TF2's 1.5 MiB, frequently reallocated
16 * vertex buffer (VM flush (?) decreases performance dramatically).
19 #define MM_MIN_ORDER 7 /* >= 6 to not violate ARB_map_buffer_alignment */
20 #define MM_MAX_ORDER 21
22 #define MM_NUM_BUCKETS (MM_MAX_ORDER - MM_MIN_ORDER + 1)
24 #define MM_MIN_SIZE (1 << MM_MIN_ORDER)
25 #define MM_MAX_SIZE (1 << MM_MAX_ORDER)
28 struct list_head free
;
29 struct list_head used
;
30 struct list_head full
;
35 struct nouveau_device
*dev
;
36 struct mm_bucket bucket
[MM_NUM_BUCKETS
];
38 union nouveau_bo_config config
;
43 struct list_head head
;
44 struct nouveau_bo
*bo
;
45 struct nouveau_mman
*cache
;
53 mm_slab_alloc(struct mm_slab
*slab
)
60 for (i
= 0; i
< (slab
->count
+ 31) / 32; ++i
) {
61 b
= ffs(slab
->bits
[i
]) - 1;
64 assert(n
< slab
->count
);
66 slab
->bits
[i
] &= ~(1 << b
);
74 mm_slab_free(struct mm_slab
*slab
, int i
)
76 assert(i
< slab
->count
);
77 slab
->bits
[i
/ 32] |= 1 << (i
% 32);
79 assert(slab
->free
<= slab
->count
);
83 mm_get_order(uint32_t size
)
85 int s
= __builtin_clz(size
) ^ 31;
92 static struct mm_bucket
*
93 mm_bucket_by_order(struct nouveau_mman
*cache
, int order
)
95 if (order
> MM_MAX_ORDER
)
97 return &cache
->bucket
[MAX2(order
, MM_MIN_ORDER
) - MM_MIN_ORDER
];
100 static struct mm_bucket
*
101 mm_bucket_by_size(struct nouveau_mman
*cache
, unsigned size
)
103 return mm_bucket_by_order(cache
, mm_get_order(size
));
106 /* size of bo allocation for slab with chunks of (1 << chunk_order) bytes */
107 static inline uint32_t
108 mm_default_slab_size(unsigned chunk_order
)
110 static const int8_t slab_order
[MM_MAX_ORDER
- MM_MIN_ORDER
+ 1] =
112 12, 12, 13, 14, 14, 17, 17, 17, 17, 19, 19, 20, 21, 22, 22
115 assert(chunk_order
<= MM_MAX_ORDER
&& chunk_order
>= MM_MIN_ORDER
);
117 return 1 << slab_order
[chunk_order
- MM_MIN_ORDER
];
121 mm_slab_new(struct nouveau_mman
*cache
, int chunk_order
)
123 struct mm_slab
*slab
;
125 const uint32_t size
= mm_default_slab_size(chunk_order
);
127 words
= ((size
>> chunk_order
) + 31) / 32;
130 slab
= MALLOC(sizeof(struct mm_slab
) + words
* 4);
132 return PIPE_ERROR_OUT_OF_MEMORY
;
134 memset(&slab
->bits
[0], ~0, words
* 4);
138 ret
= nouveau_bo_new(cache
->dev
, cache
->domain
, 0, size
, &cache
->config
,
142 return PIPE_ERROR_OUT_OF_MEMORY
;
145 LIST_INITHEAD(&slab
->head
);
148 slab
->order
= chunk_order
;
149 slab
->count
= slab
->free
= size
>> chunk_order
;
151 LIST_ADD(&slab
->head
, &mm_bucket_by_order(cache
, chunk_order
)->free
);
153 cache
->allocated
+= size
;
155 if (nouveau_mesa_debug
)
156 debug_printf("MM: new slab, total memory = %"PRIu64
" KiB\n",
157 cache
->allocated
/ 1024);
162 /* @return token to identify slab or NULL if we just allocated a new bo */
163 struct nouveau_mm_allocation
*
164 nouveau_mm_allocate(struct nouveau_mman
*cache
,
165 uint32_t size
, struct nouveau_bo
**bo
, uint32_t *offset
)
167 struct mm_bucket
*bucket
;
168 struct mm_slab
*slab
;
169 struct nouveau_mm_allocation
*alloc
;
172 bucket
= mm_bucket_by_size(cache
, size
);
174 ret
= nouveau_bo_new(cache
->dev
, cache
->domain
, 0, size
, &cache
->config
,
177 debug_printf("bo_new(%x, %x): %i\n",
178 size
, cache
->config
.nv50
.memtype
, ret
);
184 if (!LIST_IS_EMPTY(&bucket
->used
)) {
185 slab
= LIST_ENTRY(struct mm_slab
, bucket
->used
.next
, head
);
187 if (LIST_IS_EMPTY(&bucket
->free
)) {
188 mm_slab_new(cache
, MAX2(mm_get_order(size
), MM_MIN_ORDER
));
190 slab
= LIST_ENTRY(struct mm_slab
, bucket
->free
.next
, head
);
192 LIST_DEL(&slab
->head
);
193 LIST_ADD(&slab
->head
, &bucket
->used
);
196 *offset
= mm_slab_alloc(slab
) << slab
->order
;
198 alloc
= MALLOC_STRUCT(nouveau_mm_allocation
);
202 nouveau_bo_ref(slab
->bo
, bo
);
204 if (slab
->free
== 0) {
205 LIST_DEL(&slab
->head
);
206 LIST_ADD(&slab
->head
, &bucket
->full
);
210 alloc
->offset
= *offset
;
211 alloc
->priv
= (void *)slab
;
217 nouveau_mm_free(struct nouveau_mm_allocation
*alloc
)
219 struct mm_slab
*slab
= (struct mm_slab
*)alloc
->priv
;
220 struct mm_bucket
*bucket
= mm_bucket_by_order(slab
->cache
, slab
->order
);
222 mm_slab_free(slab
, alloc
->offset
>> slab
->order
);
224 if (slab
->free
== slab
->count
) {
225 LIST_DEL(&slab
->head
);
226 LIST_ADDTAIL(&slab
->head
, &bucket
->free
);
228 if (slab
->free
== 1) {
229 LIST_DEL(&slab
->head
);
230 LIST_ADDTAIL(&slab
->head
, &bucket
->used
);
237 nouveau_mm_free_work(void *data
)
239 nouveau_mm_free(data
);
242 struct nouveau_mman
*
243 nouveau_mm_create(struct nouveau_device
*dev
, uint32_t domain
,
244 union nouveau_bo_config
*config
)
246 struct nouveau_mman
*cache
= MALLOC_STRUCT(nouveau_mman
);
253 cache
->domain
= domain
;
254 cache
->config
= *config
;
255 cache
->allocated
= 0;
257 for (i
= 0; i
< MM_NUM_BUCKETS
; ++i
) {
258 LIST_INITHEAD(&cache
->bucket
[i
].free
);
259 LIST_INITHEAD(&cache
->bucket
[i
].used
);
260 LIST_INITHEAD(&cache
->bucket
[i
].full
);
267 nouveau_mm_free_slabs(struct list_head
*head
)
269 struct mm_slab
*slab
, *next
;
271 LIST_FOR_EACH_ENTRY_SAFE(slab
, next
, head
, head
) {
272 LIST_DEL(&slab
->head
);
273 nouveau_bo_ref(NULL
, &slab
->bo
);
279 nouveau_mm_destroy(struct nouveau_mman
*cache
)
286 for (i
= 0; i
< MM_NUM_BUCKETS
; ++i
) {
287 if (!LIST_IS_EMPTY(&cache
->bucket
[i
].used
) ||
288 !LIST_IS_EMPTY(&cache
->bucket
[i
].full
))
289 debug_printf("WARNING: destroying GPU memory cache "
290 "with some buffers still in use\n");
292 nouveau_mm_free_slabs(&cache
->bucket
[i
].free
);
293 nouveau_mm_free_slabs(&cache
->bucket
[i
].used
);
294 nouveau_mm_free_slabs(&cache
->bucket
[i
].full
);