2 * Permission is hereby granted, free of charge, to any person obtaining a
3 * copy of this software and associated documentation files (the "Software"),
4 * to deal in the Software without restriction, including without limitation
5 * on the rights to use, copy, modify, merge, publish, distribute, sub
6 * license, and/or sell copies of the Software, and to permit persons to whom
7 * the Software is furnished to do so, subject to the following conditions:
9 * The above copyright notice and this permission notice (including the next
10 * paragraph) shall be included in all copies or substantial portions of the
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19 * USE OR OTHER DEALINGS IN THE SOFTWARE.
22 * Adam Rak <adam.rak@streamnovation.com>
25 #include "pipe/p_defines.h"
26 #include "pipe/p_state.h"
27 #include "pipe/p_context.h"
28 #include "util/u_blitter.h"
29 #include "util/u_double_list.h"
30 #include "util/u_transfer.h"
31 #include "util/u_surface.h"
32 #include "util/u_pack_color.h"
33 #include "util/u_memory.h"
34 #include "util/u_inlines.h"
35 #include "util/u_framebuffer.h"
36 #include "r600_resource.h"
37 #include "r600_shader.h"
38 #include "r600_pipe.h"
39 #include "r600_formats.h"
40 #include "compute_memory_pool.h"
41 #include "evergreen_compute.h"
42 #include "evergreen_compute_internal.h"
48 struct compute_memory_pool
* compute_memory_pool_new(
49 struct r600_screen
* rscreen
)
51 struct compute_memory_pool
* pool
= (struct compute_memory_pool
*)
52 CALLOC(sizeof(struct compute_memory_pool
), 1);
54 COMPUTE_DBG(rscreen
, "* compute_memory_pool_new()\n");
56 pool
->screen
= rscreen
;
60 static void compute_memory_pool_init(struct compute_memory_pool
* pool
,
61 unsigned initial_size_in_dw
)
64 COMPUTE_DBG(pool
->screen
, "* compute_memory_pool_init() initial_size_in_dw = %ld\n",
67 pool
->shadow
= (uint32_t*)CALLOC(initial_size_in_dw
, 4);
69 pool
->size_in_dw
= initial_size_in_dw
;
70 pool
->bo
= (struct r600_resource
*)r600_compute_buffer_alloc_vram(pool
->screen
,
71 pool
->size_in_dw
* 4);
75 * Frees all stuff in the pool and the pool struct itself too
77 void compute_memory_pool_delete(struct compute_memory_pool
* pool
)
79 COMPUTE_DBG(pool
->screen
, "* compute_memory_pool_delete()\n");
82 pool
->screen
->b
.b
.resource_destroy((struct pipe_screen
*)
83 pool
->screen
, (struct pipe_resource
*)pool
->bo
);
89 * Searches for an empty space in the pool, return with the pointer to the
90 * allocatable space in the pool, returns -1 on failure.
92 int64_t compute_memory_prealloc_chunk(
93 struct compute_memory_pool
* pool
,
96 struct compute_memory_item
*item
;
100 assert(size_in_dw
<= pool
->size_in_dw
);
102 COMPUTE_DBG(pool
->screen
, "* compute_memory_prealloc_chunk() size_in_dw = %ld\n",
105 for (item
= pool
->item_list
; item
; item
= item
->next
) {
106 if (item
->start_in_dw
> -1) {
107 if (item
->start_in_dw
-last_end
> size_in_dw
) {
111 last_end
= item
->start_in_dw
+ item
->size_in_dw
;
112 last_end
+= (1024 - last_end
% 1024);
116 if (pool
->size_in_dw
- last_end
< size_in_dw
) {
124 * Search for the chunk where we can link our new chunk after it.
126 struct compute_memory_item
* compute_memory_postalloc_chunk(
127 struct compute_memory_pool
* pool
,
130 struct compute_memory_item
* item
;
132 COMPUTE_DBG(pool
->screen
, "* compute_memory_postalloc_chunck() start_in_dw = %ld\n",
135 /* Check if we can insert it in the front of the list */
136 if (pool
->item_list
&& pool
->item_list
->start_in_dw
> start_in_dw
) {
140 for (item
= pool
->item_list
; item
; item
= item
->next
) {
142 if (item
->start_in_dw
< start_in_dw
143 && item
->next
->start_in_dw
> start_in_dw
) {
149 assert(item
->start_in_dw
< start_in_dw
);
154 assert(0 && "unreachable");
159 * Reallocates pool, conserves data
161 void compute_memory_grow_pool(struct compute_memory_pool
* pool
,
162 struct pipe_context
* pipe
, int new_size_in_dw
)
164 COMPUTE_DBG(pool
->screen
, "* compute_memory_grow_pool() "
165 "new_size_in_dw = %d (%d bytes)\n",
166 new_size_in_dw
, new_size_in_dw
* 4);
168 assert(new_size_in_dw
>= pool
->size_in_dw
);
171 compute_memory_pool_init(pool
, MAX2(new_size_in_dw
, 1024 * 16));
173 new_size_in_dw
+= 1024 - (new_size_in_dw
% 1024);
175 COMPUTE_DBG(pool
->screen
, " Aligned size = %d (%d bytes)\n",
176 new_size_in_dw
, new_size_in_dw
* 4);
178 compute_memory_shadow(pool
, pipe
, 1);
179 pool
->shadow
= realloc(pool
->shadow
, new_size_in_dw
*4);
180 pool
->size_in_dw
= new_size_in_dw
;
181 pool
->screen
->b
.b
.resource_destroy(
182 (struct pipe_screen
*)pool
->screen
,
183 (struct pipe_resource
*)pool
->bo
);
184 pool
->bo
= (struct r600_resource
*)r600_compute_buffer_alloc_vram(
186 pool
->size_in_dw
* 4);
187 compute_memory_shadow(pool
, pipe
, 0);
192 * Copy pool from device to host, or host to device.
194 void compute_memory_shadow(struct compute_memory_pool
* pool
,
195 struct pipe_context
* pipe
, int device_to_host
)
197 struct compute_memory_item chunk
;
199 COMPUTE_DBG(pool
->screen
, "* compute_memory_shadow() device_to_host = %d\n",
203 chunk
.start_in_dw
= 0;
204 chunk
.size_in_dw
= pool
->size_in_dw
;
205 chunk
.prev
= chunk
.next
= NULL
;
206 compute_memory_transfer(pool
, pipe
, device_to_host
, &chunk
,
207 pool
->shadow
, 0, pool
->size_in_dw
*4);
211 * Allocates pending allocations in the pool
213 void compute_memory_finalize_pending(struct compute_memory_pool
* pool
,
214 struct pipe_context
* pipe
)
216 struct compute_memory_item
*pending_list
= NULL
, *end_p
= NULL
;
217 struct compute_memory_item
*item
, *next
;
219 int64_t allocated
= 0;
220 int64_t unallocated
= 0;
222 int64_t start_in_dw
= 0;
224 COMPUTE_DBG(pool
->screen
, "* compute_memory_finalize_pending()\n");
226 for (item
= pool
->item_list
; item
; item
= item
->next
) {
227 COMPUTE_DBG(pool
->screen
, " + list: offset = %i id = %i size = %i "
228 "(%i bytes)\n",item
->start_in_dw
, item
->id
,
229 item
->size_in_dw
, item
->size_in_dw
* 4);
232 /* Search through the list of memory items in the pool */
233 for (item
= pool
->item_list
; item
; item
= next
) {
236 /* Check if the item is pending. */
237 if (item
->start_in_dw
== -1) {
238 /* It is pending, so add it to the pending_list... */
246 /* ... and then remove it from the item list. */
248 item
->prev
->next
= next
;
251 pool
->item_list
= next
;
255 next
->prev
= item
->prev
;
258 /* This sequence makes the item be at the end of the list */
263 /* Update the amount of space we will need to allocate. */
264 unallocated
+= item
->size_in_dw
+1024;
267 /* The item is not pendng, so update the amount of space
268 * that has already been allocated. */
269 allocated
+= item
->size_in_dw
;
273 /* If we require more space than the size of the pool, then grow the
276 * XXX: I'm pretty sure this won't work. Imagine this scenario:
284 * Allocated size = 150
285 * Pending Item D Size = 200
287 * In this case, there are 300 units of free space in the pool, but
288 * they aren't contiguous, so it will be impossible to allocate Item D.
290 if (pool
->size_in_dw
< allocated
+unallocated
) {
291 compute_memory_grow_pool(pool
, pipe
, allocated
+unallocated
);
294 /* Loop through all the pending items, allocate space for them and
295 * add them back to the item_list. */
296 for (item
= pending_list
; item
; item
= next
) {
299 /* Search for free space in the pool for this item. */
300 while ((start_in_dw
=compute_memory_prealloc_chunk(pool
,
301 item
->size_in_dw
)) == -1) {
302 int64_t need
= item
->size_in_dw
+2048 -
303 (pool
->size_in_dw
- allocated
);
305 need
+= 1024 - (need
% 1024);
308 compute_memory_grow_pool(pool
,
310 pool
->size_in_dw
+ need
);
313 need
= pool
->size_in_dw
/ 10;
314 need
+= 1024 - (need
% 1024);
315 compute_memory_grow_pool(pool
,
317 pool
->size_in_dw
+ need
);
320 COMPUTE_DBG(pool
->screen
, " + Found space for Item %p id = %u "
321 "start_in_dw = %u (%u bytes) size_in_dw = %u (%u bytes)\n",
322 item
, item
->id
, start_in_dw
, start_in_dw
* 4,
323 item
->size_in_dw
, item
->size_in_dw
* 4);
325 item
->start_in_dw
= start_in_dw
;
329 if (pool
->item_list
) {
330 struct compute_memory_item
*pos
;
332 pos
= compute_memory_postalloc_chunk(pool
, start_in_dw
);
335 item
->next
= pos
->next
;
338 item
->next
->prev
= item
;
341 /* Add item to the front of the list */
342 item
->next
= pool
->item_list
;
343 item
->prev
= pool
->item_list
->prev
;
344 pool
->item_list
->prev
= item
;
345 pool
->item_list
= item
;
349 pool
->item_list
= item
;
352 allocated
+= item
->size_in_dw
;
357 void compute_memory_free(struct compute_memory_pool
* pool
, int64_t id
)
359 struct compute_memory_item
*item
, *next
;
361 COMPUTE_DBG(pool
->screen
, "* compute_memory_free() id + %ld \n", id
);
363 for (item
= pool
->item_list
; item
; item
= next
) {
366 if (item
->id
== id
) {
368 item
->prev
->next
= item
->next
;
371 pool
->item_list
= item
->next
;
375 item
->next
->prev
= item
->prev
;
384 fprintf(stderr
, "Internal error, invalid id %"PRIi64
" "
385 "for compute_memory_free\n", id
);
387 assert(0 && "error");
391 * Creates pending allocations
393 struct compute_memory_item
* compute_memory_alloc(
394 struct compute_memory_pool
* pool
,
397 struct compute_memory_item
*new_item
= NULL
, *last_item
= NULL
;
399 COMPUTE_DBG(pool
->screen
, "* compute_memory_alloc() size_in_dw = %ld (%ld bytes)\n",
400 size_in_dw
, 4 * size_in_dw
);
402 new_item
= (struct compute_memory_item
*)
403 CALLOC(sizeof(struct compute_memory_item
), 1);
404 new_item
->size_in_dw
= size_in_dw
;
405 new_item
->start_in_dw
= -1; /* mark pending */
406 new_item
->id
= pool
->next_id
++;
407 new_item
->pool
= pool
;
409 if (pool
->item_list
) {
410 for (last_item
= pool
->item_list
; last_item
->next
;
411 last_item
= last_item
->next
);
413 last_item
->next
= new_item
;
414 new_item
->prev
= last_item
;
417 pool
->item_list
= new_item
;
420 COMPUTE_DBG(pool
->screen
, " + Adding item %p id = %u size = %u (%u bytes)\n",
421 new_item
, new_item
->id
, new_item
->size_in_dw
,
422 new_item
->size_in_dw
* 4);
427 * Transfer data host<->device, offset and size is in bytes
429 void compute_memory_transfer(
430 struct compute_memory_pool
* pool
,
431 struct pipe_context
* pipe
,
433 struct compute_memory_item
* chunk
,
438 int64_t aligned_size
= pool
->size_in_dw
;
439 struct pipe_resource
* gart
= (struct pipe_resource
*)pool
->bo
;
440 int64_t internal_offset
= chunk
->start_in_dw
*4 + offset_in_chunk
;
442 struct pipe_transfer
*xfer
;
447 COMPUTE_DBG(pool
->screen
, "* compute_memory_transfer() device_to_host = %d, "
448 "offset_in_chunk = %d, size = %d\n", device_to_host
,
449 offset_in_chunk
, size
);
451 if (device_to_host
) {
452 map
= pipe
->transfer_map(pipe
, gart
, 0, PIPE_TRANSFER_READ
,
453 &(struct pipe_box
) { .width
= aligned_size
,
454 .height
= 1, .depth
= 1 }, &xfer
);
457 memcpy(data
, map
+ internal_offset
, size
);
458 pipe
->transfer_unmap(pipe
, xfer
);
460 map
= pipe
->transfer_map(pipe
, gart
, 0, PIPE_TRANSFER_WRITE
,
461 &(struct pipe_box
) { .width
= aligned_size
,
462 .height
= 1, .depth
= 1 }, &xfer
);
465 memcpy(map
+ internal_offset
, data
, size
);
466 pipe
->transfer_unmap(pipe
, xfer
);
471 * Transfer data between chunk<->data, it is for VRAM<->GART transfers
473 void compute_memory_transfer_direct(
474 struct compute_memory_pool
* pool
,
476 struct compute_memory_item
* chunk
,
477 struct r600_resource
* data
,