src/gallium/drivers/r600/compute_memory_pool.c

   1 /*
   2  * Permission is hereby granted, free of charge, to any person obtaining a
   3  * copy of this software and associated documentation files (the "Software"),
   4  * to deal in the Software without restriction, including without limitation
   5  * on the rights to use, copy, modify, merge, publish, distribute, sub
   6  * license, and/or sell copies of the Software, and to permit persons to whom
   7  * the Software is furnished to do so, subject to the following conditions:
   8  *
   9  * The above copyright notice and this permission notice (including the next
  10  * paragraph) shall be included in all copies or substantial portions of the
  11  * Software.
  12  *
  13  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  14  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  15  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  16  * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
  17  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  18  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  19  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  20  *
  21  * Authors:
  22  *      Adam Rak <adam.rak@streamnovation.com>
  23  */
  24
  25 #include "pipe/p_defines.h"
  26 #include "pipe/p_state.h"
  27 #include "pipe/p_context.h"
  28 #include "util/u_blitter.h"
  29 #include "util/u_double_list.h"
  30 #include "util/u_transfer.h"
  31 #include "util/u_surface.h"
  32 #include "util/u_pack_color.h"
  33 #include "util/u_memory.h"
  34 #include "util/u_inlines.h"
  35 #include "util/u_framebuffer.h"
  36 #include "r600.h"
  37 #include "r600_resource.h"
  38 #include "r600_shader.h"
  39 #include "r600_pipe.h"
  40 #include "r600_formats.h"
  41 #include "compute_memory_pool.h"
  42 #include "evergreen_compute.h"
  43 #include "evergreen_compute_internal.h"
  44 #include <inttypes.h>
  45
  46 /**
  47  * Creates a new pool
  48  */
  49 struct compute_memory_pool* compute_memory_pool_new(
  50         struct r600_screen * rscreen)
  51 {
  52         struct compute_memory_pool* pool = (struct compute_memory_pool*)
  53                                 CALLOC(sizeof(struct compute_memory_pool), 1);
  54
  55         COMPUTE_DBG(rscreen, "* compute_memory_pool_new()\n");
  56
  57         pool->screen = rscreen;
  58         return pool;
  59 }
  60
  61 static void compute_memory_pool_init(struct compute_memory_pool * pool,
  62         unsigned initial_size_in_dw)
  63 {
  64
  65         COMPUTE_DBG(pool->screen, "* compute_memory_pool_init() initial_size_in_dw = %ld\n",
  66                 initial_size_in_dw);
  67
  68         pool->shadow = (uint32_t*)CALLOC(initial_size_in_dw, 4);
  69         pool->next_id = 1;
  70         pool->size_in_dw = initial_size_in_dw;
  71         pool->bo = (struct r600_resource*)r600_compute_buffer_alloc_vram(pool->screen,
  72                                                         pool->size_in_dw * 4);
  73 }
  74
  75 /**
  76  * Frees all stuff in the pool and the pool struct itself too
  77  */
  78 void compute_memory_pool_delete(struct compute_memory_pool* pool)
  79 {
  80         COMPUTE_DBG(pool->screen, "* compute_memory_pool_delete()\n");
  81         free(pool->shadow);
  82         if (pool->bo) {
  83                 pool->screen->screen.resource_destroy((struct pipe_screen *)
  84                         pool->screen, (struct pipe_resource *)pool->bo);
  85         }
  86         free(pool);
  87 }
  88
  89 /**
  90  * Searches for an empty space in the pool, return with the pointer to the
  91  * allocatable space in the pool, returns -1 on failure.
  92  */
  93 int64_t compute_memory_prealloc_chunk(
  94         struct compute_memory_pool* pool,
  95         int64_t size_in_dw)
  96 {
  97         struct compute_memory_item *item;
  98
  99         int last_end = 0;
 100
 101         assert(size_in_dw <= pool->size_in_dw);
 102
 103         COMPUTE_DBG(pool->screen, "* compute_memory_prealloc_chunk() size_in_dw = %ld\n",
 104                 size_in_dw);
 105
 106         for (item = pool->item_list; item; item = item->next) {
 107                 if (item->start_in_dw > -1) {
 108                         if (item->start_in_dw-last_end > size_in_dw) {
 109                                 return last_end;
 110                         }
 111
 112                         last_end = item->start_in_dw + item->size_in_dw;
 113                         last_end += (1024 - last_end % 1024);
 114                 }
 115         }
 116
 117         if (pool->size_in_dw - last_end < size_in_dw) {
 118                 return -1;
 119         }
 120
 121         return last_end;
 122 }
 123
 124 /**
 125  *  Search for the chunk where we can link our new chunk after it.
 126  */
 127 struct compute_memory_item* compute_memory_postalloc_chunk(
 128         struct compute_memory_pool* pool,
 129         int64_t start_in_dw)
 130 {
 131         struct compute_memory_item* item;
 132
 133         COMPUTE_DBG(pool->screen, "* compute_memory_postalloc_chunck() start_in_dw = %ld\n",
 134                 start_in_dw);
 135
 136         /* Check if we can insert it in the front of the list */
 137         if (pool->item_list && pool->item_list->start_in_dw > start_in_dw) {
 138                 return NULL;
 139         }
 140
 141         for (item = pool->item_list; item; item = item->next) {
 142                 if (item->next) {
 143                         if (item->start_in_dw < start_in_dw
 144                                 && item->next->start_in_dw > start_in_dw) {
 145                                 return item;
 146                         }
 147                 }
 148                 else {
 149                         /* end of chain */
 150                         assert(item->start_in_dw < start_in_dw);
 151                         return item;
 152                 }
 153         }
 154
 155         assert(0 && "unreachable");
 156         return NULL;
 157 }
 158
 159 /**
 160  * Reallocates pool, conserves data
 161  */
 162 void compute_memory_grow_pool(struct compute_memory_pool* pool,
 163         struct pipe_context * pipe, int new_size_in_dw)
 164 {
 165         COMPUTE_DBG(pool->screen, "* compute_memory_grow_pool() new_size_in_dw = %d\n",
 166                 new_size_in_dw);
 167
 168         assert(new_size_in_dw >= pool->size_in_dw);
 169
 170         if (!pool->bo) {
 171                 compute_memory_pool_init(pool, MAX2(new_size_in_dw, 1024 * 16));
 172         } else {
 173                 new_size_in_dw += 1024 - (new_size_in_dw % 1024);
 174
 175                 COMPUTE_DBG(pool->screen, "  Aligned size = %d\n", new_size_in_dw);
 176
 177                 compute_memory_shadow(pool, pipe, 1);
 178                 pool->shadow = realloc(pool->shadow, new_size_in_dw*4);
 179                 pool->size_in_dw = new_size_in_dw;
 180                 pool->screen->screen.resource_destroy(
 181                         (struct pipe_screen *)pool->screen,
 182                         (struct pipe_resource *)pool->bo);
 183                 pool->bo = (struct r600_resource*)r600_compute_buffer_alloc_vram(
 184                                                         pool->screen,
 185                                                         pool->size_in_dw * 4);
 186                 compute_memory_shadow(pool, pipe, 0);
 187         }
 188 }
 189
 190 /**
 191  * Copy pool from device to host, or host to device.
 192  */
 193 void compute_memory_shadow(struct compute_memory_pool* pool,
 194         struct pipe_context * pipe, int device_to_host)
 195 {
 196         struct compute_memory_item chunk;
 197
 198         COMPUTE_DBG(pool->screen, "* compute_memory_shadow() device_to_host = %d\n",
 199                 device_to_host);
 200
 201         chunk.id = 0;
 202         chunk.start_in_dw = 0;
 203         chunk.size_in_dw = pool->size_in_dw;
 204         chunk.prev = chunk.next = NULL;
 205         compute_memory_transfer(pool, pipe, device_to_host, &chunk,
 206                                 pool->shadow, 0, pool->size_in_dw*4);
 207 }
 208
 209 /**
 210  * Allocates pending allocations in the pool
 211  */
 212 void compute_memory_finalize_pending(struct compute_memory_pool* pool,
 213         struct pipe_context * pipe)
 214 {
 215         struct compute_memory_item *pending_list = NULL, *end_p = NULL;
 216         struct compute_memory_item *item, *next;
 217
 218         int64_t allocated = 0;
 219         int64_t unallocated = 0;
 220
 221         int64_t start_in_dw = 0;
 222
 223         COMPUTE_DBG(pool->screen, "* compute_memory_finalize_pending()\n");
 224
 225         for (item = pool->item_list; item; item = item->next) {
 226                 COMPUTE_DBG(pool->screen, "  + list: offset = %i id = %i size = %i "
 227                         "(%i bytes)\n",item->start_in_dw, item->id,
 228                         item->size_in_dw, item->size_in_dw * 4);
 229         }
 230
 231         /* Search through the list of memory items in the pool */
 232         for (item = pool->item_list; item; item = next) {
 233                 next = item->next;
 234
 235                 /* Check if the item is pending. */
 236                 if (item->start_in_dw == -1) {
 237                         /* It is pending, so add it to the pending_list... */
 238                         if (end_p) {
 239                                 end_p->next = item;
 240                         }
 241                         else {
 242                                 pending_list = item;
 243                         }
 244
 245                         /* ... and then remove it from the item list. */
 246                         if (item->prev) {
 247                                 item->prev->next = next;
 248                         }
 249                         else {
 250                                 pool->item_list = next;
 251                         }
 252
 253                         if (next) {
 254                                 next->prev = item->prev;
 255                         }
 256
 257                         /* This sequence makes the item be at the end of the list */
 258                         item->prev = end_p;
 259                         item->next = NULL;
 260                         end_p = item;
 261
 262                         /* Update the amount of space we will need to allocate. */
 263                         unallocated += item->size_in_dw+1024;
 264                 }
 265                 else {
 266                         /* The item is not pendng, so update the amount of space
 267                          * that has already been allocated. */
 268                         allocated += item->size_in_dw;
 269                 }
 270         }
 271
 272         /* If we require more space than the size of the pool, then grow the
 273          * pool.
 274          *
 275          * XXX: I'm pretty sure this won't work.  Imagine this scenario:
 276          *
 277          * Offset Item Size
 278          *   0    A    50
 279          * 200    B    50
 280          * 400    C    50
 281          *
 282          * Total size = 450
 283          * Allocated size = 150
 284          * Pending Item D Size = 200
 285          *
 286          * In this case, there are 300 units of free space in the pool, but
 287          * they aren't contiguous, so it will be impossible to allocate Item D.
 288          */
 289         if (pool->size_in_dw < allocated+unallocated) {
 290                 compute_memory_grow_pool(pool, pipe, allocated+unallocated);
 291         }
 292
 293         /* Loop through all the pending items, allocate space for them and
 294          * add them back to the item_list. */
 295         for (item = pending_list; item; item = next) {
 296                 next = item->next;
 297
 298                 /* Search for free space in the pool for this item. */
 299                 while ((start_in_dw=compute_memory_prealloc_chunk(pool,
 300                                                 item->size_in_dw)) == -1) {
 301                         int64_t need = item->size_in_dw+2048 -
 302                                                 (pool->size_in_dw - allocated);
 303
 304                         need += 1024 - (need % 1024);
 305
 306                         if (need > 0) {
 307                                 compute_memory_grow_pool(pool,
 308                                                 pipe,
 309                                                 pool->size_in_dw + need);
 310                         }
 311                         else {
 312                                 need = pool->size_in_dw / 10;
 313                                 need += 1024 - (need % 1024);
 314                                 compute_memory_grow_pool(pool,
 315                                                 pipe,
 316                                                 pool->size_in_dw + need);
 317                         }
 318                 }
 319                 COMPUTE_DBG(pool->screen, "  + Found space for Item %p id = %u "
 320                         "start_in_dw = %u (%u bytes) size_in_dw = %u (%u bytes)\n",
 321                         item, item->id, start_in_dw, start_in_dw * 4,
 322                         item->size_in_dw, item->size_in_dw * 4);
 323
 324                 item->start_in_dw = start_in_dw;
 325                 item->next = NULL;
 326                 item->prev = NULL;
 327
 328                 if (pool->item_list) {
 329                         struct compute_memory_item *pos;
 330
 331                         pos = compute_memory_postalloc_chunk(pool, start_in_dw);
 332                         if (pos) {
 333                                 item->prev = pos;
 334                                 item->next = pos->next;
 335                                 pos->next = item;
 336                                 if (item->next) {
 337                                         item->next->prev = item;
 338                                 }
 339                         } else {
 340                                 /* Add item to the front of the list */
 341                                 item->next = pool->item_list->next;
 342                                 if (pool->item_list->next) {
 343                                         pool->item_list->next->prev = item;
 344                                 }
 345                                 item->prev = pool->item_list->prev;
 346                                 if (pool->item_list->prev) {
 347                                         pool->item_list->prev->next = item;
 348                                 }
 349                                 pool->item_list = item;
 350                         }
 351                 }
 352                 else {
 353                         pool->item_list = item;
 354                 }
 355
 356                 allocated += item->size_in_dw;
 357         }
 358 }
 359
 360
 361 void compute_memory_free(struct compute_memory_pool* pool, int64_t id)
 362 {
 363         struct compute_memory_item *item, *next;
 364
 365         COMPUTE_DBG(pool->screen, "* compute_memory_free() id + %ld \n", id);
 366
 367         for (item = pool->item_list; item; item = next) {
 368                 next = item->next;
 369
 370                 if (item->id == id) {
 371                         if (item->prev) {
 372                                 item->prev->next = item->next;
 373                         }
 374                         else {
 375                                 pool->item_list = item->next;
 376                         }
 377
 378                         if (item->next) {
 379                                 item->next->prev = item->prev;
 380                         }
 381
 382                         free(item);
 383
 384                         return;
 385                 }
 386         }
 387
 388         fprintf(stderr, "Internal error, invalid id %"PRIi64" "
 389                 "for compute_memory_free\n", id);
 390
 391         assert(0 && "error");
 392 }
 393
 394 /**
 395  * Creates pending allocations
 396  */
 397 struct compute_memory_item* compute_memory_alloc(
 398         struct compute_memory_pool* pool,
 399         int64_t size_in_dw)
 400 {
 401         struct compute_memory_item *new_item = NULL, *last_item = NULL;
 402
 403         COMPUTE_DBG(pool->screen, "* compute_memory_alloc() size_in_dw = %ld (%ld bytes)\n",
 404                         size_in_dw, 4 * size_in_dw);
 405
 406         new_item = (struct compute_memory_item *)
 407                                 CALLOC(sizeof(struct compute_memory_item), 1);
 408         new_item->size_in_dw = size_in_dw;
 409         new_item->start_in_dw = -1; /* mark pending */
 410         new_item->id = pool->next_id++;
 411         new_item->pool = pool;
 412
 413         if (pool->item_list) {
 414                 for (last_item = pool->item_list; last_item->next;
 415                                                 last_item = last_item->next);
 416
 417                 last_item->next = new_item;
 418                 new_item->prev = last_item;
 419         }
 420         else {
 421                 pool->item_list = new_item;
 422         }
 423
 424         COMPUTE_DBG(pool->screen, "  + Adding item %p id = %u size = %u (%u bytes)\n",
 425                         new_item, new_item->id, new_item->size_in_dw,
 426                         new_item->size_in_dw * 4);
 427         return new_item;
 428 }
 429
 430 /**
 431  * Transfer data host<->device, offset and size is in bytes
 432  */
 433 void compute_memory_transfer(
 434         struct compute_memory_pool* pool,
 435         struct pipe_context * pipe,
 436         int device_to_host,
 437         struct compute_memory_item* chunk,
 438         void* data,
 439         int offset_in_chunk,
 440         int size)
 441 {
 442         int64_t aligned_size = pool->size_in_dw;
 443         struct pipe_resource* gart = (struct pipe_resource*)pool->bo;
 444         int64_t internal_offset = chunk->start_in_dw*4 + offset_in_chunk;
 445
 446         struct pipe_transfer *xfer;
 447         uint32_t *map;
 448
 449         assert(gart);
 450
 451         COMPUTE_DBG(pool->screen, "* compute_memory_transfer() device_to_host = %d, "
 452                 "offset_in_chunk = %d, size = %d\n", device_to_host,
 453                 offset_in_chunk, size);
 454
 455         if (device_to_host) {
 456                 map = pipe->transfer_map(pipe, gart, 0, PIPE_TRANSFER_READ,
 457                         &(struct pipe_box) { .width = aligned_size,
 458                         .height = 1, .depth = 1 }, &xfer);
 459                 assert(xfer);
 460                 assert(map);
 461                 memcpy(data, map + internal_offset, size);
 462                 pipe->transfer_unmap(pipe, xfer);
 463         } else {
 464                 map = pipe->transfer_map(pipe, gart, 0, PIPE_TRANSFER_WRITE,
 465                         &(struct pipe_box) { .width = aligned_size,
 466                         .height = 1, .depth = 1 }, &xfer);
 467                 assert(xfer);
 468                 assert(map);
 469                 memcpy(map + internal_offset, data, size);
 470                 pipe->transfer_unmap(pipe, xfer);
 471         }
 472 }
 473
 474 /**
 475  * Transfer data between chunk<->data, it is for VRAM<->GART transfers
 476  */
 477 void compute_memory_transfer_direct(
 478         struct compute_memory_pool* pool,
 479         int chunk_to_data,
 480         struct compute_memory_item* chunk,
 481         struct r600_resource* data,
 482         int offset_in_chunk,
 483         int offset_in_data,
 484         int size)
 485 {
 486         ///TODO: DMA
 487 }