src/gallium/drivers/r600/compute_memory_pool.c

   1 /*
   2  * Permission is hereby granted, free of charge, to any person obtaining a
   3  * copy of this software and associated documentation files (the "Software"),
   4  * to deal in the Software without restriction, including without limitation
   5  * on the rights to use, copy, modify, merge, publish, distribute, sub
   6  * license, and/or sell copies of the Software, and to permit persons to whom
   7  * the Software is furnished to do so, subject to the following conditions:
   8  *
   9  * The above copyright notice and this permission notice (including the next
  10  * paragraph) shall be included in all copies or substantial portions of the
  11  * Software.
  12  *
  13  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  14  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  15  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  16  * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
  17  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  18  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  19  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  20  *
  21  * Authors:
  22  *      Adam Rak <adam.rak@streamnovation.com>
  23  */
  24
  25 #include "pipe/p_defines.h"
  26 #include "pipe/p_state.h"
  27 #include "pipe/p_context.h"
  28 #include "util/u_blitter.h"
  29 #include "util/u_double_list.h"
  30 #include "util/u_transfer.h"
  31 #include "util/u_surface.h"
  32 #include "util/u_pack_color.h"
  33 #include "util/u_memory.h"
  34 #include "util/u_inlines.h"
  35 #include "util/u_framebuffer.h"
  36 #include "r600.h"
  37 #include "r600_resource.h"
  38 #include "r600_shader.h"
  39 #include "r600_pipe.h"
  40 #include "r600_formats.h"
  41 #include "compute_memory_pool.h"
  42 #include "evergreen_compute_internal.h"
  43 #include <inttypes.h>
  44
  45 /**
  46  * Creates a new pool
  47  */
  48 struct compute_memory_pool* compute_memory_pool_new(
  49         struct r600_screen * rscreen)
  50 {
  51         struct compute_memory_pool* pool = (struct compute_memory_pool*)
  52                                 CALLOC(sizeof(struct compute_memory_pool), 1);
  53
  54         COMPUTE_DBG("* compute_memory_pool_new()\n");
  55
  56         pool->screen = rscreen;
  57         return pool;
  58 }
  59
  60 static void compute_memory_pool_init(struct compute_memory_pool * pool,
  61         unsigned initial_size_in_dw)
  62 {
  63
  64         COMPUTE_DBG("* compute_memory_pool_init() initial_size_in_dw = %ld\n",
  65                 initial_size_in_dw);
  66
  67         /* XXX: pool->shadow is used when the buffer needs to be resized, but
  68          * resizing does not work at the moment.
  69          * pool->shadow = (uint32_t*)CALLOC(4, pool->size_in_dw);
  70          */
  71         pool->next_id = 1;
  72         pool->size_in_dw = initial_size_in_dw;
  73         pool->bo = (struct r600_resource*)r600_compute_buffer_alloc_vram(pool->screen,
  74                                                         pool->size_in_dw * 4);
  75 }
  76
  77 /**
  78  * Frees all stuff in the pool and the pool struct itself too
  79  */
  80 void compute_memory_pool_delete(struct compute_memory_pool* pool)
  81 {
  82         COMPUTE_DBG("* compute_memory_pool_delete()\n");
  83         free(pool->shadow);
  84         if (pool->bo) {
  85                 pool->screen->screen.resource_destroy((struct pipe_screen *)
  86                         pool->screen, (struct pipe_resource *)pool->bo);
  87         }
  88         free(pool);
  89 }
  90
  91 /**
  92  * Searches for an empty space in the pool, return with the pointer to the
  93  * allocatable space in the pool, returns -1 on failure.
  94  */
  95 int64_t compute_memory_prealloc_chunk(
  96         struct compute_memory_pool* pool,
  97         int64_t size_in_dw)
  98 {
  99         assert(size_in_dw <= pool->size_in_dw);
 100
 101         struct compute_memory_item *item;
 102
 103         int last_end = 0;
 104
 105         COMPUTE_DBG("* compute_memory_prealloc_chunk() size_in_dw = %ld\n",
 106                 size_in_dw);
 107
 108         for (item = pool->item_list; item; item = item->next) {
 109                 if (item->start_in_dw > -1) {
 110                         if (item->start_in_dw-last_end > size_in_dw) {
 111                                 return last_end;
 112                         }
 113
 114                         last_end = item->start_in_dw + item->size_in_dw;
 115                         last_end += (1024 - last_end % 1024);
 116                 }
 117         }
 118
 119         if (pool->size_in_dw - last_end < size_in_dw) {
 120                 return -1;
 121         }
 122
 123         return last_end;
 124 }
 125
 126 /**
 127  *  Search for the chunk where we can link our new chunk after it.
 128  */
 129 struct compute_memory_item* compute_memory_postalloc_chunk(
 130         struct compute_memory_pool* pool,
 131         int64_t start_in_dw)
 132 {
 133         struct compute_memory_item* item;
 134
 135         COMPUTE_DBG("* compute_memory_postalloc_chunck() start_in_dw = %ld\n",
 136                 start_in_dw);
 137
 138         for (item = pool->item_list; item; item = item->next) {
 139                 if (item->next) {
 140                         if (item->start_in_dw < start_in_dw
 141                                 && item->next->start_in_dw > start_in_dw) {
 142                                 return item;
 143                         }
 144                 }
 145                 else {
 146                         /* end of chain */
 147                         assert(item->start_in_dw < start_in_dw);
 148                         return item;
 149                 }
 150         }
 151
 152         assert(0 && "unreachable");
 153         return NULL;
 154 }
 155
 156 /**
 157  * Reallocates pool, conserves data
 158  */
 159 void compute_memory_grow_pool(struct compute_memory_pool* pool,
 160         struct pipe_context * pipe, int new_size_in_dw)
 161 {
 162         COMPUTE_DBG("* compute_memory_grow_pool() new_size_in_dw = %d\n",
 163                 new_size_in_dw);
 164
 165         assert(new_size_in_dw >= pool->size_in_dw);
 166
 167         assert(!pool->bo && "Growing the global memory pool is not yet "
 168                 "supported.  You will see this message if you are trying to"
 169                 "use more than 64 kb of memory");
 170
 171         if (!pool->bo) {
 172                 compute_memory_pool_init(pool, MAX2(new_size_in_dw, 1024 * 16));
 173         } else {
 174                 /* XXX: Growing memory pools does not work at the moment.  I think
 175                  * it is because we are using fragment shaders to copy data to
 176                  * the new texture and some of the compute registers are being
 177                  * included in the 3D command stream. */
 178                 fprintf(stderr, "Warning: growing the global memory pool to"
 179                                 "more than 64 kb is not yet supported\n");
 180                 new_size_in_dw += 1024 - (new_size_in_dw % 1024);
 181
 182                 COMPUTE_DBG("  Aligned size = %d\n", new_size_in_dw);
 183
 184                 compute_memory_shadow(pool, pipe, 1);
 185                 pool->shadow = realloc(pool->shadow, new_size_in_dw*4);
 186                 pool->size_in_dw = new_size_in_dw;
 187                 pool->screen->screen.resource_destroy(
 188                         (struct pipe_screen *)pool->screen,
 189                         (struct pipe_resource *)pool->bo);
 190                 pool->bo = (struct r600_resource*)r600_compute_buffer_alloc_vram(
 191                                                         pool->screen,
 192                                                         pool->size_in_dw * 4);
 193                 compute_memory_shadow(pool, pipe, 0);
 194         }
 195 }
 196
 197 /**
 198  * Copy pool from device to host, or host to device.
 199  */
 200 void compute_memory_shadow(struct compute_memory_pool* pool,
 201         struct pipe_context * pipe, int device_to_host)
 202 {
 203         struct compute_memory_item chunk;
 204
 205         COMPUTE_DBG("* compute_memory_shadow() device_to_host = %d\n",
 206                 device_to_host);
 207
 208         chunk.id = 0;
 209         chunk.start_in_dw = 0;
 210         chunk.size_in_dw = pool->size_in_dw;
 211         chunk.prev = chunk.next = NULL;
 212         compute_memory_transfer(pool, pipe, device_to_host, &chunk,
 213                                 pool->shadow, 0, pool->size_in_dw*4);
 214 }
 215
 216 /**
 217  * Allocates pending allocations in the pool
 218  */
 219 void compute_memory_finalize_pending(struct compute_memory_pool* pool,
 220         struct pipe_context * pipe)
 221 {
 222         struct compute_memory_item *pending_list = NULL, *end_p = NULL;
 223         struct compute_memory_item *item, *next;
 224
 225         int64_t allocated = 0;
 226         int64_t unallocated = 0;
 227
 228         COMPUTE_DBG("* compute_memory_finalize_pending()\n");
 229
 230         for (item = pool->item_list; item; item = item->next) {
 231                 COMPUTE_DBG("  + list: offset = %i id = %i size = %i "
 232                         "(%i bytes)\n",item->start_in_dw, item->id,
 233                         item->size_in_dw, item->size_in_dw * 4);
 234         }
 235
 236         /* Search through the list of memory items in the pool */
 237         for (item = pool->item_list; item; item = next) {
 238                 next = item->next;
 239
 240                 /* Check if the item is pending. */
 241                 if (item->start_in_dw == -1) {
 242                         /* It is pending, so add it to the pending_list... */
 243                         if (end_p) {
 244                                 end_p->next = item;
 245                         }
 246                         else {
 247                                 pending_list = item;
 248                         }
 249
 250                         /* ... and then remove it from the item list. */
 251                         if (item->prev) {
 252                                 item->prev->next = next;
 253                         }
 254                         else {
 255                                 pool->item_list = next;
 256                         }
 257
 258                         if (next) {
 259                                 next->prev = item->prev;
 260                         }
 261
 262                         /* This sequence makes the item be at the end of the list */
 263                         item->prev = end_p;
 264                         item->next = NULL;
 265                         end_p = item;
 266
 267                         /* Update the amount of space we will need to allocate. */
 268                         unallocated += item->size_in_dw+1024;
 269                 }
 270                 else {
 271                         /* The item is not pendng, so update the amount of space
 272                          * that has already been allocated. */
 273                         allocated += item->size_in_dw;
 274                 }
 275         }
 276
 277         /* If we require more space than the size of the pool, then grow the
 278          * pool.
 279          *
 280          * XXX: I'm pretty sure this won't work.  Imagine this scenario:
 281          *
 282          * Offset Item Size
 283          *   0    A    50
 284          * 200    B    50
 285          * 400    C    50
 286          *
 287          * Total size = 450
 288          * Allocated size = 150
 289          * Pending Item D Size = 200
 290          *
 291          * In this case, there are 300 units of free space in the pool, but
 292          * they aren't contiguous, so it will be impossible to allocate Item D.
 293          */
 294         if (pool->size_in_dw < allocated+unallocated) {
 295                 compute_memory_grow_pool(pool, pipe, allocated+unallocated);
 296         }
 297
 298         /* Loop through all the pending items, allocate space for them and
 299          * add them back to the item_list. */
 300         for (item = pending_list; item; item = next) {
 301                 next = item->next;
 302
 303                 int64_t start_in_dw;
 304
 305                 /* Search for free space in the pool for this item. */
 306                 while ((start_in_dw=compute_memory_prealloc_chunk(pool,
 307                                                 item->size_in_dw)) == -1) {
 308                         int64_t need = item->size_in_dw+2048 -
 309                                                 (pool->size_in_dw - allocated);
 310
 311                         need += 1024 - (need % 1024);
 312
 313                         if (need > 0) {
 314                                 compute_memory_grow_pool(pool,
 315                                                 pipe,
 316                                                 pool->size_in_dw + need);
 317                         }
 318                         else {
 319                                 need = pool->size_in_dw / 10;
 320                                 need += 1024 - (need % 1024);
 321                                 compute_memory_grow_pool(pool,
 322                                                 pipe,
 323                                                 pool->size_in_dw + need);
 324                         }
 325                 }
 326                 COMPUTE_DBG("  + Found space for Item %p id = %u "
 327                         "start_in_dw = %u (%u bytes) size_in_dw = %u (%u bytes)\n",
 328                         item, item->id, start_in_dw, start_in_dw * 4,
 329                         item->size_in_dw, item->size_in_dw * 4);
 330
 331                 item->start_in_dw = start_in_dw;
 332                 item->next = NULL;
 333                 item->prev = NULL;
 334
 335                 if (pool->item_list) {
 336                         struct compute_memory_item *pos;
 337
 338                         pos = compute_memory_postalloc_chunk(pool, start_in_dw);
 339                         item->prev = pos;
 340                         item->next = pos->next;
 341                         pos->next = item;
 342
 343                         if (item->next) {
 344                                 item->next->prev = item;
 345                         }
 346                 }
 347                 else {
 348                         pool->item_list = item;
 349                 }
 350
 351                 allocated += item->size_in_dw;
 352         }
 353 }
 354
 355
 356 void compute_memory_free(struct compute_memory_pool* pool, int64_t id)
 357 {
 358         struct compute_memory_item *item, *next;
 359
 360         COMPUTE_DBG("* compute_memory_free() id + %ld \n", id);
 361
 362         for (item = pool->item_list; item; item = next) {
 363                 next = item->next;
 364
 365                 if (item->id == id) {
 366                         if (item->prev) {
 367                                 item->prev->next = item->next;
 368                         }
 369                         else {
 370                                 pool->item_list = item->next;
 371                         }
 372
 373                         if (item->next) {
 374                                 item->next->prev = item->prev;
 375                         }
 376
 377                         free(item);
 378
 379                         return;
 380                 }
 381         }
 382
 383         fprintf(stderr, "Internal error, invalid id %"PRIi64" "
 384                 "for compute_memory_free\n", id);
 385
 386         assert(0 && "error");
 387 }
 388
 389 /**
 390  * Creates pending allocations
 391  */
 392 struct compute_memory_item* compute_memory_alloc(
 393         struct compute_memory_pool* pool,
 394         int64_t size_in_dw)
 395 {
 396         struct compute_memory_item *new_item;
 397
 398         COMPUTE_DBG("* compute_memory_alloc() size_in_dw = %ld (%ld bytes)\n",
 399                         size_in_dw, 4 * size_in_dw);
 400
 401         new_item = (struct compute_memory_item *)
 402                                 CALLOC(sizeof(struct compute_memory_item), 1);
 403         new_item->size_in_dw = size_in_dw;
 404         new_item->start_in_dw = -1; /* mark pending */
 405         new_item->id = pool->next_id++;
 406         new_item->pool = pool;
 407
 408         struct compute_memory_item *last_item;
 409
 410         if (pool->item_list) {
 411                 for (last_item = pool->item_list; last_item->next;
 412                                                 last_item = last_item->next);
 413
 414                 last_item->next = new_item;
 415                 new_item->prev = last_item;
 416         }
 417         else {
 418                 pool->item_list = new_item;
 419         }
 420
 421         COMPUTE_DBG("  + Adding item %p id = %u size = %u (%u bytes)\n",
 422                         new_item, new_item->id, new_item->size_in_dw,
 423                         new_item->size_in_dw * 4);
 424         return new_item;
 425 }
 426
 427 /**
 428  * Transfer data host<->device, offset and size is in bytes
 429  */
 430 void compute_memory_transfer(
 431         struct compute_memory_pool* pool,
 432         struct pipe_context * pipe,
 433         int device_to_host,
 434         struct compute_memory_item* chunk,
 435         void* data,
 436         int offset_in_chunk,
 437         int size)
 438 {
 439         int64_t aligned_size = pool->size_in_dw;
 440         struct pipe_resource* gart = (struct pipe_resource*)pool->bo;
 441         int64_t internal_offset = chunk->start_in_dw*4 + offset_in_chunk;
 442
 443         struct pipe_transfer *xfer;
 444         uint32_t *map;
 445
 446         assert(gart);
 447
 448         COMPUTE_DBG("* compute_memory_transfer() device_to_host = %d, "
 449                 "offset_in_chunk = %d, size = %d\n", device_to_host,
 450                 offset_in_chunk, size);
 451
 452         if (device_to_host)
 453         {
 454                 xfer = pipe->get_transfer(pipe, gart, 0, PIPE_TRANSFER_READ,
 455                         &(struct pipe_box) { .width = aligned_size,
 456                         .height = 1, .depth = 1 });
 457                 assert(xfer);
 458                 map = pipe->transfer_map(pipe, xfer);
 459                 assert(map);
 460                 memcpy(data, map + internal_offset, size);
 461                 pipe->transfer_unmap(pipe, xfer);
 462                 pipe->transfer_destroy(pipe, xfer);
 463         } else {
 464                 xfer = pipe->get_transfer(pipe, gart, 0, PIPE_TRANSFER_WRITE,
 465                         &(struct pipe_box) { .width = aligned_size,
 466                         .height = 1, .depth = 1 });
 467                 assert(xfer);
 468                 map = pipe->transfer_map(pipe, xfer);
 469                 assert(map);
 470                 memcpy(map + internal_offset, data, size);
 471                 pipe->transfer_unmap(pipe, xfer);
 472                 pipe->transfer_destroy(pipe, xfer);
 473         }
 474 }
 475
 476 /**
 477  * Transfer data between chunk<->data, it is for VRAM<->GART transfers
 478  */
 479 void compute_memory_transfer_direct(
 480         struct compute_memory_pool* pool,
 481         int chunk_to_data,
 482         struct compute_memory_item* chunk,
 483         struct r600_resource* data,
 484         int offset_in_chunk,
 485         int offset_in_data,
 486         int size)
 487 {
 488         ///TODO: DMA
 489 }