src/gallium/drivers/r600/compute_memory_pool.c

   1 /*
   2  * Permission is hereby granted, free of charge, to any person obtaining a
   3  * copy of this software and associated documentation files (the "Software"),
   4  * to deal in the Software without restriction, including without limitation
   5  * on the rights to use, copy, modify, merge, publish, distribute, sub
   6  * license, and/or sell copies of the Software, and to permit persons to whom
   7  * the Software is furnished to do so, subject to the following conditions:
   8  *
   9  * The above copyright notice and this permission notice (including the next
  10  * paragraph) shall be included in all copies or substantial portions of the
  11  * Software.
  12  *
  13  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  14  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  15  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  16  * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
  17  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  18  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  19  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  20  *
  21  * Authors:
  22  *      Adam Rak <adam.rak@streamnovation.com>
  23  */
  24
  25 #include "pipe/p_defines.h"
  26 #include "pipe/p_state.h"
  27 #include "pipe/p_context.h"
  28 #include "util/u_blitter.h"
  29 #include "util/u_double_list.h"
  30 #include "util/u_transfer.h"
  31 #include "util/u_surface.h"
  32 #include "util/u_pack_color.h"
  33 #include "util/u_memory.h"
  34 #include "util/u_inlines.h"
  35 #include "util/u_framebuffer.h"
  36 #include "r600.h"
  37 #include "r600_resource.h"
  38 #include "r600_shader.h"
  39 #include "r600_pipe.h"
  40 #include "r600_formats.h"
  41 #include "compute_memory_pool.h"
  42 #include "evergreen_compute_internal.h"
  43 #include <inttypes.h>
  44
  45 static struct r600_texture * create_pool_texture(struct r600_screen * screen,
  46                 unsigned size_in_dw)
  47 {
  48
  49         struct pipe_resource templ;
  50         struct r600_texture * tex;
  51
  52         if (size_in_dw == 0) {
  53                 return NULL;
  54         }
  55         memset(&templ, 0, sizeof(templ));
  56         templ.target = PIPE_TEXTURE_1D;
  57         templ.format = PIPE_FORMAT_R32_UINT;
  58         templ.bind = PIPE_BIND_CUSTOM;
  59         templ.usage = PIPE_USAGE_IMMUTABLE;
  60         templ.flags = 0;
  61         templ.width0 = size_in_dw;
  62         templ.height0 = 1;
  63         templ.depth0 = 1;
  64         templ.array_size = 1;
  65
  66         tex = (struct r600_texture *)r600_texture_create(
  67                                                 &screen->screen, &templ);
  68         /* XXX: Propagate this error */
  69         assert(tex && "Out of memory");
  70         tex->is_rat = 1;
  71         return tex;
  72 }
  73
  74 /**
  75  * Creates a new pool
  76  */
  77 struct compute_memory_pool* compute_memory_pool_new(
  78         struct r600_screen * rscreen)
  79 {
  80         struct compute_memory_pool* pool = (struct compute_memory_pool*)
  81                                 CALLOC(sizeof(struct compute_memory_pool), 1);
  82
  83         COMPUTE_DBG("* compute_memory_pool_new()\n");
  84
  85         pool->screen = rscreen;
  86         return pool;
  87 }
  88
  89 static void compute_memory_pool_init(struct compute_memory_pool * pool,
  90         unsigned initial_size_in_dw)
  91 {
  92
  93         COMPUTE_DBG("* compute_memory_pool_init() initial_size_in_dw = %ld\n",
  94                 initial_size_in_dw);
  95
  96         /* XXX: pool->shadow is used when the buffer needs to be resized, but
  97          * resizing does not work at the moment.
  98          * pool->shadow = (uint32_t*)CALLOC(4, pool->size_in_dw);
  99          */
 100         pool->next_id = 1;
 101         pool->size_in_dw = initial_size_in_dw;
 102         pool->bo = (struct r600_resource*)create_pool_texture(pool->screen,
 103                                                         pool->size_in_dw);
 104 }
 105
 106 /**
 107  * Frees all stuff in the pool and the pool struct itself too
 108  */
 109 void compute_memory_pool_delete(struct compute_memory_pool* pool)
 110 {
 111         COMPUTE_DBG("* compute_memory_pool_delete()\n");
 112         free(pool->shadow);
 113         if (pool->bo) {
 114                 pool->screen->screen.resource_destroy((struct pipe_screen *)
 115                         pool->screen, (struct pipe_resource *)pool->bo);
 116         }
 117         free(pool);
 118 }
 119
 120 /**
 121  * Searches for an empty space in the pool, return with the pointer to the
 122  * allocatable space in the pool, returns -1 on failure.
 123  */
 124 int64_t compute_memory_prealloc_chunk(
 125         struct compute_memory_pool* pool,
 126         int64_t size_in_dw)
 127 {
 128         assert(size_in_dw <= pool->size_in_dw);
 129
 130         struct compute_memory_item *item;
 131
 132         int last_end = 0;
 133
 134         COMPUTE_DBG("* compute_memory_prealloc_chunk() size_in_dw = %ld\n",
 135                 size_in_dw);
 136
 137         for (item = pool->item_list; item; item = item->next) {
 138                 if (item->start_in_dw > -1) {
 139                         if (item->start_in_dw-last_end > size_in_dw) {
 140                                 return last_end;
 141                         }
 142
 143                         last_end = item->start_in_dw + item->size_in_dw;
 144                         last_end += (1024 - last_end % 1024);
 145                 }
 146         }
 147
 148         if (pool->size_in_dw - last_end < size_in_dw) {
 149                 return -1;
 150         }
 151
 152         return last_end;
 153 }
 154
 155 /**
 156  *  Search for the chunk where we can link our new chunk after it.
 157  */
 158 struct compute_memory_item* compute_memory_postalloc_chunk(
 159         struct compute_memory_pool* pool,
 160         int64_t start_in_dw)
 161 {
 162         struct compute_memory_item* item;
 163
 164         COMPUTE_DBG("* compute_memory_postalloc_chunck() start_in_dw = %ld\n",
 165                 start_in_dw);
 166
 167         for (item = pool->item_list; item; item = item->next) {
 168                 if (item->next) {
 169                         if (item->start_in_dw < start_in_dw
 170                                 && item->next->start_in_dw > start_in_dw) {
 171                                 return item;
 172                         }
 173                 }
 174                 else {
 175                         /* end of chain */
 176                         assert(item->start_in_dw < start_in_dw);
 177                         return item;
 178                 }
 179         }
 180
 181         assert(0 && "unreachable");
 182         return NULL;
 183 }
 184
 185 /**
 186  * Reallocates pool, conserves data
 187  */
 188 void compute_memory_grow_pool(struct compute_memory_pool* pool,
 189         struct pipe_context * pipe, int new_size_in_dw)
 190 {
 191         COMPUTE_DBG("* compute_memory_grow_pool() new_size_in_dw = %d\n",
 192                 new_size_in_dw);
 193
 194         assert(new_size_in_dw >= pool->size_in_dw);
 195
 196         assert(!pool->bo && "Growing the global memory pool is not yet "
 197                 "supported.  You will see this message if you are trying to"
 198                 "use more than 64 kb of memory");
 199
 200         if (!pool->bo) {
 201                 compute_memory_pool_init(pool, 1024 * 16);
 202         } else {
 203                 /* XXX: Growing memory pools does not work at the moment.  I think
 204                  * it is because we are using fragment shaders to copy data to
 205                  * the new texture and some of the compute registers are being
 206                  * included in the 3D command stream. */
 207                 fprintf(stderr, "Warning: growing the global memory pool to"
 208                                 "more than 64 kb is not yet supported\n");
 209                 new_size_in_dw += 1024 - (new_size_in_dw % 1024);
 210
 211                 COMPUTE_DBG("  Aligned size = %d\n", new_size_in_dw);
 212
 213                 compute_memory_shadow(pool, pipe, 1);
 214                 pool->shadow = realloc(pool->shadow, new_size_in_dw*4);
 215                 pool->size_in_dw = new_size_in_dw;
 216                 pool->screen->screen.resource_destroy(
 217                         (struct pipe_screen *)pool->screen,
 218                         (struct pipe_resource *)pool->bo);
 219                 pool->bo = (struct r600_resource*)create_pool_texture(
 220                                                         pool->screen,
 221                                                         pool->size_in_dw);
 222                 compute_memory_shadow(pool, pipe, 0);
 223         }
 224 }
 225
 226 /**
 227  * Copy pool from device to host, or host to device.
 228  */
 229 void compute_memory_shadow(struct compute_memory_pool* pool,
 230         struct pipe_context * pipe, int device_to_host)
 231 {
 232         struct compute_memory_item chunk;
 233
 234         COMPUTE_DBG("* compute_memory_shadow() device_to_host = %d\n",
 235                 device_to_host);
 236
 237         chunk.id = 0;
 238         chunk.start_in_dw = 0;
 239         chunk.size_in_dw = pool->size_in_dw;
 240         chunk.prev = chunk.next = NULL;
 241         compute_memory_transfer(pool, pipe, device_to_host, &chunk,
 242                                 pool->shadow, 0, pool->size_in_dw*4);
 243 }
 244
 245 /**
 246  * Allocates pending allocations in the pool
 247  */
 248 void compute_memory_finalize_pending(struct compute_memory_pool* pool,
 249         struct pipe_context * pipe)
 250 {
 251         struct compute_memory_item *pending_list = NULL, *end_p = NULL;
 252         struct compute_memory_item *item, *next;
 253
 254         int64_t allocated = 0;
 255         int64_t unallocated = 0;
 256
 257         COMPUTE_DBG("* compute_memory_finalize_pending()\n");
 258
 259         for (item = pool->item_list; item; item = item->next) {
 260                 COMPUTE_DBG("list: %i %p\n", item->start_in_dw, item->next);
 261         }
 262
 263         for (item = pool->item_list; item; item = next) {
 264                 next = item->next;
 265
 266
 267                 if (item->start_in_dw == -1) {
 268                         if (end_p) {
 269                                 end_p->next = item;
 270                         }
 271                         else {
 272                                 pending_list = item;
 273                         }
 274
 275                         if (item->prev) {
 276                                 item->prev->next = next;
 277                         }
 278                         else {
 279                                 pool->item_list = next;
 280                         }
 281
 282                         if (next) {
 283                                 next->prev = item->prev;
 284                         }
 285
 286                         item->prev = end_p;
 287                         item->next = NULL;
 288                         end_p = item;
 289
 290                         unallocated += item->size_in_dw+1024;
 291                 }
 292                 else {
 293                         allocated += item->size_in_dw;
 294                 }
 295         }
 296
 297         if (pool->size_in_dw < allocated+unallocated) {
 298                 compute_memory_grow_pool(pool, pipe, allocated+unallocated);
 299         }
 300
 301         for (item = pending_list; item; item = next) {
 302                 next = item->next;
 303
 304                 int64_t start_in_dw;
 305
 306                 while ((start_in_dw=compute_memory_prealloc_chunk(pool,
 307                                                 item->size_in_dw)) == -1) {
 308                         int64_t need = item->size_in_dw+2048 -
 309                                                 (pool->size_in_dw - allocated);
 310
 311                         need += 1024 - (need % 1024);
 312
 313                         if (need > 0) {
 314                                 compute_memory_grow_pool(pool,
 315                                                 pipe,
 316                                                 pool->size_in_dw + need);
 317                         }
 318                         else {
 319                                 need = pool->size_in_dw / 10;
 320                                 need += 1024 - (need % 1024);
 321                                 compute_memory_grow_pool(pool,
 322                                                 pipe,
 323                                                 pool->size_in_dw + need);
 324                         }
 325                 }
 326
 327                 item->start_in_dw = start_in_dw;
 328                 item->next = NULL;
 329                 item->prev = NULL;
 330
 331                 if (pool->item_list) {
 332                         struct compute_memory_item *pos;
 333
 334                         pos = compute_memory_postalloc_chunk(pool, start_in_dw);
 335                         item->prev = pos;
 336                         item->next = pos->next;
 337                         pos->next = item;
 338
 339                         if (item->next) {
 340                                 item->next->prev = item;
 341                         }
 342                 }
 343                 else {
 344                         pool->item_list = item;
 345                 }
 346
 347                 allocated += item->size_in_dw;
 348         }
 349 }
 350
 351
 352 void compute_memory_free(struct compute_memory_pool* pool, int64_t id)
 353 {
 354         struct compute_memory_item *item, *next;
 355
 356         COMPUTE_DBG("* compute_memory_free() id + %ld \n", id);
 357
 358         for (item = pool->item_list; item; item = next) {
 359                 next = item->next;
 360
 361                 if (item->id == id) {
 362                         if (item->prev) {
 363                                 item->prev->next = item->next;
 364                         }
 365                         else {
 366                                 pool->item_list = item->next;
 367                         }
 368
 369                         if (item->next) {
 370                                 item->next->prev = item->prev;
 371                         }
 372
 373                         free(item);
 374
 375                         return;
 376                 }
 377         }
 378
 379         fprintf(stderr, "Internal error, invalid id %"PRIi64" "
 380                 "for compute_memory_free\n", id);
 381
 382         assert(0 && "error");
 383 }
 384
 385 /**
 386  * Creates pending allocations
 387  */
 388 struct compute_memory_item* compute_memory_alloc(
 389         struct compute_memory_pool* pool,
 390         int64_t size_in_dw)
 391 {
 392         struct compute_memory_item *new_item;
 393
 394         COMPUTE_DBG("* compute_memory_alloc() size_in_dw = %ld\n", size_in_dw);
 395
 396         new_item = (struct compute_memory_item *)
 397                                 CALLOC(sizeof(struct compute_memory_item), 1);
 398         new_item->size_in_dw = size_in_dw;
 399         new_item->start_in_dw = -1; /* mark pending */
 400         new_item->id = pool->next_id++;
 401         new_item->pool = pool;
 402
 403         struct compute_memory_item *last_item;
 404
 405         if (pool->item_list) {
 406                 for (last_item = pool->item_list; last_item->next;
 407                                                 last_item = last_item->next);
 408
 409                 last_item->next = new_item;
 410                 new_item->prev = last_item;
 411         }
 412         else {
 413                 pool->item_list = new_item;
 414         }
 415
 416         return new_item;
 417 }
 418
 419 /**
 420  * Transfer data host<->device, offset and size is in bytes
 421  */
 422 void compute_memory_transfer(
 423         struct compute_memory_pool* pool,
 424         struct pipe_context * pipe,
 425         int device_to_host,
 426         struct compute_memory_item* chunk,
 427         void* data,
 428         int offset_in_chunk,
 429         int size)
 430 {
 431         int64_t aligned_size = pool->size_in_dw;
 432         struct pipe_resource* gart = (struct pipe_resource*)pool->bo;
 433         int64_t internal_offset = chunk->start_in_dw*4 + offset_in_chunk;
 434
 435         struct pipe_transfer *xfer;
 436         uint32_t *map;
 437
 438         assert(gart);
 439
 440         COMPUTE_DBG("* compute_memory_transfer() device_to_host = %d, "
 441                 "offset_in_chunk = %d, size = %d\n", device_to_host,
 442                 offset_in_chunk, size);
 443
 444         if (device_to_host)
 445         {
 446                 xfer = pipe->get_transfer(pipe, gart, 0, PIPE_TRANSFER_READ,
 447                         &(struct pipe_box) { .width = aligned_size,
 448                         .height = 1, .depth = 1 });
 449                 assert(xfer);
 450                 map = pipe->transfer_map(pipe, xfer);
 451                 assert(map);
 452                 memcpy(data, map + internal_offset, size);
 453                 pipe->transfer_unmap(pipe, xfer);
 454                 pipe->transfer_destroy(pipe, xfer);
 455         } else {
 456                 xfer = pipe->get_transfer(pipe, gart, 0, PIPE_TRANSFER_WRITE,
 457                         &(struct pipe_box) { .width = aligned_size,
 458                         .height = 1, .depth = 1 });
 459                 assert(xfer);
 460                 map = pipe->transfer_map(pipe, xfer);
 461                 assert(map);
 462                 memcpy(map + internal_offset, data, size);
 463                 pipe->transfer_unmap(pipe, xfer);
 464                 pipe->transfer_destroy(pipe, xfer);
 465         }
 466 }
 467
 468 /**
 469  * Transfer data between chunk<->data, it is for VRAM<->GART transfers
 470  */
 471 void compute_memory_transfer_direct(
 472         struct compute_memory_pool* pool,
 473         int chunk_to_data,
 474         struct compute_memory_item* chunk,
 475         struct r600_resource* data,
 476         int offset_in_chunk,
 477         int offset_in_data,
 478         int size)
 479 {
 480         ///TODO: DMA
 481 }