src/gallium/drivers/r600/compute_memory_pool.c

   1 /*
   2  * Permission is hereby granted, free of charge, to any person obtaining a
   3  * copy of this software and associated documentation files (the "Software"),
   4  * to deal in the Software without restriction, including without limitation
   5  * on the rights to use, copy, modify, merge, publish, distribute, sub
   6  * license, and/or sell copies of the Software, and to permit persons to whom
   7  * the Software is furnished to do so, subject to the following conditions:
   8  *
   9  * The above copyright notice and this permission notice (including the next
  10  * paragraph) shall be included in all copies or substantial portions of the
  11  * Software.
  12  *
  13  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  14  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  15  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  16  * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
  17  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  18  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  19  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  20  *
  21  * Authors:
  22  *      Adam Rak <adam.rak@streamnovation.com>
  23  */
  24
  25 #include "pipe/p_defines.h"
  26 #include "pipe/p_state.h"
  27 #include "pipe/p_context.h"
  28 #include "util/u_blitter.h"
  29 #include "util/u_double_list.h"
  30 #include "util/u_transfer.h"
  31 #include "util/u_surface.h"
  32 #include "util/u_pack_color.h"
  33 #include "util/u_memory.h"
  34 #include "util/u_inlines.h"
  35 #include "util/u_framebuffer.h"
  36 #include "r600.h"
  37 #include "r600_resource.h"
  38 #include "r600_shader.h"
  39 #include "r600_pipe.h"
  40 #include "r600_formats.h"
  41 #include "compute_memory_pool.h"
  42 #include "evergreen_compute_internal.h"
  43
  44 /**
  45  * Creates a new pool
  46  */
  47 struct compute_memory_pool* compute_memory_pool_new(
  48         int64_t initial_size_in_dw,
  49         struct r600_screen * rscreen)
  50 {
  51         struct compute_memory_pool* pool = (struct compute_memory_pool*)
  52                                 CALLOC(sizeof(struct compute_memory_pool), 1);
  53
  54         pool->next_id = 1;
  55         pool->size_in_dw = initial_size_in_dw;
  56         pool->screen = rscreen;
  57         pool->bo = (struct r600_resource*)r600_compute_buffer_alloc_vram(
  58                                         pool->screen, pool->size_in_dw*4);
  59         pool->shadow = (uint32_t*)CALLOC(4, pool->size_in_dw);
  60
  61         return pool;
  62 }
  63
  64 /**
  65  * Frees all stuff in the pool and the pool struct itself too
  66  */
  67 void compute_memory_pool_delete(struct compute_memory_pool* pool)
  68 {
  69         free(pool->shadow);
  70         pool->screen->screen.resource_destroy((struct pipe_screen *)
  71                         pool->screen, (struct pipe_resource *)pool->bo);
  72         free(pool);
  73 }
  74
  75 /**
  76  * Searches for an empty space in the pool, return with the pointer to the
  77  * allocatable space in the pool, returns -1 on failure.
  78  */
  79 int64_t compute_memory_prealloc_chunk(
  80         struct compute_memory_pool* pool,
  81         int64_t size_in_dw)
  82 {
  83         assert(size_in_dw <= pool->size_in_dw);
  84
  85         struct compute_memory_item *item;
  86
  87         int last_end = 0;
  88
  89         for (item = pool->item_list; item; item = item->next) {
  90                 if (item->start_in_dw > -1) {
  91                         if (item->start_in_dw-last_end > size_in_dw) {
  92                                 return last_end;
  93                         }
  94
  95                         last_end = item->start_in_dw + item->size_in_dw;
  96                         last_end += (1024 - last_end % 1024);
  97                 }
  98         }
  99
 100         if (pool->size_in_dw - last_end < size_in_dw) {
 101                 return -1;
 102         }
 103
 104         return last_end;
 105 }
 106
 107 /**
 108  *  Search for the chunk where we can link our new chunk after it.
 109  */
 110 struct compute_memory_item* compute_memory_postalloc_chunk(
 111         struct compute_memory_pool* pool,
 112         int64_t start_in_dw)
 113 {
 114         struct compute_memory_item* item;
 115
 116         for (item = pool->item_list; item; item = item->next) {
 117                 if (item->next) {
 118                         if (item->start_in_dw < start_in_dw
 119                                 && item->next->start_in_dw > start_in_dw) {
 120                                 return item;
 121                         }
 122                 }
 123                 else {
 124                         /* end of chain */
 125                         assert(item->start_in_dw < start_in_dw);
 126                         return item;
 127                 }
 128         }
 129
 130         assert(0 && "unreachable");
 131         return NULL;
 132 }
 133
 134 /**
 135  * Reallocates pool, conserves data
 136  */
 137 void compute_memory_grow_pool(struct compute_memory_pool* pool,
 138         struct pipe_context * pipe, int new_size_in_dw)
 139 {
 140         assert(new_size_in_dw >= pool->size_in_dw);
 141
 142         new_size_in_dw += 1024 - (new_size_in_dw % 1024);
 143
 144         compute_memory_shadow(pool, pipe, 1);
 145         pool->shadow = (uint32_t*)realloc(pool->shadow, new_size_in_dw*4);
 146         pool->size_in_dw = new_size_in_dw;
 147         pool->screen->screen.resource_destroy(
 148                 (struct pipe_screen *)pool->screen,
 149                 (struct pipe_resource *)pool->bo);
 150         pool->bo = r600_compute_buffer_alloc_vram(pool->screen,
 151                                                 pool->size_in_dw*4);
 152         compute_memory_shadow(pool, pipe, 0);
 153 }
 154
 155 /**
 156  * Copy pool from device to host, or host to device.
 157  */
 158 void compute_memory_shadow(struct compute_memory_pool* pool,
 159         struct pipe_context * pipe, int device_to_host)
 160 {
 161         struct compute_memory_item chunk;
 162
 163         chunk.id = 0;
 164         chunk.start_in_dw = 0;
 165         chunk.size_in_dw = pool->size_in_dw;
 166         chunk.prev = chunk.next = NULL;
 167         compute_memory_transfer(pool, pipe, device_to_host, &chunk,
 168                                 pool->shadow, 0, pool->size_in_dw*4);
 169 }
 170
 171 /**
 172  * Allocates pending allocations in the pool
 173  */
 174 void compute_memory_finalize_pending(struct compute_memory_pool* pool,
 175         struct pipe_context * pipe)
 176 {
 177         struct compute_memory_item *pending_list = NULL, *end_p = NULL;
 178         struct compute_memory_item *item, *next;
 179
 180         int64_t allocated = 0;
 181         int64_t unallocated = 0;
 182
 183         for (item = pool->item_list; item; item = item->next) {
 184                 COMPUTE_DBG("list: %i %p\n", item->start_in_dw, item->next);
 185         }
 186
 187         for (item = pool->item_list; item; item = next) {
 188                 next = item->next;
 189
 190
 191                 if (item->start_in_dw == -1) {
 192                         if (end_p) {
 193                                 end_p->next = item;
 194                         }
 195                         else {
 196                                 pending_list = item;
 197                         }
 198
 199                         if (item->prev) {
 200                                 item->prev->next = next;
 201                         }
 202                         else {
 203                                 pool->item_list = next;
 204                         }
 205
 206                         if (next) {
 207                                 next->prev = item->prev;
 208                         }
 209
 210                         item->prev = end_p;
 211                         item->next = NULL;
 212                         end_p = item;
 213
 214                         unallocated += item->size_in_dw+1024;
 215                 }
 216                 else {
 217                         allocated += item->size_in_dw;
 218                 }
 219         }
 220
 221         if (pool->size_in_dw < allocated+unallocated) {
 222                 compute_memory_grow_pool(pool, pipe, allocated+unallocated);
 223         }
 224
 225         for (item = pending_list; item; item = next) {
 226                 next = item->next;
 227
 228                 int64_t start_in_dw;
 229
 230                 while ((start_in_dw=compute_memory_prealloc_chunk(pool,
 231                                                 item->size_in_dw)) == -1) {
 232                         int64_t need = item->size_in_dw+2048 -
 233                                                 (pool->size_in_dw - allocated);
 234
 235                         need += 1024 - (need % 1024);
 236
 237                         if (need > 0) {
 238                                 compute_memory_grow_pool(pool,
 239                                                 pipe,
 240                                                 pool->size_in_dw + need);
 241                         }
 242                         else {
 243                                 need = pool->size_in_dw / 10;
 244                                 need += 1024 - (need % 1024);
 245                                 compute_memory_grow_pool(pool,
 246                                                 pipe,
 247                                                 pool->size_in_dw + need);
 248                         }
 249                 }
 250
 251                 item->start_in_dw = start_in_dw;
 252                 item->next = NULL;
 253                 item->prev = NULL;
 254
 255                 if (pool->item_list) {
 256                         struct compute_memory_item *pos;
 257
 258                         pos = compute_memory_postalloc_chunk(pool, start_in_dw);
 259                         item->prev = pos;
 260                         item->next = pos->next;
 261                         pos->next = item;
 262
 263                         if (item->next) {
 264                                 item->next->prev = item;
 265                         }
 266                 }
 267                 else {
 268                         pool->item_list = item;
 269                 }
 270
 271                 allocated += item->size_in_dw;
 272         }
 273 }
 274
 275
 276 void compute_memory_free(struct compute_memory_pool* pool, int64_t id)
 277 {
 278         struct compute_memory_item *item, *next;
 279
 280         for (item = pool->item_list; item; item = next) {
 281                 next = item->next;
 282
 283                 if (item->id == id) {
 284                         if (item->prev) {
 285                                 item->prev->next = item->next;
 286                         }
 287                         else {
 288                                 pool->item_list = item->next;
 289                         }
 290
 291                         if (item->next) {
 292                                 item->next->prev = item->prev;
 293                         }
 294
 295                         free(item);
 296
 297                         return;
 298                 }
 299         }
 300
 301         fprintf(stderr, "Internal error, invalid id %ld "
 302                 "for compute_memory_free\n", id);
 303
 304         assert(0 && "error");
 305 }
 306
 307 /**
 308  * Creates pending allocations
 309  */
 310 struct compute_memory_item* compute_memory_alloc(
 311         struct compute_memory_pool* pool,
 312         int64_t size_in_dw)
 313 {
 314         struct compute_memory_item *new_item;
 315
 316         COMPUTE_DBG("Alloc: %i\n", size_in_dw);
 317
 318         new_item = (struct compute_memory_item *)
 319                                 CALLOC(sizeof(struct compute_memory_item), 1);
 320         new_item->size_in_dw = size_in_dw;
 321         new_item->start_in_dw = -1; /* mark pending */
 322         new_item->id = pool->next_id++;
 323         new_item->pool = pool;
 324
 325         struct compute_memory_item *last_item;
 326
 327         if (pool->item_list) {
 328                 for (last_item = pool->item_list; last_item->next;
 329                                                 last_item = last_item->next);
 330
 331                 last_item->next = new_item;
 332                 new_item->prev = last_item;
 333         }
 334         else {
 335                 pool->item_list = new_item;
 336         }
 337
 338         return new_item;
 339 }
 340
 341 /**
 342  * Transfer data host<->device, offset and size is in bytes
 343  */
 344 void compute_memory_transfer(
 345         struct compute_memory_pool* pool,
 346         struct pipe_context * pipe,
 347         int device_to_host,
 348         struct compute_memory_item* chunk,
 349         void* data,
 350         int offset_in_chunk,
 351         int size)
 352 {
 353         int64_t aligned_size = pool->size_in_dw;
 354         struct pipe_resource* gart = (struct pipe_resource*)pool->bo;
 355         int64_t internal_offset = chunk->start_in_dw*4 + offset_in_chunk;
 356
 357         struct pipe_transfer *xfer;
 358         uint32_t *map;
 359
 360         if (device_to_host)
 361         {
 362                 xfer = pipe->get_transfer(pipe, gart, 0, PIPE_TRANSFER_READ,
 363                         &(struct pipe_box) { .width = aligned_size,
 364                         .height = 1, .depth = 1 });
 365                 assert(xfer);
 366                 map = pipe->transfer_map(pipe, xfer);
 367                 assert(map);
 368                 memcpy(data, map + internal_offset, size);
 369                 pipe->transfer_unmap(pipe, xfer);
 370                 pipe->transfer_destroy(pipe, xfer);
 371         } else {
 372                 xfer = pipe->get_transfer(pipe, gart, 0, PIPE_TRANSFER_WRITE,
 373                         &(struct pipe_box) { .width = aligned_size,
 374                         .height = 1, .depth = 1 });
 375                 assert(xfer);
 376                 map = pipe->transfer_map(pipe, xfer);
 377                 assert(map);
 378                 memcpy(map + internal_offset, data, size);
 379                 pipe->transfer_unmap(pipe, xfer);
 380                 pipe->transfer_destroy(pipe, xfer);
 381         }
 382 }
 383
 384 /**
 385  * Transfer data between chunk<->data, it is for VRAM<->GART transfers
 386  */
 387 void compute_memory_transfer_direct(
 388         struct compute_memory_pool* pool,
 389         int chunk_to_data,
 390         struct compute_memory_item* chunk,
 391         struct r600_resource* data,
 392         int offset_in_chunk,
 393         int offset_in_data,
 394         int size)
 395 {
 396         ///TODO: DMA
 397 }