src/gallium/drivers/r600/compute_memory_pool.c

   1 /*
   2  * Permission is hereby granted, free of charge, to any person obtaining a
   3  * copy of this software and associated documentation files (the "Software"),
   4  * to deal in the Software without restriction, including without limitation
   5  * on the rights to use, copy, modify, merge, publish, distribute, sub
   6  * license, and/or sell copies of the Software, and to permit persons to whom
   7  * the Software is furnished to do so, subject to the following conditions:
   8  *
   9  * The above copyright notice and this permission notice (including the next
  10  * paragraph) shall be included in all copies or substantial portions of the
  11  * Software.
  12  *
  13  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  14  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  15  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  16  * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
  17  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  18  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  19  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  20  *
  21  * Authors:
  22  *      Adam Rak <adam.rak@streamnovation.com>
  23  */
  24
  25 #include "pipe/p_defines.h"
  26 #include "pipe/p_state.h"
  27 #include "pipe/p_context.h"
  28 #include "util/u_blitter.h"
  29 #include "util/u_double_list.h"
  30 #include "util/u_transfer.h"
  31 #include "util/u_surface.h"
  32 #include "util/u_pack_color.h"
  33 #include "util/u_memory.h"
  34 #include "util/u_inlines.h"
  35 #include "util/u_framebuffer.h"
  36 #include "r600.h"
  37 #include "r600_resource.h"
  38 #include "r600_shader.h"
  39 #include "r600_pipe.h"
  40 #include "r600_formats.h"
  41 #include "compute_memory_pool.h"
  42 #include "evergreen_compute_internal.h"
  43
  44 static struct r600_resource_texture * create_pool_texture(struct r600_screen * screen,
  45                 unsigned size_in_dw)
  46 {
  47
  48         struct pipe_resource templ;
  49         struct r600_resource_texture * tex;
  50
  51         memset(&templ, 0, sizeof(templ));
  52         templ.target = PIPE_TEXTURE_1D;
  53         templ.format = PIPE_FORMAT_R32_UINT;
  54         templ.bind = PIPE_BIND_CUSTOM;
  55         templ.usage = PIPE_USAGE_IMMUTABLE;
  56         templ.flags = 0;
  57         templ.width0 = size_in_dw;
  58         templ.height0 = 1;
  59         templ.depth0 = 1;
  60         templ.array_size = 1;
  61
  62         tex = (struct r600_resource_texture *)r600_texture_create(
  63                                                 &screen->screen, &templ);
  64         /* XXX: Propagate this error */
  65         assert(tex && "Out of memory");
  66         tex->is_rat = 1;
  67         return tex;
  68 }
  69
  70 /**
  71  * Creates a new pool
  72  */
  73 struct compute_memory_pool* compute_memory_pool_new(
  74         int64_t initial_size_in_dw,
  75         struct r600_screen * rscreen)
  76 {
  77         struct compute_memory_pool* pool = (struct compute_memory_pool*)
  78                                 CALLOC(sizeof(struct compute_memory_pool), 1);
  79
  80         pool->next_id = 1;
  81         pool->size_in_dw = initial_size_in_dw;
  82         pool->screen = rscreen;
  83         pool->bo = (struct r600_resource*)create_pool_texture(pool->screen,
  84                                                         pool->size_in_dw);
  85         pool->shadow = (uint32_t*)CALLOC(4, pool->size_in_dw);
  86
  87         return pool;
  88 }
  89
  90 /**
  91  * Frees all stuff in the pool and the pool struct itself too
  92  */
  93 void compute_memory_pool_delete(struct compute_memory_pool* pool)
  94 {
  95         free(pool->shadow);
  96         pool->screen->screen.resource_destroy((struct pipe_screen *)
  97                         pool->screen, (struct pipe_resource *)pool->bo);
  98         free(pool);
  99 }
 100
 101 /**
 102  * Searches for an empty space in the pool, return with the pointer to the
 103  * allocatable space in the pool, returns -1 on failure.
 104  */
 105 int64_t compute_memory_prealloc_chunk(
 106         struct compute_memory_pool* pool,
 107         int64_t size_in_dw)
 108 {
 109         assert(size_in_dw <= pool->size_in_dw);
 110
 111         struct compute_memory_item *item;
 112
 113         int last_end = 0;
 114
 115         for (item = pool->item_list; item; item = item->next) {
 116                 if (item->start_in_dw > -1) {
 117                         if (item->start_in_dw-last_end > size_in_dw) {
 118                                 return last_end;
 119                         }
 120
 121                         last_end = item->start_in_dw + item->size_in_dw;
 122                         last_end += (1024 - last_end % 1024);
 123                 }
 124         }
 125
 126         if (pool->size_in_dw - last_end < size_in_dw) {
 127                 return -1;
 128         }
 129
 130         return last_end;
 131 }
 132
 133 /**
 134  *  Search for the chunk where we can link our new chunk after it.
 135  */
 136 struct compute_memory_item* compute_memory_postalloc_chunk(
 137         struct compute_memory_pool* pool,
 138         int64_t start_in_dw)
 139 {
 140         struct compute_memory_item* item;
 141
 142         for (item = pool->item_list; item; item = item->next) {
 143                 if (item->next) {
 144                         if (item->start_in_dw < start_in_dw
 145                                 && item->next->start_in_dw > start_in_dw) {
 146                                 return item;
 147                         }
 148                 }
 149                 else {
 150                         /* end of chain */
 151                         assert(item->start_in_dw < start_in_dw);
 152                         return item;
 153                 }
 154         }
 155
 156         assert(0 && "unreachable");
 157         return NULL;
 158 }
 159
 160 /**
 161  * Reallocates pool, conserves data
 162  */
 163 void compute_memory_grow_pool(struct compute_memory_pool* pool,
 164         struct pipe_context * pipe, int new_size_in_dw)
 165 {
 166         assert(new_size_in_dw >= pool->size_in_dw);
 167
 168         new_size_in_dw += 1024 - (new_size_in_dw % 1024);
 169
 170         compute_memory_shadow(pool, pipe, 1);
 171         pool->shadow = (uint32_t*)realloc(pool->shadow, new_size_in_dw*4);
 172         pool->size_in_dw = new_size_in_dw;
 173         pool->screen->screen.resource_destroy(
 174                 (struct pipe_screen *)pool->screen,
 175                 (struct pipe_resource *)pool->bo);
 176         pool->bo = (struct r600_resource*)create_pool_texture(pool->screen,
 177                                                         pool->size_in_dw);
 178         compute_memory_shadow(pool, pipe, 0);
 179 }
 180
 181 /**
 182  * Copy pool from device to host, or host to device.
 183  */
 184 void compute_memory_shadow(struct compute_memory_pool* pool,
 185         struct pipe_context * pipe, int device_to_host)
 186 {
 187         struct compute_memory_item chunk;
 188
 189         chunk.id = 0;
 190         chunk.start_in_dw = 0;
 191         chunk.size_in_dw = pool->size_in_dw;
 192         chunk.prev = chunk.next = NULL;
 193         compute_memory_transfer(pool, pipe, device_to_host, &chunk,
 194                                 pool->shadow, 0, pool->size_in_dw*4);
 195 }
 196
 197 /**
 198  * Allocates pending allocations in the pool
 199  */
 200 void compute_memory_finalize_pending(struct compute_memory_pool* pool,
 201         struct pipe_context * pipe)
 202 {
 203         struct compute_memory_item *pending_list = NULL, *end_p = NULL;
 204         struct compute_memory_item *item, *next;
 205
 206         int64_t allocated = 0;
 207         int64_t unallocated = 0;
 208
 209         for (item = pool->item_list; item; item = item->next) {
 210                 COMPUTE_DBG("list: %i %p\n", item->start_in_dw, item->next);
 211         }
 212
 213         for (item = pool->item_list; item; item = next) {
 214                 next = item->next;
 215
 216
 217                 if (item->start_in_dw == -1) {
 218                         if (end_p) {
 219                                 end_p->next = item;
 220                         }
 221                         else {
 222                                 pending_list = item;
 223                         }
 224
 225                         if (item->prev) {
 226                                 item->prev->next = next;
 227                         }
 228                         else {
 229                                 pool->item_list = next;
 230                         }
 231
 232                         if (next) {
 233                                 next->prev = item->prev;
 234                         }
 235
 236                         item->prev = end_p;
 237                         item->next = NULL;
 238                         end_p = item;
 239
 240                         unallocated += item->size_in_dw+1024;
 241                 }
 242                 else {
 243                         allocated += item->size_in_dw;
 244                 }
 245         }
 246
 247         if (pool->size_in_dw < allocated+unallocated) {
 248                 compute_memory_grow_pool(pool, pipe, allocated+unallocated);
 249         }
 250
 251         for (item = pending_list; item; item = next) {
 252                 next = item->next;
 253
 254                 int64_t start_in_dw;
 255
 256                 while ((start_in_dw=compute_memory_prealloc_chunk(pool,
 257                                                 item->size_in_dw)) == -1) {
 258                         int64_t need = item->size_in_dw+2048 -
 259                                                 (pool->size_in_dw - allocated);
 260
 261                         need += 1024 - (need % 1024);
 262
 263                         if (need > 0) {
 264                                 compute_memory_grow_pool(pool,
 265                                                 pipe,
 266                                                 pool->size_in_dw + need);
 267                         }
 268                         else {
 269                                 need = pool->size_in_dw / 10;
 270                                 need += 1024 - (need % 1024);
 271                                 compute_memory_grow_pool(pool,
 272                                                 pipe,
 273                                                 pool->size_in_dw + need);
 274                         }
 275                 }
 276
 277                 item->start_in_dw = start_in_dw;
 278                 item->next = NULL;
 279                 item->prev = NULL;
 280
 281                 if (pool->item_list) {
 282                         struct compute_memory_item *pos;
 283
 284                         pos = compute_memory_postalloc_chunk(pool, start_in_dw);
 285                         item->prev = pos;
 286                         item->next = pos->next;
 287                         pos->next = item;
 288
 289                         if (item->next) {
 290                                 item->next->prev = item;
 291                         }
 292                 }
 293                 else {
 294                         pool->item_list = item;
 295                 }
 296
 297                 allocated += item->size_in_dw;
 298         }
 299 }
 300
 301
 302 void compute_memory_free(struct compute_memory_pool* pool, int64_t id)
 303 {
 304         struct compute_memory_item *item, *next;
 305
 306         for (item = pool->item_list; item; item = next) {
 307                 next = item->next;
 308
 309                 if (item->id == id) {
 310                         if (item->prev) {
 311                                 item->prev->next = item->next;
 312                         }
 313                         else {
 314                                 pool->item_list = item->next;
 315                         }
 316
 317                         if (item->next) {
 318                                 item->next->prev = item->prev;
 319                         }
 320
 321                         free(item);
 322
 323                         return;
 324                 }
 325         }
 326
 327         fprintf(stderr, "Internal error, invalid id %ld "
 328                 "for compute_memory_free\n", id);
 329
 330         assert(0 && "error");
 331 }
 332
 333 /**
 334  * Creates pending allocations
 335  */
 336 struct compute_memory_item* compute_memory_alloc(
 337         struct compute_memory_pool* pool,
 338         int64_t size_in_dw)
 339 {
 340         struct compute_memory_item *new_item;
 341
 342         COMPUTE_DBG("Alloc: %i\n", size_in_dw);
 343
 344         new_item = (struct compute_memory_item *)
 345                                 CALLOC(sizeof(struct compute_memory_item), 1);
 346         new_item->size_in_dw = size_in_dw;
 347         new_item->start_in_dw = -1; /* mark pending */
 348         new_item->id = pool->next_id++;
 349         new_item->pool = pool;
 350
 351         struct compute_memory_item *last_item;
 352
 353         if (pool->item_list) {
 354                 for (last_item = pool->item_list; last_item->next;
 355                                                 last_item = last_item->next);
 356
 357                 last_item->next = new_item;
 358                 new_item->prev = last_item;
 359         }
 360         else {
 361                 pool->item_list = new_item;
 362         }
 363
 364         return new_item;
 365 }
 366
 367 /**
 368  * Transfer data host<->device, offset and size is in bytes
 369  */
 370 void compute_memory_transfer(
 371         struct compute_memory_pool* pool,
 372         struct pipe_context * pipe,
 373         int device_to_host,
 374         struct compute_memory_item* chunk,
 375         void* data,
 376         int offset_in_chunk,
 377         int size)
 378 {
 379         int64_t aligned_size = pool->size_in_dw;
 380         struct pipe_resource* gart = (struct pipe_resource*)pool->bo;
 381         int64_t internal_offset = chunk->start_in_dw*4 + offset_in_chunk;
 382
 383         struct pipe_transfer *xfer;
 384         uint32_t *map;
 385
 386         if (device_to_host)
 387         {
 388                 xfer = pipe->get_transfer(pipe, gart, 0, PIPE_TRANSFER_READ,
 389                         &(struct pipe_box) { .width = aligned_size,
 390                         .height = 1, .depth = 1 });
 391                 assert(xfer);
 392                 map = pipe->transfer_map(pipe, xfer);
 393                 assert(map);
 394                 memcpy(data, map + internal_offset, size);
 395                 pipe->transfer_unmap(pipe, xfer);
 396                 pipe->transfer_destroy(pipe, xfer);
 397         } else {
 398                 xfer = pipe->get_transfer(pipe, gart, 0, PIPE_TRANSFER_WRITE,
 399                         &(struct pipe_box) { .width = aligned_size,
 400                         .height = 1, .depth = 1 });
 401                 assert(xfer);
 402                 map = pipe->transfer_map(pipe, xfer);
 403                 assert(map);
 404                 memcpy(map + internal_offset, data, size);
 405                 pipe->transfer_unmap(pipe, xfer);
 406                 pipe->transfer_destroy(pipe, xfer);
 407         }
 408 }
 409
 410 /**
 411  * Transfer data between chunk<->data, it is for VRAM<->GART transfers
 412  */
 413 void compute_memory_transfer_direct(
 414         struct compute_memory_pool* pool,
 415         int chunk_to_data,
 416         struct compute_memory_item* chunk,
 417         struct r600_resource* data,
 418         int offset_in_chunk,
 419         int offset_in_data,
 420         int size)
 421 {
 422         ///TODO: DMA
 423 }