src/gallium/drivers/iris/iris_batch.c

   1 /*
   2  * Copyright © 2017 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the
   6  * "Software"), to deal in the Software without restriction, including
   7  * without limitation the rights to use, copy, modify, merge, publish,
   8  * distribute, sublicense, and/or sell copies of the Software, and to
   9  * permit persons to whom the Software is furnished to do so, subject to
  10  * the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the
  13  * next paragraph) shall be included in all copies or substantial portions
  14  * of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  17  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  18  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  19  * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
  20  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  21  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  22  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  23  */
  24
  25 #include "iris_batch.h"
  26 #include "iris_bufmgr.h"
  27 #include "iris_context.h"
  28
  29 #include "drm-uapi/i915_drm.h"
  30
  31 #include "util/hash_table.h"
  32 #include "util/set.h"
  33 #include "main/macros.h"
  34
  35 #include <errno.h>
  36 #include <xf86drm.h>
  37
  38 #define FILE_DEBUG_FLAG DEBUG_BUFMGR
  39
  40 #define BATCH_SZ (20 * 1024)
  41
  42 /* Terminating the batch takes either 4 bytes for MI_BATCH_BUFFER_END
  43  * or 12 bytes for MI_BATCH_BUFFER_START (when chaining).  Plus, we may
  44  * need an extra 4 bytes to pad out to the nearest QWord.  So reserve 16.
  45  */
  46 #define BATCH_RESERVED 16
  47
  48 static void decode_batch(struct iris_batch *batch);
  49
  50 static void
  51 iris_batch_reset(struct iris_batch *batch);
  52
  53 UNUSED static void
  54 dump_validation_list(struct iris_batch *batch)
  55 {
  56    fprintf(stderr, "Validation list (length %d):\n", batch->exec_count);
  57
  58    for (int i = 0; i < batch->exec_count; i++) {
  59       uint64_t flags = batch->validation_list[i].flags;
  60       assert(batch->validation_list[i].handle ==
  61              batch->exec_bos[i]->gem_handle);
  62       fprintf(stderr, "[%2d]: %2d %-14s %p %-7s @ 0x%016llx (%"PRIu64"B)\n",
  63               i,
  64               batch->validation_list[i].handle,
  65               batch->exec_bos[i]->name,
  66               batch->exec_bos[i],
  67               (flags & EXEC_OBJECT_WRITE) ? "(write)" : "",
  68               batch->validation_list[i].offset,
  69               batch->exec_bos[i]->size);
  70    }
  71 }
  72
  73 static struct gen_batch_decode_bo
  74 decode_get_bo(void *v_batch, uint64_t address)
  75 {
  76    struct iris_batch *batch = v_batch;
  77
  78    for (int i = 0; i < batch->exec_count; i++) {
  79       struct iris_bo *bo = batch->exec_bos[i];
  80       /* The decoder zeroes out the top 16 bits, so we need to as well */
  81       uint64_t bo_address = bo->gtt_offset & (~0ull >> 16);
  82
  83       if (address >= bo_address && address < bo_address + bo->size) {
  84          return (struct gen_batch_decode_bo) {
  85             .addr = address,
  86             .size = bo->size,
  87             .map = iris_bo_map(batch->dbg, bo, MAP_READ) +
  88                    (address - bo_address),
  89          };
  90       }
  91    }
  92
  93    return (struct gen_batch_decode_bo) { };
  94 }
  95
  96 static bool
  97 uint_key_compare(const void *a, const void *b)
  98 {
  99    return a == b;
 100 }
 101
 102 static uint32_t
 103 uint_key_hash(const void *key)
 104 {
 105    return (uintptr_t) key;
 106 }
 107
 108 void
 109 iris_init_batch(struct iris_batch *batch,
 110                 struct iris_screen *screen,
 111                 struct iris_vtable *vtbl,
 112                 struct pipe_debug_callback *dbg,
 113                 uint8_t ring)
 114 {
 115    batch->screen = screen;
 116    batch->vtbl = vtbl;
 117    batch->dbg = dbg;
 118
 119    /* ring should be one of I915_EXEC_RENDER, I915_EXEC_BLT, etc. */
 120    assert((ring & ~I915_EXEC_RING_MASK) == 0);
 121    assert(util_bitcount(ring) == 1);
 122    batch->ring = ring;
 123
 124    batch->exec_count = 0;
 125    batch->exec_array_size = 100;
 126    batch->exec_bos =
 127       malloc(batch->exec_array_size * sizeof(batch->exec_bos[0]));
 128    batch->validation_list =
 129       malloc(batch->exec_array_size * sizeof(batch->validation_list[0]));
 130
 131    batch->cache.render = _mesa_hash_table_create(NULL, _mesa_hash_pointer,
 132                                                  _mesa_key_pointer_equal);
 133    batch->cache.depth = _mesa_set_create(NULL, _mesa_hash_pointer,
 134                                          _mesa_key_pointer_equal);
 135    if (unlikely(INTEL_DEBUG)) {
 136       batch->state_sizes =
 137          _mesa_hash_table_create(NULL, uint_key_hash, uint_key_compare);
 138
 139       const unsigned decode_flags =
 140          GEN_BATCH_DECODE_FULL |
 141          ((INTEL_DEBUG & DEBUG_COLOR) ? GEN_BATCH_DECODE_IN_COLOR : 0) |
 142          GEN_BATCH_DECODE_OFFSETS |
 143          GEN_BATCH_DECODE_FLOATS;
 144
 145       gen_batch_decode_ctx_init(&batch->decoder, &screen->devinfo,
 146                                 stderr, decode_flags, NULL,
 147                                 decode_get_bo, NULL, batch);
 148       batch->decoder.max_vbo_decoded_lines = 32;
 149    }
 150
 151    iris_batch_reset(batch);
 152 }
 153
 154 #define READ_ONCE(x) (*(volatile __typeof__(x) *)&(x))
 155
 156 static unsigned
 157 add_exec_bo(struct iris_batch *batch, struct iris_bo *bo)
 158 {
 159    unsigned index = READ_ONCE(bo->index);
 160
 161    if (index < batch->exec_count && batch->exec_bos[index] == bo)
 162       return index;
 163
 164    /* May have been shared between multiple active batches */
 165    for (index = 0; index < batch->exec_count; index++) {
 166       if (batch->exec_bos[index] == bo)
 167          return index;
 168    }
 169
 170    iris_bo_reference(bo);
 171
 172    if (batch->exec_count == batch->exec_array_size) {
 173       batch->exec_array_size *= 2;
 174       batch->exec_bos =
 175          realloc(batch->exec_bos,
 176                  batch->exec_array_size * sizeof(batch->exec_bos[0]));
 177       batch->validation_list =
 178          realloc(batch->validation_list,
 179                  batch->exec_array_size * sizeof(batch->validation_list[0]));
 180    }
 181
 182    batch->validation_list[batch->exec_count] =
 183       (struct drm_i915_gem_exec_object2) {
 184          .handle = bo->gem_handle,
 185          .offset = bo->gtt_offset,
 186          .flags = bo->kflags,
 187       };
 188
 189    bo->index = batch->exec_count;
 190    batch->exec_bos[batch->exec_count] = bo;
 191    batch->aperture_space += bo->size;
 192
 193    return batch->exec_count++;
 194 }
 195
 196 static void
 197 create_batch(struct iris_batch *batch)
 198 {
 199    struct iris_screen *screen = batch->screen;
 200    struct iris_bufmgr *bufmgr = screen->bufmgr;
 201
 202    batch->bo = iris_bo_alloc(bufmgr, "command buffer",
 203                              BATCH_SZ + BATCH_RESERVED, IRIS_MEMZONE_OTHER);
 204    batch->bo->kflags |= EXEC_OBJECT_CAPTURE;
 205    batch->map = iris_bo_map(NULL, batch->bo, MAP_READ | MAP_WRITE);
 206    batch->map_next = batch->map;
 207
 208    add_exec_bo(batch, batch->bo);
 209 }
 210
 211 static void
 212 iris_batch_reset(struct iris_batch *batch)
 213 {
 214    if (batch->last_bo != NULL) {
 215       iris_bo_unreference(batch->last_bo);
 216       batch->last_bo = NULL;
 217    }
 218    batch->last_bo = batch->bo;
 219    batch->primary_batch_size = 0;
 220
 221    create_batch(batch);
 222    assert(batch->bo->index == 0);
 223
 224    if (batch->state_sizes)
 225       _mesa_hash_table_clear(batch->state_sizes, NULL);
 226
 227    iris_cache_sets_clear(batch);
 228 }
 229
 230 void
 231 iris_batch_free(struct iris_batch *batch)
 232 {
 233    for (int i = 0; i < batch->exec_count; i++) {
 234       iris_bo_unreference(batch->exec_bos[i]);
 235    }
 236    free(batch->exec_bos);
 237    free(batch->validation_list);
 238    iris_bo_unreference(batch->bo);
 239    batch->bo = NULL;
 240    batch->map = NULL;
 241    batch->map_next = NULL;
 242
 243    iris_bo_unreference(batch->last_bo);
 244
 245    _mesa_hash_table_destroy(batch->cache.render, NULL);
 246    _mesa_set_destroy(batch->cache.depth, NULL);
 247
 248    if (batch->state_sizes) {
 249       _mesa_hash_table_destroy(batch->state_sizes, NULL);
 250       gen_batch_decode_ctx_finish(&batch->decoder);
 251    }
 252 }
 253
 254 static unsigned
 255 batch_bytes_used(struct iris_batch *batch)
 256 {
 257    return batch->map_next - batch->map;
 258 }
 259
 260 /**
 261  * If we've chained to a secondary batch, or are getting near to the end,
 262  * then flush.  This should only be called between draws.
 263  */
 264 void
 265 iris_batch_maybe_flush(struct iris_batch *batch, unsigned estimate)
 266 {
 267    if (batch->bo != batch->exec_bos[0] ||
 268        batch_bytes_used(batch) + estimate >= BATCH_SZ) {
 269       iris_batch_flush(batch);
 270    }
 271 }
 272
 273 void
 274 iris_require_command_space(struct iris_batch *batch, unsigned size)
 275 {
 276    const unsigned required_bytes = batch_bytes_used(batch) + size;
 277
 278    if (required_bytes >= BATCH_SZ) {
 279       /* We only support chaining a single time. */
 280       assert(batch->bo == batch->exec_bos[0]);
 281
 282       uint32_t *cmd = batch->map_next;
 283       uint64_t *addr = batch->map_next + 4;
 284       uint32_t *noop = batch->map_next + 12;
 285       batch->map_next += 12;
 286
 287       /* No longer held by batch->bo, still held by validation list */
 288       iris_bo_unreference(batch->bo);
 289       batch->primary_batch_size = ALIGN(batch_bytes_used(batch), 8);
 290       create_batch(batch);
 291
 292       /* Emit MI_BATCH_BUFFER_START to chain to another batch. */
 293       *cmd = (0x31 << 23) | (1 << 8) | (3 - 2);
 294       *addr = batch->bo->gtt_offset;
 295       *noop = 0;
 296    }
 297 }
 298
 299 void *
 300 iris_get_command_space(struct iris_batch *batch, unsigned bytes)
 301 {
 302    iris_require_command_space(batch, bytes);
 303    void *map = batch->map_next;
 304    batch->map_next += bytes;
 305    return map;
 306 }
 307
 308 void
 309 iris_batch_emit(struct iris_batch *batch, const void *data, unsigned size)
 310 {
 311    void *map = iris_get_command_space(batch, size);
 312    memcpy(map, data, size);
 313 }
 314
 315 /**
 316  * Called from iris_batch_flush before emitting MI_BATCHBUFFER_END and
 317  * sending it off.
 318  *
 319  * This function can emit state (say, to preserve registers that aren't saved
 320  * between batches).
 321  */
 322 static void
 323 iris_finish_batch(struct iris_batch *batch)
 324 {
 325    if (batch->bo == batch->exec_bos[0])
 326       batch->primary_batch_size = batch_bytes_used(batch);
 327
 328    // XXX: ISP DIS
 329
 330    /* Emit MI_BATCH_BUFFER_END to finish our batch.  Note that execbuf2
 331     * requires our batch size to be QWord aligned, so we pad it out if
 332     * necessary by emitting an extra MI_NOOP after the end.
 333     */
 334    const bool qword_aligned = (batch_bytes_used(batch) % 8) == 0;
 335    uint32_t *map = batch->map_next;
 336
 337    map[0] = (0xA << 23);
 338    map[1] = 0;
 339
 340    batch->map_next += qword_aligned ? 8 : 4;
 341 }
 342
 343 static int
 344 submit_batch(struct iris_batch *batch, int in_fence_fd, int *out_fence_fd)
 345 {
 346    iris_bo_unmap(batch->bo);
 347
 348    /* The requirement for using I915_EXEC_NO_RELOC are:
 349     *
 350     *   The addresses written in the objects must match the corresponding
 351     *   reloc.gtt_offset which in turn must match the corresponding
 352     *   execobject.offset.
 353     *
 354     *   Any render targets written to in the batch must be flagged with
 355     *   EXEC_OBJECT_WRITE.
 356     *
 357     *   To avoid stalling, execobject.offset should match the current
 358     *   address of that object within the active context.
 359     */
 360    struct drm_i915_gem_execbuffer2 execbuf = {
 361       .buffers_ptr = (uintptr_t) batch->validation_list,
 362       .buffer_count = batch->exec_count,
 363       .batch_start_offset = 0,
 364       .batch_len = batch->primary_batch_size,
 365       .flags = batch->ring |
 366                I915_EXEC_NO_RELOC |
 367                I915_EXEC_BATCH_FIRST |
 368                I915_EXEC_HANDLE_LUT,
 369       .rsvd1 = batch->hw_ctx_id, /* rsvd1 is actually the context ID */
 370    };
 371
 372    unsigned long cmd = DRM_IOCTL_I915_GEM_EXECBUFFER2;
 373
 374    if (in_fence_fd != -1) {
 375       execbuf.rsvd2 = in_fence_fd;
 376       execbuf.flags |= I915_EXEC_FENCE_IN;
 377    }
 378
 379    if (out_fence_fd != NULL) {
 380       cmd = DRM_IOCTL_I915_GEM_EXECBUFFER2_WR;
 381       *out_fence_fd = -1;
 382       execbuf.flags |= I915_EXEC_FENCE_OUT;
 383    }
 384
 385    int ret = drm_ioctl(batch->screen->fd, cmd, &execbuf);
 386    if (ret != 0) {
 387       ret = -errno;
 388       DBG("execbuf FAILED: errno = %d\n", -ret);
 389    } else {
 390       DBG("execbuf succeeded\n");
 391    }
 392
 393    for (int i = 0; i < batch->exec_count; i++) {
 394       struct iris_bo *bo = batch->exec_bos[i];
 395
 396       bo->idle = false;
 397       bo->index = -1;
 398    }
 399
 400    if (ret == 0 && out_fence_fd != NULL)
 401       *out_fence_fd = execbuf.rsvd2 >> 32;
 402
 403    return ret;
 404 }
 405
 406 /**
 407  * The in_fence_fd is ignored if -1.  Otherwise this function takes ownership
 408  * of the fd.
 409  *
 410  * The out_fence_fd is ignored if NULL. Otherwise, the caller takes ownership
 411  * of the returned fd.
 412  */
 413 int
 414 _iris_batch_flush_fence(struct iris_batch *batch,
 415                         int in_fence_fd, int *out_fence_fd,
 416                         const char *file, int line)
 417 {
 418    if (batch_bytes_used(batch) == 0)
 419       return 0;
 420
 421    iris_finish_batch(batch);
 422
 423    if (unlikely(INTEL_DEBUG & (DEBUG_BATCH | DEBUG_SUBMIT))) {
 424       int bytes_for_commands = batch_bytes_used(batch);
 425       if (batch->bo != batch->exec_bos[0])
 426          bytes_for_commands += batch->primary_batch_size;
 427       fprintf(stderr, "%19s:%-3d: Batchbuffer flush with %5db (%0.1f%%), "
 428               "%4d BOs (%0.1fMb aperture)\n",
 429               file, line,
 430               bytes_for_commands, 100.0f * bytes_for_commands / BATCH_SZ,
 431               batch->exec_count,
 432               (float) batch->aperture_space / (1024 * 1024));
 433       dump_validation_list(batch);
 434    }
 435
 436    if (unlikely(INTEL_DEBUG & DEBUG_BATCH)) {
 437       decode_batch(batch);
 438    }
 439
 440    int ret = submit_batch(batch, in_fence_fd, out_fence_fd);
 441
 442    //throttle(iris);
 443
 444    if (ret < 0)
 445       return ret;
 446
 447    //if (iris->ctx.Const.ResetStrategy == GL_LOSE_CONTEXT_ON_RESET_ARB)
 448       //iris_check_for_reset(ice);
 449
 450    if (unlikely(INTEL_DEBUG & DEBUG_SYNC)) {
 451       dbg_printf("waiting for idle\n");
 452       iris_bo_wait_rendering(batch->bo);
 453    }
 454
 455    /* Clean up after the batch we submitted and prepare for a new one. */
 456    for (int i = 0; i < batch->exec_count; i++) {
 457       iris_bo_unreference(batch->exec_bos[i]);
 458       batch->exec_bos[i] = NULL;
 459    }
 460    batch->exec_count = 0;
 461    batch->aperture_space = 0;
 462
 463    /* Start a new batch buffer. */
 464    iris_batch_reset(batch);
 465
 466    return 0;
 467 }
 468
 469 bool
 470 iris_batch_references(struct iris_batch *batch, struct iris_bo *bo)
 471 {
 472    unsigned index = READ_ONCE(bo->index);
 473    if (index < batch->exec_count && batch->exec_bos[index] == bo)
 474       return true;
 475
 476    for (int i = 0; i < batch->exec_count; i++) {
 477       if (batch->exec_bos[i] == bo)
 478          return true;
 479    }
 480    return false;
 481 }
 482
 483 /* This is the only way buffers get added to the validate list.
 484  */
 485 void
 486 iris_use_pinned_bo(struct iris_batch *batch,
 487                    struct iris_bo *bo,
 488                    bool writable)
 489 {
 490    assert(bo->kflags & EXEC_OBJECT_PINNED);
 491    unsigned index = add_exec_bo(batch, bo);
 492    if (writable)
 493       batch->validation_list[index].flags |= EXEC_OBJECT_WRITE;
 494 }
 495
 496 static void
 497 decode_batch(struct iris_batch *batch)
 498 {
 499    //if (batch->bo != batch->exec_bos[0]) {
 500    void *map = iris_bo_map(batch->dbg, batch->exec_bos[0], MAP_READ);
 501    gen_print_batch(&batch->decoder, map, batch->primary_batch_size,
 502                    batch->exec_bos[0]->gtt_offset);
 503
 504       //fprintf(stderr, "Secondary batch...\n");
 505    //}
 506
 507    //gen_print_batch(&batch->decoder, batch->map, batch_bytes_used(batch),
 508                    //batch->bo->gtt_offset);
 509 }