src/mesa/drivers/dri/i965/intel_batchbuffer.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2006 Tungsten Graphics, Inc., Cedar Park, Texas.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  19  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  21  * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
  22  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  23  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  24  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27
  28 #include "intel_context.h"
  29 #include "intel_batchbuffer.h"
  30 #include "intel_buffer_objects.h"
  31 #include "intel_reg.h"
  32 #include "intel_bufmgr.h"
  33 #include "intel_buffers.h"
  34
  35 static void
  36 intel_batchbuffer_reset(struct intel_context *intel);
  37
  38 struct cached_batch_item {
  39    struct cached_batch_item *next;
  40    uint16_t header;
  41    uint16_t size;
  42 };
  43
  44 static void clear_cache( struct intel_context *intel )
  45 {
  46    struct cached_batch_item *item = intel->batch.cached_items;
  47
  48    while (item) {
  49       struct cached_batch_item *next = item->next;
  50       free(item);
  51       item = next;
  52    }
  53
  54    intel->batch.cached_items = NULL;
  55 }
  56
  57 void
  58 intel_batchbuffer_init(struct intel_context *intel)
  59 {
  60    intel_batchbuffer_reset(intel);
  61
  62    if (intel->gen >= 6) {
  63       /* We can't just use brw_state_batch to get a chunk of space for
  64        * the gen6 workaround because it involves actually writing to
  65        * the buffer, and the kernel doesn't let us write to the batch.
  66        */
  67       intel->batch.workaround_bo = drm_intel_bo_alloc(intel->bufmgr,
  68                                                       "pipe_control workaround",
  69                                                       4096, 4096);
  70    }
  71
  72    if (!intel->has_llc) {
  73       intel->batch.cpu_map = malloc(intel->maxBatchSize);
  74       intel->batch.map = intel->batch.cpu_map;
  75    }
  76 }
  77
  78 static void
  79 intel_batchbuffer_reset(struct intel_context *intel)
  80 {
  81    if (intel->batch.last_bo != NULL) {
  82       drm_intel_bo_unreference(intel->batch.last_bo);
  83       intel->batch.last_bo = NULL;
  84    }
  85    intel->batch.last_bo = intel->batch.bo;
  86
  87    clear_cache(intel);
  88
  89    intel->batch.bo = drm_intel_bo_alloc(intel->bufmgr, "batchbuffer",
  90                                         intel->maxBatchSize, 4096);
  91    if (intel->has_llc) {
  92       drm_intel_bo_map(intel->batch.bo, true);
  93       intel->batch.map = intel->batch.bo->virtual;
  94    }
  95
  96    intel->batch.reserved_space = BATCH_RESERVED;
  97    intel->batch.state_batch_offset = intel->batch.bo->size;
  98    intel->batch.used = 0;
  99    intel->batch.needs_sol_reset = false;
 100 }
 101
 102 void
 103 intel_batchbuffer_save_state(struct intel_context *intel)
 104 {
 105    intel->batch.saved.used = intel->batch.used;
 106    intel->batch.saved.reloc_count =
 107       drm_intel_gem_bo_get_reloc_count(intel->batch.bo);
 108 }
 109
 110 void
 111 intel_batchbuffer_reset_to_saved(struct intel_context *intel)
 112 {
 113    drm_intel_gem_bo_clear_relocs(intel->batch.bo, intel->batch.saved.reloc_count);
 114
 115    intel->batch.used = intel->batch.saved.used;
 116
 117    /* Cached batch state is dead, since we just cleared some unknown part of the
 118     * batchbuffer.  Assume that the caller resets any other state necessary.
 119     */
 120    clear_cache(intel);
 121 }
 122
 123 void
 124 intel_batchbuffer_free(struct intel_context *intel)
 125 {
 126    free(intel->batch.cpu_map);
 127    drm_intel_bo_unreference(intel->batch.last_bo);
 128    drm_intel_bo_unreference(intel->batch.bo);
 129    drm_intel_bo_unreference(intel->batch.workaround_bo);
 130    clear_cache(intel);
 131 }
 132
 133 static void
 134 do_batch_dump(struct intel_context *intel)
 135 {
 136    struct drm_intel_decode *decode;
 137    struct intel_batchbuffer *batch = &intel->batch;
 138    int ret;
 139
 140    decode = drm_intel_decode_context_alloc(intel->intelScreen->deviceID);
 141    if (!decode)
 142       return;
 143
 144    ret = drm_intel_bo_map(batch->bo, false);
 145    if (ret == 0) {
 146       drm_intel_decode_set_batch_pointer(decode,
 147                                          batch->bo->virtual,
 148                                          batch->bo->offset,
 149                                          batch->used);
 150    } else {
 151       fprintf(stderr,
 152               "WARNING: failed to map batchbuffer (%s), "
 153               "dumping uploaded data instead.\n", strerror(ret));
 154
 155       drm_intel_decode_set_batch_pointer(decode,
 156                                          batch->map,
 157                                          batch->bo->offset,
 158                                          batch->used);
 159    }
 160
 161    drm_intel_decode(decode);
 162
 163    drm_intel_decode_context_free(decode);
 164
 165    if (ret == 0) {
 166       drm_intel_bo_unmap(batch->bo);
 167
 168       if (intel->vtbl.debug_batch != NULL)
 169          intel->vtbl.debug_batch(intel);
 170    }
 171 }
 172
 173 /* TODO: Push this whole function into bufmgr.
 174  */
 175 static int
 176 do_flush_locked(struct intel_context *intel)
 177 {
 178    struct intel_batchbuffer *batch = &intel->batch;
 179    int ret = 0;
 180
 181    if (intel->has_llc) {
 182       drm_intel_bo_unmap(batch->bo);
 183    } else {
 184       ret = drm_intel_bo_subdata(batch->bo, 0, 4*batch->used, batch->map);
 185       if (ret == 0 && batch->state_batch_offset != batch->bo->size) {
 186          ret = drm_intel_bo_subdata(batch->bo,
 187                                     batch->state_batch_offset,
 188                                     batch->bo->size - batch->state_batch_offset,
 189                                     (char *)batch->map + batch->state_batch_offset);
 190       }
 191    }
 192
 193    if (!intel->intelScreen->no_hw) {
 194       int flags;
 195
 196       if (intel->gen < 6 || !batch->is_blit) {
 197          flags = I915_EXEC_RENDER;
 198       } else {
 199          flags = I915_EXEC_BLT;
 200       }
 201
 202       if (batch->needs_sol_reset)
 203          flags |= I915_EXEC_GEN7_SOL_RESET;
 204
 205       if (ret == 0) {
 206          if (unlikely(INTEL_DEBUG & DEBUG_AUB) && intel->vtbl.annotate_aub)
 207             intel->vtbl.annotate_aub(intel);
 208          if (intel->hw_ctx == NULL || batch->is_blit) {
 209             ret = drm_intel_bo_mrb_exec(batch->bo, 4 * batch->used, NULL, 0, 0,
 210                                         flags);
 211          } else {
 212             ret = drm_intel_gem_bo_context_exec(batch->bo, intel->hw_ctx,
 213                                                 4 * batch->used, flags);
 214          }
 215       }
 216    }
 217
 218    if (unlikely(INTEL_DEBUG & DEBUG_BATCH))
 219       do_batch_dump(intel);
 220
 221    if (ret != 0) {
 222       fprintf(stderr, "intel_do_flush_locked failed: %s\n", strerror(-ret));
 223       exit(1);
 224    }
 225    intel->vtbl.new_batch(intel);
 226
 227    return ret;
 228 }
 229
 230 int
 231 _intel_batchbuffer_flush(struct intel_context *intel,
 232                          const char *file, int line)
 233 {
 234    int ret;
 235
 236    if (intel->batch.used == 0)
 237       return 0;
 238
 239    if (intel->first_post_swapbuffers_batch == NULL) {
 240       intel->first_post_swapbuffers_batch = intel->batch.bo;
 241       drm_intel_bo_reference(intel->first_post_swapbuffers_batch);
 242    }
 243
 244    if (unlikely(INTEL_DEBUG & DEBUG_BATCH))
 245       fprintf(stderr, "%s:%d: Batchbuffer flush with %db used\n", file, line,
 246               4*intel->batch.used);
 247
 248    intel->batch.reserved_space = 0;
 249
 250    if (intel->vtbl.finish_batch)
 251       intel->vtbl.finish_batch(intel);
 252
 253    /* Mark the end of the buffer. */
 254    intel_batchbuffer_emit_dword(intel, MI_BATCH_BUFFER_END);
 255    if (intel->batch.used & 1) {
 256       /* Round batchbuffer usage to 2 DWORDs. */
 257       intel_batchbuffer_emit_dword(intel, MI_NOOP);
 258    }
 259
 260    intel_upload_finish(intel);
 261
 262    /* Check that we didn't just wrap our batchbuffer at a bad time. */
 263    assert(!intel->no_batch_wrap);
 264
 265    ret = do_flush_locked(intel);
 266
 267    if (unlikely(INTEL_DEBUG & DEBUG_SYNC)) {
 268       fprintf(stderr, "waiting for idle\n");
 269       drm_intel_bo_wait_rendering(intel->batch.bo);
 270    }
 271
 272    /* Reset the buffer:
 273     */
 274    intel_batchbuffer_reset(intel);
 275
 276    return ret;
 277 }
 278
 279
 280 /*  This is the only way buffers get added to the validate list.
 281  */
 282 bool
 283 intel_batchbuffer_emit_reloc(struct intel_context *intel,
 284                              drm_intel_bo *buffer,
 285                              uint32_t read_domains, uint32_t write_domain,
 286                              uint32_t delta)
 287 {
 288    int ret;
 289
 290    ret = drm_intel_bo_emit_reloc(intel->batch.bo, 4*intel->batch.used,
 291                                  buffer, delta,
 292                                  read_domains, write_domain);
 293    assert(ret == 0);
 294    (void)ret;
 295
 296    /*
 297     * Using the old buffer offset, write in what the right data would be, in case
 298     * the buffer doesn't move and we can short-circuit the relocation processing
 299     * in the kernel
 300     */
 301    intel_batchbuffer_emit_dword(intel, buffer->offset + delta);
 302
 303    return true;
 304 }
 305
 306 bool
 307 intel_batchbuffer_emit_reloc_fenced(struct intel_context *intel,
 308                                     drm_intel_bo *buffer,
 309                                     uint32_t read_domains,
 310                                     uint32_t write_domain,
 311                                     uint32_t delta)
 312 {
 313    int ret;
 314
 315    ret = drm_intel_bo_emit_reloc_fence(intel->batch.bo, 4*intel->batch.used,
 316                                        buffer, delta,
 317                                        read_domains, write_domain);
 318    assert(ret == 0);
 319    (void)ret;
 320
 321    /*
 322     * Using the old buffer offset, write in what the right data would
 323     * be, in case the buffer doesn't move and we can short-circuit the
 324     * relocation processing in the kernel
 325     */
 326    intel_batchbuffer_emit_dword(intel, buffer->offset + delta);
 327
 328    return true;
 329 }
 330
 331 void
 332 intel_batchbuffer_data(struct intel_context *intel,
 333                        const void *data, GLuint bytes, bool is_blit)
 334 {
 335    assert((bytes & 3) == 0);
 336    intel_batchbuffer_require_space(intel, bytes, is_blit);
 337    __memcpy(intel->batch.map + intel->batch.used, data, bytes);
 338    intel->batch.used += bytes >> 2;
 339 }
 340
 341 void
 342 intel_batchbuffer_cached_advance(struct intel_context *intel)
 343 {
 344    struct cached_batch_item **prev = &intel->batch.cached_items, *item;
 345    uint32_t sz = (intel->batch.used - intel->batch.emit) * sizeof(uint32_t);
 346    uint32_t *start = intel->batch.map + intel->batch.emit;
 347    uint16_t op = *start >> 16;
 348
 349    while (*prev) {
 350       uint32_t *old;
 351
 352       item = *prev;
 353       old = intel->batch.map + item->header;
 354       if (op == *old >> 16) {
 355          if (item->size == sz && memcmp(old, start, sz) == 0) {
 356             if (prev != &intel->batch.cached_items) {
 357                *prev = item->next;
 358                item->next = intel->batch.cached_items;
 359                intel->batch.cached_items = item;
 360             }
 361             intel->batch.used = intel->batch.emit;
 362             return;
 363          }
 364
 365          goto emit;
 366       }
 367       prev = &item->next;
 368    }
 369
 370    item = malloc(sizeof(struct cached_batch_item));
 371    if (item == NULL)
 372       return;
 373
 374    item->next = intel->batch.cached_items;
 375    intel->batch.cached_items = item;
 376
 377 emit:
 378    item->size = sz;
 379    item->header = intel->batch.emit;
 380 }
 381
 382 /**
 383  * Restriction [DevSNB, DevIVB]:
 384  *
 385  * Prior to changing Depth/Stencil Buffer state (i.e. any combination of
 386  * 3DSTATE_DEPTH_BUFFER, 3DSTATE_CLEAR_PARAMS, 3DSTATE_STENCIL_BUFFER,
 387  * 3DSTATE_HIER_DEPTH_BUFFER) SW must first issue a pipelined depth stall
 388  * (PIPE_CONTROL with Depth Stall bit set), followed by a pipelined depth
 389  * cache flush (PIPE_CONTROL with Depth Flush Bit set), followed by
 390  * another pipelined depth stall (PIPE_CONTROL with Depth Stall bit set),
 391  * unless SW can otherwise guarantee that the pipeline from WM onwards is
 392  * already flushed (e.g., via a preceding MI_FLUSH).
 393  */
 394 void
 395 intel_emit_depth_stall_flushes(struct intel_context *intel)
 396 {
 397    assert(intel->gen >= 6 && intel->gen <= 7);
 398
 399    BEGIN_BATCH(4);
 400    OUT_BATCH(_3DSTATE_PIPE_CONTROL | (4 - 2));
 401    OUT_BATCH(PIPE_CONTROL_DEPTH_STALL);
 402    OUT_BATCH(0); /* address */
 403    OUT_BATCH(0); /* write data */
 404    ADVANCE_BATCH()
 405
 406    BEGIN_BATCH(4);
 407    OUT_BATCH(_3DSTATE_PIPE_CONTROL | (4 - 2));
 408    OUT_BATCH(PIPE_CONTROL_DEPTH_CACHE_FLUSH);
 409    OUT_BATCH(0); /* address */
 410    OUT_BATCH(0); /* write data */
 411    ADVANCE_BATCH();
 412
 413    BEGIN_BATCH(4);
 414    OUT_BATCH(_3DSTATE_PIPE_CONTROL | (4 - 2));
 415    OUT_BATCH(PIPE_CONTROL_DEPTH_STALL);
 416    OUT_BATCH(0); /* address */
 417    OUT_BATCH(0); /* write data */
 418    ADVANCE_BATCH();
 419 }
 420
 421 /**
 422  * From the BSpec, volume 2a.03: VS Stage Input / State:
 423  * "[DevIVB] A PIPE_CONTROL with Post-Sync Operation set to 1h and a depth
 424  *  stall needs to be sent just prior to any 3DSTATE_VS, 3DSTATE_URB_VS,
 425  *  3DSTATE_CONSTANT_VS, 3DSTATE_BINDING_TABLE_POINTER_VS,
 426  *  3DSTATE_SAMPLER_STATE_POINTER_VS command.  Only one PIPE_CONTROL needs
 427  *  to be sent before any combination of VS associated 3DSTATE."
 428  */
 429 void
 430 gen7_emit_vs_workaround_flush(struct intel_context *intel)
 431 {
 432    assert(intel->gen == 7);
 433
 434    BEGIN_BATCH(4);
 435    OUT_BATCH(_3DSTATE_PIPE_CONTROL | (4 - 2));
 436    OUT_BATCH(PIPE_CONTROL_DEPTH_STALL | PIPE_CONTROL_WRITE_IMMEDIATE);
 437    OUT_RELOC(intel->batch.workaround_bo,
 438              I915_GEM_DOMAIN_INSTRUCTION, I915_GEM_DOMAIN_INSTRUCTION, 0);
 439    OUT_BATCH(0); /* write data */
 440    ADVANCE_BATCH();
 441 }
 442
 443 /**
 444  * Emits a PIPE_CONTROL with a non-zero post-sync operation, for
 445  * implementing two workarounds on gen6.  From section 1.4.7.1
 446  * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1:
 447  *
 448  * [DevSNB-C+{W/A}] Before any depth stall flush (including those
 449  * produced by non-pipelined state commands), software needs to first
 450  * send a PIPE_CONTROL with no bits set except Post-Sync Operation !=
 451  * 0.
 452  *
 453  * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable
 454  * =1, a PIPE_CONTROL with any non-zero post-sync-op is required.
 455  *
 456  * And the workaround for these two requires this workaround first:
 457  *
 458  * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent
 459  * BEFORE the pipe-control with a post-sync op and no write-cache
 460  * flushes.
 461  *
 462  * And this last workaround is tricky because of the requirements on
 463  * that bit.  From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM
 464  * volume 2 part 1:
 465  *
 466  *     "1 of the following must also be set:
 467  *      - Render Target Cache Flush Enable ([12] of DW1)
 468  *      - Depth Cache Flush Enable ([0] of DW1)
 469  *      - Stall at Pixel Scoreboard ([1] of DW1)
 470  *      - Depth Stall ([13] of DW1)
 471  *      - Post-Sync Operation ([13] of DW1)
 472  *      - Notify Enable ([8] of DW1)"
 473  *
 474  * The cache flushes require the workaround flush that triggered this
 475  * one, so we can't use it.  Depth stall would trigger the same.
 476  * Post-sync nonzero is what triggered this second workaround, so we
 477  * can't use that one either.  Notify enable is IRQs, which aren't
 478  * really our business.  That leaves only stall at scoreboard.
 479  */
 480 void
 481 intel_emit_post_sync_nonzero_flush(struct intel_context *intel)
 482 {
 483    if (!intel->batch.need_workaround_flush)
 484       return;
 485
 486    BEGIN_BATCH(4);
 487    OUT_BATCH(_3DSTATE_PIPE_CONTROL | (4 - 2));
 488    OUT_BATCH(PIPE_CONTROL_CS_STALL |
 489              PIPE_CONTROL_STALL_AT_SCOREBOARD);
 490    OUT_BATCH(0); /* address */
 491    OUT_BATCH(0); /* write data */
 492    ADVANCE_BATCH();
 493
 494    BEGIN_BATCH(4);
 495    OUT_BATCH(_3DSTATE_PIPE_CONTROL | (4 - 2));
 496    OUT_BATCH(PIPE_CONTROL_WRITE_IMMEDIATE);
 497    OUT_RELOC(intel->batch.workaround_bo,
 498              I915_GEM_DOMAIN_INSTRUCTION, I915_GEM_DOMAIN_INSTRUCTION, 0);
 499    OUT_BATCH(0); /* write data */
 500    ADVANCE_BATCH();
 501
 502    intel->batch.need_workaround_flush = false;
 503 }
 504
 505 /* Emit a pipelined flush to either flush render and texture cache for
 506  * reading from a FBO-drawn texture, or flush so that frontbuffer
 507  * render appears on the screen in DRI1.
 508  *
 509  * This is also used for the always_flush_cache driconf debug option.
 510  */
 511 void
 512 intel_batchbuffer_emit_mi_flush(struct intel_context *intel)
 513 {
 514    if (intel->gen >= 6) {
 515       if (intel->batch.is_blit) {
 516          BEGIN_BATCH_BLT(4);
 517          OUT_BATCH(MI_FLUSH_DW);
 518          OUT_BATCH(0);
 519          OUT_BATCH(0);
 520          OUT_BATCH(0);
 521          ADVANCE_BATCH();
 522       } else {
 523          if (intel->gen == 6) {
 524             /* Hardware workaround: SNB B-Spec says:
 525              *
 526              * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache
 527              * Flush Enable =1, a PIPE_CONTROL with any non-zero
 528              * post-sync-op is required.
 529              */
 530             intel_emit_post_sync_nonzero_flush(intel);
 531          }
 532
 533          BEGIN_BATCH(4);
 534          OUT_BATCH(_3DSTATE_PIPE_CONTROL | (4 - 2));
 535          OUT_BATCH(PIPE_CONTROL_INSTRUCTION_FLUSH |
 536                    PIPE_CONTROL_WRITE_FLUSH |
 537                    PIPE_CONTROL_DEPTH_CACHE_FLUSH |
 538                    PIPE_CONTROL_VF_CACHE_INVALIDATE |
 539                    PIPE_CONTROL_TC_FLUSH |
 540                    PIPE_CONTROL_NO_WRITE |
 541                    PIPE_CONTROL_CS_STALL);
 542          OUT_BATCH(0); /* write address */
 543          OUT_BATCH(0); /* write data */
 544          ADVANCE_BATCH();
 545       }
 546    } else {
 547       BEGIN_BATCH(4);
 548       OUT_BATCH(_3DSTATE_PIPE_CONTROL | (4 - 2) |
 549                 PIPE_CONTROL_WRITE_FLUSH |
 550                 PIPE_CONTROL_NO_WRITE);
 551       OUT_BATCH(0); /* write address */
 552       OUT_BATCH(0); /* write data */
 553       OUT_BATCH(0); /* write data */
 554       ADVANCE_BATCH();
 555    }
 556 }