src/gallium/drivers/radeonsi/r600_hw_context.c

   1 /*
   2  * Copyright 2010 Jerome Glisse <glisse@freedesktop.org>
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * on the rights to use, copy, modify, merge, publish, distribute, sub
   8  * license, and/or sell copies of the Software, and to permit persons to whom
   9  * the Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  18  * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
  19  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  20  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  21  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *      Jerome Glisse
  25  */
  26 #include "r600_hw_context_priv.h"
  27 #include "radeonsi_pm4.h"
  28 #include "radeonsi_pipe.h"
  29 #include "sid.h"
  30 #include "util/u_memory.h"
  31 #include <errno.h>
  32
  33 #define GROUP_FORCE_NEW_BLOCK   0
  34
  35 /* Get backends mask */
  36 void si_get_backend_mask(struct r600_context *ctx)
  37 {
  38         struct radeon_winsys_cs *cs = ctx->cs;
  39         struct si_resource *buffer;
  40         uint32_t *results;
  41         unsigned num_backends = ctx->screen->info.r600_num_backends;
  42         unsigned i, mask = 0;
  43
  44         /* if backend_map query is supported by the kernel */
  45         if (ctx->screen->info.r600_backend_map_valid) {
  46                 unsigned num_tile_pipes = ctx->screen->info.r600_num_tile_pipes;
  47                 unsigned backend_map = ctx->screen->info.r600_backend_map;
  48                 unsigned item_width = 4, item_mask = 0x7;
  49
  50                 while(num_tile_pipes--) {
  51                         i = backend_map & item_mask;
  52                         mask |= (1<<i);
  53                         backend_map >>= item_width;
  54                 }
  55                 if (mask != 0) {
  56                         ctx->backend_mask = mask;
  57                         return;
  58                 }
  59         }
  60
  61         /* otherwise backup path for older kernels */
  62
  63         /* create buffer for event data */
  64         buffer = si_resource_create_custom(&ctx->screen->screen,
  65                                            PIPE_USAGE_STAGING,
  66                                            ctx->max_db*16);
  67         if (!buffer)
  68                 goto err;
  69
  70         /* initialize buffer with zeroes */
  71         results = ctx->ws->buffer_map(buffer->cs_buf, ctx->cs, PIPE_TRANSFER_WRITE);
  72         if (results) {
  73                 uint64_t va = 0;
  74
  75                 memset(results, 0, ctx->max_db * 4 * 4);
  76                 ctx->ws->buffer_unmap(buffer->cs_buf);
  77
  78                 /* emit EVENT_WRITE for ZPASS_DONE */
  79                 va = r600_resource_va(&ctx->screen->screen, (void *)buffer);
  80                 cs->buf[cs->cdw++] = PKT3(PKT3_EVENT_WRITE, 2, 0);
  81                 cs->buf[cs->cdw++] = EVENT_TYPE(EVENT_TYPE_ZPASS_DONE) | EVENT_INDEX(1);
  82                 cs->buf[cs->cdw++] = va;
  83                 cs->buf[cs->cdw++] = va >> 32;
  84
  85                 cs->buf[cs->cdw++] = PKT3(PKT3_NOP, 0, 0);
  86                 cs->buf[cs->cdw++] = r600_context_bo_reloc(ctx, buffer, RADEON_USAGE_WRITE);
  87
  88                 /* analyze results */
  89                 results = ctx->ws->buffer_map(buffer->cs_buf, ctx->cs, PIPE_TRANSFER_READ);
  90                 if (results) {
  91                         for(i = 0; i < ctx->max_db; i++) {
  92                                 /* at least highest bit will be set if backend is used */
  93                                 if (results[i*4 + 1])
  94                                         mask |= (1<<i);
  95                         }
  96                         ctx->ws->buffer_unmap(buffer->cs_buf);
  97                 }
  98         }
  99
 100         si_resource_reference(&buffer, NULL);
 101
 102         if (mask != 0) {
 103                 ctx->backend_mask = mask;
 104                 return;
 105         }
 106
 107 err:
 108         /* fallback to old method - set num_backends lower bits to 1 */
 109         ctx->backend_mask = (~((uint32_t)0))>>(32-num_backends);
 110         return;
 111 }
 112
 113 bool si_is_timer_query(unsigned type)
 114 {
 115         return type == PIPE_QUERY_TIME_ELAPSED ||
 116                 type == PIPE_QUERY_TIMESTAMP ||
 117                 type == PIPE_QUERY_TIMESTAMP_DISJOINT;
 118 }
 119
 120 bool si_query_needs_begin(unsigned type)
 121 {
 122         return type != PIPE_QUERY_TIMESTAMP;
 123 }
 124
 125 /* initialize */
 126 void si_need_cs_space(struct r600_context *ctx, unsigned num_dw,
 127                         boolean count_draw_in)
 128 {
 129         int i;
 130
 131         /* The number of dwords we already used in the CS so far. */
 132         num_dw += ctx->cs->cdw;
 133
 134         for (i = 0; i < SI_NUM_ATOMS(ctx); i++) {
 135                 if (ctx->atoms.array[i]->dirty) {
 136                         num_dw += ctx->atoms.array[i]->num_dw;
 137                 }
 138         }
 139
 140         if (count_draw_in) {
 141                 /* The number of dwords all the dirty states would take. */
 142                 num_dw += ctx->pm4_dirty_cdwords;
 143
 144                 /* The upper-bound of how much a draw command would take. */
 145                 num_dw += SI_MAX_DRAW_CS_DWORDS;
 146         }
 147
 148         /* Count in queries_suspend. */
 149         num_dw += ctx->num_cs_dw_nontimer_queries_suspend;
 150
 151         /* Count in streamout_end at the end of CS. */
 152         num_dw += ctx->num_cs_dw_streamout_end;
 153
 154         /* Count in render_condition(NULL) at the end of CS. */
 155         if (ctx->predicate_drawing) {
 156                 num_dw += 3;
 157         }
 158
 159         /* Count in framebuffer cache flushes at the end of CS. */
 160         num_dw += 7; /* one SURFACE_SYNC and CACHE_FLUSH_AND_INV (r6xx-only) */
 161
 162         /* Save 16 dwords for the fence mechanism. */
 163         num_dw += 16;
 164
 165 #if R600_TRACE_CS
 166         if (ctx->screen->trace_bo) {
 167                 num_dw += R600_TRACE_CS_DWORDS;
 168         }
 169 #endif
 170
 171         /* Flush if there's not enough space. */
 172         if (num_dw > RADEON_MAX_CMDBUF_DWORDS) {
 173                 radeonsi_flush(&ctx->context, NULL, RADEON_FLUSH_ASYNC);
 174         }
 175 }
 176
 177 static void r600_flush_framebuffer(struct r600_context *ctx)
 178 {
 179         struct si_pm4_state *pm4;
 180
 181         if (!(ctx->flags & R600_CONTEXT_DST_CACHES_DIRTY))
 182                 return;
 183
 184         pm4 = si_pm4_alloc_state(ctx);
 185
 186         if (pm4 == NULL)
 187                 return;
 188
 189         si_cmd_surface_sync(pm4, S_0085F0_CB0_DEST_BASE_ENA(1) |
 190                                 S_0085F0_CB1_DEST_BASE_ENA(1) |
 191                                 S_0085F0_CB2_DEST_BASE_ENA(1) |
 192                                 S_0085F0_CB3_DEST_BASE_ENA(1) |
 193                                 S_0085F0_CB4_DEST_BASE_ENA(1) |
 194                                 S_0085F0_CB5_DEST_BASE_ENA(1) |
 195                                 S_0085F0_CB6_DEST_BASE_ENA(1) |
 196                                 S_0085F0_CB7_DEST_BASE_ENA(1) |
 197                                 S_0085F0_DB_ACTION_ENA(1) |
 198                                 S_0085F0_DB_DEST_BASE_ENA(1));
 199         si_cmd_flush_and_inv_cb_meta(pm4);
 200
 201         si_pm4_emit(ctx, pm4);
 202         si_pm4_free_state(ctx, pm4, ~0);
 203
 204         ctx->flags &= ~R600_CONTEXT_DST_CACHES_DIRTY;
 205         ctx->flush_and_inv_cb_meta = false;
 206 }
 207
 208 void si_context_flush(struct r600_context *ctx, unsigned flags)
 209 {
 210         struct radeon_winsys_cs *cs = ctx->cs;
 211         bool queries_suspended = false;
 212
 213 #if 0
 214         bool streamout_suspended = false;
 215 #endif
 216
 217         if (!cs->cdw)
 218                 return;
 219
 220         /* suspend queries */
 221         if (ctx->num_cs_dw_nontimer_queries_suspend) {
 222                 r600_context_queries_suspend(ctx);
 223                 queries_suspended = true;
 224         }
 225
 226 #if 0
 227         if (ctx->num_cs_dw_streamout_end) {
 228                 r600_context_streamout_end(ctx);
 229                 streamout_suspended = true;
 230         }
 231 #endif
 232
 233         r600_flush_framebuffer(ctx);
 234
 235         /* partial flush is needed to avoid lockups on some chips with user fences */
 236         cs->buf[cs->cdw++] = PKT3(PKT3_EVENT_WRITE, 0, 0);
 237         cs->buf[cs->cdw++] = EVENT_TYPE(EVENT_TYPE_PS_PARTIAL_FLUSH) | EVENT_INDEX(4);
 238
 239         /* force to keep tiling flags */
 240         flags |= RADEON_FLUSH_KEEP_TILING_FLAGS;
 241
 242 #if R600_TRACE_CS
 243         if (ctx->screen->trace_bo) {
 244                 struct r600_screen *rscreen = ctx->screen;
 245                 unsigned i;
 246
 247                 for (i = 0; i < cs->cdw; i++) {
 248                         fprintf(stderr, "[%4d] [%5d] 0x%08x\n", rscreen->cs_count, i, cs->buf[i]);
 249                 }
 250                 rscreen->cs_count++;
 251         }
 252 #endif
 253
 254         /* Flush the CS. */
 255         ctx->ws->cs_flush(ctx->cs, flags, 0);
 256
 257 #if R600_TRACE_CS
 258         if (ctx->screen->trace_bo) {
 259                 struct r600_screen *rscreen = ctx->screen;
 260                 unsigned i;
 261
 262                 for (i = 0; i < 10; i++) {
 263                         usleep(5);
 264                         if (!ctx->ws->buffer_is_busy(rscreen->trace_bo->buf, RADEON_USAGE_READWRITE)) {
 265                                 break;
 266                         }
 267                 }
 268                 if (i == 10) {
 269                         fprintf(stderr, "timeout on cs lockup likely happen at cs %d dw %d\n",
 270                                 rscreen->trace_ptr[1], rscreen->trace_ptr[0]);
 271                 } else {
 272                         fprintf(stderr, "cs %d executed in %dms\n", rscreen->trace_ptr[1], i * 5);
 273                 }
 274         }
 275 #endif
 276
 277         ctx->pm4_dirty_cdwords = 0;
 278         ctx->flags = 0;
 279
 280         /* set all valid group as dirty so they get reemited on
 281          * next draw command
 282          */
 283         si_pm4_reset_emitted(ctx);
 284
 285         /* The CS initialization should be emitted before everything else. */
 286         si_pm4_emit(ctx, ctx->queued.named.init);
 287         ctx->emitted.named.init = ctx->queued.named.init;
 288
 289 #if 0
 290         if (streamout_suspended) {
 291                 ctx->streamout_start = TRUE;
 292                 ctx->streamout_append_bitmask = ~0;
 293         }
 294 #endif
 295
 296         /* resume queries */
 297         if (queries_suspended) {
 298                 r600_context_queries_resume(ctx);
 299         }
 300
 301         si_all_descriptors_begin_new_cs(ctx);
 302 }
 303
 304 void si_context_emit_fence(struct r600_context *ctx, struct si_resource *fence_bo, unsigned offset, unsigned value)
 305 {
 306         struct radeon_winsys_cs *cs = ctx->cs;
 307         uint64_t va;
 308
 309         si_need_cs_space(ctx, 10, FALSE);
 310
 311         va = r600_resource_va(&ctx->screen->screen, (void*)fence_bo);
 312         va = va + (offset << 2);
 313
 314         cs->buf[cs->cdw++] = PKT3(PKT3_EVENT_WRITE, 0, 0);
 315         cs->buf[cs->cdw++] = EVENT_TYPE(EVENT_TYPE_PS_PARTIAL_FLUSH) | EVENT_INDEX(4);
 316         cs->buf[cs->cdw++] = PKT3(PKT3_EVENT_WRITE_EOP, 4, 0);
 317         cs->buf[cs->cdw++] = EVENT_TYPE(EVENT_TYPE_CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5);
 318         cs->buf[cs->cdw++] = va & 0xFFFFFFFFUL;       /* ADDRESS_LO */
 319         /* DATA_SEL | INT_EN | ADDRESS_HI */
 320         cs->buf[cs->cdw++] = (1 << 29) | (0 << 24) | ((va >> 32UL) & 0xFF);
 321         cs->buf[cs->cdw++] = value;                   /* DATA_LO */
 322         cs->buf[cs->cdw++] = 0;                       /* DATA_HI */
 323         cs->buf[cs->cdw++] = PKT3(PKT3_NOP, 0, 0);
 324         cs->buf[cs->cdw++] = r600_context_bo_reloc(ctx, fence_bo, RADEON_USAGE_WRITE);
 325 }
 326
 327 static unsigned r600_query_read_result(char *map, unsigned start_index, unsigned end_index,
 328                                        bool test_status_bit)
 329 {
 330         uint32_t *current_result = (uint32_t*)map;
 331         uint64_t start, end;
 332
 333         start = (uint64_t)current_result[start_index] |
 334                 (uint64_t)current_result[start_index+1] << 32;
 335         end = (uint64_t)current_result[end_index] |
 336               (uint64_t)current_result[end_index+1] << 32;
 337
 338         if (!test_status_bit ||
 339             ((start & 0x8000000000000000UL) && (end & 0x8000000000000000UL))) {
 340                 return end - start;
 341         }
 342         return 0;
 343 }
 344
 345 static boolean r600_query_result(struct r600_context *ctx, struct r600_query *query, boolean wait)
 346 {
 347         unsigned results_base = query->results_start;
 348         char *map;
 349
 350         map = ctx->ws->buffer_map(query->buffer->cs_buf, ctx->cs,
 351                                   PIPE_TRANSFER_READ |
 352                                   (wait ? 0 : PIPE_TRANSFER_DONTBLOCK));
 353         if (!map)
 354                 return FALSE;
 355
 356         /* count all results across all data blocks */
 357         switch (query->type) {
 358         case PIPE_QUERY_OCCLUSION_COUNTER:
 359                 while (results_base != query->results_end) {
 360                         query->result.u64 +=
 361                                 r600_query_read_result(map + results_base, 0, 2, true);
 362                         results_base = (results_base + 16) % query->buffer->b.b.width0;
 363                 }
 364                 break;
 365         case PIPE_QUERY_OCCLUSION_PREDICATE:
 366                 while (results_base != query->results_end) {
 367                         query->result.b = query->result.b ||
 368                                 r600_query_read_result(map + results_base, 0, 2, true) != 0;
 369                         results_base = (results_base + 16) % query->buffer->b.b.width0;
 370                 }
 371                 break;
 372         case PIPE_QUERY_TIMESTAMP:
 373         {
 374                 uint32_t *current_result = (uint32_t*)map;
 375                 query->result.u64 = (uint64_t)current_result[0] | (uint64_t)current_result[1] << 32;
 376                 break;
 377         }
 378         case PIPE_QUERY_TIME_ELAPSED:
 379                 while (results_base != query->results_end) {
 380                         query->result.u64 +=
 381                                 r600_query_read_result(map + results_base, 0, 2, false);
 382                         results_base = (results_base + query->result_size) % query->buffer->b.b.width0;
 383                 }
 384                 break;
 385         case PIPE_QUERY_PRIMITIVES_EMITTED:
 386                 /* SAMPLE_STREAMOUTSTATS stores this structure:
 387                  * {
 388                  *    u64 NumPrimitivesWritten;
 389                  *    u64 PrimitiveStorageNeeded;
 390                  * }
 391                  * We only need NumPrimitivesWritten here. */
 392                 while (results_base != query->results_end) {
 393                         query->result.u64 +=
 394                                 r600_query_read_result(map + results_base, 2, 6, true);
 395                         results_base = (results_base + query->result_size) % query->buffer->b.b.width0;
 396                 }
 397                 break;
 398         case PIPE_QUERY_PRIMITIVES_GENERATED:
 399                 /* Here we read PrimitiveStorageNeeded. */
 400                 while (results_base != query->results_end) {
 401                         query->result.u64 +=
 402                                 r600_query_read_result(map + results_base, 0, 4, true);
 403                         results_base = (results_base + query->result_size) % query->buffer->b.b.width0;
 404                 }
 405                 break;
 406         case PIPE_QUERY_SO_STATISTICS:
 407                 while (results_base != query->results_end) {
 408                         query->result.so.num_primitives_written +=
 409                                 r600_query_read_result(map + results_base, 2, 6, true);
 410                         query->result.so.primitives_storage_needed +=
 411                                 r600_query_read_result(map + results_base, 0, 4, true);
 412                         results_base = (results_base + query->result_size) % query->buffer->b.b.width0;
 413                 }
 414                 break;
 415         case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
 416                 while (results_base != query->results_end) {
 417                         query->result.b = query->result.b ||
 418                                 r600_query_read_result(map + results_base, 2, 6, true) !=
 419                                 r600_query_read_result(map + results_base, 0, 4, true);
 420                         results_base = (results_base + query->result_size) % query->buffer->b.b.width0;
 421                 }
 422                 break;
 423         default:
 424                 assert(0);
 425         }
 426
 427         query->results_start = query->results_end;
 428         ctx->ws->buffer_unmap(query->buffer->cs_buf);
 429         return TRUE;
 430 }
 431
 432 void r600_query_begin(struct r600_context *ctx, struct r600_query *query)
 433 {
 434         struct radeon_winsys_cs *cs = ctx->cs;
 435         unsigned new_results_end, i;
 436         uint32_t *results;
 437         uint64_t va;
 438
 439         si_need_cs_space(ctx, query->num_cs_dw * 2, TRUE);
 440
 441         new_results_end = (query->results_end + query->result_size) % query->buffer->b.b.width0;
 442
 443         /* collect current results if query buffer is full */
 444         if (new_results_end == query->results_start) {
 445                 r600_query_result(ctx, query, TRUE);
 446         }
 447
 448         switch (query->type) {
 449         case PIPE_QUERY_OCCLUSION_COUNTER:
 450         case PIPE_QUERY_OCCLUSION_PREDICATE:
 451                 results = ctx->ws->buffer_map(query->buffer->cs_buf, ctx->cs, PIPE_TRANSFER_WRITE);
 452                 if (results) {
 453                         results = (uint32_t*)((char*)results + query->results_end);
 454                         memset(results, 0, query->result_size);
 455
 456                         /* Set top bits for unused backends */
 457                         for (i = 0; i < ctx->max_db; i++) {
 458                                 if (!(ctx->backend_mask & (1<<i))) {
 459                                         results[(i * 4)+1] = 0x80000000;
 460                                         results[(i * 4)+3] = 0x80000000;
 461                                 }
 462                         }
 463                         ctx->ws->buffer_unmap(query->buffer->cs_buf);
 464                 }
 465                 break;
 466         case PIPE_QUERY_TIME_ELAPSED:
 467                 break;
 468         case PIPE_QUERY_PRIMITIVES_EMITTED:
 469         case PIPE_QUERY_PRIMITIVES_GENERATED:
 470         case PIPE_QUERY_SO_STATISTICS:
 471         case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
 472                 results = ctx->ws->buffer_map(query->buffer->cs_buf, ctx->cs, PIPE_TRANSFER_WRITE);
 473                 results = (uint32_t*)((char*)results + query->results_end);
 474                 memset(results, 0, query->result_size);
 475                 ctx->ws->buffer_unmap(query->buffer->cs_buf);
 476                 break;
 477         default:
 478                 assert(0);
 479         }
 480
 481         /* emit begin query */
 482         va = r600_resource_va(&ctx->screen->screen, (void*)query->buffer);
 483         va += query->results_end;
 484
 485         switch (query->type) {
 486         case PIPE_QUERY_OCCLUSION_COUNTER:
 487         case PIPE_QUERY_OCCLUSION_PREDICATE:
 488                 cs->buf[cs->cdw++] = PKT3(PKT3_EVENT_WRITE, 2, 0);
 489                 cs->buf[cs->cdw++] = EVENT_TYPE(EVENT_TYPE_ZPASS_DONE) | EVENT_INDEX(1);
 490                 cs->buf[cs->cdw++] = va;
 491                 cs->buf[cs->cdw++] = (va >> 32UL) & 0xFF;
 492                 break;
 493         case PIPE_QUERY_PRIMITIVES_EMITTED:
 494         case PIPE_QUERY_PRIMITIVES_GENERATED:
 495         case PIPE_QUERY_SO_STATISTICS:
 496         case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
 497                 cs->buf[cs->cdw++] = PKT3(PKT3_EVENT_WRITE, 2, 0);
 498                 cs->buf[cs->cdw++] = EVENT_TYPE(EVENT_TYPE_SAMPLE_STREAMOUTSTATS) | EVENT_INDEX(3);
 499                 cs->buf[cs->cdw++] = query->results_end;
 500                 cs->buf[cs->cdw++] = 0;
 501                 break;
 502         case PIPE_QUERY_TIME_ELAPSED:
 503                 cs->buf[cs->cdw++] = PKT3(PKT3_EVENT_WRITE_EOP, 4, 0);
 504                 cs->buf[cs->cdw++] = EVENT_TYPE(EVENT_TYPE_CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5);
 505                 cs->buf[cs->cdw++] = va;
 506                 cs->buf[cs->cdw++] = (3 << 29) | ((va >> 32UL) & 0xFF);
 507                 cs->buf[cs->cdw++] = 0;
 508                 cs->buf[cs->cdw++] = 0;
 509                 break;
 510         default:
 511                 assert(0);
 512         }
 513         cs->buf[cs->cdw++] = PKT3(PKT3_NOP, 0, 0);
 514         cs->buf[cs->cdw++] = r600_context_bo_reloc(ctx, query->buffer, RADEON_USAGE_WRITE);
 515
 516         if (!si_is_timer_query(query->type)) {
 517                 ctx->num_cs_dw_nontimer_queries_suspend += query->num_cs_dw;
 518         }
 519 }
 520
 521 void r600_query_end(struct r600_context *ctx, struct r600_query *query)
 522 {
 523         struct radeon_winsys_cs *cs = ctx->cs;
 524         uint64_t va;
 525         unsigned new_results_end;
 526
 527         /* The queries which need begin already called this in begin_query. */
 528         if (!si_query_needs_begin(query->type)) {
 529                 si_need_cs_space(ctx, query->num_cs_dw, TRUE);
 530
 531                 new_results_end = (query->results_end + query->result_size) % query->buffer->b.b.width0;
 532
 533                 /* collect current results if query buffer is full */
 534                 if (new_results_end == query->results_start) {
 535                 r600_query_result(ctx, query, TRUE);
 536                 }
 537         }
 538
 539         va = r600_resource_va(&ctx->screen->screen, (void*)query->buffer);
 540         /* emit end query */
 541         switch (query->type) {
 542         case PIPE_QUERY_OCCLUSION_COUNTER:
 543         case PIPE_QUERY_OCCLUSION_PREDICATE:
 544                 va += query->results_end + 8;
 545                 cs->buf[cs->cdw++] = PKT3(PKT3_EVENT_WRITE, 2, 0);
 546                 cs->buf[cs->cdw++] = EVENT_TYPE(EVENT_TYPE_ZPASS_DONE) | EVENT_INDEX(1);
 547                 cs->buf[cs->cdw++] = va;
 548                 cs->buf[cs->cdw++] = (va >> 32UL) & 0xFF;
 549                 break;
 550         case PIPE_QUERY_PRIMITIVES_EMITTED:
 551         case PIPE_QUERY_PRIMITIVES_GENERATED:
 552         case PIPE_QUERY_SO_STATISTICS:
 553         case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
 554                 cs->buf[cs->cdw++] = PKT3(PKT3_EVENT_WRITE, 2, 0);
 555                 cs->buf[cs->cdw++] = EVENT_TYPE(EVENT_TYPE_SAMPLE_STREAMOUTSTATS) | EVENT_INDEX(3);
 556                 cs->buf[cs->cdw++] = query->results_end + query->result_size/2;
 557                 cs->buf[cs->cdw++] = 0;
 558                 break;
 559         case PIPE_QUERY_TIME_ELAPSED:
 560                 va += query->results_end + query->result_size/2;
 561                 /* fall through */
 562         case PIPE_QUERY_TIMESTAMP:
 563                 cs->buf[cs->cdw++] = PKT3(PKT3_EVENT_WRITE_EOP, 4, 0);
 564                 cs->buf[cs->cdw++] = EVENT_TYPE(EVENT_TYPE_CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5);
 565                 cs->buf[cs->cdw++] = va;
 566                 cs->buf[cs->cdw++] = (3 << 29) | ((va >> 32UL) & 0xFF);
 567                 cs->buf[cs->cdw++] = 0;
 568                 cs->buf[cs->cdw++] = 0;
 569                 break;
 570         default:
 571                 assert(0);
 572         }
 573         cs->buf[cs->cdw++] = PKT3(PKT3_NOP, 0, 0);
 574         cs->buf[cs->cdw++] = r600_context_bo_reloc(ctx, query->buffer, RADEON_USAGE_WRITE);
 575
 576         query->results_end = (query->results_end + query->result_size) % query->buffer->b.b.width0;
 577
 578         if (si_query_needs_begin(query->type) && !si_is_timer_query(query->type)) {
 579                 ctx->num_cs_dw_nontimer_queries_suspend -= query->num_cs_dw;
 580         }
 581 }
 582
 583 void r600_query_predication(struct r600_context *ctx, struct r600_query *query, int operation,
 584                             int flag_wait)
 585 {
 586         struct radeon_winsys_cs *cs = ctx->cs;
 587         uint64_t va;
 588
 589         if (operation == PREDICATION_OP_CLEAR) {
 590                 si_need_cs_space(ctx, 3, FALSE);
 591
 592                 cs->buf[cs->cdw++] = PKT3(PKT3_SET_PREDICATION, 1, 0);
 593                 cs->buf[cs->cdw++] = 0;
 594                 cs->buf[cs->cdw++] = PRED_OP(PREDICATION_OP_CLEAR);
 595         } else {
 596                 unsigned results_base = query->results_start;
 597                 unsigned count;
 598                 uint32_t op;
 599
 600                 /* find count of the query data blocks */
 601                 count = (query->buffer->b.b.width0 + query->results_end - query->results_start) % query->buffer->b.b.width0;
 602                 count /= query->result_size;
 603
 604                 si_need_cs_space(ctx, 5 * count, TRUE);
 605
 606                 op = PRED_OP(operation) | PREDICATION_DRAW_VISIBLE |
 607                                 (flag_wait ? PREDICATION_HINT_WAIT : PREDICATION_HINT_NOWAIT_DRAW);
 608                 va = r600_resource_va(&ctx->screen->screen, (void*)query->buffer);
 609
 610                 /* emit predicate packets for all data blocks */
 611                 while (results_base != query->results_end) {
 612                         cs->buf[cs->cdw++] = PKT3(PKT3_SET_PREDICATION, 1, 0);
 613                         cs->buf[cs->cdw++] = (va + results_base) & 0xFFFFFFFFUL;
 614                         cs->buf[cs->cdw++] = op | (((va + results_base) >> 32UL) & 0xFF);
 615                         cs->buf[cs->cdw++] = PKT3(PKT3_NOP, 0, 0);
 616                         cs->buf[cs->cdw++] = r600_context_bo_reloc(ctx, query->buffer,
 617                                                                              RADEON_USAGE_READ);
 618                         results_base = (results_base + query->result_size) % query->buffer->b.b.width0;
 619
 620                         /* set CONTINUE bit for all packets except the first */
 621                         op |= PREDICATION_CONTINUE;
 622                 }
 623         }
 624 }
 625
 626 struct r600_query *r600_context_query_create(struct r600_context *ctx, unsigned query_type)
 627 {
 628         struct r600_query *query;
 629         unsigned buffer_size = 4096;
 630
 631         query = CALLOC_STRUCT(r600_query);
 632         if (query == NULL)
 633                 return NULL;
 634
 635         query->type = query_type;
 636
 637         switch (query_type) {
 638         case PIPE_QUERY_OCCLUSION_COUNTER:
 639         case PIPE_QUERY_OCCLUSION_PREDICATE:
 640                 query->result_size = 16 * ctx->max_db;
 641                 query->num_cs_dw = 6;
 642                 break;
 643         case PIPE_QUERY_TIMESTAMP:
 644                 query->result_size = 8;
 645                 query->num_cs_dw = 8;
 646                 break;
 647         case PIPE_QUERY_TIME_ELAPSED:
 648                 query->result_size = 16;
 649                 query->num_cs_dw = 8;
 650                 break;
 651         case PIPE_QUERY_PRIMITIVES_EMITTED:
 652         case PIPE_QUERY_PRIMITIVES_GENERATED:
 653         case PIPE_QUERY_SO_STATISTICS:
 654         case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
 655                 /* NumPrimitivesWritten, PrimitiveStorageNeeded. */
 656                 query->result_size = 32;
 657                 query->num_cs_dw = 6;
 658                 break;
 659         default:
 660                 assert(0);
 661                 FREE(query);
 662                 return NULL;
 663         }
 664
 665         /* adjust buffer size to simplify offsets wrapping math */
 666         buffer_size -= buffer_size % query->result_size;
 667
 668         /* Queries are normally read by the CPU after
 669          * being written by the gpu, hence staging is probably a good
 670          * usage pattern.
 671          */
 672         query->buffer = si_resource_create_custom(&ctx->screen->screen,
 673                                                   PIPE_USAGE_STAGING,
 674                                                   buffer_size);
 675         if (!query->buffer) {
 676                 FREE(query);
 677                 return NULL;
 678         }
 679         return query;
 680 }
 681
 682 void r600_context_query_destroy(struct r600_context *ctx, struct r600_query *query)
 683 {
 684         si_resource_reference(&query->buffer, NULL);
 685         free(query);
 686 }
 687
 688 boolean r600_context_query_result(struct r600_context *ctx,
 689                                 struct r600_query *query,
 690                                 boolean wait, void *vresult)
 691 {
 692         boolean *result_b = (boolean*)vresult;
 693         uint64_t *result_u64 = (uint64_t*)vresult;
 694         struct pipe_query_data_so_statistics *result_so =
 695                 (struct pipe_query_data_so_statistics*)vresult;
 696
 697         if (!r600_query_result(ctx, query, wait))
 698                 return FALSE;
 699
 700         switch (query->type) {
 701         case PIPE_QUERY_OCCLUSION_COUNTER:
 702         case PIPE_QUERY_PRIMITIVES_EMITTED:
 703         case PIPE_QUERY_PRIMITIVES_GENERATED:
 704                 *result_u64 = query->result.u64;
 705                 break;
 706         case PIPE_QUERY_OCCLUSION_PREDICATE:
 707         case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
 708                 *result_b = query->result.b;
 709                 break;
 710         case PIPE_QUERY_TIMESTAMP:
 711         case PIPE_QUERY_TIME_ELAPSED:
 712                 *result_u64 = (1000000 * query->result.u64) / ctx->screen->info.r600_clock_crystal_freq;
 713                 break;
 714         case PIPE_QUERY_SO_STATISTICS:
 715                 *result_so = query->result.so;
 716                 break;
 717         default:
 718                 assert(0);
 719         }
 720         return TRUE;
 721 }
 722
 723 void r600_context_queries_suspend(struct r600_context *ctx)
 724 {
 725         struct r600_query *query;
 726
 727         LIST_FOR_EACH_ENTRY(query, &ctx->active_nontimer_query_list, list) {
 728                 r600_query_end(ctx, query);
 729         }
 730         assert(ctx->num_cs_dw_nontimer_queries_suspend == 0);
 731 }
 732
 733 void r600_context_queries_resume(struct r600_context *ctx)
 734 {
 735         struct r600_query *query;
 736
 737         assert(ctx->num_cs_dw_nontimer_queries_suspend == 0);
 738
 739         LIST_FOR_EACH_ENTRY(query, &ctx->active_nontimer_query_list, list) {
 740                 r600_query_begin(ctx, query);
 741         }
 742 }
 743
 744 void r600_context_draw_opaque_count(struct r600_context *ctx, struct r600_so_target *t)
 745 {
 746         struct radeon_winsys_cs *cs = ctx->cs;
 747         si_need_cs_space(ctx, 14 + 21, TRUE);
 748
 749         cs->buf[cs->cdw++] = PKT3(PKT3_SET_CONTEXT_REG, 1, 0);
 750         cs->buf[cs->cdw++] = (R_028B28_VGT_STRMOUT_DRAW_OPAQUE_OFFSET - SI_CONTEXT_REG_OFFSET) >> 2;
 751         cs->buf[cs->cdw++] = 0;
 752
 753         cs->buf[cs->cdw++] = PKT3(PKT3_SET_CONTEXT_REG, 1, 0);
 754         cs->buf[cs->cdw++] = (R_028B30_VGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE - SI_CONTEXT_REG_OFFSET) >> 2;
 755         cs->buf[cs->cdw++] = t->stride >> 2;
 756
 757 #if 0
 758         cs->buf[cs->cdw++] = PKT3(PKT3_COPY_DW, 4, 0);
 759         cs->buf[cs->cdw++] = COPY_DW_SRC_IS_MEM | COPY_DW_DST_IS_REG;
 760         cs->buf[cs->cdw++] = 0; /* src address lo */
 761         cs->buf[cs->cdw++] = 0; /* src address hi */
 762         cs->buf[cs->cdw++] = R_028B2C_VGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE >> 2; /* dst register */
 763         cs->buf[cs->cdw++] = 0; /* unused */
 764 #endif
 765
 766         cs->buf[cs->cdw++] = PKT3(PKT3_NOP, 0, 0);
 767         cs->buf[cs->cdw++] = r600_context_bo_reloc(ctx, t->filled_size, RADEON_USAGE_READ);
 768
 769 }
 770
 771 #if R600_TRACE_CS
 772 void r600_trace_emit(struct r600_context *rctx)
 773 {
 774         struct r600_screen *rscreen = rctx->screen;
 775         struct radeon_winsys_cs *cs = rctx->cs;
 776         uint64_t va;
 777
 778         va = r600_resource_va(&rscreen->screen, (void*)rscreen->trace_bo);
 779         r600_context_bo_reloc(rctx, rscreen->trace_bo, RADEON_USAGE_READWRITE);
 780         cs->buf[cs->cdw++] = PKT3(PKT3_WRITE_DATA, 4, 0);
 781         cs->buf[cs->cdw++] = PKT3_WRITE_DATA_DST_SEL(PKT3_WRITE_DATA_DST_SEL_MEM_SYNC) |
 782                                 PKT3_WRITE_DATA_WR_CONFIRM |
 783                                 PKT3_WRITE_DATA_ENGINE_SEL(PKT3_WRITE_DATA_ENGINE_SEL_ME);
 784         cs->buf[cs->cdw++] = va & 0xFFFFFFFFUL;
 785         cs->buf[cs->cdw++] = (va >> 32UL) & 0xFFFFFFFFUL;
 786         cs->buf[cs->cdw++] = cs->cdw;
 787         cs->buf[cs->cdw++] = rscreen->cs_count;
 788 }
 789 #endif