src/gallium/drivers/radeonsi/si_test_dma_perf.c

   1 /*
   2  * Copyright 2018 Advanced Micro Devices, Inc.
   3  * All Rights Reserved.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  22  * SOFTWARE.
  23  *
  24  */
  25
  26 /* This file implements tests on the si_clearbuffer function. */
  27
  28 #include "si_pipe.h"
  29 #include "si_query.h"
  30
  31 #define MIN_SIZE   512
  32 #define MAX_SIZE   (128 * 1024 * 1024)
  33 #define SIZE_SHIFT 1
  34 #define NUM_RUNS   128
  35
  36 static double get_MBps_rate(unsigned num_bytes, unsigned ns)
  37 {
  38    return (num_bytes / (1024.0 * 1024.0)) / (ns / 1000000000.0);
  39 }
  40
  41 void si_test_dma_perf(struct si_screen *sscreen)
  42 {
  43    struct pipe_screen *screen = &sscreen->b;
  44    struct pipe_context *ctx = screen->context_create(screen, NULL, 0);
  45    struct si_context *sctx = (struct si_context *)ctx;
  46    const uint32_t clear_value = 0x12345678;
  47    static const unsigned cs_dwords_per_thread_list[] = {64, 32, 16, 8, 4, 2, 1};
  48    static const unsigned cs_waves_per_sh_list[] = {1, 2, 4, 8, 16, 0};
  49
  50 #define NUM_SHADERS ARRAY_SIZE(cs_dwords_per_thread_list)
  51 #define NUM_METHODS (4 + 2 * NUM_SHADERS * ARRAY_SIZE(cs_waves_per_sh_list))
  52
  53    static const char *method_str[] = {
  54       "CP MC   ",
  55       "CP L2   ",
  56       "CP L2   ",
  57       "SDMA    ",
  58    };
  59    static const char *placement_str[] = {
  60       /* Clear */
  61       "fill->VRAM",
  62       "fill->GTT ",
  63       /* Copy */
  64       "VRAM->VRAM",
  65       "VRAM->GTT ",
  66       "GTT ->VRAM",
  67    };
  68
  69    printf("DMA rate is in MB/s for each size. Slow cases are skipped and print 0.\n");
  70    printf("Heap       ,Method  ,L2p,Wa,");
  71    for (unsigned size = MIN_SIZE; size <= MAX_SIZE; size <<= SIZE_SHIFT) {
  72       if (size >= 1024)
  73          printf("%6uKB,", size / 1024);
  74       else
  75          printf(" %6uB,", size);
  76    }
  77    printf("\n");
  78
  79    /* results[log2(size)][placement][method][] */
  80    struct si_result {
  81       bool is_valid;
  82       bool is_cp;
  83       bool is_sdma;
  84       bool is_cs;
  85       unsigned cache_policy;
  86       unsigned dwords_per_thread;
  87       unsigned waves_per_sh;
  88       unsigned score;
  89       unsigned index; /* index in results[x][y][index] */
  90    } results[32][ARRAY_SIZE(placement_str)][NUM_METHODS] = {};
  91
  92    /* Run benchmarks. */
  93    for (unsigned placement = 0; placement < ARRAY_SIZE(placement_str); placement++) {
  94       bool is_copy = placement >= 2;
  95
  96       printf("-----------,--------,---,--,");
  97       for (unsigned size = MIN_SIZE; size <= MAX_SIZE; size <<= SIZE_SHIFT)
  98          printf("--------,");
  99       printf("\n");
 100
 101       for (unsigned method = 0; method < NUM_METHODS; method++) {
 102          bool test_cp = method <= 2;
 103          bool test_sdma = method == 3;
 104          bool test_cs = method >= 4;
 105          unsigned cs_method = method - 4;
 106          STATIC_ASSERT(L2_STREAM + 1 == L2_LRU);
 107          unsigned cs_waves_per_sh =
 108             test_cs ? cs_waves_per_sh_list[cs_method / (2 * NUM_SHADERS)] : 0;
 109          cs_method %= 2 * NUM_SHADERS;
 110          unsigned cache_policy =
 111             test_cp ? method % 3 : test_cs ? L2_STREAM + (cs_method / NUM_SHADERS) : 0;
 112          unsigned cs_dwords_per_thread =
 113             test_cs ? cs_dwords_per_thread_list[cs_method % NUM_SHADERS] : 0;
 114
 115          if (test_sdma && !sctx->sdma_cs)
 116             continue;
 117
 118          if (sctx->chip_class == GFX6) {
 119             /* GFX6 doesn't support CP DMA operations through L2. */
 120             if (test_cp && cache_policy != L2_BYPASS)
 121                continue;
 122             /* WAVES_PER_SH is in multiples of 16 on GFX6. */
 123             if (test_cs && cs_waves_per_sh % 16 != 0)
 124                continue;
 125          }
 126
 127          printf("%s ,", placement_str[placement]);
 128          if (test_cs) {
 129             printf("CS x%-4u,%3s,", cs_dwords_per_thread,
 130                    cache_policy == L2_LRU ? "LRU" : cache_policy == L2_STREAM ? "Str" : "");
 131          } else {
 132             printf("%s,%3s,", method_str[method],
 133                    method == L2_LRU ? "LRU" : method == L2_STREAM ? "Str" : "");
 134          }
 135          if (test_cs && cs_waves_per_sh)
 136             printf("%2u,", cs_waves_per_sh);
 137          else
 138             printf("  ,");
 139
 140          double score = 0;
 141          for (unsigned size = MIN_SIZE; size <= MAX_SIZE; size <<= SIZE_SHIFT) {
 142             /* Don't test bigger sizes if it's too slow. Print 0. */
 143             if (size >= 512 * 1024 && score < 400 * (size / (4 * 1024 * 1024))) {
 144                printf("%7.0f ,", 0.0);
 145                continue;
 146             }
 147
 148             enum pipe_resource_usage dst_usage, src_usage;
 149             struct pipe_resource *dst, *src;
 150             struct pipe_query *q[NUM_RUNS];
 151             unsigned query_type = PIPE_QUERY_TIME_ELAPSED;
 152
 153             if (test_sdma) {
 154                if (sctx->chip_class == GFX6)
 155                   query_type = SI_QUERY_TIME_ELAPSED_SDMA_SI;
 156                else
 157                   query_type = SI_QUERY_TIME_ELAPSED_SDMA;
 158             }
 159
 160             if (placement == 0 || placement == 2 || placement == 4)
 161                dst_usage = PIPE_USAGE_DEFAULT;
 162             else
 163                dst_usage = PIPE_USAGE_STREAM;
 164
 165             if (placement == 2 || placement == 3)
 166                src_usage = PIPE_USAGE_DEFAULT;
 167             else
 168                src_usage = PIPE_USAGE_STREAM;
 169
 170             dst = pipe_buffer_create(screen, 0, dst_usage, size);
 171             src = is_copy ? pipe_buffer_create(screen, 0, src_usage, size) : NULL;
 172
 173             /* Run tests. */
 174             for (unsigned iter = 0; iter < NUM_RUNS; iter++) {
 175                q[iter] = ctx->create_query(ctx, query_type, 0);
 176                ctx->begin_query(ctx, q[iter]);
 177
 178                if (test_cp) {
 179                   /* CP DMA */
 180                   if (is_copy) {
 181                      si_cp_dma_copy_buffer(sctx, dst, src, 0, 0, size, 0, SI_COHERENCY_NONE,
 182                                            cache_policy);
 183                   } else {
 184                      si_cp_dma_clear_buffer(sctx, sctx->gfx_cs, dst, 0, size, clear_value, 0,
 185                                             SI_COHERENCY_NONE, cache_policy);
 186                   }
 187                } else if (test_sdma) {
 188                   /* SDMA */
 189                   if (is_copy) {
 190                      si_sdma_copy_buffer(sctx, dst, src, 0, 0, size);
 191                   } else {
 192                      si_sdma_clear_buffer(sctx, dst, 0, size, clear_value);
 193                   }
 194                } else {
 195                   /* Compute */
 196                   /* The memory accesses are coalesced, meaning that the 1st instruction writes
 197                    * the 1st contiguous block of data for the whole wave, the 2nd instruction
 198                    * writes the 2nd contiguous block of data, etc.
 199                    */
 200                   unsigned instructions_per_thread = MAX2(1, cs_dwords_per_thread / 4);
 201                   unsigned dwords_per_instruction = cs_dwords_per_thread / instructions_per_thread;
 202                   unsigned dwords_per_wave = cs_dwords_per_thread * 64;
 203
 204                   unsigned num_dwords = size / 4;
 205                   unsigned num_instructions = DIV_ROUND_UP(num_dwords, dwords_per_instruction);
 206
 207                   void *cs = si_create_dma_compute_shader(ctx, cs_dwords_per_thread,
 208                                                           cache_policy == L2_STREAM, is_copy);
 209
 210                   struct pipe_grid_info info = {};
 211                   info.block[0] = MIN2(64, num_instructions);
 212                   info.block[1] = 1;
 213                   info.block[2] = 1;
 214                   info.grid[0] = DIV_ROUND_UP(num_dwords, dwords_per_wave);
 215                   info.grid[1] = 1;
 216                   info.grid[2] = 1;
 217
 218                   struct pipe_shader_buffer sb[2] = {};
 219                   sb[0].buffer = dst;
 220                   sb[0].buffer_size = size;
 221
 222                   if (is_copy) {
 223                      sb[1].buffer = src;
 224                      sb[1].buffer_size = size;
 225                   } else {
 226                      for (unsigned i = 0; i < 4; i++)
 227                         sctx->cs_user_data[i] = clear_value;
 228                   }
 229
 230                   sctx->flags |= SI_CONTEXT_INV_VCACHE | SI_CONTEXT_INV_SCACHE;
 231
 232                   ctx->set_shader_buffers(ctx, PIPE_SHADER_COMPUTE, 0, is_copy ? 2 : 1, sb, 0x1);
 233                   ctx->bind_compute_state(ctx, cs);
 234                   sctx->cs_max_waves_per_sh = cs_waves_per_sh;
 235
 236                   ctx->launch_grid(ctx, &info);
 237
 238                   ctx->bind_compute_state(ctx, NULL);
 239                   ctx->delete_compute_state(ctx, cs);
 240                   sctx->cs_max_waves_per_sh = 0; /* disable the limit */
 241
 242                   sctx->flags |= SI_CONTEXT_CS_PARTIAL_FLUSH;
 243                }
 244
 245                /* Flush L2, so that we don't just test L2 cache performance. */
 246                if (!test_sdma) {
 247                   sctx->flags |= SI_CONTEXT_WB_L2;
 248                   sctx->emit_cache_flush(sctx);
 249                }
 250
 251                ctx->end_query(ctx, q[iter]);
 252                ctx->flush(ctx, NULL, PIPE_FLUSH_ASYNC);
 253             }
 254             pipe_resource_reference(&dst, NULL);
 255             pipe_resource_reference(&src, NULL);
 256
 257             /* Get results. */
 258             uint64_t min = ~0ull, max = 0, total = 0;
 259
 260             for (unsigned iter = 0; iter < NUM_RUNS; iter++) {
 261                union pipe_query_result result;
 262
 263                ctx->get_query_result(ctx, q[iter], true, &result);
 264                ctx->destroy_query(ctx, q[iter]);
 265
 266                min = MIN2(min, result.u64);
 267                max = MAX2(max, result.u64);
 268                total += result.u64;
 269             }
 270
 271             score = get_MBps_rate(size, total / (double)NUM_RUNS);
 272             printf("%7.0f ,", score);
 273             fflush(stdout);
 274
 275             struct si_result *r = &results[util_logbase2(size)][placement][method];
 276             r->is_valid = true;
 277             r->is_cp = test_cp;
 278             r->is_sdma = test_sdma;
 279             r->is_cs = test_cs;
 280             r->cache_policy = cache_policy;
 281             r->dwords_per_thread = cs_dwords_per_thread;
 282             r->waves_per_sh = cs_waves_per_sh;
 283             r->score = score;
 284             r->index = method;
 285          }
 286          puts("");
 287       }
 288    }
 289
 290    puts("");
 291    puts("static struct si_method");
 292    printf("get_best_clear_for_%s(enum radeon_bo_domain dst, uint64_t size64, bool async, bool "
 293           "cached)\n",
 294           sctx->screen->info.name);
 295    puts("{");
 296    puts("   unsigned size = MIN2(size64, UINT_MAX);\n");
 297
 298    /* Analyze results and find the best methods. */
 299    for (unsigned placement = 0; placement < ARRAY_SIZE(placement_str); placement++) {
 300       if (placement == 0)
 301          puts("   if (dst == RADEON_DOMAIN_VRAM) {");
 302       else if (placement == 1)
 303          puts("   } else { /* GTT */");
 304       else if (placement == 2) {
 305          puts("}");
 306          puts("");
 307          puts("static struct si_method");
 308          printf("get_best_copy_for_%s(enum radeon_bo_domain dst, enum radeon_bo_domain src,\n",
 309                 sctx->screen->info.name);
 310          printf("                     uint64_t size64, bool async, bool cached)\n");
 311          puts("{");
 312          puts("   unsigned size = MIN2(size64, UINT_MAX);\n");
 313          puts("   if (src == RADEON_DOMAIN_VRAM && dst == RADEON_DOMAIN_VRAM) {");
 314       } else if (placement == 3)
 315          puts("   } else if (src == RADEON_DOMAIN_VRAM && dst == RADEON_DOMAIN_GTT) {");
 316       else
 317          puts("   } else { /* GTT -> VRAM */");
 318
 319       for (unsigned mode = 0; mode < 3; mode++) {
 320          bool async = mode == 0;
 321          bool cached = mode == 1;
 322
 323          if (async)
 324             puts("      if (async) { /* SDMA or async compute */");
 325          else if (cached)
 326             puts("      if (cached) { /* gfx ring */");
 327          else
 328             puts("      } else { /* gfx ring - uncached */");
 329
 330          /* The list of best chosen methods. */
 331          struct si_result *methods[32];
 332          unsigned method_max_size[32];
 333          unsigned num_methods = 0;
 334
 335          for (unsigned size = MIN_SIZE; size <= MAX_SIZE; size <<= SIZE_SHIFT) {
 336             /* Find the best method. */
 337             struct si_result *best = NULL;
 338
 339             for (unsigned i = 0; i < NUM_METHODS; i++) {
 340                struct si_result *r = &results[util_logbase2(size)][placement][i];
 341
 342                if (!r->is_valid)
 343                   continue;
 344
 345                /* Ban CP DMA clears via MC on <= GFX8. They are super slow
 346                 * on GTT, which we can get due to BO evictions.
 347                 */
 348                if (sctx->chip_class <= GFX8 && placement == 1 && r->is_cp &&
 349                    r->cache_policy == L2_BYPASS)
 350                   continue;
 351
 352                if (async) {
 353                   /* The following constraints for compute IBs try to limit
 354                    * resource usage so as not to decrease the performance
 355                    * of gfx IBs too much.
 356                    */
 357
 358                   /* Don't use CP DMA on asynchronous rings, because
 359                    * the engine is shared with gfx IBs.
 360                    */
 361                   if (r->is_cp)
 362                      continue;
 363
 364                   /* Don't use L2 caching on asynchronous rings to minimize
 365                    * L2 usage.
 366                    */
 367                   if (r->cache_policy == L2_LRU)
 368                      continue;
 369
 370                   /* Asynchronous compute recommends waves_per_sh != 0
 371                    * to limit CU usage. */
 372                   if (r->is_cs && r->waves_per_sh == 0)
 373                      continue;
 374                } else {
 375                   /* SDMA is always asynchronous */
 376                   if (r->is_sdma)
 377                      continue;
 378
 379                   if (cached && r->cache_policy == L2_BYPASS)
 380                      continue;
 381                   if (!cached && r->cache_policy == L2_LRU)
 382                      continue;
 383                }
 384
 385                if (!best) {
 386                   best = r;
 387                   continue;
 388                }
 389
 390                /* Assume some measurement error. Earlier methods occupy fewer
 391                 * resources, so the next method is always more greedy, and we
 392                 * don't want to select it due to a measurement error.
 393                 */
 394                double min_improvement = 1.03;
 395
 396                if (best->score * min_improvement < r->score)
 397                   best = r;
 398             }
 399
 400             if (num_methods > 0) {
 401                unsigned prev_index = num_methods - 1;
 402                struct si_result *prev = methods[prev_index];
 403                struct si_result *prev_this_size =
 404                   &results[util_logbase2(size)][placement][prev->index];
 405
 406                /* If the best one is also the best for the previous size,
 407                 * just bump the size for the previous one.
 408                 *
 409                 * If there is no best, it means all methods were too slow
 410                 * for this size and were not tested. Use the best one for
 411                 * the previous size.
 412                 */
 413                if (!best ||
 414                    /* If it's the same method as for the previous size: */
 415                    (prev->is_cp == best->is_cp && prev->is_sdma == best->is_sdma &&
 416                     prev->is_cs == best->is_cs && prev->cache_policy == best->cache_policy &&
 417                     prev->dwords_per_thread == best->dwords_per_thread &&
 418                     prev->waves_per_sh == best->waves_per_sh) ||
 419                    /* If the method for the previous size is also the best
 420                     * for this size: */
 421                    (prev_this_size->is_valid && prev_this_size->score * 1.03 > best->score)) {
 422                   method_max_size[prev_index] = size;
 423                   continue;
 424                }
 425             }
 426
 427             /* Add it to the list. */
 428             assert(num_methods < ARRAY_SIZE(methods));
 429             methods[num_methods] = best;
 430             method_max_size[num_methods] = size;
 431             num_methods++;
 432          }
 433
 434          for (unsigned i = 0; i < num_methods; i++) {
 435             struct si_result *best = methods[i];
 436             unsigned size = method_max_size[i];
 437
 438             /* The size threshold is between the current benchmarked
 439              * size and the next benchmarked size. */
 440             if (i < num_methods - 1)
 441                printf("         if (size <= %9u) ", (size + (size << SIZE_SHIFT)) / 2);
 442             else if (i > 0)
 443                printf("         else                   ");
 444             else
 445                printf("         ");
 446             printf("return ");
 447
 448             assert(best);
 449             if (best->is_cp) {
 450                printf("CP_DMA(%s);\n",
 451                       best->cache_policy == L2_BYPASS
 452                          ? "L2_BYPASS"
 453                          : best->cache_policy == L2_LRU ? "L2_LRU   " : "L2_STREAM");
 454             }
 455             if (best->is_sdma)
 456                printf("SDMA;\n");
 457             if (best->is_cs) {
 458                printf("COMPUTE(%s, %u, %u);\n",
 459                       best->cache_policy == L2_LRU ? "L2_LRU   " : "L2_STREAM",
 460                       best->dwords_per_thread, best->waves_per_sh);
 461             }
 462          }
 463       }
 464       puts("      }");
 465    }
 466    puts("   }");
 467    puts("}");
 468
 469    ctx->destroy(ctx);
 470    exit(0);
 471 }