return ureg_create_shader_and_destroy(ureg, &sctx->b);
}
+/* Create a compute shader implementing clear_buffer or copy_buffer. */
+void *si_create_dma_compute_shader(struct pipe_context *ctx,
+ unsigned num_dwords_per_thread,
+ bool dst_stream_cache_policy, bool is_copy)
+{
+ assert(util_is_power_of_two_nonzero(num_dwords_per_thread));
+
+ unsigned store_qualifier = TGSI_MEMORY_COHERENT | TGSI_MEMORY_RESTRICT;
+ if (dst_stream_cache_policy)
+ store_qualifier |= TGSI_MEMORY_STREAM_CACHE_POLICY;
+
+ /* Don't cache loads, because there is no reuse. */
+ unsigned load_qualifier = store_qualifier | TGSI_MEMORY_STREAM_CACHE_POLICY;
+
+ unsigned num_mem_ops = MAX2(1, num_dwords_per_thread / 4);
+ unsigned *inst_dwords = alloca(num_mem_ops * sizeof(unsigned));
+
+ for (unsigned i = 0; i < num_mem_ops; i++) {
+ if (i*4 < num_dwords_per_thread)
+ inst_dwords[i] = MIN2(4, num_dwords_per_thread - i*4);
+ }
+
+ struct ureg_program *ureg = ureg_create(PIPE_SHADER_COMPUTE);
+ if (!ureg)
+ return NULL;
+
+ ureg_property(ureg, TGSI_PROPERTY_CS_FIXED_BLOCK_WIDTH, 64);
+ ureg_property(ureg, TGSI_PROPERTY_CS_FIXED_BLOCK_HEIGHT, 1);
+ ureg_property(ureg, TGSI_PROPERTY_CS_FIXED_BLOCK_DEPTH, 1);
+
+ struct ureg_src value;
+ if (!is_copy) {
+ ureg_property(ureg, TGSI_PROPERTY_CS_USER_DATA_DWORDS, inst_dwords[0]);
+ value = ureg_DECL_system_value(ureg, TGSI_SEMANTIC_CS_USER_DATA, 0);
+ }
+
+ struct ureg_src tid = ureg_DECL_system_value(ureg, TGSI_SEMANTIC_THREAD_ID, 0);
+ struct ureg_src blk = ureg_DECL_system_value(ureg, TGSI_SEMANTIC_BLOCK_ID, 0);
+ struct ureg_dst store_addr = ureg_writemask(ureg_DECL_temporary(ureg), TGSI_WRITEMASK_X);
+ struct ureg_dst load_addr = ureg_writemask(ureg_DECL_temporary(ureg), TGSI_WRITEMASK_X);
+ struct ureg_dst dstbuf = ureg_dst(ureg_DECL_buffer(ureg, 0, false));
+ struct ureg_src srcbuf;
+ struct ureg_src *values = NULL;
+
+ if (is_copy) {
+ srcbuf = ureg_DECL_buffer(ureg, 1, false);
+ values = malloc(num_mem_ops * sizeof(struct ureg_src));
+ }
+
+ /* If there are multiple stores, the first store writes into 0+tid,
+ * the 2nd store writes into 64+tid, the 3rd store writes into 128+tid, etc.
+ */
+ ureg_UMAD(ureg, store_addr, blk, ureg_imm1u(ureg, 64 * num_mem_ops), tid);
+ /* Convert from a "store size unit" into bytes. */
+ ureg_UMUL(ureg, store_addr, ureg_src(store_addr),
+ ureg_imm1u(ureg, 4 * inst_dwords[0]));
+ ureg_MOV(ureg, load_addr, ureg_src(store_addr));
+
+ /* Distance between a load and a store for latency hiding. */
+ unsigned load_store_distance = is_copy ? 8 : 0;
+
+ for (unsigned i = 0; i < num_mem_ops + load_store_distance; i++) {
+ int d = i - load_store_distance;
+
+ if (is_copy && i < num_mem_ops) {
+ if (i) {
+ ureg_UADD(ureg, load_addr, ureg_src(load_addr),
+ ureg_imm1u(ureg, 4 * inst_dwords[i] * 64));
+ }
+
+ values[i] = ureg_src(ureg_DECL_temporary(ureg));
+ struct ureg_dst dst =
+ ureg_writemask(ureg_dst(values[i]),
+ u_bit_consecutive(0, inst_dwords[i]));
+ struct ureg_src srcs[] = {srcbuf, ureg_src(load_addr)};
+ ureg_memory_insn(ureg, TGSI_OPCODE_LOAD, &dst, 1, srcs, 2,
+ load_qualifier, TGSI_TEXTURE_BUFFER, 0);
+ }
+
+ if (d >= 0) {
+ if (d) {
+ ureg_UADD(ureg, store_addr, ureg_src(store_addr),
+ ureg_imm1u(ureg, 4 * inst_dwords[d] * 64));
+ }
+
+ struct ureg_dst dst =
+ ureg_writemask(dstbuf, u_bit_consecutive(0, inst_dwords[d]));
+ struct ureg_src srcs[] =
+ {ureg_src(store_addr), is_copy ? values[d] : value};
+ ureg_memory_insn(ureg, TGSI_OPCODE_STORE, &dst, 1, srcs, 2,
+ store_qualifier, TGSI_TEXTURE_BUFFER, 0);
+ }
+ }
+ ureg_END(ureg);
+
+ struct pipe_compute_state state = {};
+ state.ir_type = PIPE_SHADER_IR_TGSI;
+ state.prog = ureg_get_tokens(ureg, NULL);
+
+ void *cs = ctx->create_compute_state(ctx, &state);
+ ureg_destroy(ureg);
+ free(values);
+ return cs;
+}
+
/* Create the compute shader that is used to collect the results.
*
* One compute grid with a single thread is launched for every query result
+++ /dev/null
-/*
- * Copyright 2018 Advanced Micro Devices, Inc.
- * All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- *
- */
-
-/* This file implements tests on the si_clearbuffer function. */
-
-#include "si_pipe.h"
-
-#define CLEARBUF_MIN 32
-#define CLEARBUF_COUNT 16
-#define CLEARBUF_MEMSZ 1024
-
-static uint64_t
-measure_clearbuf_time(struct pipe_context *ctx,
- uint64_t memory_size)
-{
- struct pipe_query *query_te;
- union pipe_query_result qresult;
- struct pipe_resource *buf;
-
- struct si_context *sctx = (struct si_context*)ctx;
- struct pipe_screen *screen = ctx->screen;
-
- buf = pipe_buffer_create(screen, 0, PIPE_USAGE_DEFAULT, memory_size);
-
- query_te = ctx->create_query(ctx, PIPE_QUERY_TIME_ELAPSED, 0);
-
- ctx->begin_query(ctx, query_te);
- /* operation */
- si_cp_dma_clear_buffer(sctx, buf, 0, memory_size, 0x00,
- SI_COHERENCY_SHADER, L2_LRU);
- ctx->end_query(ctx, query_te);
- ctx->get_query_result(ctx, query_te, true, &qresult);
-
- /* Cleanup. */
- ctx->destroy_query(ctx, query_te);
- pipe_resource_reference(&buf, NULL);
-
- /* Report Results */
- return qresult.u64;
-}
-
-/**
- * @brief Analyze rate of clearing a 1K Buffer averaged over 16 iterations
- * @param ctx Context of pipe to perform analysis on
- */
-static void
-analyze_clearbuf_perf_avg(struct pipe_context *ctx)
-{
- uint index = 0;
- uint64_t result[CLEARBUF_COUNT];
- uint64_t sum = 0;
- long long int rate_kBps;
-
- /* Run Tests. */
- for (index = 0 ; index < CLEARBUF_COUNT ; index++) {
- result[index] = measure_clearbuf_time(ctx, CLEARBUF_MEMSZ);
- sum += result[index];
- }
-
- /* Calculate Results. */
- /* kBps = (size(bytes))/(1000) / (time(ns)/(1000*1000*1000)) */
- rate_kBps = CLEARBUF_COUNT*CLEARBUF_MEMSZ;
- rate_kBps *= 1000UL*1000UL;
- rate_kBps /= sum;
-
- /* Display Results. */
- printf("CP DMA clear_buffer performance (buffer %lu ,repeat %u ):",
- (uint64_t)CLEARBUF_MEMSZ,
- CLEARBUF_COUNT );
- printf(" %llu kB/s\n", rate_kBps );
-}
-
-/**
- * @brief Analyze rate of clearing a range of Buffer sizes
- * @param ctx Context of pipe to perform analysis on
- */
-static void
-analyze_clearbuf_perf_rng(struct pipe_context *ctx)
-{
- uint index = 0;
- uint64_t result[CLEARBUF_COUNT];
- uint64_t mem_size;
- long long int rate_kBps;
-
- /* Run Tests. */
- mem_size = CLEARBUF_MIN;
- for (index = 0 ; index < CLEARBUF_COUNT ; index++ ) {
- result[index] = measure_clearbuf_time(ctx, mem_size);
- mem_size <<= 1;
- }
-
- /* Calculate & Display Results. */
- /* kBps = (size(bytes))/(1000) / (time(ns)/(1000*1000*1000)) */
- mem_size = CLEARBUF_MIN;
- for (index = 0 ; index < CLEARBUF_COUNT ; index++ ) {
- rate_kBps = mem_size;
- rate_kBps *= 1000UL*1000UL;
- rate_kBps /= result[index];
-
- printf("CP DMA clear_buffer performance (buffer %lu):",
- mem_size );
- printf(" %llu kB/s\n", rate_kBps );
-
- mem_size <<= 1;
- }
-}
-
-void si_test_clearbuffer_perf(struct si_screen *sscreen)
-{
- struct pipe_screen *screen = &sscreen->b;
- struct pipe_context *ctx = screen->context_create(screen, NULL, 0);
-
- analyze_clearbuf_perf_avg(ctx);
- analyze_clearbuf_perf_rng(ctx);
-
- exit(0);
-}
--- /dev/null
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+/* This file implements tests on the si_clearbuffer function. */
+
+#include "si_pipe.h"
+#include "si_query.h"
+
+#define MIN_SIZE 512
+#define MAX_SIZE (128 * 1024 * 1024)
+#define SIZE_SHIFT 1
+#define NUM_RUNS 128
+
+static double get_MBps_rate(unsigned num_bytes, unsigned ns)
+{
+ return (num_bytes / (1024.0 * 1024.0)) / (ns / 1000000000.0);
+}
+
+void si_test_dma_perf(struct si_screen *sscreen)
+{
+ struct pipe_screen *screen = &sscreen->b;
+ struct pipe_context *ctx = screen->context_create(screen, NULL, 0);
+ struct si_context *sctx = (struct si_context*)ctx;
+ const uint32_t clear_value = 0x12345678;
+ static const unsigned cs_dwords_per_thread_list[] = {64, 32, 16, 8, 4, 2, 1};
+ static const unsigned cs_waves_per_sh_list[] = {1, 2, 4, 8, 16, 0};
+
+#define NUM_SHADERS ARRAY_SIZE(cs_dwords_per_thread_list)
+#define NUM_METHODS (4 + 2*NUM_SHADERS * ARRAY_SIZE(cs_waves_per_sh_list))
+
+ static const char *method_str[] = {
+ "CP MC ",
+ "CP L2 ",
+ "CP L2 ",
+ "SDMA ",
+ };
+ static const char *placement_str[] = {
+ /* Clear */
+ "fill->VRAM",
+ "fill->GTT ",
+ /* Copy */
+ "VRAM->VRAM",
+ "VRAM->GTT ",
+ "GTT ->VRAM",
+ };
+
+ printf("DMA rate is in MB/s for each size. Slow cases are skipped and print 0.\n");
+ printf("Heap ,Method ,L2p,Wa,");
+ for (unsigned size = MIN_SIZE; size <= MAX_SIZE; size <<= SIZE_SHIFT) {
+ if (size >= 1024)
+ printf("%6uKB,", size / 1024);
+ else
+ printf(" %6uB,", size);
+ }
+ printf("\n");
+
+ /* results[log2(size)][placement][method][] */
+ struct si_result {
+ bool is_valid;
+ bool is_cp;
+ bool is_sdma;
+ bool is_cs;
+ unsigned cache_policy;
+ unsigned dwords_per_thread;
+ unsigned waves_per_sh;
+ unsigned score;
+ unsigned index; /* index in results[x][y][index] */
+ } results[32][ARRAY_SIZE(placement_str)][NUM_METHODS] = {};
+
+ /* Run benchmarks. */
+ for (unsigned placement = 0; placement < ARRAY_SIZE(placement_str); placement++) {
+ bool is_copy = placement >= 2;
+
+ printf("-----------,--------,---,--,");
+ for (unsigned size = MIN_SIZE; size <= MAX_SIZE; size <<= SIZE_SHIFT)
+ printf("--------,");
+ printf("\n");
+
+ for (unsigned method = 0; method < NUM_METHODS; method++) {
+ bool test_cp = method <= 2;
+ bool test_sdma = method == 3;
+ bool test_cs = method >= 4;
+ unsigned cs_method = method - 4;
+ STATIC_ASSERT(L2_STREAM + 1 == L2_LRU);
+ unsigned cs_waves_per_sh =
+ test_cs ? cs_waves_per_sh_list[cs_method / (2*NUM_SHADERS)] : 0;
+ cs_method %= 2*NUM_SHADERS;
+ unsigned cache_policy = test_cp ? method % 3 :
+ test_cs ? L2_STREAM + (cs_method / NUM_SHADERS) : 0;
+ unsigned cs_dwords_per_thread =
+ test_cs ? cs_dwords_per_thread_list[cs_method % NUM_SHADERS] : 0;
+
+ if (sctx->chip_class == SI) {
+ /* SI doesn't support CP DMA operations through L2. */
+ if (test_cp && cache_policy != L2_BYPASS)
+ continue;
+ /* WAVES_PER_SH is in multiples of 16 on SI. */
+ if (test_cs && cs_waves_per_sh % 16 != 0)
+ continue;
+ }
+
+ printf("%s ,", placement_str[placement]);
+ if (test_cs) {
+ printf("CS x%-4u,%3s,", cs_dwords_per_thread,
+ cache_policy == L2_LRU ? "LRU" :
+ cache_policy == L2_STREAM ? "Str" : "");
+ } else {
+ printf("%s,%3s,", method_str[method],
+ method == L2_LRU ? "LRU" :
+ method == L2_STREAM ? "Str" : "");
+ }
+ if (test_cs && cs_waves_per_sh)
+ printf("%2u,", cs_waves_per_sh);
+ else
+ printf(" ,");
+
+ double score = 0;
+ for (unsigned size = MIN_SIZE; size <= MAX_SIZE; size <<= SIZE_SHIFT) {
+ /* Don't test bigger sizes if it's too slow. Print 0. */
+ if (size >= 512*1024 &&
+ score < 400 * (size / (4*1024*1024))) {
+ printf("%7.0f ,", 0.0);
+ continue;
+ }
+
+ enum pipe_resource_usage dst_usage, src_usage;
+ struct pipe_resource *dst, *src;
+ struct pipe_query *q[NUM_RUNS];
+ unsigned query_type = PIPE_QUERY_TIME_ELAPSED;
+
+ if (test_sdma) {
+ if (sctx->chip_class == SI)
+ query_type = SI_QUERY_TIME_ELAPSED_SDMA_SI;
+ else
+ query_type = SI_QUERY_TIME_ELAPSED_SDMA;
+ }
+
+ if (placement == 0 || placement == 2 || placement == 4)
+ dst_usage = PIPE_USAGE_DEFAULT;
+ else
+ dst_usage = PIPE_USAGE_STREAM;
+
+ if (placement == 2 || placement == 3)
+ src_usage = PIPE_USAGE_DEFAULT;
+ else
+ src_usage = PIPE_USAGE_STREAM;
+
+ dst = pipe_buffer_create(screen, 0, dst_usage, size);
+ src = is_copy ? pipe_buffer_create(screen, 0, src_usage, size) : NULL;
+
+ /* Run tests. */
+ for (unsigned iter = 0; iter < NUM_RUNS; iter++) {
+ q[iter] = ctx->create_query(ctx, query_type, 0);
+ ctx->begin_query(ctx, q[iter]);
+
+ if (test_cp) {
+ /* CP DMA */
+ if (is_copy) {
+ si_copy_buffer(sctx, dst, src, 0, 0, size, 0,
+ cache_policy);
+ } else {
+ si_cp_dma_clear_buffer(sctx, dst, 0, size, clear_value,
+ SI_COHERENCY_NONE, cache_policy);
+ }
+ } else if (test_sdma) {
+ /* SDMA */
+ if (is_copy) {
+ struct pipe_box box;
+ u_box_1d(0, size, &box);
+ sctx->dma_copy(ctx, dst, 0, 0, 0, 0, src, 0, &box);
+ } else {
+ sctx->dma_clear_buffer(sctx, dst, 0, size, clear_value);
+ }
+ } else {
+ /* Compute */
+ /* The memory accesses are coalesced, meaning that the 1st instruction writes
+ * the 1st contiguous block of data for the whole wave, the 2nd instruction
+ * writes the 2nd contiguous block of data, etc.
+ */
+ unsigned instructions_per_thread = MAX2(1, cs_dwords_per_thread / 4);
+ unsigned dwords_per_instruction = cs_dwords_per_thread / instructions_per_thread;
+ unsigned dwords_per_wave = cs_dwords_per_thread * 64;
+
+ unsigned num_dwords = size / 4;
+ unsigned num_instructions = DIV_ROUND_UP(num_dwords, dwords_per_instruction);
+
+ void *cs = si_create_dma_compute_shader(ctx, cs_dwords_per_thread,
+ cache_policy == L2_STREAM, is_copy);
+
+ struct pipe_grid_info info = {};
+ info.block[0] = MIN2(64, num_instructions);
+ info.block[1] = 1;
+ info.block[2] = 1;
+ info.grid[0] = DIV_ROUND_UP(num_dwords, dwords_per_wave);
+ info.grid[1] = 1;
+ info.grid[2] = 1;
+
+ struct pipe_shader_buffer sb[2] = {};
+ sb[0].buffer = dst;
+ sb[0].buffer_size = size;
+
+ if (is_copy) {
+ sb[1].buffer = src;
+ sb[1].buffer_size = size;
+ } else {
+ for (unsigned i = 0; i < 4; i++)
+ sctx->cs_user_data[i] = clear_value;
+ }
+
+ sctx->flags |= SI_CONTEXT_INV_VMEM_L1 |
+ SI_CONTEXT_INV_SMEM_L1;
+
+ ctx->set_shader_buffers(ctx, PIPE_SHADER_COMPUTE, 0, is_copy ? 2 : 1, sb);
+ ctx->bind_compute_state(ctx, cs);
+ sctx->cs_max_waves_per_sh = cs_waves_per_sh;
+
+ ctx->launch_grid(ctx, &info);
+
+ ctx->bind_compute_state(ctx, NULL);
+ ctx->delete_compute_state(ctx, cs);
+ sctx->cs_max_waves_per_sh = 0; /* disable the limit */
+
+ sctx->flags |= SI_CONTEXT_CS_PARTIAL_FLUSH;
+ }
+
+ /* Flush L2, so that we don't just test L2 cache performance. */
+ if (!test_sdma) {
+ sctx->flags |= SI_CONTEXT_WRITEBACK_GLOBAL_L2;
+ si_emit_cache_flush(sctx);
+ }
+
+ ctx->end_query(ctx, q[iter]);
+ ctx->flush(ctx, NULL, PIPE_FLUSH_ASYNC);
+ }
+ pipe_resource_reference(&dst, NULL);
+ pipe_resource_reference(&src, NULL);
+
+ /* Get results. */
+ uint64_t min = ~0ull, max = 0, total = 0;
+
+ for (unsigned iter = 0; iter < NUM_RUNS; iter++) {
+ union pipe_query_result result;
+
+ ctx->get_query_result(ctx, q[iter], true, &result);
+ ctx->destroy_query(ctx, q[iter]);
+
+ min = MIN2(min, result.u64);
+ max = MAX2(max, result.u64);
+ total += result.u64;
+ }
+
+ score = get_MBps_rate(size, total / (double)NUM_RUNS);
+ printf("%7.0f ,", score);
+ fflush(stdout);
+
+ struct si_result *r = &results[util_logbase2(size)][placement][method];
+ r->is_valid = true;
+ r->is_cp = test_cp;
+ r->is_sdma = test_sdma;
+ r->is_cs = test_cs;
+ r->cache_policy = cache_policy;
+ r->dwords_per_thread = cs_dwords_per_thread;
+ r->waves_per_sh = cs_waves_per_sh;
+ r->score = score;
+ r->index = method;
+ }
+ puts("");
+ }
+ }
+
+ puts("");
+ puts("static struct si_method");
+ printf("get_best_clear_for_%s(enum radeon_bo_domain dst, uint64_t size64, bool async, bool cached)\n",
+ sctx->screen->info.name);
+ puts("{");
+ puts(" unsigned size = MIN2(size64, UINT_MAX);\n");
+
+ /* Analyze results and find the best methods. */
+ for (unsigned placement = 0; placement < ARRAY_SIZE(placement_str); placement++) {
+ if (placement == 0)
+ puts(" if (dst == RADEON_DOMAIN_VRAM) {");
+ else if (placement == 1)
+ puts(" } else { /* GTT */");
+ else if (placement == 2) {
+ puts("}");
+ puts("");
+ puts("static struct si_method");
+ printf("get_best_copy_for_%s(enum radeon_bo_domain dst, enum radeon_bo_domain src,\n",
+ sctx->screen->info.name);
+ printf(" uint64_t size64, bool async, bool cached)\n");
+ puts("{");
+ puts(" unsigned size = MIN2(size64, UINT_MAX);\n");
+ puts(" if (src == RADEON_DOMAIN_VRAM && dst == RADEON_DOMAIN_VRAM) {");
+ } else if (placement == 3)
+ puts(" } else if (src == RADEON_DOMAIN_VRAM && dst == RADEON_DOMAIN_GTT) {");
+ else
+ puts(" } else { /* GTT -> VRAM */");
+
+ for (unsigned mode = 0; mode < 3; mode++) {
+ bool async = mode == 0;
+ bool cached = mode == 1;
+
+ if (async)
+ puts(" if (async) { /* SDMA or async compute */");
+ else if (cached)
+ puts(" if (cached) { /* gfx ring */");
+ else
+ puts(" } else { /* gfx ring - uncached */");
+
+ /* The list of best chosen methods. */
+ struct si_result *methods[32];
+ unsigned method_max_size[32];
+ unsigned num_methods = 0;
+
+ for (unsigned size = MIN_SIZE; size <= MAX_SIZE; size <<= SIZE_SHIFT) {
+ /* Find the best method. */
+ struct si_result *best = NULL;
+
+ for (unsigned i = 0; i < NUM_METHODS; i++) {
+ struct si_result *r = &results[util_logbase2(size)][placement][i];
+
+ if (!r->is_valid)
+ continue;
+
+ /* Ban CP DMA clears via MC on <= VI. They are super slow
+ * on GTT, which we can get due to BO evictions.
+ */
+ if (sctx->chip_class <= VI && placement == 1 &&
+ r->is_cp && r->cache_policy == L2_BYPASS)
+ continue;
+
+ if (async) {
+ /* The following constraints for compute IBs try to limit
+ * resource usage so as not to decrease the performance
+ * of gfx IBs too much.
+ */
+
+ /* Don't use CP DMA on asynchronous rings, because
+ * the engine is shared with gfx IBs.
+ */
+ if (r->is_cp)
+ continue;
+
+ /* Don't use L2 caching on asynchronous rings to minimize
+ * L2 usage.
+ */
+ if (r->cache_policy == L2_LRU)
+ continue;
+
+ /* Asynchronous compute recommends waves_per_sh != 0
+ * to limit CU usage. */
+ if (r->is_cs && r->waves_per_sh == 0)
+ continue;
+ } else {
+ /* SDMA is always asynchronous */
+ if (r->is_sdma)
+ continue;
+
+ if (cached && r->cache_policy == L2_BYPASS)
+ continue;
+ if (!cached && r->cache_policy == L2_LRU)
+ continue;
+ }
+
+ if (!best) {
+ best = r;
+ continue;
+ }
+
+ /* Assume some measurement error. Earlier methods occupy fewer
+ * resources, so the next method is always more greedy, and we
+ * don't want to select it due to a measurement error.
+ */
+ double min_improvement = 1.03;
+
+ if (best->score * min_improvement < r->score)
+ best = r;
+ }
+
+ if (num_methods > 0) {
+ unsigned prev_index = num_methods - 1;
+ struct si_result *prev = methods[prev_index];
+ struct si_result *prev_this_size = &results[util_logbase2(size)][placement][prev->index];
+
+ /* If the best one is also the best for the previous size,
+ * just bump the size for the previous one.
+ *
+ * If there is no best, it means all methods were too slow
+ * for this size and were not tested. Use the best one for
+ * the previous size.
+ */
+ if (!best ||
+ /* If it's the same method as for the previous size: */
+ (prev->is_cp == best->is_cp &&
+ prev->is_sdma == best->is_sdma &&
+ prev->is_cs == best->is_cs &&
+ prev->cache_policy == best->cache_policy &&
+ prev->dwords_per_thread == best->dwords_per_thread &&
+ prev->waves_per_sh == best->waves_per_sh) ||
+ /* If the method for the previous size is also the best
+ * for this size: */
+ (prev_this_size->is_valid &&
+ prev_this_size->score * 1.03 > best->score)) {
+ method_max_size[prev_index] = size;
+ continue;
+ }
+ }
+
+ /* Add it to the list. */
+ assert(num_methods < ARRAY_SIZE(methods));
+ methods[num_methods] = best;
+ method_max_size[num_methods] = size;
+ num_methods++;
+ }
+
+ for (unsigned i = 0; i < num_methods; i++) {
+ struct si_result *best = methods[i];
+ unsigned size = method_max_size[i];
+
+ /* The size threshold is between the current benchmarked
+ * size and the next benchmarked size. */
+ if (i < num_methods - 1)
+ printf(" if (size <= %9u) ", (size + (size << SIZE_SHIFT)) / 2);
+ else if (i > 0)
+ printf(" else ");
+ else
+ printf(" ");
+ printf("return ");
+
+ assert(best);
+ if (best->is_cp) {
+ printf("CP_DMA(%s);\n",
+ best->cache_policy == L2_BYPASS ? "L2_BYPASS" :
+ best->cache_policy == L2_LRU ? "L2_LRU " : "L2_STREAM");
+ }
+ if (best->is_sdma)
+ printf("SDMA;\n");
+ if (best->is_cs) {
+ printf("COMPUTE(%s, %u, %u);\n",
+ best->cache_policy == L2_LRU ? "L2_LRU " : "L2_STREAM",
+ best->dwords_per_thread,
+ best->waves_per_sh);
+ }
+ }
+ }
+ puts(" }");
+ }
+ puts(" }");
+ puts("}");
+
+ ctx->destroy(ctx);
+ exit(0);
+}
projects / mesa.git / commitdiff