radeonsi/compute: Allocate the scratch buffer during state creation
[mesa.git] / src / gallium / drivers / radeonsi / si_compute.c
1 /*
2 * Copyright 2013 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * on the rights to use, copy, modify, merge, publish, distribute, sub
8 * license, and/or sell copies of the Software, and to permit persons to whom
9 * the Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
19 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
20 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE.
22 *
23 */
24
25 #include "util/u_memory.h"
26 #include "radeon/r600_pipe_common.h"
27 #include "radeon/radeon_elf_util.h"
28 #include "radeon/radeon_llvm_util.h"
29
30 #include "radeon/r600_cs.h"
31 #include "si_pipe.h"
32 #include "si_shader.h"
33 #include "sid.h"
34
35 #define MAX_GLOBAL_BUFFERS 20
36 #if HAVE_LLVM < 0x0305
37 #define NUM_USER_SGPRS 2
38 #else
39 /* XXX: Even though we don't pass the scratch buffer via user sgprs any more
40 * LLVM still expects that we specify 4 USER_SGPRS so it can remain compatible
41 * with older mesa. */
42 #define NUM_USER_SGPRS 4
43 #endif
44
45 static const char *scratch_rsrc_dword0_symbol =
46 "SCRATCH_RSRC_DWORD0";
47
48 static const char *scratch_rsrc_dword1_symbol =
49 "SCRATCH_RSRC_DWORD1";
50
51 struct si_compute {
52 struct si_context *ctx;
53
54 unsigned local_size;
55 unsigned private_size;
56 unsigned input_size;
57 struct si_shader shader;
58 unsigned num_user_sgprs;
59
60 struct r600_resource *input_buffer;
61 struct pipe_resource *global_buffers[MAX_GLOBAL_BUFFERS];
62
63 #if HAVE_LLVM < 0x0306
64 unsigned num_kernels;
65 struct si_shader *kernels;
66 LLVMContextRef llvm_ctx;
67 #endif
68 };
69
70 static void apply_scratch_relocs(const struct si_screen *sscreen,
71 struct si_shader *shader, uint64_t scratch_va);
72 static void init_scratch_buffer(struct si_context *sctx, struct si_compute *program)
73 {
74 unsigned scratch_bytes = 0;
75 uint64_t scratch_buffer_va;
76 unsigned i;
77
78 /* Compute the scratch buffer size using the maximum number of waves.
79 * This way we don't need to recompute it for each kernel launch. */
80 unsigned scratch_waves = 32 * sctx->screen->b.info.max_compute_units;
81 for (i = 0; i < program->shader.binary.global_symbol_count; i++) {
82 unsigned offset =
83 program->shader.binary.global_symbol_offsets[i];
84 unsigned scratch_bytes_needed;
85
86 si_shader_binary_read_config(&program->shader.binary,
87 &program->shader, offset);
88 scratch_bytes_needed = program->shader.scratch_bytes_per_wave;
89 scratch_bytes = MAX2(scratch_bytes, scratch_bytes_needed);
90 }
91
92 if (scratch_bytes == 0)
93 return;
94
95 program->shader.scratch_bo = (struct r600_resource*)
96 si_resource_create_custom(sctx->b.b.screen,
97 PIPE_USAGE_DEFAULT,
98 scratch_bytes * scratch_waves);
99
100 scratch_buffer_va = program->shader.scratch_bo->gpu_address;
101
102 /* apply_scratch_relocs needs scratch_bytes_per_wave to be set
103 * to the maximum bytes needed, so it can compute the stride
104 * correctly.
105 */
106 program->shader.scratch_bytes_per_wave = scratch_bytes;
107
108 /* Patch the shader with the scratch buffer address. */
109 apply_scratch_relocs(sctx->screen, &program->shader, scratch_buffer_va);
110
111 }
112
113 static void *si_create_compute_state(
114 struct pipe_context *ctx,
115 const struct pipe_compute_state *cso)
116 {
117 struct si_context *sctx = (struct si_context *)ctx;
118 struct si_compute *program = CALLOC_STRUCT(si_compute);
119 const struct pipe_llvm_program_header *header;
120 const char *code;
121
122 header = cso->prog;
123 code = cso->prog + sizeof(struct pipe_llvm_program_header);
124
125 program->ctx = sctx;
126 program->local_size = cso->req_local_mem;
127 program->private_size = cso->req_private_mem;
128 program->input_size = cso->req_input_mem;
129
130 #if HAVE_LLVM < 0x0306
131 {
132 unsigned i;
133 program->llvm_ctx = LLVMContextCreate();
134 program->num_kernels = radeon_llvm_get_num_kernels(program->llvm_ctx,
135 code, header->num_bytes);
136 program->kernels = CALLOC(sizeof(struct si_shader),
137 program->num_kernels);
138 for (i = 0; i < program->num_kernels; i++) {
139 LLVMModuleRef mod = radeon_llvm_get_kernel_module(program->llvm_ctx, i,
140 code, header->num_bytes);
141 si_compile_llvm(sctx->screen, &program->kernels[i], mod);
142 LLVMDisposeModule(mod);
143 }
144 }
145 #else
146
147 radeon_elf_read(code, header->num_bytes, &program->shader.binary, true);
148
149 /* init_scratch_buffer patches the shader code with the scratch address,
150 * so we need to call it before si_shader_binary_read() which uploads
151 * the shader code to the GPU.
152 */
153 init_scratch_buffer(sctx, program);
154 si_shader_binary_read(sctx->screen, &program->shader, &program->shader.binary);
155
156 #endif
157 program->input_buffer = si_resource_create_custom(sctx->b.b.screen,
158 PIPE_USAGE_IMMUTABLE, program->input_size);
159
160 return program;
161 }
162
163 static void si_bind_compute_state(struct pipe_context *ctx, void *state)
164 {
165 struct si_context *sctx = (struct si_context*)ctx;
166 sctx->cs_shader_state.program = (struct si_compute*)state;
167 }
168
169 static void si_set_global_binding(
170 struct pipe_context *ctx, unsigned first, unsigned n,
171 struct pipe_resource **resources,
172 uint32_t **handles)
173 {
174 unsigned i;
175 struct si_context *sctx = (struct si_context*)ctx;
176 struct si_compute *program = sctx->cs_shader_state.program;
177
178 if (!resources) {
179 for (i = first; i < first + n; i++) {
180 pipe_resource_reference(&program->global_buffers[i], NULL);
181 }
182 return;
183 }
184
185 for (i = first; i < first + n; i++) {
186 uint64_t va;
187 uint32_t offset;
188 pipe_resource_reference(&program->global_buffers[i], resources[i]);
189 va = r600_resource(resources[i])->gpu_address;
190 offset = util_le32_to_cpu(*handles[i]);
191 va += offset;
192 va = util_cpu_to_le64(va);
193 memcpy(handles[i], &va, sizeof(va));
194 }
195 }
196
197 /**
198 * This function computes the value for R_00B860_COMPUTE_TMPRING_SIZE.WAVES
199 * /p block_layout is the number of threads in each work group.
200 * /p grid layout is the number of work groups.
201 */
202 static unsigned compute_num_waves_for_scratch(
203 const struct radeon_info *info,
204 const uint *block_layout,
205 const uint *grid_layout)
206 {
207 unsigned num_sh = MAX2(info->max_sh_per_se, 1);
208 unsigned num_se = MAX2(info->max_se, 1);
209 unsigned num_blocks = 1;
210 unsigned threads_per_block = 1;
211 unsigned waves_per_block;
212 unsigned waves_per_sh;
213 unsigned waves;
214 unsigned scratch_waves;
215 unsigned i;
216
217 for (i = 0; i < 3; i++) {
218 threads_per_block *= block_layout[i];
219 num_blocks *= grid_layout[i];
220 }
221
222 waves_per_block = align(threads_per_block, 64) / 64;
223 waves = waves_per_block * num_blocks;
224 waves_per_sh = align(waves, num_sh * num_se) / (num_sh * num_se);
225 scratch_waves = waves_per_sh * num_sh * num_se;
226
227 if (waves_per_block > waves_per_sh) {
228 scratch_waves = waves_per_block * num_sh * num_se;
229 }
230
231 return scratch_waves;
232 }
233
234 static void apply_scratch_relocs(const struct si_screen *sscreen,
235 struct si_shader *shader, uint64_t scratch_va) {
236 unsigned i;
237 uint32_t scratch_rsrc_dword0 = scratch_va & 0xffffffff;
238 uint32_t scratch_rsrc_dword1 =
239 S_008F04_BASE_ADDRESS_HI(scratch_va >> 32)
240 | S_008F04_STRIDE(shader->scratch_bytes_per_wave / 64);
241
242 if (!shader->binary.reloc_count) {
243 return;
244 }
245
246 for (i = 0 ; i < shader->binary.reloc_count; i++) {
247 const struct radeon_shader_reloc *reloc = &shader->binary.relocs[i];
248 if (!strcmp(scratch_rsrc_dword0_symbol, reloc->name)) {
249 util_memcpy_cpu_to_le32(shader->binary.code + reloc->offset,
250 &scratch_rsrc_dword0, 4);
251 } else if (!strcmp(scratch_rsrc_dword1_symbol, reloc->name)) {
252 util_memcpy_cpu_to_le32(shader->binary.code + reloc->offset,
253 &scratch_rsrc_dword1, 4);
254 }
255 }
256 }
257
258 static void si_launch_grid(
259 struct pipe_context *ctx,
260 const uint *block_layout, const uint *grid_layout,
261 uint32_t pc, const void *input)
262 {
263 struct si_context *sctx = (struct si_context*)ctx;
264 struct radeon_winsys_cs *cs = sctx->b.rings.gfx.cs;
265 struct si_compute *program = sctx->cs_shader_state.program;
266 struct si_pm4_state *pm4 = CALLOC_STRUCT(si_pm4_state);
267 struct r600_resource *input_buffer = program->input_buffer;
268 unsigned kernel_args_size;
269 unsigned num_work_size_bytes = 36;
270 uint32_t kernel_args_offset = 0;
271 uint32_t *kernel_args;
272 uint64_t kernel_args_va;
273 uint64_t scratch_buffer_va = 0;
274 uint64_t shader_va;
275 unsigned arg_user_sgpr_count = NUM_USER_SGPRS;
276 unsigned i;
277 struct si_shader *shader = &program->shader;
278 unsigned lds_blocks;
279 unsigned num_waves_for_scratch;
280
281 #if HAVE_LLVM < 0x0306
282 shader = &program->kernels[pc];
283 #endif
284
285
286 radeon_emit(cs, PKT3(PKT3_CONTEXT_CONTROL, 1, 0) | PKT3_SHADER_TYPE_S(1));
287 radeon_emit(cs, 0x80000000);
288 radeon_emit(cs, 0x80000000);
289
290 sctx->b.flags |= SI_CONTEXT_INV_TC_L1 |
291 SI_CONTEXT_INV_TC_L2 |
292 SI_CONTEXT_INV_ICACHE |
293 SI_CONTEXT_INV_KCACHE |
294 SI_CONTEXT_FLUSH_WITH_INV_L2 |
295 SI_CONTEXT_FLAG_COMPUTE;
296 si_emit_cache_flush(&sctx->b, NULL);
297
298 pm4->compute_pkt = true;
299
300 #if HAVE_LLVM >= 0x0306
301 /* Read the config information */
302 si_shader_binary_read_config(&program->shader.binary, shader, pc);
303 #endif
304
305 /* Upload the kernel arguments */
306
307 /* The extra num_work_size_bytes are for work group / work item size information */
308 kernel_args_size = program->input_size + num_work_size_bytes + 8 /* For scratch va */;
309
310 kernel_args = sctx->b.ws->buffer_map(input_buffer->cs_buf,
311 sctx->b.rings.gfx.cs, PIPE_TRANSFER_WRITE);
312 for (i = 0; i < 3; i++) {
313 kernel_args[i] = grid_layout[i];
314 kernel_args[i + 3] = grid_layout[i] * block_layout[i];
315 kernel_args[i + 6] = block_layout[i];
316 }
317
318 num_waves_for_scratch = compute_num_waves_for_scratch(
319 &sctx->screen->b.info, block_layout, grid_layout);
320
321 memcpy(kernel_args + (num_work_size_bytes / 4), input, program->input_size);
322
323 if (shader->scratch_bytes_per_wave > 0) {
324
325 COMPUTE_DBG(sctx->screen, "Waves: %u; Scratch per wave: %u bytes; "
326 "Total Scratch: %u bytes\n", num_waves_for_scratch,
327 shader->scratch_bytes_per_wave,
328 shader->scratch_bytes_per_wave *
329 num_waves_for_scratch);
330
331 si_pm4_add_bo(pm4, shader->scratch_bo,
332 RADEON_USAGE_READWRITE,
333 RADEON_PRIO_SHADER_RESOURCE_RW);
334
335 scratch_buffer_va = shader->scratch_bo->gpu_address;
336 }
337
338 for (i = 0; i < (kernel_args_size / 4); i++) {
339 COMPUTE_DBG(sctx->screen, "input %u : %u\n", i,
340 kernel_args[i]);
341 }
342
343 sctx->b.ws->buffer_unmap(input_buffer->cs_buf);
344
345 kernel_args_va = input_buffer->gpu_address;
346 kernel_args_va += kernel_args_offset;
347
348 si_pm4_add_bo(pm4, input_buffer, RADEON_USAGE_READ,
349 RADEON_PRIO_SHADER_DATA);
350
351 si_pm4_set_reg(pm4, R_00B900_COMPUTE_USER_DATA_0, kernel_args_va);
352 si_pm4_set_reg(pm4, R_00B900_COMPUTE_USER_DATA_0 + 4, S_008F04_BASE_ADDRESS_HI (kernel_args_va >> 32) | S_008F04_STRIDE(0));
353 si_pm4_set_reg(pm4, R_00B900_COMPUTE_USER_DATA_0 + 8, scratch_buffer_va);
354 si_pm4_set_reg(pm4, R_00B900_COMPUTE_USER_DATA_0 + 12,
355 S_008F04_BASE_ADDRESS_HI(scratch_buffer_va >> 32)
356 | S_008F04_STRIDE(shader->scratch_bytes_per_wave / 64));
357
358 si_pm4_set_reg(pm4, R_00B810_COMPUTE_START_X, 0);
359 si_pm4_set_reg(pm4, R_00B814_COMPUTE_START_Y, 0);
360 si_pm4_set_reg(pm4, R_00B818_COMPUTE_START_Z, 0);
361
362 si_pm4_set_reg(pm4, R_00B81C_COMPUTE_NUM_THREAD_X,
363 S_00B81C_NUM_THREAD_FULL(block_layout[0]));
364 si_pm4_set_reg(pm4, R_00B820_COMPUTE_NUM_THREAD_Y,
365 S_00B820_NUM_THREAD_FULL(block_layout[1]));
366 si_pm4_set_reg(pm4, R_00B824_COMPUTE_NUM_THREAD_Z,
367 S_00B824_NUM_THREAD_FULL(block_layout[2]));
368
369 /* Global buffers */
370 for (i = 0; i < MAX_GLOBAL_BUFFERS; i++) {
371 struct r600_resource *buffer =
372 (struct r600_resource*)program->global_buffers[i];
373 if (!buffer) {
374 continue;
375 }
376 si_pm4_add_bo(pm4, buffer, RADEON_USAGE_READWRITE, RADEON_PRIO_SHADER_RESOURCE_RW);
377 }
378
379 /* This register has been moved to R_00CD20_COMPUTE_MAX_WAVE_ID
380 * and is now per pipe, so it should be handled in the
381 * kernel if we want to use something other than the default value,
382 * which is now 0x22f.
383 */
384 if (sctx->b.chip_class <= SI) {
385 /* XXX: This should be:
386 * (number of compute units) * 4 * (waves per simd) - 1 */
387
388 si_pm4_set_reg(pm4, R_00B82C_COMPUTE_MAX_WAVE_ID,
389 0x190 /* Default value */);
390 }
391
392 shader_va = shader->bo->gpu_address;
393
394 #if HAVE_LLVM >= 0x0306
395 shader_va += pc;
396 #endif
397 si_pm4_add_bo(pm4, shader->bo, RADEON_USAGE_READ, RADEON_PRIO_SHADER_DATA);
398 si_pm4_set_reg(pm4, R_00B830_COMPUTE_PGM_LO, (shader_va >> 8) & 0xffffffff);
399 si_pm4_set_reg(pm4, R_00B834_COMPUTE_PGM_HI, shader_va >> 40);
400
401 si_pm4_set_reg(pm4, R_00B848_COMPUTE_PGM_RSRC1,
402 /* We always use at least 3 VGPRS, these come from
403 * TIDIG_COMP_CNT.
404 * XXX: The compiler should account for this.
405 */
406 S_00B848_VGPRS((MAX2(3, shader->num_vgprs) - 1) / 4)
407 /* We always use at least 4 + arg_user_sgpr_count. The 4 extra
408 * sgprs are from TGID_X_EN, TGID_Y_EN, TGID_Z_EN, TG_SIZE_EN
409 * XXX: The compiler should account for this.
410 */
411 | S_00B848_SGPRS(((MAX2(4 + arg_user_sgpr_count,
412 shader->num_sgprs)) - 1) / 8))
413 ;
414
415 lds_blocks = shader->lds_size;
416 /* XXX: We are over allocating LDS. For SI, the shader reports LDS in
417 * blocks of 256 bytes, so if there are 4 bytes lds allocated in
418 * the shader and 4 bytes allocated by the state tracker, then
419 * we will set LDS_SIZE to 512 bytes rather than 256.
420 */
421 if (sctx->b.chip_class <= SI) {
422 lds_blocks += align(program->local_size, 256) >> 8;
423 } else {
424 lds_blocks += align(program->local_size, 512) >> 9;
425 }
426
427 assert(lds_blocks <= 0xFF);
428
429 si_pm4_set_reg(pm4, R_00B84C_COMPUTE_PGM_RSRC2,
430 S_00B84C_SCRATCH_EN(shader->scratch_bytes_per_wave > 0)
431 | S_00B84C_USER_SGPR(arg_user_sgpr_count)
432 | S_00B84C_TGID_X_EN(1)
433 | S_00B84C_TGID_Y_EN(1)
434 | S_00B84C_TGID_Z_EN(1)
435 | S_00B84C_TG_SIZE_EN(1)
436 | S_00B84C_TIDIG_COMP_CNT(2)
437 | S_00B84C_LDS_SIZE(lds_blocks)
438 | S_00B84C_EXCP_EN(0))
439 ;
440 si_pm4_set_reg(pm4, R_00B854_COMPUTE_RESOURCE_LIMITS, 0);
441
442 si_pm4_set_reg(pm4, R_00B858_COMPUTE_STATIC_THREAD_MGMT_SE0,
443 S_00B858_SH0_CU_EN(0xffff /* Default value */)
444 | S_00B858_SH1_CU_EN(0xffff /* Default value */))
445 ;
446
447 si_pm4_set_reg(pm4, R_00B85C_COMPUTE_STATIC_THREAD_MGMT_SE1,
448 S_00B85C_SH0_CU_EN(0xffff /* Default value */)
449 | S_00B85C_SH1_CU_EN(0xffff /* Default value */))
450 ;
451
452 num_waves_for_scratch =
453 MIN2(num_waves_for_scratch,
454 32 * sctx->screen->b.info.max_compute_units);
455 si_pm4_set_reg(pm4, R_00B860_COMPUTE_TMPRING_SIZE,
456 /* The maximum value for WAVES is 32 * num CU.
457 * If you program this value incorrectly, the GPU will hang if
458 * COMPUTE_PGM_RSRC2.SCRATCH_EN is enabled.
459 */
460 S_00B860_WAVES(num_waves_for_scratch)
461 | S_00B860_WAVESIZE(shader->scratch_bytes_per_wave >> 10))
462 ;
463
464 si_pm4_cmd_begin(pm4, PKT3_DISPATCH_DIRECT);
465 si_pm4_cmd_add(pm4, grid_layout[0]); /* Thread groups DIM_X */
466 si_pm4_cmd_add(pm4, grid_layout[1]); /* Thread groups DIM_Y */
467 si_pm4_cmd_add(pm4, grid_layout[2]); /* Thread gropus DIM_Z */
468 si_pm4_cmd_add(pm4, 1); /* DISPATCH_INITIATOR */
469 si_pm4_cmd_end(pm4, false);
470
471 si_pm4_emit(sctx, pm4);
472
473 #if 0
474 fprintf(stderr, "cdw: %i\n", sctx->cs->cdw);
475 for (i = 0; i < sctx->cs->cdw; i++) {
476 fprintf(stderr, "%4i : 0x%08X\n", i, sctx->cs->buf[i]);
477 }
478 #endif
479
480 si_pm4_free_state(sctx, pm4, ~0);
481
482 sctx->b.flags |= SI_CONTEXT_CS_PARTIAL_FLUSH |
483 SI_CONTEXT_INV_TC_L1 |
484 SI_CONTEXT_INV_TC_L2 |
485 SI_CONTEXT_INV_ICACHE |
486 SI_CONTEXT_INV_KCACHE |
487 SI_CONTEXT_FLAG_COMPUTE;
488 si_emit_cache_flush(&sctx->b, NULL);
489 }
490
491
492 static void si_delete_compute_state(struct pipe_context *ctx, void* state){
493 struct si_compute *program = (struct si_compute *)state;
494
495 if (!state) {
496 return;
497 }
498
499 #if HAVE_LLVM < 0x0306
500 if (program->kernels) {
501 for (int i = 0; i < program->num_kernels; i++){
502 if (program->kernels[i].bo){
503 si_shader_destroy(ctx, &program->kernels[i]);
504 }
505 }
506 FREE(program->kernels);
507 }
508
509 if (program->llvm_ctx){
510 LLVMContextDispose(program->llvm_ctx);
511 }
512 #else
513 si_shader_destroy(ctx, &program->shader);
514 #endif
515
516 pipe_resource_reference(
517 (struct pipe_resource **)&program->input_buffer, NULL);
518
519 radeon_shader_binary_free_members(&program->shader.binary, true);
520 FREE(program);
521 }
522
523 static void si_set_compute_resources(struct pipe_context * ctx_,
524 unsigned start, unsigned count,
525 struct pipe_surface ** surfaces) { }
526
527 void si_init_compute_functions(struct si_context *sctx)
528 {
529 sctx->b.b.create_compute_state = si_create_compute_state;
530 sctx->b.b.delete_compute_state = si_delete_compute_state;
531 sctx->b.b.bind_compute_state = si_bind_compute_state;
532 /* ctx->context.create_sampler_view = evergreen_compute_create_sampler_view; */
533 sctx->b.b.set_compute_resources = si_set_compute_resources;
534 sctx->b.b.set_global_binding = si_set_global_binding;
535 sctx->b.b.launch_grid = si_launch_grid;
536 }