2 * Copyright 2011 Adam Rak <adam.rak@streamnovation.com>
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * on the rights to use, copy, modify, merge, publish, distribute, sub
8 * license, and/or sell copies of the Software, and to permit persons to whom
9 * the Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
19 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
20 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE.
24 * Adam Rak <adam.rak@streamnovation.com>
29 #include "pipe/p_defines.h"
30 #include "pipe/p_state.h"
31 #include "pipe/p_context.h"
32 #include "util/u_blitter.h"
33 #include "util/u_double_list.h"
34 #include "util/u_transfer.h"
35 #include "util/u_surface.h"
36 #include "util/u_pack_color.h"
37 #include "util/u_memory.h"
38 #include "util/u_inlines.h"
39 #include "util/u_framebuffer.h"
40 #include "pipebuffer/pb_buffer.h"
41 #include "evergreend.h"
42 #include "r600_resource.h"
43 #include "r600_shader.h"
44 #include "r600_pipe.h"
45 #include "r600_formats.h"
46 #include "evergreen_compute.h"
47 #include "evergreen_compute_internal.h"
48 #include "compute_memory_pool.h"
50 #include "radeon_llvm_util.h"
54 RAT0 is for global binding write
55 VTX1 is for global binding read
57 for wrting images RAT1...
58 for reading images TEX2...
61 TEX2... consumes the same fetch resources, that VTX2... would consume
63 CONST0 and VTX0 is for parameters
64 CONST0 is binding smaller input parameter buffer, and for constant indexing,
66 VTX0 is for indirect/non-constant indexing, or if the input is bigger than
67 the constant cache can handle
69 RAT-s are limited to 12, so we can only bind at most 11 texture for writing
70 because we reserve RAT0 for global bindings. With byteaddressing enabled,
71 we should reserve another one too.=> 10 image binding for writing max.
74 CL_DEVICE_MAX_READ_IMAGE_ARGS: 128
75 CL_DEVICE_MAX_WRITE_IMAGE_ARGS: 8
77 so 10 for writing is enough. 176 is the max for reading according to the docs
79 writable images should be listed first < 10, so their id corresponds to RAT(id+1)
80 writable images will consume TEX slots, VTX slots too because of linear indexing
84 static void evergreen_cs_set_vertex_buffer(
85 struct r600_context
* rctx
,
88 struct pipe_resource
* buffer
)
90 struct r600_vertexbuf_state
*state
= &rctx
->cs_vertex_buffer_state
;
91 struct pipe_vertex_buffer
*vb
= &state
->vb
[vb_index
];
93 vb
->buffer_offset
= offset
;
95 vb
->user_buffer
= NULL
;
97 /* The vertex instructions in the compute shaders use the texture cache,
98 * so we need to invalidate it. */
99 rctx
->flags
|= R600_CONTEXT_INVAL_READ_CACHES
;
100 state
->enabled_mask
|= 1 << vb_index
;
101 state
->dirty_mask
|= 1 << vb_index
;
102 state
->atom
.dirty
= true;
105 static const struct u_resource_vtbl r600_global_buffer_vtbl
=
107 u_default_resource_get_handle
, /* get_handle */
108 r600_compute_global_buffer_destroy
, /* resource_destroy */
109 r600_compute_global_transfer_map
, /* transfer_map */
110 r600_compute_global_transfer_flush_region
,/* transfer_flush_region */
111 r600_compute_global_transfer_unmap
, /* transfer_unmap */
112 r600_compute_global_transfer_inline_write
/* transfer_inline_write */
116 void *evergreen_create_compute_state(
117 struct pipe_context
*ctx_
,
118 const const struct pipe_compute_state
*cso
)
120 struct r600_context
*ctx
= (struct r600_context
*)ctx_
;
121 struct r600_pipe_compute
*shader
= CALLOC_STRUCT(r600_pipe_compute
);
124 const struct pipe_llvm_program_header
* header
;
125 const unsigned char * code
;
128 COMPUTE_DBG(ctx
->screen
, "*** evergreen_create_compute_state\n");
131 code
= cso
->prog
+ sizeof(struct pipe_llvm_program_header
);
134 shader
->ctx
= (struct r600_context
*)ctx
;
135 shader
->resources
= (struct evergreen_compute_resource
*)
136 CALLOC(sizeof(struct evergreen_compute_resource
),
137 get_compute_resource_num());
138 shader
->local_size
= cso
->req_local_mem
; ///TODO: assert it
139 shader
->private_size
= cso
->req_private_mem
;
140 shader
->input_size
= cso
->req_input_mem
;
143 shader
->num_kernels
= radeon_llvm_get_num_kernels(code
, header
->num_bytes
);
144 shader
->kernels
= CALLOC(sizeof(struct r600_kernel
), shader
->num_kernels
);
146 for (i
= 0; i
< shader
->num_kernels
; i
++) {
147 struct r600_kernel
*kernel
= &shader
->kernels
[i
];
148 kernel
->llvm_module
= radeon_llvm_get_kernel_module(i
, code
,
155 void evergreen_delete_compute_state(struct pipe_context
*ctx
, void* state
)
157 struct r600_pipe_compute
*shader
= (struct r600_pipe_compute
*)state
;
159 free(shader
->resources
);
163 static void evergreen_bind_compute_state(struct pipe_context
*ctx_
, void *state
)
165 struct r600_context
*ctx
= (struct r600_context
*)ctx_
;
167 COMPUTE_DBG(ctx
->screen
, "*** evergreen_bind_compute_state\n");
169 ctx
->cs_shader_state
.shader
= (struct r600_pipe_compute
*)state
;
172 /* The kernel parameters are stored a vtx buffer (ID=0), besides the explicit
173 * kernel parameters there are inplicit parameters that need to be stored
174 * in the vertex buffer as well. Here is how these parameters are organized in
177 * DWORDS 0-2: Number of work groups in each dimension (x,y,z)
178 * DWORDS 3-5: Number of global work items in each dimension (x,y,z)
179 * DWORDS 6-8: Number of work items within each work group in each dimension
181 * DWORDS 9+ : Kernel parameters
183 void evergreen_compute_upload_input(
184 struct pipe_context
*ctx_
,
185 const uint
*block_layout
,
186 const uint
*grid_layout
,
189 struct r600_context
*ctx
= (struct r600_context
*)ctx_
;
190 struct r600_pipe_compute
*shader
= ctx
->cs_shader_state
.shader
;
192 unsigned kernel_parameters_offset_bytes
= 36;
193 uint32_t * num_work_groups_start
;
194 uint32_t * global_size_start
;
195 uint32_t * local_size_start
;
196 uint32_t * kernel_parameters_start
;
198 if (shader
->input_size
== 0) {
202 if (!shader
->kernel_param
) {
203 unsigned buffer_size
= shader
->input_size
;
205 /* Add space for the grid dimensions */
206 buffer_size
+= kernel_parameters_offset_bytes
;
207 shader
->kernel_param
= r600_compute_buffer_alloc_vram(
208 ctx
->screen
, buffer_size
);
211 num_work_groups_start
= r600_buffer_mmap_sync_with_rings(ctx
, shader
->kernel_param
, PIPE_TRANSFER_WRITE
);
212 global_size_start
= num_work_groups_start
+ (3 * (sizeof(uint
) /4));
213 local_size_start
= global_size_start
+ (3 * (sizeof(uint
)) / 4);
214 kernel_parameters_start
= local_size_start
+ (3 * (sizeof(uint
)) / 4);
216 /* Copy the work group size */
217 memcpy(num_work_groups_start
, grid_layout
, 3 * sizeof(uint
));
219 /* Copy the global size */
220 for (i
= 0; i
< 3; i
++) {
221 global_size_start
[i
] = grid_layout
[i
] * block_layout
[i
];
224 /* Copy the local dimensions */
225 memcpy(local_size_start
, block_layout
, 3 * sizeof(uint
));
227 /* Copy the kernel inputs */
228 memcpy(kernel_parameters_start
, input
, shader
->input_size
);
230 for (i
= 0; i
< (kernel_parameters_offset_bytes
/ 4) +
231 (shader
->input_size
/ 4); i
++) {
232 COMPUTE_DBG(ctx
->screen
, "input %i : %i\n", i
,
233 ((unsigned*)num_work_groups_start
)[i
]);
236 ctx
->ws
->buffer_unmap(shader
->kernel_param
->cs_buf
);
238 ///ID=0 is reserved for the parameters
239 evergreen_cs_set_vertex_buffer(ctx
, 0, 0,
240 (struct pipe_resource
*)shader
->kernel_param
);
241 ///ID=0 is reserved for parameters
242 evergreen_set_const_cache(shader
, 0, shader
->kernel_param
,
243 shader
->input_size
, 0);
246 static void evergreen_emit_direct_dispatch(
247 struct r600_context
*rctx
,
248 const uint
*block_layout
, const uint
*grid_layout
)
251 struct radeon_winsys_cs
*cs
= rctx
->rings
.gfx
.cs
;
253 unsigned num_pipes
= rctx
->screen
->info
.r600_max_pipes
;
254 unsigned wave_divisor
= (16 * num_pipes
);
257 /* XXX: Enable lds and get size from cs_shader_state */
258 unsigned lds_size
= 0;
260 /* Calculate group_size/grid_size */
261 for (i
= 0; i
< 3; i
++) {
262 group_size
*= block_layout
[i
];
265 for (i
= 0; i
< 3; i
++) {
266 grid_size
*= grid_layout
[i
];
269 /* num_waves = ceil((tg_size.x * tg_size.y, tg_size.z) / (16 * num_pipes)) */
270 num_waves
= (block_layout
[0] * block_layout
[1] * block_layout
[2] +
271 wave_divisor
- 1) / wave_divisor
;
273 COMPUTE_DBG(rctx
->screen
, "Using %u pipes, there are %u wavefronts per thread block\n",
274 num_pipes
, num_waves
);
276 /* XXX: Partition the LDS between PS/CS. By default half (4096 dwords
277 * on Evergreen) oes to Pixel Shaders and half goes to Compute Shaders.
278 * We may need to allocat the entire LDS space for Compute Shaders.
280 * EG: R_008E2C_SQ_LDS_RESOURCE_MGMT := S_008E2C_NUM_LS_LDS(lds_dwords)
281 * CM: CM_R_0286FC_SPI_LDS_MGMT := S_0286FC_NUM_LS_LDS(lds_dwords)
284 r600_write_config_reg(cs
, R_008970_VGT_NUM_INDICES
, group_size
);
286 r600_write_config_reg_seq(cs
, R_00899C_VGT_COMPUTE_START_X
, 3);
287 r600_write_value(cs
, 0); /* R_00899C_VGT_COMPUTE_START_X */
288 r600_write_value(cs
, 0); /* R_0089A0_VGT_COMPUTE_START_Y */
289 r600_write_value(cs
, 0); /* R_0089A4_VGT_COMPUTE_START_Z */
291 r600_write_config_reg(cs
, R_0089AC_VGT_COMPUTE_THREAD_GROUP_SIZE
,
294 r600_write_compute_context_reg_seq(cs
, R_0286EC_SPI_COMPUTE_NUM_THREAD_X
, 3);
295 r600_write_value(cs
, block_layout
[0]); /* R_0286EC_SPI_COMPUTE_NUM_THREAD_X */
296 r600_write_value(cs
, block_layout
[1]); /* R_0286F0_SPI_COMPUTE_NUM_THREAD_Y */
297 r600_write_value(cs
, block_layout
[2]); /* R_0286F4_SPI_COMPUTE_NUM_THREAD_Z */
299 r600_write_compute_context_reg(cs
, CM_R_0288E8_SQ_LDS_ALLOC
,
300 lds_size
| (num_waves
<< 14));
302 /* Dispatch packet */
303 r600_write_value(cs
, PKT3C(PKT3_DISPATCH_DIRECT
, 3, 0));
304 r600_write_value(cs
, grid_layout
[0]);
305 r600_write_value(cs
, grid_layout
[1]);
306 r600_write_value(cs
, grid_layout
[2]);
307 /* VGT_DISPATCH_INITIATOR = COMPUTE_SHADER_EN */
308 r600_write_value(cs
, 1);
311 static void compute_emit_cs(struct r600_context
*ctx
, const uint
*block_layout
,
312 const uint
*grid_layout
)
314 struct radeon_winsys_cs
*cs
= ctx
->rings
.gfx
.cs
;
315 unsigned flush_flags
= 0;
317 struct r600_resource
*onebo
= NULL
;
318 struct evergreen_compute_resource
*resources
=
319 ctx
->cs_shader_state
.shader
->resources
;
321 /* make sure that the gfx ring is only one active */
322 if (ctx
->rings
.dma
.cs
) {
323 ctx
->rings
.dma
.flush(ctx
, RADEON_FLUSH_ASYNC
);
326 /* Initialize all the compute-related registers.
328 * See evergreen_init_atom_start_compute_cs() in this file for the list
329 * of registers initialized by the start_compute_cs_cmd atom.
331 r600_emit_command_buffer(cs
, &ctx
->start_compute_cs_cmd
);
333 ctx
->flags
|= R600_CONTEXT_WAIT_3D_IDLE
| R600_CONTEXT_FLUSH_AND_INV
;
334 r600_flush_emit(ctx
);
336 /* Emit colorbuffers. */
337 for (i
= 0; i
< ctx
->framebuffer
.state
.nr_cbufs
; i
++) {
338 struct r600_surface
*cb
= (struct r600_surface
*)ctx
->framebuffer
.state
.cbufs
[i
];
339 unsigned reloc
= r600_context_bo_reloc(ctx
, &ctx
->rings
.gfx
,
340 (struct r600_resource
*)cb
->base
.texture
,
341 RADEON_USAGE_READWRITE
);
343 r600_write_compute_context_reg_seq(cs
, R_028C60_CB_COLOR0_BASE
+ i
* 0x3C, 7);
344 r600_write_value(cs
, cb
->cb_color_base
); /* R_028C60_CB_COLOR0_BASE */
345 r600_write_value(cs
, cb
->cb_color_pitch
); /* R_028C64_CB_COLOR0_PITCH */
346 r600_write_value(cs
, cb
->cb_color_slice
); /* R_028C68_CB_COLOR0_SLICE */
347 r600_write_value(cs
, cb
->cb_color_view
); /* R_028C6C_CB_COLOR0_VIEW */
348 r600_write_value(cs
, cb
->cb_color_info
); /* R_028C70_CB_COLOR0_INFO */
349 r600_write_value(cs
, cb
->cb_color_attrib
); /* R_028C74_CB_COLOR0_ATTRIB */
350 r600_write_value(cs
, cb
->cb_color_dim
); /* R_028C78_CB_COLOR0_DIM */
352 r600_write_value(cs
, PKT3(PKT3_NOP
, 0, 0)); /* R_028C60_CB_COLOR0_BASE */
353 r600_write_value(cs
, reloc
);
355 if (!ctx
->keep_tiling_flags
) {
356 r600_write_value(cs
, PKT3(PKT3_NOP
, 0, 0)); /* R_028C70_CB_COLOR0_INFO */
357 r600_write_value(cs
, reloc
);
360 r600_write_value(cs
, PKT3(PKT3_NOP
, 0, 0)); /* R_028C74_CB_COLOR0_ATTRIB */
361 r600_write_value(cs
, reloc
);
364 /* Set CB_TARGET_MASK XXX: Use cb_misc_state */
365 r600_write_compute_context_reg(cs
, R_028238_CB_TARGET_MASK
,
366 ctx
->compute_cb_target_mask
);
369 /* Emit vertex buffer state */
370 ctx
->cs_vertex_buffer_state
.atom
.num_dw
= 12 * util_bitcount(ctx
->cs_vertex_buffer_state
.dirty_mask
);
371 r600_emit_atom(ctx
, &ctx
->cs_vertex_buffer_state
.atom
);
373 /* Emit compute shader state */
374 r600_emit_atom(ctx
, &ctx
->cs_shader_state
.atom
);
376 for (i
= 0; i
< get_compute_resource_num(); i
++) {
377 if (resources
[i
].enabled
) {
379 COMPUTE_DBG(ctx
->screen
, "resnum: %i, cdw: %i\n", i
, cs
->cdw
);
381 for (j
= 0; j
< resources
[i
].cs_end
; j
++) {
382 if (resources
[i
].do_reloc
[j
]) {
383 assert(resources
[i
].bo
);
384 evergreen_emit_ctx_reloc(ctx
,
389 cs
->buf
[cs
->cdw
++] = resources
[i
].cs
[j
];
392 if (resources
[i
].bo
) {
393 onebo
= resources
[i
].bo
;
394 evergreen_emit_ctx_reloc(ctx
,
398 ///special case for textures
399 if (resources
[i
].do_reloc
400 [resources
[i
].cs_end
] == 2) {
401 evergreen_emit_ctx_reloc(ctx
,
409 /* Emit dispatch state and dispatch packet */
410 evergreen_emit_direct_dispatch(ctx
, block_layout
, grid_layout
);
412 /* XXX evergreen_flush_emit() hardcodes the CP_COHER_SIZE to 0xffffffff
414 ctx
->flags
|= R600_CONTEXT_INVAL_READ_CACHES
;
415 r600_flush_emit(ctx
);
418 COMPUTE_DBG(ctx
->screen
, "cdw: %i\n", cs
->cdw
);
419 for (i
= 0; i
< cs
->cdw
; i
++) {
420 COMPUTE_DBG(ctx
->screen
, "%4i : 0x%08X\n", i
, ctx
->cs
->buf
[i
]);
424 flush_flags
= RADEON_FLUSH_ASYNC
| RADEON_FLUSH_COMPUTE
;
425 if (ctx
->keep_tiling_flags
) {
426 flush_flags
|= RADEON_FLUSH_KEEP_TILING_FLAGS
;
429 ctx
->ws
->cs_flush(ctx
->rings
.gfx
.cs
, flush_flags
);
433 COMPUTE_DBG(ctx
->screen
, "shader started\n");
435 ctx
->ws
->buffer_wait(onebo
->buf
, 0);
437 COMPUTE_DBG(ctx
->screen
, "...\n");
442 * Emit function for r600_cs_shader_state atom
444 void evergreen_emit_cs_shader(
445 struct r600_context
*rctx
,
446 struct r600_atom
*atom
)
448 struct r600_cs_shader_state
*state
=
449 (struct r600_cs_shader_state
*)atom
;
450 struct r600_pipe_compute
*shader
= state
->shader
;
451 struct r600_kernel
*kernel
= &shader
->kernels
[state
->kernel_index
];
452 struct radeon_winsys_cs
*cs
= rctx
->rings
.gfx
.cs
;
455 va
= r600_resource_va(&rctx
->screen
->screen
, &kernel
->code_bo
->b
.b
);
457 r600_write_compute_context_reg_seq(cs
, R_0288D0_SQ_PGM_START_LS
, 3);
458 r600_write_value(cs
, va
>> 8); /* R_0288D0_SQ_PGM_START_LS */
459 r600_write_value(cs
, /* R_0288D4_SQ_PGM_RESOURCES_LS */
460 S_0288D4_NUM_GPRS(kernel
->bc
.ngpr
)
461 | S_0288D4_STACK_SIZE(kernel
->bc
.nstack
));
462 r600_write_value(cs
, 0); /* R_0288D8_SQ_PGM_RESOURCES_LS_2 */
464 r600_write_value(cs
, PKT3C(PKT3_NOP
, 0, 0));
465 r600_write_value(cs
, r600_context_bo_reloc(rctx
, &rctx
->rings
.gfx
,
466 kernel
->code_bo
, RADEON_USAGE_READ
));
468 rctx
->flags
|= R600_CONTEXT_INVAL_READ_CACHES
;
471 static void evergreen_launch_grid(
472 struct pipe_context
*ctx_
,
473 const uint
*block_layout
, const uint
*grid_layout
,
474 uint32_t pc
, const void *input
)
476 struct r600_context
*ctx
= (struct r600_context
*)ctx_
;
479 COMPUTE_DBG(ctx
->screen
, "*** evergreen_launch_grid: pc = %u\n", pc
);
481 struct r600_pipe_compute
*shader
= ctx
->cs_shader_state
.shader
;
482 if (!shader
->kernels
[pc
].code_bo
) {
484 struct r600_kernel
*kernel
= &shader
->kernels
[pc
];
485 r600_compute_shader_create(ctx_
, kernel
->llvm_module
, &kernel
->bc
);
486 kernel
->code_bo
= r600_compute_buffer_alloc_vram(ctx
->screen
,
488 p
= r600_buffer_mmap_sync_with_rings(ctx
, kernel
->code_bo
, PIPE_TRANSFER_WRITE
);
489 memcpy(p
, kernel
->bc
.bytecode
, kernel
->bc
.ndw
* 4);
490 ctx
->ws
->buffer_unmap(kernel
->code_bo
->cs_buf
);
494 ctx
->cs_shader_state
.kernel_index
= pc
;
495 evergreen_compute_upload_input(ctx_
, block_layout
, grid_layout
, input
);
496 compute_emit_cs(ctx
, block_layout
, grid_layout
);
499 static void evergreen_set_compute_resources(struct pipe_context
* ctx_
,
500 unsigned start
, unsigned count
,
501 struct pipe_surface
** surfaces
)
503 struct r600_context
*ctx
= (struct r600_context
*)ctx_
;
504 struct r600_surface
**resources
= (struct r600_surface
**)surfaces
;
506 COMPUTE_DBG(ctx
->screen
, "*** evergreen_set_compute_resources: start = %u count = %u\n",
509 for (int i
= 0; i
< count
; i
++) {
510 /* The First two vertex buffers are reserved for parameters and
512 unsigned vtx_id
= 2 + i
;
514 struct r600_resource_global
*buffer
=
515 (struct r600_resource_global
*)
516 resources
[i
]->base
.texture
;
517 if (resources
[i
]->base
.writable
) {
520 evergreen_set_rat(ctx
->cs_shader_state
.shader
, i
+1,
521 (struct r600_resource
*)resources
[i
]->base
.texture
,
522 buffer
->chunk
->start_in_dw
*4,
523 resources
[i
]->base
.texture
->width0
);
526 evergreen_cs_set_vertex_buffer(ctx
, vtx_id
,
527 buffer
->chunk
->start_in_dw
* 4,
528 resources
[i
]->base
.texture
);
533 static void evergreen_set_cs_sampler_view(struct pipe_context
*ctx_
,
534 unsigned start_slot
, unsigned count
,
535 struct pipe_sampler_view
**views
)
537 struct r600_context
*ctx
= (struct r600_context
*)ctx_
;
538 struct r600_pipe_sampler_view
**resource
=
539 (struct r600_pipe_sampler_view
**)views
;
541 for (int i
= 0; i
< count
; i
++) {
544 ///FETCH0 = VTX0 (param buffer),
545 //FETCH1 = VTX1 (global buffer pool), FETCH2... = TEX
546 evergreen_set_tex_resource(ctx
->cs_shader_state
.shader
, resource
[i
], i
+2);
551 static void evergreen_bind_compute_sampler_states(
552 struct pipe_context
*ctx_
,
554 unsigned num_samplers
,
557 struct r600_context
*ctx
= (struct r600_context
*)ctx_
;
558 struct compute_sampler_state
** samplers
=
559 (struct compute_sampler_state
**)samplers_
;
561 for (int i
= 0; i
< num_samplers
; i
++) {
563 evergreen_set_sampler_resource(
564 ctx
->cs_shader_state
.shader
, samplers
[i
], i
);
569 static void evergreen_set_global_binding(
570 struct pipe_context
*ctx_
, unsigned first
, unsigned n
,
571 struct pipe_resource
**resources
,
574 struct r600_context
*ctx
= (struct r600_context
*)ctx_
;
575 struct compute_memory_pool
*pool
= ctx
->screen
->global_pool
;
576 struct r600_resource_global
**buffers
=
577 (struct r600_resource_global
**)resources
;
579 COMPUTE_DBG(ctx
->screen
, "*** evergreen_set_global_binding first = %u n = %u\n",
587 compute_memory_finalize_pending(pool
, ctx_
);
589 for (int i
= 0; i
< n
; i
++)
591 assert(resources
[i
]->target
== PIPE_BUFFER
);
592 assert(resources
[i
]->bind
& PIPE_BIND_GLOBAL
);
594 *(handles
[i
]) = buffers
[i
]->chunk
->start_in_dw
* 4;
597 evergreen_set_rat(ctx
->cs_shader_state
.shader
, 0, pool
->bo
, 0, pool
->size_in_dw
* 4);
598 evergreen_cs_set_vertex_buffer(ctx
, 1, 0,
599 (struct pipe_resource
*)pool
->bo
);
603 * This function initializes all the compute specific registers that need to
604 * be initialized for each compute command stream. Registers that are common
605 * to both compute and 3D will be initialized at the beginning of each compute
606 * command stream by the start_cs_cmd atom. However, since the SET_CONTEXT_REG
607 * packet requires that the shader type bit be set, we must initialize all
608 * context registers needed for compute in this function. The registers
609 * intialized by the start_cs_cmd atom can be found in evereen_state.c in the
610 * functions evergreen_init_atom_start_cs or cayman_init_atom_start_cs depending
613 void evergreen_init_atom_start_compute_cs(struct r600_context
*ctx
)
615 struct r600_command_buffer
*cb
= &ctx
->start_compute_cs_cmd
;
617 int num_stack_entries
;
619 /* since all required registers are initialised in the
620 * start_compute_cs_cmd atom, we can EMIT_EARLY here.
622 r600_init_command_buffer(cb
, 256);
623 cb
->pkt_flags
= RADEON_CP_PACKET3_COMPUTE_MODE
;
625 /* This must be first. */
626 r600_store_value(cb
, PKT3(PKT3_CONTEXT_CONTROL
, 1, 0));
627 r600_store_value(cb
, 0x80000000);
628 r600_store_value(cb
, 0x80000000);
630 /* We're setting config registers here. */
631 r600_store_value(cb
, PKT3(PKT3_EVENT_WRITE
, 0, 0));
632 r600_store_value(cb
, EVENT_TYPE(EVENT_TYPE_CS_PARTIAL_FLUSH
) | EVENT_INDEX(4));
634 switch (ctx
->family
) {
638 num_stack_entries
= 256;
642 num_stack_entries
= 256;
646 num_stack_entries
= 512;
651 num_stack_entries
= 512;
655 num_stack_entries
= 256;
659 num_stack_entries
= 256;
663 num_stack_entries
= 512;
667 num_stack_entries
= 512;
671 num_stack_entries
= 256;
675 num_stack_entries
= 256;
679 /* Config Registers */
680 if (ctx
->chip_class
< CAYMAN
)
681 evergreen_init_common_regs(cb
, ctx
->chip_class
, ctx
->family
,
682 ctx
->screen
->info
.drm_minor
);
684 cayman_init_common_regs(cb
, ctx
->chip_class
, ctx
->family
,
685 ctx
->screen
->info
.drm_minor
);
687 /* The primitive type always needs to be POINTLIST for compute. */
688 r600_store_config_reg(cb
, R_008958_VGT_PRIMITIVE_TYPE
,
689 V_008958_DI_PT_POINTLIST
);
691 if (ctx
->chip_class
< CAYMAN
) {
693 /* These registers control which simds can be used by each stage.
694 * The default for these registers is 0xffffffff, which means
695 * all simds are available for each stage. It's possible we may
696 * want to play around with these in the future, but for now
697 * the default value is fine.
699 * R_008E20_SQ_STATIC_THREAD_MGMT1
700 * R_008E24_SQ_STATIC_THREAD_MGMT2
701 * R_008E28_SQ_STATIC_THREAD_MGMT3
704 /* XXX: We may need to adjust the thread and stack resouce
705 * values for 3D/compute interop */
707 r600_store_config_reg_seq(cb
, R_008C18_SQ_THREAD_RESOURCE_MGMT_1
, 5);
709 /* R_008C18_SQ_THREAD_RESOURCE_MGMT_1
710 * Set the number of threads used by the PS/VS/GS/ES stage to
713 r600_store_value(cb
, 0);
715 /* R_008C1C_SQ_THREAD_RESOURCE_MGMT_2
716 * Set the number of threads used by the CS (aka LS) stage to
717 * the maximum number of threads and set the number of threads
718 * for the HS stage to 0. */
719 r600_store_value(cb
, S_008C1C_NUM_LS_THREADS(num_threads
));
721 /* R_008C20_SQ_STACK_RESOURCE_MGMT_1
722 * Set the Control Flow stack entries to 0 for PS/VS stages */
723 r600_store_value(cb
, 0);
725 /* R_008C24_SQ_STACK_RESOURCE_MGMT_2
726 * Set the Control Flow stack entries to 0 for GS/ES stages */
727 r600_store_value(cb
, 0);
729 /* R_008C28_SQ_STACK_RESOURCE_MGMT_3
730 * Set the Contol Flow stack entries to 0 for the HS stage, and
731 * set it to the maximum value for the CS (aka LS) stage. */
733 S_008C28_NUM_LS_STACK_ENTRIES(num_stack_entries
));
736 /* Context Registers */
738 if (ctx
->chip_class
< CAYMAN
) {
739 /* workaround for hw issues with dyn gpr - must set all limits
740 * to 240 instead of 0, 0x1e == 240 / 8
742 r600_store_context_reg(cb
, R_028838_SQ_DYN_GPR_RESOURCE_LIMIT_1
,
743 S_028838_PS_GPRS(0x1e) |
744 S_028838_VS_GPRS(0x1e) |
745 S_028838_GS_GPRS(0x1e) |
746 S_028838_ES_GPRS(0x1e) |
747 S_028838_HS_GPRS(0x1e) |
748 S_028838_LS_GPRS(0x1e));
751 /* XXX: Investigate setting bit 15, which is FAST_COMPUTE_MODE */
752 r600_store_context_reg(cb
, R_028A40_VGT_GS_MODE
,
753 S_028A40_COMPUTE_MODE(1) | S_028A40_PARTIAL_THD_AT_EOI(1));
755 r600_store_context_reg(cb
, R_028B54_VGT_SHADER_STAGES_EN
, 2/*CS_ON*/);
757 r600_store_context_reg(cb
, R_0286E8_SPI_COMPUTE_INPUT_CNTL
,
758 S_0286E8_TID_IN_GROUP_ENA
760 | S_0286E8_DISABLE_INDEX_PACK
)
763 /* The LOOP_CONST registers are an optimizations for loops that allows
764 * you to store the initial counter, increment value, and maximum
765 * counter value in a register so that hardware can calculate the
766 * correct number of iterations for the loop, so that you don't need
767 * to have the loop counter in your shader code. We don't currently use
768 * this optimization, so we must keep track of the counter in the
769 * shader and use a break instruction to exit loops. However, the
770 * hardware will still uses this register to determine when to exit a
771 * loop, so we need to initialize the counter to 0, set the increment
772 * value to 1 and the maximum counter value to the 4095 (0xfff) which
773 * is the maximum value allowed. This gives us a maximum of 4096
774 * iterations for our loops, but hopefully our break instruction will
775 * execute before some time before the 4096th iteration.
777 eg_store_loop_const(cb
, R_03A200_SQ_LOOP_CONST_0
+ (160 * 4), 0x1000FFF);
780 void evergreen_init_compute_state_functions(struct r600_context
*ctx
)
782 ctx
->context
.create_compute_state
= evergreen_create_compute_state
;
783 ctx
->context
.delete_compute_state
= evergreen_delete_compute_state
;
784 ctx
->context
.bind_compute_state
= evergreen_bind_compute_state
;
785 // ctx->context.create_sampler_view = evergreen_compute_create_sampler_view;
786 ctx
->context
.set_compute_resources
= evergreen_set_compute_resources
;
787 ctx
->context
.set_compute_sampler_views
= evergreen_set_cs_sampler_view
;
788 ctx
->context
.bind_compute_sampler_states
= evergreen_bind_compute_sampler_states
;
789 ctx
->context
.set_global_binding
= evergreen_set_global_binding
;
790 ctx
->context
.launch_grid
= evergreen_launch_grid
;
792 /* We always use at least two vertex buffers for compute, one for
793 * parameters and one for global memory */
794 ctx
->cs_vertex_buffer_state
.enabled_mask
=
795 ctx
->cs_vertex_buffer_state
.dirty_mask
= 1 | 2;
799 struct pipe_resource
*r600_compute_global_buffer_create(
800 struct pipe_screen
*screen
,
801 const struct pipe_resource
*templ
)
803 struct r600_resource_global
* result
= NULL
;
804 struct r600_screen
* rscreen
= NULL
;
807 assert(templ
->target
== PIPE_BUFFER
);
808 assert(templ
->bind
& PIPE_BIND_GLOBAL
);
809 assert(templ
->array_size
== 1 || templ
->array_size
== 0);
810 assert(templ
->depth0
== 1 || templ
->depth0
== 0);
811 assert(templ
->height0
== 1 || templ
->height0
== 0);
813 result
= (struct r600_resource_global
*)
814 CALLOC(sizeof(struct r600_resource_global
), 1);
815 rscreen
= (struct r600_screen
*)screen
;
817 COMPUTE_DBG(rscreen
, "*** r600_compute_global_buffer_create\n");
818 COMPUTE_DBG(rscreen
, "width = %u array_size = %u\n", templ
->width0
,
821 result
->base
.b
.vtbl
= &r600_global_buffer_vtbl
;
822 result
->base
.b
.b
.screen
= screen
;
823 result
->base
.b
.b
= *templ
;
824 pipe_reference_init(&result
->base
.b
.b
.reference
, 1);
826 size_in_dw
= (templ
->width0
+3) / 4;
828 result
->chunk
= compute_memory_alloc(rscreen
->global_pool
, size_in_dw
);
830 if (result
->chunk
== NULL
)
836 return &result
->base
.b
.b
;
839 void r600_compute_global_buffer_destroy(
840 struct pipe_screen
*screen
,
841 struct pipe_resource
*res
)
843 struct r600_resource_global
* buffer
= NULL
;
844 struct r600_screen
* rscreen
= NULL
;
846 assert(res
->target
== PIPE_BUFFER
);
847 assert(res
->bind
& PIPE_BIND_GLOBAL
);
849 buffer
= (struct r600_resource_global
*)res
;
850 rscreen
= (struct r600_screen
*)screen
;
852 compute_memory_free(rscreen
->global_pool
, buffer
->chunk
->id
);
854 buffer
->chunk
= NULL
;
858 void *r600_compute_global_transfer_map(
859 struct pipe_context
*ctx_
,
860 struct pipe_resource
*resource
,
863 const struct pipe_box
*box
,
864 struct pipe_transfer
**ptransfer
)
866 struct r600_context
*rctx
= (struct r600_context
*)ctx_
;
867 struct compute_memory_pool
*pool
= rctx
->screen
->global_pool
;
868 struct pipe_transfer
*transfer
= util_slab_alloc(&rctx
->pool_transfers
);
869 struct r600_resource_global
* buffer
=
870 (struct r600_resource_global
*)resource
;
873 compute_memory_finalize_pending(pool
, ctx_
);
875 assert(resource
->target
== PIPE_BUFFER
);
877 COMPUTE_DBG(rctx
->screen
, "* r600_compute_global_get_transfer()\n"
878 "level = %u, usage = %u, box(x = %u, y = %u, z = %u "
879 "width = %u, height = %u, depth = %u)\n", level
, usage
,
880 box
->x
, box
->y
, box
->z
, box
->width
, box
->height
,
883 transfer
->resource
= resource
;
884 transfer
->level
= level
;
885 transfer
->usage
= usage
;
886 transfer
->box
= *box
;
887 transfer
->stride
= 0;
888 transfer
->layer_stride
= 0;
890 assert(transfer
->resource
->target
== PIPE_BUFFER
);
891 assert(transfer
->resource
->bind
& PIPE_BIND_GLOBAL
);
892 assert(transfer
->box
.x
>= 0);
893 assert(transfer
->box
.y
== 0);
894 assert(transfer
->box
.z
== 0);
896 ///TODO: do it better, mapping is not possible if the pool is too big
898 COMPUTE_DBG(rctx
->screen
, "* r600_compute_global_transfer_map()\n");
900 if (!(map
= r600_buffer_mmap_sync_with_rings(rctx
, buffer
->chunk
->pool
->bo
, transfer
->usage
))) {
901 util_slab_free(&rctx
->pool_transfers
, transfer
);
905 *ptransfer
= transfer
;
907 COMPUTE_DBG(rctx
->screen
, "Buffer: %p + %u (buffer offset in global memory) "
908 "+ %u (box.x)\n", map
, buffer
->chunk
->start_in_dw
, transfer
->box
.x
);
909 return ((char*)(map
+ buffer
->chunk
->start_in_dw
)) + transfer
->box
.x
;
912 void r600_compute_global_transfer_unmap(
913 struct pipe_context
*ctx_
,
914 struct pipe_transfer
* transfer
)
916 struct r600_context
*ctx
= NULL
;
917 struct r600_resource_global
* buffer
= NULL
;
919 assert(transfer
->resource
->target
== PIPE_BUFFER
);
920 assert(transfer
->resource
->bind
& PIPE_BIND_GLOBAL
);
922 ctx
= (struct r600_context
*)ctx_
;
923 buffer
= (struct r600_resource_global
*)transfer
->resource
;
925 COMPUTE_DBG(ctx
->screen
, "* r600_compute_global_transfer_unmap()\n");
927 ctx
->ws
->buffer_unmap(buffer
->chunk
->pool
->bo
->cs_buf
);
928 util_slab_free(&ctx
->pool_transfers
, transfer
);
931 void r600_compute_global_transfer_flush_region(
932 struct pipe_context
*ctx_
,
933 struct pipe_transfer
*transfer
,
934 const struct pipe_box
*box
)
939 void r600_compute_global_transfer_inline_write(
940 struct pipe_context
*pipe
,
941 struct pipe_resource
*resource
,
944 const struct pipe_box
*box
,
947 unsigned layer_stride
)