2 * Copyright 2011 Adam Rak <adam.rak@streamnovation.com>
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * on the rights to use, copy, modify, merge, publish, distribute, sub
8 * license, and/or sell copies of the Software, and to permit persons to whom
9 * the Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
19 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
20 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE.
24 * Adam Rak <adam.rak@streamnovation.com>
29 #include "pipe/p_defines.h"
30 #include "pipe/p_state.h"
31 #include "pipe/p_context.h"
32 #include "util/u_blitter.h"
33 #include "util/u_double_list.h"
34 #include "util/u_transfer.h"
35 #include "util/u_surface.h"
36 #include "util/u_pack_color.h"
37 #include "util/u_memory.h"
38 #include "util/u_inlines.h"
39 #include "util/u_framebuffer.h"
40 #include "pipebuffer/pb_buffer.h"
41 #include "evergreend.h"
42 #include "r600_resource.h"
43 #include "r600_shader.h"
44 #include "r600_pipe.h"
45 #include "r600_formats.h"
46 #include "evergreen_compute.h"
47 #include "evergreen_compute_internal.h"
48 #include "compute_memory_pool.h"
50 #include "radeon_llvm_util.h"
54 RAT0 is for global binding write
55 VTX1 is for global binding read
57 for wrting images RAT1...
58 for reading images TEX2...
61 TEX2... consumes the same fetch resources, that VTX2... would consume
63 CONST0 and VTX0 is for parameters
64 CONST0 is binding smaller input parameter buffer, and for constant indexing,
66 VTX0 is for indirect/non-constant indexing, or if the input is bigger than
67 the constant cache can handle
69 RAT-s are limited to 12, so we can only bind at most 11 texture for writing
70 because we reserve RAT0 for global bindings. With byteaddressing enabled,
71 we should reserve another one too.=> 10 image binding for writing max.
74 CL_DEVICE_MAX_READ_IMAGE_ARGS: 128
75 CL_DEVICE_MAX_WRITE_IMAGE_ARGS: 8
77 so 10 for writing is enough. 176 is the max for reading according to the docs
79 writable images should be listed first < 10, so their id corresponds to RAT(id+1)
80 writable images will consume TEX slots, VTX slots too because of linear indexing
84 static void evergreen_cs_set_vertex_buffer(
85 struct r600_context
* rctx
,
88 struct pipe_resource
* buffer
)
90 struct r600_vertexbuf_state
*state
= &rctx
->cs_vertex_buffer_state
;
91 struct pipe_vertex_buffer
*vb
= &state
->vb
[vb_index
];
93 vb
->buffer_offset
= offset
;
95 vb
->user_buffer
= NULL
;
97 /* The vertex instructions in the compute shaders use the texture cache,
98 * so we need to invalidate it. */
99 rctx
->flags
|= R600_CONTEXT_INVAL_READ_CACHES
;
100 state
->enabled_mask
|= 1 << vb_index
;
101 state
->dirty_mask
|= 1 << vb_index
;
102 state
->atom
.dirty
= true;
105 static void evergreen_cs_set_constant_buffer(
106 struct r600_context
* rctx
,
110 struct pipe_resource
* buffer
)
112 struct pipe_constant_buffer cb
;
113 cb
.buffer_size
= size
;
114 cb
.buffer_offset
= offset
;
116 cb
.user_buffer
= NULL
;
118 rctx
->context
.set_constant_buffer(&rctx
->context
, PIPE_SHADER_COMPUTE
, cb_index
, &cb
);
121 static const struct u_resource_vtbl r600_global_buffer_vtbl
=
123 u_default_resource_get_handle
, /* get_handle */
124 r600_compute_global_buffer_destroy
, /* resource_destroy */
125 r600_compute_global_transfer_map
, /* transfer_map */
126 r600_compute_global_transfer_flush_region
,/* transfer_flush_region */
127 r600_compute_global_transfer_unmap
, /* transfer_unmap */
128 r600_compute_global_transfer_inline_write
/* transfer_inline_write */
132 void *evergreen_create_compute_state(
133 struct pipe_context
*ctx_
,
134 const const struct pipe_compute_state
*cso
)
136 struct r600_context
*ctx
= (struct r600_context
*)ctx_
;
137 struct r600_pipe_compute
*shader
= CALLOC_STRUCT(r600_pipe_compute
);
140 const struct pipe_llvm_program_header
* header
;
141 const unsigned char * code
;
144 COMPUTE_DBG(ctx
->screen
, "*** evergreen_create_compute_state\n");
147 code
= cso
->prog
+ sizeof(struct pipe_llvm_program_header
);
150 shader
->ctx
= (struct r600_context
*)ctx
;
151 shader
->resources
= (struct evergreen_compute_resource
*)
152 CALLOC(sizeof(struct evergreen_compute_resource
),
153 get_compute_resource_num());
154 shader
->local_size
= cso
->req_local_mem
; ///TODO: assert it
155 shader
->private_size
= cso
->req_private_mem
;
156 shader
->input_size
= cso
->req_input_mem
;
159 shader
->num_kernels
= radeon_llvm_get_num_kernels(code
, header
->num_bytes
);
160 shader
->kernels
= CALLOC(sizeof(struct r600_kernel
), shader
->num_kernels
);
162 for (i
= 0; i
< shader
->num_kernels
; i
++) {
163 struct r600_kernel
*kernel
= &shader
->kernels
[i
];
164 kernel
->llvm_module
= radeon_llvm_get_kernel_module(i
, code
,
171 void evergreen_delete_compute_state(struct pipe_context
*ctx
, void* state
)
173 struct r600_pipe_compute
*shader
= (struct r600_pipe_compute
*)state
;
175 free(shader
->resources
);
179 static void evergreen_bind_compute_state(struct pipe_context
*ctx_
, void *state
)
181 struct r600_context
*ctx
= (struct r600_context
*)ctx_
;
183 COMPUTE_DBG(ctx
->screen
, "*** evergreen_bind_compute_state\n");
185 ctx
->cs_shader_state
.shader
= (struct r600_pipe_compute
*)state
;
188 /* The kernel parameters are stored a vtx buffer (ID=0), besides the explicit
189 * kernel parameters there are inplicit parameters that need to be stored
190 * in the vertex buffer as well. Here is how these parameters are organized in
193 * DWORDS 0-2: Number of work groups in each dimension (x,y,z)
194 * DWORDS 3-5: Number of global work items in each dimension (x,y,z)
195 * DWORDS 6-8: Number of work items within each work group in each dimension
197 * DWORDS 9+ : Kernel parameters
199 void evergreen_compute_upload_input(
200 struct pipe_context
*ctx_
,
201 const uint
*block_layout
,
202 const uint
*grid_layout
,
205 struct r600_context
*ctx
= (struct r600_context
*)ctx_
;
206 struct r600_pipe_compute
*shader
= ctx
->cs_shader_state
.shader
;
208 /* We need to reserve 9 dwords (36 bytes) for implicit kernel
211 unsigned input_size
= shader
->input_size
+ 36;
212 uint32_t * num_work_groups_start
;
213 uint32_t * global_size_start
;
214 uint32_t * local_size_start
;
215 uint32_t * kernel_parameters_start
;
217 if (shader
->input_size
== 0) {
221 if (!shader
->kernel_param
) {
222 /* Add space for the grid dimensions */
223 shader
->kernel_param
= r600_compute_buffer_alloc_vram(
224 ctx
->screen
, input_size
);
227 num_work_groups_start
= r600_buffer_mmap_sync_with_rings(ctx
, shader
->kernel_param
, PIPE_TRANSFER_WRITE
);
228 global_size_start
= num_work_groups_start
+ (3 * (sizeof(uint
) /4));
229 local_size_start
= global_size_start
+ (3 * (sizeof(uint
)) / 4);
230 kernel_parameters_start
= local_size_start
+ (3 * (sizeof(uint
)) / 4);
232 /* Copy the work group size */
233 memcpy(num_work_groups_start
, grid_layout
, 3 * sizeof(uint
));
235 /* Copy the global size */
236 for (i
= 0; i
< 3; i
++) {
237 global_size_start
[i
] = grid_layout
[i
] * block_layout
[i
];
240 /* Copy the local dimensions */
241 memcpy(local_size_start
, block_layout
, 3 * sizeof(uint
));
243 /* Copy the kernel inputs */
244 memcpy(kernel_parameters_start
, input
, shader
->input_size
);
246 for (i
= 0; i
< (input_size
/ 4); i
++) {
247 COMPUTE_DBG(ctx
->screen
, "input %i : %i\n", i
,
248 ((unsigned*)num_work_groups_start
)[i
]);
251 ctx
->ws
->buffer_unmap(shader
->kernel_param
->cs_buf
);
253 /* ID=0 is reserved for the parameters */
254 evergreen_cs_set_constant_buffer(ctx
, 0, 0, input_size
,
255 (struct pipe_resource
*)shader
->kernel_param
);
258 static void evergreen_emit_direct_dispatch(
259 struct r600_context
*rctx
,
260 const uint
*block_layout
, const uint
*grid_layout
)
263 struct radeon_winsys_cs
*cs
= rctx
->rings
.gfx
.cs
;
265 unsigned num_pipes
= rctx
->screen
->info
.r600_max_pipes
;
266 unsigned wave_divisor
= (16 * num_pipes
);
269 /* XXX: Enable lds and get size from cs_shader_state */
270 unsigned lds_size
= 0;
272 /* Calculate group_size/grid_size */
273 for (i
= 0; i
< 3; i
++) {
274 group_size
*= block_layout
[i
];
277 for (i
= 0; i
< 3; i
++) {
278 grid_size
*= grid_layout
[i
];
281 /* num_waves = ceil((tg_size.x * tg_size.y, tg_size.z) / (16 * num_pipes)) */
282 num_waves
= (block_layout
[0] * block_layout
[1] * block_layout
[2] +
283 wave_divisor
- 1) / wave_divisor
;
285 COMPUTE_DBG(rctx
->screen
, "Using %u pipes, there are %u wavefronts per thread block\n",
286 num_pipes
, num_waves
);
288 /* XXX: Partition the LDS between PS/CS. By default half (4096 dwords
289 * on Evergreen) oes to Pixel Shaders and half goes to Compute Shaders.
290 * We may need to allocat the entire LDS space for Compute Shaders.
292 * EG: R_008E2C_SQ_LDS_RESOURCE_MGMT := S_008E2C_NUM_LS_LDS(lds_dwords)
293 * CM: CM_R_0286FC_SPI_LDS_MGMT := S_0286FC_NUM_LS_LDS(lds_dwords)
296 r600_write_config_reg(cs
, R_008970_VGT_NUM_INDICES
, group_size
);
298 r600_write_config_reg_seq(cs
, R_00899C_VGT_COMPUTE_START_X
, 3);
299 r600_write_value(cs
, 0); /* R_00899C_VGT_COMPUTE_START_X */
300 r600_write_value(cs
, 0); /* R_0089A0_VGT_COMPUTE_START_Y */
301 r600_write_value(cs
, 0); /* R_0089A4_VGT_COMPUTE_START_Z */
303 r600_write_config_reg(cs
, R_0089AC_VGT_COMPUTE_THREAD_GROUP_SIZE
,
306 r600_write_compute_context_reg_seq(cs
, R_0286EC_SPI_COMPUTE_NUM_THREAD_X
, 3);
307 r600_write_value(cs
, block_layout
[0]); /* R_0286EC_SPI_COMPUTE_NUM_THREAD_X */
308 r600_write_value(cs
, block_layout
[1]); /* R_0286F0_SPI_COMPUTE_NUM_THREAD_Y */
309 r600_write_value(cs
, block_layout
[2]); /* R_0286F4_SPI_COMPUTE_NUM_THREAD_Z */
311 r600_write_compute_context_reg(cs
, CM_R_0288E8_SQ_LDS_ALLOC
,
312 lds_size
| (num_waves
<< 14));
314 /* Dispatch packet */
315 r600_write_value(cs
, PKT3C(PKT3_DISPATCH_DIRECT
, 3, 0));
316 r600_write_value(cs
, grid_layout
[0]);
317 r600_write_value(cs
, grid_layout
[1]);
318 r600_write_value(cs
, grid_layout
[2]);
319 /* VGT_DISPATCH_INITIATOR = COMPUTE_SHADER_EN */
320 r600_write_value(cs
, 1);
323 static void compute_emit_cs(struct r600_context
*ctx
, const uint
*block_layout
,
324 const uint
*grid_layout
)
326 struct radeon_winsys_cs
*cs
= ctx
->rings
.gfx
.cs
;
327 unsigned flush_flags
= 0;
329 struct evergreen_compute_resource
*resources
=
330 ctx
->cs_shader_state
.shader
->resources
;
332 /* make sure that the gfx ring is only one active */
333 if (ctx
->rings
.dma
.cs
) {
334 ctx
->rings
.dma
.flush(ctx
, RADEON_FLUSH_ASYNC
);
337 /* Initialize all the compute-related registers.
339 * See evergreen_init_atom_start_compute_cs() in this file for the list
340 * of registers initialized by the start_compute_cs_cmd atom.
342 r600_emit_command_buffer(cs
, &ctx
->start_compute_cs_cmd
);
344 ctx
->flags
|= R600_CONTEXT_WAIT_3D_IDLE
| R600_CONTEXT_FLUSH_AND_INV
;
345 r600_flush_emit(ctx
);
347 /* Emit colorbuffers. */
348 for (i
= 0; i
< ctx
->framebuffer
.state
.nr_cbufs
; i
++) {
349 struct r600_surface
*cb
= (struct r600_surface
*)ctx
->framebuffer
.state
.cbufs
[i
];
350 unsigned reloc
= r600_context_bo_reloc(ctx
, &ctx
->rings
.gfx
,
351 (struct r600_resource
*)cb
->base
.texture
,
352 RADEON_USAGE_READWRITE
);
354 r600_write_compute_context_reg_seq(cs
, R_028C60_CB_COLOR0_BASE
+ i
* 0x3C, 7);
355 r600_write_value(cs
, cb
->cb_color_base
); /* R_028C60_CB_COLOR0_BASE */
356 r600_write_value(cs
, cb
->cb_color_pitch
); /* R_028C64_CB_COLOR0_PITCH */
357 r600_write_value(cs
, cb
->cb_color_slice
); /* R_028C68_CB_COLOR0_SLICE */
358 r600_write_value(cs
, cb
->cb_color_view
); /* R_028C6C_CB_COLOR0_VIEW */
359 r600_write_value(cs
, cb
->cb_color_info
); /* R_028C70_CB_COLOR0_INFO */
360 r600_write_value(cs
, cb
->cb_color_attrib
); /* R_028C74_CB_COLOR0_ATTRIB */
361 r600_write_value(cs
, cb
->cb_color_dim
); /* R_028C78_CB_COLOR0_DIM */
363 r600_write_value(cs
, PKT3(PKT3_NOP
, 0, 0)); /* R_028C60_CB_COLOR0_BASE */
364 r600_write_value(cs
, reloc
);
366 if (!ctx
->keep_tiling_flags
) {
367 r600_write_value(cs
, PKT3(PKT3_NOP
, 0, 0)); /* R_028C70_CB_COLOR0_INFO */
368 r600_write_value(cs
, reloc
);
371 r600_write_value(cs
, PKT3(PKT3_NOP
, 0, 0)); /* R_028C74_CB_COLOR0_ATTRIB */
372 r600_write_value(cs
, reloc
);
375 /* Set CB_TARGET_MASK XXX: Use cb_misc_state */
376 r600_write_compute_context_reg(cs
, R_028238_CB_TARGET_MASK
,
377 ctx
->compute_cb_target_mask
);
380 /* Emit vertex buffer state */
381 ctx
->cs_vertex_buffer_state
.atom
.num_dw
= 12 * util_bitcount(ctx
->cs_vertex_buffer_state
.dirty_mask
);
382 r600_emit_atom(ctx
, &ctx
->cs_vertex_buffer_state
.atom
);
384 /* Emit constant buffer state */
385 r600_emit_atom(ctx
, &ctx
->constbuf_state
[PIPE_SHADER_COMPUTE
].atom
);
387 /* Emit compute shader state */
388 r600_emit_atom(ctx
, &ctx
->cs_shader_state
.atom
);
390 for (i
= 0; i
< get_compute_resource_num(); i
++) {
391 if (resources
[i
].enabled
) {
393 COMPUTE_DBG(ctx
->screen
, "resnum: %i, cdw: %i\n", i
, cs
->cdw
);
395 for (j
= 0; j
< resources
[i
].cs_end
; j
++) {
396 if (resources
[i
].do_reloc
[j
]) {
397 assert(resources
[i
].bo
);
398 evergreen_emit_ctx_reloc(ctx
,
403 cs
->buf
[cs
->cdw
++] = resources
[i
].cs
[j
];
406 if (resources
[i
].bo
) {
407 evergreen_emit_ctx_reloc(ctx
,
411 ///special case for textures
412 if (resources
[i
].do_reloc
413 [resources
[i
].cs_end
] == 2) {
414 evergreen_emit_ctx_reloc(ctx
,
422 /* Emit dispatch state and dispatch packet */
423 evergreen_emit_direct_dispatch(ctx
, block_layout
, grid_layout
);
425 /* XXX evergreen_flush_emit() hardcodes the CP_COHER_SIZE to 0xffffffff
427 ctx
->flags
|= R600_CONTEXT_INVAL_READ_CACHES
;
428 r600_flush_emit(ctx
);
431 COMPUTE_DBG(ctx
->screen
, "cdw: %i\n", cs
->cdw
);
432 for (i
= 0; i
< cs
->cdw
; i
++) {
433 COMPUTE_DBG(ctx
->screen
, "%4i : 0x%08X\n", i
, ctx
->cs
->buf
[i
]);
437 flush_flags
= RADEON_FLUSH_ASYNC
| RADEON_FLUSH_COMPUTE
;
438 if (ctx
->keep_tiling_flags
) {
439 flush_flags
|= RADEON_FLUSH_KEEP_TILING_FLAGS
;
442 ctx
->ws
->cs_flush(ctx
->rings
.gfx
.cs
, flush_flags
);
446 COMPUTE_DBG(ctx
->screen
, "shader started\n");
451 * Emit function for r600_cs_shader_state atom
453 void evergreen_emit_cs_shader(
454 struct r600_context
*rctx
,
455 struct r600_atom
*atom
)
457 struct r600_cs_shader_state
*state
=
458 (struct r600_cs_shader_state
*)atom
;
459 struct r600_pipe_compute
*shader
= state
->shader
;
460 struct r600_kernel
*kernel
= &shader
->kernels
[state
->kernel_index
];
461 struct radeon_winsys_cs
*cs
= rctx
->rings
.gfx
.cs
;
464 va
= r600_resource_va(&rctx
->screen
->screen
, &kernel
->code_bo
->b
.b
);
466 r600_write_compute_context_reg_seq(cs
, R_0288D0_SQ_PGM_START_LS
, 3);
467 r600_write_value(cs
, va
>> 8); /* R_0288D0_SQ_PGM_START_LS */
468 r600_write_value(cs
, /* R_0288D4_SQ_PGM_RESOURCES_LS */
469 S_0288D4_NUM_GPRS(kernel
->bc
.ngpr
)
470 | S_0288D4_STACK_SIZE(kernel
->bc
.nstack
));
471 r600_write_value(cs
, 0); /* R_0288D8_SQ_PGM_RESOURCES_LS_2 */
473 r600_write_value(cs
, PKT3C(PKT3_NOP
, 0, 0));
474 r600_write_value(cs
, r600_context_bo_reloc(rctx
, &rctx
->rings
.gfx
,
475 kernel
->code_bo
, RADEON_USAGE_READ
));
477 rctx
->flags
|= R600_CONTEXT_INVAL_READ_CACHES
;
480 static void evergreen_launch_grid(
481 struct pipe_context
*ctx_
,
482 const uint
*block_layout
, const uint
*grid_layout
,
483 uint32_t pc
, const void *input
)
485 struct r600_context
*ctx
= (struct r600_context
*)ctx_
;
488 COMPUTE_DBG(ctx
->screen
, "*** evergreen_launch_grid: pc = %u\n", pc
);
490 struct r600_pipe_compute
*shader
= ctx
->cs_shader_state
.shader
;
491 if (!shader
->kernels
[pc
].code_bo
) {
493 struct r600_kernel
*kernel
= &shader
->kernels
[pc
];
494 r600_compute_shader_create(ctx_
, kernel
->llvm_module
, &kernel
->bc
);
495 kernel
->code_bo
= r600_compute_buffer_alloc_vram(ctx
->screen
,
497 p
= r600_buffer_mmap_sync_with_rings(ctx
, kernel
->code_bo
, PIPE_TRANSFER_WRITE
);
498 memcpy(p
, kernel
->bc
.bytecode
, kernel
->bc
.ndw
* 4);
499 ctx
->ws
->buffer_unmap(kernel
->code_bo
->cs_buf
);
503 ctx
->cs_shader_state
.kernel_index
= pc
;
504 evergreen_compute_upload_input(ctx_
, block_layout
, grid_layout
, input
);
505 compute_emit_cs(ctx
, block_layout
, grid_layout
);
508 static void evergreen_set_compute_resources(struct pipe_context
* ctx_
,
509 unsigned start
, unsigned count
,
510 struct pipe_surface
** surfaces
)
512 struct r600_context
*ctx
= (struct r600_context
*)ctx_
;
513 struct r600_surface
**resources
= (struct r600_surface
**)surfaces
;
515 COMPUTE_DBG(ctx
->screen
, "*** evergreen_set_compute_resources: start = %u count = %u\n",
518 for (int i
= 0; i
< count
; i
++) {
519 /* The First two vertex buffers are reserved for parameters and
521 unsigned vtx_id
= 2 + i
;
523 struct r600_resource_global
*buffer
=
524 (struct r600_resource_global
*)
525 resources
[i
]->base
.texture
;
526 if (resources
[i
]->base
.writable
) {
529 evergreen_set_rat(ctx
->cs_shader_state
.shader
, i
+1,
530 (struct r600_resource
*)resources
[i
]->base
.texture
,
531 buffer
->chunk
->start_in_dw
*4,
532 resources
[i
]->base
.texture
->width0
);
535 evergreen_cs_set_vertex_buffer(ctx
, vtx_id
,
536 buffer
->chunk
->start_in_dw
* 4,
537 resources
[i
]->base
.texture
);
542 static void evergreen_set_cs_sampler_view(struct pipe_context
*ctx_
,
543 unsigned start_slot
, unsigned count
,
544 struct pipe_sampler_view
**views
)
546 struct r600_context
*ctx
= (struct r600_context
*)ctx_
;
547 struct r600_pipe_sampler_view
**resource
=
548 (struct r600_pipe_sampler_view
**)views
;
550 for (int i
= 0; i
< count
; i
++) {
553 ///FETCH0 = VTX0 (param buffer),
554 //FETCH1 = VTX1 (global buffer pool), FETCH2... = TEX
555 evergreen_set_tex_resource(ctx
->cs_shader_state
.shader
, resource
[i
], i
+2);
560 static void evergreen_bind_compute_sampler_states(
561 struct pipe_context
*ctx_
,
563 unsigned num_samplers
,
566 struct r600_context
*ctx
= (struct r600_context
*)ctx_
;
567 struct compute_sampler_state
** samplers
=
568 (struct compute_sampler_state
**)samplers_
;
570 for (int i
= 0; i
< num_samplers
; i
++) {
572 evergreen_set_sampler_resource(
573 ctx
->cs_shader_state
.shader
, samplers
[i
], i
);
578 static void evergreen_set_global_binding(
579 struct pipe_context
*ctx_
, unsigned first
, unsigned n
,
580 struct pipe_resource
**resources
,
583 struct r600_context
*ctx
= (struct r600_context
*)ctx_
;
584 struct compute_memory_pool
*pool
= ctx
->screen
->global_pool
;
585 struct r600_resource_global
**buffers
=
586 (struct r600_resource_global
**)resources
;
588 COMPUTE_DBG(ctx
->screen
, "*** evergreen_set_global_binding first = %u n = %u\n",
596 compute_memory_finalize_pending(pool
, ctx_
);
598 for (int i
= 0; i
< n
; i
++)
600 assert(resources
[i
]->target
== PIPE_BUFFER
);
601 assert(resources
[i
]->bind
& PIPE_BIND_GLOBAL
);
603 *(handles
[i
]) = buffers
[i
]->chunk
->start_in_dw
* 4;
606 evergreen_set_rat(ctx
->cs_shader_state
.shader
, 0, pool
->bo
, 0, pool
->size_in_dw
* 4);
607 evergreen_cs_set_vertex_buffer(ctx
, 1, 0,
608 (struct pipe_resource
*)pool
->bo
);
612 * This function initializes all the compute specific registers that need to
613 * be initialized for each compute command stream. Registers that are common
614 * to both compute and 3D will be initialized at the beginning of each compute
615 * command stream by the start_cs_cmd atom. However, since the SET_CONTEXT_REG
616 * packet requires that the shader type bit be set, we must initialize all
617 * context registers needed for compute in this function. The registers
618 * intialized by the start_cs_cmd atom can be found in evereen_state.c in the
619 * functions evergreen_init_atom_start_cs or cayman_init_atom_start_cs depending
622 void evergreen_init_atom_start_compute_cs(struct r600_context
*ctx
)
624 struct r600_command_buffer
*cb
= &ctx
->start_compute_cs_cmd
;
626 int num_stack_entries
;
628 /* since all required registers are initialised in the
629 * start_compute_cs_cmd atom, we can EMIT_EARLY here.
631 r600_init_command_buffer(cb
, 256);
632 cb
->pkt_flags
= RADEON_CP_PACKET3_COMPUTE_MODE
;
634 /* This must be first. */
635 r600_store_value(cb
, PKT3(PKT3_CONTEXT_CONTROL
, 1, 0));
636 r600_store_value(cb
, 0x80000000);
637 r600_store_value(cb
, 0x80000000);
639 /* We're setting config registers here. */
640 r600_store_value(cb
, PKT3(PKT3_EVENT_WRITE
, 0, 0));
641 r600_store_value(cb
, EVENT_TYPE(EVENT_TYPE_CS_PARTIAL_FLUSH
) | EVENT_INDEX(4));
643 switch (ctx
->family
) {
647 num_stack_entries
= 256;
651 num_stack_entries
= 256;
655 num_stack_entries
= 512;
660 num_stack_entries
= 512;
664 num_stack_entries
= 256;
668 num_stack_entries
= 256;
672 num_stack_entries
= 512;
676 num_stack_entries
= 512;
680 num_stack_entries
= 256;
684 num_stack_entries
= 256;
688 /* Config Registers */
689 if (ctx
->chip_class
< CAYMAN
)
690 evergreen_init_common_regs(cb
, ctx
->chip_class
, ctx
->family
,
691 ctx
->screen
->info
.drm_minor
);
693 cayman_init_common_regs(cb
, ctx
->chip_class
, ctx
->family
,
694 ctx
->screen
->info
.drm_minor
);
696 /* The primitive type always needs to be POINTLIST for compute. */
697 r600_store_config_reg(cb
, R_008958_VGT_PRIMITIVE_TYPE
,
698 V_008958_DI_PT_POINTLIST
);
700 if (ctx
->chip_class
< CAYMAN
) {
702 /* These registers control which simds can be used by each stage.
703 * The default for these registers is 0xffffffff, which means
704 * all simds are available for each stage. It's possible we may
705 * want to play around with these in the future, but for now
706 * the default value is fine.
708 * R_008E20_SQ_STATIC_THREAD_MGMT1
709 * R_008E24_SQ_STATIC_THREAD_MGMT2
710 * R_008E28_SQ_STATIC_THREAD_MGMT3
713 /* XXX: We may need to adjust the thread and stack resouce
714 * values for 3D/compute interop */
716 r600_store_config_reg_seq(cb
, R_008C18_SQ_THREAD_RESOURCE_MGMT_1
, 5);
718 /* R_008C18_SQ_THREAD_RESOURCE_MGMT_1
719 * Set the number of threads used by the PS/VS/GS/ES stage to
722 r600_store_value(cb
, 0);
724 /* R_008C1C_SQ_THREAD_RESOURCE_MGMT_2
725 * Set the number of threads used by the CS (aka LS) stage to
726 * the maximum number of threads and set the number of threads
727 * for the HS stage to 0. */
728 r600_store_value(cb
, S_008C1C_NUM_LS_THREADS(num_threads
));
730 /* R_008C20_SQ_STACK_RESOURCE_MGMT_1
731 * Set the Control Flow stack entries to 0 for PS/VS stages */
732 r600_store_value(cb
, 0);
734 /* R_008C24_SQ_STACK_RESOURCE_MGMT_2
735 * Set the Control Flow stack entries to 0 for GS/ES stages */
736 r600_store_value(cb
, 0);
738 /* R_008C28_SQ_STACK_RESOURCE_MGMT_3
739 * Set the Contol Flow stack entries to 0 for the HS stage, and
740 * set it to the maximum value for the CS (aka LS) stage. */
742 S_008C28_NUM_LS_STACK_ENTRIES(num_stack_entries
));
745 /* Context Registers */
747 if (ctx
->chip_class
< CAYMAN
) {
748 /* workaround for hw issues with dyn gpr - must set all limits
749 * to 240 instead of 0, 0x1e == 240 / 8
751 r600_store_context_reg(cb
, R_028838_SQ_DYN_GPR_RESOURCE_LIMIT_1
,
752 S_028838_PS_GPRS(0x1e) |
753 S_028838_VS_GPRS(0x1e) |
754 S_028838_GS_GPRS(0x1e) |
755 S_028838_ES_GPRS(0x1e) |
756 S_028838_HS_GPRS(0x1e) |
757 S_028838_LS_GPRS(0x1e));
760 /* XXX: Investigate setting bit 15, which is FAST_COMPUTE_MODE */
761 r600_store_context_reg(cb
, R_028A40_VGT_GS_MODE
,
762 S_028A40_COMPUTE_MODE(1) | S_028A40_PARTIAL_THD_AT_EOI(1));
764 r600_store_context_reg(cb
, R_028B54_VGT_SHADER_STAGES_EN
, 2/*CS_ON*/);
766 r600_store_context_reg(cb
, R_0286E8_SPI_COMPUTE_INPUT_CNTL
,
767 S_0286E8_TID_IN_GROUP_ENA
769 | S_0286E8_DISABLE_INDEX_PACK
)
772 /* The LOOP_CONST registers are an optimizations for loops that allows
773 * you to store the initial counter, increment value, and maximum
774 * counter value in a register so that hardware can calculate the
775 * correct number of iterations for the loop, so that you don't need
776 * to have the loop counter in your shader code. We don't currently use
777 * this optimization, so we must keep track of the counter in the
778 * shader and use a break instruction to exit loops. However, the
779 * hardware will still uses this register to determine when to exit a
780 * loop, so we need to initialize the counter to 0, set the increment
781 * value to 1 and the maximum counter value to the 4095 (0xfff) which
782 * is the maximum value allowed. This gives us a maximum of 4096
783 * iterations for our loops, but hopefully our break instruction will
784 * execute before some time before the 4096th iteration.
786 eg_store_loop_const(cb
, R_03A200_SQ_LOOP_CONST_0
+ (160 * 4), 0x1000FFF);
789 void evergreen_init_compute_state_functions(struct r600_context
*ctx
)
791 ctx
->context
.create_compute_state
= evergreen_create_compute_state
;
792 ctx
->context
.delete_compute_state
= evergreen_delete_compute_state
;
793 ctx
->context
.bind_compute_state
= evergreen_bind_compute_state
;
794 // ctx->context.create_sampler_view = evergreen_compute_create_sampler_view;
795 ctx
->context
.set_compute_resources
= evergreen_set_compute_resources
;
796 ctx
->context
.set_compute_sampler_views
= evergreen_set_cs_sampler_view
;
797 ctx
->context
.bind_compute_sampler_states
= evergreen_bind_compute_sampler_states
;
798 ctx
->context
.set_global_binding
= evergreen_set_global_binding
;
799 ctx
->context
.launch_grid
= evergreen_launch_grid
;
801 /* We always use at least one vertex buffer for parameters (id = 1)*/
802 ctx
->cs_vertex_buffer_state
.enabled_mask
=
803 ctx
->cs_vertex_buffer_state
.dirty_mask
= 0x2;
807 struct pipe_resource
*r600_compute_global_buffer_create(
808 struct pipe_screen
*screen
,
809 const struct pipe_resource
*templ
)
811 struct r600_resource_global
* result
= NULL
;
812 struct r600_screen
* rscreen
= NULL
;
815 assert(templ
->target
== PIPE_BUFFER
);
816 assert(templ
->bind
& PIPE_BIND_GLOBAL
);
817 assert(templ
->array_size
== 1 || templ
->array_size
== 0);
818 assert(templ
->depth0
== 1 || templ
->depth0
== 0);
819 assert(templ
->height0
== 1 || templ
->height0
== 0);
821 result
= (struct r600_resource_global
*)
822 CALLOC(sizeof(struct r600_resource_global
), 1);
823 rscreen
= (struct r600_screen
*)screen
;
825 COMPUTE_DBG(rscreen
, "*** r600_compute_global_buffer_create\n");
826 COMPUTE_DBG(rscreen
, "width = %u array_size = %u\n", templ
->width0
,
829 result
->base
.b
.vtbl
= &r600_global_buffer_vtbl
;
830 result
->base
.b
.b
.screen
= screen
;
831 result
->base
.b
.b
= *templ
;
832 pipe_reference_init(&result
->base
.b
.b
.reference
, 1);
834 size_in_dw
= (templ
->width0
+3) / 4;
836 result
->chunk
= compute_memory_alloc(rscreen
->global_pool
, size_in_dw
);
838 if (result
->chunk
== NULL
)
844 return &result
->base
.b
.b
;
847 void r600_compute_global_buffer_destroy(
848 struct pipe_screen
*screen
,
849 struct pipe_resource
*res
)
851 struct r600_resource_global
* buffer
= NULL
;
852 struct r600_screen
* rscreen
= NULL
;
854 assert(res
->target
== PIPE_BUFFER
);
855 assert(res
->bind
& PIPE_BIND_GLOBAL
);
857 buffer
= (struct r600_resource_global
*)res
;
858 rscreen
= (struct r600_screen
*)screen
;
860 compute_memory_free(rscreen
->global_pool
, buffer
->chunk
->id
);
862 buffer
->chunk
= NULL
;
866 void *r600_compute_global_transfer_map(
867 struct pipe_context
*ctx_
,
868 struct pipe_resource
*resource
,
871 const struct pipe_box
*box
,
872 struct pipe_transfer
**ptransfer
)
874 struct r600_context
*rctx
= (struct r600_context
*)ctx_
;
875 struct compute_memory_pool
*pool
= rctx
->screen
->global_pool
;
876 struct pipe_transfer
*transfer
= util_slab_alloc(&rctx
->pool_transfers
);
877 struct r600_resource_global
* buffer
=
878 (struct r600_resource_global
*)resource
;
881 compute_memory_finalize_pending(pool
, ctx_
);
883 assert(resource
->target
== PIPE_BUFFER
);
885 COMPUTE_DBG(rctx
->screen
, "* r600_compute_global_get_transfer()\n"
886 "level = %u, usage = %u, box(x = %u, y = %u, z = %u "
887 "width = %u, height = %u, depth = %u)\n", level
, usage
,
888 box
->x
, box
->y
, box
->z
, box
->width
, box
->height
,
891 transfer
->resource
= resource
;
892 transfer
->level
= level
;
893 transfer
->usage
= usage
;
894 transfer
->box
= *box
;
895 transfer
->stride
= 0;
896 transfer
->layer_stride
= 0;
898 assert(transfer
->resource
->target
== PIPE_BUFFER
);
899 assert(transfer
->resource
->bind
& PIPE_BIND_GLOBAL
);
900 assert(transfer
->box
.x
>= 0);
901 assert(transfer
->box
.y
== 0);
902 assert(transfer
->box
.z
== 0);
904 ///TODO: do it better, mapping is not possible if the pool is too big
906 COMPUTE_DBG(rctx
->screen
, "* r600_compute_global_transfer_map()\n");
908 if (!(map
= r600_buffer_mmap_sync_with_rings(rctx
, buffer
->chunk
->pool
->bo
, transfer
->usage
))) {
909 util_slab_free(&rctx
->pool_transfers
, transfer
);
913 *ptransfer
= transfer
;
915 COMPUTE_DBG(rctx
->screen
, "Buffer: %p + %u (buffer offset in global memory) "
916 "+ %u (box.x)\n", map
, buffer
->chunk
->start_in_dw
, transfer
->box
.x
);
917 return ((char*)(map
+ buffer
->chunk
->start_in_dw
)) + transfer
->box
.x
;
920 void r600_compute_global_transfer_unmap(
921 struct pipe_context
*ctx_
,
922 struct pipe_transfer
* transfer
)
924 struct r600_context
*ctx
= NULL
;
925 struct r600_resource_global
* buffer
= NULL
;
927 assert(transfer
->resource
->target
== PIPE_BUFFER
);
928 assert(transfer
->resource
->bind
& PIPE_BIND_GLOBAL
);
930 ctx
= (struct r600_context
*)ctx_
;
931 buffer
= (struct r600_resource_global
*)transfer
->resource
;
933 COMPUTE_DBG(ctx
->screen
, "* r600_compute_global_transfer_unmap()\n");
935 ctx
->ws
->buffer_unmap(buffer
->chunk
->pool
->bo
->cs_buf
);
936 util_slab_free(&ctx
->pool_transfers
, transfer
);
939 void r600_compute_global_transfer_flush_region(
940 struct pipe_context
*ctx_
,
941 struct pipe_transfer
*transfer
,
942 const struct pipe_box
*box
)
947 void r600_compute_global_transfer_inline_write(
948 struct pipe_context
*pipe
,
949 struct pipe_resource
*resource
,
952 const struct pipe_box
*box
,
955 unsigned layer_stride
)