projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
llvmpipe: add initial nir support
[mesa.git] / src / gallium / drivers / llvmpipe / lp_state_cs.c
1 /**************************************************************************
2 *
3 * Copyright 2019 Red Hat.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included
14 * in all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
17 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 *
24 **************************************************************************/
25 #include "util/u_memory.h"
26 #include "util/simple_list.h"
27 #include "util/os_time.h"
28 #include "util/u_dump.h"
29 #include "util/u_string.h"
30 #include "tgsi/tgsi_dump.h"
31 #include "tgsi/tgsi_parse.h"
32 #include "gallivm/lp_bld_const.h"
33 #include "gallivm/lp_bld_debug.h"
34 #include "gallivm/lp_bld_intr.h"
35 #include "gallivm/lp_bld_flow.h"
36 #include "gallivm/lp_bld_gather.h"
37 #include "gallivm/lp_bld_coro.h"
38 #include "gallivm/lp_bld_nir.h"
39 #include "lp_state_cs.h"
40 #include "lp_context.h"
41 #include "lp_debug.h"
42 #include "lp_state.h"
43 #include "lp_perf.h"
44 #include "lp_screen.h"
45 #include "lp_memory.h"
46 #include "lp_cs_tpool.h"
47 #include "state_tracker/sw_winsys.h"
48 #include "nir/nir_to_tgsi_info.h"
49
50 struct lp_cs_job_info {
51 unsigned grid_size[3];
52 unsigned block_size[3];
53 unsigned req_local_mem;
54 struct lp_cs_exec *current;
55 };
56
57 static void
58 generate_compute(struct llvmpipe_context *lp,
59 struct lp_compute_shader *shader,
60 struct lp_compute_shader_variant *variant)
61 {
62 struct gallivm_state *gallivm = variant->gallivm;
63 const struct lp_compute_shader_variant_key *key = &variant->key;
64 char func_name[64], func_name_coro[64];
65 LLVMTypeRef arg_types[13];
66 LLVMTypeRef func_type, coro_func_type;
67 LLVMTypeRef int32_type = LLVMInt32TypeInContext(gallivm->context);
68 LLVMValueRef context_ptr;
69 LLVMValueRef x_size_arg, y_size_arg, z_size_arg;
70 LLVMValueRef grid_x_arg, grid_y_arg, grid_z_arg;
71 LLVMValueRef grid_size_x_arg, grid_size_y_arg, grid_size_z_arg;
72 LLVMValueRef thread_data_ptr;
73 LLVMBasicBlockRef block;
74 LLVMBuilderRef builder;
75 struct lp_build_sampler_soa *sampler;
76 struct lp_build_image_soa *image;
77 LLVMValueRef function, coro;
78 struct lp_type cs_type;
79 unsigned i;
80
81 /*
82 * This function has two parts
83 * a) setup the coroutine execution environment loop.
84 * b) build the compute shader llvm for use inside the coroutine.
85 */
86 assert(lp_native_vector_width / 32 >= 4);
87
88 memset(&cs_type, 0, sizeof cs_type);
89 cs_type.floating = TRUE; /* floating point values */
90 cs_type.sign = TRUE; /* values are signed */
91 cs_type.norm = FALSE; /* values are not limited to [0,1] or [-1,1] */
92 cs_type.width = 32; /* 32-bit float */
93 cs_type.length = MIN2(lp_native_vector_width / 32, 16); /* n*4 elements per vector */
94 snprintf(func_name, sizeof(func_name), "cs%u_variant%u",
95 shader->no, variant->no);
96
97 snprintf(func_name_coro, sizeof(func_name), "cs_co_%u_variant%u",
98 shader->no, variant->no);
99
100 arg_types[0] = variant->jit_cs_context_ptr_type; /* context */
101 arg_types[1] = int32_type; /* block_x_size */
102 arg_types[2] = int32_type; /* block_y_size */
103 arg_types[3] = int32_type; /* block_z_size */
104 arg_types[4] = int32_type; /* grid_x */
105 arg_types[5] = int32_type; /* grid_y */
106 arg_types[6] = int32_type; /* grid_z */
107 arg_types[7] = int32_type; /* grid_size_x */
108 arg_types[8] = int32_type; /* grid_size_y */
109 arg_types[9] = int32_type; /* grid_size_z */
110 arg_types[10] = variant->jit_cs_thread_data_ptr_type; /* per thread data */
111 arg_types[11] = int32_type;
112 arg_types[12] = int32_type;
113 func_type = LLVMFunctionType(LLVMVoidTypeInContext(gallivm->context),
114 arg_types, ARRAY_SIZE(arg_types) - 2, 0);
115
116 coro_func_type = LLVMFunctionType(LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0),
117 arg_types, ARRAY_SIZE(arg_types), 0);
118
119 function = LLVMAddFunction(gallivm->module, func_name, func_type);
120 LLVMSetFunctionCallConv(function, LLVMCCallConv);
121
122 coro = LLVMAddFunction(gallivm->module, func_name_coro, coro_func_type);
123 LLVMSetFunctionCallConv(coro, LLVMCCallConv);
124
125 variant->function = function;
126
127 for(i = 0; i < ARRAY_SIZE(arg_types); ++i) {
128 if(LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind) {
129 lp_add_function_attr(coro, i + 1, LP_FUNC_ATTR_NOALIAS);
130 lp_add_function_attr(function, i + 1, LP_FUNC_ATTR_NOALIAS);
131 }
132 }
133
134 context_ptr = LLVMGetParam(function, 0);
135 x_size_arg = LLVMGetParam(function, 1);
136 y_size_arg = LLVMGetParam(function, 2);
137 z_size_arg = LLVMGetParam(function, 3);
138 grid_x_arg = LLVMGetParam(function, 4);
139 grid_y_arg = LLVMGetParam(function, 5);
140 grid_z_arg = LLVMGetParam(function, 6);
141 grid_size_x_arg = LLVMGetParam(function, 7);
142 grid_size_y_arg = LLVMGetParam(function, 8);
143 grid_size_z_arg = LLVMGetParam(function, 9);
144 thread_data_ptr = LLVMGetParam(function, 10);
145
146 lp_build_name(context_ptr, "context");
147 lp_build_name(x_size_arg, "x_size");
148 lp_build_name(y_size_arg, "y_size");
149 lp_build_name(z_size_arg, "z_size");
150 lp_build_name(grid_x_arg, "grid_x");
151 lp_build_name(grid_y_arg, "grid_y");
152 lp_build_name(grid_z_arg, "grid_z");
153 lp_build_name(grid_size_x_arg, "grid_size_x");
154 lp_build_name(grid_size_y_arg, "grid_size_y");
155 lp_build_name(grid_size_z_arg, "grid_size_z");
156 lp_build_name(thread_data_ptr, "thread_data");
157
158 block = LLVMAppendBasicBlockInContext(gallivm->context, function, "entry");
159 builder = gallivm->builder;
160 assert(builder);
161 LLVMPositionBuilderAtEnd(builder, block);
162 sampler = lp_llvm_sampler_soa_create(key->state);
163 image = lp_llvm_image_soa_create(key->image_state);
164
165 struct lp_build_loop_state loop_state[4];
166 LLVMValueRef num_x_loop;
167 LLVMValueRef vec_length = lp_build_const_int32(gallivm, cs_type.length);
168 num_x_loop = LLVMBuildAdd(gallivm->builder, x_size_arg, vec_length, "");
169 num_x_loop = LLVMBuildSub(gallivm->builder, num_x_loop, lp_build_const_int32(gallivm, 1), "");
170 num_x_loop = LLVMBuildUDiv(gallivm->builder, num_x_loop, vec_length, "");
171 LLVMValueRef partials = LLVMBuildURem(gallivm->builder, x_size_arg, vec_length, "");
172
173 LLVMValueRef coro_num_hdls = LLVMBuildMul(gallivm->builder, num_x_loop, y_size_arg, "");
174 coro_num_hdls = LLVMBuildMul(gallivm->builder, coro_num_hdls, z_size_arg, "");
175
176 LLVMTypeRef hdl_ptr_type = LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0);
177 LLVMValueRef coro_hdls = LLVMBuildArrayAlloca(gallivm->builder, hdl_ptr_type, coro_num_hdls, "coro_hdls");
178
179 unsigned end_coroutine = INT_MAX;
180
181 /*
182 * This is the main coroutine execution loop. It iterates over the dimensions
183 * and calls the coroutine main entrypoint on the first pass, but in subsequent
184 * passes it checks if the coroutine has completed and resumes it if not.
185 */
186 /* take x_width - round up to type.length width */
187 lp_build_loop_begin(&loop_state[3], gallivm,
188 lp_build_const_int32(gallivm, 0)); /* coroutine reentry loop */
189 lp_build_loop_begin(&loop_state[2], gallivm,
190 lp_build_const_int32(gallivm, 0)); /* z loop */
191 lp_build_loop_begin(&loop_state[1], gallivm,
192 lp_build_const_int32(gallivm, 0)); /* y loop */
193 lp_build_loop_begin(&loop_state[0], gallivm,
194 lp_build_const_int32(gallivm, 0)); /* x loop */
195 {
196 LLVMValueRef args[13];
197 args[0] = context_ptr;
198 args[1] = loop_state[0].counter;
199 args[2] = loop_state[1].counter;
200 args[3] = loop_state[2].counter;
201 args[4] = grid_x_arg;
202 args[5] = grid_y_arg;
203 args[6] = grid_z_arg;
204 args[7] = grid_size_x_arg;
205 args[8] = grid_size_y_arg;
206 args[9] = grid_size_z_arg;
207 args[10] = thread_data_ptr;
208 args[11] = num_x_loop;
209 args[12] = partials;
210
211 /* idx = (z * (size_x * size_y) + y * size_x + x */
212 LLVMValueRef coro_hdl_idx = LLVMBuildMul(gallivm->builder, loop_state[2].counter,
213 LLVMBuildMul(gallivm->builder, num_x_loop, y_size_arg, ""), "");
214 coro_hdl_idx = LLVMBuildAdd(gallivm->builder, coro_hdl_idx,
215 LLVMBuildMul(gallivm->builder, loop_state[1].counter,
216 num_x_loop, ""), "");
217 coro_hdl_idx = LLVMBuildAdd(gallivm->builder, coro_hdl_idx,
218 loop_state[0].counter, "");
219
220 LLVMValueRef coro_entry = LLVMBuildGEP(gallivm->builder, coro_hdls, &coro_hdl_idx, 1, "");
221
222 LLVMValueRef coro_hdl = LLVMBuildLoad(gallivm->builder, coro_entry, "coro_hdl");
223
224 struct lp_build_if_state ifstate;
225 LLVMValueRef cmp = LLVMBuildICmp(gallivm->builder, LLVMIntEQ, loop_state[3].counter,
226 lp_build_const_int32(gallivm, 0), "");
227 /* first time here - call the coroutine function entry point */
228 lp_build_if(&ifstate, gallivm, cmp);
229 LLVMValueRef coro_ret = LLVMBuildCall(gallivm->builder, coro, args, 13, "");
230 LLVMBuildStore(gallivm->builder, coro_ret, coro_entry);
231 lp_build_else(&ifstate);
232 /* subsequent calls for this invocation - check if done. */
233 LLVMValueRef coro_done = lp_build_coro_done(gallivm, coro_hdl);
234 struct lp_build_if_state ifstate2;
235 lp_build_if(&ifstate2, gallivm, coro_done);
236 /* if done destroy and force loop exit */
237 lp_build_coro_destroy(gallivm, coro_hdl);
238 lp_build_loop_force_set_counter(&loop_state[3], lp_build_const_int32(gallivm, end_coroutine - 1));
239 lp_build_else(&ifstate2);
240 /* otherwise resume the coroutine */
241 lp_build_coro_resume(gallivm, coro_hdl);
242 lp_build_endif(&ifstate2);
243 lp_build_endif(&ifstate);
244 lp_build_loop_force_reload_counter(&loop_state[3]);
245 }
246 lp_build_loop_end_cond(&loop_state[0],
247 num_x_loop,
248 NULL, LLVMIntUGE);
249 lp_build_loop_end_cond(&loop_state[1],
250 y_size_arg,
251 NULL, LLVMIntUGE);
252 lp_build_loop_end_cond(&loop_state[2],
253 z_size_arg,
254 NULL, LLVMIntUGE);
255 lp_build_loop_end_cond(&loop_state[3],
256 lp_build_const_int32(gallivm, end_coroutine),
257 NULL, LLVMIntEQ);
258 LLVMBuildRetVoid(builder);
259
260 /* This is stage (b) - generate the compute shader code inside the coroutine. */
261 context_ptr = LLVMGetParam(coro, 0);
262 x_size_arg = LLVMGetParam(coro, 1);
263 y_size_arg = LLVMGetParam(coro, 2);
264 z_size_arg = LLVMGetParam(coro, 3);
265 grid_x_arg = LLVMGetParam(coro, 4);
266 grid_y_arg = LLVMGetParam(coro, 5);
267 grid_z_arg = LLVMGetParam(coro, 6);
268 grid_size_x_arg = LLVMGetParam(coro, 7);
269 grid_size_y_arg = LLVMGetParam(coro, 8);
270 grid_size_z_arg = LLVMGetParam(coro, 9);
271 thread_data_ptr = LLVMGetParam(coro, 10);
272 num_x_loop = LLVMGetParam(coro, 11);
273 partials = LLVMGetParam(coro, 12);
274 block = LLVMAppendBasicBlockInContext(gallivm->context, coro, "entry");
275 LLVMPositionBuilderAtEnd(builder, block);
276 {
277 LLVMValueRef consts_ptr, num_consts_ptr;
278 LLVMValueRef ssbo_ptr, num_ssbo_ptr;
279 LLVMValueRef shared_ptr;
280 struct lp_build_mask_context mask;
281 struct lp_bld_tgsi_system_values system_values;
282
283 memset(&system_values, 0, sizeof(system_values));
284 consts_ptr = lp_jit_cs_context_constants(gallivm, context_ptr);
285 num_consts_ptr = lp_jit_cs_context_num_constants(gallivm, context_ptr);
286 ssbo_ptr = lp_jit_cs_context_ssbos(gallivm, context_ptr);
287 num_ssbo_ptr = lp_jit_cs_context_num_ssbos(gallivm, context_ptr);
288 shared_ptr = lp_jit_cs_thread_data_shared(gallivm, thread_data_ptr);
289
290 /* these are coroutine entrypoint necessities */
291 LLVMValueRef coro_id = lp_build_coro_id(gallivm);
292 LLVMValueRef coro_hdl = lp_build_coro_begin_alloc_mem(gallivm, coro_id);
293
294 LLVMValueRef has_partials = LLVMBuildICmp(gallivm->builder, LLVMIntNE, partials, lp_build_const_int32(gallivm, 0), "");
295 LLVMValueRef tid_vals[3];
296 LLVMValueRef tids_x[LP_MAX_VECTOR_LENGTH], tids_y[LP_MAX_VECTOR_LENGTH], tids_z[LP_MAX_VECTOR_LENGTH];
297 LLVMValueRef base_val = LLVMBuildMul(gallivm->builder, x_size_arg, vec_length, "");
298 for (i = 0; i < cs_type.length; i++) {
299 tids_x[i] = LLVMBuildAdd(gallivm->builder, base_val, lp_build_const_int32(gallivm, i), "");
300 tids_y[i] = y_size_arg;
301 tids_z[i] = z_size_arg;
302 }
303 tid_vals[0] = lp_build_gather_values(gallivm, tids_x, cs_type.length);
304 tid_vals[1] = lp_build_gather_values(gallivm, tids_y, cs_type.length);
305 tid_vals[2] = lp_build_gather_values(gallivm, tids_z, cs_type.length);
306 system_values.thread_id = LLVMGetUndef(LLVMArrayType(LLVMVectorType(int32_type, cs_type.length), 3));
307 for (i = 0; i < 3; i++)
308 system_values.thread_id = LLVMBuildInsertValue(builder, system_values.thread_id, tid_vals[i], i, "");
309
310 LLVMValueRef gtids[3] = { grid_x_arg, grid_y_arg, grid_z_arg };
311 system_values.block_id = LLVMGetUndef(LLVMVectorType(int32_type, 3));
312 for (i = 0; i < 3; i++)
313 system_values.block_id = LLVMBuildInsertElement(builder, system_values.block_id, gtids[i], lp_build_const_int32(gallivm, i), "");
314
315 LLVMValueRef gstids[3] = { grid_size_x_arg, grid_size_y_arg, grid_size_z_arg };
316 system_values.grid_size = LLVMGetUndef(LLVMVectorType(int32_type, 3));
317 for (i = 0; i < 3; i++)
318 system_values.grid_size = LLVMBuildInsertElement(builder, system_values.grid_size, gstids[i], lp_build_const_int32(gallivm, i), "");
319
320 LLVMValueRef last_x_loop = LLVMBuildICmp(gallivm->builder, LLVMIntEQ, x_size_arg, LLVMBuildSub(gallivm->builder, num_x_loop, lp_build_const_int32(gallivm, 1), ""), "");
321 LLVMValueRef use_partial_mask = LLVMBuildAnd(gallivm->builder, last_x_loop, has_partials, "");
322 struct lp_build_if_state if_state;
323 LLVMValueRef mask_val = lp_build_alloca(gallivm, LLVMVectorType(int32_type, cs_type.length), "mask");
324 LLVMValueRef full_mask_val = lp_build_const_int_vec(gallivm, cs_type, ~0);
325 LLVMBuildStore(gallivm->builder, full_mask_val, mask_val);
326
327 lp_build_if(&if_state, gallivm, use_partial_mask);
328 struct lp_build_loop_state mask_loop_state;
329 lp_build_loop_begin(&mask_loop_state, gallivm, partials);
330 LLVMValueRef tmask_val = LLVMBuildLoad(gallivm->builder, mask_val, "");
331 tmask_val = LLVMBuildInsertElement(gallivm->builder, tmask_val, lp_build_const_int32(gallivm, 0), mask_loop_state.counter, "");
332 LLVMBuildStore(gallivm->builder, tmask_val, mask_val);
333 lp_build_loop_end_cond(&mask_loop_state, vec_length, NULL, LLVMIntUGE);
334 lp_build_endif(&if_state);
335
336 mask_val = LLVMBuildLoad(gallivm->builder, mask_val, "");
337 lp_build_mask_begin(&mask, gallivm, cs_type, mask_val);
338
339 struct lp_build_coro_suspend_info coro_info;
340
341 LLVMBasicBlockRef sus_block = LLVMAppendBasicBlockInContext(gallivm->context, coro, "suspend");
342 LLVMBasicBlockRef clean_block = LLVMAppendBasicBlockInContext(gallivm->context, coro, "cleanup");
343
344 coro_info.suspend = sus_block;
345 coro_info.cleanup = clean_block;
346
347 struct lp_build_tgsi_params params;
348 memset(&params, 0, sizeof(params));
349
350 params.type = cs_type;
351 params.mask = &mask;
352 params.consts_ptr = consts_ptr;
353 params.const_sizes_ptr = num_consts_ptr;
354 params.system_values = &system_values;
355 params.context_ptr = context_ptr;
356 params.sampler = sampler;
357 params.info = &shader->info.base;
358 params.ssbo_ptr = ssbo_ptr;
359 params.ssbo_sizes_ptr = num_ssbo_ptr;
360 params.image = image;
361 params.shared_ptr = shared_ptr;
362 params.coro = &coro_info;
363
364 if (shader->base.type == PIPE_SHADER_IR_TGSI)
365 lp_build_tgsi_soa(gallivm, shader->base.tokens, &params, NULL);
366 else
367 lp_build_nir_soa(gallivm, shader->base.ir.nir, &params,
368 NULL);
369
370 mask_val = lp_build_mask_end(&mask);
371
372 lp_build_coro_suspend_switch(gallivm, &coro_info, NULL, true);
373 LLVMPositionBuilderAtEnd(builder, clean_block);
374
375 lp_build_coro_free_mem(gallivm, coro_id, coro_hdl);
376
377 LLVMBuildBr(builder, sus_block);
378 LLVMPositionBuilderAtEnd(builder, sus_block);
379
380 lp_build_coro_end(gallivm, coro_hdl);
381 LLVMBuildRet(builder, coro_hdl);
382 }
383
384 sampler->destroy(sampler);
385 image->destroy(image);
386
387 gallivm_verify_function(gallivm, coro);
388 gallivm_verify_function(gallivm, function);
389 }
390
391 static void *
392 llvmpipe_create_compute_state(struct pipe_context *pipe,
393 const struct pipe_compute_state *templ)
394 {
395 struct lp_compute_shader *shader;
396 int nr_samplers, nr_sampler_views;
397 shader = CALLOC_STRUCT(lp_compute_shader);
398 if (!shader)
399 return NULL;
400
401 shader->base.type = templ->ir_type;
402 if (templ->ir_type == PIPE_SHADER_IR_TGSI) {
403 /* get/save the summary info for this shader */
404 lp_build_tgsi_info(templ->prog, &shader->info);
405
406 /* we need to keep a local copy of the tokens */
407 shader->base.tokens = tgsi_dup_tokens(templ->prog);
408 } else {
409 shader->base.ir.nir = (struct nir_shader *)templ->prog;
410 nir_tgsi_scan_shader(templ->prog, &shader->info.base, false);
411 }
412
413 shader->req_local_mem = templ->req_local_mem;
414 make_empty_list(&shader->variants);
415
416 nr_samplers = shader->info.base.file_max[TGSI_FILE_SAMPLER] + 1;
417 nr_sampler_views = shader->info.base.file_max[TGSI_FILE_SAMPLER_VIEW] + 1;
418 shader->variant_key_size = Offset(struct lp_compute_shader_variant_key,
419 state[MAX2(nr_samplers, nr_sampler_views)]);
420 return shader;
421 }
422
423 static void
424 llvmpipe_bind_compute_state(struct pipe_context *pipe,
425 void *cs)
426 {
427 struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe);
428
429 if (llvmpipe->cs == cs)
430 return;
431
432 llvmpipe->cs = (struct lp_compute_shader *)cs;
433 llvmpipe->cs_dirty |= LP_CSNEW_CS;
434 }
435
436 /**
437 * Remove shader variant from two lists: the shader's variant list
438 * and the context's variant list.
439 */
440 static void
441 llvmpipe_remove_cs_shader_variant(struct llvmpipe_context *lp,
442 struct lp_compute_shader_variant *variant)
443 {
444 if ((LP_DEBUG & DEBUG_CS) || (gallivm_debug & GALLIVM_DEBUG_IR)) {
445 debug_printf("llvmpipe: del cs #%u var %u v created %u v cached %u "
446 "v total cached %u inst %u total inst %u\n",
447 variant->shader->no, variant->no,
448 variant->shader->variants_created,
449 variant->shader->variants_cached,
450 lp->nr_cs_variants, variant->nr_instrs, lp->nr_cs_instrs);
451 }
452
453 gallivm_destroy(variant->gallivm);
454
455 /* remove from shader's list */
456 remove_from_list(&variant->list_item_local);
457 variant->shader->variants_cached--;
458
459 /* remove from context's list */
460 remove_from_list(&variant->list_item_global);
461 lp->nr_fs_variants--;
462 lp->nr_fs_instrs -= variant->nr_instrs;
463
464 FREE(variant);
465 }
466
467 static void
468 llvmpipe_delete_compute_state(struct pipe_context *pipe,
469 void *cs)
470 {
471 struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe);
472 struct lp_compute_shader *shader = cs;
473 struct lp_cs_variant_list_item *li;
474
475 /* Delete all the variants */
476 li = first_elem(&shader->variants);
477 while(!at_end(&shader->variants, li)) {
478 struct lp_cs_variant_list_item *next = next_elem(li);
479 llvmpipe_remove_cs_shader_variant(llvmpipe, li->base);
480 li = next;
481 }
482 tgsi_free_tokens(shader->base.tokens);
483 FREE(shader);
484 }
485
486 static void
487 make_variant_key(struct llvmpipe_context *lp,
488 struct lp_compute_shader *shader,
489 struct lp_compute_shader_variant_key *key)
490 {
491 int i;
492
493 memset(key, 0, shader->variant_key_size);
494
495 /* This value will be the same for all the variants of a given shader:
496 */
497 key->nr_samplers = shader->info.base.file_max[TGSI_FILE_SAMPLER] + 1;
498
499 for(i = 0; i < key->nr_samplers; ++i) {
500 if(shader->info.base.file_mask[TGSI_FILE_SAMPLER] & (1 << i)) {
501 lp_sampler_static_sampler_state(&key->state[i].sampler_state,
502 lp->samplers[PIPE_SHADER_COMPUTE][i]);
503 }
504 }
505
506 /*
507 * XXX If TGSI_FILE_SAMPLER_VIEW exists assume all texture opcodes
508 * are dx10-style? Can't really have mixed opcodes, at least not
509 * if we want to skip the holes here (without rescanning tgsi).
510 */
511 if (shader->info.base.file_max[TGSI_FILE_SAMPLER_VIEW] != -1) {
512 key->nr_sampler_views = shader->info.base.file_max[TGSI_FILE_SAMPLER_VIEW] + 1;
513 for(i = 0; i < key->nr_sampler_views; ++i) {
514 /*
515 * Note sview may exceed what's representable by file_mask.
516 * This will still work, the only downside is that not actually
517 * used views may be included in the shader key.
518 */
519 if(shader->info.base.file_mask[TGSI_FILE_SAMPLER_VIEW] & (1u << (i & 31))) {
520 lp_sampler_static_texture_state(&key->state[i].texture_state,
521 lp->sampler_views[PIPE_SHADER_COMPUTE][i]);
522 }
523 }
524 }
525 else {
526 key->nr_sampler_views = key->nr_samplers;
527 for(i = 0; i < key->nr_sampler_views; ++i) {
528 if(shader->info.base.file_mask[TGSI_FILE_SAMPLER] & (1 << i)) {
529 lp_sampler_static_texture_state(&key->state[i].texture_state,
530 lp->sampler_views[PIPE_SHADER_COMPUTE][i]);
531 }
532 }
533 }
534
535 key->nr_images = shader->info.base.file_max[TGSI_FILE_IMAGE] + 1;
536 for (i = 0; i < key->nr_images; ++i) {
537 if (shader->info.base.file_mask[TGSI_FILE_IMAGE] & (1 << i)) {
538 lp_sampler_static_texture_state_image(&key->image_state[i].image_state,
539 &lp->images[PIPE_SHADER_COMPUTE][i]);
540 }
541 }
542 }
543
544 static void
545 dump_cs_variant_key(const struct lp_compute_shader_variant_key *key)
546 {
547 int i;
548 debug_printf("cs variant %p:\n", (void *) key);
549
550 for (i = 0; i < key->nr_samplers; ++i) {
551 const struct lp_static_sampler_state *sampler = &key->state[i].sampler_state;
552 debug_printf("sampler[%u] = \n", i);
553 debug_printf(" .wrap = %s %s %s\n",
554 util_str_tex_wrap(sampler->wrap_s, TRUE),
555 util_str_tex_wrap(sampler->wrap_t, TRUE),
556 util_str_tex_wrap(sampler->wrap_r, TRUE));
557 debug_printf(" .min_img_filter = %s\n",
558 util_str_tex_filter(sampler->min_img_filter, TRUE));
559 debug_printf(" .min_mip_filter = %s\n",
560 util_str_tex_mipfilter(sampler->min_mip_filter, TRUE));
561 debug_printf(" .mag_img_filter = %s\n",
562 util_str_tex_filter(sampler->mag_img_filter, TRUE));
563 if (sampler->compare_mode != PIPE_TEX_COMPARE_NONE)
564 debug_printf(" .compare_func = %s\n", util_str_func(sampler->compare_func, TRUE));
565 debug_printf(" .normalized_coords = %u\n", sampler->normalized_coords);
566 debug_printf(" .min_max_lod_equal = %u\n", sampler->min_max_lod_equal);
567 debug_printf(" .lod_bias_non_zero = %u\n", sampler->lod_bias_non_zero);
568 debug_printf(" .apply_min_lod = %u\n", sampler->apply_min_lod);
569 debug_printf(" .apply_max_lod = %u\n", sampler->apply_max_lod);
570 }
571 for (i = 0; i < key->nr_sampler_views; ++i) {
572 const struct lp_static_texture_state *texture = &key->state[i].texture_state;
573 debug_printf("texture[%u] = \n", i);
574 debug_printf(" .format = %s\n",
575 util_format_name(texture->format));
576 debug_printf(" .target = %s\n",
577 util_str_tex_target(texture->target, TRUE));
578 debug_printf(" .level_zero_only = %u\n",
579 texture->level_zero_only);
580 debug_printf(" .pot = %u %u %u\n",
581 texture->pot_width,
582 texture->pot_height,
583 texture->pot_depth);
584 }
585 for (i = 0; i < key->nr_images; ++i) {
586 const struct lp_static_texture_state *image = &key->image_state[i].image_state;
587 debug_printf("image[%u] = \n", i);
588 debug_printf(" .format = %s\n",
589 util_format_name(image->format));
590 debug_printf(" .target = %s\n",
591 util_str_tex_target(image->target, TRUE));
592 debug_printf(" .level_zero_only = %u\n",
593 image->level_zero_only);
594 debug_printf(" .pot = %u %u %u\n",
595 image->pot_width,
596 image->pot_height,
597 image->pot_depth);
598 }
599 }
600
601 static void
602 lp_debug_cs_variant(const struct lp_compute_shader_variant *variant)
603 {
604 debug_printf("llvmpipe: Compute shader #%u variant #%u:\n",
605 variant->shader->no, variant->no);
606 if (variant->shader->base.type == PIPE_SHADER_IR_TGSI)
607 tgsi_dump(variant->shader->base.tokens, 0);
608 else
609 nir_print_shader(variant->shader->base.ir.nir, stderr);
610 dump_cs_variant_key(&variant->key);
611 debug_printf("\n");
612 }
613
614 static struct lp_compute_shader_variant *
615 generate_variant(struct llvmpipe_context *lp,
616 struct lp_compute_shader *shader,
617 const struct lp_compute_shader_variant_key *key)
618 {
619 struct lp_compute_shader_variant *variant;
620 char module_name[64];
621
622 variant = CALLOC_STRUCT(lp_compute_shader_variant);
623 if (!variant)
624 return NULL;
625
626 snprintf(module_name, sizeof(module_name), "cs%u_variant%u",
627 shader->no, shader->variants_created);
628
629 variant->gallivm = gallivm_create(module_name, lp->context);
630 if (!variant->gallivm) {
631 FREE(variant);
632 return NULL;
633 }
634
635 variant->shader = shader;
636 variant->list_item_global.base = variant;
637 variant->list_item_local.base = variant;
638 variant->no = shader->variants_created++;
639
640 memcpy(&variant->key, key, shader->variant_key_size);
641
642 if ((LP_DEBUG & DEBUG_CS) || (gallivm_debug & GALLIVM_DEBUG_IR)) {
643 lp_debug_cs_variant(variant);
644 }
645
646 lp_jit_init_cs_types(variant);
647
648 generate_compute(lp, shader, variant);
649
650 gallivm_compile_module(variant->gallivm);
651
652 variant->nr_instrs += lp_build_count_ir_module(variant->gallivm->module);
653
654 variant->jit_function = (lp_jit_cs_func)gallivm_jit_function(variant->gallivm, variant->function);
655
656 gallivm_free_ir(variant->gallivm);
657 return variant;
658 }
659
660 static void
661 lp_cs_ctx_set_cs_variant( struct lp_cs_context *csctx,
662 struct lp_compute_shader_variant *variant)
663 {
664 csctx->cs.current.variant = variant;
665 }
666
667 static void
668 llvmpipe_update_cs(struct llvmpipe_context *lp)
669 {
670 struct lp_compute_shader *shader = lp->cs;
671
672 struct lp_compute_shader_variant_key key;
673 struct lp_compute_shader_variant *variant = NULL;
674 struct lp_cs_variant_list_item *li;
675
676 make_variant_key(lp, shader, &key);
677
678 /* Search the variants for one which matches the key */
679 li = first_elem(&shader->variants);
680 while(!at_end(&shader->variants, li)) {
681 if(memcmp(&li->base->key, &key, shader->variant_key_size) == 0) {
682 variant = li->base;
683 break;
684 }
685 li = next_elem(li);
686 }
687
688 if (variant) {
689 /* Move this variant to the head of the list to implement LRU
690 * deletion of shader's when we have too many.
691 */
692 move_to_head(&lp->cs_variants_list, &variant->list_item_global);
693 }
694 else {
695 /* variant not found, create it now */
696 int64_t t0, t1, dt;
697 unsigned i;
698 unsigned variants_to_cull;
699
700 if (LP_DEBUG & DEBUG_CS) {
701 debug_printf("%u variants,\t%u instrs,\t%u instrs/variant\n",
702 lp->nr_cs_variants,
703 lp->nr_cs_instrs,
704 lp->nr_cs_variants ? lp->nr_cs_instrs / lp->nr_cs_variants : 0);
705 }
706
707 /* First, check if we've exceeded the max number of shader variants.
708 * If so, free 6.25% of them (the least recently used ones).
709 */
710 variants_to_cull = lp->nr_cs_variants >= LP_MAX_SHADER_VARIANTS ? LP_MAX_SHADER_VARIANTS / 16 : 0;
711
712 if (variants_to_cull ||
713 lp->nr_cs_instrs >= LP_MAX_SHADER_INSTRUCTIONS) {
714 if (gallivm_debug & GALLIVM_DEBUG_PERF) {
715 debug_printf("Evicting CS: %u cs variants,\t%u total variants,"
716 "\t%u instrs,\t%u instrs/variant\n",
717 shader->variants_cached,
718 lp->nr_cs_variants, lp->nr_cs_instrs,
719 lp->nr_cs_instrs / lp->nr_cs_variants);
720 }
721
722 /*
723 * We need to re-check lp->nr_cs_variants because an arbitrarliy large
724 * number of shader variants (potentially all of them) could be
725 * pending for destruction on flush.
726 */
727
728 for (i = 0; i < variants_to_cull || lp->nr_cs_instrs >= LP_MAX_SHADER_INSTRUCTIONS; i++) {
729 struct lp_cs_variant_list_item *item;
730 if (is_empty_list(&lp->cs_variants_list)) {
731 break;
732 }
733 item = last_elem(&lp->cs_variants_list);
734 assert(item);
735 assert(item->base);
736 llvmpipe_remove_cs_shader_variant(lp, item->base);
737 }
738 }
739 /*
740 * Generate the new variant.
741 */
742 t0 = os_time_get();
743 variant = generate_variant(lp, shader, &key);
744 t1 = os_time_get();
745 dt = t1 - t0;
746 LP_COUNT_ADD(llvm_compile_time, dt);
747 LP_COUNT_ADD(nr_llvm_compiles, 2); /* emit vs. omit in/out test */
748
749 /* Put the new variant into the list */
750 if (variant) {
751 insert_at_head(&shader->variants, &variant->list_item_local);
752 insert_at_head(&lp->cs_variants_list, &variant->list_item_global);
753 lp->nr_cs_variants++;
754 lp->nr_cs_instrs += variant->nr_instrs;
755 shader->variants_cached++;
756 }
757 }
758 /* Bind this variant */
759 lp_cs_ctx_set_cs_variant(lp->csctx, variant);
760 }
761
762 /**
763 * Called during state validation when LP_CSNEW_SAMPLER_VIEW is set.
764 */
765 static void
766 lp_csctx_set_sampler_views(struct lp_cs_context *csctx,
767 unsigned num,
768 struct pipe_sampler_view **views)
769 {
770 unsigned i, max_tex_num;
771
772 LP_DBG(DEBUG_SETUP, "%s\n", __FUNCTION__);
773
774 assert(num <= PIPE_MAX_SHADER_SAMPLER_VIEWS);
775
776 max_tex_num = MAX2(num, csctx->cs.current_tex_num);
777
778 for (i = 0; i < max_tex_num; i++) {
779 struct pipe_sampler_view *view = i < num ? views[i] : NULL;
780
781 if (view) {
782 struct pipe_resource *res = view->texture;
783 struct llvmpipe_resource *lp_tex = llvmpipe_resource(res);
784 struct lp_jit_texture *jit_tex;
785 jit_tex = &csctx->cs.current.jit_context.textures[i];
786
787 /* We're referencing the texture's internal data, so save a
788 * reference to it.
789 */
790 pipe_resource_reference(&csctx->cs.current_tex[i], res);
791
792 if (!lp_tex->dt) {
793 /* regular texture - csctx array of mipmap level offsets */
794 int j;
795 unsigned first_level = 0;
796 unsigned last_level = 0;
797
798 if (llvmpipe_resource_is_texture(res)) {
799 first_level = view->u.tex.first_level;
800 last_level = view->u.tex.last_level;
801 assert(first_level <= last_level);
802 assert(last_level <= res->last_level);
803 jit_tex->base = lp_tex->tex_data;
804 }
805 else {
806 jit_tex->base = lp_tex->data;
807 }
808 if (LP_PERF & PERF_TEX_MEM) {
809 /* use dummy tile memory */
810 jit_tex->base = lp_dummy_tile;
811 jit_tex->width = TILE_SIZE/8;
812 jit_tex->height = TILE_SIZE/8;
813 jit_tex->depth = 1;
814 jit_tex->first_level = 0;
815 jit_tex->last_level = 0;
816 jit_tex->mip_offsets[0] = 0;
817 jit_tex->row_stride[0] = 0;
818 jit_tex->img_stride[0] = 0;
819 }
820 else {
821 jit_tex->width = res->width0;
822 jit_tex->height = res->height0;
823 jit_tex->depth = res->depth0;
824 jit_tex->first_level = first_level;
825 jit_tex->last_level = last_level;
826
827 if (llvmpipe_resource_is_texture(res)) {
828 for (j = first_level; j <= last_level; j++) {
829 jit_tex->mip_offsets[j] = lp_tex->mip_offsets[j];
830 jit_tex->row_stride[j] = lp_tex->row_stride[j];
831 jit_tex->img_stride[j] = lp_tex->img_stride[j];
832 }
833
834 if (res->target == PIPE_TEXTURE_1D_ARRAY ||
835 res->target == PIPE_TEXTURE_2D_ARRAY ||
836 res->target == PIPE_TEXTURE_CUBE ||
837 res->target == PIPE_TEXTURE_CUBE_ARRAY) {
838 /*
839 * For array textures, we don't have first_layer, instead
840 * adjust last_layer (stored as depth) plus the mip level offsets
841 * (as we have mip-first layout can't just adjust base ptr).
842 * XXX For mip levels, could do something similar.
843 */
844 jit_tex->depth = view->u.tex.last_layer - view->u.tex.first_layer + 1;
845 for (j = first_level; j <= last_level; j++) {
846 jit_tex->mip_offsets[j] += view->u.tex.first_layer *
847 lp_tex->img_stride[j];
848 }
849 if (view->target == PIPE_TEXTURE_CUBE ||
850 view->target == PIPE_TEXTURE_CUBE_ARRAY) {
851 assert(jit_tex->depth % 6 == 0);
852 }
853 assert(view->u.tex.first_layer <= view->u.tex.last_layer);
854 assert(view->u.tex.last_layer < res->array_size);
855 }
856 }
857 else {
858 /*
859 * For buffers, we don't have "offset", instead adjust
860 * the size (stored as width) plus the base pointer.
861 */
862 unsigned view_blocksize = util_format_get_blocksize(view->format);
863 /* probably don't really need to fill that out */
864 jit_tex->mip_offsets[0] = 0;
865 jit_tex->row_stride[0] = 0;
866 jit_tex->img_stride[0] = 0;
867
868 /* everything specified in number of elements here. */
869 jit_tex->width = view->u.buf.size / view_blocksize;
870 jit_tex->base = (uint8_t *)jit_tex->base + view->u.buf.offset;
871 /* XXX Unsure if we need to sanitize parameters? */
872 assert(view->u.buf.offset + view->u.buf.size <= res->width0);
873 }
874 }
875 }
876 else {
877 /* display target texture/surface */
878 /*
879 * XXX: Where should this be unmapped?
880 */
881 struct llvmpipe_screen *screen = llvmpipe_screen(res->screen);
882 struct sw_winsys *winsys = screen->winsys;
883 jit_tex->base = winsys->displaytarget_map(winsys, lp_tex->dt,
884 PIPE_TRANSFER_READ);
885 jit_tex->row_stride[0] = lp_tex->row_stride[0];
886 jit_tex->img_stride[0] = lp_tex->img_stride[0];
887 jit_tex->mip_offsets[0] = 0;
888 jit_tex->width = res->width0;
889 jit_tex->height = res->height0;
890 jit_tex->depth = res->depth0;
891 jit_tex->first_level = jit_tex->last_level = 0;
892 assert(jit_tex->base);
893 }
894 }
895 else {
896 pipe_resource_reference(&csctx->cs.current_tex[i], NULL);
897 }
898 }
899 csctx->cs.current_tex_num = num;
900 }
901
902
903 /**
904 * Called during state validation when LP_NEW_SAMPLER is set.
905 */
906 static void
907 lp_csctx_set_sampler_state(struct lp_cs_context *csctx,
908 unsigned num,
909 struct pipe_sampler_state **samplers)
910 {
911 unsigned i;
912
913 LP_DBG(DEBUG_SETUP, "%s\n", __FUNCTION__);
914
915 assert(num <= PIPE_MAX_SAMPLERS);
916
917 for (i = 0; i < PIPE_MAX_SAMPLERS; i++) {
918 const struct pipe_sampler_state *sampler = i < num ? samplers[i] : NULL;
919
920 if (sampler) {
921 struct lp_jit_sampler *jit_sam;
922 jit_sam = &csctx->cs.current.jit_context.samplers[i];
923
924 jit_sam->min_lod = sampler->min_lod;
925 jit_sam->max_lod = sampler->max_lod;
926 jit_sam->lod_bias = sampler->lod_bias;
927 COPY_4V(jit_sam->border_color, sampler->border_color.f);
928 }
929 }
930 }
931
932 static void
933 lp_csctx_set_cs_constants(struct lp_cs_context *csctx,
934 unsigned num,
935 struct pipe_constant_buffer *buffers)
936 {
937 unsigned i;
938
939 LP_DBG(DEBUG_SETUP, "%s %p\n", __FUNCTION__, (void *) buffers);
940
941 assert(num <= ARRAY_SIZE(csctx->constants));
942
943 for (i = 0; i < num; ++i) {
944 util_copy_constant_buffer(&csctx->constants[i].current, &buffers[i]);
945 }
946 for (; i < ARRAY_SIZE(csctx->constants); i++) {
947 util_copy_constant_buffer(&csctx->constants[i].current, NULL);
948 }
949 }
950
951 static void
952 lp_csctx_set_cs_ssbos(struct lp_cs_context *csctx,
953 unsigned num,
954 struct pipe_shader_buffer *buffers)
955 {
956 int i;
957 LP_DBG(DEBUG_SETUP, "%s %p\n", __FUNCTION__, (void *)buffers);
958
959 assert (num <= ARRAY_SIZE(csctx->ssbos));
960
961 for (i = 0; i < num; ++i) {
962 util_copy_shader_buffer(&csctx->ssbos[i].current, &buffers[i]);
963 }
964 for (; i < ARRAY_SIZE(csctx->ssbos); i++) {
965 util_copy_shader_buffer(&csctx->ssbos[i].current, NULL);
966 }
967 }
968
969 static void
970 lp_csctx_set_cs_images(struct lp_cs_context *csctx,
971 unsigned num,
972 struct pipe_image_view *images)
973 {
974 unsigned i;
975
976 LP_DBG(DEBUG_SETUP, "%s %p\n", __FUNCTION__, (void *) images);
977
978 assert(num <= ARRAY_SIZE(csctx->images));
979
980 for (i = 0; i < num; ++i) {
981 struct pipe_image_view *image = &images[i];
982 util_copy_image_view(&csctx->images[i].current, &images[i]);
983
984 struct pipe_resource *res = image->resource;
985 struct llvmpipe_resource *lp_res = llvmpipe_resource(res);
986 struct lp_jit_image *jit_image;
987
988 jit_image = &csctx->cs.current.jit_context.images[i];
989 if (!lp_res)
990 continue;
991 if (!lp_res->dt) {
992 /* regular texture - csctx array of mipmap level offsets */
993 if (llvmpipe_resource_is_texture(res)) {
994 jit_image->base = lp_res->tex_data;
995 } else
996 jit_image->base = lp_res->data;
997
998 jit_image->width = res->width0;
999 jit_image->height = res->height0;
1000 jit_image->depth = res->depth0;
1001
1002 if (llvmpipe_resource_is_texture(res)) {
1003 uint32_t mip_offset = lp_res->mip_offsets[image->u.tex.level];
1004
1005 jit_image->width = u_minify(jit_image->width, image->u.tex.level);
1006 jit_image->height = u_minify(jit_image->height, image->u.tex.level);
1007
1008 if (res->target == PIPE_TEXTURE_1D_ARRAY ||
1009 res->target == PIPE_TEXTURE_2D_ARRAY ||
1010 res->target == PIPE_TEXTURE_3D ||
1011 res->target == PIPE_TEXTURE_CUBE ||
1012 res->target == PIPE_TEXTURE_CUBE_ARRAY) {
1013 /*
1014 * For array textures, we don't have first_layer, instead
1015 * adjust last_layer (stored as depth) plus the mip level offsets
1016 * (as we have mip-first layout can't just adjust base ptr).
1017 * XXX For mip levels, could do something similar.
1018 */
1019 jit_image->depth = image->u.tex.last_layer - image->u.tex.first_layer + 1;
1020 mip_offset += image->u.tex.first_layer * lp_res->img_stride[image->u.tex.level];
1021 } else
1022 jit_image->depth = u_minify(jit_image->depth, image->u.tex.level);
1023
1024 jit_image->row_stride = lp_res->row_stride[image->u.tex.level];
1025 jit_image->img_stride = lp_res->img_stride[image->u.tex.level];
1026 jit_image->base = (uint8_t *)jit_image->base + mip_offset;
1027 } else {
1028 unsigned view_blocksize = util_format_get_blocksize(image->format);
1029 jit_image->width = image->u.buf.size / view_blocksize;
1030 jit_image->base = (uint8_t *)jit_image->base + image->u.buf.offset;
1031 }
1032 }
1033 }
1034 for (; i < ARRAY_SIZE(csctx->images); i++) {
1035 util_copy_image_view(&csctx->images[i].current, NULL);
1036 }
1037 }
1038
1039 static void
1040 update_csctx_consts(struct llvmpipe_context *llvmpipe)
1041 {
1042 struct lp_cs_context *csctx = llvmpipe->csctx;
1043 int i;
1044
1045 for (i = 0; i < ARRAY_SIZE(csctx->constants); ++i) {
1046 struct pipe_resource *buffer = csctx->constants[i].current.buffer;
1047 const ubyte *current_data = NULL;
1048
1049 if (buffer) {
1050 /* resource buffer */
1051 current_data = (ubyte *) llvmpipe_resource_data(buffer);
1052 }
1053 else if (csctx->constants[i].current.user_buffer) {
1054 /* user-space buffer */
1055 current_data = (ubyte *) csctx->constants[i].current.user_buffer;
1056 }
1057
1058 if (current_data) {
1059 current_data += csctx->constants[i].current.buffer_offset;
1060
1061 csctx->cs.current.jit_context.constants[i] = (const float *)current_data;
1062 csctx->cs.current.jit_context.num_constants[i] = csctx->constants[i].current.buffer_size;
1063 } else {
1064 csctx->cs.current.jit_context.constants[i] = NULL;
1065 csctx->cs.current.jit_context.num_constants[i] = 0;
1066 }
1067 }
1068 }
1069
1070 static void
1071 update_csctx_ssbo(struct llvmpipe_context *llvmpipe)
1072 {
1073 struct lp_cs_context *csctx = llvmpipe->csctx;
1074 int i;
1075 for (i = 0; i < ARRAY_SIZE(csctx->ssbos); ++i) {
1076 struct pipe_resource *buffer = csctx->ssbos[i].current.buffer;
1077 const ubyte *current_data = NULL;
1078
1079 if (!buffer)
1080 continue;
1081 /* resource buffer */
1082 current_data = (ubyte *) llvmpipe_resource_data(buffer);
1083 if (current_data) {
1084 current_data += csctx->ssbos[i].current.buffer_offset;
1085
1086 csctx->cs.current.jit_context.ssbos[i] = (const uint32_t *)current_data;
1087 csctx->cs.current.jit_context.num_ssbos[i] = csctx->ssbos[i].current.buffer_size;
1088 } else {
1089 csctx->cs.current.jit_context.ssbos[i] = NULL;
1090 csctx->cs.current.jit_context.num_ssbos[i] = 0;
1091 }
1092 }
1093 }
1094
1095 static void
1096 llvmpipe_cs_update_derived(struct llvmpipe_context *llvmpipe)
1097 {
1098 if (llvmpipe->cs_dirty & (LP_CSNEW_CS))
1099 llvmpipe_update_cs(llvmpipe);
1100
1101 if (llvmpipe->cs_dirty & LP_CSNEW_CONSTANTS) {
1102 lp_csctx_set_cs_constants(llvmpipe->csctx,
1103 ARRAY_SIZE(llvmpipe->constants[PIPE_SHADER_COMPUTE]),
1104 llvmpipe->constants[PIPE_SHADER_COMPUTE]);
1105 update_csctx_consts(llvmpipe);
1106 }
1107
1108 if (llvmpipe->cs_dirty & LP_CSNEW_SSBOS) {
1109 lp_csctx_set_cs_ssbos(llvmpipe->csctx,
1110 ARRAY_SIZE(llvmpipe->ssbos[PIPE_SHADER_COMPUTE]),
1111 llvmpipe->ssbos[PIPE_SHADER_COMPUTE]);
1112 update_csctx_ssbo(llvmpipe);
1113 }
1114
1115 if (llvmpipe->cs_dirty & LP_CSNEW_SAMPLER_VIEW)
1116 lp_csctx_set_sampler_views(llvmpipe->csctx,
1117 llvmpipe->num_sampler_views[PIPE_SHADER_COMPUTE],
1118 llvmpipe->sampler_views[PIPE_SHADER_COMPUTE]);
1119
1120 if (llvmpipe->cs_dirty & LP_CSNEW_SAMPLER)
1121 lp_csctx_set_sampler_state(llvmpipe->csctx,
1122 llvmpipe->num_samplers[PIPE_SHADER_COMPUTE],
1123 llvmpipe->samplers[PIPE_SHADER_COMPUTE]);
1124
1125 if (llvmpipe->cs_dirty & LP_CSNEW_IMAGES)
1126 lp_csctx_set_cs_images(llvmpipe->csctx,
1127 ARRAY_SIZE(llvmpipe->images[PIPE_SHADER_COMPUTE]),
1128 llvmpipe->images[PIPE_SHADER_COMPUTE]);
1129
1130 llvmpipe->cs_dirty = 0;
1131 }
1132
1133 static void
1134 cs_exec_fn(void *init_data, int iter_idx, struct lp_cs_local_mem *lmem)
1135 {
1136 struct lp_cs_job_info *job_info = init_data;
1137 struct lp_jit_cs_thread_data thread_data;
1138
1139 memset(&thread_data, 0, sizeof(thread_data));
1140
1141 if (lmem->local_size < job_info->req_local_mem) {
1142 lmem->local_mem_ptr = REALLOC(lmem->local_mem_ptr, lmem->local_size,
1143 job_info->req_local_mem);
1144 lmem->local_size = job_info->req_local_mem;
1145 }
1146 thread_data.shared = lmem->local_mem_ptr;
1147
1148 unsigned grid_z = iter_idx / (job_info->grid_size[0] * job_info->grid_size[1]);
1149 unsigned grid_y = (iter_idx - (grid_z * (job_info->grid_size[0] * job_info->grid_size[1]))) / job_info->grid_size[0];
1150 unsigned grid_x = (iter_idx - (grid_z * (job_info->grid_size[0] * job_info->grid_size[1])) - (grid_y * job_info->grid_size[0]));
1151 struct lp_compute_shader_variant *variant = job_info->current->variant;
1152 variant->jit_function(&job_info->current->jit_context,
1153 job_info->block_size[0], job_info->block_size[1], job_info->block_size[2],
1154 grid_x, grid_y, grid_z,
1155 job_info->grid_size[0], job_info->grid_size[1], job_info->grid_size[2],
1156 &thread_data);
1157 }
1158
1159 static void
1160 fill_grid_size(struct pipe_context *pipe,
1161 const struct pipe_grid_info *info,
1162 uint32_t grid_size[3])
1163 {
1164 struct pipe_transfer *transfer;
1165 uint32_t *params;
1166 if (!info->indirect) {
1167 grid_size[0] = info->grid[0];
1168 grid_size[1] = info->grid[1];
1169 grid_size[2] = info->grid[2];
1170 return;
1171 }
1172 params = pipe_buffer_map_range(pipe, info->indirect,
1173 info->indirect_offset,
1174 3 * sizeof(uint32_t),
1175 PIPE_TRANSFER_READ,
1176 &transfer);
1177
1178 if (!transfer)
1179 return;
1180
1181 grid_size[0] = params[0];
1182 grid_size[1] = params[1];
1183 grid_size[2] = params[2];
1184 pipe_buffer_unmap(pipe, transfer);
1185 }
1186
1187 static void llvmpipe_launch_grid(struct pipe_context *pipe,
1188 const struct pipe_grid_info *info)
1189 {
1190 struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe);
1191 struct llvmpipe_screen *screen = llvmpipe_screen(pipe->screen);
1192 struct lp_cs_job_info job_info;
1193
1194 memset(&job_info, 0, sizeof(job_info));
1195
1196 llvmpipe_cs_update_derived(llvmpipe);
1197
1198 fill_grid_size(pipe, info, job_info.grid_size);
1199
1200 job_info.block_size[0] = info->block[0];
1201 job_info.block_size[1] = info->block[1];
1202 job_info.block_size[2] = info->block[2];
1203 job_info.req_local_mem = llvmpipe->cs->req_local_mem;
1204 job_info.current = &llvmpipe->csctx->cs.current;
1205
1206 int num_tasks = job_info.grid_size[2] * job_info.grid_size[1] * job_info.grid_size[0];
1207 if (num_tasks) {
1208 struct lp_cs_tpool_task *task;
1209 mtx_lock(&screen->cs_mutex);
1210 task = lp_cs_tpool_queue_task(screen->cs_tpool, cs_exec_fn, &job_info, num_tasks);
1211
1212 lp_cs_tpool_wait_for_task(screen->cs_tpool, &task);
1213 mtx_unlock(&screen->cs_mutex);
1214 }
1215 llvmpipe->pipeline_statistics.cs_invocations += num_tasks * info->block[0] * info->block[1] * info->block[2];
1216 }
1217
1218 void
1219 llvmpipe_init_compute_funcs(struct llvmpipe_context *llvmpipe)
1220 {
1221 llvmpipe->pipe.create_compute_state = llvmpipe_create_compute_state;
1222 llvmpipe->pipe.bind_compute_state = llvmpipe_bind_compute_state;
1223 llvmpipe->pipe.delete_compute_state = llvmpipe_delete_compute_state;
1224 llvmpipe->pipe.launch_grid = llvmpipe_launch_grid;
1225 }
1226
1227 void
1228 lp_csctx_destroy(struct lp_cs_context *csctx)
1229 {
1230 unsigned i;
1231 for (i = 0; i < ARRAY_SIZE(csctx->cs.current_tex); i++) {
1232 pipe_resource_reference(&csctx->cs.current_tex[i], NULL);
1233 }
1234 for (i = 0; i < ARRAY_SIZE(csctx->constants); i++) {
1235 pipe_resource_reference(&csctx->constants[i].current.buffer, NULL);
1236 }
1237 for (i = 0; i < ARRAY_SIZE(csctx->ssbos); i++) {
1238 pipe_resource_reference(&csctx->ssbos[i].current.buffer, NULL);
1239 }
1240 FREE(csctx);
1241 }
1242
1243 struct lp_cs_context *lp_csctx_create(struct pipe_context *pipe)
1244 {
1245 struct lp_cs_context *csctx;
1246
1247 csctx = CALLOC_STRUCT(lp_cs_context);
1248 if (!csctx)
1249 return NULL;
1250
1251 csctx->pipe = pipe;
1252 return csctx;
1253 }