projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
llvmpipe: fix CALLOC vs. free mismatches
[mesa.git] / src / gallium / drivers / llvmpipe / lp_state_cs.c
1 /**************************************************************************
2 *
3 * Copyright 2019 Red Hat.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included
14 * in all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
17 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 *
24 **************************************************************************/
25 #include "util/u_memory.h"
26 #include "util/simple_list.h"
27 #include "util/os_time.h"
28 #include "util/u_dump.h"
29 #include "util/u_string.h"
30 #include "tgsi/tgsi_dump.h"
31 #include "tgsi/tgsi_parse.h"
32 #include "gallivm/lp_bld_const.h"
33 #include "gallivm/lp_bld_debug.h"
34 #include "gallivm/lp_bld_intr.h"
35 #include "gallivm/lp_bld_flow.h"
36 #include "gallivm/lp_bld_gather.h"
37 #include "gallivm/lp_bld_coro.h"
38 #include "lp_state_cs.h"
39 #include "lp_context.h"
40 #include "lp_debug.h"
41 #include "lp_state.h"
42 #include "lp_perf.h"
43 #include "lp_screen.h"
44 #include "lp_memory.h"
45 #include "lp_cs_tpool.h"
46 #include "state_tracker/sw_winsys.h"
47
48 struct lp_cs_job_info {
49 unsigned grid_size[3];
50 unsigned block_size[3];
51 unsigned req_local_mem;
52 struct lp_cs_exec *current;
53 };
54
55 static void
56 generate_compute(struct llvmpipe_context *lp,
57 struct lp_compute_shader *shader,
58 struct lp_compute_shader_variant *variant)
59 {
60 struct gallivm_state *gallivm = variant->gallivm;
61 const struct lp_compute_shader_variant_key *key = &variant->key;
62 char func_name[64], func_name_coro[64];
63 LLVMTypeRef arg_types[13];
64 LLVMTypeRef func_type, coro_func_type;
65 LLVMTypeRef int32_type = LLVMInt32TypeInContext(gallivm->context);
66 LLVMValueRef context_ptr;
67 LLVMValueRef x_size_arg, y_size_arg, z_size_arg;
68 LLVMValueRef grid_x_arg, grid_y_arg, grid_z_arg;
69 LLVMValueRef grid_size_x_arg, grid_size_y_arg, grid_size_z_arg;
70 LLVMValueRef thread_data_ptr;
71 LLVMBasicBlockRef block;
72 LLVMBuilderRef builder;
73 struct lp_build_sampler_soa *sampler;
74 struct lp_build_image_soa *image;
75 LLVMValueRef function, coro;
76 struct lp_type cs_type;
77 unsigned i;
78
79 /*
80 * This function has two parts
81 * a) setup the coroutine execution environment loop.
82 * b) build the compute shader llvm for use inside the coroutine.
83 */
84 assert(lp_native_vector_width / 32 >= 4);
85
86 memset(&cs_type, 0, sizeof cs_type);
87 cs_type.floating = TRUE; /* floating point values */
88 cs_type.sign = TRUE; /* values are signed */
89 cs_type.norm = FALSE; /* values are not limited to [0,1] or [-1,1] */
90 cs_type.width = 32; /* 32-bit float */
91 cs_type.length = MIN2(lp_native_vector_width / 32, 16); /* n*4 elements per vector */
92 snprintf(func_name, sizeof(func_name), "cs%u_variant%u",
93 shader->no, variant->no);
94
95 snprintf(func_name_coro, sizeof(func_name), "cs_co_%u_variant%u",
96 shader->no, variant->no);
97
98 arg_types[0] = variant->jit_cs_context_ptr_type; /* context */
99 arg_types[1] = int32_type; /* block_x_size */
100 arg_types[2] = int32_type; /* block_y_size */
101 arg_types[3] = int32_type; /* block_z_size */
102 arg_types[4] = int32_type; /* grid_x */
103 arg_types[5] = int32_type; /* grid_y */
104 arg_types[6] = int32_type; /* grid_z */
105 arg_types[7] = int32_type; /* grid_size_x */
106 arg_types[8] = int32_type; /* grid_size_y */
107 arg_types[9] = int32_type; /* grid_size_z */
108 arg_types[10] = variant->jit_cs_thread_data_ptr_type; /* per thread data */
109 arg_types[11] = int32_type;
110 arg_types[12] = int32_type;
111 func_type = LLVMFunctionType(LLVMVoidTypeInContext(gallivm->context),
112 arg_types, ARRAY_SIZE(arg_types) - 2, 0);
113
114 coro_func_type = LLVMFunctionType(LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0),
115 arg_types, ARRAY_SIZE(arg_types), 0);
116
117 function = LLVMAddFunction(gallivm->module, func_name, func_type);
118 LLVMSetFunctionCallConv(function, LLVMCCallConv);
119
120 coro = LLVMAddFunction(gallivm->module, func_name_coro, coro_func_type);
121 LLVMSetFunctionCallConv(coro, LLVMCCallConv);
122
123 variant->function = function;
124
125 for(i = 0; i < ARRAY_SIZE(arg_types); ++i) {
126 if(LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind) {
127 lp_add_function_attr(coro, i + 1, LP_FUNC_ATTR_NOALIAS);
128 lp_add_function_attr(function, i + 1, LP_FUNC_ATTR_NOALIAS);
129 }
130 }
131
132 context_ptr = LLVMGetParam(function, 0);
133 x_size_arg = LLVMGetParam(function, 1);
134 y_size_arg = LLVMGetParam(function, 2);
135 z_size_arg = LLVMGetParam(function, 3);
136 grid_x_arg = LLVMGetParam(function, 4);
137 grid_y_arg = LLVMGetParam(function, 5);
138 grid_z_arg = LLVMGetParam(function, 6);
139 grid_size_x_arg = LLVMGetParam(function, 7);
140 grid_size_y_arg = LLVMGetParam(function, 8);
141 grid_size_z_arg = LLVMGetParam(function, 9);
142 thread_data_ptr = LLVMGetParam(function, 10);
143
144 lp_build_name(context_ptr, "context");
145 lp_build_name(x_size_arg, "x_size");
146 lp_build_name(y_size_arg, "y_size");
147 lp_build_name(z_size_arg, "z_size");
148 lp_build_name(grid_x_arg, "grid_x");
149 lp_build_name(grid_y_arg, "grid_y");
150 lp_build_name(grid_z_arg, "grid_z");
151 lp_build_name(grid_size_x_arg, "grid_size_x");
152 lp_build_name(grid_size_y_arg, "grid_size_y");
153 lp_build_name(grid_size_z_arg, "grid_size_z");
154 lp_build_name(thread_data_ptr, "thread_data");
155
156 block = LLVMAppendBasicBlockInContext(gallivm->context, function, "entry");
157 builder = gallivm->builder;
158 assert(builder);
159 LLVMPositionBuilderAtEnd(builder, block);
160 sampler = lp_llvm_sampler_soa_create(key->state);
161 image = lp_llvm_image_soa_create(key->image_state);
162
163 struct lp_build_loop_state loop_state[4];
164 LLVMValueRef num_x_loop;
165 LLVMValueRef vec_length = lp_build_const_int32(gallivm, cs_type.length);
166 num_x_loop = LLVMBuildAdd(gallivm->builder, x_size_arg, vec_length, "");
167 num_x_loop = LLVMBuildSub(gallivm->builder, num_x_loop, lp_build_const_int32(gallivm, 1), "");
168 num_x_loop = LLVMBuildUDiv(gallivm->builder, num_x_loop, vec_length, "");
169 LLVMValueRef partials = LLVMBuildURem(gallivm->builder, x_size_arg, vec_length, "");
170
171 LLVMValueRef coro_num_hdls = LLVMBuildMul(gallivm->builder, num_x_loop, y_size_arg, "");
172 coro_num_hdls = LLVMBuildMul(gallivm->builder, coro_num_hdls, z_size_arg, "");
173
174 LLVMTypeRef hdl_ptr_type = LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0);
175 LLVMValueRef coro_hdls = LLVMBuildArrayAlloca(gallivm->builder, hdl_ptr_type, coro_num_hdls, "coro_hdls");
176
177 unsigned end_coroutine = INT_MAX;
178
179 /*
180 * This is the main coroutine execution loop. It iterates over the dimensions
181 * and calls the coroutine main entrypoint on the first pass, but in subsequent
182 * passes it checks if the coroutine has completed and resumes it if not.
183 */
184 /* take x_width - round up to type.length width */
185 lp_build_loop_begin(&loop_state[3], gallivm,
186 lp_build_const_int32(gallivm, 0)); /* coroutine reentry loop */
187 lp_build_loop_begin(&loop_state[2], gallivm,
188 lp_build_const_int32(gallivm, 0)); /* z loop */
189 lp_build_loop_begin(&loop_state[1], gallivm,
190 lp_build_const_int32(gallivm, 0)); /* y loop */
191 lp_build_loop_begin(&loop_state[0], gallivm,
192 lp_build_const_int32(gallivm, 0)); /* x loop */
193 {
194 LLVMValueRef args[13];
195 args[0] = context_ptr;
196 args[1] = loop_state[0].counter;
197 args[2] = loop_state[1].counter;
198 args[3] = loop_state[2].counter;
199 args[4] = grid_x_arg;
200 args[5] = grid_y_arg;
201 args[6] = grid_z_arg;
202 args[7] = grid_size_x_arg;
203 args[8] = grid_size_y_arg;
204 args[9] = grid_size_z_arg;
205 args[10] = thread_data_ptr;
206 args[11] = num_x_loop;
207 args[12] = partials;
208
209 /* idx = (z * (size_x * size_y) + y * size_x + x */
210 LLVMValueRef coro_hdl_idx = LLVMBuildMul(gallivm->builder, loop_state[2].counter,
211 LLVMBuildMul(gallivm->builder, num_x_loop, y_size_arg, ""), "");
212 coro_hdl_idx = LLVMBuildAdd(gallivm->builder, coro_hdl_idx,
213 LLVMBuildMul(gallivm->builder, loop_state[1].counter,
214 num_x_loop, ""), "");
215 coro_hdl_idx = LLVMBuildAdd(gallivm->builder, coro_hdl_idx,
216 loop_state[0].counter, "");
217
218 LLVMValueRef coro_entry = LLVMBuildGEP(gallivm->builder, coro_hdls, &coro_hdl_idx, 1, "");
219
220 LLVMValueRef coro_hdl = LLVMBuildLoad(gallivm->builder, coro_entry, "coro_hdl");
221
222 struct lp_build_if_state ifstate;
223 LLVMValueRef cmp = LLVMBuildICmp(gallivm->builder, LLVMIntEQ, loop_state[3].counter,
224 lp_build_const_int32(gallivm, 0), "");
225 /* first time here - call the coroutine function entry point */
226 lp_build_if(&ifstate, gallivm, cmp);
227 LLVMValueRef coro_ret = LLVMBuildCall(gallivm->builder, coro, args, 13, "");
228 LLVMBuildStore(gallivm->builder, coro_ret, coro_entry);
229 lp_build_else(&ifstate);
230 /* subsequent calls for this invocation - check if done. */
231 LLVMValueRef coro_done = lp_build_coro_done(gallivm, coro_hdl);
232 struct lp_build_if_state ifstate2;
233 lp_build_if(&ifstate2, gallivm, coro_done);
234 /* if done destroy and force loop exit */
235 lp_build_coro_destroy(gallivm, coro_hdl);
236 lp_build_loop_force_set_counter(&loop_state[3], lp_build_const_int32(gallivm, end_coroutine - 1));
237 lp_build_else(&ifstate2);
238 /* otherwise resume the coroutine */
239 lp_build_coro_resume(gallivm, coro_hdl);
240 lp_build_endif(&ifstate2);
241 lp_build_endif(&ifstate);
242 lp_build_loop_force_reload_counter(&loop_state[3]);
243 }
244 lp_build_loop_end_cond(&loop_state[0],
245 num_x_loop,
246 NULL, LLVMIntUGE);
247 lp_build_loop_end_cond(&loop_state[1],
248 y_size_arg,
249 NULL, LLVMIntUGE);
250 lp_build_loop_end_cond(&loop_state[2],
251 z_size_arg,
252 NULL, LLVMIntUGE);
253 lp_build_loop_end_cond(&loop_state[3],
254 lp_build_const_int32(gallivm, end_coroutine),
255 NULL, LLVMIntEQ);
256 LLVMBuildRetVoid(builder);
257
258 /* This is stage (b) - generate the compute shader code inside the coroutine. */
259 context_ptr = LLVMGetParam(coro, 0);
260 x_size_arg = LLVMGetParam(coro, 1);
261 y_size_arg = LLVMGetParam(coro, 2);
262 z_size_arg = LLVMGetParam(coro, 3);
263 grid_x_arg = LLVMGetParam(coro, 4);
264 grid_y_arg = LLVMGetParam(coro, 5);
265 grid_z_arg = LLVMGetParam(coro, 6);
266 grid_size_x_arg = LLVMGetParam(coro, 7);
267 grid_size_y_arg = LLVMGetParam(coro, 8);
268 grid_size_z_arg = LLVMGetParam(coro, 9);
269 thread_data_ptr = LLVMGetParam(coro, 10);
270 num_x_loop = LLVMGetParam(coro, 11);
271 partials = LLVMGetParam(coro, 12);
272 block = LLVMAppendBasicBlockInContext(gallivm->context, coro, "entry");
273 LLVMPositionBuilderAtEnd(builder, block);
274 {
275 const struct tgsi_token *tokens = shader->base.tokens;
276 LLVMValueRef consts_ptr, num_consts_ptr;
277 LLVMValueRef ssbo_ptr, num_ssbo_ptr;
278 LLVMValueRef shared_ptr;
279 struct lp_build_mask_context mask;
280 struct lp_bld_tgsi_system_values system_values;
281
282 memset(&system_values, 0, sizeof(system_values));
283 consts_ptr = lp_jit_cs_context_constants(gallivm, context_ptr);
284 num_consts_ptr = lp_jit_cs_context_num_constants(gallivm, context_ptr);
285 ssbo_ptr = lp_jit_cs_context_ssbos(gallivm, context_ptr);
286 num_ssbo_ptr = lp_jit_cs_context_num_ssbos(gallivm, context_ptr);
287 shared_ptr = lp_jit_cs_thread_data_shared(gallivm, thread_data_ptr);
288
289 /* these are coroutine entrypoint necessities */
290 LLVMValueRef coro_id = lp_build_coro_id(gallivm);
291 LLVMValueRef coro_hdl = lp_build_coro_begin_alloc_mem(gallivm, coro_id);
292
293 LLVMValueRef has_partials = LLVMBuildICmp(gallivm->builder, LLVMIntNE, partials, lp_build_const_int32(gallivm, 0), "");
294 LLVMValueRef tid_vals[3];
295 LLVMValueRef tids_x[LP_MAX_VECTOR_LENGTH], tids_y[LP_MAX_VECTOR_LENGTH], tids_z[LP_MAX_VECTOR_LENGTH];
296 LLVMValueRef base_val = LLVMBuildMul(gallivm->builder, x_size_arg, vec_length, "");
297 for (i = 0; i < cs_type.length; i++) {
298 tids_x[i] = LLVMBuildAdd(gallivm->builder, base_val, lp_build_const_int32(gallivm, i), "");
299 tids_y[i] = y_size_arg;
300 tids_z[i] = z_size_arg;
301 }
302 tid_vals[0] = lp_build_gather_values(gallivm, tids_x, cs_type.length);
303 tid_vals[1] = lp_build_gather_values(gallivm, tids_y, cs_type.length);
304 tid_vals[2] = lp_build_gather_values(gallivm, tids_z, cs_type.length);
305 system_values.thread_id = LLVMGetUndef(LLVMArrayType(LLVMVectorType(int32_type, cs_type.length), 3));
306 for (i = 0; i < 3; i++)
307 system_values.thread_id = LLVMBuildInsertValue(builder, system_values.thread_id, tid_vals[i], i, "");
308
309 LLVMValueRef gtids[3] = { grid_x_arg, grid_y_arg, grid_z_arg };
310 system_values.block_id = LLVMGetUndef(LLVMVectorType(int32_type, 3));
311 for (i = 0; i < 3; i++)
312 system_values.block_id = LLVMBuildInsertElement(builder, system_values.block_id, gtids[i], lp_build_const_int32(gallivm, i), "");
313
314 LLVMValueRef gstids[3] = { grid_size_x_arg, grid_size_y_arg, grid_size_z_arg };
315 system_values.grid_size = LLVMGetUndef(LLVMVectorType(int32_type, 3));
316 for (i = 0; i < 3; i++)
317 system_values.grid_size = LLVMBuildInsertElement(builder, system_values.grid_size, gstids[i], lp_build_const_int32(gallivm, i), "");
318
319 LLVMValueRef last_x_loop = LLVMBuildICmp(gallivm->builder, LLVMIntEQ, x_size_arg, LLVMBuildSub(gallivm->builder, num_x_loop, lp_build_const_int32(gallivm, 1), ""), "");
320 LLVMValueRef use_partial_mask = LLVMBuildAnd(gallivm->builder, last_x_loop, has_partials, "");
321 struct lp_build_if_state if_state;
322 LLVMValueRef mask_val = lp_build_alloca(gallivm, LLVMVectorType(int32_type, cs_type.length), "mask");
323 LLVMValueRef full_mask_val = lp_build_const_int_vec(gallivm, cs_type, ~0);
324 LLVMBuildStore(gallivm->builder, full_mask_val, mask_val);
325
326 lp_build_if(&if_state, gallivm, use_partial_mask);
327 struct lp_build_loop_state mask_loop_state;
328 lp_build_loop_begin(&mask_loop_state, gallivm, partials);
329 LLVMValueRef tmask_val = LLVMBuildLoad(gallivm->builder, mask_val, "");
330 tmask_val = LLVMBuildInsertElement(gallivm->builder, tmask_val, lp_build_const_int32(gallivm, 0), mask_loop_state.counter, "");
331 LLVMBuildStore(gallivm->builder, tmask_val, mask_val);
332 lp_build_loop_end_cond(&mask_loop_state, vec_length, NULL, LLVMIntUGE);
333 lp_build_endif(&if_state);
334
335 mask_val = LLVMBuildLoad(gallivm->builder, mask_val, "");
336 lp_build_mask_begin(&mask, gallivm, cs_type, mask_val);
337
338 struct lp_build_coro_suspend_info coro_info;
339
340 LLVMBasicBlockRef sus_block = LLVMAppendBasicBlockInContext(gallivm->context, coro, "suspend");
341 LLVMBasicBlockRef clean_block = LLVMAppendBasicBlockInContext(gallivm->context, coro, "cleanup");
342
343 coro_info.suspend = sus_block;
344 coro_info.cleanup = clean_block;
345
346 struct lp_build_tgsi_params params;
347 memset(&params, 0, sizeof(params));
348
349 params.type = cs_type;
350 params.mask = &mask;
351 params.consts_ptr = consts_ptr;
352 params.const_sizes_ptr = num_consts_ptr;
353 params.system_values = &system_values;
354 params.context_ptr = context_ptr;
355 params.sampler = sampler;
356 params.info = &shader->info.base;
357 params.ssbo_ptr = ssbo_ptr;
358 params.ssbo_sizes_ptr = num_ssbo_ptr;
359 params.image = image;
360 params.shared_ptr = shared_ptr;
361 params.coro = &coro_info;
362
363 lp_build_tgsi_soa(gallivm, tokens, &params, NULL);
364
365 mask_val = lp_build_mask_end(&mask);
366
367 lp_build_coro_suspend_switch(gallivm, &coro_info, NULL, true);
368 LLVMPositionBuilderAtEnd(builder, clean_block);
369
370 lp_build_coro_free_mem(gallivm, coro_id, coro_hdl);
371
372 LLVMBuildBr(builder, sus_block);
373 LLVMPositionBuilderAtEnd(builder, sus_block);
374
375 lp_build_coro_end(gallivm, coro_hdl);
376 LLVMBuildRet(builder, coro_hdl);
377 }
378
379 sampler->destroy(sampler);
380 image->destroy(image);
381
382 gallivm_verify_function(gallivm, coro);
383 gallivm_verify_function(gallivm, function);
384 }
385
386 static void *
387 llvmpipe_create_compute_state(struct pipe_context *pipe,
388 const struct pipe_compute_state *templ)
389 {
390 struct lp_compute_shader *shader;
391 int nr_samplers, nr_sampler_views;
392 shader = CALLOC_STRUCT(lp_compute_shader);
393 if (!shader)
394 return NULL;
395
396 assert(templ->ir_type == PIPE_SHADER_IR_TGSI);
397 shader->base.tokens = tgsi_dup_tokens(templ->prog);
398
399 shader->req_local_mem = templ->req_local_mem;
400 lp_build_tgsi_info(shader->base.tokens, &shader->info);
401 make_empty_list(&shader->variants);
402
403 nr_samplers = shader->info.base.file_max[TGSI_FILE_SAMPLER] + 1;
404 nr_sampler_views = shader->info.base.file_max[TGSI_FILE_SAMPLER_VIEW] + 1;
405 shader->variant_key_size = Offset(struct lp_compute_shader_variant_key,
406 state[MAX2(nr_samplers, nr_sampler_views)]);
407 return shader;
408 }
409
410 static void
411 llvmpipe_bind_compute_state(struct pipe_context *pipe,
412 void *cs)
413 {
414 struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe);
415
416 if (llvmpipe->cs == cs)
417 return;
418
419 llvmpipe->cs = (struct lp_compute_shader *)cs;
420 llvmpipe->cs_dirty |= LP_CSNEW_CS;
421 }
422
423 /**
424 * Remove shader variant from two lists: the shader's variant list
425 * and the context's variant list.
426 */
427 static void
428 llvmpipe_remove_cs_shader_variant(struct llvmpipe_context *lp,
429 struct lp_compute_shader_variant *variant)
430 {
431 if ((LP_DEBUG & DEBUG_CS) || (gallivm_debug & GALLIVM_DEBUG_IR)) {
432 debug_printf("llvmpipe: del cs #%u var %u v created %u v cached %u "
433 "v total cached %u inst %u total inst %u\n",
434 variant->shader->no, variant->no,
435 variant->shader->variants_created,
436 variant->shader->variants_cached,
437 lp->nr_cs_variants, variant->nr_instrs, lp->nr_cs_instrs);
438 }
439
440 gallivm_destroy(variant->gallivm);
441
442 /* remove from shader's list */
443 remove_from_list(&variant->list_item_local);
444 variant->shader->variants_cached--;
445
446 /* remove from context's list */
447 remove_from_list(&variant->list_item_global);
448 lp->nr_fs_variants--;
449 lp->nr_fs_instrs -= variant->nr_instrs;
450
451 FREE(variant);
452 }
453
454 static void
455 llvmpipe_delete_compute_state(struct pipe_context *pipe,
456 void *cs)
457 {
458 struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe);
459 struct lp_compute_shader *shader = cs;
460 struct lp_cs_variant_list_item *li;
461
462 /* Delete all the variants */
463 li = first_elem(&shader->variants);
464 while(!at_end(&shader->variants, li)) {
465 struct lp_cs_variant_list_item *next = next_elem(li);
466 llvmpipe_remove_cs_shader_variant(llvmpipe, li->base);
467 li = next;
468 }
469 tgsi_free_tokens(shader->base.tokens);
470 FREE(shader);
471 }
472
473 static void
474 make_variant_key(struct llvmpipe_context *lp,
475 struct lp_compute_shader *shader,
476 struct lp_compute_shader_variant_key *key)
477 {
478 int i;
479
480 memset(key, 0, shader->variant_key_size);
481
482 /* This value will be the same for all the variants of a given shader:
483 */
484 key->nr_samplers = shader->info.base.file_max[TGSI_FILE_SAMPLER] + 1;
485
486 for(i = 0; i < key->nr_samplers; ++i) {
487 if(shader->info.base.file_mask[TGSI_FILE_SAMPLER] & (1 << i)) {
488 lp_sampler_static_sampler_state(&key->state[i].sampler_state,
489 lp->samplers[PIPE_SHADER_COMPUTE][i]);
490 }
491 }
492
493 /*
494 * XXX If TGSI_FILE_SAMPLER_VIEW exists assume all texture opcodes
495 * are dx10-style? Can't really have mixed opcodes, at least not
496 * if we want to skip the holes here (without rescanning tgsi).
497 */
498 if (shader->info.base.file_max[TGSI_FILE_SAMPLER_VIEW] != -1) {
499 key->nr_sampler_views = shader->info.base.file_max[TGSI_FILE_SAMPLER_VIEW] + 1;
500 for(i = 0; i < key->nr_sampler_views; ++i) {
501 /*
502 * Note sview may exceed what's representable by file_mask.
503 * This will still work, the only downside is that not actually
504 * used views may be included in the shader key.
505 */
506 if(shader->info.base.file_mask[TGSI_FILE_SAMPLER_VIEW] & (1u << (i & 31))) {
507 lp_sampler_static_texture_state(&key->state[i].texture_state,
508 lp->sampler_views[PIPE_SHADER_COMPUTE][i]);
509 }
510 }
511 }
512 else {
513 key->nr_sampler_views = key->nr_samplers;
514 for(i = 0; i < key->nr_sampler_views; ++i) {
515 if(shader->info.base.file_mask[TGSI_FILE_SAMPLER] & (1 << i)) {
516 lp_sampler_static_texture_state(&key->state[i].texture_state,
517 lp->sampler_views[PIPE_SHADER_COMPUTE][i]);
518 }
519 }
520 }
521
522 key->nr_images = shader->info.base.file_max[TGSI_FILE_IMAGE] + 1;
523 for (i = 0; i < key->nr_images; ++i) {
524 if (shader->info.base.file_mask[TGSI_FILE_IMAGE] & (1 << i)) {
525 lp_sampler_static_texture_state_image(&key->image_state[i].image_state,
526 &lp->images[PIPE_SHADER_COMPUTE][i]);
527 }
528 }
529 }
530
531 static void
532 dump_cs_variant_key(const struct lp_compute_shader_variant_key *key)
533 {
534 int i;
535 debug_printf("cs variant %p:\n", (void *) key);
536
537 for (i = 0; i < key->nr_samplers; ++i) {
538 const struct lp_static_sampler_state *sampler = &key->state[i].sampler_state;
539 debug_printf("sampler[%u] = \n", i);
540 debug_printf(" .wrap = %s %s %s\n",
541 util_str_tex_wrap(sampler->wrap_s, TRUE),
542 util_str_tex_wrap(sampler->wrap_t, TRUE),
543 util_str_tex_wrap(sampler->wrap_r, TRUE));
544 debug_printf(" .min_img_filter = %s\n",
545 util_str_tex_filter(sampler->min_img_filter, TRUE));
546 debug_printf(" .min_mip_filter = %s\n",
547 util_str_tex_mipfilter(sampler->min_mip_filter, TRUE));
548 debug_printf(" .mag_img_filter = %s\n",
549 util_str_tex_filter(sampler->mag_img_filter, TRUE));
550 if (sampler->compare_mode != PIPE_TEX_COMPARE_NONE)
551 debug_printf(" .compare_func = %s\n", util_str_func(sampler->compare_func, TRUE));
552 debug_printf(" .normalized_coords = %u\n", sampler->normalized_coords);
553 debug_printf(" .min_max_lod_equal = %u\n", sampler->min_max_lod_equal);
554 debug_printf(" .lod_bias_non_zero = %u\n", sampler->lod_bias_non_zero);
555 debug_printf(" .apply_min_lod = %u\n", sampler->apply_min_lod);
556 debug_printf(" .apply_max_lod = %u\n", sampler->apply_max_lod);
557 }
558 for (i = 0; i < key->nr_sampler_views; ++i) {
559 const struct lp_static_texture_state *texture = &key->state[i].texture_state;
560 debug_printf("texture[%u] = \n", i);
561 debug_printf(" .format = %s\n",
562 util_format_name(texture->format));
563 debug_printf(" .target = %s\n",
564 util_str_tex_target(texture->target, TRUE));
565 debug_printf(" .level_zero_only = %u\n",
566 texture->level_zero_only);
567 debug_printf(" .pot = %u %u %u\n",
568 texture->pot_width,
569 texture->pot_height,
570 texture->pot_depth);
571 }
572 for (i = 0; i < key->nr_images; ++i) {
573 const struct lp_static_texture_state *image = &key->image_state[i].image_state;
574 debug_printf("image[%u] = \n", i);
575 debug_printf(" .format = %s\n",
576 util_format_name(image->format));
577 debug_printf(" .target = %s\n",
578 util_str_tex_target(image->target, TRUE));
579 debug_printf(" .level_zero_only = %u\n",
580 image->level_zero_only);
581 debug_printf(" .pot = %u %u %u\n",
582 image->pot_width,
583 image->pot_height,
584 image->pot_depth);
585 }
586 }
587
588 static void
589 lp_debug_cs_variant(const struct lp_compute_shader_variant *variant)
590 {
591 debug_printf("llvmpipe: Compute shader #%u variant #%u:\n",
592 variant->shader->no, variant->no);
593 tgsi_dump(variant->shader->base.tokens, 0);
594 dump_cs_variant_key(&variant->key);
595 debug_printf("\n");
596 }
597
598 static struct lp_compute_shader_variant *
599 generate_variant(struct llvmpipe_context *lp,
600 struct lp_compute_shader *shader,
601 const struct lp_compute_shader_variant_key *key)
602 {
603 struct lp_compute_shader_variant *variant;
604 char module_name[64];
605
606 variant = CALLOC_STRUCT(lp_compute_shader_variant);
607 if (!variant)
608 return NULL;
609
610 snprintf(module_name, sizeof(module_name), "cs%u_variant%u",
611 shader->no, shader->variants_created);
612
613 variant->gallivm = gallivm_create(module_name, lp->context);
614 if (!variant->gallivm) {
615 FREE(variant);
616 return NULL;
617 }
618
619 variant->shader = shader;
620 variant->list_item_global.base = variant;
621 variant->list_item_local.base = variant;
622 variant->no = shader->variants_created++;
623
624 memcpy(&variant->key, key, shader->variant_key_size);
625
626 if ((LP_DEBUG & DEBUG_CS) || (gallivm_debug & GALLIVM_DEBUG_IR)) {
627 lp_debug_cs_variant(variant);
628 }
629
630 lp_jit_init_cs_types(variant);
631
632 generate_compute(lp, shader, variant);
633
634 gallivm_compile_module(variant->gallivm);
635
636 variant->nr_instrs += lp_build_count_ir_module(variant->gallivm->module);
637
638 variant->jit_function = (lp_jit_cs_func)gallivm_jit_function(variant->gallivm, variant->function);
639
640 gallivm_free_ir(variant->gallivm);
641 return variant;
642 }
643
644 static void
645 lp_cs_ctx_set_cs_variant( struct lp_cs_context *csctx,
646 struct lp_compute_shader_variant *variant)
647 {
648 csctx->cs.current.variant = variant;
649 }
650
651 static void
652 llvmpipe_update_cs(struct llvmpipe_context *lp)
653 {
654 struct lp_compute_shader *shader = lp->cs;
655
656 struct lp_compute_shader_variant_key key;
657 struct lp_compute_shader_variant *variant = NULL;
658 struct lp_cs_variant_list_item *li;
659
660 make_variant_key(lp, shader, &key);
661
662 /* Search the variants for one which matches the key */
663 li = first_elem(&shader->variants);
664 while(!at_end(&shader->variants, li)) {
665 if(memcmp(&li->base->key, &key, shader->variant_key_size) == 0) {
666 variant = li->base;
667 break;
668 }
669 li = next_elem(li);
670 }
671
672 if (variant) {
673 /* Move this variant to the head of the list to implement LRU
674 * deletion of shader's when we have too many.
675 */
676 move_to_head(&lp->cs_variants_list, &variant->list_item_global);
677 }
678 else {
679 /* variant not found, create it now */
680 int64_t t0, t1, dt;
681 unsigned i;
682 unsigned variants_to_cull;
683
684 if (LP_DEBUG & DEBUG_CS) {
685 debug_printf("%u variants,\t%u instrs,\t%u instrs/variant\n",
686 lp->nr_cs_variants,
687 lp->nr_cs_instrs,
688 lp->nr_cs_variants ? lp->nr_cs_instrs / lp->nr_cs_variants : 0);
689 }
690
691 /* First, check if we've exceeded the max number of shader variants.
692 * If so, free 6.25% of them (the least recently used ones).
693 */
694 variants_to_cull = lp->nr_cs_variants >= LP_MAX_SHADER_VARIANTS ? LP_MAX_SHADER_VARIANTS / 16 : 0;
695
696 if (variants_to_cull ||
697 lp->nr_cs_instrs >= LP_MAX_SHADER_INSTRUCTIONS) {
698 if (gallivm_debug & GALLIVM_DEBUG_PERF) {
699 debug_printf("Evicting CS: %u cs variants,\t%u total variants,"
700 "\t%u instrs,\t%u instrs/variant\n",
701 shader->variants_cached,
702 lp->nr_cs_variants, lp->nr_cs_instrs,
703 lp->nr_cs_instrs / lp->nr_cs_variants);
704 }
705
706 /*
707 * We need to re-check lp->nr_cs_variants because an arbitrarliy large
708 * number of shader variants (potentially all of them) could be
709 * pending for destruction on flush.
710 */
711
712 for (i = 0; i < variants_to_cull || lp->nr_cs_instrs >= LP_MAX_SHADER_INSTRUCTIONS; i++) {
713 struct lp_cs_variant_list_item *item;
714 if (is_empty_list(&lp->cs_variants_list)) {
715 break;
716 }
717 item = last_elem(&lp->cs_variants_list);
718 assert(item);
719 assert(item->base);
720 llvmpipe_remove_cs_shader_variant(lp, item->base);
721 }
722 }
723 /*
724 * Generate the new variant.
725 */
726 t0 = os_time_get();
727 variant = generate_variant(lp, shader, &key);
728 t1 = os_time_get();
729 dt = t1 - t0;
730 LP_COUNT_ADD(llvm_compile_time, dt);
731 LP_COUNT_ADD(nr_llvm_compiles, 2); /* emit vs. omit in/out test */
732
733 /* Put the new variant into the list */
734 if (variant) {
735 insert_at_head(&shader->variants, &variant->list_item_local);
736 insert_at_head(&lp->cs_variants_list, &variant->list_item_global);
737 lp->nr_cs_variants++;
738 lp->nr_cs_instrs += variant->nr_instrs;
739 shader->variants_cached++;
740 }
741 }
742 /* Bind this variant */
743 lp_cs_ctx_set_cs_variant(lp->csctx, variant);
744 }
745
746 /**
747 * Called during state validation when LP_CSNEW_SAMPLER_VIEW is set.
748 */
749 static void
750 lp_csctx_set_sampler_views(struct lp_cs_context *csctx,
751 unsigned num,
752 struct pipe_sampler_view **views)
753 {
754 unsigned i, max_tex_num;
755
756 LP_DBG(DEBUG_SETUP, "%s\n", __FUNCTION__);
757
758 assert(num <= PIPE_MAX_SHADER_SAMPLER_VIEWS);
759
760 max_tex_num = MAX2(num, csctx->cs.current_tex_num);
761
762 for (i = 0; i < max_tex_num; i++) {
763 struct pipe_sampler_view *view = i < num ? views[i] : NULL;
764
765 if (view) {
766 struct pipe_resource *res = view->texture;
767 struct llvmpipe_resource *lp_tex = llvmpipe_resource(res);
768 struct lp_jit_texture *jit_tex;
769 jit_tex = &csctx->cs.current.jit_context.textures[i];
770
771 /* We're referencing the texture's internal data, so save a
772 * reference to it.
773 */
774 pipe_resource_reference(&csctx->cs.current_tex[i], res);
775
776 if (!lp_tex->dt) {
777 /* regular texture - csctx array of mipmap level offsets */
778 int j;
779 unsigned first_level = 0;
780 unsigned last_level = 0;
781
782 if (llvmpipe_resource_is_texture(res)) {
783 first_level = view->u.tex.first_level;
784 last_level = view->u.tex.last_level;
785 assert(first_level <= last_level);
786 assert(last_level <= res->last_level);
787 jit_tex->base = lp_tex->tex_data;
788 }
789 else {
790 jit_tex->base = lp_tex->data;
791 }
792 if (LP_PERF & PERF_TEX_MEM) {
793 /* use dummy tile memory */
794 jit_tex->base = lp_dummy_tile;
795 jit_tex->width = TILE_SIZE/8;
796 jit_tex->height = TILE_SIZE/8;
797 jit_tex->depth = 1;
798 jit_tex->first_level = 0;
799 jit_tex->last_level = 0;
800 jit_tex->mip_offsets[0] = 0;
801 jit_tex->row_stride[0] = 0;
802 jit_tex->img_stride[0] = 0;
803 }
804 else {
805 jit_tex->width = res->width0;
806 jit_tex->height = res->height0;
807 jit_tex->depth = res->depth0;
808 jit_tex->first_level = first_level;
809 jit_tex->last_level = last_level;
810
811 if (llvmpipe_resource_is_texture(res)) {
812 for (j = first_level; j <= last_level; j++) {
813 jit_tex->mip_offsets[j] = lp_tex->mip_offsets[j];
814 jit_tex->row_stride[j] = lp_tex->row_stride[j];
815 jit_tex->img_stride[j] = lp_tex->img_stride[j];
816 }
817
818 if (res->target == PIPE_TEXTURE_1D_ARRAY ||
819 res->target == PIPE_TEXTURE_2D_ARRAY ||
820 res->target == PIPE_TEXTURE_CUBE ||
821 res->target == PIPE_TEXTURE_CUBE_ARRAY) {
822 /*
823 * For array textures, we don't have first_layer, instead
824 * adjust last_layer (stored as depth) plus the mip level offsets
825 * (as we have mip-first layout can't just adjust base ptr).
826 * XXX For mip levels, could do something similar.
827 */
828 jit_tex->depth = view->u.tex.last_layer - view->u.tex.first_layer + 1;
829 for (j = first_level; j <= last_level; j++) {
830 jit_tex->mip_offsets[j] += view->u.tex.first_layer *
831 lp_tex->img_stride[j];
832 }
833 if (view->target == PIPE_TEXTURE_CUBE ||
834 view->target == PIPE_TEXTURE_CUBE_ARRAY) {
835 assert(jit_tex->depth % 6 == 0);
836 }
837 assert(view->u.tex.first_layer <= view->u.tex.last_layer);
838 assert(view->u.tex.last_layer < res->array_size);
839 }
840 }
841 else {
842 /*
843 * For buffers, we don't have "offset", instead adjust
844 * the size (stored as width) plus the base pointer.
845 */
846 unsigned view_blocksize = util_format_get_blocksize(view->format);
847 /* probably don't really need to fill that out */
848 jit_tex->mip_offsets[0] = 0;
849 jit_tex->row_stride[0] = 0;
850 jit_tex->img_stride[0] = 0;
851
852 /* everything specified in number of elements here. */
853 jit_tex->width = view->u.buf.size / view_blocksize;
854 jit_tex->base = (uint8_t *)jit_tex->base + view->u.buf.offset;
855 /* XXX Unsure if we need to sanitize parameters? */
856 assert(view->u.buf.offset + view->u.buf.size <= res->width0);
857 }
858 }
859 }
860 else {
861 /* display target texture/surface */
862 /*
863 * XXX: Where should this be unmapped?
864 */
865 struct llvmpipe_screen *screen = llvmpipe_screen(res->screen);
866 struct sw_winsys *winsys = screen->winsys;
867 jit_tex->base = winsys->displaytarget_map(winsys, lp_tex->dt,
868 PIPE_TRANSFER_READ);
869 jit_tex->row_stride[0] = lp_tex->row_stride[0];
870 jit_tex->img_stride[0] = lp_tex->img_stride[0];
871 jit_tex->mip_offsets[0] = 0;
872 jit_tex->width = res->width0;
873 jit_tex->height = res->height0;
874 jit_tex->depth = res->depth0;
875 jit_tex->first_level = jit_tex->last_level = 0;
876 assert(jit_tex->base);
877 }
878 }
879 else {
880 pipe_resource_reference(&csctx->cs.current_tex[i], NULL);
881 }
882 }
883 csctx->cs.current_tex_num = num;
884 }
885
886
887 /**
888 * Called during state validation when LP_NEW_SAMPLER is set.
889 */
890 static void
891 lp_csctx_set_sampler_state(struct lp_cs_context *csctx,
892 unsigned num,
893 struct pipe_sampler_state **samplers)
894 {
895 unsigned i;
896
897 LP_DBG(DEBUG_SETUP, "%s\n", __FUNCTION__);
898
899 assert(num <= PIPE_MAX_SAMPLERS);
900
901 for (i = 0; i < PIPE_MAX_SAMPLERS; i++) {
902 const struct pipe_sampler_state *sampler = i < num ? samplers[i] : NULL;
903
904 if (sampler) {
905 struct lp_jit_sampler *jit_sam;
906 jit_sam = &csctx->cs.current.jit_context.samplers[i];
907
908 jit_sam->min_lod = sampler->min_lod;
909 jit_sam->max_lod = sampler->max_lod;
910 jit_sam->lod_bias = sampler->lod_bias;
911 COPY_4V(jit_sam->border_color, sampler->border_color.f);
912 }
913 }
914 }
915
916 static void
917 lp_csctx_set_cs_constants(struct lp_cs_context *csctx,
918 unsigned num,
919 struct pipe_constant_buffer *buffers)
920 {
921 unsigned i;
922
923 LP_DBG(DEBUG_SETUP, "%s %p\n", __FUNCTION__, (void *) buffers);
924
925 assert(num <= ARRAY_SIZE(csctx->constants));
926
927 for (i = 0; i < num; ++i) {
928 util_copy_constant_buffer(&csctx->constants[i].current, &buffers[i]);
929 }
930 for (; i < ARRAY_SIZE(csctx->constants); i++) {
931 util_copy_constant_buffer(&csctx->constants[i].current, NULL);
932 }
933 }
934
935 static void
936 lp_csctx_set_cs_ssbos(struct lp_cs_context *csctx,
937 unsigned num,
938 struct pipe_shader_buffer *buffers)
939 {
940 int i;
941 LP_DBG(DEBUG_SETUP, "%s %p\n", __FUNCTION__, (void *)buffers);
942
943 assert (num <= ARRAY_SIZE(csctx->ssbos));
944
945 for (i = 0; i < num; ++i) {
946 util_copy_shader_buffer(&csctx->ssbos[i].current, &buffers[i]);
947 }
948 for (; i < ARRAY_SIZE(csctx->ssbos); i++) {
949 util_copy_shader_buffer(&csctx->ssbos[i].current, NULL);
950 }
951 }
952
953 static void
954 lp_csctx_set_cs_images(struct lp_cs_context *csctx,
955 unsigned num,
956 struct pipe_image_view *images)
957 {
958 unsigned i;
959
960 LP_DBG(DEBUG_SETUP, "%s %p\n", __FUNCTION__, (void *) images);
961
962 assert(num <= ARRAY_SIZE(csctx->images));
963
964 for (i = 0; i < num; ++i) {
965 struct pipe_image_view *image = &images[i];
966 util_copy_image_view(&csctx->images[i].current, &images[i]);
967
968 struct pipe_resource *res = image->resource;
969 struct llvmpipe_resource *lp_res = llvmpipe_resource(res);
970 struct lp_jit_image *jit_image;
971
972 jit_image = &csctx->cs.current.jit_context.images[i];
973 if (!lp_res)
974 continue;
975 if (!lp_res->dt) {
976 /* regular texture - csctx array of mipmap level offsets */
977 if (llvmpipe_resource_is_texture(res)) {
978 jit_image->base = lp_res->tex_data;
979 } else
980 jit_image->base = lp_res->data;
981
982 jit_image->width = res->width0;
983 jit_image->height = res->height0;
984 jit_image->depth = res->depth0;
985
986 if (llvmpipe_resource_is_texture(res)) {
987 uint32_t mip_offset = lp_res->mip_offsets[image->u.tex.level];
988
989 jit_image->width = u_minify(jit_image->width, image->u.tex.level);
990 jit_image->height = u_minify(jit_image->height, image->u.tex.level);
991
992 if (res->target == PIPE_TEXTURE_1D_ARRAY ||
993 res->target == PIPE_TEXTURE_2D_ARRAY ||
994 res->target == PIPE_TEXTURE_3D ||
995 res->target == PIPE_TEXTURE_CUBE ||
996 res->target == PIPE_TEXTURE_CUBE_ARRAY) {
997 /*
998 * For array textures, we don't have first_layer, instead
999 * adjust last_layer (stored as depth) plus the mip level offsets
1000 * (as we have mip-first layout can't just adjust base ptr).
1001 * XXX For mip levels, could do something similar.
1002 */
1003 jit_image->depth = image->u.tex.last_layer - image->u.tex.first_layer + 1;
1004 mip_offset += image->u.tex.first_layer * lp_res->img_stride[image->u.tex.level];
1005 } else
1006 jit_image->depth = u_minify(jit_image->depth, image->u.tex.level);
1007
1008 jit_image->row_stride = lp_res->row_stride[image->u.tex.level];
1009 jit_image->img_stride = lp_res->img_stride[image->u.tex.level];
1010 jit_image->base = (uint8_t *)jit_image->base + mip_offset;
1011 } else {
1012 unsigned view_blocksize = util_format_get_blocksize(image->format);
1013 jit_image->width = image->u.buf.size / view_blocksize;
1014 jit_image->base = (uint8_t *)jit_image->base + image->u.buf.offset;
1015 }
1016 }
1017 }
1018 for (; i < ARRAY_SIZE(csctx->images); i++) {
1019 util_copy_image_view(&csctx->images[i].current, NULL);
1020 }
1021 }
1022
1023 static void
1024 update_csctx_consts(struct llvmpipe_context *llvmpipe)
1025 {
1026 struct lp_cs_context *csctx = llvmpipe->csctx;
1027 int i;
1028
1029 for (i = 0; i < ARRAY_SIZE(csctx->constants); ++i) {
1030 struct pipe_resource *buffer = csctx->constants[i].current.buffer;
1031 const ubyte *current_data = NULL;
1032
1033 if (buffer) {
1034 /* resource buffer */
1035 current_data = (ubyte *) llvmpipe_resource_data(buffer);
1036 }
1037 else if (csctx->constants[i].current.user_buffer) {
1038 /* user-space buffer */
1039 current_data = (ubyte *) csctx->constants[i].current.user_buffer;
1040 }
1041
1042 if (current_data) {
1043 current_data += csctx->constants[i].current.buffer_offset;
1044
1045 csctx->cs.current.jit_context.constants[i] = (const float *)current_data;
1046 csctx->cs.current.jit_context.num_constants[i] = csctx->constants[i].current.buffer_size;
1047 } else {
1048 csctx->cs.current.jit_context.constants[i] = NULL;
1049 csctx->cs.current.jit_context.num_constants[i] = 0;
1050 }
1051 }
1052 }
1053
1054 static void
1055 update_csctx_ssbo(struct llvmpipe_context *llvmpipe)
1056 {
1057 struct lp_cs_context *csctx = llvmpipe->csctx;
1058 int i;
1059 for (i = 0; i < ARRAY_SIZE(csctx->ssbos); ++i) {
1060 struct pipe_resource *buffer = csctx->ssbos[i].current.buffer;
1061 const ubyte *current_data = NULL;
1062
1063 if (!buffer)
1064 continue;
1065 /* resource buffer */
1066 current_data = (ubyte *) llvmpipe_resource_data(buffer);
1067 if (current_data) {
1068 current_data += csctx->ssbos[i].current.buffer_offset;
1069
1070 csctx->cs.current.jit_context.ssbos[i] = (const uint32_t *)current_data;
1071 csctx->cs.current.jit_context.num_ssbos[i] = csctx->ssbos[i].current.buffer_size;
1072 } else {
1073 csctx->cs.current.jit_context.ssbos[i] = NULL;
1074 csctx->cs.current.jit_context.num_ssbos[i] = 0;
1075 }
1076 }
1077 }
1078
1079 static void
1080 llvmpipe_cs_update_derived(struct llvmpipe_context *llvmpipe)
1081 {
1082 if (llvmpipe->cs_dirty & (LP_CSNEW_CS))
1083 llvmpipe_update_cs(llvmpipe);
1084
1085 if (llvmpipe->cs_dirty & LP_CSNEW_CONSTANTS) {
1086 lp_csctx_set_cs_constants(llvmpipe->csctx,
1087 ARRAY_SIZE(llvmpipe->constants[PIPE_SHADER_COMPUTE]),
1088 llvmpipe->constants[PIPE_SHADER_COMPUTE]);
1089 update_csctx_consts(llvmpipe);
1090 }
1091
1092 if (llvmpipe->cs_dirty & LP_CSNEW_SSBOS) {
1093 lp_csctx_set_cs_ssbos(llvmpipe->csctx,
1094 ARRAY_SIZE(llvmpipe->ssbos[PIPE_SHADER_COMPUTE]),
1095 llvmpipe->ssbos[PIPE_SHADER_COMPUTE]);
1096 update_csctx_ssbo(llvmpipe);
1097 }
1098
1099 if (llvmpipe->cs_dirty & LP_CSNEW_SAMPLER_VIEW)
1100 lp_csctx_set_sampler_views(llvmpipe->csctx,
1101 llvmpipe->num_sampler_views[PIPE_SHADER_COMPUTE],
1102 llvmpipe->sampler_views[PIPE_SHADER_COMPUTE]);
1103
1104 if (llvmpipe->cs_dirty & LP_CSNEW_SAMPLER)
1105 lp_csctx_set_sampler_state(llvmpipe->csctx,
1106 llvmpipe->num_samplers[PIPE_SHADER_COMPUTE],
1107 llvmpipe->samplers[PIPE_SHADER_COMPUTE]);
1108
1109 if (llvmpipe->cs_dirty & LP_CSNEW_IMAGES)
1110 lp_csctx_set_cs_images(llvmpipe->csctx,
1111 ARRAY_SIZE(llvmpipe->images[PIPE_SHADER_COMPUTE]),
1112 llvmpipe->images[PIPE_SHADER_COMPUTE]);
1113
1114 llvmpipe->cs_dirty = 0;
1115 }
1116
1117 static void
1118 cs_exec_fn(void *init_data, int iter_idx, struct lp_cs_local_mem *lmem)
1119 {
1120 struct lp_cs_job_info *job_info = init_data;
1121 struct lp_jit_cs_thread_data thread_data;
1122
1123 memset(&thread_data, 0, sizeof(thread_data));
1124
1125 if (lmem->local_size < job_info->req_local_mem) {
1126 lmem->local_mem_ptr = REALLOC(lmem->local_mem_ptr, lmem->local_size,
1127 job_info->req_local_mem);
1128 lmem->local_size = job_info->req_local_mem;
1129 }
1130 thread_data.shared = lmem->local_mem_ptr;
1131
1132 unsigned grid_z = iter_idx / (job_info->grid_size[0] * job_info->grid_size[1]);
1133 unsigned grid_y = (iter_idx - (grid_z * (job_info->grid_size[0] * job_info->grid_size[1]))) / job_info->grid_size[0];
1134 unsigned grid_x = (iter_idx - (grid_z * (job_info->grid_size[0] * job_info->grid_size[1])) - (grid_y * job_info->grid_size[0]));
1135 struct lp_compute_shader_variant *variant = job_info->current->variant;
1136 variant->jit_function(&job_info->current->jit_context,
1137 job_info->block_size[0], job_info->block_size[1], job_info->block_size[2],
1138 grid_x, grid_y, grid_z,
1139 job_info->grid_size[0], job_info->grid_size[1], job_info->grid_size[2],
1140 &thread_data);
1141 }
1142
1143 static void
1144 fill_grid_size(struct pipe_context *pipe,
1145 const struct pipe_grid_info *info,
1146 uint32_t grid_size[3])
1147 {
1148 struct pipe_transfer *transfer;
1149 uint32_t *params;
1150 if (!info->indirect) {
1151 grid_size[0] = info->grid[0];
1152 grid_size[1] = info->grid[1];
1153 grid_size[2] = info->grid[2];
1154 return;
1155 }
1156 params = pipe_buffer_map_range(pipe, info->indirect,
1157 info->indirect_offset,
1158 3 * sizeof(uint32_t),
1159 PIPE_TRANSFER_READ,
1160 &transfer);
1161
1162 if (!transfer)
1163 return;
1164
1165 grid_size[0] = params[0];
1166 grid_size[1] = params[1];
1167 grid_size[2] = params[2];
1168 pipe_buffer_unmap(pipe, transfer);
1169 }
1170
1171 static void llvmpipe_launch_grid(struct pipe_context *pipe,
1172 const struct pipe_grid_info *info)
1173 {
1174 struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe);
1175 struct llvmpipe_screen *screen = llvmpipe_screen(pipe->screen);
1176 struct lp_cs_job_info job_info;
1177
1178 memset(&job_info, 0, sizeof(job_info));
1179
1180 llvmpipe_cs_update_derived(llvmpipe);
1181
1182 fill_grid_size(pipe, info, job_info.grid_size);
1183
1184 job_info.block_size[0] = info->block[0];
1185 job_info.block_size[1] = info->block[1];
1186 job_info.block_size[2] = info->block[2];
1187 job_info.req_local_mem = llvmpipe->cs->req_local_mem;
1188 job_info.current = &llvmpipe->csctx->cs.current;
1189
1190 int num_tasks = job_info.grid_size[2] * job_info.grid_size[1] * job_info.grid_size[0];
1191 if (num_tasks) {
1192 struct lp_cs_tpool_task *task;
1193 mtx_lock(&screen->cs_mutex);
1194 task = lp_cs_tpool_queue_task(screen->cs_tpool, cs_exec_fn, &job_info, num_tasks);
1195
1196 lp_cs_tpool_wait_for_task(screen->cs_tpool, &task);
1197 mtx_unlock(&screen->cs_mutex);
1198 }
1199 llvmpipe->pipeline_statistics.cs_invocations += num_tasks * info->block[0] * info->block[1] * info->block[2];
1200 }
1201
1202 void
1203 llvmpipe_init_compute_funcs(struct llvmpipe_context *llvmpipe)
1204 {
1205 llvmpipe->pipe.create_compute_state = llvmpipe_create_compute_state;
1206 llvmpipe->pipe.bind_compute_state = llvmpipe_bind_compute_state;
1207 llvmpipe->pipe.delete_compute_state = llvmpipe_delete_compute_state;
1208 llvmpipe->pipe.launch_grid = llvmpipe_launch_grid;
1209 }
1210
1211 void
1212 lp_csctx_destroy(struct lp_cs_context *csctx)
1213 {
1214 unsigned i;
1215 for (i = 0; i < ARRAY_SIZE(csctx->cs.current_tex); i++) {
1216 pipe_resource_reference(&csctx->cs.current_tex[i], NULL);
1217 }
1218 for (i = 0; i < ARRAY_SIZE(csctx->constants); i++) {
1219 pipe_resource_reference(&csctx->constants[i].current.buffer, NULL);
1220 }
1221 for (i = 0; i < ARRAY_SIZE(csctx->ssbos); i++) {
1222 pipe_resource_reference(&csctx->ssbos[i].current.buffer, NULL);
1223 }
1224 FREE(csctx);
1225 }
1226
1227 struct lp_cs_context *lp_csctx_create(struct pipe_context *pipe)
1228 {
1229 struct lp_cs_context *csctx;
1230
1231 csctx = CALLOC_STRUCT(lp_cs_context);
1232 if (!csctx)
1233 return NULL;
1234
1235 csctx->pipe = pipe;
1236 return csctx;
1237 }