projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
gallivm: rework coroutine malloc/free callouts.
[mesa.git] / src / gallium / drivers / llvmpipe / lp_state_cs.c
1 /**************************************************************************
2 *
3 * Copyright 2019 Red Hat.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included
14 * in all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
17 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 *
24 **************************************************************************/
25 #include "util/u_memory.h"
26 #include "util/simple_list.h"
27 #include "util/os_time.h"
28 #include "util/u_dump.h"
29 #include "util/u_string.h"
30 #include "tgsi/tgsi_dump.h"
31 #include "tgsi/tgsi_parse.h"
32 #include "gallivm/lp_bld_const.h"
33 #include "gallivm/lp_bld_debug.h"
34 #include "gallivm/lp_bld_intr.h"
35 #include "gallivm/lp_bld_flow.h"
36 #include "gallivm/lp_bld_gather.h"
37 #include "gallivm/lp_bld_coro.h"
38 #include "gallivm/lp_bld_nir.h"
39 #include "lp_state_cs.h"
40 #include "lp_context.h"
41 #include "lp_debug.h"
42 #include "lp_state.h"
43 #include "lp_perf.h"
44 #include "lp_screen.h"
45 #include "lp_memory.h"
46 #include "lp_cs_tpool.h"
47 #include "frontend/sw_winsys.h"
48 #include "nir/nir_to_tgsi_info.h"
49 #include "nir_serialize.h"
50
51 /** Fragment shader number (for debugging) */
52 static unsigned cs_no = 0;
53
54 struct lp_cs_job_info {
55 unsigned grid_size[3];
56 unsigned block_size[3];
57 unsigned req_local_mem;
58 unsigned work_dim;
59 struct lp_cs_exec *current;
60 };
61
62 static void
63 generate_compute(struct llvmpipe_context *lp,
64 struct lp_compute_shader *shader,
65 struct lp_compute_shader_variant *variant)
66 {
67 struct gallivm_state *gallivm = variant->gallivm;
68 const struct lp_compute_shader_variant_key *key = &variant->key;
69 char func_name[64], func_name_coro[64];
70 LLVMTypeRef arg_types[17];
71 LLVMTypeRef func_type, coro_func_type;
72 LLVMTypeRef int32_type = LLVMInt32TypeInContext(gallivm->context);
73 LLVMValueRef context_ptr;
74 LLVMValueRef x_size_arg, y_size_arg, z_size_arg;
75 LLVMValueRef grid_x_arg, grid_y_arg, grid_z_arg;
76 LLVMValueRef grid_size_x_arg, grid_size_y_arg, grid_size_z_arg;
77 LLVMValueRef work_dim_arg, thread_data_ptr;
78 LLVMBasicBlockRef block;
79 LLVMBuilderRef builder;
80 struct lp_build_sampler_soa *sampler;
81 struct lp_build_image_soa *image;
82 LLVMValueRef function, coro;
83 struct lp_type cs_type;
84 unsigned i;
85
86 /*
87 * This function has two parts
88 * a) setup the coroutine execution environment loop.
89 * b) build the compute shader llvm for use inside the coroutine.
90 */
91 assert(lp_native_vector_width / 32 >= 4);
92
93 memset(&cs_type, 0, sizeof cs_type);
94 cs_type.floating = TRUE; /* floating point values */
95 cs_type.sign = TRUE; /* values are signed */
96 cs_type.norm = FALSE; /* values are not limited to [0,1] or [-1,1] */
97 cs_type.width = 32; /* 32-bit float */
98 cs_type.length = MIN2(lp_native_vector_width / 32, 16); /* n*4 elements per vector */
99 snprintf(func_name, sizeof(func_name), "cs_variant");
100
101 snprintf(func_name_coro, sizeof(func_name), "cs_co_variant");
102
103 arg_types[0] = variant->jit_cs_context_ptr_type; /* context */
104 arg_types[1] = int32_type; /* block_x_size */
105 arg_types[2] = int32_type; /* block_y_size */
106 arg_types[3] = int32_type; /* block_z_size */
107 arg_types[4] = int32_type; /* grid_x */
108 arg_types[5] = int32_type; /* grid_y */
109 arg_types[6] = int32_type; /* grid_z */
110 arg_types[7] = int32_type; /* grid_size_x */
111 arg_types[8] = int32_type; /* grid_size_y */
112 arg_types[9] = int32_type; /* grid_size_z */
113 arg_types[10] = int32_type; /* work dim */
114 arg_types[11] = variant->jit_cs_thread_data_ptr_type; /* per thread data */
115 arg_types[12] = int32_type; /* coro only - num X loops */
116 arg_types[13] = int32_type; /* coro only - partials */
117 arg_types[14] = int32_type; /* coro block_x_size */
118 arg_types[15] = int32_type; /* coro block_y_size */
119 arg_types[16] = int32_type; /* coro block_z_size */
120 func_type = LLVMFunctionType(LLVMVoidTypeInContext(gallivm->context),
121 arg_types, ARRAY_SIZE(arg_types) - 5, 0);
122
123 coro_func_type = LLVMFunctionType(LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0),
124 arg_types, ARRAY_SIZE(arg_types), 0);
125
126 function = LLVMAddFunction(gallivm->module, func_name, func_type);
127 LLVMSetFunctionCallConv(function, LLVMCCallConv);
128
129 coro = LLVMAddFunction(gallivm->module, func_name_coro, coro_func_type);
130 LLVMSetFunctionCallConv(coro, LLVMCCallConv);
131
132 variant->function = function;
133
134 for(i = 0; i < ARRAY_SIZE(arg_types); ++i) {
135 if(LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind) {
136 lp_add_function_attr(coro, i + 1, LP_FUNC_ATTR_NOALIAS);
137 lp_add_function_attr(function, i + 1, LP_FUNC_ATTR_NOALIAS);
138 }
139 }
140
141 lp_build_coro_declare_malloc_hooks(gallivm);
142
143 context_ptr = LLVMGetParam(function, 0);
144 x_size_arg = LLVMGetParam(function, 1);
145 y_size_arg = LLVMGetParam(function, 2);
146 z_size_arg = LLVMGetParam(function, 3);
147 grid_x_arg = LLVMGetParam(function, 4);
148 grid_y_arg = LLVMGetParam(function, 5);
149 grid_z_arg = LLVMGetParam(function, 6);
150 grid_size_x_arg = LLVMGetParam(function, 7);
151 grid_size_y_arg = LLVMGetParam(function, 8);
152 grid_size_z_arg = LLVMGetParam(function, 9);
153 work_dim_arg = LLVMGetParam(function, 10);
154 thread_data_ptr = LLVMGetParam(function, 11);
155
156 lp_build_name(context_ptr, "context");
157 lp_build_name(x_size_arg, "x_size");
158 lp_build_name(y_size_arg, "y_size");
159 lp_build_name(z_size_arg, "z_size");
160 lp_build_name(grid_x_arg, "grid_x");
161 lp_build_name(grid_y_arg, "grid_y");
162 lp_build_name(grid_z_arg, "grid_z");
163 lp_build_name(grid_size_x_arg, "grid_size_x");
164 lp_build_name(grid_size_y_arg, "grid_size_y");
165 lp_build_name(grid_size_z_arg, "grid_size_z");
166 lp_build_name(work_dim_arg, "work_dim");
167 lp_build_name(thread_data_ptr, "thread_data");
168
169 block = LLVMAppendBasicBlockInContext(gallivm->context, function, "entry");
170 builder = gallivm->builder;
171 assert(builder);
172 LLVMPositionBuilderAtEnd(builder, block);
173 sampler = lp_llvm_sampler_soa_create(key->samplers);
174 image = lp_llvm_image_soa_create(lp_cs_variant_key_images(key));
175
176 struct lp_build_loop_state loop_state[4];
177 LLVMValueRef num_x_loop;
178 LLVMValueRef vec_length = lp_build_const_int32(gallivm, cs_type.length);
179 num_x_loop = LLVMBuildAdd(gallivm->builder, x_size_arg, vec_length, "");
180 num_x_loop = LLVMBuildSub(gallivm->builder, num_x_loop, lp_build_const_int32(gallivm, 1), "");
181 num_x_loop = LLVMBuildUDiv(gallivm->builder, num_x_loop, vec_length, "");
182 LLVMValueRef partials = LLVMBuildURem(gallivm->builder, x_size_arg, vec_length, "");
183
184 LLVMValueRef coro_num_hdls = LLVMBuildMul(gallivm->builder, num_x_loop, y_size_arg, "");
185 coro_num_hdls = LLVMBuildMul(gallivm->builder, coro_num_hdls, z_size_arg, "");
186
187 LLVMTypeRef hdl_ptr_type = LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0);
188 LLVMValueRef coro_hdls = LLVMBuildArrayAlloca(gallivm->builder, hdl_ptr_type, coro_num_hdls, "coro_hdls");
189
190 unsigned end_coroutine = INT_MAX;
191
192 /*
193 * This is the main coroutine execution loop. It iterates over the dimensions
194 * and calls the coroutine main entrypoint on the first pass, but in subsequent
195 * passes it checks if the coroutine has completed and resumes it if not.
196 */
197 /* take x_width - round up to type.length width */
198 lp_build_loop_begin(&loop_state[3], gallivm,
199 lp_build_const_int32(gallivm, 0)); /* coroutine reentry loop */
200 lp_build_loop_begin(&loop_state[2], gallivm,
201 lp_build_const_int32(gallivm, 0)); /* z loop */
202 lp_build_loop_begin(&loop_state[1], gallivm,
203 lp_build_const_int32(gallivm, 0)); /* y loop */
204 lp_build_loop_begin(&loop_state[0], gallivm,
205 lp_build_const_int32(gallivm, 0)); /* x loop */
206 {
207 LLVMValueRef args[17];
208 args[0] = context_ptr;
209 args[1] = loop_state[0].counter;
210 args[2] = loop_state[1].counter;
211 args[3] = loop_state[2].counter;
212 args[4] = grid_x_arg;
213 args[5] = grid_y_arg;
214 args[6] = grid_z_arg;
215 args[7] = grid_size_x_arg;
216 args[8] = grid_size_y_arg;
217 args[9] = grid_size_z_arg;
218 args[10] = work_dim_arg;
219 args[11] = thread_data_ptr;
220 args[12] = num_x_loop;
221 args[13] = partials;
222 args[14] = x_size_arg;
223 args[15] = y_size_arg;
224 args[16] = z_size_arg;
225
226 /* idx = (z * (size_x * size_y) + y * size_x + x */
227 LLVMValueRef coro_hdl_idx = LLVMBuildMul(gallivm->builder, loop_state[2].counter,
228 LLVMBuildMul(gallivm->builder, num_x_loop, y_size_arg, ""), "");
229 coro_hdl_idx = LLVMBuildAdd(gallivm->builder, coro_hdl_idx,
230 LLVMBuildMul(gallivm->builder, loop_state[1].counter,
231 num_x_loop, ""), "");
232 coro_hdl_idx = LLVMBuildAdd(gallivm->builder, coro_hdl_idx,
233 loop_state[0].counter, "");
234
235 LLVMValueRef coro_entry = LLVMBuildGEP(gallivm->builder, coro_hdls, &coro_hdl_idx, 1, "");
236
237 LLVMValueRef coro_hdl = LLVMBuildLoad(gallivm->builder, coro_entry, "coro_hdl");
238
239 struct lp_build_if_state ifstate;
240 LLVMValueRef cmp = LLVMBuildICmp(gallivm->builder, LLVMIntEQ, loop_state[3].counter,
241 lp_build_const_int32(gallivm, 0), "");
242 /* first time here - call the coroutine function entry point */
243 lp_build_if(&ifstate, gallivm, cmp);
244 LLVMValueRef coro_ret = LLVMBuildCall(gallivm->builder, coro, args, 17, "");
245 LLVMBuildStore(gallivm->builder, coro_ret, coro_entry);
246 lp_build_else(&ifstate);
247 /* subsequent calls for this invocation - check if done. */
248 LLVMValueRef coro_done = lp_build_coro_done(gallivm, coro_hdl);
249 struct lp_build_if_state ifstate2;
250 lp_build_if(&ifstate2, gallivm, coro_done);
251 /* if done destroy and force loop exit */
252 lp_build_coro_destroy(gallivm, coro_hdl);
253 lp_build_loop_force_set_counter(&loop_state[3], lp_build_const_int32(gallivm, end_coroutine - 1));
254 lp_build_else(&ifstate2);
255 /* otherwise resume the coroutine */
256 lp_build_coro_resume(gallivm, coro_hdl);
257 lp_build_endif(&ifstate2);
258 lp_build_endif(&ifstate);
259 lp_build_loop_force_reload_counter(&loop_state[3]);
260 }
261 lp_build_loop_end_cond(&loop_state[0],
262 num_x_loop,
263 NULL, LLVMIntUGE);
264 lp_build_loop_end_cond(&loop_state[1],
265 y_size_arg,
266 NULL, LLVMIntUGE);
267 lp_build_loop_end_cond(&loop_state[2],
268 z_size_arg,
269 NULL, LLVMIntUGE);
270 lp_build_loop_end_cond(&loop_state[3],
271 lp_build_const_int32(gallivm, end_coroutine),
272 NULL, LLVMIntEQ);
273 LLVMBuildRetVoid(builder);
274
275 /* This is stage (b) - generate the compute shader code inside the coroutine. */
276 LLVMValueRef block_x_size_arg, block_y_size_arg, block_z_size_arg;
277 context_ptr = LLVMGetParam(coro, 0);
278 x_size_arg = LLVMGetParam(coro, 1);
279 y_size_arg = LLVMGetParam(coro, 2);
280 z_size_arg = LLVMGetParam(coro, 3);
281 grid_x_arg = LLVMGetParam(coro, 4);
282 grid_y_arg = LLVMGetParam(coro, 5);
283 grid_z_arg = LLVMGetParam(coro, 6);
284 grid_size_x_arg = LLVMGetParam(coro, 7);
285 grid_size_y_arg = LLVMGetParam(coro, 8);
286 grid_size_z_arg = LLVMGetParam(coro, 9);
287 work_dim_arg = LLVMGetParam(coro, 10);
288 thread_data_ptr = LLVMGetParam(coro, 11);
289 num_x_loop = LLVMGetParam(coro, 12);
290 partials = LLVMGetParam(coro, 13);
291 block_x_size_arg = LLVMGetParam(coro, 14);
292 block_y_size_arg = LLVMGetParam(coro, 15);
293 block_z_size_arg = LLVMGetParam(coro, 16);
294 block = LLVMAppendBasicBlockInContext(gallivm->context, coro, "entry");
295 LLVMPositionBuilderAtEnd(builder, block);
296 {
297 LLVMValueRef consts_ptr, num_consts_ptr;
298 LLVMValueRef ssbo_ptr, num_ssbo_ptr;
299 LLVMValueRef shared_ptr;
300 LLVMValueRef kernel_args_ptr;
301 struct lp_build_mask_context mask;
302 struct lp_bld_tgsi_system_values system_values;
303
304 memset(&system_values, 0, sizeof(system_values));
305 consts_ptr = lp_jit_cs_context_constants(gallivm, context_ptr);
306 num_consts_ptr = lp_jit_cs_context_num_constants(gallivm, context_ptr);
307 ssbo_ptr = lp_jit_cs_context_ssbos(gallivm, context_ptr);
308 num_ssbo_ptr = lp_jit_cs_context_num_ssbos(gallivm, context_ptr);
309 kernel_args_ptr = lp_jit_cs_context_kernel_args(gallivm, context_ptr);
310
311 shared_ptr = lp_jit_cs_thread_data_shared(gallivm, thread_data_ptr);
312
313 /* these are coroutine entrypoint necessities */
314 LLVMValueRef coro_id = lp_build_coro_id(gallivm);
315 LLVMValueRef coro_hdl = lp_build_coro_begin_alloc_mem(gallivm, coro_id);
316
317 LLVMValueRef has_partials = LLVMBuildICmp(gallivm->builder, LLVMIntNE, partials, lp_build_const_int32(gallivm, 0), "");
318 LLVMValueRef tid_vals[3];
319 LLVMValueRef tids_x[LP_MAX_VECTOR_LENGTH], tids_y[LP_MAX_VECTOR_LENGTH], tids_z[LP_MAX_VECTOR_LENGTH];
320 LLVMValueRef base_val = LLVMBuildMul(gallivm->builder, x_size_arg, vec_length, "");
321 for (i = 0; i < cs_type.length; i++) {
322 tids_x[i] = LLVMBuildAdd(gallivm->builder, base_val, lp_build_const_int32(gallivm, i), "");
323 tids_y[i] = y_size_arg;
324 tids_z[i] = z_size_arg;
325 }
326 tid_vals[0] = lp_build_gather_values(gallivm, tids_x, cs_type.length);
327 tid_vals[1] = lp_build_gather_values(gallivm, tids_y, cs_type.length);
328 tid_vals[2] = lp_build_gather_values(gallivm, tids_z, cs_type.length);
329 system_values.thread_id = LLVMGetUndef(LLVMArrayType(LLVMVectorType(int32_type, cs_type.length), 3));
330 for (i = 0; i < 3; i++)
331 system_values.thread_id = LLVMBuildInsertValue(builder, system_values.thread_id, tid_vals[i], i, "");
332
333 LLVMValueRef gtids[3] = { grid_x_arg, grid_y_arg, grid_z_arg };
334 system_values.block_id = LLVMGetUndef(LLVMVectorType(int32_type, 3));
335 for (i = 0; i < 3; i++)
336 system_values.block_id = LLVMBuildInsertElement(builder, system_values.block_id, gtids[i], lp_build_const_int32(gallivm, i), "");
337
338 LLVMValueRef gstids[3] = { grid_size_x_arg, grid_size_y_arg, grid_size_z_arg };
339 system_values.grid_size = LLVMGetUndef(LLVMVectorType(int32_type, 3));
340 for (i = 0; i < 3; i++)
341 system_values.grid_size = LLVMBuildInsertElement(builder, system_values.grid_size, gstids[i], lp_build_const_int32(gallivm, i), "");
342
343 system_values.work_dim = work_dim_arg;
344
345 LLVMValueRef bsize[3] = { block_x_size_arg, block_y_size_arg, block_z_size_arg };
346 system_values.block_size = LLVMGetUndef(LLVMVectorType(int32_type, 3));
347 for (i = 0; i < 3; i++)
348 system_values.block_size = LLVMBuildInsertElement(builder, system_values.block_size, bsize[i], lp_build_const_int32(gallivm, i), "");
349
350 LLVMValueRef last_x_loop = LLVMBuildICmp(gallivm->builder, LLVMIntEQ, x_size_arg, LLVMBuildSub(gallivm->builder, num_x_loop, lp_build_const_int32(gallivm, 1), ""), "");
351 LLVMValueRef use_partial_mask = LLVMBuildAnd(gallivm->builder, last_x_loop, has_partials, "");
352 struct lp_build_if_state if_state;
353 LLVMValueRef mask_val = lp_build_alloca(gallivm, LLVMVectorType(int32_type, cs_type.length), "mask");
354 LLVMValueRef full_mask_val = lp_build_const_int_vec(gallivm, cs_type, ~0);
355 LLVMBuildStore(gallivm->builder, full_mask_val, mask_val);
356
357 lp_build_if(&if_state, gallivm, use_partial_mask);
358 struct lp_build_loop_state mask_loop_state;
359 lp_build_loop_begin(&mask_loop_state, gallivm, partials);
360 LLVMValueRef tmask_val = LLVMBuildLoad(gallivm->builder, mask_val, "");
361 tmask_val = LLVMBuildInsertElement(gallivm->builder, tmask_val, lp_build_const_int32(gallivm, 0), mask_loop_state.counter, "");
362 LLVMBuildStore(gallivm->builder, tmask_val, mask_val);
363 lp_build_loop_end_cond(&mask_loop_state, vec_length, NULL, LLVMIntUGE);
364 lp_build_endif(&if_state);
365
366 mask_val = LLVMBuildLoad(gallivm->builder, mask_val, "");
367 lp_build_mask_begin(&mask, gallivm, cs_type, mask_val);
368
369 struct lp_build_coro_suspend_info coro_info;
370
371 LLVMBasicBlockRef sus_block = LLVMAppendBasicBlockInContext(gallivm->context, coro, "suspend");
372 LLVMBasicBlockRef clean_block = LLVMAppendBasicBlockInContext(gallivm->context, coro, "cleanup");
373
374 coro_info.suspend = sus_block;
375 coro_info.cleanup = clean_block;
376
377 struct lp_build_tgsi_params params;
378 memset(&params, 0, sizeof(params));
379
380 params.type = cs_type;
381 params.mask = &mask;
382 params.consts_ptr = consts_ptr;
383 params.const_sizes_ptr = num_consts_ptr;
384 params.system_values = &system_values;
385 params.context_ptr = context_ptr;
386 params.sampler = sampler;
387 params.info = &shader->info.base;
388 params.ssbo_ptr = ssbo_ptr;
389 params.ssbo_sizes_ptr = num_ssbo_ptr;
390 params.image = image;
391 params.shared_ptr = shared_ptr;
392 params.coro = &coro_info;
393 params.kernel_args = kernel_args_ptr;
394
395 if (shader->base.type == PIPE_SHADER_IR_TGSI)
396 lp_build_tgsi_soa(gallivm, shader->base.tokens, &params, NULL);
397 else
398 lp_build_nir_soa(gallivm, shader->base.ir.nir, &params,
399 NULL);
400
401 mask_val = lp_build_mask_end(&mask);
402
403 lp_build_coro_suspend_switch(gallivm, &coro_info, NULL, true);
404 LLVMPositionBuilderAtEnd(builder, clean_block);
405
406 lp_build_coro_free_mem(gallivm, coro_id, coro_hdl);
407
408 LLVMBuildBr(builder, sus_block);
409 LLVMPositionBuilderAtEnd(builder, sus_block);
410
411 lp_build_coro_end(gallivm, coro_hdl);
412 LLVMBuildRet(builder, coro_hdl);
413 }
414
415 sampler->destroy(sampler);
416 image->destroy(image);
417
418 gallivm_verify_function(gallivm, coro);
419 gallivm_verify_function(gallivm, function);
420 }
421
422 static void *
423 llvmpipe_create_compute_state(struct pipe_context *pipe,
424 const struct pipe_compute_state *templ)
425 {
426 struct lp_compute_shader *shader;
427 int nr_samplers, nr_sampler_views;
428
429 shader = CALLOC_STRUCT(lp_compute_shader);
430 if (!shader)
431 return NULL;
432
433 shader->no = cs_no++;
434
435 shader->base.type = templ->ir_type;
436 if (templ->ir_type == PIPE_SHADER_IR_NIR_SERIALIZED) {
437 struct blob_reader reader;
438 const struct pipe_binary_program_header *hdr = templ->prog;
439
440 blob_reader_init(&reader, hdr->blob, hdr->num_bytes);
441 shader->base.ir.nir = nir_deserialize(NULL, pipe->screen->get_compiler_options(pipe->screen, PIPE_SHADER_IR_NIR, PIPE_SHADER_COMPUTE), &reader);
442 shader->base.type = PIPE_SHADER_IR_NIR;
443
444 pipe->screen->finalize_nir(pipe->screen, shader->base.ir.nir, false);
445 } else if (templ->ir_type == PIPE_SHADER_IR_NIR)
446 shader->base.ir.nir = (struct nir_shader *)templ->prog;
447
448 if (shader->base.type == PIPE_SHADER_IR_TGSI) {
449 /* get/save the summary info for this shader */
450 lp_build_tgsi_info(templ->prog, &shader->info);
451
452 /* we need to keep a local copy of the tokens */
453 shader->base.tokens = tgsi_dup_tokens(templ->prog);
454 } else {
455 nir_tgsi_scan_shader(shader->base.ir.nir, &shader->info.base, false);
456 }
457
458 shader->req_local_mem = templ->req_local_mem;
459 make_empty_list(&shader->variants);
460
461 nr_samplers = shader->info.base.file_max[TGSI_FILE_SAMPLER] + 1;
462 nr_sampler_views = shader->info.base.file_max[TGSI_FILE_SAMPLER_VIEW] + 1;
463 int nr_images = shader->info.base.file_max[TGSI_FILE_IMAGE] + 1;
464 shader->variant_key_size = lp_cs_variant_key_size(MAX2(nr_samplers, nr_sampler_views), nr_images);
465
466 return shader;
467 }
468
469 static void
470 llvmpipe_bind_compute_state(struct pipe_context *pipe,
471 void *cs)
472 {
473 struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe);
474
475 if (llvmpipe->cs == cs)
476 return;
477
478 llvmpipe->cs = (struct lp_compute_shader *)cs;
479 llvmpipe->cs_dirty |= LP_CSNEW_CS;
480 }
481
482 /**
483 * Remove shader variant from two lists: the shader's variant list
484 * and the context's variant list.
485 */
486 static void
487 llvmpipe_remove_cs_shader_variant(struct llvmpipe_context *lp,
488 struct lp_compute_shader_variant *variant)
489 {
490 if ((LP_DEBUG & DEBUG_CS) || (gallivm_debug & GALLIVM_DEBUG_IR)) {
491 debug_printf("llvmpipe: del cs #%u var %u v created %u v cached %u "
492 "v total cached %u inst %u total inst %u\n",
493 variant->shader->no, variant->no,
494 variant->shader->variants_created,
495 variant->shader->variants_cached,
496 lp->nr_cs_variants, variant->nr_instrs, lp->nr_cs_instrs);
497 }
498
499 gallivm_destroy(variant->gallivm);
500
501 /* remove from shader's list */
502 remove_from_list(&variant->list_item_local);
503 variant->shader->variants_cached--;
504
505 /* remove from context's list */
506 remove_from_list(&variant->list_item_global);
507 lp->nr_fs_variants--;
508 lp->nr_fs_instrs -= variant->nr_instrs;
509
510 FREE(variant);
511 }
512
513 static void
514 llvmpipe_delete_compute_state(struct pipe_context *pipe,
515 void *cs)
516 {
517 struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe);
518 struct lp_compute_shader *shader = cs;
519 struct lp_cs_variant_list_item *li;
520
521 if (llvmpipe->cs == cs)
522 llvmpipe->cs = NULL;
523 for (unsigned i = 0; i < shader->max_global_buffers; i++)
524 pipe_resource_reference(&shader->global_buffers[i], NULL);
525 FREE(shader->global_buffers);
526
527 /* Delete all the variants */
528 li = first_elem(&shader->variants);
529 while(!at_end(&shader->variants, li)) {
530 struct lp_cs_variant_list_item *next = next_elem(li);
531 llvmpipe_remove_cs_shader_variant(llvmpipe, li->base);
532 li = next;
533 }
534 if (shader->base.ir.nir)
535 ralloc_free(shader->base.ir.nir);
536 tgsi_free_tokens(shader->base.tokens);
537 FREE(shader);
538 }
539
540 static struct lp_compute_shader_variant_key *
541 make_variant_key(struct llvmpipe_context *lp,
542 struct lp_compute_shader *shader,
543 char *store)
544 {
545 int i;
546 struct lp_compute_shader_variant_key *key;
547 key = (struct lp_compute_shader_variant_key *)store;
548 memset(key, 0, offsetof(struct lp_compute_shader_variant_key, samplers[1]));
549
550 /* This value will be the same for all the variants of a given shader:
551 */
552 key->nr_samplers = shader->info.base.file_max[TGSI_FILE_SAMPLER] + 1;
553
554 struct lp_sampler_static_state *cs_sampler;
555
556 cs_sampler = key->samplers;
557 for(i = 0; i < key->nr_samplers; ++i) {
558 if(shader->info.base.file_mask[TGSI_FILE_SAMPLER] & (1 << i)) {
559 lp_sampler_static_sampler_state(&cs_sampler[i].sampler_state,
560 lp->samplers[PIPE_SHADER_COMPUTE][i]);
561 }
562 }
563
564 /*
565 * XXX If TGSI_FILE_SAMPLER_VIEW exists assume all texture opcodes
566 * are dx10-style? Can't really have mixed opcodes, at least not
567 * if we want to skip the holes here (without rescanning tgsi).
568 */
569 if (shader->info.base.file_max[TGSI_FILE_SAMPLER_VIEW] != -1) {
570 key->nr_sampler_views = shader->info.base.file_max[TGSI_FILE_SAMPLER_VIEW] + 1;
571 for(i = 0; i < key->nr_sampler_views; ++i) {
572 /*
573 * Note sview may exceed what's representable by file_mask.
574 * This will still work, the only downside is that not actually
575 * used views may be included in the shader key.
576 */
577 if(shader->info.base.file_mask[TGSI_FILE_SAMPLER_VIEW] & (1u << (i & 31))) {
578 lp_sampler_static_texture_state(&cs_sampler[i].texture_state,
579 lp->sampler_views[PIPE_SHADER_COMPUTE][i]);
580 }
581 }
582 }
583 else {
584 key->nr_sampler_views = key->nr_samplers;
585 for(i = 0; i < key->nr_sampler_views; ++i) {
586 if(shader->info.base.file_mask[TGSI_FILE_SAMPLER] & (1 << i)) {
587 lp_sampler_static_texture_state(&cs_sampler[i].texture_state,
588 lp->sampler_views[PIPE_SHADER_COMPUTE][i]);
589 }
590 }
591 }
592
593 struct lp_image_static_state *lp_image;
594 lp_image = lp_cs_variant_key_images(key);
595 key->nr_images = shader->info.base.file_max[TGSI_FILE_IMAGE] + 1;
596 for (i = 0; i < key->nr_images; ++i) {
597 if (shader->info.base.file_mask[TGSI_FILE_IMAGE] & (1 << i)) {
598 lp_sampler_static_texture_state_image(&lp_image[i].image_state,
599 &lp->images[PIPE_SHADER_COMPUTE][i]);
600 }
601 }
602 return key;
603 }
604
605 static void
606 dump_cs_variant_key(const struct lp_compute_shader_variant_key *key)
607 {
608 int i;
609 debug_printf("cs variant %p:\n", (void *) key);
610
611 for (i = 0; i < key->nr_samplers; ++i) {
612 const struct lp_static_sampler_state *sampler = &key->samplers[i].sampler_state;
613 debug_printf("sampler[%u] = \n", i);
614 debug_printf(" .wrap = %s %s %s\n",
615 util_str_tex_wrap(sampler->wrap_s, TRUE),
616 util_str_tex_wrap(sampler->wrap_t, TRUE),
617 util_str_tex_wrap(sampler->wrap_r, TRUE));
618 debug_printf(" .min_img_filter = %s\n",
619 util_str_tex_filter(sampler->min_img_filter, TRUE));
620 debug_printf(" .min_mip_filter = %s\n",
621 util_str_tex_mipfilter(sampler->min_mip_filter, TRUE));
622 debug_printf(" .mag_img_filter = %s\n",
623 util_str_tex_filter(sampler->mag_img_filter, TRUE));
624 if (sampler->compare_mode != PIPE_TEX_COMPARE_NONE)
625 debug_printf(" .compare_func = %s\n", util_str_func(sampler->compare_func, TRUE));
626 debug_printf(" .normalized_coords = %u\n", sampler->normalized_coords);
627 debug_printf(" .min_max_lod_equal = %u\n", sampler->min_max_lod_equal);
628 debug_printf(" .lod_bias_non_zero = %u\n", sampler->lod_bias_non_zero);
629 debug_printf(" .apply_min_lod = %u\n", sampler->apply_min_lod);
630 debug_printf(" .apply_max_lod = %u\n", sampler->apply_max_lod);
631 }
632 for (i = 0; i < key->nr_sampler_views; ++i) {
633 const struct lp_static_texture_state *texture = &key->samplers[i].texture_state;
634 debug_printf("texture[%u] = \n", i);
635 debug_printf(" .format = %s\n",
636 util_format_name(texture->format));
637 debug_printf(" .target = %s\n",
638 util_str_tex_target(texture->target, TRUE));
639 debug_printf(" .level_zero_only = %u\n",
640 texture->level_zero_only);
641 debug_printf(" .pot = %u %u %u\n",
642 texture->pot_width,
643 texture->pot_height,
644 texture->pot_depth);
645 }
646 struct lp_image_static_state *images = lp_cs_variant_key_images(key);
647 for (i = 0; i < key->nr_images; ++i) {
648 const struct lp_static_texture_state *image = &images[i].image_state;
649 debug_printf("image[%u] = \n", i);
650 debug_printf(" .format = %s\n",
651 util_format_name(image->format));
652 debug_printf(" .target = %s\n",
653 util_str_tex_target(image->target, TRUE));
654 debug_printf(" .level_zero_only = %u\n",
655 image->level_zero_only);
656 debug_printf(" .pot = %u %u %u\n",
657 image->pot_width,
658 image->pot_height,
659 image->pot_depth);
660 }
661 }
662
663 static void
664 lp_debug_cs_variant(const struct lp_compute_shader_variant *variant)
665 {
666 debug_printf("llvmpipe: Compute shader #%u variant #%u:\n",
667 variant->shader->no, variant->no);
668 if (variant->shader->base.type == PIPE_SHADER_IR_TGSI)
669 tgsi_dump(variant->shader->base.tokens, 0);
670 else
671 nir_print_shader(variant->shader->base.ir.nir, stderr);
672 dump_cs_variant_key(&variant->key);
673 debug_printf("\n");
674 }
675
676 static struct lp_compute_shader_variant *
677 generate_variant(struct llvmpipe_context *lp,
678 struct lp_compute_shader *shader,
679 const struct lp_compute_shader_variant_key *key)
680 {
681 struct lp_compute_shader_variant *variant;
682 char module_name[64];
683 variant = MALLOC(sizeof *variant + shader->variant_key_size - sizeof variant->key);
684 if (!variant)
685 return NULL;
686
687 memset(variant, 0, sizeof(*variant));
688 snprintf(module_name, sizeof(module_name), "cs%u_variant%u",
689 shader->no, shader->variants_created);
690
691 variant->gallivm = gallivm_create(module_name, lp->context);
692 if (!variant->gallivm) {
693 FREE(variant);
694 return NULL;
695 }
696
697 variant->shader = shader;
698 variant->list_item_global.base = variant;
699 variant->list_item_local.base = variant;
700 variant->no = shader->variants_created++;
701
702 memcpy(&variant->key, key, shader->variant_key_size);
703
704 if ((LP_DEBUG & DEBUG_CS) || (gallivm_debug & GALLIVM_DEBUG_IR)) {
705 lp_debug_cs_variant(variant);
706 }
707
708 lp_jit_init_cs_types(variant);
709
710 generate_compute(lp, shader, variant);
711
712 gallivm_compile_module(variant->gallivm);
713
714 lp_build_coro_add_malloc_hooks(variant->gallivm);
715 variant->nr_instrs += lp_build_count_ir_module(variant->gallivm->module);
716
717 variant->jit_function = (lp_jit_cs_func)gallivm_jit_function(variant->gallivm, variant->function);
718
719 gallivm_free_ir(variant->gallivm);
720 return variant;
721 }
722
723 static void
724 lp_cs_ctx_set_cs_variant( struct lp_cs_context *csctx,
725 struct lp_compute_shader_variant *variant)
726 {
727 csctx->cs.current.variant = variant;
728 }
729
730 static void
731 llvmpipe_update_cs(struct llvmpipe_context *lp)
732 {
733 struct lp_compute_shader *shader = lp->cs;
734
735 struct lp_compute_shader_variant_key *key;
736 struct lp_compute_shader_variant *variant = NULL;
737 struct lp_cs_variant_list_item *li;
738 char store[LP_CS_MAX_VARIANT_KEY_SIZE];
739
740 key = make_variant_key(lp, shader, store);
741
742 /* Search the variants for one which matches the key */
743 li = first_elem(&shader->variants);
744 while(!at_end(&shader->variants, li)) {
745 if(memcmp(&li->base->key, key, shader->variant_key_size) == 0) {
746 variant = li->base;
747 break;
748 }
749 li = next_elem(li);
750 }
751
752 if (variant) {
753 /* Move this variant to the head of the list to implement LRU
754 * deletion of shader's when we have too many.
755 */
756 move_to_head(&lp->cs_variants_list, &variant->list_item_global);
757 }
758 else {
759 /* variant not found, create it now */
760 int64_t t0, t1, dt;
761 unsigned i;
762 unsigned variants_to_cull;
763
764 if (LP_DEBUG & DEBUG_CS) {
765 debug_printf("%u variants,\t%u instrs,\t%u instrs/variant\n",
766 lp->nr_cs_variants,
767 lp->nr_cs_instrs,
768 lp->nr_cs_variants ? lp->nr_cs_instrs / lp->nr_cs_variants : 0);
769 }
770
771 /* First, check if we've exceeded the max number of shader variants.
772 * If so, free 6.25% of them (the least recently used ones).
773 */
774 variants_to_cull = lp->nr_cs_variants >= LP_MAX_SHADER_VARIANTS ? LP_MAX_SHADER_VARIANTS / 16 : 0;
775
776 if (variants_to_cull ||
777 lp->nr_cs_instrs >= LP_MAX_SHADER_INSTRUCTIONS) {
778 if (gallivm_debug & GALLIVM_DEBUG_PERF) {
779 debug_printf("Evicting CS: %u cs variants,\t%u total variants,"
780 "\t%u instrs,\t%u instrs/variant\n",
781 shader->variants_cached,
782 lp->nr_cs_variants, lp->nr_cs_instrs,
783 lp->nr_cs_instrs / lp->nr_cs_variants);
784 }
785
786 /*
787 * We need to re-check lp->nr_cs_variants because an arbitrarliy large
788 * number of shader variants (potentially all of them) could be
789 * pending for destruction on flush.
790 */
791
792 for (i = 0; i < variants_to_cull || lp->nr_cs_instrs >= LP_MAX_SHADER_INSTRUCTIONS; i++) {
793 struct lp_cs_variant_list_item *item;
794 if (is_empty_list(&lp->cs_variants_list)) {
795 break;
796 }
797 item = last_elem(&lp->cs_variants_list);
798 assert(item);
799 assert(item->base);
800 llvmpipe_remove_cs_shader_variant(lp, item->base);
801 }
802 }
803 /*
804 * Generate the new variant.
805 */
806 t0 = os_time_get();
807 variant = generate_variant(lp, shader, key);
808 t1 = os_time_get();
809 dt = t1 - t0;
810 LP_COUNT_ADD(llvm_compile_time, dt);
811 LP_COUNT_ADD(nr_llvm_compiles, 2); /* emit vs. omit in/out test */
812
813 /* Put the new variant into the list */
814 if (variant) {
815 insert_at_head(&shader->variants, &variant->list_item_local);
816 insert_at_head(&lp->cs_variants_list, &variant->list_item_global);
817 lp->nr_cs_variants++;
818 lp->nr_cs_instrs += variant->nr_instrs;
819 shader->variants_cached++;
820 }
821 }
822 /* Bind this variant */
823 lp_cs_ctx_set_cs_variant(lp->csctx, variant);
824 }
825
826 /**
827 * Called during state validation when LP_CSNEW_SAMPLER_VIEW is set.
828 */
829 static void
830 lp_csctx_set_sampler_views(struct lp_cs_context *csctx,
831 unsigned num,
832 struct pipe_sampler_view **views)
833 {
834 unsigned i, max_tex_num;
835
836 LP_DBG(DEBUG_SETUP, "%s\n", __FUNCTION__);
837
838 assert(num <= PIPE_MAX_SHADER_SAMPLER_VIEWS);
839
840 max_tex_num = MAX2(num, csctx->cs.current_tex_num);
841
842 for (i = 0; i < max_tex_num; i++) {
843 struct pipe_sampler_view *view = i < num ? views[i] : NULL;
844
845 if (view) {
846 struct pipe_resource *res = view->texture;
847 struct llvmpipe_resource *lp_tex = llvmpipe_resource(res);
848 struct lp_jit_texture *jit_tex;
849 jit_tex = &csctx->cs.current.jit_context.textures[i];
850
851 /* We're referencing the texture's internal data, so save a
852 * reference to it.
853 */
854 pipe_resource_reference(&csctx->cs.current_tex[i], res);
855
856 if (!lp_tex->dt) {
857 /* regular texture - csctx array of mipmap level offsets */
858 int j;
859 unsigned first_level = 0;
860 unsigned last_level = 0;
861
862 if (llvmpipe_resource_is_texture(res)) {
863 first_level = view->u.tex.first_level;
864 last_level = view->u.tex.last_level;
865 assert(first_level <= last_level);
866 assert(last_level <= res->last_level);
867 jit_tex->base = lp_tex->tex_data;
868 }
869 else {
870 jit_tex->base = lp_tex->data;
871 }
872 if (LP_PERF & PERF_TEX_MEM) {
873 /* use dummy tile memory */
874 jit_tex->base = lp_dummy_tile;
875 jit_tex->width = TILE_SIZE/8;
876 jit_tex->height = TILE_SIZE/8;
877 jit_tex->depth = 1;
878 jit_tex->first_level = 0;
879 jit_tex->last_level = 0;
880 jit_tex->mip_offsets[0] = 0;
881 jit_tex->row_stride[0] = 0;
882 jit_tex->img_stride[0] = 0;
883 jit_tex->num_samples = 0;
884 jit_tex->sample_stride = 0;
885 }
886 else {
887 jit_tex->width = res->width0;
888 jit_tex->height = res->height0;
889 jit_tex->depth = res->depth0;
890 jit_tex->first_level = first_level;
891 jit_tex->last_level = last_level;
892 jit_tex->num_samples = res->nr_samples;
893 jit_tex->sample_stride = 0;
894
895 if (llvmpipe_resource_is_texture(res)) {
896 for (j = first_level; j <= last_level; j++) {
897 jit_tex->mip_offsets[j] = lp_tex->mip_offsets[j];
898 jit_tex->row_stride[j] = lp_tex->row_stride[j];
899 jit_tex->img_stride[j] = lp_tex->img_stride[j];
900 }
901 jit_tex->sample_stride = lp_tex->sample_stride;
902
903 if (res->target == PIPE_TEXTURE_1D_ARRAY ||
904 res->target == PIPE_TEXTURE_2D_ARRAY ||
905 res->target == PIPE_TEXTURE_CUBE ||
906 res->target == PIPE_TEXTURE_CUBE_ARRAY) {
907 /*
908 * For array textures, we don't have first_layer, instead
909 * adjust last_layer (stored as depth) plus the mip level offsets
910 * (as we have mip-first layout can't just adjust base ptr).
911 * XXX For mip levels, could do something similar.
912 */
913 jit_tex->depth = view->u.tex.last_layer - view->u.tex.first_layer + 1;
914 for (j = first_level; j <= last_level; j++) {
915 jit_tex->mip_offsets[j] += view->u.tex.first_layer *
916 lp_tex->img_stride[j];
917 }
918 if (view->target == PIPE_TEXTURE_CUBE ||
919 view->target == PIPE_TEXTURE_CUBE_ARRAY) {
920 assert(jit_tex->depth % 6 == 0);
921 }
922 assert(view->u.tex.first_layer <= view->u.tex.last_layer);
923 assert(view->u.tex.last_layer < res->array_size);
924 }
925 }
926 else {
927 /*
928 * For buffers, we don't have "offset", instead adjust
929 * the size (stored as width) plus the base pointer.
930 */
931 unsigned view_blocksize = util_format_get_blocksize(view->format);
932 /* probably don't really need to fill that out */
933 jit_tex->mip_offsets[0] = 0;
934 jit_tex->row_stride[0] = 0;
935 jit_tex->img_stride[0] = 0;
936
937 /* everything specified in number of elements here. */
938 jit_tex->width = view->u.buf.size / view_blocksize;
939 jit_tex->base = (uint8_t *)jit_tex->base + view->u.buf.offset;
940 /* XXX Unsure if we need to sanitize parameters? */
941 assert(view->u.buf.offset + view->u.buf.size <= res->width0);
942 }
943 }
944 }
945 else {
946 /* display target texture/surface */
947 /*
948 * XXX: Where should this be unmapped?
949 */
950 struct llvmpipe_screen *screen = llvmpipe_screen(res->screen);
951 struct sw_winsys *winsys = screen->winsys;
952 jit_tex->base = winsys->displaytarget_map(winsys, lp_tex->dt,
953 PIPE_TRANSFER_READ);
954 jit_tex->row_stride[0] = lp_tex->row_stride[0];
955 jit_tex->img_stride[0] = lp_tex->img_stride[0];
956 jit_tex->mip_offsets[0] = 0;
957 jit_tex->width = res->width0;
958 jit_tex->height = res->height0;
959 jit_tex->depth = res->depth0;
960 jit_tex->first_level = jit_tex->last_level = 0;
961 jit_tex->num_samples = res->nr_samples;
962 jit_tex->sample_stride = 0;
963 assert(jit_tex->base);
964 }
965 }
966 else {
967 pipe_resource_reference(&csctx->cs.current_tex[i], NULL);
968 }
969 }
970 csctx->cs.current_tex_num = num;
971 }
972
973
974 /**
975 * Called during state validation when LP_NEW_SAMPLER is set.
976 */
977 static void
978 lp_csctx_set_sampler_state(struct lp_cs_context *csctx,
979 unsigned num,
980 struct pipe_sampler_state **samplers)
981 {
982 unsigned i;
983
984 LP_DBG(DEBUG_SETUP, "%s\n", __FUNCTION__);
985
986 assert(num <= PIPE_MAX_SAMPLERS);
987
988 for (i = 0; i < PIPE_MAX_SAMPLERS; i++) {
989 const struct pipe_sampler_state *sampler = i < num ? samplers[i] : NULL;
990
991 if (sampler) {
992 struct lp_jit_sampler *jit_sam;
993 jit_sam = &csctx->cs.current.jit_context.samplers[i];
994
995 jit_sam->min_lod = sampler->min_lod;
996 jit_sam->max_lod = sampler->max_lod;
997 jit_sam->lod_bias = sampler->lod_bias;
998 COPY_4V(jit_sam->border_color, sampler->border_color.f);
999 }
1000 }
1001 }
1002
1003 static void
1004 lp_csctx_set_cs_constants(struct lp_cs_context *csctx,
1005 unsigned num,
1006 struct pipe_constant_buffer *buffers)
1007 {
1008 unsigned i;
1009
1010 LP_DBG(DEBUG_SETUP, "%s %p\n", __FUNCTION__, (void *) buffers);
1011
1012 assert(num <= ARRAY_SIZE(csctx->constants));
1013
1014 for (i = 0; i < num; ++i) {
1015 util_copy_constant_buffer(&csctx->constants[i].current, &buffers[i]);
1016 }
1017 for (; i < ARRAY_SIZE(csctx->constants); i++) {
1018 util_copy_constant_buffer(&csctx->constants[i].current, NULL);
1019 }
1020 }
1021
1022 static void
1023 lp_csctx_set_cs_ssbos(struct lp_cs_context *csctx,
1024 unsigned num,
1025 struct pipe_shader_buffer *buffers)
1026 {
1027 int i;
1028 LP_DBG(DEBUG_SETUP, "%s %p\n", __FUNCTION__, (void *)buffers);
1029
1030 assert (num <= ARRAY_SIZE(csctx->ssbos));
1031
1032 for (i = 0; i < num; ++i) {
1033 util_copy_shader_buffer(&csctx->ssbos[i].current, &buffers[i]);
1034 }
1035 for (; i < ARRAY_SIZE(csctx->ssbos); i++) {
1036 util_copy_shader_buffer(&csctx->ssbos[i].current, NULL);
1037 }
1038 }
1039
1040 static void
1041 lp_csctx_set_cs_images(struct lp_cs_context *csctx,
1042 unsigned num,
1043 struct pipe_image_view *images)
1044 {
1045 unsigned i;
1046
1047 LP_DBG(DEBUG_SETUP, "%s %p\n", __FUNCTION__, (void *) images);
1048
1049 assert(num <= ARRAY_SIZE(csctx->images));
1050
1051 for (i = 0; i < num; ++i) {
1052 struct pipe_image_view *image = &images[i];
1053 util_copy_image_view(&csctx->images[i].current, &images[i]);
1054
1055 struct pipe_resource *res = image->resource;
1056 struct llvmpipe_resource *lp_res = llvmpipe_resource(res);
1057 struct lp_jit_image *jit_image;
1058
1059 jit_image = &csctx->cs.current.jit_context.images[i];
1060 if (!lp_res)
1061 continue;
1062 if (!lp_res->dt) {
1063 /* regular texture - csctx array of mipmap level offsets */
1064 if (llvmpipe_resource_is_texture(res)) {
1065 jit_image->base = lp_res->tex_data;
1066 } else
1067 jit_image->base = lp_res->data;
1068
1069 jit_image->width = res->width0;
1070 jit_image->height = res->height0;
1071 jit_image->depth = res->depth0;
1072 jit_image->num_samples = res->nr_samples;
1073
1074 if (llvmpipe_resource_is_texture(res)) {
1075 uint32_t mip_offset = lp_res->mip_offsets[image->u.tex.level];
1076
1077 jit_image->width = u_minify(jit_image->width, image->u.tex.level);
1078 jit_image->height = u_minify(jit_image->height, image->u.tex.level);
1079
1080 if (res->target == PIPE_TEXTURE_1D_ARRAY ||
1081 res->target == PIPE_TEXTURE_2D_ARRAY ||
1082 res->target == PIPE_TEXTURE_3D ||
1083 res->target == PIPE_TEXTURE_CUBE ||
1084 res->target == PIPE_TEXTURE_CUBE_ARRAY) {
1085 /*
1086 * For array textures, we don't have first_layer, instead
1087 * adjust last_layer (stored as depth) plus the mip level offsets
1088 * (as we have mip-first layout can't just adjust base ptr).
1089 * XXX For mip levels, could do something similar.
1090 */
1091 jit_image->depth = image->u.tex.last_layer - image->u.tex.first_layer + 1;
1092 mip_offset += image->u.tex.first_layer * lp_res->img_stride[image->u.tex.level];
1093 } else
1094 jit_image->depth = u_minify(jit_image->depth, image->u.tex.level);
1095
1096 jit_image->row_stride = lp_res->row_stride[image->u.tex.level];
1097 jit_image->img_stride = lp_res->img_stride[image->u.tex.level];
1098 jit_image->sample_stride = lp_res->sample_stride;
1099 jit_image->base = (uint8_t *)jit_image->base + mip_offset;
1100 } else {
1101 unsigned view_blocksize = util_format_get_blocksize(image->format);
1102 jit_image->width = image->u.buf.size / view_blocksize;
1103 jit_image->base = (uint8_t *)jit_image->base + image->u.buf.offset;
1104 }
1105 }
1106 }
1107 for (; i < ARRAY_SIZE(csctx->images); i++) {
1108 util_copy_image_view(&csctx->images[i].current, NULL);
1109 }
1110 }
1111
1112 static void
1113 update_csctx_consts(struct llvmpipe_context *llvmpipe)
1114 {
1115 struct lp_cs_context *csctx = llvmpipe->csctx;
1116 int i;
1117
1118 for (i = 0; i < ARRAY_SIZE(csctx->constants); ++i) {
1119 struct pipe_resource *buffer = csctx->constants[i].current.buffer;
1120 const ubyte *current_data = NULL;
1121
1122 if (buffer) {
1123 /* resource buffer */
1124 current_data = (ubyte *) llvmpipe_resource_data(buffer);
1125 }
1126 else if (csctx->constants[i].current.user_buffer) {
1127 /* user-space buffer */
1128 current_data = (ubyte *) csctx->constants[i].current.user_buffer;
1129 }
1130
1131 if (current_data) {
1132 current_data += csctx->constants[i].current.buffer_offset;
1133
1134 csctx->cs.current.jit_context.constants[i] = (const float *)current_data;
1135 csctx->cs.current.jit_context.num_constants[i] = csctx->constants[i].current.buffer_size;
1136 } else {
1137 csctx->cs.current.jit_context.constants[i] = NULL;
1138 csctx->cs.current.jit_context.num_constants[i] = 0;
1139 }
1140 }
1141 }
1142
1143 static void
1144 update_csctx_ssbo(struct llvmpipe_context *llvmpipe)
1145 {
1146 struct lp_cs_context *csctx = llvmpipe->csctx;
1147 int i;
1148 for (i = 0; i < ARRAY_SIZE(csctx->ssbos); ++i) {
1149 struct pipe_resource *buffer = csctx->ssbos[i].current.buffer;
1150 const ubyte *current_data = NULL;
1151
1152 if (!buffer)
1153 continue;
1154 /* resource buffer */
1155 current_data = (ubyte *) llvmpipe_resource_data(buffer);
1156 if (current_data) {
1157 current_data += csctx->ssbos[i].current.buffer_offset;
1158
1159 csctx->cs.current.jit_context.ssbos[i] = (const uint32_t *)current_data;
1160 csctx->cs.current.jit_context.num_ssbos[i] = csctx->ssbos[i].current.buffer_size;
1161 } else {
1162 csctx->cs.current.jit_context.ssbos[i] = NULL;
1163 csctx->cs.current.jit_context.num_ssbos[i] = 0;
1164 }
1165 }
1166 }
1167
1168 static void
1169 llvmpipe_cs_update_derived(struct llvmpipe_context *llvmpipe, void *input)
1170 {
1171 if (llvmpipe->cs_dirty & (LP_CSNEW_CS))
1172 llvmpipe_update_cs(llvmpipe);
1173
1174 if (llvmpipe->cs_dirty & LP_CSNEW_CONSTANTS) {
1175 lp_csctx_set_cs_constants(llvmpipe->csctx,
1176 ARRAY_SIZE(llvmpipe->constants[PIPE_SHADER_COMPUTE]),
1177 llvmpipe->constants[PIPE_SHADER_COMPUTE]);
1178 update_csctx_consts(llvmpipe);
1179 }
1180
1181 if (llvmpipe->cs_dirty & LP_CSNEW_SSBOS) {
1182 lp_csctx_set_cs_ssbos(llvmpipe->csctx,
1183 ARRAY_SIZE(llvmpipe->ssbos[PIPE_SHADER_COMPUTE]),
1184 llvmpipe->ssbos[PIPE_SHADER_COMPUTE]);
1185 update_csctx_ssbo(llvmpipe);
1186 }
1187
1188 if (llvmpipe->cs_dirty & LP_CSNEW_SAMPLER_VIEW)
1189 lp_csctx_set_sampler_views(llvmpipe->csctx,
1190 llvmpipe->num_sampler_views[PIPE_SHADER_COMPUTE],
1191 llvmpipe->sampler_views[PIPE_SHADER_COMPUTE]);
1192
1193 if (llvmpipe->cs_dirty & LP_CSNEW_SAMPLER)
1194 lp_csctx_set_sampler_state(llvmpipe->csctx,
1195 llvmpipe->num_samplers[PIPE_SHADER_COMPUTE],
1196 llvmpipe->samplers[PIPE_SHADER_COMPUTE]);
1197
1198 if (llvmpipe->cs_dirty & LP_CSNEW_IMAGES)
1199 lp_csctx_set_cs_images(llvmpipe->csctx,
1200 ARRAY_SIZE(llvmpipe->images[PIPE_SHADER_COMPUTE]),
1201 llvmpipe->images[PIPE_SHADER_COMPUTE]);
1202
1203 if (input) {
1204 struct lp_cs_context *csctx = llvmpipe->csctx;
1205 csctx->input = input;
1206 csctx->cs.current.jit_context.kernel_args = input;
1207 }
1208
1209 llvmpipe->cs_dirty = 0;
1210 }
1211
1212 static void
1213 cs_exec_fn(void *init_data, int iter_idx, struct lp_cs_local_mem *lmem)
1214 {
1215 struct lp_cs_job_info *job_info = init_data;
1216 struct lp_jit_cs_thread_data thread_data;
1217
1218 memset(&thread_data, 0, sizeof(thread_data));
1219
1220 if (lmem->local_size < job_info->req_local_mem) {
1221 lmem->local_mem_ptr = REALLOC(lmem->local_mem_ptr, lmem->local_size,
1222 job_info->req_local_mem);
1223 lmem->local_size = job_info->req_local_mem;
1224 }
1225 thread_data.shared = lmem->local_mem_ptr;
1226
1227 unsigned grid_z = iter_idx / (job_info->grid_size[0] * job_info->grid_size[1]);
1228 unsigned grid_y = (iter_idx - (grid_z * (job_info->grid_size[0] * job_info->grid_size[1]))) / job_info->grid_size[0];
1229 unsigned grid_x = (iter_idx - (grid_z * (job_info->grid_size[0] * job_info->grid_size[1])) - (grid_y * job_info->grid_size[0]));
1230 struct lp_compute_shader_variant *variant = job_info->current->variant;
1231 variant->jit_function(&job_info->current->jit_context,
1232 job_info->block_size[0], job_info->block_size[1], job_info->block_size[2],
1233 grid_x, grid_y, grid_z,
1234 job_info->grid_size[0], job_info->grid_size[1], job_info->grid_size[2], job_info->work_dim,
1235 &thread_data);
1236 }
1237
1238 static void
1239 fill_grid_size(struct pipe_context *pipe,
1240 const struct pipe_grid_info *info,
1241 uint32_t grid_size[3])
1242 {
1243 struct pipe_transfer *transfer;
1244 uint32_t *params;
1245 if (!info->indirect) {
1246 grid_size[0] = info->grid[0];
1247 grid_size[1] = info->grid[1];
1248 grid_size[2] = info->grid[2];
1249 return;
1250 }
1251 params = pipe_buffer_map_range(pipe, info->indirect,
1252 info->indirect_offset,
1253 3 * sizeof(uint32_t),
1254 PIPE_TRANSFER_READ,
1255 &transfer);
1256
1257 if (!transfer)
1258 return;
1259
1260 grid_size[0] = params[0];
1261 grid_size[1] = params[1];
1262 grid_size[2] = params[2];
1263 pipe_buffer_unmap(pipe, transfer);
1264 }
1265
1266 static void llvmpipe_launch_grid(struct pipe_context *pipe,
1267 const struct pipe_grid_info *info)
1268 {
1269 struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe);
1270 struct llvmpipe_screen *screen = llvmpipe_screen(pipe->screen);
1271 struct lp_cs_job_info job_info;
1272
1273 memset(&job_info, 0, sizeof(job_info));
1274
1275 llvmpipe_cs_update_derived(llvmpipe, info->input);
1276
1277 fill_grid_size(pipe, info, job_info.grid_size);
1278
1279 job_info.block_size[0] = info->block[0];
1280 job_info.block_size[1] = info->block[1];
1281 job_info.block_size[2] = info->block[2];
1282 job_info.work_dim = info->work_dim;
1283 job_info.req_local_mem = llvmpipe->cs->req_local_mem;
1284 job_info.current = &llvmpipe->csctx->cs.current;
1285
1286 int num_tasks = job_info.grid_size[2] * job_info.grid_size[1] * job_info.grid_size[0];
1287 if (num_tasks) {
1288 struct lp_cs_tpool_task *task;
1289 mtx_lock(&screen->cs_mutex);
1290 task = lp_cs_tpool_queue_task(screen->cs_tpool, cs_exec_fn, &job_info, num_tasks);
1291
1292 lp_cs_tpool_wait_for_task(screen->cs_tpool, &task);
1293 mtx_unlock(&screen->cs_mutex);
1294 }
1295 llvmpipe->pipeline_statistics.cs_invocations += num_tasks * info->block[0] * info->block[1] * info->block[2];
1296 }
1297
1298 static void
1299 llvmpipe_set_compute_resources(struct pipe_context *pipe,
1300 unsigned start, unsigned count,
1301 struct pipe_surface **resources)
1302 {
1303
1304
1305 }
1306
1307 static void
1308 llvmpipe_set_global_binding(struct pipe_context *pipe,
1309 unsigned first, unsigned count,
1310 struct pipe_resource **resources,
1311 uint32_t **handles)
1312 {
1313 struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe);
1314 struct lp_compute_shader *cs = llvmpipe->cs;
1315 unsigned i;
1316
1317 if (first + count > cs->max_global_buffers) {
1318 unsigned old_max = cs->max_global_buffers;
1319 cs->max_global_buffers = first + count;
1320 cs->global_buffers = realloc(cs->global_buffers,
1321 cs->max_global_buffers * sizeof(cs->global_buffers[0]));
1322 if (!cs->global_buffers) {
1323 return;
1324 }
1325
1326 memset(&cs->global_buffers[old_max], 0, (cs->max_global_buffers - old_max) * sizeof(cs->global_buffers[0]));
1327 }
1328
1329 if (!resources) {
1330 for (i = 0; i < count; i++)
1331 pipe_resource_reference(&cs->global_buffers[first + i], NULL);
1332 return;
1333 }
1334
1335 for (i = 0; i < count; i++) {
1336 uintptr_t va;
1337 uint32_t offset;
1338 pipe_resource_reference(&cs->global_buffers[first + i], resources[i]);
1339 struct llvmpipe_resource *lp_res = llvmpipe_resource(resources[i]);
1340 offset = *handles[i];
1341 va = (uintptr_t)((char *)lp_res->data + offset);
1342 memcpy(handles[i], &va, sizeof(va));
1343 }
1344 }
1345
1346 void
1347 llvmpipe_init_compute_funcs(struct llvmpipe_context *llvmpipe)
1348 {
1349 llvmpipe->pipe.create_compute_state = llvmpipe_create_compute_state;
1350 llvmpipe->pipe.bind_compute_state = llvmpipe_bind_compute_state;
1351 llvmpipe->pipe.delete_compute_state = llvmpipe_delete_compute_state;
1352 llvmpipe->pipe.set_compute_resources = llvmpipe_set_compute_resources;
1353 llvmpipe->pipe.set_global_binding = llvmpipe_set_global_binding;
1354 llvmpipe->pipe.launch_grid = llvmpipe_launch_grid;
1355 }
1356
1357 void
1358 lp_csctx_destroy(struct lp_cs_context *csctx)
1359 {
1360 unsigned i;
1361 for (i = 0; i < ARRAY_SIZE(csctx->cs.current_tex); i++) {
1362 pipe_resource_reference(&csctx->cs.current_tex[i], NULL);
1363 }
1364 for (i = 0; i < ARRAY_SIZE(csctx->constants); i++) {
1365 pipe_resource_reference(&csctx->constants[i].current.buffer, NULL);
1366 }
1367 for (i = 0; i < ARRAY_SIZE(csctx->ssbos); i++) {
1368 pipe_resource_reference(&csctx->ssbos[i].current.buffer, NULL);
1369 }
1370 FREE(csctx);
1371 }
1372
1373 struct lp_cs_context *lp_csctx_create(struct pipe_context *pipe)
1374 {
1375 struct lp_cs_context *csctx;
1376
1377 csctx = CALLOC_STRUCT(lp_cs_context);
1378 if (!csctx)
1379 return NULL;
1380
1381 csctx->pipe = pipe;
1382 return csctx;
1383 }