projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
llvmpipo/nir: free compute shader NIR
[mesa.git] / src / gallium / drivers / llvmpipe / lp_state_cs.c
1 /**************************************************************************
2 *
3 * Copyright 2019 Red Hat.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included
14 * in all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
17 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 *
24 **************************************************************************/
25 #include "util/u_memory.h"
26 #include "util/simple_list.h"
27 #include "util/os_time.h"
28 #include "util/u_dump.h"
29 #include "util/u_string.h"
30 #include "tgsi/tgsi_dump.h"
31 #include "tgsi/tgsi_parse.h"
32 #include "gallivm/lp_bld_const.h"
33 #include "gallivm/lp_bld_debug.h"
34 #include "gallivm/lp_bld_intr.h"
35 #include "gallivm/lp_bld_flow.h"
36 #include "gallivm/lp_bld_gather.h"
37 #include "gallivm/lp_bld_coro.h"
38 #include "gallivm/lp_bld_nir.h"
39 #include "lp_state_cs.h"
40 #include "lp_context.h"
41 #include "lp_debug.h"
42 #include "lp_state.h"
43 #include "lp_perf.h"
44 #include "lp_screen.h"
45 #include "lp_memory.h"
46 #include "lp_cs_tpool.h"
47 #include "state_tracker/sw_winsys.h"
48 #include "nir/nir_to_tgsi_info.h"
49 #include "nir_serialize.h"
50 struct lp_cs_job_info {
51 unsigned grid_size[3];
52 unsigned block_size[3];
53 unsigned req_local_mem;
54 unsigned work_dim;
55 struct lp_cs_exec *current;
56 };
57
58 static void
59 generate_compute(struct llvmpipe_context *lp,
60 struct lp_compute_shader *shader,
61 struct lp_compute_shader_variant *variant)
62 {
63 struct gallivm_state *gallivm = variant->gallivm;
64 const struct lp_compute_shader_variant_key *key = &variant->key;
65 char func_name[64], func_name_coro[64];
66 LLVMTypeRef arg_types[17];
67 LLVMTypeRef func_type, coro_func_type;
68 LLVMTypeRef int32_type = LLVMInt32TypeInContext(gallivm->context);
69 LLVMValueRef context_ptr;
70 LLVMValueRef x_size_arg, y_size_arg, z_size_arg;
71 LLVMValueRef grid_x_arg, grid_y_arg, grid_z_arg;
72 LLVMValueRef grid_size_x_arg, grid_size_y_arg, grid_size_z_arg;
73 LLVMValueRef work_dim_arg, thread_data_ptr;
74 LLVMBasicBlockRef block;
75 LLVMBuilderRef builder;
76 struct lp_build_sampler_soa *sampler;
77 struct lp_build_image_soa *image;
78 LLVMValueRef function, coro;
79 struct lp_type cs_type;
80 unsigned i;
81
82 /*
83 * This function has two parts
84 * a) setup the coroutine execution environment loop.
85 * b) build the compute shader llvm for use inside the coroutine.
86 */
87 assert(lp_native_vector_width / 32 >= 4);
88
89 memset(&cs_type, 0, sizeof cs_type);
90 cs_type.floating = TRUE; /* floating point values */
91 cs_type.sign = TRUE; /* values are signed */
92 cs_type.norm = FALSE; /* values are not limited to [0,1] or [-1,1] */
93 cs_type.width = 32; /* 32-bit float */
94 cs_type.length = MIN2(lp_native_vector_width / 32, 16); /* n*4 elements per vector */
95 snprintf(func_name, sizeof(func_name), "cs%u_variant%u",
96 shader->no, variant->no);
97
98 snprintf(func_name_coro, sizeof(func_name), "cs_co_%u_variant%u",
99 shader->no, variant->no);
100
101 arg_types[0] = variant->jit_cs_context_ptr_type; /* context */
102 arg_types[1] = int32_type; /* block_x_size */
103 arg_types[2] = int32_type; /* block_y_size */
104 arg_types[3] = int32_type; /* block_z_size */
105 arg_types[4] = int32_type; /* grid_x */
106 arg_types[5] = int32_type; /* grid_y */
107 arg_types[6] = int32_type; /* grid_z */
108 arg_types[7] = int32_type; /* grid_size_x */
109 arg_types[8] = int32_type; /* grid_size_y */
110 arg_types[9] = int32_type; /* grid_size_z */
111 arg_types[10] = int32_type; /* work dim */
112 arg_types[11] = variant->jit_cs_thread_data_ptr_type; /* per thread data */
113 arg_types[12] = int32_type; /* coro only - num X loops */
114 arg_types[13] = int32_type; /* coro only - partials */
115 arg_types[14] = int32_type; /* coro block_x_size */
116 arg_types[15] = int32_type; /* coro block_y_size */
117 arg_types[16] = int32_type; /* coro block_z_size */
118 func_type = LLVMFunctionType(LLVMVoidTypeInContext(gallivm->context),
119 arg_types, ARRAY_SIZE(arg_types) - 5, 0);
120
121 coro_func_type = LLVMFunctionType(LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0),
122 arg_types, ARRAY_SIZE(arg_types), 0);
123
124 function = LLVMAddFunction(gallivm->module, func_name, func_type);
125 LLVMSetFunctionCallConv(function, LLVMCCallConv);
126
127 coro = LLVMAddFunction(gallivm->module, func_name_coro, coro_func_type);
128 LLVMSetFunctionCallConv(coro, LLVMCCallConv);
129
130 variant->function = function;
131
132 for(i = 0; i < ARRAY_SIZE(arg_types); ++i) {
133 if(LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind) {
134 lp_add_function_attr(coro, i + 1, LP_FUNC_ATTR_NOALIAS);
135 lp_add_function_attr(function, i + 1, LP_FUNC_ATTR_NOALIAS);
136 }
137 }
138
139 context_ptr = LLVMGetParam(function, 0);
140 x_size_arg = LLVMGetParam(function, 1);
141 y_size_arg = LLVMGetParam(function, 2);
142 z_size_arg = LLVMGetParam(function, 3);
143 grid_x_arg = LLVMGetParam(function, 4);
144 grid_y_arg = LLVMGetParam(function, 5);
145 grid_z_arg = LLVMGetParam(function, 6);
146 grid_size_x_arg = LLVMGetParam(function, 7);
147 grid_size_y_arg = LLVMGetParam(function, 8);
148 grid_size_z_arg = LLVMGetParam(function, 9);
149 work_dim_arg = LLVMGetParam(function, 10);
150 thread_data_ptr = LLVMGetParam(function, 11);
151
152 lp_build_name(context_ptr, "context");
153 lp_build_name(x_size_arg, "x_size");
154 lp_build_name(y_size_arg, "y_size");
155 lp_build_name(z_size_arg, "z_size");
156 lp_build_name(grid_x_arg, "grid_x");
157 lp_build_name(grid_y_arg, "grid_y");
158 lp_build_name(grid_z_arg, "grid_z");
159 lp_build_name(grid_size_x_arg, "grid_size_x");
160 lp_build_name(grid_size_y_arg, "grid_size_y");
161 lp_build_name(grid_size_z_arg, "grid_size_z");
162 lp_build_name(work_dim_arg, "work_dim");
163 lp_build_name(thread_data_ptr, "thread_data");
164
165 block = LLVMAppendBasicBlockInContext(gallivm->context, function, "entry");
166 builder = gallivm->builder;
167 assert(builder);
168 LLVMPositionBuilderAtEnd(builder, block);
169 sampler = lp_llvm_sampler_soa_create(key->state);
170 image = lp_llvm_image_soa_create(key->image_state);
171
172 struct lp_build_loop_state loop_state[4];
173 LLVMValueRef num_x_loop;
174 LLVMValueRef vec_length = lp_build_const_int32(gallivm, cs_type.length);
175 num_x_loop = LLVMBuildAdd(gallivm->builder, x_size_arg, vec_length, "");
176 num_x_loop = LLVMBuildSub(gallivm->builder, num_x_loop, lp_build_const_int32(gallivm, 1), "");
177 num_x_loop = LLVMBuildUDiv(gallivm->builder, num_x_loop, vec_length, "");
178 LLVMValueRef partials = LLVMBuildURem(gallivm->builder, x_size_arg, vec_length, "");
179
180 LLVMValueRef coro_num_hdls = LLVMBuildMul(gallivm->builder, num_x_loop, y_size_arg, "");
181 coro_num_hdls = LLVMBuildMul(gallivm->builder, coro_num_hdls, z_size_arg, "");
182
183 LLVMTypeRef hdl_ptr_type = LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0);
184 LLVMValueRef coro_hdls = LLVMBuildArrayAlloca(gallivm->builder, hdl_ptr_type, coro_num_hdls, "coro_hdls");
185
186 unsigned end_coroutine = INT_MAX;
187
188 /*
189 * This is the main coroutine execution loop. It iterates over the dimensions
190 * and calls the coroutine main entrypoint on the first pass, but in subsequent
191 * passes it checks if the coroutine has completed and resumes it if not.
192 */
193 /* take x_width - round up to type.length width */
194 lp_build_loop_begin(&loop_state[3], gallivm,
195 lp_build_const_int32(gallivm, 0)); /* coroutine reentry loop */
196 lp_build_loop_begin(&loop_state[2], gallivm,
197 lp_build_const_int32(gallivm, 0)); /* z loop */
198 lp_build_loop_begin(&loop_state[1], gallivm,
199 lp_build_const_int32(gallivm, 0)); /* y loop */
200 lp_build_loop_begin(&loop_state[0], gallivm,
201 lp_build_const_int32(gallivm, 0)); /* x loop */
202 {
203 LLVMValueRef args[17];
204 args[0] = context_ptr;
205 args[1] = loop_state[0].counter;
206 args[2] = loop_state[1].counter;
207 args[3] = loop_state[2].counter;
208 args[4] = grid_x_arg;
209 args[5] = grid_y_arg;
210 args[6] = grid_z_arg;
211 args[7] = grid_size_x_arg;
212 args[8] = grid_size_y_arg;
213 args[9] = grid_size_z_arg;
214 args[10] = work_dim_arg;
215 args[11] = thread_data_ptr;
216 args[12] = num_x_loop;
217 args[13] = partials;
218 args[14] = x_size_arg;
219 args[15] = y_size_arg;
220 args[16] = z_size_arg;
221
222 /* idx = (z * (size_x * size_y) + y * size_x + x */
223 LLVMValueRef coro_hdl_idx = LLVMBuildMul(gallivm->builder, loop_state[2].counter,
224 LLVMBuildMul(gallivm->builder, num_x_loop, y_size_arg, ""), "");
225 coro_hdl_idx = LLVMBuildAdd(gallivm->builder, coro_hdl_idx,
226 LLVMBuildMul(gallivm->builder, loop_state[1].counter,
227 num_x_loop, ""), "");
228 coro_hdl_idx = LLVMBuildAdd(gallivm->builder, coro_hdl_idx,
229 loop_state[0].counter, "");
230
231 LLVMValueRef coro_entry = LLVMBuildGEP(gallivm->builder, coro_hdls, &coro_hdl_idx, 1, "");
232
233 LLVMValueRef coro_hdl = LLVMBuildLoad(gallivm->builder, coro_entry, "coro_hdl");
234
235 struct lp_build_if_state ifstate;
236 LLVMValueRef cmp = LLVMBuildICmp(gallivm->builder, LLVMIntEQ, loop_state[3].counter,
237 lp_build_const_int32(gallivm, 0), "");
238 /* first time here - call the coroutine function entry point */
239 lp_build_if(&ifstate, gallivm, cmp);
240 LLVMValueRef coro_ret = LLVMBuildCall(gallivm->builder, coro, args, 17, "");
241 LLVMBuildStore(gallivm->builder, coro_ret, coro_entry);
242 lp_build_else(&ifstate);
243 /* subsequent calls for this invocation - check if done. */
244 LLVMValueRef coro_done = lp_build_coro_done(gallivm, coro_hdl);
245 struct lp_build_if_state ifstate2;
246 lp_build_if(&ifstate2, gallivm, coro_done);
247 /* if done destroy and force loop exit */
248 lp_build_coro_destroy(gallivm, coro_hdl);
249 lp_build_loop_force_set_counter(&loop_state[3], lp_build_const_int32(gallivm, end_coroutine - 1));
250 lp_build_else(&ifstate2);
251 /* otherwise resume the coroutine */
252 lp_build_coro_resume(gallivm, coro_hdl);
253 lp_build_endif(&ifstate2);
254 lp_build_endif(&ifstate);
255 lp_build_loop_force_reload_counter(&loop_state[3]);
256 }
257 lp_build_loop_end_cond(&loop_state[0],
258 num_x_loop,
259 NULL, LLVMIntUGE);
260 lp_build_loop_end_cond(&loop_state[1],
261 y_size_arg,
262 NULL, LLVMIntUGE);
263 lp_build_loop_end_cond(&loop_state[2],
264 z_size_arg,
265 NULL, LLVMIntUGE);
266 lp_build_loop_end_cond(&loop_state[3],
267 lp_build_const_int32(gallivm, end_coroutine),
268 NULL, LLVMIntEQ);
269 LLVMBuildRetVoid(builder);
270
271 /* This is stage (b) - generate the compute shader code inside the coroutine. */
272 LLVMValueRef block_x_size_arg, block_y_size_arg, block_z_size_arg;
273 context_ptr = LLVMGetParam(coro, 0);
274 x_size_arg = LLVMGetParam(coro, 1);
275 y_size_arg = LLVMGetParam(coro, 2);
276 z_size_arg = LLVMGetParam(coro, 3);
277 grid_x_arg = LLVMGetParam(coro, 4);
278 grid_y_arg = LLVMGetParam(coro, 5);
279 grid_z_arg = LLVMGetParam(coro, 6);
280 grid_size_x_arg = LLVMGetParam(coro, 7);
281 grid_size_y_arg = LLVMGetParam(coro, 8);
282 grid_size_z_arg = LLVMGetParam(coro, 9);
283 work_dim_arg = LLVMGetParam(coro, 10);
284 thread_data_ptr = LLVMGetParam(coro, 11);
285 num_x_loop = LLVMGetParam(coro, 12);
286 partials = LLVMGetParam(coro, 13);
287 block_x_size_arg = LLVMGetParam(coro, 14);
288 block_y_size_arg = LLVMGetParam(coro, 15);
289 block_z_size_arg = LLVMGetParam(coro, 16);
290 block = LLVMAppendBasicBlockInContext(gallivm->context, coro, "entry");
291 LLVMPositionBuilderAtEnd(builder, block);
292 {
293 LLVMValueRef consts_ptr, num_consts_ptr;
294 LLVMValueRef ssbo_ptr, num_ssbo_ptr;
295 LLVMValueRef shared_ptr;
296 LLVMValueRef kernel_args_ptr;
297 struct lp_build_mask_context mask;
298 struct lp_bld_tgsi_system_values system_values;
299
300 memset(&system_values, 0, sizeof(system_values));
301 consts_ptr = lp_jit_cs_context_constants(gallivm, context_ptr);
302 num_consts_ptr = lp_jit_cs_context_num_constants(gallivm, context_ptr);
303 ssbo_ptr = lp_jit_cs_context_ssbos(gallivm, context_ptr);
304 num_ssbo_ptr = lp_jit_cs_context_num_ssbos(gallivm, context_ptr);
305 kernel_args_ptr = lp_jit_cs_context_kernel_args(gallivm, context_ptr);
306
307 shared_ptr = lp_jit_cs_thread_data_shared(gallivm, thread_data_ptr);
308
309 /* these are coroutine entrypoint necessities */
310 LLVMValueRef coro_id = lp_build_coro_id(gallivm);
311 LLVMValueRef coro_hdl = lp_build_coro_begin_alloc_mem(gallivm, coro_id);
312
313 LLVMValueRef has_partials = LLVMBuildICmp(gallivm->builder, LLVMIntNE, partials, lp_build_const_int32(gallivm, 0), "");
314 LLVMValueRef tid_vals[3];
315 LLVMValueRef tids_x[LP_MAX_VECTOR_LENGTH], tids_y[LP_MAX_VECTOR_LENGTH], tids_z[LP_MAX_VECTOR_LENGTH];
316 LLVMValueRef base_val = LLVMBuildMul(gallivm->builder, x_size_arg, vec_length, "");
317 for (i = 0; i < cs_type.length; i++) {
318 tids_x[i] = LLVMBuildAdd(gallivm->builder, base_val, lp_build_const_int32(gallivm, i), "");
319 tids_y[i] = y_size_arg;
320 tids_z[i] = z_size_arg;
321 }
322 tid_vals[0] = lp_build_gather_values(gallivm, tids_x, cs_type.length);
323 tid_vals[1] = lp_build_gather_values(gallivm, tids_y, cs_type.length);
324 tid_vals[2] = lp_build_gather_values(gallivm, tids_z, cs_type.length);
325 system_values.thread_id = LLVMGetUndef(LLVMArrayType(LLVMVectorType(int32_type, cs_type.length), 3));
326 for (i = 0; i < 3; i++)
327 system_values.thread_id = LLVMBuildInsertValue(builder, system_values.thread_id, tid_vals[i], i, "");
328
329 LLVMValueRef gtids[3] = { grid_x_arg, grid_y_arg, grid_z_arg };
330 system_values.block_id = LLVMGetUndef(LLVMVectorType(int32_type, 3));
331 for (i = 0; i < 3; i++)
332 system_values.block_id = LLVMBuildInsertElement(builder, system_values.block_id, gtids[i], lp_build_const_int32(gallivm, i), "");
333
334 LLVMValueRef gstids[3] = { grid_size_x_arg, grid_size_y_arg, grid_size_z_arg };
335 system_values.grid_size = LLVMGetUndef(LLVMVectorType(int32_type, 3));
336 for (i = 0; i < 3; i++)
337 system_values.grid_size = LLVMBuildInsertElement(builder, system_values.grid_size, gstids[i], lp_build_const_int32(gallivm, i), "");
338
339 system_values.work_dim = work_dim_arg;
340
341 LLVMValueRef bsize[3] = { block_x_size_arg, block_y_size_arg, block_z_size_arg };
342 system_values.block_size = LLVMGetUndef(LLVMVectorType(int32_type, 3));
343 for (i = 0; i < 3; i++)
344 system_values.block_size = LLVMBuildInsertElement(builder, system_values.block_size, bsize[i], lp_build_const_int32(gallivm, i), "");
345
346 LLVMValueRef last_x_loop = LLVMBuildICmp(gallivm->builder, LLVMIntEQ, x_size_arg, LLVMBuildSub(gallivm->builder, num_x_loop, lp_build_const_int32(gallivm, 1), ""), "");
347 LLVMValueRef use_partial_mask = LLVMBuildAnd(gallivm->builder, last_x_loop, has_partials, "");
348 struct lp_build_if_state if_state;
349 LLVMValueRef mask_val = lp_build_alloca(gallivm, LLVMVectorType(int32_type, cs_type.length), "mask");
350 LLVMValueRef full_mask_val = lp_build_const_int_vec(gallivm, cs_type, ~0);
351 LLVMBuildStore(gallivm->builder, full_mask_val, mask_val);
352
353 lp_build_if(&if_state, gallivm, use_partial_mask);
354 struct lp_build_loop_state mask_loop_state;
355 lp_build_loop_begin(&mask_loop_state, gallivm, partials);
356 LLVMValueRef tmask_val = LLVMBuildLoad(gallivm->builder, mask_val, "");
357 tmask_val = LLVMBuildInsertElement(gallivm->builder, tmask_val, lp_build_const_int32(gallivm, 0), mask_loop_state.counter, "");
358 LLVMBuildStore(gallivm->builder, tmask_val, mask_val);
359 lp_build_loop_end_cond(&mask_loop_state, vec_length, NULL, LLVMIntUGE);
360 lp_build_endif(&if_state);
361
362 mask_val = LLVMBuildLoad(gallivm->builder, mask_val, "");
363 lp_build_mask_begin(&mask, gallivm, cs_type, mask_val);
364
365 struct lp_build_coro_suspend_info coro_info;
366
367 LLVMBasicBlockRef sus_block = LLVMAppendBasicBlockInContext(gallivm->context, coro, "suspend");
368 LLVMBasicBlockRef clean_block = LLVMAppendBasicBlockInContext(gallivm->context, coro, "cleanup");
369
370 coro_info.suspend = sus_block;
371 coro_info.cleanup = clean_block;
372
373 struct lp_build_tgsi_params params;
374 memset(&params, 0, sizeof(params));
375
376 params.type = cs_type;
377 params.mask = &mask;
378 params.consts_ptr = consts_ptr;
379 params.const_sizes_ptr = num_consts_ptr;
380 params.system_values = &system_values;
381 params.context_ptr = context_ptr;
382 params.sampler = sampler;
383 params.info = &shader->info.base;
384 params.ssbo_ptr = ssbo_ptr;
385 params.ssbo_sizes_ptr = num_ssbo_ptr;
386 params.image = image;
387 params.shared_ptr = shared_ptr;
388 params.coro = &coro_info;
389 params.kernel_args = kernel_args_ptr;
390
391 if (shader->base.type == PIPE_SHADER_IR_TGSI)
392 lp_build_tgsi_soa(gallivm, shader->base.tokens, &params, NULL);
393 else
394 lp_build_nir_soa(gallivm, shader->base.ir.nir, &params,
395 NULL);
396
397 mask_val = lp_build_mask_end(&mask);
398
399 lp_build_coro_suspend_switch(gallivm, &coro_info, NULL, true);
400 LLVMPositionBuilderAtEnd(builder, clean_block);
401
402 lp_build_coro_free_mem(gallivm, coro_id, coro_hdl);
403
404 LLVMBuildBr(builder, sus_block);
405 LLVMPositionBuilderAtEnd(builder, sus_block);
406
407 lp_build_coro_end(gallivm, coro_hdl);
408 LLVMBuildRet(builder, coro_hdl);
409 }
410
411 sampler->destroy(sampler);
412 image->destroy(image);
413
414 gallivm_verify_function(gallivm, coro);
415 gallivm_verify_function(gallivm, function);
416 }
417
418 static void *
419 llvmpipe_create_compute_state(struct pipe_context *pipe,
420 const struct pipe_compute_state *templ)
421 {
422 struct lp_compute_shader *shader;
423 int nr_samplers, nr_sampler_views;
424 shader = CALLOC_STRUCT(lp_compute_shader);
425 if (!shader)
426 return NULL;
427
428 shader->base.type = templ->ir_type;
429 if (templ->ir_type == PIPE_SHADER_IR_NIR_SERIALIZED) {
430 struct blob_reader reader;
431 const struct pipe_binary_program_header *hdr = templ->prog;
432
433 blob_reader_init(&reader, hdr->blob, hdr->num_bytes);
434 shader->base.ir.nir = nir_deserialize(NULL, pipe->screen->get_compiler_options(pipe->screen, PIPE_SHADER_IR_NIR, PIPE_SHADER_COMPUTE), &reader);
435 shader->base.type = PIPE_SHADER_IR_NIR;
436
437 pipe->screen->finalize_nir(pipe->screen, shader->base.ir.nir, false);
438 } else if (templ->ir_type == PIPE_SHADER_IR_NIR)
439 shader->base.ir.nir = (struct nir_shader *)templ->prog;
440
441 if (shader->base.type == PIPE_SHADER_IR_TGSI) {
442 /* get/save the summary info for this shader */
443 lp_build_tgsi_info(templ->prog, &shader->info);
444
445 /* we need to keep a local copy of the tokens */
446 shader->base.tokens = tgsi_dup_tokens(templ->prog);
447 } else {
448 nir_tgsi_scan_shader(shader->base.ir.nir, &shader->info.base, false);
449 }
450
451 shader->req_local_mem = templ->req_local_mem;
452 make_empty_list(&shader->variants);
453
454 nr_samplers = shader->info.base.file_max[TGSI_FILE_SAMPLER] + 1;
455 nr_sampler_views = shader->info.base.file_max[TGSI_FILE_SAMPLER_VIEW] + 1;
456 shader->variant_key_size = Offset(struct lp_compute_shader_variant_key,
457 state[MAX2(nr_samplers, nr_sampler_views)]);
458 return shader;
459 }
460
461 static void
462 llvmpipe_bind_compute_state(struct pipe_context *pipe,
463 void *cs)
464 {
465 struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe);
466
467 if (llvmpipe->cs == cs)
468 return;
469
470 llvmpipe->cs = (struct lp_compute_shader *)cs;
471 llvmpipe->cs_dirty |= LP_CSNEW_CS;
472 }
473
474 /**
475 * Remove shader variant from two lists: the shader's variant list
476 * and the context's variant list.
477 */
478 static void
479 llvmpipe_remove_cs_shader_variant(struct llvmpipe_context *lp,
480 struct lp_compute_shader_variant *variant)
481 {
482 if ((LP_DEBUG & DEBUG_CS) || (gallivm_debug & GALLIVM_DEBUG_IR)) {
483 debug_printf("llvmpipe: del cs #%u var %u v created %u v cached %u "
484 "v total cached %u inst %u total inst %u\n",
485 variant->shader->no, variant->no,
486 variant->shader->variants_created,
487 variant->shader->variants_cached,
488 lp->nr_cs_variants, variant->nr_instrs, lp->nr_cs_instrs);
489 }
490
491 gallivm_destroy(variant->gallivm);
492
493 /* remove from shader's list */
494 remove_from_list(&variant->list_item_local);
495 variant->shader->variants_cached--;
496
497 /* remove from context's list */
498 remove_from_list(&variant->list_item_global);
499 lp->nr_fs_variants--;
500 lp->nr_fs_instrs -= variant->nr_instrs;
501
502 FREE(variant);
503 }
504
505 static void
506 llvmpipe_delete_compute_state(struct pipe_context *pipe,
507 void *cs)
508 {
509 struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe);
510 struct lp_compute_shader *shader = cs;
511 struct lp_cs_variant_list_item *li;
512
513 if (llvmpipe->cs == cs)
514 llvmpipe->cs = NULL;
515 for (unsigned i = 0; i < shader->max_global_buffers; i++)
516 pipe_resource_reference(&shader->global_buffers[i], NULL);
517 FREE(shader->global_buffers);
518
519 /* Delete all the variants */
520 li = first_elem(&shader->variants);
521 while(!at_end(&shader->variants, li)) {
522 struct lp_cs_variant_list_item *next = next_elem(li);
523 llvmpipe_remove_cs_shader_variant(llvmpipe, li->base);
524 li = next;
525 }
526 if (shader->base.ir.nir)
527 ralloc_free(shader->base.ir.nir);
528 tgsi_free_tokens(shader->base.tokens);
529 FREE(shader);
530 }
531
532 static void
533 make_variant_key(struct llvmpipe_context *lp,
534 struct lp_compute_shader *shader,
535 struct lp_compute_shader_variant_key *key)
536 {
537 int i;
538
539 memset(key, 0, shader->variant_key_size);
540
541 /* This value will be the same for all the variants of a given shader:
542 */
543 key->nr_samplers = shader->info.base.file_max[TGSI_FILE_SAMPLER] + 1;
544
545 for(i = 0; i < key->nr_samplers; ++i) {
546 if(shader->info.base.file_mask[TGSI_FILE_SAMPLER] & (1 << i)) {
547 lp_sampler_static_sampler_state(&key->state[i].sampler_state,
548 lp->samplers[PIPE_SHADER_COMPUTE][i]);
549 }
550 }
551
552 /*
553 * XXX If TGSI_FILE_SAMPLER_VIEW exists assume all texture opcodes
554 * are dx10-style? Can't really have mixed opcodes, at least not
555 * if we want to skip the holes here (without rescanning tgsi).
556 */
557 if (shader->info.base.file_max[TGSI_FILE_SAMPLER_VIEW] != -1) {
558 key->nr_sampler_views = shader->info.base.file_max[TGSI_FILE_SAMPLER_VIEW] + 1;
559 for(i = 0; i < key->nr_sampler_views; ++i) {
560 /*
561 * Note sview may exceed what's representable by file_mask.
562 * This will still work, the only downside is that not actually
563 * used views may be included in the shader key.
564 */
565 if(shader->info.base.file_mask[TGSI_FILE_SAMPLER_VIEW] & (1u << (i & 31))) {
566 lp_sampler_static_texture_state(&key->state[i].texture_state,
567 lp->sampler_views[PIPE_SHADER_COMPUTE][i]);
568 }
569 }
570 }
571 else {
572 key->nr_sampler_views = key->nr_samplers;
573 for(i = 0; i < key->nr_sampler_views; ++i) {
574 if(shader->info.base.file_mask[TGSI_FILE_SAMPLER] & (1 << i)) {
575 lp_sampler_static_texture_state(&key->state[i].texture_state,
576 lp->sampler_views[PIPE_SHADER_COMPUTE][i]);
577 }
578 }
579 }
580
581 key->nr_images = shader->info.base.file_max[TGSI_FILE_IMAGE] + 1;
582 for (i = 0; i < key->nr_images; ++i) {
583 if (shader->info.base.file_mask[TGSI_FILE_IMAGE] & (1 << i)) {
584 lp_sampler_static_texture_state_image(&key->image_state[i].image_state,
585 &lp->images[PIPE_SHADER_COMPUTE][i]);
586 }
587 }
588 }
589
590 static void
591 dump_cs_variant_key(const struct lp_compute_shader_variant_key *key)
592 {
593 int i;
594 debug_printf("cs variant %p:\n", (void *) key);
595
596 for (i = 0; i < key->nr_samplers; ++i) {
597 const struct lp_static_sampler_state *sampler = &key->state[i].sampler_state;
598 debug_printf("sampler[%u] = \n", i);
599 debug_printf(" .wrap = %s %s %s\n",
600 util_str_tex_wrap(sampler->wrap_s, TRUE),
601 util_str_tex_wrap(sampler->wrap_t, TRUE),
602 util_str_tex_wrap(sampler->wrap_r, TRUE));
603 debug_printf(" .min_img_filter = %s\n",
604 util_str_tex_filter(sampler->min_img_filter, TRUE));
605 debug_printf(" .min_mip_filter = %s\n",
606 util_str_tex_mipfilter(sampler->min_mip_filter, TRUE));
607 debug_printf(" .mag_img_filter = %s\n",
608 util_str_tex_filter(sampler->mag_img_filter, TRUE));
609 if (sampler->compare_mode != PIPE_TEX_COMPARE_NONE)
610 debug_printf(" .compare_func = %s\n", util_str_func(sampler->compare_func, TRUE));
611 debug_printf(" .normalized_coords = %u\n", sampler->normalized_coords);
612 debug_printf(" .min_max_lod_equal = %u\n", sampler->min_max_lod_equal);
613 debug_printf(" .lod_bias_non_zero = %u\n", sampler->lod_bias_non_zero);
614 debug_printf(" .apply_min_lod = %u\n", sampler->apply_min_lod);
615 debug_printf(" .apply_max_lod = %u\n", sampler->apply_max_lod);
616 }
617 for (i = 0; i < key->nr_sampler_views; ++i) {
618 const struct lp_static_texture_state *texture = &key->state[i].texture_state;
619 debug_printf("texture[%u] = \n", i);
620 debug_printf(" .format = %s\n",
621 util_format_name(texture->format));
622 debug_printf(" .target = %s\n",
623 util_str_tex_target(texture->target, TRUE));
624 debug_printf(" .level_zero_only = %u\n",
625 texture->level_zero_only);
626 debug_printf(" .pot = %u %u %u\n",
627 texture->pot_width,
628 texture->pot_height,
629 texture->pot_depth);
630 }
631 for (i = 0; i < key->nr_images; ++i) {
632 const struct lp_static_texture_state *image = &key->image_state[i].image_state;
633 debug_printf("image[%u] = \n", i);
634 debug_printf(" .format = %s\n",
635 util_format_name(image->format));
636 debug_printf(" .target = %s\n",
637 util_str_tex_target(image->target, TRUE));
638 debug_printf(" .level_zero_only = %u\n",
639 image->level_zero_only);
640 debug_printf(" .pot = %u %u %u\n",
641 image->pot_width,
642 image->pot_height,
643 image->pot_depth);
644 }
645 }
646
647 static void
648 lp_debug_cs_variant(const struct lp_compute_shader_variant *variant)
649 {
650 debug_printf("llvmpipe: Compute shader #%u variant #%u:\n",
651 variant->shader->no, variant->no);
652 if (variant->shader->base.type == PIPE_SHADER_IR_TGSI)
653 tgsi_dump(variant->shader->base.tokens, 0);
654 else
655 nir_print_shader(variant->shader->base.ir.nir, stderr);
656 dump_cs_variant_key(&variant->key);
657 debug_printf("\n");
658 }
659
660 static struct lp_compute_shader_variant *
661 generate_variant(struct llvmpipe_context *lp,
662 struct lp_compute_shader *shader,
663 const struct lp_compute_shader_variant_key *key)
664 {
665 struct lp_compute_shader_variant *variant;
666 char module_name[64];
667
668 variant = CALLOC_STRUCT(lp_compute_shader_variant);
669 if (!variant)
670 return NULL;
671
672 snprintf(module_name, sizeof(module_name), "cs%u_variant%u",
673 shader->no, shader->variants_created);
674
675 variant->gallivm = gallivm_create(module_name, lp->context);
676 if (!variant->gallivm) {
677 FREE(variant);
678 return NULL;
679 }
680
681 variant->shader = shader;
682 variant->list_item_global.base = variant;
683 variant->list_item_local.base = variant;
684 variant->no = shader->variants_created++;
685
686 memcpy(&variant->key, key, shader->variant_key_size);
687
688 if ((LP_DEBUG & DEBUG_CS) || (gallivm_debug & GALLIVM_DEBUG_IR)) {
689 lp_debug_cs_variant(variant);
690 }
691
692 lp_jit_init_cs_types(variant);
693
694 generate_compute(lp, shader, variant);
695
696 gallivm_compile_module(variant->gallivm);
697
698 variant->nr_instrs += lp_build_count_ir_module(variant->gallivm->module);
699
700 variant->jit_function = (lp_jit_cs_func)gallivm_jit_function(variant->gallivm, variant->function);
701
702 gallivm_free_ir(variant->gallivm);
703 return variant;
704 }
705
706 static void
707 lp_cs_ctx_set_cs_variant( struct lp_cs_context *csctx,
708 struct lp_compute_shader_variant *variant)
709 {
710 csctx->cs.current.variant = variant;
711 }
712
713 static void
714 llvmpipe_update_cs(struct llvmpipe_context *lp)
715 {
716 struct lp_compute_shader *shader = lp->cs;
717
718 struct lp_compute_shader_variant_key key;
719 struct lp_compute_shader_variant *variant = NULL;
720 struct lp_cs_variant_list_item *li;
721
722 make_variant_key(lp, shader, &key);
723
724 /* Search the variants for one which matches the key */
725 li = first_elem(&shader->variants);
726 while(!at_end(&shader->variants, li)) {
727 if(memcmp(&li->base->key, &key, shader->variant_key_size) == 0) {
728 variant = li->base;
729 break;
730 }
731 li = next_elem(li);
732 }
733
734 if (variant) {
735 /* Move this variant to the head of the list to implement LRU
736 * deletion of shader's when we have too many.
737 */
738 move_to_head(&lp->cs_variants_list, &variant->list_item_global);
739 }
740 else {
741 /* variant not found, create it now */
742 int64_t t0, t1, dt;
743 unsigned i;
744 unsigned variants_to_cull;
745
746 if (LP_DEBUG & DEBUG_CS) {
747 debug_printf("%u variants,\t%u instrs,\t%u instrs/variant\n",
748 lp->nr_cs_variants,
749 lp->nr_cs_instrs,
750 lp->nr_cs_variants ? lp->nr_cs_instrs / lp->nr_cs_variants : 0);
751 }
752
753 /* First, check if we've exceeded the max number of shader variants.
754 * If so, free 6.25% of them (the least recently used ones).
755 */
756 variants_to_cull = lp->nr_cs_variants >= LP_MAX_SHADER_VARIANTS ? LP_MAX_SHADER_VARIANTS / 16 : 0;
757
758 if (variants_to_cull ||
759 lp->nr_cs_instrs >= LP_MAX_SHADER_INSTRUCTIONS) {
760 if (gallivm_debug & GALLIVM_DEBUG_PERF) {
761 debug_printf("Evicting CS: %u cs variants,\t%u total variants,"
762 "\t%u instrs,\t%u instrs/variant\n",
763 shader->variants_cached,
764 lp->nr_cs_variants, lp->nr_cs_instrs,
765 lp->nr_cs_instrs / lp->nr_cs_variants);
766 }
767
768 /*
769 * We need to re-check lp->nr_cs_variants because an arbitrarliy large
770 * number of shader variants (potentially all of them) could be
771 * pending for destruction on flush.
772 */
773
774 for (i = 0; i < variants_to_cull || lp->nr_cs_instrs >= LP_MAX_SHADER_INSTRUCTIONS; i++) {
775 struct lp_cs_variant_list_item *item;
776 if (is_empty_list(&lp->cs_variants_list)) {
777 break;
778 }
779 item = last_elem(&lp->cs_variants_list);
780 assert(item);
781 assert(item->base);
782 llvmpipe_remove_cs_shader_variant(lp, item->base);
783 }
784 }
785 /*
786 * Generate the new variant.
787 */
788 t0 = os_time_get();
789 variant = generate_variant(lp, shader, &key);
790 t1 = os_time_get();
791 dt = t1 - t0;
792 LP_COUNT_ADD(llvm_compile_time, dt);
793 LP_COUNT_ADD(nr_llvm_compiles, 2); /* emit vs. omit in/out test */
794
795 /* Put the new variant into the list */
796 if (variant) {
797 insert_at_head(&shader->variants, &variant->list_item_local);
798 insert_at_head(&lp->cs_variants_list, &variant->list_item_global);
799 lp->nr_cs_variants++;
800 lp->nr_cs_instrs += variant->nr_instrs;
801 shader->variants_cached++;
802 }
803 }
804 /* Bind this variant */
805 lp_cs_ctx_set_cs_variant(lp->csctx, variant);
806 }
807
808 /**
809 * Called during state validation when LP_CSNEW_SAMPLER_VIEW is set.
810 */
811 static void
812 lp_csctx_set_sampler_views(struct lp_cs_context *csctx,
813 unsigned num,
814 struct pipe_sampler_view **views)
815 {
816 unsigned i, max_tex_num;
817
818 LP_DBG(DEBUG_SETUP, "%s\n", __FUNCTION__);
819
820 assert(num <= PIPE_MAX_SHADER_SAMPLER_VIEWS);
821
822 max_tex_num = MAX2(num, csctx->cs.current_tex_num);
823
824 for (i = 0; i < max_tex_num; i++) {
825 struct pipe_sampler_view *view = i < num ? views[i] : NULL;
826
827 if (view) {
828 struct pipe_resource *res = view->texture;
829 struct llvmpipe_resource *lp_tex = llvmpipe_resource(res);
830 struct lp_jit_texture *jit_tex;
831 jit_tex = &csctx->cs.current.jit_context.textures[i];
832
833 /* We're referencing the texture's internal data, so save a
834 * reference to it.
835 */
836 pipe_resource_reference(&csctx->cs.current_tex[i], res);
837
838 if (!lp_tex->dt) {
839 /* regular texture - csctx array of mipmap level offsets */
840 int j;
841 unsigned first_level = 0;
842 unsigned last_level = 0;
843
844 if (llvmpipe_resource_is_texture(res)) {
845 first_level = view->u.tex.first_level;
846 last_level = view->u.tex.last_level;
847 assert(first_level <= last_level);
848 assert(last_level <= res->last_level);
849 jit_tex->base = lp_tex->tex_data;
850 }
851 else {
852 jit_tex->base = lp_tex->data;
853 }
854 if (LP_PERF & PERF_TEX_MEM) {
855 /* use dummy tile memory */
856 jit_tex->base = lp_dummy_tile;
857 jit_tex->width = TILE_SIZE/8;
858 jit_tex->height = TILE_SIZE/8;
859 jit_tex->depth = 1;
860 jit_tex->first_level = 0;
861 jit_tex->last_level = 0;
862 jit_tex->mip_offsets[0] = 0;
863 jit_tex->row_stride[0] = 0;
864 jit_tex->img_stride[0] = 0;
865 }
866 else {
867 jit_tex->width = res->width0;
868 jit_tex->height = res->height0;
869 jit_tex->depth = res->depth0;
870 jit_tex->first_level = first_level;
871 jit_tex->last_level = last_level;
872
873 if (llvmpipe_resource_is_texture(res)) {
874 for (j = first_level; j <= last_level; j++) {
875 jit_tex->mip_offsets[j] = lp_tex->mip_offsets[j];
876 jit_tex->row_stride[j] = lp_tex->row_stride[j];
877 jit_tex->img_stride[j] = lp_tex->img_stride[j];
878 }
879
880 if (res->target == PIPE_TEXTURE_1D_ARRAY ||
881 res->target == PIPE_TEXTURE_2D_ARRAY ||
882 res->target == PIPE_TEXTURE_CUBE ||
883 res->target == PIPE_TEXTURE_CUBE_ARRAY) {
884 /*
885 * For array textures, we don't have first_layer, instead
886 * adjust last_layer (stored as depth) plus the mip level offsets
887 * (as we have mip-first layout can't just adjust base ptr).
888 * XXX For mip levels, could do something similar.
889 */
890 jit_tex->depth = view->u.tex.last_layer - view->u.tex.first_layer + 1;
891 for (j = first_level; j <= last_level; j++) {
892 jit_tex->mip_offsets[j] += view->u.tex.first_layer *
893 lp_tex->img_stride[j];
894 }
895 if (view->target == PIPE_TEXTURE_CUBE ||
896 view->target == PIPE_TEXTURE_CUBE_ARRAY) {
897 assert(jit_tex->depth % 6 == 0);
898 }
899 assert(view->u.tex.first_layer <= view->u.tex.last_layer);
900 assert(view->u.tex.last_layer < res->array_size);
901 }
902 }
903 else {
904 /*
905 * For buffers, we don't have "offset", instead adjust
906 * the size (stored as width) plus the base pointer.
907 */
908 unsigned view_blocksize = util_format_get_blocksize(view->format);
909 /* probably don't really need to fill that out */
910 jit_tex->mip_offsets[0] = 0;
911 jit_tex->row_stride[0] = 0;
912 jit_tex->img_stride[0] = 0;
913
914 /* everything specified in number of elements here. */
915 jit_tex->width = view->u.buf.size / view_blocksize;
916 jit_tex->base = (uint8_t *)jit_tex->base + view->u.buf.offset;
917 /* XXX Unsure if we need to sanitize parameters? */
918 assert(view->u.buf.offset + view->u.buf.size <= res->width0);
919 }
920 }
921 }
922 else {
923 /* display target texture/surface */
924 /*
925 * XXX: Where should this be unmapped?
926 */
927 struct llvmpipe_screen *screen = llvmpipe_screen(res->screen);
928 struct sw_winsys *winsys = screen->winsys;
929 jit_tex->base = winsys->displaytarget_map(winsys, lp_tex->dt,
930 PIPE_TRANSFER_READ);
931 jit_tex->row_stride[0] = lp_tex->row_stride[0];
932 jit_tex->img_stride[0] = lp_tex->img_stride[0];
933 jit_tex->mip_offsets[0] = 0;
934 jit_tex->width = res->width0;
935 jit_tex->height = res->height0;
936 jit_tex->depth = res->depth0;
937 jit_tex->first_level = jit_tex->last_level = 0;
938 assert(jit_tex->base);
939 }
940 }
941 else {
942 pipe_resource_reference(&csctx->cs.current_tex[i], NULL);
943 }
944 }
945 csctx->cs.current_tex_num = num;
946 }
947
948
949 /**
950 * Called during state validation when LP_NEW_SAMPLER is set.
951 */
952 static void
953 lp_csctx_set_sampler_state(struct lp_cs_context *csctx,
954 unsigned num,
955 struct pipe_sampler_state **samplers)
956 {
957 unsigned i;
958
959 LP_DBG(DEBUG_SETUP, "%s\n", __FUNCTION__);
960
961 assert(num <= PIPE_MAX_SAMPLERS);
962
963 for (i = 0; i < PIPE_MAX_SAMPLERS; i++) {
964 const struct pipe_sampler_state *sampler = i < num ? samplers[i] : NULL;
965
966 if (sampler) {
967 struct lp_jit_sampler *jit_sam;
968 jit_sam = &csctx->cs.current.jit_context.samplers[i];
969
970 jit_sam->min_lod = sampler->min_lod;
971 jit_sam->max_lod = sampler->max_lod;
972 jit_sam->lod_bias = sampler->lod_bias;
973 COPY_4V(jit_sam->border_color, sampler->border_color.f);
974 }
975 }
976 }
977
978 static void
979 lp_csctx_set_cs_constants(struct lp_cs_context *csctx,
980 unsigned num,
981 struct pipe_constant_buffer *buffers)
982 {
983 unsigned i;
984
985 LP_DBG(DEBUG_SETUP, "%s %p\n", __FUNCTION__, (void *) buffers);
986
987 assert(num <= ARRAY_SIZE(csctx->constants));
988
989 for (i = 0; i < num; ++i) {
990 util_copy_constant_buffer(&csctx->constants[i].current, &buffers[i]);
991 }
992 for (; i < ARRAY_SIZE(csctx->constants); i++) {
993 util_copy_constant_buffer(&csctx->constants[i].current, NULL);
994 }
995 }
996
997 static void
998 lp_csctx_set_cs_ssbos(struct lp_cs_context *csctx,
999 unsigned num,
1000 struct pipe_shader_buffer *buffers)
1001 {
1002 int i;
1003 LP_DBG(DEBUG_SETUP, "%s %p\n", __FUNCTION__, (void *)buffers);
1004
1005 assert (num <= ARRAY_SIZE(csctx->ssbos));
1006
1007 for (i = 0; i < num; ++i) {
1008 util_copy_shader_buffer(&csctx->ssbos[i].current, &buffers[i]);
1009 }
1010 for (; i < ARRAY_SIZE(csctx->ssbos); i++) {
1011 util_copy_shader_buffer(&csctx->ssbos[i].current, NULL);
1012 }
1013 }
1014
1015 static void
1016 lp_csctx_set_cs_images(struct lp_cs_context *csctx,
1017 unsigned num,
1018 struct pipe_image_view *images)
1019 {
1020 unsigned i;
1021
1022 LP_DBG(DEBUG_SETUP, "%s %p\n", __FUNCTION__, (void *) images);
1023
1024 assert(num <= ARRAY_SIZE(csctx->images));
1025
1026 for (i = 0; i < num; ++i) {
1027 struct pipe_image_view *image = &images[i];
1028 util_copy_image_view(&csctx->images[i].current, &images[i]);
1029
1030 struct pipe_resource *res = image->resource;
1031 struct llvmpipe_resource *lp_res = llvmpipe_resource(res);
1032 struct lp_jit_image *jit_image;
1033
1034 jit_image = &csctx->cs.current.jit_context.images[i];
1035 if (!lp_res)
1036 continue;
1037 if (!lp_res->dt) {
1038 /* regular texture - csctx array of mipmap level offsets */
1039 if (llvmpipe_resource_is_texture(res)) {
1040 jit_image->base = lp_res->tex_data;
1041 } else
1042 jit_image->base = lp_res->data;
1043
1044 jit_image->width = res->width0;
1045 jit_image->height = res->height0;
1046 jit_image->depth = res->depth0;
1047
1048 if (llvmpipe_resource_is_texture(res)) {
1049 uint32_t mip_offset = lp_res->mip_offsets[image->u.tex.level];
1050
1051 jit_image->width = u_minify(jit_image->width, image->u.tex.level);
1052 jit_image->height = u_minify(jit_image->height, image->u.tex.level);
1053
1054 if (res->target == PIPE_TEXTURE_1D_ARRAY ||
1055 res->target == PIPE_TEXTURE_2D_ARRAY ||
1056 res->target == PIPE_TEXTURE_3D ||
1057 res->target == PIPE_TEXTURE_CUBE ||
1058 res->target == PIPE_TEXTURE_CUBE_ARRAY) {
1059 /*
1060 * For array textures, we don't have first_layer, instead
1061 * adjust last_layer (stored as depth) plus the mip level offsets
1062 * (as we have mip-first layout can't just adjust base ptr).
1063 * XXX For mip levels, could do something similar.
1064 */
1065 jit_image->depth = image->u.tex.last_layer - image->u.tex.first_layer + 1;
1066 mip_offset += image->u.tex.first_layer * lp_res->img_stride[image->u.tex.level];
1067 } else
1068 jit_image->depth = u_minify(jit_image->depth, image->u.tex.level);
1069
1070 jit_image->row_stride = lp_res->row_stride[image->u.tex.level];
1071 jit_image->img_stride = lp_res->img_stride[image->u.tex.level];
1072 jit_image->base = (uint8_t *)jit_image->base + mip_offset;
1073 } else {
1074 unsigned view_blocksize = util_format_get_blocksize(image->format);
1075 jit_image->width = image->u.buf.size / view_blocksize;
1076 jit_image->base = (uint8_t *)jit_image->base + image->u.buf.offset;
1077 }
1078 }
1079 }
1080 for (; i < ARRAY_SIZE(csctx->images); i++) {
1081 util_copy_image_view(&csctx->images[i].current, NULL);
1082 }
1083 }
1084
1085 static void
1086 update_csctx_consts(struct llvmpipe_context *llvmpipe)
1087 {
1088 struct lp_cs_context *csctx = llvmpipe->csctx;
1089 int i;
1090
1091 for (i = 0; i < ARRAY_SIZE(csctx->constants); ++i) {
1092 struct pipe_resource *buffer = csctx->constants[i].current.buffer;
1093 const ubyte *current_data = NULL;
1094
1095 if (buffer) {
1096 /* resource buffer */
1097 current_data = (ubyte *) llvmpipe_resource_data(buffer);
1098 }
1099 else if (csctx->constants[i].current.user_buffer) {
1100 /* user-space buffer */
1101 current_data = (ubyte *) csctx->constants[i].current.user_buffer;
1102 }
1103
1104 if (current_data) {
1105 current_data += csctx->constants[i].current.buffer_offset;
1106
1107 csctx->cs.current.jit_context.constants[i] = (const float *)current_data;
1108 csctx->cs.current.jit_context.num_constants[i] = csctx->constants[i].current.buffer_size;
1109 } else {
1110 csctx->cs.current.jit_context.constants[i] = NULL;
1111 csctx->cs.current.jit_context.num_constants[i] = 0;
1112 }
1113 }
1114 }
1115
1116 static void
1117 update_csctx_ssbo(struct llvmpipe_context *llvmpipe)
1118 {
1119 struct lp_cs_context *csctx = llvmpipe->csctx;
1120 int i;
1121 for (i = 0; i < ARRAY_SIZE(csctx->ssbos); ++i) {
1122 struct pipe_resource *buffer = csctx->ssbos[i].current.buffer;
1123 const ubyte *current_data = NULL;
1124
1125 if (!buffer)
1126 continue;
1127 /* resource buffer */
1128 current_data = (ubyte *) llvmpipe_resource_data(buffer);
1129 if (current_data) {
1130 current_data += csctx->ssbos[i].current.buffer_offset;
1131
1132 csctx->cs.current.jit_context.ssbos[i] = (const uint32_t *)current_data;
1133 csctx->cs.current.jit_context.num_ssbos[i] = csctx->ssbos[i].current.buffer_size;
1134 } else {
1135 csctx->cs.current.jit_context.ssbos[i] = NULL;
1136 csctx->cs.current.jit_context.num_ssbos[i] = 0;
1137 }
1138 }
1139 }
1140
1141 static void
1142 llvmpipe_cs_update_derived(struct llvmpipe_context *llvmpipe, void *input)
1143 {
1144 if (llvmpipe->cs_dirty & (LP_CSNEW_CS))
1145 llvmpipe_update_cs(llvmpipe);
1146
1147 if (llvmpipe->cs_dirty & LP_CSNEW_CONSTANTS) {
1148 lp_csctx_set_cs_constants(llvmpipe->csctx,
1149 ARRAY_SIZE(llvmpipe->constants[PIPE_SHADER_COMPUTE]),
1150 llvmpipe->constants[PIPE_SHADER_COMPUTE]);
1151 update_csctx_consts(llvmpipe);
1152 }
1153
1154 if (llvmpipe->cs_dirty & LP_CSNEW_SSBOS) {
1155 lp_csctx_set_cs_ssbos(llvmpipe->csctx,
1156 ARRAY_SIZE(llvmpipe->ssbos[PIPE_SHADER_COMPUTE]),
1157 llvmpipe->ssbos[PIPE_SHADER_COMPUTE]);
1158 update_csctx_ssbo(llvmpipe);
1159 }
1160
1161 if (llvmpipe->cs_dirty & LP_CSNEW_SAMPLER_VIEW)
1162 lp_csctx_set_sampler_views(llvmpipe->csctx,
1163 llvmpipe->num_sampler_views[PIPE_SHADER_COMPUTE],
1164 llvmpipe->sampler_views[PIPE_SHADER_COMPUTE]);
1165
1166 if (llvmpipe->cs_dirty & LP_CSNEW_SAMPLER)
1167 lp_csctx_set_sampler_state(llvmpipe->csctx,
1168 llvmpipe->num_samplers[PIPE_SHADER_COMPUTE],
1169 llvmpipe->samplers[PIPE_SHADER_COMPUTE]);
1170
1171 if (llvmpipe->cs_dirty & LP_CSNEW_IMAGES)
1172 lp_csctx_set_cs_images(llvmpipe->csctx,
1173 ARRAY_SIZE(llvmpipe->images[PIPE_SHADER_COMPUTE]),
1174 llvmpipe->images[PIPE_SHADER_COMPUTE]);
1175
1176 if (input) {
1177 struct lp_cs_context *csctx = llvmpipe->csctx;
1178 csctx->input = input;
1179 csctx->cs.current.jit_context.kernel_args = input;
1180 }
1181
1182 llvmpipe->cs_dirty = 0;
1183 }
1184
1185 static void
1186 cs_exec_fn(void *init_data, int iter_idx, struct lp_cs_local_mem *lmem)
1187 {
1188 struct lp_cs_job_info *job_info = init_data;
1189 struct lp_jit_cs_thread_data thread_data;
1190
1191 memset(&thread_data, 0, sizeof(thread_data));
1192
1193 if (lmem->local_size < job_info->req_local_mem) {
1194 lmem->local_mem_ptr = REALLOC(lmem->local_mem_ptr, lmem->local_size,
1195 job_info->req_local_mem);
1196 lmem->local_size = job_info->req_local_mem;
1197 }
1198 thread_data.shared = lmem->local_mem_ptr;
1199
1200 unsigned grid_z = iter_idx / (job_info->grid_size[0] * job_info->grid_size[1]);
1201 unsigned grid_y = (iter_idx - (grid_z * (job_info->grid_size[0] * job_info->grid_size[1]))) / job_info->grid_size[0];
1202 unsigned grid_x = (iter_idx - (grid_z * (job_info->grid_size[0] * job_info->grid_size[1])) - (grid_y * job_info->grid_size[0]));
1203 struct lp_compute_shader_variant *variant = job_info->current->variant;
1204 variant->jit_function(&job_info->current->jit_context,
1205 job_info->block_size[0], job_info->block_size[1], job_info->block_size[2],
1206 grid_x, grid_y, grid_z,
1207 job_info->grid_size[0], job_info->grid_size[1], job_info->grid_size[2], job_info->work_dim,
1208 &thread_data);
1209 }
1210
1211 static void
1212 fill_grid_size(struct pipe_context *pipe,
1213 const struct pipe_grid_info *info,
1214 uint32_t grid_size[3])
1215 {
1216 struct pipe_transfer *transfer;
1217 uint32_t *params;
1218 if (!info->indirect) {
1219 grid_size[0] = info->grid[0];
1220 grid_size[1] = info->grid[1];
1221 grid_size[2] = info->grid[2];
1222 return;
1223 }
1224 params = pipe_buffer_map_range(pipe, info->indirect,
1225 info->indirect_offset,
1226 3 * sizeof(uint32_t),
1227 PIPE_TRANSFER_READ,
1228 &transfer);
1229
1230 if (!transfer)
1231 return;
1232
1233 grid_size[0] = params[0];
1234 grid_size[1] = params[1];
1235 grid_size[2] = params[2];
1236 pipe_buffer_unmap(pipe, transfer);
1237 }
1238
1239 static void llvmpipe_launch_grid(struct pipe_context *pipe,
1240 const struct pipe_grid_info *info)
1241 {
1242 struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe);
1243 struct llvmpipe_screen *screen = llvmpipe_screen(pipe->screen);
1244 struct lp_cs_job_info job_info;
1245
1246 memset(&job_info, 0, sizeof(job_info));
1247
1248 llvmpipe_cs_update_derived(llvmpipe, info->input);
1249
1250 fill_grid_size(pipe, info, job_info.grid_size);
1251
1252 job_info.block_size[0] = info->block[0];
1253 job_info.block_size[1] = info->block[1];
1254 job_info.block_size[2] = info->block[2];
1255 job_info.work_dim = info->work_dim;
1256 job_info.req_local_mem = llvmpipe->cs->req_local_mem;
1257 job_info.current = &llvmpipe->csctx->cs.current;
1258
1259 int num_tasks = job_info.grid_size[2] * job_info.grid_size[1] * job_info.grid_size[0];
1260 if (num_tasks) {
1261 struct lp_cs_tpool_task *task;
1262 mtx_lock(&screen->cs_mutex);
1263 task = lp_cs_tpool_queue_task(screen->cs_tpool, cs_exec_fn, &job_info, num_tasks);
1264
1265 lp_cs_tpool_wait_for_task(screen->cs_tpool, &task);
1266 mtx_unlock(&screen->cs_mutex);
1267 }
1268 llvmpipe->pipeline_statistics.cs_invocations += num_tasks * info->block[0] * info->block[1] * info->block[2];
1269 }
1270
1271 static void
1272 llvmpipe_set_compute_resources(struct pipe_context *pipe,
1273 unsigned start, unsigned count,
1274 struct pipe_surface **resources)
1275 {
1276
1277
1278 }
1279
1280 static void
1281 llvmpipe_set_global_binding(struct pipe_context *pipe,
1282 unsigned first, unsigned count,
1283 struct pipe_resource **resources,
1284 uint32_t **handles)
1285 {
1286 struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe);
1287 struct lp_compute_shader *cs = llvmpipe->cs;
1288 unsigned i;
1289
1290 if (first + count > cs->max_global_buffers) {
1291 unsigned old_max = cs->max_global_buffers;
1292 cs->max_global_buffers = first + count;
1293 cs->global_buffers = realloc(cs->global_buffers,
1294 cs->max_global_buffers * sizeof(cs->global_buffers[0]));
1295 if (!cs->global_buffers) {
1296 return;
1297 }
1298
1299 memset(&cs->global_buffers[old_max], 0, (cs->max_global_buffers - old_max) * sizeof(cs->global_buffers[0]));
1300 }
1301
1302 if (!resources) {
1303 for (i = 0; i < count; i++)
1304 pipe_resource_reference(&cs->global_buffers[first + i], NULL);
1305 return;
1306 }
1307
1308 for (i = 0; i < count; i++) {
1309 uintptr_t va;
1310 uint32_t offset;
1311 pipe_resource_reference(&cs->global_buffers[first + i], resources[i]);
1312 struct llvmpipe_resource *lp_res = llvmpipe_resource(resources[i]);
1313 offset = *handles[i];
1314 va = (uintptr_t)((char *)lp_res->data + offset);
1315 memcpy(handles[i], &va, sizeof(va));
1316 }
1317 }
1318
1319 void
1320 llvmpipe_init_compute_funcs(struct llvmpipe_context *llvmpipe)
1321 {
1322 llvmpipe->pipe.create_compute_state = llvmpipe_create_compute_state;
1323 llvmpipe->pipe.bind_compute_state = llvmpipe_bind_compute_state;
1324 llvmpipe->pipe.delete_compute_state = llvmpipe_delete_compute_state;
1325 llvmpipe->pipe.set_compute_resources = llvmpipe_set_compute_resources;
1326 llvmpipe->pipe.set_global_binding = llvmpipe_set_global_binding;
1327 llvmpipe->pipe.launch_grid = llvmpipe_launch_grid;
1328 }
1329
1330 void
1331 lp_csctx_destroy(struct lp_cs_context *csctx)
1332 {
1333 unsigned i;
1334 for (i = 0; i < ARRAY_SIZE(csctx->cs.current_tex); i++) {
1335 pipe_resource_reference(&csctx->cs.current_tex[i], NULL);
1336 }
1337 for (i = 0; i < ARRAY_SIZE(csctx->constants); i++) {
1338 pipe_resource_reference(&csctx->constants[i].current.buffer, NULL);
1339 }
1340 for (i = 0; i < ARRAY_SIZE(csctx->ssbos); i++) {
1341 pipe_resource_reference(&csctx->ssbos[i].current.buffer, NULL);
1342 }
1343 FREE(csctx);
1344 }
1345
1346 struct lp_cs_context *lp_csctx_create(struct pipe_context *pipe)
1347 {
1348 struct lp_cs_context *csctx;
1349
1350 csctx = CALLOC_STRUCT(lp_cs_context);
1351 if (!csctx)
1352 return NULL;
1353
1354 csctx->pipe = pipe;
1355 return csctx;
1356 }