nir/vtn: SPIR-V bit count opcodes (core and extension) dest size mismatches nir
[mesa.git] / src / compiler / spirv / vtn_opencl.c
1 /*
2 * Copyright © 2018 Red Hat
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Rob Clark (robdclark@gmail.com)
25 */
26
27 #include "math.h"
28 #include "nir/nir_builtin_builder.h"
29
30 #include "vtn_private.h"
31 #include "OpenCL.std.h"
32
33 typedef nir_ssa_def *(*nir_handler)(struct vtn_builder *b,
34 enum OpenCLstd_Entrypoints opcode,
35 unsigned num_srcs, nir_ssa_def **srcs,
36 const struct glsl_type *dest_type);
37
38 static void
39 handle_instr(struct vtn_builder *b, enum OpenCLstd_Entrypoints opcode,
40 const uint32_t *w, unsigned count, nir_handler handler)
41 {
42 const struct glsl_type *dest_type = vtn_get_type(b, w[1])->type;
43
44 unsigned num_srcs = count - 5;
45 nir_ssa_def *srcs[3] = { NULL };
46 vtn_assert(num_srcs <= ARRAY_SIZE(srcs));
47 for (unsigned i = 0; i < num_srcs; i++) {
48 srcs[i] = vtn_get_nir_ssa(b, w[i + 5]);
49 }
50
51 nir_ssa_def *result = handler(b, opcode, num_srcs, srcs, dest_type);
52 if (result) {
53 vtn_push_nir_ssa(b, w[2], result);
54 } else {
55 vtn_assert(dest_type == glsl_void_type());
56 }
57 }
58
59 static nir_op
60 nir_alu_op_for_opencl_opcode(struct vtn_builder *b,
61 enum OpenCLstd_Entrypoints opcode)
62 {
63 switch (opcode) {
64 case OpenCLstd_Fabs: return nir_op_fabs;
65 case OpenCLstd_SAbs: return nir_op_iabs;
66 case OpenCLstd_SAdd_sat: return nir_op_iadd_sat;
67 case OpenCLstd_UAdd_sat: return nir_op_uadd_sat;
68 case OpenCLstd_Ceil: return nir_op_fceil;
69 case OpenCLstd_Cos: return nir_op_fcos;
70 case OpenCLstd_Exp2: return nir_op_fexp2;
71 case OpenCLstd_Log2: return nir_op_flog2;
72 case OpenCLstd_Floor: return nir_op_ffloor;
73 case OpenCLstd_SHadd: return nir_op_ihadd;
74 case OpenCLstd_UHadd: return nir_op_uhadd;
75 case OpenCLstd_Fma: return nir_op_ffma;
76 case OpenCLstd_Fmax: return nir_op_fmax;
77 case OpenCLstd_SMax: return nir_op_imax;
78 case OpenCLstd_UMax: return nir_op_umax;
79 case OpenCLstd_Fmin: return nir_op_fmin;
80 case OpenCLstd_SMin: return nir_op_imin;
81 case OpenCLstd_UMin: return nir_op_umin;
82 case OpenCLstd_Fmod: return nir_op_fmod;
83 case OpenCLstd_Mix: return nir_op_flrp;
84 case OpenCLstd_Native_cos: return nir_op_fcos;
85 case OpenCLstd_Native_divide: return nir_op_fdiv;
86 case OpenCLstd_Native_exp2: return nir_op_fexp2;
87 case OpenCLstd_Native_log2: return nir_op_flog2;
88 case OpenCLstd_Native_powr: return nir_op_fpow;
89 case OpenCLstd_Native_recip: return nir_op_frcp;
90 case OpenCLstd_Native_rsqrt: return nir_op_frsq;
91 case OpenCLstd_Native_sin: return nir_op_fsin;
92 case OpenCLstd_Native_sqrt: return nir_op_fsqrt;
93 case OpenCLstd_SMul_hi: return nir_op_imul_high;
94 case OpenCLstd_UMul_hi: return nir_op_umul_high;
95 case OpenCLstd_Popcount: return nir_op_bit_count;
96 case OpenCLstd_Pow: return nir_op_fpow;
97 case OpenCLstd_Remainder: return nir_op_frem;
98 case OpenCLstd_SRhadd: return nir_op_irhadd;
99 case OpenCLstd_URhadd: return nir_op_urhadd;
100 case OpenCLstd_Rsqrt: return nir_op_frsq;
101 case OpenCLstd_Sign: return nir_op_fsign;
102 case OpenCLstd_Sin: return nir_op_fsin;
103 case OpenCLstd_Sqrt: return nir_op_fsqrt;
104 case OpenCLstd_SSub_sat: return nir_op_isub_sat;
105 case OpenCLstd_USub_sat: return nir_op_usub_sat;
106 case OpenCLstd_Trunc: return nir_op_ftrunc;
107 case OpenCLstd_Rint: return nir_op_fround_even;
108 /* uhm... */
109 case OpenCLstd_UAbs: return nir_op_mov;
110 default:
111 vtn_fail("No NIR equivalent");
112 }
113 }
114
115 static nir_ssa_def *
116 handle_alu(struct vtn_builder *b, enum OpenCLstd_Entrypoints opcode,
117 unsigned num_srcs, nir_ssa_def **srcs,
118 const struct glsl_type *dest_type)
119 {
120 nir_ssa_def *ret = nir_build_alu(&b->nb, nir_alu_op_for_opencl_opcode(b, opcode),
121 srcs[0], srcs[1], srcs[2], NULL);
122 if (opcode == OpenCLstd_Popcount)
123 ret = nir_u2u(&b->nb, ret, glsl_get_bit_size(dest_type));
124 return ret;
125 }
126
127 static nir_ssa_def *
128 handle_special(struct vtn_builder *b, enum OpenCLstd_Entrypoints opcode,
129 unsigned num_srcs, nir_ssa_def **srcs,
130 const struct glsl_type *dest_type)
131 {
132 nir_builder *nb = &b->nb;
133
134 switch (opcode) {
135 case OpenCLstd_SAbs_diff:
136 return nir_iabs_diff(nb, srcs[0], srcs[1]);
137 case OpenCLstd_UAbs_diff:
138 return nir_uabs_diff(nb, srcs[0], srcs[1]);
139 case OpenCLstd_Bitselect:
140 return nir_bitselect(nb, srcs[0], srcs[1], srcs[2]);
141 case OpenCLstd_SMad_hi:
142 return nir_imad_hi(nb, srcs[0], srcs[1], srcs[2]);
143 case OpenCLstd_UMad_hi:
144 return nir_umad_hi(nb, srcs[0], srcs[1], srcs[2]);
145 case OpenCLstd_SMul24:
146 return nir_imul24(nb, srcs[0], srcs[1]);
147 case OpenCLstd_UMul24:
148 return nir_umul24(nb, srcs[0], srcs[1]);
149 case OpenCLstd_SMad24:
150 return nir_imad24(nb, srcs[0], srcs[1], srcs[2]);
151 case OpenCLstd_UMad24:
152 return nir_umad24(nb, srcs[0], srcs[1], srcs[2]);
153 case OpenCLstd_FClamp:
154 return nir_fclamp(nb, srcs[0], srcs[1], srcs[2]);
155 case OpenCLstd_SClamp:
156 return nir_iclamp(nb, srcs[0], srcs[1], srcs[2]);
157 case OpenCLstd_UClamp:
158 return nir_uclamp(nb, srcs[0], srcs[1], srcs[2]);
159 case OpenCLstd_Copysign:
160 return nir_copysign(nb, srcs[0], srcs[1]);
161 case OpenCLstd_Cross:
162 if (glsl_get_components(dest_type) == 4)
163 return nir_cross4(nb, srcs[0], srcs[1]);
164 return nir_cross3(nb, srcs[0], srcs[1]);
165 case OpenCLstd_Degrees:
166 return nir_degrees(nb, srcs[0]);
167 case OpenCLstd_Fdim:
168 return nir_fdim(nb, srcs[0], srcs[1]);
169 case OpenCLstd_Distance:
170 return nir_distance(nb, srcs[0], srcs[1]);
171 case OpenCLstd_Fast_distance:
172 return nir_fast_distance(nb, srcs[0], srcs[1]);
173 case OpenCLstd_Fast_length:
174 return nir_fast_length(nb, srcs[0]);
175 case OpenCLstd_Fast_normalize:
176 return nir_fast_normalize(nb, srcs[0]);
177 case OpenCLstd_Length:
178 return nir_length(nb, srcs[0]);
179 case OpenCLstd_Mad:
180 return nir_fmad(nb, srcs[0], srcs[1], srcs[2]);
181 case OpenCLstd_Maxmag:
182 return nir_maxmag(nb, srcs[0], srcs[1]);
183 case OpenCLstd_Minmag:
184 return nir_minmag(nb, srcs[0], srcs[1]);
185 case OpenCLstd_Nan:
186 return nir_nan(nb, srcs[0]);
187 case OpenCLstd_Nextafter:
188 return nir_nextafter(nb, srcs[0], srcs[1]);
189 case OpenCLstd_Normalize:
190 return nir_normalize(nb, srcs[0]);
191 case OpenCLstd_Radians:
192 return nir_radians(nb, srcs[0]);
193 case OpenCLstd_Rotate:
194 return nir_rotate(nb, srcs[0], srcs[1]);
195 case OpenCLstd_Smoothstep:
196 return nir_smoothstep(nb, srcs[0], srcs[1], srcs[2]);
197 case OpenCLstd_Clz:
198 return nir_clz_u(nb, srcs[0]);
199 case OpenCLstd_Select:
200 return nir_select(nb, srcs[0], srcs[1], srcs[2]);
201 case OpenCLstd_Step:
202 return nir_sge(nb, srcs[1], srcs[0]);
203 case OpenCLstd_S_Upsample:
204 case OpenCLstd_U_Upsample:
205 return nir_upsample(nb, srcs[0], srcs[1]);
206 case OpenCLstd_Native_exp:
207 return nir_fexp(nb, srcs[0]);
208 case OpenCLstd_Native_exp10:
209 return nir_fexp2(nb, nir_fmul_imm(nb, srcs[0], log(10) / log(2)));
210 case OpenCLstd_Native_log:
211 return nir_flog(nb, srcs[0]);
212 case OpenCLstd_Native_log10:
213 return nir_fmul_imm(nb, nir_flog2(nb, srcs[0]), log(2) / log(10));
214 case OpenCLstd_Native_tan:
215 return nir_ftan(nb, srcs[0]);
216 default:
217 vtn_fail("No NIR equivalent");
218 return NULL;
219 }
220 }
221
222 static void
223 _handle_v_load_store(struct vtn_builder *b, enum OpenCLstd_Entrypoints opcode,
224 const uint32_t *w, unsigned count, bool load)
225 {
226 struct vtn_type *type;
227 if (load)
228 type = vtn_get_type(b, w[1]);
229 else
230 type = vtn_get_value_type(b, w[5]);
231 unsigned a = load ? 0 : 1;
232
233 const struct glsl_type *dest_type = type->type;
234 unsigned components = glsl_get_vector_elements(dest_type);
235
236 nir_ssa_def *offset = vtn_get_nir_ssa(b, w[5 + a]);
237 struct vtn_value *p = vtn_value(b, w[6 + a], vtn_value_type_pointer);
238
239 struct vtn_ssa_value *comps[NIR_MAX_VEC_COMPONENTS];
240 nir_ssa_def *ncomps[NIR_MAX_VEC_COMPONENTS];
241
242 nir_ssa_def *moffset = nir_imul_imm(&b->nb, offset, components);
243 nir_deref_instr *deref = vtn_pointer_to_deref(b, p->pointer);
244
245 for (int i = 0; i < components; i++) {
246 nir_ssa_def *coffset = nir_iadd_imm(&b->nb, moffset, i);
247 nir_deref_instr *arr_deref = nir_build_deref_ptr_as_array(&b->nb, deref, coffset);
248
249 if (load) {
250 comps[i] = vtn_local_load(b, arr_deref, p->type->access);
251 ncomps[i] = comps[i]->def;
252 } else {
253 struct vtn_ssa_value *ssa = vtn_create_ssa_value(b, glsl_scalar_type(glsl_get_base_type(dest_type)));
254 struct vtn_ssa_value *val = vtn_ssa_value(b, w[5]);
255 ssa->def = nir_channel(&b->nb, val->def, i);
256 vtn_local_store(b, ssa, arr_deref, p->type->access);
257 }
258 }
259 if (load) {
260 vtn_push_nir_ssa(b, w[2], nir_vec(&b->nb, ncomps, components));
261 }
262 }
263
264 static void
265 vtn_handle_opencl_vload(struct vtn_builder *b, enum OpenCLstd_Entrypoints opcode,
266 const uint32_t *w, unsigned count)
267 {
268 _handle_v_load_store(b, opcode, w, count, true);
269 }
270
271 static void
272 vtn_handle_opencl_vstore(struct vtn_builder *b, enum OpenCLstd_Entrypoints opcode,
273 const uint32_t *w, unsigned count)
274 {
275 _handle_v_load_store(b, opcode, w, count, false);
276 }
277
278 static nir_ssa_def *
279 handle_printf(struct vtn_builder *b, enum OpenCLstd_Entrypoints opcode,
280 unsigned num_srcs, nir_ssa_def **srcs,
281 const struct glsl_type *dest_type)
282 {
283 /* hahah, yeah, right.. */
284 return nir_imm_int(&b->nb, -1);
285 }
286
287 static nir_ssa_def *
288 handle_round(struct vtn_builder *b, enum OpenCLstd_Entrypoints opcode,
289 unsigned num_srcs, nir_ssa_def **srcs,
290 const struct glsl_type *dest_type)
291 {
292 nir_ssa_def *src = srcs[0];
293 nir_builder *nb = &b->nb;
294 nir_ssa_def *half = nir_imm_floatN_t(nb, 0.5, src->bit_size);
295 nir_ssa_def *truncated = nir_ftrunc(nb, src);
296 nir_ssa_def *remainder = nir_fsub(nb, src, truncated);
297
298 return nir_bcsel(nb, nir_fge(nb, nir_fabs(nb, remainder), half),
299 nir_fadd(nb, truncated, nir_fsign(nb, src)), truncated);
300 }
301
302 static nir_ssa_def *
303 handle_shuffle(struct vtn_builder *b, enum OpenCLstd_Entrypoints opcode, unsigned num_srcs,
304 nir_ssa_def **srcs, const struct glsl_type *dest_type)
305 {
306 struct nir_ssa_def *input = srcs[0];
307 struct nir_ssa_def *mask = srcs[1];
308
309 unsigned out_elems = glsl_get_vector_elements(dest_type);
310 nir_ssa_def *outres[NIR_MAX_VEC_COMPONENTS];
311 unsigned in_elems = input->num_components;
312 if (mask->bit_size != 32)
313 mask = nir_u2u32(&b->nb, mask);
314 mask = nir_iand(&b->nb, mask, nir_imm_intN_t(&b->nb, in_elems - 1, mask->bit_size));
315 for (unsigned i = 0; i < out_elems; i++)
316 outres[i] = nir_vector_extract(&b->nb, input, nir_channel(&b->nb, mask, i));
317
318 return nir_vec(&b->nb, outres, out_elems);
319 }
320
321 static nir_ssa_def *
322 handle_shuffle2(struct vtn_builder *b, enum OpenCLstd_Entrypoints opcode, unsigned num_srcs,
323 nir_ssa_def **srcs, const struct glsl_type *dest_type)
324 {
325 struct nir_ssa_def *input0 = srcs[0];
326 struct nir_ssa_def *input1 = srcs[1];
327 struct nir_ssa_def *mask = srcs[2];
328
329 unsigned out_elems = glsl_get_vector_elements(dest_type);
330 nir_ssa_def *outres[NIR_MAX_VEC_COMPONENTS];
331 unsigned in_elems = input0->num_components;
332 unsigned total_mask = 2 * in_elems - 1;
333 unsigned half_mask = in_elems - 1;
334 if (mask->bit_size != 32)
335 mask = nir_u2u32(&b->nb, mask);
336 mask = nir_iand(&b->nb, mask, nir_imm_intN_t(&b->nb, total_mask, mask->bit_size));
337 for (unsigned i = 0; i < out_elems; i++) {
338 nir_ssa_def *this_mask = nir_channel(&b->nb, mask, i);
339 nir_ssa_def *vmask = nir_iand(&b->nb, this_mask, nir_imm_intN_t(&b->nb, half_mask, mask->bit_size));
340 nir_ssa_def *val0 = nir_vector_extract(&b->nb, input0, vmask);
341 nir_ssa_def *val1 = nir_vector_extract(&b->nb, input1, vmask);
342 nir_ssa_def *sel = nir_ilt(&b->nb, this_mask, nir_imm_intN_t(&b->nb, in_elems, mask->bit_size));
343 outres[i] = nir_bcsel(&b->nb, sel, val0, val1);
344 }
345 return nir_vec(&b->nb, outres, out_elems);
346 }
347
348 bool
349 vtn_handle_opencl_instruction(struct vtn_builder *b, SpvOp ext_opcode,
350 const uint32_t *w, unsigned count)
351 {
352 enum OpenCLstd_Entrypoints cl_opcode = (enum OpenCLstd_Entrypoints) ext_opcode;
353
354 switch (cl_opcode) {
355 case OpenCLstd_Fabs:
356 case OpenCLstd_SAbs:
357 case OpenCLstd_UAbs:
358 case OpenCLstd_SAdd_sat:
359 case OpenCLstd_UAdd_sat:
360 case OpenCLstd_Ceil:
361 case OpenCLstd_Cos:
362 case OpenCLstd_Exp2:
363 case OpenCLstd_Log2:
364 case OpenCLstd_Floor:
365 case OpenCLstd_Fma:
366 case OpenCLstd_Fmax:
367 case OpenCLstd_SHadd:
368 case OpenCLstd_UHadd:
369 case OpenCLstd_SMax:
370 case OpenCLstd_UMax:
371 case OpenCLstd_Fmin:
372 case OpenCLstd_SMin:
373 case OpenCLstd_UMin:
374 case OpenCLstd_Mix:
375 case OpenCLstd_Native_cos:
376 case OpenCLstd_Native_divide:
377 case OpenCLstd_Native_exp2:
378 case OpenCLstd_Native_log2:
379 case OpenCLstd_Native_powr:
380 case OpenCLstd_Native_recip:
381 case OpenCLstd_Native_rsqrt:
382 case OpenCLstd_Native_sin:
383 case OpenCLstd_Native_sqrt:
384 case OpenCLstd_Fmod:
385 case OpenCLstd_SMul_hi:
386 case OpenCLstd_UMul_hi:
387 case OpenCLstd_Popcount:
388 case OpenCLstd_Pow:
389 case OpenCLstd_Remainder:
390 case OpenCLstd_SRhadd:
391 case OpenCLstd_URhadd:
392 case OpenCLstd_Rsqrt:
393 case OpenCLstd_Sign:
394 case OpenCLstd_Sin:
395 case OpenCLstd_Sqrt:
396 case OpenCLstd_SSub_sat:
397 case OpenCLstd_USub_sat:
398 case OpenCLstd_Trunc:
399 case OpenCLstd_Rint:
400 handle_instr(b, cl_opcode, w, count, handle_alu);
401 return true;
402 case OpenCLstd_SAbs_diff:
403 case OpenCLstd_UAbs_diff:
404 case OpenCLstd_SMad_hi:
405 case OpenCLstd_UMad_hi:
406 case OpenCLstd_SMad24:
407 case OpenCLstd_UMad24:
408 case OpenCLstd_SMul24:
409 case OpenCLstd_UMul24:
410 case OpenCLstd_Bitselect:
411 case OpenCLstd_FClamp:
412 case OpenCLstd_SClamp:
413 case OpenCLstd_UClamp:
414 case OpenCLstd_Copysign:
415 case OpenCLstd_Cross:
416 case OpenCLstd_Degrees:
417 case OpenCLstd_Fdim:
418 case OpenCLstd_Distance:
419 case OpenCLstd_Fast_distance:
420 case OpenCLstd_Fast_length:
421 case OpenCLstd_Fast_normalize:
422 case OpenCLstd_Length:
423 case OpenCLstd_Mad:
424 case OpenCLstd_Maxmag:
425 case OpenCLstd_Minmag:
426 case OpenCLstd_Nan:
427 case OpenCLstd_Nextafter:
428 case OpenCLstd_Normalize:
429 case OpenCLstd_Radians:
430 case OpenCLstd_Rotate:
431 case OpenCLstd_Select:
432 case OpenCLstd_Step:
433 case OpenCLstd_Smoothstep:
434 case OpenCLstd_S_Upsample:
435 case OpenCLstd_U_Upsample:
436 case OpenCLstd_Clz:
437 case OpenCLstd_Native_exp:
438 case OpenCLstd_Native_exp10:
439 case OpenCLstd_Native_log:
440 case OpenCLstd_Native_log10:
441 case OpenCLstd_Native_tan:
442 handle_instr(b, cl_opcode, w, count, handle_special);
443 return true;
444 case OpenCLstd_Vloadn:
445 vtn_handle_opencl_vload(b, cl_opcode, w, count);
446 return true;
447 case OpenCLstd_Vstoren:
448 vtn_handle_opencl_vstore(b, cl_opcode, w, count);
449 return true;
450 case OpenCLstd_Shuffle:
451 handle_instr(b, cl_opcode, w, count, handle_shuffle);
452 return true;
453 case OpenCLstd_Shuffle2:
454 handle_instr(b, cl_opcode, w, count, handle_shuffle2);
455 return true;
456 case OpenCLstd_Round:
457 handle_instr(b, cl_opcode, w, count, handle_round);
458 return true;
459 case OpenCLstd_Printf:
460 handle_instr(b, cl_opcode, w, count, handle_printf);
461 return true;
462 case OpenCLstd_Prefetch:
463 /* TODO maybe add a nir instruction for this? */
464 return true;
465 default:
466 vtn_fail("unhandled opencl opc: %u\n", ext_opcode);
467 return false;
468 }
469 }