nir/vtn: SPIR-V bit count opcodes (core and extension) dest size mismatches nir
[mesa.git] / src / compiler / spirv / vtn_alu.c
1 /*
2 * Copyright © 2016 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include <math.h>
25 #include "vtn_private.h"
26 #include "spirv_info.h"
27
28 /*
29 * Normally, column vectors in SPIR-V correspond to a single NIR SSA
30 * definition. But for matrix multiplies, we want to do one routine for
31 * multiplying a matrix by a matrix and then pretend that vectors are matrices
32 * with one column. So we "wrap" these things, and unwrap the result before we
33 * send it off.
34 */
35
36 static struct vtn_ssa_value *
37 wrap_matrix(struct vtn_builder *b, struct vtn_ssa_value *val)
38 {
39 if (val == NULL)
40 return NULL;
41
42 if (glsl_type_is_matrix(val->type))
43 return val;
44
45 struct vtn_ssa_value *dest = rzalloc(b, struct vtn_ssa_value);
46 dest->type = glsl_get_bare_type(val->type);
47 dest->elems = ralloc_array(b, struct vtn_ssa_value *, 1);
48 dest->elems[0] = val;
49
50 return dest;
51 }
52
53 static struct vtn_ssa_value *
54 unwrap_matrix(struct vtn_ssa_value *val)
55 {
56 if (glsl_type_is_matrix(val->type))
57 return val;
58
59 return val->elems[0];
60 }
61
62 static struct vtn_ssa_value *
63 matrix_multiply(struct vtn_builder *b,
64 struct vtn_ssa_value *_src0, struct vtn_ssa_value *_src1)
65 {
66
67 struct vtn_ssa_value *src0 = wrap_matrix(b, _src0);
68 struct vtn_ssa_value *src1 = wrap_matrix(b, _src1);
69 struct vtn_ssa_value *src0_transpose = wrap_matrix(b, _src0->transposed);
70 struct vtn_ssa_value *src1_transpose = wrap_matrix(b, _src1->transposed);
71
72 unsigned src0_rows = glsl_get_vector_elements(src0->type);
73 unsigned src0_columns = glsl_get_matrix_columns(src0->type);
74 unsigned src1_columns = glsl_get_matrix_columns(src1->type);
75
76 const struct glsl_type *dest_type;
77 if (src1_columns > 1) {
78 dest_type = glsl_matrix_type(glsl_get_base_type(src0->type),
79 src0_rows, src1_columns);
80 } else {
81 dest_type = glsl_vector_type(glsl_get_base_type(src0->type), src0_rows);
82 }
83 struct vtn_ssa_value *dest = vtn_create_ssa_value(b, dest_type);
84
85 dest = wrap_matrix(b, dest);
86
87 bool transpose_result = false;
88 if (src0_transpose && src1_transpose) {
89 /* transpose(A) * transpose(B) = transpose(B * A) */
90 src1 = src0_transpose;
91 src0 = src1_transpose;
92 src0_transpose = NULL;
93 src1_transpose = NULL;
94 transpose_result = true;
95 }
96
97 if (src0_transpose && !src1_transpose &&
98 glsl_get_base_type(src0->type) == GLSL_TYPE_FLOAT) {
99 /* We already have the rows of src0 and the columns of src1 available,
100 * so we can just take the dot product of each row with each column to
101 * get the result.
102 */
103
104 for (unsigned i = 0; i < src1_columns; i++) {
105 nir_ssa_def *vec_src[4];
106 for (unsigned j = 0; j < src0_rows; j++) {
107 vec_src[j] = nir_fdot(&b->nb, src0_transpose->elems[j]->def,
108 src1->elems[i]->def);
109 }
110 dest->elems[i]->def = nir_vec(&b->nb, vec_src, src0_rows);
111 }
112 } else {
113 /* We don't handle the case where src1 is transposed but not src0, since
114 * the general case only uses individual components of src1 so the
115 * optimizer should chew through the transpose we emitted for src1.
116 */
117
118 for (unsigned i = 0; i < src1_columns; i++) {
119 /* dest[i] = sum(src0[j] * src1[i][j] for all j) */
120 dest->elems[i]->def =
121 nir_fmul(&b->nb, src0->elems[0]->def,
122 nir_channel(&b->nb, src1->elems[i]->def, 0));
123 for (unsigned j = 1; j < src0_columns; j++) {
124 dest->elems[i]->def =
125 nir_fadd(&b->nb, dest->elems[i]->def,
126 nir_fmul(&b->nb, src0->elems[j]->def,
127 nir_channel(&b->nb, src1->elems[i]->def, j)));
128 }
129 }
130 }
131
132 dest = unwrap_matrix(dest);
133
134 if (transpose_result)
135 dest = vtn_ssa_transpose(b, dest);
136
137 return dest;
138 }
139
140 static struct vtn_ssa_value *
141 mat_times_scalar(struct vtn_builder *b,
142 struct vtn_ssa_value *mat,
143 nir_ssa_def *scalar)
144 {
145 struct vtn_ssa_value *dest = vtn_create_ssa_value(b, mat->type);
146 for (unsigned i = 0; i < glsl_get_matrix_columns(mat->type); i++) {
147 if (glsl_base_type_is_integer(glsl_get_base_type(mat->type)))
148 dest->elems[i]->def = nir_imul(&b->nb, mat->elems[i]->def, scalar);
149 else
150 dest->elems[i]->def = nir_fmul(&b->nb, mat->elems[i]->def, scalar);
151 }
152
153 return dest;
154 }
155
156 static struct vtn_ssa_value *
157 vtn_handle_matrix_alu(struct vtn_builder *b, SpvOp opcode,
158 struct vtn_ssa_value *src0, struct vtn_ssa_value *src1)
159 {
160 switch (opcode) {
161 case SpvOpFNegate: {
162 struct vtn_ssa_value *dest = vtn_create_ssa_value(b, src0->type);
163 unsigned cols = glsl_get_matrix_columns(src0->type);
164 for (unsigned i = 0; i < cols; i++)
165 dest->elems[i]->def = nir_fneg(&b->nb, src0->elems[i]->def);
166 return dest;
167 }
168
169 case SpvOpFAdd: {
170 struct vtn_ssa_value *dest = vtn_create_ssa_value(b, src0->type);
171 unsigned cols = glsl_get_matrix_columns(src0->type);
172 for (unsigned i = 0; i < cols; i++)
173 dest->elems[i]->def =
174 nir_fadd(&b->nb, src0->elems[i]->def, src1->elems[i]->def);
175 return dest;
176 }
177
178 case SpvOpFSub: {
179 struct vtn_ssa_value *dest = vtn_create_ssa_value(b, src0->type);
180 unsigned cols = glsl_get_matrix_columns(src0->type);
181 for (unsigned i = 0; i < cols; i++)
182 dest->elems[i]->def =
183 nir_fsub(&b->nb, src0->elems[i]->def, src1->elems[i]->def);
184 return dest;
185 }
186
187 case SpvOpTranspose:
188 return vtn_ssa_transpose(b, src0);
189
190 case SpvOpMatrixTimesScalar:
191 if (src0->transposed) {
192 return vtn_ssa_transpose(b, mat_times_scalar(b, src0->transposed,
193 src1->def));
194 } else {
195 return mat_times_scalar(b, src0, src1->def);
196 }
197 break;
198
199 case SpvOpVectorTimesMatrix:
200 case SpvOpMatrixTimesVector:
201 case SpvOpMatrixTimesMatrix:
202 if (opcode == SpvOpVectorTimesMatrix) {
203 return matrix_multiply(b, vtn_ssa_transpose(b, src1), src0);
204 } else {
205 return matrix_multiply(b, src0, src1);
206 }
207 break;
208
209 default: vtn_fail_with_opcode("unknown matrix opcode", opcode);
210 }
211 }
212
213 nir_op
214 vtn_nir_alu_op_for_spirv_opcode(struct vtn_builder *b,
215 SpvOp opcode, bool *swap,
216 unsigned src_bit_size, unsigned dst_bit_size)
217 {
218 /* Indicates that the first two arguments should be swapped. This is
219 * used for implementing greater-than and less-than-or-equal.
220 */
221 *swap = false;
222
223 switch (opcode) {
224 case SpvOpSNegate: return nir_op_ineg;
225 case SpvOpFNegate: return nir_op_fneg;
226 case SpvOpNot: return nir_op_inot;
227 case SpvOpIAdd: return nir_op_iadd;
228 case SpvOpFAdd: return nir_op_fadd;
229 case SpvOpISub: return nir_op_isub;
230 case SpvOpFSub: return nir_op_fsub;
231 case SpvOpIMul: return nir_op_imul;
232 case SpvOpFMul: return nir_op_fmul;
233 case SpvOpUDiv: return nir_op_udiv;
234 case SpvOpSDiv: return nir_op_idiv;
235 case SpvOpFDiv: return nir_op_fdiv;
236 case SpvOpUMod: return nir_op_umod;
237 case SpvOpSMod: return nir_op_imod;
238 case SpvOpFMod: return nir_op_fmod;
239 case SpvOpSRem: return nir_op_irem;
240 case SpvOpFRem: return nir_op_frem;
241
242 case SpvOpShiftRightLogical: return nir_op_ushr;
243 case SpvOpShiftRightArithmetic: return nir_op_ishr;
244 case SpvOpShiftLeftLogical: return nir_op_ishl;
245 case SpvOpLogicalOr: return nir_op_ior;
246 case SpvOpLogicalEqual: return nir_op_ieq;
247 case SpvOpLogicalNotEqual: return nir_op_ine;
248 case SpvOpLogicalAnd: return nir_op_iand;
249 case SpvOpLogicalNot: return nir_op_inot;
250 case SpvOpBitwiseOr: return nir_op_ior;
251 case SpvOpBitwiseXor: return nir_op_ixor;
252 case SpvOpBitwiseAnd: return nir_op_iand;
253 case SpvOpSelect: return nir_op_bcsel;
254 case SpvOpIEqual: return nir_op_ieq;
255
256 case SpvOpBitFieldInsert: return nir_op_bitfield_insert;
257 case SpvOpBitFieldSExtract: return nir_op_ibitfield_extract;
258 case SpvOpBitFieldUExtract: return nir_op_ubitfield_extract;
259 case SpvOpBitReverse: return nir_op_bitfield_reverse;
260
261 case SpvOpUCountLeadingZerosINTEL: return nir_op_uclz;
262 /* SpvOpUCountTrailingZerosINTEL is handled elsewhere. */
263 case SpvOpAbsISubINTEL: return nir_op_uabs_isub;
264 case SpvOpAbsUSubINTEL: return nir_op_uabs_usub;
265 case SpvOpIAddSatINTEL: return nir_op_iadd_sat;
266 case SpvOpUAddSatINTEL: return nir_op_uadd_sat;
267 case SpvOpIAverageINTEL: return nir_op_ihadd;
268 case SpvOpUAverageINTEL: return nir_op_uhadd;
269 case SpvOpIAverageRoundedINTEL: return nir_op_irhadd;
270 case SpvOpUAverageRoundedINTEL: return nir_op_urhadd;
271 case SpvOpISubSatINTEL: return nir_op_isub_sat;
272 case SpvOpUSubSatINTEL: return nir_op_usub_sat;
273 case SpvOpIMul32x16INTEL: return nir_op_imul_32x16;
274 case SpvOpUMul32x16INTEL: return nir_op_umul_32x16;
275
276 /* The ordered / unordered operators need special implementation besides
277 * the logical operator to use since they also need to check if operands are
278 * ordered.
279 */
280 case SpvOpFOrdEqual: return nir_op_feq;
281 case SpvOpFUnordEqual: return nir_op_feq;
282 case SpvOpINotEqual: return nir_op_ine;
283 case SpvOpLessOrGreater: /* Deprecated, use OrdNotEqual */
284 case SpvOpFOrdNotEqual: return nir_op_fneu;
285 case SpvOpFUnordNotEqual: return nir_op_fneu;
286 case SpvOpULessThan: return nir_op_ult;
287 case SpvOpSLessThan: return nir_op_ilt;
288 case SpvOpFOrdLessThan: return nir_op_flt;
289 case SpvOpFUnordLessThan: return nir_op_flt;
290 case SpvOpUGreaterThan: *swap = true; return nir_op_ult;
291 case SpvOpSGreaterThan: *swap = true; return nir_op_ilt;
292 case SpvOpFOrdGreaterThan: *swap = true; return nir_op_flt;
293 case SpvOpFUnordGreaterThan: *swap = true; return nir_op_flt;
294 case SpvOpULessThanEqual: *swap = true; return nir_op_uge;
295 case SpvOpSLessThanEqual: *swap = true; return nir_op_ige;
296 case SpvOpFOrdLessThanEqual: *swap = true; return nir_op_fge;
297 case SpvOpFUnordLessThanEqual: *swap = true; return nir_op_fge;
298 case SpvOpUGreaterThanEqual: return nir_op_uge;
299 case SpvOpSGreaterThanEqual: return nir_op_ige;
300 case SpvOpFOrdGreaterThanEqual: return nir_op_fge;
301 case SpvOpFUnordGreaterThanEqual: return nir_op_fge;
302
303 /* Conversions: */
304 case SpvOpQuantizeToF16: return nir_op_fquantize2f16;
305 case SpvOpUConvert:
306 case SpvOpConvertFToU:
307 case SpvOpConvertFToS:
308 case SpvOpConvertSToF:
309 case SpvOpConvertUToF:
310 case SpvOpSConvert:
311 case SpvOpFConvert: {
312 nir_alu_type src_type;
313 nir_alu_type dst_type;
314
315 switch (opcode) {
316 case SpvOpConvertFToS:
317 src_type = nir_type_float;
318 dst_type = nir_type_int;
319 break;
320 case SpvOpConvertFToU:
321 src_type = nir_type_float;
322 dst_type = nir_type_uint;
323 break;
324 case SpvOpFConvert:
325 src_type = dst_type = nir_type_float;
326 break;
327 case SpvOpConvertSToF:
328 src_type = nir_type_int;
329 dst_type = nir_type_float;
330 break;
331 case SpvOpSConvert:
332 src_type = dst_type = nir_type_int;
333 break;
334 case SpvOpConvertUToF:
335 src_type = nir_type_uint;
336 dst_type = nir_type_float;
337 break;
338 case SpvOpUConvert:
339 src_type = dst_type = nir_type_uint;
340 break;
341 default:
342 unreachable("Invalid opcode");
343 }
344 src_type |= src_bit_size;
345 dst_type |= dst_bit_size;
346 return nir_type_conversion_op(src_type, dst_type, nir_rounding_mode_undef);
347 }
348 /* Derivatives: */
349 case SpvOpDPdx: return nir_op_fddx;
350 case SpvOpDPdy: return nir_op_fddy;
351 case SpvOpDPdxFine: return nir_op_fddx_fine;
352 case SpvOpDPdyFine: return nir_op_fddy_fine;
353 case SpvOpDPdxCoarse: return nir_op_fddx_coarse;
354 case SpvOpDPdyCoarse: return nir_op_fddy_coarse;
355
356 case SpvOpIsNormal: return nir_op_fisnormal;
357 case SpvOpIsFinite: return nir_op_fisfinite;
358
359 default:
360 vtn_fail("No NIR equivalent: %u", opcode);
361 }
362 }
363
364 static void
365 handle_no_contraction(struct vtn_builder *b, struct vtn_value *val, int member,
366 const struct vtn_decoration *dec, void *_void)
367 {
368 vtn_assert(dec->scope == VTN_DEC_DECORATION);
369 if (dec->decoration != SpvDecorationNoContraction)
370 return;
371
372 b->nb.exact = true;
373 }
374
375 static void
376 handle_rounding_mode(struct vtn_builder *b, struct vtn_value *val, int member,
377 const struct vtn_decoration *dec, void *_out_rounding_mode)
378 {
379 nir_rounding_mode *out_rounding_mode = _out_rounding_mode;
380 assert(dec->scope == VTN_DEC_DECORATION);
381 if (dec->decoration != SpvDecorationFPRoundingMode)
382 return;
383 switch (dec->operands[0]) {
384 case SpvFPRoundingModeRTE:
385 *out_rounding_mode = nir_rounding_mode_rtne;
386 break;
387 case SpvFPRoundingModeRTZ:
388 *out_rounding_mode = nir_rounding_mode_rtz;
389 break;
390 default:
391 unreachable("Not supported rounding mode");
392 break;
393 }
394 }
395
396 static void
397 handle_no_wrap(struct vtn_builder *b, struct vtn_value *val, int member,
398 const struct vtn_decoration *dec, void *_alu)
399 {
400 nir_alu_instr *alu = _alu;
401 switch (dec->decoration) {
402 case SpvDecorationNoSignedWrap:
403 alu->no_signed_wrap = true;
404 break;
405 case SpvDecorationNoUnsignedWrap:
406 alu->no_unsigned_wrap = true;
407 break;
408 default:
409 /* Do nothing. */
410 break;
411 }
412 }
413
414 void
415 vtn_handle_alu(struct vtn_builder *b, SpvOp opcode,
416 const uint32_t *w, unsigned count)
417 {
418 struct vtn_value *dest_val = vtn_untyped_value(b, w[2]);
419 const struct glsl_type *dest_type = vtn_get_type(b, w[1])->type;
420
421 vtn_foreach_decoration(b, dest_val, handle_no_contraction, NULL);
422
423 /* Collect the various SSA sources */
424 const unsigned num_inputs = count - 3;
425 struct vtn_ssa_value *vtn_src[4] = { NULL, };
426 for (unsigned i = 0; i < num_inputs; i++)
427 vtn_src[i] = vtn_ssa_value(b, w[i + 3]);
428
429 if (glsl_type_is_matrix(vtn_src[0]->type) ||
430 (num_inputs >= 2 && glsl_type_is_matrix(vtn_src[1]->type))) {
431 vtn_push_ssa_value(b, w[2],
432 vtn_handle_matrix_alu(b, opcode, vtn_src[0], vtn_src[1]));
433 b->nb.exact = b->exact;
434 return;
435 }
436
437 struct vtn_ssa_value *dest = vtn_create_ssa_value(b, dest_type);
438 nir_ssa_def *src[4] = { NULL, };
439 for (unsigned i = 0; i < num_inputs; i++) {
440 vtn_assert(glsl_type_is_vector_or_scalar(vtn_src[i]->type));
441 src[i] = vtn_src[i]->def;
442 }
443
444 switch (opcode) {
445 case SpvOpAny:
446 dest->def = nir_bany(&b->nb, src[0]);
447 break;
448
449 case SpvOpAll:
450 dest->def = nir_ball(&b->nb, src[0]);
451 break;
452
453 case SpvOpOuterProduct: {
454 for (unsigned i = 0; i < src[1]->num_components; i++) {
455 dest->elems[i]->def =
456 nir_fmul(&b->nb, src[0], nir_channel(&b->nb, src[1], i));
457 }
458 break;
459 }
460
461 case SpvOpDot:
462 dest->def = nir_fdot(&b->nb, src[0], src[1]);
463 break;
464
465 case SpvOpIAddCarry:
466 vtn_assert(glsl_type_is_struct_or_ifc(dest_type));
467 dest->elems[0]->def = nir_iadd(&b->nb, src[0], src[1]);
468 dest->elems[1]->def = nir_uadd_carry(&b->nb, src[0], src[1]);
469 break;
470
471 case SpvOpISubBorrow:
472 vtn_assert(glsl_type_is_struct_or_ifc(dest_type));
473 dest->elems[0]->def = nir_isub(&b->nb, src[0], src[1]);
474 dest->elems[1]->def = nir_usub_borrow(&b->nb, src[0], src[1]);
475 break;
476
477 case SpvOpUMulExtended: {
478 vtn_assert(glsl_type_is_struct_or_ifc(dest_type));
479 nir_ssa_def *umul = nir_umul_2x32_64(&b->nb, src[0], src[1]);
480 dest->elems[0]->def = nir_unpack_64_2x32_split_x(&b->nb, umul);
481 dest->elems[1]->def = nir_unpack_64_2x32_split_y(&b->nb, umul);
482 break;
483 }
484
485 case SpvOpSMulExtended: {
486 vtn_assert(glsl_type_is_struct_or_ifc(dest_type));
487 nir_ssa_def *smul = nir_imul_2x32_64(&b->nb, src[0], src[1]);
488 dest->elems[0]->def = nir_unpack_64_2x32_split_x(&b->nb, smul);
489 dest->elems[1]->def = nir_unpack_64_2x32_split_y(&b->nb, smul);
490 break;
491 }
492
493 case SpvOpFwidth:
494 dest->def = nir_fadd(&b->nb,
495 nir_fabs(&b->nb, nir_fddx(&b->nb, src[0])),
496 nir_fabs(&b->nb, nir_fddy(&b->nb, src[0])));
497 break;
498 case SpvOpFwidthFine:
499 dest->def = nir_fadd(&b->nb,
500 nir_fabs(&b->nb, nir_fddx_fine(&b->nb, src[0])),
501 nir_fabs(&b->nb, nir_fddy_fine(&b->nb, src[0])));
502 break;
503 case SpvOpFwidthCoarse:
504 dest->def = nir_fadd(&b->nb,
505 nir_fabs(&b->nb, nir_fddx_coarse(&b->nb, src[0])),
506 nir_fabs(&b->nb, nir_fddy_coarse(&b->nb, src[0])));
507 break;
508
509 case SpvOpVectorTimesScalar:
510 /* The builder will take care of splatting for us. */
511 dest->def = nir_fmul(&b->nb, src[0], src[1]);
512 break;
513
514 case SpvOpIsNan:
515 dest->def = nir_fneu(&b->nb, src[0], src[0]);
516 break;
517
518 case SpvOpOrdered:
519 dest->def = nir_iand(&b->nb, nir_feq(&b->nb, src[0], src[0]),
520 nir_feq(&b->nb, src[1], src[1]));
521 break;
522
523 case SpvOpUnordered:
524 dest->def = nir_ior(&b->nb, nir_fneu(&b->nb, src[0], src[0]),
525 nir_fneu(&b->nb, src[1], src[1]));
526 break;
527
528 case SpvOpIsInf: {
529 nir_ssa_def *inf = nir_imm_floatN_t(&b->nb, INFINITY, src[0]->bit_size);
530 dest->def = nir_ieq(&b->nb, nir_fabs(&b->nb, src[0]), inf);
531 break;
532 }
533
534 case SpvOpFUnordEqual:
535 case SpvOpFUnordNotEqual:
536 case SpvOpFUnordLessThan:
537 case SpvOpFUnordGreaterThan:
538 case SpvOpFUnordLessThanEqual:
539 case SpvOpFUnordGreaterThanEqual: {
540 bool swap;
541 unsigned src_bit_size = glsl_get_bit_size(vtn_src[0]->type);
542 unsigned dst_bit_size = glsl_get_bit_size(dest_type);
543 nir_op op = vtn_nir_alu_op_for_spirv_opcode(b, opcode, &swap,
544 src_bit_size, dst_bit_size);
545
546 if (swap) {
547 nir_ssa_def *tmp = src[0];
548 src[0] = src[1];
549 src[1] = tmp;
550 }
551
552 dest->def =
553 nir_ior(&b->nb,
554 nir_build_alu(&b->nb, op, src[0], src[1], NULL, NULL),
555 nir_ior(&b->nb,
556 nir_fneu(&b->nb, src[0], src[0]),
557 nir_fneu(&b->nb, src[1], src[1])));
558 break;
559 }
560
561 case SpvOpLessOrGreater:
562 case SpvOpFOrdNotEqual: {
563 /* For all the SpvOpFOrd* comparisons apart from NotEqual, the value
564 * from the ALU will probably already be false if the operands are not
565 * ordered so we don’t need to handle it specially.
566 */
567 bool swap;
568 unsigned src_bit_size = glsl_get_bit_size(vtn_src[0]->type);
569 unsigned dst_bit_size = glsl_get_bit_size(dest_type);
570 nir_op op = vtn_nir_alu_op_for_spirv_opcode(b, opcode, &swap,
571 src_bit_size, dst_bit_size);
572
573 assert(!swap);
574
575 dest->def =
576 nir_iand(&b->nb,
577 nir_build_alu(&b->nb, op, src[0], src[1], NULL, NULL),
578 nir_iand(&b->nb,
579 nir_feq(&b->nb, src[0], src[0]),
580 nir_feq(&b->nb, src[1], src[1])));
581 break;
582 }
583
584 case SpvOpFConvert: {
585 nir_alu_type src_alu_type = nir_get_nir_type_for_glsl_type(vtn_src[0]->type);
586 nir_alu_type dst_alu_type = nir_get_nir_type_for_glsl_type(dest_type);
587 nir_rounding_mode rounding_mode = nir_rounding_mode_undef;
588
589 vtn_foreach_decoration(b, dest_val, handle_rounding_mode, &rounding_mode);
590 nir_op op = nir_type_conversion_op(src_alu_type, dst_alu_type, rounding_mode);
591
592 dest->def = nir_build_alu(&b->nb, op, src[0], src[1], NULL, NULL);
593 break;
594 }
595
596 case SpvOpBitFieldInsert:
597 case SpvOpBitFieldSExtract:
598 case SpvOpBitFieldUExtract:
599 case SpvOpShiftLeftLogical:
600 case SpvOpShiftRightArithmetic:
601 case SpvOpShiftRightLogical: {
602 bool swap;
603 unsigned src0_bit_size = glsl_get_bit_size(vtn_src[0]->type);
604 unsigned dst_bit_size = glsl_get_bit_size(dest_type);
605 nir_op op = vtn_nir_alu_op_for_spirv_opcode(b, opcode, &swap,
606 src0_bit_size, dst_bit_size);
607
608 assert (op == nir_op_ushr || op == nir_op_ishr || op == nir_op_ishl ||
609 op == nir_op_bitfield_insert || op == nir_op_ubitfield_extract ||
610 op == nir_op_ibitfield_extract);
611
612 for (unsigned i = 0; i < nir_op_infos[op].num_inputs; i++) {
613 unsigned src_bit_size =
614 nir_alu_type_get_type_size(nir_op_infos[op].input_types[i]);
615 if (src_bit_size == 0)
616 continue;
617 if (src_bit_size != src[i]->bit_size) {
618 assert(src_bit_size == 32);
619 /* Convert the Shift, Offset and Count operands to 32 bits, which is the bitsize
620 * supported by the NIR instructions. See discussion here:
621 *
622 * https://lists.freedesktop.org/archives/mesa-dev/2018-April/193026.html
623 */
624 src[i] = nir_u2u32(&b->nb, src[i]);
625 }
626 }
627 dest->def = nir_build_alu(&b->nb, op, src[0], src[1], src[2], src[3]);
628 break;
629 }
630
631 case SpvOpSignBitSet:
632 dest->def = nir_i2b(&b->nb,
633 nir_ushr(&b->nb, src[0], nir_imm_int(&b->nb, src[0]->bit_size - 1)));
634 break;
635
636 case SpvOpUCountTrailingZerosINTEL:
637 dest->def = nir_umin(&b->nb,
638 nir_find_lsb(&b->nb, src[0]),
639 nir_imm_int(&b->nb, 32u));
640 break;
641
642 case SpvOpBitCount: {
643 /* bit_count always returns int32, but the SPIR-V opcode just says the return
644 * value needs to be big enough to store the number of bits.
645 */
646 dest->def = nir_u2u(&b->nb, nir_bit_count(&b->nb, src[0]), glsl_get_bit_size(dest_type));
647 break;
648 }
649
650 default: {
651 bool swap;
652 unsigned src_bit_size = glsl_get_bit_size(vtn_src[0]->type);
653 unsigned dst_bit_size = glsl_get_bit_size(dest_type);
654 nir_op op = vtn_nir_alu_op_for_spirv_opcode(b, opcode, &swap,
655 src_bit_size, dst_bit_size);
656
657 if (swap) {
658 nir_ssa_def *tmp = src[0];
659 src[0] = src[1];
660 src[1] = tmp;
661 }
662
663 switch (op) {
664 case nir_op_ishl:
665 case nir_op_ishr:
666 case nir_op_ushr:
667 if (src[1]->bit_size != 32)
668 src[1] = nir_u2u32(&b->nb, src[1]);
669 break;
670 default:
671 break;
672 }
673
674 dest->def = nir_build_alu(&b->nb, op, src[0], src[1], src[2], src[3]);
675 break;
676 } /* default */
677 }
678
679 switch (opcode) {
680 case SpvOpIAdd:
681 case SpvOpIMul:
682 case SpvOpISub:
683 case SpvOpShiftLeftLogical:
684 case SpvOpSNegate: {
685 nir_alu_instr *alu = nir_instr_as_alu(dest->def->parent_instr);
686 vtn_foreach_decoration(b, dest_val, handle_no_wrap, alu);
687 break;
688 }
689 default:
690 /* Do nothing. */
691 break;
692 }
693
694 vtn_push_ssa_value(b, w[2], dest);
695
696 b->nb.exact = b->exact;
697 }
698
699 void
700 vtn_handle_bitcast(struct vtn_builder *b, const uint32_t *w, unsigned count)
701 {
702 vtn_assert(count == 4);
703 /* From the definition of OpBitcast in the SPIR-V 1.2 spec:
704 *
705 * "If Result Type has the same number of components as Operand, they
706 * must also have the same component width, and results are computed per
707 * component.
708 *
709 * If Result Type has a different number of components than Operand, the
710 * total number of bits in Result Type must equal the total number of
711 * bits in Operand. Let L be the type, either Result Type or Operand’s
712 * type, that has the larger number of components. Let S be the other
713 * type, with the smaller number of components. The number of components
714 * in L must be an integer multiple of the number of components in S.
715 * The first component (that is, the only or lowest-numbered component)
716 * of S maps to the first components of L, and so on, up to the last
717 * component of S mapping to the last components of L. Within this
718 * mapping, any single component of S (mapping to multiple components of
719 * L) maps its lower-ordered bits to the lower-numbered components of L."
720 */
721
722 struct vtn_type *type = vtn_get_type(b, w[1]);
723 struct nir_ssa_def *src = vtn_get_nir_ssa(b, w[3]);
724
725 vtn_fail_if(src->num_components * src->bit_size !=
726 glsl_get_vector_elements(type->type) * glsl_get_bit_size(type->type),
727 "Source and destination of OpBitcast must have the same "
728 "total number of bits");
729 nir_ssa_def *val =
730 nir_bitcast_vector(&b->nb, src, glsl_get_bit_size(type->type));
731 vtn_push_nir_ssa(b, w[2], val);
732 }