2 * Copyright © 2018 Red Hat
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 * Rob Clark (robdclark@gmail.com)
28 #include "nir/nir_builtin_builder.h"
30 #include "vtn_private.h"
31 #include "OpenCL.std.h"
33 typedef nir_ssa_def
*(*nir_handler
)(struct vtn_builder
*b
,
34 enum OpenCLstd_Entrypoints opcode
,
35 unsigned num_srcs
, nir_ssa_def
**srcs
,
36 const struct glsl_type
*dest_type
);
39 handle_instr(struct vtn_builder
*b
, enum OpenCLstd_Entrypoints opcode
,
40 const uint32_t *w
, unsigned count
, nir_handler handler
)
42 const struct glsl_type
*dest_type
= vtn_get_type(b
, w
[1])->type
;
44 unsigned num_srcs
= count
- 5;
45 nir_ssa_def
*srcs
[3] = { NULL
};
46 vtn_assert(num_srcs
<= ARRAY_SIZE(srcs
));
47 for (unsigned i
= 0; i
< num_srcs
; i
++) {
48 srcs
[i
] = vtn_get_nir_ssa(b
, w
[i
+ 5]);
51 nir_ssa_def
*result
= handler(b
, opcode
, num_srcs
, srcs
, dest_type
);
53 vtn_push_nir_ssa(b
, w
[2], result
);
55 vtn_assert(dest_type
== glsl_void_type());
60 nir_alu_op_for_opencl_opcode(struct vtn_builder
*b
,
61 enum OpenCLstd_Entrypoints opcode
)
64 case OpenCLstd_Fabs
: return nir_op_fabs
;
65 case OpenCLstd_SAbs
: return nir_op_iabs
;
66 case OpenCLstd_SAdd_sat
: return nir_op_iadd_sat
;
67 case OpenCLstd_UAdd_sat
: return nir_op_uadd_sat
;
68 case OpenCLstd_Ceil
: return nir_op_fceil
;
69 case OpenCLstd_Cos
: return nir_op_fcos
;
70 case OpenCLstd_Exp2
: return nir_op_fexp2
;
71 case OpenCLstd_Log2
: return nir_op_flog2
;
72 case OpenCLstd_Floor
: return nir_op_ffloor
;
73 case OpenCLstd_SHadd
: return nir_op_ihadd
;
74 case OpenCLstd_UHadd
: return nir_op_uhadd
;
75 case OpenCLstd_Fma
: return nir_op_ffma
;
76 case OpenCLstd_Fmax
: return nir_op_fmax
;
77 case OpenCLstd_SMax
: return nir_op_imax
;
78 case OpenCLstd_UMax
: return nir_op_umax
;
79 case OpenCLstd_Fmin
: return nir_op_fmin
;
80 case OpenCLstd_SMin
: return nir_op_imin
;
81 case OpenCLstd_UMin
: return nir_op_umin
;
82 case OpenCLstd_Fmod
: return nir_op_fmod
;
83 case OpenCLstd_Mix
: return nir_op_flrp
;
84 case OpenCLstd_Native_cos
: return nir_op_fcos
;
85 case OpenCLstd_Native_divide
: return nir_op_fdiv
;
86 case OpenCLstd_Native_exp2
: return nir_op_fexp2
;
87 case OpenCLstd_Native_log2
: return nir_op_flog2
;
88 case OpenCLstd_Native_powr
: return nir_op_fpow
;
89 case OpenCLstd_Native_recip
: return nir_op_frcp
;
90 case OpenCLstd_Native_rsqrt
: return nir_op_frsq
;
91 case OpenCLstd_Native_sin
: return nir_op_fsin
;
92 case OpenCLstd_Native_sqrt
: return nir_op_fsqrt
;
93 case OpenCLstd_SMul_hi
: return nir_op_imul_high
;
94 case OpenCLstd_UMul_hi
: return nir_op_umul_high
;
95 case OpenCLstd_Popcount
: return nir_op_bit_count
;
96 case OpenCLstd_Pow
: return nir_op_fpow
;
97 case OpenCLstd_Remainder
: return nir_op_frem
;
98 case OpenCLstd_SRhadd
: return nir_op_irhadd
;
99 case OpenCLstd_URhadd
: return nir_op_urhadd
;
100 case OpenCLstd_Rsqrt
: return nir_op_frsq
;
101 case OpenCLstd_Sign
: return nir_op_fsign
;
102 case OpenCLstd_Sin
: return nir_op_fsin
;
103 case OpenCLstd_Sqrt
: return nir_op_fsqrt
;
104 case OpenCLstd_SSub_sat
: return nir_op_isub_sat
;
105 case OpenCLstd_USub_sat
: return nir_op_usub_sat
;
106 case OpenCLstd_Trunc
: return nir_op_ftrunc
;
107 case OpenCLstd_Rint
: return nir_op_fround_even
;
109 case OpenCLstd_UAbs
: return nir_op_mov
;
111 vtn_fail("No NIR equivalent");
116 handle_alu(struct vtn_builder
*b
, enum OpenCLstd_Entrypoints opcode
,
117 unsigned num_srcs
, nir_ssa_def
**srcs
,
118 const struct glsl_type
*dest_type
)
120 nir_ssa_def
*ret
= nir_build_alu(&b
->nb
, nir_alu_op_for_opencl_opcode(b
, opcode
),
121 srcs
[0], srcs
[1], srcs
[2], NULL
);
122 if (opcode
== OpenCLstd_Popcount
)
123 ret
= nir_u2u(&b
->nb
, ret
, glsl_get_bit_size(dest_type
));
128 handle_special(struct vtn_builder
*b
, enum OpenCLstd_Entrypoints opcode
,
129 unsigned num_srcs
, nir_ssa_def
**srcs
,
130 const struct glsl_type
*dest_type
)
132 nir_builder
*nb
= &b
->nb
;
135 case OpenCLstd_SAbs_diff
:
136 return nir_iabs_diff(nb
, srcs
[0], srcs
[1]);
137 case OpenCLstd_UAbs_diff
:
138 return nir_uabs_diff(nb
, srcs
[0], srcs
[1]);
139 case OpenCLstd_Bitselect
:
140 return nir_bitselect(nb
, srcs
[0], srcs
[1], srcs
[2]);
141 case OpenCLstd_SMad_hi
:
142 return nir_imad_hi(nb
, srcs
[0], srcs
[1], srcs
[2]);
143 case OpenCLstd_UMad_hi
:
144 return nir_umad_hi(nb
, srcs
[0], srcs
[1], srcs
[2]);
145 case OpenCLstd_SMul24
:
146 return nir_imul24(nb
, srcs
[0], srcs
[1]);
147 case OpenCLstd_UMul24
:
148 return nir_umul24(nb
, srcs
[0], srcs
[1]);
149 case OpenCLstd_SMad24
:
150 return nir_imad24(nb
, srcs
[0], srcs
[1], srcs
[2]);
151 case OpenCLstd_UMad24
:
152 return nir_umad24(nb
, srcs
[0], srcs
[1], srcs
[2]);
153 case OpenCLstd_FClamp
:
154 return nir_fclamp(nb
, srcs
[0], srcs
[1], srcs
[2]);
155 case OpenCLstd_SClamp
:
156 return nir_iclamp(nb
, srcs
[0], srcs
[1], srcs
[2]);
157 case OpenCLstd_UClamp
:
158 return nir_uclamp(nb
, srcs
[0], srcs
[1], srcs
[2]);
159 case OpenCLstd_Copysign
:
160 return nir_copysign(nb
, srcs
[0], srcs
[1]);
161 case OpenCLstd_Cross
:
162 if (glsl_get_components(dest_type
) == 4)
163 return nir_cross4(nb
, srcs
[0], srcs
[1]);
164 return nir_cross3(nb
, srcs
[0], srcs
[1]);
165 case OpenCLstd_Degrees
:
166 return nir_degrees(nb
, srcs
[0]);
168 return nir_fdim(nb
, srcs
[0], srcs
[1]);
169 case OpenCLstd_Distance
:
170 return nir_distance(nb
, srcs
[0], srcs
[1]);
171 case OpenCLstd_Fast_distance
:
172 return nir_fast_distance(nb
, srcs
[0], srcs
[1]);
173 case OpenCLstd_Fast_length
:
174 return nir_fast_length(nb
, srcs
[0]);
175 case OpenCLstd_Fast_normalize
:
176 return nir_fast_normalize(nb
, srcs
[0]);
177 case OpenCLstd_Length
:
178 return nir_length(nb
, srcs
[0]);
180 return nir_fmad(nb
, srcs
[0], srcs
[1], srcs
[2]);
181 case OpenCLstd_Maxmag
:
182 return nir_maxmag(nb
, srcs
[0], srcs
[1]);
183 case OpenCLstd_Minmag
:
184 return nir_minmag(nb
, srcs
[0], srcs
[1]);
186 return nir_nan(nb
, srcs
[0]);
187 case OpenCLstd_Nextafter
:
188 return nir_nextafter(nb
, srcs
[0], srcs
[1]);
189 case OpenCLstd_Normalize
:
190 return nir_normalize(nb
, srcs
[0]);
191 case OpenCLstd_Radians
:
192 return nir_radians(nb
, srcs
[0]);
193 case OpenCLstd_Rotate
:
194 return nir_rotate(nb
, srcs
[0], srcs
[1]);
195 case OpenCLstd_Smoothstep
:
196 return nir_smoothstep(nb
, srcs
[0], srcs
[1], srcs
[2]);
198 return nir_clz_u(nb
, srcs
[0]);
199 case OpenCLstd_Select
:
200 return nir_select(nb
, srcs
[0], srcs
[1], srcs
[2]);
202 return nir_sge(nb
, srcs
[1], srcs
[0]);
203 case OpenCLstd_S_Upsample
:
204 case OpenCLstd_U_Upsample
:
205 return nir_upsample(nb
, srcs
[0], srcs
[1]);
206 case OpenCLstd_Native_exp
:
207 return nir_fexp(nb
, srcs
[0]);
208 case OpenCLstd_Native_exp10
:
209 return nir_fexp2(nb
, nir_fmul_imm(nb
, srcs
[0], log(10) / log(2)));
210 case OpenCLstd_Native_log
:
211 return nir_flog(nb
, srcs
[0]);
212 case OpenCLstd_Native_log10
:
213 return nir_fmul_imm(nb
, nir_flog2(nb
, srcs
[0]), log(2) / log(10));
214 case OpenCLstd_Native_tan
:
215 return nir_ftan(nb
, srcs
[0]);
217 vtn_fail("No NIR equivalent");
223 _handle_v_load_store(struct vtn_builder
*b
, enum OpenCLstd_Entrypoints opcode
,
224 const uint32_t *w
, unsigned count
, bool load
)
226 struct vtn_type
*type
;
228 type
= vtn_get_type(b
, w
[1]);
230 type
= vtn_get_value_type(b
, w
[5]);
231 unsigned a
= load
? 0 : 1;
233 const struct glsl_type
*dest_type
= type
->type
;
234 unsigned components
= glsl_get_vector_elements(dest_type
);
236 nir_ssa_def
*offset
= vtn_get_nir_ssa(b
, w
[5 + a
]);
237 struct vtn_value
*p
= vtn_value(b
, w
[6 + a
], vtn_value_type_pointer
);
239 struct vtn_ssa_value
*comps
[NIR_MAX_VEC_COMPONENTS
];
240 nir_ssa_def
*ncomps
[NIR_MAX_VEC_COMPONENTS
];
242 nir_ssa_def
*moffset
= nir_imul_imm(&b
->nb
, offset
, components
);
243 nir_deref_instr
*deref
= vtn_pointer_to_deref(b
, p
->pointer
);
245 for (int i
= 0; i
< components
; i
++) {
246 nir_ssa_def
*coffset
= nir_iadd_imm(&b
->nb
, moffset
, i
);
247 nir_deref_instr
*arr_deref
= nir_build_deref_ptr_as_array(&b
->nb
, deref
, coffset
);
250 comps
[i
] = vtn_local_load(b
, arr_deref
, p
->type
->access
);
251 ncomps
[i
] = comps
[i
]->def
;
253 struct vtn_ssa_value
*ssa
= vtn_create_ssa_value(b
, glsl_scalar_type(glsl_get_base_type(dest_type
)));
254 struct vtn_ssa_value
*val
= vtn_ssa_value(b
, w
[5]);
255 ssa
->def
= nir_channel(&b
->nb
, val
->def
, i
);
256 vtn_local_store(b
, ssa
, arr_deref
, p
->type
->access
);
260 vtn_push_nir_ssa(b
, w
[2], nir_vec(&b
->nb
, ncomps
, components
));
265 vtn_handle_opencl_vload(struct vtn_builder
*b
, enum OpenCLstd_Entrypoints opcode
,
266 const uint32_t *w
, unsigned count
)
268 _handle_v_load_store(b
, opcode
, w
, count
, true);
272 vtn_handle_opencl_vstore(struct vtn_builder
*b
, enum OpenCLstd_Entrypoints opcode
,
273 const uint32_t *w
, unsigned count
)
275 _handle_v_load_store(b
, opcode
, w
, count
, false);
279 handle_printf(struct vtn_builder
*b
, enum OpenCLstd_Entrypoints opcode
,
280 unsigned num_srcs
, nir_ssa_def
**srcs
,
281 const struct glsl_type
*dest_type
)
283 /* hahah, yeah, right.. */
284 return nir_imm_int(&b
->nb
, -1);
288 handle_round(struct vtn_builder
*b
, enum OpenCLstd_Entrypoints opcode
,
289 unsigned num_srcs
, nir_ssa_def
**srcs
,
290 const struct glsl_type
*dest_type
)
292 nir_ssa_def
*src
= srcs
[0];
293 nir_builder
*nb
= &b
->nb
;
294 nir_ssa_def
*half
= nir_imm_floatN_t(nb
, 0.5, src
->bit_size
);
295 nir_ssa_def
*truncated
= nir_ftrunc(nb
, src
);
296 nir_ssa_def
*remainder
= nir_fsub(nb
, src
, truncated
);
298 return nir_bcsel(nb
, nir_fge(nb
, nir_fabs(nb
, remainder
), half
),
299 nir_fadd(nb
, truncated
, nir_fsign(nb
, src
)), truncated
);
303 handle_shuffle(struct vtn_builder
*b
, enum OpenCLstd_Entrypoints opcode
, unsigned num_srcs
,
304 nir_ssa_def
**srcs
, const struct glsl_type
*dest_type
)
306 struct nir_ssa_def
*input
= srcs
[0];
307 struct nir_ssa_def
*mask
= srcs
[1];
309 unsigned out_elems
= glsl_get_vector_elements(dest_type
);
310 nir_ssa_def
*outres
[NIR_MAX_VEC_COMPONENTS
];
311 unsigned in_elems
= input
->num_components
;
312 if (mask
->bit_size
!= 32)
313 mask
= nir_u2u32(&b
->nb
, mask
);
314 mask
= nir_iand(&b
->nb
, mask
, nir_imm_intN_t(&b
->nb
, in_elems
- 1, mask
->bit_size
));
315 for (unsigned i
= 0; i
< out_elems
; i
++)
316 outres
[i
] = nir_vector_extract(&b
->nb
, input
, nir_channel(&b
->nb
, mask
, i
));
318 return nir_vec(&b
->nb
, outres
, out_elems
);
322 handle_shuffle2(struct vtn_builder
*b
, enum OpenCLstd_Entrypoints opcode
, unsigned num_srcs
,
323 nir_ssa_def
**srcs
, const struct glsl_type
*dest_type
)
325 struct nir_ssa_def
*input0
= srcs
[0];
326 struct nir_ssa_def
*input1
= srcs
[1];
327 struct nir_ssa_def
*mask
= srcs
[2];
329 unsigned out_elems
= glsl_get_vector_elements(dest_type
);
330 nir_ssa_def
*outres
[NIR_MAX_VEC_COMPONENTS
];
331 unsigned in_elems
= input0
->num_components
;
332 unsigned total_mask
= 2 * in_elems
- 1;
333 unsigned half_mask
= in_elems
- 1;
334 if (mask
->bit_size
!= 32)
335 mask
= nir_u2u32(&b
->nb
, mask
);
336 mask
= nir_iand(&b
->nb
, mask
, nir_imm_intN_t(&b
->nb
, total_mask
, mask
->bit_size
));
337 for (unsigned i
= 0; i
< out_elems
; i
++) {
338 nir_ssa_def
*this_mask
= nir_channel(&b
->nb
, mask
, i
);
339 nir_ssa_def
*vmask
= nir_iand(&b
->nb
, this_mask
, nir_imm_intN_t(&b
->nb
, half_mask
, mask
->bit_size
));
340 nir_ssa_def
*val0
= nir_vector_extract(&b
->nb
, input0
, vmask
);
341 nir_ssa_def
*val1
= nir_vector_extract(&b
->nb
, input1
, vmask
);
342 nir_ssa_def
*sel
= nir_ilt(&b
->nb
, this_mask
, nir_imm_intN_t(&b
->nb
, in_elems
, mask
->bit_size
));
343 outres
[i
] = nir_bcsel(&b
->nb
, sel
, val0
, val1
);
345 return nir_vec(&b
->nb
, outres
, out_elems
);
349 vtn_handle_opencl_instruction(struct vtn_builder
*b
, SpvOp ext_opcode
,
350 const uint32_t *w
, unsigned count
)
352 enum OpenCLstd_Entrypoints cl_opcode
= (enum OpenCLstd_Entrypoints
) ext_opcode
;
358 case OpenCLstd_SAdd_sat
:
359 case OpenCLstd_UAdd_sat
:
364 case OpenCLstd_Floor
:
367 case OpenCLstd_SHadd
:
368 case OpenCLstd_UHadd
:
375 case OpenCLstd_Native_cos
:
376 case OpenCLstd_Native_divide
:
377 case OpenCLstd_Native_exp2
:
378 case OpenCLstd_Native_log2
:
379 case OpenCLstd_Native_powr
:
380 case OpenCLstd_Native_recip
:
381 case OpenCLstd_Native_rsqrt
:
382 case OpenCLstd_Native_sin
:
383 case OpenCLstd_Native_sqrt
:
385 case OpenCLstd_SMul_hi
:
386 case OpenCLstd_UMul_hi
:
387 case OpenCLstd_Popcount
:
389 case OpenCLstd_Remainder
:
390 case OpenCLstd_SRhadd
:
391 case OpenCLstd_URhadd
:
392 case OpenCLstd_Rsqrt
:
396 case OpenCLstd_SSub_sat
:
397 case OpenCLstd_USub_sat
:
398 case OpenCLstd_Trunc
:
400 handle_instr(b
, cl_opcode
, w
, count
, handle_alu
);
402 case OpenCLstd_SAbs_diff
:
403 case OpenCLstd_UAbs_diff
:
404 case OpenCLstd_SMad_hi
:
405 case OpenCLstd_UMad_hi
:
406 case OpenCLstd_SMad24
:
407 case OpenCLstd_UMad24
:
408 case OpenCLstd_SMul24
:
409 case OpenCLstd_UMul24
:
410 case OpenCLstd_Bitselect
:
411 case OpenCLstd_FClamp
:
412 case OpenCLstd_SClamp
:
413 case OpenCLstd_UClamp
:
414 case OpenCLstd_Copysign
:
415 case OpenCLstd_Cross
:
416 case OpenCLstd_Degrees
:
418 case OpenCLstd_Distance
:
419 case OpenCLstd_Fast_distance
:
420 case OpenCLstd_Fast_length
:
421 case OpenCLstd_Fast_normalize
:
422 case OpenCLstd_Length
:
424 case OpenCLstd_Maxmag
:
425 case OpenCLstd_Minmag
:
427 case OpenCLstd_Nextafter
:
428 case OpenCLstd_Normalize
:
429 case OpenCLstd_Radians
:
430 case OpenCLstd_Rotate
:
431 case OpenCLstd_Select
:
433 case OpenCLstd_Smoothstep
:
434 case OpenCLstd_S_Upsample
:
435 case OpenCLstd_U_Upsample
:
437 case OpenCLstd_Native_exp
:
438 case OpenCLstd_Native_exp10
:
439 case OpenCLstd_Native_log
:
440 case OpenCLstd_Native_log10
:
441 case OpenCLstd_Native_tan
:
442 handle_instr(b
, cl_opcode
, w
, count
, handle_special
);
444 case OpenCLstd_Vloadn
:
445 vtn_handle_opencl_vload(b
, cl_opcode
, w
, count
);
447 case OpenCLstd_Vstoren
:
448 vtn_handle_opencl_vstore(b
, cl_opcode
, w
, count
);
450 case OpenCLstd_Shuffle
:
451 handle_instr(b
, cl_opcode
, w
, count
, handle_shuffle
);
453 case OpenCLstd_Shuffle2
:
454 handle_instr(b
, cl_opcode
, w
, count
, handle_shuffle2
);
456 case OpenCLstd_Round
:
457 handle_instr(b
, cl_opcode
, w
, count
, handle_round
);
459 case OpenCLstd_Printf
:
460 handle_instr(b
, cl_opcode
, w
, count
, handle_printf
);
462 case OpenCLstd_Prefetch
:
463 /* TODO maybe add a nir instruction for this? */
466 vtn_fail("unhandled opencl opc: %u\n", ext_opcode
);