4ba23d157749a60d62ae0a79ba527c9ca771e3f2
[mesa.git] / src / intel / compiler / brw_vec4_nir.cpp
1 /*
2 * Copyright © 2015 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "brw_nir.h"
25 #include "brw_vec4.h"
26 #include "brw_vec4_builder.h"
27 #include "brw_vec4_surface_builder.h"
28 #include "brw_eu.h"
29
30 using namespace brw;
31 using namespace brw::surface_access;
32
33 namespace brw {
34
35 void
36 vec4_visitor::emit_nir_code()
37 {
38 if (nir->num_uniforms > 0)
39 nir_setup_uniforms();
40
41 nir_emit_impl(nir_shader_get_entrypoint((nir_shader *)nir));
42 }
43
44 void
45 vec4_visitor::nir_setup_uniforms()
46 {
47 uniforms = nir->num_uniforms / 16;
48 }
49
50 void
51 vec4_visitor::nir_emit_impl(nir_function_impl *impl)
52 {
53 nir_locals = ralloc_array(mem_ctx, dst_reg, impl->reg_alloc);
54 for (unsigned i = 0; i < impl->reg_alloc; i++) {
55 nir_locals[i] = dst_reg();
56 }
57
58 foreach_list_typed(nir_register, reg, node, &impl->registers) {
59 unsigned array_elems =
60 reg->num_array_elems == 0 ? 1 : reg->num_array_elems;
61 const unsigned num_regs = array_elems * DIV_ROUND_UP(reg->bit_size, 32);
62 nir_locals[reg->index] = dst_reg(VGRF, alloc.allocate(num_regs));
63
64 if (reg->bit_size == 64)
65 nir_locals[reg->index].type = BRW_REGISTER_TYPE_DF;
66 }
67
68 nir_ssa_values = ralloc_array(mem_ctx, dst_reg, impl->ssa_alloc);
69
70 nir_emit_cf_list(&impl->body);
71 }
72
73 void
74 vec4_visitor::nir_emit_cf_list(exec_list *list)
75 {
76 exec_list_validate(list);
77 foreach_list_typed(nir_cf_node, node, node, list) {
78 switch (node->type) {
79 case nir_cf_node_if:
80 nir_emit_if(nir_cf_node_as_if(node));
81 break;
82
83 case nir_cf_node_loop:
84 nir_emit_loop(nir_cf_node_as_loop(node));
85 break;
86
87 case nir_cf_node_block:
88 nir_emit_block(nir_cf_node_as_block(node));
89 break;
90
91 default:
92 unreachable("Invalid CFG node block");
93 }
94 }
95 }
96
97 void
98 vec4_visitor::nir_emit_if(nir_if *if_stmt)
99 {
100 /* First, put the condition in f0 */
101 src_reg condition = get_nir_src(if_stmt->condition, BRW_REGISTER_TYPE_D, 1);
102 vec4_instruction *inst = emit(MOV(dst_null_d(), condition));
103 inst->conditional_mod = BRW_CONDITIONAL_NZ;
104
105 /* We can just predicate based on the X channel, as the condition only
106 * goes on its own line */
107 emit(IF(BRW_PREDICATE_ALIGN16_REPLICATE_X));
108
109 nir_emit_cf_list(&if_stmt->then_list);
110
111 /* note: if the else is empty, dead CF elimination will remove it */
112 emit(BRW_OPCODE_ELSE);
113
114 nir_emit_cf_list(&if_stmt->else_list);
115
116 emit(BRW_OPCODE_ENDIF);
117 }
118
119 void
120 vec4_visitor::nir_emit_loop(nir_loop *loop)
121 {
122 emit(BRW_OPCODE_DO);
123
124 nir_emit_cf_list(&loop->body);
125
126 emit(BRW_OPCODE_WHILE);
127 }
128
129 void
130 vec4_visitor::nir_emit_block(nir_block *block)
131 {
132 nir_foreach_instr(instr, block) {
133 nir_emit_instr(instr);
134 }
135 }
136
137 void
138 vec4_visitor::nir_emit_instr(nir_instr *instr)
139 {
140 base_ir = instr;
141
142 switch (instr->type) {
143 case nir_instr_type_load_const:
144 nir_emit_load_const(nir_instr_as_load_const(instr));
145 break;
146
147 case nir_instr_type_intrinsic:
148 nir_emit_intrinsic(nir_instr_as_intrinsic(instr));
149 break;
150
151 case nir_instr_type_alu:
152 nir_emit_alu(nir_instr_as_alu(instr));
153 break;
154
155 case nir_instr_type_jump:
156 nir_emit_jump(nir_instr_as_jump(instr));
157 break;
158
159 case nir_instr_type_tex:
160 nir_emit_texture(nir_instr_as_tex(instr));
161 break;
162
163 case nir_instr_type_ssa_undef:
164 nir_emit_undef(nir_instr_as_ssa_undef(instr));
165 break;
166
167 default:
168 unreachable("VS instruction not yet implemented by NIR->vec4");
169 }
170 }
171
172 static dst_reg
173 dst_reg_for_nir_reg(vec4_visitor *v, nir_register *nir_reg,
174 unsigned base_offset, nir_src *indirect)
175 {
176 dst_reg reg;
177
178 reg = v->nir_locals[nir_reg->index];
179 if (nir_reg->bit_size == 64)
180 reg.type = BRW_REGISTER_TYPE_DF;
181 reg = offset(reg, 8, base_offset);
182 if (indirect) {
183 reg.reladdr =
184 new(v->mem_ctx) src_reg(v->get_nir_src(*indirect,
185 BRW_REGISTER_TYPE_D,
186 1));
187 }
188 return reg;
189 }
190
191 dst_reg
192 vec4_visitor::get_nir_dest(const nir_dest &dest)
193 {
194 if (dest.is_ssa) {
195 dst_reg dst =
196 dst_reg(VGRF, alloc.allocate(DIV_ROUND_UP(dest.ssa.bit_size, 32)));
197 if (dest.ssa.bit_size == 64)
198 dst.type = BRW_REGISTER_TYPE_DF;
199 nir_ssa_values[dest.ssa.index] = dst;
200 return dst;
201 } else {
202 return dst_reg_for_nir_reg(this, dest.reg.reg, dest.reg.base_offset,
203 dest.reg.indirect);
204 }
205 }
206
207 dst_reg
208 vec4_visitor::get_nir_dest(const nir_dest &dest, enum brw_reg_type type)
209 {
210 return retype(get_nir_dest(dest), type);
211 }
212
213 dst_reg
214 vec4_visitor::get_nir_dest(const nir_dest &dest, nir_alu_type type)
215 {
216 return get_nir_dest(dest, brw_type_for_nir_type(devinfo, type));
217 }
218
219 src_reg
220 vec4_visitor::get_nir_src(const nir_src &src, enum brw_reg_type type,
221 unsigned num_components)
222 {
223 dst_reg reg;
224
225 if (src.is_ssa) {
226 assert(src.ssa != NULL);
227 reg = nir_ssa_values[src.ssa->index];
228 }
229 else {
230 reg = dst_reg_for_nir_reg(this, src.reg.reg, src.reg.base_offset,
231 src.reg.indirect);
232 }
233
234 reg = retype(reg, type);
235
236 src_reg reg_as_src = src_reg(reg);
237 reg_as_src.swizzle = brw_swizzle_for_size(num_components);
238 return reg_as_src;
239 }
240
241 src_reg
242 vec4_visitor::get_nir_src(const nir_src &src, nir_alu_type type,
243 unsigned num_components)
244 {
245 return get_nir_src(src, brw_type_for_nir_type(devinfo, type),
246 num_components);
247 }
248
249 src_reg
250 vec4_visitor::get_nir_src(const nir_src &src, unsigned num_components)
251 {
252 /* if type is not specified, default to signed int */
253 return get_nir_src(src, nir_type_int32, num_components);
254 }
255
256 src_reg
257 vec4_visitor::get_nir_src_imm(const nir_src &src)
258 {
259 assert(nir_src_num_components(src) == 1);
260 assert(nir_src_bit_size(src) == 32);
261 return nir_src_is_const(src) ? src_reg(brw_imm_d(nir_src_as_int(src))) :
262 get_nir_src(src, 1);
263 }
264
265 src_reg
266 vec4_visitor::get_indirect_offset(nir_intrinsic_instr *instr)
267 {
268 nir_src *offset_src = nir_get_io_offset_src(instr);
269
270 if (nir_src_is_const(*offset_src)) {
271 /* The only constant offset we should find is 0. brw_nir.c's
272 * add_const_offset_to_base() will fold other constant offsets
273 * into instr->const_index[0].
274 */
275 assert(nir_src_as_uint(*offset_src) == 0);
276 return src_reg();
277 }
278
279 return get_nir_src(*offset_src, BRW_REGISTER_TYPE_UD, 1);
280 }
281
282 static src_reg
283 setup_imm_df(const vec4_builder &bld, double v)
284 {
285 const gen_device_info *devinfo = bld.shader->devinfo;
286 assert(devinfo->gen >= 7);
287
288 if (devinfo->gen >= 8)
289 return brw_imm_df(v);
290
291 /* gen7.5 does not support DF immediates straighforward but the DIM
292 * instruction allows to set the 64-bit immediate value.
293 */
294 if (devinfo->is_haswell) {
295 const vec4_builder ubld = bld.exec_all();
296 const dst_reg dst = bld.vgrf(BRW_REGISTER_TYPE_DF);
297 ubld.DIM(dst, brw_imm_df(v));
298 return swizzle(src_reg(dst), BRW_SWIZZLE_XXXX);
299 }
300
301 /* gen7 does not support DF immediates */
302 union {
303 double d;
304 struct {
305 uint32_t i1;
306 uint32_t i2;
307 };
308 } di;
309
310 di.d = v;
311
312 /* Write the low 32-bit of the constant to the X:UD channel and the
313 * high 32-bit to the Y:UD channel to build the constant in a VGRF.
314 * We have to do this twice (offset 0 and offset 1), since a DF VGRF takes
315 * two SIMD8 registers in SIMD4x2 execution. Finally, return a swizzle
316 * XXXX so any access to the VGRF only reads the constant data in these
317 * channels.
318 */
319 const dst_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_UD, 2);
320 for (unsigned n = 0; n < 2; n++) {
321 const vec4_builder ubld = bld.exec_all().group(4, n);
322 ubld.MOV(writemask(offset(tmp, 8, n), WRITEMASK_X), brw_imm_ud(di.i1));
323 ubld.MOV(writemask(offset(tmp, 8, n), WRITEMASK_Y), brw_imm_ud(di.i2));
324 }
325
326 return swizzle(src_reg(retype(tmp, BRW_REGISTER_TYPE_DF)), BRW_SWIZZLE_XXXX);
327 }
328
329 void
330 vec4_visitor::nir_emit_load_const(nir_load_const_instr *instr)
331 {
332 dst_reg reg;
333
334 if (instr->def.bit_size == 64) {
335 reg = dst_reg(VGRF, alloc.allocate(2));
336 reg.type = BRW_REGISTER_TYPE_DF;
337 } else {
338 reg = dst_reg(VGRF, alloc.allocate(1));
339 reg.type = BRW_REGISTER_TYPE_D;
340 }
341
342 const vec4_builder ibld = vec4_builder(this).at_end();
343 unsigned remaining = brw_writemask_for_size(instr->def.num_components);
344
345 /* @FIXME: consider emitting vector operations to save some MOVs in
346 * cases where the components are representable in 8 bits.
347 * For now, we emit a MOV for each distinct value.
348 */
349 for (unsigned i = 0; i < instr->def.num_components; i++) {
350 unsigned writemask = 1 << i;
351
352 if ((remaining & writemask) == 0)
353 continue;
354
355 for (unsigned j = i; j < instr->def.num_components; j++) {
356 if ((instr->def.bit_size == 32 &&
357 instr->value[i].u32 == instr->value[j].u32) ||
358 (instr->def.bit_size == 64 &&
359 instr->value[i].f64 == instr->value[j].f64)) {
360 writemask |= 1 << j;
361 }
362 }
363
364 reg.writemask = writemask;
365 if (instr->def.bit_size == 64) {
366 emit(MOV(reg, setup_imm_df(ibld, instr->value[i].f64)));
367 } else {
368 emit(MOV(reg, brw_imm_d(instr->value[i].i32)));
369 }
370
371 remaining &= ~writemask;
372 }
373
374 /* Set final writemask */
375 reg.writemask = brw_writemask_for_size(instr->def.num_components);
376
377 nir_ssa_values[instr->def.index] = reg;
378 }
379
380 src_reg
381 vec4_visitor::get_nir_ssbo_intrinsic_index(nir_intrinsic_instr *instr)
382 {
383 /* SSBO stores are weird in that their index is in src[1] */
384 const unsigned src = instr->intrinsic == nir_intrinsic_store_ssbo ? 1 : 0;
385
386 src_reg surf_index;
387 if (nir_src_is_const(instr->src[src])) {
388 unsigned index = prog_data->base.binding_table.ssbo_start +
389 nir_src_as_uint(instr->src[src]);
390 surf_index = brw_imm_ud(index);
391 } else {
392 surf_index = src_reg(this, glsl_type::uint_type);
393 emit(ADD(dst_reg(surf_index), get_nir_src(instr->src[src], 1),
394 brw_imm_ud(prog_data->base.binding_table.ssbo_start)));
395 surf_index = emit_uniformize(surf_index);
396 }
397
398 return surf_index;
399 }
400
401 void
402 vec4_visitor::nir_emit_intrinsic(nir_intrinsic_instr *instr)
403 {
404 dst_reg dest;
405 src_reg src;
406
407 switch (instr->intrinsic) {
408
409 case nir_intrinsic_load_input: {
410 assert(nir_dest_bit_size(instr->dest) == 32);
411 /* We set EmitNoIndirectInput for VS */
412 unsigned load_offset = nir_src_as_uint(instr->src[0]);
413
414 dest = get_nir_dest(instr->dest);
415 dest.writemask = brw_writemask_for_size(instr->num_components);
416
417 src = src_reg(ATTR, instr->const_index[0] + load_offset,
418 glsl_type::uvec4_type);
419 src = retype(src, dest.type);
420
421 /* Swizzle source based on component layout qualifier */
422 src.swizzle = BRW_SWZ_COMP_INPUT(nir_intrinsic_component(instr));
423 emit(MOV(dest, src));
424 break;
425 }
426
427 case nir_intrinsic_store_output: {
428 assert(nir_src_bit_size(instr->src[0]) == 32);
429 unsigned store_offset = nir_src_as_uint(instr->src[1]);
430 int varying = instr->const_index[0] + store_offset;
431 src = get_nir_src(instr->src[0], BRW_REGISTER_TYPE_F,
432 instr->num_components);
433
434 unsigned c = nir_intrinsic_component(instr);
435 output_reg[varying][c] = dst_reg(src);
436 output_num_components[varying][c] = instr->num_components;
437 break;
438 }
439
440 case nir_intrinsic_get_buffer_size: {
441 assert(nir_src_num_components(instr->src[0]) == 1);
442 unsigned ssbo_index = nir_src_is_const(instr->src[0]) ?
443 nir_src_as_uint(instr->src[0]) : 0;
444
445 const unsigned index =
446 prog_data->base.binding_table.ssbo_start + ssbo_index;
447 dst_reg result_dst = get_nir_dest(instr->dest);
448 vec4_instruction *inst = new(mem_ctx)
449 vec4_instruction(SHADER_OPCODE_GET_BUFFER_SIZE, result_dst);
450
451 inst->base_mrf = 2;
452 inst->mlen = 1; /* always at least one */
453 inst->src[1] = brw_imm_ud(index);
454
455 /* MRF for the first parameter */
456 src_reg lod = brw_imm_d(0);
457 int param_base = inst->base_mrf;
458 int writemask = WRITEMASK_X;
459 emit(MOV(dst_reg(MRF, param_base, glsl_type::int_type, writemask), lod));
460
461 emit(inst);
462 break;
463 }
464
465 case nir_intrinsic_store_ssbo: {
466 assert(devinfo->gen >= 7);
467
468 /* brw_nir_lower_mem_access_bit_sizes takes care of this */
469 assert(nir_src_bit_size(instr->src[0]) == 32);
470 assert(nir_intrinsic_write_mask(instr) ==
471 (1u << instr->num_components) - 1);
472
473 src_reg surf_index = get_nir_ssbo_intrinsic_index(instr);
474 src_reg offset_reg = retype(get_nir_src_imm(instr->src[2]),
475 BRW_REGISTER_TYPE_UD);
476
477 /* Value */
478 src_reg val_reg = get_nir_src(instr->src[0], BRW_REGISTER_TYPE_F, 4);
479
480 /* IvyBridge does not have a native SIMD4x2 untyped write message so untyped
481 * writes will use SIMD8 mode. In order to hide this and keep symmetry across
482 * typed and untyped messages and across hardware platforms, the
483 * current implementation of the untyped messages will transparently convert
484 * the SIMD4x2 payload into an equivalent SIMD8 payload by transposing it
485 * and enabling only channel X on the SEND instruction.
486 *
487 * The above, works well for full vector writes, but not for partial writes
488 * where we want to write some channels and not others, like when we have
489 * code such as v.xyw = vec3(1,2,4). Because the untyped write messages are
490 * quite restrictive with regards to the channel enables we can configure in
491 * the message descriptor (not all combinations are allowed) we cannot simply
492 * implement these scenarios with a single message while keeping the
493 * aforementioned symmetry in the implementation. For now we de decided that
494 * it is better to keep the symmetry to reduce complexity, so in situations
495 * such as the one described we end up emitting two untyped write messages
496 * (one for xy and another for w).
497 *
498 * The code below packs consecutive channels into a single write message,
499 * detects gaps in the vector write and if needed, sends a second message
500 * with the remaining channels. If in the future we decide that we want to
501 * emit a single message at the expense of losing the symmetry in the
502 * implementation we can:
503 *
504 * 1) For IvyBridge: Only use the red channel of the untyped write SIMD8
505 * message payload. In this mode we can write up to 8 offsets and dwords
506 * to the red channel only (for the two vec4s in the SIMD4x2 execution)
507 * and select which of the 8 channels carry data to write by setting the
508 * appropriate writemask in the dst register of the SEND instruction.
509 * It would require to write a new generator opcode specifically for
510 * IvyBridge since we would need to prepare a SIMD8 payload that could
511 * use any channel, not just X.
512 *
513 * 2) For Haswell+: Simply send a single write message but set the writemask
514 * on the dst of the SEND instruction to select the channels we want to
515 * write. It would require to modify the current messages to receive
516 * and honor the writemask provided.
517 */
518 const vec4_builder bld = vec4_builder(this).at_end()
519 .annotate(current_annotation, base_ir);
520
521 emit_untyped_write(bld, surf_index, offset_reg, val_reg,
522 1 /* dims */, instr->num_components /* size */,
523 BRW_PREDICATE_NONE);
524 break;
525 }
526
527 case nir_intrinsic_load_ssbo: {
528 assert(devinfo->gen >= 7);
529
530 /* brw_nir_lower_mem_access_bit_sizes takes care of this */
531 assert(nir_dest_bit_size(instr->dest) == 32);
532
533 src_reg surf_index = get_nir_ssbo_intrinsic_index(instr);
534 src_reg offset_reg = retype(get_nir_src_imm(instr->src[1]),
535 BRW_REGISTER_TYPE_UD);
536
537 /* Read the vector */
538 const vec4_builder bld = vec4_builder(this).at_end()
539 .annotate(current_annotation, base_ir);
540
541 src_reg read_result = emit_untyped_read(bld, surf_index, offset_reg,
542 1 /* dims */, 4 /* size*/,
543 BRW_PREDICATE_NONE);
544 dst_reg dest = get_nir_dest(instr->dest);
545 read_result.type = dest.type;
546 read_result.swizzle = brw_swizzle_for_size(instr->num_components);
547 emit(MOV(dest, read_result));
548 break;
549 }
550
551 case nir_intrinsic_ssbo_atomic_add:
552 case nir_intrinsic_ssbo_atomic_imin:
553 case nir_intrinsic_ssbo_atomic_umin:
554 case nir_intrinsic_ssbo_atomic_imax:
555 case nir_intrinsic_ssbo_atomic_umax:
556 case nir_intrinsic_ssbo_atomic_and:
557 case nir_intrinsic_ssbo_atomic_or:
558 case nir_intrinsic_ssbo_atomic_xor:
559 case nir_intrinsic_ssbo_atomic_exchange:
560 case nir_intrinsic_ssbo_atomic_comp_swap:
561 nir_emit_ssbo_atomic(brw_aop_for_nir_intrinsic(instr), instr);
562 break;
563
564 case nir_intrinsic_load_vertex_id:
565 unreachable("should be lowered by lower_vertex_id()");
566
567 case nir_intrinsic_load_vertex_id_zero_base:
568 case nir_intrinsic_load_base_vertex:
569 case nir_intrinsic_load_instance_id:
570 case nir_intrinsic_load_base_instance:
571 case nir_intrinsic_load_draw_id:
572 case nir_intrinsic_load_invocation_id:
573 unreachable("should be lowered by brw_nir_lower_vs_inputs()");
574
575 case nir_intrinsic_load_uniform: {
576 /* Offsets are in bytes but they should always be multiples of 4 */
577 assert(nir_intrinsic_base(instr) % 4 == 0);
578
579 dest = get_nir_dest(instr->dest);
580
581 src = src_reg(dst_reg(UNIFORM, nir_intrinsic_base(instr) / 16));
582 src.type = dest.type;
583
584 /* Uniforms don't actually have to be vec4 aligned. In the case that
585 * it isn't, we have to use a swizzle to shift things around. They
586 * do still have the std140 alignment requirement that vec2's have to
587 * be vec2-aligned and vec3's and vec4's have to be vec4-aligned.
588 *
589 * The swizzle also works in the indirect case as the generator adds
590 * the swizzle to the offset for us.
591 */
592 const int type_size = type_sz(src.type);
593 unsigned shift = (nir_intrinsic_base(instr) % 16) / type_size;
594 assert(shift + instr->num_components <= 4);
595
596 if (nir_src_is_const(instr->src[0])) {
597 const unsigned load_offset = nir_src_as_uint(instr->src[0]);
598 /* Offsets are in bytes but they should always be multiples of 4 */
599 assert(load_offset % 4 == 0);
600
601 src.swizzle = brw_swizzle_for_size(instr->num_components);
602 dest.writemask = brw_writemask_for_size(instr->num_components);
603 unsigned offset = load_offset + shift * type_size;
604 src.offset = ROUND_DOWN_TO(offset, 16);
605 shift = (offset % 16) / type_size;
606 assert(shift + instr->num_components <= 4);
607 src.swizzle += BRW_SWIZZLE4(shift, shift, shift, shift);
608
609 emit(MOV(dest, src));
610 } else {
611 /* Uniform arrays are vec4 aligned, because of std140 alignment
612 * rules.
613 */
614 assert(shift == 0);
615
616 src_reg indirect = get_nir_src(instr->src[0], BRW_REGISTER_TYPE_UD, 1);
617
618 /* MOV_INDIRECT is going to stomp the whole thing anyway */
619 dest.writemask = WRITEMASK_XYZW;
620
621 emit(SHADER_OPCODE_MOV_INDIRECT, dest, src,
622 indirect, brw_imm_ud(instr->const_index[1]));
623 }
624 break;
625 }
626
627 case nir_intrinsic_load_ubo: {
628 src_reg surf_index;
629
630 prog_data->base.has_ubo_pull = true;
631
632 dest = get_nir_dest(instr->dest);
633
634 if (nir_src_is_const(instr->src[0])) {
635 /* The block index is a constant, so just emit the binding table entry
636 * as an immediate.
637 */
638 const unsigned index = prog_data->base.binding_table.ubo_start +
639 nir_src_as_uint(instr->src[0]);
640 surf_index = brw_imm_ud(index);
641 } else {
642 /* The block index is not a constant. Evaluate the index expression
643 * per-channel and add the base UBO index; we have to select a value
644 * from any live channel.
645 */
646 surf_index = src_reg(this, glsl_type::uint_type);
647 emit(ADD(dst_reg(surf_index), get_nir_src(instr->src[0], nir_type_int32,
648 instr->num_components),
649 brw_imm_ud(prog_data->base.binding_table.ubo_start)));
650 surf_index = emit_uniformize(surf_index);
651 }
652
653 src_reg offset_reg;
654 if (nir_src_is_const(instr->src[1])) {
655 unsigned load_offset = nir_src_as_uint(instr->src[1]);
656 offset_reg = brw_imm_ud(load_offset & ~15);
657 } else {
658 offset_reg = src_reg(this, glsl_type::uint_type);
659 emit(MOV(dst_reg(offset_reg),
660 get_nir_src(instr->src[1], nir_type_uint32, 1)));
661 }
662
663 src_reg packed_consts;
664 if (nir_dest_bit_size(instr->dest) == 32) {
665 packed_consts = src_reg(this, glsl_type::vec4_type);
666 emit_pull_constant_load_reg(dst_reg(packed_consts),
667 surf_index,
668 offset_reg,
669 NULL, NULL /* before_block/inst */);
670 } else {
671 src_reg temp = src_reg(this, glsl_type::dvec4_type);
672 src_reg temp_float = retype(temp, BRW_REGISTER_TYPE_F);
673
674 emit_pull_constant_load_reg(dst_reg(temp_float),
675 surf_index, offset_reg, NULL, NULL);
676 if (offset_reg.file == IMM)
677 offset_reg.ud += 16;
678 else
679 emit(ADD(dst_reg(offset_reg), offset_reg, brw_imm_ud(16u)));
680 emit_pull_constant_load_reg(dst_reg(byte_offset(temp_float, REG_SIZE)),
681 surf_index, offset_reg, NULL, NULL);
682
683 packed_consts = src_reg(this, glsl_type::dvec4_type);
684 shuffle_64bit_data(dst_reg(packed_consts), temp, false);
685 }
686
687 packed_consts.swizzle = brw_swizzle_for_size(instr->num_components);
688 if (nir_src_is_const(instr->src[1])) {
689 unsigned load_offset = nir_src_as_uint(instr->src[1]);
690 unsigned type_size = type_sz(dest.type);
691 packed_consts.swizzle +=
692 BRW_SWIZZLE4(load_offset % 16 / type_size,
693 load_offset % 16 / type_size,
694 load_offset % 16 / type_size,
695 load_offset % 16 / type_size);
696 }
697
698 emit(MOV(dest, retype(packed_consts, dest.type)));
699
700 break;
701 }
702
703 case nir_intrinsic_scoped_barrier:
704 assert(nir_intrinsic_execution_scope(instr) == NIR_SCOPE_NONE);
705 /* Fall through. */
706 case nir_intrinsic_memory_barrier: {
707 const vec4_builder bld =
708 vec4_builder(this).at_end().annotate(current_annotation, base_ir);
709 const dst_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_UD);
710 vec4_instruction *fence =
711 bld.emit(SHADER_OPCODE_MEMORY_FENCE, tmp, brw_vec8_grf(0, 0));
712 fence->sfid = GEN7_SFID_DATAPORT_DATA_CACHE;
713 break;
714 }
715
716 case nir_intrinsic_shader_clock: {
717 /* We cannot do anything if there is an event, so ignore it for now */
718 const src_reg shader_clock = get_timestamp();
719 const enum brw_reg_type type = brw_type_for_base_type(glsl_type::uvec2_type);
720
721 dest = get_nir_dest(instr->dest, type);
722 emit(MOV(dest, shader_clock));
723 break;
724 }
725
726 default:
727 unreachable("Unknown intrinsic");
728 }
729 }
730
731 void
732 vec4_visitor::nir_emit_ssbo_atomic(int op, nir_intrinsic_instr *instr)
733 {
734 dst_reg dest;
735 if (nir_intrinsic_infos[instr->intrinsic].has_dest)
736 dest = get_nir_dest(instr->dest);
737
738 src_reg surface = get_nir_ssbo_intrinsic_index(instr);
739 src_reg offset = get_nir_src(instr->src[1], 1);
740 src_reg data1;
741 if (op != BRW_AOP_INC && op != BRW_AOP_DEC && op != BRW_AOP_PREDEC)
742 data1 = get_nir_src(instr->src[2], 1);
743 src_reg data2;
744 if (op == BRW_AOP_CMPWR)
745 data2 = get_nir_src(instr->src[3], 1);
746
747 /* Emit the actual atomic operation operation */
748 const vec4_builder bld =
749 vec4_builder(this).at_end().annotate(current_annotation, base_ir);
750
751 src_reg atomic_result = emit_untyped_atomic(bld, surface, offset,
752 data1, data2,
753 1 /* dims */, 1 /* rsize */,
754 op,
755 BRW_PREDICATE_NONE);
756 dest.type = atomic_result.type;
757 bld.MOV(dest, atomic_result);
758 }
759
760 static unsigned
761 brw_swizzle_for_nir_swizzle(uint8_t swizzle[4])
762 {
763 return BRW_SWIZZLE4(swizzle[0], swizzle[1], swizzle[2], swizzle[3]);
764 }
765
766 bool
767 vec4_visitor::optimize_predicate(nir_alu_instr *instr,
768 enum brw_predicate *predicate)
769 {
770 if (!instr->src[0].src.is_ssa ||
771 instr->src[0].src.ssa->parent_instr->type != nir_instr_type_alu)
772 return false;
773
774 nir_alu_instr *cmp_instr =
775 nir_instr_as_alu(instr->src[0].src.ssa->parent_instr);
776
777 switch (cmp_instr->op) {
778 case nir_op_b32any_fnequal2:
779 case nir_op_b32any_inequal2:
780 case nir_op_b32any_fnequal3:
781 case nir_op_b32any_inequal3:
782 case nir_op_b32any_fnequal4:
783 case nir_op_b32any_inequal4:
784 *predicate = BRW_PREDICATE_ALIGN16_ANY4H;
785 break;
786 case nir_op_b32all_fequal2:
787 case nir_op_b32all_iequal2:
788 case nir_op_b32all_fequal3:
789 case nir_op_b32all_iequal3:
790 case nir_op_b32all_fequal4:
791 case nir_op_b32all_iequal4:
792 *predicate = BRW_PREDICATE_ALIGN16_ALL4H;
793 break;
794 default:
795 return false;
796 }
797
798 unsigned size_swizzle =
799 brw_swizzle_for_size(nir_op_infos[cmp_instr->op].input_sizes[0]);
800
801 src_reg op[2];
802 assert(nir_op_infos[cmp_instr->op].num_inputs == 2);
803 for (unsigned i = 0; i < 2; i++) {
804 nir_alu_type type = nir_op_infos[cmp_instr->op].input_types[i];
805 unsigned bit_size = nir_src_bit_size(cmp_instr->src[i].src);
806 type = (nir_alu_type) (((unsigned) type) | bit_size);
807 op[i] = get_nir_src(cmp_instr->src[i].src, type, 4);
808 unsigned base_swizzle =
809 brw_swizzle_for_nir_swizzle(cmp_instr->src[i].swizzle);
810 op[i].swizzle = brw_compose_swizzle(size_swizzle, base_swizzle);
811 }
812
813 emit(CMP(dst_null_d(), op[0], op[1],
814 brw_cmod_for_nir_comparison(cmp_instr->op)));
815
816 return true;
817 }
818
819 static void
820 emit_find_msb_using_lzd(const vec4_builder &bld,
821 const dst_reg &dst,
822 const src_reg &src,
823 bool is_signed)
824 {
825 vec4_instruction *inst;
826 src_reg temp = src;
827
828 if (is_signed) {
829 /* LZD of an absolute value source almost always does the right
830 * thing. There are two problem values:
831 *
832 * * 0x80000000. Since abs(0x80000000) == 0x80000000, LZD returns
833 * 0. However, findMSB(int(0x80000000)) == 30.
834 *
835 * * 0xffffffff. Since abs(0xffffffff) == 1, LZD returns
836 * 31. Section 8.8 (Integer Functions) of the GLSL 4.50 spec says:
837 *
838 * For a value of zero or negative one, -1 will be returned.
839 *
840 * * Negative powers of two. LZD(abs(-(1<<x))) returns x, but
841 * findMSB(-(1<<x)) should return x-1.
842 *
843 * For all negative number cases, including 0x80000000 and
844 * 0xffffffff, the correct value is obtained from LZD if instead of
845 * negating the (already negative) value the logical-not is used. A
846 * conditonal logical-not can be achieved in two instructions.
847 */
848 temp = src_reg(bld.vgrf(BRW_REGISTER_TYPE_D));
849
850 bld.ASR(dst_reg(temp), src, brw_imm_d(31));
851 bld.XOR(dst_reg(temp), temp, src);
852 }
853
854 bld.LZD(retype(dst, BRW_REGISTER_TYPE_UD),
855 retype(temp, BRW_REGISTER_TYPE_UD));
856
857 /* LZD counts from the MSB side, while GLSL's findMSB() wants the count
858 * from the LSB side. Subtract the result from 31 to convert the MSB count
859 * into an LSB count. If no bits are set, LZD will return 32. 31-32 = -1,
860 * which is exactly what findMSB() is supposed to return.
861 */
862 inst = bld.ADD(dst, retype(src_reg(dst), BRW_REGISTER_TYPE_D),
863 brw_imm_d(31));
864 inst->src[0].negate = true;
865 }
866
867 void
868 vec4_visitor::emit_conversion_from_double(dst_reg dst, src_reg src)
869 {
870 /* BDW PRM vol 15 - workarounds:
871 * DF->f format conversion for Align16 has wrong emask calculation when
872 * source is immediate.
873 */
874 if (devinfo->gen == 8 && dst.type == BRW_REGISTER_TYPE_F &&
875 src.file == BRW_IMMEDIATE_VALUE) {
876 emit(MOV(dst, brw_imm_f(src.df)));
877 return;
878 }
879
880 enum opcode op;
881 switch (dst.type) {
882 case BRW_REGISTER_TYPE_D:
883 op = VEC4_OPCODE_DOUBLE_TO_D32;
884 break;
885 case BRW_REGISTER_TYPE_UD:
886 op = VEC4_OPCODE_DOUBLE_TO_U32;
887 break;
888 case BRW_REGISTER_TYPE_F:
889 op = VEC4_OPCODE_DOUBLE_TO_F32;
890 break;
891 default:
892 unreachable("Unknown conversion");
893 }
894
895 dst_reg temp = dst_reg(this, glsl_type::dvec4_type);
896 emit(MOV(temp, src));
897 dst_reg temp2 = dst_reg(this, glsl_type::dvec4_type);
898 emit(op, temp2, src_reg(temp));
899
900 emit(VEC4_OPCODE_PICK_LOW_32BIT, retype(temp2, dst.type), src_reg(temp2));
901 emit(MOV(dst, src_reg(retype(temp2, dst.type))));
902 }
903
904 void
905 vec4_visitor::emit_conversion_to_double(dst_reg dst, src_reg src)
906 {
907 dst_reg tmp_dst = dst_reg(src_reg(this, glsl_type::dvec4_type));
908 src_reg tmp_src = retype(src_reg(this, glsl_type::vec4_type), src.type);
909 emit(MOV(dst_reg(tmp_src), src));
910 emit(VEC4_OPCODE_TO_DOUBLE, tmp_dst, tmp_src);
911 emit(MOV(dst, src_reg(tmp_dst)));
912 }
913
914 /**
915 * Try to use an immediate value for a source
916 *
917 * In cases of flow control, constant propagation is sometimes unable to
918 * determine that a register contains a constant value. To work around this,
919 * try to emit a literal as one of the sources. If \c try_src0_also is set,
920 * \c op[0] will also be tried for an immediate value.
921 *
922 * If \c op[0] is modified, the operands will be exchanged so that \c op[1]
923 * will always be the immediate value.
924 *
925 * \return The index of the source that was modified, 0 or 1, if successful.
926 * Otherwise, -1.
927 *
928 * \param op - Operands to the instruction
929 * \param try_src0_also - True if \c op[0] should also be a candidate for
930 * getting an immediate value. This should only be set
931 * for commutative operations.
932 */
933 static int
934 try_immediate_source(const nir_alu_instr *instr, src_reg *op,
935 bool try_src0_also,
936 ASSERTED const gen_device_info *devinfo)
937 {
938 unsigned idx;
939
940 /* MOV should be the only single-source instruction passed to this
941 * function. Any other unary instruction with a constant source should
942 * have been constant-folded away!
943 */
944 assert(nir_op_infos[instr->op].num_inputs > 1 ||
945 instr->op == nir_op_mov);
946
947 if (instr->op != nir_op_mov &&
948 nir_src_bit_size(instr->src[1].src) == 32 &&
949 nir_src_is_const(instr->src[1].src)) {
950 idx = 1;
951 } else if (try_src0_also &&
952 nir_src_bit_size(instr->src[0].src) == 32 &&
953 nir_src_is_const(instr->src[0].src)) {
954 idx = 0;
955 } else {
956 return -1;
957 }
958
959 const enum brw_reg_type old_type = op[idx].type;
960
961 switch (old_type) {
962 case BRW_REGISTER_TYPE_D:
963 case BRW_REGISTER_TYPE_UD: {
964 int first_comp = -1;
965 int d = 0;
966
967 for (unsigned i = 0; i < NIR_MAX_VEC_COMPONENTS; i++) {
968 if (nir_alu_instr_channel_used(instr, idx, i)) {
969 if (first_comp < 0) {
970 first_comp = i;
971 d = nir_src_comp_as_int(instr->src[idx].src,
972 instr->src[idx].swizzle[i]);
973 } else if (d != nir_src_comp_as_int(instr->src[idx].src,
974 instr->src[idx].swizzle[i])) {
975 return -1;
976 }
977 }
978 }
979
980 assert(first_comp >= 0);
981
982 if (op[idx].abs)
983 d = MAX2(-d, d);
984
985 if (op[idx].negate) {
986 /* On Gen8+ a negation source modifier on a logical operation means
987 * something different. Nothing should generate this, so assert that
988 * it does not occur.
989 */
990 assert(devinfo->gen < 8 || (instr->op != nir_op_iand &&
991 instr->op != nir_op_ior &&
992 instr->op != nir_op_ixor));
993 d = -d;
994 }
995
996 op[idx] = retype(src_reg(brw_imm_d(d)), old_type);
997 break;
998 }
999
1000 case BRW_REGISTER_TYPE_F: {
1001 int first_comp = -1;
1002 float f[NIR_MAX_VEC_COMPONENTS] = { 0.0f };
1003 bool is_scalar = true;
1004
1005 for (unsigned i = 0; i < NIR_MAX_VEC_COMPONENTS; i++) {
1006 if (nir_alu_instr_channel_used(instr, idx, i)) {
1007 f[i] = nir_src_comp_as_float(instr->src[idx].src,
1008 instr->src[idx].swizzle[i]);
1009 if (first_comp < 0) {
1010 first_comp = i;
1011 } else if (f[first_comp] != f[i]) {
1012 is_scalar = false;
1013 }
1014 }
1015 }
1016
1017 if (is_scalar) {
1018 if (op[idx].abs)
1019 f[first_comp] = fabs(f[first_comp]);
1020
1021 if (op[idx].negate)
1022 f[first_comp] = -f[first_comp];
1023
1024 op[idx] = src_reg(brw_imm_f(f[first_comp]));
1025 assert(op[idx].type == old_type);
1026 } else {
1027 uint8_t vf_values[4] = { 0, 0, 0, 0 };
1028
1029 for (unsigned i = 0; i < ARRAY_SIZE(vf_values); i++) {
1030
1031 if (op[idx].abs)
1032 f[i] = fabs(f[i]);
1033
1034 if (op[idx].negate)
1035 f[i] = -f[i];
1036
1037 const int vf = brw_float_to_vf(f[i]);
1038 if (vf == -1)
1039 return -1;
1040
1041 vf_values[i] = vf;
1042 }
1043
1044 op[idx] = src_reg(brw_imm_vf4(vf_values[0], vf_values[1],
1045 vf_values[2], vf_values[3]));
1046 }
1047 break;
1048 }
1049
1050 default:
1051 unreachable("Non-32bit type.");
1052 }
1053
1054 /* If the instruction has more than one source, the instruction format only
1055 * allows source 1 to be an immediate value. If the immediate value was
1056 * source 0, then the sources must be exchanged.
1057 */
1058 if (idx == 0 && instr->op != nir_op_mov) {
1059 src_reg tmp = op[0];
1060 op[0] = op[1];
1061 op[1] = tmp;
1062 }
1063
1064 return idx;
1065 }
1066
1067 void
1068 vec4_visitor::fix_float_operands(src_reg op[3], nir_alu_instr *instr)
1069 {
1070 bool fixed[3] = { false, false, false };
1071
1072 for (unsigned i = 0; i < 2; i++) {
1073 if (!nir_src_is_const(instr->src[i].src))
1074 continue;
1075
1076 for (unsigned j = i + 1; j < 3; j++) {
1077 if (fixed[j])
1078 continue;
1079
1080 if (!nir_src_is_const(instr->src[j].src))
1081 continue;
1082
1083 if (nir_alu_srcs_equal(instr, instr, i, j)) {
1084 if (!fixed[i])
1085 op[i] = fix_3src_operand(op[i]);
1086
1087 op[j] = op[i];
1088
1089 fixed[i] = true;
1090 fixed[j] = true;
1091 } else if (nir_alu_srcs_negative_equal(instr, instr, i, j)) {
1092 if (!fixed[i])
1093 op[i] = fix_3src_operand(op[i]);
1094
1095 op[j] = op[i];
1096 op[j].negate = !op[j].negate;
1097
1098 fixed[i] = true;
1099 fixed[j] = true;
1100 }
1101 }
1102 }
1103
1104 for (unsigned i = 0; i < 3; i++) {
1105 if (!fixed[i])
1106 op[i] = fix_3src_operand(op[i]);
1107 }
1108 }
1109
1110 static bool
1111 const_src_fits_in_16_bits(const nir_src &src, brw_reg_type type)
1112 {
1113 assert(nir_src_is_const(src));
1114 if (type_is_unsigned_int(type)) {
1115 return nir_src_comp_as_uint(src, 0) <= UINT16_MAX;
1116 } else {
1117 const int64_t c = nir_src_comp_as_int(src, 0);
1118 return c <= INT16_MAX && c >= INT16_MIN;
1119 }
1120 }
1121
1122 void
1123 vec4_visitor::nir_emit_alu(nir_alu_instr *instr)
1124 {
1125 vec4_instruction *inst;
1126
1127 nir_alu_type dst_type = (nir_alu_type) (nir_op_infos[instr->op].output_type |
1128 nir_dest_bit_size(instr->dest.dest));
1129 dst_reg dst = get_nir_dest(instr->dest.dest, dst_type);
1130 dst.writemask = instr->dest.write_mask;
1131
1132 assert(!instr->dest.saturate);
1133
1134 src_reg op[4];
1135 for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
1136 /* We don't lower to source modifiers, so they shouldn't exist. */
1137 assert(!instr->src[i].abs);
1138 assert(!instr->src[i].negate);
1139
1140 nir_alu_type src_type = (nir_alu_type)
1141 (nir_op_infos[instr->op].input_types[i] |
1142 nir_src_bit_size(instr->src[i].src));
1143 op[i] = get_nir_src(instr->src[i].src, src_type, 4);
1144 op[i].swizzle = brw_swizzle_for_nir_swizzle(instr->src[i].swizzle);
1145 }
1146
1147 switch (instr->op) {
1148 case nir_op_mov:
1149 try_immediate_source(instr, &op[0], true, devinfo);
1150 inst = emit(MOV(dst, op[0]));
1151 break;
1152
1153 case nir_op_vec2:
1154 case nir_op_vec3:
1155 case nir_op_vec4:
1156 unreachable("not reached: should be handled by lower_vec_to_movs()");
1157
1158 case nir_op_i2f32:
1159 case nir_op_u2f32:
1160 inst = emit(MOV(dst, op[0]));
1161 break;
1162
1163 case nir_op_f2f32:
1164 case nir_op_f2i32:
1165 case nir_op_f2u32:
1166 if (nir_src_bit_size(instr->src[0].src) == 64)
1167 emit_conversion_from_double(dst, op[0]);
1168 else
1169 inst = emit(MOV(dst, op[0]));
1170 break;
1171
1172 case nir_op_f2f64:
1173 case nir_op_i2f64:
1174 case nir_op_u2f64:
1175 emit_conversion_to_double(dst, op[0]);
1176 break;
1177
1178 case nir_op_fsat:
1179 inst = emit(MOV(dst, op[0]));
1180 inst->saturate = true;
1181 break;
1182
1183 case nir_op_fneg:
1184 case nir_op_ineg:
1185 op[0].negate = true;
1186 inst = emit(MOV(dst, op[0]));
1187 break;
1188
1189 case nir_op_fabs:
1190 case nir_op_iabs:
1191 op[0].negate = false;
1192 op[0].abs = true;
1193 inst = emit(MOV(dst, op[0]));
1194 break;
1195
1196 case nir_op_iadd:
1197 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1198 /* fall through */
1199 case nir_op_fadd:
1200 try_immediate_source(instr, op, true, devinfo);
1201 inst = emit(ADD(dst, op[0], op[1]));
1202 break;
1203
1204 case nir_op_uadd_sat:
1205 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1206 inst = emit(ADD(dst, op[0], op[1]));
1207 inst->saturate = true;
1208 break;
1209
1210 case nir_op_fmul:
1211 try_immediate_source(instr, op, true, devinfo);
1212 inst = emit(MUL(dst, op[0], op[1]));
1213 break;
1214
1215 case nir_op_imul: {
1216 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1217 if (devinfo->gen < 8) {
1218 /* For integer multiplication, the MUL uses the low 16 bits of one of
1219 * the operands (src0 through SNB, src1 on IVB and later). The MACH
1220 * accumulates in the contribution of the upper 16 bits of that
1221 * operand. If we can determine that one of the args is in the low
1222 * 16 bits, though, we can just emit a single MUL.
1223 */
1224 if (nir_src_is_const(instr->src[0].src) &&
1225 nir_alu_instr_src_read_mask(instr, 0) == 1 &&
1226 const_src_fits_in_16_bits(instr->src[0].src, op[0].type)) {
1227 if (devinfo->gen < 7)
1228 emit(MUL(dst, op[0], op[1]));
1229 else
1230 emit(MUL(dst, op[1], op[0]));
1231 } else if (nir_src_is_const(instr->src[1].src) &&
1232 nir_alu_instr_src_read_mask(instr, 1) == 1 &&
1233 const_src_fits_in_16_bits(instr->src[1].src, op[1].type)) {
1234 if (devinfo->gen < 7)
1235 emit(MUL(dst, op[1], op[0]));
1236 else
1237 emit(MUL(dst, op[0], op[1]));
1238 } else {
1239 struct brw_reg acc = retype(brw_acc_reg(8), dst.type);
1240
1241 emit(MUL(acc, op[0], op[1]));
1242 emit(MACH(dst_null_d(), op[0], op[1]));
1243 emit(MOV(dst, src_reg(acc)));
1244 }
1245 } else {
1246 emit(MUL(dst, op[0], op[1]));
1247 }
1248 break;
1249 }
1250
1251 case nir_op_imul_high:
1252 case nir_op_umul_high: {
1253 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1254 struct brw_reg acc = retype(brw_acc_reg(8), dst.type);
1255
1256 if (devinfo->gen >= 8)
1257 emit(MUL(acc, op[0], retype(op[1], BRW_REGISTER_TYPE_UW)));
1258 else
1259 emit(MUL(acc, op[0], op[1]));
1260
1261 emit(MACH(dst, op[0], op[1]));
1262 break;
1263 }
1264
1265 case nir_op_frcp:
1266 inst = emit_math(SHADER_OPCODE_RCP, dst, op[0]);
1267 break;
1268
1269 case nir_op_fexp2:
1270 inst = emit_math(SHADER_OPCODE_EXP2, dst, op[0]);
1271 break;
1272
1273 case nir_op_flog2:
1274 inst = emit_math(SHADER_OPCODE_LOG2, dst, op[0]);
1275 break;
1276
1277 case nir_op_fsin:
1278 inst = emit_math(SHADER_OPCODE_SIN, dst, op[0]);
1279 break;
1280
1281 case nir_op_fcos:
1282 inst = emit_math(SHADER_OPCODE_COS, dst, op[0]);
1283 break;
1284
1285 case nir_op_idiv:
1286 case nir_op_udiv:
1287 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1288 emit_math(SHADER_OPCODE_INT_QUOTIENT, dst, op[0], op[1]);
1289 break;
1290
1291 case nir_op_umod:
1292 case nir_op_irem:
1293 /* According to the sign table for INT DIV in the Ivy Bridge PRM, it
1294 * appears that our hardware just does the right thing for signed
1295 * remainder.
1296 */
1297 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1298 emit_math(SHADER_OPCODE_INT_REMAINDER, dst, op[0], op[1]);
1299 break;
1300
1301 case nir_op_imod: {
1302 /* Get a regular C-style remainder. If a % b == 0, set the predicate. */
1303 inst = emit_math(SHADER_OPCODE_INT_REMAINDER, dst, op[0], op[1]);
1304
1305 /* Math instructions don't support conditional mod */
1306 inst = emit(MOV(dst_null_d(), src_reg(dst)));
1307 inst->conditional_mod = BRW_CONDITIONAL_NZ;
1308
1309 /* Now, we need to determine if signs of the sources are different.
1310 * When we XOR the sources, the top bit is 0 if they are the same and 1
1311 * if they are different. We can then use a conditional modifier to
1312 * turn that into a predicate. This leads us to an XOR.l instruction.
1313 *
1314 * Technically, according to the PRM, you're not allowed to use .l on a
1315 * XOR instruction. However, emperical experiments and Curro's reading
1316 * of the simulator source both indicate that it's safe.
1317 */
1318 src_reg tmp = src_reg(this, glsl_type::ivec4_type);
1319 inst = emit(XOR(dst_reg(tmp), op[0], op[1]));
1320 inst->predicate = BRW_PREDICATE_NORMAL;
1321 inst->conditional_mod = BRW_CONDITIONAL_L;
1322
1323 /* If the result of the initial remainder operation is non-zero and the
1324 * two sources have different signs, add in a copy of op[1] to get the
1325 * final integer modulus value.
1326 */
1327 inst = emit(ADD(dst, src_reg(dst), op[1]));
1328 inst->predicate = BRW_PREDICATE_NORMAL;
1329 break;
1330 }
1331
1332 case nir_op_ldexp:
1333 unreachable("not reached: should be handled by ldexp_to_arith()");
1334
1335 case nir_op_fsqrt:
1336 inst = emit_math(SHADER_OPCODE_SQRT, dst, op[0]);
1337 break;
1338
1339 case nir_op_frsq:
1340 inst = emit_math(SHADER_OPCODE_RSQ, dst, op[0]);
1341 break;
1342
1343 case nir_op_fpow:
1344 inst = emit_math(SHADER_OPCODE_POW, dst, op[0], op[1]);
1345 break;
1346
1347 case nir_op_uadd_carry: {
1348 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1349 struct brw_reg acc = retype(brw_acc_reg(8), BRW_REGISTER_TYPE_UD);
1350
1351 emit(ADDC(dst_null_ud(), op[0], op[1]));
1352 emit(MOV(dst, src_reg(acc)));
1353 break;
1354 }
1355
1356 case nir_op_usub_borrow: {
1357 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1358 struct brw_reg acc = retype(brw_acc_reg(8), BRW_REGISTER_TYPE_UD);
1359
1360 emit(SUBB(dst_null_ud(), op[0], op[1]));
1361 emit(MOV(dst, src_reg(acc)));
1362 break;
1363 }
1364
1365 case nir_op_ftrunc:
1366 inst = emit(RNDZ(dst, op[0]));
1367 if (devinfo->gen < 6) {
1368 inst->conditional_mod = BRW_CONDITIONAL_R;
1369 inst = emit(ADD(dst, src_reg(dst), brw_imm_f(1.0f)));
1370 inst->predicate = BRW_PREDICATE_NORMAL;
1371 inst = emit(MOV(dst, src_reg(dst))); /* for potential saturation */
1372 }
1373 break;
1374
1375 case nir_op_fceil: {
1376 src_reg tmp = src_reg(this, glsl_type::float_type);
1377 tmp.swizzle =
1378 brw_swizzle_for_size(instr->src[0].src.is_ssa ?
1379 instr->src[0].src.ssa->num_components :
1380 instr->src[0].src.reg.reg->num_components);
1381
1382 op[0].negate = !op[0].negate;
1383 emit(RNDD(dst_reg(tmp), op[0]));
1384 tmp.negate = true;
1385 inst = emit(MOV(dst, tmp));
1386 break;
1387 }
1388
1389 case nir_op_ffloor:
1390 inst = emit(RNDD(dst, op[0]));
1391 break;
1392
1393 case nir_op_ffract:
1394 inst = emit(FRC(dst, op[0]));
1395 break;
1396
1397 case nir_op_fround_even:
1398 inst = emit(RNDE(dst, op[0]));
1399 if (devinfo->gen < 6) {
1400 inst->conditional_mod = BRW_CONDITIONAL_R;
1401 inst = emit(ADD(dst, src_reg(dst), brw_imm_f(1.0f)));
1402 inst->predicate = BRW_PREDICATE_NORMAL;
1403 inst = emit(MOV(dst, src_reg(dst))); /* for potential saturation */
1404 }
1405 break;
1406
1407 case nir_op_fquantize2f16: {
1408 /* See also vec4_visitor::emit_pack_half_2x16() */
1409 src_reg tmp16 = src_reg(this, glsl_type::uvec4_type);
1410 src_reg tmp32 = src_reg(this, glsl_type::vec4_type);
1411 src_reg zero = src_reg(this, glsl_type::vec4_type);
1412
1413 /* Check for denormal */
1414 src_reg abs_src0 = op[0];
1415 abs_src0.abs = true;
1416 emit(CMP(dst_null_f(), abs_src0, brw_imm_f(ldexpf(1.0, -14)),
1417 BRW_CONDITIONAL_L));
1418 /* Get the appropriately signed zero */
1419 emit(AND(retype(dst_reg(zero), BRW_REGISTER_TYPE_UD),
1420 retype(op[0], BRW_REGISTER_TYPE_UD),
1421 brw_imm_ud(0x80000000)));
1422 /* Do the actual F32 -> F16 -> F32 conversion */
1423 emit(F32TO16(dst_reg(tmp16), op[0]));
1424 emit(F16TO32(dst_reg(tmp32), tmp16));
1425 /* Select that or zero based on normal status */
1426 inst = emit(BRW_OPCODE_SEL, dst, zero, tmp32);
1427 inst->predicate = BRW_PREDICATE_NORMAL;
1428 break;
1429 }
1430
1431 case nir_op_imin:
1432 case nir_op_umin:
1433 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1434 /* fall through */
1435 case nir_op_fmin:
1436 try_immediate_source(instr, op, true, devinfo);
1437 inst = emit_minmax(BRW_CONDITIONAL_L, dst, op[0], op[1]);
1438 break;
1439
1440 case nir_op_imax:
1441 case nir_op_umax:
1442 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1443 /* fall through */
1444 case nir_op_fmax:
1445 try_immediate_source(instr, op, true, devinfo);
1446 inst = emit_minmax(BRW_CONDITIONAL_GE, dst, op[0], op[1]);
1447 break;
1448
1449 case nir_op_fddx:
1450 case nir_op_fddx_coarse:
1451 case nir_op_fddx_fine:
1452 case nir_op_fddy:
1453 case nir_op_fddy_coarse:
1454 case nir_op_fddy_fine:
1455 unreachable("derivatives are not valid in vertex shaders");
1456
1457 case nir_op_ilt32:
1458 case nir_op_ult32:
1459 case nir_op_ige32:
1460 case nir_op_uge32:
1461 case nir_op_ieq32:
1462 case nir_op_ine32:
1463 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1464 /* Fallthrough */
1465 case nir_op_flt32:
1466 case nir_op_fge32:
1467 case nir_op_feq32:
1468 case nir_op_fneu32: {
1469 enum brw_conditional_mod conditional_mod =
1470 brw_cmod_for_nir_comparison(instr->op);
1471
1472 if (nir_src_bit_size(instr->src[0].src) < 64) {
1473 /* If the order of the sources is changed due to an immediate value,
1474 * then the condition must also be changed.
1475 */
1476 if (try_immediate_source(instr, op, true, devinfo) == 0)
1477 conditional_mod = brw_swap_cmod(conditional_mod);
1478
1479 emit(CMP(dst, op[0], op[1], conditional_mod));
1480 } else {
1481 /* Produce a 32-bit boolean result from the DF comparison by selecting
1482 * only the low 32-bit in each DF produced. Do this in a temporary
1483 * so we can then move from there to the result using align16 again
1484 * to honor the original writemask.
1485 */
1486 dst_reg temp = dst_reg(this, glsl_type::dvec4_type);
1487 emit(CMP(temp, op[0], op[1], conditional_mod));
1488 dst_reg result = dst_reg(this, glsl_type::bvec4_type);
1489 emit(VEC4_OPCODE_PICK_LOW_32BIT, result, src_reg(temp));
1490 emit(MOV(dst, src_reg(result)));
1491 }
1492 break;
1493 }
1494
1495 case nir_op_b32all_iequal2:
1496 case nir_op_b32all_iequal3:
1497 case nir_op_b32all_iequal4:
1498 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1499 /* Fallthrough */
1500 case nir_op_b32all_fequal2:
1501 case nir_op_b32all_fequal3:
1502 case nir_op_b32all_fequal4: {
1503 unsigned swiz =
1504 brw_swizzle_for_size(nir_op_infos[instr->op].input_sizes[0]);
1505
1506 emit(CMP(dst_null_d(), swizzle(op[0], swiz), swizzle(op[1], swiz),
1507 brw_cmod_for_nir_comparison(instr->op)));
1508 emit(MOV(dst, brw_imm_d(0)));
1509 inst = emit(MOV(dst, brw_imm_d(~0)));
1510 inst->predicate = BRW_PREDICATE_ALIGN16_ALL4H;
1511 break;
1512 }
1513
1514 case nir_op_b32any_inequal2:
1515 case nir_op_b32any_inequal3:
1516 case nir_op_b32any_inequal4:
1517 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1518 /* Fallthrough */
1519 case nir_op_b32any_fnequal2:
1520 case nir_op_b32any_fnequal3:
1521 case nir_op_b32any_fnequal4: {
1522 unsigned swiz =
1523 brw_swizzle_for_size(nir_op_infos[instr->op].input_sizes[0]);
1524
1525 emit(CMP(dst_null_d(), swizzle(op[0], swiz), swizzle(op[1], swiz),
1526 brw_cmod_for_nir_comparison(instr->op)));
1527
1528 emit(MOV(dst, brw_imm_d(0)));
1529 inst = emit(MOV(dst, brw_imm_d(~0)));
1530 inst->predicate = BRW_PREDICATE_ALIGN16_ANY4H;
1531 break;
1532 }
1533
1534 case nir_op_inot:
1535 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1536 if (devinfo->gen >= 8) {
1537 op[0] = resolve_source_modifiers(op[0]);
1538 }
1539 emit(NOT(dst, op[0]));
1540 break;
1541
1542 case nir_op_ixor:
1543 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1544 if (devinfo->gen >= 8) {
1545 op[0] = resolve_source_modifiers(op[0]);
1546 op[1] = resolve_source_modifiers(op[1]);
1547 }
1548 try_immediate_source(instr, op, true, devinfo);
1549 emit(XOR(dst, op[0], op[1]));
1550 break;
1551
1552 case nir_op_ior:
1553 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1554 if (devinfo->gen >= 8) {
1555 op[0] = resolve_source_modifiers(op[0]);
1556 op[1] = resolve_source_modifiers(op[1]);
1557 }
1558 try_immediate_source(instr, op, true, devinfo);
1559 emit(OR(dst, op[0], op[1]));
1560 break;
1561
1562 case nir_op_iand:
1563 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1564 if (devinfo->gen >= 8) {
1565 op[0] = resolve_source_modifiers(op[0]);
1566 op[1] = resolve_source_modifiers(op[1]);
1567 }
1568 try_immediate_source(instr, op, true, devinfo);
1569 emit(AND(dst, op[0], op[1]));
1570 break;
1571
1572 case nir_op_b2i32:
1573 case nir_op_b2f32:
1574 case nir_op_b2f64:
1575 if (nir_dest_bit_size(instr->dest.dest) > 32) {
1576 assert(dst.type == BRW_REGISTER_TYPE_DF);
1577 emit_conversion_to_double(dst, negate(op[0]));
1578 } else {
1579 emit(MOV(dst, negate(op[0])));
1580 }
1581 break;
1582
1583 case nir_op_f2b32:
1584 if (nir_src_bit_size(instr->src[0].src) == 64) {
1585 /* We use a MOV with conditional_mod to check if the provided value is
1586 * 0.0. We want this to flush denormalized numbers to zero, so we set a
1587 * source modifier on the source operand to trigger this, as source
1588 * modifiers don't affect the result of the testing against 0.0.
1589 */
1590 src_reg value = op[0];
1591 value.abs = true;
1592 vec4_instruction *inst = emit(MOV(dst_null_df(), value));
1593 inst->conditional_mod = BRW_CONDITIONAL_NZ;
1594
1595 src_reg one = src_reg(this, glsl_type::ivec4_type);
1596 emit(MOV(dst_reg(one), brw_imm_d(~0)));
1597 inst = emit(BRW_OPCODE_SEL, dst, one, brw_imm_d(0));
1598 inst->predicate = BRW_PREDICATE_NORMAL;
1599 } else {
1600 emit(CMP(dst, op[0], brw_imm_f(0.0f), BRW_CONDITIONAL_NZ));
1601 }
1602 break;
1603
1604 case nir_op_i2b32:
1605 emit(CMP(dst, op[0], brw_imm_d(0), BRW_CONDITIONAL_NZ));
1606 break;
1607
1608 case nir_op_unpack_half_2x16_split_x:
1609 case nir_op_unpack_half_2x16_split_y:
1610 case nir_op_pack_half_2x16_split:
1611 unreachable("not reached: should not occur in vertex shader");
1612
1613 case nir_op_unpack_snorm_2x16:
1614 case nir_op_unpack_unorm_2x16:
1615 case nir_op_pack_snorm_2x16:
1616 case nir_op_pack_unorm_2x16:
1617 unreachable("not reached: should be handled by lower_packing_builtins");
1618
1619 case nir_op_pack_uvec4_to_uint:
1620 unreachable("not reached");
1621
1622 case nir_op_pack_uvec2_to_uint: {
1623 dst_reg tmp1 = dst_reg(this, glsl_type::uint_type);
1624 tmp1.writemask = WRITEMASK_X;
1625 op[0].swizzle = BRW_SWIZZLE_YYYY;
1626 emit(SHL(tmp1, op[0], src_reg(brw_imm_ud(16u))));
1627
1628 dst_reg tmp2 = dst_reg(this, glsl_type::uint_type);
1629 tmp2.writemask = WRITEMASK_X;
1630 op[0].swizzle = BRW_SWIZZLE_XXXX;
1631 emit(AND(tmp2, op[0], src_reg(brw_imm_ud(0xffffu))));
1632
1633 emit(OR(dst, src_reg(tmp1), src_reg(tmp2)));
1634 break;
1635 }
1636
1637 case nir_op_pack_64_2x32_split: {
1638 dst_reg result = dst_reg(this, glsl_type::dvec4_type);
1639 dst_reg tmp = dst_reg(this, glsl_type::uvec4_type);
1640 emit(MOV(tmp, retype(op[0], BRW_REGISTER_TYPE_UD)));
1641 emit(VEC4_OPCODE_SET_LOW_32BIT, result, src_reg(tmp));
1642 emit(MOV(tmp, retype(op[1], BRW_REGISTER_TYPE_UD)));
1643 emit(VEC4_OPCODE_SET_HIGH_32BIT, result, src_reg(tmp));
1644 emit(MOV(dst, src_reg(result)));
1645 break;
1646 }
1647
1648 case nir_op_unpack_64_2x32_split_x:
1649 case nir_op_unpack_64_2x32_split_y: {
1650 enum opcode oper = (instr->op == nir_op_unpack_64_2x32_split_x) ?
1651 VEC4_OPCODE_PICK_LOW_32BIT : VEC4_OPCODE_PICK_HIGH_32BIT;
1652 dst_reg tmp = dst_reg(this, glsl_type::dvec4_type);
1653 emit(MOV(tmp, op[0]));
1654 dst_reg tmp2 = dst_reg(this, glsl_type::uvec4_type);
1655 emit(oper, tmp2, src_reg(tmp));
1656 emit(MOV(dst, src_reg(tmp2)));
1657 break;
1658 }
1659
1660 case nir_op_unpack_half_2x16:
1661 /* As NIR does not guarantee that we have a correct swizzle outside the
1662 * boundaries of a vector, and the implementation of emit_unpack_half_2x16
1663 * uses the source operand in an operation with WRITEMASK_Y while our
1664 * source operand has only size 1, it accessed incorrect data producing
1665 * regressions in Piglit. We repeat the swizzle of the first component on the
1666 * rest of components to avoid regressions. In the vec4_visitor IR code path
1667 * this is not needed because the operand has already the correct swizzle.
1668 */
1669 op[0].swizzle = brw_compose_swizzle(BRW_SWIZZLE_XXXX, op[0].swizzle);
1670 emit_unpack_half_2x16(dst, op[0]);
1671 break;
1672
1673 case nir_op_pack_half_2x16:
1674 emit_pack_half_2x16(dst, op[0]);
1675 break;
1676
1677 case nir_op_unpack_unorm_4x8:
1678 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1679 emit_unpack_unorm_4x8(dst, op[0]);
1680 break;
1681
1682 case nir_op_pack_unorm_4x8:
1683 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1684 emit_pack_unorm_4x8(dst, op[0]);
1685 break;
1686
1687 case nir_op_unpack_snorm_4x8:
1688 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1689 emit_unpack_snorm_4x8(dst, op[0]);
1690 break;
1691
1692 case nir_op_pack_snorm_4x8:
1693 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1694 emit_pack_snorm_4x8(dst, op[0]);
1695 break;
1696
1697 case nir_op_bitfield_reverse:
1698 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1699 emit(BFREV(dst, op[0]));
1700 break;
1701
1702 case nir_op_bit_count:
1703 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1704 emit(CBIT(dst, op[0]));
1705 break;
1706
1707 case nir_op_ufind_msb:
1708 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1709 emit_find_msb_using_lzd(vec4_builder(this).at_end(), dst, op[0], false);
1710 break;
1711
1712 case nir_op_ifind_msb: {
1713 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1714 vec4_builder bld = vec4_builder(this).at_end();
1715 src_reg src(dst);
1716
1717 if (devinfo->gen < 7) {
1718 emit_find_msb_using_lzd(bld, dst, op[0], true);
1719 } else {
1720 emit(FBH(retype(dst, BRW_REGISTER_TYPE_UD), op[0]));
1721
1722 /* FBH counts from the MSB side, while GLSL's findMSB() wants the
1723 * count from the LSB side. If FBH didn't return an error
1724 * (0xFFFFFFFF), then subtract the result from 31 to convert the MSB
1725 * count into an LSB count.
1726 */
1727 bld.CMP(dst_null_d(), src, brw_imm_d(-1), BRW_CONDITIONAL_NZ);
1728
1729 inst = bld.ADD(dst, src, brw_imm_d(31));
1730 inst->predicate = BRW_PREDICATE_NORMAL;
1731 inst->src[0].negate = true;
1732 }
1733 break;
1734 }
1735
1736 case nir_op_find_lsb: {
1737 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1738 vec4_builder bld = vec4_builder(this).at_end();
1739
1740 if (devinfo->gen < 7) {
1741 dst_reg temp = bld.vgrf(BRW_REGISTER_TYPE_D);
1742
1743 /* (x & -x) generates a value that consists of only the LSB of x.
1744 * For all powers of 2, findMSB(y) == findLSB(y).
1745 */
1746 src_reg src = src_reg(retype(op[0], BRW_REGISTER_TYPE_D));
1747 src_reg negated_src = src;
1748
1749 /* One must be negated, and the other must be non-negated. It
1750 * doesn't matter which is which.
1751 */
1752 negated_src.negate = true;
1753 src.negate = false;
1754
1755 bld.AND(temp, src, negated_src);
1756 emit_find_msb_using_lzd(bld, dst, src_reg(temp), false);
1757 } else {
1758 bld.FBL(dst, op[0]);
1759 }
1760 break;
1761 }
1762
1763 case nir_op_ubitfield_extract:
1764 case nir_op_ibitfield_extract:
1765 unreachable("should have been lowered");
1766 case nir_op_ubfe:
1767 case nir_op_ibfe:
1768 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1769 op[0] = fix_3src_operand(op[0]);
1770 op[1] = fix_3src_operand(op[1]);
1771 op[2] = fix_3src_operand(op[2]);
1772
1773 emit(BFE(dst, op[2], op[1], op[0]));
1774 break;
1775
1776 case nir_op_bfm:
1777 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1778 emit(BFI1(dst, op[0], op[1]));
1779 break;
1780
1781 case nir_op_bfi:
1782 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1783 op[0] = fix_3src_operand(op[0]);
1784 op[1] = fix_3src_operand(op[1]);
1785 op[2] = fix_3src_operand(op[2]);
1786
1787 emit(BFI2(dst, op[0], op[1], op[2]));
1788 break;
1789
1790 case nir_op_bitfield_insert:
1791 unreachable("not reached: should have been lowered");
1792
1793 case nir_op_fsign:
1794 if (type_sz(op[0].type) < 8) {
1795 /* AND(val, 0x80000000) gives the sign bit.
1796 *
1797 * Predicated OR ORs 1.0 (0x3f800000) with the sign bit if val is not
1798 * zero.
1799 */
1800 emit(CMP(dst_null_f(), op[0], brw_imm_f(0.0f), BRW_CONDITIONAL_NZ));
1801
1802 op[0].type = BRW_REGISTER_TYPE_UD;
1803 dst.type = BRW_REGISTER_TYPE_UD;
1804 emit(AND(dst, op[0], brw_imm_ud(0x80000000u)));
1805
1806 inst = emit(OR(dst, src_reg(dst), brw_imm_ud(0x3f800000u)));
1807 inst->predicate = BRW_PREDICATE_NORMAL;
1808 dst.type = BRW_REGISTER_TYPE_F;
1809 } else {
1810 /* For doubles we do the same but we need to consider:
1811 *
1812 * - We use a MOV with conditional_mod instead of a CMP so that we can
1813 * skip loading a 0.0 immediate. We use a source modifier on the
1814 * source of the MOV so that we flush denormalized values to 0.
1815 * Since we want to compare against 0, this won't alter the result.
1816 * - We need to extract the high 32-bit of each DF where the sign
1817 * is stored.
1818 * - We need to produce a DF result.
1819 */
1820
1821 /* Check for zero */
1822 src_reg value = op[0];
1823 value.abs = true;
1824 inst = emit(MOV(dst_null_df(), value));
1825 inst->conditional_mod = BRW_CONDITIONAL_NZ;
1826
1827 /* AND each high 32-bit channel with 0x80000000u */
1828 dst_reg tmp = dst_reg(this, glsl_type::uvec4_type);
1829 emit(VEC4_OPCODE_PICK_HIGH_32BIT, tmp, op[0]);
1830 emit(AND(tmp, src_reg(tmp), brw_imm_ud(0x80000000u)));
1831
1832 /* Add 1.0 to each channel, predicated to skip the cases where the
1833 * channel's value was 0
1834 */
1835 inst = emit(OR(tmp, src_reg(tmp), brw_imm_ud(0x3f800000u)));
1836 inst->predicate = BRW_PREDICATE_NORMAL;
1837
1838 /* Now convert the result from float to double */
1839 emit_conversion_to_double(dst, retype(src_reg(tmp),
1840 BRW_REGISTER_TYPE_F));
1841 }
1842 break;
1843
1844 case nir_op_ishl:
1845 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1846 try_immediate_source(instr, op, false, devinfo);
1847 emit(SHL(dst, op[0], op[1]));
1848 break;
1849
1850 case nir_op_ishr:
1851 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1852 try_immediate_source(instr, op, false, devinfo);
1853 emit(ASR(dst, op[0], op[1]));
1854 break;
1855
1856 case nir_op_ushr:
1857 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1858 try_immediate_source(instr, op, false, devinfo);
1859 emit(SHR(dst, op[0], op[1]));
1860 break;
1861
1862 case nir_op_ffma:
1863 if (type_sz(dst.type) == 8) {
1864 dst_reg mul_dst = dst_reg(this, glsl_type::dvec4_type);
1865 emit(MUL(mul_dst, op[1], op[0]));
1866 inst = emit(ADD(dst, src_reg(mul_dst), op[2]));
1867 } else {
1868 fix_float_operands(op, instr);
1869 inst = emit(MAD(dst, op[2], op[1], op[0]));
1870 }
1871 break;
1872
1873 case nir_op_flrp:
1874 fix_float_operands(op, instr);
1875 inst = emit(LRP(dst, op[2], op[1], op[0]));
1876 break;
1877
1878 case nir_op_b32csel:
1879 enum brw_predicate predicate;
1880 if (!optimize_predicate(instr, &predicate)) {
1881 emit(CMP(dst_null_d(), op[0], brw_imm_d(0), BRW_CONDITIONAL_NZ));
1882 switch (dst.writemask) {
1883 case WRITEMASK_X:
1884 predicate = BRW_PREDICATE_ALIGN16_REPLICATE_X;
1885 break;
1886 case WRITEMASK_Y:
1887 predicate = BRW_PREDICATE_ALIGN16_REPLICATE_Y;
1888 break;
1889 case WRITEMASK_Z:
1890 predicate = BRW_PREDICATE_ALIGN16_REPLICATE_Z;
1891 break;
1892 case WRITEMASK_W:
1893 predicate = BRW_PREDICATE_ALIGN16_REPLICATE_W;
1894 break;
1895 default:
1896 predicate = BRW_PREDICATE_NORMAL;
1897 break;
1898 }
1899 }
1900 inst = emit(BRW_OPCODE_SEL, dst, op[1], op[2]);
1901 inst->predicate = predicate;
1902 break;
1903
1904 case nir_op_fdot_replicated2:
1905 try_immediate_source(instr, op, true, devinfo);
1906 inst = emit(BRW_OPCODE_DP2, dst, op[0], op[1]);
1907 break;
1908
1909 case nir_op_fdot_replicated3:
1910 try_immediate_source(instr, op, true, devinfo);
1911 inst = emit(BRW_OPCODE_DP3, dst, op[0], op[1]);
1912 break;
1913
1914 case nir_op_fdot_replicated4:
1915 try_immediate_source(instr, op, true, devinfo);
1916 inst = emit(BRW_OPCODE_DP4, dst, op[0], op[1]);
1917 break;
1918
1919 case nir_op_fdph_replicated:
1920 try_immediate_source(instr, op, false, devinfo);
1921 inst = emit(BRW_OPCODE_DPH, dst, op[0], op[1]);
1922 break;
1923
1924 case nir_op_fdiv:
1925 unreachable("not reached: should be lowered by DIV_TO_MUL_RCP in the compiler");
1926
1927 case nir_op_fmod:
1928 unreachable("not reached: should be lowered by MOD_TO_FLOOR in the compiler");
1929
1930 case nir_op_fsub:
1931 case nir_op_isub:
1932 unreachable("not reached: should be handled by ir_sub_to_add_neg");
1933
1934 default:
1935 unreachable("Unimplemented ALU operation");
1936 }
1937
1938 /* If we need to do a boolean resolve, replace the result with -(x & 1)
1939 * to sign extend the low bit to 0/~0
1940 */
1941 if (devinfo->gen <= 5 &&
1942 (instr->instr.pass_flags & BRW_NIR_BOOLEAN_MASK) ==
1943 BRW_NIR_BOOLEAN_NEEDS_RESOLVE) {
1944 dst_reg masked = dst_reg(this, glsl_type::int_type);
1945 masked.writemask = dst.writemask;
1946 emit(AND(masked, src_reg(dst), brw_imm_d(1)));
1947 src_reg masked_neg = src_reg(masked);
1948 masked_neg.negate = true;
1949 emit(MOV(retype(dst, BRW_REGISTER_TYPE_D), masked_neg));
1950 }
1951 }
1952
1953 void
1954 vec4_visitor::nir_emit_jump(nir_jump_instr *instr)
1955 {
1956 switch (instr->type) {
1957 case nir_jump_break:
1958 emit(BRW_OPCODE_BREAK);
1959 break;
1960
1961 case nir_jump_continue:
1962 emit(BRW_OPCODE_CONTINUE);
1963 break;
1964
1965 case nir_jump_return:
1966 /* fall through */
1967 default:
1968 unreachable("unknown jump");
1969 }
1970 }
1971
1972 static enum ir_texture_opcode
1973 ir_texture_opcode_for_nir_texop(nir_texop texop)
1974 {
1975 enum ir_texture_opcode op;
1976
1977 switch (texop) {
1978 case nir_texop_lod: op = ir_lod; break;
1979 case nir_texop_query_levels: op = ir_query_levels; break;
1980 case nir_texop_texture_samples: op = ir_texture_samples; break;
1981 case nir_texop_tex: op = ir_tex; break;
1982 case nir_texop_tg4: op = ir_tg4; break;
1983 case nir_texop_txb: op = ir_txb; break;
1984 case nir_texop_txd: op = ir_txd; break;
1985 case nir_texop_txf: op = ir_txf; break;
1986 case nir_texop_txf_ms: op = ir_txf_ms; break;
1987 case nir_texop_txl: op = ir_txl; break;
1988 case nir_texop_txs: op = ir_txs; break;
1989 case nir_texop_samples_identical: op = ir_samples_identical; break;
1990 default:
1991 unreachable("unknown texture opcode");
1992 }
1993
1994 return op;
1995 }
1996
1997 static const glsl_type *
1998 glsl_type_for_nir_alu_type(nir_alu_type alu_type,
1999 unsigned components)
2000 {
2001 return glsl_type::get_instance(brw_glsl_base_type_for_nir_type(alu_type),
2002 components, 1);
2003 }
2004
2005 void
2006 vec4_visitor::nir_emit_texture(nir_tex_instr *instr)
2007 {
2008 unsigned texture = instr->texture_index;
2009 unsigned sampler = instr->sampler_index;
2010 src_reg texture_reg = brw_imm_ud(texture);
2011 src_reg sampler_reg = brw_imm_ud(sampler);
2012 src_reg coordinate;
2013 const glsl_type *coord_type = NULL;
2014 src_reg shadow_comparator;
2015 src_reg offset_value;
2016 src_reg lod, lod2;
2017 src_reg sample_index;
2018 src_reg mcs;
2019
2020 const glsl_type *dest_type =
2021 glsl_type_for_nir_alu_type(instr->dest_type,
2022 nir_tex_instr_dest_size(instr));
2023 dst_reg dest = get_nir_dest(instr->dest, instr->dest_type);
2024
2025 /* The hardware requires a LOD for buffer textures */
2026 if (instr->sampler_dim == GLSL_SAMPLER_DIM_BUF)
2027 lod = brw_imm_d(0);
2028
2029 /* Load the texture operation sources */
2030 uint32_t constant_offset = 0;
2031 for (unsigned i = 0; i < instr->num_srcs; i++) {
2032 switch (instr->src[i].src_type) {
2033 case nir_tex_src_comparator:
2034 shadow_comparator = get_nir_src(instr->src[i].src,
2035 BRW_REGISTER_TYPE_F, 1);
2036 break;
2037
2038 case nir_tex_src_coord: {
2039 unsigned src_size = nir_tex_instr_src_size(instr, i);
2040
2041 switch (instr->op) {
2042 case nir_texop_txf:
2043 case nir_texop_txf_ms:
2044 case nir_texop_samples_identical:
2045 coordinate = get_nir_src(instr->src[i].src, BRW_REGISTER_TYPE_D,
2046 src_size);
2047 coord_type = glsl_type::ivec(src_size);
2048 break;
2049
2050 default:
2051 coordinate = get_nir_src(instr->src[i].src, BRW_REGISTER_TYPE_F,
2052 src_size);
2053 coord_type = glsl_type::vec(src_size);
2054 break;
2055 }
2056 break;
2057 }
2058
2059 case nir_tex_src_ddx:
2060 lod = get_nir_src(instr->src[i].src, BRW_REGISTER_TYPE_F,
2061 nir_tex_instr_src_size(instr, i));
2062 break;
2063
2064 case nir_tex_src_ddy:
2065 lod2 = get_nir_src(instr->src[i].src, BRW_REGISTER_TYPE_F,
2066 nir_tex_instr_src_size(instr, i));
2067 break;
2068
2069 case nir_tex_src_lod:
2070 switch (instr->op) {
2071 case nir_texop_txs:
2072 case nir_texop_txf:
2073 lod = get_nir_src(instr->src[i].src, BRW_REGISTER_TYPE_D, 1);
2074 break;
2075
2076 default:
2077 lod = get_nir_src(instr->src[i].src, BRW_REGISTER_TYPE_F, 1);
2078 break;
2079 }
2080 break;
2081
2082 case nir_tex_src_ms_index: {
2083 sample_index = get_nir_src(instr->src[i].src, BRW_REGISTER_TYPE_D, 1);
2084 break;
2085 }
2086
2087 case nir_tex_src_offset:
2088 if (!brw_texture_offset(instr, i, &constant_offset)) {
2089 offset_value =
2090 get_nir_src(instr->src[i].src, BRW_REGISTER_TYPE_D, 2);
2091 }
2092 break;
2093
2094 case nir_tex_src_texture_offset: {
2095 /* Emit code to evaluate the actual indexing expression */
2096 src_reg src = get_nir_src(instr->src[i].src, 1);
2097 src_reg temp(this, glsl_type::uint_type);
2098 emit(ADD(dst_reg(temp), src, brw_imm_ud(texture)));
2099 texture_reg = emit_uniformize(temp);
2100 break;
2101 }
2102
2103 case nir_tex_src_sampler_offset: {
2104 /* Emit code to evaluate the actual indexing expression */
2105 src_reg src = get_nir_src(instr->src[i].src, 1);
2106 src_reg temp(this, glsl_type::uint_type);
2107 emit(ADD(dst_reg(temp), src, brw_imm_ud(sampler)));
2108 sampler_reg = emit_uniformize(temp);
2109 break;
2110 }
2111
2112 case nir_tex_src_projector:
2113 unreachable("Should be lowered by do_lower_texture_projection");
2114
2115 case nir_tex_src_bias:
2116 unreachable("LOD bias is not valid for vertex shaders.\n");
2117
2118 default:
2119 unreachable("unknown texture source");
2120 }
2121 }
2122
2123 if (instr->op == nir_texop_txf_ms ||
2124 instr->op == nir_texop_samples_identical) {
2125 assert(coord_type != NULL);
2126 if (devinfo->gen >= 7 &&
2127 key_tex->compressed_multisample_layout_mask & (1 << texture)) {
2128 mcs = emit_mcs_fetch(coord_type, coordinate, texture_reg);
2129 } else {
2130 mcs = brw_imm_ud(0u);
2131 }
2132 }
2133
2134 /* Stuff the channel select bits in the top of the texture offset */
2135 if (instr->op == nir_texop_tg4) {
2136 if (instr->component == 1 &&
2137 (key_tex->gather_channel_quirk_mask & (1 << texture))) {
2138 /* gather4 sampler is broken for green channel on RG32F --
2139 * we must ask for blue instead.
2140 */
2141 constant_offset |= 2 << 16;
2142 } else {
2143 constant_offset |= instr->component << 16;
2144 }
2145 }
2146
2147 ir_texture_opcode op = ir_texture_opcode_for_nir_texop(instr->op);
2148
2149 emit_texture(op, dest, dest_type, coordinate, instr->coord_components,
2150 shadow_comparator,
2151 lod, lod2, sample_index,
2152 constant_offset, offset_value, mcs,
2153 texture, texture_reg, sampler_reg);
2154 }
2155
2156 void
2157 vec4_visitor::nir_emit_undef(nir_ssa_undef_instr *instr)
2158 {
2159 nir_ssa_values[instr->def.index] =
2160 dst_reg(VGRF, alloc.allocate(DIV_ROUND_UP(instr->def.bit_size, 32)));
2161 }
2162
2163 /* SIMD4x2 64bit data is stored in register space like this:
2164 *
2165 * r0.0:DF x0 y0 z0 w0
2166 * r1.0:DF x1 y1 z1 w1
2167 *
2168 * When we need to write data such as this to memory using 32-bit write
2169 * messages we need to shuffle it in this fashion:
2170 *
2171 * r0.0:DF x0 y0 x1 y1 (to be written at base offset)
2172 * r0.0:DF z0 w0 z1 w1 (to be written at base offset + 16)
2173 *
2174 * We need to do the inverse operation when we read using 32-bit messages,
2175 * which we can do by applying the same exact shuffling on the 64-bit data
2176 * read, only that because the data for each vertex is positioned differently
2177 * we need to apply different channel enables.
2178 *
2179 * This function takes 64bit data and shuffles it as explained above.
2180 *
2181 * The @for_write parameter is used to specify if the shuffling is being done
2182 * for proper SIMD4x2 64-bit data that needs to be shuffled prior to a 32-bit
2183 * write message (for_write = true), or instead we are doing the inverse
2184 * operation and we have just read 64-bit data using a 32-bit messages that we
2185 * need to shuffle to create valid SIMD4x2 64-bit data (for_write = false).
2186 *
2187 * If @block and @ref are non-NULL, then the shuffling is done after @ref,
2188 * otherwise the instructions are emitted normally at the end. The function
2189 * returns the last instruction inserted.
2190 *
2191 * Notice that @src and @dst cannot be the same register.
2192 */
2193 vec4_instruction *
2194 vec4_visitor::shuffle_64bit_data(dst_reg dst, src_reg src, bool for_write,
2195 bblock_t *block, vec4_instruction *ref)
2196 {
2197 assert(type_sz(src.type) == 8);
2198 assert(type_sz(dst.type) == 8);
2199 assert(!regions_overlap(dst, 2 * REG_SIZE, src, 2 * REG_SIZE));
2200 assert(!ref == !block);
2201
2202 const vec4_builder bld = !ref ? vec4_builder(this).at_end() :
2203 vec4_builder(this).at(block, ref->next);
2204
2205 /* Resolve swizzle in src */
2206 vec4_instruction *inst;
2207 if (src.swizzle != BRW_SWIZZLE_XYZW) {
2208 dst_reg data = dst_reg(this, glsl_type::dvec4_type);
2209 inst = bld.MOV(data, src);
2210 src = src_reg(data);
2211 }
2212
2213 /* dst+0.XY = src+0.XY */
2214 inst = bld.group(4, 0).MOV(writemask(dst, WRITEMASK_XY), src);
2215
2216 /* dst+0.ZW = src+1.XY */
2217 inst = bld.group(4, for_write ? 1 : 0)
2218 .MOV(writemask(dst, WRITEMASK_ZW),
2219 swizzle(byte_offset(src, REG_SIZE), BRW_SWIZZLE_XYXY));
2220
2221 /* dst+1.XY = src+0.ZW */
2222 inst = bld.group(4, for_write ? 0 : 1)
2223 .MOV(writemask(byte_offset(dst, REG_SIZE), WRITEMASK_XY),
2224 swizzle(src, BRW_SWIZZLE_ZWZW));
2225
2226 /* dst+1.ZW = src+1.ZW */
2227 inst = bld.group(4, 1)
2228 .MOV(writemask(byte_offset(dst, REG_SIZE), WRITEMASK_ZW),
2229 byte_offset(src, REG_SIZE));
2230
2231 return inst;
2232 }
2233
2234 }