nir: Move intel's intrinsic_image_coordinate_components() to core nir.
[mesa.git] / src / intel / compiler / brw_fs_nir.cpp
1 /*
2 * Copyright © 2010 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "compiler/glsl/ir.h"
25 #include "brw_fs.h"
26 #include "brw_nir.h"
27 #include "brw_eu.h"
28 #include "nir_search_helpers.h"
29 #include "util/u_math.h"
30 #include "util/bitscan.h"
31
32 using namespace brw;
33
34 void
35 fs_visitor::emit_nir_code()
36 {
37 emit_shader_float_controls_execution_mode();
38
39 /* emit the arrays used for inputs and outputs - load/store intrinsics will
40 * be converted to reads/writes of these arrays
41 */
42 nir_setup_outputs();
43 nir_setup_uniforms();
44 nir_emit_system_values();
45 last_scratch = ALIGN(nir->scratch_size, 4) * dispatch_width;
46
47 nir_emit_impl(nir_shader_get_entrypoint((nir_shader *)nir));
48 }
49
50 void
51 fs_visitor::nir_setup_outputs()
52 {
53 if (stage == MESA_SHADER_TESS_CTRL || stage == MESA_SHADER_FRAGMENT)
54 return;
55
56 unsigned vec4s[VARYING_SLOT_TESS_MAX] = { 0, };
57
58 /* Calculate the size of output registers in a separate pass, before
59 * allocating them. With ARB_enhanced_layouts, multiple output variables
60 * may occupy the same slot, but have different type sizes.
61 */
62 nir_foreach_variable(var, &nir->outputs) {
63 const int loc = var->data.driver_location;
64 const unsigned var_vec4s =
65 var->data.compact ? DIV_ROUND_UP(glsl_get_length(var->type), 4)
66 : type_size_vec4(var->type, true);
67 vec4s[loc] = MAX2(vec4s[loc], var_vec4s);
68 }
69
70 for (unsigned loc = 0; loc < ARRAY_SIZE(vec4s);) {
71 if (vec4s[loc] == 0) {
72 loc++;
73 continue;
74 }
75
76 unsigned reg_size = vec4s[loc];
77
78 /* Check if there are any ranges that start within this range and extend
79 * past it. If so, include them in this allocation.
80 */
81 for (unsigned i = 1; i < reg_size; i++)
82 reg_size = MAX2(vec4s[i + loc] + i, reg_size);
83
84 fs_reg reg = bld.vgrf(BRW_REGISTER_TYPE_F, 4 * reg_size);
85 for (unsigned i = 0; i < reg_size; i++)
86 outputs[loc + i] = offset(reg, bld, 4 * i);
87
88 loc += reg_size;
89 }
90 }
91
92 void
93 fs_visitor::nir_setup_uniforms()
94 {
95 /* Only the first compile gets to set up uniforms. */
96 if (push_constant_loc) {
97 assert(pull_constant_loc);
98 return;
99 }
100
101 uniforms = nir->num_uniforms / 4;
102
103 if (stage == MESA_SHADER_COMPUTE) {
104 /* Add a uniform for the thread local id. It must be the last uniform
105 * on the list.
106 */
107 assert(uniforms == prog_data->nr_params);
108 uint32_t *param = brw_stage_prog_data_add_params(prog_data, 1);
109 *param = BRW_PARAM_BUILTIN_SUBGROUP_ID;
110 subgroup_id = fs_reg(UNIFORM, uniforms++, BRW_REGISTER_TYPE_UD);
111 }
112 }
113
114 static bool
115 emit_system_values_block(nir_block *block, fs_visitor *v)
116 {
117 fs_reg *reg;
118
119 nir_foreach_instr(instr, block) {
120 if (instr->type != nir_instr_type_intrinsic)
121 continue;
122
123 nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
124 switch (intrin->intrinsic) {
125 case nir_intrinsic_load_vertex_id:
126 case nir_intrinsic_load_base_vertex:
127 unreachable("should be lowered by nir_lower_system_values().");
128
129 case nir_intrinsic_load_vertex_id_zero_base:
130 case nir_intrinsic_load_is_indexed_draw:
131 case nir_intrinsic_load_first_vertex:
132 case nir_intrinsic_load_instance_id:
133 case nir_intrinsic_load_base_instance:
134 case nir_intrinsic_load_draw_id:
135 unreachable("should be lowered by brw_nir_lower_vs_inputs().");
136
137 case nir_intrinsic_load_invocation_id:
138 if (v->stage == MESA_SHADER_TESS_CTRL)
139 break;
140 assert(v->stage == MESA_SHADER_GEOMETRY);
141 reg = &v->nir_system_values[SYSTEM_VALUE_INVOCATION_ID];
142 if (reg->file == BAD_FILE) {
143 const fs_builder abld = v->bld.annotate("gl_InvocationID", NULL);
144 fs_reg g1(retype(brw_vec8_grf(1, 0), BRW_REGISTER_TYPE_UD));
145 fs_reg iid = abld.vgrf(BRW_REGISTER_TYPE_UD, 1);
146 abld.SHR(iid, g1, brw_imm_ud(27u));
147 *reg = iid;
148 }
149 break;
150
151 case nir_intrinsic_load_sample_pos:
152 assert(v->stage == MESA_SHADER_FRAGMENT);
153 reg = &v->nir_system_values[SYSTEM_VALUE_SAMPLE_POS];
154 if (reg->file == BAD_FILE)
155 *reg = *v->emit_samplepos_setup();
156 break;
157
158 case nir_intrinsic_load_sample_id:
159 assert(v->stage == MESA_SHADER_FRAGMENT);
160 reg = &v->nir_system_values[SYSTEM_VALUE_SAMPLE_ID];
161 if (reg->file == BAD_FILE)
162 *reg = *v->emit_sampleid_setup();
163 break;
164
165 case nir_intrinsic_load_sample_mask_in:
166 assert(v->stage == MESA_SHADER_FRAGMENT);
167 assert(v->devinfo->gen >= 7);
168 reg = &v->nir_system_values[SYSTEM_VALUE_SAMPLE_MASK_IN];
169 if (reg->file == BAD_FILE)
170 *reg = *v->emit_samplemaskin_setup();
171 break;
172
173 case nir_intrinsic_load_work_group_id:
174 assert(v->stage == MESA_SHADER_COMPUTE);
175 reg = &v->nir_system_values[SYSTEM_VALUE_WORK_GROUP_ID];
176 if (reg->file == BAD_FILE)
177 *reg = *v->emit_cs_work_group_id_setup();
178 break;
179
180 case nir_intrinsic_load_helper_invocation:
181 assert(v->stage == MESA_SHADER_FRAGMENT);
182 reg = &v->nir_system_values[SYSTEM_VALUE_HELPER_INVOCATION];
183 if (reg->file == BAD_FILE) {
184 const fs_builder abld =
185 v->bld.annotate("gl_HelperInvocation", NULL);
186
187 /* On Gen6+ (gl_HelperInvocation is only exposed on Gen7+) the
188 * pixel mask is in g1.7 of the thread payload.
189 *
190 * We move the per-channel pixel enable bit to the low bit of each
191 * channel by shifting the byte containing the pixel mask by the
192 * vector immediate 0x76543210UV.
193 *
194 * The region of <1,8,0> reads only 1 byte (the pixel masks for
195 * subspans 0 and 1) in SIMD8 and an additional byte (the pixel
196 * masks for 2 and 3) in SIMD16.
197 */
198 fs_reg shifted = abld.vgrf(BRW_REGISTER_TYPE_UW, 1);
199
200 for (unsigned i = 0; i < DIV_ROUND_UP(v->dispatch_width, 16); i++) {
201 const fs_builder hbld = abld.group(MIN2(16, v->dispatch_width), i);
202 hbld.SHR(offset(shifted, hbld, i),
203 stride(retype(brw_vec1_grf(1 + i, 7),
204 BRW_REGISTER_TYPE_UB),
205 1, 8, 0),
206 brw_imm_v(0x76543210));
207 }
208
209 /* A set bit in the pixel mask means the channel is enabled, but
210 * that is the opposite of gl_HelperInvocation so we need to invert
211 * the mask.
212 *
213 * The negate source-modifier bit of logical instructions on Gen8+
214 * performs 1's complement negation, so we can use that instead of
215 * a NOT instruction.
216 */
217 fs_reg inverted = negate(shifted);
218 if (v->devinfo->gen < 8) {
219 inverted = abld.vgrf(BRW_REGISTER_TYPE_UW);
220 abld.NOT(inverted, shifted);
221 }
222
223 /* We then resolve the 0/1 result to 0/~0 boolean values by ANDing
224 * with 1 and negating.
225 */
226 fs_reg anded = abld.vgrf(BRW_REGISTER_TYPE_UD, 1);
227 abld.AND(anded, inverted, brw_imm_uw(1));
228
229 fs_reg dst = abld.vgrf(BRW_REGISTER_TYPE_D, 1);
230 abld.MOV(dst, negate(retype(anded, BRW_REGISTER_TYPE_D)));
231 *reg = dst;
232 }
233 break;
234
235 default:
236 break;
237 }
238 }
239
240 return true;
241 }
242
243 void
244 fs_visitor::nir_emit_system_values()
245 {
246 nir_system_values = ralloc_array(mem_ctx, fs_reg, SYSTEM_VALUE_MAX);
247 for (unsigned i = 0; i < SYSTEM_VALUE_MAX; i++) {
248 nir_system_values[i] = fs_reg();
249 }
250
251 /* Always emit SUBGROUP_INVOCATION. Dead code will clean it up if we
252 * never end up using it.
253 */
254 {
255 const fs_builder abld = bld.annotate("gl_SubgroupInvocation", NULL);
256 fs_reg &reg = nir_system_values[SYSTEM_VALUE_SUBGROUP_INVOCATION];
257 reg = abld.vgrf(BRW_REGISTER_TYPE_UW);
258
259 const fs_builder allbld8 = abld.group(8, 0).exec_all();
260 allbld8.MOV(reg, brw_imm_v(0x76543210));
261 if (dispatch_width > 8)
262 allbld8.ADD(byte_offset(reg, 16), reg, brw_imm_uw(8u));
263 if (dispatch_width > 16) {
264 const fs_builder allbld16 = abld.group(16, 0).exec_all();
265 allbld16.ADD(byte_offset(reg, 32), reg, brw_imm_uw(16u));
266 }
267 }
268
269 nir_function_impl *impl = nir_shader_get_entrypoint((nir_shader *)nir);
270 nir_foreach_block(block, impl)
271 emit_system_values_block(block, this);
272 }
273
274 /*
275 * Returns a type based on a reference_type (word, float, half-float) and a
276 * given bit_size.
277 *
278 * Reference BRW_REGISTER_TYPE are HF,F,DF,W,D,UW,UD.
279 *
280 * @FIXME: 64-bit return types are always DF on integer types to maintain
281 * compability with uses of DF previously to the introduction of int64
282 * support.
283 */
284 static brw_reg_type
285 brw_reg_type_from_bit_size(const unsigned bit_size,
286 const brw_reg_type reference_type)
287 {
288 switch(reference_type) {
289 case BRW_REGISTER_TYPE_HF:
290 case BRW_REGISTER_TYPE_F:
291 case BRW_REGISTER_TYPE_DF:
292 switch(bit_size) {
293 case 16:
294 return BRW_REGISTER_TYPE_HF;
295 case 32:
296 return BRW_REGISTER_TYPE_F;
297 case 64:
298 return BRW_REGISTER_TYPE_DF;
299 default:
300 unreachable("Invalid bit size");
301 }
302 case BRW_REGISTER_TYPE_B:
303 case BRW_REGISTER_TYPE_W:
304 case BRW_REGISTER_TYPE_D:
305 case BRW_REGISTER_TYPE_Q:
306 switch(bit_size) {
307 case 8:
308 return BRW_REGISTER_TYPE_B;
309 case 16:
310 return BRW_REGISTER_TYPE_W;
311 case 32:
312 return BRW_REGISTER_TYPE_D;
313 case 64:
314 return BRW_REGISTER_TYPE_Q;
315 default:
316 unreachable("Invalid bit size");
317 }
318 case BRW_REGISTER_TYPE_UB:
319 case BRW_REGISTER_TYPE_UW:
320 case BRW_REGISTER_TYPE_UD:
321 case BRW_REGISTER_TYPE_UQ:
322 switch(bit_size) {
323 case 8:
324 return BRW_REGISTER_TYPE_UB;
325 case 16:
326 return BRW_REGISTER_TYPE_UW;
327 case 32:
328 return BRW_REGISTER_TYPE_UD;
329 case 64:
330 return BRW_REGISTER_TYPE_UQ;
331 default:
332 unreachable("Invalid bit size");
333 }
334 default:
335 unreachable("Unknown type");
336 }
337 }
338
339 void
340 fs_visitor::nir_emit_impl(nir_function_impl *impl)
341 {
342 nir_locals = ralloc_array(mem_ctx, fs_reg, impl->reg_alloc);
343 for (unsigned i = 0; i < impl->reg_alloc; i++) {
344 nir_locals[i] = fs_reg();
345 }
346
347 foreach_list_typed(nir_register, reg, node, &impl->registers) {
348 unsigned array_elems =
349 reg->num_array_elems == 0 ? 1 : reg->num_array_elems;
350 unsigned size = array_elems * reg->num_components;
351 const brw_reg_type reg_type = reg->bit_size == 8 ? BRW_REGISTER_TYPE_B :
352 brw_reg_type_from_bit_size(reg->bit_size, BRW_REGISTER_TYPE_F);
353 nir_locals[reg->index] = bld.vgrf(reg_type, size);
354 }
355
356 nir_ssa_values = reralloc(mem_ctx, nir_ssa_values, fs_reg,
357 impl->ssa_alloc);
358
359 nir_emit_cf_list(&impl->body);
360 }
361
362 void
363 fs_visitor::nir_emit_cf_list(exec_list *list)
364 {
365 exec_list_validate(list);
366 foreach_list_typed(nir_cf_node, node, node, list) {
367 switch (node->type) {
368 case nir_cf_node_if:
369 nir_emit_if(nir_cf_node_as_if(node));
370 break;
371
372 case nir_cf_node_loop:
373 nir_emit_loop(nir_cf_node_as_loop(node));
374 break;
375
376 case nir_cf_node_block:
377 nir_emit_block(nir_cf_node_as_block(node));
378 break;
379
380 default:
381 unreachable("Invalid CFG node block");
382 }
383 }
384 }
385
386 void
387 fs_visitor::nir_emit_if(nir_if *if_stmt)
388 {
389 bool invert;
390 fs_reg cond_reg;
391
392 /* If the condition has the form !other_condition, use other_condition as
393 * the source, but invert the predicate on the if instruction.
394 */
395 nir_alu_instr *cond = nir_src_as_alu_instr(if_stmt->condition);
396 if (cond != NULL && cond->op == nir_op_inot) {
397 assert(!cond->src[0].negate);
398 assert(!cond->src[0].abs);
399
400 invert = true;
401 cond_reg = get_nir_src(cond->src[0].src);
402 } else {
403 invert = false;
404 cond_reg = get_nir_src(if_stmt->condition);
405 }
406
407 /* first, put the condition into f0 */
408 fs_inst *inst = bld.MOV(bld.null_reg_d(),
409 retype(cond_reg, BRW_REGISTER_TYPE_D));
410 inst->conditional_mod = BRW_CONDITIONAL_NZ;
411
412 bld.IF(BRW_PREDICATE_NORMAL)->predicate_inverse = invert;
413
414 nir_emit_cf_list(&if_stmt->then_list);
415
416 if (!nir_cf_list_is_empty_block(&if_stmt->else_list)) {
417 bld.emit(BRW_OPCODE_ELSE);
418 nir_emit_cf_list(&if_stmt->else_list);
419 }
420
421 bld.emit(BRW_OPCODE_ENDIF);
422
423 if (devinfo->gen < 7)
424 limit_dispatch_width(16, "Non-uniform control flow unsupported "
425 "in SIMD32 mode.");
426 }
427
428 void
429 fs_visitor::nir_emit_loop(nir_loop *loop)
430 {
431 bld.emit(BRW_OPCODE_DO);
432
433 nir_emit_cf_list(&loop->body);
434
435 bld.emit(BRW_OPCODE_WHILE);
436
437 if (devinfo->gen < 7)
438 limit_dispatch_width(16, "Non-uniform control flow unsupported "
439 "in SIMD32 mode.");
440 }
441
442 void
443 fs_visitor::nir_emit_block(nir_block *block)
444 {
445 nir_foreach_instr(instr, block) {
446 nir_emit_instr(instr);
447 }
448 }
449
450 void
451 fs_visitor::nir_emit_instr(nir_instr *instr)
452 {
453 const fs_builder abld = bld.annotate(NULL, instr);
454
455 switch (instr->type) {
456 case nir_instr_type_alu:
457 nir_emit_alu(abld, nir_instr_as_alu(instr), true);
458 break;
459
460 case nir_instr_type_deref:
461 unreachable("All derefs should've been lowered");
462 break;
463
464 case nir_instr_type_intrinsic:
465 switch (stage) {
466 case MESA_SHADER_VERTEX:
467 nir_emit_vs_intrinsic(abld, nir_instr_as_intrinsic(instr));
468 break;
469 case MESA_SHADER_TESS_CTRL:
470 nir_emit_tcs_intrinsic(abld, nir_instr_as_intrinsic(instr));
471 break;
472 case MESA_SHADER_TESS_EVAL:
473 nir_emit_tes_intrinsic(abld, nir_instr_as_intrinsic(instr));
474 break;
475 case MESA_SHADER_GEOMETRY:
476 nir_emit_gs_intrinsic(abld, nir_instr_as_intrinsic(instr));
477 break;
478 case MESA_SHADER_FRAGMENT:
479 nir_emit_fs_intrinsic(abld, nir_instr_as_intrinsic(instr));
480 break;
481 case MESA_SHADER_COMPUTE:
482 nir_emit_cs_intrinsic(abld, nir_instr_as_intrinsic(instr));
483 break;
484 default:
485 unreachable("unsupported shader stage");
486 }
487 break;
488
489 case nir_instr_type_tex:
490 nir_emit_texture(abld, nir_instr_as_tex(instr));
491 break;
492
493 case nir_instr_type_load_const:
494 nir_emit_load_const(abld, nir_instr_as_load_const(instr));
495 break;
496
497 case nir_instr_type_ssa_undef:
498 /* We create a new VGRF for undefs on every use (by handling
499 * them in get_nir_src()), rather than for each definition.
500 * This helps register coalescing eliminate MOVs from undef.
501 */
502 break;
503
504 case nir_instr_type_jump:
505 nir_emit_jump(abld, nir_instr_as_jump(instr));
506 break;
507
508 default:
509 unreachable("unknown instruction type");
510 }
511 }
512
513 /**
514 * Recognizes a parent instruction of nir_op_extract_* and changes the type to
515 * match instr.
516 */
517 bool
518 fs_visitor::optimize_extract_to_float(nir_alu_instr *instr,
519 const fs_reg &result)
520 {
521 if (!instr->src[0].src.is_ssa ||
522 !instr->src[0].src.ssa->parent_instr)
523 return false;
524
525 if (instr->src[0].src.ssa->parent_instr->type != nir_instr_type_alu)
526 return false;
527
528 nir_alu_instr *src0 =
529 nir_instr_as_alu(instr->src[0].src.ssa->parent_instr);
530
531 if (src0->op != nir_op_extract_u8 && src0->op != nir_op_extract_u16 &&
532 src0->op != nir_op_extract_i8 && src0->op != nir_op_extract_i16)
533 return false;
534
535 /* If either opcode has source modifiers, bail.
536 *
537 * TODO: We can potentially handle source modifiers if both of the opcodes
538 * we're combining are signed integers.
539 */
540 if (instr->src[0].abs || instr->src[0].negate ||
541 src0->src[0].abs || src0->src[0].negate)
542 return false;
543
544 unsigned element = nir_src_as_uint(src0->src[1].src);
545
546 /* Element type to extract.*/
547 const brw_reg_type type = brw_int_type(
548 src0->op == nir_op_extract_u16 || src0->op == nir_op_extract_i16 ? 2 : 1,
549 src0->op == nir_op_extract_i16 || src0->op == nir_op_extract_i8);
550
551 fs_reg op0 = get_nir_src(src0->src[0].src);
552 op0.type = brw_type_for_nir_type(devinfo,
553 (nir_alu_type)(nir_op_infos[src0->op].input_types[0] |
554 nir_src_bit_size(src0->src[0].src)));
555 op0 = offset(op0, bld, src0->src[0].swizzle[0]);
556
557 set_saturate(instr->dest.saturate,
558 bld.MOV(result, subscript(op0, type, element)));
559 return true;
560 }
561
562 bool
563 fs_visitor::optimize_frontfacing_ternary(nir_alu_instr *instr,
564 const fs_reg &result)
565 {
566 nir_intrinsic_instr *src0 = nir_src_as_intrinsic(instr->src[0].src);
567 if (src0 == NULL || src0->intrinsic != nir_intrinsic_load_front_face)
568 return false;
569
570 if (!nir_src_is_const(instr->src[1].src) ||
571 !nir_src_is_const(instr->src[2].src))
572 return false;
573
574 const float value1 = nir_src_as_float(instr->src[1].src);
575 const float value2 = nir_src_as_float(instr->src[2].src);
576 if (fabsf(value1) != 1.0f || fabsf(value2) != 1.0f)
577 return false;
578
579 /* nir_opt_algebraic should have gotten rid of bcsel(b, a, a) */
580 assert(value1 == -value2);
581
582 fs_reg tmp = vgrf(glsl_type::int_type);
583
584 if (devinfo->gen >= 12) {
585 /* Bit 15 of g1.1 is 0 if the polygon is front facing. */
586 fs_reg g1 = fs_reg(retype(brw_vec1_grf(1, 1), BRW_REGISTER_TYPE_W));
587
588 /* For (gl_FrontFacing ? 1.0 : -1.0), emit:
589 *
590 * or(8) tmp.1<2>W g0.0<0,1,0>W 0x00003f80W
591 * and(8) dst<1>D tmp<8,8,1>D 0xbf800000D
592 *
593 * and negate the result for (gl_FrontFacing ? -1.0 : 1.0).
594 */
595 bld.OR(subscript(tmp, BRW_REGISTER_TYPE_W, 1),
596 g1, brw_imm_uw(0x3f80));
597
598 if (value1 == -1.0f)
599 bld.MOV(tmp, negate(tmp));
600
601 } else if (devinfo->gen >= 6) {
602 /* Bit 15 of g0.0 is 0 if the polygon is front facing. */
603 fs_reg g0 = fs_reg(retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_W));
604
605 /* For (gl_FrontFacing ? 1.0 : -1.0), emit:
606 *
607 * or(8) tmp.1<2>W g0.0<0,1,0>W 0x00003f80W
608 * and(8) dst<1>D tmp<8,8,1>D 0xbf800000D
609 *
610 * and negate g0.0<0,1,0>W for (gl_FrontFacing ? -1.0 : 1.0).
611 *
612 * This negation looks like it's safe in practice, because bits 0:4 will
613 * surely be TRIANGLES
614 */
615
616 if (value1 == -1.0f) {
617 g0.negate = true;
618 }
619
620 bld.OR(subscript(tmp, BRW_REGISTER_TYPE_W, 1),
621 g0, brw_imm_uw(0x3f80));
622 } else {
623 /* Bit 31 of g1.6 is 0 if the polygon is front facing. */
624 fs_reg g1_6 = fs_reg(retype(brw_vec1_grf(1, 6), BRW_REGISTER_TYPE_D));
625
626 /* For (gl_FrontFacing ? 1.0 : -1.0), emit:
627 *
628 * or(8) tmp<1>D g1.6<0,1,0>D 0x3f800000D
629 * and(8) dst<1>D tmp<8,8,1>D 0xbf800000D
630 *
631 * and negate g1.6<0,1,0>D for (gl_FrontFacing ? -1.0 : 1.0).
632 *
633 * This negation looks like it's safe in practice, because bits 0:4 will
634 * surely be TRIANGLES
635 */
636
637 if (value1 == -1.0f) {
638 g1_6.negate = true;
639 }
640
641 bld.OR(tmp, g1_6, brw_imm_d(0x3f800000));
642 }
643 bld.AND(retype(result, BRW_REGISTER_TYPE_D), tmp, brw_imm_d(0xbf800000));
644
645 return true;
646 }
647
648 static void
649 emit_find_msb_using_lzd(const fs_builder &bld,
650 const fs_reg &result,
651 const fs_reg &src,
652 bool is_signed)
653 {
654 fs_inst *inst;
655 fs_reg temp = src;
656
657 if (is_signed) {
658 /* LZD of an absolute value source almost always does the right
659 * thing. There are two problem values:
660 *
661 * * 0x80000000. Since abs(0x80000000) == 0x80000000, LZD returns
662 * 0. However, findMSB(int(0x80000000)) == 30.
663 *
664 * * 0xffffffff. Since abs(0xffffffff) == 1, LZD returns
665 * 31. Section 8.8 (Integer Functions) of the GLSL 4.50 spec says:
666 *
667 * For a value of zero or negative one, -1 will be returned.
668 *
669 * * Negative powers of two. LZD(abs(-(1<<x))) returns x, but
670 * findMSB(-(1<<x)) should return x-1.
671 *
672 * For all negative number cases, including 0x80000000 and
673 * 0xffffffff, the correct value is obtained from LZD if instead of
674 * negating the (already negative) value the logical-not is used. A
675 * conditonal logical-not can be achieved in two instructions.
676 */
677 temp = bld.vgrf(BRW_REGISTER_TYPE_D);
678
679 bld.ASR(temp, src, brw_imm_d(31));
680 bld.XOR(temp, temp, src);
681 }
682
683 bld.LZD(retype(result, BRW_REGISTER_TYPE_UD),
684 retype(temp, BRW_REGISTER_TYPE_UD));
685
686 /* LZD counts from the MSB side, while GLSL's findMSB() wants the count
687 * from the LSB side. Subtract the result from 31 to convert the MSB
688 * count into an LSB count. If no bits are set, LZD will return 32.
689 * 31-32 = -1, which is exactly what findMSB() is supposed to return.
690 */
691 inst = bld.ADD(result, retype(result, BRW_REGISTER_TYPE_D), brw_imm_d(31));
692 inst->src[0].negate = true;
693 }
694
695 static brw_rnd_mode
696 brw_rnd_mode_from_nir_op (const nir_op op) {
697 switch (op) {
698 case nir_op_f2f16_rtz:
699 return BRW_RND_MODE_RTZ;
700 case nir_op_f2f16_rtne:
701 return BRW_RND_MODE_RTNE;
702 default:
703 unreachable("Operation doesn't support rounding mode");
704 }
705 }
706
707 static brw_rnd_mode
708 brw_rnd_mode_from_execution_mode(unsigned execution_mode)
709 {
710 if (nir_has_any_rounding_mode_rtne(execution_mode))
711 return BRW_RND_MODE_RTNE;
712 if (nir_has_any_rounding_mode_rtz(execution_mode))
713 return BRW_RND_MODE_RTZ;
714 return BRW_RND_MODE_UNSPECIFIED;
715 }
716
717 fs_reg
718 fs_visitor::prepare_alu_destination_and_sources(const fs_builder &bld,
719 nir_alu_instr *instr,
720 fs_reg *op,
721 bool need_dest)
722 {
723 fs_reg result =
724 need_dest ? get_nir_dest(instr->dest.dest) : bld.null_reg_ud();
725
726 result.type = brw_type_for_nir_type(devinfo,
727 (nir_alu_type)(nir_op_infos[instr->op].output_type |
728 nir_dest_bit_size(instr->dest.dest)));
729
730 for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
731 op[i] = get_nir_src(instr->src[i].src);
732 op[i].type = brw_type_for_nir_type(devinfo,
733 (nir_alu_type)(nir_op_infos[instr->op].input_types[i] |
734 nir_src_bit_size(instr->src[i].src)));
735 op[i].abs = instr->src[i].abs;
736 op[i].negate = instr->src[i].negate;
737 }
738
739 /* Move and vecN instrutions may still be vectored. Return the raw,
740 * vectored source and destination so that fs_visitor::nir_emit_alu can
741 * handle it. Other callers should not have to handle these kinds of
742 * instructions.
743 */
744 switch (instr->op) {
745 case nir_op_mov:
746 case nir_op_vec2:
747 case nir_op_vec3:
748 case nir_op_vec4:
749 return result;
750 default:
751 break;
752 }
753
754 /* At this point, we have dealt with any instruction that operates on
755 * more than a single channel. Therefore, we can just adjust the source
756 * and destination registers for that channel and emit the instruction.
757 */
758 unsigned channel = 0;
759 if (nir_op_infos[instr->op].output_size == 0) {
760 /* Since NIR is doing the scalarizing for us, we should only ever see
761 * vectorized operations with a single channel.
762 */
763 assert(util_bitcount(instr->dest.write_mask) == 1);
764 channel = ffs(instr->dest.write_mask) - 1;
765
766 result = offset(result, bld, channel);
767 }
768
769 for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
770 assert(nir_op_infos[instr->op].input_sizes[i] < 2);
771 op[i] = offset(op[i], bld, instr->src[i].swizzle[channel]);
772 }
773
774 return result;
775 }
776
777 void
778 fs_visitor::resolve_inot_sources(const fs_builder &bld, nir_alu_instr *instr,
779 fs_reg *op)
780 {
781 for (unsigned i = 0; i < 2; i++) {
782 nir_alu_instr *inot_instr = nir_src_as_alu_instr(instr->src[i].src);
783
784 if (inot_instr != NULL && inot_instr->op == nir_op_inot &&
785 !inot_instr->src[0].abs && !inot_instr->src[0].negate) {
786 /* The source of the inot is now the source of instr. */
787 prepare_alu_destination_and_sources(bld, inot_instr, &op[i], false);
788
789 assert(!op[i].negate);
790 op[i].negate = true;
791 } else {
792 op[i] = resolve_source_modifiers(op[i]);
793 }
794 }
795 }
796
797 bool
798 fs_visitor::try_emit_b2fi_of_inot(const fs_builder &bld,
799 fs_reg result,
800 nir_alu_instr *instr)
801 {
802 if (devinfo->gen < 6 || devinfo->gen >= 12)
803 return false;
804
805 nir_alu_instr *inot_instr = nir_src_as_alu_instr(instr->src[0].src);
806
807 if (inot_instr == NULL || inot_instr->op != nir_op_inot)
808 return false;
809
810 /* HF is also possible as a destination on BDW+. For nir_op_b2i, the set
811 * of valid size-changing combinations is a bit more complex.
812 *
813 * The source restriction is just because I was lazy about generating the
814 * constant below.
815 */
816 if (nir_dest_bit_size(instr->dest.dest) != 32 ||
817 nir_src_bit_size(inot_instr->src[0].src) != 32)
818 return false;
819
820 /* b2[fi](inot(a)) maps a=0 => 1, a=-1 => 0. Since a can only be 0 or -1,
821 * this is float(1 + a).
822 */
823 fs_reg op;
824
825 prepare_alu_destination_and_sources(bld, inot_instr, &op, false);
826
827 /* Ignore the saturate modifier, if there is one. The result of the
828 * arithmetic can only be 0 or 1, so the clamping will do nothing anyway.
829 */
830 bld.ADD(result, op, brw_imm_d(1));
831
832 return true;
833 }
834
835 /**
836 * Emit code for nir_op_fsign possibly fused with a nir_op_fmul
837 *
838 * If \c instr is not the \c nir_op_fsign, then \c fsign_src is the index of
839 * the source of \c instr that is a \c nir_op_fsign.
840 */
841 void
842 fs_visitor::emit_fsign(const fs_builder &bld, const nir_alu_instr *instr,
843 fs_reg result, fs_reg *op, unsigned fsign_src)
844 {
845 fs_inst *inst;
846
847 assert(instr->op == nir_op_fsign || instr->op == nir_op_fmul);
848 assert(fsign_src < nir_op_infos[instr->op].num_inputs);
849
850 if (instr->op != nir_op_fsign) {
851 const nir_alu_instr *const fsign_instr =
852 nir_src_as_alu_instr(instr->src[fsign_src].src);
853
854 assert(!fsign_instr->dest.saturate);
855
856 /* op[fsign_src] has the nominal result of the fsign, and op[1 -
857 * fsign_src] has the other multiply source. This must be rearranged so
858 * that op[0] is the source of the fsign op[1] is the other multiply
859 * source.
860 */
861 if (fsign_src != 0)
862 op[1] = op[0];
863
864 op[0] = get_nir_src(fsign_instr->src[0].src);
865
866 const nir_alu_type t =
867 (nir_alu_type)(nir_op_infos[instr->op].input_types[0] |
868 nir_src_bit_size(fsign_instr->src[0].src));
869
870 op[0].type = brw_type_for_nir_type(devinfo, t);
871 op[0].abs = fsign_instr->src[0].abs;
872 op[0].negate = fsign_instr->src[0].negate;
873
874 unsigned channel = 0;
875 if (nir_op_infos[instr->op].output_size == 0) {
876 /* Since NIR is doing the scalarizing for us, we should only ever see
877 * vectorized operations with a single channel.
878 */
879 assert(util_bitcount(instr->dest.write_mask) == 1);
880 channel = ffs(instr->dest.write_mask) - 1;
881 }
882
883 op[0] = offset(op[0], bld, fsign_instr->src[0].swizzle[channel]);
884
885 /* Resolve any source modifiers. We could do slightly better on Gen8+
886 * if the only source modifier is negation, but *shrug*.
887 */
888 if (op[1].negate || op[1].abs) {
889 fs_reg tmp = bld.vgrf(op[1].type);
890
891 bld.MOV(tmp, op[1]);
892 op[1] = tmp;
893 }
894 } else {
895 assert(!instr->dest.saturate);
896 }
897
898 if (op[0].abs) {
899 /* Straightforward since the source can be assumed to be either strictly
900 * >= 0 or strictly <= 0 depending on the setting of the negate flag.
901 */
902 set_condmod(BRW_CONDITIONAL_NZ, bld.MOV(result, op[0]));
903
904 if (instr->op == nir_op_fsign) {
905 inst = (op[0].negate)
906 ? bld.MOV(result, brw_imm_f(-1.0f))
907 : bld.MOV(result, brw_imm_f(1.0f));
908 } else {
909 op[1].negate = (op[0].negate != op[1].negate);
910 inst = bld.MOV(result, op[1]);
911 }
912
913 set_predicate(BRW_PREDICATE_NORMAL, inst);
914 } else if (type_sz(op[0].type) == 2) {
915 /* AND(val, 0x8000) gives the sign bit.
916 *
917 * Predicated OR ORs 1.0 (0x3c00) with the sign bit if val is not zero.
918 */
919 fs_reg zero = retype(brw_imm_uw(0), BRW_REGISTER_TYPE_HF);
920 bld.CMP(bld.null_reg_f(), op[0], zero, BRW_CONDITIONAL_NZ);
921
922 op[0].type = BRW_REGISTER_TYPE_UW;
923 result.type = BRW_REGISTER_TYPE_UW;
924 bld.AND(result, op[0], brw_imm_uw(0x8000u));
925
926 if (instr->op == nir_op_fsign)
927 inst = bld.OR(result, result, brw_imm_uw(0x3c00u));
928 else {
929 /* Use XOR here to get the result sign correct. */
930 inst = bld.XOR(result, result, retype(op[1], BRW_REGISTER_TYPE_UW));
931 }
932
933 inst->predicate = BRW_PREDICATE_NORMAL;
934 } else if (type_sz(op[0].type) == 4) {
935 /* AND(val, 0x80000000) gives the sign bit.
936 *
937 * Predicated OR ORs 1.0 (0x3f800000) with the sign bit if val is not
938 * zero.
939 */
940 bld.CMP(bld.null_reg_f(), op[0], brw_imm_f(0.0f), BRW_CONDITIONAL_NZ);
941
942 op[0].type = BRW_REGISTER_TYPE_UD;
943 result.type = BRW_REGISTER_TYPE_UD;
944 bld.AND(result, op[0], brw_imm_ud(0x80000000u));
945
946 if (instr->op == nir_op_fsign)
947 inst = bld.OR(result, result, brw_imm_ud(0x3f800000u));
948 else {
949 /* Use XOR here to get the result sign correct. */
950 inst = bld.XOR(result, result, retype(op[1], BRW_REGISTER_TYPE_UD));
951 }
952
953 inst->predicate = BRW_PREDICATE_NORMAL;
954 } else {
955 /* For doubles we do the same but we need to consider:
956 *
957 * - 2-src instructions can't operate with 64-bit immediates
958 * - The sign is encoded in the high 32-bit of each DF
959 * - We need to produce a DF result.
960 */
961
962 fs_reg zero = vgrf(glsl_type::double_type);
963 bld.MOV(zero, setup_imm_df(bld, 0.0));
964 bld.CMP(bld.null_reg_df(), op[0], zero, BRW_CONDITIONAL_NZ);
965
966 bld.MOV(result, zero);
967
968 fs_reg r = subscript(result, BRW_REGISTER_TYPE_UD, 1);
969 bld.AND(r, subscript(op[0], BRW_REGISTER_TYPE_UD, 1),
970 brw_imm_ud(0x80000000u));
971
972 if (instr->op == nir_op_fsign) {
973 set_predicate(BRW_PREDICATE_NORMAL,
974 bld.OR(r, r, brw_imm_ud(0x3ff00000u)));
975 } else {
976 /* This could be done better in some cases. If the scale is an
977 * immediate with the low 32-bits all 0, emitting a separate XOR and
978 * OR would allow an algebraic optimization to remove the OR. There
979 * are currently zero instances of fsign(double(x))*IMM in shader-db
980 * or any test suite, so it is hard to care at this time.
981 */
982 fs_reg result_int64 = retype(result, BRW_REGISTER_TYPE_UQ);
983 inst = bld.XOR(result_int64, result_int64,
984 retype(op[1], BRW_REGISTER_TYPE_UQ));
985 }
986 }
987 }
988
989 /**
990 * Deteremine whether sources of a nir_op_fmul can be fused with a nir_op_fsign
991 *
992 * Checks the operands of a \c nir_op_fmul to determine whether or not
993 * \c emit_fsign could fuse the multiplication with the \c sign() calculation.
994 *
995 * \param instr The multiplication instruction
996 *
997 * \param fsign_src The source of \c instr that may or may not be a
998 * \c nir_op_fsign
999 */
1000 static bool
1001 can_fuse_fmul_fsign(nir_alu_instr *instr, unsigned fsign_src)
1002 {
1003 assert(instr->op == nir_op_fmul);
1004
1005 nir_alu_instr *const fsign_instr =
1006 nir_src_as_alu_instr(instr->src[fsign_src].src);
1007
1008 /* Rules:
1009 *
1010 * 1. instr->src[fsign_src] must be a nir_op_fsign.
1011 * 2. The nir_op_fsign can only be used by this multiplication.
1012 * 3. The source that is the nir_op_fsign does not have source modifiers.
1013 * \c emit_fsign only examines the source modifiers of the source of the
1014 * \c nir_op_fsign.
1015 *
1016 * The nir_op_fsign must also not have the saturate modifier, but steps
1017 * have already been taken (in nir_opt_algebraic) to ensure that.
1018 */
1019 return fsign_instr != NULL && fsign_instr->op == nir_op_fsign &&
1020 is_used_once(fsign_instr) &&
1021 !instr->src[fsign_src].abs && !instr->src[fsign_src].negate;
1022 }
1023
1024 void
1025 fs_visitor::nir_emit_alu(const fs_builder &bld, nir_alu_instr *instr,
1026 bool need_dest)
1027 {
1028 struct brw_wm_prog_key *fs_key = (struct brw_wm_prog_key *) this->key;
1029 fs_inst *inst;
1030 unsigned execution_mode =
1031 bld.shader->nir->info.float_controls_execution_mode;
1032
1033 fs_reg op[4];
1034 fs_reg result = prepare_alu_destination_and_sources(bld, instr, op, need_dest);
1035
1036 switch (instr->op) {
1037 case nir_op_mov:
1038 case nir_op_vec2:
1039 case nir_op_vec3:
1040 case nir_op_vec4: {
1041 fs_reg temp = result;
1042 bool need_extra_copy = false;
1043 for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
1044 if (!instr->src[i].src.is_ssa &&
1045 instr->dest.dest.reg.reg == instr->src[i].src.reg.reg) {
1046 need_extra_copy = true;
1047 temp = bld.vgrf(result.type, 4);
1048 break;
1049 }
1050 }
1051
1052 for (unsigned i = 0; i < 4; i++) {
1053 if (!(instr->dest.write_mask & (1 << i)))
1054 continue;
1055
1056 if (instr->op == nir_op_mov) {
1057 inst = bld.MOV(offset(temp, bld, i),
1058 offset(op[0], bld, instr->src[0].swizzle[i]));
1059 } else {
1060 inst = bld.MOV(offset(temp, bld, i),
1061 offset(op[i], bld, instr->src[i].swizzle[0]));
1062 }
1063 inst->saturate = instr->dest.saturate;
1064 }
1065
1066 /* In this case the source and destination registers were the same,
1067 * so we need to insert an extra set of moves in order to deal with
1068 * any swizzling.
1069 */
1070 if (need_extra_copy) {
1071 for (unsigned i = 0; i < 4; i++) {
1072 if (!(instr->dest.write_mask & (1 << i)))
1073 continue;
1074
1075 bld.MOV(offset(result, bld, i), offset(temp, bld, i));
1076 }
1077 }
1078 return;
1079 }
1080
1081 case nir_op_i2f32:
1082 case nir_op_u2f32:
1083 if (optimize_extract_to_float(instr, result))
1084 return;
1085 inst = bld.MOV(result, op[0]);
1086 inst->saturate = instr->dest.saturate;
1087 break;
1088
1089 case nir_op_f2f16_rtne:
1090 case nir_op_f2f16_rtz:
1091 case nir_op_f2f16: {
1092 brw_rnd_mode rnd = BRW_RND_MODE_UNSPECIFIED;
1093
1094 if (nir_op_f2f16 == instr->op)
1095 rnd = brw_rnd_mode_from_execution_mode(execution_mode);
1096 else
1097 rnd = brw_rnd_mode_from_nir_op(instr->op);
1098
1099 if (BRW_RND_MODE_UNSPECIFIED != rnd)
1100 bld.emit(SHADER_OPCODE_RND_MODE, bld.null_reg_ud(), brw_imm_d(rnd));
1101
1102 /* In theory, it would be better to use BRW_OPCODE_F32TO16. Depending
1103 * on the HW gen, it is a special hw opcode or just a MOV, and
1104 * brw_F32TO16 (at brw_eu_emit) would do the work to chose.
1105 *
1106 * But if we want to use that opcode, we need to provide support on
1107 * different optimizations and lowerings. As right now HF support is
1108 * only for gen8+, it will be better to use directly the MOV, and use
1109 * BRW_OPCODE_F32TO16 when/if we work for HF support on gen7.
1110 */
1111 assert(type_sz(op[0].type) < 8); /* brw_nir_lower_conversions */
1112 inst = bld.MOV(result, op[0]);
1113 inst->saturate = instr->dest.saturate;
1114 break;
1115 }
1116
1117 case nir_op_b2i8:
1118 case nir_op_b2i16:
1119 case nir_op_b2i32:
1120 case nir_op_b2i64:
1121 case nir_op_b2f16:
1122 case nir_op_b2f32:
1123 case nir_op_b2f64:
1124 if (try_emit_b2fi_of_inot(bld, result, instr))
1125 break;
1126 op[0].type = BRW_REGISTER_TYPE_D;
1127 op[0].negate = !op[0].negate;
1128 /* fallthrough */
1129 case nir_op_i2f64:
1130 case nir_op_i2i64:
1131 case nir_op_u2f64:
1132 case nir_op_u2u64:
1133 case nir_op_f2f64:
1134 case nir_op_f2i64:
1135 case nir_op_f2u64:
1136 case nir_op_i2i32:
1137 case nir_op_u2u32:
1138 case nir_op_f2i32:
1139 case nir_op_f2u32:
1140 case nir_op_i2f16:
1141 case nir_op_i2i16:
1142 case nir_op_u2f16:
1143 case nir_op_u2u16:
1144 case nir_op_f2i16:
1145 case nir_op_f2u16:
1146 case nir_op_i2i8:
1147 case nir_op_u2u8:
1148 case nir_op_f2i8:
1149 case nir_op_f2u8:
1150 if (result.type == BRW_REGISTER_TYPE_B ||
1151 result.type == BRW_REGISTER_TYPE_UB ||
1152 result.type == BRW_REGISTER_TYPE_HF)
1153 assert(type_sz(op[0].type) < 8); /* brw_nir_lower_conversions */
1154
1155 if (op[0].type == BRW_REGISTER_TYPE_B ||
1156 op[0].type == BRW_REGISTER_TYPE_UB ||
1157 op[0].type == BRW_REGISTER_TYPE_HF)
1158 assert(type_sz(result.type) < 8); /* brw_nir_lower_conversions */
1159
1160 inst = bld.MOV(result, op[0]);
1161 inst->saturate = instr->dest.saturate;
1162 break;
1163
1164 case nir_op_fsat:
1165 inst = bld.MOV(result, op[0]);
1166 inst->saturate = true;
1167 break;
1168
1169 case nir_op_fneg:
1170 case nir_op_ineg:
1171 op[0].negate = true;
1172 inst = bld.MOV(result, op[0]);
1173 if (instr->op == nir_op_fneg)
1174 inst->saturate = instr->dest.saturate;
1175 break;
1176
1177 case nir_op_fabs:
1178 case nir_op_iabs:
1179 op[0].negate = false;
1180 op[0].abs = true;
1181 inst = bld.MOV(result, op[0]);
1182 if (instr->op == nir_op_fabs)
1183 inst->saturate = instr->dest.saturate;
1184 break;
1185
1186 case nir_op_f2f32:
1187 if (nir_has_any_rounding_mode_enabled(execution_mode)) {
1188 brw_rnd_mode rnd =
1189 brw_rnd_mode_from_execution_mode(execution_mode);
1190 bld.emit(SHADER_OPCODE_RND_MODE, bld.null_reg_ud(),
1191 brw_imm_d(rnd));
1192 }
1193
1194 if (op[0].type == BRW_REGISTER_TYPE_HF)
1195 assert(type_sz(result.type) < 8); /* brw_nir_lower_conversions */
1196
1197 inst = bld.MOV(result, op[0]);
1198 inst->saturate = instr->dest.saturate;
1199 break;
1200
1201 case nir_op_fsign:
1202 emit_fsign(bld, instr, result, op, 0);
1203 break;
1204
1205 case nir_op_frcp:
1206 inst = bld.emit(SHADER_OPCODE_RCP, result, op[0]);
1207 inst->saturate = instr->dest.saturate;
1208 break;
1209
1210 case nir_op_fexp2:
1211 inst = bld.emit(SHADER_OPCODE_EXP2, result, op[0]);
1212 inst->saturate = instr->dest.saturate;
1213 break;
1214
1215 case nir_op_flog2:
1216 inst = bld.emit(SHADER_OPCODE_LOG2, result, op[0]);
1217 inst->saturate = instr->dest.saturate;
1218 break;
1219
1220 case nir_op_fsin:
1221 inst = bld.emit(SHADER_OPCODE_SIN, result, op[0]);
1222 inst->saturate = instr->dest.saturate;
1223 break;
1224
1225 case nir_op_fcos:
1226 inst = bld.emit(SHADER_OPCODE_COS, result, op[0]);
1227 inst->saturate = instr->dest.saturate;
1228 break;
1229
1230 case nir_op_fddx:
1231 if (fs_key->high_quality_derivatives) {
1232 inst = bld.emit(FS_OPCODE_DDX_FINE, result, op[0]);
1233 } else {
1234 inst = bld.emit(FS_OPCODE_DDX_COARSE, result, op[0]);
1235 }
1236 inst->saturate = instr->dest.saturate;
1237 break;
1238 case nir_op_fddx_fine:
1239 inst = bld.emit(FS_OPCODE_DDX_FINE, result, op[0]);
1240 inst->saturate = instr->dest.saturate;
1241 break;
1242 case nir_op_fddx_coarse:
1243 inst = bld.emit(FS_OPCODE_DDX_COARSE, result, op[0]);
1244 inst->saturate = instr->dest.saturate;
1245 break;
1246 case nir_op_fddy:
1247 if (fs_key->high_quality_derivatives) {
1248 inst = bld.emit(FS_OPCODE_DDY_FINE, result, op[0]);
1249 } else {
1250 inst = bld.emit(FS_OPCODE_DDY_COARSE, result, op[0]);
1251 }
1252 inst->saturate = instr->dest.saturate;
1253 break;
1254 case nir_op_fddy_fine:
1255 inst = bld.emit(FS_OPCODE_DDY_FINE, result, op[0]);
1256 inst->saturate = instr->dest.saturate;
1257 break;
1258 case nir_op_fddy_coarse:
1259 inst = bld.emit(FS_OPCODE_DDY_COARSE, result, op[0]);
1260 inst->saturate = instr->dest.saturate;
1261 break;
1262
1263 case nir_op_fadd:
1264 if (nir_has_any_rounding_mode_enabled(execution_mode)) {
1265 brw_rnd_mode rnd =
1266 brw_rnd_mode_from_execution_mode(execution_mode);
1267 bld.emit(SHADER_OPCODE_RND_MODE, bld.null_reg_ud(),
1268 brw_imm_d(rnd));
1269 }
1270 /* fallthrough */
1271 case nir_op_iadd:
1272 inst = bld.ADD(result, op[0], op[1]);
1273 inst->saturate = instr->dest.saturate;
1274 break;
1275
1276 case nir_op_iadd_sat:
1277 case nir_op_uadd_sat:
1278 inst = bld.ADD(result, op[0], op[1]);
1279 inst->saturate = true;
1280 break;
1281
1282 case nir_op_isub_sat:
1283 bld.emit(SHADER_OPCODE_ISUB_SAT, result, op[0], op[1]);
1284 break;
1285
1286 case nir_op_usub_sat:
1287 bld.emit(SHADER_OPCODE_USUB_SAT, result, op[0], op[1]);
1288 break;
1289
1290 case nir_op_irhadd:
1291 case nir_op_urhadd:
1292 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1293 inst = bld.AVG(result, op[0], op[1]);
1294 break;
1295
1296 case nir_op_ihadd:
1297 case nir_op_uhadd: {
1298 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1299 fs_reg tmp = bld.vgrf(result.type);
1300
1301 if (devinfo->gen >= 8) {
1302 op[0] = resolve_source_modifiers(op[0]);
1303 op[1] = resolve_source_modifiers(op[1]);
1304 }
1305
1306 /* AVG(x, y) - ((x ^ y) & 1) */
1307 bld.XOR(tmp, op[0], op[1]);
1308 bld.AND(tmp, tmp, retype(brw_imm_ud(1), result.type));
1309 bld.AVG(result, op[0], op[1]);
1310 inst = bld.ADD(result, result, tmp);
1311 inst->src[1].negate = true;
1312 break;
1313 }
1314
1315 case nir_op_fmul:
1316 for (unsigned i = 0; i < 2; i++) {
1317 if (can_fuse_fmul_fsign(instr, i)) {
1318 emit_fsign(bld, instr, result, op, i);
1319 return;
1320 }
1321 }
1322
1323 /* We emit the rounding mode after the previous fsign optimization since
1324 * it won't result in a MUL, but will try to negate the value by other
1325 * means.
1326 */
1327 if (nir_has_any_rounding_mode_enabled(execution_mode)) {
1328 brw_rnd_mode rnd =
1329 brw_rnd_mode_from_execution_mode(execution_mode);
1330 bld.emit(SHADER_OPCODE_RND_MODE, bld.null_reg_ud(),
1331 brw_imm_d(rnd));
1332 }
1333
1334 inst = bld.MUL(result, op[0], op[1]);
1335 inst->saturate = instr->dest.saturate;
1336 break;
1337
1338 case nir_op_imul_2x32_64:
1339 case nir_op_umul_2x32_64:
1340 bld.MUL(result, op[0], op[1]);
1341 break;
1342
1343 case nir_op_imul_32x16:
1344 case nir_op_umul_32x16: {
1345 const bool ud = instr->op == nir_op_umul_32x16;
1346
1347 assert(nir_dest_bit_size(instr->dest.dest) == 32);
1348
1349 /* Before Gen7, the order of the 32-bit source and the 16-bit source was
1350 * swapped. The extension isn't enabled on those platforms, so don't
1351 * pretend to support the differences.
1352 */
1353 assert(devinfo->gen >= 7);
1354
1355 if (op[1].file == IMM)
1356 op[1] = ud ? brw_imm_uw(op[1].ud) : brw_imm_w(op[1].d);
1357 else {
1358 const enum brw_reg_type word_type =
1359 ud ? BRW_REGISTER_TYPE_UW : BRW_REGISTER_TYPE_W;
1360
1361 op[1] = subscript(op[1], word_type, 0);
1362 }
1363
1364 const enum brw_reg_type dword_type =
1365 ud ? BRW_REGISTER_TYPE_UD : BRW_REGISTER_TYPE_D;
1366
1367 bld.MUL(result, retype(op[0], dword_type), op[1]);
1368 break;
1369 }
1370
1371 case nir_op_imul:
1372 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1373 bld.MUL(result, op[0], op[1]);
1374 break;
1375
1376 case nir_op_imul_high:
1377 case nir_op_umul_high:
1378 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1379 bld.emit(SHADER_OPCODE_MULH, result, op[0], op[1]);
1380 break;
1381
1382 case nir_op_idiv:
1383 case nir_op_udiv:
1384 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1385 bld.emit(SHADER_OPCODE_INT_QUOTIENT, result, op[0], op[1]);
1386 break;
1387
1388 case nir_op_uadd_carry:
1389 unreachable("Should have been lowered by carry_to_arith().");
1390
1391 case nir_op_usub_borrow:
1392 unreachable("Should have been lowered by borrow_to_arith().");
1393
1394 case nir_op_umod:
1395 case nir_op_irem:
1396 /* According to the sign table for INT DIV in the Ivy Bridge PRM, it
1397 * appears that our hardware just does the right thing for signed
1398 * remainder.
1399 */
1400 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1401 bld.emit(SHADER_OPCODE_INT_REMAINDER, result, op[0], op[1]);
1402 break;
1403
1404 case nir_op_imod: {
1405 /* Get a regular C-style remainder. If a % b == 0, set the predicate. */
1406 bld.emit(SHADER_OPCODE_INT_REMAINDER, result, op[0], op[1]);
1407
1408 /* Math instructions don't support conditional mod */
1409 inst = bld.MOV(bld.null_reg_d(), result);
1410 inst->conditional_mod = BRW_CONDITIONAL_NZ;
1411
1412 /* Now, we need to determine if signs of the sources are different.
1413 * When we XOR the sources, the top bit is 0 if they are the same and 1
1414 * if they are different. We can then use a conditional modifier to
1415 * turn that into a predicate. This leads us to an XOR.l instruction.
1416 *
1417 * Technically, according to the PRM, you're not allowed to use .l on a
1418 * XOR instruction. However, emperical experiments and Curro's reading
1419 * of the simulator source both indicate that it's safe.
1420 */
1421 fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_D);
1422 inst = bld.XOR(tmp, op[0], op[1]);
1423 inst->predicate = BRW_PREDICATE_NORMAL;
1424 inst->conditional_mod = BRW_CONDITIONAL_L;
1425
1426 /* If the result of the initial remainder operation is non-zero and the
1427 * two sources have different signs, add in a copy of op[1] to get the
1428 * final integer modulus value.
1429 */
1430 inst = bld.ADD(result, result, op[1]);
1431 inst->predicate = BRW_PREDICATE_NORMAL;
1432 break;
1433 }
1434
1435 case nir_op_flt32:
1436 case nir_op_fge32:
1437 case nir_op_feq32:
1438 case nir_op_fne32: {
1439 fs_reg dest = result;
1440
1441 const uint32_t bit_size = nir_src_bit_size(instr->src[0].src);
1442 if (bit_size != 32)
1443 dest = bld.vgrf(op[0].type, 1);
1444
1445 bld.CMP(dest, op[0], op[1], brw_cmod_for_nir_comparison(instr->op));
1446
1447 if (bit_size > 32) {
1448 bld.MOV(result, subscript(dest, BRW_REGISTER_TYPE_UD, 0));
1449 } else if(bit_size < 32) {
1450 /* When we convert the result to 32-bit we need to be careful and do
1451 * it as a signed conversion to get sign extension (for 32-bit true)
1452 */
1453 const brw_reg_type src_type =
1454 brw_reg_type_from_bit_size(bit_size, BRW_REGISTER_TYPE_D);
1455
1456 bld.MOV(retype(result, BRW_REGISTER_TYPE_D), retype(dest, src_type));
1457 }
1458 break;
1459 }
1460
1461 case nir_op_ilt32:
1462 case nir_op_ult32:
1463 case nir_op_ige32:
1464 case nir_op_uge32:
1465 case nir_op_ieq32:
1466 case nir_op_ine32: {
1467 fs_reg dest = result;
1468
1469 /* On Gen11 we have an additional issue being that src1 cannot be a byte
1470 * type. So we convert both operands for the comparison.
1471 */
1472 fs_reg temp_op[2];
1473 temp_op[0] = bld.fix_byte_src(op[0]);
1474 temp_op[1] = bld.fix_byte_src(op[1]);
1475
1476 const uint32_t bit_size = type_sz(temp_op[0].type) * 8;
1477 if (bit_size != 32)
1478 dest = bld.vgrf(temp_op[0].type, 1);
1479
1480 bld.CMP(dest, temp_op[0], temp_op[1],
1481 brw_cmod_for_nir_comparison(instr->op));
1482
1483 if (bit_size > 32) {
1484 bld.MOV(result, subscript(dest, BRW_REGISTER_TYPE_UD, 0));
1485 } else if (bit_size < 32) {
1486 /* When we convert the result to 32-bit we need to be careful and do
1487 * it as a signed conversion to get sign extension (for 32-bit true)
1488 */
1489 const brw_reg_type src_type =
1490 brw_reg_type_from_bit_size(bit_size, BRW_REGISTER_TYPE_D);
1491
1492 bld.MOV(retype(result, BRW_REGISTER_TYPE_D), retype(dest, src_type));
1493 }
1494 break;
1495 }
1496
1497 case nir_op_inot:
1498 if (devinfo->gen >= 8) {
1499 nir_alu_instr *inot_src_instr = nir_src_as_alu_instr(instr->src[0].src);
1500
1501 if (inot_src_instr != NULL &&
1502 (inot_src_instr->op == nir_op_ior ||
1503 inot_src_instr->op == nir_op_ixor ||
1504 inot_src_instr->op == nir_op_iand) &&
1505 !inot_src_instr->src[0].abs &&
1506 !inot_src_instr->src[0].negate &&
1507 !inot_src_instr->src[1].abs &&
1508 !inot_src_instr->src[1].negate) {
1509 /* The sources of the source logical instruction are now the
1510 * sources of the instruction that will be generated.
1511 */
1512 prepare_alu_destination_and_sources(bld, inot_src_instr, op, false);
1513 resolve_inot_sources(bld, inot_src_instr, op);
1514
1515 /* Smash all of the sources and destination to be signed. This
1516 * doesn't matter for the operation of the instruction, but cmod
1517 * propagation fails on unsigned sources with negation (due to
1518 * fs_inst::can_do_cmod returning false).
1519 */
1520 result.type =
1521 brw_type_for_nir_type(devinfo,
1522 (nir_alu_type)(nir_type_int |
1523 nir_dest_bit_size(instr->dest.dest)));
1524 op[0].type =
1525 brw_type_for_nir_type(devinfo,
1526 (nir_alu_type)(nir_type_int |
1527 nir_src_bit_size(inot_src_instr->src[0].src)));
1528 op[1].type =
1529 brw_type_for_nir_type(devinfo,
1530 (nir_alu_type)(nir_type_int |
1531 nir_src_bit_size(inot_src_instr->src[1].src)));
1532
1533 /* For XOR, only invert one of the sources. Arbitrarily choose
1534 * the first source.
1535 */
1536 op[0].negate = !op[0].negate;
1537 if (inot_src_instr->op != nir_op_ixor)
1538 op[1].negate = !op[1].negate;
1539
1540 switch (inot_src_instr->op) {
1541 case nir_op_ior:
1542 bld.AND(result, op[0], op[1]);
1543 return;
1544
1545 case nir_op_iand:
1546 bld.OR(result, op[0], op[1]);
1547 return;
1548
1549 case nir_op_ixor:
1550 bld.XOR(result, op[0], op[1]);
1551 return;
1552
1553 default:
1554 unreachable("impossible opcode");
1555 }
1556 }
1557 op[0] = resolve_source_modifiers(op[0]);
1558 }
1559 bld.NOT(result, op[0]);
1560 break;
1561 case nir_op_ixor:
1562 if (devinfo->gen >= 8) {
1563 resolve_inot_sources(bld, instr, op);
1564 }
1565 bld.XOR(result, op[0], op[1]);
1566 break;
1567 case nir_op_ior:
1568 if (devinfo->gen >= 8) {
1569 resolve_inot_sources(bld, instr, op);
1570 }
1571 bld.OR(result, op[0], op[1]);
1572 break;
1573 case nir_op_iand:
1574 if (devinfo->gen >= 8) {
1575 resolve_inot_sources(bld, instr, op);
1576 }
1577 bld.AND(result, op[0], op[1]);
1578 break;
1579
1580 case nir_op_fdot2:
1581 case nir_op_fdot3:
1582 case nir_op_fdot4:
1583 case nir_op_b32all_fequal2:
1584 case nir_op_b32all_iequal2:
1585 case nir_op_b32all_fequal3:
1586 case nir_op_b32all_iequal3:
1587 case nir_op_b32all_fequal4:
1588 case nir_op_b32all_iequal4:
1589 case nir_op_b32any_fnequal2:
1590 case nir_op_b32any_inequal2:
1591 case nir_op_b32any_fnequal3:
1592 case nir_op_b32any_inequal3:
1593 case nir_op_b32any_fnequal4:
1594 case nir_op_b32any_inequal4:
1595 unreachable("Lowered by nir_lower_alu_reductions");
1596
1597 case nir_op_fnoise1_1:
1598 case nir_op_fnoise1_2:
1599 case nir_op_fnoise1_3:
1600 case nir_op_fnoise1_4:
1601 case nir_op_fnoise2_1:
1602 case nir_op_fnoise2_2:
1603 case nir_op_fnoise2_3:
1604 case nir_op_fnoise2_4:
1605 case nir_op_fnoise3_1:
1606 case nir_op_fnoise3_2:
1607 case nir_op_fnoise3_3:
1608 case nir_op_fnoise3_4:
1609 case nir_op_fnoise4_1:
1610 case nir_op_fnoise4_2:
1611 case nir_op_fnoise4_3:
1612 case nir_op_fnoise4_4:
1613 unreachable("not reached: should be handled by lower_noise");
1614
1615 case nir_op_ldexp:
1616 unreachable("not reached: should be handled by ldexp_to_arith()");
1617
1618 case nir_op_fsqrt:
1619 inst = bld.emit(SHADER_OPCODE_SQRT, result, op[0]);
1620 inst->saturate = instr->dest.saturate;
1621 break;
1622
1623 case nir_op_frsq:
1624 inst = bld.emit(SHADER_OPCODE_RSQ, result, op[0]);
1625 inst->saturate = instr->dest.saturate;
1626 break;
1627
1628 case nir_op_i2b32:
1629 case nir_op_f2b32: {
1630 uint32_t bit_size = nir_src_bit_size(instr->src[0].src);
1631 if (bit_size == 64) {
1632 /* two-argument instructions can't take 64-bit immediates */
1633 fs_reg zero;
1634 fs_reg tmp;
1635
1636 if (instr->op == nir_op_f2b32) {
1637 zero = vgrf(glsl_type::double_type);
1638 tmp = vgrf(glsl_type::double_type);
1639 bld.MOV(zero, setup_imm_df(bld, 0.0));
1640 } else {
1641 zero = vgrf(glsl_type::int64_t_type);
1642 tmp = vgrf(glsl_type::int64_t_type);
1643 bld.MOV(zero, brw_imm_q(0));
1644 }
1645
1646 /* A SIMD16 execution needs to be split in two instructions, so use
1647 * a vgrf instead of the flag register as dst so instruction splitting
1648 * works
1649 */
1650 bld.CMP(tmp, op[0], zero, BRW_CONDITIONAL_NZ);
1651 bld.MOV(result, subscript(tmp, BRW_REGISTER_TYPE_UD, 0));
1652 } else {
1653 fs_reg zero;
1654 if (bit_size == 32) {
1655 zero = instr->op == nir_op_f2b32 ? brw_imm_f(0.0f) : brw_imm_d(0);
1656 } else {
1657 assert(bit_size == 16);
1658 zero = instr->op == nir_op_f2b32 ?
1659 retype(brw_imm_w(0), BRW_REGISTER_TYPE_HF) : brw_imm_w(0);
1660 }
1661 bld.CMP(result, op[0], zero, BRW_CONDITIONAL_NZ);
1662 }
1663 break;
1664 }
1665
1666 case nir_op_ftrunc:
1667 inst = bld.RNDZ(result, op[0]);
1668 if (devinfo->gen < 6) {
1669 set_condmod(BRW_CONDITIONAL_R, inst);
1670 set_predicate(BRW_PREDICATE_NORMAL,
1671 bld.ADD(result, result, brw_imm_f(1.0f)));
1672 inst = bld.MOV(result, result); /* for potential saturation */
1673 }
1674 inst->saturate = instr->dest.saturate;
1675 break;
1676
1677 case nir_op_fceil: {
1678 op[0].negate = !op[0].negate;
1679 fs_reg temp = vgrf(glsl_type::float_type);
1680 bld.RNDD(temp, op[0]);
1681 temp.negate = true;
1682 inst = bld.MOV(result, temp);
1683 inst->saturate = instr->dest.saturate;
1684 break;
1685 }
1686 case nir_op_ffloor:
1687 inst = bld.RNDD(result, op[0]);
1688 inst->saturate = instr->dest.saturate;
1689 break;
1690 case nir_op_ffract:
1691 inst = bld.FRC(result, op[0]);
1692 inst->saturate = instr->dest.saturate;
1693 break;
1694 case nir_op_fround_even:
1695 inst = bld.RNDE(result, op[0]);
1696 if (devinfo->gen < 6) {
1697 set_condmod(BRW_CONDITIONAL_R, inst);
1698 set_predicate(BRW_PREDICATE_NORMAL,
1699 bld.ADD(result, result, brw_imm_f(1.0f)));
1700 inst = bld.MOV(result, result); /* for potential saturation */
1701 }
1702 inst->saturate = instr->dest.saturate;
1703 break;
1704
1705 case nir_op_fquantize2f16: {
1706 fs_reg tmp16 = bld.vgrf(BRW_REGISTER_TYPE_D);
1707 fs_reg tmp32 = bld.vgrf(BRW_REGISTER_TYPE_F);
1708 fs_reg zero = bld.vgrf(BRW_REGISTER_TYPE_F);
1709
1710 /* The destination stride must be at least as big as the source stride. */
1711 tmp16.type = BRW_REGISTER_TYPE_W;
1712 tmp16.stride = 2;
1713
1714 /* Check for denormal */
1715 fs_reg abs_src0 = op[0];
1716 abs_src0.abs = true;
1717 bld.CMP(bld.null_reg_f(), abs_src0, brw_imm_f(ldexpf(1.0, -14)),
1718 BRW_CONDITIONAL_L);
1719 /* Get the appropriately signed zero */
1720 bld.AND(retype(zero, BRW_REGISTER_TYPE_UD),
1721 retype(op[0], BRW_REGISTER_TYPE_UD),
1722 brw_imm_ud(0x80000000));
1723 /* Do the actual F32 -> F16 -> F32 conversion */
1724 bld.emit(BRW_OPCODE_F32TO16, tmp16, op[0]);
1725 bld.emit(BRW_OPCODE_F16TO32, tmp32, tmp16);
1726 /* Select that or zero based on normal status */
1727 inst = bld.SEL(result, zero, tmp32);
1728 inst->predicate = BRW_PREDICATE_NORMAL;
1729 inst->saturate = instr->dest.saturate;
1730 break;
1731 }
1732
1733 case nir_op_imin:
1734 case nir_op_umin:
1735 case nir_op_fmin:
1736 inst = bld.emit_minmax(result, op[0], op[1], BRW_CONDITIONAL_L);
1737 inst->saturate = instr->dest.saturate;
1738 break;
1739
1740 case nir_op_imax:
1741 case nir_op_umax:
1742 case nir_op_fmax:
1743 inst = bld.emit_minmax(result, op[0], op[1], BRW_CONDITIONAL_GE);
1744 inst->saturate = instr->dest.saturate;
1745 break;
1746
1747 case nir_op_pack_snorm_2x16:
1748 case nir_op_pack_snorm_4x8:
1749 case nir_op_pack_unorm_2x16:
1750 case nir_op_pack_unorm_4x8:
1751 case nir_op_unpack_snorm_2x16:
1752 case nir_op_unpack_snorm_4x8:
1753 case nir_op_unpack_unorm_2x16:
1754 case nir_op_unpack_unorm_4x8:
1755 case nir_op_unpack_half_2x16:
1756 case nir_op_pack_half_2x16:
1757 unreachable("not reached: should be handled by lower_packing_builtins");
1758
1759 case nir_op_unpack_half_2x16_split_x_flush_to_zero:
1760 assert(FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP16 & execution_mode);
1761 /* Fall-through */
1762 case nir_op_unpack_half_2x16_split_x:
1763 inst = bld.emit(BRW_OPCODE_F16TO32, result,
1764 subscript(op[0], BRW_REGISTER_TYPE_UW, 0));
1765 inst->saturate = instr->dest.saturate;
1766 break;
1767
1768 case nir_op_unpack_half_2x16_split_y_flush_to_zero:
1769 assert(FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP16 & execution_mode);
1770 /* Fall-through */
1771 case nir_op_unpack_half_2x16_split_y:
1772 inst = bld.emit(BRW_OPCODE_F16TO32, result,
1773 subscript(op[0], BRW_REGISTER_TYPE_UW, 1));
1774 inst->saturate = instr->dest.saturate;
1775 break;
1776
1777 case nir_op_pack_64_2x32_split:
1778 case nir_op_pack_32_2x16_split:
1779 bld.emit(FS_OPCODE_PACK, result, op[0], op[1]);
1780 break;
1781
1782 case nir_op_unpack_64_2x32_split_x:
1783 case nir_op_unpack_64_2x32_split_y: {
1784 if (instr->op == nir_op_unpack_64_2x32_split_x)
1785 bld.MOV(result, subscript(op[0], BRW_REGISTER_TYPE_UD, 0));
1786 else
1787 bld.MOV(result, subscript(op[0], BRW_REGISTER_TYPE_UD, 1));
1788 break;
1789 }
1790
1791 case nir_op_unpack_32_2x16_split_x:
1792 case nir_op_unpack_32_2x16_split_y: {
1793 if (instr->op == nir_op_unpack_32_2x16_split_x)
1794 bld.MOV(result, subscript(op[0], BRW_REGISTER_TYPE_UW, 0));
1795 else
1796 bld.MOV(result, subscript(op[0], BRW_REGISTER_TYPE_UW, 1));
1797 break;
1798 }
1799
1800 case nir_op_fpow:
1801 inst = bld.emit(SHADER_OPCODE_POW, result, op[0], op[1]);
1802 inst->saturate = instr->dest.saturate;
1803 break;
1804
1805 case nir_op_bitfield_reverse:
1806 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1807 bld.BFREV(result, op[0]);
1808 break;
1809
1810 case nir_op_bit_count:
1811 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1812 bld.CBIT(result, op[0]);
1813 break;
1814
1815 case nir_op_ufind_msb: {
1816 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1817 emit_find_msb_using_lzd(bld, result, op[0], false);
1818 break;
1819 }
1820
1821 case nir_op_uclz:
1822 assert(nir_dest_bit_size(instr->dest.dest) == 32);
1823 bld.LZD(retype(result, BRW_REGISTER_TYPE_UD), op[0]);
1824 break;
1825
1826 case nir_op_ifind_msb: {
1827 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1828
1829 if (devinfo->gen < 7) {
1830 emit_find_msb_using_lzd(bld, result, op[0], true);
1831 } else {
1832 bld.FBH(retype(result, BRW_REGISTER_TYPE_UD), op[0]);
1833
1834 /* FBH counts from the MSB side, while GLSL's findMSB() wants the
1835 * count from the LSB side. If FBH didn't return an error
1836 * (0xFFFFFFFF), then subtract the result from 31 to convert the MSB
1837 * count into an LSB count.
1838 */
1839 bld.CMP(bld.null_reg_d(), result, brw_imm_d(-1), BRW_CONDITIONAL_NZ);
1840
1841 inst = bld.ADD(result, result, brw_imm_d(31));
1842 inst->predicate = BRW_PREDICATE_NORMAL;
1843 inst->src[0].negate = true;
1844 }
1845 break;
1846 }
1847
1848 case nir_op_find_lsb:
1849 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1850
1851 if (devinfo->gen < 7) {
1852 fs_reg temp = vgrf(glsl_type::int_type);
1853
1854 /* (x & -x) generates a value that consists of only the LSB of x.
1855 * For all powers of 2, findMSB(y) == findLSB(y).
1856 */
1857 fs_reg src = retype(op[0], BRW_REGISTER_TYPE_D);
1858 fs_reg negated_src = src;
1859
1860 /* One must be negated, and the other must be non-negated. It
1861 * doesn't matter which is which.
1862 */
1863 negated_src.negate = true;
1864 src.negate = false;
1865
1866 bld.AND(temp, src, negated_src);
1867 emit_find_msb_using_lzd(bld, result, temp, false);
1868 } else {
1869 bld.FBL(result, op[0]);
1870 }
1871 break;
1872
1873 case nir_op_ubitfield_extract:
1874 case nir_op_ibitfield_extract:
1875 unreachable("should have been lowered");
1876 case nir_op_ubfe:
1877 case nir_op_ibfe:
1878 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1879 bld.BFE(result, op[2], op[1], op[0]);
1880 break;
1881 case nir_op_bfm:
1882 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1883 bld.BFI1(result, op[0], op[1]);
1884 break;
1885 case nir_op_bfi:
1886 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1887 bld.BFI2(result, op[0], op[1], op[2]);
1888 break;
1889
1890 case nir_op_bitfield_insert:
1891 unreachable("not reached: should have been lowered");
1892
1893 case nir_op_ishl:
1894 bld.SHL(result, op[0], op[1]);
1895 break;
1896 case nir_op_ishr:
1897 bld.ASR(result, op[0], op[1]);
1898 break;
1899 case nir_op_ushr:
1900 bld.SHR(result, op[0], op[1]);
1901 break;
1902
1903 case nir_op_urol:
1904 bld.ROL(result, op[0], op[1]);
1905 break;
1906 case nir_op_uror:
1907 bld.ROR(result, op[0], op[1]);
1908 break;
1909
1910 case nir_op_pack_half_2x16_split:
1911 bld.emit(FS_OPCODE_PACK_HALF_2x16_SPLIT, result, op[0], op[1]);
1912 break;
1913
1914 case nir_op_ffma:
1915 if (nir_has_any_rounding_mode_enabled(execution_mode)) {
1916 brw_rnd_mode rnd =
1917 brw_rnd_mode_from_execution_mode(execution_mode);
1918 bld.emit(SHADER_OPCODE_RND_MODE, bld.null_reg_ud(),
1919 brw_imm_d(rnd));
1920 }
1921
1922 inst = bld.MAD(result, op[2], op[1], op[0]);
1923 inst->saturate = instr->dest.saturate;
1924 break;
1925
1926 case nir_op_flrp:
1927 if (nir_has_any_rounding_mode_enabled(execution_mode)) {
1928 brw_rnd_mode rnd =
1929 brw_rnd_mode_from_execution_mode(execution_mode);
1930 bld.emit(SHADER_OPCODE_RND_MODE, bld.null_reg_ud(),
1931 brw_imm_d(rnd));
1932 }
1933
1934 inst = bld.LRP(result, op[0], op[1], op[2]);
1935 inst->saturate = instr->dest.saturate;
1936 break;
1937
1938 case nir_op_b32csel:
1939 if (optimize_frontfacing_ternary(instr, result))
1940 return;
1941
1942 bld.CMP(bld.null_reg_d(), op[0], brw_imm_d(0), BRW_CONDITIONAL_NZ);
1943 inst = bld.SEL(result, op[1], op[2]);
1944 inst->predicate = BRW_PREDICATE_NORMAL;
1945 break;
1946
1947 case nir_op_extract_u8:
1948 case nir_op_extract_i8: {
1949 unsigned byte = nir_src_as_uint(instr->src[1].src);
1950
1951 /* The PRMs say:
1952 *
1953 * BDW+
1954 * There is no direct conversion from B/UB to Q/UQ or Q/UQ to B/UB.
1955 * Use two instructions and a word or DWord intermediate integer type.
1956 */
1957 if (nir_dest_bit_size(instr->dest.dest) == 64) {
1958 const brw_reg_type type = brw_int_type(1, instr->op == nir_op_extract_i8);
1959
1960 if (instr->op == nir_op_extract_i8) {
1961 /* If we need to sign extend, extract to a word first */
1962 fs_reg w_temp = bld.vgrf(BRW_REGISTER_TYPE_W);
1963 bld.MOV(w_temp, subscript(op[0], type, byte));
1964 bld.MOV(result, w_temp);
1965 } else if (byte & 1) {
1966 /* Extract the high byte from the word containing the desired byte
1967 * offset.
1968 */
1969 bld.SHR(result,
1970 subscript(op[0], BRW_REGISTER_TYPE_UW, byte / 2),
1971 brw_imm_uw(8));
1972 } else {
1973 /* Otherwise use an AND with 0xff and a word type */
1974 bld.AND(result,
1975 subscript(op[0], BRW_REGISTER_TYPE_UW, byte / 2),
1976 brw_imm_uw(0xff));
1977 }
1978 } else {
1979 const brw_reg_type type = brw_int_type(1, instr->op == nir_op_extract_i8);
1980 bld.MOV(result, subscript(op[0], type, byte));
1981 }
1982 break;
1983 }
1984
1985 case nir_op_extract_u16:
1986 case nir_op_extract_i16: {
1987 const brw_reg_type type = brw_int_type(2, instr->op == nir_op_extract_i16);
1988 unsigned word = nir_src_as_uint(instr->src[1].src);
1989 bld.MOV(result, subscript(op[0], type, word));
1990 break;
1991 }
1992
1993 default:
1994 unreachable("unhandled instruction");
1995 }
1996
1997 /* If we need to do a boolean resolve, replace the result with -(x & 1)
1998 * to sign extend the low bit to 0/~0
1999 */
2000 if (devinfo->gen <= 5 &&
2001 !result.is_null() &&
2002 (instr->instr.pass_flags & BRW_NIR_BOOLEAN_MASK) == BRW_NIR_BOOLEAN_NEEDS_RESOLVE) {
2003 fs_reg masked = vgrf(glsl_type::int_type);
2004 bld.AND(masked, result, brw_imm_d(1));
2005 masked.negate = true;
2006 bld.MOV(retype(result, BRW_REGISTER_TYPE_D), masked);
2007 }
2008 }
2009
2010 void
2011 fs_visitor::nir_emit_load_const(const fs_builder &bld,
2012 nir_load_const_instr *instr)
2013 {
2014 const brw_reg_type reg_type =
2015 brw_reg_type_from_bit_size(instr->def.bit_size, BRW_REGISTER_TYPE_D);
2016 fs_reg reg = bld.vgrf(reg_type, instr->def.num_components);
2017
2018 switch (instr->def.bit_size) {
2019 case 8:
2020 for (unsigned i = 0; i < instr->def.num_components; i++)
2021 bld.MOV(offset(reg, bld, i), setup_imm_b(bld, instr->value[i].i8));
2022 break;
2023
2024 case 16:
2025 for (unsigned i = 0; i < instr->def.num_components; i++)
2026 bld.MOV(offset(reg, bld, i), brw_imm_w(instr->value[i].i16));
2027 break;
2028
2029 case 32:
2030 for (unsigned i = 0; i < instr->def.num_components; i++)
2031 bld.MOV(offset(reg, bld, i), brw_imm_d(instr->value[i].i32));
2032 break;
2033
2034 case 64:
2035 assert(devinfo->gen >= 7);
2036 if (devinfo->gen == 7) {
2037 /* We don't get 64-bit integer types until gen8 */
2038 for (unsigned i = 0; i < instr->def.num_components; i++) {
2039 bld.MOV(retype(offset(reg, bld, i), BRW_REGISTER_TYPE_DF),
2040 setup_imm_df(bld, instr->value[i].f64));
2041 }
2042 } else {
2043 for (unsigned i = 0; i < instr->def.num_components; i++)
2044 bld.MOV(offset(reg, bld, i), brw_imm_q(instr->value[i].i64));
2045 }
2046 break;
2047
2048 default:
2049 unreachable("Invalid bit size");
2050 }
2051
2052 nir_ssa_values[instr->def.index] = reg;
2053 }
2054
2055 fs_reg
2056 fs_visitor::get_nir_src(const nir_src &src)
2057 {
2058 fs_reg reg;
2059 if (src.is_ssa) {
2060 if (src.ssa->parent_instr->type == nir_instr_type_ssa_undef) {
2061 const brw_reg_type reg_type =
2062 brw_reg_type_from_bit_size(src.ssa->bit_size, BRW_REGISTER_TYPE_D);
2063 reg = bld.vgrf(reg_type, src.ssa->num_components);
2064 } else {
2065 reg = nir_ssa_values[src.ssa->index];
2066 }
2067 } else {
2068 /* We don't handle indirects on locals */
2069 assert(src.reg.indirect == NULL);
2070 reg = offset(nir_locals[src.reg.reg->index], bld,
2071 src.reg.base_offset * src.reg.reg->num_components);
2072 }
2073
2074 if (nir_src_bit_size(src) == 64 && devinfo->gen == 7) {
2075 /* The only 64-bit type available on gen7 is DF, so use that. */
2076 reg.type = BRW_REGISTER_TYPE_DF;
2077 } else {
2078 /* To avoid floating-point denorm flushing problems, set the type by
2079 * default to an integer type - instructions that need floating point
2080 * semantics will set this to F if they need to
2081 */
2082 reg.type = brw_reg_type_from_bit_size(nir_src_bit_size(src),
2083 BRW_REGISTER_TYPE_D);
2084 }
2085
2086 return reg;
2087 }
2088
2089 /**
2090 * Return an IMM for constants; otherwise call get_nir_src() as normal.
2091 *
2092 * This function should not be called on any value which may be 64 bits.
2093 * We could theoretically support 64-bit on gen8+ but we choose not to
2094 * because it wouldn't work in general (no gen7 support) and there are
2095 * enough restrictions in 64-bit immediates that you can't take the return
2096 * value and treat it the same as the result of get_nir_src().
2097 */
2098 fs_reg
2099 fs_visitor::get_nir_src_imm(const nir_src &src)
2100 {
2101 assert(nir_src_bit_size(src) == 32);
2102 return nir_src_is_const(src) ?
2103 fs_reg(brw_imm_d(nir_src_as_int(src))) : get_nir_src(src);
2104 }
2105
2106 fs_reg
2107 fs_visitor::get_nir_dest(const nir_dest &dest)
2108 {
2109 if (dest.is_ssa) {
2110 const brw_reg_type reg_type =
2111 brw_reg_type_from_bit_size(dest.ssa.bit_size,
2112 dest.ssa.bit_size == 8 ?
2113 BRW_REGISTER_TYPE_D :
2114 BRW_REGISTER_TYPE_F);
2115 nir_ssa_values[dest.ssa.index] =
2116 bld.vgrf(reg_type, dest.ssa.num_components);
2117 bld.UNDEF(nir_ssa_values[dest.ssa.index]);
2118 return nir_ssa_values[dest.ssa.index];
2119 } else {
2120 /* We don't handle indirects on locals */
2121 assert(dest.reg.indirect == NULL);
2122 return offset(nir_locals[dest.reg.reg->index], bld,
2123 dest.reg.base_offset * dest.reg.reg->num_components);
2124 }
2125 }
2126
2127 void
2128 fs_visitor::emit_percomp(const fs_builder &bld, const fs_inst &inst,
2129 unsigned wr_mask)
2130 {
2131 for (unsigned i = 0; i < 4; i++) {
2132 if (!((wr_mask >> i) & 1))
2133 continue;
2134
2135 fs_inst *new_inst = new(mem_ctx) fs_inst(inst);
2136 new_inst->dst = offset(new_inst->dst, bld, i);
2137 for (unsigned j = 0; j < new_inst->sources; j++)
2138 if (new_inst->src[j].file == VGRF)
2139 new_inst->src[j] = offset(new_inst->src[j], bld, i);
2140
2141 bld.emit(new_inst);
2142 }
2143 }
2144
2145 static fs_inst *
2146 emit_pixel_interpolater_send(const fs_builder &bld,
2147 enum opcode opcode,
2148 const fs_reg &dst,
2149 const fs_reg &src,
2150 const fs_reg &desc,
2151 glsl_interp_mode interpolation)
2152 {
2153 struct brw_wm_prog_data *wm_prog_data =
2154 brw_wm_prog_data(bld.shader->stage_prog_data);
2155
2156 fs_inst *inst = bld.emit(opcode, dst, src, desc);
2157 /* 2 floats per slot returned */
2158 inst->size_written = 2 * dst.component_size(inst->exec_size);
2159 inst->pi_noperspective = interpolation == INTERP_MODE_NOPERSPECTIVE;
2160
2161 wm_prog_data->pulls_bary = true;
2162
2163 return inst;
2164 }
2165
2166 /**
2167 * Computes 1 << x, given a D/UD register containing some value x.
2168 */
2169 static fs_reg
2170 intexp2(const fs_builder &bld, const fs_reg &x)
2171 {
2172 assert(x.type == BRW_REGISTER_TYPE_UD || x.type == BRW_REGISTER_TYPE_D);
2173
2174 fs_reg result = bld.vgrf(x.type, 1);
2175 fs_reg one = bld.vgrf(x.type, 1);
2176
2177 bld.MOV(one, retype(brw_imm_d(1), one.type));
2178 bld.SHL(result, one, x);
2179 return result;
2180 }
2181
2182 void
2183 fs_visitor::emit_gs_end_primitive(const nir_src &vertex_count_nir_src)
2184 {
2185 assert(stage == MESA_SHADER_GEOMETRY);
2186
2187 struct brw_gs_prog_data *gs_prog_data = brw_gs_prog_data(prog_data);
2188
2189 if (gs_compile->control_data_header_size_bits == 0)
2190 return;
2191
2192 /* We can only do EndPrimitive() functionality when the control data
2193 * consists of cut bits. Fortunately, the only time it isn't is when the
2194 * output type is points, in which case EndPrimitive() is a no-op.
2195 */
2196 if (gs_prog_data->control_data_format !=
2197 GEN7_GS_CONTROL_DATA_FORMAT_GSCTL_CUT) {
2198 return;
2199 }
2200
2201 /* Cut bits use one bit per vertex. */
2202 assert(gs_compile->control_data_bits_per_vertex == 1);
2203
2204 fs_reg vertex_count = get_nir_src(vertex_count_nir_src);
2205 vertex_count.type = BRW_REGISTER_TYPE_UD;
2206
2207 /* Cut bit n should be set to 1 if EndPrimitive() was called after emitting
2208 * vertex n, 0 otherwise. So all we need to do here is mark bit
2209 * (vertex_count - 1) % 32 in the cut_bits register to indicate that
2210 * EndPrimitive() was called after emitting vertex (vertex_count - 1);
2211 * vec4_gs_visitor::emit_control_data_bits() will take care of the rest.
2212 *
2213 * Note that if EndPrimitive() is called before emitting any vertices, this
2214 * will cause us to set bit 31 of the control_data_bits register to 1.
2215 * That's fine because:
2216 *
2217 * - If max_vertices < 32, then vertex number 31 (zero-based) will never be
2218 * output, so the hardware will ignore cut bit 31.
2219 *
2220 * - If max_vertices == 32, then vertex number 31 is guaranteed to be the
2221 * last vertex, so setting cut bit 31 has no effect (since the primitive
2222 * is automatically ended when the GS terminates).
2223 *
2224 * - If max_vertices > 32, then the ir_emit_vertex visitor will reset the
2225 * control_data_bits register to 0 when the first vertex is emitted.
2226 */
2227
2228 const fs_builder abld = bld.annotate("end primitive");
2229
2230 /* control_data_bits |= 1 << ((vertex_count - 1) % 32) */
2231 fs_reg prev_count = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2232 abld.ADD(prev_count, vertex_count, brw_imm_ud(0xffffffffu));
2233 fs_reg mask = intexp2(abld, prev_count);
2234 /* Note: we're relying on the fact that the GEN SHL instruction only pays
2235 * attention to the lower 5 bits of its second source argument, so on this
2236 * architecture, 1 << (vertex_count - 1) is equivalent to 1 <<
2237 * ((vertex_count - 1) % 32).
2238 */
2239 abld.OR(this->control_data_bits, this->control_data_bits, mask);
2240 }
2241
2242 void
2243 fs_visitor::emit_gs_control_data_bits(const fs_reg &vertex_count)
2244 {
2245 assert(stage == MESA_SHADER_GEOMETRY);
2246 assert(gs_compile->control_data_bits_per_vertex != 0);
2247
2248 struct brw_gs_prog_data *gs_prog_data = brw_gs_prog_data(prog_data);
2249
2250 const fs_builder abld = bld.annotate("emit control data bits");
2251 const fs_builder fwa_bld = bld.exec_all();
2252
2253 /* We use a single UD register to accumulate control data bits (32 bits
2254 * for each of the SIMD8 channels). So we need to write a DWord (32 bits)
2255 * at a time.
2256 *
2257 * Unfortunately, the URB_WRITE_SIMD8 message uses 128-bit (OWord) offsets.
2258 * We have select a 128-bit group via the Global and Per-Slot Offsets, then
2259 * use the Channel Mask phase to enable/disable which DWord within that
2260 * group to write. (Remember, different SIMD8 channels may have emitted
2261 * different numbers of vertices, so we may need per-slot offsets.)
2262 *
2263 * Channel masking presents an annoying problem: we may have to replicate
2264 * the data up to 4 times:
2265 *
2266 * Msg = Handles, Per-Slot Offsets, Channel Masks, Data, Data, Data, Data.
2267 *
2268 * To avoid penalizing shaders that emit a small number of vertices, we
2269 * can avoid these sometimes: if the size of the control data header is
2270 * <= 128 bits, then there is only 1 OWord. All SIMD8 channels will land
2271 * land in the same 128-bit group, so we can skip per-slot offsets.
2272 *
2273 * Similarly, if the control data header is <= 32 bits, there is only one
2274 * DWord, so we can skip channel masks.
2275 */
2276 enum opcode opcode = SHADER_OPCODE_URB_WRITE_SIMD8;
2277
2278 fs_reg channel_mask, per_slot_offset;
2279
2280 if (gs_compile->control_data_header_size_bits > 32) {
2281 opcode = SHADER_OPCODE_URB_WRITE_SIMD8_MASKED;
2282 channel_mask = vgrf(glsl_type::uint_type);
2283 }
2284
2285 if (gs_compile->control_data_header_size_bits > 128) {
2286 opcode = SHADER_OPCODE_URB_WRITE_SIMD8_MASKED_PER_SLOT;
2287 per_slot_offset = vgrf(glsl_type::uint_type);
2288 }
2289
2290 /* Figure out which DWord we're trying to write to using the formula:
2291 *
2292 * dword_index = (vertex_count - 1) * bits_per_vertex / 32
2293 *
2294 * Since bits_per_vertex is a power of two, and is known at compile
2295 * time, this can be optimized to:
2296 *
2297 * dword_index = (vertex_count - 1) >> (6 - log2(bits_per_vertex))
2298 */
2299 if (opcode != SHADER_OPCODE_URB_WRITE_SIMD8) {
2300 fs_reg dword_index = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2301 fs_reg prev_count = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2302 abld.ADD(prev_count, vertex_count, brw_imm_ud(0xffffffffu));
2303 unsigned log2_bits_per_vertex =
2304 util_last_bit(gs_compile->control_data_bits_per_vertex);
2305 abld.SHR(dword_index, prev_count, brw_imm_ud(6u - log2_bits_per_vertex));
2306
2307 if (per_slot_offset.file != BAD_FILE) {
2308 /* Set the per-slot offset to dword_index / 4, so that we'll write to
2309 * the appropriate OWord within the control data header.
2310 */
2311 abld.SHR(per_slot_offset, dword_index, brw_imm_ud(2u));
2312 }
2313
2314 /* Set the channel masks to 1 << (dword_index % 4), so that we'll
2315 * write to the appropriate DWORD within the OWORD.
2316 */
2317 fs_reg channel = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2318 fwa_bld.AND(channel, dword_index, brw_imm_ud(3u));
2319 channel_mask = intexp2(fwa_bld, channel);
2320 /* Then the channel masks need to be in bits 23:16. */
2321 fwa_bld.SHL(channel_mask, channel_mask, brw_imm_ud(16u));
2322 }
2323
2324 /* Store the control data bits in the message payload and send it. */
2325 unsigned mlen = 2;
2326 if (channel_mask.file != BAD_FILE)
2327 mlen += 4; /* channel masks, plus 3 extra copies of the data */
2328 if (per_slot_offset.file != BAD_FILE)
2329 mlen++;
2330
2331 fs_reg payload = bld.vgrf(BRW_REGISTER_TYPE_UD, mlen);
2332 fs_reg *sources = ralloc_array(mem_ctx, fs_reg, mlen);
2333 unsigned i = 0;
2334 sources[i++] = fs_reg(retype(brw_vec8_grf(1, 0), BRW_REGISTER_TYPE_UD));
2335 if (per_slot_offset.file != BAD_FILE)
2336 sources[i++] = per_slot_offset;
2337 if (channel_mask.file != BAD_FILE)
2338 sources[i++] = channel_mask;
2339 while (i < mlen) {
2340 sources[i++] = this->control_data_bits;
2341 }
2342
2343 abld.LOAD_PAYLOAD(payload, sources, mlen, mlen);
2344 fs_inst *inst = abld.emit(opcode, reg_undef, payload);
2345 inst->mlen = mlen;
2346 /* We need to increment Global Offset by 256-bits to make room for
2347 * Broadwell's extra "Vertex Count" payload at the beginning of the
2348 * URB entry. Since this is an OWord message, Global Offset is counted
2349 * in 128-bit units, so we must set it to 2.
2350 */
2351 if (gs_prog_data->static_vertex_count == -1)
2352 inst->offset = 2;
2353 }
2354
2355 void
2356 fs_visitor::set_gs_stream_control_data_bits(const fs_reg &vertex_count,
2357 unsigned stream_id)
2358 {
2359 /* control_data_bits |= stream_id << ((2 * (vertex_count - 1)) % 32) */
2360
2361 /* Note: we are calling this *before* increasing vertex_count, so
2362 * this->vertex_count == vertex_count - 1 in the formula above.
2363 */
2364
2365 /* Stream mode uses 2 bits per vertex */
2366 assert(gs_compile->control_data_bits_per_vertex == 2);
2367
2368 /* Must be a valid stream */
2369 assert(stream_id < MAX_VERTEX_STREAMS);
2370
2371 /* Control data bits are initialized to 0 so we don't have to set any
2372 * bits when sending vertices to stream 0.
2373 */
2374 if (stream_id == 0)
2375 return;
2376
2377 const fs_builder abld = bld.annotate("set stream control data bits", NULL);
2378
2379 /* reg::sid = stream_id */
2380 fs_reg sid = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2381 abld.MOV(sid, brw_imm_ud(stream_id));
2382
2383 /* reg:shift_count = 2 * (vertex_count - 1) */
2384 fs_reg shift_count = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2385 abld.SHL(shift_count, vertex_count, brw_imm_ud(1u));
2386
2387 /* Note: we're relying on the fact that the GEN SHL instruction only pays
2388 * attention to the lower 5 bits of its second source argument, so on this
2389 * architecture, stream_id << 2 * (vertex_count - 1) is equivalent to
2390 * stream_id << ((2 * (vertex_count - 1)) % 32).
2391 */
2392 fs_reg mask = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2393 abld.SHL(mask, sid, shift_count);
2394 abld.OR(this->control_data_bits, this->control_data_bits, mask);
2395 }
2396
2397 void
2398 fs_visitor::emit_gs_vertex(const nir_src &vertex_count_nir_src,
2399 unsigned stream_id)
2400 {
2401 assert(stage == MESA_SHADER_GEOMETRY);
2402
2403 struct brw_gs_prog_data *gs_prog_data = brw_gs_prog_data(prog_data);
2404
2405 fs_reg vertex_count = get_nir_src(vertex_count_nir_src);
2406 vertex_count.type = BRW_REGISTER_TYPE_UD;
2407
2408 /* Haswell and later hardware ignores the "Render Stream Select" bits
2409 * from the 3DSTATE_STREAMOUT packet when the SOL stage is disabled,
2410 * and instead sends all primitives down the pipeline for rasterization.
2411 * If the SOL stage is enabled, "Render Stream Select" is honored and
2412 * primitives bound to non-zero streams are discarded after stream output.
2413 *
2414 * Since the only purpose of primives sent to non-zero streams is to
2415 * be recorded by transform feedback, we can simply discard all geometry
2416 * bound to these streams when transform feedback is disabled.
2417 */
2418 if (stream_id > 0 && !nir->info.has_transform_feedback_varyings)
2419 return;
2420
2421 /* If we're outputting 32 control data bits or less, then we can wait
2422 * until the shader is over to output them all. Otherwise we need to
2423 * output them as we go. Now is the time to do it, since we're about to
2424 * output the vertex_count'th vertex, so it's guaranteed that the
2425 * control data bits associated with the (vertex_count - 1)th vertex are
2426 * correct.
2427 */
2428 if (gs_compile->control_data_header_size_bits > 32) {
2429 const fs_builder abld =
2430 bld.annotate("emit vertex: emit control data bits");
2431
2432 /* Only emit control data bits if we've finished accumulating a batch
2433 * of 32 bits. This is the case when:
2434 *
2435 * (vertex_count * bits_per_vertex) % 32 == 0
2436 *
2437 * (in other words, when the last 5 bits of vertex_count *
2438 * bits_per_vertex are 0). Assuming bits_per_vertex == 2^n for some
2439 * integer n (which is always the case, since bits_per_vertex is
2440 * always 1 or 2), this is equivalent to requiring that the last 5-n
2441 * bits of vertex_count are 0:
2442 *
2443 * vertex_count & (2^(5-n) - 1) == 0
2444 *
2445 * 2^(5-n) == 2^5 / 2^n == 32 / bits_per_vertex, so this is
2446 * equivalent to:
2447 *
2448 * vertex_count & (32 / bits_per_vertex - 1) == 0
2449 *
2450 * TODO: If vertex_count is an immediate, we could do some of this math
2451 * at compile time...
2452 */
2453 fs_inst *inst =
2454 abld.AND(bld.null_reg_d(), vertex_count,
2455 brw_imm_ud(32u / gs_compile->control_data_bits_per_vertex - 1u));
2456 inst->conditional_mod = BRW_CONDITIONAL_Z;
2457
2458 abld.IF(BRW_PREDICATE_NORMAL);
2459 /* If vertex_count is 0, then no control data bits have been
2460 * accumulated yet, so we can skip emitting them.
2461 */
2462 abld.CMP(bld.null_reg_d(), vertex_count, brw_imm_ud(0u),
2463 BRW_CONDITIONAL_NEQ);
2464 abld.IF(BRW_PREDICATE_NORMAL);
2465 emit_gs_control_data_bits(vertex_count);
2466 abld.emit(BRW_OPCODE_ENDIF);
2467
2468 /* Reset control_data_bits to 0 so we can start accumulating a new
2469 * batch.
2470 *
2471 * Note: in the case where vertex_count == 0, this neutralizes the
2472 * effect of any call to EndPrimitive() that the shader may have
2473 * made before outputting its first vertex.
2474 */
2475 inst = abld.MOV(this->control_data_bits, brw_imm_ud(0u));
2476 inst->force_writemask_all = true;
2477 abld.emit(BRW_OPCODE_ENDIF);
2478 }
2479
2480 emit_urb_writes(vertex_count);
2481
2482 /* In stream mode we have to set control data bits for all vertices
2483 * unless we have disabled control data bits completely (which we do
2484 * do for GL_POINTS outputs that don't use streams).
2485 */
2486 if (gs_compile->control_data_header_size_bits > 0 &&
2487 gs_prog_data->control_data_format ==
2488 GEN7_GS_CONTROL_DATA_FORMAT_GSCTL_SID) {
2489 set_gs_stream_control_data_bits(vertex_count, stream_id);
2490 }
2491 }
2492
2493 void
2494 fs_visitor::emit_gs_input_load(const fs_reg &dst,
2495 const nir_src &vertex_src,
2496 unsigned base_offset,
2497 const nir_src &offset_src,
2498 unsigned num_components,
2499 unsigned first_component)
2500 {
2501 assert(type_sz(dst.type) == 4);
2502 struct brw_gs_prog_data *gs_prog_data = brw_gs_prog_data(prog_data);
2503 const unsigned push_reg_count = gs_prog_data->base.urb_read_length * 8;
2504
2505 /* TODO: figure out push input layout for invocations == 1 */
2506 if (gs_prog_data->invocations == 1 &&
2507 nir_src_is_const(offset_src) && nir_src_is_const(vertex_src) &&
2508 4 * (base_offset + nir_src_as_uint(offset_src)) < push_reg_count) {
2509 int imm_offset = (base_offset + nir_src_as_uint(offset_src)) * 4 +
2510 nir_src_as_uint(vertex_src) * push_reg_count;
2511 for (unsigned i = 0; i < num_components; i++) {
2512 bld.MOV(offset(dst, bld, i),
2513 fs_reg(ATTR, imm_offset + i + first_component, dst.type));
2514 }
2515 return;
2516 }
2517
2518 /* Resort to the pull model. Ensure the VUE handles are provided. */
2519 assert(gs_prog_data->base.include_vue_handles);
2520
2521 unsigned first_icp_handle = gs_prog_data->include_primitive_id ? 3 : 2;
2522 fs_reg icp_handle = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2523
2524 if (gs_prog_data->invocations == 1) {
2525 if (nir_src_is_const(vertex_src)) {
2526 /* The vertex index is constant; just select the proper URB handle. */
2527 icp_handle =
2528 retype(brw_vec8_grf(first_icp_handle + nir_src_as_uint(vertex_src), 0),
2529 BRW_REGISTER_TYPE_UD);
2530 } else {
2531 /* The vertex index is non-constant. We need to use indirect
2532 * addressing to fetch the proper URB handle.
2533 *
2534 * First, we start with the sequence <7, 6, 5, 4, 3, 2, 1, 0>
2535 * indicating that channel <n> should read the handle from
2536 * DWord <n>. We convert that to bytes by multiplying by 4.
2537 *
2538 * Next, we convert the vertex index to bytes by multiplying
2539 * by 32 (shifting by 5), and add the two together. This is
2540 * the final indirect byte offset.
2541 */
2542 fs_reg sequence = bld.vgrf(BRW_REGISTER_TYPE_UW, 1);
2543 fs_reg channel_offsets = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2544 fs_reg vertex_offset_bytes = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2545 fs_reg icp_offset_bytes = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2546
2547 /* sequence = <7, 6, 5, 4, 3, 2, 1, 0> */
2548 bld.MOV(sequence, fs_reg(brw_imm_v(0x76543210)));
2549 /* channel_offsets = 4 * sequence = <28, 24, 20, 16, 12, 8, 4, 0> */
2550 bld.SHL(channel_offsets, sequence, brw_imm_ud(2u));
2551 /* Convert vertex_index to bytes (multiply by 32) */
2552 bld.SHL(vertex_offset_bytes,
2553 retype(get_nir_src(vertex_src), BRW_REGISTER_TYPE_UD),
2554 brw_imm_ud(5u));
2555 bld.ADD(icp_offset_bytes, vertex_offset_bytes, channel_offsets);
2556
2557 /* Use first_icp_handle as the base offset. There is one register
2558 * of URB handles per vertex, so inform the register allocator that
2559 * we might read up to nir->info.gs.vertices_in registers.
2560 */
2561 bld.emit(SHADER_OPCODE_MOV_INDIRECT, icp_handle,
2562 retype(brw_vec8_grf(first_icp_handle, 0), icp_handle.type),
2563 fs_reg(icp_offset_bytes),
2564 brw_imm_ud(nir->info.gs.vertices_in * REG_SIZE));
2565 }
2566 } else {
2567 assert(gs_prog_data->invocations > 1);
2568
2569 if (nir_src_is_const(vertex_src)) {
2570 unsigned vertex = nir_src_as_uint(vertex_src);
2571 assert(devinfo->gen >= 9 || vertex <= 5);
2572 bld.MOV(icp_handle,
2573 retype(brw_vec1_grf(first_icp_handle + vertex / 8, vertex % 8),
2574 BRW_REGISTER_TYPE_UD));
2575 } else {
2576 /* The vertex index is non-constant. We need to use indirect
2577 * addressing to fetch the proper URB handle.
2578 *
2579 */
2580 fs_reg icp_offset_bytes = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2581
2582 /* Convert vertex_index to bytes (multiply by 4) */
2583 bld.SHL(icp_offset_bytes,
2584 retype(get_nir_src(vertex_src), BRW_REGISTER_TYPE_UD),
2585 brw_imm_ud(2u));
2586
2587 /* Use first_icp_handle as the base offset. There is one DWord
2588 * of URB handles per vertex, so inform the register allocator that
2589 * we might read up to ceil(nir->info.gs.vertices_in / 8) registers.
2590 */
2591 bld.emit(SHADER_OPCODE_MOV_INDIRECT, icp_handle,
2592 retype(brw_vec8_grf(first_icp_handle, 0), icp_handle.type),
2593 fs_reg(icp_offset_bytes),
2594 brw_imm_ud(DIV_ROUND_UP(nir->info.gs.vertices_in, 8) *
2595 REG_SIZE));
2596 }
2597 }
2598
2599 fs_inst *inst;
2600 fs_reg indirect_offset = get_nir_src(offset_src);
2601
2602 if (nir_src_is_const(offset_src)) {
2603 /* Constant indexing - use global offset. */
2604 if (first_component != 0) {
2605 unsigned read_components = num_components + first_component;
2606 fs_reg tmp = bld.vgrf(dst.type, read_components);
2607 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8, tmp, icp_handle);
2608 inst->size_written = read_components *
2609 tmp.component_size(inst->exec_size);
2610 for (unsigned i = 0; i < num_components; i++) {
2611 bld.MOV(offset(dst, bld, i),
2612 offset(tmp, bld, i + first_component));
2613 }
2614 } else {
2615 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8, dst, icp_handle);
2616 inst->size_written = num_components *
2617 dst.component_size(inst->exec_size);
2618 }
2619 inst->offset = base_offset + nir_src_as_uint(offset_src);
2620 inst->mlen = 1;
2621 } else {
2622 /* Indirect indexing - use per-slot offsets as well. */
2623 const fs_reg srcs[] = { icp_handle, indirect_offset };
2624 unsigned read_components = num_components + first_component;
2625 fs_reg tmp = bld.vgrf(dst.type, read_components);
2626 fs_reg payload = bld.vgrf(BRW_REGISTER_TYPE_UD, 2);
2627 bld.LOAD_PAYLOAD(payload, srcs, ARRAY_SIZE(srcs), 0);
2628 if (first_component != 0) {
2629 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT, tmp,
2630 payload);
2631 inst->size_written = read_components *
2632 tmp.component_size(inst->exec_size);
2633 for (unsigned i = 0; i < num_components; i++) {
2634 bld.MOV(offset(dst, bld, i),
2635 offset(tmp, bld, i + first_component));
2636 }
2637 } else {
2638 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT, dst, payload);
2639 inst->size_written = num_components *
2640 dst.component_size(inst->exec_size);
2641 }
2642 inst->offset = base_offset;
2643 inst->mlen = 2;
2644 }
2645 }
2646
2647 fs_reg
2648 fs_visitor::get_indirect_offset(nir_intrinsic_instr *instr)
2649 {
2650 nir_src *offset_src = nir_get_io_offset_src(instr);
2651
2652 if (nir_src_is_const(*offset_src)) {
2653 /* The only constant offset we should find is 0. brw_nir.c's
2654 * add_const_offset_to_base() will fold other constant offsets
2655 * into instr->const_index[0].
2656 */
2657 assert(nir_src_as_uint(*offset_src) == 0);
2658 return fs_reg();
2659 }
2660
2661 return get_nir_src(*offset_src);
2662 }
2663
2664 void
2665 fs_visitor::nir_emit_vs_intrinsic(const fs_builder &bld,
2666 nir_intrinsic_instr *instr)
2667 {
2668 assert(stage == MESA_SHADER_VERTEX);
2669
2670 fs_reg dest;
2671 if (nir_intrinsic_infos[instr->intrinsic].has_dest)
2672 dest = get_nir_dest(instr->dest);
2673
2674 switch (instr->intrinsic) {
2675 case nir_intrinsic_load_vertex_id:
2676 case nir_intrinsic_load_base_vertex:
2677 unreachable("should be lowered by nir_lower_system_values()");
2678
2679 case nir_intrinsic_load_input: {
2680 assert(nir_dest_bit_size(instr->dest) == 32);
2681 fs_reg src = fs_reg(ATTR, nir_intrinsic_base(instr) * 4, dest.type);
2682 src = offset(src, bld, nir_intrinsic_component(instr));
2683 src = offset(src, bld, nir_src_as_uint(instr->src[0]));
2684
2685 for (unsigned i = 0; i < instr->num_components; i++)
2686 bld.MOV(offset(dest, bld, i), offset(src, bld, i));
2687 break;
2688 }
2689
2690 case nir_intrinsic_load_vertex_id_zero_base:
2691 case nir_intrinsic_load_instance_id:
2692 case nir_intrinsic_load_base_instance:
2693 case nir_intrinsic_load_draw_id:
2694 case nir_intrinsic_load_first_vertex:
2695 case nir_intrinsic_load_is_indexed_draw:
2696 unreachable("lowered by brw_nir_lower_vs_inputs");
2697
2698 default:
2699 nir_emit_intrinsic(bld, instr);
2700 break;
2701 }
2702 }
2703
2704 fs_reg
2705 fs_visitor::get_tcs_single_patch_icp_handle(const fs_builder &bld,
2706 nir_intrinsic_instr *instr)
2707 {
2708 struct brw_tcs_prog_data *tcs_prog_data = brw_tcs_prog_data(prog_data);
2709 const nir_src &vertex_src = instr->src[0];
2710 nir_intrinsic_instr *vertex_intrin = nir_src_as_intrinsic(vertex_src);
2711 fs_reg icp_handle;
2712
2713 if (nir_src_is_const(vertex_src)) {
2714 /* Emit a MOV to resolve <0,1,0> regioning. */
2715 icp_handle = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2716 unsigned vertex = nir_src_as_uint(vertex_src);
2717 bld.MOV(icp_handle,
2718 retype(brw_vec1_grf(1 + (vertex >> 3), vertex & 7),
2719 BRW_REGISTER_TYPE_UD));
2720 } else if (tcs_prog_data->instances == 1 && vertex_intrin &&
2721 vertex_intrin->intrinsic == nir_intrinsic_load_invocation_id) {
2722 /* For the common case of only 1 instance, an array index of
2723 * gl_InvocationID means reading g1. Skip all the indirect work.
2724 */
2725 icp_handle = retype(brw_vec8_grf(1, 0), BRW_REGISTER_TYPE_UD);
2726 } else {
2727 /* The vertex index is non-constant. We need to use indirect
2728 * addressing to fetch the proper URB handle.
2729 */
2730 icp_handle = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2731
2732 /* Each ICP handle is a single DWord (4 bytes) */
2733 fs_reg vertex_offset_bytes = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2734 bld.SHL(vertex_offset_bytes,
2735 retype(get_nir_src(vertex_src), BRW_REGISTER_TYPE_UD),
2736 brw_imm_ud(2u));
2737
2738 /* Start at g1. We might read up to 4 registers. */
2739 bld.emit(SHADER_OPCODE_MOV_INDIRECT, icp_handle,
2740 retype(brw_vec8_grf(1, 0), icp_handle.type), vertex_offset_bytes,
2741 brw_imm_ud(4 * REG_SIZE));
2742 }
2743
2744 return icp_handle;
2745 }
2746
2747 fs_reg
2748 fs_visitor::get_tcs_eight_patch_icp_handle(const fs_builder &bld,
2749 nir_intrinsic_instr *instr)
2750 {
2751 struct brw_tcs_prog_key *tcs_key = (struct brw_tcs_prog_key *) key;
2752 struct brw_tcs_prog_data *tcs_prog_data = brw_tcs_prog_data(prog_data);
2753 const nir_src &vertex_src = instr->src[0];
2754
2755 unsigned first_icp_handle = tcs_prog_data->include_primitive_id ? 3 : 2;
2756
2757 if (nir_src_is_const(vertex_src)) {
2758 return fs_reg(retype(brw_vec8_grf(first_icp_handle +
2759 nir_src_as_uint(vertex_src), 0),
2760 BRW_REGISTER_TYPE_UD));
2761 }
2762
2763 /* The vertex index is non-constant. We need to use indirect
2764 * addressing to fetch the proper URB handle.
2765 *
2766 * First, we start with the sequence <7, 6, 5, 4, 3, 2, 1, 0>
2767 * indicating that channel <n> should read the handle from
2768 * DWord <n>. We convert that to bytes by multiplying by 4.
2769 *
2770 * Next, we convert the vertex index to bytes by multiplying
2771 * by 32 (shifting by 5), and add the two together. This is
2772 * the final indirect byte offset.
2773 */
2774 fs_reg icp_handle = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2775 fs_reg sequence = bld.vgrf(BRW_REGISTER_TYPE_UW, 1);
2776 fs_reg channel_offsets = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2777 fs_reg vertex_offset_bytes = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2778 fs_reg icp_offset_bytes = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2779
2780 /* sequence = <7, 6, 5, 4, 3, 2, 1, 0> */
2781 bld.MOV(sequence, fs_reg(brw_imm_v(0x76543210)));
2782 /* channel_offsets = 4 * sequence = <28, 24, 20, 16, 12, 8, 4, 0> */
2783 bld.SHL(channel_offsets, sequence, brw_imm_ud(2u));
2784 /* Convert vertex_index to bytes (multiply by 32) */
2785 bld.SHL(vertex_offset_bytes,
2786 retype(get_nir_src(vertex_src), BRW_REGISTER_TYPE_UD),
2787 brw_imm_ud(5u));
2788 bld.ADD(icp_offset_bytes, vertex_offset_bytes, channel_offsets);
2789
2790 /* Use first_icp_handle as the base offset. There is one register
2791 * of URB handles per vertex, so inform the register allocator that
2792 * we might read up to nir->info.gs.vertices_in registers.
2793 */
2794 bld.emit(SHADER_OPCODE_MOV_INDIRECT, icp_handle,
2795 retype(brw_vec8_grf(first_icp_handle, 0), icp_handle.type),
2796 icp_offset_bytes, brw_imm_ud(tcs_key->input_vertices * REG_SIZE));
2797
2798 return icp_handle;
2799 }
2800
2801 struct brw_reg
2802 fs_visitor::get_tcs_output_urb_handle()
2803 {
2804 struct brw_vue_prog_data *vue_prog_data = brw_vue_prog_data(prog_data);
2805
2806 if (vue_prog_data->dispatch_mode == DISPATCH_MODE_TCS_SINGLE_PATCH) {
2807 return retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_UD);
2808 } else {
2809 assert(vue_prog_data->dispatch_mode == DISPATCH_MODE_TCS_8_PATCH);
2810 return retype(brw_vec8_grf(1, 0), BRW_REGISTER_TYPE_UD);
2811 }
2812 }
2813
2814 void
2815 fs_visitor::nir_emit_tcs_intrinsic(const fs_builder &bld,
2816 nir_intrinsic_instr *instr)
2817 {
2818 assert(stage == MESA_SHADER_TESS_CTRL);
2819 struct brw_tcs_prog_key *tcs_key = (struct brw_tcs_prog_key *) key;
2820 struct brw_tcs_prog_data *tcs_prog_data = brw_tcs_prog_data(prog_data);
2821 struct brw_vue_prog_data *vue_prog_data = &tcs_prog_data->base;
2822
2823 bool eight_patch =
2824 vue_prog_data->dispatch_mode == DISPATCH_MODE_TCS_8_PATCH;
2825
2826 fs_reg dst;
2827 if (nir_intrinsic_infos[instr->intrinsic].has_dest)
2828 dst = get_nir_dest(instr->dest);
2829
2830 switch (instr->intrinsic) {
2831 case nir_intrinsic_load_primitive_id:
2832 bld.MOV(dst, fs_reg(eight_patch ? brw_vec8_grf(2, 0)
2833 : brw_vec1_grf(0, 1)));
2834 break;
2835 case nir_intrinsic_load_invocation_id:
2836 bld.MOV(retype(dst, invocation_id.type), invocation_id);
2837 break;
2838 case nir_intrinsic_load_patch_vertices_in:
2839 bld.MOV(retype(dst, BRW_REGISTER_TYPE_D),
2840 brw_imm_d(tcs_key->input_vertices));
2841 break;
2842
2843 case nir_intrinsic_control_barrier: {
2844 if (tcs_prog_data->instances == 1)
2845 break;
2846
2847 fs_reg m0 = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2848 fs_reg m0_2 = component(m0, 2);
2849
2850 const fs_builder chanbld = bld.exec_all().group(1, 0);
2851
2852 /* Zero the message header */
2853 bld.exec_all().MOV(m0, brw_imm_ud(0u));
2854
2855 if (devinfo->gen < 11) {
2856 /* Copy "Barrier ID" from r0.2, bits 16:13 */
2857 chanbld.AND(m0_2, retype(brw_vec1_grf(0, 2), BRW_REGISTER_TYPE_UD),
2858 brw_imm_ud(INTEL_MASK(16, 13)));
2859
2860 /* Shift it up to bits 27:24. */
2861 chanbld.SHL(m0_2, m0_2, brw_imm_ud(11));
2862 } else {
2863 chanbld.AND(m0_2, retype(brw_vec1_grf(0, 2), BRW_REGISTER_TYPE_UD),
2864 brw_imm_ud(INTEL_MASK(30, 24)));
2865 }
2866
2867 /* Set the Barrier Count and the enable bit */
2868 if (devinfo->gen < 11) {
2869 chanbld.OR(m0_2, m0_2,
2870 brw_imm_ud(tcs_prog_data->instances << 9 | (1 << 15)));
2871 } else {
2872 chanbld.OR(m0_2, m0_2,
2873 brw_imm_ud(tcs_prog_data->instances << 8 | (1 << 15)));
2874 }
2875
2876 bld.emit(SHADER_OPCODE_BARRIER, bld.null_reg_ud(), m0);
2877 break;
2878 }
2879
2880 case nir_intrinsic_load_input:
2881 unreachable("nir_lower_io should never give us these.");
2882 break;
2883
2884 case nir_intrinsic_load_per_vertex_input: {
2885 assert(nir_dest_bit_size(instr->dest) == 32);
2886 fs_reg indirect_offset = get_indirect_offset(instr);
2887 unsigned imm_offset = instr->const_index[0];
2888 fs_inst *inst;
2889
2890 fs_reg icp_handle =
2891 eight_patch ? get_tcs_eight_patch_icp_handle(bld, instr)
2892 : get_tcs_single_patch_icp_handle(bld, instr);
2893
2894 /* We can only read two double components with each URB read, so
2895 * we send two read messages in that case, each one loading up to
2896 * two double components.
2897 */
2898 unsigned num_components = instr->num_components;
2899 unsigned first_component = nir_intrinsic_component(instr);
2900
2901 if (indirect_offset.file == BAD_FILE) {
2902 /* Constant indexing - use global offset. */
2903 if (first_component != 0) {
2904 unsigned read_components = num_components + first_component;
2905 fs_reg tmp = bld.vgrf(dst.type, read_components);
2906 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8, tmp, icp_handle);
2907 for (unsigned i = 0; i < num_components; i++) {
2908 bld.MOV(offset(dst, bld, i),
2909 offset(tmp, bld, i + first_component));
2910 }
2911 } else {
2912 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8, dst, icp_handle);
2913 }
2914 inst->offset = imm_offset;
2915 inst->mlen = 1;
2916 } else {
2917 /* Indirect indexing - use per-slot offsets as well. */
2918 const fs_reg srcs[] = { icp_handle, indirect_offset };
2919 fs_reg payload = bld.vgrf(BRW_REGISTER_TYPE_UD, 2);
2920 bld.LOAD_PAYLOAD(payload, srcs, ARRAY_SIZE(srcs), 0);
2921 if (first_component != 0) {
2922 unsigned read_components = num_components + first_component;
2923 fs_reg tmp = bld.vgrf(dst.type, read_components);
2924 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT, tmp,
2925 payload);
2926 for (unsigned i = 0; i < num_components; i++) {
2927 bld.MOV(offset(dst, bld, i),
2928 offset(tmp, bld, i + first_component));
2929 }
2930 } else {
2931 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT, dst,
2932 payload);
2933 }
2934 inst->offset = imm_offset;
2935 inst->mlen = 2;
2936 }
2937 inst->size_written = (num_components + first_component) *
2938 inst->dst.component_size(inst->exec_size);
2939
2940 /* Copy the temporary to the destination to deal with writemasking.
2941 *
2942 * Also attempt to deal with gl_PointSize being in the .w component.
2943 */
2944 if (inst->offset == 0 && indirect_offset.file == BAD_FILE) {
2945 assert(type_sz(dst.type) == 4);
2946 inst->dst = bld.vgrf(dst.type, 4);
2947 inst->size_written = 4 * REG_SIZE;
2948 bld.MOV(dst, offset(inst->dst, bld, 3));
2949 }
2950 break;
2951 }
2952
2953 case nir_intrinsic_load_output:
2954 case nir_intrinsic_load_per_vertex_output: {
2955 assert(nir_dest_bit_size(instr->dest) == 32);
2956 fs_reg indirect_offset = get_indirect_offset(instr);
2957 unsigned imm_offset = instr->const_index[0];
2958 unsigned first_component = nir_intrinsic_component(instr);
2959
2960 struct brw_reg output_handles = get_tcs_output_urb_handle();
2961
2962 fs_inst *inst;
2963 if (indirect_offset.file == BAD_FILE) {
2964 /* This MOV replicates the output handle to all enabled channels
2965 * is SINGLE_PATCH mode.
2966 */
2967 fs_reg patch_handle = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2968 bld.MOV(patch_handle, output_handles);
2969
2970 {
2971 if (first_component != 0) {
2972 unsigned read_components =
2973 instr->num_components + first_component;
2974 fs_reg tmp = bld.vgrf(dst.type, read_components);
2975 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8, tmp,
2976 patch_handle);
2977 inst->size_written = read_components * REG_SIZE;
2978 for (unsigned i = 0; i < instr->num_components; i++) {
2979 bld.MOV(offset(dst, bld, i),
2980 offset(tmp, bld, i + first_component));
2981 }
2982 } else {
2983 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8, dst,
2984 patch_handle);
2985 inst->size_written = instr->num_components * REG_SIZE;
2986 }
2987 inst->offset = imm_offset;
2988 inst->mlen = 1;
2989 }
2990 } else {
2991 /* Indirect indexing - use per-slot offsets as well. */
2992 const fs_reg srcs[] = { output_handles, indirect_offset };
2993 fs_reg payload = bld.vgrf(BRW_REGISTER_TYPE_UD, 2);
2994 bld.LOAD_PAYLOAD(payload, srcs, ARRAY_SIZE(srcs), 0);
2995 if (first_component != 0) {
2996 unsigned read_components =
2997 instr->num_components + first_component;
2998 fs_reg tmp = bld.vgrf(dst.type, read_components);
2999 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT, tmp,
3000 payload);
3001 inst->size_written = read_components * REG_SIZE;
3002 for (unsigned i = 0; i < instr->num_components; i++) {
3003 bld.MOV(offset(dst, bld, i),
3004 offset(tmp, bld, i + first_component));
3005 }
3006 } else {
3007 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT, dst,
3008 payload);
3009 inst->size_written = instr->num_components * REG_SIZE;
3010 }
3011 inst->offset = imm_offset;
3012 inst->mlen = 2;
3013 }
3014 break;
3015 }
3016
3017 case nir_intrinsic_store_output:
3018 case nir_intrinsic_store_per_vertex_output: {
3019 assert(nir_src_bit_size(instr->src[0]) == 32);
3020 fs_reg value = get_nir_src(instr->src[0]);
3021 fs_reg indirect_offset = get_indirect_offset(instr);
3022 unsigned imm_offset = instr->const_index[0];
3023 unsigned mask = instr->const_index[1];
3024 unsigned header_regs = 0;
3025 struct brw_reg output_handles = get_tcs_output_urb_handle();
3026
3027 fs_reg srcs[7];
3028 srcs[header_regs++] = output_handles;
3029
3030 if (indirect_offset.file != BAD_FILE) {
3031 srcs[header_regs++] = indirect_offset;
3032 }
3033
3034 if (mask == 0)
3035 break;
3036
3037 unsigned num_components = util_last_bit(mask);
3038 enum opcode opcode;
3039
3040 /* We can only pack two 64-bit components in a single message, so send
3041 * 2 messages if we have more components
3042 */
3043 unsigned first_component = nir_intrinsic_component(instr);
3044 mask = mask << first_component;
3045
3046 if (mask != WRITEMASK_XYZW) {
3047 srcs[header_regs++] = brw_imm_ud(mask << 16);
3048 opcode = indirect_offset.file != BAD_FILE ?
3049 SHADER_OPCODE_URB_WRITE_SIMD8_MASKED_PER_SLOT :
3050 SHADER_OPCODE_URB_WRITE_SIMD8_MASKED;
3051 } else {
3052 opcode = indirect_offset.file != BAD_FILE ?
3053 SHADER_OPCODE_URB_WRITE_SIMD8_PER_SLOT :
3054 SHADER_OPCODE_URB_WRITE_SIMD8;
3055 }
3056
3057 for (unsigned i = 0; i < num_components; i++) {
3058 if (!(mask & (1 << (i + first_component))))
3059 continue;
3060
3061 srcs[header_regs + i + first_component] = offset(value, bld, i);
3062 }
3063
3064 unsigned mlen = header_regs + num_components + first_component;
3065 fs_reg payload =
3066 bld.vgrf(BRW_REGISTER_TYPE_UD, mlen);
3067 bld.LOAD_PAYLOAD(payload, srcs, mlen, header_regs);
3068
3069 fs_inst *inst = bld.emit(opcode, bld.null_reg_ud(), payload);
3070 inst->offset = imm_offset;
3071 inst->mlen = mlen;
3072 break;
3073 }
3074
3075 default:
3076 nir_emit_intrinsic(bld, instr);
3077 break;
3078 }
3079 }
3080
3081 void
3082 fs_visitor::nir_emit_tes_intrinsic(const fs_builder &bld,
3083 nir_intrinsic_instr *instr)
3084 {
3085 assert(stage == MESA_SHADER_TESS_EVAL);
3086 struct brw_tes_prog_data *tes_prog_data = brw_tes_prog_data(prog_data);
3087
3088 fs_reg dest;
3089 if (nir_intrinsic_infos[instr->intrinsic].has_dest)
3090 dest = get_nir_dest(instr->dest);
3091
3092 switch (instr->intrinsic) {
3093 case nir_intrinsic_load_primitive_id:
3094 bld.MOV(dest, fs_reg(brw_vec1_grf(0, 1)));
3095 break;
3096 case nir_intrinsic_load_tess_coord:
3097 /* gl_TessCoord is part of the payload in g1-3 */
3098 for (unsigned i = 0; i < 3; i++) {
3099 bld.MOV(offset(dest, bld, i), fs_reg(brw_vec8_grf(1 + i, 0)));
3100 }
3101 break;
3102
3103 case nir_intrinsic_load_input:
3104 case nir_intrinsic_load_per_vertex_input: {
3105 assert(nir_dest_bit_size(instr->dest) == 32);
3106 fs_reg indirect_offset = get_indirect_offset(instr);
3107 unsigned imm_offset = instr->const_index[0];
3108 unsigned first_component = nir_intrinsic_component(instr);
3109
3110 fs_inst *inst;
3111 if (indirect_offset.file == BAD_FILE) {
3112 /* Arbitrarily only push up to 32 vec4 slots worth of data,
3113 * which is 16 registers (since each holds 2 vec4 slots).
3114 */
3115 const unsigned max_push_slots = 32;
3116 if (imm_offset < max_push_slots) {
3117 fs_reg src = fs_reg(ATTR, imm_offset / 2, dest.type);
3118 for (int i = 0; i < instr->num_components; i++) {
3119 unsigned comp = 4 * (imm_offset % 2) + i + first_component;
3120 bld.MOV(offset(dest, bld, i), component(src, comp));
3121 }
3122
3123 tes_prog_data->base.urb_read_length =
3124 MAX2(tes_prog_data->base.urb_read_length,
3125 (imm_offset / 2) + 1);
3126 } else {
3127 /* Replicate the patch handle to all enabled channels */
3128 const fs_reg srcs[] = {
3129 retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_UD)
3130 };
3131 fs_reg patch_handle = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
3132 bld.LOAD_PAYLOAD(patch_handle, srcs, ARRAY_SIZE(srcs), 0);
3133
3134 if (first_component != 0) {
3135 unsigned read_components =
3136 instr->num_components + first_component;
3137 fs_reg tmp = bld.vgrf(dest.type, read_components);
3138 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8, tmp,
3139 patch_handle);
3140 inst->size_written = read_components * REG_SIZE;
3141 for (unsigned i = 0; i < instr->num_components; i++) {
3142 bld.MOV(offset(dest, bld, i),
3143 offset(tmp, bld, i + first_component));
3144 }
3145 } else {
3146 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8, dest,
3147 patch_handle);
3148 inst->size_written = instr->num_components * REG_SIZE;
3149 }
3150 inst->mlen = 1;
3151 inst->offset = imm_offset;
3152 }
3153 } else {
3154 /* Indirect indexing - use per-slot offsets as well. */
3155
3156 /* We can only read two double components with each URB read, so
3157 * we send two read messages in that case, each one loading up to
3158 * two double components.
3159 */
3160 unsigned num_components = instr->num_components;
3161 const fs_reg srcs[] = {
3162 retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_UD),
3163 indirect_offset
3164 };
3165 fs_reg payload = bld.vgrf(BRW_REGISTER_TYPE_UD, 2);
3166 bld.LOAD_PAYLOAD(payload, srcs, ARRAY_SIZE(srcs), 0);
3167
3168 if (first_component != 0) {
3169 unsigned read_components =
3170 num_components + first_component;
3171 fs_reg tmp = bld.vgrf(dest.type, read_components);
3172 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT, tmp,
3173 payload);
3174 for (unsigned i = 0; i < num_components; i++) {
3175 bld.MOV(offset(dest, bld, i),
3176 offset(tmp, bld, i + first_component));
3177 }
3178 } else {
3179 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT, dest,
3180 payload);
3181 }
3182 inst->mlen = 2;
3183 inst->offset = imm_offset;
3184 inst->size_written = (num_components + first_component) *
3185 inst->dst.component_size(inst->exec_size);
3186 }
3187 break;
3188 }
3189 default:
3190 nir_emit_intrinsic(bld, instr);
3191 break;
3192 }
3193 }
3194
3195 void
3196 fs_visitor::nir_emit_gs_intrinsic(const fs_builder &bld,
3197 nir_intrinsic_instr *instr)
3198 {
3199 assert(stage == MESA_SHADER_GEOMETRY);
3200 fs_reg indirect_offset;
3201
3202 fs_reg dest;
3203 if (nir_intrinsic_infos[instr->intrinsic].has_dest)
3204 dest = get_nir_dest(instr->dest);
3205
3206 switch (instr->intrinsic) {
3207 case nir_intrinsic_load_primitive_id:
3208 assert(stage == MESA_SHADER_GEOMETRY);
3209 assert(brw_gs_prog_data(prog_data)->include_primitive_id);
3210 bld.MOV(retype(dest, BRW_REGISTER_TYPE_UD),
3211 retype(fs_reg(brw_vec8_grf(2, 0)), BRW_REGISTER_TYPE_UD));
3212 break;
3213
3214 case nir_intrinsic_load_input:
3215 unreachable("load_input intrinsics are invalid for the GS stage");
3216
3217 case nir_intrinsic_load_per_vertex_input:
3218 emit_gs_input_load(dest, instr->src[0], instr->const_index[0],
3219 instr->src[1], instr->num_components,
3220 nir_intrinsic_component(instr));
3221 break;
3222
3223 case nir_intrinsic_emit_vertex_with_counter:
3224 emit_gs_vertex(instr->src[0], instr->const_index[0]);
3225 break;
3226
3227 case nir_intrinsic_end_primitive_with_counter:
3228 emit_gs_end_primitive(instr->src[0]);
3229 break;
3230
3231 case nir_intrinsic_set_vertex_count:
3232 bld.MOV(this->final_gs_vertex_count, get_nir_src(instr->src[0]));
3233 break;
3234
3235 case nir_intrinsic_load_invocation_id: {
3236 fs_reg val = nir_system_values[SYSTEM_VALUE_INVOCATION_ID];
3237 assert(val.file != BAD_FILE);
3238 dest.type = val.type;
3239 bld.MOV(dest, val);
3240 break;
3241 }
3242
3243 default:
3244 nir_emit_intrinsic(bld, instr);
3245 break;
3246 }
3247 }
3248
3249 /**
3250 * Fetch the current render target layer index.
3251 */
3252 static fs_reg
3253 fetch_render_target_array_index(const fs_builder &bld)
3254 {
3255 if (bld.shader->devinfo->gen >= 6) {
3256 /* The render target array index is provided in the thread payload as
3257 * bits 26:16 of r0.0.
3258 */
3259 const fs_reg idx = bld.vgrf(BRW_REGISTER_TYPE_UD);
3260 bld.AND(idx, brw_uw1_reg(BRW_GENERAL_REGISTER_FILE, 0, 1),
3261 brw_imm_uw(0x7ff));
3262 return idx;
3263 } else {
3264 /* Pre-SNB we only ever render into the first layer of the framebuffer
3265 * since layered rendering is not implemented.
3266 */
3267 return brw_imm_ud(0);
3268 }
3269 }
3270
3271 /**
3272 * Fake non-coherent framebuffer read implemented using TXF to fetch from the
3273 * framebuffer at the current fragment coordinates and sample index.
3274 */
3275 fs_inst *
3276 fs_visitor::emit_non_coherent_fb_read(const fs_builder &bld, const fs_reg &dst,
3277 unsigned target)
3278 {
3279 const struct gen_device_info *devinfo = bld.shader->devinfo;
3280
3281 assert(bld.shader->stage == MESA_SHADER_FRAGMENT);
3282 const brw_wm_prog_key *wm_key =
3283 reinterpret_cast<const brw_wm_prog_key *>(key);
3284 assert(!wm_key->coherent_fb_fetch);
3285 const struct brw_wm_prog_data *wm_prog_data =
3286 brw_wm_prog_data(stage_prog_data);
3287
3288 /* Calculate the surface index relative to the start of the texture binding
3289 * table block, since that's what the texturing messages expect.
3290 */
3291 const unsigned surface = target +
3292 wm_prog_data->binding_table.render_target_read_start -
3293 wm_prog_data->base.binding_table.texture_start;
3294
3295 /* Calculate the fragment coordinates. */
3296 const fs_reg coords = bld.vgrf(BRW_REGISTER_TYPE_UD, 3);
3297 bld.MOV(offset(coords, bld, 0), pixel_x);
3298 bld.MOV(offset(coords, bld, 1), pixel_y);
3299 bld.MOV(offset(coords, bld, 2), fetch_render_target_array_index(bld));
3300
3301 /* Calculate the sample index and MCS payload when multisampling. Luckily
3302 * the MCS fetch message behaves deterministically for UMS surfaces, so it
3303 * shouldn't be necessary to recompile based on whether the framebuffer is
3304 * CMS or UMS.
3305 */
3306 if (wm_key->multisample_fbo &&
3307 nir_system_values[SYSTEM_VALUE_SAMPLE_ID].file == BAD_FILE)
3308 nir_system_values[SYSTEM_VALUE_SAMPLE_ID] = *emit_sampleid_setup();
3309
3310 const fs_reg sample = nir_system_values[SYSTEM_VALUE_SAMPLE_ID];
3311 const fs_reg mcs = wm_key->multisample_fbo ?
3312 emit_mcs_fetch(coords, 3, brw_imm_ud(surface), fs_reg()) : fs_reg();
3313
3314 /* Use either a normal or a CMS texel fetch message depending on whether
3315 * the framebuffer is single or multisample. On SKL+ use the wide CMS
3316 * message just in case the framebuffer uses 16x multisampling, it should
3317 * be equivalent to the normal CMS fetch for lower multisampling modes.
3318 */
3319 const opcode op = !wm_key->multisample_fbo ? SHADER_OPCODE_TXF_LOGICAL :
3320 devinfo->gen >= 9 ? SHADER_OPCODE_TXF_CMS_W_LOGICAL :
3321 SHADER_OPCODE_TXF_CMS_LOGICAL;
3322
3323 /* Emit the instruction. */
3324 fs_reg srcs[TEX_LOGICAL_NUM_SRCS];
3325 srcs[TEX_LOGICAL_SRC_COORDINATE] = coords;
3326 srcs[TEX_LOGICAL_SRC_LOD] = brw_imm_ud(0);
3327 srcs[TEX_LOGICAL_SRC_SAMPLE_INDEX] = sample;
3328 srcs[TEX_LOGICAL_SRC_MCS] = mcs;
3329 srcs[TEX_LOGICAL_SRC_SURFACE] = brw_imm_ud(surface);
3330 srcs[TEX_LOGICAL_SRC_SAMPLER] = brw_imm_ud(0);
3331 srcs[TEX_LOGICAL_SRC_COORD_COMPONENTS] = brw_imm_ud(3);
3332 srcs[TEX_LOGICAL_SRC_GRAD_COMPONENTS] = brw_imm_ud(0);
3333
3334 fs_inst *inst = bld.emit(op, dst, srcs, ARRAY_SIZE(srcs));
3335 inst->size_written = 4 * inst->dst.component_size(inst->exec_size);
3336
3337 return inst;
3338 }
3339
3340 /**
3341 * Actual coherent framebuffer read implemented using the native render target
3342 * read message. Requires SKL+.
3343 */
3344 static fs_inst *
3345 emit_coherent_fb_read(const fs_builder &bld, const fs_reg &dst, unsigned target)
3346 {
3347 assert(bld.shader->devinfo->gen >= 9);
3348 fs_inst *inst = bld.emit(FS_OPCODE_FB_READ_LOGICAL, dst);
3349 inst->target = target;
3350 inst->size_written = 4 * inst->dst.component_size(inst->exec_size);
3351
3352 return inst;
3353 }
3354
3355 static fs_reg
3356 alloc_temporary(const fs_builder &bld, unsigned size, fs_reg *regs, unsigned n)
3357 {
3358 if (n && regs[0].file != BAD_FILE) {
3359 return regs[0];
3360
3361 } else {
3362 const fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_F, size);
3363
3364 for (unsigned i = 0; i < n; i++)
3365 regs[i] = tmp;
3366
3367 return tmp;
3368 }
3369 }
3370
3371 static fs_reg
3372 alloc_frag_output(fs_visitor *v, unsigned location)
3373 {
3374 assert(v->stage == MESA_SHADER_FRAGMENT);
3375 const brw_wm_prog_key *const key =
3376 reinterpret_cast<const brw_wm_prog_key *>(v->key);
3377 const unsigned l = GET_FIELD(location, BRW_NIR_FRAG_OUTPUT_LOCATION);
3378 const unsigned i = GET_FIELD(location, BRW_NIR_FRAG_OUTPUT_INDEX);
3379
3380 if (i > 0 || (key->force_dual_color_blend && l == FRAG_RESULT_DATA1))
3381 return alloc_temporary(v->bld, 4, &v->dual_src_output, 1);
3382
3383 else if (l == FRAG_RESULT_COLOR)
3384 return alloc_temporary(v->bld, 4, v->outputs,
3385 MAX2(key->nr_color_regions, 1));
3386
3387 else if (l == FRAG_RESULT_DEPTH)
3388 return alloc_temporary(v->bld, 1, &v->frag_depth, 1);
3389
3390 else if (l == FRAG_RESULT_STENCIL)
3391 return alloc_temporary(v->bld, 1, &v->frag_stencil, 1);
3392
3393 else if (l == FRAG_RESULT_SAMPLE_MASK)
3394 return alloc_temporary(v->bld, 1, &v->sample_mask, 1);
3395
3396 else if (l >= FRAG_RESULT_DATA0 &&
3397 l < FRAG_RESULT_DATA0 + BRW_MAX_DRAW_BUFFERS)
3398 return alloc_temporary(v->bld, 4,
3399 &v->outputs[l - FRAG_RESULT_DATA0], 1);
3400
3401 else
3402 unreachable("Invalid location");
3403 }
3404
3405 void
3406 fs_visitor::nir_emit_fs_intrinsic(const fs_builder &bld,
3407 nir_intrinsic_instr *instr)
3408 {
3409 assert(stage == MESA_SHADER_FRAGMENT);
3410
3411 fs_reg dest;
3412 if (nir_intrinsic_infos[instr->intrinsic].has_dest)
3413 dest = get_nir_dest(instr->dest);
3414
3415 switch (instr->intrinsic) {
3416 case nir_intrinsic_load_front_face:
3417 bld.MOV(retype(dest, BRW_REGISTER_TYPE_D),
3418 *emit_frontfacing_interpolation());
3419 break;
3420
3421 case nir_intrinsic_load_sample_pos: {
3422 fs_reg sample_pos = nir_system_values[SYSTEM_VALUE_SAMPLE_POS];
3423 assert(sample_pos.file != BAD_FILE);
3424 dest.type = sample_pos.type;
3425 bld.MOV(dest, sample_pos);
3426 bld.MOV(offset(dest, bld, 1), offset(sample_pos, bld, 1));
3427 break;
3428 }
3429
3430 case nir_intrinsic_load_layer_id:
3431 dest.type = BRW_REGISTER_TYPE_UD;
3432 bld.MOV(dest, fetch_render_target_array_index(bld));
3433 break;
3434
3435 case nir_intrinsic_is_helper_invocation: {
3436 /* Unlike the regular gl_HelperInvocation, that is defined at dispatch,
3437 * the helperInvocationEXT() (aka SpvOpIsHelperInvocationEXT) takes into
3438 * consideration demoted invocations. That information is stored in
3439 * f0.1.
3440 */
3441 dest.type = BRW_REGISTER_TYPE_UD;
3442
3443 bld.MOV(dest, brw_imm_ud(0));
3444
3445 fs_inst *mov = bld.MOV(dest, brw_imm_ud(~0));
3446 mov->predicate = BRW_PREDICATE_NORMAL;
3447 mov->predicate_inverse = true;
3448 mov->flag_subreg = sample_mask_flag_subreg(this);
3449 break;
3450 }
3451
3452 case nir_intrinsic_load_helper_invocation:
3453 case nir_intrinsic_load_sample_mask_in:
3454 case nir_intrinsic_load_sample_id: {
3455 gl_system_value sv = nir_system_value_from_intrinsic(instr->intrinsic);
3456 fs_reg val = nir_system_values[sv];
3457 assert(val.file != BAD_FILE);
3458 dest.type = val.type;
3459 bld.MOV(dest, val);
3460 break;
3461 }
3462
3463 case nir_intrinsic_store_output: {
3464 const fs_reg src = get_nir_src(instr->src[0]);
3465 const unsigned store_offset = nir_src_as_uint(instr->src[1]);
3466 const unsigned location = nir_intrinsic_base(instr) +
3467 SET_FIELD(store_offset, BRW_NIR_FRAG_OUTPUT_LOCATION);
3468 const fs_reg new_dest = retype(alloc_frag_output(this, location),
3469 src.type);
3470
3471 for (unsigned j = 0; j < instr->num_components; j++)
3472 bld.MOV(offset(new_dest, bld, nir_intrinsic_component(instr) + j),
3473 offset(src, bld, j));
3474
3475 break;
3476 }
3477
3478 case nir_intrinsic_load_output: {
3479 const unsigned l = GET_FIELD(nir_intrinsic_base(instr),
3480 BRW_NIR_FRAG_OUTPUT_LOCATION);
3481 assert(l >= FRAG_RESULT_DATA0);
3482 const unsigned load_offset = nir_src_as_uint(instr->src[0]);
3483 const unsigned target = l - FRAG_RESULT_DATA0 + load_offset;
3484 const fs_reg tmp = bld.vgrf(dest.type, 4);
3485
3486 if (reinterpret_cast<const brw_wm_prog_key *>(key)->coherent_fb_fetch)
3487 emit_coherent_fb_read(bld, tmp, target);
3488 else
3489 emit_non_coherent_fb_read(bld, tmp, target);
3490
3491 for (unsigned j = 0; j < instr->num_components; j++) {
3492 bld.MOV(offset(dest, bld, j),
3493 offset(tmp, bld, nir_intrinsic_component(instr) + j));
3494 }
3495
3496 break;
3497 }
3498
3499 case nir_intrinsic_demote:
3500 case nir_intrinsic_discard:
3501 case nir_intrinsic_demote_if:
3502 case nir_intrinsic_discard_if: {
3503 /* We track our discarded pixels in f0.1/f1.0. By predicating on it, we
3504 * can update just the flag bits that aren't yet discarded. If there's
3505 * no condition, we emit a CMP of g0 != g0, so all currently executing
3506 * channels will get turned off.
3507 */
3508 fs_inst *cmp = NULL;
3509 if (instr->intrinsic == nir_intrinsic_demote_if ||
3510 instr->intrinsic == nir_intrinsic_discard_if) {
3511 nir_alu_instr *alu = nir_src_as_alu_instr(instr->src[0]);
3512
3513 if (alu != NULL &&
3514 alu->op != nir_op_bcsel &&
3515 alu->op != nir_op_inot &&
3516 (devinfo->gen > 5 ||
3517 (alu->instr.pass_flags & BRW_NIR_BOOLEAN_MASK) != BRW_NIR_BOOLEAN_NEEDS_RESOLVE ||
3518 alu->op == nir_op_fne32 || alu->op == nir_op_feq32 ||
3519 alu->op == nir_op_flt32 || alu->op == nir_op_fge32 ||
3520 alu->op == nir_op_ine32 || alu->op == nir_op_ieq32 ||
3521 alu->op == nir_op_ilt32 || alu->op == nir_op_ige32 ||
3522 alu->op == nir_op_ult32 || alu->op == nir_op_uge32)) {
3523 /* Re-emit the instruction that generated the Boolean value, but
3524 * do not store it. Since this instruction will be conditional,
3525 * other instructions that want to use the real Boolean value may
3526 * get garbage. This was a problem for piglit's fs-discard-exit-2
3527 * test.
3528 *
3529 * Ideally we'd detect that the instruction cannot have a
3530 * conditional modifier before emitting the instructions. Alas,
3531 * that is nigh impossible. Instead, we're going to assume the
3532 * instruction (or last instruction) generated can have a
3533 * conditional modifier. If it cannot, fallback to the old-style
3534 * compare, and hope dead code elimination will clean up the
3535 * extra instructions generated.
3536 */
3537 nir_emit_alu(bld, alu, false);
3538
3539 cmp = (fs_inst *) instructions.get_tail();
3540 if (cmp->conditional_mod == BRW_CONDITIONAL_NONE) {
3541 if (cmp->can_do_cmod())
3542 cmp->conditional_mod = BRW_CONDITIONAL_Z;
3543 else
3544 cmp = NULL;
3545 } else {
3546 /* The old sequence that would have been generated is,
3547 * basically, bool_result == false. This is equivalent to
3548 * !bool_result, so negate the old modifier.
3549 */
3550 cmp->conditional_mod = brw_negate_cmod(cmp->conditional_mod);
3551 }
3552 }
3553
3554 if (cmp == NULL) {
3555 cmp = bld.CMP(bld.null_reg_f(), get_nir_src(instr->src[0]),
3556 brw_imm_d(0), BRW_CONDITIONAL_Z);
3557 }
3558 } else {
3559 fs_reg some_reg = fs_reg(retype(brw_vec8_grf(0, 0),
3560 BRW_REGISTER_TYPE_UW));
3561 cmp = bld.CMP(bld.null_reg_f(), some_reg, some_reg, BRW_CONDITIONAL_NZ);
3562 }
3563
3564 cmp->predicate = BRW_PREDICATE_NORMAL;
3565 cmp->flag_subreg = sample_mask_flag_subreg(this);
3566
3567 if (devinfo->gen >= 6) {
3568 /* Due to the way we implement discard, the jump will only happen
3569 * when the whole quad is discarded. So we can do this even for
3570 * demote as it won't break its uniformity promises.
3571 */
3572 emit_discard_jump();
3573 }
3574
3575 if (devinfo->gen < 7)
3576 limit_dispatch_width(
3577 16, "Fragment discard/demote not implemented in SIMD32 mode.\n");
3578 break;
3579 }
3580
3581 case nir_intrinsic_load_input: {
3582 /* load_input is only used for flat inputs */
3583 assert(nir_dest_bit_size(instr->dest) == 32);
3584 unsigned base = nir_intrinsic_base(instr);
3585 unsigned comp = nir_intrinsic_component(instr);
3586 unsigned num_components = instr->num_components;
3587
3588 /* Special case fields in the VUE header */
3589 if (base == VARYING_SLOT_LAYER)
3590 comp = 1;
3591 else if (base == VARYING_SLOT_VIEWPORT)
3592 comp = 2;
3593
3594 for (unsigned int i = 0; i < num_components; i++) {
3595 bld.MOV(offset(dest, bld, i),
3596 retype(component(interp_reg(base, comp + i), 3), dest.type));
3597 }
3598 break;
3599 }
3600
3601 case nir_intrinsic_load_fs_input_interp_deltas: {
3602 assert(stage == MESA_SHADER_FRAGMENT);
3603 assert(nir_src_as_uint(instr->src[0]) == 0);
3604 fs_reg interp = interp_reg(nir_intrinsic_base(instr),
3605 nir_intrinsic_component(instr));
3606 dest.type = BRW_REGISTER_TYPE_F;
3607 bld.MOV(offset(dest, bld, 0), component(interp, 3));
3608 bld.MOV(offset(dest, bld, 1), component(interp, 1));
3609 bld.MOV(offset(dest, bld, 2), component(interp, 0));
3610 break;
3611 }
3612
3613 case nir_intrinsic_load_barycentric_pixel:
3614 case nir_intrinsic_load_barycentric_centroid:
3615 case nir_intrinsic_load_barycentric_sample: {
3616 /* Use the delta_xy values computed from the payload */
3617 const glsl_interp_mode interp_mode =
3618 (enum glsl_interp_mode) nir_intrinsic_interp_mode(instr);
3619 enum brw_barycentric_mode bary =
3620 brw_barycentric_mode(interp_mode, instr->intrinsic);
3621 const fs_reg srcs[] = { offset(this->delta_xy[bary], bld, 0),
3622 offset(this->delta_xy[bary], bld, 1) };
3623 bld.LOAD_PAYLOAD(dest, srcs, ARRAY_SIZE(srcs), 0);
3624 break;
3625 }
3626
3627 case nir_intrinsic_load_barycentric_at_sample: {
3628 const glsl_interp_mode interpolation =
3629 (enum glsl_interp_mode) nir_intrinsic_interp_mode(instr);
3630
3631 if (nir_src_is_const(instr->src[0])) {
3632 unsigned msg_data = nir_src_as_uint(instr->src[0]) << 4;
3633
3634 emit_pixel_interpolater_send(bld,
3635 FS_OPCODE_INTERPOLATE_AT_SAMPLE,
3636 dest,
3637 fs_reg(), /* src */
3638 brw_imm_ud(msg_data),
3639 interpolation);
3640 } else {
3641 const fs_reg sample_src = retype(get_nir_src(instr->src[0]),
3642 BRW_REGISTER_TYPE_UD);
3643
3644 if (nir_src_is_dynamically_uniform(instr->src[0])) {
3645 const fs_reg sample_id = bld.emit_uniformize(sample_src);
3646 const fs_reg msg_data = vgrf(glsl_type::uint_type);
3647 bld.exec_all().group(1, 0)
3648 .SHL(msg_data, sample_id, brw_imm_ud(4u));
3649 emit_pixel_interpolater_send(bld,
3650 FS_OPCODE_INTERPOLATE_AT_SAMPLE,
3651 dest,
3652 fs_reg(), /* src */
3653 component(msg_data, 0),
3654 interpolation);
3655 } else {
3656 /* Make a loop that sends a message to the pixel interpolater
3657 * for the sample number in each live channel. If there are
3658 * multiple channels with the same sample number then these
3659 * will be handled simultaneously with a single interation of
3660 * the loop.
3661 */
3662 bld.emit(BRW_OPCODE_DO);
3663
3664 /* Get the next live sample number into sample_id_reg */
3665 const fs_reg sample_id = bld.emit_uniformize(sample_src);
3666
3667 /* Set the flag register so that we can perform the send
3668 * message on all channels that have the same sample number
3669 */
3670 bld.CMP(bld.null_reg_ud(),
3671 sample_src, sample_id,
3672 BRW_CONDITIONAL_EQ);
3673 const fs_reg msg_data = vgrf(glsl_type::uint_type);
3674 bld.exec_all().group(1, 0)
3675 .SHL(msg_data, sample_id, brw_imm_ud(4u));
3676 fs_inst *inst =
3677 emit_pixel_interpolater_send(bld,
3678 FS_OPCODE_INTERPOLATE_AT_SAMPLE,
3679 dest,
3680 fs_reg(), /* src */
3681 component(msg_data, 0),
3682 interpolation);
3683 set_predicate(BRW_PREDICATE_NORMAL, inst);
3684
3685 /* Continue the loop if there are any live channels left */
3686 set_predicate_inv(BRW_PREDICATE_NORMAL,
3687 true, /* inverse */
3688 bld.emit(BRW_OPCODE_WHILE));
3689 }
3690 }
3691 break;
3692 }
3693
3694 case nir_intrinsic_load_barycentric_at_offset: {
3695 const glsl_interp_mode interpolation =
3696 (enum glsl_interp_mode) nir_intrinsic_interp_mode(instr);
3697
3698 nir_const_value *const_offset = nir_src_as_const_value(instr->src[0]);
3699
3700 if (const_offset) {
3701 assert(nir_src_bit_size(instr->src[0]) == 32);
3702 unsigned off_x = MIN2((int)(const_offset[0].f32 * 16), 7) & 0xf;
3703 unsigned off_y = MIN2((int)(const_offset[1].f32 * 16), 7) & 0xf;
3704
3705 emit_pixel_interpolater_send(bld,
3706 FS_OPCODE_INTERPOLATE_AT_SHARED_OFFSET,
3707 dest,
3708 fs_reg(), /* src */
3709 brw_imm_ud(off_x | (off_y << 4)),
3710 interpolation);
3711 } else {
3712 fs_reg src = vgrf(glsl_type::ivec2_type);
3713 fs_reg offset_src = retype(get_nir_src(instr->src[0]),
3714 BRW_REGISTER_TYPE_F);
3715 for (int i = 0; i < 2; i++) {
3716 fs_reg temp = vgrf(glsl_type::float_type);
3717 bld.MUL(temp, offset(offset_src, bld, i), brw_imm_f(16.0f));
3718 fs_reg itemp = vgrf(glsl_type::int_type);
3719 /* float to int */
3720 bld.MOV(itemp, temp);
3721
3722 /* Clamp the upper end of the range to +7/16.
3723 * ARB_gpu_shader5 requires that we support a maximum offset
3724 * of +0.5, which isn't representable in a S0.4 value -- if
3725 * we didn't clamp it, we'd end up with -8/16, which is the
3726 * opposite of what the shader author wanted.
3727 *
3728 * This is legal due to ARB_gpu_shader5's quantization
3729 * rules:
3730 *
3731 * "Not all values of <offset> may be supported; x and y
3732 * offsets may be rounded to fixed-point values with the
3733 * number of fraction bits given by the
3734 * implementation-dependent constant
3735 * FRAGMENT_INTERPOLATION_OFFSET_BITS"
3736 */
3737 set_condmod(BRW_CONDITIONAL_L,
3738 bld.SEL(offset(src, bld, i), itemp, brw_imm_d(7)));
3739 }
3740
3741 const enum opcode opcode = FS_OPCODE_INTERPOLATE_AT_PER_SLOT_OFFSET;
3742 emit_pixel_interpolater_send(bld,
3743 opcode,
3744 dest,
3745 src,
3746 brw_imm_ud(0u),
3747 interpolation);
3748 }
3749 break;
3750 }
3751
3752 case nir_intrinsic_load_frag_coord:
3753 emit_fragcoord_interpolation(dest);
3754 break;
3755
3756 case nir_intrinsic_load_interpolated_input: {
3757 assert(instr->src[0].ssa &&
3758 instr->src[0].ssa->parent_instr->type == nir_instr_type_intrinsic);
3759 nir_intrinsic_instr *bary_intrinsic =
3760 nir_instr_as_intrinsic(instr->src[0].ssa->parent_instr);
3761 nir_intrinsic_op bary_intrin = bary_intrinsic->intrinsic;
3762 enum glsl_interp_mode interp_mode =
3763 (enum glsl_interp_mode) nir_intrinsic_interp_mode(bary_intrinsic);
3764 fs_reg dst_xy;
3765
3766 if (bary_intrin == nir_intrinsic_load_barycentric_at_offset ||
3767 bary_intrin == nir_intrinsic_load_barycentric_at_sample) {
3768 /* Use the result of the PI message. */
3769 dst_xy = retype(get_nir_src(instr->src[0]), BRW_REGISTER_TYPE_F);
3770 } else {
3771 /* Use the delta_xy values computed from the payload */
3772 enum brw_barycentric_mode bary =
3773 brw_barycentric_mode(interp_mode, bary_intrin);
3774 dst_xy = this->delta_xy[bary];
3775 }
3776
3777 for (unsigned int i = 0; i < instr->num_components; i++) {
3778 fs_reg interp =
3779 component(interp_reg(nir_intrinsic_base(instr),
3780 nir_intrinsic_component(instr) + i), 0);
3781 interp.type = BRW_REGISTER_TYPE_F;
3782 dest.type = BRW_REGISTER_TYPE_F;
3783
3784 if (devinfo->gen < 6 && interp_mode == INTERP_MODE_SMOOTH) {
3785 fs_reg tmp = vgrf(glsl_type::float_type);
3786 bld.emit(FS_OPCODE_LINTERP, tmp, dst_xy, interp);
3787 bld.MUL(offset(dest, bld, i), tmp, this->pixel_w);
3788 } else {
3789 bld.emit(FS_OPCODE_LINTERP, offset(dest, bld, i), dst_xy, interp);
3790 }
3791 }
3792 break;
3793 }
3794
3795 default:
3796 nir_emit_intrinsic(bld, instr);
3797 break;
3798 }
3799 }
3800
3801 void
3802 fs_visitor::nir_emit_cs_intrinsic(const fs_builder &bld,
3803 nir_intrinsic_instr *instr)
3804 {
3805 assert(stage == MESA_SHADER_COMPUTE);
3806 struct brw_cs_prog_data *cs_prog_data = brw_cs_prog_data(prog_data);
3807
3808 fs_reg dest;
3809 if (nir_intrinsic_infos[instr->intrinsic].has_dest)
3810 dest = get_nir_dest(instr->dest);
3811
3812 switch (instr->intrinsic) {
3813 case nir_intrinsic_control_barrier:
3814 /* The whole workgroup fits in a single HW thread, so all the
3815 * invocations are already executed lock-step. Instead of an actual
3816 * barrier just emit a scheduling fence, that will generate no code.
3817 */
3818 if (workgroup_size() <= dispatch_width) {
3819 bld.exec_all().group(1, 0).emit(FS_OPCODE_SCHEDULING_FENCE);
3820 break;
3821 }
3822
3823 emit_barrier();
3824 cs_prog_data->uses_barrier = true;
3825 break;
3826
3827 case nir_intrinsic_load_subgroup_id:
3828 bld.MOV(retype(dest, BRW_REGISTER_TYPE_UD), subgroup_id);
3829 break;
3830
3831 case nir_intrinsic_load_local_invocation_id:
3832 case nir_intrinsic_load_work_group_id: {
3833 gl_system_value sv = nir_system_value_from_intrinsic(instr->intrinsic);
3834 fs_reg val = nir_system_values[sv];
3835 assert(val.file != BAD_FILE);
3836 dest.type = val.type;
3837 for (unsigned i = 0; i < 3; i++)
3838 bld.MOV(offset(dest, bld, i), offset(val, bld, i));
3839 break;
3840 }
3841
3842 case nir_intrinsic_load_num_work_groups: {
3843 const unsigned surface =
3844 cs_prog_data->binding_table.work_groups_start;
3845
3846 cs_prog_data->uses_num_work_groups = true;
3847
3848 fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS];
3849 srcs[SURFACE_LOGICAL_SRC_SURFACE] = brw_imm_ud(surface);
3850 srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = brw_imm_ud(1);
3851 srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(1); /* num components */
3852
3853 /* Read the 3 GLuint components of gl_NumWorkGroups */
3854 for (unsigned i = 0; i < 3; i++) {
3855 srcs[SURFACE_LOGICAL_SRC_ADDRESS] = brw_imm_ud(i << 2);
3856 bld.emit(SHADER_OPCODE_UNTYPED_SURFACE_READ_LOGICAL,
3857 offset(dest, bld, i), srcs, SURFACE_LOGICAL_NUM_SRCS);
3858 }
3859 break;
3860 }
3861
3862 case nir_intrinsic_shared_atomic_add:
3863 case nir_intrinsic_shared_atomic_imin:
3864 case nir_intrinsic_shared_atomic_umin:
3865 case nir_intrinsic_shared_atomic_imax:
3866 case nir_intrinsic_shared_atomic_umax:
3867 case nir_intrinsic_shared_atomic_and:
3868 case nir_intrinsic_shared_atomic_or:
3869 case nir_intrinsic_shared_atomic_xor:
3870 case nir_intrinsic_shared_atomic_exchange:
3871 case nir_intrinsic_shared_atomic_comp_swap:
3872 nir_emit_shared_atomic(bld, brw_aop_for_nir_intrinsic(instr), instr);
3873 break;
3874 case nir_intrinsic_shared_atomic_fmin:
3875 case nir_intrinsic_shared_atomic_fmax:
3876 case nir_intrinsic_shared_atomic_fcomp_swap:
3877 nir_emit_shared_atomic_float(bld, brw_aop_for_nir_intrinsic(instr), instr);
3878 break;
3879
3880 case nir_intrinsic_load_shared: {
3881 assert(devinfo->gen >= 7);
3882 assert(stage == MESA_SHADER_COMPUTE);
3883
3884 const unsigned bit_size = nir_dest_bit_size(instr->dest);
3885 fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS];
3886 srcs[SURFACE_LOGICAL_SRC_SURFACE] = brw_imm_ud(GEN7_BTI_SLM);
3887 srcs[SURFACE_LOGICAL_SRC_ADDRESS] = get_nir_src(instr->src[0]);
3888 srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = brw_imm_ud(1);
3889
3890 /* Make dest unsigned because that's what the temporary will be */
3891 dest.type = brw_reg_type_from_bit_size(bit_size, BRW_REGISTER_TYPE_UD);
3892
3893 /* Read the vector */
3894 if (nir_intrinsic_align(instr) >= 4) {
3895 assert(nir_dest_bit_size(instr->dest) == 32);
3896 srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(instr->num_components);
3897 fs_inst *inst =
3898 bld.emit(SHADER_OPCODE_UNTYPED_SURFACE_READ_LOGICAL,
3899 dest, srcs, SURFACE_LOGICAL_NUM_SRCS);
3900 inst->size_written = instr->num_components * dispatch_width * 4;
3901 } else {
3902 assert(nir_dest_bit_size(instr->dest) <= 32);
3903 assert(nir_dest_num_components(instr->dest) == 1);
3904 srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(bit_size);
3905
3906 fs_reg read_result = bld.vgrf(BRW_REGISTER_TYPE_UD);
3907 bld.emit(SHADER_OPCODE_BYTE_SCATTERED_READ_LOGICAL,
3908 read_result, srcs, SURFACE_LOGICAL_NUM_SRCS);
3909 bld.MOV(dest, subscript(read_result, dest.type, 0));
3910 }
3911 break;
3912 }
3913
3914 case nir_intrinsic_store_shared: {
3915 assert(devinfo->gen >= 7);
3916 assert(stage == MESA_SHADER_COMPUTE);
3917
3918 const unsigned bit_size = nir_src_bit_size(instr->src[0]);
3919 fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS];
3920 srcs[SURFACE_LOGICAL_SRC_SURFACE] = brw_imm_ud(GEN7_BTI_SLM);
3921 srcs[SURFACE_LOGICAL_SRC_ADDRESS] = get_nir_src(instr->src[1]);
3922 srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = brw_imm_ud(1);
3923
3924 fs_reg data = get_nir_src(instr->src[0]);
3925 data.type = brw_reg_type_from_bit_size(bit_size, BRW_REGISTER_TYPE_UD);
3926
3927 assert(nir_intrinsic_write_mask(instr) ==
3928 (1u << instr->num_components) - 1);
3929 if (nir_intrinsic_align(instr) >= 4) {
3930 assert(nir_src_bit_size(instr->src[0]) == 32);
3931 assert(nir_src_num_components(instr->src[0]) <= 4);
3932 srcs[SURFACE_LOGICAL_SRC_DATA] = data;
3933 srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(instr->num_components);
3934 bld.emit(SHADER_OPCODE_UNTYPED_SURFACE_WRITE_LOGICAL,
3935 fs_reg(), srcs, SURFACE_LOGICAL_NUM_SRCS);
3936 } else {
3937 assert(nir_src_bit_size(instr->src[0]) <= 32);
3938 assert(nir_src_num_components(instr->src[0]) == 1);
3939 srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(bit_size);
3940
3941 srcs[SURFACE_LOGICAL_SRC_DATA] = bld.vgrf(BRW_REGISTER_TYPE_UD);
3942 bld.MOV(srcs[SURFACE_LOGICAL_SRC_DATA], data);
3943
3944 bld.emit(SHADER_OPCODE_BYTE_SCATTERED_WRITE_LOGICAL,
3945 fs_reg(), srcs, SURFACE_LOGICAL_NUM_SRCS);
3946 }
3947 break;
3948 }
3949
3950 default:
3951 nir_emit_intrinsic(bld, instr);
3952 break;
3953 }
3954 }
3955
3956 static fs_reg
3957 brw_nir_reduction_op_identity(const fs_builder &bld,
3958 nir_op op, brw_reg_type type)
3959 {
3960 nir_const_value value = nir_alu_binop_identity(op, type_sz(type) * 8);
3961 switch (type_sz(type)) {
3962 case 1:
3963 if (type == BRW_REGISTER_TYPE_UB) {
3964 return brw_imm_uw(value.u8);
3965 } else {
3966 assert(type == BRW_REGISTER_TYPE_B);
3967 return brw_imm_w(value.i8);
3968 }
3969 case 2:
3970 return retype(brw_imm_uw(value.u16), type);
3971 case 4:
3972 return retype(brw_imm_ud(value.u32), type);
3973 case 8:
3974 if (type == BRW_REGISTER_TYPE_DF)
3975 return setup_imm_df(bld, value.f64);
3976 else
3977 return retype(brw_imm_u64(value.u64), type);
3978 default:
3979 unreachable("Invalid type size");
3980 }
3981 }
3982
3983 static opcode
3984 brw_op_for_nir_reduction_op(nir_op op)
3985 {
3986 switch (op) {
3987 case nir_op_iadd: return BRW_OPCODE_ADD;
3988 case nir_op_fadd: return BRW_OPCODE_ADD;
3989 case nir_op_imul: return BRW_OPCODE_MUL;
3990 case nir_op_fmul: return BRW_OPCODE_MUL;
3991 case nir_op_imin: return BRW_OPCODE_SEL;
3992 case nir_op_umin: return BRW_OPCODE_SEL;
3993 case nir_op_fmin: return BRW_OPCODE_SEL;
3994 case nir_op_imax: return BRW_OPCODE_SEL;
3995 case nir_op_umax: return BRW_OPCODE_SEL;
3996 case nir_op_fmax: return BRW_OPCODE_SEL;
3997 case nir_op_iand: return BRW_OPCODE_AND;
3998 case nir_op_ior: return BRW_OPCODE_OR;
3999 case nir_op_ixor: return BRW_OPCODE_XOR;
4000 default:
4001 unreachable("Invalid reduction operation");
4002 }
4003 }
4004
4005 static brw_conditional_mod
4006 brw_cond_mod_for_nir_reduction_op(nir_op op)
4007 {
4008 switch (op) {
4009 case nir_op_iadd: return BRW_CONDITIONAL_NONE;
4010 case nir_op_fadd: return BRW_CONDITIONAL_NONE;
4011 case nir_op_imul: return BRW_CONDITIONAL_NONE;
4012 case nir_op_fmul: return BRW_CONDITIONAL_NONE;
4013 case nir_op_imin: return BRW_CONDITIONAL_L;
4014 case nir_op_umin: return BRW_CONDITIONAL_L;
4015 case nir_op_fmin: return BRW_CONDITIONAL_L;
4016 case nir_op_imax: return BRW_CONDITIONAL_GE;
4017 case nir_op_umax: return BRW_CONDITIONAL_GE;
4018 case nir_op_fmax: return BRW_CONDITIONAL_GE;
4019 case nir_op_iand: return BRW_CONDITIONAL_NONE;
4020 case nir_op_ior: return BRW_CONDITIONAL_NONE;
4021 case nir_op_ixor: return BRW_CONDITIONAL_NONE;
4022 default:
4023 unreachable("Invalid reduction operation");
4024 }
4025 }
4026
4027 fs_reg
4028 fs_visitor::get_nir_image_intrinsic_image(const brw::fs_builder &bld,
4029 nir_intrinsic_instr *instr)
4030 {
4031 fs_reg image = retype(get_nir_src_imm(instr->src[0]), BRW_REGISTER_TYPE_UD);
4032 fs_reg surf_index = image;
4033
4034 if (stage_prog_data->binding_table.image_start > 0) {
4035 if (image.file == BRW_IMMEDIATE_VALUE) {
4036 surf_index =
4037 brw_imm_ud(image.d + stage_prog_data->binding_table.image_start);
4038 } else {
4039 surf_index = vgrf(glsl_type::uint_type);
4040 bld.ADD(surf_index, image,
4041 brw_imm_d(stage_prog_data->binding_table.image_start));
4042 }
4043 }
4044
4045 return bld.emit_uniformize(surf_index);
4046 }
4047
4048 fs_reg
4049 fs_visitor::get_nir_ssbo_intrinsic_index(const brw::fs_builder &bld,
4050 nir_intrinsic_instr *instr)
4051 {
4052 /* SSBO stores are weird in that their index is in src[1] */
4053 const unsigned src = instr->intrinsic == nir_intrinsic_store_ssbo ? 1 : 0;
4054
4055 fs_reg surf_index;
4056 if (nir_src_is_const(instr->src[src])) {
4057 unsigned index = stage_prog_data->binding_table.ssbo_start +
4058 nir_src_as_uint(instr->src[src]);
4059 surf_index = brw_imm_ud(index);
4060 } else {
4061 surf_index = vgrf(glsl_type::uint_type);
4062 bld.ADD(surf_index, get_nir_src(instr->src[src]),
4063 brw_imm_ud(stage_prog_data->binding_table.ssbo_start));
4064 }
4065
4066 return bld.emit_uniformize(surf_index);
4067 }
4068
4069 /**
4070 * The offsets we get from NIR act as if each SIMD channel has it's own blob
4071 * of contiguous space. However, if we actually place each SIMD channel in
4072 * it's own space, we end up with terrible cache performance because each SIMD
4073 * channel accesses a different cache line even when they're all accessing the
4074 * same byte offset. To deal with this problem, we swizzle the address using
4075 * a simple algorithm which ensures that any time a SIMD message reads or
4076 * writes the same address, it's all in the same cache line. We have to keep
4077 * the bottom two bits fixed so that we can read/write up to a dword at a time
4078 * and the individual element is contiguous. We do this by splitting the
4079 * address as follows:
4080 *
4081 * 31 4-6 2 0
4082 * +-------------------------------+------------+----------+
4083 * | Hi address bits | chan index | addr low |
4084 * +-------------------------------+------------+----------+
4085 *
4086 * In other words, the bottom two address bits stay, and the top 30 get
4087 * shifted up so that we can stick the SIMD channel index in the middle. This
4088 * way, we can access 8, 16, or 32-bit elements and, when accessing a 32-bit
4089 * at the same logical offset, the scratch read/write instruction acts on
4090 * continuous elements and we get good cache locality.
4091 */
4092 fs_reg
4093 fs_visitor::swizzle_nir_scratch_addr(const brw::fs_builder &bld,
4094 const fs_reg &nir_addr,
4095 bool in_dwords)
4096 {
4097 const fs_reg &chan_index =
4098 nir_system_values[SYSTEM_VALUE_SUBGROUP_INVOCATION];
4099 const unsigned chan_index_bits = ffs(dispatch_width) - 1;
4100
4101 fs_reg addr = bld.vgrf(BRW_REGISTER_TYPE_UD);
4102 if (in_dwords) {
4103 /* In this case, we know the address is aligned to a DWORD and we want
4104 * the final address in DWORDs.
4105 */
4106 bld.SHL(addr, nir_addr, brw_imm_ud(chan_index_bits - 2));
4107 bld.OR(addr, addr, chan_index);
4108 } else {
4109 /* This case substantially more annoying because we have to pay
4110 * attention to those pesky two bottom bits.
4111 */
4112 fs_reg addr_hi = bld.vgrf(BRW_REGISTER_TYPE_UD);
4113 bld.AND(addr_hi, nir_addr, brw_imm_ud(~0x3u));
4114 bld.SHL(addr_hi, addr_hi, brw_imm_ud(chan_index_bits));
4115 fs_reg chan_addr = bld.vgrf(BRW_REGISTER_TYPE_UD);
4116 bld.SHL(chan_addr, chan_index, brw_imm_ud(2));
4117 bld.AND(addr, nir_addr, brw_imm_ud(0x3u));
4118 bld.OR(addr, addr, addr_hi);
4119 bld.OR(addr, addr, chan_addr);
4120 }
4121 return addr;
4122 }
4123
4124 void
4125 fs_visitor::nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr)
4126 {
4127 fs_reg dest;
4128 if (nir_intrinsic_infos[instr->intrinsic].has_dest)
4129 dest = get_nir_dest(instr->dest);
4130
4131 switch (instr->intrinsic) {
4132 case nir_intrinsic_image_load:
4133 case nir_intrinsic_image_store:
4134 case nir_intrinsic_image_atomic_add:
4135 case nir_intrinsic_image_atomic_imin:
4136 case nir_intrinsic_image_atomic_umin:
4137 case nir_intrinsic_image_atomic_imax:
4138 case nir_intrinsic_image_atomic_umax:
4139 case nir_intrinsic_image_atomic_and:
4140 case nir_intrinsic_image_atomic_or:
4141 case nir_intrinsic_image_atomic_xor:
4142 case nir_intrinsic_image_atomic_exchange:
4143 case nir_intrinsic_image_atomic_comp_swap:
4144 case nir_intrinsic_bindless_image_load:
4145 case nir_intrinsic_bindless_image_store:
4146 case nir_intrinsic_bindless_image_atomic_add:
4147 case nir_intrinsic_bindless_image_atomic_imin:
4148 case nir_intrinsic_bindless_image_atomic_umin:
4149 case nir_intrinsic_bindless_image_atomic_imax:
4150 case nir_intrinsic_bindless_image_atomic_umax:
4151 case nir_intrinsic_bindless_image_atomic_and:
4152 case nir_intrinsic_bindless_image_atomic_or:
4153 case nir_intrinsic_bindless_image_atomic_xor:
4154 case nir_intrinsic_bindless_image_atomic_exchange:
4155 case nir_intrinsic_bindless_image_atomic_comp_swap: {
4156 if (stage == MESA_SHADER_FRAGMENT &&
4157 instr->intrinsic != nir_intrinsic_image_load)
4158 brw_wm_prog_data(prog_data)->has_side_effects = true;
4159
4160 /* Get some metadata from the image intrinsic. */
4161 const nir_intrinsic_info *info = &nir_intrinsic_infos[instr->intrinsic];
4162
4163 fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS];
4164
4165 switch (instr->intrinsic) {
4166 case nir_intrinsic_image_load:
4167 case nir_intrinsic_image_store:
4168 case nir_intrinsic_image_atomic_add:
4169 case nir_intrinsic_image_atomic_imin:
4170 case nir_intrinsic_image_atomic_umin:
4171 case nir_intrinsic_image_atomic_imax:
4172 case nir_intrinsic_image_atomic_umax:
4173 case nir_intrinsic_image_atomic_and:
4174 case nir_intrinsic_image_atomic_or:
4175 case nir_intrinsic_image_atomic_xor:
4176 case nir_intrinsic_image_atomic_exchange:
4177 case nir_intrinsic_image_atomic_comp_swap:
4178 srcs[SURFACE_LOGICAL_SRC_SURFACE] =
4179 get_nir_image_intrinsic_image(bld, instr);
4180 break;
4181
4182 default:
4183 /* Bindless */
4184 srcs[SURFACE_LOGICAL_SRC_SURFACE_HANDLE] =
4185 bld.emit_uniformize(get_nir_src(instr->src[0]));
4186 break;
4187 }
4188
4189 srcs[SURFACE_LOGICAL_SRC_ADDRESS] = get_nir_src(instr->src[1]);
4190 srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] =
4191 brw_imm_ud(nir_image_intrinsic_coord_components(instr));
4192
4193 /* Emit an image load, store or atomic op. */
4194 if (instr->intrinsic == nir_intrinsic_image_load ||
4195 instr->intrinsic == nir_intrinsic_bindless_image_load) {
4196 srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(instr->num_components);
4197 fs_inst *inst =
4198 bld.emit(SHADER_OPCODE_TYPED_SURFACE_READ_LOGICAL,
4199 dest, srcs, SURFACE_LOGICAL_NUM_SRCS);
4200 inst->size_written = instr->num_components * dispatch_width * 4;
4201 } else if (instr->intrinsic == nir_intrinsic_image_store ||
4202 instr->intrinsic == nir_intrinsic_bindless_image_store) {
4203 srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(instr->num_components);
4204 srcs[SURFACE_LOGICAL_SRC_DATA] = get_nir_src(instr->src[3]);
4205 bld.emit(SHADER_OPCODE_TYPED_SURFACE_WRITE_LOGICAL,
4206 fs_reg(), srcs, SURFACE_LOGICAL_NUM_SRCS);
4207 } else {
4208 unsigned num_srcs = info->num_srcs;
4209 int op = brw_aop_for_nir_intrinsic(instr);
4210 if (op == BRW_AOP_INC || op == BRW_AOP_DEC) {
4211 assert(num_srcs == 4);
4212 num_srcs = 3;
4213 }
4214
4215 srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(op);
4216
4217 fs_reg data;
4218 if (num_srcs >= 4)
4219 data = get_nir_src(instr->src[3]);
4220 if (num_srcs >= 5) {
4221 fs_reg tmp = bld.vgrf(data.type, 2);
4222 fs_reg sources[2] = { data, get_nir_src(instr->src[4]) };
4223 bld.LOAD_PAYLOAD(tmp, sources, 2, 0);
4224 data = tmp;
4225 }
4226 srcs[SURFACE_LOGICAL_SRC_DATA] = data;
4227
4228 bld.emit(SHADER_OPCODE_TYPED_ATOMIC_LOGICAL,
4229 dest, srcs, SURFACE_LOGICAL_NUM_SRCS);
4230 }
4231 break;
4232 }
4233
4234 case nir_intrinsic_image_size:
4235 case nir_intrinsic_bindless_image_size: {
4236 /* Unlike the [un]typed load and store opcodes, the TXS that this turns
4237 * into will handle the binding table index for us in the geneerator.
4238 * Incidentally, this means that we can handle bindless with exactly the
4239 * same code.
4240 */
4241 fs_reg image = retype(get_nir_src_imm(instr->src[0]),
4242 BRW_REGISTER_TYPE_UD);
4243 image = bld.emit_uniformize(image);
4244
4245 fs_reg srcs[TEX_LOGICAL_NUM_SRCS];
4246 if (instr->intrinsic == nir_intrinsic_image_size)
4247 srcs[TEX_LOGICAL_SRC_SURFACE] = image;
4248 else
4249 srcs[TEX_LOGICAL_SRC_SURFACE_HANDLE] = image;
4250 srcs[TEX_LOGICAL_SRC_SAMPLER] = brw_imm_d(0);
4251 srcs[TEX_LOGICAL_SRC_COORD_COMPONENTS] = brw_imm_d(0);
4252 srcs[TEX_LOGICAL_SRC_GRAD_COMPONENTS] = brw_imm_d(0);
4253
4254 /* Since the image size is always uniform, we can just emit a SIMD8
4255 * query instruction and splat the result out.
4256 */
4257 const fs_builder ubld = bld.exec_all().group(8, 0);
4258
4259 fs_reg tmp = ubld.vgrf(BRW_REGISTER_TYPE_UD, 4);
4260 fs_inst *inst = ubld.emit(SHADER_OPCODE_IMAGE_SIZE_LOGICAL,
4261 tmp, srcs, ARRAY_SIZE(srcs));
4262 inst->size_written = 4 * REG_SIZE;
4263
4264 for (unsigned c = 0; c < instr->dest.ssa.num_components; ++c) {
4265 if (c == 2 && nir_intrinsic_image_dim(instr) == GLSL_SAMPLER_DIM_CUBE) {
4266 bld.emit(SHADER_OPCODE_INT_QUOTIENT,
4267 offset(retype(dest, tmp.type), bld, c),
4268 component(offset(tmp, ubld, c), 0), brw_imm_ud(6));
4269 } else {
4270 bld.MOV(offset(retype(dest, tmp.type), bld, c),
4271 component(offset(tmp, ubld, c), 0));
4272 }
4273 }
4274 break;
4275 }
4276
4277 case nir_intrinsic_image_load_raw_intel: {
4278 fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS];
4279 srcs[SURFACE_LOGICAL_SRC_SURFACE] =
4280 get_nir_image_intrinsic_image(bld, instr);
4281 srcs[SURFACE_LOGICAL_SRC_ADDRESS] = get_nir_src(instr->src[1]);
4282 srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = brw_imm_ud(1);
4283 srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(instr->num_components);
4284
4285 fs_inst *inst =
4286 bld.emit(SHADER_OPCODE_UNTYPED_SURFACE_READ_LOGICAL,
4287 dest, srcs, SURFACE_LOGICAL_NUM_SRCS);
4288 inst->size_written = instr->num_components * dispatch_width * 4;
4289 break;
4290 }
4291
4292 case nir_intrinsic_image_store_raw_intel: {
4293 if (stage == MESA_SHADER_FRAGMENT)
4294 brw_wm_prog_data(prog_data)->has_side_effects = true;
4295
4296 fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS];
4297 srcs[SURFACE_LOGICAL_SRC_SURFACE] =
4298 get_nir_image_intrinsic_image(bld, instr);
4299 srcs[SURFACE_LOGICAL_SRC_ADDRESS] = get_nir_src(instr->src[1]);
4300 srcs[SURFACE_LOGICAL_SRC_DATA] = get_nir_src(instr->src[2]);
4301 srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = brw_imm_ud(1);
4302 srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(instr->num_components);
4303
4304 bld.emit(SHADER_OPCODE_UNTYPED_SURFACE_WRITE_LOGICAL,
4305 fs_reg(), srcs, SURFACE_LOGICAL_NUM_SRCS);
4306 break;
4307 }
4308
4309 case nir_intrinsic_scoped_memory_barrier:
4310 case nir_intrinsic_group_memory_barrier:
4311 case nir_intrinsic_memory_barrier_shared:
4312 case nir_intrinsic_memory_barrier_buffer:
4313 case nir_intrinsic_memory_barrier_image:
4314 case nir_intrinsic_memory_barrier: {
4315 bool l3_fence, slm_fence;
4316 if (instr->intrinsic == nir_intrinsic_scoped_memory_barrier) {
4317 nir_variable_mode modes = nir_intrinsic_memory_modes(instr);
4318 l3_fence = modes & (nir_var_shader_out |
4319 nir_var_mem_ssbo |
4320 nir_var_mem_global);
4321 slm_fence = modes & nir_var_mem_shared;
4322 } else {
4323 l3_fence = instr->intrinsic != nir_intrinsic_memory_barrier_shared;
4324 slm_fence = instr->intrinsic == nir_intrinsic_group_memory_barrier ||
4325 instr->intrinsic == nir_intrinsic_memory_barrier ||
4326 instr->intrinsic == nir_intrinsic_memory_barrier_shared;
4327 }
4328
4329 if (stage != MESA_SHADER_COMPUTE)
4330 slm_fence = false;
4331
4332 /* If the workgroup fits in a single HW thread, the messages for SLM are
4333 * processed in-order and the shader itself is already synchronized so
4334 * the memory fence is not necessary.
4335 *
4336 * TODO: Check if applies for many HW threads sharing same Data Port.
4337 */
4338 if (slm_fence && workgroup_size() <= dispatch_width)
4339 slm_fence = false;
4340
4341 /* Prior to Gen11, there's only L3 fence, so emit that instead. */
4342 if (slm_fence && devinfo->gen < 11) {
4343 slm_fence = false;
4344 l3_fence = true;
4345 }
4346
4347 /* Be conservative in Gen11+ and always stall in a fence. Since there
4348 * are two different fences, and shader might want to synchronize
4349 * between them.
4350 *
4351 * TODO: Improve NIR so that scope and visibility information for the
4352 * barriers is available here to make a better decision.
4353 *
4354 * TODO: When emitting more than one fence, it might help emit all
4355 * the fences first and then generate the stall moves.
4356 */
4357 const bool stall = devinfo->gen >= 11;
4358
4359 const fs_builder ubld = bld.group(8, 0);
4360 const fs_reg tmp = ubld.vgrf(BRW_REGISTER_TYPE_UD, 2);
4361
4362 if (l3_fence) {
4363 ubld.emit(SHADER_OPCODE_MEMORY_FENCE, tmp,
4364 brw_vec8_grf(0, 0), brw_imm_ud(stall),
4365 /* bti */ brw_imm_ud(0))
4366 ->size_written = 2 * REG_SIZE;
4367 }
4368
4369 if (slm_fence) {
4370 ubld.emit(SHADER_OPCODE_MEMORY_FENCE, tmp,
4371 brw_vec8_grf(0, 0), brw_imm_ud(stall),
4372 brw_imm_ud(GEN7_BTI_SLM))
4373 ->size_written = 2 * REG_SIZE;
4374 }
4375
4376 if (!l3_fence && !slm_fence)
4377 ubld.emit(FS_OPCODE_SCHEDULING_FENCE);
4378
4379 break;
4380 }
4381
4382 case nir_intrinsic_memory_barrier_tcs_patch:
4383 break;
4384
4385 case nir_intrinsic_shader_clock: {
4386 /* We cannot do anything if there is an event, so ignore it for now */
4387 const fs_reg shader_clock = get_timestamp(bld);
4388 const fs_reg srcs[] = { component(shader_clock, 0),
4389 component(shader_clock, 1) };
4390 bld.LOAD_PAYLOAD(dest, srcs, ARRAY_SIZE(srcs), 0);
4391 break;
4392 }
4393
4394 case nir_intrinsic_image_samples:
4395 /* The driver does not support multi-sampled images. */
4396 bld.MOV(retype(dest, BRW_REGISTER_TYPE_D), brw_imm_d(1));
4397 break;
4398
4399 case nir_intrinsic_load_uniform: {
4400 /* Offsets are in bytes but they should always aligned to
4401 * the type size
4402 */
4403 assert(instr->const_index[0] % 4 == 0 ||
4404 instr->const_index[0] % type_sz(dest.type) == 0);
4405
4406 fs_reg src(UNIFORM, instr->const_index[0] / 4, dest.type);
4407
4408 if (nir_src_is_const(instr->src[0])) {
4409 unsigned load_offset = nir_src_as_uint(instr->src[0]);
4410 assert(load_offset % type_sz(dest.type) == 0);
4411 /* For 16-bit types we add the module of the const_index[0]
4412 * offset to access to not 32-bit aligned element
4413 */
4414 src.offset = load_offset + instr->const_index[0] % 4;
4415
4416 for (unsigned j = 0; j < instr->num_components; j++) {
4417 bld.MOV(offset(dest, bld, j), offset(src, bld, j));
4418 }
4419 } else {
4420 fs_reg indirect = retype(get_nir_src(instr->src[0]),
4421 BRW_REGISTER_TYPE_UD);
4422
4423 /* We need to pass a size to the MOV_INDIRECT but we don't want it to
4424 * go past the end of the uniform. In order to keep the n'th
4425 * component from running past, we subtract off the size of all but
4426 * one component of the vector.
4427 */
4428 assert(instr->const_index[1] >=
4429 instr->num_components * (int) type_sz(dest.type));
4430 unsigned read_size = instr->const_index[1] -
4431 (instr->num_components - 1) * type_sz(dest.type);
4432
4433 bool supports_64bit_indirects =
4434 !devinfo->is_cherryview && !gen_device_info_is_9lp(devinfo);
4435
4436 if (type_sz(dest.type) != 8 || supports_64bit_indirects) {
4437 for (unsigned j = 0; j < instr->num_components; j++) {
4438 bld.emit(SHADER_OPCODE_MOV_INDIRECT,
4439 offset(dest, bld, j), offset(src, bld, j),
4440 indirect, brw_imm_ud(read_size));
4441 }
4442 } else {
4443 const unsigned num_mov_indirects =
4444 type_sz(dest.type) / type_sz(BRW_REGISTER_TYPE_UD);
4445 /* We read a little bit less per MOV INDIRECT, as they are now
4446 * 32-bits ones instead of 64-bit. Fix read_size then.
4447 */
4448 const unsigned read_size_32bit = read_size -
4449 (num_mov_indirects - 1) * type_sz(BRW_REGISTER_TYPE_UD);
4450 for (unsigned j = 0; j < instr->num_components; j++) {
4451 for (unsigned i = 0; i < num_mov_indirects; i++) {
4452 bld.emit(SHADER_OPCODE_MOV_INDIRECT,
4453 subscript(offset(dest, bld, j), BRW_REGISTER_TYPE_UD, i),
4454 subscript(offset(src, bld, j), BRW_REGISTER_TYPE_UD, i),
4455 indirect, brw_imm_ud(read_size_32bit));
4456 }
4457 }
4458 }
4459 }
4460 break;
4461 }
4462
4463 case nir_intrinsic_load_ubo: {
4464 fs_reg surf_index;
4465 if (nir_src_is_const(instr->src[0])) {
4466 const unsigned index = stage_prog_data->binding_table.ubo_start +
4467 nir_src_as_uint(instr->src[0]);
4468 surf_index = brw_imm_ud(index);
4469 } else {
4470 /* The block index is not a constant. Evaluate the index expression
4471 * per-channel and add the base UBO index; we have to select a value
4472 * from any live channel.
4473 */
4474 surf_index = vgrf(glsl_type::uint_type);
4475 bld.ADD(surf_index, get_nir_src(instr->src[0]),
4476 brw_imm_ud(stage_prog_data->binding_table.ubo_start));
4477 surf_index = bld.emit_uniformize(surf_index);
4478 }
4479
4480 if (!nir_src_is_const(instr->src[1])) {
4481 fs_reg base_offset = retype(get_nir_src(instr->src[1]),
4482 BRW_REGISTER_TYPE_UD);
4483
4484 for (int i = 0; i < instr->num_components; i++)
4485 VARYING_PULL_CONSTANT_LOAD(bld, offset(dest, bld, i), surf_index,
4486 base_offset, i * type_sz(dest.type));
4487
4488 prog_data->has_ubo_pull = true;
4489 } else {
4490 /* Even if we are loading doubles, a pull constant load will load
4491 * a 32-bit vec4, so should only reserve vgrf space for that. If we
4492 * need to load a full dvec4 we will have to emit 2 loads. This is
4493 * similar to demote_pull_constants(), except that in that case we
4494 * see individual accesses to each component of the vector and then
4495 * we let CSE deal with duplicate loads. Here we see a vector access
4496 * and we have to split it if necessary.
4497 */
4498 const unsigned type_size = type_sz(dest.type);
4499 const unsigned load_offset = nir_src_as_uint(instr->src[1]);
4500
4501 /* See if we've selected this as a push constant candidate */
4502 if (nir_src_is_const(instr->src[0])) {
4503 const unsigned ubo_block = nir_src_as_uint(instr->src[0]);
4504 const unsigned offset_256b = load_offset / 32;
4505
4506 fs_reg push_reg;
4507 for (int i = 0; i < 4; i++) {
4508 const struct brw_ubo_range *range = &prog_data->ubo_ranges[i];
4509 if (range->block == ubo_block &&
4510 offset_256b >= range->start &&
4511 offset_256b < range->start + range->length) {
4512
4513 push_reg = fs_reg(UNIFORM, UBO_START + i, dest.type);
4514 push_reg.offset = load_offset - 32 * range->start;
4515 break;
4516 }
4517 }
4518
4519 if (push_reg.file != BAD_FILE) {
4520 for (unsigned i = 0; i < instr->num_components; i++) {
4521 bld.MOV(offset(dest, bld, i),
4522 byte_offset(push_reg, i * type_size));
4523 }
4524 break;
4525 }
4526 }
4527
4528 prog_data->has_ubo_pull = true;
4529
4530 const unsigned block_sz = 64; /* Fetch one cacheline at a time. */
4531 const fs_builder ubld = bld.exec_all().group(block_sz / 4, 0);
4532 const fs_reg packed_consts = ubld.vgrf(BRW_REGISTER_TYPE_UD);
4533
4534 for (unsigned c = 0; c < instr->num_components;) {
4535 const unsigned base = load_offset + c * type_size;
4536 /* Number of usable components in the next block-aligned load. */
4537 const unsigned count = MIN2(instr->num_components - c,
4538 (block_sz - base % block_sz) / type_size);
4539
4540 ubld.emit(FS_OPCODE_UNIFORM_PULL_CONSTANT_LOAD,
4541 packed_consts, surf_index,
4542 brw_imm_ud(base & ~(block_sz - 1)));
4543
4544 const fs_reg consts =
4545 retype(byte_offset(packed_consts, base & (block_sz - 1)),
4546 dest.type);
4547
4548 for (unsigned d = 0; d < count; d++)
4549 bld.MOV(offset(dest, bld, c + d), component(consts, d));
4550
4551 c += count;
4552 }
4553 }
4554 break;
4555 }
4556
4557 case nir_intrinsic_load_global: {
4558 assert(devinfo->gen >= 8);
4559
4560 if (nir_intrinsic_align(instr) >= 4) {
4561 assert(nir_dest_bit_size(instr->dest) == 32);
4562 fs_inst *inst = bld.emit(SHADER_OPCODE_A64_UNTYPED_READ_LOGICAL,
4563 dest,
4564 get_nir_src(instr->src[0]), /* Address */
4565 fs_reg(), /* No source data */
4566 brw_imm_ud(instr->num_components));
4567 inst->size_written = instr->num_components *
4568 inst->dst.component_size(inst->exec_size);
4569 } else {
4570 const unsigned bit_size = nir_dest_bit_size(instr->dest);
4571 assert(bit_size <= 32);
4572 assert(nir_dest_num_components(instr->dest) == 1);
4573 fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_UD);
4574 bld.emit(SHADER_OPCODE_A64_BYTE_SCATTERED_READ_LOGICAL,
4575 tmp,
4576 get_nir_src(instr->src[0]), /* Address */
4577 fs_reg(), /* No source data */
4578 brw_imm_ud(bit_size));
4579 bld.MOV(dest, subscript(tmp, dest.type, 0));
4580 }
4581 break;
4582 }
4583
4584 case nir_intrinsic_store_global:
4585 assert(devinfo->gen >= 8);
4586
4587 if (stage == MESA_SHADER_FRAGMENT)
4588 brw_wm_prog_data(prog_data)->has_side_effects = true;
4589
4590 if (nir_intrinsic_align(instr) >= 4) {
4591 assert(nir_src_bit_size(instr->src[0]) == 32);
4592 bld.emit(SHADER_OPCODE_A64_UNTYPED_WRITE_LOGICAL,
4593 fs_reg(),
4594 get_nir_src(instr->src[1]), /* Address */
4595 get_nir_src(instr->src[0]), /* Data */
4596 brw_imm_ud(instr->num_components));
4597 } else {
4598 const unsigned bit_size = nir_src_bit_size(instr->src[0]);
4599 assert(bit_size <= 32);
4600 assert(nir_src_num_components(instr->src[0]) == 1);
4601 brw_reg_type data_type =
4602 brw_reg_type_from_bit_size(bit_size, BRW_REGISTER_TYPE_UD);
4603 fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_UD);
4604 bld.MOV(tmp, retype(get_nir_src(instr->src[0]), data_type));
4605 bld.emit(SHADER_OPCODE_A64_BYTE_SCATTERED_WRITE_LOGICAL,
4606 fs_reg(),
4607 get_nir_src(instr->src[1]), /* Address */
4608 tmp, /* Data */
4609 brw_imm_ud(nir_src_bit_size(instr->src[0])));
4610 }
4611 break;
4612
4613 case nir_intrinsic_global_atomic_add:
4614 case nir_intrinsic_global_atomic_imin:
4615 case nir_intrinsic_global_atomic_umin:
4616 case nir_intrinsic_global_atomic_imax:
4617 case nir_intrinsic_global_atomic_umax:
4618 case nir_intrinsic_global_atomic_and:
4619 case nir_intrinsic_global_atomic_or:
4620 case nir_intrinsic_global_atomic_xor:
4621 case nir_intrinsic_global_atomic_exchange:
4622 case nir_intrinsic_global_atomic_comp_swap:
4623 nir_emit_global_atomic(bld, brw_aop_for_nir_intrinsic(instr), instr);
4624 break;
4625 case nir_intrinsic_global_atomic_fmin:
4626 case nir_intrinsic_global_atomic_fmax:
4627 case nir_intrinsic_global_atomic_fcomp_swap:
4628 nir_emit_global_atomic_float(bld, brw_aop_for_nir_intrinsic(instr), instr);
4629 break;
4630
4631 case nir_intrinsic_load_ssbo: {
4632 assert(devinfo->gen >= 7);
4633
4634 const unsigned bit_size = nir_dest_bit_size(instr->dest);
4635 fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS];
4636 srcs[SURFACE_LOGICAL_SRC_SURFACE] =
4637 get_nir_ssbo_intrinsic_index(bld, instr);
4638 srcs[SURFACE_LOGICAL_SRC_ADDRESS] = get_nir_src(instr->src[1]);
4639 srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = brw_imm_ud(1);
4640
4641 /* Make dest unsigned because that's what the temporary will be */
4642 dest.type = brw_reg_type_from_bit_size(bit_size, BRW_REGISTER_TYPE_UD);
4643
4644 /* Read the vector */
4645 if (nir_intrinsic_align(instr) >= 4) {
4646 assert(nir_dest_bit_size(instr->dest) == 32);
4647 srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(instr->num_components);
4648 fs_inst *inst =
4649 bld.emit(SHADER_OPCODE_UNTYPED_SURFACE_READ_LOGICAL,
4650 dest, srcs, SURFACE_LOGICAL_NUM_SRCS);
4651 inst->size_written = instr->num_components * dispatch_width * 4;
4652 } else {
4653 assert(nir_dest_bit_size(instr->dest) <= 32);
4654 assert(nir_dest_num_components(instr->dest) == 1);
4655 srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(bit_size);
4656
4657 fs_reg read_result = bld.vgrf(BRW_REGISTER_TYPE_UD);
4658 bld.emit(SHADER_OPCODE_BYTE_SCATTERED_READ_LOGICAL,
4659 read_result, srcs, SURFACE_LOGICAL_NUM_SRCS);
4660 bld.MOV(dest, subscript(read_result, dest.type, 0));
4661 }
4662 break;
4663 }
4664
4665 case nir_intrinsic_store_ssbo: {
4666 assert(devinfo->gen >= 7);
4667
4668 if (stage == MESA_SHADER_FRAGMENT)
4669 brw_wm_prog_data(prog_data)->has_side_effects = true;
4670
4671 const unsigned bit_size = nir_src_bit_size(instr->src[0]);
4672 fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS];
4673 srcs[SURFACE_LOGICAL_SRC_SURFACE] =
4674 get_nir_ssbo_intrinsic_index(bld, instr);
4675 srcs[SURFACE_LOGICAL_SRC_ADDRESS] = get_nir_src(instr->src[2]);
4676 srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = brw_imm_ud(1);
4677
4678 fs_reg data = get_nir_src(instr->src[0]);
4679 data.type = brw_reg_type_from_bit_size(bit_size, BRW_REGISTER_TYPE_UD);
4680
4681 assert(nir_intrinsic_write_mask(instr) ==
4682 (1u << instr->num_components) - 1);
4683 if (nir_intrinsic_align(instr) >= 4) {
4684 assert(nir_src_bit_size(instr->src[0]) == 32);
4685 assert(nir_src_num_components(instr->src[0]) <= 4);
4686 srcs[SURFACE_LOGICAL_SRC_DATA] = data;
4687 srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(instr->num_components);
4688 bld.emit(SHADER_OPCODE_UNTYPED_SURFACE_WRITE_LOGICAL,
4689 fs_reg(), srcs, SURFACE_LOGICAL_NUM_SRCS);
4690 } else {
4691 assert(nir_src_bit_size(instr->src[0]) <= 32);
4692 assert(nir_src_num_components(instr->src[0]) == 1);
4693 srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(bit_size);
4694
4695 srcs[SURFACE_LOGICAL_SRC_DATA] = bld.vgrf(BRW_REGISTER_TYPE_UD);
4696 bld.MOV(srcs[SURFACE_LOGICAL_SRC_DATA], data);
4697
4698 bld.emit(SHADER_OPCODE_BYTE_SCATTERED_WRITE_LOGICAL,
4699 fs_reg(), srcs, SURFACE_LOGICAL_NUM_SRCS);
4700 }
4701 break;
4702 }
4703
4704 case nir_intrinsic_store_output: {
4705 assert(nir_src_bit_size(instr->src[0]) == 32);
4706 fs_reg src = get_nir_src(instr->src[0]);
4707
4708 unsigned store_offset = nir_src_as_uint(instr->src[1]);
4709 unsigned num_components = instr->num_components;
4710 unsigned first_component = nir_intrinsic_component(instr);
4711
4712 fs_reg new_dest = retype(offset(outputs[instr->const_index[0]], bld,
4713 4 * store_offset), src.type);
4714 for (unsigned j = 0; j < num_components; j++) {
4715 bld.MOV(offset(new_dest, bld, j + first_component),
4716 offset(src, bld, j));
4717 }
4718 break;
4719 }
4720
4721 case nir_intrinsic_ssbo_atomic_add:
4722 case nir_intrinsic_ssbo_atomic_imin:
4723 case nir_intrinsic_ssbo_atomic_umin:
4724 case nir_intrinsic_ssbo_atomic_imax:
4725 case nir_intrinsic_ssbo_atomic_umax:
4726 case nir_intrinsic_ssbo_atomic_and:
4727 case nir_intrinsic_ssbo_atomic_or:
4728 case nir_intrinsic_ssbo_atomic_xor:
4729 case nir_intrinsic_ssbo_atomic_exchange:
4730 case nir_intrinsic_ssbo_atomic_comp_swap:
4731 nir_emit_ssbo_atomic(bld, brw_aop_for_nir_intrinsic(instr), instr);
4732 break;
4733 case nir_intrinsic_ssbo_atomic_fmin:
4734 case nir_intrinsic_ssbo_atomic_fmax:
4735 case nir_intrinsic_ssbo_atomic_fcomp_swap:
4736 nir_emit_ssbo_atomic_float(bld, brw_aop_for_nir_intrinsic(instr), instr);
4737 break;
4738
4739 case nir_intrinsic_get_buffer_size: {
4740 assert(nir_src_num_components(instr->src[0]) == 1);
4741 unsigned ssbo_index = nir_src_is_const(instr->src[0]) ?
4742 nir_src_as_uint(instr->src[0]) : 0;
4743
4744 /* A resinfo's sampler message is used to get the buffer size. The
4745 * SIMD8's writeback message consists of four registers and SIMD16's
4746 * writeback message consists of 8 destination registers (two per each
4747 * component). Because we are only interested on the first channel of
4748 * the first returned component, where resinfo returns the buffer size
4749 * for SURFTYPE_BUFFER, we can just use the SIMD8 variant regardless of
4750 * the dispatch width.
4751 */
4752 const fs_builder ubld = bld.exec_all().group(8, 0);
4753 fs_reg src_payload = ubld.vgrf(BRW_REGISTER_TYPE_UD);
4754 fs_reg ret_payload = ubld.vgrf(BRW_REGISTER_TYPE_UD, 4);
4755
4756 /* Set LOD = 0 */
4757 ubld.MOV(src_payload, brw_imm_d(0));
4758
4759 const unsigned index = prog_data->binding_table.ssbo_start + ssbo_index;
4760 fs_inst *inst = ubld.emit(SHADER_OPCODE_GET_BUFFER_SIZE, ret_payload,
4761 src_payload, brw_imm_ud(index));
4762 inst->header_size = 0;
4763 inst->mlen = 1;
4764 inst->size_written = 4 * REG_SIZE;
4765
4766 /* SKL PRM, vol07, 3D Media GPGPU Engine, Bounds Checking and Faulting:
4767 *
4768 * "Out-of-bounds checking is always performed at a DWord granularity. If
4769 * any part of the DWord is out-of-bounds then the whole DWord is
4770 * considered out-of-bounds."
4771 *
4772 * This implies that types with size smaller than 4-bytes need to be
4773 * padded if they don't complete the last dword of the buffer. But as we
4774 * need to maintain the original size we need to reverse the padding
4775 * calculation to return the correct size to know the number of elements
4776 * of an unsized array. As we stored in the last two bits of the surface
4777 * size the needed padding for the buffer, we calculate here the
4778 * original buffer_size reversing the surface_size calculation:
4779 *
4780 * surface_size = isl_align(buffer_size, 4) +
4781 * (isl_align(buffer_size) - buffer_size)
4782 *
4783 * buffer_size = surface_size & ~3 - surface_size & 3
4784 */
4785
4786 fs_reg size_aligned4 = ubld.vgrf(BRW_REGISTER_TYPE_UD);
4787 fs_reg size_padding = ubld.vgrf(BRW_REGISTER_TYPE_UD);
4788 fs_reg buffer_size = ubld.vgrf(BRW_REGISTER_TYPE_UD);
4789
4790 ubld.AND(size_padding, ret_payload, brw_imm_ud(3));
4791 ubld.AND(size_aligned4, ret_payload, brw_imm_ud(~3));
4792 ubld.ADD(buffer_size, size_aligned4, negate(size_padding));
4793
4794 bld.MOV(retype(dest, ret_payload.type), component(buffer_size, 0));
4795 break;
4796 }
4797
4798 case nir_intrinsic_load_scratch: {
4799 assert(devinfo->gen >= 7);
4800
4801 assert(nir_dest_num_components(instr->dest) == 1);
4802 const unsigned bit_size = nir_dest_bit_size(instr->dest);
4803 fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS];
4804
4805 if (devinfo->gen >= 8) {
4806 srcs[SURFACE_LOGICAL_SRC_SURFACE] =
4807 brw_imm_ud(GEN8_BTI_STATELESS_NON_COHERENT);
4808 } else {
4809 srcs[SURFACE_LOGICAL_SRC_SURFACE] = brw_imm_ud(BRW_BTI_STATELESS);
4810 }
4811
4812 srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = brw_imm_ud(1);
4813 srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(bit_size);
4814 const fs_reg nir_addr = get_nir_src(instr->src[0]);
4815
4816 /* Make dest unsigned because that's what the temporary will be */
4817 dest.type = brw_reg_type_from_bit_size(bit_size, BRW_REGISTER_TYPE_UD);
4818
4819 /* Read the vector */
4820 if (nir_intrinsic_align(instr) >= 4) {
4821 assert(nir_dest_bit_size(instr->dest) == 32);
4822
4823 /* The offset for a DWORD scattered message is in dwords. */
4824 srcs[SURFACE_LOGICAL_SRC_ADDRESS] =
4825 swizzle_nir_scratch_addr(bld, nir_addr, true);
4826
4827 bld.emit(SHADER_OPCODE_DWORD_SCATTERED_READ_LOGICAL,
4828 dest, srcs, SURFACE_LOGICAL_NUM_SRCS);
4829 } else {
4830 assert(nir_dest_bit_size(instr->dest) <= 32);
4831
4832 srcs[SURFACE_LOGICAL_SRC_ADDRESS] =
4833 swizzle_nir_scratch_addr(bld, nir_addr, false);
4834
4835 fs_reg read_result = bld.vgrf(BRW_REGISTER_TYPE_UD);
4836 bld.emit(SHADER_OPCODE_BYTE_SCATTERED_READ_LOGICAL,
4837 read_result, srcs, SURFACE_LOGICAL_NUM_SRCS);
4838 bld.MOV(dest, read_result);
4839 }
4840 break;
4841 }
4842
4843 case nir_intrinsic_store_scratch: {
4844 assert(devinfo->gen >= 7);
4845
4846 assert(nir_src_num_components(instr->src[0]) == 1);
4847 const unsigned bit_size = nir_src_bit_size(instr->src[0]);
4848 fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS];
4849
4850 if (devinfo->gen >= 8) {
4851 srcs[SURFACE_LOGICAL_SRC_SURFACE] =
4852 brw_imm_ud(GEN8_BTI_STATELESS_NON_COHERENT);
4853 } else {
4854 srcs[SURFACE_LOGICAL_SRC_SURFACE] = brw_imm_ud(BRW_BTI_STATELESS);
4855 }
4856
4857 srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = brw_imm_ud(1);
4858 srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(bit_size);
4859 const fs_reg nir_addr = get_nir_src(instr->src[1]);
4860
4861 fs_reg data = get_nir_src(instr->src[0]);
4862 data.type = brw_reg_type_from_bit_size(bit_size, BRW_REGISTER_TYPE_UD);
4863
4864 assert(nir_intrinsic_write_mask(instr) ==
4865 (1u << instr->num_components) - 1);
4866 if (nir_intrinsic_align(instr) >= 4) {
4867 assert(nir_src_bit_size(instr->src[0]) == 32);
4868 srcs[SURFACE_LOGICAL_SRC_DATA] = data;
4869
4870 /* The offset for a DWORD scattered message is in dwords. */
4871 srcs[SURFACE_LOGICAL_SRC_ADDRESS] =
4872 swizzle_nir_scratch_addr(bld, nir_addr, true);
4873
4874 bld.emit(SHADER_OPCODE_DWORD_SCATTERED_WRITE_LOGICAL,
4875 fs_reg(), srcs, SURFACE_LOGICAL_NUM_SRCS);
4876 } else {
4877 assert(nir_src_bit_size(instr->src[0]) <= 32);
4878
4879 srcs[SURFACE_LOGICAL_SRC_DATA] = bld.vgrf(BRW_REGISTER_TYPE_UD);
4880 bld.MOV(srcs[SURFACE_LOGICAL_SRC_DATA], data);
4881
4882 srcs[SURFACE_LOGICAL_SRC_ADDRESS] =
4883 swizzle_nir_scratch_addr(bld, nir_addr, false);
4884
4885 bld.emit(SHADER_OPCODE_BYTE_SCATTERED_WRITE_LOGICAL,
4886 fs_reg(), srcs, SURFACE_LOGICAL_NUM_SRCS);
4887 }
4888 break;
4889 }
4890
4891 case nir_intrinsic_load_subgroup_size:
4892 /* This should only happen for fragment shaders because every other case
4893 * is lowered in NIR so we can optimize on it.
4894 */
4895 assert(stage == MESA_SHADER_FRAGMENT);
4896 bld.MOV(retype(dest, BRW_REGISTER_TYPE_D), brw_imm_d(dispatch_width));
4897 break;
4898
4899 case nir_intrinsic_load_subgroup_invocation:
4900 bld.MOV(retype(dest, BRW_REGISTER_TYPE_D),
4901 nir_system_values[SYSTEM_VALUE_SUBGROUP_INVOCATION]);
4902 break;
4903
4904 case nir_intrinsic_load_subgroup_eq_mask:
4905 case nir_intrinsic_load_subgroup_ge_mask:
4906 case nir_intrinsic_load_subgroup_gt_mask:
4907 case nir_intrinsic_load_subgroup_le_mask:
4908 case nir_intrinsic_load_subgroup_lt_mask:
4909 unreachable("not reached");
4910
4911 case nir_intrinsic_vote_any: {
4912 const fs_builder ubld = bld.exec_all().group(1, 0);
4913
4914 /* The any/all predicates do not consider channel enables. To prevent
4915 * dead channels from affecting the result, we initialize the flag with
4916 * with the identity value for the logical operation.
4917 */
4918 if (dispatch_width == 32) {
4919 /* For SIMD32, we use a UD type so we fill both f0.0 and f0.1. */
4920 ubld.MOV(retype(brw_flag_reg(0, 0), BRW_REGISTER_TYPE_UD),
4921 brw_imm_ud(0));
4922 } else {
4923 ubld.MOV(brw_flag_reg(0, 0), brw_imm_uw(0));
4924 }
4925 bld.CMP(bld.null_reg_d(), get_nir_src(instr->src[0]), brw_imm_d(0), BRW_CONDITIONAL_NZ);
4926
4927 /* For some reason, the any/all predicates don't work properly with
4928 * SIMD32. In particular, it appears that a SEL with a QtrCtrl of 2H
4929 * doesn't read the correct subset of the flag register and you end up
4930 * getting garbage in the second half. Work around this by using a pair
4931 * of 1-wide MOVs and scattering the result.
4932 */
4933 fs_reg res1 = ubld.vgrf(BRW_REGISTER_TYPE_D);
4934 ubld.MOV(res1, brw_imm_d(0));
4935 set_predicate(dispatch_width == 8 ? BRW_PREDICATE_ALIGN1_ANY8H :
4936 dispatch_width == 16 ? BRW_PREDICATE_ALIGN1_ANY16H :
4937 BRW_PREDICATE_ALIGN1_ANY32H,
4938 ubld.MOV(res1, brw_imm_d(-1)));
4939
4940 bld.MOV(retype(dest, BRW_REGISTER_TYPE_D), component(res1, 0));
4941 break;
4942 }
4943 case nir_intrinsic_vote_all: {
4944 const fs_builder ubld = bld.exec_all().group(1, 0);
4945
4946 /* The any/all predicates do not consider channel enables. To prevent
4947 * dead channels from affecting the result, we initialize the flag with
4948 * with the identity value for the logical operation.
4949 */
4950 if (dispatch_width == 32) {
4951 /* For SIMD32, we use a UD type so we fill both f0.0 and f0.1. */
4952 ubld.MOV(retype(brw_flag_reg(0, 0), BRW_REGISTER_TYPE_UD),
4953 brw_imm_ud(0xffffffff));
4954 } else {
4955 ubld.MOV(brw_flag_reg(0, 0), brw_imm_uw(0xffff));
4956 }
4957 bld.CMP(bld.null_reg_d(), get_nir_src(instr->src[0]), brw_imm_d(0), BRW_CONDITIONAL_NZ);
4958
4959 /* For some reason, the any/all predicates don't work properly with
4960 * SIMD32. In particular, it appears that a SEL with a QtrCtrl of 2H
4961 * doesn't read the correct subset of the flag register and you end up
4962 * getting garbage in the second half. Work around this by using a pair
4963 * of 1-wide MOVs and scattering the result.
4964 */
4965 fs_reg res1 = ubld.vgrf(BRW_REGISTER_TYPE_D);
4966 ubld.MOV(res1, brw_imm_d(0));
4967 set_predicate(dispatch_width == 8 ? BRW_PREDICATE_ALIGN1_ALL8H :
4968 dispatch_width == 16 ? BRW_PREDICATE_ALIGN1_ALL16H :
4969 BRW_PREDICATE_ALIGN1_ALL32H,
4970 ubld.MOV(res1, brw_imm_d(-1)));
4971
4972 bld.MOV(retype(dest, BRW_REGISTER_TYPE_D), component(res1, 0));
4973 break;
4974 }
4975 case nir_intrinsic_vote_feq:
4976 case nir_intrinsic_vote_ieq: {
4977 fs_reg value = get_nir_src(instr->src[0]);
4978 if (instr->intrinsic == nir_intrinsic_vote_feq) {
4979 const unsigned bit_size = nir_src_bit_size(instr->src[0]);
4980 value.type = bit_size == 8 ? BRW_REGISTER_TYPE_B :
4981 brw_reg_type_from_bit_size(bit_size, BRW_REGISTER_TYPE_F);
4982 }
4983
4984 fs_reg uniformized = bld.emit_uniformize(value);
4985 const fs_builder ubld = bld.exec_all().group(1, 0);
4986
4987 /* The any/all predicates do not consider channel enables. To prevent
4988 * dead channels from affecting the result, we initialize the flag with
4989 * with the identity value for the logical operation.
4990 */
4991 if (dispatch_width == 32) {
4992 /* For SIMD32, we use a UD type so we fill both f0.0 and f0.1. */
4993 ubld.MOV(retype(brw_flag_reg(0, 0), BRW_REGISTER_TYPE_UD),
4994 brw_imm_ud(0xffffffff));
4995 } else {
4996 ubld.MOV(brw_flag_reg(0, 0), brw_imm_uw(0xffff));
4997 }
4998 bld.CMP(bld.null_reg_d(), value, uniformized, BRW_CONDITIONAL_Z);
4999
5000 /* For some reason, the any/all predicates don't work properly with
5001 * SIMD32. In particular, it appears that a SEL with a QtrCtrl of 2H
5002 * doesn't read the correct subset of the flag register and you end up
5003 * getting garbage in the second half. Work around this by using a pair
5004 * of 1-wide MOVs and scattering the result.
5005 */
5006 fs_reg res1 = ubld.vgrf(BRW_REGISTER_TYPE_D);
5007 ubld.MOV(res1, brw_imm_d(0));
5008 set_predicate(dispatch_width == 8 ? BRW_PREDICATE_ALIGN1_ALL8H :
5009 dispatch_width == 16 ? BRW_PREDICATE_ALIGN1_ALL16H :
5010 BRW_PREDICATE_ALIGN1_ALL32H,
5011 ubld.MOV(res1, brw_imm_d(-1)));
5012
5013 bld.MOV(retype(dest, BRW_REGISTER_TYPE_D), component(res1, 0));
5014 break;
5015 }
5016
5017 case nir_intrinsic_ballot: {
5018 const fs_reg value = retype(get_nir_src(instr->src[0]),
5019 BRW_REGISTER_TYPE_UD);
5020 struct brw_reg flag = brw_flag_reg(0, 0);
5021 /* FIXME: For SIMD32 programs, this causes us to stomp on f0.1 as well
5022 * as f0.0. This is a problem for fragment programs as we currently use
5023 * f0.1 for discards. Fortunately, we don't support SIMD32 fragment
5024 * programs yet so this isn't a problem. When we do, something will
5025 * have to change.
5026 */
5027 if (dispatch_width == 32)
5028 flag.type = BRW_REGISTER_TYPE_UD;
5029
5030 bld.exec_all().group(1, 0).MOV(flag, brw_imm_ud(0u));
5031 bld.CMP(bld.null_reg_ud(), value, brw_imm_ud(0u), BRW_CONDITIONAL_NZ);
5032
5033 if (instr->dest.ssa.bit_size > 32) {
5034 dest.type = BRW_REGISTER_TYPE_UQ;
5035 } else {
5036 dest.type = BRW_REGISTER_TYPE_UD;
5037 }
5038 bld.MOV(dest, flag);
5039 break;
5040 }
5041
5042 case nir_intrinsic_read_invocation: {
5043 const fs_reg value = get_nir_src(instr->src[0]);
5044 const fs_reg invocation = get_nir_src(instr->src[1]);
5045 fs_reg tmp = bld.vgrf(value.type);
5046
5047 bld.exec_all().emit(SHADER_OPCODE_BROADCAST, tmp, value,
5048 bld.emit_uniformize(invocation));
5049
5050 bld.MOV(retype(dest, value.type), fs_reg(component(tmp, 0)));
5051 break;
5052 }
5053
5054 case nir_intrinsic_read_first_invocation: {
5055 const fs_reg value = get_nir_src(instr->src[0]);
5056 bld.MOV(retype(dest, value.type), bld.emit_uniformize(value));
5057 break;
5058 }
5059
5060 case nir_intrinsic_shuffle: {
5061 const fs_reg value = get_nir_src(instr->src[0]);
5062 const fs_reg index = get_nir_src(instr->src[1]);
5063
5064 bld.emit(SHADER_OPCODE_SHUFFLE, retype(dest, value.type), value, index);
5065 break;
5066 }
5067
5068 case nir_intrinsic_first_invocation: {
5069 fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_UD);
5070 bld.exec_all().emit(SHADER_OPCODE_FIND_LIVE_CHANNEL, tmp);
5071 bld.MOV(retype(dest, BRW_REGISTER_TYPE_UD),
5072 fs_reg(component(tmp, 0)));
5073 break;
5074 }
5075
5076 case nir_intrinsic_quad_broadcast: {
5077 const fs_reg value = get_nir_src(instr->src[0]);
5078 const unsigned index = nir_src_as_uint(instr->src[1]);
5079
5080 bld.emit(SHADER_OPCODE_CLUSTER_BROADCAST, retype(dest, value.type),
5081 value, brw_imm_ud(index), brw_imm_ud(4));
5082 break;
5083 }
5084
5085 case nir_intrinsic_quad_swap_horizontal: {
5086 const fs_reg value = get_nir_src(instr->src[0]);
5087 const fs_reg tmp = bld.vgrf(value.type);
5088 if (devinfo->gen <= 7) {
5089 /* The hardware doesn't seem to support these crazy regions with
5090 * compressed instructions on gen7 and earlier so we fall back to
5091 * using quad swizzles. Fortunately, we don't support 64-bit
5092 * anything in Vulkan on gen7.
5093 */
5094 assert(nir_src_bit_size(instr->src[0]) == 32);
5095 const fs_builder ubld = bld.exec_all();
5096 ubld.emit(SHADER_OPCODE_QUAD_SWIZZLE, tmp, value,
5097 brw_imm_ud(BRW_SWIZZLE4(1,0,3,2)));
5098 bld.MOV(retype(dest, value.type), tmp);
5099 } else {
5100 const fs_builder ubld = bld.exec_all().group(dispatch_width / 2, 0);
5101
5102 const fs_reg src_left = horiz_stride(value, 2);
5103 const fs_reg src_right = horiz_stride(horiz_offset(value, 1), 2);
5104 const fs_reg tmp_left = horiz_stride(tmp, 2);
5105 const fs_reg tmp_right = horiz_stride(horiz_offset(tmp, 1), 2);
5106
5107 ubld.MOV(tmp_left, src_right);
5108 ubld.MOV(tmp_right, src_left);
5109
5110 }
5111 bld.MOV(retype(dest, value.type), tmp);
5112 break;
5113 }
5114
5115 case nir_intrinsic_quad_swap_vertical: {
5116 const fs_reg value = get_nir_src(instr->src[0]);
5117 if (nir_src_bit_size(instr->src[0]) == 32) {
5118 /* For 32-bit, we can use a SIMD4x2 instruction to do this easily */
5119 const fs_reg tmp = bld.vgrf(value.type);
5120 const fs_builder ubld = bld.exec_all();
5121 ubld.emit(SHADER_OPCODE_QUAD_SWIZZLE, tmp, value,
5122 brw_imm_ud(BRW_SWIZZLE4(2,3,0,1)));
5123 bld.MOV(retype(dest, value.type), tmp);
5124 } else {
5125 /* For larger data types, we have to either emit dispatch_width many
5126 * MOVs or else fall back to doing indirects.
5127 */
5128 fs_reg idx = bld.vgrf(BRW_REGISTER_TYPE_W);
5129 bld.XOR(idx, nir_system_values[SYSTEM_VALUE_SUBGROUP_INVOCATION],
5130 brw_imm_w(0x2));
5131 bld.emit(SHADER_OPCODE_SHUFFLE, retype(dest, value.type), value, idx);
5132 }
5133 break;
5134 }
5135
5136 case nir_intrinsic_quad_swap_diagonal: {
5137 const fs_reg value = get_nir_src(instr->src[0]);
5138 if (nir_src_bit_size(instr->src[0]) == 32) {
5139 /* For 32-bit, we can use a SIMD4x2 instruction to do this easily */
5140 const fs_reg tmp = bld.vgrf(value.type);
5141 const fs_builder ubld = bld.exec_all();
5142 ubld.emit(SHADER_OPCODE_QUAD_SWIZZLE, tmp, value,
5143 brw_imm_ud(BRW_SWIZZLE4(3,2,1,0)));
5144 bld.MOV(retype(dest, value.type), tmp);
5145 } else {
5146 /* For larger data types, we have to either emit dispatch_width many
5147 * MOVs or else fall back to doing indirects.
5148 */
5149 fs_reg idx = bld.vgrf(BRW_REGISTER_TYPE_W);
5150 bld.XOR(idx, nir_system_values[SYSTEM_VALUE_SUBGROUP_INVOCATION],
5151 brw_imm_w(0x3));
5152 bld.emit(SHADER_OPCODE_SHUFFLE, retype(dest, value.type), value, idx);
5153 }
5154 break;
5155 }
5156
5157 case nir_intrinsic_reduce: {
5158 fs_reg src = get_nir_src(instr->src[0]);
5159 nir_op redop = (nir_op)nir_intrinsic_reduction_op(instr);
5160 unsigned cluster_size = nir_intrinsic_cluster_size(instr);
5161 if (cluster_size == 0 || cluster_size > dispatch_width)
5162 cluster_size = dispatch_width;
5163
5164 /* Figure out the source type */
5165 src.type = brw_type_for_nir_type(devinfo,
5166 (nir_alu_type)(nir_op_infos[redop].input_types[0] |
5167 nir_src_bit_size(instr->src[0])));
5168
5169 fs_reg identity = brw_nir_reduction_op_identity(bld, redop, src.type);
5170 opcode brw_op = brw_op_for_nir_reduction_op(redop);
5171 brw_conditional_mod cond_mod = brw_cond_mod_for_nir_reduction_op(redop);
5172
5173 /* There are a couple of register region issues that make things
5174 * complicated for 8-bit types:
5175 *
5176 * 1. Only raw moves are allowed to write to a packed 8-bit
5177 * destination.
5178 * 2. If we use a strided destination, the efficient way to do scan
5179 * operations ends up using strides that are too big to encode in
5180 * an instruction.
5181 *
5182 * To get around these issues, we just do all 8-bit scan operations in
5183 * 16 bits. It's actually fewer instructions than what we'd have to do
5184 * if we were trying to do it in native 8-bit types and the results are
5185 * the same once we truncate to 8 bits at the end.
5186 */
5187 brw_reg_type scan_type = src.type;
5188 if (type_sz(scan_type) == 1)
5189 scan_type = brw_reg_type_from_bit_size(16, src.type);
5190
5191 /* Set up a register for all of our scratching around and initialize it
5192 * to reduction operation's identity value.
5193 */
5194 fs_reg scan = bld.vgrf(scan_type);
5195 bld.exec_all().emit(SHADER_OPCODE_SEL_EXEC, scan, src, identity);
5196
5197 bld.emit_scan(brw_op, scan, cluster_size, cond_mod);
5198
5199 dest.type = src.type;
5200 if (cluster_size * type_sz(src.type) >= REG_SIZE * 2) {
5201 /* In this case, CLUSTER_BROADCAST instruction isn't needed because
5202 * the distance between clusters is at least 2 GRFs. In this case,
5203 * we don't need the weird striding of the CLUSTER_BROADCAST
5204 * instruction and can just do regular MOVs.
5205 */
5206 assert((cluster_size * type_sz(src.type)) % (REG_SIZE * 2) == 0);
5207 const unsigned groups =
5208 (dispatch_width * type_sz(src.type)) / (REG_SIZE * 2);
5209 const unsigned group_size = dispatch_width / groups;
5210 for (unsigned i = 0; i < groups; i++) {
5211 const unsigned cluster = (i * group_size) / cluster_size;
5212 const unsigned comp = cluster * cluster_size + (cluster_size - 1);
5213 bld.group(group_size, i).MOV(horiz_offset(dest, i * group_size),
5214 component(scan, comp));
5215 }
5216 } else {
5217 bld.emit(SHADER_OPCODE_CLUSTER_BROADCAST, dest, scan,
5218 brw_imm_ud(cluster_size - 1), brw_imm_ud(cluster_size));
5219 }
5220 break;
5221 }
5222
5223 case nir_intrinsic_inclusive_scan:
5224 case nir_intrinsic_exclusive_scan: {
5225 fs_reg src = get_nir_src(instr->src[0]);
5226 nir_op redop = (nir_op)nir_intrinsic_reduction_op(instr);
5227
5228 /* Figure out the source type */
5229 src.type = brw_type_for_nir_type(devinfo,
5230 (nir_alu_type)(nir_op_infos[redop].input_types[0] |
5231 nir_src_bit_size(instr->src[0])));
5232
5233 fs_reg identity = brw_nir_reduction_op_identity(bld, redop, src.type);
5234 opcode brw_op = brw_op_for_nir_reduction_op(redop);
5235 brw_conditional_mod cond_mod = brw_cond_mod_for_nir_reduction_op(redop);
5236
5237 /* There are a couple of register region issues that make things
5238 * complicated for 8-bit types:
5239 *
5240 * 1. Only raw moves are allowed to write to a packed 8-bit
5241 * destination.
5242 * 2. If we use a strided destination, the efficient way to do scan
5243 * operations ends up using strides that are too big to encode in
5244 * an instruction.
5245 *
5246 * To get around these issues, we just do all 8-bit scan operations in
5247 * 16 bits. It's actually fewer instructions than what we'd have to do
5248 * if we were trying to do it in native 8-bit types and the results are
5249 * the same once we truncate to 8 bits at the end.
5250 */
5251 brw_reg_type scan_type = src.type;
5252 if (type_sz(scan_type) == 1)
5253 scan_type = brw_reg_type_from_bit_size(16, src.type);
5254
5255 /* Set up a register for all of our scratching around and initialize it
5256 * to reduction operation's identity value.
5257 */
5258 fs_reg scan = bld.vgrf(scan_type);
5259 const fs_builder allbld = bld.exec_all();
5260 allbld.emit(SHADER_OPCODE_SEL_EXEC, scan, src, identity);
5261
5262 if (instr->intrinsic == nir_intrinsic_exclusive_scan) {
5263 /* Exclusive scan is a bit harder because we have to do an annoying
5264 * shift of the contents before we can begin. To make things worse,
5265 * we can't do this with a normal stride; we have to use indirects.
5266 */
5267 fs_reg shifted = bld.vgrf(scan_type);
5268 fs_reg idx = bld.vgrf(BRW_REGISTER_TYPE_W);
5269 allbld.ADD(idx, nir_system_values[SYSTEM_VALUE_SUBGROUP_INVOCATION],
5270 brw_imm_w(-1));
5271 allbld.emit(SHADER_OPCODE_SHUFFLE, shifted, scan, idx);
5272 allbld.group(1, 0).MOV(component(shifted, 0), identity);
5273 scan = shifted;
5274 }
5275
5276 bld.emit_scan(brw_op, scan, dispatch_width, cond_mod);
5277
5278 bld.MOV(retype(dest, src.type), scan);
5279 break;
5280 }
5281
5282 case nir_intrinsic_begin_invocation_interlock: {
5283 const fs_builder ubld = bld.group(8, 0);
5284 const fs_reg tmp = ubld.vgrf(BRW_REGISTER_TYPE_UD, 2);
5285
5286 ubld.emit(SHADER_OPCODE_INTERLOCK, tmp, brw_vec8_grf(0, 0))
5287 ->size_written = 2 * REG_SIZE;
5288 break;
5289 }
5290
5291 case nir_intrinsic_end_invocation_interlock: {
5292 /* For endInvocationInterlock(), we need to insert a memory fence which
5293 * stalls in the shader until the memory transactions prior to that
5294 * fence are complete. This ensures that the shader does not end before
5295 * any writes from its critical section have landed. Otherwise, you can
5296 * end up with a case where the next invocation on that pixel properly
5297 * stalls for previous FS invocation on its pixel to complete but
5298 * doesn't actually wait for the dataport memory transactions from that
5299 * thread to land before submitting its own.
5300 */
5301 const fs_builder ubld = bld.group(8, 0);
5302 const fs_reg tmp = ubld.vgrf(BRW_REGISTER_TYPE_UD, 2);
5303 ubld.emit(SHADER_OPCODE_MEMORY_FENCE, tmp,
5304 brw_vec8_grf(0, 0), brw_imm_ud(1), brw_imm_ud(0))
5305 ->size_written = 2 * REG_SIZE;
5306 break;
5307 }
5308
5309 default:
5310 unreachable("unknown intrinsic");
5311 }
5312 }
5313
5314 void
5315 fs_visitor::nir_emit_ssbo_atomic(const fs_builder &bld,
5316 int op, nir_intrinsic_instr *instr)
5317 {
5318 if (stage == MESA_SHADER_FRAGMENT)
5319 brw_wm_prog_data(prog_data)->has_side_effects = true;
5320
5321 /* The BTI untyped atomic messages only support 32-bit atomics. If you
5322 * just look at the big table of messages in the Vol 7 of the SKL PRM, they
5323 * appear to exist. However, if you look at Vol 2a, there are no message
5324 * descriptors provided for Qword atomic ops except for A64 messages.
5325 */
5326 assert(nir_dest_bit_size(instr->dest) == 32);
5327
5328 fs_reg dest;
5329 if (nir_intrinsic_infos[instr->intrinsic].has_dest)
5330 dest = get_nir_dest(instr->dest);
5331
5332 fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS];
5333 srcs[SURFACE_LOGICAL_SRC_SURFACE] = get_nir_ssbo_intrinsic_index(bld, instr);
5334 srcs[SURFACE_LOGICAL_SRC_ADDRESS] = get_nir_src(instr->src[1]);
5335 srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = brw_imm_ud(1);
5336 srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(op);
5337
5338 fs_reg data;
5339 if (op != BRW_AOP_INC && op != BRW_AOP_DEC && op != BRW_AOP_PREDEC)
5340 data = get_nir_src(instr->src[2]);
5341
5342 if (op == BRW_AOP_CMPWR) {
5343 fs_reg tmp = bld.vgrf(data.type, 2);
5344 fs_reg sources[2] = { data, get_nir_src(instr->src[3]) };
5345 bld.LOAD_PAYLOAD(tmp, sources, 2, 0);
5346 data = tmp;
5347 }
5348 srcs[SURFACE_LOGICAL_SRC_DATA] = data;
5349
5350 /* Emit the actual atomic operation */
5351
5352 bld.emit(SHADER_OPCODE_UNTYPED_ATOMIC_LOGICAL,
5353 dest, srcs, SURFACE_LOGICAL_NUM_SRCS);
5354 }
5355
5356 void
5357 fs_visitor::nir_emit_ssbo_atomic_float(const fs_builder &bld,
5358 int op, nir_intrinsic_instr *instr)
5359 {
5360 if (stage == MESA_SHADER_FRAGMENT)
5361 brw_wm_prog_data(prog_data)->has_side_effects = true;
5362
5363 fs_reg dest;
5364 if (nir_intrinsic_infos[instr->intrinsic].has_dest)
5365 dest = get_nir_dest(instr->dest);
5366
5367 fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS];
5368 srcs[SURFACE_LOGICAL_SRC_SURFACE] = get_nir_ssbo_intrinsic_index(bld, instr);
5369 srcs[SURFACE_LOGICAL_SRC_ADDRESS] = get_nir_src(instr->src[1]);
5370 srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = brw_imm_ud(1);
5371 srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(op);
5372
5373 fs_reg data = get_nir_src(instr->src[2]);
5374 if (op == BRW_AOP_FCMPWR) {
5375 fs_reg tmp = bld.vgrf(data.type, 2);
5376 fs_reg sources[2] = { data, get_nir_src(instr->src[3]) };
5377 bld.LOAD_PAYLOAD(tmp, sources, 2, 0);
5378 data = tmp;
5379 }
5380 srcs[SURFACE_LOGICAL_SRC_DATA] = data;
5381
5382 /* Emit the actual atomic operation */
5383
5384 bld.emit(SHADER_OPCODE_UNTYPED_ATOMIC_FLOAT_LOGICAL,
5385 dest, srcs, SURFACE_LOGICAL_NUM_SRCS);
5386 }
5387
5388 void
5389 fs_visitor::nir_emit_shared_atomic(const fs_builder &bld,
5390 int op, nir_intrinsic_instr *instr)
5391 {
5392 fs_reg dest;
5393 if (nir_intrinsic_infos[instr->intrinsic].has_dest)
5394 dest = get_nir_dest(instr->dest);
5395
5396 fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS];
5397 srcs[SURFACE_LOGICAL_SRC_SURFACE] = brw_imm_ud(GEN7_BTI_SLM);
5398 srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = brw_imm_ud(1);
5399 srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(op);
5400
5401 fs_reg data;
5402 if (op != BRW_AOP_INC && op != BRW_AOP_DEC && op != BRW_AOP_PREDEC)
5403 data = get_nir_src(instr->src[1]);
5404 if (op == BRW_AOP_CMPWR) {
5405 fs_reg tmp = bld.vgrf(data.type, 2);
5406 fs_reg sources[2] = { data, get_nir_src(instr->src[2]) };
5407 bld.LOAD_PAYLOAD(tmp, sources, 2, 0);
5408 data = tmp;
5409 }
5410 srcs[SURFACE_LOGICAL_SRC_DATA] = data;
5411
5412 /* Get the offset */
5413 if (nir_src_is_const(instr->src[0])) {
5414 srcs[SURFACE_LOGICAL_SRC_ADDRESS] =
5415 brw_imm_ud(instr->const_index[0] + nir_src_as_uint(instr->src[0]));
5416 } else {
5417 srcs[SURFACE_LOGICAL_SRC_ADDRESS] = vgrf(glsl_type::uint_type);
5418 bld.ADD(srcs[SURFACE_LOGICAL_SRC_ADDRESS],
5419 retype(get_nir_src(instr->src[0]), BRW_REGISTER_TYPE_UD),
5420 brw_imm_ud(instr->const_index[0]));
5421 }
5422
5423 /* Emit the actual atomic operation operation */
5424
5425 bld.emit(SHADER_OPCODE_UNTYPED_ATOMIC_LOGICAL,
5426 dest, srcs, SURFACE_LOGICAL_NUM_SRCS);
5427 }
5428
5429 void
5430 fs_visitor::nir_emit_shared_atomic_float(const fs_builder &bld,
5431 int op, nir_intrinsic_instr *instr)
5432 {
5433 fs_reg dest;
5434 if (nir_intrinsic_infos[instr->intrinsic].has_dest)
5435 dest = get_nir_dest(instr->dest);
5436
5437 fs_reg srcs[SURFACE_LOGICAL_NUM_SRCS];
5438 srcs[SURFACE_LOGICAL_SRC_SURFACE] = brw_imm_ud(GEN7_BTI_SLM);
5439 srcs[SURFACE_LOGICAL_SRC_IMM_DIMS] = brw_imm_ud(1);
5440 srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(op);
5441
5442 fs_reg data = get_nir_src(instr->src[1]);
5443 if (op == BRW_AOP_FCMPWR) {
5444 fs_reg tmp = bld.vgrf(data.type, 2);
5445 fs_reg sources[2] = { data, get_nir_src(instr->src[2]) };
5446 bld.LOAD_PAYLOAD(tmp, sources, 2, 0);
5447 data = tmp;
5448 }
5449 srcs[SURFACE_LOGICAL_SRC_DATA] = data;
5450
5451 /* Get the offset */
5452 if (nir_src_is_const(instr->src[0])) {
5453 srcs[SURFACE_LOGICAL_SRC_ADDRESS] =
5454 brw_imm_ud(instr->const_index[0] + nir_src_as_uint(instr->src[0]));
5455 } else {
5456 srcs[SURFACE_LOGICAL_SRC_ADDRESS] = vgrf(glsl_type::uint_type);
5457 bld.ADD(srcs[SURFACE_LOGICAL_SRC_ADDRESS],
5458 retype(get_nir_src(instr->src[0]), BRW_REGISTER_TYPE_UD),
5459 brw_imm_ud(instr->const_index[0]));
5460 }
5461
5462 /* Emit the actual atomic operation operation */
5463
5464 bld.emit(SHADER_OPCODE_UNTYPED_ATOMIC_FLOAT_LOGICAL,
5465 dest, srcs, SURFACE_LOGICAL_NUM_SRCS);
5466 }
5467
5468 void
5469 fs_visitor::nir_emit_global_atomic(const fs_builder &bld,
5470 int op, nir_intrinsic_instr *instr)
5471 {
5472 if (stage == MESA_SHADER_FRAGMENT)
5473 brw_wm_prog_data(prog_data)->has_side_effects = true;
5474
5475 fs_reg dest;
5476 if (nir_intrinsic_infos[instr->intrinsic].has_dest)
5477 dest = get_nir_dest(instr->dest);
5478
5479 fs_reg addr = get_nir_src(instr->src[0]);
5480
5481 fs_reg data;
5482 if (op != BRW_AOP_INC && op != BRW_AOP_DEC && op != BRW_AOP_PREDEC)
5483 data = get_nir_src(instr->src[1]);
5484
5485 if (op == BRW_AOP_CMPWR) {
5486 fs_reg tmp = bld.vgrf(data.type, 2);
5487 fs_reg sources[2] = { data, get_nir_src(instr->src[2]) };
5488 bld.LOAD_PAYLOAD(tmp, sources, 2, 0);
5489 data = tmp;
5490 }
5491
5492 if (nir_dest_bit_size(instr->dest) == 64) {
5493 bld.emit(SHADER_OPCODE_A64_UNTYPED_ATOMIC_INT64_LOGICAL,
5494 dest, addr, data, brw_imm_ud(op));
5495 } else {
5496 assert(nir_dest_bit_size(instr->dest) == 32);
5497 bld.emit(SHADER_OPCODE_A64_UNTYPED_ATOMIC_LOGICAL,
5498 dest, addr, data, brw_imm_ud(op));
5499 }
5500 }
5501
5502 void
5503 fs_visitor::nir_emit_global_atomic_float(const fs_builder &bld,
5504 int op, nir_intrinsic_instr *instr)
5505 {
5506 if (stage == MESA_SHADER_FRAGMENT)
5507 brw_wm_prog_data(prog_data)->has_side_effects = true;
5508
5509 assert(nir_intrinsic_infos[instr->intrinsic].has_dest);
5510 fs_reg dest = get_nir_dest(instr->dest);
5511
5512 fs_reg addr = get_nir_src(instr->src[0]);
5513
5514 assert(op != BRW_AOP_INC && op != BRW_AOP_DEC && op != BRW_AOP_PREDEC);
5515 fs_reg data = get_nir_src(instr->src[1]);
5516
5517 if (op == BRW_AOP_FCMPWR) {
5518 fs_reg tmp = bld.vgrf(data.type, 2);
5519 fs_reg sources[2] = { data, get_nir_src(instr->src[2]) };
5520 bld.LOAD_PAYLOAD(tmp, sources, 2, 0);
5521 data = tmp;
5522 }
5523
5524 bld.emit(SHADER_OPCODE_A64_UNTYPED_ATOMIC_LOGICAL,
5525 dest, addr, data, brw_imm_ud(op));
5526 }
5527
5528 void
5529 fs_visitor::nir_emit_texture(const fs_builder &bld, nir_tex_instr *instr)
5530 {
5531 unsigned texture = instr->texture_index;
5532 unsigned sampler = instr->sampler_index;
5533
5534 fs_reg srcs[TEX_LOGICAL_NUM_SRCS];
5535
5536 srcs[TEX_LOGICAL_SRC_SURFACE] = brw_imm_ud(texture);
5537 srcs[TEX_LOGICAL_SRC_SAMPLER] = brw_imm_ud(sampler);
5538
5539 int lod_components = 0;
5540
5541 /* The hardware requires a LOD for buffer textures */
5542 if (instr->sampler_dim == GLSL_SAMPLER_DIM_BUF)
5543 srcs[TEX_LOGICAL_SRC_LOD] = brw_imm_d(0);
5544
5545 uint32_t header_bits = 0;
5546 for (unsigned i = 0; i < instr->num_srcs; i++) {
5547 fs_reg src = get_nir_src(instr->src[i].src);
5548 switch (instr->src[i].src_type) {
5549 case nir_tex_src_bias:
5550 srcs[TEX_LOGICAL_SRC_LOD] =
5551 retype(get_nir_src_imm(instr->src[i].src), BRW_REGISTER_TYPE_F);
5552 break;
5553 case nir_tex_src_comparator:
5554 srcs[TEX_LOGICAL_SRC_SHADOW_C] = retype(src, BRW_REGISTER_TYPE_F);
5555 break;
5556 case nir_tex_src_coord:
5557 switch (instr->op) {
5558 case nir_texop_txf:
5559 case nir_texop_txf_ms:
5560 case nir_texop_txf_ms_mcs:
5561 case nir_texop_samples_identical:
5562 srcs[TEX_LOGICAL_SRC_COORDINATE] = retype(src, BRW_REGISTER_TYPE_D);
5563 break;
5564 default:
5565 srcs[TEX_LOGICAL_SRC_COORDINATE] = retype(src, BRW_REGISTER_TYPE_F);
5566 break;
5567 }
5568 break;
5569 case nir_tex_src_ddx:
5570 srcs[TEX_LOGICAL_SRC_LOD] = retype(src, BRW_REGISTER_TYPE_F);
5571 lod_components = nir_tex_instr_src_size(instr, i);
5572 break;
5573 case nir_tex_src_ddy:
5574 srcs[TEX_LOGICAL_SRC_LOD2] = retype(src, BRW_REGISTER_TYPE_F);
5575 break;
5576 case nir_tex_src_lod:
5577 switch (instr->op) {
5578 case nir_texop_txs:
5579 srcs[TEX_LOGICAL_SRC_LOD] =
5580 retype(get_nir_src_imm(instr->src[i].src), BRW_REGISTER_TYPE_UD);
5581 break;
5582 case nir_texop_txf:
5583 srcs[TEX_LOGICAL_SRC_LOD] =
5584 retype(get_nir_src_imm(instr->src[i].src), BRW_REGISTER_TYPE_D);
5585 break;
5586 default:
5587 srcs[TEX_LOGICAL_SRC_LOD] =
5588 retype(get_nir_src_imm(instr->src[i].src), BRW_REGISTER_TYPE_F);
5589 break;
5590 }
5591 break;
5592 case nir_tex_src_min_lod:
5593 srcs[TEX_LOGICAL_SRC_MIN_LOD] =
5594 retype(get_nir_src_imm(instr->src[i].src), BRW_REGISTER_TYPE_F);
5595 break;
5596 case nir_tex_src_ms_index:
5597 srcs[TEX_LOGICAL_SRC_SAMPLE_INDEX] = retype(src, BRW_REGISTER_TYPE_UD);
5598 break;
5599
5600 case nir_tex_src_offset: {
5601 uint32_t offset_bits = 0;
5602 if (brw_texture_offset(instr, i, &offset_bits)) {
5603 header_bits |= offset_bits;
5604 } else {
5605 srcs[TEX_LOGICAL_SRC_TG4_OFFSET] =
5606 retype(src, BRW_REGISTER_TYPE_D);
5607 }
5608 break;
5609 }
5610
5611 case nir_tex_src_projector:
5612 unreachable("should be lowered");
5613
5614 case nir_tex_src_texture_offset: {
5615 /* Emit code to evaluate the actual indexing expression */
5616 fs_reg tmp = vgrf(glsl_type::uint_type);
5617 bld.ADD(tmp, src, brw_imm_ud(texture));
5618 srcs[TEX_LOGICAL_SRC_SURFACE] = bld.emit_uniformize(tmp);
5619 break;
5620 }
5621
5622 case nir_tex_src_sampler_offset: {
5623 /* Emit code to evaluate the actual indexing expression */
5624 fs_reg tmp = vgrf(glsl_type::uint_type);
5625 bld.ADD(tmp, src, brw_imm_ud(sampler));
5626 srcs[TEX_LOGICAL_SRC_SAMPLER] = bld.emit_uniformize(tmp);
5627 break;
5628 }
5629
5630 case nir_tex_src_texture_handle:
5631 assert(nir_tex_instr_src_index(instr, nir_tex_src_texture_offset) == -1);
5632 srcs[TEX_LOGICAL_SRC_SURFACE] = fs_reg();
5633 srcs[TEX_LOGICAL_SRC_SURFACE_HANDLE] = bld.emit_uniformize(src);
5634 break;
5635
5636 case nir_tex_src_sampler_handle:
5637 assert(nir_tex_instr_src_index(instr, nir_tex_src_sampler_offset) == -1);
5638 srcs[TEX_LOGICAL_SRC_SAMPLER] = fs_reg();
5639 srcs[TEX_LOGICAL_SRC_SAMPLER_HANDLE] = bld.emit_uniformize(src);
5640 break;
5641
5642 case nir_tex_src_ms_mcs:
5643 assert(instr->op == nir_texop_txf_ms);
5644 srcs[TEX_LOGICAL_SRC_MCS] = retype(src, BRW_REGISTER_TYPE_D);
5645 break;
5646
5647 case nir_tex_src_plane: {
5648 const uint32_t plane = nir_src_as_uint(instr->src[i].src);
5649 const uint32_t texture_index =
5650 instr->texture_index +
5651 stage_prog_data->binding_table.plane_start[plane] -
5652 stage_prog_data->binding_table.texture_start;
5653
5654 srcs[TEX_LOGICAL_SRC_SURFACE] = brw_imm_ud(texture_index);
5655 break;
5656 }
5657
5658 default:
5659 unreachable("unknown texture source");
5660 }
5661 }
5662
5663 if (srcs[TEX_LOGICAL_SRC_MCS].file == BAD_FILE &&
5664 (instr->op == nir_texop_txf_ms ||
5665 instr->op == nir_texop_samples_identical)) {
5666 if (devinfo->gen >= 7 &&
5667 key_tex->compressed_multisample_layout_mask & (1 << texture)) {
5668 srcs[TEX_LOGICAL_SRC_MCS] =
5669 emit_mcs_fetch(srcs[TEX_LOGICAL_SRC_COORDINATE],
5670 instr->coord_components,
5671 srcs[TEX_LOGICAL_SRC_SURFACE],
5672 srcs[TEX_LOGICAL_SRC_SURFACE_HANDLE]);
5673 } else {
5674 srcs[TEX_LOGICAL_SRC_MCS] = brw_imm_ud(0u);
5675 }
5676 }
5677
5678 srcs[TEX_LOGICAL_SRC_COORD_COMPONENTS] = brw_imm_d(instr->coord_components);
5679 srcs[TEX_LOGICAL_SRC_GRAD_COMPONENTS] = brw_imm_d(lod_components);
5680
5681 enum opcode opcode;
5682 switch (instr->op) {
5683 case nir_texop_tex:
5684 opcode = SHADER_OPCODE_TEX_LOGICAL;
5685 break;
5686 case nir_texop_txb:
5687 opcode = FS_OPCODE_TXB_LOGICAL;
5688 break;
5689 case nir_texop_txl:
5690 opcode = SHADER_OPCODE_TXL_LOGICAL;
5691 break;
5692 case nir_texop_txd:
5693 opcode = SHADER_OPCODE_TXD_LOGICAL;
5694 break;
5695 case nir_texop_txf:
5696 opcode = SHADER_OPCODE_TXF_LOGICAL;
5697 break;
5698 case nir_texop_txf_ms:
5699 if ((key_tex->msaa_16 & (1 << sampler)))
5700 opcode = SHADER_OPCODE_TXF_CMS_W_LOGICAL;
5701 else
5702 opcode = SHADER_OPCODE_TXF_CMS_LOGICAL;
5703 break;
5704 case nir_texop_txf_ms_mcs:
5705 opcode = SHADER_OPCODE_TXF_MCS_LOGICAL;
5706 break;
5707 case nir_texop_query_levels:
5708 case nir_texop_txs:
5709 opcode = SHADER_OPCODE_TXS_LOGICAL;
5710 break;
5711 case nir_texop_lod:
5712 opcode = SHADER_OPCODE_LOD_LOGICAL;
5713 break;
5714 case nir_texop_tg4:
5715 if (srcs[TEX_LOGICAL_SRC_TG4_OFFSET].file != BAD_FILE)
5716 opcode = SHADER_OPCODE_TG4_OFFSET_LOGICAL;
5717 else
5718 opcode = SHADER_OPCODE_TG4_LOGICAL;
5719 break;
5720 case nir_texop_texture_samples:
5721 opcode = SHADER_OPCODE_SAMPLEINFO_LOGICAL;
5722 break;
5723 case nir_texop_samples_identical: {
5724 fs_reg dst = retype(get_nir_dest(instr->dest), BRW_REGISTER_TYPE_D);
5725
5726 /* If mcs is an immediate value, it means there is no MCS. In that case
5727 * just return false.
5728 */
5729 if (srcs[TEX_LOGICAL_SRC_MCS].file == BRW_IMMEDIATE_VALUE) {
5730 bld.MOV(dst, brw_imm_ud(0u));
5731 } else if ((key_tex->msaa_16 & (1 << sampler))) {
5732 fs_reg tmp = vgrf(glsl_type::uint_type);
5733 bld.OR(tmp, srcs[TEX_LOGICAL_SRC_MCS],
5734 offset(srcs[TEX_LOGICAL_SRC_MCS], bld, 1));
5735 bld.CMP(dst, tmp, brw_imm_ud(0u), BRW_CONDITIONAL_EQ);
5736 } else {
5737 bld.CMP(dst, srcs[TEX_LOGICAL_SRC_MCS], brw_imm_ud(0u),
5738 BRW_CONDITIONAL_EQ);
5739 }
5740 return;
5741 }
5742 default:
5743 unreachable("unknown texture opcode");
5744 }
5745
5746 if (instr->op == nir_texop_tg4) {
5747 if (instr->component == 1 &&
5748 key_tex->gather_channel_quirk_mask & (1 << texture)) {
5749 /* gather4 sampler is broken for green channel on RG32F --
5750 * we must ask for blue instead.
5751 */
5752 header_bits |= 2 << 16;
5753 } else {
5754 header_bits |= instr->component << 16;
5755 }
5756 }
5757
5758 fs_reg dst = bld.vgrf(brw_type_for_nir_type(devinfo, instr->dest_type), 4);
5759 fs_inst *inst = bld.emit(opcode, dst, srcs, ARRAY_SIZE(srcs));
5760 inst->offset = header_bits;
5761
5762 const unsigned dest_size = nir_tex_instr_dest_size(instr);
5763 if (devinfo->gen >= 9 &&
5764 instr->op != nir_texop_tg4 && instr->op != nir_texop_query_levels) {
5765 unsigned write_mask = instr->dest.is_ssa ?
5766 nir_ssa_def_components_read(&instr->dest.ssa):
5767 (1 << dest_size) - 1;
5768 assert(write_mask != 0); /* dead code should have been eliminated */
5769 inst->size_written = util_last_bit(write_mask) *
5770 inst->dst.component_size(inst->exec_size);
5771 } else {
5772 inst->size_written = 4 * inst->dst.component_size(inst->exec_size);
5773 }
5774
5775 if (srcs[TEX_LOGICAL_SRC_SHADOW_C].file != BAD_FILE)
5776 inst->shadow_compare = true;
5777
5778 if (instr->op == nir_texop_tg4 && devinfo->gen == 6)
5779 emit_gen6_gather_wa(key_tex->gen6_gather_wa[texture], dst);
5780
5781 fs_reg nir_dest[4];
5782 for (unsigned i = 0; i < dest_size; i++)
5783 nir_dest[i] = offset(dst, bld, i);
5784
5785 if (instr->op == nir_texop_query_levels) {
5786 /* # levels is in .w */
5787 nir_dest[0] = offset(dst, bld, 3);
5788 } else if (instr->op == nir_texop_txs &&
5789 dest_size >= 3 && devinfo->gen < 7) {
5790 /* Gen4-6 return 0 instead of 1 for single layer surfaces. */
5791 fs_reg depth = offset(dst, bld, 2);
5792 nir_dest[2] = vgrf(glsl_type::int_type);
5793 bld.emit_minmax(nir_dest[2], depth, brw_imm_d(1), BRW_CONDITIONAL_GE);
5794 }
5795
5796 bld.LOAD_PAYLOAD(get_nir_dest(instr->dest), nir_dest, dest_size, 0);
5797 }
5798
5799 void
5800 fs_visitor::nir_emit_jump(const fs_builder &bld, nir_jump_instr *instr)
5801 {
5802 switch (instr->type) {
5803 case nir_jump_break:
5804 bld.emit(BRW_OPCODE_BREAK);
5805 break;
5806 case nir_jump_continue:
5807 bld.emit(BRW_OPCODE_CONTINUE);
5808 break;
5809 case nir_jump_return:
5810 default:
5811 unreachable("unknown jump");
5812 }
5813 }
5814
5815 /*
5816 * This helper takes a source register and un/shuffles it into the destination
5817 * register.
5818 *
5819 * If source type size is smaller than destination type size the operation
5820 * needed is a component shuffle. The opposite case would be an unshuffle. If
5821 * source/destination type size is equal a shuffle is done that would be
5822 * equivalent to a simple MOV.
5823 *
5824 * For example, if source is a 16-bit type and destination is 32-bit. A 3
5825 * components .xyz 16-bit vector on SIMD8 would be.
5826 *
5827 * |x1|x2|x3|x4|x5|x6|x7|x8|y1|y2|y3|y4|y5|y6|y7|y8|
5828 * |z1|z2|z3|z4|z5|z6|z7|z8| | | | | | | | |
5829 *
5830 * This helper will return the following 2 32-bit components with the 16-bit
5831 * values shuffled:
5832 *
5833 * |x1 y1|x2 y2|x3 y3|x4 y4|x5 y5|x6 y6|x7 y7|x8 y8|
5834 * |z1 |z2 |z3 |z4 |z5 |z6 |z7 |z8 |
5835 *
5836 * For unshuffle, the example would be the opposite, a 64-bit type source
5837 * and a 32-bit destination. A 2 component .xy 64-bit vector on SIMD8
5838 * would be:
5839 *
5840 * | x1l x1h | x2l x2h | x3l x3h | x4l x4h |
5841 * | x5l x5h | x6l x6h | x7l x7h | x8l x8h |
5842 * | y1l y1h | y2l y2h | y3l y3h | y4l y4h |
5843 * | y5l y5h | y6l y6h | y7l y7h | y8l y8h |
5844 *
5845 * The returned result would be the following 4 32-bit components unshuffled:
5846 *
5847 * | x1l | x2l | x3l | x4l | x5l | x6l | x7l | x8l |
5848 * | x1h | x2h | x3h | x4h | x5h | x6h | x7h | x8h |
5849 * | y1l | y2l | y3l | y4l | y5l | y6l | y7l | y8l |
5850 * | y1h | y2h | y3h | y4h | y5h | y6h | y7h | y8h |
5851 *
5852 * - Source and destination register must not be overlapped.
5853 * - components units are measured in terms of the smaller type between
5854 * source and destination because we are un/shuffling the smaller
5855 * components from/into the bigger ones.
5856 * - first_component parameter allows skipping source components.
5857 */
5858 void
5859 shuffle_src_to_dst(const fs_builder &bld,
5860 const fs_reg &dst,
5861 const fs_reg &src,
5862 uint32_t first_component,
5863 uint32_t components)
5864 {
5865 if (type_sz(src.type) == type_sz(dst.type)) {
5866 assert(!regions_overlap(dst,
5867 type_sz(dst.type) * bld.dispatch_width() * components,
5868 offset(src, bld, first_component),
5869 type_sz(src.type) * bld.dispatch_width() * components));
5870 for (unsigned i = 0; i < components; i++) {
5871 bld.MOV(retype(offset(dst, bld, i), src.type),
5872 offset(src, bld, i + first_component));
5873 }
5874 } else if (type_sz(src.type) < type_sz(dst.type)) {
5875 /* Source is shuffled into destination */
5876 unsigned size_ratio = type_sz(dst.type) / type_sz(src.type);
5877 assert(!regions_overlap(dst,
5878 type_sz(dst.type) * bld.dispatch_width() *
5879 DIV_ROUND_UP(components, size_ratio),
5880 offset(src, bld, first_component),
5881 type_sz(src.type) * bld.dispatch_width() * components));
5882
5883 brw_reg_type shuffle_type =
5884 brw_reg_type_from_bit_size(8 * type_sz(src.type),
5885 BRW_REGISTER_TYPE_D);
5886 for (unsigned i = 0; i < components; i++) {
5887 fs_reg shuffle_component_i =
5888 subscript(offset(dst, bld, i / size_ratio),
5889 shuffle_type, i % size_ratio);
5890 bld.MOV(shuffle_component_i,
5891 retype(offset(src, bld, i + first_component), shuffle_type));
5892 }
5893 } else {
5894 /* Source is unshuffled into destination */
5895 unsigned size_ratio = type_sz(src.type) / type_sz(dst.type);
5896 assert(!regions_overlap(dst,
5897 type_sz(dst.type) * bld.dispatch_width() * components,
5898 offset(src, bld, first_component / size_ratio),
5899 type_sz(src.type) * bld.dispatch_width() *
5900 DIV_ROUND_UP(components + (first_component % size_ratio),
5901 size_ratio)));
5902
5903 brw_reg_type shuffle_type =
5904 brw_reg_type_from_bit_size(8 * type_sz(dst.type),
5905 BRW_REGISTER_TYPE_D);
5906 for (unsigned i = 0; i < components; i++) {
5907 fs_reg shuffle_component_i =
5908 subscript(offset(src, bld, (first_component + i) / size_ratio),
5909 shuffle_type, (first_component + i) % size_ratio);
5910 bld.MOV(retype(offset(dst, bld, i), shuffle_type),
5911 shuffle_component_i);
5912 }
5913 }
5914 }
5915
5916 void
5917 shuffle_from_32bit_read(const fs_builder &bld,
5918 const fs_reg &dst,
5919 const fs_reg &src,
5920 uint32_t first_component,
5921 uint32_t components)
5922 {
5923 assert(type_sz(src.type) == 4);
5924
5925 /* This function takes components in units of the destination type while
5926 * shuffle_src_to_dst takes components in units of the smallest type
5927 */
5928 if (type_sz(dst.type) > 4) {
5929 assert(type_sz(dst.type) == 8);
5930 first_component *= 2;
5931 components *= 2;
5932 }
5933
5934 shuffle_src_to_dst(bld, dst, src, first_component, components);
5935 }
5936
5937 fs_reg
5938 setup_imm_df(const fs_builder &bld, double v)
5939 {
5940 const struct gen_device_info *devinfo = bld.shader->devinfo;
5941 assert(devinfo->gen >= 7);
5942
5943 if (devinfo->gen >= 8)
5944 return brw_imm_df(v);
5945
5946 /* gen7.5 does not support DF immediates straighforward but the DIM
5947 * instruction allows to set the 64-bit immediate value.
5948 */
5949 if (devinfo->is_haswell) {
5950 const fs_builder ubld = bld.exec_all().group(1, 0);
5951 fs_reg dst = ubld.vgrf(BRW_REGISTER_TYPE_DF, 1);
5952 ubld.DIM(dst, brw_imm_df(v));
5953 return component(dst, 0);
5954 }
5955
5956 /* gen7 does not support DF immediates, so we generate a 64-bit constant by
5957 * writing the low 32-bit of the constant to suboffset 0 of a VGRF and
5958 * the high 32-bit to suboffset 4 and then applying a stride of 0.
5959 *
5960 * Alternatively, we could also produce a normal VGRF (without stride 0)
5961 * by writing to all the channels in the VGRF, however, that would hit the
5962 * gen7 bug where we have to split writes that span more than 1 register
5963 * into instructions with a width of 4 (otherwise the write to the second
5964 * register written runs into an execmask hardware bug) which isn't very
5965 * nice.
5966 */
5967 union {
5968 double d;
5969 struct {
5970 uint32_t i1;
5971 uint32_t i2;
5972 };
5973 } di;
5974
5975 di.d = v;
5976
5977 const fs_builder ubld = bld.exec_all().group(1, 0);
5978 const fs_reg tmp = ubld.vgrf(BRW_REGISTER_TYPE_UD, 2);
5979 ubld.MOV(tmp, brw_imm_ud(di.i1));
5980 ubld.MOV(horiz_offset(tmp, 1), brw_imm_ud(di.i2));
5981
5982 return component(retype(tmp, BRW_REGISTER_TYPE_DF), 0);
5983 }
5984
5985 fs_reg
5986 setup_imm_b(const fs_builder &bld, int8_t v)
5987 {
5988 const fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_B);
5989 bld.MOV(tmp, brw_imm_w(v));
5990 return tmp;
5991 }
5992
5993 fs_reg
5994 setup_imm_ub(const fs_builder &bld, uint8_t v)
5995 {
5996 const fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_UB);
5997 bld.MOV(tmp, brw_imm_uw(v));
5998 return tmp;
5999 }