intel/fs: Add support for vec8 and vec16 ops
[mesa.git] / src / intel / compiler / brw_fs_nir.cpp
1 /*
2 * Copyright © 2010 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "compiler/glsl/ir.h"
25 #include "brw_fs.h"
26 #include "brw_nir.h"
27 #include "brw_eu.h"
28 #include "nir_search_helpers.h"
29 #include "util/u_math.h"
30 #include "util/bitscan.h"
31
32 using namespace brw;
33
34 void
35 fs_visitor::emit_nir_code()
36 {
37 emit_shader_float_controls_execution_mode();
38
39 /* emit the arrays used for inputs and outputs - load/store intrinsics will
40 * be converted to reads/writes of these arrays
41 */
42 nir_setup_outputs();
43 nir_setup_uniforms();
44 nir_emit_system_values();
45 last_scratch = ALIGN(nir->scratch_size, 4) * dispatch_width;
46
47 nir_emit_impl(nir_shader_get_entrypoint((nir_shader *)nir));
48 }
49
50 void
51 fs_visitor::nir_setup_outputs()
52 {
53 if (stage == MESA_SHADER_TESS_CTRL || stage == MESA_SHADER_FRAGMENT)
54 return;
55
56 unsigned vec4s[VARYING_SLOT_TESS_MAX] = { 0, };
57
58 /* Calculate the size of output registers in a separate pass, before
59 * allocating them. With ARB_enhanced_layouts, multiple output variables
60 * may occupy the same slot, but have different type sizes.
61 */
62 nir_foreach_shader_out_variable(var, nir) {
63 const int loc = var->data.driver_location;
64 const unsigned var_vec4s =
65 var->data.compact ? DIV_ROUND_UP(glsl_get_length(var->type), 4)
66 : type_size_vec4(var->type, true);
67 vec4s[loc] = MAX2(vec4s[loc], var_vec4s);
68 }
69
70 for (unsigned loc = 0; loc < ARRAY_SIZE(vec4s);) {
71 if (vec4s[loc] == 0) {
72 loc++;
73 continue;
74 }
75
76 unsigned reg_size = vec4s[loc];
77
78 /* Check if there are any ranges that start within this range and extend
79 * past it. If so, include them in this allocation.
80 */
81 for (unsigned i = 1; i < reg_size; i++)
82 reg_size = MAX2(vec4s[i + loc] + i, reg_size);
83
84 fs_reg reg = bld.vgrf(BRW_REGISTER_TYPE_F, 4 * reg_size);
85 for (unsigned i = 0; i < reg_size; i++)
86 outputs[loc + i] = offset(reg, bld, 4 * i);
87
88 loc += reg_size;
89 }
90 }
91
92 void
93 fs_visitor::nir_setup_uniforms()
94 {
95 /* Only the first compile gets to set up uniforms. */
96 if (push_constant_loc) {
97 assert(pull_constant_loc);
98 return;
99 }
100
101 uniforms = nir->num_uniforms / 4;
102
103 if (stage == MESA_SHADER_COMPUTE || stage == MESA_SHADER_KERNEL) {
104 /* Add uniforms for builtins after regular NIR uniforms. */
105 assert(uniforms == prog_data->nr_params);
106
107 uint32_t *param;
108 if (nir->info.cs.local_size_variable &&
109 compiler->lower_variable_group_size) {
110 param = brw_stage_prog_data_add_params(prog_data, 3);
111 for (unsigned i = 0; i < 3; i++) {
112 param[i] = (BRW_PARAM_BUILTIN_WORK_GROUP_SIZE_X + i);
113 group_size[i] = fs_reg(UNIFORM, uniforms++, BRW_REGISTER_TYPE_UD);
114 }
115 }
116
117 /* Subgroup ID must be the last uniform on the list. This will make
118 * easier later to split between cross thread and per thread
119 * uniforms.
120 */
121 param = brw_stage_prog_data_add_params(prog_data, 1);
122 *param = BRW_PARAM_BUILTIN_SUBGROUP_ID;
123 subgroup_id = fs_reg(UNIFORM, uniforms++, BRW_REGISTER_TYPE_UD);
124 }
125 }
126
127 static bool
128 emit_system_values_block(nir_block *block, fs_visitor *v)
129 {
130 fs_reg *reg;
131
132 nir_foreach_instr(instr, block) {
133 if (instr->type != nir_instr_type_intrinsic)
134 continue;
135
136 nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
137 switch (intrin->intrinsic) {
138 case nir_intrinsic_load_vertex_id:
139 case nir_intrinsic_load_base_vertex:
140 unreachable("should be lowered by nir_lower_system_values().");
141
142 case nir_intrinsic_load_vertex_id_zero_base:
143 case nir_intrinsic_load_is_indexed_draw:
144 case nir_intrinsic_load_first_vertex:
145 case nir_intrinsic_load_instance_id:
146 case nir_intrinsic_load_base_instance:
147 case nir_intrinsic_load_draw_id:
148 unreachable("should be lowered by brw_nir_lower_vs_inputs().");
149
150 case nir_intrinsic_load_invocation_id:
151 if (v->stage == MESA_SHADER_TESS_CTRL)
152 break;
153 assert(v->stage == MESA_SHADER_GEOMETRY);
154 reg = &v->nir_system_values[SYSTEM_VALUE_INVOCATION_ID];
155 if (reg->file == BAD_FILE) {
156 const fs_builder abld = v->bld.annotate("gl_InvocationID", NULL);
157 fs_reg g1(retype(brw_vec8_grf(1, 0), BRW_REGISTER_TYPE_UD));
158 fs_reg iid = abld.vgrf(BRW_REGISTER_TYPE_UD, 1);
159 abld.SHR(iid, g1, brw_imm_ud(27u));
160 *reg = iid;
161 }
162 break;
163
164 case nir_intrinsic_load_sample_pos:
165 assert(v->stage == MESA_SHADER_FRAGMENT);
166 reg = &v->nir_system_values[SYSTEM_VALUE_SAMPLE_POS];
167 if (reg->file == BAD_FILE)
168 *reg = *v->emit_samplepos_setup();
169 break;
170
171 case nir_intrinsic_load_sample_id:
172 assert(v->stage == MESA_SHADER_FRAGMENT);
173 reg = &v->nir_system_values[SYSTEM_VALUE_SAMPLE_ID];
174 if (reg->file == BAD_FILE)
175 *reg = *v->emit_sampleid_setup();
176 break;
177
178 case nir_intrinsic_load_sample_mask_in:
179 assert(v->stage == MESA_SHADER_FRAGMENT);
180 assert(v->devinfo->gen >= 7);
181 reg = &v->nir_system_values[SYSTEM_VALUE_SAMPLE_MASK_IN];
182 if (reg->file == BAD_FILE)
183 *reg = *v->emit_samplemaskin_setup();
184 break;
185
186 case nir_intrinsic_load_work_group_id:
187 assert(v->stage == MESA_SHADER_COMPUTE ||
188 v->stage == MESA_SHADER_KERNEL);
189 reg = &v->nir_system_values[SYSTEM_VALUE_WORK_GROUP_ID];
190 if (reg->file == BAD_FILE)
191 *reg = *v->emit_cs_work_group_id_setup();
192 break;
193
194 case nir_intrinsic_load_helper_invocation:
195 assert(v->stage == MESA_SHADER_FRAGMENT);
196 reg = &v->nir_system_values[SYSTEM_VALUE_HELPER_INVOCATION];
197 if (reg->file == BAD_FILE) {
198 const fs_builder abld =
199 v->bld.annotate("gl_HelperInvocation", NULL);
200
201 /* On Gen6+ (gl_HelperInvocation is only exposed on Gen7+) the
202 * pixel mask is in g1.7 of the thread payload.
203 *
204 * We move the per-channel pixel enable bit to the low bit of each
205 * channel by shifting the byte containing the pixel mask by the
206 * vector immediate 0x76543210UV.
207 *
208 * The region of <1,8,0> reads only 1 byte (the pixel masks for
209 * subspans 0 and 1) in SIMD8 and an additional byte (the pixel
210 * masks for 2 and 3) in SIMD16.
211 */
212 fs_reg shifted = abld.vgrf(BRW_REGISTER_TYPE_UW, 1);
213
214 for (unsigned i = 0; i < DIV_ROUND_UP(v->dispatch_width, 16); i++) {
215 const fs_builder hbld = abld.group(MIN2(16, v->dispatch_width), i);
216 hbld.SHR(offset(shifted, hbld, i),
217 stride(retype(brw_vec1_grf(1 + i, 7),
218 BRW_REGISTER_TYPE_UB),
219 1, 8, 0),
220 brw_imm_v(0x76543210));
221 }
222
223 /* A set bit in the pixel mask means the channel is enabled, but
224 * that is the opposite of gl_HelperInvocation so we need to invert
225 * the mask.
226 *
227 * The negate source-modifier bit of logical instructions on Gen8+
228 * performs 1's complement negation, so we can use that instead of
229 * a NOT instruction.
230 */
231 fs_reg inverted = negate(shifted);
232 if (v->devinfo->gen < 8) {
233 inverted = abld.vgrf(BRW_REGISTER_TYPE_UW);
234 abld.NOT(inverted, shifted);
235 }
236
237 /* We then resolve the 0/1 result to 0/~0 boolean values by ANDing
238 * with 1 and negating.
239 */
240 fs_reg anded = abld.vgrf(BRW_REGISTER_TYPE_UD, 1);
241 abld.AND(anded, inverted, brw_imm_uw(1));
242
243 fs_reg dst = abld.vgrf(BRW_REGISTER_TYPE_D, 1);
244 abld.MOV(dst, negate(retype(anded, BRW_REGISTER_TYPE_D)));
245 *reg = dst;
246 }
247 break;
248
249 default:
250 break;
251 }
252 }
253
254 return true;
255 }
256
257 void
258 fs_visitor::nir_emit_system_values()
259 {
260 nir_system_values = ralloc_array(mem_ctx, fs_reg, SYSTEM_VALUE_MAX);
261 for (unsigned i = 0; i < SYSTEM_VALUE_MAX; i++) {
262 nir_system_values[i] = fs_reg();
263 }
264
265 /* Always emit SUBGROUP_INVOCATION. Dead code will clean it up if we
266 * never end up using it.
267 */
268 {
269 const fs_builder abld = bld.annotate("gl_SubgroupInvocation", NULL);
270 fs_reg &reg = nir_system_values[SYSTEM_VALUE_SUBGROUP_INVOCATION];
271 reg = abld.vgrf(BRW_REGISTER_TYPE_UW);
272
273 const fs_builder allbld8 = abld.group(8, 0).exec_all();
274 allbld8.MOV(reg, brw_imm_v(0x76543210));
275 if (dispatch_width > 8)
276 allbld8.ADD(byte_offset(reg, 16), reg, brw_imm_uw(8u));
277 if (dispatch_width > 16) {
278 const fs_builder allbld16 = abld.group(16, 0).exec_all();
279 allbld16.ADD(byte_offset(reg, 32), reg, brw_imm_uw(16u));
280 }
281 }
282
283 nir_function_impl *impl = nir_shader_get_entrypoint((nir_shader *)nir);
284 nir_foreach_block(block, impl)
285 emit_system_values_block(block, this);
286 }
287
288 /*
289 * Returns a type based on a reference_type (word, float, half-float) and a
290 * given bit_size.
291 *
292 * Reference BRW_REGISTER_TYPE are HF,F,DF,W,D,UW,UD.
293 *
294 * @FIXME: 64-bit return types are always DF on integer types to maintain
295 * compability with uses of DF previously to the introduction of int64
296 * support.
297 */
298 static brw_reg_type
299 brw_reg_type_from_bit_size(const unsigned bit_size,
300 const brw_reg_type reference_type)
301 {
302 switch(reference_type) {
303 case BRW_REGISTER_TYPE_HF:
304 case BRW_REGISTER_TYPE_F:
305 case BRW_REGISTER_TYPE_DF:
306 switch(bit_size) {
307 case 16:
308 return BRW_REGISTER_TYPE_HF;
309 case 32:
310 return BRW_REGISTER_TYPE_F;
311 case 64:
312 return BRW_REGISTER_TYPE_DF;
313 default:
314 unreachable("Invalid bit size");
315 }
316 case BRW_REGISTER_TYPE_B:
317 case BRW_REGISTER_TYPE_W:
318 case BRW_REGISTER_TYPE_D:
319 case BRW_REGISTER_TYPE_Q:
320 switch(bit_size) {
321 case 8:
322 return BRW_REGISTER_TYPE_B;
323 case 16:
324 return BRW_REGISTER_TYPE_W;
325 case 32:
326 return BRW_REGISTER_TYPE_D;
327 case 64:
328 return BRW_REGISTER_TYPE_Q;
329 default:
330 unreachable("Invalid bit size");
331 }
332 case BRW_REGISTER_TYPE_UB:
333 case BRW_REGISTER_TYPE_UW:
334 case BRW_REGISTER_TYPE_UD:
335 case BRW_REGISTER_TYPE_UQ:
336 switch(bit_size) {
337 case 8:
338 return BRW_REGISTER_TYPE_UB;
339 case 16:
340 return BRW_REGISTER_TYPE_UW;
341 case 32:
342 return BRW_REGISTER_TYPE_UD;
343 case 64:
344 return BRW_REGISTER_TYPE_UQ;
345 default:
346 unreachable("Invalid bit size");
347 }
348 default:
349 unreachable("Unknown type");
350 }
351 }
352
353 void
354 fs_visitor::nir_emit_impl(nir_function_impl *impl)
355 {
356 nir_locals = ralloc_array(mem_ctx, fs_reg, impl->reg_alloc);
357 for (unsigned i = 0; i < impl->reg_alloc; i++) {
358 nir_locals[i] = fs_reg();
359 }
360
361 foreach_list_typed(nir_register, reg, node, &impl->registers) {
362 unsigned array_elems =
363 reg->num_array_elems == 0 ? 1 : reg->num_array_elems;
364 unsigned size = array_elems * reg->num_components;
365 const brw_reg_type reg_type = reg->bit_size == 8 ? BRW_REGISTER_TYPE_B :
366 brw_reg_type_from_bit_size(reg->bit_size, BRW_REGISTER_TYPE_F);
367 nir_locals[reg->index] = bld.vgrf(reg_type, size);
368 }
369
370 nir_ssa_values = reralloc(mem_ctx, nir_ssa_values, fs_reg,
371 impl->ssa_alloc);
372
373 nir_emit_cf_list(&impl->body);
374 }
375
376 void
377 fs_visitor::nir_emit_cf_list(exec_list *list)
378 {
379 exec_list_validate(list);
380 foreach_list_typed(nir_cf_node, node, node, list) {
381 switch (node->type) {
382 case nir_cf_node_if:
383 nir_emit_if(nir_cf_node_as_if(node));
384 break;
385
386 case nir_cf_node_loop:
387 nir_emit_loop(nir_cf_node_as_loop(node));
388 break;
389
390 case nir_cf_node_block:
391 nir_emit_block(nir_cf_node_as_block(node));
392 break;
393
394 default:
395 unreachable("Invalid CFG node block");
396 }
397 }
398 }
399
400 void
401 fs_visitor::nir_emit_if(nir_if *if_stmt)
402 {
403 bool invert;
404 fs_reg cond_reg;
405
406 /* If the condition has the form !other_condition, use other_condition as
407 * the source, but invert the predicate on the if instruction.
408 */
409 nir_alu_instr *cond = nir_src_as_alu_instr(if_stmt->condition);
410 if (cond != NULL && cond->op == nir_op_inot) {
411 invert = true;
412 cond_reg = get_nir_src(cond->src[0].src);
413 } else {
414 invert = false;
415 cond_reg = get_nir_src(if_stmt->condition);
416 }
417
418 /* first, put the condition into f0 */
419 fs_inst *inst = bld.MOV(bld.null_reg_d(),
420 retype(cond_reg, BRW_REGISTER_TYPE_D));
421 inst->conditional_mod = BRW_CONDITIONAL_NZ;
422
423 bld.IF(BRW_PREDICATE_NORMAL)->predicate_inverse = invert;
424
425 nir_emit_cf_list(&if_stmt->then_list);
426
427 if (!nir_cf_list_is_empty_block(&if_stmt->else_list)) {
428 bld.emit(BRW_OPCODE_ELSE);
429 nir_emit_cf_list(&if_stmt->else_list);
430 }
431
432 bld.emit(BRW_OPCODE_ENDIF);
433
434 if (devinfo->gen < 7)
435 limit_dispatch_width(16, "Non-uniform control flow unsupported "
436 "in SIMD32 mode.");
437 }
438
439 void
440 fs_visitor::nir_emit_loop(nir_loop *loop)
441 {
442 bld.emit(BRW_OPCODE_DO);
443
444 nir_emit_cf_list(&loop->body);
445
446 bld.emit(BRW_OPCODE_WHILE);
447
448 if (devinfo->gen < 7)
449 limit_dispatch_width(16, "Non-uniform control flow unsupported "
450 "in SIMD32 mode.");
451 }
452
453 void
454 fs_visitor::nir_emit_block(nir_block *block)
455 {
456 nir_foreach_instr(instr, block) {
457 nir_emit_instr(instr);
458 }
459 }
460
461 void
462 fs_visitor::nir_emit_instr(nir_instr *instr)
463 {
464 const fs_builder abld = bld.annotate(NULL, instr);
465
466 switch (instr->type) {
467 case nir_instr_type_alu:
468 nir_emit_alu(abld, nir_instr_as_alu(instr), true);
469 break;
470
471 case nir_instr_type_deref:
472 unreachable("All derefs should've been lowered");
473 break;
474
475 case nir_instr_type_intrinsic:
476 switch (stage) {
477 case MESA_SHADER_VERTEX:
478 nir_emit_vs_intrinsic(abld, nir_instr_as_intrinsic(instr));
479 break;
480 case MESA_SHADER_TESS_CTRL:
481 nir_emit_tcs_intrinsic(abld, nir_instr_as_intrinsic(instr));
482 break;
483 case MESA_SHADER_TESS_EVAL:
484 nir_emit_tes_intrinsic(abld, nir_instr_as_intrinsic(instr));
485 break;
486 case MESA_SHADER_GEOMETRY:
487 nir_emit_gs_intrinsic(abld, nir_instr_as_intrinsic(instr));
488 break;
489 case MESA_SHADER_FRAGMENT:
490 nir_emit_fs_intrinsic(abld, nir_instr_as_intrinsic(instr));
491 break;
492 case MESA_SHADER_COMPUTE:
493 case MESA_SHADER_KERNEL:
494 nir_emit_cs_intrinsic(abld, nir_instr_as_intrinsic(instr));
495 break;
496 default:
497 unreachable("unsupported shader stage");
498 }
499 break;
500
501 case nir_instr_type_tex:
502 nir_emit_texture(abld, nir_instr_as_tex(instr));
503 break;
504
505 case nir_instr_type_load_const:
506 nir_emit_load_const(abld, nir_instr_as_load_const(instr));
507 break;
508
509 case nir_instr_type_ssa_undef:
510 /* We create a new VGRF for undefs on every use (by handling
511 * them in get_nir_src()), rather than for each definition.
512 * This helps register coalescing eliminate MOVs from undef.
513 */
514 break;
515
516 case nir_instr_type_jump:
517 nir_emit_jump(abld, nir_instr_as_jump(instr));
518 break;
519
520 default:
521 unreachable("unknown instruction type");
522 }
523 }
524
525 /**
526 * Recognizes a parent instruction of nir_op_extract_* and changes the type to
527 * match instr.
528 */
529 bool
530 fs_visitor::optimize_extract_to_float(nir_alu_instr *instr,
531 const fs_reg &result)
532 {
533 if (!instr->src[0].src.is_ssa ||
534 !instr->src[0].src.ssa->parent_instr)
535 return false;
536
537 if (instr->src[0].src.ssa->parent_instr->type != nir_instr_type_alu)
538 return false;
539
540 nir_alu_instr *src0 =
541 nir_instr_as_alu(instr->src[0].src.ssa->parent_instr);
542
543 if (src0->op != nir_op_extract_u8 && src0->op != nir_op_extract_u16 &&
544 src0->op != nir_op_extract_i8 && src0->op != nir_op_extract_i16)
545 return false;
546
547 unsigned element = nir_src_as_uint(src0->src[1].src);
548
549 /* Element type to extract.*/
550 const brw_reg_type type = brw_int_type(
551 src0->op == nir_op_extract_u16 || src0->op == nir_op_extract_i16 ? 2 : 1,
552 src0->op == nir_op_extract_i16 || src0->op == nir_op_extract_i8);
553
554 fs_reg op0 = get_nir_src(src0->src[0].src);
555 op0.type = brw_type_for_nir_type(devinfo,
556 (nir_alu_type)(nir_op_infos[src0->op].input_types[0] |
557 nir_src_bit_size(src0->src[0].src)));
558 op0 = offset(op0, bld, src0->src[0].swizzle[0]);
559
560 bld.MOV(result, subscript(op0, type, element));
561 return true;
562 }
563
564 bool
565 fs_visitor::optimize_frontfacing_ternary(nir_alu_instr *instr,
566 const fs_reg &result)
567 {
568 nir_intrinsic_instr *src0 = nir_src_as_intrinsic(instr->src[0].src);
569 if (src0 == NULL || src0->intrinsic != nir_intrinsic_load_front_face)
570 return false;
571
572 if (!nir_src_is_const(instr->src[1].src) ||
573 !nir_src_is_const(instr->src[2].src))
574 return false;
575
576 const float value1 = nir_src_as_float(instr->src[1].src);
577 const float value2 = nir_src_as_float(instr->src[2].src);
578 if (fabsf(value1) != 1.0f || fabsf(value2) != 1.0f)
579 return false;
580
581 /* nir_opt_algebraic should have gotten rid of bcsel(b, a, a) */
582 assert(value1 == -value2);
583
584 fs_reg tmp = vgrf(glsl_type::int_type);
585
586 if (devinfo->gen >= 12) {
587 /* Bit 15 of g1.1 is 0 if the polygon is front facing. */
588 fs_reg g1 = fs_reg(retype(brw_vec1_grf(1, 1), BRW_REGISTER_TYPE_W));
589
590 /* For (gl_FrontFacing ? 1.0 : -1.0), emit:
591 *
592 * or(8) tmp.1<2>W g0.0<0,1,0>W 0x00003f80W
593 * and(8) dst<1>D tmp<8,8,1>D 0xbf800000D
594 *
595 * and negate the result for (gl_FrontFacing ? -1.0 : 1.0).
596 */
597 bld.OR(subscript(tmp, BRW_REGISTER_TYPE_W, 1),
598 g1, brw_imm_uw(0x3f80));
599
600 if (value1 == -1.0f)
601 bld.MOV(tmp, negate(tmp));
602
603 } else if (devinfo->gen >= 6) {
604 /* Bit 15 of g0.0 is 0 if the polygon is front facing. */
605 fs_reg g0 = fs_reg(retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_W));
606
607 /* For (gl_FrontFacing ? 1.0 : -1.0), emit:
608 *
609 * or(8) tmp.1<2>W g0.0<0,1,0>W 0x00003f80W
610 * and(8) dst<1>D tmp<8,8,1>D 0xbf800000D
611 *
612 * and negate g0.0<0,1,0>W for (gl_FrontFacing ? -1.0 : 1.0).
613 *
614 * This negation looks like it's safe in practice, because bits 0:4 will
615 * surely be TRIANGLES
616 */
617
618 if (value1 == -1.0f) {
619 g0.negate = true;
620 }
621
622 bld.OR(subscript(tmp, BRW_REGISTER_TYPE_W, 1),
623 g0, brw_imm_uw(0x3f80));
624 } else {
625 /* Bit 31 of g1.6 is 0 if the polygon is front facing. */
626 fs_reg g1_6 = fs_reg(retype(brw_vec1_grf(1, 6), BRW_REGISTER_TYPE_D));
627
628 /* For (gl_FrontFacing ? 1.0 : -1.0), emit:
629 *
630 * or(8) tmp<1>D g1.6<0,1,0>D 0x3f800000D
631 * and(8) dst<1>D tmp<8,8,1>D 0xbf800000D
632 *
633 * and negate g1.6<0,1,0>D for (gl_FrontFacing ? -1.0 : 1.0).
634 *
635 * This negation looks like it's safe in practice, because bits 0:4 will
636 * surely be TRIANGLES
637 */
638
639 if (value1 == -1.0f) {
640 g1_6.negate = true;
641 }
642
643 bld.OR(tmp, g1_6, brw_imm_d(0x3f800000));
644 }
645 bld.AND(retype(result, BRW_REGISTER_TYPE_D), tmp, brw_imm_d(0xbf800000));
646
647 return true;
648 }
649
650 static void
651 emit_find_msb_using_lzd(const fs_builder &bld,
652 const fs_reg &result,
653 const fs_reg &src,
654 bool is_signed)
655 {
656 fs_inst *inst;
657 fs_reg temp = src;
658
659 if (is_signed) {
660 /* LZD of an absolute value source almost always does the right
661 * thing. There are two problem values:
662 *
663 * * 0x80000000. Since abs(0x80000000) == 0x80000000, LZD returns
664 * 0. However, findMSB(int(0x80000000)) == 30.
665 *
666 * * 0xffffffff. Since abs(0xffffffff) == 1, LZD returns
667 * 31. Section 8.8 (Integer Functions) of the GLSL 4.50 spec says:
668 *
669 * For a value of zero or negative one, -1 will be returned.
670 *
671 * * Negative powers of two. LZD(abs(-(1<<x))) returns x, but
672 * findMSB(-(1<<x)) should return x-1.
673 *
674 * For all negative number cases, including 0x80000000 and
675 * 0xffffffff, the correct value is obtained from LZD if instead of
676 * negating the (already negative) value the logical-not is used. A
677 * conditonal logical-not can be achieved in two instructions.
678 */
679 temp = bld.vgrf(BRW_REGISTER_TYPE_D);
680
681 bld.ASR(temp, src, brw_imm_d(31));
682 bld.XOR(temp, temp, src);
683 }
684
685 bld.LZD(retype(result, BRW_REGISTER_TYPE_UD),
686 retype(temp, BRW_REGISTER_TYPE_UD));
687
688 /* LZD counts from the MSB side, while GLSL's findMSB() wants the count
689 * from the LSB side. Subtract the result from 31 to convert the MSB
690 * count into an LSB count. If no bits are set, LZD will return 32.
691 * 31-32 = -1, which is exactly what findMSB() is supposed to return.
692 */
693 inst = bld.ADD(result, retype(result, BRW_REGISTER_TYPE_D), brw_imm_d(31));
694 inst->src[0].negate = true;
695 }
696
697 static brw_rnd_mode
698 brw_rnd_mode_from_nir_op (const nir_op op) {
699 switch (op) {
700 case nir_op_f2f16_rtz:
701 return BRW_RND_MODE_RTZ;
702 case nir_op_f2f16_rtne:
703 return BRW_RND_MODE_RTNE;
704 default:
705 unreachable("Operation doesn't support rounding mode");
706 }
707 }
708
709 static brw_rnd_mode
710 brw_rnd_mode_from_execution_mode(unsigned execution_mode)
711 {
712 if (nir_has_any_rounding_mode_rtne(execution_mode))
713 return BRW_RND_MODE_RTNE;
714 if (nir_has_any_rounding_mode_rtz(execution_mode))
715 return BRW_RND_MODE_RTZ;
716 return BRW_RND_MODE_UNSPECIFIED;
717 }
718
719 fs_reg
720 fs_visitor::prepare_alu_destination_and_sources(const fs_builder &bld,
721 nir_alu_instr *instr,
722 fs_reg *op,
723 bool need_dest)
724 {
725 fs_reg result =
726 need_dest ? get_nir_dest(instr->dest.dest) : bld.null_reg_ud();
727
728 result.type = brw_type_for_nir_type(devinfo,
729 (nir_alu_type)(nir_op_infos[instr->op].output_type |
730 nir_dest_bit_size(instr->dest.dest)));
731
732 assert(!instr->dest.saturate);
733
734 for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
735 /* We don't lower to source modifiers so they should not exist. */
736 assert(!instr->src[i].abs);
737 assert(!instr->src[i].negate);
738
739 op[i] = get_nir_src(instr->src[i].src);
740 op[i].type = brw_type_for_nir_type(devinfo,
741 (nir_alu_type)(nir_op_infos[instr->op].input_types[i] |
742 nir_src_bit_size(instr->src[i].src)));
743 }
744
745 /* Move and vecN instrutions may still be vectored. Return the raw,
746 * vectored source and destination so that fs_visitor::nir_emit_alu can
747 * handle it. Other callers should not have to handle these kinds of
748 * instructions.
749 */
750 switch (instr->op) {
751 case nir_op_mov:
752 case nir_op_vec2:
753 case nir_op_vec3:
754 case nir_op_vec4:
755 case nir_op_vec8:
756 case nir_op_vec16:
757 return result;
758 default:
759 break;
760 }
761
762 /* At this point, we have dealt with any instruction that operates on
763 * more than a single channel. Therefore, we can just adjust the source
764 * and destination registers for that channel and emit the instruction.
765 */
766 unsigned channel = 0;
767 if (nir_op_infos[instr->op].output_size == 0) {
768 /* Since NIR is doing the scalarizing for us, we should only ever see
769 * vectorized operations with a single channel.
770 */
771 assert(util_bitcount(instr->dest.write_mask) == 1);
772 channel = ffs(instr->dest.write_mask) - 1;
773
774 result = offset(result, bld, channel);
775 }
776
777 for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
778 assert(nir_op_infos[instr->op].input_sizes[i] < 2);
779 op[i] = offset(op[i], bld, instr->src[i].swizzle[channel]);
780 }
781
782 return result;
783 }
784
785 void
786 fs_visitor::resolve_inot_sources(const fs_builder &bld, nir_alu_instr *instr,
787 fs_reg *op)
788 {
789 for (unsigned i = 0; i < 2; i++) {
790 nir_alu_instr *inot_instr = nir_src_as_alu_instr(instr->src[i].src);
791
792 if (inot_instr != NULL && inot_instr->op == nir_op_inot) {
793 /* The source of the inot is now the source of instr. */
794 prepare_alu_destination_and_sources(bld, inot_instr, &op[i], false);
795
796 assert(!op[i].negate);
797 op[i].negate = true;
798 } else {
799 op[i] = resolve_source_modifiers(op[i]);
800 }
801 }
802 }
803
804 bool
805 fs_visitor::try_emit_b2fi_of_inot(const fs_builder &bld,
806 fs_reg result,
807 nir_alu_instr *instr)
808 {
809 if (devinfo->gen < 6 || devinfo->gen >= 12)
810 return false;
811
812 nir_alu_instr *inot_instr = nir_src_as_alu_instr(instr->src[0].src);
813
814 if (inot_instr == NULL || inot_instr->op != nir_op_inot)
815 return false;
816
817 /* HF is also possible as a destination on BDW+. For nir_op_b2i, the set
818 * of valid size-changing combinations is a bit more complex.
819 *
820 * The source restriction is just because I was lazy about generating the
821 * constant below.
822 */
823 if (nir_dest_bit_size(instr->dest.dest) != 32 ||
824 nir_src_bit_size(inot_instr->src[0].src) != 32)
825 return false;
826
827 /* b2[fi](inot(a)) maps a=0 => 1, a=-1 => 0. Since a can only be 0 or -1,
828 * this is float(1 + a).
829 */
830 fs_reg op;
831
832 prepare_alu_destination_and_sources(bld, inot_instr, &op, false);
833
834 /* Ignore the saturate modifier, if there is one. The result of the
835 * arithmetic can only be 0 or 1, so the clamping will do nothing anyway.
836 */
837 bld.ADD(result, op, brw_imm_d(1));
838
839 return true;
840 }
841
842 /**
843 * Emit code for nir_op_fsign possibly fused with a nir_op_fmul
844 *
845 * If \c instr is not the \c nir_op_fsign, then \c fsign_src is the index of
846 * the source of \c instr that is a \c nir_op_fsign.
847 */
848 void
849 fs_visitor::emit_fsign(const fs_builder &bld, const nir_alu_instr *instr,
850 fs_reg result, fs_reg *op, unsigned fsign_src)
851 {
852 fs_inst *inst;
853
854 assert(instr->op == nir_op_fsign || instr->op == nir_op_fmul);
855 assert(fsign_src < nir_op_infos[instr->op].num_inputs);
856
857 if (instr->op != nir_op_fsign) {
858 const nir_alu_instr *const fsign_instr =
859 nir_src_as_alu_instr(instr->src[fsign_src].src);
860
861 /* op[fsign_src] has the nominal result of the fsign, and op[1 -
862 * fsign_src] has the other multiply source. This must be rearranged so
863 * that op[0] is the source of the fsign op[1] is the other multiply
864 * source.
865 */
866 if (fsign_src != 0)
867 op[1] = op[0];
868
869 op[0] = get_nir_src(fsign_instr->src[0].src);
870
871 const nir_alu_type t =
872 (nir_alu_type)(nir_op_infos[instr->op].input_types[0] |
873 nir_src_bit_size(fsign_instr->src[0].src));
874
875 op[0].type = brw_type_for_nir_type(devinfo, t);
876
877 unsigned channel = 0;
878 if (nir_op_infos[instr->op].output_size == 0) {
879 /* Since NIR is doing the scalarizing for us, we should only ever see
880 * vectorized operations with a single channel.
881 */
882 assert(util_bitcount(instr->dest.write_mask) == 1);
883 channel = ffs(instr->dest.write_mask) - 1;
884 }
885
886 op[0] = offset(op[0], bld, fsign_instr->src[0].swizzle[channel]);
887 }
888
889 if (type_sz(op[0].type) == 2) {
890 /* AND(val, 0x8000) gives the sign bit.
891 *
892 * Predicated OR ORs 1.0 (0x3c00) with the sign bit if val is not zero.
893 */
894 fs_reg zero = retype(brw_imm_uw(0), BRW_REGISTER_TYPE_HF);
895 bld.CMP(bld.null_reg_f(), op[0], zero, BRW_CONDITIONAL_NZ);
896
897 op[0].type = BRW_REGISTER_TYPE_UW;
898 result.type = BRW_REGISTER_TYPE_UW;
899 bld.AND(result, op[0], brw_imm_uw(0x8000u));
900
901 if (instr->op == nir_op_fsign)
902 inst = bld.OR(result, result, brw_imm_uw(0x3c00u));
903 else {
904 /* Use XOR here to get the result sign correct. */
905 inst = bld.XOR(result, result, retype(op[1], BRW_REGISTER_TYPE_UW));
906 }
907
908 inst->predicate = BRW_PREDICATE_NORMAL;
909 } else if (type_sz(op[0].type) == 4) {
910 /* AND(val, 0x80000000) gives the sign bit.
911 *
912 * Predicated OR ORs 1.0 (0x3f800000) with the sign bit if val is not
913 * zero.
914 */
915 bld.CMP(bld.null_reg_f(), op[0], brw_imm_f(0.0f), BRW_CONDITIONAL_NZ);
916
917 op[0].type = BRW_REGISTER_TYPE_UD;
918 result.type = BRW_REGISTER_TYPE_UD;
919 bld.AND(result, op[0], brw_imm_ud(0x80000000u));
920
921 if (instr->op == nir_op_fsign)
922 inst = bld.OR(result, result, brw_imm_ud(0x3f800000u));
923 else {
924 /* Use XOR here to get the result sign correct. */
925 inst = bld.XOR(result, result, retype(op[1], BRW_REGISTER_TYPE_UD));
926 }
927
928 inst->predicate = BRW_PREDICATE_NORMAL;
929 } else {
930 /* For doubles we do the same but we need to consider:
931 *
932 * - 2-src instructions can't operate with 64-bit immediates
933 * - The sign is encoded in the high 32-bit of each DF
934 * - We need to produce a DF result.
935 */
936
937 fs_reg zero = vgrf(glsl_type::double_type);
938 bld.MOV(zero, setup_imm_df(bld, 0.0));
939 bld.CMP(bld.null_reg_df(), op[0], zero, BRW_CONDITIONAL_NZ);
940
941 bld.MOV(result, zero);
942
943 fs_reg r = subscript(result, BRW_REGISTER_TYPE_UD, 1);
944 bld.AND(r, subscript(op[0], BRW_REGISTER_TYPE_UD, 1),
945 brw_imm_ud(0x80000000u));
946
947 if (instr->op == nir_op_fsign) {
948 set_predicate(BRW_PREDICATE_NORMAL,
949 bld.OR(r, r, brw_imm_ud(0x3ff00000u)));
950 } else {
951 /* This could be done better in some cases. If the scale is an
952 * immediate with the low 32-bits all 0, emitting a separate XOR and
953 * OR would allow an algebraic optimization to remove the OR. There
954 * are currently zero instances of fsign(double(x))*IMM in shader-db
955 * or any test suite, so it is hard to care at this time.
956 */
957 fs_reg result_int64 = retype(result, BRW_REGISTER_TYPE_UQ);
958 inst = bld.XOR(result_int64, result_int64,
959 retype(op[1], BRW_REGISTER_TYPE_UQ));
960 }
961 }
962 }
963
964 /**
965 * Deteremine whether sources of a nir_op_fmul can be fused with a nir_op_fsign
966 *
967 * Checks the operands of a \c nir_op_fmul to determine whether or not
968 * \c emit_fsign could fuse the multiplication with the \c sign() calculation.
969 *
970 * \param instr The multiplication instruction
971 *
972 * \param fsign_src The source of \c instr that may or may not be a
973 * \c nir_op_fsign
974 */
975 static bool
976 can_fuse_fmul_fsign(nir_alu_instr *instr, unsigned fsign_src)
977 {
978 assert(instr->op == nir_op_fmul);
979
980 nir_alu_instr *const fsign_instr =
981 nir_src_as_alu_instr(instr->src[fsign_src].src);
982
983 /* Rules:
984 *
985 * 1. instr->src[fsign_src] must be a nir_op_fsign.
986 * 2. The nir_op_fsign can only be used by this multiplication.
987 * 3. The source that is the nir_op_fsign does not have source modifiers.
988 * \c emit_fsign only examines the source modifiers of the source of the
989 * \c nir_op_fsign.
990 *
991 * The nir_op_fsign must also not have the saturate modifier, but steps
992 * have already been taken (in nir_opt_algebraic) to ensure that.
993 */
994 return fsign_instr != NULL && fsign_instr->op == nir_op_fsign &&
995 is_used_once(fsign_instr);
996 }
997
998 void
999 fs_visitor::nir_emit_alu(const fs_builder &bld, nir_alu_instr *instr,
1000 bool need_dest)
1001 {
1002 struct brw_wm_prog_key *fs_key = (struct brw_wm_prog_key *) this->key;
1003 fs_inst *inst;
1004 unsigned execution_mode =
1005 bld.shader->nir->info.float_controls_execution_mode;
1006
1007 fs_reg op[NIR_MAX_VEC_COMPONENTS];
1008 fs_reg result = prepare_alu_destination_and_sources(bld, instr, op, need_dest);
1009
1010 switch (instr->op) {
1011 case nir_op_mov:
1012 case nir_op_vec2:
1013 case nir_op_vec3:
1014 case nir_op_vec4:
1015 case nir_op_vec8:
1016 case nir_op_vec16: {
1017 fs_reg temp = result;
1018 bool need_extra_copy = false;
1019 for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
1020 if (!instr->src[i].src.is_ssa &&
1021 instr->dest.dest.reg.reg == instr->src[i].src.reg.reg) {
1022 need_extra_copy = true;
1023 temp = bld.vgrf(result.type, 4);
1024 break;
1025 }
1026 }
1027
1028 for (unsigned i = 0; i < 4; i++) {
1029 if (!(instr->dest.write_mask & (1 << i)))
1030 continue;
1031
1032 if (instr->op == nir_op_mov) {
1033 inst = bld.MOV(offset(temp, bld, i),
1034 offset(op[0], bld, instr->src[0].swizzle[i]));
1035 } else {
1036 inst = bld.MOV(offset(temp, bld, i),
1037 offset(op[i], bld, instr->src[i].swizzle[0]));
1038 }
1039 }
1040
1041 /* In this case the source and destination registers were the same,
1042 * so we need to insert an extra set of moves in order to deal with
1043 * any swizzling.
1044 */
1045 if (need_extra_copy) {
1046 for (unsigned i = 0; i < 4; i++) {
1047 if (!(instr->dest.write_mask & (1 << i)))
1048 continue;
1049
1050 bld.MOV(offset(result, bld, i), offset(temp, bld, i));
1051 }
1052 }
1053 return;
1054 }
1055
1056 case nir_op_i2f32:
1057 case nir_op_u2f32:
1058 if (optimize_extract_to_float(instr, result))
1059 return;
1060 inst = bld.MOV(result, op[0]);
1061 break;
1062
1063 case nir_op_f2f16_rtne:
1064 case nir_op_f2f16_rtz:
1065 case nir_op_f2f16: {
1066 brw_rnd_mode rnd = BRW_RND_MODE_UNSPECIFIED;
1067
1068 if (nir_op_f2f16 == instr->op)
1069 rnd = brw_rnd_mode_from_execution_mode(execution_mode);
1070 else
1071 rnd = brw_rnd_mode_from_nir_op(instr->op);
1072
1073 if (BRW_RND_MODE_UNSPECIFIED != rnd)
1074 bld.emit(SHADER_OPCODE_RND_MODE, bld.null_reg_ud(), brw_imm_d(rnd));
1075
1076 /* In theory, it would be better to use BRW_OPCODE_F32TO16. Depending
1077 * on the HW gen, it is a special hw opcode or just a MOV, and
1078 * brw_F32TO16 (at brw_eu_emit) would do the work to chose.
1079 *
1080 * But if we want to use that opcode, we need to provide support on
1081 * different optimizations and lowerings. As right now HF support is
1082 * only for gen8+, it will be better to use directly the MOV, and use
1083 * BRW_OPCODE_F32TO16 when/if we work for HF support on gen7.
1084 */
1085 assert(type_sz(op[0].type) < 8); /* brw_nir_lower_conversions */
1086 inst = bld.MOV(result, op[0]);
1087 break;
1088 }
1089
1090 case nir_op_b2i8:
1091 case nir_op_b2i16:
1092 case nir_op_b2i32:
1093 case nir_op_b2i64:
1094 case nir_op_b2f16:
1095 case nir_op_b2f32:
1096 case nir_op_b2f64:
1097 if (try_emit_b2fi_of_inot(bld, result, instr))
1098 break;
1099 op[0].type = BRW_REGISTER_TYPE_D;
1100 op[0].negate = !op[0].negate;
1101 /* fallthrough */
1102 case nir_op_i2f64:
1103 case nir_op_i2i64:
1104 case nir_op_u2f64:
1105 case nir_op_u2u64:
1106 case nir_op_f2f64:
1107 case nir_op_f2i64:
1108 case nir_op_f2u64:
1109 case nir_op_i2i32:
1110 case nir_op_u2u32:
1111 case nir_op_f2i32:
1112 case nir_op_f2u32:
1113 case nir_op_i2f16:
1114 case nir_op_i2i16:
1115 case nir_op_u2f16:
1116 case nir_op_u2u16:
1117 case nir_op_f2i16:
1118 case nir_op_f2u16:
1119 case nir_op_i2i8:
1120 case nir_op_u2u8:
1121 case nir_op_f2i8:
1122 case nir_op_f2u8:
1123 if (result.type == BRW_REGISTER_TYPE_B ||
1124 result.type == BRW_REGISTER_TYPE_UB ||
1125 result.type == BRW_REGISTER_TYPE_HF)
1126 assert(type_sz(op[0].type) < 8); /* brw_nir_lower_conversions */
1127
1128 if (op[0].type == BRW_REGISTER_TYPE_B ||
1129 op[0].type == BRW_REGISTER_TYPE_UB ||
1130 op[0].type == BRW_REGISTER_TYPE_HF)
1131 assert(type_sz(result.type) < 8); /* brw_nir_lower_conversions */
1132
1133 inst = bld.MOV(result, op[0]);
1134 break;
1135
1136 case nir_op_fsat:
1137 inst = bld.MOV(result, op[0]);
1138 inst->saturate = true;
1139 break;
1140
1141 case nir_op_fneg:
1142 case nir_op_ineg:
1143 op[0].negate = true;
1144 inst = bld.MOV(result, op[0]);
1145 break;
1146
1147 case nir_op_fabs:
1148 case nir_op_iabs:
1149 op[0].negate = false;
1150 op[0].abs = true;
1151 inst = bld.MOV(result, op[0]);
1152 break;
1153
1154 case nir_op_f2f32:
1155 if (nir_has_any_rounding_mode_enabled(execution_mode)) {
1156 brw_rnd_mode rnd =
1157 brw_rnd_mode_from_execution_mode(execution_mode);
1158 bld.emit(SHADER_OPCODE_RND_MODE, bld.null_reg_ud(),
1159 brw_imm_d(rnd));
1160 }
1161
1162 if (op[0].type == BRW_REGISTER_TYPE_HF)
1163 assert(type_sz(result.type) < 8); /* brw_nir_lower_conversions */
1164
1165 inst = bld.MOV(result, op[0]);
1166 break;
1167
1168 case nir_op_fsign:
1169 emit_fsign(bld, instr, result, op, 0);
1170 break;
1171
1172 case nir_op_frcp:
1173 inst = bld.emit(SHADER_OPCODE_RCP, result, op[0]);
1174 break;
1175
1176 case nir_op_fexp2:
1177 inst = bld.emit(SHADER_OPCODE_EXP2, result, op[0]);
1178 break;
1179
1180 case nir_op_flog2:
1181 inst = bld.emit(SHADER_OPCODE_LOG2, result, op[0]);
1182 break;
1183
1184 case nir_op_fsin:
1185 inst = bld.emit(SHADER_OPCODE_SIN, result, op[0]);
1186 break;
1187
1188 case nir_op_fcos:
1189 inst = bld.emit(SHADER_OPCODE_COS, result, op[0]);
1190 break;
1191
1192 case nir_op_fddx:
1193 if (fs_key->high_quality_derivatives) {
1194 inst = bld.emit(FS_OPCODE_DDX_FINE, result, op[0]);
1195 } else {
1196 inst = bld.emit(FS_OPCODE_DDX_COARSE, result, op[0]);
1197 }
1198 break;
1199 case nir_op_fddx_fine:
1200 inst = bld.emit(FS_OPCODE_DDX_FINE, result, op[0]);
1201 break;
1202 case nir_op_fddx_coarse:
1203 inst = bld.emit(FS_OPCODE_DDX_COARSE, result, op[0]);
1204 break;
1205 case nir_op_fddy:
1206 if (fs_key->high_quality_derivatives) {
1207 inst = bld.emit(FS_OPCODE_DDY_FINE, result, op[0]);
1208 } else {
1209 inst = bld.emit(FS_OPCODE_DDY_COARSE, result, op[0]);
1210 }
1211 break;
1212 case nir_op_fddy_fine:
1213 inst = bld.emit(FS_OPCODE_DDY_FINE, result, op[0]);
1214 break;
1215 case nir_op_fddy_coarse:
1216 inst = bld.emit(FS_OPCODE_DDY_COARSE, result, op[0]);
1217 break;
1218
1219 case nir_op_fadd:
1220 if (nir_has_any_rounding_mode_enabled(execution_mode)) {
1221 brw_rnd_mode rnd =
1222 brw_rnd_mode_from_execution_mode(execution_mode);
1223 bld.emit(SHADER_OPCODE_RND_MODE, bld.null_reg_ud(),
1224 brw_imm_d(rnd));
1225 }
1226 /* fallthrough */
1227 case nir_op_iadd:
1228 inst = bld.ADD(result, op[0], op[1]);
1229 break;
1230
1231 case nir_op_iadd_sat:
1232 case nir_op_uadd_sat:
1233 inst = bld.ADD(result, op[0], op[1]);
1234 inst->saturate = true;
1235 break;
1236
1237 case nir_op_isub_sat:
1238 bld.emit(SHADER_OPCODE_ISUB_SAT, result, op[0], op[1]);
1239 break;
1240
1241 case nir_op_usub_sat:
1242 bld.emit(SHADER_OPCODE_USUB_SAT, result, op[0], op[1]);
1243 break;
1244
1245 case nir_op_irhadd:
1246 case nir_op_urhadd:
1247 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1248 inst = bld.AVG(result, op[0], op[1]);
1249 break;
1250
1251 case nir_op_ihadd:
1252 case nir_op_uhadd: {
1253 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1254 fs_reg tmp = bld.vgrf(result.type);
1255
1256 if (devinfo->gen >= 8) {
1257 op[0] = resolve_source_modifiers(op[0]);
1258 op[1] = resolve_source_modifiers(op[1]);
1259 }
1260
1261 /* AVG(x, y) - ((x ^ y) & 1) */
1262 bld.XOR(tmp, op[0], op[1]);
1263 bld.AND(tmp, tmp, retype(brw_imm_ud(1), result.type));
1264 bld.AVG(result, op[0], op[1]);
1265 inst = bld.ADD(result, result, tmp);
1266 inst->src[1].negate = true;
1267 break;
1268 }
1269
1270 case nir_op_fmul:
1271 for (unsigned i = 0; i < 2; i++) {
1272 if (can_fuse_fmul_fsign(instr, i)) {
1273 emit_fsign(bld, instr, result, op, i);
1274 return;
1275 }
1276 }
1277
1278 /* We emit the rounding mode after the previous fsign optimization since
1279 * it won't result in a MUL, but will try to negate the value by other
1280 * means.
1281 */
1282 if (nir_has_any_rounding_mode_enabled(execution_mode)) {
1283 brw_rnd_mode rnd =
1284 brw_rnd_mode_from_execution_mode(execution_mode);
1285 bld.emit(SHADER_OPCODE_RND_MODE, bld.null_reg_ud(),
1286 brw_imm_d(rnd));
1287 }
1288
1289 inst = bld.MUL(result, op[0], op[1]);
1290 break;
1291
1292 case nir_op_imul_2x32_64:
1293 case nir_op_umul_2x32_64:
1294 bld.MUL(result, op[0], op[1]);
1295 break;
1296
1297 case nir_op_imul_32x16:
1298 case nir_op_umul_32x16: {
1299 const bool ud = instr->op == nir_op_umul_32x16;
1300
1301 assert(nir_dest_bit_size(instr->dest.dest) == 32);
1302
1303 /* Before Gen7, the order of the 32-bit source and the 16-bit source was
1304 * swapped. The extension isn't enabled on those platforms, so don't
1305 * pretend to support the differences.
1306 */
1307 assert(devinfo->gen >= 7);
1308
1309 if (op[1].file == IMM)
1310 op[1] = ud ? brw_imm_uw(op[1].ud) : brw_imm_w(op[1].d);
1311 else {
1312 const enum brw_reg_type word_type =
1313 ud ? BRW_REGISTER_TYPE_UW : BRW_REGISTER_TYPE_W;
1314
1315 op[1] = subscript(op[1], word_type, 0);
1316 }
1317
1318 const enum brw_reg_type dword_type =
1319 ud ? BRW_REGISTER_TYPE_UD : BRW_REGISTER_TYPE_D;
1320
1321 bld.MUL(result, retype(op[0], dword_type), op[1]);
1322 break;
1323 }
1324
1325 case nir_op_imul:
1326 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1327 bld.MUL(result, op[0], op[1]);
1328 break;
1329
1330 case nir_op_imul_high:
1331 case nir_op_umul_high:
1332 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1333 bld.emit(SHADER_OPCODE_MULH, result, op[0], op[1]);
1334 break;
1335
1336 case nir_op_idiv:
1337 case nir_op_udiv:
1338 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1339 bld.emit(SHADER_OPCODE_INT_QUOTIENT, result, op[0], op[1]);
1340 break;
1341
1342 case nir_op_uadd_carry:
1343 unreachable("Should have been lowered by carry_to_arith().");
1344
1345 case nir_op_usub_borrow:
1346 unreachable("Should have been lowered by borrow_to_arith().");
1347
1348 case nir_op_umod:
1349 case nir_op_irem:
1350 /* According to the sign table for INT DIV in the Ivy Bridge PRM, it
1351 * appears that our hardware just does the right thing for signed
1352 * remainder.
1353 */
1354 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1355 bld.emit(SHADER_OPCODE_INT_REMAINDER, result, op[0], op[1]);
1356 break;
1357
1358 case nir_op_imod: {
1359 /* Get a regular C-style remainder. If a % b == 0, set the predicate. */
1360 bld.emit(SHADER_OPCODE_INT_REMAINDER, result, op[0], op[1]);
1361
1362 /* Math instructions don't support conditional mod */
1363 inst = bld.MOV(bld.null_reg_d(), result);
1364 inst->conditional_mod = BRW_CONDITIONAL_NZ;
1365
1366 /* Now, we need to determine if signs of the sources are different.
1367 * When we XOR the sources, the top bit is 0 if they are the same and 1
1368 * if they are different. We can then use a conditional modifier to
1369 * turn that into a predicate. This leads us to an XOR.l instruction.
1370 *
1371 * Technically, according to the PRM, you're not allowed to use .l on a
1372 * XOR instruction. However, emperical experiments and Curro's reading
1373 * of the simulator source both indicate that it's safe.
1374 */
1375 fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_D);
1376 inst = bld.XOR(tmp, op[0], op[1]);
1377 inst->predicate = BRW_PREDICATE_NORMAL;
1378 inst->conditional_mod = BRW_CONDITIONAL_L;
1379
1380 /* If the result of the initial remainder operation is non-zero and the
1381 * two sources have different signs, add in a copy of op[1] to get the
1382 * final integer modulus value.
1383 */
1384 inst = bld.ADD(result, result, op[1]);
1385 inst->predicate = BRW_PREDICATE_NORMAL;
1386 break;
1387 }
1388
1389 case nir_op_flt32:
1390 case nir_op_fge32:
1391 case nir_op_feq32:
1392 case nir_op_fneu32: {
1393 fs_reg dest = result;
1394
1395 const uint32_t bit_size = nir_src_bit_size(instr->src[0].src);
1396 if (bit_size != 32)
1397 dest = bld.vgrf(op[0].type, 1);
1398
1399 bld.CMP(dest, op[0], op[1], brw_cmod_for_nir_comparison(instr->op));
1400
1401 if (bit_size > 32) {
1402 bld.MOV(result, subscript(dest, BRW_REGISTER_TYPE_UD, 0));
1403 } else if(bit_size < 32) {
1404 /* When we convert the result to 32-bit we need to be careful and do
1405 * it as a signed conversion to get sign extension (for 32-bit true)
1406 */
1407 const brw_reg_type src_type =
1408 brw_reg_type_from_bit_size(bit_size, BRW_REGISTER_TYPE_D);
1409
1410 bld.MOV(retype(result, BRW_REGISTER_TYPE_D), retype(dest, src_type));
1411 }
1412 break;
1413 }
1414
1415 case nir_op_ilt32:
1416 case nir_op_ult32:
1417 case nir_op_ige32:
1418 case nir_op_uge32:
1419 case nir_op_ieq32:
1420 case nir_op_ine32: {
1421 fs_reg dest = result;
1422
1423 /* On Gen11 we have an additional issue being that src1 cannot be a byte
1424 * type. So we convert both operands for the comparison.
1425 */
1426 fs_reg temp_op[2];
1427 temp_op[0] = bld.fix_byte_src(op[0]);
1428 temp_op[1] = bld.fix_byte_src(op[1]);
1429
1430 const uint32_t bit_size = type_sz(temp_op[0].type) * 8;
1431 if (bit_size != 32)
1432 dest = bld.vgrf(temp_op[0].type, 1);
1433
1434 bld.CMP(dest, temp_op[0], temp_op[1],
1435 brw_cmod_for_nir_comparison(instr->op));
1436
1437 if (bit_size > 32) {
1438 bld.MOV(result, subscript(dest, BRW_REGISTER_TYPE_UD, 0));
1439 } else if (bit_size < 32) {
1440 /* When we convert the result to 32-bit we need to be careful and do
1441 * it as a signed conversion to get sign extension (for 32-bit true)
1442 */
1443 const brw_reg_type src_type =
1444 brw_reg_type_from_bit_size(bit_size, BRW_REGISTER_TYPE_D);
1445
1446 bld.MOV(retype(result, BRW_REGISTER_TYPE_D), retype(dest, src_type));
1447 }
1448 break;
1449 }
1450
1451 case nir_op_inot:
1452 if (devinfo->gen >= 8) {
1453 nir_alu_instr *inot_src_instr = nir_src_as_alu_instr(instr->src[0].src);
1454
1455 if (inot_src_instr != NULL &&
1456 (inot_src_instr->op == nir_op_ior ||
1457 inot_src_instr->op == nir_op_ixor ||
1458 inot_src_instr->op == nir_op_iand)) {
1459 /* The sources of the source logical instruction are now the
1460 * sources of the instruction that will be generated.
1461 */
1462 prepare_alu_destination_and_sources(bld, inot_src_instr, op, false);
1463 resolve_inot_sources(bld, inot_src_instr, op);
1464
1465 /* Smash all of the sources and destination to be signed. This
1466 * doesn't matter for the operation of the instruction, but cmod
1467 * propagation fails on unsigned sources with negation (due to
1468 * fs_inst::can_do_cmod returning false).
1469 */
1470 result.type =
1471 brw_type_for_nir_type(devinfo,
1472 (nir_alu_type)(nir_type_int |
1473 nir_dest_bit_size(instr->dest.dest)));
1474 op[0].type =
1475 brw_type_for_nir_type(devinfo,
1476 (nir_alu_type)(nir_type_int |
1477 nir_src_bit_size(inot_src_instr->src[0].src)));
1478 op[1].type =
1479 brw_type_for_nir_type(devinfo,
1480 (nir_alu_type)(nir_type_int |
1481 nir_src_bit_size(inot_src_instr->src[1].src)));
1482
1483 /* For XOR, only invert one of the sources. Arbitrarily choose
1484 * the first source.
1485 */
1486 op[0].negate = !op[0].negate;
1487 if (inot_src_instr->op != nir_op_ixor)
1488 op[1].negate = !op[1].negate;
1489
1490 switch (inot_src_instr->op) {
1491 case nir_op_ior:
1492 bld.AND(result, op[0], op[1]);
1493 return;
1494
1495 case nir_op_iand:
1496 bld.OR(result, op[0], op[1]);
1497 return;
1498
1499 case nir_op_ixor:
1500 bld.XOR(result, op[0], op[1]);
1501 return;
1502
1503 default:
1504 unreachable("impossible opcode");
1505 }
1506 }
1507 op[0] = resolve_source_modifiers(op[0]);
1508 }
1509 bld.NOT(result, op[0]);
1510 break;
1511 case nir_op_ixor:
1512 if (devinfo->gen >= 8) {
1513 resolve_inot_sources(bld, instr, op);
1514 }
1515 bld.XOR(result, op[0], op[1]);
1516 break;
1517 case nir_op_ior:
1518 if (devinfo->gen >= 8) {
1519 resolve_inot_sources(bld, instr, op);
1520 }
1521 bld.OR(result, op[0], op[1]);
1522 break;
1523 case nir_op_iand:
1524 if (devinfo->gen >= 8) {
1525 resolve_inot_sources(bld, instr, op);
1526 }
1527 bld.AND(result, op[0], op[1]);
1528 break;
1529
1530 case nir_op_fdot2:
1531 case nir_op_fdot3:
1532 case nir_op_fdot4:
1533 case nir_op_b32all_fequal2:
1534 case nir_op_b32all_iequal2:
1535 case nir_op_b32all_fequal3:
1536 case nir_op_b32all_iequal3:
1537 case nir_op_b32all_fequal4:
1538 case nir_op_b32all_iequal4:
1539 case nir_op_b32any_fnequal2:
1540 case nir_op_b32any_inequal2:
1541 case nir_op_b32any_fnequal3:
1542 case nir_op_b32any_inequal3:
1543 case nir_op_b32any_fnequal4:
1544 case nir_op_b32any_inequal4:
1545 unreachable("Lowered by nir_lower_alu_reductions");
1546
1547 case nir_op_ldexp:
1548 unreachable("not reached: should be handled by ldexp_to_arith()");
1549
1550 case nir_op_fsqrt:
1551 inst = bld.emit(SHADER_OPCODE_SQRT, result, op[0]);
1552 break;
1553
1554 case nir_op_frsq:
1555 inst = bld.emit(SHADER_OPCODE_RSQ, result, op[0]);
1556 break;
1557
1558 case nir_op_i2b32:
1559 case nir_op_f2b32: {
1560 uint32_t bit_size = nir_src_bit_size(instr->src[0].src);
1561 if (bit_size == 64) {
1562 /* two-argument instructions can't take 64-bit immediates */
1563 fs_reg zero;
1564 fs_reg tmp;
1565
1566 if (instr->op == nir_op_f2b32) {
1567 zero = vgrf(glsl_type::double_type);
1568 tmp = vgrf(glsl_type::double_type);
1569 bld.MOV(zero, setup_imm_df(bld, 0.0));
1570 } else {
1571 zero = vgrf(glsl_type::int64_t_type);
1572 tmp = vgrf(glsl_type::int64_t_type);
1573 bld.MOV(zero, brw_imm_q(0));
1574 }
1575
1576 /* A SIMD16 execution needs to be split in two instructions, so use
1577 * a vgrf instead of the flag register as dst so instruction splitting
1578 * works
1579 */
1580 bld.CMP(tmp, op[0], zero, BRW_CONDITIONAL_NZ);
1581 bld.MOV(result, subscript(tmp, BRW_REGISTER_TYPE_UD, 0));
1582 } else {
1583 fs_reg zero;
1584 if (bit_size == 32) {
1585 zero = instr->op == nir_op_f2b32 ? brw_imm_f(0.0f) : brw_imm_d(0);
1586 } else {
1587 assert(bit_size == 16);
1588 zero = instr->op == nir_op_f2b32 ?
1589 retype(brw_imm_w(0), BRW_REGISTER_TYPE_HF) : brw_imm_w(0);
1590 }
1591 bld.CMP(result, op[0], zero, BRW_CONDITIONAL_NZ);
1592 }
1593 break;
1594 }
1595
1596 case nir_op_ftrunc:
1597 inst = bld.RNDZ(result, op[0]);
1598 if (devinfo->gen < 6) {
1599 set_condmod(BRW_CONDITIONAL_R, inst);
1600 set_predicate(BRW_PREDICATE_NORMAL,
1601 bld.ADD(result, result, brw_imm_f(1.0f)));
1602 inst = bld.MOV(result, result); /* for potential saturation */
1603 }
1604 break;
1605
1606 case nir_op_fceil: {
1607 op[0].negate = !op[0].negate;
1608 fs_reg temp = vgrf(glsl_type::float_type);
1609 bld.RNDD(temp, op[0]);
1610 temp.negate = true;
1611 inst = bld.MOV(result, temp);
1612 break;
1613 }
1614 case nir_op_ffloor:
1615 inst = bld.RNDD(result, op[0]);
1616 break;
1617 case nir_op_ffract:
1618 inst = bld.FRC(result, op[0]);
1619 break;
1620 case nir_op_fround_even:
1621 inst = bld.RNDE(result, op[0]);
1622 if (devinfo->gen < 6) {
1623 set_condmod(BRW_CONDITIONAL_R, inst);
1624 set_predicate(BRW_PREDICATE_NORMAL,
1625 bld.ADD(result, result, brw_imm_f(1.0f)));
1626 inst = bld.MOV(result, result); /* for potential saturation */
1627 }
1628 break;
1629
1630 case nir_op_fquantize2f16: {
1631 fs_reg tmp16 = bld.vgrf(BRW_REGISTER_TYPE_D);
1632 fs_reg tmp32 = bld.vgrf(BRW_REGISTER_TYPE_F);
1633 fs_reg zero = bld.vgrf(BRW_REGISTER_TYPE_F);
1634
1635 /* The destination stride must be at least as big as the source stride. */
1636 tmp16.type = BRW_REGISTER_TYPE_W;
1637 tmp16.stride = 2;
1638
1639 /* Check for denormal */
1640 fs_reg abs_src0 = op[0];
1641 abs_src0.abs = true;
1642 bld.CMP(bld.null_reg_f(), abs_src0, brw_imm_f(ldexpf(1.0, -14)),
1643 BRW_CONDITIONAL_L);
1644 /* Get the appropriately signed zero */
1645 bld.AND(retype(zero, BRW_REGISTER_TYPE_UD),
1646 retype(op[0], BRW_REGISTER_TYPE_UD),
1647 brw_imm_ud(0x80000000));
1648 /* Do the actual F32 -> F16 -> F32 conversion */
1649 bld.emit(BRW_OPCODE_F32TO16, tmp16, op[0]);
1650 bld.emit(BRW_OPCODE_F16TO32, tmp32, tmp16);
1651 /* Select that or zero based on normal status */
1652 inst = bld.SEL(result, zero, tmp32);
1653 inst->predicate = BRW_PREDICATE_NORMAL;
1654 break;
1655 }
1656
1657 case nir_op_imin:
1658 case nir_op_umin:
1659 case nir_op_fmin:
1660 inst = bld.emit_minmax(result, op[0], op[1], BRW_CONDITIONAL_L);
1661 break;
1662
1663 case nir_op_imax:
1664 case nir_op_umax:
1665 case nir_op_fmax:
1666 inst = bld.emit_minmax(result, op[0], op[1], BRW_CONDITIONAL_GE);
1667 break;
1668
1669 case nir_op_pack_snorm_2x16:
1670 case nir_op_pack_snorm_4x8:
1671 case nir_op_pack_unorm_2x16:
1672 case nir_op_pack_unorm_4x8:
1673 case nir_op_unpack_snorm_2x16:
1674 case nir_op_unpack_snorm_4x8:
1675 case nir_op_unpack_unorm_2x16:
1676 case nir_op_unpack_unorm_4x8:
1677 case nir_op_unpack_half_2x16:
1678 case nir_op_pack_half_2x16:
1679 unreachable("not reached: should be handled by lower_packing_builtins");
1680
1681 case nir_op_unpack_half_2x16_split_x_flush_to_zero:
1682 assert(FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP16 & execution_mode);
1683 /* Fall-through */
1684 case nir_op_unpack_half_2x16_split_x:
1685 inst = bld.emit(BRW_OPCODE_F16TO32, result,
1686 subscript(op[0], BRW_REGISTER_TYPE_UW, 0));
1687 break;
1688
1689 case nir_op_unpack_half_2x16_split_y_flush_to_zero:
1690 assert(FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP16 & execution_mode);
1691 /* Fall-through */
1692 case nir_op_unpack_half_2x16_split_y:
1693 inst = bld.emit(BRW_OPCODE_F16TO32, result,
1694 subscript(op[0], BRW_REGISTER_TYPE_UW, 1));
1695 break;
1696
1697 case nir_op_pack_64_2x32_split:
1698 case nir_op_pack_32_2x16_split:
1699 bld.emit(FS_OPCODE_PACK, result, op[0], op[1]);
1700 break;
1701
1702 case nir_op_unpack_64_2x32_split_x:
1703 case nir_op_unpack_64_2x32_split_y: {
1704 if (instr->op == nir_op_unpack_64_2x32_split_x)
1705 bld.MOV(result, subscript(op[0], BRW_REGISTER_TYPE_UD, 0));
1706 else
1707 bld.MOV(result, subscript(op[0], BRW_REGISTER_TYPE_UD, 1));
1708 break;
1709 }
1710
1711 case nir_op_unpack_32_2x16_split_x:
1712 case nir_op_unpack_32_2x16_split_y: {
1713 if (instr->op == nir_op_unpack_32_2x16_split_x)
1714 bld.MOV(result, subscript(op[0], BRW_REGISTER_TYPE_UW, 0));
1715 else
1716 bld.MOV(result, subscript(op[0], BRW_REGISTER_TYPE_UW, 1));
1717 break;
1718 }
1719
1720 case nir_op_fpow:
1721 inst = bld.emit(SHADER_OPCODE_POW, result, op[0], op[1]);
1722 break;
1723
1724 case nir_op_bitfield_reverse:
1725 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1726 bld.BFREV(result, op[0]);
1727 break;
1728
1729 case nir_op_bit_count:
1730 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1731 bld.CBIT(result, op[0]);
1732 break;
1733
1734 case nir_op_ufind_msb: {
1735 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1736 emit_find_msb_using_lzd(bld, result, op[0], false);
1737 break;
1738 }
1739
1740 case nir_op_uclz:
1741 assert(nir_dest_bit_size(instr->dest.dest) == 32);
1742 bld.LZD(retype(result, BRW_REGISTER_TYPE_UD), op[0]);
1743 break;
1744
1745 case nir_op_ifind_msb: {
1746 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1747
1748 if (devinfo->gen < 7) {
1749 emit_find_msb_using_lzd(bld, result, op[0], true);
1750 } else {
1751 bld.FBH(retype(result, BRW_REGISTER_TYPE_UD), op[0]);
1752
1753 /* FBH counts from the MSB side, while GLSL's findMSB() wants the
1754 * count from the LSB side. If FBH didn't return an error
1755 * (0xFFFFFFFF), then subtract the result from 31 to convert the MSB
1756 * count into an LSB count.
1757 */
1758 bld.CMP(bld.null_reg_d(), result, brw_imm_d(-1), BRW_CONDITIONAL_NZ);
1759
1760 inst = bld.ADD(result, result, brw_imm_d(31));
1761 inst->predicate = BRW_PREDICATE_NORMAL;
1762 inst->src[0].negate = true;
1763 }
1764 break;
1765 }
1766
1767 case nir_op_find_lsb:
1768 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1769
1770 if (devinfo->gen < 7) {
1771 fs_reg temp = vgrf(glsl_type::int_type);
1772
1773 /* (x & -x) generates a value that consists of only the LSB of x.
1774 * For all powers of 2, findMSB(y) == findLSB(y).
1775 */
1776 fs_reg src = retype(op[0], BRW_REGISTER_TYPE_D);
1777 fs_reg negated_src = src;
1778
1779 /* One must be negated, and the other must be non-negated. It
1780 * doesn't matter which is which.
1781 */
1782 negated_src.negate = true;
1783 src.negate = false;
1784
1785 bld.AND(temp, src, negated_src);
1786 emit_find_msb_using_lzd(bld, result, temp, false);
1787 } else {
1788 bld.FBL(result, op[0]);
1789 }
1790 break;
1791
1792 case nir_op_ubitfield_extract:
1793 case nir_op_ibitfield_extract:
1794 unreachable("should have been lowered");
1795 case nir_op_ubfe:
1796 case nir_op_ibfe:
1797 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1798 bld.BFE(result, op[2], op[1], op[0]);
1799 break;
1800 case nir_op_bfm:
1801 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1802 bld.BFI1(result, op[0], op[1]);
1803 break;
1804 case nir_op_bfi:
1805 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1806 bld.BFI2(result, op[0], op[1], op[2]);
1807 break;
1808
1809 case nir_op_bitfield_insert:
1810 unreachable("not reached: should have been lowered");
1811
1812 case nir_op_ishl:
1813 bld.SHL(result, op[0], op[1]);
1814 break;
1815 case nir_op_ishr:
1816 bld.ASR(result, op[0], op[1]);
1817 break;
1818 case nir_op_ushr:
1819 bld.SHR(result, op[0], op[1]);
1820 break;
1821
1822 case nir_op_urol:
1823 bld.ROL(result, op[0], op[1]);
1824 break;
1825 case nir_op_uror:
1826 bld.ROR(result, op[0], op[1]);
1827 break;
1828
1829 case nir_op_pack_half_2x16_split:
1830 bld.emit(FS_OPCODE_PACK_HALF_2x16_SPLIT, result, op[0], op[1]);
1831 break;
1832
1833 case nir_op_ffma:
1834 if (nir_has_any_rounding_mode_enabled(execution_mode)) {
1835 brw_rnd_mode rnd =
1836 brw_rnd_mode_from_execution_mode(execution_mode);
1837 bld.emit(SHADER_OPCODE_RND_MODE, bld.null_reg_ud(),
1838 brw_imm_d(rnd));
1839 }
1840
1841 inst = bld.MAD(result, op[2], op[1], op[0]);
1842 break;
1843
1844 case nir_op_flrp:
1845 if (nir_has_any_rounding_mode_enabled(execution_mode)) {
1846 brw_rnd_mode rnd =
1847 brw_rnd_mode_from_execution_mode(execution_mode);
1848 bld.emit(SHADER_OPCODE_RND_MODE, bld.null_reg_ud(),
1849 brw_imm_d(rnd));
1850 }
1851
1852 inst = bld.LRP(result, op[0], op[1], op[2]);
1853 break;
1854
1855 case nir_op_b32csel:
1856 if (optimize_frontfacing_ternary(instr, result))
1857 return;
1858
1859 bld.CMP(bld.null_reg_d(), op[0], brw_imm_d(0), BRW_CONDITIONAL_NZ);
1860 inst = bld.SEL(result, op[1], op[2]);
1861 inst->predicate = BRW_PREDICATE_NORMAL;
1862 break;
1863
1864 case nir_op_extract_u8:
1865 case nir_op_extract_i8: {
1866 unsigned byte = nir_src_as_uint(instr->src[1].src);
1867
1868 /* The PRMs say:
1869 *
1870 * BDW+
1871 * There is no direct conversion from B/UB to Q/UQ or Q/UQ to B/UB.
1872 * Use two instructions and a word or DWord intermediate integer type.
1873 */
1874 if (nir_dest_bit_size(instr->dest.dest) == 64) {
1875 const brw_reg_type type = brw_int_type(1, instr->op == nir_op_extract_i8);
1876
1877 if (instr->op == nir_op_extract_i8) {
1878 /* If we need to sign extend, extract to a word first */
1879 fs_reg w_temp = bld.vgrf(BRW_REGISTER_TYPE_W);
1880 bld.MOV(w_temp, subscript(op[0], type, byte));
1881 bld.MOV(result, w_temp);
1882 } else if (byte & 1) {
1883 /* Extract the high byte from the word containing the desired byte
1884 * offset.
1885 */
1886 bld.SHR(result,
1887 subscript(op[0], BRW_REGISTER_TYPE_UW, byte / 2),
1888 brw_imm_uw(8));
1889 } else {
1890 /* Otherwise use an AND with 0xff and a word type */
1891 bld.AND(result,
1892 subscript(op[0], BRW_REGISTER_TYPE_UW, byte / 2),
1893 brw_imm_uw(0xff));
1894 }
1895 } else {
1896 const brw_reg_type type = brw_int_type(1, instr->op == nir_op_extract_i8);
1897 bld.MOV(result, subscript(op[0], type, byte));
1898 }
1899 break;
1900 }
1901
1902 case nir_op_extract_u16:
1903 case nir_op_extract_i16: {
1904 const brw_reg_type type = brw_int_type(2, instr->op == nir_op_extract_i16);
1905 unsigned word = nir_src_as_uint(instr->src[1].src);
1906 bld.MOV(result, subscript(op[0], type, word));
1907 break;
1908 }
1909
1910 default:
1911 unreachable("unhandled instruction");
1912 }
1913
1914 /* If we need to do a boolean resolve, replace the result with -(x & 1)
1915 * to sign extend the low bit to 0/~0
1916 */
1917 if (devinfo->gen <= 5 &&
1918 !result.is_null() &&
1919 (instr->instr.pass_flags & BRW_NIR_BOOLEAN_MASK) == BRW_NIR_BOOLEAN_NEEDS_RESOLVE) {
1920 fs_reg masked = vgrf(glsl_type::int_type);
1921 bld.AND(masked, result, brw_imm_d(1));
1922 masked.negate = true;
1923 bld.MOV(retype(result, BRW_REGISTER_TYPE_D), masked);
1924 }
1925 }
1926
1927 void
1928 fs_visitor::nir_emit_load_const(const fs_builder &bld,
1929 nir_load_const_instr *instr)
1930 {
1931 const brw_reg_type reg_type =
1932 brw_reg_type_from_bit_size(instr->def.bit_size, BRW_REGISTER_TYPE_D);
1933 fs_reg reg = bld.vgrf(reg_type, instr->def.num_components);
1934
1935 switch (instr->def.bit_size) {
1936 case 8:
1937 for (unsigned i = 0; i < instr->def.num_components; i++)
1938 bld.MOV(offset(reg, bld, i), setup_imm_b(bld, instr->value[i].i8));
1939 break;
1940
1941 case 16:
1942 for (unsigned i = 0; i < instr->def.num_components; i++)
1943 bld.MOV(offset(reg, bld, i), brw_imm_w(instr->value[i].i16));
1944 break;
1945
1946 case 32:
1947 for (unsigned i = 0; i < instr->def.num_components; i++)
1948 bld.MOV(offset(reg, bld, i), brw_imm_d(instr->value[i].i32));
1949 break;
1950
1951 case 64:
1952 assert(devinfo->gen >= 7);
1953 if (devinfo->gen == 7) {
1954 /* We don't get 64-bit integer types until gen8 */
1955 for (unsigned i = 0; i < instr->def.num_components; i++) {
1956 bld.MOV(retype(offset(reg, bld, i), BRW_REGISTER_TYPE_DF),
1957 setup_imm_df(bld, instr->value[i].f64));
1958 }
1959 } else {
1960 for (unsigned i = 0; i < instr->def.num_components; i++)
1961 bld.MOV(offset(reg, bld, i), brw_imm_q(instr->value[i].i64));
1962 }
1963 break;
1964
1965 default:
1966 unreachable("Invalid bit size");
1967 }
1968
1969 nir_ssa_values[instr->def.index] = reg;
1970 }
1971
1972 fs_reg
1973 fs_visitor::get_nir_src(const nir_src &src)
1974 {
1975 fs_reg reg;
1976 if (src.is_ssa) {
1977 if (src.ssa->parent_instr->type == nir_instr_type_ssa_undef) {
1978 const brw_reg_type reg_type =
1979 brw_reg_type_from_bit_size(src.ssa->bit_size, BRW_REGISTER_TYPE_D);
1980 reg = bld.vgrf(reg_type, src.ssa->num_components);
1981 } else {
1982 reg = nir_ssa_values[src.ssa->index];
1983 }
1984 } else {
1985 /* We don't handle indirects on locals */
1986 assert(src.reg.indirect == NULL);
1987 reg = offset(nir_locals[src.reg.reg->index], bld,
1988 src.reg.base_offset * src.reg.reg->num_components);
1989 }
1990
1991 if (nir_src_bit_size(src) == 64 && devinfo->gen == 7) {
1992 /* The only 64-bit type available on gen7 is DF, so use that. */
1993 reg.type = BRW_REGISTER_TYPE_DF;
1994 } else {
1995 /* To avoid floating-point denorm flushing problems, set the type by
1996 * default to an integer type - instructions that need floating point
1997 * semantics will set this to F if they need to
1998 */
1999 reg.type = brw_reg_type_from_bit_size(nir_src_bit_size(src),
2000 BRW_REGISTER_TYPE_D);
2001 }
2002
2003 return reg;
2004 }
2005
2006 /**
2007 * Return an IMM for constants; otherwise call get_nir_src() as normal.
2008 *
2009 * This function should not be called on any value which may be 64 bits.
2010 * We could theoretically support 64-bit on gen8+ but we choose not to
2011 * because it wouldn't work in general (no gen7 support) and there are
2012 * enough restrictions in 64-bit immediates that you can't take the return
2013 * value and treat it the same as the result of get_nir_src().
2014 */
2015 fs_reg
2016 fs_visitor::get_nir_src_imm(const nir_src &src)
2017 {
2018 assert(nir_src_bit_size(src) == 32);
2019 return nir_src_is_const(src) ?
2020 fs_reg(brw_imm_d(nir_src_as_int(src))) : get_nir_src(src);
2021 }
2022
2023 fs_reg
2024 fs_visitor::get_nir_dest(const nir_dest &dest)
2025 {
2026 if (dest.is_ssa) {
2027 const brw_reg_type reg_type =
2028 brw_reg_type_from_bit_size(dest.ssa.bit_size,
2029 dest.ssa.bit_size == 8 ?
2030 BRW_REGISTER_TYPE_D :
2031 BRW_REGISTER_TYPE_F);
2032 nir_ssa_values[dest.ssa.index] =
2033 bld.vgrf(reg_type, dest.ssa.num_components);
2034 bld.UNDEF(nir_ssa_values[dest.ssa.index]);
2035 return nir_ssa_values[dest.ssa.index];
2036 } else {
2037 /* We don't handle indirects on locals */
2038 assert(dest.reg.indirect == NULL);
2039 return offset(nir_locals[dest.reg.reg->index], bld,
2040 dest.reg.base_offset * dest.reg.reg->num_components);
2041 }
2042 }
2043
2044 void
2045 fs_visitor::emit_percomp(const fs_builder &bld, const fs_inst &inst,
2046 unsigned wr_mask)
2047 {
2048 for (unsigned i = 0; i < 4; i++) {
2049 if (!((wr_mask >> i) & 1))
2050 continue;
2051
2052 fs_inst *new_inst = new(mem_ctx) fs_inst(inst);
2053 new_inst->dst = offset(new_inst->dst, bld, i);
2054 for (unsigned j = 0; j < new_inst->sources; j++)
2055 if (new_inst->src[j].file == VGRF)
2056 new_inst->src[j] = offset(new_inst->src[j], bld, i);
2057
2058 bld.emit(new_inst);
2059 }
2060 }
2061
2062 static fs_inst *
2063 emit_pixel_interpolater_send(const fs_builder &bld,
2064 enum opcode opcode,
2065 const fs_reg &dst,
2066 const fs_reg &src,
2067 const fs_reg &desc,
2068 glsl_interp_mode interpolation)
2069 {
2070 struct brw_wm_prog_data *wm_prog_data =
2071 brw_wm_prog_data(bld.shader->stage_prog_data);
2072
2073 fs_inst *inst = bld.emit(opcode, dst, src, desc);
2074 /* 2 floats per slot returned */
2075 inst->size_written = 2 * dst.component_size(inst->exec_size);
2076 inst->pi_noperspective = interpolation == INTERP_MODE_NOPERSPECTIVE;
2077
2078 wm_prog_data->pulls_bary = true;
2079
2080 return inst;
2081 }
2082
2083 /**
2084 * Computes 1 << x, given a D/UD register containing some value x.
2085 */
2086 static fs_reg
2087 intexp2(const fs_builder &bld, const fs_reg &x)
2088 {
2089 assert(x.type == BRW_REGISTER_TYPE_UD || x.type == BRW_REGISTER_TYPE_D);
2090
2091 fs_reg result = bld.vgrf(x.type, 1);
2092 fs_reg one = bld.vgrf(x.type, 1);
2093
2094 bld.MOV(one, retype(brw_imm_d(1), one.type));
2095 bld.SHL(result, one, x);
2096 return result;
2097 }
2098
2099 void
2100 fs_visitor::emit_gs_end_primitive(const nir_src &vertex_count_nir_src)
2101 {
2102 assert(stage == MESA_SHADER_GEOMETRY);
2103
2104 struct brw_gs_prog_data *gs_prog_data = brw_gs_prog_data(prog_data);
2105
2106 if (gs_compile->control_data_header_size_bits == 0)
2107 return;
2108
2109 /* We can only do EndPrimitive() functionality when the control data
2110 * consists of cut bits. Fortunately, the only time it isn't is when the
2111 * output type is points, in which case EndPrimitive() is a no-op.
2112 */
2113 if (gs_prog_data->control_data_format !=
2114 GEN7_GS_CONTROL_DATA_FORMAT_GSCTL_CUT) {
2115 return;
2116 }
2117
2118 /* Cut bits use one bit per vertex. */
2119 assert(gs_compile->control_data_bits_per_vertex == 1);
2120
2121 fs_reg vertex_count = get_nir_src(vertex_count_nir_src);
2122 vertex_count.type = BRW_REGISTER_TYPE_UD;
2123
2124 /* Cut bit n should be set to 1 if EndPrimitive() was called after emitting
2125 * vertex n, 0 otherwise. So all we need to do here is mark bit
2126 * (vertex_count - 1) % 32 in the cut_bits register to indicate that
2127 * EndPrimitive() was called after emitting vertex (vertex_count - 1);
2128 * vec4_gs_visitor::emit_control_data_bits() will take care of the rest.
2129 *
2130 * Note that if EndPrimitive() is called before emitting any vertices, this
2131 * will cause us to set bit 31 of the control_data_bits register to 1.
2132 * That's fine because:
2133 *
2134 * - If max_vertices < 32, then vertex number 31 (zero-based) will never be
2135 * output, so the hardware will ignore cut bit 31.
2136 *
2137 * - If max_vertices == 32, then vertex number 31 is guaranteed to be the
2138 * last vertex, so setting cut bit 31 has no effect (since the primitive
2139 * is automatically ended when the GS terminates).
2140 *
2141 * - If max_vertices > 32, then the ir_emit_vertex visitor will reset the
2142 * control_data_bits register to 0 when the first vertex is emitted.
2143 */
2144
2145 const fs_builder abld = bld.annotate("end primitive");
2146
2147 /* control_data_bits |= 1 << ((vertex_count - 1) % 32) */
2148 fs_reg prev_count = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2149 abld.ADD(prev_count, vertex_count, brw_imm_ud(0xffffffffu));
2150 fs_reg mask = intexp2(abld, prev_count);
2151 /* Note: we're relying on the fact that the GEN SHL instruction only pays
2152 * attention to the lower 5 bits of its second source argument, so on this
2153 * architecture, 1 << (vertex_count - 1) is equivalent to 1 <<
2154 * ((vertex_count - 1) % 32).
2155 */
2156 abld.OR(this->control_data_bits, this->control_data_bits, mask);
2157 }
2158
2159 void
2160 fs_visitor::emit_gs_control_data_bits(const fs_reg &vertex_count)
2161 {
2162 assert(stage == MESA_SHADER_GEOMETRY);
2163 assert(gs_compile->control_data_bits_per_vertex != 0);
2164
2165 struct brw_gs_prog_data *gs_prog_data = brw_gs_prog_data(prog_data);
2166
2167 const fs_builder abld = bld.annotate("emit control data bits");
2168 const fs_builder fwa_bld = bld.exec_all();
2169
2170 /* We use a single UD register to accumulate control data bits (32 bits
2171 * for each of the SIMD8 channels). So we need to write a DWord (32 bits)
2172 * at a time.
2173 *
2174 * Unfortunately, the URB_WRITE_SIMD8 message uses 128-bit (OWord) offsets.
2175 * We have select a 128-bit group via the Global and Per-Slot Offsets, then
2176 * use the Channel Mask phase to enable/disable which DWord within that
2177 * group to write. (Remember, different SIMD8 channels may have emitted
2178 * different numbers of vertices, so we may need per-slot offsets.)
2179 *
2180 * Channel masking presents an annoying problem: we may have to replicate
2181 * the data up to 4 times:
2182 *
2183 * Msg = Handles, Per-Slot Offsets, Channel Masks, Data, Data, Data, Data.
2184 *
2185 * To avoid penalizing shaders that emit a small number of vertices, we
2186 * can avoid these sometimes: if the size of the control data header is
2187 * <= 128 bits, then there is only 1 OWord. All SIMD8 channels will land
2188 * land in the same 128-bit group, so we can skip per-slot offsets.
2189 *
2190 * Similarly, if the control data header is <= 32 bits, there is only one
2191 * DWord, so we can skip channel masks.
2192 */
2193 enum opcode opcode = SHADER_OPCODE_URB_WRITE_SIMD8;
2194
2195 fs_reg channel_mask, per_slot_offset;
2196
2197 if (gs_compile->control_data_header_size_bits > 32) {
2198 opcode = SHADER_OPCODE_URB_WRITE_SIMD8_MASKED;
2199 channel_mask = vgrf(glsl_type::uint_type);
2200 }
2201
2202 if (gs_compile->control_data_header_size_bits > 128) {
2203 opcode = SHADER_OPCODE_URB_WRITE_SIMD8_MASKED_PER_SLOT;
2204 per_slot_offset = vgrf(glsl_type::uint_type);
2205 }
2206
2207 /* Figure out which DWord we're trying to write to using the formula:
2208 *
2209 * dword_index = (vertex_count - 1) * bits_per_vertex / 32
2210 *
2211 * Since bits_per_vertex is a power of two, and is known at compile
2212 * time, this can be optimized to:
2213 *
2214 * dword_index = (vertex_count - 1) >> (6 - log2(bits_per_vertex))
2215 */
2216 if (opcode != SHADER_OPCODE_URB_WRITE_SIMD8) {
2217 fs_reg dword_index = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2218 fs_reg prev_count = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2219 abld.ADD(prev_count, vertex_count, brw_imm_ud(0xffffffffu));
2220 unsigned log2_bits_per_vertex =
2221 util_last_bit(gs_compile->control_data_bits_per_vertex);
2222 abld.SHR(dword_index, prev_count, brw_imm_ud(6u - log2_bits_per_vertex));
2223
2224 if (per_slot_offset.file != BAD_FILE) {
2225 /* Set the per-slot offset to dword_index / 4, so that we'll write to
2226 * the appropriate OWord within the control data header.
2227 */
2228 abld.SHR(per_slot_offset, dword_index, brw_imm_ud(2u));
2229 }
2230
2231 /* Set the channel masks to 1 << (dword_index % 4), so that we'll
2232 * write to the appropriate DWORD within the OWORD.
2233 */
2234 fs_reg channel = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2235 fwa_bld.AND(channel, dword_index, brw_imm_ud(3u));
2236 channel_mask = intexp2(fwa_bld, channel);
2237 /* Then the channel masks need to be in bits 23:16. */
2238 fwa_bld.SHL(channel_mask, channel_mask, brw_imm_ud(16u));
2239 }
2240
2241 /* Store the control data bits in the message payload and send it. */
2242 unsigned mlen = 2;
2243 if (channel_mask.file != BAD_FILE)
2244 mlen += 4; /* channel masks, plus 3 extra copies of the data */
2245 if (per_slot_offset.file != BAD_FILE)
2246 mlen++;
2247
2248 fs_reg payload = bld.vgrf(BRW_REGISTER_TYPE_UD, mlen);
2249 fs_reg *sources = ralloc_array(mem_ctx, fs_reg, mlen);
2250 unsigned i = 0;
2251 sources[i++] = fs_reg(retype(brw_vec8_grf(1, 0), BRW_REGISTER_TYPE_UD));
2252 if (per_slot_offset.file != BAD_FILE)
2253 sources[i++] = per_slot_offset;
2254 if (channel_mask.file != BAD_FILE)
2255 sources[i++] = channel_mask;
2256 while (i < mlen) {
2257 sources[i++] = this->control_data_bits;
2258 }
2259
2260 abld.LOAD_PAYLOAD(payload, sources, mlen, mlen);
2261 fs_inst *inst = abld.emit(opcode, reg_undef, payload);
2262 inst->mlen = mlen;
2263 /* We need to increment Global Offset by 256-bits to make room for
2264 * Broadwell's extra "Vertex Count" payload at the beginning of the
2265 * URB entry. Since this is an OWord message, Global Offset is counted
2266 * in 128-bit units, so we must set it to 2.
2267 */
2268 if (gs_prog_data->static_vertex_count == -1)
2269 inst->offset = 2;
2270 }
2271
2272 void
2273 fs_visitor::set_gs_stream_control_data_bits(const fs_reg &vertex_count,
2274 unsigned stream_id)
2275 {
2276 /* control_data_bits |= stream_id << ((2 * (vertex_count - 1)) % 32) */
2277
2278 /* Note: we are calling this *before* increasing vertex_count, so
2279 * this->vertex_count == vertex_count - 1 in the formula above.
2280 */
2281
2282 /* Stream mode uses 2 bits per vertex */
2283 assert(gs_compile->control_data_bits_per_vertex == 2);
2284
2285 /* Must be a valid stream */
2286 assert(stream_id < MAX_VERTEX_STREAMS);
2287
2288 /* Control data bits are initialized to 0 so we don't have to set any
2289 * bits when sending vertices to stream 0.
2290 */
2291 if (stream_id == 0)
2292 return;
2293
2294 const fs_builder abld = bld.annotate("set stream control data bits", NULL);
2295
2296 /* reg::sid = stream_id */
2297 fs_reg sid = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2298 abld.MOV(sid, brw_imm_ud(stream_id));
2299
2300 /* reg:shift_count = 2 * (vertex_count - 1) */
2301 fs_reg shift_count = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2302 abld.SHL(shift_count, vertex_count, brw_imm_ud(1u));
2303
2304 /* Note: we're relying on the fact that the GEN SHL instruction only pays
2305 * attention to the lower 5 bits of its second source argument, so on this
2306 * architecture, stream_id << 2 * (vertex_count - 1) is equivalent to
2307 * stream_id << ((2 * (vertex_count - 1)) % 32).
2308 */
2309 fs_reg mask = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2310 abld.SHL(mask, sid, shift_count);
2311 abld.OR(this->control_data_bits, this->control_data_bits, mask);
2312 }
2313
2314 void
2315 fs_visitor::emit_gs_vertex(const nir_src &vertex_count_nir_src,
2316 unsigned stream_id)
2317 {
2318 assert(stage == MESA_SHADER_GEOMETRY);
2319
2320 struct brw_gs_prog_data *gs_prog_data = brw_gs_prog_data(prog_data);
2321
2322 fs_reg vertex_count = get_nir_src(vertex_count_nir_src);
2323 vertex_count.type = BRW_REGISTER_TYPE_UD;
2324
2325 /* Haswell and later hardware ignores the "Render Stream Select" bits
2326 * from the 3DSTATE_STREAMOUT packet when the SOL stage is disabled,
2327 * and instead sends all primitives down the pipeline for rasterization.
2328 * If the SOL stage is enabled, "Render Stream Select" is honored and
2329 * primitives bound to non-zero streams are discarded after stream output.
2330 *
2331 * Since the only purpose of primives sent to non-zero streams is to
2332 * be recorded by transform feedback, we can simply discard all geometry
2333 * bound to these streams when transform feedback is disabled.
2334 */
2335 if (stream_id > 0 && !nir->info.has_transform_feedback_varyings)
2336 return;
2337
2338 /* If we're outputting 32 control data bits or less, then we can wait
2339 * until the shader is over to output them all. Otherwise we need to
2340 * output them as we go. Now is the time to do it, since we're about to
2341 * output the vertex_count'th vertex, so it's guaranteed that the
2342 * control data bits associated with the (vertex_count - 1)th vertex are
2343 * correct.
2344 */
2345 if (gs_compile->control_data_header_size_bits > 32) {
2346 const fs_builder abld =
2347 bld.annotate("emit vertex: emit control data bits");
2348
2349 /* Only emit control data bits if we've finished accumulating a batch
2350 * of 32 bits. This is the case when:
2351 *
2352 * (vertex_count * bits_per_vertex) % 32 == 0
2353 *
2354 * (in other words, when the last 5 bits of vertex_count *
2355 * bits_per_vertex are 0). Assuming bits_per_vertex == 2^n for some
2356 * integer n (which is always the case, since bits_per_vertex is
2357 * always 1 or 2), this is equivalent to requiring that the last 5-n
2358 * bits of vertex_count are 0:
2359 *
2360 * vertex_count & (2^(5-n) - 1) == 0
2361 *
2362 * 2^(5-n) == 2^5 / 2^n == 32 / bits_per_vertex, so this is
2363 * equivalent to:
2364 *
2365 * vertex_count & (32 / bits_per_vertex - 1) == 0
2366 *
2367 * TODO: If vertex_count is an immediate, we could do some of this math
2368 * at compile time...
2369 */
2370 fs_inst *inst =
2371 abld.AND(bld.null_reg_d(), vertex_count,
2372 brw_imm_ud(32u / gs_compile->control_data_bits_per_vertex - 1u));
2373 inst->conditional_mod = BRW_CONDITIONAL_Z;
2374
2375 abld.IF(BRW_PREDICATE_NORMAL);
2376 /* If vertex_count is 0, then no control data bits have been
2377 * accumulated yet, so we can skip emitting them.
2378 */
2379 abld.CMP(bld.null_reg_d(), vertex_count, brw_imm_ud(0u),
2380 BRW_CONDITIONAL_NEQ);
2381 abld.IF(BRW_PREDICATE_NORMAL);
2382 emit_gs_control_data_bits(vertex_count);
2383 abld.emit(BRW_OPCODE_ENDIF);
2384
2385 /* Reset control_data_bits to 0 so we can start accumulating a new
2386 * batch.
2387 *
2388 * Note: in the case where vertex_count == 0, this neutralizes the
2389 * effect of any call to EndPrimitive() that the shader may have
2390 * made before outputting its first vertex.
2391 */
2392 inst = abld.MOV(this->control_data_bits, brw_imm_ud(0u));
2393 inst->force_writemask_all = true;
2394 abld.emit(BRW_OPCODE_ENDIF);
2395 }
2396
2397 emit_urb_writes(vertex_count);
2398
2399 /* In stream mode we have to set control data bits for all vertices
2400 * unless we have disabled control data bits completely (which we do
2401 * do for GL_POINTS outputs that don't use streams).
2402 */
2403 if (gs_compile->control_data_header_size_bits > 0 &&
2404 gs_prog_data->control_data_format ==
2405 GEN7_GS_CONTROL_DATA_FORMAT_GSCTL_SID) {
2406 set_gs_stream_control_data_bits(vertex_count, stream_id);
2407 }
2408 }
2409
2410 void
2411 fs_visitor::emit_gs_input_load(const fs_reg &dst,
2412 const nir_src &vertex_src,
2413 unsigned base_offset,
2414 const nir_src &offset_src,
2415 unsigned num_components,
2416 unsigned first_component)
2417 {
2418 assert(type_sz(dst.type) == 4);
2419 struct brw_gs_prog_data *gs_prog_data = brw_gs_prog_data(prog_data);
2420 const unsigned push_reg_count = gs_prog_data->base.urb_read_length * 8;
2421
2422 /* TODO: figure out push input layout for invocations == 1 */
2423 if (gs_prog_data->invocations == 1 &&
2424 nir_src_is_const(offset_src) && nir_src_is_const(vertex_src) &&
2425 4 * (base_offset + nir_src_as_uint(offset_src)) < push_reg_count) {
2426 int imm_offset = (base_offset + nir_src_as_uint(offset_src)) * 4 +
2427 nir_src_as_uint(vertex_src) * push_reg_count;
2428 for (unsigned i = 0; i < num_components; i++) {
2429 bld.MOV(offset(dst, bld, i),
2430 fs_reg(ATTR, imm_offset + i + first_component, dst.type));
2431 }
2432 return;
2433 }
2434
2435 /* Resort to the pull model. Ensure the VUE handles are provided. */
2436 assert(gs_prog_data->base.include_vue_handles);
2437
2438 unsigned first_icp_handle = gs_prog_data->include_primitive_id ? 3 : 2;
2439 fs_reg icp_handle = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2440
2441 if (gs_prog_data->invocations == 1) {
2442 if (nir_src_is_const(vertex_src)) {
2443 /* The vertex index is constant; just select the proper URB handle. */
2444 icp_handle =
2445 retype(brw_vec8_grf(first_icp_handle + nir_src_as_uint(vertex_src), 0),
2446 BRW_REGISTER_TYPE_UD);
2447 } else {
2448 /* The vertex index is non-constant. We need to use indirect
2449 * addressing to fetch the proper URB handle.
2450 *
2451 * First, we start with the sequence <7, 6, 5, 4, 3, 2, 1, 0>
2452 * indicating that channel <n> should read the handle from
2453 * DWord <n>. We convert that to bytes by multiplying by 4.
2454 *
2455 * Next, we convert the vertex index to bytes by multiplying
2456 * by 32 (shifting by 5), and add the two together. This is
2457 * the final indirect byte offset.
2458 */
2459 fs_reg sequence = bld.vgrf(BRW_REGISTER_TYPE_UW, 1);
2460 fs_reg channel_offsets = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2461 fs_reg vertex_offset_bytes = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2462 fs_reg icp_offset_bytes = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2463
2464 /* sequence = <7, 6, 5, 4, 3, 2, 1, 0> */
2465 bld.MOV(sequence, fs_reg(brw_imm_v(0x76543210)));
2466 /* channel_offsets = 4 * sequence = <28, 24, 20, 16, 12, 8, 4, 0> */
2467 bld.SHL(channel_offsets, sequence, brw_imm_ud(2u));
2468 /* Convert vertex_index to bytes (multiply by 32) */
2469 bld.SHL(vertex_offset_bytes,
2470 retype(get_nir_src(vertex_src), BRW_REGISTER_TYPE_UD),
2471 brw_imm_ud(5u));
2472 bld.ADD(icp_offset_bytes, vertex_offset_bytes, channel_offsets);
2473
2474 /* Use first_icp_handle as the base offset. There is one register
2475 * of URB handles per vertex, so inform the register allocator that
2476 * we might read up to nir->info.gs.vertices_in registers.
2477 */
2478 bld.emit(SHADER_OPCODE_MOV_INDIRECT, icp_handle,
2479 retype(brw_vec8_grf(first_icp_handle, 0), icp_handle.type),
2480 fs_reg(icp_offset_bytes),
2481 brw_imm_ud(nir->info.gs.vertices_in * REG_SIZE));
2482 }
2483 } else {
2484 assert(gs_prog_data->invocations > 1);
2485
2486 if (nir_src_is_const(vertex_src)) {
2487 unsigned vertex = nir_src_as_uint(vertex_src);
2488 assert(devinfo->gen >= 9 || vertex <= 5);
2489 bld.MOV(icp_handle,
2490 retype(brw_vec1_grf(first_icp_handle + vertex / 8, vertex % 8),
2491 BRW_REGISTER_TYPE_UD));
2492 } else {
2493 /* The vertex index is non-constant. We need to use indirect
2494 * addressing to fetch the proper URB handle.
2495 *
2496 */
2497 fs_reg icp_offset_bytes = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2498
2499 /* Convert vertex_index to bytes (multiply by 4) */
2500 bld.SHL(icp_offset_bytes,
2501 retype(get_nir_src(vertex_src), BRW_REGISTER_TYPE_UD),
2502 brw_imm_ud(2u));
2503
2504 /* Use first_icp_handle as the base offset. There is one DWord
2505 * of URB handles per vertex, so inform the register allocator that
2506 * we might read up to ceil(nir->info.gs.vertices_in / 8) registers.
2507 */
2508 bld.emit(SHADER_OPCODE_MOV_INDIRECT, icp_handle,
2509 retype(brw_vec8_grf(first_icp_handle, 0), icp_handle.type),
2510 fs_reg(icp_offset_bytes),
2511 brw_imm_ud(DIV_ROUND_UP(nir->info.gs.vertices_in, 8) *
2512 REG_SIZE));
2513 }
2514 }
2515
2516 fs_inst *inst;
2517 fs_reg indirect_offset = get_nir_src(offset_src);
2518
2519 if (nir_src_is_const(offset_src)) {
2520 /* Constant indexing - use global offset. */
2521 if (first_component != 0) {
2522 unsigned read_components = num_components + first_component;
2523 fs_reg tmp = bld.vgrf(dst.type, read_components);
2524 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8, tmp, icp_handle);
2525 inst->size_written = read_components *
2526 tmp.component_size(inst->exec_size);
2527 for (unsigned i = 0; i < num_components; i++) {
2528 bld.MOV(offset(dst, bld, i),
2529 offset(tmp, bld, i + first_component));
2530 }
2531 } else {
2532 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8, dst, icp_handle);
2533 inst->size_written = num_components *
2534 dst.component_size(inst->exec_size);
2535 }
2536 inst->offset = base_offset + nir_src_as_uint(offset_src);
2537 inst->mlen = 1;
2538 } else {
2539 /* Indirect indexing - use per-slot offsets as well. */
2540 const fs_reg srcs[] = { icp_handle, indirect_offset };
2541 unsigned read_components = num_components + first_component;
2542 fs_reg tmp = bld.vgrf(dst.type, read_components);
2543 fs_reg payload = bld.vgrf(BRW_REGISTER_TYPE_UD, 2);
2544 bld.LOAD_PAYLOAD(payload, srcs, ARRAY_SIZE(srcs), 0);
2545 if (first_component != 0) {
2546 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT, tmp,
2547 payload);
2548 inst->size_written = read_components *
2549 tmp.component_size(inst->exec_size);
2550 for (unsigned i = 0; i < num_components; i++) {
2551 bld.MOV(offset(dst, bld, i),
2552 offset(tmp, bld, i + first_component));
2553 }
2554 } else {
2555 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT, dst, payload);
2556 inst->size_written = num_components *
2557 dst.component_size(inst->exec_size);
2558 }
2559 inst->offset = base_offset;
2560 inst->mlen = 2;
2561 }
2562 }
2563
2564 fs_reg
2565 fs_visitor::get_indirect_offset(nir_intrinsic_instr *instr)
2566 {
2567 nir_src *offset_src = nir_get_io_offset_src(instr);
2568
2569 if (nir_src_is_const(*offset_src)) {
2570 /* The only constant offset we should find is 0. brw_nir.c's
2571 * add_const_offset_to_base() will fold other constant offsets
2572 * into instr->const_index[0].
2573 */
2574 assert(nir_src_as_uint(*offset_src) == 0);
2575 return fs_reg();
2576 }
2577
2578 return get_nir_src(*offset_src);
2579 }
2580
2581 void
2582 fs_visitor::nir_emit_vs_intrinsic(const fs_builder &bld,
2583 nir_intrinsic_instr *instr)
2584 {
2585 assert(stage == MESA_SHADER_VERTEX);
2586
2587 fs_reg dest;
2588 if (nir_intrinsic_infos[instr->intrinsic].has_dest)
2589 dest = get_nir_dest(instr->dest);
2590
2591 switch (instr->intrinsic) {
2592 case nir_intrinsic_load_vertex_id:
2593 case nir_intrinsic_load_base_vertex:
2594 unreachable("should be lowered by nir_lower_system_values()");
2595
2596 case nir_intrinsic_load_input: {
2597 assert(nir_dest_bit_size(instr->dest) == 32);
2598 fs_reg src = fs_reg(ATTR, nir_intrinsic_base(instr) * 4, dest.type);
2599 src = offset(src, bld, nir_intrinsic_component(instr));
2600 src = offset(src, bld, nir_src_as_uint(instr->src[0]));
2601
2602 for (unsigned i = 0; i < instr->num_components; i++)
2603 bld.MOV(offset(dest, bld, i), offset(src, bld, i));
2604 break;
2605 }
2606
2607 case nir_intrinsic_load_vertex_id_zero_base:
2608 case nir_intrinsic_load_instance_id:
2609 case nir_intrinsic_load_base_instance:
2610 case nir_intrinsic_load_draw_id:
2611 case nir_intrinsic_load_first_vertex:
2612 case nir_intrinsic_load_is_indexed_draw:
2613 unreachable("lowered by brw_nir_lower_vs_inputs");
2614
2615 default:
2616 nir_emit_intrinsic(bld, instr);
2617 break;
2618 }
2619 }
2620
2621 fs_reg
2622 fs_visitor::get_tcs_single_patch_icp_handle(const fs_builder &bld,
2623 nir_intrinsic_instr *instr)
2624 {
2625 struct brw_tcs_prog_data *tcs_prog_data = brw_tcs_prog_data(prog_data);
2626 const nir_src &vertex_src = instr->src[0];
2627 nir_intrinsic_instr *vertex_intrin = nir_src_as_intrinsic(vertex_src);
2628 fs_reg icp_handle;
2629
2630 if (nir_src_is_const(vertex_src)) {
2631 /* Emit a MOV to resolve <0,1,0> regioning. */
2632 icp_handle = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2633 unsigned vertex = nir_src_as_uint(vertex_src);
2634 bld.MOV(icp_handle,
2635 retype(brw_vec1_grf(1 + (vertex >> 3), vertex & 7),
2636 BRW_REGISTER_TYPE_UD));
2637 } else if (tcs_prog_data->instances == 1 && vertex_intrin &&
2638 vertex_intrin->intrinsic == nir_intrinsic_load_invocation_id) {
2639 /* For the common case of only 1 instance, an array index of
2640 * gl_InvocationID means reading g1. Skip all the indirect work.
2641 */
2642 icp_handle = retype(brw_vec8_grf(1, 0), BRW_REGISTER_TYPE_UD);
2643 } else {
2644 /* The vertex index is non-constant. We need to use indirect
2645 * addressing to fetch the proper URB handle.
2646 */
2647 icp_handle = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2648
2649 /* Each ICP handle is a single DWord (4 bytes) */
2650 fs_reg vertex_offset_bytes = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2651 bld.SHL(vertex_offset_bytes,
2652 retype(get_nir_src(vertex_src), BRW_REGISTER_TYPE_UD),
2653 brw_imm_ud(2u));
2654
2655 /* Start at g1. We might read up to 4 registers. */
2656 bld.emit(SHADER_OPCODE_MOV_INDIRECT, icp_handle,
2657 retype(brw_vec8_grf(1, 0), icp_handle.type), vertex_offset_bytes,
2658 brw_imm_ud(4 * REG_SIZE));
2659 }
2660
2661 return icp_handle;
2662 }
2663
2664 fs_reg
2665 fs_visitor::get_tcs_eight_patch_icp_handle(const fs_builder &bld,
2666 nir_intrinsic_instr *instr)
2667 {
2668 struct brw_tcs_prog_key *tcs_key = (struct brw_tcs_prog_key *) key;
2669 struct brw_tcs_prog_data *tcs_prog_data = brw_tcs_prog_data(prog_data);
2670 const nir_src &vertex_src = instr->src[0];
2671
2672 unsigned first_icp_handle = tcs_prog_data->include_primitive_id ? 3 : 2;
2673
2674 if (nir_src_is_const(vertex_src)) {
2675 return fs_reg(retype(brw_vec8_grf(first_icp_handle +
2676 nir_src_as_uint(vertex_src), 0),
2677 BRW_REGISTER_TYPE_UD));
2678 }
2679
2680 /* The vertex index is non-constant. We need to use indirect
2681 * addressing to fetch the proper URB handle.
2682 *
2683 * First, we start with the sequence <7, 6, 5, 4, 3, 2, 1, 0>
2684 * indicating that channel <n> should read the handle from
2685 * DWord <n>. We convert that to bytes by multiplying by 4.
2686 *
2687 * Next, we convert the vertex index to bytes by multiplying
2688 * by 32 (shifting by 5), and add the two together. This is
2689 * the final indirect byte offset.
2690 */
2691 fs_reg icp_handle = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2692 fs_reg sequence = bld.vgrf(BRW_REGISTER_TYPE_UW, 1);
2693 fs_reg channel_offsets = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2694 fs_reg vertex_offset_bytes = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2695 fs_reg icp_offset_bytes = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2696
2697 /* sequence = <7, 6, 5, 4, 3, 2, 1, 0> */
2698 bld.MOV(sequence, fs_reg(brw_imm_v(0x76543210)));
2699 /* channel_offsets = 4 * sequence = <28, 24, 20, 16, 12, 8, 4, 0> */
2700 bld.SHL(channel_offsets, sequence, brw_imm_ud(2u));
2701 /* Convert vertex_index to bytes (multiply by 32) */
2702 bld.SHL(vertex_offset_bytes,
2703 retype(get_nir_src(vertex_src), BRW_REGISTER_TYPE_UD),
2704 brw_imm_ud(5u));
2705 bld.ADD(icp_offset_bytes, vertex_offset_bytes, channel_offsets);
2706
2707 /* Use first_icp_handle as the base offset. There is one register
2708 * of URB handles per vertex, so inform the register allocator that
2709 * we might read up to nir->info.gs.vertices_in registers.
2710 */
2711 bld.emit(SHADER_OPCODE_MOV_INDIRECT, icp_handle,
2712 retype(brw_vec8_grf(first_icp_handle, 0), icp_handle.type),
2713 icp_offset_bytes, brw_imm_ud(tcs_key->input_vertices * REG_SIZE));
2714
2715 return icp_handle;
2716 }
2717
2718 struct brw_reg
2719 fs_visitor::get_tcs_output_urb_handle()
2720 {
2721 struct brw_vue_prog_data *vue_prog_data = brw_vue_prog_data(prog_data);
2722
2723 if (vue_prog_data->dispatch_mode == DISPATCH_MODE_TCS_SINGLE_PATCH) {
2724 return retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_UD);
2725 } else {
2726 assert(vue_prog_data->dispatch_mode == DISPATCH_MODE_TCS_8_PATCH);
2727 return retype(brw_vec8_grf(1, 0), BRW_REGISTER_TYPE_UD);
2728 }
2729 }
2730
2731 void
2732 fs_visitor::nir_emit_tcs_intrinsic(const fs_builder &bld,
2733 nir_intrinsic_instr *instr)
2734 {
2735 assert(stage == MESA_SHADER_TESS_CTRL);
2736 struct brw_tcs_prog_key *tcs_key = (struct brw_tcs_prog_key *) key;
2737 struct brw_tcs_prog_data *tcs_prog_data = brw_tcs_prog_data(prog_data);
2738 struct brw_vue_prog_data *vue_prog_data = &tcs_prog_data->base;
2739
2740 bool eight_patch =
2741 vue_prog_data->dispatch_mode == DISPATCH_MODE_TCS_8_PATCH;
2742
2743 fs_reg dst;
2744 if (nir_intrinsic_infos[instr->intrinsic].has_dest)
2745 dst = get_nir_dest(instr->dest);
2746
2747 switch (instr->intrinsic) {
2748 case nir_intrinsic_load_primitive_id:
2749 bld.MOV(dst, fs_reg(eight_patch ? brw_vec8_grf(2, 0)
2750 : brw_vec1_grf(0, 1)));
2751 break;
2752 case nir_intrinsic_load_invocation_id:
2753 bld.MOV(retype(dst, invocation_id.type), invocation_id);
2754 break;
2755 case nir_intrinsic_load_patch_vertices_in:
2756 bld.MOV(retype(dst, BRW_REGISTER_TYPE_D),
2757 brw_imm_d(tcs_key->input_vertices));
2758 break;
2759
2760 case nir_intrinsic_control_barrier: {
2761 if (tcs_prog_data->instances == 1)
2762 break;
2763
2764 fs_reg m0 = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2765 fs_reg m0_2 = component(m0, 2);
2766
2767 const fs_builder chanbld = bld.exec_all().group(1, 0);
2768
2769 /* Zero the message header */
2770 bld.exec_all().MOV(m0, brw_imm_ud(0u));
2771
2772 if (devinfo->gen < 11) {
2773 /* Copy "Barrier ID" from r0.2, bits 16:13 */
2774 chanbld.AND(m0_2, retype(brw_vec1_grf(0, 2), BRW_REGISTER_TYPE_UD),
2775 brw_imm_ud(INTEL_MASK(16, 13)));
2776
2777 /* Shift it up to bits 27:24. */
2778 chanbld.SHL(m0_2, m0_2, brw_imm_ud(11));
2779 } else {
2780 chanbld.AND(m0_2, retype(brw_vec1_grf(0, 2), BRW_REGISTER_TYPE_UD),
2781 brw_imm_ud(INTEL_MASK(30, 24)));
2782 }
2783
2784 /* Set the Barrier Count and the enable bit */
2785 if (devinfo->gen < 11) {
2786 chanbld.OR(m0_2, m0_2,
2787 brw_imm_ud(tcs_prog_data->instances << 9 | (1 << 15)));
2788 } else {
2789 chanbld.OR(m0_2, m0_2,
2790 brw_imm_ud(tcs_prog_data->instances << 8 | (1 << 15)));
2791 }
2792
2793 bld.emit(SHADER_OPCODE_BARRIER, bld.null_reg_ud(), m0);
2794 break;
2795 }
2796
2797 case nir_intrinsic_load_input:
2798 unreachable("nir_lower_io should never give us these.");
2799 break;
2800
2801 case nir_intrinsic_load_per_vertex_input: {
2802 assert(nir_dest_bit_size(instr->dest) == 32);
2803 fs_reg indirect_offset = get_indirect_offset(instr);
2804 unsigned imm_offset = instr->const_index[0];
2805 fs_inst *inst;
2806
2807 fs_reg icp_handle =
2808 eight_patch ? get_tcs_eight_patch_icp_handle(bld, instr)
2809 : get_tcs_single_patch_icp_handle(bld, instr);
2810
2811 /* We can only read two double components with each URB read, so
2812 * we send two read messages in that case, each one loading up to
2813 * two double components.
2814 */
2815 unsigned num_components = instr->num_components;
2816 unsigned first_component = nir_intrinsic_component(instr);
2817
2818 if (indirect_offset.file == BAD_FILE) {
2819 /* Constant indexing - use global offset. */
2820 if (first_component != 0) {
2821 unsigned read_components = num_components + first_component;
2822 fs_reg tmp = bld.vgrf(dst.type, read_components);
2823 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8, tmp, icp_handle);
2824 for (unsigned i = 0; i < num_components; i++) {
2825 bld.MOV(offset(dst, bld, i),
2826 offset(tmp, bld, i + first_component));
2827 }
2828 } else {
2829 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8, dst, icp_handle);
2830 }
2831 inst->offset = imm_offset;
2832 inst->mlen = 1;
2833 } else {
2834 /* Indirect indexing - use per-slot offsets as well. */
2835 const fs_reg srcs[] = { icp_handle, indirect_offset };
2836 fs_reg payload = bld.vgrf(BRW_REGISTER_TYPE_UD, 2);
2837 bld.LOAD_PAYLOAD(payload, srcs, ARRAY_SIZE(srcs), 0);
2838 if (first_component != 0) {
2839 unsigned read_components = num_components + first_component;
2840 fs_reg tmp = bld.vgrf(dst.type, read_components);
2841 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT, tmp,
2842 payload);
2843 for (unsigned i = 0; i < num_components; i++) {
2844 bld.MOV(offset(dst, bld, i),
2845 offset(tmp, bld, i + first_component));
2846 }
2847 } else {
2848 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT, dst,
2849 payload);
2850 }
2851 inst->offset = imm_offset;
2852 inst->mlen = 2;
2853 }
2854 inst->size_written = (num_components + first_component) *
2855 inst->dst.component_size(inst->exec_size);
2856
2857 /* Copy the temporary to the destination to deal with writemasking.
2858 *
2859 * Also attempt to deal with gl_PointSize being in the .w component.
2860 */
2861 if (inst->offset == 0 && indirect_offset.file == BAD_FILE) {
2862 assert(type_sz(dst.type) == 4);
2863 inst->dst = bld.vgrf(dst.type, 4);
2864 inst->size_written = 4 * REG_SIZE;
2865 bld.MOV(dst, offset(inst->dst, bld, 3));
2866 }
2867 break;
2868 }
2869
2870 case nir_intrinsic_load_output:
2871 case nir_intrinsic_load_per_vertex_output: {
2872 assert(nir_dest_bit_size(instr->dest) == 32);
2873 fs_reg indirect_offset = get_indirect_offset(instr);
2874 unsigned imm_offset = instr->const_index[0];
2875 unsigned first_component = nir_intrinsic_component(instr);
2876
2877 struct brw_reg output_handles = get_tcs_output_urb_handle();
2878
2879 fs_inst *inst;
2880 if (indirect_offset.file == BAD_FILE) {
2881 /* This MOV replicates the output handle to all enabled channels
2882 * is SINGLE_PATCH mode.
2883 */
2884 fs_reg patch_handle = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2885 bld.MOV(patch_handle, output_handles);
2886
2887 {
2888 if (first_component != 0) {
2889 unsigned read_components =
2890 instr->num_components + first_component;
2891 fs_reg tmp = bld.vgrf(dst.type, read_components);
2892 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8, tmp,
2893 patch_handle);
2894 inst->size_written = read_components * REG_SIZE;
2895 for (unsigned i = 0; i < instr->num_components; i++) {
2896 bld.MOV(offset(dst, bld, i),
2897 offset(tmp, bld, i + first_component));
2898 }
2899 } else {
2900 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8, dst,
2901 patch_handle);
2902 inst->size_written = instr->num_components * REG_SIZE;
2903 }
2904 inst->offset = imm_offset;
2905 inst->mlen = 1;
2906 }
2907 } else {
2908 /* Indirect indexing - use per-slot offsets as well. */
2909 const fs_reg srcs[] = { output_handles, indirect_offset };
2910 fs_reg payload = bld.vgrf(BRW_REGISTER_TYPE_UD, 2);
2911 bld.LOAD_PAYLOAD(payload, srcs, ARRAY_SIZE(srcs), 0);
2912 if (first_component != 0) {
2913 unsigned read_components =
2914 instr->num_components + first_component;
2915 fs_reg tmp = bld.vgrf(dst.type, read_components);
2916 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT, tmp,
2917 payload);
2918 inst->size_written = read_components * REG_SIZE;
2919 for (unsigned i = 0; i < instr->num_components; i++) {
2920 bld.MOV(offset(dst, bld, i),
2921 offset(tmp, bld, i + first_component));
2922 }
2923 } else {
2924 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT, dst,
2925 payload);
2926 inst->size_written = instr->num_components * REG_SIZE;
2927 }
2928 inst->offset = imm_offset;
2929 inst->mlen = 2;
2930 }
2931 break;
2932 }
2933
2934 case nir_intrinsic_store_output:
2935 case nir_intrinsic_store_per_vertex_output: {
2936 assert(nir_src_bit_size(instr->src[0]) == 32);
2937 fs_reg value = get_nir_src(instr->src[0]);
2938 fs_reg indirect_offset = get_indirect_offset(instr);
2939 unsigned imm_offset = instr->const_index[0];
2940 unsigned mask = instr->const_index[1];
2941 unsigned header_regs = 0;
2942 struct brw_reg output_handles = get_tcs_output_urb_handle();
2943
2944 fs_reg srcs[7];
2945 srcs[header_regs++] = output_handles;
2946
2947 if (indirect_offset.file != BAD_FILE) {
2948 srcs[header_regs++] = indirect_offset;
2949 }
2950
2951 if (mask == 0)
2952 break;
2953
2954 unsigned num_components = util_last_bit(mask);
2955 enum opcode opcode;
2956
2957 /* We can only pack two 64-bit components in a single message, so send
2958 * 2 messages if we have more components
2959 */
2960 unsigned first_component = nir_intrinsic_component(instr);
2961 mask = mask << first_component;
2962
2963 if (mask != WRITEMASK_XYZW) {
2964 srcs[header_regs++] = brw_imm_ud(mask << 16);
2965 opcode = indirect_offset.file != BAD_FILE ?
2966 SHADER_OPCODE_URB_WRITE_SIMD8_MASKED_PER_SLOT :
2967 SHADER_OPCODE_URB_WRITE_SIMD8_MASKED;
2968 } else {
2969 opcode = indirect_offset.file != BAD_FILE ?
2970 SHADER_OPCODE_URB_WRITE_SIMD8_PER_SLOT :
2971 SHADER_OPCODE_URB_WRITE_SIMD8;
2972 }
2973
2974 for (unsigned i = 0; i < num_components; i++) {
2975 if (!(mask & (1 << (i + first_component))))
2976 continue;
2977
2978 srcs[header_regs + i + first_component] = offset(value, bld, i);
2979 }
2980
2981 unsigned mlen = header_regs + num_components + first_component;
2982 fs_reg payload =
2983 bld.vgrf(BRW_REGISTER_TYPE_UD, mlen);
2984 bld.LOAD_PAYLOAD(payload, srcs, mlen, header_regs);
2985
2986 fs_inst *inst = bld.emit(opcode, bld.null_reg_ud(), payload);
2987 inst->offset = imm_offset;
2988 inst->mlen = mlen;
2989 break;
2990 }
2991
2992 default:
2993 nir_emit_intrinsic(bld, instr);
2994 break;
2995 }
2996 }
2997
2998 void
2999 fs_visitor::nir_emit_tes_intrinsic(const fs_builder &bld,
3000 nir_intrinsic_instr *instr)
3001 {
3002 assert(stage == MESA_SHADER_TESS_EVAL);
3003 struct brw_tes_prog_data *tes_prog_data = brw_tes_prog_data(prog_data);
3004
3005 fs_reg dest;
3006 if (nir_intrinsic_infos[instr->intrinsic].has_dest)
3007 dest = get_nir_dest(instr->dest);
3008
3009 switch (instr->intrinsic) {
3010 case nir_intrinsic_load_primitive_id:
3011 bld.MOV(dest, fs_reg(brw_vec1_grf(0, 1)));
3012 break;
3013 case nir_intrinsic_load_tess_coord:
3014 /* gl_TessCoord is part of the payload in g1-3 */
3015 for (unsigned i = 0; i < 3; i++) {
3016 bld.MOV(offset(dest, bld, i), fs_reg(brw_vec8_grf(1 + i, 0)));
3017 }
3018 break;
3019
3020 case nir_intrinsic_load_input:
3021 case nir_intrinsic_load_per_vertex_input: {
3022 assert(nir_dest_bit_size(instr->dest) == 32);
3023 fs_reg indirect_offset = get_indirect_offset(instr);
3024 unsigned imm_offset = instr->const_index[0];
3025 unsigned first_component = nir_intrinsic_component(instr);
3026
3027 fs_inst *inst;
3028 if (indirect_offset.file == BAD_FILE) {
3029 /* Arbitrarily only push up to 32 vec4 slots worth of data,
3030 * which is 16 registers (since each holds 2 vec4 slots).
3031 */
3032 const unsigned max_push_slots = 32;
3033 if (imm_offset < max_push_slots) {
3034 fs_reg src = fs_reg(ATTR, imm_offset / 2, dest.type);
3035 for (int i = 0; i < instr->num_components; i++) {
3036 unsigned comp = 4 * (imm_offset % 2) + i + first_component;
3037 bld.MOV(offset(dest, bld, i), component(src, comp));
3038 }
3039
3040 tes_prog_data->base.urb_read_length =
3041 MAX2(tes_prog_data->base.urb_read_length,
3042 (imm_offset / 2) + 1);
3043 } else {
3044 /* Replicate the patch handle to all enabled channels */
3045 const fs_reg srcs[] = {
3046 retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_UD)
3047 };
3048 fs_reg patch_handle = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
3049 bld.LOAD_PAYLOAD(patch_handle, srcs, ARRAY_SIZE(srcs), 0);
3050
3051 if (first_component != 0) {
3052 unsigned read_components =
3053 instr->num_components + first_component;
3054 fs_reg tmp = bld.vgrf(dest.type, read_components);
3055 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8, tmp,
3056 patch_handle);