i965: Rewrite FS input handling to use the new NIR intrinsics.
[mesa.git] / src / mesa / drivers / dri / i965 / brw_fs_nir.cpp
1 /*
2 * Copyright © 2010 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "compiler/glsl/ir.h"
25 #include "brw_fs.h"
26 #include "brw_fs_surface_builder.h"
27 #include "brw_nir.h"
28 #include "brw_program.h"
29
30 using namespace brw;
31 using namespace brw::surface_access;
32
33 void
34 fs_visitor::emit_nir_code()
35 {
36 /* emit the arrays used for inputs and outputs - load/store intrinsics will
37 * be converted to reads/writes of these arrays
38 */
39 nir_setup_outputs();
40 nir_setup_uniforms();
41 nir_emit_system_values();
42
43 /* get the main function and emit it */
44 nir_foreach_function(function, nir) {
45 assert(strcmp(function->name, "main") == 0);
46 assert(function->impl);
47 nir_emit_impl(function->impl);
48 }
49 }
50
51 void
52 fs_visitor::nir_setup_single_output_varying(fs_reg *reg,
53 const glsl_type *type,
54 unsigned *location)
55 {
56 if (type->is_array() || type->is_matrix()) {
57 const struct glsl_type *elem_type = glsl_get_array_element(type);
58 const unsigned length = glsl_get_length(type);
59
60 for (unsigned i = 0; i < length; i++) {
61 nir_setup_single_output_varying(reg, elem_type, location);
62 }
63 } else if (type->is_record()) {
64 for (unsigned i = 0; i < type->length; i++) {
65 const struct glsl_type *field_type = type->fields.structure[i].type;
66 nir_setup_single_output_varying(reg, field_type, location);
67 }
68 } else {
69 assert(type->is_scalar() || type->is_vector());
70 unsigned num_elements = type->vector_elements;
71 if (type->is_double())
72 num_elements *= 2;
73 for (unsigned count = 0; count < num_elements; count += 4) {
74 this->outputs[*location] = *reg;
75 this->output_components[*location] = MIN2(4, num_elements - count);
76 *reg = offset(*reg, bld, this->output_components[*location]);
77 (*location)++;
78 }
79 }
80 }
81
82 void
83 fs_visitor::nir_setup_outputs()
84 {
85 if (stage == MESA_SHADER_TESS_CTRL)
86 return;
87
88 brw_wm_prog_key *key = (brw_wm_prog_key*) this->key;
89
90 nir_outputs = bld.vgrf(BRW_REGISTER_TYPE_F, nir->num_outputs);
91
92 nir_foreach_variable(var, &nir->outputs) {
93 fs_reg reg = offset(nir_outputs, bld, var->data.driver_location);
94
95 switch (stage) {
96 case MESA_SHADER_VERTEX:
97 case MESA_SHADER_TESS_EVAL:
98 case MESA_SHADER_GEOMETRY: {
99 unsigned location = var->data.location;
100 nir_setup_single_output_varying(&reg, var->type, &location);
101 break;
102 }
103 case MESA_SHADER_FRAGMENT:
104 if (key->force_dual_color_blend &&
105 var->data.location == FRAG_RESULT_DATA1) {
106 this->dual_src_output = reg;
107 this->do_dual_src = true;
108 } else if (var->data.index > 0) {
109 assert(var->data.location == FRAG_RESULT_DATA0);
110 assert(var->data.index == 1);
111 this->dual_src_output = reg;
112 this->do_dual_src = true;
113 } else if (var->data.location == FRAG_RESULT_COLOR) {
114 /* Writing gl_FragColor outputs to all color regions. */
115 for (unsigned int i = 0; i < MAX2(key->nr_color_regions, 1); i++) {
116 this->outputs[i] = reg;
117 this->output_components[i] = 4;
118 }
119 } else if (var->data.location == FRAG_RESULT_DEPTH) {
120 this->frag_depth = reg;
121 } else if (var->data.location == FRAG_RESULT_STENCIL) {
122 this->frag_stencil = reg;
123 } else if (var->data.location == FRAG_RESULT_SAMPLE_MASK) {
124 this->sample_mask = reg;
125 } else {
126 int vector_elements = var->type->without_array()->vector_elements;
127
128 /* gl_FragData or a user-defined FS output */
129 assert(var->data.location >= FRAG_RESULT_DATA0 &&
130 var->data.location < FRAG_RESULT_DATA0+BRW_MAX_DRAW_BUFFERS);
131
132 /* General color output. */
133 for (unsigned int i = 0; i < MAX2(1, var->type->length); i++) {
134 int output = var->data.location - FRAG_RESULT_DATA0 + i;
135 this->outputs[output] = offset(reg, bld, vector_elements * i);
136 this->output_components[output] = vector_elements;
137 }
138 }
139 break;
140 default:
141 unreachable("unhandled shader stage");
142 }
143 }
144 }
145
146 void
147 fs_visitor::nir_setup_uniforms()
148 {
149 if (dispatch_width != min_dispatch_width)
150 return;
151
152 uniforms = nir->num_uniforms / 4;
153 }
154
155 static bool
156 emit_system_values_block(nir_block *block, fs_visitor *v)
157 {
158 fs_reg *reg;
159
160 nir_foreach_instr(instr, block) {
161 if (instr->type != nir_instr_type_intrinsic)
162 continue;
163
164 nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
165 switch (intrin->intrinsic) {
166 case nir_intrinsic_load_vertex_id:
167 unreachable("should be lowered by lower_vertex_id().");
168
169 case nir_intrinsic_load_vertex_id_zero_base:
170 assert(v->stage == MESA_SHADER_VERTEX);
171 reg = &v->nir_system_values[SYSTEM_VALUE_VERTEX_ID_ZERO_BASE];
172 if (reg->file == BAD_FILE)
173 *reg = *v->emit_vs_system_value(SYSTEM_VALUE_VERTEX_ID_ZERO_BASE);
174 break;
175
176 case nir_intrinsic_load_base_vertex:
177 assert(v->stage == MESA_SHADER_VERTEX);
178 reg = &v->nir_system_values[SYSTEM_VALUE_BASE_VERTEX];
179 if (reg->file == BAD_FILE)
180 *reg = *v->emit_vs_system_value(SYSTEM_VALUE_BASE_VERTEX);
181 break;
182
183 case nir_intrinsic_load_instance_id:
184 assert(v->stage == MESA_SHADER_VERTEX);
185 reg = &v->nir_system_values[SYSTEM_VALUE_INSTANCE_ID];
186 if (reg->file == BAD_FILE)
187 *reg = *v->emit_vs_system_value(SYSTEM_VALUE_INSTANCE_ID);
188 break;
189
190 case nir_intrinsic_load_base_instance:
191 assert(v->stage == MESA_SHADER_VERTEX);
192 reg = &v->nir_system_values[SYSTEM_VALUE_BASE_INSTANCE];
193 if (reg->file == BAD_FILE)
194 *reg = *v->emit_vs_system_value(SYSTEM_VALUE_BASE_INSTANCE);
195 break;
196
197 case nir_intrinsic_load_draw_id:
198 assert(v->stage == MESA_SHADER_VERTEX);
199 reg = &v->nir_system_values[SYSTEM_VALUE_DRAW_ID];
200 if (reg->file == BAD_FILE)
201 *reg = *v->emit_vs_system_value(SYSTEM_VALUE_DRAW_ID);
202 break;
203
204 case nir_intrinsic_load_invocation_id:
205 if (v->stage == MESA_SHADER_TESS_CTRL)
206 break;
207 assert(v->stage == MESA_SHADER_GEOMETRY);
208 reg = &v->nir_system_values[SYSTEM_VALUE_INVOCATION_ID];
209 if (reg->file == BAD_FILE) {
210 const fs_builder abld = v->bld.annotate("gl_InvocationID", NULL);
211 fs_reg g1(retype(brw_vec8_grf(1, 0), BRW_REGISTER_TYPE_UD));
212 fs_reg iid = abld.vgrf(BRW_REGISTER_TYPE_UD, 1);
213 abld.SHR(iid, g1, brw_imm_ud(27u));
214 *reg = iid;
215 }
216 break;
217
218 case nir_intrinsic_load_sample_pos:
219 assert(v->stage == MESA_SHADER_FRAGMENT);
220 reg = &v->nir_system_values[SYSTEM_VALUE_SAMPLE_POS];
221 if (reg->file == BAD_FILE)
222 *reg = *v->emit_samplepos_setup();
223 break;
224
225 case nir_intrinsic_load_sample_id:
226 assert(v->stage == MESA_SHADER_FRAGMENT);
227 reg = &v->nir_system_values[SYSTEM_VALUE_SAMPLE_ID];
228 if (reg->file == BAD_FILE)
229 *reg = *v->emit_sampleid_setup();
230 break;
231
232 case nir_intrinsic_load_sample_mask_in:
233 assert(v->stage == MESA_SHADER_FRAGMENT);
234 assert(v->devinfo->gen >= 7);
235 reg = &v->nir_system_values[SYSTEM_VALUE_SAMPLE_MASK_IN];
236 if (reg->file == BAD_FILE)
237 *reg = *v->emit_samplemaskin_setup();
238 break;
239
240 case nir_intrinsic_load_work_group_id:
241 assert(v->stage == MESA_SHADER_COMPUTE);
242 reg = &v->nir_system_values[SYSTEM_VALUE_WORK_GROUP_ID];
243 if (reg->file == BAD_FILE)
244 *reg = *v->emit_cs_work_group_id_setup();
245 break;
246
247 case nir_intrinsic_load_helper_invocation:
248 assert(v->stage == MESA_SHADER_FRAGMENT);
249 reg = &v->nir_system_values[SYSTEM_VALUE_HELPER_INVOCATION];
250 if (reg->file == BAD_FILE) {
251 const fs_builder abld =
252 v->bld.annotate("gl_HelperInvocation", NULL);
253
254 /* On Gen6+ (gl_HelperInvocation is only exposed on Gen7+) the
255 * pixel mask is in g1.7 of the thread payload.
256 *
257 * We move the per-channel pixel enable bit to the low bit of each
258 * channel by shifting the byte containing the pixel mask by the
259 * vector immediate 0x76543210UV.
260 *
261 * The region of <1,8,0> reads only 1 byte (the pixel masks for
262 * subspans 0 and 1) in SIMD8 and an additional byte (the pixel
263 * masks for 2 and 3) in SIMD16.
264 */
265 fs_reg shifted = abld.vgrf(BRW_REGISTER_TYPE_UW, 1);
266 abld.SHR(shifted,
267 stride(byte_offset(retype(brw_vec1_grf(1, 0),
268 BRW_REGISTER_TYPE_UB), 28),
269 1, 8, 0),
270 brw_imm_v(0x76543210));
271
272 /* A set bit in the pixel mask means the channel is enabled, but
273 * that is the opposite of gl_HelperInvocation so we need to invert
274 * the mask.
275 *
276 * The negate source-modifier bit of logical instructions on Gen8+
277 * performs 1's complement negation, so we can use that instead of
278 * a NOT instruction.
279 */
280 fs_reg inverted = negate(shifted);
281 if (v->devinfo->gen < 8) {
282 inverted = abld.vgrf(BRW_REGISTER_TYPE_UW);
283 abld.NOT(inverted, shifted);
284 }
285
286 /* We then resolve the 0/1 result to 0/~0 boolean values by ANDing
287 * with 1 and negating.
288 */
289 fs_reg anded = abld.vgrf(BRW_REGISTER_TYPE_UD, 1);
290 abld.AND(anded, inverted, brw_imm_uw(1));
291
292 fs_reg dst = abld.vgrf(BRW_REGISTER_TYPE_D, 1);
293 abld.MOV(dst, negate(retype(anded, BRW_REGISTER_TYPE_D)));
294 *reg = dst;
295 }
296 break;
297
298 default:
299 break;
300 }
301 }
302
303 return true;
304 }
305
306 void
307 fs_visitor::nir_emit_system_values()
308 {
309 nir_system_values = ralloc_array(mem_ctx, fs_reg, SYSTEM_VALUE_MAX);
310 for (unsigned i = 0; i < SYSTEM_VALUE_MAX; i++) {
311 nir_system_values[i] = fs_reg();
312 }
313
314 nir_foreach_function(function, nir) {
315 assert(strcmp(function->name, "main") == 0);
316 assert(function->impl);
317 nir_foreach_block(block, function->impl) {
318 emit_system_values_block(block, this);
319 }
320 }
321 }
322
323 void
324 fs_visitor::nir_emit_impl(nir_function_impl *impl)
325 {
326 nir_locals = ralloc_array(mem_ctx, fs_reg, impl->reg_alloc);
327 for (unsigned i = 0; i < impl->reg_alloc; i++) {
328 nir_locals[i] = fs_reg();
329 }
330
331 foreach_list_typed(nir_register, reg, node, &impl->registers) {
332 unsigned array_elems =
333 reg->num_array_elems == 0 ? 1 : reg->num_array_elems;
334 unsigned size = array_elems * reg->num_components;
335 const brw_reg_type reg_type =
336 reg->bit_size == 32 ? BRW_REGISTER_TYPE_F : BRW_REGISTER_TYPE_DF;
337 nir_locals[reg->index] = bld.vgrf(reg_type, size);
338 }
339
340 nir_ssa_values = reralloc(mem_ctx, nir_ssa_values, fs_reg,
341 impl->ssa_alloc);
342
343 nir_emit_cf_list(&impl->body);
344 }
345
346 void
347 fs_visitor::nir_emit_cf_list(exec_list *list)
348 {
349 exec_list_validate(list);
350 foreach_list_typed(nir_cf_node, node, node, list) {
351 switch (node->type) {
352 case nir_cf_node_if:
353 nir_emit_if(nir_cf_node_as_if(node));
354 break;
355
356 case nir_cf_node_loop:
357 nir_emit_loop(nir_cf_node_as_loop(node));
358 break;
359
360 case nir_cf_node_block:
361 nir_emit_block(nir_cf_node_as_block(node));
362 break;
363
364 default:
365 unreachable("Invalid CFG node block");
366 }
367 }
368 }
369
370 void
371 fs_visitor::nir_emit_if(nir_if *if_stmt)
372 {
373 /* first, put the condition into f0 */
374 fs_inst *inst = bld.MOV(bld.null_reg_d(),
375 retype(get_nir_src(if_stmt->condition),
376 BRW_REGISTER_TYPE_D));
377 inst->conditional_mod = BRW_CONDITIONAL_NZ;
378
379 bld.IF(BRW_PREDICATE_NORMAL);
380
381 nir_emit_cf_list(&if_stmt->then_list);
382
383 /* note: if the else is empty, dead CF elimination will remove it */
384 bld.emit(BRW_OPCODE_ELSE);
385
386 nir_emit_cf_list(&if_stmt->else_list);
387
388 bld.emit(BRW_OPCODE_ENDIF);
389 }
390
391 void
392 fs_visitor::nir_emit_loop(nir_loop *loop)
393 {
394 bld.emit(BRW_OPCODE_DO);
395
396 nir_emit_cf_list(&loop->body);
397
398 bld.emit(BRW_OPCODE_WHILE);
399 }
400
401 void
402 fs_visitor::nir_emit_block(nir_block *block)
403 {
404 nir_foreach_instr(instr, block) {
405 nir_emit_instr(instr);
406 }
407 }
408
409 void
410 fs_visitor::nir_emit_instr(nir_instr *instr)
411 {
412 const fs_builder abld = bld.annotate(NULL, instr);
413
414 switch (instr->type) {
415 case nir_instr_type_alu:
416 nir_emit_alu(abld, nir_instr_as_alu(instr));
417 break;
418
419 case nir_instr_type_intrinsic:
420 switch (stage) {
421 case MESA_SHADER_VERTEX:
422 nir_emit_vs_intrinsic(abld, nir_instr_as_intrinsic(instr));
423 break;
424 case MESA_SHADER_TESS_CTRL:
425 nir_emit_tcs_intrinsic(abld, nir_instr_as_intrinsic(instr));
426 break;
427 case MESA_SHADER_TESS_EVAL:
428 nir_emit_tes_intrinsic(abld, nir_instr_as_intrinsic(instr));
429 break;
430 case MESA_SHADER_GEOMETRY:
431 nir_emit_gs_intrinsic(abld, nir_instr_as_intrinsic(instr));
432 break;
433 case MESA_SHADER_FRAGMENT:
434 nir_emit_fs_intrinsic(abld, nir_instr_as_intrinsic(instr));
435 break;
436 case MESA_SHADER_COMPUTE:
437 nir_emit_cs_intrinsic(abld, nir_instr_as_intrinsic(instr));
438 break;
439 default:
440 unreachable("unsupported shader stage");
441 }
442 break;
443
444 case nir_instr_type_tex:
445 nir_emit_texture(abld, nir_instr_as_tex(instr));
446 break;
447
448 case nir_instr_type_load_const:
449 nir_emit_load_const(abld, nir_instr_as_load_const(instr));
450 break;
451
452 case nir_instr_type_ssa_undef:
453 nir_emit_undef(abld, nir_instr_as_ssa_undef(instr));
454 break;
455
456 case nir_instr_type_jump:
457 nir_emit_jump(abld, nir_instr_as_jump(instr));
458 break;
459
460 default:
461 unreachable("unknown instruction type");
462 }
463 }
464
465 /**
466 * Recognizes a parent instruction of nir_op_extract_* and changes the type to
467 * match instr.
468 */
469 bool
470 fs_visitor::optimize_extract_to_float(nir_alu_instr *instr,
471 const fs_reg &result)
472 {
473 if (!instr->src[0].src.is_ssa ||
474 !instr->src[0].src.ssa->parent_instr)
475 return false;
476
477 if (instr->src[0].src.ssa->parent_instr->type != nir_instr_type_alu)
478 return false;
479
480 nir_alu_instr *src0 =
481 nir_instr_as_alu(instr->src[0].src.ssa->parent_instr);
482
483 if (src0->op != nir_op_extract_u8 && src0->op != nir_op_extract_u16 &&
484 src0->op != nir_op_extract_i8 && src0->op != nir_op_extract_i16)
485 return false;
486
487 nir_const_value *element = nir_src_as_const_value(src0->src[1].src);
488 assert(element != NULL);
489
490 /* Element type to extract.*/
491 const brw_reg_type type = brw_int_type(
492 src0->op == nir_op_extract_u16 || src0->op == nir_op_extract_i16 ? 2 : 1,
493 src0->op == nir_op_extract_i16 || src0->op == nir_op_extract_i8);
494
495 fs_reg op0 = get_nir_src(src0->src[0].src);
496 op0.type = brw_type_for_nir_type(
497 (nir_alu_type)(nir_op_infos[src0->op].input_types[0] |
498 nir_src_bit_size(src0->src[0].src)));
499 op0 = offset(op0, bld, src0->src[0].swizzle[0]);
500
501 set_saturate(instr->dest.saturate,
502 bld.MOV(result, subscript(op0, type, element->u32[0])));
503 return true;
504 }
505
506 bool
507 fs_visitor::optimize_frontfacing_ternary(nir_alu_instr *instr,
508 const fs_reg &result)
509 {
510 if (!instr->src[0].src.is_ssa ||
511 instr->src[0].src.ssa->parent_instr->type != nir_instr_type_intrinsic)
512 return false;
513
514 nir_intrinsic_instr *src0 =
515 nir_instr_as_intrinsic(instr->src[0].src.ssa->parent_instr);
516
517 if (src0->intrinsic != nir_intrinsic_load_front_face)
518 return false;
519
520 nir_const_value *value1 = nir_src_as_const_value(instr->src[1].src);
521 if (!value1 || fabsf(value1->f32[0]) != 1.0f)
522 return false;
523
524 nir_const_value *value2 = nir_src_as_const_value(instr->src[2].src);
525 if (!value2 || fabsf(value2->f32[0]) != 1.0f)
526 return false;
527
528 fs_reg tmp = vgrf(glsl_type::int_type);
529
530 if (devinfo->gen >= 6) {
531 /* Bit 15 of g0.0 is 0 if the polygon is front facing. */
532 fs_reg g0 = fs_reg(retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_W));
533
534 /* For (gl_FrontFacing ? 1.0 : -1.0), emit:
535 *
536 * or(8) tmp.1<2>W g0.0<0,1,0>W 0x00003f80W
537 * and(8) dst<1>D tmp<8,8,1>D 0xbf800000D
538 *
539 * and negate g0.0<0,1,0>W for (gl_FrontFacing ? -1.0 : 1.0).
540 *
541 * This negation looks like it's safe in practice, because bits 0:4 will
542 * surely be TRIANGLES
543 */
544
545 if (value1->f32[0] == -1.0f) {
546 g0.negate = true;
547 }
548
549 tmp.type = BRW_REGISTER_TYPE_W;
550 tmp.subreg_offset = 2;
551 tmp.stride = 2;
552
553 bld.OR(tmp, g0, brw_imm_uw(0x3f80));
554
555 tmp.type = BRW_REGISTER_TYPE_D;
556 tmp.subreg_offset = 0;
557 tmp.stride = 1;
558 } else {
559 /* Bit 31 of g1.6 is 0 if the polygon is front facing. */
560 fs_reg g1_6 = fs_reg(retype(brw_vec1_grf(1, 6), BRW_REGISTER_TYPE_D));
561
562 /* For (gl_FrontFacing ? 1.0 : -1.0), emit:
563 *
564 * or(8) tmp<1>D g1.6<0,1,0>D 0x3f800000D
565 * and(8) dst<1>D tmp<8,8,1>D 0xbf800000D
566 *
567 * and negate g1.6<0,1,0>D for (gl_FrontFacing ? -1.0 : 1.0).
568 *
569 * This negation looks like it's safe in practice, because bits 0:4 will
570 * surely be TRIANGLES
571 */
572
573 if (value1->f32[0] == -1.0f) {
574 g1_6.negate = true;
575 }
576
577 bld.OR(tmp, g1_6, brw_imm_d(0x3f800000));
578 }
579 bld.AND(retype(result, BRW_REGISTER_TYPE_D), tmp, brw_imm_d(0xbf800000));
580
581 return true;
582 }
583
584 static void
585 emit_find_msb_using_lzd(const fs_builder &bld,
586 const fs_reg &result,
587 const fs_reg &src,
588 bool is_signed)
589 {
590 fs_inst *inst;
591 fs_reg temp = src;
592
593 if (is_signed) {
594 /* LZD of an absolute value source almost always does the right
595 * thing. There are two problem values:
596 *
597 * * 0x80000000. Since abs(0x80000000) == 0x80000000, LZD returns
598 * 0. However, findMSB(int(0x80000000)) == 30.
599 *
600 * * 0xffffffff. Since abs(0xffffffff) == 1, LZD returns
601 * 31. Section 8.8 (Integer Functions) of the GLSL 4.50 spec says:
602 *
603 * For a value of zero or negative one, -1 will be returned.
604 *
605 * * Negative powers of two. LZD(abs(-(1<<x))) returns x, but
606 * findMSB(-(1<<x)) should return x-1.
607 *
608 * For all negative number cases, including 0x80000000 and
609 * 0xffffffff, the correct value is obtained from LZD if instead of
610 * negating the (already negative) value the logical-not is used. A
611 * conditonal logical-not can be achieved in two instructions.
612 */
613 temp = bld.vgrf(BRW_REGISTER_TYPE_D);
614
615 bld.ASR(temp, src, brw_imm_d(31));
616 bld.XOR(temp, temp, src);
617 }
618
619 bld.LZD(retype(result, BRW_REGISTER_TYPE_UD),
620 retype(temp, BRW_REGISTER_TYPE_UD));
621
622 /* LZD counts from the MSB side, while GLSL's findMSB() wants the count
623 * from the LSB side. Subtract the result from 31 to convert the MSB
624 * count into an LSB count. If no bits are set, LZD will return 32.
625 * 31-32 = -1, which is exactly what findMSB() is supposed to return.
626 */
627 inst = bld.ADD(result, retype(result, BRW_REGISTER_TYPE_D), brw_imm_d(31));
628 inst->src[0].negate = true;
629 }
630
631 void
632 fs_visitor::nir_emit_alu(const fs_builder &bld, nir_alu_instr *instr)
633 {
634 struct brw_wm_prog_key *fs_key = (struct brw_wm_prog_key *) this->key;
635 fs_inst *inst;
636
637 fs_reg result = get_nir_dest(instr->dest.dest);
638 result.type = brw_type_for_nir_type(
639 (nir_alu_type)(nir_op_infos[instr->op].output_type |
640 nir_dest_bit_size(instr->dest.dest)));
641
642 fs_reg op[4];
643 for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
644 op[i] = get_nir_src(instr->src[i].src);
645 op[i].type = brw_type_for_nir_type(
646 (nir_alu_type)(nir_op_infos[instr->op].input_types[i] |
647 nir_src_bit_size(instr->src[i].src)));
648 op[i].abs = instr->src[i].abs;
649 op[i].negate = instr->src[i].negate;
650 }
651
652 /* We get a bunch of mov's out of the from_ssa pass and they may still
653 * be vectorized. We'll handle them as a special-case. We'll also
654 * handle vecN here because it's basically the same thing.
655 */
656 switch (instr->op) {
657 case nir_op_imov:
658 case nir_op_fmov:
659 case nir_op_vec2:
660 case nir_op_vec3:
661 case nir_op_vec4: {
662 fs_reg temp = result;
663 bool need_extra_copy = false;
664 for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
665 if (!instr->src[i].src.is_ssa &&
666 instr->dest.dest.reg.reg == instr->src[i].src.reg.reg) {
667 need_extra_copy = true;
668 temp = bld.vgrf(result.type, 4);
669 break;
670 }
671 }
672
673 for (unsigned i = 0; i < 4; i++) {
674 if (!(instr->dest.write_mask & (1 << i)))
675 continue;
676
677 if (instr->op == nir_op_imov || instr->op == nir_op_fmov) {
678 inst = bld.MOV(offset(temp, bld, i),
679 offset(op[0], bld, instr->src[0].swizzle[i]));
680 } else {
681 inst = bld.MOV(offset(temp, bld, i),
682 offset(op[i], bld, instr->src[i].swizzle[0]));
683 }
684 inst->saturate = instr->dest.saturate;
685 }
686
687 /* In this case the source and destination registers were the same,
688 * so we need to insert an extra set of moves in order to deal with
689 * any swizzling.
690 */
691 if (need_extra_copy) {
692 for (unsigned i = 0; i < 4; i++) {
693 if (!(instr->dest.write_mask & (1 << i)))
694 continue;
695
696 bld.MOV(offset(result, bld, i), offset(temp, bld, i));
697 }
698 }
699 return;
700 }
701 default:
702 break;
703 }
704
705 /* At this point, we have dealt with any instruction that operates on
706 * more than a single channel. Therefore, we can just adjust the source
707 * and destination registers for that channel and emit the instruction.
708 */
709 unsigned channel = 0;
710 if (nir_op_infos[instr->op].output_size == 0) {
711 /* Since NIR is doing the scalarizing for us, we should only ever see
712 * vectorized operations with a single channel.
713 */
714 assert(_mesa_bitcount(instr->dest.write_mask) == 1);
715 channel = ffs(instr->dest.write_mask) - 1;
716
717 result = offset(result, bld, channel);
718 }
719
720 for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
721 assert(nir_op_infos[instr->op].input_sizes[i] < 2);
722 op[i] = offset(op[i], bld, instr->src[i].swizzle[channel]);
723 }
724
725 switch (instr->op) {
726 case nir_op_i2f:
727 case nir_op_u2f:
728 if (optimize_extract_to_float(instr, result))
729 return;
730 inst = bld.MOV(result, op[0]);
731 inst->saturate = instr->dest.saturate;
732 break;
733
734 case nir_op_f2d:
735 case nir_op_i2d:
736 case nir_op_u2d:
737 /* CHV PRM, vol07, 3D Media GPGPU Engine, Register Region Restrictions:
738 *
739 * "When source or destination is 64b (...), regioning in Align1
740 * must follow these rules:
741 *
742 * 1. Source and destination horizontal stride must be aligned to
743 * the same qword.
744 * (...)"
745 *
746 * This means that 32-bit to 64-bit conversions need to have the 32-bit
747 * data elements aligned to 64-bit. This restriction does not apply to
748 * BDW and later.
749 */
750 if (devinfo->is_cherryview || devinfo->is_broxton) {
751 fs_reg tmp = bld.vgrf(result.type, 1);
752 tmp = subscript(tmp, op[0].type, 0);
753 inst = bld.MOV(tmp, op[0]);
754 inst = bld.MOV(result, tmp);
755 inst->saturate = instr->dest.saturate;
756 break;
757 }
758 /* fallthrough */
759 case nir_op_d2f:
760 case nir_op_d2i:
761 case nir_op_d2u:
762 inst = bld.MOV(result, op[0]);
763 inst->saturate = instr->dest.saturate;
764 break;
765
766 case nir_op_f2i:
767 case nir_op_f2u:
768 bld.MOV(result, op[0]);
769 break;
770
771 case nir_op_fsign: {
772 if (type_sz(op[0].type) < 8) {
773 /* AND(val, 0x80000000) gives the sign bit.
774 *
775 * Predicated OR ORs 1.0 (0x3f800000) with the sign bit if val is not
776 * zero.
777 */
778 bld.CMP(bld.null_reg_f(), op[0], brw_imm_f(0.0f), BRW_CONDITIONAL_NZ);
779
780 fs_reg result_int = retype(result, BRW_REGISTER_TYPE_UD);
781 op[0].type = BRW_REGISTER_TYPE_UD;
782 result.type = BRW_REGISTER_TYPE_UD;
783 bld.AND(result_int, op[0], brw_imm_ud(0x80000000u));
784
785 inst = bld.OR(result_int, result_int, brw_imm_ud(0x3f800000u));
786 inst->predicate = BRW_PREDICATE_NORMAL;
787 if (instr->dest.saturate) {
788 inst = bld.MOV(result, result);
789 inst->saturate = true;
790 }
791 } else {
792 /* For doubles we do the same but we need to consider:
793 *
794 * - 2-src instructions can't operate with 64-bit immediates
795 * - The sign is encoded in the high 32-bit of each DF
796 * - CMP with DF requires special handling in SIMD16
797 * - We need to produce a DF result.
798 */
799
800 /* 2-src instructions can't have 64-bit immediates, so put 0.0 in
801 * a register and compare with that.
802 */
803 fs_reg tmp = vgrf(glsl_type::double_type);
804 bld.MOV(tmp, setup_imm_df(bld, 0.0));
805
806 /* A direct DF CMP using the flag register (null dst) won't work in
807 * SIMD16 because the CMP will be split in two by lower_simd_width,
808 * resulting in two CMP instructions with the same dst (NULL),
809 * leading to dead code elimination of the first one. In SIMD8,
810 * however, there is no need to split the CMP and we can save some
811 * work.
812 */
813 fs_reg dst_tmp = vgrf(glsl_type::double_type);
814 bld.CMP(dst_tmp, op[0], tmp, BRW_CONDITIONAL_NZ);
815
816 /* In SIMD16 we want to avoid using a NULL dst register with DF CMP,
817 * so we store the result of the comparison in a vgrf instead and
818 * then we generate a UD comparison from that that won't have to
819 * be split by lower_simd_width. This is what NIR does to handle
820 * double comparisons in the general case.
821 */
822 if (bld.dispatch_width() == 16 ) {
823 fs_reg dst_tmp_ud = retype(dst_tmp, BRW_REGISTER_TYPE_UD);
824 bld.MOV(dst_tmp_ud, subscript(dst_tmp, BRW_REGISTER_TYPE_UD, 0));
825 bld.CMP(bld.null_reg_ud(),
826 dst_tmp_ud, brw_imm_ud(0), BRW_CONDITIONAL_NZ);
827 }
828
829 /* Get the high 32-bit of each double component where the sign is */
830 fs_reg result_int = retype(result, BRW_REGISTER_TYPE_UD);
831 bld.MOV(result_int, subscript(op[0], BRW_REGISTER_TYPE_UD, 1));
832
833 /* Get the sign bit */
834 bld.AND(result_int, result_int, brw_imm_ud(0x80000000u));
835
836 /* Add 1.0 to the sign, predicated to skip the case of op[0] == 0.0 */
837 inst = bld.OR(result_int, result_int, brw_imm_ud(0x3f800000u));
838 inst->predicate = BRW_PREDICATE_NORMAL;
839
840 /* Convert from 32-bit float to 64-bit double */
841 result.type = BRW_REGISTER_TYPE_DF;
842 inst = bld.MOV(result, retype(result_int, BRW_REGISTER_TYPE_F));
843
844 if (instr->dest.saturate) {
845 inst = bld.MOV(result, result);
846 inst->saturate = true;
847 }
848 }
849 break;
850 }
851
852 case nir_op_isign:
853 /* ASR(val, 31) -> negative val generates 0xffffffff (signed -1).
854 * -> non-negative val generates 0x00000000.
855 * Predicated OR sets 1 if val is positive.
856 */
857 assert(nir_dest_bit_size(instr->dest.dest) < 64);
858 bld.CMP(bld.null_reg_d(), op[0], brw_imm_d(0), BRW_CONDITIONAL_G);
859 bld.ASR(result, op[0], brw_imm_d(31));
860 inst = bld.OR(result, result, brw_imm_d(1));
861 inst->predicate = BRW_PREDICATE_NORMAL;
862 break;
863
864 case nir_op_frcp:
865 inst = bld.emit(SHADER_OPCODE_RCP, result, op[0]);
866 inst->saturate = instr->dest.saturate;
867 break;
868
869 case nir_op_fexp2:
870 inst = bld.emit(SHADER_OPCODE_EXP2, result, op[0]);
871 inst->saturate = instr->dest.saturate;
872 break;
873
874 case nir_op_flog2:
875 inst = bld.emit(SHADER_OPCODE_LOG2, result, op[0]);
876 inst->saturate = instr->dest.saturate;
877 break;
878
879 case nir_op_fsin:
880 inst = bld.emit(SHADER_OPCODE_SIN, result, op[0]);
881 inst->saturate = instr->dest.saturate;
882 break;
883
884 case nir_op_fcos:
885 inst = bld.emit(SHADER_OPCODE_COS, result, op[0]);
886 inst->saturate = instr->dest.saturate;
887 break;
888
889 case nir_op_fddx:
890 if (fs_key->high_quality_derivatives) {
891 inst = bld.emit(FS_OPCODE_DDX_FINE, result, op[0]);
892 } else {
893 inst = bld.emit(FS_OPCODE_DDX_COARSE, result, op[0]);
894 }
895 inst->saturate = instr->dest.saturate;
896 break;
897 case nir_op_fddx_fine:
898 inst = bld.emit(FS_OPCODE_DDX_FINE, result, op[0]);
899 inst->saturate = instr->dest.saturate;
900 break;
901 case nir_op_fddx_coarse:
902 inst = bld.emit(FS_OPCODE_DDX_COARSE, result, op[0]);
903 inst->saturate = instr->dest.saturate;
904 break;
905 case nir_op_fddy:
906 if (fs_key->high_quality_derivatives) {
907 inst = bld.emit(FS_OPCODE_DDY_FINE, result, op[0]);
908 } else {
909 inst = bld.emit(FS_OPCODE_DDY_COARSE, result, op[0]);
910 }
911 inst->saturate = instr->dest.saturate;
912 break;
913 case nir_op_fddy_fine:
914 inst = bld.emit(FS_OPCODE_DDY_FINE, result, op[0]);
915 inst->saturate = instr->dest.saturate;
916 break;
917 case nir_op_fddy_coarse:
918 inst = bld.emit(FS_OPCODE_DDY_COARSE, result, op[0]);
919 inst->saturate = instr->dest.saturate;
920 break;
921
922 case nir_op_iadd:
923 assert(nir_dest_bit_size(instr->dest.dest) < 64);
924 case nir_op_fadd:
925 inst = bld.ADD(result, op[0], op[1]);
926 inst->saturate = instr->dest.saturate;
927 break;
928
929 case nir_op_fmul:
930 inst = bld.MUL(result, op[0], op[1]);
931 inst->saturate = instr->dest.saturate;
932 break;
933
934 case nir_op_imul:
935 assert(nir_dest_bit_size(instr->dest.dest) < 64);
936 bld.MUL(result, op[0], op[1]);
937 break;
938
939 case nir_op_imul_high:
940 case nir_op_umul_high:
941 assert(nir_dest_bit_size(instr->dest.dest) < 64);
942 bld.emit(SHADER_OPCODE_MULH, result, op[0], op[1]);
943 break;
944
945 case nir_op_idiv:
946 case nir_op_udiv:
947 assert(nir_dest_bit_size(instr->dest.dest) < 64);
948 bld.emit(SHADER_OPCODE_INT_QUOTIENT, result, op[0], op[1]);
949 break;
950
951 case nir_op_uadd_carry:
952 unreachable("Should have been lowered by carry_to_arith().");
953
954 case nir_op_usub_borrow:
955 unreachable("Should have been lowered by borrow_to_arith().");
956
957 case nir_op_umod:
958 case nir_op_irem:
959 /* According to the sign table for INT DIV in the Ivy Bridge PRM, it
960 * appears that our hardware just does the right thing for signed
961 * remainder.
962 */
963 assert(nir_dest_bit_size(instr->dest.dest) < 64);
964 bld.emit(SHADER_OPCODE_INT_REMAINDER, result, op[0], op[1]);
965 break;
966
967 case nir_op_imod: {
968 /* Get a regular C-style remainder. If a % b == 0, set the predicate. */
969 bld.emit(SHADER_OPCODE_INT_REMAINDER, result, op[0], op[1]);
970
971 /* Math instructions don't support conditional mod */
972 inst = bld.MOV(bld.null_reg_d(), result);
973 inst->conditional_mod = BRW_CONDITIONAL_NZ;
974
975 /* Now, we need to determine if signs of the sources are different.
976 * When we XOR the sources, the top bit is 0 if they are the same and 1
977 * if they are different. We can then use a conditional modifier to
978 * turn that into a predicate. This leads us to an XOR.l instruction.
979 *
980 * Technically, according to the PRM, you're not allowed to use .l on a
981 * XOR instruction. However, emperical experiments and Curro's reading
982 * of the simulator source both indicate that it's safe.
983 */
984 fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_D);
985 inst = bld.XOR(tmp, op[0], op[1]);
986 inst->predicate = BRW_PREDICATE_NORMAL;
987 inst->conditional_mod = BRW_CONDITIONAL_L;
988
989 /* If the result of the initial remainder operation is non-zero and the
990 * two sources have different signs, add in a copy of op[1] to get the
991 * final integer modulus value.
992 */
993 inst = bld.ADD(result, result, op[1]);
994 inst->predicate = BRW_PREDICATE_NORMAL;
995 break;
996 }
997
998 case nir_op_flt:
999 case nir_op_fge:
1000 case nir_op_feq:
1001 case nir_op_fne: {
1002 fs_reg dest = result;
1003 if (nir_src_bit_size(instr->src[0].src) > 32) {
1004 dest = bld.vgrf(BRW_REGISTER_TYPE_DF, 1);
1005 }
1006 brw_conditional_mod cond;
1007 switch (instr->op) {
1008 case nir_op_flt:
1009 cond = BRW_CONDITIONAL_L;
1010 break;
1011 case nir_op_fge:
1012 cond = BRW_CONDITIONAL_GE;
1013 break;
1014 case nir_op_feq:
1015 cond = BRW_CONDITIONAL_Z;
1016 break;
1017 case nir_op_fne:
1018 cond = BRW_CONDITIONAL_NZ;
1019 break;
1020 default:
1021 unreachable("bad opcode");
1022 }
1023 bld.CMP(dest, op[0], op[1], cond);
1024 if (nir_src_bit_size(instr->src[0].src) > 32) {
1025 bld.MOV(result, subscript(dest, BRW_REGISTER_TYPE_UD, 0));
1026 }
1027 break;
1028 }
1029
1030 case nir_op_ilt:
1031 case nir_op_ult:
1032 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1033 bld.CMP(result, op[0], op[1], BRW_CONDITIONAL_L);
1034 break;
1035
1036 case nir_op_ige:
1037 case nir_op_uge:
1038 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1039 bld.CMP(result, op[0], op[1], BRW_CONDITIONAL_GE);
1040 break;
1041
1042 case nir_op_ieq:
1043 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1044 bld.CMP(result, op[0], op[1], BRW_CONDITIONAL_Z);
1045 break;
1046
1047 case nir_op_ine:
1048 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1049 bld.CMP(result, op[0], op[1], BRW_CONDITIONAL_NZ);
1050 break;
1051
1052 case nir_op_inot:
1053 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1054 if (devinfo->gen >= 8) {
1055 op[0] = resolve_source_modifiers(op[0]);
1056 }
1057 bld.NOT(result, op[0]);
1058 break;
1059 case nir_op_ixor:
1060 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1061 if (devinfo->gen >= 8) {
1062 op[0] = resolve_source_modifiers(op[0]);
1063 op[1] = resolve_source_modifiers(op[1]);
1064 }
1065 bld.XOR(result, op[0], op[1]);
1066 break;
1067 case nir_op_ior:
1068 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1069 if (devinfo->gen >= 8) {
1070 op[0] = resolve_source_modifiers(op[0]);
1071 op[1] = resolve_source_modifiers(op[1]);
1072 }
1073 bld.OR(result, op[0], op[1]);
1074 break;
1075 case nir_op_iand:
1076 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1077 if (devinfo->gen >= 8) {
1078 op[0] = resolve_source_modifiers(op[0]);
1079 op[1] = resolve_source_modifiers(op[1]);
1080 }
1081 bld.AND(result, op[0], op[1]);
1082 break;
1083
1084 case nir_op_fdot2:
1085 case nir_op_fdot3:
1086 case nir_op_fdot4:
1087 case nir_op_ball_fequal2:
1088 case nir_op_ball_iequal2:
1089 case nir_op_ball_fequal3:
1090 case nir_op_ball_iequal3:
1091 case nir_op_ball_fequal4:
1092 case nir_op_ball_iequal4:
1093 case nir_op_bany_fnequal2:
1094 case nir_op_bany_inequal2:
1095 case nir_op_bany_fnequal3:
1096 case nir_op_bany_inequal3:
1097 case nir_op_bany_fnequal4:
1098 case nir_op_bany_inequal4:
1099 unreachable("Lowered by nir_lower_alu_reductions");
1100
1101 case nir_op_fnoise1_1:
1102 case nir_op_fnoise1_2:
1103 case nir_op_fnoise1_3:
1104 case nir_op_fnoise1_4:
1105 case nir_op_fnoise2_1:
1106 case nir_op_fnoise2_2:
1107 case nir_op_fnoise2_3:
1108 case nir_op_fnoise2_4:
1109 case nir_op_fnoise3_1:
1110 case nir_op_fnoise3_2:
1111 case nir_op_fnoise3_3:
1112 case nir_op_fnoise3_4:
1113 case nir_op_fnoise4_1:
1114 case nir_op_fnoise4_2:
1115 case nir_op_fnoise4_3:
1116 case nir_op_fnoise4_4:
1117 unreachable("not reached: should be handled by lower_noise");
1118
1119 case nir_op_ldexp:
1120 unreachable("not reached: should be handled by ldexp_to_arith()");
1121
1122 case nir_op_fsqrt:
1123 inst = bld.emit(SHADER_OPCODE_SQRT, result, op[0]);
1124 inst->saturate = instr->dest.saturate;
1125 break;
1126
1127 case nir_op_frsq:
1128 inst = bld.emit(SHADER_OPCODE_RSQ, result, op[0]);
1129 inst->saturate = instr->dest.saturate;
1130 break;
1131
1132 case nir_op_b2i:
1133 case nir_op_b2f:
1134 bld.MOV(result, negate(op[0]));
1135 break;
1136
1137 case nir_op_f2b:
1138 bld.CMP(result, op[0], brw_imm_f(0.0f), BRW_CONDITIONAL_NZ);
1139 break;
1140 case nir_op_d2b: {
1141 /* two-argument instructions can't take 64-bit immediates */
1142 fs_reg zero = vgrf(glsl_type::double_type);
1143 bld.MOV(zero, setup_imm_df(bld, 0.0));
1144 /* A SIMD16 execution needs to be split in two instructions, so use
1145 * a vgrf instead of the flag register as dst so instruction splitting
1146 * works
1147 */
1148 fs_reg tmp = vgrf(glsl_type::double_type);
1149 bld.CMP(tmp, op[0], zero, BRW_CONDITIONAL_NZ);
1150 bld.MOV(result, subscript(tmp, BRW_REGISTER_TYPE_UD, 0));
1151 break;
1152 }
1153 case nir_op_i2b:
1154 bld.CMP(result, op[0], brw_imm_d(0), BRW_CONDITIONAL_NZ);
1155 break;
1156
1157 case nir_op_ftrunc:
1158 inst = bld.RNDZ(result, op[0]);
1159 inst->saturate = instr->dest.saturate;
1160 break;
1161
1162 case nir_op_fceil: {
1163 op[0].negate = !op[0].negate;
1164 fs_reg temp = vgrf(glsl_type::float_type);
1165 bld.RNDD(temp, op[0]);
1166 temp.negate = true;
1167 inst = bld.MOV(result, temp);
1168 inst->saturate = instr->dest.saturate;
1169 break;
1170 }
1171 case nir_op_ffloor:
1172 inst = bld.RNDD(result, op[0]);
1173 inst->saturate = instr->dest.saturate;
1174 break;
1175 case nir_op_ffract:
1176 inst = bld.FRC(result, op[0]);
1177 inst->saturate = instr->dest.saturate;
1178 break;
1179 case nir_op_fround_even:
1180 inst = bld.RNDE(result, op[0]);
1181 inst->saturate = instr->dest.saturate;
1182 break;
1183
1184 case nir_op_fquantize2f16: {
1185 fs_reg tmp16 = bld.vgrf(BRW_REGISTER_TYPE_D);
1186 fs_reg tmp32 = bld.vgrf(BRW_REGISTER_TYPE_F);
1187 fs_reg zero = bld.vgrf(BRW_REGISTER_TYPE_F);
1188
1189 /* The destination stride must be at least as big as the source stride. */
1190 tmp16.type = BRW_REGISTER_TYPE_W;
1191 tmp16.stride = 2;
1192
1193 /* Check for denormal */
1194 fs_reg abs_src0 = op[0];
1195 abs_src0.abs = true;
1196 bld.CMP(bld.null_reg_f(), abs_src0, brw_imm_f(ldexpf(1.0, -14)),
1197 BRW_CONDITIONAL_L);
1198 /* Get the appropriately signed zero */
1199 bld.AND(retype(zero, BRW_REGISTER_TYPE_UD),
1200 retype(op[0], BRW_REGISTER_TYPE_UD),
1201 brw_imm_ud(0x80000000));
1202 /* Do the actual F32 -> F16 -> F32 conversion */
1203 bld.emit(BRW_OPCODE_F32TO16, tmp16, op[0]);
1204 bld.emit(BRW_OPCODE_F16TO32, tmp32, tmp16);
1205 /* Select that or zero based on normal status */
1206 inst = bld.SEL(result, zero, tmp32);
1207 inst->predicate = BRW_PREDICATE_NORMAL;
1208 inst->saturate = instr->dest.saturate;
1209 break;
1210 }
1211
1212 case nir_op_imin:
1213 case nir_op_umin:
1214 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1215 case nir_op_fmin:
1216 inst = bld.emit_minmax(result, op[0], op[1], BRW_CONDITIONAL_L);
1217 inst->saturate = instr->dest.saturate;
1218 break;
1219
1220 case nir_op_imax:
1221 case nir_op_umax:
1222 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1223 case nir_op_fmax:
1224 inst = bld.emit_minmax(result, op[0], op[1], BRW_CONDITIONAL_GE);
1225 inst->saturate = instr->dest.saturate;
1226 break;
1227
1228 case nir_op_pack_snorm_2x16:
1229 case nir_op_pack_snorm_4x8:
1230 case nir_op_pack_unorm_2x16:
1231 case nir_op_pack_unorm_4x8:
1232 case nir_op_unpack_snorm_2x16:
1233 case nir_op_unpack_snorm_4x8:
1234 case nir_op_unpack_unorm_2x16:
1235 case nir_op_unpack_unorm_4x8:
1236 case nir_op_unpack_half_2x16:
1237 case nir_op_pack_half_2x16:
1238 unreachable("not reached: should be handled by lower_packing_builtins");
1239
1240 case nir_op_unpack_half_2x16_split_x:
1241 inst = bld.emit(FS_OPCODE_UNPACK_HALF_2x16_SPLIT_X, result, op[0]);
1242 inst->saturate = instr->dest.saturate;
1243 break;
1244 case nir_op_unpack_half_2x16_split_y:
1245 inst = bld.emit(FS_OPCODE_UNPACK_HALF_2x16_SPLIT_Y, result, op[0]);
1246 inst->saturate = instr->dest.saturate;
1247 break;
1248
1249 case nir_op_pack_double_2x32_split:
1250 /* Optimize the common case where we are re-packing a double with
1251 * the result of a previous double unpack. In this case we can take the
1252 * 32-bit value to use in the re-pack from the original double and bypass
1253 * the unpack operation.
1254 */
1255 for (int i = 0; i < 2; i++) {
1256 if (instr->src[i].src.is_ssa)
1257 continue;
1258
1259 const nir_instr *parent_instr = instr->src[i].src.ssa->parent_instr;
1260 if (parent_instr->type == nir_instr_type_alu)
1261 continue;
1262
1263 const nir_alu_instr *alu_parent = nir_instr_as_alu(parent_instr);
1264 if (alu_parent->op == nir_op_unpack_double_2x32_split_x ||
1265 alu_parent->op == nir_op_unpack_double_2x32_split_y)
1266 continue;
1267
1268 if (!alu_parent->src[0].src.is_ssa)
1269 continue;
1270
1271 op[i] = get_nir_src(alu_parent->src[0].src);
1272 op[i] = offset(retype(op[i], BRW_REGISTER_TYPE_DF), bld,
1273 alu_parent->src[0].swizzle[channel]);
1274 if (alu_parent->op == nir_op_unpack_double_2x32_split_y)
1275 op[i] = subscript(op[i], BRW_REGISTER_TYPE_UD, 1);
1276 else
1277 op[i] = subscript(op[i], BRW_REGISTER_TYPE_UD, 0);
1278 }
1279 bld.emit(FS_OPCODE_PACK, result, op[0], op[1]);
1280 break;
1281
1282 case nir_op_unpack_double_2x32_split_x:
1283 case nir_op_unpack_double_2x32_split_y: {
1284 /* Optimize the common case where we are unpacking from a double we have
1285 * previously packed. In this case we can just bypass the pack operation
1286 * and source directly from its arguments.
1287 */
1288 unsigned index = (instr->op == nir_op_unpack_double_2x32_split_x) ? 0 : 1;
1289 if (instr->src[0].src.is_ssa) {
1290 nir_instr *parent_instr = instr->src[0].src.ssa->parent_instr;
1291 if (parent_instr->type == nir_instr_type_alu) {
1292 nir_alu_instr *alu_parent = nir_instr_as_alu(parent_instr);
1293 if (alu_parent->op == nir_op_pack_double_2x32_split &&
1294 alu_parent->src[index].src.is_ssa) {
1295 op[0] = retype(get_nir_src(alu_parent->src[index].src),
1296 BRW_REGISTER_TYPE_UD);
1297 op[0] =
1298 offset(op[0], bld, alu_parent->src[index].swizzle[channel]);
1299 bld.MOV(result, op[0]);
1300 break;
1301 }
1302 }
1303 }
1304
1305 if (instr->op == nir_op_unpack_double_2x32_split_x)
1306 bld.MOV(result, subscript(op[0], BRW_REGISTER_TYPE_UD, 0));
1307 else
1308 bld.MOV(result, subscript(op[0], BRW_REGISTER_TYPE_UD, 1));
1309 break;
1310 }
1311
1312 case nir_op_fpow:
1313 inst = bld.emit(SHADER_OPCODE_POW, result, op[0], op[1]);
1314 inst->saturate = instr->dest.saturate;
1315 break;
1316
1317 case nir_op_bitfield_reverse:
1318 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1319 bld.BFREV(result, op[0]);
1320 break;
1321
1322 case nir_op_bit_count:
1323 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1324 bld.CBIT(result, op[0]);
1325 break;
1326
1327 case nir_op_ufind_msb: {
1328 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1329 emit_find_msb_using_lzd(bld, result, op[0], false);
1330 break;
1331 }
1332
1333 case nir_op_ifind_msb: {
1334 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1335
1336 if (devinfo->gen < 7) {
1337 emit_find_msb_using_lzd(bld, result, op[0], true);
1338 } else {
1339 bld.FBH(retype(result, BRW_REGISTER_TYPE_UD), op[0]);
1340
1341 /* FBH counts from the MSB side, while GLSL's findMSB() wants the
1342 * count from the LSB side. If FBH didn't return an error
1343 * (0xFFFFFFFF), then subtract the result from 31 to convert the MSB
1344 * count into an LSB count.
1345 */
1346 bld.CMP(bld.null_reg_d(), result, brw_imm_d(-1), BRW_CONDITIONAL_NZ);
1347
1348 inst = bld.ADD(result, result, brw_imm_d(31));
1349 inst->predicate = BRW_PREDICATE_NORMAL;
1350 inst->src[0].negate = true;
1351 }
1352 break;
1353 }
1354
1355 case nir_op_find_lsb:
1356 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1357
1358 if (devinfo->gen < 7) {
1359 fs_reg temp = vgrf(glsl_type::int_type);
1360
1361 /* (x & -x) generates a value that consists of only the LSB of x.
1362 * For all powers of 2, findMSB(y) == findLSB(y).
1363 */
1364 fs_reg src = retype(op[0], BRW_REGISTER_TYPE_D);
1365 fs_reg negated_src = src;
1366
1367 /* One must be negated, and the other must be non-negated. It
1368 * doesn't matter which is which.
1369 */
1370 negated_src.negate = true;
1371 src.negate = false;
1372
1373 bld.AND(temp, src, negated_src);
1374 emit_find_msb_using_lzd(bld, result, temp, false);
1375 } else {
1376 bld.FBL(result, op[0]);
1377 }
1378 break;
1379
1380 case nir_op_ubitfield_extract:
1381 case nir_op_ibitfield_extract:
1382 unreachable("should have been lowered");
1383 case nir_op_ubfe:
1384 case nir_op_ibfe:
1385 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1386 bld.BFE(result, op[2], op[1], op[0]);
1387 break;
1388 case nir_op_bfm:
1389 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1390 bld.BFI1(result, op[0], op[1]);
1391 break;
1392 case nir_op_bfi:
1393 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1394 bld.BFI2(result, op[0], op[1], op[2]);
1395 break;
1396
1397 case nir_op_bitfield_insert:
1398 unreachable("not reached: should have been lowered");
1399
1400 case nir_op_ishl:
1401 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1402 bld.SHL(result, op[0], op[1]);
1403 break;
1404 case nir_op_ishr:
1405 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1406 bld.ASR(result, op[0], op[1]);
1407 break;
1408 case nir_op_ushr:
1409 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1410 bld.SHR(result, op[0], op[1]);
1411 break;
1412
1413 case nir_op_pack_half_2x16_split:
1414 bld.emit(FS_OPCODE_PACK_HALF_2x16_SPLIT, result, op[0], op[1]);
1415 break;
1416
1417 case nir_op_ffma:
1418 inst = bld.MAD(result, op[2], op[1], op[0]);
1419 inst->saturate = instr->dest.saturate;
1420 break;
1421
1422 case nir_op_flrp:
1423 inst = bld.LRP(result, op[0], op[1], op[2]);
1424 inst->saturate = instr->dest.saturate;
1425 break;
1426
1427 case nir_op_bcsel:
1428 if (optimize_frontfacing_ternary(instr, result))
1429 return;
1430
1431 bld.CMP(bld.null_reg_d(), op[0], brw_imm_d(0), BRW_CONDITIONAL_NZ);
1432 inst = bld.SEL(result, op[1], op[2]);
1433 inst->predicate = BRW_PREDICATE_NORMAL;
1434 break;
1435
1436 case nir_op_extract_u8:
1437 case nir_op_extract_i8: {
1438 const brw_reg_type type = brw_int_type(1, instr->op == nir_op_extract_i8);
1439 nir_const_value *byte = nir_src_as_const_value(instr->src[1].src);
1440 assert(byte != NULL);
1441 bld.MOV(result, subscript(op[0], type, byte->u32[0]));
1442 break;
1443 }
1444
1445 case nir_op_extract_u16:
1446 case nir_op_extract_i16: {
1447 const brw_reg_type type = brw_int_type(2, instr->op == nir_op_extract_i16);
1448 nir_const_value *word = nir_src_as_const_value(instr->src[1].src);
1449 assert(word != NULL);
1450 bld.MOV(result, subscript(op[0], type, word->u32[0]));
1451 break;
1452 }
1453
1454 default:
1455 unreachable("unhandled instruction");
1456 }
1457
1458 /* If we need to do a boolean resolve, replace the result with -(x & 1)
1459 * to sign extend the low bit to 0/~0
1460 */
1461 if (devinfo->gen <= 5 &&
1462 (instr->instr.pass_flags & BRW_NIR_BOOLEAN_MASK) == BRW_NIR_BOOLEAN_NEEDS_RESOLVE) {
1463 fs_reg masked = vgrf(glsl_type::int_type);
1464 bld.AND(masked, result, brw_imm_d(1));
1465 masked.negate = true;
1466 bld.MOV(retype(result, BRW_REGISTER_TYPE_D), masked);
1467 }
1468 }
1469
1470 void
1471 fs_visitor::nir_emit_load_const(const fs_builder &bld,
1472 nir_load_const_instr *instr)
1473 {
1474 const brw_reg_type reg_type =
1475 instr->def.bit_size == 32 ? BRW_REGISTER_TYPE_D : BRW_REGISTER_TYPE_DF;
1476 fs_reg reg = bld.vgrf(reg_type, instr->def.num_components);
1477
1478 switch (instr->def.bit_size) {
1479 case 32:
1480 for (unsigned i = 0; i < instr->def.num_components; i++)
1481 bld.MOV(offset(reg, bld, i), brw_imm_d(instr->value.i32[i]));
1482 break;
1483
1484 case 64:
1485 for (unsigned i = 0; i < instr->def.num_components; i++)
1486 bld.MOV(offset(reg, bld, i),
1487 setup_imm_df(bld, instr->value.f64[i]));
1488 break;
1489
1490 default:
1491 unreachable("Invalid bit size");
1492 }
1493
1494 nir_ssa_values[instr->def.index] = reg;
1495 }
1496
1497 void
1498 fs_visitor::nir_emit_undef(const fs_builder &bld, nir_ssa_undef_instr *instr)
1499 {
1500 const brw_reg_type reg_type =
1501 instr->def.bit_size == 32 ? BRW_REGISTER_TYPE_D : BRW_REGISTER_TYPE_DF;
1502 nir_ssa_values[instr->def.index] =
1503 bld.vgrf(reg_type, instr->def.num_components);
1504 }
1505
1506 fs_reg
1507 fs_visitor::get_nir_src(const nir_src &src)
1508 {
1509 fs_reg reg;
1510 if (src.is_ssa) {
1511 reg = nir_ssa_values[src.ssa->index];
1512 } else {
1513 /* We don't handle indirects on locals */
1514 assert(src.reg.indirect == NULL);
1515 reg = offset(nir_locals[src.reg.reg->index], bld,
1516 src.reg.base_offset * src.reg.reg->num_components);
1517 }
1518
1519 /* to avoid floating-point denorm flushing problems, set the type by
1520 * default to D - instructions that need floating point semantics will set
1521 * this to F if they need to
1522 */
1523 return retype(reg, BRW_REGISTER_TYPE_D);
1524 }
1525
1526 /**
1527 * Return an IMM for constants; otherwise call get_nir_src() as normal.
1528 */
1529 fs_reg
1530 fs_visitor::get_nir_src_imm(const nir_src &src)
1531 {
1532 nir_const_value *val = nir_src_as_const_value(src);
1533 return val ? fs_reg(brw_imm_d(val->i32[0])) : get_nir_src(src);
1534 }
1535
1536 fs_reg
1537 fs_visitor::get_nir_dest(const nir_dest &dest)
1538 {
1539 if (dest.is_ssa) {
1540 const brw_reg_type reg_type =
1541 dest.ssa.bit_size == 32 ? BRW_REGISTER_TYPE_F : BRW_REGISTER_TYPE_DF;
1542 nir_ssa_values[dest.ssa.index] =
1543 bld.vgrf(reg_type, dest.ssa.num_components);
1544 return nir_ssa_values[dest.ssa.index];
1545 } else {
1546 /* We don't handle indirects on locals */
1547 assert(dest.reg.indirect == NULL);
1548 return offset(nir_locals[dest.reg.reg->index], bld,
1549 dest.reg.base_offset * dest.reg.reg->num_components);
1550 }
1551 }
1552
1553 fs_reg
1554 fs_visitor::get_nir_image_deref(const nir_deref_var *deref)
1555 {
1556 fs_reg image(UNIFORM, deref->var->data.driver_location / 4,
1557 BRW_REGISTER_TYPE_UD);
1558 fs_reg indirect;
1559 unsigned indirect_max = 0;
1560
1561 for (const nir_deref *tail = &deref->deref; tail->child;
1562 tail = tail->child) {
1563 const nir_deref_array *deref_array = nir_deref_as_array(tail->child);
1564 assert(tail->child->deref_type == nir_deref_type_array);
1565 const unsigned size = glsl_get_length(tail->type);
1566 const unsigned element_size = type_size_scalar(deref_array->deref.type);
1567 const unsigned base = MIN2(deref_array->base_offset, size - 1);
1568 image = offset(image, bld, base * element_size);
1569
1570 if (deref_array->deref_array_type == nir_deref_array_type_indirect) {
1571 fs_reg tmp = vgrf(glsl_type::uint_type);
1572
1573 /* Accessing an invalid surface index with the dataport can result
1574 * in a hang. According to the spec "if the index used to
1575 * select an individual element is negative or greater than or
1576 * equal to the size of the array, the results of the operation
1577 * are undefined but may not lead to termination" -- which is one
1578 * of the possible outcomes of the hang. Clamp the index to
1579 * prevent access outside of the array bounds.
1580 */
1581 bld.emit_minmax(tmp, retype(get_nir_src(deref_array->indirect),
1582 BRW_REGISTER_TYPE_UD),
1583 brw_imm_ud(size - base - 1), BRW_CONDITIONAL_L);
1584
1585 indirect_max += element_size * (tail->type->length - 1);
1586
1587 bld.MUL(tmp, tmp, brw_imm_ud(element_size * 4));
1588 if (indirect.file == BAD_FILE) {
1589 indirect = tmp;
1590 } else {
1591 bld.ADD(indirect, indirect, tmp);
1592 }
1593 }
1594 }
1595
1596 if (indirect.file == BAD_FILE) {
1597 return image;
1598 } else {
1599 /* Emit a pile of MOVs to load the uniform into a temporary. The
1600 * dead-code elimination pass will get rid of what we don't use.
1601 */
1602 fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_UD, BRW_IMAGE_PARAM_SIZE);
1603 for (unsigned j = 0; j < BRW_IMAGE_PARAM_SIZE; j++) {
1604 bld.emit(SHADER_OPCODE_MOV_INDIRECT,
1605 offset(tmp, bld, j), offset(image, bld, j),
1606 indirect, brw_imm_ud((indirect_max + 1) * 4));
1607 }
1608 return tmp;
1609 }
1610 }
1611
1612 void
1613 fs_visitor::emit_percomp(const fs_builder &bld, const fs_inst &inst,
1614 unsigned wr_mask)
1615 {
1616 for (unsigned i = 0; i < 4; i++) {
1617 if (!((wr_mask >> i) & 1))
1618 continue;
1619
1620 fs_inst *new_inst = new(mem_ctx) fs_inst(inst);
1621 new_inst->dst = offset(new_inst->dst, bld, i);
1622 for (unsigned j = 0; j < new_inst->sources; j++)
1623 if (new_inst->src[j].file == VGRF)
1624 new_inst->src[j] = offset(new_inst->src[j], bld, i);
1625
1626 bld.emit(new_inst);
1627 }
1628 }
1629
1630 /**
1631 * Get the matching channel register datatype for an image intrinsic of the
1632 * specified GLSL image type.
1633 */
1634 static brw_reg_type
1635 get_image_base_type(const glsl_type *type)
1636 {
1637 switch ((glsl_base_type)type->sampled_type) {
1638 case GLSL_TYPE_UINT:
1639 return BRW_REGISTER_TYPE_UD;
1640 case GLSL_TYPE_INT:
1641 return BRW_REGISTER_TYPE_D;
1642 case GLSL_TYPE_FLOAT:
1643 return BRW_REGISTER_TYPE_F;
1644 default:
1645 unreachable("Not reached.");
1646 }
1647 }
1648
1649 /**
1650 * Get the appropriate atomic op for an image atomic intrinsic.
1651 */
1652 static unsigned
1653 get_image_atomic_op(nir_intrinsic_op op, const glsl_type *type)
1654 {
1655 switch (op) {
1656 case nir_intrinsic_image_atomic_add:
1657 return BRW_AOP_ADD;
1658 case nir_intrinsic_image_atomic_min:
1659 return (get_image_base_type(type) == BRW_REGISTER_TYPE_D ?
1660 BRW_AOP_IMIN : BRW_AOP_UMIN);
1661 case nir_intrinsic_image_atomic_max:
1662 return (get_image_base_type(type) == BRW_REGISTER_TYPE_D ?
1663 BRW_AOP_IMAX : BRW_AOP_UMAX);
1664 case nir_intrinsic_image_atomic_and:
1665 return BRW_AOP_AND;
1666 case nir_intrinsic_image_atomic_or:
1667 return BRW_AOP_OR;
1668 case nir_intrinsic_image_atomic_xor:
1669 return BRW_AOP_XOR;
1670 case nir_intrinsic_image_atomic_exchange:
1671 return BRW_AOP_MOV;
1672 case nir_intrinsic_image_atomic_comp_swap:
1673 return BRW_AOP_CMPWR;
1674 default:
1675 unreachable("Not reachable.");
1676 }
1677 }
1678
1679 static fs_inst *
1680 emit_pixel_interpolater_send(const fs_builder &bld,
1681 enum opcode opcode,
1682 const fs_reg &dst,
1683 const fs_reg &src,
1684 const fs_reg &desc,
1685 glsl_interp_mode interpolation)
1686 {
1687 struct brw_wm_prog_data *wm_prog_data =
1688 (struct brw_wm_prog_data *) bld.shader->stage_prog_data;
1689 fs_inst *inst;
1690 fs_reg payload;
1691 int mlen;
1692
1693 if (src.file == BAD_FILE) {
1694 /* Dummy payload */
1695 payload = bld.vgrf(BRW_REGISTER_TYPE_F, 1);
1696 mlen = 1;
1697 } else {
1698 payload = src;
1699 mlen = 2 * bld.dispatch_width() / 8;
1700 }
1701
1702 inst = bld.emit(opcode, dst, payload, desc);
1703 inst->mlen = mlen;
1704 /* 2 floats per slot returned */
1705 inst->regs_written = 2 * bld.dispatch_width() / 8;
1706 inst->pi_noperspective = interpolation == INTERP_MODE_NOPERSPECTIVE;
1707
1708 wm_prog_data->pulls_bary = true;
1709
1710 return inst;
1711 }
1712
1713 /**
1714 * Computes 1 << x, given a D/UD register containing some value x.
1715 */
1716 static fs_reg
1717 intexp2(const fs_builder &bld, const fs_reg &x)
1718 {
1719 assert(x.type == BRW_REGISTER_TYPE_UD || x.type == BRW_REGISTER_TYPE_D);
1720
1721 fs_reg result = bld.vgrf(x.type, 1);
1722 fs_reg one = bld.vgrf(x.type, 1);
1723
1724 bld.MOV(one, retype(brw_imm_d(1), one.type));
1725 bld.SHL(result, one, x);
1726 return result;
1727 }
1728
1729 void
1730 fs_visitor::emit_gs_end_primitive(const nir_src &vertex_count_nir_src)
1731 {
1732 assert(stage == MESA_SHADER_GEOMETRY);
1733
1734 struct brw_gs_prog_data *gs_prog_data =
1735 (struct brw_gs_prog_data *) prog_data;
1736
1737 if (gs_compile->control_data_header_size_bits == 0)
1738 return;
1739
1740 /* We can only do EndPrimitive() functionality when the control data
1741 * consists of cut bits. Fortunately, the only time it isn't is when the
1742 * output type is points, in which case EndPrimitive() is a no-op.
1743 */
1744 if (gs_prog_data->control_data_format !=
1745 GEN7_GS_CONTROL_DATA_FORMAT_GSCTL_CUT) {
1746 return;
1747 }
1748
1749 /* Cut bits use one bit per vertex. */
1750 assert(gs_compile->control_data_bits_per_vertex == 1);
1751
1752 fs_reg vertex_count = get_nir_src(vertex_count_nir_src);
1753 vertex_count.type = BRW_REGISTER_TYPE_UD;
1754
1755 /* Cut bit n should be set to 1 if EndPrimitive() was called after emitting
1756 * vertex n, 0 otherwise. So all we need to do here is mark bit
1757 * (vertex_count - 1) % 32 in the cut_bits register to indicate that
1758 * EndPrimitive() was called after emitting vertex (vertex_count - 1);
1759 * vec4_gs_visitor::emit_control_data_bits() will take care of the rest.
1760 *
1761 * Note that if EndPrimitive() is called before emitting any vertices, this
1762 * will cause us to set bit 31 of the control_data_bits register to 1.
1763 * That's fine because:
1764 *
1765 * - If max_vertices < 32, then vertex number 31 (zero-based) will never be
1766 * output, so the hardware will ignore cut bit 31.
1767 *
1768 * - If max_vertices == 32, then vertex number 31 is guaranteed to be the
1769 * last vertex, so setting cut bit 31 has no effect (since the primitive
1770 * is automatically ended when the GS terminates).
1771 *
1772 * - If max_vertices > 32, then the ir_emit_vertex visitor will reset the
1773 * control_data_bits register to 0 when the first vertex is emitted.
1774 */
1775
1776 const fs_builder abld = bld.annotate("end primitive");
1777
1778 /* control_data_bits |= 1 << ((vertex_count - 1) % 32) */
1779 fs_reg prev_count = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
1780 abld.ADD(prev_count, vertex_count, brw_imm_ud(0xffffffffu));
1781 fs_reg mask = intexp2(abld, prev_count);
1782 /* Note: we're relying on the fact that the GEN SHL instruction only pays
1783 * attention to the lower 5 bits of its second source argument, so on this
1784 * architecture, 1 << (vertex_count - 1) is equivalent to 1 <<
1785 * ((vertex_count - 1) % 32).
1786 */
1787 abld.OR(this->control_data_bits, this->control_data_bits, mask);
1788 }
1789
1790 void
1791 fs_visitor::emit_gs_control_data_bits(const fs_reg &vertex_count)
1792 {
1793 assert(stage == MESA_SHADER_GEOMETRY);
1794 assert(gs_compile->control_data_bits_per_vertex != 0);
1795
1796 struct brw_gs_prog_data *gs_prog_data =
1797 (struct brw_gs_prog_data *) prog_data;
1798
1799 const fs_builder abld = bld.annotate("emit control data bits");
1800 const fs_builder fwa_bld = bld.exec_all();
1801
1802 /* We use a single UD register to accumulate control data bits (32 bits
1803 * for each of the SIMD8 channels). So we need to write a DWord (32 bits)
1804 * at a time.
1805 *
1806 * Unfortunately, the URB_WRITE_SIMD8 message uses 128-bit (OWord) offsets.
1807 * We have select a 128-bit group via the Global and Per-Slot Offsets, then
1808 * use the Channel Mask phase to enable/disable which DWord within that
1809 * group to write. (Remember, different SIMD8 channels may have emitted
1810 * different numbers of vertices, so we may need per-slot offsets.)
1811 *
1812 * Channel masking presents an annoying problem: we may have to replicate
1813 * the data up to 4 times:
1814 *
1815 * Msg = Handles, Per-Slot Offsets, Channel Masks, Data, Data, Data, Data.
1816 *
1817 * To avoid penalizing shaders that emit a small number of vertices, we
1818 * can avoid these sometimes: if the size of the control data header is
1819 * <= 128 bits, then there is only 1 OWord. All SIMD8 channels will land
1820 * land in the same 128-bit group, so we can skip per-slot offsets.
1821 *
1822 * Similarly, if the control data header is <= 32 bits, there is only one
1823 * DWord, so we can skip channel masks.
1824 */
1825 enum opcode opcode = SHADER_OPCODE_URB_WRITE_SIMD8;
1826
1827 fs_reg channel_mask, per_slot_offset;
1828
1829 if (gs_compile->control_data_header_size_bits > 32) {
1830 opcode = SHADER_OPCODE_URB_WRITE_SIMD8_MASKED;
1831 channel_mask = vgrf(glsl_type::uint_type);
1832 }
1833
1834 if (gs_compile->control_data_header_size_bits > 128) {
1835 opcode = SHADER_OPCODE_URB_WRITE_SIMD8_MASKED_PER_SLOT;
1836 per_slot_offset = vgrf(glsl_type::uint_type);
1837 }
1838
1839 /* Figure out which DWord we're trying to write to using the formula:
1840 *
1841 * dword_index = (vertex_count - 1) * bits_per_vertex / 32
1842 *
1843 * Since bits_per_vertex is a power of two, and is known at compile
1844 * time, this can be optimized to:
1845 *
1846 * dword_index = (vertex_count - 1) >> (6 - log2(bits_per_vertex))
1847 */
1848 if (opcode != SHADER_OPCODE_URB_WRITE_SIMD8) {
1849 fs_reg dword_index = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
1850 fs_reg prev_count = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
1851 abld.ADD(prev_count, vertex_count, brw_imm_ud(0xffffffffu));
1852 unsigned log2_bits_per_vertex =
1853 _mesa_fls(gs_compile->control_data_bits_per_vertex);
1854 abld.SHR(dword_index, prev_count, brw_imm_ud(6u - log2_bits_per_vertex));
1855
1856 if (per_slot_offset.file != BAD_FILE) {
1857 /* Set the per-slot offset to dword_index / 4, so that we'll write to
1858 * the appropriate OWord within the control data header.
1859 */
1860 abld.SHR(per_slot_offset, dword_index, brw_imm_ud(2u));
1861 }
1862
1863 /* Set the channel masks to 1 << (dword_index % 4), so that we'll
1864 * write to the appropriate DWORD within the OWORD.
1865 */
1866 fs_reg channel = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
1867 fwa_bld.AND(channel, dword_index, brw_imm_ud(3u));
1868 channel_mask = intexp2(fwa_bld, channel);
1869 /* Then the channel masks need to be in bits 23:16. */
1870 fwa_bld.SHL(channel_mask, channel_mask, brw_imm_ud(16u));
1871 }
1872
1873 /* Store the control data bits in the message payload and send it. */
1874 int mlen = 2;
1875 if (channel_mask.file != BAD_FILE)
1876 mlen += 4; /* channel masks, plus 3 extra copies of the data */
1877 if (per_slot_offset.file != BAD_FILE)
1878 mlen++;
1879
1880 fs_reg payload = bld.vgrf(BRW_REGISTER_TYPE_UD, mlen);
1881 fs_reg *sources = ralloc_array(mem_ctx, fs_reg, mlen);
1882 int i = 0;
1883 sources[i++] = fs_reg(retype(brw_vec8_grf(1, 0), BRW_REGISTER_TYPE_UD));
1884 if (per_slot_offset.file != BAD_FILE)
1885 sources[i++] = per_slot_offset;
1886 if (channel_mask.file != BAD_FILE)
1887 sources[i++] = channel_mask;
1888 while (i < mlen) {
1889 sources[i++] = this->control_data_bits;
1890 }
1891
1892 abld.LOAD_PAYLOAD(payload, sources, mlen, mlen);
1893 fs_inst *inst = abld.emit(opcode, reg_undef, payload);
1894 inst->mlen = mlen;
1895 /* We need to increment Global Offset by 256-bits to make room for
1896 * Broadwell's extra "Vertex Count" payload at the beginning of the
1897 * URB entry. Since this is an OWord message, Global Offset is counted
1898 * in 128-bit units, so we must set it to 2.
1899 */
1900 if (gs_prog_data->static_vertex_count == -1)
1901 inst->offset = 2;
1902 }
1903
1904 void
1905 fs_visitor::set_gs_stream_control_data_bits(const fs_reg &vertex_count,
1906 unsigned stream_id)
1907 {
1908 /* control_data_bits |= stream_id << ((2 * (vertex_count - 1)) % 32) */
1909
1910 /* Note: we are calling this *before* increasing vertex_count, so
1911 * this->vertex_count == vertex_count - 1 in the formula above.
1912 */
1913
1914 /* Stream mode uses 2 bits per vertex */
1915 assert(gs_compile->control_data_bits_per_vertex == 2);
1916
1917 /* Must be a valid stream */
1918 assert(stream_id >= 0 && stream_id < MAX_VERTEX_STREAMS);
1919
1920 /* Control data bits are initialized to 0 so we don't have to set any
1921 * bits when sending vertices to stream 0.
1922 */
1923 if (stream_id == 0)
1924 return;
1925
1926 const fs_builder abld = bld.annotate("set stream control data bits", NULL);
1927
1928 /* reg::sid = stream_id */
1929 fs_reg sid = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
1930 abld.MOV(sid, brw_imm_ud(stream_id));
1931
1932 /* reg:shift_count = 2 * (vertex_count - 1) */
1933 fs_reg shift_count = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
1934 abld.SHL(shift_count, vertex_count, brw_imm_ud(1u));
1935
1936 /* Note: we're relying on the fact that the GEN SHL instruction only pays
1937 * attention to the lower 5 bits of its second source argument, so on this
1938 * architecture, stream_id << 2 * (vertex_count - 1) is equivalent to
1939 * stream_id << ((2 * (vertex_count - 1)) % 32).
1940 */
1941 fs_reg mask = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
1942 abld.SHL(mask, sid, shift_count);
1943 abld.OR(this->control_data_bits, this->control_data_bits, mask);
1944 }
1945
1946 void
1947 fs_visitor::emit_gs_vertex(const nir_src &vertex_count_nir_src,
1948 unsigned stream_id)
1949 {
1950 assert(stage == MESA_SHADER_GEOMETRY);
1951
1952 struct brw_gs_prog_data *gs_prog_data =
1953 (struct brw_gs_prog_data *) prog_data;
1954
1955 fs_reg vertex_count = get_nir_src(vertex_count_nir_src);
1956 vertex_count.type = BRW_REGISTER_TYPE_UD;
1957
1958 /* Haswell and later hardware ignores the "Render Stream Select" bits
1959 * from the 3DSTATE_STREAMOUT packet when the SOL stage is disabled,
1960 * and instead sends all primitives down the pipeline for rasterization.
1961 * If the SOL stage is enabled, "Render Stream Select" is honored and
1962 * primitives bound to non-zero streams are discarded after stream output.
1963 *
1964 * Since the only purpose of primives sent to non-zero streams is to
1965 * be recorded by transform feedback, we can simply discard all geometry
1966 * bound to these streams when transform feedback is disabled.
1967 */
1968 if (stream_id > 0 && !nir->info.has_transform_feedback_varyings)
1969 return;
1970
1971 /* If we're outputting 32 control data bits or less, then we can wait
1972 * until the shader is over to output them all. Otherwise we need to
1973 * output them as we go. Now is the time to do it, since we're about to
1974 * output the vertex_count'th vertex, so it's guaranteed that the
1975 * control data bits associated with the (vertex_count - 1)th vertex are
1976 * correct.
1977 */
1978 if (gs_compile->control_data_header_size_bits > 32) {
1979 const fs_builder abld =
1980 bld.annotate("emit vertex: emit control data bits");
1981
1982 /* Only emit control data bits if we've finished accumulating a batch
1983 * of 32 bits. This is the case when:
1984 *
1985 * (vertex_count * bits_per_vertex) % 32 == 0
1986 *
1987 * (in other words, when the last 5 bits of vertex_count *
1988 * bits_per_vertex are 0). Assuming bits_per_vertex == 2^n for some
1989 * integer n (which is always the case, since bits_per_vertex is
1990 * always 1 or 2), this is equivalent to requiring that the last 5-n
1991 * bits of vertex_count are 0:
1992 *
1993 * vertex_count & (2^(5-n) - 1) == 0
1994 *
1995 * 2^(5-n) == 2^5 / 2^n == 32 / bits_per_vertex, so this is
1996 * equivalent to:
1997 *
1998 * vertex_count & (32 / bits_per_vertex - 1) == 0
1999 *
2000 * TODO: If vertex_count is an immediate, we could do some of this math
2001 * at compile time...
2002 */
2003 fs_inst *inst =
2004 abld.AND(bld.null_reg_d(), vertex_count,
2005 brw_imm_ud(32u / gs_compile->control_data_bits_per_vertex - 1u));
2006 inst->conditional_mod = BRW_CONDITIONAL_Z;
2007
2008 abld.IF(BRW_PREDICATE_NORMAL);
2009 /* If vertex_count is 0, then no control data bits have been
2010 * accumulated yet, so we can skip emitting them.
2011 */
2012 abld.CMP(bld.null_reg_d(), vertex_count, brw_imm_ud(0u),
2013 BRW_CONDITIONAL_NEQ);
2014 abld.IF(BRW_PREDICATE_NORMAL);
2015 emit_gs_control_data_bits(vertex_count);
2016 abld.emit(BRW_OPCODE_ENDIF);
2017
2018 /* Reset control_data_bits to 0 so we can start accumulating a new
2019 * batch.
2020 *
2021 * Note: in the case where vertex_count == 0, this neutralizes the
2022 * effect of any call to EndPrimitive() that the shader may have
2023 * made before outputting its first vertex.
2024 */
2025 inst = abld.MOV(this->control_data_bits, brw_imm_ud(0u));
2026 inst->force_writemask_all = true;
2027 abld.emit(BRW_OPCODE_ENDIF);
2028 }
2029
2030 emit_urb_writes(vertex_count);
2031
2032 /* In stream mode we have to set control data bits for all vertices
2033 * unless we have disabled control data bits completely (which we do
2034 * do for GL_POINTS outputs that don't use streams).
2035 */
2036 if (gs_compile->control_data_header_size_bits > 0 &&
2037 gs_prog_data->control_data_format ==
2038 GEN7_GS_CONTROL_DATA_FORMAT_GSCTL_SID) {
2039 set_gs_stream_control_data_bits(vertex_count, stream_id);
2040 }
2041 }
2042
2043 void
2044 fs_visitor::emit_gs_input_load(const fs_reg &dst,
2045 const nir_src &vertex_src,
2046 unsigned base_offset,
2047 const nir_src &offset_src,
2048 unsigned num_components,
2049 unsigned first_component)
2050 {
2051 struct brw_gs_prog_data *gs_prog_data = (struct brw_gs_prog_data *) prog_data;
2052
2053 nir_const_value *vertex_const = nir_src_as_const_value(vertex_src);
2054 nir_const_value *offset_const = nir_src_as_const_value(offset_src);
2055 const unsigned push_reg_count = gs_prog_data->base.urb_read_length * 8;
2056
2057 /* Offset 0 is the VUE header, which contains VARYING_SLOT_LAYER [.y],
2058 * VARYING_SLOT_VIEWPORT [.z], and VARYING_SLOT_PSIZ [.w]. Only
2059 * gl_PointSize is available as a GS input, however, so it must be that.
2060 */
2061 const bool is_point_size = (base_offset == 0);
2062
2063 /* TODO: figure out push input layout for invocations == 1 */
2064 if (gs_prog_data->invocations == 1 &&
2065 offset_const != NULL && vertex_const != NULL &&
2066 4 * (base_offset + offset_const->u32[0]) < push_reg_count) {
2067 int imm_offset = (base_offset + offset_const->u32[0]) * 4 +
2068 vertex_const->u32[0] * push_reg_count;
2069 /* This input was pushed into registers. */
2070 if (is_point_size) {
2071 /* gl_PointSize comes in .w */
2072 bld.MOV(dst, fs_reg(ATTR, imm_offset + 3, dst.type));
2073 } else {
2074 for (unsigned i = 0; i < num_components; i++) {
2075 bld.MOV(offset(dst, bld, i),
2076 fs_reg(ATTR, imm_offset + i, dst.type));
2077 }
2078 }
2079 return;
2080 }
2081
2082 /* Resort to the pull model. Ensure the VUE handles are provided. */
2083 gs_prog_data->base.include_vue_handles = true;
2084
2085 unsigned first_icp_handle = gs_prog_data->include_primitive_id ? 3 : 2;
2086 fs_reg icp_handle = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2087
2088 if (gs_prog_data->invocations == 1) {
2089 if (vertex_const) {
2090 /* The vertex index is constant; just select the proper URB handle. */
2091 icp_handle =
2092 retype(brw_vec8_grf(first_icp_handle + vertex_const->i32[0], 0),
2093 BRW_REGISTER_TYPE_UD);
2094 } else {
2095 /* The vertex index is non-constant. We need to use indirect
2096 * addressing to fetch the proper URB handle.
2097 *
2098 * First, we start with the sequence <7, 6, 5, 4, 3, 2, 1, 0>
2099 * indicating that channel <n> should read the handle from
2100 * DWord <n>. We convert that to bytes by multiplying by 4.
2101 *
2102 * Next, we convert the vertex index to bytes by multiplying
2103 * by 32 (shifting by 5), and add the two together. This is
2104 * the final indirect byte offset.
2105 */
2106 fs_reg sequence = bld.vgrf(BRW_REGISTER_TYPE_W, 1);
2107 fs_reg channel_offsets = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2108 fs_reg vertex_offset_bytes = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2109 fs_reg icp_offset_bytes = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2110
2111 /* sequence = <7, 6, 5, 4, 3, 2, 1, 0> */
2112 bld.MOV(sequence, fs_reg(brw_imm_v(0x76543210)));
2113 /* channel_offsets = 4 * sequence = <28, 24, 20, 16, 12, 8, 4, 0> */
2114 bld.SHL(channel_offsets, sequence, brw_imm_ud(2u));
2115 /* Convert vertex_index to bytes (multiply by 32) */
2116 bld.SHL(vertex_offset_bytes,
2117 retype(get_nir_src(vertex_src), BRW_REGISTER_TYPE_UD),
2118 brw_imm_ud(5u));
2119 bld.ADD(icp_offset_bytes, vertex_offset_bytes, channel_offsets);
2120
2121 /* Use first_icp_handle as the base offset. There is one register
2122 * of URB handles per vertex, so inform the register allocator that
2123 * we might read up to nir->info.gs.vertices_in registers.
2124 */
2125 bld.emit(SHADER_OPCODE_MOV_INDIRECT, icp_handle,
2126 fs_reg(brw_vec8_grf(first_icp_handle, 0)),
2127 fs_reg(icp_offset_bytes),
2128 brw_imm_ud(nir->info.gs.vertices_in * REG_SIZE));
2129 }
2130 } else {
2131 assert(gs_prog_data->invocations > 1);
2132
2133 if (vertex_const) {
2134 assert(devinfo->gen >= 9 || vertex_const->i32[0] <= 5);
2135 bld.MOV(icp_handle,
2136 retype(brw_vec1_grf(first_icp_handle +
2137 vertex_const->i32[0] / 8,
2138 vertex_const->i32[0] % 8),
2139 BRW_REGISTER_TYPE_UD));
2140 } else {
2141 /* The vertex index is non-constant. We need to use indirect
2142 * addressing to fetch the proper URB handle.
2143 *
2144 */
2145 fs_reg icp_offset_bytes = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2146
2147 /* Convert vertex_index to bytes (multiply by 4) */
2148 bld.SHL(icp_offset_bytes,
2149 retype(get_nir_src(vertex_src), BRW_REGISTER_TYPE_UD),
2150 brw_imm_ud(2u));
2151
2152 /* Use first_icp_handle as the base offset. There is one DWord
2153 * of URB handles per vertex, so inform the register allocator that
2154 * we might read up to ceil(nir->info.gs.vertices_in / 8) registers.
2155 */
2156 bld.emit(SHADER_OPCODE_MOV_INDIRECT, icp_handle,
2157 fs_reg(brw_vec8_grf(first_icp_handle, 0)),
2158 fs_reg(icp_offset_bytes),
2159 brw_imm_ud(DIV_ROUND_UP(nir->info.gs.vertices_in, 8) *
2160 REG_SIZE));
2161 }
2162 }
2163
2164 fs_inst *inst;
2165
2166 fs_reg tmp_dst = dst;
2167 fs_reg indirect_offset = get_nir_src(offset_src);
2168 unsigned num_iterations = 1;
2169 unsigned orig_num_components = num_components;
2170
2171 if (type_sz(dst.type) == 8) {
2172 if (num_components > 2) {
2173 num_iterations = 2;
2174 num_components = 2;
2175 }
2176 fs_reg tmp = fs_reg(VGRF, alloc.allocate(4), dst.type);
2177 tmp_dst = tmp;
2178 first_component = first_component / 2;
2179 }
2180
2181 for (unsigned iter = 0; iter < num_iterations; iter++) {
2182 if (offset_const) {
2183 /* Constant indexing - use global offset. */
2184 if (first_component != 0) {
2185 unsigned read_components = num_components + first_component;
2186 fs_reg tmp = bld.vgrf(dst.type, read_components);
2187 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8, tmp, icp_handle);
2188 inst->regs_written = read_components * type_sz(tmp_dst.type) / 4;
2189 for (unsigned i = 0; i < num_components; i++) {
2190 bld.MOV(offset(tmp_dst, bld, i),
2191 offset(tmp, bld, i + first_component));
2192 }
2193 } else {
2194 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8, tmp_dst,
2195 icp_handle);
2196 inst->regs_written = num_components * type_sz(tmp_dst.type) / 4;
2197 }
2198 inst->offset = base_offset + offset_const->u32[0];
2199 inst->mlen = 1;
2200 } else {
2201 /* Indirect indexing - use per-slot offsets as well. */
2202 const fs_reg srcs[] = { icp_handle, indirect_offset };
2203 unsigned read_components = num_components + first_component;
2204 fs_reg tmp = bld.vgrf(dst.type, read_components);
2205 fs_reg payload = bld.vgrf(BRW_REGISTER_TYPE_UD, 2);
2206 bld.LOAD_PAYLOAD(payload, srcs, ARRAY_SIZE(srcs), 0);
2207 if (first_component != 0) {
2208 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT, tmp,
2209 payload);
2210 inst->regs_written = read_components * type_sz(tmp_dst.type) / 4;
2211 for (unsigned i = 0; i < num_components; i++) {
2212 bld.MOV(offset(tmp_dst, bld, i),
2213 offset(tmp, bld, i + first_component));
2214 }
2215 } else {
2216 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT, tmp_dst,
2217 payload);
2218 inst->regs_written = num_components * type_sz(tmp_dst.type) / 4;
2219 }
2220 inst->offset = base_offset;
2221 inst->mlen = 2;
2222 }
2223
2224 if (type_sz(dst.type) == 8) {
2225 shuffle_32bit_load_result_to_64bit_data(
2226 bld, tmp_dst, retype(tmp_dst, BRW_REGISTER_TYPE_F), num_components);
2227
2228 for (unsigned c = 0; c < num_components; c++)
2229 bld.MOV(offset(dst, bld, iter * 2 + c), offset(tmp_dst, bld, c));
2230 }
2231
2232 if (num_iterations > 1) {
2233 num_components = orig_num_components - 2;
2234 if(offset_const) {
2235 base_offset++;
2236 } else {
2237 fs_reg new_indirect = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2238 bld.ADD(new_indirect, indirect_offset, brw_imm_ud(1u));
2239 indirect_offset = new_indirect;
2240 }
2241 }
2242 }
2243
2244 if (is_point_size) {
2245 /* Read the whole VUE header (because of alignment) and read .w. */
2246 fs_reg tmp = bld.vgrf(dst.type, 4);
2247 inst->dst = tmp;
2248 inst->regs_written = 4;
2249 bld.MOV(dst, offset(tmp, bld, 3));
2250 }
2251 }
2252
2253 fs_reg
2254 fs_visitor::get_indirect_offset(nir_intrinsic_instr *instr)
2255 {
2256 nir_src *offset_src = nir_get_io_offset_src(instr);
2257 nir_const_value *const_value = nir_src_as_const_value(*offset_src);
2258
2259 if (const_value) {
2260 /* The only constant offset we should find is 0. brw_nir.c's
2261 * add_const_offset_to_base() will fold other constant offsets
2262 * into instr->const_index[0].
2263 */
2264 assert(const_value->u32[0] == 0);
2265 return fs_reg();
2266 }
2267
2268 return get_nir_src(*offset_src);
2269 }
2270
2271 static void
2272 do_untyped_vector_read(const fs_builder &bld,
2273 const fs_reg dest,
2274 const fs_reg surf_index,
2275 const fs_reg offset_reg,
2276 unsigned num_components)
2277 {
2278 if (type_sz(dest.type) == 4) {
2279 fs_reg read_result = emit_untyped_read(bld, surf_index, offset_reg,
2280 1 /* dims */,
2281 num_components,
2282 BRW_PREDICATE_NONE);
2283 read_result.type = dest.type;
2284 for (unsigned i = 0; i < num_components; i++)
2285 bld.MOV(offset(dest, bld, i), offset(read_result, bld, i));
2286 } else if (type_sz(dest.type) == 8) {
2287 /* Reading a dvec, so we need to:
2288 *
2289 * 1. Multiply num_components by 2, to account for the fact that we
2290 * need to read 64-bit components.
2291 * 2. Shuffle the result of the load to form valid 64-bit elements
2292 * 3. Emit a second load (for components z/w) if needed.
2293 */
2294 fs_reg read_offset = bld.vgrf(BRW_REGISTER_TYPE_UD);
2295 bld.MOV(read_offset, offset_reg);
2296
2297 int iters = num_components <= 2 ? 1 : 2;
2298
2299 /* Load the dvec, the first iteration loads components x/y, the second
2300 * iteration, if needed, loads components z/w
2301 */
2302 for (int it = 0; it < iters; it++) {
2303 /* Compute number of components to read in this iteration */
2304 int iter_components = MIN2(2, num_components);
2305 num_components -= iter_components;
2306
2307 /* Read. Since this message reads 32-bit components, we need to
2308 * read twice as many components.
2309 */
2310 fs_reg read_result = emit_untyped_read(bld, surf_index, read_offset,
2311 1 /* dims */,
2312 iter_components * 2,
2313 BRW_PREDICATE_NONE);
2314
2315 /* Shuffle the 32-bit load result into valid 64-bit data */
2316 const fs_reg packed_result = bld.vgrf(dest.type, iter_components);
2317 shuffle_32bit_load_result_to_64bit_data(
2318 bld, packed_result, read_result, iter_components);
2319
2320 /* Move each component to its destination */
2321 read_result = retype(read_result, BRW_REGISTER_TYPE_DF);
2322 for (int c = 0; c < iter_components; c++) {
2323 bld.MOV(offset(dest, bld, it * 2 + c),
2324 offset(packed_result, bld, c));
2325 }
2326
2327 bld.ADD(read_offset, read_offset, brw_imm_ud(16));
2328 }
2329 } else {
2330 unreachable("Unsupported type");
2331 }
2332 }
2333
2334 void
2335 fs_visitor::nir_emit_vs_intrinsic(const fs_builder &bld,
2336 nir_intrinsic_instr *instr)
2337 {
2338 assert(stage == MESA_SHADER_VERTEX);
2339
2340 fs_reg dest;
2341 if (nir_intrinsic_infos[instr->intrinsic].has_dest)
2342 dest = get_nir_dest(instr->dest);
2343
2344 switch (instr->intrinsic) {
2345 case nir_intrinsic_load_vertex_id:
2346 unreachable("should be lowered by lower_vertex_id()");
2347
2348 case nir_intrinsic_load_vertex_id_zero_base:
2349 case nir_intrinsic_load_base_vertex:
2350 case nir_intrinsic_load_instance_id:
2351 case nir_intrinsic_load_base_instance:
2352 case nir_intrinsic_load_draw_id: {
2353 gl_system_value sv = nir_system_value_from_intrinsic(instr->intrinsic);
2354 fs_reg val = nir_system_values[sv];
2355 assert(val.file != BAD_FILE);
2356 dest.type = val.type;
2357 bld.MOV(dest, val);
2358 break;
2359 }
2360
2361 default:
2362 nir_emit_intrinsic(bld, instr);
2363 break;
2364 }
2365 }
2366
2367 void
2368 fs_visitor::nir_emit_tcs_intrinsic(const fs_builder &bld,
2369 nir_intrinsic_instr *instr)
2370 {
2371 assert(stage == MESA_SHADER_TESS_CTRL);
2372 struct brw_tcs_prog_key *tcs_key = (struct brw_tcs_prog_key *) key;
2373 struct brw_tcs_prog_data *tcs_prog_data =
2374 (struct brw_tcs_prog_data *) prog_data;
2375
2376 fs_reg dst;
2377 if (nir_intrinsic_infos[instr->intrinsic].has_dest)
2378 dst = get_nir_dest(instr->dest);
2379
2380 switch (instr->intrinsic) {
2381 case nir_intrinsic_load_primitive_id:
2382 bld.MOV(dst, fs_reg(brw_vec1_grf(0, 1)));
2383 break;
2384 case nir_intrinsic_load_invocation_id:
2385 bld.MOV(retype(dst, invocation_id.type), invocation_id);
2386 break;
2387 case nir_intrinsic_load_patch_vertices_in:
2388 bld.MOV(retype(dst, BRW_REGISTER_TYPE_D),
2389 brw_imm_d(tcs_key->input_vertices));
2390 break;
2391
2392 case nir_intrinsic_barrier: {
2393 if (tcs_prog_data->instances == 1)
2394 break;
2395
2396 fs_reg m0 = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2397 fs_reg m0_2 = byte_offset(m0, 2 * sizeof(uint32_t));
2398
2399 const fs_builder fwa_bld = bld.exec_all();
2400
2401 /* Zero the message header */
2402 fwa_bld.MOV(m0, brw_imm_ud(0u));
2403
2404 /* Copy "Barrier ID" from r0.2, bits 16:13 */
2405 fwa_bld.AND(m0_2, retype(brw_vec1_grf(0, 2), BRW_REGISTER_TYPE_UD),
2406 brw_imm_ud(INTEL_MASK(16, 13)));
2407
2408 /* Shift it up to bits 27:24. */
2409 fwa_bld.SHL(m0_2, m0_2, brw_imm_ud(11));
2410
2411 /* Set the Barrier Count and the enable bit */
2412 fwa_bld.OR(m0_2, m0_2,
2413 brw_imm_ud(tcs_prog_data->instances << 8 | (1 << 15)));
2414
2415 bld.emit(SHADER_OPCODE_BARRIER, bld.null_reg_ud(), m0);
2416 break;
2417 }
2418
2419 case nir_intrinsic_load_input:
2420 unreachable("nir_lower_io should never give us these.");
2421 break;
2422
2423 case nir_intrinsic_load_per_vertex_input: {
2424 fs_reg indirect_offset = get_indirect_offset(instr);
2425 unsigned imm_offset = instr->const_index[0];
2426
2427 const nir_src &vertex_src = instr->src[0];
2428 nir_const_value *vertex_const = nir_src_as_const_value(vertex_src);
2429
2430 fs_inst *inst;
2431
2432 fs_reg icp_handle;
2433
2434 if (vertex_const) {
2435 /* Emit a MOV to resolve <0,1,0> regioning. */
2436 icp_handle = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2437 bld.MOV(icp_handle,
2438 retype(brw_vec1_grf(1 + (vertex_const->i32[0] >> 3),
2439 vertex_const->i32[0] & 7),
2440 BRW_REGISTER_TYPE_UD));
2441 } else if (tcs_prog_data->instances == 1 &&
2442 vertex_src.is_ssa &&
2443 vertex_src.ssa->parent_instr->type == nir_instr_type_intrinsic &&
2444 nir_instr_as_intrinsic(vertex_src.ssa->parent_instr)->intrinsic == nir_intrinsic_load_invocation_id) {
2445 /* For the common case of only 1 instance, an array index of
2446 * gl_InvocationID means reading g1. Skip all the indirect work.
2447 */
2448 icp_handle = retype(brw_vec8_grf(1, 0), BRW_REGISTER_TYPE_UD);
2449 } else {
2450 /* The vertex index is non-constant. We need to use indirect
2451 * addressing to fetch the proper URB handle.
2452 */
2453 icp_handle = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2454
2455 /* Each ICP handle is a single DWord (4 bytes) */
2456 fs_reg vertex_offset_bytes = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2457 bld.SHL(vertex_offset_bytes,
2458 retype(get_nir_src(vertex_src), BRW_REGISTER_TYPE_UD),
2459 brw_imm_ud(2u));
2460
2461 /* Start at g1. We might read up to 4 registers. */
2462 bld.emit(SHADER_OPCODE_MOV_INDIRECT, icp_handle,
2463 fs_reg(brw_vec8_grf(1, 0)), vertex_offset_bytes,
2464 brw_imm_ud(4 * REG_SIZE));
2465 }
2466
2467 /* We can only read two double components with each URB read, so
2468 * we send two read messages in that case, each one loading up to
2469 * two double components.
2470 */
2471 unsigned num_iterations = 1;
2472 unsigned num_components = instr->num_components;
2473 unsigned first_component = nir_intrinsic_component(instr);
2474 fs_reg orig_dst = dst;
2475 if (type_sz(dst.type) == 8) {
2476 first_component = first_component / 2;
2477 if (instr->num_components > 2) {
2478 num_iterations = 2;
2479 num_components = 2;
2480 }
2481
2482 fs_reg tmp = fs_reg(VGRF, alloc.allocate(4), dst.type);
2483 dst = tmp;
2484 }
2485
2486 for (unsigned iter = 0; iter < num_iterations; iter++) {
2487 if (indirect_offset.file == BAD_FILE) {
2488 /* Constant indexing - use global offset. */
2489 if (first_component != 0) {
2490 unsigned read_components = num_components + first_component;
2491 fs_reg tmp = bld.vgrf(dst.type, read_components);
2492 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8, tmp, icp_handle);
2493 for (unsigned i = 0; i < num_components; i++) {
2494 bld.MOV(offset(dst, bld, i),
2495 offset(tmp, bld, i + first_component));
2496 }
2497 } else {
2498 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8, dst, icp_handle);
2499 }
2500 inst->offset = imm_offset;
2501 inst->mlen = 1;
2502 } else {
2503 /* Indirect indexing - use per-slot offsets as well. */
2504 const fs_reg srcs[] = { icp_handle, indirect_offset };
2505 fs_reg payload = bld.vgrf(BRW_REGISTER_TYPE_UD, 2);
2506 bld.LOAD_PAYLOAD(payload, srcs, ARRAY_SIZE(srcs), 0);
2507 if (first_component != 0) {
2508 unsigned read_components = num_components + first_component;
2509 fs_reg tmp = bld.vgrf(dst.type, read_components);
2510 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT, tmp,
2511 payload);
2512 for (unsigned i = 0; i < num_components; i++) {
2513 bld.MOV(offset(dst, bld, i),
2514 offset(tmp, bld, i + first_component));
2515 }
2516 } else {
2517 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT, dst,
2518 payload);
2519 }
2520 inst->offset = imm_offset;
2521 inst->mlen = 2;
2522 }
2523 inst->regs_written =
2524 ((num_components + first_component) * type_sz(dst.type) / 4);
2525
2526 /* If we are reading 64-bit data using 32-bit read messages we need
2527 * build proper 64-bit data elements by shuffling the low and high
2528 * 32-bit components around like we do for other things like UBOs
2529 * or SSBOs.
2530 */
2531 if (type_sz(dst.type) == 8) {
2532 shuffle_32bit_load_result_to_64bit_data(
2533 bld, dst, retype(dst, BRW_REGISTER_TYPE_F), num_components);
2534
2535 for (unsigned c = 0; c < num_components; c++) {
2536 bld.MOV(offset(orig_dst, bld, iter * 2 + c),
2537 offset(dst, bld, c));
2538 }
2539 }
2540
2541 /* Copy the temporary to the destination to deal with writemasking.
2542 *
2543 * Also attempt to deal with gl_PointSize being in the .w component.
2544 */
2545 if (inst->offset == 0 && indirect_offset.file == BAD_FILE) {
2546 assert(type_sz(dst.type) < 8);
2547 inst->dst = bld.vgrf(dst.type, 4);
2548 inst->regs_written = 4;
2549 bld.MOV(dst, offset(inst->dst, bld, 3));
2550 }
2551
2552 /* If we are loading double data and we need a second read message
2553 * adjust the write offset
2554 */
2555 if (num_iterations > 1) {
2556 num_components = instr->num_components - 2;
2557 imm_offset++;
2558 }
2559 }
2560 break;
2561 }
2562
2563 case nir_intrinsic_load_output:
2564 case nir_intrinsic_load_per_vertex_output: {
2565 fs_reg indirect_offset = get_indirect_offset(instr);
2566 unsigned imm_offset = instr->const_index[0];
2567
2568 fs_inst *inst;
2569 if (indirect_offset.file == BAD_FILE) {
2570 /* Replicate the patch handle to all enabled channels */
2571 fs_reg patch_handle = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2572 bld.MOV(patch_handle,
2573 retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_UD));
2574
2575 if (imm_offset == 0) {
2576 /* This is a read of gl_TessLevelInner[], which lives in the
2577 * Patch URB header. The layout depends on the domain.
2578 */
2579 dst.type = BRW_REGISTER_TYPE_F;
2580 switch (tcs_key->tes_primitive_mode) {
2581 case GL_QUADS: {
2582 /* DWords 3-2 (reversed) */
2583 fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_F, 4);
2584
2585 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8, tmp, patch_handle);
2586 inst->offset = 0;
2587 inst->mlen = 1;
2588 inst->regs_written = 4;
2589
2590 /* dst.xy = tmp.wz */
2591 bld.MOV(dst, offset(tmp, bld, 3));
2592 bld.MOV(offset(dst, bld, 1), offset(tmp, bld, 2));
2593 break;
2594 }
2595 case GL_TRIANGLES:
2596 /* DWord 4; hardcode offset = 1 and regs_written = 1 */
2597 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8, dst, patch_handle);
2598 inst->offset = 1;
2599 inst->mlen = 1;
2600 inst->regs_written = 1;
2601 break;
2602 case GL_ISOLINES:
2603 /* All channels are undefined. */
2604 break;
2605 default:
2606 unreachable("Bogus tessellation domain");
2607 }
2608 } else if (imm_offset == 1) {
2609 /* This is a read of gl_TessLevelOuter[], which lives in the
2610 * Patch URB header. The layout depends on the domain.
2611 */
2612 dst.type = BRW_REGISTER_TYPE_F;
2613
2614 fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_F, 4);
2615 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8, tmp, patch_handle);
2616 inst->offset = 1;
2617 inst->mlen = 1;
2618 inst->regs_written = 4;
2619
2620 /* Reswizzle: WZYX */
2621 fs_reg srcs[4] = {
2622 offset(tmp, bld, 3),
2623 offset(tmp, bld, 2),
2624 offset(tmp, bld, 1),
2625 offset(tmp, bld, 0),
2626 };
2627
2628 unsigned num_components;
2629 switch (tcs_key->tes_primitive_mode) {
2630 case GL_QUADS:
2631 num_components = 4;
2632 break;
2633 case GL_TRIANGLES:
2634 num_components = 3;
2635 break;
2636 case GL_ISOLINES:
2637 /* Isolines are not reversed; swizzle .zw -> .xy */
2638 srcs[0] = offset(tmp, bld, 2);
2639 srcs[1] = offset(tmp, bld, 3);
2640 num_components = 2;
2641 break;
2642 default:
2643 unreachable("Bogus tessellation domain");
2644 }
2645 bld.LOAD_PAYLOAD(dst, srcs, num_components, 0);
2646 } else {
2647 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8, dst, patch_handle);
2648 inst->offset = imm_offset;
2649 inst->mlen = 1;
2650 inst->regs_written = instr->num_components;
2651 }
2652 } else {
2653 /* Indirect indexing - use per-slot offsets as well. */
2654 const fs_reg srcs[] = {
2655 retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_UD),
2656 indirect_offset
2657 };
2658 fs_reg payload = bld.vgrf(BRW_REGISTER_TYPE_UD, 2);
2659 bld.LOAD_PAYLOAD(payload, srcs, ARRAY_SIZE(srcs), 0);
2660
2661 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT, dst, payload);
2662 inst->offset = imm_offset;
2663 inst->mlen = 2;
2664 inst->regs_written = instr->num_components;
2665 }
2666 break;
2667 }
2668
2669 case nir_intrinsic_store_output:
2670 case nir_intrinsic_store_per_vertex_output: {
2671 fs_reg value = get_nir_src(instr->src[0]);
2672 bool is_64bit = (instr->src[0].is_ssa ?
2673 instr->src[0].ssa->bit_size : instr->src[0].reg.reg->bit_size) == 64;
2674 fs_reg indirect_offset = get_indirect_offset(instr);
2675 unsigned imm_offset = instr->const_index[0];
2676 unsigned swiz = BRW_SWIZZLE_XYZW;
2677 unsigned mask = instr->const_index[1];
2678 unsigned header_regs = 0;
2679 fs_reg srcs[7];
2680 srcs[header_regs++] = retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_UD);
2681
2682 if (indirect_offset.file != BAD_FILE) {
2683 srcs[header_regs++] = indirect_offset;
2684 } else if (!is_passthrough_shader) {
2685 if (imm_offset == 0) {
2686 value.type = BRW_REGISTER_TYPE_F;
2687
2688 mask &= (1 << tesslevel_inner_components(tcs_key->tes_primitive_mode)) - 1;
2689
2690 /* This is a write to gl_TessLevelInner[], which lives in the
2691 * Patch URB header. The layout depends on the domain.
2692 */
2693 switch (tcs_key->tes_primitive_mode) {
2694 case GL_QUADS:
2695 /* gl_TessLevelInner[].xy lives at DWords 3-2 (reversed).
2696 * We use an XXYX swizzle to reverse put .xy in the .wz
2697 * channels, and use a .zw writemask.
2698 */
2699 mask = writemask_for_backwards_vector(mask);
2700 swiz = BRW_SWIZZLE4(0, 0, 1, 0);
2701 break;
2702 case GL_TRIANGLES:
2703 /* gl_TessLevelInner[].x lives at DWord 4, so we set the
2704 * writemask to X and bump the URB offset by 1.
2705 */
2706 imm_offset = 1;
2707 break;
2708 case GL_ISOLINES:
2709 /* Skip; gl_TessLevelInner[] doesn't exist for isolines. */
2710 return;
2711 default:
2712 unreachable("Bogus tessellation domain");
2713 }
2714 } else if (imm_offset == 1) {
2715 /* This is a write to gl_TessLevelOuter[] which lives in the
2716 * Patch URB Header at DWords 4-7. However, it's reversed, so
2717 * instead of .xyzw we have .wzyx.
2718 */
2719 value.type = BRW_REGISTER_TYPE_F;
2720
2721 mask &= (1 << tesslevel_outer_components(tcs_key->tes_primitive_mode)) - 1;
2722
2723 if (tcs_key->tes_primitive_mode == GL_ISOLINES) {
2724 /* Isolines .xy should be stored in .zw, in order. */
2725 swiz = BRW_SWIZZLE4(0, 0, 0, 1);
2726 mask <<= 2;
2727 } else {
2728 /* Other domains are reversed; store .wzyx instead of .xyzw */
2729 swiz = BRW_SWIZZLE_WZYX;
2730 mask = writemask_for_backwards_vector(mask);
2731 }
2732 }
2733 }
2734
2735 if (mask == 0)
2736 break;
2737
2738 unsigned num_components = _mesa_fls(mask);
2739 enum opcode opcode;
2740
2741 /* We can only pack two 64-bit components in a single message, so send
2742 * 2 messages if we have more components
2743 */
2744 unsigned num_iterations = 1;
2745 unsigned iter_components = num_components;
2746 unsigned first_component = nir_intrinsic_component(instr);
2747 if (is_64bit) {
2748 first_component = first_component / 2;
2749 if (instr->num_components > 2) {
2750 num_iterations = 2;
2751 iter_components = 2;
2752 }
2753 }
2754
2755 /* 64-bit data needs to me shuffled before we can write it to the URB.
2756 * We will use this temporary to shuffle the components in each
2757 * iteration.
2758 */
2759 fs_reg tmp =
2760 fs_reg(VGRF, alloc.allocate(2 * iter_components), value.type);
2761
2762 mask = mask << first_component;
2763
2764 for (unsigned iter = 0; iter < num_iterations; iter++) {
2765 if (!is_64bit && mask != WRITEMASK_XYZW) {
2766 srcs[header_regs++] = brw_imm_ud(mask << 16);
2767 opcode = indirect_offset.file != BAD_FILE ?
2768 SHADER_OPCODE_URB_WRITE_SIMD8_MASKED_PER_SLOT :
2769 SHADER_OPCODE_URB_WRITE_SIMD8_MASKED;
2770 } else if (is_64bit && ((mask & WRITEMASK_XY) != WRITEMASK_XY)) {
2771 /* Expand the 64-bit mask to 32-bit channels. We only handle
2772 * two channels in each iteration, so we only care about X/Y.
2773 */
2774 unsigned mask32 = 0;
2775 if (mask & WRITEMASK_X)
2776 mask32 |= WRITEMASK_XY;
2777 if (mask & WRITEMASK_Y)
2778 mask32 |= WRITEMASK_ZW;
2779
2780 /* If the mask does not include any of the channels X or Y there
2781 * is nothing to do in this iteration. Move on to the next couple
2782 * of 64-bit channels.
2783 */
2784 if (!mask32) {
2785 mask >>= 2;
2786 imm_offset++;
2787 continue;
2788 }
2789
2790 srcs[header_regs++] = brw_imm_ud(mask32 << 16);
2791 opcode = indirect_offset.file != BAD_FILE ?
2792 SHADER_OPCODE_URB_WRITE_SIMD8_MASKED_PER_SLOT :
2793 SHADER_OPCODE_URB_WRITE_SIMD8_MASKED;
2794 } else {
2795 opcode = indirect_offset.file != BAD_FILE ?
2796 SHADER_OPCODE_URB_WRITE_SIMD8_PER_SLOT :
2797 SHADER_OPCODE_URB_WRITE_SIMD8;
2798 }
2799
2800 for (unsigned i = 0; i < iter_components; i++) {
2801 if (!(mask & (1 << (i + first_component))))
2802 continue;
2803
2804 if (!is_64bit) {
2805 srcs[header_regs + i + first_component] =
2806 offset(value, bld, BRW_GET_SWZ(swiz, i));
2807 } else {
2808 /* We need to shuffle the 64-bit data to match the layout
2809 * expected by our 32-bit URB write messages. We use a temporary
2810 * for that.
2811 */
2812 unsigned channel = BRW_GET_SWZ(swiz, iter * 2 + i);
2813 shuffle_64bit_data_for_32bit_write(bld,
2814 retype(offset(tmp, bld, 2 * i), BRW_REGISTER_TYPE_F),
2815 retype(offset(value, bld, 2 * channel), BRW_REGISTER_TYPE_DF),
2816 1);
2817
2818 /* Now copy the data to the destination */
2819 fs_reg dest = fs_reg(VGRF, alloc.allocate(2), value.type);
2820 unsigned idx = 2 * i;
2821 bld.MOV(dest, offset(tmp, bld, idx));
2822 bld.MOV(offset(dest, bld, 1), offset(tmp, bld, idx + 1));
2823 srcs[header_regs + idx + first_component * 2] = dest;
2824 srcs[header_regs + idx + 1 + first_component * 2] =
2825 offset(dest, bld, 1);
2826 }
2827 }
2828
2829 unsigned mlen =
2830 header_regs + (is_64bit ? 2 * iter_components : iter_components) +
2831 (is_64bit ? 2 * first_component : first_component);
2832 fs_reg payload =
2833 bld.vgrf(BRW_REGISTER_TYPE_UD, mlen);
2834 bld.LOAD_PAYLOAD(payload, srcs, mlen, header_regs);
2835
2836 fs_inst *inst = bld.emit(opcode, bld.null_reg_ud(), payload);
2837 inst->offset = imm_offset;
2838 inst->mlen = mlen;
2839
2840 /* If this is a 64-bit attribute, select the next two 64-bit channels
2841 * to be handled in the next iteration.
2842 */
2843 if (is_64bit) {
2844 mask >>= 2;
2845 imm_offset++;
2846 }
2847 }
2848 break;
2849 }
2850
2851 default:
2852 nir_emit_intrinsic(bld, instr);
2853 break;
2854 }
2855 }
2856
2857 void
2858 fs_visitor::nir_emit_tes_intrinsic(const fs_builder &bld,
2859 nir_intrinsic_instr *instr)
2860 {
2861 assert(stage == MESA_SHADER_TESS_EVAL);
2862 struct brw_tes_prog_data *tes_prog_data = (struct brw_tes_prog_data *) prog_data;
2863
2864 fs_reg dest;
2865 if (nir_intrinsic_infos[instr->intrinsic].has_dest)
2866 dest = get_nir_dest(instr->dest);
2867
2868 switch (instr->intrinsic) {
2869 case nir_intrinsic_load_primitive_id:
2870 bld.MOV(dest, fs_reg(brw_vec1_grf(0, 1)));
2871 break;
2872 case nir_intrinsic_load_tess_coord:
2873 /* gl_TessCoord is part of the payload in g1-3 */
2874 for (unsigned i = 0; i < 3; i++) {
2875 bld.MOV(offset(dest, bld, i), fs_reg(brw_vec8_grf(1 + i, 0)));
2876 }
2877 break;
2878
2879 case nir_intrinsic_load_tess_level_outer:
2880 /* When the TES reads gl_TessLevelOuter, we ensure that the patch header
2881 * appears as a push-model input. So, we can simply use the ATTR file
2882 * rather than issuing URB read messages. The data is stored in the
2883 * high DWords in reverse order - DWord 7 contains .x, DWord 6 contains
2884 * .y, and so on.
2885 */
2886 switch (tes_prog_data->domain) {
2887 case BRW_TESS_DOMAIN_QUAD:
2888 for (unsigned i = 0; i < 4; i++)
2889 bld.MOV(offset(dest, bld, i), component(fs_reg(ATTR, 0), 7 - i));
2890 break;
2891 case BRW_TESS_DOMAIN_TRI:
2892 for (unsigned i = 0; i < 3; i++)
2893 bld.MOV(offset(dest, bld, i), component(fs_reg(ATTR, 0), 7 - i));
2894 break;
2895 case BRW_TESS_DOMAIN_ISOLINE:
2896 for (unsigned i = 0; i < 2; i++)
2897 bld.MOV(offset(dest, bld, i), component(fs_reg(ATTR, 0), 6 + i));
2898 break;
2899 }
2900 break;
2901
2902 case nir_intrinsic_load_tess_level_inner:
2903 /* When the TES reads gl_TessLevelInner, we ensure that the patch header
2904 * appears as a push-model input. So, we can simply use the ATTR file
2905 * rather than issuing URB read messages.
2906 */
2907 switch (tes_prog_data->domain) {
2908 case BRW_TESS_DOMAIN_QUAD:
2909 bld.MOV(dest, component(fs_reg(ATTR, 0), 3));
2910 bld.MOV(offset(dest, bld, 1), component(fs_reg(ATTR, 0), 2));
2911 break;
2912 case BRW_TESS_DOMAIN_TRI:
2913 bld.MOV(dest, component(fs_reg(ATTR, 0), 4));
2914 break;
2915 case BRW_TESS_DOMAIN_ISOLINE:
2916 /* ignore - value is undefined */
2917 break;
2918 }
2919 break;
2920
2921 case nir_intrinsic_load_input:
2922 case nir_intrinsic_load_per_vertex_input: {
2923 fs_reg indirect_offset = get_indirect_offset(instr);
2924 unsigned imm_offset = instr->const_index[0];
2925 unsigned first_component = nir_intrinsic_component(instr);
2926
2927 if (type_sz(dest.type) == 8) {
2928 first_component = first_component / 2;
2929 }
2930
2931 fs_inst *inst;
2932 if (indirect_offset.file == BAD_FILE) {
2933 /* Arbitrarily only push up to 32 vec4 slots worth of data,
2934 * which is 16 registers (since each holds 2 vec4 slots).
2935 */
2936 const unsigned max_push_slots = 32;
2937 if (imm_offset < max_push_slots) {
2938 fs_reg src = fs_reg(ATTR, imm_offset / 2, dest.type);
2939 for (int i = 0; i < instr->num_components; i++) {
2940 unsigned comp = 16 / type_sz(dest.type) * (imm_offset % 2) +
2941 i + first_component;
2942 bld.MOV(offset(dest, bld, i), component(src, comp));
2943 }
2944 tes_prog_data->base.urb_read_length =
2945 MAX2(tes_prog_data->base.urb_read_length,
2946 DIV_ROUND_UP(imm_offset + 1, 2));
2947 } else {
2948 /* Replicate the patch handle to all enabled channels */
2949 const fs_reg srcs[] = {
2950 retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_UD)
2951 };
2952 fs_reg patch_handle = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2953 bld.LOAD_PAYLOAD(patch_handle, srcs, ARRAY_SIZE(srcs), 0);
2954
2955 if (first_component != 0) {
2956 unsigned read_components =
2957 instr->num_components + first_component;
2958 fs_reg tmp = bld.vgrf(dest.type, read_components);
2959 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8, tmp,
2960 patch_handle);
2961 inst->regs_written = read_components;
2962 for (unsigned i = 0; i < instr->num_components; i++) {
2963 bld.MOV(offset(dest, bld, i),
2964 offset(tmp, bld, i + first_component));
2965 }
2966 } else {
2967 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8, dest,
2968 patch_handle);
2969 inst->regs_written = instr->num_components;
2970 }
2971 inst->mlen = 1;
2972 inst->offset = imm_offset;
2973 }
2974 } else {
2975 /* Indirect indexing - use per-slot offsets as well. */
2976
2977 /* We can only read two double components with each URB read, so
2978 * we send two read messages in that case, each one loading up to
2979 * two double components.
2980 */
2981 unsigned num_iterations = 1;
2982 unsigned num_components = instr->num_components;
2983 fs_reg orig_dest = dest;
2984 if (type_sz(dest.type) == 8) {
2985 if (instr->num_components > 2) {
2986 num_iterations = 2;
2987 num_components = 2;
2988 }
2989 fs_reg tmp = fs_reg(VGRF, alloc.allocate(4), dest.type);
2990 dest = tmp;
2991 }
2992
2993 for (unsigned iter = 0; iter < num_iterations; iter++) {
2994 const fs_reg srcs[] = {
2995 retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_UD),
2996 indirect_offset
2997 };
2998 fs_reg payload = bld.vgrf(BRW_REGISTER_TYPE_UD, 2);
2999 bld.LOAD_PAYLOAD(payload, srcs, ARRAY_SIZE(srcs), 0);
3000
3001 if (first_component != 0) {
3002 unsigned read_components =
3003 num_components + first_component;
3004 fs_reg tmp = bld.vgrf(dest.type, read_components);
3005 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT, tmp,
3006 payload);
3007 for (unsigned i = 0; i < num_components; i++) {
3008 bld.MOV(offset(dest, bld, i),
3009 offset(tmp, bld, i + first_component));
3010 }
3011 } else {
3012 inst = bld.emit(SHADER_OPCODE_URB_READ_SIMD8_PER_SLOT, dest,
3013 payload);
3014 }
3015 inst->mlen = 2;
3016 inst->offset = imm_offset;
3017 inst->regs_written =
3018 ((num_components + first_component) * type_sz(dest.type) / 4);
3019
3020 /* If we are reading 64-bit data using 32-bit read messages we need
3021 * build proper 64-bit data elements by shuffling the low and high
3022 * 32-bit components around like we do for other things like UBOs
3023 * or SSBOs.
3024 */
3025 if (type_sz(dest.type) == 8) {
3026 shuffle_32bit_load_result_to_64bit_data(
3027 bld, dest, retype(dest, BRW_REGISTER_TYPE_F), num_components);
3028
3029 for (unsigned c = 0; c < num_components; c++) {
3030 bld.MOV(offset(orig_dest, bld, iter * 2 + c),
3031 offset(dest, bld, c));
3032 }
3033 }
3034
3035 /* If we are loading double data and we need a second read message
3036 * adjust the offset
3037 */
3038 if (num_iterations > 1) {
3039 num_components = instr->num_components - 2;
3040 imm_offset++;
3041 }
3042 }
3043 }
3044 break;
3045 }
3046 default:
3047 nir_emit_intrinsic(bld, instr);
3048 break;
3049 }
3050 }
3051
3052 void
3053 fs_visitor::nir_emit_gs_intrinsic(const fs_builder &bld,
3054 nir_intrinsic_instr *instr)
3055 {
3056 assert(stage == MESA_SHADER_GEOMETRY);
3057 fs_reg indirect_offset;
3058
3059 fs_reg dest;
3060 if (nir_intrinsic_infos[instr->intrinsic].has_dest)
3061 dest = get_nir_dest(instr->dest);
3062
3063 switch (instr->intrinsic) {
3064 case nir_intrinsic_load_primitive_id:
3065 assert(stage == MESA_SHADER_GEOMETRY);
3066 assert(((struct brw_gs_prog_data *)prog_data)->include_primitive_id);
3067 bld.MOV(retype(dest, BRW_REGISTER_TYPE_UD),
3068 retype(fs_reg(brw_vec8_grf(2, 0)), BRW_REGISTER_TYPE_UD));
3069 break;
3070
3071 case nir_intrinsic_load_input:
3072 unreachable("load_input intrinsics are invalid for the GS stage");
3073
3074 case nir_intrinsic_load_per_vertex_input:
3075 emit_gs_input_load(dest, instr->src[0], instr->const_index[0],
3076 instr->src[1], instr->num_components,
3077 nir_intrinsic_component(instr));
3078 break;
3079
3080 case nir_intrinsic_emit_vertex_with_counter:
3081 emit_gs_vertex(instr->src[0], instr->const_index[0]);
3082 break;
3083
3084 case nir_intrinsic_end_primitive_with_counter:
3085 emit_gs_end_primitive(instr->src[0]);
3086 break;
3087
3088 case nir_intrinsic_set_vertex_count:
3089 bld.MOV(this->final_gs_vertex_count, get_nir_src(instr->src[0]));
3090 break;
3091
3092 case nir_intrinsic_load_invocation_id: {
3093 fs_reg val = nir_system_values[SYSTEM_VALUE_INVOCATION_ID];
3094 assert(val.file != BAD_FILE);
3095 dest.type = val.type;
3096 bld.MOV(dest, val);
3097 break;
3098 }
3099
3100 default:
3101 nir_emit_intrinsic(bld, instr);
3102 break;
3103 }
3104 }
3105
3106 void
3107 fs_visitor::nir_emit_fs_intrinsic(const fs_builder &bld,
3108 nir_intrinsic_instr *instr)
3109 {
3110 assert(stage == MESA_SHADER_FRAGMENT);
3111
3112 fs_reg dest;
3113 if (nir_intrinsic_infos[instr->intrinsic].has_dest)
3114 dest = get_nir_dest(instr->dest);
3115
3116 switch (instr->intrinsic) {
3117 case nir_intrinsic_load_front_face:
3118 bld.MOV(retype(dest, BRW_REGISTER_TYPE_D),
3119 *emit_frontfacing_interpolation());
3120 break;
3121
3122 case nir_intrinsic_load_sample_pos: {
3123 fs_reg sample_pos = nir_system_values[SYSTEM_VALUE_SAMPLE_POS];
3124 assert(sample_pos.file != BAD_FILE);
3125 dest.type = sample_pos.type;
3126 bld.MOV(dest, sample_pos);
3127 bld.MOV(offset(dest, bld, 1), offset(sample_pos, bld, 1));
3128 break;
3129 }
3130
3131 case nir_intrinsic_load_helper_invocation:
3132 case nir_intrinsic_load_sample_mask_in:
3133 case nir_intrinsic_load_sample_id: {
3134 gl_system_value sv = nir_system_value_from_intrinsic(instr->intrinsic);
3135 fs_reg val = nir_system_values[sv];
3136 assert(val.file != BAD_FILE);
3137 dest.type = val.type;
3138 bld.MOV(dest, val);
3139 break;
3140 }
3141
3142 case nir_intrinsic_discard:
3143 case nir_intrinsic_discard_if: {
3144 /* We track our discarded pixels in f0.1. By predicating on it, we can
3145 * update just the flag bits that aren't yet discarded. If there's no
3146 * condition, we emit a CMP of g0 != g0, so all currently executing
3147 * channels will get turned off.
3148 */
3149 fs_inst *cmp;
3150 if (instr->intrinsic == nir_intrinsic_discard_if) {
3151 cmp = bld.CMP(bld.null_reg_f(), get_nir_src(instr->src[0]),
3152 brw_imm_d(0), BRW_CONDITIONAL_Z);
3153 } else {
3154 fs_reg some_reg = fs_reg(retype(brw_vec8_grf(0, 0),
3155 BRW_REGISTER_TYPE_UW));
3156 cmp = bld.CMP(bld.null_reg_f(), some_reg, some_reg, BRW_CONDITIONAL_NZ);
3157 }
3158 cmp->predicate = BRW_PREDICATE_NORMAL;
3159 cmp->flag_subreg = 1;
3160
3161 if (devinfo->gen >= 6) {
3162 emit_discard_jump();
3163 }
3164 break;
3165 }
3166
3167 case nir_intrinsic_load_input: {
3168 /* load_input is only used for flat inputs */
3169 unsigned base = nir_intrinsic_base(instr);
3170 unsigned component = nir_intrinsic_component(instr);
3171 unsigned num_components = instr->num_components;
3172 enum brw_reg_type type = dest.type;
3173
3174 /* Special case fields in the VUE header */
3175 if (base == VARYING_SLOT_LAYER)
3176 component = 1;
3177 else if (base == VARYING_SLOT_VIEWPORT)
3178 component = 2;
3179
3180 if (nir_dest_bit_size(instr->dest) == 64) {
3181 /* const_index is in 32-bit type size units that could not be aligned
3182 * with DF. We need to read the double vector as if it was a float
3183 * vector of twice the number of components to fetch the right data.
3184 */
3185 type = BRW_REGISTER_TYPE_F;
3186 num_components *= 2;
3187 }
3188
3189 for (unsigned int i = 0; i < num_components; i++) {
3190 struct brw_reg interp = interp_reg(base, component + i);
3191 interp = suboffset(interp, 3);
3192 bld.emit(FS_OPCODE_CINTERP, offset(retype(dest, type), bld, i),
3193 retype(fs_reg(interp), type));
3194 }
3195
3196 if (nir_dest_bit_size(instr->dest) == 64) {
3197 shuffle_32bit_load_result_to_64bit_data(bld,
3198 dest,
3199 retype(dest, type),
3200 instr->num_components);
3201 }
3202 break;
3203 }
3204
3205 case nir_intrinsic_load_barycentric_pixel:
3206 case nir_intrinsic_load_barycentric_centroid:
3207 case nir_intrinsic_load_barycentric_sample:
3208 /* Do nothing - load_interpolated_input handling will handle it later. */
3209 break;
3210
3211 case nir_intrinsic_load_barycentric_at_sample: {
3212 const glsl_interp_mode interpolation =
3213 (enum glsl_interp_mode) nir_intrinsic_interp_mode(instr);
3214
3215 nir_const_value *const_sample = nir_src_as_const_value(instr->src[0]);
3216
3217 if (const_sample) {
3218 unsigned msg_data = const_sample->i32[0] << 4;
3219
3220 emit_pixel_interpolater_send(bld,
3221 FS_OPCODE_INTERPOLATE_AT_SAMPLE,
3222 dest,
3223 fs_reg(), /* src */
3224 brw_imm_ud(msg_data),
3225 interpolation);
3226 } else {
3227 const fs_reg sample_src = retype(get_nir_src(instr->src[0]),
3228 BRW_REGISTER_TYPE_UD);
3229
3230 if (nir_src_is_dynamically_uniform(instr->src[0])) {
3231 const fs_reg sample_id = bld.emit_uniformize(sample_src);
3232 const fs_reg msg_data = vgrf(glsl_type::uint_type);
3233 bld.exec_all().group(1, 0)
3234 .SHL(msg_data, sample_id, brw_imm_ud(4u));
3235 emit_pixel_interpolater_send(bld,
3236 FS_OPCODE_INTERPOLATE_AT_SAMPLE,
3237 dest,
3238 fs_reg(), /* src */
3239 msg_data,
3240 interpolation);
3241 } else {
3242 /* Make a loop that sends a message to the pixel interpolater
3243 * for the sample number in each live channel. If there are
3244 * multiple channels with the same sample number then these
3245 * will be handled simultaneously with a single interation of
3246 * the loop.
3247 */
3248 bld.emit(BRW_OPCODE_DO);
3249
3250 /* Get the next live sample number into sample_id_reg */
3251 const fs_reg sample_id = bld.emit_uniformize(sample_src);
3252
3253 /* Set the flag register so that we can perform the send
3254 * message on all channels that have the same sample number
3255 */
3256 bld.CMP(bld.null_reg_ud(),
3257 sample_src, sample_id,
3258 BRW_CONDITIONAL_EQ);
3259 const fs_reg msg_data = vgrf(glsl_type::uint_type);
3260 bld.exec_all().group(1, 0)
3261 .SHL(msg_data, sample_id, brw_imm_ud(4u));
3262 fs_inst *inst =
3263 emit_pixel_interpolater_send(bld,
3264 FS_OPCODE_INTERPOLATE_AT_SAMPLE,
3265 dest,
3266 fs_reg(), /* src */
3267 msg_data,
3268 interpolation);
3269 set_predicate(BRW_PREDICATE_NORMAL, inst);
3270
3271 /* Continue the loop if there are any live channels left */
3272 set_predicate_inv(BRW_PREDICATE_NORMAL,
3273 true, /* inverse */
3274 bld.emit(BRW_OPCODE_WHILE));
3275 }
3276 }
3277 break;
3278 }
3279
3280 case nir_intrinsic_load_barycentric_at_offset: {
3281 const glsl_interp_mode interpolation =
3282 (enum glsl_interp_mode) nir_intrinsic_interp_mode(instr);
3283
3284 nir_const_value *const_offset = nir_src_as_const_value(instr->src[0]);
3285
3286 if (const_offset) {
3287 unsigned off_x = MIN2((int)(const_offset->f32[0] * 16), 7) & 0xf;
3288 unsigned off_y = MIN2((int)(const_offset->f32[1] * 16), 7) & 0xf;
3289
3290 emit_pixel_interpolater_send(bld,
3291 FS_OPCODE_INTERPOLATE_AT_SHARED_OFFSET,
3292 dest,
3293 fs_reg(), /* src */
3294 brw_imm_ud(off_x | (off_y << 4)),
3295 interpolation);
3296 } else {
3297 fs_reg src = vgrf(glsl_type::ivec2_type);
3298 fs_reg offset_src = retype(get_nir_src(instr->src[0]),
3299 BRW_REGISTER_TYPE_F);
3300 for (int i = 0; i < 2; i++) {
3301 fs_reg temp = vgrf(glsl_type::float_type);
3302 bld.MUL(temp, offset(offset_src, bld, i), brw_imm_f(16.0f));
3303 fs_reg itemp = vgrf(glsl_type::int_type);
3304 /* float to int */
3305 bld.MOV(itemp, temp);
3306
3307 /* Clamp the upper end of the range to +7/16.
3308 * ARB_gpu_shader5 requires that we support a maximum offset
3309 * of +0.5, which isn't representable in a S0.4 value -- if
3310 * we didn't clamp it, we'd end up with -8/16, which is the
3311 * opposite of what the shader author wanted.
3312 *
3313 * This is legal due to ARB_gpu_shader5's quantization
3314 * rules:
3315 *
3316 * "Not all values of <offset> may be supported; x and y
3317 * offsets may be rounded to fixed-point values with the
3318 * number of fraction bits given by the
3319 * implementation-dependent constant
3320 * FRAGMENT_INTERPOLATION_OFFSET_BITS"
3321 */
3322 set_condmod(BRW_CONDITIONAL_L,
3323 bld.SEL(offset(src, bld, i), itemp, brw_imm_d(7)));
3324 }
3325
3326 const enum opcode opcode = FS_OPCODE_INTERPOLATE_AT_PER_SLOT_OFFSET;
3327 emit_pixel_interpolater_send(bld,
3328 opcode,
3329 dest,
3330 src,
3331 brw_imm_ud(0u),
3332 interpolation);
3333 }
3334 break;
3335 }
3336
3337 case nir_intrinsic_load_interpolated_input: {
3338 if (nir_intrinsic_base(instr) == VARYING_SLOT_POS) {
3339 emit_fragcoord_interpolation(dest);
3340 break;
3341 }
3342
3343 assert(instr->src[0].ssa &&
3344 instr->src[0].ssa->parent_instr->type == nir_instr_type_intrinsic);
3345 nir_intrinsic_instr *bary_intrinsic =
3346 nir_instr_as_intrinsic(instr->src[0].ssa->parent_instr);
3347 nir_intrinsic_op bary_intrin = bary_intrinsic->intrinsic;
3348 enum glsl_interp_mode interp_mode =
3349 (enum glsl_interp_mode) nir_intrinsic_interp_mode(bary_intrinsic);
3350 fs_reg dst_xy;
3351
3352 if (bary_intrin == nir_intrinsic_load_barycentric_at_offset ||
3353 bary_intrin == nir_intrinsic_load_barycentric_at_sample) {
3354 /* Use the result of the PI message */
3355 dst_xy = retype(get_nir_src(instr->src[0]), BRW_REGISTER_TYPE_F);
3356 } else {
3357 /* Use the delta_xy values computed from the payload */
3358 enum brw_barycentric_mode bary =
3359 brw_barycentric_mode(interp_mode, bary_intrin);
3360
3361 dst_xy = this->delta_xy[bary];
3362 }
3363
3364 for (unsigned int i = 0; i < instr->num_components; i++) {
3365 fs_reg interp =
3366 fs_reg(interp_reg(nir_intrinsic_base(instr),
3367 nir_intrinsic_component(instr) + i));
3368 interp.type = BRW_REGISTER_TYPE_F;
3369 dest.type = BRW_REGISTER_TYPE_F;
3370
3371 if (devinfo->needs_unlit_centroid_workaround &&
3372 bary_intrin == nir_intrinsic_load_barycentric_centroid) {
3373
3374 /* Get the pixel/sample mask into f0 so that we know which
3375 * pixels are lit. Then, for each channel that is unlit,
3376 * replace the centroid data with non-centroid data.
3377 */
3378 bld.emit(FS_OPCODE_MOV_DISPATCH_TO_FLAGS);
3379
3380 fs_reg dest_i = offset(dest, bld, i);
3381 fs_reg dst_xy_pixel =
3382 delta_xy[brw_barycentric_mode(interp_mode,
3383 nir_intrinsic_load_barycentric_pixel)];
3384
3385 fs_inst *inst;
3386 inst = bld.emit(FS_OPCODE_LINTERP, dest_i, dst_xy_pixel, interp);
3387 inst->predicate = BRW_PREDICATE_NORMAL;
3388 inst->predicate_inverse = true;
3389 inst->no_dd_clear = true;
3390
3391 inst = bld.emit(FS_OPCODE_LINTERP, dest_i, dst_xy, interp);
3392 inst->predicate = BRW_PREDICATE_NORMAL;
3393 inst->predicate_inverse = false;
3394 inst->no_dd_check = true;
3395 } else if (devinfo->gen < 6 && interp_mode == INTERP_MODE_SMOOTH) {
3396 fs_reg tmp = vgrf(glsl_type::float_type);
3397 bld.emit(FS_OPCODE_LINTERP, tmp, dst_xy, interp);
3398 bld.MUL(offset(dest, bld, i), tmp, this->pixel_w);
3399 } else {
3400 bld.emit(FS_OPCODE_LINTERP, offset(dest, bld, i), dst_xy, interp);
3401 }
3402 }
3403 break;
3404 }
3405
3406 default:
3407 nir_emit_intrinsic(bld, instr);
3408 break;
3409 }
3410 }
3411
3412 void
3413 fs_visitor::nir_emit_cs_intrinsic(const fs_builder &bld,
3414 nir_intrinsic_instr *instr)
3415 {
3416 assert(stage == MESA_SHADER_COMPUTE);
3417 struct brw_cs_prog_data *cs_prog_data =
3418 (struct brw_cs_prog_data *) prog_data;
3419
3420 fs_reg dest;
3421 if (nir_intrinsic_infos[instr->intrinsic].has_dest)
3422 dest = get_nir_dest(instr->dest);
3423
3424 switch (instr->intrinsic) {
3425 case nir_intrinsic_barrier:
3426 emit_barrier();
3427 cs_prog_data->uses_barrier = true;
3428 break;
3429
3430 case nir_intrinsic_load_local_invocation_id:
3431 case nir_intrinsic_load_work_group_id: {
3432 gl_system_value sv = nir_system_value_from_intrinsic(instr->intrinsic);
3433 fs_reg val = nir_system_values[sv];
3434 assert(val.file != BAD_FILE);
3435 dest.type = val.type;
3436 for (unsigned i = 0; i < 3; i++)
3437 bld.MOV(offset(dest, bld, i), offset(val, bld, i));
3438 break;
3439 }
3440
3441 case nir_intrinsic_load_num_work_groups: {
3442 const unsigned surface =
3443 cs_prog_data->binding_table.work_groups_start;
3444
3445 cs_prog_data->uses_num_work_groups = true;
3446
3447 fs_reg surf_index = brw_imm_ud(surface);
3448 brw_mark_surface_used(prog_data, surface);
3449
3450 /* Read the 3 GLuint components of gl_NumWorkGroups */
3451 for (unsigned i = 0; i < 3; i++) {
3452 fs_reg read_result =
3453 emit_untyped_read(bld, surf_index,
3454 brw_imm_ud(i << 2),
3455 1 /* dims */, 1 /* size */,
3456 BRW_PREDICATE_NONE);
3457 read_result.type = dest.type;
3458 bld.MOV(dest, read_result);
3459 dest = offset(dest, bld, 1);
3460 }
3461 break;
3462 }
3463
3464 case nir_intrinsic_shared_atomic_add:
3465 nir_emit_shared_atomic(bld, BRW_AOP_ADD, instr);
3466 break;
3467 case nir_intrinsic_shared_atomic_imin:
3468 nir_emit_shared_atomic(bld, BRW_AOP_IMIN, instr);
3469 break;
3470 case nir_intrinsic_shared_atomic_umin:
3471 nir_emit_shared_atomic(bld, BRW_AOP_UMIN, instr);
3472 break;
3473 case nir_intrinsic_shared_atomic_imax:
3474 nir_emit_shared_atomic(bld, BRW_AOP_IMAX, instr);
3475 break;
3476 case nir_intrinsic_shared_atomic_umax:
3477 nir_emit_shared_atomic(bld, BRW_AOP_UMAX, instr);
3478 break;
3479 case nir_intrinsic_shared_atomic_and:
3480 nir_emit_shared_atomic(bld, BRW_AOP_AND, instr);
3481 break;
3482 case nir_intrinsic_shared_atomic_or:
3483 nir_emit_shared_atomic(bld, BRW_AOP_OR, instr);
3484 break;
3485 case nir_intrinsic_shared_atomic_xor:
3486 nir_emit_shared_atomic(bld, BRW_AOP_XOR, instr);
3487 break;
3488 case nir_intrinsic_shared_atomic_exchange:
3489 nir_emit_shared_atomic(bld, BRW_AOP_MOV, instr);
3490 break;
3491 case nir_intrinsic_shared_atomic_comp_swap:
3492 nir_emit_shared_atomic(bld, BRW_AOP_CMPWR, instr);
3493 break;
3494
3495 case nir_intrinsic_load_shared: {
3496 assert(devinfo->gen >= 7);
3497
3498 fs_reg surf_index = brw_imm_ud(GEN7_BTI_SLM);
3499
3500 /* Get the offset to read from */
3501 fs_reg offset_reg;
3502 nir_const_value *const_offset = nir_src_as_const_value(instr->src[0]);
3503 if (const_offset) {
3504 offset_reg = brw_imm_ud(instr->const_index[0] + const_offset->u32[0]);
3505 } else {
3506 offset_reg = vgrf(glsl_type::uint_type);
3507 bld.ADD(offset_reg,
3508 retype(get_nir_src(instr->src[0]), BRW_REGISTER_TYPE_UD),
3509 brw_imm_ud(instr->const_index[0]));
3510 }
3511
3512 /* Read the vector */
3513 do_untyped_vector_read(bld, dest, surf_index, offset_reg,
3514 instr->num_components);
3515 break;
3516 }
3517
3518 case nir_intrinsic_store_shared: {
3519 assert(devinfo->gen >= 7);
3520
3521 /* Block index */
3522 fs_reg surf_index = brw_imm_ud(GEN7_BTI_SLM);
3523
3524 /* Value */
3525 fs_reg val_reg = get_nir_src(instr->src[0]);
3526
3527 /* Writemask */
3528 unsigned writemask = instr->const_index[1];
3529
3530 /* get_nir_src() retypes to integer. Be wary of 64-bit types though
3531 * since the untyped writes below operate in units of 32-bits, which
3532 * means that we need to write twice as many components each time.
3533 * Also, we have to suffle 64-bit data to be in the appropriate layout
3534 * expected by our 32-bit write messages.
3535 */
3536 unsigned type_size = 4;
3537 unsigned bit_size = instr->src[0].is_ssa ?
3538 instr->src[0].ssa->bit_size : instr->src[0].reg.reg->bit_size;
3539 if (bit_size == 64) {
3540 type_size = 8;
3541 fs_reg tmp =
3542 fs_reg(VGRF, alloc.allocate(alloc.sizes[val_reg.nr]), val_reg.type);
3543 shuffle_64bit_data_for_32bit_write(
3544 bld,
3545 retype(tmp, BRW_REGISTER_TYPE_F),
3546 retype(val_reg, BRW_REGISTER_TYPE_DF),
3547 instr->num_components);
3548 val_reg = tmp;
3549 }
3550
3551 unsigned type_slots = type_size / 4;
3552
3553 /* Combine groups of consecutive enabled channels in one write
3554 * message. We use ffs to find the first enabled channel and then ffs on
3555 * the bit-inverse, down-shifted writemask to determine the length of
3556 * the block of enabled bits.
3557 */
3558 while (writemask) {
3559 unsigned first_component = ffs(writemask) - 1;
3560 unsigned length = ffs(~(writemask >> first_component)) - 1;
3561
3562 /* We can't write more than 2 64-bit components at once. Limit the
3563 * length of the write to what we can do and let the next iteration
3564 * handle the rest
3565 */
3566 if (type_size > 4)
3567 length = MIN2(2, length);
3568
3569 fs_reg offset_reg;
3570 nir_const_value *const_offset = nir_src_as_const_value(instr->src[1]);
3571 if (const_offset) {
3572 offset_reg = brw_imm_ud(instr->const_index[0] + const_offset->u32[0] +
3573 type_size * first_component);
3574 } else {
3575 offset_reg = vgrf(glsl_type::uint_type);
3576 bld.ADD(offset_reg,
3577 retype(get_nir_src(instr->src[1]), BRW_REGISTER_TYPE_UD),
3578 brw_imm_ud(instr->const_index[0] + type_size * first_component));
3579 }
3580
3581 emit_untyped_write(bld, surf_index, offset_reg,
3582 offset(val_reg, bld, first_component * type_slots),
3583 1 /* dims */, length * type_slots,
3584 BRW_PREDICATE_NONE);
3585
3586 /* Clear the bits in the writemask that we just wrote, then try
3587 * again to see if more channels are left.
3588 */
3589 writemask &= (15 << (first_component + length));
3590 }
3591
3592 break;
3593 }
3594
3595 default:
3596 nir_emit_intrinsic(bld, instr);
3597 break;
3598 }
3599 }
3600
3601 void
3602 fs_visitor::nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr)
3603 {
3604 fs_reg dest;
3605 if (nir_intrinsic_infos[instr->intrinsic].has_dest)
3606 dest = get_nir_dest(instr->dest);
3607
3608 switch (instr->intrinsic) {
3609 case nir_intrinsic_atomic_counter_inc:
3610 case nir_intrinsic_atomic_counter_dec:
3611 case nir_intrinsic_atomic_counter_read: {
3612 if (stage == MESA_SHADER_FRAGMENT &&
3613 instr->intrinsic != nir_intrinsic_atomic_counter_read)
3614 ((struct brw_wm_prog_data *)prog_data)->has_side_effects = true;
3615
3616 /* Get the arguments of the atomic intrinsic. */
3617 const fs_reg offset = get_nir_src(instr->src[0]);
3618 const unsigned surface = (stage_prog_data->binding_table.abo_start +
3619 instr->const_index[0]);
3620 fs_reg tmp;
3621
3622 /* Emit a surface read or atomic op. */
3623 switch (instr->intrinsic) {
3624 case nir_intrinsic_atomic_counter_read:
3625 tmp = emit_untyped_read(bld, brw_imm_ud(surface), offset, 1, 1);
3626 break;
3627
3628 case nir_intrinsic_atomic_counter_inc:
3629 tmp = emit_untyped_atomic(bld, brw_imm_ud(surface), offset, fs_reg(),
3630 fs_reg(), 1, 1, BRW_AOP_INC);
3631 break;
3632
3633 case nir_intrinsic_atomic_counter_dec:
3634 tmp = emit_untyped_atomic(bld, brw_imm_ud(surface), offset, fs_reg(),
3635 fs_reg(), 1, 1, BRW_AOP_PREDEC);
3636 break;
3637
3638 default:
3639 unreachable("Unreachable");
3640 }
3641
3642 /* Assign the result. */
3643 bld.MOV(retype(dest, BRW_REGISTER_TYPE_UD), tmp);
3644
3645 /* Mark the surface as used. */
3646 brw_mark_surface_used(stage_prog_data, surface);
3647 break;
3648 }
3649
3650 case nir_intrinsic_image_load:
3651 case nir_intrinsic_image_store:
3652 case nir_intrinsic_image_atomic_add:
3653 case nir_intrinsic_image_atomic_min:
3654 case nir_intrinsic_image_atomic_max:
3655 case nir_intrinsic_image_atomic_and:
3656 case nir_intrinsic_image_atomic_or:
3657 case nir_intrinsic_image_atomic_xor:
3658 case nir_intrinsic_image_atomic_exchange:
3659 case nir_intrinsic_image_atomic_comp_swap: {
3660 using namespace image_access;
3661
3662 if (stage == MESA_SHADER_FRAGMENT &&
3663 instr->intrinsic != nir_intrinsic_image_load)
3664 ((struct brw_wm_prog_data *)prog_data)->has_side_effects = true;
3665
3666 /* Get the referenced image variable and type. */
3667 const nir_variable *var = instr->variables[0]->var;
3668 const glsl_type *type = var->type->without_array();
3669 const brw_reg_type base_type = get_image_base_type(type);
3670
3671 /* Get some metadata from the image intrinsic. */
3672 const nir_intrinsic_info *info = &nir_intrinsic_infos[instr->intrinsic];
3673 const unsigned arr_dims = type->sampler_array ? 1 : 0;
3674 const unsigned surf_dims = type->coordinate_components() - arr_dims;
3675 const unsigned format = var->data.image.format;
3676
3677 /* Get the arguments of the image intrinsic. */
3678 const fs_reg image = get_nir_image_deref(instr->variables[0]);
3679 const fs_reg addr = retype(get_nir_src(instr->src[0]),
3680 BRW_REGISTER_TYPE_UD);
3681 const fs_reg src0 = (info->num_srcs >= 3 ?
3682 retype(get_nir_src(instr->src[2]), base_type) :
3683 fs_reg());
3684 const fs_reg src1 = (info->num_srcs >= 4 ?
3685 retype(get_nir_src(instr->src[3]), base_type) :
3686 fs_reg());
3687 fs_reg tmp;
3688
3689 /* Emit an image load, store or atomic op. */
3690 if (instr->intrinsic == nir_intrinsic_image_load)
3691 tmp = emit_image_load(bld, image, addr, surf_dims, arr_dims, format);
3692
3693 else if (instr->intrinsic == nir_intrinsic_image_store)
3694 emit_image_store(bld, image, addr, src0, surf_dims, arr_dims,
3695 var->data.image.write_only ? GL_NONE : format);
3696
3697 else
3698 tmp = emit_image_atomic(bld, image, addr, src0, src1,
3699 surf_dims, arr_dims, info->dest_components,
3700 get_image_atomic_op(instr->intrinsic, type));
3701
3702 /* Assign the result. */
3703 for (unsigned c = 0; c < info->dest_components; ++c)
3704 bld.MOV(offset(retype(dest, base_type), bld, c),
3705 offset(tmp, bld, c));
3706 break;
3707 }
3708
3709 case nir_intrinsic_memory_barrier_atomic_counter:
3710 case nir_intrinsic_memory_barrier_buffer:
3711 case nir_intrinsic_memory_barrier_image:
3712 case nir_intrinsic_memory_barrier: {
3713 const fs_builder ubld = bld.group(8, 0);
3714 const fs_reg tmp = ubld.vgrf(BRW_REGISTER_TYPE_UD, 2);
3715 ubld.emit(SHADER_OPCODE_MEMORY_FENCE, tmp)
3716 ->regs_written = 2;
3717 break;
3718 }
3719
3720 case nir_intrinsic_group_memory_barrier:
3721 case nir_intrinsic_memory_barrier_shared:
3722 /* We treat these workgroup-level barriers as no-ops. This should be
3723 * safe at present and as long as:
3724 *
3725 * - Memory access instructions are not subsequently reordered by the
3726 * compiler back-end.
3727 *
3728 * - All threads from a given compute shader workgroup fit within a
3729 * single subslice and therefore talk to the same HDC shared unit
3730 * what supposedly guarantees ordering and coherency between threads
3731 * from the same workgroup. This may change in the future when we
3732 * start splitting workgroups across multiple subslices.
3733 *
3734 * - The context is not in fault-and-stream mode, which could cause
3735 * memory transactions (including to SLM) prior to the barrier to be
3736 * replayed after the barrier if a pagefault occurs. This shouldn't
3737 * be a problem up to and including SKL because fault-and-stream is
3738 * not usable due to hardware issues, but that's likely to change in
3739 * the future.
3740 */
3741 break;
3742
3743 case nir_intrinsic_shader_clock: {
3744 /* We cannot do anything if there is an event, so ignore it for now */
3745 fs_reg shader_clock = get_timestamp(bld);
3746 const fs_reg srcs[] = { shader_clock.set_smear(0), shader_clock.set_smear(1) };
3747
3748 bld.LOAD_PAYLOAD(dest, srcs, ARRAY_SIZE(srcs), 0);
3749 break;
3750 }
3751
3752 case nir_intrinsic_image_size: {
3753 /* Get the referenced image variable and type. */
3754 const nir_variable *var = instr->variables[0]->var;
3755 const glsl_type *type = var->type->without_array();
3756
3757 /* Get the size of the image. */
3758 const fs_reg image = get_nir_image_deref(instr->variables[0]);
3759 const fs_reg size = offset(image, bld, BRW_IMAGE_PARAM_SIZE_OFFSET);
3760
3761 /* For 1DArray image types, the array index is stored in the Z component.
3762 * Fix this by swizzling the Z component to the Y component.
3763 */
3764 const bool is_1d_array_image =
3765 type->sampler_dimensionality == GLSL_SAMPLER_DIM_1D &&
3766 type->sampler_array;
3767
3768 /* For CubeArray images, we should count the number of cubes instead
3769 * of the number of faces. Fix it by dividing the (Z component) by 6.
3770 */
3771 const bool is_cube_array_image =
3772 type->sampler_dimensionality == GLSL_SAMPLER_DIM_CUBE &&
3773 type->sampler_array;
3774
3775 /* Copy all the components. */
3776 const nir_intrinsic_info *info = &nir_intrinsic_infos[instr->intrinsic];
3777 for (unsigned c = 0; c < info->dest_components; ++c) {
3778 if ((int)c >= type->coordinate_components()) {
3779 bld.MOV(offset(retype(dest, BRW_REGISTER_TYPE_D), bld, c),
3780 brw_imm_d(1));
3781 } else if (c == 1 && is_1d_array_image) {
3782 bld.MOV(offset(retype(dest, BRW_REGISTER_TYPE_D), bld, c),
3783 offset(size, bld, 2));
3784 } else if (c == 2 && is_cube_array_image) {
3785 bld.emit(SHADER_OPCODE_INT_QUOTIENT,
3786 offset(retype(dest, BRW_REGISTER_TYPE_D), bld, c),
3787 offset(size, bld, c), brw_imm_d(6));
3788 } else {
3789 bld.MOV(offset(retype(dest, BRW_REGISTER_TYPE_D), bld, c),
3790 offset(size, bld, c));
3791 }
3792 }
3793
3794 break;
3795 }
3796
3797 case nir_intrinsic_image_samples:
3798 /* The driver does not support multi-sampled images. */
3799 bld.MOV(retype(dest, BRW_REGISTER_TYPE_D), brw_imm_d(1));
3800 break;
3801
3802 case nir_intrinsic_load_uniform: {
3803 /* Offsets are in bytes but they should always be multiples of 4 */
3804 assert(instr->const_index[0] % 4 == 0);
3805
3806 fs_reg src(UNIFORM, instr->const_index[0] / 4, dest.type);
3807
3808 nir_const_value *const_offset = nir_src_as_const_value(instr->src[0]);
3809 if (const_offset) {
3810 /* Offsets are in bytes but they should always be multiples of 4 */
3811 assert(const_offset->u32[0] % 4 == 0);
3812 src.reg_offset = const_offset->u32[0] / 4;
3813
3814 for (unsigned j = 0; j < instr->num_components; j++) {
3815 bld.MOV(offset(dest, bld, j), offset(src, bld, j));
3816 }
3817 } else {
3818 fs_reg indirect = retype(get_nir_src(instr->src[0]),
3819 BRW_REGISTER_TYPE_UD);
3820
3821 /* We need to pass a size to the MOV_INDIRECT but we don't want it to
3822 * go past the end of the uniform. In order to keep the n'th
3823 * component from running past, we subtract off the size of all but
3824 * one component of the vector.
3825 */
3826 assert(instr->const_index[1] >=
3827 instr->num_components * (int) type_sz(dest.type));
3828 unsigned read_size = instr->const_index[1] -
3829 (instr->num_components - 1) * type_sz(dest.type);
3830
3831 fs_reg indirect_chv_high_32bit;
3832 bool is_chv_bxt_64bit =
3833 (devinfo->is_cherryview || devinfo->is_broxton) &&
3834 type_sz(dest.type) == 8;
3835 if (is_chv_bxt_64bit) {
3836 indirect_chv_high_32bit = vgrf(glsl_type::uint_type);
3837 /* Calculate indirect address to read high 32 bits */
3838 bld.ADD(indirect_chv_high_32bit, indirect, brw_imm_ud(4));
3839 }
3840
3841 for (unsigned j = 0; j < instr->num_components; j++) {
3842 if (!is_chv_bxt_64bit) {
3843 bld.emit(SHADER_OPCODE_MOV_INDIRECT,
3844 offset(dest, bld, j), offset(src, bld, j),
3845 indirect, brw_imm_ud(read_size));
3846 } else {
3847 bld.emit(SHADER_OPCODE_MOV_INDIRECT,
3848 subscript(offset(dest, bld, j), BRW_REGISTER_TYPE_UD, 0),
3849 offset(src, bld, j),
3850 indirect, brw_imm_ud(read_size));
3851
3852 bld.emit(SHADER_OPCODE_MOV_INDIRECT,
3853 subscript(offset(dest, bld, j), BRW_REGISTER_TYPE_UD, 1),
3854 offset(src, bld, j),
3855 indirect_chv_high_32bit, brw_imm_ud(read_size));
3856 }
3857 }
3858 }
3859 break;
3860 }
3861
3862 case nir_intrinsic_load_ubo: {
3863 nir_const_value *const_index = nir_src_as_const_value(instr->src[0]);
3864 fs_reg surf_index;
3865
3866 if (const_index) {
3867 const unsigned index = stage_prog_data->binding_table.ubo_start +
3868 const_index->u32[0];
3869 surf_index = brw_imm_ud(index);
3870 brw_mark_surface_used(prog_data, index);
3871 } else {
3872 /* The block index is not a constant. Evaluate the index expression
3873 * per-channel and add the base UBO index; we have to select a value
3874 * from any live channel.
3875 */
3876 surf_index = vgrf(glsl_type::uint_type);
3877 bld.ADD(surf_index, get_nir_src(instr->src[0]),
3878 brw_imm_ud(stage_prog_data->binding_table.ubo_start));
3879 surf_index = bld.emit_uniformize(surf_index);
3880
3881 /* Assume this may touch any UBO. It would be nice to provide
3882 * a tighter bound, but the array information is already lowered away.
3883 */
3884 brw_mark_surface_used(prog_data,
3885 stage_prog_data->binding_table.ubo_start +
3886 nir->info.num_ubos - 1);
3887 }
3888
3889 nir_const_value *const_offset = nir_src_as_const_value(instr->src[1]);
3890 if (const_offset == NULL) {
3891 fs_reg base_offset = retype(get_nir_src(instr->src[1]),
3892 BRW_REGISTER_TYPE_UD);
3893
3894 for (int i = 0; i < instr->num_components; i++)
3895 VARYING_PULL_CONSTANT_LOAD(bld, offset(dest, bld, i), surf_index,
3896 base_offset, i * type_sz(dest.type));
3897 } else {
3898 /* Even if we are loading doubles, a pull constant load will load
3899 * a 32-bit vec4, so should only reserve vgrf space for that. If we
3900 * need to load a full dvec4 we will have to emit 2 loads. This is
3901 * similar to demote_pull_constants(), except that in that case we
3902 * see individual accesses to each component of the vector and then
3903 * we let CSE deal with duplicate loads. Here we see a vector access
3904 * and we have to split it if necessary.
3905 */
3906 const unsigned type_size = type_sz(dest.type);
3907 const fs_reg packed_consts = bld.vgrf(BRW_REGISTER_TYPE_F);
3908 for (unsigned c = 0; c < instr->num_components;) {
3909 const unsigned base = const_offset->u32[0] + c * type_size;
3910
3911 /* Number of usable components in the next 16B-aligned load */
3912 const unsigned count = MIN2(instr->num_components - c,
3913 (16 - base % 16) / type_size);
3914
3915 bld.exec_all()
3916 .emit(FS_OPCODE_UNIFORM_PULL_CONSTANT_LOAD,
3917 packed_consts, surf_index, brw_imm_ud(base & ~15));
3918
3919 const fs_reg consts =
3920 retype(byte_offset(packed_consts, base & 15), dest.type);
3921
3922 for (unsigned d = 0; d < count; d++)
3923 bld.MOV(offset(dest, bld, c + d), component(consts, d));
3924
3925 c += count;
3926 }
3927 }
3928 break;
3929 }
3930
3931 case nir_intrinsic_load_ssbo: {
3932 assert(devinfo->gen >= 7);
3933
3934 nir_const_value *const_uniform_block =
3935 nir_src_as_const_value(instr->src[0]);
3936
3937 fs_reg surf_index;
3938 if (const_uniform_block) {
3939 unsigned index = stage_prog_data->binding_table.ssbo_start +
3940 const_uniform_block->u32[0];
3941 surf_index = brw_imm_ud(index);
3942 brw_mark_surface_used(prog_data, index);
3943 } else {
3944 surf_index = vgrf(glsl_type::uint_type);
3945 bld.ADD(surf_index, get_nir_src(instr->src[0]),
3946 brw_imm_ud(stage_prog_data->binding_table.ssbo_start));
3947
3948 /* Assume this may touch any UBO. It would be nice to provide
3949 * a tighter bound, but the array information is already lowered away.
3950 */
3951 brw_mark_surface_used(prog_data,
3952 stage_prog_data->binding_table.ssbo_start +
3953 nir->info.num_ssbos - 1);
3954 }
3955
3956 fs_reg offset_reg;
3957 nir_const_value *const_offset = nir_src_as_const_value(instr->src[1]);
3958 if (const_offset) {
3959 offset_reg = brw_imm_ud(const_offset->u32[0]);
3960 } else {
3961 offset_reg = get_nir_src(instr->src[1]);
3962 }
3963
3964 /* Read the vector */
3965 do_untyped_vector_read(bld, dest, surf_index, offset_reg,
3966 instr->num_components);
3967
3968 break;
3969 }
3970
3971 case nir_intrinsic_load_input: {
3972 fs_reg src = fs_reg(ATTR, instr->const_index[0], dest.type);
3973 unsigned num_components = instr->num_components;
3974 enum brw_reg_type type = dest.type;
3975
3976 nir_const_value *const_offset = nir_src_as_const_value(instr->src[0]);
3977 assert(const_offset && "Indirect input loads not allowed");
3978 src = offset(src, bld, const_offset->u32[0]);
3979
3980 for (unsigned j = 0; j < num_components; j++) {
3981 bld.MOV(offset(dest, bld, j), offset(src, bld, j));
3982 }
3983
3984 if (type == BRW_REGISTER_TYPE_DF) {
3985 /* Once the double vector is read, set again its original register
3986 * type to continue with normal execution.
3987 */
3988 src = retype(src, type);
3989 dest = retype(dest, type);
3990 }
3991
3992 if (type_sz(src.type) == 8) {
3993 shuffle_32bit_load_result_to_64bit_data(bld,
3994 dest,
3995 retype(dest, BRW_REGISTER_TYPE_F),
3996 instr->num_components);
3997 }
3998
3999 break;
4000 }
4001
4002 case nir_intrinsic_store_ssbo: {
4003 assert(devinfo->gen >= 7);
4004
4005 if (stage == MESA_SHADER_FRAGMENT)
4006 ((struct brw_wm_prog_data *)prog_data)->has_side_effects = true;
4007
4008 /* Block index */
4009 fs_reg surf_index;
4010 nir_const_value *const_uniform_block =
4011 nir_src_as_const_value(instr->src[1]);
4012 if (const_uniform_block) {
4013 unsigned index = stage_prog_data->binding_table.ssbo_start +
4014 const_uniform_block->u32[0];
4015 surf_index = brw_imm_ud(index);
4016 brw_mark_surface_used(prog_data, index);
4017 } else {
4018 surf_index = vgrf(glsl_type::uint_type);
4019 bld.ADD(surf_index, get_nir_src(instr->src[1]),
4020 brw_imm_ud(stage_prog_data->binding_table.ssbo_start));
4021
4022 brw_mark_surface_used(prog_data,
4023 stage_prog_data->binding_table.ssbo_start +
4024 nir->info.num_ssbos - 1);
4025 }
4026
4027 /* Value */
4028 fs_reg val_reg = get_nir_src(instr->src[0]);
4029
4030 /* Writemask */
4031 unsigned writemask = instr->const_index[0];
4032
4033 /* get_nir_src() retypes to integer. Be wary of 64-bit types though
4034 * since the untyped writes below operate in units of 32-bits, which
4035 * means that we need to write twice as many components each time.
4036 * Also, we have to suffle 64-bit data to be in the appropriate layout
4037 * expected by our 32-bit write messages.
4038 */
4039 unsigned type_size = 4;
4040 unsigned bit_size = instr->src[0].is_ssa ?
4041 instr->src[0].ssa->bit_size : instr->src[0].reg.reg->bit_size;
4042 if (bit_size == 64) {
4043 type_size = 8;
4044 fs_reg tmp =
4045 fs_reg(VGRF, alloc.allocate(alloc.sizes[val_reg.nr]), val_reg.type);
4046 shuffle_64bit_data_for_32bit_write(bld,
4047 retype(tmp, BRW_REGISTER_TYPE_F),
4048 retype(val_reg, BRW_REGISTER_TYPE_DF),
4049 instr->num_components);
4050 val_reg = tmp;
4051 }
4052
4053 unsigned type_slots = type_size / 4;
4054
4055 /* Combine groups of consecutive enabled channels in one write
4056 * message. We use ffs to find the first enabled channel and then ffs on
4057 * the bit-inverse, down-shifted writemask to determine the length of
4058 * the block of enabled bits.
4059 */
4060 while (writemask) {
4061 unsigned first_component = ffs(writemask) - 1;
4062 unsigned length = ffs(~(writemask >> first_component)) - 1;
4063
4064 /* We can't write more than 2 64-bit components at once. Limit the
4065 * length of the write to what we can do and let the next iteration
4066 * handle the rest
4067 */
4068 if (type_size > 4)
4069 length = MIN2(2, length);
4070
4071 fs_reg offset_reg;
4072 nir_const_value *const_offset = nir_src_as_const_value(instr->src[2]);
4073 if (const_offset) {
4074 offset_reg = brw_imm_ud(const_offset->u32[0] +
4075 type_size * first_component);
4076 } else {
4077 offset_reg = vgrf(glsl_type::uint_type);
4078 bld.ADD(offset_reg,
4079 retype(get_nir_src(instr->src[2]), BRW_REGISTER_TYPE_UD),
4080 brw_imm_ud(type_size * first_component));
4081 }
4082
4083
4084 emit_untyped_write(bld, surf_index, offset_reg,
4085 offset(val_reg, bld, first_component * type_slots),
4086 1 /* dims */, length * type_slots,
4087 BRW_PREDICATE_NONE);
4088
4089 /* Clear the bits in the writemask that we just wrote, then try
4090 * again to see if more channels are left.
4091 */
4092 writemask &= (15 << (first_component + length));
4093 }
4094 break;
4095 }
4096
4097 case nir_intrinsic_store_output: {
4098 fs_reg src = get_nir_src(instr->src[0]);
4099 fs_reg new_dest = offset(retype(nir_outputs, src.type), bld,
4100 instr->const_index[0]);
4101
4102 nir_const_value *const_offset = nir_src_as_const_value(instr->src[1]);
4103 assert(const_offset && "Indirect output stores not allowed");
4104 new_dest = offset(new_dest, bld, const_offset->u32[0]);
4105
4106 unsigned num_components = instr->num_components;
4107 unsigned bit_size = instr->src[0].is_ssa ?
4108 instr->src[0].ssa->bit_size : instr->src[0].reg.reg->bit_size;
4109 if (bit_size == 64) {
4110 fs_reg tmp =
4111 fs_reg(VGRF, alloc.allocate(2 * num_components),
4112 BRW_REGISTER_TYPE_F);
4113 shuffle_64bit_data_for_32bit_write(
4114 bld, tmp, retype(src, BRW_REGISTER_TYPE_DF), num_components);
4115 src = retype(tmp, src.type);
4116 num_components *= 2;
4117 }
4118
4119 for (unsigned j = 0; j < num_components; j++) {
4120 bld.MOV(offset(new_dest, bld, j), offset(src, bld, j));
4121 }
4122 break;
4123 }
4124
4125 case nir_intrinsic_ssbo_atomic_add:
4126 nir_emit_ssbo_atomic(bld, BRW_AOP_ADD, instr);
4127 break;
4128 case nir_intrinsic_ssbo_atomic_imin:
4129 nir_emit_ssbo_atomic(bld, BRW_AOP_IMIN, instr);
4130 break;
4131 case nir_intrinsic_ssbo_atomic_umin:
4132 nir_emit_ssbo_atomic(bld, BRW_AOP_UMIN, instr);
4133 break;
4134 case nir_intrinsic_ssbo_atomic_imax:
4135 nir_emit_ssbo_atomic(bld, BRW_AOP_IMAX, instr);
4136 break;
4137 case nir_intrinsic_ssbo_atomic_umax:
4138 nir_emit_ssbo_atomic(bld, BRW_AOP_UMAX, instr);
4139 break;
4140 case nir_intrinsic_ssbo_atomic_and:
4141 nir_emit_ssbo_atomic(bld, BRW_AOP_AND, instr);
4142 break;
4143 case nir_intrinsic_ssbo_atomic_or:
4144 nir_emit_ssbo_atomic(bld, BRW_AOP_OR, instr);
4145 break;
4146 case nir_intrinsic_ssbo_atomic_xor:
4147 nir_emit_ssbo_atomic(bld, BRW_AOP_XOR, instr);
4148 break;
4149 case nir_intrinsic_ssbo_atomic_exchange:
4150 nir_emit_ssbo_atomic(bld, BRW_AOP_MOV, instr);
4151 break;
4152 case nir_intrinsic_ssbo_atomic_comp_swap:
4153 nir_emit_ssbo_atomic(bld, BRW_AOP_CMPWR, instr);
4154 break;
4155
4156 case nir_intrinsic_get_buffer_size: {
4157 nir_const_value *const_uniform_block = nir_src_as_const_value(instr->src[0]);
4158 unsigned ssbo_index = const_uniform_block ? const_uniform_block->u32[0] : 0;
4159
4160 /* A resinfo's sampler message is used to get the buffer size. The
4161 * SIMD8's writeback message consists of four registers and SIMD16's
4162 * writeback message consists of 8 destination registers (two per each
4163 * component). Because we are only interested on the first channel of
4164 * the first returned component, where resinfo returns the buffer size
4165 * for SURFTYPE_BUFFER, we can just use the SIMD8 variant regardless of
4166 * the dispatch width.
4167 */
4168 const fs_builder ubld = bld.exec_all().group(8, 0);
4169 fs_reg src_payload = ubld.vgrf(BRW_REGISTER_TYPE_UD);
4170 fs_reg ret_payload = ubld.vgrf(BRW_REGISTER_TYPE_UD, 4);
4171
4172 /* Set LOD = 0 */
4173 ubld.MOV(src_payload, brw_imm_d(0));
4174
4175 const unsigned index = prog_data->binding_table.ssbo_start + ssbo_index;
4176 fs_inst *inst = ubld.emit(FS_OPCODE_GET_BUFFER_SIZE, ret_payload,
4177 src_payload, brw_imm_ud(index));
4178 inst->header_size = 0;
4179 inst->mlen = 1;
4180 inst->regs_written = 4;
4181
4182 bld.MOV(retype(dest, ret_payload.type), component(ret_payload, 0));
4183 brw_mark_surface_used(prog_data, index);
4184 break;
4185 }
4186
4187 case nir_intrinsic_load_channel_num: {
4188 fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_UW);
4189 dest = retype(dest, BRW_REGISTER_TYPE_UD);
4190 const fs_builder allbld8 = bld.group(8, 0).exec_all();
4191 allbld8.MOV(tmp, brw_imm_v(0x76543210));
4192 if (dispatch_width > 8)
4193 allbld8.ADD(byte_offset(tmp, 16), tmp, brw_imm_uw(8u));
4194 if (dispatch_width > 16) {
4195 const fs_builder allbld16 = bld.group(16, 0).exec_all();
4196 allbld16.ADD(byte_offset(tmp, 32), tmp, brw_imm_uw(16u));
4197 }
4198 bld.MOV(dest, tmp);
4199 break;
4200 }
4201
4202 default:
4203 unreachable("unknown intrinsic");
4204 }
4205 }
4206
4207 void
4208 fs_visitor::nir_emit_ssbo_atomic(const fs_builder &bld,
4209 int op, nir_intrinsic_instr *instr)
4210 {
4211 if (stage == MESA_SHADER_FRAGMENT)
4212 ((struct brw_wm_prog_data *)prog_data)->has_side_effects = true;
4213
4214 fs_reg dest;
4215 if (nir_intrinsic_infos[instr->intrinsic].has_dest)
4216 dest = get_nir_dest(instr->dest);
4217
4218 fs_reg surface;
4219 nir_const_value *const_surface = nir_src_as_const_value(instr->src[0]);
4220 if (const_surface) {
4221 unsigned surf_index = stage_prog_data->binding_table.ssbo_start +
4222 const_surface->u32[0];
4223 surface = brw_imm_ud(surf_index);
4224 brw_mark_surface_used(prog_data, surf_index);
4225 } else {
4226 surface = vgrf(glsl_type::uint_type);
4227 bld.ADD(surface, get_nir_src(instr->src[0]),
4228 brw_imm_ud(stage_prog_data->binding_table.ssbo_start));
4229
4230 /* Assume this may touch any SSBO. This is the same we do for other
4231 * UBO/SSBO accesses with non-constant surface.
4232 */
4233 brw_mark_surface_used(prog_data,
4234 stage_prog_data->binding_table.ssbo_start +
4235 nir->info.num_ssbos - 1);
4236 }
4237
4238 fs_reg offset = get_nir_src(instr->src[1]);
4239 fs_reg data1 = get_nir_src(instr->src[2]);
4240 fs_reg data2;
4241 if (op == BRW_AOP_CMPWR)
4242 data2 = get_nir_src(instr->src[3]);
4243
4244 /* Emit the actual atomic operation operation */
4245
4246 fs_reg atomic_result = emit_untyped_atomic(bld, surface, offset,
4247 data1, data2,
4248 1 /* dims */, 1 /* rsize */,
4249 op,
4250 BRW_PREDICATE_NONE);
4251 dest.type = atomic_result.type;
4252 bld.MOV(dest, atomic_result);
4253 }
4254
4255 void
4256 fs_visitor::nir_emit_shared_atomic(const fs_builder &bld,
4257 int op, nir_intrinsic_instr *instr)
4258 {
4259 fs_reg dest;
4260 if (nir_intrinsic_infos[instr->intrinsic].has_dest)
4261 dest = get_nir_dest(instr->dest);
4262
4263 fs_reg surface = brw_imm_ud(GEN7_BTI_SLM);
4264 fs_reg offset = get_nir_src(instr->src[0]);
4265 fs_reg data1 = get_nir_src(instr->src[1]);
4266 fs_reg data2;
4267 if (op == BRW_AOP_CMPWR)
4268 data2 = get_nir_src(instr->src[2]);
4269
4270 /* Emit the actual atomic operation operation */
4271
4272 fs_reg atomic_result = emit_untyped_atomic(bld, surface, offset,
4273 data1, data2,
4274 1 /* dims */, 1 /* rsize */,
4275 op,
4276 BRW_PREDICATE_NONE);
4277 dest.type = atomic_result.type;
4278 bld.MOV(dest, atomic_result);
4279 }
4280
4281 void
4282 fs_visitor::nir_emit_texture(const fs_builder &bld, nir_tex_instr *instr)
4283 {
4284 unsigned texture = instr->texture_index;
4285 unsigned sampler = instr->sampler_index;
4286
4287 fs_reg srcs[TEX_LOGICAL_NUM_SRCS];
4288
4289 srcs[TEX_LOGICAL_SRC_SURFACE] = brw_imm_ud(texture);
4290 srcs[TEX_LOGICAL_SRC_SAMPLER] = brw_imm_ud(sampler);
4291
4292 int lod_components = 0;
4293
4294 /* The hardware requires a LOD for buffer textures */
4295 if (instr->sampler_dim == GLSL_SAMPLER_DIM_BUF)
4296 srcs[TEX_LOGICAL_SRC_LOD] = brw_imm_d(0);
4297
4298 for (unsigned i = 0; i < instr->num_srcs; i++) {
4299 fs_reg src = get_nir_src(instr->src[i].src);
4300 switch (instr->src[i].src_type) {
4301 case nir_tex_src_bias:
4302 srcs[TEX_LOGICAL_SRC_LOD] =
4303 retype(get_nir_src_imm(instr->src[i].src), BRW_REGISTER_TYPE_F);
4304 break;
4305 case nir_tex_src_comparitor:
4306 srcs[TEX_LOGICAL_SRC_SHADOW_C] = retype(src, BRW_REGISTER_TYPE_F);
4307 break;
4308 case nir_tex_src_coord:
4309 switch (instr->op) {
4310 case nir_texop_txf:
4311 case nir_texop_txf_ms:
4312 case nir_texop_txf_ms_mcs:
4313 case nir_texop_samples_identical:
4314 srcs[TEX_LOGICAL_SRC_COORDINATE] = retype(src, BRW_REGISTER_TYPE_D);
4315 break;
4316 default:
4317 srcs[TEX_LOGICAL_SRC_COORDINATE] = retype(src, BRW_REGISTER_TYPE_F);
4318 break;
4319 }
4320 break;
4321 case nir_tex_src_ddx:
4322 srcs[TEX_LOGICAL_SRC_LOD] = retype(src, BRW_REGISTER_TYPE_F);
4323 lod_components = nir_tex_instr_src_size(instr, i);
4324 break;
4325 case nir_tex_src_ddy:
4326 srcs[TEX_LOGICAL_SRC_LOD2] = retype(src, BRW_REGISTER_TYPE_F);
4327 break;
4328 case nir_tex_src_lod:
4329 switch (instr->op) {
4330 case nir_texop_txs:
4331 srcs[TEX_LOGICAL_SRC_LOD] =
4332 retype(get_nir_src_imm(instr->src[i].src), BRW_REGISTER_TYPE_UD);
4333 break;
4334 case nir_texop_txf:
4335 srcs[TEX_LOGICAL_SRC_LOD] =
4336 retype(get_nir_src_imm(instr->src[i].src), BRW_REGISTER_TYPE_D);
4337 break;
4338 default:
4339 srcs[TEX_LOGICAL_SRC_LOD] =
4340 retype(get_nir_src_imm(instr->src[i].src), BRW_REGISTER_TYPE_F);
4341 break;
4342 }
4343 break;
4344 case nir_tex_src_ms_index:
4345 srcs[TEX_LOGICAL_SRC_SAMPLE_INDEX] = retype(src, BRW_REGISTER_TYPE_UD);
4346 break;
4347
4348 case nir_tex_src_offset: {
4349 nir_const_value *const_offset =
4350 nir_src_as_const_value(instr->src[i].src);
4351 if (const_offset) {
4352 unsigned header_bits = brw_texture_offset(const_offset->i32, 3);
4353 if (header_bits != 0)
4354 srcs[TEX_LOGICAL_SRC_OFFSET_VALUE] = brw_imm_ud(header_bits);
4355 } else {
4356 srcs[TEX_LOGICAL_SRC_OFFSET_VALUE] =
4357 retype(src, BRW_REGISTER_TYPE_D);
4358 }
4359 break;
4360 }
4361
4362 case nir_tex_src_projector:
4363 unreachable("should be lowered");
4364
4365 case nir_tex_src_texture_offset: {
4366 /* Figure out the highest possible texture index and mark it as used */
4367 uint32_t max_used = texture + instr->texture_array_size - 1;
4368 if (instr->op == nir_texop_tg4 && devinfo->gen < 8) {
4369 max_used += stage_prog_data->binding_table.gather_texture_start;
4370 } else {
4371 max_used += stage_prog_data->binding_table.texture_start;
4372 }
4373 brw_mark_surface_used(prog_data, max_used);
4374
4375 /* Emit code to evaluate the actual indexing expression */
4376 fs_reg tmp = vgrf(glsl_type::uint_type);
4377 bld.ADD(tmp, src, brw_imm_ud(texture));
4378 srcs[TEX_LOGICAL_SRC_SURFACE] = bld.emit_uniformize(tmp);
4379 break;
4380 }
4381
4382 case nir_tex_src_sampler_offset: {
4383 /* Emit code to evaluate the actual indexing expression */
4384 fs_reg tmp = vgrf(glsl_type::uint_type);
4385 bld.ADD(tmp, src, brw_imm_ud(sampler));
4386 srcs[TEX_LOGICAL_SRC_SAMPLER] = bld.emit_uniformize(tmp);
4387 break;
4388 }
4389
4390 case nir_tex_src_ms_mcs:
4391 assert(instr->op == nir_texop_txf_ms);
4392 srcs[TEX_LOGICAL_SRC_MCS] = retype(src, BRW_REGISTER_TYPE_D);
4393 break;
4394
4395 case nir_tex_src_plane: {
4396 nir_const_value *const_plane =
4397 nir_src_as_const_value(instr->src[i].src);
4398 const uint32_t plane = const_plane->u32[0];
4399 const uint32_t texture_index =
4400 instr->texture_index +
4401 stage_prog_data->binding_table.plane_start[plane] -
4402 stage_prog_data->binding_table.texture_start;
4403
4404 srcs[TEX_LOGICAL_SRC_SURFACE] = brw_imm_ud(texture_index);
4405 break;
4406 }
4407
4408 default:
4409 unreachable("unknown texture source");
4410 }
4411 }
4412
4413 if (srcs[TEX_LOGICAL_SRC_MCS].file == BAD_FILE &&
4414 (instr->op == nir_texop_txf_ms ||
4415 instr->op == nir_texop_samples_identical)) {
4416 if (devinfo->gen >= 7 &&
4417 key_tex->compressed_multisample_layout_mask & (1 << texture)) {
4418 srcs[TEX_LOGICAL_SRC_MCS] =
4419 emit_mcs_fetch(srcs[TEX_LOGICAL_SRC_COORDINATE],
4420 instr->coord_components,
4421 srcs[TEX_LOGICAL_SRC_SURFACE]);
4422 } else {
4423 srcs[TEX_LOGICAL_SRC_MCS] = brw_imm_ud(0u);
4424 }
4425 }
4426
4427 srcs[TEX_LOGICAL_SRC_COORD_COMPONENTS] = brw_imm_d(instr->coord_components);
4428 srcs[TEX_LOGICAL_SRC_GRAD_COMPONENTS] = brw_imm_d(lod_components);
4429
4430 if (instr->op == nir_texop_query_levels) {
4431 /* textureQueryLevels() is implemented in terms of TXS so we need to
4432 * pass a valid LOD argument.
4433 */
4434 assert(srcs[TEX_LOGICAL_SRC_LOD].file == BAD_FILE);
4435 srcs[TEX_LOGICAL_SRC_LOD] = brw_imm_ud(0u);
4436 }
4437
4438 enum opcode opcode;
4439 switch (instr->op) {
4440 case nir_texop_tex:
4441 opcode = SHADER_OPCODE_TEX_LOGICAL;
4442 break;
4443 case nir_texop_txb:
4444 opcode = FS_OPCODE_TXB_LOGICAL;
4445 break;
4446 case nir_texop_txl:
4447 opcode = SHADER_OPCODE_TXL_LOGICAL;
4448 break;
4449 case nir_texop_txd:
4450 opcode = SHADER_OPCODE_TXD_LOGICAL;
4451 break;
4452 case nir_texop_txf:
4453 opcode = SHADER_OPCODE_TXF_LOGICAL;
4454 break;
4455 case nir_texop_txf_ms:
4456 if ((key_tex->msaa_16 & (1 << sampler)))
4457 opcode = SHADER_OPCODE_TXF_CMS_W_LOGICAL;
4458 else
4459 opcode = SHADER_OPCODE_TXF_CMS_LOGICAL;
4460 break;
4461 case nir_texop_txf_ms_mcs:
4462 opcode = SHADER_OPCODE_TXF_MCS_LOGICAL;
4463 break;
4464 case nir_texop_query_levels:
4465 case nir_texop_txs:
4466 opcode = SHADER_OPCODE_TXS_LOGICAL;
4467 break;
4468 case nir_texop_lod:
4469 opcode = SHADER_OPCODE_LOD_LOGICAL;
4470 break;
4471 case nir_texop_tg4:
4472 if (srcs[TEX_LOGICAL_SRC_OFFSET_VALUE].file != BAD_FILE &&
4473 srcs[TEX_LOGICAL_SRC_OFFSET_VALUE].file != IMM)
4474 opcode = SHADER_OPCODE_TG4_OFFSET_LOGICAL;
4475 else
4476 opcode = SHADER_OPCODE_TG4_LOGICAL;
4477 break;
4478 case nir_texop_texture_samples:
4479 opcode = SHADER_OPCODE_SAMPLEINFO_LOGICAL;
4480 break;
4481 case nir_texop_samples_identical: {
4482 fs_reg dst = retype(get_nir_dest(instr->dest), BRW_REGISTER_TYPE_D);
4483
4484 /* If mcs is an immediate value, it means there is no MCS. In that case
4485 * just return false.
4486 */
4487 if (srcs[TEX_LOGICAL_SRC_MCS].file == BRW_IMMEDIATE_VALUE) {
4488 bld.MOV(dst, brw_imm_ud(0u));
4489 } else if ((key_tex->msaa_16 & (1 << sampler))) {
4490 fs_reg tmp = vgrf(glsl_type::uint_type);
4491 bld.OR(tmp, srcs[TEX_LOGICAL_SRC_MCS],
4492 offset(srcs[TEX_LOGICAL_SRC_MCS], bld, 1));
4493 bld.CMP(dst, tmp, brw_imm_ud(0u), BRW_CONDITIONAL_EQ);
4494 } else {
4495 bld.CMP(dst, srcs[TEX_LOGICAL_SRC_MCS], brw_imm_ud(0u),
4496 BRW_CONDITIONAL_EQ);
4497 }
4498 return;
4499 }
4500 default:
4501 unreachable("unknown texture opcode");
4502 }
4503
4504 fs_reg dst = bld.vgrf(brw_type_for_nir_type(instr->dest_type), 4);
4505 fs_inst *inst = bld.emit(opcode, dst, srcs, ARRAY_SIZE(srcs));
4506
4507 const unsigned dest_size = nir_tex_instr_dest_size(instr);
4508 if (devinfo->gen >= 9 &&
4509 instr->op != nir_texop_tg4 && instr->op != nir_texop_query_levels) {
4510 unsigned write_mask = instr->dest.is_ssa ?
4511 nir_ssa_def_components_read(&instr->dest.ssa):
4512 (1 << dest_size) - 1;
4513 assert(write_mask != 0); /* dead code should have been eliminated */
4514 inst->regs_written = _mesa_fls(write_mask) * dispatch_width / 8;
4515 } else {
4516 inst->regs_written = 4 * dispatch_width / 8;
4517 }
4518
4519 if (srcs[TEX_LOGICAL_SRC_SHADOW_C].file != BAD_FILE)
4520 inst->shadow_compare = true;
4521
4522 if (srcs[TEX_LOGICAL_SRC_OFFSET_VALUE].file == IMM)
4523 inst->offset = srcs[TEX_LOGICAL_SRC_OFFSET_VALUE].ud;
4524
4525 if (instr->op == nir_texop_tg4) {
4526 if (instr->component == 1 &&
4527 key_tex->gather_channel_quirk_mask & (1 << texture)) {
4528 /* gather4 sampler is broken for green channel on RG32F --
4529 * we must ask for blue instead.
4530 */
4531 inst->offset |= 2 << 16;
4532 } else {
4533 inst->offset |= instr->component << 16;
4534 }
4535
4536 if (devinfo->gen == 6)
4537 emit_gen6_gather_wa(key_tex->gen6_gather_wa[texture], dst);
4538 }
4539
4540 fs_reg nir_dest[4];
4541 for (unsigned i = 0; i < dest_size; i++)
4542 nir_dest[i] = offset(dst, bld, i);
4543
4544 if (instr->op == nir_texop_query_levels) {
4545 /* # levels is in .w */
4546 nir_dest[0] = offset(dst, bld, 3);
4547 } else if (instr->op == nir_texop_txs &&
4548 dest_size >= 3 && devinfo->gen < 7) {
4549 /* Gen4-6 return 0 instead of 1 for single layer surfaces. */
4550 fs_reg depth = offset(dst, bld, 2);
4551 nir_dest[2] = vgrf(glsl_type::int_type);
4552 bld.emit_minmax(nir_dest[2], depth, brw_imm_d(1), BRW_CONDITIONAL_GE);
4553 }
4554
4555 bld.LOAD_PAYLOAD(get_nir_dest(instr->dest), nir_dest, dest_size, 0);
4556 }
4557
4558 void
4559 fs_visitor::nir_emit_jump(const fs_builder &bld, nir_jump_instr *instr)
4560 {
4561 switch (instr->type) {
4562 case nir_jump_break:
4563 bld.emit(BRW_OPCODE_BREAK);
4564 break;
4565 case nir_jump_continue:
4566 bld.emit(BRW_OPCODE_CONTINUE);
4567 break;
4568 case nir_jump_return:
4569 default:
4570 unreachable("unknown jump");
4571 }
4572 }
4573
4574 /**
4575 * This helper takes the result of a load operation that reads 32-bit elements
4576 * in this format:
4577 *
4578 * x x x x x x x x
4579 * y y y y y y y y
4580 * z z z z z z z z
4581 * w w w w w w w w
4582 *
4583 * and shuffles the data to get this:
4584 *
4585 * x y x y x y x y
4586 * x y x y x y x y
4587 * z w z w z w z w
4588 * z w z w z w z w
4589 *
4590 * Which is exactly what we want if the load is reading 64-bit components
4591 * like doubles, where x represents the low 32-bit of the x double component
4592 * and y represents the high 32-bit of the x double component (likewise with
4593 * z and w for double component y). The parameter @components represents
4594 * the number of 64-bit components present in @src. This would typically be
4595 * 2 at most, since we can only fit 2 double elements in the result of a
4596 * vec4 load.
4597 *
4598 * Notice that @dst and @src can be the same register.
4599 */
4600 void
4601 shuffle_32bit_load_result_to_64bit_data(const fs_builder &bld,
4602 const fs_reg &dst,
4603 const fs_reg &src,
4604 uint32_t components)
4605 {
4606 assert(type_sz(src.type) == 4);
4607 assert(type_sz(dst.type) == 8);
4608
4609 /* A temporary that we will use to shuffle the 32-bit data of each
4610 * component in the vector into valid 64-bit data. We can't write directly
4611 * to dst because dst can be (and would usually be) the same as src
4612 * and in that case the first MOV in the loop below would overwrite the
4613 * data read in the second MOV.
4614 */
4615 fs_reg tmp = bld.vgrf(dst.type);
4616
4617 for (unsigned i = 0; i < components; i++) {
4618 const fs_reg component_i = offset(src, bld, 2 * i);
4619
4620 bld.MOV(subscript(tmp, src.type, 0), component_i);
4621 bld.MOV(subscript(tmp, src.type, 1), offset(component_i, bld, 1));
4622
4623 bld.MOV(offset(dst, bld, i), tmp);
4624 }
4625 }
4626
4627 /**
4628 * This helper does the inverse operation of
4629 * SHUFFLE_32BIT_LOAD_RESULT_TO_64BIT_DATA.
4630 *
4631 * We need to do this when we are going to use untyped write messsages that
4632 * operate with 32-bit components in order to arrange our 64-bit data to be
4633 * in the expected layout.
4634 *
4635 * Notice that callers of this function, unlike in the case of the inverse
4636 * operation, would typically need to call this with dst and src being
4637 * different registers, since they would otherwise corrupt the original
4638 * 64-bit data they are about to write. Because of this the function checks
4639 * that the src and dst regions involved in the operation do not overlap.
4640 */
4641 void
4642 shuffle_64bit_data_for_32bit_write(const fs_builder &bld,
4643 const fs_reg &dst,
4644 const fs_reg &src,
4645 uint32_t components)
4646 {
4647 assert(type_sz(src.type) == 8);
4648 assert(type_sz(dst.type) == 4);
4649
4650 assert(!src.in_range(dst, 2 * components * bld.dispatch_width() / 8));
4651
4652 for (unsigned i = 0; i < components; i++) {
4653 const fs_reg component_i = offset(src, bld, i);
4654 bld.MOV(offset(dst, bld, 2 * i), subscript(component_i, dst.type, 0));
4655 bld.MOV(offset(dst, bld, 2 * i + 1), subscript(component_i, dst.type, 1));
4656 }
4657 }
4658
4659 fs_reg
4660 setup_imm_df(const fs_builder &bld, double v)
4661 {
4662 const struct brw_device_info *devinfo = bld.shader->devinfo;
4663 assert(devinfo->gen >= 7);
4664
4665 if (devinfo->gen >= 8)
4666 return brw_imm_df(v);
4667
4668 /* gen7.5 does not support DF immediates straighforward but the DIM
4669 * instruction allows to set the 64-bit immediate value.
4670 */
4671 if (devinfo->is_haswell) {
4672 const fs_builder ubld = bld.exec_all();
4673 fs_reg dst = ubld.vgrf(BRW_REGISTER_TYPE_DF, 1);
4674 ubld.DIM(dst, brw_imm_df(v));
4675 return component(dst, 0);
4676 }
4677
4678 /* gen7 does not support DF immediates, so we generate a 64-bit constant by
4679 * writing the low 32-bit of the constant to suboffset 0 of a VGRF and
4680 * the high 32-bit to suboffset 4 and then applying a stride of 0.
4681 *
4682 * Alternatively, we could also produce a normal VGRF (without stride 0)
4683 * by writing to all the channels in the VGRF, however, that would hit the
4684 * gen7 bug where we have to split writes that span more than 1 register
4685 * into instructions with a width of 4 (otherwise the write to the second
4686 * register written runs into an execmask hardware bug) which isn't very
4687 * nice.
4688 */
4689 union {
4690 double d;
4691 struct {
4692 uint32_t i1;
4693 uint32_t i2;
4694 };
4695 } di;
4696
4697 di.d = v;
4698
4699 const fs_builder ubld = bld.exec_all().group(1, 0);
4700 const fs_reg tmp = ubld.vgrf(BRW_REGISTER_TYPE_UD, 2);
4701 ubld.MOV(tmp, brw_imm_ud(di.i1));
4702 ubld.MOV(horiz_offset(tmp, 1), brw_imm_ud(di.i2));
4703
4704 return component(retype(tmp, BRW_REGISTER_TYPE_DF), 0);
4705 }