Merge remote-tracking branch 'mesa-public/master' into vulkan
[mesa.git] / src / mesa / drivers / dri / i965 / brw_fs_nir.cpp
1 /*
2 * Copyright © 2010 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "glsl/ir.h"
25 #include "glsl/ir_optimization.h"
26 #include "glsl/nir/glsl_to_nir.h"
27 #include "main/shaderimage.h"
28 #include "program/prog_to_nir.h"
29 #include "brw_fs.h"
30 #include "brw_fs_surface_builder.h"
31 #include "brw_nir.h"
32 #include "brw_fs_surface_builder.h"
33
34 using namespace brw;
35 using namespace brw::surface_access;
36
37 void
38 fs_visitor::emit_nir_code()
39 {
40 /* emit the arrays used for inputs and outputs - load/store intrinsics will
41 * be converted to reads/writes of these arrays
42 */
43 nir_setup_inputs();
44 nir_setup_outputs();
45 nir_setup_uniforms();
46 nir_emit_system_values();
47
48 /* get the main function and emit it */
49 nir_foreach_overload(nir, overload) {
50 assert(strcmp(overload->function->name, "main") == 0);
51 assert(overload->impl);
52 nir_emit_impl(overload->impl);
53 }
54 }
55
56 void
57 fs_visitor::nir_setup_inputs()
58 {
59 if (stage != MESA_SHADER_FRAGMENT)
60 return;
61
62 nir_inputs = bld.vgrf(BRW_REGISTER_TYPE_F, nir->num_inputs);
63
64 nir_foreach_variable(var, &nir->inputs) {
65 fs_reg input = offset(nir_inputs, bld, var->data.driver_location);
66
67 fs_reg reg;
68 if (var->data.location == VARYING_SLOT_POS) {
69 reg = *emit_fragcoord_interpolation(var->data.pixel_center_integer,
70 var->data.origin_upper_left);
71 emit_percomp(bld, fs_inst(BRW_OPCODE_MOV, bld.dispatch_width(),
72 input, reg), 0xF);
73 } else {
74 emit_general_interpolation(input, var->name, var->type,
75 (glsl_interp_qualifier) var->data.interpolation,
76 var->data.location, var->data.centroid,
77 var->data.sample);
78 }
79 }
80 }
81
82 void
83 fs_visitor::nir_setup_outputs()
84 {
85 brw_wm_prog_key *key = (brw_wm_prog_key*) this->key;
86
87 nir_outputs = bld.vgrf(BRW_REGISTER_TYPE_F, nir->num_outputs);
88
89 nir_foreach_variable(var, &nir->outputs) {
90 fs_reg reg = offset(nir_outputs, bld, var->data.driver_location);
91
92 int vector_elements = var->type->without_array()->vector_elements;
93
94 switch (stage) {
95 case MESA_SHADER_VERTEX:
96 for (unsigned int i = 0; i < ALIGN(type_size_scalar(var->type), 4) / 4; i++) {
97 int output = var->data.location + i;
98 this->outputs[output] = offset(reg, bld, 4 * i);
99 this->output_components[output] = vector_elements;
100 }
101 break;
102 case MESA_SHADER_FRAGMENT:
103 if (var->data.index > 0) {
104 assert(var->data.location == FRAG_RESULT_DATA0);
105 assert(var->data.index == 1);
106 this->dual_src_output = reg;
107 this->do_dual_src = true;
108 } else if (var->data.location == FRAG_RESULT_COLOR) {
109 /* Writing gl_FragColor outputs to all color regions. */
110 for (unsigned int i = 0; i < MAX2(key->nr_color_regions, 1); i++) {
111 this->outputs[i] = reg;
112 this->output_components[i] = 4;
113 }
114 } else if (var->data.location == FRAG_RESULT_DEPTH) {
115 this->frag_depth = reg;
116 } else if (var->data.location == FRAG_RESULT_SAMPLE_MASK) {
117 this->sample_mask = reg;
118 } else {
119 /* gl_FragData or a user-defined FS output */
120 assert(var->data.location >= FRAG_RESULT_DATA0 &&
121 var->data.location < FRAG_RESULT_DATA0+BRW_MAX_DRAW_BUFFERS);
122
123 /* General color output. */
124 for (unsigned int i = 0; i < MAX2(1, var->type->length); i++) {
125 int output = var->data.location - FRAG_RESULT_DATA0 + i;
126 this->outputs[output] = offset(reg, bld, vector_elements * i);
127 this->output_components[output] = vector_elements;
128 }
129 }
130 break;
131 default:
132 unreachable("unhandled shader stage");
133 }
134 }
135 }
136
137 void
138 fs_visitor::nir_setup_uniforms()
139 {
140 if (dispatch_width != 8)
141 return;
142
143 uniforms = nir->num_uniforms;
144
145 nir_foreach_variable(var, &nir->uniforms) {
146 /* UBO's and atomics don't take up space in the uniform file */
147 if (var->interface_type != NULL || var->type->contains_atomic())
148 continue;
149
150 if (type_size_scalar(var->type) > 0)
151 param_size[var->data.driver_location] = type_size_scalar(var->type);
152 }
153 }
154
155 static bool
156 emit_system_values_block(nir_block *block, void *void_visitor)
157 {
158 fs_visitor *v = (fs_visitor *)void_visitor;
159 fs_reg *reg;
160
161 nir_foreach_instr(block, instr) {
162 if (instr->type != nir_instr_type_intrinsic)
163 continue;
164
165 nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
166 switch (intrin->intrinsic) {
167 case nir_intrinsic_load_vertex_id:
168 unreachable("should be lowered by lower_vertex_id().");
169
170 case nir_intrinsic_load_vertex_id_zero_base:
171 assert(v->stage == MESA_SHADER_VERTEX);
172 reg = &v->nir_system_values[SYSTEM_VALUE_VERTEX_ID_ZERO_BASE];
173 if (reg->file == BAD_FILE)
174 *reg = *v->emit_vs_system_value(SYSTEM_VALUE_VERTEX_ID_ZERO_BASE);
175 break;
176
177 case nir_intrinsic_load_base_vertex:
178 assert(v->stage == MESA_SHADER_VERTEX);
179 reg = &v->nir_system_values[SYSTEM_VALUE_BASE_VERTEX];
180 if (reg->file == BAD_FILE)
181 *reg = *v->emit_vs_system_value(SYSTEM_VALUE_BASE_VERTEX);
182 break;
183
184 case nir_intrinsic_load_instance_id:
185 assert(v->stage == MESA_SHADER_VERTEX);
186 reg = &v->nir_system_values[SYSTEM_VALUE_INSTANCE_ID];
187 if (reg->file == BAD_FILE)
188 *reg = *v->emit_vs_system_value(SYSTEM_VALUE_INSTANCE_ID);
189 break;
190
191 case nir_intrinsic_load_sample_pos:
192 assert(v->stage == MESA_SHADER_FRAGMENT);
193 reg = &v->nir_system_values[SYSTEM_VALUE_SAMPLE_POS];
194 if (reg->file == BAD_FILE)
195 *reg = *v->emit_samplepos_setup();
196 break;
197
198 case nir_intrinsic_load_sample_id:
199 assert(v->stage == MESA_SHADER_FRAGMENT);
200 reg = &v->nir_system_values[SYSTEM_VALUE_SAMPLE_ID];
201 if (reg->file == BAD_FILE)
202 *reg = *v->emit_sampleid_setup();
203 break;
204
205 case nir_intrinsic_load_sample_mask_in:
206 assert(v->stage == MESA_SHADER_FRAGMENT);
207 assert(v->devinfo->gen >= 7);
208 reg = &v->nir_system_values[SYSTEM_VALUE_SAMPLE_MASK_IN];
209 if (reg->file == BAD_FILE)
210 *reg = fs_reg(retype(brw_vec8_grf(v->payload.sample_mask_in_reg, 0),
211 BRW_REGISTER_TYPE_D));
212 break;
213
214 case nir_intrinsic_load_local_invocation_id:
215 assert(v->stage == MESA_SHADER_COMPUTE);
216 reg = &v->nir_system_values[SYSTEM_VALUE_LOCAL_INVOCATION_ID];
217 if (reg->file == BAD_FILE)
218 *reg = *v->emit_cs_local_invocation_id_setup();
219 break;
220
221 case nir_intrinsic_load_work_group_id:
222 assert(v->stage == MESA_SHADER_COMPUTE);
223 reg = &v->nir_system_values[SYSTEM_VALUE_WORK_GROUP_ID];
224 if (reg->file == BAD_FILE)
225 *reg = *v->emit_cs_work_group_id_setup();
226 break;
227
228 default:
229 break;
230 }
231 }
232
233 return true;
234 }
235
236 void
237 fs_visitor::nir_emit_system_values()
238 {
239 nir_system_values = ralloc_array(mem_ctx, fs_reg, SYSTEM_VALUE_MAX);
240 nir_foreach_overload(nir, overload) {
241 assert(strcmp(overload->function->name, "main") == 0);
242 assert(overload->impl);
243 nir_foreach_block(overload->impl, emit_system_values_block, this);
244 }
245 }
246
247 void
248 fs_visitor::nir_emit_impl(nir_function_impl *impl)
249 {
250 nir_locals = reralloc(mem_ctx, nir_locals, fs_reg, impl->reg_alloc);
251 foreach_list_typed(nir_register, reg, node, &impl->registers) {
252 unsigned array_elems =
253 reg->num_array_elems == 0 ? 1 : reg->num_array_elems;
254 unsigned size = array_elems * reg->num_components;
255 nir_locals[reg->index] = bld.vgrf(BRW_REGISTER_TYPE_F, size);
256 }
257
258 nir_ssa_values = reralloc(mem_ctx, nir_ssa_values, fs_reg,
259 impl->ssa_alloc);
260
261 nir_emit_cf_list(&impl->body);
262 }
263
264 void
265 fs_visitor::nir_emit_cf_list(exec_list *list)
266 {
267 exec_list_validate(list);
268 foreach_list_typed(nir_cf_node, node, node, list) {
269 switch (node->type) {
270 case nir_cf_node_if:
271 nir_emit_if(nir_cf_node_as_if(node));
272 break;
273
274 case nir_cf_node_loop:
275 nir_emit_loop(nir_cf_node_as_loop(node));
276 break;
277
278 case nir_cf_node_block:
279 nir_emit_block(nir_cf_node_as_block(node));
280 break;
281
282 default:
283 unreachable("Invalid CFG node block");
284 }
285 }
286 }
287
288 void
289 fs_visitor::nir_emit_if(nir_if *if_stmt)
290 {
291 /* first, put the condition into f0 */
292 fs_inst *inst = bld.MOV(bld.null_reg_d(),
293 retype(get_nir_src(if_stmt->condition),
294 BRW_REGISTER_TYPE_D));
295 inst->conditional_mod = BRW_CONDITIONAL_NZ;
296
297 bld.IF(BRW_PREDICATE_NORMAL);
298
299 nir_emit_cf_list(&if_stmt->then_list);
300
301 /* note: if the else is empty, dead CF elimination will remove it */
302 bld.emit(BRW_OPCODE_ELSE);
303
304 nir_emit_cf_list(&if_stmt->else_list);
305
306 bld.emit(BRW_OPCODE_ENDIF);
307 }
308
309 void
310 fs_visitor::nir_emit_loop(nir_loop *loop)
311 {
312 bld.emit(BRW_OPCODE_DO);
313
314 nir_emit_cf_list(&loop->body);
315
316 bld.emit(BRW_OPCODE_WHILE);
317 }
318
319 void
320 fs_visitor::nir_emit_block(nir_block *block)
321 {
322 nir_foreach_instr(block, instr) {
323 nir_emit_instr(instr);
324 }
325 }
326
327 void
328 fs_visitor::nir_emit_instr(nir_instr *instr)
329 {
330 const fs_builder abld = bld.annotate(NULL, instr);
331
332 switch (instr->type) {
333 case nir_instr_type_alu:
334 nir_emit_alu(abld, nir_instr_as_alu(instr));
335 break;
336
337 case nir_instr_type_intrinsic:
338 nir_emit_intrinsic(abld, nir_instr_as_intrinsic(instr));
339 break;
340
341 case nir_instr_type_tex:
342 nir_emit_texture(abld, nir_instr_as_tex(instr));
343 break;
344
345 case nir_instr_type_load_const:
346 nir_emit_load_const(abld, nir_instr_as_load_const(instr));
347 break;
348
349 case nir_instr_type_ssa_undef:
350 nir_emit_undef(abld, nir_instr_as_ssa_undef(instr));
351 break;
352
353 case nir_instr_type_jump:
354 nir_emit_jump(abld, nir_instr_as_jump(instr));
355 break;
356
357 default:
358 unreachable("unknown instruction type");
359 }
360 }
361
362 bool
363 fs_visitor::optimize_frontfacing_ternary(nir_alu_instr *instr,
364 const fs_reg &result)
365 {
366 if (!instr->src[0].src.is_ssa ||
367 instr->src[0].src.ssa->parent_instr->type != nir_instr_type_intrinsic)
368 return false;
369
370 nir_intrinsic_instr *src0 =
371 nir_instr_as_intrinsic(instr->src[0].src.ssa->parent_instr);
372
373 if (src0->intrinsic != nir_intrinsic_load_front_face)
374 return false;
375
376 nir_const_value *value1 = nir_src_as_const_value(instr->src[1].src);
377 if (!value1 || fabsf(value1->f[0]) != 1.0f)
378 return false;
379
380 nir_const_value *value2 = nir_src_as_const_value(instr->src[2].src);
381 if (!value2 || fabsf(value2->f[0]) != 1.0f)
382 return false;
383
384 fs_reg tmp = vgrf(glsl_type::int_type);
385
386 if (devinfo->gen >= 6) {
387 /* Bit 15 of g0.0 is 0 if the polygon is front facing. */
388 fs_reg g0 = fs_reg(retype(brw_vec1_grf(0, 0), BRW_REGISTER_TYPE_W));
389
390 /* For (gl_FrontFacing ? 1.0 : -1.0), emit:
391 *
392 * or(8) tmp.1<2>W g0.0<0,1,0>W 0x00003f80W
393 * and(8) dst<1>D tmp<8,8,1>D 0xbf800000D
394 *
395 * and negate g0.0<0,1,0>W for (gl_FrontFacing ? -1.0 : 1.0).
396 *
397 * This negation looks like it's safe in practice, because bits 0:4 will
398 * surely be TRIANGLES
399 */
400
401 if (value1->f[0] == -1.0f) {
402 g0.negate = true;
403 }
404
405 tmp.type = BRW_REGISTER_TYPE_W;
406 tmp.subreg_offset = 2;
407 tmp.stride = 2;
408
409 fs_inst *or_inst = bld.OR(tmp, g0, fs_reg(0x3f80));
410 or_inst->src[1].type = BRW_REGISTER_TYPE_UW;
411
412 tmp.type = BRW_REGISTER_TYPE_D;
413 tmp.subreg_offset = 0;
414 tmp.stride = 1;
415 } else {
416 /* Bit 31 of g1.6 is 0 if the polygon is front facing. */
417 fs_reg g1_6 = fs_reg(retype(brw_vec1_grf(1, 6), BRW_REGISTER_TYPE_D));
418
419 /* For (gl_FrontFacing ? 1.0 : -1.0), emit:
420 *
421 * or(8) tmp<1>D g1.6<0,1,0>D 0x3f800000D
422 * and(8) dst<1>D tmp<8,8,1>D 0xbf800000D
423 *
424 * and negate g1.6<0,1,0>D for (gl_FrontFacing ? -1.0 : 1.0).
425 *
426 * This negation looks like it's safe in practice, because bits 0:4 will
427 * surely be TRIANGLES
428 */
429
430 if (value1->f[0] == -1.0f) {
431 g1_6.negate = true;
432 }
433
434 bld.OR(tmp, g1_6, fs_reg(0x3f800000));
435 }
436 bld.AND(retype(result, BRW_REGISTER_TYPE_D), tmp, fs_reg(0xbf800000));
437
438 return true;
439 }
440
441 void
442 fs_visitor::nir_emit_alu(const fs_builder &bld, nir_alu_instr *instr)
443 {
444 struct brw_wm_prog_key *fs_key = (struct brw_wm_prog_key *) this->key;
445 fs_inst *inst;
446
447 fs_reg result = get_nir_dest(instr->dest.dest);
448 result.type = brw_type_for_nir_type(nir_op_infos[instr->op].output_type);
449
450 fs_reg op[4];
451 for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
452 op[i] = get_nir_src(instr->src[i].src);
453 op[i].type = brw_type_for_nir_type(nir_op_infos[instr->op].input_types[i]);
454 op[i].abs = instr->src[i].abs;
455 op[i].negate = instr->src[i].negate;
456 }
457
458 /* We get a bunch of mov's out of the from_ssa pass and they may still
459 * be vectorized. We'll handle them as a special-case. We'll also
460 * handle vecN here because it's basically the same thing.
461 */
462 switch (instr->op) {
463 case nir_op_imov:
464 case nir_op_fmov:
465 case nir_op_vec2:
466 case nir_op_vec3:
467 case nir_op_vec4: {
468 fs_reg temp = result;
469 bool need_extra_copy = false;
470 for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
471 if (!instr->src[i].src.is_ssa &&
472 instr->dest.dest.reg.reg == instr->src[i].src.reg.reg) {
473 need_extra_copy = true;
474 temp = bld.vgrf(result.type, 4);
475 break;
476 }
477 }
478
479 for (unsigned i = 0; i < 4; i++) {
480 if (!(instr->dest.write_mask & (1 << i)))
481 continue;
482
483 if (instr->op == nir_op_imov || instr->op == nir_op_fmov) {
484 inst = bld.MOV(offset(temp, bld, i),
485 offset(op[0], bld, instr->src[0].swizzle[i]));
486 } else {
487 inst = bld.MOV(offset(temp, bld, i),
488 offset(op[i], bld, instr->src[i].swizzle[0]));
489 }
490 inst->saturate = instr->dest.saturate;
491 }
492
493 /* In this case the source and destination registers were the same,
494 * so we need to insert an extra set of moves in order to deal with
495 * any swizzling.
496 */
497 if (need_extra_copy) {
498 for (unsigned i = 0; i < 4; i++) {
499 if (!(instr->dest.write_mask & (1 << i)))
500 continue;
501
502 bld.MOV(offset(result, bld, i), offset(temp, bld, i));
503 }
504 }
505 return;
506 }
507 default:
508 break;
509 }
510
511 /* At this point, we have dealt with any instruction that operates on
512 * more than a single channel. Therefore, we can just adjust the source
513 * and destination registers for that channel and emit the instruction.
514 */
515 unsigned channel = 0;
516 if (nir_op_infos[instr->op].output_size == 0) {
517 /* Since NIR is doing the scalarizing for us, we should only ever see
518 * vectorized operations with a single channel.
519 */
520 assert(_mesa_bitcount(instr->dest.write_mask) == 1);
521 channel = ffs(instr->dest.write_mask) - 1;
522
523 result = offset(result, bld, channel);
524 }
525
526 for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
527 assert(nir_op_infos[instr->op].input_sizes[i] < 2);
528 op[i] = offset(op[i], bld, instr->src[i].swizzle[channel]);
529 }
530
531 switch (instr->op) {
532 case nir_op_i2f:
533 case nir_op_u2f:
534 inst = bld.MOV(result, op[0]);
535 inst->saturate = instr->dest.saturate;
536 break;
537
538 case nir_op_f2i:
539 case nir_op_f2u:
540 bld.MOV(result, op[0]);
541 break;
542
543 case nir_op_fsign: {
544 /* AND(val, 0x80000000) gives the sign bit.
545 *
546 * Predicated OR ORs 1.0 (0x3f800000) with the sign bit if val is not
547 * zero.
548 */
549 bld.CMP(bld.null_reg_f(), op[0], fs_reg(0.0f), BRW_CONDITIONAL_NZ);
550
551 fs_reg result_int = retype(result, BRW_REGISTER_TYPE_UD);
552 op[0].type = BRW_REGISTER_TYPE_UD;
553 result.type = BRW_REGISTER_TYPE_UD;
554 bld.AND(result_int, op[0], fs_reg(0x80000000u));
555
556 inst = bld.OR(result_int, result_int, fs_reg(0x3f800000u));
557 inst->predicate = BRW_PREDICATE_NORMAL;
558 if (instr->dest.saturate) {
559 inst = bld.MOV(result, result);
560 inst->saturate = true;
561 }
562 break;
563 }
564
565 case nir_op_isign:
566 /* ASR(val, 31) -> negative val generates 0xffffffff (signed -1).
567 * -> non-negative val generates 0x00000000.
568 * Predicated OR sets 1 if val is positive.
569 */
570 bld.CMP(bld.null_reg_d(), op[0], fs_reg(0), BRW_CONDITIONAL_G);
571 bld.ASR(result, op[0], fs_reg(31));
572 inst = bld.OR(result, result, fs_reg(1));
573 inst->predicate = BRW_PREDICATE_NORMAL;
574 break;
575
576 case nir_op_frcp:
577 inst = bld.emit(SHADER_OPCODE_RCP, result, op[0]);
578 inst->saturate = instr->dest.saturate;
579 break;
580
581 case nir_op_fexp2:
582 inst = bld.emit(SHADER_OPCODE_EXP2, result, op[0]);
583 inst->saturate = instr->dest.saturate;
584 break;
585
586 case nir_op_flog2:
587 inst = bld.emit(SHADER_OPCODE_LOG2, result, op[0]);
588 inst->saturate = instr->dest.saturate;
589 break;
590
591 case nir_op_fsin:
592 inst = bld.emit(SHADER_OPCODE_SIN, result, op[0]);
593 inst->saturate = instr->dest.saturate;
594 break;
595
596 case nir_op_fcos:
597 inst = bld.emit(SHADER_OPCODE_COS, result, op[0]);
598 inst->saturate = instr->dest.saturate;
599 break;
600
601 case nir_op_fddx:
602 if (fs_key->high_quality_derivatives) {
603 inst = bld.emit(FS_OPCODE_DDX_FINE, result, op[0]);
604 } else {
605 inst = bld.emit(FS_OPCODE_DDX_COARSE, result, op[0]);
606 }
607 inst->saturate = instr->dest.saturate;
608 break;
609 case nir_op_fddx_fine:
610 inst = bld.emit(FS_OPCODE_DDX_FINE, result, op[0]);
611 inst->saturate = instr->dest.saturate;
612 break;
613 case nir_op_fddx_coarse:
614 inst = bld.emit(FS_OPCODE_DDX_COARSE, result, op[0]);
615 inst->saturate = instr->dest.saturate;
616 break;
617 case nir_op_fddy:
618 if (fs_key->high_quality_derivatives) {
619 inst = bld.emit(FS_OPCODE_DDY_FINE, result, op[0],
620 fs_reg(fs_key->render_to_fbo));
621 } else {
622 inst = bld.emit(FS_OPCODE_DDY_COARSE, result, op[0],
623 fs_reg(fs_key->render_to_fbo));
624 }
625 inst->saturate = instr->dest.saturate;
626 break;
627 case nir_op_fddy_fine:
628 inst = bld.emit(FS_OPCODE_DDY_FINE, result, op[0],
629 fs_reg(fs_key->render_to_fbo));
630 inst->saturate = instr->dest.saturate;
631 break;
632 case nir_op_fddy_coarse:
633 inst = bld.emit(FS_OPCODE_DDY_COARSE, result, op[0],
634 fs_reg(fs_key->render_to_fbo));
635 inst->saturate = instr->dest.saturate;
636 break;
637
638 case nir_op_fadd:
639 case nir_op_iadd:
640 inst = bld.ADD(result, op[0], op[1]);
641 inst->saturate = instr->dest.saturate;
642 break;
643
644 case nir_op_fmul:
645 inst = bld.MUL(result, op[0], op[1]);
646 inst->saturate = instr->dest.saturate;
647 break;
648
649 case nir_op_imul:
650 bld.MUL(result, op[0], op[1]);
651 break;
652
653 case nir_op_imul_high:
654 case nir_op_umul_high:
655 bld.emit(SHADER_OPCODE_MULH, result, op[0], op[1]);
656 break;
657
658 case nir_op_idiv:
659 case nir_op_udiv:
660 bld.emit(SHADER_OPCODE_INT_QUOTIENT, result, op[0], op[1]);
661 break;
662
663 case nir_op_uadd_carry:
664 unreachable("Should have been lowered by carry_to_arith().");
665
666 case nir_op_usub_borrow:
667 unreachable("Should have been lowered by borrow_to_arith().");
668
669 case nir_op_umod:
670 bld.emit(SHADER_OPCODE_INT_REMAINDER, result, op[0], op[1]);
671 break;
672
673 case nir_op_flt:
674 case nir_op_ilt:
675 case nir_op_ult:
676 bld.CMP(result, op[0], op[1], BRW_CONDITIONAL_L);
677 break;
678
679 case nir_op_fge:
680 case nir_op_ige:
681 case nir_op_uge:
682 bld.CMP(result, op[0], op[1], BRW_CONDITIONAL_GE);
683 break;
684
685 case nir_op_feq:
686 case nir_op_ieq:
687 bld.CMP(result, op[0], op[1], BRW_CONDITIONAL_Z);
688 break;
689
690 case nir_op_fne:
691 case nir_op_ine:
692 bld.CMP(result, op[0], op[1], BRW_CONDITIONAL_NZ);
693 break;
694
695 case nir_op_inot:
696 if (devinfo->gen >= 8) {
697 op[0] = resolve_source_modifiers(op[0]);
698 }
699 bld.NOT(result, op[0]);
700 break;
701 case nir_op_ixor:
702 if (devinfo->gen >= 8) {
703 op[0] = resolve_source_modifiers(op[0]);
704 op[1] = resolve_source_modifiers(op[1]);
705 }
706 bld.XOR(result, op[0], op[1]);
707 break;
708 case nir_op_ior:
709 if (devinfo->gen >= 8) {
710 op[0] = resolve_source_modifiers(op[0]);
711 op[1] = resolve_source_modifiers(op[1]);
712 }
713 bld.OR(result, op[0], op[1]);
714 break;
715 case nir_op_iand:
716 if (devinfo->gen >= 8) {
717 op[0] = resolve_source_modifiers(op[0]);
718 op[1] = resolve_source_modifiers(op[1]);
719 }
720 bld.AND(result, op[0], op[1]);
721 break;
722
723 case nir_op_fdot2:
724 case nir_op_fdot3:
725 case nir_op_fdot4:
726 case nir_op_bany2:
727 case nir_op_bany3:
728 case nir_op_bany4:
729 case nir_op_ball2:
730 case nir_op_ball3:
731 case nir_op_ball4:
732 case nir_op_ball_fequal2:
733 case nir_op_ball_iequal2:
734 case nir_op_ball_fequal3:
735 case nir_op_ball_iequal3:
736 case nir_op_ball_fequal4:
737 case nir_op_ball_iequal4:
738 case nir_op_bany_fnequal2:
739 case nir_op_bany_inequal2:
740 case nir_op_bany_fnequal3:
741 case nir_op_bany_inequal3:
742 case nir_op_bany_fnequal4:
743 case nir_op_bany_inequal4:
744 unreachable("Lowered by nir_lower_alu_reductions");
745
746 case nir_op_fnoise1_1:
747 case nir_op_fnoise1_2:
748 case nir_op_fnoise1_3:
749 case nir_op_fnoise1_4:
750 case nir_op_fnoise2_1:
751 case nir_op_fnoise2_2:
752 case nir_op_fnoise2_3:
753 case nir_op_fnoise2_4:
754 case nir_op_fnoise3_1:
755 case nir_op_fnoise3_2:
756 case nir_op_fnoise3_3:
757 case nir_op_fnoise3_4:
758 case nir_op_fnoise4_1:
759 case nir_op_fnoise4_2:
760 case nir_op_fnoise4_3:
761 case nir_op_fnoise4_4:
762 unreachable("not reached: should be handled by lower_noise");
763
764 case nir_op_ldexp:
765 unreachable("not reached: should be handled by ldexp_to_arith()");
766
767 case nir_op_fsqrt:
768 inst = bld.emit(SHADER_OPCODE_SQRT, result, op[0]);
769 inst->saturate = instr->dest.saturate;
770 break;
771
772 case nir_op_frsq:
773 inst = bld.emit(SHADER_OPCODE_RSQ, result, op[0]);
774 inst->saturate = instr->dest.saturate;
775 break;
776
777 case nir_op_b2i:
778 case nir_op_b2f:
779 bld.MOV(result, negate(op[0]));
780 break;
781
782 case nir_op_f2b:
783 bld.CMP(result, op[0], fs_reg(0.0f), BRW_CONDITIONAL_NZ);
784 break;
785 case nir_op_i2b:
786 bld.CMP(result, op[0], fs_reg(0), BRW_CONDITIONAL_NZ);
787 break;
788
789 case nir_op_ftrunc:
790 inst = bld.RNDZ(result, op[0]);
791 inst->saturate = instr->dest.saturate;
792 break;
793
794 case nir_op_fceil: {
795 op[0].negate = !op[0].negate;
796 fs_reg temp = vgrf(glsl_type::float_type);
797 bld.RNDD(temp, op[0]);
798 temp.negate = true;
799 inst = bld.MOV(result, temp);
800 inst->saturate = instr->dest.saturate;
801 break;
802 }
803 case nir_op_ffloor:
804 inst = bld.RNDD(result, op[0]);
805 inst->saturate = instr->dest.saturate;
806 break;
807 case nir_op_ffract:
808 inst = bld.FRC(result, op[0]);
809 inst->saturate = instr->dest.saturate;
810 break;
811 case nir_op_fround_even:
812 inst = bld.RNDE(result, op[0]);
813 inst->saturate = instr->dest.saturate;
814 break;
815
816 case nir_op_fmin:
817 case nir_op_imin:
818 case nir_op_umin:
819 if (devinfo->gen >= 6) {
820 inst = bld.emit(BRW_OPCODE_SEL, result, op[0], op[1]);
821 inst->conditional_mod = BRW_CONDITIONAL_L;
822 } else {
823 bld.CMP(bld.null_reg_d(), op[0], op[1], BRW_CONDITIONAL_L);
824 inst = bld.SEL(result, op[0], op[1]);
825 inst->predicate = BRW_PREDICATE_NORMAL;
826 }
827 inst->saturate = instr->dest.saturate;
828 break;
829
830 case nir_op_fmax:
831 case nir_op_imax:
832 case nir_op_umax:
833 if (devinfo->gen >= 6) {
834 inst = bld.emit(BRW_OPCODE_SEL, result, op[0], op[1]);
835 inst->conditional_mod = BRW_CONDITIONAL_GE;
836 } else {
837 bld.CMP(bld.null_reg_d(), op[0], op[1], BRW_CONDITIONAL_GE);
838 inst = bld.SEL(result, op[0], op[1]);
839 inst->predicate = BRW_PREDICATE_NORMAL;
840 }
841 inst->saturate = instr->dest.saturate;
842 break;
843
844 case nir_op_pack_snorm_2x16:
845 case nir_op_pack_snorm_4x8:
846 case nir_op_pack_unorm_2x16:
847 case nir_op_pack_unorm_4x8:
848 case nir_op_unpack_snorm_2x16:
849 case nir_op_unpack_snorm_4x8:
850 case nir_op_unpack_unorm_2x16:
851 case nir_op_unpack_unorm_4x8:
852 case nir_op_unpack_half_2x16:
853 case nir_op_pack_half_2x16:
854 unreachable("not reached: should be handled by lower_packing_builtins");
855
856 case nir_op_unpack_half_2x16_split_x:
857 inst = bld.emit(FS_OPCODE_UNPACK_HALF_2x16_SPLIT_X, result, op[0]);
858 inst->saturate = instr->dest.saturate;
859 break;
860 case nir_op_unpack_half_2x16_split_y:
861 inst = bld.emit(FS_OPCODE_UNPACK_HALF_2x16_SPLIT_Y, result, op[0]);
862 inst->saturate = instr->dest.saturate;
863 break;
864
865 case nir_op_fpow:
866 inst = bld.emit(SHADER_OPCODE_POW, result, op[0], op[1]);
867 inst->saturate = instr->dest.saturate;
868 break;
869
870 case nir_op_bitfield_reverse:
871 bld.BFREV(result, op[0]);
872 break;
873
874 case nir_op_bit_count:
875 bld.CBIT(result, op[0]);
876 break;
877
878 case nir_op_ufind_msb:
879 case nir_op_ifind_msb: {
880 bld.FBH(retype(result, BRW_REGISTER_TYPE_UD), op[0]);
881
882 /* FBH counts from the MSB side, while GLSL's findMSB() wants the count
883 * from the LSB side. If FBH didn't return an error (0xFFFFFFFF), then
884 * subtract the result from 31 to convert the MSB count into an LSB count.
885 */
886
887 bld.CMP(bld.null_reg_d(), result, fs_reg(-1), BRW_CONDITIONAL_NZ);
888 fs_reg neg_result(result);
889 neg_result.negate = true;
890 inst = bld.ADD(result, neg_result, fs_reg(31));
891 inst->predicate = BRW_PREDICATE_NORMAL;
892 break;
893 }
894
895 case nir_op_find_lsb:
896 bld.FBL(result, op[0]);
897 break;
898
899 case nir_op_ubitfield_extract:
900 case nir_op_ibitfield_extract:
901 bld.BFE(result, op[2], op[1], op[0]);
902 break;
903 case nir_op_bfm:
904 bld.BFI1(result, op[0], op[1]);
905 break;
906 case nir_op_bfi:
907 bld.BFI2(result, op[0], op[1], op[2]);
908 break;
909
910 case nir_op_bitfield_insert:
911 unreachable("not reached: should be handled by "
912 "lower_instructions::bitfield_insert_to_bfm_bfi");
913
914 case nir_op_ishl:
915 bld.SHL(result, op[0], op[1]);
916 break;
917 case nir_op_ishr:
918 bld.ASR(result, op[0], op[1]);
919 break;
920 case nir_op_ushr:
921 bld.SHR(result, op[0], op[1]);
922 break;
923
924 case nir_op_pack_half_2x16_split:
925 bld.emit(FS_OPCODE_PACK_HALF_2x16_SPLIT, result, op[0], op[1]);
926 break;
927
928 case nir_op_ffma:
929 inst = bld.MAD(result, op[2], op[1], op[0]);
930 inst->saturate = instr->dest.saturate;
931 break;
932
933 case nir_op_flrp:
934 inst = bld.LRP(result, op[0], op[1], op[2]);
935 inst->saturate = instr->dest.saturate;
936 break;
937
938 case nir_op_bcsel:
939 if (optimize_frontfacing_ternary(instr, result))
940 return;
941
942 bld.CMP(bld.null_reg_d(), op[0], fs_reg(0), BRW_CONDITIONAL_NZ);
943 inst = bld.SEL(result, op[1], op[2]);
944 inst->predicate = BRW_PREDICATE_NORMAL;
945 break;
946
947 default:
948 unreachable("unhandled instruction");
949 }
950
951 /* If we need to do a boolean resolve, replace the result with -(x & 1)
952 * to sign extend the low bit to 0/~0
953 */
954 if (devinfo->gen <= 5 &&
955 (instr->instr.pass_flags & BRW_NIR_BOOLEAN_MASK) == BRW_NIR_BOOLEAN_NEEDS_RESOLVE) {
956 fs_reg masked = vgrf(glsl_type::int_type);
957 bld.AND(masked, result, fs_reg(1));
958 masked.negate = true;
959 bld.MOV(retype(result, BRW_REGISTER_TYPE_D), masked);
960 }
961 }
962
963 void
964 fs_visitor::nir_emit_load_const(const fs_builder &bld,
965 nir_load_const_instr *instr)
966 {
967 fs_reg reg = bld.vgrf(BRW_REGISTER_TYPE_D, instr->def.num_components);
968
969 for (unsigned i = 0; i < instr->def.num_components; i++)
970 bld.MOV(offset(reg, bld, i), fs_reg(instr->value.i[i]));
971
972 nir_ssa_values[instr->def.index] = reg;
973 }
974
975 void
976 fs_visitor::nir_emit_undef(const fs_builder &bld, nir_ssa_undef_instr *instr)
977 {
978 nir_ssa_values[instr->def.index] = bld.vgrf(BRW_REGISTER_TYPE_D,
979 instr->def.num_components);
980 }
981
982 static fs_reg
983 fs_reg_for_nir_reg(fs_visitor *v, nir_register *nir_reg,
984 unsigned base_offset, nir_src *indirect)
985 {
986 fs_reg reg;
987
988 assert(!nir_reg->is_global);
989
990 reg = v->nir_locals[nir_reg->index];
991
992 reg = offset(reg, v->bld, base_offset * nir_reg->num_components);
993 if (indirect) {
994 int multiplier = nir_reg->num_components * (v->dispatch_width / 8);
995
996 reg.reladdr = new(v->mem_ctx) fs_reg(v->vgrf(glsl_type::int_type));
997 v->bld.MUL(*reg.reladdr, v->get_nir_src(*indirect),
998 fs_reg(multiplier));
999 }
1000
1001 return reg;
1002 }
1003
1004 fs_reg
1005 fs_visitor::get_nir_src(nir_src src)
1006 {
1007 fs_reg reg;
1008 if (src.is_ssa) {
1009 reg = nir_ssa_values[src.ssa->index];
1010 } else {
1011 reg = fs_reg_for_nir_reg(this, src.reg.reg, src.reg.base_offset,
1012 src.reg.indirect);
1013 }
1014
1015 /* to avoid floating-point denorm flushing problems, set the type by
1016 * default to D - instructions that need floating point semantics will set
1017 * this to F if they need to
1018 */
1019 return retype(reg, BRW_REGISTER_TYPE_D);
1020 }
1021
1022 fs_reg
1023 fs_visitor::get_nir_dest(nir_dest dest)
1024 {
1025 if (dest.is_ssa) {
1026 nir_ssa_values[dest.ssa.index] = bld.vgrf(BRW_REGISTER_TYPE_F,
1027 dest.ssa.num_components);
1028 return nir_ssa_values[dest.ssa.index];
1029 }
1030
1031 return fs_reg_for_nir_reg(this, dest.reg.reg, dest.reg.base_offset,
1032 dest.reg.indirect);
1033 }
1034
1035 fs_reg
1036 fs_visitor::get_nir_image_deref(const nir_deref_var *deref)
1037 {
1038 fs_reg image(UNIFORM, deref->var->data.driver_location,
1039 BRW_REGISTER_TYPE_UD);
1040
1041 if (deref->deref.child) {
1042 const nir_deref_array *deref_array =
1043 nir_deref_as_array(deref->deref.child);
1044 assert(deref->deref.child->deref_type == nir_deref_type_array &&
1045 deref_array->deref.child == NULL);
1046 const unsigned size = glsl_get_length(deref->var->type);
1047 const unsigned base = MIN2(deref_array->base_offset, size - 1);
1048
1049 image = offset(image, bld, base * BRW_IMAGE_PARAM_SIZE);
1050
1051 if (deref_array->deref_array_type == nir_deref_array_type_indirect) {
1052 fs_reg *tmp = new(mem_ctx) fs_reg(vgrf(glsl_type::int_type));
1053
1054 if (devinfo->gen == 7 && !devinfo->is_haswell) {
1055 /* IVB hangs when trying to access an invalid surface index with
1056 * the dataport. According to the spec "if the index used to
1057 * select an individual element is negative or greater than or
1058 * equal to the size of the array, the results of the operation
1059 * are undefined but may not lead to termination" -- which is one
1060 * of the possible outcomes of the hang. Clamp the index to
1061 * prevent access outside of the array bounds.
1062 */
1063 bld.emit_minmax(*tmp, retype(get_nir_src(deref_array->indirect),
1064 BRW_REGISTER_TYPE_UD),
1065 fs_reg(size - base - 1), BRW_CONDITIONAL_L);
1066 } else {
1067 bld.MOV(*tmp, get_nir_src(deref_array->indirect));
1068 }
1069
1070 bld.MUL(*tmp, *tmp, fs_reg(BRW_IMAGE_PARAM_SIZE));
1071 image.reladdr = tmp;
1072 }
1073 }
1074
1075 return image;
1076 }
1077
1078 void
1079 fs_visitor::emit_percomp(const fs_builder &bld, const fs_inst &inst,
1080 unsigned wr_mask)
1081 {
1082 for (unsigned i = 0; i < 4; i++) {
1083 if (!((wr_mask >> i) & 1))
1084 continue;
1085
1086 fs_inst *new_inst = new(mem_ctx) fs_inst(inst);
1087 new_inst->dst = offset(new_inst->dst, bld, i);
1088 for (unsigned j = 0; j < new_inst->sources; j++)
1089 if (new_inst->src[j].file == GRF)
1090 new_inst->src[j] = offset(new_inst->src[j], bld, i);
1091
1092 bld.emit(new_inst);
1093 }
1094 }
1095
1096 /**
1097 * Get the matching channel register datatype for an image intrinsic of the
1098 * specified GLSL image type.
1099 */
1100 static brw_reg_type
1101 get_image_base_type(const glsl_type *type)
1102 {
1103 switch ((glsl_base_type)type->sampler_type) {
1104 case GLSL_TYPE_UINT:
1105 return BRW_REGISTER_TYPE_UD;
1106 case GLSL_TYPE_INT:
1107 return BRW_REGISTER_TYPE_D;
1108 case GLSL_TYPE_FLOAT:
1109 return BRW_REGISTER_TYPE_F;
1110 default:
1111 unreachable("Not reached.");
1112 }
1113 }
1114
1115 /**
1116 * Get the appropriate atomic op for an image atomic intrinsic.
1117 */
1118 static unsigned
1119 get_image_atomic_op(nir_intrinsic_op op, const glsl_type *type)
1120 {
1121 switch (op) {
1122 case nir_intrinsic_image_atomic_add:
1123 return BRW_AOP_ADD;
1124 case nir_intrinsic_image_atomic_min:
1125 return (get_image_base_type(type) == BRW_REGISTER_TYPE_D ?
1126 BRW_AOP_IMIN : BRW_AOP_UMIN);
1127 case nir_intrinsic_image_atomic_max:
1128 return (get_image_base_type(type) == BRW_REGISTER_TYPE_D ?
1129 BRW_AOP_IMAX : BRW_AOP_UMAX);
1130 case nir_intrinsic_image_atomic_and:
1131 return BRW_AOP_AND;
1132 case nir_intrinsic_image_atomic_or:
1133 return BRW_AOP_OR;
1134 case nir_intrinsic_image_atomic_xor:
1135 return BRW_AOP_XOR;
1136 case nir_intrinsic_image_atomic_exchange:
1137 return BRW_AOP_MOV;
1138 case nir_intrinsic_image_atomic_comp_swap:
1139 return BRW_AOP_CMPWR;
1140 default:
1141 unreachable("Not reachable.");
1142 }
1143 }
1144
1145 static fs_inst *
1146 emit_pixel_interpolater_send(const fs_builder &bld,
1147 enum opcode opcode,
1148 const fs_reg &dst,
1149 const fs_reg &src,
1150 const fs_reg &desc,
1151 glsl_interp_qualifier interpolation)
1152 {
1153 fs_inst *inst;
1154 fs_reg payload;
1155 int mlen;
1156
1157 if (src.file == BAD_FILE) {
1158 /* Dummy payload */
1159 payload = bld.vgrf(BRW_REGISTER_TYPE_F, 1);
1160 mlen = 1;
1161 } else {
1162 payload = src;
1163 mlen = 2 * bld.dispatch_width() / 8;
1164 }
1165
1166 inst = bld.emit(opcode, dst, payload, desc);
1167 inst->mlen = mlen;
1168 /* 2 floats per slot returned */
1169 inst->regs_written = 2 * bld.dispatch_width() / 8;
1170 inst->pi_noperspective = interpolation == INTERP_QUALIFIER_NOPERSPECTIVE;
1171
1172 return inst;
1173 }
1174
1175 void
1176 fs_visitor::nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr)
1177 {
1178 fs_reg dest;
1179 if (nir_intrinsic_infos[instr->intrinsic].has_dest)
1180 dest = get_nir_dest(instr->dest);
1181
1182 bool has_indirect = false;
1183
1184 switch (instr->intrinsic) {
1185 case nir_intrinsic_discard:
1186 case nir_intrinsic_discard_if: {
1187 /* We track our discarded pixels in f0.1. By predicating on it, we can
1188 * update just the flag bits that aren't yet discarded. If there's no
1189 * condition, we emit a CMP of g0 != g0, so all currently executing
1190 * channels will get turned off.
1191 */
1192 fs_inst *cmp;
1193 if (instr->intrinsic == nir_intrinsic_discard_if) {
1194 cmp = bld.CMP(bld.null_reg_f(), get_nir_src(instr->src[0]),
1195 fs_reg(0), BRW_CONDITIONAL_Z);
1196 } else {
1197 fs_reg some_reg = fs_reg(retype(brw_vec8_grf(0, 0),
1198 BRW_REGISTER_TYPE_UW));
1199 cmp = bld.CMP(bld.null_reg_f(), some_reg, some_reg, BRW_CONDITIONAL_NZ);
1200 }
1201 cmp->predicate = BRW_PREDICATE_NORMAL;
1202 cmp->flag_subreg = 1;
1203
1204 if (devinfo->gen >= 6) {
1205 emit_discard_jump();
1206 }
1207 break;
1208 }
1209
1210 case nir_intrinsic_atomic_counter_inc:
1211 case nir_intrinsic_atomic_counter_dec:
1212 case nir_intrinsic_atomic_counter_read: {
1213 using namespace surface_access;
1214
1215 /* Get the arguments of the atomic intrinsic. */
1216 const fs_reg offset = get_nir_src(instr->src[0]);
1217 const unsigned surface = (stage_prog_data->binding_table.abo_start +
1218 instr->const_index[0]);
1219 fs_reg tmp;
1220
1221 /* Emit a surface read or atomic op. */
1222 switch (instr->intrinsic) {
1223 case nir_intrinsic_atomic_counter_read:
1224 tmp = emit_untyped_read(bld, fs_reg(surface), offset, 1, 1);
1225 break;
1226
1227 case nir_intrinsic_atomic_counter_inc:
1228 tmp = emit_untyped_atomic(bld, fs_reg(surface), offset, fs_reg(),
1229 fs_reg(), 1, 1, BRW_AOP_INC);
1230 break;
1231
1232 case nir_intrinsic_atomic_counter_dec:
1233 tmp = emit_untyped_atomic(bld, fs_reg(surface), offset, fs_reg(),
1234 fs_reg(), 1, 1, BRW_AOP_PREDEC);
1235 break;
1236
1237 default:
1238 unreachable("Unreachable");
1239 }
1240
1241 /* Assign the result. */
1242 bld.MOV(retype(dest, BRW_REGISTER_TYPE_UD), tmp);
1243
1244 /* Mark the surface as used. */
1245 brw_mark_surface_used(stage_prog_data, surface);
1246 break;
1247 }
1248
1249 case nir_intrinsic_image_load:
1250 case nir_intrinsic_image_store:
1251 case nir_intrinsic_image_atomic_add:
1252 case nir_intrinsic_image_atomic_min:
1253 case nir_intrinsic_image_atomic_max:
1254 case nir_intrinsic_image_atomic_and:
1255 case nir_intrinsic_image_atomic_or:
1256 case nir_intrinsic_image_atomic_xor:
1257 case nir_intrinsic_image_atomic_exchange:
1258 case nir_intrinsic_image_atomic_comp_swap: {
1259 using namespace image_access;
1260
1261 /* Get the referenced image variable and type. */
1262 const nir_variable *var = instr->variables[0]->var;
1263 const glsl_type *type = var->type->without_array();
1264 const brw_reg_type base_type = get_image_base_type(type);
1265
1266 /* Get some metadata from the image intrinsic. */
1267 const nir_intrinsic_info *info = &nir_intrinsic_infos[instr->intrinsic];
1268 const unsigned arr_dims = type->sampler_array ? 1 : 0;
1269 const unsigned surf_dims = type->coordinate_components() - arr_dims;
1270 const mesa_format format =
1271 (var->data.image.write_only ? MESA_FORMAT_NONE :
1272 _mesa_get_shader_image_format(var->data.image.format));
1273
1274 /* Get the arguments of the image intrinsic. */
1275 const fs_reg image = get_nir_image_deref(instr->variables[0]);
1276 const fs_reg addr = retype(get_nir_src(instr->src[0]),
1277 BRW_REGISTER_TYPE_UD);
1278 const fs_reg src0 = (info->num_srcs >= 3 ?
1279 retype(get_nir_src(instr->src[2]), base_type) :
1280 fs_reg());
1281 const fs_reg src1 = (info->num_srcs >= 4 ?
1282 retype(get_nir_src(instr->src[3]), base_type) :
1283 fs_reg());
1284 fs_reg tmp;
1285
1286 /* Emit an image load, store or atomic op. */
1287 if (instr->intrinsic == nir_intrinsic_image_load)
1288 tmp = emit_image_load(bld, image, addr, surf_dims, arr_dims, format);
1289
1290 else if (instr->intrinsic == nir_intrinsic_image_store)
1291 emit_image_store(bld, image, addr, src0, surf_dims, arr_dims, format);
1292
1293 else
1294 tmp = emit_image_atomic(bld, image, addr, src0, src1,
1295 surf_dims, arr_dims, info->dest_components,
1296 get_image_atomic_op(instr->intrinsic, type));
1297
1298 /* Assign the result. */
1299 for (unsigned c = 0; c < info->dest_components; ++c)
1300 bld.MOV(offset(retype(dest, base_type), bld, c),
1301 offset(tmp, bld, c));
1302 break;
1303 }
1304
1305 case nir_intrinsic_memory_barrier: {
1306 const fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_UD, 16 / dispatch_width);
1307 bld.emit(SHADER_OPCODE_MEMORY_FENCE, tmp)
1308 ->regs_written = 2;
1309 break;
1310 }
1311
1312 case nir_intrinsic_image_size: {
1313 /* Get the referenced image variable and type. */
1314 const nir_variable *var = instr->variables[0]->var;
1315 const glsl_type *type = var->type->without_array();
1316
1317 /* Get the size of the image. */
1318 const fs_reg image = get_nir_image_deref(instr->variables[0]);
1319 const fs_reg size = offset(image, bld, BRW_IMAGE_PARAM_SIZE_OFFSET);
1320
1321 /* For 1DArray image types, the array index is stored in the Z component.
1322 * Fix this by swizzling the Z component to the Y component.
1323 */
1324 const bool is_1d_array_image =
1325 type->sampler_dimensionality == GLSL_SAMPLER_DIM_1D &&
1326 type->sampler_array;
1327
1328 /* For CubeArray images, we should count the number of cubes instead
1329 * of the number of faces. Fix it by dividing the (Z component) by 6.
1330 */
1331 const bool is_cube_array_image =
1332 type->sampler_dimensionality == GLSL_SAMPLER_DIM_CUBE &&
1333 type->sampler_array;
1334
1335 /* Copy all the components. */
1336 const nir_intrinsic_info *info = &nir_intrinsic_infos[instr->intrinsic];
1337 for (unsigned c = 0; c < info->dest_components; ++c) {
1338 if ((int)c >= type->coordinate_components()) {
1339 bld.MOV(offset(retype(dest, BRW_REGISTER_TYPE_D), bld, c),
1340 fs_reg(1));
1341 } else if (c == 1 && is_1d_array_image) {
1342 bld.MOV(offset(retype(dest, BRW_REGISTER_TYPE_D), bld, c),
1343 offset(size, bld, 2));
1344 } else if (c == 2 && is_cube_array_image) {
1345 bld.emit(SHADER_OPCODE_INT_QUOTIENT,
1346 offset(retype(dest, BRW_REGISTER_TYPE_D), bld, c),
1347 offset(size, bld, c), fs_reg(6));
1348 } else {
1349 bld.MOV(offset(retype(dest, BRW_REGISTER_TYPE_D), bld, c),
1350 offset(size, bld, c));
1351 }
1352 }
1353
1354 break;
1355 }
1356
1357 case nir_intrinsic_image_samples:
1358 /* The driver does not support multi-sampled images. */
1359 bld.MOV(retype(dest, BRW_REGISTER_TYPE_D), fs_reg(1));
1360 break;
1361
1362 case nir_intrinsic_load_front_face:
1363 bld.MOV(retype(dest, BRW_REGISTER_TYPE_D),
1364 *emit_frontfacing_interpolation());
1365 break;
1366
1367 case nir_intrinsic_load_vertex_id:
1368 unreachable("should be lowered by lower_vertex_id()");
1369
1370 case nir_intrinsic_load_vertex_id_zero_base:
1371 case nir_intrinsic_load_base_vertex:
1372 case nir_intrinsic_load_instance_id:
1373 case nir_intrinsic_load_sample_mask_in:
1374 case nir_intrinsic_load_sample_id: {
1375 gl_system_value sv = nir_system_value_from_intrinsic(instr->intrinsic);
1376 fs_reg val = nir_system_values[sv];
1377 assert(val.file != BAD_FILE);
1378 dest.type = val.type;
1379 bld.MOV(dest, val);
1380 break;
1381 }
1382
1383 case nir_intrinsic_load_sample_pos: {
1384 fs_reg sample_pos = nir_system_values[SYSTEM_VALUE_SAMPLE_POS];
1385 assert(sample_pos.file != BAD_FILE);
1386 dest.type = sample_pos.type;
1387 bld.MOV(dest, sample_pos);
1388 bld.MOV(offset(dest, bld, 1), offset(sample_pos, bld, 1));
1389 break;
1390 }
1391
1392 case nir_intrinsic_load_uniform_indirect:
1393 has_indirect = true;
1394 /* fallthrough */
1395 case nir_intrinsic_load_uniform: {
1396 fs_reg uniform_reg(UNIFORM, instr->const_index[0]);
1397 uniform_reg.reg_offset = instr->const_index[1];
1398
1399 for (unsigned j = 0; j < instr->num_components; j++) {
1400 fs_reg src = offset(retype(uniform_reg, dest.type), bld, j);
1401 if (has_indirect)
1402 src.reladdr = new(mem_ctx) fs_reg(get_nir_src(instr->src[0]));
1403
1404 bld.MOV(dest, src);
1405 dest = offset(dest, bld, 1);
1406 }
1407 break;
1408 }
1409
1410 case nir_intrinsic_load_ubo_indirect:
1411 has_indirect = true;
1412 /* fallthrough */
1413 case nir_intrinsic_load_ubo: {
1414 nir_const_value *const_index = nir_src_as_const_value(instr->src[0]);
1415 fs_reg surf_index;
1416
1417 if (const_index) {
1418 surf_index = fs_reg(stage_prog_data->binding_table.ubo_start +
1419 const_index->u[0]);
1420 } else {
1421 /* The block index is not a constant. Evaluate the index expression
1422 * per-channel and add the base UBO index; we have to select a value
1423 * from any live channel.
1424 */
1425 surf_index = vgrf(glsl_type::uint_type);
1426 bld.ADD(surf_index, get_nir_src(instr->src[0]),
1427 fs_reg(stage_prog_data->binding_table.ubo_start));
1428 surf_index = bld.emit_uniformize(surf_index);
1429
1430 /* Assume this may touch any UBO. It would be nice to provide
1431 * a tighter bound, but the array information is already lowered away.
1432 */
1433 brw_mark_surface_used(prog_data,
1434 stage_prog_data->binding_table.ubo_start +
1435 nir->info.num_ubos - 1);
1436 }
1437
1438 if (has_indirect) {
1439 /* Turn the byte offset into a dword offset. */
1440 fs_reg base_offset = vgrf(glsl_type::int_type);
1441 bld.SHR(base_offset, retype(get_nir_src(instr->src[1]),
1442 BRW_REGISTER_TYPE_D),
1443 fs_reg(2));
1444
1445 unsigned vec4_offset = instr->const_index[0] / 4;
1446 for (int i = 0; i < instr->num_components; i++)
1447 VARYING_PULL_CONSTANT_LOAD(bld, offset(dest, bld, i), surf_index,
1448 base_offset, vec4_offset + i);
1449 } else {
1450 fs_reg packed_consts = vgrf(glsl_type::float_type);
1451 packed_consts.type = dest.type;
1452
1453 fs_reg const_offset_reg((unsigned) instr->const_index[0] & ~15);
1454 bld.emit(FS_OPCODE_UNIFORM_PULL_CONSTANT_LOAD, packed_consts,
1455 surf_index, const_offset_reg);
1456
1457 for (unsigned i = 0; i < instr->num_components; i++) {
1458 packed_consts.set_smear(instr->const_index[0] % 16 / 4 + i);
1459
1460 /* The std140 packing rules don't allow vectors to cross 16-byte
1461 * boundaries, and a reg is 32 bytes.
1462 */
1463 assert(packed_consts.subreg_offset < 32);
1464
1465 bld.MOV(dest, packed_consts);
1466 dest = offset(dest, bld, 1);
1467 }
1468 }
1469 break;
1470 }
1471
1472 case nir_intrinsic_load_ssbo_indirect:
1473 has_indirect = true;
1474 /* fallthrough */
1475 case nir_intrinsic_load_ssbo: {
1476 assert(devinfo->gen >= 7);
1477
1478 nir_const_value *const_uniform_block =
1479 nir_src_as_const_value(instr->src[0]);
1480
1481 fs_reg surf_index;
1482 if (const_uniform_block) {
1483 unsigned index = stage_prog_data->binding_table.ssbo_start +
1484 const_uniform_block->u[0];
1485 surf_index = fs_reg(index);
1486 brw_mark_surface_used(prog_data, index);
1487 } else {
1488 surf_index = vgrf(glsl_type::uint_type);
1489 bld.ADD(surf_index, get_nir_src(instr->src[0]),
1490 fs_reg(stage_prog_data->binding_table.ssbo_start));
1491 surf_index = bld.emit_uniformize(surf_index);
1492
1493 /* Assume this may touch any UBO. It would be nice to provide
1494 * a tighter bound, but the array information is already lowered away.
1495 */
1496 brw_mark_surface_used(prog_data,
1497 stage_prog_data->binding_table.ssbo_start +
1498 nir->info.num_ssbos - 1);
1499 }
1500
1501 /* Get the offset to read from */
1502 fs_reg offset_reg = vgrf(glsl_type::uint_type);
1503 unsigned const_offset_bytes = 0;
1504 if (has_indirect) {
1505 bld.MOV(offset_reg, get_nir_src(instr->src[1]));
1506 } else {
1507 const_offset_bytes = instr->const_index[0];
1508 bld.MOV(offset_reg, fs_reg(const_offset_bytes));
1509 }
1510
1511 /* Read the vector */
1512 for (int i = 0; i < instr->num_components; i++) {
1513 fs_reg read_result = emit_untyped_read(bld, surf_index, offset_reg,
1514 1 /* dims */, 1 /* size */,
1515 BRW_PREDICATE_NONE);
1516 read_result.type = dest.type;
1517 bld.MOV(dest, read_result);
1518 dest = offset(dest, bld, 1);
1519
1520 /* Vector components are stored contiguous in memory */
1521 if (i < instr->num_components) {
1522 if (!has_indirect) {
1523 const_offset_bytes += 4;
1524 bld.MOV(offset_reg, fs_reg(const_offset_bytes));
1525 } else {
1526 bld.ADD(offset_reg, offset_reg, brw_imm_ud(4));
1527 }
1528 }
1529 }
1530
1531 break;
1532 }
1533
1534 case nir_intrinsic_load_input_indirect:
1535 has_indirect = true;
1536 /* fallthrough */
1537 case nir_intrinsic_load_input: {
1538 unsigned index = 0;
1539 for (unsigned j = 0; j < instr->num_components; j++) {
1540 fs_reg src;
1541 if (stage == MESA_SHADER_VERTEX) {
1542 src = offset(fs_reg(ATTR, instr->const_index[0], dest.type), bld, index);
1543 } else {
1544 src = offset(retype(nir_inputs, dest.type), bld,
1545 instr->const_index[0] + index);
1546 }
1547 if (has_indirect)
1548 src.reladdr = new(mem_ctx) fs_reg(get_nir_src(instr->src[0]));
1549 index++;
1550
1551 bld.MOV(dest, src);
1552 dest = offset(dest, bld, 1);
1553 }
1554 break;
1555 }
1556
1557 /* Handle ARB_gpu_shader5 interpolation intrinsics
1558 *
1559 * It's worth a quick word of explanation as to why we handle the full
1560 * variable-based interpolation intrinsic rather than a lowered version
1561 * with like we do for other inputs. We have to do that because the way
1562 * we set up inputs doesn't allow us to use the already setup inputs for
1563 * interpolation. At the beginning of the shader, we go through all of
1564 * the input variables and do the initial interpolation and put it in
1565 * the nir_inputs array based on its location as determined in
1566 * nir_lower_io. If the input isn't used, dead code cleans up and
1567 * everything works fine. However, when we get to the ARB_gpu_shader5
1568 * interpolation intrinsics, we need to reinterpolate the input
1569 * differently. If we used an intrinsic that just had an index it would
1570 * only give us the offset into the nir_inputs array. However, this is
1571 * useless because that value is post-interpolation and we need
1572 * pre-interpolation. In order to get the actual location of the bits
1573 * we get from the vertex fetching hardware, we need the variable.
1574 */
1575 case nir_intrinsic_interp_var_at_centroid:
1576 case nir_intrinsic_interp_var_at_sample:
1577 case nir_intrinsic_interp_var_at_offset: {
1578 assert(stage == MESA_SHADER_FRAGMENT);
1579
1580 ((struct brw_wm_prog_data *) prog_data)->pulls_bary = true;
1581
1582 fs_reg dst_xy = bld.vgrf(BRW_REGISTER_TYPE_F, 2);
1583 const glsl_interp_qualifier interpolation =
1584 (glsl_interp_qualifier) instr->variables[0]->var->data.interpolation;
1585
1586 switch (instr->intrinsic) {
1587 case nir_intrinsic_interp_var_at_centroid:
1588 emit_pixel_interpolater_send(bld,
1589 FS_OPCODE_INTERPOLATE_AT_CENTROID,
1590 dst_xy,
1591 fs_reg(), /* src */
1592 fs_reg(0u),
1593 interpolation);
1594 break;
1595
1596 case nir_intrinsic_interp_var_at_sample: {
1597 nir_const_value *const_sample = nir_src_as_const_value(instr->src[0]);
1598
1599 if (const_sample) {
1600 unsigned msg_data = const_sample->i[0] << 4;
1601
1602 emit_pixel_interpolater_send(bld,
1603 FS_OPCODE_INTERPOLATE_AT_SAMPLE,
1604 dst_xy,
1605 fs_reg(), /* src */
1606 fs_reg(msg_data),
1607 interpolation);
1608 } else {
1609 const fs_reg sample_src = retype(get_nir_src(instr->src[0]),
1610 BRW_REGISTER_TYPE_UD);
1611
1612 if (nir_src_is_dynamically_uniform(instr->src[0])) {
1613 const fs_reg sample_id = bld.emit_uniformize(sample_src);
1614 const fs_reg msg_data = vgrf(glsl_type::uint_type);
1615 bld.exec_all().group(1, 0).SHL(msg_data, sample_id, fs_reg(4u));
1616 emit_pixel_interpolater_send(bld,
1617 FS_OPCODE_INTERPOLATE_AT_SAMPLE,
1618 dst_xy,
1619 fs_reg(), /* src */
1620 msg_data,
1621 interpolation);
1622 } else {
1623 /* Make a loop that sends a message to the pixel interpolater
1624 * for the sample number in each live channel. If there are
1625 * multiple channels with the same sample number then these
1626 * will be handled simultaneously with a single interation of
1627 * the loop.
1628 */
1629 bld.emit(BRW_OPCODE_DO);
1630
1631 /* Get the next live sample number into sample_id_reg */
1632 const fs_reg sample_id = bld.emit_uniformize(sample_src);
1633
1634 /* Set the flag register so that we can perform the send
1635 * message on all channels that have the same sample number
1636 */
1637 bld.CMP(bld.null_reg_ud(),
1638 sample_src, sample_id,
1639 BRW_CONDITIONAL_EQ);
1640 const fs_reg msg_data = vgrf(glsl_type::uint_type);
1641 bld.exec_all().group(1, 0).SHL(msg_data, sample_id, fs_reg(4u));
1642 fs_inst *inst =
1643 emit_pixel_interpolater_send(bld,
1644 FS_OPCODE_INTERPOLATE_AT_SAMPLE,
1645 dst_xy,
1646 fs_reg(), /* src */
1647 msg_data,
1648 interpolation);
1649 set_predicate(BRW_PREDICATE_NORMAL, inst);
1650
1651 /* Continue the loop if there are any live channels left */
1652 set_predicate_inv(BRW_PREDICATE_NORMAL,
1653 true, /* inverse */
1654 bld.emit(BRW_OPCODE_WHILE));
1655 }
1656 }
1657
1658 break;
1659 }
1660
1661 case nir_intrinsic_interp_var_at_offset: {
1662 nir_const_value *const_offset = nir_src_as_const_value(instr->src[0]);
1663
1664 if (const_offset) {
1665 unsigned off_x = MIN2((int)(const_offset->f[0] * 16), 7) & 0xf;
1666 unsigned off_y = MIN2((int)(const_offset->f[1] * 16), 7) & 0xf;
1667
1668 emit_pixel_interpolater_send(bld,
1669 FS_OPCODE_INTERPOLATE_AT_SHARED_OFFSET,
1670 dst_xy,
1671 fs_reg(), /* src */
1672 fs_reg(off_x | (off_y << 4)),
1673 interpolation);
1674 } else {
1675 fs_reg src = vgrf(glsl_type::ivec2_type);
1676 fs_reg offset_src = retype(get_nir_src(instr->src[0]),
1677 BRW_REGISTER_TYPE_F);
1678 for (int i = 0; i < 2; i++) {
1679 fs_reg temp = vgrf(glsl_type::float_type);
1680 bld.MUL(temp, offset(offset_src, bld, i), fs_reg(16.0f));
1681 fs_reg itemp = vgrf(glsl_type::int_type);
1682 bld.MOV(itemp, temp); /* float to int */
1683
1684 /* Clamp the upper end of the range to +7/16.
1685 * ARB_gpu_shader5 requires that we support a maximum offset
1686 * of +0.5, which isn't representable in a S0.4 value -- if
1687 * we didn't clamp it, we'd end up with -8/16, which is the
1688 * opposite of what the shader author wanted.
1689 *
1690 * This is legal due to ARB_gpu_shader5's quantization
1691 * rules:
1692 *
1693 * "Not all values of <offset> may be supported; x and y
1694 * offsets may be rounded to fixed-point values with the
1695 * number of fraction bits given by the
1696 * implementation-dependent constant
1697 * FRAGMENT_INTERPOLATION_OFFSET_BITS"
1698 */
1699 set_condmod(BRW_CONDITIONAL_L,
1700 bld.SEL(offset(src, bld, i), itemp, fs_reg(7)));
1701 }
1702
1703 const enum opcode opcode = FS_OPCODE_INTERPOLATE_AT_PER_SLOT_OFFSET;
1704 emit_pixel_interpolater_send(bld,
1705 opcode,
1706 dst_xy,
1707 src,
1708 fs_reg(0u),
1709 interpolation);
1710 }
1711 break;
1712 }
1713
1714 default:
1715 unreachable("Invalid intrinsic");
1716 }
1717
1718 for (unsigned j = 0; j < instr->num_components; j++) {
1719 fs_reg src = interp_reg(instr->variables[0]->var->data.location, j);
1720 src.type = dest.type;
1721
1722 bld.emit(FS_OPCODE_LINTERP, dest, dst_xy, src);
1723 dest = offset(dest, bld, 1);
1724 }
1725 break;
1726 }
1727
1728 case nir_intrinsic_store_ssbo_indirect:
1729 has_indirect = true;
1730 /* fallthrough */
1731 case nir_intrinsic_store_ssbo: {
1732 assert(devinfo->gen >= 7);
1733
1734 /* Block index */
1735 fs_reg surf_index;
1736 nir_const_value *const_uniform_block =
1737 nir_src_as_const_value(instr->src[1]);
1738 if (const_uniform_block) {
1739 unsigned index = stage_prog_data->binding_table.ssbo_start +
1740 const_uniform_block->u[0];
1741 surf_index = fs_reg(index);
1742 brw_mark_surface_used(prog_data, index);
1743 } else {
1744 surf_index = vgrf(glsl_type::uint_type);
1745 bld.ADD(surf_index, get_nir_src(instr->src[1]),
1746 fs_reg(stage_prog_data->binding_table.ssbo_start));
1747 surf_index = bld.emit_uniformize(surf_index);
1748
1749 brw_mark_surface_used(prog_data,
1750 stage_prog_data->binding_table.ssbo_start +
1751 nir->info.num_ssbos - 1);
1752 }
1753
1754 /* Offset */
1755 fs_reg offset_reg = vgrf(glsl_type::uint_type);
1756 unsigned const_offset_bytes = 0;
1757 if (has_indirect) {
1758 bld.MOV(offset_reg, get_nir_src(instr->src[2]));
1759 } else {
1760 const_offset_bytes = instr->const_index[0];
1761 bld.MOV(offset_reg, fs_reg(const_offset_bytes));
1762 }
1763
1764 /* Value */
1765 fs_reg val_reg = get_nir_src(instr->src[0]);
1766
1767 /* Writemask */
1768 unsigned writemask = instr->const_index[1];
1769
1770 /* Write each component present in the writemask */
1771 unsigned skipped_channels = 0;
1772 for (int i = 0; i < instr->num_components; i++) {
1773 int component_mask = 1 << i;
1774 if (writemask & component_mask) {
1775 if (skipped_channels) {
1776 if (!has_indirect) {
1777 const_offset_bytes += 4 * skipped_channels;
1778 bld.MOV(offset_reg, fs_reg(const_offset_bytes));
1779 } else {
1780 bld.ADD(offset_reg, offset_reg,
1781 brw_imm_ud(4 * skipped_channels));
1782 }
1783 skipped_channels = 0;
1784 }
1785
1786 emit_untyped_write(bld, surf_index, offset_reg,
1787 offset(val_reg, bld, i),
1788 1 /* dims */, 1 /* size */,
1789 BRW_PREDICATE_NONE);
1790 }
1791
1792 skipped_channels++;
1793 }
1794 break;
1795 }
1796
1797 case nir_intrinsic_store_output_indirect:
1798 has_indirect = true;
1799 /* fallthrough */
1800 case nir_intrinsic_store_output: {
1801 fs_reg src = get_nir_src(instr->src[0]);
1802 unsigned index = 0;
1803 for (unsigned j = 0; j < instr->num_components; j++) {
1804 fs_reg new_dest = offset(retype(nir_outputs, src.type), bld,
1805 instr->const_index[0] + index);
1806 if (has_indirect)
1807 src.reladdr = new(mem_ctx) fs_reg(get_nir_src(instr->src[1]));
1808 index++;
1809 bld.MOV(new_dest, src);
1810 src = offset(src, bld, 1);
1811 }
1812 break;
1813 }
1814
1815 case nir_intrinsic_barrier:
1816 emit_barrier();
1817 if (stage == MESA_SHADER_COMPUTE)
1818 ((struct brw_cs_prog_data *) prog_data)->uses_barrier = true;
1819 break;
1820
1821 case nir_intrinsic_load_local_invocation_id:
1822 case nir_intrinsic_load_work_group_id: {
1823 gl_system_value sv = nir_system_value_from_intrinsic(instr->intrinsic);
1824 fs_reg val = nir_system_values[sv];
1825 assert(val.file != BAD_FILE);
1826 dest.type = val.type;
1827 for (unsigned i = 0; i < 3; i++)
1828 bld.MOV(offset(dest, bld, i), offset(val, bld, i));
1829 break;
1830 }
1831
1832 case nir_intrinsic_ssbo_atomic_add:
1833 nir_emit_ssbo_atomic(bld, BRW_AOP_ADD, instr);
1834 break;
1835 case nir_intrinsic_ssbo_atomic_imin:
1836 nir_emit_ssbo_atomic(bld, BRW_AOP_IMIN, instr);
1837 break;
1838 case nir_intrinsic_ssbo_atomic_umin:
1839 nir_emit_ssbo_atomic(bld, BRW_AOP_UMIN, instr);
1840 break;
1841 case nir_intrinsic_ssbo_atomic_imax:
1842 nir_emit_ssbo_atomic(bld, BRW_AOP_IMAX, instr);
1843 break;
1844 case nir_intrinsic_ssbo_atomic_umax:
1845 nir_emit_ssbo_atomic(bld, BRW_AOP_UMAX, instr);
1846 break;
1847 case nir_intrinsic_ssbo_atomic_and:
1848 nir_emit_ssbo_atomic(bld, BRW_AOP_AND, instr);
1849 break;
1850 case nir_intrinsic_ssbo_atomic_or:
1851 nir_emit_ssbo_atomic(bld, BRW_AOP_OR, instr);
1852 break;
1853 case nir_intrinsic_ssbo_atomic_xor:
1854 nir_emit_ssbo_atomic(bld, BRW_AOP_XOR, instr);
1855 break;
1856 case nir_intrinsic_ssbo_atomic_exchange:
1857 nir_emit_ssbo_atomic(bld, BRW_AOP_MOV, instr);
1858 break;
1859 case nir_intrinsic_ssbo_atomic_comp_swap:
1860 nir_emit_ssbo_atomic(bld, BRW_AOP_CMPWR, instr);
1861 break;
1862
1863 case nir_intrinsic_get_buffer_size: {
1864 nir_const_value *const_uniform_block = nir_src_as_const_value(instr->src[0]);
1865 unsigned ssbo_index = const_uniform_block ? const_uniform_block->u[0] : 0;
1866 int reg_width = dispatch_width / 8;
1867
1868 /* Set LOD = 0 */
1869 fs_reg source = fs_reg(0);
1870
1871 int mlen = 1 * reg_width;
1872 fs_reg src_payload = fs_reg(GRF, alloc.allocate(mlen),
1873 BRW_REGISTER_TYPE_UD);
1874 bld.LOAD_PAYLOAD(src_payload, &source, 1, 0);
1875
1876 fs_reg surf_index = fs_reg(prog_data->binding_table.ssbo_start + ssbo_index);
1877 fs_inst *inst = bld.emit(FS_OPCODE_GET_BUFFER_SIZE, dest,
1878 src_payload, surf_index);
1879 inst->header_size = 0;
1880 inst->mlen = mlen;
1881 bld.emit(inst);
1882 break;
1883 }
1884
1885 case nir_intrinsic_load_num_work_groups: {
1886 assert(devinfo->gen >= 7);
1887 assert(stage == MESA_SHADER_COMPUTE);
1888
1889 struct brw_cs_prog_data *cs_prog_data =
1890 (struct brw_cs_prog_data *) prog_data;
1891 const unsigned surface =
1892 cs_prog_data->binding_table.work_groups_start;
1893
1894 cs_prog_data->uses_num_work_groups = true;
1895
1896 fs_reg surf_index = fs_reg(surface);
1897 brw_mark_surface_used(prog_data, surface);
1898
1899 /* Read the 3 GLuint components of gl_NumWorkGroups */
1900 for (unsigned i = 0; i < 3; i++) {
1901 fs_reg read_result =
1902 emit_untyped_read(bld, surf_index,
1903 fs_reg(i << 2),
1904 1 /* dims */, 1 /* size */,
1905 BRW_PREDICATE_NONE);
1906 read_result.type = dest.type;
1907 bld.MOV(dest, read_result);
1908 dest = offset(dest, bld, 1);
1909 }
1910 break;
1911 }
1912
1913 default:
1914 unreachable("unknown intrinsic");
1915 }
1916 }
1917
1918 void
1919 fs_visitor::nir_emit_ssbo_atomic(const fs_builder &bld,
1920 int op, nir_intrinsic_instr *instr)
1921 {
1922 fs_reg dest;
1923 if (nir_intrinsic_infos[instr->intrinsic].has_dest)
1924 dest = get_nir_dest(instr->dest);
1925
1926 fs_reg surface;
1927 nir_const_value *const_surface = nir_src_as_const_value(instr->src[0]);
1928 if (const_surface) {
1929 unsigned surf_index = stage_prog_data->binding_table.ssbo_start +
1930 const_surface->u[0];
1931 surface = fs_reg(surf_index);
1932 brw_mark_surface_used(prog_data, surf_index);
1933 } else {
1934 surface = vgrf(glsl_type::uint_type);
1935 bld.ADD(surface, get_nir_src(instr->src[0]),
1936 fs_reg(stage_prog_data->binding_table.ssbo_start));
1937
1938 /* Assume this may touch any SSBO. This is the same we do for other
1939 * UBO/SSBO accesses with non-constant surface.
1940 */
1941 brw_mark_surface_used(prog_data,
1942 stage_prog_data->binding_table.ssbo_start +
1943 nir->info.num_ssbos - 1);
1944 }
1945
1946 fs_reg offset = get_nir_src(instr->src[1]);
1947 fs_reg data1 = get_nir_src(instr->src[2]);
1948 fs_reg data2;
1949 if (op == BRW_AOP_CMPWR)
1950 data2 = get_nir_src(instr->src[3]);
1951
1952 /* Emit the actual atomic operation operation */
1953
1954 fs_reg atomic_result =
1955 surface_access::emit_untyped_atomic(bld, surface, offset,
1956 data1, data2,
1957 1 /* dims */, 1 /* rsize */,
1958 op,
1959 BRW_PREDICATE_NONE);
1960 dest.type = atomic_result.type;
1961 bld.MOV(dest, atomic_result);
1962 }
1963
1964 void
1965 fs_visitor::nir_emit_texture(const fs_builder &bld, nir_tex_instr *instr)
1966 {
1967 unsigned sampler = instr->sampler_index;
1968 fs_reg sampler_reg(sampler);
1969
1970 int gather_component = instr->component;
1971
1972 bool is_rect = instr->sampler_dim == GLSL_SAMPLER_DIM_RECT;
1973
1974 bool is_cube_array = instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE &&
1975 instr->is_array;
1976
1977 int lod_components = 0;
1978 int UNUSED offset_components = 0;
1979
1980 fs_reg coordinate, shadow_comparitor, lod, lod2, sample_index, mcs, tex_offset;
1981
1982 for (unsigned i = 0; i < instr->num_srcs; i++) {
1983 fs_reg src = get_nir_src(instr->src[i].src);
1984 switch (instr->src[i].src_type) {
1985 case nir_tex_src_bias:
1986 lod = retype(src, BRW_REGISTER_TYPE_F);
1987 break;
1988 case nir_tex_src_comparitor:
1989 shadow_comparitor = retype(src, BRW_REGISTER_TYPE_F);
1990 break;
1991 case nir_tex_src_coord:
1992 switch (instr->op) {
1993 case nir_texop_txf:
1994 case nir_texop_txf_ms:
1995 coordinate = retype(src, BRW_REGISTER_TYPE_D);
1996 break;
1997 default:
1998 coordinate = retype(src, BRW_REGISTER_TYPE_F);
1999 break;
2000 }
2001 break;
2002 case nir_tex_src_ddx:
2003 lod = retype(src, BRW_REGISTER_TYPE_F);
2004 lod_components = nir_tex_instr_src_size(instr, i);
2005 break;
2006 case nir_tex_src_ddy:
2007 lod2 = retype(src, BRW_REGISTER_TYPE_F);
2008 break;
2009 case nir_tex_src_lod:
2010 switch (instr->op) {
2011 case nir_texop_txs:
2012 lod = retype(src, BRW_REGISTER_TYPE_UD);
2013 break;
2014 case nir_texop_txf:
2015 lod = retype(src, BRW_REGISTER_TYPE_D);
2016 break;
2017 default:
2018 lod = retype(src, BRW_REGISTER_TYPE_F);
2019 break;
2020 }
2021 break;
2022 case nir_tex_src_ms_index:
2023 sample_index = retype(src, BRW_REGISTER_TYPE_UD);
2024 break;
2025 case nir_tex_src_offset:
2026 tex_offset = retype(src, BRW_REGISTER_TYPE_D);
2027 if (instr->is_array)
2028 offset_components = instr->coord_components - 1;
2029 else
2030 offset_components = instr->coord_components;
2031 break;
2032 case nir_tex_src_projector:
2033 unreachable("should be lowered");
2034
2035 case nir_tex_src_sampler_offset: {
2036 /* Figure out the highest possible sampler index and mark it as used */
2037 uint32_t max_used = sampler + instr->sampler_array_size - 1;
2038 if (instr->op == nir_texop_tg4 && devinfo->gen < 8) {
2039 max_used += stage_prog_data->binding_table.gather_texture_start;
2040 } else {
2041 max_used += stage_prog_data->binding_table.texture_start;
2042 }
2043 brw_mark_surface_used(prog_data, max_used);
2044
2045 /* Emit code to evaluate the actual indexing expression */
2046 sampler_reg = vgrf(glsl_type::uint_type);
2047 bld.ADD(sampler_reg, src, fs_reg(sampler));
2048 sampler_reg = bld.emit_uniformize(sampler_reg);
2049 break;
2050 }
2051
2052 default:
2053 unreachable("unknown texture source");
2054 }
2055 }
2056
2057 if (instr->op == nir_texop_txf_ms) {
2058 if (devinfo->gen >= 7 &&
2059 key_tex->compressed_multisample_layout_mask & (1 << sampler)) {
2060 mcs = emit_mcs_fetch(coordinate, instr->coord_components, sampler_reg);
2061 } else {
2062 mcs = fs_reg(0u);
2063 }
2064 }
2065
2066 for (unsigned i = 0; i < 3; i++) {
2067 if (instr->const_offset[i] != 0) {
2068 assert(offset_components == 0);
2069 tex_offset = fs_reg(brw_texture_offset(instr->const_offset, 3));
2070 break;
2071 }
2072 }
2073
2074 enum glsl_base_type dest_base_type =
2075 brw_glsl_base_type_for_nir_type (instr->dest_type);
2076
2077 const glsl_type *dest_type =
2078 glsl_type::get_instance(dest_base_type, nir_tex_instr_dest_size(instr),
2079 1);
2080
2081 ir_texture_opcode op;
2082 switch (instr->op) {
2083 case nir_texop_lod: op = ir_lod; break;
2084 case nir_texop_query_levels: op = ir_query_levels; break;
2085 case nir_texop_tex: op = ir_tex; break;
2086 case nir_texop_tg4: op = ir_tg4; break;
2087 case nir_texop_txb: op = ir_txb; break;
2088 case nir_texop_txd: op = ir_txd; break;
2089 case nir_texop_txf: op = ir_txf; break;
2090 case nir_texop_txf_ms: op = ir_txf_ms; break;
2091 case nir_texop_txl: op = ir_txl; break;
2092 case nir_texop_txs: op = ir_txs; break;
2093 case nir_texop_texture_samples: {
2094 fs_reg dst = retype(get_nir_dest(instr->dest), BRW_REGISTER_TYPE_D);
2095 fs_inst *inst = bld.emit(SHADER_OPCODE_SAMPLEINFO, dst,
2096 bld.vgrf(BRW_REGISTER_TYPE_D, 1),
2097 sampler_reg);
2098 inst->mlen = 1;
2099 inst->header_size = 1;
2100 inst->base_mrf = -1;
2101 return;
2102 }
2103 default:
2104 unreachable("unknown texture opcode");
2105 }
2106
2107 emit_texture(op, dest_type, coordinate, instr->coord_components,
2108 shadow_comparitor, lod, lod2, lod_components, sample_index,
2109 tex_offset, mcs, gather_component,
2110 is_cube_array, is_rect, sampler, sampler_reg);
2111
2112 fs_reg dest = get_nir_dest(instr->dest);
2113 dest.type = this->result.type;
2114 unsigned num_components = nir_tex_instr_dest_size(instr);
2115 emit_percomp(bld, fs_inst(BRW_OPCODE_MOV, bld.dispatch_width(),
2116 dest, this->result),
2117 (1 << num_components) - 1);
2118 }
2119
2120 void
2121 fs_visitor::nir_emit_jump(const fs_builder &bld, nir_jump_instr *instr)
2122 {
2123 switch (instr->type) {
2124 case nir_jump_break:
2125 bld.emit(BRW_OPCODE_BREAK);
2126 break;
2127 case nir_jump_continue:
2128 bld.emit(BRW_OPCODE_CONTINUE);
2129 break;
2130 case nir_jump_return:
2131 /* This has to be the last block in the shader. We don't handle
2132 * early returns.
2133 */
2134 assert(nir_cf_node_next(&instr->instr.block->cf_node) == NULL &&
2135 instr->instr.block->cf_node.parent->type == nir_cf_node_function);
2136 break;
2137 default:
2138 unreachable("unknown jump");
2139 }
2140 }