projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
i965/vec4: Assert that fine/coarse derivative ops don't appear
[mesa.git] / src / mesa / drivers / dri / i965 / brw_vec4_visitor.cpp
1 /*
2 * Copyright © 2011 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "brw_vec4.h"
25 #include "glsl/ir_uniform.h"
26 extern "C" {
27 #include "program/sampler.h"
28 }
29
30 namespace brw {
31
32 vec4_instruction::vec4_instruction(vec4_visitor *v,
33 enum opcode opcode, const dst_reg &dst,
34 const src_reg &src0, const src_reg &src1,
35 const src_reg &src2)
36 {
37 this->opcode = opcode;
38 this->dst = dst;
39 this->src[0] = src0;
40 this->src[1] = src1;
41 this->src[2] = src2;
42 this->saturate = false;
43 this->force_writemask_all = false;
44 this->no_dd_clear = false;
45 this->no_dd_check = false;
46 this->writes_accumulator = false;
47 this->conditional_mod = BRW_CONDITIONAL_NONE;
48 this->texture_offset = 0;
49 this->target = 0;
50 this->shadow_compare = false;
51 this->ir = v->base_ir;
52 this->urb_write_flags = BRW_URB_WRITE_NO_FLAGS;
53 this->header_present = false;
54 this->mlen = 0;
55 this->base_mrf = 0;
56 this->offset = 0;
57 this->annotation = v->current_annotation;
58 }
59
60 vec4_instruction *
61 vec4_visitor::emit(vec4_instruction *inst)
62 {
63 this->instructions.push_tail(inst);
64
65 return inst;
66 }
67
68 vec4_instruction *
69 vec4_visitor::emit_before(vec4_instruction *inst, vec4_instruction *new_inst)
70 {
71 new_inst->ir = inst->ir;
72 new_inst->annotation = inst->annotation;
73
74 inst->insert_before(new_inst);
75
76 return inst;
77 }
78
79 vec4_instruction *
80 vec4_visitor::emit(enum opcode opcode, dst_reg dst,
81 src_reg src0, src_reg src1, src_reg src2)
82 {
83 return emit(new(mem_ctx) vec4_instruction(this, opcode, dst,
84 src0, src1, src2));
85 }
86
87
88 vec4_instruction *
89 vec4_visitor::emit(enum opcode opcode, dst_reg dst, src_reg src0, src_reg src1)
90 {
91 return emit(new(mem_ctx) vec4_instruction(this, opcode, dst, src0, src1));
92 }
93
94 vec4_instruction *
95 vec4_visitor::emit(enum opcode opcode, dst_reg dst, src_reg src0)
96 {
97 return emit(new(mem_ctx) vec4_instruction(this, opcode, dst, src0));
98 }
99
100 vec4_instruction *
101 vec4_visitor::emit(enum opcode opcode, dst_reg dst)
102 {
103 return emit(new(mem_ctx) vec4_instruction(this, opcode, dst));
104 }
105
106 vec4_instruction *
107 vec4_visitor::emit(enum opcode opcode)
108 {
109 return emit(new(mem_ctx) vec4_instruction(this, opcode, dst_reg()));
110 }
111
112 #define ALU1(op) \
113 vec4_instruction * \
114 vec4_visitor::op(const dst_reg &dst, const src_reg &src0) \
115 { \
116 return new(mem_ctx) vec4_instruction(this, BRW_OPCODE_##op, dst, \
117 src0); \
118 }
119
120 #define ALU2(op) \
121 vec4_instruction * \
122 vec4_visitor::op(const dst_reg &dst, const src_reg &src0, \
123 const src_reg &src1) \
124 { \
125 return new(mem_ctx) vec4_instruction(this, BRW_OPCODE_##op, dst, \
126 src0, src1); \
127 }
128
129 #define ALU2_ACC(op) \
130 vec4_instruction * \
131 vec4_visitor::op(const dst_reg &dst, const src_reg &src0, \
132 const src_reg &src1) \
133 { \
134 vec4_instruction *inst = new(mem_ctx) vec4_instruction(this, \
135 BRW_OPCODE_##op, dst, src0, src1); \
136 inst->writes_accumulator = true; \
137 return inst; \
138 }
139
140 #define ALU3(op) \
141 vec4_instruction * \
142 vec4_visitor::op(const dst_reg &dst, const src_reg &src0, \
143 const src_reg &src1, const src_reg &src2) \
144 { \
145 assert(brw->gen >= 6); \
146 return new(mem_ctx) vec4_instruction(this, BRW_OPCODE_##op, dst, \
147 src0, src1, src2); \
148 }
149
150 ALU1(NOT)
151 ALU1(MOV)
152 ALU1(FRC)
153 ALU1(RNDD)
154 ALU1(RNDE)
155 ALU1(RNDZ)
156 ALU1(F32TO16)
157 ALU1(F16TO32)
158 ALU2(ADD)
159 ALU2(MUL)
160 ALU2_ACC(MACH)
161 ALU2(AND)
162 ALU2(OR)
163 ALU2(XOR)
164 ALU2(DP3)
165 ALU2(DP4)
166 ALU2(DPH)
167 ALU2(SHL)
168 ALU2(SHR)
169 ALU2(ASR)
170 ALU3(LRP)
171 ALU1(BFREV)
172 ALU3(BFE)
173 ALU2(BFI1)
174 ALU3(BFI2)
175 ALU1(FBH)
176 ALU1(FBL)
177 ALU1(CBIT)
178 ALU3(MAD)
179 ALU2_ACC(ADDC)
180 ALU2_ACC(SUBB)
181 ALU2(MAC)
182
183 /** Gen4 predicated IF. */
184 vec4_instruction *
185 vec4_visitor::IF(enum brw_predicate predicate)
186 {
187 vec4_instruction *inst;
188
189 inst = new(mem_ctx) vec4_instruction(this, BRW_OPCODE_IF);
190 inst->predicate = predicate;
191
192 return inst;
193 }
194
195 /** Gen6 IF with embedded comparison. */
196 vec4_instruction *
197 vec4_visitor::IF(src_reg src0, src_reg src1,
198 enum brw_conditional_mod condition)
199 {
200 assert(brw->gen == 6);
201
202 vec4_instruction *inst;
203
204 resolve_ud_negate(&src0);
205 resolve_ud_negate(&src1);
206
207 inst = new(mem_ctx) vec4_instruction(this, BRW_OPCODE_IF, dst_null_d(),
208 src0, src1);
209 inst->conditional_mod = condition;
210
211 return inst;
212 }
213
214 /**
215 * CMP: Sets the low bit of the destination channels with the result
216 * of the comparison, while the upper bits are undefined, and updates
217 * the flag register with the packed 16 bits of the result.
218 */
219 vec4_instruction *
220 vec4_visitor::CMP(dst_reg dst, src_reg src0, src_reg src1,
221 enum brw_conditional_mod condition)
222 {
223 vec4_instruction *inst;
224
225 /* original gen4 does type conversion to the destination type
226 * before before comparison, producing garbage results for floating
227 * point comparisons.
228 */
229 if (brw->gen == 4) {
230 dst.type = src0.type;
231 if (dst.file == HW_REG)
232 dst.fixed_hw_reg.type = dst.type;
233 }
234
235 resolve_ud_negate(&src0);
236 resolve_ud_negate(&src1);
237
238 inst = new(mem_ctx) vec4_instruction(this, BRW_OPCODE_CMP, dst, src0, src1);
239 inst->conditional_mod = condition;
240
241 return inst;
242 }
243
244 vec4_instruction *
245 vec4_visitor::SCRATCH_READ(const dst_reg &dst, const src_reg &index)
246 {
247 vec4_instruction *inst;
248
249 inst = new(mem_ctx) vec4_instruction(this, SHADER_OPCODE_GEN4_SCRATCH_READ,
250 dst, index);
251 inst->base_mrf = 14;
252 inst->mlen = 2;
253
254 return inst;
255 }
256
257 vec4_instruction *
258 vec4_visitor::SCRATCH_WRITE(const dst_reg &dst, const src_reg &src,
259 const src_reg &index)
260 {
261 vec4_instruction *inst;
262
263 inst = new(mem_ctx) vec4_instruction(this, SHADER_OPCODE_GEN4_SCRATCH_WRITE,
264 dst, src, index);
265 inst->base_mrf = 13;
266 inst->mlen = 3;
267
268 return inst;
269 }
270
271 void
272 vec4_visitor::emit_dp(dst_reg dst, src_reg src0, src_reg src1, unsigned elements)
273 {
274 static enum opcode dot_opcodes[] = {
275 BRW_OPCODE_DP2, BRW_OPCODE_DP3, BRW_OPCODE_DP4
276 };
277
278 emit(dot_opcodes[elements - 2], dst, src0, src1);
279 }
280
281 src_reg
282 vec4_visitor::fix_3src_operand(src_reg src)
283 {
284 /* Using vec4 uniforms in SIMD4x2 programs is difficult. You'd like to be
285 * able to use vertical stride of zero to replicate the vec4 uniform, like
286 *
287 * g3<0;4,1>:f - [0, 4][1, 5][2, 6][3, 7]
288 *
289 * But you can't, since vertical stride is always four in three-source
290 * instructions. Instead, insert a MOV instruction to do the replication so
291 * that the three-source instruction can consume it.
292 */
293
294 /* The MOV is only needed if the source is a uniform or immediate. */
295 if (src.file != UNIFORM && src.file != IMM)
296 return src;
297
298 if (src.file == UNIFORM && brw_is_single_value_swizzle(src.swizzle))
299 return src;
300
301 dst_reg expanded = dst_reg(this, glsl_type::vec4_type);
302 expanded.type = src.type;
303 emit(MOV(expanded, src));
304 return src_reg(expanded);
305 }
306
307 src_reg
308 vec4_visitor::fix_math_operand(src_reg src)
309 {
310 /* The gen6 math instruction ignores the source modifiers --
311 * swizzle, abs, negate, and at least some parts of the register
312 * region description.
313 *
314 * Rather than trying to enumerate all these cases, *always* expand the
315 * operand to a temp GRF for gen6.
316 *
317 * For gen7, keep the operand as-is, except if immediate, which gen7 still
318 * can't use.
319 */
320
321 if (brw->gen == 7 && src.file != IMM)
322 return src;
323
324 dst_reg expanded = dst_reg(this, glsl_type::vec4_type);
325 expanded.type = src.type;
326 emit(MOV(expanded, src));
327 return src_reg(expanded);
328 }
329
330 void
331 vec4_visitor::emit_math1_gen6(enum opcode opcode, dst_reg dst, src_reg src)
332 {
333 src = fix_math_operand(src);
334
335 if (brw->gen == 6 && dst.writemask != WRITEMASK_XYZW) {
336 /* The gen6 math instruction must be align1, so we can't do
337 * writemasks.
338 */
339 dst_reg temp_dst = dst_reg(this, glsl_type::vec4_type);
340
341 emit(opcode, temp_dst, src);
342
343 emit(MOV(dst, src_reg(temp_dst)));
344 } else {
345 emit(opcode, dst, src);
346 }
347 }
348
349 void
350 vec4_visitor::emit_math1_gen4(enum opcode opcode, dst_reg dst, src_reg src)
351 {
352 vec4_instruction *inst = emit(opcode, dst, src);
353 inst->base_mrf = 1;
354 inst->mlen = 1;
355 }
356
357 void
358 vec4_visitor::emit_math(opcode opcode, dst_reg dst, src_reg src)
359 {
360 switch (opcode) {
361 case SHADER_OPCODE_RCP:
362 case SHADER_OPCODE_RSQ:
363 case SHADER_OPCODE_SQRT:
364 case SHADER_OPCODE_EXP2:
365 case SHADER_OPCODE_LOG2:
366 case SHADER_OPCODE_SIN:
367 case SHADER_OPCODE_COS:
368 break;
369 default:
370 unreachable("not reached: bad math opcode");
371 }
372
373 if (brw->gen >= 8) {
374 emit(opcode, dst, src);
375 } else if (brw->gen >= 6) {
376 emit_math1_gen6(opcode, dst, src);
377 } else {
378 emit_math1_gen4(opcode, dst, src);
379 }
380 }
381
382 void
383 vec4_visitor::emit_math2_gen6(enum opcode opcode,
384 dst_reg dst, src_reg src0, src_reg src1)
385 {
386 src0 = fix_math_operand(src0);
387 src1 = fix_math_operand(src1);
388
389 if (brw->gen == 6 && dst.writemask != WRITEMASK_XYZW) {
390 /* The gen6 math instruction must be align1, so we can't do
391 * writemasks.
392 */
393 dst_reg temp_dst = dst_reg(this, glsl_type::vec4_type);
394 temp_dst.type = dst.type;
395
396 emit(opcode, temp_dst, src0, src1);
397
398 emit(MOV(dst, src_reg(temp_dst)));
399 } else {
400 emit(opcode, dst, src0, src1);
401 }
402 }
403
404 void
405 vec4_visitor::emit_math2_gen4(enum opcode opcode,
406 dst_reg dst, src_reg src0, src_reg src1)
407 {
408 vec4_instruction *inst = emit(opcode, dst, src0, src1);
409 inst->base_mrf = 1;
410 inst->mlen = 2;
411 }
412
413 void
414 vec4_visitor::emit_math(enum opcode opcode,
415 dst_reg dst, src_reg src0, src_reg src1)
416 {
417 switch (opcode) {
418 case SHADER_OPCODE_POW:
419 case SHADER_OPCODE_INT_QUOTIENT:
420 case SHADER_OPCODE_INT_REMAINDER:
421 break;
422 default:
423 unreachable("not reached: unsupported binary math opcode");
424 }
425
426 if (brw->gen >= 8) {
427 emit(opcode, dst, src0, src1);
428 } else if (brw->gen >= 6) {
429 emit_math2_gen6(opcode, dst, src0, src1);
430 } else {
431 emit_math2_gen4(opcode, dst, src0, src1);
432 }
433 }
434
435 void
436 vec4_visitor::emit_pack_half_2x16(dst_reg dst, src_reg src0)
437 {
438 if (brw->gen < 7) {
439 unreachable("ir_unop_pack_half_2x16 should be lowered");
440 }
441
442 assert(dst.type == BRW_REGISTER_TYPE_UD);
443 assert(src0.type == BRW_REGISTER_TYPE_F);
444
445 /* From the Ivybridge PRM, Vol4, Part3, Section 6.27 f32to16:
446 *
447 * Because this instruction does not have a 16-bit floating-point type,
448 * the destination data type must be Word (W).
449 *
450 * The destination must be DWord-aligned and specify a horizontal stride
451 * (HorzStride) of 2. The 16-bit result is stored in the lower word of
452 * each destination channel and the upper word is not modified.
453 *
454 * The above restriction implies that the f32to16 instruction must use
455 * align1 mode, because only in align1 mode is it possible to specify
456 * horizontal stride. We choose here to defy the hardware docs and emit
457 * align16 instructions.
458 *
459 * (I [chadv] did attempt to emit align1 instructions for VS f32to16
460 * instructions. I was partially successful in that the code passed all
461 * tests. However, the code was dubiously correct and fragile, and the
462 * tests were not harsh enough to probe that frailty. Not trusting the
463 * code, I chose instead to remain in align16 mode in defiance of the hw
464 * docs).
465 *
466 * I've [chadv] experimentally confirmed that, on gen7 hardware and the
467 * simulator, emitting a f32to16 in align16 mode with UD as destination
468 * data type is safe. The behavior differs from that specified in the PRM
469 * in that the upper word of each destination channel is cleared to 0.
470 */
471
472 dst_reg tmp_dst(this, glsl_type::uvec2_type);
473 src_reg tmp_src(tmp_dst);
474
475 #if 0
476 /* Verify the undocumented behavior on which the following instructions
477 * rely. If f32to16 fails to clear the upper word of the X and Y channels,
478 * then the result of the bit-or instruction below will be incorrect.
479 *
480 * You should inspect the disasm output in order to verify that the MOV is
481 * not optimized away.
482 */
483 emit(MOV(tmp_dst, src_reg(0x12345678u)));
484 #endif
485
486 /* Give tmp the form below, where "." means untouched.
487 *
488 * w z y x w z y x
489 * |.|.|0x0000hhhh|0x0000llll|.|.|0x0000hhhh|0x0000llll|
490 *
491 * That the upper word of each write-channel be 0 is required for the
492 * following bit-shift and bit-or instructions to work. Note that this
493 * relies on the undocumented hardware behavior mentioned above.
494 */
495 tmp_dst.writemask = WRITEMASK_XY;
496 emit(F32TO16(tmp_dst, src0));
497
498 /* Give the write-channels of dst the form:
499 * 0xhhhh0000
500 */
501 tmp_src.swizzle = BRW_SWIZZLE_YYYY;
502 emit(SHL(dst, tmp_src, src_reg(16u)));
503
504 /* Finally, give the write-channels of dst the form of packHalf2x16's
505 * output:
506 * 0xhhhhllll
507 */
508 tmp_src.swizzle = BRW_SWIZZLE_XXXX;
509 emit(OR(dst, src_reg(dst), tmp_src));
510 }
511
512 void
513 vec4_visitor::emit_unpack_half_2x16(dst_reg dst, src_reg src0)
514 {
515 if (brw->gen < 7) {
516 unreachable("ir_unop_unpack_half_2x16 should be lowered");
517 }
518
519 assert(dst.type == BRW_REGISTER_TYPE_F);
520 assert(src0.type == BRW_REGISTER_TYPE_UD);
521
522 /* From the Ivybridge PRM, Vol4, Part3, Section 6.26 f16to32:
523 *
524 * Because this instruction does not have a 16-bit floating-point type,
525 * the source data type must be Word (W). The destination type must be
526 * F (Float).
527 *
528 * To use W as the source data type, we must adjust horizontal strides,
529 * which is only possible in align1 mode. All my [chadv] attempts at
530 * emitting align1 instructions for unpackHalf2x16 failed to pass the
531 * Piglit tests, so I gave up.
532 *
533 * I've verified that, on gen7 hardware and the simulator, it is safe to
534 * emit f16to32 in align16 mode with UD as source data type.
535 */
536
537 dst_reg tmp_dst(this, glsl_type::uvec2_type);
538 src_reg tmp_src(tmp_dst);
539
540 tmp_dst.writemask = WRITEMASK_X;
541 emit(AND(tmp_dst, src0, src_reg(0xffffu)));
542
543 tmp_dst.writemask = WRITEMASK_Y;
544 emit(SHR(tmp_dst, src0, src_reg(16u)));
545
546 dst.writemask = WRITEMASK_XY;
547 emit(F16TO32(dst, tmp_src));
548 }
549
550 void
551 vec4_visitor::visit_instructions(const exec_list *list)
552 {
553 foreach_in_list(ir_instruction, ir, list) {
554 base_ir = ir;
555 ir->accept(this);
556 }
557 }
558
559
560 static int
561 type_size(const struct glsl_type *type)
562 {
563 unsigned int i;
564 int size;
565
566 switch (type->base_type) {
567 case GLSL_TYPE_UINT:
568 case GLSL_TYPE_INT:
569 case GLSL_TYPE_FLOAT:
570 case GLSL_TYPE_BOOL:
571 if (type->is_matrix()) {
572 return type->matrix_columns;
573 } else {
574 /* Regardless of size of vector, it gets a vec4. This is bad
575 * packing for things like floats, but otherwise arrays become a
576 * mess. Hopefully a later pass over the code can pack scalars
577 * down if appropriate.
578 */
579 return 1;
580 }
581 case GLSL_TYPE_ARRAY:
582 assert(type->length > 0);
583 return type_size(type->fields.array) * type->length;
584 case GLSL_TYPE_STRUCT:
585 size = 0;
586 for (i = 0; i < type->length; i++) {
587 size += type_size(type->fields.structure[i].type);
588 }
589 return size;
590 case GLSL_TYPE_SAMPLER:
591 /* Samplers take up one slot in UNIFORMS[], but they're baked in
592 * at link time.
593 */
594 return 1;
595 case GLSL_TYPE_ATOMIC_UINT:
596 return 0;
597 case GLSL_TYPE_IMAGE:
598 case GLSL_TYPE_VOID:
599 case GLSL_TYPE_ERROR:
600 case GLSL_TYPE_INTERFACE:
601 unreachable("not reached");
602 }
603
604 return 0;
605 }
606
607 int
608 vec4_visitor::virtual_grf_alloc(int size)
609 {
610 if (virtual_grf_array_size <= virtual_grf_count) {
611 if (virtual_grf_array_size == 0)
612 virtual_grf_array_size = 16;
613 else
614 virtual_grf_array_size *= 2;
615 virtual_grf_sizes = reralloc(mem_ctx, virtual_grf_sizes, int,
616 virtual_grf_array_size);
617 virtual_grf_reg_map = reralloc(mem_ctx, virtual_grf_reg_map, int,
618 virtual_grf_array_size);
619 }
620 virtual_grf_reg_map[virtual_grf_count] = virtual_grf_reg_count;
621 virtual_grf_reg_count += size;
622 virtual_grf_sizes[virtual_grf_count] = size;
623 return virtual_grf_count++;
624 }
625
626 src_reg::src_reg(class vec4_visitor *v, const struct glsl_type *type)
627 {
628 init();
629
630 this->file = GRF;
631 this->reg = v->virtual_grf_alloc(type_size(type));
632
633 if (type->is_array() || type->is_record()) {
634 this->swizzle = BRW_SWIZZLE_NOOP;
635 } else {
636 this->swizzle = swizzle_for_size(type->vector_elements);
637 }
638
639 this->type = brw_type_for_base_type(type);
640 }
641
642 dst_reg::dst_reg(class vec4_visitor *v, const struct glsl_type *type)
643 {
644 init();
645
646 this->file = GRF;
647 this->reg = v->virtual_grf_alloc(type_size(type));
648
649 if (type->is_array() || type->is_record()) {
650 this->writemask = WRITEMASK_XYZW;
651 } else {
652 this->writemask = (1 << type->vector_elements) - 1;
653 }
654
655 this->type = brw_type_for_base_type(type);
656 }
657
658 /* Our support for uniforms is piggy-backed on the struct
659 * gl_fragment_program, because that's where the values actually
660 * get stored, rather than in some global gl_shader_program uniform
661 * store.
662 */
663 void
664 vec4_visitor::setup_uniform_values(ir_variable *ir)
665 {
666 int namelen = strlen(ir->name);
667
668 /* The data for our (non-builtin) uniforms is stored in a series of
669 * gl_uniform_driver_storage structs for each subcomponent that
670 * glGetUniformLocation() could name. We know it's been set up in the same
671 * order we'd walk the type, so walk the list of storage and find anything
672 * with our name, or the prefix of a component that starts with our name.
673 */
674 for (unsigned u = 0; u < shader_prog->NumUserUniformStorage; u++) {
675 struct gl_uniform_storage *storage = &shader_prog->UniformStorage[u];
676
677 if (strncmp(ir->name, storage->name, namelen) != 0 ||
678 (storage->name[namelen] != 0 &&
679 storage->name[namelen] != '.' &&
680 storage->name[namelen] != '[')) {
681 continue;
682 }
683
684 gl_constant_value *components = storage->storage;
685 unsigned vector_count = (MAX2(storage->array_elements, 1) *
686 storage->type->matrix_columns);
687
688 for (unsigned s = 0; s < vector_count; s++) {
689 assert(uniforms < uniform_array_size);
690 uniform_vector_size[uniforms] = storage->type->vector_elements;
691
692 int i;
693 for (i = 0; i < uniform_vector_size[uniforms]; i++) {
694 stage_prog_data->param[uniforms * 4 + i] = components;
695 components++;
696 }
697 for (; i < 4; i++) {
698 static gl_constant_value zero = { 0.0 };
699 stage_prog_data->param[uniforms * 4 + i] = &zero;
700 }
701
702 uniforms++;
703 }
704 }
705 }
706
707 void
708 vec4_visitor::setup_uniform_clipplane_values()
709 {
710 gl_clip_plane *clip_planes = brw_select_clip_planes(ctx);
711
712 for (int i = 0; i < key->nr_userclip_plane_consts; ++i) {
713 assert(this->uniforms < uniform_array_size);
714 this->uniform_vector_size[this->uniforms] = 4;
715 this->userplane[i] = dst_reg(UNIFORM, this->uniforms);
716 this->userplane[i].type = BRW_REGISTER_TYPE_F;
717 for (int j = 0; j < 4; ++j) {
718 stage_prog_data->param[this->uniforms * 4 + j] =
719 (gl_constant_value *) &clip_planes[i][j];
720 }
721 ++this->uniforms;
722 }
723 }
724
725 /* Our support for builtin uniforms is even scarier than non-builtin.
726 * It sits on top of the PROG_STATE_VAR parameters that are
727 * automatically updated from GL context state.
728 */
729 void
730 vec4_visitor::setup_builtin_uniform_values(ir_variable *ir)
731 {
732 const ir_state_slot *const slots = ir->state_slots;
733 assert(ir->state_slots != NULL);
734
735 for (unsigned int i = 0; i < ir->num_state_slots; i++) {
736 /* This state reference has already been setup by ir_to_mesa,
737 * but we'll get the same index back here. We can reference
738 * ParameterValues directly, since unlike brw_fs.cpp, we never
739 * add new state references during compile.
740 */
741 int index = _mesa_add_state_reference(this->prog->Parameters,
742 (gl_state_index *)slots[i].tokens);
743 gl_constant_value *values =
744 &this->prog->Parameters->ParameterValues[index][0];
745
746 assert(this->uniforms < uniform_array_size);
747 this->uniform_vector_size[this->uniforms] = 0;
748 /* Add each of the unique swizzled channels of the element.
749 * This will end up matching the size of the glsl_type of this field.
750 */
751 int last_swiz = -1;
752 for (unsigned int j = 0; j < 4; j++) {
753 int swiz = GET_SWZ(slots[i].swizzle, j);
754 last_swiz = swiz;
755
756 stage_prog_data->param[this->uniforms * 4 + j] = &values[swiz];
757 assert(this->uniforms < uniform_array_size);
758 if (swiz <= last_swiz)
759 this->uniform_vector_size[this->uniforms]++;
760 }
761 this->uniforms++;
762 }
763 }
764
765 dst_reg *
766 vec4_visitor::variable_storage(ir_variable *var)
767 {
768 return (dst_reg *)hash_table_find(this->variable_ht, var);
769 }
770
771 void
772 vec4_visitor::emit_bool_to_cond_code(ir_rvalue *ir,
773 enum brw_predicate *predicate)
774 {
775 ir_expression *expr = ir->as_expression();
776
777 *predicate = BRW_PREDICATE_NORMAL;
778
779 if (expr) {
780 src_reg op[2];
781 vec4_instruction *inst;
782
783 assert(expr->get_num_operands() <= 2);
784 for (unsigned int i = 0; i < expr->get_num_operands(); i++) {
785 expr->operands[i]->accept(this);
786 op[i] = this->result;
787
788 resolve_ud_negate(&op[i]);
789 }
790
791 switch (expr->operation) {
792 case ir_unop_logic_not:
793 inst = emit(AND(dst_null_d(), op[0], src_reg(1)));
794 inst->conditional_mod = BRW_CONDITIONAL_Z;
795 break;
796
797 case ir_binop_logic_xor:
798 inst = emit(XOR(dst_null_d(), op[0], op[1]));
799 inst->conditional_mod = BRW_CONDITIONAL_NZ;
800 break;
801
802 case ir_binop_logic_or:
803 inst = emit(OR(dst_null_d(), op[0], op[1]));
804 inst->conditional_mod = BRW_CONDITIONAL_NZ;
805 break;
806
807 case ir_binop_logic_and:
808 inst = emit(AND(dst_null_d(), op[0], op[1]));
809 inst->conditional_mod = BRW_CONDITIONAL_NZ;
810 break;
811
812 case ir_unop_f2b:
813 if (brw->gen >= 6) {
814 emit(CMP(dst_null_d(), op[0], src_reg(0.0f), BRW_CONDITIONAL_NZ));
815 } else {
816 inst = emit(MOV(dst_null_f(), op[0]));
817 inst->conditional_mod = BRW_CONDITIONAL_NZ;
818 }
819 break;
820
821 case ir_unop_i2b:
822 if (brw->gen >= 6) {
823 emit(CMP(dst_null_d(), op[0], src_reg(0), BRW_CONDITIONAL_NZ));
824 } else {
825 inst = emit(MOV(dst_null_d(), op[0]));
826 inst->conditional_mod = BRW_CONDITIONAL_NZ;
827 }
828 break;
829
830 case ir_binop_all_equal:
831 inst = emit(CMP(dst_null_d(), op[0], op[1], BRW_CONDITIONAL_Z));
832 *predicate = BRW_PREDICATE_ALIGN16_ALL4H;
833 break;
834
835 case ir_binop_any_nequal:
836 inst = emit(CMP(dst_null_d(), op[0], op[1], BRW_CONDITIONAL_NZ));
837 *predicate = BRW_PREDICATE_ALIGN16_ANY4H;
838 break;
839
840 case ir_unop_any:
841 inst = emit(CMP(dst_null_d(), op[0], src_reg(0), BRW_CONDITIONAL_NZ));
842 *predicate = BRW_PREDICATE_ALIGN16_ANY4H;
843 break;
844
845 case ir_binop_greater:
846 case ir_binop_gequal:
847 case ir_binop_less:
848 case ir_binop_lequal:
849 case ir_binop_equal:
850 case ir_binop_nequal:
851 emit(CMP(dst_null_d(), op[0], op[1],
852 brw_conditional_for_comparison(expr->operation)));
853 break;
854
855 default:
856 unreachable("not reached");
857 }
858 return;
859 }
860
861 ir->accept(this);
862
863 resolve_ud_negate(&this->result);
864
865 if (brw->gen >= 6) {
866 vec4_instruction *inst = emit(AND(dst_null_d(),
867 this->result, src_reg(1)));
868 inst->conditional_mod = BRW_CONDITIONAL_NZ;
869 } else {
870 vec4_instruction *inst = emit(MOV(dst_null_d(), this->result));
871 inst->conditional_mod = BRW_CONDITIONAL_NZ;
872 }
873 }
874
875 /**
876 * Emit a gen6 IF statement with the comparison folded into the IF
877 * instruction.
878 */
879 void
880 vec4_visitor::emit_if_gen6(ir_if *ir)
881 {
882 ir_expression *expr = ir->condition->as_expression();
883
884 if (expr) {
885 src_reg op[2];
886 dst_reg temp;
887
888 assert(expr->get_num_operands() <= 2);
889 for (unsigned int i = 0; i < expr->get_num_operands(); i++) {
890 expr->operands[i]->accept(this);
891 op[i] = this->result;
892 }
893
894 switch (expr->operation) {
895 case ir_unop_logic_not:
896 emit(IF(op[0], src_reg(0), BRW_CONDITIONAL_Z));
897 return;
898
899 case ir_binop_logic_xor:
900 emit(IF(op[0], op[1], BRW_CONDITIONAL_NZ));
901 return;
902
903 case ir_binop_logic_or:
904 temp = dst_reg(this, glsl_type::bool_type);
905 emit(OR(temp, op[0], op[1]));
906 emit(IF(src_reg(temp), src_reg(0), BRW_CONDITIONAL_NZ));
907 return;
908
909 case ir_binop_logic_and:
910 temp = dst_reg(this, glsl_type::bool_type);
911 emit(AND(temp, op[0], op[1]));
912 emit(IF(src_reg(temp), src_reg(0), BRW_CONDITIONAL_NZ));
913 return;
914
915 case ir_unop_f2b:
916 emit(IF(op[0], src_reg(0), BRW_CONDITIONAL_NZ));
917 return;
918
919 case ir_unop_i2b:
920 emit(IF(op[0], src_reg(0), BRW_CONDITIONAL_NZ));
921 return;
922
923 case ir_binop_greater:
924 case ir_binop_gequal:
925 case ir_binop_less:
926 case ir_binop_lequal:
927 case ir_binop_equal:
928 case ir_binop_nequal:
929 emit(IF(op[0], op[1],
930 brw_conditional_for_comparison(expr->operation)));
931 return;
932
933 case ir_binop_all_equal:
934 emit(CMP(dst_null_d(), op[0], op[1], BRW_CONDITIONAL_Z));
935 emit(IF(BRW_PREDICATE_ALIGN16_ALL4H));
936 return;
937
938 case ir_binop_any_nequal:
939 emit(CMP(dst_null_d(), op[0], op[1], BRW_CONDITIONAL_NZ));
940 emit(IF(BRW_PREDICATE_ALIGN16_ANY4H));
941 return;
942
943 case ir_unop_any:
944 emit(CMP(dst_null_d(), op[0], src_reg(0), BRW_CONDITIONAL_NZ));
945 emit(IF(BRW_PREDICATE_ALIGN16_ANY4H));
946 return;
947
948 default:
949 unreachable("not reached");
950 }
951 return;
952 }
953
954 ir->condition->accept(this);
955
956 emit(IF(this->result, src_reg(0), BRW_CONDITIONAL_NZ));
957 }
958
959 void
960 vec4_visitor::visit(ir_variable *ir)
961 {
962 dst_reg *reg = NULL;
963
964 if (variable_storage(ir))
965 return;
966
967 switch (ir->data.mode) {
968 case ir_var_shader_in:
969 reg = new(mem_ctx) dst_reg(ATTR, ir->data.location);
970 break;
971
972 case ir_var_shader_out:
973 reg = new(mem_ctx) dst_reg(this, ir->type);
974
975 for (int i = 0; i < type_size(ir->type); i++) {
976 output_reg[ir->data.location + i] = *reg;
977 output_reg[ir->data.location + i].reg_offset = i;
978 output_reg[ir->data.location + i].type =
979 brw_type_for_base_type(ir->type->get_scalar_type());
980 output_reg_annotation[ir->data.location + i] = ir->name;
981 }
982 break;
983
984 case ir_var_auto:
985 case ir_var_temporary:
986 reg = new(mem_ctx) dst_reg(this, ir->type);
987 break;
988
989 case ir_var_uniform:
990 reg = new(this->mem_ctx) dst_reg(UNIFORM, this->uniforms);
991
992 /* Thanks to the lower_ubo_reference pass, we will see only
993 * ir_binop_ubo_load expressions and not ir_dereference_variable for UBO
994 * variables, so no need for them to be in variable_ht.
995 *
996 * Atomic counters take no uniform storage, no need to do
997 * anything here.
998 */
999 if (ir->is_in_uniform_block() || ir->type->contains_atomic())
1000 return;
1001
1002 /* Track how big the whole uniform variable is, in case we need to put a
1003 * copy of its data into pull constants for array access.
1004 */
1005 assert(this->uniforms < uniform_array_size);
1006 this->uniform_size[this->uniforms] = type_size(ir->type);
1007
1008 if (!strncmp(ir->name, "gl_", 3)) {
1009 setup_builtin_uniform_values(ir);
1010 } else {
1011 setup_uniform_values(ir);
1012 }
1013 break;
1014
1015 case ir_var_system_value:
1016 reg = make_reg_for_system_value(ir);
1017 break;
1018
1019 default:
1020 unreachable("not reached");
1021 }
1022
1023 reg->type = brw_type_for_base_type(ir->type);
1024 hash_table_insert(this->variable_ht, reg, ir);
1025 }
1026
1027 void
1028 vec4_visitor::visit(ir_loop *ir)
1029 {
1030 /* We don't want debugging output to print the whole body of the
1031 * loop as the annotation.
1032 */
1033 this->base_ir = NULL;
1034
1035 emit(BRW_OPCODE_DO);
1036
1037 visit_instructions(&ir->body_instructions);
1038
1039 emit(BRW_OPCODE_WHILE);
1040 }
1041
1042 void
1043 vec4_visitor::visit(ir_loop_jump *ir)
1044 {
1045 switch (ir->mode) {
1046 case ir_loop_jump::jump_break:
1047 emit(BRW_OPCODE_BREAK);
1048 break;
1049 case ir_loop_jump::jump_continue:
1050 emit(BRW_OPCODE_CONTINUE);
1051 break;
1052 }
1053 }
1054
1055
1056 void
1057 vec4_visitor::visit(ir_function_signature *)
1058 {
1059 unreachable("not reached");
1060 }
1061
1062 void
1063 vec4_visitor::visit(ir_function *ir)
1064 {
1065 /* Ignore function bodies other than main() -- we shouldn't see calls to
1066 * them since they should all be inlined.
1067 */
1068 if (strcmp(ir->name, "main") == 0) {
1069 const ir_function_signature *sig;
1070 exec_list empty;
1071
1072 sig = ir->matching_signature(NULL, &empty, false);
1073
1074 assert(sig);
1075
1076 visit_instructions(&sig->body);
1077 }
1078 }
1079
1080 bool
1081 vec4_visitor::try_emit_sat(ir_expression *ir)
1082 {
1083 ir_rvalue *sat_src = ir->as_rvalue_to_saturate();
1084 if (!sat_src)
1085 return false;
1086
1087 sat_src->accept(this);
1088 src_reg src = this->result;
1089
1090 this->result = src_reg(this, ir->type);
1091 vec4_instruction *inst;
1092 inst = emit(MOV(dst_reg(this->result), src));
1093 inst->saturate = true;
1094
1095 return true;
1096 }
1097
1098 bool
1099 vec4_visitor::try_emit_mad(ir_expression *ir)
1100 {
1101 /* 3-src instructions were introduced in gen6. */
1102 if (brw->gen < 6)
1103 return false;
1104
1105 /* MAD can only handle floating-point data. */
1106 if (ir->type->base_type != GLSL_TYPE_FLOAT)
1107 return false;
1108
1109 ir_rvalue *nonmul = ir->operands[1];
1110 ir_expression *mul = ir->operands[0]->as_expression();
1111
1112 if (!mul || mul->operation != ir_binop_mul) {
1113 nonmul = ir->operands[0];
1114 mul = ir->operands[1]->as_expression();
1115
1116 if (!mul || mul->operation != ir_binop_mul)
1117 return false;
1118 }
1119
1120 nonmul->accept(this);
1121 src_reg src0 = fix_3src_operand(this->result);
1122
1123 mul->operands[0]->accept(this);
1124 src_reg src1 = fix_3src_operand(this->result);
1125
1126 mul->operands[1]->accept(this);
1127 src_reg src2 = fix_3src_operand(this->result);
1128
1129 this->result = src_reg(this, ir->type);
1130 emit(BRW_OPCODE_MAD, dst_reg(this->result), src0, src1, src2);
1131
1132 return true;
1133 }
1134
1135 bool
1136 vec4_visitor::try_emit_b2f_of_compare(ir_expression *ir)
1137 {
1138 ir_expression *const cmp = ir->operands[0]->as_expression();
1139
1140 if (cmp == NULL)
1141 return false;
1142
1143 switch (cmp->operation) {
1144 case ir_binop_less:
1145 case ir_binop_greater:
1146 case ir_binop_lequal:
1147 case ir_binop_gequal:
1148 case ir_binop_equal:
1149 case ir_binop_nequal:
1150 break;
1151
1152 default:
1153 return false;
1154 }
1155
1156 cmp->operands[0]->accept(this);
1157 const src_reg cmp_src0 = this->result;
1158
1159 cmp->operands[1]->accept(this);
1160 const src_reg cmp_src1 = this->result;
1161
1162 this->result = src_reg(this, ir->type);
1163
1164 emit(CMP(dst_reg(this->result), cmp_src0, cmp_src1,
1165 brw_conditional_for_comparison(cmp->operation)));
1166
1167 /* If the comparison is false, this->result will just happen to be zero.
1168 */
1169 vec4_instruction *const inst = emit(BRW_OPCODE_SEL, dst_reg(this->result),
1170 this->result, src_reg(1.0f));
1171 inst->predicate = BRW_PREDICATE_NORMAL;
1172 inst->predicate_inverse = true;
1173
1174 return true;
1175 }
1176
1177 void
1178 vec4_visitor::emit_minmax(enum brw_conditional_mod conditionalmod, dst_reg dst,
1179 src_reg src0, src_reg src1)
1180 {
1181 vec4_instruction *inst;
1182
1183 if (brw->gen >= 6) {
1184 inst = emit(BRW_OPCODE_SEL, dst, src0, src1);
1185 inst->conditional_mod = conditionalmod;
1186 } else {
1187 emit(CMP(dst, src0, src1, conditionalmod));
1188
1189 inst = emit(BRW_OPCODE_SEL, dst, src0, src1);
1190 inst->predicate = BRW_PREDICATE_NORMAL;
1191 }
1192 }
1193
1194 void
1195 vec4_visitor::emit_lrp(const dst_reg &dst,
1196 const src_reg &x, const src_reg &y, const src_reg &a)
1197 {
1198 if (brw->gen >= 6) {
1199 /* Note that the instruction's argument order is reversed from GLSL
1200 * and the IR.
1201 */
1202 emit(LRP(dst,
1203 fix_3src_operand(a), fix_3src_operand(y), fix_3src_operand(x)));
1204 } else {
1205 /* Earlier generations don't support three source operations, so we
1206 * need to emit x*(1-a) + y*a.
1207 */
1208 dst_reg y_times_a = dst_reg(this, glsl_type::vec4_type);
1209 dst_reg one_minus_a = dst_reg(this, glsl_type::vec4_type);
1210 dst_reg x_times_one_minus_a = dst_reg(this, glsl_type::vec4_type);
1211 y_times_a.writemask = dst.writemask;
1212 one_minus_a.writemask = dst.writemask;
1213 x_times_one_minus_a.writemask = dst.writemask;
1214
1215 emit(MUL(y_times_a, y, a));
1216 emit(ADD(one_minus_a, negate(a), src_reg(1.0f)));
1217 emit(MUL(x_times_one_minus_a, x, src_reg(one_minus_a)));
1218 emit(ADD(dst, src_reg(x_times_one_minus_a), src_reg(y_times_a)));
1219 }
1220 }
1221
1222 void
1223 vec4_visitor::visit(ir_expression *ir)
1224 {
1225 unsigned int operand;
1226 src_reg op[Elements(ir->operands)];
1227 src_reg result_src;
1228 dst_reg result_dst;
1229 vec4_instruction *inst;
1230
1231 if (try_emit_sat(ir))
1232 return;
1233
1234 if (ir->operation == ir_binop_add) {
1235 if (try_emit_mad(ir))
1236 return;
1237 }
1238
1239 if (ir->operation == ir_unop_b2f) {
1240 if (try_emit_b2f_of_compare(ir))
1241 return;
1242 }
1243
1244 for (operand = 0; operand < ir->get_num_operands(); operand++) {
1245 this->result.file = BAD_FILE;
1246 ir->operands[operand]->accept(this);
1247 if (this->result.file == BAD_FILE) {
1248 fprintf(stderr, "Failed to get tree for expression operand:\n");
1249 ir->operands[operand]->fprint(stderr);
1250 exit(1);
1251 }
1252 op[operand] = this->result;
1253
1254 /* Matrix expression operands should have been broken down to vector
1255 * operations already.
1256 */
1257 assert(!ir->operands[operand]->type->is_matrix());
1258 }
1259
1260 int vector_elements = ir->operands[0]->type->vector_elements;
1261 if (ir->operands[1]) {
1262 vector_elements = MAX2(vector_elements,
1263 ir->operands[1]->type->vector_elements);
1264 }
1265
1266 this->result.file = BAD_FILE;
1267
1268 /* Storage for our result. Ideally for an assignment we'd be using
1269 * the actual storage for the result here, instead.
1270 */
1271 result_src = src_reg(this, ir->type);
1272 /* convenience for the emit functions below. */
1273 result_dst = dst_reg(result_src);
1274 /* If nothing special happens, this is the result. */
1275 this->result = result_src;
1276 /* Limit writes to the channels that will be used by result_src later.
1277 * This does limit this temp's use as a temporary for multi-instruction
1278 * sequences.
1279 */
1280 result_dst.writemask = (1 << ir->type->vector_elements) - 1;
1281
1282 switch (ir->operation) {
1283 case ir_unop_logic_not:
1284 /* Note that BRW_OPCODE_NOT is not appropriate here, since it is
1285 * ones complement of the whole register, not just bit 0.
1286 */
1287 emit(XOR(result_dst, op[0], src_reg(1)));
1288 break;
1289 case ir_unop_neg:
1290 op[0].negate = !op[0].negate;
1291 emit(MOV(result_dst, op[0]));
1292 break;
1293 case ir_unop_abs:
1294 op[0].abs = true;
1295 op[0].negate = false;
1296 emit(MOV(result_dst, op[0]));
1297 break;
1298
1299 case ir_unop_sign:
1300 if (ir->type->is_float()) {
1301 /* AND(val, 0x80000000) gives the sign bit.
1302 *
1303 * Predicated OR ORs 1.0 (0x3f800000) with the sign bit if val is not
1304 * zero.
1305 */
1306 emit(CMP(dst_null_f(), op[0], src_reg(0.0f), BRW_CONDITIONAL_NZ));
1307
1308 op[0].type = BRW_REGISTER_TYPE_UD;
1309 result_dst.type = BRW_REGISTER_TYPE_UD;
1310 emit(AND(result_dst, op[0], src_reg(0x80000000u)));
1311
1312 inst = emit(OR(result_dst, src_reg(result_dst), src_reg(0x3f800000u)));
1313 inst->predicate = BRW_PREDICATE_NORMAL;
1314
1315 this->result.type = BRW_REGISTER_TYPE_F;
1316 } else {
1317 /* ASR(val, 31) -> negative val generates 0xffffffff (signed -1).
1318 * -> non-negative val generates 0x00000000.
1319 * Predicated OR sets 1 if val is positive.
1320 */
1321 emit(CMP(dst_null_d(), op[0], src_reg(0), BRW_CONDITIONAL_G));
1322
1323 emit(ASR(result_dst, op[0], src_reg(31)));
1324
1325 inst = emit(OR(result_dst, src_reg(result_dst), src_reg(1)));
1326 inst->predicate = BRW_PREDICATE_NORMAL;
1327 }
1328 break;
1329
1330 case ir_unop_rcp:
1331 emit_math(SHADER_OPCODE_RCP, result_dst, op[0]);
1332 break;
1333
1334 case ir_unop_exp2:
1335 emit_math(SHADER_OPCODE_EXP2, result_dst, op[0]);
1336 break;
1337 case ir_unop_log2:
1338 emit_math(SHADER_OPCODE_LOG2, result_dst, op[0]);
1339 break;
1340 case ir_unop_exp:
1341 case ir_unop_log:
1342 unreachable("not reached: should be handled by ir_explog_to_explog2");
1343 case ir_unop_sin:
1344 case ir_unop_sin_reduced:
1345 emit_math(SHADER_OPCODE_SIN, result_dst, op[0]);
1346 break;
1347 case ir_unop_cos:
1348 case ir_unop_cos_reduced:
1349 emit_math(SHADER_OPCODE_COS, result_dst, op[0]);
1350 break;
1351
1352 case ir_unop_dFdx:
1353 case ir_unop_dFdx_coarse:
1354 case ir_unop_dFdx_fine:
1355 case ir_unop_dFdy:
1356 case ir_unop_dFdy_coarse:
1357 case ir_unop_dFdy_fine:
1358 unreachable("derivatives not valid in vertex shader");
1359
1360 case ir_unop_bitfield_reverse:
1361 emit(BFREV(result_dst, op[0]));
1362 break;
1363 case ir_unop_bit_count:
1364 emit(CBIT(result_dst, op[0]));
1365 break;
1366 case ir_unop_find_msb: {
1367 src_reg temp = src_reg(this, glsl_type::uint_type);
1368
1369 inst = emit(FBH(dst_reg(temp), op[0]));
1370 inst->dst.writemask = WRITEMASK_XYZW;
1371
1372 /* FBH counts from the MSB side, while GLSL's findMSB() wants the count
1373 * from the LSB side. If FBH didn't return an error (0xFFFFFFFF), then
1374 * subtract the result from 31 to convert the MSB count into an LSB count.
1375 */
1376
1377 /* FBH only supports UD type for dst, so use a MOV to convert UD to D. */
1378 temp.swizzle = BRW_SWIZZLE_NOOP;
1379 emit(MOV(result_dst, temp));
1380
1381 src_reg src_tmp = src_reg(result_dst);
1382 emit(CMP(dst_null_d(), src_tmp, src_reg(-1), BRW_CONDITIONAL_NZ));
1383
1384 src_tmp.negate = true;
1385 inst = emit(ADD(result_dst, src_tmp, src_reg(31)));
1386 inst->predicate = BRW_PREDICATE_NORMAL;
1387 break;
1388 }
1389 case ir_unop_find_lsb:
1390 emit(FBL(result_dst, op[0]));
1391 break;
1392
1393 case ir_unop_noise:
1394 unreachable("not reached: should be handled by lower_noise");
1395
1396 case ir_binop_add:
1397 emit(ADD(result_dst, op[0], op[1]));
1398 break;
1399 case ir_binop_sub:
1400 unreachable("not reached: should be handled by ir_sub_to_add_neg");
1401
1402 case ir_binop_mul:
1403 if (brw->gen < 8 && ir->type->is_integer()) {
1404 /* For integer multiplication, the MUL uses the low 16 bits of one of
1405 * the operands (src0 through SNB, src1 on IVB and later). The MACH
1406 * accumulates in the contribution of the upper 16 bits of that
1407 * operand. If we can determine that one of the args is in the low
1408 * 16 bits, though, we can just emit a single MUL.
1409 */
1410 if (ir->operands[0]->is_uint16_constant()) {
1411 if (brw->gen < 7)
1412 emit(MUL(result_dst, op[0], op[1]));
1413 else
1414 emit(MUL(result_dst, op[1], op[0]));
1415 } else if (ir->operands[1]->is_uint16_constant()) {
1416 if (brw->gen < 7)
1417 emit(MUL(result_dst, op[1], op[0]));
1418 else
1419 emit(MUL(result_dst, op[0], op[1]));
1420 } else {
1421 struct brw_reg acc = retype(brw_acc_reg(), result_dst.type);
1422
1423 emit(MUL(acc, op[0], op[1]));
1424 emit(MACH(dst_null_d(), op[0], op[1]));
1425 emit(MOV(result_dst, src_reg(acc)));
1426 }
1427 } else {
1428 emit(MUL(result_dst, op[0], op[1]));
1429 }
1430 break;
1431 case ir_binop_imul_high: {
1432 struct brw_reg acc = retype(brw_acc_reg(), result_dst.type);
1433
1434 emit(MUL(acc, op[0], op[1]));
1435 emit(MACH(result_dst, op[0], op[1]));
1436 break;
1437 }
1438 case ir_binop_div:
1439 /* Floating point should be lowered by DIV_TO_MUL_RCP in the compiler. */
1440 assert(ir->type->is_integer());
1441 emit_math(SHADER_OPCODE_INT_QUOTIENT, result_dst, op[0], op[1]);
1442 break;
1443 case ir_binop_carry: {
1444 struct brw_reg acc = retype(brw_acc_reg(), BRW_REGISTER_TYPE_UD);
1445
1446 emit(ADDC(dst_null_ud(), op[0], op[1]));
1447 emit(MOV(result_dst, src_reg(acc)));
1448 break;
1449 }
1450 case ir_binop_borrow: {
1451 struct brw_reg acc = retype(brw_acc_reg(), BRW_REGISTER_TYPE_UD);
1452
1453 emit(SUBB(dst_null_ud(), op[0], op[1]));
1454 emit(MOV(result_dst, src_reg(acc)));
1455 break;
1456 }
1457 case ir_binop_mod:
1458 /* Floating point should be lowered by MOD_TO_FRACT in the compiler. */
1459 assert(ir->type->is_integer());
1460 emit_math(SHADER_OPCODE_INT_REMAINDER, result_dst, op[0], op[1]);
1461 break;
1462
1463 case ir_binop_less:
1464 case ir_binop_greater:
1465 case ir_binop_lequal:
1466 case ir_binop_gequal:
1467 case ir_binop_equal:
1468 case ir_binop_nequal: {
1469 emit(CMP(result_dst, op[0], op[1],
1470 brw_conditional_for_comparison(ir->operation)));
1471 emit(AND(result_dst, result_src, src_reg(0x1)));
1472 break;
1473 }
1474
1475 case ir_binop_all_equal:
1476 /* "==" operator producing a scalar boolean. */
1477 if (ir->operands[0]->type->is_vector() ||
1478 ir->operands[1]->type->is_vector()) {
1479 emit(CMP(dst_null_d(), op[0], op[1], BRW_CONDITIONAL_Z));
1480 emit(MOV(result_dst, src_reg(0)));
1481 inst = emit(MOV(result_dst, src_reg(1)));
1482 inst->predicate = BRW_PREDICATE_ALIGN16_ALL4H;
1483 } else {
1484 emit(CMP(result_dst, op[0], op[1], BRW_CONDITIONAL_Z));
1485 emit(AND(result_dst, result_src, src_reg(0x1)));
1486 }
1487 break;
1488 case ir_binop_any_nequal:
1489 /* "!=" operator producing a scalar boolean. */
1490 if (ir->operands[0]->type->is_vector() ||
1491 ir->operands[1]->type->is_vector()) {
1492 emit(CMP(dst_null_d(), op[0], op[1], BRW_CONDITIONAL_NZ));
1493
1494 emit(MOV(result_dst, src_reg(0)));
1495 inst = emit(MOV(result_dst, src_reg(1)));
1496 inst->predicate = BRW_PREDICATE_ALIGN16_ANY4H;
1497 } else {
1498 emit(CMP(result_dst, op[0], op[1], BRW_CONDITIONAL_NZ));
1499 emit(AND(result_dst, result_src, src_reg(0x1)));
1500 }
1501 break;
1502
1503 case ir_unop_any:
1504 emit(CMP(dst_null_d(), op[0], src_reg(0), BRW_CONDITIONAL_NZ));
1505 emit(MOV(result_dst, src_reg(0)));
1506
1507 inst = emit(MOV(result_dst, src_reg(1)));
1508 inst->predicate = BRW_PREDICATE_ALIGN16_ANY4H;
1509 break;
1510
1511 case ir_binop_logic_xor:
1512 emit(XOR(result_dst, op[0], op[1]));
1513 break;
1514
1515 case ir_binop_logic_or:
1516 emit(OR(result_dst, op[0], op[1]));
1517 break;
1518
1519 case ir_binop_logic_and:
1520 emit(AND(result_dst, op[0], op[1]));
1521 break;
1522
1523 case ir_binop_dot:
1524 assert(ir->operands[0]->type->is_vector());
1525 assert(ir->operands[0]->type == ir->operands[1]->type);
1526 emit_dp(result_dst, op[0], op[1], ir->operands[0]->type->vector_elements);
1527 break;
1528
1529 case ir_unop_sqrt:
1530 emit_math(SHADER_OPCODE_SQRT, result_dst, op[0]);
1531 break;
1532 case ir_unop_rsq:
1533 emit_math(SHADER_OPCODE_RSQ, result_dst, op[0]);
1534 break;
1535
1536 case ir_unop_bitcast_i2f:
1537 case ir_unop_bitcast_u2f:
1538 this->result = op[0];
1539 this->result.type = BRW_REGISTER_TYPE_F;
1540 break;
1541
1542 case ir_unop_bitcast_f2i:
1543 this->result = op[0];
1544 this->result.type = BRW_REGISTER_TYPE_D;
1545 break;
1546
1547 case ir_unop_bitcast_f2u:
1548 this->result = op[0];
1549 this->result.type = BRW_REGISTER_TYPE_UD;
1550 break;
1551
1552 case ir_unop_i2f:
1553 case ir_unop_i2u:
1554 case ir_unop_u2i:
1555 case ir_unop_u2f:
1556 case ir_unop_b2f:
1557 case ir_unop_b2i:
1558 case ir_unop_f2i:
1559 case ir_unop_f2u:
1560 emit(MOV(result_dst, op[0]));
1561 break;
1562 case ir_unop_f2b:
1563 case ir_unop_i2b: {
1564 emit(CMP(result_dst, op[0], src_reg(0.0f), BRW_CONDITIONAL_NZ));
1565 emit(AND(result_dst, result_src, src_reg(1)));
1566 break;
1567 }
1568
1569 case ir_unop_trunc:
1570 emit(RNDZ(result_dst, op[0]));
1571 break;
1572 case ir_unop_ceil:
1573 op[0].negate = !op[0].negate;
1574 inst = emit(RNDD(result_dst, op[0]));
1575 this->result.negate = true;
1576 break;
1577 case ir_unop_floor:
1578 inst = emit(RNDD(result_dst, op[0]));
1579 break;
1580 case ir_unop_fract:
1581 inst = emit(FRC(result_dst, op[0]));
1582 break;
1583 case ir_unop_round_even:
1584 emit(RNDE(result_dst, op[0]));
1585 break;
1586
1587 case ir_binop_min:
1588 emit_minmax(BRW_CONDITIONAL_L, result_dst, op[0], op[1]);
1589 break;
1590 case ir_binop_max:
1591 emit_minmax(BRW_CONDITIONAL_G, result_dst, op[0], op[1]);
1592 break;
1593
1594 case ir_binop_pow:
1595 emit_math(SHADER_OPCODE_POW, result_dst, op[0], op[1]);
1596 break;
1597
1598 case ir_unop_bit_not:
1599 inst = emit(NOT(result_dst, op[0]));
1600 break;
1601 case ir_binop_bit_and:
1602 inst = emit(AND(result_dst, op[0], op[1]));
1603 break;
1604 case ir_binop_bit_xor:
1605 inst = emit(XOR(result_dst, op[0], op[1]));
1606 break;
1607 case ir_binop_bit_or:
1608 inst = emit(OR(result_dst, op[0], op[1]));
1609 break;
1610
1611 case ir_binop_lshift:
1612 inst = emit(SHL(result_dst, op[0], op[1]));
1613 break;
1614
1615 case ir_binop_rshift:
1616 if (ir->type->base_type == GLSL_TYPE_INT)
1617 inst = emit(ASR(result_dst, op[0], op[1]));
1618 else
1619 inst = emit(SHR(result_dst, op[0], op[1]));
1620 break;
1621
1622 case ir_binop_bfm:
1623 emit(BFI1(result_dst, op[0], op[1]));
1624 break;
1625
1626 case ir_binop_ubo_load: {
1627 ir_constant *uniform_block = ir->operands[0]->as_constant();
1628 ir_constant *const_offset_ir = ir->operands[1]->as_constant();
1629 unsigned const_offset = const_offset_ir ? const_offset_ir->value.u[0] : 0;
1630 src_reg offset;
1631
1632 /* Now, load the vector from that offset. */
1633 assert(ir->type->is_vector() || ir->type->is_scalar());
1634
1635 src_reg packed_consts = src_reg(this, glsl_type::vec4_type);
1636 packed_consts.type = result.type;
1637 src_reg surf_index =
1638 src_reg(prog_data->base.binding_table.ubo_start + uniform_block->value.u[0]);
1639 if (const_offset_ir) {
1640 if (brw->gen >= 8) {
1641 /* Store the offset in a GRF so we can send-from-GRF. */
1642 offset = src_reg(this, glsl_type::int_type);
1643 emit(MOV(dst_reg(offset), src_reg(const_offset / 16)));
1644 } else {
1645 /* Immediates are fine on older generations since they'll be moved
1646 * to a (potentially fake) MRF at the generator level.
1647 */
1648 offset = src_reg(const_offset / 16);
1649 }
1650 } else {
1651 offset = src_reg(this, glsl_type::uint_type);
1652 emit(SHR(dst_reg(offset), op[1], src_reg(4)));
1653 }
1654
1655 if (brw->gen >= 7) {
1656 dst_reg grf_offset = dst_reg(this, glsl_type::int_type);
1657 grf_offset.type = offset.type;
1658
1659 emit(MOV(grf_offset, offset));
1660
1661 emit(new(mem_ctx) vec4_instruction(this,
1662 VS_OPCODE_PULL_CONSTANT_LOAD_GEN7,
1663 dst_reg(packed_consts),
1664 surf_index,
1665 src_reg(grf_offset)));
1666 } else {
1667 vec4_instruction *pull =
1668 emit(new(mem_ctx) vec4_instruction(this,
1669 VS_OPCODE_PULL_CONSTANT_LOAD,
1670 dst_reg(packed_consts),
1671 surf_index,
1672 offset));
1673 pull->base_mrf = 14;
1674 pull->mlen = 1;
1675 }
1676
1677 packed_consts.swizzle = swizzle_for_size(ir->type->vector_elements);
1678 packed_consts.swizzle += BRW_SWIZZLE4(const_offset % 16 / 4,
1679 const_offset % 16 / 4,
1680 const_offset % 16 / 4,
1681 const_offset % 16 / 4);
1682
1683 /* UBO bools are any nonzero int. We store bools as either 0 or 1. */
1684 if (ir->type->base_type == GLSL_TYPE_BOOL) {
1685 emit(CMP(result_dst, packed_consts, src_reg(0u),
1686 BRW_CONDITIONAL_NZ));
1687 emit(AND(result_dst, result, src_reg(0x1)));
1688 } else {
1689 emit(MOV(result_dst, packed_consts));
1690 }
1691 break;
1692 }
1693
1694 case ir_binop_vector_extract:
1695 unreachable("should have been lowered by vec_index_to_cond_assign");
1696
1697 case ir_triop_fma:
1698 op[0] = fix_3src_operand(op[0]);
1699 op[1] = fix_3src_operand(op[1]);
1700 op[2] = fix_3src_operand(op[2]);
1701 /* Note that the instruction's argument order is reversed from GLSL
1702 * and the IR.
1703 */
1704 emit(MAD(result_dst, op[2], op[1], op[0]));
1705 break;
1706
1707 case ir_triop_lrp:
1708 emit_lrp(result_dst, op[0], op[1], op[2]);
1709 break;
1710
1711 case ir_triop_csel:
1712 emit(CMP(dst_null_d(), op[0], src_reg(0), BRW_CONDITIONAL_NZ));
1713 inst = emit(BRW_OPCODE_SEL, result_dst, op[1], op[2]);
1714 inst->predicate = BRW_PREDICATE_NORMAL;
1715 break;
1716
1717 case ir_triop_bfi:
1718 op[0] = fix_3src_operand(op[0]);
1719 op[1] = fix_3src_operand(op[1]);
1720 op[2] = fix_3src_operand(op[2]);
1721 emit(BFI2(result_dst, op[0], op[1], op[2]));
1722 break;
1723
1724 case ir_triop_bitfield_extract:
1725 op[0] = fix_3src_operand(op[0]);
1726 op[1] = fix_3src_operand(op[1]);
1727 op[2] = fix_3src_operand(op[2]);
1728 /* Note that the instruction's argument order is reversed from GLSL
1729 * and the IR.
1730 */
1731 emit(BFE(result_dst, op[2], op[1], op[0]));
1732 break;
1733
1734 case ir_triop_vector_insert:
1735 unreachable("should have been lowered by lower_vector_insert");
1736
1737 case ir_quadop_bitfield_insert:
1738 unreachable("not reached: should be handled by "
1739 "bitfield_insert_to_bfm_bfi\n");
1740
1741 case ir_quadop_vector:
1742 unreachable("not reached: should be handled by lower_quadop_vector");
1743
1744 case ir_unop_pack_half_2x16:
1745 emit_pack_half_2x16(result_dst, op[0]);
1746 break;
1747 case ir_unop_unpack_half_2x16:
1748 emit_unpack_half_2x16(result_dst, op[0]);
1749 break;
1750 case ir_unop_pack_snorm_2x16:
1751 case ir_unop_pack_snorm_4x8:
1752 case ir_unop_pack_unorm_2x16:
1753 case ir_unop_pack_unorm_4x8:
1754 case ir_unop_unpack_snorm_2x16:
1755 case ir_unop_unpack_snorm_4x8:
1756 case ir_unop_unpack_unorm_2x16:
1757 case ir_unop_unpack_unorm_4x8:
1758 unreachable("not reached: should be handled by lower_packing_builtins");
1759 case ir_unop_unpack_half_2x16_split_x:
1760 case ir_unop_unpack_half_2x16_split_y:
1761 case ir_binop_pack_half_2x16_split:
1762 case ir_unop_interpolate_at_centroid:
1763 case ir_binop_interpolate_at_sample:
1764 case ir_binop_interpolate_at_offset:
1765 unreachable("not reached: should not occur in vertex shader");
1766 case ir_binop_ldexp:
1767 unreachable("not reached: should be handled by ldexp_to_arith()");
1768 }
1769 }
1770
1771
1772 void
1773 vec4_visitor::visit(ir_swizzle *ir)
1774 {
1775 src_reg src;
1776 int i = 0;
1777 int swizzle[4];
1778
1779 /* Note that this is only swizzles in expressions, not those on the left
1780 * hand side of an assignment, which do write masking. See ir_assignment
1781 * for that.
1782 */
1783
1784 ir->val->accept(this);
1785 src = this->result;
1786 assert(src.file != BAD_FILE);
1787
1788 for (i = 0; i < ir->type->vector_elements; i++) {
1789 switch (i) {
1790 case 0:
1791 swizzle[i] = BRW_GET_SWZ(src.swizzle, ir->mask.x);
1792 break;
1793 case 1:
1794 swizzle[i] = BRW_GET_SWZ(src.swizzle, ir->mask.y);
1795 break;
1796 case 2:
1797 swizzle[i] = BRW_GET_SWZ(src.swizzle, ir->mask.z);
1798 break;
1799 case 3:
1800 swizzle[i] = BRW_GET_SWZ(src.swizzle, ir->mask.w);
1801 break;
1802 }
1803 }
1804 for (; i < 4; i++) {
1805 /* Replicate the last channel out. */
1806 swizzle[i] = swizzle[ir->type->vector_elements - 1];
1807 }
1808
1809 src.swizzle = BRW_SWIZZLE4(swizzle[0], swizzle[1], swizzle[2], swizzle[3]);
1810
1811 this->result = src;
1812 }
1813
1814 void
1815 vec4_visitor::visit(ir_dereference_variable *ir)
1816 {
1817 const struct glsl_type *type = ir->type;
1818 dst_reg *reg = variable_storage(ir->var);
1819
1820 if (!reg) {
1821 fail("Failed to find variable storage for %s\n", ir->var->name);
1822 this->result = src_reg(brw_null_reg());
1823 return;
1824 }
1825
1826 this->result = src_reg(*reg);
1827
1828 /* System values get their swizzle from the dst_reg writemask */
1829 if (ir->var->data.mode == ir_var_system_value)
1830 return;
1831
1832 if (type->is_scalar() || type->is_vector() || type->is_matrix())
1833 this->result.swizzle = swizzle_for_size(type->vector_elements);
1834 }
1835
1836
1837 int
1838 vec4_visitor::compute_array_stride(ir_dereference_array *ir)
1839 {
1840 /* Under normal circumstances array elements are stored consecutively, so
1841 * the stride is equal to the size of the array element.
1842 */
1843 return type_size(ir->type);
1844 }
1845
1846
1847 void
1848 vec4_visitor::visit(ir_dereference_array *ir)
1849 {
1850 ir_constant *constant_index;
1851 src_reg src;
1852 int array_stride = compute_array_stride(ir);
1853
1854 constant_index = ir->array_index->constant_expression_value();
1855
1856 ir->array->accept(this);
1857 src = this->result;
1858
1859 if (constant_index) {
1860 src.reg_offset += constant_index->value.i[0] * array_stride;
1861 } else {
1862 /* Variable index array dereference. It eats the "vec4" of the
1863 * base of the array and an index that offsets the Mesa register
1864 * index.
1865 */
1866 ir->array_index->accept(this);
1867
1868 src_reg index_reg;
1869
1870 if (array_stride == 1) {
1871 index_reg = this->result;
1872 } else {
1873 index_reg = src_reg(this, glsl_type::int_type);
1874
1875 emit(MUL(dst_reg(index_reg), this->result, src_reg(array_stride)));
1876 }
1877
1878 if (src.reladdr) {
1879 src_reg temp = src_reg(this, glsl_type::int_type);
1880
1881 emit(ADD(dst_reg(temp), *src.reladdr, index_reg));
1882
1883 index_reg = temp;
1884 }
1885
1886 src.reladdr = ralloc(mem_ctx, src_reg);
1887 memcpy(src.reladdr, &index_reg, sizeof(index_reg));
1888 }
1889
1890 /* If the type is smaller than a vec4, replicate the last channel out. */
1891 if (ir->type->is_scalar() || ir->type->is_vector() || ir->type->is_matrix())
1892 src.swizzle = swizzle_for_size(ir->type->vector_elements);
1893 else
1894 src.swizzle = BRW_SWIZZLE_NOOP;
1895 src.type = brw_type_for_base_type(ir->type);
1896
1897 this->result = src;
1898 }
1899
1900 void
1901 vec4_visitor::visit(ir_dereference_record *ir)
1902 {
1903 unsigned int i;
1904 const glsl_type *struct_type = ir->record->type;
1905 int offset = 0;
1906
1907 ir->record->accept(this);
1908
1909 for (i = 0; i < struct_type->length; i++) {
1910 if (strcmp(struct_type->fields.structure[i].name, ir->field) == 0)
1911 break;
1912 offset += type_size(struct_type->fields.structure[i].type);
1913 }
1914
1915 /* If the type is smaller than a vec4, replicate the last channel out. */
1916 if (ir->type->is_scalar() || ir->type->is_vector() || ir->type->is_matrix())
1917 this->result.swizzle = swizzle_for_size(ir->type->vector_elements);
1918 else
1919 this->result.swizzle = BRW_SWIZZLE_NOOP;
1920 this->result.type = brw_type_for_base_type(ir->type);
1921
1922 this->result.reg_offset += offset;
1923 }
1924
1925 /**
1926 * We want to be careful in assignment setup to hit the actual storage
1927 * instead of potentially using a temporary like we might with the
1928 * ir_dereference handler.
1929 */
1930 static dst_reg
1931 get_assignment_lhs(ir_dereference *ir, vec4_visitor *v)
1932 {
1933 /* The LHS must be a dereference. If the LHS is a variable indexed array
1934 * access of a vector, it must be separated into a series conditional moves
1935 * before reaching this point (see ir_vec_index_to_cond_assign).
1936 */
1937 assert(ir->as_dereference());
1938 ir_dereference_array *deref_array = ir->as_dereference_array();
1939 if (deref_array) {
1940 assert(!deref_array->array->type->is_vector());
1941 }
1942
1943 /* Use the rvalue deref handler for the most part. We'll ignore
1944 * swizzles in it and write swizzles using writemask, though.
1945 */
1946 ir->accept(v);
1947 return dst_reg(v->result);
1948 }
1949
1950 void
1951 vec4_visitor::emit_block_move(dst_reg *dst, src_reg *src,
1952 const struct glsl_type *type,
1953 enum brw_predicate predicate)
1954 {
1955 if (type->base_type == GLSL_TYPE_STRUCT) {
1956 for (unsigned int i = 0; i < type->length; i++) {
1957 emit_block_move(dst, src, type->fields.structure[i].type, predicate);
1958 }
1959 return;
1960 }
1961
1962 if (type->is_array()) {
1963 for (unsigned int i = 0; i < type->length; i++) {
1964 emit_block_move(dst, src, type->fields.array, predicate);
1965 }
1966 return;
1967 }
1968
1969 if (type->is_matrix()) {
1970 const struct glsl_type *vec_type;
1971
1972 vec_type = glsl_type::get_instance(GLSL_TYPE_FLOAT,
1973 type->vector_elements, 1);
1974
1975 for (int i = 0; i < type->matrix_columns; i++) {
1976 emit_block_move(dst, src, vec_type, predicate);
1977 }
1978 return;
1979 }
1980
1981 assert(type->is_scalar() || type->is_vector());
1982
1983 dst->type = brw_type_for_base_type(type);
1984 src->type = dst->type;
1985
1986 dst->writemask = (1 << type->vector_elements) - 1;
1987
1988 src->swizzle = swizzle_for_size(type->vector_elements);
1989
1990 vec4_instruction *inst = emit(MOV(*dst, *src));
1991 inst->predicate = predicate;
1992
1993 dst->reg_offset++;
1994 src->reg_offset++;
1995 }
1996
1997
1998 /* If the RHS processing resulted in an instruction generating a
1999 * temporary value, and it would be easy to rewrite the instruction to
2000 * generate its result right into the LHS instead, do so. This ends
2001 * up reliably removing instructions where it can be tricky to do so
2002 * later without real UD chain information.
2003 */
2004 bool
2005 vec4_visitor::try_rewrite_rhs_to_dst(ir_assignment *ir,
2006 dst_reg dst,
2007 src_reg src,
2008 vec4_instruction *pre_rhs_inst,
2009 vec4_instruction *last_rhs_inst)
2010 {
2011 /* This could be supported, but it would take more smarts. */
2012 if (ir->condition)
2013 return false;
2014
2015 if (pre_rhs_inst == last_rhs_inst)
2016 return false; /* No instructions generated to work with. */
2017
2018 /* Make sure the last instruction generated our source reg. */
2019 if (src.file != GRF ||
2020 src.file != last_rhs_inst->dst.file ||
2021 src.reg != last_rhs_inst->dst.reg ||
2022 src.reg_offset != last_rhs_inst->dst.reg_offset ||
2023 src.reladdr ||
2024 src.abs ||
2025 src.negate ||
2026 last_rhs_inst->predicate != BRW_PREDICATE_NONE)
2027 return false;
2028
2029 /* Check that that last instruction fully initialized the channels
2030 * we want to use, in the order we want to use them. We could
2031 * potentially reswizzle the operands of many instructions so that
2032 * we could handle out of order channels, but don't yet.
2033 */
2034
2035 for (unsigned i = 0; i < 4; i++) {
2036 if (dst.writemask & (1 << i)) {
2037 if (!(last_rhs_inst->dst.writemask & (1 << i)))
2038 return false;
2039
2040 if (BRW_GET_SWZ(src.swizzle, i) != i)
2041 return false;
2042 }
2043 }
2044
2045 /* Success! Rewrite the instruction. */
2046 last_rhs_inst->dst.file = dst.file;
2047 last_rhs_inst->dst.reg = dst.reg;
2048 last_rhs_inst->dst.reg_offset = dst.reg_offset;
2049 last_rhs_inst->dst.reladdr = dst.reladdr;
2050 last_rhs_inst->dst.writemask &= dst.writemask;
2051
2052 return true;
2053 }
2054
2055 void
2056 vec4_visitor::visit(ir_assignment *ir)
2057 {
2058 dst_reg dst = get_assignment_lhs(ir->lhs, this);
2059 enum brw_predicate predicate = BRW_PREDICATE_NONE;
2060
2061 if (!ir->lhs->type->is_scalar() &&
2062 !ir->lhs->type->is_vector()) {
2063 ir->rhs->accept(this);
2064 src_reg src = this->result;
2065
2066 if (ir->condition) {
2067 emit_bool_to_cond_code(ir->condition, &predicate);
2068 }
2069
2070 /* emit_block_move doesn't account for swizzles in the source register.
2071 * This should be ok, since the source register is a structure or an
2072 * array, and those can't be swizzled. But double-check to be sure.
2073 */
2074 assert(src.swizzle ==
2075 (ir->rhs->type->is_matrix()
2076 ? swizzle_for_size(ir->rhs->type->vector_elements)
2077 : BRW_SWIZZLE_NOOP));
2078
2079 emit_block_move(&dst, &src, ir->rhs->type, predicate);
2080 return;
2081 }
2082
2083 /* Now we're down to just a scalar/vector with writemasks. */
2084 int i;
2085
2086 vec4_instruction *pre_rhs_inst, *last_rhs_inst;
2087 pre_rhs_inst = (vec4_instruction *)this->instructions.get_tail();
2088
2089 ir->rhs->accept(this);
2090
2091 last_rhs_inst = (vec4_instruction *)this->instructions.get_tail();
2092
2093 src_reg src = this->result;
2094
2095 int swizzles[4];
2096 int first_enabled_chan = 0;
2097 int src_chan = 0;
2098
2099 assert(ir->lhs->type->is_vector() ||
2100 ir->lhs->type->is_scalar());
2101 dst.writemask = ir->write_mask;
2102
2103 for (int i = 0; i < 4; i++) {
2104 if (dst.writemask & (1 << i)) {
2105 first_enabled_chan = BRW_GET_SWZ(src.swizzle, i);
2106 break;
2107 }
2108 }
2109
2110 /* Swizzle a small RHS vector into the channels being written.
2111 *
2112 * glsl ir treats write_mask as dictating how many channels are
2113 * present on the RHS while in our instructions we need to make
2114 * those channels appear in the slots of the vec4 they're written to.
2115 */
2116 for (int i = 0; i < 4; i++) {
2117 if (dst.writemask & (1 << i))
2118 swizzles[i] = BRW_GET_SWZ(src.swizzle, src_chan++);
2119 else
2120 swizzles[i] = first_enabled_chan;
2121 }
2122 src.swizzle = BRW_SWIZZLE4(swizzles[0], swizzles[1],
2123 swizzles[2], swizzles[3]);
2124
2125 if (try_rewrite_rhs_to_dst(ir, dst, src, pre_rhs_inst, last_rhs_inst)) {
2126 return;
2127 }
2128
2129 if (ir->condition) {
2130 emit_bool_to_cond_code(ir->condition, &predicate);
2131 }
2132
2133 for (i = 0; i < type_size(ir->lhs->type); i++) {
2134 vec4_instruction *inst = emit(MOV(dst, src));
2135 inst->predicate = predicate;
2136
2137 dst.reg_offset++;
2138 src.reg_offset++;
2139 }
2140 }
2141
2142 void
2143 vec4_visitor::emit_constant_values(dst_reg *dst, ir_constant *ir)
2144 {
2145 if (ir->type->base_type == GLSL_TYPE_STRUCT) {
2146 foreach_in_list(ir_constant, field_value, &ir->components) {
2147 emit_constant_values(dst, field_value);
2148 }
2149 return;
2150 }
2151
2152 if (ir->type->is_array()) {
2153 for (unsigned int i = 0; i < ir->type->length; i++) {
2154 emit_constant_values(dst, ir->array_elements[i]);
2155 }
2156 return;
2157 }
2158
2159 if (ir->type->is_matrix()) {
2160 for (int i = 0; i < ir->type->matrix_columns; i++) {
2161 float *vec = &ir->value.f[i * ir->type->vector_elements];
2162
2163 for (int j = 0; j < ir->type->vector_elements; j++) {
2164 dst->writemask = 1 << j;
2165 dst->type = BRW_REGISTER_TYPE_F;
2166
2167 emit(MOV(*dst, src_reg(vec[j])));
2168 }
2169 dst->reg_offset++;
2170 }
2171 return;
2172 }
2173
2174 int remaining_writemask = (1 << ir->type->vector_elements) - 1;
2175
2176 for (int i = 0; i < ir->type->vector_elements; i++) {
2177 if (!(remaining_writemask & (1 << i)))
2178 continue;
2179
2180 dst->writemask = 1 << i;
2181 dst->type = brw_type_for_base_type(ir->type);
2182
2183 /* Find other components that match the one we're about to
2184 * write. Emits fewer instructions for things like vec4(0.5,
2185 * 1.5, 1.5, 1.5).
2186 */
2187 for (int j = i + 1; j < ir->type->vector_elements; j++) {
2188 if (ir->type->base_type == GLSL_TYPE_BOOL) {
2189 if (ir->value.b[i] == ir->value.b[j])
2190 dst->writemask |= (1 << j);
2191 } else {
2192 /* u, i, and f storage all line up, so no need for a
2193 * switch case for comparing each type.
2194 */
2195 if (ir->value.u[i] == ir->value.u[j])
2196 dst->writemask |= (1 << j);
2197 }
2198 }
2199
2200 switch (ir->type->base_type) {
2201 case GLSL_TYPE_FLOAT:
2202 emit(MOV(*dst, src_reg(ir->value.f[i])));
2203 break;
2204 case GLSL_TYPE_INT:
2205 emit(MOV(*dst, src_reg(ir->value.i[i])));
2206 break;
2207 case GLSL_TYPE_UINT:
2208 emit(MOV(*dst, src_reg(ir->value.u[i])));
2209 break;
2210 case GLSL_TYPE_BOOL:
2211 emit(MOV(*dst, src_reg(ir->value.b[i])));
2212 break;
2213 default:
2214 unreachable("Non-float/uint/int/bool constant");
2215 }
2216
2217 remaining_writemask &= ~dst->writemask;
2218 }
2219 dst->reg_offset++;
2220 }
2221
2222 void
2223 vec4_visitor::visit(ir_constant *ir)
2224 {
2225 dst_reg dst = dst_reg(this, ir->type);
2226 this->result = src_reg(dst);
2227
2228 emit_constant_values(&dst, ir);
2229 }
2230
2231 void
2232 vec4_visitor::visit_atomic_counter_intrinsic(ir_call *ir)
2233 {
2234 ir_dereference *deref = static_cast<ir_dereference *>(
2235 ir->actual_parameters.get_head());
2236 ir_variable *location = deref->variable_referenced();
2237 unsigned surf_index = (prog_data->base.binding_table.abo_start +
2238 location->data.atomic.buffer_index);
2239
2240 /* Calculate the surface offset */
2241 src_reg offset(this, glsl_type::uint_type);
2242 ir_dereference_array *deref_array = deref->as_dereference_array();
2243 if (deref_array) {
2244 deref_array->array_index->accept(this);
2245
2246 src_reg tmp(this, glsl_type::uint_type);
2247 emit(MUL(dst_reg(tmp), this->result, ATOMIC_COUNTER_SIZE));
2248 emit(ADD(dst_reg(offset), tmp, location->data.atomic.offset));
2249 } else {
2250 offset = location->data.atomic.offset;
2251 }
2252
2253 /* Emit the appropriate machine instruction */
2254 const char *callee = ir->callee->function_name();
2255 dst_reg dst = get_assignment_lhs(ir->return_deref, this);
2256
2257 if (!strcmp("__intrinsic_atomic_read", callee)) {
2258 emit_untyped_surface_read(surf_index, dst, offset);
2259
2260 } else if (!strcmp("__intrinsic_atomic_increment", callee)) {
2261 emit_untyped_atomic(BRW_AOP_INC, surf_index, dst, offset,
2262 src_reg(), src_reg());
2263
2264 } else if (!strcmp("__intrinsic_atomic_predecrement", callee)) {
2265 emit_untyped_atomic(BRW_AOP_PREDEC, surf_index, dst, offset,
2266 src_reg(), src_reg());
2267 }
2268 }
2269
2270 void
2271 vec4_visitor::visit(ir_call *ir)
2272 {
2273 const char *callee = ir->callee->function_name();
2274
2275 if (!strcmp("__intrinsic_atomic_read", callee) ||
2276 !strcmp("__intrinsic_atomic_increment", callee) ||
2277 !strcmp("__intrinsic_atomic_predecrement", callee)) {
2278 visit_atomic_counter_intrinsic(ir);
2279 } else {
2280 unreachable("Unsupported intrinsic.");
2281 }
2282 }
2283
2284 src_reg
2285 vec4_visitor::emit_mcs_fetch(ir_texture *ir, src_reg coordinate, uint32_t sampler)
2286 {
2287 vec4_instruction *inst = new(mem_ctx) vec4_instruction(this, SHADER_OPCODE_TXF_MCS);
2288 inst->base_mrf = 2;
2289 inst->mlen = 1;
2290 inst->dst = dst_reg(this, glsl_type::uvec4_type);
2291 inst->dst.writemask = WRITEMASK_XYZW;
2292
2293 inst->src[1] = src_reg(sampler);
2294
2295 /* parameters are: u, v, r, lod; lod will always be zero due to api restrictions */
2296 int param_base = inst->base_mrf;
2297 int coord_mask = (1 << ir->coordinate->type->vector_elements) - 1;
2298 int zero_mask = 0xf & ~coord_mask;
2299
2300 emit(MOV(dst_reg(MRF, param_base, ir->coordinate->type, coord_mask),
2301 coordinate));
2302
2303 emit(MOV(dst_reg(MRF, param_base, ir->coordinate->type, zero_mask),
2304 src_reg(0)));
2305
2306 emit(inst);
2307 return src_reg(inst->dst);
2308 }
2309
2310 void
2311 vec4_visitor::visit(ir_texture *ir)
2312 {
2313 uint32_t sampler =
2314 _mesa_get_sampler_uniform_value(ir->sampler, shader_prog, prog);
2315
2316 /* When tg4 is used with the degenerate ZERO/ONE swizzles, don't bother
2317 * emitting anything other than setting up the constant result.
2318 */
2319 if (ir->op == ir_tg4) {
2320 ir_constant *chan = ir->lod_info.component->as_constant();
2321 int swiz = GET_SWZ(key->tex.swizzles[sampler], chan->value.i[0]);
2322 if (swiz == SWIZZLE_ZERO || swiz == SWIZZLE_ONE) {
2323 dst_reg result(this, ir->type);
2324 this->result = src_reg(result);
2325 emit(MOV(result, src_reg(swiz == SWIZZLE_ONE ? 1.0f : 0.0f)));
2326 return;
2327 }
2328 }
2329
2330 /* Should be lowered by do_lower_texture_projection */
2331 assert(!ir->projector);
2332
2333 /* Should be lowered */
2334 assert(!ir->offset || !ir->offset->type->is_array());
2335
2336 /* Generate code to compute all the subexpression trees. This has to be
2337 * done before loading any values into MRFs for the sampler message since
2338 * generating these values may involve SEND messages that need the MRFs.
2339 */
2340 src_reg coordinate;
2341 if (ir->coordinate) {
2342 ir->coordinate->accept(this);
2343 coordinate = this->result;
2344 }
2345
2346 src_reg shadow_comparitor;
2347 if (ir->shadow_comparitor) {
2348 ir->shadow_comparitor->accept(this);
2349 shadow_comparitor = this->result;
2350 }
2351
2352 bool has_nonconstant_offset = ir->offset && !ir->offset->as_constant();
2353 src_reg offset_value;
2354 if (has_nonconstant_offset) {
2355 ir->offset->accept(this);
2356 offset_value = src_reg(this->result);
2357 }
2358
2359 const glsl_type *lod_type = NULL, *sample_index_type = NULL;
2360 src_reg lod, dPdx, dPdy, sample_index, mcs;
2361 switch (ir->op) {
2362 case ir_tex:
2363 lod = src_reg(0.0f);
2364 lod_type = glsl_type::float_type;
2365 break;
2366 case ir_txf:
2367 case ir_txl:
2368 case ir_txs:
2369 ir->lod_info.lod->accept(this);
2370 lod = this->result;
2371 lod_type = ir->lod_info.lod->type;
2372 break;
2373 case ir_query_levels:
2374 lod = src_reg(0);
2375 lod_type = glsl_type::int_type;
2376 break;
2377 case ir_txf_ms:
2378 ir->lod_info.sample_index->accept(this);
2379 sample_index = this->result;
2380 sample_index_type = ir->lod_info.sample_index->type;
2381
2382 if (brw->gen >= 7 && key->tex.compressed_multisample_layout_mask & (1<<sampler))
2383 mcs = emit_mcs_fetch(ir, coordinate, sampler);
2384 else
2385 mcs = src_reg(0u);
2386 break;
2387 case ir_txd:
2388 ir->lod_info.grad.dPdx->accept(this);
2389 dPdx = this->result;
2390
2391 ir->lod_info.grad.dPdy->accept(this);
2392 dPdy = this->result;
2393
2394 lod_type = ir->lod_info.grad.dPdx->type;
2395 break;
2396 case ir_txb:
2397 case ir_lod:
2398 case ir_tg4:
2399 break;
2400 }
2401
2402 enum opcode opcode;
2403 switch (ir->op) {
2404 case ir_tex: opcode = SHADER_OPCODE_TXL; break;
2405 case ir_txl: opcode = SHADER_OPCODE_TXL; break;
2406 case ir_txd: opcode = SHADER_OPCODE_TXD; break;
2407 case ir_txf: opcode = SHADER_OPCODE_TXF; break;
2408 case ir_txf_ms: opcode = SHADER_OPCODE_TXF_CMS; break;
2409 case ir_txs: opcode = SHADER_OPCODE_TXS; break;
2410 case ir_tg4: opcode = has_nonconstant_offset
2411 ? SHADER_OPCODE_TG4_OFFSET : SHADER_OPCODE_TG4; break;
2412 case ir_query_levels: opcode = SHADER_OPCODE_TXS; break;
2413 case ir_txb:
2414 unreachable("TXB is not valid for vertex shaders.");
2415 case ir_lod:
2416 unreachable("LOD is not valid for vertex shaders.");
2417 default:
2418 unreachable("Unrecognized tex op");
2419 }
2420
2421 vec4_instruction *inst = new(mem_ctx) vec4_instruction(this, opcode);
2422
2423 if (ir->offset != NULL && ir->op != ir_txf)
2424 inst->texture_offset = brw_texture_offset(ctx, ir->offset->as_constant());
2425
2426 /* Stuff the channel select bits in the top of the texture offset */
2427 if (ir->op == ir_tg4)
2428 inst->texture_offset |= gather_channel(ir, sampler) << 16;
2429
2430 /* The message header is necessary for:
2431 * - Gen4 (always)
2432 * - Texel offsets
2433 * - Gather channel selection
2434 * - Sampler indices too large to fit in a 4-bit value.
2435 */
2436 inst->header_present =
2437 brw->gen < 5 || inst->texture_offset != 0 || ir->op == ir_tg4 ||
2438 sampler >= 16;
2439 inst->base_mrf = 2;
2440 inst->mlen = inst->header_present + 1; /* always at least one */
2441 inst->dst = dst_reg(this, ir->type);
2442 inst->dst.writemask = WRITEMASK_XYZW;
2443 inst->shadow_compare = ir->shadow_comparitor != NULL;
2444
2445 inst->src[1] = src_reg(sampler);
2446
2447 /* MRF for the first parameter */
2448 int param_base = inst->base_mrf + inst->header_present;
2449
2450 if (ir->op == ir_txs || ir->op == ir_query_levels) {
2451 int writemask = brw->gen == 4 ? WRITEMASK_W : WRITEMASK_X;
2452 emit(MOV(dst_reg(MRF, param_base, lod_type, writemask), lod));
2453 } else {
2454 /* Load the coordinate */
2455 /* FINISHME: gl_clamp_mask and saturate */
2456 int coord_mask = (1 << ir->coordinate->type->vector_elements) - 1;
2457 int zero_mask = 0xf & ~coord_mask;
2458
2459 emit(MOV(dst_reg(MRF, param_base, ir->coordinate->type, coord_mask),
2460 coordinate));
2461
2462 if (zero_mask != 0) {
2463 emit(MOV(dst_reg(MRF, param_base, ir->coordinate->type, zero_mask),
2464 src_reg(0)));
2465 }
2466 /* Load the shadow comparitor */
2467 if (ir->shadow_comparitor && ir->op != ir_txd && (ir->op != ir_tg4 || !has_nonconstant_offset)) {
2468 emit(MOV(dst_reg(MRF, param_base + 1, ir->shadow_comparitor->type,
2469 WRITEMASK_X),
2470 shadow_comparitor));
2471 inst->mlen++;
2472 }
2473
2474 /* Load the LOD info */
2475 if (ir->op == ir_tex || ir->op == ir_txl) {
2476 int mrf, writemask;
2477 if (brw->gen >= 5) {
2478 mrf = param_base + 1;
2479 if (ir->shadow_comparitor) {
2480 writemask = WRITEMASK_Y;
2481 /* mlen already incremented */
2482 } else {
2483 writemask = WRITEMASK_X;
2484 inst->mlen++;
2485 }
2486 } else /* brw->gen == 4 */ {
2487 mrf = param_base;
2488 writemask = WRITEMASK_W;
2489 }
2490 emit(MOV(dst_reg(MRF, mrf, lod_type, writemask), lod));
2491 } else if (ir->op == ir_txf) {
2492 emit(MOV(dst_reg(MRF, param_base, lod_type, WRITEMASK_W), lod));
2493 } else if (ir->op == ir_txf_ms) {
2494 emit(MOV(dst_reg(MRF, param_base + 1, sample_index_type, WRITEMASK_X),
2495 sample_index));
2496 if (brw->gen >= 7)
2497 /* MCS data is in the first channel of `mcs`, but we need to get it into
2498 * the .y channel of the second vec4 of params, so replicate .x across
2499 * the whole vec4 and then mask off everything except .y
2500 */
2501 mcs.swizzle = BRW_SWIZZLE_XXXX;
2502 emit(MOV(dst_reg(MRF, param_base + 1, glsl_type::uint_type, WRITEMASK_Y),
2503 mcs));
2504 inst->mlen++;
2505 } else if (ir->op == ir_txd) {
2506 const glsl_type *type = lod_type;
2507
2508 if (brw->gen >= 5) {
2509 dPdx.swizzle = BRW_SWIZZLE4(SWIZZLE_X,SWIZZLE_X,SWIZZLE_Y,SWIZZLE_Y);
2510 dPdy.swizzle = BRW_SWIZZLE4(SWIZZLE_X,SWIZZLE_X,SWIZZLE_Y,SWIZZLE_Y);
2511 emit(MOV(dst_reg(MRF, param_base + 1, type, WRITEMASK_XZ), dPdx));
2512 emit(MOV(dst_reg(MRF, param_base + 1, type, WRITEMASK_YW), dPdy));
2513 inst->mlen++;
2514
2515 if (ir->type->vector_elements == 3 || ir->shadow_comparitor) {
2516 dPdx.swizzle = BRW_SWIZZLE_ZZZZ;
2517 dPdy.swizzle = BRW_SWIZZLE_ZZZZ;
2518 emit(MOV(dst_reg(MRF, param_base + 2, type, WRITEMASK_X), dPdx));
2519 emit(MOV(dst_reg(MRF, param_base + 2, type, WRITEMASK_Y), dPdy));
2520 inst->mlen++;
2521
2522 if (ir->shadow_comparitor) {
2523 emit(MOV(dst_reg(MRF, param_base + 2,
2524 ir->shadow_comparitor->type, WRITEMASK_Z),
2525 shadow_comparitor));
2526 }
2527 }
2528 } else /* brw->gen == 4 */ {
2529 emit(MOV(dst_reg(MRF, param_base + 1, type, WRITEMASK_XYZ), dPdx));
2530 emit(MOV(dst_reg(MRF, param_base + 2, type, WRITEMASK_XYZ), dPdy));
2531 inst->mlen += 2;
2532 }
2533 } else if (ir->op == ir_tg4 && has_nonconstant_offset) {
2534 if (ir->shadow_comparitor) {
2535 emit(MOV(dst_reg(MRF, param_base, ir->shadow_comparitor->type, WRITEMASK_W),
2536 shadow_comparitor));
2537 }
2538
2539 emit(MOV(dst_reg(MRF, param_base + 1, glsl_type::ivec2_type, WRITEMASK_XY),
2540 offset_value));
2541 inst->mlen++;
2542 }
2543 }
2544
2545 emit(inst);
2546
2547 /* fixup num layers (z) for cube arrays: hardware returns faces * layers;
2548 * spec requires layers.
2549 */
2550 if (ir->op == ir_txs) {
2551 glsl_type const *type = ir->sampler->type;
2552 if (type->sampler_dimensionality == GLSL_SAMPLER_DIM_CUBE &&
2553 type->sampler_array) {
2554 emit_math(SHADER_OPCODE_INT_QUOTIENT,
2555 writemask(inst->dst, WRITEMASK_Z),
2556 src_reg(inst->dst), src_reg(6));
2557 }
2558 }
2559
2560 if (brw->gen == 6 && ir->op == ir_tg4) {
2561 emit_gen6_gather_wa(key->tex.gen6_gather_wa[sampler], inst->dst);
2562 }
2563
2564 swizzle_result(ir, src_reg(inst->dst), sampler);
2565 }
2566
2567 /**
2568 * Apply workarounds for Gen6 gather with UINT/SINT
2569 */
2570 void
2571 vec4_visitor::emit_gen6_gather_wa(uint8_t wa, dst_reg dst)
2572 {
2573 if (!wa)
2574 return;
2575
2576 int width = (wa & WA_8BIT) ? 8 : 16;
2577 dst_reg dst_f = dst;
2578 dst_f.type = BRW_REGISTER_TYPE_F;
2579
2580 /* Convert from UNORM to UINT */
2581 emit(MUL(dst_f, src_reg(dst_f), src_reg((float)((1 << width) - 1))));
2582 emit(MOV(dst, src_reg(dst_f)));
2583
2584 if (wa & WA_SIGN) {
2585 /* Reinterpret the UINT value as a signed INT value by
2586 * shifting the sign bit into place, then shifting back
2587 * preserving sign.
2588 */
2589 emit(SHL(dst, src_reg(dst), src_reg(32 - width)));
2590 emit(ASR(dst, src_reg(dst), src_reg(32 - width)));
2591 }
2592 }
2593
2594 /**
2595 * Set up the gather channel based on the swizzle, for gather4.
2596 */
2597 uint32_t
2598 vec4_visitor::gather_channel(ir_texture *ir, uint32_t sampler)
2599 {
2600 ir_constant *chan = ir->lod_info.component->as_constant();
2601 int swiz = GET_SWZ(key->tex.swizzles[sampler], chan->value.i[0]);
2602 switch (swiz) {
2603 case SWIZZLE_X: return 0;
2604 case SWIZZLE_Y:
2605 /* gather4 sampler is broken for green channel on RG32F --
2606 * we must ask for blue instead.
2607 */
2608 if (key->tex.gather_channel_quirk_mask & (1<<sampler))
2609 return 2;
2610 return 1;
2611 case SWIZZLE_Z: return 2;
2612 case SWIZZLE_W: return 3;
2613 default:
2614 unreachable("Not reached"); /* zero, one swizzles handled already */
2615 }
2616 }
2617
2618 void
2619 vec4_visitor::swizzle_result(ir_texture *ir, src_reg orig_val, uint32_t sampler)
2620 {
2621 int s = key->tex.swizzles[sampler];
2622
2623 this->result = src_reg(this, ir->type);
2624 dst_reg swizzled_result(this->result);
2625
2626 if (ir->op == ir_query_levels) {
2627 /* # levels is in .w */
2628 orig_val.swizzle = BRW_SWIZZLE4(SWIZZLE_W, SWIZZLE_W, SWIZZLE_W, SWIZZLE_W);
2629 emit(MOV(swizzled_result, orig_val));
2630 return;
2631 }
2632
2633 if (ir->op == ir_txs || ir->type == glsl_type::float_type
2634 || s == SWIZZLE_NOOP || ir->op == ir_tg4) {
2635 emit(MOV(swizzled_result, orig_val));
2636 return;
2637 }
2638
2639
2640 int zero_mask = 0, one_mask = 0, copy_mask = 0;
2641 int swizzle[4] = {0};
2642
2643 for (int i = 0; i < 4; i++) {
2644 switch (GET_SWZ(s, i)) {
2645 case SWIZZLE_ZERO:
2646 zero_mask |= (1 << i);
2647 break;
2648 case SWIZZLE_ONE:
2649 one_mask |= (1 << i);
2650 break;
2651 default:
2652 copy_mask |= (1 << i);
2653 swizzle[i] = GET_SWZ(s, i);
2654 break;
2655 }
2656 }
2657
2658 if (copy_mask) {
2659 orig_val.swizzle = BRW_SWIZZLE4(swizzle[0], swizzle[1], swizzle[2], swizzle[3]);
2660 swizzled_result.writemask = copy_mask;
2661 emit(MOV(swizzled_result, orig_val));
2662 }
2663
2664 if (zero_mask) {
2665 swizzled_result.writemask = zero_mask;
2666 emit(MOV(swizzled_result, src_reg(0.0f)));
2667 }
2668
2669 if (one_mask) {
2670 swizzled_result.writemask = one_mask;
2671 emit(MOV(swizzled_result, src_reg(1.0f)));
2672 }
2673 }
2674
2675 void
2676 vec4_visitor::visit(ir_return *)
2677 {
2678 unreachable("not reached");
2679 }
2680
2681 void
2682 vec4_visitor::visit(ir_discard *)
2683 {
2684 unreachable("not reached");
2685 }
2686
2687 void
2688 vec4_visitor::visit(ir_if *ir)
2689 {
2690 /* Don't point the annotation at the if statement, because then it plus
2691 * the then and else blocks get printed.
2692 */
2693 this->base_ir = ir->condition;
2694
2695 if (brw->gen == 6) {
2696 emit_if_gen6(ir);
2697 } else {
2698 enum brw_predicate predicate;
2699 emit_bool_to_cond_code(ir->condition, &predicate);
2700 emit(IF(predicate));
2701 }
2702
2703 visit_instructions(&ir->then_instructions);
2704
2705 if (!ir->else_instructions.is_empty()) {
2706 this->base_ir = ir->condition;
2707 emit(BRW_OPCODE_ELSE);
2708
2709 visit_instructions(&ir->else_instructions);
2710 }
2711
2712 this->base_ir = ir->condition;
2713 emit(BRW_OPCODE_ENDIF);
2714 }
2715
2716 void
2717 vec4_visitor::visit(ir_emit_vertex *)
2718 {
2719 unreachable("not reached");
2720 }
2721
2722 void
2723 vec4_visitor::visit(ir_end_primitive *)
2724 {
2725 unreachable("not reached");
2726 }
2727
2728 void
2729 vec4_visitor::emit_untyped_atomic(unsigned atomic_op, unsigned surf_index,
2730 dst_reg dst, src_reg offset,
2731 src_reg src0, src_reg src1)
2732 {
2733 unsigned mlen = 0;
2734
2735 /* Set the atomic operation offset. */
2736 emit(MOV(brw_writemask(brw_uvec_mrf(8, mlen, 0), WRITEMASK_X), offset));
2737 mlen++;
2738
2739 /* Set the atomic operation arguments. */
2740 if (src0.file != BAD_FILE) {
2741 emit(MOV(brw_writemask(brw_uvec_mrf(8, mlen, 0), WRITEMASK_X), src0));
2742 mlen++;
2743 }
2744
2745 if (src1.file != BAD_FILE) {
2746 emit(MOV(brw_writemask(brw_uvec_mrf(8, mlen, 0), WRITEMASK_X), src1));
2747 mlen++;
2748 }
2749
2750 /* Emit the instruction. Note that this maps to the normal SIMD8
2751 * untyped atomic message on Ivy Bridge, but that's OK because
2752 * unused channels will be masked out.
2753 */
2754 vec4_instruction *inst = emit(SHADER_OPCODE_UNTYPED_ATOMIC, dst,
2755 src_reg(atomic_op), src_reg(surf_index));
2756 inst->base_mrf = 0;
2757 inst->mlen = mlen;
2758 }
2759
2760 void
2761 vec4_visitor::emit_untyped_surface_read(unsigned surf_index, dst_reg dst,
2762 src_reg offset)
2763 {
2764 /* Set the surface read offset. */
2765 emit(MOV(brw_writemask(brw_uvec_mrf(8, 0, 0), WRITEMASK_X), offset));
2766
2767 /* Emit the instruction. Note that this maps to the normal SIMD8
2768 * untyped surface read message, but that's OK because unused
2769 * channels will be masked out.
2770 */
2771 vec4_instruction *inst = emit(SHADER_OPCODE_UNTYPED_SURFACE_READ,
2772 dst, src_reg(surf_index));
2773 inst->base_mrf = 0;
2774 inst->mlen = 1;
2775 }
2776
2777 void
2778 vec4_visitor::emit_ndc_computation()
2779 {
2780 /* Get the position */
2781 src_reg pos = src_reg(output_reg[VARYING_SLOT_POS]);
2782
2783 /* Build ndc coords, which are (x/w, y/w, z/w, 1/w) */
2784 dst_reg ndc = dst_reg(this, glsl_type::vec4_type);
2785 output_reg[BRW_VARYING_SLOT_NDC] = ndc;
2786
2787 current_annotation = "NDC";
2788 dst_reg ndc_w = ndc;
2789 ndc_w.writemask = WRITEMASK_W;
2790 src_reg pos_w = pos;
2791 pos_w.swizzle = BRW_SWIZZLE4(SWIZZLE_W, SWIZZLE_W, SWIZZLE_W, SWIZZLE_W);
2792 emit_math(SHADER_OPCODE_RCP, ndc_w, pos_w);
2793
2794 dst_reg ndc_xyz = ndc;
2795 ndc_xyz.writemask = WRITEMASK_XYZ;
2796
2797 emit(MUL(ndc_xyz, pos, src_reg(ndc_w)));
2798 }
2799
2800 void
2801 vec4_visitor::emit_psiz_and_flags(struct brw_reg reg)
2802 {
2803 if (brw->gen < 6 &&
2804 ((prog_data->vue_map.slots_valid & VARYING_BIT_PSIZ) ||
2805 key->userclip_active || brw->has_negative_rhw_bug)) {
2806 dst_reg header1 = dst_reg(this, glsl_type::uvec4_type);
2807 dst_reg header1_w = header1;
2808 header1_w.writemask = WRITEMASK_W;
2809
2810 emit(MOV(header1, 0u));
2811
2812 if (prog_data->vue_map.slots_valid & VARYING_BIT_PSIZ) {
2813 src_reg psiz = src_reg(output_reg[VARYING_SLOT_PSIZ]);
2814
2815 current_annotation = "Point size";
2816 emit(MUL(header1_w, psiz, src_reg((float)(1 << 11))));
2817 emit(AND(header1_w, src_reg(header1_w), 0x7ff << 8));
2818 }
2819
2820 if (key->userclip_active) {
2821 current_annotation = "Clipping flags";
2822 dst_reg flags0 = dst_reg(this, glsl_type::uint_type);
2823 dst_reg flags1 = dst_reg(this, glsl_type::uint_type);
2824
2825 emit(CMP(dst_null_f(), src_reg(output_reg[VARYING_SLOT_CLIP_DIST0]), src_reg(0.0f), BRW_CONDITIONAL_L));
2826 emit(VS_OPCODE_UNPACK_FLAGS_SIMD4X2, flags0, src_reg(0));
2827 emit(OR(header1_w, src_reg(header1_w), src_reg(flags0)));
2828
2829 emit(CMP(dst_null_f(), src_reg(output_reg[VARYING_SLOT_CLIP_DIST1]), src_reg(0.0f), BRW_CONDITIONAL_L));
2830 emit(VS_OPCODE_UNPACK_FLAGS_SIMD4X2, flags1, src_reg(0));
2831 emit(SHL(flags1, src_reg(flags1), src_reg(4)));
2832 emit(OR(header1_w, src_reg(header1_w), src_reg(flags1)));
2833 }
2834
2835 /* i965 clipping workaround:
2836 * 1) Test for -ve rhw
2837 * 2) If set,
2838 * set ndc = (0,0,0,0)
2839 * set ucp[6] = 1
2840 *
2841 * Later, clipping will detect ucp[6] and ensure the primitive is
2842 * clipped against all fixed planes.
2843 */
2844 if (brw->has_negative_rhw_bug) {
2845 src_reg ndc_w = src_reg(output_reg[BRW_VARYING_SLOT_NDC]);
2846 ndc_w.swizzle = BRW_SWIZZLE_WWWW;
2847 emit(CMP(dst_null_f(), ndc_w, src_reg(0.0f), BRW_CONDITIONAL_L));
2848 vec4_instruction *inst;
2849 inst = emit(OR(header1_w, src_reg(header1_w), src_reg(1u << 6)));
2850 inst->predicate = BRW_PREDICATE_NORMAL;
2851 inst = emit(MOV(output_reg[BRW_VARYING_SLOT_NDC], src_reg(0.0f)));
2852 inst->predicate = BRW_PREDICATE_NORMAL;
2853 }
2854
2855 emit(MOV(retype(reg, BRW_REGISTER_TYPE_UD), src_reg(header1)));
2856 } else if (brw->gen < 6) {
2857 emit(MOV(retype(reg, BRW_REGISTER_TYPE_UD), 0u));
2858 } else {
2859 emit(MOV(retype(reg, BRW_REGISTER_TYPE_D), src_reg(0)));
2860 if (prog_data->vue_map.slots_valid & VARYING_BIT_PSIZ) {
2861 emit(MOV(brw_writemask(reg, WRITEMASK_W),
2862 src_reg(output_reg[VARYING_SLOT_PSIZ])));
2863 }
2864 if (prog_data->vue_map.slots_valid & VARYING_BIT_LAYER) {
2865 emit(MOV(retype(brw_writemask(reg, WRITEMASK_Y), BRW_REGISTER_TYPE_D),
2866 src_reg(output_reg[VARYING_SLOT_LAYER])));
2867 }
2868 if (prog_data->vue_map.slots_valid & VARYING_BIT_VIEWPORT) {
2869 emit(MOV(retype(brw_writemask(reg, WRITEMASK_Z), BRW_REGISTER_TYPE_D),
2870 src_reg(output_reg[VARYING_SLOT_VIEWPORT])));
2871 }
2872 }
2873 }
2874
2875 void
2876 vec4_visitor::emit_clip_distances(dst_reg reg, int offset)
2877 {
2878 /* From the GLSL 1.30 spec, section 7.1 (Vertex Shader Special Variables):
2879 *
2880 * "If a linked set of shaders forming the vertex stage contains no
2881 * static write to gl_ClipVertex or gl_ClipDistance, but the
2882 * application has requested clipping against user clip planes through
2883 * the API, then the coordinate written to gl_Position is used for
2884 * comparison against the user clip planes."
2885 *
2886 * This function is only called if the shader didn't write to
2887 * gl_ClipDistance. Accordingly, we use gl_ClipVertex to perform clipping
2888 * if the user wrote to it; otherwise we use gl_Position.
2889 */
2890 gl_varying_slot clip_vertex = VARYING_SLOT_CLIP_VERTEX;
2891 if (!(prog_data->vue_map.slots_valid & VARYING_BIT_CLIP_VERTEX)) {
2892 clip_vertex = VARYING_SLOT_POS;
2893 }
2894
2895 for (int i = 0; i + offset < key->nr_userclip_plane_consts && i < 4;
2896 ++i) {
2897 reg.writemask = 1 << i;
2898 emit(DP4(reg,
2899 src_reg(output_reg[clip_vertex]),
2900 src_reg(this->userplane[i + offset])));
2901 }
2902 }
2903
2904 void
2905 vec4_visitor::emit_generic_urb_slot(dst_reg reg, int varying)
2906 {
2907 assert (varying < VARYING_SLOT_MAX);
2908 reg.type = output_reg[varying].type;
2909 current_annotation = output_reg_annotation[varying];
2910 /* Copy the register, saturating if necessary */
2911 vec4_instruction *inst = emit(MOV(reg,
2912 src_reg(output_reg[varying])));
2913 if ((varying == VARYING_SLOT_COL0 ||
2914 varying == VARYING_SLOT_COL1 ||
2915 varying == VARYING_SLOT_BFC0 ||
2916 varying == VARYING_SLOT_BFC1) &&
2917 key->clamp_vertex_color) {
2918 inst->saturate = true;
2919 }
2920 }
2921
2922 void
2923 vec4_visitor::emit_urb_slot(int mrf, int varying)
2924 {
2925 struct brw_reg hw_reg = brw_message_reg(mrf);
2926 dst_reg reg = dst_reg(MRF, mrf);
2927 reg.type = BRW_REGISTER_TYPE_F;
2928
2929 switch (varying) {
2930 case VARYING_SLOT_PSIZ:
2931 /* PSIZ is always in slot 0, and is coupled with other flags. */
2932 current_annotation = "indices, point width, clip flags";
2933 emit_psiz_and_flags(hw_reg);
2934 break;
2935 case BRW_VARYING_SLOT_NDC:
2936 current_annotation = "NDC";
2937 emit(MOV(reg, src_reg(output_reg[BRW_VARYING_SLOT_NDC])));
2938 break;
2939 case VARYING_SLOT_POS:
2940 current_annotation = "gl_Position";
2941 emit(MOV(reg, src_reg(output_reg[VARYING_SLOT_POS])));
2942 break;
2943 case VARYING_SLOT_EDGE:
2944 /* This is present when doing unfilled polygons. We're supposed to copy
2945 * the edge flag from the user-provided vertex array
2946 * (glEdgeFlagPointer), or otherwise we'll copy from the current value
2947 * of that attribute (starts as 1.0f). This is then used in clipping to
2948 * determine which edges should be drawn as wireframe.
2949 */
2950 current_annotation = "edge flag";
2951 emit(MOV(reg, src_reg(dst_reg(ATTR, VERT_ATTRIB_EDGEFLAG,
2952 glsl_type::float_type, WRITEMASK_XYZW))));
2953 break;
2954 case BRW_VARYING_SLOT_PAD:
2955 /* No need to write to this slot */
2956 break;
2957 default:
2958 emit_generic_urb_slot(reg, varying);
2959 break;
2960 }
2961 }
2962
2963 static int
2964 align_interleaved_urb_mlen(struct brw_context *brw, int mlen)
2965 {
2966 if (brw->gen >= 6) {
2967 /* URB data written (does not include the message header reg) must
2968 * be a multiple of 256 bits, or 2 VS registers. See vol5c.5,
2969 * section 5.4.3.2.2: URB_INTERLEAVED.
2970 *
2971 * URB entries are allocated on a multiple of 1024 bits, so an
2972 * extra 128 bits written here to make the end align to 256 is
2973 * no problem.
2974 */
2975 if ((mlen % 2) != 1)
2976 mlen++;
2977 }
2978
2979 return mlen;
2980 }
2981
2982
2983 /**
2984 * Generates the VUE payload plus the necessary URB write instructions to
2985 * output it.
2986 *
2987 * The VUE layout is documented in Volume 2a.
2988 */
2989 void
2990 vec4_visitor::emit_vertex()
2991 {
2992 /* MRF 0 is reserved for the debugger, so start with message header
2993 * in MRF 1.
2994 */
2995 int base_mrf = 1;
2996 int mrf = base_mrf;
2997 /* In the process of generating our URB write message contents, we
2998 * may need to unspill a register or load from an array. Those
2999 * reads would use MRFs 14-15.
3000 */
3001 int max_usable_mrf = 13;
3002
3003 /* The following assertion verifies that max_usable_mrf causes an
3004 * even-numbered amount of URB write data, which will meet gen6's
3005 * requirements for length alignment.
3006 */
3007 assert ((max_usable_mrf - base_mrf) % 2 == 0);
3008
3009 /* First mrf is the g0-based message header containing URB handles and
3010 * such.
3011 */
3012 emit_urb_write_header(mrf++);
3013
3014 if (brw->gen < 6) {
3015 emit_ndc_computation();
3016 }
3017
3018 /* Lower legacy ff and ClipVertex clipping to clip distances */
3019 if (key->userclip_active && !prog->UsesClipDistanceOut) {
3020 current_annotation = "user clip distances";
3021
3022 output_reg[VARYING_SLOT_CLIP_DIST0] = dst_reg(this, glsl_type::vec4_type);
3023 output_reg[VARYING_SLOT_CLIP_DIST1] = dst_reg(this, glsl_type::vec4_type);
3024
3025 emit_clip_distances(output_reg[VARYING_SLOT_CLIP_DIST0], 0);
3026 emit_clip_distances(output_reg[VARYING_SLOT_CLIP_DIST1], 4);
3027 }
3028
3029 /* We may need to split this up into several URB writes, so do them in a
3030 * loop.
3031 */
3032 int slot = 0;
3033 bool complete = false;
3034 do {
3035 /* URB offset is in URB row increments, and each of our MRFs is half of
3036 * one of those, since we're doing interleaved writes.
3037 */
3038 int offset = slot / 2;
3039
3040 mrf = base_mrf + 1;
3041 for (; slot < prog_data->vue_map.num_slots; ++slot) {
3042 emit_urb_slot(mrf++, prog_data->vue_map.slot_to_varying[slot]);
3043
3044 /* If this was max_usable_mrf, we can't fit anything more into this
3045 * URB WRITE.
3046 */
3047 if (mrf > max_usable_mrf) {
3048 slot++;
3049 break;
3050 }
3051 }
3052
3053 complete = slot >= prog_data->vue_map.num_slots;
3054 current_annotation = "URB write";
3055 vec4_instruction *inst = emit_urb_write_opcode(complete);
3056 inst->base_mrf = base_mrf;
3057 inst->mlen = align_interleaved_urb_mlen(brw, mrf - base_mrf);
3058 inst->offset += offset;
3059 } while(!complete);
3060 }
3061
3062
3063 src_reg
3064 vec4_visitor::get_scratch_offset(vec4_instruction *inst,
3065 src_reg *reladdr, int reg_offset)
3066 {
3067 /* Because we store the values to scratch interleaved like our
3068 * vertex data, we need to scale the vec4 index by 2.
3069 */
3070 int message_header_scale = 2;
3071
3072 /* Pre-gen6, the message header uses byte offsets instead of vec4
3073 * (16-byte) offset units.
3074 */
3075 if (brw->gen < 6)
3076 message_header_scale *= 16;
3077
3078 if (reladdr) {
3079 src_reg index = src_reg(this, glsl_type::int_type);
3080
3081 emit_before(inst, ADD(dst_reg(index), *reladdr, src_reg(reg_offset)));
3082 emit_before(inst, MUL(dst_reg(index),
3083 index, src_reg(message_header_scale)));
3084
3085 return index;
3086 } else {
3087 return src_reg(reg_offset * message_header_scale);
3088 }
3089 }
3090
3091 src_reg
3092 vec4_visitor::get_pull_constant_offset(vec4_instruction *inst,
3093 src_reg *reladdr, int reg_offset)
3094 {
3095 if (reladdr) {
3096 src_reg index = src_reg(this, glsl_type::int_type);
3097
3098 emit_before(inst, ADD(dst_reg(index), *reladdr, src_reg(reg_offset)));
3099
3100 /* Pre-gen6, the message header uses byte offsets instead of vec4
3101 * (16-byte) offset units.
3102 */
3103 if (brw->gen < 6) {
3104 emit_before(inst, MUL(dst_reg(index), index, src_reg(16)));
3105 }
3106
3107 return index;
3108 } else if (brw->gen >= 8) {
3109 /* Store the offset in a GRF so we can send-from-GRF. */
3110 src_reg offset = src_reg(this, glsl_type::int_type);
3111 emit_before(inst, MOV(dst_reg(offset), src_reg(reg_offset)));
3112 return offset;
3113 } else {
3114 int message_header_scale = brw->gen < 6 ? 16 : 1;
3115 return src_reg(reg_offset * message_header_scale);
3116 }
3117 }
3118
3119 /**
3120 * Emits an instruction before @inst to load the value named by @orig_src
3121 * from scratch space at @base_offset to @temp.
3122 *
3123 * @base_offset is measured in 32-byte units (the size of a register).
3124 */
3125 void
3126 vec4_visitor::emit_scratch_read(vec4_instruction *inst,
3127 dst_reg temp, src_reg orig_src,
3128 int base_offset)
3129 {
3130 int reg_offset = base_offset + orig_src.reg_offset;
3131 src_reg index = get_scratch_offset(inst, orig_src.reladdr, reg_offset);
3132
3133 emit_before(inst, SCRATCH_READ(temp, index));
3134 }
3135
3136 /**
3137 * Emits an instruction after @inst to store the value to be written
3138 * to @orig_dst to scratch space at @base_offset, from @temp.
3139 *
3140 * @base_offset is measured in 32-byte units (the size of a register).
3141 */
3142 void
3143 vec4_visitor::emit_scratch_write(vec4_instruction *inst, int base_offset)
3144 {
3145 int reg_offset = base_offset + inst->dst.reg_offset;
3146 src_reg index = get_scratch_offset(inst, inst->dst.reladdr, reg_offset);
3147
3148 /* Create a temporary register to store *inst's result in.
3149 *
3150 * We have to be careful in MOVing from our temporary result register in
3151 * the scratch write. If we swizzle from channels of the temporary that
3152 * weren't initialized, it will confuse live interval analysis, which will
3153 * make spilling fail to make progress.
3154 */
3155 src_reg temp = src_reg(this, glsl_type::vec4_type);
3156 temp.type = inst->dst.type;
3157 int first_writemask_chan = ffs(inst->dst.writemask) - 1;
3158 int swizzles[4];
3159 for (int i = 0; i < 4; i++)
3160 if (inst->dst.writemask & (1 << i))
3161 swizzles[i] = i;
3162 else
3163 swizzles[i] = first_writemask_chan;
3164 temp.swizzle = BRW_SWIZZLE4(swizzles[0], swizzles[1],
3165 swizzles[2], swizzles[3]);
3166
3167 dst_reg dst = dst_reg(brw_writemask(brw_vec8_grf(0, 0),
3168 inst->dst.writemask));
3169 vec4_instruction *write = SCRATCH_WRITE(dst, temp, index);
3170 write->predicate = inst->predicate;
3171 write->ir = inst->ir;
3172 write->annotation = inst->annotation;
3173 inst->insert_after(write);
3174
3175 inst->dst.file = temp.file;
3176 inst->dst.reg = temp.reg;
3177 inst->dst.reg_offset = temp.reg_offset;
3178 inst->dst.reladdr = NULL;
3179 }
3180
3181 /**
3182 * We can't generally support array access in GRF space, because a
3183 * single instruction's destination can only span 2 contiguous
3184 * registers. So, we send all GRF arrays that get variable index
3185 * access to scratch space.
3186 */
3187 void
3188 vec4_visitor::move_grf_array_access_to_scratch()
3189 {
3190 int scratch_loc[this->virtual_grf_count];
3191
3192 for (int i = 0; i < this->virtual_grf_count; i++) {
3193 scratch_loc[i] = -1;
3194 }
3195
3196 /* First, calculate the set of virtual GRFs that need to be punted
3197 * to scratch due to having any array access on them, and where in
3198 * scratch.
3199 */
3200 foreach_in_list(vec4_instruction, inst, &instructions) {
3201 if (inst->dst.file == GRF && inst->dst.reladdr &&
3202 scratch_loc[inst->dst.reg] == -1) {
3203 scratch_loc[inst->dst.reg] = c->last_scratch;
3204 c->last_scratch += this->virtual_grf_sizes[inst->dst.reg];
3205 }
3206
3207 for (int i = 0 ; i < 3; i++) {
3208 src_reg *src = &inst->src[i];
3209
3210 if (src->file == GRF && src->reladdr &&
3211 scratch_loc[src->reg] == -1) {
3212 scratch_loc[src->reg] = c->last_scratch;
3213 c->last_scratch += this->virtual_grf_sizes[src->reg];
3214 }
3215 }
3216 }
3217
3218 /* Now, for anything that will be accessed through scratch, rewrite
3219 * it to load/store. Note that this is a _safe list walk, because
3220 * we may generate a new scratch_write instruction after the one
3221 * we're processing.
3222 */
3223 foreach_in_list_safe(vec4_instruction, inst, &instructions) {
3224 /* Set up the annotation tracking for new generated instructions. */
3225 base_ir = inst->ir;
3226 current_annotation = inst->annotation;
3227
3228 if (inst->dst.file == GRF && scratch_loc[inst->dst.reg] != -1) {
3229 emit_scratch_write(inst, scratch_loc[inst->dst.reg]);
3230 }
3231
3232 for (int i = 0 ; i < 3; i++) {
3233 if (inst->src[i].file != GRF || scratch_loc[inst->src[i].reg] == -1)
3234 continue;
3235
3236 dst_reg temp = dst_reg(this, glsl_type::vec4_type);
3237
3238 emit_scratch_read(inst, temp, inst->src[i],
3239 scratch_loc[inst->src[i].reg]);
3240
3241 inst->src[i].file = temp.file;
3242 inst->src[i].reg = temp.reg;
3243 inst->src[i].reg_offset = temp.reg_offset;
3244 inst->src[i].reladdr = NULL;
3245 }
3246 }
3247 }
3248
3249 /**
3250 * Emits an instruction before @inst to load the value named by @orig_src
3251 * from the pull constant buffer (surface) at @base_offset to @temp.
3252 */
3253 void
3254 vec4_visitor::emit_pull_constant_load(vec4_instruction *inst,
3255 dst_reg temp, src_reg orig_src,
3256 int base_offset)
3257 {
3258 int reg_offset = base_offset + orig_src.reg_offset;
3259 src_reg index = src_reg(prog_data->base.binding_table.pull_constants_start);
3260 src_reg offset = get_pull_constant_offset(inst, orig_src.reladdr, reg_offset);
3261 vec4_instruction *load;
3262
3263 if (brw->gen >= 7) {
3264 dst_reg grf_offset = dst_reg(this, glsl_type::int_type);
3265 grf_offset.type = offset.type;
3266 emit_before(inst, MOV(grf_offset, offset));
3267
3268 load = new(mem_ctx) vec4_instruction(this,
3269 VS_OPCODE_PULL_CONSTANT_LOAD_GEN7,
3270 temp, index, src_reg(grf_offset));
3271 } else {
3272 load = new(mem_ctx) vec4_instruction(this, VS_OPCODE_PULL_CONSTANT_LOAD,
3273 temp, index, offset);
3274 load->base_mrf = 14;
3275 load->mlen = 1;
3276 }
3277 emit_before(inst, load);
3278 }
3279
3280 /**
3281 * Implements array access of uniforms by inserting a
3282 * PULL_CONSTANT_LOAD instruction.
3283 *
3284 * Unlike temporary GRF array access (where we don't support it due to
3285 * the difficulty of doing relative addressing on instruction
3286 * destinations), we could potentially do array access of uniforms
3287 * that were loaded in GRF space as push constants. In real-world
3288 * usage we've seen, though, the arrays being used are always larger
3289 * than we could load as push constants, so just always move all
3290 * uniform array access out to a pull constant buffer.
3291 */
3292 void
3293 vec4_visitor::move_uniform_array_access_to_pull_constants()
3294 {
3295 int pull_constant_loc[this->uniforms];
3296
3297 for (int i = 0; i < this->uniforms; i++) {
3298 pull_constant_loc[i] = -1;
3299 }
3300
3301 /* Walk through and find array access of uniforms. Put a copy of that
3302 * uniform in the pull constant buffer.
3303 *
3304 * Note that we don't move constant-indexed accesses to arrays. No
3305 * testing has been done of the performance impact of this choice.
3306 */
3307 foreach_in_list_safe(vec4_instruction, inst, &instructions) {
3308 for (int i = 0 ; i < 3; i++) {
3309 if (inst->src[i].file != UNIFORM || !inst->src[i].reladdr)
3310 continue;
3311
3312 int uniform = inst->src[i].reg;
3313
3314 /* If this array isn't already present in the pull constant buffer,
3315 * add it.
3316 */
3317 if (pull_constant_loc[uniform] == -1) {
3318 const gl_constant_value **values =
3319 &stage_prog_data->param[uniform * 4];
3320
3321 pull_constant_loc[uniform] = stage_prog_data->nr_pull_params / 4;
3322
3323 assert(uniform < uniform_array_size);
3324 for (int j = 0; j < uniform_size[uniform] * 4; j++) {
3325 stage_prog_data->pull_param[stage_prog_data->nr_pull_params++]
3326 = values[j];
3327 }
3328 }
3329
3330 /* Set up the annotation tracking for new generated instructions. */
3331 base_ir = inst->ir;
3332 current_annotation = inst->annotation;
3333
3334 dst_reg temp = dst_reg(this, glsl_type::vec4_type);
3335
3336 emit_pull_constant_load(inst, temp, inst->src[i],
3337 pull_constant_loc[uniform]);
3338
3339 inst->src[i].file = temp.file;
3340 inst->src[i].reg = temp.reg;
3341 inst->src[i].reg_offset = temp.reg_offset;
3342 inst->src[i].reladdr = NULL;
3343 }
3344 }
3345
3346 /* Now there are no accesses of the UNIFORM file with a reladdr, so
3347 * no need to track them as larger-than-vec4 objects. This will be
3348 * relied on in cutting out unused uniform vectors from push
3349 * constants.
3350 */
3351 split_uniform_registers();
3352 }
3353
3354 void
3355 vec4_visitor::resolve_ud_negate(src_reg *reg)
3356 {
3357 if (reg->type != BRW_REGISTER_TYPE_UD ||
3358 !reg->negate)
3359 return;
3360
3361 src_reg temp = src_reg(this, glsl_type::uvec4_type);
3362 emit(BRW_OPCODE_MOV, dst_reg(temp), *reg);
3363 *reg = temp;
3364 }
3365
3366 vec4_visitor::vec4_visitor(struct brw_context *brw,
3367 struct brw_vec4_compile *c,
3368 struct gl_program *prog,
3369 const struct brw_vec4_prog_key *key,
3370 struct brw_vec4_prog_data *prog_data,
3371 struct gl_shader_program *shader_prog,
3372 gl_shader_stage stage,
3373 void *mem_ctx,
3374 bool debug_flag,
3375 bool no_spills,
3376 shader_time_shader_type st_base,
3377 shader_time_shader_type st_written,
3378 shader_time_shader_type st_reset)
3379 : backend_visitor(brw, shader_prog, prog, &prog_data->base, stage),
3380 c(c),
3381 key(key),
3382 prog_data(prog_data),
3383 sanity_param_count(0),
3384 fail_msg(NULL),
3385 first_non_payload_grf(0),
3386 need_all_constants_in_pull_buffer(false),
3387 debug_flag(debug_flag),
3388 no_spills(no_spills),
3389 st_base(st_base),
3390 st_written(st_written),
3391 st_reset(st_reset)
3392 {
3393 this->mem_ctx = mem_ctx;
3394 this->failed = false;
3395
3396 this->base_ir = NULL;
3397 this->current_annotation = NULL;
3398 memset(this->output_reg_annotation, 0, sizeof(this->output_reg_annotation));
3399
3400 this->variable_ht = hash_table_ctor(0,
3401 hash_table_pointer_hash,
3402 hash_table_pointer_compare);
3403
3404 this->virtual_grf_start = NULL;
3405 this->virtual_grf_end = NULL;
3406 this->virtual_grf_sizes = NULL;
3407 this->virtual_grf_count = 0;
3408 this->virtual_grf_reg_map = NULL;
3409 this->virtual_grf_reg_count = 0;
3410 this->virtual_grf_array_size = 0;
3411 this->live_intervals_valid = false;
3412
3413 this->max_grf = brw->gen >= 7 ? GEN7_MRF_HACK_START : BRW_MAX_GRF;
3414
3415 this->uniforms = 0;
3416
3417 /* Initialize uniform_array_size to at least 1 because pre-gen6 VS requires
3418 * at least one. See setup_uniforms() in brw_vec4.cpp.
3419 */
3420 this->uniform_array_size = 1;
3421 if (prog_data) {
3422 this->uniform_array_size = MAX2(stage_prog_data->nr_params, 1);
3423 }
3424
3425 this->uniform_size = rzalloc_array(mem_ctx, int, this->uniform_array_size);
3426 this->uniform_vector_size = rzalloc_array(mem_ctx, int, this->uniform_array_size);
3427 }
3428
3429 vec4_visitor::~vec4_visitor()
3430 {
3431 hash_table_dtor(this->variable_ht);
3432 }
3433
3434
3435 void
3436 vec4_visitor::fail(const char *format, ...)
3437 {
3438 va_list va;
3439 char *msg;
3440
3441 if (failed)
3442 return;
3443
3444 failed = true;
3445
3446 va_start(va, format);
3447 msg = ralloc_vasprintf(mem_ctx, format, va);
3448 va_end(va);
3449 msg = ralloc_asprintf(mem_ctx, "vec4 compile failed: %s\n", msg);
3450
3451 this->fail_msg = msg;
3452
3453 if (debug_flag) {
3454 fprintf(stderr, "%s", msg);
3455 }
3456 }
3457
3458 } /* namespace brw */