glsl: pass mem_ctx to constant_expression_value(...) and friends
[mesa.git] / src / mesa / state_tracker / st_glsl_to_tgsi.cpp
1 /*
2 * Copyright (C) 2005-2007 Brian Paul All Rights Reserved.
3 * Copyright (C) 2008 VMware, Inc. All Rights Reserved.
4 * Copyright © 2010 Intel Corporation
5 * Copyright © 2011 Bryan Cain
6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the "Software"),
9 * to deal in the Software without restriction, including without limitation
10 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
11 * and/or sell copies of the Software, and to permit persons to whom the
12 * Software is furnished to do so, subject to the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the next
15 * paragraph) shall be included in all copies or substantial portions of the
16 * Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
21 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
23 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
24 * DEALINGS IN THE SOFTWARE.
25 */
26
27 /**
28 * \file glsl_to_tgsi.cpp
29 *
30 * Translate GLSL IR to TGSI.
31 */
32
33 #include "st_glsl_to_tgsi.h"
34
35 #include "compiler/glsl/glsl_parser_extras.h"
36 #include "compiler/glsl/ir_optimization.h"
37 #include "compiler/glsl/program.h"
38
39 #include "main/errors.h"
40 #include "main/shaderobj.h"
41 #include "main/uniforms.h"
42 #include "main/shaderapi.h"
43 #include "main/shaderimage.h"
44 #include "program/prog_instruction.h"
45
46 #include "pipe/p_context.h"
47 #include "pipe/p_screen.h"
48 #include "tgsi/tgsi_ureg.h"
49 #include "tgsi/tgsi_info.h"
50 #include "util/u_math.h"
51 #include "util/u_memory.h"
52 #include "st_program.h"
53 #include "st_mesa_to_tgsi.h"
54 #include "st_format.h"
55 #include "st_nir.h"
56 #include "st_shader_cache.h"
57
58 #include "util/hash_table.h"
59 #include <algorithm>
60
61 #define PROGRAM_ANY_CONST ((1 << PROGRAM_STATE_VAR) | \
62 (1 << PROGRAM_CONSTANT) | \
63 (1 << PROGRAM_UNIFORM))
64
65 #define MAX_GLSL_TEXTURE_OFFSET 4
66
67 class st_src_reg;
68 class st_dst_reg;
69
70 static int swizzle_for_size(int size);
71
72 static int swizzle_for_type(const glsl_type *type, int component = 0)
73 {
74 unsigned num_elements = 4;
75
76 if (type) {
77 type = type->without_array();
78 if (type->is_scalar() || type->is_vector() || type->is_matrix())
79 num_elements = type->vector_elements;
80 }
81
82 int swizzle = swizzle_for_size(num_elements);
83 assert(num_elements + component <= 4);
84
85 swizzle += component * MAKE_SWIZZLE4(1, 1, 1, 1);
86 return swizzle;
87 }
88
89 static unsigned is_precise(const ir_variable *ir)
90 {
91 if (!ir)
92 return 0;
93 return ir->data.precise || ir->data.invariant;
94 }
95
96 /**
97 * This struct is a corresponding struct to TGSI ureg_src.
98 */
99 class st_src_reg {
100 public:
101 st_src_reg(gl_register_file file, int index, const glsl_type *type,
102 int component = 0, unsigned array_id = 0)
103 {
104 assert(file != PROGRAM_ARRAY || array_id != 0);
105 this->file = file;
106 this->index = index;
107 this->swizzle = swizzle_for_type(type, component);
108 this->negate = 0;
109 this->abs = 0;
110 this->index2D = 0;
111 this->type = type ? type->base_type : GLSL_TYPE_ERROR;
112 this->reladdr = NULL;
113 this->reladdr2 = NULL;
114 this->has_index2 = false;
115 this->double_reg2 = false;
116 this->array_id = array_id;
117 this->is_double_vertex_input = false;
118 }
119
120 st_src_reg(gl_register_file file, int index, enum glsl_base_type type)
121 {
122 assert(file != PROGRAM_ARRAY); /* need array_id > 0 */
123 this->type = type;
124 this->file = file;
125 this->index = index;
126 this->index2D = 0;
127 this->swizzle = SWIZZLE_XYZW;
128 this->negate = 0;
129 this->abs = 0;
130 this->reladdr = NULL;
131 this->reladdr2 = NULL;
132 this->has_index2 = false;
133 this->double_reg2 = false;
134 this->array_id = 0;
135 this->is_double_vertex_input = false;
136 }
137
138 st_src_reg(gl_register_file file, int index, enum glsl_base_type type, int index2D)
139 {
140 assert(file != PROGRAM_ARRAY); /* need array_id > 0 */
141 this->type = type;
142 this->file = file;
143 this->index = index;
144 this->index2D = index2D;
145 this->swizzle = SWIZZLE_XYZW;
146 this->negate = 0;
147 this->abs = 0;
148 this->reladdr = NULL;
149 this->reladdr2 = NULL;
150 this->has_index2 = false;
151 this->double_reg2 = false;
152 this->array_id = 0;
153 this->is_double_vertex_input = false;
154 }
155
156 st_src_reg()
157 {
158 this->type = GLSL_TYPE_ERROR;
159 this->file = PROGRAM_UNDEFINED;
160 this->index = 0;
161 this->index2D = 0;
162 this->swizzle = 0;
163 this->negate = 0;
164 this->abs = 0;
165 this->reladdr = NULL;
166 this->reladdr2 = NULL;
167 this->has_index2 = false;
168 this->double_reg2 = false;
169 this->array_id = 0;
170 this->is_double_vertex_input = false;
171 }
172
173 explicit st_src_reg(st_dst_reg reg);
174
175 int32_t index; /**< temporary index, VERT_ATTRIB_*, VARYING_SLOT_*, etc. */
176 int16_t index2D;
177 uint16_t swizzle; /**< SWIZZLE_XYZWONEZERO swizzles from Mesa. */
178 int negate:4; /**< NEGATE_XYZW mask from mesa */
179 unsigned abs:1;
180 enum glsl_base_type type:5; /** GLSL_TYPE_* from GLSL IR (enum glsl_base_type) */
181 unsigned has_index2:1;
182 gl_register_file file:5; /**< PROGRAM_* from Mesa */
183 /*
184 * Is this the second half of a double register pair?
185 * currently used for input mapping only.
186 */
187 unsigned double_reg2:1;
188 unsigned is_double_vertex_input:1;
189 unsigned array_id:10;
190
191 /** Register index should be offset by the integer in this reg. */
192 st_src_reg *reladdr;
193 st_src_reg *reladdr2;
194
195 st_src_reg get_abs()
196 {
197 st_src_reg reg = *this;
198 reg.negate = 0;
199 reg.abs = 1;
200 return reg;
201 }
202 };
203
204 class st_dst_reg {
205 public:
206 st_dst_reg(gl_register_file file, int writemask, enum glsl_base_type type, int index)
207 {
208 assert(file != PROGRAM_ARRAY); /* need array_id > 0 */
209 this->file = file;
210 this->index = index;
211 this->index2D = 0;
212 this->writemask = writemask;
213 this->reladdr = NULL;
214 this->reladdr2 = NULL;
215 this->has_index2 = false;
216 this->type = type;
217 this->array_id = 0;
218 }
219
220 st_dst_reg(gl_register_file file, int writemask, enum glsl_base_type type)
221 {
222 assert(file != PROGRAM_ARRAY); /* need array_id > 0 */
223 this->file = file;
224 this->index = 0;
225 this->index2D = 0;
226 this->writemask = writemask;
227 this->reladdr = NULL;
228 this->reladdr2 = NULL;
229 this->has_index2 = false;
230 this->type = type;
231 this->array_id = 0;
232 }
233
234 st_dst_reg()
235 {
236 this->type = GLSL_TYPE_ERROR;
237 this->file = PROGRAM_UNDEFINED;
238 this->index = 0;
239 this->index2D = 0;
240 this->writemask = 0;
241 this->reladdr = NULL;
242 this->reladdr2 = NULL;
243 this->has_index2 = false;
244 this->array_id = 0;
245 }
246
247 explicit st_dst_reg(st_src_reg reg);
248
249 int32_t index; /**< temporary index, VERT_ATTRIB_*, VARYING_SLOT_*, etc. */
250 int16_t index2D;
251 gl_register_file file:5; /**< PROGRAM_* from Mesa */
252 unsigned writemask:4; /**< Bitfield of WRITEMASK_[XYZW] */
253 enum glsl_base_type type:5; /** GLSL_TYPE_* from GLSL IR (enum glsl_base_type) */
254 unsigned has_index2:1;
255 unsigned array_id:10;
256
257 /** Register index should be offset by the integer in this reg. */
258 st_src_reg *reladdr;
259 st_src_reg *reladdr2;
260 };
261
262 st_src_reg::st_src_reg(st_dst_reg reg)
263 {
264 this->type = reg.type;
265 this->file = reg.file;
266 this->index = reg.index;
267 this->swizzle = SWIZZLE_XYZW;
268 this->negate = 0;
269 this->abs = 0;
270 this->reladdr = reg.reladdr;
271 this->index2D = reg.index2D;
272 this->reladdr2 = reg.reladdr2;
273 this->has_index2 = reg.has_index2;
274 this->double_reg2 = false;
275 this->array_id = reg.array_id;
276 this->is_double_vertex_input = false;
277 }
278
279 st_dst_reg::st_dst_reg(st_src_reg reg)
280 {
281 this->type = reg.type;
282 this->file = reg.file;
283 this->index = reg.index;
284 this->writemask = WRITEMASK_XYZW;
285 this->reladdr = reg.reladdr;
286 this->index2D = reg.index2D;
287 this->reladdr2 = reg.reladdr2;
288 this->has_index2 = reg.has_index2;
289 this->array_id = reg.array_id;
290 }
291
292 class glsl_to_tgsi_instruction : public exec_node {
293 public:
294 DECLARE_RALLOC_CXX_OPERATORS(glsl_to_tgsi_instruction)
295
296 st_dst_reg dst[2];
297 st_src_reg src[4];
298 st_src_reg resource; /**< sampler, image or buffer register */
299 st_src_reg *tex_offsets;
300
301 /** Pointer to the ir source this tree came from for debugging */
302 ir_instruction *ir;
303
304 unsigned op:8; /**< TGSI opcode */
305 unsigned precise:1;
306 unsigned saturate:1;
307 unsigned is_64bit_expanded:1;
308 unsigned sampler_base:5;
309 unsigned sampler_array_size:6; /**< 1-based size of sampler array, 1 if not array */
310 unsigned tex_target:4; /**< One of TEXTURE_*_INDEX */
311 glsl_base_type tex_type:5;
312 unsigned tex_shadow:1;
313 unsigned image_format:9;
314 unsigned tex_offset_num_offset:3;
315 unsigned dead_mask:4; /**< Used in dead code elimination */
316 unsigned buffer_access:3; /**< buffer access type */
317
318 const struct tgsi_opcode_info *info;
319 };
320
321 class variable_storage {
322 DECLARE_RZALLOC_CXX_OPERATORS(variable_storage)
323
324 public:
325 variable_storage(ir_variable *var, gl_register_file file, int index,
326 unsigned array_id = 0)
327 : file(file), index(index), component(0), var(var), array_id(array_id)
328 {
329 assert(file != PROGRAM_ARRAY || array_id != 0);
330 }
331
332 gl_register_file file;
333 int index;
334
335 /* Explicit component location. This is given in terms of the GLSL-style
336 * swizzles where each double is a single component, i.e. for 64-bit types
337 * it can only be 0 or 1.
338 */
339 int component;
340 ir_variable *var; /* variable that maps to this, if any */
341 unsigned array_id;
342 };
343
344 class immediate_storage : public exec_node {
345 public:
346 immediate_storage(gl_constant_value *values, int size32, int type)
347 {
348 memcpy(this->values, values, size32 * sizeof(gl_constant_value));
349 this->size32 = size32;
350 this->type = type;
351 }
352
353 /* doubles are stored across 2 gl_constant_values */
354 gl_constant_value values[4];
355 int size32; /**< Number of 32-bit components (1-4) */
356 int type; /**< GL_DOUBLE, GL_FLOAT, GL_INT, GL_BOOL, or GL_UNSIGNED_INT */
357 };
358
359 static const st_src_reg undef_src = st_src_reg(PROGRAM_UNDEFINED, 0, GLSL_TYPE_ERROR);
360 static const st_dst_reg undef_dst = st_dst_reg(PROGRAM_UNDEFINED, SWIZZLE_NOOP, GLSL_TYPE_ERROR);
361
362 struct inout_decl {
363 unsigned mesa_index;
364 unsigned array_id; /* TGSI ArrayID; 1-based: 0 means not an array */
365 unsigned size;
366 unsigned interp_loc;
367 unsigned gs_out_streams;
368 enum glsl_interp_mode interp;
369 enum glsl_base_type base_type;
370 ubyte usage_mask; /* GLSL-style usage-mask, i.e. single bit per double */
371 };
372
373 static struct inout_decl *
374 find_inout_array(struct inout_decl *decls, unsigned count, unsigned array_id)
375 {
376 assert(array_id != 0);
377
378 for (unsigned i = 0; i < count; i++) {
379 struct inout_decl *decl = &decls[i];
380
381 if (array_id == decl->array_id) {
382 return decl;
383 }
384 }
385
386 return NULL;
387 }
388
389 static enum glsl_base_type
390 find_array_type(struct inout_decl *decls, unsigned count, unsigned array_id)
391 {
392 if (!array_id)
393 return GLSL_TYPE_ERROR;
394 struct inout_decl *decl = find_inout_array(decls, count, array_id);
395 if (decl)
396 return decl->base_type;
397 return GLSL_TYPE_ERROR;
398 }
399
400 struct rename_reg_pair {
401 int old_reg;
402 int new_reg;
403 };
404
405 struct glsl_to_tgsi_visitor : public ir_visitor {
406 public:
407 glsl_to_tgsi_visitor();
408 ~glsl_to_tgsi_visitor();
409
410 struct gl_context *ctx;
411 struct gl_program *prog;
412 struct gl_shader_program *shader_program;
413 struct gl_linked_shader *shader;
414 struct gl_shader_compiler_options *options;
415
416 int next_temp;
417
418 unsigned *array_sizes;
419 unsigned max_num_arrays;
420 unsigned next_array;
421
422 struct inout_decl inputs[4 * PIPE_MAX_SHADER_INPUTS];
423 unsigned num_inputs;
424 unsigned num_input_arrays;
425 struct inout_decl outputs[4 * PIPE_MAX_SHADER_OUTPUTS];
426 unsigned num_outputs;
427 unsigned num_output_arrays;
428
429 int num_address_regs;
430 uint32_t samplers_used;
431 glsl_base_type sampler_types[PIPE_MAX_SAMPLERS];
432 int sampler_targets[PIPE_MAX_SAMPLERS]; /**< One of TGSI_TEXTURE_* */
433 int images_used;
434 int image_targets[PIPE_MAX_SHADER_IMAGES];
435 unsigned image_formats[PIPE_MAX_SHADER_IMAGES];
436 bool indirect_addr_consts;
437 int wpos_transform_const;
438
439 int glsl_version;
440 bool native_integers;
441 bool have_sqrt;
442 bool have_fma;
443 bool use_shared_memory;
444 bool has_tex_txf_lz;
445 bool precise;
446
447 variable_storage *find_variable_storage(ir_variable *var);
448
449 int add_constant(gl_register_file file, gl_constant_value values[8],
450 int size, int datatype, uint16_t *swizzle_out);
451
452 st_src_reg get_temp(const glsl_type *type);
453 void reladdr_to_temp(ir_instruction *ir, st_src_reg *reg, int *num_reladdr);
454
455 st_src_reg st_src_reg_for_double(double val);
456 st_src_reg st_src_reg_for_float(float val);
457 st_src_reg st_src_reg_for_int(int val);
458 st_src_reg st_src_reg_for_type(enum glsl_base_type type, int val);
459
460 /**
461 * \name Visit methods
462 *
463 * As typical for the visitor pattern, there must be one \c visit method for
464 * each concrete subclass of \c ir_instruction. Virtual base classes within
465 * the hierarchy should not have \c visit methods.
466 */
467 /*@{*/
468 virtual void visit(ir_variable *);
469 virtual void visit(ir_loop *);
470 virtual void visit(ir_loop_jump *);
471 virtual void visit(ir_function_signature *);
472 virtual void visit(ir_function *);
473 virtual void visit(ir_expression *);
474 virtual void visit(ir_swizzle *);
475 virtual void visit(ir_dereference_variable *);
476 virtual void visit(ir_dereference_array *);
477 virtual void visit(ir_dereference_record *);
478 virtual void visit(ir_assignment *);
479 virtual void visit(ir_constant *);
480 virtual void visit(ir_call *);
481 virtual void visit(ir_return *);
482 virtual void visit(ir_discard *);
483 virtual void visit(ir_texture *);
484 virtual void visit(ir_if *);
485 virtual void visit(ir_emit_vertex *);
486 virtual void visit(ir_end_primitive *);
487 virtual void visit(ir_barrier *);
488 /*@}*/
489
490 void visit_expression(ir_expression *, st_src_reg *) ATTRIBUTE_NOINLINE;
491
492 void visit_atomic_counter_intrinsic(ir_call *);
493 void visit_ssbo_intrinsic(ir_call *);
494 void visit_membar_intrinsic(ir_call *);
495 void visit_shared_intrinsic(ir_call *);
496 void visit_image_intrinsic(ir_call *);
497 void visit_generic_intrinsic(ir_call *, unsigned op);
498
499 st_src_reg result;
500
501 /** List of variable_storage */
502 struct hash_table *variables;
503
504 /** List of immediate_storage */
505 exec_list immediates;
506 unsigned num_immediates;
507
508 /** List of glsl_to_tgsi_instruction */
509 exec_list instructions;
510
511 glsl_to_tgsi_instruction *emit_asm(ir_instruction *ir, unsigned op,
512 st_dst_reg dst = undef_dst,
513 st_src_reg src0 = undef_src,
514 st_src_reg src1 = undef_src,
515 st_src_reg src2 = undef_src,
516 st_src_reg src3 = undef_src);
517
518 glsl_to_tgsi_instruction *emit_asm(ir_instruction *ir, unsigned op,
519 st_dst_reg dst, st_dst_reg dst1,
520 st_src_reg src0 = undef_src,
521 st_src_reg src1 = undef_src,
522 st_src_reg src2 = undef_src,
523 st_src_reg src3 = undef_src);
524
525 unsigned get_opcode(unsigned op,
526 st_dst_reg dst,
527 st_src_reg src0, st_src_reg src1);
528
529 /**
530 * Emit the correct dot-product instruction for the type of arguments
531 */
532 glsl_to_tgsi_instruction *emit_dp(ir_instruction *ir,
533 st_dst_reg dst,
534 st_src_reg src0,
535 st_src_reg src1,
536 unsigned elements);
537
538 void emit_scalar(ir_instruction *ir, unsigned op,
539 st_dst_reg dst, st_src_reg src0);
540
541 void emit_scalar(ir_instruction *ir, unsigned op,
542 st_dst_reg dst, st_src_reg src0, st_src_reg src1);
543
544 void emit_arl(ir_instruction *ir, st_dst_reg dst, st_src_reg src0);
545
546 void get_deref_offsets(ir_dereference *ir,
547 unsigned *array_size,
548 unsigned *base,
549 uint16_t *index,
550 st_src_reg *reladdr,
551 bool opaque);
552 void calc_deref_offsets(ir_dereference *tail,
553 unsigned *array_elements,
554 uint16_t *index,
555 st_src_reg *indirect,
556 unsigned *location);
557 st_src_reg canonicalize_gather_offset(st_src_reg offset);
558
559 bool try_emit_mad(ir_expression *ir,
560 int mul_operand);
561 bool try_emit_mad_for_and_not(ir_expression *ir,
562 int mul_operand);
563
564 void emit_swz(ir_expression *ir);
565
566 bool process_move_condition(ir_rvalue *ir);
567
568 void simplify_cmp(void);
569
570 void rename_temp_registers(int num_renames, struct rename_reg_pair *renames);
571 void get_first_temp_read(int *first_reads);
572 void get_first_temp_write(int *first_writes);
573 void get_last_temp_read_first_temp_write(int *last_reads, int *first_writes);
574 void get_last_temp_write(int *last_writes);
575
576 void copy_propagate(void);
577 int eliminate_dead_code(void);
578
579 void merge_two_dsts(void);
580 void merge_registers(void);
581 void renumber_registers(void);
582
583 void emit_block_mov(ir_assignment *ir, const struct glsl_type *type,
584 st_dst_reg *l, st_src_reg *r,
585 st_src_reg *cond, bool cond_swap);
586
587 void *mem_ctx;
588 };
589
590 static st_dst_reg address_reg = st_dst_reg(PROGRAM_ADDRESS, WRITEMASK_X, GLSL_TYPE_FLOAT, 0);
591 static st_dst_reg address_reg2 = st_dst_reg(PROGRAM_ADDRESS, WRITEMASK_X, GLSL_TYPE_FLOAT, 1);
592 static st_dst_reg sampler_reladdr = st_dst_reg(PROGRAM_ADDRESS, WRITEMASK_X, GLSL_TYPE_FLOAT, 2);
593
594 static void
595 fail_link(struct gl_shader_program *prog, const char *fmt, ...) PRINTFLIKE(2, 3);
596
597 static void
598 fail_link(struct gl_shader_program *prog, const char *fmt, ...)
599 {
600 va_list args;
601 va_start(args, fmt);
602 ralloc_vasprintf_append(&prog->data->InfoLog, fmt, args);
603 va_end(args);
604
605 prog->data->LinkStatus = linking_failure;
606 }
607
608 static int
609 swizzle_for_size(int size)
610 {
611 static const int size_swizzles[4] = {
612 MAKE_SWIZZLE4(SWIZZLE_X, SWIZZLE_X, SWIZZLE_X, SWIZZLE_X),
613 MAKE_SWIZZLE4(SWIZZLE_X, SWIZZLE_Y, SWIZZLE_Y, SWIZZLE_Y),
614 MAKE_SWIZZLE4(SWIZZLE_X, SWIZZLE_Y, SWIZZLE_Z, SWIZZLE_Z),
615 MAKE_SWIZZLE4(SWIZZLE_X, SWIZZLE_Y, SWIZZLE_Z, SWIZZLE_W),
616 };
617
618 assert((size >= 1) && (size <= 4));
619 return size_swizzles[size - 1];
620 }
621
622 static bool
623 is_resource_instruction(unsigned opcode)
624 {
625 switch (opcode) {
626 case TGSI_OPCODE_RESQ:
627 case TGSI_OPCODE_LOAD:
628 case TGSI_OPCODE_ATOMUADD:
629 case TGSI_OPCODE_ATOMXCHG:
630 case TGSI_OPCODE_ATOMCAS:
631 case TGSI_OPCODE_ATOMAND:
632 case TGSI_OPCODE_ATOMOR:
633 case TGSI_OPCODE_ATOMXOR:
634 case TGSI_OPCODE_ATOMUMIN:
635 case TGSI_OPCODE_ATOMUMAX:
636 case TGSI_OPCODE_ATOMIMIN:
637 case TGSI_OPCODE_ATOMIMAX:
638 return true;
639 default:
640 return false;
641 }
642 }
643
644 static unsigned
645 num_inst_dst_regs(const glsl_to_tgsi_instruction *op)
646 {
647 return op->info->num_dst;
648 }
649
650 static unsigned
651 num_inst_src_regs(const glsl_to_tgsi_instruction *op)
652 {
653 return op->info->is_tex || is_resource_instruction(op->op) ?
654 op->info->num_src - 1 : op->info->num_src;
655 }
656
657 glsl_to_tgsi_instruction *
658 glsl_to_tgsi_visitor::emit_asm(ir_instruction *ir, unsigned op,
659 st_dst_reg dst, st_dst_reg dst1,
660 st_src_reg src0, st_src_reg src1,
661 st_src_reg src2, st_src_reg src3)
662 {
663 glsl_to_tgsi_instruction *inst = new(mem_ctx) glsl_to_tgsi_instruction();
664 int num_reladdr = 0, i, j;
665 bool dst_is_64bit[2];
666
667 op = get_opcode(op, dst, src0, src1);
668
669 /* If we have to do relative addressing, we want to load the ARL
670 * reg directly for one of the regs, and preload the other reladdr
671 * sources into temps.
672 */
673 num_reladdr += dst.reladdr != NULL || dst.reladdr2;
674 num_reladdr += dst1.reladdr != NULL || dst1.reladdr2;
675 num_reladdr += src0.reladdr != NULL || src0.reladdr2 != NULL;
676 num_reladdr += src1.reladdr != NULL || src1.reladdr2 != NULL;
677 num_reladdr += src2.reladdr != NULL || src2.reladdr2 != NULL;
678 num_reladdr += src3.reladdr != NULL || src3.reladdr2 != NULL;
679
680 reladdr_to_temp(ir, &src3, &num_reladdr);
681 reladdr_to_temp(ir, &src2, &num_reladdr);
682 reladdr_to_temp(ir, &src1, &num_reladdr);
683 reladdr_to_temp(ir, &src0, &num_reladdr);
684
685 if (dst.reladdr || dst.reladdr2) {
686 if (dst.reladdr)
687 emit_arl(ir, address_reg, *dst.reladdr);
688 if (dst.reladdr2)
689 emit_arl(ir, address_reg2, *dst.reladdr2);
690 num_reladdr--;
691 }
692 if (dst1.reladdr) {
693 emit_arl(ir, address_reg, *dst1.reladdr);
694 num_reladdr--;
695 }
696 assert(num_reladdr == 0);
697
698 /* inst->op has only 8 bits. */
699 STATIC_ASSERT(TGSI_OPCODE_LAST <= 255);
700
701 inst->op = op;
702 inst->precise = this->precise;
703 inst->info = tgsi_get_opcode_info(op);
704 inst->dst[0] = dst;
705 inst->dst[1] = dst1;
706 inst->src[0] = src0;
707 inst->src[1] = src1;
708 inst->src[2] = src2;
709 inst->src[3] = src3;
710 inst->is_64bit_expanded = false;
711 inst->ir = ir;
712 inst->dead_mask = 0;
713 inst->tex_offsets = NULL;
714 inst->tex_offset_num_offset = 0;
715 inst->saturate = 0;
716 inst->tex_shadow = 0;
717 /* default to float, for paths where this is not initialized
718 * (since 0==UINT which is likely wrong):
719 */
720 inst->tex_type = GLSL_TYPE_FLOAT;
721
722 /* Update indirect addressing status used by TGSI */
723 if (dst.reladdr || dst.reladdr2) {
724 switch(dst.file) {
725 case PROGRAM_STATE_VAR:
726 case PROGRAM_CONSTANT:
727 case PROGRAM_UNIFORM:
728 this->indirect_addr_consts = true;
729 break;
730 case PROGRAM_IMMEDIATE:
731 assert(!"immediates should not have indirect addressing");
732 break;
733 default:
734 break;
735 }
736 }
737 else {
738 for (i = 0; i < 4; i++) {
739 if(inst->src[i].reladdr) {
740 switch(inst->src[i].file) {
741 case PROGRAM_STATE_VAR:
742 case PROGRAM_CONSTANT:
743 case PROGRAM_UNIFORM:
744 this->indirect_addr_consts = true;
745 break;
746 case PROGRAM_IMMEDIATE:
747 assert(!"immediates should not have indirect addressing");
748 break;
749 default:
750 break;
751 }
752 }
753 }
754 }
755
756 /*
757 * This section contains the double processing.
758 * GLSL just represents doubles as single channel values,
759 * however most HW and TGSI represent doubles as pairs of register channels.
760 *
761 * so we have to fixup destination writemask/index and src swizzle/indexes.
762 * dest writemasks need to translate from single channel write mask
763 * to a dual-channel writemask, but also need to modify the index,
764 * if we are touching the Z,W fields in the pre-translated writemask.
765 *
766 * src channels have similiar index modifications along with swizzle
767 * changes to we pick the XY, ZW pairs from the correct index.
768 *
769 * GLSL [0].x -> TGSI [0].xy
770 * GLSL [0].y -> TGSI [0].zw
771 * GLSL [0].z -> TGSI [1].xy
772 * GLSL [0].w -> TGSI [1].zw
773 */
774 for (j = 0; j < 2; j++) {
775 dst_is_64bit[j] = glsl_base_type_is_64bit(inst->dst[j].type);
776 if (!dst_is_64bit[j] && inst->dst[j].file == PROGRAM_OUTPUT && inst->dst[j].type == GLSL_TYPE_ARRAY) {
777 enum glsl_base_type type = find_array_type(this->outputs, this->num_outputs, inst->dst[j].array_id);
778 if (glsl_base_type_is_64bit(type))
779 dst_is_64bit[j] = true;
780 }
781 }
782
783 if (dst_is_64bit[0] || dst_is_64bit[1] ||
784 glsl_base_type_is_64bit(inst->src[0].type)) {
785 glsl_to_tgsi_instruction *dinst = NULL;
786 int initial_src_swz[4], initial_src_idx[4];
787 int initial_dst_idx[2], initial_dst_writemask[2];
788 /* select the writemask for dst0 or dst1 */
789 unsigned writemask = inst->dst[1].file == PROGRAM_UNDEFINED ? inst->dst[0].writemask : inst->dst[1].writemask;
790
791 /* copy out the writemask, index and swizzles for all src/dsts. */
792 for (j = 0; j < 2; j++) {
793 initial_dst_writemask[j] = inst->dst[j].writemask;
794 initial_dst_idx[j] = inst->dst[j].index;
795 }
796
797 for (j = 0; j < 4; j++) {
798 initial_src_swz[j] = inst->src[j].swizzle;
799 initial_src_idx[j] = inst->src[j].index;
800 }
801
802 /*
803 * scan all the components in the dst writemask
804 * generate an instruction for each of them if required.
805 */
806 st_src_reg addr;
807 while (writemask) {
808
809 int i = u_bit_scan(&writemask);
810
811 /* before emitting the instruction, see if we have to adjust load / store
812 * address */
813 if (i > 1 && (inst->op == TGSI_OPCODE_LOAD || inst->op == TGSI_OPCODE_STORE) &&
814 addr.file == PROGRAM_UNDEFINED) {
815 /* We have to advance the buffer address by 16 */
816 addr = get_temp(glsl_type::uint_type);
817 emit_asm(ir, TGSI_OPCODE_UADD, st_dst_reg(addr),
818 inst->src[0], st_src_reg_for_int(16));
819 }
820
821 /* first time use previous instruction */
822 if (dinst == NULL) {
823 dinst = inst;
824 } else {
825 /* create a new instructions for subsequent attempts */
826 dinst = new(mem_ctx) glsl_to_tgsi_instruction();
827 *dinst = *inst;
828 dinst->next = NULL;
829 dinst->prev = NULL;
830 }
831 this->instructions.push_tail(dinst);
832 dinst->is_64bit_expanded = true;
833
834 /* modify the destination if we are splitting */
835 for (j = 0; j < 2; j++) {
836 if (dst_is_64bit[j]) {
837 dinst->dst[j].writemask = (i & 1) ? WRITEMASK_ZW : WRITEMASK_XY;
838 dinst->dst[j].index = initial_dst_idx[j];
839 if (i > 1) {
840 if (dinst->op == TGSI_OPCODE_LOAD || dinst->op == TGSI_OPCODE_STORE)
841 dinst->src[0] = addr;
842 if (dinst->op != TGSI_OPCODE_STORE)
843 dinst->dst[j].index++;
844 }
845 } else {
846 /* if we aren't writing to a double, just get the bit of the initial writemask
847 for this channel */
848 dinst->dst[j].writemask = initial_dst_writemask[j] & (1 << i);
849 }
850 }
851
852 /* modify the src registers */
853 for (j = 0; j < 4; j++) {
854 int swz = GET_SWZ(initial_src_swz[j], i);
855
856 if (glsl_base_type_is_64bit(dinst->src[j].type)) {
857 dinst->src[j].index = initial_src_idx[j];
858 if (swz > 1) {
859 dinst->src[j].double_reg2 = true;
860 dinst->src[j].index++;
861 }
862
863 if (swz & 1)
864 dinst->src[j].swizzle = MAKE_SWIZZLE4(SWIZZLE_Z, SWIZZLE_W, SWIZZLE_Z, SWIZZLE_W);
865 else
866 dinst->src[j].swizzle = MAKE_SWIZZLE4(SWIZZLE_X, SWIZZLE_Y, SWIZZLE_X, SWIZZLE_Y);
867
868 } else {
869 /* some opcodes are special case in what they use as sources
870 - [FUI]2D/[UI]2I64 is a float/[u]int src0, DLDEXP is integer src1 */
871 if (op == TGSI_OPCODE_F2D || op == TGSI_OPCODE_U2D || op == TGSI_OPCODE_I2D ||
872 op == TGSI_OPCODE_I2I64 || op == TGSI_OPCODE_U2I64 ||
873 op == TGSI_OPCODE_DLDEXP ||
874 (op == TGSI_OPCODE_UCMP && dst_is_64bit[0])) {
875 dinst->src[j].swizzle = MAKE_SWIZZLE4(swz, swz, swz, swz);
876 }
877 }
878 }
879 }
880 inst = dinst;
881 } else {
882 this->instructions.push_tail(inst);
883 }
884
885
886 return inst;
887 }
888
889 glsl_to_tgsi_instruction *
890 glsl_to_tgsi_visitor::emit_asm(ir_instruction *ir, unsigned op,
891 st_dst_reg dst,
892 st_src_reg src0, st_src_reg src1,
893 st_src_reg src2, st_src_reg src3)
894 {
895 return emit_asm(ir, op, dst, undef_dst, src0, src1, src2, src3);
896 }
897
898 /**
899 * Determines whether to use an integer, unsigned integer, or float opcode
900 * based on the operands and input opcode, then emits the result.
901 */
902 unsigned
903 glsl_to_tgsi_visitor::get_opcode(unsigned op,
904 st_dst_reg dst,
905 st_src_reg src0, st_src_reg src1)
906 {
907 enum glsl_base_type type = GLSL_TYPE_FLOAT;
908
909 if (op == TGSI_OPCODE_MOV)
910 return op;
911
912 assert(src0.type != GLSL_TYPE_ARRAY);
913 assert(src0.type != GLSL_TYPE_STRUCT);
914 assert(src1.type != GLSL_TYPE_ARRAY);
915 assert(src1.type != GLSL_TYPE_STRUCT);
916
917 if (is_resource_instruction(op))
918 type = src1.type;
919 else if (src0.type == GLSL_TYPE_INT64 || src1.type == GLSL_TYPE_INT64)
920 type = GLSL_TYPE_INT64;
921 else if (src0.type == GLSL_TYPE_UINT64 || src1.type == GLSL_TYPE_UINT64)
922 type = GLSL_TYPE_UINT64;
923 else if (src0.type == GLSL_TYPE_DOUBLE || src1.type == GLSL_TYPE_DOUBLE)
924 type = GLSL_TYPE_DOUBLE;
925 else if (src0.type == GLSL_TYPE_FLOAT || src1.type == GLSL_TYPE_FLOAT)
926 type = GLSL_TYPE_FLOAT;
927 else if (native_integers)
928 type = src0.type == GLSL_TYPE_BOOL ? GLSL_TYPE_INT : src0.type;
929
930 #define case7(c, f, i, u, d, i64, ui64) \
931 case TGSI_OPCODE_##c: \
932 if (type == GLSL_TYPE_UINT64) \
933 op = TGSI_OPCODE_##ui64; \
934 else if (type == GLSL_TYPE_INT64) \
935 op = TGSI_OPCODE_##i64; \
936 else if (type == GLSL_TYPE_DOUBLE) \
937 op = TGSI_OPCODE_##d; \
938 else if (type == GLSL_TYPE_INT) \
939 op = TGSI_OPCODE_##i; \
940 else if (type == GLSL_TYPE_UINT) \
941 op = TGSI_OPCODE_##u; \
942 else \
943 op = TGSI_OPCODE_##f; \
944 break;
945 #define case5(c, f, i, u, d) \
946 case TGSI_OPCODE_##c: \
947 if (type == GLSL_TYPE_DOUBLE) \
948 op = TGSI_OPCODE_##d; \
949 else if (type == GLSL_TYPE_INT) \
950 op = TGSI_OPCODE_##i; \
951 else if (type == GLSL_TYPE_UINT) \
952 op = TGSI_OPCODE_##u; \
953 else \
954 op = TGSI_OPCODE_##f; \
955 break;
956
957 #define case4(c, f, i, u) \
958 case TGSI_OPCODE_##c: \
959 if (type == GLSL_TYPE_INT) \
960 op = TGSI_OPCODE_##i; \
961 else if (type == GLSL_TYPE_UINT) \
962 op = TGSI_OPCODE_##u; \
963 else \
964 op = TGSI_OPCODE_##f; \
965 break;
966
967 #define case3(f, i, u) case4(f, f, i, u)
968 #define case6d(f, i, u, d, i64, u64) case7(f, f, i, u, d, i64, u64)
969 #define case3fid(f, i, d) case5(f, f, i, i, d)
970 #define case3fid64(f, i, d, i64) case7(f, f, i, i, d, i64, i64)
971 #define case2fi(f, i) case4(f, f, i, i)
972 #define case2iu(i, u) case4(i, LAST, i, u)
973
974 #define case2iu64(i, i64) case7(i, LAST, i, i, LAST, i64, i64)
975 #define case4iu64(i, u, i64, u64) case7(i, LAST, i, u, LAST, i64, u64)
976
977 #define casecomp(c, f, i, u, d, i64, ui64) \
978 case TGSI_OPCODE_##c: \
979 if (type == GLSL_TYPE_INT64) \
980 op = TGSI_OPCODE_##i64; \
981 else if (type == GLSL_TYPE_UINT64) \
982 op = TGSI_OPCODE_##ui64; \
983 else if (type == GLSL_TYPE_DOUBLE) \
984 op = TGSI_OPCODE_##d; \
985 else if (type == GLSL_TYPE_INT || type == GLSL_TYPE_SUBROUTINE) \
986 op = TGSI_OPCODE_##i; \
987 else if (type == GLSL_TYPE_UINT) \
988 op = TGSI_OPCODE_##u; \
989 else if (native_integers) \
990 op = TGSI_OPCODE_##f; \
991 else \
992 op = TGSI_OPCODE_##c; \
993 break;
994
995 switch(op) {
996 case3fid64(ADD, UADD, DADD, U64ADD);
997 case3fid64(MUL, UMUL, DMUL, U64MUL);
998 case3fid(MAD, UMAD, DMAD);
999 case3fid(FMA, UMAD, DFMA);
1000 case6d(DIV, IDIV, UDIV, DDIV, I64DIV, U64DIV);
1001 case6d(MAX, IMAX, UMAX, DMAX, I64MAX, U64MAX);
1002 case6d(MIN, IMIN, UMIN, DMIN, I64MIN, U64MIN);
1003 case4iu64(MOD, UMOD, I64MOD, U64MOD);
1004
1005 casecomp(SEQ, FSEQ, USEQ, USEQ, DSEQ, U64SEQ, U64SEQ);
1006 casecomp(SNE, FSNE, USNE, USNE, DSNE, U64SNE, U64SNE);
1007 casecomp(SGE, FSGE, ISGE, USGE, DSGE, I64SGE, U64SGE);
1008 casecomp(SLT, FSLT, ISLT, USLT, DSLT, I64SLT, U64SLT);
1009
1010 case2iu64(SHL, U64SHL);
1011 case4iu64(ISHR, USHR, I64SHR, U64SHR);
1012
1013 case3fid64(SSG, ISSG, DSSG, I64SSG);
1014
1015 case2iu(IBFE, UBFE);
1016 case2iu(IMSB, UMSB);
1017 case2iu(IMUL_HI, UMUL_HI);
1018
1019 case3fid(SQRT, SQRT, DSQRT);
1020
1021 case3fid(RCP, RCP, DRCP);
1022 case3fid(RSQ, RSQ, DRSQ);
1023
1024 case3fid(FRC, FRC, DFRAC);
1025 case3fid(TRUNC, TRUNC, DTRUNC);
1026 case3fid(CEIL, CEIL, DCEIL);
1027 case3fid(FLR, FLR, DFLR);
1028 case3fid(ROUND, ROUND, DROUND);
1029
1030 case2iu(ATOMIMAX, ATOMUMAX);
1031 case2iu(ATOMIMIN, ATOMUMIN);
1032
1033 default: break;
1034 }
1035
1036 assert(op != TGSI_OPCODE_LAST);
1037 return op;
1038 }
1039
1040 glsl_to_tgsi_instruction *
1041 glsl_to_tgsi_visitor::emit_dp(ir_instruction *ir,
1042 st_dst_reg dst, st_src_reg src0, st_src_reg src1,
1043 unsigned elements)
1044 {
1045 static const unsigned dot_opcodes[] = {
1046 TGSI_OPCODE_DP2, TGSI_OPCODE_DP3, TGSI_OPCODE_DP4
1047 };
1048
1049 return emit_asm(ir, dot_opcodes[elements - 2], dst, src0, src1);
1050 }
1051
1052 /**
1053 * Emits TGSI scalar opcodes to produce unique answers across channels.
1054 *
1055 * Some TGSI opcodes are scalar-only, like ARB_fp/vp. The src X
1056 * channel determines the result across all channels. So to do a vec4
1057 * of this operation, we want to emit a scalar per source channel used
1058 * to produce dest channels.
1059 */
1060 void
1061 glsl_to_tgsi_visitor::emit_scalar(ir_instruction *ir, unsigned op,
1062 st_dst_reg dst,
1063 st_src_reg orig_src0, st_src_reg orig_src1)
1064 {
1065 int i, j;
1066 int done_mask = ~dst.writemask;
1067
1068 /* TGSI RCP is a scalar operation splatting results to all channels,
1069 * like ARB_fp/vp. So emit as many RCPs as necessary to cover our
1070 * dst channels.
1071 */
1072 for (i = 0; i < 4; i++) {
1073 GLuint this_mask = (1 << i);
1074 st_src_reg src0 = orig_src0;
1075 st_src_reg src1 = orig_src1;
1076
1077 if (done_mask & this_mask)
1078 continue;
1079
1080 GLuint src0_swiz = GET_SWZ(src0.swizzle, i);
1081 GLuint src1_swiz = GET_SWZ(src1.swizzle, i);
1082 for (j = i + 1; j < 4; j++) {
1083 /* If there is another enabled component in the destination that is
1084 * derived from the same inputs, generate its value on this pass as
1085 * well.
1086 */
1087 if (!(done_mask & (1 << j)) &&
1088 GET_SWZ(src0.swizzle, j) == src0_swiz &&
1089 GET_SWZ(src1.swizzle, j) == src1_swiz) {
1090 this_mask |= (1 << j);
1091 }
1092 }
1093 src0.swizzle = MAKE_SWIZZLE4(src0_swiz, src0_swiz,
1094 src0_swiz, src0_swiz);
1095 src1.swizzle = MAKE_SWIZZLE4(src1_swiz, src1_swiz,
1096 src1_swiz, src1_swiz);
1097
1098 dst.writemask = this_mask;
1099 emit_asm(ir, op, dst, src0, src1);
1100 done_mask |= this_mask;
1101 }
1102 }
1103
1104 void
1105 glsl_to_tgsi_visitor::emit_scalar(ir_instruction *ir, unsigned op,
1106 st_dst_reg dst, st_src_reg src0)
1107 {
1108 st_src_reg undef = undef_src;
1109
1110 undef.swizzle = SWIZZLE_XXXX;
1111
1112 emit_scalar(ir, op, dst, src0, undef);
1113 }
1114
1115 void
1116 glsl_to_tgsi_visitor::emit_arl(ir_instruction *ir,
1117 st_dst_reg dst, st_src_reg src0)
1118 {
1119 int op = TGSI_OPCODE_ARL;
1120
1121 if (src0.type == GLSL_TYPE_INT || src0.type == GLSL_TYPE_UINT)
1122 op = TGSI_OPCODE_UARL;
1123
1124 assert(dst.file == PROGRAM_ADDRESS);
1125 if (dst.index >= this->num_address_regs)
1126 this->num_address_regs = dst.index + 1;
1127
1128 emit_asm(NULL, op, dst, src0);
1129 }
1130
1131 int
1132 glsl_to_tgsi_visitor::add_constant(gl_register_file file,
1133 gl_constant_value values[8], int size, int datatype,
1134 uint16_t *swizzle_out)
1135 {
1136 if (file == PROGRAM_CONSTANT) {
1137 GLuint swizzle = swizzle_out ? *swizzle_out : 0;
1138 int result = _mesa_add_typed_unnamed_constant(this->prog->Parameters, values,
1139 size, datatype, &swizzle);
1140 if (swizzle_out)
1141 *swizzle_out = swizzle;
1142 return result;
1143 }
1144
1145 assert(file == PROGRAM_IMMEDIATE);
1146
1147 int index = 0;
1148 immediate_storage *entry;
1149 int size32 = size * ((datatype == GL_DOUBLE ||
1150 datatype == GL_INT64_ARB ||
1151 datatype == GL_UNSIGNED_INT64_ARB)? 2 : 1);
1152 int i;
1153
1154 /* Search immediate storage to see if we already have an identical
1155 * immediate that we can use instead of adding a duplicate entry.
1156 */
1157 foreach_in_list(immediate_storage, entry, &this->immediates) {
1158 immediate_storage *tmp = entry;
1159
1160 for (i = 0; i * 4 < size32; i++) {
1161 int slot_size = MIN2(size32 - (i * 4), 4);
1162 if (tmp->type != datatype || tmp->size32 != slot_size)
1163 break;
1164 if (memcmp(tmp->values, &values[i * 4],
1165 slot_size * sizeof(gl_constant_value)))
1166 break;
1167
1168 /* Everything matches, keep going until the full size is matched */
1169 tmp = (immediate_storage *)tmp->next;
1170 }
1171
1172 /* The full value matched */
1173 if (i * 4 >= size32)
1174 return index;
1175
1176 index++;
1177 }
1178
1179 for (i = 0; i * 4 < size32; i++) {
1180 int slot_size = MIN2(size32 - (i * 4), 4);
1181 /* Add this immediate to the list. */
1182 entry = new(mem_ctx) immediate_storage(&values[i * 4], slot_size, datatype);
1183 this->immediates.push_tail(entry);
1184 this->num_immediates++;
1185 }
1186 return index;
1187 }
1188
1189 st_src_reg
1190 glsl_to_tgsi_visitor::st_src_reg_for_float(float val)
1191 {
1192 st_src_reg src(PROGRAM_IMMEDIATE, -1, GLSL_TYPE_FLOAT);
1193 union gl_constant_value uval;
1194
1195 uval.f = val;
1196 src.index = add_constant(src.file, &uval, 1, GL_FLOAT, &src.swizzle);
1197
1198 return src;
1199 }
1200
1201 st_src_reg
1202 glsl_to_tgsi_visitor::st_src_reg_for_double(double val)
1203 {
1204 st_src_reg src(PROGRAM_IMMEDIATE, -1, GLSL_TYPE_DOUBLE);
1205 union gl_constant_value uval[2];
1206
1207 memcpy(uval, &val, sizeof(uval));
1208 src.index = add_constant(src.file, uval, 1, GL_DOUBLE, &src.swizzle);
1209 src.swizzle = MAKE_SWIZZLE4(SWIZZLE_X, SWIZZLE_Y, SWIZZLE_X, SWIZZLE_Y);
1210 return src;
1211 }
1212
1213 st_src_reg
1214 glsl_to_tgsi_visitor::st_src_reg_for_int(int val)
1215 {
1216 st_src_reg src(PROGRAM_IMMEDIATE, -1, GLSL_TYPE_INT);
1217 union gl_constant_value uval;
1218
1219 assert(native_integers);
1220
1221 uval.i = val;
1222 src.index = add_constant(src.file, &uval, 1, GL_INT, &src.swizzle);
1223
1224 return src;
1225 }
1226
1227 st_src_reg
1228 glsl_to_tgsi_visitor::st_src_reg_for_type(enum glsl_base_type type, int val)
1229 {
1230 if (native_integers)
1231 return type == GLSL_TYPE_FLOAT ? st_src_reg_for_float(val) :
1232 st_src_reg_for_int(val);
1233 else
1234 return st_src_reg_for_float(val);
1235 }
1236
1237 static int
1238 attrib_type_size(const struct glsl_type *type, bool is_vs_input)
1239 {
1240 return type->count_attribute_slots(is_vs_input);
1241 }
1242
1243 static int
1244 type_size(const struct glsl_type *type)
1245 {
1246 return type->count_attribute_slots(false);
1247 }
1248
1249 /**
1250 * If the given GLSL type is an array or matrix or a structure containing
1251 * an array/matrix member, return true. Else return false.
1252 *
1253 * This is used to determine which kind of temp storage (PROGRAM_TEMPORARY
1254 * or PROGRAM_ARRAY) should be used for variables of this type. Anytime
1255 * we have an array that might be indexed with a variable, we need to use
1256 * the later storage type.
1257 */
1258 static bool
1259 type_has_array_or_matrix(const glsl_type *type)
1260 {
1261 if (type->is_array() || type->is_matrix())
1262 return true;
1263
1264 if (type->is_record()) {
1265 for (unsigned i = 0; i < type->length; i++) {
1266 if (type_has_array_or_matrix(type->fields.structure[i].type)) {
1267 return true;
1268 }
1269 }
1270 }
1271
1272 return false;
1273 }
1274
1275
1276 /**
1277 * In the initial pass of codegen, we assign temporary numbers to
1278 * intermediate results. (not SSA -- variable assignments will reuse
1279 * storage).
1280 */
1281 st_src_reg
1282 glsl_to_tgsi_visitor::get_temp(const glsl_type *type)
1283 {
1284 st_src_reg src;
1285
1286 src.type = native_integers ? type->base_type : GLSL_TYPE_FLOAT;
1287 src.reladdr = NULL;
1288 src.negate = 0;
1289 src.abs = 0;
1290
1291 if (!options->EmitNoIndirectTemp && type_has_array_or_matrix(type)) {
1292 if (next_array >= max_num_arrays) {
1293 max_num_arrays += 32;
1294 array_sizes = (unsigned*)
1295 realloc(array_sizes, sizeof(array_sizes[0]) * max_num_arrays);
1296 }
1297
1298 src.file = PROGRAM_ARRAY;
1299 src.index = 0;
1300 src.array_id = next_array + 1;
1301 array_sizes[next_array] = type_size(type);
1302 ++next_array;
1303
1304 } else {
1305 src.file = PROGRAM_TEMPORARY;
1306 src.index = next_temp;
1307 next_temp += type_size(type);
1308 }
1309
1310 if (type->is_array() || type->is_record()) {
1311 src.swizzle = SWIZZLE_NOOP;
1312 } else {
1313 src.swizzle = swizzle_for_size(type->vector_elements);
1314 }
1315
1316 return src;
1317 }
1318
1319 variable_storage *
1320 glsl_to_tgsi_visitor::find_variable_storage(ir_variable *var)
1321 {
1322 struct hash_entry *entry;
1323
1324 entry = _mesa_hash_table_search(this->variables, var);
1325 if (!entry)
1326 return NULL;
1327
1328 return (variable_storage *)entry->data;
1329 }
1330
1331 void
1332 glsl_to_tgsi_visitor::visit(ir_variable *ir)
1333 {
1334 if (strcmp(ir->name, "gl_FragCoord") == 0) {
1335 this->prog->OriginUpperLeft = ir->data.origin_upper_left;
1336 this->prog->PixelCenterInteger = ir->data.pixel_center_integer;
1337 }
1338
1339 if (ir->data.mode == ir_var_uniform && strncmp(ir->name, "gl_", 3) == 0) {
1340 unsigned int i;
1341 const ir_state_slot *const slots = ir->get_state_slots();
1342 assert(slots != NULL);
1343
1344 /* Check if this statevar's setup in the STATE file exactly
1345 * matches how we'll want to reference it as a
1346 * struct/array/whatever. If not, then we need to move it into
1347 * temporary storage and hope that it'll get copy-propagated
1348 * out.
1349 */
1350 for (i = 0; i < ir->get_num_state_slots(); i++) {
1351 if (slots[i].swizzle != SWIZZLE_XYZW) {
1352 break;
1353 }
1354 }
1355
1356 variable_storage *storage;
1357 st_dst_reg dst;
1358 if (i == ir->get_num_state_slots()) {
1359 /* We'll set the index later. */
1360 storage = new(mem_ctx) variable_storage(ir, PROGRAM_STATE_VAR, -1);
1361
1362 _mesa_hash_table_insert(this->variables, ir, storage);
1363
1364 dst = undef_dst;
1365 } else {
1366 /* The variable_storage constructor allocates slots based on the size
1367 * of the type. However, this had better match the number of state
1368 * elements that we're going to copy into the new temporary.
1369 */
1370 assert((int) ir->get_num_state_slots() == type_size(ir->type));
1371
1372 dst = st_dst_reg(get_temp(ir->type));
1373
1374 storage = new(mem_ctx) variable_storage(ir, dst.file, dst.index,
1375 dst.array_id);
1376
1377 _mesa_hash_table_insert(this->variables, ir, storage);
1378 }
1379
1380
1381 for (unsigned int i = 0; i < ir->get_num_state_slots(); i++) {
1382 int index = _mesa_add_state_reference(this->prog->Parameters,
1383 (gl_state_index *)slots[i].tokens);
1384
1385 if (storage->file == PROGRAM_STATE_VAR) {
1386 if (storage->index == -1) {
1387 storage->index = index;
1388 } else {
1389 assert(index == storage->index + (int)i);
1390 }
1391 } else {
1392 /* We use GLSL_TYPE_FLOAT here regardless of the actual type of
1393 * the data being moved since MOV does not care about the type of
1394 * data it is moving, and we don't want to declare registers with
1395 * array or struct types.
1396 */
1397 st_src_reg src(PROGRAM_STATE_VAR, index, GLSL_TYPE_FLOAT);
1398 src.swizzle = slots[i].swizzle;
1399 emit_asm(ir, TGSI_OPCODE_MOV, dst, src);
1400 /* even a float takes up a whole vec4 reg in a struct/array. */
1401 dst.index++;
1402 }
1403 }
1404
1405 if (storage->file == PROGRAM_TEMPORARY &&
1406 dst.index != storage->index + (int) ir->get_num_state_slots()) {
1407 fail_link(this->shader_program,
1408 "failed to load builtin uniform `%s' (%d/%d regs loaded)\n",
1409 ir->name, dst.index - storage->index,
1410 type_size(ir->type));
1411 }
1412 }
1413 }
1414
1415 void
1416 glsl_to_tgsi_visitor::visit(ir_loop *ir)
1417 {
1418 emit_asm(NULL, TGSI_OPCODE_BGNLOOP);
1419
1420 visit_exec_list(&ir->body_instructions, this);
1421
1422 emit_asm(NULL, TGSI_OPCODE_ENDLOOP);
1423 }
1424
1425 void
1426 glsl_to_tgsi_visitor::visit(ir_loop_jump *ir)
1427 {
1428 switch (ir->mode) {
1429 case ir_loop_jump::jump_break:
1430 emit_asm(NULL, TGSI_OPCODE_BRK);
1431 break;
1432 case ir_loop_jump::jump_continue:
1433 emit_asm(NULL, TGSI_OPCODE_CONT);
1434 break;
1435 }
1436 }
1437
1438
1439 void
1440 glsl_to_tgsi_visitor::visit(ir_function_signature *ir)
1441 {
1442 assert(0);
1443 (void)ir;
1444 }
1445
1446 void
1447 glsl_to_tgsi_visitor::visit(ir_function *ir)
1448 {
1449 /* Ignore function bodies other than main() -- we shouldn't see calls to
1450 * them since they should all be inlined before we get to glsl_to_tgsi.
1451 */
1452 if (strcmp(ir->name, "main") == 0) {
1453 const ir_function_signature *sig;
1454 exec_list empty;
1455
1456 sig = ir->matching_signature(NULL, &empty, false);
1457
1458 assert(sig);
1459
1460 foreach_in_list(ir_instruction, ir, &sig->body) {
1461 ir->accept(this);
1462 }
1463 }
1464 }
1465
1466 bool
1467 glsl_to_tgsi_visitor::try_emit_mad(ir_expression *ir, int mul_operand)
1468 {
1469 int nonmul_operand = 1 - mul_operand;
1470 st_src_reg a, b, c;
1471 st_dst_reg result_dst;
1472
1473 ir_expression *expr = ir->operands[mul_operand]->as_expression();
1474 if (!expr || expr->operation != ir_binop_mul)
1475 return false;
1476
1477 expr->operands[0]->accept(this);
1478 a = this->result;
1479 expr->operands[1]->accept(this);
1480 b = this->result;
1481 ir->operands[nonmul_operand]->accept(this);
1482 c = this->result;
1483
1484 this->result = get_temp(ir->type);
1485 result_dst = st_dst_reg(this->result);
1486 result_dst.writemask = (1 << ir->type->vector_elements) - 1;
1487 emit_asm(ir, TGSI_OPCODE_MAD, result_dst, a, b, c);
1488
1489 return true;
1490 }
1491
1492 /**
1493 * Emit MAD(a, -b, a) instead of AND(a, NOT(b))
1494 *
1495 * The logic values are 1.0 for true and 0.0 for false. Logical-and is
1496 * implemented using multiplication, and logical-or is implemented using
1497 * addition. Logical-not can be implemented as (true - x), or (1.0 - x).
1498 * As result, the logical expression (a & !b) can be rewritten as:
1499 *
1500 * - a * !b
1501 * - a * (1 - b)
1502 * - (a * 1) - (a * b)
1503 * - a + -(a * b)
1504 * - a + (a * -b)
1505 *
1506 * This final expression can be implemented as a single MAD(a, -b, a)
1507 * instruction.
1508 */
1509 bool
1510 glsl_to_tgsi_visitor::try_emit_mad_for_and_not(ir_expression *ir, int try_operand)
1511 {
1512 const int other_operand = 1 - try_operand;
1513 st_src_reg a, b;
1514
1515 ir_expression *expr = ir->operands[try_operand]->as_expression();
1516 if (!expr || expr->operation != ir_unop_logic_not)
1517 return false;
1518
1519 ir->operands[other_operand]->accept(this);
1520 a = this->result;
1521 expr->operands[0]->accept(this);
1522 b = this->result;
1523
1524 b.negate = ~b.negate;
1525
1526 this->result = get_temp(ir->type);
1527 emit_asm(ir, TGSI_OPCODE_MAD, st_dst_reg(this->result), a, b, a);
1528
1529 return true;
1530 }
1531
1532 void
1533 glsl_to_tgsi_visitor::reladdr_to_temp(ir_instruction *ir,
1534 st_src_reg *reg, int *num_reladdr)
1535 {
1536 if (!reg->reladdr && !reg->reladdr2)
1537 return;
1538
1539 if (reg->reladdr) emit_arl(ir, address_reg, *reg->reladdr);
1540 if (reg->reladdr2) emit_arl(ir, address_reg2, *reg->reladdr2);
1541
1542 if (*num_reladdr != 1) {
1543 st_src_reg temp = get_temp(reg->type == GLSL_TYPE_DOUBLE ? glsl_type::dvec4_type : glsl_type::vec4_type);
1544
1545 emit_asm(ir, TGSI_OPCODE_MOV, st_dst_reg(temp), *reg);
1546 *reg = temp;
1547 }
1548
1549 (*num_reladdr)--;
1550 }
1551
1552 void
1553 glsl_to_tgsi_visitor::visit(ir_expression *ir)
1554 {
1555 st_src_reg op[ARRAY_SIZE(ir->operands)];
1556
1557 /* Quick peephole: Emit MAD(a, b, c) instead of ADD(MUL(a, b), c)
1558 */
1559 if (!this->precise && ir->operation == ir_binop_add) {
1560 if (try_emit_mad(ir, 1))
1561 return;
1562 if (try_emit_mad(ir, 0))
1563 return;
1564 }
1565
1566 /* Quick peephole: Emit OPCODE_MAD(-a, -b, a) instead of AND(a, NOT(b))
1567 */
1568 if (!native_integers && ir->operation == ir_binop_logic_and) {
1569 if (try_emit_mad_for_and_not(ir, 1))
1570 return;
1571 if (try_emit_mad_for_and_not(ir, 0))
1572 return;
1573 }
1574
1575 if (ir->operation == ir_quadop_vector)
1576 assert(!"ir_quadop_vector should have been lowered");
1577
1578 for (unsigned int operand = 0; operand < ir->num_operands; operand++) {
1579 this->result.file = PROGRAM_UNDEFINED;
1580 ir->operands[operand]->accept(this);
1581 if (this->result.file == PROGRAM_UNDEFINED) {
1582 printf("Failed to get tree for expression operand:\n");
1583 ir->operands[operand]->print();
1584 printf("\n");
1585 exit(1);
1586 }
1587 op[operand] = this->result;
1588
1589 /* Matrix expression operands should have been broken down to vector
1590 * operations already.
1591 */
1592 assert(!ir->operands[operand]->type->is_matrix());
1593 }
1594
1595 visit_expression(ir, op);
1596 }
1597
1598 /* The non-recursive part of the expression visitor lives in a separate
1599 * function and should be prevented from being inlined, to avoid a stack
1600 * explosion when deeply nested expressions are visited.
1601 */
1602 void
1603 glsl_to_tgsi_visitor::visit_expression(ir_expression* ir, st_src_reg *op)
1604 {
1605 st_src_reg result_src;
1606 st_dst_reg result_dst;
1607
1608 int vector_elements = ir->operands[0]->type->vector_elements;
1609 if (ir->operands[1]) {
1610 vector_elements = MAX2(vector_elements,
1611 ir->operands[1]->type->vector_elements);
1612 }
1613
1614 this->result.file = PROGRAM_UNDEFINED;
1615
1616 /* Storage for our result. Ideally for an assignment we'd be using
1617 * the actual storage for the result here, instead.
1618 */
1619 result_src = get_temp(ir->type);
1620 /* convenience for the emit functions below. */
1621 result_dst = st_dst_reg(result_src);
1622 /* Limit writes to the channels that will be used by result_src later.
1623 * This does limit this temp's use as a temporary for multi-instruction
1624 * sequences.
1625 */
1626 result_dst.writemask = (1 << ir->type->vector_elements) - 1;
1627
1628 switch (ir->operation) {
1629 case ir_unop_logic_not:
1630 if (result_dst.type != GLSL_TYPE_FLOAT)
1631 emit_asm(ir, TGSI_OPCODE_NOT, result_dst, op[0]);
1632 else {
1633 /* Previously 'SEQ dst, src, 0.0' was used for this. However, many
1634 * older GPUs implement SEQ using multiple instructions (i915 uses two
1635 * SGE instructions and a MUL instruction). Since our logic values are
1636 * 0.0 and 1.0, 1-x also implements !x.
1637 */
1638 op[0].negate = ~op[0].negate;
1639 emit_asm(ir, TGSI_OPCODE_ADD, result_dst, op[0], st_src_reg_for_float(1.0));
1640 }
1641 break;
1642 case ir_unop_neg:
1643 if (result_dst.type == GLSL_TYPE_INT64 || result_dst.type == GLSL_TYPE_UINT64)
1644 emit_asm(ir, TGSI_OPCODE_I64NEG, result_dst, op[0]);
1645 else if (result_dst.type == GLSL_TYPE_INT || result_dst.type == GLSL_TYPE_UINT)
1646 emit_asm(ir, TGSI_OPCODE_INEG, result_dst, op[0]);
1647 else if (result_dst.type == GLSL_TYPE_DOUBLE)
1648 emit_asm(ir, TGSI_OPCODE_DNEG, result_dst, op[0]);
1649 else {
1650 op[0].negate = ~op[0].negate;
1651 result_src = op[0];
1652 }
1653 break;
1654 case ir_unop_subroutine_to_int:
1655 emit_asm(ir, TGSI_OPCODE_MOV, result_dst, op[0]);
1656 break;
1657 case ir_unop_abs:
1658 if (result_dst.type == GLSL_TYPE_FLOAT)
1659 emit_asm(ir, TGSI_OPCODE_MOV, result_dst, op[0].get_abs());
1660 else if (result_dst.type == GLSL_TYPE_DOUBLE)
1661 emit_asm(ir, TGSI_OPCODE_DABS, result_dst, op[0]);
1662 else if (result_dst.type == GLSL_TYPE_INT64 || result_dst.type == GLSL_TYPE_UINT64)
1663 emit_asm(ir, TGSI_OPCODE_I64ABS, result_dst, op[0]);
1664 else
1665 emit_asm(ir, TGSI_OPCODE_IABS, result_dst, op[0]);
1666 break;
1667 case ir_unop_sign:
1668 emit_asm(ir, TGSI_OPCODE_SSG, result_dst, op[0]);
1669 break;
1670 case ir_unop_rcp:
1671 emit_scalar(ir, TGSI_OPCODE_RCP, result_dst, op[0]);
1672 break;
1673
1674 case ir_unop_exp2:
1675 emit_scalar(ir, TGSI_OPCODE_EX2, result_dst, op[0]);
1676 break;
1677 case ir_unop_exp:
1678 assert(!"not reached: should be handled by exp_to_exp2");
1679 break;
1680 case ir_unop_log:
1681 assert(!"not reached: should be handled by log_to_log2");
1682 break;
1683 case ir_unop_log2:
1684 emit_scalar(ir, TGSI_OPCODE_LG2, result_dst, op[0]);
1685 break;
1686 case ir_unop_sin:
1687 emit_scalar(ir, TGSI_OPCODE_SIN, result_dst, op[0]);
1688 break;
1689 case ir_unop_cos:
1690 emit_scalar(ir, TGSI_OPCODE_COS, result_dst, op[0]);
1691 break;
1692 case ir_unop_saturate: {
1693 glsl_to_tgsi_instruction *inst;
1694 inst = emit_asm(ir, TGSI_OPCODE_MOV, result_dst, op[0]);
1695 inst->saturate = true;
1696 break;
1697 }
1698
1699 case ir_unop_dFdx:
1700 case ir_unop_dFdx_coarse:
1701 emit_asm(ir, TGSI_OPCODE_DDX, result_dst, op[0]);
1702 break;
1703 case ir_unop_dFdx_fine:
1704 emit_asm(ir, TGSI_OPCODE_DDX_FINE, result_dst, op[0]);
1705 break;
1706 case ir_unop_dFdy:
1707 case ir_unop_dFdy_coarse:
1708 case ir_unop_dFdy_fine:
1709 {
1710 /* The X component contains 1 or -1 depending on whether the framebuffer
1711 * is a FBO or the window system buffer, respectively.
1712 * It is then multiplied with the source operand of DDY.
1713 */
1714 static const gl_state_index transform_y_state[STATE_LENGTH]
1715 = { STATE_INTERNAL, STATE_FB_WPOS_Y_TRANSFORM };
1716
1717 unsigned transform_y_index =
1718 _mesa_add_state_reference(this->prog->Parameters,
1719 transform_y_state);
1720
1721 st_src_reg transform_y = st_src_reg(PROGRAM_STATE_VAR,
1722 transform_y_index,
1723 glsl_type::vec4_type);
1724 transform_y.swizzle = SWIZZLE_XXXX;
1725
1726 st_src_reg temp = get_temp(glsl_type::vec4_type);
1727
1728 emit_asm(ir, TGSI_OPCODE_MUL, st_dst_reg(temp), transform_y, op[0]);
1729 emit_asm(ir, ir->operation == ir_unop_dFdy_fine ?
1730 TGSI_OPCODE_DDY_FINE : TGSI_OPCODE_DDY, result_dst, temp);
1731 break;
1732 }
1733
1734 case ir_unop_frexp_sig:
1735 emit_asm(ir, TGSI_OPCODE_DFRACEXP, result_dst, undef_dst, op[0]);
1736 break;
1737
1738 case ir_unop_frexp_exp:
1739 emit_asm(ir, TGSI_OPCODE_DFRACEXP, undef_dst, result_dst, op[0]);
1740 break;
1741
1742 case ir_unop_noise: {
1743 /* At some point, a motivated person could add a better
1744 * implementation of noise. Currently not even the nvidia
1745 * binary drivers do anything more than this. In any case, the
1746 * place to do this is in the GL state tracker, not the poor
1747 * driver.
1748 */
1749 emit_asm(ir, TGSI_OPCODE_MOV, result_dst, st_src_reg_for_float(0.5));
1750 break;
1751 }
1752
1753 case ir_binop_add:
1754 emit_asm(ir, TGSI_OPCODE_ADD, result_dst, op[0], op[1]);
1755 break;
1756 case ir_binop_sub:
1757 op[1].negate = ~op[1].negate;
1758 emit_asm(ir, TGSI_OPCODE_ADD, result_dst, op[0], op[1]);
1759 break;
1760
1761 case ir_binop_mul:
1762 emit_asm(ir, TGSI_OPCODE_MUL, result_dst, op[0], op[1]);
1763 break;
1764 case ir_binop_div:
1765 emit_asm(ir, TGSI_OPCODE_DIV, result_dst, op[0], op[1]);
1766 break;
1767 case ir_binop_mod:
1768 if (result_dst.type == GLSL_TYPE_FLOAT)
1769 assert(!"ir_binop_mod should have been converted to b * fract(a/b)");
1770 else
1771 emit_asm(ir, TGSI_OPCODE_MOD, result_dst, op[0], op[1]);
1772 break;
1773
1774 case ir_binop_less:
1775 emit_asm(ir, TGSI_OPCODE_SLT, result_dst, op[0], op[1]);
1776 break;
1777 case ir_binop_greater:
1778 emit_asm(ir, TGSI_OPCODE_SLT, result_dst, op[1], op[0]);
1779 break;
1780 case ir_binop_lequal:
1781 emit_asm(ir, TGSI_OPCODE_SGE, result_dst, op[1], op[0]);
1782 break;
1783 case ir_binop_gequal:
1784 emit_asm(ir, TGSI_OPCODE_SGE, result_dst, op[0], op[1]);
1785 break;
1786 case ir_binop_equal:
1787 emit_asm(ir, TGSI_OPCODE_SEQ, result_dst, op[0], op[1]);
1788 break;
1789 case ir_binop_nequal:
1790 emit_asm(ir, TGSI_OPCODE_SNE, result_dst, op[0], op[1]);
1791 break;
1792 case ir_binop_all_equal:
1793 /* "==" operator producing a scalar boolean. */
1794 if (ir->operands[0]->type->is_vector() ||
1795 ir->operands[1]->type->is_vector()) {
1796 st_src_reg temp = get_temp(native_integers ?
1797 glsl_type::uvec4_type :
1798 glsl_type::vec4_type);
1799
1800 if (native_integers) {
1801 st_dst_reg temp_dst = st_dst_reg(temp);
1802 st_src_reg temp1 = st_src_reg(temp), temp2 = st_src_reg(temp);
1803
1804 if (ir->operands[0]->type->is_boolean() &&
1805 ir->operands[1]->as_constant() &&
1806 ir->operands[1]->as_constant()->is_one()) {
1807 emit_asm(ir, TGSI_OPCODE_MOV, st_dst_reg(temp), op[0]);
1808 } else {
1809 emit_asm(ir, TGSI_OPCODE_SEQ, st_dst_reg(temp), op[0], op[1]);
1810 }
1811
1812 /* Emit 1-3 AND operations to combine the SEQ results. */
1813 switch (ir->operands[0]->type->vector_elements) {
1814 case 2:
1815 break;
1816 case 3:
1817 temp_dst.writemask = WRITEMASK_Y;
1818 temp1.swizzle = SWIZZLE_YYYY;
1819 temp2.swizzle = SWIZZLE_ZZZZ;
1820 emit_asm(ir, TGSI_OPCODE_AND, temp_dst, temp1, temp2);
1821 break;
1822 case 4:
1823 temp_dst.writemask = WRITEMASK_X;
1824 temp1.swizzle = SWIZZLE_XXXX;
1825 temp2.swizzle = SWIZZLE_YYYY;
1826 emit_asm(ir, TGSI_OPCODE_AND, temp_dst, temp1, temp2);
1827 temp_dst.writemask = WRITEMASK_Y;
1828 temp1.swizzle = SWIZZLE_ZZZZ;
1829 temp2.swizzle = SWIZZLE_WWWW;
1830 emit_asm(ir, TGSI_OPCODE_AND, temp_dst, temp1, temp2);
1831 }
1832
1833 temp1.swizzle = SWIZZLE_XXXX;
1834 temp2.swizzle = SWIZZLE_YYYY;
1835 emit_asm(ir, TGSI_OPCODE_AND, result_dst, temp1, temp2);
1836 } else {
1837 emit_asm(ir, TGSI_OPCODE_SNE, st_dst_reg(temp), op[0], op[1]);
1838
1839 /* After the dot-product, the value will be an integer on the
1840 * range [0,4]. Zero becomes 1.0, and positive values become zero.
1841 */
1842 emit_dp(ir, result_dst, temp, temp, vector_elements);
1843
1844 /* Negating the result of the dot-product gives values on the range
1845 * [-4, 0]. Zero becomes 1.0, and negative values become zero.
1846 * This is achieved using SGE.
1847 */
1848 st_src_reg sge_src = result_src;
1849 sge_src.negate = ~sge_src.negate;
1850 emit_asm(ir, TGSI_OPCODE_SGE, result_dst, sge_src, st_src_reg_for_float(0.0));
1851 }
1852 } else {
1853 emit_asm(ir, TGSI_OPCODE_SEQ, result_dst, op[0], op[1]);
1854 }
1855 break;
1856 case ir_binop_any_nequal:
1857 /* "!=" operator producing a scalar boolean. */
1858 if (ir->operands[0]->type->is_vector() ||
1859 ir->operands[1]->type->is_vector()) {
1860 st_src_reg temp = get_temp(native_integers ?
1861 glsl_type::uvec4_type :
1862 glsl_type::vec4_type);
1863 if (ir->operands[0]->type->is_boolean() &&
1864 ir->operands[1]->as_constant() &&
1865 ir->operands[1]->as_constant()->is_zero()) {
1866 emit_asm(ir, TGSI_OPCODE_MOV, st_dst_reg(temp), op[0]);
1867 } else {
1868 emit_asm(ir, TGSI_OPCODE_SNE, st_dst_reg(temp), op[0], op[1]);
1869 }
1870
1871 if (native_integers) {
1872 st_dst_reg temp_dst = st_dst_reg(temp);
1873 st_src_reg temp1 = st_src_reg(temp), temp2 = st_src_reg(temp);
1874
1875 /* Emit 1-3 OR operations to combine the SNE results. */
1876 switch (ir->operands[0]->type->vector_elements) {
1877 case 2:
1878 break;
1879 case 3:
1880 temp_dst.writemask = WRITEMASK_Y;
1881 temp1.swizzle = SWIZZLE_YYYY;
1882 temp2.swizzle = SWIZZLE_ZZZZ;
1883 emit_asm(ir, TGSI_OPCODE_OR, temp_dst, temp1, temp2);
1884 break;
1885 case 4:
1886 temp_dst.writemask = WRITEMASK_X;
1887 temp1.swizzle = SWIZZLE_XXXX;
1888 temp2.swizzle = SWIZZLE_YYYY;
1889 emit_asm(ir, TGSI_OPCODE_OR, temp_dst, temp1, temp2);
1890 temp_dst.writemask = WRITEMASK_Y;
1891 temp1.swizzle = SWIZZLE_ZZZZ;
1892 temp2.swizzle = SWIZZLE_WWWW;
1893 emit_asm(ir, TGSI_OPCODE_OR, temp_dst, temp1, temp2);
1894 }
1895
1896 temp1.swizzle = SWIZZLE_XXXX;
1897 temp2.swizzle = SWIZZLE_YYYY;
1898 emit_asm(ir, TGSI_OPCODE_OR, result_dst, temp1, temp2);
1899 } else {
1900 /* After the dot-product, the value will be an integer on the
1901 * range [0,4]. Zero stays zero, and positive values become 1.0.
1902 */
1903 glsl_to_tgsi_instruction *const dp =
1904 emit_dp(ir, result_dst, temp, temp, vector_elements);
1905 if (this->prog->Target == GL_FRAGMENT_PROGRAM_ARB) {
1906 /* The clamping to [0,1] can be done for free in the fragment
1907 * shader with a saturate.
1908 */
1909 dp->saturate = true;
1910 } else {
1911 /* Negating the result of the dot-product gives values on the range
1912 * [-4, 0]. Zero stays zero, and negative values become 1.0. This
1913 * achieved using SLT.
1914 */
1915 st_src_reg slt_src = result_src;
1916 slt_src.negate = ~slt_src.negate;
1917 emit_asm(ir, TGSI_OPCODE_SLT, result_dst, slt_src, st_src_reg_for_float(0.0));
1918 }
1919 }
1920 } else {
1921 emit_asm(ir, TGSI_OPCODE_SNE, result_dst, op[0], op[1]);
1922 }
1923 break;
1924
1925 case ir_binop_logic_xor:
1926 if (native_integers)
1927 emit_asm(ir, TGSI_OPCODE_XOR, result_dst, op[0], op[1]);
1928 else
1929 emit_asm(ir, TGSI_OPCODE_SNE, result_dst, op[0], op[1]);
1930 break;
1931
1932 case ir_binop_logic_or: {
1933 if (native_integers) {
1934 /* If integers are used as booleans, we can use an actual "or"
1935 * instruction.
1936 */
1937 assert(native_integers);
1938 emit_asm(ir, TGSI_OPCODE_OR, result_dst, op[0], op[1]);
1939 } else {
1940 /* After the addition, the value will be an integer on the
1941 * range [0,2]. Zero stays zero, and positive values become 1.0.
1942 */
1943 glsl_to_tgsi_instruction *add =
1944 emit_asm(ir, TGSI_OPCODE_ADD, result_dst, op[0], op[1]);
1945 if (this->prog->Target == GL_FRAGMENT_PROGRAM_ARB) {
1946 /* The clamping to [0,1] can be done for free in the fragment
1947 * shader with a saturate if floats are being used as boolean values.
1948 */
1949 add->saturate = true;
1950 } else {
1951 /* Negating the result of the addition gives values on the range
1952 * [-2, 0]. Zero stays zero, and negative values become 1.0. This
1953 * is achieved using SLT.
1954 */
1955 st_src_reg slt_src = result_src;
1956 slt_src.negate = ~slt_src.negate;
1957 emit_asm(ir, TGSI_OPCODE_SLT, result_dst, slt_src, st_src_reg_for_float(0.0));
1958 }
1959 }
1960 break;
1961 }
1962
1963 case ir_binop_logic_and:
1964 /* If native integers are disabled, the bool args are stored as float 0.0
1965 * or 1.0, so "mul" gives us "and". If they're enabled, just use the
1966 * actual AND opcode.
1967 */
1968 if (native_integers)
1969 emit_asm(ir, TGSI_OPCODE_AND, result_dst, op[0], op[1]);
1970 else
1971 emit_asm(ir, TGSI_OPCODE_MUL, result_dst, op[0], op[1]);
1972 break;
1973
1974 case ir_binop_dot:
1975 assert(ir->operands[0]->type->is_vector());
1976 assert(ir->operands[0]->type == ir->operands[1]->type);
1977 emit_dp(ir, result_dst, op[0], op[1],
1978 ir->operands[0]->type->vector_elements);
1979 break;
1980
1981 case ir_unop_sqrt:
1982 if (have_sqrt) {
1983 emit_scalar(ir, TGSI_OPCODE_SQRT, result_dst, op[0]);
1984 } else {
1985 /* This is the only instruction sequence that makes the game "Risen"
1986 * render correctly. ABS is not required for the game, but since GLSL
1987 * declares negative values as "undefined", allowing us to do whatever
1988 * we want, I choose to use ABS to match DX9 and pre-GLSL RSQ
1989 * behavior.
1990 */
1991 emit_scalar(ir, TGSI_OPCODE_RSQ, result_dst, op[0].get_abs());
1992 emit_scalar(ir, TGSI_OPCODE_RCP, result_dst, result_src);
1993 }
1994 break;
1995 case ir_unop_rsq:
1996 emit_scalar(ir, TGSI_OPCODE_RSQ, result_dst, op[0]);
1997 break;
1998 case ir_unop_i2f:
1999 if (native_integers) {
2000 emit_asm(ir, TGSI_OPCODE_I2F, result_dst, op[0]);
2001 break;
2002 }
2003 /* fallthrough to next case otherwise */
2004 case ir_unop_b2f:
2005 if (native_integers) {
2006 emit_asm(ir, TGSI_OPCODE_AND, result_dst, op[0], st_src_reg_for_float(1.0));
2007 break;
2008 }
2009 /* fallthrough to next case otherwise */
2010 case ir_unop_i2u:
2011 case ir_unop_u2i:
2012 case ir_unop_i642u64:
2013 case ir_unop_u642i64:
2014 /* Converting between signed and unsigned integers is a no-op. */
2015 result_src = op[0];
2016 result_src.type = result_dst.type;
2017 break;
2018 case ir_unop_b2i:
2019 if (native_integers) {
2020 /* Booleans are stored as integers using ~0 for true and 0 for false.
2021 * GLSL requires that int(bool) return 1 for true and 0 for false.
2022 * This conversion is done with AND, but it could be done with NEG.
2023 */
2024 emit_asm(ir, TGSI_OPCODE_AND, result_dst, op[0], st_src_reg_for_int(1));
2025 } else {
2026 /* Booleans and integers are both stored as floats when native
2027 * integers are disabled.
2028 */
2029 result_src = op[0];
2030 }
2031 break;
2032 case ir_unop_f2i:
2033 if (native_integers)
2034 emit_asm(ir, TGSI_OPCODE_F2I, result_dst, op[0]);
2035 else
2036 emit_asm(ir, TGSI_OPCODE_TRUNC, result_dst, op[0]);
2037 break;
2038 case ir_unop_f2u:
2039 if (native_integers)
2040 emit_asm(ir, TGSI_OPCODE_F2U, result_dst, op[0]);
2041 else
2042 emit_asm(ir, TGSI_OPCODE_TRUNC, result_dst, op[0]);
2043 break;
2044 case ir_unop_bitcast_f2i:
2045 case ir_unop_bitcast_f2u:
2046 /* Make sure we don't propagate the negate modifier to integer opcodes. */
2047 if (op[0].negate || op[0].abs)
2048 emit_asm(ir, TGSI_OPCODE_MOV, result_dst, op[0]);
2049 else
2050 result_src = op[0];
2051 result_src.type = ir->operation == ir_unop_bitcast_f2i ? GLSL_TYPE_INT :
2052 GLSL_TYPE_UINT;
2053 break;
2054 case ir_unop_bitcast_i2f:
2055 case ir_unop_bitcast_u2f:
2056 result_src = op[0];
2057 result_src.type = GLSL_TYPE_FLOAT;
2058 break;
2059 case ir_unop_f2b:
2060 emit_asm(ir, TGSI_OPCODE_SNE, result_dst, op[0], st_src_reg_for_float(0.0));
2061 break;
2062 case ir_unop_d2b:
2063 emit_asm(ir, TGSI_OPCODE_SNE, result_dst, op[0], st_src_reg_for_double(0.0));
2064 break;
2065 case ir_unop_i2b:
2066 if (native_integers)
2067 emit_asm(ir, TGSI_OPCODE_USNE, result_dst, op[0], st_src_reg_for_int(0));
2068 else
2069 emit_asm(ir, TGSI_OPCODE_SNE, result_dst, op[0], st_src_reg_for_float(0.0));
2070 break;
2071 case ir_unop_bitcast_u642d:
2072 case ir_unop_bitcast_i642d:
2073 result_src = op[0];
2074 result_src.type = GLSL_TYPE_DOUBLE;
2075 break;
2076 case ir_unop_bitcast_d2i64:
2077 result_src = op[0];
2078 result_src.type = GLSL_TYPE_INT64;
2079 break;
2080 case ir_unop_bitcast_d2u64:
2081 result_src = op[0];
2082 result_src.type = GLSL_TYPE_UINT64;
2083 break;
2084 case ir_unop_trunc:
2085 emit_asm(ir, TGSI_OPCODE_TRUNC, result_dst, op[0]);
2086 break;
2087 case ir_unop_ceil:
2088 emit_asm(ir, TGSI_OPCODE_CEIL, result_dst, op[0]);
2089 break;
2090 case ir_unop_floor:
2091 emit_asm(ir, TGSI_OPCODE_FLR, result_dst, op[0]);
2092 break;
2093 case ir_unop_round_even:
2094 emit_asm(ir, TGSI_OPCODE_ROUND, result_dst, op[0]);
2095 break;
2096 case ir_unop_fract:
2097 emit_asm(ir, TGSI_OPCODE_FRC, result_dst, op[0]);
2098 break;
2099
2100 case ir_binop_min:
2101 emit_asm(ir, TGSI_OPCODE_MIN, result_dst, op[0], op[1]);
2102 break;
2103 case ir_binop_max:
2104 emit_asm(ir, TGSI_OPCODE_MAX, result_dst, op[0], op[1]);
2105 break;
2106 case ir_binop_pow:
2107 emit_scalar(ir, TGSI_OPCODE_POW, result_dst, op[0], op[1]);
2108 break;
2109
2110 case ir_unop_bit_not:
2111 if (native_integers) {
2112 emit_asm(ir, TGSI_OPCODE_NOT, result_dst, op[0]);
2113 break;
2114 }
2115 case ir_unop_u2f:
2116 if (native_integers) {
2117 emit_asm(ir, TGSI_OPCODE_U2F, result_dst, op[0]);
2118 break;
2119 }
2120 case ir_binop_lshift:
2121 case ir_binop_rshift:
2122 if (native_integers) {
2123 unsigned opcode = ir->operation == ir_binop_lshift ? TGSI_OPCODE_SHL
2124 : TGSI_OPCODE_ISHR;
2125 st_src_reg count;
2126
2127 if (glsl_base_type_is_64bit(op[0].type)) {
2128 /* GLSL shift operations have 32-bit shift counts, but TGSI uses
2129 * 64 bits.
2130 */
2131 count = get_temp(glsl_type::u64vec(ir->operands[1]->type->components()));
2132 emit_asm(ir, TGSI_OPCODE_U2I64, st_dst_reg(count), op[1]);
2133 } else {
2134 count = op[1];
2135 }
2136
2137 emit_asm(ir, opcode, result_dst, op[0], count);
2138 break;
2139 }
2140 case ir_binop_bit_and:
2141 if (native_integers) {
2142 emit_asm(ir, TGSI_OPCODE_AND, result_dst, op[0], op[1]);
2143 break;
2144 }
2145 case ir_binop_bit_xor:
2146 if (native_integers) {
2147 emit_asm(ir, TGSI_OPCODE_XOR, result_dst, op[0], op[1]);
2148 break;
2149 }
2150 case ir_binop_bit_or:
2151 if (native_integers) {
2152 emit_asm(ir, TGSI_OPCODE_OR, result_dst, op[0], op[1]);
2153 break;
2154 }
2155
2156 assert(!"GLSL 1.30 features unsupported");
2157 break;
2158
2159 case ir_binop_ubo_load: {
2160 ir_constant *const_uniform_block = ir->operands[0]->as_constant();
2161 ir_constant *const_offset_ir = ir->operands[1]->as_constant();
2162 unsigned const_offset = const_offset_ir ? const_offset_ir->value.u[0] : 0;
2163 unsigned const_block = const_uniform_block ? const_uniform_block->value.u[0] + 1 : 0;
2164 st_src_reg index_reg = get_temp(glsl_type::uint_type);
2165 st_src_reg cbuf;
2166
2167 cbuf.type = ir->type->base_type;
2168 cbuf.file = PROGRAM_CONSTANT;
2169 cbuf.index = 0;
2170 cbuf.reladdr = NULL;
2171 cbuf.negate = 0;
2172 cbuf.abs = 0;
2173
2174 assert(ir->type->is_vector() || ir->type->is_scalar());
2175
2176 if (const_offset_ir) {
2177 /* Constant index into constant buffer */
2178 cbuf.reladdr = NULL;
2179 cbuf.index = const_offset / 16;
2180 }
2181 else {
2182 ir_expression *offset_expr = ir->operands[1]->as_expression();
2183 st_src_reg offset = op[1];
2184
2185 /* The OpenGL spec is written in such a way that accesses with
2186 * non-constant offset are almost always vec4-aligned. The only
2187 * exception to this are members of structs in arrays of structs:
2188 * each struct in an array of structs is at least vec4-aligned,
2189 * but single-element and [ui]vec2 members of the struct may be at
2190 * an offset that is not a multiple of 16 bytes.
2191 *
2192 * Here, we extract that offset, relying on previous passes to always
2193 * generate offset expressions of the form (+ expr constant_offset).
2194 *
2195 * Note that the std430 layout, which allows more cases of alignment
2196 * less than vec4 in arrays, is not supported for uniform blocks, so
2197 * we do not have to deal with it here.
2198 */
2199 if (offset_expr && offset_expr->operation == ir_binop_add) {
2200 const_offset_ir = offset_expr->operands[1]->as_constant();
2201 if (const_offset_ir) {
2202 const_offset = const_offset_ir->value.u[0];
2203 cbuf.index = const_offset / 16;
2204 offset_expr->operands[0]->accept(this);
2205 offset = this->result;
2206 }
2207 }
2208
2209 /* Relative/variable index into constant buffer */
2210 emit_asm(ir, TGSI_OPCODE_USHR, st_dst_reg(index_reg), offset,
2211 st_src_reg_for_int(4));
2212 cbuf.reladdr = ralloc(mem_ctx, st_src_reg);
2213 memcpy(cbuf.reladdr, &index_reg, sizeof(index_reg));
2214 }
2215
2216 if (const_uniform_block) {
2217 /* Constant constant buffer */
2218 cbuf.reladdr2 = NULL;
2219 cbuf.index2D = const_block;
2220 cbuf.has_index2 = true;
2221 }
2222 else {
2223 /* Relative/variable constant buffer */
2224 cbuf.reladdr2 = ralloc(mem_ctx, st_src_reg);
2225 cbuf.index2D = 1;
2226 memcpy(cbuf.reladdr2, &op[0], sizeof(st_src_reg));
2227 cbuf.has_index2 = true;
2228 }
2229
2230 cbuf.swizzle = swizzle_for_size(ir->type->vector_elements);
2231 if (glsl_base_type_is_64bit(cbuf.type))
2232 cbuf.swizzle += MAKE_SWIZZLE4(const_offset % 16 / 8,
2233 const_offset % 16 / 8,
2234 const_offset % 16 / 8,
2235 const_offset % 16 / 8);
2236 else
2237 cbuf.swizzle += MAKE_SWIZZLE4(const_offset % 16 / 4,
2238 const_offset % 16 / 4,
2239 const_offset % 16 / 4,
2240 const_offset % 16 / 4);
2241
2242 if (ir->type->is_boolean()) {
2243 emit_asm(ir, TGSI_OPCODE_USNE, result_dst, cbuf, st_src_reg_for_int(0));
2244 } else {
2245 emit_asm(ir, TGSI_OPCODE_MOV, result_dst, cbuf);
2246 }
2247 break;
2248 }
2249 case ir_triop_lrp:
2250 /* note: we have to reorder the three args here */
2251 emit_asm(ir, TGSI_OPCODE_LRP, result_dst, op[2], op[1], op[0]);
2252 break;
2253 case ir_triop_csel:
2254 if (this->ctx->Const.NativeIntegers)
2255 emit_asm(ir, TGSI_OPCODE_UCMP, result_dst, op[0], op[1], op[2]);
2256 else {
2257 op[0].negate = ~op[0].negate;
2258 emit_asm(ir, TGSI_OPCODE_CMP, result_dst, op[0], op[1], op[2]);
2259 }
2260 break;
2261 case ir_triop_bitfield_extract:
2262 emit_asm(ir, TGSI_OPCODE_IBFE, result_dst, op[0], op[1], op[2]);
2263 break;
2264 case ir_quadop_bitfield_insert:
2265 emit_asm(ir, TGSI_OPCODE_BFI, result_dst, op[0], op[1], op[2], op[3]);
2266 break;
2267 case ir_unop_bitfield_reverse:
2268 emit_asm(ir, TGSI_OPCODE_BREV, result_dst, op[0]);
2269 break;
2270 case ir_unop_bit_count:
2271 emit_asm(ir, TGSI_OPCODE_POPC, result_dst, op[0]);
2272 break;
2273 case ir_unop_find_msb:
2274 emit_asm(ir, TGSI_OPCODE_IMSB, result_dst, op[0]);
2275 break;
2276 case ir_unop_find_lsb:
2277 emit_asm(ir, TGSI_OPCODE_LSB, result_dst, op[0]);
2278 break;
2279 case ir_binop_imul_high:
2280 emit_asm(ir, TGSI_OPCODE_IMUL_HI, result_dst, op[0], op[1]);
2281 break;
2282 case ir_triop_fma:
2283 /* In theory, MAD is incorrect here. */
2284 if (have_fma)
2285 emit_asm(ir, TGSI_OPCODE_FMA, result_dst, op[0], op[1], op[2]);
2286 else
2287 emit_asm(ir, TGSI_OPCODE_MAD, result_dst, op[0], op[1], op[2]);
2288 break;
2289 case ir_unop_interpolate_at_centroid:
2290 emit_asm(ir, TGSI_OPCODE_INTERP_CENTROID, result_dst, op[0]);
2291 break;
2292 case ir_binop_interpolate_at_offset: {
2293 /* The y coordinate needs to be flipped for the default fb */
2294 static const gl_state_index transform_y_state[STATE_LENGTH]
2295 = { STATE_INTERNAL, STATE_FB_WPOS_Y_TRANSFORM };
2296
2297 unsigned transform_y_index =
2298 _mesa_add_state_reference(this->prog->Parameters,
2299 transform_y_state);
2300
2301 st_src_reg transform_y = st_src_reg(PROGRAM_STATE_VAR,
2302 transform_y_index,
2303 glsl_type::vec4_type);
2304 transform_y.swizzle = SWIZZLE_XXXX;
2305
2306 st_src_reg temp = get_temp(glsl_type::vec2_type);
2307 st_dst_reg temp_dst = st_dst_reg(temp);
2308
2309 emit_asm(ir, TGSI_OPCODE_MOV, temp_dst, op[1]);
2310 temp_dst.writemask = WRITEMASK_Y;
2311 emit_asm(ir, TGSI_OPCODE_MUL, temp_dst, transform_y, op[1]);
2312 emit_asm(ir, TGSI_OPCODE_INTERP_OFFSET, result_dst, op[0], temp);
2313 break;
2314 }
2315 case ir_binop_interpolate_at_sample:
2316 emit_asm(ir, TGSI_OPCODE_INTERP_SAMPLE, result_dst, op[0], op[1]);
2317 break;
2318
2319 case ir_unop_d2f:
2320 emit_asm(ir, TGSI_OPCODE_D2F, result_dst, op[0]);
2321 break;
2322 case ir_unop_f2d:
2323 emit_asm(ir, TGSI_OPCODE_F2D, result_dst, op[0]);
2324 break;
2325 case ir_unop_d2i:
2326 emit_asm(ir, TGSI_OPCODE_D2I, result_dst, op[0]);
2327 break;
2328 case ir_unop_i2d:
2329 emit_asm(ir, TGSI_OPCODE_I2D, result_dst, op[0]);
2330 break;
2331 case ir_unop_d2u:
2332 emit_asm(ir, TGSI_OPCODE_D2U, result_dst, op[0]);
2333 break;
2334 case ir_unop_u2d:
2335 emit_asm(ir, TGSI_OPCODE_U2D, result_dst, op[0]);
2336 break;
2337 case ir_unop_unpack_double_2x32:
2338 case ir_unop_pack_double_2x32:
2339 case ir_unop_unpack_int_2x32:
2340 case ir_unop_pack_int_2x32:
2341 case ir_unop_unpack_uint_2x32:
2342 case ir_unop_pack_uint_2x32:
2343 case ir_unop_unpack_sampler_2x32:
2344 case ir_unop_pack_sampler_2x32:
2345 case ir_unop_unpack_image_2x32:
2346 case ir_unop_pack_image_2x32:
2347 emit_asm(ir, TGSI_OPCODE_MOV, result_dst, op[0]);
2348 break;
2349
2350 case ir_binop_ldexp:
2351 if (ir->operands[0]->type->is_double()) {
2352 emit_asm(ir, TGSI_OPCODE_DLDEXP, result_dst, op[0], op[1]);
2353 } else {
2354 assert(!"Invalid ldexp for non-double opcode in glsl_to_tgsi_visitor::visit()");
2355 }
2356 break;
2357
2358 case ir_unop_pack_half_2x16:
2359 emit_asm(ir, TGSI_OPCODE_PK2H, result_dst, op[0]);
2360 break;
2361 case ir_unop_unpack_half_2x16:
2362 emit_asm(ir, TGSI_OPCODE_UP2H, result_dst, op[0]);
2363 break;
2364
2365 case ir_unop_get_buffer_size: {
2366 ir_constant *const_offset = ir->operands[0]->as_constant();
2367 st_src_reg buffer(
2368 PROGRAM_BUFFER,
2369 ctx->Const.Program[shader->Stage].MaxAtomicBuffers +
2370 (const_offset ? const_offset->value.u[0] : 0),
2371 GLSL_TYPE_UINT);
2372 if (!const_offset) {
2373 buffer.reladdr = ralloc(mem_ctx, st_src_reg);
2374 *buffer.reladdr = op[0];
2375 emit_arl(ir, sampler_reladdr, op[0]);
2376 }
2377 emit_asm(ir, TGSI_OPCODE_RESQ, result_dst)->resource = buffer;
2378 break;
2379 }
2380
2381 case ir_unop_u2i64:
2382 case ir_unop_u2u64:
2383 case ir_unop_b2i64: {
2384 st_src_reg temp = get_temp(glsl_type::uvec4_type);
2385 st_dst_reg temp_dst = st_dst_reg(temp);
2386 unsigned orig_swz = op[0].swizzle;
2387 /*
2388 * To convert unsigned to 64-bit:
2389 * zero Y channel, copy X channel.
2390 */
2391 temp_dst.writemask = WRITEMASK_Y;
2392 if (vector_elements > 1)
2393 temp_dst.writemask |= WRITEMASK_W;
2394 emit_asm(ir, TGSI_OPCODE_MOV, temp_dst, st_src_reg_for_int(0));
2395 temp_dst.writemask = WRITEMASK_X;
2396 if (vector_elements > 1)
2397 temp_dst.writemask |= WRITEMASK_Z;
2398 op[0].swizzle = MAKE_SWIZZLE4(GET_SWZ(orig_swz, 0), GET_SWZ(orig_swz, 0),
2399 GET_SWZ(orig_swz, 1), GET_SWZ(orig_swz, 1));
2400 if (ir->operation == ir_unop_u2i64 || ir->operation == ir_unop_u2u64)
2401 emit_asm(ir, TGSI_OPCODE_MOV, temp_dst, op[0]);
2402 else
2403 emit_asm(ir, TGSI_OPCODE_AND, temp_dst, op[0], st_src_reg_for_int(1));
2404 result_src = temp;
2405 result_src.type = GLSL_TYPE_UINT64;
2406 if (vector_elements > 2) {
2407 /* Subtle: We rely on the fact that get_temp here returns the next
2408 * TGSI temporary register directly after the temp register used for
2409 * the first two components, so that the result gets picked up
2410 * automatically.
2411 */
2412 st_src_reg temp = get_temp(glsl_type::uvec4_type);
2413 st_dst_reg temp_dst = st_dst_reg(temp);
2414 temp_dst.writemask = WRITEMASK_Y;
2415 if (vector_elements > 3)
2416 temp_dst.writemask |= WRITEMASK_W;
2417 emit_asm(ir, TGSI_OPCODE_MOV, temp_dst, st_src_reg_for_int(0));
2418
2419 temp_dst.writemask = WRITEMASK_X;
2420 if (vector_elements > 3)
2421 temp_dst.writemask |= WRITEMASK_Z;
2422 op[0].swizzle = MAKE_SWIZZLE4(GET_SWZ(orig_swz, 2), GET_SWZ(orig_swz, 2),
2423 GET_SWZ(orig_swz, 3), GET_SWZ(orig_swz, 3));
2424 if (ir->operation == ir_unop_u2i64 || ir->operation == ir_unop_u2u64)
2425 emit_asm(ir, TGSI_OPCODE_MOV, temp_dst, op[0]);
2426 else
2427 emit_asm(ir, TGSI_OPCODE_AND, temp_dst, op[0], st_src_reg_for_int(1));
2428 }
2429 break;
2430 }
2431 case ir_unop_i642i:
2432 case ir_unop_u642i:
2433 case ir_unop_u642u:
2434 case ir_unop_i642u: {
2435 st_src_reg temp = get_temp(glsl_type::uvec4_type);
2436 st_dst_reg temp_dst = st_dst_reg(temp);
2437 unsigned orig_swz = op[0].swizzle;
2438 unsigned orig_idx = op[0].index;
2439 int el;
2440 temp_dst.writemask = WRITEMASK_X;
2441
2442 for (el = 0; el < vector_elements; el++) {
2443 unsigned swz = GET_SWZ(orig_swz, el);
2444 if (swz & 1)
2445 op[0].swizzle = MAKE_SWIZZLE4(SWIZZLE_Z, SWIZZLE_Z, SWIZZLE_Z, SWIZZLE_Z);
2446 else
2447 op[0].swizzle = MAKE_SWIZZLE4(SWIZZLE_X, SWIZZLE_X, SWIZZLE_X, SWIZZLE_X);
2448 if (swz > 2)
2449 op[0].index = orig_idx + 1;
2450 op[0].type = GLSL_TYPE_UINT;
2451 temp_dst.writemask = WRITEMASK_X << el;
2452 emit_asm(ir, TGSI_OPCODE_MOV, temp_dst, op[0]);
2453 }
2454 result_src = temp;
2455 if (ir->operation == ir_unop_u642u || ir->operation == ir_unop_i642u)
2456 result_src.type = GLSL_TYPE_UINT;
2457 else
2458 result_src.type = GLSL_TYPE_INT;
2459 break;
2460 }
2461 case ir_unop_i642b:
2462 emit_asm(ir, TGSI_OPCODE_U64SNE, result_dst, op[0], st_src_reg_for_int(0));
2463 break;
2464 case ir_unop_i642f:
2465 emit_asm(ir, TGSI_OPCODE_I642F, result_dst, op[0]);
2466 break;
2467 case ir_unop_u642f:
2468 emit_asm(ir, TGSI_OPCODE_U642F, result_dst, op[0]);
2469 break;
2470 case ir_unop_i642d:
2471 emit_asm(ir, TGSI_OPCODE_I642D, result_dst, op[0]);
2472 break;
2473 case ir_unop_u642d:
2474 emit_asm(ir, TGSI_OPCODE_U642D, result_dst, op[0]);
2475 break;
2476 case ir_unop_i2i64:
2477 emit_asm(ir, TGSI_OPCODE_I2I64, result_dst, op[0]);
2478 break;
2479 case ir_unop_f2i64:
2480 emit_asm(ir, TGSI_OPCODE_F2I64, result_dst, op[0]);
2481 break;
2482 case ir_unop_d2i64:
2483 emit_asm(ir, TGSI_OPCODE_D2I64, result_dst, op[0]);
2484 break;
2485 case ir_unop_i2u64:
2486 emit_asm(ir, TGSI_OPCODE_I2I64, result_dst, op[0]);
2487 break;
2488 case ir_unop_f2u64:
2489 emit_asm(ir, TGSI_OPCODE_F2U64, result_dst, op[0]);
2490 break;
2491 case ir_unop_d2u64:
2492 emit_asm(ir, TGSI_OPCODE_D2U64, result_dst, op[0]);
2493 break;
2494 /* these might be needed */
2495 case ir_unop_pack_snorm_2x16:
2496 case ir_unop_pack_unorm_2x16:
2497 case ir_unop_pack_snorm_4x8:
2498 case ir_unop_pack_unorm_4x8:
2499
2500 case ir_unop_unpack_snorm_2x16:
2501 case ir_unop_unpack_unorm_2x16:
2502 case ir_unop_unpack_snorm_4x8:
2503 case ir_unop_unpack_unorm_4x8:
2504
2505 case ir_quadop_vector:
2506 case ir_binop_vector_extract:
2507 case ir_triop_vector_insert:
2508 case ir_binop_carry:
2509 case ir_binop_borrow:
2510 case ir_unop_ssbo_unsized_array_length:
2511 /* This operation is not supported, or should have already been handled.
2512 */
2513 assert(!"Invalid ir opcode in glsl_to_tgsi_visitor::visit()");
2514 break;
2515 }
2516
2517 this->result = result_src;
2518 }
2519
2520
2521 void
2522 glsl_to_tgsi_visitor::visit(ir_swizzle *ir)
2523 {
2524 st_src_reg src;
2525 int i;
2526 int swizzle[4];
2527
2528 /* Note that this is only swizzles in expressions, not those on the left
2529 * hand side of an assignment, which do write masking. See ir_assignment
2530 * for that.
2531 */
2532
2533 ir->val->accept(this);
2534 src = this->result;
2535 assert(src.file != PROGRAM_UNDEFINED);
2536 assert(ir->type->vector_elements > 0);
2537
2538 for (i = 0; i < 4; i++) {
2539 if (i < ir->type->vector_elements) {
2540 switch (i) {
2541 case 0:
2542 swizzle[i] = GET_SWZ(src.swizzle, ir->mask.x);
2543 break;
2544 case 1:
2545 swizzle[i] = GET_SWZ(src.swizzle, ir->mask.y);
2546 break;
2547 case 2:
2548 swizzle[i] = GET_SWZ(src.swizzle, ir->mask.z);
2549 break;
2550 case 3:
2551 swizzle[i] = GET_SWZ(src.swizzle, ir->mask.w);
2552 break;
2553 }
2554 } else {
2555 /* If the type is smaller than a vec4, replicate the last
2556 * channel out.
2557 */
2558 swizzle[i] = swizzle[ir->type->vector_elements - 1];
2559 }
2560 }
2561
2562 src.swizzle = MAKE_SWIZZLE4(swizzle[0], swizzle[1], swizzle[2], swizzle[3]);
2563
2564 this->result = src;
2565 }
2566
2567 /* Test if the variable is an array. Note that geometry and
2568 * tessellation shader inputs are outputs are always arrays (except
2569 * for patch inputs), so only the array element type is considered.
2570 */
2571 static bool
2572 is_inout_array(unsigned stage, ir_variable *var, bool *remove_array)
2573 {
2574 const glsl_type *type = var->type;
2575
2576 *remove_array = false;
2577
2578 if ((stage == MESA_SHADER_VERTEX && var->data.mode == ir_var_shader_in) ||
2579 (stage == MESA_SHADER_FRAGMENT && var->data.mode == ir_var_shader_out))
2580 return false;
2581
2582 if (((stage == MESA_SHADER_GEOMETRY && var->data.mode == ir_var_shader_in) ||
2583 (stage == MESA_SHADER_TESS_EVAL && var->data.mode == ir_var_shader_in) ||
2584 stage == MESA_SHADER_TESS_CTRL) &&
2585 !var->data.patch) {
2586 if (!var->type->is_array())
2587 return false; /* a system value probably */
2588
2589 type = var->type->fields.array;
2590 *remove_array = true;
2591 }
2592
2593 return type->is_array() || type->is_matrix();
2594 }
2595
2596 static unsigned
2597 st_translate_interp_loc(ir_variable *var)
2598 {
2599 if (var->data.centroid)
2600 return TGSI_INTERPOLATE_LOC_CENTROID;
2601 else if (var->data.sample)
2602 return TGSI_INTERPOLATE_LOC_SAMPLE;
2603 else
2604 return TGSI_INTERPOLATE_LOC_CENTER;
2605 }
2606
2607 void
2608 glsl_to_tgsi_visitor::visit(ir_dereference_variable *ir)
2609 {
2610 variable_storage *entry = find_variable_storage(ir->var);
2611 ir_variable *var = ir->var;
2612 bool remove_array;
2613
2614 if (!entry) {
2615 switch (var->data.mode) {
2616 case ir_var_uniform:
2617 entry = new(mem_ctx) variable_storage(var, PROGRAM_UNIFORM,
2618 var->data.param_index);
2619 _mesa_hash_table_insert(this->variables, var, entry);
2620 break;
2621 case ir_var_shader_in: {
2622 /* The linker assigns locations for varyings and attributes,
2623 * including deprecated builtins (like gl_Color), user-assign
2624 * generic attributes (glBindVertexLocation), and
2625 * user-defined varyings.
2626 */
2627 assert(var->data.location != -1);
2628
2629 const glsl_type *type_without_array = var->type->without_array();
2630 struct inout_decl *decl = &inputs[num_inputs];
2631 unsigned component = var->data.location_frac;
2632 unsigned num_components;
2633 num_inputs++;
2634
2635 if (type_without_array->is_64bit())
2636 component = component / 2;
2637 if (type_without_array->vector_elements)
2638 num_components = type_without_array->vector_elements;
2639 else
2640 num_components = 4;
2641
2642 decl->mesa_index = var->data.location;
2643 decl->interp = (glsl_interp_mode) var->data.interpolation;
2644 decl->interp_loc = st_translate_interp_loc(var);
2645 decl->base_type = type_without_array->base_type;
2646 decl->usage_mask = u_bit_consecutive(component, num_components);
2647
2648 if (is_inout_array(shader->Stage, var, &remove_array)) {
2649 decl->array_id = num_input_arrays + 1;
2650 num_input_arrays++;
2651 } else {
2652 decl->array_id = 0;
2653 }
2654
2655 if (remove_array)
2656 decl->size = type_size(var->type->fields.array);
2657 else
2658 decl->size = type_size(var->type);
2659
2660 entry = new(mem_ctx) variable_storage(var,
2661 PROGRAM_INPUT,
2662 decl->mesa_index,
2663 decl->array_id);
2664 entry->component = component;
2665
2666 _mesa_hash_table_insert(this->variables, var, entry);
2667
2668 break;
2669 }
2670 case ir_var_shader_out: {
2671 assert(var->data.location != -1);
2672
2673 const glsl_type *type_without_array = var->type->without_array();
2674 struct inout_decl *decl = &outputs[num_outputs];
2675 unsigned component = var->data.location_frac;
2676 unsigned num_components;
2677 num_outputs++;
2678
2679 if (type_without_array->is_64bit())
2680 component = component / 2;
2681 if (type_without_array->vector_elements)
2682 num_components = type_without_array->vector_elements;
2683 else
2684 num_components = 4;
2685
2686 decl->mesa_index = var->data.location + FRAG_RESULT_MAX * var->data.index;
2687 decl->base_type = type_without_array->base_type;
2688 decl->usage_mask = u_bit_consecutive(component, num_components);
2689 if (var->data.stream & (1u << 31)) {
2690 decl->gs_out_streams = var->data.stream & ~(1u << 31);
2691 } else {
2692 assert(var->data.stream < 4);
2693 decl->gs_out_streams = 0;
2694 for (unsigned i = 0; i < num_components; ++i)
2695 decl->gs_out_streams |= var->data.stream << (2 * (component + i));
2696 }
2697
2698 if (is_inout_array(shader->Stage, var, &remove_array)) {
2699 decl->array_id = num_output_arrays + 1;
2700 num_output_arrays++;
2701 } else {
2702 decl->array_id = 0;
2703 }
2704
2705 if (remove_array)
2706 decl->size = type_size(var->type->fields.array);
2707 else
2708 decl->size = type_size(var->type);
2709
2710 if (var->data.fb_fetch_output) {
2711 st_dst_reg dst = st_dst_reg(get_temp(var->type));
2712 st_src_reg src = st_src_reg(PROGRAM_OUTPUT, decl->mesa_index,
2713 var->type, component, decl->array_id);
2714 emit_asm(NULL, TGSI_OPCODE_FBFETCH, dst, src);
2715 entry = new(mem_ctx) variable_storage(var, dst.file, dst.index,
2716 dst.array_id);
2717 } else {
2718 entry = new(mem_ctx) variable_storage(var,
2719 PROGRAM_OUTPUT,
2720 decl->mesa_index,
2721 decl->array_id);
2722 }
2723 entry->component = component;
2724
2725 _mesa_hash_table_insert(this->variables, var, entry);
2726
2727 break;
2728 }
2729 case ir_var_system_value:
2730 entry = new(mem_ctx) variable_storage(var,
2731 PROGRAM_SYSTEM_VALUE,
2732 var->data.location);
2733 break;
2734 case ir_var_auto:
2735 case ir_var_temporary:
2736 st_src_reg src = get_temp(var->type);
2737
2738 entry = new(mem_ctx) variable_storage(var, src.file, src.index,
2739 src.array_id);
2740 _mesa_hash_table_insert(this->variables, var, entry);
2741
2742 break;
2743 }
2744
2745 if (!entry) {
2746 printf("Failed to make storage for %s\n", var->name);
2747 exit(1);
2748 }
2749 }
2750
2751 this->result = st_src_reg(entry->file, entry->index, var->type,
2752 entry->component, entry->array_id);
2753 if (this->shader->Stage == MESA_SHADER_VERTEX && var->data.mode == ir_var_shader_in && var->type->is_double())
2754 this->result.is_double_vertex_input = true;
2755 if (!native_integers)
2756 this->result.type = GLSL_TYPE_FLOAT;
2757 }
2758
2759 static void
2760 shrink_array_declarations(struct inout_decl *decls, unsigned count,
2761 GLbitfield64* usage_mask,
2762 GLbitfield64 double_usage_mask,
2763 GLbitfield* patch_usage_mask)
2764 {
2765 unsigned i;
2766 int j;
2767
2768 /* Fix array declarations by removing unused array elements at both ends
2769 * of the arrays. For example, mat4[3] where only mat[1] is used.
2770 */
2771 for (i = 0; i < count; i++) {
2772 struct inout_decl *decl = &decls[i];
2773 if (!decl->array_id)
2774 continue;
2775
2776 /* Shrink the beginning. */
2777 for (j = 0; j < (int)decl->size; j++) {
2778 if (decl->mesa_index >= VARYING_SLOT_PATCH0) {
2779 if (*patch_usage_mask &
2780 BITFIELD64_BIT(decl->mesa_index - VARYING_SLOT_PATCH0 + j))
2781 break;
2782 }
2783 else {
2784 if (*usage_mask & BITFIELD64_BIT(decl->mesa_index+j))
2785 break;
2786 if (double_usage_mask & BITFIELD64_BIT(decl->mesa_index+j-1))
2787 break;
2788 }
2789
2790 decl->mesa_index++;
2791 decl->size--;
2792 j--;
2793 }
2794
2795 /* Shrink the end. */
2796 for (j = decl->size-1; j >= 0; j--) {
2797 if (decl->mesa_index >= VARYING_SLOT_PATCH0) {
2798 if (*patch_usage_mask &
2799 BITFIELD64_BIT(decl->mesa_index - VARYING_SLOT_PATCH0 + j))
2800 break;
2801 }
2802 else {
2803 if (*usage_mask & BITFIELD64_BIT(decl->mesa_index+j))
2804 break;
2805 if (double_usage_mask & BITFIELD64_BIT(decl->mesa_index+j-1))
2806 break;
2807 }
2808
2809 decl->size--;
2810 }
2811
2812 /* When not all entries of an array are accessed, we mark them as used
2813 * here anyway, to ensure that the input/output mapping logic doesn't get
2814 * confused.
2815 *
2816 * TODO This happens when an array isn't used via indirect access, which
2817 * some game ports do (at least eON-based). There is an optimization
2818 * opportunity here by replacing the array declaration with non-array
2819 * declarations of those slots that are actually used.
2820 */
2821 for (j = 1; j < (int)decl->size; ++j) {
2822 if (decl->mesa_index >= VARYING_SLOT_PATCH0)
2823 *patch_usage_mask |= BITFIELD64_BIT(decl->mesa_index - VARYING_SLOT_PATCH0 + j);
2824 else
2825 *usage_mask |= BITFIELD64_BIT(decl->mesa_index + j);
2826 }
2827 }
2828 }
2829
2830 void
2831 glsl_to_tgsi_visitor::visit(ir_dereference_array *ir)
2832 {
2833 ir_constant *index;
2834 st_src_reg src;
2835 int element_size = type_size(ir->type);
2836 bool is_2D = false;
2837
2838 index = ir->array_index->constant_expression_value(ralloc_parent(ir));
2839
2840 ir->array->accept(this);
2841 src = this->result;
2842
2843 if (ir->array->ir_type != ir_type_dereference_array) {
2844 switch (this->prog->Target) {
2845 case GL_TESS_CONTROL_PROGRAM_NV:
2846 is_2D = (src.file == PROGRAM_INPUT || src.file == PROGRAM_OUTPUT) &&
2847 !ir->variable_referenced()->data.patch;
2848 break;
2849 case GL_TESS_EVALUATION_PROGRAM_NV:
2850 is_2D = src.file == PROGRAM_INPUT &&
2851 !ir->variable_referenced()->data.patch;
2852 break;
2853 case GL_GEOMETRY_PROGRAM_NV:
2854 is_2D = src.file == PROGRAM_INPUT;
2855 break;
2856 }
2857 }
2858
2859 if (is_2D)
2860 element_size = 1;
2861
2862 if (index) {
2863
2864 if (this->prog->Target == GL_VERTEX_PROGRAM_ARB &&
2865 src.file == PROGRAM_INPUT)
2866 element_size = attrib_type_size(ir->type, true);
2867 if (is_2D) {
2868 src.index2D = index->value.i[0];
2869 src.has_index2 = true;
2870 } else
2871 src.index += index->value.i[0] * element_size;
2872 } else {
2873 /* Variable index array dereference. It eats the "vec4" of the
2874 * base of the array and an index that offsets the TGSI register
2875 * index.
2876 */
2877 ir->array_index->accept(this);
2878
2879 st_src_reg index_reg;
2880
2881 if (element_size == 1) {
2882 index_reg = this->result;
2883 } else {
2884 index_reg = get_temp(native_integers ?
2885 glsl_type::int_type : glsl_type::float_type);
2886
2887 emit_asm(ir, TGSI_OPCODE_MUL, st_dst_reg(index_reg),
2888 this->result, st_src_reg_for_type(index_reg.type, element_size));
2889 }
2890
2891 /* If there was already a relative address register involved, add the
2892 * new and the old together to get the new offset.
2893 */
2894 if (!is_2D && src.reladdr != NULL) {
2895 st_src_reg accum_reg = get_temp(native_integers ?
2896 glsl_type::int_type : glsl_type::float_type);
2897
2898 emit_asm(ir, TGSI_OPCODE_ADD, st_dst_reg(accum_reg),
2899 index_reg, *src.reladdr);
2900
2901 index_reg = accum_reg;
2902 }
2903
2904 if (is_2D) {
2905 src.reladdr2 = ralloc(mem_ctx, st_src_reg);
2906 memcpy(src.reladdr2, &index_reg, sizeof(index_reg));
2907 src.index2D = 0;
2908 src.has_index2 = true;
2909 } else {
2910 src.reladdr = ralloc(mem_ctx, st_src_reg);
2911 memcpy(src.reladdr, &index_reg, sizeof(index_reg));
2912 }
2913 }
2914
2915 /* Change the register type to the element type of the array. */
2916 src.type = ir->type->base_type;
2917
2918 this->result = src;
2919 }
2920
2921 void
2922 glsl_to_tgsi_visitor::visit(ir_dereference_record *ir)
2923 {
2924 unsigned int i;
2925 const glsl_type *struct_type = ir->record->type;
2926 int offset = 0;
2927
2928 ir->record->accept(this);
2929
2930 assert(ir->field_idx >= 0);
2931 for (i = 0; i < struct_type->length; i++) {
2932 if (i == (unsigned) ir->field_idx)
2933 break;
2934 offset += type_size(struct_type->fields.structure[i].type);
2935 }
2936
2937 /* If the type is smaller than a vec4, replicate the last channel out. */
2938 if (ir->type->is_scalar() || ir->type->is_vector())
2939 this->result.swizzle = swizzle_for_size(ir->type->vector_elements);
2940 else
2941 this->result.swizzle = SWIZZLE_NOOP;
2942
2943 this->result.index += offset;
2944 this->result.type = ir->type->base_type;
2945 }
2946
2947 /**
2948 * We want to be careful in assignment setup to hit the actual storage
2949 * instead of potentially using a temporary like we might with the
2950 * ir_dereference handler.
2951 */
2952 static st_dst_reg
2953 get_assignment_lhs(ir_dereference *ir, glsl_to_tgsi_visitor *v, int *component)
2954 {
2955 /* The LHS must be a dereference. If the LHS is a variable indexed array
2956 * access of a vector, it must be separated into a series conditional moves
2957 * before reaching this point (see ir_vec_index_to_cond_assign).
2958 */
2959 assert(ir->as_dereference());
2960 ir_dereference_array *deref_array = ir->as_dereference_array();
2961 if (deref_array) {
2962 assert(!deref_array->array->type->is_vector());
2963 }
2964
2965 /* Use the rvalue deref handler for the most part. We write swizzles using
2966 * the writemask, but we do extract the base component for enhanced layouts
2967 * from the source swizzle.
2968 */
2969 ir->accept(v);
2970 *component = GET_SWZ(v->result.swizzle, 0);
2971 return st_dst_reg(v->result);
2972 }
2973
2974 /**
2975 * Process the condition of a conditional assignment
2976 *
2977 * Examines the condition of a conditional assignment to generate the optimal
2978 * first operand of a \c CMP instruction. If the condition is a relational
2979 * operator with 0 (e.g., \c ir_binop_less), the value being compared will be
2980 * used as the source for the \c CMP instruction. Otherwise the comparison
2981 * is processed to a boolean result, and the boolean result is used as the
2982 * operand to the CMP instruction.
2983 */
2984 bool
2985 glsl_to_tgsi_visitor::process_move_condition(ir_rvalue *ir)
2986 {
2987 ir_rvalue *src_ir = ir;
2988 bool negate = true;
2989 bool switch_order = false;
2990
2991 ir_expression *const expr = ir->as_expression();
2992
2993 if (native_integers) {
2994 if ((expr != NULL) && (expr->num_operands == 2)) {
2995 enum glsl_base_type type = expr->operands[0]->type->base_type;
2996 if (type == GLSL_TYPE_INT || type == GLSL_TYPE_UINT ||
2997 type == GLSL_TYPE_BOOL) {
2998 if (expr->operation == ir_binop_equal) {
2999 if (expr->operands[0]->is_zero()) {
3000 src_ir = expr->operands[1];
3001 switch_order = true;
3002 }
3003 else if (expr->operands[1]->is_zero()) {
3004 src_ir = expr->operands[0];
3005 switch_order = true;
3006 }
3007 }
3008 else if (expr->operation == ir_binop_nequal) {
3009 if (expr->operands[0]->is_zero()) {
3010 src_ir = expr->operands[1];
3011 }
3012 else if (expr->operands[1]->is_zero()) {
3013 src_ir = expr->operands[0];
3014 }
3015 }
3016 }
3017 }
3018
3019 src_ir->accept(this);
3020 return switch_order;
3021 }
3022
3023 if ((expr != NULL) && (expr->num_operands == 2)) {
3024 bool zero_on_left = false;
3025
3026 if (expr->operands[0]->is_zero()) {
3027 src_ir = expr->operands[1];
3028 zero_on_left = true;
3029 } else if (expr->operands[1]->is_zero()) {
3030 src_ir = expr->operands[0];
3031 zero_on_left = false;
3032 }
3033
3034 /* a is - 0 + - 0 +
3035 * (a < 0) T F F ( a < 0) T F F
3036 * (0 < a) F F T (-a < 0) F F T
3037 * (a <= 0) T T F (-a < 0) F F T (swap order of other operands)
3038 * (0 <= a) F T T ( a < 0) T F F (swap order of other operands)
3039 * (a > 0) F F T (-a < 0) F F T
3040 * (0 > a) T F F ( a < 0) T F F
3041 * (a >= 0) F T T ( a < 0) T F F (swap order of other operands)
3042 * (0 >= a) T T F (-a < 0) F F T (swap order of other operands)
3043 *
3044 * Note that exchanging the order of 0 and 'a' in the comparison simply
3045 * means that the value of 'a' should be negated.
3046 */
3047 if (src_ir != ir) {
3048 switch (expr->operation) {
3049 case ir_binop_less:
3050 switch_order = false;
3051 negate = zero_on_left;
3052 break;
3053
3054 case ir_binop_greater:
3055 switch_order = false;
3056 negate = !zero_on_left;
3057 break;
3058
3059 case ir_binop_lequal:
3060 switch_order = true;
3061 negate = !zero_on_left;
3062 break;
3063
3064 case ir_binop_gequal:
3065 switch_order = true;
3066 negate = zero_on_left;
3067 break;
3068
3069 default:
3070 /* This isn't the right kind of comparison afterall, so make sure
3071 * the whole condition is visited.
3072 */
3073 src_ir = ir;
3074 break;
3075 }
3076 }
3077 }
3078
3079 src_ir->accept(this);
3080
3081 /* We use the TGSI_OPCODE_CMP (a < 0 ? b : c) for conditional moves, and the
3082 * condition we produced is 0.0 or 1.0. By flipping the sign, we can
3083 * choose which value TGSI_OPCODE_CMP produces without an extra instruction
3084 * computing the condition.
3085 */
3086 if (negate)
3087 this->result.negate = ~this->result.negate;
3088
3089 return switch_order;
3090 }
3091
3092 void
3093 glsl_to_tgsi_visitor::emit_block_mov(ir_assignment *ir, const struct glsl_type *type,
3094 st_dst_reg *l, st_src_reg *r,
3095 st_src_reg *cond, bool cond_swap)
3096 {
3097 if (type->is_record()) {
3098 for (unsigned int i = 0; i < type->length; i++) {
3099 emit_block_mov(ir, type->fields.structure[i].type, l, r,
3100 cond, cond_swap);
3101 }
3102 return;
3103 }
3104
3105 if (type->is_array()) {
3106 for (unsigned int i = 0; i < type->length; i++) {
3107 emit_block_mov(ir, type->fields.array, l, r, cond, cond_swap);
3108 }
3109 return;
3110 }
3111
3112 if (type->is_matrix()) {
3113 const struct glsl_type *vec_type;
3114
3115 vec_type = glsl_type::get_instance(type->is_double() ? GLSL_TYPE_DOUBLE : GLSL_TYPE_FLOAT,
3116 type->vector_elements, 1);
3117
3118 for (int i = 0; i < type->matrix_columns; i++) {
3119 emit_block_mov(ir, vec_type, l, r, cond, cond_swap);
3120 }
3121 return;
3122 }
3123
3124 assert(type->is_scalar() || type->is_vector());
3125
3126 l->type = type->base_type;
3127 r->type = type->base_type;
3128 if (cond) {
3129 st_src_reg l_src = st_src_reg(*l);
3130 l_src.swizzle = swizzle_for_size(type->vector_elements);
3131
3132 if (native_integers) {
3133 emit_asm(ir, TGSI_OPCODE_UCMP, *l, *cond,
3134 cond_swap ? l_src : *r,
3135 cond_swap ? *r : l_src);
3136 } else {
3137 emit_asm(ir, TGSI_OPCODE_CMP, *l, *cond,
3138 cond_swap ? l_src : *r,
3139 cond_swap ? *r : l_src);
3140 }
3141 } else {
3142 emit_asm(ir, TGSI_OPCODE_MOV, *l, *r);
3143 }
3144 l->index++;
3145 r->index++;
3146 if (type->is_dual_slot()) {
3147 l->index++;
3148 if (r->is_double_vertex_input == false)
3149 r->index++;
3150 }
3151 }
3152
3153 void
3154 glsl_to_tgsi_visitor::visit(ir_assignment *ir)
3155 {
3156 int dst_component;
3157 st_dst_reg l;
3158 st_src_reg r;
3159
3160 /* all generated instructions need to be flaged as precise */
3161 this->precise = is_precise(ir->lhs->variable_referenced());
3162 ir->rhs->accept(this);
3163 r = this->result;
3164
3165 l = get_assignment_lhs(ir->lhs, this, &dst_component);
3166
3167 {
3168 int swizzles[4];
3169 int first_enabled_chan = 0;
3170 int rhs_chan = 0;
3171 ir_variable *variable = ir->lhs->variable_referenced();
3172
3173 if (shader->Stage == MESA_SHADER_FRAGMENT &&
3174 variable->data.mode == ir_var_shader_out &&
3175 (variable->data.location == FRAG_RESULT_DEPTH ||
3176 variable->data.location == FRAG_RESULT_STENCIL)) {
3177 assert(ir->lhs->type->is_scalar());
3178 assert(ir->write_mask == WRITEMASK_X);
3179
3180 if (variable->data.location == FRAG_RESULT_DEPTH)
3181 l.writemask = WRITEMASK_Z;
3182 else {
3183 assert(variable->data.location == FRAG_RESULT_STENCIL);
3184 l.writemask = WRITEMASK_Y;
3185 }
3186 } else if (ir->write_mask == 0) {
3187 assert(!ir->lhs->type->is_scalar() && !ir->lhs->type->is_vector());
3188
3189 unsigned num_elements = ir->lhs->type->without_array()->vector_elements;
3190
3191 if (num_elements) {
3192 l.writemask = u_bit_consecutive(0, num_elements);
3193 } else {
3194 /* The type is a struct or an array of (array of) structs. */
3195 l.writemask = WRITEMASK_XYZW;
3196 }
3197 } else {
3198 l.writemask = ir->write_mask;
3199 }
3200
3201 for (int i = 0; i < 4; i++) {
3202 if (l.writemask & (1 << i)) {
3203 first_enabled_chan = GET_SWZ(r.swizzle, i);
3204 break;
3205 }
3206 }
3207
3208 l.writemask = l.writemask << dst_component;
3209
3210 /* Swizzle a small RHS vector into the channels being written.
3211 *
3212 * glsl ir treats write_mask as dictating how many channels are
3213 * present on the RHS while TGSI treats write_mask as just
3214 * showing which channels of the vec4 RHS get written.
3215 */
3216 for (int i = 0; i < 4; i++) {
3217 if (l.writemask & (1 << i))
3218 swizzles[i] = GET_SWZ(r.swizzle, rhs_chan++);
3219 else
3220 swizzles[i] = first_enabled_chan;
3221 }
3222 r.swizzle = MAKE_SWIZZLE4(swizzles[0], swizzles[1],
3223 swizzles[2], swizzles[3]);
3224 }
3225
3226 assert(l.file != PROGRAM_UNDEFINED);
3227 assert(r.file != PROGRAM_UNDEFINED);
3228
3229 if (ir->condition) {
3230 const bool switch_order = this->process_move_condition(ir->condition);
3231 st_src_reg condition = this->result;
3232
3233 emit_block_mov(ir, ir->lhs->type, &l, &r, &condition, switch_order);
3234 } else if (ir->rhs->as_expression() &&
3235 this->instructions.get_tail() &&
3236 ir->rhs == ((glsl_to_tgsi_instruction *)this->instructions.get_tail())->ir &&
3237 !((glsl_to_tgsi_instruction *)this->instructions.get_tail())->is_64bit_expanded &&
3238 type_size(ir->lhs->type) == 1 &&
3239 l.writemask == ((glsl_to_tgsi_instruction *)this->instructions.get_tail())->dst[0].writemask) {
3240 /* To avoid emitting an extra MOV when assigning an expression to a
3241 * variable, emit the last instruction of the expression again, but
3242 * replace the destination register with the target of the assignment.
3243 * Dead code elimination will remove the original instruction.
3244 */
3245 glsl_to_tgsi_instruction *inst, *new_inst;
3246 inst = (glsl_to_tgsi_instruction *)this->instructions.get_tail();
3247 new_inst = emit_asm(ir, inst->op, l, inst->src[0], inst->src[1], inst->src[2], inst->src[3]);
3248 new_inst->saturate = inst->saturate;
3249 inst->dead_mask = inst->dst[0].writemask;
3250 } else {
3251 emit_block_mov(ir, ir->rhs->type, &l, &r, NULL, false);
3252 }
3253 this->precise = 0;
3254 }
3255
3256
3257 void
3258 glsl_to_tgsi_visitor::visit(ir_constant *ir)
3259 {
3260 st_src_reg src;
3261 GLdouble stack_vals[4] = { 0 };
3262 gl_constant_value *values = (gl_constant_value *) stack_vals;
3263 GLenum gl_type = GL_NONE;
3264 unsigned int i;
3265 static int in_array = 0;
3266 gl_register_file file = in_array ? PROGRAM_CONSTANT : PROGRAM_IMMEDIATE;
3267
3268 /* Unfortunately, 4 floats is all we can get into
3269 * _mesa_add_typed_unnamed_constant. So, make a temp to store an
3270 * aggregate constant and move each constant value into it. If we
3271 * get lucky, copy propagation will eliminate the extra moves.
3272 */
3273 if (ir->type->is_record()) {
3274 st_src_reg temp_base = get_temp(ir->type);
3275 st_dst_reg temp = st_dst_reg(temp_base);
3276
3277 foreach_in_list(ir_constant, field_value, &ir->components) {
3278 int size = type_size(field_value->type);
3279
3280 assert(size > 0);
3281
3282 field_value->accept(this);
3283 src = this->result;
3284
3285 for (i = 0; i < (unsigned int)size; i++) {
3286 emit_asm(ir, TGSI_OPCODE_MOV, temp, src);
3287
3288 src.index++;
3289 temp.index++;
3290 }
3291 }
3292 this->result = temp_base;
3293 return;
3294 }
3295
3296 if (ir->type->is_array()) {
3297 st_src_reg temp_base = get_temp(ir->type);
3298 st_dst_reg temp = st_dst_reg(temp_base);
3299 int size = type_size(ir->type->fields.array);
3300
3301 assert(size > 0);
3302 in_array++;
3303
3304 for (i = 0; i < ir->type->length; i++) {
3305 ir->array_elements[i]->accept(this);
3306 src = this->result;
3307 for (int j = 0; j < size; j++) {
3308 emit_asm(ir, TGSI_OPCODE_MOV, temp, src);
3309
3310 src.index++;
3311 temp.index++;
3312 }
3313 }
3314 this->result = temp_base;
3315 in_array--;
3316 return;
3317 }
3318
3319 if (ir->type->is_matrix()) {
3320 st_src_reg mat = get_temp(ir->type);
3321 st_dst_reg mat_column = st_dst_reg(mat);
3322
3323 for (i = 0; i < ir->type->matrix_columns; i++) {
3324 switch (ir->type->base_type) {
3325 case GLSL_TYPE_FLOAT:
3326 values = (gl_constant_value *) &ir->value.f[i * ir->type->vector_elements];
3327
3328 src = st_src_reg(file, -1, ir->type->base_type);
3329 src.index = add_constant(file,
3330 values,
3331 ir->type->vector_elements,
3332 GL_FLOAT,
3333 &src.swizzle);
3334 emit_asm(ir, TGSI_OPCODE_MOV, mat_column, src);
3335 break;
3336 case GLSL_TYPE_DOUBLE:
3337 values = (gl_constant_value *) &ir->value.d[i * ir->type->vector_elements];
3338 src = st_src_reg(file, -1, ir->type->base_type);
3339 src.index = add_constant(file,
3340 values,
3341 ir->type->vector_elements,
3342 GL_DOUBLE,
3343 &src.swizzle);
3344 if (ir->type->vector_elements >= 2) {
3345 mat_column.writemask = WRITEMASK_XY;
3346 src.swizzle = MAKE_SWIZZLE4(SWIZZLE_X, SWIZZLE_Y, SWIZZLE_X, SWIZZLE_Y);
3347 emit_asm(ir, TGSI_OPCODE_MOV, mat_column, src);
3348 } else {
3349 mat_column.writemask = WRITEMASK_X;
3350 src.swizzle = MAKE_SWIZZLE4(SWIZZLE_X, SWIZZLE_X, SWIZZLE_X, SWIZZLE_X);
3351 emit_asm(ir, TGSI_OPCODE_MOV, mat_column, src);
3352 }
3353 src.index++;
3354 if (ir->type->vector_elements > 2) {
3355 if (ir->type->vector_elements == 4) {
3356 mat_column.writemask = WRITEMASK_ZW;
3357 src.swizzle = MAKE_SWIZZLE4(SWIZZLE_X, SWIZZLE_Y, SWIZZLE_X, SWIZZLE_Y);
3358 emit_asm(ir, TGSI_OPCODE_MOV, mat_column, src);
3359 } else {
3360 mat_column.writemask = WRITEMASK_Z;
3361 src.swizzle = MAKE_SWIZZLE4(SWIZZLE_Y, SWIZZLE_Y, SWIZZLE_Y, SWIZZLE_Y);
3362 emit_asm(ir, TGSI_OPCODE_MOV, mat_column, src);
3363 mat_column.writemask = WRITEMASK_XYZW;
3364 src.swizzle = SWIZZLE_XYZW;
3365 }
3366 mat_column.index++;
3367 }
3368 break;
3369 default:
3370 unreachable("Illegal matrix constant type.\n");
3371 break;
3372 }
3373 mat_column.index++;
3374 }
3375 this->result = mat;
3376 return;
3377 }
3378
3379 switch (ir->type->base_type) {
3380 case GLSL_TYPE_FLOAT:
3381 gl_type = GL_FLOAT;
3382 for (i = 0; i < ir->type->vector_elements; i++) {
3383 values[i].f = ir->value.f[i];
3384 }
3385 break;
3386 case GLSL_TYPE_DOUBLE:
3387 gl_type = GL_DOUBLE;
3388 for (i = 0; i < ir->type->vector_elements; i++) {
3389 memcpy(&values[i * 2], &ir->value.d[i], sizeof(double));
3390 }
3391 break;
3392 case GLSL_TYPE_INT64:
3393 gl_type = GL_INT64_ARB;
3394 for (i = 0; i < ir->type->vector_elements; i++) {
3395 memcpy(&values[i * 2], &ir->value.d[i], sizeof(int64_t));
3396 }
3397 break;
3398 case GLSL_TYPE_UINT64:
3399 gl_type = GL_UNSIGNED_INT64_ARB;
3400 for (i = 0; i < ir->type->vector_elements; i++) {
3401 memcpy(&values[i * 2], &ir->value.d[i], sizeof(uint64_t));
3402 }
3403 break;
3404 case GLSL_TYPE_UINT:
3405 gl_type = native_integers ? GL_UNSIGNED_INT : GL_FLOAT;
3406 for (i = 0; i < ir->type->vector_elements; i++) {
3407 if (native_integers)
3408 values[i].u = ir->value.u[i];
3409 else
3410 values[i].f = ir->value.u[i];
3411 }
3412 break;
3413 case GLSL_TYPE_INT:
3414 gl_type = native_integers ? GL_INT : GL_FLOAT;
3415 for (i = 0; i < ir->type->vector_elements; i++) {
3416 if (native_integers)
3417 values[i].i = ir->value.i[i];
3418 else
3419 values[i].f = ir->value.i[i];
3420 }
3421 break;
3422 case GLSL_TYPE_BOOL:
3423 gl_type = native_integers ? GL_BOOL : GL_FLOAT;
3424 for (i = 0; i < ir->type->vector_elements; i++) {
3425 values[i].u = ir->value.b[i] ? ctx->Const.UniformBooleanTrue : 0;
3426 }
3427 break;
3428 default:
3429 assert(!"Non-float/uint/int/bool constant");
3430 }
3431
3432 this->result = st_src_reg(file, -1, ir->type);
3433 this->result.index = add_constant(file,
3434 values,
3435 ir->type->vector_elements,
3436 gl_type,
3437 &this->result.swizzle);
3438 }
3439
3440 void
3441 glsl_to_tgsi_visitor::visit_atomic_counter_intrinsic(ir_call *ir)
3442 {
3443 exec_node *param = ir->actual_parameters.get_head();
3444 ir_dereference *deref = static_cast<ir_dereference *>(param);
3445 ir_variable *location = deref->variable_referenced();
3446
3447 st_src_reg buffer(
3448 PROGRAM_BUFFER, location->data.binding, GLSL_TYPE_ATOMIC_UINT);
3449
3450 /* Calculate the surface offset */
3451 st_src_reg offset;
3452 unsigned array_size = 0, base = 0;
3453 uint16_t index = 0;
3454
3455 get_deref_offsets(deref, &array_size, &base, &index, &offset, false);
3456
3457 if (offset.file != PROGRAM_UNDEFINED) {
3458 emit_asm(ir, TGSI_OPCODE_MUL, st_dst_reg(offset),
3459 offset, st_src_reg_for_int(ATOMIC_COUNTER_SIZE));
3460 emit_asm(ir, TGSI_OPCODE_ADD, st_dst_reg(offset),
3461 offset, st_src_reg_for_int(location->data.offset + index * ATOMIC_COUNTER_SIZE));
3462 } else {
3463 offset = st_src_reg_for_int(location->data.offset + index * ATOMIC_COUNTER_SIZE);
3464 }
3465
3466 ir->return_deref->accept(this);
3467 st_dst_reg dst(this->result);
3468 dst.writemask = WRITEMASK_X;
3469
3470 glsl_to_tgsi_instruction *inst;
3471
3472 if (ir->callee->intrinsic_id == ir_intrinsic_atomic_counter_read) {
3473 inst = emit_asm(ir, TGSI_OPCODE_LOAD, dst, offset);
3474 } else if (ir->callee->intrinsic_id == ir_intrinsic_atomic_counter_increment) {
3475 inst = emit_asm(ir, TGSI_OPCODE_ATOMUADD, dst, offset,
3476 st_src_reg_for_int(1));
3477 } else if (ir->callee->intrinsic_id == ir_intrinsic_atomic_counter_predecrement) {
3478 inst = emit_asm(ir, TGSI_OPCODE_ATOMUADD, dst, offset,
3479 st_src_reg_for_int(-1));
3480 emit_asm(ir, TGSI_OPCODE_ADD, dst, this->result, st_src_reg_for_int(-1));
3481 } else {
3482 param = param->get_next();
3483 ir_rvalue *val = ((ir_instruction *)param)->as_rvalue();
3484 val->accept(this);
3485
3486 st_src_reg data = this->result, data2 = undef_src;
3487 unsigned opcode;
3488 switch (ir->callee->intrinsic_id) {
3489 case ir_intrinsic_atomic_counter_add:
3490 opcode = TGSI_OPCODE_ATOMUADD;
3491 break;
3492 case ir_intrinsic_atomic_counter_min:
3493 opcode = TGSI_OPCODE_ATOMIMIN;
3494 break;
3495 case ir_intrinsic_atomic_counter_max:
3496 opcode = TGSI_OPCODE_ATOMIMAX;
3497 break;
3498 case ir_intrinsic_atomic_counter_and:
3499 opcode = TGSI_OPCODE_ATOMAND;
3500 break;
3501 case ir_intrinsic_atomic_counter_or:
3502 opcode = TGSI_OPCODE_ATOMOR;
3503 break;
3504 case ir_intrinsic_atomic_counter_xor:
3505 opcode = TGSI_OPCODE_ATOMXOR;
3506 break;
3507 case ir_intrinsic_atomic_counter_exchange:
3508 opcode = TGSI_OPCODE_ATOMXCHG;
3509 break;
3510 case ir_intrinsic_atomic_counter_comp_swap: {
3511 opcode = TGSI_OPCODE_ATOMCAS;
3512 param = param->get_next();
3513 val = ((ir_instruction *)param)->as_rvalue();
3514 val->accept(this);
3515 data2 = this->result;
3516 break;
3517 }
3518 default:
3519 assert(!"Unexpected intrinsic");
3520 return;
3521 }
3522
3523 inst = emit_asm(ir, opcode, dst, offset, data, data2);
3524 }
3525
3526 inst->resource = buffer;
3527 }
3528
3529 void
3530 glsl_to_tgsi_visitor::visit_ssbo_intrinsic(ir_call *ir)
3531 {
3532 exec_node *param = ir->actual_parameters.get_head();
3533
3534 ir_rvalue *block = ((ir_instruction *)param)->as_rvalue();
3535
3536 param = param->get_next();
3537 ir_rvalue *offset = ((ir_instruction *)param)->as_rvalue();
3538
3539 ir_constant *const_block = block->as_constant();
3540
3541 st_src_reg buffer(
3542 PROGRAM_BUFFER,
3543 ctx->Const.Program[shader->Stage].MaxAtomicBuffers +
3544 (const_block ? const_block->value.u[0] : 0),
3545 GLSL_TYPE_UINT);
3546
3547 if (!const_block) {
3548 block->accept(this);
3549 buffer.reladdr = ralloc(mem_ctx, st_src_reg);
3550 *buffer.reladdr = this->result;
3551 emit_arl(ir, sampler_reladdr, this->result);
3552 }
3553
3554 /* Calculate the surface offset */
3555 offset->accept(this);
3556 st_src_reg off = this->result;
3557
3558 st_dst_reg dst = undef_dst;
3559 if (ir->return_deref) {
3560 ir->return_deref->accept(this);
3561 dst = st_dst_reg(this->result);
3562 dst.writemask = (1 << ir->return_deref->type->vector_elements) - 1;
3563 }
3564
3565 glsl_to_tgsi_instruction *inst;
3566
3567 if (ir->callee->intrinsic_id == ir_intrinsic_ssbo_load) {
3568 inst = emit_asm(ir, TGSI_OPCODE_LOAD, dst, off);
3569 if (dst.type == GLSL_TYPE_BOOL)
3570 emit_asm(ir, TGSI_OPCODE_USNE, dst, st_src_reg(dst), st_src_reg_for_int(0));
3571 } else if (ir->callee->intrinsic_id == ir_intrinsic_ssbo_store) {
3572 param = param->get_next();
3573 ir_rvalue *val = ((ir_instruction *)param)->as_rvalue();
3574 val->accept(this);
3575
3576 param = param->get_next();
3577 ir_constant *write_mask = ((ir_instruction *)param)->as_constant();
3578 assert(write_mask);
3579 dst.writemask = write_mask->value.u[0];
3580
3581 dst.type = this->result.type;
3582 inst = emit_asm(ir, TGSI_OPCODE_STORE, dst, off, this->result);
3583 } else {
3584 param = param->get_next();
3585 ir_rvalue *val = ((ir_instruction *)param)->as_rvalue();
3586 val->accept(this);
3587
3588 st_src_reg data = this->result, data2 = undef_src;
3589 unsigned opcode;
3590 switch (ir->callee->intrinsic_id) {
3591 case ir_intrinsic_ssbo_atomic_add:
3592 opcode = TGSI_OPCODE_ATOMUADD;
3593 break;
3594 case ir_intrinsic_ssbo_atomic_min:
3595 opcode = TGSI_OPCODE_ATOMIMIN;
3596 break;
3597 case ir_intrinsic_ssbo_atomic_max:
3598 opcode = TGSI_OPCODE_ATOMIMAX;
3599 break;
3600 case ir_intrinsic_ssbo_atomic_and:
3601 opcode = TGSI_OPCODE_ATOMAND;
3602 break;
3603 case ir_intrinsic_ssbo_atomic_or:
3604 opcode = TGSI_OPCODE_ATOMOR;
3605 break;
3606 case ir_intrinsic_ssbo_atomic_xor:
3607 opcode = TGSI_OPCODE_ATOMXOR;
3608 break;
3609 case ir_intrinsic_ssbo_atomic_exchange:
3610 opcode = TGSI_OPCODE_ATOMXCHG;
3611 break;
3612 case ir_intrinsic_ssbo_atomic_comp_swap:
3613 opcode = TGSI_OPCODE_ATOMCAS;
3614 param = param->get_next();
3615 val = ((ir_instruction *)param)->as_rvalue();
3616 val->accept(this);
3617 data2 = this->result;
3618 break;
3619 default:
3620 assert(!"Unexpected intrinsic");
3621 return;
3622 }
3623
3624 inst = emit_asm(ir, opcode, dst, off, data, data2);
3625 }
3626
3627 param = param->get_next();
3628 ir_constant *access = NULL;
3629 if (!param->is_tail_sentinel()) {
3630 access = ((ir_instruction *)param)->as_constant();
3631 assert(access);
3632 }
3633
3634 /* The emit_asm() might have actually split the op into pieces, e.g. for
3635 * double stores. We have to go back and fix up all the generated ops.
3636 */
3637 unsigned op = inst->op;
3638 do {
3639 inst->resource = buffer;
3640 if (access)
3641 inst->buffer_access = access->value.u[0];
3642
3643 if (inst == this->instructions.get_head_raw())
3644 break;
3645 inst = (glsl_to_tgsi_instruction *)inst->get_prev();
3646
3647 if (inst->op == TGSI_OPCODE_UADD) {
3648 if (inst == this->instructions.get_head_raw())
3649 break;
3650 inst = (glsl_to_tgsi_instruction *)inst->get_prev();
3651 }
3652 } while (inst->op == op && inst->resource.file == PROGRAM_UNDEFINED);
3653 }
3654
3655 void
3656 glsl_to_tgsi_visitor::visit_membar_intrinsic(ir_call *ir)
3657 {
3658 switch (ir->callee->intrinsic_id) {
3659 case ir_intrinsic_memory_barrier:
3660 emit_asm(ir, TGSI_OPCODE_MEMBAR, undef_dst,
3661 st_src_reg_for_int(TGSI_MEMBAR_SHADER_BUFFER |
3662 TGSI_MEMBAR_ATOMIC_BUFFER |
3663 TGSI_MEMBAR_SHADER_IMAGE |
3664 TGSI_MEMBAR_SHARED));
3665 break;
3666 case ir_intrinsic_memory_barrier_atomic_counter:
3667 emit_asm(ir, TGSI_OPCODE_MEMBAR, undef_dst,
3668 st_src_reg_for_int(TGSI_MEMBAR_ATOMIC_BUFFER));
3669 break;
3670 case ir_intrinsic_memory_barrier_buffer:
3671 emit_asm(ir, TGSI_OPCODE_MEMBAR, undef_dst,
3672 st_src_reg_for_int(TGSI_MEMBAR_SHADER_BUFFER));
3673 break;
3674 case ir_intrinsic_memory_barrier_image:
3675 emit_asm(ir, TGSI_OPCODE_MEMBAR, undef_dst,
3676 st_src_reg_for_int(TGSI_MEMBAR_SHADER_IMAGE));
3677 break;
3678 case ir_intrinsic_memory_barrier_shared:
3679 emit_asm(ir, TGSI_OPCODE_MEMBAR, undef_dst,
3680 st_src_reg_for_int(TGSI_MEMBAR_SHARED));
3681 break;
3682 case ir_intrinsic_group_memory_barrier:
3683 emit_asm(ir, TGSI_OPCODE_MEMBAR, undef_dst,
3684 st_src_reg_for_int(TGSI_MEMBAR_SHADER_BUFFER |
3685 TGSI_MEMBAR_ATOMIC_BUFFER |
3686 TGSI_MEMBAR_SHADER_IMAGE |
3687 TGSI_MEMBAR_SHARED |
3688 TGSI_MEMBAR_THREAD_GROUP));
3689 break;
3690 default:
3691 assert(!"Unexpected memory barrier intrinsic");
3692 }
3693 }
3694
3695 void
3696 glsl_to_tgsi_visitor::visit_shared_intrinsic(ir_call *ir)
3697 {
3698 exec_node *param = ir->actual_parameters.get_head();
3699
3700 ir_rvalue *offset = ((ir_instruction *)param)->as_rvalue();
3701
3702 st_src_reg buffer(PROGRAM_MEMORY, 0, GLSL_TYPE_UINT);
3703
3704 /* Calculate the surface offset */
3705 offset->accept(this);
3706 st_src_reg off = this->result;
3707
3708 st_dst_reg dst = undef_dst;
3709 if (ir->return_deref) {
3710 ir->return_deref->accept(this);
3711 dst = st_dst_reg(this->result);
3712 dst.writemask = (1 << ir->return_deref->type->vector_elements) - 1;
3713 }
3714
3715 glsl_to_tgsi_instruction *inst;
3716
3717 if (ir->callee->intrinsic_id == ir_intrinsic_shared_load) {
3718 inst = emit_asm(ir, TGSI_OPCODE_LOAD, dst, off);
3719 inst->resource = buffer;
3720 } else if (ir->callee->intrinsic_id == ir_intrinsic_shared_store) {
3721 param = param->get_next();
3722 ir_rvalue *val = ((ir_instruction *)param)->as_rvalue();
3723 val->accept(this);
3724
3725 param = param->get_next();
3726 ir_constant *write_mask = ((ir_instruction *)param)->as_constant();
3727 assert(write_mask);
3728 dst.writemask = write_mask->value.u[0];
3729
3730 dst.type = this->result.type;
3731 inst = emit_asm(ir, TGSI_OPCODE_STORE, dst, off, this->result);
3732 inst->resource = buffer;
3733 } else {
3734 param = param->get_next();
3735 ir_rvalue *val = ((ir_instruction *)param)->as_rvalue();
3736 val->accept(this);
3737
3738 st_src_reg data = this->result, data2 = undef_src;
3739 unsigned opcode;
3740 switch (ir->callee->intrinsic_id) {
3741 case ir_intrinsic_shared_atomic_add:
3742 opcode = TGSI_OPCODE_ATOMUADD;
3743 break;
3744 case ir_intrinsic_shared_atomic_min:
3745 opcode = TGSI_OPCODE_ATOMIMIN;
3746 break;
3747 case ir_intrinsic_shared_atomic_max:
3748 opcode = TGSI_OPCODE_ATOMIMAX;
3749 break;
3750 case ir_intrinsic_shared_atomic_and:
3751 opcode = TGSI_OPCODE_ATOMAND;
3752 break;
3753 case ir_intrinsic_shared_atomic_or:
3754 opcode = TGSI_OPCODE_ATOMOR;
3755 break;
3756 case ir_intrinsic_shared_atomic_xor:
3757 opcode = TGSI_OPCODE_ATOMXOR;
3758 break;
3759 case ir_intrinsic_shared_atomic_exchange:
3760 opcode = TGSI_OPCODE_ATOMXCHG;
3761 break;
3762 case ir_intrinsic_shared_atomic_comp_swap:
3763 opcode = TGSI_OPCODE_ATOMCAS;
3764 param = param->get_next();
3765 val = ((ir_instruction *)param)->as_rvalue();
3766 val->accept(this);
3767 data2 = this->result;
3768 break;
3769 default:
3770 assert(!"Unexpected intrinsic");
3771 return;
3772 }
3773
3774 inst = emit_asm(ir, opcode, dst, off, data, data2);
3775 inst->resource = buffer;
3776 }
3777 }
3778
3779 static void
3780 get_image_qualifiers(ir_dereference *ir, const glsl_type **type,
3781 bool *memory_coherent, bool *memory_volatile,
3782 bool *memory_restrict, unsigned *image_format)
3783 {
3784
3785 switch (ir->ir_type) {
3786 case ir_type_dereference_record: {
3787 ir_dereference_record *deref_record = ir->as_dereference_record();
3788
3789 *type = deref_record->type;
3790
3791 const glsl_type *struct_type =
3792 deref_record->record->type->without_array();
3793 int fild_idx = deref_record->field_idx;
3794 *memory_coherent =
3795 struct_type->fields.structure[fild_idx].memory_coherent;
3796 *memory_volatile =
3797 struct_type->fields.structure[fild_idx].memory_volatile;
3798 *memory_restrict =
3799 struct_type->fields.structure[fild_idx].memory_restrict;
3800 *image_format =
3801 struct_type->fields.structure[fild_idx].image_format;
3802 break;
3803 }
3804
3805 case ir_type_dereference_array: {
3806 ir_dereference_array *deref_arr = ir->as_dereference_array();
3807 get_image_qualifiers((ir_dereference *)deref_arr->array, type,
3808 memory_coherent, memory_volatile, memory_restrict,
3809 image_format);
3810 break;
3811 }
3812
3813 case ir_type_dereference_variable: {
3814 ir_variable *var = ir->variable_referenced();
3815
3816 *type = var->type->without_array();
3817 *memory_coherent = var->data.memory_coherent;
3818 *memory_volatile = var->data.memory_volatile;
3819 *memory_restrict = var->data.memory_restrict;
3820 *image_format = var->data.image_format;
3821 break;
3822 }
3823
3824 default:
3825 break;
3826 }
3827 }
3828
3829 void
3830 glsl_to_tgsi_visitor::visit_image_intrinsic(ir_call *ir)
3831 {
3832 exec_node *param = ir->actual_parameters.get_head();
3833
3834 ir_dereference *img = (ir_dereference *)param;
3835 const ir_variable *imgvar = img->variable_referenced();
3836 unsigned sampler_array_size = 1, sampler_base = 0;
3837 bool memory_coherent = false, memory_volatile = false, memory_restrict = false;
3838 unsigned image_format = 0;
3839 const glsl_type *type = NULL;
3840
3841 get_image_qualifiers(img, &type, &memory_coherent, &memory_volatile,
3842 &memory_restrict, &image_format);
3843
3844 st_src_reg reladdr;
3845 st_src_reg image(PROGRAM_IMAGE, 0, GLSL_TYPE_UINT);
3846 uint16_t index = 0;
3847 get_deref_offsets(img, &sampler_array_size, &sampler_base,
3848 &index, &reladdr, !imgvar->contains_bindless());
3849
3850 image.index = index;
3851 if (reladdr.file != PROGRAM_UNDEFINED) {
3852 image.reladdr = ralloc(mem_ctx, st_src_reg);
3853 *image.reladdr = reladdr;
3854 emit_arl(ir, sampler_reladdr, reladdr);
3855 }
3856
3857 st_dst_reg dst = undef_dst;
3858 if (ir->return_deref) {
3859 ir->return_deref->accept(this);
3860 dst = st_dst_reg(this->result);
3861 dst.writemask = (1 << ir->return_deref->type->vector_elements) - 1;
3862 }
3863
3864 glsl_to_tgsi_instruction *inst;
3865
3866 if (ir->callee->intrinsic_id == ir_intrinsic_image_size) {
3867 dst.writemask = WRITEMASK_XYZ;
3868 inst = emit_asm(ir, TGSI_OPCODE_RESQ, dst);
3869 } else if (ir->callee->intrinsic_id == ir_intrinsic_image_samples) {
3870 st_src_reg res = get_temp(glsl_type::ivec4_type);
3871 st_dst_reg dstres = st_dst_reg(res);
3872 dstres.writemask = WRITEMASK_W;
3873 inst = emit_asm(ir, TGSI_OPCODE_RESQ, dstres);
3874 res.swizzle = SWIZZLE_WWWW;
3875 emit_asm(ir, TGSI_OPCODE_MOV, dst, res);
3876 } else {
3877 st_src_reg arg1 = undef_src, arg2 = undef_src;
3878 st_src_reg coord;
3879 st_dst_reg coord_dst;
3880 coord = get_temp(glsl_type::ivec4_type);
3881 coord_dst = st_dst_reg(coord);
3882 coord_dst.writemask = (1 << type->coordinate_components()) - 1;
3883 param = param->get_next();
3884 ((ir_dereference *)param)->accept(this);
3885 emit_asm(ir, TGSI_OPCODE_MOV, coord_dst, this->result);
3886 coord.swizzle = SWIZZLE_XXXX;
3887 switch (type->coordinate_components()) {
3888 case 4: assert(!"unexpected coord count");
3889 /* fallthrough */
3890 case 3: coord.swizzle |= SWIZZLE_Z << 6;
3891 /* fallthrough */
3892 case 2: coord.swizzle |= SWIZZLE_Y << 3;
3893 }
3894
3895 if (type->sampler_dimensionality == GLSL_SAMPLER_DIM_MS) {
3896 param = param->get_next();
3897 ((ir_dereference *)param)->accept(this);
3898 st_src_reg sample = this->result;
3899 sample.swizzle = SWIZZLE_XXXX;
3900 coord_dst.writemask = WRITEMASK_W;
3901 emit_asm(ir, TGSI_OPCODE_MOV, coord_dst, sample);
3902 coord.swizzle |= SWIZZLE_W << 9;
3903 }
3904
3905 param = param->get_next();
3906 if (!param->is_tail_sentinel()) {
3907 ((ir_dereference *)param)->accept(this);
3908 arg1 = this->result;
3909 param = param->get_next();
3910 }
3911
3912 if (!param->is_tail_sentinel()) {
3913 ((ir_dereference *)param)->accept(this);
3914 arg2 = this->result;
3915 param = param->get_next();
3916 }
3917
3918 assert(param->is_tail_sentinel());
3919
3920 unsigned opcode;
3921 switch (ir->callee->intrinsic_id) {
3922 case ir_intrinsic_image_load:
3923 opcode = TGSI_OPCODE_LOAD;
3924 break;
3925 case ir_intrinsic_image_store:
3926 opcode = TGSI_OPCODE_STORE;
3927 break;
3928 case ir_intrinsic_image_atomic_add:
3929 opcode = TGSI_OPCODE_ATOMUADD;
3930 break;
3931 case ir_intrinsic_image_atomic_min:
3932 opcode = TGSI_OPCODE_ATOMIMIN;
3933 break;
3934 case ir_intrinsic_image_atomic_max:
3935 opcode = TGSI_OPCODE_ATOMIMAX;
3936 break;
3937 case ir_intrinsic_image_atomic_and:
3938 opcode = TGSI_OPCODE_ATOMAND;
3939 break;
3940 case ir_intrinsic_image_atomic_or:
3941 opcode = TGSI_OPCODE_ATOMOR;
3942 break;
3943 case ir_intrinsic_image_atomic_xor:
3944 opcode = TGSI_OPCODE_ATOMXOR;
3945 break;
3946 case ir_intrinsic_image_atomic_exchange:
3947 opcode = TGSI_OPCODE_ATOMXCHG;
3948 break;
3949 case ir_intrinsic_image_atomic_comp_swap:
3950 opcode = TGSI_OPCODE_ATOMCAS;
3951 break;
3952 default:
3953 assert(!"Unexpected intrinsic");
3954 return;
3955 }
3956
3957 inst = emit_asm(ir, opcode, dst, coord, arg1, arg2);
3958 if (opcode == TGSI_OPCODE_STORE)
3959 inst->dst[0].writemask = WRITEMASK_XYZW;
3960 }
3961
3962 if (imgvar->contains_bindless()) {
3963 img->accept(this);
3964 inst->resource = this->result;
3965 inst->resource.swizzle = MAKE_SWIZZLE4(SWIZZLE_X, SWIZZLE_Y,
3966 SWIZZLE_X, SWIZZLE_Y);
3967 } else {
3968 inst->resource = image;
3969 inst->sampler_array_size = sampler_array_size;
3970 inst->sampler_base = sampler_base;
3971 }
3972
3973 inst->tex_target = type->sampler_index();
3974 inst->image_format = st_mesa_format_to_pipe_format(st_context(ctx),
3975 _mesa_get_shader_image_format(image_format));
3976
3977 if (memory_coherent)
3978 inst->buffer_access |= TGSI_MEMORY_COHERENT;
3979 if (memory_restrict)
3980 inst->buffer_access |= TGSI_MEMORY_RESTRICT;
3981 if (memory_volatile)
3982 inst->buffer_access |= TGSI_MEMORY_VOLATILE;
3983 }
3984
3985 void
3986 glsl_to_tgsi_visitor::visit_generic_intrinsic(ir_call *ir, unsigned op)
3987 {
3988 ir->return_deref->accept(this);
3989 st_dst_reg dst = st_dst_reg(this->result);
3990
3991 dst.writemask = u_bit_consecutive(0, ir->return_deref->var->type->vector_elements);
3992
3993 st_src_reg src[4] = { undef_src, undef_src, undef_src, undef_src };
3994 unsigned num_src = 0;
3995 foreach_in_list(ir_rvalue, param, &ir->actual_parameters) {
3996 assert(num_src < ARRAY_SIZE(src));
3997
3998 this->result.file = PROGRAM_UNDEFINED;
3999 param->accept(this);
4000 assert(this->result.file != PROGRAM_UNDEFINED);
4001
4002 src[num_src] = this->result;
4003 num_src++;
4004 }
4005
4006 emit_asm(ir, op, dst, src[0], src[1], src[2], src[3]);
4007 }
4008
4009 void
4010 glsl_to_tgsi_visitor::visit(ir_call *ir)
4011 {
4012 ir_function_signature *sig = ir->callee;
4013
4014 /* Filter out intrinsics */
4015 switch (sig->intrinsic_id) {
4016 case ir_intrinsic_atomic_counter_read:
4017 case ir_intrinsic_atomic_counter_increment:
4018 case ir_intrinsic_atomic_counter_predecrement:
4019 case ir_intrinsic_atomic_counter_add:
4020 case ir_intrinsic_atomic_counter_min:
4021 case ir_intrinsic_atomic_counter_max:
4022 case ir_intrinsic_atomic_counter_and:
4023 case ir_intrinsic_atomic_counter_or:
4024 case ir_intrinsic_atomic_counter_xor:
4025 case ir_intrinsic_atomic_counter_exchange:
4026 case ir_intrinsic_atomic_counter_comp_swap:
4027 visit_atomic_counter_intrinsic(ir);
4028 return;
4029
4030 case ir_intrinsic_ssbo_load:
4031 case ir_intrinsic_ssbo_store:
4032 case ir_intrinsic_ssbo_atomic_add:
4033 case ir_intrinsic_ssbo_atomic_min:
4034 case ir_intrinsic_ssbo_atomic_max:
4035 case ir_intrinsic_ssbo_atomic_and:
4036 case ir_intrinsic_ssbo_atomic_or:
4037 case ir_intrinsic_ssbo_atomic_xor:
4038 case ir_intrinsic_ssbo_atomic_exchange:
4039 case ir_intrinsic_ssbo_atomic_comp_swap:
4040 visit_ssbo_intrinsic(ir);
4041 return;
4042
4043 case ir_intrinsic_memory_barrier:
4044 case ir_intrinsic_memory_barrier_atomic_counter:
4045 case ir_intrinsic_memory_barrier_buffer:
4046 case ir_intrinsic_memory_barrier_image:
4047 case ir_intrinsic_memory_barrier_shared:
4048 case ir_intrinsic_group_memory_barrier:
4049 visit_membar_intrinsic(ir);
4050 return;
4051
4052 case ir_intrinsic_shared_load:
4053 case ir_intrinsic_shared_store:
4054 case ir_intrinsic_shared_atomic_add:
4055 case ir_intrinsic_shared_atomic_min:
4056 case ir_intrinsic_shared_atomic_max:
4057 case ir_intrinsic_shared_atomic_and:
4058 case ir_intrinsic_shared_atomic_or:
4059 case ir_intrinsic_shared_atomic_xor:
4060 case ir_intrinsic_shared_atomic_exchange:
4061 case ir_intrinsic_shared_atomic_comp_swap:
4062 visit_shared_intrinsic(ir);
4063 return;
4064
4065 case ir_intrinsic_image_load:
4066 case ir_intrinsic_image_store:
4067 case ir_intrinsic_image_atomic_add:
4068 case ir_intrinsic_image_atomic_min:
4069 case ir_intrinsic_image_atomic_max:
4070 case ir_intrinsic_image_atomic_and:
4071 case ir_intrinsic_image_atomic_or:
4072 case ir_intrinsic_image_atomic_xor:
4073 case ir_intrinsic_image_atomic_exchange:
4074 case ir_intrinsic_image_atomic_comp_swap:
4075 case ir_intrinsic_image_size:
4076 case ir_intrinsic_image_samples:
4077 visit_image_intrinsic(ir);
4078 return;
4079
4080 case ir_intrinsic_shader_clock:
4081 visit_generic_intrinsic(ir, TGSI_OPCODE_CLOCK);
4082 return;
4083
4084 case ir_intrinsic_vote_all:
4085 visit_generic_intrinsic(ir, TGSI_OPCODE_VOTE_ALL);
4086 return;
4087 case ir_intrinsic_vote_any:
4088 visit_generic_intrinsic(ir, TGSI_OPCODE_VOTE_ANY);
4089 return;
4090 case ir_intrinsic_vote_eq:
4091 visit_generic_intrinsic(ir, TGSI_OPCODE_VOTE_EQ);
4092 return;
4093 case ir_intrinsic_ballot:
4094 visit_generic_intrinsic(ir, TGSI_OPCODE_BALLOT);
4095 return;
4096 case ir_intrinsic_read_first_invocation:
4097 visit_generic_intrinsic(ir, TGSI_OPCODE_READ_FIRST);
4098 return;
4099 case ir_intrinsic_read_invocation:
4100 visit_generic_intrinsic(ir, TGSI_OPCODE_READ_INVOC);
4101 return;
4102
4103 case ir_intrinsic_invalid:
4104 case ir_intrinsic_generic_load:
4105 case ir_intrinsic_generic_store:
4106 case ir_intrinsic_generic_atomic_add:
4107 case ir_intrinsic_generic_atomic_and:
4108 case ir_intrinsic_generic_atomic_or:
4109 case ir_intrinsic_generic_atomic_xor:
4110 case ir_intrinsic_generic_atomic_min:
4111 case ir_intrinsic_generic_atomic_max:
4112 case ir_intrinsic_generic_atomic_exchange:
4113 case ir_intrinsic_generic_atomic_comp_swap:
4114 unreachable("Invalid intrinsic");
4115 }
4116 }
4117
4118 void
4119 glsl_to_tgsi_visitor::calc_deref_offsets(ir_dereference *tail,
4120 unsigned *array_elements,
4121 uint16_t *index,
4122 st_src_reg *indirect,
4123 unsigned *location)
4124 {
4125 switch (tail->ir_type) {
4126 case ir_type_dereference_record: {
4127 ir_dereference_record *deref_record = tail->as_dereference_record();
4128 const glsl_type *struct_type = deref_record->record->type;
4129 int field_index = deref_record->field_idx;
4130
4131 calc_deref_offsets(deref_record->record->as_dereference(), array_elements, index, indirect, location);
4132
4133 assert(field_index >= 0);
4134 *location += struct_type->record_location_offset(field_index);
4135 break;
4136 }
4137
4138 case ir_type_dereference_array: {
4139 ir_dereference_array *deref_arr = tail->as_dereference_array();
4140
4141 void *mem_ctx = ralloc_parent(deref_arr);
4142 ir_constant *array_index =
4143 deref_arr->array_index->constant_expression_value(mem_ctx);
4144
4145 if (!array_index) {
4146 st_src_reg temp_reg;
4147 st_dst_reg temp_dst;
4148
4149 temp_reg = get_temp(glsl_type::uint_type);
4150 temp_dst = st_dst_reg(temp_reg);
4151 temp_dst.writemask = 1;
4152
4153 deref_arr->array_index->accept(this);
4154 if (*array_elements != 1)
4155 emit_asm(NULL, TGSI_OPCODE_MUL, temp_dst, this->result, st_src_reg_for_int(*array_elements));
4156 else
4157 emit_asm(NULL, TGSI_OPCODE_MOV, temp_dst, this->result);
4158
4159 if (indirect->file == PROGRAM_UNDEFINED)
4160 *indirect = temp_reg;
4161 else {
4162 temp_dst = st_dst_reg(*indirect);
4163 temp_dst.writemask = 1;
4164 emit_asm(NULL, TGSI_OPCODE_ADD, temp_dst, *indirect, temp_reg);
4165 }
4166 } else
4167 *index += array_index->value.u[0] * *array_elements;
4168
4169 *array_elements *= deref_arr->array->type->length;
4170
4171 calc_deref_offsets(deref_arr->array->as_dereference(), array_elements, index, indirect, location);
4172 break;
4173 }
4174 default:
4175 break;
4176 }
4177 }
4178
4179 void
4180 glsl_to_tgsi_visitor::get_deref_offsets(ir_dereference *ir,
4181 unsigned *array_size,
4182 unsigned *base,
4183 uint16_t *index,
4184 st_src_reg *reladdr,
4185 bool opaque)
4186 {
4187 GLuint shader = _mesa_program_enum_to_shader_stage(this->prog->Target);
4188 unsigned location = 0;
4189 ir_variable *var = ir->variable_referenced();
4190
4191 memset(reladdr, 0, sizeof(*reladdr));
4192 reladdr->file = PROGRAM_UNDEFINED;
4193
4194 *base = 0;
4195 *array_size = 1;
4196
4197 assert(var);
4198 location = var->data.location;
4199 calc_deref_offsets(ir, array_size, index, reladdr, &location);
4200
4201 /*
4202 * If we end up with no indirect then adjust the base to the index,
4203 * and set the array size to 1.
4204 */
4205 if (reladdr->file == PROGRAM_UNDEFINED) {
4206 *base = *index;
4207 *array_size = 1;
4208 }
4209
4210 if (opaque) {
4211 assert(location != 0xffffffff);
4212 *base += this->shader_program->data->UniformStorage[location].opaque[shader].index;
4213 *index += this->shader_program->data->UniformStorage[location].opaque[shader].index;
4214 }
4215 }
4216
4217 st_src_reg
4218 glsl_to_tgsi_visitor::canonicalize_gather_offset(st_src_reg offset)
4219 {
4220 if (offset.reladdr || offset.reladdr2) {
4221 st_src_reg tmp = get_temp(glsl_type::ivec2_type);
4222 st_dst_reg tmp_dst = st_dst_reg(tmp);
4223 tmp_dst.writemask = WRITEMASK_XY;
4224 emit_asm(NULL, TGSI_OPCODE_MOV, tmp_dst, offset);
4225 return tmp;
4226 }
4227
4228 return offset;
4229 }
4230
4231 void
4232 glsl_to_tgsi_visitor::visit(ir_texture *ir)
4233 {
4234 st_src_reg result_src, coord, cube_sc, lod_info, projector, dx, dy;
4235 st_src_reg offset[MAX_GLSL_TEXTURE_OFFSET], sample_index, component;
4236 st_src_reg levels_src, reladdr;
4237 st_dst_reg result_dst, coord_dst, cube_sc_dst;
4238 glsl_to_tgsi_instruction *inst = NULL;
4239 unsigned opcode = TGSI_OPCODE_NOP;
4240 const glsl_type *sampler_type = ir->sampler->type;
4241 unsigned sampler_array_size = 1, sampler_base = 0;
4242 bool is_cube_array = false, is_cube_shadow = false;
4243 ir_variable *var = ir->sampler->variable_referenced();
4244 unsigned i;
4245
4246 /* if we are a cube array sampler or a cube shadow */
4247 if (sampler_type->sampler_dimensionality == GLSL_SAMPLER_DIM_CUBE) {
4248 is_cube_array = sampler_type->sampler_array;
4249 is_cube_shadow = sampler_type->sampler_shadow;
4250 }
4251
4252 if (ir->coordinate) {
4253 ir->coordinate->accept(this);
4254
4255 /* Put our coords in a temp. We'll need to modify them for shadow,
4256 * projection, or LOD, so the only case we'd use it as-is is if
4257 * we're doing plain old texturing. The optimization passes on
4258 * glsl_to_tgsi_visitor should handle cleaning up our mess in that case.
4259 */
4260 coord = get_temp(glsl_type::vec4_type);
4261 coord_dst = st_dst_reg(coord);
4262 coord_dst.writemask = (1 << ir->coordinate->type->vector_elements) - 1;
4263 emit_asm(ir, TGSI_OPCODE_MOV, coord_dst, this->result);
4264 }
4265
4266 if (ir->projector) {
4267 ir->projector->accept(this);
4268 projector = this->result;
4269 }
4270
4271 /* Storage for our result. Ideally for an assignment we'd be using
4272 * the actual storage for the result here, instead.
4273 */
4274 result_src = get_temp(ir->type);
4275 result_dst = st_dst_reg(result_src);
4276 result_dst.writemask = (1 << ir->type->vector_elements) - 1;
4277
4278 switch (ir->op) {
4279 case ir_tex:
4280 opcode = (is_cube_array && ir->shadow_comparator) ? TGSI_OPCODE_TEX2 : TGSI_OPCODE_TEX;
4281 if (ir->offset) {
4282 ir->offset->accept(this);
4283 offset[0] = this->result;
4284 }
4285 break;
4286 case ir_txb:
4287 if (is_cube_array || is_cube_shadow) {
4288 opcode = TGSI_OPCODE_TXB2;
4289 }
4290 else {
4291 opcode = TGSI_OPCODE_TXB;
4292 }
4293 ir->lod_info.bias->accept(this);
4294 lod_info = this->result;
4295 if (ir->offset) {
4296 ir->offset->accept(this);
4297 offset[0] = this->result;
4298 }
4299 break;
4300 case ir_txl:
4301 if (this->has_tex_txf_lz && ir->lod_info.lod->is_zero()) {
4302 opcode = TGSI_OPCODE_TEX_LZ;
4303 } else {
4304 opcode = is_cube_array ? TGSI_OPCODE_TXL2 : TGSI_OPCODE_TXL;
4305 ir->lod_info.lod->accept(this);
4306 lod_info = this->result;
4307 }
4308 if (ir->offset) {
4309 ir->offset->accept(this);
4310 offset[0] = this->result;
4311 }
4312 break;
4313 case ir_txd:
4314 opcode = TGSI_OPCODE_TXD;
4315 ir->lod_info.grad.dPdx->accept(this);
4316 dx = this->result;
4317 ir->lod_info.grad.dPdy->accept(this);
4318 dy = this->result;
4319 if (ir->offset) {
4320 ir->offset->accept(this);
4321 offset[0] = this->result;
4322 }
4323 break;
4324 case ir_txs:
4325 opcode = TGSI_OPCODE_TXQ;
4326 ir->lod_info.lod->accept(this);
4327 lod_info = this->result;
4328 break;
4329 case ir_query_levels:
4330 opcode = TGSI_OPCODE_TXQ;
4331 lod_info = undef_src;
4332 levels_src = get_temp(ir->type);
4333 break;
4334 case ir_txf:
4335 if (this->has_tex_txf_lz && ir->lod_info.lod->is_zero()) {
4336 opcode = TGSI_OPCODE_TXF_LZ;
4337 } else {
4338 opcode = TGSI_OPCODE_TXF;
4339 ir->lod_info.lod->accept(this);
4340 lod_info = this->result;
4341 }
4342 if (ir->offset) {
4343 ir->offset->accept(this);
4344 offset[0] = this->result;
4345 }
4346 break;
4347 case ir_txf_ms:
4348 opcode = TGSI_OPCODE_TXF;
4349 ir->lod_info.sample_index->accept(this);
4350 sample_index = this->result;
4351 break;
4352 case ir_tg4:
4353 opcode = TGSI_OPCODE_TG4;
4354 ir->lod_info.component->accept(this);
4355 component = this->result;
4356 if (ir->offset) {
4357 ir->offset->accept(this);
4358 if (ir->offset->type->is_array()) {
4359 const glsl_type *elt_type = ir->offset->type->fields.array;
4360 for (i = 0; i < ir->offset->type->length; i++) {
4361 offset[i] = this->result;
4362 offset[i].index += i * type_size(elt_type);
4363 offset[i].type = elt_type->base_type;
4364 offset[i].swizzle = swizzle_for_size(elt_type->vector_elements);
4365 offset[i] = canonicalize_gather_offset(offset[i]);
4366 }
4367 } else {
4368 offset[0] = canonicalize_gather_offset(this->result);
4369 }
4370 }
4371 break;
4372 case ir_lod:
4373 opcode = TGSI_OPCODE_LODQ;
4374 break;
4375 case ir_texture_samples:
4376 opcode = TGSI_OPCODE_TXQS;
4377 break;
4378 case ir_samples_identical:
4379 unreachable("Unexpected ir_samples_identical opcode");
4380 }
4381
4382 if (ir->projector) {
4383 if (opcode == TGSI_OPCODE_TEX) {
4384 /* Slot the projector in as the last component of the coord. */
4385 coord_dst.writemask = WRITEMASK_W;
4386 emit_asm(ir, TGSI_OPCODE_MOV, coord_dst, projector);
4387 coord_dst.writemask = WRITEMASK_XYZW;
4388 opcode = TGSI_OPCODE_TXP;
4389 } else {
4390 st_src_reg coord_w = coord;
4391 coord_w.swizzle = SWIZZLE_WWWW;
4392
4393 /* For the other TEX opcodes there's no projective version
4394 * since the last slot is taken up by LOD info. Do the
4395 * projective divide now.
4396 */
4397 coord_dst.writemask = WRITEMASK_W;
4398 emit_asm(ir, TGSI_OPCODE_RCP, coord_dst, projector);
4399
4400 /* In the case where we have to project the coordinates "by hand,"
4401 * the shadow comparator value must also be projected.
4402 */
4403 st_src_reg tmp_src = coord;
4404 if (ir->shadow_comparator) {
4405 /* Slot the shadow value in as the second to last component of the
4406 * coord.
4407 */
4408 ir->shadow_comparator->accept(this);
4409
4410 tmp_src = get_temp(glsl_type::vec4_type);
4411 st_dst_reg tmp_dst = st_dst_reg(tmp_src);
4412
4413 /* Projective division not allowed for array samplers. */
4414 assert(!sampler_type->sampler_array);
4415
4416 tmp_dst.writemask = WRITEMASK_Z;
4417 emit_asm(ir, TGSI_OPCODE_MOV, tmp_dst, this->result);
4418
4419 tmp_dst.writemask = WRITEMASK_XY;
4420 emit_asm(ir, TGSI_OPCODE_MOV, tmp_dst, coord);
4421 }
4422
4423 coord_dst.writemask = WRITEMASK_XYZ;
4424 emit_asm(ir, TGSI_OPCODE_MUL, coord_dst, tmp_src, coord_w);
4425
4426 coord_dst.writemask = WRITEMASK_XYZW;
4427 coord.swizzle = SWIZZLE_XYZW;
4428 }
4429 }
4430
4431 /* If projection is done and the opcode is not TGSI_OPCODE_TXP, then the shadow
4432 * comparator was put in the correct place (and projected) by the code,
4433 * above, that handles by-hand projection.
4434 */
4435 if (ir->shadow_comparator && (!ir->projector || opcode == TGSI_OPCODE_TXP)) {
4436 /* Slot the shadow value in as the second to last component of the
4437 * coord.
4438 */
4439 ir->shadow_comparator->accept(this);
4440
4441 if (is_cube_array) {
4442 cube_sc = get_temp(glsl_type::float_type);
4443 cube_sc_dst = st_dst_reg(cube_sc);
4444 cube_sc_dst.writemask = WRITEMASK_X;
4445 emit_asm(ir, TGSI_OPCODE_MOV, cube_sc_dst, this->result);
4446 cube_sc_dst.writemask = WRITEMASK_X;
4447 }
4448 else {
4449 if ((sampler_type->sampler_dimensionality == GLSL_SAMPLER_DIM_2D &&
4450 sampler_type->sampler_array) ||
4451 sampler_type->sampler_dimensionality == GLSL_SAMPLER_DIM_CUBE) {
4452 coord_dst.writemask = WRITEMASK_W;
4453 } else {
4454 coord_dst.writemask = WRITEMASK_Z;
4455 }
4456 emit_asm(ir, TGSI_OPCODE_MOV, coord_dst, this->result);
4457 coord_dst.writemask = WRITEMASK_XYZW;
4458 }
4459 }
4460
4461 if (ir->op == ir_txf_ms) {
4462 coord_dst.writemask = WRITEMASK_W;
4463 emit_asm(ir, TGSI_OPCODE_MOV, coord_dst, sample_index);
4464 coord_dst.writemask = WRITEMASK_XYZW;
4465 } else if (opcode == TGSI_OPCODE_TXL || opcode == TGSI_OPCODE_TXB ||
4466 opcode == TGSI_OPCODE_TXF) {
4467 /* TGSI stores LOD or LOD bias in the last channel of the coords. */
4468 coord_dst.writemask = WRITEMASK_W;
4469 emit_asm(ir, TGSI_OPCODE_MOV, coord_dst, lod_info);
4470 coord_dst.writemask = WRITEMASK_XYZW;
4471 }
4472
4473 st_src_reg sampler(PROGRAM_SAMPLER, 0, GLSL_TYPE_UINT);
4474
4475 uint16_t index = 0;
4476 get_deref_offsets(ir->sampler, &sampler_array_size, &sampler_base,
4477 &index, &reladdr, !var->contains_bindless());
4478
4479 sampler.index = index;
4480 if (reladdr.file != PROGRAM_UNDEFINED) {
4481 sampler.reladdr = ralloc(mem_ctx, st_src_reg);
4482 *sampler.reladdr = reladdr;
4483 emit_arl(ir, sampler_reladdr, reladdr);
4484 }
4485
4486 if (opcode == TGSI_OPCODE_TXD)
4487 inst = emit_asm(ir, opcode, result_dst, coord, dx, dy);
4488 else if (opcode == TGSI_OPCODE_TXQ) {
4489 if (ir->op == ir_query_levels) {
4490 /* the level is stored in W */
4491 inst = emit_asm(ir, opcode, st_dst_reg(levels_src), lod_info);
4492 result_dst.writemask = WRITEMASK_X;
4493 levels_src.swizzle = SWIZZLE_WWWW;
4494 emit_asm(ir, TGSI_OPCODE_MOV, result_dst, levels_src);
4495 } else
4496 inst = emit_asm(ir, opcode, result_dst, lod_info);
4497 } else if (opcode == TGSI_OPCODE_TXQS) {
4498 inst = emit_asm(ir, opcode, result_dst);
4499 } else if (opcode == TGSI_OPCODE_TXL2 || opcode == TGSI_OPCODE_TXB2) {
4500 inst = emit_asm(ir, opcode, result_dst, coord, lod_info);
4501 } else if (opcode == TGSI_OPCODE_TEX2) {
4502 inst = emit_asm(ir, opcode, result_dst, coord, cube_sc);
4503 } else if (opcode == TGSI_OPCODE_TG4) {
4504 if (is_cube_array && ir->shadow_comparator) {
4505 inst = emit_asm(ir, opcode, result_dst, coord, cube_sc);
4506 } else {
4507 inst = emit_asm(ir, opcode, result_dst, coord, component);
4508 }
4509 } else
4510 inst = emit_asm(ir, opcode, result_dst, coord);
4511
4512 if (ir->shadow_comparator)
4513 inst->tex_shadow = GL_TRUE;
4514
4515 if (var->contains_bindless()) {
4516 ir->sampler->accept(this);
4517 inst->resource = this->result;
4518 inst->resource.swizzle = MAKE_SWIZZLE4(SWIZZLE_X, SWIZZLE_Y,
4519 SWIZZLE_X, SWIZZLE_Y);
4520 } else {
4521 inst->resource = sampler;
4522 inst->sampler_array_size = sampler_array_size;
4523 inst->sampler_base = sampler_base;
4524 }
4525
4526 if (ir->offset) {
4527 if (!inst->tex_offsets)
4528 inst->tex_offsets = rzalloc_array(inst, st_src_reg, MAX_GLSL_TEXTURE_OFFSET);
4529
4530 for (i = 0; i < MAX_GLSL_TEXTURE_OFFSET && offset[i].file != PROGRAM_UNDEFINED; i++)
4531 inst->tex_offsets[i] = offset[i];
4532 inst->tex_offset_num_offset = i;
4533 }
4534
4535 inst->tex_target = sampler_type->sampler_index();
4536 inst->tex_type = ir->type->base_type;
4537
4538 this->result = result_src;
4539 }
4540
4541 void
4542 glsl_to_tgsi_visitor::visit(ir_return *ir)
4543 {
4544 assert(!ir->get_value());
4545
4546 emit_asm(ir, TGSI_OPCODE_RET);
4547 }
4548
4549 void
4550 glsl_to_tgsi_visitor::visit(ir_discard *ir)
4551 {
4552 if (ir->condition) {
4553 ir->condition->accept(this);
4554 st_src_reg condition = this->result;
4555
4556 /* Convert the bool condition to a float so we can negate. */
4557 if (native_integers) {
4558 st_src_reg temp = get_temp(ir->condition->type);
4559 emit_asm(ir, TGSI_OPCODE_AND, st_dst_reg(temp),
4560 condition, st_src_reg_for_float(1.0));
4561 condition = temp;
4562 }
4563
4564 condition.negate = ~condition.negate;
4565 emit_asm(ir, TGSI_OPCODE_KILL_IF, undef_dst, condition);
4566 } else {
4567 /* unconditional kil */
4568 emit_asm(ir, TGSI_OPCODE_KILL);
4569 }
4570 }
4571
4572 void
4573 glsl_to_tgsi_visitor::visit(ir_if *ir)
4574 {
4575 unsigned if_opcode;
4576 glsl_to_tgsi_instruction *if_inst;
4577
4578 ir->condition->accept(this);
4579 assert(this->result.file != PROGRAM_UNDEFINED);
4580
4581 if_opcode = native_integers ? TGSI_OPCODE_UIF : TGSI_OPCODE_IF;
4582
4583 if_inst = emit_asm(ir->condition, if_opcode, undef_dst, this->result);
4584
4585 this->instructions.push_tail(if_inst);
4586
4587 visit_exec_list(&ir->then_instructions, this);
4588
4589 if (!ir->else_instructions.is_empty()) {
4590 emit_asm(ir->condition, TGSI_OPCODE_ELSE);
4591 visit_exec_list(&ir->else_instructions, this);
4592 }
4593
4594 if_inst = emit_asm(ir->condition, TGSI_OPCODE_ENDIF);
4595 }
4596
4597
4598 void
4599 glsl_to_tgsi_visitor::visit(ir_emit_vertex *ir)
4600 {
4601 assert(this->prog->Target == GL_GEOMETRY_PROGRAM_NV);
4602
4603 ir->stream->accept(this);
4604 emit_asm(ir, TGSI_OPCODE_EMIT, undef_dst, this->result);
4605 }
4606
4607 void
4608 glsl_to_tgsi_visitor::visit(ir_end_primitive *ir)
4609 {
4610 assert(this->prog->Target == GL_GEOMETRY_PROGRAM_NV);
4611
4612 ir->stream->accept(this);
4613 emit_asm(ir, TGSI_OPCODE_ENDPRIM, undef_dst, this->result);
4614 }
4615
4616 void
4617 glsl_to_tgsi_visitor::visit(ir_barrier *ir)
4618 {
4619 assert(this->prog->Target == GL_TESS_CONTROL_PROGRAM_NV ||
4620 this->prog->Target == GL_COMPUTE_PROGRAM_NV);
4621
4622 emit_asm(ir, TGSI_OPCODE_BARRIER);
4623 }
4624
4625 glsl_to_tgsi_visitor::glsl_to_tgsi_visitor()
4626 {
4627 STATIC_ASSERT(sizeof(samplers_used) * 8 >= PIPE_MAX_SAMPLERS);
4628
4629 result.file = PROGRAM_UNDEFINED;
4630 next_temp = 1;
4631 array_sizes = NULL;
4632 max_num_arrays = 0;
4633 next_array = 0;
4634 num_inputs = 0;
4635 num_outputs = 0;
4636 num_input_arrays = 0;
4637 num_output_arrays = 0;
4638 num_immediates = 0;
4639 num_address_regs = 0;
4640 samplers_used = 0;
4641 images_used = 0;
4642 indirect_addr_consts = false;
4643 wpos_transform_const = -1;
4644 glsl_version = 0;
4645 native_integers = false;
4646 mem_ctx = ralloc_context(NULL);
4647 ctx = NULL;
4648 prog = NULL;
4649 precise = 0;
4650 shader_program = NULL;
4651 shader = NULL;
4652 options = NULL;
4653 have_sqrt = false;
4654 have_fma = false;
4655 use_shared_memory = false;
4656 has_tex_txf_lz = false;
4657 variables = NULL;
4658 }
4659
4660 static void var_destroy(struct hash_entry *entry)
4661 {
4662 variable_storage *storage = (variable_storage *)entry->data;
4663
4664 delete storage;
4665 }
4666
4667 glsl_to_tgsi_visitor::~glsl_to_tgsi_visitor()
4668 {
4669 _mesa_hash_table_destroy(variables, var_destroy);
4670 free(array_sizes);
4671 ralloc_free(mem_ctx);
4672 }
4673
4674 extern "C" void free_glsl_to_tgsi_visitor(glsl_to_tgsi_visitor *v)
4675 {
4676 delete v;
4677 }
4678
4679
4680 /**
4681 * Count resources used by the given gpu program (number of texture
4682 * samplers, etc).
4683 */
4684 static void
4685 count_resources(glsl_to_tgsi_visitor *v, gl_program *prog)
4686 {
4687 v->samplers_used = 0;
4688 v->images_used = 0;
4689
4690 foreach_in_list(glsl_to_tgsi_instruction, inst, &v->instructions) {
4691 if (inst->info->is_tex) {
4692 for (int i = 0; i < inst->sampler_array_size; i++) {
4693 unsigned idx = inst->sampler_base + i;
4694 v->samplers_used |= 1u << idx;
4695
4696 debug_assert(idx < (int)ARRAY_SIZE(v->sampler_types));
4697 v->sampler_types[idx] = inst->tex_type;
4698 v->sampler_targets[idx] =
4699 st_translate_texture_target(inst->tex_target, inst->tex_shadow);
4700
4701 if (inst->tex_shadow) {
4702 prog->ShadowSamplers |= 1 << (inst->resource.index + i);
4703 }
4704 }
4705 }
4706
4707 if (inst->tex_target == TEXTURE_EXTERNAL_INDEX)
4708 prog->ExternalSamplersUsed |= 1 << inst->resource.index;
4709
4710 if (inst->resource.file != PROGRAM_UNDEFINED && (
4711 is_resource_instruction(inst->op) ||
4712 inst->op == TGSI_OPCODE_STORE)) {
4713 if (inst->resource.file == PROGRAM_MEMORY) {
4714 v->use_shared_memory = true;
4715 } else if (inst->resource.file == PROGRAM_IMAGE) {
4716 for (int i = 0; i < inst->sampler_array_size; i++) {
4717 unsigned idx = inst->sampler_base + i;
4718 v->images_used |= 1 << idx;
4719 v->image_targets[idx] =
4720 st_translate_texture_target(inst->tex_target, false);
4721 v->image_formats[idx] = inst->image_format;
4722 }
4723 }
4724 }
4725 }
4726 prog->SamplersUsed = v->samplers_used;
4727
4728 if (v->shader_program != NULL)
4729 _mesa_update_shader_textures_used(v->shader_program, prog);
4730 }
4731
4732 /**
4733 * Returns the mask of channels (bitmask of WRITEMASK_X,Y,Z,W) which
4734 * are read from the given src in this instruction
4735 */
4736 static int
4737 get_src_arg_mask(st_dst_reg dst, st_src_reg src)
4738 {
4739 int read_mask = 0, comp;
4740
4741 /* Now, given the src swizzle and the written channels, find which
4742 * components are actually read
4743 */
4744 for (comp = 0; comp < 4; ++comp) {
4745 const unsigned coord = GET_SWZ(src.swizzle, comp);
4746 assert(coord < 4);
4747 if (dst.writemask & (1 << comp) && coord <= SWIZZLE_W)
4748 read_mask |= 1 << coord;
4749 }
4750
4751 return read_mask;
4752 }
4753
4754 /**
4755 * This pass replaces CMP T0, T1 T2 T0 with MOV T0, T2 when the CMP
4756 * instruction is the first instruction to write to register T0. There are
4757 * several lowering passes done in GLSL IR (e.g. branches and
4758 * relative addressing) that create a large number of conditional assignments
4759 * that ir_to_mesa converts to CMP instructions like the one mentioned above.
4760 *
4761 * Here is why this conversion is safe:
4762 * CMP T0, T1 T2 T0 can be expanded to:
4763 * if (T1 < 0.0)
4764 * MOV T0, T2;
4765 * else
4766 * MOV T0, T0;
4767 *
4768 * If (T1 < 0.0) evaluates to true then our replacement MOV T0, T2 is the same
4769 * as the original program. If (T1 < 0.0) evaluates to false, executing
4770 * MOV T0, T0 will store a garbage value in T0 since T0 is uninitialized.
4771 * Therefore, it doesn't matter that we are replacing MOV T0, T0 with MOV T0, T2
4772 * because any instruction that was going to read from T0 after this was going
4773 * to read a garbage value anyway.
4774 */
4775 void
4776 glsl_to_tgsi_visitor::simplify_cmp(void)
4777 {
4778 int tempWritesSize = 0;
4779 unsigned *tempWrites = NULL;
4780 unsigned outputWrites[VARYING_SLOT_TESS_MAX];
4781
4782 memset(outputWrites, 0, sizeof(outputWrites));
4783
4784 foreach_in_list(glsl_to_tgsi_instruction, inst, &this->instructions) {
4785 unsigned prevWriteMask = 0;
4786
4787 /* Give up if we encounter relative addressing or flow control. */
4788 if (inst->dst[0].reladdr || inst->dst[0].reladdr2 ||
4789 inst->dst[1].reladdr || inst->dst[1].reladdr2 ||
4790 inst->info->is_branch ||
4791 inst->op == TGSI_OPCODE_CONT ||
4792 inst->op == TGSI_OPCODE_END ||
4793 inst->op == TGSI_OPCODE_RET) {
4794 break;
4795 }
4796
4797 if (inst->dst[0].file == PROGRAM_OUTPUT) {
4798 assert(inst->dst[0].index < (signed)ARRAY_SIZE(outputWrites));
4799 prevWriteMask = outputWrites[inst->dst[0].index];
4800 outputWrites[inst->dst[0].index] |= inst->dst[0].writemask;
4801 } else if (inst->dst[0].file == PROGRAM_TEMPORARY) {
4802 if (inst->dst[0].index >= tempWritesSize) {
4803 const int inc = 4096;
4804
4805 tempWrites = (unsigned*)
4806 realloc(tempWrites,
4807 (tempWritesSize + inc) * sizeof(unsigned));
4808 if (!tempWrites)
4809 return;
4810
4811 memset(tempWrites + tempWritesSize, 0, inc * sizeof(unsigned));
4812 tempWritesSize += inc;
4813 }
4814
4815 prevWriteMask = tempWrites[inst->dst[0].index];
4816 tempWrites[inst->dst[0].index] |= inst->dst[0].writemask;
4817 } else
4818 continue;
4819
4820 /* For a CMP to be considered a conditional write, the destination
4821 * register and source register two must be the same. */
4822 if (inst->op == TGSI_OPCODE_CMP
4823 && !(inst->dst[0].writemask & prevWriteMask)
4824 && inst->src[2].file == inst->dst[0].file
4825 && inst->src[2].index == inst->dst[0].index
4826 && inst->dst[0].writemask == get_src_arg_mask(inst->dst[0], inst->src[2])) {
4827
4828 inst->op = TGSI_OPCODE_MOV;
4829 inst->info = tgsi_get_opcode_info(inst->op);
4830 inst->src[0] = inst->src[1];
4831 }
4832 }
4833
4834 free(tempWrites);
4835 }
4836
4837 /* Replaces all references to a temporary register index with another index. */
4838 void
4839 glsl_to_tgsi_visitor::rename_temp_registers(int num_renames, struct rename_reg_pair *renames)
4840 {
4841 foreach_in_list(glsl_to_tgsi_instruction, inst, &this->instructions) {
4842 unsigned j;
4843 int k;
4844 for (j = 0; j < num_inst_src_regs(inst); j++) {
4845 if (inst->src[j].file == PROGRAM_TEMPORARY)
4846 for (k = 0; k < num_renames; k++)
4847 if (inst->src[j].index == renames[k].old_reg)
4848 inst->src[j].index = renames[k].new_reg;
4849 }
4850
4851 for (j = 0; j < inst->tex_offset_num_offset; j++) {
4852 if (inst->tex_offsets[j].file == PROGRAM_TEMPORARY)
4853 for (k = 0; k < num_renames; k++)
4854 if (inst->tex_offsets[j].index == renames[k].old_reg)
4855 inst->tex_offsets[j].index = renames[k].new_reg;
4856 }
4857
4858 if (inst->resource.file == PROGRAM_TEMPORARY) {
4859 for (k = 0; k < num_renames; k++)
4860 if (inst->resource.index == renames[k].old_reg)
4861 inst->resource.index = renames[k].new_reg;
4862 }
4863
4864 for (j = 0; j < num_inst_dst_regs(inst); j++) {
4865 if (inst->dst[j].file == PROGRAM_TEMPORARY)
4866 for (k = 0; k < num_renames; k++)
4867 if (inst->dst[j].index == renames[k].old_reg)
4868 inst->dst[j].index = renames[k].new_reg;
4869 }
4870 }
4871 }
4872
4873 void
4874 glsl_to_tgsi_visitor::get_first_temp_write(int *first_writes)
4875 {
4876 int depth = 0; /* loop depth */
4877 int loop_start = -1; /* index of the first active BGNLOOP (if any) */
4878 unsigned i = 0, j;
4879
4880 foreach_in_list(glsl_to_tgsi_instruction, inst, &this->instructions) {
4881 for (j = 0; j < num_inst_dst_regs(inst); j++) {
4882 if (inst->dst[j].file == PROGRAM_TEMPORARY) {
4883 if (first_writes[inst->dst[j].index] == -1)
4884 first_writes[inst->dst[j].index] = (depth == 0) ? i : loop_start;
4885 }
4886 }
4887
4888 if (inst->op == TGSI_OPCODE_BGNLOOP) {
4889 if(depth++ == 0)
4890 loop_start = i;
4891 } else if (inst->op == TGSI_OPCODE_ENDLOOP) {
4892 if (--depth == 0)
4893 loop_start = -1;
4894 }
4895 assert(depth >= 0);
4896 i++;
4897 }
4898 }
4899
4900 void
4901 glsl_to_tgsi_visitor::get_first_temp_read(int *first_reads)
4902 {
4903 int depth = 0; /* loop depth */
4904 int loop_start = -1; /* index of the first active BGNLOOP (if any) */
4905 unsigned i = 0, j;
4906
4907 foreach_in_list(glsl_to_tgsi_instruction, inst, &this->instructions) {
4908 for (j = 0; j < num_inst_src_regs(inst); j++) {
4909 if (inst->src[j].file == PROGRAM_TEMPORARY) {
4910 if (first_reads[inst->src[j].index] == -1)
4911 first_reads[inst->src[j].index] = (depth == 0) ? i : loop_start;
4912 }
4913 }
4914 for (j = 0; j < inst->tex_offset_num_offset; j++) {
4915 if (inst->tex_offsets[j].file == PROGRAM_TEMPORARY) {
4916 if (first_reads[inst->tex_offsets[j].index] == -1)
4917 first_reads[inst->tex_offsets[j].index] = (depth == 0) ? i : loop_start;
4918 }
4919 }
4920 if (inst->op == TGSI_OPCODE_BGNLOOP) {
4921 if(depth++ == 0)
4922 loop_start = i;
4923 } else if (inst->op == TGSI_OPCODE_ENDLOOP) {
4924 if (--depth == 0)
4925 loop_start = -1;
4926 }
4927 assert(depth >= 0);
4928 i++;
4929 }
4930 }
4931
4932 void
4933 glsl_to_tgsi_visitor::get_last_temp_read_first_temp_write(int *last_reads, int *first_writes)
4934 {
4935 int depth = 0; /* loop depth */
4936 int loop_start = -1; /* index of the first active BGNLOOP (if any) */
4937 unsigned i = 0, j;
4938 int k;
4939 foreach_in_list(glsl_to_tgsi_instruction, inst, &this->instructions) {
4940 for (j = 0; j < num_inst_src_regs(inst); j++) {
4941 if (inst->src[j].file == PROGRAM_TEMPORARY)
4942 last_reads[inst->src[j].index] = (depth == 0) ? i : -2;
4943 }
4944 for (j = 0; j < num_inst_dst_regs(inst); j++) {
4945 if (inst->dst[j].file == PROGRAM_TEMPORARY) {
4946 if (first_writes[inst->dst[j].index] == -1)
4947 first_writes[inst->dst[j].index] = (depth == 0) ? i : loop_start;
4948 last_reads[inst->dst[j].index] = (depth == 0) ? i : -2;
4949 }
4950 }
4951 for (j = 0; j < inst->tex_offset_num_offset; j++) {
4952 if (inst->tex_offsets[j].file == PROGRAM_TEMPORARY)
4953 last_reads[inst->tex_offsets[j].index] = (depth == 0) ? i : -2;
4954 }
4955 if (inst->op == TGSI_OPCODE_BGNLOOP) {
4956 if(depth++ == 0)
4957 loop_start = i;
4958 } else if (inst->op == TGSI_OPCODE_ENDLOOP) {
4959 if (--depth == 0) {
4960 loop_start = -1;
4961 for (k = 0; k < this->next_temp; k++) {
4962 if (last_reads[k] == -2) {
4963 last_reads[k] = i;
4964 }
4965 }
4966 }
4967 }
4968 assert(depth >= 0);
4969 i++;
4970 }
4971 }
4972
4973 void
4974 glsl_to_tgsi_visitor::get_last_temp_write(int *last_writes)
4975 {
4976 int depth = 0; /* loop depth */
4977 int i = 0, k;
4978 unsigned j;
4979
4980 foreach_in_list(glsl_to_tgsi_instruction, inst, &this->instructions) {
4981 for (j = 0; j < num_inst_dst_regs(inst); j++) {
4982 if (inst->dst[j].file == PROGRAM_TEMPORARY)
4983 last_writes[inst->dst[j].index] = (depth == 0) ? i : -2;
4984 }
4985
4986 if (inst->op == TGSI_OPCODE_BGNLOOP)
4987 depth++;
4988 else if (inst->op == TGSI_OPCODE_ENDLOOP)
4989 if (--depth == 0) {
4990 for (k = 0; k < this->next_temp; k++) {
4991 if (last_writes[k] == -2) {
4992 last_writes[k] = i;
4993 }
4994 }
4995 }
4996 assert(depth >= 0);
4997 i++;
4998 }
4999 }
5000
5001 /*
5002 * On a basic block basis, tracks available PROGRAM_TEMPORARY register
5003 * channels for copy propagation and updates following instructions to
5004 * use the original versions.
5005 *
5006 * The glsl_to_tgsi_visitor lazily produces code assuming that this pass
5007 * will occur. As an example, a TXP production before this pass:
5008 *
5009 * 0: MOV TEMP[1], INPUT[4].xyyy;
5010 * 1: MOV TEMP[1].w, INPUT[4].wwww;
5011 * 2: TXP TEMP[2], TEMP[1], texture[0], 2D;
5012 *
5013 * and after:
5014 *
5015 * 0: MOV TEMP[1], INPUT[4].xyyy;
5016 * 1: MOV TEMP[1].w, INPUT[4].wwww;
5017 * 2: TXP TEMP[2], INPUT[4].xyyw, texture[0], 2D;
5018 *
5019 * which allows for dead code elimination on TEMP[1]'s writes.
5020 */
5021 void
5022 glsl_to_tgsi_visitor::copy_propagate(void)
5023 {
5024 glsl_to_tgsi_instruction **acp = rzalloc_array(mem_ctx,
5025 glsl_to_tgsi_instruction *,
5026 this->next_temp * 4);
5027 int *acp_level = rzalloc_array(mem_ctx, int, this->next_temp * 4);
5028 int level = 0;
5029
5030 foreach_in_list(glsl_to_tgsi_instruction, inst, &this->instructions) {
5031 assert(inst->dst[0].file != PROGRAM_TEMPORARY
5032 || inst->dst[0].index < this->next_temp);
5033
5034 /* First, do any copy propagation possible into the src regs. */
5035 for (int r = 0; r < 3; r++) {
5036 glsl_to_tgsi_instruction *first = NULL;
5037 bool good = true;
5038 int acp_base = inst->src[r].index * 4;
5039
5040 if (inst->src[r].file != PROGRAM_TEMPORARY ||
5041 inst->src[r].reladdr ||
5042 inst->src[r].reladdr2)
5043 continue;
5044
5045 /* See if we can find entries in the ACP consisting of MOVs
5046 * from the same src register for all the swizzled channels
5047 * of this src register reference.
5048 */
5049 for (int i = 0; i < 4; i++) {
5050 int src_chan = GET_SWZ(inst->src[r].swizzle, i);
5051 glsl_to_tgsi_instruction *copy_chan = acp[acp_base + src_chan];
5052
5053 if (!copy_chan) {
5054 good = false;
5055 break;
5056 }
5057
5058 assert(acp_level[acp_base + src_chan] <= level);
5059
5060 if (!first) {
5061 first = copy_chan;
5062 } else {
5063 if (first->src[0].file != copy_chan->src[0].file ||
5064 first->src[0].index != copy_chan->src[0].index ||
5065 first->src[0].double_reg2 != copy_chan->src[0].double_reg2 ||
5066 first->src[0].index2D != copy_chan->src[0].index2D) {
5067 good = false;
5068 break;
5069 }
5070 }
5071 }
5072
5073 if (good) {
5074 /* We've now validated that we can copy-propagate to
5075 * replace this src register reference. Do it.
5076 */
5077 inst->src[r].file = first->src[0].file;
5078 inst->src[r].index = first->src[0].index;
5079 inst->src[r].index2D = first->src[0].index2D;
5080 inst->src[r].has_index2 = first->src[0].has_index2;
5081 inst->src[r].double_reg2 = first->src[0].double_reg2;
5082 inst->src[r].array_id = first->src[0].array_id;
5083
5084 int swizzle = 0;
5085 for (int i = 0; i < 4; i++) {
5086 int src_chan = GET_SWZ(inst->src[r].swizzle, i);
5087 glsl_to_tgsi_instruction *copy_inst = acp[acp_base + src_chan];
5088 swizzle |= (GET_SWZ(copy_inst->src[0].swizzle, src_chan) << (3 * i));
5089 }
5090 inst->src[r].swizzle = swizzle;
5091 }
5092 }
5093
5094 switch (inst->op) {
5095 case TGSI_OPCODE_BGNLOOP:
5096 case TGSI_OPCODE_ENDLOOP:
5097 /* End of a basic block, clear the ACP entirely. */
5098 memset(acp, 0, sizeof(*acp) * this->next_temp * 4);
5099 break;
5100
5101 case TGSI_OPCODE_IF:
5102 case TGSI_OPCODE_UIF:
5103 ++level;
5104 break;
5105
5106 case TGSI_OPCODE_ENDIF:
5107 case TGSI_OPCODE_ELSE:
5108 /* Clear all channels written inside the block from the ACP, but
5109 * leaving those that were not touched.
5110 */
5111 for (int r = 0; r < this->next_temp; r++) {
5112 for (int c = 0; c < 4; c++) {
5113 if (!acp[4 * r + c])
5114 continue;
5115
5116 if (acp_level[4 * r + c] >= level)
5117 acp[4 * r + c] = NULL;
5118 }
5119 }
5120 if (inst->op == TGSI_OPCODE_ENDIF)
5121 --level;
5122 break;
5123
5124 default:
5125 /* Continuing the block, clear any written channels from
5126 * the ACP.
5127 */
5128 for (int d = 0; d < 2; d++) {
5129 if (inst->dst[d].file == PROGRAM_TEMPORARY && inst->dst[d].reladdr) {
5130 /* Any temporary might be written, so no copy propagation
5131 * across this instruction.
5132 */
5133 memset(acp, 0, sizeof(*acp) * this->next_temp * 4);
5134 } else if (inst->dst[d].file == PROGRAM_OUTPUT &&
5135 inst->dst[d].reladdr) {
5136 /* Any output might be written, so no copy propagation
5137 * from outputs across this instruction.
5138 */
5139 for (int r = 0; r < this->next_temp; r++) {
5140 for (int c = 0; c < 4; c++) {
5141 if (!acp[4 * r + c])
5142 continue;
5143
5144 if (acp[4 * r + c]->src[0].file == PROGRAM_OUTPUT)
5145 acp[4 * r + c] = NULL;
5146 }
5147 }
5148 } else if (inst->dst[d].file == PROGRAM_TEMPORARY ||
5149 inst->dst[d].file == PROGRAM_OUTPUT) {
5150 /* Clear where it's used as dst. */
5151 if (inst->dst[d].file == PROGRAM_TEMPORARY) {
5152 for (int c = 0; c < 4; c++) {
5153 if (inst->dst[d].writemask & (1 << c))
5154 acp[4 * inst->dst[d].index + c] = NULL;
5155 }
5156 }
5157
5158 /* Clear where it's used as src. */
5159 for (int r = 0; r < this->next_temp; r++) {
5160 for (int c = 0; c < 4; c++) {
5161 if (!acp[4 * r + c])
5162 continue;
5163
5164 int src_chan = GET_SWZ(acp[4 * r + c]->src[0].swizzle, c);
5165
5166 if (acp[4 * r + c]->src[0].file == inst->dst[d].file &&
5167 acp[4 * r + c]->src[0].index == inst->dst[d].index &&
5168 inst->dst[d].writemask & (1 << src_chan)) {
5169 acp[4 * r + c] = NULL;
5170 }
5171 }
5172 }
5173 }
5174 }
5175 break;
5176 }
5177
5178 /* If this is a copy, add it to the ACP. */
5179 if (inst->op == TGSI_OPCODE_MOV &&
5180 inst->dst[0].file == PROGRAM_TEMPORARY &&
5181 !(inst->dst[0].file == inst->src[0].file &&
5182 inst->dst[0].index == inst->src[0].index) &&
5183 !inst->dst[0].reladdr &&
5184 !inst->dst[0].reladdr2 &&
5185 !inst->saturate &&
5186 inst->src[0].file != PROGRAM_ARRAY &&
5187 !inst->src[0].reladdr &&
5188 !inst->src[0].reladdr2 &&
5189 !inst->src[0].negate &&
5190 !inst->src[0].abs) {
5191 for (int i = 0; i < 4; i++) {
5192 if (inst->dst[0].writemask & (1 << i)) {
5193 acp[4 * inst->dst[0].index + i] = inst;
5194 acp_level[4 * inst->dst[0].index + i] = level;
5195 }
5196 }
5197 }
5198 }
5199
5200 ralloc_free(acp_level);
5201 ralloc_free(acp);
5202 }
5203
5204 /*
5205 * On a basic block basis, tracks available PROGRAM_TEMPORARY registers for dead
5206 * code elimination.
5207 *
5208 * The glsl_to_tgsi_visitor lazily produces code assuming that this pass
5209 * will occur. As an example, a TXP production after copy propagation but
5210 * before this pass:
5211 *
5212 * 0: MOV TEMP[1], INPUT[4].xyyy;
5213 * 1: MOV TEMP[1].w, INPUT[4].wwww;
5214 * 2: TXP TEMP[2], INPUT[4].xyyw, texture[0], 2D;
5215 *
5216 * and after this pass:
5217 *
5218 * 0: TXP TEMP[2], INPUT[4].xyyw, texture[0], 2D;
5219 */
5220 int
5221 glsl_to_tgsi_visitor::eliminate_dead_code(void)
5222 {
5223 glsl_to_tgsi_instruction **writes = rzalloc_array(mem_ctx,
5224 glsl_to_tgsi_instruction *,
5225 this->next_temp * 4);
5226 int *write_level = rzalloc_array(mem_ctx, int, this->next_temp * 4);
5227 int level = 0;
5228 int removed = 0;
5229
5230 foreach_in_list(glsl_to_tgsi_instruction, inst, &this->instructions) {
5231 assert(inst->dst[0].file != PROGRAM_TEMPORARY
5232 || inst->dst[0].index < this->next_temp);
5233
5234 switch (inst->op) {
5235 case TGSI_OPCODE_BGNLOOP:
5236 case TGSI_OPCODE_ENDLOOP:
5237 case TGSI_OPCODE_CONT:
5238 case TGSI_OPCODE_BRK:
5239 /* End of a basic block, clear the write array entirely.
5240 *
5241 * This keeps us from killing dead code when the writes are
5242 * on either side of a loop, even when the register isn't touched
5243 * inside the loop. However, glsl_to_tgsi_visitor doesn't seem to emit
5244 * dead code of this type, so it shouldn't make a difference as long as
5245 * the dead code elimination pass in the GLSL compiler does its job.
5246 */
5247 memset(writes, 0, sizeof(*writes) * this->next_temp * 4);
5248 break;
5249
5250 case TGSI_OPCODE_ENDIF:
5251 case TGSI_OPCODE_ELSE:
5252 /* Promote the recorded level of all channels written inside the
5253 * preceding if or else block to the level above the if/else block.
5254 */
5255 for (int r = 0; r < this->next_temp; r++) {
5256 for (int c = 0; c < 4; c++) {
5257 if (!writes[4 * r + c])
5258 continue;
5259
5260 if (write_level[4 * r + c] == level)
5261 write_level[4 * r + c] = level-1;
5262 }
5263 }
5264 if(inst->op == TGSI_OPCODE_ENDIF)
5265 --level;
5266 break;
5267
5268 case TGSI_OPCODE_IF:
5269 case TGSI_OPCODE_UIF:
5270 ++level;
5271 /* fallthrough to default case to mark the condition as read */
5272 default:
5273 /* Continuing the block, clear any channels from the write array that
5274 * are read by this instruction.
5275 */
5276 for (unsigned i = 0; i < ARRAY_SIZE(inst->src); i++) {
5277 if (inst->src[i].file == PROGRAM_TEMPORARY && inst->src[i].reladdr){
5278 /* Any temporary might be read, so no dead code elimination
5279 * across this instruction.
5280 */
5281 memset(writes, 0, sizeof(*writes) * this->next_temp * 4);
5282 } else if (inst->src[i].file == PROGRAM_TEMPORARY) {
5283 /* Clear where it's used as src. */
5284 int src_chans = 1 << GET_SWZ(inst->src[i].swizzle, 0);
5285 src_chans |= 1 << GET_SWZ(inst->src[i].swizzle, 1);
5286 src_chans |= 1 << GET_SWZ(inst->src[i].swizzle, 2);
5287 src_chans |= 1 << GET_SWZ(inst->src[i].swizzle, 3);
5288
5289 for (int c = 0; c < 4; c++) {
5290 if (src_chans & (1 << c))
5291 writes[4 * inst->src[i].index + c] = NULL;
5292 }
5293 }
5294 }
5295 for (unsigned i = 0; i < inst->tex_offset_num_offset; i++) {
5296 if (inst->tex_offsets[i].file == PROGRAM_TEMPORARY && inst->tex_offsets[i].reladdr){
5297 /* Any temporary might be read, so no dead code elimination
5298 * across this instruction.
5299 */
5300 memset(writes, 0, sizeof(*writes) * this->next_temp * 4);
5301 } else if (inst->tex_offsets[i].file == PROGRAM_TEMPORARY) {
5302 /* Clear where it's used as src. */
5303 int src_chans = 1 << GET_SWZ(inst->tex_offsets[i].swizzle, 0);
5304 src_chans |= 1 << GET_SWZ(inst->tex_offsets[i].swizzle, 1);
5305 src_chans |= 1 << GET_SWZ(inst->tex_offsets[i].swizzle, 2);
5306 src_chans |= 1 << GET_SWZ(inst->tex_offsets[i].swizzle, 3);
5307
5308 for (int c = 0; c < 4; c++) {
5309 if (src_chans & (1 << c))
5310 writes[4 * inst->tex_offsets[i].index + c] = NULL;
5311 }
5312 }
5313 }
5314
5315 if (inst->resource.file == PROGRAM_TEMPORARY) {
5316 int src_chans;
5317
5318 src_chans = 1 << GET_SWZ(inst->resource.swizzle, 0);
5319 src_chans |= 1 << GET_SWZ(inst->resource.swizzle, 1);
5320 src_chans |= 1 << GET_SWZ(inst->resource.swizzle, 2);
5321 src_chans |= 1 << GET_SWZ(inst->resource.swizzle, 3);
5322
5323 for (int c = 0; c < 4; c++) {
5324 if (src_chans & (1 << c))
5325 writes[4 * inst->resource.index + c] = NULL;
5326 }
5327 }
5328
5329 break;
5330 }
5331
5332 /* If this instruction writes to a temporary, add it to the write array.
5333 * If there is already an instruction in the write array for one or more
5334 * of the channels, flag that channel write as dead.
5335 */
5336 for (unsigned i = 0; i < ARRAY_SIZE(inst->dst); i++) {
5337 if (inst->dst[i].file == PROGRAM_TEMPORARY &&
5338 !inst->dst[i].reladdr) {
5339 for (int c = 0; c < 4; c++) {
5340 if (inst->dst[i].writemask & (1 << c)) {
5341 if (writes[4 * inst->dst[i].index + c]) {
5342 if (write_level[4 * inst->dst[i].index + c] < level)
5343 continue;
5344 else
5345 writes[4 * inst->dst[i].index + c]->dead_mask |= (1 << c);
5346 }
5347 writes[4 * inst->dst[i].index + c] = inst;
5348 write_level[4 * inst->dst[i].index + c] = level;
5349 }
5350 }
5351 }
5352 }
5353 }
5354
5355 /* Anything still in the write array at this point is dead code. */
5356 for (int r = 0; r < this->next_temp; r++) {
5357 for (int c = 0; c < 4; c++) {
5358 glsl_to_tgsi_instruction *inst = writes[4 * r + c];
5359 if (inst)
5360 inst->dead_mask |= (1 << c);
5361 }
5362 }
5363
5364 /* Now actually remove the instructions that are completely dead and update
5365 * the writemask of other instructions with dead channels.
5366 */
5367 foreach_in_list_safe(glsl_to_tgsi_instruction, inst, &this->instructions) {
5368 if (!inst->dead_mask || !inst->dst[0].writemask)
5369 continue;
5370 /* No amount of dead masks should remove memory stores */
5371 if (inst->info->is_store)
5372 continue;
5373
5374 if ((inst->dst[0].writemask & ~inst->dead_mask) == 0) {
5375 inst->remove();
5376 delete inst;
5377 removed++;
5378 } else {
5379 if (glsl_base_type_is_64bit(inst->dst[0].type)) {
5380 if (inst->dead_mask == WRITEMASK_XY ||
5381 inst->dead_mask == WRITEMASK_ZW)
5382 inst->dst[0].writemask &= ~(inst->dead_mask);
5383 } else
5384 inst->dst[0].writemask &= ~(inst->dead_mask);
5385 }
5386 }
5387
5388 ralloc_free(write_level);
5389 ralloc_free(writes);
5390
5391 return removed;
5392 }
5393
5394 /* merge DFRACEXP instructions into one. */
5395 void
5396 glsl_to_tgsi_visitor::merge_two_dsts(void)
5397 {
5398 foreach_in_list_safe(glsl_to_tgsi_instruction, inst, &this->instructions) {
5399 glsl_to_tgsi_instruction *inst2;
5400 bool merged;
5401 if (num_inst_dst_regs(inst) != 2)
5402 continue;
5403
5404 if (inst->dst[0].file != PROGRAM_UNDEFINED &&
5405 inst->dst[1].file != PROGRAM_UNDEFINED)
5406 continue;
5407
5408 inst2 = (glsl_to_tgsi_instruction *) inst->next;
5409 do {
5410
5411 if (inst->src[0].file == inst2->src[0].file &&
5412 inst->src[0].index == inst2->src[0].index &&
5413 inst->src[0].type == inst2->src[0].type &&
5414 inst->src[0].swizzle == inst2->src[0].swizzle)
5415 break;
5416 inst2 = (glsl_to_tgsi_instruction *) inst2->next;
5417 } while (inst2);
5418
5419 if (!inst2)
5420 continue;
5421 merged = false;
5422 if (inst->dst[0].file == PROGRAM_UNDEFINED) {
5423 merged = true;
5424 inst->dst[0] = inst2->dst[0];
5425 } else if (inst->dst[1].file == PROGRAM_UNDEFINED) {
5426 inst->dst[1] = inst2->dst[1];
5427 merged = true;
5428 }
5429
5430 if (merged) {
5431 inst2->remove();
5432 delete inst2;
5433 }
5434 }
5435 }
5436
5437 /* Merges temporary registers together where possible to reduce the number of
5438 * registers needed to run a program.
5439 *
5440 * Produces optimal code only after copy propagation and dead code elimination
5441 * have been run. */
5442 void
5443 glsl_to_tgsi_visitor::merge_registers(void)
5444 {
5445 int *last_reads = ralloc_array(mem_ctx, int, this->next_temp);
5446 int *first_writes = ralloc_array(mem_ctx, int, this->next_temp);
5447 struct rename_reg_pair *renames = rzalloc_array(mem_ctx, struct rename_reg_pair, this->next_temp);
5448 int i, j;
5449 int num_renames = 0;
5450
5451 /* Read the indices of the last read and first write to each temp register
5452 * into an array so that we don't have to traverse the instruction list as
5453 * much. */
5454 for (i = 0; i < this->next_temp; i++) {
5455 last_reads[i] = -1;
5456 first_writes[i] = -1;
5457 }
5458 get_last_temp_read_first_temp_write(last_reads, first_writes);
5459
5460 /* Start looking for registers with non-overlapping usages that can be
5461 * merged together. */
5462 for (i = 0; i < this->next_temp; i++) {
5463 /* Don't touch unused registers. */
5464 if (last_reads[i] < 0 || first_writes[i] < 0) continue;
5465
5466 for (j = 0; j < this->next_temp; j++) {
5467 /* Don't touch unused registers. */
5468 if (last_reads[j] < 0 || first_writes[j] < 0) continue;
5469
5470 /* We can merge the two registers if the first write to j is after or
5471 * in the same instruction as the last read from i. Note that the
5472 * register at index i will always be used earlier or at the same time
5473 * as the register at index j. */
5474 if (first_writes[i] <= first_writes[j] &&
5475 last_reads[i] <= first_writes[j]) {
5476 renames[num_renames].old_reg = j;
5477 renames[num_renames].new_reg = i;
5478 num_renames++;
5479
5480 /* Update the first_writes and last_reads arrays with the new
5481 * values for the merged register index, and mark the newly unused
5482 * register index as such. */
5483 assert(last_reads[j] >= last_reads[i]);
5484 last_reads[i] = last_reads[j];
5485 first_writes[j] = -1;
5486 last_reads[j] = -1;
5487 }
5488 }
5489 }
5490
5491 rename_temp_registers(num_renames, renames);
5492 ralloc_free(renames);
5493 ralloc_free(last_reads);
5494 ralloc_free(first_writes);
5495 }
5496
5497 /* Reassign indices to temporary registers by reusing unused indices created
5498 * by optimization passes. */
5499 void
5500 glsl_to_tgsi_visitor::renumber_registers(void)
5501 {
5502 int i = 0;
5503 int new_index = 0;
5504 int *first_writes = ralloc_array(mem_ctx, int, this->next_temp);
5505 struct rename_reg_pair *renames = rzalloc_array(mem_ctx, struct rename_reg_pair, this->next_temp);
5506 int num_renames = 0;
5507
5508 for (i = 0; i < this->next_temp; i++) {
5509 first_writes[i] = -1;
5510 }
5511 get_first_temp_write(first_writes);
5512
5513 for (i = 0; i < this->next_temp; i++) {
5514 if (first_writes[i] < 0) continue;
5515 if (i != new_index) {
5516 renames[num_renames].old_reg = i;
5517 renames[num_renames].new_reg = new_index;
5518 num_renames++;
5519 }
5520 new_index++;
5521 }
5522
5523 rename_temp_registers(num_renames, renames);
5524 this->next_temp = new_index;
5525 ralloc_free(renames);
5526 ralloc_free(first_writes);
5527 }
5528
5529 /* ------------------------- TGSI conversion stuff -------------------------- */
5530
5531 /**
5532 * Intermediate state used during shader translation.
5533 */
5534 struct st_translate {
5535 struct ureg_program *ureg;
5536
5537 unsigned temps_size;
5538 struct ureg_dst *temps;
5539
5540 struct ureg_dst *arrays;
5541 unsigned num_temp_arrays;
5542 struct ureg_src *constants;
5543 int num_constants;
5544 struct ureg_src *immediates;
5545 int num_immediates;
5546 struct ureg_dst outputs[PIPE_MAX_SHADER_OUTPUTS];
5547 struct ureg_src inputs[PIPE_MAX_SHADER_INPUTS];
5548 struct ureg_dst address[3];
5549 struct ureg_src samplers[PIPE_MAX_SAMPLERS];
5550 struct ureg_src buffers[PIPE_MAX_SHADER_BUFFERS];
5551 struct ureg_src images[PIPE_MAX_SHADER_IMAGES];
5552 struct ureg_src systemValues[SYSTEM_VALUE_MAX];
5553 struct ureg_src shared_memory;
5554 unsigned *array_sizes;
5555 struct inout_decl *input_decls;
5556 unsigned num_input_decls;
5557 struct inout_decl *output_decls;
5558 unsigned num_output_decls;
5559
5560 const ubyte *inputMapping;
5561 const ubyte *outputMapping;
5562
5563 unsigned procType; /**< PIPE_SHADER_VERTEX/FRAGMENT */
5564 };
5565
5566 /** Map Mesa's SYSTEM_VALUE_x to TGSI_SEMANTIC_x */
5567 unsigned
5568 _mesa_sysval_to_semantic(unsigned sysval)
5569 {
5570 switch (sysval) {
5571 /* Vertex shader */
5572 case SYSTEM_VALUE_VERTEX_ID:
5573 return TGSI_SEMANTIC_VERTEXID;
5574 case SYSTEM_VALUE_INSTANCE_ID:
5575 return TGSI_SEMANTIC_INSTANCEID;
5576 case SYSTEM_VALUE_VERTEX_ID_ZERO_BASE:
5577 return TGSI_SEMANTIC_VERTEXID_NOBASE;
5578 case SYSTEM_VALUE_BASE_VERTEX:
5579 return TGSI_SEMANTIC_BASEVERTEX;
5580 case SYSTEM_VALUE_BASE_INSTANCE:
5581 return TGSI_SEMANTIC_BASEINSTANCE;
5582 case SYSTEM_VALUE_DRAW_ID:
5583 return TGSI_SEMANTIC_DRAWID;
5584
5585 /* Geometry shader */
5586 case SYSTEM_VALUE_INVOCATION_ID:
5587 return TGSI_SEMANTIC_INVOCATIONID;
5588
5589 /* Fragment shader */
5590 case SYSTEM_VALUE_FRAG_COORD:
5591 return TGSI_SEMANTIC_POSITION;
5592 case SYSTEM_VALUE_FRONT_FACE:
5593 return TGSI_SEMANTIC_FACE;
5594 case SYSTEM_VALUE_SAMPLE_ID:
5595 return TGSI_SEMANTIC_SAMPLEID;
5596 case SYSTEM_VALUE_SAMPLE_POS:
5597 return TGSI_SEMANTIC_SAMPLEPOS;
5598 case SYSTEM_VALUE_SAMPLE_MASK_IN:
5599 return TGSI_SEMANTIC_SAMPLEMASK;
5600 case SYSTEM_VALUE_HELPER_INVOCATION:
5601 return TGSI_SEMANTIC_HELPER_INVOCATION;
5602
5603 /* Tessellation shader */
5604 case SYSTEM_VALUE_TESS_COORD:
5605 return TGSI_SEMANTIC_TESSCOORD;
5606 case SYSTEM_VALUE_VERTICES_IN:
5607 return TGSI_SEMANTIC_VERTICESIN;
5608 case SYSTEM_VALUE_PRIMITIVE_ID:
5609 return TGSI_SEMANTIC_PRIMID;
5610 case SYSTEM_VALUE_TESS_LEVEL_OUTER:
5611 return TGSI_SEMANTIC_TESSOUTER;
5612 case SYSTEM_VALUE_TESS_LEVEL_INNER:
5613 return TGSI_SEMANTIC_TESSINNER;
5614
5615 /* Compute shader */
5616 case SYSTEM_VALUE_LOCAL_INVOCATION_ID:
5617 return TGSI_SEMANTIC_THREAD_ID;
5618 case SYSTEM_VALUE_WORK_GROUP_ID:
5619 return TGSI_SEMANTIC_BLOCK_ID;
5620 case SYSTEM_VALUE_NUM_WORK_GROUPS:
5621 return TGSI_SEMANTIC_GRID_SIZE;
5622 case SYSTEM_VALUE_LOCAL_GROUP_SIZE:
5623 return TGSI_SEMANTIC_BLOCK_SIZE;
5624
5625 /* ARB_shader_ballot */
5626 case SYSTEM_VALUE_SUBGROUP_SIZE:
5627 return TGSI_SEMANTIC_SUBGROUP_SIZE;
5628 case SYSTEM_VALUE_SUBGROUP_INVOCATION:
5629 return TGSI_SEMANTIC_SUBGROUP_INVOCATION;
5630 case SYSTEM_VALUE_SUBGROUP_EQ_MASK:
5631 return TGSI_SEMANTIC_SUBGROUP_EQ_MASK;
5632 case SYSTEM_VALUE_SUBGROUP_GE_MASK:
5633 return TGSI_SEMANTIC_SUBGROUP_GE_MASK;
5634 case SYSTEM_VALUE_SUBGROUP_GT_MASK:
5635 return TGSI_SEMANTIC_SUBGROUP_GT_MASK;
5636 case SYSTEM_VALUE_SUBGROUP_LE_MASK:
5637 return TGSI_SEMANTIC_SUBGROUP_LE_MASK;
5638 case SYSTEM_VALUE_SUBGROUP_LT_MASK:
5639 return TGSI_SEMANTIC_SUBGROUP_LT_MASK;
5640
5641 /* Unhandled */
5642 case SYSTEM_VALUE_LOCAL_INVOCATION_INDEX:
5643 case SYSTEM_VALUE_GLOBAL_INVOCATION_ID:
5644 case SYSTEM_VALUE_VERTEX_CNT:
5645 default:
5646 assert(!"Unexpected SYSTEM_VALUE_ enum");
5647 return TGSI_SEMANTIC_COUNT;
5648 }
5649 }
5650
5651 /**
5652 * Map a glsl_to_tgsi constant/immediate to a TGSI immediate.
5653 */
5654 static struct ureg_src
5655 emit_immediate(struct st_translate *t,
5656 gl_constant_value values[4],
5657 int type, int size)
5658 {
5659 struct ureg_program *ureg = t->ureg;
5660
5661 switch(type)
5662 {
5663 case GL_FLOAT:
5664 return ureg_DECL_immediate(ureg, &values[0].f, size);
5665 case GL_DOUBLE:
5666 return ureg_DECL_immediate_f64(ureg, (double *)&values[0].f, size);
5667 case GL_INT64_ARB:
5668 return ureg_DECL_immediate_int64(ureg, (int64_t *)&values[0].f, size);
5669 case GL_UNSIGNED_INT64_ARB:
5670 return ureg_DECL_immediate_uint64(ureg, (uint64_t *)&values[0].f, size);
5671 case GL_INT:
5672 return ureg_DECL_immediate_int(ureg, &values[0].i, size);
5673 case GL_UNSIGNED_INT:
5674 case GL_BOOL:
5675 return ureg_DECL_immediate_uint(ureg, &values[0].u, size);
5676 default:
5677 assert(!"should not get here - type must be float, int, uint, or bool");
5678 return ureg_src_undef();
5679 }
5680 }
5681
5682 /**
5683 * Map a glsl_to_tgsi dst register to a TGSI ureg_dst register.
5684 */
5685 static struct ureg_dst
5686 dst_register(struct st_translate *t, gl_register_file file, unsigned index,
5687 unsigned array_id)
5688 {
5689 unsigned array;
5690
5691 switch(file) {
5692 case PROGRAM_UNDEFINED:
5693 return ureg_dst_undef();
5694
5695 case PROGRAM_TEMPORARY:
5696 /* Allocate space for temporaries on demand. */
5697 if (index >= t->temps_size) {
5698 const int inc = align(index - t->temps_size + 1, 4096);
5699
5700 t->temps = (struct ureg_dst*)
5701 realloc(t->temps,
5702 (t->temps_size + inc) * sizeof(struct ureg_dst));
5703 if (!t->temps)
5704 return ureg_dst_undef();
5705
5706 memset(t->temps + t->temps_size, 0, inc * sizeof(struct ureg_dst));
5707 t->temps_size += inc;
5708 }
5709
5710 if (ureg_dst_is_undef(t->temps[index]))
5711 t->temps[index] = ureg_DECL_local_temporary(t->ureg);
5712
5713 return t->temps[index];
5714
5715 case PROGRAM_ARRAY:
5716 assert(array_id && array_id <= t->num_temp_arrays);
5717 array = array_id - 1;
5718
5719 if (ureg_dst_is_undef(t->arrays[array]))
5720 t->arrays[array] = ureg_DECL_array_temporary(
5721 t->ureg, t->array_sizes[array], TRUE);
5722
5723 return ureg_dst_array_offset(t->arrays[array], index);
5724
5725 case PROGRAM_OUTPUT:
5726 if (!array_id) {
5727 if (t->procType == PIPE_SHADER_FRAGMENT)
5728 assert(index < 2 * FRAG_RESULT_MAX);
5729 else if (t->procType == PIPE_SHADER_TESS_CTRL ||
5730 t->procType == PIPE_SHADER_TESS_EVAL)
5731 assert(index < VARYING_SLOT_TESS_MAX);
5732 else
5733 assert(index < VARYING_SLOT_MAX);
5734
5735 assert(t->outputMapping[index] < ARRAY_SIZE(t->outputs));
5736 assert(t->outputs[t->outputMapping[index]].File != TGSI_FILE_NULL);
5737 return t->outputs[t->outputMapping[index]];
5738 }
5739 else {
5740 struct inout_decl *decl = find_inout_array(t->output_decls, t->num_output_decls, array_id);
5741 unsigned mesa_index = decl->mesa_index;
5742 int slot = t->outputMapping[mesa_index];
5743
5744 assert(slot != -1 && t->outputs[slot].File == TGSI_FILE_OUTPUT);
5745
5746 struct ureg_dst dst = t->outputs[slot];
5747 dst.ArrayID = array_id;
5748 return ureg_dst_array_offset(dst, index - mesa_index);
5749 }
5750
5751 case PROGRAM_ADDRESS:
5752 return t->address[index];
5753
5754 default:
5755 assert(!"unknown dst register file");
5756 return ureg_dst_undef();
5757 }
5758 }
5759
5760 /**
5761 * Map a glsl_to_tgsi src register to a TGSI ureg_src register.
5762 */
5763 static struct ureg_src
5764 src_register(struct st_translate *t, const st_src_reg *reg)
5765 {
5766 int index = reg->index;
5767 int double_reg2 = reg->double_reg2 ? 1 : 0;
5768
5769 switch(reg->file) {
5770 case PROGRAM_UNDEFINED:
5771 return ureg_imm4f(t->ureg, 0, 0, 0, 0);
5772
5773 case PROGRAM_TEMPORARY:
5774 case PROGRAM_ARRAY:
5775 return ureg_src(dst_register(t, reg->file, reg->index, reg->array_id));
5776
5777 case PROGRAM_OUTPUT: {
5778 struct ureg_dst dst = dst_register(t, reg->file, reg->index, reg->array_id);
5779 assert(dst.WriteMask != 0);
5780 unsigned shift = ffs(dst.WriteMask) - 1;
5781 return ureg_swizzle(ureg_src(dst),
5782 shift,
5783 MIN2(shift + 1, 3),
5784 MIN2(shift + 2, 3),
5785 MIN2(shift + 3, 3));
5786 }
5787
5788 case PROGRAM_UNIFORM:
5789 assert(reg->index >= 0);
5790 return reg->index < t->num_constants ?
5791 t->constants[reg->index] : ureg_imm4f(t->ureg, 0, 0, 0, 0);
5792 case PROGRAM_STATE_VAR:
5793 case PROGRAM_CONSTANT: /* ie, immediate */
5794 if (reg->has_index2)
5795 return ureg_src_register(TGSI_FILE_CONSTANT, reg->index);
5796 else
5797 return reg->index >= 0 && reg->index < t->num_constants ?
5798 t->constants[reg->index] : ureg_imm4f(t->ureg, 0, 0, 0, 0);
5799
5800 case PROGRAM_IMMEDIATE:
5801 assert(reg->index >= 0 && reg->index < t->num_immediates);
5802 return t->immediates[reg->index];
5803
5804 case PROGRAM_INPUT:
5805 /* GLSL inputs are 64-bit containers, so we have to
5806 * map back to the original index and add the offset after
5807 * mapping. */
5808 index -= double_reg2;
5809 if (!reg->array_id) {
5810 assert(t->inputMapping[index] < ARRAY_SIZE(t->inputs));
5811 assert(t->inputs[t->inputMapping[index]].File != TGSI_FILE_NULL);
5812 return t->inputs[t->inputMapping[index] + double_reg2];
5813 }
5814 else {
5815 struct inout_decl *decl = find_inout_array(t->input_decls, t->num_input_decls, reg->array_id);
5816 unsigned mesa_index = decl->mesa_index;
5817 int slot = t->inputMapping[mesa_index];
5818
5819 assert(slot != -1 && t->inputs[slot].File == TGSI_FILE_INPUT);
5820
5821 struct ureg_src src = t->inputs[slot];
5822 src.ArrayID = reg->array_id;
5823 return ureg_src_array_offset(src, index + double_reg2 - mesa_index);
5824 }
5825
5826 case PROGRAM_ADDRESS:
5827 return ureg_src(t->address[reg->index]);
5828
5829 case PROGRAM_SYSTEM_VALUE:
5830 assert(reg->index < (int) ARRAY_SIZE(t->systemValues));
5831 return t->systemValues[reg->index];
5832
5833 default:
5834 assert(!"unknown src register file");
5835 return ureg_src_undef();
5836 }
5837 }
5838
5839 /**
5840 * Create a TGSI ureg_dst register from an st_dst_reg.
5841 */
5842 static struct ureg_dst
5843 translate_dst(struct st_translate *t,
5844 const st_dst_reg *dst_reg,
5845 bool saturate)
5846 {
5847 struct ureg_dst dst = dst_register(t, dst_reg->file, dst_reg->index,
5848 dst_reg->array_id);
5849
5850 if (dst.File == TGSI_FILE_NULL)
5851 return dst;
5852
5853 dst = ureg_writemask(dst, dst_reg->writemask);
5854
5855 if (saturate)
5856 dst = ureg_saturate(dst);
5857
5858 if (dst_reg->reladdr != NULL) {
5859 assert(dst_reg->file != PROGRAM_TEMPORARY);
5860 dst = ureg_dst_indirect(dst, ureg_src(t->address[0]));
5861 }
5862
5863 if (dst_reg->has_index2) {
5864 if (dst_reg->reladdr2)
5865 dst = ureg_dst_dimension_indirect(dst, ureg_src(t->address[1]),
5866 dst_reg->index2D);
5867 else
5868 dst = ureg_dst_dimension(dst, dst_reg->index2D);
5869 }
5870
5871 return dst;
5872 }
5873
5874 /**
5875 * Create a TGSI ureg_src register from an st_src_reg.
5876 */
5877 static struct ureg_src
5878 translate_src(struct st_translate *t, const st_src_reg *src_reg)
5879 {
5880 struct ureg_src src = src_register(t, src_reg);
5881
5882 if (src_reg->has_index2) {
5883 /* 2D indexes occur with geometry shader inputs (attrib, vertex)
5884 * and UBO constant buffers (buffer, position).
5885 */
5886 if (src_reg->reladdr2)
5887 src = ureg_src_dimension_indirect(src, ureg_src(t->address[1]),
5888 src_reg->index2D);
5889 else
5890 src = ureg_src_dimension(src, src_reg->index2D);
5891 }
5892
5893 src = ureg_swizzle(src,
5894 GET_SWZ(src_reg->swizzle, 0) & 0x3,
5895 GET_SWZ(src_reg->swizzle, 1) & 0x3,
5896 GET_SWZ(src_reg->swizzle, 2) & 0x3,
5897 GET_SWZ(src_reg->swizzle, 3) & 0x3);
5898
5899 if (src_reg->abs)
5900 src = ureg_abs(src);
5901
5902 if ((src_reg->negate & 0xf) == NEGATE_XYZW)
5903 src = ureg_negate(src);
5904
5905 if (src_reg->reladdr != NULL) {
5906 assert(src_reg->file != PROGRAM_TEMPORARY);
5907 src = ureg_src_indirect(src, ureg_src(t->address[0]));
5908 }
5909
5910 return src;
5911 }
5912
5913 static struct tgsi_texture_offset
5914 translate_tex_offset(struct st_translate *t,
5915 const st_src_reg *in_offset)
5916 {
5917 struct tgsi_texture_offset offset;
5918 struct ureg_src src = translate_src(t, in_offset);
5919
5920 offset.File = src.File;
5921 offset.Index = src.Index;
5922 offset.SwizzleX = src.SwizzleX;
5923 offset.SwizzleY = src.SwizzleY;
5924 offset.SwizzleZ = src.SwizzleZ;
5925 offset.Padding = 0;
5926
5927 assert(!src.Indirect);
5928 assert(!src.DimIndirect);
5929 assert(!src.Dimension);
5930 assert(!src.Absolute); /* those shouldn't be used with integers anyway */
5931 assert(!src.Negate);
5932
5933 return offset;
5934 }
5935
5936 static void
5937 compile_tgsi_instruction(struct st_translate *t,
5938 const glsl_to_tgsi_instruction *inst)
5939 {
5940 struct ureg_program *ureg = t->ureg;
5941 int i;
5942 struct ureg_dst dst[2];
5943 struct ureg_src src[4];
5944 struct tgsi_texture_offset texoffsets[MAX_GLSL_TEXTURE_OFFSET];
5945
5946 int num_dst;
5947 int num_src;
5948 unsigned tex_target = 0;
5949
5950 num_dst = num_inst_dst_regs(inst);
5951 num_src = num_inst_src_regs(inst);
5952
5953 for (i = 0; i < num_dst; i++)
5954 dst[i] = translate_dst(t,
5955 &inst->dst[i],
5956 inst->saturate);
5957
5958 for (i = 0; i < num_src; i++)
5959 src[i] = translate_src(t, &inst->src[i]);
5960
5961 switch(inst->op) {
5962 case TGSI_OPCODE_BGNLOOP:
5963 case TGSI_OPCODE_ELSE:
5964 case TGSI_OPCODE_ENDLOOP:
5965 case TGSI_OPCODE_IF:
5966 case TGSI_OPCODE_UIF:
5967 assert(num_dst == 0);
5968 ureg_insn(ureg, inst->op, NULL, 0, src, num_src, inst->precise);
5969 return;
5970
5971 case TGSI_OPCODE_TEX:
5972 case TGSI_OPCODE_TEX_LZ:
5973 case TGSI_OPCODE_TXB:
5974 case TGSI_OPCODE_TXD:
5975 case TGSI_OPCODE_TXL:
5976 case TGSI_OPCODE_TXP:
5977 case TGSI_OPCODE_TXQ:
5978 case TGSI_OPCODE_TXQS:
5979 case TGSI_OPCODE_TXF:
5980 case TGSI_OPCODE_TXF_LZ:
5981 case TGSI_OPCODE_TEX2:
5982 case TGSI_OPCODE_TXB2:
5983 case TGSI_OPCODE_TXL2:
5984 case TGSI_OPCODE_TG4:
5985 case TGSI_OPCODE_LODQ:
5986 if (inst->resource.file == PROGRAM_SAMPLER) {
5987 src[num_src] = t->samplers[inst->resource.index];
5988 } else {
5989 /* Bindless samplers. */
5990 src[num_src] = translate_src(t, &inst->resource);
5991 }
5992 assert(src[num_src].File != TGSI_FILE_NULL);
5993 if (inst->resource.reladdr)
5994 src[num_src] =
5995 ureg_src_indirect(src[num_src], ureg_src(t->address[2]));
5996 num_src++;
5997 for (i = 0; i < (int)inst->tex_offset_num_offset; i++) {
5998 texoffsets[i] = translate_tex_offset(t, &inst->tex_offsets[i]);
5999 }
6000 tex_target = st_translate_texture_target(inst->tex_target, inst->tex_shadow);
6001
6002 ureg_tex_insn(ureg,
6003 inst->op,
6004 dst, num_dst,
6005 tex_target,
6006 st_translate_texture_type(inst->tex_type),
6007 texoffsets, inst->tex_offset_num_offset,
6008 src, num_src);
6009 return;
6010
6011 case TGSI_OPCODE_RESQ:
6012 case TGSI_OPCODE_LOAD:
6013 case TGSI_OPCODE_ATOMUADD:
6014 case TGSI_OPCODE_ATOMXCHG:
6015 case TGSI_OPCODE_ATOMCAS:
6016 case TGSI_OPCODE_ATOMAND:
6017 case TGSI_OPCODE_ATOMOR:
6018 case TGSI_OPCODE_ATOMXOR:
6019 case TGSI_OPCODE_ATOMUMIN:
6020 case TGSI_OPCODE_ATOMUMAX:
6021 case TGSI_OPCODE_ATOMIMIN:
6022 case TGSI_OPCODE_ATOMIMAX:
6023 for (i = num_src - 1; i >= 0; i--)
6024 src[i + 1] = src[i];
6025 num_src++;
6026 if (inst->resource.file == PROGRAM_MEMORY) {
6027 src[0] = t->shared_memory;
6028 } else if (inst->resource.file == PROGRAM_BUFFER) {
6029 src[0] = t->buffers[inst->resource.index];
6030 } else {
6031 if (inst->resource.file == PROGRAM_IMAGE) {
6032 src[0] = t->images[inst->resource.index];
6033 } else {
6034 /* Bindless images. */
6035 src[0] = translate_src(t, &inst->resource);
6036 }
6037 tex_target = st_translate_texture_target(inst->tex_target, inst->tex_shadow);
6038 }
6039 if (inst->resource.reladdr)
6040 src[0] = ureg_src_indirect(src[0], ureg_src(t->address[2]));
6041 assert(src[0].File != TGSI_FILE_NULL);
6042 ureg_memory_insn(ureg, inst->op, dst, num_dst, src, num_src,
6043 inst->buffer_access,
6044 tex_target, inst->image_format);
6045 break;
6046
6047 case TGSI_OPCODE_STORE:
6048 if (inst->resource.file == PROGRAM_MEMORY) {
6049 dst[0] = ureg_dst(t->shared_memory);
6050 } else if (inst->resource.file == PROGRAM_BUFFER) {
6051 dst[0] = ureg_dst(t->buffers[inst->resource.index]);
6052 } else {
6053 if (inst->resource.file == PROGRAM_IMAGE) {
6054 dst[0] = ureg_dst(t->images[inst->resource.index]);
6055 } else {
6056 /* Bindless images. */
6057 dst[0] = ureg_dst(translate_src(t, &inst->resource));
6058 }
6059 tex_target = st_translate_texture_target(inst->tex_target, inst->tex_shadow);
6060 }
6061 dst[0] = ureg_writemask(dst[0], inst->dst[0].writemask);
6062 if (inst->resource.reladdr)
6063 dst[0] = ureg_dst_indirect(dst[0], ureg_src(t->address[2]));
6064 assert(dst[0].File != TGSI_FILE_NULL);
6065 ureg_memory_insn(ureg, inst->op, dst, num_dst, src, num_src,
6066 inst->buffer_access,
6067 tex_target, inst->image_format);
6068 break;
6069
6070 case TGSI_OPCODE_SCS:
6071 dst[0] = ureg_writemask(dst[0], TGSI_WRITEMASK_XY);
6072 ureg_insn(ureg, inst->op, dst, num_dst, src, num_src, inst->precise);
6073 break;
6074
6075 default:
6076 ureg_insn(ureg,
6077 inst->op,
6078 dst, num_dst,
6079 src, num_src, inst->precise);
6080 break;
6081 }
6082 }
6083
6084 /**
6085 * Emit the TGSI instructions for inverting and adjusting WPOS.
6086 * This code is unavoidable because it also depends on whether
6087 * a FBO is bound (STATE_FB_WPOS_Y_TRANSFORM).
6088 */
6089 static void
6090 emit_wpos_adjustment(struct gl_context *ctx,
6091 struct st_translate *t,
6092 int wpos_transform_const,
6093 boolean invert,
6094 GLfloat adjX, GLfloat adjY[2])
6095 {
6096 struct ureg_program *ureg = t->ureg;
6097
6098 assert(wpos_transform_const >= 0);
6099
6100 /* Fragment program uses fragment position input.
6101 * Need to replace instances of INPUT[WPOS] with temp T
6102 * where T = INPUT[WPOS] is inverted by Y.
6103 */
6104 struct ureg_src wpostrans = ureg_DECL_constant(ureg, wpos_transform_const);
6105 struct ureg_dst wpos_temp = ureg_DECL_temporary( ureg );
6106 struct ureg_src *wpos =
6107 ctx->Const.GLSLFragCoordIsSysVal ?
6108 &t->systemValues[SYSTEM_VALUE_FRAG_COORD] :
6109 &t->inputs[t->inputMapping[VARYING_SLOT_POS]];
6110 struct ureg_src wpos_input = *wpos;
6111
6112 /* First, apply the coordinate shift: */
6113 if (adjX || adjY[0] || adjY[1]) {
6114 if (adjY[0] != adjY[1]) {
6115 /* Adjust the y coordinate by adjY[1] or adjY[0] respectively
6116 * depending on whether inversion is actually going to be applied
6117 * or not, which is determined by testing against the inversion
6118 * state variable used below, which will be either +1 or -1.
6119 */
6120 struct ureg_dst adj_temp = ureg_DECL_local_temporary(ureg);
6121
6122 ureg_CMP(ureg, adj_temp,
6123 ureg_scalar(wpostrans, invert ? 2 : 0),
6124 ureg_imm4f(ureg, adjX, adjY[0], 0.0f, 0.0f),
6125 ureg_imm4f(ureg, adjX, adjY[1], 0.0f, 0.0f));
6126 ureg_ADD(ureg, wpos_temp, wpos_input, ureg_src(adj_temp));
6127 } else {
6128 ureg_ADD(ureg, wpos_temp, wpos_input,
6129 ureg_imm4f(ureg, adjX, adjY[0], 0.0f, 0.0f));
6130 }
6131 wpos_input = ureg_src(wpos_temp);
6132 } else {
6133 /* MOV wpos_temp, input[wpos]
6134 */
6135 ureg_MOV( ureg, wpos_temp, wpos_input );
6136 }
6137
6138 /* Now the conditional y flip: STATE_FB_WPOS_Y_TRANSFORM.xy/zw will be
6139 * inversion/identity, or the other way around if we're drawing to an FBO.
6140 */
6141 if (invert) {
6142 /* MAD wpos_temp.y, wpos_input, wpostrans.xxxx, wpostrans.yyyy
6143 */
6144 ureg_MAD( ureg,
6145 ureg_writemask(wpos_temp, TGSI_WRITEMASK_Y ),
6146 wpos_input,
6147 ureg_scalar(wpostrans, 0),
6148 ureg_scalar(wpostrans, 1));
6149 } else {
6150 /* MAD wpos_temp.y, wpos_input, wpostrans.zzzz, wpostrans.wwww
6151 */
6152 ureg_MAD( ureg,
6153 ureg_writemask(wpos_temp, TGSI_WRITEMASK_Y ),
6154 wpos_input,
6155 ureg_scalar(wpostrans, 2),
6156 ureg_scalar(wpostrans, 3));
6157 }
6158
6159 /* Use wpos_temp as position input from here on:
6160 */
6161 *wpos = ureg_src(wpos_temp);
6162 }
6163
6164
6165 /**
6166 * Emit fragment position/ooordinate code.
6167 */
6168 static void
6169 emit_wpos(struct st_context *st,
6170 struct st_translate *t,
6171 const struct gl_program *program,
6172 struct ureg_program *ureg,
6173 int wpos_transform_const)
6174 {
6175 struct pipe_screen *pscreen = st->pipe->screen;
6176 GLfloat adjX = 0.0f;
6177 GLfloat adjY[2] = { 0.0f, 0.0f };
6178 boolean invert = FALSE;
6179
6180 /* Query the pixel center conventions supported by the pipe driver and set
6181 * adjX, adjY to help out if it cannot handle the requested one internally.
6182 *
6183 * The bias of the y-coordinate depends on whether y-inversion takes place
6184 * (adjY[1]) or not (adjY[0]), which is in turn dependent on whether we are
6185 * drawing to an FBO (causes additional inversion), and whether the pipe
6186 * driver origin and the requested origin differ (the latter condition is
6187 * stored in the 'invert' variable).
6188 *
6189 * For height = 100 (i = integer, h = half-integer, l = lower, u = upper):
6190 *
6191 * center shift only:
6192 * i -> h: +0.5
6193 * h -> i: -0.5
6194 *
6195 * inversion only:
6196 * l,i -> u,i: ( 0.0 + 1.0) * -1 + 100 = 99
6197 * l,h -> u,h: ( 0.5 + 0.0) * -1 + 100 = 99.5
6198 * u,i -> l,i: (99.0 + 1.0) * -1 + 100 = 0
6199 * u,h -> l,h: (99.5 + 0.0) * -1 + 100 = 0.5
6200 *
6201 * inversion and center shift:
6202 * l,i -> u,h: ( 0.0 + 0.5) * -1 + 100 = 99.5
6203 * l,h -> u,i: ( 0.5 + 0.5) * -1 + 100 = 99
6204 * u,i -> l,h: (99.0 + 0.5) * -1 + 100 = 0.5
6205 * u,h -> l,i: (99.5 + 0.5) * -1 + 100 = 0
6206 */
6207 if (program->OriginUpperLeft) {
6208 /* Fragment shader wants origin in upper-left */
6209 if (pscreen->get_param(pscreen, PIPE_CAP_TGSI_FS_COORD_ORIGIN_UPPER_LEFT)) {
6210 /* the driver supports upper-left origin */
6211 }
6212 else if (pscreen->get_param(pscreen, PIPE_CAP_TGSI_FS_COORD_ORIGIN_LOWER_LEFT)) {
6213 /* the driver supports lower-left origin, need to invert Y */
6214 ureg_property(ureg, TGSI_PROPERTY_FS_COORD_ORIGIN,
6215 TGSI_FS_COORD_ORIGIN_LOWER_LEFT);
6216 invert = TRUE;
6217 }
6218 else
6219 assert(0);
6220 }
6221 else {
6222 /* Fragment shader wants origin in lower-left */
6223 if (pscreen->get_param(pscreen, PIPE_CAP_TGSI_FS_COORD_ORIGIN_LOWER_LEFT))
6224 /* the driver supports lower-left origin */
6225 ureg_property(ureg, TGSI_PROPERTY_FS_COORD_ORIGIN,
6226 TGSI_FS_COORD_ORIGIN_LOWER_LEFT);
6227 else if (pscreen->get_param(pscreen, PIPE_CAP_TGSI_FS_COORD_ORIGIN_UPPER_LEFT))
6228 /* the driver supports upper-left origin, need to invert Y */
6229 invert = TRUE;
6230 else
6231 assert(0);
6232 }
6233
6234 if (program->PixelCenterInteger) {
6235 /* Fragment shader wants pixel center integer */
6236 if (pscreen->get_param(pscreen, PIPE_CAP_TGSI_FS_COORD_PIXEL_CENTER_INTEGER)) {
6237 /* the driver supports pixel center integer */
6238 adjY[1] = 1.0f;
6239 ureg_property(ureg, TGSI_PROPERTY_FS_COORD_PIXEL_CENTER,
6240 TGSI_FS_COORD_PIXEL_CENTER_INTEGER);
6241 }
6242 else if (pscreen->get_param(pscreen, PIPE_CAP_TGSI_FS_COORD_PIXEL_CENTER_HALF_INTEGER)) {
6243 /* the driver supports pixel center half integer, need to bias X,Y */
6244 adjX = -0.5f;
6245 adjY[0] = -0.5f;
6246 adjY[1] = 0.5f;
6247 }
6248 else
6249 assert(0);
6250 }
6251 else {
6252 /* Fragment shader wants pixel center half integer */
6253 if (pscreen->get_param(pscreen, PIPE_CAP_TGSI_FS_COORD_PIXEL_CENTER_HALF_INTEGER)) {
6254 /* the driver supports pixel center half integer */
6255 }
6256 else if (pscreen->get_param(pscreen, PIPE_CAP_TGSI_FS_COORD_PIXEL_CENTER_INTEGER)) {
6257 /* the driver supports pixel center integer, need to bias X,Y */
6258 adjX = adjY[0] = adjY[1] = 0.5f;
6259 ureg_property(ureg, TGSI_PROPERTY_FS_COORD_PIXEL_CENTER,
6260 TGSI_FS_COORD_PIXEL_CENTER_INTEGER);
6261 }
6262 else
6263 assert(0);
6264 }
6265
6266 /* we invert after adjustment so that we avoid the MOV to temporary,
6267 * and reuse the adjustment ADD instead */
6268 emit_wpos_adjustment(st->ctx, t, wpos_transform_const, invert, adjX, adjY);
6269 }
6270
6271 /**
6272 * OpenGL's fragment gl_FrontFace input is 1 for front-facing, 0 for back.
6273 * TGSI uses +1 for front, -1 for back.
6274 * This function converts the TGSI value to the GL value. Simply clamping/
6275 * saturating the value to [0,1] does the job.
6276 */
6277 static void
6278 emit_face_var(struct gl_context *ctx, struct st_translate *t)
6279 {
6280 struct ureg_program *ureg = t->ureg;
6281 struct ureg_dst face_temp = ureg_DECL_temporary(ureg);
6282 struct ureg_src face_input = t->inputs[t->inputMapping[VARYING_SLOT_FACE]];
6283
6284 if (ctx->Const.NativeIntegers) {
6285 ureg_FSGE(ureg, face_temp, face_input, ureg_imm1f(ureg, 0));
6286 }
6287 else {
6288 /* MOV_SAT face_temp, input[face] */
6289 ureg_MOV(ureg, ureg_saturate(face_temp), face_input);
6290 }
6291
6292 /* Use face_temp as face input from here on: */
6293 t->inputs[t->inputMapping[VARYING_SLOT_FACE]] = ureg_src(face_temp);
6294 }
6295
6296 static void
6297 emit_compute_block_size(const struct gl_program *prog,
6298 struct ureg_program *ureg) {
6299 ureg_property(ureg, TGSI_PROPERTY_CS_FIXED_BLOCK_WIDTH,
6300 prog->info.cs.local_size[0]);
6301 ureg_property(ureg, TGSI_PROPERTY_CS_FIXED_BLOCK_HEIGHT,
6302 prog->info.cs.local_size[1]);
6303 ureg_property(ureg, TGSI_PROPERTY_CS_FIXED_BLOCK_DEPTH,
6304 prog->info.cs.local_size[2]);
6305 }
6306
6307 struct sort_inout_decls {
6308 bool operator()(const struct inout_decl &a, const struct inout_decl &b) const {
6309 return mapping[a.mesa_index] < mapping[b.mesa_index];
6310 }
6311
6312 const ubyte *mapping;
6313 };
6314
6315 /* Sort the given array of decls by the corresponding slot (TGSI file index).
6316 *
6317 * This is for the benefit of older drivers which are broken when the
6318 * declarations aren't sorted in this way.
6319 */
6320 static void
6321 sort_inout_decls_by_slot(struct inout_decl *decls,
6322 unsigned count,
6323 const ubyte mapping[])
6324 {
6325 sort_inout_decls sorter;
6326 sorter.mapping = mapping;
6327 std::sort(decls, decls + count, sorter);
6328 }
6329
6330 static unsigned
6331 st_translate_interp(enum glsl_interp_mode glsl_qual, GLuint varying)
6332 {
6333 switch (glsl_qual) {
6334 case INTERP_MODE_NONE:
6335 if (varying == VARYING_SLOT_COL0 || varying == VARYING_SLOT_COL1)
6336 return TGSI_INTERPOLATE_COLOR;
6337 return TGSI_INTERPOLATE_PERSPECTIVE;
6338 case INTERP_MODE_SMOOTH:
6339 return TGSI_INTERPOLATE_PERSPECTIVE;
6340 case INTERP_MODE_FLAT:
6341 return TGSI_INTERPOLATE_CONSTANT;
6342 case INTERP_MODE_NOPERSPECTIVE:
6343 return TGSI_INTERPOLATE_LINEAR;
6344 default:
6345 assert(0 && "unexpected interp mode in st_translate_interp()");
6346 return TGSI_INTERPOLATE_PERSPECTIVE;
6347 }
6348 }
6349
6350 /**
6351 * Translate intermediate IR (glsl_to_tgsi_instruction) to TGSI format.
6352 * \param program the program to translate
6353 * \param numInputs number of input registers used
6354 * \param inputMapping maps Mesa fragment program inputs to TGSI generic
6355 * input indexes
6356 * \param inputSemanticName the TGSI_SEMANTIC flag for each input
6357 * \param inputSemanticIndex the semantic index (ex: which texcoord) for
6358 * each input
6359 * \param interpMode the TGSI_INTERPOLATE_LINEAR/PERSP mode for each input
6360 * \param numOutputs number of output registers used
6361 * \param outputMapping maps Mesa fragment program outputs to TGSI
6362 * generic outputs
6363 * \param outputSemanticName the TGSI_SEMANTIC flag for each output
6364 * \param outputSemanticIndex the semantic index (ex: which texcoord) for
6365 * each output
6366 *
6367 * \return PIPE_OK or PIPE_ERROR_OUT_OF_MEMORY
6368 */
6369 extern "C" enum pipe_error
6370 st_translate_program(
6371 struct gl_context *ctx,
6372 uint procType,
6373 struct ureg_program *ureg,
6374 glsl_to_tgsi_visitor *program,
6375 const struct gl_program *proginfo,
6376 GLuint numInputs,
6377 const ubyte inputMapping[],
6378 const ubyte inputSlotToAttr[],
6379 const ubyte inputSemanticName[],
6380 const ubyte inputSemanticIndex[],
6381 const ubyte interpMode[],
6382 GLuint numOutputs,
6383 const ubyte outputMapping[],
6384 const ubyte outputSemanticName[],
6385 const ubyte outputSemanticIndex[])
6386 {
6387 struct st_translate *t;
6388 unsigned i;
6389 struct gl_program_constants *frag_const =
6390 &ctx->Const.Program[MESA_SHADER_FRAGMENT];
6391 enum pipe_error ret = PIPE_OK;
6392
6393 assert(numInputs <= ARRAY_SIZE(t->inputs));
6394 assert(numOutputs <= ARRAY_SIZE(t->outputs));
6395
6396 t = CALLOC_STRUCT(st_translate);
6397 if (!t) {
6398 ret = PIPE_ERROR_OUT_OF_MEMORY;
6399 goto out;
6400 }
6401
6402 t->procType = procType;
6403 t->inputMapping = inputMapping;
6404 t->outputMapping = outputMapping;
6405 t->ureg = ureg;
6406 t->num_temp_arrays = program->next_array;
6407 if (t->num_temp_arrays)
6408 t->arrays = (struct ureg_dst*)
6409 calloc(t->num_temp_arrays, sizeof(t->arrays[0]));
6410
6411 /*
6412 * Declare input attributes.
6413 */
6414 switch (procType) {
6415 case PIPE_SHADER_FRAGMENT:
6416 case PIPE_SHADER_GEOMETRY:
6417 case PIPE_SHADER_TESS_EVAL:
6418 case PIPE_SHADER_TESS_CTRL:
6419 sort_inout_decls_by_slot(program->inputs, program->num_inputs, inputMapping);
6420
6421 for (i = 0; i < program->num_inputs; ++i) {
6422 struct inout_decl *decl = &program->inputs[i];
6423 unsigned slot = inputMapping[decl->mesa_index];
6424 struct ureg_src src;
6425 ubyte tgsi_usage_mask = decl->usage_mask;
6426
6427 if (glsl_base_type_is_64bit(decl->base_type)) {
6428 if (tgsi_usage_mask == 1)
6429 tgsi_usage_mask = TGSI_WRITEMASK_XY;
6430 else if (tgsi_usage_mask == 2)
6431 tgsi_usage_mask = TGSI_WRITEMASK_ZW;
6432 else
6433 tgsi_usage_mask = TGSI_WRITEMASK_XYZW;
6434 }
6435
6436 unsigned interp_mode = 0;
6437 unsigned interp_location = 0;
6438 if (procType == PIPE_SHADER_FRAGMENT) {
6439 assert(interpMode);
6440 interp_mode = interpMode[slot] != TGSI_INTERPOLATE_COUNT ?
6441 interpMode[slot] :
6442 st_translate_interp(decl->interp, inputSlotToAttr[slot]);
6443
6444 interp_location = decl->interp_loc;
6445 }
6446
6447 src = ureg_DECL_fs_input_cyl_centroid_layout(ureg,
6448 inputSemanticName[slot], inputSemanticIndex[slot],
6449 interp_mode, 0, interp_location, slot, tgsi_usage_mask,
6450 decl->array_id, decl->size);
6451
6452 for (unsigned j = 0; j < decl->size; ++j) {
6453 if (t->inputs[slot + j].File != TGSI_FILE_INPUT) {
6454 /* The ArrayID is set up in dst_register */
6455 t->inputs[slot + j] = src;
6456 t->inputs[slot + j].ArrayID = 0;
6457 t->inputs[slot + j].Index += j;
6458 }
6459 }
6460 }
6461 break;
6462 case PIPE_SHADER_VERTEX:
6463 for (i = 0; i < numInputs; i++) {
6464 t->inputs[i] = ureg_DECL_vs_input(ureg, i);
6465 }
6466 break;
6467 case PIPE_SHADER_COMPUTE:
6468 break;
6469 default:
6470 assert(0);
6471 }
6472
6473 /*
6474 * Declare output attributes.
6475 */
6476 switch (procType) {
6477 case PIPE_SHADER_FRAGMENT:
6478 case PIPE_SHADER_COMPUTE:
6479 break;
6480 case PIPE_SHADER_GEOMETRY:
6481 case PIPE_SHADER_TESS_EVAL:
6482 case PIPE_SHADER_TESS_CTRL:
6483 case PIPE_SHADER_VERTEX:
6484 sort_inout_decls_by_slot(program->outputs, program->num_outputs, outputMapping);
6485
6486 for (i = 0; i < program->num_outputs; ++i) {
6487 struct inout_decl *decl = &program->outputs[i];
6488 unsigned slot = outputMapping[decl->mesa_index];
6489 struct ureg_dst dst;
6490 ubyte tgsi_usage_mask = decl->usage_mask;
6491
6492 if (glsl_base_type_is_64bit(decl->base_type)) {
6493 if (tgsi_usage_mask == 1)
6494 tgsi_usage_mask = TGSI_WRITEMASK_XY;
6495 else if (tgsi_usage_mask == 2)
6496 tgsi_usage_mask = TGSI_WRITEMASK_ZW;
6497 else
6498 tgsi_usage_mask = TGSI_WRITEMASK_XYZW;
6499 }
6500
6501 dst = ureg_DECL_output_layout(ureg,
6502 outputSemanticName[slot], outputSemanticIndex[slot],
6503 decl->gs_out_streams,
6504 slot, tgsi_usage_mask, decl->array_id, decl->size);
6505
6506 for (unsigned j = 0; j < decl->size; ++j) {
6507 if (t->outputs[slot + j].File != TGSI_FILE_OUTPUT) {
6508 /* The ArrayID is set up in dst_register */
6509 t->outputs[slot + j] = dst;
6510 t->outputs[slot + j].ArrayID = 0;
6511 t->outputs[slot + j].Index += j;
6512 }
6513 }
6514 }
6515 break;
6516 default:
6517 assert(0);
6518 }
6519
6520 if (procType == PIPE_SHADER_FRAGMENT) {
6521 if (program->shader->Program->info.fs.early_fragment_tests ||
6522 program->shader->Program->info.fs.post_depth_coverage) {
6523 ureg_property(ureg, TGSI_PROPERTY_FS_EARLY_DEPTH_STENCIL, 1);
6524
6525 if (program->shader->Program->info.fs.post_depth_coverage)
6526 ureg_property(ureg, TGSI_PROPERTY_FS_POST_DEPTH_COVERAGE, 1);
6527 }
6528
6529 if (proginfo->info.inputs_read & VARYING_BIT_POS) {
6530 /* Must do this after setting up t->inputs. */
6531 emit_wpos(st_context(ctx), t, proginfo, ureg,
6532 program->wpos_transform_const);
6533 }
6534
6535 if (proginfo->info.inputs_read & VARYING_BIT_FACE)
6536 emit_face_var(ctx, t);
6537
6538 for (i = 0; i < numOutputs; i++) {
6539 switch (outputSemanticName[i]) {
6540 case TGSI_SEMANTIC_POSITION:
6541 t->outputs[i] = ureg_DECL_output(ureg,
6542 TGSI_SEMANTIC_POSITION, /* Z/Depth */
6543 outputSemanticIndex[i]);
6544 t->outputs[i] = ureg_writemask(t->outputs[i], TGSI_WRITEMASK_Z);
6545 break;
6546 case TGSI_SEMANTIC_STENCIL:
6547 t->outputs[i] = ureg_DECL_output(ureg,
6548 TGSI_SEMANTIC_STENCIL, /* Stencil */
6549 outputSemanticIndex[i]);
6550 t->outputs[i] = ureg_writemask(t->outputs[i], TGSI_WRITEMASK_Y);
6551 break;
6552 case TGSI_SEMANTIC_COLOR:
6553 t->outputs[i] = ureg_DECL_output(ureg,
6554 TGSI_SEMANTIC_COLOR,
6555 outputSemanticIndex[i]);
6556 break;
6557 case TGSI_SEMANTIC_SAMPLEMASK:
6558 t->outputs[i] = ureg_DECL_output(ureg,
6559 TGSI_SEMANTIC_SAMPLEMASK,
6560 outputSemanticIndex[i]);
6561 /* TODO: If we ever support more than 32 samples, this will have
6562 * to become an array.
6563 */
6564 t->outputs[i] = ureg_writemask(t->outputs[i], TGSI_WRITEMASK_X);
6565 break;
6566 default:
6567 assert(!"fragment shader outputs must be POSITION/STENCIL/COLOR");
6568 ret = PIPE_ERROR_BAD_INPUT;
6569 goto out;
6570 }
6571 }
6572 }
6573 else if (procType == PIPE_SHADER_VERTEX) {
6574 for (i = 0; i < numOutputs; i++) {
6575 if (outputSemanticName[i] == TGSI_SEMANTIC_FOG) {
6576 /* force register to contain a fog coordinate in the form (F, 0, 0, 1). */
6577 ureg_MOV(ureg,
6578 ureg_writemask(t->outputs[i], TGSI_WRITEMASK_YZW),
6579 ureg_imm4f(ureg, 0.0f, 0.0f, 0.0f, 1.0f));
6580 t->outputs[i] = ureg_writemask(t->outputs[i], TGSI_WRITEMASK_X);
6581 }
6582 }
6583 }
6584
6585 if (procType == PIPE_SHADER_COMPUTE) {
6586 emit_compute_block_size(proginfo, ureg);
6587 }
6588
6589 /* Declare address register.
6590 */
6591 if (program->num_address_regs > 0) {
6592 assert(program->num_address_regs <= 3);
6593 for (int i = 0; i < program->num_address_regs; i++)
6594 t->address[i] = ureg_DECL_address(ureg);
6595 }
6596
6597 /* Declare misc input registers
6598 */
6599 {
6600 GLbitfield sysInputs = proginfo->info.system_values_read;
6601
6602 for (i = 0; sysInputs; i++) {
6603 if (sysInputs & (1 << i)) {
6604 unsigned semName = _mesa_sysval_to_semantic(i);
6605
6606 t->systemValues[i] = ureg_DECL_system_value(ureg, semName, 0);
6607
6608 if (semName == TGSI_SEMANTIC_INSTANCEID ||
6609 semName == TGSI_SEMANTIC_VERTEXID) {
6610 /* From Gallium perspective, these system values are always
6611 * integer, and require native integer support. However, if
6612 * native integer is supported on the vertex stage but not the
6613 * pixel stage (e.g, i915g + draw), Mesa will generate IR that
6614 * assumes these system values are floats. To resolve the
6615 * inconsistency, we insert a U2F.
6616 */
6617 struct st_context *st = st_context(ctx);
6618 struct pipe_screen *pscreen = st->pipe->screen;
6619 assert(procType == PIPE_SHADER_VERTEX);
6620 assert(pscreen->get_shader_param(pscreen, PIPE_SHADER_VERTEX, PIPE_SHADER_CAP_INTEGERS));
6621 (void) pscreen;
6622 if (!ctx->Const.NativeIntegers) {
6623 struct ureg_dst temp = ureg_DECL_local_temporary(t->ureg);
6624 ureg_U2F( t->ureg, ureg_writemask(temp, TGSI_WRITEMASK_X), t->systemValues[i]);
6625 t->systemValues[i] = ureg_scalar(ureg_src(temp), 0);
6626 }
6627 }
6628
6629 if (procType == PIPE_SHADER_FRAGMENT &&
6630 semName == TGSI_SEMANTIC_POSITION)
6631 emit_wpos(st_context(ctx), t, proginfo, ureg,
6632 program->wpos_transform_const);
6633
6634 sysInputs &= ~(1 << i);
6635 }
6636 }
6637 }
6638
6639 t->array_sizes = program->array_sizes;
6640 t->input_decls = program->inputs;
6641 t->num_input_decls = program->num_inputs;
6642 t->output_decls = program->outputs;
6643 t->num_output_decls = program->num_outputs;
6644
6645 /* Emit constants and uniforms. TGSI uses a single index space for these,
6646 * so we put all the translated regs in t->constants.
6647 */
6648 if (proginfo->Parameters) {
6649 t->constants = (struct ureg_src *)
6650 calloc(proginfo->Parameters->NumParameters, sizeof(t->constants[0]));
6651 if (t->constants == NULL) {
6652 ret = PIPE_ERROR_OUT_OF_MEMORY;
6653 goto out;
6654 }
6655 t->num_constants = proginfo->Parameters->NumParameters;
6656
6657 for (i = 0; i < proginfo->Parameters->NumParameters; i++) {
6658 switch (proginfo->Parameters->Parameters[i].Type) {
6659 case PROGRAM_STATE_VAR:
6660 case PROGRAM_UNIFORM:
6661 t->constants[i] = ureg_DECL_constant(ureg, i);
6662 break;
6663
6664 /* Emit immediates for PROGRAM_CONSTANT only when there's no indirect
6665 * addressing of the const buffer.
6666 * FIXME: Be smarter and recognize param arrays:
6667 * indirect addressing is only valid within the referenced
6668 * array.
6669 */
6670 case PROGRAM_CONSTANT:
6671 if (program->indirect_addr_consts)
6672 t->constants[i] = ureg_DECL_constant(ureg, i);
6673 else
6674 t->constants[i] = emit_immediate(t,
6675 proginfo->Parameters->ParameterValues[i],
6676 proginfo->Parameters->Parameters[i].DataType,
6677 4);
6678 break;
6679 default:
6680 break;
6681 }
6682 }
6683 }
6684
6685 for (i = 0; i < proginfo->info.num_ubos; i++) {
6686 unsigned size = proginfo->sh.UniformBlocks[i]->UniformBufferSize;
6687 unsigned num_const_vecs = (size + 15) / 16;
6688 unsigned first, last;
6689 assert(num_const_vecs > 0);
6690 first = 0;
6691 last = num_const_vecs > 0 ? num_const_vecs - 1 : 0;
6692 ureg_DECL_constant2D(t->ureg, first, last, i + 1);
6693 }
6694
6695 /* Emit immediate values.
6696 */
6697 t->immediates = (struct ureg_src *)
6698 calloc(program->num_immediates, sizeof(struct ureg_src));
6699 if (t->immediates == NULL) {
6700 ret = PIPE_ERROR_OUT_OF_MEMORY;
6701 goto out;
6702 }
6703 t->num_immediates = program->num_immediates;
6704
6705 i = 0;
6706 foreach_in_list(immediate_storage, imm, &program->immediates) {
6707 assert(i < program->num_immediates);
6708 t->immediates[i++] = emit_immediate(t, imm->values, imm->type, imm->size32);
6709 }
6710 assert(i == program->num_immediates);
6711
6712 /* texture samplers */
6713 for (i = 0; i < frag_const->MaxTextureImageUnits; i++) {
6714 if (program->samplers_used & (1u << i)) {
6715 unsigned type = st_translate_texture_type(program->sampler_types[i]);
6716
6717 t->samplers[i] = ureg_DECL_sampler(ureg, i);
6718
6719 ureg_DECL_sampler_view( ureg, i, program->sampler_targets[i],
6720 type, type, type, type );
6721 }
6722 }
6723
6724 /* Declare atomic and shader storage buffers. */
6725 {
6726 struct gl_program *prog = program->prog;
6727
6728 for (i = 0; i < prog->info.num_abos; i++) {
6729 unsigned index = prog->sh.AtomicBuffers[i]->Binding;
6730 assert(index < frag_const->MaxAtomicBuffers);
6731 t->buffers[index] = ureg_DECL_buffer(ureg, index, true);
6732 }
6733
6734 assert(prog->info.num_ssbos <= frag_const->MaxShaderStorageBlocks);
6735 for (i = 0; i < prog->info.num_ssbos; i++) {
6736 unsigned index = frag_const->MaxAtomicBuffers + i;
6737 t->buffers[index] = ureg_DECL_buffer(ureg, index, false);
6738 }
6739 }
6740
6741 if (program->use_shared_memory)
6742 t->shared_memory = ureg_DECL_memory(ureg, TGSI_MEMORY_TYPE_SHARED);
6743
6744 for (i = 0; i < program->shader->Program->info.num_images; i++) {
6745 if (program->images_used & (1 << i)) {
6746 t->images[i] = ureg_DECL_image(ureg, i,
6747 program->image_targets[i],
6748 program->image_formats[i],
6749 true, false);
6750 }
6751 }
6752
6753 /* Emit each instruction in turn:
6754 */
6755 foreach_in_list(glsl_to_tgsi_instruction, inst, &program->instructions)
6756 compile_tgsi_instruction(t, inst);
6757
6758 /* Set the next shader stage hint for VS and TES. */
6759 switch (procType) {
6760 case PIPE_SHADER_VERTEX:
6761 case PIPE_SHADER_TESS_EVAL:
6762 if (program->shader_program->SeparateShader)
6763 break;
6764
6765 for (i = program->shader->Stage+1; i <= MESA_SHADER_FRAGMENT; i++) {
6766 if (program->shader_program->_LinkedShaders[i]) {
6767 ureg_set_next_shader_processor(
6768 ureg, pipe_shader_type_from_mesa((gl_shader_stage)i));
6769 break;
6770 }
6771 }
6772 break;
6773 }
6774
6775 out:
6776 if (t) {
6777 free(t->arrays);
6778 free(t->temps);
6779 free(t->constants);
6780 t->num_constants = 0;
6781 free(t->immediates);
6782 t->num_immediates = 0;
6783 FREE(t);
6784 }
6785
6786 return ret;
6787 }
6788 /* ----------------------------- End TGSI code ------------------------------ */
6789
6790
6791 /**
6792 * Convert a shader's GLSL IR into a Mesa gl_program, although without
6793 * generating Mesa IR.
6794 */
6795 static struct gl_program *
6796 get_mesa_program_tgsi(struct gl_context *ctx,
6797 struct gl_shader_program *shader_program,
6798 struct gl_linked_shader *shader)
6799 {
6800 glsl_to_tgsi_visitor* v;
6801 struct gl_program *prog;
6802 struct gl_shader_compiler_options *options =
6803 &ctx->Const.ShaderCompilerOptions[shader->Stage];
6804 struct pipe_screen *pscreen = ctx->st->pipe->screen;
6805 enum pipe_shader_type ptarget = pipe_shader_type_from_mesa(shader->Stage);
6806 unsigned skip_merge_registers;
6807
6808 validate_ir_tree(shader->ir);
6809
6810 prog = shader->Program;
6811
6812 prog->Parameters = _mesa_new_parameter_list();
6813 v = new glsl_to_tgsi_visitor();
6814 v->ctx = ctx;
6815 v->prog = prog;
6816 v->shader_program = shader_program;
6817 v->shader = shader;
6818 v->options = options;
6819 v->glsl_version = ctx->Const.GLSLVersion;
6820 v->native_integers = ctx->Const.NativeIntegers;
6821
6822 v->have_sqrt = pscreen->get_shader_param(pscreen, ptarget,
6823 PIPE_SHADER_CAP_TGSI_SQRT_SUPPORTED);
6824 v->have_fma = pscreen->get_shader_param(pscreen, ptarget,
6825 PIPE_SHADER_CAP_TGSI_FMA_SUPPORTED);
6826 v->has_tex_txf_lz = pscreen->get_param(pscreen,
6827 PIPE_CAP_TGSI_TEX_TXF_LZ);
6828
6829 v->variables = _mesa_hash_table_create(v->mem_ctx, _mesa_hash_pointer,
6830 _mesa_key_pointer_equal);
6831 skip_merge_registers =
6832 pscreen->get_shader_param(pscreen, ptarget,
6833 PIPE_SHADER_CAP_TGSI_SKIP_MERGE_REGISTERS);
6834
6835 _mesa_generate_parameters_list_for_uniforms(shader_program, shader,
6836 prog->Parameters);
6837
6838 /* Remove reads from output registers. */
6839 if (!pscreen->get_param(pscreen, PIPE_CAP_TGSI_CAN_READ_OUTPUTS))
6840 lower_output_reads(shader->Stage, shader->ir);
6841
6842 /* Emit intermediate IR for main(). */
6843 visit_exec_list(shader->ir, v);
6844
6845 #if 0
6846 /* Print out some information (for debugging purposes) used by the
6847 * optimization passes. */
6848 {
6849 int i;
6850 int *first_writes = ralloc_array(v->mem_ctx, int, v->next_temp);
6851 int *first_reads = ralloc_array(v->mem_ctx, int, v->next_temp);
6852 int *last_writes = ralloc_array(v->mem_ctx, int, v->next_temp);
6853 int *last_reads = ralloc_array(v->mem_ctx, int, v->next_temp);
6854
6855 for (i = 0; i < v->next_temp; i++) {
6856 first_writes[i] = -1;
6857 first_reads[i] = -1;
6858 last_writes[i] = -1;
6859 last_reads[i] = -1;
6860 }
6861 v->get_first_temp_read(first_reads);
6862 v->get_last_temp_read_first_temp_write(last_reads, first_writes);
6863 v->get_last_temp_write(last_writes);
6864 for (i = 0; i < v->next_temp; i++)
6865 printf("Temp %d: FR=%3d FW=%3d LR=%3d LW=%3d\n", i, first_reads[i],
6866 first_writes[i],
6867 last_reads[i],
6868 last_writes[i]);
6869 ralloc_free(first_writes);
6870 ralloc_free(first_reads);
6871 ralloc_free(last_writes);
6872 ralloc_free(last_reads);
6873 }
6874 #endif
6875
6876 /* Perform optimizations on the instructions in the glsl_to_tgsi_visitor. */
6877 v->simplify_cmp();
6878
6879 if (shader->Stage != MESA_SHADER_TESS_CTRL &&
6880 shader->Stage != MESA_SHADER_TESS_EVAL)
6881 v->copy_propagate();
6882
6883 while (v->eliminate_dead_code());
6884
6885 v->merge_two_dsts();
6886 if (!skip_merge_registers)
6887 v->merge_registers();
6888 v->renumber_registers();
6889
6890 /* Write the END instruction. */
6891 v->emit_asm(NULL, TGSI_OPCODE_END);
6892
6893 if (ctx->_Shader->Flags & GLSL_DUMP) {
6894 _mesa_log("\n");
6895 _mesa_log("GLSL IR for linked %s program %d:\n",
6896 _mesa_shader_stage_to_string(shader->Stage),
6897 shader_program->Name);
6898 _mesa_print_ir(_mesa_get_log_file(), shader->ir, NULL);
6899 _mesa_log("\n\n");
6900 }
6901
6902 do_set_program_inouts(shader->ir, prog, shader->Stage);
6903 _mesa_copy_linked_program_data(shader_program, shader);
6904 shrink_array_declarations(v->inputs, v->num_inputs,
6905 &prog->info.inputs_read,
6906 prog->info.double_inputs_read,
6907 &prog->info.patch_inputs_read);
6908 shrink_array_declarations(v->outputs, v->num_outputs,
6909 &prog->info.outputs_written, 0ULL,
6910 &prog->info.patch_outputs_written);
6911 count_resources(v, prog);
6912
6913 /* The GLSL IR won't be needed anymore. */
6914 ralloc_free(shader->ir);
6915 shader->ir = NULL;
6916
6917 /* This must be done before the uniform storage is associated. */
6918 if (shader->Stage == MESA_SHADER_FRAGMENT &&
6919 (prog->info.inputs_read & VARYING_BIT_POS ||
6920 prog->info.system_values_read & (1 << SYSTEM_VALUE_FRAG_COORD))) {
6921 static const gl_state_index wposTransformState[STATE_LENGTH] = {
6922 STATE_INTERNAL, STATE_FB_WPOS_Y_TRANSFORM
6923 };
6924
6925 v->wpos_transform_const = _mesa_add_state_reference(prog->Parameters,
6926 wposTransformState);
6927 }
6928
6929 /* Avoid reallocation of the program parameter list, because the uniform
6930 * storage is only associated with the original parameter list.
6931 * This should be enough for Bitmap and DrawPixels constants.
6932 */
6933 _mesa_reserve_parameter_storage(prog->Parameters, 8);
6934
6935 /* This has to be done last. Any operation the can cause
6936 * prog->ParameterValues to get reallocated (e.g., anything that adds a
6937 * program constant) has to happen before creating this linkage.
6938 */
6939 _mesa_associate_uniform_storage(ctx, shader_program, prog, true);
6940 if (!shader_program->data->LinkStatus) {
6941 free_glsl_to_tgsi_visitor(v);
6942 _mesa_reference_program(ctx, &shader->Program, NULL);
6943 return NULL;
6944 }
6945
6946 struct st_vertex_program *stvp;
6947 struct st_fragment_program *stfp;
6948 struct st_common_program *stp;
6949 struct st_compute_program *stcp;
6950
6951 switch (shader->Stage) {
6952 case MESA_SHADER_VERTEX:
6953 stvp = (struct st_vertex_program *)prog;
6954 stvp->glsl_to_tgsi = v;
6955 break;
6956 case MESA_SHADER_FRAGMENT:
6957 stfp = (struct st_fragment_program *)prog;
6958 stfp->glsl_to_tgsi = v;
6959 break;
6960 case MESA_SHADER_TESS_CTRL:
6961 case MESA_SHADER_TESS_EVAL:
6962 case MESA_SHADER_GEOMETRY:
6963 stp = st_common_program(prog);
6964 stp->glsl_to_tgsi = v;
6965 break;
6966 case MESA_SHADER_COMPUTE:
6967 stcp = (struct st_compute_program *)prog;
6968 stcp->glsl_to_tgsi = v;
6969 break;
6970 default:
6971 assert(!"should not be reached");
6972 return NULL;
6973 }
6974
6975 return prog;
6976 }
6977
6978 /* See if there are unsupported control flow statements. */
6979 class ir_control_flow_info_visitor : public ir_hierarchical_visitor {
6980 private:
6981 const struct gl_shader_compiler_options *options;
6982 public:
6983 ir_control_flow_info_visitor(const struct gl_shader_compiler_options *options)
6984 : options(options),
6985 unsupported(false)
6986 {
6987 }
6988
6989 virtual ir_visitor_status visit_enter(ir_function *ir)
6990 {
6991 /* Other functions are skipped (same as glsl_to_tgsi). */
6992 if (strcmp(ir->name, "main") == 0)
6993 return visit_continue;
6994
6995 return visit_continue_with_parent;
6996 }
6997
6998 virtual ir_visitor_status visit_enter(ir_call *ir)
6999 {
7000 if (!ir->callee->is_intrinsic()) {
7001 unsupported = true; /* it's a function call */
7002 return visit_stop;
7003 }
7004 return visit_continue;
7005 }
7006
7007 virtual ir_visitor_status visit_enter(ir_return *ir)
7008 {
7009 if (options->EmitNoMainReturn) {
7010 unsupported = true;
7011 return visit_stop;
7012 }
7013 return visit_continue;
7014 }
7015
7016 bool unsupported;
7017 };
7018
7019 static bool
7020 has_unsupported_control_flow(exec_list *ir,
7021 const struct gl_shader_compiler_options *options)
7022 {
7023 ir_control_flow_info_visitor visitor(options);
7024 visit_list_elements(&visitor, ir);
7025 return visitor.unsupported;
7026 }
7027
7028 extern "C" {
7029
7030 /**
7031 * Link a shader.
7032 * Called via ctx->Driver.LinkShader()
7033 * This actually involves converting GLSL IR into an intermediate TGSI-like IR
7034 * with code lowering and other optimizations.
7035 */
7036 GLboolean
7037 st_link_shader(struct gl_context *ctx, struct gl_shader_program *prog)
7038 {
7039 /* Return early if we are loading the shader from on-disk cache */
7040 if (st_load_tgsi_from_disk_cache(ctx, prog)) {
7041 return GL_TRUE;
7042 }
7043
7044 struct pipe_screen *pscreen = ctx->st->pipe->screen;
7045 assert(prog->data->LinkStatus);
7046
7047 for (unsigned i = 0; i < MESA_SHADER_STAGES; i++) {
7048 if (prog->_LinkedShaders[i] == NULL)
7049 continue;
7050
7051 struct gl_linked_shader *shader = prog->_LinkedShaders[i];
7052 exec_list *ir = shader->ir;
7053 gl_shader_stage stage = shader->Stage;
7054 const struct gl_shader_compiler_options *options =
7055 &ctx->Const.ShaderCompilerOptions[stage];
7056 enum pipe_shader_type ptarget = pipe_shader_type_from_mesa(stage);
7057 bool have_dround = pscreen->get_shader_param(pscreen, ptarget,
7058 PIPE_SHADER_CAP_TGSI_DROUND_SUPPORTED);
7059 bool have_dfrexp = pscreen->get_shader_param(pscreen, ptarget,
7060 PIPE_SHADER_CAP_TGSI_DFRACEXP_DLDEXP_SUPPORTED);
7061 unsigned if_threshold = pscreen->get_shader_param(pscreen, ptarget,
7062 PIPE_SHADER_CAP_LOWER_IF_THRESHOLD);
7063
7064 /* If there are forms of indirect addressing that the driver
7065 * cannot handle, perform the lowering pass.
7066 */
7067 if (options->EmitNoIndirectInput || options->EmitNoIndirectOutput ||
7068 options->EmitNoIndirectTemp || options->EmitNoIndirectUniform) {
7069 lower_variable_index_to_cond_assign(stage, ir,
7070 options->EmitNoIndirectInput,
7071 options->EmitNoIndirectOutput,
7072 options->EmitNoIndirectTemp,
7073 options->EmitNoIndirectUniform);
7074 }
7075
7076 if (!pscreen->get_param(pscreen, PIPE_CAP_INT64_DIVMOD))
7077 lower_64bit_integer_instructions(ir, DIV64 | MOD64);
7078
7079 if (ctx->Extensions.ARB_shading_language_packing) {
7080 unsigned lower_inst = LOWER_PACK_SNORM_2x16 |
7081 LOWER_UNPACK_SNORM_2x16 |
7082 LOWER_PACK_UNORM_2x16 |
7083 LOWER_UNPACK_UNORM_2x16 |
7084 LOWER_PACK_SNORM_4x8 |
7085 LOWER_UNPACK_SNORM_4x8 |
7086 LOWER_UNPACK_UNORM_4x8 |
7087 LOWER_PACK_UNORM_4x8;
7088
7089 if (ctx->Extensions.ARB_gpu_shader5)
7090 lower_inst |= LOWER_PACK_USE_BFI |
7091 LOWER_PACK_USE_BFE;
7092 if (!ctx->st->has_half_float_packing)
7093 lower_inst |= LOWER_PACK_HALF_2x16 |
7094 LOWER_UNPACK_HALF_2x16;
7095
7096 lower_packing_builtins(ir, lower_inst);
7097 }
7098
7099 if (!pscreen->get_param(pscreen, PIPE_CAP_TEXTURE_GATHER_OFFSETS))
7100 lower_offset_arrays(ir);
7101 do_mat_op_to_vec(ir);
7102
7103 if (stage == MESA_SHADER_FRAGMENT)
7104 lower_blend_equation_advanced(shader);
7105
7106 lower_instructions(ir,
7107 MOD_TO_FLOOR |
7108 FDIV_TO_MUL_RCP |
7109 EXP_TO_EXP2 |
7110 LOG_TO_LOG2 |
7111 LDEXP_TO_ARITH |
7112 (have_dfrexp ? 0 : DFREXP_DLDEXP_TO_ARITH) |
7113 CARRY_TO_ARITH |
7114 BORROW_TO_ARITH |
7115 (have_dround ? 0 : DOPS_TO_DFRAC) |
7116 (options->EmitNoPow ? POW_TO_EXP2 : 0) |
7117 (!ctx->Const.NativeIntegers ? INT_DIV_TO_MUL_RCP : 0) |
7118 (options->EmitNoSat ? SAT_TO_CLAMP : 0) |
7119 (ctx->Const.ForceGLSLAbsSqrt ? SQRT_TO_ABS_SQRT : 0) |
7120 /* Assume that if ARB_gpu_shader5 is not supported
7121 * then all of the extended integer functions need
7122 * lowering. It may be necessary to add some caps
7123 * for individual instructions.
7124 */
7125 (!ctx->Extensions.ARB_gpu_shader5
7126 ? BIT_COUNT_TO_MATH |
7127 EXTRACT_TO_SHIFTS |
7128 INSERT_TO_SHIFTS |
7129 REVERSE_TO_SHIFTS |
7130 FIND_LSB_TO_FLOAT_CAST |
7131 FIND_MSB_TO_FLOAT_CAST |
7132 IMUL_HIGH_TO_MUL
7133 : 0));
7134
7135 do_vec_index_to_cond_assign(ir);
7136 lower_vector_insert(ir, true);
7137 lower_quadop_vector(ir, false);
7138 lower_noise(ir);
7139 if (options->MaxIfDepth == 0) {
7140 lower_discard(ir);
7141 }
7142
7143 if (ctx->Const.GLSLOptimizeConservatively) {
7144 /* Do it once and repeat only if there's unsupported control flow. */
7145 do {
7146 do_common_optimization(ir, true, true, options,
7147 ctx->Const.NativeIntegers);
7148 lower_if_to_cond_assign((gl_shader_stage)i, ir,
7149 options->MaxIfDepth, if_threshold);
7150 } while (has_unsupported_control_flow(ir, options));
7151 } else {
7152 /* Repeat it until it stops making changes. */
7153 bool progress;
7154 do {
7155 progress = do_common_optimization(ir, true, true, options,
7156 ctx->Const.NativeIntegers);
7157 progress |= lower_if_to_cond_assign((gl_shader_stage)i, ir,
7158 options->MaxIfDepth, if_threshold);
7159 } while (progress);
7160 }
7161
7162 validate_ir_tree(ir);
7163 }
7164
7165 build_program_resource_list(ctx, prog);
7166
7167 for (unsigned i = 0; i < MESA_SHADER_STAGES; i++) {
7168 struct gl_linked_shader *shader = prog->_LinkedShaders[i];
7169 if (shader == NULL)
7170 continue;
7171
7172 enum pipe_shader_type ptarget =
7173 pipe_shader_type_from_mesa(shader->Stage);
7174 enum pipe_shader_ir preferred_ir = (enum pipe_shader_ir)
7175 pscreen->get_shader_param(pscreen, ptarget,
7176 PIPE_SHADER_CAP_PREFERRED_IR);
7177
7178 struct gl_program *linked_prog = NULL;
7179 if (preferred_ir == PIPE_SHADER_IR_NIR) {
7180 /* TODO only for GLSL VS/FS/CS for now: */
7181 switch (shader->Stage) {
7182 case MESA_SHADER_VERTEX:
7183 case MESA_SHADER_FRAGMENT:
7184 case MESA_SHADER_COMPUTE:
7185 linked_prog = st_nir_get_mesa_program(ctx, prog, shader);
7186 default:
7187 break;
7188 }
7189 } else {
7190 linked_prog = get_mesa_program_tgsi(ctx, prog, shader);
7191 }
7192
7193 if (linked_prog) {
7194 st_set_prog_affected_state_flags(linked_prog);
7195 if (!ctx->Driver.ProgramStringNotify(ctx,
7196 _mesa_shader_stage_to_program(i),
7197 linked_prog)) {
7198 _mesa_reference_program(ctx, &shader->Program, NULL);
7199 return GL_FALSE;
7200 }
7201 }
7202 }
7203
7204 return GL_TRUE;
7205 }
7206
7207 void
7208 st_translate_stream_output_info(glsl_to_tgsi_visitor *glsl_to_tgsi,
7209 const ubyte outputMapping[],
7210 struct pipe_stream_output_info *so)
7211 {
7212 if (!glsl_to_tgsi->shader_program->last_vert_prog)
7213 return;
7214
7215 struct gl_transform_feedback_info *info =
7216 glsl_to_tgsi->shader_program->last_vert_prog->sh.LinkedTransformFeedback;
7217 st_translate_stream_output_info2(info, outputMapping, so);
7218 }
7219
7220 void
7221 st_translate_stream_output_info2(struct gl_transform_feedback_info *info,
7222 const ubyte outputMapping[],
7223 struct pipe_stream_output_info *so)
7224 {
7225 unsigned i;
7226
7227 for (i = 0; i < info->NumOutputs; i++) {
7228 so->output[i].register_index =
7229 outputMapping[info->Outputs[i].OutputRegister];
7230 so->output[i].start_component = info->Outputs[i].ComponentOffset;
7231 so->output[i].num_components = info->Outputs[i].NumComponents;
7232 so->output[i].output_buffer = info->Outputs[i].OutputBuffer;
7233 so->output[i].dst_offset = info->Outputs[i].DstOffset;
7234 so->output[i].stream = info->Outputs[i].StreamId;
7235 }
7236
7237 for (i = 0; i < PIPE_MAX_SO_BUFFERS; i++) {
7238 so->stride[i] = info->Buffers[i].Stride;
7239 }
7240 so->num_outputs = info->NumOutputs;
7241 }
7242
7243 } /* extern "C" */