2 * Copyright (C) 2015-2018 Rob Clark <robclark@freedesktop.org>
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
24 * Rob Clark <robclark@freedesktop.org>
27 #include "ir3_compiler.h"
28 #include "ir3_context.h"
29 #include "ir3_image.h"
30 #include "ir3_shader.h"
34 ir3_context_init(struct ir3_compiler
*compiler
,
35 struct ir3_shader_variant
*so
)
37 struct ir3_context
*ctx
= rzalloc(NULL
, struct ir3_context
);
39 if (compiler
->gpu_id
>= 400) {
40 if (so
->type
== MESA_SHADER_VERTEX
) {
41 ctx
->astc_srgb
= so
->key
.vastc_srgb
;
42 } else if (so
->type
== MESA_SHADER_FRAGMENT
) {
43 ctx
->astc_srgb
= so
->key
.fastc_srgb
;
47 if (so
->type
== MESA_SHADER_VERTEX
) {
48 ctx
->samples
= so
->key
.vsamples
;
49 } else if (so
->type
== MESA_SHADER_FRAGMENT
) {
50 ctx
->samples
= so
->key
.fsamples
;
54 if (compiler
->gpu_id
>= 600) {
55 ctx
->funcs
= &ir3_a6xx_funcs
;
56 } else if (compiler
->gpu_id
>= 400) {
57 ctx
->funcs
= &ir3_a4xx_funcs
;
60 ctx
->compiler
= compiler
;
62 ctx
->def_ht
= _mesa_hash_table_create(ctx
,
63 _mesa_hash_pointer
, _mesa_key_pointer_equal
);
64 ctx
->block_ht
= _mesa_hash_table_create(ctx
,
65 _mesa_hash_pointer
, _mesa_key_pointer_equal
);
66 ctx
->sel_cond_conversions
= _mesa_hash_table_create(ctx
,
67 _mesa_hash_pointer
, _mesa_key_pointer_equal
);
69 /* TODO: maybe generate some sort of bitmask of what key
70 * lowers vs what shader has (ie. no need to lower
71 * texture clamp lowering if no texture sample instrs)..
72 * although should be done further up the stack to avoid
73 * creating duplicate variants..
76 ctx
->s
= nir_shader_clone(ctx
, so
->shader
->nir
);
77 if (ir3_key_lowers_nir(&so
->key
))
78 ir3_optimize_nir(so
->shader
, ctx
->s
, &so
->key
);
80 /* this needs to be the last pass run, so do this here instead of
81 * in ir3_optimize_nir():
83 bool progress
= false;
84 NIR_PASS(progress
, ctx
->s
, nir_lower_locals_to_regs
);
86 /* we could need cleanup after lower_locals_to_regs */
89 NIR_PASS(progress
, ctx
->s
, nir_opt_algebraic
);
90 NIR_PASS(progress
, ctx
->s
, nir_opt_constant_folding
);
93 /* We want to lower nir_op_imul as late as possible, to catch also
94 * those generated by earlier passes (e.g, nir_lower_locals_to_regs).
95 * However, we want a final swing of a few passes to have a chance
96 * at optimizing the result.
99 NIR_PASS(progress
, ctx
->s
, ir3_nir_lower_imul
);
102 NIR_PASS(progress
, ctx
->s
, nir_opt_algebraic
);
103 NIR_PASS(progress
, ctx
->s
, nir_opt_copy_prop_vars
);
104 NIR_PASS(progress
, ctx
->s
, nir_opt_dead_write_vars
);
105 NIR_PASS(progress
, ctx
->s
, nir_opt_dce
);
106 NIR_PASS(progress
, ctx
->s
, nir_opt_constant_folding
);
109 /* Enable the texture pre-fetch feature only a4xx onwards. But
110 * only enable it on generations that have been tested:
112 if ((so
->type
== MESA_SHADER_FRAGMENT
) && (compiler
->gpu_id
>= 600))
113 NIR_PASS_V(ctx
->s
, ir3_nir_lower_tex_prefetch
);
115 NIR_PASS_V(ctx
->s
, nir_convert_from_ssa
, true);
117 /* Super crude heuristic to limit # of tex prefetch in small
118 * shaders. This completely ignores loops.. but that's really
119 * not the worst of it's problems. (A frag shader that has
120 * loops is probably going to be big enough to not trigger a
123 * 1) probably want to do this in terms of ir3 instructions
124 * 2) probably really want to decide this after scheduling
125 * (or at least pre-RA sched) so we have a rough idea about
126 * nops, and don't count things that get cp'd away
127 * 3) blob seems to use higher thresholds with a mix of more
128 * SFU instructions. Which partly makes sense, more SFU
129 * instructions probably means you want to get the real
130 * shader started sooner, but that considers where in the
131 * shader the SFU instructions are, which blob doesn't seem
134 * This uses more conservative thresholds assuming a more alu
135 * than sfu heavy instruction mix.
137 if (so
->type
== MESA_SHADER_FRAGMENT
) {
138 nir_function_impl
*fxn
= nir_shader_get_entrypoint(ctx
->s
);
140 unsigned instruction_count
= 0;
141 nir_foreach_block (block
, fxn
) {
142 instruction_count
+= exec_list_length(&block
->instr_list
);
145 if (instruction_count
< 50) {
146 ctx
->prefetch_limit
= 2;
147 } else if (instruction_count
< 70) {
148 ctx
->prefetch_limit
= 3;
150 ctx
->prefetch_limit
= IR3_MAX_SAMPLER_PREFETCH
;
154 if (shader_debug_enabled(so
->type
)) {
155 fprintf(stdout
, "NIR (final form) for %s shader %s:\n",
156 ir3_shader_stage(so
), so
->shader
->nir
->info
.name
);
157 nir_print_shader(ctx
->s
, stdout
);
160 ir3_ibo_mapping_init(&so
->image_mapping
, ctx
->s
->info
.num_textures
);
166 ir3_context_free(struct ir3_context
*ctx
)
175 /* allocate a n element value array (to be populated by caller) and
178 struct ir3_instruction
**
179 ir3_get_dst_ssa(struct ir3_context
*ctx
, nir_ssa_def
*dst
, unsigned n
)
181 struct ir3_instruction
**value
=
182 ralloc_array(ctx
->def_ht
, struct ir3_instruction
*, n
);
183 _mesa_hash_table_insert(ctx
->def_ht
, dst
, value
);
187 struct ir3_instruction
**
188 ir3_get_dst(struct ir3_context
*ctx
, nir_dest
*dst
, unsigned n
)
190 struct ir3_instruction
**value
;
193 value
= ir3_get_dst_ssa(ctx
, &dst
->ssa
, n
);
195 value
= ralloc_array(ctx
, struct ir3_instruction
*, n
);
198 /* NOTE: in non-ssa case, we don't really need to store last_dst
199 * but this helps us catch cases where put_dst() call is forgotten
201 compile_assert(ctx
, !ctx
->last_dst
);
202 ctx
->last_dst
= value
;
208 struct ir3_instruction
* const *
209 ir3_get_src(struct ir3_context
*ctx
, nir_src
*src
)
212 struct hash_entry
*entry
;
213 entry
= _mesa_hash_table_search(ctx
->def_ht
, src
->ssa
);
214 compile_assert(ctx
, entry
);
217 nir_register
*reg
= src
->reg
.reg
;
218 struct ir3_array
*arr
= ir3_get_array(ctx
, reg
);
219 unsigned num_components
= arr
->r
->num_components
;
220 struct ir3_instruction
*addr
= NULL
;
221 struct ir3_instruction
**value
=
222 ralloc_array(ctx
, struct ir3_instruction
*, num_components
);
224 if (src
->reg
.indirect
)
225 addr
= ir3_get_addr0(ctx
, ir3_get_src(ctx
, src
->reg
.indirect
)[0],
226 reg
->num_components
);
228 for (unsigned i
= 0; i
< num_components
; i
++) {
229 unsigned n
= src
->reg
.base_offset
* reg
->num_components
+ i
;
230 compile_assert(ctx
, n
< arr
->length
);
231 value
[i
] = ir3_create_array_load(ctx
, arr
, n
, addr
, reg
->bit_size
);
239 ir3_put_dst(struct ir3_context
*ctx
, nir_dest
*dst
)
241 unsigned bit_size
= nir_dest_bit_size(*dst
);
243 /* add extra mov if dst value is HIGH reg.. in some cases not all
244 * instructions can read from HIGH regs, in cases where they can
245 * ir3_cp will clean up the extra mov:
247 for (unsigned i
= 0; i
< ctx
->last_dst_n
; i
++) {
248 if (!ctx
->last_dst
[i
])
250 if (ctx
->last_dst
[i
]->regs
[0]->flags
& IR3_REG_HIGH
) {
251 ctx
->last_dst
[i
] = ir3_MOV(ctx
->block
, ctx
->last_dst
[i
], TYPE_U32
);
255 /* Note: 1-bit bools are stored in 32-bit regs */
256 if (bit_size
== 16) {
257 for (unsigned i
= 0; i
< ctx
->last_dst_n
; i
++) {
258 struct ir3_instruction
*dst
= ctx
->last_dst
[i
];
259 dst
->regs
[0]->flags
|= IR3_REG_HALF
;
260 if (ctx
->last_dst
[i
]->opc
== OPC_META_SPLIT
)
261 dst
->regs
[1]->instr
->regs
[0]->flags
|= IR3_REG_HALF
;
266 nir_register
*reg
= dst
->reg
.reg
;
267 struct ir3_array
*arr
= ir3_get_array(ctx
, reg
);
268 unsigned num_components
= ctx
->last_dst_n
;
269 struct ir3_instruction
*addr
= NULL
;
271 if (dst
->reg
.indirect
)
272 addr
= ir3_get_addr0(ctx
, ir3_get_src(ctx
, dst
->reg
.indirect
)[0],
273 reg
->num_components
);
275 for (unsigned i
= 0; i
< num_components
; i
++) {
276 unsigned n
= dst
->reg
.base_offset
* reg
->num_components
+ i
;
277 compile_assert(ctx
, n
< arr
->length
);
278 if (!ctx
->last_dst
[i
])
280 ir3_create_array_store(ctx
, arr
, n
, ctx
->last_dst
[i
], addr
);
283 ralloc_free(ctx
->last_dst
);
286 ctx
->last_dst
= NULL
;
291 dest_flags(struct ir3_instruction
*instr
)
293 return instr
->regs
[0]->flags
& (IR3_REG_HALF
| IR3_REG_HIGH
);
296 struct ir3_instruction
*
297 ir3_create_collect(struct ir3_context
*ctx
, struct ir3_instruction
*const *arr
,
300 struct ir3_block
*block
= ctx
->block
;
301 struct ir3_instruction
*collect
;
306 unsigned flags
= dest_flags(arr
[0]);
308 collect
= ir3_instr_create2(block
, OPC_META_COLLECT
, 1 + arrsz
);
309 __ssa_dst(collect
)->flags
|= flags
;
310 for (unsigned i
= 0; i
< arrsz
; i
++) {
311 struct ir3_instruction
*elem
= arr
[i
];
313 /* Since arrays are pre-colored in RA, we can't assume that
314 * things will end up in the right place. (Ie. if a collect
315 * joins elements from two different arrays.) So insert an
318 * We could possibly skip this if all the collected elements
319 * are contiguous elements in a single array.. not sure how
320 * likely that is to happen.
322 * Fixes a problem with glamor shaders, that in effect do
329 * color = texture2D(tex, texcoord);
331 * In this case, texcoord will end up as nir registers (which
332 * translate to ir3 array's of length 1. And we can't assume
333 * the two (or more) arrays will get allocated in consecutive
337 if (elem
->regs
[0]->flags
& IR3_REG_ARRAY
) {
338 type_t type
= (flags
& IR3_REG_HALF
) ? TYPE_U16
: TYPE_U32
;
339 elem
= ir3_MOV(block
, elem
, type
);
342 compile_assert(ctx
, dest_flags(elem
) == flags
);
343 __ssa_src(collect
, elem
, flags
);
346 collect
->regs
[0]->wrmask
= MASK(arrsz
);
351 /* helper for instructions that produce multiple consecutive scalar
352 * outputs which need to have a split meta instruction inserted
355 ir3_split_dest(struct ir3_block
*block
, struct ir3_instruction
**dst
,
356 struct ir3_instruction
*src
, unsigned base
, unsigned n
)
358 struct ir3_instruction
*prev
= NULL
;
360 if ((n
== 1) && (src
->regs
[0]->wrmask
== 0x1)) {
365 if (src
->opc
== OPC_META_COLLECT
) {
366 debug_assert((base
+ n
) < src
->regs_count
);
368 for (int i
= 0; i
< n
; i
++) {
369 dst
[i
] = ssa(src
->regs
[i
+ base
+ 1]);
375 unsigned flags
= dest_flags(src
);
377 for (int i
= 0, j
= 0; i
< n
; i
++) {
378 struct ir3_instruction
*split
=
379 ir3_instr_create(block
, OPC_META_SPLIT
);
380 __ssa_dst(split
)->flags
|= flags
;
381 __ssa_src(split
, src
, flags
);
382 split
->split
.off
= i
+ base
;
385 split
->cp
.left
= prev
;
386 split
->cp
.left_cnt
++;
387 prev
->cp
.right
= split
;
388 prev
->cp
.right_cnt
++;
392 if (src
->regs
[0]->wrmask
& (1 << (i
+ base
)))
398 ir3_context_error(struct ir3_context
*ctx
, const char *format
, ...)
400 struct hash_table
*errors
= NULL
;
402 va_start(ap
, format
);
403 if (ctx
->cur_instr
) {
404 errors
= _mesa_hash_table_create(NULL
,
406 _mesa_key_pointer_equal
);
407 char *msg
= ralloc_vasprintf(errors
, format
, ap
);
408 _mesa_hash_table_insert(errors
, ctx
->cur_instr
, msg
);
410 _debug_vprintf(format
, ap
);
413 nir_print_shader_annotated(ctx
->s
, stdout
, errors
);
419 static struct ir3_instruction
*
420 create_addr0(struct ir3_block
*block
, struct ir3_instruction
*src
, int align
)
422 struct ir3_instruction
*instr
, *immed
;
424 /* TODO in at least some cases, the backend could probably be
425 * made clever enough to propagate IR3_REG_HALF..
427 instr
= ir3_COV(block
, src
, TYPE_U32
, TYPE_S16
);
428 instr
->regs
[0]->flags
|= IR3_REG_HALF
;
435 /* src *= 2 => src <<= 1: */
436 immed
= create_immed(block
, 1);
437 immed
->regs
[0]->flags
|= IR3_REG_HALF
;
439 instr
= ir3_SHL_B(block
, instr
, 0, immed
, 0);
440 instr
->regs
[0]->flags
|= IR3_REG_HALF
;
441 instr
->regs
[1]->flags
|= IR3_REG_HALF
;
445 immed
= create_immed(block
, 3);
446 immed
->regs
[0]->flags
|= IR3_REG_HALF
;
448 instr
= ir3_MULL_U(block
, instr
, 0, immed
, 0);
449 instr
->regs
[0]->flags
|= IR3_REG_HALF
;
450 instr
->regs
[1]->flags
|= IR3_REG_HALF
;
453 /* src *= 4 => src <<= 2: */
454 immed
= create_immed(block
, 2);
455 immed
->regs
[0]->flags
|= IR3_REG_HALF
;
457 instr
= ir3_SHL_B(block
, instr
, 0, immed
, 0);
458 instr
->regs
[0]->flags
|= IR3_REG_HALF
;
459 instr
->regs
[1]->flags
|= IR3_REG_HALF
;
462 unreachable("bad align");
466 instr
= ir3_MOV(block
, instr
, TYPE_S16
);
467 instr
->regs
[0]->num
= regid(REG_A0
, 0);
468 instr
->regs
[0]->flags
&= ~IR3_REG_SSA
;
469 instr
->regs
[0]->flags
|= IR3_REG_HALF
;
470 instr
->regs
[1]->flags
|= IR3_REG_HALF
;
475 static struct ir3_instruction
*
476 create_addr1(struct ir3_block
*block
, unsigned const_val
)
479 struct ir3_instruction
*immed
= create_immed(block
, const_val
);
480 struct ir3_instruction
*instr
= ir3_MOV(block
, immed
, TYPE_S16
);
481 instr
->regs
[0]->num
= regid(REG_A0
, 1);
482 instr
->regs
[0]->flags
&= ~IR3_REG_SSA
;
483 instr
->regs
[0]->flags
|= IR3_REG_HALF
;
484 instr
->regs
[1]->flags
|= IR3_REG_HALF
;
488 /* caches addr values to avoid generating multiple cov/shl/mova
489 * sequences for each use of a given NIR level src as address
491 struct ir3_instruction
*
492 ir3_get_addr0(struct ir3_context
*ctx
, struct ir3_instruction
*src
, int align
)
494 struct ir3_instruction
*addr
;
495 unsigned idx
= align
- 1;
497 compile_assert(ctx
, idx
< ARRAY_SIZE(ctx
->addr0_ht
));
499 if (!ctx
->addr0_ht
[idx
]) {
500 ctx
->addr0_ht
[idx
] = _mesa_hash_table_create(ctx
,
501 _mesa_hash_pointer
, _mesa_key_pointer_equal
);
503 struct hash_entry
*entry
;
504 entry
= _mesa_hash_table_search(ctx
->addr0_ht
[idx
], src
);
509 addr
= create_addr0(ctx
->block
, src
, align
);
510 _mesa_hash_table_insert(ctx
->addr0_ht
[idx
], src
, addr
);
515 /* Similar to ir3_get_addr0, but for a1.x. */
516 struct ir3_instruction
*
517 ir3_get_addr1(struct ir3_context
*ctx
, unsigned const_val
)
519 struct ir3_instruction
*addr
;
521 if (!ctx
->addr1_ht
) {
522 ctx
->addr1_ht
= _mesa_hash_table_u64_create(ctx
);
524 addr
= _mesa_hash_table_u64_search(ctx
->addr1_ht
, const_val
);
529 addr
= create_addr1(ctx
->block
, const_val
);
530 _mesa_hash_table_u64_insert(ctx
->addr1_ht
, const_val
, addr
);
535 struct ir3_instruction
*
536 ir3_get_predicate(struct ir3_context
*ctx
, struct ir3_instruction
*src
)
538 struct ir3_block
*b
= ctx
->block
;
539 struct ir3_instruction
*cond
;
541 /* NOTE: only cmps.*.* can write p0.x: */
542 cond
= ir3_CMPS_S(b
, src
, 0, create_immed(b
, 0), 0);
543 cond
->cat2
.condition
= IR3_COND_NE
;
545 /* condition always goes in predicate register: */
546 cond
->regs
[0]->num
= regid(REG_P0
, 0);
547 cond
->regs
[0]->flags
&= ~IR3_REG_SSA
;
557 ir3_declare_array(struct ir3_context
*ctx
, nir_register
*reg
)
559 struct ir3_array
*arr
= rzalloc(ctx
, struct ir3_array
);
560 arr
->id
= ++ctx
->num_arrays
;
561 /* NOTE: sometimes we get non array regs, for example for arrays of
562 * length 1. See fs-const-array-of-struct-of-array.shader_test. So
563 * treat a non-array as if it was an array of length 1.
565 * It would be nice if there was a nir pass to convert arrays of
568 arr
->length
= reg
->num_components
* MAX2(1, reg
->num_array_elems
);
569 compile_assert(ctx
, arr
->length
> 0);
571 list_addtail(&arr
->node
, &ctx
->ir
->array_list
);
575 ir3_get_array(struct ir3_context
*ctx
, nir_register
*reg
)
577 foreach_array (arr
, &ctx
->ir
->array_list
) {
581 ir3_context_error(ctx
, "bogus reg: %s\n", reg
->name
);
585 /* relative (indirect) if address!=NULL */
586 struct ir3_instruction
*
587 ir3_create_array_load(struct ir3_context
*ctx
, struct ir3_array
*arr
, int n
,
588 struct ir3_instruction
*address
, unsigned bitsize
)
590 struct ir3_block
*block
= ctx
->block
;
591 struct ir3_instruction
*mov
;
592 struct ir3_register
*src
;
595 mov
= ir3_instr_create(block
, OPC_MOV
);
597 mov
->cat1
.src_type
= TYPE_U16
;
598 mov
->cat1
.dst_type
= TYPE_U16
;
599 flags
|= IR3_REG_HALF
;
602 mov
->cat1
.src_type
= TYPE_U32
;
603 mov
->cat1
.dst_type
= TYPE_U32
;
606 mov
->barrier_class
= IR3_BARRIER_ARRAY_R
;
607 mov
->barrier_conflict
= IR3_BARRIER_ARRAY_W
;
608 __ssa_dst(mov
)->flags
|= flags
;
609 src
= ir3_reg_create(mov
, 0, IR3_REG_ARRAY
|
610 COND(address
, IR3_REG_RELATIV
) | flags
);
611 src
->instr
= arr
->last_write
;
612 src
->size
= arr
->length
;
613 src
->array
.id
= arr
->id
;
614 src
->array
.offset
= n
;
617 ir3_instr_set_address(mov
, address
);
622 /* relative (indirect) if address!=NULL */
624 ir3_create_array_store(struct ir3_context
*ctx
, struct ir3_array
*arr
, int n
,
625 struct ir3_instruction
*src
, struct ir3_instruction
*address
)
627 struct ir3_block
*block
= ctx
->block
;
628 struct ir3_instruction
*mov
;
629 struct ir3_register
*dst
;
631 /* if not relative store, don't create an extra mov, since that
632 * ends up being difficult for cp to remove.
634 * Also, don't skip the mov if the src is meta (like fanout/split),
635 * since that creates a situation that RA can't really handle properly.
637 if (!address
&& !is_meta(src
)) {
640 src
->barrier_class
|= IR3_BARRIER_ARRAY_W
;
641 src
->barrier_conflict
|= IR3_BARRIER_ARRAY_R
| IR3_BARRIER_ARRAY_W
;
643 dst
->flags
|= IR3_REG_ARRAY
;
644 dst
->instr
= arr
->last_write
;
645 dst
->size
= arr
->length
;
646 dst
->array
.id
= arr
->id
;
647 dst
->array
.offset
= n
;
649 arr
->last_write
= src
;
651 array_insert(block
, block
->keeps
, src
);
656 mov
= ir3_instr_create(block
, OPC_MOV
);
657 mov
->cat1
.src_type
= TYPE_U32
;
658 mov
->cat1
.dst_type
= TYPE_U32
;
659 mov
->barrier_class
= IR3_BARRIER_ARRAY_W
;
660 mov
->barrier_conflict
= IR3_BARRIER_ARRAY_R
| IR3_BARRIER_ARRAY_W
;
661 dst
= ir3_reg_create(mov
, 0, IR3_REG_ARRAY
|
662 COND(address
, IR3_REG_RELATIV
));
663 dst
->instr
= arr
->last_write
;
664 dst
->size
= arr
->length
;
665 dst
->array
.id
= arr
->id
;
666 dst
->array
.offset
= n
;
667 ir3_reg_create(mov
, 0, IR3_REG_SSA
)->instr
= src
;
670 ir3_instr_set_address(mov
, address
);
672 arr
->last_write
= mov
;
674 /* the array store may only matter to something in an earlier
675 * block (ie. loops), but since arrays are not in SSA, depth
676 * pass won't know this.. so keep all array stores:
678 array_insert(block
, block
->keeps
, mov
);