2 * Copyright (C) 2015-2018 Rob Clark <robclark@freedesktop.org>
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
24 * Rob Clark <robclark@freedesktop.org>
27 #include "util/u_math.h"
29 #include "ir3_compiler.h"
30 #include "ir3_context.h"
31 #include "ir3_image.h"
32 #include "ir3_shader.h"
36 ir3_context_init(struct ir3_compiler
*compiler
,
37 struct ir3_shader_variant
*so
)
39 struct ir3_context
*ctx
= rzalloc(NULL
, struct ir3_context
);
41 if (compiler
->gpu_id
>= 400) {
42 if (so
->type
== MESA_SHADER_VERTEX
) {
43 ctx
->astc_srgb
= so
->key
.vastc_srgb
;
44 } else if (so
->type
== MESA_SHADER_FRAGMENT
) {
45 ctx
->astc_srgb
= so
->key
.fastc_srgb
;
49 if (so
->type
== MESA_SHADER_VERTEX
) {
50 ctx
->samples
= so
->key
.vsamples
;
51 } else if (so
->type
== MESA_SHADER_FRAGMENT
) {
52 ctx
->samples
= so
->key
.fsamples
;
56 if (compiler
->gpu_id
>= 600) {
57 ctx
->funcs
= &ir3_a6xx_funcs
;
58 } else if (compiler
->gpu_id
>= 400) {
59 ctx
->funcs
= &ir3_a4xx_funcs
;
62 ctx
->compiler
= compiler
;
64 ctx
->def_ht
= _mesa_hash_table_create(ctx
,
65 _mesa_hash_pointer
, _mesa_key_pointer_equal
);
66 ctx
->block_ht
= _mesa_hash_table_create(ctx
,
67 _mesa_hash_pointer
, _mesa_key_pointer_equal
);
69 /* TODO: maybe generate some sort of bitmask of what key
70 * lowers vs what shader has (ie. no need to lower
71 * texture clamp lowering if no texture sample instrs)..
72 * although should be done further up the stack to avoid
73 * creating duplicate variants..
76 if (ir3_key_lowers_nir(&so
->key
)) {
77 nir_shader
*s
= nir_shader_clone(ctx
, so
->shader
->nir
);
78 ctx
->s
= ir3_optimize_nir(so
->shader
, s
, &so
->key
);
80 /* fast-path for shader key that lowers nothing in NIR: */
81 ctx
->s
= nir_shader_clone(ctx
, so
->shader
->nir
);
84 /* this needs to be the last pass run, so do this here instead of
85 * in ir3_optimize_nir():
87 NIR_PASS_V(ctx
->s
, nir_lower_bool_to_int32
);
88 NIR_PASS_V(ctx
->s
, nir_lower_locals_to_regs
);
89 NIR_PASS_V(ctx
->s
, nir_convert_from_ssa
, true);
91 if (ir3_shader_debug
& IR3_DBG_DISASM
) {
92 DBG("dump nir%dv%d: type=%d, k={cts=%u,hp=%u}",
93 so
->shader
->id
, so
->id
, so
->type
,
94 so
->key
.color_two_side
, so
->key
.half_precision
);
95 nir_print_shader(ctx
->s
, stdout
);
98 if (shader_debug_enabled(so
->type
)) {
99 fprintf(stderr
, "NIR (final form) for %s shader:\n",
100 _mesa_shader_stage_to_string(so
->type
));
101 nir_print_shader(ctx
->s
, stderr
);
104 ir3_nir_scan_driver_consts(ctx
->s
, &so
->const_layout
);
106 so
->num_uniforms
= ctx
->s
->num_uniforms
;
107 so
->num_ubos
= ctx
->s
->info
.num_ubos
;
109 ir3_ibo_mapping_init(&so
->image_mapping
, ctx
->s
->info
.num_textures
);
111 /* Layout of constant registers, each section aligned to vec4. Note
112 * that pointer size (ubo, etc) changes depending on generation.
117 * if (vertex shader) {
118 * driver params (IR3_DP_*)
119 * if (stream_output.num_outputs > 0)
120 * stream-out addresses
124 * Immediates go last mostly because they are inserted in the CP pass
125 * after the nir -> ir3 frontend.
127 * Note UBO size in bytes should be aligned to vec4
129 debug_assert((ctx
->so
->shader
->ubo_state
.size
% 16) == 0);
130 unsigned constoff
= align(ctx
->so
->shader
->ubo_state
.size
/ 16, 4);
131 unsigned ptrsz
= ir3_pointer_size(ctx
->compiler
);
133 memset(&so
->constbase
, ~0, sizeof(so
->constbase
));
135 if (so
->num_ubos
> 0) {
136 so
->constbase
.ubo
= constoff
;
137 constoff
+= align(ctx
->s
->info
.num_ubos
* ptrsz
, 4) / 4;
140 if (so
->const_layout
.ssbo_size
.count
> 0) {
141 unsigned cnt
= so
->const_layout
.ssbo_size
.count
;
142 so
->constbase
.ssbo_sizes
= constoff
;
143 constoff
+= align(cnt
, 4) / 4;
146 if (so
->const_layout
.image_dims
.count
> 0) {
147 unsigned cnt
= so
->const_layout
.image_dims
.count
;
148 so
->constbase
.image_dims
= constoff
;
149 constoff
+= align(cnt
, 4) / 4;
152 unsigned num_driver_params
= 0;
153 if (so
->type
== MESA_SHADER_VERTEX
) {
154 num_driver_params
= IR3_DP_VS_COUNT
;
155 } else if (so
->type
== MESA_SHADER_COMPUTE
) {
156 num_driver_params
= IR3_DP_CS_COUNT
;
159 so
->constbase
.driver_param
= constoff
;
160 constoff
+= align(num_driver_params
, 4) / 4;
162 if ((so
->type
== MESA_SHADER_VERTEX
) &&
163 (compiler
->gpu_id
< 500) &&
164 so
->shader
->stream_output
.num_outputs
> 0) {
165 so
->constbase
.tfbo
= constoff
;
166 constoff
+= align(IR3_MAX_SO_BUFFERS
* ptrsz
, 4) / 4;
169 so
->constbase
.immediate
= constoff
;
175 ir3_context_free(struct ir3_context
*ctx
)
184 /* allocate a n element value array (to be populated by caller) and
187 struct ir3_instruction
**
188 ir3_get_dst_ssa(struct ir3_context
*ctx
, nir_ssa_def
*dst
, unsigned n
)
190 struct ir3_instruction
**value
=
191 ralloc_array(ctx
->def_ht
, struct ir3_instruction
*, n
);
192 _mesa_hash_table_insert(ctx
->def_ht
, dst
, value
);
196 struct ir3_instruction
**
197 ir3_get_dst(struct ir3_context
*ctx
, nir_dest
*dst
, unsigned n
)
199 struct ir3_instruction
**value
;
202 value
= ir3_get_dst_ssa(ctx
, &dst
->ssa
, n
);
204 value
= ralloc_array(ctx
, struct ir3_instruction
*, n
);
207 /* NOTE: in non-ssa case, we don't really need to store last_dst
208 * but this helps us catch cases where put_dst() call is forgotten
210 compile_assert(ctx
, !ctx
->last_dst
);
211 ctx
->last_dst
= value
;
217 struct ir3_instruction
* const *
218 ir3_get_src(struct ir3_context
*ctx
, nir_src
*src
)
221 struct hash_entry
*entry
;
222 entry
= _mesa_hash_table_search(ctx
->def_ht
, src
->ssa
);
223 compile_assert(ctx
, entry
);
226 nir_register
*reg
= src
->reg
.reg
;
227 struct ir3_array
*arr
= ir3_get_array(ctx
, reg
);
228 unsigned num_components
= arr
->r
->num_components
;
229 struct ir3_instruction
*addr
= NULL
;
230 struct ir3_instruction
**value
=
231 ralloc_array(ctx
, struct ir3_instruction
*, num_components
);
233 if (src
->reg
.indirect
)
234 addr
= ir3_get_addr(ctx
, ir3_get_src(ctx
, src
->reg
.indirect
)[0],
235 reg
->num_components
);
237 for (unsigned i
= 0; i
< num_components
; i
++) {
238 unsigned n
= src
->reg
.base_offset
* reg
->num_components
+ i
;
239 compile_assert(ctx
, n
< arr
->length
);
240 value
[i
] = ir3_create_array_load(ctx
, arr
, n
, addr
);
248 ir3_put_dst(struct ir3_context
*ctx
, nir_dest
*dst
)
250 unsigned bit_size
= nir_dest_bit_size(*dst
);
252 /* add extra mov if dst value is HIGH reg.. in some cases not all
253 * instructions can read from HIGH regs, in cases where they can
254 * ir3_cp will clean up the extra mov:
256 for (unsigned i
= 0; i
< ctx
->last_dst_n
; i
++) {
257 if (!ctx
->last_dst
[i
])
259 if (ctx
->last_dst
[i
]->regs
[0]->flags
& IR3_REG_HIGH
) {
260 ctx
->last_dst
[i
] = ir3_MOV(ctx
->block
, ctx
->last_dst
[i
], TYPE_U32
);
265 for (unsigned i
= 0; i
< ctx
->last_dst_n
; i
++) {
266 struct ir3_instruction
*dst
= ctx
->last_dst
[i
];
267 dst
->regs
[0]->flags
|= IR3_REG_HALF
;
268 if (ctx
->last_dst
[i
]->opc
== OPC_META_FO
)
269 dst
->regs
[1]->instr
->regs
[0]->flags
|= IR3_REG_HALF
;
274 nir_register
*reg
= dst
->reg
.reg
;
275 struct ir3_array
*arr
= ir3_get_array(ctx
, reg
);
276 unsigned num_components
= ctx
->last_dst_n
;
277 struct ir3_instruction
*addr
= NULL
;
279 if (dst
->reg
.indirect
)
280 addr
= ir3_get_addr(ctx
, ir3_get_src(ctx
, dst
->reg
.indirect
)[0],
281 reg
->num_components
);
283 for (unsigned i
= 0; i
< num_components
; i
++) {
284 unsigned n
= dst
->reg
.base_offset
* reg
->num_components
+ i
;
285 compile_assert(ctx
, n
< arr
->length
);
286 if (!ctx
->last_dst
[i
])
288 ir3_create_array_store(ctx
, arr
, n
, ctx
->last_dst
[i
], addr
);
291 ralloc_free(ctx
->last_dst
);
294 ctx
->last_dst
= NULL
;
298 struct ir3_instruction
*
299 ir3_create_collect(struct ir3_context
*ctx
, struct ir3_instruction
*const *arr
,
302 struct ir3_block
*block
= ctx
->block
;
303 struct ir3_instruction
*collect
;
308 unsigned flags
= arr
[0]->regs
[0]->flags
& IR3_REG_HALF
;
310 collect
= ir3_instr_create2(block
, OPC_META_FI
, 1 + arrsz
);
311 ir3_reg_create(collect
, 0, flags
); /* dst */
312 for (unsigned i
= 0; i
< arrsz
; i
++) {
313 struct ir3_instruction
*elem
= arr
[i
];
315 /* Since arrays are pre-colored in RA, we can't assume that
316 * things will end up in the right place. (Ie. if a collect
317 * joins elements from two different arrays.) So insert an
320 * We could possibly skip this if all the collected elements
321 * are contiguous elements in a single array.. not sure how
322 * likely that is to happen.
324 * Fixes a problem with glamor shaders, that in effect do
331 * color = texture2D(tex, texcoord);
333 * In this case, texcoord will end up as nir registers (which
334 * translate to ir3 array's of length 1. And we can't assume
335 * the two (or more) arrays will get allocated in consecutive
339 if (elem
->regs
[0]->flags
& IR3_REG_ARRAY
) {
340 type_t type
= (flags
& IR3_REG_HALF
) ? TYPE_U16
: TYPE_U32
;
341 elem
= ir3_MOV(block
, elem
, type
);
344 compile_assert(ctx
, (elem
->regs
[0]->flags
& IR3_REG_HALF
) == flags
);
345 ir3_reg_create(collect
, 0, IR3_REG_SSA
| flags
)->instr
= elem
;
348 collect
->regs
[0]->wrmask
= MASK(arrsz
);
353 /* helper for instructions that produce multiple consecutive scalar
354 * outputs which need to have a split/fanout meta instruction inserted
357 ir3_split_dest(struct ir3_block
*block
, struct ir3_instruction
**dst
,
358 struct ir3_instruction
*src
, unsigned base
, unsigned n
)
360 struct ir3_instruction
*prev
= NULL
;
362 if ((n
== 1) && (src
->regs
[0]->wrmask
== 0x1)) {
367 unsigned flags
= src
->regs
[0]->flags
& (IR3_REG_HALF
| IR3_REG_HIGH
);
369 for (int i
= 0, j
= 0; i
< n
; i
++) {
370 struct ir3_instruction
*split
= ir3_instr_create(block
, OPC_META_FO
);
371 ir3_reg_create(split
, 0, IR3_REG_SSA
| flags
);
372 ir3_reg_create(split
, 0, IR3_REG_SSA
| flags
)->instr
= src
;
373 split
->fo
.off
= i
+ base
;
376 split
->cp
.left
= prev
;
377 split
->cp
.left_cnt
++;
378 prev
->cp
.right
= split
;
379 prev
->cp
.right_cnt
++;
383 if (src
->regs
[0]->wrmask
& (1 << (i
+ base
)))
389 ir3_context_error(struct ir3_context
*ctx
, const char *format
, ...)
391 struct hash_table
*errors
= NULL
;
393 va_start(ap
, format
);
394 if (ctx
->cur_instr
) {
395 errors
= _mesa_hash_table_create(NULL
,
397 _mesa_key_pointer_equal
);
398 char *msg
= ralloc_vasprintf(errors
, format
, ap
);
399 _mesa_hash_table_insert(errors
, ctx
->cur_instr
, msg
);
401 _debug_vprintf(format
, ap
);
404 nir_print_shader_annotated(ctx
->s
, stdout
, errors
);
410 static struct ir3_instruction
*
411 create_addr(struct ir3_block
*block
, struct ir3_instruction
*src
, int align
)
413 struct ir3_instruction
*instr
, *immed
;
415 /* TODO in at least some cases, the backend could probably be
416 * made clever enough to propagate IR3_REG_HALF..
418 instr
= ir3_COV(block
, src
, TYPE_U32
, TYPE_S16
);
419 instr
->regs
[0]->flags
|= IR3_REG_HALF
;
426 /* src *= 2 => src <<= 1: */
427 immed
= create_immed(block
, 1);
428 immed
->regs
[0]->flags
|= IR3_REG_HALF
;
430 instr
= ir3_SHL_B(block
, instr
, 0, immed
, 0);
431 instr
->regs
[0]->flags
|= IR3_REG_HALF
;
432 instr
->regs
[1]->flags
|= IR3_REG_HALF
;
436 immed
= create_immed(block
, 3);
437 immed
->regs
[0]->flags
|= IR3_REG_HALF
;
439 instr
= ir3_MULL_U(block
, instr
, 0, immed
, 0);
440 instr
->regs
[0]->flags
|= IR3_REG_HALF
;
441 instr
->regs
[1]->flags
|= IR3_REG_HALF
;
444 /* src *= 4 => src <<= 2: */
445 immed
= create_immed(block
, 2);
446 immed
->regs
[0]->flags
|= IR3_REG_HALF
;
448 instr
= ir3_SHL_B(block
, instr
, 0, immed
, 0);
449 instr
->regs
[0]->flags
|= IR3_REG_HALF
;
450 instr
->regs
[1]->flags
|= IR3_REG_HALF
;
453 unreachable("bad align");
457 instr
= ir3_MOV(block
, instr
, TYPE_S16
);
458 instr
->regs
[0]->num
= regid(REG_A0
, 0);
459 instr
->regs
[0]->flags
|= IR3_REG_HALF
;
460 instr
->regs
[1]->flags
|= IR3_REG_HALF
;
465 /* caches addr values to avoid generating multiple cov/shl/mova
466 * sequences for each use of a given NIR level src as address
468 struct ir3_instruction
*
469 ir3_get_addr(struct ir3_context
*ctx
, struct ir3_instruction
*src
, int align
)
471 struct ir3_instruction
*addr
;
472 unsigned idx
= align
- 1;
474 compile_assert(ctx
, idx
< ARRAY_SIZE(ctx
->addr_ht
));
476 if (!ctx
->addr_ht
[idx
]) {
477 ctx
->addr_ht
[idx
] = _mesa_hash_table_create(ctx
,
478 _mesa_hash_pointer
, _mesa_key_pointer_equal
);
480 struct hash_entry
*entry
;
481 entry
= _mesa_hash_table_search(ctx
->addr_ht
[idx
], src
);
486 addr
= create_addr(ctx
->block
, src
, align
);
487 _mesa_hash_table_insert(ctx
->addr_ht
[idx
], src
, addr
);
492 struct ir3_instruction
*
493 ir3_get_predicate(struct ir3_context
*ctx
, struct ir3_instruction
*src
)
495 struct ir3_block
*b
= ctx
->block
;
496 struct ir3_instruction
*cond
;
498 /* NOTE: only cmps.*.* can write p0.x: */
499 cond
= ir3_CMPS_S(b
, src
, 0, create_immed(b
, 0), 0);
500 cond
->cat2
.condition
= IR3_COND_NE
;
502 /* condition always goes in predicate register: */
503 cond
->regs
[0]->num
= regid(REG_P0
, 0);
513 ir3_declare_array(struct ir3_context
*ctx
, nir_register
*reg
)
515 struct ir3_array
*arr
= rzalloc(ctx
, struct ir3_array
);
516 arr
->id
= ++ctx
->num_arrays
;
517 /* NOTE: sometimes we get non array regs, for example for arrays of
518 * length 1. See fs-const-array-of-struct-of-array.shader_test. So
519 * treat a non-array as if it was an array of length 1.
521 * It would be nice if there was a nir pass to convert arrays of
524 arr
->length
= reg
->num_components
* MAX2(1, reg
->num_array_elems
);
525 compile_assert(ctx
, arr
->length
> 0);
527 list_addtail(&arr
->node
, &ctx
->ir
->array_list
);
531 ir3_get_array(struct ir3_context
*ctx
, nir_register
*reg
)
533 list_for_each_entry (struct ir3_array
, arr
, &ctx
->ir
->array_list
, node
) {
537 ir3_context_error(ctx
, "bogus reg: %s\n", reg
->name
);
541 /* relative (indirect) if address!=NULL */
542 struct ir3_instruction
*
543 ir3_create_array_load(struct ir3_context
*ctx
, struct ir3_array
*arr
, int n
,
544 struct ir3_instruction
*address
)
546 struct ir3_block
*block
= ctx
->block
;
547 struct ir3_instruction
*mov
;
548 struct ir3_register
*src
;
550 mov
= ir3_instr_create(block
, OPC_MOV
);
551 mov
->cat1
.src_type
= TYPE_U32
;
552 mov
->cat1
.dst_type
= TYPE_U32
;
553 mov
->barrier_class
= IR3_BARRIER_ARRAY_R
;
554 mov
->barrier_conflict
= IR3_BARRIER_ARRAY_W
;
555 ir3_reg_create(mov
, 0, 0);
556 src
= ir3_reg_create(mov
, 0, IR3_REG_ARRAY
|
557 COND(address
, IR3_REG_RELATIV
));
558 src
->instr
= arr
->last_write
;
559 src
->size
= arr
->length
;
560 src
->array
.id
= arr
->id
;
561 src
->array
.offset
= n
;
564 ir3_instr_set_address(mov
, address
);
569 /* relative (indirect) if address!=NULL */
571 ir3_create_array_store(struct ir3_context
*ctx
, struct ir3_array
*arr
, int n
,
572 struct ir3_instruction
*src
, struct ir3_instruction
*address
)
574 struct ir3_block
*block
= ctx
->block
;
575 struct ir3_instruction
*mov
;
576 struct ir3_register
*dst
;
578 /* if not relative store, don't create an extra mov, since that
579 * ends up being difficult for cp to remove.
584 src
->barrier_class
|= IR3_BARRIER_ARRAY_W
;
585 src
->barrier_conflict
|= IR3_BARRIER_ARRAY_R
| IR3_BARRIER_ARRAY_W
;
587 dst
->flags
|= IR3_REG_ARRAY
;
588 dst
->instr
= arr
->last_write
;
589 dst
->size
= arr
->length
;
590 dst
->array
.id
= arr
->id
;
591 dst
->array
.offset
= n
;
593 arr
->last_write
= src
;
595 array_insert(block
, block
->keeps
, src
);
600 mov
= ir3_instr_create(block
, OPC_MOV
);
601 mov
->cat1
.src_type
= TYPE_U32
;
602 mov
->cat1
.dst_type
= TYPE_U32
;
603 mov
->barrier_class
= IR3_BARRIER_ARRAY_W
;
604 mov
->barrier_conflict
= IR3_BARRIER_ARRAY_R
| IR3_BARRIER_ARRAY_W
;
605 dst
= ir3_reg_create(mov
, 0, IR3_REG_ARRAY
|
606 COND(address
, IR3_REG_RELATIV
));
607 dst
->instr
= arr
->last_write
;
608 dst
->size
= arr
->length
;
609 dst
->array
.id
= arr
->id
;
610 dst
->array
.offset
= n
;
611 ir3_reg_create(mov
, 0, IR3_REG_SSA
)->instr
= src
;
614 ir3_instr_set_address(mov
, address
);
616 arr
->last_write
= mov
;
618 /* the array store may only matter to something in an earlier
619 * block (ie. loops), but since arrays are not in SSA, depth
620 * pass won't know this.. so keep all array stores:
622 array_insert(block
, block
->keeps
, mov
);