2 * Copyright © 2016-2017 Broadcom
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 #include "broadcom/common/v3d_device_info.h"
25 #include "v3d_compiler.h"
26 #include "util/u_prim.h"
27 #include "compiler/nir/nir_schedule.h"
30 vir_get_nsrc(struct qinst
*inst
)
32 switch (inst
->qpu
.type
) {
33 case V3D_QPU_INSTR_TYPE_BRANCH
:
35 case V3D_QPU_INSTR_TYPE_ALU
:
36 if (inst
->qpu
.alu
.add
.op
!= V3D_QPU_A_NOP
)
37 return v3d_qpu_add_op_num_src(inst
->qpu
.alu
.add
.op
);
39 return v3d_qpu_mul_op_num_src(inst
->qpu
.alu
.mul
.op
);
46 * Returns whether the instruction has any side effects that must be
50 vir_has_side_effects(struct v3d_compile
*c
, struct qinst
*inst
)
52 switch (inst
->qpu
.type
) {
53 case V3D_QPU_INSTR_TYPE_BRANCH
:
55 case V3D_QPU_INSTR_TYPE_ALU
:
56 switch (inst
->qpu
.alu
.add
.op
) {
57 case V3D_QPU_A_SETREVF
:
58 case V3D_QPU_A_SETMSF
:
59 case V3D_QPU_A_VPMSETUP
:
60 case V3D_QPU_A_STVPMV
:
61 case V3D_QPU_A_STVPMD
:
62 case V3D_QPU_A_STVPMP
:
70 switch (inst
->qpu
.alu
.mul
.op
) {
71 case V3D_QPU_M_MULTOP
:
78 if (inst
->qpu
.sig
.ldtmu
||
79 inst
->qpu
.sig
.ldvary
||
80 inst
->qpu
.sig
.ldtlbu
||
81 inst
->qpu
.sig
.ldtlb
||
82 inst
->qpu
.sig
.wrtmuc
||
83 inst
->qpu
.sig
.thrsw
) {
91 vir_is_raw_mov(struct qinst
*inst
)
93 if (inst
->qpu
.type
!= V3D_QPU_INSTR_TYPE_ALU
||
94 (inst
->qpu
.alu
.mul
.op
!= V3D_QPU_M_FMOV
&&
95 inst
->qpu
.alu
.mul
.op
!= V3D_QPU_M_MOV
)) {
99 if (inst
->qpu
.alu
.add
.output_pack
!= V3D_QPU_PACK_NONE
||
100 inst
->qpu
.alu
.mul
.output_pack
!= V3D_QPU_PACK_NONE
) {
104 if (inst
->qpu
.alu
.add
.a_unpack
!= V3D_QPU_UNPACK_NONE
||
105 inst
->qpu
.alu
.add
.b_unpack
!= V3D_QPU_UNPACK_NONE
||
106 inst
->qpu
.alu
.mul
.a_unpack
!= V3D_QPU_UNPACK_NONE
||
107 inst
->qpu
.alu
.mul
.b_unpack
!= V3D_QPU_UNPACK_NONE
) {
111 if (inst
->qpu
.flags
.ac
!= V3D_QPU_COND_NONE
||
112 inst
->qpu
.flags
.mc
!= V3D_QPU_COND_NONE
)
119 vir_is_add(struct qinst
*inst
)
121 return (inst
->qpu
.type
== V3D_QPU_INSTR_TYPE_ALU
&&
122 inst
->qpu
.alu
.add
.op
!= V3D_QPU_A_NOP
);
126 vir_is_mul(struct qinst
*inst
)
128 return (inst
->qpu
.type
== V3D_QPU_INSTR_TYPE_ALU
&&
129 inst
->qpu
.alu
.mul
.op
!= V3D_QPU_M_NOP
);
133 vir_is_tex(struct qinst
*inst
)
135 if (inst
->dst
.file
== QFILE_MAGIC
)
136 return v3d_qpu_magic_waddr_is_tmu(inst
->dst
.index
);
138 if (inst
->qpu
.type
== V3D_QPU_INSTR_TYPE_ALU
&&
139 inst
->qpu
.alu
.add
.op
== V3D_QPU_A_TMUWT
) {
147 vir_writes_r3(const struct v3d_device_info
*devinfo
, struct qinst
*inst
)
149 for (int i
= 0; i
< vir_get_nsrc(inst
); i
++) {
150 switch (inst
->src
[i
].file
) {
158 if (devinfo
->ver
< 41 && (inst
->qpu
.sig
.ldvary
||
159 inst
->qpu
.sig
.ldtlb
||
160 inst
->qpu
.sig
.ldtlbu
||
161 inst
->qpu
.sig
.ldvpm
)) {
169 vir_writes_r4(const struct v3d_device_info
*devinfo
, struct qinst
*inst
)
171 switch (inst
->dst
.file
) {
173 switch (inst
->dst
.index
) {
174 case V3D_QPU_WADDR_RECIP
:
175 case V3D_QPU_WADDR_RSQRT
:
176 case V3D_QPU_WADDR_EXP
:
177 case V3D_QPU_WADDR_LOG
:
178 case V3D_QPU_WADDR_SIN
:
186 if (devinfo
->ver
< 41 && inst
->qpu
.sig
.ldtmu
)
193 vir_set_unpack(struct qinst
*inst
, int src
,
194 enum v3d_qpu_input_unpack unpack
)
196 assert(src
== 0 || src
== 1);
198 if (vir_is_add(inst
)) {
200 inst
->qpu
.alu
.add
.a_unpack
= unpack
;
202 inst
->qpu
.alu
.add
.b_unpack
= unpack
;
204 assert(vir_is_mul(inst
));
206 inst
->qpu
.alu
.mul
.a_unpack
= unpack
;
208 inst
->qpu
.alu
.mul
.b_unpack
= unpack
;
213 vir_set_cond(struct qinst
*inst
, enum v3d_qpu_cond cond
)
215 if (vir_is_add(inst
)) {
216 inst
->qpu
.flags
.ac
= cond
;
218 assert(vir_is_mul(inst
));
219 inst
->qpu
.flags
.mc
= cond
;
224 vir_set_pf(struct qinst
*inst
, enum v3d_qpu_pf pf
)
226 if (vir_is_add(inst
)) {
227 inst
->qpu
.flags
.apf
= pf
;
229 assert(vir_is_mul(inst
));
230 inst
->qpu
.flags
.mpf
= pf
;
235 vir_set_uf(struct qinst
*inst
, enum v3d_qpu_uf uf
)
237 if (vir_is_add(inst
)) {
238 inst
->qpu
.flags
.auf
= uf
;
240 assert(vir_is_mul(inst
));
241 inst
->qpu
.flags
.muf
= uf
;
247 vir_channels_written(struct qinst
*inst
)
249 if (vir_is_mul(inst
)) {
250 switch (inst
->dst
.pack
) {
251 case QPU_PACK_MUL_NOP
:
252 case QPU_PACK_MUL_8888
:
254 case QPU_PACK_MUL_8A
:
256 case QPU_PACK_MUL_8B
:
258 case QPU_PACK_MUL_8C
:
260 case QPU_PACK_MUL_8D
:
264 switch (inst
->dst
.pack
) {
266 case QPU_PACK_A_8888
:
267 case QPU_PACK_A_8888_SAT
:
268 case QPU_PACK_A_32_SAT
:
271 case QPU_PACK_A_8A_SAT
:
274 case QPU_PACK_A_8B_SAT
:
277 case QPU_PACK_A_8C_SAT
:
280 case QPU_PACK_A_8D_SAT
:
283 case QPU_PACK_A_16A_SAT
:
286 case QPU_PACK_A_16B_SAT
:
290 unreachable("Bad pack field");
295 vir_get_temp(struct v3d_compile
*c
)
299 reg
.file
= QFILE_TEMP
;
300 reg
.index
= c
->num_temps
++;
302 if (c
->num_temps
> c
->defs_array_size
) {
303 uint32_t old_size
= c
->defs_array_size
;
304 c
->defs_array_size
= MAX2(old_size
* 2, 16);
306 c
->defs
= reralloc(c
, c
->defs
, struct qinst
*,
308 memset(&c
->defs
[old_size
], 0,
309 sizeof(c
->defs
[0]) * (c
->defs_array_size
- old_size
));
311 c
->spillable
= reralloc(c
, c
->spillable
,
313 BITSET_WORDS(c
->defs_array_size
));
314 for (int i
= old_size
; i
< c
->defs_array_size
; i
++)
315 BITSET_SET(c
->spillable
, i
);
322 vir_add_inst(enum v3d_qpu_add_op op
, struct qreg dst
, struct qreg src0
, struct qreg src1
)
324 struct qinst
*inst
= calloc(1, sizeof(*inst
));
326 inst
->qpu
= v3d_qpu_nop();
327 inst
->qpu
.alu
.add
.op
= op
;
338 vir_mul_inst(enum v3d_qpu_mul_op op
, struct qreg dst
, struct qreg src0
, struct qreg src1
)
340 struct qinst
*inst
= calloc(1, sizeof(*inst
));
342 inst
->qpu
= v3d_qpu_nop();
343 inst
->qpu
.alu
.mul
.op
= op
;
354 vir_branch_inst(struct v3d_compile
*c
, enum v3d_qpu_branch_cond cond
)
356 struct qinst
*inst
= calloc(1, sizeof(*inst
));
358 inst
->qpu
= v3d_qpu_nop();
359 inst
->qpu
.type
= V3D_QPU_INSTR_TYPE_BRANCH
;
360 inst
->qpu
.branch
.cond
= cond
;
361 inst
->qpu
.branch
.msfign
= V3D_QPU_MSFIGN_NONE
;
362 inst
->qpu
.branch
.bdi
= V3D_QPU_BRANCH_DEST_REL
;
363 inst
->qpu
.branch
.ub
= true;
364 inst
->qpu
.branch
.bdu
= V3D_QPU_BRANCH_DEST_REL
;
366 inst
->dst
= vir_nop_reg();
367 inst
->uniform
= vir_get_uniform_index(c
, QUNIFORM_CONSTANT
, 0);
373 vir_emit(struct v3d_compile
*c
, struct qinst
*inst
)
375 switch (c
->cursor
.mode
) {
377 list_add(&inst
->link
, c
->cursor
.link
);
379 case vir_cursor_addtail
:
380 list_addtail(&inst
->link
, c
->cursor
.link
);
384 c
->cursor
= vir_after_inst(inst
);
385 c
->live_intervals_valid
= false;
388 /* Updates inst to write to a new temporary, emits it, and notes the def. */
390 vir_emit_def(struct v3d_compile
*c
, struct qinst
*inst
)
392 assert(inst
->dst
.file
== QFILE_NULL
);
394 /* If we're emitting an instruction that's a def, it had better be
395 * writing a register.
397 if (inst
->qpu
.type
== V3D_QPU_INSTR_TYPE_ALU
) {
398 assert(inst
->qpu
.alu
.add
.op
== V3D_QPU_A_NOP
||
399 v3d_qpu_add_op_has_dst(inst
->qpu
.alu
.add
.op
));
400 assert(inst
->qpu
.alu
.mul
.op
== V3D_QPU_M_NOP
||
401 v3d_qpu_mul_op_has_dst(inst
->qpu
.alu
.mul
.op
));
404 inst
->dst
= vir_get_temp(c
);
406 if (inst
->dst
.file
== QFILE_TEMP
)
407 c
->defs
[inst
->dst
.index
] = inst
;
415 vir_emit_nondef(struct v3d_compile
*c
, struct qinst
*inst
)
417 if (inst
->dst
.file
== QFILE_TEMP
)
418 c
->defs
[inst
->dst
.index
] = NULL
;
426 vir_new_block(struct v3d_compile
*c
)
428 struct qblock
*block
= rzalloc(c
, struct qblock
);
430 list_inithead(&block
->instructions
);
432 block
->predecessors
= _mesa_set_create(block
,
434 _mesa_key_pointer_equal
);
436 block
->index
= c
->next_block_index
++;
442 vir_set_emit_block(struct v3d_compile
*c
, struct qblock
*block
)
444 c
->cur_block
= block
;
445 c
->cursor
= vir_after_block(block
);
446 list_addtail(&block
->link
, &c
->blocks
);
450 vir_entry_block(struct v3d_compile
*c
)
452 return list_first_entry(&c
->blocks
, struct qblock
, link
);
456 vir_exit_block(struct v3d_compile
*c
)
458 return list_last_entry(&c
->blocks
, struct qblock
, link
);
462 vir_link_blocks(struct qblock
*predecessor
, struct qblock
*successor
)
464 _mesa_set_add(successor
->predecessors
, predecessor
);
465 if (predecessor
->successors
[0]) {
466 assert(!predecessor
->successors
[1]);
467 predecessor
->successors
[1] = successor
;
469 predecessor
->successors
[0] = successor
;
473 const struct v3d_compiler
*
474 v3d_compiler_init(const struct v3d_device_info
*devinfo
)
476 struct v3d_compiler
*compiler
= rzalloc(NULL
, struct v3d_compiler
);
480 compiler
->devinfo
= devinfo
;
482 if (!vir_init_reg_sets(compiler
)) {
483 ralloc_free(compiler
);
491 v3d_compiler_free(const struct v3d_compiler
*compiler
)
493 ralloc_free((void *)compiler
);
496 static struct v3d_compile
*
497 vir_compile_init(const struct v3d_compiler
*compiler
,
500 void (*debug_output
)(const char *msg
,
501 void *debug_output_data
),
502 void *debug_output_data
,
503 int program_id
, int variant_id
,
504 bool fallback_scheduler
)
506 struct v3d_compile
*c
= rzalloc(NULL
, struct v3d_compile
);
508 c
->compiler
= compiler
;
509 c
->devinfo
= compiler
->devinfo
;
511 c
->program_id
= program_id
;
512 c
->variant_id
= variant_id
;
514 c
->debug_output
= debug_output
;
515 c
->debug_output_data
= debug_output_data
;
516 c
->compilation_result
= V3D_COMPILATION_SUCCEEDED
;
517 c
->fallback_scheduler
= fallback_scheduler
;
519 s
= nir_shader_clone(c
, s
);
522 list_inithead(&c
->blocks
);
523 vir_set_emit_block(c
, vir_new_block(c
));
525 c
->output_position_index
= -1;
526 c
->output_sample_mask_index
= -1;
528 c
->def_ht
= _mesa_hash_table_create(c
, _mesa_hash_pointer
,
529 _mesa_key_pointer_equal
);
535 type_size_vec4(const struct glsl_type
*type
, bool bindless
)
537 return glsl_count_attribute_slots(type
, false);
541 v3d_lower_nir(struct v3d_compile
*c
)
543 struct nir_lower_tex_options tex_options
= {
545 .lower_tg4_broadcom_swizzle
= true,
547 .lower_rect
= false, /* XXX: Use this on V3D 3.x */
549 /* Apply swizzles to all samplers. */
550 .swizzle_result
= ~0,
553 /* Lower the format swizzle and (for 32-bit returns)
554 * ARB_texture_swizzle-style swizzle.
556 for (int i
= 0; i
< ARRAY_SIZE(c
->key
->tex
); i
++) {
557 for (int j
= 0; j
< 4; j
++)
558 tex_options
.swizzles
[i
][j
] = c
->key
->tex
[i
].swizzle
[j
];
560 if (c
->key
->tex
[i
].clamp_s
)
561 tex_options
.saturate_s
|= 1 << i
;
562 if (c
->key
->tex
[i
].clamp_t
)
563 tex_options
.saturate_t
|= 1 << i
;
564 if (c
->key
->tex
[i
].clamp_r
)
565 tex_options
.saturate_r
|= 1 << i
;
566 if (c
->key
->tex
[i
].return_size
== 16) {
567 tex_options
.lower_tex_packing
[i
] =
568 nir_lower_tex_packing_16
;
572 /* CS textures may not have return_size reflecting the shadow state. */
573 nir_foreach_variable(var
, &c
->s
->uniforms
) {
574 const struct glsl_type
*type
= glsl_without_array(var
->type
);
575 unsigned array_len
= MAX2(glsl_get_length(var
->type
), 1);
577 if (!glsl_type_is_sampler(type
) ||
578 !glsl_sampler_type_is_shadow(type
))
581 for (int i
= 0; i
< array_len
; i
++) {
582 tex_options
.lower_tex_packing
[var
->data
.binding
+ i
] =
583 nir_lower_tex_packing_16
;
587 NIR_PASS_V(c
->s
, nir_lower_tex
, &tex_options
);
588 NIR_PASS_V(c
->s
, nir_lower_system_values
);
590 NIR_PASS_V(c
->s
, nir_lower_vars_to_scratch
,
591 nir_var_function_temp
,
593 glsl_get_natural_size_align_bytes
);
594 NIR_PASS_V(c
->s
, v3d_nir_lower_scratch
);
598 v3d_set_prog_data_uniforms(struct v3d_compile
*c
,
599 struct v3d_prog_data
*prog_data
)
601 int count
= c
->num_uniforms
;
602 struct v3d_uniform_list
*ulist
= &prog_data
->uniforms
;
604 ulist
->count
= count
;
605 ulist
->data
= ralloc_array(prog_data
, uint32_t, count
);
606 memcpy(ulist
->data
, c
->uniform_data
,
607 count
* sizeof(*ulist
->data
));
608 ulist
->contents
= ralloc_array(prog_data
, enum quniform_contents
, count
);
609 memcpy(ulist
->contents
, c
->uniform_contents
,
610 count
* sizeof(*ulist
->contents
));
614 v3d_vs_set_prog_data(struct v3d_compile
*c
,
615 struct v3d_vs_prog_data
*prog_data
)
617 /* The vertex data gets format converted by the VPM so that
618 * each attribute channel takes up a VPM column. Precompute
619 * the sizes for the shader record.
621 for (int i
= 0; i
< ARRAY_SIZE(prog_data
->vattr_sizes
); i
++) {
622 prog_data
->vattr_sizes
[i
] = c
->vattr_sizes
[i
];
623 prog_data
->vpm_input_size
+= c
->vattr_sizes
[i
];
626 prog_data
->uses_vid
= (c
->s
->info
.system_values_read
&
627 (1ull << SYSTEM_VALUE_VERTEX_ID
));
628 prog_data
->uses_iid
= (c
->s
->info
.system_values_read
&
629 (1ull << SYSTEM_VALUE_INSTANCE_ID
));
631 if (prog_data
->uses_vid
)
632 prog_data
->vpm_input_size
++;
633 if (prog_data
->uses_iid
)
634 prog_data
->vpm_input_size
++;
636 /* Input/output segment size are in sectors (8 rows of 32 bits per
639 prog_data
->vpm_input_size
= align(prog_data
->vpm_input_size
, 8) / 8;
640 prog_data
->vpm_output_size
= align(c
->vpm_output_size
, 8) / 8;
642 /* Set us up for shared input/output segments. This is apparently
643 * necessary for our VCM setup to avoid varying corruption.
645 prog_data
->separate_segments
= false;
646 prog_data
->vpm_output_size
= MAX2(prog_data
->vpm_output_size
,
647 prog_data
->vpm_input_size
);
648 prog_data
->vpm_input_size
= 0;
650 /* Compute VCM cache size. We set up our program to take up less than
651 * half of the VPM, so that any set of bin and render programs won't
652 * run out of space. We need space for at least one input segment,
653 * and then allocate the rest to output segments (one for the current
654 * program, the rest to VCM). The valid range of the VCM cache size
655 * field is 1-4 16-vertex batches, but GFXH-1744 limits us to 2-4
658 assert(c
->devinfo
->vpm_size
);
659 int sector_size
= V3D_CHANNELS
* sizeof(uint32_t) * 8;
660 int vpm_size_in_sectors
= c
->devinfo
->vpm_size
/ sector_size
;
661 int half_vpm
= vpm_size_in_sectors
/ 2;
662 int vpm_output_sectors
= half_vpm
- prog_data
->vpm_input_size
;
663 int vpm_output_batches
= vpm_output_sectors
/ prog_data
->vpm_output_size
;
664 assert(vpm_output_batches
>= 2);
665 prog_data
->vcm_cache_size
= CLAMP(vpm_output_batches
- 1, 2, 4);
669 v3d_gs_set_prog_data(struct v3d_compile
*c
,
670 struct v3d_gs_prog_data
*prog_data
)
672 prog_data
->num_inputs
= c
->num_inputs
;
673 memcpy(prog_data
->input_slots
, c
->input_slots
,
674 c
->num_inputs
* sizeof(*c
->input_slots
));
676 /* gl_PrimitiveIdIn is written by the GBG into the first word of the
677 * VPM output header automatically and the shader will overwrite
678 * it after reading it if necessary, so it doesn't add to the VPM
681 prog_data
->uses_pid
= (c
->s
->info
.system_values_read
&
682 (1ull << SYSTEM_VALUE_PRIMITIVE_ID
));
684 /* Output segment size is in sectors (8 rows of 32 bits per channel) */
685 prog_data
->vpm_output_size
= align(c
->vpm_output_size
, 8) / 8;
687 /* Compute SIMD dispatch width and update VPM output size accordingly
688 * to ensure we can fit our program in memory. Available widths are
691 * Notice that at draw time we will have to consider VPM memory
692 * requirements from other stages and choose a smaller dispatch
693 * width if needed to fit the program in VPM memory.
695 prog_data
->simd_width
= 16;
696 while ((prog_data
->simd_width
> 1 && prog_data
->vpm_output_size
> 16) ||
697 prog_data
->simd_width
== 2) {
698 prog_data
->simd_width
>>= 1;
699 prog_data
->vpm_output_size
=
700 align(prog_data
->vpm_output_size
, 2) / 2;
702 assert(prog_data
->vpm_output_size
<= 16);
703 assert(prog_data
->simd_width
!= 2);
705 prog_data
->out_prim_type
= c
->s
->info
.gs
.output_primitive
;
706 prog_data
->num_invocations
= c
->s
->info
.gs
.invocations
;
710 v3d_set_fs_prog_data_inputs(struct v3d_compile
*c
,
711 struct v3d_fs_prog_data
*prog_data
)
713 prog_data
->num_inputs
= c
->num_inputs
;
714 memcpy(prog_data
->input_slots
, c
->input_slots
,
715 c
->num_inputs
* sizeof(*c
->input_slots
));
717 STATIC_ASSERT(ARRAY_SIZE(prog_data
->flat_shade_flags
) >
718 (V3D_MAX_FS_INPUTS
- 1) / 24);
719 for (int i
= 0; i
< V3D_MAX_FS_INPUTS
; i
++) {
720 if (BITSET_TEST(c
->flat_shade_flags
, i
))
721 prog_data
->flat_shade_flags
[i
/ 24] |= 1 << (i
% 24);
723 if (BITSET_TEST(c
->noperspective_flags
, i
))
724 prog_data
->noperspective_flags
[i
/ 24] |= 1 << (i
% 24);
726 if (BITSET_TEST(c
->centroid_flags
, i
))
727 prog_data
->centroid_flags
[i
/ 24] |= 1 << (i
% 24);
732 v3d_fs_set_prog_data(struct v3d_compile
*c
,
733 struct v3d_fs_prog_data
*prog_data
)
735 v3d_set_fs_prog_data_inputs(c
, prog_data
);
736 prog_data
->writes_z
= c
->writes_z
;
737 prog_data
->disable_ez
= !c
->s
->info
.fs
.early_fragment_tests
;
738 prog_data
->uses_center_w
= c
->uses_center_w
;
739 prog_data
->uses_implicit_point_line_varyings
=
740 c
->uses_implicit_point_line_varyings
;
741 prog_data
->lock_scoreboard_on_first_thrsw
=
742 c
->lock_scoreboard_on_first_thrsw
;
746 v3d_cs_set_prog_data(struct v3d_compile
*c
,
747 struct v3d_compute_prog_data
*prog_data
)
749 prog_data
->shared_size
= c
->s
->info
.cs
.shared_size
;
753 v3d_set_prog_data(struct v3d_compile
*c
,
754 struct v3d_prog_data
*prog_data
)
756 prog_data
->threads
= c
->threads
;
757 prog_data
->single_seg
= !c
->last_thrsw
;
758 prog_data
->spill_size
= c
->spill_size
;
759 prog_data
->tmu_dirty_rcl
= c
->tmu_dirty_rcl
;
761 v3d_set_prog_data_uniforms(c
, prog_data
);
763 switch (c
->s
->info
.stage
) {
764 case MESA_SHADER_VERTEX
:
765 v3d_vs_set_prog_data(c
, (struct v3d_vs_prog_data
*)prog_data
);
767 case MESA_SHADER_GEOMETRY
:
768 v3d_gs_set_prog_data(c
, (struct v3d_gs_prog_data
*)prog_data
);
770 case MESA_SHADER_FRAGMENT
:
771 v3d_fs_set_prog_data(c
, (struct v3d_fs_prog_data
*)prog_data
);
773 case MESA_SHADER_COMPUTE
:
774 v3d_cs_set_prog_data(c
, (struct v3d_compute_prog_data
*)prog_data
);
777 unreachable("unsupported shader stage");
782 v3d_return_qpu_insts(struct v3d_compile
*c
, uint32_t *final_assembly_size
)
784 *final_assembly_size
= c
->qpu_inst_count
* sizeof(uint64_t);
786 uint64_t *qpu_insts
= malloc(*final_assembly_size
);
790 memcpy(qpu_insts
, c
->qpu_insts
, *final_assembly_size
);
792 vir_compile_destroy(c
);
798 v3d_nir_lower_vs_early(struct v3d_compile
*c
)
800 /* Split our I/O vars and dead code eliminate the unused
803 NIR_PASS_V(c
->s
, nir_lower_io_to_scalar_early
,
804 nir_var_shader_in
| nir_var_shader_out
);
805 uint64_t used_outputs
[4] = {0};
806 for (int i
= 0; i
< c
->vs_key
->num_used_outputs
; i
++) {
807 int slot
= v3d_slot_get_slot(c
->vs_key
->used_outputs
[i
]);
808 int comp
= v3d_slot_get_component(c
->vs_key
->used_outputs
[i
]);
809 used_outputs
[comp
] |= 1ull << slot
;
811 NIR_PASS_V(c
->s
, nir_remove_unused_io_vars
,
812 nir_var_shader_out
, used_outputs
, NULL
); /* demotes to globals */
813 NIR_PASS_V(c
->s
, nir_lower_global_vars_to_local
);
814 v3d_optimize_nir(c
->s
);
815 NIR_PASS_V(c
->s
, nir_remove_dead_variables
, nir_var_shader_in
, NULL
);
817 /* This must go before nir_lower_io */
818 if (c
->vs_key
->per_vertex_point_size
)
819 NIR_PASS_V(c
->s
, nir_lower_point_size
, 1.0f
, 0.0f
);
821 NIR_PASS_V(c
->s
, nir_lower_io
, nir_var_shader_in
| nir_var_shader_out
,
823 (nir_lower_io_options
)0);
824 /* clean up nir_lower_io's deref_var remains */
825 NIR_PASS_V(c
->s
, nir_opt_dce
);
829 v3d_nir_lower_gs_early(struct v3d_compile
*c
)
831 /* Split our I/O vars and dead code eliminate the unused
834 NIR_PASS_V(c
->s
, nir_lower_io_to_scalar_early
,
835 nir_var_shader_in
| nir_var_shader_out
);
836 uint64_t used_outputs
[4] = {0};
837 for (int i
= 0; i
< c
->gs_key
->num_used_outputs
; i
++) {
838 int slot
= v3d_slot_get_slot(c
->gs_key
->used_outputs
[i
]);
839 int comp
= v3d_slot_get_component(c
->gs_key
->used_outputs
[i
]);
840 used_outputs
[comp
] |= 1ull << slot
;
842 NIR_PASS_V(c
->s
, nir_remove_unused_io_vars
,
843 nir_var_shader_out
, used_outputs
, NULL
); /* demotes to globals */
844 NIR_PASS_V(c
->s
, nir_lower_global_vars_to_local
);
845 v3d_optimize_nir(c
->s
);
846 NIR_PASS_V(c
->s
, nir_remove_dead_variables
, nir_var_shader_in
, NULL
);
848 /* This must go before nir_lower_io */
849 if (c
->gs_key
->per_vertex_point_size
)
850 NIR_PASS_V(c
->s
, nir_lower_point_size
, 1.0f
, 0.0f
);
852 NIR_PASS_V(c
->s
, nir_lower_io
, nir_var_shader_in
| nir_var_shader_out
,
854 (nir_lower_io_options
)0);
855 /* clean up nir_lower_io's deref_var remains */
856 NIR_PASS_V(c
->s
, nir_opt_dce
);
860 v3d_fixup_fs_output_types(struct v3d_compile
*c
)
862 nir_foreach_shader_out_variable(var
, c
->s
) {
865 switch (var
->data
.location
) {
866 case FRAG_RESULT_COLOR
:
869 case FRAG_RESULT_DATA0
:
870 case FRAG_RESULT_DATA1
:
871 case FRAG_RESULT_DATA2
:
872 case FRAG_RESULT_DATA3
:
873 mask
= 1 << (var
->data
.location
- FRAG_RESULT_DATA0
);
877 if (c
->fs_key
->int_color_rb
& mask
) {
879 glsl_vector_type(GLSL_TYPE_INT
,
880 glsl_get_components(var
->type
));
881 } else if (c
->fs_key
->uint_color_rb
& mask
) {
883 glsl_vector_type(GLSL_TYPE_UINT
,
884 glsl_get_components(var
->type
));
890 v3d_nir_lower_fs_early(struct v3d_compile
*c
)
892 if (c
->fs_key
->int_color_rb
|| c
->fs_key
->uint_color_rb
)
893 v3d_fixup_fs_output_types(c
);
895 NIR_PASS_V(c
->s
, v3d_nir_lower_logic_ops
, c
);
897 if (c
->fs_key
->line_smoothing
) {
898 v3d_nir_lower_line_smooth(c
->s
);
899 NIR_PASS_V(c
->s
, nir_lower_global_vars_to_local
);
900 /* The lowering pass can introduce new sysval reads */
901 nir_shader_gather_info(c
->s
, nir_shader_get_entrypoint(c
->s
));
904 /* If the shader has no non-TLB side effects, we can promote it to
905 * enabling early_fragment_tests even if the user didn't.
907 if (!(c
->s
->info
.num_images
||
908 c
->s
->info
.num_ssbos
)) {
909 c
->s
->info
.fs
.early_fragment_tests
= true;
914 v3d_nir_lower_gs_late(struct v3d_compile
*c
)
916 if (c
->key
->ucp_enables
) {
917 NIR_PASS_V(c
->s
, nir_lower_clip_gs
, c
->key
->ucp_enables
,
921 /* Note: GS output scalarizing must happen after nir_lower_clip_gs. */
922 NIR_PASS_V(c
->s
, nir_lower_io_to_scalar
, nir_var_shader_out
);
926 v3d_nir_lower_vs_late(struct v3d_compile
*c
)
928 if (c
->vs_key
->clamp_color
)
929 NIR_PASS_V(c
->s
, nir_lower_clamp_color_outputs
);
931 if (c
->key
->ucp_enables
) {
932 NIR_PASS_V(c
->s
, nir_lower_clip_vs
, c
->key
->ucp_enables
,
934 NIR_PASS_V(c
->s
, nir_lower_io_to_scalar
,
938 /* Note: VS output scalarizing must happen after nir_lower_clip_vs. */
939 NIR_PASS_V(c
->s
, nir_lower_io_to_scalar
, nir_var_shader_out
);
943 v3d_nir_lower_fs_late(struct v3d_compile
*c
)
945 if (c
->fs_key
->light_twoside
)
946 NIR_PASS_V(c
->s
, nir_lower_two_sided_color
, true);
948 if (c
->fs_key
->clamp_color
)
949 NIR_PASS_V(c
->s
, nir_lower_clamp_color_outputs
);
951 if (c
->fs_key
->alpha_test
) {
952 NIR_PASS_V(c
->s
, nir_lower_alpha_test
,
953 c
->fs_key
->alpha_test_func
,
957 /* In OpenGL the fragment shader can't read gl_ClipDistance[], but
958 * Vulkan allows it, in which case the SPIR-V compiler will declare
959 * VARING_SLOT_CLIP_DIST0 as compact array variable. Pass true as
960 * the last parameter to always operate with a compact array in both
961 * OpenGL and Vulkan so we do't have to care about the API we
964 if (c
->key
->ucp_enables
)
965 NIR_PASS_V(c
->s
, nir_lower_clip_fs
, c
->key
->ucp_enables
, true);
967 /* Note: FS input scalarizing must happen after
968 * nir_lower_two_sided_color, which only handles a vec4 at a time.
970 NIR_PASS_V(c
->s
, nir_lower_io_to_scalar
, nir_var_shader_in
);
974 vir_get_max_temps(struct v3d_compile
*c
)
977 vir_for_each_inst_inorder(inst
, c
)
980 uint32_t *pressure
= rzalloc_array(NULL
, uint32_t, max_ip
);
982 for (int t
= 0; t
< c
->num_temps
; t
++) {
983 for (int i
= c
->temp_start
[t
]; (i
< c
->temp_end
[t
] &&
991 uint32_t max_temps
= 0;
992 for (int i
= 0; i
< max_ip
; i
++)
993 max_temps
= MAX2(max_temps
, pressure
[i
]);
995 ralloc_free(pressure
);
1000 enum v3d_dependency_class
{
1001 V3D_DEPENDENCY_CLASS_GS_VPM_OUTPUT_0
1005 v3d_intrinsic_dependency_cb(nir_intrinsic_instr
*intr
,
1006 nir_schedule_dependency
*dep
,
1009 struct v3d_compile
*c
= user_data
;
1011 switch (intr
->intrinsic
) {
1012 case nir_intrinsic_store_output
:
1013 /* Writing to location 0 overwrites the value passed in for
1014 * gl_PrimitiveID on geometry shaders
1016 if (c
->s
->info
.stage
!= MESA_SHADER_GEOMETRY
||
1017 nir_intrinsic_base(intr
) != 0)
1020 nir_const_value
*const_value
=
1021 nir_src_as_const_value(intr
->src
[1]);
1023 if (const_value
== NULL
)
1027 nir_const_value_as_uint(*const_value
,
1028 nir_src_bit_size(intr
->src
[1]));
1032 dep
->klass
= V3D_DEPENDENCY_CLASS_GS_VPM_OUTPUT_0
;
1033 dep
->type
= NIR_SCHEDULE_WRITE_DEPENDENCY
;
1036 case nir_intrinsic_load_primitive_id
:
1037 if (c
->s
->info
.stage
!= MESA_SHADER_GEOMETRY
)
1040 dep
->klass
= V3D_DEPENDENCY_CLASS_GS_VPM_OUTPUT_0
;
1041 dep
->type
= NIR_SCHEDULE_READ_DEPENDENCY
;
1052 v3d_attempt_compile(struct v3d_compile
*c
)
1054 switch (c
->s
->info
.stage
) {
1055 case MESA_SHADER_VERTEX
:
1056 c
->vs_key
= (struct v3d_vs_key
*) c
->key
;
1058 case MESA_SHADER_GEOMETRY
:
1059 c
->gs_key
= (struct v3d_gs_key
*) c
->key
;
1061 case MESA_SHADER_FRAGMENT
:
1062 c
->fs_key
= (struct v3d_fs_key
*) c
->key
;
1064 case MESA_SHADER_COMPUTE
:
1067 unreachable("unsupported shader stage");
1070 switch (c
->s
->info
.stage
) {
1071 case MESA_SHADER_VERTEX
:
1072 v3d_nir_lower_vs_early(c
);
1074 case MESA_SHADER_GEOMETRY
:
1075 v3d_nir_lower_gs_early(c
);
1077 case MESA_SHADER_FRAGMENT
:
1078 v3d_nir_lower_fs_early(c
);
1086 switch (c
->s
->info
.stage
) {
1087 case MESA_SHADER_VERTEX
:
1088 v3d_nir_lower_vs_late(c
);
1090 case MESA_SHADER_GEOMETRY
:
1091 v3d_nir_lower_gs_late(c
);
1093 case MESA_SHADER_FRAGMENT
:
1094 v3d_nir_lower_fs_late(c
);
1100 NIR_PASS_V(c
->s
, v3d_nir_lower_io
, c
);
1101 NIR_PASS_V(c
->s
, v3d_nir_lower_txf_ms
, c
);
1102 NIR_PASS_V(c
->s
, v3d_nir_lower_image_load_store
);
1103 NIR_PASS_V(c
->s
, nir_lower_idiv
, nir_lower_idiv_fast
);
1105 v3d_optimize_nir(c
->s
);
1107 /* Do late algebraic optimization to turn add(a, neg(b)) back into
1108 * subs, then the mandatory cleanup after algebraic. Note that it may
1109 * produce fnegs, and if so then we need to keep running to squash
1112 bool more_late_algebraic
= true;
1113 while (more_late_algebraic
) {
1114 more_late_algebraic
= false;
1115 NIR_PASS(more_late_algebraic
, c
->s
, nir_opt_algebraic_late
);
1116 NIR_PASS_V(c
->s
, nir_opt_constant_folding
);
1117 NIR_PASS_V(c
->s
, nir_copy_prop
);
1118 NIR_PASS_V(c
->s
, nir_opt_dce
);
1119 NIR_PASS_V(c
->s
, nir_opt_cse
);
1122 NIR_PASS_V(c
->s
, nir_lower_bool_to_int32
);
1123 NIR_PASS_V(c
->s
, nir_convert_from_ssa
, true);
1125 struct nir_schedule_options schedule_options
= {
1126 /* Schedule for about half our register space, to enable more
1127 * shaders to hit 4 threads.
1131 /* Vertex shaders share the same memory for inputs and outputs,
1132 * fragement and geometry shaders do not.
1134 .stages_with_shared_io_memory
=
1135 (((1 << MESA_ALL_SHADER_STAGES
) - 1) &
1136 ~((1 << MESA_SHADER_FRAGMENT
) |
1137 (1 << MESA_SHADER_GEOMETRY
))),
1139 .fallback
= c
->fallback_scheduler
,
1141 .intrinsic_cb
= v3d_intrinsic_dependency_cb
,
1142 .intrinsic_cb_data
= c
,
1144 NIR_PASS_V(c
->s
, nir_schedule
, &schedule_options
);
1149 uint64_t *v3d_compile(const struct v3d_compiler
*compiler
,
1150 struct v3d_key
*key
,
1151 struct v3d_prog_data
**out_prog_data
,
1153 void (*debug_output
)(const char *msg
,
1154 void *debug_output_data
),
1155 void *debug_output_data
,
1156 int program_id
, int variant_id
,
1157 uint32_t *final_assembly_size
)
1159 struct v3d_compile
*c
;
1161 for (int i
= 0; true; i
++) {
1162 c
= vir_compile_init(compiler
, key
, s
,
1163 debug_output
, debug_output_data
,
1164 program_id
, variant_id
,
1165 i
> 0 /* fallback_scheduler */);
1167 v3d_attempt_compile(c
);
1170 c
->compilation_result
!=
1171 V3D_COMPILATION_FAILED_REGISTER_ALLOCATION
)
1175 int ret
= asprintf(&debug_msg
,
1176 "Using fallback scheduler for %s",
1177 vir_get_stage_name(c
));
1180 if (unlikely(V3D_DEBUG
& V3D_DEBUG_PERF
))
1181 fprintf(stderr
, "%s\n", debug_msg
);
1183 c
->debug_output(debug_msg
, c
->debug_output_data
);
1187 vir_compile_destroy(c
);
1190 struct v3d_prog_data
*prog_data
;
1192 static const int prog_data_size
[] = {
1193 [MESA_SHADER_VERTEX
] = sizeof(struct v3d_vs_prog_data
),
1194 [MESA_SHADER_GEOMETRY
] = sizeof(struct v3d_gs_prog_data
),
1195 [MESA_SHADER_FRAGMENT
] = sizeof(struct v3d_fs_prog_data
),
1196 [MESA_SHADER_COMPUTE
] = sizeof(struct v3d_compute_prog_data
),
1199 assert(c
->s
->info
.stage
>= 0 &&
1200 c
->s
->info
.stage
< ARRAY_SIZE(prog_data_size
) &&
1201 prog_data_size
[c
->s
->info
.stage
]);
1203 prog_data
= rzalloc_size(NULL
, prog_data_size
[c
->s
->info
.stage
]);
1205 v3d_set_prog_data(c
, prog_data
);
1207 *out_prog_data
= prog_data
;
1210 int ret
= asprintf(&shaderdb
,
1211 "%s shader: %d inst, %d threads, %d loops, "
1212 "%d uniforms, %d max-temps, %d:%d spills:fills, "
1213 "%d sfu-stalls, %d inst-and-stalls",
1214 vir_get_stage_name(c
),
1219 vir_get_max_temps(c
),
1222 c
->qpu_inst_stalled_count
,
1223 c
->qpu_inst_count
+ c
->qpu_inst_stalled_count
);
1225 if (V3D_DEBUG
& V3D_DEBUG_SHADERDB
)
1226 fprintf(stderr
, "SHADER-DB: %s\n", shaderdb
);
1228 c
->debug_output(shaderdb
, c
->debug_output_data
);
1232 return v3d_return_qpu_insts(c
, final_assembly_size
);
1236 vir_remove_instruction(struct v3d_compile
*c
, struct qinst
*qinst
)
1238 if (qinst
->dst
.file
== QFILE_TEMP
)
1239 c
->defs
[qinst
->dst
.index
] = NULL
;
1241 assert(&qinst
->link
!= c
->cursor
.link
);
1243 list_del(&qinst
->link
);
1246 c
->live_intervals_valid
= false;
1250 vir_follow_movs(struct v3d_compile
*c
, struct qreg reg
)
1253 int pack = reg.pack;
1255 while (reg.file == QFILE_TEMP &&
1256 c->defs[reg.index] &&
1257 (c->defs[reg.index]->op == QOP_MOV ||
1258 c->defs[reg.index]->op == QOP_FMOV) &&
1259 !c->defs[reg.index]->dst.pack &&
1260 !c->defs[reg.index]->src[0].pack) {
1261 reg = c->defs[reg.index]->src[0];
1270 vir_compile_destroy(struct v3d_compile
*c
)
1272 /* Defuse the assert that we aren't removing the cursor's instruction.
1274 c
->cursor
.link
= NULL
;
1276 vir_for_each_block(block
, c
) {
1277 while (!list_is_empty(&block
->instructions
)) {
1278 struct qinst
*qinst
=
1279 list_first_entry(&block
->instructions
,
1280 struct qinst
, link
);
1281 vir_remove_instruction(c
, qinst
);
1289 vir_get_uniform_index(struct v3d_compile
*c
,
1290 enum quniform_contents contents
,
1293 for (int i
= 0; i
< c
->num_uniforms
; i
++) {
1294 if (c
->uniform_contents
[i
] == contents
&&
1295 c
->uniform_data
[i
] == data
) {
1300 uint32_t uniform
= c
->num_uniforms
++;
1302 if (uniform
>= c
->uniform_array_size
) {
1303 c
->uniform_array_size
= MAX2(MAX2(16, uniform
+ 1),
1304 c
->uniform_array_size
* 2);
1306 c
->uniform_data
= reralloc(c
, c
->uniform_data
,
1308 c
->uniform_array_size
);
1309 c
->uniform_contents
= reralloc(c
, c
->uniform_contents
,
1310 enum quniform_contents
,
1311 c
->uniform_array_size
);
1314 c
->uniform_contents
[uniform
] = contents
;
1315 c
->uniform_data
[uniform
] = data
;
1321 vir_uniform(struct v3d_compile
*c
,
1322 enum quniform_contents contents
,
1325 struct qinst
*inst
= vir_NOP(c
);
1326 inst
->qpu
.sig
.ldunif
= true;
1327 inst
->uniform
= vir_get_uniform_index(c
, contents
, data
);
1328 inst
->dst
= vir_get_temp(c
);
1329 c
->defs
[inst
->dst
.index
] = inst
;
1333 #define OPTPASS(func) \
1335 bool stage_progress = func(c); \
1336 if (stage_progress) { \
1338 if (print_opt_debug) { \
1340 "VIR opt pass %2d: %s progress\n", \
1343 /*XXX vir_validate(c);*/ \
1348 vir_optimize(struct v3d_compile
*c
)
1350 bool print_opt_debug
= false;
1354 bool progress
= false;
1356 OPTPASS(vir_opt_copy_propagate
);
1357 OPTPASS(vir_opt_redundant_flags
);
1358 OPTPASS(vir_opt_dead_code
);
1359 OPTPASS(vir_opt_small_immediates
);
1369 vir_get_stage_name(struct v3d_compile
*c
)
1371 if (c
->vs_key
&& c
->vs_key
->is_coord
)
1372 return "MESA_SHADER_VERTEX_BIN";
1373 else if (c
->gs_key
&& c
->gs_key
->is_coord
)
1374 return "MESA_SHADER_GEOMETRY_BIN";
1376 return gl_shader_stage_name(c
->s
->info
.stage
);