2 * Copyright © 2014 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 * Connor Abbott (cwabbott0@gmail.com)
28 #include "compiler/nir/nir.h"
30 #include "ir_uniform.h"
32 #include "main/config.h"
33 #include "main/mtypes.h"
37 * replace atomic counter intrinsics that use a variable with intrinsics
38 * that directly store the buffer index and byte offset
42 lower_instr(nir_intrinsic_instr
*instr
,
43 const struct gl_shader_program
*shader_program
,
44 nir_shader
*shader
, bool use_binding_as_idx
)
47 switch (instr
->intrinsic
) {
48 case nir_intrinsic_atomic_counter_read_var
:
49 op
= nir_intrinsic_atomic_counter_read
;
52 case nir_intrinsic_atomic_counter_inc_var
:
53 op
= nir_intrinsic_atomic_counter_inc
;
56 case nir_intrinsic_atomic_counter_dec_var
:
57 op
= nir_intrinsic_atomic_counter_dec
;
60 case nir_intrinsic_atomic_counter_add_var
:
61 op
= nir_intrinsic_atomic_counter_add
;
64 case nir_intrinsic_atomic_counter_min_var
:
65 op
= nir_intrinsic_atomic_counter_min
;
68 case nir_intrinsic_atomic_counter_max_var
:
69 op
= nir_intrinsic_atomic_counter_max
;
72 case nir_intrinsic_atomic_counter_and_var
:
73 op
= nir_intrinsic_atomic_counter_and
;
76 case nir_intrinsic_atomic_counter_or_var
:
77 op
= nir_intrinsic_atomic_counter_or
;
80 case nir_intrinsic_atomic_counter_xor_var
:
81 op
= nir_intrinsic_atomic_counter_xor
;
84 case nir_intrinsic_atomic_counter_exchange_var
:
85 op
= nir_intrinsic_atomic_counter_exchange
;
88 case nir_intrinsic_atomic_counter_comp_swap_var
:
89 op
= nir_intrinsic_atomic_counter_comp_swap
;
96 if (instr
->variables
[0]->var
->data
.mode
!= nir_var_uniform
&&
97 instr
->variables
[0]->var
->data
.mode
!= nir_var_shader_storage
&&
98 instr
->variables
[0]->var
->data
.mode
!= nir_var_shared
)
99 return false; /* atomics passed as function arguments can't be lowered */
101 void *mem_ctx
= ralloc_parent(instr
);
102 unsigned uniform_loc
= instr
->variables
[0]->var
->data
.location
;
104 unsigned idx
= use_binding_as_idx
?
105 instr
->variables
[0]->var
->data
.binding
:
106 shader_program
->data
->UniformStorage
[uniform_loc
].opaque
[shader
->info
.stage
].index
;
108 nir_intrinsic_instr
*new_instr
= nir_intrinsic_instr_create(mem_ctx
, op
);
109 nir_intrinsic_set_base(new_instr
, idx
);
111 nir_load_const_instr
*offset_const
=
112 nir_load_const_instr_create(mem_ctx
, 1, 32);
113 offset_const
->value
.u32
[0] = instr
->variables
[0]->var
->data
.offset
;
115 nir_instr_insert_before(&instr
->instr
, &offset_const
->instr
);
117 nir_ssa_def
*offset_def
= &offset_const
->def
;
119 nir_deref
*tail
= &instr
->variables
[0]->deref
;
120 while (tail
->child
!= NULL
) {
121 nir_deref_array
*deref_array
= nir_deref_as_array(tail
->child
);
124 unsigned child_array_elements
= tail
->child
!= NULL
?
125 glsl_get_aoa_size(tail
->type
) : 1;
127 offset_const
->value
.u32
[0] += deref_array
->base_offset
*
128 child_array_elements
* ATOMIC_COUNTER_SIZE
;
130 if (deref_array
->deref_array_type
== nir_deref_array_type_indirect
) {
131 nir_load_const_instr
*atomic_counter_size
=
132 nir_load_const_instr_create(mem_ctx
, 1, 32);
133 atomic_counter_size
->value
.u32
[0] = child_array_elements
* ATOMIC_COUNTER_SIZE
;
134 nir_instr_insert_before(&instr
->instr
, &atomic_counter_size
->instr
);
136 nir_alu_instr
*mul
= nir_alu_instr_create(mem_ctx
, nir_op_imul
);
137 nir_ssa_dest_init(&mul
->instr
, &mul
->dest
.dest
, 1, 32, NULL
);
138 mul
->dest
.write_mask
= 0x1;
139 nir_src_copy(&mul
->src
[0].src
, &deref_array
->indirect
, mul
);
140 mul
->src
[1].src
.is_ssa
= true;
141 mul
->src
[1].src
.ssa
= &atomic_counter_size
->def
;
142 nir_instr_insert_before(&instr
->instr
, &mul
->instr
);
144 nir_alu_instr
*add
= nir_alu_instr_create(mem_ctx
, nir_op_iadd
);
145 nir_ssa_dest_init(&add
->instr
, &add
->dest
.dest
, 1, 32, NULL
);
146 add
->dest
.write_mask
= 0x1;
147 add
->src
[0].src
.is_ssa
= true;
148 add
->src
[0].src
.ssa
= &mul
->dest
.dest
.ssa
;
149 add
->src
[1].src
.is_ssa
= true;
150 add
->src
[1].src
.ssa
= offset_def
;
151 nir_instr_insert_before(&instr
->instr
, &add
->instr
);
153 offset_def
= &add
->dest
.dest
.ssa
;
157 new_instr
->src
[0].is_ssa
= true;
158 new_instr
->src
[0].ssa
= offset_def
;
160 /* Copy the other sources, if any, from the original instruction to the new
163 for (unsigned i
= 0; i
< nir_intrinsic_infos
[instr
->intrinsic
].num_srcs
; i
++)
164 nir_src_copy(&new_instr
->src
[i
+ 1], &instr
->src
[i
], new_instr
);
166 if (instr
->dest
.is_ssa
) {
167 nir_ssa_dest_init(&new_instr
->instr
, &new_instr
->dest
,
168 instr
->dest
.ssa
.num_components
, 32, NULL
);
169 nir_ssa_def_rewrite_uses(&instr
->dest
.ssa
,
170 nir_src_for_ssa(&new_instr
->dest
.ssa
));
172 nir_dest_copy(&new_instr
->dest
, &instr
->dest
, mem_ctx
);
175 nir_instr_insert_before(&instr
->instr
, &new_instr
->instr
);
176 nir_instr_remove(&instr
->instr
);
182 gl_nir_lower_atomics(nir_shader
*shader
,
183 const struct gl_shader_program
*shader_program
,
184 bool use_binding_as_idx
)
186 bool progress
= false;
188 nir_assert_lowered_derefs(shader
, nir_lower_atomic_counter_derefs
);
190 nir_foreach_function(function
, shader
) {
194 bool impl_progress
= false;
196 nir_foreach_block(block
, function
->impl
) {
197 nir_foreach_instr_safe(instr
, block
) {
198 if (instr
->type
!= nir_instr_type_intrinsic
)
201 impl_progress
|= lower_instr(nir_instr_as_intrinsic(instr
),
202 shader_program
, shader
,
208 nir_metadata_preserve(function
->impl
, nir_metadata_block_index
|
209 nir_metadata_dominance
);