2 * Copyright © 2014 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 * Connor Abbott (cwabbott0@gmail.com)
28 #include "compiler/nir/nir.h"
29 #include "compiler/nir/nir_builder.h"
31 #include "ir_uniform.h"
32 #include "main/config.h"
33 #include "main/mtypes.h"
37 * replace atomic counter intrinsics that use a variable with intrinsics
38 * that directly store the buffer index and byte offset
42 lower_deref_instr(nir_builder
*b
, nir_intrinsic_instr
*instr
,
43 const struct gl_shader_program
*shader_program
,
44 nir_shader
*shader
, bool use_binding_as_idx
)
47 switch (instr
->intrinsic
) {
48 case nir_intrinsic_atomic_counter_read_deref
:
49 op
= nir_intrinsic_atomic_counter_read
;
52 case nir_intrinsic_atomic_counter_inc_deref
:
53 op
= nir_intrinsic_atomic_counter_inc
;
56 case nir_intrinsic_atomic_counter_dec_deref
:
57 op
= nir_intrinsic_atomic_counter_dec
;
60 case nir_intrinsic_atomic_counter_add_deref
:
61 op
= nir_intrinsic_atomic_counter_add
;
64 case nir_intrinsic_atomic_counter_min_deref
:
65 op
= nir_intrinsic_atomic_counter_min
;
68 case nir_intrinsic_atomic_counter_max_deref
:
69 op
= nir_intrinsic_atomic_counter_max
;
72 case nir_intrinsic_atomic_counter_and_deref
:
73 op
= nir_intrinsic_atomic_counter_and
;
76 case nir_intrinsic_atomic_counter_or_deref
:
77 op
= nir_intrinsic_atomic_counter_or
;
80 case nir_intrinsic_atomic_counter_xor_deref
:
81 op
= nir_intrinsic_atomic_counter_xor
;
84 case nir_intrinsic_atomic_counter_exchange_deref
:
85 op
= nir_intrinsic_atomic_counter_exchange
;
88 case nir_intrinsic_atomic_counter_comp_swap_deref
:
89 op
= nir_intrinsic_atomic_counter_comp_swap
;
96 nir_deref_instr
*deref
= nir_src_as_deref(instr
->src
[0]);
97 nir_variable
*var
= nir_deref_instr_get_variable(deref
);
99 if (var
->data
.mode
!= nir_var_uniform
&&
100 var
->data
.mode
!= nir_var_shader_storage
&&
101 var
->data
.mode
!= nir_var_shared
)
102 return false; /* atomics passed as function arguments can't be lowered */
104 const unsigned uniform_loc
= var
->data
.location
;
105 const unsigned idx
= use_binding_as_idx
? var
->data
.binding
:
106 shader_program
->data
->UniformStorage
[uniform_loc
].opaque
[shader
->info
.stage
].index
;
108 b
->cursor
= nir_before_instr(&instr
->instr
);
110 nir_ssa_def
*offset
= nir_imm_int(b
, var
->data
.offset
);
111 for (nir_deref_instr
*d
= deref
; d
->deref_type
!= nir_deref_type_var
;
112 d
= nir_deref_instr_parent(d
)) {
113 assert(d
->deref_type
== nir_deref_type_array
);
114 assert(d
->arr
.index
.is_ssa
);
116 unsigned array_stride
= ATOMIC_COUNTER_SIZE
;
117 if (glsl_type_is_array(d
->type
))
118 array_stride
*= glsl_get_aoa_size(d
->type
);
120 offset
= nir_iadd(b
, offset
, nir_imul(b
, d
->arr
.index
.ssa
,
121 nir_imm_int(b
, array_stride
)));
124 /* Since the first source is a deref and the first source in the lowered
125 * instruction is the offset, we can just swap it out and change the
128 instr
->intrinsic
= op
;
129 nir_instr_rewrite_src(&instr
->instr
, &instr
->src
[0],
130 nir_src_for_ssa(offset
));
131 nir_intrinsic_set_base(instr
, idx
);
133 nir_deref_instr_remove_if_unused(deref
);
139 gl_nir_lower_atomics(nir_shader
*shader
,
140 const struct gl_shader_program
*shader_program
,
141 bool use_binding_as_idx
)
143 bool progress
= false;
145 nir_foreach_function(function
, shader
) {
149 bool impl_progress
= false;
152 nir_builder_init(&build
, function
->impl
);
154 nir_foreach_block(block
, function
->impl
) {
155 nir_foreach_instr_safe(instr
, block
) {
156 if (instr
->type
!= nir_instr_type_intrinsic
)
159 impl_progress
|= lower_deref_instr(&build
,
160 nir_instr_as_intrinsic(instr
),
161 shader_program
, shader
,
167 nir_metadata_preserve(function
->impl
, nir_metadata_block_index
|
168 nir_metadata_dominance
);