2 * Copyright © 2018 Intel Corporation
3 * Copyright © 2018 Broadcom
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
25 #include "v3d_compiler.h"
26 #include "compiler/nir/nir_builder.h"
27 #include "compiler/nir/nir_format_convert.h"
29 /** @file v3d_nir_lower_scratch.c
31 * Swizzles around the addresses of
32 * nir_intrinsic_load_scratch/nir_intrinsic_store_scratch so that a QPU stores
33 * a cacheline at a time per dword of scratch access, scalarizing and removing
34 * writemasks in the process.
38 v3d_nir_scratch_offset(nir_builder
*b
, nir_intrinsic_instr
*instr
)
40 bool is_store
= instr
->intrinsic
== nir_intrinsic_store_scratch
;
41 nir_ssa_def
*offset
= nir_ssa_for_src(b
, instr
->src
[is_store
? 1 : 0], 1);
43 assert(nir_intrinsic_align_mul(instr
) >= 4);
44 assert(nir_intrinsic_align_offset(instr
) == 0);
46 /* The spill_offset register will already have the subgroup ID (EIDX)
47 * shifted and ORed in at bit 2, so all we need to do is to move the
48 * dword index up above V3D_CHANNELS.
50 return nir_imul_imm(b
, offset
, V3D_CHANNELS
);
54 v3d_nir_lower_load_scratch(nir_builder
*b
, nir_intrinsic_instr
*instr
)
56 b
->cursor
= nir_before_instr(&instr
->instr
);
58 nir_ssa_def
*offset
= v3d_nir_scratch_offset(b
,instr
);
60 nir_ssa_def
*chans
[NIR_MAX_VEC_COMPONENTS
];
61 for (int i
= 0; i
< instr
->num_components
; i
++) {
62 nir_ssa_def
*chan_offset
=
63 nir_iadd_imm(b
, offset
, V3D_CHANNELS
* i
* 4);
65 nir_intrinsic_instr
*chan_instr
=
66 nir_intrinsic_instr_create(b
->shader
, instr
->intrinsic
);
67 chan_instr
->num_components
= 1;
68 nir_ssa_dest_init(&chan_instr
->instr
, &chan_instr
->dest
, 1,
69 instr
->dest
.ssa
.bit_size
, NULL
);
71 chan_instr
->src
[0] = nir_src_for_ssa(chan_offset
);
73 nir_intrinsic_set_align(chan_instr
, 4, 0);
75 nir_builder_instr_insert(b
, &chan_instr
->instr
);
77 chans
[i
] = &chan_instr
->dest
.ssa
;
80 nir_ssa_def
*result
= nir_vec(b
, chans
, instr
->num_components
);
81 nir_ssa_def_rewrite_uses(&instr
->dest
.ssa
, nir_src_for_ssa(result
));
82 nir_instr_remove(&instr
->instr
);
86 v3d_nir_lower_store_scratch(nir_builder
*b
, nir_intrinsic_instr
*instr
)
88 b
->cursor
= nir_before_instr(&instr
->instr
);
90 nir_ssa_def
*offset
= v3d_nir_scratch_offset(b
, instr
);
91 nir_ssa_def
*value
= nir_ssa_for_src(b
, instr
->src
[0],
92 instr
->num_components
);
94 for (int i
= 0; i
< instr
->num_components
; i
++) {
95 if (!(nir_intrinsic_write_mask(instr
) & (1 << i
)))
98 nir_ssa_def
*chan_offset
=
99 nir_iadd_imm(b
, offset
, V3D_CHANNELS
* i
* 4);
101 nir_intrinsic_instr
*chan_instr
=
102 nir_intrinsic_instr_create(b
->shader
, instr
->intrinsic
);
103 chan_instr
->num_components
= 1;
105 chan_instr
->src
[0] = nir_src_for_ssa(nir_channel(b
,
108 chan_instr
->src
[1] = nir_src_for_ssa(chan_offset
);
109 nir_intrinsic_set_write_mask(chan_instr
, 0x1);
110 nir_intrinsic_set_align(chan_instr
, 4, 0);
112 nir_builder_instr_insert(b
, &chan_instr
->instr
);
115 nir_instr_remove(&instr
->instr
);
119 v3d_nir_lower_scratch(nir_shader
*s
)
121 nir_foreach_function(function
, s
) {
126 nir_builder_init(&b
, function
->impl
);
128 nir_foreach_block(block
, function
->impl
) {
129 nir_foreach_instr_safe(instr
, block
) {
130 if (instr
->type
!= nir_instr_type_intrinsic
)
133 nir_intrinsic_instr
*intr
=
134 nir_instr_as_intrinsic(instr
);
136 switch (intr
->intrinsic
) {
137 case nir_intrinsic_load_scratch
:
138 v3d_nir_lower_load_scratch(&b
, intr
);
140 case nir_intrinsic_store_scratch
:
141 v3d_nir_lower_store_scratch(&b
, intr
);
149 nir_metadata_preserve(function
->impl
,
150 nir_metadata_block_index
|
151 nir_metadata_dominance
);