nir/load_store_vectorize: rework alignment calculation
[mesa.git] / src / compiler / nir / nir_opt_load_store_vectorize.c
1 /*
2 * Copyright © 2019 Valve Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 /**
25 * Although it's called a load/store "vectorization" pass, this also combines
26 * intersecting and identical loads/stores. It currently supports derefs, ubo,
27 * ssbo and push constant loads/stores.
28 *
29 * This doesn't handle copy_deref intrinsics and assumes that
30 * nir_lower_alu_to_scalar() has been called and that the IR is free from ALU
31 * modifiers. It also assumes that derefs have explicitly laid out types.
32 *
33 * After vectorization, the backend may want to call nir_lower_alu_to_scalar()
34 * and nir_lower_pack(). Also this creates cast instructions taking derefs as a
35 * source and some parts of NIR may not be able to handle that well.
36 *
37 * There are a few situations where this doesn't vectorize as well as it could:
38 * - It won't turn four consecutive vec3 loads into 3 vec4 loads.
39 * - It doesn't do global vectorization.
40 * Handling these cases probably wouldn't provide much benefit though.
41 *
42 * This probably doesn't handle big-endian GPUs correctly.
43 */
44
45 #include "nir.h"
46 #include "nir_deref.h"
47 #include "nir_builder.h"
48 #include "nir_worklist.h"
49 #include "util/u_dynarray.h"
50
51 #include <stdlib.h>
52
53 struct intrinsic_info {
54 nir_variable_mode mode; /* 0 if the mode is obtained from the deref. */
55 nir_intrinsic_op op;
56 bool is_atomic;
57 /* Indices into nir_intrinsic::src[] or -1 if not applicable. */
58 int resource_src; /* resource (e.g. from vulkan_resource_index) */
59 int base_src; /* offset which it loads/stores from */
60 int deref_src; /* deref which is loads/stores from */
61 int value_src; /* the data it is storing */
62 };
63
64 static const struct intrinsic_info *
65 get_info(nir_intrinsic_op op) {
66 switch (op) {
67 #define INFO(mode, op, atomic, res, base, deref, val) \
68 case nir_intrinsic_##op: {\
69 static const struct intrinsic_info op##_info = {mode, nir_intrinsic_##op, atomic, res, base, deref, val};\
70 return &op##_info;\
71 }
72 #define LOAD(mode, op, res, base, deref) INFO(mode, load_##op, false, res, base, deref, -1)
73 #define STORE(mode, op, res, base, deref, val) INFO(mode, store_##op, false, res, base, deref, val)
74 #define ATOMIC(mode, type, op, res, base, deref, val) INFO(mode, type##_atomic_##op, true, res, base, deref, val)
75 LOAD(nir_var_mem_push_const, push_constant, -1, 0, -1)
76 LOAD(nir_var_mem_ubo, ubo, 0, 1, -1)
77 LOAD(nir_var_mem_ssbo, ssbo, 0, 1, -1)
78 STORE(nir_var_mem_ssbo, ssbo, 1, 2, -1, 0)
79 LOAD(0, deref, -1, -1, 0)
80 STORE(0, deref, -1, -1, 0, 1)
81 LOAD(nir_var_mem_shared, shared, -1, 0, -1)
82 STORE(nir_var_mem_shared, shared, -1, 1, -1, 0)
83 LOAD(nir_var_mem_global, global, -1, 0, -1)
84 STORE(nir_var_mem_global, global, -1, 1, -1, 0)
85 ATOMIC(nir_var_mem_ssbo, ssbo, add, 0, 1, -1, 2)
86 ATOMIC(nir_var_mem_ssbo, ssbo, imin, 0, 1, -1, 2)
87 ATOMIC(nir_var_mem_ssbo, ssbo, umin, 0, 1, -1, 2)
88 ATOMIC(nir_var_mem_ssbo, ssbo, imax, 0, 1, -1, 2)
89 ATOMIC(nir_var_mem_ssbo, ssbo, umax, 0, 1, -1, 2)
90 ATOMIC(nir_var_mem_ssbo, ssbo, and, 0, 1, -1, 2)
91 ATOMIC(nir_var_mem_ssbo, ssbo, or, 0, 1, -1, 2)
92 ATOMIC(nir_var_mem_ssbo, ssbo, xor, 0, 1, -1, 2)
93 ATOMIC(nir_var_mem_ssbo, ssbo, exchange, 0, 1, -1, 2)
94 ATOMIC(nir_var_mem_ssbo, ssbo, comp_swap, 0, 1, -1, 2)
95 ATOMIC(nir_var_mem_ssbo, ssbo, fadd, 0, 1, -1, 2)
96 ATOMIC(nir_var_mem_ssbo, ssbo, fmin, 0, 1, -1, 2)
97 ATOMIC(nir_var_mem_ssbo, ssbo, fmax, 0, 1, -1, 2)
98 ATOMIC(nir_var_mem_ssbo, ssbo, fcomp_swap, 0, 1, -1, 2)
99 ATOMIC(0, deref, add, -1, -1, 0, 1)
100 ATOMIC(0, deref, imin, -1, -1, 0, 1)
101 ATOMIC(0, deref, umin, -1, -1, 0, 1)
102 ATOMIC(0, deref, imax, -1, -1, 0, 1)
103 ATOMIC(0, deref, umax, -1, -1, 0, 1)
104 ATOMIC(0, deref, and, -1, -1, 0, 1)
105 ATOMIC(0, deref, or, -1, -1, 0, 1)
106 ATOMIC(0, deref, xor, -1, -1, 0, 1)
107 ATOMIC(0, deref, exchange, -1, -1, 0, 1)
108 ATOMIC(0, deref, comp_swap, -1, -1, 0, 1)
109 ATOMIC(0, deref, fadd, -1, -1, 0, 1)
110 ATOMIC(0, deref, fmin, -1, -1, 0, 1)
111 ATOMIC(0, deref, fmax, -1, -1, 0, 1)
112 ATOMIC(0, deref, fcomp_swap, -1, -1, 0, 1)
113 ATOMIC(nir_var_mem_shared, shared, add, -1, 0, -1, 1)
114 ATOMIC(nir_var_mem_shared, shared, imin, -1, 0, -1, 1)
115 ATOMIC(nir_var_mem_shared, shared, umin, -1, 0, -1, 1)
116 ATOMIC(nir_var_mem_shared, shared, imax, -1, 0, -1, 1)
117 ATOMIC(nir_var_mem_shared, shared, umax, -1, 0, -1, 1)
118 ATOMIC(nir_var_mem_shared, shared, and, -1, 0, -1, 1)
119 ATOMIC(nir_var_mem_shared, shared, or, -1, 0, -1, 1)
120 ATOMIC(nir_var_mem_shared, shared, xor, -1, 0, -1, 1)
121 ATOMIC(nir_var_mem_shared, shared, exchange, -1, 0, -1, 1)
122 ATOMIC(nir_var_mem_shared, shared, comp_swap, -1, 0, -1, 1)
123 ATOMIC(nir_var_mem_shared, shared, fadd, -1, 0, -1, 1)
124 ATOMIC(nir_var_mem_shared, shared, fmin, -1, 0, -1, 1)
125 ATOMIC(nir_var_mem_shared, shared, fmax, -1, 0, -1, 1)
126 ATOMIC(nir_var_mem_shared, shared, fcomp_swap, -1, 0, -1, 1)
127 ATOMIC(nir_var_mem_global, global, add, -1, 0, -1, 1)
128 ATOMIC(nir_var_mem_global, global, imin, -1, 0, -1, 1)
129 ATOMIC(nir_var_mem_global, global, umin, -1, 0, -1, 1)
130 ATOMIC(nir_var_mem_global, global, imax, -1, 0, -1, 1)
131 ATOMIC(nir_var_mem_global, global, umax, -1, 0, -1, 1)
132 ATOMIC(nir_var_mem_global, global, and, -1, 0, -1, 1)
133 ATOMIC(nir_var_mem_global, global, or, -1, 0, -1, 1)
134 ATOMIC(nir_var_mem_global, global, xor, -1, 0, -1, 1)
135 ATOMIC(nir_var_mem_global, global, exchange, -1, 0, -1, 1)
136 ATOMIC(nir_var_mem_global, global, comp_swap, -1, 0, -1, 1)
137 ATOMIC(nir_var_mem_global, global, fadd, -1, 0, -1, 1)
138 ATOMIC(nir_var_mem_global, global, fmin, -1, 0, -1, 1)
139 ATOMIC(nir_var_mem_global, global, fmax, -1, 0, -1, 1)
140 ATOMIC(nir_var_mem_global, global, fcomp_swap, -1, 0, -1, 1)
141 default:
142 break;
143 #undef ATOMIC
144 #undef STORE
145 #undef LOAD
146 #undef INFO
147 }
148 return NULL;
149 }
150
151 /*
152 * Information used to compare memory operations.
153 * It canonically represents an offset as:
154 * `offset_defs[0]*offset_defs_mul[0] + offset_defs[1]*offset_defs_mul[1] + ...`
155 * "offset_defs" is sorted in ascenting order by the ssa definition's index.
156 * "resource" or "var" may be NULL.
157 */
158 struct entry_key {
159 nir_ssa_def *resource;
160 nir_variable *var;
161 unsigned offset_def_count;
162 nir_ssa_def **offset_defs;
163 uint64_t *offset_defs_mul;
164 };
165
166 /* Information on a single memory operation. */
167 struct entry {
168 struct list_head head;
169 unsigned index;
170
171 struct entry_key *key;
172 union {
173 uint64_t offset; /* sign-extended */
174 int64_t offset_signed;
175 };
176 uint32_t align_mul;
177 uint32_t align_offset;
178
179 nir_instr *instr;
180 nir_intrinsic_instr *intrin;
181 const struct intrinsic_info *info;
182 enum gl_access_qualifier access;
183 bool is_store;
184
185 nir_deref_instr *deref;
186 };
187
188 struct vectorize_ctx {
189 nir_variable_mode modes;
190 nir_should_vectorize_mem_func callback;
191 nir_variable_mode robust_modes;
192 struct list_head entries[nir_num_variable_modes];
193 struct hash_table *loads[nir_num_variable_modes];
194 struct hash_table *stores[nir_num_variable_modes];
195 };
196
197 static uint32_t hash_entry_key(const void *key_)
198 {
199 /* this is careful to not include pointers in the hash calculation so that
200 * the order of the hash table walk is deterministic */
201 struct entry_key *key = (struct entry_key*)key_;
202
203 uint32_t hash = 0;
204 if (key->resource)
205 hash = XXH32(&key->resource->index, sizeof(key->resource->index), hash);
206 if (key->var) {
207 hash = XXH32(&key->var->index, sizeof(key->var->index), hash);
208 unsigned mode = key->var->data.mode;
209 hash = XXH32(&mode, sizeof(mode), hash);
210 }
211
212 for (unsigned i = 0; i < key->offset_def_count; i++)
213 hash = XXH32(&key->offset_defs[i]->index, sizeof(key->offset_defs[i]->index), hash);
214
215 hash = XXH32(key->offset_defs_mul, key->offset_def_count * sizeof(uint64_t), hash);
216
217 return hash;
218 }
219
220 static bool entry_key_equals(const void *a_, const void *b_)
221 {
222 struct entry_key *a = (struct entry_key*)a_;
223 struct entry_key *b = (struct entry_key*)b_;
224
225 if (a->var != b->var || a->resource != b->resource)
226 return false;
227
228 if (a->offset_def_count != b->offset_def_count)
229 return false;
230
231 size_t offset_def_size = a->offset_def_count * sizeof(nir_ssa_def *);
232 size_t offset_def_mul_size = a->offset_def_count * sizeof(uint64_t);
233 if (a->offset_def_count &&
234 (memcmp(a->offset_defs, b->offset_defs, offset_def_size) ||
235 memcmp(a->offset_defs_mul, b->offset_defs_mul, offset_def_mul_size)))
236 return false;
237
238 return true;
239 }
240
241 static void delete_entry_dynarray(struct hash_entry *entry)
242 {
243 struct util_dynarray *arr = (struct util_dynarray *)entry->data;
244 ralloc_free(arr);
245 }
246
247 static int sort_entries(const void *a_, const void *b_)
248 {
249 struct entry *a = *(struct entry*const*)a_;
250 struct entry *b = *(struct entry*const*)b_;
251
252 if (a->offset_signed > b->offset_signed)
253 return 1;
254 else if (a->offset_signed < b->offset_signed)
255 return -1;
256 else
257 return 0;
258 }
259
260 static unsigned
261 get_bit_size(struct entry *entry)
262 {
263 unsigned size = entry->is_store ?
264 entry->intrin->src[entry->info->value_src].ssa->bit_size :
265 entry->intrin->dest.ssa.bit_size;
266 return size == 1 ? 32u : size;
267 }
268
269 /* If "def" is from an alu instruction with the opcode "op" and one of it's
270 * sources is a constant, update "def" to be the non-constant source, fill "c"
271 * with the constant and return true. */
272 static bool
273 parse_alu(nir_ssa_def **def, nir_op op, uint64_t *c)
274 {
275 nir_ssa_scalar scalar;
276 scalar.def = *def;
277 scalar.comp = 0;
278
279 if (!nir_ssa_scalar_is_alu(scalar) || nir_ssa_scalar_alu_op(scalar) != op)
280 return false;
281
282 nir_ssa_scalar src0 = nir_ssa_scalar_chase_alu_src(scalar, 0);
283 nir_ssa_scalar src1 = nir_ssa_scalar_chase_alu_src(scalar, 1);
284 if (op != nir_op_ishl && nir_ssa_scalar_is_const(src0) && src1.comp == 0) {
285 *c = nir_ssa_scalar_as_uint(src0);
286 *def = src1.def;
287 } else if (nir_ssa_scalar_is_const(src1) && src0.comp == 0) {
288 *c = nir_ssa_scalar_as_uint(src1);
289 *def = src0.def;
290 } else {
291 return false;
292 }
293 return true;
294 }
295
296 /* Parses an offset expression such as "a * 16 + 4" and "(a * 16 + 4) * 64 + 32". */
297 static void
298 parse_offset(nir_ssa_def **base, uint64_t *base_mul, uint64_t *offset)
299 {
300 if ((*base)->parent_instr->type == nir_instr_type_load_const) {
301 *offset = nir_src_comp_as_uint(nir_src_for_ssa(*base), 0);
302 *base = NULL;
303 return;
304 }
305
306 uint64_t mul = 1;
307 uint64_t add = 0;
308 bool progress = false;
309 do {
310 uint64_t mul2 = 1, add2 = 0;
311
312 progress = parse_alu(base, nir_op_imul, &mul2);
313 mul *= mul2;
314
315 mul2 = 0;
316 progress |= parse_alu(base, nir_op_ishl, &mul2);
317 mul <<= mul2;
318
319 progress |= parse_alu(base, nir_op_iadd, &add2);
320 add += add2 * mul;
321 } while (progress);
322
323 *base_mul = mul;
324 *offset = add;
325 }
326
327 static unsigned
328 type_scalar_size_bytes(const struct glsl_type *type)
329 {
330 assert(glsl_type_is_vector_or_scalar(type) ||
331 glsl_type_is_matrix(type));
332 return glsl_type_is_boolean(type) ? 4u : glsl_get_bit_size(type) / 8u;
333 }
334
335 static uint64_t
336 mask_sign_extend(uint64_t val, unsigned bit_size)
337 {
338 return (int64_t)(val << (64 - bit_size)) >> (64 - bit_size);
339 }
340
341 static unsigned
342 add_to_entry_key(nir_ssa_def **offset_defs, uint64_t *offset_defs_mul,
343 unsigned offset_def_count, nir_ssa_def *def, uint64_t mul)
344 {
345 mul = mask_sign_extend(mul, def->bit_size);
346
347 for (unsigned i = 0; i <= offset_def_count; i++) {
348 if (i == offset_def_count || def->index > offset_defs[i]->index) {
349 /* insert before i */
350 memmove(offset_defs + i + 1, offset_defs + i,
351 (offset_def_count - i) * sizeof(nir_ssa_def *));
352 memmove(offset_defs_mul + i + 1, offset_defs_mul + i,
353 (offset_def_count - i) * sizeof(uint64_t));
354 offset_defs[i] = def;
355 offset_defs_mul[i] = mul;
356 return 1;
357 } else if (def->index == offset_defs[i]->index) {
358 /* merge with offset_def at i */
359 offset_defs_mul[i] += mul;
360 return 0;
361 }
362 }
363 unreachable("Unreachable.");
364 return 0;
365 }
366
367 static struct entry_key *
368 create_entry_key_from_deref(void *mem_ctx,
369 struct vectorize_ctx *ctx,
370 nir_deref_path *path,
371 uint64_t *offset_base)
372 {
373 unsigned path_len = 0;
374 while (path->path[path_len])
375 path_len++;
376
377 nir_ssa_def *offset_defs_stack[32];
378 uint64_t offset_defs_mul_stack[32];
379 nir_ssa_def **offset_defs = offset_defs_stack;
380 uint64_t *offset_defs_mul = offset_defs_mul_stack;
381 if (path_len > 32) {
382 offset_defs = malloc(path_len * sizeof(nir_ssa_def *));
383 offset_defs_mul = malloc(path_len * sizeof(uint64_t));
384 }
385 unsigned offset_def_count = 0;
386
387 struct entry_key* key = ralloc(mem_ctx, struct entry_key);
388 key->resource = NULL;
389 key->var = NULL;
390 *offset_base = 0;
391
392 for (unsigned i = 0; i < path_len; i++) {
393 nir_deref_instr *parent = i ? path->path[i - 1] : NULL;
394 nir_deref_instr *deref = path->path[i];
395
396 switch (deref->deref_type) {
397 case nir_deref_type_var: {
398 assert(!parent);
399 key->var = deref->var;
400 break;
401 }
402 case nir_deref_type_array:
403 case nir_deref_type_ptr_as_array: {
404 assert(parent);
405 nir_ssa_def *index = deref->arr.index.ssa;
406 uint32_t stride = nir_deref_instr_array_stride(deref);
407
408 nir_ssa_def *base = index;
409 uint64_t offset = 0, base_mul = 1;
410 parse_offset(&base, &base_mul, &offset);
411 offset = mask_sign_extend(offset, index->bit_size);
412
413 *offset_base += offset * stride;
414 if (base) {
415 offset_def_count += add_to_entry_key(offset_defs, offset_defs_mul,
416 offset_def_count,
417 base, base_mul * stride);
418 }
419 break;
420 }
421 case nir_deref_type_struct: {
422 assert(parent);
423 int offset = glsl_get_struct_field_offset(parent->type, deref->strct.index);
424 *offset_base += offset;
425 break;
426 }
427 case nir_deref_type_cast: {
428 if (!parent)
429 key->resource = deref->parent.ssa;
430 break;
431 }
432 default:
433 unreachable("Unhandled deref type");
434 }
435 }
436
437 key->offset_def_count = offset_def_count;
438 key->offset_defs = ralloc_array(mem_ctx, nir_ssa_def *, offset_def_count);
439 key->offset_defs_mul = ralloc_array(mem_ctx, uint64_t, offset_def_count);
440 memcpy(key->offset_defs, offset_defs, offset_def_count * sizeof(nir_ssa_def *));
441 memcpy(key->offset_defs_mul, offset_defs_mul, offset_def_count * sizeof(uint64_t));
442
443 if (offset_defs != offset_defs_stack)
444 free(offset_defs);
445 if (offset_defs_mul != offset_defs_mul_stack)
446 free(offset_defs_mul);
447
448 return key;
449 }
450
451 static unsigned
452 parse_entry_key_from_offset(struct entry_key *key, unsigned size, unsigned left,
453 nir_ssa_def *base, uint64_t base_mul, uint64_t *offset)
454 {
455 uint64_t new_mul;
456 uint64_t new_offset;
457 parse_offset(&base, &new_mul, &new_offset);
458 *offset += new_offset * base_mul;
459
460 if (!base)
461 return 0;
462
463 base_mul *= new_mul;
464
465 assert(left >= 1);
466
467 if (left >= 2) {
468 nir_ssa_scalar scalar;
469 scalar.def = base;
470 scalar.comp = 0;
471 if (nir_ssa_scalar_is_alu(scalar) && nir_ssa_scalar_alu_op(scalar) == nir_op_iadd) {
472 nir_ssa_scalar src0 = nir_ssa_scalar_chase_alu_src(scalar, 0);
473 nir_ssa_scalar src1 = nir_ssa_scalar_chase_alu_src(scalar, 1);
474 if (src0.comp == 0 && src1.comp == 0) {
475 unsigned amount = parse_entry_key_from_offset(key, size, left - 1, src0.def, base_mul, offset);
476 amount += parse_entry_key_from_offset(key, size + amount, left - amount, src1.def, base_mul, offset);
477 return amount;
478 }
479 }
480 }
481
482 return add_to_entry_key(key->offset_defs, key->offset_defs_mul, size, base, base_mul);
483 }
484
485 static struct entry_key *
486 create_entry_key_from_offset(void *mem_ctx, nir_ssa_def *base, uint64_t base_mul, uint64_t *offset)
487 {
488 struct entry_key *key = ralloc(mem_ctx, struct entry_key);
489 key->resource = NULL;
490 key->var = NULL;
491 if (base) {
492 nir_ssa_def *offset_defs[32];
493 uint64_t offset_defs_mul[32];
494 key->offset_defs = offset_defs;
495 key->offset_defs_mul = offset_defs_mul;
496
497 key->offset_def_count = parse_entry_key_from_offset(key, 0, 32, base, base_mul, offset);
498
499 key->offset_defs = ralloc_array(mem_ctx, nir_ssa_def *, key->offset_def_count);
500 key->offset_defs_mul = ralloc_array(mem_ctx, uint64_t, key->offset_def_count);
501 memcpy(key->offset_defs, offset_defs, key->offset_def_count * sizeof(nir_ssa_def *));
502 memcpy(key->offset_defs_mul, offset_defs_mul, key->offset_def_count * sizeof(uint64_t));
503 } else {
504 key->offset_def_count = 0;
505 key->offset_defs = NULL;
506 key->offset_defs_mul = NULL;
507 }
508 return key;
509 }
510
511 static nir_variable_mode
512 get_variable_mode(struct entry *entry)
513 {
514 if (entry->info->mode)
515 return entry->info->mode;
516 assert(entry->deref);
517 return entry->deref->mode;
518 }
519
520 static unsigned
521 mode_to_index(nir_variable_mode mode)
522 {
523 assert(util_bitcount(mode) == 1);
524
525 /* Globals and SSBOs should be tracked together */
526 if (mode == nir_var_mem_global)
527 mode = nir_var_mem_ssbo;
528
529 return ffs(mode) - 1;
530 }
531
532 static nir_variable_mode
533 aliasing_modes(nir_variable_mode modes)
534 {
535 /* Global and SSBO can alias */
536 if (modes & (nir_var_mem_ssbo | nir_var_mem_global))
537 modes |= nir_var_mem_ssbo | nir_var_mem_global;
538 return modes;
539 }
540
541 static void
542 calc_alignment(struct entry *entry)
543 {
544 uint32_t align_mul = 31;
545 for (unsigned i = 0; i < entry->key->offset_def_count; i++) {
546 if (entry->key->offset_defs_mul[i])
547 align_mul = MIN2(align_mul, ffsll(entry->key->offset_defs_mul[i]));
548 }
549
550 entry->align_mul = 1u << (align_mul - 1);
551 bool has_align = nir_intrinsic_infos[entry->intrin->intrinsic].index_map[NIR_INTRINSIC_ALIGN_MUL];
552 if (!has_align || entry->align_mul >= nir_intrinsic_align_mul(entry->intrin)) {
553 entry->align_offset = entry->offset % entry->align_mul;
554 } else {
555 entry->align_mul = nir_intrinsic_align_mul(entry->intrin);
556 entry->align_offset = nir_intrinsic_align_offset(entry->intrin);
557 }
558 }
559
560 static struct entry *
561 create_entry(struct vectorize_ctx *ctx,
562 const struct intrinsic_info *info,
563 nir_intrinsic_instr *intrin)
564 {
565 struct entry *entry = rzalloc(ctx, struct entry);
566 entry->intrin = intrin;
567 entry->instr = &intrin->instr;
568 entry->info = info;
569 entry->is_store = entry->info->value_src >= 0;
570
571 if (entry->info->deref_src >= 0) {
572 entry->deref = nir_src_as_deref(intrin->src[entry->info->deref_src]);
573 nir_deref_path path;
574 nir_deref_path_init(&path, entry->deref, NULL);
575 entry->key = create_entry_key_from_deref(entry, ctx, &path, &entry->offset);
576 nir_deref_path_finish(&path);
577 } else {
578 nir_ssa_def *base = entry->info->base_src >= 0 ?
579 intrin->src[entry->info->base_src].ssa : NULL;
580 uint64_t offset = 0;
581 if (nir_intrinsic_has_base(intrin))
582 offset += nir_intrinsic_base(intrin);
583 entry->key = create_entry_key_from_offset(entry, base, 1, &offset);
584 entry->offset = offset;
585
586 if (base)
587 entry->offset = mask_sign_extend(entry->offset, base->bit_size);
588 }
589
590 if (entry->info->resource_src >= 0)
591 entry->key->resource = intrin->src[entry->info->resource_src].ssa;
592
593 if (nir_intrinsic_has_access(intrin))
594 entry->access = nir_intrinsic_access(intrin);
595 else if (entry->key->var)
596 entry->access = entry->key->var->data.access;
597
598 uint32_t restrict_modes = nir_var_shader_in | nir_var_shader_out;
599 restrict_modes |= nir_var_shader_temp | nir_var_function_temp;
600 restrict_modes |= nir_var_uniform | nir_var_mem_push_const;
601 restrict_modes |= nir_var_system_value | nir_var_mem_shared;
602 if (get_variable_mode(entry) & restrict_modes)
603 entry->access |= ACCESS_RESTRICT;
604
605 calc_alignment(entry);
606
607 return entry;
608 }
609
610 static nir_deref_instr *
611 cast_deref(nir_builder *b, unsigned num_components, unsigned bit_size, nir_deref_instr *deref)
612 {
613 if (glsl_get_components(deref->type) == num_components &&
614 type_scalar_size_bytes(deref->type)*8u == bit_size)
615 return deref;
616
617 enum glsl_base_type types[] = {
618 GLSL_TYPE_UINT8, GLSL_TYPE_UINT16, GLSL_TYPE_UINT, GLSL_TYPE_UINT64};
619 enum glsl_base_type base = types[ffs(bit_size / 8u) - 1u];
620 const struct glsl_type *type = glsl_vector_type(base, num_components);
621
622 if (deref->type == type)
623 return deref;
624
625 return nir_build_deref_cast(b, &deref->dest.ssa, deref->mode, type, 0);
626 }
627
628 /* Return true if the write mask "write_mask" of a store with "old_bit_size"
629 * bits per element can be represented for a store with "new_bit_size" bits per
630 * element. */
631 static bool
632 writemask_representable(unsigned write_mask, unsigned old_bit_size, unsigned new_bit_size)
633 {
634 while (write_mask) {
635 int start, count;
636 u_bit_scan_consecutive_range(&write_mask, &start, &count);
637 start *= old_bit_size;
638 count *= old_bit_size;
639 if (start % new_bit_size != 0)
640 return false;
641 if (count % new_bit_size != 0)
642 return false;
643 }
644 return true;
645 }
646
647 /* Return true if "new_bit_size" is a usable bit size for a vectorized load/store
648 * of "low" and "high". */
649 static bool
650 new_bitsize_acceptable(struct vectorize_ctx *ctx, unsigned new_bit_size,
651 struct entry *low, struct entry *high, unsigned size)
652 {
653 if (size % new_bit_size != 0)
654 return false;
655
656 unsigned new_num_components = size / new_bit_size;
657 if (!nir_num_components_valid(new_num_components))
658 return false;
659
660 unsigned high_offset = high->offset_signed - low->offset_signed;
661
662 /* check nir_extract_bits limitations */
663 unsigned common_bit_size = MIN2(get_bit_size(low), get_bit_size(high));
664 common_bit_size = MIN2(common_bit_size, new_bit_size);
665 if (high_offset > 0)
666 common_bit_size = MIN2(common_bit_size, (1u << (ffs(high_offset * 8) - 1)));
667 if (new_bit_size / common_bit_size > NIR_MAX_VEC_COMPONENTS)
668 return false;
669
670 uint32_t align = low->align_offset ? 1 << (ffs(low->align_offset) - 1) : low->align_mul;
671 if (!ctx->callback(align, new_bit_size, new_num_components,
672 high_offset, low->intrin, high->intrin))
673 return false;
674
675 if (low->is_store) {
676 unsigned low_size = low->intrin->num_components * get_bit_size(low);
677 unsigned high_size = high->intrin->num_components * get_bit_size(high);
678
679 if (low_size % new_bit_size != 0)
680 return false;
681 if (high_size % new_bit_size != 0)
682 return false;
683
684 unsigned write_mask = nir_intrinsic_write_mask(low->intrin);
685 if (!writemask_representable(write_mask, low_size, new_bit_size))
686 return false;
687
688 write_mask = nir_intrinsic_write_mask(high->intrin);
689 if (!writemask_representable(write_mask, high_size, new_bit_size))
690 return false;
691 }
692
693 return true;
694 }
695
696 /* Updates a write mask, "write_mask", so that it can be used with a
697 * "new_bit_size"-bit store instead of a "old_bit_size"-bit store. */
698 static uint32_t
699 update_writemask(unsigned write_mask, unsigned old_bit_size, unsigned new_bit_size)
700 {
701 uint32_t res = 0;
702 while (write_mask) {
703 int start, count;
704 u_bit_scan_consecutive_range(&write_mask, &start, &count);
705 start = start * old_bit_size / new_bit_size;
706 count = count * old_bit_size / new_bit_size;
707 res |= ((1 << count) - 1) << start;
708 }
709 return res;
710 }
711
712 static nir_deref_instr *subtract_deref(nir_builder *b, nir_deref_instr *deref, int64_t offset)
713 {
714 /* avoid adding another deref to the path */
715 if (deref->deref_type == nir_deref_type_ptr_as_array &&
716 nir_src_is_const(deref->arr.index) &&
717 offset % nir_deref_instr_array_stride(deref) == 0) {
718 unsigned stride = nir_deref_instr_array_stride(deref);
719 nir_ssa_def *index = nir_imm_intN_t(b, nir_src_as_int(deref->arr.index) - offset / stride,
720 deref->dest.ssa.bit_size);
721 return nir_build_deref_ptr_as_array(b, nir_deref_instr_parent(deref), index);
722 }
723
724 if (deref->deref_type == nir_deref_type_array &&
725 nir_src_is_const(deref->arr.index)) {
726 nir_deref_instr *parent = nir_deref_instr_parent(deref);
727 unsigned stride = glsl_get_explicit_stride(parent->type);
728 if (offset % stride == 0)
729 return nir_build_deref_array_imm(
730 b, parent, nir_src_as_int(deref->arr.index) - offset / stride);
731 }
732
733
734 deref = nir_build_deref_cast(b, &deref->dest.ssa, deref->mode,
735 glsl_scalar_type(GLSL_TYPE_UINT8), 1);
736 return nir_build_deref_ptr_as_array(
737 b, deref, nir_imm_intN_t(b, -offset, deref->dest.ssa.bit_size));
738 }
739
740 static void
741 vectorize_loads(nir_builder *b, struct vectorize_ctx *ctx,
742 struct entry *low, struct entry *high,
743 struct entry *first, struct entry *second,
744 unsigned new_bit_size, unsigned new_num_components,
745 unsigned high_start)
746 {
747 unsigned low_bit_size = get_bit_size(low);
748 unsigned high_bit_size = get_bit_size(high);
749 bool low_bool = low->intrin->dest.ssa.bit_size == 1;
750 bool high_bool = high->intrin->dest.ssa.bit_size == 1;
751 nir_ssa_def *data = &first->intrin->dest.ssa;
752
753 b->cursor = nir_after_instr(first->instr);
754
755 /* update the load's destination size and extract data for each of the original loads */
756 data->num_components = new_num_components;
757 data->bit_size = new_bit_size;
758
759 nir_ssa_def *low_def = nir_extract_bits(
760 b, &data, 1, 0, low->intrin->num_components, low_bit_size);
761 nir_ssa_def *high_def = nir_extract_bits(
762 b, &data, 1, high_start, high->intrin->num_components, high_bit_size);
763
764 /* convert booleans */
765 low_def = low_bool ? nir_i2b(b, low_def) : nir_mov(b, low_def);
766 high_def = high_bool ? nir_i2b(b, high_def) : nir_mov(b, high_def);
767
768 /* update uses */
769 if (first == low) {
770 nir_ssa_def_rewrite_uses_after(&low->intrin->dest.ssa, nir_src_for_ssa(low_def),
771 high_def->parent_instr);
772 nir_ssa_def_rewrite_uses(&high->intrin->dest.ssa, nir_src_for_ssa(high_def));
773 } else {
774 nir_ssa_def_rewrite_uses(&low->intrin->dest.ssa, nir_src_for_ssa(low_def));
775 nir_ssa_def_rewrite_uses_after(&high->intrin->dest.ssa, nir_src_for_ssa(high_def),
776 high_def->parent_instr);
777 }
778
779 /* update the intrinsic */
780 first->intrin->num_components = new_num_components;
781
782 const struct intrinsic_info *info = first->info;
783
784 /* update the offset */
785 if (first != low && info->base_src >= 0) {
786 /* let nir_opt_algebraic() remove this addition. this doesn't have much
787 * issues with subtracting 16 from expressions like "(i + 1) * 16" because
788 * nir_opt_algebraic() turns them into "i * 16 + 16" */
789 b->cursor = nir_before_instr(first->instr);
790
791 nir_ssa_def *new_base = first->intrin->src[info->base_src].ssa;
792 new_base = nir_iadd_imm(b, new_base, -(int)(high_start / 8u));
793
794 nir_instr_rewrite_src(first->instr, &first->intrin->src[info->base_src],
795 nir_src_for_ssa(new_base));
796 }
797
798 /* update the deref */
799 if (info->deref_src >= 0) {
800 b->cursor = nir_before_instr(first->instr);
801
802 nir_deref_instr *deref = nir_src_as_deref(first->intrin->src[info->deref_src]);
803 if (first != low && high_start != 0)
804 deref = subtract_deref(b, deref, high_start / 8u);
805 first->deref = cast_deref(b, new_num_components, new_bit_size, deref);
806
807 nir_instr_rewrite_src(first->instr, &first->intrin->src[info->deref_src],
808 nir_src_for_ssa(&first->deref->dest.ssa));
809 }
810
811 /* update base/align */
812 if (first != low && nir_intrinsic_has_base(first->intrin))
813 nir_intrinsic_set_base(first->intrin, nir_intrinsic_base(low->intrin));
814
815 first->key = low->key;
816 first->offset = low->offset;
817
818 first->align_mul = low->align_mul;
819 first->align_offset = low->align_offset;
820
821 nir_instr_remove(second->instr);
822 }
823
824 static void
825 vectorize_stores(nir_builder *b, struct vectorize_ctx *ctx,
826 struct entry *low, struct entry *high,
827 struct entry *first, struct entry *second,
828 unsigned new_bit_size, unsigned new_num_components,
829 unsigned high_start)
830 {
831 ASSERTED unsigned low_size = low->intrin->num_components * get_bit_size(low);
832 assert(low_size % new_bit_size == 0);
833
834 b->cursor = nir_before_instr(second->instr);
835
836 /* get new writemasks */
837 uint32_t low_write_mask = nir_intrinsic_write_mask(low->intrin);
838 uint32_t high_write_mask = nir_intrinsic_write_mask(high->intrin);
839 low_write_mask = update_writemask(low_write_mask, get_bit_size(low), new_bit_size);
840 high_write_mask = update_writemask(high_write_mask, get_bit_size(high), new_bit_size);
841 high_write_mask <<= high_start / new_bit_size;
842
843 uint32_t write_mask = low_write_mask | high_write_mask;
844
845 /* convert booleans */
846 nir_ssa_def *low_val = low->intrin->src[low->info->value_src].ssa;
847 nir_ssa_def *high_val = high->intrin->src[high->info->value_src].ssa;
848 low_val = low_val->bit_size == 1 ? nir_b2i(b, low_val, 32) : low_val;
849 high_val = high_val->bit_size == 1 ? nir_b2i(b, high_val, 32) : high_val;
850
851 /* combine the data */
852 nir_ssa_def *data_channels[NIR_MAX_VEC_COMPONENTS];
853 for (unsigned i = 0; i < new_num_components; i++) {
854 bool set_low = low_write_mask & (1 << i);
855 bool set_high = high_write_mask & (1 << i);
856
857 if (set_low && (!set_high || low == second)) {
858 unsigned offset = i * new_bit_size;
859 data_channels[i] = nir_extract_bits(b, &low_val, 1, offset, 1, new_bit_size);
860 } else if (set_high) {
861 assert(!set_low || high == second);
862 unsigned offset = i * new_bit_size - high_start;
863 data_channels[i] = nir_extract_bits(b, &high_val, 1, offset, 1, new_bit_size);
864 } else {
865 data_channels[i] = nir_ssa_undef(b, 1, new_bit_size);
866 }
867 }
868 nir_ssa_def *data = nir_vec(b, data_channels, new_num_components);
869
870 /* update the intrinsic */
871 nir_intrinsic_set_write_mask(second->intrin, write_mask);
872 second->intrin->num_components = data->num_components;
873
874 const struct intrinsic_info *info = second->info;
875 assert(info->value_src >= 0);
876 nir_instr_rewrite_src(second->instr, &second->intrin->src[info->value_src],
877 nir_src_for_ssa(data));
878
879 /* update the offset */
880 if (second != low && info->base_src >= 0)
881 nir_instr_rewrite_src(second->instr, &second->intrin->src[info->base_src],
882 low->intrin->src[info->base_src]);
883
884 /* update the deref */
885 if (info->deref_src >= 0) {
886 b->cursor = nir_before_instr(second->instr);
887 second->deref = cast_deref(b, new_num_components, new_bit_size,
888 nir_src_as_deref(low->intrin->src[info->deref_src]));
889 nir_instr_rewrite_src(second->instr, &second->intrin->src[info->deref_src],
890 nir_src_for_ssa(&second->deref->dest.ssa));
891 }
892
893 /* update base/align */
894 if (second != low && nir_intrinsic_has_base(second->intrin))
895 nir_intrinsic_set_base(second->intrin, nir_intrinsic_base(low->intrin));
896
897 second->key = low->key;
898 second->offset = low->offset;
899
900 second->align_mul = low->align_mul;
901 second->align_offset = low->align_offset;
902
903 list_del(&first->head);
904 nir_instr_remove(first->instr);
905 }
906
907 /* Returns true if it can prove that "a" and "b" point to different resources. */
908 static bool
909 resources_different(nir_ssa_def *a, nir_ssa_def *b)
910 {
911 if (!a || !b)
912 return false;
913
914 if (a->parent_instr->type == nir_instr_type_load_const &&
915 b->parent_instr->type == nir_instr_type_load_const) {
916 return nir_src_as_uint(nir_src_for_ssa(a)) != nir_src_as_uint(nir_src_for_ssa(b));
917 }
918
919 if (a->parent_instr->type == nir_instr_type_intrinsic &&
920 b->parent_instr->type == nir_instr_type_intrinsic) {
921 nir_intrinsic_instr *aintrin = nir_instr_as_intrinsic(a->parent_instr);
922 nir_intrinsic_instr *bintrin = nir_instr_as_intrinsic(b->parent_instr);
923 if (aintrin->intrinsic == nir_intrinsic_vulkan_resource_index &&
924 bintrin->intrinsic == nir_intrinsic_vulkan_resource_index) {
925 return nir_intrinsic_desc_set(aintrin) != nir_intrinsic_desc_set(bintrin) ||
926 nir_intrinsic_binding(aintrin) != nir_intrinsic_binding(bintrin) ||
927 resources_different(aintrin->src[0].ssa, bintrin->src[0].ssa);
928 }
929 }
930
931 return false;
932 }
933
934 static int64_t
935 compare_entries(struct entry *a, struct entry *b)
936 {
937 if (!entry_key_equals(a->key, b->key))
938 return INT64_MAX;
939 return b->offset_signed - a->offset_signed;
940 }
941
942 static bool
943 may_alias(struct entry *a, struct entry *b)
944 {
945 assert(mode_to_index(get_variable_mode(a)) ==
946 mode_to_index(get_variable_mode(b)));
947
948 /* if the resources/variables are definitively different and both have
949 * ACCESS_RESTRICT, we can assume they do not alias. */
950 bool res_different = a->key->var != b->key->var ||
951 resources_different(a->key->resource, b->key->resource);
952 if (res_different && (a->access & ACCESS_RESTRICT) && (b->access & ACCESS_RESTRICT))
953 return false;
954
955 /* we can't compare offsets if the resources/variables might be different */
956 if (a->key->var != b->key->var || a->key->resource != b->key->resource)
957 return true;
958
959 /* use adjacency information */
960 /* TODO: we can look closer at the entry keys */
961 int64_t diff = compare_entries(a, b);
962 if (diff != INT64_MAX) {
963 /* with atomics, intrin->num_components can be 0 */
964 if (diff < 0)
965 return llabs(diff) < MAX2(b->intrin->num_components, 1u) * (get_bit_size(b) / 8u);
966 else
967 return diff < MAX2(a->intrin->num_components, 1u) * (get_bit_size(a) / 8u);
968 }
969
970 /* TODO: we can use deref information */
971
972 return true;
973 }
974
975 static bool
976 check_for_aliasing(struct vectorize_ctx *ctx, struct entry *first, struct entry *second)
977 {
978 nir_variable_mode mode = get_variable_mode(first);
979 if (mode & (nir_var_uniform | nir_var_system_value |
980 nir_var_mem_push_const | nir_var_mem_ubo))
981 return false;
982
983 unsigned mode_index = mode_to_index(mode);
984 if (first->is_store) {
985 /* find first entry that aliases "first" */
986 list_for_each_entry_from(struct entry, next, first, &ctx->entries[mode_index], head) {
987 if (next == first)
988 continue;
989 if (next == second)
990 return false;
991 if (may_alias(first, next))
992 return true;
993 }
994 } else {
995 /* find previous store that aliases this load */
996 list_for_each_entry_from_rev(struct entry, prev, second, &ctx->entries[mode_index], head) {
997 if (prev == second)
998 continue;
999 if (prev == first)
1000 return false;
1001 if (prev->is_store && may_alias(second, prev))
1002 return true;
1003 }
1004 }
1005
1006 return false;
1007 }
1008
1009 static bool
1010 check_for_robustness(struct vectorize_ctx *ctx, struct entry *low)
1011 {
1012 nir_variable_mode mode = get_variable_mode(low);
1013 if (mode & ctx->robust_modes) {
1014 unsigned low_bit_size = get_bit_size(low);
1015 unsigned low_size = low->intrin->num_components * low_bit_size;
1016
1017 /* don't attempt to vectorize accesses if the offset can overflow. */
1018 /* TODO: handle indirect accesses. */
1019 return low->offset_signed < 0 && low->offset_signed + low_size >= 0;
1020 }
1021
1022 return false;
1023 }
1024
1025 static bool
1026 is_strided_vector(const struct glsl_type *type)
1027 {
1028 if (glsl_type_is_vector(type)) {
1029 unsigned explicit_stride = glsl_get_explicit_stride(type);
1030 return explicit_stride != 0 && explicit_stride !=
1031 type_scalar_size_bytes(glsl_get_array_element(type));
1032 } else {
1033 return false;
1034 }
1035 }
1036
1037 static bool
1038 try_vectorize(nir_function_impl *impl, struct vectorize_ctx *ctx,
1039 struct entry *low, struct entry *high,
1040 struct entry *first, struct entry *second)
1041 {
1042 if (!(get_variable_mode(first) & ctx->modes) ||
1043 !(get_variable_mode(second) & ctx->modes))
1044 return false;
1045
1046 if (check_for_aliasing(ctx, first, second))
1047 return false;
1048
1049 if (check_for_robustness(ctx, low))
1050 return false;
1051
1052 /* we can only vectorize non-volatile loads/stores of the same type and with
1053 * the same access */
1054 if (first->info != second->info || first->access != second->access ||
1055 (first->access & ACCESS_VOLATILE) || first->info->is_atomic)
1056 return false;
1057
1058 /* don't attempt to vectorize accesses of row-major matrix columns */
1059 if (first->deref) {
1060 const struct glsl_type *first_type = first->deref->type;
1061 const struct glsl_type *second_type = second->deref->type;
1062 if (is_strided_vector(first_type) || is_strided_vector(second_type))
1063 return false;
1064 }
1065
1066 /* gather information */
1067 uint64_t diff = high->offset_signed - low->offset_signed;
1068 unsigned low_bit_size = get_bit_size(low);
1069 unsigned high_bit_size = get_bit_size(high);
1070 unsigned low_size = low->intrin->num_components * low_bit_size;
1071 unsigned high_size = high->intrin->num_components * high_bit_size;
1072 unsigned new_size = MAX2(diff * 8u + high_size, low_size);
1073
1074 /* find a good bit size for the new load/store */
1075 unsigned new_bit_size = 0;
1076 if (new_bitsize_acceptable(ctx, low_bit_size, low, high, new_size)) {
1077 new_bit_size = low_bit_size;
1078 } else if (low_bit_size != high_bit_size &&
1079 new_bitsize_acceptable(ctx, high_bit_size, low, high, new_size)) {
1080 new_bit_size = high_bit_size;
1081 } else {
1082 new_bit_size = 64;
1083 for (; new_bit_size >= 8; new_bit_size /= 2) {
1084 /* don't repeat trying out bitsizes */
1085 if (new_bit_size == low_bit_size || new_bit_size == high_bit_size)
1086 continue;
1087 if (new_bitsize_acceptable(ctx, new_bit_size, low, high, new_size))
1088 break;
1089 }
1090 if (new_bit_size < 8)
1091 return false;
1092 }
1093 unsigned new_num_components = new_size / new_bit_size;
1094
1095 /* vectorize the loads/stores */
1096 nir_builder b;
1097 nir_builder_init(&b, impl);
1098
1099 if (first->is_store)
1100 vectorize_stores(&b, ctx, low, high, first, second,
1101 new_bit_size, new_num_components, diff * 8u);
1102 else
1103 vectorize_loads(&b, ctx, low, high, first, second,
1104 new_bit_size, new_num_components, diff * 8u);
1105
1106 return true;
1107 }
1108
1109 static bool
1110 update_align(struct entry *entry)
1111 {
1112 if (nir_intrinsic_has_align_mul(entry->intrin) &&
1113 (entry->align_mul != nir_intrinsic_align_mul(entry->intrin) ||
1114 entry->align_offset != nir_intrinsic_align_offset(entry->intrin))) {
1115 nir_intrinsic_set_align(entry->intrin, entry->align_mul, entry->align_offset);
1116 return true;
1117 }
1118 return false;
1119 }
1120
1121 static bool
1122 vectorize_entries(struct vectorize_ctx *ctx, nir_function_impl *impl, struct hash_table *ht)
1123 {
1124 if (!ht)
1125 return false;
1126
1127 bool progress = false;
1128 hash_table_foreach(ht, entry) {
1129 struct util_dynarray *arr = entry->data;
1130 if (!arr->size)
1131 continue;
1132
1133 qsort(util_dynarray_begin(arr),
1134 util_dynarray_num_elements(arr, struct entry *),
1135 sizeof(struct entry *), &sort_entries);
1136
1137 unsigned i = 0;
1138 for (; i < util_dynarray_num_elements(arr, struct entry*) - 1; i++) {
1139 struct entry *low = *util_dynarray_element(arr, struct entry *, i);
1140 struct entry *high = *util_dynarray_element(arr, struct entry *, i + 1);
1141
1142 uint64_t diff = high->offset_signed - low->offset_signed;
1143 if (diff > get_bit_size(low) / 8u * low->intrin->num_components)
1144 continue;
1145
1146 struct entry *first = low->index < high->index ? low : high;
1147 struct entry *second = low->index < high->index ? high : low;
1148
1149 if (try_vectorize(impl, ctx, low, high, first, second)) {
1150 *util_dynarray_element(arr, struct entry *, i) = NULL;
1151 *util_dynarray_element(arr, struct entry *, i + 1) = low->is_store ? second : first;
1152 progress = true;
1153 }
1154 }
1155
1156 util_dynarray_foreach(arr, struct entry *, elem) {
1157 if (*elem)
1158 progress |= update_align(*elem);
1159 }
1160 }
1161
1162 _mesa_hash_table_clear(ht, delete_entry_dynarray);
1163
1164 return progress;
1165 }
1166
1167 static bool
1168 handle_barrier(struct vectorize_ctx *ctx, bool *progress, nir_function_impl *impl, nir_instr *instr)
1169 {
1170 unsigned modes = 0;
1171 bool acquire = true;
1172 bool release = true;
1173 if (instr->type == nir_instr_type_intrinsic) {
1174 nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
1175 switch (intrin->intrinsic) {
1176 case nir_intrinsic_group_memory_barrier:
1177 case nir_intrinsic_memory_barrier:
1178 modes = nir_var_mem_ssbo | nir_var_mem_shared | nir_var_mem_global;
1179 break;
1180 /* prevent speculative loads/stores */
1181 case nir_intrinsic_discard_if:
1182 case nir_intrinsic_discard:
1183 modes = nir_var_all;
1184 break;
1185 case nir_intrinsic_memory_barrier_buffer:
1186 modes = nir_var_mem_ssbo | nir_var_mem_global;
1187 break;
1188 case nir_intrinsic_memory_barrier_shared:
1189 modes = nir_var_mem_shared;
1190 break;
1191 case nir_intrinsic_scoped_barrier:
1192 if (nir_intrinsic_memory_scope(intrin) == NIR_SCOPE_NONE)
1193 break;
1194
1195 modes = nir_intrinsic_memory_modes(intrin) & (nir_var_mem_ssbo |
1196 nir_var_mem_shared |
1197 nir_var_mem_global);
1198 acquire = nir_intrinsic_memory_semantics(intrin) & NIR_MEMORY_ACQUIRE;
1199 release = nir_intrinsic_memory_semantics(intrin) & NIR_MEMORY_RELEASE;
1200 switch (nir_intrinsic_memory_scope(intrin)) {
1201 case NIR_SCOPE_INVOCATION:
1202 case NIR_SCOPE_SUBGROUP:
1203 /* a barier should never be required for correctness with these scopes */
1204 modes = 0;
1205 break;
1206 default:
1207 break;
1208 }
1209 break;
1210 default:
1211 return false;
1212 }
1213 } else if (instr->type == nir_instr_type_call) {
1214 modes = nir_var_all;
1215 } else {
1216 return false;
1217 }
1218
1219 while (modes) {
1220 unsigned mode_index = u_bit_scan(&modes);
1221 if ((1 << mode_index) == nir_var_mem_global) {
1222 /* Global should be rolled in with SSBO */
1223 assert(list_is_empty(&ctx->entries[mode_index]));
1224 assert(ctx->loads[mode_index] == NULL);
1225 assert(ctx->stores[mode_index] == NULL);
1226 continue;
1227 }
1228
1229 if (acquire)
1230 *progress |= vectorize_entries(ctx, impl, ctx->loads[mode_index]);
1231 if (release)
1232 *progress |= vectorize_entries(ctx, impl, ctx->stores[mode_index]);
1233 }
1234
1235 return true;
1236 }
1237
1238 static bool
1239 process_block(nir_function_impl *impl, struct vectorize_ctx *ctx, nir_block *block)
1240 {
1241 bool progress = false;
1242
1243 for (unsigned i = 0; i < nir_num_variable_modes; i++) {
1244 list_inithead(&ctx->entries[i]);
1245 if (ctx->loads[i])
1246 _mesa_hash_table_clear(ctx->loads[i], delete_entry_dynarray);
1247 if (ctx->stores[i])
1248 _mesa_hash_table_clear(ctx->stores[i], delete_entry_dynarray);
1249 }
1250
1251 /* create entries */
1252 unsigned next_index = 0;
1253
1254 nir_foreach_instr_safe(instr, block) {
1255 if (handle_barrier(ctx, &progress, impl, instr))
1256 continue;
1257
1258 /* gather information */
1259 if (instr->type != nir_instr_type_intrinsic)
1260 continue;
1261 nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
1262
1263 const struct intrinsic_info *info = get_info(intrin->intrinsic);
1264 if (!info)
1265 continue;
1266
1267 nir_variable_mode mode = info->mode;
1268 if (!mode)
1269 mode = nir_src_as_deref(intrin->src[info->deref_src])->mode;
1270 if (!(mode & aliasing_modes(ctx->modes)))
1271 continue;
1272 unsigned mode_index = mode_to_index(mode);
1273
1274 /* create entry */
1275 struct entry *entry = create_entry(ctx, info, intrin);
1276 entry->index = next_index++;
1277
1278 list_addtail(&entry->head, &ctx->entries[mode_index]);
1279
1280 /* add the entry to a hash table */
1281
1282 struct hash_table *adj_ht = NULL;
1283 if (entry->is_store) {
1284 if (!ctx->stores[mode_index])
1285 ctx->stores[mode_index] = _mesa_hash_table_create(ctx, &hash_entry_key, &entry_key_equals);
1286 adj_ht = ctx->stores[mode_index];
1287 } else {
1288 if (!ctx->loads[mode_index])
1289 ctx->loads[mode_index] = _mesa_hash_table_create(ctx, &hash_entry_key, &entry_key_equals);
1290 adj_ht = ctx->loads[mode_index];
1291 }
1292
1293 uint32_t key_hash = hash_entry_key(entry->key);
1294 struct hash_entry *adj_entry = _mesa_hash_table_search_pre_hashed(adj_ht, key_hash, entry->key);
1295 struct util_dynarray *arr;
1296 if (adj_entry && adj_entry->data) {
1297 arr = (struct util_dynarray *)adj_entry->data;
1298 } else {
1299 arr = ralloc(ctx, struct util_dynarray);
1300 util_dynarray_init(arr, arr);
1301 _mesa_hash_table_insert_pre_hashed(adj_ht, key_hash, entry->key, arr);
1302 }
1303 util_dynarray_append(arr, struct entry *, entry);
1304 }
1305
1306 /* sort and combine entries */
1307 for (unsigned i = 0; i < nir_num_variable_modes; i++) {
1308 progress |= vectorize_entries(ctx, impl, ctx->loads[i]);
1309 progress |= vectorize_entries(ctx, impl, ctx->stores[i]);
1310 }
1311
1312 return progress;
1313 }
1314
1315 bool
1316 nir_opt_load_store_vectorize(nir_shader *shader, nir_variable_mode modes,
1317 nir_should_vectorize_mem_func callback,
1318 nir_variable_mode robust_modes)
1319 {
1320 bool progress = false;
1321
1322 struct vectorize_ctx *ctx = rzalloc(NULL, struct vectorize_ctx);
1323 ctx->modes = modes;
1324 ctx->callback = callback;
1325 ctx->robust_modes = robust_modes;
1326
1327 nir_shader_index_vars(shader, modes);
1328
1329 nir_foreach_function(function, shader) {
1330 if (function->impl) {
1331 if (modes & nir_var_function_temp)
1332 nir_function_impl_index_vars(function->impl);
1333
1334 nir_foreach_block(block, function->impl)
1335 progress |= process_block(function->impl, ctx, block);
1336
1337 nir_metadata_preserve(function->impl,
1338 nir_metadata_block_index |
1339 nir_metadata_dominance |
1340 nir_metadata_live_ssa_defs);
1341 }
1342 }
1343
1344 ralloc_free(ctx);
1345 return progress;
1346 }