src/freedreno/ir3/ir3_nir_analyze_ubo_ranges.c

   1 /*
   2  * Copyright © 2019 Google, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  21  * SOFTWARE.
  22  */
  23
  24 #include "ir3_nir.h"
  25 #include "ir3_compiler.h"
  26 #include "compiler/nir/nir.h"
  27 #include "compiler/nir/nir_builder.h"
  28 #include "util/u_math.h"
  29
  30 static bool
  31 range_is_gl_uniforms(struct ir3_ubo_range *r)
  32 {
  33         return !r->bindless && r->block == 0;
  34 }
  35
  36 static inline struct ir3_ubo_range
  37 get_ubo_load_range(nir_shader *nir, nir_intrinsic_instr *instr, uint32_t alignment)
  38 {
  39         struct ir3_ubo_range r;
  40
  41         if (nir_src_is_const(instr->src[1])) {
  42                 int offset = nir_src_as_uint(instr->src[1]);
  43                 const int bytes = nir_intrinsic_dest_components(instr) * 4;
  44
  45                 r.start = ROUND_DOWN_TO(offset, alignment * 16);
  46                 r.end = ALIGN(offset + bytes, alignment * 16);
  47         } else {
  48                 /* The other valid place to call this is on the GL default uniform block */
  49                 assert(nir_src_as_uint(instr->src[0]) == 0);
  50                 r.start = 0;
  51                 r.end = ALIGN(nir->num_uniforms * 16, alignment * 16);
  52         }
  53
  54         return r;
  55 }
  56
  57 static struct ir3_ubo_range *
  58 get_existing_range(nir_intrinsic_instr *instr,
  59                                    struct ir3_ubo_analysis_state *state,
  60                                    bool create_new)
  61 {
  62         unsigned block, base = 0;
  63         bool bindless;
  64         if (nir_src_is_const(instr->src[0])) {
  65                 block = nir_src_as_uint(instr->src[0]);
  66                 bindless = false;
  67         } else {
  68                 nir_intrinsic_instr *rsrc = ir3_bindless_resource(instr->src[0]);
  69                 if (rsrc && nir_src_is_const(rsrc->src[0])) {
  70                         block = nir_src_as_uint(rsrc->src[0]);
  71                         base = nir_intrinsic_desc_set(rsrc);
  72                         bindless = true;
  73                 } else {
  74                         return NULL;
  75                 }
  76         }
  77         for (int i = 0; i < IR3_MAX_UBO_PUSH_RANGES; i++) {
  78                 struct ir3_ubo_range *range = &state->range[i];
  79                 if (range->end < range->start) {
  80                         /* We don't have a matching range, but there are more available.
  81                          */
  82                         if (create_new) {
  83                                 range->block = block;
  84                                 range->bindless_base = base;
  85                                 range->bindless = bindless;
  86                                 return range;
  87                         } else {
  88                                 return NULL;
  89                         }
  90                 } else if (range->block == block && range->bindless_base == base &&
  91                                    range->bindless == bindless) {
  92                         return range;
  93                 }
  94         }
  95
  96         return NULL;
  97 }
  98
  99 static void
 100 gather_ubo_ranges(nir_shader *nir, nir_intrinsic_instr *instr,
 101                                   struct ir3_ubo_analysis_state *state, uint32_t alignment)
 102 {
 103         if (ir3_shader_debug & IR3_DBG_NOUBOOPT)
 104                 return;
 105
 106         struct ir3_ubo_range *old_r = get_existing_range(instr, state, true);
 107         if (!old_r)
 108                 return;
 109
 110         /* We don't know how to get the size of UBOs being indirected on, other
 111          * than on the GL uniforms where we have some other shader_info data.
 112          */
 113         if (!nir_src_is_const(instr->src[1]) && !range_is_gl_uniforms(old_r))
 114                 return;
 115
 116         const struct ir3_ubo_range r = get_ubo_load_range(nir, instr, alignment);
 117
 118         if (r.start < old_r->start)
 119                 old_r->start = r.start;
 120         if (old_r->end < r.end)
 121                 old_r->end = r.end;
 122 }
 123
 124 /* For indirect offset, it is common to see a pattern of multiple
 125  * loads with the same base, but different constant offset, ie:
 126  *
 127  *    vec1 32 ssa_33 = iadd ssa_base, const_offset
 128  *    vec4 32 ssa_34 = intrinsic load_uniform (ssa_33) (base=N, 0, 0)
 129  *
 130  * Detect this, and peel out the const_offset part, to end up with:
 131  *
 132  *    vec4 32 ssa_34 = intrinsic load_uniform (ssa_base) (base=N+const_offset, 0, 0)
 133  *
 134  * Or similarly:
 135  *
 136  *    vec1 32 ssa_33 = imad24_ir3 a, b, const_offset
 137  *    vec4 32 ssa_34 = intrinsic load_uniform (ssa_33) (base=N, 0, 0)
 138  *
 139  * Can be converted to:
 140  *
 141  *    vec1 32 ssa_base = imul24 a, b
 142  *    vec4 32 ssa_34 = intrinsic load_uniform (ssa_base) (base=N+const_offset, 0, 0)
 143  *
 144  * This gives the other opt passes something much easier to work
 145  * with (ie. not requiring value range tracking)
 146  */
 147 static void
 148 handle_partial_const(nir_builder *b, nir_ssa_def **srcp, int *offp)
 149 {
 150         if ((*srcp)->parent_instr->type != nir_instr_type_alu)
 151                 return;
 152
 153         nir_alu_instr *alu = nir_instr_as_alu((*srcp)->parent_instr);
 154
 155         if (alu->op == nir_op_imad24_ir3) {
 156                 /* This case is slightly more complicated as we need to
 157                  * replace the imad24_ir3 with an imul24:
 158                  */
 159                 if (!nir_src_is_const(alu->src[2].src))
 160                         return;
 161
 162                 *offp += nir_src_as_uint(alu->src[2].src);
 163                 *srcp = nir_imul24(b, nir_ssa_for_alu_src(b, alu, 0),
 164                                 nir_ssa_for_alu_src(b, alu, 1));
 165
 166                 return;
 167         }
 168
 169         if (alu->op != nir_op_iadd)
 170                 return;
 171
 172         if (!(alu->src[0].src.is_ssa && alu->src[1].src.is_ssa))
 173                 return;
 174
 175         if (nir_src_is_const(alu->src[0].src)) {
 176                 *offp += nir_src_as_uint(alu->src[0].src);
 177                 *srcp = alu->src[1].src.ssa;
 178         } else if (nir_src_is_const(alu->src[1].src)) {
 179                 *srcp = alu->src[0].src.ssa;
 180                 *offp += nir_src_as_uint(alu->src[1].src);
 181         }
 182 }
 183
 184 /* Tracks the maximum bindful UBO accessed so that we reduce the UBO
 185  * descriptors emitted in the fast path for GL.
 186  */
 187 static void
 188 track_ubo_use(nir_intrinsic_instr *instr, nir_builder *b, int *num_ubos)
 189 {
 190         if (ir3_bindless_resource(instr->src[0])) {
 191                 assert(!b->shader->info.first_ubo_is_default_ubo); /* only set for GL */
 192                 return;
 193         }
 194
 195         if (nir_src_is_const(instr->src[0])) {
 196                 int block = nir_src_as_uint(instr->src[0]);
 197                 *num_ubos = MAX2(*num_ubos, block + 1);
 198         } else {
 199                 *num_ubos = b->shader->info.num_ubos;
 200         }
 201 }
 202
 203 static void
 204 lower_ubo_load_to_uniform(nir_intrinsic_instr *instr, nir_builder *b,
 205                 struct ir3_ubo_analysis_state *state, int *num_ubos, uint32_t alignment)
 206 {
 207         b->cursor = nir_before_instr(&instr->instr);
 208
 209         /* We don't lower dynamic block index UBO loads to load_uniform, but we
 210          * could probably with some effort determine a block stride in number of
 211          * registers.
 212          */
 213         struct ir3_ubo_range *range = get_existing_range(instr, state, false);
 214         if (!range) {
 215                 track_ubo_use(instr, b, num_ubos);
 216                 return;
 217         }
 218
 219         /* We don't have a good way of determining the range of the dynamic
 220          * access in general, so for now just fall back to pulling.
 221          */
 222         if (!nir_src_is_const(instr->src[1]) && !range_is_gl_uniforms(range))
 223                 return;
 224
 225         /* After gathering the UBO access ranges, we limit the total
 226          * upload. Don't lower if this load is outside the range.
 227          */
 228         const struct ir3_ubo_range r = get_ubo_load_range(b->shader,
 229                         instr, alignment);
 230         if (!(range->start <= r.start && r.end <= range->end)) {
 231                 track_ubo_use(instr, b, num_ubos);
 232                 return;
 233         }
 234
 235         nir_ssa_def *ubo_offset = nir_ssa_for_src(b, instr->src[1], 1);
 236         int const_offset = 0;
 237
 238         handle_partial_const(b, &ubo_offset, &const_offset);
 239
 240         /* UBO offset is in bytes, but uniform offset is in units of
 241          * dwords, so we need to divide by 4 (right-shift by 2). For ldc the
 242          * offset is in units of 16 bytes, so we need to multiply by 4. And
 243          * also the same for the constant part of the offset:
 244          */
 245         const int shift = -2;
 246         nir_ssa_def *new_offset = ir3_nir_try_propagate_bit_shift(b, ubo_offset, -2);
 247         nir_ssa_def *uniform_offset = NULL;
 248         if (new_offset) {
 249                 uniform_offset = new_offset;
 250         } else {
 251                 uniform_offset = shift > 0 ?
 252                         nir_ishl(b, ubo_offset, nir_imm_int(b,  shift)) :
 253                         nir_ushr(b, ubo_offset, nir_imm_int(b, -shift));
 254         }
 255
 256         debug_assert(!(const_offset & 0x3));
 257         const_offset >>= 2;
 258
 259         const int range_offset = ((int)range->offset - (int)range->start) / 4;
 260         const_offset += range_offset;
 261
 262         /* The range_offset could be negative, if if only part of the UBO
 263          * block is accessed, range->start can be greater than range->offset.
 264          * But we can't underflow const_offset.  If necessary we need to
 265          * insert nir instructions to compensate (which can hopefully be
 266          * optimized away)
 267          */
 268         if (const_offset < 0) {
 269                 uniform_offset = nir_iadd_imm(b, uniform_offset, const_offset);
 270                 const_offset = 0;
 271         }
 272
 273         nir_intrinsic_instr *uniform =
 274                 nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_uniform);
 275         uniform->num_components = instr->num_components;
 276         uniform->src[0] = nir_src_for_ssa(uniform_offset);
 277         nir_intrinsic_set_base(uniform, const_offset);
 278         nir_ssa_dest_init(&uniform->instr, &uniform->dest,
 279                                           uniform->num_components, instr->dest.ssa.bit_size,
 280                                           instr->dest.ssa.name);
 281         nir_builder_instr_insert(b, &uniform->instr);
 282         nir_ssa_def_rewrite_uses(&instr->dest.ssa,
 283                                                          nir_src_for_ssa(&uniform->dest.ssa));
 284
 285         nir_instr_remove(&instr->instr);
 286
 287         state->lower_count++;
 288 }
 289
 290 static bool
 291 instr_is_load_ubo(nir_instr *instr)
 292 {
 293         if (instr->type != nir_instr_type_intrinsic)
 294                 return false;
 295
 296         nir_intrinsic_op op = nir_instr_as_intrinsic(instr)->intrinsic;
 297
 298         /* ir3_nir_lower_io_offsets happens after this pass. */
 299         assert(op != nir_intrinsic_load_ubo_ir3);
 300
 301         return op == nir_intrinsic_load_ubo;
 302 }
 303
 304 bool
 305 ir3_nir_analyze_ubo_ranges(nir_shader *nir, struct ir3_shader *shader)
 306 {
 307         struct ir3_const_state *const_state = shader->const_state;
 308         struct ir3_ubo_analysis_state *state = &const_state->ubo_state;
 309
 310         memset(state, 0, sizeof(*state));
 311         for (int i = 0; i < IR3_MAX_UBO_PUSH_RANGES; i++) {
 312                 state->range[i].start = UINT32_MAX;
 313         }
 314
 315         nir_foreach_function (function, nir) {
 316                 if (function->impl) {
 317                         nir_foreach_block (block, function->impl) {
 318                                 nir_foreach_instr (instr, block) {
 319                                         if (instr_is_load_ubo(instr))
 320                                                 gather_ubo_ranges(nir, nir_instr_as_intrinsic(instr),
 321                                                                 state, shader->compiler->const_upload_unit);
 322                                 }
 323                         }
 324                 }
 325         }
 326
 327         /* For now, everything we upload is accessed statically and thus will be
 328          * used by the shader. Once we can upload dynamically indexed data, we may
 329          * upload sparsely accessed arrays, at which point we probably want to
 330          * give priority to smaller UBOs, on the assumption that big UBOs will be
 331          * accessed dynamically.  Alternatively, we can track statically and
 332          * dynamically accessed ranges separately and upload static rangtes
 333          * first.
 334          */
 335
 336         /* Limit our uploads to the amount of constant buffer space available in
 337          * the hardware, minus what the shader compiler may need for various
 338          * driver params.  We do this UBO-to-push-constant before the real
 339          * allocation of the driver params' const space, because UBO pointers can
 340          * be driver params but this pass usually eliminatings them.
 341          */
 342         struct ir3_const_state worst_case_const_state = { };
 343         ir3_setup_const_state(shader, nir, &worst_case_const_state);
 344         const uint32_t max_upload = (shader->compiler->max_const -
 345                         worst_case_const_state.offsets.immediate) * 16;
 346
 347         uint32_t offset = shader->num_reserved_user_consts * 16;
 348         state->num_enabled = ARRAY_SIZE(state->range);
 349         for (uint32_t i = 0; i < ARRAY_SIZE(state->range); i++) {
 350                 if (state->range[i].start >= state->range[i].end) {
 351                         state->num_enabled = i;
 352                         break;
 353                 }
 354
 355                 uint32_t range_size = state->range[i].end - state->range[i].start;
 356
 357                 debug_assert(offset <= max_upload);
 358                 state->range[i].offset = offset;
 359                 if (offset + range_size > max_upload) {
 360                         range_size = max_upload - offset;
 361                         state->range[i].end = state->range[i].start + range_size;
 362                 }
 363                 offset += range_size;
 364
 365         }
 366         state->size = offset;
 367
 368         int num_ubos = 0;
 369         nir_foreach_function (function, nir) {
 370                 if (function->impl) {
 371                         nir_builder builder;
 372                         nir_builder_init(&builder, function->impl);
 373                         nir_foreach_block (block, function->impl) {
 374                                 nir_foreach_instr_safe (instr, block) {
 375                                         if (instr_is_load_ubo(instr))
 376                                                 lower_ubo_load_to_uniform(nir_instr_as_intrinsic(instr),
 377                                                                 &builder, state, &num_ubos,
 378                                                                 shader->compiler->const_upload_unit);
 379                                 }
 380                         }
 381
 382                         nir_metadata_preserve(function->impl, nir_metadata_block_index |
 383                                                                   nir_metadata_dominance);
 384                 }
 385         }
 386         /* Update the num_ubos field for GL (first_ubo_is_default_ubo).  With
 387          * Vulkan's bindless, we don't use the num_ubos field, so we can leave it
 388          * incremented.
 389          */
 390         if (nir->info.first_ubo_is_default_ubo)
 391             nir->info.num_ubos = num_ubos;
 392
 393         return state->lower_count > 0;
 394 }