src/intel/vulkan/anv_nir_apply_pipeline_layout.c

   1 /*
   2  * Copyright © 2015 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include "anv_nir.h"
  25 #include "program/prog_parameter.h"
  26 #include "nir/nir_builder.h"
  27
  28 struct apply_pipeline_layout_state {
  29    nir_shader *shader;
  30    nir_builder builder;
  31
  32    struct {
  33       BITSET_WORD *used;
  34       uint8_t *surface_offsets;
  35       uint8_t *sampler_offsets;
  36       uint8_t *image_offsets;
  37    } set[MAX_SETS];
  38 };
  39
  40 static void
  41 add_binding(struct apply_pipeline_layout_state *state,
  42             uint32_t set, uint32_t binding)
  43 {
  44    BITSET_SET(state->set[set].used, binding);
  45 }
  46
  47 static void
  48 add_var_binding(struct apply_pipeline_layout_state *state, nir_variable *var)
  49 {
  50    add_binding(state, var->data.descriptor_set, var->data.binding);
  51 }
  52
  53 static bool
  54 get_used_bindings_block(nir_block *block, void *void_state)
  55 {
  56    struct apply_pipeline_layout_state *state = void_state;
  57
  58    nir_foreach_instr_safe(block, instr) {
  59       switch (instr->type) {
  60       case nir_instr_type_intrinsic: {
  61          nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
  62          switch (intrin->intrinsic) {
  63          case nir_intrinsic_vulkan_resource_index:
  64             add_binding(state, nir_intrinsic_desc_set(intrin),
  65                         nir_intrinsic_binding(intrin));
  66             break;
  67
  68          case nir_intrinsic_image_load:
  69          case nir_intrinsic_image_store:
  70          case nir_intrinsic_image_atomic_add:
  71          case nir_intrinsic_image_atomic_min:
  72          case nir_intrinsic_image_atomic_max:
  73          case nir_intrinsic_image_atomic_and:
  74          case nir_intrinsic_image_atomic_or:
  75          case nir_intrinsic_image_atomic_xor:
  76          case nir_intrinsic_image_atomic_exchange:
  77          case nir_intrinsic_image_atomic_comp_swap:
  78          case nir_intrinsic_image_size:
  79          case nir_intrinsic_image_samples:
  80             add_var_binding(state, intrin->variables[0]->var);
  81             break;
  82
  83          default:
  84             break;
  85          }
  86          break;
  87       }
  88       case nir_instr_type_tex: {
  89          nir_tex_instr *tex = nir_instr_as_tex(instr);
  90          assert(tex->texture);
  91          add_var_binding(state, tex->texture->var);
  92          if (tex->sampler)
  93             add_var_binding(state, tex->sampler->var);
  94          break;
  95       }
  96       default:
  97          continue;
  98       }
  99    }
 100
 101    return true;
 102 }
 103
 104 static void
 105 lower_res_index_intrinsic(nir_intrinsic_instr *intrin,
 106                           struct apply_pipeline_layout_state *state)
 107 {
 108    nir_builder *b = &state->builder;
 109
 110    b->cursor = nir_before_instr(&intrin->instr);
 111
 112    uint32_t set = nir_intrinsic_desc_set(intrin);
 113    uint32_t binding = nir_intrinsic_binding(intrin);
 114
 115    uint32_t surface_index = state->set[set].surface_offsets[binding];
 116
 117    nir_const_value *const_block_idx =
 118       nir_src_as_const_value(intrin->src[0]);
 119
 120    nir_ssa_def *block_index;
 121    if (const_block_idx) {
 122       block_index = nir_imm_int(b, surface_index + const_block_idx->u[0]);
 123    } else {
 124       block_index = nir_iadd(b, nir_imm_int(b, surface_index),
 125                              nir_ssa_for_src(b, intrin->src[0], 1));
 126    }
 127
 128    assert(intrin->dest.is_ssa);
 129    nir_ssa_def_rewrite_uses(&intrin->dest.ssa, nir_src_for_ssa(block_index));
 130    nir_instr_remove(&intrin->instr);
 131 }
 132
 133 static void
 134 lower_tex_deref(nir_tex_instr *tex, nir_deref_var *deref,
 135                 unsigned *const_index, nir_tex_src_type src_type,
 136                 struct apply_pipeline_layout_state *state)
 137 {
 138    if (deref->deref.child) {
 139       assert(deref->deref.child->deref_type == nir_deref_type_array);
 140       nir_deref_array *deref_array = nir_deref_as_array(deref->deref.child);
 141
 142       *const_index += deref_array->base_offset;
 143
 144       if (deref_array->deref_array_type == nir_deref_array_type_indirect) {
 145          nir_tex_src *new_srcs = rzalloc_array(tex, nir_tex_src,
 146                                                tex->num_srcs + 1);
 147
 148          for (unsigned i = 0; i < tex->num_srcs; i++) {
 149             new_srcs[i].src_type = tex->src[i].src_type;
 150             nir_instr_move_src(&tex->instr, &new_srcs[i].src, &tex->src[i].src);
 151          }
 152
 153          ralloc_free(tex->src);
 154          tex->src = new_srcs;
 155
 156          /* Now we can go ahead and move the source over to being a
 157           * first-class texture source.
 158           */
 159          tex->src[tex->num_srcs].src_type = src_type;
 160          tex->num_srcs++;
 161          assert(deref_array->indirect.is_ssa);
 162          nir_instr_rewrite_src(&tex->instr, &tex->src[tex->num_srcs - 1].src,
 163                                deref_array->indirect);
 164       }
 165    }
 166 }
 167
 168 static void
 169 cleanup_tex_deref(nir_tex_instr *tex, nir_deref_var *deref)
 170 {
 171    if (deref->deref.child == NULL)
 172       return;
 173
 174    nir_deref_array *deref_array = nir_deref_as_array(deref->deref.child);
 175
 176    if (deref_array->deref_array_type != nir_deref_array_type_indirect)
 177       return;
 178
 179    nir_instr_rewrite_src(&tex->instr, &deref_array->indirect, NIR_SRC_INIT);
 180 }
 181
 182 static void
 183 lower_tex(nir_tex_instr *tex, struct apply_pipeline_layout_state *state)
 184 {
 185    /* No one should have come by and lowered it already */
 186    assert(tex->texture);
 187
 188    unsigned set = tex->texture->var->data.descriptor_set;
 189    unsigned binding = tex->texture->var->data.binding;
 190    tex->texture_index = state->set[set].surface_offsets[binding];
 191    lower_tex_deref(tex, tex->texture, &tex->texture_index,
 192                    nir_tex_src_texture_offset, state);
 193
 194    if (tex->sampler) {
 195       unsigned set = tex->sampler->var->data.descriptor_set;
 196       unsigned binding = tex->sampler->var->data.binding;
 197       tex->sampler_index = state->set[set].surface_offsets[binding];
 198       lower_tex_deref(tex, tex->sampler, &tex->sampler_index,
 199                       nir_tex_src_sampler_offset, state);
 200    }
 201
 202    /* The backend only ever uses this to mark used surfaces.  We don't care
 203     * about that little optimization so it just needs to be non-zero.
 204     */
 205    tex->texture_array_size = 1;
 206
 207    cleanup_tex_deref(tex, tex->texture);
 208    if (tex->sampler)
 209       cleanup_tex_deref(tex, tex->sampler);
 210    tex->texture = NULL;
 211    tex->sampler = NULL;
 212 }
 213
 214 static bool
 215 apply_pipeline_layout_block(nir_block *block, void *void_state)
 216 {
 217    struct apply_pipeline_layout_state *state = void_state;
 218
 219    nir_foreach_instr_safe(block, instr) {
 220       switch (instr->type) {
 221       case nir_instr_type_intrinsic: {
 222          nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
 223          if (intrin->intrinsic == nir_intrinsic_vulkan_resource_index) {
 224             lower_res_index_intrinsic(intrin, state);
 225          }
 226          break;
 227       }
 228       case nir_instr_type_tex:
 229          lower_tex(nir_instr_as_tex(instr), state);
 230          break;
 231       default:
 232          continue;
 233       }
 234    }
 235
 236    return true;
 237 }
 238
 239 static void
 240 setup_vec4_uniform_value(const union gl_constant_value **params,
 241                          const union gl_constant_value *values,
 242                          unsigned n)
 243 {
 244    static const gl_constant_value zero = { 0 };
 245
 246    for (unsigned i = 0; i < n; ++i)
 247       params[i] = &values[i];
 248
 249    for (unsigned i = n; i < 4; ++i)
 250       params[i] = &zero;
 251 }
 252
 253 void
 254 anv_nir_apply_pipeline_layout(struct anv_pipeline *pipeline,
 255                               nir_shader *shader,
 256                               struct brw_stage_prog_data *prog_data)
 257 {
 258    struct anv_pipeline_layout *layout = pipeline->layout;
 259
 260    struct apply_pipeline_layout_state state = {
 261       .shader = shader,
 262    };
 263
 264    void *mem_ctx = ralloc_context(NULL);
 265
 266    for (unsigned s = 0; s < layout->num_sets; s++) {
 267       const unsigned count = layout->set[s].layout->binding_count;
 268       const unsigned words = BITSET_WORDS(count);
 269       state.set[s].used = rzalloc_array(mem_ctx, BITSET_WORD, words);
 270       state.set[s].surface_offsets = rzalloc_array(mem_ctx, uint8_t, count);
 271       state.set[s].sampler_offsets = rzalloc_array(mem_ctx, uint8_t, count);
 272       state.set[s].image_offsets = rzalloc_array(mem_ctx, uint8_t, count);
 273    }
 274
 275    nir_foreach_function(shader, function) {
 276       if (function->impl)
 277          nir_foreach_block(function->impl, get_used_bindings_block, &state);
 278    }
 279
 280    struct anv_pipeline_bind_map map = {
 281       .surface_count = 0,
 282       .sampler_count = 0,
 283       .image_count = 0,
 284    };
 285
 286    for (uint32_t set = 0; set < layout->num_sets; set++) {
 287       struct anv_descriptor_set_layout *set_layout = layout->set[set].layout;
 288
 289       BITSET_WORD b, _tmp;
 290       BITSET_FOREACH_SET(b, _tmp, state.set[set].used,
 291                          set_layout->binding_count) {
 292          if (set_layout->binding[b].stage[shader->stage].surface_index >= 0)
 293             map.surface_count += set_layout->binding[b].array_size;
 294          if (set_layout->binding[b].stage[shader->stage].sampler_index >= 0)
 295             map.sampler_count += set_layout->binding[b].array_size;
 296          if (set_layout->binding[b].stage[shader->stage].image_index >= 0)
 297             map.image_count += set_layout->binding[b].array_size;
 298       }
 299    }
 300
 301    map.surface_to_descriptor =
 302       malloc(map.surface_count * sizeof(struct anv_pipeline_binding));
 303    map.sampler_to_descriptor =
 304       malloc(map.sampler_count * sizeof(struct anv_pipeline_binding));
 305
 306    pipeline->bindings[shader->stage] = map;
 307
 308    unsigned surface = 0;
 309    unsigned sampler = 0;
 310    unsigned image = 0;
 311    for (uint32_t set = 0; set < layout->num_sets; set++) {
 312       struct anv_descriptor_set_layout *set_layout = layout->set[set].layout;
 313
 314       BITSET_WORD b, _tmp;
 315       BITSET_FOREACH_SET(b, _tmp, state.set[set].used,
 316                          set_layout->binding_count) {
 317          unsigned array_size = set_layout->binding[b].array_size;
 318          unsigned set_offset = set_layout->binding[b].descriptor_index;
 319
 320          if (set_layout->binding[b].stage[shader->stage].surface_index >= 0) {
 321             state.set[set].surface_offsets[b] = surface;
 322             for (unsigned i = 0; i < array_size; i++) {
 323                map.surface_to_descriptor[surface + i].set = set;
 324                map.surface_to_descriptor[surface + i].offset = set_offset + i;
 325             }
 326             surface += array_size;
 327          }
 328
 329          if (set_layout->binding[b].stage[shader->stage].sampler_index >= 0) {
 330             state.set[set].sampler_offsets[b] = sampler;
 331             for (unsigned i = 0; i < array_size; i++) {
 332                map.sampler_to_descriptor[sampler + i].set = set;
 333                map.sampler_to_descriptor[sampler + i].offset = set_offset + i;
 334             }
 335             sampler += array_size;
 336          }
 337
 338          if (set_layout->binding[b].stage[shader->stage].image_index >= 0) {
 339             state.set[set].image_offsets[b] = image;
 340             image += array_size;
 341          }
 342       }
 343    }
 344
 345    nir_foreach_function(shader, function) {
 346       if (function->impl) {
 347          nir_builder_init(&state.builder, function->impl);
 348          nir_foreach_block(function->impl, apply_pipeline_layout_block, &state);
 349          nir_metadata_preserve(function->impl, nir_metadata_block_index |
 350                                                nir_metadata_dominance);
 351       }
 352    }
 353
 354    if (map.image_count > 0) {
 355       assert(map.image_count <= MAX_IMAGES);
 356       nir_foreach_variable(var, &shader->uniforms) {
 357          if (glsl_type_is_image(var->type) ||
 358              (glsl_type_is_array(var->type) &&
 359               glsl_type_is_image(glsl_get_array_element(var->type)))) {
 360             /* Images are represented as uniform push constants and the actual
 361              * information required for reading/writing to/from the image is
 362              * storred in the uniform.
 363              */
 364             unsigned set = var->data.descriptor_set;
 365             unsigned binding = var->data.binding;
 366             unsigned image_index = state.set[set].image_offsets[binding];
 367
 368             var->data.driver_location = shader->num_uniforms +
 369                                         image_index * BRW_IMAGE_PARAM_SIZE * 4;
 370          }
 371       }
 372
 373       struct anv_push_constants *null_data = NULL;
 374       const gl_constant_value **param =
 375          prog_data->param + (shader->num_uniforms / 4);
 376       const struct brw_image_param *image_param = null_data->images;
 377       for (uint32_t i = 0; i < map.image_count; i++) {
 378          setup_vec4_uniform_value(param + BRW_IMAGE_PARAM_SURFACE_IDX_OFFSET,
 379             (const union gl_constant_value *)&image_param->surface_idx, 1);
 380          setup_vec4_uniform_value(param + BRW_IMAGE_PARAM_OFFSET_OFFSET,
 381             (const union gl_constant_value *)image_param->offset, 2);
 382          setup_vec4_uniform_value(param + BRW_IMAGE_PARAM_SIZE_OFFSET,
 383             (const union gl_constant_value *)image_param->size, 3);
 384          setup_vec4_uniform_value(param + BRW_IMAGE_PARAM_STRIDE_OFFSET,
 385             (const union gl_constant_value *)image_param->stride, 4);
 386          setup_vec4_uniform_value(param + BRW_IMAGE_PARAM_TILING_OFFSET,
 387             (const union gl_constant_value *)image_param->tiling, 3);
 388          setup_vec4_uniform_value(param + BRW_IMAGE_PARAM_SWIZZLING_OFFSET,
 389             (const union gl_constant_value *)image_param->swizzling, 2);
 390
 391          param += BRW_IMAGE_PARAM_SIZE;
 392          image_param ++;
 393       }
 394
 395       shader->num_uniforms += map.image_count * BRW_IMAGE_PARAM_SIZE * 4;
 396    }
 397
 398    ralloc_free(mem_ctx);
 399 }