src/intel/vulkan/anv_nir_apply_pipeline_layout.c

   1 /*
   2  * Copyright © 2015 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include "anv_nir.h"
  25 #include "program/prog_parameter.h"
  26 #include "nir/nir_builder.h"
  27
  28 struct apply_pipeline_layout_state {
  29    nir_shader *shader;
  30    nir_builder builder;
  31
  32    struct anv_pipeline_layout *layout;
  33    bool add_bounds_checks;
  34
  35    struct {
  36       BITSET_WORD *used;
  37       uint8_t *surface_offsets;
  38       uint8_t *sampler_offsets;
  39       uint8_t *image_offsets;
  40    } set[MAX_SETS];
  41 };
  42
  43 static void
  44 add_binding(struct apply_pipeline_layout_state *state,
  45             uint32_t set, uint32_t binding)
  46 {
  47    BITSET_SET(state->set[set].used, binding);
  48 }
  49
  50 static void
  51 add_var_binding(struct apply_pipeline_layout_state *state, nir_variable *var)
  52 {
  53    add_binding(state, var->data.descriptor_set, var->data.binding);
  54 }
  55
  56 static void
  57 get_used_bindings_block(nir_block *block,
  58                         struct apply_pipeline_layout_state *state)
  59 {
  60    nir_foreach_instr_safe(instr, block) {
  61       switch (instr->type) {
  62       case nir_instr_type_intrinsic: {
  63          nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
  64          switch (intrin->intrinsic) {
  65          case nir_intrinsic_vulkan_resource_index:
  66             add_binding(state, nir_intrinsic_desc_set(intrin),
  67                         nir_intrinsic_binding(intrin));
  68             break;
  69
  70          case nir_intrinsic_image_load:
  71          case nir_intrinsic_image_store:
  72          case nir_intrinsic_image_atomic_add:
  73          case nir_intrinsic_image_atomic_min:
  74          case nir_intrinsic_image_atomic_max:
  75          case nir_intrinsic_image_atomic_and:
  76          case nir_intrinsic_image_atomic_or:
  77          case nir_intrinsic_image_atomic_xor:
  78          case nir_intrinsic_image_atomic_exchange:
  79          case nir_intrinsic_image_atomic_comp_swap:
  80          case nir_intrinsic_image_size:
  81          case nir_intrinsic_image_samples:
  82             add_var_binding(state, intrin->variables[0]->var);
  83             break;
  84
  85          default:
  86             break;
  87          }
  88          break;
  89       }
  90       case nir_instr_type_tex: {
  91          nir_tex_instr *tex = nir_instr_as_tex(instr);
  92          assert(tex->texture);
  93          add_var_binding(state, tex->texture->var);
  94          if (tex->sampler)
  95             add_var_binding(state, tex->sampler->var);
  96          break;
  97       }
  98       default:
  99          continue;
 100       }
 101    }
 102 }
 103
 104 static void
 105 lower_res_index_intrinsic(nir_intrinsic_instr *intrin,
 106                           struct apply_pipeline_layout_state *state)
 107 {
 108    nir_builder *b = &state->builder;
 109
 110    b->cursor = nir_before_instr(&intrin->instr);
 111
 112    uint32_t set = nir_intrinsic_desc_set(intrin);
 113    uint32_t binding = nir_intrinsic_binding(intrin);
 114
 115    uint32_t surface_index = state->set[set].surface_offsets[binding];
 116    uint32_t array_size =
 117       state->layout->set[set].layout->binding[binding].array_size;
 118
 119    nir_ssa_def *block_index = nir_ssa_for_src(b, intrin->src[0], 1);
 120
 121    if (state->add_bounds_checks)
 122       block_index = nir_umin(b, block_index, nir_imm_int(b, array_size - 1));
 123
 124    block_index = nir_iadd(b, nir_imm_int(b, surface_index), block_index);
 125
 126    assert(intrin->dest.is_ssa);
 127    nir_ssa_def_rewrite_uses(&intrin->dest.ssa, nir_src_for_ssa(block_index));
 128    nir_instr_remove(&intrin->instr);
 129 }
 130
 131 static void
 132 lower_tex_deref(nir_tex_instr *tex, nir_deref_var *deref,
 133                 unsigned *const_index, unsigned array_size,
 134                 nir_tex_src_type src_type,
 135                 struct apply_pipeline_layout_state *state)
 136 {
 137    nir_builder *b = &state->builder;
 138
 139    if (deref->deref.child) {
 140       assert(deref->deref.child->deref_type == nir_deref_type_array);
 141       nir_deref_array *deref_array = nir_deref_as_array(deref->deref.child);
 142
 143       if (deref_array->deref_array_type == nir_deref_array_type_indirect) {
 144          nir_ssa_def *index =
 145             nir_iadd(b, nir_imm_int(b, deref_array->base_offset),
 146                         nir_ssa_for_src(b, deref_array->indirect, 1));
 147
 148          if (state->add_bounds_checks)
 149             index = nir_umin(b, index, nir_imm_int(b, array_size - 1));
 150
 151          nir_tex_src *new_srcs = rzalloc_array(tex, nir_tex_src,
 152                                                tex->num_srcs + 1);
 153
 154          for (unsigned i = 0; i < tex->num_srcs; i++) {
 155             new_srcs[i].src_type = tex->src[i].src_type;
 156             nir_instr_move_src(&tex->instr, &new_srcs[i].src, &tex->src[i].src);
 157          }
 158
 159          ralloc_free(tex->src);
 160          tex->src = new_srcs;
 161
 162          /* Now we can go ahead and move the source over to being a
 163           * first-class texture source.
 164           */
 165          tex->src[tex->num_srcs].src_type = src_type;
 166          nir_instr_rewrite_src(&tex->instr, &tex->src[tex->num_srcs].src,
 167                                nir_src_for_ssa(index));
 168          tex->num_srcs++;
 169       } else {
 170          *const_index += MIN2(deref_array->base_offset, array_size - 1);
 171       }
 172    }
 173 }
 174
 175 static void
 176 cleanup_tex_deref(nir_tex_instr *tex, nir_deref_var *deref)
 177 {
 178    if (deref->deref.child == NULL)
 179       return;
 180
 181    nir_deref_array *deref_array = nir_deref_as_array(deref->deref.child);
 182
 183    if (deref_array->deref_array_type != nir_deref_array_type_indirect)
 184       return;
 185
 186    nir_instr_rewrite_src(&tex->instr, &deref_array->indirect, NIR_SRC_INIT);
 187 }
 188
 189 static void
 190 lower_tex(nir_tex_instr *tex, struct apply_pipeline_layout_state *state)
 191 {
 192    /* No one should have come by and lowered it already */
 193    assert(tex->texture);
 194
 195    state->builder.cursor = nir_before_instr(&tex->instr);
 196
 197    unsigned set = tex->texture->var->data.descriptor_set;
 198    unsigned binding = tex->texture->var->data.binding;
 199    unsigned array_size =
 200       state->layout->set[set].layout->binding[binding].array_size;
 201    tex->texture_index = state->set[set].surface_offsets[binding];
 202    lower_tex_deref(tex, tex->texture, &tex->texture_index, array_size,
 203                    nir_tex_src_texture_offset, state);
 204
 205    if (tex->sampler) {
 206       unsigned set = tex->sampler->var->data.descriptor_set;
 207       unsigned binding = tex->sampler->var->data.binding;
 208       unsigned array_size =
 209          state->layout->set[set].layout->binding[binding].array_size;
 210       tex->sampler_index = state->set[set].sampler_offsets[binding];
 211       lower_tex_deref(tex, tex->sampler, &tex->sampler_index, array_size,
 212                       nir_tex_src_sampler_offset, state);
 213    }
 214
 215    /* The backend only ever uses this to mark used surfaces.  We don't care
 216     * about that little optimization so it just needs to be non-zero.
 217     */
 218    tex->texture_array_size = 1;
 219
 220    cleanup_tex_deref(tex, tex->texture);
 221    if (tex->sampler)
 222       cleanup_tex_deref(tex, tex->sampler);
 223    tex->texture = NULL;
 224    tex->sampler = NULL;
 225 }
 226
 227 static void
 228 apply_pipeline_layout_block(nir_block *block,
 229                             struct apply_pipeline_layout_state *state)
 230 {
 231    nir_foreach_instr_safe(instr, block) {
 232       switch (instr->type) {
 233       case nir_instr_type_intrinsic: {
 234          nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
 235          if (intrin->intrinsic == nir_intrinsic_vulkan_resource_index) {
 236             lower_res_index_intrinsic(intrin, state);
 237          }
 238          break;
 239       }
 240       case nir_instr_type_tex:
 241          lower_tex(nir_instr_as_tex(instr), state);
 242          break;
 243       default:
 244          continue;
 245       }
 246    }
 247 }
 248
 249 static void
 250 setup_vec4_uniform_value(const union gl_constant_value **params,
 251                          const union gl_constant_value *values,
 252                          unsigned n)
 253 {
 254    static const gl_constant_value zero = { 0 };
 255
 256    for (unsigned i = 0; i < n; ++i)
 257       params[i] = &values[i];
 258
 259    for (unsigned i = n; i < 4; ++i)
 260       params[i] = &zero;
 261 }
 262
 263 void
 264 anv_nir_apply_pipeline_layout(struct anv_pipeline *pipeline,
 265                               nir_shader *shader,
 266                               struct brw_stage_prog_data *prog_data,
 267                               struct anv_pipeline_bind_map *map)
 268 {
 269    struct anv_pipeline_layout *layout = pipeline->layout;
 270
 271    struct apply_pipeline_layout_state state = {
 272       .shader = shader,
 273       .layout = layout,
 274       .add_bounds_checks = pipeline->device->robust_buffer_access,
 275    };
 276
 277    void *mem_ctx = ralloc_context(NULL);
 278
 279    for (unsigned s = 0; s < layout->num_sets; s++) {
 280       const unsigned count = layout->set[s].layout->binding_count;
 281       const unsigned words = BITSET_WORDS(count);
 282       state.set[s].used = rzalloc_array(mem_ctx, BITSET_WORD, words);
 283       state.set[s].surface_offsets = rzalloc_array(mem_ctx, uint8_t, count);
 284       state.set[s].sampler_offsets = rzalloc_array(mem_ctx, uint8_t, count);
 285       state.set[s].image_offsets = rzalloc_array(mem_ctx, uint8_t, count);
 286    }
 287
 288    nir_foreach_function(function, shader) {
 289       if (!function->impl)
 290          continue;
 291
 292       nir_foreach_block(block, function->impl)
 293          get_used_bindings_block(block, &state);
 294    }
 295
 296    for (uint32_t set = 0; set < layout->num_sets; set++) {
 297       struct anv_descriptor_set_layout *set_layout = layout->set[set].layout;
 298
 299       BITSET_WORD b, _tmp;
 300       BITSET_FOREACH_SET(b, _tmp, state.set[set].used,
 301                          set_layout->binding_count) {
 302          if (set_layout->binding[b].stage[shader->stage].surface_index >= 0)
 303             map->surface_count += set_layout->binding[b].array_size;
 304          if (set_layout->binding[b].stage[shader->stage].sampler_index >= 0)
 305             map->sampler_count += set_layout->binding[b].array_size;
 306          if (set_layout->binding[b].stage[shader->stage].image_index >= 0)
 307             map->image_count += set_layout->binding[b].array_size;
 308       }
 309    }
 310
 311    unsigned surface = 0;
 312    unsigned sampler = 0;
 313    unsigned image = 0;
 314    for (uint32_t set = 0; set < layout->num_sets; set++) {
 315       struct anv_descriptor_set_layout *set_layout = layout->set[set].layout;
 316
 317       BITSET_WORD b, _tmp;
 318       BITSET_FOREACH_SET(b, _tmp, state.set[set].used,
 319                          set_layout->binding_count) {
 320          unsigned array_size = set_layout->binding[b].array_size;
 321          unsigned set_offset = set_layout->binding[b].descriptor_index;
 322
 323          if (set_layout->binding[b].stage[shader->stage].surface_index >= 0) {
 324             state.set[set].surface_offsets[b] = surface;
 325             for (unsigned i = 0; i < array_size; i++) {
 326                map->surface_to_descriptor[surface + i].set = set;
 327                map->surface_to_descriptor[surface + i].offset = set_offset + i;
 328             }
 329             surface += array_size;
 330          }
 331
 332          if (set_layout->binding[b].stage[shader->stage].sampler_index >= 0) {
 333             state.set[set].sampler_offsets[b] = sampler;
 334             for (unsigned i = 0; i < array_size; i++) {
 335                map->sampler_to_descriptor[sampler + i].set = set;
 336                map->sampler_to_descriptor[sampler + i].offset = set_offset + i;
 337             }
 338             sampler += array_size;
 339          }
 340
 341          if (set_layout->binding[b].stage[shader->stage].image_index >= 0) {
 342             state.set[set].image_offsets[b] = image;
 343             image += array_size;
 344          }
 345       }
 346    }
 347
 348    nir_foreach_function(function, shader) {
 349       if (!function->impl)
 350          continue;
 351
 352       nir_builder_init(&state.builder, function->impl);
 353       nir_foreach_block(block, function->impl)
 354          apply_pipeline_layout_block(block, &state);
 355       nir_metadata_preserve(function->impl, nir_metadata_block_index |
 356                                             nir_metadata_dominance);
 357    }
 358
 359    if (map->image_count > 0) {
 360       assert(map->image_count <= MAX_IMAGES);
 361       nir_foreach_variable(var, &shader->uniforms) {
 362          if (glsl_type_is_image(var->type) ||
 363              (glsl_type_is_array(var->type) &&
 364               glsl_type_is_image(glsl_get_array_element(var->type)))) {
 365             /* Images are represented as uniform push constants and the actual
 366              * information required for reading/writing to/from the image is
 367              * storred in the uniform.
 368              */
 369             unsigned set = var->data.descriptor_set;
 370             unsigned binding = var->data.binding;
 371             unsigned image_index = state.set[set].image_offsets[binding];
 372
 373             var->data.driver_location = shader->num_uniforms +
 374                                         image_index * BRW_IMAGE_PARAM_SIZE * 4;
 375          }
 376       }
 377
 378       struct anv_push_constants *null_data = NULL;
 379       const gl_constant_value **param =
 380          prog_data->param + (shader->num_uniforms / 4);
 381       const struct brw_image_param *image_param = null_data->images;
 382       for (uint32_t i = 0; i < map->image_count; i++) {
 383          setup_vec4_uniform_value(param + BRW_IMAGE_PARAM_SURFACE_IDX_OFFSET,
 384             (const union gl_constant_value *)&image_param->surface_idx, 1);
 385          setup_vec4_uniform_value(param + BRW_IMAGE_PARAM_OFFSET_OFFSET,
 386             (const union gl_constant_value *)image_param->offset, 2);
 387          setup_vec4_uniform_value(param + BRW_IMAGE_PARAM_SIZE_OFFSET,
 388             (const union gl_constant_value *)image_param->size, 3);
 389          setup_vec4_uniform_value(param + BRW_IMAGE_PARAM_STRIDE_OFFSET,
 390             (const union gl_constant_value *)image_param->stride, 4);
 391          setup_vec4_uniform_value(param + BRW_IMAGE_PARAM_TILING_OFFSET,
 392             (const union gl_constant_value *)image_param->tiling, 3);
 393          setup_vec4_uniform_value(param + BRW_IMAGE_PARAM_SWIZZLING_OFFSET,
 394             (const union gl_constant_value *)image_param->swizzling, 2);
 395
 396          param += BRW_IMAGE_PARAM_SIZE;
 397          image_param ++;
 398       }
 399
 400       shader->num_uniforms += map->image_count * BRW_IMAGE_PARAM_SIZE * 4;
 401    }
 402
 403    ralloc_free(mem_ctx);
 404 }