src/intel/vulkan/anv_nir_apply_pipeline_layout.c

   1 /*
   2  * Copyright © 2015 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include "anv_nir.h"
  25 #include "program/prog_parameter.h"
  26 #include "nir/nir_builder.h"
  27
  28 struct apply_pipeline_layout_state {
  29    nir_shader *shader;
  30    nir_builder builder;
  31
  32    struct {
  33       BITSET_WORD *used;
  34       uint8_t *surface_offsets;
  35       uint8_t *sampler_offsets;
  36       uint8_t *image_offsets;
  37    } set[MAX_SETS];
  38 };
  39
  40 static void
  41 add_binding(struct apply_pipeline_layout_state *state,
  42             uint32_t set, uint32_t binding)
  43 {
  44    BITSET_SET(state->set[set].used, binding);
  45 }
  46
  47 static void
  48 add_var_binding(struct apply_pipeline_layout_state *state, nir_variable *var)
  49 {
  50    add_binding(state, var->data.descriptor_set, var->data.binding);
  51 }
  52
  53 static bool
  54 get_used_bindings_block(nir_block *block, void *void_state)
  55 {
  56    struct apply_pipeline_layout_state *state = void_state;
  57
  58    nir_foreach_instr_safe(block, instr) {
  59       switch (instr->type) {
  60       case nir_instr_type_intrinsic: {
  61          nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
  62          switch (intrin->intrinsic) {
  63          case nir_intrinsic_vulkan_resource_index:
  64             add_binding(state, nir_intrinsic_desc_set(intrin),
  65                         nir_intrinsic_binding(intrin));
  66             break;
  67
  68          case nir_intrinsic_image_load:
  69          case nir_intrinsic_image_store:
  70          case nir_intrinsic_image_atomic_add:
  71          case nir_intrinsic_image_atomic_min:
  72          case nir_intrinsic_image_atomic_max:
  73          case nir_intrinsic_image_atomic_and:
  74          case nir_intrinsic_image_atomic_or:
  75          case nir_intrinsic_image_atomic_xor:
  76          case nir_intrinsic_image_atomic_exchange:
  77          case nir_intrinsic_image_atomic_comp_swap:
  78          case nir_intrinsic_image_size:
  79          case nir_intrinsic_image_samples:
  80             add_var_binding(state, intrin->variables[0]->var);
  81             break;
  82
  83          default:
  84             break;
  85          }
  86          break;
  87       }
  88       case nir_instr_type_tex: {
  89          nir_tex_instr *tex = nir_instr_as_tex(instr);
  90          assert(tex->texture);
  91          add_var_binding(state, tex->texture->var);
  92          if (tex->sampler)
  93             add_var_binding(state, tex->sampler->var);
  94          break;
  95       }
  96       default:
  97          continue;
  98       }
  99    }
 100
 101    return true;
 102 }
 103
 104 static void
 105 lower_res_index_intrinsic(nir_intrinsic_instr *intrin,
 106                           struct apply_pipeline_layout_state *state)
 107 {
 108    nir_builder *b = &state->builder;
 109
 110    b->cursor = nir_before_instr(&intrin->instr);
 111
 112    uint32_t set = nir_intrinsic_desc_set(intrin);
 113    uint32_t binding = nir_intrinsic_binding(intrin);
 114
 115    uint32_t surface_index = state->set[set].surface_offsets[binding];
 116
 117    nir_const_value *const_block_idx =
 118       nir_src_as_const_value(intrin->src[0]);
 119
 120    nir_ssa_def *block_index;
 121    if (const_block_idx) {
 122       block_index = nir_imm_int(b, surface_index + const_block_idx->u[0]);
 123    } else {
 124       block_index = nir_iadd(b, nir_imm_int(b, surface_index),
 125                              nir_ssa_for_src(b, intrin->src[0], 1));
 126    }
 127
 128    assert(intrin->dest.is_ssa);
 129    nir_ssa_def_rewrite_uses(&intrin->dest.ssa, nir_src_for_ssa(block_index));
 130    nir_instr_remove(&intrin->instr);
 131 }
 132
 133 static void
 134 lower_tex_deref(nir_tex_instr *tex, nir_deref_var *deref,
 135                 unsigned *const_index, nir_tex_src_type src_type,
 136                 struct apply_pipeline_layout_state *state)
 137 {
 138    if (deref->deref.child) {
 139       assert(deref->deref.child->deref_type == nir_deref_type_array);
 140       nir_deref_array *deref_array = nir_deref_as_array(deref->deref.child);
 141
 142       *const_index += deref_array->base_offset;
 143
 144       if (deref_array->deref_array_type == nir_deref_array_type_indirect) {
 145          nir_tex_src *new_srcs = rzalloc_array(tex, nir_tex_src,
 146                                                tex->num_srcs + 1);
 147
 148          for (unsigned i = 0; i < tex->num_srcs; i++) {
 149             new_srcs[i].src_type = tex->src[i].src_type;
 150             nir_instr_move_src(&tex->instr, &new_srcs[i].src, &tex->src[i].src);
 151          }
 152
 153          ralloc_free(tex->src);
 154          tex->src = new_srcs;
 155
 156          /* Now we can go ahead and move the source over to being a
 157           * first-class texture source.
 158           */
 159          tex->src[tex->num_srcs].src_type = src_type;
 160          tex->num_srcs++;
 161          assert(deref_array->indirect.is_ssa);
 162          nir_instr_rewrite_src(&tex->instr, &tex->src[tex->num_srcs - 1].src,
 163                                deref_array->indirect);
 164       }
 165    }
 166 }
 167
 168 static void
 169 cleanup_tex_deref(nir_tex_instr *tex, nir_deref_var *deref)
 170 {
 171    if (deref->deref.child == NULL)
 172       return;
 173
 174    nir_deref_array *deref_array = nir_deref_as_array(deref->deref.child);
 175
 176    if (deref_array->deref_array_type != nir_deref_array_type_indirect)
 177       return;
 178
 179    nir_instr_rewrite_src(&tex->instr, &deref_array->indirect, NIR_SRC_INIT);
 180 }
 181
 182 static void
 183 lower_tex(nir_tex_instr *tex, struct apply_pipeline_layout_state *state)
 184 {
 185    /* No one should have come by and lowered it already */
 186    assert(tex->texture);
 187
 188    unsigned set = tex->texture->var->data.descriptor_set;
 189    unsigned binding = tex->texture->var->data.binding;
 190    tex->texture_index = state->set[set].surface_offsets[binding];
 191    lower_tex_deref(tex, tex->texture, &tex->texture_index,
 192                    nir_tex_src_texture_offset, state);
 193
 194    if (tex->sampler) {
 195       unsigned set = tex->sampler->var->data.descriptor_set;
 196       unsigned binding = tex->sampler->var->data.binding;
 197       tex->sampler_index = state->set[set].surface_offsets[binding];
 198       lower_tex_deref(tex, tex->sampler, &tex->sampler_index,
 199                       nir_tex_src_sampler_offset, state);
 200    }
 201
 202    /* The backend only ever uses this to mark used surfaces.  We don't care
 203     * about that little optimization so it just needs to be non-zero.
 204     */
 205    tex->texture_array_size = 1;
 206
 207    cleanup_tex_deref(tex, tex->texture);
 208    if (tex->sampler)
 209       cleanup_tex_deref(tex, tex->sampler);
 210    tex->texture = NULL;
 211    tex->sampler = NULL;
 212 }
 213
 214 static bool
 215 apply_pipeline_layout_block(nir_block *block, void *void_state)
 216 {
 217    struct apply_pipeline_layout_state *state = void_state;
 218
 219    nir_foreach_instr_safe(block, instr) {
 220       switch (instr->type) {
 221       case nir_instr_type_intrinsic: {
 222          nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
 223          if (intrin->intrinsic == nir_intrinsic_vulkan_resource_index) {
 224             lower_res_index_intrinsic(intrin, state);
 225          }
 226          break;
 227       }
 228       case nir_instr_type_tex:
 229          lower_tex(nir_instr_as_tex(instr), state);
 230          break;
 231       default:
 232          continue;
 233       }
 234    }
 235
 236    return true;
 237 }
 238
 239 static void
 240 setup_vec4_uniform_value(const union gl_constant_value **params,
 241                          const union gl_constant_value *values,
 242                          unsigned n)
 243 {
 244    static const gl_constant_value zero = { 0 };
 245
 246    for (unsigned i = 0; i < n; ++i)
 247       params[i] = &values[i];
 248
 249    for (unsigned i = n; i < 4; ++i)
 250       params[i] = &zero;
 251 }
 252
 253 void
 254 anv_nir_apply_pipeline_layout(struct anv_pipeline *pipeline,
 255                               nir_shader *shader,
 256                               struct brw_stage_prog_data *prog_data)
 257 {
 258    struct anv_pipeline_layout *layout = pipeline->layout;
 259
 260    struct apply_pipeline_layout_state state = {
 261       .shader = shader,
 262    };
 263
 264    void *mem_ctx = ralloc_context(NULL);
 265
 266    for (unsigned s = 0; s < layout->num_sets; s++) {
 267       const unsigned count = layout->set[s].layout->binding_count;
 268       const unsigned words = BITSET_WORDS(count);
 269       state.set[s].used = rzalloc_array(mem_ctx, BITSET_WORD, words);
 270       state.set[s].surface_offsets = rzalloc_array(mem_ctx, uint8_t, count);
 271       state.set[s].sampler_offsets = rzalloc_array(mem_ctx, uint8_t, count);
 272       state.set[s].image_offsets = rzalloc_array(mem_ctx, uint8_t, count);
 273    }
 274
 275    nir_foreach_function(shader, function) {
 276       if (function->impl)
 277          nir_foreach_block(function->impl, get_used_bindings_block, &state);
 278    }
 279
 280    struct anv_pipeline_bind_map map = {
 281       .surface_count = 0,
 282       .sampler_count = 0,
 283    };
 284
 285    for (uint32_t set = 0; set < layout->num_sets; set++) {
 286       struct anv_descriptor_set_layout *set_layout = layout->set[set].layout;
 287
 288       BITSET_WORD b, _tmp;
 289       BITSET_FOREACH_SET(b, _tmp, state.set[set].used,
 290                          set_layout->binding_count) {
 291          if (set_layout->binding[b].stage[shader->stage].surface_index >= 0)
 292             map.surface_count += set_layout->binding[b].array_size;
 293          if (set_layout->binding[b].stage[shader->stage].sampler_index >= 0)
 294             map.sampler_count += set_layout->binding[b].array_size;
 295          if (set_layout->binding[b].stage[shader->stage].image_index >= 0)
 296             map.image_count += set_layout->binding[b].array_size;
 297       }
 298    }
 299
 300    map.surface_to_descriptor =
 301       malloc(map.surface_count * sizeof(struct anv_pipeline_binding));
 302    map.sampler_to_descriptor =
 303       malloc(map.sampler_count * sizeof(struct anv_pipeline_binding));
 304
 305    pipeline->bindings[shader->stage] = map;
 306
 307    unsigned surface = 0;
 308    unsigned sampler = 0;
 309    unsigned image = 0;
 310    for (uint32_t set = 0; set < layout->num_sets; set++) {
 311       struct anv_descriptor_set_layout *set_layout = layout->set[set].layout;
 312
 313       BITSET_WORD b, _tmp;
 314       BITSET_FOREACH_SET(b, _tmp, state.set[set].used,
 315                          set_layout->binding_count) {
 316          unsigned array_size = set_layout->binding[b].array_size;
 317          unsigned set_offset = set_layout->binding[b].descriptor_index;
 318
 319          if (set_layout->binding[b].stage[shader->stage].surface_index >= 0) {
 320             state.set[set].surface_offsets[b] = surface;
 321             for (unsigned i = 0; i < array_size; i++) {
 322                map.surface_to_descriptor[surface + i].set = set;
 323                map.surface_to_descriptor[surface + i].offset = set_offset + i;
 324             }
 325             surface += array_size;
 326          }
 327
 328          if (set_layout->binding[b].stage[shader->stage].sampler_index >= 0) {
 329             state.set[set].sampler_offsets[b] = sampler;
 330             for (unsigned i = 0; i < array_size; i++) {
 331                map.sampler_to_descriptor[sampler + i].set = set;
 332                map.sampler_to_descriptor[sampler + i].offset = set_offset + i;
 333             }
 334             sampler += array_size;
 335          }
 336
 337          if (set_layout->binding[b].stage[shader->stage].image_index >= 0) {
 338             state.set[set].image_offsets[b] = image;
 339             image += array_size;
 340          }
 341       }
 342    }
 343
 344    nir_foreach_function(shader, function) {
 345       if (function->impl) {
 346          nir_builder_init(&state.builder, function->impl);
 347          nir_foreach_block(function->impl, apply_pipeline_layout_block, &state);
 348          nir_metadata_preserve(function->impl, nir_metadata_block_index |
 349                                                nir_metadata_dominance);
 350       }
 351    }
 352
 353    if (map.image_count > 0) {
 354       nir_foreach_variable(var, &shader->uniforms) {
 355          if (glsl_type_is_image(var->type) ||
 356              (glsl_type_is_array(var->type) &&
 357               glsl_type_is_image(glsl_get_array_element(var->type)))) {
 358             /* Images are represented as uniform push constants and the actual
 359              * information required for reading/writing to/from the image is
 360              * storred in the uniform.
 361              */
 362             unsigned set = var->data.descriptor_set;
 363             unsigned binding = var->data.binding;
 364             unsigned image_index = state.set[set].image_offsets[binding];
 365
 366             var->data.driver_location = shader->num_uniforms +
 367                                         image_index * BRW_IMAGE_PARAM_SIZE * 4;
 368          }
 369       }
 370
 371       struct anv_push_constants *null_data = NULL;
 372       const gl_constant_value **param = prog_data->param + shader->num_uniforms;
 373       const struct brw_image_param *image_param = null_data->images;
 374       for (uint32_t i = 0; i < map.image_count; i++) {
 375          setup_vec4_uniform_value(param + BRW_IMAGE_PARAM_SURFACE_IDX_OFFSET,
 376             (const union gl_constant_value *)&image_param->surface_idx, 1);
 377          setup_vec4_uniform_value(param + BRW_IMAGE_PARAM_OFFSET_OFFSET,
 378             (const union gl_constant_value *)image_param->offset, 2);
 379          setup_vec4_uniform_value(param + BRW_IMAGE_PARAM_SIZE_OFFSET,
 380             (const union gl_constant_value *)image_param->size, 3);
 381          setup_vec4_uniform_value(param + BRW_IMAGE_PARAM_STRIDE_OFFSET,
 382             (const union gl_constant_value *)image_param->stride, 4);
 383          setup_vec4_uniform_value(param + BRW_IMAGE_PARAM_TILING_OFFSET,
 384             (const union gl_constant_value *)image_param->tiling, 3);
 385          setup_vec4_uniform_value(param + BRW_IMAGE_PARAM_SWIZZLING_OFFSET,
 386             (const union gl_constant_value *)image_param->swizzling, 2);
 387
 388          param += BRW_IMAGE_PARAM_SIZE;
 389          image_param ++;
 390       }
 391
 392       shader->num_uniforms += map.image_count * BRW_IMAGE_PARAM_SIZE * 4;
 393    }
 394 }