src/glsl/lower_shared_reference.cpp

   1 /*
   2  * Copyright (c) 2015 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  */
  23
  24 /**
  25  * \file lower_shared_reference.cpp
  26  *
  27  * IR lower pass to replace dereferences of compute shader shared variables
  28  * with intrinsic function calls.
  29  *
  30  * This relieves drivers of the responsibility of allocating space for the
  31  * shared variables in the shared memory region.
  32  */
  33
  34 #include "lower_buffer_access.h"
  35 #include "ir_builder.h"
  36 #include "main/macros.h"
  37 #include "util/list.h"
  38 #include "glsl_parser_extras.h"
  39
  40 using namespace ir_builder;
  41
  42 namespace {
  43
  44 struct var_offset {
  45    struct list_head node;
  46    const ir_variable *var;
  47    unsigned offset;
  48 };
  49
  50 class lower_shared_reference_visitor :
  51       public lower_buffer_access::lower_buffer_access {
  52 public:
  53
  54    lower_shared_reference_visitor(struct gl_shader *shader)
  55       : list_ctx(ralloc_context(NULL)), shader(shader), shared_size(0u)
  56    {
  57       list_inithead(&var_offsets);
  58    }
  59
  60    ~lower_shared_reference_visitor()
  61    {
  62       ralloc_free(list_ctx);
  63    }
  64
  65    enum {
  66       shared_load_access,
  67       shared_store_access,
  68       shared_atomic_access,
  69    } buffer_access_type;
  70
  71    void insert_buffer_access(void *mem_ctx, ir_dereference *deref,
  72                              const glsl_type *type, ir_rvalue *offset,
  73                              unsigned mask, int channel);
  74
  75    void handle_rvalue(ir_rvalue **rvalue);
  76    ir_visitor_status visit_enter(ir_assignment *ir);
  77    void handle_assignment(ir_assignment *ir);
  78
  79    unsigned get_shared_offset(const ir_variable *);
  80
  81    ir_call *shared_load(void *mem_ctx, const struct glsl_type *type,
  82                         ir_rvalue *offset);
  83    ir_call *shared_store(void *mem_ctx, ir_rvalue *deref, ir_rvalue *offset,
  84                          unsigned write_mask);
  85
  86    void *list_ctx;
  87    struct gl_shader *shader;
  88    struct list_head var_offsets;
  89    unsigned shared_size;
  90    bool progress;
  91 };
  92
  93 unsigned
  94 lower_shared_reference_visitor::get_shared_offset(const ir_variable *var)
  95 {
  96    list_for_each_entry(var_offset, var_entry, &var_offsets, node) {
  97       if (var_entry->var == var)
  98          return var_entry->offset;
  99    }
 100
 101    struct var_offset *new_entry = rzalloc(list_ctx, struct var_offset);
 102    list_add(&new_entry->node, &var_offsets);
 103    new_entry->var = var;
 104
 105    unsigned var_align = var->type->std430_base_alignment(false);
 106    new_entry->offset = glsl_align(shared_size, var_align);
 107
 108    unsigned var_size = var->type->std430_size(false);
 109    shared_size = new_entry->offset + var_size;
 110
 111    return new_entry->offset;
 112 }
 113
 114 void
 115 lower_shared_reference_visitor::handle_rvalue(ir_rvalue **rvalue)
 116 {
 117    if (!*rvalue)
 118       return;
 119
 120    ir_dereference *deref = (*rvalue)->as_dereference();
 121    if (!deref)
 122       return;
 123
 124    ir_variable *var = deref->variable_referenced();
 125    if (!var || var->data.mode != ir_var_shader_shared)
 126       return;
 127
 128    buffer_access_type = shared_load_access;
 129
 130    void *mem_ctx = ralloc_parent(shader->ir);
 131
 132    ir_rvalue *offset = NULL;
 133    unsigned const_offset = get_shared_offset(var);
 134    bool row_major;
 135    int matrix_columns;
 136    assert(var->get_interface_type() == NULL);
 137    const unsigned packing = GLSL_INTERFACE_PACKING_STD430;
 138
 139    setup_buffer_access(mem_ctx, var, deref,
 140                        &offset, &const_offset,
 141                        &row_major, &matrix_columns, packing);
 142
 143    /* Now that we've calculated the offset to the start of the
 144     * dereference, walk over the type and emit loads into a temporary.
 145     */
 146    const glsl_type *type = (*rvalue)->type;
 147    ir_variable *load_var = new(mem_ctx) ir_variable(type,
 148                                                     "shared_load_temp",
 149                                                     ir_var_temporary);
 150    base_ir->insert_before(load_var);
 151
 152    ir_variable *load_offset = new(mem_ctx) ir_variable(glsl_type::uint_type,
 153                                                        "shared_load_temp_offset",
 154                                                        ir_var_temporary);
 155    base_ir->insert_before(load_offset);
 156    base_ir->insert_before(assign(load_offset, offset));
 157
 158    deref = new(mem_ctx) ir_dereference_variable(load_var);
 159
 160    emit_access(mem_ctx, false, deref, load_offset, const_offset, row_major,
 161                matrix_columns, packing, 0);
 162
 163    *rvalue = deref;
 164
 165    progress = true;
 166 }
 167
 168 void
 169 lower_shared_reference_visitor::handle_assignment(ir_assignment *ir)
 170 {
 171    if (!ir || !ir->lhs)
 172       return;
 173
 174    ir_rvalue *rvalue = ir->lhs->as_rvalue();
 175    if (!rvalue)
 176       return;
 177
 178    ir_dereference *deref = ir->lhs->as_dereference();
 179    if (!deref)
 180       return;
 181
 182    ir_variable *var = ir->lhs->variable_referenced();
 183    if (!var || var->data.mode != ir_var_shader_shared)
 184       return;
 185
 186    buffer_access_type = shared_store_access;
 187
 188    /* We have a write to a shared variable, so declare a temporary and rewrite
 189     * the assignment so that the temporary is the LHS.
 190     */
 191    void *mem_ctx = ralloc_parent(shader->ir);
 192
 193    const glsl_type *type = rvalue->type;
 194    ir_variable *store_var = new(mem_ctx) ir_variable(type,
 195                                                      "shared_store_temp",
 196                                                      ir_var_temporary);
 197    base_ir->insert_before(store_var);
 198    ir->lhs = new(mem_ctx) ir_dereference_variable(store_var);
 199
 200    ir_rvalue *offset = NULL;
 201    unsigned const_offset = get_shared_offset(var);
 202    bool row_major;
 203    int matrix_columns;
 204    assert(var->get_interface_type() == NULL);
 205    const unsigned packing = GLSL_INTERFACE_PACKING_STD430;
 206
 207    setup_buffer_access(mem_ctx, var, deref,
 208                        &offset, &const_offset,
 209                        &row_major, &matrix_columns, packing);
 210
 211    deref = new(mem_ctx) ir_dereference_variable(store_var);
 212
 213    ir_variable *store_offset = new(mem_ctx) ir_variable(glsl_type::uint_type,
 214                                                         "shared_store_temp_offset",
 215                                                         ir_var_temporary);
 216    base_ir->insert_before(store_offset);
 217    base_ir->insert_before(assign(store_offset, offset));
 218
 219    /* Now we have to write the value assigned to the temporary back to memory */
 220    emit_access(mem_ctx, true, deref, store_offset, const_offset, row_major,
 221                matrix_columns, packing, ir->write_mask);
 222
 223    progress = true;
 224 }
 225
 226 ir_visitor_status
 227 lower_shared_reference_visitor::visit_enter(ir_assignment *ir)
 228 {
 229    handle_assignment(ir);
 230    return rvalue_visit(ir);
 231 }
 232
 233 void
 234 lower_shared_reference_visitor::insert_buffer_access(void *mem_ctx,
 235                                                      ir_dereference *deref,
 236                                                      const glsl_type *type,
 237                                                      ir_rvalue *offset,
 238                                                      unsigned mask,
 239                                                      int channel)
 240 {
 241    if (buffer_access_type == shared_store_access) {
 242       ir_call *store = shared_store(mem_ctx, deref, offset, mask);
 243       base_ir->insert_after(store);
 244    } else {
 245       ir_call *load = shared_load(mem_ctx, type, offset);
 246       base_ir->insert_before(load);
 247       ir_rvalue *value = load->return_deref->as_rvalue()->clone(mem_ctx, NULL);
 248       base_ir->insert_before(assign(deref->clone(mem_ctx, NULL),
 249                                     value));
 250    }
 251 }
 252
 253 static bool
 254 compute_shader_enabled(const _mesa_glsl_parse_state *state)
 255 {
 256    return state->stage == MESA_SHADER_COMPUTE;
 257 }
 258
 259 ir_call *
 260 lower_shared_reference_visitor::shared_store(void *mem_ctx,
 261                                              ir_rvalue *deref,
 262                                              ir_rvalue *offset,
 263                                              unsigned write_mask)
 264 {
 265    exec_list sig_params;
 266
 267    ir_variable *offset_ref = new(mem_ctx)
 268       ir_variable(glsl_type::uint_type, "offset" , ir_var_function_in);
 269    sig_params.push_tail(offset_ref);
 270
 271    ir_variable *val_ref = new(mem_ctx)
 272       ir_variable(deref->type, "value" , ir_var_function_in);
 273    sig_params.push_tail(val_ref);
 274
 275    ir_variable *writemask_ref = new(mem_ctx)
 276       ir_variable(glsl_type::uint_type, "write_mask" , ir_var_function_in);
 277    sig_params.push_tail(writemask_ref);
 278
 279    ir_function_signature *sig = new(mem_ctx)
 280       ir_function_signature(glsl_type::void_type, compute_shader_enabled);
 281    assert(sig);
 282    sig->replace_parameters(&sig_params);
 283    sig->is_intrinsic = true;
 284
 285    ir_function *f = new(mem_ctx) ir_function("__intrinsic_store_shared");
 286    f->add_signature(sig);
 287
 288    exec_list call_params;
 289    call_params.push_tail(offset->clone(mem_ctx, NULL));
 290    call_params.push_tail(deref->clone(mem_ctx, NULL));
 291    call_params.push_tail(new(mem_ctx) ir_constant(write_mask));
 292    return new(mem_ctx) ir_call(sig, NULL, &call_params);
 293 }
 294
 295 ir_call *
 296 lower_shared_reference_visitor::shared_load(void *mem_ctx,
 297                                             const struct glsl_type *type,
 298                                             ir_rvalue *offset)
 299 {
 300    exec_list sig_params;
 301
 302    ir_variable *offset_ref = new(mem_ctx)
 303       ir_variable(glsl_type::uint_type, "offset_ref" , ir_var_function_in);
 304    sig_params.push_tail(offset_ref);
 305
 306    ir_function_signature *sig =
 307       new(mem_ctx) ir_function_signature(type, compute_shader_enabled);
 308    assert(sig);
 309    sig->replace_parameters(&sig_params);
 310    sig->is_intrinsic = true;
 311
 312    ir_function *f = new(mem_ctx) ir_function("__intrinsic_load_shared");
 313    f->add_signature(sig);
 314
 315    ir_variable *result = new(mem_ctx)
 316       ir_variable(type, "shared_load_result", ir_var_temporary);
 317    base_ir->insert_before(result);
 318    ir_dereference_variable *deref_result = new(mem_ctx)
 319       ir_dereference_variable(result);
 320
 321    exec_list call_params;
 322    call_params.push_tail(offset->clone(mem_ctx, NULL));
 323
 324    return new(mem_ctx) ir_call(sig, deref_result, &call_params);
 325 }
 326
 327 } /* unnamed namespace */
 328
 329 void
 330 lower_shared_reference(struct gl_shader *shader, unsigned *shared_size)
 331 {
 332    if (shader->Stage != MESA_SHADER_COMPUTE)
 333       return;
 334
 335    lower_shared_reference_visitor v(shader);
 336
 337    /* Loop over the instructions lowering references, because we take a deref
 338     * of an shared variable array using a shared variable dereference as the
 339     * index will produce a collection of instructions all of which have cloned
 340     * shared variable dereferences for that array index.
 341     */
 342    do {
 343       v.progress = false;
 344       visit_list_elements(&v, shader->ir);
 345    } while (v.progress);
 346
 347    *shared_size = v.shared_size;
 348 }