src/gallium/frontends/clover/nir/invocation.cpp

   1 //
   2 // Copyright 2019 Karol Herbst
   3 //
   4 // Permission is hereby granted, free of charge, to any person obtaining a
   5 // copy of this software and associated documentation files (the "Software"),
   6 // to deal in the Software without restriction, including without limitation
   7 // the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 // and/or sell copies of the Software, and to permit persons to whom the
   9 // Software is furnished to do so, subject to the following conditions:
  10 //
  11 // The above copyright notice and this permission notice shall be included in
  12 // all copies or substantial portions of the Software.
  13 //
  14 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 // THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 // OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 // ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 // OTHER DEALINGS IN THE SOFTWARE.
  21 //
  22
  23 #include "invocation.hpp"
  24
  25 #include <tuple>
  26
  27 #include "core/device.hpp"
  28 #include "core/error.hpp"
  29 #include "core/module.hpp"
  30 #include "pipe/p_state.h"
  31 #include "util/algorithm.hpp"
  32 #include "util/functional.hpp"
  33
  34 #include <compiler/glsl_types.h>
  35 #include <compiler/nir/nir_builder.h>
  36 #include <compiler/nir/nir_serialize.h>
  37 #include <compiler/spirv/nir_spirv.h>
  38 #include <util/u_math.h>
  39
  40 using namespace clover;
  41
  42 #ifdef HAVE_CLOVER_SPIRV
  43
  44 // Refs and unrefs the glsl_type_singleton.
  45 static class glsl_type_ref {
  46 public:
  47    glsl_type_ref() {
  48       glsl_type_singleton_init_or_ref();
  49    }
  50
  51    ~glsl_type_ref() {
  52       glsl_type_singleton_decref();
  53    }
  54 } glsl_type_ref;
  55
  56 static const nir_shader_compiler_options *
  57 dev_get_nir_compiler_options(const device &dev)
  58 {
  59    const void *co = dev.get_compiler_options(PIPE_SHADER_IR_NIR);
  60    return static_cast<const nir_shader_compiler_options*>(co);
  61 }
  62
  63 static void debug_function(void *private_data,
  64                    enum nir_spirv_debug_level level, size_t spirv_offset,
  65                    const char *message)
  66 {
  67    assert(private_data);
  68    auto r_log = reinterpret_cast<std::string *>(private_data);
  69    *r_log += message;
  70 }
  71
  72 struct clover_lower_nir_state {
  73    std::vector<module::argument> &args;
  74    uint32_t global_dims;
  75    nir_variable *offset_vars[3];
  76 };
  77
  78 static bool
  79 clover_lower_nir_filter(const nir_instr *instr, const void *)
  80 {
  81    return instr->type == nir_instr_type_intrinsic;
  82 }
  83
  84 static nir_ssa_def *
  85 clover_lower_nir_instr(nir_builder *b, nir_instr *instr, void *_state)
  86 {
  87    clover_lower_nir_state *state = reinterpret_cast<clover_lower_nir_state*>(_state);
  88    nir_intrinsic_instr *intrinsic = nir_instr_as_intrinsic(instr);
  89
  90    switch (intrinsic->intrinsic) {
  91    case nir_intrinsic_load_base_global_invocation_id: {
  92       nir_ssa_def *loads[3];
  93
  94       /* create variables if we didn't do so alrady */
  95       if (!state->offset_vars[0]) {
  96          /* TODO: fix for 64 bit */
  97          /* Even though we only place one scalar argument, clover will bind up to
  98           * three 32 bit values
  99          */
 100          unsigned location = state->args.size();
 101          state->args.emplace_back(module::argument::scalar, 4, 4, 4,
 102                                   module::argument::zero_ext,
 103                                   module::argument::grid_offset);
 104
 105          const glsl_type *type = glsl_uint_type();
 106          for (uint32_t i = 0; i < 3; i++) {
 107             state->offset_vars[i] =
 108                nir_variable_create(b->shader, nir_var_uniform, type,
 109                                    "global_invocation_id_offsets");
 110             state->offset_vars[i]->data.location = location + i;
 111          }
 112       }
 113
 114       for (int i = 0; i < 3; i++) {
 115          nir_variable *var = state->offset_vars[i];
 116          loads[i] = var ? nir_load_var(b, var) : nir_imm_int(b, 0);
 117       }
 118
 119       return nir_u2u(b, nir_vec(b, loads, state->global_dims),
 120                      nir_dest_bit_size(intrinsic->dest));
 121    }
 122    default:
 123       return NULL;
 124    }
 125 }
 126
 127 static bool
 128 clover_lower_nir(nir_shader *nir, std::vector<module::argument> &args, uint32_t dims)
 129 {
 130    clover_lower_nir_state state = { args, dims };
 131    return nir_shader_lower_instructions(nir,
 132       clover_lower_nir_filter, clover_lower_nir_instr, &state);
 133 }
 134
 135 module clover::nir::spirv_to_nir(const module &mod, const device &dev,
 136                                  std::string &r_log)
 137 {
 138    struct spirv_to_nir_options spirv_options = {};
 139    spirv_options.environment = NIR_SPIRV_OPENCL;
 140    if (dev.address_bits() == 32u) {
 141       spirv_options.shared_addr_format = nir_address_format_32bit_offset;
 142       spirv_options.global_addr_format = nir_address_format_32bit_global;
 143       spirv_options.temp_addr_format = nir_address_format_32bit_offset;
 144       spirv_options.constant_addr_format = nir_address_format_32bit_global;
 145    } else {
 146       spirv_options.shared_addr_format = nir_address_format_32bit_offset_as_64bit;
 147       spirv_options.global_addr_format = nir_address_format_64bit_global;
 148       spirv_options.temp_addr_format = nir_address_format_32bit_offset_as_64bit;
 149       spirv_options.constant_addr_format = nir_address_format_64bit_global;
 150    }
 151    spirv_options.caps.address = true;
 152    spirv_options.caps.float64 = true;
 153    spirv_options.caps.int8 = true;
 154    spirv_options.caps.int16 = true;
 155    spirv_options.caps.int64 = true;
 156    spirv_options.caps.kernel = true;
 157    spirv_options.caps.int64_atomics = dev.has_int64_atomics();
 158    spirv_options.debug.func = &debug_function;
 159    spirv_options.debug.private_data = &r_log;
 160
 161    module m;
 162    // We only insert one section.
 163    assert(mod.secs.size() == 1);
 164    auto &section = mod.secs[0];
 165
 166    module::resource_id section_id = 0;
 167    for (const auto &sym : mod.syms) {
 168       assert(sym.section == 0);
 169
 170       const auto *binary =
 171          reinterpret_cast<const pipe_binary_program_header *>(section.data.data());
 172       const uint32_t *data = reinterpret_cast<const uint32_t *>(binary->blob);
 173       const size_t num_words = binary->num_bytes / 4;
 174       const char *name = sym.name.c_str();
 175       auto *compiler_options = dev_get_nir_compiler_options(dev);
 176
 177       nir_shader *nir = spirv_to_nir(data, num_words, nullptr, 0,
 178                                      MESA_SHADER_KERNEL, name,
 179                                      &spirv_options, compiler_options);
 180       if (!nir) {
 181          r_log += "Translation from SPIR-V to NIR for kernel \"" + sym.name +
 182                   "\" failed.\n";
 183          throw build_error();
 184       }
 185
 186       nir->info.cs.local_size_variable = true;
 187       nir_validate_shader(nir, "clover");
 188
 189       // Inline all functions first.
 190       // according to the comment on nir_inline_functions
 191       NIR_PASS_V(nir, nir_lower_variable_initializers, nir_var_function_temp);
 192       NIR_PASS_V(nir, nir_lower_returns);
 193       NIR_PASS_V(nir, nir_inline_functions);
 194       NIR_PASS_V(nir, nir_copy_prop);
 195       NIR_PASS_V(nir, nir_opt_deref);
 196
 197       // Pick off the single entrypoint that we want.
 198       foreach_list_typed_safe(nir_function, func, node, &nir->functions) {
 199          if (!func->is_entrypoint)
 200             exec_node_remove(&func->node);
 201       }
 202       assert(exec_list_length(&nir->functions) == 1);
 203
 204       nir_validate_shader(nir, "clover after function inlining");
 205
 206       NIR_PASS_V(nir, nir_lower_variable_initializers, ~nir_var_function_temp);
 207
 208       // copy propagate to prepare for lower_explicit_io
 209       NIR_PASS_V(nir, nir_split_var_copies);
 210       NIR_PASS_V(nir, nir_opt_copy_prop_vars);
 211       NIR_PASS_V(nir, nir_lower_var_copies);
 212       NIR_PASS_V(nir, nir_lower_vars_to_ssa);
 213       NIR_PASS_V(nir, nir_opt_dce);
 214
 215       NIR_PASS_V(nir, nir_lower_system_values);
 216       nir_lower_compute_system_values_options sysval_options = { 0 };
 217       sysval_options.has_base_global_invocation_id = true;
 218       NIR_PASS_V(nir, nir_lower_compute_system_values, &sysval_options);
 219
 220       auto args = sym.args;
 221       NIR_PASS_V(nir, clover_lower_nir, args, dev.max_block_size().size());
 222
 223       // Calculate input offsets.
 224       unsigned offset = 0;
 225       nir_foreach_uniform_variable(var, nir) {
 226          offset = align(offset, glsl_get_cl_alignment(var->type));
 227          var->data.driver_location = offset;
 228          offset += glsl_get_cl_size(var->type);
 229       }
 230
 231       NIR_PASS_V(nir, nir_lower_mem_constant_vars,
 232                  glsl_get_cl_type_size_align);
 233       NIR_PASS_V(nir, nir_lower_vars_to_explicit_types,
 234                  nir_var_mem_shared | nir_var_function_temp,
 235                  glsl_get_cl_type_size_align);
 236
 237       /* use offsets for kernel inputs (uniform) */
 238       NIR_PASS_V(nir, nir_lower_explicit_io, nir_var_uniform,
 239                  nir->info.cs.ptr_size == 64 ?
 240                  nir_address_format_32bit_offset_as_64bit :
 241                  nir_address_format_32bit_offset);
 242
 243       NIR_PASS_V(nir, nir_lower_explicit_io, nir_var_mem_constant,
 244                  spirv_options.constant_addr_format);
 245       NIR_PASS_V(nir, nir_lower_explicit_io, nir_var_mem_shared,
 246                  spirv_options.shared_addr_format);
 247
 248       NIR_PASS_V(nir, nir_lower_explicit_io, nir_var_function_temp,
 249                  spirv_options.temp_addr_format);
 250
 251       NIR_PASS_V(nir, nir_lower_explicit_io, nir_var_mem_global,
 252                  spirv_options.global_addr_format);
 253
 254       NIR_PASS_V(nir, nir_remove_dead_variables, nir_var_all, NULL);
 255
 256       if (compiler_options->lower_int64_options)
 257          NIR_PASS_V(nir, nir_lower_int64);
 258
 259       NIR_PASS_V(nir, nir_opt_dce);
 260
 261       struct blob blob;
 262       blob_init(&blob);
 263       nir_serialize(&blob, nir, false);
 264
 265       const pipe_binary_program_header header { uint32_t(blob.size) };
 266       module::section text { section_id, module::section::text_executable, header.num_bytes, {} };
 267       text.data.insert(text.data.end(), reinterpret_cast<const char *>(&header),
 268                        reinterpret_cast<const char *>(&header) + sizeof(header));
 269       text.data.insert(text.data.end(), blob.data, blob.data + blob.size);
 270
 271       m.syms.emplace_back(sym.name, section_id, 0, args);
 272       m.secs.push_back(text);
 273       section_id++;
 274    }
 275    return m;
 276 }
 277 #else
 278 module clover::nir::spirv_to_nir(const module &mod, const device &dev, std::string &r_log)
 279 {
 280    r_log += "SPIR-V support in clover is not enabled.\n";
 281    throw error(CL_LINKER_NOT_AVAILABLE);
 282 }
 283 #endif