src/gallium/frontends/clover/nir/invocation.cpp

   1 //
   2 // Copyright 2019 Karol Herbst
   3 //
   4 // Permission is hereby granted, free of charge, to any person obtaining a
   5 // copy of this software and associated documentation files (the "Software"),
   6 // to deal in the Software without restriction, including without limitation
   7 // the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 // and/or sell copies of the Software, and to permit persons to whom the
   9 // Software is furnished to do so, subject to the following conditions:
  10 //
  11 // The above copyright notice and this permission notice shall be included in
  12 // all copies or substantial portions of the Software.
  13 //
  14 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 // THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 // OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 // ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 // OTHER DEALINGS IN THE SOFTWARE.
  21 //
  22
  23 #include "invocation.hpp"
  24
  25 #include <tuple>
  26
  27 #include "core/device.hpp"
  28 #include "core/error.hpp"
  29 #include "core/module.hpp"
  30 #include "pipe/p_state.h"
  31 #include "util/algorithm.hpp"
  32 #include "util/functional.hpp"
  33
  34 #include <compiler/glsl_types.h>
  35 #include <compiler/nir/nir_builder.h>
  36 #include <compiler/nir/nir_serialize.h>
  37 #include <compiler/spirv/nir_spirv.h>
  38 #include <util/u_math.h>
  39
  40 using namespace clover;
  41
  42 #ifdef HAVE_CLOVER_SPIRV
  43
  44 // Refs and unrefs the glsl_type_singleton.
  45 static class glsl_type_ref {
  46 public:
  47    glsl_type_ref() {
  48       glsl_type_singleton_init_or_ref();
  49    }
  50
  51    ~glsl_type_ref() {
  52       glsl_type_singleton_decref();
  53    }
  54 } glsl_type_ref;
  55
  56 static const nir_shader_compiler_options *
  57 dev_get_nir_compiler_options(const device &dev)
  58 {
  59    const void *co = dev.get_compiler_options(PIPE_SHADER_IR_NIR);
  60    return static_cast<const nir_shader_compiler_options*>(co);
  61 }
  62
  63 static void debug_function(void *private_data,
  64                    enum nir_spirv_debug_level level, size_t spirv_offset,
  65                    const char *message)
  66 {
  67    assert(private_data);
  68    auto r_log = reinterpret_cast<std::string *>(private_data);
  69    *r_log += message;
  70 }
  71
  72 struct clover_lower_nir_state {
  73    std::vector<module::argument> &args;
  74    uint32_t global_dims;
  75    nir_variable *offset_vars[3];
  76 };
  77
  78 static bool
  79 clover_lower_nir_filter(const nir_instr *instr, const void *)
  80 {
  81    return instr->type == nir_instr_type_intrinsic;
  82 }
  83
  84 static nir_ssa_def *
  85 clover_lower_nir_instr(nir_builder *b, nir_instr *instr, void *_state)
  86 {
  87    clover_lower_nir_state *state = reinterpret_cast<clover_lower_nir_state*>(_state);
  88    nir_intrinsic_instr *intrinsic = nir_instr_as_intrinsic(instr);
  89
  90    switch (intrinsic->intrinsic) {
  91    case nir_intrinsic_load_base_global_invocation_id: {
  92       nir_ssa_def *loads[3];
  93
  94       /* create variables if we didn't do so alrady */
  95       if (!state->offset_vars[0]) {
  96          /* TODO: fix for 64 bit */
  97          /* Even though we only place one scalar argument, clover will bind up to
  98           * three 32 bit values
  99          */
 100          state->args.emplace_back(module::argument::scalar, 4, 4, 4,
 101                                   module::argument::zero_ext,
 102                                   module::argument::grid_offset);
 103
 104          const glsl_type *type = glsl_uint_type();
 105          for (uint32_t i = 0; i < 3; i++) {
 106             state->offset_vars[i] =
 107                nir_variable_create(b->shader, nir_var_uniform, type,
 108                                    "global_invocation_id_offsets");
 109             state->offset_vars[i]->data.location = b->shader->num_uniforms++;
 110          }
 111       }
 112
 113       for (int i = 0; i < 3; i++) {
 114          nir_variable *var = state->offset_vars[i];
 115          loads[i] = var ? nir_load_var(b, var) : nir_imm_int(b, 0);
 116       }
 117
 118       return nir_u2u(b, nir_vec(b, loads, state->global_dims),
 119                      nir_dest_bit_size(intrinsic->dest));
 120    }
 121    default:
 122       return NULL;
 123    }
 124 }
 125
 126 static bool
 127 clover_lower_nir(nir_shader *nir, std::vector<module::argument> &args, uint32_t dims)
 128 {
 129    clover_lower_nir_state state = { args, dims };
 130    return nir_shader_lower_instructions(nir,
 131       clover_lower_nir_filter, clover_lower_nir_instr, &state);
 132 }
 133
 134 module clover::nir::spirv_to_nir(const module &mod, const device &dev,
 135                                  std::string &r_log)
 136 {
 137    struct spirv_to_nir_options spirv_options = {};
 138    spirv_options.environment = NIR_SPIRV_OPENCL;
 139    if (dev.address_bits() == 32u) {
 140       spirv_options.shared_addr_format = nir_address_format_32bit_offset;
 141       spirv_options.global_addr_format = nir_address_format_32bit_global;
 142       spirv_options.temp_addr_format = nir_address_format_32bit_offset;
 143       spirv_options.constant_addr_format = nir_address_format_32bit_global;
 144    } else {
 145       spirv_options.shared_addr_format = nir_address_format_32bit_offset_as_64bit;
 146       spirv_options.global_addr_format = nir_address_format_64bit_global;
 147       spirv_options.temp_addr_format = nir_address_format_32bit_offset_as_64bit;
 148       spirv_options.constant_addr_format = nir_address_format_64bit_global;
 149    }
 150    spirv_options.caps.address = true;
 151    spirv_options.caps.float64 = true;
 152    spirv_options.caps.int8 = true;
 153    spirv_options.caps.int16 = true;
 154    spirv_options.caps.int64 = true;
 155    spirv_options.caps.kernel = true;
 156    spirv_options.caps.int64_atomics = dev.has_int64_atomics();
 157    spirv_options.debug.func = &debug_function;
 158    spirv_options.debug.private_data = &r_log;
 159
 160    module m;
 161    // We only insert one section.
 162    assert(mod.secs.size() == 1);
 163    auto &section = mod.secs[0];
 164
 165    module::resource_id section_id = 0;
 166    for (const auto &sym : mod.syms) {
 167       assert(sym.section == 0);
 168
 169       const auto *binary =
 170          reinterpret_cast<const pipe_binary_program_header *>(section.data.data());
 171       const uint32_t *data = reinterpret_cast<const uint32_t *>(binary->blob);
 172       const size_t num_words = binary->num_bytes / 4;
 173       const char *name = sym.name.c_str();
 174       auto *compiler_options = dev_get_nir_compiler_options(dev);
 175
 176       nir_shader *nir = spirv_to_nir(data, num_words, nullptr, 0,
 177                                      MESA_SHADER_KERNEL, name,
 178                                      &spirv_options, compiler_options);
 179       if (!nir) {
 180          r_log += "Translation from SPIR-V to NIR for kernel \"" + sym.name +
 181                   "\" failed.\n";
 182          throw build_error();
 183       }
 184
 185       nir->info.cs.local_size_variable = true;
 186       nir_validate_shader(nir, "clover");
 187
 188       // Inline all functions first.
 189       // according to the comment on nir_inline_functions
 190       NIR_PASS_V(nir, nir_lower_variable_initializers, nir_var_function_temp);
 191       NIR_PASS_V(nir, nir_lower_returns);
 192       NIR_PASS_V(nir, nir_inline_functions);
 193       NIR_PASS_V(nir, nir_copy_prop);
 194       NIR_PASS_V(nir, nir_opt_deref);
 195
 196       // Pick off the single entrypoint that we want.
 197       foreach_list_typed_safe(nir_function, func, node, &nir->functions) {
 198          if (!func->is_entrypoint)
 199             exec_node_remove(&func->node);
 200       }
 201       assert(exec_list_length(&nir->functions) == 1);
 202
 203       nir_validate_shader(nir, "clover after function inlining");
 204
 205       NIR_PASS_V(nir, nir_lower_variable_initializers, ~nir_var_function_temp);
 206
 207       // copy propagate to prepare for lower_explicit_io
 208       NIR_PASS_V(nir, nir_split_var_copies);
 209       NIR_PASS_V(nir, nir_opt_copy_prop_vars);
 210       NIR_PASS_V(nir, nir_lower_var_copies);
 211       NIR_PASS_V(nir, nir_lower_vars_to_ssa);
 212       NIR_PASS_V(nir, nir_opt_dce);
 213
 214       NIR_PASS_V(nir, nir_lower_system_values);
 215       nir_lower_compute_system_values_options sysval_options = { 0 };
 216       sysval_options.has_base_global_invocation_id = true;
 217       NIR_PASS_V(nir, nir_lower_compute_system_values, &sysval_options);
 218
 219       auto args = sym.args;
 220       NIR_PASS_V(nir, clover_lower_nir, args, dev.max_block_size().size());
 221
 222       // Calculate input offsets.
 223       unsigned offset = 0;
 224       nir_foreach_uniform_variable(var, nir) {
 225          offset = align(offset, glsl_get_cl_alignment(var->type));
 226          var->data.driver_location = offset;
 227          offset += glsl_get_cl_size(var->type);
 228       }
 229
 230       NIR_PASS_V(nir, nir_lower_mem_constant_vars,
 231                  glsl_get_cl_type_size_align);
 232       NIR_PASS_V(nir, nir_lower_vars_to_explicit_types,
 233                  nir_var_mem_shared | nir_var_function_temp,
 234                  glsl_get_cl_type_size_align);
 235
 236       /* use offsets for kernel inputs (uniform) */
 237       NIR_PASS_V(nir, nir_lower_explicit_io, nir_var_uniform,
 238                  nir->info.cs.ptr_size == 64 ?
 239                  nir_address_format_32bit_offset_as_64bit :
 240                  nir_address_format_32bit_offset);
 241
 242       NIR_PASS_V(nir, nir_lower_explicit_io, nir_var_mem_constant,
 243                  spirv_options.constant_addr_format);
 244       NIR_PASS_V(nir, nir_lower_explicit_io, nir_var_mem_shared,
 245                  spirv_options.shared_addr_format);
 246
 247       NIR_PASS_V(nir, nir_lower_explicit_io, nir_var_function_temp,
 248                  spirv_options.temp_addr_format);
 249
 250       NIR_PASS_V(nir, nir_lower_explicit_io, nir_var_mem_global,
 251                  spirv_options.global_addr_format);
 252
 253       NIR_PASS_V(nir, nir_remove_dead_variables, nir_var_all, NULL);
 254
 255       if (compiler_options->lower_int64_options)
 256          NIR_PASS_V(nir, nir_lower_int64);
 257
 258       NIR_PASS_V(nir, nir_opt_dce);
 259
 260       struct blob blob;
 261       blob_init(&blob);
 262       nir_serialize(&blob, nir, false);
 263
 264       const pipe_binary_program_header header { uint32_t(blob.size) };
 265       module::section text { section_id, module::section::text_executable, header.num_bytes, {} };
 266       text.data.insert(text.data.end(), reinterpret_cast<const char *>(&header),
 267                        reinterpret_cast<const char *>(&header) + sizeof(header));
 268       text.data.insert(text.data.end(), blob.data, blob.data + blob.size);
 269
 270       m.syms.emplace_back(sym.name, section_id, 0, args);
 271       m.secs.push_back(text);
 272       section_id++;
 273    }
 274    return m;
 275 }
 276 #else
 277 module clover::nir::spirv_to_nir(const module &mod, const device &dev, std::string &r_log)
 278 {
 279    r_log += "SPIR-V support in clover is not enabled.\n";
 280    throw error(CL_LINKER_NOT_AVAILABLE);
 281 }
 282 #endif