src/gallium/drivers/r600/sfn/sfn_nir_lower_fs_out_to_vector.cpp

   1 /* -*- mesa-c++  -*-
   2  *
   3  * Copyright (c) 2019 Collabora LTD
   4  *
   5  * Author: Gert Wollny <gert.wollny@collabora.com>
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the "Software"),
   9  * to deal in the Software without restriction, including without limitation
  10  * on the rights to use, copy, modify, merge, publish, distribute, sub
  11  * license, and/or sell copies of the Software, and to permit persons to whom
  12  * the Software is furnished to do so, subject to the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the next
  15  * paragraph) shall be included in all copies or substantial portions of the
  16  * Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  21  * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
  22  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  23  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  24  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  */
  26
  27 #include "sfn_nir_lower_fs_out_to_vector.h"
  28
  29 #include "nir_builder.h"
  30 #include "nir_deref.h"
  31 #include "util/u_math.h"
  32
  33 #include <set>
  34 #include <vector>
  35 #include <array>
  36 #include <algorithm>
  37
  38 namespace r600 {
  39
  40 using std::multiset;
  41 using std::vector;
  42 using std::array;
  43
  44 struct nir_intrinsic_instr_less  {
  45    bool operator () (const nir_intrinsic_instr *lhs, const nir_intrinsic_instr *rhs) const
  46    {
  47       nir_variable *vlhs = nir_deref_instr_get_variable(nir_src_as_deref(lhs->src[0]));
  48       nir_variable *vrhs = nir_deref_instr_get_variable(nir_src_as_deref(rhs->src[0]));
  49
  50       auto ltype = glsl_get_base_type(vlhs->type);
  51       auto rtype = glsl_get_base_type(vrhs->type);
  52
  53       if (ltype != rtype)
  54          return ltype < rtype;
  55       return vlhs->data.location < vrhs->data.location;
  56    }
  57 };
  58
  59 class NirLowerIOToVector {
  60 public:
  61    NirLowerIOToVector(int base_slot);
  62    bool run(nir_function_impl *shader);
  63
  64 protected:
  65    bool var_can_merge(const nir_variable *lhs, const nir_variable *rhs);
  66    bool var_can_rewrite(nir_variable *var) const;
  67    void create_new_io_vars(nir_shader *shader);
  68    void create_new_io_var(nir_shader *shader, unsigned location, unsigned comps);
  69
  70    nir_deref_instr *clone_deref_array(nir_builder *b, nir_deref_instr *dst_tail,
  71                                       const nir_deref_instr *src_head);
  72
  73    bool vectorize_block(nir_builder *b, nir_block *block);
  74    bool instr_can_rewrite(nir_instr *instr);
  75    bool vec_instr_set_remove(nir_builder *b,nir_instr *instr);
  76
  77    using InstrSet  = multiset<nir_intrinsic_instr *, nir_intrinsic_instr_less>;
  78    using InstrSubSet = std::pair<InstrSet::iterator, InstrSet::iterator>;
  79
  80    bool vec_instr_stack_pop(nir_builder *b, InstrSubSet& ir_set,
  81                             nir_intrinsic_instr *instr);
  82
  83    array<array<nir_variable *, 4>, 16> m_vars;
  84    InstrSet m_block_io;
  85    int m_next_index;
  86 private:
  87    virtual exec_list *get_io_list(nir_shader *shader) const  = 0;
  88    virtual bool instr_can_rewrite_type(nir_intrinsic_instr *intr) const  = 0;
  89    virtual bool var_can_rewrite_slot(nir_variable *var) const = 0;
  90    virtual void create_new_io(nir_builder *b, nir_intrinsic_instr *intr, nir_variable *var,
  91                               nir_ssa_def **srcs, unsigned first_comp, unsigned num_comps) = 0;
  92
  93    int m_base_slot;
  94 };
  95
  96 class NirLowerFSOutToVector : public NirLowerIOToVector {
  97 public:
  98    NirLowerFSOutToVector();
  99
 100 private:
 101    exec_list *get_io_list(nir_shader *shader) const  override;
 102    bool var_can_rewrite_slot(nir_variable *var) const override;
 103    void create_new_io(nir_builder *b, nir_intrinsic_instr *intr, nir_variable *var,
 104                          nir_ssa_def **srcs, unsigned first_comp, unsigned num_comps) override;
 105    bool instr_can_rewrite_type(nir_intrinsic_instr *intr) const  override;
 106
 107    nir_ssa_def *create_combined_vector(nir_builder *b, nir_ssa_def **srcs,
 108                                        int first_comp, int num_comp);
 109 };
 110
 111 bool r600_lower_fs_out_to_vector(nir_shader *shader)
 112 {
 113    NirLowerFSOutToVector processor;
 114
 115    assert(shader->info.stage == MESA_SHADER_FRAGMENT);
 116    bool progress = false;
 117
 118    nir_foreach_function(function, shader) {
 119       if (function->impl)
 120          progress |= processor.run(function->impl);
 121    }
 122    return progress;
 123 }
 124
 125 NirLowerIOToVector::NirLowerIOToVector(int base_slot):
 126    m_next_index(0),
 127    m_base_slot(base_slot)
 128 {
 129    for(auto& a : m_vars)
 130       for(auto& aa : a)
 131          aa = nullptr;
 132 }
 133
 134 bool NirLowerIOToVector::run(nir_function_impl *impl)
 135 {
 136    nir_builder b;
 137    nir_builder_init(&b, impl);
 138
 139    nir_metadata_require(impl, nir_metadata_dominance);
 140    create_new_io_vars(impl->function->shader);
 141
 142    bool progress = vectorize_block(&b, nir_start_block(impl));
 143    if (progress) {
 144       nir_metadata_preserve(impl, (nir_metadata )
 145                             (nir_metadata_block_index |
 146                              nir_metadata_dominance));
 147    }
 148    return progress;
 149 }
 150
 151 void NirLowerIOToVector::create_new_io_vars(nir_shader *shader)
 152 {
 153    struct exec_list *io_list = get_io_list(shader);
 154    if (exec_list_is_empty(io_list))
 155       return;
 156
 157    nir_foreach_variable(var, io_list) {
 158       if (var_can_rewrite(var)) {
 159          unsigned loc = var->data.location - m_base_slot;
 160          m_vars[loc][var->data.location_frac] = var;
 161       }
 162    }
 163
 164    /* We don't handle combining vars of different type e.g. different array
 165     * lengths.
 166     */
 167    for (unsigned i = 0; i < 16; i++) {
 168       unsigned comps = 0;
 169
 170       for (unsigned j = 0; j < 3; j++) {
 171          if (!m_vars[i][j])
 172             continue;
 173
 174          for (unsigned k = j + 1; k < 4; k++) {
 175             if (!m_vars[i][k])
 176                continue;
 177
 178             if (!var_can_merge(m_vars[i][j], m_vars[i][k]))
 179                continue;
 180
 181             /* Set comps */
 182             for (unsigned n = 0; n < glsl_get_components(m_vars[i][j]->type); ++n)
 183                comps |= 1 << (m_vars[i][j]->data.location_frac + n);
 184
 185             for (unsigned n = 0; n < glsl_get_components(m_vars[i][k]->type); ++n)
 186                comps |= 1 << (m_vars[i][k]->data.location_frac + n);
 187
 188          }
 189       }
 190       if (comps)
 191          create_new_io_var(shader, i, comps);
 192    }
 193 }
 194
 195 bool
 196 NirLowerIOToVector::var_can_merge(const nir_variable *lhs,
 197                                      const nir_variable *rhs)
 198 {
 199    return (glsl_get_base_type(lhs->type) == glsl_get_base_type(rhs->type));
 200 }
 201
 202 void
 203 NirLowerIOToVector::create_new_io_var(nir_shader *shader,
 204                                     unsigned location, unsigned comps)
 205 {
 206    unsigned num_comps = util_bitcount(comps);
 207    assert(num_comps > 1);
 208
 209    /* Note: u_bit_scan() strips a component of the comps bitfield here */
 210    unsigned first_comp = u_bit_scan(&comps);
 211
 212    nir_variable *var = nir_variable_clone(m_vars[location][first_comp], shader);
 213    var->data.location_frac = first_comp;
 214    var->type = glsl_replace_vector_type(var->type, num_comps);
 215
 216    nir_shader_add_variable(shader, var);
 217
 218    m_vars[location][first_comp] = var;
 219
 220    while (comps) {
 221       const int comp = u_bit_scan(&comps);
 222       if (m_vars[location][comp]) {
 223          m_vars[location][comp] = var;
 224       }
 225    }
 226 }
 227
 228 bool NirLowerIOToVector::var_can_rewrite(nir_variable *var) const
 229 {
 230    /* Skip complex types we don't split in the first place */
 231    if (!glsl_type_is_vector_or_scalar(glsl_without_array(var->type)))
 232       return false;
 233
 234    if (glsl_get_bit_size(glsl_without_array(var->type)) != 32)
 235       return false;
 236
 237    return var_can_rewrite_slot(var);
 238 }
 239
 240 bool
 241 NirLowerIOToVector::vectorize_block(nir_builder *b, nir_block *block)
 242 {
 243    bool progress = false;
 244
 245    nir_foreach_instr_safe(instr, block) {
 246       if (instr_can_rewrite(instr)) {
 247          instr->index = m_next_index++;
 248          nir_intrinsic_instr *ir = nir_instr_as_intrinsic(instr);
 249          m_block_io.insert(ir);
 250       }
 251    }
 252
 253    for (unsigned i = 0; i < block->num_dom_children; i++) {
 254       nir_block *child = block->dom_children[i];
 255       progress |= vectorize_block(b, child);
 256    }
 257
 258    nir_foreach_instr_reverse_safe(instr, block) {
 259       progress |= vec_instr_set_remove(b, instr);
 260    }
 261    m_block_io.clear();
 262
 263    return progress;
 264 }
 265
 266 bool NirLowerIOToVector::instr_can_rewrite(nir_instr *instr)
 267 {
 268    if (instr->type != nir_instr_type_intrinsic)
 269       return false;
 270
 271    nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
 272
 273    if (intr->num_components > 3)
 274       return false;
 275
 276    return instr_can_rewrite_type(intr);
 277 }
 278
 279 bool NirLowerIOToVector::vec_instr_set_remove(nir_builder *b,nir_instr *instr)
 280 {
 281    if (!instr_can_rewrite(instr))
 282       return false;
 283
 284    nir_intrinsic_instr *ir = nir_instr_as_intrinsic(instr);
 285    auto entry = m_block_io.equal_range(ir);
 286    if (entry.first != m_block_io.end()) {
 287       vec_instr_stack_pop(b, entry, ir);
 288    }
 289    return true;
 290 }
 291
 292 nir_deref_instr *
 293 NirLowerIOToVector::clone_deref_array(nir_builder *b, nir_deref_instr *dst_tail,
 294                                     const nir_deref_instr *src_head)
 295 {
 296    const nir_deref_instr *parent = nir_deref_instr_parent(src_head);
 297
 298    if (!parent)
 299       return dst_tail;
 300
 301    assert(src_head->deref_type == nir_deref_type_array);
 302
 303    dst_tail = clone_deref_array(b, dst_tail, parent);
 304
 305    return nir_build_deref_array(b, dst_tail,
 306                                 nir_ssa_for_src(b, src_head->arr.index, 1));
 307 }
 308
 309 NirLowerFSOutToVector::NirLowerFSOutToVector():
 310   NirLowerIOToVector(FRAG_RESULT_COLOR)
 311 {
 312
 313 }
 314
 315 bool NirLowerFSOutToVector::var_can_rewrite_slot(nir_variable *var) const
 316 {
 317    return ((var->data.mode == nir_var_shader_out) &&
 318            ((var->data.location == FRAG_RESULT_COLOR) ||
 319               ((var->data.location >= FRAG_RESULT_DATA0) &&
 320                (var->data.location <= FRAG_RESULT_DATA7))));
 321 }
 322
 323 bool NirLowerIOToVector::vec_instr_stack_pop(nir_builder *b, InstrSubSet &ir_set,
 324                                            nir_intrinsic_instr *instr)
 325 {
 326    vector< nir_intrinsic_instr *> ir_sorted_set(ir_set.first, ir_set.second);
 327    std::sort(ir_sorted_set.begin(), ir_sorted_set.end(),
 328              [](const nir_intrinsic_instr *lhs, const nir_intrinsic_instr *rhs) {
 329                   return lhs->instr.index > rhs->instr.index;
 330              }
 331    );
 332
 333    nir_intrinsic_instr *intr = *ir_sorted_set.begin();
 334    nir_variable *var = nir_deref_instr_get_variable(nir_src_as_deref(intr->src[0]));
 335
 336    unsigned loc = var->data.location - m_base_slot;
 337
 338    nir_variable *new_var = m_vars[loc][var->data.location_frac];
 339    unsigned num_comps = glsl_get_vector_elements(glsl_without_array(new_var->type));
 340    unsigned old_num_comps = glsl_get_vector_elements(glsl_without_array(var->type));
 341
 342    /* Don't bother walking the stack if this component can't be vectorised. */
 343    if (old_num_comps > 3) {
 344       return false;
 345    }
 346
 347    if (new_var == var) {
 348       return false;
 349    }
 350
 351    b->cursor = nir_after_instr(&intr->instr);
 352    nir_ssa_undef_instr *instr_undef =
 353       nir_ssa_undef_instr_create(b->shader, 1, 32);
 354    nir_builder_instr_insert(b, &instr_undef->instr);
 355
 356    nir_ssa_def *srcs[4];
 357    for (int i = 0; i < 4; i++) {
 358       srcs[i] = &instr_undef->def;
 359    }
 360    srcs[var->data.location_frac] = intr->src[1].ssa;
 361
 362    for (auto k = ir_sorted_set.begin() + 1; k != ir_sorted_set.end(); ++k) {
 363       nir_intrinsic_instr *intr2 = *k;
 364       nir_variable *var2 =
 365          nir_deref_instr_get_variable(nir_src_as_deref(intr2->src[0]));
 366       unsigned loc2 = var->data.location - m_base_slot;
 367
 368       if (m_vars[loc][var->data.location_frac] !=
 369           m_vars[loc2][var2->data.location_frac]) {
 370          continue;
 371       }
 372
 373      assert(glsl_get_vector_elements(glsl_without_array(var2->type)) < 4);
 374
 375       if (srcs[var2->data.location_frac] == &instr_undef->def) {
 376          assert(intr2->src[1].is_ssa);
 377          assert(intr2->src[1].ssa);
 378          srcs[var2->data.location_frac] = intr2->src[1].ssa;
 379       }
 380       nir_instr_remove(&intr2->instr);
 381    }
 382
 383    create_new_io(b, intr, new_var, srcs, new_var->data.location_frac,
 384                  num_comps);
 385    return true;
 386 }
 387
 388 exec_list *NirLowerFSOutToVector::get_io_list(nir_shader *shader) const
 389 {
 390    return &shader->outputs;
 391 }
 392
 393 void
 394 NirLowerFSOutToVector::create_new_io(nir_builder *b, nir_intrinsic_instr *intr, nir_variable *var,
 395                                         nir_ssa_def **srcs, unsigned first_comp, unsigned num_comps)
 396 {
 397    b->cursor = nir_before_instr(&intr->instr);
 398
 399    nir_intrinsic_instr *new_intr =
 400       nir_intrinsic_instr_create(b->shader, intr->intrinsic);
 401    new_intr->num_components = num_comps;
 402
 403    nir_intrinsic_set_write_mask(new_intr, (1 << num_comps) - 1);
 404
 405    nir_deref_instr *deref = nir_build_deref_var(b, var);
 406    deref = clone_deref_array(b, deref, nir_src_as_deref(intr->src[0]));
 407
 408    new_intr->src[0] = nir_src_for_ssa(&deref->dest.ssa);
 409    new_intr->src[1] = nir_src_for_ssa(create_combined_vector(b, srcs, first_comp, num_comps));
 410
 411    nir_builder_instr_insert(b, &new_intr->instr);
 412
 413    /* Remove the old store intrinsic */
 414    nir_instr_remove(&intr->instr);
 415 }
 416
 417 bool NirLowerFSOutToVector::instr_can_rewrite_type(nir_intrinsic_instr *intr) const
 418 {
 419    if (intr->intrinsic != nir_intrinsic_store_deref)
 420       return false;
 421
 422    nir_deref_instr *deref = nir_src_as_deref(intr->src[0]);
 423    if (deref->mode != nir_var_shader_out)
 424       return false;
 425
 426    return var_can_rewrite(nir_deref_instr_get_variable(deref));
 427 }
 428
 429 nir_ssa_def *NirLowerFSOutToVector::create_combined_vector(nir_builder *b, nir_ssa_def **srcs,
 430                                                            int first_comp, int num_comp)
 431 {
 432    nir_op op;
 433    switch (num_comp) {
 434    case 2: op = nir_op_vec2; break;
 435    case 3: op = nir_op_vec3; break;
 436    case 4: op = nir_op_vec4; break;
 437    default:
 438       assert(0 && "combined vector must have 2 to 4 components");
 439
 440    }
 441    nir_alu_instr * instr = nir_alu_instr_create(b->shader, op);
 442    instr->exact = b->exact;
 443
 444    int i = 0;
 445    unsigned k = 0;
 446    while (i < num_comp) {
 447       nir_ssa_def *s = srcs[first_comp + k];
 448       for(uint8_t kk = 0; kk < s->num_components && i < num_comp; ++kk) {
 449          instr->src[i].src  = nir_src_for_ssa(s);
 450          instr->src[i].swizzle[0] = kk;
 451          ++i;
 452       }
 453       k += s->num_components;
 454    }
 455
 456    nir_ssa_dest_init(&instr->instr, &instr->dest.dest, num_comp, 32, NULL);
 457    instr->dest.write_mask = (1 << num_comp) - 1;
 458    nir_builder_instr_insert(b, &instr->instr);
 459    return &instr->dest.dest.ssa;
 460 }
 461
 462 }