src/gallium/drivers/r600/sfn/sfn_valuepool.cpp

   1 /* -*- mesa-c++  -*-
   2  *
   3  * Copyright (c) 2018-2019 Collabora LTD
   4  *
   5  * Author: Gert Wollny <gert.wollny@collabora.com>
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the "Software"),
   9  * to deal in the Software without restriction, including without limitation
  10  * on the rights to use, copy, modify, merge, publish, distribute, sub
  11  * license, and/or sell copies of the Software, and to permit persons to whom
  12  * the Software is furnished to do so, subject to the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the next
  15  * paragraph) shall be included in all copies or substantial portions of the
  16  * Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  21  * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
  22  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  23  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  24  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  */
  26
  27 #include "sfn_debug.h"
  28 #include "sfn_value_gpr.h"
  29 #include "sfn_valuepool.h"
  30
  31 #include <iostream>
  32 #include <queue>
  33
  34 namespace r600 {
  35
  36 using std::vector;
  37 using std::pair;
  38 using std::make_pair;
  39 using std::queue;
  40
  41 ValuePool::ValuePool():
  42    m_next_register_index(0),
  43    current_temp_reg_index(0),
  44    next_temp_reg_comp(4)
  45 {
  46 }
  47
  48 PValue ValuePool::m_undef = Value::zero;
  49
  50 GPRVector ValuePool::vec_from_nir(const nir_dest& dst, int num_components)
  51 {
  52    std::array<PValue, 4> result;
  53    for (int i = 0; i < 4; ++i)
  54       result[i] = from_nir(dst, i < num_components ? i : 7);
  55    return GPRVector(result);
  56
  57 }
  58
  59 PValue ValuePool::from_nir(const nir_src& v, unsigned component, unsigned swizzled)
  60 {
  61    sfn_log << SfnLog::reg << "Search " << (v.is_ssa ? "ssa_reg " : "reg ")
  62            << (v.is_ssa ? v.ssa->index : v.reg.reg->index);
  63
  64    if (!v.is_ssa) {
  65       int idx = lookup_register_index(v);
  66       sfn_log << SfnLog::reg << "  -> got index " <<  idx << "\n";
  67       if (idx >= 0) {
  68          auto reg = lookup_register(idx, swizzled, false);
  69          if (reg) {
  70             if (reg->type() == Value::gpr_vector) {
  71                auto& array = static_cast<GPRArray&>(*reg);
  72                reg = array.get_indirect(v.reg.base_offset,
  73                                         v.reg.indirect ?
  74                                            from_nir(*v.reg.indirect, 0, 0) : nullptr,
  75                                         component);
  76             }
  77             return reg;
  78          }
  79       }
  80       assert(0 && "local registers should always be found");
  81    }
  82
  83    unsigned index = v.ssa->index;
  84    /* For undefs we use zero and let ()yet to be implemeneted dce deal with it */
  85    if (m_ssa_undef.find(index) != m_ssa_undef.end())
  86       return Value::zero;
  87
  88
  89    int idx = lookup_register_index(v);
  90    sfn_log << SfnLog::reg << "  -> got index " <<  idx << "\n";
  91    if (idx >= 0) {
  92       auto reg = lookup_register(idx, swizzled, false);
  93       if (reg)
  94          return reg;
  95    }
  96
  97
  98    auto literal_val = m_literal_constants.find(index);
  99    if (literal_val != m_literal_constants.end()) {
 100       switch (literal_val->second->def.bit_size) {
 101       case 1:
 102          return PValue(new LiteralValue(literal_val->second->value[swizzled].b ? 0xffffffff : 0, component));
 103       case 32:
 104          return literal(literal_val->second->value[swizzled].u32);
 105       default:
 106          sfn_log << SfnLog::reg << "Unsupported bit size " << literal_val->second->def.bit_size
 107                  << " fall back to 32\n";
 108          return PValue(new LiteralValue(literal_val->second->value[swizzled].u32, component));
 109       }
 110    }
 111
 112    unsigned uindex = (index << 2) + swizzled;
 113    auto u = m_uniforms.find(uindex);
 114    if (u != m_uniforms.end())
 115       return u->second;
 116
 117    return PValue();
 118 }
 119
 120 PValue ValuePool::from_nir(const nir_src& v, unsigned component)
 121 {
 122    return from_nir(v, component, component);
 123 }
 124
 125 PValue ValuePool::from_nir(const nir_tex_src &v, unsigned component)
 126 {
 127    return from_nir(v.src, component, component);
 128 }
 129
 130 PValue ValuePool::from_nir(const nir_alu_src &v, unsigned component)
 131 {
 132    return from_nir(v.src, component, v.swizzle[component]);
 133 }
 134
 135 PValue ValuePool::get_temp_register()
 136 {
 137    if (next_temp_reg_comp > 3) {
 138       current_temp_reg_index = allocate_temp_register();
 139       next_temp_reg_comp = 0;
 140    }
 141    return PValue(new GPRValue(current_temp_reg_index, next_temp_reg_comp++));
 142 }
 143
 144 GPRVector ValuePool::get_temp_vec4()
 145 {
 146    int sel = allocate_temp_register();
 147    return GPRVector(sel, {0,1,2,3});
 148 }
 149
 150 PValue ValuePool::create_register_from_nir_src(const nir_src& src, int comp)
 151 {
 152    int idx = src.is_ssa ? get_dst_ssa_register_index(*src.ssa):
 153                           get_local_register_index(*src.reg.reg);
 154
 155    auto retval = lookup_register(idx, comp, false);
 156    if (!retval)
 157       retval = create_register(idx, comp);
 158    return retval;
 159 }
 160
 161 PValue ValuePool::from_nir(const nir_alu_dest &v, unsigned component)
 162 {
 163    //assert(v->write_mask & (1 << component));
 164    return from_nir(v.dest, component);
 165 }
 166
 167 int ValuePool::lookup_register_index(const nir_dest& dst)
 168 {
 169    return dst.is_ssa ? get_dst_ssa_register_index(dst.ssa):
 170                        get_local_register_index(*dst.reg.reg);
 171 }
 172
 173 int ValuePool::lookup_register_index(const nir_src& src) const
 174 {
 175    int index = 0;
 176
 177    index = src.is_ssa ?
 178               get_ssa_register_index(*src.ssa) :
 179               get_local_register_index(*src.reg.reg);
 180
 181    sfn_log << SfnLog::reg << " LIDX:" << index;
 182
 183    auto r = m_register_map.find(index);
 184    if (r == m_register_map.end()) {
 185       return -1;
 186    }
 187    return static_cast<int>(r->second.index);
 188 }
 189
 190
 191 int ValuePool::allocate_component(unsigned index, unsigned comp, bool pre_alloc)
 192 {
 193    assert(comp < 8);
 194    return allocate_with_mask(index, 1 << comp, pre_alloc);
 195 }
 196
 197 int ValuePool::allocate_temp_register()
 198 {
 199    return m_next_register_index++;
 200 }
 201
 202
 203 PValue ValuePool::from_nir(const nir_dest& v, unsigned component)
 204 {
 205    int idx = lookup_register_index(v);
 206    sfn_log << SfnLog::reg << __func__  << ": ";
 207    if (v.is_ssa)
 208       sfn_log << "ssa_" << v.ssa.index;
 209    else
 210       sfn_log << "r" << v.reg.reg->index;
 211    sfn_log << " -> " << idx << "\n";
 212
 213    auto retval = lookup_register(idx, component, false);
 214    if (!retval)
 215       retval = create_register(idx, component);
 216
 217    if (retval->type() == Value::gpr_vector) {
 218       assert(!v.is_ssa);
 219       auto& array = static_cast<GPRArray&>(*retval);
 220       retval = array.get_indirect(v.reg.base_offset,
 221                                   v.reg.indirect ?
 222                                   from_nir(*v.reg.indirect, 0, 0) : nullptr,
 223                                   component);
 224    }
 225
 226    return retval;
 227 }
 228
 229 ValueMap ValuePool::get_temp_registers() const
 230 {
 231    ValueMap result;
 232
 233    for (auto& v : m_registers) {
 234       if (v.second->type() == Value::gpr)
 235          result.insert(v.second);
 236       else if (v.second->type() == Value::gpr_vector) {
 237          auto& array = static_cast<GPRArray&>(*v.second);
 238          array.collect_registers(result);
 239       }
 240    }
 241    return result;
 242 }
 243
 244 static const char swz[] = "xyzw01?_";
 245
 246 PValue ValuePool::create_register(unsigned sel, unsigned swizzle)
 247 {
 248    sfn_log << SfnLog::reg
 249            <<"Create register " << sel  << '.' << swz[swizzle] << "\n";
 250    auto retval = PValue(new GPRValue(sel, swizzle));
 251    m_registers[(sel << 3) + swizzle] = retval;
 252    return retval;
 253 }
 254
 255 bool ValuePool::inject_register(unsigned sel, unsigned swizzle,
 256                                 const PValue& reg, bool map)
 257 {
 258    uint32_t ssa_index = sel;
 259
 260    if (map) {
 261       auto pos = m_ssa_register_map.find(sel);
 262       if (pos == m_ssa_register_map.end())
 263          ssa_index = m_next_register_index++;
 264       else
 265          ssa_index = pos->second;
 266    }
 267
 268    sfn_log << SfnLog::reg
 269            << "Inject register " << sel  << '.' << swz[swizzle]
 270            << " at index " <<  ssa_index << " ...";
 271
 272    if (map)
 273       m_ssa_register_map[sel] = ssa_index;
 274
 275    allocate_with_mask(ssa_index, swizzle, true);
 276
 277    unsigned idx = (ssa_index << 3) + swizzle;
 278    auto p = m_registers.find(idx);
 279    if ( (p != m_registers.end()) && *p->second != *reg) {
 280       std::cerr << "Register location (" << ssa_index << ", " << swizzle << ") was already reserved\n";
 281       assert(0);
 282       return false;
 283    }
 284    sfn_log << SfnLog::reg << " at idx:" << idx << " to " << *reg << "\n";
 285    m_registers[idx] = reg;
 286
 287    if (m_next_register_index <= ssa_index)
 288       m_next_register_index = ssa_index + 1;
 289    return true;
 290 }
 291
 292
 293 PValue ValuePool::lookup_register(unsigned sel, unsigned swizzle,
 294                                   bool required)
 295 {
 296
 297    PValue retval;
 298    sfn_log << SfnLog::reg
 299            << "lookup register " << sel  << '.' << swz[swizzle] << "("
 300            << ((sel << 3) + swizzle) << ")...";
 301
 302
 303    auto reg = m_registers.find((sel << 3) + swizzle);
 304    if (reg != m_registers.end()) {
 305       sfn_log << SfnLog::reg << " -> Found " << *reg->second << "\n";
 306       retval = reg->second;
 307    } else if (swizzle == 7) {
 308       PValue retval = create_register(sel, swizzle);
 309       sfn_log << SfnLog::reg << " -> Created " << *retval << "\n";
 310    } else if (required) {
 311       sfn_log << SfnLog::reg << "Register (" << sel << ", "
 312               << swizzle << ") not found but required\n";
 313       assert(0 && "Unallocated register value requested\n");
 314    }
 315    sfn_log << SfnLog::reg << " -> Not required and not  allocated\n";
 316    return retval;
 317 }
 318
 319 unsigned ValuePool::get_dst_ssa_register_index(const nir_ssa_def& ssa)
 320 {
 321    sfn_log << SfnLog::reg << __func__ << ": search dst ssa "
 322            << ssa.index;
 323
 324    auto pos = m_ssa_register_map.find(ssa.index);
 325    if (pos == m_ssa_register_map.end()) {
 326       sfn_log << SfnLog::reg << " Need to allocate ...";
 327       allocate_ssa_register(ssa);
 328       pos = m_ssa_register_map.find(ssa.index);
 329       assert(pos != m_ssa_register_map.end());
 330    }
 331    sfn_log << SfnLog::reg << "... got " << pos->second << "\n";
 332    return pos->second;
 333 }
 334
 335 unsigned ValuePool::get_ssa_register_index(const nir_ssa_def& ssa) const
 336 {
 337    sfn_log << SfnLog::reg << __func__ << ": search ssa "
 338            << ssa.index;
 339
 340    auto pos = m_ssa_register_map.find(ssa.index);
 341    sfn_log << SfnLog::reg << " got " << pos->second<< "\n";
 342    if (pos == m_ssa_register_map.end()) {
 343       sfn_log << SfnLog::reg << __func__ << ": ssa register "
 344               << ssa.index << " lookup failed\n";
 345       return -1;
 346    }
 347    return pos->second;
 348 }
 349
 350 unsigned ValuePool::get_local_register_index(const nir_register& reg)
 351 {
 352    auto pos = m_local_register_map.find(reg.index);
 353    if (pos == m_local_register_map.end()) {
 354       allocate_local_register(reg);
 355       pos = m_local_register_map.find(reg.index);
 356       assert(pos != m_local_register_map.end());
 357    }
 358    return pos->second;
 359 }
 360
 361 unsigned ValuePool::get_local_register_index(const nir_register& reg) const
 362 {
 363    auto pos = m_local_register_map.find(reg.index);
 364    if (pos == m_local_register_map.end()) {
 365       sfn_log << SfnLog::err << __func__ << ": local register "
 366               << reg.index << " lookup failed";
 367       return -1;
 368    }
 369    return pos->second;
 370 }
 371
 372 void ValuePool::allocate_ssa_register(const nir_ssa_def& ssa)
 373 {
 374    sfn_log << SfnLog::reg << "ValuePool: Allocate ssa register " << ssa.index
 375            << " as " << m_next_register_index << "\n";
 376    int index = m_next_register_index++;
 377    m_ssa_register_map[ssa.index] = index;
 378    allocate_with_mask(index, 0xf, true);
 379 }
 380
 381 void ValuePool::allocate_arrays(array_list& arrays)
 382 {
 383    int ncomponents = 0;
 384    int current_index = m_next_register_index;
 385    unsigned instance = 0;
 386
 387    while (!arrays.empty()) {
 388       auto a = arrays.top();
 389       arrays.pop();
 390
 391       /* This is a bit hackish, return an id that encodes the array merge. To make sure
 392        * that the mapping doesn't go wrong we have to make sure the arrays is longer than
 393        * the number of instances in this arrays slot */
 394       if (a.ncomponents + ncomponents > 4 ||
 395           a.length < instance) {
 396          current_index = m_next_register_index;
 397          ncomponents = 0;
 398          instance = 0;
 399       }
 400
 401       if (ncomponents == 0)
 402          m_next_register_index += a.length;
 403
 404       uint32_t mask = ((1 << a.ncomponents) - 1) << ncomponents;
 405
 406       PValue  array = PValue(new GPRArray(current_index, a.length, mask, ncomponents));
 407
 408       sfn_log << SfnLog::reg << "Add array at "<< current_index
 409               << " of size " << a.length << " with " << a.ncomponents
 410               << " components, mask " << mask << "\n";
 411
 412       m_local_register_map[a.index] = current_index + instance;
 413
 414       for (unsigned  i = 0; i < a.ncomponents; ++i)
 415          m_registers[((current_index  + instance) << 3) + i] = array;
 416
 417       VRec next_reg = {current_index + instance, mask, mask};
 418       m_register_map[current_index + instance] = next_reg;
 419
 420       ncomponents += a.ncomponents;
 421       ++instance;
 422    }
 423 }
 424
 425 void ValuePool::allocate_local_register(const nir_register& reg)
 426 {
 427    int index = m_next_register_index++;
 428    m_local_register_map[reg.index] = index;
 429    allocate_with_mask(index, 0xf, true);
 430
 431    /* Create actual register and map it */;
 432    for (int i = 0; i < 4; ++i) {
 433       int k = (index << 3) + i;
 434       m_registers[k] = PValue(new GPRValue(index, i));
 435    }
 436 }
 437
 438 void ValuePool::allocate_local_register(const nir_register& reg, array_list& arrays)
 439 {
 440    sfn_log << SfnLog::reg << "ValuePool: Allocate local register " << reg.index
 441            << " as " << m_next_register_index << "\n";
 442
 443    if (reg.num_array_elems) {
 444       array_entry ae = {reg.index, reg.num_array_elems, reg.num_components};
 445       arrays.push(ae);
 446    }
 447    else
 448       allocate_local_register(reg);
 449 }
 450
 451 bool ValuePool::create_undef(nir_ssa_undef_instr* instr)
 452 {
 453    m_ssa_undef.insert(instr->def.index);
 454    return true;
 455 }
 456
 457 bool ValuePool::set_literal_constant(nir_load_const_instr* instr)
 458 {
 459    sfn_log << SfnLog::reg << "Add literal " <<  instr->def.index << "\n";
 460    m_literal_constants[instr->def.index] = instr;
 461    return true;
 462 }
 463
 464 const nir_load_const_instr* ValuePool::get_literal_constant(int index)
 465 {
 466    sfn_log << SfnLog::reg << "Try to locate literal " << index  << "...";
 467    auto literal = m_literal_constants.find(index);
 468    if (literal == m_literal_constants.end()) {
 469       sfn_log << SfnLog::reg << " not found\n";
 470       return nullptr;
 471    }
 472    sfn_log << SfnLog::reg << " found\n";
 473    return literal->second;
 474 }
 475
 476 void ValuePool::add_uniform(unsigned index, const PValue& value)
 477 {
 478    sfn_log << SfnLog::reg << "Reserve " << *value << " as " << index << "\n";
 479    m_uniforms[index] = value;
 480 }
 481
 482 PValue ValuePool::uniform(unsigned index)
 483 {
 484    sfn_log << SfnLog::reg << "Search index " << index << "\n";
 485    auto i = m_uniforms.find(index);
 486    return i == m_uniforms.end() ? PValue() : i->second;
 487 }
 488
 489 int ValuePool::allocate_with_mask(unsigned index, unsigned mask, bool pre_alloc)
 490 {
 491    int retval;
 492    VRec next_register = { index, mask };
 493
 494    sfn_log << SfnLog::reg << (pre_alloc ? "Pre-alloc" : "Allocate")
 495            << " register (" << index << ", " << mask << ")\n";
 496    retval = index;
 497    auto r = m_register_map.find(index);
 498
 499    if (r != m_register_map.end()) {
 500       if ((r->second.mask & next_register.mask) &&
 501           !(r->second.pre_alloc_mask & next_register.mask)) {
 502          std::cerr << "r600 ERR: register ("
 503                    << index << ", " << mask
 504                    << ") already allocated as (" << r->second.index << ", "
 505                    << r->second.mask << ", " << r->second.pre_alloc_mask
 506                    << ") \n";
 507          retval = -1;
 508       } else {
 509          r->second.mask |= next_register.mask;
 510          if (pre_alloc)
 511             r->second.pre_alloc_mask |= next_register.mask;
 512          retval = r->second.index;
 513       }
 514    } else  {
 515       if (pre_alloc)
 516          next_register.pre_alloc_mask = mask;
 517       m_register_map[index] = next_register;
 518       retval = next_register.index;
 519    }
 520
 521    sfn_log << SfnLog::reg << "Allocate register (" << index << "," << mask << ") in R"
 522            << retval << "\n";
 523
 524    return retval;
 525 }
 526
 527 PValue ValuePool::literal(uint32_t value)
 528 {
 529    const uint32_t float_1 = 0x3f800000;
 530    const uint32_t float_05 = 0x3f000000;
 531
 532    auto l = m_literals.find(value);
 533    if (l != m_literals.end())
 534       return l->second;
 535
 536    switch (value) {
 537    case 0:
 538       m_literals[0] = PValue(new InlineConstValue(ALU_SRC_0, 0));
 539       return m_literals[0];
 540    case 1:
 541       m_literals[1] = PValue(new InlineConstValue(ALU_SRC_1_INT, 0));
 542       return m_literals[1];
 543    case float_1:
 544       m_literals[float_1] = PValue(new InlineConstValue(ALU_SRC_1, 0));
 545       return m_literals[float_1];
 546    case float_05:
 547       m_literals[float_05] = PValue(new InlineConstValue(ALU_SRC_0_5, 0));
 548       return m_literals[float_05];
 549    case 0xffffffff:
 550       m_literals[0xffffffff] = PValue(new InlineConstValue(ALU_SRC_M_1_INT, 0));
 551       return m_literals[0xffffffff];
 552    default:
 553       m_literals[value] = PValue(new LiteralValue(value));
 554       return m_literals[value];
 555    }
 556 }
 557
 558 }