src/gallium/drivers/r600/sfn/sfn_valuepool.cpp

   1 /* -*- mesa-c++  -*-
   2  *
   3  * Copyright (c) 2018-2019 Collabora LTD
   4  *
   5  * Author: Gert Wollny <gert.wollny@collabora.com>
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the "Software"),
   9  * to deal in the Software without restriction, including without limitation
  10  * on the rights to use, copy, modify, merge, publish, distribute, sub
  11  * license, and/or sell copies of the Software, and to permit persons to whom
  12  * the Software is furnished to do so, subject to the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the next
  15  * paragraph) shall be included in all copies or substantial portions of the
  16  * Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  21  * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
  22  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  23  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  24  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  */
  26
  27 #include "sfn_debug.h"
  28 #include "sfn_value_gpr.h"
  29 #include "sfn_valuepool.h"
  30
  31 #include <iostream>
  32 #include <queue>
  33
  34 namespace r600 {
  35
  36 using std::vector;
  37 using std::pair;
  38 using std::make_pair;
  39 using std::queue;
  40
  41 ValuePool::ValuePool():
  42    m_next_register_index(0),
  43    current_temp_reg_index(0),
  44    next_temp_reg_comp(4)
  45 {
  46 }
  47
  48 PValue ValuePool::m_undef = Value::zero;
  49
  50 GPRVector ValuePool::vec_from_nir(const nir_dest& dst, int num_components)
  51 {
  52    std::array<PValue, 4> result;
  53    for (int i = 0; i < 4; ++i)
  54       result[i] = from_nir(dst, i < num_components ? i : 7);
  55    return GPRVector(result);
  56 }
  57
  58 std::vector<PValue> ValuePool::varvec_from_nir(const nir_dest& dst, int num_components)
  59 {
  60    std::vector<PValue> result(num_components);
  61    for (int i = 0; i < num_components; ++i)
  62       result[i] = from_nir(dst, i);
  63    return result;
  64 }
  65
  66
  67 std::vector<PValue> ValuePool::varvec_from_nir(const nir_src& src, int num_components)
  68 {
  69    std::vector<PValue> result(num_components);
  70    int i;
  71    for (i = 0; i < num_components; ++i)
  72       result[i] = from_nir(src, i);
  73
  74    return result;
  75 }
  76
  77
  78 PValue ValuePool::from_nir(const nir_src& v, unsigned component, unsigned swizzled)
  79 {
  80    sfn_log << SfnLog::reg << "Search " << (v.is_ssa ? "ssa_reg " : "reg ")
  81            << (v.is_ssa ? v.ssa->index : v.reg.reg->index);
  82
  83    if (!v.is_ssa) {
  84       int idx = lookup_register_index(v);
  85       sfn_log << SfnLog::reg << "  -> got index " <<  idx << "\n";
  86       if (idx >= 0) {
  87          auto reg = lookup_register(idx, swizzled, false);
  88          if (reg) {
  89             if (reg->type() == Value::gpr_vector) {
  90                auto& array = static_cast<GPRArray&>(*reg);
  91                reg = array.get_indirect(v.reg.base_offset,
  92                                         v.reg.indirect ?
  93                                            from_nir(*v.reg.indirect, 0, 0) : nullptr,
  94                                         component);
  95             }
  96             return reg;
  97          }
  98       }
  99       assert(0 && "local registers should always be found");
 100    }
 101
 102    unsigned index = v.ssa->index;
 103    /* For undefs we use zero and let ()yet to be implemeneted dce deal with it */
 104    if (m_ssa_undef.find(index) != m_ssa_undef.end())
 105       return Value::zero;
 106
 107
 108    int idx = lookup_register_index(v);
 109    sfn_log << SfnLog::reg << "  -> got index " <<  idx << "\n";
 110    if (idx >= 0) {
 111       auto reg = lookup_register(idx, swizzled, false);
 112       if (reg)
 113          return reg;
 114    }
 115
 116
 117    auto literal_val = m_literal_constants.find(index);
 118    if (literal_val != m_literal_constants.end()) {
 119       switch (literal_val->second->def.bit_size) {
 120       case 1:
 121          return PValue(new LiteralValue(literal_val->second->value[swizzled].b ? 0xffffffff : 0, component));
 122       case 32:
 123          return literal(literal_val->second->value[swizzled].u32);
 124       default:
 125          sfn_log << SfnLog::reg << "Unsupported bit size " << literal_val->second->def.bit_size
 126                  << " fall back to 32\n";
 127          return PValue(new LiteralValue(literal_val->second->value[swizzled].u32, component));
 128       }
 129    }
 130
 131    unsigned uindex = (index << 2) + swizzled;
 132    auto u = m_uniforms.find(uindex);
 133    if (u != m_uniforms.end())
 134       return u->second;
 135
 136    return PValue();
 137 }
 138
 139 PValue ValuePool::from_nir(const nir_src& v, unsigned component)
 140 {
 141    return from_nir(v, component, component);
 142 }
 143
 144 PValue ValuePool::from_nir(const nir_tex_src &v, unsigned component)
 145 {
 146    return from_nir(v.src, component, component);
 147 }
 148
 149 PValue ValuePool::from_nir(const nir_alu_src &v, unsigned component)
 150 {
 151    return from_nir(v.src, component, v.swizzle[component]);
 152 }
 153
 154 PValue ValuePool::get_temp_register()
 155 {
 156    if (next_temp_reg_comp > 3) {
 157       current_temp_reg_index = allocate_temp_register();
 158       next_temp_reg_comp = 0;
 159    }
 160    return PValue(new GPRValue(current_temp_reg_index, next_temp_reg_comp++));
 161 }
 162
 163 GPRVector ValuePool::get_temp_vec4()
 164 {
 165    int sel = allocate_temp_register();
 166    return GPRVector(sel, {0,1,2,3});
 167 }
 168
 169 PValue ValuePool::create_register_from_nir_src(const nir_src& src, int comp)
 170 {
 171    int idx = src.is_ssa ? get_dst_ssa_register_index(*src.ssa):
 172                           get_local_register_index(*src.reg.reg);
 173
 174    auto retval = lookup_register(idx, comp, false);
 175    if (!retval || retval->type() != Value::gpr || retval->type() != Value::gpr_array_value)
 176       retval = create_register(idx, comp);
 177    return retval;
 178 }
 179
 180 PValue ValuePool::from_nir(const nir_alu_dest &v, unsigned component)
 181 {
 182    //assert(v->write_mask & (1 << component));
 183    return from_nir(v.dest, component);
 184 }
 185
 186 int ValuePool::lookup_register_index(const nir_dest& dst)
 187 {
 188    return dst.is_ssa ? get_dst_ssa_register_index(dst.ssa):
 189                        get_local_register_index(*dst.reg.reg);
 190 }
 191
 192 int ValuePool::lookup_register_index(const nir_src& src) const
 193 {
 194    int index = 0;
 195
 196    index = src.is_ssa ?
 197               get_ssa_register_index(*src.ssa) :
 198               get_local_register_index(*src.reg.reg);
 199
 200    sfn_log << SfnLog::reg << " LIDX:" << index;
 201
 202    auto r = m_register_map.find(index);
 203    if (r == m_register_map.end()) {
 204       return -1;
 205    }
 206    return static_cast<int>(r->second.index);
 207 }
 208
 209
 210 int ValuePool::allocate_component(unsigned index, unsigned comp, bool pre_alloc)
 211 {
 212    assert(comp < 8);
 213    return allocate_with_mask(index, 1 << comp, pre_alloc);
 214 }
 215
 216 int ValuePool::allocate_temp_register()
 217 {
 218    return m_next_register_index++;
 219 }
 220
 221
 222 PValue ValuePool::from_nir(const nir_dest& v, unsigned component)
 223 {
 224    int idx = lookup_register_index(v);
 225    sfn_log << SfnLog::reg << __func__  << ": ";
 226    if (v.is_ssa)
 227       sfn_log << "ssa_" << v.ssa.index;
 228    else
 229       sfn_log << "r" << v.reg.reg->index;
 230    sfn_log << " -> " << idx << "\n";
 231
 232    auto retval = lookup_register(idx, component, false);
 233    if (!retval)
 234       retval = create_register(idx, component);
 235
 236    if (retval->type() == Value::gpr_vector) {
 237       assert(!v.is_ssa);
 238       auto& array = static_cast<GPRArray&>(*retval);
 239       retval = array.get_indirect(v.reg.base_offset,
 240                                   v.reg.indirect ?
 241                                   from_nir(*v.reg.indirect, 0, 0) : nullptr,
 242                                   component);
 243    }
 244
 245    return retval;
 246 }
 247
 248 ValueMap ValuePool::get_temp_registers() const
 249 {
 250    ValueMap result;
 251
 252    for (auto& v : m_registers) {
 253       if (v.second->type() == Value::gpr)
 254          result.insert(v.second);
 255       else if (v.second->type() == Value::gpr_vector) {
 256          auto& array = static_cast<GPRArray&>(*v.second);
 257          array.collect_registers(result);
 258       }
 259    }
 260    return result;
 261 }
 262
 263 static const char swz[] = "xyzw01?_";
 264
 265 PValue ValuePool::create_register(unsigned sel, unsigned swizzle)
 266 {
 267    sfn_log << SfnLog::reg
 268            <<"Create register " << sel  << '.' << swz[swizzle] << "\n";
 269    auto retval = PValue(new GPRValue(sel, swizzle));
 270    m_registers[(sel << 3) + swizzle] = retval;
 271    return retval;
 272 }
 273
 274 bool ValuePool::inject_register(unsigned sel, unsigned swizzle,
 275                                 const PValue& reg, bool map)
 276 {
 277    uint32_t ssa_index = sel;
 278
 279    if (map) {
 280       auto pos = m_ssa_register_map.find(sel);
 281       if (pos == m_ssa_register_map.end())
 282          ssa_index = m_next_register_index++;
 283       else
 284          ssa_index = pos->second;
 285    }
 286
 287    sfn_log << SfnLog::reg
 288            << "Inject register " << sel  << '.' << swz[swizzle]
 289            << " at index " <<  ssa_index << " ...";
 290
 291    if (map)
 292       m_ssa_register_map[sel] = ssa_index;
 293
 294    allocate_with_mask(ssa_index, swizzle, true);
 295
 296    unsigned idx = (ssa_index << 3) + swizzle;
 297    auto p = m_registers.find(idx);
 298    if ( (p != m_registers.end()) && *p->second != *reg) {
 299       std::cerr << "Register location (" << ssa_index << ", " << swizzle << ") was already reserved\n";
 300       assert(0);
 301       return false;
 302    }
 303    sfn_log << SfnLog::reg << " at idx:" << idx << " to " << *reg << "\n";
 304    m_registers[idx] = reg;
 305
 306    if (m_next_register_index <= ssa_index)
 307       m_next_register_index = ssa_index + 1;
 308    return true;
 309 }
 310
 311
 312 PValue ValuePool::lookup_register(unsigned sel, unsigned swizzle,
 313                                   bool required)
 314 {
 315
 316    PValue retval;
 317    sfn_log << SfnLog::reg
 318            << "lookup register " << sel  << '.' << swz[swizzle] << "("
 319            << ((sel << 3) + swizzle) << ")...";
 320
 321
 322    auto reg = m_registers.find((sel << 3) + swizzle);
 323    if (reg != m_registers.end()) {
 324       sfn_log << SfnLog::reg << " -> Found " << *reg->second << "\n";
 325       retval = reg->second;
 326    } else if (swizzle == 7) {
 327       PValue retval = create_register(sel, swizzle);
 328       sfn_log << SfnLog::reg << " -> Created " << *retval << "\n";
 329    } else if (required) {
 330       sfn_log << SfnLog::reg << "Register (" << sel << ", "
 331               << swizzle << ") not found but required\n";
 332       assert(0 && "Unallocated register value requested\n");
 333    }
 334    sfn_log << SfnLog::reg << " -> Not required and not  allocated\n";
 335    return retval;
 336 }
 337
 338 unsigned ValuePool::get_dst_ssa_register_index(const nir_ssa_def& ssa)
 339 {
 340    sfn_log << SfnLog::reg << __func__ << ": search dst ssa "
 341            << ssa.index;
 342
 343    auto pos = m_ssa_register_map.find(ssa.index);
 344    if (pos == m_ssa_register_map.end()) {
 345       sfn_log << SfnLog::reg << " Need to allocate ...";
 346       allocate_ssa_register(ssa);
 347       pos = m_ssa_register_map.find(ssa.index);
 348       assert(pos != m_ssa_register_map.end());
 349    }
 350    sfn_log << SfnLog::reg << "... got " << pos->second << "\n";
 351    return pos->second;
 352 }
 353
 354 unsigned ValuePool::get_ssa_register_index(const nir_ssa_def& ssa) const
 355 {
 356    sfn_log << SfnLog::reg << __func__ << ": search ssa "
 357            << ssa.index;
 358
 359    auto pos = m_ssa_register_map.find(ssa.index);
 360    sfn_log << SfnLog::reg << " got " << pos->second<< "\n";
 361    if (pos == m_ssa_register_map.end()) {
 362       sfn_log << SfnLog::reg << __func__ << ": ssa register "
 363               << ssa.index << " lookup failed\n";
 364       return -1;
 365    }
 366    return pos->second;
 367 }
 368
 369 unsigned ValuePool::get_local_register_index(const nir_register& reg)
 370 {
 371    auto pos = m_local_register_map.find(reg.index);
 372    if (pos == m_local_register_map.end()) {
 373       allocate_local_register(reg);
 374       pos = m_local_register_map.find(reg.index);
 375       assert(pos != m_local_register_map.end());
 376    }
 377    return pos->second;
 378 }
 379
 380 unsigned ValuePool::get_local_register_index(const nir_register& reg) const
 381 {
 382    auto pos = m_local_register_map.find(reg.index);
 383    if (pos == m_local_register_map.end()) {
 384       sfn_log << SfnLog::err << __func__ << ": local register "
 385               << reg.index << " lookup failed";
 386       return -1;
 387    }
 388    return pos->second;
 389 }
 390
 391 void ValuePool::allocate_ssa_register(const nir_ssa_def& ssa)
 392 {
 393    sfn_log << SfnLog::reg << "ValuePool: Allocate ssa register " << ssa.index
 394            << " as " << m_next_register_index << "\n";
 395    int index = m_next_register_index++;
 396    m_ssa_register_map[ssa.index] = index;
 397    allocate_with_mask(index, 0xf, true);
 398 }
 399
 400 void ValuePool::allocate_arrays(array_list& arrays)
 401 {
 402    int ncomponents = 0;
 403    int current_index = m_next_register_index;
 404    unsigned instance = 0;
 405
 406    while (!arrays.empty()) {
 407       auto a = arrays.top();
 408       arrays.pop();
 409
 410       /* This is a bit hackish, return an id that encodes the array merge. To make sure
 411        * that the mapping doesn't go wrong we have to make sure the arrays is longer than
 412        * the number of instances in this arrays slot */
 413       if (a.ncomponents + ncomponents > 4 ||
 414           a.length < instance) {
 415          current_index = m_next_register_index;
 416          ncomponents = 0;
 417          instance = 0;
 418       }
 419
 420       if (ncomponents == 0)
 421          m_next_register_index += a.length;
 422
 423       uint32_t mask = ((1 << a.ncomponents) - 1) << ncomponents;
 424
 425       PValue  array = PValue(new GPRArray(current_index, a.length, mask, ncomponents));
 426
 427       sfn_log << SfnLog::reg << "Add array at "<< current_index
 428               << " of size " << a.length << " with " << a.ncomponents
 429               << " components, mask " << mask << "\n";
 430
 431       m_local_register_map[a.index] = current_index + instance;
 432
 433       for (unsigned  i = 0; i < a.ncomponents; ++i)
 434          m_registers[((current_index  + instance) << 3) + i] = array;
 435
 436       VRec next_reg = {current_index + instance, mask, mask};
 437       m_register_map[current_index + instance] = next_reg;
 438
 439       ncomponents += a.ncomponents;
 440       ++instance;
 441    }
 442 }
 443
 444 void ValuePool::allocate_local_register(const nir_register& reg)
 445 {
 446    int index = m_next_register_index++;
 447    m_local_register_map[reg.index] = index;
 448    allocate_with_mask(index, 0xf, true);
 449
 450    /* Create actual register and map it */;
 451    for (int i = 0; i < 4; ++i) {
 452       int k = (index << 3) + i;
 453       m_registers[k] = PValue(new GPRValue(index, i));
 454    }
 455 }
 456
 457 void ValuePool::allocate_local_register(const nir_register& reg, array_list& arrays)
 458 {
 459    sfn_log << SfnLog::reg << "ValuePool: Allocate local register " << reg.index
 460            << " as " << m_next_register_index << "\n";
 461
 462    if (reg.num_array_elems) {
 463       array_entry ae = {reg.index, reg.num_array_elems, reg.num_components};
 464       arrays.push(ae);
 465    }
 466    else
 467       allocate_local_register(reg);
 468 }
 469
 470 bool ValuePool::create_undef(nir_ssa_undef_instr* instr)
 471 {
 472    m_ssa_undef.insert(instr->def.index);
 473    return true;
 474 }
 475
 476 bool ValuePool::set_literal_constant(nir_load_const_instr* instr)
 477 {
 478    sfn_log << SfnLog::reg << "Add literal " <<  instr->def.index << "\n";
 479    m_literal_constants[instr->def.index] = instr;
 480    return true;
 481 }
 482
 483 const nir_load_const_instr* ValuePool::get_literal_constant(int index)
 484 {
 485    sfn_log << SfnLog::reg << "Try to locate literal " << index  << "...";
 486    auto literal = m_literal_constants.find(index);
 487    if (literal == m_literal_constants.end()) {
 488       sfn_log << SfnLog::reg << " not found\n";
 489       return nullptr;
 490    }
 491    sfn_log << SfnLog::reg << " found\n";
 492    return literal->second;
 493 }
 494
 495 void ValuePool::add_uniform(unsigned index, const PValue& value)
 496 {
 497    sfn_log << SfnLog::reg << "Reserve " << *value << " as " << index << "\n";
 498    m_uniforms[index] = value;
 499 }
 500
 501 PValue ValuePool::uniform(unsigned index)
 502 {
 503    sfn_log << SfnLog::reg << "Search index " << index << "\n";
 504    auto i = m_uniforms.find(index);
 505    return i == m_uniforms.end() ? PValue() : i->second;
 506 }
 507
 508 int ValuePool::allocate_with_mask(unsigned index, unsigned mask, bool pre_alloc)
 509 {
 510    int retval;
 511    VRec next_register = { index, mask };
 512
 513    sfn_log << SfnLog::reg << (pre_alloc ? "Pre-alloc" : "Allocate")
 514            << " register (" << index << ", " << mask << ")\n";
 515    retval = index;
 516    auto r = m_register_map.find(index);
 517
 518    if (r != m_register_map.end()) {
 519       if ((r->second.mask & next_register.mask) &&
 520           !(r->second.pre_alloc_mask & next_register.mask)) {
 521          std::cerr << "r600 ERR: register ("
 522                    << index << ", " << mask
 523                    << ") already allocated as (" << r->second.index << ", "
 524                    << r->second.mask << ", " << r->second.pre_alloc_mask
 525                    << ") \n";
 526          retval = -1;
 527       } else {
 528          r->second.mask |= next_register.mask;
 529          if (pre_alloc)
 530             r->second.pre_alloc_mask |= next_register.mask;
 531          retval = r->second.index;
 532       }
 533    } else  {
 534       if (pre_alloc)
 535          next_register.pre_alloc_mask = mask;
 536       m_register_map[index] = next_register;
 537       retval = next_register.index;
 538    }
 539
 540    sfn_log << SfnLog::reg << "Allocate register (" << index << "," << mask << ") in R"
 541            << retval << "\n";
 542
 543    return retval;
 544 }
 545
 546 PValue ValuePool::literal(uint32_t value)
 547 {
 548    auto l = m_literals.find(value);
 549    if (l != m_literals.end())
 550       return l->second;
 551
 552    m_literals[value] = PValue(new LiteralValue(value));
 553    return m_literals[value];
 554 }
 555
 556 }