src/compiler/nir/nir_search.c

   1 /*
   2  * Copyright © 2014 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Jason Ekstrand (jason@jlekstrand.net)
  25  *
  26  */
  27
  28 #include <inttypes.h>
  29 #include "nir_search.h"
  30
  31 struct match_state {
  32    bool inexact_match;
  33    bool has_exact_alu;
  34    unsigned variables_seen;
  35    nir_alu_src variables[NIR_SEARCH_MAX_VARIABLES];
  36 };
  37
  38 static bool
  39 match_expression(const nir_search_expression *expr, nir_alu_instr *instr,
  40                  unsigned num_components, const uint8_t *swizzle,
  41                  struct match_state *state);
  42
  43 static const uint8_t identity_swizzle[] = { 0, 1, 2, 3 };
  44
  45 /**
  46  * Check if a source produces a value of the given type.
  47  *
  48  * Used for satisfying 'a@type' constraints.
  49  */
  50 static bool
  51 src_is_type(nir_src src, nir_alu_type type)
  52 {
  53    assert(type != nir_type_invalid);
  54
  55    if (!src.is_ssa)
  56       return false;
  57
  58    /* Turn nir_type_bool32 into nir_type_bool...they're the same thing. */
  59    if (nir_alu_type_get_base_type(type) == nir_type_bool)
  60       type = nir_type_bool;
  61
  62    if (src.ssa->parent_instr->type == nir_instr_type_alu) {
  63       nir_alu_instr *src_alu = nir_instr_as_alu(src.ssa->parent_instr);
  64       nir_alu_type output_type = nir_op_infos[src_alu->op].output_type;
  65
  66       if (type == nir_type_bool) {
  67          switch (src_alu->op) {
  68          case nir_op_iand:
  69          case nir_op_ior:
  70          case nir_op_ixor:
  71             return src_is_type(src_alu->src[0].src, nir_type_bool) &&
  72                    src_is_type(src_alu->src[1].src, nir_type_bool);
  73          case nir_op_inot:
  74             return src_is_type(src_alu->src[0].src, nir_type_bool);
  75          default:
  76             break;
  77          }
  78       }
  79
  80       return nir_alu_type_get_base_type(output_type) == type;
  81    } else if (src.ssa->parent_instr->type == nir_instr_type_intrinsic) {
  82       nir_intrinsic_instr *intr = nir_instr_as_intrinsic(src.ssa->parent_instr);
  83
  84       if (type == nir_type_bool) {
  85          return intr->intrinsic == nir_intrinsic_load_front_face ||
  86                 intr->intrinsic == nir_intrinsic_load_helper_invocation;
  87       }
  88    }
  89
  90    /* don't know */
  91    return false;
  92 }
  93
  94 static bool
  95 match_value(const nir_search_value *value, nir_alu_instr *instr, unsigned src,
  96             unsigned num_components, const uint8_t *swizzle,
  97             struct match_state *state)
  98 {
  99    uint8_t new_swizzle[4];
 100
 101    /* Searching only works on SSA values because, if it's not SSA, we can't
 102     * know if the value changed between one instance of that value in the
 103     * expression and another.  Also, the replace operation will place reads of
 104     * that value right before the last instruction in the expression we're
 105     * replacing so those reads will happen after the original reads and may
 106     * not be valid if they're register reads.
 107     */
 108    if (!instr->src[src].src.is_ssa)
 109       return false;
 110
 111    /* If the source is an explicitly sized source, then we need to reset
 112     * both the number of components and the swizzle.
 113     */
 114    if (nir_op_infos[instr->op].input_sizes[src] != 0) {
 115       num_components = nir_op_infos[instr->op].input_sizes[src];
 116       swizzle = identity_swizzle;
 117    }
 118
 119    for (unsigned i = 0; i < num_components; ++i)
 120       new_swizzle[i] = instr->src[src].swizzle[swizzle[i]];
 121
 122    /* If the value has a specific bit size and it doesn't match, bail */
 123    if (value->bit_size &&
 124        nir_src_bit_size(instr->src[src].src) != value->bit_size)
 125       return false;
 126
 127    switch (value->type) {
 128    case nir_search_value_expression:
 129       if (instr->src[src].src.ssa->parent_instr->type != nir_instr_type_alu)
 130          return false;
 131
 132       return match_expression(nir_search_value_as_expression(value),
 133                               nir_instr_as_alu(instr->src[src].src.ssa->parent_instr),
 134                               num_components, new_swizzle, state);
 135
 136    case nir_search_value_variable: {
 137       nir_search_variable *var = nir_search_value_as_variable(value);
 138       assert(var->variable < NIR_SEARCH_MAX_VARIABLES);
 139
 140       if (state->variables_seen & (1 << var->variable)) {
 141          if (state->variables[var->variable].src.ssa != instr->src[src].src.ssa)
 142             return false;
 143
 144          assert(!instr->src[src].abs && !instr->src[src].negate);
 145
 146          for (unsigned i = 0; i < num_components; ++i) {
 147             if (state->variables[var->variable].swizzle[i] != new_swizzle[i])
 148                return false;
 149          }
 150
 151          return true;
 152       } else {
 153          if (var->is_constant &&
 154              instr->src[src].src.ssa->parent_instr->type != nir_instr_type_load_const)
 155             return false;
 156
 157          if (var->cond && !var->cond(instr, src, num_components, new_swizzle))
 158             return false;
 159
 160          if (var->type != nir_type_invalid &&
 161              !src_is_type(instr->src[src].src, var->type))
 162             return false;
 163
 164          state->variables_seen |= (1 << var->variable);
 165          state->variables[var->variable].src = instr->src[src].src;
 166          state->variables[var->variable].abs = false;
 167          state->variables[var->variable].negate = false;
 168
 169          for (unsigned i = 0; i < 4; ++i) {
 170             if (i < num_components)
 171                state->variables[var->variable].swizzle[i] = new_swizzle[i];
 172             else
 173                state->variables[var->variable].swizzle[i] = 0;
 174          }
 175
 176          return true;
 177       }
 178    }
 179
 180    case nir_search_value_constant: {
 181       nir_search_constant *const_val = nir_search_value_as_constant(value);
 182
 183       if (!instr->src[src].src.is_ssa)
 184          return false;
 185
 186       if (instr->src[src].src.ssa->parent_instr->type != nir_instr_type_load_const)
 187          return false;
 188
 189       nir_load_const_instr *load =
 190          nir_instr_as_load_const(instr->src[src].src.ssa->parent_instr);
 191
 192       switch (const_val->type) {
 193       case nir_type_float:
 194          for (unsigned i = 0; i < num_components; ++i) {
 195             double val;
 196             switch (load->def.bit_size) {
 197             case 32:
 198                val = load->value.f32[new_swizzle[i]];
 199                break;
 200             case 64:
 201                val = load->value.f64[new_swizzle[i]];
 202                break;
 203             default:
 204                unreachable("unknown bit size");
 205             }
 206
 207             if (val != const_val->data.d)
 208                return false;
 209          }
 210          return true;
 211
 212       case nir_type_int:
 213          for (unsigned i = 0; i < num_components; ++i) {
 214             int64_t val;
 215             switch (load->def.bit_size) {
 216             case 32:
 217                val = load->value.i32[new_swizzle[i]];
 218                break;
 219             case 64:
 220                val = load->value.i64[new_swizzle[i]];
 221                break;
 222             default:
 223                unreachable("unknown bit size");
 224             }
 225
 226             if (val != const_val->data.i)
 227                return false;
 228          }
 229          return true;
 230
 231       case nir_type_uint:
 232       case nir_type_bool32:
 233          for (unsigned i = 0; i < num_components; ++i) {
 234             uint64_t val;
 235             switch (load->def.bit_size) {
 236             case 32:
 237                val = load->value.u32[new_swizzle[i]];
 238                break;
 239             case 64:
 240                val = load->value.u64[new_swizzle[i]];
 241                break;
 242             default:
 243                unreachable("unknown bit size");
 244             }
 245
 246             if (val != const_val->data.u)
 247                return false;
 248          }
 249          return true;
 250
 251       default:
 252          unreachable("Invalid alu source type");
 253       }
 254    }
 255
 256    default:
 257       unreachable("Invalid search value type");
 258    }
 259 }
 260
 261 static bool
 262 match_expression(const nir_search_expression *expr, nir_alu_instr *instr,
 263                  unsigned num_components, const uint8_t *swizzle,
 264                  struct match_state *state)
 265 {
 266    if (instr->op != expr->opcode)
 267       return false;
 268
 269    assert(instr->dest.dest.is_ssa);
 270
 271    if (expr->value.bit_size &&
 272        instr->dest.dest.ssa.bit_size != expr->value.bit_size)
 273       return false;
 274
 275    state->inexact_match = expr->inexact || state->inexact_match;
 276    state->has_exact_alu = instr->exact || state->has_exact_alu;
 277    if (state->inexact_match && state->has_exact_alu)
 278       return false;
 279
 280    assert(!instr->dest.saturate);
 281    assert(nir_op_infos[instr->op].num_inputs > 0);
 282
 283    /* If we have an explicitly sized destination, we can only handle the
 284     * identity swizzle.  While dot(vec3(a, b, c).zxy) is a valid
 285     * expression, we don't have the information right now to propagate that
 286     * swizzle through.  We can only properly propagate swizzles if the
 287     * instruction is vectorized.
 288     */
 289    if (nir_op_infos[instr->op].output_size != 0) {
 290       for (unsigned i = 0; i < num_components; i++) {
 291          if (swizzle[i] != i)
 292             return false;
 293       }
 294    }
 295
 296    /* Stash off the current variables_seen bitmask.  This way we can
 297     * restore it prior to matching in the commutative case below.
 298     */
 299    unsigned variables_seen_stash = state->variables_seen;
 300
 301    bool matched = true;
 302    for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
 303       if (!match_value(expr->srcs[i], instr, i, num_components,
 304                        swizzle, state)) {
 305          matched = false;
 306          break;
 307       }
 308    }
 309
 310    if (matched)
 311       return true;
 312
 313    if (nir_op_infos[instr->op].algebraic_properties & NIR_OP_IS_COMMUTATIVE) {
 314       assert(nir_op_infos[instr->op].num_inputs == 2);
 315
 316       /* Restore the variables_seen bitmask.  If we don't do this, then we
 317        * could end up with an erroneous failure due to variables found in the
 318        * first match attempt above not matching those in the second.
 319        */
 320       state->variables_seen = variables_seen_stash;
 321
 322       if (!match_value(expr->srcs[0], instr, 1, num_components,
 323                        swizzle, state))
 324          return false;
 325
 326       return match_value(expr->srcs[1], instr, 0, num_components,
 327                          swizzle, state);
 328    } else {
 329       return false;
 330    }
 331 }
 332
 333 typedef struct bitsize_tree {
 334    unsigned num_srcs;
 335    struct bitsize_tree *srcs[4];
 336
 337    unsigned common_size;
 338    bool is_src_sized[4];
 339    bool is_dest_sized;
 340
 341    unsigned dest_size;
 342    unsigned src_size[4];
 343 } bitsize_tree;
 344
 345 static bitsize_tree *
 346 build_bitsize_tree(void *mem_ctx, struct match_state *state,
 347                    const nir_search_value *value)
 348 {
 349    bitsize_tree *tree = rzalloc(mem_ctx, bitsize_tree);
 350
 351    switch (value->type) {
 352    case nir_search_value_expression: {
 353       nir_search_expression *expr = nir_search_value_as_expression(value);
 354       nir_op_info info = nir_op_infos[expr->opcode];
 355       tree->num_srcs = info.num_inputs;
 356       tree->common_size = 0;
 357       for (unsigned i = 0; i < info.num_inputs; i++) {
 358          tree->is_src_sized[i] = !!nir_alu_type_get_type_size(info.input_types[i]);
 359          if (tree->is_src_sized[i])
 360             tree->src_size[i] = nir_alu_type_get_type_size(info.input_types[i]);
 361          tree->srcs[i] = build_bitsize_tree(mem_ctx, state, expr->srcs[i]);
 362       }
 363       tree->is_dest_sized = !!nir_alu_type_get_type_size(info.output_type);
 364       if (tree->is_dest_sized)
 365          tree->dest_size = nir_alu_type_get_type_size(info.output_type);
 366       break;
 367    }
 368
 369    case nir_search_value_variable: {
 370       nir_search_variable *var = nir_search_value_as_variable(value);
 371       tree->num_srcs = 0;
 372       tree->is_dest_sized = true;
 373       tree->dest_size = nir_src_bit_size(state->variables[var->variable].src);
 374       break;
 375    }
 376
 377    case nir_search_value_constant: {
 378       tree->num_srcs = 0;
 379       tree->is_dest_sized = false;
 380       tree->common_size = 0;
 381       break;
 382    }
 383    }
 384
 385    if (value->bit_size) {
 386       assert(!tree->is_dest_sized || tree->dest_size == value->bit_size);
 387       tree->common_size = value->bit_size;
 388    }
 389
 390    return tree;
 391 }
 392
 393 static unsigned
 394 bitsize_tree_filter_up(bitsize_tree *tree)
 395 {
 396    for (unsigned i = 0; i < tree->num_srcs; i++) {
 397       unsigned src_size = bitsize_tree_filter_up(tree->srcs[i]);
 398       if (src_size == 0)
 399          continue;
 400
 401       if (tree->is_src_sized[i]) {
 402          assert(src_size == tree->src_size[i]);
 403       } else if (tree->common_size != 0) {
 404          assert(src_size == tree->common_size);
 405          tree->src_size[i] = src_size;
 406       } else {
 407          tree->common_size = src_size;
 408          tree->src_size[i] = src_size;
 409       }
 410    }
 411
 412    if (tree->num_srcs && tree->common_size) {
 413       if (tree->dest_size == 0)
 414          tree->dest_size = tree->common_size;
 415       else if (!tree->is_dest_sized)
 416          assert(tree->dest_size == tree->common_size);
 417
 418       for (unsigned i = 0; i < tree->num_srcs; i++) {
 419          if (!tree->src_size[i])
 420             tree->src_size[i] = tree->common_size;
 421       }
 422    }
 423
 424    return tree->dest_size;
 425 }
 426
 427 static void
 428 bitsize_tree_filter_down(bitsize_tree *tree, unsigned size)
 429 {
 430    if (tree->dest_size)
 431       assert(tree->dest_size == size);
 432    else
 433       tree->dest_size = size;
 434
 435    if (!tree->is_dest_sized) {
 436       if (tree->common_size)
 437          assert(tree->common_size == size);
 438       else
 439          tree->common_size = size;
 440    }
 441
 442    for (unsigned i = 0; i < tree->num_srcs; i++) {
 443       if (!tree->src_size[i]) {
 444          assert(tree->common_size);
 445          tree->src_size[i] = tree->common_size;
 446       }
 447       bitsize_tree_filter_down(tree->srcs[i], tree->src_size[i]);
 448    }
 449 }
 450
 451 static nir_alu_src
 452 construct_value(const nir_search_value *value,
 453                 unsigned num_components, bitsize_tree *bitsize,
 454                 struct match_state *state,
 455                 nir_instr *instr, void *mem_ctx)
 456 {
 457    switch (value->type) {
 458    case nir_search_value_expression: {
 459       const nir_search_expression *expr = nir_search_value_as_expression(value);
 460
 461       if (nir_op_infos[expr->opcode].output_size != 0)
 462          num_components = nir_op_infos[expr->opcode].output_size;
 463
 464       nir_alu_instr *alu = nir_alu_instr_create(mem_ctx, expr->opcode);
 465       nir_ssa_dest_init(&alu->instr, &alu->dest.dest, num_components,
 466                         bitsize->dest_size, NULL);
 467       alu->dest.write_mask = (1 << num_components) - 1;
 468       alu->dest.saturate = false;
 469
 470       /* We have no way of knowing what values in a given search expression
 471        * map to a particular replacement value.  Therefore, if the
 472        * expression we are replacing has any exact values, the entire
 473        * replacement should be exact.
 474        */
 475       alu->exact = state->has_exact_alu;
 476
 477       for (unsigned i = 0; i < nir_op_infos[expr->opcode].num_inputs; i++) {
 478          /* If the source is an explicitly sized source, then we need to reset
 479           * the number of components to match.
 480           */
 481          if (nir_op_infos[alu->op].input_sizes[i] != 0)
 482             num_components = nir_op_infos[alu->op].input_sizes[i];
 483
 484          alu->src[i] = construct_value(expr->srcs[i],
 485                                        num_components, bitsize->srcs[i],
 486                                        state, instr, mem_ctx);
 487       }
 488
 489       nir_instr_insert_before(instr, &alu->instr);
 490
 491       nir_alu_src val;
 492       val.src = nir_src_for_ssa(&alu->dest.dest.ssa);
 493       val.negate = false;
 494       val.abs = false,
 495       memcpy(val.swizzle, identity_swizzle, sizeof val.swizzle);
 496
 497       return val;
 498    }
 499
 500    case nir_search_value_variable: {
 501       const nir_search_variable *var = nir_search_value_as_variable(value);
 502       assert(state->variables_seen & (1 << var->variable));
 503
 504       nir_alu_src val = { NIR_SRC_INIT };
 505       nir_alu_src_copy(&val, &state->variables[var->variable], mem_ctx);
 506
 507       assert(!var->is_constant);
 508
 509       return val;
 510    }
 511
 512    case nir_search_value_constant: {
 513       const nir_search_constant *c = nir_search_value_as_constant(value);
 514       nir_load_const_instr *load =
 515          nir_load_const_instr_create(mem_ctx, 1, bitsize->dest_size);
 516
 517       switch (c->type) {
 518       case nir_type_float:
 519          load->def.name = ralloc_asprintf(load, "%f", c->data.d);
 520          switch (bitsize->dest_size) {
 521          case 32:
 522             load->value.f32[0] = c->data.d;
 523             break;
 524          case 64:
 525             load->value.f64[0] = c->data.d;
 526             break;
 527          default:
 528             unreachable("unknown bit size");
 529          }
 530          break;
 531
 532       case nir_type_int:
 533          load->def.name = ralloc_asprintf(load, "%" PRIi64, c->data.i);
 534          switch (bitsize->dest_size) {
 535          case 32:
 536             load->value.i32[0] = c->data.i;
 537             break;
 538          case 64:
 539             load->value.i64[0] = c->data.i;
 540             break;
 541          default:
 542             unreachable("unknown bit size");
 543          }
 544          break;
 545
 546       case nir_type_uint:
 547          load->def.name = ralloc_asprintf(load, "%" PRIu64, c->data.u);
 548          switch (bitsize->dest_size) {
 549          case 32:
 550             load->value.u32[0] = c->data.u;
 551             break;
 552          case 64:
 553             load->value.u64[0] = c->data.u;
 554             break;
 555          default:
 556             unreachable("unknown bit size");
 557          }
 558          break;
 559
 560       case nir_type_bool32:
 561          load->value.u32[0] = c->data.u;
 562          break;
 563       default:
 564          unreachable("Invalid alu source type");
 565       }
 566
 567       nir_instr_insert_before(instr, &load->instr);
 568
 569       nir_alu_src val;
 570       val.src = nir_src_for_ssa(&load->def);
 571       val.negate = false;
 572       val.abs = false,
 573       memset(val.swizzle, 0, sizeof val.swizzle);
 574
 575       return val;
 576    }
 577
 578    default:
 579       unreachable("Invalid search value type");
 580    }
 581 }
 582
 583 nir_alu_instr *
 584 nir_replace_instr(nir_alu_instr *instr, const nir_search_expression *search,
 585                   const nir_search_value *replace, void *mem_ctx)
 586 {
 587    uint8_t swizzle[4] = { 0, 0, 0, 0 };
 588
 589    for (unsigned i = 0; i < instr->dest.dest.ssa.num_components; ++i)
 590       swizzle[i] = i;
 591
 592    assert(instr->dest.dest.is_ssa);
 593
 594    struct match_state state;
 595    state.inexact_match = false;
 596    state.has_exact_alu = false;
 597    state.variables_seen = 0;
 598
 599    if (!match_expression(search, instr, instr->dest.dest.ssa.num_components,
 600                          swizzle, &state))
 601       return NULL;
 602
 603    void *bitsize_ctx = ralloc_context(NULL);
 604    bitsize_tree *tree = build_bitsize_tree(bitsize_ctx, &state, replace);
 605    bitsize_tree_filter_up(tree);
 606    bitsize_tree_filter_down(tree, instr->dest.dest.ssa.bit_size);
 607
 608    /* Inserting a mov may be unnecessary.  However, it's much easier to
 609     * simply let copy propagation clean this up than to try to go through
 610     * and rewrite swizzles ourselves.
 611     */
 612    nir_alu_instr *mov = nir_alu_instr_create(mem_ctx, nir_op_imov);
 613    mov->dest.write_mask = instr->dest.write_mask;
 614    nir_ssa_dest_init(&mov->instr, &mov->dest.dest,
 615                      instr->dest.dest.ssa.num_components,
 616                      instr->dest.dest.ssa.bit_size, NULL);
 617
 618    mov->src[0] = construct_value(replace,
 619                                  instr->dest.dest.ssa.num_components, tree,
 620                                  &state, &instr->instr, mem_ctx);
 621    nir_instr_insert_before(&instr->instr, &mov->instr);
 622
 623    nir_ssa_def_rewrite_uses(&instr->dest.dest.ssa,
 624                             nir_src_for_ssa(&mov->dest.dest.ssa));
 625
 626    /* We know this one has no more uses because we just rewrote them all,
 627     * so we can remove it.  The rest of the matched expression, however, we
 628     * don't know so much about.  We'll just let dead code clean them up.
 629     */
 630    nir_instr_remove(&instr->instr);
 631
 632    ralloc_free(bitsize_ctx);
 633
 634    return mov;
 635 }