src/compiler/nir/nir_lower_flrp.c

   1 /*
   2  * Copyright © 2018 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23 #include <math.h>
  24 #include "nir.h"
  25 #include "nir_builder.h"
  26 #include "util/u_vector.h"
  27
  28 /**
  29  * Lower flrp instructions.
  30  *
  31  * Unlike the lowerings that are possible in nir_opt_algrbraic, this pass can
  32  * examine more global information to determine a possibly more efficient
  33  * lowering for each flrp.
  34  */
  35
  36 static void
  37 append_flrp_to_dead_list(struct u_vector *dead_flrp, struct nir_alu_instr *alu)
  38 {
  39    struct nir_alu_instr **tail = u_vector_add(dead_flrp);
  40    *tail = alu;
  41 }
  42
  43 /**
  44  * Replace flrp(a, b, c) with ffma(b, c, ffma(-a, c, a)).
  45  */
  46 static void
  47 replace_with_strict_ffma(struct nir_builder *bld, struct u_vector *dead_flrp,
  48                          struct nir_alu_instr *alu)
  49 {
  50    nir_ssa_def *const a = nir_ssa_for_alu_src(bld, alu, 0);
  51    nir_ssa_def *const b = nir_ssa_for_alu_src(bld, alu, 1);
  52    nir_ssa_def *const c = nir_ssa_for_alu_src(bld, alu, 2);
  53
  54    nir_ssa_def *const neg_a = nir_fneg(bld, a);
  55    nir_instr_as_alu(neg_a->parent_instr)->exact = alu->exact;
  56
  57    nir_ssa_def *const inner_ffma = nir_ffma(bld, neg_a, c, a);
  58    nir_instr_as_alu(inner_ffma->parent_instr)->exact = alu->exact;
  59
  60    nir_ssa_def *const outer_ffma = nir_ffma(bld, b, c, inner_ffma);
  61    nir_instr_as_alu(outer_ffma->parent_instr)->exact = alu->exact;
  62
  63    nir_ssa_def_rewrite_uses(&alu->dest.dest.ssa, nir_src_for_ssa(outer_ffma));
  64
  65    /* DO NOT REMOVE the original flrp yet.  Many of the lowering choices are
  66     * based on other uses of the sources.  Removing the flrp may cause the
  67     * last flrp in a sequence to make a different, incorrect choice.
  68     */
  69    append_flrp_to_dead_list(dead_flrp, alu);
  70 }
  71
  72 /**
  73  * Replace flrp(a, b, c) with ffma(a, (1 - c), bc)
  74  */
  75 static void
  76 replace_with_single_ffma(struct nir_builder *bld, struct u_vector *dead_flrp,
  77                          struct nir_alu_instr *alu)
  78 {
  79    nir_ssa_def *const a = nir_ssa_for_alu_src(bld, alu, 0);
  80    nir_ssa_def *const b = nir_ssa_for_alu_src(bld, alu, 1);
  81    nir_ssa_def *const c = nir_ssa_for_alu_src(bld, alu, 2);
  82
  83    nir_ssa_def *const neg_c = nir_fneg(bld, c);
  84    nir_instr_as_alu(neg_c->parent_instr)->exact = alu->exact;
  85
  86    nir_ssa_def *const one_minus_c =
  87       nir_fadd(bld, nir_imm_floatN_t(bld, 1.0f, c->bit_size), neg_c);
  88    nir_instr_as_alu(one_minus_c->parent_instr)->exact = alu->exact;
  89
  90    nir_ssa_def *const b_times_c = nir_fmul(bld, b, c);
  91    nir_instr_as_alu(b_times_c->parent_instr)->exact = alu->exact;
  92
  93    nir_ssa_def *const final_ffma = nir_ffma(bld, a, one_minus_c, b_times_c);
  94    nir_instr_as_alu(final_ffma->parent_instr)->exact = alu->exact;
  95
  96    nir_ssa_def_rewrite_uses(&alu->dest.dest.ssa, nir_src_for_ssa(final_ffma));
  97
  98    /* DO NOT REMOVE the original flrp yet.  Many of the lowering choices are
  99     * based on other uses of the sources.  Removing the flrp may cause the
 100     * last flrp in a sequence to make a different, incorrect choice.
 101     */
 102    append_flrp_to_dead_list(dead_flrp, alu);
 103 }
 104
 105 /**
 106  * Replace flrp(a, b, c) with a(1-c) + bc.
 107  */
 108 static void
 109 replace_with_strict(struct nir_builder *bld, struct u_vector *dead_flrp,
 110                     struct nir_alu_instr *alu)
 111 {
 112    nir_ssa_def *const a = nir_ssa_for_alu_src(bld, alu, 0);
 113    nir_ssa_def *const b = nir_ssa_for_alu_src(bld, alu, 1);
 114    nir_ssa_def *const c = nir_ssa_for_alu_src(bld, alu, 2);
 115
 116    nir_ssa_def *const neg_c = nir_fneg(bld, c);
 117    nir_instr_as_alu(neg_c->parent_instr)->exact = alu->exact;
 118
 119    nir_ssa_def *const one_minus_c =
 120       nir_fadd(bld, nir_imm_floatN_t(bld, 1.0f, c->bit_size), neg_c);
 121    nir_instr_as_alu(one_minus_c->parent_instr)->exact = alu->exact;
 122
 123    nir_ssa_def *const first_product = nir_fmul(bld, a, one_minus_c);
 124    nir_instr_as_alu(first_product->parent_instr)->exact = alu->exact;
 125
 126    nir_ssa_def *const second_product = nir_fmul(bld, b, c);
 127    nir_instr_as_alu(second_product->parent_instr)->exact = alu->exact;
 128
 129    nir_ssa_def *const sum = nir_fadd(bld, first_product, second_product);
 130    nir_instr_as_alu(sum->parent_instr)->exact = alu->exact;
 131
 132    nir_ssa_def_rewrite_uses(&alu->dest.dest.ssa, nir_src_for_ssa(sum));
 133
 134    /* DO NOT REMOVE the original flrp yet.  Many of the lowering choices are
 135     * based on other uses of the sources.  Removing the flrp may cause the
 136     * last flrp in a sequence to make a different, incorrect choice.
 137     */
 138    append_flrp_to_dead_list(dead_flrp, alu);
 139 }
 140
 141 /**
 142  * Replace flrp(a, b, c) with a + c(b-a).
 143  */
 144 static void
 145 replace_with_fast(struct nir_builder *bld, struct u_vector *dead_flrp,
 146                   struct nir_alu_instr *alu)
 147 {
 148    nir_ssa_def *const a = nir_ssa_for_alu_src(bld, alu, 0);
 149    nir_ssa_def *const b = nir_ssa_for_alu_src(bld, alu, 1);
 150    nir_ssa_def *const c = nir_ssa_for_alu_src(bld, alu, 2);
 151
 152    nir_ssa_def *const neg_a = nir_fneg(bld, a);
 153    nir_instr_as_alu(neg_a->parent_instr)->exact = alu->exact;
 154
 155    nir_ssa_def *const b_minus_a = nir_fadd(bld, b, neg_a);
 156    nir_instr_as_alu(b_minus_a->parent_instr)->exact = alu->exact;
 157
 158    nir_ssa_def *const product = nir_fmul(bld, c, b_minus_a);
 159    nir_instr_as_alu(product->parent_instr)->exact = alu->exact;
 160
 161    nir_ssa_def *const sum = nir_fadd(bld, a, product);
 162    nir_instr_as_alu(sum->parent_instr)->exact = alu->exact;
 163
 164    nir_ssa_def_rewrite_uses(&alu->dest.dest.ssa, nir_src_for_ssa(sum));
 165
 166    /* DO NOT REMOVE the original flrp yet.  Many of the lowering choices are
 167     * based on other uses of the sources.  Removing the flrp may cause the
 168     * last flrp in a sequence to make a different, incorrect choice.
 169     */
 170    append_flrp_to_dead_list(dead_flrp, alu);
 171 }
 172
 173 /**
 174  * Replace flrp(a, b, c) with (b*c ± c) + a => b*c + (a ± c)
 175  *
 176  * \note: This only works if a = ±1.
 177  */
 178 static void
 179 replace_with_expanded_ffma_and_add(struct nir_builder *bld,
 180                                    struct u_vector *dead_flrp,
 181                                    struct nir_alu_instr *alu, bool subtract_c)
 182 {
 183    nir_ssa_def *const a = nir_ssa_for_alu_src(bld, alu, 0);
 184    nir_ssa_def *const b = nir_ssa_for_alu_src(bld, alu, 1);
 185    nir_ssa_def *const c = nir_ssa_for_alu_src(bld, alu, 2);
 186
 187    nir_ssa_def *const b_times_c = nir_fmul(bld, b, c);
 188    nir_instr_as_alu(b_times_c->parent_instr)->exact = alu->exact;
 189
 190    nir_ssa_def *inner_sum;
 191
 192    if (subtract_c) {
 193       nir_ssa_def *const neg_c = nir_fneg(bld, c);
 194       nir_instr_as_alu(neg_c->parent_instr)->exact = alu->exact;
 195
 196       inner_sum = nir_fadd(bld, a, neg_c);
 197    } else {
 198       inner_sum = nir_fadd(bld, a, c);
 199    }
 200
 201    nir_instr_as_alu(inner_sum->parent_instr)->exact = alu->exact;
 202
 203    nir_ssa_def *const outer_sum = nir_fadd(bld, inner_sum, b_times_c);
 204    nir_instr_as_alu(outer_sum->parent_instr)->exact = alu->exact;
 205
 206    nir_ssa_def_rewrite_uses(&alu->dest.dest.ssa, nir_src_for_ssa(outer_sum));
 207
 208    /* DO NOT REMOVE the original flrp yet.  Many of the lowering choices are
 209     * based on other uses of the sources.  Removing the flrp may cause the
 210     * last flrp in a sequence to make a different, incorrect choice.
 211     */
 212    append_flrp_to_dead_list(dead_flrp, alu);
 213 }
 214
 215 /**
 216  * Determines whether a swizzled source is constant w/ all components the same.
 217  *
 218  * The value of the constant is stored in \c result.
 219  *
 220  * \return
 221  * True if all components of the swizzled source are the same constant.
 222  * Otherwise false is returned.
 223  */
 224 static bool
 225 all_same_constant(const nir_alu_instr *instr, unsigned src, double *result)
 226 {
 227    nir_const_value *val = nir_src_as_const_value(instr->src[src].src);
 228
 229    if (!val)
 230       return false;
 231
 232    const uint8_t *const swizzle = instr->src[src].swizzle;
 233    const unsigned num_components = nir_dest_num_components(instr->dest.dest);
 234
 235    if (instr->dest.dest.ssa.bit_size == 32) {
 236       const float first = val[swizzle[0]].f32;
 237
 238       for (unsigned i = 1; i < num_components; i++) {
 239          if (val[swizzle[i]].f32 != first)
 240             return false;
 241       }
 242
 243       *result = first;
 244    } else {
 245       const double first = val[swizzle[0]].f64;
 246
 247       for (unsigned i = 1; i < num_components; i++) {
 248          if (val[swizzle[i]].f64 != first)
 249             return false;
 250       }
 251
 252       *result = first;
 253    }
 254
 255    return true;
 256 }
 257
 258 static bool
 259 sources_are_constants_with_similar_magnitudes(const nir_alu_instr *instr)
 260 {
 261    nir_const_value *val0 = nir_src_as_const_value(instr->src[0].src);
 262    nir_const_value *val1 = nir_src_as_const_value(instr->src[1].src);
 263
 264    if (val0 == NULL || val1 == NULL)
 265       return false;
 266
 267    const uint8_t *const swizzle0 = instr->src[0].swizzle;
 268    const uint8_t *const swizzle1 = instr->src[1].swizzle;
 269    const unsigned num_components = nir_dest_num_components(instr->dest.dest);
 270
 271    if (instr->dest.dest.ssa.bit_size == 32) {
 272       for (unsigned i = 0; i < num_components; i++) {
 273          int exp0;
 274          int exp1;
 275
 276          frexpf(val0[swizzle0[i]].f32, &exp0);
 277          frexpf(val1[swizzle1[i]].f32, &exp1);
 278
 279          /* If the difference between exponents is >= 24, then A+B will always
 280           * have the value whichever between A and B has the largest absolute
 281           * value.  So, [0, 23] is the valid range.  The smaller the limit
 282           * value, the more precision will be maintained at a potential
 283           * performance cost.  Somewhat arbitrarilly split the range in half.
 284           */
 285          if (abs(exp0 - exp1) > (23 / 2))
 286             return false;
 287       }
 288    } else {
 289       for (unsigned i = 0; i < num_components; i++) {
 290          int exp0;
 291          int exp1;
 292
 293          frexp(val0[swizzle0[i]].f64, &exp0);
 294          frexp(val1[swizzle1[i]].f64, &exp1);
 295
 296          /* If the difference between exponents is >= 53, then A+B will always
 297           * have the value whichever between A and B has the largest absolute
 298           * value.  So, [0, 52] is the valid range.  The smaller the limit
 299           * value, the more precision will be maintained at a potential
 300           * performance cost.  Somewhat arbitrarilly split the range in half.
 301           */
 302          if (abs(exp0 - exp1) > (52 / 2))
 303             return false;
 304       }
 305    }
 306
 307    return true;
 308 }
 309
 310 /**
 311  * Counts of similar types of nir_op_flrp instructions
 312  *
 313  * If a similar instruction fits into more than one category, it will only be
 314  * counted once.  The assumption is that no other instruction will have all
 315  * sources the same, or CSE would have removed one of the instructions.
 316  */
 317 struct similar_flrp_stats {
 318    unsigned src2;
 319    unsigned src0_and_src2;
 320    unsigned src1_and_src2;
 321 };
 322
 323 /**
 324  * Collection counts of similar FLRP instructions.
 325  *
 326  * This function only cares about similar instructions that have src2 in
 327  * common.
 328  */
 329 static void
 330 get_similar_flrp_stats(nir_alu_instr *alu, struct similar_flrp_stats *st)
 331 {
 332    memset(st, 0, sizeof(*st));
 333
 334    nir_foreach_use(other_use, alu->src[2].src.ssa) {
 335       /* Is the use also a flrp? */
 336       nir_instr *const other_instr = other_use->parent_instr;
 337       if (other_instr->type != nir_instr_type_alu)
 338          continue;
 339
 340       /* Eh-hem... don't match the instruction with itself. */
 341       if (other_instr == &alu->instr)
 342          continue;
 343
 344       nir_alu_instr *const other_alu = nir_instr_as_alu(other_instr);
 345       if (other_alu->op != nir_op_flrp)
 346          continue;
 347
 348       /* Does the other flrp use source 2 from the first flrp as its source 2
 349        * as well?
 350        */
 351       if (!nir_alu_srcs_equal(alu, other_alu, 2, 2))
 352          continue;
 353
 354       if (nir_alu_srcs_equal(alu, other_alu, 0, 0))
 355          st->src0_and_src2++;
 356       else if (nir_alu_srcs_equal(alu, other_alu, 1, 1))
 357          st->src1_and_src2++;
 358       else
 359          st->src2++;
 360    }
 361 }
 362
 363 static void
 364 convert_flrp_instruction(nir_builder *bld,
 365                          struct u_vector *dead_flrp,
 366                          nir_alu_instr *alu,
 367                          bool always_precise)
 368 {
 369    bool have_ffma = !bld->shader->options->lower_ffma;
 370
 371    bld->cursor = nir_before_instr(&alu->instr);
 372
 373    /* There are two methods to implement flrp(x, y, t).  The strictly correct
 374     * implementation according to the GLSL spec is:
 375     *
 376     *    x(1 - t) + yt
 377     *
 378     * This can also be implemented using two chained FMAs
 379     *
 380     *    fma(y, t, fma(-x, t, x))
 381     *
 382     * This method, using either formulation, has better precision when the
 383     * difference between x and y is very large.  It guarantess that flrp(x, y,
 384     * 1) = y.  For example, flrp(1e38, 1.0, 1.0) is 1.0.  This is correct.
 385     *
 386     * The other possible implementation is:
 387     *
 388     *    x + t(y - x)
 389     *
 390     * This can also be formuated as an FMA:
 391     *
 392     *    fma(y - x, t, x)
 393     *
 394     * For this implementation, flrp(1e38, 1.0, 1.0) is 0.0.  Since 1.0 was
 395     * expected, that's a pretty significant error.
 396     *
 397     * The choice made for lowering depends on a number of factors.
 398     *
 399     * - If the flrp is marked precise and FMA is supported:
 400     *
 401     *        fma(y, t, fma(-x, t, x))
 402     *
 403     *   This is strictly correct (maybe?), and the cost is two FMA
 404     *   instructions.  It at least maintains the flrp(x, y, 1.0) == y
 405     *   condition.
 406     *
 407     * - If the flrp is marked precise and FMA is not supported:
 408     *
 409     *        x(1 - t) + yt
 410     *
 411     *   This is strictly correct, and the cost is 4 instructions.  If FMA is
 412     *   supported, this may or may not be reduced to 3 instructions (a
 413     *   subtract, a multiply, and an FMA)... but in that case the other
 414     *   formulation should have been used.
 415     */
 416    if (alu->exact) {
 417       if (have_ffma)
 418          replace_with_strict_ffma(bld, dead_flrp, alu);
 419       else
 420          replace_with_strict(bld, dead_flrp, alu);
 421
 422       return;
 423    }
 424
 425    /*
 426     * - If x and y are both immediates and the relative magnitude of the
 427     *   values is similar (such that x-y does not lose too much precision):
 428     *
 429     *        x + t(x - y)
 430     *
 431     *   We rely on constant folding to eliminate x-y, and we rely on
 432     *   nir_opt_algebraic to possibly generate an FMA.  The cost is either one
 433     *   FMA or two instructions.
 434     */
 435    if (sources_are_constants_with_similar_magnitudes(alu)) {
 436       replace_with_fast(bld, dead_flrp, alu);
 437       return;
 438    }
 439
 440    /*
 441     * - If x = 1:
 442     *
 443     *        (yt + -t) + 1
 444     *
 445     * - If x = -1:
 446     *
 447     *        (yt + t) - 1
 448     *
 449     *   In both cases, x is used in place of ±1 for simplicity.  Both forms
 450     *   lend to ffma generation on platforms that support ffma.
 451     */
 452    double src0_as_constant;
 453    if (all_same_constant(alu, 0, &src0_as_constant)) {
 454       if (src0_as_constant == 1.0) {
 455          replace_with_expanded_ffma_and_add(bld, dead_flrp, alu,
 456                                             true /* subtract t */);
 457          return;
 458       } else if (src0_as_constant == -1.0) {
 459          replace_with_expanded_ffma_and_add(bld, dead_flrp, alu,
 460                                             false /* add t */);
 461          return;
 462       }
 463    }
 464
 465    /*
 466     * - If y = ±1:
 467     *
 468     *        x(1 - t) + yt
 469     *
 470     *   In this case either the multiply in yt will be eliminated by
 471     *   nir_opt_algebraic.  If FMA is supported, this results in fma(x, (1 -
 472     *   t), ±t) for two instructions.  If FMA is not supported, then the cost
 473     *   is 3 instructions.  We rely on nir_opt_algebraic to generate the FMA
 474     *   instructions as well.
 475     *
 476     *   Another possible replacement is
 477     *
 478     *        -xt + x ± t
 479     *
 480     *   Some groupings of this may be better on some platforms in some
 481     *   circumstances, bit it is probably dependent on scheduling.  Futher
 482     *   investigation may be required.
 483     */
 484    double src1_as_constant;
 485    if ((all_same_constant(alu, 1, &src1_as_constant) &&
 486         (src1_as_constant == -1.0 || src1_as_constant == 1.0))) {
 487       replace_with_strict(bld, dead_flrp, alu);
 488       return;
 489    }
 490
 491    if (have_ffma) {
 492       if (always_precise) {
 493          replace_with_strict_ffma(bld, dead_flrp, alu);
 494          return;
 495       }
 496
 497       /*
 498        * - If FMA is supported and other flrp(x, _, t) exists:
 499        *
 500        *        fma(y, t, fma(-x, t, x))
 501        *
 502        *   The hope is that the inner FMA calculation will be shared with the
 503        *   other lowered flrp.  This results in two FMA instructions for the
 504        *   first flrp and one FMA instruction for each additional flrp.  It
 505        *   also means that the live range for x might be complete after the
 506        *   inner ffma instead of after the last flrp.
 507        */
 508       struct similar_flrp_stats st;
 509
 510       get_similar_flrp_stats(alu, &st);
 511       if (st.src0_and_src2 > 0) {
 512          replace_with_strict_ffma(bld, dead_flrp, alu);
 513          return;
 514       }
 515
 516       /*
 517        * - If FMA is supported and another flrp(_, y, t) exists:
 518        *
 519        *        fma(x, (1 - t), yt)
 520        *
 521        *   The hope is that the (1 - t) and the yt will be shared with the
 522        *   other lowered flrp.  This results in 3 insructions for the first
 523        *   flrp and 1 for each additional flrp.
 524        */
 525       if (st.src1_and_src2 > 0) {
 526          replace_with_single_ffma(bld, dead_flrp, alu);
 527          return;
 528       }
 529    } else {
 530       if (always_precise) {
 531          replace_with_strict(bld, dead_flrp, alu);
 532          return;
 533       }
 534
 535       /*
 536        * - If FMA is not supported and another flrp(x, _, t) exists:
 537        *
 538        *        x(1 - t) + yt
 539        *
 540        *   The hope is that the x(1 - t) will be shared with the other lowered
 541        *   flrp.  This results in 4 insructions for the first flrp and 2 for
 542        *   each additional flrp.
 543        *
 544        * - If FMA is not supported and another flrp(_, y, t) exists:
 545        *
 546        *        x(1 - t) + yt
 547        *
 548        *   The hope is that the (1 - t) and the yt will be shared with the
 549        *   other lowered flrp.  This results in 4 insructions for the first
 550        *   flrp and 2 for each additional flrp.
 551        */
 552       struct similar_flrp_stats st;
 553
 554       get_similar_flrp_stats(alu, &st);
 555       if (st.src0_and_src2 > 0 || st.src1_and_src2 > 0) {
 556          replace_with_strict(bld, dead_flrp, alu);
 557          return;
 558       }
 559    }
 560
 561    /*
 562     * - If t is constant:
 563     *
 564     *        x(1 - t) + yt
 565     *
 566     *   The cost is three instructions without FMA or two instructions with
 567     *   FMA.  This is the same cost as the imprecise lowering, but it gives
 568     *   the instruction scheduler a little more freedom.
 569     *
 570     *   There is no need to handle t = 0.5 specially.  nir_opt_algebraic
 571     *   already has optimizations to convert 0.5x + 0.5y to 0.5(x + y).
 572     */
 573    if (alu->src[2].src.ssa->parent_instr->type == nir_instr_type_load_const) {
 574       replace_with_strict(bld, dead_flrp, alu);
 575       return;
 576    }
 577
 578    /*
 579     * - Otherwise
 580     *
 581     *        x + t(x - y)
 582     */
 583    replace_with_fast(bld, dead_flrp, alu);
 584 }
 585
 586 static void
 587 lower_flrp_impl(nir_function_impl *impl,
 588                 struct u_vector *dead_flrp,
 589                 unsigned lowering_mask,
 590                 bool always_precise)
 591 {
 592    nir_builder b;
 593    nir_builder_init(&b, impl);
 594
 595    nir_foreach_block(block, impl) {
 596       nir_foreach_instr_safe(instr, block) {
 597          if (instr->type == nir_instr_type_alu) {
 598             nir_alu_instr *const alu = nir_instr_as_alu(instr);
 599
 600             if (alu->op == nir_op_flrp &&
 601                 (alu->dest.dest.ssa.bit_size & lowering_mask)) {
 602                convert_flrp_instruction(&b, dead_flrp, alu, always_precise);
 603             }
 604          }
 605       }
 606    }
 607
 608    nir_metadata_preserve(impl, nir_metadata_block_index |
 609                                nir_metadata_dominance);
 610 }
 611
 612 /**
 613  * \param lowering_mask - Bitwise-or of the bit sizes that need to be lowered
 614  *                        (e.g., 16 | 64 if only 16-bit and 64-bit flrp need
 615  *                        lowering).
 616  * \param always_precise - Always require precise lowering for flrp.  This
 617  *                        will always lower flrp to (a * (1 - c)) + (b * c).
 618  * \param have_ffma - Set to true if the GPU has an FFMA instruction that
 619  *                    should be used.
 620  */
 621 bool
 622 nir_lower_flrp(nir_shader *shader,
 623                unsigned lowering_mask,
 624                bool always_precise)
 625 {
 626    struct u_vector dead_flrp;
 627
 628    if (!u_vector_init(&dead_flrp, sizeof(struct nir_alu_instr *), 64))
 629       return false;
 630
 631    nir_foreach_function(function, shader) {
 632       if (function->impl) {
 633          lower_flrp_impl(function->impl, &dead_flrp, lowering_mask,
 634                          always_precise);
 635       }
 636    }
 637
 638    /* Progress was made if the dead list is not empty.  Remove all the
 639     * instructions from the dead list.
 640     */
 641    const bool progress = u_vector_length(&dead_flrp) != 0;
 642
 643    struct nir_alu_instr **instr;
 644    u_vector_foreach(instr, &dead_flrp)
 645       nir_instr_remove(&(*instr)->instr);
 646
 647    u_vector_finish(&dead_flrp);
 648
 649    return progress;
 650 }