src/compiler/nir/nir_lower_flrp.c

   1 /*
   2  * Copyright © 2018 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23 #include <math.h>
  24 #include "nir.h"
  25 #include "nir_builder.h"
  26 #include "util/u_vector.h"
  27
  28 /**
  29  * Lower flrp instructions.
  30  *
  31  * Unlike the lowerings that are possible in nir_opt_algrbraic, this pass can
  32  * examine more global information to determine a possibly more efficient
  33  * lowering for each flrp.
  34  */
  35
  36 static void
  37 append_flrp_to_dead_list(struct u_vector *dead_flrp, struct nir_alu_instr *alu)
  38 {
  39    struct nir_alu_instr **tail = u_vector_add(dead_flrp);
  40    *tail = alu;
  41 }
  42
  43 /**
  44  * Replace flrp(a, b, c) with ffma(b, c, ffma(-a, c, a)).
  45  */
  46 static void
  47 replace_with_strict_ffma(struct nir_builder *bld, struct u_vector *dead_flrp,
  48                          struct nir_alu_instr *alu)
  49 {
  50    nir_ssa_def *const a = nir_ssa_for_alu_src(bld, alu, 0);
  51    nir_ssa_def *const b = nir_ssa_for_alu_src(bld, alu, 1);
  52    nir_ssa_def *const c = nir_ssa_for_alu_src(bld, alu, 2);
  53
  54    nir_ssa_def *const neg_a = nir_fneg(bld, a);
  55    nir_instr_as_alu(neg_a->parent_instr)->exact = alu->exact;
  56
  57    nir_ssa_def *const inner_ffma = nir_ffma(bld, neg_a, c, a);
  58    nir_instr_as_alu(inner_ffma->parent_instr)->exact = alu->exact;
  59
  60    nir_ssa_def *const outer_ffma = nir_ffma(bld, b, c, inner_ffma);
  61    nir_instr_as_alu(outer_ffma->parent_instr)->exact = alu->exact;
  62
  63    nir_ssa_def_rewrite_uses(&alu->dest.dest.ssa, nir_src_for_ssa(outer_ffma));
  64
  65    /* DO NOT REMOVE the original flrp yet.  Many of the lowering choices are
  66     * based on other uses of the sources.  Removing the flrp may cause the
  67     * last flrp in a sequence to make a different, incorrect choice.
  68     */
  69    append_flrp_to_dead_list(dead_flrp, alu);
  70 }
  71
  72 /**
  73  * Replace flrp(a, b, c) with ffma(a, (1 - c), bc)
  74  */
  75 static void
  76 replace_with_single_ffma(struct nir_builder *bld, struct u_vector *dead_flrp,
  77                          struct nir_alu_instr *alu)
  78 {
  79    nir_ssa_def *const a = nir_ssa_for_alu_src(bld, alu, 0);
  80    nir_ssa_def *const b = nir_ssa_for_alu_src(bld, alu, 1);
  81    nir_ssa_def *const c = nir_ssa_for_alu_src(bld, alu, 2);
  82
  83    nir_ssa_def *const neg_c = nir_fneg(bld, c);
  84    nir_instr_as_alu(neg_c->parent_instr)->exact = alu->exact;
  85
  86    nir_ssa_def *const one_minus_c =
  87       nir_fadd(bld, nir_imm_floatN_t(bld, 1.0f, c->bit_size), neg_c);
  88    nir_instr_as_alu(one_minus_c->parent_instr)->exact = alu->exact;
  89
  90    nir_ssa_def *const b_times_c = nir_fmul(bld, b, c);
  91    nir_instr_as_alu(b_times_c->parent_instr)->exact = alu->exact;
  92
  93    nir_ssa_def *const final_ffma = nir_ffma(bld, a, one_minus_c, b_times_c);
  94    nir_instr_as_alu(final_ffma->parent_instr)->exact = alu->exact;
  95
  96    nir_ssa_def_rewrite_uses(&alu->dest.dest.ssa, nir_src_for_ssa(final_ffma));
  97
  98    /* DO NOT REMOVE the original flrp yet.  Many of the lowering choices are
  99     * based on other uses of the sources.  Removing the flrp may cause the
 100     * last flrp in a sequence to make a different, incorrect choice.
 101     */
 102    append_flrp_to_dead_list(dead_flrp, alu);
 103 }
 104
 105 /**
 106  * Replace flrp(a, b, c) with a(1-c) + bc.
 107  */
 108 static void
 109 replace_with_strict(struct nir_builder *bld, struct u_vector *dead_flrp,
 110                     struct nir_alu_instr *alu)
 111 {
 112    nir_ssa_def *const a = nir_ssa_for_alu_src(bld, alu, 0);
 113    nir_ssa_def *const b = nir_ssa_for_alu_src(bld, alu, 1);
 114    nir_ssa_def *const c = nir_ssa_for_alu_src(bld, alu, 2);
 115
 116    nir_ssa_def *const neg_c = nir_fneg(bld, c);
 117    nir_instr_as_alu(neg_c->parent_instr)->exact = alu->exact;
 118
 119    nir_ssa_def *const one_minus_c =
 120       nir_fadd(bld, nir_imm_floatN_t(bld, 1.0f, c->bit_size), neg_c);
 121    nir_instr_as_alu(one_minus_c->parent_instr)->exact = alu->exact;
 122
 123    nir_ssa_def *const first_product = nir_fmul(bld, a, one_minus_c);
 124    nir_instr_as_alu(first_product->parent_instr)->exact = alu->exact;
 125
 126    nir_ssa_def *const second_product = nir_fmul(bld, b, c);
 127    nir_instr_as_alu(second_product->parent_instr)->exact = alu->exact;
 128
 129    nir_ssa_def *const sum = nir_fadd(bld, first_product, second_product);
 130    nir_instr_as_alu(sum->parent_instr)->exact = alu->exact;
 131
 132    nir_ssa_def_rewrite_uses(&alu->dest.dest.ssa, nir_src_for_ssa(sum));
 133
 134    /* DO NOT REMOVE the original flrp yet.  Many of the lowering choices are
 135     * based on other uses of the sources.  Removing the flrp may cause the
 136     * last flrp in a sequence to make a different, incorrect choice.
 137     */
 138    append_flrp_to_dead_list(dead_flrp, alu);
 139 }
 140
 141 /**
 142  * Replace flrp(a, b, c) with a + c(b-a).
 143  */
 144 static void
 145 replace_with_fast(struct nir_builder *bld, struct u_vector *dead_flrp,
 146                   struct nir_alu_instr *alu)
 147 {
 148    nir_ssa_def *const a = nir_ssa_for_alu_src(bld, alu, 0);
 149    nir_ssa_def *const b = nir_ssa_for_alu_src(bld, alu, 1);
 150    nir_ssa_def *const c = nir_ssa_for_alu_src(bld, alu, 2);
 151
 152    nir_ssa_def *const neg_a = nir_fneg(bld, a);
 153    nir_instr_as_alu(neg_a->parent_instr)->exact = alu->exact;
 154
 155    nir_ssa_def *const b_minus_a = nir_fadd(bld, b, neg_a);
 156    nir_instr_as_alu(b_minus_a->parent_instr)->exact = alu->exact;
 157
 158    nir_ssa_def *const product = nir_fmul(bld, c, b_minus_a);
 159    nir_instr_as_alu(product->parent_instr)->exact = alu->exact;
 160
 161    nir_ssa_def *const sum = nir_fadd(bld, a, product);
 162    nir_instr_as_alu(sum->parent_instr)->exact = alu->exact;
 163
 164    nir_ssa_def_rewrite_uses(&alu->dest.dest.ssa, nir_src_for_ssa(sum));
 165
 166    /* DO NOT REMOVE the original flrp yet.  Many of the lowering choices are
 167     * based on other uses of the sources.  Removing the flrp may cause the
 168     * last flrp in a sequence to make a different, incorrect choice.
 169     */
 170    append_flrp_to_dead_list(dead_flrp, alu);
 171 }
 172
 173 /**
 174  * Replace flrp(a, b, c) with (b*c ± c) + a => b*c + (a ± c)
 175  *
 176  * \note: This only works if a = ±1.
 177  */
 178 static void
 179 replace_with_expanded_ffma_and_add(struct nir_builder *bld,
 180                                    struct u_vector *dead_flrp,
 181                                    struct nir_alu_instr *alu, bool subtract_c)
 182 {
 183    nir_ssa_def *const a = nir_ssa_for_alu_src(bld, alu, 0);
 184    nir_ssa_def *const b = nir_ssa_for_alu_src(bld, alu, 1);
 185    nir_ssa_def *const c = nir_ssa_for_alu_src(bld, alu, 2);
 186
 187    nir_ssa_def *const b_times_c = nir_fmul(bld, b, c);
 188    nir_instr_as_alu(b_times_c->parent_instr)->exact = alu->exact;
 189
 190    nir_ssa_def *inner_sum;
 191
 192    if (subtract_c) {
 193       nir_ssa_def *const neg_c = nir_fneg(bld, c);
 194       nir_instr_as_alu(neg_c->parent_instr)->exact = alu->exact;
 195
 196       inner_sum = nir_fadd(bld, a, neg_c);
 197    } else {
 198       inner_sum = nir_fadd(bld, a, c);
 199    }
 200
 201    nir_instr_as_alu(inner_sum->parent_instr)->exact = alu->exact;
 202
 203    nir_ssa_def *const outer_sum = nir_fadd(bld, inner_sum, b_times_c);
 204    nir_instr_as_alu(outer_sum->parent_instr)->exact = alu->exact;
 205
 206    nir_ssa_def_rewrite_uses(&alu->dest.dest.ssa, nir_src_for_ssa(outer_sum));
 207
 208    /* DO NOT REMOVE the original flrp yet.  Many of the lowering choices are
 209     * based on other uses of the sources.  Removing the flrp may cause the
 210     * last flrp in a sequence to make a different, incorrect choice.
 211     */
 212    append_flrp_to_dead_list(dead_flrp, alu);
 213 }
 214
 215 /**
 216  * Determines whether a swizzled source is constant w/ all components the same.
 217  *
 218  * The value of the constant is stored in \c result.
 219  *
 220  * \return
 221  * True if all components of the swizzled source are the same constant.
 222  * Otherwise false is returned.
 223  */
 224 static bool
 225 all_same_constant(const nir_alu_instr *instr, unsigned src, double *result)
 226 {
 227    nir_const_value *val = nir_src_as_const_value(instr->src[src].src);
 228
 229    if (!val)
 230       return false;
 231
 232    const uint8_t *const swizzle = instr->src[src].swizzle;
 233    const unsigned num_components = nir_dest_num_components(instr->dest.dest);
 234
 235    if (instr->dest.dest.ssa.bit_size == 32) {
 236       const float first = val[swizzle[0]].f32;
 237
 238       for (unsigned i = 1; i < num_components; i++) {
 239          if (val[swizzle[i]].f32 != first)
 240             return false;
 241       }
 242
 243       *result = first;
 244    } else {
 245       const double first = val[swizzle[0]].f64;
 246
 247       for (unsigned i = 1; i < num_components; i++) {
 248          if (val[swizzle[i]].f64 != first)
 249             return false;
 250       }
 251
 252       *result = first;
 253    }
 254
 255    return true;
 256 }
 257
 258 static bool
 259 sources_are_constants_with_similar_magnitudes(const nir_alu_instr *instr)
 260 {
 261    nir_const_value *val0 = nir_src_as_const_value(instr->src[0].src);
 262    nir_const_value *val1 = nir_src_as_const_value(instr->src[1].src);
 263
 264    if (val0 == NULL || val1 == NULL)
 265       return false;
 266
 267    const uint8_t *const swizzle0 = instr->src[0].swizzle;
 268    const uint8_t *const swizzle1 = instr->src[1].swizzle;
 269    const unsigned num_components = nir_dest_num_components(instr->dest.dest);
 270
 271    if (instr->dest.dest.ssa.bit_size == 32) {
 272       for (unsigned i = 0; i < num_components; i++) {
 273          int exp0;
 274          int exp1;
 275
 276          frexpf(val0[swizzle0[i]].f32, &exp0);
 277          frexpf(val1[swizzle1[i]].f32, &exp1);
 278
 279          /* If the difference between exponents is >= 24, then A+B will always
 280           * have the value whichever between A and B has the largest absolute
 281           * value.  So, [0, 23] is the valid range.  The smaller the limit
 282           * value, the more precision will be maintained at a potential
 283           * performance cost.  Somewhat arbitrarilly split the range in half.
 284           */
 285          if (abs(exp0 - exp1) > (23 / 2))
 286             return false;
 287       }
 288    } else {
 289       for (unsigned i = 0; i < num_components; i++) {
 290          int exp0;
 291          int exp1;
 292
 293          frexp(val0[swizzle0[i]].f64, &exp0);
 294          frexp(val1[swizzle1[i]].f64, &exp1);
 295
 296          /* If the difference between exponents is >= 53, then A+B will always
 297           * have the value whichever between A and B has the largest absolute
 298           * value.  So, [0, 52] is the valid range.  The smaller the limit
 299           * value, the more precision will be maintained at a potential
 300           * performance cost.  Somewhat arbitrarilly split the range in half.
 301           */
 302          if (abs(exp0 - exp1) > (52 / 2))
 303             return false;
 304       }
 305    }
 306
 307    return true;
 308 }
 309
 310 /**
 311  * Counts of similar types of nir_op_flrp instructions
 312  *
 313  * If a similar instruction fits into more than one category, it will only be
 314  * counted once.  The assumption is that no other instruction will have all
 315  * sources the same, or CSE would have removed one of the instructions.
 316  */
 317 struct similar_flrp_stats {
 318    unsigned src2;
 319    unsigned src0_and_src2;
 320    unsigned src1_and_src2;
 321 };
 322
 323 /**
 324  * Collection counts of similar FLRP instructions.
 325  *
 326  * This function only cares about similar instructions that have src2 in
 327  * common.
 328  */
 329 static void
 330 get_similar_flrp_stats(nir_alu_instr *alu, struct similar_flrp_stats *st)
 331 {
 332    memset(st, 0, sizeof(*st));
 333
 334    nir_foreach_use(other_use, alu->src[2].src.ssa) {
 335       /* Is the use also a flrp? */
 336       nir_instr *const other_instr = other_use->parent_instr;
 337       if (other_instr->type != nir_instr_type_alu)
 338          continue;
 339
 340       /* Eh-hem... don't match the instruction with itself. */
 341       if (other_instr == &alu->instr)
 342          continue;
 343
 344       nir_alu_instr *const other_alu = nir_instr_as_alu(other_instr);
 345       if (other_alu->op != nir_op_flrp)
 346          continue;
 347
 348       /* Does the other flrp use source 2 from the first flrp as its source 2
 349        * as well?
 350        */
 351       if (!nir_alu_srcs_equal(alu, other_alu, 2, 2))
 352          continue;
 353
 354       if (nir_alu_srcs_equal(alu, other_alu, 0, 0))
 355          st->src0_and_src2++;
 356       else if (nir_alu_srcs_equal(alu, other_alu, 1, 1))
 357          st->src1_and_src2++;
 358       else
 359          st->src2++;
 360    }
 361 }
 362
 363 static void
 364 convert_flrp_instruction(nir_builder *bld,
 365                          struct u_vector *dead_flrp,
 366                          nir_alu_instr *alu,
 367                          bool always_precise,
 368                          bool have_ffma)
 369 {
 370    bld->cursor = nir_before_instr(&alu->instr);
 371
 372    /* There are two methods to implement flrp(x, y, t).  The strictly correct
 373     * implementation according to the GLSL spec is:
 374     *
 375     *    x(1 - t) + yt
 376     *
 377     * This can also be implemented using two chained FMAs
 378     *
 379     *    fma(y, t, fma(-x, t, x))
 380     *
 381     * This method, using either formulation, has better precision when the
 382     * difference between x and y is very large.  It guarantess that flrp(x, y,
 383     * 1) = y.  For example, flrp(1e38, 1.0, 1.0) is 1.0.  This is correct.
 384     *
 385     * The other possible implementation is:
 386     *
 387     *    x + t(y - x)
 388     *
 389     * This can also be formuated as an FMA:
 390     *
 391     *    fma(y - x, t, x)
 392     *
 393     * For this implementation, flrp(1e38, 1.0, 1.0) is 0.0.  Since 1.0 was
 394     * expected, that's a pretty significant error.
 395     *
 396     * The choice made for lowering depends on a number of factors.
 397     *
 398     * - If the flrp is marked precise and FMA is supported:
 399     *
 400     *        fma(y, t, fma(-x, t, x))
 401     *
 402     *   This is strictly correct (maybe?), and the cost is two FMA
 403     *   instructions.  It at least maintains the flrp(x, y, 1.0) == y
 404     *   condition.
 405     *
 406     * - If the flrp is marked precise and FMA is not supported:
 407     *
 408     *        x(1 - t) + yt
 409     *
 410     *   This is strictly correct, and the cost is 4 instructions.  If FMA is
 411     *   supported, this may or may not be reduced to 3 instructions (a
 412     *   subtract, a multiply, and an FMA)... but in that case the other
 413     *   formulation should have been used.
 414     */
 415    if (alu->exact) {
 416       if (have_ffma)
 417          replace_with_strict_ffma(bld, dead_flrp, alu);
 418       else
 419          replace_with_strict(bld, dead_flrp, alu);
 420
 421       return;
 422    }
 423
 424    /*
 425     * - If x and y are both immediates and the relative magnitude of the
 426     *   values is similar (such that x-y does not lose too much precision):
 427     *
 428     *        x + t(x - y)
 429     *
 430     *   We rely on constant folding to eliminate x-y, and we rely on
 431     *   nir_opt_algebraic to possibly generate an FMA.  The cost is either one
 432     *   FMA or two instructions.
 433     */
 434    if (sources_are_constants_with_similar_magnitudes(alu)) {
 435       replace_with_fast(bld, dead_flrp, alu);
 436       return;
 437    }
 438
 439    /*
 440     * - If x = 1:
 441     *
 442     *        (yt + -t) + 1
 443     *
 444     * - If x = -1:
 445     *
 446     *        (yt + t) - 1
 447     *
 448     *   In both cases, x is used in place of ±1 for simplicity.  Both forms
 449     *   lend to ffma generation on platforms that support ffma.
 450     */
 451    double src0_as_constant;
 452    if (all_same_constant(alu, 0, &src0_as_constant)) {
 453       if (src0_as_constant == 1.0) {
 454          replace_with_expanded_ffma_and_add(bld, dead_flrp, alu,
 455                                             true /* subtract t */);
 456          return;
 457       } else if (src0_as_constant == -1.0) {
 458          replace_with_expanded_ffma_and_add(bld, dead_flrp, alu,
 459                                             false /* add t */);
 460          return;
 461       }
 462    }
 463
 464    /*
 465     * - If y = ±1:
 466     *
 467     *        x(1 - t) + yt
 468     *
 469     *   In this case either the multiply in yt will be eliminated by
 470     *   nir_opt_algebraic.  If FMA is supported, this results in fma(x, (1 -
 471     *   t), ±t) for two instructions.  If FMA is not supported, then the cost
 472     *   is 3 instructions.  We rely on nir_opt_algebraic to generate the FMA
 473     *   instructions as well.
 474     *
 475     *   Another possible replacement is
 476     *
 477     *        -xt + x ± t
 478     *
 479     *   Some groupings of this may be better on some platforms in some
 480     *   circumstances, bit it is probably dependent on scheduling.  Futher
 481     *   investigation may be required.
 482     */
 483    double src1_as_constant;
 484    if ((all_same_constant(alu, 1, &src1_as_constant) &&
 485         (src1_as_constant == -1.0 || src1_as_constant == 1.0))) {
 486       replace_with_strict(bld, dead_flrp, alu);
 487       return;
 488    }
 489
 490    if (have_ffma) {
 491       if (always_precise) {
 492          replace_with_strict_ffma(bld, dead_flrp, alu);
 493          return;
 494       }
 495
 496       /*
 497        * - If FMA is supported and other flrp(x, _, t) exists:
 498        *
 499        *        fma(y, t, fma(-x, t, x))
 500        *
 501        *   The hope is that the inner FMA calculation will be shared with the
 502        *   other lowered flrp.  This results in two FMA instructions for the
 503        *   first flrp and one FMA instruction for each additional flrp.  It
 504        *   also means that the live range for x might be complete after the
 505        *   inner ffma instead of after the last flrp.
 506        */
 507       struct similar_flrp_stats st;
 508
 509       get_similar_flrp_stats(alu, &st);
 510       if (st.src0_and_src2 > 0) {
 511          replace_with_strict_ffma(bld, dead_flrp, alu);
 512          return;
 513       }
 514
 515       /*
 516        * - If FMA is supported and another flrp(_, y, t) exists:
 517        *
 518        *        fma(x, (1 - t), yt)
 519        *
 520        *   The hope is that the (1 - t) and the yt will be shared with the
 521        *   other lowered flrp.  This results in 3 insructions for the first
 522        *   flrp and 1 for each additional flrp.
 523        */
 524       if (st.src1_and_src2 > 0) {
 525          replace_with_single_ffma(bld, dead_flrp, alu);
 526          return;
 527       }
 528    } else {
 529       if (always_precise) {
 530          replace_with_strict(bld, dead_flrp, alu);
 531          return;
 532       }
 533
 534       /*
 535        * - If FMA is not supported and another flrp(x, _, t) exists:
 536        *
 537        *        x(1 - t) + yt
 538        *
 539        *   The hope is that the x(1 - t) will be shared with the other lowered
 540        *   flrp.  This results in 4 insructions for the first flrp and 2 for
 541        *   each additional flrp.
 542        *
 543        * - If FMA is not supported and another flrp(_, y, t) exists:
 544        *
 545        *        x(1 - t) + yt
 546        *
 547        *   The hope is that the (1 - t) and the yt will be shared with the
 548        *   other lowered flrp.  This results in 4 insructions for the first
 549        *   flrp and 2 for each additional flrp.
 550        */
 551       struct similar_flrp_stats st;
 552
 553       get_similar_flrp_stats(alu, &st);
 554       if (st.src0_and_src2 > 0 || st.src1_and_src2 > 0) {
 555          replace_with_strict(bld, dead_flrp, alu);
 556          return;
 557       }
 558    }
 559
 560    /*
 561     * - If t is constant:
 562     *
 563     *        x(1 - t) + yt
 564     *
 565     *   The cost is three instructions without FMA or two instructions with
 566     *   FMA.  This is the same cost as the imprecise lowering, but it gives
 567     *   the instruction scheduler a little more freedom.
 568     *
 569     *   There is no need to handle t = 0.5 specially.  nir_opt_algebraic
 570     *   already has optimizations to convert 0.5x + 0.5y to 0.5(x + y).
 571     */
 572    if (alu->src[2].src.ssa->parent_instr->type == nir_instr_type_load_const) {
 573       replace_with_strict(bld, dead_flrp, alu);
 574       return;
 575    }
 576
 577    /*
 578     * - Otherwise
 579     *
 580     *        x + t(x - y)
 581     */
 582    replace_with_fast(bld, dead_flrp, alu);
 583 }
 584
 585 static void
 586 lower_flrp_impl(nir_function_impl *impl,
 587                 struct u_vector *dead_flrp,
 588                 unsigned lowering_mask,
 589                 bool always_precise,
 590                 bool have_ffma)
 591 {
 592    nir_builder b;
 593    nir_builder_init(&b, impl);
 594
 595    nir_foreach_block(block, impl) {
 596       nir_foreach_instr_safe(instr, block) {
 597          if (instr->type == nir_instr_type_alu) {
 598             nir_alu_instr *const alu = nir_instr_as_alu(instr);
 599
 600             if (alu->op == nir_op_flrp &&
 601                 (alu->dest.dest.ssa.bit_size & lowering_mask)) {
 602                convert_flrp_instruction(&b, dead_flrp, alu, always_precise,
 603                                         have_ffma);
 604             }
 605          }
 606       }
 607    }
 608
 609    nir_metadata_preserve(impl, nir_metadata_block_index |
 610                                nir_metadata_dominance);
 611 }
 612
 613 /**
 614  * \param lowering_mask - Bitwise-or of the bit sizes that need to be lowered
 615  *                        (e.g., 16 | 64 if only 16-bit and 64-bit flrp need
 616  *                        lowering).
 617  * \param always_precise - Always require precise lowering for flrp.  This
 618  *                        will always lower flrp to (a * (1 - c)) + (b * c).
 619  * \param have_ffma - Set to true if the GPU has an FFMA instruction that
 620  *                    should be used.
 621  */
 622 bool
 623 nir_lower_flrp(nir_shader *shader,
 624                unsigned lowering_mask,
 625                bool always_precise,
 626                bool have_ffma)
 627 {
 628    struct u_vector dead_flrp;
 629
 630    if (!u_vector_init(&dead_flrp, sizeof(struct nir_alu_instr *), 64))
 631       return false;
 632
 633    nir_foreach_function(function, shader) {
 634       if (function->impl) {
 635          lower_flrp_impl(function->impl, &dead_flrp, lowering_mask,
 636                          always_precise, have_ffma);
 637       }
 638    }
 639
 640    /* Progress was made if the dead list is not empty.  Remove all the
 641     * instructions from the dead list.
 642     */
 643    const bool progress = u_vector_length(&dead_flrp) != 0;
 644
 645    struct nir_alu_instr **instr;
 646    u_vector_foreach(instr, &dead_flrp)
 647       nir_instr_remove(&(*instr)->instr);
 648
 649    u_vector_finish(&dead_flrp);
 650
 651    return progress;
 652 }