2 * Copyright © 2015 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
26 #include "nir_builder.h"
32 * Lowers some unsupported double operations, using only:
34 * - pack/unpackDouble2x32
35 * - conversion to/from single-precision
36 * - double add, mul, and fma
37 * - conditional select
38 * - 32-bit integer and floating point arithmetic
41 /* Creates a double with the exponent bits set to a given integer value */
43 set_exponent(nir_builder
*b
, nir_ssa_def
*src
, nir_ssa_def
*exp
)
45 /* Split into bits 0-31 and 32-63 */
46 nir_ssa_def
*lo
= nir_unpack_64_2x32_split_x(b
, src
);
47 nir_ssa_def
*hi
= nir_unpack_64_2x32_split_y(b
, src
);
49 /* The exponent is bits 52-62, or 20-30 of the high word, so set the exponent
52 nir_ssa_def
*new_hi
= nir_bitfield_insert(b
, hi
, exp
,
56 return nir_pack_64_2x32_split(b
, lo
, new_hi
);
60 get_exponent(nir_builder
*b
, nir_ssa_def
*src
)
63 nir_ssa_def
*hi
= nir_unpack_64_2x32_split_y(b
, src
);
65 /* extract bits 20-30 of the high word */
66 return nir_ubitfield_extract(b
, hi
, nir_imm_int(b
, 20), nir_imm_int(b
, 11));
69 /* Return infinity with the sign of the given source which is +/-0 */
72 get_signed_inf(nir_builder
*b
, nir_ssa_def
*zero
)
74 nir_ssa_def
*zero_hi
= nir_unpack_64_2x32_split_y(b
, zero
);
76 /* The bit pattern for infinity is 0x7ff0000000000000, where the sign bit
77 * is the highest bit. Only the sign bit can be non-zero in the passed in
78 * source. So we essentially need to OR the infinity and the zero, except
79 * the low 32 bits are always 0 so we can construct the correct high 32
80 * bits and then pack it together with zero low 32 bits.
82 nir_ssa_def
*inf_hi
= nir_ior(b
, nir_imm_int(b
, 0x7ff00000), zero_hi
);
83 return nir_pack_64_2x32_split(b
, nir_imm_int(b
, 0), inf_hi
);
87 * Generates the correctly-signed infinity if the source was zero, and flushes
88 * the result to 0 if the source was infinity or the calculated exponent was
89 * too small to be representable.
93 fix_inv_result(nir_builder
*b
, nir_ssa_def
*res
, nir_ssa_def
*src
,
96 /* If the exponent is too small or the original input was infinity/NaN,
97 * force the result to 0 (flush denorms) to avoid the work of handling
98 * denorms properly. Note that this doesn't preserve positive/negative
99 * zeros, but GLSL doesn't require it.
101 res
= nir_bcsel(b
, nir_ior(b
, nir_ige(b
, nir_imm_int(b
, 0), exp
),
102 nir_feq(b
, nir_fabs(b
, src
),
103 nir_imm_double(b
, INFINITY
))),
104 nir_imm_double(b
, 0.0f
), res
);
106 /* If the original input was 0, generate the correctly-signed infinity */
107 res
= nir_bcsel(b
, nir_fne(b
, src
, nir_imm_double(b
, 0.0f
)),
108 res
, get_signed_inf(b
, src
));
115 lower_rcp(nir_builder
*b
, nir_ssa_def
*src
)
117 /* normalize the input to avoid range issues */
118 nir_ssa_def
*src_norm
= set_exponent(b
, src
, nir_imm_int(b
, 1023));
120 /* cast to float, do an rcp, and then cast back to get an approximate
123 nir_ssa_def
*ra
= nir_f2f64(b
, nir_frcp(b
, nir_f2f32(b
, src_norm
)));
125 /* Fixup the exponent of the result - note that we check if this is too
128 nir_ssa_def
*new_exp
= nir_isub(b
, get_exponent(b
, ra
),
129 nir_isub(b
, get_exponent(b
, src
),
130 nir_imm_int(b
, 1023)));
132 ra
= set_exponent(b
, ra
, new_exp
);
134 /* Do a few Newton-Raphson steps to improve precision.
136 * Each step doubles the precision, and we started off with around 24 bits,
137 * so we only need to do 2 steps to get to full precision. The step is:
139 * x_new = x * (2 - x*src)
141 * But we can re-arrange this to improve precision by using another fused
144 * x_new = x + x * (1 - x*src)
146 * See https://en.wikipedia.org/wiki/Division_algorithm for more details.
149 ra
= nir_ffma(b
, nir_fneg(b
, ra
), nir_ffma(b
, ra
, src
, nir_imm_double(b
, -1)), ra
);
150 ra
= nir_ffma(b
, nir_fneg(b
, ra
), nir_ffma(b
, ra
, src
, nir_imm_double(b
, -1)), ra
);
152 return fix_inv_result(b
, ra
, src
, new_exp
);
156 lower_sqrt_rsq(nir_builder
*b
, nir_ssa_def
*src
, bool sqrt
)
158 /* We want to compute:
162 * When the exponent is even, this is equivalent to:
164 * 1/sqrt(m) * 2^(-e/2)
166 * and then the exponent is odd, this is equal to:
168 * 1/sqrt(m * 2) * 2^(-(e - 1)/2)
170 * where the m * 2 is absorbed into the exponent. So we want the exponent
171 * inside the square root to be 1 if e is odd and 0 if e is even, and we
172 * want to subtract off e/2 from the final exponent, rounded to negative
173 * infinity. We can do the former by first computing the unbiased exponent,
174 * and then AND'ing it with 1 to get 0 or 1, and we can do the latter by
175 * shifting right by 1.
178 nir_ssa_def
*unbiased_exp
= nir_isub(b
, get_exponent(b
, src
),
179 nir_imm_int(b
, 1023));
180 nir_ssa_def
*even
= nir_iand(b
, unbiased_exp
, nir_imm_int(b
, 1));
181 nir_ssa_def
*half
= nir_ishr(b
, unbiased_exp
, nir_imm_int(b
, 1));
183 nir_ssa_def
*src_norm
= set_exponent(b
, src
,
184 nir_iadd(b
, nir_imm_int(b
, 1023),
187 nir_ssa_def
*ra
= nir_f2f64(b
, nir_frsq(b
, nir_f2f32(b
, src_norm
)));
188 nir_ssa_def
*new_exp
= nir_isub(b
, get_exponent(b
, ra
), half
);
189 ra
= set_exponent(b
, ra
, new_exp
);
192 * The following implements an iterative algorithm that's very similar
193 * between sqrt and rsqrt. We start with an iteration of Goldschmit's
194 * algorithm, which looks like:
197 * y_0 = initial (single-precision) rsqrt estimate
201 * r_0 = .5 - h_0 * g_0
202 * g_1 = g_0 * r_0 + g_0
203 * h_1 = h_0 * r_0 + h_0
205 * Now g_1 ~= sqrt(a), and h_1 ~= 1/(2 * sqrt(a)). We could continue
206 * applying another round of Goldschmit, but since we would never refer
207 * back to a (the original source), we would add too much rounding error.
208 * So instead, we do one last round of Newton-Raphson, which has better
209 * rounding characteristics, to get the final rounding correct. This is
210 * split into two cases:
214 * Normally, doing a round of Newton-Raphson for sqrt involves taking a
215 * reciprocal of the original estimate, which is slow since it isn't
216 * supported in HW. But we can take advantage of the fact that we already
217 * computed a good estimate of 1/(2 * g_1) by rearranging it like so:
219 * g_2 = .5 * (g_1 + a / g_1)
220 * = g_1 + .5 * (a / g_1 - g_1)
221 * = g_1 + (.5 / g_1) * (a - g_1^2)
222 * = g_1 + h_1 * (a - g_1^2)
224 * The second term represents the error, and by splitting it out we can get
225 * better precision by computing it as part of a fused multiply-add. Since
226 * both Newton-Raphson and Goldschmit approximately double the precision of
227 * the result, these two steps should be enough.
231 * First off, note that the first round of the Goldschmit algorithm is
232 * really just a Newton-Raphson step in disguise:
234 * h_1 = h_0 * (.5 - h_0 * g_0) + h_0
235 * = h_0 * (1.5 - h_0 * g_0)
236 * = h_0 * (1.5 - .5 * a * y_0^2)
237 * = (.5 * y_0) * (1.5 - .5 * a * y_0^2)
239 * which is the standard formula multiplied by .5. Unlike in the sqrt case,
240 * we don't need the inverse to do a Newton-Raphson step; we just need h_1,
241 * so we can skip the calculation of g_1. Instead, we simply do another
242 * Newton-Raphson step:
245 * r_1 = .5 - h_1 * y_1 * a
246 * y_2 = y_1 * r_1 + y_1
248 * Where the difference from Goldschmit is that we calculate y_1 * a
249 * instead of using g_1. Doing it this way should be as fast as computing
250 * y_1 up front instead of h_1, and it lets us share the code for the
251 * initial Goldschmit step with the sqrt case.
253 * Putting it together, the computations are:
257 * r_0 = .5 - h_0 * g_0
258 * h_1 = h_0 * r_0 + h_0
260 * g_1 = g_0 * r_0 + g_0
261 * r_1 = a - g_1 * g_1
262 * g_2 = h_1 * r_1 + g_1
265 * r_1 = .5 - y_1 * (h_1 * a)
266 * y_2 = y_1 * r_1 + y_1
268 * For more on the ideas behind this, see "Software Division and Square
269 * Root Using Goldschmit's Algorithms" by Markstein and the Wikipedia page
271 * (https://en.wikipedia.org/wiki/Methods_of_computing_square_roots).
274 nir_ssa_def
*one_half
= nir_imm_double(b
, 0.5);
275 nir_ssa_def
*h_0
= nir_fmul(b
, one_half
, ra
);
276 nir_ssa_def
*g_0
= nir_fmul(b
, src
, ra
);
277 nir_ssa_def
*r_0
= nir_ffma(b
, nir_fneg(b
, h_0
), g_0
, one_half
);
278 nir_ssa_def
*h_1
= nir_ffma(b
, h_0
, r_0
, h_0
);
281 nir_ssa_def
*g_1
= nir_ffma(b
, g_0
, r_0
, g_0
);
282 nir_ssa_def
*r_1
= nir_ffma(b
, nir_fneg(b
, g_1
), g_1
, src
);
283 res
= nir_ffma(b
, h_1
, r_1
, g_1
);
285 nir_ssa_def
*y_1
= nir_fmul(b
, nir_imm_double(b
, 2.0), h_1
);
286 nir_ssa_def
*r_1
= nir_ffma(b
, nir_fneg(b
, y_1
), nir_fmul(b
, h_1
, src
),
288 res
= nir_ffma(b
, y_1
, r_1
, y_1
);
292 /* Here, the special cases we need to handle are
296 const bool preserve_denorms
=
297 b
->shader
->info
.float_controls_execution_mode
&
298 FLOAT_CONTROLS_DENORM_PRESERVE_FP64
;
299 nir_ssa_def
*src_flushed
= src
;
300 if (!preserve_denorms
) {
301 src_flushed
= nir_bcsel(b
,
302 nir_flt(b
, nir_fabs(b
, src
),
303 nir_imm_double(b
, DBL_MIN
)),
304 nir_imm_double(b
, 0.0),
307 res
= nir_bcsel(b
, nir_ior(b
, nir_feq(b
, src_flushed
, nir_imm_double(b
, 0.0)),
308 nir_feq(b
, src
, nir_imm_double(b
, INFINITY
))),
311 res
= fix_inv_result(b
, res
, src
, new_exp
);
318 lower_trunc(nir_builder
*b
, nir_ssa_def
*src
)
320 nir_ssa_def
*unbiased_exp
= nir_isub(b
, get_exponent(b
, src
),
321 nir_imm_int(b
, 1023));
323 nir_ssa_def
*frac_bits
= nir_isub(b
, nir_imm_int(b
, 52), unbiased_exp
);
326 * Decide the operation to apply depending on the unbiased exponent:
328 * if (unbiased_exp < 0)
330 * else if (unbiased_exp > 52)
333 * return src & (~0 << frac_bits)
335 * Notice that the else branch is a 64-bit integer operation that we need
336 * to implement in terms of 32-bit integer arithmetics (at least until we
337 * support 64-bit integer arithmetics).
340 /* Compute "~0 << frac_bits" in terms of hi/lo 32-bit integer math */
341 nir_ssa_def
*mask_lo
=
343 nir_ige(b
, frac_bits
, nir_imm_int(b
, 32)),
345 nir_ishl(b
, nir_imm_int(b
, ~0), frac_bits
));
347 nir_ssa_def
*mask_hi
=
349 nir_ilt(b
, frac_bits
, nir_imm_int(b
, 33)),
353 nir_isub(b
, frac_bits
, nir_imm_int(b
, 32))));
355 nir_ssa_def
*src_lo
= nir_unpack_64_2x32_split_x(b
, src
);
356 nir_ssa_def
*src_hi
= nir_unpack_64_2x32_split_y(b
, src
);
360 nir_ilt(b
, unbiased_exp
, nir_imm_int(b
, 0)),
361 nir_imm_double(b
, 0.0),
362 nir_bcsel(b
, nir_ige(b
, unbiased_exp
, nir_imm_int(b
, 53)),
364 nir_pack_64_2x32_split(b
,
365 nir_iand(b
, mask_lo
, src_lo
),
366 nir_iand(b
, mask_hi
, src_hi
))));
370 lower_floor(nir_builder
*b
, nir_ssa_def
*src
)
373 * For x >= 0, floor(x) = trunc(x)
375 * - if x is integer, floor(x) = x
376 * - otherwise, floor(x) = trunc(x) - 1
378 nir_ssa_def
*tr
= nir_ftrunc(b
, src
);
379 nir_ssa_def
*positive
= nir_fge(b
, src
, nir_imm_double(b
, 0.0));
381 nir_ior(b
, positive
, nir_feq(b
, src
, tr
)),
383 nir_fsub(b
, tr
, nir_imm_double(b
, 1.0)));
387 lower_ceil(nir_builder
*b
, nir_ssa_def
*src
)
389 /* if x < 0, ceil(x) = trunc(x)
390 * else if (x - trunc(x) == 0), ceil(x) = x
391 * else, ceil(x) = trunc(x) + 1
393 nir_ssa_def
*tr
= nir_ftrunc(b
, src
);
394 nir_ssa_def
*negative
= nir_flt(b
, src
, nir_imm_double(b
, 0.0));
396 nir_ior(b
, negative
, nir_feq(b
, src
, tr
)),
398 nir_fadd(b
, tr
, nir_imm_double(b
, 1.0)));
402 lower_fract(nir_builder
*b
, nir_ssa_def
*src
)
404 return nir_fsub(b
, src
, nir_ffloor(b
, src
));
408 lower_round_even(nir_builder
*b
, nir_ssa_def
*src
)
410 /* Add and subtract 2**52 to round off any fractional bits. */
411 nir_ssa_def
*two52
= nir_imm_double(b
, (double)(1ull << 52));
412 nir_ssa_def
*sign
= nir_iand(b
, nir_unpack_64_2x32_split_y(b
, src
),
413 nir_imm_int(b
, 1ull << 31));
416 nir_ssa_def
*res
= nir_fsub(b
, nir_fadd(b
, nir_fabs(b
, src
), two52
), two52
);
419 return nir_bcsel(b
, nir_flt(b
, nir_fabs(b
, src
), two52
),
420 nir_pack_64_2x32_split(b
, nir_unpack_64_2x32_split_x(b
, res
),
421 nir_ior(b
, nir_unpack_64_2x32_split_y(b
, res
), sign
)), src
);
425 lower_mod(nir_builder
*b
, nir_ssa_def
*src0
, nir_ssa_def
*src1
)
427 /* mod(x,y) = x - y * floor(x/y)
429 * If the division is lowered, it could add some rounding errors that make
430 * floor() to return the quotient minus one when x = N * y. If this is the
431 * case, we should return zero because mod(x, y) output value is [0, y).
432 * But fortunately Vulkan spec allows this kind of errors; from Vulkan
433 * spec, appendix A (Precision and Operation of SPIR-V instructions:
435 * "The OpFRem and OpFMod instructions use cheap approximations of
436 * remainder, and the error can be large due to the discontinuity in
437 * trunc() and floor(). This can produce mathematically unexpected
438 * results in some cases, such as FMod(x,x) computing x rather than 0,
439 * and can also cause the result to have a different sign than the
440 * infinitely precise result."
442 * In practice this means the output value is actually in the interval
445 * While Vulkan states this behaviour explicitly, OpenGL does not, and thus
446 * we need to assume that value should be in range [0, y); but on the other
447 * hand, mod(a,b) is defined as "a - b * floor(a/b)" and OpenGL allows for
448 * some error in division, so a/a could actually end up being 1.0 - 1ULP;
449 * so in this case floor(a/a) would end up as 0, and hence mod(a,a) == a.
451 * In summary, in the practice mod(a,a) can be "a" both for OpenGL and
454 nir_ssa_def
*floor
= nir_ffloor(b
, nir_fdiv(b
, src0
, src1
));
456 return nir_fsub(b
, src0
, nir_fmul(b
, src1
, floor
));
460 lower_doubles_instr_to_soft(nir_builder
*b
, nir_alu_instr
*instr
,
461 const nir_shader
*softfp64
,
462 nir_lower_doubles_options options
)
464 if (!(options
& nir_lower_fp64_full_software
))
467 assert(instr
->dest
.dest
.is_ssa
);
470 const struct glsl_type
*return_type
= glsl_uint64_t_type();
474 if (instr
->src
[0].src
.ssa
->bit_size
!= 64)
476 name
= "__fp64_to_int64";
477 return_type
= glsl_int64_t_type();
480 if (instr
->src
[0].src
.ssa
->bit_size
!= 64)
482 name
= "__fp64_to_uint64";
485 name
= "__fp32_to_fp64";
488 name
= "__fp64_to_fp32";
489 return_type
= glsl_float_type();
492 name
= "__fp64_to_int";
493 return_type
= glsl_int_type();
496 name
= "__fp64_to_uint";
497 return_type
= glsl_uint_type();
501 name
= "__fp64_to_bool";
502 return_type
= glsl_bool_type();
505 name
= "__bool_to_fp64";
508 if (instr
->src
[0].src
.ssa
->bit_size
== 64)
509 name
= "__int64_to_fp64";
511 name
= "__int_to_fp64";
514 if (instr
->src
[0].src
.ssa
->bit_size
== 64)
515 name
= "__uint64_to_fp64";
517 name
= "__uint_to_fp64";
525 case nir_op_fround_even
:
542 return_type
= glsl_bool_type();
546 return_type
= glsl_bool_type();
550 return_type
= glsl_bool_type();
554 return_type
= glsl_bool_type();
578 nir_function
*func
= NULL
;
579 nir_foreach_function(function
, softfp64
) {
580 if (strcmp(function
->name
, name
) == 0) {
585 if (!func
|| !func
->impl
) {
586 fprintf(stderr
, "Cannot find function \"%s\"\n", name
);
590 nir_ssa_def
*params
[4] = { NULL
, };
592 nir_variable
*ret_tmp
=
593 nir_local_variable_create(b
->impl
, return_type
, "return_tmp");
594 nir_deref_instr
*ret_deref
= nir_build_deref_var(b
, ret_tmp
);
595 params
[0] = &ret_deref
->dest
.ssa
;
597 assert(nir_op_infos
[instr
->op
].num_inputs
+ 1 == func
->num_params
);
598 for (unsigned i
= 0; i
< nir_op_infos
[instr
->op
].num_inputs
; i
++) {
599 assert(i
+ 1 < ARRAY_SIZE(params
));
600 params
[i
+ 1] = nir_mov_alu(b
, instr
->src
[i
], 1);
603 nir_inline_function_impl(b
, func
->impl
, params
);
605 return nir_load_deref(b
, ret_deref
);
608 nir_lower_doubles_options
609 nir_lower_doubles_op_to_options_mask(nir_op opcode
)
612 case nir_op_frcp
: return nir_lower_drcp
;
613 case nir_op_fsqrt
: return nir_lower_dsqrt
;
614 case nir_op_frsq
: return nir_lower_drsq
;
615 case nir_op_ftrunc
: return nir_lower_dtrunc
;
616 case nir_op_ffloor
: return nir_lower_dfloor
;
617 case nir_op_fceil
: return nir_lower_dceil
;
618 case nir_op_ffract
: return nir_lower_dfract
;
619 case nir_op_fround_even
: return nir_lower_dround_even
;
620 case nir_op_fmod
: return nir_lower_dmod
;
621 case nir_op_fsub
: return nir_lower_dsub
;
622 case nir_op_fdiv
: return nir_lower_ddiv
;
627 struct lower_doubles_data
{
628 const nir_shader
*softfp64
;
629 nir_lower_doubles_options options
;
633 should_lower_double_instr(const nir_instr
*instr
, const void *_data
)
635 const struct lower_doubles_data
*data
= _data
;
636 const nir_lower_doubles_options options
= data
->options
;
638 if (instr
->type
!= nir_instr_type_alu
)
641 const nir_alu_instr
*alu
= nir_instr_as_alu(instr
);
643 assert(alu
->dest
.dest
.is_ssa
);
644 bool is_64
= alu
->dest
.dest
.ssa
.bit_size
== 64;
646 unsigned num_srcs
= nir_op_infos
[alu
->op
].num_inputs
;
647 for (unsigned i
= 0; i
< num_srcs
; i
++) {
648 is_64
|= (nir_src_bit_size(alu
->src
[i
].src
) == 64);
654 if (options
& nir_lower_fp64_full_software
)
657 return options
& nir_lower_doubles_op_to_options_mask(alu
->op
);
661 lower_doubles_instr(nir_builder
*b
, nir_instr
*instr
, void *_data
)
663 const struct lower_doubles_data
*data
= _data
;
664 const nir_lower_doubles_options options
= data
->options
;
665 nir_alu_instr
*alu
= nir_instr_as_alu(instr
);
667 nir_ssa_def
*soft_def
=
668 lower_doubles_instr_to_soft(b
, alu
, data
->softfp64
, options
);
672 if (!(options
& nir_lower_doubles_op_to_options_mask(alu
->op
)))
675 nir_ssa_def
*src
= nir_mov_alu(b
, alu
->src
[0],
676 alu
->dest
.dest
.ssa
.num_components
);
680 return lower_rcp(b
, src
);
682 return lower_sqrt_rsq(b
, src
, true);
684 return lower_sqrt_rsq(b
, src
, false);
686 return lower_trunc(b
, src
);
688 return lower_floor(b
, src
);
690 return lower_ceil(b
, src
);
692 return lower_fract(b
, src
);
693 case nir_op_fround_even
:
694 return lower_round_even(b
, src
);
699 nir_ssa_def
*src1
= nir_mov_alu(b
, alu
->src
[1],
700 alu
->dest
.dest
.ssa
.num_components
);
703 return nir_fmul(b
, src
, nir_frcp(b
, src1
));
705 return nir_fadd(b
, src
, nir_fneg(b
, src1
));
707 return lower_mod(b
, src
, src1
);
709 unreachable("unhandled opcode");
713 unreachable("unhandled opcode");
718 nir_lower_doubles_impl(nir_function_impl
*impl
,
719 const nir_shader
*softfp64
,
720 nir_lower_doubles_options options
)
722 struct lower_doubles_data data
= {
723 .softfp64
= softfp64
,
728 nir_function_impl_lower_instructions(impl
,
729 should_lower_double_instr
,
733 if (progress
&& (options
& nir_lower_fp64_full_software
)) {
734 /* SSA and register indices are completely messed up now */
735 nir_index_ssa_defs(impl
);
736 nir_index_local_regs(impl
);
738 nir_metadata_preserve(impl
, nir_metadata_none
);
740 /* And we have deref casts we need to clean up thanks to function
743 nir_opt_deref_impl(impl
);
744 } else if (progress
) {
745 nir_metadata_preserve(impl
, nir_metadata_block_index
|
746 nir_metadata_dominance
);
748 nir_metadata_preserve(impl
, nir_metadata_all
);
755 nir_lower_doubles(nir_shader
*shader
,
756 const nir_shader
*softfp64
,
757 nir_lower_doubles_options options
)
759 bool progress
= false;
761 nir_foreach_function(function
, shader
) {
762 if (function
->impl
) {
763 progress
|= nir_lower_doubles_impl(function
->impl
, softfp64
, options
);