2 * Copyright © 2010 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
25 * \file lower_instructions.cpp
27 * Many GPUs lack native instructions for certain expression operations, and
28 * must replace them with some other expression tree. This pass lowers some
29 * of the most common cases, allowing the lowering code to be implemented once
30 * rather than in each driver backend.
32 * Currently supported transformations:
35 * - INT_DIV_TO_MUL_RCP
49 * Breaks an ir_binop_sub expression down to add(op0, neg(op1))
51 * This simplifies expression reassociation, and for many backends
52 * there is no subtract operation separate from adding the negation.
53 * For backends with native subtract operations, they will probably
54 * want to recognize add(op0, neg(op1)) or the other way around to
55 * produce a subtract anyway.
57 * DIV_TO_MUL_RCP and INT_DIV_TO_MUL_RCP:
58 * --------------------------------------
59 * Breaks an ir_binop_div expression down to op0 * (rcp(op1)).
61 * Many GPUs don't have a divide instruction (945 and 965 included),
62 * but they do have an RCP instruction to compute an approximate
63 * reciprocal. By breaking the operation down, constant reciprocals
64 * can get constant folded.
66 * DIV_TO_MUL_RCP only lowers floating point division; INT_DIV_TO_MUL_RCP
67 * handles the integer case, converting to and from floating point so that
70 * EXP_TO_EXP2 and LOG_TO_LOG2:
71 * ----------------------------
72 * Many GPUs don't have a base e log or exponent instruction, but they
73 * do have base 2 versions, so this pass converts exp and log to exp2
74 * and log2 operations.
78 * Many older GPUs don't have an x**y instruction. For these GPUs, convert
79 * x**y to 2**(y * log2(x)).
83 * Breaks an ir_binop_mod expression down to (op0 - op1 * floor(op0 / op1))
85 * Many GPUs don't have a MOD instruction (945 and 965 included), and
86 * if we have to break it down like this anyway, it gives an
87 * opportunity to do things like constant fold the (1.0 / op1) easily.
89 * Note: before we used to implement this as op1 * fract(op / op1) but this
90 * implementation had significant precision errors.
94 * Converts ir_binop_ldexp to arithmetic and bit operations for float sources.
96 * DFREXP_DLDEXP_TO_ARITH:
98 * Converts ir_binop_ldexp, ir_unop_frexp_sig, and ir_unop_frexp_exp to
99 * arithmetic and bit ops for double arguments.
103 * Converts ir_carry into (x + y) < x.
107 * Converts ir_borrow into (x < y).
111 * Converts ir_unop_saturate into min(max(x, 0.0), 1.0)
115 * Converts double trunc, ceil, floor, round to fract
118 #include "c99_math.h"
119 #include "program/prog_instruction.h" /* for swizzle */
120 #include "compiler/glsl_types.h"
122 #include "ir_builder.h"
123 #include "ir_optimization.h"
125 using namespace ir_builder
;
129 class lower_instructions_visitor
: public ir_hierarchical_visitor
{
131 lower_instructions_visitor(unsigned lower
)
132 : progress(false), lower(lower
) { }
134 ir_visitor_status
visit_leave(ir_expression
*);
139 unsigned lower
; /** Bitfield of which operations to lower */
141 void sub_to_add_neg(ir_expression
*);
142 void div_to_mul_rcp(ir_expression
*);
143 void int_div_to_mul_rcp(ir_expression
*);
144 void mod_to_floor(ir_expression
*);
145 void exp_to_exp2(ir_expression
*);
146 void pow_to_exp2(ir_expression
*);
147 void log_to_log2(ir_expression
*);
148 void ldexp_to_arith(ir_expression
*);
149 void dldexp_to_arith(ir_expression
*);
150 void dfrexp_sig_to_arith(ir_expression
*);
151 void dfrexp_exp_to_arith(ir_expression
*);
152 void carry_to_arith(ir_expression
*);
153 void borrow_to_arith(ir_expression
*);
154 void sat_to_clamp(ir_expression
*);
155 void double_dot_to_fma(ir_expression
*);
156 void double_lrp(ir_expression
*);
157 void dceil_to_dfrac(ir_expression
*);
158 void dfloor_to_dfrac(ir_expression
*);
159 void dround_even_to_dfrac(ir_expression
*);
160 void dtrunc_to_dfrac(ir_expression
*);
161 void dsign_to_csel(ir_expression
*);
162 void bit_count_to_math(ir_expression
*);
165 } /* anonymous namespace */
168 * Determine if a particular type of lowering should occur
170 #define lowering(x) (this->lower & x)
173 lower_instructions(exec_list
*instructions
, unsigned what_to_lower
)
175 lower_instructions_visitor
v(what_to_lower
);
177 visit_list_elements(&v
, instructions
);
182 lower_instructions_visitor::sub_to_add_neg(ir_expression
*ir
)
184 ir
->operation
= ir_binop_add
;
185 ir
->operands
[1] = new(ir
) ir_expression(ir_unop_neg
, ir
->operands
[1]->type
,
186 ir
->operands
[1], NULL
);
187 this->progress
= true;
191 lower_instructions_visitor::div_to_mul_rcp(ir_expression
*ir
)
193 assert(ir
->operands
[1]->type
->is_float() || ir
->operands
[1]->type
->is_double());
195 /* New expression for the 1.0 / op1 */
197 expr
= new(ir
) ir_expression(ir_unop_rcp
,
198 ir
->operands
[1]->type
,
201 /* op0 / op1 -> op0 * (1.0 / op1) */
202 ir
->operation
= ir_binop_mul
;
203 ir
->operands
[1] = expr
;
205 this->progress
= true;
209 lower_instructions_visitor::int_div_to_mul_rcp(ir_expression
*ir
)
211 assert(ir
->operands
[1]->type
->is_integer());
213 /* Be careful with integer division -- we need to do it as a
214 * float and re-truncate, since rcp(n > 1) of an integer would
217 ir_rvalue
*op0
, *op1
;
218 const struct glsl_type
*vec_type
;
220 vec_type
= glsl_type::get_instance(GLSL_TYPE_FLOAT
,
221 ir
->operands
[1]->type
->vector_elements
,
222 ir
->operands
[1]->type
->matrix_columns
);
224 if (ir
->operands
[1]->type
->base_type
== GLSL_TYPE_INT
)
225 op1
= new(ir
) ir_expression(ir_unop_i2f
, vec_type
, ir
->operands
[1], NULL
);
227 op1
= new(ir
) ir_expression(ir_unop_u2f
, vec_type
, ir
->operands
[1], NULL
);
229 op1
= new(ir
) ir_expression(ir_unop_rcp
, op1
->type
, op1
, NULL
);
231 vec_type
= glsl_type::get_instance(GLSL_TYPE_FLOAT
,
232 ir
->operands
[0]->type
->vector_elements
,
233 ir
->operands
[0]->type
->matrix_columns
);
235 if (ir
->operands
[0]->type
->base_type
== GLSL_TYPE_INT
)
236 op0
= new(ir
) ir_expression(ir_unop_i2f
, vec_type
, ir
->operands
[0], NULL
);
238 op0
= new(ir
) ir_expression(ir_unop_u2f
, vec_type
, ir
->operands
[0], NULL
);
240 vec_type
= glsl_type::get_instance(GLSL_TYPE_FLOAT
,
241 ir
->type
->vector_elements
,
242 ir
->type
->matrix_columns
);
244 op0
= new(ir
) ir_expression(ir_binop_mul
, vec_type
, op0
, op1
);
246 if (ir
->operands
[1]->type
->base_type
== GLSL_TYPE_INT
) {
247 ir
->operation
= ir_unop_f2i
;
248 ir
->operands
[0] = op0
;
250 ir
->operation
= ir_unop_i2u
;
251 ir
->operands
[0] = new(ir
) ir_expression(ir_unop_f2i
, op0
);
253 ir
->operands
[1] = NULL
;
255 this->progress
= true;
259 lower_instructions_visitor::exp_to_exp2(ir_expression
*ir
)
261 ir_constant
*log2_e
= new(ir
) ir_constant(float(M_LOG2E
));
263 ir
->operation
= ir_unop_exp2
;
264 ir
->operands
[0] = new(ir
) ir_expression(ir_binop_mul
, ir
->operands
[0]->type
,
265 ir
->operands
[0], log2_e
);
266 this->progress
= true;
270 lower_instructions_visitor::pow_to_exp2(ir_expression
*ir
)
272 ir_expression
*const log2_x
=
273 new(ir
) ir_expression(ir_unop_log2
, ir
->operands
[0]->type
,
276 ir
->operation
= ir_unop_exp2
;
277 ir
->operands
[0] = new(ir
) ir_expression(ir_binop_mul
, ir
->operands
[1]->type
,
278 ir
->operands
[1], log2_x
);
279 ir
->operands
[1] = NULL
;
280 this->progress
= true;
284 lower_instructions_visitor::log_to_log2(ir_expression
*ir
)
286 ir
->operation
= ir_binop_mul
;
287 ir
->operands
[0] = new(ir
) ir_expression(ir_unop_log2
, ir
->operands
[0]->type
,
288 ir
->operands
[0], NULL
);
289 ir
->operands
[1] = new(ir
) ir_constant(float(1.0 / M_LOG2E
));
290 this->progress
= true;
294 lower_instructions_visitor::mod_to_floor(ir_expression
*ir
)
296 ir_variable
*x
= new(ir
) ir_variable(ir
->operands
[0]->type
, "mod_x",
298 ir_variable
*y
= new(ir
) ir_variable(ir
->operands
[1]->type
, "mod_y",
300 this->base_ir
->insert_before(x
);
301 this->base_ir
->insert_before(y
);
303 ir_assignment
*const assign_x
=
304 new(ir
) ir_assignment(new(ir
) ir_dereference_variable(x
),
305 ir
->operands
[0], NULL
);
306 ir_assignment
*const assign_y
=
307 new(ir
) ir_assignment(new(ir
) ir_dereference_variable(y
),
308 ir
->operands
[1], NULL
);
310 this->base_ir
->insert_before(assign_x
);
311 this->base_ir
->insert_before(assign_y
);
313 ir_expression
*const div_expr
=
314 new(ir
) ir_expression(ir_binop_div
, x
->type
,
315 new(ir
) ir_dereference_variable(x
),
316 new(ir
) ir_dereference_variable(y
));
318 /* Don't generate new IR that would need to be lowered in an additional
321 if (lowering(DIV_TO_MUL_RCP
) && (ir
->type
->is_float() || ir
->type
->is_double()))
322 div_to_mul_rcp(div_expr
);
324 ir_expression
*const floor_expr
=
325 new(ir
) ir_expression(ir_unop_floor
, x
->type
, div_expr
);
327 if (lowering(DOPS_TO_DFRAC
) && ir
->type
->is_double())
328 dfloor_to_dfrac(floor_expr
);
330 ir_expression
*const mul_expr
=
331 new(ir
) ir_expression(ir_binop_mul
,
332 new(ir
) ir_dereference_variable(y
),
335 ir
->operation
= ir_binop_sub
;
336 ir
->operands
[0] = new(ir
) ir_dereference_variable(x
);
337 ir
->operands
[1] = mul_expr
;
338 this->progress
= true;
342 lower_instructions_visitor::ldexp_to_arith(ir_expression
*ir
)
345 * ir_binop_ldexp x exp
348 * extracted_biased_exp = rshift(bitcast_f2i(abs(x)), exp_shift);
349 * resulting_biased_exp = extracted_biased_exp + exp;
351 * if (resulting_biased_exp < 1 || x == 0.0f) {
352 * return copysign(0.0, x);
355 * return bitcast_u2f((bitcast_f2u(x) & sign_mantissa_mask) |
356 * lshift(i2u(resulting_biased_exp), exp_shift));
358 * which we can't actually implement as such, since the GLSL IR doesn't
359 * have vectorized if-statements. We actually implement it without branches
360 * using conditional-select:
362 * extracted_biased_exp = rshift(bitcast_f2i(abs(x)), exp_shift);
363 * resulting_biased_exp = extracted_biased_exp + exp;
365 * is_not_zero_or_underflow = logic_and(nequal(x, 0.0f),
366 * gequal(resulting_biased_exp, 1);
367 * x = csel(is_not_zero_or_underflow, x, copysign(0.0f, x));
368 * resulting_biased_exp = csel(is_not_zero_or_underflow,
369 * resulting_biased_exp, 0);
371 * return bitcast_u2f((bitcast_f2u(x) & sign_mantissa_mask) |
372 * lshift(i2u(resulting_biased_exp), exp_shift));
375 const unsigned vec_elem
= ir
->type
->vector_elements
;
378 const glsl_type
*ivec
= glsl_type::get_instance(GLSL_TYPE_INT
, vec_elem
, 1);
379 const glsl_type
*bvec
= glsl_type::get_instance(GLSL_TYPE_BOOL
, vec_elem
, 1);
382 ir_constant
*zeroi
= ir_constant::zero(ir
, ivec
);
384 ir_constant
*sign_mask
= new(ir
) ir_constant(0x80000000u
, vec_elem
);
386 ir_constant
*exp_shift
= new(ir
) ir_constant(23, vec_elem
);
387 ir_constant
*exp_width
= new(ir
) ir_constant(8, vec_elem
);
389 /* Temporary variables */
390 ir_variable
*x
= new(ir
) ir_variable(ir
->type
, "x", ir_var_temporary
);
391 ir_variable
*exp
= new(ir
) ir_variable(ivec
, "exp", ir_var_temporary
);
393 ir_variable
*zero_sign_x
= new(ir
) ir_variable(ir
->type
, "zero_sign_x",
396 ir_variable
*extracted_biased_exp
=
397 new(ir
) ir_variable(ivec
, "extracted_biased_exp", ir_var_temporary
);
398 ir_variable
*resulting_biased_exp
=
399 new(ir
) ir_variable(ivec
, "resulting_biased_exp", ir_var_temporary
);
401 ir_variable
*is_not_zero_or_underflow
=
402 new(ir
) ir_variable(bvec
, "is_not_zero_or_underflow", ir_var_temporary
);
404 ir_instruction
&i
= *base_ir
;
406 /* Copy <x> and <exp> arguments. */
408 i
.insert_before(assign(x
, ir
->operands
[0]));
409 i
.insert_before(exp
);
410 i
.insert_before(assign(exp
, ir
->operands
[1]));
412 /* Extract the biased exponent from <x>. */
413 i
.insert_before(extracted_biased_exp
);
414 i
.insert_before(assign(extracted_biased_exp
,
415 rshift(bitcast_f2i(abs(x
)), exp_shift
)));
417 i
.insert_before(resulting_biased_exp
);
418 i
.insert_before(assign(resulting_biased_exp
,
419 add(extracted_biased_exp
, exp
)));
421 /* Test if result is ±0.0, subnormal, or underflow by checking if the
422 * resulting biased exponent would be less than 0x1. If so, the result is
423 * 0.0 with the sign of x. (Actually, invert the conditions so that
424 * immediate values are the second arguments, which is better for i965)
426 i
.insert_before(zero_sign_x
);
427 i
.insert_before(assign(zero_sign_x
,
428 bitcast_u2f(bit_and(bitcast_f2u(x
), sign_mask
))));
430 i
.insert_before(is_not_zero_or_underflow
);
431 i
.insert_before(assign(is_not_zero_or_underflow
,
432 logic_and(nequal(x
, new(ir
) ir_constant(0.0f
, vec_elem
)),
433 gequal(resulting_biased_exp
,
434 new(ir
) ir_constant(0x1, vec_elem
)))));
435 i
.insert_before(assign(x
, csel(is_not_zero_or_underflow
,
437 i
.insert_before(assign(resulting_biased_exp
,
438 csel(is_not_zero_or_underflow
,
439 resulting_biased_exp
, zeroi
)));
441 /* We could test for overflows by checking if the resulting biased exponent
442 * would be greater than 0xFE. Turns out we don't need to because the GLSL
445 * "If this product is too large to be represented in the
446 * floating-point type, the result is undefined."
449 ir_constant
*exp_shift_clone
= exp_shift
->clone(ir
, NULL
);
450 ir
->operation
= ir_unop_bitcast_i2f
;
451 ir
->operands
[0] = bitfield_insert(bitcast_f2i(x
), resulting_biased_exp
,
452 exp_shift_clone
, exp_width
);
453 ir
->operands
[1] = NULL
;
455 this->progress
= true;
459 lower_instructions_visitor::dldexp_to_arith(ir_expression
*ir
)
461 /* See ldexp_to_arith for structure. Uses frexp_exp to extract the exponent
462 * from the significand.
465 const unsigned vec_elem
= ir
->type
->vector_elements
;
468 const glsl_type
*ivec
= glsl_type::get_instance(GLSL_TYPE_INT
, vec_elem
, 1);
469 const glsl_type
*bvec
= glsl_type::get_instance(GLSL_TYPE_BOOL
, vec_elem
, 1);
472 ir_constant
*zeroi
= ir_constant::zero(ir
, ivec
);
474 ir_constant
*sign_mask
= new(ir
) ir_constant(0x80000000u
);
476 ir_constant
*exp_shift
= new(ir
) ir_constant(20u);
477 ir_constant
*exp_width
= new(ir
) ir_constant(11u);
478 ir_constant
*exp_bias
= new(ir
) ir_constant(1022, vec_elem
);
480 /* Temporary variables */
481 ir_variable
*x
= new(ir
) ir_variable(ir
->type
, "x", ir_var_temporary
);
482 ir_variable
*exp
= new(ir
) ir_variable(ivec
, "exp", ir_var_temporary
);
484 ir_variable
*zero_sign_x
= new(ir
) ir_variable(ir
->type
, "zero_sign_x",
487 ir_variable
*extracted_biased_exp
=
488 new(ir
) ir_variable(ivec
, "extracted_biased_exp", ir_var_temporary
);
489 ir_variable
*resulting_biased_exp
=
490 new(ir
) ir_variable(ivec
, "resulting_biased_exp", ir_var_temporary
);
492 ir_variable
*is_not_zero_or_underflow
=
493 new(ir
) ir_variable(bvec
, "is_not_zero_or_underflow", ir_var_temporary
);
495 ir_instruction
&i
= *base_ir
;
497 /* Copy <x> and <exp> arguments. */
499 i
.insert_before(assign(x
, ir
->operands
[0]));
500 i
.insert_before(exp
);
501 i
.insert_before(assign(exp
, ir
->operands
[1]));
503 ir_expression
*frexp_exp
= expr(ir_unop_frexp_exp
, x
);
504 if (lowering(DFREXP_DLDEXP_TO_ARITH
))
505 dfrexp_exp_to_arith(frexp_exp
);
507 /* Extract the biased exponent from <x>. */
508 i
.insert_before(extracted_biased_exp
);
509 i
.insert_before(assign(extracted_biased_exp
, add(frexp_exp
, exp_bias
)));
511 i
.insert_before(resulting_biased_exp
);
512 i
.insert_before(assign(resulting_biased_exp
,
513 add(extracted_biased_exp
, exp
)));
515 /* Test if result is ±0.0, subnormal, or underflow by checking if the
516 * resulting biased exponent would be less than 0x1. If so, the result is
517 * 0.0 with the sign of x. (Actually, invert the conditions so that
518 * immediate values are the second arguments, which is better for i965)
519 * TODO: Implement in a vector fashion.
521 i
.insert_before(zero_sign_x
);
522 for (unsigned elem
= 0; elem
< vec_elem
; elem
++) {
523 ir_variable
*unpacked
=
524 new(ir
) ir_variable(glsl_type::uvec2_type
, "unpacked", ir_var_temporary
);
525 i
.insert_before(unpacked
);
528 expr(ir_unop_unpack_double_2x32
, swizzle(x
, elem
, 1))));
529 i
.insert_before(assign(unpacked
, bit_and(swizzle_y(unpacked
), sign_mask
->clone(ir
, NULL
)),
531 i
.insert_before(assign(unpacked
, ir_constant::zero(ir
, glsl_type::uint_type
), WRITEMASK_X
));
532 i
.insert_before(assign(zero_sign_x
,
533 expr(ir_unop_pack_double_2x32
, unpacked
),
536 i
.insert_before(is_not_zero_or_underflow
);
537 i
.insert_before(assign(is_not_zero_or_underflow
,
538 gequal(resulting_biased_exp
,
539 new(ir
) ir_constant(0x1, vec_elem
))));
540 i
.insert_before(assign(x
, csel(is_not_zero_or_underflow
,
542 i
.insert_before(assign(resulting_biased_exp
,
543 csel(is_not_zero_or_underflow
,
544 resulting_biased_exp
, zeroi
)));
546 /* We could test for overflows by checking if the resulting biased exponent
547 * would be greater than 0xFE. Turns out we don't need to because the GLSL
550 * "If this product is too large to be represented in the
551 * floating-point type, the result is undefined."
554 ir_rvalue
*results
[4] = {NULL
};
555 for (unsigned elem
= 0; elem
< vec_elem
; elem
++) {
556 ir_variable
*unpacked
=
557 new(ir
) ir_variable(glsl_type::uvec2_type
, "unpacked", ir_var_temporary
);
558 i
.insert_before(unpacked
);
561 expr(ir_unop_unpack_double_2x32
, swizzle(x
, elem
, 1))));
563 ir_expression
*bfi
= bitfield_insert(
565 i2u(swizzle(resulting_biased_exp
, elem
, 1)),
566 exp_shift
->clone(ir
, NULL
),
567 exp_width
->clone(ir
, NULL
));
569 i
.insert_before(assign(unpacked
, bfi
, WRITEMASK_Y
));
571 results
[elem
] = expr(ir_unop_pack_double_2x32
, unpacked
);
574 ir
->operation
= ir_quadop_vector
;
575 ir
->operands
[0] = results
[0];
576 ir
->operands
[1] = results
[1];
577 ir
->operands
[2] = results
[2];
578 ir
->operands
[3] = results
[3];
580 /* Don't generate new IR that would need to be lowered in an additional
584 this->progress
= true;
588 lower_instructions_visitor::dfrexp_sig_to_arith(ir_expression
*ir
)
590 const unsigned vec_elem
= ir
->type
->vector_elements
;
591 const glsl_type
*bvec
= glsl_type::get_instance(GLSL_TYPE_BOOL
, vec_elem
, 1);
593 /* Double-precision floating-point values are stored as
598 * We're just extracting the significand here, so we only need to modify
599 * the upper 32-bit uint. Unfortunately we must extract each double
600 * independently as there is no vector version of unpackDouble.
603 ir_instruction
&i
= *base_ir
;
605 ir_variable
*is_not_zero
=
606 new(ir
) ir_variable(bvec
, "is_not_zero", ir_var_temporary
);
607 ir_rvalue
*results
[4] = {NULL
};
609 ir_constant
*dzero
= new(ir
) ir_constant(0.0, vec_elem
);
610 i
.insert_before(is_not_zero
);
613 nequal(abs(ir
->operands
[0]->clone(ir
, NULL
)), dzero
)));
615 /* TODO: Remake this as more vector-friendly when int64 support is
618 for (unsigned elem
= 0; elem
< vec_elem
; elem
++) {
619 ir_constant
*zero
= new(ir
) ir_constant(0u, 1);
620 ir_constant
*sign_mantissa_mask
= new(ir
) ir_constant(0x800fffffu
, 1);
622 /* Exponent of double floating-point values in the range [0.5, 1.0). */
623 ir_constant
*exponent_value
= new(ir
) ir_constant(0x3fe00000u
, 1);
626 new(ir
) ir_variable(glsl_type::uint_type
, "bits", ir_var_temporary
);
627 ir_variable
*unpacked
=
628 new(ir
) ir_variable(glsl_type::uvec2_type
, "unpacked", ir_var_temporary
);
630 ir_rvalue
*x
= swizzle(ir
->operands
[0]->clone(ir
, NULL
), elem
, 1);
632 i
.insert_before(bits
);
633 i
.insert_before(unpacked
);
634 i
.insert_before(assign(unpacked
, expr(ir_unop_unpack_double_2x32
, x
)));
636 /* Manipulate the high uint to remove the exponent and replace it with
637 * either the default exponent or zero.
639 i
.insert_before(assign(bits
, swizzle_y(unpacked
)));
640 i
.insert_before(assign(bits
, bit_and(bits
, sign_mantissa_mask
)));
641 i
.insert_before(assign(bits
, bit_or(bits
,
642 csel(swizzle(is_not_zero
, elem
, 1),
645 i
.insert_before(assign(unpacked
, bits
, WRITEMASK_Y
));
646 results
[elem
] = expr(ir_unop_pack_double_2x32
, unpacked
);
649 /* Put the dvec back together */
650 ir
->operation
= ir_quadop_vector
;
651 ir
->operands
[0] = results
[0];
652 ir
->operands
[1] = results
[1];
653 ir
->operands
[2] = results
[2];
654 ir
->operands
[3] = results
[3];
656 this->progress
= true;
660 lower_instructions_visitor::dfrexp_exp_to_arith(ir_expression
*ir
)
662 const unsigned vec_elem
= ir
->type
->vector_elements
;
663 const glsl_type
*bvec
= glsl_type::get_instance(GLSL_TYPE_BOOL
, vec_elem
, 1);
664 const glsl_type
*uvec
= glsl_type::get_instance(GLSL_TYPE_UINT
, vec_elem
, 1);
666 /* Double-precision floating-point values are stored as
671 * We're just extracting the exponent here, so we only care about the upper
675 ir_instruction
&i
= *base_ir
;
677 ir_variable
*is_not_zero
=
678 new(ir
) ir_variable(bvec
, "is_not_zero", ir_var_temporary
);
679 ir_variable
*high_words
=
680 new(ir
) ir_variable(uvec
, "high_words", ir_var_temporary
);
681 ir_constant
*dzero
= new(ir
) ir_constant(0.0, vec_elem
);
682 ir_constant
*izero
= new(ir
) ir_constant(0, vec_elem
);
684 ir_rvalue
*absval
= abs(ir
->operands
[0]);
686 i
.insert_before(is_not_zero
);
687 i
.insert_before(high_words
);
688 i
.insert_before(assign(is_not_zero
, nequal(absval
->clone(ir
, NULL
), dzero
)));
690 /* Extract all of the upper uints. */
691 for (unsigned elem
= 0; elem
< vec_elem
; elem
++) {
692 ir_rvalue
*x
= swizzle(absval
->clone(ir
, NULL
), elem
, 1);
694 i
.insert_before(assign(high_words
,
695 swizzle_y(expr(ir_unop_unpack_double_2x32
, x
)),
699 ir_constant
*exponent_shift
= new(ir
) ir_constant(20, vec_elem
);
700 ir_constant
*exponent_bias
= new(ir
) ir_constant(-1022, vec_elem
);
702 /* For non-zero inputs, shift the exponent down and apply bias. */
703 ir
->operation
= ir_triop_csel
;
704 ir
->operands
[0] = new(ir
) ir_dereference_variable(is_not_zero
);
705 ir
->operands
[1] = add(exponent_bias
, u2i(rshift(high_words
, exponent_shift
)));
706 ir
->operands
[2] = izero
;
708 this->progress
= true;
712 lower_instructions_visitor::carry_to_arith(ir_expression
*ir
)
717 * sum = ir_binop_add x y
718 * bcarry = ir_binop_less sum x
719 * carry = ir_unop_b2i bcarry
722 ir_rvalue
*x_clone
= ir
->operands
[0]->clone(ir
, NULL
);
723 ir
->operation
= ir_unop_i2u
;
724 ir
->operands
[0] = b2i(less(add(ir
->operands
[0], ir
->operands
[1]), x_clone
));
725 ir
->operands
[1] = NULL
;
727 this->progress
= true;
731 lower_instructions_visitor::borrow_to_arith(ir_expression
*ir
)
734 * ir_binop_borrow x y
736 * bcarry = ir_binop_less x y
737 * carry = ir_unop_b2i bcarry
740 ir
->operation
= ir_unop_i2u
;
741 ir
->operands
[0] = b2i(less(ir
->operands
[0], ir
->operands
[1]));
742 ir
->operands
[1] = NULL
;
744 this->progress
= true;
748 lower_instructions_visitor::sat_to_clamp(ir_expression
*ir
)
753 * ir_binop_min (ir_binop_max(x, 0.0), 1.0)
756 ir
->operation
= ir_binop_min
;
757 ir
->operands
[0] = new(ir
) ir_expression(ir_binop_max
, ir
->operands
[0]->type
,
759 new(ir
) ir_constant(0.0f
));
760 ir
->operands
[1] = new(ir
) ir_constant(1.0f
);
762 this->progress
= true;
766 lower_instructions_visitor::double_dot_to_fma(ir_expression
*ir
)
768 ir_variable
*temp
= new(ir
) ir_variable(ir
->operands
[0]->type
->get_base_type(), "dot_res",
770 this->base_ir
->insert_before(temp
);
772 int nc
= ir
->operands
[0]->type
->components();
773 for (int i
= nc
- 1; i
>= 1; i
--) {
774 ir_assignment
*assig
;
776 assig
= assign(temp
, mul(swizzle(ir
->operands
[0]->clone(ir
, NULL
), i
, 1),
777 swizzle(ir
->operands
[1]->clone(ir
, NULL
), i
, 1)));
779 assig
= assign(temp
, fma(swizzle(ir
->operands
[0]->clone(ir
, NULL
), i
, 1),
780 swizzle(ir
->operands
[1]->clone(ir
, NULL
), i
, 1),
783 this->base_ir
->insert_before(assig
);
786 ir
->operation
= ir_triop_fma
;
787 ir
->operands
[0] = swizzle(ir
->operands
[0], 0, 1);
788 ir
->operands
[1] = swizzle(ir
->operands
[1], 0, 1);
789 ir
->operands
[2] = new(ir
) ir_dereference_variable(temp
);
791 this->progress
= true;
796 lower_instructions_visitor::double_lrp(ir_expression
*ir
)
799 ir_rvalue
*op0
= ir
->operands
[0], *op2
= ir
->operands
[2];
800 ir_constant
*one
= new(ir
) ir_constant(1.0, op2
->type
->vector_elements
);
802 switch (op2
->type
->vector_elements
) {
804 swizval
= SWIZZLE_XXXX
;
807 assert(op0
->type
->vector_elements
== op2
->type
->vector_elements
);
808 swizval
= SWIZZLE_XYZW
;
812 ir
->operation
= ir_triop_fma
;
813 ir
->operands
[0] = swizzle(op2
, swizval
, op0
->type
->vector_elements
);
814 ir
->operands
[2] = mul(sub(one
, op2
->clone(ir
, NULL
)), op0
);
816 this->progress
= true;
820 lower_instructions_visitor::dceil_to_dfrac(ir_expression
*ir
)
824 * temp = sub(x, frtemp);
825 * result = temp + ((frtemp != 0.0) ? 1.0 : 0.0);
827 ir_instruction
&i
= *base_ir
;
828 ir_constant
*zero
= new(ir
) ir_constant(0.0, ir
->operands
[0]->type
->vector_elements
);
829 ir_constant
*one
= new(ir
) ir_constant(1.0, ir
->operands
[0]->type
->vector_elements
);
830 ir_variable
*frtemp
= new(ir
) ir_variable(ir
->operands
[0]->type
, "frtemp",
833 i
.insert_before(frtemp
);
834 i
.insert_before(assign(frtemp
, fract(ir
->operands
[0])));
836 ir
->operation
= ir_binop_add
;
837 ir
->operands
[0] = sub(ir
->operands
[0]->clone(ir
, NULL
), frtemp
);
838 ir
->operands
[1] = csel(nequal(frtemp
, zero
), one
, zero
->clone(ir
, NULL
));
840 this->progress
= true;
844 lower_instructions_visitor::dfloor_to_dfrac(ir_expression
*ir
)
848 * result = sub(x, frtemp);
850 ir
->operation
= ir_binop_sub
;
851 ir
->operands
[1] = fract(ir
->operands
[0]->clone(ir
, NULL
));
853 this->progress
= true;
856 lower_instructions_visitor::dround_even_to_dfrac(ir_expression
*ir
)
861 * frtemp = frac(temp);
862 * t2 = sub(temp, frtemp);
863 * if (frac(x) == 0.5)
864 * result = frac(t2 * 0.5) == 0 ? t2 : t2 - 1;
869 ir_instruction
&i
= *base_ir
;
870 ir_variable
*frtemp
= new(ir
) ir_variable(ir
->operands
[0]->type
, "frtemp",
872 ir_variable
*temp
= new(ir
) ir_variable(ir
->operands
[0]->type
, "temp",
874 ir_variable
*t2
= new(ir
) ir_variable(ir
->operands
[0]->type
, "t2",
876 ir_constant
*p5
= new(ir
) ir_constant(0.5, ir
->operands
[0]->type
->vector_elements
);
877 ir_constant
*one
= new(ir
) ir_constant(1.0, ir
->operands
[0]->type
->vector_elements
);
878 ir_constant
*zero
= new(ir
) ir_constant(0.0, ir
->operands
[0]->type
->vector_elements
);
880 i
.insert_before(temp
);
881 i
.insert_before(assign(temp
, add(ir
->operands
[0], p5
)));
883 i
.insert_before(frtemp
);
884 i
.insert_before(assign(frtemp
, fract(temp
)));
887 i
.insert_before(assign(t2
, sub(temp
, frtemp
)));
889 ir
->operation
= ir_triop_csel
;
890 ir
->operands
[0] = equal(fract(ir
->operands
[0]->clone(ir
, NULL
)),
891 p5
->clone(ir
, NULL
));
892 ir
->operands
[1] = csel(equal(fract(mul(t2
, p5
->clone(ir
, NULL
))),
896 ir
->operands
[2] = new(ir
) ir_dereference_variable(t2
);
898 this->progress
= true;
902 lower_instructions_visitor::dtrunc_to_dfrac(ir_expression
*ir
)
906 * temp = sub(x, frtemp);
907 * result = x >= 0 ? temp : temp + (frtemp == 0.0) ? 0 : 1;
909 ir_rvalue
*arg
= ir
->operands
[0];
910 ir_instruction
&i
= *base_ir
;
912 ir_constant
*zero
= new(ir
) ir_constant(0.0, arg
->type
->vector_elements
);
913 ir_constant
*one
= new(ir
) ir_constant(1.0, arg
->type
->vector_elements
);
914 ir_variable
*frtemp
= new(ir
) ir_variable(arg
->type
, "frtemp",
916 ir_variable
*temp
= new(ir
) ir_variable(ir
->operands
[0]->type
, "temp",
919 i
.insert_before(frtemp
);
920 i
.insert_before(assign(frtemp
, fract(arg
)));
921 i
.insert_before(temp
);
922 i
.insert_before(assign(temp
, sub(arg
->clone(ir
, NULL
), frtemp
)));
924 ir
->operation
= ir_triop_csel
;
925 ir
->operands
[0] = gequal(arg
->clone(ir
, NULL
), zero
);
926 ir
->operands
[1] = new (ir
) ir_dereference_variable(temp
);
927 ir
->operands
[2] = add(temp
,
928 csel(equal(frtemp
, zero
->clone(ir
, NULL
)),
929 zero
->clone(ir
, NULL
),
932 this->progress
= true;
936 lower_instructions_visitor::dsign_to_csel(ir_expression
*ir
)
939 * temp = x > 0.0 ? 1.0 : 0.0;
940 * result = x < 0.0 ? -1.0 : temp;
942 ir_rvalue
*arg
= ir
->operands
[0];
943 ir_constant
*zero
= new(ir
) ir_constant(0.0, arg
->type
->vector_elements
);
944 ir_constant
*one
= new(ir
) ir_constant(1.0, arg
->type
->vector_elements
);
945 ir_constant
*neg_one
= new(ir
) ir_constant(-1.0, arg
->type
->vector_elements
);
947 ir
->operation
= ir_triop_csel
;
948 ir
->operands
[0] = less(arg
->clone(ir
, NULL
),
949 zero
->clone(ir
, NULL
));
950 ir
->operands
[1] = neg_one
;
951 ir
->operands
[2] = csel(greater(arg
, zero
),
953 zero
->clone(ir
, NULL
));
955 this->progress
= true;
959 lower_instructions_visitor::bit_count_to_math(ir_expression
*ir
)
961 /* For more details, see:
963 * http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetPaallel
965 const unsigned elements
= ir
->operands
[0]->type
->vector_elements
;
966 ir_variable
*temp
= new(ir
) ir_variable(glsl_type::uvec(elements
), "temp",
968 ir_constant
*c55555555
= new(ir
) ir_constant(0x55555555u
);
969 ir_constant
*c33333333
= new(ir
) ir_constant(0x33333333u
);
970 ir_constant
*c0F0F0F0F
= new(ir
) ir_constant(0x0F0F0F0Fu
);
971 ir_constant
*c01010101
= new(ir
) ir_constant(0x01010101u
);
972 ir_constant
*c1
= new(ir
) ir_constant(1u);
973 ir_constant
*c2
= new(ir
) ir_constant(2u);
974 ir_constant
*c4
= new(ir
) ir_constant(4u);
975 ir_constant
*c24
= new(ir
) ir_constant(24u);
977 base_ir
->insert_before(temp
);
979 if (ir
->operands
[0]->type
->base_type
== GLSL_TYPE_UINT
) {
980 base_ir
->insert_before(assign(temp
, ir
->operands
[0]));
982 assert(ir
->operands
[0]->type
->base_type
== GLSL_TYPE_INT
);
983 base_ir
->insert_before(assign(temp
, i2u(ir
->operands
[0])));
986 /* temp = temp - ((temp >> 1) & 0x55555555u); */
987 base_ir
->insert_before(assign(temp
, sub(temp
, bit_and(rshift(temp
, c1
),
990 /* temp = (temp & 0x33333333u) + ((temp >> 2) & 0x33333333u); */
991 base_ir
->insert_before(assign(temp
, add(bit_and(temp
, c33333333
),
992 bit_and(rshift(temp
, c2
),
993 c33333333
->clone(ir
, NULL
)))));
995 /* int(((temp + (temp >> 4) & 0xF0F0F0Fu) * 0x1010101u) >> 24); */
996 ir
->operation
= ir_unop_u2i
;
997 ir
->operands
[0] = rshift(mul(bit_and(add(temp
, rshift(temp
, c4
)), c0F0F0F0F
),
1001 this->progress
= true;
1005 lower_instructions_visitor::visit_leave(ir_expression
*ir
)
1007 switch (ir
->operation
) {
1009 if (ir
->operands
[0]->type
->is_double())
1010 double_dot_to_fma(ir
);
1013 if (ir
->operands
[0]->type
->is_double())
1017 if (lowering(SUB_TO_ADD_NEG
))
1022 if (ir
->operands
[1]->type
->is_integer() && lowering(INT_DIV_TO_MUL_RCP
))
1023 int_div_to_mul_rcp(ir
);
1024 else if ((ir
->operands
[1]->type
->is_float() ||
1025 ir
->operands
[1]->type
->is_double()) && lowering(DIV_TO_MUL_RCP
))
1030 if (lowering(EXP_TO_EXP2
))
1035 if (lowering(LOG_TO_LOG2
))
1040 if (lowering(MOD_TO_FLOOR
) && (ir
->type
->is_float() || ir
->type
->is_double()))
1045 if (lowering(POW_TO_EXP2
))
1049 case ir_binop_ldexp
:
1050 if (lowering(LDEXP_TO_ARITH
) && ir
->type
->is_float())
1052 if (lowering(DFREXP_DLDEXP_TO_ARITH
) && ir
->type
->is_double())
1053 dldexp_to_arith(ir
);
1056 case ir_unop_frexp_exp
:
1057 if (lowering(DFREXP_DLDEXP_TO_ARITH
) && ir
->operands
[0]->type
->is_double())
1058 dfrexp_exp_to_arith(ir
);
1061 case ir_unop_frexp_sig
:
1062 if (lowering(DFREXP_DLDEXP_TO_ARITH
) && ir
->operands
[0]->type
->is_double())
1063 dfrexp_sig_to_arith(ir
);
1066 case ir_binop_carry
:
1067 if (lowering(CARRY_TO_ARITH
))
1071 case ir_binop_borrow
:
1072 if (lowering(BORROW_TO_ARITH
))
1073 borrow_to_arith(ir
);
1076 case ir_unop_saturate
:
1077 if (lowering(SAT_TO_CLAMP
))
1082 if (lowering(DOPS_TO_DFRAC
) && ir
->type
->is_double())
1083 dtrunc_to_dfrac(ir
);
1087 if (lowering(DOPS_TO_DFRAC
) && ir
->type
->is_double())
1092 if (lowering(DOPS_TO_DFRAC
) && ir
->type
->is_double())
1093 dfloor_to_dfrac(ir
);
1096 case ir_unop_round_even
:
1097 if (lowering(DOPS_TO_DFRAC
) && ir
->type
->is_double())
1098 dround_even_to_dfrac(ir
);
1102 if (lowering(DOPS_TO_DFRAC
) && ir
->type
->is_double())
1106 case ir_unop_bit_count
:
1107 if (lowering(BIT_COUNT_TO_MATH
))
1108 bit_count_to_math(ir
);
1112 return visit_continue
;
1115 return visit_continue
;