2 * Copyright © 2010 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
25 * \file lower_instructions.cpp
27 * Many GPUs lack native instructions for certain expression operations, and
28 * must replace them with some other expression tree. This pass lowers some
29 * of the most common cases, allowing the lowering code to be implemented once
30 * rather than in each driver backend.
32 * Currently supported transformations:
35 * - INT_DIV_TO_MUL_RCP
49 * Breaks an ir_binop_sub expression down to add(op0, neg(op1))
51 * This simplifies expression reassociation, and for many backends
52 * there is no subtract operation separate from adding the negation.
53 * For backends with native subtract operations, they will probably
54 * want to recognize add(op0, neg(op1)) or the other way around to
55 * produce a subtract anyway.
57 * DIV_TO_MUL_RCP and INT_DIV_TO_MUL_RCP:
58 * --------------------------------------
59 * Breaks an ir_binop_div expression down to op0 * (rcp(op1)).
61 * Many GPUs don't have a divide instruction (945 and 965 included),
62 * but they do have an RCP instruction to compute an approximate
63 * reciprocal. By breaking the operation down, constant reciprocals
64 * can get constant folded.
66 * DIV_TO_MUL_RCP only lowers floating point division; INT_DIV_TO_MUL_RCP
67 * handles the integer case, converting to and from floating point so that
70 * EXP_TO_EXP2 and LOG_TO_LOG2:
71 * ----------------------------
72 * Many GPUs don't have a base e log or exponent instruction, but they
73 * do have base 2 versions, so this pass converts exp and log to exp2
74 * and log2 operations.
78 * Many older GPUs don't have an x**y instruction. For these GPUs, convert
79 * x**y to 2**(y * log2(x)).
83 * Breaks an ir_binop_mod expression down to (op0 - op1 * floor(op0 / op1))
85 * Many GPUs don't have a MOD instruction (945 and 965 included), and
86 * if we have to break it down like this anyway, it gives an
87 * opportunity to do things like constant fold the (1.0 / op1) easily.
89 * Note: before we used to implement this as op1 * fract(op / op1) but this
90 * implementation had significant precision errors.
94 * Converts ir_binop_ldexp to arithmetic and bit operations for float sources.
96 * DFREXP_DLDEXP_TO_ARITH:
98 * Converts ir_binop_ldexp, ir_unop_frexp_sig, and ir_unop_frexp_exp to
99 * arithmetic and bit ops for double arguments.
103 * Converts ir_carry into (x + y) < x.
107 * Converts ir_borrow into (x < y).
111 * Converts ir_unop_saturate into min(max(x, 0.0), 1.0)
115 * Converts double trunc, ceil, floor, round to fract
118 #include "c99_math.h"
119 #include "program/prog_instruction.h" /* for swizzle */
120 #include "compiler/glsl_types.h"
122 #include "ir_builder.h"
123 #include "ir_optimization.h"
125 using namespace ir_builder
;
129 class lower_instructions_visitor
: public ir_hierarchical_visitor
{
131 lower_instructions_visitor(unsigned lower
)
132 : progress(false), lower(lower
) { }
134 ir_visitor_status
visit_leave(ir_expression
*);
139 unsigned lower
; /** Bitfield of which operations to lower */
141 void sub_to_add_neg(ir_expression
*);
142 void div_to_mul_rcp(ir_expression
*);
143 void int_div_to_mul_rcp(ir_expression
*);
144 void mod_to_floor(ir_expression
*);
145 void exp_to_exp2(ir_expression
*);
146 void pow_to_exp2(ir_expression
*);
147 void log_to_log2(ir_expression
*);
148 void ldexp_to_arith(ir_expression
*);
149 void dldexp_to_arith(ir_expression
*);
150 void dfrexp_sig_to_arith(ir_expression
*);
151 void dfrexp_exp_to_arith(ir_expression
*);
152 void carry_to_arith(ir_expression
*);
153 void borrow_to_arith(ir_expression
*);
154 void sat_to_clamp(ir_expression
*);
155 void double_dot_to_fma(ir_expression
*);
156 void double_lrp(ir_expression
*);
157 void dceil_to_dfrac(ir_expression
*);
158 void dfloor_to_dfrac(ir_expression
*);
159 void dround_even_to_dfrac(ir_expression
*);
160 void dtrunc_to_dfrac(ir_expression
*);
161 void dsign_to_csel(ir_expression
*);
162 void bit_count_to_math(ir_expression
*);
163 void extract_to_shifts(ir_expression
*);
164 void insert_to_shifts(ir_expression
*);
165 void reverse_to_shifts(ir_expression
*ir
);
166 void find_lsb_to_float_cast(ir_expression
*ir
);
167 void find_msb_to_float_cast(ir_expression
*ir
);
170 } /* anonymous namespace */
173 * Determine if a particular type of lowering should occur
175 #define lowering(x) (this->lower & x)
178 lower_instructions(exec_list
*instructions
, unsigned what_to_lower
)
180 lower_instructions_visitor
v(what_to_lower
);
182 visit_list_elements(&v
, instructions
);
187 lower_instructions_visitor::sub_to_add_neg(ir_expression
*ir
)
189 ir
->operation
= ir_binop_add
;
190 ir
->operands
[1] = new(ir
) ir_expression(ir_unop_neg
, ir
->operands
[1]->type
,
191 ir
->operands
[1], NULL
);
192 this->progress
= true;
196 lower_instructions_visitor::div_to_mul_rcp(ir_expression
*ir
)
198 assert(ir
->operands
[1]->type
->is_float() || ir
->operands
[1]->type
->is_double());
200 /* New expression for the 1.0 / op1 */
202 expr
= new(ir
) ir_expression(ir_unop_rcp
,
203 ir
->operands
[1]->type
,
206 /* op0 / op1 -> op0 * (1.0 / op1) */
207 ir
->operation
= ir_binop_mul
;
208 ir
->operands
[1] = expr
;
210 this->progress
= true;
214 lower_instructions_visitor::int_div_to_mul_rcp(ir_expression
*ir
)
216 assert(ir
->operands
[1]->type
->is_integer());
218 /* Be careful with integer division -- we need to do it as a
219 * float and re-truncate, since rcp(n > 1) of an integer would
222 ir_rvalue
*op0
, *op1
;
223 const struct glsl_type
*vec_type
;
225 vec_type
= glsl_type::get_instance(GLSL_TYPE_FLOAT
,
226 ir
->operands
[1]->type
->vector_elements
,
227 ir
->operands
[1]->type
->matrix_columns
);
229 if (ir
->operands
[1]->type
->base_type
== GLSL_TYPE_INT
)
230 op1
= new(ir
) ir_expression(ir_unop_i2f
, vec_type
, ir
->operands
[1], NULL
);
232 op1
= new(ir
) ir_expression(ir_unop_u2f
, vec_type
, ir
->operands
[1], NULL
);
234 op1
= new(ir
) ir_expression(ir_unop_rcp
, op1
->type
, op1
, NULL
);
236 vec_type
= glsl_type::get_instance(GLSL_TYPE_FLOAT
,
237 ir
->operands
[0]->type
->vector_elements
,
238 ir
->operands
[0]->type
->matrix_columns
);
240 if (ir
->operands
[0]->type
->base_type
== GLSL_TYPE_INT
)
241 op0
= new(ir
) ir_expression(ir_unop_i2f
, vec_type
, ir
->operands
[0], NULL
);
243 op0
= new(ir
) ir_expression(ir_unop_u2f
, vec_type
, ir
->operands
[0], NULL
);
245 vec_type
= glsl_type::get_instance(GLSL_TYPE_FLOAT
,
246 ir
->type
->vector_elements
,
247 ir
->type
->matrix_columns
);
249 op0
= new(ir
) ir_expression(ir_binop_mul
, vec_type
, op0
, op1
);
251 if (ir
->operands
[1]->type
->base_type
== GLSL_TYPE_INT
) {
252 ir
->operation
= ir_unop_f2i
;
253 ir
->operands
[0] = op0
;
255 ir
->operation
= ir_unop_i2u
;
256 ir
->operands
[0] = new(ir
) ir_expression(ir_unop_f2i
, op0
);
258 ir
->operands
[1] = NULL
;
260 this->progress
= true;
264 lower_instructions_visitor::exp_to_exp2(ir_expression
*ir
)
266 ir_constant
*log2_e
= new(ir
) ir_constant(float(M_LOG2E
));
268 ir
->operation
= ir_unop_exp2
;
269 ir
->operands
[0] = new(ir
) ir_expression(ir_binop_mul
, ir
->operands
[0]->type
,
270 ir
->operands
[0], log2_e
);
271 this->progress
= true;
275 lower_instructions_visitor::pow_to_exp2(ir_expression
*ir
)
277 ir_expression
*const log2_x
=
278 new(ir
) ir_expression(ir_unop_log2
, ir
->operands
[0]->type
,
281 ir
->operation
= ir_unop_exp2
;
282 ir
->operands
[0] = new(ir
) ir_expression(ir_binop_mul
, ir
->operands
[1]->type
,
283 ir
->operands
[1], log2_x
);
284 ir
->operands
[1] = NULL
;
285 this->progress
= true;
289 lower_instructions_visitor::log_to_log2(ir_expression
*ir
)
291 ir
->operation
= ir_binop_mul
;
292 ir
->operands
[0] = new(ir
) ir_expression(ir_unop_log2
, ir
->operands
[0]->type
,
293 ir
->operands
[0], NULL
);
294 ir
->operands
[1] = new(ir
) ir_constant(float(1.0 / M_LOG2E
));
295 this->progress
= true;
299 lower_instructions_visitor::mod_to_floor(ir_expression
*ir
)
301 ir_variable
*x
= new(ir
) ir_variable(ir
->operands
[0]->type
, "mod_x",
303 ir_variable
*y
= new(ir
) ir_variable(ir
->operands
[1]->type
, "mod_y",
305 this->base_ir
->insert_before(x
);
306 this->base_ir
->insert_before(y
);
308 ir_assignment
*const assign_x
=
309 new(ir
) ir_assignment(new(ir
) ir_dereference_variable(x
),
310 ir
->operands
[0], NULL
);
311 ir_assignment
*const assign_y
=
312 new(ir
) ir_assignment(new(ir
) ir_dereference_variable(y
),
313 ir
->operands
[1], NULL
);
315 this->base_ir
->insert_before(assign_x
);
316 this->base_ir
->insert_before(assign_y
);
318 ir_expression
*const div_expr
=
319 new(ir
) ir_expression(ir_binop_div
, x
->type
,
320 new(ir
) ir_dereference_variable(x
),
321 new(ir
) ir_dereference_variable(y
));
323 /* Don't generate new IR that would need to be lowered in an additional
326 if (lowering(DIV_TO_MUL_RCP
) && (ir
->type
->is_float() || ir
->type
->is_double()))
327 div_to_mul_rcp(div_expr
);
329 ir_expression
*const floor_expr
=
330 new(ir
) ir_expression(ir_unop_floor
, x
->type
, div_expr
);
332 if (lowering(DOPS_TO_DFRAC
) && ir
->type
->is_double())
333 dfloor_to_dfrac(floor_expr
);
335 ir_expression
*const mul_expr
=
336 new(ir
) ir_expression(ir_binop_mul
,
337 new(ir
) ir_dereference_variable(y
),
340 ir
->operation
= ir_binop_sub
;
341 ir
->operands
[0] = new(ir
) ir_dereference_variable(x
);
342 ir
->operands
[1] = mul_expr
;
343 this->progress
= true;
347 lower_instructions_visitor::ldexp_to_arith(ir_expression
*ir
)
350 * ir_binop_ldexp x exp
353 * extracted_biased_exp = rshift(bitcast_f2i(abs(x)), exp_shift);
354 * resulting_biased_exp = extracted_biased_exp + exp;
356 * if (resulting_biased_exp < 1 || x == 0.0f) {
357 * return copysign(0.0, x);
360 * return bitcast_u2f((bitcast_f2u(x) & sign_mantissa_mask) |
361 * lshift(i2u(resulting_biased_exp), exp_shift));
363 * which we can't actually implement as such, since the GLSL IR doesn't
364 * have vectorized if-statements. We actually implement it without branches
365 * using conditional-select:
367 * extracted_biased_exp = rshift(bitcast_f2i(abs(x)), exp_shift);
368 * resulting_biased_exp = extracted_biased_exp + exp;
370 * is_not_zero_or_underflow = logic_and(nequal(x, 0.0f),
371 * gequal(resulting_biased_exp, 1);
372 * x = csel(is_not_zero_or_underflow, x, copysign(0.0f, x));
373 * resulting_biased_exp = csel(is_not_zero_or_underflow,
374 * resulting_biased_exp, 0);
376 * return bitcast_u2f((bitcast_f2u(x) & sign_mantissa_mask) |
377 * lshift(i2u(resulting_biased_exp), exp_shift));
380 const unsigned vec_elem
= ir
->type
->vector_elements
;
383 const glsl_type
*ivec
= glsl_type::get_instance(GLSL_TYPE_INT
, vec_elem
, 1);
384 const glsl_type
*bvec
= glsl_type::get_instance(GLSL_TYPE_BOOL
, vec_elem
, 1);
387 ir_constant
*zeroi
= ir_constant::zero(ir
, ivec
);
389 ir_constant
*sign_mask
= new(ir
) ir_constant(0x80000000u
, vec_elem
);
391 ir_constant
*exp_shift
= new(ir
) ir_constant(23, vec_elem
);
392 ir_constant
*exp_width
= new(ir
) ir_constant(8, vec_elem
);
394 /* Temporary variables */
395 ir_variable
*x
= new(ir
) ir_variable(ir
->type
, "x", ir_var_temporary
);
396 ir_variable
*exp
= new(ir
) ir_variable(ivec
, "exp", ir_var_temporary
);
398 ir_variable
*zero_sign_x
= new(ir
) ir_variable(ir
->type
, "zero_sign_x",
401 ir_variable
*extracted_biased_exp
=
402 new(ir
) ir_variable(ivec
, "extracted_biased_exp", ir_var_temporary
);
403 ir_variable
*resulting_biased_exp
=
404 new(ir
) ir_variable(ivec
, "resulting_biased_exp", ir_var_temporary
);
406 ir_variable
*is_not_zero_or_underflow
=
407 new(ir
) ir_variable(bvec
, "is_not_zero_or_underflow", ir_var_temporary
);
409 ir_instruction
&i
= *base_ir
;
411 /* Copy <x> and <exp> arguments. */
413 i
.insert_before(assign(x
, ir
->operands
[0]));
414 i
.insert_before(exp
);
415 i
.insert_before(assign(exp
, ir
->operands
[1]));
417 /* Extract the biased exponent from <x>. */
418 i
.insert_before(extracted_biased_exp
);
419 i
.insert_before(assign(extracted_biased_exp
,
420 rshift(bitcast_f2i(abs(x
)), exp_shift
)));
422 i
.insert_before(resulting_biased_exp
);
423 i
.insert_before(assign(resulting_biased_exp
,
424 add(extracted_biased_exp
, exp
)));
426 /* Test if result is ±0.0, subnormal, or underflow by checking if the
427 * resulting biased exponent would be less than 0x1. If so, the result is
428 * 0.0 with the sign of x. (Actually, invert the conditions so that
429 * immediate values are the second arguments, which is better for i965)
431 i
.insert_before(zero_sign_x
);
432 i
.insert_before(assign(zero_sign_x
,
433 bitcast_u2f(bit_and(bitcast_f2u(x
), sign_mask
))));
435 i
.insert_before(is_not_zero_or_underflow
);
436 i
.insert_before(assign(is_not_zero_or_underflow
,
437 logic_and(nequal(x
, new(ir
) ir_constant(0.0f
, vec_elem
)),
438 gequal(resulting_biased_exp
,
439 new(ir
) ir_constant(0x1, vec_elem
)))));
440 i
.insert_before(assign(x
, csel(is_not_zero_or_underflow
,
442 i
.insert_before(assign(resulting_biased_exp
,
443 csel(is_not_zero_or_underflow
,
444 resulting_biased_exp
, zeroi
)));
446 /* We could test for overflows by checking if the resulting biased exponent
447 * would be greater than 0xFE. Turns out we don't need to because the GLSL
450 * "If this product is too large to be represented in the
451 * floating-point type, the result is undefined."
454 ir_constant
*exp_shift_clone
= exp_shift
->clone(ir
, NULL
);
455 ir
->operation
= ir_unop_bitcast_i2f
;
456 ir
->operands
[0] = bitfield_insert(bitcast_f2i(x
), resulting_biased_exp
,
457 exp_shift_clone
, exp_width
);
458 ir
->operands
[1] = NULL
;
460 this->progress
= true;
464 lower_instructions_visitor::dldexp_to_arith(ir_expression
*ir
)
466 /* See ldexp_to_arith for structure. Uses frexp_exp to extract the exponent
467 * from the significand.
470 const unsigned vec_elem
= ir
->type
->vector_elements
;
473 const glsl_type
*ivec
= glsl_type::get_instance(GLSL_TYPE_INT
, vec_elem
, 1);
474 const glsl_type
*bvec
= glsl_type::get_instance(GLSL_TYPE_BOOL
, vec_elem
, 1);
477 ir_constant
*zeroi
= ir_constant::zero(ir
, ivec
);
479 ir_constant
*sign_mask
= new(ir
) ir_constant(0x80000000u
);
481 ir_constant
*exp_shift
= new(ir
) ir_constant(20u);
482 ir_constant
*exp_width
= new(ir
) ir_constant(11u);
483 ir_constant
*exp_bias
= new(ir
) ir_constant(1022, vec_elem
);
485 /* Temporary variables */
486 ir_variable
*x
= new(ir
) ir_variable(ir
->type
, "x", ir_var_temporary
);
487 ir_variable
*exp
= new(ir
) ir_variable(ivec
, "exp", ir_var_temporary
);
489 ir_variable
*zero_sign_x
= new(ir
) ir_variable(ir
->type
, "zero_sign_x",
492 ir_variable
*extracted_biased_exp
=
493 new(ir
) ir_variable(ivec
, "extracted_biased_exp", ir_var_temporary
);
494 ir_variable
*resulting_biased_exp
=
495 new(ir
) ir_variable(ivec
, "resulting_biased_exp", ir_var_temporary
);
497 ir_variable
*is_not_zero_or_underflow
=
498 new(ir
) ir_variable(bvec
, "is_not_zero_or_underflow", ir_var_temporary
);
500 ir_instruction
&i
= *base_ir
;
502 /* Copy <x> and <exp> arguments. */
504 i
.insert_before(assign(x
, ir
->operands
[0]));
505 i
.insert_before(exp
);
506 i
.insert_before(assign(exp
, ir
->operands
[1]));
508 ir_expression
*frexp_exp
= expr(ir_unop_frexp_exp
, x
);
509 if (lowering(DFREXP_DLDEXP_TO_ARITH
))
510 dfrexp_exp_to_arith(frexp_exp
);
512 /* Extract the biased exponent from <x>. */
513 i
.insert_before(extracted_biased_exp
);
514 i
.insert_before(assign(extracted_biased_exp
, add(frexp_exp
, exp_bias
)));
516 i
.insert_before(resulting_biased_exp
);
517 i
.insert_before(assign(resulting_biased_exp
,
518 add(extracted_biased_exp
, exp
)));
520 /* Test if result is ±0.0, subnormal, or underflow by checking if the
521 * resulting biased exponent would be less than 0x1. If so, the result is
522 * 0.0 with the sign of x. (Actually, invert the conditions so that
523 * immediate values are the second arguments, which is better for i965)
524 * TODO: Implement in a vector fashion.
526 i
.insert_before(zero_sign_x
);
527 for (unsigned elem
= 0; elem
< vec_elem
; elem
++) {
528 ir_variable
*unpacked
=
529 new(ir
) ir_variable(glsl_type::uvec2_type
, "unpacked", ir_var_temporary
);
530 i
.insert_before(unpacked
);
533 expr(ir_unop_unpack_double_2x32
, swizzle(x
, elem
, 1))));
534 i
.insert_before(assign(unpacked
, bit_and(swizzle_y(unpacked
), sign_mask
->clone(ir
, NULL
)),
536 i
.insert_before(assign(unpacked
, ir_constant::zero(ir
, glsl_type::uint_type
), WRITEMASK_X
));
537 i
.insert_before(assign(zero_sign_x
,
538 expr(ir_unop_pack_double_2x32
, unpacked
),
541 i
.insert_before(is_not_zero_or_underflow
);
542 i
.insert_before(assign(is_not_zero_or_underflow
,
543 gequal(resulting_biased_exp
,
544 new(ir
) ir_constant(0x1, vec_elem
))));
545 i
.insert_before(assign(x
, csel(is_not_zero_or_underflow
,
547 i
.insert_before(assign(resulting_biased_exp
,
548 csel(is_not_zero_or_underflow
,
549 resulting_biased_exp
, zeroi
)));
551 /* We could test for overflows by checking if the resulting biased exponent
552 * would be greater than 0xFE. Turns out we don't need to because the GLSL
555 * "If this product is too large to be represented in the
556 * floating-point type, the result is undefined."
559 ir_rvalue
*results
[4] = {NULL
};
560 for (unsigned elem
= 0; elem
< vec_elem
; elem
++) {
561 ir_variable
*unpacked
=
562 new(ir
) ir_variable(glsl_type::uvec2_type
, "unpacked", ir_var_temporary
);
563 i
.insert_before(unpacked
);
566 expr(ir_unop_unpack_double_2x32
, swizzle(x
, elem
, 1))));
568 ir_expression
*bfi
= bitfield_insert(
570 i2u(swizzle(resulting_biased_exp
, elem
, 1)),
571 exp_shift
->clone(ir
, NULL
),
572 exp_width
->clone(ir
, NULL
));
574 i
.insert_before(assign(unpacked
, bfi
, WRITEMASK_Y
));
576 results
[elem
] = expr(ir_unop_pack_double_2x32
, unpacked
);
579 ir
->operation
= ir_quadop_vector
;
580 ir
->operands
[0] = results
[0];
581 ir
->operands
[1] = results
[1];
582 ir
->operands
[2] = results
[2];
583 ir
->operands
[3] = results
[3];
585 /* Don't generate new IR that would need to be lowered in an additional
589 this->progress
= true;
593 lower_instructions_visitor::dfrexp_sig_to_arith(ir_expression
*ir
)
595 const unsigned vec_elem
= ir
->type
->vector_elements
;
596 const glsl_type
*bvec
= glsl_type::get_instance(GLSL_TYPE_BOOL
, vec_elem
, 1);
598 /* Double-precision floating-point values are stored as
603 * We're just extracting the significand here, so we only need to modify
604 * the upper 32-bit uint. Unfortunately we must extract each double
605 * independently as there is no vector version of unpackDouble.
608 ir_instruction
&i
= *base_ir
;
610 ir_variable
*is_not_zero
=
611 new(ir
) ir_variable(bvec
, "is_not_zero", ir_var_temporary
);
612 ir_rvalue
*results
[4] = {NULL
};
614 ir_constant
*dzero
= new(ir
) ir_constant(0.0, vec_elem
);
615 i
.insert_before(is_not_zero
);
618 nequal(abs(ir
->operands
[0]->clone(ir
, NULL
)), dzero
)));
620 /* TODO: Remake this as more vector-friendly when int64 support is
623 for (unsigned elem
= 0; elem
< vec_elem
; elem
++) {
624 ir_constant
*zero
= new(ir
) ir_constant(0u, 1);
625 ir_constant
*sign_mantissa_mask
= new(ir
) ir_constant(0x800fffffu
, 1);
627 /* Exponent of double floating-point values in the range [0.5, 1.0). */
628 ir_constant
*exponent_value
= new(ir
) ir_constant(0x3fe00000u
, 1);
631 new(ir
) ir_variable(glsl_type::uint_type
, "bits", ir_var_temporary
);
632 ir_variable
*unpacked
=
633 new(ir
) ir_variable(glsl_type::uvec2_type
, "unpacked", ir_var_temporary
);
635 ir_rvalue
*x
= swizzle(ir
->operands
[0]->clone(ir
, NULL
), elem
, 1);
637 i
.insert_before(bits
);
638 i
.insert_before(unpacked
);
639 i
.insert_before(assign(unpacked
, expr(ir_unop_unpack_double_2x32
, x
)));
641 /* Manipulate the high uint to remove the exponent and replace it with
642 * either the default exponent or zero.
644 i
.insert_before(assign(bits
, swizzle_y(unpacked
)));
645 i
.insert_before(assign(bits
, bit_and(bits
, sign_mantissa_mask
)));
646 i
.insert_before(assign(bits
, bit_or(bits
,
647 csel(swizzle(is_not_zero
, elem
, 1),
650 i
.insert_before(assign(unpacked
, bits
, WRITEMASK_Y
));
651 results
[elem
] = expr(ir_unop_pack_double_2x32
, unpacked
);
654 /* Put the dvec back together */
655 ir
->operation
= ir_quadop_vector
;
656 ir
->operands
[0] = results
[0];
657 ir
->operands
[1] = results
[1];
658 ir
->operands
[2] = results
[2];
659 ir
->operands
[3] = results
[3];
661 this->progress
= true;
665 lower_instructions_visitor::dfrexp_exp_to_arith(ir_expression
*ir
)
667 const unsigned vec_elem
= ir
->type
->vector_elements
;
668 const glsl_type
*bvec
= glsl_type::get_instance(GLSL_TYPE_BOOL
, vec_elem
, 1);
669 const glsl_type
*uvec
= glsl_type::get_instance(GLSL_TYPE_UINT
, vec_elem
, 1);
671 /* Double-precision floating-point values are stored as
676 * We're just extracting the exponent here, so we only care about the upper
680 ir_instruction
&i
= *base_ir
;
682 ir_variable
*is_not_zero
=
683 new(ir
) ir_variable(bvec
, "is_not_zero", ir_var_temporary
);
684 ir_variable
*high_words
=
685 new(ir
) ir_variable(uvec
, "high_words", ir_var_temporary
);
686 ir_constant
*dzero
= new(ir
) ir_constant(0.0, vec_elem
);
687 ir_constant
*izero
= new(ir
) ir_constant(0, vec_elem
);
689 ir_rvalue
*absval
= abs(ir
->operands
[0]);
691 i
.insert_before(is_not_zero
);
692 i
.insert_before(high_words
);
693 i
.insert_before(assign(is_not_zero
, nequal(absval
->clone(ir
, NULL
), dzero
)));
695 /* Extract all of the upper uints. */
696 for (unsigned elem
= 0; elem
< vec_elem
; elem
++) {
697 ir_rvalue
*x
= swizzle(absval
->clone(ir
, NULL
), elem
, 1);
699 i
.insert_before(assign(high_words
,
700 swizzle_y(expr(ir_unop_unpack_double_2x32
, x
)),
704 ir_constant
*exponent_shift
= new(ir
) ir_constant(20, vec_elem
);
705 ir_constant
*exponent_bias
= new(ir
) ir_constant(-1022, vec_elem
);
707 /* For non-zero inputs, shift the exponent down and apply bias. */
708 ir
->operation
= ir_triop_csel
;
709 ir
->operands
[0] = new(ir
) ir_dereference_variable(is_not_zero
);
710 ir
->operands
[1] = add(exponent_bias
, u2i(rshift(high_words
, exponent_shift
)));
711 ir
->operands
[2] = izero
;
713 this->progress
= true;
717 lower_instructions_visitor::carry_to_arith(ir_expression
*ir
)
722 * sum = ir_binop_add x y
723 * bcarry = ir_binop_less sum x
724 * carry = ir_unop_b2i bcarry
727 ir_rvalue
*x_clone
= ir
->operands
[0]->clone(ir
, NULL
);
728 ir
->operation
= ir_unop_i2u
;
729 ir
->operands
[0] = b2i(less(add(ir
->operands
[0], ir
->operands
[1]), x_clone
));
730 ir
->operands
[1] = NULL
;
732 this->progress
= true;
736 lower_instructions_visitor::borrow_to_arith(ir_expression
*ir
)
739 * ir_binop_borrow x y
741 * bcarry = ir_binop_less x y
742 * carry = ir_unop_b2i bcarry
745 ir
->operation
= ir_unop_i2u
;
746 ir
->operands
[0] = b2i(less(ir
->operands
[0], ir
->operands
[1]));
747 ir
->operands
[1] = NULL
;
749 this->progress
= true;
753 lower_instructions_visitor::sat_to_clamp(ir_expression
*ir
)
758 * ir_binop_min (ir_binop_max(x, 0.0), 1.0)
761 ir
->operation
= ir_binop_min
;
762 ir
->operands
[0] = new(ir
) ir_expression(ir_binop_max
, ir
->operands
[0]->type
,
764 new(ir
) ir_constant(0.0f
));
765 ir
->operands
[1] = new(ir
) ir_constant(1.0f
);
767 this->progress
= true;
771 lower_instructions_visitor::double_dot_to_fma(ir_expression
*ir
)
773 ir_variable
*temp
= new(ir
) ir_variable(ir
->operands
[0]->type
->get_base_type(), "dot_res",
775 this->base_ir
->insert_before(temp
);
777 int nc
= ir
->operands
[0]->type
->components();
778 for (int i
= nc
- 1; i
>= 1; i
--) {
779 ir_assignment
*assig
;
781 assig
= assign(temp
, mul(swizzle(ir
->operands
[0]->clone(ir
, NULL
), i
, 1),
782 swizzle(ir
->operands
[1]->clone(ir
, NULL
), i
, 1)));
784 assig
= assign(temp
, fma(swizzle(ir
->operands
[0]->clone(ir
, NULL
), i
, 1),
785 swizzle(ir
->operands
[1]->clone(ir
, NULL
), i
, 1),
788 this->base_ir
->insert_before(assig
);
791 ir
->operation
= ir_triop_fma
;
792 ir
->operands
[0] = swizzle(ir
->operands
[0], 0, 1);
793 ir
->operands
[1] = swizzle(ir
->operands
[1], 0, 1);
794 ir
->operands
[2] = new(ir
) ir_dereference_variable(temp
);
796 this->progress
= true;
801 lower_instructions_visitor::double_lrp(ir_expression
*ir
)
804 ir_rvalue
*op0
= ir
->operands
[0], *op2
= ir
->operands
[2];
805 ir_constant
*one
= new(ir
) ir_constant(1.0, op2
->type
->vector_elements
);
807 switch (op2
->type
->vector_elements
) {
809 swizval
= SWIZZLE_XXXX
;
812 assert(op0
->type
->vector_elements
== op2
->type
->vector_elements
);
813 swizval
= SWIZZLE_XYZW
;
817 ir
->operation
= ir_triop_fma
;
818 ir
->operands
[0] = swizzle(op2
, swizval
, op0
->type
->vector_elements
);
819 ir
->operands
[2] = mul(sub(one
, op2
->clone(ir
, NULL
)), op0
);
821 this->progress
= true;
825 lower_instructions_visitor::dceil_to_dfrac(ir_expression
*ir
)
829 * temp = sub(x, frtemp);
830 * result = temp + ((frtemp != 0.0) ? 1.0 : 0.0);
832 ir_instruction
&i
= *base_ir
;
833 ir_constant
*zero
= new(ir
) ir_constant(0.0, ir
->operands
[0]->type
->vector_elements
);
834 ir_constant
*one
= new(ir
) ir_constant(1.0, ir
->operands
[0]->type
->vector_elements
);
835 ir_variable
*frtemp
= new(ir
) ir_variable(ir
->operands
[0]->type
, "frtemp",
838 i
.insert_before(frtemp
);
839 i
.insert_before(assign(frtemp
, fract(ir
->operands
[0])));
841 ir
->operation
= ir_binop_add
;
842 ir
->operands
[0] = sub(ir
->operands
[0]->clone(ir
, NULL
), frtemp
);
843 ir
->operands
[1] = csel(nequal(frtemp
, zero
), one
, zero
->clone(ir
, NULL
));
845 this->progress
= true;
849 lower_instructions_visitor::dfloor_to_dfrac(ir_expression
*ir
)
853 * result = sub(x, frtemp);
855 ir
->operation
= ir_binop_sub
;
856 ir
->operands
[1] = fract(ir
->operands
[0]->clone(ir
, NULL
));
858 this->progress
= true;
861 lower_instructions_visitor::dround_even_to_dfrac(ir_expression
*ir
)
866 * frtemp = frac(temp);
867 * t2 = sub(temp, frtemp);
868 * if (frac(x) == 0.5)
869 * result = frac(t2 * 0.5) == 0 ? t2 : t2 - 1;
874 ir_instruction
&i
= *base_ir
;
875 ir_variable
*frtemp
= new(ir
) ir_variable(ir
->operands
[0]->type
, "frtemp",
877 ir_variable
*temp
= new(ir
) ir_variable(ir
->operands
[0]->type
, "temp",
879 ir_variable
*t2
= new(ir
) ir_variable(ir
->operands
[0]->type
, "t2",
881 ir_constant
*p5
= new(ir
) ir_constant(0.5, ir
->operands
[0]->type
->vector_elements
);
882 ir_constant
*one
= new(ir
) ir_constant(1.0, ir
->operands
[0]->type
->vector_elements
);
883 ir_constant
*zero
= new(ir
) ir_constant(0.0, ir
->operands
[0]->type
->vector_elements
);
885 i
.insert_before(temp
);
886 i
.insert_before(assign(temp
, add(ir
->operands
[0], p5
)));
888 i
.insert_before(frtemp
);
889 i
.insert_before(assign(frtemp
, fract(temp
)));
892 i
.insert_before(assign(t2
, sub(temp
, frtemp
)));
894 ir
->operation
= ir_triop_csel
;
895 ir
->operands
[0] = equal(fract(ir
->operands
[0]->clone(ir
, NULL
)),
896 p5
->clone(ir
, NULL
));
897 ir
->operands
[1] = csel(equal(fract(mul(t2
, p5
->clone(ir
, NULL
))),
901 ir
->operands
[2] = new(ir
) ir_dereference_variable(t2
);
903 this->progress
= true;
907 lower_instructions_visitor::dtrunc_to_dfrac(ir_expression
*ir
)
911 * temp = sub(x, frtemp);
912 * result = x >= 0 ? temp : temp + (frtemp == 0.0) ? 0 : 1;
914 ir_rvalue
*arg
= ir
->operands
[0];
915 ir_instruction
&i
= *base_ir
;
917 ir_constant
*zero
= new(ir
) ir_constant(0.0, arg
->type
->vector_elements
);
918 ir_constant
*one
= new(ir
) ir_constant(1.0, arg
->type
->vector_elements
);
919 ir_variable
*frtemp
= new(ir
) ir_variable(arg
->type
, "frtemp",
921 ir_variable
*temp
= new(ir
) ir_variable(ir
->operands
[0]->type
, "temp",
924 i
.insert_before(frtemp
);
925 i
.insert_before(assign(frtemp
, fract(arg
)));
926 i
.insert_before(temp
);
927 i
.insert_before(assign(temp
, sub(arg
->clone(ir
, NULL
), frtemp
)));
929 ir
->operation
= ir_triop_csel
;
930 ir
->operands
[0] = gequal(arg
->clone(ir
, NULL
), zero
);
931 ir
->operands
[1] = new (ir
) ir_dereference_variable(temp
);
932 ir
->operands
[2] = add(temp
,
933 csel(equal(frtemp
, zero
->clone(ir
, NULL
)),
934 zero
->clone(ir
, NULL
),
937 this->progress
= true;
941 lower_instructions_visitor::dsign_to_csel(ir_expression
*ir
)
944 * temp = x > 0.0 ? 1.0 : 0.0;
945 * result = x < 0.0 ? -1.0 : temp;
947 ir_rvalue
*arg
= ir
->operands
[0];
948 ir_constant
*zero
= new(ir
) ir_constant(0.0, arg
->type
->vector_elements
);
949 ir_constant
*one
= new(ir
) ir_constant(1.0, arg
->type
->vector_elements
);
950 ir_constant
*neg_one
= new(ir
) ir_constant(-1.0, arg
->type
->vector_elements
);
952 ir
->operation
= ir_triop_csel
;
953 ir
->operands
[0] = less(arg
->clone(ir
, NULL
),
954 zero
->clone(ir
, NULL
));
955 ir
->operands
[1] = neg_one
;
956 ir
->operands
[2] = csel(greater(arg
, zero
),
958 zero
->clone(ir
, NULL
));
960 this->progress
= true;
964 lower_instructions_visitor::bit_count_to_math(ir_expression
*ir
)
966 /* For more details, see:
968 * http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetPaallel
970 const unsigned elements
= ir
->operands
[0]->type
->vector_elements
;
971 ir_variable
*temp
= new(ir
) ir_variable(glsl_type::uvec(elements
), "temp",
973 ir_constant
*c55555555
= new(ir
) ir_constant(0x55555555u
);
974 ir_constant
*c33333333
= new(ir
) ir_constant(0x33333333u
);
975 ir_constant
*c0F0F0F0F
= new(ir
) ir_constant(0x0F0F0F0Fu
);
976 ir_constant
*c01010101
= new(ir
) ir_constant(0x01010101u
);
977 ir_constant
*c1
= new(ir
) ir_constant(1u);
978 ir_constant
*c2
= new(ir
) ir_constant(2u);
979 ir_constant
*c4
= new(ir
) ir_constant(4u);
980 ir_constant
*c24
= new(ir
) ir_constant(24u);
982 base_ir
->insert_before(temp
);
984 if (ir
->operands
[0]->type
->base_type
== GLSL_TYPE_UINT
) {
985 base_ir
->insert_before(assign(temp
, ir
->operands
[0]));
987 assert(ir
->operands
[0]->type
->base_type
== GLSL_TYPE_INT
);
988 base_ir
->insert_before(assign(temp
, i2u(ir
->operands
[0])));
991 /* temp = temp - ((temp >> 1) & 0x55555555u); */
992 base_ir
->insert_before(assign(temp
, sub(temp
, bit_and(rshift(temp
, c1
),
995 /* temp = (temp & 0x33333333u) + ((temp >> 2) & 0x33333333u); */
996 base_ir
->insert_before(assign(temp
, add(bit_and(temp
, c33333333
),
997 bit_and(rshift(temp
, c2
),
998 c33333333
->clone(ir
, NULL
)))));
1000 /* int(((temp + (temp >> 4) & 0xF0F0F0Fu) * 0x1010101u) >> 24); */
1001 ir
->operation
= ir_unop_u2i
;
1002 ir
->operands
[0] = rshift(mul(bit_and(add(temp
, rshift(temp
, c4
)), c0F0F0F0F
),
1006 this->progress
= true;
1010 lower_instructions_visitor::extract_to_shifts(ir_expression
*ir
)
1013 new(ir
) ir_variable(ir
->operands
[0]->type
, "bits", ir_var_temporary
);
1015 base_ir
->insert_before(bits
);
1016 base_ir
->insert_before(assign(bits
, ir
->operands
[2]));
1018 if (ir
->operands
[0]->type
->base_type
== GLSL_TYPE_UINT
) {
1020 new(ir
) ir_constant(1u, ir
->operands
[0]->type
->vector_elements
);
1022 new(ir
) ir_constant(32u, ir
->operands
[0]->type
->vector_elements
);
1023 ir_constant
*cFFFFFFFF
=
1024 new(ir
) ir_constant(0xFFFFFFFFu
, ir
->operands
[0]->type
->vector_elements
);
1026 /* At least some hardware treats (x << y) as (x << (y%32)). This means
1027 * we'd get a mask of 0 when bits is 32. Special case it.
1029 * mask = bits == 32 ? 0xffffffff : (1u << bits) - 1u;
1031 ir_expression
*mask
= csel(equal(bits
, c32
),
1033 sub(lshift(c1
, bits
), c1
->clone(ir
, NULL
)));
1035 /* Section 8.8 (Integer Functions) of the GLSL 4.50 spec says:
1037 * If bits is zero, the result will be zero.
1039 * Since (1 << 0) - 1 == 0, we don't need to bother with the conditional
1040 * select as in the signed integer case.
1042 * (value >> offset) & mask;
1044 ir
->operation
= ir_binop_bit_and
;
1045 ir
->operands
[0] = rshift(ir
->operands
[0], ir
->operands
[1]);
1046 ir
->operands
[1] = mask
;
1047 ir
->operands
[2] = NULL
;
1050 new(ir
) ir_constant(int(0), ir
->operands
[0]->type
->vector_elements
);
1052 new(ir
) ir_constant(int(32), ir
->operands
[0]->type
->vector_elements
);
1054 new(ir
) ir_variable(ir
->operands
[0]->type
, "temp", ir_var_temporary
);
1056 /* temp = 32 - bits; */
1057 base_ir
->insert_before(temp
);
1058 base_ir
->insert_before(assign(temp
, sub(c32
, bits
)));
1060 /* expr = value << (temp - offset)) >> temp; */
1061 ir_expression
*expr
=
1062 rshift(lshift(ir
->operands
[0], sub(temp
, ir
->operands
[1])), temp
);
1064 /* Section 8.8 (Integer Functions) of the GLSL 4.50 spec says:
1066 * If bits is zero, the result will be zero.
1068 * Due to the (x << (y%32)) behavior mentioned before, the (value <<
1069 * (32-0)) doesn't "erase" all of the data as we would like, so finish
1072 * (bits == 0) ? 0 : e;
1074 ir
->operation
= ir_triop_csel
;
1075 ir
->operands
[0] = equal(c0
, bits
);
1076 ir
->operands
[1] = c0
->clone(ir
, NULL
);
1077 ir
->operands
[2] = expr
;
1080 this->progress
= true;
1084 lower_instructions_visitor::insert_to_shifts(ir_expression
*ir
)
1088 ir_constant
*cFFFFFFFF
;
1089 ir_variable
*offset
=
1090 new(ir
) ir_variable(ir
->operands
[0]->type
, "offset", ir_var_temporary
);
1092 new(ir
) ir_variable(ir
->operands
[0]->type
, "bits", ir_var_temporary
);
1094 new(ir
) ir_variable(ir
->operands
[0]->type
, "mask", ir_var_temporary
);
1096 if (ir
->operands
[0]->type
->base_type
== GLSL_TYPE_INT
) {
1097 c1
= new(ir
) ir_constant(int(1), ir
->operands
[0]->type
->vector_elements
);
1098 c32
= new(ir
) ir_constant(int(32), ir
->operands
[0]->type
->vector_elements
);
1099 cFFFFFFFF
= new(ir
) ir_constant(int(0xFFFFFFFF), ir
->operands
[0]->type
->vector_elements
);
1101 assert(ir
->operands
[0]->type
->base_type
== GLSL_TYPE_UINT
);
1103 c1
= new(ir
) ir_constant(1u, ir
->operands
[0]->type
->vector_elements
);
1104 c32
= new(ir
) ir_constant(32u, ir
->operands
[0]->type
->vector_elements
);
1105 cFFFFFFFF
= new(ir
) ir_constant(0xFFFFFFFFu
, ir
->operands
[0]->type
->vector_elements
);
1108 base_ir
->insert_before(offset
);
1109 base_ir
->insert_before(assign(offset
, ir
->operands
[2]));
1111 base_ir
->insert_before(bits
);
1112 base_ir
->insert_before(assign(bits
, ir
->operands
[3]));
1114 /* At least some hardware treats (x << y) as (x << (y%32)). This means
1115 * we'd get a mask of 0 when bits is 32. Special case it.
1117 * mask = (bits == 32 ? 0xffffffff : (1u << bits) - 1u) << offset;
1119 * Section 8.8 (Integer Functions) of the GLSL 4.50 spec says:
1121 * The result will be undefined if offset or bits is negative, or if the
1122 * sum of offset and bits is greater than the number of bits used to
1123 * store the operand.
1125 * Since it's undefined, there are a couple other ways this could be
1126 * implemented. The other way that was considered was to put the csel
1127 * around the whole thing:
1129 * final_result = bits == 32 ? insert : ... ;
1131 base_ir
->insert_before(mask
);
1133 base_ir
->insert_before(assign(mask
, csel(equal(bits
, c32
),
1135 lshift(sub(lshift(c1
, bits
),
1136 c1
->clone(ir
, NULL
)),
1139 /* (base & ~mask) | ((insert << offset) & mask) */
1140 ir
->operation
= ir_binop_bit_or
;
1141 ir
->operands
[0] = bit_and(ir
->operands
[0], bit_not(mask
));
1142 ir
->operands
[1] = bit_and(lshift(ir
->operands
[1], offset
), mask
);
1143 ir
->operands
[2] = NULL
;
1144 ir
->operands
[3] = NULL
;
1146 this->progress
= true;
1150 lower_instructions_visitor::reverse_to_shifts(ir_expression
*ir
)
1152 /* For more details, see:
1154 * http://graphics.stanford.edu/~seander/bithacks.html#ReverseParallel
1157 new(ir
) ir_constant(1u, ir
->operands
[0]->type
->vector_elements
);
1159 new(ir
) ir_constant(2u, ir
->operands
[0]->type
->vector_elements
);
1161 new(ir
) ir_constant(4u, ir
->operands
[0]->type
->vector_elements
);
1163 new(ir
) ir_constant(8u, ir
->operands
[0]->type
->vector_elements
);
1165 new(ir
) ir_constant(16u, ir
->operands
[0]->type
->vector_elements
);
1166 ir_constant
*c33333333
=
1167 new(ir
) ir_constant(0x33333333u
, ir
->operands
[0]->type
->vector_elements
);
1168 ir_constant
*c55555555
=
1169 new(ir
) ir_constant(0x55555555u
, ir
->operands
[0]->type
->vector_elements
);
1170 ir_constant
*c0F0F0F0F
=
1171 new(ir
) ir_constant(0x0F0F0F0Fu
, ir
->operands
[0]->type
->vector_elements
);
1172 ir_constant
*c00FF00FF
=
1173 new(ir
) ir_constant(0x00FF00FFu
, ir
->operands
[0]->type
->vector_elements
);
1175 new(ir
) ir_variable(glsl_type::uvec(ir
->operands
[0]->type
->vector_elements
),
1176 "temp", ir_var_temporary
);
1177 ir_instruction
&i
= *base_ir
;
1179 i
.insert_before(temp
);
1181 if (ir
->operands
[0]->type
->base_type
== GLSL_TYPE_UINT
) {
1182 i
.insert_before(assign(temp
, ir
->operands
[0]));
1184 assert(ir
->operands
[0]->type
->base_type
== GLSL_TYPE_INT
);
1185 i
.insert_before(assign(temp
, i2u(ir
->operands
[0])));
1188 /* Swap odd and even bits.
1190 * temp = ((temp >> 1) & 0x55555555u) | ((temp & 0x55555555u) << 1);
1192 i
.insert_before(assign(temp
, bit_or(bit_and(rshift(temp
, c1
), c55555555
),
1193 lshift(bit_and(temp
, c55555555
->clone(ir
, NULL
)),
1194 c1
->clone(ir
, NULL
)))));
1195 /* Swap consecutive pairs.
1197 * temp = ((temp >> 2) & 0x33333333u) | ((temp & 0x33333333u) << 2);
1199 i
.insert_before(assign(temp
, bit_or(bit_and(rshift(temp
, c2
), c33333333
),
1200 lshift(bit_and(temp
, c33333333
->clone(ir
, NULL
)),
1201 c2
->clone(ir
, NULL
)))));
1205 * temp = ((temp >> 4) & 0x0F0F0F0Fu) | ((temp & 0x0F0F0F0Fu) << 4);
1207 i
.insert_before(assign(temp
, bit_or(bit_and(rshift(temp
, c4
), c0F0F0F0F
),
1208 lshift(bit_and(temp
, c0F0F0F0F
->clone(ir
, NULL
)),
1209 c4
->clone(ir
, NULL
)))));
1211 /* The last step is, basically, bswap. Swap the bytes, then swap the
1212 * words. When this code is run through GCC on x86, it does generate a
1213 * bswap instruction.
1215 * temp = ((temp >> 8) & 0x00FF00FFu) | ((temp & 0x00FF00FFu) << 8);
1216 * temp = ( temp >> 16 ) | ( temp << 16);
1218 i
.insert_before(assign(temp
, bit_or(bit_and(rshift(temp
, c8
), c00FF00FF
),
1219 lshift(bit_and(temp
, c00FF00FF
->clone(ir
, NULL
)),
1220 c8
->clone(ir
, NULL
)))));
1222 if (ir
->operands
[0]->type
->base_type
== GLSL_TYPE_UINT
) {
1223 ir
->operation
= ir_binop_bit_or
;
1224 ir
->operands
[0] = rshift(temp
, c16
);
1225 ir
->operands
[1] = lshift(temp
, c16
->clone(ir
, NULL
));
1227 ir
->operation
= ir_unop_u2i
;
1228 ir
->operands
[0] = bit_or(rshift(temp
, c16
),
1229 lshift(temp
, c16
->clone(ir
, NULL
)));
1232 this->progress
= true;
1236 lower_instructions_visitor::find_lsb_to_float_cast(ir_expression
*ir
)
1238 /* For more details, see:
1240 * http://graphics.stanford.edu/~seander/bithacks.html#ZerosOnRightFloatCast
1242 const unsigned elements
= ir
->operands
[0]->type
->vector_elements
;
1243 ir_constant
*c0
= new(ir
) ir_constant(unsigned(0), elements
);
1244 ir_constant
*cminus1
= new(ir
) ir_constant(int(-1), elements
);
1245 ir_constant
*c23
= new(ir
) ir_constant(int(23), elements
);
1246 ir_constant
*c7F
= new(ir
) ir_constant(int(0x7F), elements
);
1248 new(ir
) ir_variable(glsl_type::ivec(elements
), "temp", ir_var_temporary
);
1249 ir_variable
*lsb_only
=
1250 new(ir
) ir_variable(glsl_type::uvec(elements
), "lsb_only", ir_var_temporary
);
1251 ir_variable
*as_float
=
1252 new(ir
) ir_variable(glsl_type::vec(elements
), "as_float", ir_var_temporary
);
1254 new(ir
) ir_variable(glsl_type::ivec(elements
), "lsb", ir_var_temporary
);
1256 ir_instruction
&i
= *base_ir
;
1258 i
.insert_before(temp
);
1260 if (ir
->operands
[0]->type
->base_type
== GLSL_TYPE_INT
) {
1261 i
.insert_before(assign(temp
, ir
->operands
[0]));
1263 assert(ir
->operands
[0]->type
->base_type
== GLSL_TYPE_UINT
);
1264 i
.insert_before(assign(temp
, u2i(ir
->operands
[0])));
1267 /* The int-to-float conversion is lossless because (value & -value) is
1268 * either a power of two or zero. We don't use the result in the zero
1269 * case. The uint() cast is necessary so that 0x80000000 does not
1270 * generate a negative value.
1272 * uint lsb_only = uint(value & -value);
1273 * float as_float = float(lsb_only);
1275 i
.insert_before(lsb_only
);
1276 i
.insert_before(assign(lsb_only
, i2u(bit_and(temp
, neg(temp
)))));
1278 i
.insert_before(as_float
);
1279 i
.insert_before(assign(as_float
, u2f(lsb_only
)));
1281 /* This is basically an open-coded frexp. Implementations that have a
1282 * native frexp instruction would be better served by that. This is
1283 * optimized versus a full-featured open-coded implementation in two ways:
1285 * - We don't care about a correct result from subnormal numbers (including
1286 * 0.0), so the raw exponent can always be safely unbiased.
1288 * - The value cannot be negative, so it does not need to be masked off to
1289 * extract the exponent.
1291 * int lsb = (floatBitsToInt(as_float) >> 23) - 0x7f;
1293 i
.insert_before(lsb
);
1294 i
.insert_before(assign(lsb
, sub(rshift(bitcast_f2i(as_float
), c23
), c7F
)));
1296 /* Use lsb_only in the comparison instead of temp so that the & (far above)
1297 * can possibly generate the result without an explicit comparison.
1299 * (lsb_only == 0) ? -1 : lsb;
1301 * Since our input values are all integers, the unbiased exponent must not
1302 * be negative. It will only be negative (-0x7f, in fact) if lsb_only is
1303 * 0. Instead of using (lsb_only == 0), we could use (lsb >= 0). Which is
1304 * better is likely GPU dependent. Either way, the difference should be
1307 ir
->operation
= ir_triop_csel
;
1308 ir
->operands
[0] = equal(lsb_only
, c0
);
1309 ir
->operands
[1] = cminus1
;
1310 ir
->operands
[2] = new(ir
) ir_dereference_variable(lsb
);
1312 this->progress
= true;
1316 lower_instructions_visitor::find_msb_to_float_cast(ir_expression
*ir
)
1318 /* For more details, see:
1320 * http://graphics.stanford.edu/~seander/bithacks.html#ZerosOnRightFloatCast
1322 const unsigned elements
= ir
->operands
[0]->type
->vector_elements
;
1323 ir_constant
*c0
= new(ir
) ir_constant(int(0), elements
);
1324 ir_constant
*cminus1
= new(ir
) ir_constant(int(-1), elements
);
1325 ir_constant
*c23
= new(ir
) ir_constant(int(23), elements
);
1326 ir_constant
*c7F
= new(ir
) ir_constant(int(0x7F), elements
);
1327 ir_constant
*c000000FF
= new(ir
) ir_constant(0x000000FFu
, elements
);
1328 ir_constant
*cFFFFFF00
= new(ir
) ir_constant(0xFFFFFF00u
, elements
);
1330 new(ir
) ir_variable(glsl_type::uvec(elements
), "temp", ir_var_temporary
);
1331 ir_variable
*as_float
=
1332 new(ir
) ir_variable(glsl_type::vec(elements
), "as_float", ir_var_temporary
);
1334 new(ir
) ir_variable(glsl_type::ivec(elements
), "msb", ir_var_temporary
);
1336 ir_instruction
&i
= *base_ir
;
1338 i
.insert_before(temp
);
1340 if (ir
->operands
[0]->type
->base_type
== GLSL_TYPE_UINT
) {
1341 i
.insert_before(assign(temp
, ir
->operands
[0]));
1343 assert(ir
->operands
[0]->type
->base_type
== GLSL_TYPE_INT
);
1345 /* findMSB(uint(abs(some_int))) almost always does the right thing.
1346 * There are two problem values:
1348 * * 0x80000000. Since abs(0x80000000) == 0x80000000, findMSB returns
1349 * 31. However, findMSB(int(0x80000000)) == 30.
1351 * * 0xffffffff. Since abs(0xffffffff) == 1, findMSB returns
1352 * 31. Section 8.8 (Integer Functions) of the GLSL 4.50 spec says:
1354 * For a value of zero or negative one, -1 will be returned.
1356 * For all negative number cases, including 0x80000000 and 0xffffffff,
1357 * the correct value is obtained from findMSB if instead of negating the
1358 * (already negative) value the logical-not is used. A conditonal
1359 * logical-not can be achieved in two instructions.
1361 ir_variable
*as_int
=
1362 new(ir
) ir_variable(glsl_type::ivec(elements
), "as_int", ir_var_temporary
);
1363 ir_constant
*c31
= new(ir
) ir_constant(int(31), elements
);
1365 i
.insert_before(as_int
);
1366 i
.insert_before(assign(as_int
, ir
->operands
[0]));
1367 i
.insert_before(assign(temp
, i2u(expr(ir_binop_bit_xor
,
1369 rshift(as_int
, c31
)))));
1372 /* The int-to-float conversion is lossless because bits are conditionally
1373 * masked off the bottom of temp to ensure the value has at most 24 bits of
1374 * data or is zero. We don't use the result in the zero case. The uint()
1375 * cast is necessary so that 0x80000000 does not generate a negative value.
1377 * float as_float = float(temp > 255 ? temp & ~255 : temp);
1379 i
.insert_before(as_float
);
1380 i
.insert_before(assign(as_float
, u2f(csel(greater(temp
, c000000FF
),
1381 bit_and(temp
, cFFFFFF00
),
1384 /* This is basically an open-coded frexp. Implementations that have a
1385 * native frexp instruction would be better served by that. This is
1386 * optimized versus a full-featured open-coded implementation in two ways:
1388 * - We don't care about a correct result from subnormal numbers (including
1389 * 0.0), so the raw exponent can always be safely unbiased.
1391 * - The value cannot be negative, so it does not need to be masked off to
1392 * extract the exponent.
1394 * int msb = (floatBitsToInt(as_float) >> 23) - 0x7f;
1396 i
.insert_before(msb
);
1397 i
.insert_before(assign(msb
, sub(rshift(bitcast_f2i(as_float
), c23
), c7F
)));
1399 /* Use msb in the comparison instead of temp so that the subtract can
1400 * possibly generate the result without an explicit comparison.
1402 * (msb < 0) ? -1 : msb;
1404 * Since our input values are all integers, the unbiased exponent must not
1405 * be negative. It will only be negative (-0x7f, in fact) if temp is 0.
1407 ir
->operation
= ir_triop_csel
;
1408 ir
->operands
[0] = less(msb
, c0
);
1409 ir
->operands
[1] = cminus1
;
1410 ir
->operands
[2] = new(ir
) ir_dereference_variable(msb
);
1412 this->progress
= true;
1416 lower_instructions_visitor::visit_leave(ir_expression
*ir
)
1418 switch (ir
->operation
) {
1420 if (ir
->operands
[0]->type
->is_double())
1421 double_dot_to_fma(ir
);
1424 if (ir
->operands
[0]->type
->is_double())
1428 if (lowering(SUB_TO_ADD_NEG
))
1433 if (ir
->operands
[1]->type
->is_integer() && lowering(INT_DIV_TO_MUL_RCP
))
1434 int_div_to_mul_rcp(ir
);
1435 else if ((ir
->operands
[1]->type
->is_float() ||
1436 ir
->operands
[1]->type
->is_double()) && lowering(DIV_TO_MUL_RCP
))
1441 if (lowering(EXP_TO_EXP2
))
1446 if (lowering(LOG_TO_LOG2
))
1451 if (lowering(MOD_TO_FLOOR
) && (ir
->type
->is_float() || ir
->type
->is_double()))
1456 if (lowering(POW_TO_EXP2
))
1460 case ir_binop_ldexp
:
1461 if (lowering(LDEXP_TO_ARITH
) && ir
->type
->is_float())
1463 if (lowering(DFREXP_DLDEXP_TO_ARITH
) && ir
->type
->is_double())
1464 dldexp_to_arith(ir
);
1467 case ir_unop_frexp_exp
:
1468 if (lowering(DFREXP_DLDEXP_TO_ARITH
) && ir
->operands
[0]->type
->is_double())
1469 dfrexp_exp_to_arith(ir
);
1472 case ir_unop_frexp_sig
:
1473 if (lowering(DFREXP_DLDEXP_TO_ARITH
) && ir
->operands
[0]->type
->is_double())
1474 dfrexp_sig_to_arith(ir
);
1477 case ir_binop_carry
:
1478 if (lowering(CARRY_TO_ARITH
))
1482 case ir_binop_borrow
:
1483 if (lowering(BORROW_TO_ARITH
))
1484 borrow_to_arith(ir
);
1487 case ir_unop_saturate
:
1488 if (lowering(SAT_TO_CLAMP
))
1493 if (lowering(DOPS_TO_DFRAC
) && ir
->type
->is_double())
1494 dtrunc_to_dfrac(ir
);
1498 if (lowering(DOPS_TO_DFRAC
) && ir
->type
->is_double())
1503 if (lowering(DOPS_TO_DFRAC
) && ir
->type
->is_double())
1504 dfloor_to_dfrac(ir
);
1507 case ir_unop_round_even
:
1508 if (lowering(DOPS_TO_DFRAC
) && ir
->type
->is_double())
1509 dround_even_to_dfrac(ir
);
1513 if (lowering(DOPS_TO_DFRAC
) && ir
->type
->is_double())
1517 case ir_unop_bit_count
:
1518 if (lowering(BIT_COUNT_TO_MATH
))
1519 bit_count_to_math(ir
);
1522 case ir_triop_bitfield_extract
:
1523 if (lowering(EXTRACT_TO_SHIFTS
))
1524 extract_to_shifts(ir
);
1527 case ir_quadop_bitfield_insert
:
1528 if (lowering(INSERT_TO_SHIFTS
))
1529 insert_to_shifts(ir
);
1532 case ir_unop_bitfield_reverse
:
1533 if (lowering(REVERSE_TO_SHIFTS
))
1534 reverse_to_shifts(ir
);
1537 case ir_unop_find_lsb
:
1538 if (lowering(FIND_LSB_TO_FLOAT_CAST
))
1539 find_lsb_to_float_cast(ir
);
1542 case ir_unop_find_msb
:
1543 if (lowering(FIND_MSB_TO_FLOAT_CAST
))
1544 find_msb_to_float_cast(ir
);
1548 return visit_continue
;
1551 return visit_continue
;