2 * Copyright © 2010 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
25 * \file lower_instructions.cpp
27 * Many GPUs lack native instructions for certain expression operations, and
28 * must replace them with some other expression tree. This pass lowers some
29 * of the most common cases, allowing the lowering code to be implemented once
30 * rather than in each driver backend.
32 * Currently supported transformations:
35 * - INT_DIV_TO_MUL_RCP
49 * Breaks an ir_binop_sub expression down to add(op0, neg(op1))
51 * This simplifies expression reassociation, and for many backends
52 * there is no subtract operation separate from adding the negation.
53 * For backends with native subtract operations, they will probably
54 * want to recognize add(op0, neg(op1)) or the other way around to
55 * produce a subtract anyway.
57 * FDIV_TO_MUL_RCP, DDIV_TO_MUL_RCP, and INT_DIV_TO_MUL_RCP:
58 * ---------------------------------------------------------
59 * Breaks an ir_binop_div expression down to op0 * (rcp(op1)).
61 * Many GPUs don't have a divide instruction (945 and 965 included),
62 * but they do have an RCP instruction to compute an approximate
63 * reciprocal. By breaking the operation down, constant reciprocals
64 * can get constant folded.
66 * FDIV_TO_MUL_RCP only lowers single-precision floating point division;
67 * DDIV_TO_MUL_RCP only lowers double-precision floating point division.
68 * DIV_TO_MUL_RCP is a convenience macro that sets both flags.
69 * INT_DIV_TO_MUL_RCP handles the integer case, converting to and from floating
70 * point so that RCP is possible.
72 * EXP_TO_EXP2 and LOG_TO_LOG2:
73 * ----------------------------
74 * Many GPUs don't have a base e log or exponent instruction, but they
75 * do have base 2 versions, so this pass converts exp and log to exp2
76 * and log2 operations.
80 * Many older GPUs don't have an x**y instruction. For these GPUs, convert
81 * x**y to 2**(y * log2(x)).
85 * Breaks an ir_binop_mod expression down to (op0 - op1 * floor(op0 / op1))
87 * Many GPUs don't have a MOD instruction (945 and 965 included), and
88 * if we have to break it down like this anyway, it gives an
89 * opportunity to do things like constant fold the (1.0 / op1) easily.
91 * Note: before we used to implement this as op1 * fract(op / op1) but this
92 * implementation had significant precision errors.
96 * Converts ir_binop_ldexp to arithmetic and bit operations for float sources.
98 * DFREXP_DLDEXP_TO_ARITH:
100 * Converts ir_binop_ldexp, ir_unop_frexp_sig, and ir_unop_frexp_exp to
101 * arithmetic and bit ops for double arguments.
105 * Converts ir_carry into (x + y) < x.
109 * Converts ir_borrow into (x < y).
113 * Converts ir_unop_saturate into min(max(x, 0.0), 1.0)
117 * Converts double trunc, ceil, floor, round to fract
120 #include "c99_math.h"
121 #include "program/prog_instruction.h" /* for swizzle */
122 #include "compiler/glsl_types.h"
124 #include "ir_builder.h"
125 #include "ir_optimization.h"
127 using namespace ir_builder
;
131 class lower_instructions_visitor
: public ir_hierarchical_visitor
{
133 lower_instructions_visitor(unsigned lower
)
134 : progress(false), lower(lower
) { }
136 ir_visitor_status
visit_leave(ir_expression
*);
141 unsigned lower
; /** Bitfield of which operations to lower */
143 void sub_to_add_neg(ir_expression
*);
144 void div_to_mul_rcp(ir_expression
*);
145 void int_div_to_mul_rcp(ir_expression
*);
146 void mod_to_floor(ir_expression
*);
147 void exp_to_exp2(ir_expression
*);
148 void pow_to_exp2(ir_expression
*);
149 void log_to_log2(ir_expression
*);
150 void ldexp_to_arith(ir_expression
*);
151 void dldexp_to_arith(ir_expression
*);
152 void dfrexp_sig_to_arith(ir_expression
*);
153 void dfrexp_exp_to_arith(ir_expression
*);
154 void carry_to_arith(ir_expression
*);
155 void borrow_to_arith(ir_expression
*);
156 void sat_to_clamp(ir_expression
*);
157 void double_dot_to_fma(ir_expression
*);
158 void double_lrp(ir_expression
*);
159 void dceil_to_dfrac(ir_expression
*);
160 void dfloor_to_dfrac(ir_expression
*);
161 void dround_even_to_dfrac(ir_expression
*);
162 void dtrunc_to_dfrac(ir_expression
*);
163 void dsign_to_csel(ir_expression
*);
164 void bit_count_to_math(ir_expression
*);
165 void extract_to_shifts(ir_expression
*);
166 void insert_to_shifts(ir_expression
*);
167 void reverse_to_shifts(ir_expression
*ir
);
168 void find_lsb_to_float_cast(ir_expression
*ir
);
169 void find_msb_to_float_cast(ir_expression
*ir
);
170 void imul_high_to_mul(ir_expression
*ir
);
171 void sqrt_to_abs_sqrt(ir_expression
*ir
);
173 ir_expression
*_carry(operand a
, operand b
);
176 } /* anonymous namespace */
179 * Determine if a particular type of lowering should occur
181 #define lowering(x) (this->lower & x)
184 lower_instructions(exec_list
*instructions
, unsigned what_to_lower
)
186 lower_instructions_visitor
v(what_to_lower
);
188 visit_list_elements(&v
, instructions
);
193 lower_instructions_visitor::sub_to_add_neg(ir_expression
*ir
)
195 ir
->operation
= ir_binop_add
;
196 ir
->init_num_operands();
197 ir
->operands
[1] = new(ir
) ir_expression(ir_unop_neg
, ir
->operands
[1]->type
,
198 ir
->operands
[1], NULL
);
199 this->progress
= true;
203 lower_instructions_visitor::div_to_mul_rcp(ir_expression
*ir
)
205 assert(ir
->operands
[1]->type
->is_float() || ir
->operands
[1]->type
->is_double());
207 /* New expression for the 1.0 / op1 */
209 expr
= new(ir
) ir_expression(ir_unop_rcp
,
210 ir
->operands
[1]->type
,
213 /* op0 / op1 -> op0 * (1.0 / op1) */
214 ir
->operation
= ir_binop_mul
;
215 ir
->init_num_operands();
216 ir
->operands
[1] = expr
;
218 this->progress
= true;
222 lower_instructions_visitor::int_div_to_mul_rcp(ir_expression
*ir
)
224 assert(ir
->operands
[1]->type
->is_integer());
226 /* Be careful with integer division -- we need to do it as a
227 * float and re-truncate, since rcp(n > 1) of an integer would
230 ir_rvalue
*op0
, *op1
;
231 const struct glsl_type
*vec_type
;
233 vec_type
= glsl_type::get_instance(GLSL_TYPE_FLOAT
,
234 ir
->operands
[1]->type
->vector_elements
,
235 ir
->operands
[1]->type
->matrix_columns
);
237 if (ir
->operands
[1]->type
->base_type
== GLSL_TYPE_INT
)
238 op1
= new(ir
) ir_expression(ir_unop_i2f
, vec_type
, ir
->operands
[1], NULL
);
240 op1
= new(ir
) ir_expression(ir_unop_u2f
, vec_type
, ir
->operands
[1], NULL
);
242 op1
= new(ir
) ir_expression(ir_unop_rcp
, op1
->type
, op1
, NULL
);
244 vec_type
= glsl_type::get_instance(GLSL_TYPE_FLOAT
,
245 ir
->operands
[0]->type
->vector_elements
,
246 ir
->operands
[0]->type
->matrix_columns
);
248 if (ir
->operands
[0]->type
->base_type
== GLSL_TYPE_INT
)
249 op0
= new(ir
) ir_expression(ir_unop_i2f
, vec_type
, ir
->operands
[0], NULL
);
251 op0
= new(ir
) ir_expression(ir_unop_u2f
, vec_type
, ir
->operands
[0], NULL
);
253 vec_type
= glsl_type::get_instance(GLSL_TYPE_FLOAT
,
254 ir
->type
->vector_elements
,
255 ir
->type
->matrix_columns
);
257 op0
= new(ir
) ir_expression(ir_binop_mul
, vec_type
, op0
, op1
);
259 if (ir
->operands
[1]->type
->base_type
== GLSL_TYPE_INT
) {
260 ir
->operation
= ir_unop_f2i
;
261 ir
->operands
[0] = op0
;
263 ir
->operation
= ir_unop_i2u
;
264 ir
->operands
[0] = new(ir
) ir_expression(ir_unop_f2i
, op0
);
266 ir
->init_num_operands();
267 ir
->operands
[1] = NULL
;
269 this->progress
= true;
273 lower_instructions_visitor::exp_to_exp2(ir_expression
*ir
)
275 ir_constant
*log2_e
= new(ir
) ir_constant(float(M_LOG2E
));
277 ir
->operation
= ir_unop_exp2
;
278 ir
->init_num_operands();
279 ir
->operands
[0] = new(ir
) ir_expression(ir_binop_mul
, ir
->operands
[0]->type
,
280 ir
->operands
[0], log2_e
);
281 this->progress
= true;
285 lower_instructions_visitor::pow_to_exp2(ir_expression
*ir
)
287 ir_expression
*const log2_x
=
288 new(ir
) ir_expression(ir_unop_log2
, ir
->operands
[0]->type
,
291 ir
->operation
= ir_unop_exp2
;
292 ir
->init_num_operands();
293 ir
->operands
[0] = new(ir
) ir_expression(ir_binop_mul
, ir
->operands
[1]->type
,
294 ir
->operands
[1], log2_x
);
295 ir
->operands
[1] = NULL
;
296 this->progress
= true;
300 lower_instructions_visitor::log_to_log2(ir_expression
*ir
)
302 ir
->operation
= ir_binop_mul
;
303 ir
->init_num_operands();
304 ir
->operands
[0] = new(ir
) ir_expression(ir_unop_log2
, ir
->operands
[0]->type
,
305 ir
->operands
[0], NULL
);
306 ir
->operands
[1] = new(ir
) ir_constant(float(1.0 / M_LOG2E
));
307 this->progress
= true;
311 lower_instructions_visitor::mod_to_floor(ir_expression
*ir
)
313 ir_variable
*x
= new(ir
) ir_variable(ir
->operands
[0]->type
, "mod_x",
315 ir_variable
*y
= new(ir
) ir_variable(ir
->operands
[1]->type
, "mod_y",
317 this->base_ir
->insert_before(x
);
318 this->base_ir
->insert_before(y
);
320 ir_assignment
*const assign_x
=
321 new(ir
) ir_assignment(new(ir
) ir_dereference_variable(x
),
322 ir
->operands
[0], NULL
);
323 ir_assignment
*const assign_y
=
324 new(ir
) ir_assignment(new(ir
) ir_dereference_variable(y
),
325 ir
->operands
[1], NULL
);
327 this->base_ir
->insert_before(assign_x
);
328 this->base_ir
->insert_before(assign_y
);
330 ir_expression
*const div_expr
=
331 new(ir
) ir_expression(ir_binop_div
, x
->type
,
332 new(ir
) ir_dereference_variable(x
),
333 new(ir
) ir_dereference_variable(y
));
335 /* Don't generate new IR that would need to be lowered in an additional
338 if ((lowering(FDIV_TO_MUL_RCP
) && ir
->type
->is_float()) ||
339 (lowering(DDIV_TO_MUL_RCP
) && ir
->type
->is_double()))
340 div_to_mul_rcp(div_expr
);
342 ir_expression
*const floor_expr
=
343 new(ir
) ir_expression(ir_unop_floor
, x
->type
, div_expr
);
345 if (lowering(DOPS_TO_DFRAC
) && ir
->type
->is_double())
346 dfloor_to_dfrac(floor_expr
);
348 ir_expression
*const mul_expr
=
349 new(ir
) ir_expression(ir_binop_mul
,
350 new(ir
) ir_dereference_variable(y
),
353 ir
->operation
= ir_binop_sub
;
354 ir
->init_num_operands();
355 ir
->operands
[0] = new(ir
) ir_dereference_variable(x
);
356 ir
->operands
[1] = mul_expr
;
357 this->progress
= true;
361 lower_instructions_visitor::ldexp_to_arith(ir_expression
*ir
)
364 * ir_binop_ldexp x exp
367 * extracted_biased_exp = rshift(bitcast_f2i(abs(x)), exp_shift);
368 * resulting_biased_exp = extracted_biased_exp + exp;
370 * if (resulting_biased_exp < 1 || x == 0.0f) {
371 * return copysign(0.0, x);
374 * return bitcast_u2f((bitcast_f2u(x) & sign_mantissa_mask) |
375 * lshift(i2u(resulting_biased_exp), exp_shift));
377 * which we can't actually implement as such, since the GLSL IR doesn't
378 * have vectorized if-statements. We actually implement it without branches
379 * using conditional-select:
381 * extracted_biased_exp = rshift(bitcast_f2i(abs(x)), exp_shift);
382 * resulting_biased_exp = extracted_biased_exp + exp;
384 * is_not_zero_or_underflow = logic_and(nequal(x, 0.0f),
385 * gequal(resulting_biased_exp, 1);
386 * x = csel(is_not_zero_or_underflow, x, copysign(0.0f, x));
387 * resulting_biased_exp = csel(is_not_zero_or_underflow,
388 * resulting_biased_exp, 0);
390 * return bitcast_u2f((bitcast_f2u(x) & sign_mantissa_mask) |
391 * lshift(i2u(resulting_biased_exp), exp_shift));
394 const unsigned vec_elem
= ir
->type
->vector_elements
;
397 const glsl_type
*ivec
= glsl_type::get_instance(GLSL_TYPE_INT
, vec_elem
, 1);
398 const glsl_type
*bvec
= glsl_type::get_instance(GLSL_TYPE_BOOL
, vec_elem
, 1);
401 ir_constant
*zeroi
= ir_constant::zero(ir
, ivec
);
403 ir_constant
*sign_mask
= new(ir
) ir_constant(0x80000000u
, vec_elem
);
405 ir_constant
*exp_shift
= new(ir
) ir_constant(23, vec_elem
);
407 /* Temporary variables */
408 ir_variable
*x
= new(ir
) ir_variable(ir
->type
, "x", ir_var_temporary
);
409 ir_variable
*exp
= new(ir
) ir_variable(ivec
, "exp", ir_var_temporary
);
411 ir_variable
*zero_sign_x
= new(ir
) ir_variable(ir
->type
, "zero_sign_x",
414 ir_variable
*extracted_biased_exp
=
415 new(ir
) ir_variable(ivec
, "extracted_biased_exp", ir_var_temporary
);
416 ir_variable
*resulting_biased_exp
=
417 new(ir
) ir_variable(ivec
, "resulting_biased_exp", ir_var_temporary
);
419 ir_variable
*is_not_zero_or_underflow
=
420 new(ir
) ir_variable(bvec
, "is_not_zero_or_underflow", ir_var_temporary
);
422 ir_instruction
&i
= *base_ir
;
424 /* Copy <x> and <exp> arguments. */
426 i
.insert_before(assign(x
, ir
->operands
[0]));
427 i
.insert_before(exp
);
428 i
.insert_before(assign(exp
, ir
->operands
[1]));
430 /* Extract the biased exponent from <x>. */
431 i
.insert_before(extracted_biased_exp
);
432 i
.insert_before(assign(extracted_biased_exp
,
433 rshift(bitcast_f2i(abs(x
)), exp_shift
)));
435 i
.insert_before(resulting_biased_exp
);
436 i
.insert_before(assign(resulting_biased_exp
,
437 add(extracted_biased_exp
, exp
)));
439 /* Test if result is ±0.0, subnormal, or underflow by checking if the
440 * resulting biased exponent would be less than 0x1. If so, the result is
441 * 0.0 with the sign of x. (Actually, invert the conditions so that
442 * immediate values are the second arguments, which is better for i965)
444 i
.insert_before(zero_sign_x
);
445 i
.insert_before(assign(zero_sign_x
,
446 bitcast_u2f(bit_and(bitcast_f2u(x
), sign_mask
))));
448 i
.insert_before(is_not_zero_or_underflow
);
449 i
.insert_before(assign(is_not_zero_or_underflow
,
450 logic_and(nequal(x
, new(ir
) ir_constant(0.0f
, vec_elem
)),
451 gequal(resulting_biased_exp
,
452 new(ir
) ir_constant(0x1, vec_elem
)))));
453 i
.insert_before(assign(x
, csel(is_not_zero_or_underflow
,
455 i
.insert_before(assign(resulting_biased_exp
,
456 csel(is_not_zero_or_underflow
,
457 resulting_biased_exp
, zeroi
)));
459 /* We could test for overflows by checking if the resulting biased exponent
460 * would be greater than 0xFE. Turns out we don't need to because the GLSL
463 * "If this product is too large to be represented in the
464 * floating-point type, the result is undefined."
467 ir_constant
*exp_shift_clone
= exp_shift
->clone(ir
, NULL
);
469 /* Don't generate new IR that would need to be lowered in an additional
472 if (!lowering(INSERT_TO_SHIFTS
)) {
473 ir_constant
*exp_width
= new(ir
) ir_constant(8, vec_elem
);
474 ir
->operation
= ir_unop_bitcast_i2f
;
475 ir
->init_num_operands();
476 ir
->operands
[0] = bitfield_insert(bitcast_f2i(x
), resulting_biased_exp
,
477 exp_shift_clone
, exp_width
);
478 ir
->operands
[1] = NULL
;
480 ir_constant
*sign_mantissa_mask
= new(ir
) ir_constant(0x807fffffu
, vec_elem
);
481 ir
->operation
= ir_unop_bitcast_u2f
;
482 ir
->init_num_operands();
483 ir
->operands
[0] = bit_or(bit_and(bitcast_f2u(x
), sign_mantissa_mask
),
484 lshift(i2u(resulting_biased_exp
), exp_shift_clone
));
485 ir
->operands
[1] = NULL
;
488 this->progress
= true;
492 lower_instructions_visitor::dldexp_to_arith(ir_expression
*ir
)
494 /* See ldexp_to_arith for structure. Uses frexp_exp to extract the exponent
495 * from the significand.
498 const unsigned vec_elem
= ir
->type
->vector_elements
;
501 const glsl_type
*ivec
= glsl_type::get_instance(GLSL_TYPE_INT
, vec_elem
, 1);
502 const glsl_type
*bvec
= glsl_type::get_instance(GLSL_TYPE_BOOL
, vec_elem
, 1);
505 ir_constant
*zeroi
= ir_constant::zero(ir
, ivec
);
507 ir_constant
*sign_mask
= new(ir
) ir_constant(0x80000000u
);
509 ir_constant
*exp_shift
= new(ir
) ir_constant(20u);
510 ir_constant
*exp_width
= new(ir
) ir_constant(11u);
511 ir_constant
*exp_bias
= new(ir
) ir_constant(1022, vec_elem
);
513 /* Temporary variables */
514 ir_variable
*x
= new(ir
) ir_variable(ir
->type
, "x", ir_var_temporary
);
515 ir_variable
*exp
= new(ir
) ir_variable(ivec
, "exp", ir_var_temporary
);
517 ir_variable
*zero_sign_x
= new(ir
) ir_variable(ir
->type
, "zero_sign_x",
520 ir_variable
*extracted_biased_exp
=
521 new(ir
) ir_variable(ivec
, "extracted_biased_exp", ir_var_temporary
);
522 ir_variable
*resulting_biased_exp
=
523 new(ir
) ir_variable(ivec
, "resulting_biased_exp", ir_var_temporary
);
525 ir_variable
*is_not_zero_or_underflow
=
526 new(ir
) ir_variable(bvec
, "is_not_zero_or_underflow", ir_var_temporary
);
528 ir_instruction
&i
= *base_ir
;
530 /* Copy <x> and <exp> arguments. */
532 i
.insert_before(assign(x
, ir
->operands
[0]));
533 i
.insert_before(exp
);
534 i
.insert_before(assign(exp
, ir
->operands
[1]));
536 ir_expression
*frexp_exp
= expr(ir_unop_frexp_exp
, x
);
537 if (lowering(DFREXP_DLDEXP_TO_ARITH
))
538 dfrexp_exp_to_arith(frexp_exp
);
540 /* Extract the biased exponent from <x>. */
541 i
.insert_before(extracted_biased_exp
);
542 i
.insert_before(assign(extracted_biased_exp
, add(frexp_exp
, exp_bias
)));
544 i
.insert_before(resulting_biased_exp
);
545 i
.insert_before(assign(resulting_biased_exp
,
546 add(extracted_biased_exp
, exp
)));
548 /* Test if result is ±0.0, subnormal, or underflow by checking if the
549 * resulting biased exponent would be less than 0x1. If so, the result is
550 * 0.0 with the sign of x. (Actually, invert the conditions so that
551 * immediate values are the second arguments, which is better for i965)
552 * TODO: Implement in a vector fashion.
554 i
.insert_before(zero_sign_x
);
555 for (unsigned elem
= 0; elem
< vec_elem
; elem
++) {
556 ir_variable
*unpacked
=
557 new(ir
) ir_variable(glsl_type::uvec2_type
, "unpacked", ir_var_temporary
);
558 i
.insert_before(unpacked
);
561 expr(ir_unop_unpack_double_2x32
, swizzle(x
, elem
, 1))));
562 i
.insert_before(assign(unpacked
, bit_and(swizzle_y(unpacked
), sign_mask
->clone(ir
, NULL
)),
564 i
.insert_before(assign(unpacked
, ir_constant::zero(ir
, glsl_type::uint_type
), WRITEMASK_X
));
565 i
.insert_before(assign(zero_sign_x
,
566 expr(ir_unop_pack_double_2x32
, unpacked
),
569 i
.insert_before(is_not_zero_or_underflow
);
570 i
.insert_before(assign(is_not_zero_or_underflow
,
571 gequal(resulting_biased_exp
,
572 new(ir
) ir_constant(0x1, vec_elem
))));
573 i
.insert_before(assign(x
, csel(is_not_zero_or_underflow
,
575 i
.insert_before(assign(resulting_biased_exp
,
576 csel(is_not_zero_or_underflow
,
577 resulting_biased_exp
, zeroi
)));
579 /* We could test for overflows by checking if the resulting biased exponent
580 * would be greater than 0xFE. Turns out we don't need to because the GLSL
583 * "If this product is too large to be represented in the
584 * floating-point type, the result is undefined."
587 ir_rvalue
*results
[4] = {NULL
};
588 for (unsigned elem
= 0; elem
< vec_elem
; elem
++) {
589 ir_variable
*unpacked
=
590 new(ir
) ir_variable(glsl_type::uvec2_type
, "unpacked", ir_var_temporary
);
591 i
.insert_before(unpacked
);
594 expr(ir_unop_unpack_double_2x32
, swizzle(x
, elem
, 1))));
596 ir_expression
*bfi
= bitfield_insert(
598 i2u(swizzle(resulting_biased_exp
, elem
, 1)),
599 exp_shift
->clone(ir
, NULL
),
600 exp_width
->clone(ir
, NULL
));
602 i
.insert_before(assign(unpacked
, bfi
, WRITEMASK_Y
));
604 results
[elem
] = expr(ir_unop_pack_double_2x32
, unpacked
);
607 ir
->operation
= ir_quadop_vector
;
608 ir
->init_num_operands();
609 ir
->operands
[0] = results
[0];
610 ir
->operands
[1] = results
[1];
611 ir
->operands
[2] = results
[2];
612 ir
->operands
[3] = results
[3];
614 /* Don't generate new IR that would need to be lowered in an additional
618 this->progress
= true;
622 lower_instructions_visitor::dfrexp_sig_to_arith(ir_expression
*ir
)
624 const unsigned vec_elem
= ir
->type
->vector_elements
;
625 const glsl_type
*bvec
= glsl_type::get_instance(GLSL_TYPE_BOOL
, vec_elem
, 1);
627 /* Double-precision floating-point values are stored as
632 * We're just extracting the significand here, so we only need to modify
633 * the upper 32-bit uint. Unfortunately we must extract each double
634 * independently as there is no vector version of unpackDouble.
637 ir_instruction
&i
= *base_ir
;
639 ir_variable
*is_not_zero
=
640 new(ir
) ir_variable(bvec
, "is_not_zero", ir_var_temporary
);
641 ir_rvalue
*results
[4] = {NULL
};
643 ir_constant
*dzero
= new(ir
) ir_constant(0.0, vec_elem
);
644 i
.insert_before(is_not_zero
);
647 nequal(abs(ir
->operands
[0]->clone(ir
, NULL
)), dzero
)));
649 /* TODO: Remake this as more vector-friendly when int64 support is
652 for (unsigned elem
= 0; elem
< vec_elem
; elem
++) {
653 ir_constant
*zero
= new(ir
) ir_constant(0u, 1);
654 ir_constant
*sign_mantissa_mask
= new(ir
) ir_constant(0x800fffffu
, 1);
656 /* Exponent of double floating-point values in the range [0.5, 1.0). */
657 ir_constant
*exponent_value
= new(ir
) ir_constant(0x3fe00000u
, 1);
660 new(ir
) ir_variable(glsl_type::uint_type
, "bits", ir_var_temporary
);
661 ir_variable
*unpacked
=
662 new(ir
) ir_variable(glsl_type::uvec2_type
, "unpacked", ir_var_temporary
);
664 ir_rvalue
*x
= swizzle(ir
->operands
[0]->clone(ir
, NULL
), elem
, 1);
666 i
.insert_before(bits
);
667 i
.insert_before(unpacked
);
668 i
.insert_before(assign(unpacked
, expr(ir_unop_unpack_double_2x32
, x
)));
670 /* Manipulate the high uint to remove the exponent and replace it with
671 * either the default exponent or zero.
673 i
.insert_before(assign(bits
, swizzle_y(unpacked
)));
674 i
.insert_before(assign(bits
, bit_and(bits
, sign_mantissa_mask
)));
675 i
.insert_before(assign(bits
, bit_or(bits
,
676 csel(swizzle(is_not_zero
, elem
, 1),
679 i
.insert_before(assign(unpacked
, bits
, WRITEMASK_Y
));
680 results
[elem
] = expr(ir_unop_pack_double_2x32
, unpacked
);
683 /* Put the dvec back together */
684 ir
->operation
= ir_quadop_vector
;
685 ir
->init_num_operands();
686 ir
->operands
[0] = results
[0];
687 ir
->operands
[1] = results
[1];
688 ir
->operands
[2] = results
[2];
689 ir
->operands
[3] = results
[3];
691 this->progress
= true;
695 lower_instructions_visitor::dfrexp_exp_to_arith(ir_expression
*ir
)
697 const unsigned vec_elem
= ir
->type
->vector_elements
;
698 const glsl_type
*bvec
= glsl_type::get_instance(GLSL_TYPE_BOOL
, vec_elem
, 1);
699 const glsl_type
*uvec
= glsl_type::get_instance(GLSL_TYPE_UINT
, vec_elem
, 1);
701 /* Double-precision floating-point values are stored as
706 * We're just extracting the exponent here, so we only care about the upper
710 ir_instruction
&i
= *base_ir
;
712 ir_variable
*is_not_zero
=
713 new(ir
) ir_variable(bvec
, "is_not_zero", ir_var_temporary
);
714 ir_variable
*high_words
=
715 new(ir
) ir_variable(uvec
, "high_words", ir_var_temporary
);
716 ir_constant
*dzero
= new(ir
) ir_constant(0.0, vec_elem
);
717 ir_constant
*izero
= new(ir
) ir_constant(0, vec_elem
);
719 ir_rvalue
*absval
= abs(ir
->operands
[0]);
721 i
.insert_before(is_not_zero
);
722 i
.insert_before(high_words
);
723 i
.insert_before(assign(is_not_zero
, nequal(absval
->clone(ir
, NULL
), dzero
)));
725 /* Extract all of the upper uints. */
726 for (unsigned elem
= 0; elem
< vec_elem
; elem
++) {
727 ir_rvalue
*x
= swizzle(absval
->clone(ir
, NULL
), elem
, 1);
729 i
.insert_before(assign(high_words
,
730 swizzle_y(expr(ir_unop_unpack_double_2x32
, x
)),
734 ir_constant
*exponent_shift
= new(ir
) ir_constant(20, vec_elem
);
735 ir_constant
*exponent_bias
= new(ir
) ir_constant(-1022, vec_elem
);
737 /* For non-zero inputs, shift the exponent down and apply bias. */
738 ir
->operation
= ir_triop_csel
;
739 ir
->init_num_operands();
740 ir
->operands
[0] = new(ir
) ir_dereference_variable(is_not_zero
);
741 ir
->operands
[1] = add(exponent_bias
, u2i(rshift(high_words
, exponent_shift
)));
742 ir
->operands
[2] = izero
;
744 this->progress
= true;
748 lower_instructions_visitor::carry_to_arith(ir_expression
*ir
)
753 * sum = ir_binop_add x y
754 * bcarry = ir_binop_less sum x
755 * carry = ir_unop_b2i bcarry
758 ir_rvalue
*x_clone
= ir
->operands
[0]->clone(ir
, NULL
);
759 ir
->operation
= ir_unop_i2u
;
760 ir
->init_num_operands();
761 ir
->operands
[0] = b2i(less(add(ir
->operands
[0], ir
->operands
[1]), x_clone
));
762 ir
->operands
[1] = NULL
;
764 this->progress
= true;
768 lower_instructions_visitor::borrow_to_arith(ir_expression
*ir
)
771 * ir_binop_borrow x y
773 * bcarry = ir_binop_less x y
774 * carry = ir_unop_b2i bcarry
777 ir
->operation
= ir_unop_i2u
;
778 ir
->init_num_operands();
779 ir
->operands
[0] = b2i(less(ir
->operands
[0], ir
->operands
[1]));
780 ir
->operands
[1] = NULL
;
782 this->progress
= true;
786 lower_instructions_visitor::sat_to_clamp(ir_expression
*ir
)
791 * ir_binop_min (ir_binop_max(x, 0.0), 1.0)
794 ir
->operation
= ir_binop_min
;
795 ir
->init_num_operands();
796 ir
->operands
[0] = new(ir
) ir_expression(ir_binop_max
, ir
->operands
[0]->type
,
798 new(ir
) ir_constant(0.0f
));
799 ir
->operands
[1] = new(ir
) ir_constant(1.0f
);
801 this->progress
= true;
805 lower_instructions_visitor::double_dot_to_fma(ir_expression
*ir
)
807 ir_variable
*temp
= new(ir
) ir_variable(ir
->operands
[0]->type
->get_base_type(), "dot_res",
809 this->base_ir
->insert_before(temp
);
811 int nc
= ir
->operands
[0]->type
->components();
812 for (int i
= nc
- 1; i
>= 1; i
--) {
813 ir_assignment
*assig
;
815 assig
= assign(temp
, mul(swizzle(ir
->operands
[0]->clone(ir
, NULL
), i
, 1),
816 swizzle(ir
->operands
[1]->clone(ir
, NULL
), i
, 1)));
818 assig
= assign(temp
, fma(swizzle(ir
->operands
[0]->clone(ir
, NULL
), i
, 1),
819 swizzle(ir
->operands
[1]->clone(ir
, NULL
), i
, 1),
822 this->base_ir
->insert_before(assig
);
825 ir
->operation
= ir_triop_fma
;
826 ir
->init_num_operands();
827 ir
->operands
[0] = swizzle(ir
->operands
[0], 0, 1);
828 ir
->operands
[1] = swizzle(ir
->operands
[1], 0, 1);
829 ir
->operands
[2] = new(ir
) ir_dereference_variable(temp
);
831 this->progress
= true;
836 lower_instructions_visitor::double_lrp(ir_expression
*ir
)
839 ir_rvalue
*op0
= ir
->operands
[0], *op2
= ir
->operands
[2];
840 ir_constant
*one
= new(ir
) ir_constant(1.0, op2
->type
->vector_elements
);
842 switch (op2
->type
->vector_elements
) {
844 swizval
= SWIZZLE_XXXX
;
847 assert(op0
->type
->vector_elements
== op2
->type
->vector_elements
);
848 swizval
= SWIZZLE_XYZW
;
852 ir
->operation
= ir_triop_fma
;
853 ir
->init_num_operands();
854 ir
->operands
[0] = swizzle(op2
, swizval
, op0
->type
->vector_elements
);
855 ir
->operands
[2] = mul(sub(one
, op2
->clone(ir
, NULL
)), op0
);
857 this->progress
= true;
861 lower_instructions_visitor::dceil_to_dfrac(ir_expression
*ir
)
865 * temp = sub(x, frtemp);
866 * result = temp + ((frtemp != 0.0) ? 1.0 : 0.0);
868 ir_instruction
&i
= *base_ir
;
869 ir_constant
*zero
= new(ir
) ir_constant(0.0, ir
->operands
[0]->type
->vector_elements
);
870 ir_constant
*one
= new(ir
) ir_constant(1.0, ir
->operands
[0]->type
->vector_elements
);
871 ir_variable
*frtemp
= new(ir
) ir_variable(ir
->operands
[0]->type
, "frtemp",
874 i
.insert_before(frtemp
);
875 i
.insert_before(assign(frtemp
, fract(ir
->operands
[0])));
877 ir
->operation
= ir_binop_add
;
878 ir
->init_num_operands();
879 ir
->operands
[0] = sub(ir
->operands
[0]->clone(ir
, NULL
), frtemp
);
880 ir
->operands
[1] = csel(nequal(frtemp
, zero
), one
, zero
->clone(ir
, NULL
));
882 this->progress
= true;
886 lower_instructions_visitor::dfloor_to_dfrac(ir_expression
*ir
)
890 * result = sub(x, frtemp);
892 ir
->operation
= ir_binop_sub
;
893 ir
->init_num_operands();
894 ir
->operands
[1] = fract(ir
->operands
[0]->clone(ir
, NULL
));
896 this->progress
= true;
899 lower_instructions_visitor::dround_even_to_dfrac(ir_expression
*ir
)
904 * frtemp = frac(temp);
905 * t2 = sub(temp, frtemp);
906 * if (frac(x) == 0.5)
907 * result = frac(t2 * 0.5) == 0 ? t2 : t2 - 1;
912 ir_instruction
&i
= *base_ir
;
913 ir_variable
*frtemp
= new(ir
) ir_variable(ir
->operands
[0]->type
, "frtemp",
915 ir_variable
*temp
= new(ir
) ir_variable(ir
->operands
[0]->type
, "temp",
917 ir_variable
*t2
= new(ir
) ir_variable(ir
->operands
[0]->type
, "t2",
919 ir_constant
*p5
= new(ir
) ir_constant(0.5, ir
->operands
[0]->type
->vector_elements
);
920 ir_constant
*one
= new(ir
) ir_constant(1.0, ir
->operands
[0]->type
->vector_elements
);
921 ir_constant
*zero
= new(ir
) ir_constant(0.0, ir
->operands
[0]->type
->vector_elements
);
923 i
.insert_before(temp
);
924 i
.insert_before(assign(temp
, add(ir
->operands
[0], p5
)));
926 i
.insert_before(frtemp
);
927 i
.insert_before(assign(frtemp
, fract(temp
)));
930 i
.insert_before(assign(t2
, sub(temp
, frtemp
)));
932 ir
->operation
= ir_triop_csel
;
933 ir
->init_num_operands();
934 ir
->operands
[0] = equal(fract(ir
->operands
[0]->clone(ir
, NULL
)),
935 p5
->clone(ir
, NULL
));
936 ir
->operands
[1] = csel(equal(fract(mul(t2
, p5
->clone(ir
, NULL
))),
940 ir
->operands
[2] = new(ir
) ir_dereference_variable(t2
);
942 this->progress
= true;
946 lower_instructions_visitor::dtrunc_to_dfrac(ir_expression
*ir
)
950 * temp = sub(x, frtemp);
951 * result = x >= 0 ? temp : temp + (frtemp == 0.0) ? 0 : 1;
953 ir_rvalue
*arg
= ir
->operands
[0];
954 ir_instruction
&i
= *base_ir
;
956 ir_constant
*zero
= new(ir
) ir_constant(0.0, arg
->type
->vector_elements
);
957 ir_constant
*one
= new(ir
) ir_constant(1.0, arg
->type
->vector_elements
);
958 ir_variable
*frtemp
= new(ir
) ir_variable(arg
->type
, "frtemp",
960 ir_variable
*temp
= new(ir
) ir_variable(ir
->operands
[0]->type
, "temp",
963 i
.insert_before(frtemp
);
964 i
.insert_before(assign(frtemp
, fract(arg
)));
965 i
.insert_before(temp
);
966 i
.insert_before(assign(temp
, sub(arg
->clone(ir
, NULL
), frtemp
)));
968 ir
->operation
= ir_triop_csel
;
969 ir
->init_num_operands();
970 ir
->operands
[0] = gequal(arg
->clone(ir
, NULL
), zero
);
971 ir
->operands
[1] = new (ir
) ir_dereference_variable(temp
);
972 ir
->operands
[2] = add(temp
,
973 csel(equal(frtemp
, zero
->clone(ir
, NULL
)),
974 zero
->clone(ir
, NULL
),
977 this->progress
= true;
981 lower_instructions_visitor::dsign_to_csel(ir_expression
*ir
)
984 * temp = x > 0.0 ? 1.0 : 0.0;
985 * result = x < 0.0 ? -1.0 : temp;
987 ir_rvalue
*arg
= ir
->operands
[0];
988 ir_constant
*zero
= new(ir
) ir_constant(0.0, arg
->type
->vector_elements
);
989 ir_constant
*one
= new(ir
) ir_constant(1.0, arg
->type
->vector_elements
);
990 ir_constant
*neg_one
= new(ir
) ir_constant(-1.0, arg
->type
->vector_elements
);
992 ir
->operation
= ir_triop_csel
;
993 ir
->init_num_operands();
994 ir
->operands
[0] = less(arg
->clone(ir
, NULL
),
995 zero
->clone(ir
, NULL
));
996 ir
->operands
[1] = neg_one
;
997 ir
->operands
[2] = csel(greater(arg
, zero
),
999 zero
->clone(ir
, NULL
));
1001 this->progress
= true;
1005 lower_instructions_visitor::bit_count_to_math(ir_expression
*ir
)
1007 /* For more details, see:
1009 * http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetPaallel
1011 const unsigned elements
= ir
->operands
[0]->type
->vector_elements
;
1012 ir_variable
*temp
= new(ir
) ir_variable(glsl_type::uvec(elements
), "temp",
1014 ir_constant
*c55555555
= new(ir
) ir_constant(0x55555555u
);
1015 ir_constant
*c33333333
= new(ir
) ir_constant(0x33333333u
);
1016 ir_constant
*c0F0F0F0F
= new(ir
) ir_constant(0x0F0F0F0Fu
);
1017 ir_constant
*c01010101
= new(ir
) ir_constant(0x01010101u
);
1018 ir_constant
*c1
= new(ir
) ir_constant(1u);
1019 ir_constant
*c2
= new(ir
) ir_constant(2u);
1020 ir_constant
*c4
= new(ir
) ir_constant(4u);
1021 ir_constant
*c24
= new(ir
) ir_constant(24u);
1023 base_ir
->insert_before(temp
);
1025 if (ir
->operands
[0]->type
->base_type
== GLSL_TYPE_UINT
) {
1026 base_ir
->insert_before(assign(temp
, ir
->operands
[0]));
1028 assert(ir
->operands
[0]->type
->base_type
== GLSL_TYPE_INT
);
1029 base_ir
->insert_before(assign(temp
, i2u(ir
->operands
[0])));
1032 /* temp = temp - ((temp >> 1) & 0x55555555u); */
1033 base_ir
->insert_before(assign(temp
, sub(temp
, bit_and(rshift(temp
, c1
),
1036 /* temp = (temp & 0x33333333u) + ((temp >> 2) & 0x33333333u); */
1037 base_ir
->insert_before(assign(temp
, add(bit_and(temp
, c33333333
),
1038 bit_and(rshift(temp
, c2
),
1039 c33333333
->clone(ir
, NULL
)))));
1041 /* int(((temp + (temp >> 4) & 0xF0F0F0Fu) * 0x1010101u) >> 24); */
1042 ir
->operation
= ir_unop_u2i
;
1043 ir
->init_num_operands();
1044 ir
->operands
[0] = rshift(mul(bit_and(add(temp
, rshift(temp
, c4
)), c0F0F0F0F
),
1048 this->progress
= true;
1052 lower_instructions_visitor::extract_to_shifts(ir_expression
*ir
)
1055 new(ir
) ir_variable(ir
->operands
[0]->type
, "bits", ir_var_temporary
);
1057 base_ir
->insert_before(bits
);
1058 base_ir
->insert_before(assign(bits
, ir
->operands
[2]));
1060 if (ir
->operands
[0]->type
->base_type
== GLSL_TYPE_UINT
) {
1062 new(ir
) ir_constant(1u, ir
->operands
[0]->type
->vector_elements
);
1064 new(ir
) ir_constant(32u, ir
->operands
[0]->type
->vector_elements
);
1065 ir_constant
*cFFFFFFFF
=
1066 new(ir
) ir_constant(0xFFFFFFFFu
, ir
->operands
[0]->type
->vector_elements
);
1068 /* At least some hardware treats (x << y) as (x << (y%32)). This means
1069 * we'd get a mask of 0 when bits is 32. Special case it.
1071 * mask = bits == 32 ? 0xffffffff : (1u << bits) - 1u;
1073 ir_expression
*mask
= csel(equal(bits
, c32
),
1075 sub(lshift(c1
, bits
), c1
->clone(ir
, NULL
)));
1077 /* Section 8.8 (Integer Functions) of the GLSL 4.50 spec says:
1079 * If bits is zero, the result will be zero.
1081 * Since (1 << 0) - 1 == 0, we don't need to bother with the conditional
1082 * select as in the signed integer case.
1084 * (value >> offset) & mask;
1086 ir
->operation
= ir_binop_bit_and
;
1087 ir
->init_num_operands();
1088 ir
->operands
[0] = rshift(ir
->operands
[0], ir
->operands
[1]);
1089 ir
->operands
[1] = mask
;
1090 ir
->operands
[2] = NULL
;
1093 new(ir
) ir_constant(int(0), ir
->operands
[0]->type
->vector_elements
);
1095 new(ir
) ir_constant(int(32), ir
->operands
[0]->type
->vector_elements
);
1097 new(ir
) ir_variable(ir
->operands
[0]->type
, "temp", ir_var_temporary
);
1099 /* temp = 32 - bits; */
1100 base_ir
->insert_before(temp
);
1101 base_ir
->insert_before(assign(temp
, sub(c32
, bits
)));
1103 /* expr = value << (temp - offset)) >> temp; */
1104 ir_expression
*expr
=
1105 rshift(lshift(ir
->operands
[0], sub(temp
, ir
->operands
[1])), temp
);
1107 /* Section 8.8 (Integer Functions) of the GLSL 4.50 spec says:
1109 * If bits is zero, the result will be zero.
1111 * Due to the (x << (y%32)) behavior mentioned before, the (value <<
1112 * (32-0)) doesn't "erase" all of the data as we would like, so finish
1115 * (bits == 0) ? 0 : e;
1117 ir
->operation
= ir_triop_csel
;
1118 ir
->init_num_operands();
1119 ir
->operands
[0] = equal(c0
, bits
);
1120 ir
->operands
[1] = c0
->clone(ir
, NULL
);
1121 ir
->operands
[2] = expr
;
1124 this->progress
= true;
1128 lower_instructions_visitor::insert_to_shifts(ir_expression
*ir
)
1132 ir_constant
*cFFFFFFFF
;
1133 ir_variable
*offset
=
1134 new(ir
) ir_variable(ir
->operands
[0]->type
, "offset", ir_var_temporary
);
1136 new(ir
) ir_variable(ir
->operands
[0]->type
, "bits", ir_var_temporary
);
1138 new(ir
) ir_variable(ir
->operands
[0]->type
, "mask", ir_var_temporary
);
1140 if (ir
->operands
[0]->type
->base_type
== GLSL_TYPE_INT
) {
1141 c1
= new(ir
) ir_constant(int(1), ir
->operands
[0]->type
->vector_elements
);
1142 c32
= new(ir
) ir_constant(int(32), ir
->operands
[0]->type
->vector_elements
);
1143 cFFFFFFFF
= new(ir
) ir_constant(int(0xFFFFFFFF), ir
->operands
[0]->type
->vector_elements
);
1145 assert(ir
->operands
[0]->type
->base_type
== GLSL_TYPE_UINT
);
1147 c1
= new(ir
) ir_constant(1u, ir
->operands
[0]->type
->vector_elements
);
1148 c32
= new(ir
) ir_constant(32u, ir
->operands
[0]->type
->vector_elements
);
1149 cFFFFFFFF
= new(ir
) ir_constant(0xFFFFFFFFu
, ir
->operands
[0]->type
->vector_elements
);
1152 base_ir
->insert_before(offset
);
1153 base_ir
->insert_before(assign(offset
, ir
->operands
[2]));
1155 base_ir
->insert_before(bits
);
1156 base_ir
->insert_before(assign(bits
, ir
->operands
[3]));
1158 /* At least some hardware treats (x << y) as (x << (y%32)). This means
1159 * we'd get a mask of 0 when bits is 32. Special case it.
1161 * mask = (bits == 32 ? 0xffffffff : (1u << bits) - 1u) << offset;
1163 * Section 8.8 (Integer Functions) of the GLSL 4.50 spec says:
1165 * The result will be undefined if offset or bits is negative, or if the
1166 * sum of offset and bits is greater than the number of bits used to
1167 * store the operand.
1169 * Since it's undefined, there are a couple other ways this could be
1170 * implemented. The other way that was considered was to put the csel
1171 * around the whole thing:
1173 * final_result = bits == 32 ? insert : ... ;
1175 base_ir
->insert_before(mask
);
1177 base_ir
->insert_before(assign(mask
, csel(equal(bits
, c32
),
1179 lshift(sub(lshift(c1
, bits
),
1180 c1
->clone(ir
, NULL
)),
1183 /* (base & ~mask) | ((insert << offset) & mask) */
1184 ir
->operation
= ir_binop_bit_or
;
1185 ir
->init_num_operands();
1186 ir
->operands
[0] = bit_and(ir
->operands
[0], bit_not(mask
));
1187 ir
->operands
[1] = bit_and(lshift(ir
->operands
[1], offset
), mask
);
1188 ir
->operands
[2] = NULL
;
1189 ir
->operands
[3] = NULL
;
1191 this->progress
= true;
1195 lower_instructions_visitor::reverse_to_shifts(ir_expression
*ir
)
1197 /* For more details, see:
1199 * http://graphics.stanford.edu/~seander/bithacks.html#ReverseParallel
1202 new(ir
) ir_constant(1u, ir
->operands
[0]->type
->vector_elements
);
1204 new(ir
) ir_constant(2u, ir
->operands
[0]->type
->vector_elements
);
1206 new(ir
) ir_constant(4u, ir
->operands
[0]->type
->vector_elements
);
1208 new(ir
) ir_constant(8u, ir
->operands
[0]->type
->vector_elements
);
1210 new(ir
) ir_constant(16u, ir
->operands
[0]->type
->vector_elements
);
1211 ir_constant
*c33333333
=
1212 new(ir
) ir_constant(0x33333333u
, ir
->operands
[0]->type
->vector_elements
);
1213 ir_constant
*c55555555
=
1214 new(ir
) ir_constant(0x55555555u
, ir
->operands
[0]->type
->vector_elements
);
1215 ir_constant
*c0F0F0F0F
=
1216 new(ir
) ir_constant(0x0F0F0F0Fu
, ir
->operands
[0]->type
->vector_elements
);
1217 ir_constant
*c00FF00FF
=
1218 new(ir
) ir_constant(0x00FF00FFu
, ir
->operands
[0]->type
->vector_elements
);
1220 new(ir
) ir_variable(glsl_type::uvec(ir
->operands
[0]->type
->vector_elements
),
1221 "temp", ir_var_temporary
);
1222 ir_instruction
&i
= *base_ir
;
1224 i
.insert_before(temp
);
1226 if (ir
->operands
[0]->type
->base_type
== GLSL_TYPE_UINT
) {
1227 i
.insert_before(assign(temp
, ir
->operands
[0]));
1229 assert(ir
->operands
[0]->type
->base_type
== GLSL_TYPE_INT
);
1230 i
.insert_before(assign(temp
, i2u(ir
->operands
[0])));
1233 /* Swap odd and even bits.
1235 * temp = ((temp >> 1) & 0x55555555u) | ((temp & 0x55555555u) << 1);
1237 i
.insert_before(assign(temp
, bit_or(bit_and(rshift(temp
, c1
), c55555555
),
1238 lshift(bit_and(temp
, c55555555
->clone(ir
, NULL
)),
1239 c1
->clone(ir
, NULL
)))));
1240 /* Swap consecutive pairs.
1242 * temp = ((temp >> 2) & 0x33333333u) | ((temp & 0x33333333u) << 2);
1244 i
.insert_before(assign(temp
, bit_or(bit_and(rshift(temp
, c2
), c33333333
),
1245 lshift(bit_and(temp
, c33333333
->clone(ir
, NULL
)),
1246 c2
->clone(ir
, NULL
)))));
1250 * temp = ((temp >> 4) & 0x0F0F0F0Fu) | ((temp & 0x0F0F0F0Fu) << 4);
1252 i
.insert_before(assign(temp
, bit_or(bit_and(rshift(temp
, c4
), c0F0F0F0F
),
1253 lshift(bit_and(temp
, c0F0F0F0F
->clone(ir
, NULL
)),
1254 c4
->clone(ir
, NULL
)))));
1256 /* The last step is, basically, bswap. Swap the bytes, then swap the
1257 * words. When this code is run through GCC on x86, it does generate a
1258 * bswap instruction.
1260 * temp = ((temp >> 8) & 0x00FF00FFu) | ((temp & 0x00FF00FFu) << 8);
1261 * temp = ( temp >> 16 ) | ( temp << 16);
1263 i
.insert_before(assign(temp
, bit_or(bit_and(rshift(temp
, c8
), c00FF00FF
),
1264 lshift(bit_and(temp
, c00FF00FF
->clone(ir
, NULL
)),
1265 c8
->clone(ir
, NULL
)))));
1267 if (ir
->operands
[0]->type
->base_type
== GLSL_TYPE_UINT
) {
1268 ir
->operation
= ir_binop_bit_or
;
1269 ir
->init_num_operands();
1270 ir
->operands
[0] = rshift(temp
, c16
);
1271 ir
->operands
[1] = lshift(temp
, c16
->clone(ir
, NULL
));
1273 ir
->operation
= ir_unop_u2i
;
1274 ir
->init_num_operands();
1275 ir
->operands
[0] = bit_or(rshift(temp
, c16
),
1276 lshift(temp
, c16
->clone(ir
, NULL
)));
1279 this->progress
= true;
1283 lower_instructions_visitor::find_lsb_to_float_cast(ir_expression
*ir
)
1285 /* For more details, see:
1287 * http://graphics.stanford.edu/~seander/bithacks.html#ZerosOnRightFloatCast
1289 const unsigned elements
= ir
->operands
[0]->type
->vector_elements
;
1290 ir_constant
*c0
= new(ir
) ir_constant(unsigned(0), elements
);
1291 ir_constant
*cminus1
= new(ir
) ir_constant(int(-1), elements
);
1292 ir_constant
*c23
= new(ir
) ir_constant(int(23), elements
);
1293 ir_constant
*c7F
= new(ir
) ir_constant(int(0x7F), elements
);
1295 new(ir
) ir_variable(glsl_type::ivec(elements
), "temp", ir_var_temporary
);
1296 ir_variable
*lsb_only
=
1297 new(ir
) ir_variable(glsl_type::uvec(elements
), "lsb_only", ir_var_temporary
);
1298 ir_variable
*as_float
=
1299 new(ir
) ir_variable(glsl_type::vec(elements
), "as_float", ir_var_temporary
);
1301 new(ir
) ir_variable(glsl_type::ivec(elements
), "lsb", ir_var_temporary
);
1303 ir_instruction
&i
= *base_ir
;
1305 i
.insert_before(temp
);
1307 if (ir
->operands
[0]->type
->base_type
== GLSL_TYPE_INT
) {
1308 i
.insert_before(assign(temp
, ir
->operands
[0]));
1310 assert(ir
->operands
[0]->type
->base_type
== GLSL_TYPE_UINT
);
1311 i
.insert_before(assign(temp
, u2i(ir
->operands
[0])));
1314 /* The int-to-float conversion is lossless because (value & -value) is
1315 * either a power of two or zero. We don't use the result in the zero
1316 * case. The uint() cast is necessary so that 0x80000000 does not
1317 * generate a negative value.
1319 * uint lsb_only = uint(value & -value);
1320 * float as_float = float(lsb_only);
1322 i
.insert_before(lsb_only
);
1323 i
.insert_before(assign(lsb_only
, i2u(bit_and(temp
, neg(temp
)))));
1325 i
.insert_before(as_float
);
1326 i
.insert_before(assign(as_float
, u2f(lsb_only
)));
1328 /* This is basically an open-coded frexp. Implementations that have a
1329 * native frexp instruction would be better served by that. This is
1330 * optimized versus a full-featured open-coded implementation in two ways:
1332 * - We don't care about a correct result from subnormal numbers (including
1333 * 0.0), so the raw exponent can always be safely unbiased.
1335 * - The value cannot be negative, so it does not need to be masked off to
1336 * extract the exponent.
1338 * int lsb = (floatBitsToInt(as_float) >> 23) - 0x7f;
1340 i
.insert_before(lsb
);
1341 i
.insert_before(assign(lsb
, sub(rshift(bitcast_f2i(as_float
), c23
), c7F
)));
1343 /* Use lsb_only in the comparison instead of temp so that the & (far above)
1344 * can possibly generate the result without an explicit comparison.
1346 * (lsb_only == 0) ? -1 : lsb;
1348 * Since our input values are all integers, the unbiased exponent must not
1349 * be negative. It will only be negative (-0x7f, in fact) if lsb_only is
1350 * 0. Instead of using (lsb_only == 0), we could use (lsb >= 0). Which is
1351 * better is likely GPU dependent. Either way, the difference should be
1354 ir
->operation
= ir_triop_csel
;
1355 ir
->init_num_operands();
1356 ir
->operands
[0] = equal(lsb_only
, c0
);
1357 ir
->operands
[1] = cminus1
;
1358 ir
->operands
[2] = new(ir
) ir_dereference_variable(lsb
);
1360 this->progress
= true;
1364 lower_instructions_visitor::find_msb_to_float_cast(ir_expression
*ir
)
1366 /* For more details, see:
1368 * http://graphics.stanford.edu/~seander/bithacks.html#ZerosOnRightFloatCast
1370 const unsigned elements
= ir
->operands
[0]->type
->vector_elements
;
1371 ir_constant
*c0
= new(ir
) ir_constant(int(0), elements
);
1372 ir_constant
*cminus1
= new(ir
) ir_constant(int(-1), elements
);
1373 ir_constant
*c23
= new(ir
) ir_constant(int(23), elements
);
1374 ir_constant
*c7F
= new(ir
) ir_constant(int(0x7F), elements
);
1375 ir_constant
*c000000FF
= new(ir
) ir_constant(0x000000FFu
, elements
);
1376 ir_constant
*cFFFFFF00
= new(ir
) ir_constant(0xFFFFFF00u
, elements
);
1378 new(ir
) ir_variable(glsl_type::uvec(elements
), "temp", ir_var_temporary
);
1379 ir_variable
*as_float
=
1380 new(ir
) ir_variable(glsl_type::vec(elements
), "as_float", ir_var_temporary
);
1382 new(ir
) ir_variable(glsl_type::ivec(elements
), "msb", ir_var_temporary
);
1384 ir_instruction
&i
= *base_ir
;
1386 i
.insert_before(temp
);
1388 if (ir
->operands
[0]->type
->base_type
== GLSL_TYPE_UINT
) {
1389 i
.insert_before(assign(temp
, ir
->operands
[0]));
1391 assert(ir
->operands
[0]->type
->base_type
== GLSL_TYPE_INT
);
1393 /* findMSB(uint(abs(some_int))) almost always does the right thing.
1394 * There are two problem values:
1396 * * 0x80000000. Since abs(0x80000000) == 0x80000000, findMSB returns
1397 * 31. However, findMSB(int(0x80000000)) == 30.
1399 * * 0xffffffff. Since abs(0xffffffff) == 1, findMSB returns
1400 * 31. Section 8.8 (Integer Functions) of the GLSL 4.50 spec says:
1402 * For a value of zero or negative one, -1 will be returned.
1404 * For all negative number cases, including 0x80000000 and 0xffffffff,
1405 * the correct value is obtained from findMSB if instead of negating the
1406 * (already negative) value the logical-not is used. A conditonal
1407 * logical-not can be achieved in two instructions.
1409 ir_variable
*as_int
=
1410 new(ir
) ir_variable(glsl_type::ivec(elements
), "as_int", ir_var_temporary
);
1411 ir_constant
*c31
= new(ir
) ir_constant(int(31), elements
);
1413 i
.insert_before(as_int
);
1414 i
.insert_before(assign(as_int
, ir
->operands
[0]));
1415 i
.insert_before(assign(temp
, i2u(expr(ir_binop_bit_xor
,
1417 rshift(as_int
, c31
)))));
1420 /* The int-to-float conversion is lossless because bits are conditionally
1421 * masked off the bottom of temp to ensure the value has at most 24 bits of
1422 * data or is zero. We don't use the result in the zero case. The uint()
1423 * cast is necessary so that 0x80000000 does not generate a negative value.
1425 * float as_float = float(temp > 255 ? temp & ~255 : temp);
1427 i
.insert_before(as_float
);
1428 i
.insert_before(assign(as_float
, u2f(csel(greater(temp
, c000000FF
),
1429 bit_and(temp
, cFFFFFF00
),
1432 /* This is basically an open-coded frexp. Implementations that have a
1433 * native frexp instruction would be better served by that. This is
1434 * optimized versus a full-featured open-coded implementation in two ways:
1436 * - We don't care about a correct result from subnormal numbers (including
1437 * 0.0), so the raw exponent can always be safely unbiased.
1439 * - The value cannot be negative, so it does not need to be masked off to
1440 * extract the exponent.
1442 * int msb = (floatBitsToInt(as_float) >> 23) - 0x7f;
1444 i
.insert_before(msb
);
1445 i
.insert_before(assign(msb
, sub(rshift(bitcast_f2i(as_float
), c23
), c7F
)));
1447 /* Use msb in the comparison instead of temp so that the subtract can
1448 * possibly generate the result without an explicit comparison.
1450 * (msb < 0) ? -1 : msb;
1452 * Since our input values are all integers, the unbiased exponent must not
1453 * be negative. It will only be negative (-0x7f, in fact) if temp is 0.
1455 ir
->operation
= ir_triop_csel
;
1456 ir
->init_num_operands();
1457 ir
->operands
[0] = less(msb
, c0
);
1458 ir
->operands
[1] = cminus1
;
1459 ir
->operands
[2] = new(ir
) ir_dereference_variable(msb
);
1461 this->progress
= true;
1465 lower_instructions_visitor::_carry(operand a
, operand b
)
1467 if (lowering(CARRY_TO_ARITH
))
1468 return i2u(b2i(less(add(a
, b
),
1469 a
.val
->clone(ralloc_parent(a
.val
), NULL
))));
1475 lower_instructions_visitor::imul_high_to_mul(ir_expression
*ir
)
1480 * (GH * CD) + (GH * AB) << 16 + (EF * CD) << 16 + (EF * AB) << 32
1482 * In GLSL, (a * b) becomes
1484 * uint m1 = (a & 0x0000ffffu) * (b & 0x0000ffffu);
1485 * uint m2 = (a & 0x0000ffffu) * (b >> 16);
1486 * uint m3 = (a >> 16) * (b & 0x0000ffffu);
1487 * uint m4 = (a >> 16) * (b >> 16);
1494 * lo_result = uaddCarry(m1, m2 << 16, c1);
1495 * hi_result = m4 + c1;
1496 * lo_result = uaddCarry(lo_result, m3 << 16, c2);
1497 * hi_result = hi_result + c2;
1498 * hi_result = hi_result + (m2 >> 16) + (m3 >> 16);
1500 const unsigned elements
= ir
->operands
[0]->type
->vector_elements
;
1502 new(ir
) ir_variable(glsl_type::uvec(elements
), "src1", ir_var_temporary
);
1503 ir_variable
*src1h
=
1504 new(ir
) ir_variable(glsl_type::uvec(elements
), "src1h", ir_var_temporary
);
1505 ir_variable
*src1l
=
1506 new(ir
) ir_variable(glsl_type::uvec(elements
), "src1l", ir_var_temporary
);
1508 new(ir
) ir_variable(glsl_type::uvec(elements
), "src2", ir_var_temporary
);
1509 ir_variable
*src2h
=
1510 new(ir
) ir_variable(glsl_type::uvec(elements
), "src2h", ir_var_temporary
);
1511 ir_variable
*src2l
=
1512 new(ir
) ir_variable(glsl_type::uvec(elements
), "src2l", ir_var_temporary
);
1514 new(ir
) ir_variable(glsl_type::uvec(elements
), "t1", ir_var_temporary
);
1516 new(ir
) ir_variable(glsl_type::uvec(elements
), "t2", ir_var_temporary
);
1518 new(ir
) ir_variable(glsl_type::uvec(elements
), "lo", ir_var_temporary
);
1520 new(ir
) ir_variable(glsl_type::uvec(elements
), "hi", ir_var_temporary
);
1521 ir_variable
*different_signs
= NULL
;
1522 ir_constant
*c0000FFFF
= new(ir
) ir_constant(0x0000FFFFu
, elements
);
1523 ir_constant
*c16
= new(ir
) ir_constant(16u, elements
);
1525 ir_instruction
&i
= *base_ir
;
1527 i
.insert_before(src1
);
1528 i
.insert_before(src2
);
1529 i
.insert_before(src1h
);
1530 i
.insert_before(src2h
);
1531 i
.insert_before(src1l
);
1532 i
.insert_before(src2l
);
1534 if (ir
->operands
[0]->type
->base_type
== GLSL_TYPE_UINT
) {
1535 i
.insert_before(assign(src1
, ir
->operands
[0]));
1536 i
.insert_before(assign(src2
, ir
->operands
[1]));
1538 assert(ir
->operands
[0]->type
->base_type
== GLSL_TYPE_INT
);
1540 ir_variable
*itmp1
=
1541 new(ir
) ir_variable(glsl_type::ivec(elements
), "itmp1", ir_var_temporary
);
1542 ir_variable
*itmp2
=
1543 new(ir
) ir_variable(glsl_type::ivec(elements
), "itmp2", ir_var_temporary
);
1544 ir_constant
*c0
= new(ir
) ir_constant(int(0), elements
);
1546 i
.insert_before(itmp1
);
1547 i
.insert_before(itmp2
);
1548 i
.insert_before(assign(itmp1
, ir
->operands
[0]));
1549 i
.insert_before(assign(itmp2
, ir
->operands
[1]));
1552 new(ir
) ir_variable(glsl_type::bvec(elements
), "different_signs",
1555 i
.insert_before(different_signs
);
1556 i
.insert_before(assign(different_signs
, expr(ir_binop_logic_xor
,
1558 less(itmp2
, c0
->clone(ir
, NULL
)))));
1560 i
.insert_before(assign(src1
, i2u(abs(itmp1
))));
1561 i
.insert_before(assign(src2
, i2u(abs(itmp2
))));
1564 i
.insert_before(assign(src1l
, bit_and(src1
, c0000FFFF
)));
1565 i
.insert_before(assign(src2l
, bit_and(src2
, c0000FFFF
->clone(ir
, NULL
))));
1566 i
.insert_before(assign(src1h
, rshift(src1
, c16
)));
1567 i
.insert_before(assign(src2h
, rshift(src2
, c16
->clone(ir
, NULL
))));
1569 i
.insert_before(lo
);
1570 i
.insert_before(hi
);
1571 i
.insert_before(t1
);
1572 i
.insert_before(t2
);
1574 i
.insert_before(assign(lo
, mul(src1l
, src2l
)));
1575 i
.insert_before(assign(t1
, mul(src1l
, src2h
)));
1576 i
.insert_before(assign(t2
, mul(src1h
, src2l
)));
1577 i
.insert_before(assign(hi
, mul(src1h
, src2h
)));
1579 i
.insert_before(assign(hi
, add(hi
, _carry(lo
, lshift(t1
, c16
->clone(ir
, NULL
))))));
1580 i
.insert_before(assign(lo
, add(lo
, lshift(t1
, c16
->clone(ir
, NULL
)))));
1582 i
.insert_before(assign(hi
, add(hi
, _carry(lo
, lshift(t2
, c16
->clone(ir
, NULL
))))));
1583 i
.insert_before(assign(lo
, add(lo
, lshift(t2
, c16
->clone(ir
, NULL
)))));
1585 if (different_signs
== NULL
) {
1586 assert(ir
->operands
[0]->type
->base_type
== GLSL_TYPE_UINT
);
1588 ir
->operation
= ir_binop_add
;
1589 ir
->init_num_operands();
1590 ir
->operands
[0] = add(hi
, rshift(t1
, c16
->clone(ir
, NULL
)));
1591 ir
->operands
[1] = rshift(t2
, c16
->clone(ir
, NULL
));
1593 assert(ir
->operands
[0]->type
->base_type
== GLSL_TYPE_INT
);
1595 i
.insert_before(assign(hi
, add(add(hi
, rshift(t1
, c16
->clone(ir
, NULL
))),
1596 rshift(t2
, c16
->clone(ir
, NULL
)))));
1598 /* For channels where different_signs is set we have to perform a 64-bit
1599 * negation. This is *not* the same as just negating the high 32-bits.
1600 * Consider -3 * 2. The high 32-bits is 0, but the desired result is
1601 * -1, not -0! Recall -x == ~x + 1.
1603 ir_variable
*neg_hi
=
1604 new(ir
) ir_variable(glsl_type::ivec(elements
), "neg_hi", ir_var_temporary
);
1605 ir_constant
*c1
= new(ir
) ir_constant(1u, elements
);
1607 i
.insert_before(neg_hi
);
1608 i
.insert_before(assign(neg_hi
, add(bit_not(u2i(hi
)),
1609 u2i(_carry(bit_not(lo
), c1
)))));
1611 ir
->operation
= ir_triop_csel
;
1612 ir
->init_num_operands();
1613 ir
->operands
[0] = new(ir
) ir_dereference_variable(different_signs
);
1614 ir
->operands
[1] = new(ir
) ir_dereference_variable(neg_hi
);
1615 ir
->operands
[2] = u2i(hi
);
1620 lower_instructions_visitor::sqrt_to_abs_sqrt(ir_expression
*ir
)
1622 ir
->operands
[0] = new(ir
) ir_expression(ir_unop_abs
, ir
->operands
[0]);
1623 this->progress
= true;
1627 lower_instructions_visitor::visit_leave(ir_expression
*ir
)
1629 switch (ir
->operation
) {
1631 if (ir
->operands
[0]->type
->is_double())
1632 double_dot_to_fma(ir
);
1635 if (ir
->operands
[0]->type
->is_double())
1639 if (lowering(SUB_TO_ADD_NEG
))
1644 if (ir
->operands
[1]->type
->is_integer() && lowering(INT_DIV_TO_MUL_RCP
))
1645 int_div_to_mul_rcp(ir
);
1646 else if ((ir
->operands
[1]->type
->is_float() && lowering(FDIV_TO_MUL_RCP
)) ||
1647 (ir
->operands
[1]->type
->is_double() && lowering(DDIV_TO_MUL_RCP
)))
1652 if (lowering(EXP_TO_EXP2
))
1657 if (lowering(LOG_TO_LOG2
))
1662 if (lowering(MOD_TO_FLOOR
) && (ir
->type
->is_float() || ir
->type
->is_double()))
1667 if (lowering(POW_TO_EXP2
))
1671 case ir_binop_ldexp
:
1672 if (lowering(LDEXP_TO_ARITH
) && ir
->type
->is_float())
1674 if (lowering(DFREXP_DLDEXP_TO_ARITH
) && ir
->type
->is_double())
1675 dldexp_to_arith(ir
);
1678 case ir_unop_frexp_exp
:
1679 if (lowering(DFREXP_DLDEXP_TO_ARITH
) && ir
->operands
[0]->type
->is_double())
1680 dfrexp_exp_to_arith(ir
);
1683 case ir_unop_frexp_sig
:
1684 if (lowering(DFREXP_DLDEXP_TO_ARITH
) && ir
->operands
[0]->type
->is_double())
1685 dfrexp_sig_to_arith(ir
);
1688 case ir_binop_carry
:
1689 if (lowering(CARRY_TO_ARITH
))
1693 case ir_binop_borrow
:
1694 if (lowering(BORROW_TO_ARITH
))
1695 borrow_to_arith(ir
);
1698 case ir_unop_saturate
:
1699 if (lowering(SAT_TO_CLAMP
))
1704 if (lowering(DOPS_TO_DFRAC
) && ir
->type
->is_double())
1705 dtrunc_to_dfrac(ir
);
1709 if (lowering(DOPS_TO_DFRAC
) && ir
->type
->is_double())
1714 if (lowering(DOPS_TO_DFRAC
) && ir
->type
->is_double())
1715 dfloor_to_dfrac(ir
);
1718 case ir_unop_round_even
:
1719 if (lowering(DOPS_TO_DFRAC
) && ir
->type
->is_double())
1720 dround_even_to_dfrac(ir
);
1724 if (lowering(DOPS_TO_DFRAC
) && ir
->type
->is_double())
1728 case ir_unop_bit_count
:
1729 if (lowering(BIT_COUNT_TO_MATH
))
1730 bit_count_to_math(ir
);
1733 case ir_triop_bitfield_extract
:
1734 if (lowering(EXTRACT_TO_SHIFTS
))
1735 extract_to_shifts(ir
);
1738 case ir_quadop_bitfield_insert
:
1739 if (lowering(INSERT_TO_SHIFTS
))
1740 insert_to_shifts(ir
);
1743 case ir_unop_bitfield_reverse
:
1744 if (lowering(REVERSE_TO_SHIFTS
))
1745 reverse_to_shifts(ir
);
1748 case ir_unop_find_lsb
:
1749 if (lowering(FIND_LSB_TO_FLOAT_CAST
))
1750 find_lsb_to_float_cast(ir
);
1753 case ir_unop_find_msb
:
1754 if (lowering(FIND_MSB_TO_FLOAT_CAST
))
1755 find_msb_to_float_cast(ir
);
1758 case ir_binop_imul_high
:
1759 if (lowering(IMUL_HIGH_TO_MUL
))
1760 imul_high_to_mul(ir
);
1765 if (lowering(SQRT_TO_ABS_SQRT
))
1766 sqrt_to_abs_sqrt(ir
);
1770 return visit_continue
;
1773 return visit_continue
;