glsl: set old ldexp operand to NULL when lowering
[mesa.git] / src / compiler / glsl / lower_instructions.cpp
1 /*
2 * Copyright © 2010 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 */
23
24 /**
25 * \file lower_instructions.cpp
26 *
27 * Many GPUs lack native instructions for certain expression operations, and
28 * must replace them with some other expression tree. This pass lowers some
29 * of the most common cases, allowing the lowering code to be implemented once
30 * rather than in each driver backend.
31 *
32 * Currently supported transformations:
33 * - SUB_TO_ADD_NEG
34 * - DIV_TO_MUL_RCP
35 * - INT_DIV_TO_MUL_RCP
36 * - EXP_TO_EXP2
37 * - POW_TO_EXP2
38 * - LOG_TO_LOG2
39 * - MOD_TO_FLOOR
40 * - LDEXP_TO_ARITH
41 * - DFREXP_TO_ARITH
42 * - CARRY_TO_ARITH
43 * - BORROW_TO_ARITH
44 * - SAT_TO_CLAMP
45 * - DOPS_TO_DFRAC
46 *
47 * SUB_TO_ADD_NEG:
48 * ---------------
49 * Breaks an ir_binop_sub expression down to add(op0, neg(op1))
50 *
51 * This simplifies expression reassociation, and for many backends
52 * there is no subtract operation separate from adding the negation.
53 * For backends with native subtract operations, they will probably
54 * want to recognize add(op0, neg(op1)) or the other way around to
55 * produce a subtract anyway.
56 *
57 * FDIV_TO_MUL_RCP, DDIV_TO_MUL_RCP, and INT_DIV_TO_MUL_RCP:
58 * ---------------------------------------------------------
59 * Breaks an ir_binop_div expression down to op0 * (rcp(op1)).
60 *
61 * Many GPUs don't have a divide instruction (945 and 965 included),
62 * but they do have an RCP instruction to compute an approximate
63 * reciprocal. By breaking the operation down, constant reciprocals
64 * can get constant folded.
65 *
66 * FDIV_TO_MUL_RCP only lowers single-precision floating point division;
67 * DDIV_TO_MUL_RCP only lowers double-precision floating point division.
68 * DIV_TO_MUL_RCP is a convenience macro that sets both flags.
69 * INT_DIV_TO_MUL_RCP handles the integer case, converting to and from floating
70 * point so that RCP is possible.
71 *
72 * EXP_TO_EXP2 and LOG_TO_LOG2:
73 * ----------------------------
74 * Many GPUs don't have a base e log or exponent instruction, but they
75 * do have base 2 versions, so this pass converts exp and log to exp2
76 * and log2 operations.
77 *
78 * POW_TO_EXP2:
79 * -----------
80 * Many older GPUs don't have an x**y instruction. For these GPUs, convert
81 * x**y to 2**(y * log2(x)).
82 *
83 * MOD_TO_FLOOR:
84 * -------------
85 * Breaks an ir_binop_mod expression down to (op0 - op1 * floor(op0 / op1))
86 *
87 * Many GPUs don't have a MOD instruction (945 and 965 included), and
88 * if we have to break it down like this anyway, it gives an
89 * opportunity to do things like constant fold the (1.0 / op1) easily.
90 *
91 * Note: before we used to implement this as op1 * fract(op / op1) but this
92 * implementation had significant precision errors.
93 *
94 * LDEXP_TO_ARITH:
95 * -------------
96 * Converts ir_binop_ldexp to arithmetic and bit operations for float sources.
97 *
98 * DFREXP_DLDEXP_TO_ARITH:
99 * ---------------
100 * Converts ir_binop_ldexp, ir_unop_frexp_sig, and ir_unop_frexp_exp to
101 * arithmetic and bit ops for double arguments.
102 *
103 * CARRY_TO_ARITH:
104 * ---------------
105 * Converts ir_carry into (x + y) < x.
106 *
107 * BORROW_TO_ARITH:
108 * ----------------
109 * Converts ir_borrow into (x < y).
110 *
111 * SAT_TO_CLAMP:
112 * -------------
113 * Converts ir_unop_saturate into min(max(x, 0.0), 1.0)
114 *
115 * DOPS_TO_DFRAC:
116 * --------------
117 * Converts double trunc, ceil, floor, round to fract
118 */
119
120 #include "c99_math.h"
121 #include "program/prog_instruction.h" /* for swizzle */
122 #include "compiler/glsl_types.h"
123 #include "ir.h"
124 #include "ir_builder.h"
125 #include "ir_optimization.h"
126
127 using namespace ir_builder;
128
129 namespace {
130
131 class lower_instructions_visitor : public ir_hierarchical_visitor {
132 public:
133 lower_instructions_visitor(unsigned lower)
134 : progress(false), lower(lower) { }
135
136 ir_visitor_status visit_leave(ir_expression *);
137
138 bool progress;
139
140 private:
141 unsigned lower; /** Bitfield of which operations to lower */
142
143 void sub_to_add_neg(ir_expression *);
144 void div_to_mul_rcp(ir_expression *);
145 void int_div_to_mul_rcp(ir_expression *);
146 void mod_to_floor(ir_expression *);
147 void exp_to_exp2(ir_expression *);
148 void pow_to_exp2(ir_expression *);
149 void log_to_log2(ir_expression *);
150 void ldexp_to_arith(ir_expression *);
151 void dldexp_to_arith(ir_expression *);
152 void dfrexp_sig_to_arith(ir_expression *);
153 void dfrexp_exp_to_arith(ir_expression *);
154 void carry_to_arith(ir_expression *);
155 void borrow_to_arith(ir_expression *);
156 void sat_to_clamp(ir_expression *);
157 void double_dot_to_fma(ir_expression *);
158 void double_lrp(ir_expression *);
159 void dceil_to_dfrac(ir_expression *);
160 void dfloor_to_dfrac(ir_expression *);
161 void dround_even_to_dfrac(ir_expression *);
162 void dtrunc_to_dfrac(ir_expression *);
163 void dsign_to_csel(ir_expression *);
164 void bit_count_to_math(ir_expression *);
165 void extract_to_shifts(ir_expression *);
166 void insert_to_shifts(ir_expression *);
167 void reverse_to_shifts(ir_expression *ir);
168 void find_lsb_to_float_cast(ir_expression *ir);
169 void find_msb_to_float_cast(ir_expression *ir);
170 void imul_high_to_mul(ir_expression *ir);
171 void sqrt_to_abs_sqrt(ir_expression *ir);
172
173 ir_expression *_carry(operand a, operand b);
174 };
175
176 } /* anonymous namespace */
177
178 /**
179 * Determine if a particular type of lowering should occur
180 */
181 #define lowering(x) (this->lower & x)
182
183 bool
184 lower_instructions(exec_list *instructions, unsigned what_to_lower)
185 {
186 lower_instructions_visitor v(what_to_lower);
187
188 visit_list_elements(&v, instructions);
189 return v.progress;
190 }
191
192 void
193 lower_instructions_visitor::sub_to_add_neg(ir_expression *ir)
194 {
195 ir->operation = ir_binop_add;
196 ir->init_num_operands();
197 ir->operands[1] = new(ir) ir_expression(ir_unop_neg, ir->operands[1]->type,
198 ir->operands[1], NULL);
199 this->progress = true;
200 }
201
202 void
203 lower_instructions_visitor::div_to_mul_rcp(ir_expression *ir)
204 {
205 assert(ir->operands[1]->type->is_float() || ir->operands[1]->type->is_double());
206
207 /* New expression for the 1.0 / op1 */
208 ir_rvalue *expr;
209 expr = new(ir) ir_expression(ir_unop_rcp,
210 ir->operands[1]->type,
211 ir->operands[1]);
212
213 /* op0 / op1 -> op0 * (1.0 / op1) */
214 ir->operation = ir_binop_mul;
215 ir->init_num_operands();
216 ir->operands[1] = expr;
217
218 this->progress = true;
219 }
220
221 void
222 lower_instructions_visitor::int_div_to_mul_rcp(ir_expression *ir)
223 {
224 assert(ir->operands[1]->type->is_integer());
225
226 /* Be careful with integer division -- we need to do it as a
227 * float and re-truncate, since rcp(n > 1) of an integer would
228 * just be 0.
229 */
230 ir_rvalue *op0, *op1;
231 const struct glsl_type *vec_type;
232
233 vec_type = glsl_type::get_instance(GLSL_TYPE_FLOAT,
234 ir->operands[1]->type->vector_elements,
235 ir->operands[1]->type->matrix_columns);
236
237 if (ir->operands[1]->type->base_type == GLSL_TYPE_INT)
238 op1 = new(ir) ir_expression(ir_unop_i2f, vec_type, ir->operands[1], NULL);
239 else
240 op1 = new(ir) ir_expression(ir_unop_u2f, vec_type, ir->operands[1], NULL);
241
242 op1 = new(ir) ir_expression(ir_unop_rcp, op1->type, op1, NULL);
243
244 vec_type = glsl_type::get_instance(GLSL_TYPE_FLOAT,
245 ir->operands[0]->type->vector_elements,
246 ir->operands[0]->type->matrix_columns);
247
248 if (ir->operands[0]->type->base_type == GLSL_TYPE_INT)
249 op0 = new(ir) ir_expression(ir_unop_i2f, vec_type, ir->operands[0], NULL);
250 else
251 op0 = new(ir) ir_expression(ir_unop_u2f, vec_type, ir->operands[0], NULL);
252
253 vec_type = glsl_type::get_instance(GLSL_TYPE_FLOAT,
254 ir->type->vector_elements,
255 ir->type->matrix_columns);
256
257 op0 = new(ir) ir_expression(ir_binop_mul, vec_type, op0, op1);
258
259 if (ir->operands[1]->type->base_type == GLSL_TYPE_INT) {
260 ir->operation = ir_unop_f2i;
261 ir->operands[0] = op0;
262 } else {
263 ir->operation = ir_unop_i2u;
264 ir->operands[0] = new(ir) ir_expression(ir_unop_f2i, op0);
265 }
266 ir->init_num_operands();
267 ir->operands[1] = NULL;
268
269 this->progress = true;
270 }
271
272 void
273 lower_instructions_visitor::exp_to_exp2(ir_expression *ir)
274 {
275 ir_constant *log2_e = new(ir) ir_constant(float(M_LOG2E));
276
277 ir->operation = ir_unop_exp2;
278 ir->init_num_operands();
279 ir->operands[0] = new(ir) ir_expression(ir_binop_mul, ir->operands[0]->type,
280 ir->operands[0], log2_e);
281 this->progress = true;
282 }
283
284 void
285 lower_instructions_visitor::pow_to_exp2(ir_expression *ir)
286 {
287 ir_expression *const log2_x =
288 new(ir) ir_expression(ir_unop_log2, ir->operands[0]->type,
289 ir->operands[0]);
290
291 ir->operation = ir_unop_exp2;
292 ir->init_num_operands();
293 ir->operands[0] = new(ir) ir_expression(ir_binop_mul, ir->operands[1]->type,
294 ir->operands[1], log2_x);
295 ir->operands[1] = NULL;
296 this->progress = true;
297 }
298
299 void
300 lower_instructions_visitor::log_to_log2(ir_expression *ir)
301 {
302 ir->operation = ir_binop_mul;
303 ir->init_num_operands();
304 ir->operands[0] = new(ir) ir_expression(ir_unop_log2, ir->operands[0]->type,
305 ir->operands[0], NULL);
306 ir->operands[1] = new(ir) ir_constant(float(1.0 / M_LOG2E));
307 this->progress = true;
308 }
309
310 void
311 lower_instructions_visitor::mod_to_floor(ir_expression *ir)
312 {
313 ir_variable *x = new(ir) ir_variable(ir->operands[0]->type, "mod_x",
314 ir_var_temporary);
315 ir_variable *y = new(ir) ir_variable(ir->operands[1]->type, "mod_y",
316 ir_var_temporary);
317 this->base_ir->insert_before(x);
318 this->base_ir->insert_before(y);
319
320 ir_assignment *const assign_x =
321 new(ir) ir_assignment(new(ir) ir_dereference_variable(x),
322 ir->operands[0], NULL);
323 ir_assignment *const assign_y =
324 new(ir) ir_assignment(new(ir) ir_dereference_variable(y),
325 ir->operands[1], NULL);
326
327 this->base_ir->insert_before(assign_x);
328 this->base_ir->insert_before(assign_y);
329
330 ir_expression *const div_expr =
331 new(ir) ir_expression(ir_binop_div, x->type,
332 new(ir) ir_dereference_variable(x),
333 new(ir) ir_dereference_variable(y));
334
335 /* Don't generate new IR that would need to be lowered in an additional
336 * pass.
337 */
338 if ((lowering(FDIV_TO_MUL_RCP) && ir->type->is_float()) ||
339 (lowering(DDIV_TO_MUL_RCP) && ir->type->is_double()))
340 div_to_mul_rcp(div_expr);
341
342 ir_expression *const floor_expr =
343 new(ir) ir_expression(ir_unop_floor, x->type, div_expr);
344
345 if (lowering(DOPS_TO_DFRAC) && ir->type->is_double())
346 dfloor_to_dfrac(floor_expr);
347
348 ir_expression *const mul_expr =
349 new(ir) ir_expression(ir_binop_mul,
350 new(ir) ir_dereference_variable(y),
351 floor_expr);
352
353 ir->operation = ir_binop_sub;
354 ir->init_num_operands();
355 ir->operands[0] = new(ir) ir_dereference_variable(x);
356 ir->operands[1] = mul_expr;
357 this->progress = true;
358 }
359
360 void
361 lower_instructions_visitor::ldexp_to_arith(ir_expression *ir)
362 {
363 /* Translates
364 * ir_binop_ldexp x exp
365 * into
366 *
367 * extracted_biased_exp = rshift(bitcast_f2i(abs(x)), exp_shift);
368 * resulting_biased_exp = extracted_biased_exp + exp;
369 *
370 * if (resulting_biased_exp < 1 || x == 0.0f) {
371 * return copysign(0.0, x);
372 * }
373 *
374 * return bitcast_u2f((bitcast_f2u(x) & sign_mantissa_mask) |
375 * lshift(i2u(resulting_biased_exp), exp_shift));
376 *
377 * which we can't actually implement as such, since the GLSL IR doesn't
378 * have vectorized if-statements. We actually implement it without branches
379 * using conditional-select:
380 *
381 * extracted_biased_exp = rshift(bitcast_f2i(abs(x)), exp_shift);
382 * resulting_biased_exp = extracted_biased_exp + exp;
383 *
384 * is_not_zero_or_underflow = logic_and(nequal(x, 0.0f),
385 * gequal(resulting_biased_exp, 1);
386 * x = csel(is_not_zero_or_underflow, x, copysign(0.0f, x));
387 * resulting_biased_exp = csel(is_not_zero_or_underflow,
388 * resulting_biased_exp, 0);
389 *
390 * return bitcast_u2f((bitcast_f2u(x) & sign_mantissa_mask) |
391 * lshift(i2u(resulting_biased_exp), exp_shift));
392 */
393
394 const unsigned vec_elem = ir->type->vector_elements;
395
396 /* Types */
397 const glsl_type *ivec = glsl_type::get_instance(GLSL_TYPE_INT, vec_elem, 1);
398 const glsl_type *bvec = glsl_type::get_instance(GLSL_TYPE_BOOL, vec_elem, 1);
399
400 /* Constants */
401 ir_constant *zeroi = ir_constant::zero(ir, ivec);
402
403 ir_constant *sign_mask = new(ir) ir_constant(0x80000000u, vec_elem);
404
405 ir_constant *exp_shift = new(ir) ir_constant(23, vec_elem);
406
407 /* Temporary variables */
408 ir_variable *x = new(ir) ir_variable(ir->type, "x", ir_var_temporary);
409 ir_variable *exp = new(ir) ir_variable(ivec, "exp", ir_var_temporary);
410
411 ir_variable *zero_sign_x = new(ir) ir_variable(ir->type, "zero_sign_x",
412 ir_var_temporary);
413
414 ir_variable *extracted_biased_exp =
415 new(ir) ir_variable(ivec, "extracted_biased_exp", ir_var_temporary);
416 ir_variable *resulting_biased_exp =
417 new(ir) ir_variable(ivec, "resulting_biased_exp", ir_var_temporary);
418
419 ir_variable *is_not_zero_or_underflow =
420 new(ir) ir_variable(bvec, "is_not_zero_or_underflow", ir_var_temporary);
421
422 ir_instruction &i = *base_ir;
423
424 /* Copy <x> and <exp> arguments. */
425 i.insert_before(x);
426 i.insert_before(assign(x, ir->operands[0]));
427 i.insert_before(exp);
428 i.insert_before(assign(exp, ir->operands[1]));
429
430 /* Extract the biased exponent from <x>. */
431 i.insert_before(extracted_biased_exp);
432 i.insert_before(assign(extracted_biased_exp,
433 rshift(bitcast_f2i(abs(x)), exp_shift)));
434
435 i.insert_before(resulting_biased_exp);
436 i.insert_before(assign(resulting_biased_exp,
437 add(extracted_biased_exp, exp)));
438
439 /* Test if result is ±0.0, subnormal, or underflow by checking if the
440 * resulting biased exponent would be less than 0x1. If so, the result is
441 * 0.0 with the sign of x. (Actually, invert the conditions so that
442 * immediate values are the second arguments, which is better for i965)
443 */
444 i.insert_before(zero_sign_x);
445 i.insert_before(assign(zero_sign_x,
446 bitcast_u2f(bit_and(bitcast_f2u(x), sign_mask))));
447
448 i.insert_before(is_not_zero_or_underflow);
449 i.insert_before(assign(is_not_zero_or_underflow,
450 logic_and(nequal(x, new(ir) ir_constant(0.0f, vec_elem)),
451 gequal(resulting_biased_exp,
452 new(ir) ir_constant(0x1, vec_elem)))));
453 i.insert_before(assign(x, csel(is_not_zero_or_underflow,
454 x, zero_sign_x)));
455 i.insert_before(assign(resulting_biased_exp,
456 csel(is_not_zero_or_underflow,
457 resulting_biased_exp, zeroi)));
458
459 /* We could test for overflows by checking if the resulting biased exponent
460 * would be greater than 0xFE. Turns out we don't need to because the GLSL
461 * spec says:
462 *
463 * "If this product is too large to be represented in the
464 * floating-point type, the result is undefined."
465 */
466
467 ir_constant *exp_shift_clone = exp_shift->clone(ir, NULL);
468
469 /* Don't generate new IR that would need to be lowered in an additional
470 * pass.
471 */
472 if (!lowering(INSERT_TO_SHIFTS)) {
473 ir_constant *exp_width = new(ir) ir_constant(8, vec_elem);
474 ir->operation = ir_unop_bitcast_i2f;
475 ir->init_num_operands();
476 ir->operands[0] = bitfield_insert(bitcast_f2i(x), resulting_biased_exp,
477 exp_shift_clone, exp_width);
478 ir->operands[1] = NULL;
479 } else {
480 ir_constant *sign_mantissa_mask = new(ir) ir_constant(0x807fffffu, vec_elem);
481 ir->operation = ir_unop_bitcast_u2f;
482 ir->init_num_operands();
483 ir->operands[0] = bit_or(bit_and(bitcast_f2u(x), sign_mantissa_mask),
484 lshift(i2u(resulting_biased_exp), exp_shift_clone));
485 ir->operands[1] = NULL;
486 }
487
488 this->progress = true;
489 }
490
491 void
492 lower_instructions_visitor::dldexp_to_arith(ir_expression *ir)
493 {
494 /* See ldexp_to_arith for structure. Uses frexp_exp to extract the exponent
495 * from the significand.
496 */
497
498 const unsigned vec_elem = ir->type->vector_elements;
499
500 /* Types */
501 const glsl_type *ivec = glsl_type::get_instance(GLSL_TYPE_INT, vec_elem, 1);
502 const glsl_type *bvec = glsl_type::get_instance(GLSL_TYPE_BOOL, vec_elem, 1);
503
504 /* Constants */
505 ir_constant *zeroi = ir_constant::zero(ir, ivec);
506
507 ir_constant *sign_mask = new(ir) ir_constant(0x80000000u);
508
509 ir_constant *exp_shift = new(ir) ir_constant(20u);
510 ir_constant *exp_width = new(ir) ir_constant(11u);
511 ir_constant *exp_bias = new(ir) ir_constant(1022, vec_elem);
512
513 /* Temporary variables */
514 ir_variable *x = new(ir) ir_variable(ir->type, "x", ir_var_temporary);
515 ir_variable *exp = new(ir) ir_variable(ivec, "exp", ir_var_temporary);
516
517 ir_variable *zero_sign_x = new(ir) ir_variable(ir->type, "zero_sign_x",
518 ir_var_temporary);
519
520 ir_variable *extracted_biased_exp =
521 new(ir) ir_variable(ivec, "extracted_biased_exp", ir_var_temporary);
522 ir_variable *resulting_biased_exp =
523 new(ir) ir_variable(ivec, "resulting_biased_exp", ir_var_temporary);
524
525 ir_variable *is_not_zero_or_underflow =
526 new(ir) ir_variable(bvec, "is_not_zero_or_underflow", ir_var_temporary);
527
528 ir_instruction &i = *base_ir;
529
530 /* Copy <x> and <exp> arguments. */
531 i.insert_before(x);
532 i.insert_before(assign(x, ir->operands[0]));
533 i.insert_before(exp);
534 i.insert_before(assign(exp, ir->operands[1]));
535
536 ir_expression *frexp_exp = expr(ir_unop_frexp_exp, x);
537 if (lowering(DFREXP_DLDEXP_TO_ARITH))
538 dfrexp_exp_to_arith(frexp_exp);
539
540 /* Extract the biased exponent from <x>. */
541 i.insert_before(extracted_biased_exp);
542 i.insert_before(assign(extracted_biased_exp, add(frexp_exp, exp_bias)));
543
544 i.insert_before(resulting_biased_exp);
545 i.insert_before(assign(resulting_biased_exp,
546 add(extracted_biased_exp, exp)));
547
548 /* Test if result is ±0.0, subnormal, or underflow by checking if the
549 * resulting biased exponent would be less than 0x1. If so, the result is
550 * 0.0 with the sign of x. (Actually, invert the conditions so that
551 * immediate values are the second arguments, which is better for i965)
552 * TODO: Implement in a vector fashion.
553 */
554 i.insert_before(zero_sign_x);
555 for (unsigned elem = 0; elem < vec_elem; elem++) {
556 ir_variable *unpacked =
557 new(ir) ir_variable(glsl_type::uvec2_type, "unpacked", ir_var_temporary);
558 i.insert_before(unpacked);
559 i.insert_before(
560 assign(unpacked,
561 expr(ir_unop_unpack_double_2x32, swizzle(x, elem, 1))));
562 i.insert_before(assign(unpacked, bit_and(swizzle_y(unpacked), sign_mask->clone(ir, NULL)),
563 WRITEMASK_Y));
564 i.insert_before(assign(unpacked, ir_constant::zero(ir, glsl_type::uint_type), WRITEMASK_X));
565 i.insert_before(assign(zero_sign_x,
566 expr(ir_unop_pack_double_2x32, unpacked),
567 1 << elem));
568 }
569 i.insert_before(is_not_zero_or_underflow);
570 i.insert_before(assign(is_not_zero_or_underflow,
571 gequal(resulting_biased_exp,
572 new(ir) ir_constant(0x1, vec_elem))));
573 i.insert_before(assign(x, csel(is_not_zero_or_underflow,
574 x, zero_sign_x)));
575 i.insert_before(assign(resulting_biased_exp,
576 csel(is_not_zero_or_underflow,
577 resulting_biased_exp, zeroi)));
578
579 /* We could test for overflows by checking if the resulting biased exponent
580 * would be greater than 0xFE. Turns out we don't need to because the GLSL
581 * spec says:
582 *
583 * "If this product is too large to be represented in the
584 * floating-point type, the result is undefined."
585 */
586
587 ir_rvalue *results[4] = {NULL};
588 for (unsigned elem = 0; elem < vec_elem; elem++) {
589 ir_variable *unpacked =
590 new(ir) ir_variable(glsl_type::uvec2_type, "unpacked", ir_var_temporary);
591 i.insert_before(unpacked);
592 i.insert_before(
593 assign(unpacked,
594 expr(ir_unop_unpack_double_2x32, swizzle(x, elem, 1))));
595
596 ir_expression *bfi = bitfield_insert(
597 swizzle_y(unpacked),
598 i2u(swizzle(resulting_biased_exp, elem, 1)),
599 exp_shift->clone(ir, NULL),
600 exp_width->clone(ir, NULL));
601
602 i.insert_before(assign(unpacked, bfi, WRITEMASK_Y));
603
604 results[elem] = expr(ir_unop_pack_double_2x32, unpacked);
605 }
606
607 ir->operation = ir_quadop_vector;
608 ir->init_num_operands();
609 ir->operands[0] = results[0];
610 ir->operands[1] = results[1];
611 ir->operands[2] = results[2];
612 ir->operands[3] = results[3];
613
614 /* Don't generate new IR that would need to be lowered in an additional
615 * pass.
616 */
617
618 this->progress = true;
619 }
620
621 void
622 lower_instructions_visitor::dfrexp_sig_to_arith(ir_expression *ir)
623 {
624 const unsigned vec_elem = ir->type->vector_elements;
625 const glsl_type *bvec = glsl_type::get_instance(GLSL_TYPE_BOOL, vec_elem, 1);
626
627 /* Double-precision floating-point values are stored as
628 * 1 sign bit;
629 * 11 exponent bits;
630 * 52 mantissa bits.
631 *
632 * We're just extracting the significand here, so we only need to modify
633 * the upper 32-bit uint. Unfortunately we must extract each double
634 * independently as there is no vector version of unpackDouble.
635 */
636
637 ir_instruction &i = *base_ir;
638
639 ir_variable *is_not_zero =
640 new(ir) ir_variable(bvec, "is_not_zero", ir_var_temporary);
641 ir_rvalue *results[4] = {NULL};
642
643 ir_constant *dzero = new(ir) ir_constant(0.0, vec_elem);
644 i.insert_before(is_not_zero);
645 i.insert_before(
646 assign(is_not_zero,
647 nequal(abs(ir->operands[0]->clone(ir, NULL)), dzero)));
648
649 /* TODO: Remake this as more vector-friendly when int64 support is
650 * available.
651 */
652 for (unsigned elem = 0; elem < vec_elem; elem++) {
653 ir_constant *zero = new(ir) ir_constant(0u, 1);
654 ir_constant *sign_mantissa_mask = new(ir) ir_constant(0x800fffffu, 1);
655
656 /* Exponent of double floating-point values in the range [0.5, 1.0). */
657 ir_constant *exponent_value = new(ir) ir_constant(0x3fe00000u, 1);
658
659 ir_variable *bits =
660 new(ir) ir_variable(glsl_type::uint_type, "bits", ir_var_temporary);
661 ir_variable *unpacked =
662 new(ir) ir_variable(glsl_type::uvec2_type, "unpacked", ir_var_temporary);
663
664 ir_rvalue *x = swizzle(ir->operands[0]->clone(ir, NULL), elem, 1);
665
666 i.insert_before(bits);
667 i.insert_before(unpacked);
668 i.insert_before(assign(unpacked, expr(ir_unop_unpack_double_2x32, x)));
669
670 /* Manipulate the high uint to remove the exponent and replace it with
671 * either the default exponent or zero.
672 */
673 i.insert_before(assign(bits, swizzle_y(unpacked)));
674 i.insert_before(assign(bits, bit_and(bits, sign_mantissa_mask)));
675 i.insert_before(assign(bits, bit_or(bits,
676 csel(swizzle(is_not_zero, elem, 1),
677 exponent_value,
678 zero))));
679 i.insert_before(assign(unpacked, bits, WRITEMASK_Y));
680 results[elem] = expr(ir_unop_pack_double_2x32, unpacked);
681 }
682
683 /* Put the dvec back together */
684 ir->operation = ir_quadop_vector;
685 ir->init_num_operands();
686 ir->operands[0] = results[0];
687 ir->operands[1] = results[1];
688 ir->operands[2] = results[2];
689 ir->operands[3] = results[3];
690
691 this->progress = true;
692 }
693
694 void
695 lower_instructions_visitor::dfrexp_exp_to_arith(ir_expression *ir)
696 {
697 const unsigned vec_elem = ir->type->vector_elements;
698 const glsl_type *bvec = glsl_type::get_instance(GLSL_TYPE_BOOL, vec_elem, 1);
699 const glsl_type *uvec = glsl_type::get_instance(GLSL_TYPE_UINT, vec_elem, 1);
700
701 /* Double-precision floating-point values are stored as
702 * 1 sign bit;
703 * 11 exponent bits;
704 * 52 mantissa bits.
705 *
706 * We're just extracting the exponent here, so we only care about the upper
707 * 32-bit uint.
708 */
709
710 ir_instruction &i = *base_ir;
711
712 ir_variable *is_not_zero =
713 new(ir) ir_variable(bvec, "is_not_zero", ir_var_temporary);
714 ir_variable *high_words =
715 new(ir) ir_variable(uvec, "high_words", ir_var_temporary);
716 ir_constant *dzero = new(ir) ir_constant(0.0, vec_elem);
717 ir_constant *izero = new(ir) ir_constant(0, vec_elem);
718
719 ir_rvalue *absval = abs(ir->operands[0]);
720
721 i.insert_before(is_not_zero);
722 i.insert_before(high_words);
723 i.insert_before(assign(is_not_zero, nequal(absval->clone(ir, NULL), dzero)));
724
725 /* Extract all of the upper uints. */
726 for (unsigned elem = 0; elem < vec_elem; elem++) {
727 ir_rvalue *x = swizzle(absval->clone(ir, NULL), elem, 1);
728
729 i.insert_before(assign(high_words,
730 swizzle_y(expr(ir_unop_unpack_double_2x32, x)),
731 1 << elem));
732
733 }
734 ir_constant *exponent_shift = new(ir) ir_constant(20, vec_elem);
735 ir_constant *exponent_bias = new(ir) ir_constant(-1022, vec_elem);
736
737 /* For non-zero inputs, shift the exponent down and apply bias. */
738 ir->operation = ir_triop_csel;
739 ir->init_num_operands();
740 ir->operands[0] = new(ir) ir_dereference_variable(is_not_zero);
741 ir->operands[1] = add(exponent_bias, u2i(rshift(high_words, exponent_shift)));
742 ir->operands[2] = izero;
743
744 this->progress = true;
745 }
746
747 void
748 lower_instructions_visitor::carry_to_arith(ir_expression *ir)
749 {
750 /* Translates
751 * ir_binop_carry x y
752 * into
753 * sum = ir_binop_add x y
754 * bcarry = ir_binop_less sum x
755 * carry = ir_unop_b2i bcarry
756 */
757
758 ir_rvalue *x_clone = ir->operands[0]->clone(ir, NULL);
759 ir->operation = ir_unop_i2u;
760 ir->init_num_operands();
761 ir->operands[0] = b2i(less(add(ir->operands[0], ir->operands[1]), x_clone));
762 ir->operands[1] = NULL;
763
764 this->progress = true;
765 }
766
767 void
768 lower_instructions_visitor::borrow_to_arith(ir_expression *ir)
769 {
770 /* Translates
771 * ir_binop_borrow x y
772 * into
773 * bcarry = ir_binop_less x y
774 * carry = ir_unop_b2i bcarry
775 */
776
777 ir->operation = ir_unop_i2u;
778 ir->init_num_operands();
779 ir->operands[0] = b2i(less(ir->operands[0], ir->operands[1]));
780 ir->operands[1] = NULL;
781
782 this->progress = true;
783 }
784
785 void
786 lower_instructions_visitor::sat_to_clamp(ir_expression *ir)
787 {
788 /* Translates
789 * ir_unop_saturate x
790 * into
791 * ir_binop_min (ir_binop_max(x, 0.0), 1.0)
792 */
793
794 ir->operation = ir_binop_min;
795 ir->init_num_operands();
796 ir->operands[0] = new(ir) ir_expression(ir_binop_max, ir->operands[0]->type,
797 ir->operands[0],
798 new(ir) ir_constant(0.0f));
799 ir->operands[1] = new(ir) ir_constant(1.0f);
800
801 this->progress = true;
802 }
803
804 void
805 lower_instructions_visitor::double_dot_to_fma(ir_expression *ir)
806 {
807 ir_variable *temp = new(ir) ir_variable(ir->operands[0]->type->get_base_type(), "dot_res",
808 ir_var_temporary);
809 this->base_ir->insert_before(temp);
810
811 int nc = ir->operands[0]->type->components();
812 for (int i = nc - 1; i >= 1; i--) {
813 ir_assignment *assig;
814 if (i == (nc - 1)) {
815 assig = assign(temp, mul(swizzle(ir->operands[0]->clone(ir, NULL), i, 1),
816 swizzle(ir->operands[1]->clone(ir, NULL), i, 1)));
817 } else {
818 assig = assign(temp, fma(swizzle(ir->operands[0]->clone(ir, NULL), i, 1),
819 swizzle(ir->operands[1]->clone(ir, NULL), i, 1),
820 temp));
821 }
822 this->base_ir->insert_before(assig);
823 }
824
825 ir->operation = ir_triop_fma;
826 ir->init_num_operands();
827 ir->operands[0] = swizzle(ir->operands[0], 0, 1);
828 ir->operands[1] = swizzle(ir->operands[1], 0, 1);
829 ir->operands[2] = new(ir) ir_dereference_variable(temp);
830
831 this->progress = true;
832
833 }
834
835 void
836 lower_instructions_visitor::double_lrp(ir_expression *ir)
837 {
838 int swizval;
839 ir_rvalue *op0 = ir->operands[0], *op2 = ir->operands[2];
840 ir_constant *one = new(ir) ir_constant(1.0, op2->type->vector_elements);
841
842 switch (op2->type->vector_elements) {
843 case 1:
844 swizval = SWIZZLE_XXXX;
845 break;
846 default:
847 assert(op0->type->vector_elements == op2->type->vector_elements);
848 swizval = SWIZZLE_XYZW;
849 break;
850 }
851
852 ir->operation = ir_triop_fma;
853 ir->init_num_operands();
854 ir->operands[0] = swizzle(op2, swizval, op0->type->vector_elements);
855 ir->operands[2] = mul(sub(one, op2->clone(ir, NULL)), op0);
856
857 this->progress = true;
858 }
859
860 void
861 lower_instructions_visitor::dceil_to_dfrac(ir_expression *ir)
862 {
863 /*
864 * frtemp = frac(x);
865 * temp = sub(x, frtemp);
866 * result = temp + ((frtemp != 0.0) ? 1.0 : 0.0);
867 */
868 ir_instruction &i = *base_ir;
869 ir_constant *zero = new(ir) ir_constant(0.0, ir->operands[0]->type->vector_elements);
870 ir_constant *one = new(ir) ir_constant(1.0, ir->operands[0]->type->vector_elements);
871 ir_variable *frtemp = new(ir) ir_variable(ir->operands[0]->type, "frtemp",
872 ir_var_temporary);
873
874 i.insert_before(frtemp);
875 i.insert_before(assign(frtemp, fract(ir->operands[0])));
876
877 ir->operation = ir_binop_add;
878 ir->init_num_operands();
879 ir->operands[0] = sub(ir->operands[0]->clone(ir, NULL), frtemp);
880 ir->operands[1] = csel(nequal(frtemp, zero), one, zero->clone(ir, NULL));
881
882 this->progress = true;
883 }
884
885 void
886 lower_instructions_visitor::dfloor_to_dfrac(ir_expression *ir)
887 {
888 /*
889 * frtemp = frac(x);
890 * result = sub(x, frtemp);
891 */
892 ir->operation = ir_binop_sub;
893 ir->init_num_operands();
894 ir->operands[1] = fract(ir->operands[0]->clone(ir, NULL));
895
896 this->progress = true;
897 }
898 void
899 lower_instructions_visitor::dround_even_to_dfrac(ir_expression *ir)
900 {
901 /*
902 * insane but works
903 * temp = x + 0.5;
904 * frtemp = frac(temp);
905 * t2 = sub(temp, frtemp);
906 * if (frac(x) == 0.5)
907 * result = frac(t2 * 0.5) == 0 ? t2 : t2 - 1;
908 * else
909 * result = t2;
910
911 */
912 ir_instruction &i = *base_ir;
913 ir_variable *frtemp = new(ir) ir_variable(ir->operands[0]->type, "frtemp",
914 ir_var_temporary);
915 ir_variable *temp = new(ir) ir_variable(ir->operands[0]->type, "temp",
916 ir_var_temporary);
917 ir_variable *t2 = new(ir) ir_variable(ir->operands[0]->type, "t2",
918 ir_var_temporary);
919 ir_constant *p5 = new(ir) ir_constant(0.5, ir->operands[0]->type->vector_elements);
920 ir_constant *one = new(ir) ir_constant(1.0, ir->operands[0]->type->vector_elements);
921 ir_constant *zero = new(ir) ir_constant(0.0, ir->operands[0]->type->vector_elements);
922
923 i.insert_before(temp);
924 i.insert_before(assign(temp, add(ir->operands[0], p5)));
925
926 i.insert_before(frtemp);
927 i.insert_before(assign(frtemp, fract(temp)));
928
929 i.insert_before(t2);
930 i.insert_before(assign(t2, sub(temp, frtemp)));
931
932 ir->operation = ir_triop_csel;
933 ir->init_num_operands();
934 ir->operands[0] = equal(fract(ir->operands[0]->clone(ir, NULL)),
935 p5->clone(ir, NULL));
936 ir->operands[1] = csel(equal(fract(mul(t2, p5->clone(ir, NULL))),
937 zero),
938 t2,
939 sub(t2, one));
940 ir->operands[2] = new(ir) ir_dereference_variable(t2);
941
942 this->progress = true;
943 }
944
945 void
946 lower_instructions_visitor::dtrunc_to_dfrac(ir_expression *ir)
947 {
948 /*
949 * frtemp = frac(x);
950 * temp = sub(x, frtemp);
951 * result = x >= 0 ? temp : temp + (frtemp == 0.0) ? 0 : 1;
952 */
953 ir_rvalue *arg = ir->operands[0];
954 ir_instruction &i = *base_ir;
955
956 ir_constant *zero = new(ir) ir_constant(0.0, arg->type->vector_elements);
957 ir_constant *one = new(ir) ir_constant(1.0, arg->type->vector_elements);
958 ir_variable *frtemp = new(ir) ir_variable(arg->type, "frtemp",
959 ir_var_temporary);
960 ir_variable *temp = new(ir) ir_variable(ir->operands[0]->type, "temp",
961 ir_var_temporary);
962
963 i.insert_before(frtemp);
964 i.insert_before(assign(frtemp, fract(arg)));
965 i.insert_before(temp);
966 i.insert_before(assign(temp, sub(arg->clone(ir, NULL), frtemp)));
967
968 ir->operation = ir_triop_csel;
969 ir->init_num_operands();
970 ir->operands[0] = gequal(arg->clone(ir, NULL), zero);
971 ir->operands[1] = new (ir) ir_dereference_variable(temp);
972 ir->operands[2] = add(temp,
973 csel(equal(frtemp, zero->clone(ir, NULL)),
974 zero->clone(ir, NULL),
975 one));
976
977 this->progress = true;
978 }
979
980 void
981 lower_instructions_visitor::dsign_to_csel(ir_expression *ir)
982 {
983 /*
984 * temp = x > 0.0 ? 1.0 : 0.0;
985 * result = x < 0.0 ? -1.0 : temp;
986 */
987 ir_rvalue *arg = ir->operands[0];
988 ir_constant *zero = new(ir) ir_constant(0.0, arg->type->vector_elements);
989 ir_constant *one = new(ir) ir_constant(1.0, arg->type->vector_elements);
990 ir_constant *neg_one = new(ir) ir_constant(-1.0, arg->type->vector_elements);
991
992 ir->operation = ir_triop_csel;
993 ir->init_num_operands();
994 ir->operands[0] = less(arg->clone(ir, NULL),
995 zero->clone(ir, NULL));
996 ir->operands[1] = neg_one;
997 ir->operands[2] = csel(greater(arg, zero),
998 one,
999 zero->clone(ir, NULL));
1000
1001 this->progress = true;
1002 }
1003
1004 void
1005 lower_instructions_visitor::bit_count_to_math(ir_expression *ir)
1006 {
1007 /* For more details, see:
1008 *
1009 * http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetPaallel
1010 */
1011 const unsigned elements = ir->operands[0]->type->vector_elements;
1012 ir_variable *temp = new(ir) ir_variable(glsl_type::uvec(elements), "temp",
1013 ir_var_temporary);
1014 ir_constant *c55555555 = new(ir) ir_constant(0x55555555u);
1015 ir_constant *c33333333 = new(ir) ir_constant(0x33333333u);
1016 ir_constant *c0F0F0F0F = new(ir) ir_constant(0x0F0F0F0Fu);
1017 ir_constant *c01010101 = new(ir) ir_constant(0x01010101u);
1018 ir_constant *c1 = new(ir) ir_constant(1u);
1019 ir_constant *c2 = new(ir) ir_constant(2u);
1020 ir_constant *c4 = new(ir) ir_constant(4u);
1021 ir_constant *c24 = new(ir) ir_constant(24u);
1022
1023 base_ir->insert_before(temp);
1024
1025 if (ir->operands[0]->type->base_type == GLSL_TYPE_UINT) {
1026 base_ir->insert_before(assign(temp, ir->operands[0]));
1027 } else {
1028 assert(ir->operands[0]->type->base_type == GLSL_TYPE_INT);
1029 base_ir->insert_before(assign(temp, i2u(ir->operands[0])));
1030 }
1031
1032 /* temp = temp - ((temp >> 1) & 0x55555555u); */
1033 base_ir->insert_before(assign(temp, sub(temp, bit_and(rshift(temp, c1),
1034 c55555555))));
1035
1036 /* temp = (temp & 0x33333333u) + ((temp >> 2) & 0x33333333u); */
1037 base_ir->insert_before(assign(temp, add(bit_and(temp, c33333333),
1038 bit_and(rshift(temp, c2),
1039 c33333333->clone(ir, NULL)))));
1040
1041 /* int(((temp + (temp >> 4) & 0xF0F0F0Fu) * 0x1010101u) >> 24); */
1042 ir->operation = ir_unop_u2i;
1043 ir->init_num_operands();
1044 ir->operands[0] = rshift(mul(bit_and(add(temp, rshift(temp, c4)), c0F0F0F0F),
1045 c01010101),
1046 c24);
1047
1048 this->progress = true;
1049 }
1050
1051 void
1052 lower_instructions_visitor::extract_to_shifts(ir_expression *ir)
1053 {
1054 ir_variable *bits =
1055 new(ir) ir_variable(ir->operands[0]->type, "bits", ir_var_temporary);
1056
1057 base_ir->insert_before(bits);
1058 base_ir->insert_before(assign(bits, ir->operands[2]));
1059
1060 if (ir->operands[0]->type->base_type == GLSL_TYPE_UINT) {
1061 ir_constant *c1 =
1062 new(ir) ir_constant(1u, ir->operands[0]->type->vector_elements);
1063 ir_constant *c32 =
1064 new(ir) ir_constant(32u, ir->operands[0]->type->vector_elements);
1065 ir_constant *cFFFFFFFF =
1066 new(ir) ir_constant(0xFFFFFFFFu, ir->operands[0]->type->vector_elements);
1067
1068 /* At least some hardware treats (x << y) as (x << (y%32)). This means
1069 * we'd get a mask of 0 when bits is 32. Special case it.
1070 *
1071 * mask = bits == 32 ? 0xffffffff : (1u << bits) - 1u;
1072 */
1073 ir_expression *mask = csel(equal(bits, c32),
1074 cFFFFFFFF,
1075 sub(lshift(c1, bits), c1->clone(ir, NULL)));
1076
1077 /* Section 8.8 (Integer Functions) of the GLSL 4.50 spec says:
1078 *
1079 * If bits is zero, the result will be zero.
1080 *
1081 * Since (1 << 0) - 1 == 0, we don't need to bother with the conditional
1082 * select as in the signed integer case.
1083 *
1084 * (value >> offset) & mask;
1085 */
1086 ir->operation = ir_binop_bit_and;
1087 ir->init_num_operands();
1088 ir->operands[0] = rshift(ir->operands[0], ir->operands[1]);
1089 ir->operands[1] = mask;
1090 ir->operands[2] = NULL;
1091 } else {
1092 ir_constant *c0 =
1093 new(ir) ir_constant(int(0), ir->operands[0]->type->vector_elements);
1094 ir_constant *c32 =
1095 new(ir) ir_constant(int(32), ir->operands[0]->type->vector_elements);
1096 ir_variable *temp =
1097 new(ir) ir_variable(ir->operands[0]->type, "temp", ir_var_temporary);
1098
1099 /* temp = 32 - bits; */
1100 base_ir->insert_before(temp);
1101 base_ir->insert_before(assign(temp, sub(c32, bits)));
1102
1103 /* expr = value << (temp - offset)) >> temp; */
1104 ir_expression *expr =
1105 rshift(lshift(ir->operands[0], sub(temp, ir->operands[1])), temp);
1106
1107 /* Section 8.8 (Integer Functions) of the GLSL 4.50 spec says:
1108 *
1109 * If bits is zero, the result will be zero.
1110 *
1111 * Due to the (x << (y%32)) behavior mentioned before, the (value <<
1112 * (32-0)) doesn't "erase" all of the data as we would like, so finish
1113 * up with:
1114 *
1115 * (bits == 0) ? 0 : e;
1116 */
1117 ir->operation = ir_triop_csel;
1118 ir->init_num_operands();
1119 ir->operands[0] = equal(c0, bits);
1120 ir->operands[1] = c0->clone(ir, NULL);
1121 ir->operands[2] = expr;
1122 }
1123
1124 this->progress = true;
1125 }
1126
1127 void
1128 lower_instructions_visitor::insert_to_shifts(ir_expression *ir)
1129 {
1130 ir_constant *c1;
1131 ir_constant *c32;
1132 ir_constant *cFFFFFFFF;
1133 ir_variable *offset =
1134 new(ir) ir_variable(ir->operands[0]->type, "offset", ir_var_temporary);
1135 ir_variable *bits =
1136 new(ir) ir_variable(ir->operands[0]->type, "bits", ir_var_temporary);
1137 ir_variable *mask =
1138 new(ir) ir_variable(ir->operands[0]->type, "mask", ir_var_temporary);
1139
1140 if (ir->operands[0]->type->base_type == GLSL_TYPE_INT) {
1141 c1 = new(ir) ir_constant(int(1), ir->operands[0]->type->vector_elements);
1142 c32 = new(ir) ir_constant(int(32), ir->operands[0]->type->vector_elements);
1143 cFFFFFFFF = new(ir) ir_constant(int(0xFFFFFFFF), ir->operands[0]->type->vector_elements);
1144 } else {
1145 assert(ir->operands[0]->type->base_type == GLSL_TYPE_UINT);
1146
1147 c1 = new(ir) ir_constant(1u, ir->operands[0]->type->vector_elements);
1148 c32 = new(ir) ir_constant(32u, ir->operands[0]->type->vector_elements);
1149 cFFFFFFFF = new(ir) ir_constant(0xFFFFFFFFu, ir->operands[0]->type->vector_elements);
1150 }
1151
1152 base_ir->insert_before(offset);
1153 base_ir->insert_before(assign(offset, ir->operands[2]));
1154
1155 base_ir->insert_before(bits);
1156 base_ir->insert_before(assign(bits, ir->operands[3]));
1157
1158 /* At least some hardware treats (x << y) as (x << (y%32)). This means
1159 * we'd get a mask of 0 when bits is 32. Special case it.
1160 *
1161 * mask = (bits == 32 ? 0xffffffff : (1u << bits) - 1u) << offset;
1162 *
1163 * Section 8.8 (Integer Functions) of the GLSL 4.50 spec says:
1164 *
1165 * The result will be undefined if offset or bits is negative, or if the
1166 * sum of offset and bits is greater than the number of bits used to
1167 * store the operand.
1168 *
1169 * Since it's undefined, there are a couple other ways this could be
1170 * implemented. The other way that was considered was to put the csel
1171 * around the whole thing:
1172 *
1173 * final_result = bits == 32 ? insert : ... ;
1174 */
1175 base_ir->insert_before(mask);
1176
1177 base_ir->insert_before(assign(mask, csel(equal(bits, c32),
1178 cFFFFFFFF,
1179 lshift(sub(lshift(c1, bits),
1180 c1->clone(ir, NULL)),
1181 offset))));
1182
1183 /* (base & ~mask) | ((insert << offset) & mask) */
1184 ir->operation = ir_binop_bit_or;
1185 ir->init_num_operands();
1186 ir->operands[0] = bit_and(ir->operands[0], bit_not(mask));
1187 ir->operands[1] = bit_and(lshift(ir->operands[1], offset), mask);
1188 ir->operands[2] = NULL;
1189 ir->operands[3] = NULL;
1190
1191 this->progress = true;
1192 }
1193
1194 void
1195 lower_instructions_visitor::reverse_to_shifts(ir_expression *ir)
1196 {
1197 /* For more details, see:
1198 *
1199 * http://graphics.stanford.edu/~seander/bithacks.html#ReverseParallel
1200 */
1201 ir_constant *c1 =
1202 new(ir) ir_constant(1u, ir->operands[0]->type->vector_elements);
1203 ir_constant *c2 =
1204 new(ir) ir_constant(2u, ir->operands[0]->type->vector_elements);
1205 ir_constant *c4 =
1206 new(ir) ir_constant(4u, ir->operands[0]->type->vector_elements);
1207 ir_constant *c8 =
1208 new(ir) ir_constant(8u, ir->operands[0]->type->vector_elements);
1209 ir_constant *c16 =
1210 new(ir) ir_constant(16u, ir->operands[0]->type->vector_elements);
1211 ir_constant *c33333333 =
1212 new(ir) ir_constant(0x33333333u, ir->operands[0]->type->vector_elements);
1213 ir_constant *c55555555 =
1214 new(ir) ir_constant(0x55555555u, ir->operands[0]->type->vector_elements);
1215 ir_constant *c0F0F0F0F =
1216 new(ir) ir_constant(0x0F0F0F0Fu, ir->operands[0]->type->vector_elements);
1217 ir_constant *c00FF00FF =
1218 new(ir) ir_constant(0x00FF00FFu, ir->operands[0]->type->vector_elements);
1219 ir_variable *temp =
1220 new(ir) ir_variable(glsl_type::uvec(ir->operands[0]->type->vector_elements),
1221 "temp", ir_var_temporary);
1222 ir_instruction &i = *base_ir;
1223
1224 i.insert_before(temp);
1225
1226 if (ir->operands[0]->type->base_type == GLSL_TYPE_UINT) {
1227 i.insert_before(assign(temp, ir->operands[0]));
1228 } else {
1229 assert(ir->operands[0]->type->base_type == GLSL_TYPE_INT);
1230 i.insert_before(assign(temp, i2u(ir->operands[0])));
1231 }
1232
1233 /* Swap odd and even bits.
1234 *
1235 * temp = ((temp >> 1) & 0x55555555u) | ((temp & 0x55555555u) << 1);
1236 */
1237 i.insert_before(assign(temp, bit_or(bit_and(rshift(temp, c1), c55555555),
1238 lshift(bit_and(temp, c55555555->clone(ir, NULL)),
1239 c1->clone(ir, NULL)))));
1240 /* Swap consecutive pairs.
1241 *
1242 * temp = ((temp >> 2) & 0x33333333u) | ((temp & 0x33333333u) << 2);
1243 */
1244 i.insert_before(assign(temp, bit_or(bit_and(rshift(temp, c2), c33333333),
1245 lshift(bit_and(temp, c33333333->clone(ir, NULL)),
1246 c2->clone(ir, NULL)))));
1247
1248 /* Swap nibbles.
1249 *
1250 * temp = ((temp >> 4) & 0x0F0F0F0Fu) | ((temp & 0x0F0F0F0Fu) << 4);
1251 */
1252 i.insert_before(assign(temp, bit_or(bit_and(rshift(temp, c4), c0F0F0F0F),
1253 lshift(bit_and(temp, c0F0F0F0F->clone(ir, NULL)),
1254 c4->clone(ir, NULL)))));
1255
1256 /* The last step is, basically, bswap. Swap the bytes, then swap the
1257 * words. When this code is run through GCC on x86, it does generate a
1258 * bswap instruction.
1259 *
1260 * temp = ((temp >> 8) & 0x00FF00FFu) | ((temp & 0x00FF00FFu) << 8);
1261 * temp = ( temp >> 16 ) | ( temp << 16);
1262 */
1263 i.insert_before(assign(temp, bit_or(bit_and(rshift(temp, c8), c00FF00FF),
1264 lshift(bit_and(temp, c00FF00FF->clone(ir, NULL)),
1265 c8->clone(ir, NULL)))));
1266
1267 if (ir->operands[0]->type->base_type == GLSL_TYPE_UINT) {
1268 ir->operation = ir_binop_bit_or;
1269 ir->init_num_operands();
1270 ir->operands[0] = rshift(temp, c16);
1271 ir->operands[1] = lshift(temp, c16->clone(ir, NULL));
1272 } else {
1273 ir->operation = ir_unop_u2i;
1274 ir->init_num_operands();
1275 ir->operands[0] = bit_or(rshift(temp, c16),
1276 lshift(temp, c16->clone(ir, NULL)));
1277 }
1278
1279 this->progress = true;
1280 }
1281
1282 void
1283 lower_instructions_visitor::find_lsb_to_float_cast(ir_expression *ir)
1284 {
1285 /* For more details, see:
1286 *
1287 * http://graphics.stanford.edu/~seander/bithacks.html#ZerosOnRightFloatCast
1288 */
1289 const unsigned elements = ir->operands[0]->type->vector_elements;
1290 ir_constant *c0 = new(ir) ir_constant(unsigned(0), elements);
1291 ir_constant *cminus1 = new(ir) ir_constant(int(-1), elements);
1292 ir_constant *c23 = new(ir) ir_constant(int(23), elements);
1293 ir_constant *c7F = new(ir) ir_constant(int(0x7F), elements);
1294 ir_variable *temp =
1295 new(ir) ir_variable(glsl_type::ivec(elements), "temp", ir_var_temporary);
1296 ir_variable *lsb_only =
1297 new(ir) ir_variable(glsl_type::uvec(elements), "lsb_only", ir_var_temporary);
1298 ir_variable *as_float =
1299 new(ir) ir_variable(glsl_type::vec(elements), "as_float", ir_var_temporary);
1300 ir_variable *lsb =
1301 new(ir) ir_variable(glsl_type::ivec(elements), "lsb", ir_var_temporary);
1302
1303 ir_instruction &i = *base_ir;
1304
1305 i.insert_before(temp);
1306
1307 if (ir->operands[0]->type->base_type == GLSL_TYPE_INT) {
1308 i.insert_before(assign(temp, ir->operands[0]));
1309 } else {
1310 assert(ir->operands[0]->type->base_type == GLSL_TYPE_UINT);
1311 i.insert_before(assign(temp, u2i(ir->operands[0])));
1312 }
1313
1314 /* The int-to-float conversion is lossless because (value & -value) is
1315 * either a power of two or zero. We don't use the result in the zero
1316 * case. The uint() cast is necessary so that 0x80000000 does not
1317 * generate a negative value.
1318 *
1319 * uint lsb_only = uint(value & -value);
1320 * float as_float = float(lsb_only);
1321 */
1322 i.insert_before(lsb_only);
1323 i.insert_before(assign(lsb_only, i2u(bit_and(temp, neg(temp)))));
1324
1325 i.insert_before(as_float);
1326 i.insert_before(assign(as_float, u2f(lsb_only)));
1327
1328 /* This is basically an open-coded frexp. Implementations that have a
1329 * native frexp instruction would be better served by that. This is
1330 * optimized versus a full-featured open-coded implementation in two ways:
1331 *
1332 * - We don't care about a correct result from subnormal numbers (including
1333 * 0.0), so the raw exponent can always be safely unbiased.
1334 *
1335 * - The value cannot be negative, so it does not need to be masked off to
1336 * extract the exponent.
1337 *
1338 * int lsb = (floatBitsToInt(as_float) >> 23) - 0x7f;
1339 */
1340 i.insert_before(lsb);
1341 i.insert_before(assign(lsb, sub(rshift(bitcast_f2i(as_float), c23), c7F)));
1342
1343 /* Use lsb_only in the comparison instead of temp so that the & (far above)
1344 * can possibly generate the result without an explicit comparison.
1345 *
1346 * (lsb_only == 0) ? -1 : lsb;
1347 *
1348 * Since our input values are all integers, the unbiased exponent must not
1349 * be negative. It will only be negative (-0x7f, in fact) if lsb_only is
1350 * 0. Instead of using (lsb_only == 0), we could use (lsb >= 0). Which is
1351 * better is likely GPU dependent. Either way, the difference should be
1352 * small.
1353 */
1354 ir->operation = ir_triop_csel;
1355 ir->init_num_operands();
1356 ir->operands[0] = equal(lsb_only, c0);
1357 ir->operands[1] = cminus1;
1358 ir->operands[2] = new(ir) ir_dereference_variable(lsb);
1359
1360 this->progress = true;
1361 }
1362
1363 void
1364 lower_instructions_visitor::find_msb_to_float_cast(ir_expression *ir)
1365 {
1366 /* For more details, see:
1367 *
1368 * http://graphics.stanford.edu/~seander/bithacks.html#ZerosOnRightFloatCast
1369 */
1370 const unsigned elements = ir->operands[0]->type->vector_elements;
1371 ir_constant *c0 = new(ir) ir_constant(int(0), elements);
1372 ir_constant *cminus1 = new(ir) ir_constant(int(-1), elements);
1373 ir_constant *c23 = new(ir) ir_constant(int(23), elements);
1374 ir_constant *c7F = new(ir) ir_constant(int(0x7F), elements);
1375 ir_constant *c000000FF = new(ir) ir_constant(0x000000FFu, elements);
1376 ir_constant *cFFFFFF00 = new(ir) ir_constant(0xFFFFFF00u, elements);
1377 ir_variable *temp =
1378 new(ir) ir_variable(glsl_type::uvec(elements), "temp", ir_var_temporary);
1379 ir_variable *as_float =
1380 new(ir) ir_variable(glsl_type::vec(elements), "as_float", ir_var_temporary);
1381 ir_variable *msb =
1382 new(ir) ir_variable(glsl_type::ivec(elements), "msb", ir_var_temporary);
1383
1384 ir_instruction &i = *base_ir;
1385
1386 i.insert_before(temp);
1387
1388 if (ir->operands[0]->type->base_type == GLSL_TYPE_UINT) {
1389 i.insert_before(assign(temp, ir->operands[0]));
1390 } else {
1391 assert(ir->operands[0]->type->base_type == GLSL_TYPE_INT);
1392
1393 /* findMSB(uint(abs(some_int))) almost always does the right thing.
1394 * There are two problem values:
1395 *
1396 * * 0x80000000. Since abs(0x80000000) == 0x80000000, findMSB returns
1397 * 31. However, findMSB(int(0x80000000)) == 30.
1398 *
1399 * * 0xffffffff. Since abs(0xffffffff) == 1, findMSB returns
1400 * 31. Section 8.8 (Integer Functions) of the GLSL 4.50 spec says:
1401 *
1402 * For a value of zero or negative one, -1 will be returned.
1403 *
1404 * For all negative number cases, including 0x80000000 and 0xffffffff,
1405 * the correct value is obtained from findMSB if instead of negating the
1406 * (already negative) value the logical-not is used. A conditonal
1407 * logical-not can be achieved in two instructions.
1408 */
1409 ir_variable *as_int =
1410 new(ir) ir_variable(glsl_type::ivec(elements), "as_int", ir_var_temporary);
1411 ir_constant *c31 = new(ir) ir_constant(int(31), elements);
1412
1413 i.insert_before(as_int);
1414 i.insert_before(assign(as_int, ir->operands[0]));
1415 i.insert_before(assign(temp, i2u(expr(ir_binop_bit_xor,
1416 as_int,
1417 rshift(as_int, c31)))));
1418 }
1419
1420 /* The int-to-float conversion is lossless because bits are conditionally
1421 * masked off the bottom of temp to ensure the value has at most 24 bits of
1422 * data or is zero. We don't use the result in the zero case. The uint()
1423 * cast is necessary so that 0x80000000 does not generate a negative value.
1424 *
1425 * float as_float = float(temp > 255 ? temp & ~255 : temp);
1426 */
1427 i.insert_before(as_float);
1428 i.insert_before(assign(as_float, u2f(csel(greater(temp, c000000FF),
1429 bit_and(temp, cFFFFFF00),
1430 temp))));
1431
1432 /* This is basically an open-coded frexp. Implementations that have a
1433 * native frexp instruction would be better served by that. This is
1434 * optimized versus a full-featured open-coded implementation in two ways:
1435 *
1436 * - We don't care about a correct result from subnormal numbers (including
1437 * 0.0), so the raw exponent can always be safely unbiased.
1438 *
1439 * - The value cannot be negative, so it does not need to be masked off to
1440 * extract the exponent.
1441 *
1442 * int msb = (floatBitsToInt(as_float) >> 23) - 0x7f;
1443 */
1444 i.insert_before(msb);
1445 i.insert_before(assign(msb, sub(rshift(bitcast_f2i(as_float), c23), c7F)));
1446
1447 /* Use msb in the comparison instead of temp so that the subtract can
1448 * possibly generate the result without an explicit comparison.
1449 *
1450 * (msb < 0) ? -1 : msb;
1451 *
1452 * Since our input values are all integers, the unbiased exponent must not
1453 * be negative. It will only be negative (-0x7f, in fact) if temp is 0.
1454 */
1455 ir->operation = ir_triop_csel;
1456 ir->init_num_operands();
1457 ir->operands[0] = less(msb, c0);
1458 ir->operands[1] = cminus1;
1459 ir->operands[2] = new(ir) ir_dereference_variable(msb);
1460
1461 this->progress = true;
1462 }
1463
1464 ir_expression *
1465 lower_instructions_visitor::_carry(operand a, operand b)
1466 {
1467 if (lowering(CARRY_TO_ARITH))
1468 return i2u(b2i(less(add(a, b),
1469 a.val->clone(ralloc_parent(a.val), NULL))));
1470 else
1471 return carry(a, b);
1472 }
1473
1474 void
1475 lower_instructions_visitor::imul_high_to_mul(ir_expression *ir)
1476 {
1477 /* ABCD
1478 * * EFGH
1479 * ======
1480 * (GH * CD) + (GH * AB) << 16 + (EF * CD) << 16 + (EF * AB) << 32
1481 *
1482 * In GLSL, (a * b) becomes
1483 *
1484 * uint m1 = (a & 0x0000ffffu) * (b & 0x0000ffffu);
1485 * uint m2 = (a & 0x0000ffffu) * (b >> 16);
1486 * uint m3 = (a >> 16) * (b & 0x0000ffffu);
1487 * uint m4 = (a >> 16) * (b >> 16);
1488 *
1489 * uint c1;
1490 * uint c2;
1491 * uint lo_result;
1492 * uint hi_result;
1493 *
1494 * lo_result = uaddCarry(m1, m2 << 16, c1);
1495 * hi_result = m4 + c1;
1496 * lo_result = uaddCarry(lo_result, m3 << 16, c2);
1497 * hi_result = hi_result + c2;
1498 * hi_result = hi_result + (m2 >> 16) + (m3 >> 16);
1499 */
1500 const unsigned elements = ir->operands[0]->type->vector_elements;
1501 ir_variable *src1 =
1502 new(ir) ir_variable(glsl_type::uvec(elements), "src1", ir_var_temporary);
1503 ir_variable *src1h =
1504 new(ir) ir_variable(glsl_type::uvec(elements), "src1h", ir_var_temporary);
1505 ir_variable *src1l =
1506 new(ir) ir_variable(glsl_type::uvec(elements), "src1l", ir_var_temporary);
1507 ir_variable *src2 =
1508 new(ir) ir_variable(glsl_type::uvec(elements), "src2", ir_var_temporary);
1509 ir_variable *src2h =
1510 new(ir) ir_variable(glsl_type::uvec(elements), "src2h", ir_var_temporary);
1511 ir_variable *src2l =
1512 new(ir) ir_variable(glsl_type::uvec(elements), "src2l", ir_var_temporary);
1513 ir_variable *t1 =
1514 new(ir) ir_variable(glsl_type::uvec(elements), "t1", ir_var_temporary);
1515 ir_variable *t2 =
1516 new(ir) ir_variable(glsl_type::uvec(elements), "t2", ir_var_temporary);
1517 ir_variable *lo =
1518 new(ir) ir_variable(glsl_type::uvec(elements), "lo", ir_var_temporary);
1519 ir_variable *hi =
1520 new(ir) ir_variable(glsl_type::uvec(elements), "hi", ir_var_temporary);
1521 ir_variable *different_signs = NULL;
1522 ir_constant *c0000FFFF = new(ir) ir_constant(0x0000FFFFu, elements);
1523 ir_constant *c16 = new(ir) ir_constant(16u, elements);
1524
1525 ir_instruction &i = *base_ir;
1526
1527 i.insert_before(src1);
1528 i.insert_before(src2);
1529 i.insert_before(src1h);
1530 i.insert_before(src2h);
1531 i.insert_before(src1l);
1532 i.insert_before(src2l);
1533
1534 if (ir->operands[0]->type->base_type == GLSL_TYPE_UINT) {
1535 i.insert_before(assign(src1, ir->operands[0]));
1536 i.insert_before(assign(src2, ir->operands[1]));
1537 } else {
1538 assert(ir->operands[0]->type->base_type == GLSL_TYPE_INT);
1539
1540 ir_variable *itmp1 =
1541 new(ir) ir_variable(glsl_type::ivec(elements), "itmp1", ir_var_temporary);
1542 ir_variable *itmp2 =
1543 new(ir) ir_variable(glsl_type::ivec(elements), "itmp2", ir_var_temporary);
1544 ir_constant *c0 = new(ir) ir_constant(int(0), elements);
1545
1546 i.insert_before(itmp1);
1547 i.insert_before(itmp2);
1548 i.insert_before(assign(itmp1, ir->operands[0]));
1549 i.insert_before(assign(itmp2, ir->operands[1]));
1550
1551 different_signs =
1552 new(ir) ir_variable(glsl_type::bvec(elements), "different_signs",
1553 ir_var_temporary);
1554
1555 i.insert_before(different_signs);
1556 i.insert_before(assign(different_signs, expr(ir_binop_logic_xor,
1557 less(itmp1, c0),
1558 less(itmp2, c0->clone(ir, NULL)))));
1559
1560 i.insert_before(assign(src1, i2u(abs(itmp1))));
1561 i.insert_before(assign(src2, i2u(abs(itmp2))));
1562 }
1563
1564 i.insert_before(assign(src1l, bit_and(src1, c0000FFFF)));
1565 i.insert_before(assign(src2l, bit_and(src2, c0000FFFF->clone(ir, NULL))));
1566 i.insert_before(assign(src1h, rshift(src1, c16)));
1567 i.insert_before(assign(src2h, rshift(src2, c16->clone(ir, NULL))));
1568
1569 i.insert_before(lo);
1570 i.insert_before(hi);
1571 i.insert_before(t1);
1572 i.insert_before(t2);
1573
1574 i.insert_before(assign(lo, mul(src1l, src2l)));
1575 i.insert_before(assign(t1, mul(src1l, src2h)));
1576 i.insert_before(assign(t2, mul(src1h, src2l)));
1577 i.insert_before(assign(hi, mul(src1h, src2h)));
1578
1579 i.insert_before(assign(hi, add(hi, _carry(lo, lshift(t1, c16->clone(ir, NULL))))));
1580 i.insert_before(assign(lo, add(lo, lshift(t1, c16->clone(ir, NULL)))));
1581
1582 i.insert_before(assign(hi, add(hi, _carry(lo, lshift(t2, c16->clone(ir, NULL))))));
1583 i.insert_before(assign(lo, add(lo, lshift(t2, c16->clone(ir, NULL)))));
1584
1585 if (different_signs == NULL) {
1586 assert(ir->operands[0]->type->base_type == GLSL_TYPE_UINT);
1587
1588 ir->operation = ir_binop_add;
1589 ir->init_num_operands();
1590 ir->operands[0] = add(hi, rshift(t1, c16->clone(ir, NULL)));
1591 ir->operands[1] = rshift(t2, c16->clone(ir, NULL));
1592 } else {
1593 assert(ir->operands[0]->type->base_type == GLSL_TYPE_INT);
1594
1595 i.insert_before(assign(hi, add(add(hi, rshift(t1, c16->clone(ir, NULL))),
1596 rshift(t2, c16->clone(ir, NULL)))));
1597
1598 /* For channels where different_signs is set we have to perform a 64-bit
1599 * negation. This is *not* the same as just negating the high 32-bits.
1600 * Consider -3 * 2. The high 32-bits is 0, but the desired result is
1601 * -1, not -0! Recall -x == ~x + 1.
1602 */
1603 ir_variable *neg_hi =
1604 new(ir) ir_variable(glsl_type::ivec(elements), "neg_hi", ir_var_temporary);
1605 ir_constant *c1 = new(ir) ir_constant(1u, elements);
1606
1607 i.insert_before(neg_hi);
1608 i.insert_before(assign(neg_hi, add(bit_not(u2i(hi)),
1609 u2i(_carry(bit_not(lo), c1)))));
1610
1611 ir->operation = ir_triop_csel;
1612 ir->init_num_operands();
1613 ir->operands[0] = new(ir) ir_dereference_variable(different_signs);
1614 ir->operands[1] = new(ir) ir_dereference_variable(neg_hi);
1615 ir->operands[2] = u2i(hi);
1616 }
1617 }
1618
1619 void
1620 lower_instructions_visitor::sqrt_to_abs_sqrt(ir_expression *ir)
1621 {
1622 ir->operands[0] = new(ir) ir_expression(ir_unop_abs, ir->operands[0]);
1623 this->progress = true;
1624 }
1625
1626 ir_visitor_status
1627 lower_instructions_visitor::visit_leave(ir_expression *ir)
1628 {
1629 switch (ir->operation) {
1630 case ir_binop_dot:
1631 if (ir->operands[0]->type->is_double())
1632 double_dot_to_fma(ir);
1633 break;
1634 case ir_triop_lrp:
1635 if (ir->operands[0]->type->is_double())
1636 double_lrp(ir);
1637 break;
1638 case ir_binop_sub:
1639 if (lowering(SUB_TO_ADD_NEG))
1640 sub_to_add_neg(ir);
1641 break;
1642
1643 case ir_binop_div:
1644 if (ir->operands[1]->type->is_integer() && lowering(INT_DIV_TO_MUL_RCP))
1645 int_div_to_mul_rcp(ir);
1646 else if ((ir->operands[1]->type->is_float() && lowering(FDIV_TO_MUL_RCP)) ||
1647 (ir->operands[1]->type->is_double() && lowering(DDIV_TO_MUL_RCP)))
1648 div_to_mul_rcp(ir);
1649 break;
1650
1651 case ir_unop_exp:
1652 if (lowering(EXP_TO_EXP2))
1653 exp_to_exp2(ir);
1654 break;
1655
1656 case ir_unop_log:
1657 if (lowering(LOG_TO_LOG2))
1658 log_to_log2(ir);
1659 break;
1660
1661 case ir_binop_mod:
1662 if (lowering(MOD_TO_FLOOR) && (ir->type->is_float() || ir->type->is_double()))
1663 mod_to_floor(ir);
1664 break;
1665
1666 case ir_binop_pow:
1667 if (lowering(POW_TO_EXP2))
1668 pow_to_exp2(ir);
1669 break;
1670
1671 case ir_binop_ldexp:
1672 if (lowering(LDEXP_TO_ARITH) && ir->type->is_float())
1673 ldexp_to_arith(ir);
1674 if (lowering(DFREXP_DLDEXP_TO_ARITH) && ir->type->is_double())
1675 dldexp_to_arith(ir);
1676 break;
1677
1678 case ir_unop_frexp_exp:
1679 if (lowering(DFREXP_DLDEXP_TO_ARITH) && ir->operands[0]->type->is_double())
1680 dfrexp_exp_to_arith(ir);
1681 break;
1682
1683 case ir_unop_frexp_sig:
1684 if (lowering(DFREXP_DLDEXP_TO_ARITH) && ir->operands[0]->type->is_double())
1685 dfrexp_sig_to_arith(ir);
1686 break;
1687
1688 case ir_binop_carry:
1689 if (lowering(CARRY_TO_ARITH))
1690 carry_to_arith(ir);
1691 break;
1692
1693 case ir_binop_borrow:
1694 if (lowering(BORROW_TO_ARITH))
1695 borrow_to_arith(ir);
1696 break;
1697
1698 case ir_unop_saturate:
1699 if (lowering(SAT_TO_CLAMP))
1700 sat_to_clamp(ir);
1701 break;
1702
1703 case ir_unop_trunc:
1704 if (lowering(DOPS_TO_DFRAC) && ir->type->is_double())
1705 dtrunc_to_dfrac(ir);
1706 break;
1707
1708 case ir_unop_ceil:
1709 if (lowering(DOPS_TO_DFRAC) && ir->type->is_double())
1710 dceil_to_dfrac(ir);
1711 break;
1712
1713 case ir_unop_floor:
1714 if (lowering(DOPS_TO_DFRAC) && ir->type->is_double())
1715 dfloor_to_dfrac(ir);
1716 break;
1717
1718 case ir_unop_round_even:
1719 if (lowering(DOPS_TO_DFRAC) && ir->type->is_double())
1720 dround_even_to_dfrac(ir);
1721 break;
1722
1723 case ir_unop_sign:
1724 if (lowering(DOPS_TO_DFRAC) && ir->type->is_double())
1725 dsign_to_csel(ir);
1726 break;
1727
1728 case ir_unop_bit_count:
1729 if (lowering(BIT_COUNT_TO_MATH))
1730 bit_count_to_math(ir);
1731 break;
1732
1733 case ir_triop_bitfield_extract:
1734 if (lowering(EXTRACT_TO_SHIFTS))
1735 extract_to_shifts(ir);
1736 break;
1737
1738 case ir_quadop_bitfield_insert:
1739 if (lowering(INSERT_TO_SHIFTS))
1740 insert_to_shifts(ir);
1741 break;
1742
1743 case ir_unop_bitfield_reverse:
1744 if (lowering(REVERSE_TO_SHIFTS))
1745 reverse_to_shifts(ir);
1746 break;
1747
1748 case ir_unop_find_lsb:
1749 if (lowering(FIND_LSB_TO_FLOAT_CAST))
1750 find_lsb_to_float_cast(ir);
1751 break;
1752
1753 case ir_unop_find_msb:
1754 if (lowering(FIND_MSB_TO_FLOAT_CAST))
1755 find_msb_to_float_cast(ir);
1756 break;
1757
1758 case ir_binop_imul_high:
1759 if (lowering(IMUL_HIGH_TO_MUL))
1760 imul_high_to_mul(ir);
1761 break;
1762
1763 case ir_unop_rsq:
1764 case ir_unop_sqrt:
1765 if (lowering(SQRT_TO_ABS_SQRT))
1766 sqrt_to_abs_sqrt(ir);
1767 break;
1768
1769 default:
1770 return visit_continue;
1771 }
1772
1773 return visit_continue;
1774 }