src/compiler/glsl/glsl_to_nir.cpp

   1 /*
   2  * Copyright © 2014 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Connor Abbott (cwabbott0@gmail.com)
  25  *
  26  */
  27
  28 #include "glsl_to_nir.h"
  29 #include "ir_visitor.h"
  30 #include "ir_hierarchical_visitor.h"
  31 #include "ir.h"
  32 #include "compiler/nir/nir_control_flow.h"
  33 #include "compiler/nir/nir_builder.h"
  34 #include "main/imports.h"
  35
  36 /*
  37  * pass to lower GLSL IR to NIR
  38  *
  39  * This will lower variable dereferences to loads/stores of corresponding
  40  * variables in NIR - the variables will be converted to registers in a later
  41  * pass.
  42  */
  43
  44 namespace {
  45
  46 class nir_visitor : public ir_visitor
  47 {
  48 public:
  49    nir_visitor(nir_shader *shader);
  50    ~nir_visitor();
  51
  52    virtual void visit(ir_variable *);
  53    virtual void visit(ir_function *);
  54    virtual void visit(ir_function_signature *);
  55    virtual void visit(ir_loop *);
  56    virtual void visit(ir_if *);
  57    virtual void visit(ir_discard *);
  58    virtual void visit(ir_loop_jump *);
  59    virtual void visit(ir_return *);
  60    virtual void visit(ir_call *);
  61    virtual void visit(ir_assignment *);
  62    virtual void visit(ir_emit_vertex *);
  63    virtual void visit(ir_end_primitive *);
  64    virtual void visit(ir_expression *);
  65    virtual void visit(ir_swizzle *);
  66    virtual void visit(ir_texture *);
  67    virtual void visit(ir_constant *);
  68    virtual void visit(ir_dereference_variable *);
  69    virtual void visit(ir_dereference_record *);
  70    virtual void visit(ir_dereference_array *);
  71    virtual void visit(ir_barrier *);
  72
  73    void create_function(ir_function_signature *ir);
  74
  75 private:
  76    void add_instr(nir_instr *instr, unsigned num_components, unsigned bit_size);
  77    nir_ssa_def *evaluate_rvalue(ir_rvalue *ir);
  78
  79    nir_alu_instr *emit(nir_op op, unsigned dest_size, nir_ssa_def **srcs);
  80    nir_alu_instr *emit(nir_op op, unsigned dest_size, nir_ssa_def *src1);
  81    nir_alu_instr *emit(nir_op op, unsigned dest_size, nir_ssa_def *src1,
  82                        nir_ssa_def *src2);
  83    nir_alu_instr *emit(nir_op op, unsigned dest_size, nir_ssa_def *src1,
  84                        nir_ssa_def *src2, nir_ssa_def *src3);
  85
  86    bool supports_ints;
  87
  88    nir_shader *shader;
  89    nir_function_impl *impl;
  90    nir_builder b;
  91    nir_ssa_def *result; /* result of the expression tree last visited */
  92
  93    nir_deref_var *evaluate_deref(nir_instr *mem_ctx, ir_instruction *ir);
  94
  95    /* the head of the dereference chain we're creating */
  96    nir_deref_var *deref_head;
  97    /* the tail of the dereference chain we're creating */
  98    nir_deref *deref_tail;
  99
 100    nir_variable *var; /* variable created by ir_variable visitor */
 101
 102    /* whether the IR we're operating on is per-function or global */
 103    bool is_global;
 104
 105    /* map of ir_variable -> nir_variable */
 106    struct hash_table *var_table;
 107
 108    /* map of ir_function_signature -> nir_function_overload */
 109    struct hash_table *overload_table;
 110 };
 111
 112 /*
 113  * This visitor runs before the main visitor, calling create_function() for
 114  * each function so that the main visitor can resolve forward references in
 115  * calls.
 116  */
 117
 118 class nir_function_visitor : public ir_hierarchical_visitor
 119 {
 120 public:
 121    nir_function_visitor(nir_visitor *v) : visitor(v)
 122    {
 123    }
 124    virtual ir_visitor_status visit_enter(ir_function *);
 125
 126 private:
 127    nir_visitor *visitor;
 128 };
 129
 130 } /* end of anonymous namespace */
 131
 132 nir_shader *
 133 glsl_to_nir(const struct gl_shader_program *shader_prog,
 134             gl_shader_stage stage,
 135             const nir_shader_compiler_options *options)
 136 {
 137    struct gl_shader *sh = shader_prog->_LinkedShaders[stage];
 138
 139    nir_shader *shader = nir_shader_create(NULL, stage, options);
 140
 141    nir_visitor v1(shader);
 142    nir_function_visitor v2(&v1);
 143    v2.run(sh->ir);
 144    visit_exec_list(sh->ir, &v1);
 145
 146    shader->info.name = ralloc_asprintf(shader, "GLSL%d", shader_prog->Name);
 147    if (shader_prog->Label)
 148       shader->info.label = ralloc_strdup(shader, shader_prog->Label);
 149    shader->info.num_textures = _mesa_fls(sh->Program->SamplersUsed);
 150    shader->info.num_ubos = sh->NumUniformBlocks;
 151    shader->info.num_abos = shader_prog->NumAtomicBuffers;
 152    shader->info.num_ssbos = sh->NumShaderStorageBlocks;
 153    shader->info.num_images = sh->NumImages;
 154    shader->info.inputs_read = sh->Program->InputsRead;
 155    shader->info.double_inputs_read = sh->Program->DoubleInputsRead;
 156    shader->info.outputs_written = sh->Program->OutputsWritten;
 157    shader->info.patch_inputs_read = sh->Program->PatchInputsRead;
 158    shader->info.patch_outputs_written = sh->Program->PatchOutputsWritten;
 159    shader->info.system_values_read = sh->Program->SystemValuesRead;
 160    shader->info.uses_texture_gather = sh->Program->UsesGather;
 161    shader->info.uses_clip_distance_out =
 162       sh->Program->ClipDistanceArraySize != 0;
 163    shader->info.separate_shader = shader_prog->SeparateShader;
 164    shader->info.has_transform_feedback_varyings =
 165       shader_prog->TransformFeedback.NumVarying > 0;
 166
 167    switch (stage) {
 168    case MESA_SHADER_TESS_CTRL:
 169       shader->info.tcs.vertices_out = sh->TessCtrl.VerticesOut;
 170       break;
 171
 172    case MESA_SHADER_GEOMETRY:
 173       shader->info.gs.vertices_in = shader_prog->Geom.VerticesIn;
 174       shader->info.gs.output_primitive = sh->Geom.OutputType;
 175       shader->info.gs.vertices_out = sh->Geom.VerticesOut;
 176       shader->info.gs.invocations = sh->Geom.Invocations;
 177       shader->info.gs.uses_end_primitive = shader_prog->Geom.UsesEndPrimitive;
 178       shader->info.gs.uses_streams = shader_prog->Geom.UsesStreams;
 179       break;
 180
 181    case MESA_SHADER_FRAGMENT: {
 182       struct gl_fragment_program *fp =
 183          (struct gl_fragment_program *)sh->Program;
 184
 185       shader->info.fs.uses_discard = fp->UsesKill;
 186       shader->info.fs.uses_sample_qualifier = fp->IsSample != 0;
 187       shader->info.fs.early_fragment_tests = sh->EarlyFragmentTests;
 188       shader->info.fs.depth_layout = fp->FragDepthLayout;
 189       break;
 190    }
 191
 192    case MESA_SHADER_COMPUTE: {
 193       struct gl_compute_program *cp = (struct gl_compute_program *)sh->Program;
 194       shader->info.cs.local_size[0] = cp->LocalSize[0];
 195       shader->info.cs.local_size[1] = cp->LocalSize[1];
 196       shader->info.cs.local_size[2] = cp->LocalSize[2];
 197       break;
 198    }
 199
 200    default:
 201       break; /* No stage-specific info */
 202    }
 203
 204    return shader;
 205 }
 206
 207 nir_visitor::nir_visitor(nir_shader *shader)
 208 {
 209    this->supports_ints = shader->options->native_integers;
 210    this->shader = shader;
 211    this->is_global = true;
 212    this->var_table = _mesa_hash_table_create(NULL, _mesa_hash_pointer,
 213                                              _mesa_key_pointer_equal);
 214    this->overload_table = _mesa_hash_table_create(NULL, _mesa_hash_pointer,
 215                                                   _mesa_key_pointer_equal);
 216    this->result = NULL;
 217    this->impl = NULL;
 218    this->var = NULL;
 219    this->deref_head = NULL;
 220    this->deref_tail = NULL;
 221    memset(&this->b, 0, sizeof(this->b));
 222 }
 223
 224 nir_visitor::~nir_visitor()
 225 {
 226    _mesa_hash_table_destroy(this->var_table, NULL);
 227    _mesa_hash_table_destroy(this->overload_table, NULL);
 228 }
 229
 230 nir_deref_var *
 231 nir_visitor::evaluate_deref(nir_instr *mem_ctx, ir_instruction *ir)
 232 {
 233    ir->accept(this);
 234    ralloc_steal(mem_ctx, this->deref_head);
 235    return this->deref_head;
 236 }
 237
 238 static nir_constant *
 239 constant_copy(ir_constant *ir, void *mem_ctx)
 240 {
 241    if (ir == NULL)
 242       return NULL;
 243
 244    nir_constant *ret = ralloc(mem_ctx, nir_constant);
 245
 246    unsigned total_elems = ir->type->components();
 247    unsigned i;
 248
 249    ret->num_elements = 0;
 250    switch (ir->type->base_type) {
 251    case GLSL_TYPE_UINT:
 252       for (i = 0; i < total_elems; i++)
 253          ret->value.u[i] = ir->value.u[i];
 254       break;
 255
 256    case GLSL_TYPE_INT:
 257       for (i = 0; i < total_elems; i++)
 258          ret->value.i[i] = ir->value.i[i];
 259       break;
 260
 261    case GLSL_TYPE_FLOAT:
 262       for (i = 0; i < total_elems; i++)
 263          ret->value.f[i] = ir->value.f[i];
 264       break;
 265
 266    case GLSL_TYPE_DOUBLE:
 267       for (i = 0; i < total_elems; i++)
 268          ret->value.d[i] = ir->value.d[i];
 269       break;
 270
 271    case GLSL_TYPE_BOOL:
 272       for (i = 0; i < total_elems; i++)
 273          ret->value.b[i] = ir->value.b[i];
 274       break;
 275
 276    case GLSL_TYPE_STRUCT:
 277       ret->elements = ralloc_array(mem_ctx, nir_constant *,
 278                                    ir->type->length);
 279       ret->num_elements = ir->type->length;
 280
 281       i = 0;
 282       foreach_in_list(ir_constant, field, &ir->components) {
 283          ret->elements[i] = constant_copy(field, mem_ctx);
 284          i++;
 285       }
 286       break;
 287
 288    case GLSL_TYPE_ARRAY:
 289       ret->elements = ralloc_array(mem_ctx, nir_constant *,
 290                                    ir->type->length);
 291       ret->num_elements = ir->type->length;
 292
 293       for (i = 0; i < ir->type->length; i++)
 294          ret->elements[i] = constant_copy(ir->array_elements[i], mem_ctx);
 295       break;
 296
 297    default:
 298       unreachable("not reached");
 299    }
 300
 301    return ret;
 302 }
 303
 304 void
 305 nir_visitor::visit(ir_variable *ir)
 306 {
 307    nir_variable *var = ralloc(shader, nir_variable);
 308    var->type = ir->type;
 309    var->name = ralloc_strdup(var, ir->name);
 310
 311    var->data.read_only = ir->data.read_only;
 312    var->data.centroid = ir->data.centroid;
 313    var->data.sample = ir->data.sample;
 314    var->data.patch = ir->data.patch;
 315    var->data.invariant = ir->data.invariant;
 316    var->data.location = ir->data.location;
 317
 318    switch(ir->data.mode) {
 319    case ir_var_auto:
 320    case ir_var_temporary:
 321       if (is_global)
 322          var->data.mode = nir_var_global;
 323       else
 324          var->data.mode = nir_var_local;
 325       break;
 326
 327    case ir_var_function_in:
 328    case ir_var_function_out:
 329    case ir_var_function_inout:
 330    case ir_var_const_in:
 331       var->data.mode = nir_var_local;
 332       break;
 333
 334    case ir_var_shader_in:
 335       if (shader->stage == MESA_SHADER_FRAGMENT &&
 336           ir->data.location == VARYING_SLOT_FACE) {
 337          /* For whatever reason, GLSL IR makes gl_FrontFacing an input */
 338          var->data.location = SYSTEM_VALUE_FRONT_FACE;
 339          var->data.mode = nir_var_system_value;
 340       } else if (shader->stage == MESA_SHADER_GEOMETRY &&
 341                  ir->data.location == VARYING_SLOT_PRIMITIVE_ID) {
 342          /* For whatever reason, GLSL IR makes gl_PrimitiveIDIn an input */
 343          var->data.location = SYSTEM_VALUE_PRIMITIVE_ID;
 344          var->data.mode = nir_var_system_value;
 345       } else {
 346          var->data.mode = nir_var_shader_in;
 347       }
 348       break;
 349
 350    case ir_var_shader_out:
 351       var->data.mode = nir_var_shader_out;
 352       break;
 353
 354    case ir_var_uniform:
 355       var->data.mode = nir_var_uniform;
 356       break;
 357
 358    case ir_var_shader_storage:
 359       var->data.mode = nir_var_shader_storage;
 360       break;
 361
 362    case ir_var_system_value:
 363       var->data.mode = nir_var_system_value;
 364       break;
 365
 366    default:
 367       unreachable("not reached");
 368    }
 369
 370    var->data.interpolation = ir->data.interpolation;
 371    var->data.origin_upper_left = ir->data.origin_upper_left;
 372    var->data.pixel_center_integer = ir->data.pixel_center_integer;
 373    var->data.explicit_location = ir->data.explicit_location;
 374    var->data.explicit_index = ir->data.explicit_index;
 375    var->data.explicit_binding = ir->data.explicit_binding;
 376    var->data.has_initializer = ir->data.has_initializer;
 377    var->data.location_frac = ir->data.location_frac;
 378
 379    switch (ir->data.depth_layout) {
 380    case ir_depth_layout_none:
 381       var->data.depth_layout = nir_depth_layout_none;
 382       break;
 383    case ir_depth_layout_any:
 384       var->data.depth_layout = nir_depth_layout_any;
 385       break;
 386    case ir_depth_layout_greater:
 387       var->data.depth_layout = nir_depth_layout_greater;
 388       break;
 389    case ir_depth_layout_less:
 390       var->data.depth_layout = nir_depth_layout_less;
 391       break;
 392    case ir_depth_layout_unchanged:
 393       var->data.depth_layout = nir_depth_layout_unchanged;
 394       break;
 395    default:
 396       unreachable("not reached");
 397    }
 398
 399    var->data.index = ir->data.index;
 400    var->data.binding = ir->data.binding;
 401    var->data.offset = ir->data.offset;
 402    var->data.image.read_only = ir->data.image_read_only;
 403    var->data.image.write_only = ir->data.image_write_only;
 404    var->data.image.coherent = ir->data.image_coherent;
 405    var->data.image._volatile = ir->data.image_volatile;
 406    var->data.image.restrict_flag = ir->data.image_restrict;
 407    var->data.image.format = ir->data.image_format;
 408    var->data.max_array_access = ir->data.max_array_access;
 409
 410    var->num_state_slots = ir->get_num_state_slots();
 411    if (var->num_state_slots > 0) {
 412       var->state_slots = ralloc_array(var, nir_state_slot,
 413                                       var->num_state_slots);
 414
 415       ir_state_slot *state_slots = ir->get_state_slots();
 416       for (unsigned i = 0; i < var->num_state_slots; i++) {
 417          for (unsigned j = 0; j < 5; j++)
 418             var->state_slots[i].tokens[j] = state_slots[i].tokens[j];
 419          var->state_slots[i].swizzle = state_slots[i].swizzle;
 420       }
 421    } else {
 422       var->state_slots = NULL;
 423    }
 424
 425    var->constant_initializer = constant_copy(ir->constant_initializer, var);
 426
 427    var->interface_type = ir->get_interface_type();
 428
 429    if (var->data.mode == nir_var_local)
 430       nir_function_impl_add_variable(impl, var);
 431    else
 432       nir_shader_add_variable(shader, var);
 433
 434    _mesa_hash_table_insert(var_table, ir, var);
 435    this->var = var;
 436 }
 437
 438 ir_visitor_status
 439 nir_function_visitor::visit_enter(ir_function *ir)
 440 {
 441    foreach_in_list(ir_function_signature, sig, &ir->signatures) {
 442       visitor->create_function(sig);
 443    }
 444    return visit_continue_with_parent;
 445 }
 446
 447 void
 448 nir_visitor::create_function(ir_function_signature *ir)
 449 {
 450    if (ir->is_intrinsic)
 451       return;
 452
 453    nir_function *func = nir_function_create(shader, ir->function_name());
 454
 455    assert(ir->parameters.is_empty());
 456    assert(ir->return_type == glsl_type::void_type);
 457
 458    _mesa_hash_table_insert(this->overload_table, ir, func);
 459 }
 460
 461 void
 462 nir_visitor::visit(ir_function *ir)
 463 {
 464    foreach_in_list(ir_function_signature, sig, &ir->signatures)
 465       sig->accept(this);
 466 }
 467
 468 void
 469 nir_visitor::visit(ir_function_signature *ir)
 470 {
 471    if (ir->is_intrinsic)
 472       return;
 473
 474    struct hash_entry *entry =
 475       _mesa_hash_table_search(this->overload_table, ir);
 476
 477    assert(entry);
 478    nir_function *func = (nir_function *) entry->data;
 479
 480    if (ir->is_defined) {
 481       nir_function_impl *impl = nir_function_impl_create(func);
 482       this->impl = impl;
 483
 484       assert(strcmp(func->name, "main") == 0);
 485       assert(ir->parameters.is_empty());
 486       assert(func->return_type == glsl_type::void_type);
 487
 488       this->is_global = false;
 489
 490       nir_builder_init(&b, impl);
 491       b.cursor = nir_after_cf_list(&impl->body);
 492       visit_exec_list(&ir->body, this);
 493
 494       this->is_global = true;
 495    } else {
 496       func->impl = NULL;
 497    }
 498 }
 499
 500 void
 501 nir_visitor::visit(ir_loop *ir)
 502 {
 503    nir_loop *loop = nir_loop_create(this->shader);
 504    nir_builder_cf_insert(&b, &loop->cf_node);
 505
 506    b.cursor = nir_after_cf_list(&loop->body);
 507    visit_exec_list(&ir->body_instructions, this);
 508    b.cursor = nir_after_cf_node(&loop->cf_node);
 509 }
 510
 511 void
 512 nir_visitor::visit(ir_if *ir)
 513 {
 514    nir_src condition =
 515       nir_src_for_ssa(evaluate_rvalue(ir->condition));
 516
 517    nir_if *if_stmt = nir_if_create(this->shader);
 518    if_stmt->condition = condition;
 519    nir_builder_cf_insert(&b, &if_stmt->cf_node);
 520
 521    b.cursor = nir_after_cf_list(&if_stmt->then_list);
 522    visit_exec_list(&ir->then_instructions, this);
 523
 524    b.cursor = nir_after_cf_list(&if_stmt->else_list);
 525    visit_exec_list(&ir->else_instructions, this);
 526
 527    b.cursor = nir_after_cf_node(&if_stmt->cf_node);
 528 }
 529
 530 void
 531 nir_visitor::visit(ir_discard *ir)
 532 {
 533    /*
 534     * discards aren't treated as control flow, because before we lower them
 535     * they can appear anywhere in the shader and the stuff after them may still
 536     * be executed (yay, crazy GLSL rules!). However, after lowering, all the
 537     * discards will be immediately followed by a return.
 538     */
 539
 540    nir_intrinsic_instr *discard;
 541    if (ir->condition) {
 542       discard = nir_intrinsic_instr_create(this->shader,
 543                                            nir_intrinsic_discard_if);
 544       discard->src[0] =
 545          nir_src_for_ssa(evaluate_rvalue(ir->condition));
 546    } else {
 547       discard = nir_intrinsic_instr_create(this->shader, nir_intrinsic_discard);
 548    }
 549
 550    nir_builder_instr_insert(&b, &discard->instr);
 551 }
 552
 553 void
 554 nir_visitor::visit(ir_emit_vertex *ir)
 555 {
 556    nir_intrinsic_instr *instr =
 557       nir_intrinsic_instr_create(this->shader, nir_intrinsic_emit_vertex);
 558    nir_intrinsic_set_stream_id(instr, ir->stream_id());
 559    nir_builder_instr_insert(&b, &instr->instr);
 560 }
 561
 562 void
 563 nir_visitor::visit(ir_end_primitive *ir)
 564 {
 565    nir_intrinsic_instr *instr =
 566       nir_intrinsic_instr_create(this->shader, nir_intrinsic_end_primitive);
 567    nir_intrinsic_set_stream_id(instr, ir->stream_id());
 568    nir_builder_instr_insert(&b, &instr->instr);
 569 }
 570
 571 void
 572 nir_visitor::visit(ir_loop_jump *ir)
 573 {
 574    nir_jump_type type;
 575    switch (ir->mode) {
 576    case ir_loop_jump::jump_break:
 577       type = nir_jump_break;
 578       break;
 579    case ir_loop_jump::jump_continue:
 580       type = nir_jump_continue;
 581       break;
 582    default:
 583       unreachable("not reached");
 584    }
 585
 586    nir_jump_instr *instr = nir_jump_instr_create(this->shader, type);
 587    nir_builder_instr_insert(&b, &instr->instr);
 588 }
 589
 590 void
 591 nir_visitor::visit(ir_return *ir)
 592 {
 593    if (ir->value != NULL) {
 594       nir_intrinsic_instr *copy =
 595          nir_intrinsic_instr_create(this->shader, nir_intrinsic_copy_var);
 596
 597       copy->variables[0] = nir_deref_var_create(copy, this->impl->return_var);
 598       copy->variables[1] = evaluate_deref(&copy->instr, ir->value);
 599    }
 600
 601    nir_jump_instr *instr = nir_jump_instr_create(this->shader, nir_jump_return);
 602    nir_builder_instr_insert(&b, &instr->instr);
 603 }
 604
 605 void
 606 nir_visitor::visit(ir_call *ir)
 607 {
 608    if (ir->callee->is_intrinsic) {
 609       nir_intrinsic_op op;
 610       if (strcmp(ir->callee_name(), "__intrinsic_atomic_read") == 0) {
 611          op = nir_intrinsic_atomic_counter_read_var;
 612       } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_increment") == 0) {
 613          op = nir_intrinsic_atomic_counter_inc_var;
 614       } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_predecrement") == 0) {
 615          op = nir_intrinsic_atomic_counter_dec_var;
 616       } else if (strcmp(ir->callee_name(), "__intrinsic_image_load") == 0) {
 617          op = nir_intrinsic_image_load;
 618       } else if (strcmp(ir->callee_name(), "__intrinsic_image_store") == 0) {
 619          op = nir_intrinsic_image_store;
 620       } else if (strcmp(ir->callee_name(), "__intrinsic_image_atomic_add") == 0) {
 621          op = nir_intrinsic_image_atomic_add;
 622       } else if (strcmp(ir->callee_name(), "__intrinsic_image_atomic_min") == 0) {
 623          op = nir_intrinsic_image_atomic_min;
 624       } else if (strcmp(ir->callee_name(), "__intrinsic_image_atomic_max") == 0) {
 625          op = nir_intrinsic_image_atomic_max;
 626       } else if (strcmp(ir->callee_name(), "__intrinsic_image_atomic_and") == 0) {
 627          op = nir_intrinsic_image_atomic_and;
 628       } else if (strcmp(ir->callee_name(), "__intrinsic_image_atomic_or") == 0) {
 629          op = nir_intrinsic_image_atomic_or;
 630       } else if (strcmp(ir->callee_name(), "__intrinsic_image_atomic_xor") == 0) {
 631          op = nir_intrinsic_image_atomic_xor;
 632       } else if (strcmp(ir->callee_name(), "__intrinsic_image_atomic_exchange") == 0) {
 633          op = nir_intrinsic_image_atomic_exchange;
 634       } else if (strcmp(ir->callee_name(), "__intrinsic_image_atomic_comp_swap") == 0) {
 635          op = nir_intrinsic_image_atomic_comp_swap;
 636       } else if (strcmp(ir->callee_name(), "__intrinsic_memory_barrier") == 0) {
 637          op = nir_intrinsic_memory_barrier;
 638       } else if (strcmp(ir->callee_name(), "__intrinsic_image_size") == 0) {
 639          op = nir_intrinsic_image_size;
 640       } else if (strcmp(ir->callee_name(), "__intrinsic_image_samples") == 0) {
 641          op = nir_intrinsic_image_samples;
 642       } else if (strcmp(ir->callee_name(), "__intrinsic_store_ssbo") == 0) {
 643          op = nir_intrinsic_store_ssbo;
 644       } else if (strcmp(ir->callee_name(), "__intrinsic_load_ssbo") == 0) {
 645          op = nir_intrinsic_load_ssbo;
 646       } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_add_ssbo") == 0) {
 647          op = nir_intrinsic_ssbo_atomic_add;
 648       } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_and_ssbo") == 0) {
 649          op = nir_intrinsic_ssbo_atomic_and;
 650       } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_or_ssbo") == 0) {
 651          op = nir_intrinsic_ssbo_atomic_or;
 652       } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_xor_ssbo") == 0) {
 653          op = nir_intrinsic_ssbo_atomic_xor;
 654       } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_min_ssbo") == 0) {
 655          assert(ir->return_deref);
 656          if (ir->return_deref->type == glsl_type::int_type)
 657             op = nir_intrinsic_ssbo_atomic_imin;
 658          else if (ir->return_deref->type == glsl_type::uint_type)
 659             op = nir_intrinsic_ssbo_atomic_umin;
 660          else
 661             unreachable("Invalid type");
 662       } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_max_ssbo") == 0) {
 663          assert(ir->return_deref);
 664          if (ir->return_deref->type == glsl_type::int_type)
 665             op = nir_intrinsic_ssbo_atomic_imax;
 666          else if (ir->return_deref->type == glsl_type::uint_type)
 667             op = nir_intrinsic_ssbo_atomic_umax;
 668          else
 669             unreachable("Invalid type");
 670       } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_exchange_ssbo") == 0) {
 671          op = nir_intrinsic_ssbo_atomic_exchange;
 672       } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_comp_swap_ssbo") == 0) {
 673          op = nir_intrinsic_ssbo_atomic_comp_swap;
 674       } else if (strcmp(ir->callee_name(), "__intrinsic_shader_clock") == 0) {
 675          op = nir_intrinsic_shader_clock;
 676       } else if (strcmp(ir->callee_name(), "__intrinsic_group_memory_barrier") == 0) {
 677          op = nir_intrinsic_group_memory_barrier;
 678       } else if (strcmp(ir->callee_name(), "__intrinsic_memory_barrier_atomic_counter") == 0) {
 679          op = nir_intrinsic_memory_barrier_atomic_counter;
 680       } else if (strcmp(ir->callee_name(), "__intrinsic_memory_barrier_buffer") == 0) {
 681          op = nir_intrinsic_memory_barrier_buffer;
 682       } else if (strcmp(ir->callee_name(), "__intrinsic_memory_barrier_image") == 0) {
 683          op = nir_intrinsic_memory_barrier_image;
 684       } else if (strcmp(ir->callee_name(), "__intrinsic_memory_barrier_shared") == 0) {
 685          op = nir_intrinsic_memory_barrier_shared;
 686       } else if (strcmp(ir->callee_name(), "__intrinsic_load_shared") == 0) {
 687          op = nir_intrinsic_load_shared;
 688       } else if (strcmp(ir->callee_name(), "__intrinsic_store_shared") == 0) {
 689          op = nir_intrinsic_store_shared;
 690       } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_add_shared") == 0) {
 691          op = nir_intrinsic_shared_atomic_add;
 692       } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_and_shared") == 0) {
 693          op = nir_intrinsic_shared_atomic_and;
 694       } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_or_shared") == 0) {
 695          op = nir_intrinsic_shared_atomic_or;
 696       } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_xor_shared") == 0) {
 697          op = nir_intrinsic_shared_atomic_xor;
 698       } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_min_shared") == 0) {
 699          assert(ir->return_deref);
 700          if (ir->return_deref->type == glsl_type::int_type)
 701             op = nir_intrinsic_shared_atomic_imin;
 702          else if (ir->return_deref->type == glsl_type::uint_type)
 703             op = nir_intrinsic_shared_atomic_umin;
 704          else
 705             unreachable("Invalid type");
 706       } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_max_shared") == 0) {
 707          assert(ir->return_deref);
 708          if (ir->return_deref->type == glsl_type::int_type)
 709             op = nir_intrinsic_shared_atomic_imax;
 710          else if (ir->return_deref->type == glsl_type::uint_type)
 711             op = nir_intrinsic_shared_atomic_umax;
 712          else
 713             unreachable("Invalid type");
 714       } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_exchange_shared") == 0) {
 715          op = nir_intrinsic_shared_atomic_exchange;
 716       } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_comp_swap_shared") == 0) {
 717          op = nir_intrinsic_shared_atomic_comp_swap;
 718       } else {
 719          unreachable("not reached");
 720       }
 721
 722       nir_intrinsic_instr *instr = nir_intrinsic_instr_create(shader, op);
 723       nir_dest *dest = &instr->dest;
 724
 725       switch (op) {
 726       case nir_intrinsic_atomic_counter_read_var:
 727       case nir_intrinsic_atomic_counter_inc_var:
 728       case nir_intrinsic_atomic_counter_dec_var: {
 729          ir_dereference *param =
 730             (ir_dereference *) ir->actual_parameters.get_head();
 731          instr->variables[0] = evaluate_deref(&instr->instr, param);
 732          nir_ssa_dest_init(&instr->instr, &instr->dest, 1, 32, NULL);
 733          nir_builder_instr_insert(&b, &instr->instr);
 734          break;
 735       }
 736       case nir_intrinsic_image_load:
 737       case nir_intrinsic_image_store:
 738       case nir_intrinsic_image_atomic_add:
 739       case nir_intrinsic_image_atomic_min:
 740       case nir_intrinsic_image_atomic_max:
 741       case nir_intrinsic_image_atomic_and:
 742       case nir_intrinsic_image_atomic_or:
 743       case nir_intrinsic_image_atomic_xor:
 744       case nir_intrinsic_image_atomic_exchange:
 745       case nir_intrinsic_image_atomic_comp_swap:
 746       case nir_intrinsic_image_samples:
 747       case nir_intrinsic_image_size: {
 748          nir_ssa_undef_instr *instr_undef =
 749             nir_ssa_undef_instr_create(shader, 1, 32);
 750          nir_builder_instr_insert(&b, &instr_undef->instr);
 751
 752          /* Set the image variable dereference. */
 753          exec_node *param = ir->actual_parameters.get_head();
 754          ir_dereference *image = (ir_dereference *)param;
 755          const glsl_type *type =
 756             image->variable_referenced()->type->without_array();
 757
 758          instr->variables[0] = evaluate_deref(&instr->instr, image);
 759          param = param->get_next();
 760
 761          /* Set the intrinsic destination. */
 762          if (ir->return_deref) {
 763             const nir_intrinsic_info *info =
 764                     &nir_intrinsic_infos[instr->intrinsic];
 765             nir_ssa_dest_init(&instr->instr, &instr->dest,
 766                               info->dest_components, 32, NULL);
 767          }
 768
 769          if (op == nir_intrinsic_image_size ||
 770              op == nir_intrinsic_image_samples) {
 771             nir_builder_instr_insert(&b, &instr->instr);
 772             break;
 773          }
 774
 775          /* Set the address argument, extending the coordinate vector to four
 776           * components.
 777           */
 778          nir_ssa_def *src_addr =
 779             evaluate_rvalue((ir_dereference *)param);
 780          nir_ssa_def *srcs[4];
 781
 782          for (int i = 0; i < 4; i++) {
 783             if (i < type->coordinate_components())
 784                srcs[i] = nir_channel(&b, src_addr, i);
 785             else
 786                srcs[i] = &instr_undef->def;
 787          }
 788
 789          instr->src[0] = nir_src_for_ssa(nir_vec(&b, srcs, 4));
 790          param = param->get_next();
 791
 792          /* Set the sample argument, which is undefined for single-sample
 793           * images.
 794           */
 795          if (type->sampler_dimensionality == GLSL_SAMPLER_DIM_MS) {
 796             instr->src[1] =
 797                nir_src_for_ssa(evaluate_rvalue((ir_dereference *)param));
 798             param = param->get_next();
 799          } else {
 800             instr->src[1] = nir_src_for_ssa(&instr_undef->def);
 801          }
 802
 803          /* Set the intrinsic parameters. */
 804          if (!param->is_tail_sentinel()) {
 805             instr->src[2] =
 806                nir_src_for_ssa(evaluate_rvalue((ir_dereference *)param));
 807             param = param->get_next();
 808          }
 809
 810          if (!param->is_tail_sentinel()) {
 811             instr->src[3] =
 812                nir_src_for_ssa(evaluate_rvalue((ir_dereference *)param));
 813             param = param->get_next();
 814          }
 815          nir_builder_instr_insert(&b, &instr->instr);
 816          break;
 817       }
 818       case nir_intrinsic_memory_barrier:
 819       case nir_intrinsic_group_memory_barrier:
 820       case nir_intrinsic_memory_barrier_atomic_counter:
 821       case nir_intrinsic_memory_barrier_buffer:
 822       case nir_intrinsic_memory_barrier_image:
 823       case nir_intrinsic_memory_barrier_shared:
 824          nir_builder_instr_insert(&b, &instr->instr);
 825          break;
 826       case nir_intrinsic_shader_clock:
 827          nir_ssa_dest_init(&instr->instr, &instr->dest, 1, 32, NULL);
 828          nir_builder_instr_insert(&b, &instr->instr);
 829          break;
 830       case nir_intrinsic_store_ssbo: {
 831          exec_node *param = ir->actual_parameters.get_head();
 832          ir_rvalue *block = ((ir_instruction *)param)->as_rvalue();
 833
 834          param = param->get_next();
 835          ir_rvalue *offset = ((ir_instruction *)param)->as_rvalue();
 836
 837          param = param->get_next();
 838          ir_rvalue *val = ((ir_instruction *)param)->as_rvalue();
 839
 840          param = param->get_next();
 841          ir_constant *write_mask = ((ir_instruction *)param)->as_constant();
 842          assert(write_mask);
 843
 844          instr->src[0] = nir_src_for_ssa(evaluate_rvalue(val));
 845          instr->src[1] = nir_src_for_ssa(evaluate_rvalue(block));
 846          instr->src[2] = nir_src_for_ssa(evaluate_rvalue(offset));
 847          nir_intrinsic_set_write_mask(instr, write_mask->value.u[0]);
 848          instr->num_components = val->type->vector_elements;
 849
 850          nir_builder_instr_insert(&b, &instr->instr);
 851          break;
 852       }
 853       case nir_intrinsic_load_ssbo: {
 854          exec_node *param = ir->actual_parameters.get_head();
 855          ir_rvalue *block = ((ir_instruction *)param)->as_rvalue();
 856
 857          param = param->get_next();
 858          ir_rvalue *offset = ((ir_instruction *)param)->as_rvalue();
 859
 860          instr->src[0] = nir_src_for_ssa(evaluate_rvalue(block));
 861          instr->src[1] = nir_src_for_ssa(evaluate_rvalue(offset));
 862
 863          const glsl_type *type = ir->return_deref->var->type;
 864          instr->num_components = type->vector_elements;
 865
 866          /* Setup destination register */
 867          unsigned bit_size = glsl_get_bit_size(type);
 868          nir_ssa_dest_init(&instr->instr, &instr->dest,
 869                            type->vector_elements, bit_size, NULL);
 870
 871          /* Insert the created nir instruction now since in the case of boolean
 872           * result we will need to emit another instruction after it
 873           */
 874          nir_builder_instr_insert(&b, &instr->instr);
 875
 876          /*
 877           * In SSBO/UBO's, a true boolean value is any non-zero value, but we
 878           * consider a true boolean to be ~0. Fix this up with a != 0
 879           * comparison.
 880           */
 881          if (type->base_type == GLSL_TYPE_BOOL) {
 882             nir_alu_instr *load_ssbo_compare =
 883                nir_alu_instr_create(shader, nir_op_ine);
 884             load_ssbo_compare->src[0].src.is_ssa = true;
 885             load_ssbo_compare->src[0].src.ssa = &instr->dest.ssa;
 886             load_ssbo_compare->src[1].src =
 887                nir_src_for_ssa(nir_imm_int(&b, 0));
 888             for (unsigned i = 0; i < type->vector_elements; i++)
 889                load_ssbo_compare->src[1].swizzle[i] = 0;
 890             nir_ssa_dest_init(&load_ssbo_compare->instr,
 891                               &load_ssbo_compare->dest.dest,
 892                               type->vector_elements, bit_size, NULL);
 893             load_ssbo_compare->dest.write_mask = (1 << type->vector_elements) - 1;
 894             nir_builder_instr_insert(&b, &load_ssbo_compare->instr);
 895             dest = &load_ssbo_compare->dest.dest;
 896          }
 897          break;
 898       }
 899       case nir_intrinsic_ssbo_atomic_add:
 900       case nir_intrinsic_ssbo_atomic_imin:
 901       case nir_intrinsic_ssbo_atomic_umin:
 902       case nir_intrinsic_ssbo_atomic_imax:
 903       case nir_intrinsic_ssbo_atomic_umax:
 904       case nir_intrinsic_ssbo_atomic_and:
 905       case nir_intrinsic_ssbo_atomic_or:
 906       case nir_intrinsic_ssbo_atomic_xor:
 907       case nir_intrinsic_ssbo_atomic_exchange:
 908       case nir_intrinsic_ssbo_atomic_comp_swap: {
 909          int param_count = ir->actual_parameters.length();
 910          assert(param_count == 3 || param_count == 4);
 911
 912          /* Block index */
 913          exec_node *param = ir->actual_parameters.get_head();
 914          ir_instruction *inst = (ir_instruction *) param;
 915          instr->src[0] = nir_src_for_ssa(evaluate_rvalue(inst->as_rvalue()));
 916
 917          /* Offset */
 918          param = param->get_next();
 919          inst = (ir_instruction *) param;
 920          instr->src[1] = nir_src_for_ssa(evaluate_rvalue(inst->as_rvalue()));
 921
 922          /* data1 parameter (this is always present) */
 923          param = param->get_next();
 924          inst = (ir_instruction *) param;
 925          instr->src[2] = nir_src_for_ssa(evaluate_rvalue(inst->as_rvalue()));
 926
 927          /* data2 parameter (only with atomic_comp_swap) */
 928          if (param_count == 4) {
 929             assert(op == nir_intrinsic_ssbo_atomic_comp_swap);
 930             param = param->get_next();
 931             inst = (ir_instruction *) param;
 932             instr->src[3] = nir_src_for_ssa(evaluate_rvalue(inst->as_rvalue()));
 933          }
 934
 935          /* Atomic result */
 936          assert(ir->return_deref);
 937          nir_ssa_dest_init(&instr->instr, &instr->dest,
 938                            ir->return_deref->type->vector_elements, 32, NULL);
 939          nir_builder_instr_insert(&b, &instr->instr);
 940          break;
 941       }
 942       case nir_intrinsic_load_shared: {
 943          exec_node *param = ir->actual_parameters.get_head();
 944          ir_rvalue *offset = ((ir_instruction *)param)->as_rvalue();
 945
 946          nir_intrinsic_set_base(instr, 0);
 947          instr->src[0] = nir_src_for_ssa(evaluate_rvalue(offset));
 948
 949          const glsl_type *type = ir->return_deref->var->type;
 950          instr->num_components = type->vector_elements;
 951
 952          /* Setup destination register */
 953          unsigned bit_size = glsl_get_bit_size(type);
 954          nir_ssa_dest_init(&instr->instr, &instr->dest,
 955                            type->vector_elements, bit_size, NULL);
 956
 957          nir_builder_instr_insert(&b, &instr->instr);
 958          break;
 959       }
 960       case nir_intrinsic_store_shared: {
 961          exec_node *param = ir->actual_parameters.get_head();
 962          ir_rvalue *offset = ((ir_instruction *)param)->as_rvalue();
 963
 964          param = param->get_next();
 965          ir_rvalue *val = ((ir_instruction *)param)->as_rvalue();
 966
 967          param = param->get_next();
 968          ir_constant *write_mask = ((ir_instruction *)param)->as_constant();
 969          assert(write_mask);
 970
 971          nir_intrinsic_set_base(instr, 0);
 972          instr->src[1] = nir_src_for_ssa(evaluate_rvalue(offset));
 973
 974          nir_intrinsic_set_write_mask(instr, write_mask->value.u[0]);
 975
 976          instr->src[0] = nir_src_for_ssa(evaluate_rvalue(val));
 977          instr->num_components = val->type->vector_elements;
 978
 979          nir_builder_instr_insert(&b, &instr->instr);
 980          break;
 981       }
 982       case nir_intrinsic_shared_atomic_add:
 983       case nir_intrinsic_shared_atomic_imin:
 984       case nir_intrinsic_shared_atomic_umin:
 985       case nir_intrinsic_shared_atomic_imax:
 986       case nir_intrinsic_shared_atomic_umax:
 987       case nir_intrinsic_shared_atomic_and:
 988       case nir_intrinsic_shared_atomic_or:
 989       case nir_intrinsic_shared_atomic_xor:
 990       case nir_intrinsic_shared_atomic_exchange:
 991       case nir_intrinsic_shared_atomic_comp_swap: {
 992          int param_count = ir->actual_parameters.length();
 993          assert(param_count == 2 || param_count == 3);
 994
 995          /* Offset */
 996          exec_node *param = ir->actual_parameters.get_head();
 997          ir_instruction *inst = (ir_instruction *) param;
 998          instr->src[0] = nir_src_for_ssa(evaluate_rvalue(inst->as_rvalue()));
 999
1000          /* data1 parameter (this is always present) */
1001          param = param->get_next();
1002          inst = (ir_instruction *) param;
1003          instr->src[1] = nir_src_for_ssa(evaluate_rvalue(inst->as_rvalue()));
1004
1005          /* data2 parameter (only with atomic_comp_swap) */
1006          if (param_count == 3) {
1007             assert(op == nir_intrinsic_shared_atomic_comp_swap);
1008             param = param->get_next();
1009             inst = (ir_instruction *) param;
1010             instr->src[2] =
1011                nir_src_for_ssa(evaluate_rvalue(inst->as_rvalue()));
1012          }
1013
1014          /* Atomic result */
1015          assert(ir->return_deref);
1016          unsigned bit_size = glsl_get_bit_size(ir->return_deref->type);
1017          nir_ssa_dest_init(&instr->instr, &instr->dest,
1018                            ir->return_deref->type->vector_elements,
1019                            bit_size, NULL);
1020          nir_builder_instr_insert(&b, &instr->instr);
1021          break;
1022       }
1023       default:
1024          unreachable("not reached");
1025       }
1026
1027       if (ir->return_deref) {
1028          nir_intrinsic_instr *store_instr =
1029             nir_intrinsic_instr_create(shader, nir_intrinsic_store_var);
1030          store_instr->num_components = ir->return_deref->type->vector_elements;
1031          nir_intrinsic_set_write_mask(store_instr,
1032                                       (1 << store_instr->num_components) - 1);
1033
1034          store_instr->variables[0] =
1035             evaluate_deref(&store_instr->instr, ir->return_deref);
1036          store_instr->src[0] = nir_src_for_ssa(&dest->ssa);
1037
1038          nir_builder_instr_insert(&b, &store_instr->instr);
1039       }
1040
1041       return;
1042    }
1043
1044    struct hash_entry *entry =
1045       _mesa_hash_table_search(this->overload_table, ir->callee);
1046    assert(entry);
1047    nir_function *callee = (nir_function *) entry->data;
1048
1049    nir_call_instr *instr = nir_call_instr_create(this->shader, callee);
1050
1051    unsigned i = 0;
1052    foreach_in_list(ir_dereference, param, &ir->actual_parameters) {
1053       instr->params[i] = evaluate_deref(&instr->instr, param);
1054       i++;
1055    }
1056
1057    instr->return_deref = evaluate_deref(&instr->instr, ir->return_deref);
1058    nir_builder_instr_insert(&b, &instr->instr);
1059 }
1060
1061 void
1062 nir_visitor::visit(ir_assignment *ir)
1063 {
1064    unsigned num_components = ir->lhs->type->vector_elements;
1065
1066    b.exact = ir->lhs->variable_referenced()->data.invariant ||
1067              ir->lhs->variable_referenced()->data.precise;
1068
1069    if ((ir->rhs->as_dereference() || ir->rhs->as_constant()) &&
1070        (ir->write_mask == (1 << num_components) - 1 || ir->write_mask == 0)) {
1071       /* We're doing a plain-as-can-be copy, so emit a copy_var */
1072       nir_intrinsic_instr *copy =
1073          nir_intrinsic_instr_create(this->shader, nir_intrinsic_copy_var);
1074
1075       copy->variables[0] = evaluate_deref(&copy->instr, ir->lhs);
1076       copy->variables[1] = evaluate_deref(&copy->instr, ir->rhs);
1077
1078       if (ir->condition) {
1079          nir_if *if_stmt = nir_if_create(this->shader);
1080          if_stmt->condition = nir_src_for_ssa(evaluate_rvalue(ir->condition));
1081          nir_builder_cf_insert(&b, &if_stmt->cf_node);
1082          nir_instr_insert_after_cf_list(&if_stmt->then_list, &copy->instr);
1083          b.cursor = nir_after_cf_node(&if_stmt->cf_node);
1084       } else {
1085          nir_builder_instr_insert(&b, &copy->instr);
1086       }
1087       return;
1088    }
1089
1090    assert(ir->rhs->type->is_scalar() || ir->rhs->type->is_vector());
1091
1092    ir->lhs->accept(this);
1093    nir_deref_var *lhs_deref = this->deref_head;
1094    nir_ssa_def *src = evaluate_rvalue(ir->rhs);
1095
1096    if (ir->write_mask != (1 << num_components) - 1 && ir->write_mask != 0) {
1097       /* GLSL IR will give us the input to the write-masked assignment in a
1098        * single packed vector.  So, for example, if the writemask is xzw, then
1099        * we have to swizzle x -> x, y -> z, and z -> w and get the y component
1100        * from the load.
1101        */
1102       unsigned swiz[4];
1103       unsigned component = 0;
1104       for (unsigned i = 0; i < 4; i++) {
1105          swiz[i] = ir->write_mask & (1 << i) ? component++ : 0;
1106       }
1107       src = nir_swizzle(&b, src, swiz, num_components, !supports_ints);
1108    }
1109
1110    nir_intrinsic_instr *store =
1111       nir_intrinsic_instr_create(this->shader, nir_intrinsic_store_var);
1112    store->num_components = ir->lhs->type->vector_elements;
1113    nir_intrinsic_set_write_mask(store, ir->write_mask);
1114    nir_deref *store_deref = nir_copy_deref(store, &lhs_deref->deref);
1115    store->variables[0] = nir_deref_as_var(store_deref);
1116    store->src[0] = nir_src_for_ssa(src);
1117
1118    if (ir->condition) {
1119       nir_if *if_stmt = nir_if_create(this->shader);
1120       if_stmt->condition = nir_src_for_ssa(evaluate_rvalue(ir->condition));
1121       nir_builder_cf_insert(&b, &if_stmt->cf_node);
1122       nir_instr_insert_after_cf_list(&if_stmt->then_list, &store->instr);
1123       b.cursor = nir_after_cf_node(&if_stmt->cf_node);
1124    } else {
1125       nir_builder_instr_insert(&b, &store->instr);
1126    }
1127 }
1128
1129 /*
1130  * Given an instruction, returns a pointer to its destination or NULL if there
1131  * is no destination.
1132  *
1133  * Note that this only handles instructions we generate at this level.
1134  */
1135 static nir_dest *
1136 get_instr_dest(nir_instr *instr)
1137 {
1138    nir_alu_instr *alu_instr;
1139    nir_intrinsic_instr *intrinsic_instr;
1140    nir_tex_instr *tex_instr;
1141
1142    switch (instr->type) {
1143       case nir_instr_type_alu:
1144          alu_instr = nir_instr_as_alu(instr);
1145          return &alu_instr->dest.dest;
1146
1147       case nir_instr_type_intrinsic:
1148          intrinsic_instr = nir_instr_as_intrinsic(instr);
1149          if (nir_intrinsic_infos[intrinsic_instr->intrinsic].has_dest)
1150             return &intrinsic_instr->dest;
1151          else
1152             return NULL;
1153
1154       case nir_instr_type_tex:
1155          tex_instr = nir_instr_as_tex(instr);
1156          return &tex_instr->dest;
1157
1158       default:
1159          unreachable("not reached");
1160    }
1161
1162    return NULL;
1163 }
1164
1165 void
1166 nir_visitor::add_instr(nir_instr *instr, unsigned num_components,
1167                        unsigned bit_size)
1168 {
1169    nir_dest *dest = get_instr_dest(instr);
1170
1171    if (dest)
1172       nir_ssa_dest_init(instr, dest, num_components, bit_size, NULL);
1173
1174    nir_builder_instr_insert(&b, instr);
1175
1176    if (dest) {
1177       assert(dest->is_ssa);
1178       this->result = &dest->ssa;
1179    }
1180 }
1181
1182 nir_ssa_def *
1183 nir_visitor::evaluate_rvalue(ir_rvalue* ir)
1184 {
1185    ir->accept(this);
1186    if (ir->as_dereference() || ir->as_constant()) {
1187       /*
1188        * A dereference is being used on the right hand side, which means we
1189        * must emit a variable load.
1190        */
1191
1192       nir_intrinsic_instr *load_instr =
1193          nir_intrinsic_instr_create(this->shader, nir_intrinsic_load_var);
1194       load_instr->num_components = ir->type->vector_elements;
1195       load_instr->variables[0] = this->deref_head;
1196       ralloc_steal(load_instr, load_instr->variables[0]);
1197       unsigned bit_size = glsl_get_bit_size(ir->type);
1198       add_instr(&load_instr->instr, ir->type->vector_elements, bit_size);
1199    }
1200
1201    return this->result;
1202 }
1203
1204 static bool
1205 type_is_float(glsl_base_type type)
1206 {
1207    return type == GLSL_TYPE_FLOAT || type == GLSL_TYPE_DOUBLE;
1208 }
1209
1210 void
1211 nir_visitor::visit(ir_expression *ir)
1212 {
1213    /* Some special cases */
1214    switch (ir->operation) {
1215    case ir_binop_ubo_load: {
1216       nir_intrinsic_instr *load =
1217          nir_intrinsic_instr_create(this->shader, nir_intrinsic_load_ubo);
1218       unsigned bit_size = glsl_get_bit_size(ir->type);
1219       load->num_components = ir->type->vector_elements;
1220       load->src[0] = nir_src_for_ssa(evaluate_rvalue(ir->operands[0]));
1221       load->src[1] = nir_src_for_ssa(evaluate_rvalue(ir->operands[1]));
1222       add_instr(&load->instr, ir->type->vector_elements, bit_size);
1223
1224       /*
1225        * In UBO's, a true boolean value is any non-zero value, but we consider
1226        * a true boolean to be ~0. Fix this up with a != 0 comparison.
1227        */
1228
1229       if (ir->type->base_type == GLSL_TYPE_BOOL)
1230          this->result = nir_ine(&b, &load->dest.ssa, nir_imm_int(&b, 0));
1231
1232       return;
1233    }
1234
1235    case ir_unop_interpolate_at_centroid:
1236    case ir_binop_interpolate_at_offset:
1237    case ir_binop_interpolate_at_sample: {
1238       ir_dereference *deref = ir->operands[0]->as_dereference();
1239       ir_swizzle *swizzle = NULL;
1240       if (!deref) {
1241          /* the api does not allow a swizzle here, but the varying packing code
1242           * may have pushed one into here.
1243           */
1244          swizzle = ir->operands[0]->as_swizzle();
1245          assert(swizzle);
1246          deref = swizzle->val->as_dereference();
1247          assert(deref);
1248       }
1249
1250       deref->accept(this);
1251
1252       nir_intrinsic_op op;
1253       if (this->deref_head->var->data.mode == nir_var_shader_in) {
1254          switch (ir->operation) {
1255          case ir_unop_interpolate_at_centroid:
1256             op = nir_intrinsic_interp_var_at_centroid;
1257             break;
1258          case ir_binop_interpolate_at_offset:
1259             op = nir_intrinsic_interp_var_at_offset;
1260             break;
1261          case ir_binop_interpolate_at_sample:
1262             op = nir_intrinsic_interp_var_at_sample;
1263             break;
1264          default:
1265             unreachable("Invalid interpolation intrinsic");
1266          }
1267       } else {
1268          /* This case can happen if the vertex shader does not write the
1269           * given varying.  In this case, the linker will lower it to a
1270           * global variable.  Since interpolating a variable makes no
1271           * sense, we'll just turn it into a load which will probably
1272           * eventually end up as an SSA definition.
1273           */
1274          assert(this->deref_head->var->data.mode == nir_var_global);
1275          op = nir_intrinsic_load_var;
1276       }
1277
1278       nir_intrinsic_instr *intrin = nir_intrinsic_instr_create(shader, op);
1279       intrin->num_components = deref->type->vector_elements;
1280       intrin->variables[0] = this->deref_head;
1281       ralloc_steal(intrin, intrin->variables[0]);
1282
1283       if (intrin->intrinsic == nir_intrinsic_interp_var_at_offset ||
1284           intrin->intrinsic == nir_intrinsic_interp_var_at_sample)
1285          intrin->src[0] = nir_src_for_ssa(evaluate_rvalue(ir->operands[1]));
1286
1287       unsigned bit_size =  glsl_get_bit_size(deref->type);
1288       add_instr(&intrin->instr, deref->type->vector_elements, bit_size);
1289
1290       if (swizzle) {
1291          unsigned swiz[4] = {
1292             swizzle->mask.x, swizzle->mask.y, swizzle->mask.z, swizzle->mask.w
1293          };
1294
1295          result = nir_swizzle(&b, result, swiz,
1296                               swizzle->type->vector_elements, false);
1297       }
1298
1299       return;
1300    }
1301
1302    default:
1303       break;
1304    }
1305
1306    nir_ssa_def *srcs[4];
1307    for (unsigned i = 0; i < ir->get_num_operands(); i++)
1308       srcs[i] = evaluate_rvalue(ir->operands[i]);
1309
1310    glsl_base_type types[4];
1311    for (unsigned i = 0; i < ir->get_num_operands(); i++)
1312       if (supports_ints)
1313          types[i] = ir->operands[i]->type->base_type;
1314       else
1315          types[i] = GLSL_TYPE_FLOAT;
1316
1317    glsl_base_type out_type;
1318    if (supports_ints)
1319       out_type = ir->type->base_type;
1320    else
1321       out_type = GLSL_TYPE_FLOAT;
1322
1323    switch (ir->operation) {
1324    case ir_unop_bit_not: result = nir_inot(&b, srcs[0]); break;
1325    case ir_unop_logic_not:
1326       result = supports_ints ? nir_inot(&b, srcs[0]) : nir_fnot(&b, srcs[0]);
1327       break;
1328    case ir_unop_neg:
1329       result = type_is_float(types[0]) ? nir_fneg(&b, srcs[0])
1330                                        : nir_ineg(&b, srcs[0]);
1331       break;
1332    case ir_unop_abs:
1333       result = type_is_float(types[0]) ? nir_fabs(&b, srcs[0])
1334                                        : nir_iabs(&b, srcs[0]);
1335       break;
1336    case ir_unop_saturate:
1337       assert(type_is_float(types[0]));
1338       result = nir_fsat(&b, srcs[0]);
1339       break;
1340    case ir_unop_sign:
1341       result = type_is_float(types[0]) ? nir_fsign(&b, srcs[0])
1342                                        : nir_isign(&b, srcs[0]);
1343       break;
1344    case ir_unop_rcp:  result = nir_frcp(&b, srcs[0]);  break;
1345    case ir_unop_rsq:  result = nir_frsq(&b, srcs[0]);  break;
1346    case ir_unop_sqrt: result = nir_fsqrt(&b, srcs[0]); break;
1347    case ir_unop_exp:  unreachable("ir_unop_exp should have been lowered");
1348    case ir_unop_log:  unreachable("ir_unop_log should have been lowered");
1349    case ir_unop_exp2: result = nir_fexp2(&b, srcs[0]); break;
1350    case ir_unop_log2: result = nir_flog2(&b, srcs[0]); break;
1351    case ir_unop_i2f:
1352       result = supports_ints ? nir_i2f(&b, srcs[0]) : nir_fmov(&b, srcs[0]);
1353       break;
1354    case ir_unop_u2f:
1355       result = supports_ints ? nir_u2f(&b, srcs[0]) : nir_fmov(&b, srcs[0]);
1356       break;
1357    case ir_unop_b2f:
1358       result = supports_ints ? nir_b2f(&b, srcs[0]) : nir_fmov(&b, srcs[0]);
1359       break;
1360    case ir_unop_f2i:  result = nir_f2i(&b, srcs[0]);   break;
1361    case ir_unop_f2u:  result = nir_f2u(&b, srcs[0]);   break;
1362    case ir_unop_f2b:  result = nir_f2b(&b, srcs[0]);   break;
1363    case ir_unop_i2b:  result = nir_i2b(&b, srcs[0]);   break;
1364    case ir_unop_b2i:  result = nir_b2i(&b, srcs[0]);   break;
1365    case ir_unop_d2f:  result = nir_d2f(&b, srcs[0]);   break;
1366    case ir_unop_f2d:  result = nir_f2d(&b, srcs[0]);   break;
1367    case ir_unop_d2i:  result = nir_d2i(&b, srcs[0]);   break;
1368    case ir_unop_d2u:  result = nir_d2u(&b, srcs[0]);   break;
1369    case ir_unop_d2b:  result = nir_d2b(&b, srcs[0]);   break;
1370    case ir_unop_i2d:
1371       assert(supports_ints);
1372       result = nir_i2d(&b, srcs[0]);
1373       break;
1374    case ir_unop_u2d:
1375       assert(supports_ints);
1376       result = nir_u2d(&b, srcs[0]);
1377       break;
1378    case ir_unop_i2u:
1379    case ir_unop_u2i:
1380    case ir_unop_bitcast_i2f:
1381    case ir_unop_bitcast_f2i:
1382    case ir_unop_bitcast_u2f:
1383    case ir_unop_bitcast_f2u:
1384    case ir_unop_subroutine_to_int:
1385       /* no-op */
1386       result = nir_imov(&b, srcs[0]);
1387       break;
1388    case ir_unop_trunc: result = nir_ftrunc(&b, srcs[0]); break;
1389    case ir_unop_ceil:  result = nir_fceil(&b, srcs[0]); break;
1390    case ir_unop_floor: result = nir_ffloor(&b, srcs[0]); break;
1391    case ir_unop_fract: result = nir_ffract(&b, srcs[0]); break;
1392    case ir_unop_round_even: result = nir_fround_even(&b, srcs[0]); break;
1393    case ir_unop_sin:   result = nir_fsin(&b, srcs[0]); break;
1394    case ir_unop_cos:   result = nir_fcos(&b, srcs[0]); break;
1395    case ir_unop_dFdx:        result = nir_fddx(&b, srcs[0]); break;
1396    case ir_unop_dFdy:        result = nir_fddy(&b, srcs[0]); break;
1397    case ir_unop_dFdx_fine:   result = nir_fddx_fine(&b, srcs[0]); break;
1398    case ir_unop_dFdy_fine:   result = nir_fddy_fine(&b, srcs[0]); break;
1399    case ir_unop_dFdx_coarse: result = nir_fddx_coarse(&b, srcs[0]); break;
1400    case ir_unop_dFdy_coarse: result = nir_fddy_coarse(&b, srcs[0]); break;
1401    case ir_unop_pack_snorm_2x16:
1402       result = nir_pack_snorm_2x16(&b, srcs[0]);
1403       break;
1404    case ir_unop_pack_snorm_4x8:
1405       result = nir_pack_snorm_4x8(&b, srcs[0]);
1406       break;
1407    case ir_unop_pack_unorm_2x16:
1408       result = nir_pack_unorm_2x16(&b, srcs[0]);
1409       break;
1410    case ir_unop_pack_unorm_4x8:
1411       result = nir_pack_unorm_4x8(&b, srcs[0]);
1412       break;
1413    case ir_unop_pack_half_2x16:
1414       result = nir_pack_half_2x16(&b, srcs[0]);
1415       break;
1416    case ir_unop_unpack_snorm_2x16:
1417       result = nir_unpack_snorm_2x16(&b, srcs[0]);
1418       break;
1419    case ir_unop_unpack_snorm_4x8:
1420       result = nir_unpack_snorm_4x8(&b, srcs[0]);
1421       break;
1422    case ir_unop_unpack_unorm_2x16:
1423       result = nir_unpack_unorm_2x16(&b, srcs[0]);
1424       break;
1425    case ir_unop_unpack_unorm_4x8:
1426       result = nir_unpack_unorm_4x8(&b, srcs[0]);
1427       break;
1428    case ir_unop_unpack_half_2x16:
1429       result = nir_unpack_half_2x16(&b, srcs[0]);
1430       break;
1431    case ir_unop_pack_double_2x32:
1432       result = nir_pack_double_2x32(&b, srcs[0]);
1433       break;
1434    case ir_unop_unpack_double_2x32:
1435       result = nir_unpack_double_2x32(&b, srcs[0]);
1436       break;
1437    case ir_unop_bitfield_reverse:
1438       result = nir_bitfield_reverse(&b, srcs[0]);
1439       break;
1440    case ir_unop_bit_count:
1441       result = nir_bit_count(&b, srcs[0]);
1442       break;
1443    case ir_unop_find_msb:
1444       switch (types[0]) {
1445       case GLSL_TYPE_UINT:
1446          result = nir_ufind_msb(&b, srcs[0]);
1447          break;
1448       case GLSL_TYPE_INT:
1449          result = nir_ifind_msb(&b, srcs[0]);
1450          break;
1451       default:
1452          unreachable("Invalid type for findMSB()");
1453       }
1454       break;
1455    case ir_unop_find_lsb:
1456       result = nir_find_lsb(&b, srcs[0]);
1457       break;
1458
1459    case ir_unop_noise:
1460       switch (ir->type->vector_elements) {
1461       case 1:
1462          switch (ir->operands[0]->type->vector_elements) {
1463             case 1: result = nir_fnoise1_1(&b, srcs[0]); break;
1464             case 2: result = nir_fnoise1_2(&b, srcs[0]); break;
1465             case 3: result = nir_fnoise1_3(&b, srcs[0]); break;
1466             case 4: result = nir_fnoise1_4(&b, srcs[0]); break;
1467             default: unreachable("not reached");
1468          }
1469          break;
1470       case 2:
1471          switch (ir->operands[0]->type->vector_elements) {
1472             case 1: result = nir_fnoise2_1(&b, srcs[0]); break;
1473             case 2: result = nir_fnoise2_2(&b, srcs[0]); break;
1474             case 3: result = nir_fnoise2_3(&b, srcs[0]); break;
1475             case 4: result = nir_fnoise2_4(&b, srcs[0]); break;
1476             default: unreachable("not reached");
1477          }
1478          break;
1479       case 3:
1480          switch (ir->operands[0]->type->vector_elements) {
1481             case 1: result = nir_fnoise3_1(&b, srcs[0]); break;
1482             case 2: result = nir_fnoise3_2(&b, srcs[0]); break;
1483             case 3: result = nir_fnoise3_3(&b, srcs[0]); break;
1484             case 4: result = nir_fnoise3_4(&b, srcs[0]); break;
1485             default: unreachable("not reached");
1486          }
1487          break;
1488       case 4:
1489          switch (ir->operands[0]->type->vector_elements) {
1490             case 1: result = nir_fnoise4_1(&b, srcs[0]); break;
1491             case 2: result = nir_fnoise4_2(&b, srcs[0]); break;
1492             case 3: result = nir_fnoise4_3(&b, srcs[0]); break;
1493             case 4: result = nir_fnoise4_4(&b, srcs[0]); break;
1494             default: unreachable("not reached");
1495          }
1496          break;
1497       default:
1498          unreachable("not reached");
1499       }
1500       break;
1501    case ir_unop_get_buffer_size: {
1502       nir_intrinsic_instr *load = nir_intrinsic_instr_create(
1503          this->shader,
1504          nir_intrinsic_get_buffer_size);
1505       load->num_components = ir->type->vector_elements;
1506       load->src[0] = nir_src_for_ssa(evaluate_rvalue(ir->operands[0]));
1507       unsigned bit_size = glsl_get_bit_size(ir->type);
1508       add_instr(&load->instr, ir->type->vector_elements, bit_size);
1509       return;
1510    }
1511
1512    case ir_binop_add:
1513       result = type_is_float(out_type) ? nir_fadd(&b, srcs[0], srcs[1])
1514                                        : nir_iadd(&b, srcs[0], srcs[1]);
1515       break;
1516    case ir_binop_sub:
1517       result = type_is_float(out_type) ? nir_fsub(&b, srcs[0], srcs[1])
1518                                        : nir_isub(&b, srcs[0], srcs[1]);
1519       break;
1520    case ir_binop_mul:
1521       result = type_is_float(out_type) ? nir_fmul(&b, srcs[0], srcs[1])
1522                                        : nir_imul(&b, srcs[0], srcs[1]);
1523       break;
1524    case ir_binop_div:
1525       if (type_is_float(out_type))
1526          result = nir_fdiv(&b, srcs[0], srcs[1]);
1527       else if (out_type == GLSL_TYPE_INT)
1528          result = nir_idiv(&b, srcs[0], srcs[1]);
1529       else
1530          result = nir_udiv(&b, srcs[0], srcs[1]);
1531       break;
1532    case ir_binop_mod:
1533       result = type_is_float(out_type) ? nir_fmod(&b, srcs[0], srcs[1])
1534                                        : nir_umod(&b, srcs[0], srcs[1]);
1535       break;
1536    case ir_binop_min:
1537       if (type_is_float(out_type))
1538          result = nir_fmin(&b, srcs[0], srcs[1]);
1539       else if (out_type == GLSL_TYPE_INT)
1540          result = nir_imin(&b, srcs[0], srcs[1]);
1541       else
1542          result = nir_umin(&b, srcs[0], srcs[1]);
1543       break;
1544    case ir_binop_max:
1545       if (type_is_float(out_type))
1546          result = nir_fmax(&b, srcs[0], srcs[1]);
1547       else if (out_type == GLSL_TYPE_INT)
1548          result = nir_imax(&b, srcs[0], srcs[1]);
1549       else
1550          result = nir_umax(&b, srcs[0], srcs[1]);
1551       break;
1552    case ir_binop_pow: result = nir_fpow(&b, srcs[0], srcs[1]); break;
1553    case ir_binop_bit_and: result = nir_iand(&b, srcs[0], srcs[1]); break;
1554    case ir_binop_bit_or: result = nir_ior(&b, srcs[0], srcs[1]); break;
1555    case ir_binop_bit_xor: result = nir_ixor(&b, srcs[0], srcs[1]); break;
1556    case ir_binop_logic_and:
1557       result = supports_ints ? nir_iand(&b, srcs[0], srcs[1])
1558                              : nir_fand(&b, srcs[0], srcs[1]);
1559       break;
1560    case ir_binop_logic_or:
1561       result = supports_ints ? nir_ior(&b, srcs[0], srcs[1])
1562                              : nir_for(&b, srcs[0], srcs[1]);
1563       break;
1564    case ir_binop_logic_xor:
1565       result = supports_ints ? nir_ixor(&b, srcs[0], srcs[1])
1566                              : nir_fxor(&b, srcs[0], srcs[1]);
1567       break;
1568    case ir_binop_lshift: result = nir_ishl(&b, srcs[0], srcs[1]); break;
1569    case ir_binop_rshift:
1570       result = (out_type == GLSL_TYPE_INT) ? nir_ishr(&b, srcs[0], srcs[1])
1571                                            : nir_ushr(&b, srcs[0], srcs[1]);
1572       break;
1573    case ir_binop_imul_high:
1574       result = (out_type == GLSL_TYPE_INT) ? nir_imul_high(&b, srcs[0], srcs[1])
1575                                            : nir_umul_high(&b, srcs[0], srcs[1]);
1576       break;
1577    case ir_binop_carry:  result = nir_uadd_carry(&b, srcs[0], srcs[1]);  break;
1578    case ir_binop_borrow: result = nir_usub_borrow(&b, srcs[0], srcs[1]); break;
1579    case ir_binop_less:
1580       if (supports_ints) {
1581          if (type_is_float(types[0]))
1582             result = nir_flt(&b, srcs[0], srcs[1]);
1583          else if (types[0] == GLSL_TYPE_INT)
1584             result = nir_ilt(&b, srcs[0], srcs[1]);
1585          else
1586             result = nir_ult(&b, srcs[0], srcs[1]);
1587       } else {
1588          result = nir_slt(&b, srcs[0], srcs[1]);
1589       }
1590       break;
1591    case ir_binop_greater:
1592       if (supports_ints) {
1593          if (type_is_float(types[0]))
1594             result = nir_flt(&b, srcs[1], srcs[0]);
1595          else if (types[0] == GLSL_TYPE_INT)
1596             result = nir_ilt(&b, srcs[1], srcs[0]);
1597          else
1598             result = nir_ult(&b, srcs[1], srcs[0]);
1599       } else {
1600          result = nir_slt(&b, srcs[1], srcs[0]);
1601       }
1602       break;
1603    case ir_binop_lequal:
1604       if (supports_ints) {
1605          if (type_is_float(types[0]))
1606             result = nir_fge(&b, srcs[1], srcs[0]);
1607          else if (types[0] == GLSL_TYPE_INT)
1608             result = nir_ige(&b, srcs[1], srcs[0]);
1609          else
1610             result = nir_uge(&b, srcs[1], srcs[0]);
1611       } else {
1612          result = nir_slt(&b, srcs[1], srcs[0]);
1613       }
1614       break;
1615    case ir_binop_gequal:
1616       if (supports_ints) {
1617          if (type_is_float(types[0]))
1618             result = nir_fge(&b, srcs[0], srcs[1]);
1619          else if (types[0] == GLSL_TYPE_INT)
1620             result = nir_ige(&b, srcs[0], srcs[1]);
1621          else
1622             result = nir_uge(&b, srcs[0], srcs[1]);
1623       } else {
1624          result = nir_slt(&b, srcs[0], srcs[1]);
1625       }
1626       break;
1627    case ir_binop_equal:
1628       if (supports_ints) {
1629          if (type_is_float(types[0]))
1630             result = nir_feq(&b, srcs[0], srcs[1]);
1631          else
1632             result = nir_ieq(&b, srcs[0], srcs[1]);
1633       } else {
1634          result = nir_seq(&b, srcs[0], srcs[1]);
1635       }
1636       break;
1637    case ir_binop_nequal:
1638       if (supports_ints) {
1639          if (type_is_float(types[0]))
1640             result = nir_fne(&b, srcs[0], srcs[1]);
1641          else
1642             result = nir_ine(&b, srcs[0], srcs[1]);
1643       } else {
1644          result = nir_sne(&b, srcs[0], srcs[1]);
1645       }
1646       break;
1647    case ir_binop_all_equal:
1648       if (supports_ints) {
1649          if (type_is_float(types[0])) {
1650             switch (ir->operands[0]->type->vector_elements) {
1651                case 1: result = nir_feq(&b, srcs[0], srcs[1]); break;
1652                case 2: result = nir_ball_fequal2(&b, srcs[0], srcs[1]); break;
1653                case 3: result = nir_ball_fequal3(&b, srcs[0], srcs[1]); break;
1654                case 4: result = nir_ball_fequal4(&b, srcs[0], srcs[1]); break;
1655                default:
1656                   unreachable("not reached");
1657             }
1658          } else {
1659             switch (ir->operands[0]->type->vector_elements) {
1660                case 1: result = nir_ieq(&b, srcs[0], srcs[1]); break;
1661                case 2: result = nir_ball_iequal2(&b, srcs[0], srcs[1]); break;
1662                case 3: result = nir_ball_iequal3(&b, srcs[0], srcs[1]); break;
1663                case 4: result = nir_ball_iequal4(&b, srcs[0], srcs[1]); break;
1664                default:
1665                   unreachable("not reached");
1666             }
1667          }
1668       } else {
1669          switch (ir->operands[0]->type->vector_elements) {
1670             case 1: result = nir_seq(&b, srcs[0], srcs[1]); break;
1671             case 2: result = nir_fall_equal2(&b, srcs[0], srcs[1]); break;
1672             case 3: result = nir_fall_equal3(&b, srcs[0], srcs[1]); break;
1673             case 4: result = nir_fall_equal4(&b, srcs[0], srcs[1]); break;
1674             default:
1675                unreachable("not reached");
1676          }
1677       }
1678       break;
1679    case ir_binop_any_nequal:
1680       if (supports_ints) {
1681          if (type_is_float(types[0])) {
1682             switch (ir->operands[0]->type->vector_elements) {
1683                case 1: result = nir_fne(&b, srcs[0], srcs[1]); break;
1684                case 2: result = nir_bany_fnequal2(&b, srcs[0], srcs[1]); break;
1685                case 3: result = nir_bany_fnequal3(&b, srcs[0], srcs[1]); break;
1686                case 4: result = nir_bany_fnequal4(&b, srcs[0], srcs[1]); break;
1687                default:
1688                   unreachable("not reached");
1689             }
1690          } else {
1691             switch (ir->operands[0]->type->vector_elements) {
1692                case 1: result = nir_ine(&b, srcs[0], srcs[1]); break;
1693                case 2: result = nir_bany_inequal2(&b, srcs[0], srcs[1]); break;
1694                case 3: result = nir_bany_inequal3(&b, srcs[0], srcs[1]); break;
1695                case 4: result = nir_bany_inequal4(&b, srcs[0], srcs[1]); break;
1696                default:
1697                   unreachable("not reached");
1698             }
1699          }
1700       } else {
1701          switch (ir->operands[0]->type->vector_elements) {
1702             case 1: result = nir_sne(&b, srcs[0], srcs[1]); break;
1703             case 2: result = nir_fany_nequal2(&b, srcs[0], srcs[1]); break;
1704             case 3: result = nir_fany_nequal3(&b, srcs[0], srcs[1]); break;
1705             case 4: result = nir_fany_nequal4(&b, srcs[0], srcs[1]); break;
1706             default:
1707                unreachable("not reached");
1708          }
1709       }
1710       break;
1711    case ir_binop_dot:
1712       switch (ir->operands[0]->type->vector_elements) {
1713          case 2: result = nir_fdot2(&b, srcs[0], srcs[1]); break;
1714          case 3: result = nir_fdot3(&b, srcs[0], srcs[1]); break;
1715          case 4: result = nir_fdot4(&b, srcs[0], srcs[1]); break;
1716          default:
1717             unreachable("not reached");
1718       }
1719       break;
1720
1721    case ir_binop_ldexp: result = nir_ldexp(&b, srcs[0], srcs[1]); break;
1722    case ir_triop_fma:
1723       result = nir_ffma(&b, srcs[0], srcs[1], srcs[2]);
1724       break;
1725    case ir_triop_lrp:
1726       result = nir_flrp(&b, srcs[0], srcs[1], srcs[2]);
1727       break;
1728    case ir_triop_csel:
1729       if (supports_ints)
1730          result = nir_bcsel(&b, srcs[0], srcs[1], srcs[2]);
1731       else
1732          result = nir_fcsel(&b, srcs[0], srcs[1], srcs[2]);
1733       break;
1734    case ir_triop_bitfield_extract:
1735       result = (out_type == GLSL_TYPE_INT) ?
1736          nir_ibitfield_extract(&b, srcs[0], srcs[1], srcs[2]) :
1737          nir_ubitfield_extract(&b, srcs[0], srcs[1], srcs[2]);
1738       break;
1739    case ir_quadop_bitfield_insert:
1740       result = nir_bitfield_insert(&b, srcs[0], srcs[1], srcs[2], srcs[3]);
1741       break;
1742    case ir_quadop_vector:
1743       result = nir_vec(&b, srcs, ir->type->vector_elements);
1744       break;
1745
1746    default:
1747       unreachable("not reached");
1748    }
1749 }
1750
1751 void
1752 nir_visitor::visit(ir_swizzle *ir)
1753 {
1754    unsigned swizzle[4] = { ir->mask.x, ir->mask.y, ir->mask.z, ir->mask.w };
1755    result = nir_swizzle(&b, evaluate_rvalue(ir->val), swizzle,
1756                         ir->type->vector_elements, !supports_ints);
1757 }
1758
1759 void
1760 nir_visitor::visit(ir_texture *ir)
1761 {
1762    unsigned num_srcs;
1763    nir_texop op;
1764    switch (ir->op) {
1765    case ir_tex:
1766       op = nir_texop_tex;
1767       num_srcs = 1; /* coordinate */
1768       break;
1769
1770    case ir_txb:
1771    case ir_txl:
1772       op = (ir->op == ir_txb) ? nir_texop_txb : nir_texop_txl;
1773       num_srcs = 2; /* coordinate, bias/lod */
1774       break;
1775
1776    case ir_txd:
1777       op = nir_texop_txd; /* coordinate, dPdx, dPdy */
1778       num_srcs = 3;
1779       break;
1780
1781    case ir_txf:
1782       op = nir_texop_txf;
1783       if (ir->lod_info.lod != NULL)
1784          num_srcs = 2; /* coordinate, lod */
1785       else
1786          num_srcs = 1; /* coordinate */
1787       break;
1788
1789    case ir_txf_ms:
1790       op = nir_texop_txf_ms;
1791       num_srcs = 2; /* coordinate, sample_index */
1792       break;
1793
1794    case ir_txs:
1795       op = nir_texop_txs;
1796       if (ir->lod_info.lod != NULL)
1797          num_srcs = 1; /* lod */
1798       else
1799          num_srcs = 0;
1800       break;
1801
1802    case ir_lod:
1803       op = nir_texop_lod;
1804       num_srcs = 1; /* coordinate */
1805       break;
1806
1807    case ir_tg4:
1808       op = nir_texop_tg4;
1809       num_srcs = 1; /* coordinate */
1810       break;
1811
1812    case ir_query_levels:
1813       op = nir_texop_query_levels;
1814       num_srcs = 0;
1815       break;
1816
1817    case ir_texture_samples:
1818       op = nir_texop_texture_samples;
1819       num_srcs = 0;
1820       break;
1821
1822    case ir_samples_identical:
1823       op = nir_texop_samples_identical;
1824       num_srcs = 1; /* coordinate */
1825       break;
1826
1827    default:
1828       unreachable("not reached");
1829    }
1830
1831    if (ir->projector != NULL)
1832       num_srcs++;
1833    if (ir->shadow_comparitor != NULL)
1834       num_srcs++;
1835    if (ir->offset != NULL)
1836       num_srcs++;
1837
1838    nir_tex_instr *instr = nir_tex_instr_create(this->shader, num_srcs);
1839
1840    instr->op = op;
1841    instr->sampler_dim =
1842       (glsl_sampler_dim) ir->sampler->type->sampler_dimensionality;
1843    instr->is_array = ir->sampler->type->sampler_array;
1844    instr->is_shadow = ir->sampler->type->sampler_shadow;
1845    if (instr->is_shadow)
1846       instr->is_new_style_shadow = (ir->type->vector_elements == 1);
1847    switch (ir->type->base_type) {
1848    case GLSL_TYPE_FLOAT:
1849       instr->dest_type = nir_type_float;
1850       break;
1851    case GLSL_TYPE_INT:
1852       instr->dest_type = nir_type_int;
1853       break;
1854    case GLSL_TYPE_BOOL:
1855    case GLSL_TYPE_UINT:
1856       instr->dest_type = nir_type_uint;
1857       break;
1858    default:
1859       unreachable("not reached");
1860    }
1861
1862    instr->texture = evaluate_deref(&instr->instr, ir->sampler);
1863
1864    unsigned src_number = 0;
1865
1866    if (ir->coordinate != NULL) {
1867       instr->coord_components = ir->coordinate->type->vector_elements;
1868       instr->src[src_number].src =
1869          nir_src_for_ssa(evaluate_rvalue(ir->coordinate));
1870       instr->src[src_number].src_type = nir_tex_src_coord;
1871       src_number++;
1872    }
1873
1874    if (ir->projector != NULL) {
1875       instr->src[src_number].src =
1876          nir_src_for_ssa(evaluate_rvalue(ir->projector));
1877       instr->src[src_number].src_type = nir_tex_src_projector;
1878       src_number++;
1879    }
1880
1881    if (ir->shadow_comparitor != NULL) {
1882       instr->src[src_number].src =
1883          nir_src_for_ssa(evaluate_rvalue(ir->shadow_comparitor));
1884       instr->src[src_number].src_type = nir_tex_src_comparitor;
1885       src_number++;
1886    }
1887
1888    if (ir->offset != NULL) {
1889       /* we don't support multiple offsets yet */
1890       assert(ir->offset->type->is_vector() || ir->offset->type->is_scalar());
1891
1892       instr->src[src_number].src =
1893          nir_src_for_ssa(evaluate_rvalue(ir->offset));
1894       instr->src[src_number].src_type = nir_tex_src_offset;
1895       src_number++;
1896    }
1897
1898    switch (ir->op) {
1899    case ir_txb:
1900       instr->src[src_number].src =
1901          nir_src_for_ssa(evaluate_rvalue(ir->lod_info.bias));
1902       instr->src[src_number].src_type = nir_tex_src_bias;
1903       src_number++;
1904       break;
1905
1906    case ir_txl:
1907    case ir_txf:
1908    case ir_txs:
1909       if (ir->lod_info.lod != NULL) {
1910          instr->src[src_number].src =
1911             nir_src_for_ssa(evaluate_rvalue(ir->lod_info.lod));
1912          instr->src[src_number].src_type = nir_tex_src_lod;
1913          src_number++;
1914       }
1915       break;
1916
1917    case ir_txd:
1918       instr->src[src_number].src =
1919          nir_src_for_ssa(evaluate_rvalue(ir->lod_info.grad.dPdx));
1920       instr->src[src_number].src_type = nir_tex_src_ddx;
1921       src_number++;
1922       instr->src[src_number].src =
1923          nir_src_for_ssa(evaluate_rvalue(ir->lod_info.grad.dPdy));
1924       instr->src[src_number].src_type = nir_tex_src_ddy;
1925       src_number++;
1926       break;
1927
1928    case ir_txf_ms:
1929       instr->src[src_number].src =
1930          nir_src_for_ssa(evaluate_rvalue(ir->lod_info.sample_index));
1931       instr->src[src_number].src_type = nir_tex_src_ms_index;
1932       src_number++;
1933       break;
1934
1935    case ir_tg4:
1936       instr->component = ir->lod_info.component->as_constant()->value.u[0];
1937       break;
1938
1939    default:
1940       break;
1941    }
1942
1943    assert(src_number == num_srcs);
1944
1945    unsigned bit_size = glsl_get_bit_size(ir->type);
1946    add_instr(&instr->instr, nir_tex_instr_dest_size(instr), bit_size);
1947 }
1948
1949 void
1950 nir_visitor::visit(ir_constant *ir)
1951 {
1952    /*
1953     * We don't know if this variable is an array or struct that gets
1954     * dereferenced, so do the safe thing an make it a variable with a
1955     * constant initializer and return a dereference.
1956     */
1957
1958    nir_variable *var =
1959       nir_local_variable_create(this->impl, ir->type, "const_temp");
1960    var->data.read_only = true;
1961    var->constant_initializer = constant_copy(ir, var);
1962
1963    this->deref_head = nir_deref_var_create(this->shader, var);
1964    this->deref_tail = &this->deref_head->deref;
1965 }
1966
1967 void
1968 nir_visitor::visit(ir_dereference_variable *ir)
1969 {
1970    struct hash_entry *entry =
1971       _mesa_hash_table_search(this->var_table, ir->var);
1972    assert(entry);
1973    nir_variable *var = (nir_variable *) entry->data;
1974
1975    nir_deref_var *deref = nir_deref_var_create(this->shader, var);
1976    this->deref_head = deref;
1977    this->deref_tail = &deref->deref;
1978 }
1979
1980 void
1981 nir_visitor::visit(ir_dereference_record *ir)
1982 {
1983    ir->record->accept(this);
1984
1985    int field_index = this->deref_tail->type->field_index(ir->field);
1986    assert(field_index >= 0);
1987
1988    nir_deref_struct *deref = nir_deref_struct_create(this->deref_tail, field_index);
1989    deref->deref.type = ir->type;
1990    this->deref_tail->child = &deref->deref;
1991    this->deref_tail = &deref->deref;
1992 }
1993
1994 void
1995 nir_visitor::visit(ir_dereference_array *ir)
1996 {
1997    nir_deref_array *deref = nir_deref_array_create(this->shader);
1998    deref->deref.type = ir->type;
1999
2000    ir_constant *const_index = ir->array_index->as_constant();
2001    if (const_index != NULL) {
2002       deref->deref_array_type = nir_deref_array_type_direct;
2003       deref->base_offset = const_index->value.u[0];
2004    } else {
2005       deref->deref_array_type = nir_deref_array_type_indirect;
2006       deref->indirect =
2007          nir_src_for_ssa(evaluate_rvalue(ir->array_index));
2008    }
2009
2010    ir->array->accept(this);
2011
2012    this->deref_tail->child = &deref->deref;
2013    ralloc_steal(this->deref_tail, deref);
2014    this->deref_tail = &deref->deref;
2015 }
2016
2017 void
2018 nir_visitor::visit(ir_barrier *)
2019 {
2020    nir_intrinsic_instr *instr =
2021       nir_intrinsic_instr_create(this->shader, nir_intrinsic_barrier);
2022    nir_builder_instr_insert(&b, &instr->instr);
2023 }