src/compiler/glsl/glsl_to_nir.cpp

   1 /*
   2  * Copyright © 2014 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Connor Abbott (cwabbott0@gmail.com)
  25  *
  26  */
  27
  28 #include "glsl_to_nir.h"
  29 #include "ir_visitor.h"
  30 #include "ir_hierarchical_visitor.h"
  31 #include "ir.h"
  32 #include "compiler/nir/nir_control_flow.h"
  33 #include "compiler/nir/nir_builder.h"
  34 #include "main/imports.h"
  35
  36 /*
  37  * pass to lower GLSL IR to NIR
  38  *
  39  * This will lower variable dereferences to loads/stores of corresponding
  40  * variables in NIR - the variables will be converted to registers in a later
  41  * pass.
  42  */
  43
  44 namespace {
  45
  46 class nir_visitor : public ir_visitor
  47 {
  48 public:
  49    nir_visitor(nir_shader *shader);
  50    ~nir_visitor();
  51
  52    virtual void visit(ir_variable *);
  53    virtual void visit(ir_function *);
  54    virtual void visit(ir_function_signature *);
  55    virtual void visit(ir_loop *);
  56    virtual void visit(ir_if *);
  57    virtual void visit(ir_discard *);
  58    virtual void visit(ir_loop_jump *);
  59    virtual void visit(ir_return *);
  60    virtual void visit(ir_call *);
  61    virtual void visit(ir_assignment *);
  62    virtual void visit(ir_emit_vertex *);
  63    virtual void visit(ir_end_primitive *);
  64    virtual void visit(ir_expression *);
  65    virtual void visit(ir_swizzle *);
  66    virtual void visit(ir_texture *);
  67    virtual void visit(ir_constant *);
  68    virtual void visit(ir_dereference_variable *);
  69    virtual void visit(ir_dereference_record *);
  70    virtual void visit(ir_dereference_array *);
  71    virtual void visit(ir_barrier *);
  72
  73    void create_function(ir_function_signature *ir);
  74
  75 private:
  76    void add_instr(nir_instr *instr, unsigned num_components, unsigned bit_size);
  77    nir_ssa_def *evaluate_rvalue(ir_rvalue *ir);
  78
  79    nir_alu_instr *emit(nir_op op, unsigned dest_size, nir_ssa_def **srcs);
  80    nir_alu_instr *emit(nir_op op, unsigned dest_size, nir_ssa_def *src1);
  81    nir_alu_instr *emit(nir_op op, unsigned dest_size, nir_ssa_def *src1,
  82                        nir_ssa_def *src2);
  83    nir_alu_instr *emit(nir_op op, unsigned dest_size, nir_ssa_def *src1,
  84                        nir_ssa_def *src2, nir_ssa_def *src3);
  85
  86    bool supports_ints;
  87
  88    nir_shader *shader;
  89    nir_function_impl *impl;
  90    nir_builder b;
  91    nir_ssa_def *result; /* result of the expression tree last visited */
  92
  93    nir_deref_var *evaluate_deref(nir_instr *mem_ctx, ir_instruction *ir);
  94
  95    /* the head of the dereference chain we're creating */
  96    nir_deref_var *deref_head;
  97    /* the tail of the dereference chain we're creating */
  98    nir_deref *deref_tail;
  99
 100    nir_variable *var; /* variable created by ir_variable visitor */
 101
 102    /* whether the IR we're operating on is per-function or global */
 103    bool is_global;
 104
 105    /* map of ir_variable -> nir_variable */
 106    struct hash_table *var_table;
 107
 108    /* map of ir_function_signature -> nir_function_overload */
 109    struct hash_table *overload_table;
 110 };
 111
 112 /*
 113  * This visitor runs before the main visitor, calling create_function() for
 114  * each function so that the main visitor can resolve forward references in
 115  * calls.
 116  */
 117
 118 class nir_function_visitor : public ir_hierarchical_visitor
 119 {
 120 public:
 121    nir_function_visitor(nir_visitor *v) : visitor(v)
 122    {
 123    }
 124    virtual ir_visitor_status visit_enter(ir_function *);
 125
 126 private:
 127    nir_visitor *visitor;
 128 };
 129
 130 } /* end of anonymous namespace */
 131
 132 nir_shader *
 133 glsl_to_nir(const struct gl_shader_program *shader_prog,
 134             gl_shader_stage stage,
 135             const nir_shader_compiler_options *options)
 136 {
 137    struct gl_linked_shader *sh = shader_prog->_LinkedShaders[stage];
 138
 139    nir_shader *shader = nir_shader_create(NULL, stage, options, NULL);
 140
 141    nir_visitor v1(shader);
 142    nir_function_visitor v2(&v1);
 143    v2.run(sh->ir);
 144    visit_exec_list(sh->ir, &v1);
 145
 146    shader->info->name = ralloc_asprintf(shader, "GLSL%d", shader_prog->Name);
 147    if (shader_prog->Label)
 148       shader->info->label = ralloc_strdup(shader, shader_prog->Label);
 149    shader->info->num_textures = util_last_bit(sh->Program->SamplersUsed);
 150    shader->info->num_ubos = sh->NumUniformBlocks;
 151    shader->info->num_abos = shader_prog->NumAtomicBuffers;
 152    shader->info->num_ssbos = sh->NumShaderStorageBlocks;
 153    shader->info->num_images = sh->NumImages;
 154    shader->info->inputs_read = sh->Program->InputsRead;
 155    shader->info->double_inputs_read = sh->Program->DoubleInputsRead;
 156    shader->info->outputs_written = sh->Program->OutputsWritten;
 157    shader->info->outputs_read = sh->Program->OutputsRead;
 158    shader->info->patch_inputs_read = sh->Program->PatchInputsRead;
 159    shader->info->patch_outputs_written = sh->Program->PatchOutputsWritten;
 160    shader->info->system_values_read = sh->Program->SystemValuesRead;
 161    shader->info->uses_texture_gather = sh->Program->UsesGather;
 162    shader->info->uses_clip_distance_out =
 163       sh->Program->ClipDistanceArraySize != 0;
 164    shader->info->separate_shader = shader_prog->SeparateShader;
 165    shader->info->has_transform_feedback_varyings =
 166       shader_prog->TransformFeedback.NumVarying > 0;
 167
 168    switch (stage) {
 169    case MESA_SHADER_TESS_CTRL:
 170       shader->info->tcs.vertices_out = sh->info.TessCtrl.VerticesOut;
 171       break;
 172
 173    case MESA_SHADER_GEOMETRY:
 174       shader->info->gs.vertices_in = shader_prog->Geom.VerticesIn;
 175       shader->info->gs.output_primitive = sh->info.Geom.OutputType;
 176       shader->info->gs.vertices_out = sh->info.Geom.VerticesOut;
 177       shader->info->gs.invocations = sh->info.Geom.Invocations;
 178       shader->info->gs.uses_end_primitive = shader_prog->Geom.UsesEndPrimitive;
 179       shader->info->gs.uses_streams = shader_prog->Geom.UsesStreams;
 180       break;
 181
 182    case MESA_SHADER_FRAGMENT: {
 183       struct gl_fragment_program *fp =
 184          (struct gl_fragment_program *)sh->Program;
 185
 186       shader->info->fs.uses_discard = fp->UsesKill;
 187       shader->info->fs.uses_sample_qualifier = fp->IsSample != 0;
 188       shader->info->fs.early_fragment_tests = sh->info.EarlyFragmentTests;
 189       shader->info->fs.depth_layout = fp->FragDepthLayout;
 190       break;
 191    }
 192
 193    case MESA_SHADER_COMPUTE: {
 194       struct gl_compute_program *cp = (struct gl_compute_program *)sh->Program;
 195       shader->info->cs.local_size[0] = cp->LocalSize[0];
 196       shader->info->cs.local_size[1] = cp->LocalSize[1];
 197       shader->info->cs.local_size[2] = cp->LocalSize[2];
 198       break;
 199    }
 200
 201    default:
 202       break; /* No stage-specific info */
 203    }
 204
 205    return shader;
 206 }
 207
 208 nir_visitor::nir_visitor(nir_shader *shader)
 209 {
 210    this->supports_ints = shader->options->native_integers;
 211    this->shader = shader;
 212    this->is_global = true;
 213    this->var_table = _mesa_hash_table_create(NULL, _mesa_hash_pointer,
 214                                              _mesa_key_pointer_equal);
 215    this->overload_table = _mesa_hash_table_create(NULL, _mesa_hash_pointer,
 216                                                   _mesa_key_pointer_equal);
 217    this->result = NULL;
 218    this->impl = NULL;
 219    this->var = NULL;
 220    this->deref_head = NULL;
 221    this->deref_tail = NULL;
 222    memset(&this->b, 0, sizeof(this->b));
 223 }
 224
 225 nir_visitor::~nir_visitor()
 226 {
 227    _mesa_hash_table_destroy(this->var_table, NULL);
 228    _mesa_hash_table_destroy(this->overload_table, NULL);
 229 }
 230
 231 nir_deref_var *
 232 nir_visitor::evaluate_deref(nir_instr *mem_ctx, ir_instruction *ir)
 233 {
 234    ir->accept(this);
 235    ralloc_steal(mem_ctx, this->deref_head);
 236    return this->deref_head;
 237 }
 238
 239 static nir_constant *
 240 constant_copy(ir_constant *ir, void *mem_ctx)
 241 {
 242    if (ir == NULL)
 243       return NULL;
 244
 245    nir_constant *ret = ralloc(mem_ctx, nir_constant);
 246
 247    unsigned total_elems = ir->type->components();
 248    unsigned i;
 249
 250    ret->num_elements = 0;
 251    switch (ir->type->base_type) {
 252    case GLSL_TYPE_UINT:
 253       for (i = 0; i < total_elems; i++)
 254          ret->value.u[i] = ir->value.u[i];
 255       break;
 256
 257    case GLSL_TYPE_INT:
 258       for (i = 0; i < total_elems; i++)
 259          ret->value.i[i] = ir->value.i[i];
 260       break;
 261
 262    case GLSL_TYPE_FLOAT:
 263       for (i = 0; i < total_elems; i++)
 264          ret->value.f[i] = ir->value.f[i];
 265       break;
 266
 267    case GLSL_TYPE_DOUBLE:
 268       for (i = 0; i < total_elems; i++)
 269          ret->value.d[i] = ir->value.d[i];
 270       break;
 271
 272    case GLSL_TYPE_BOOL:
 273       for (i = 0; i < total_elems; i++)
 274          ret->value.b[i] = ir->value.b[i];
 275       break;
 276
 277    case GLSL_TYPE_STRUCT:
 278       ret->elements = ralloc_array(mem_ctx, nir_constant *,
 279                                    ir->type->length);
 280       ret->num_elements = ir->type->length;
 281
 282       i = 0;
 283       foreach_in_list(ir_constant, field, &ir->components) {
 284          ret->elements[i] = constant_copy(field, mem_ctx);
 285          i++;
 286       }
 287       break;
 288
 289    case GLSL_TYPE_ARRAY:
 290       ret->elements = ralloc_array(mem_ctx, nir_constant *,
 291                                    ir->type->length);
 292       ret->num_elements = ir->type->length;
 293
 294       for (i = 0; i < ir->type->length; i++)
 295          ret->elements[i] = constant_copy(ir->array_elements[i], mem_ctx);
 296       break;
 297
 298    default:
 299       unreachable("not reached");
 300    }
 301
 302    return ret;
 303 }
 304
 305 void
 306 nir_visitor::visit(ir_variable *ir)
 307 {
 308    nir_variable *var = ralloc(shader, nir_variable);
 309    var->type = ir->type;
 310    var->name = ralloc_strdup(var, ir->name);
 311
 312    var->data.read_only = ir->data.read_only;
 313    var->data.centroid = ir->data.centroid;
 314    var->data.sample = ir->data.sample;
 315    var->data.patch = ir->data.patch;
 316    var->data.invariant = ir->data.invariant;
 317    var->data.location = ir->data.location;
 318
 319    switch(ir->data.mode) {
 320    case ir_var_auto:
 321    case ir_var_temporary:
 322       if (is_global)
 323          var->data.mode = nir_var_global;
 324       else
 325          var->data.mode = nir_var_local;
 326       break;
 327
 328    case ir_var_function_in:
 329    case ir_var_function_out:
 330    case ir_var_function_inout:
 331    case ir_var_const_in:
 332       var->data.mode = nir_var_local;
 333       break;
 334
 335    case ir_var_shader_in:
 336       if (shader->stage == MESA_SHADER_FRAGMENT &&
 337           ir->data.location == VARYING_SLOT_FACE) {
 338          /* For whatever reason, GLSL IR makes gl_FrontFacing an input */
 339          var->data.location = SYSTEM_VALUE_FRONT_FACE;
 340          var->data.mode = nir_var_system_value;
 341       } else if (shader->stage == MESA_SHADER_GEOMETRY &&
 342                  ir->data.location == VARYING_SLOT_PRIMITIVE_ID) {
 343          /* For whatever reason, GLSL IR makes gl_PrimitiveIDIn an input */
 344          var->data.location = SYSTEM_VALUE_PRIMITIVE_ID;
 345          var->data.mode = nir_var_system_value;
 346       } else {
 347          var->data.mode = nir_var_shader_in;
 348       }
 349       break;
 350
 351    case ir_var_shader_out:
 352       var->data.mode = nir_var_shader_out;
 353       break;
 354
 355    case ir_var_uniform:
 356       var->data.mode = nir_var_uniform;
 357       break;
 358
 359    case ir_var_shader_storage:
 360       var->data.mode = nir_var_shader_storage;
 361       break;
 362
 363    case ir_var_system_value:
 364       var->data.mode = nir_var_system_value;
 365       break;
 366
 367    default:
 368       unreachable("not reached");
 369    }
 370
 371    var->data.interpolation = ir->data.interpolation;
 372    var->data.origin_upper_left = ir->data.origin_upper_left;
 373    var->data.pixel_center_integer = ir->data.pixel_center_integer;
 374    var->data.explicit_location = ir->data.explicit_location;
 375    var->data.explicit_index = ir->data.explicit_index;
 376    var->data.explicit_binding = ir->data.explicit_binding;
 377    var->data.has_initializer = ir->data.has_initializer;
 378    var->data.location_frac = ir->data.location_frac;
 379
 380    switch (ir->data.depth_layout) {
 381    case ir_depth_layout_none:
 382       var->data.depth_layout = nir_depth_layout_none;
 383       break;
 384    case ir_depth_layout_any:
 385       var->data.depth_layout = nir_depth_layout_any;
 386       break;
 387    case ir_depth_layout_greater:
 388       var->data.depth_layout = nir_depth_layout_greater;
 389       break;
 390    case ir_depth_layout_less:
 391       var->data.depth_layout = nir_depth_layout_less;
 392       break;
 393    case ir_depth_layout_unchanged:
 394       var->data.depth_layout = nir_depth_layout_unchanged;
 395       break;
 396    default:
 397       unreachable("not reached");
 398    }
 399
 400    var->data.index = ir->data.index;
 401    var->data.binding = ir->data.binding;
 402    var->data.offset = ir->data.offset;
 403    var->data.image.read_only = ir->data.image_read_only;
 404    var->data.image.write_only = ir->data.image_write_only;
 405    var->data.image.coherent = ir->data.image_coherent;
 406    var->data.image._volatile = ir->data.image_volatile;
 407    var->data.image.restrict_flag = ir->data.image_restrict;
 408    var->data.image.format = ir->data.image_format;
 409    var->data.max_array_access = ir->data.max_array_access;
 410    var->data.fb_fetch_output = ir->data.fb_fetch_output;
 411
 412    var->num_state_slots = ir->get_num_state_slots();
 413    if (var->num_state_slots > 0) {
 414       var->state_slots = ralloc_array(var, nir_state_slot,
 415                                       var->num_state_slots);
 416
 417       ir_state_slot *state_slots = ir->get_state_slots();
 418       for (unsigned i = 0; i < var->num_state_slots; i++) {
 419          for (unsigned j = 0; j < 5; j++)
 420             var->state_slots[i].tokens[j] = state_slots[i].tokens[j];
 421          var->state_slots[i].swizzle = state_slots[i].swizzle;
 422       }
 423    } else {
 424       var->state_slots = NULL;
 425    }
 426
 427    var->constant_initializer = constant_copy(ir->constant_initializer, var);
 428
 429    var->interface_type = ir->get_interface_type();
 430
 431    if (var->data.mode == nir_var_local)
 432       nir_function_impl_add_variable(impl, var);
 433    else
 434       nir_shader_add_variable(shader, var);
 435
 436    _mesa_hash_table_insert(var_table, ir, var);
 437    this->var = var;
 438 }
 439
 440 ir_visitor_status
 441 nir_function_visitor::visit_enter(ir_function *ir)
 442 {
 443    foreach_in_list(ir_function_signature, sig, &ir->signatures) {
 444       visitor->create_function(sig);
 445    }
 446    return visit_continue_with_parent;
 447 }
 448
 449 void
 450 nir_visitor::create_function(ir_function_signature *ir)
 451 {
 452    if (ir->is_intrinsic())
 453       return;
 454
 455    nir_function *func = nir_function_create(shader, ir->function_name());
 456
 457    assert(ir->parameters.is_empty());
 458    assert(ir->return_type == glsl_type::void_type);
 459
 460    _mesa_hash_table_insert(this->overload_table, ir, func);
 461 }
 462
 463 void
 464 nir_visitor::visit(ir_function *ir)
 465 {
 466    foreach_in_list(ir_function_signature, sig, &ir->signatures)
 467       sig->accept(this);
 468 }
 469
 470 void
 471 nir_visitor::visit(ir_function_signature *ir)
 472 {
 473    if (ir->is_intrinsic())
 474       return;
 475
 476    struct hash_entry *entry =
 477       _mesa_hash_table_search(this->overload_table, ir);
 478
 479    assert(entry);
 480    nir_function *func = (nir_function *) entry->data;
 481
 482    if (ir->is_defined) {
 483       nir_function_impl *impl = nir_function_impl_create(func);
 484       this->impl = impl;
 485
 486       assert(strcmp(func->name, "main") == 0);
 487       assert(ir->parameters.is_empty());
 488       assert(func->return_type == glsl_type::void_type);
 489
 490       this->is_global = false;
 491
 492       nir_builder_init(&b, impl);
 493       b.cursor = nir_after_cf_list(&impl->body);
 494       visit_exec_list(&ir->body, this);
 495
 496       this->is_global = true;
 497    } else {
 498       func->impl = NULL;
 499    }
 500 }
 501
 502 void
 503 nir_visitor::visit(ir_loop *ir)
 504 {
 505    nir_loop *loop = nir_loop_create(this->shader);
 506    nir_builder_cf_insert(&b, &loop->cf_node);
 507
 508    b.cursor = nir_after_cf_list(&loop->body);
 509    visit_exec_list(&ir->body_instructions, this);
 510    b.cursor = nir_after_cf_node(&loop->cf_node);
 511 }
 512
 513 void
 514 nir_visitor::visit(ir_if *ir)
 515 {
 516    nir_src condition =
 517       nir_src_for_ssa(evaluate_rvalue(ir->condition));
 518
 519    nir_if *if_stmt = nir_if_create(this->shader);
 520    if_stmt->condition = condition;
 521    nir_builder_cf_insert(&b, &if_stmt->cf_node);
 522
 523    b.cursor = nir_after_cf_list(&if_stmt->then_list);
 524    visit_exec_list(&ir->then_instructions, this);
 525
 526    b.cursor = nir_after_cf_list(&if_stmt->else_list);
 527    visit_exec_list(&ir->else_instructions, this);
 528
 529    b.cursor = nir_after_cf_node(&if_stmt->cf_node);
 530 }
 531
 532 void
 533 nir_visitor::visit(ir_discard *ir)
 534 {
 535    /*
 536     * discards aren't treated as control flow, because before we lower them
 537     * they can appear anywhere in the shader and the stuff after them may still
 538     * be executed (yay, crazy GLSL rules!). However, after lowering, all the
 539     * discards will be immediately followed by a return.
 540     */
 541
 542    nir_intrinsic_instr *discard;
 543    if (ir->condition) {
 544       discard = nir_intrinsic_instr_create(this->shader,
 545                                            nir_intrinsic_discard_if);
 546       discard->src[0] =
 547          nir_src_for_ssa(evaluate_rvalue(ir->condition));
 548    } else {
 549       discard = nir_intrinsic_instr_create(this->shader, nir_intrinsic_discard);
 550    }
 551
 552    nir_builder_instr_insert(&b, &discard->instr);
 553 }
 554
 555 void
 556 nir_visitor::visit(ir_emit_vertex *ir)
 557 {
 558    nir_intrinsic_instr *instr =
 559       nir_intrinsic_instr_create(this->shader, nir_intrinsic_emit_vertex);
 560    nir_intrinsic_set_stream_id(instr, ir->stream_id());
 561    nir_builder_instr_insert(&b, &instr->instr);
 562 }
 563
 564 void
 565 nir_visitor::visit(ir_end_primitive *ir)
 566 {
 567    nir_intrinsic_instr *instr =
 568       nir_intrinsic_instr_create(this->shader, nir_intrinsic_end_primitive);
 569    nir_intrinsic_set_stream_id(instr, ir->stream_id());
 570    nir_builder_instr_insert(&b, &instr->instr);
 571 }
 572
 573 void
 574 nir_visitor::visit(ir_loop_jump *ir)
 575 {
 576    nir_jump_type type;
 577    switch (ir->mode) {
 578    case ir_loop_jump::jump_break:
 579       type = nir_jump_break;
 580       break;
 581    case ir_loop_jump::jump_continue:
 582       type = nir_jump_continue;
 583       break;
 584    default:
 585       unreachable("not reached");
 586    }
 587
 588    nir_jump_instr *instr = nir_jump_instr_create(this->shader, type);
 589    nir_builder_instr_insert(&b, &instr->instr);
 590 }
 591
 592 void
 593 nir_visitor::visit(ir_return *ir)
 594 {
 595    if (ir->value != NULL) {
 596       nir_intrinsic_instr *copy =
 597          nir_intrinsic_instr_create(this->shader, nir_intrinsic_copy_var);
 598
 599       copy->variables[0] = nir_deref_var_create(copy, this->impl->return_var);
 600       copy->variables[1] = evaluate_deref(&copy->instr, ir->value);
 601    }
 602
 603    nir_jump_instr *instr = nir_jump_instr_create(this->shader, nir_jump_return);
 604    nir_builder_instr_insert(&b, &instr->instr);
 605 }
 606
 607 void
 608 nir_visitor::visit(ir_call *ir)
 609 {
 610    if (ir->callee->is_intrinsic()) {
 611       nir_intrinsic_op op;
 612
 613       switch (ir->callee->intrinsic_id) {
 614       case ir_intrinsic_atomic_counter_read:
 615          op = nir_intrinsic_atomic_counter_read_var;
 616          break;
 617       case ir_intrinsic_atomic_counter_increment:
 618          op = nir_intrinsic_atomic_counter_inc_var;
 619          break;
 620       case ir_intrinsic_atomic_counter_predecrement:
 621          op = nir_intrinsic_atomic_counter_dec_var;
 622          break;
 623       case ir_intrinsic_atomic_counter_add:
 624          op = nir_intrinsic_atomic_counter_add_var;
 625          break;
 626       case ir_intrinsic_atomic_counter_and:
 627          op = nir_intrinsic_atomic_counter_and_var;
 628          break;
 629       case ir_intrinsic_atomic_counter_or:
 630          op = nir_intrinsic_atomic_counter_or_var;
 631          break;
 632       case ir_intrinsic_atomic_counter_xor:
 633          op = nir_intrinsic_atomic_counter_xor_var;
 634          break;
 635       case ir_intrinsic_atomic_counter_min:
 636          op = nir_intrinsic_atomic_counter_min_var;
 637          break;
 638       case ir_intrinsic_atomic_counter_max:
 639          op = nir_intrinsic_atomic_counter_max_var;
 640          break;
 641       case ir_intrinsic_atomic_counter_exchange:
 642          op = nir_intrinsic_atomic_counter_exchange_var;
 643          break;
 644       case ir_intrinsic_atomic_counter_comp_swap:
 645          op = nir_intrinsic_atomic_counter_comp_swap_var;
 646          break;
 647       case ir_intrinsic_image_load:
 648          op = nir_intrinsic_image_load;
 649          break;
 650       case ir_intrinsic_image_store:
 651          op = nir_intrinsic_image_store;
 652          break;
 653       case ir_intrinsic_image_atomic_add:
 654          op = nir_intrinsic_image_atomic_add;
 655          break;
 656       case ir_intrinsic_image_atomic_min:
 657          op = nir_intrinsic_image_atomic_min;
 658          break;
 659       case ir_intrinsic_image_atomic_max:
 660          op = nir_intrinsic_image_atomic_max;
 661          break;
 662       case ir_intrinsic_image_atomic_and:
 663          op = nir_intrinsic_image_atomic_and;
 664          break;
 665       case ir_intrinsic_image_atomic_or:
 666          op = nir_intrinsic_image_atomic_or;
 667          break;
 668       case ir_intrinsic_image_atomic_xor:
 669          op = nir_intrinsic_image_atomic_xor;
 670          break;
 671       case ir_intrinsic_image_atomic_exchange:
 672          op = nir_intrinsic_image_atomic_exchange;
 673          break;
 674       case ir_intrinsic_image_atomic_comp_swap:
 675          op = nir_intrinsic_image_atomic_comp_swap;
 676          break;
 677       case ir_intrinsic_memory_barrier:
 678          op = nir_intrinsic_memory_barrier;
 679          break;
 680       case ir_intrinsic_image_size:
 681          op = nir_intrinsic_image_size;
 682          break;
 683       case ir_intrinsic_image_samples:
 684          op = nir_intrinsic_image_samples;
 685          break;
 686       case ir_intrinsic_ssbo_store:
 687          op = nir_intrinsic_store_ssbo;
 688          break;
 689       case ir_intrinsic_ssbo_load:
 690          op = nir_intrinsic_load_ssbo;
 691          break;
 692       case ir_intrinsic_ssbo_atomic_add:
 693          op = nir_intrinsic_ssbo_atomic_add;
 694          break;
 695       case ir_intrinsic_ssbo_atomic_and:
 696          op = nir_intrinsic_ssbo_atomic_and;
 697          break;
 698       case ir_intrinsic_ssbo_atomic_or:
 699          op = nir_intrinsic_ssbo_atomic_or;
 700          break;
 701       case ir_intrinsic_ssbo_atomic_xor:
 702          op = nir_intrinsic_ssbo_atomic_xor;
 703          break;
 704       case ir_intrinsic_ssbo_atomic_min:
 705          assert(ir->return_deref);
 706          if (ir->return_deref->type == glsl_type::int_type)
 707             op = nir_intrinsic_ssbo_atomic_imin;
 708          else if (ir->return_deref->type == glsl_type::uint_type)
 709             op = nir_intrinsic_ssbo_atomic_umin;
 710          else
 711             unreachable("Invalid type");
 712          break;
 713       case ir_intrinsic_ssbo_atomic_max:
 714          assert(ir->return_deref);
 715          if (ir->return_deref->type == glsl_type::int_type)
 716             op = nir_intrinsic_ssbo_atomic_imax;
 717          else if (ir->return_deref->type == glsl_type::uint_type)
 718             op = nir_intrinsic_ssbo_atomic_umax;
 719          else
 720             unreachable("Invalid type");
 721          break;
 722       case ir_intrinsic_ssbo_atomic_exchange:
 723          op = nir_intrinsic_ssbo_atomic_exchange;
 724          break;
 725       case ir_intrinsic_ssbo_atomic_comp_swap:
 726          op = nir_intrinsic_ssbo_atomic_comp_swap;
 727          break;
 728       case ir_intrinsic_shader_clock:
 729          op = nir_intrinsic_shader_clock;
 730          break;
 731       case ir_intrinsic_group_memory_barrier:
 732          op = nir_intrinsic_group_memory_barrier;
 733          break;
 734       case ir_intrinsic_memory_barrier_atomic_counter:
 735          op = nir_intrinsic_memory_barrier_atomic_counter;
 736          break;
 737       case ir_intrinsic_memory_barrier_buffer:
 738          op = nir_intrinsic_memory_barrier_buffer;
 739          break;
 740       case ir_intrinsic_memory_barrier_image:
 741          op = nir_intrinsic_memory_barrier_image;
 742          break;
 743       case ir_intrinsic_memory_barrier_shared:
 744          op = nir_intrinsic_memory_barrier_shared;
 745          break;
 746       case ir_intrinsic_shared_load:
 747          op = nir_intrinsic_load_shared;
 748          break;
 749       case ir_intrinsic_shared_store:
 750          op = nir_intrinsic_store_shared;
 751          break;
 752       case ir_intrinsic_shared_atomic_add:
 753          op = nir_intrinsic_shared_atomic_add;
 754          break;
 755       case ir_intrinsic_shared_atomic_and:
 756          op = nir_intrinsic_shared_atomic_and;
 757          break;
 758       case ir_intrinsic_shared_atomic_or:
 759          op = nir_intrinsic_shared_atomic_or;
 760          break;
 761       case ir_intrinsic_shared_atomic_xor:
 762          op = nir_intrinsic_shared_atomic_xor;
 763          break;
 764       case ir_intrinsic_shared_atomic_min:
 765          assert(ir->return_deref);
 766          if (ir->return_deref->type == glsl_type::int_type)
 767             op = nir_intrinsic_shared_atomic_imin;
 768          else if (ir->return_deref->type == glsl_type::uint_type)
 769             op = nir_intrinsic_shared_atomic_umin;
 770          else
 771             unreachable("Invalid type");
 772          break;
 773       case ir_intrinsic_shared_atomic_max:
 774          assert(ir->return_deref);
 775          if (ir->return_deref->type == glsl_type::int_type)
 776             op = nir_intrinsic_shared_atomic_imax;
 777          else if (ir->return_deref->type == glsl_type::uint_type)
 778             op = nir_intrinsic_shared_atomic_umax;
 779          else
 780             unreachable("Invalid type");
 781          break;
 782       case ir_intrinsic_shared_atomic_exchange:
 783          op = nir_intrinsic_shared_atomic_exchange;
 784          break;
 785       case ir_intrinsic_shared_atomic_comp_swap:
 786          op = nir_intrinsic_shared_atomic_comp_swap;
 787          break;
 788       default:
 789          unreachable("not reached");
 790       }
 791
 792       nir_intrinsic_instr *instr = nir_intrinsic_instr_create(shader, op);
 793       nir_dest *dest = &instr->dest;
 794
 795       switch (op) {
 796       case nir_intrinsic_atomic_counter_read_var:
 797       case nir_intrinsic_atomic_counter_inc_var:
 798       case nir_intrinsic_atomic_counter_dec_var:
 799       case nir_intrinsic_atomic_counter_add_var:
 800       case nir_intrinsic_atomic_counter_min_var:
 801       case nir_intrinsic_atomic_counter_max_var:
 802       case nir_intrinsic_atomic_counter_and_var:
 803       case nir_intrinsic_atomic_counter_or_var:
 804       case nir_intrinsic_atomic_counter_xor_var:
 805       case nir_intrinsic_atomic_counter_exchange_var:
 806       case nir_intrinsic_atomic_counter_comp_swap_var: {
 807          /* Set the counter variable dereference. */
 808          exec_node *param = ir->actual_parameters.get_head();
 809          ir_dereference *counter = (ir_dereference *)param;
 810
 811          instr->variables[0] = evaluate_deref(&instr->instr, counter);
 812          param = param->get_next();
 813
 814          /* Set the intrinsic destination. */
 815          if (ir->return_deref) {
 816             nir_ssa_dest_init(&instr->instr, &instr->dest, 1, 32, NULL);
 817          }
 818
 819          /* Set the intrinsic parameters. */
 820          if (!param->is_tail_sentinel()) {
 821             instr->src[0] =
 822                nir_src_for_ssa(evaluate_rvalue((ir_dereference *)param));
 823             param = param->get_next();
 824          }
 825
 826          if (!param->is_tail_sentinel()) {
 827             instr->src[1] =
 828                nir_src_for_ssa(evaluate_rvalue((ir_dereference *)param));
 829             param = param->get_next();
 830          }
 831
 832          nir_builder_instr_insert(&b, &instr->instr);
 833          break;
 834       }
 835       case nir_intrinsic_image_load:
 836       case nir_intrinsic_image_store:
 837       case nir_intrinsic_image_atomic_add:
 838       case nir_intrinsic_image_atomic_min:
 839       case nir_intrinsic_image_atomic_max:
 840       case nir_intrinsic_image_atomic_and:
 841       case nir_intrinsic_image_atomic_or:
 842       case nir_intrinsic_image_atomic_xor:
 843       case nir_intrinsic_image_atomic_exchange:
 844       case nir_intrinsic_image_atomic_comp_swap:
 845       case nir_intrinsic_image_samples:
 846       case nir_intrinsic_image_size: {
 847          nir_ssa_undef_instr *instr_undef =
 848             nir_ssa_undef_instr_create(shader, 1, 32);
 849          nir_builder_instr_insert(&b, &instr_undef->instr);
 850
 851          /* Set the image variable dereference. */
 852          exec_node *param = ir->actual_parameters.get_head();
 853          ir_dereference *image = (ir_dereference *)param;
 854          const glsl_type *type =
 855             image->variable_referenced()->type->without_array();
 856
 857          instr->variables[0] = evaluate_deref(&instr->instr, image);
 858          param = param->get_next();
 859
 860          /* Set the intrinsic destination. */
 861          if (ir->return_deref) {
 862             const nir_intrinsic_info *info =
 863                     &nir_intrinsic_infos[instr->intrinsic];
 864             nir_ssa_dest_init(&instr->instr, &instr->dest,
 865                               info->dest_components, 32, NULL);
 866          }
 867
 868          if (op == nir_intrinsic_image_size ||
 869              op == nir_intrinsic_image_samples) {
 870             nir_builder_instr_insert(&b, &instr->instr);
 871             break;
 872          }
 873
 874          /* Set the address argument, extending the coordinate vector to four
 875           * components.
 876           */
 877          nir_ssa_def *src_addr =
 878             evaluate_rvalue((ir_dereference *)param);
 879          nir_ssa_def *srcs[4];
 880
 881          for (int i = 0; i < 4; i++) {
 882             if (i < type->coordinate_components())
 883                srcs[i] = nir_channel(&b, src_addr, i);
 884             else
 885                srcs[i] = &instr_undef->def;
 886          }
 887
 888          instr->src[0] = nir_src_for_ssa(nir_vec(&b, srcs, 4));
 889          param = param->get_next();
 890
 891          /* Set the sample argument, which is undefined for single-sample
 892           * images.
 893           */
 894          if (type->sampler_dimensionality == GLSL_SAMPLER_DIM_MS) {
 895             instr->src[1] =
 896                nir_src_for_ssa(evaluate_rvalue((ir_dereference *)param));
 897             param = param->get_next();
 898          } else {
 899             instr->src[1] = nir_src_for_ssa(&instr_undef->def);
 900          }
 901
 902          /* Set the intrinsic parameters. */
 903          if (!param->is_tail_sentinel()) {
 904             instr->src[2] =
 905                nir_src_for_ssa(evaluate_rvalue((ir_dereference *)param));
 906             param = param->get_next();
 907          }
 908
 909          if (!param->is_tail_sentinel()) {
 910             instr->src[3] =
 911                nir_src_for_ssa(evaluate_rvalue((ir_dereference *)param));
 912             param = param->get_next();
 913          }
 914          nir_builder_instr_insert(&b, &instr->instr);
 915          break;
 916       }
 917       case nir_intrinsic_memory_barrier:
 918       case nir_intrinsic_group_memory_barrier:
 919       case nir_intrinsic_memory_barrier_atomic_counter:
 920       case nir_intrinsic_memory_barrier_buffer:
 921       case nir_intrinsic_memory_barrier_image:
 922       case nir_intrinsic_memory_barrier_shared:
 923          nir_builder_instr_insert(&b, &instr->instr);
 924          break;
 925       case nir_intrinsic_shader_clock:
 926          nir_ssa_dest_init(&instr->instr, &instr->dest, 1, 32, NULL);
 927          nir_builder_instr_insert(&b, &instr->instr);
 928          break;
 929       case nir_intrinsic_store_ssbo: {
 930          exec_node *param = ir->actual_parameters.get_head();
 931          ir_rvalue *block = ((ir_instruction *)param)->as_rvalue();
 932
 933          param = param->get_next();
 934          ir_rvalue *offset = ((ir_instruction *)param)->as_rvalue();
 935
 936          param = param->get_next();
 937          ir_rvalue *val = ((ir_instruction *)param)->as_rvalue();
 938
 939          param = param->get_next();
 940          ir_constant *write_mask = ((ir_instruction *)param)->as_constant();
 941          assert(write_mask);
 942
 943          instr->src[0] = nir_src_for_ssa(evaluate_rvalue(val));
 944          instr->src[1] = nir_src_for_ssa(evaluate_rvalue(block));
 945          instr->src[2] = nir_src_for_ssa(evaluate_rvalue(offset));
 946          nir_intrinsic_set_write_mask(instr, write_mask->value.u[0]);
 947          instr->num_components = val->type->vector_elements;
 948
 949          nir_builder_instr_insert(&b, &instr->instr);
 950          break;
 951       }
 952       case nir_intrinsic_load_ssbo: {
 953          exec_node *param = ir->actual_parameters.get_head();
 954          ir_rvalue *block = ((ir_instruction *)param)->as_rvalue();
 955
 956          param = param->get_next();
 957          ir_rvalue *offset = ((ir_instruction *)param)->as_rvalue();
 958
 959          instr->src[0] = nir_src_for_ssa(evaluate_rvalue(block));
 960          instr->src[1] = nir_src_for_ssa(evaluate_rvalue(offset));
 961
 962          const glsl_type *type = ir->return_deref->var->type;
 963          instr->num_components = type->vector_elements;
 964
 965          /* Setup destination register */
 966          unsigned bit_size = glsl_get_bit_size(type);
 967          nir_ssa_dest_init(&instr->instr, &instr->dest,
 968                            type->vector_elements, bit_size, NULL);
 969
 970          /* Insert the created nir instruction now since in the case of boolean
 971           * result we will need to emit another instruction after it
 972           */
 973          nir_builder_instr_insert(&b, &instr->instr);
 974
 975          /*
 976           * In SSBO/UBO's, a true boolean value is any non-zero value, but we
 977           * consider a true boolean to be ~0. Fix this up with a != 0
 978           * comparison.
 979           */
 980          if (type->base_type == GLSL_TYPE_BOOL) {
 981             nir_alu_instr *load_ssbo_compare =
 982                nir_alu_instr_create(shader, nir_op_ine);
 983             load_ssbo_compare->src[0].src.is_ssa = true;
 984             load_ssbo_compare->src[0].src.ssa = &instr->dest.ssa;
 985             load_ssbo_compare->src[1].src =
 986                nir_src_for_ssa(nir_imm_int(&b, 0));
 987             for (unsigned i = 0; i < type->vector_elements; i++)
 988                load_ssbo_compare->src[1].swizzle[i] = 0;
 989             nir_ssa_dest_init(&load_ssbo_compare->instr,
 990                               &load_ssbo_compare->dest.dest,
 991                               type->vector_elements, bit_size, NULL);
 992             load_ssbo_compare->dest.write_mask = (1 << type->vector_elements) - 1;
 993             nir_builder_instr_insert(&b, &load_ssbo_compare->instr);
 994             dest = &load_ssbo_compare->dest.dest;
 995          }
 996          break;
 997       }
 998       case nir_intrinsic_ssbo_atomic_add:
 999       case nir_intrinsic_ssbo_atomic_imin:
1000       case nir_intrinsic_ssbo_atomic_umin:
1001       case nir_intrinsic_ssbo_atomic_imax:
1002       case nir_intrinsic_ssbo_atomic_umax:
1003       case nir_intrinsic_ssbo_atomic_and:
1004       case nir_intrinsic_ssbo_atomic_or:
1005       case nir_intrinsic_ssbo_atomic_xor:
1006       case nir_intrinsic_ssbo_atomic_exchange:
1007       case nir_intrinsic_ssbo_atomic_comp_swap: {
1008          int param_count = ir->actual_parameters.length();
1009          assert(param_count == 3 || param_count == 4);
1010
1011          /* Block index */
1012          exec_node *param = ir->actual_parameters.get_head();
1013          ir_instruction *inst = (ir_instruction *) param;
1014          instr->src[0] = nir_src_for_ssa(evaluate_rvalue(inst->as_rvalue()));
1015
1016          /* Offset */
1017          param = param->get_next();
1018          inst = (ir_instruction *) param;
1019          instr->src[1] = nir_src_for_ssa(evaluate_rvalue(inst->as_rvalue()));
1020
1021          /* data1 parameter (this is always present) */
1022          param = param->get_next();
1023          inst = (ir_instruction *) param;
1024          instr->src[2] = nir_src_for_ssa(evaluate_rvalue(inst->as_rvalue()));
1025
1026          /* data2 parameter (only with atomic_comp_swap) */
1027          if (param_count == 4) {
1028             assert(op == nir_intrinsic_ssbo_atomic_comp_swap);
1029             param = param->get_next();
1030             inst = (ir_instruction *) param;
1031             instr->src[3] = nir_src_for_ssa(evaluate_rvalue(inst->as_rvalue()));
1032          }
1033
1034          /* Atomic result */
1035          assert(ir->return_deref);
1036          nir_ssa_dest_init(&instr->instr, &instr->dest,
1037                            ir->return_deref->type->vector_elements, 32, NULL);
1038          nir_builder_instr_insert(&b, &instr->instr);
1039          break;
1040       }
1041       case nir_intrinsic_load_shared: {
1042          exec_node *param = ir->actual_parameters.get_head();
1043          ir_rvalue *offset = ((ir_instruction *)param)->as_rvalue();
1044
1045          nir_intrinsic_set_base(instr, 0);
1046          instr->src[0] = nir_src_for_ssa(evaluate_rvalue(offset));
1047
1048          const glsl_type *type = ir->return_deref->var->type;
1049          instr->num_components = type->vector_elements;
1050
1051          /* Setup destination register */
1052          unsigned bit_size = glsl_get_bit_size(type);
1053          nir_ssa_dest_init(&instr->instr, &instr->dest,
1054                            type->vector_elements, bit_size, NULL);
1055
1056          nir_builder_instr_insert(&b, &instr->instr);
1057          break;
1058       }
1059       case nir_intrinsic_store_shared: {
1060          exec_node *param = ir->actual_parameters.get_head();
1061          ir_rvalue *offset = ((ir_instruction *)param)->as_rvalue();
1062
1063          param = param->get_next();
1064          ir_rvalue *val = ((ir_instruction *)param)->as_rvalue();
1065
1066          param = param->get_next();
1067          ir_constant *write_mask = ((ir_instruction *)param)->as_constant();
1068          assert(write_mask);
1069
1070          nir_intrinsic_set_base(instr, 0);
1071          instr->src[1] = nir_src_for_ssa(evaluate_rvalue(offset));
1072
1073          nir_intrinsic_set_write_mask(instr, write_mask->value.u[0]);
1074
1075          instr->src[0] = nir_src_for_ssa(evaluate_rvalue(val));
1076          instr->num_components = val->type->vector_elements;
1077
1078          nir_builder_instr_insert(&b, &instr->instr);
1079          break;
1080       }
1081       case nir_intrinsic_shared_atomic_add:
1082       case nir_intrinsic_shared_atomic_imin:
1083       case nir_intrinsic_shared_atomic_umin:
1084       case nir_intrinsic_shared_atomic_imax:
1085       case nir_intrinsic_shared_atomic_umax:
1086       case nir_intrinsic_shared_atomic_and:
1087       case nir_intrinsic_shared_atomic_or:
1088       case nir_intrinsic_shared_atomic_xor:
1089       case nir_intrinsic_shared_atomic_exchange:
1090       case nir_intrinsic_shared_atomic_comp_swap: {
1091          int param_count = ir->actual_parameters.length();
1092          assert(param_count == 2 || param_count == 3);
1093
1094          /* Offset */
1095          exec_node *param = ir->actual_parameters.get_head();
1096          ir_instruction *inst = (ir_instruction *) param;
1097          instr->src[0] = nir_src_for_ssa(evaluate_rvalue(inst->as_rvalue()));
1098
1099          /* data1 parameter (this is always present) */
1100          param = param->get_next();
1101          inst = (ir_instruction *) param;
1102          instr->src[1] = nir_src_for_ssa(evaluate_rvalue(inst->as_rvalue()));
1103
1104          /* data2 parameter (only with atomic_comp_swap) */
1105          if (param_count == 3) {
1106             assert(op == nir_intrinsic_shared_atomic_comp_swap);
1107             param = param->get_next();
1108             inst = (ir_instruction *) param;
1109             instr->src[2] =
1110                nir_src_for_ssa(evaluate_rvalue(inst->as_rvalue()));
1111          }
1112
1113          /* Atomic result */
1114          assert(ir->return_deref);
1115          unsigned bit_size = glsl_get_bit_size(ir->return_deref->type);
1116          nir_ssa_dest_init(&instr->instr, &instr->dest,
1117                            ir->return_deref->type->vector_elements,
1118                            bit_size, NULL);
1119          nir_builder_instr_insert(&b, &instr->instr);
1120          break;
1121       }
1122       default:
1123          unreachable("not reached");
1124       }
1125
1126       if (ir->return_deref) {
1127          nir_intrinsic_instr *store_instr =
1128             nir_intrinsic_instr_create(shader, nir_intrinsic_store_var);
1129          store_instr->num_components = ir->return_deref->type->vector_elements;
1130          nir_intrinsic_set_write_mask(store_instr,
1131                                       (1 << store_instr->num_components) - 1);
1132
1133          store_instr->variables[0] =
1134             evaluate_deref(&store_instr->instr, ir->return_deref);
1135          store_instr->src[0] = nir_src_for_ssa(&dest->ssa);
1136
1137          nir_builder_instr_insert(&b, &store_instr->instr);
1138       }
1139
1140       return;
1141    }
1142
1143    struct hash_entry *entry =
1144       _mesa_hash_table_search(this->overload_table, ir->callee);
1145    assert(entry);
1146    nir_function *callee = (nir_function *) entry->data;
1147
1148    nir_call_instr *instr = nir_call_instr_create(this->shader, callee);
1149
1150    unsigned i = 0;
1151    foreach_in_list(ir_dereference, param, &ir->actual_parameters) {
1152       instr->params[i] = evaluate_deref(&instr->instr, param);
1153       i++;
1154    }
1155
1156    instr->return_deref = evaluate_deref(&instr->instr, ir->return_deref);
1157    nir_builder_instr_insert(&b, &instr->instr);
1158 }
1159
1160 void
1161 nir_visitor::visit(ir_assignment *ir)
1162 {
1163    unsigned num_components = ir->lhs->type->vector_elements;
1164
1165    b.exact = ir->lhs->variable_referenced()->data.invariant ||
1166              ir->lhs->variable_referenced()->data.precise;
1167
1168    if ((ir->rhs->as_dereference() || ir->rhs->as_constant()) &&
1169        (ir->write_mask == (1 << num_components) - 1 || ir->write_mask == 0)) {
1170       /* We're doing a plain-as-can-be copy, so emit a copy_var */
1171       nir_intrinsic_instr *copy =
1172          nir_intrinsic_instr_create(this->shader, nir_intrinsic_copy_var);
1173
1174       copy->variables[0] = evaluate_deref(&copy->instr, ir->lhs);
1175       copy->variables[1] = evaluate_deref(&copy->instr, ir->rhs);
1176
1177       if (ir->condition) {
1178          nir_if *if_stmt = nir_if_create(this->shader);
1179          if_stmt->condition = nir_src_for_ssa(evaluate_rvalue(ir->condition));
1180          nir_builder_cf_insert(&b, &if_stmt->cf_node);
1181          nir_instr_insert_after_cf_list(&if_stmt->then_list, &copy->instr);
1182          b.cursor = nir_after_cf_node(&if_stmt->cf_node);
1183       } else {
1184          nir_builder_instr_insert(&b, &copy->instr);
1185       }
1186       return;
1187    }
1188
1189    assert(ir->rhs->type->is_scalar() || ir->rhs->type->is_vector());
1190
1191    ir->lhs->accept(this);
1192    nir_deref_var *lhs_deref = this->deref_head;
1193    nir_ssa_def *src = evaluate_rvalue(ir->rhs);
1194
1195    if (ir->write_mask != (1 << num_components) - 1 && ir->write_mask != 0) {
1196       /* GLSL IR will give us the input to the write-masked assignment in a
1197        * single packed vector.  So, for example, if the writemask is xzw, then
1198        * we have to swizzle x -> x, y -> z, and z -> w and get the y component
1199        * from the load.
1200        */
1201       unsigned swiz[4];
1202       unsigned component = 0;
1203       for (unsigned i = 0; i < 4; i++) {
1204          swiz[i] = ir->write_mask & (1 << i) ? component++ : 0;
1205       }
1206       src = nir_swizzle(&b, src, swiz, num_components, !supports_ints);
1207    }
1208
1209    nir_intrinsic_instr *store =
1210       nir_intrinsic_instr_create(this->shader, nir_intrinsic_store_var);
1211    store->num_components = ir->lhs->type->vector_elements;
1212    nir_intrinsic_set_write_mask(store, ir->write_mask);
1213    nir_deref *store_deref = nir_copy_deref(store, &lhs_deref->deref);
1214    store->variables[0] = nir_deref_as_var(store_deref);
1215    store->src[0] = nir_src_for_ssa(src);
1216
1217    if (ir->condition) {
1218       nir_if *if_stmt = nir_if_create(this->shader);
1219       if_stmt->condition = nir_src_for_ssa(evaluate_rvalue(ir->condition));
1220       nir_builder_cf_insert(&b, &if_stmt->cf_node);
1221       nir_instr_insert_after_cf_list(&if_stmt->then_list, &store->instr);
1222       b.cursor = nir_after_cf_node(&if_stmt->cf_node);
1223    } else {
1224       nir_builder_instr_insert(&b, &store->instr);
1225    }
1226 }
1227
1228 /*
1229  * Given an instruction, returns a pointer to its destination or NULL if there
1230  * is no destination.
1231  *
1232  * Note that this only handles instructions we generate at this level.
1233  */
1234 static nir_dest *
1235 get_instr_dest(nir_instr *instr)
1236 {
1237    nir_alu_instr *alu_instr;
1238    nir_intrinsic_instr *intrinsic_instr;
1239    nir_tex_instr *tex_instr;
1240
1241    switch (instr->type) {
1242       case nir_instr_type_alu:
1243          alu_instr = nir_instr_as_alu(instr);
1244          return &alu_instr->dest.dest;
1245
1246       case nir_instr_type_intrinsic:
1247          intrinsic_instr = nir_instr_as_intrinsic(instr);
1248          if (nir_intrinsic_infos[intrinsic_instr->intrinsic].has_dest)
1249             return &intrinsic_instr->dest;
1250          else
1251             return NULL;
1252
1253       case nir_instr_type_tex:
1254          tex_instr = nir_instr_as_tex(instr);
1255          return &tex_instr->dest;
1256
1257       default:
1258          unreachable("not reached");
1259    }
1260
1261    return NULL;
1262 }
1263
1264 void
1265 nir_visitor::add_instr(nir_instr *instr, unsigned num_components,
1266                        unsigned bit_size)
1267 {
1268    nir_dest *dest = get_instr_dest(instr);
1269
1270    if (dest)
1271       nir_ssa_dest_init(instr, dest, num_components, bit_size, NULL);
1272
1273    nir_builder_instr_insert(&b, instr);
1274
1275    if (dest) {
1276       assert(dest->is_ssa);
1277       this->result = &dest->ssa;
1278    }
1279 }
1280
1281 nir_ssa_def *
1282 nir_visitor::evaluate_rvalue(ir_rvalue* ir)
1283 {
1284    ir->accept(this);
1285    if (ir->as_dereference() || ir->as_constant()) {
1286       /*
1287        * A dereference is being used on the right hand side, which means we
1288        * must emit a variable load.
1289        */
1290
1291       nir_intrinsic_instr *load_instr =
1292          nir_intrinsic_instr_create(this->shader, nir_intrinsic_load_var);
1293       load_instr->num_components = ir->type->vector_elements;
1294       load_instr->variables[0] = this->deref_head;
1295       ralloc_steal(load_instr, load_instr->variables[0]);
1296       unsigned bit_size = glsl_get_bit_size(ir->type);
1297       add_instr(&load_instr->instr, ir->type->vector_elements, bit_size);
1298    }
1299
1300    return this->result;
1301 }
1302
1303 static bool
1304 type_is_float(glsl_base_type type)
1305 {
1306    return type == GLSL_TYPE_FLOAT || type == GLSL_TYPE_DOUBLE;
1307 }
1308
1309 void
1310 nir_visitor::visit(ir_expression *ir)
1311 {
1312    /* Some special cases */
1313    switch (ir->operation) {
1314    case ir_binop_ubo_load: {
1315       nir_intrinsic_instr *load =
1316          nir_intrinsic_instr_create(this->shader, nir_intrinsic_load_ubo);
1317       unsigned bit_size = glsl_get_bit_size(ir->type);
1318       load->num_components = ir->type->vector_elements;
1319       load->src[0] = nir_src_for_ssa(evaluate_rvalue(ir->operands[0]));
1320       load->src[1] = nir_src_for_ssa(evaluate_rvalue(ir->operands[1]));
1321       add_instr(&load->instr, ir->type->vector_elements, bit_size);
1322
1323       /*
1324        * In UBO's, a true boolean value is any non-zero value, but we consider
1325        * a true boolean to be ~0. Fix this up with a != 0 comparison.
1326        */
1327
1328       if (ir->type->base_type == GLSL_TYPE_BOOL)
1329          this->result = nir_ine(&b, &load->dest.ssa, nir_imm_int(&b, 0));
1330
1331       return;
1332    }
1333
1334    case ir_unop_interpolate_at_centroid:
1335    case ir_binop_interpolate_at_offset:
1336    case ir_binop_interpolate_at_sample: {
1337       ir_dereference *deref = ir->operands[0]->as_dereference();
1338       ir_swizzle *swizzle = NULL;
1339       if (!deref) {
1340          /* the api does not allow a swizzle here, but the varying packing code
1341           * may have pushed one into here.
1342           */
1343          swizzle = ir->operands[0]->as_swizzle();
1344          assert(swizzle);
1345          deref = swizzle->val->as_dereference();
1346          assert(deref);
1347       }
1348
1349       deref->accept(this);
1350
1351       nir_intrinsic_op op;
1352       if (this->deref_head->var->data.mode == nir_var_shader_in) {
1353          switch (ir->operation) {
1354          case ir_unop_interpolate_at_centroid:
1355             op = nir_intrinsic_interp_var_at_centroid;
1356             break;
1357          case ir_binop_interpolate_at_offset:
1358             op = nir_intrinsic_interp_var_at_offset;
1359             break;
1360          case ir_binop_interpolate_at_sample:
1361             op = nir_intrinsic_interp_var_at_sample;
1362             break;
1363          default:
1364             unreachable("Invalid interpolation intrinsic");
1365          }
1366       } else {
1367          /* This case can happen if the vertex shader does not write the
1368           * given varying.  In this case, the linker will lower it to a
1369           * global variable.  Since interpolating a variable makes no
1370           * sense, we'll just turn it into a load which will probably
1371           * eventually end up as an SSA definition.
1372           */
1373          assert(this->deref_head->var->data.mode == nir_var_global);
1374          op = nir_intrinsic_load_var;
1375       }
1376
1377       nir_intrinsic_instr *intrin = nir_intrinsic_instr_create(shader, op);
1378       intrin->num_components = deref->type->vector_elements;
1379       intrin->variables[0] = this->deref_head;
1380       ralloc_steal(intrin, intrin->variables[0]);
1381
1382       if (intrin->intrinsic == nir_intrinsic_interp_var_at_offset ||
1383           intrin->intrinsic == nir_intrinsic_interp_var_at_sample)
1384          intrin->src[0] = nir_src_for_ssa(evaluate_rvalue(ir->operands[1]));
1385
1386       unsigned bit_size =  glsl_get_bit_size(deref->type);
1387       add_instr(&intrin->instr, deref->type->vector_elements, bit_size);
1388
1389       if (swizzle) {
1390          unsigned swiz[4] = {
1391             swizzle->mask.x, swizzle->mask.y, swizzle->mask.z, swizzle->mask.w
1392          };
1393
1394          result = nir_swizzle(&b, result, swiz,
1395                               swizzle->type->vector_elements, false);
1396       }
1397
1398       return;
1399    }
1400
1401    default:
1402       break;
1403    }
1404
1405    nir_ssa_def *srcs[4];
1406    for (unsigned i = 0; i < ir->get_num_operands(); i++)
1407       srcs[i] = evaluate_rvalue(ir->operands[i]);
1408
1409    glsl_base_type types[4];
1410    for (unsigned i = 0; i < ir->get_num_operands(); i++)
1411       if (supports_ints)
1412          types[i] = ir->operands[i]->type->base_type;
1413       else
1414          types[i] = GLSL_TYPE_FLOAT;
1415
1416    glsl_base_type out_type;
1417    if (supports_ints)
1418       out_type = ir->type->base_type;
1419    else
1420       out_type = GLSL_TYPE_FLOAT;
1421
1422    switch (ir->operation) {
1423    case ir_unop_bit_not: result = nir_inot(&b, srcs[0]); break;
1424    case ir_unop_logic_not:
1425       result = supports_ints ? nir_inot(&b, srcs[0]) : nir_fnot(&b, srcs[0]);
1426       break;
1427    case ir_unop_neg:
1428       result = type_is_float(types[0]) ? nir_fneg(&b, srcs[0])
1429                                        : nir_ineg(&b, srcs[0]);
1430       break;
1431    case ir_unop_abs:
1432       result = type_is_float(types[0]) ? nir_fabs(&b, srcs[0])
1433                                        : nir_iabs(&b, srcs[0]);
1434       break;
1435    case ir_unop_saturate:
1436       assert(type_is_float(types[0]));
1437       result = nir_fsat(&b, srcs[0]);
1438       break;
1439    case ir_unop_sign:
1440       result = type_is_float(types[0]) ? nir_fsign(&b, srcs[0])
1441                                        : nir_isign(&b, srcs[0]);
1442       break;
1443    case ir_unop_rcp:  result = nir_frcp(&b, srcs[0]);  break;
1444    case ir_unop_rsq:  result = nir_frsq(&b, srcs[0]);  break;
1445    case ir_unop_sqrt: result = nir_fsqrt(&b, srcs[0]); break;
1446    case ir_unop_exp:  unreachable("ir_unop_exp should have been lowered");
1447    case ir_unop_log:  unreachable("ir_unop_log should have been lowered");
1448    case ir_unop_exp2: result = nir_fexp2(&b, srcs[0]); break;
1449    case ir_unop_log2: result = nir_flog2(&b, srcs[0]); break;
1450    case ir_unop_i2f:
1451       result = supports_ints ? nir_i2f(&b, srcs[0]) : nir_fmov(&b, srcs[0]);
1452       break;
1453    case ir_unop_u2f:
1454       result = supports_ints ? nir_u2f(&b, srcs[0]) : nir_fmov(&b, srcs[0]);
1455       break;
1456    case ir_unop_b2f:
1457       result = supports_ints ? nir_b2f(&b, srcs[0]) : nir_fmov(&b, srcs[0]);
1458       break;
1459    case ir_unop_f2i:  result = nir_f2i(&b, srcs[0]);   break;
1460    case ir_unop_f2u:  result = nir_f2u(&b, srcs[0]);   break;
1461    case ir_unop_f2b:  result = nir_f2b(&b, srcs[0]);   break;
1462    case ir_unop_i2b:  result = nir_i2b(&b, srcs[0]);   break;
1463    case ir_unop_b2i:  result = nir_b2i(&b, srcs[0]);   break;
1464    case ir_unop_d2f:  result = nir_d2f(&b, srcs[0]);   break;
1465    case ir_unop_f2d:  result = nir_f2d(&b, srcs[0]);   break;
1466    case ir_unop_d2i:  result = nir_d2i(&b, srcs[0]);   break;
1467    case ir_unop_d2u:  result = nir_d2u(&b, srcs[0]);   break;
1468    case ir_unop_d2b:  result = nir_d2b(&b, srcs[0]);   break;
1469    case ir_unop_i2d:
1470       assert(supports_ints);
1471       result = nir_i2d(&b, srcs[0]);
1472       break;
1473    case ir_unop_u2d:
1474       assert(supports_ints);
1475       result = nir_u2d(&b, srcs[0]);
1476       break;
1477    case ir_unop_i2u:
1478    case ir_unop_u2i:
1479    case ir_unop_bitcast_i2f:
1480    case ir_unop_bitcast_f2i:
1481    case ir_unop_bitcast_u2f:
1482    case ir_unop_bitcast_f2u:
1483    case ir_unop_subroutine_to_int:
1484       /* no-op */
1485       result = nir_imov(&b, srcs[0]);
1486       break;
1487    case ir_unop_trunc: result = nir_ftrunc(&b, srcs[0]); break;
1488    case ir_unop_ceil:  result = nir_fceil(&b, srcs[0]); break;
1489    case ir_unop_floor: result = nir_ffloor(&b, srcs[0]); break;
1490    case ir_unop_fract: result = nir_ffract(&b, srcs[0]); break;
1491    case ir_unop_round_even: result = nir_fround_even(&b, srcs[0]); break;
1492    case ir_unop_sin:   result = nir_fsin(&b, srcs[0]); break;
1493    case ir_unop_cos:   result = nir_fcos(&b, srcs[0]); break;
1494    case ir_unop_dFdx:        result = nir_fddx(&b, srcs[0]); break;
1495    case ir_unop_dFdy:        result = nir_fddy(&b, srcs[0]); break;
1496    case ir_unop_dFdx_fine:   result = nir_fddx_fine(&b, srcs[0]); break;
1497    case ir_unop_dFdy_fine:   result = nir_fddy_fine(&b, srcs[0]); break;
1498    case ir_unop_dFdx_coarse: result = nir_fddx_coarse(&b, srcs[0]); break;
1499    case ir_unop_dFdy_coarse: result = nir_fddy_coarse(&b, srcs[0]); break;
1500    case ir_unop_pack_snorm_2x16:
1501       result = nir_pack_snorm_2x16(&b, srcs[0]);
1502       break;
1503    case ir_unop_pack_snorm_4x8:
1504       result = nir_pack_snorm_4x8(&b, srcs[0]);
1505       break;
1506    case ir_unop_pack_unorm_2x16:
1507       result = nir_pack_unorm_2x16(&b, srcs[0]);
1508       break;
1509    case ir_unop_pack_unorm_4x8:
1510       result = nir_pack_unorm_4x8(&b, srcs[0]);
1511       break;
1512    case ir_unop_pack_half_2x16:
1513       result = nir_pack_half_2x16(&b, srcs[0]);
1514       break;
1515    case ir_unop_unpack_snorm_2x16:
1516       result = nir_unpack_snorm_2x16(&b, srcs[0]);
1517       break;
1518    case ir_unop_unpack_snorm_4x8:
1519       result = nir_unpack_snorm_4x8(&b, srcs[0]);
1520       break;
1521    case ir_unop_unpack_unorm_2x16:
1522       result = nir_unpack_unorm_2x16(&b, srcs[0]);
1523       break;
1524    case ir_unop_unpack_unorm_4x8:
1525       result = nir_unpack_unorm_4x8(&b, srcs[0]);
1526       break;
1527    case ir_unop_unpack_half_2x16:
1528       result = nir_unpack_half_2x16(&b, srcs[0]);
1529       break;
1530    case ir_unop_pack_double_2x32:
1531       result = nir_pack_double_2x32(&b, srcs[0]);
1532       break;
1533    case ir_unop_unpack_double_2x32:
1534       result = nir_unpack_double_2x32(&b, srcs[0]);
1535       break;
1536    case ir_unop_bitfield_reverse:
1537       result = nir_bitfield_reverse(&b, srcs[0]);
1538       break;
1539    case ir_unop_bit_count:
1540       result = nir_bit_count(&b, srcs[0]);
1541       break;
1542    case ir_unop_find_msb:
1543       switch (types[0]) {
1544       case GLSL_TYPE_UINT:
1545          result = nir_ufind_msb(&b, srcs[0]);
1546          break;
1547       case GLSL_TYPE_INT:
1548          result = nir_ifind_msb(&b, srcs[0]);
1549          break;
1550       default:
1551          unreachable("Invalid type for findMSB()");
1552       }
1553       break;
1554    case ir_unop_find_lsb:
1555       result = nir_find_lsb(&b, srcs[0]);
1556       break;
1557
1558    case ir_unop_noise:
1559       switch (ir->type->vector_elements) {
1560       case 1:
1561          switch (ir->operands[0]->type->vector_elements) {
1562             case 1: result = nir_fnoise1_1(&b, srcs[0]); break;
1563             case 2: result = nir_fnoise1_2(&b, srcs[0]); break;
1564             case 3: result = nir_fnoise1_3(&b, srcs[0]); break;
1565             case 4: result = nir_fnoise1_4(&b, srcs[0]); break;
1566             default: unreachable("not reached");
1567          }
1568          break;
1569       case 2:
1570          switch (ir->operands[0]->type->vector_elements) {
1571             case 1: result = nir_fnoise2_1(&b, srcs[0]); break;
1572             case 2: result = nir_fnoise2_2(&b, srcs[0]); break;
1573             case 3: result = nir_fnoise2_3(&b, srcs[0]); break;
1574             case 4: result = nir_fnoise2_4(&b, srcs[0]); break;
1575             default: unreachable("not reached");
1576          }
1577          break;
1578       case 3:
1579          switch (ir->operands[0]->type->vector_elements) {
1580             case 1: result = nir_fnoise3_1(&b, srcs[0]); break;
1581             case 2: result = nir_fnoise3_2(&b, srcs[0]); break;
1582             case 3: result = nir_fnoise3_3(&b, srcs[0]); break;
1583             case 4: result = nir_fnoise3_4(&b, srcs[0]); break;
1584             default: unreachable("not reached");
1585          }
1586          break;
1587       case 4:
1588          switch (ir->operands[0]->type->vector_elements) {
1589             case 1: result = nir_fnoise4_1(&b, srcs[0]); break;
1590             case 2: result = nir_fnoise4_2(&b, srcs[0]); break;
1591             case 3: result = nir_fnoise4_3(&b, srcs[0]); break;
1592             case 4: result = nir_fnoise4_4(&b, srcs[0]); break;
1593             default: unreachable("not reached");
1594          }
1595          break;
1596       default:
1597          unreachable("not reached");
1598       }
1599       break;
1600    case ir_unop_get_buffer_size: {
1601       nir_intrinsic_instr *load = nir_intrinsic_instr_create(
1602          this->shader,
1603          nir_intrinsic_get_buffer_size);
1604       load->num_components = ir->type->vector_elements;
1605       load->src[0] = nir_src_for_ssa(evaluate_rvalue(ir->operands[0]));
1606       unsigned bit_size = glsl_get_bit_size(ir->type);
1607       add_instr(&load->instr, ir->type->vector_elements, bit_size);
1608       return;
1609    }
1610
1611    case ir_binop_add:
1612       result = type_is_float(out_type) ? nir_fadd(&b, srcs[0], srcs[1])
1613                                        : nir_iadd(&b, srcs[0], srcs[1]);
1614       break;
1615    case ir_binop_sub:
1616       result = type_is_float(out_type) ? nir_fsub(&b, srcs[0], srcs[1])
1617                                        : nir_isub(&b, srcs[0], srcs[1]);
1618       break;
1619    case ir_binop_mul:
1620       result = type_is_float(out_type) ? nir_fmul(&b, srcs[0], srcs[1])
1621                                        : nir_imul(&b, srcs[0], srcs[1]);
1622       break;
1623    case ir_binop_div:
1624       if (type_is_float(out_type))
1625          result = nir_fdiv(&b, srcs[0], srcs[1]);
1626       else if (out_type == GLSL_TYPE_INT)
1627          result = nir_idiv(&b, srcs[0], srcs[1]);
1628       else
1629          result = nir_udiv(&b, srcs[0], srcs[1]);
1630       break;
1631    case ir_binop_mod:
1632       result = type_is_float(out_type) ? nir_fmod(&b, srcs[0], srcs[1])
1633                                        : nir_umod(&b, srcs[0], srcs[1]);
1634       break;
1635    case ir_binop_min:
1636       if (type_is_float(out_type))
1637          result = nir_fmin(&b, srcs[0], srcs[1]);
1638       else if (out_type == GLSL_TYPE_INT)
1639          result = nir_imin(&b, srcs[0], srcs[1]);
1640       else
1641          result = nir_umin(&b, srcs[0], srcs[1]);
1642       break;
1643    case ir_binop_max:
1644       if (type_is_float(out_type))
1645          result = nir_fmax(&b, srcs[0], srcs[1]);
1646       else if (out_type == GLSL_TYPE_INT)
1647          result = nir_imax(&b, srcs[0], srcs[1]);
1648       else
1649          result = nir_umax(&b, srcs[0], srcs[1]);
1650       break;
1651    case ir_binop_pow: result = nir_fpow(&b, srcs[0], srcs[1]); break;
1652    case ir_binop_bit_and: result = nir_iand(&b, srcs[0], srcs[1]); break;
1653    case ir_binop_bit_or: result = nir_ior(&b, srcs[0], srcs[1]); break;
1654    case ir_binop_bit_xor: result = nir_ixor(&b, srcs[0], srcs[1]); break;
1655    case ir_binop_logic_and:
1656       result = supports_ints ? nir_iand(&b, srcs[0], srcs[1])
1657                              : nir_fand(&b, srcs[0], srcs[1]);
1658       break;
1659    case ir_binop_logic_or:
1660       result = supports_ints ? nir_ior(&b, srcs[0], srcs[1])
1661                              : nir_for(&b, srcs[0], srcs[1]);
1662       break;
1663    case ir_binop_logic_xor:
1664       result = supports_ints ? nir_ixor(&b, srcs[0], srcs[1])
1665                              : nir_fxor(&b, srcs[0], srcs[1]);
1666       break;
1667    case ir_binop_lshift: result = nir_ishl(&b, srcs[0], srcs[1]); break;
1668    case ir_binop_rshift:
1669       result = (out_type == GLSL_TYPE_INT) ? nir_ishr(&b, srcs[0], srcs[1])
1670                                            : nir_ushr(&b, srcs[0], srcs[1]);
1671       break;
1672    case ir_binop_imul_high:
1673       result = (out_type == GLSL_TYPE_INT) ? nir_imul_high(&b, srcs[0], srcs[1])
1674                                            : nir_umul_high(&b, srcs[0], srcs[1]);
1675       break;
1676    case ir_binop_carry:  result = nir_uadd_carry(&b, srcs[0], srcs[1]);  break;
1677    case ir_binop_borrow: result = nir_usub_borrow(&b, srcs[0], srcs[1]); break;
1678    case ir_binop_less:
1679       if (supports_ints) {
1680          if (type_is_float(types[0]))
1681             result = nir_flt(&b, srcs[0], srcs[1]);
1682          else if (types[0] == GLSL_TYPE_INT)
1683             result = nir_ilt(&b, srcs[0], srcs[1]);
1684          else
1685             result = nir_ult(&b, srcs[0], srcs[1]);
1686       } else {
1687          result = nir_slt(&b, srcs[0], srcs[1]);
1688       }
1689       break;
1690    case ir_binop_greater:
1691       if (supports_ints) {
1692          if (type_is_float(types[0]))
1693             result = nir_flt(&b, srcs[1], srcs[0]);
1694          else if (types[0] == GLSL_TYPE_INT)
1695             result = nir_ilt(&b, srcs[1], srcs[0]);
1696          else
1697             result = nir_ult(&b, srcs[1], srcs[0]);
1698       } else {
1699          result = nir_slt(&b, srcs[1], srcs[0]);
1700       }
1701       break;
1702    case ir_binop_lequal:
1703       if (supports_ints) {
1704          if (type_is_float(types[0]))
1705             result = nir_fge(&b, srcs[1], srcs[0]);
1706          else if (types[0] == GLSL_TYPE_INT)
1707             result = nir_ige(&b, srcs[1], srcs[0]);
1708          else
1709             result = nir_uge(&b, srcs[1], srcs[0]);
1710       } else {
1711          result = nir_slt(&b, srcs[1], srcs[0]);
1712       }
1713       break;
1714    case ir_binop_gequal:
1715       if (supports_ints) {
1716          if (type_is_float(types[0]))
1717             result = nir_fge(&b, srcs[0], srcs[1]);
1718          else if (types[0] == GLSL_TYPE_INT)
1719             result = nir_ige(&b, srcs[0], srcs[1]);
1720          else
1721             result = nir_uge(&b, srcs[0], srcs[1]);
1722       } else {
1723          result = nir_slt(&b, srcs[0], srcs[1]);
1724       }
1725       break;
1726    case ir_binop_equal:
1727       if (supports_ints) {
1728          if (type_is_float(types[0]))
1729             result = nir_feq(&b, srcs[0], srcs[1]);
1730          else
1731             result = nir_ieq(&b, srcs[0], srcs[1]);
1732       } else {
1733          result = nir_seq(&b, srcs[0], srcs[1]);
1734       }
1735       break;
1736    case ir_binop_nequal:
1737       if (supports_ints) {
1738          if (type_is_float(types[0]))
1739             result = nir_fne(&b, srcs[0], srcs[1]);
1740          else
1741             result = nir_ine(&b, srcs[0], srcs[1]);
1742       } else {
1743          result = nir_sne(&b, srcs[0], srcs[1]);
1744       }
1745       break;
1746    case ir_binop_all_equal:
1747       if (supports_ints) {
1748          if (type_is_float(types[0])) {
1749             switch (ir->operands[0]->type->vector_elements) {
1750                case 1: result = nir_feq(&b, srcs[0], srcs[1]); break;
1751                case 2: result = nir_ball_fequal2(&b, srcs[0], srcs[1]); break;
1752                case 3: result = nir_ball_fequal3(&b, srcs[0], srcs[1]); break;
1753                case 4: result = nir_ball_fequal4(&b, srcs[0], srcs[1]); break;
1754                default:
1755                   unreachable("not reached");
1756             }
1757          } else {
1758             switch (ir->operands[0]->type->vector_elements) {
1759                case 1: result = nir_ieq(&b, srcs[0], srcs[1]); break;
1760                case 2: result = nir_ball_iequal2(&b, srcs[0], srcs[1]); break;
1761                case 3: result = nir_ball_iequal3(&b, srcs[0], srcs[1]); break;
1762                case 4: result = nir_ball_iequal4(&b, srcs[0], srcs[1]); break;
1763                default:
1764                   unreachable("not reached");
1765             }
1766          }
1767       } else {
1768          switch (ir->operands[0]->type->vector_elements) {
1769             case 1: result = nir_seq(&b, srcs[0], srcs[1]); break;
1770             case 2: result = nir_fall_equal2(&b, srcs[0], srcs[1]); break;
1771             case 3: result = nir_fall_equal3(&b, srcs[0], srcs[1]); break;
1772             case 4: result = nir_fall_equal4(&b, srcs[0], srcs[1]); break;
1773             default:
1774                unreachable("not reached");
1775          }
1776       }
1777       break;
1778    case ir_binop_any_nequal:
1779       if (supports_ints) {
1780          if (type_is_float(types[0])) {
1781             switch (ir->operands[0]->type->vector_elements) {
1782                case 1: result = nir_fne(&b, srcs[0], srcs[1]); break;
1783                case 2: result = nir_bany_fnequal2(&b, srcs[0], srcs[1]); break;
1784                case 3: result = nir_bany_fnequal3(&b, srcs[0], srcs[1]); break;
1785                case 4: result = nir_bany_fnequal4(&b, srcs[0], srcs[1]); break;
1786                default:
1787                   unreachable("not reached");
1788             }
1789          } else {
1790             switch (ir->operands[0]->type->vector_elements) {
1791                case 1: result = nir_ine(&b, srcs[0], srcs[1]); break;
1792                case 2: result = nir_bany_inequal2(&b, srcs[0], srcs[1]); break;
1793                case 3: result = nir_bany_inequal3(&b, srcs[0], srcs[1]); break;
1794                case 4: result = nir_bany_inequal4(&b, srcs[0], srcs[1]); break;
1795                default:
1796                   unreachable("not reached");
1797             }
1798          }
1799       } else {
1800          switch (ir->operands[0]->type->vector_elements) {
1801             case 1: result = nir_sne(&b, srcs[0], srcs[1]); break;
1802             case 2: result = nir_fany_nequal2(&b, srcs[0], srcs[1]); break;
1803             case 3: result = nir_fany_nequal3(&b, srcs[0], srcs[1]); break;
1804             case 4: result = nir_fany_nequal4(&b, srcs[0], srcs[1]); break;
1805             default:
1806                unreachable("not reached");
1807          }
1808       }
1809       break;
1810    case ir_binop_dot:
1811       switch (ir->operands[0]->type->vector_elements) {
1812          case 2: result = nir_fdot2(&b, srcs[0], srcs[1]); break;
1813          case 3: result = nir_fdot3(&b, srcs[0], srcs[1]); break;
1814          case 4: result = nir_fdot4(&b, srcs[0], srcs[1]); break;
1815          default:
1816             unreachable("not reached");
1817       }
1818       break;
1819
1820    case ir_binop_ldexp: result = nir_ldexp(&b, srcs[0], srcs[1]); break;
1821    case ir_triop_fma:
1822       result = nir_ffma(&b, srcs[0], srcs[1], srcs[2]);
1823       break;
1824    case ir_triop_lrp:
1825       result = nir_flrp(&b, srcs[0], srcs[1], srcs[2]);
1826       break;
1827    case ir_triop_csel:
1828       if (supports_ints)
1829          result = nir_bcsel(&b, srcs[0], srcs[1], srcs[2]);
1830       else
1831          result = nir_fcsel(&b, srcs[0], srcs[1], srcs[2]);
1832       break;
1833    case ir_triop_bitfield_extract:
1834       result = (out_type == GLSL_TYPE_INT) ?
1835          nir_ibitfield_extract(&b, srcs[0], srcs[1], srcs[2]) :
1836          nir_ubitfield_extract(&b, srcs[0], srcs[1], srcs[2]);
1837       break;
1838    case ir_quadop_bitfield_insert:
1839       result = nir_bitfield_insert(&b, srcs[0], srcs[1], srcs[2], srcs[3]);
1840       break;
1841    case ir_quadop_vector:
1842       result = nir_vec(&b, srcs, ir->type->vector_elements);
1843       break;
1844
1845    default:
1846       unreachable("not reached");
1847    }
1848 }
1849
1850 void
1851 nir_visitor::visit(ir_swizzle *ir)
1852 {
1853    unsigned swizzle[4] = { ir->mask.x, ir->mask.y, ir->mask.z, ir->mask.w };
1854    result = nir_swizzle(&b, evaluate_rvalue(ir->val), swizzle,
1855                         ir->type->vector_elements, !supports_ints);
1856 }
1857
1858 void
1859 nir_visitor::visit(ir_texture *ir)
1860 {
1861    unsigned num_srcs;
1862    nir_texop op;
1863    switch (ir->op) {
1864    case ir_tex:
1865       op = nir_texop_tex;
1866       num_srcs = 1; /* coordinate */
1867       break;
1868
1869    case ir_txb:
1870    case ir_txl:
1871       op = (ir->op == ir_txb) ? nir_texop_txb : nir_texop_txl;
1872       num_srcs = 2; /* coordinate, bias/lod */
1873       break;
1874
1875    case ir_txd:
1876       op = nir_texop_txd; /* coordinate, dPdx, dPdy */
1877       num_srcs = 3;
1878       break;
1879
1880    case ir_txf:
1881       op = nir_texop_txf;
1882       if (ir->lod_info.lod != NULL)
1883          num_srcs = 2; /* coordinate, lod */
1884       else
1885          num_srcs = 1; /* coordinate */
1886       break;
1887
1888    case ir_txf_ms:
1889       op = nir_texop_txf_ms;
1890       num_srcs = 2; /* coordinate, sample_index */
1891       break;
1892
1893    case ir_txs:
1894       op = nir_texop_txs;
1895       if (ir->lod_info.lod != NULL)
1896          num_srcs = 1; /* lod */
1897       else
1898          num_srcs = 0;
1899       break;
1900
1901    case ir_lod:
1902       op = nir_texop_lod;
1903       num_srcs = 1; /* coordinate */
1904       break;
1905
1906    case ir_tg4:
1907       op = nir_texop_tg4;
1908       num_srcs = 1; /* coordinate */
1909       break;
1910
1911    case ir_query_levels:
1912       op = nir_texop_query_levels;
1913       num_srcs = 0;
1914       break;
1915
1916    case ir_texture_samples:
1917       op = nir_texop_texture_samples;
1918       num_srcs = 0;
1919       break;
1920
1921    case ir_samples_identical:
1922       op = nir_texop_samples_identical;
1923       num_srcs = 1; /* coordinate */
1924       break;
1925
1926    default:
1927       unreachable("not reached");
1928    }
1929
1930    if (ir->projector != NULL)
1931       num_srcs++;
1932    if (ir->shadow_comparitor != NULL)
1933       num_srcs++;
1934    if (ir->offset != NULL)
1935       num_srcs++;
1936
1937    nir_tex_instr *instr = nir_tex_instr_create(this->shader, num_srcs);
1938
1939    instr->op = op;
1940    instr->sampler_dim =
1941       (glsl_sampler_dim) ir->sampler->type->sampler_dimensionality;
1942    instr->is_array = ir->sampler->type->sampler_array;
1943    instr->is_shadow = ir->sampler->type->sampler_shadow;
1944    if (instr->is_shadow)
1945       instr->is_new_style_shadow = (ir->type->vector_elements == 1);
1946    switch (ir->type->base_type) {
1947    case GLSL_TYPE_FLOAT:
1948       instr->dest_type = nir_type_float;
1949       break;
1950    case GLSL_TYPE_INT:
1951       instr->dest_type = nir_type_int;
1952       break;
1953    case GLSL_TYPE_BOOL:
1954    case GLSL_TYPE_UINT:
1955       instr->dest_type = nir_type_uint;
1956       break;
1957    default:
1958       unreachable("not reached");
1959    }
1960
1961    instr->texture = evaluate_deref(&instr->instr, ir->sampler);
1962
1963    unsigned src_number = 0;
1964
1965    if (ir->coordinate != NULL) {
1966       instr->coord_components = ir->coordinate->type->vector_elements;
1967       instr->src[src_number].src =
1968          nir_src_for_ssa(evaluate_rvalue(ir->coordinate));
1969       instr->src[src_number].src_type = nir_tex_src_coord;
1970       src_number++;
1971    }
1972
1973    if (ir->projector != NULL) {
1974       instr->src[src_number].src =
1975          nir_src_for_ssa(evaluate_rvalue(ir->projector));
1976       instr->src[src_number].src_type = nir_tex_src_projector;
1977       src_number++;
1978    }
1979
1980    if (ir->shadow_comparitor != NULL) {
1981       instr->src[src_number].src =
1982          nir_src_for_ssa(evaluate_rvalue(ir->shadow_comparitor));
1983       instr->src[src_number].src_type = nir_tex_src_comparitor;
1984       src_number++;
1985    }
1986
1987    if (ir->offset != NULL) {
1988       /* we don't support multiple offsets yet */
1989       assert(ir->offset->type->is_vector() || ir->offset->type->is_scalar());
1990
1991       instr->src[src_number].src =
1992          nir_src_for_ssa(evaluate_rvalue(ir->offset));
1993       instr->src[src_number].src_type = nir_tex_src_offset;
1994       src_number++;
1995    }
1996
1997    switch (ir->op) {
1998    case ir_txb:
1999       instr->src[src_number].src =
2000          nir_src_for_ssa(evaluate_rvalue(ir->lod_info.bias));
2001       instr->src[src_number].src_type = nir_tex_src_bias;
2002       src_number++;
2003       break;
2004
2005    case ir_txl:
2006    case ir_txf:
2007    case ir_txs:
2008       if (ir->lod_info.lod != NULL) {
2009          instr->src[src_number].src =
2010             nir_src_for_ssa(evaluate_rvalue(ir->lod_info.lod));
2011          instr->src[src_number].src_type = nir_tex_src_lod;
2012          src_number++;
2013       }
2014       break;
2015
2016    case ir_txd:
2017       instr->src[src_number].src =
2018          nir_src_for_ssa(evaluate_rvalue(ir->lod_info.grad.dPdx));
2019       instr->src[src_number].src_type = nir_tex_src_ddx;
2020       src_number++;
2021       instr->src[src_number].src =
2022          nir_src_for_ssa(evaluate_rvalue(ir->lod_info.grad.dPdy));
2023       instr->src[src_number].src_type = nir_tex_src_ddy;
2024       src_number++;
2025       break;
2026
2027    case ir_txf_ms:
2028       instr->src[src_number].src =
2029          nir_src_for_ssa(evaluate_rvalue(ir->lod_info.sample_index));
2030       instr->src[src_number].src_type = nir_tex_src_ms_index;
2031       src_number++;
2032       break;
2033
2034    case ir_tg4:
2035       instr->component = ir->lod_info.component->as_constant()->value.u[0];
2036       break;
2037
2038    default:
2039       break;
2040    }
2041
2042    assert(src_number == num_srcs);
2043
2044    unsigned bit_size = glsl_get_bit_size(ir->type);
2045    add_instr(&instr->instr, nir_tex_instr_dest_size(instr), bit_size);
2046 }
2047
2048 void
2049 nir_visitor::visit(ir_constant *ir)
2050 {
2051    /*
2052     * We don't know if this variable is an array or struct that gets
2053     * dereferenced, so do the safe thing an make it a variable with a
2054     * constant initializer and return a dereference.
2055     */
2056
2057    nir_variable *var =
2058       nir_local_variable_create(this->impl, ir->type, "const_temp");
2059    var->data.read_only = true;
2060    var->constant_initializer = constant_copy(ir, var);
2061
2062    this->deref_head = nir_deref_var_create(this->shader, var);
2063    this->deref_tail = &this->deref_head->deref;
2064 }
2065
2066 void
2067 nir_visitor::visit(ir_dereference_variable *ir)
2068 {
2069    struct hash_entry *entry =
2070       _mesa_hash_table_search(this->var_table, ir->var);
2071    assert(entry);
2072    nir_variable *var = (nir_variable *) entry->data;
2073
2074    nir_deref_var *deref = nir_deref_var_create(this->shader, var);
2075    this->deref_head = deref;
2076    this->deref_tail = &deref->deref;
2077 }
2078
2079 void
2080 nir_visitor::visit(ir_dereference_record *ir)
2081 {
2082    ir->record->accept(this);
2083
2084    int field_index = this->deref_tail->type->field_index(ir->field);
2085    assert(field_index >= 0);
2086
2087    nir_deref_struct *deref = nir_deref_struct_create(this->deref_tail, field_index);
2088    deref->deref.type = ir->type;
2089    this->deref_tail->child = &deref->deref;
2090    this->deref_tail = &deref->deref;
2091 }
2092
2093 void
2094 nir_visitor::visit(ir_dereference_array *ir)
2095 {
2096    nir_deref_array *deref = nir_deref_array_create(this->shader);
2097    deref->deref.type = ir->type;
2098
2099    ir_constant *const_index = ir->array_index->as_constant();
2100    if (const_index != NULL) {
2101       deref->deref_array_type = nir_deref_array_type_direct;
2102       deref->base_offset = const_index->value.u[0];
2103    } else {
2104       deref->deref_array_type = nir_deref_array_type_indirect;
2105       deref->indirect =
2106          nir_src_for_ssa(evaluate_rvalue(ir->array_index));
2107    }
2108
2109    ir->array->accept(this);
2110
2111    this->deref_tail->child = &deref->deref;
2112    ralloc_steal(this->deref_tail, deref);
2113    this->deref_tail = &deref->deref;
2114 }
2115
2116 void
2117 nir_visitor::visit(ir_barrier *)
2118 {
2119    nir_intrinsic_instr *instr =
2120       nir_intrinsic_instr_create(this->shader, nir_intrinsic_barrier);
2121    nir_builder_instr_insert(&b, &instr->instr);
2122 }