src/compiler/nir/glsl_to_nir.cpp

   1 /*
   2  * Copyright © 2014 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Connor Abbott (cwabbott0@gmail.com)
  25  *
  26  */
  27
  28 #include "glsl_to_nir.h"
  29 #include "nir_control_flow.h"
  30 #include "nir_builder.h"
  31 #include "compiler/glsl/ir_visitor.h"
  32 #include "compiler/glsl/ir_hierarchical_visitor.h"
  33 #include "compiler/glsl/ir.h"
  34 #include "main/imports.h"
  35
  36 /*
  37  * pass to lower GLSL IR to NIR
  38  *
  39  * This will lower variable dereferences to loads/stores of corresponding
  40  * variables in NIR - the variables will be converted to registers in a later
  41  * pass.
  42  */
  43
  44 namespace {
  45
  46 class nir_visitor : public ir_visitor
  47 {
  48 public:
  49    nir_visitor(nir_shader *shader, gl_shader *sh);
  50    ~nir_visitor();
  51
  52    virtual void visit(ir_variable *);
  53    virtual void visit(ir_function *);
  54    virtual void visit(ir_function_signature *);
  55    virtual void visit(ir_loop *);
  56    virtual void visit(ir_if *);
  57    virtual void visit(ir_discard *);
  58    virtual void visit(ir_loop_jump *);
  59    virtual void visit(ir_return *);
  60    virtual void visit(ir_call *);
  61    virtual void visit(ir_assignment *);
  62    virtual void visit(ir_emit_vertex *);
  63    virtual void visit(ir_end_primitive *);
  64    virtual void visit(ir_expression *);
  65    virtual void visit(ir_swizzle *);
  66    virtual void visit(ir_texture *);
  67    virtual void visit(ir_constant *);
  68    virtual void visit(ir_dereference_variable *);
  69    virtual void visit(ir_dereference_record *);
  70    virtual void visit(ir_dereference_array *);
  71    virtual void visit(ir_barrier *);
  72
  73    void create_function(ir_function_signature *ir);
  74
  75 private:
  76    void add_instr(nir_instr *instr, unsigned num_components);
  77    nir_ssa_def *evaluate_rvalue(ir_rvalue *ir);
  78
  79    nir_alu_instr *emit(nir_op op, unsigned dest_size, nir_ssa_def **srcs);
  80    nir_alu_instr *emit(nir_op op, unsigned dest_size, nir_ssa_def *src1);
  81    nir_alu_instr *emit(nir_op op, unsigned dest_size, nir_ssa_def *src1,
  82                        nir_ssa_def *src2);
  83    nir_alu_instr *emit(nir_op op, unsigned dest_size, nir_ssa_def *src1,
  84                        nir_ssa_def *src2, nir_ssa_def *src3);
  85
  86    bool supports_ints;
  87
  88    struct gl_shader *sh;
  89
  90    nir_shader *shader;
  91    nir_function_impl *impl;
  92    nir_builder b;
  93    nir_ssa_def *result; /* result of the expression tree last visited */
  94
  95    nir_deref_var *evaluate_deref(nir_instr *mem_ctx, ir_instruction *ir);
  96
  97    /* the head of the dereference chain we're creating */
  98    nir_deref_var *deref_head;
  99    /* the tail of the dereference chain we're creating */
 100    nir_deref *deref_tail;
 101
 102    nir_variable *var; /* variable created by ir_variable visitor */
 103
 104    /* whether the IR we're operating on is per-function or global */
 105    bool is_global;
 106
 107    /* map of ir_variable -> nir_variable */
 108    struct hash_table *var_table;
 109
 110    /* map of ir_function_signature -> nir_function_overload */
 111    struct hash_table *overload_table;
 112 };
 113
 114 /*
 115  * This visitor runs before the main visitor, calling create_function() for
 116  * each function so that the main visitor can resolve forward references in
 117  * calls.
 118  */
 119
 120 class nir_function_visitor : public ir_hierarchical_visitor
 121 {
 122 public:
 123    nir_function_visitor(nir_visitor *v) : visitor(v)
 124    {
 125    }
 126    virtual ir_visitor_status visit_enter(ir_function *);
 127
 128 private:
 129    nir_visitor *visitor;
 130 };
 131
 132 }; /* end of anonymous namespace */
 133
 134 nir_shader *
 135 glsl_to_nir(const struct gl_shader_program *shader_prog,
 136             gl_shader_stage stage,
 137             const nir_shader_compiler_options *options)
 138 {
 139    struct gl_shader *sh = shader_prog->_LinkedShaders[stage];
 140
 141    nir_shader *shader = nir_shader_create(NULL, stage, options);
 142
 143    nir_visitor v1(shader, sh);
 144    nir_function_visitor v2(&v1);
 145    v2.run(sh->ir);
 146    visit_exec_list(sh->ir, &v1);
 147
 148    nir_function *main = NULL;
 149    nir_foreach_function(shader, func) {
 150       if (strcmp(func->name, "main") == 0) {
 151          main = func;
 152          break;
 153       }
 154    }
 155    assert(main);
 156
 157    nir_lower_outputs_to_temporaries(shader, main);
 158
 159    shader->info.name = ralloc_asprintf(shader, "GLSL%d", shader_prog->Name);
 160    if (shader_prog->Label)
 161       shader->info.label = ralloc_strdup(shader, shader_prog->Label);
 162    shader->info.num_textures = _mesa_fls(sh->Program->SamplersUsed);
 163    shader->info.num_ubos = sh->NumUniformBlocks;
 164    shader->info.num_abos = shader_prog->NumAtomicBuffers;
 165    shader->info.num_ssbos = sh->NumShaderStorageBlocks;
 166    shader->info.num_images = sh->NumImages;
 167    shader->info.inputs_read = sh->Program->InputsRead;
 168    shader->info.outputs_written = sh->Program->OutputsWritten;
 169    shader->info.patch_inputs_read = sh->Program->PatchInputsRead;
 170    shader->info.patch_outputs_written = sh->Program->PatchOutputsWritten;
 171    shader->info.system_values_read = sh->Program->SystemValuesRead;
 172    shader->info.uses_texture_gather = sh->Program->UsesGather;
 173    shader->info.uses_clip_distance_out =
 174       sh->Program->ClipDistanceArraySize != 0;
 175    shader->info.separate_shader = shader_prog->SeparateShader;
 176    shader->info.has_transform_feedback_varyings =
 177       shader_prog->TransformFeedback.NumVarying > 0;
 178
 179    switch (stage) {
 180    case MESA_SHADER_TESS_CTRL:
 181       shader->info.tcs.vertices_out = shader_prog->TessCtrl.VerticesOut;
 182       break;
 183
 184    case MESA_SHADER_GEOMETRY:
 185       shader->info.gs.vertices_in = shader_prog->Geom.VerticesIn;
 186       shader->info.gs.output_primitive = sh->Geom.OutputType;
 187       shader->info.gs.vertices_out = sh->Geom.VerticesOut;
 188       shader->info.gs.invocations = sh->Geom.Invocations;
 189       shader->info.gs.uses_end_primitive = shader_prog->Geom.UsesEndPrimitive;
 190       shader->info.gs.uses_streams = shader_prog->Geom.UsesStreams;
 191       break;
 192
 193    case MESA_SHADER_FRAGMENT: {
 194       struct gl_fragment_program *fp =
 195          (struct gl_fragment_program *)sh->Program;
 196
 197       shader->info.fs.uses_discard = fp->UsesKill;
 198       shader->info.fs.early_fragment_tests = sh->EarlyFragmentTests;
 199       shader->info.fs.depth_layout = fp->FragDepthLayout;
 200       break;
 201    }
 202
 203    case MESA_SHADER_COMPUTE: {
 204       struct gl_compute_program *cp = (struct gl_compute_program *)sh->Program;
 205       shader->info.cs.local_size[0] = cp->LocalSize[0];
 206       shader->info.cs.local_size[1] = cp->LocalSize[1];
 207       shader->info.cs.local_size[2] = cp->LocalSize[2];
 208       break;
 209    }
 210
 211    default:
 212       break; /* No stage-specific info */
 213    }
 214
 215    return shader;
 216 }
 217
 218 nir_visitor::nir_visitor(nir_shader *shader, gl_shader *sh)
 219 {
 220    this->supports_ints = shader->options->native_integers;
 221    this->shader = shader;
 222    this->sh = sh;
 223    this->is_global = true;
 224    this->var_table = _mesa_hash_table_create(NULL, _mesa_hash_pointer,
 225                                              _mesa_key_pointer_equal);
 226    this->overload_table = _mesa_hash_table_create(NULL, _mesa_hash_pointer,
 227                                                   _mesa_key_pointer_equal);
 228 }
 229
 230 nir_visitor::~nir_visitor()
 231 {
 232    _mesa_hash_table_destroy(this->var_table, NULL);
 233    _mesa_hash_table_destroy(this->overload_table, NULL);
 234 }
 235
 236 nir_deref_var *
 237 nir_visitor::evaluate_deref(nir_instr *mem_ctx, ir_instruction *ir)
 238 {
 239    ir->accept(this);
 240    ralloc_steal(mem_ctx, this->deref_head);
 241    return this->deref_head;
 242 }
 243
 244 static nir_constant *
 245 constant_copy(ir_constant *ir, void *mem_ctx)
 246 {
 247    if (ir == NULL)
 248       return NULL;
 249
 250    nir_constant *ret = ralloc(mem_ctx, nir_constant);
 251
 252    unsigned total_elems = ir->type->components();
 253    unsigned i;
 254
 255    ret->num_elements = 0;
 256    switch (ir->type->base_type) {
 257    case GLSL_TYPE_UINT:
 258       for (i = 0; i < total_elems; i++)
 259          ret->value.u[i] = ir->value.u[i];
 260       break;
 261
 262    case GLSL_TYPE_INT:
 263       for (i = 0; i < total_elems; i++)
 264          ret->value.i[i] = ir->value.i[i];
 265       break;
 266
 267    case GLSL_TYPE_FLOAT:
 268       for (i = 0; i < total_elems; i++)
 269          ret->value.f[i] = ir->value.f[i];
 270       break;
 271
 272    case GLSL_TYPE_BOOL:
 273       for (i = 0; i < total_elems; i++)
 274          ret->value.b[i] = ir->value.b[i];
 275       break;
 276
 277    case GLSL_TYPE_STRUCT:
 278       ret->elements = ralloc_array(mem_ctx, nir_constant *,
 279                                    ir->type->length);
 280       ret->num_elements = ir->type->length;
 281
 282       i = 0;
 283       foreach_in_list(ir_constant, field, &ir->components) {
 284          ret->elements[i] = constant_copy(field, mem_ctx);
 285          i++;
 286       }
 287       break;
 288
 289    case GLSL_TYPE_ARRAY:
 290       ret->elements = ralloc_array(mem_ctx, nir_constant *,
 291                                    ir->type->length);
 292       ret->num_elements = ir->type->length;
 293
 294       for (i = 0; i < ir->type->length; i++)
 295          ret->elements[i] = constant_copy(ir->array_elements[i], mem_ctx);
 296       break;
 297
 298    default:
 299       unreachable("not reached");
 300    }
 301
 302    return ret;
 303 }
 304
 305 void
 306 nir_visitor::visit(ir_variable *ir)
 307 {
 308    nir_variable *var = ralloc(shader, nir_variable);
 309    var->type = ir->type;
 310    var->name = ralloc_strdup(var, ir->name);
 311
 312    var->data.read_only = ir->data.read_only;
 313    var->data.centroid = ir->data.centroid;
 314    var->data.sample = ir->data.sample;
 315    var->data.patch = ir->data.patch;
 316    var->data.invariant = ir->data.invariant;
 317    var->data.location = ir->data.location;
 318
 319    switch(ir->data.mode) {
 320    case ir_var_auto:
 321    case ir_var_temporary:
 322       if (is_global)
 323          var->data.mode = nir_var_global;
 324       else
 325          var->data.mode = nir_var_local;
 326       break;
 327
 328    case ir_var_function_in:
 329    case ir_var_function_out:
 330    case ir_var_function_inout:
 331    case ir_var_const_in:
 332       var->data.mode = nir_var_local;
 333       break;
 334
 335    case ir_var_shader_in:
 336       if (shader->stage == MESA_SHADER_FRAGMENT &&
 337           ir->data.location == VARYING_SLOT_FACE) {
 338          /* For whatever reason, GLSL IR makes gl_FrontFacing an input */
 339          var->data.location = SYSTEM_VALUE_FRONT_FACE;
 340          var->data.mode = nir_var_system_value;
 341       } else if (shader->stage == MESA_SHADER_GEOMETRY &&
 342                  ir->data.location == VARYING_SLOT_PRIMITIVE_ID) {
 343          /* For whatever reason, GLSL IR makes gl_PrimitiveIDIn an input */
 344          var->data.location = SYSTEM_VALUE_PRIMITIVE_ID;
 345          var->data.mode = nir_var_system_value;
 346       } else {
 347          var->data.mode = nir_var_shader_in;
 348       }
 349       break;
 350
 351    case ir_var_shader_out:
 352       var->data.mode = nir_var_shader_out;
 353       break;
 354
 355    case ir_var_uniform:
 356       var->data.mode = nir_var_uniform;
 357       break;
 358
 359    case ir_var_shader_storage:
 360       var->data.mode = nir_var_shader_storage;
 361       break;
 362
 363    case ir_var_system_value:
 364       var->data.mode = nir_var_system_value;
 365       break;
 366
 367    default:
 368       unreachable("not reached");
 369    }
 370
 371    var->data.interpolation = ir->data.interpolation;
 372    var->data.origin_upper_left = ir->data.origin_upper_left;
 373    var->data.pixel_center_integer = ir->data.pixel_center_integer;
 374    var->data.explicit_location = ir->data.explicit_location;
 375    var->data.explicit_index = ir->data.explicit_index;
 376    var->data.explicit_binding = ir->data.explicit_binding;
 377    var->data.has_initializer = ir->data.has_initializer;
 378    var->data.location_frac = ir->data.location_frac;
 379
 380    switch (ir->data.depth_layout) {
 381    case ir_depth_layout_none:
 382       var->data.depth_layout = nir_depth_layout_none;
 383       break;
 384    case ir_depth_layout_any:
 385       var->data.depth_layout = nir_depth_layout_any;
 386       break;
 387    case ir_depth_layout_greater:
 388       var->data.depth_layout = nir_depth_layout_greater;
 389       break;
 390    case ir_depth_layout_less:
 391       var->data.depth_layout = nir_depth_layout_less;
 392       break;
 393    case ir_depth_layout_unchanged:
 394       var->data.depth_layout = nir_depth_layout_unchanged;
 395       break;
 396    default:
 397       unreachable("not reached");
 398    }
 399
 400    var->data.index = ir->data.index;
 401    var->data.descriptor_set = 0;
 402    var->data.binding = ir->data.binding;
 403    var->data.offset = ir->data.offset;
 404    var->data.image.read_only = ir->data.image_read_only;
 405    var->data.image.write_only = ir->data.image_write_only;
 406    var->data.image.coherent = ir->data.image_coherent;
 407    var->data.image._volatile = ir->data.image_volatile;
 408    var->data.image.restrict_flag = ir->data.image_restrict;
 409    var->data.image.format = ir->data.image_format;
 410    var->data.max_array_access = ir->data.max_array_access;
 411
 412    var->num_state_slots = ir->get_num_state_slots();
 413    if (var->num_state_slots > 0) {
 414       var->state_slots = ralloc_array(var, nir_state_slot,
 415                                       var->num_state_slots);
 416
 417       ir_state_slot *state_slots = ir->get_state_slots();
 418       for (unsigned i = 0; i < var->num_state_slots; i++) {
 419          for (unsigned j = 0; j < 5; j++)
 420             var->state_slots[i].tokens[j] = state_slots[i].tokens[j];
 421          var->state_slots[i].swizzle = state_slots[i].swizzle;
 422       }
 423    } else {
 424       var->state_slots = NULL;
 425    }
 426
 427    var->constant_initializer = constant_copy(ir->constant_initializer, var);
 428
 429    var->interface_type = ir->get_interface_type();
 430
 431    if (var->data.mode == nir_var_local)
 432       nir_function_impl_add_variable(impl, var);
 433    else
 434       nir_shader_add_variable(shader, var);
 435
 436    _mesa_hash_table_insert(var_table, ir, var);
 437    this->var = var;
 438 }
 439
 440 ir_visitor_status
 441 nir_function_visitor::visit_enter(ir_function *ir)
 442 {
 443    foreach_in_list(ir_function_signature, sig, &ir->signatures) {
 444       visitor->create_function(sig);
 445    }
 446    return visit_continue_with_parent;
 447 }
 448
 449 void
 450 nir_visitor::create_function(ir_function_signature *ir)
 451 {
 452    if (ir->is_intrinsic)
 453       return;
 454
 455    nir_function *func = nir_function_create(shader, ir->function_name());
 456
 457    unsigned num_params = ir->parameters.length();
 458    func->num_params = num_params;
 459    func->params = ralloc_array(shader, nir_parameter, num_params);
 460
 461    unsigned i = 0;
 462    foreach_in_list(ir_variable, param, &ir->parameters) {
 463       switch (param->data.mode) {
 464       case ir_var_function_in:
 465          func->params[i].param_type = nir_parameter_in;
 466          break;
 467
 468       case ir_var_function_out:
 469          func->params[i].param_type = nir_parameter_out;
 470          break;
 471
 472       case ir_var_function_inout:
 473          func->params[i].param_type = nir_parameter_inout;
 474          break;
 475
 476       default:
 477          unreachable("not reached");
 478       }
 479
 480       func->params[i].type = param->type;
 481       i++;
 482    }
 483
 484    func->return_type = ir->return_type;
 485
 486    _mesa_hash_table_insert(this->overload_table, ir, func);
 487 }
 488
 489 void
 490 nir_visitor::visit(ir_function *ir)
 491 {
 492    foreach_in_list(ir_function_signature, sig, &ir->signatures)
 493       sig->accept(this);
 494 }
 495
 496 void
 497 nir_visitor::visit(ir_function_signature *ir)
 498 {
 499    if (ir->is_intrinsic)
 500       return;
 501
 502    struct hash_entry *entry =
 503       _mesa_hash_table_search(this->overload_table, ir);
 504
 505    assert(entry);
 506    nir_function *func = (nir_function *) entry->data;
 507
 508    if (ir->is_defined) {
 509       nir_function_impl *impl = nir_function_impl_create(func);
 510       this->impl = impl;
 511
 512       unsigned num_params = func->num_params;
 513       impl->num_params = num_params;
 514       impl->params = ralloc_array(this->shader, nir_variable *, num_params);
 515       unsigned i = 0;
 516       foreach_in_list(ir_variable, param, &ir->parameters) {
 517          param->accept(this);
 518          impl->params[i] = this->var;
 519          i++;
 520       }
 521
 522       if (func->return_type == glsl_type::void_type) {
 523          impl->return_var = NULL;
 524       } else {
 525          impl->return_var = ralloc(this->shader, nir_variable);
 526          impl->return_var->name = ralloc_strdup(impl->return_var,
 527                                                 "return_var");
 528          impl->return_var->type = func->return_type;
 529       }
 530
 531       this->is_global = false;
 532
 533       nir_builder_init(&b, impl);
 534       b.cursor = nir_after_cf_list(&impl->body);
 535       visit_exec_list(&ir->body, this);
 536
 537       this->is_global = true;
 538    } else {
 539       func->impl = NULL;
 540    }
 541 }
 542
 543 void
 544 nir_visitor::visit(ir_loop *ir)
 545 {
 546    nir_loop *loop = nir_loop_create(this->shader);
 547    nir_builder_cf_insert(&b, &loop->cf_node);
 548
 549    b.cursor = nir_after_cf_list(&loop->body);
 550    visit_exec_list(&ir->body_instructions, this);
 551    b.cursor = nir_after_cf_node(&loop->cf_node);
 552 }
 553
 554 void
 555 nir_visitor::visit(ir_if *ir)
 556 {
 557    nir_src condition =
 558       nir_src_for_ssa(evaluate_rvalue(ir->condition));
 559
 560    nir_if *if_stmt = nir_if_create(this->shader);
 561    if_stmt->condition = condition;
 562    nir_builder_cf_insert(&b, &if_stmt->cf_node);
 563
 564    b.cursor = nir_after_cf_list(&if_stmt->then_list);
 565    visit_exec_list(&ir->then_instructions, this);
 566
 567    b.cursor = nir_after_cf_list(&if_stmt->else_list);
 568    visit_exec_list(&ir->else_instructions, this);
 569
 570    b.cursor = nir_after_cf_node(&if_stmt->cf_node);
 571 }
 572
 573 void
 574 nir_visitor::visit(ir_discard *ir)
 575 {
 576    /*
 577     * discards aren't treated as control flow, because before we lower them
 578     * they can appear anywhere in the shader and the stuff after them may still
 579     * be executed (yay, crazy GLSL rules!). However, after lowering, all the
 580     * discards will be immediately followed by a return.
 581     */
 582
 583    nir_intrinsic_instr *discard;
 584    if (ir->condition) {
 585       discard = nir_intrinsic_instr_create(this->shader,
 586                                            nir_intrinsic_discard_if);
 587       discard->src[0] =
 588          nir_src_for_ssa(evaluate_rvalue(ir->condition));
 589    } else {
 590       discard = nir_intrinsic_instr_create(this->shader, nir_intrinsic_discard);
 591    }
 592
 593    nir_builder_instr_insert(&b, &discard->instr);
 594 }
 595
 596 void
 597 nir_visitor::visit(ir_emit_vertex *ir)
 598 {
 599    nir_intrinsic_instr *instr =
 600       nir_intrinsic_instr_create(this->shader, nir_intrinsic_emit_vertex);
 601    nir_intrinsic_set_stream_id(instr, ir->stream_id());
 602    nir_builder_instr_insert(&b, &instr->instr);
 603 }
 604
 605 void
 606 nir_visitor::visit(ir_end_primitive *ir)
 607 {
 608    nir_intrinsic_instr *instr =
 609       nir_intrinsic_instr_create(this->shader, nir_intrinsic_end_primitive);
 610    nir_intrinsic_set_stream_id(instr, ir->stream_id());
 611    nir_builder_instr_insert(&b, &instr->instr);
 612 }
 613
 614 void
 615 nir_visitor::visit(ir_loop_jump *ir)
 616 {
 617    nir_jump_type type;
 618    switch (ir->mode) {
 619    case ir_loop_jump::jump_break:
 620       type = nir_jump_break;
 621       break;
 622    case ir_loop_jump::jump_continue:
 623       type = nir_jump_continue;
 624       break;
 625    default:
 626       unreachable("not reached");
 627    }
 628
 629    nir_jump_instr *instr = nir_jump_instr_create(this->shader, type);
 630    nir_builder_instr_insert(&b, &instr->instr);
 631 }
 632
 633 void
 634 nir_visitor::visit(ir_return *ir)
 635 {
 636    if (ir->value != NULL) {
 637       nir_intrinsic_instr *copy =
 638          nir_intrinsic_instr_create(this->shader, nir_intrinsic_copy_var);
 639
 640       copy->variables[0] = nir_deref_var_create(copy, this->impl->return_var);
 641       copy->variables[1] = evaluate_deref(&copy->instr, ir->value);
 642    }
 643
 644    nir_jump_instr *instr = nir_jump_instr_create(this->shader, nir_jump_return);
 645    nir_builder_instr_insert(&b, &instr->instr);
 646 }
 647
 648 void
 649 nir_visitor::visit(ir_call *ir)
 650 {
 651    if (ir->callee->is_intrinsic) {
 652       nir_intrinsic_op op;
 653       if (strcmp(ir->callee_name(), "__intrinsic_atomic_read") == 0) {
 654          op = nir_intrinsic_atomic_counter_read_var;
 655       } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_increment") == 0) {
 656          op = nir_intrinsic_atomic_counter_inc_var;
 657       } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_predecrement") == 0) {
 658          op = nir_intrinsic_atomic_counter_dec_var;
 659       } else if (strcmp(ir->callee_name(), "__intrinsic_image_load") == 0) {
 660          op = nir_intrinsic_image_load;
 661       } else if (strcmp(ir->callee_name(), "__intrinsic_image_store") == 0) {
 662          op = nir_intrinsic_image_store;
 663       } else if (strcmp(ir->callee_name(), "__intrinsic_image_atomic_add") == 0) {
 664          op = nir_intrinsic_image_atomic_add;
 665       } else if (strcmp(ir->callee_name(), "__intrinsic_image_atomic_min") == 0) {
 666          op = nir_intrinsic_image_atomic_min;
 667       } else if (strcmp(ir->callee_name(), "__intrinsic_image_atomic_max") == 0) {
 668          op = nir_intrinsic_image_atomic_max;
 669       } else if (strcmp(ir->callee_name(), "__intrinsic_image_atomic_and") == 0) {
 670          op = nir_intrinsic_image_atomic_and;
 671       } else if (strcmp(ir->callee_name(), "__intrinsic_image_atomic_or") == 0) {
 672          op = nir_intrinsic_image_atomic_or;
 673       } else if (strcmp(ir->callee_name(), "__intrinsic_image_atomic_xor") == 0) {
 674          op = nir_intrinsic_image_atomic_xor;
 675       } else if (strcmp(ir->callee_name(), "__intrinsic_image_atomic_exchange") == 0) {
 676          op = nir_intrinsic_image_atomic_exchange;
 677       } else if (strcmp(ir->callee_name(), "__intrinsic_image_atomic_comp_swap") == 0) {
 678          op = nir_intrinsic_image_atomic_comp_swap;
 679       } else if (strcmp(ir->callee_name(), "__intrinsic_memory_barrier") == 0) {
 680          op = nir_intrinsic_memory_barrier;
 681       } else if (strcmp(ir->callee_name(), "__intrinsic_image_size") == 0) {
 682          op = nir_intrinsic_image_size;
 683       } else if (strcmp(ir->callee_name(), "__intrinsic_image_samples") == 0) {
 684          op = nir_intrinsic_image_samples;
 685       } else if (strcmp(ir->callee_name(), "__intrinsic_store_ssbo") == 0) {
 686          op = nir_intrinsic_store_ssbo;
 687       } else if (strcmp(ir->callee_name(), "__intrinsic_load_ssbo") == 0) {
 688          op = nir_intrinsic_load_ssbo;
 689       } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_add_ssbo") == 0) {
 690          op = nir_intrinsic_ssbo_atomic_add;
 691       } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_and_ssbo") == 0) {
 692          op = nir_intrinsic_ssbo_atomic_and;
 693       } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_or_ssbo") == 0) {
 694          op = nir_intrinsic_ssbo_atomic_or;
 695       } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_xor_ssbo") == 0) {
 696          op = nir_intrinsic_ssbo_atomic_xor;
 697       } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_min_ssbo") == 0) {
 698          assert(ir->return_deref);
 699          if (ir->return_deref->type == glsl_type::int_type)
 700             op = nir_intrinsic_ssbo_atomic_imin;
 701          else if (ir->return_deref->type == glsl_type::uint_type)
 702             op = nir_intrinsic_ssbo_atomic_umin;
 703          else
 704             unreachable("Invalid type");
 705       } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_max_ssbo") == 0) {
 706          assert(ir->return_deref);
 707          if (ir->return_deref->type == glsl_type::int_type)
 708             op = nir_intrinsic_ssbo_atomic_imax;
 709          else if (ir->return_deref->type == glsl_type::uint_type)
 710             op = nir_intrinsic_ssbo_atomic_umax;
 711          else
 712             unreachable("Invalid type");
 713       } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_exchange_ssbo") == 0) {
 714          op = nir_intrinsic_ssbo_atomic_exchange;
 715       } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_comp_swap_ssbo") == 0) {
 716          op = nir_intrinsic_ssbo_atomic_comp_swap;
 717       } else if (strcmp(ir->callee_name(), "__intrinsic_shader_clock") == 0) {
 718          op = nir_intrinsic_shader_clock;
 719       } else if (strcmp(ir->callee_name(), "__intrinsic_group_memory_barrier") == 0) {
 720          op = nir_intrinsic_group_memory_barrier;
 721       } else if (strcmp(ir->callee_name(), "__intrinsic_memory_barrier_atomic_counter") == 0) {
 722          op = nir_intrinsic_memory_barrier_atomic_counter;
 723       } else if (strcmp(ir->callee_name(), "__intrinsic_memory_barrier_buffer") == 0) {
 724          op = nir_intrinsic_memory_barrier_buffer;
 725       } else if (strcmp(ir->callee_name(), "__intrinsic_memory_barrier_image") == 0) {
 726          op = nir_intrinsic_memory_barrier_image;
 727       } else if (strcmp(ir->callee_name(), "__intrinsic_memory_barrier_shared") == 0) {
 728          op = nir_intrinsic_memory_barrier_shared;
 729       } else if (strcmp(ir->callee_name(), "__intrinsic_load_shared") == 0) {
 730          op = nir_intrinsic_load_shared;
 731       } else if (strcmp(ir->callee_name(), "__intrinsic_store_shared") == 0) {
 732          op = nir_intrinsic_store_shared;
 733       } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_add_shared") == 0) {
 734          op = nir_intrinsic_shared_atomic_add;
 735       } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_and_shared") == 0) {
 736          op = nir_intrinsic_shared_atomic_and;
 737       } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_or_shared") == 0) {
 738          op = nir_intrinsic_shared_atomic_or;
 739       } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_xor_shared") == 0) {
 740          op = nir_intrinsic_shared_atomic_xor;
 741       } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_min_shared") == 0) {
 742          assert(ir->return_deref);
 743          if (ir->return_deref->type == glsl_type::int_type)
 744             op = nir_intrinsic_shared_atomic_imin;
 745          else if (ir->return_deref->type == glsl_type::uint_type)
 746             op = nir_intrinsic_shared_atomic_umin;
 747          else
 748             unreachable("Invalid type");
 749       } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_max_shared") == 0) {
 750          assert(ir->return_deref);
 751          if (ir->return_deref->type == glsl_type::int_type)
 752             op = nir_intrinsic_shared_atomic_imax;
 753          else if (ir->return_deref->type == glsl_type::uint_type)
 754             op = nir_intrinsic_shared_atomic_umax;
 755          else
 756             unreachable("Invalid type");
 757       } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_exchange_shared") == 0) {
 758          op = nir_intrinsic_shared_atomic_exchange;
 759       } else if (strcmp(ir->callee_name(), "__intrinsic_atomic_comp_swap_shared") == 0) {
 760          op = nir_intrinsic_shared_atomic_comp_swap;
 761       } else {
 762          unreachable("not reached");
 763       }
 764
 765       nir_intrinsic_instr *instr = nir_intrinsic_instr_create(shader, op);
 766       nir_dest *dest = &instr->dest;
 767
 768       switch (op) {
 769       case nir_intrinsic_atomic_counter_read_var:
 770       case nir_intrinsic_atomic_counter_inc_var:
 771       case nir_intrinsic_atomic_counter_dec_var: {
 772          ir_dereference *param =
 773             (ir_dereference *) ir->actual_parameters.get_head();
 774          instr->variables[0] = evaluate_deref(&instr->instr, param);
 775          nir_ssa_dest_init(&instr->instr, &instr->dest, 1, NULL);
 776          nir_builder_instr_insert(&b, &instr->instr);
 777          break;
 778       }
 779       case nir_intrinsic_image_load:
 780       case nir_intrinsic_image_store:
 781       case nir_intrinsic_image_atomic_add:
 782       case nir_intrinsic_image_atomic_min:
 783       case nir_intrinsic_image_atomic_max:
 784       case nir_intrinsic_image_atomic_and:
 785       case nir_intrinsic_image_atomic_or:
 786       case nir_intrinsic_image_atomic_xor:
 787       case nir_intrinsic_image_atomic_exchange:
 788       case nir_intrinsic_image_atomic_comp_swap:
 789       case nir_intrinsic_image_samples:
 790       case nir_intrinsic_image_size: {
 791          nir_ssa_undef_instr *instr_undef =
 792             nir_ssa_undef_instr_create(shader, 1);
 793          nir_builder_instr_insert(&b, &instr_undef->instr);
 794
 795          /* Set the image variable dereference. */
 796          exec_node *param = ir->actual_parameters.get_head();
 797          ir_dereference *image = (ir_dereference *)param;
 798          const glsl_type *type =
 799             image->variable_referenced()->type->without_array();
 800
 801          instr->variables[0] = evaluate_deref(&instr->instr, image);
 802          param = param->get_next();
 803
 804          /* Set the intrinsic destination. */
 805          if (ir->return_deref) {
 806             const nir_intrinsic_info *info =
 807                     &nir_intrinsic_infos[instr->intrinsic];
 808             nir_ssa_dest_init(&instr->instr, &instr->dest,
 809                               info->dest_components, NULL);
 810          }
 811
 812          if (op == nir_intrinsic_image_size ||
 813              op == nir_intrinsic_image_samples) {
 814             nir_builder_instr_insert(&b, &instr->instr);
 815             break;
 816          }
 817
 818          /* Set the address argument, extending the coordinate vector to four
 819           * components.
 820           */
 821          nir_ssa_def *src_addr =
 822             evaluate_rvalue((ir_dereference *)param);
 823          nir_ssa_def *srcs[4];
 824
 825          for (int i = 0; i < 4; i++) {
 826             if (i < type->coordinate_components())
 827                srcs[i] = nir_channel(&b, src_addr, i);
 828             else
 829                srcs[i] = &instr_undef->def;
 830          }
 831
 832          instr->src[0] = nir_src_for_ssa(nir_vec(&b, srcs, 4));
 833          param = param->get_next();
 834
 835          /* Set the sample argument, which is undefined for single-sample
 836           * images.
 837           */
 838          if (type->sampler_dimensionality == GLSL_SAMPLER_DIM_MS) {
 839             instr->src[1] =
 840                nir_src_for_ssa(evaluate_rvalue((ir_dereference *)param));
 841             param = param->get_next();
 842          } else {
 843             instr->src[1] = nir_src_for_ssa(&instr_undef->def);
 844          }
 845
 846          /* Set the intrinsic parameters. */
 847          if (!param->is_tail_sentinel()) {
 848             instr->src[2] =
 849                nir_src_for_ssa(evaluate_rvalue((ir_dereference *)param));
 850             param = param->get_next();
 851          }
 852
 853          if (!param->is_tail_sentinel()) {
 854             instr->src[3] =
 855                nir_src_for_ssa(evaluate_rvalue((ir_dereference *)param));
 856             param = param->get_next();
 857          }
 858          nir_builder_instr_insert(&b, &instr->instr);
 859          break;
 860       }
 861       case nir_intrinsic_memory_barrier:
 862       case nir_intrinsic_group_memory_barrier:
 863       case nir_intrinsic_memory_barrier_atomic_counter:
 864       case nir_intrinsic_memory_barrier_buffer:
 865       case nir_intrinsic_memory_barrier_image:
 866       case nir_intrinsic_memory_barrier_shared:
 867          nir_builder_instr_insert(&b, &instr->instr);
 868          break;
 869       case nir_intrinsic_shader_clock:
 870          nir_ssa_dest_init(&instr->instr, &instr->dest, 1, NULL);
 871          nir_builder_instr_insert(&b, &instr->instr);
 872          break;
 873       case nir_intrinsic_store_ssbo: {
 874          exec_node *param = ir->actual_parameters.get_head();
 875          ir_rvalue *block = ((ir_instruction *)param)->as_rvalue();
 876
 877          param = param->get_next();
 878          ir_rvalue *offset = ((ir_instruction *)param)->as_rvalue();
 879
 880          param = param->get_next();
 881          ir_rvalue *val = ((ir_instruction *)param)->as_rvalue();
 882
 883          param = param->get_next();
 884          ir_constant *write_mask = ((ir_instruction *)param)->as_constant();
 885          assert(write_mask);
 886
 887          instr->src[0] = nir_src_for_ssa(evaluate_rvalue(val));
 888          instr->src[1] = nir_src_for_ssa(evaluate_rvalue(block));
 889          instr->src[2] = nir_src_for_ssa(evaluate_rvalue(offset));
 890          nir_intrinsic_set_write_mask(instr, write_mask->value.u[0]);
 891          instr->num_components = val->type->vector_elements;
 892
 893          nir_builder_instr_insert(&b, &instr->instr);
 894          break;
 895       }
 896       case nir_intrinsic_load_ssbo: {
 897          exec_node *param = ir->actual_parameters.get_head();
 898          ir_rvalue *block = ((ir_instruction *)param)->as_rvalue();
 899
 900          param = param->get_next();
 901          ir_rvalue *offset = ((ir_instruction *)param)->as_rvalue();
 902
 903          instr->src[0] = nir_src_for_ssa(evaluate_rvalue(block));
 904          instr->src[1] = nir_src_for_ssa(evaluate_rvalue(offset));
 905
 906          const glsl_type *type = ir->return_deref->var->type;
 907          instr->num_components = type->vector_elements;
 908
 909          /* Setup destination register */
 910          nir_ssa_dest_init(&instr->instr, &instr->dest,
 911                            type->vector_elements, NULL);
 912
 913          /* Insert the created nir instruction now since in the case of boolean
 914           * result we will need to emit another instruction after it
 915           */
 916          nir_builder_instr_insert(&b, &instr->instr);
 917
 918          /*
 919           * In SSBO/UBO's, a true boolean value is any non-zero value, but we
 920           * consider a true boolean to be ~0. Fix this up with a != 0
 921           * comparison.
 922           */
 923          if (type->base_type == GLSL_TYPE_BOOL) {
 924             nir_alu_instr *load_ssbo_compare =
 925                nir_alu_instr_create(shader, nir_op_ine);
 926             load_ssbo_compare->src[0].src.is_ssa = true;
 927             load_ssbo_compare->src[0].src.ssa = &instr->dest.ssa;
 928             load_ssbo_compare->src[1].src =
 929                nir_src_for_ssa(nir_imm_int(&b, 0));
 930             for (unsigned i = 0; i < type->vector_elements; i++)
 931                load_ssbo_compare->src[1].swizzle[i] = 0;
 932             nir_ssa_dest_init(&load_ssbo_compare->instr,
 933                               &load_ssbo_compare->dest.dest,
 934                               type->vector_elements, NULL);
 935             load_ssbo_compare->dest.write_mask = (1 << type->vector_elements) - 1;
 936             nir_builder_instr_insert(&b, &load_ssbo_compare->instr);
 937             dest = &load_ssbo_compare->dest.dest;
 938          }
 939          break;
 940       }
 941       case nir_intrinsic_ssbo_atomic_add:
 942       case nir_intrinsic_ssbo_atomic_imin:
 943       case nir_intrinsic_ssbo_atomic_umin:
 944       case nir_intrinsic_ssbo_atomic_imax:
 945       case nir_intrinsic_ssbo_atomic_umax:
 946       case nir_intrinsic_ssbo_atomic_and:
 947       case nir_intrinsic_ssbo_atomic_or:
 948       case nir_intrinsic_ssbo_atomic_xor:
 949       case nir_intrinsic_ssbo_atomic_exchange:
 950       case nir_intrinsic_ssbo_atomic_comp_swap: {
 951          int param_count = ir->actual_parameters.length();
 952          assert(param_count == 3 || param_count == 4);
 953
 954          /* Block index */
 955          exec_node *param = ir->actual_parameters.get_head();
 956          ir_instruction *inst = (ir_instruction *) param;
 957          instr->src[0] = nir_src_for_ssa(evaluate_rvalue(inst->as_rvalue()));
 958
 959          /* Offset */
 960          param = param->get_next();
 961          inst = (ir_instruction *) param;
 962          instr->src[1] = nir_src_for_ssa(evaluate_rvalue(inst->as_rvalue()));
 963
 964          /* data1 parameter (this is always present) */
 965          param = param->get_next();
 966          inst = (ir_instruction *) param;
 967          instr->src[2] = nir_src_for_ssa(evaluate_rvalue(inst->as_rvalue()));
 968
 969          /* data2 parameter (only with atomic_comp_swap) */
 970          if (param_count == 4) {
 971             assert(op == nir_intrinsic_ssbo_atomic_comp_swap);
 972             param = param->get_next();
 973             inst = (ir_instruction *) param;
 974             instr->src[3] = nir_src_for_ssa(evaluate_rvalue(inst->as_rvalue()));
 975          }
 976
 977          /* Atomic result */
 978          assert(ir->return_deref);
 979          nir_ssa_dest_init(&instr->instr, &instr->dest,
 980                            ir->return_deref->type->vector_elements, NULL);
 981          nir_builder_instr_insert(&b, &instr->instr);
 982          break;
 983       }
 984       case nir_intrinsic_load_shared: {
 985          exec_node *param = ir->actual_parameters.get_head();
 986          ir_rvalue *offset = ((ir_instruction *)param)->as_rvalue();
 987
 988          nir_intrinsic_set_base(instr, 0);
 989          instr->src[0] = nir_src_for_ssa(evaluate_rvalue(offset));
 990
 991          const glsl_type *type = ir->return_deref->var->type;
 992          instr->num_components = type->vector_elements;
 993
 994          /* Setup destination register */
 995          nir_ssa_dest_init(&instr->instr, &instr->dest,
 996                            type->vector_elements, NULL);
 997
 998          nir_builder_instr_insert(&b, &instr->instr);
 999          break;
1000       }
1001       case nir_intrinsic_store_shared: {
1002          exec_node *param = ir->actual_parameters.get_head();
1003          ir_rvalue *offset = ((ir_instruction *)param)->as_rvalue();
1004
1005          param = param->get_next();
1006          ir_rvalue *val = ((ir_instruction *)param)->as_rvalue();
1007
1008          param = param->get_next();
1009          ir_constant *write_mask = ((ir_instruction *)param)->as_constant();
1010          assert(write_mask);
1011
1012          nir_intrinsic_set_base(instr, 0);
1013          instr->src[1] = nir_src_for_ssa(evaluate_rvalue(offset));
1014
1015          nir_intrinsic_set_write_mask(instr, write_mask->value.u[0]);
1016
1017          instr->src[0] = nir_src_for_ssa(evaluate_rvalue(val));
1018          instr->num_components = val->type->vector_elements;
1019
1020          nir_builder_instr_insert(&b, &instr->instr);
1021          break;
1022       }
1023       case nir_intrinsic_shared_atomic_add:
1024       case nir_intrinsic_shared_atomic_imin:
1025       case nir_intrinsic_shared_atomic_umin:
1026       case nir_intrinsic_shared_atomic_imax:
1027       case nir_intrinsic_shared_atomic_umax:
1028       case nir_intrinsic_shared_atomic_and:
1029       case nir_intrinsic_shared_atomic_or:
1030       case nir_intrinsic_shared_atomic_xor:
1031       case nir_intrinsic_shared_atomic_exchange:
1032       case nir_intrinsic_shared_atomic_comp_swap: {
1033          int param_count = ir->actual_parameters.length();
1034          assert(param_count == 2 || param_count == 3);
1035
1036          /* Offset */
1037          exec_node *param = ir->actual_parameters.get_head();
1038          ir_instruction *inst = (ir_instruction *) param;
1039          instr->src[0] = nir_src_for_ssa(evaluate_rvalue(inst->as_rvalue()));
1040
1041          /* data1 parameter (this is always present) */
1042          param = param->get_next();
1043          inst = (ir_instruction *) param;
1044          instr->src[1] = nir_src_for_ssa(evaluate_rvalue(inst->as_rvalue()));
1045
1046          /* data2 parameter (only with atomic_comp_swap) */
1047          if (param_count == 3) {
1048             assert(op == nir_intrinsic_shared_atomic_comp_swap);
1049             param = param->get_next();
1050             inst = (ir_instruction *) param;
1051             instr->src[2] =
1052                nir_src_for_ssa(evaluate_rvalue(inst->as_rvalue()));
1053          }
1054
1055          /* Atomic result */
1056          assert(ir->return_deref);
1057          nir_ssa_dest_init(&instr->instr, &instr->dest,
1058                            ir->return_deref->type->vector_elements, NULL);
1059          nir_builder_instr_insert(&b, &instr->instr);
1060          break;
1061       }
1062       default:
1063          unreachable("not reached");
1064       }
1065
1066       if (ir->return_deref) {
1067          nir_intrinsic_instr *store_instr =
1068             nir_intrinsic_instr_create(shader, nir_intrinsic_store_var);
1069          store_instr->num_components = ir->return_deref->type->vector_elements;
1070          nir_intrinsic_set_write_mask(store_instr,
1071                                       (1 << store_instr->num_components) - 1);
1072
1073          store_instr->variables[0] =
1074             evaluate_deref(&store_instr->instr, ir->return_deref);
1075          store_instr->src[0] = nir_src_for_ssa(&dest->ssa);
1076
1077          nir_builder_instr_insert(&b, &store_instr->instr);
1078       }
1079
1080       return;
1081    }
1082
1083    struct hash_entry *entry =
1084       _mesa_hash_table_search(this->overload_table, ir->callee);
1085    assert(entry);
1086    nir_function *callee = (nir_function *) entry->data;
1087
1088    nir_call_instr *instr = nir_call_instr_create(this->shader, callee);
1089
1090    unsigned i = 0;
1091    foreach_in_list(ir_dereference, param, &ir->actual_parameters) {
1092       instr->params[i] = evaluate_deref(&instr->instr, param);
1093       i++;
1094    }
1095
1096    instr->return_deref = evaluate_deref(&instr->instr, ir->return_deref);
1097    nir_builder_instr_insert(&b, &instr->instr);
1098 }
1099
1100 void
1101 nir_visitor::visit(ir_assignment *ir)
1102 {
1103    unsigned num_components = ir->lhs->type->vector_elements;
1104
1105    if ((ir->rhs->as_dereference() || ir->rhs->as_constant()) &&
1106        (ir->write_mask == (1 << num_components) - 1 || ir->write_mask == 0)) {
1107       /* We're doing a plain-as-can-be copy, so emit a copy_var */
1108       nir_intrinsic_instr *copy =
1109          nir_intrinsic_instr_create(this->shader, nir_intrinsic_copy_var);
1110
1111       copy->variables[0] = evaluate_deref(&copy->instr, ir->lhs);
1112       copy->variables[1] = evaluate_deref(&copy->instr, ir->rhs);
1113
1114       if (ir->condition) {
1115          nir_if *if_stmt = nir_if_create(this->shader);
1116          if_stmt->condition = nir_src_for_ssa(evaluate_rvalue(ir->condition));
1117          nir_builder_cf_insert(&b, &if_stmt->cf_node);
1118          nir_instr_insert_after_cf_list(&if_stmt->then_list, &copy->instr);
1119          b.cursor = nir_after_cf_node(&if_stmt->cf_node);
1120       } else {
1121          nir_builder_instr_insert(&b, &copy->instr);
1122       }
1123       return;
1124    }
1125
1126    assert(ir->rhs->type->is_scalar() || ir->rhs->type->is_vector());
1127
1128    ir->lhs->accept(this);
1129    nir_deref_var *lhs_deref = this->deref_head;
1130    nir_ssa_def *src = evaluate_rvalue(ir->rhs);
1131
1132    if (ir->write_mask != (1 << num_components) - 1 && ir->write_mask != 0) {
1133       /* GLSL IR will give us the input to the write-masked assignment in a
1134        * single packed vector.  So, for example, if the writemask is xzw, then
1135        * we have to swizzle x -> x, y -> z, and z -> w and get the y component
1136        * from the load.
1137        */
1138       unsigned swiz[4];
1139       unsigned component = 0;
1140       for (unsigned i = 0; i < 4; i++) {
1141          swiz[i] = ir->write_mask & (1 << i) ? component++ : 0;
1142       }
1143       src = nir_swizzle(&b, src, swiz, num_components, !supports_ints);
1144    }
1145
1146    nir_intrinsic_instr *store =
1147       nir_intrinsic_instr_create(this->shader, nir_intrinsic_store_var);
1148    store->num_components = ir->lhs->type->vector_elements;
1149    nir_intrinsic_set_write_mask(store, ir->write_mask);
1150    nir_deref *store_deref = nir_copy_deref(store, &lhs_deref->deref);
1151    store->variables[0] = nir_deref_as_var(store_deref);
1152    store->src[0] = nir_src_for_ssa(src);
1153
1154    if (ir->condition) {
1155       nir_if *if_stmt = nir_if_create(this->shader);
1156       if_stmt->condition = nir_src_for_ssa(evaluate_rvalue(ir->condition));
1157       nir_builder_cf_insert(&b, &if_stmt->cf_node);
1158       nir_instr_insert_after_cf_list(&if_stmt->then_list, &store->instr);
1159       b.cursor = nir_after_cf_node(&if_stmt->cf_node);
1160    } else {
1161       nir_builder_instr_insert(&b, &store->instr);
1162    }
1163 }
1164
1165 /*
1166  * Given an instruction, returns a pointer to its destination or NULL if there
1167  * is no destination.
1168  *
1169  * Note that this only handles instructions we generate at this level.
1170  */
1171 static nir_dest *
1172 get_instr_dest(nir_instr *instr)
1173 {
1174    nir_alu_instr *alu_instr;
1175    nir_intrinsic_instr *intrinsic_instr;
1176    nir_tex_instr *tex_instr;
1177
1178    switch (instr->type) {
1179       case nir_instr_type_alu:
1180          alu_instr = nir_instr_as_alu(instr);
1181          return &alu_instr->dest.dest;
1182
1183       case nir_instr_type_intrinsic:
1184          intrinsic_instr = nir_instr_as_intrinsic(instr);
1185          if (nir_intrinsic_infos[intrinsic_instr->intrinsic].has_dest)
1186             return &intrinsic_instr->dest;
1187          else
1188             return NULL;
1189
1190       case nir_instr_type_tex:
1191          tex_instr = nir_instr_as_tex(instr);
1192          return &tex_instr->dest;
1193
1194       default:
1195          unreachable("not reached");
1196    }
1197
1198    return NULL;
1199 }
1200
1201 void
1202 nir_visitor::add_instr(nir_instr *instr, unsigned num_components)
1203 {
1204    nir_dest *dest = get_instr_dest(instr);
1205
1206    if (dest)
1207       nir_ssa_dest_init(instr, dest, num_components, NULL);
1208
1209    nir_builder_instr_insert(&b, instr);
1210
1211    if (dest) {
1212       assert(dest->is_ssa);
1213       this->result = &dest->ssa;
1214    }
1215 }
1216
1217 nir_ssa_def *
1218 nir_visitor::evaluate_rvalue(ir_rvalue* ir)
1219 {
1220    ir->accept(this);
1221    if (ir->as_dereference() || ir->as_constant()) {
1222       /*
1223        * A dereference is being used on the right hand side, which means we
1224        * must emit a variable load.
1225        */
1226
1227       nir_intrinsic_instr *load_instr =
1228          nir_intrinsic_instr_create(this->shader, nir_intrinsic_load_var);
1229       load_instr->num_components = ir->type->vector_elements;
1230       load_instr->variables[0] = this->deref_head;
1231       ralloc_steal(load_instr, load_instr->variables[0]);
1232       add_instr(&load_instr->instr, ir->type->vector_elements);
1233    }
1234
1235    return this->result;
1236 }
1237
1238 void
1239 nir_visitor::visit(ir_expression *ir)
1240 {
1241    /* Some special cases */
1242    switch (ir->operation) {
1243    case ir_binop_ubo_load: {
1244       nir_intrinsic_instr *load =
1245          nir_intrinsic_instr_create(this->shader, nir_intrinsic_load_ubo);
1246       load->num_components = ir->type->vector_elements;
1247       load->src[0] = nir_src_for_ssa(evaluate_rvalue(ir->operands[0]));
1248       load->src[1] = nir_src_for_ssa(evaluate_rvalue(ir->operands[1]));
1249       add_instr(&load->instr, ir->type->vector_elements);
1250
1251       /*
1252        * In UBO's, a true boolean value is any non-zero value, but we consider
1253        * a true boolean to be ~0. Fix this up with a != 0 comparison.
1254        */
1255
1256       if (ir->type->base_type == GLSL_TYPE_BOOL)
1257          this->result = nir_ine(&b, &load->dest.ssa, nir_imm_int(&b, 0));
1258
1259       return;
1260    }
1261
1262    case ir_unop_interpolate_at_centroid:
1263    case ir_binop_interpolate_at_offset:
1264    case ir_binop_interpolate_at_sample: {
1265       ir_dereference *deref = ir->operands[0]->as_dereference();
1266       ir_swizzle *swizzle = NULL;
1267       if (!deref) {
1268          /* the api does not allow a swizzle here, but the varying packing code
1269           * may have pushed one into here.
1270           */
1271          swizzle = ir->operands[0]->as_swizzle();
1272          assert(swizzle);
1273          deref = swizzle->val->as_dereference();
1274          assert(deref);
1275       }
1276
1277       deref->accept(this);
1278
1279       nir_intrinsic_op op;
1280       if (this->deref_head->var->data.mode == nir_var_shader_in) {
1281          switch (ir->operation) {
1282          case ir_unop_interpolate_at_centroid:
1283             op = nir_intrinsic_interp_var_at_centroid;
1284             break;
1285          case ir_binop_interpolate_at_offset:
1286             op = nir_intrinsic_interp_var_at_offset;
1287             break;
1288          case ir_binop_interpolate_at_sample:
1289             op = nir_intrinsic_interp_var_at_sample;
1290             break;
1291          default:
1292             unreachable("Invalid interpolation intrinsic");
1293          }
1294       } else {
1295          /* This case can happen if the vertex shader does not write the
1296           * given varying.  In this case, the linker will lower it to a
1297           * global variable.  Since interpolating a variable makes no
1298           * sense, we'll just turn it into a load which will probably
1299           * eventually end up as an SSA definition.
1300           */
1301          assert(this->deref_head->var->data.mode == nir_var_global);
1302          op = nir_intrinsic_load_var;
1303       }
1304
1305       nir_intrinsic_instr *intrin = nir_intrinsic_instr_create(shader, op);
1306       intrin->num_components = deref->type->vector_elements;
1307       intrin->variables[0] = this->deref_head;
1308       ralloc_steal(intrin, intrin->variables[0]);
1309
1310       if (intrin->intrinsic == nir_intrinsic_interp_var_at_offset ||
1311           intrin->intrinsic == nir_intrinsic_interp_var_at_sample)
1312          intrin->src[0] = nir_src_for_ssa(evaluate_rvalue(ir->operands[1]));
1313
1314       add_instr(&intrin->instr, deref->type->vector_elements);
1315
1316       if (swizzle) {
1317          unsigned swiz[4] = {
1318             swizzle->mask.x, swizzle->mask.y, swizzle->mask.z, swizzle->mask.w
1319          };
1320
1321          result = nir_swizzle(&b, result, swiz,
1322                               swizzle->type->vector_elements, false);
1323       }
1324
1325       return;
1326    }
1327
1328    default:
1329       break;
1330    }
1331
1332    nir_ssa_def *srcs[4];
1333    for (unsigned i = 0; i < ir->get_num_operands(); i++)
1334       srcs[i] = evaluate_rvalue(ir->operands[i]);
1335
1336    glsl_base_type types[4];
1337    for (unsigned i = 0; i < ir->get_num_operands(); i++)
1338       if (supports_ints)
1339          types[i] = ir->operands[i]->type->base_type;
1340       else
1341          types[i] = GLSL_TYPE_FLOAT;
1342
1343    glsl_base_type out_type;
1344    if (supports_ints)
1345       out_type = ir->type->base_type;
1346    else
1347       out_type = GLSL_TYPE_FLOAT;
1348
1349    switch (ir->operation) {
1350    case ir_unop_bit_not: result = nir_inot(&b, srcs[0]); break;
1351    case ir_unop_logic_not:
1352       result = supports_ints ? nir_inot(&b, srcs[0]) : nir_fnot(&b, srcs[0]);
1353       break;
1354    case ir_unop_neg:
1355       result = (types[0] == GLSL_TYPE_FLOAT) ? nir_fneg(&b, srcs[0])
1356                                              : nir_ineg(&b, srcs[0]);
1357       break;
1358    case ir_unop_abs:
1359       result = (types[0] == GLSL_TYPE_FLOAT) ? nir_fabs(&b, srcs[0])
1360                                              : nir_iabs(&b, srcs[0]);
1361       break;
1362    case ir_unop_saturate:
1363       assert(types[0] == GLSL_TYPE_FLOAT);
1364       result = nir_fsat(&b, srcs[0]);
1365       break;
1366    case ir_unop_sign:
1367       result = (types[0] == GLSL_TYPE_FLOAT) ? nir_fsign(&b, srcs[0])
1368                                              : nir_isign(&b, srcs[0]);
1369       break;
1370    case ir_unop_rcp:  result = nir_frcp(&b, srcs[0]);  break;
1371    case ir_unop_rsq:  result = nir_frsq(&b, srcs[0]);  break;
1372    case ir_unop_sqrt: result = nir_fsqrt(&b, srcs[0]); break;
1373    case ir_unop_exp:  unreachable("ir_unop_exp should have been lowered");
1374    case ir_unop_log:  unreachable("ir_unop_log should have been lowered");
1375    case ir_unop_exp2: result = nir_fexp2(&b, srcs[0]); break;
1376    case ir_unop_log2: result = nir_flog2(&b, srcs[0]); break;
1377    case ir_unop_i2f:
1378       result = supports_ints ? nir_i2f(&b, srcs[0]) : nir_fmov(&b, srcs[0]);
1379       break;
1380    case ir_unop_u2f:
1381       result = supports_ints ? nir_u2f(&b, srcs[0]) : nir_fmov(&b, srcs[0]);
1382       break;
1383    case ir_unop_b2f:
1384       result = supports_ints ? nir_b2f(&b, srcs[0]) : nir_fmov(&b, srcs[0]);
1385       break;
1386    case ir_unop_f2i:  result = nir_f2i(&b, srcs[0]);   break;
1387    case ir_unop_f2u:  result = nir_f2u(&b, srcs[0]);   break;
1388    case ir_unop_f2b:  result = nir_f2b(&b, srcs[0]);   break;
1389    case ir_unop_i2b:  result = nir_i2b(&b, srcs[0]);   break;
1390    case ir_unop_b2i:  result = nir_b2i(&b, srcs[0]);   break;
1391    case ir_unop_i2u:
1392    case ir_unop_u2i:
1393    case ir_unop_bitcast_i2f:
1394    case ir_unop_bitcast_f2i:
1395    case ir_unop_bitcast_u2f:
1396    case ir_unop_bitcast_f2u:
1397    case ir_unop_subroutine_to_int:
1398       /* no-op */
1399       result = nir_imov(&b, srcs[0]);
1400       break;
1401    case ir_unop_trunc: result = nir_ftrunc(&b, srcs[0]); break;
1402    case ir_unop_ceil:  result = nir_fceil(&b, srcs[0]); break;
1403    case ir_unop_floor: result = nir_ffloor(&b, srcs[0]); break;
1404    case ir_unop_fract: result = nir_ffract(&b, srcs[0]); break;
1405    case ir_unop_round_even: result = nir_fround_even(&b, srcs[0]); break;
1406    case ir_unop_sin:   result = nir_fsin(&b, srcs[0]); break;
1407    case ir_unop_cos:   result = nir_fcos(&b, srcs[0]); break;
1408    case ir_unop_dFdx:        result = nir_fddx(&b, srcs[0]); break;
1409    case ir_unop_dFdy:        result = nir_fddy(&b, srcs[0]); break;
1410    case ir_unop_dFdx_fine:   result = nir_fddx_fine(&b, srcs[0]); break;
1411    case ir_unop_dFdy_fine:   result = nir_fddy_fine(&b, srcs[0]); break;
1412    case ir_unop_dFdx_coarse: result = nir_fddx_coarse(&b, srcs[0]); break;
1413    case ir_unop_dFdy_coarse: result = nir_fddy_coarse(&b, srcs[0]); break;
1414    case ir_unop_pack_snorm_2x16:
1415       result = nir_pack_snorm_2x16(&b, srcs[0]);
1416       break;
1417    case ir_unop_pack_snorm_4x8:
1418       result = nir_pack_snorm_4x8(&b, srcs[0]);
1419       break;
1420    case ir_unop_pack_unorm_2x16:
1421       result = nir_pack_unorm_2x16(&b, srcs[0]);
1422       break;
1423    case ir_unop_pack_unorm_4x8:
1424       result = nir_pack_unorm_4x8(&b, srcs[0]);
1425       break;
1426    case ir_unop_pack_half_2x16:
1427       result = nir_pack_half_2x16(&b, srcs[0]);
1428       break;
1429    case ir_unop_unpack_snorm_2x16:
1430       result = nir_unpack_snorm_2x16(&b, srcs[0]);
1431       break;
1432    case ir_unop_unpack_snorm_4x8:
1433       result = nir_unpack_snorm_4x8(&b, srcs[0]);
1434       break;
1435    case ir_unop_unpack_unorm_2x16:
1436       result = nir_unpack_unorm_2x16(&b, srcs[0]);
1437       break;
1438    case ir_unop_unpack_unorm_4x8:
1439       result = nir_unpack_unorm_4x8(&b, srcs[0]);
1440       break;
1441    case ir_unop_unpack_half_2x16:
1442       result = nir_unpack_half_2x16(&b, srcs[0]);
1443       break;
1444    case ir_unop_bitfield_reverse:
1445       result = nir_bitfield_reverse(&b, srcs[0]);
1446       break;
1447    case ir_unop_bit_count:
1448       result = nir_bit_count(&b, srcs[0]);
1449       break;
1450    case ir_unop_find_msb:
1451       switch (types[0]) {
1452       case GLSL_TYPE_UINT:
1453          result = nir_ufind_msb(&b, srcs[0]);
1454          break;
1455       case GLSL_TYPE_INT:
1456          result = nir_ifind_msb(&b, srcs[0]);
1457          break;
1458       default:
1459          unreachable("Invalid type for findMSB()");
1460       }
1461       break;
1462    case ir_unop_find_lsb:
1463       result = nir_find_lsb(&b, srcs[0]);
1464       break;
1465
1466    case ir_unop_noise:
1467       switch (ir->type->vector_elements) {
1468       case 1:
1469          switch (ir->operands[0]->type->vector_elements) {
1470             case 1: result = nir_fnoise1_1(&b, srcs[0]); break;
1471             case 2: result = nir_fnoise1_2(&b, srcs[0]); break;
1472             case 3: result = nir_fnoise1_3(&b, srcs[0]); break;
1473             case 4: result = nir_fnoise1_4(&b, srcs[0]); break;
1474             default: unreachable("not reached");
1475          }
1476          break;
1477       case 2:
1478          switch (ir->operands[0]->type->vector_elements) {
1479             case 1: result = nir_fnoise2_1(&b, srcs[0]); break;
1480             case 2: result = nir_fnoise2_2(&b, srcs[0]); break;
1481             case 3: result = nir_fnoise2_3(&b, srcs[0]); break;
1482             case 4: result = nir_fnoise2_4(&b, srcs[0]); break;
1483             default: unreachable("not reached");
1484          }
1485          break;
1486       case 3:
1487          switch (ir->operands[0]->type->vector_elements) {
1488             case 1: result = nir_fnoise3_1(&b, srcs[0]); break;
1489             case 2: result = nir_fnoise3_2(&b, srcs[0]); break;
1490             case 3: result = nir_fnoise3_3(&b, srcs[0]); break;
1491             case 4: result = nir_fnoise3_4(&b, srcs[0]); break;
1492             default: unreachable("not reached");
1493          }
1494          break;
1495       case 4:
1496          switch (ir->operands[0]->type->vector_elements) {
1497             case 1: result = nir_fnoise4_1(&b, srcs[0]); break;
1498             case 2: result = nir_fnoise4_2(&b, srcs[0]); break;
1499             case 3: result = nir_fnoise4_3(&b, srcs[0]); break;
1500             case 4: result = nir_fnoise4_4(&b, srcs[0]); break;
1501             default: unreachable("not reached");
1502          }
1503          break;
1504       default:
1505          unreachable("not reached");
1506       }
1507       break;
1508    case ir_unop_get_buffer_size: {
1509       nir_intrinsic_instr *load = nir_intrinsic_instr_create(
1510          this->shader,
1511          nir_intrinsic_get_buffer_size);
1512       load->num_components = ir->type->vector_elements;
1513       load->src[0] = nir_src_for_ssa(evaluate_rvalue(ir->operands[0]));
1514       add_instr(&load->instr, ir->type->vector_elements);
1515       return;
1516    }
1517
1518    case ir_binop_add:
1519       result = (out_type == GLSL_TYPE_FLOAT) ? nir_fadd(&b, srcs[0], srcs[1])
1520                                              : nir_iadd(&b, srcs[0], srcs[1]);
1521       break;
1522    case ir_binop_sub:
1523       result = (out_type == GLSL_TYPE_FLOAT) ? nir_fsub(&b, srcs[0], srcs[1])
1524                                              : nir_isub(&b, srcs[0], srcs[1]);
1525       break;
1526    case ir_binop_mul:
1527       result = (out_type == GLSL_TYPE_FLOAT) ? nir_fmul(&b, srcs[0], srcs[1])
1528                                              : nir_imul(&b, srcs[0], srcs[1]);
1529       break;
1530    case ir_binop_div:
1531       if (out_type == GLSL_TYPE_FLOAT)
1532          result = nir_fdiv(&b, srcs[0], srcs[1]);
1533       else if (out_type == GLSL_TYPE_INT)
1534          result = nir_idiv(&b, srcs[0], srcs[1]);
1535       else
1536          result = nir_udiv(&b, srcs[0], srcs[1]);
1537       break;
1538    case ir_binop_mod:
1539       result = (out_type == GLSL_TYPE_FLOAT) ? nir_fmod(&b, srcs[0], srcs[1])
1540                                              : nir_umod(&b, srcs[0], srcs[1]);
1541       break;
1542    case ir_binop_min:
1543       if (out_type == GLSL_TYPE_FLOAT)
1544          result = nir_fmin(&b, srcs[0], srcs[1]);
1545       else if (out_type == GLSL_TYPE_INT)
1546          result = nir_imin(&b, srcs[0], srcs[1]);
1547       else
1548          result = nir_umin(&b, srcs[0], srcs[1]);
1549       break;
1550    case ir_binop_max:
1551       if (out_type == GLSL_TYPE_FLOAT)
1552          result = nir_fmax(&b, srcs[0], srcs[1]);
1553       else if (out_type == GLSL_TYPE_INT)
1554          result = nir_imax(&b, srcs[0], srcs[1]);
1555       else
1556          result = nir_umax(&b, srcs[0], srcs[1]);
1557       break;
1558    case ir_binop_pow: result = nir_fpow(&b, srcs[0], srcs[1]); break;
1559    case ir_binop_bit_and: result = nir_iand(&b, srcs[0], srcs[1]); break;
1560    case ir_binop_bit_or: result = nir_ior(&b, srcs[0], srcs[1]); break;
1561    case ir_binop_bit_xor: result = nir_ixor(&b, srcs[0], srcs[1]); break;
1562    case ir_binop_logic_and:
1563       result = supports_ints ? nir_iand(&b, srcs[0], srcs[1])
1564                              : nir_fand(&b, srcs[0], srcs[1]);
1565       break;
1566    case ir_binop_logic_or:
1567       result = supports_ints ? nir_ior(&b, srcs[0], srcs[1])
1568                              : nir_for(&b, srcs[0], srcs[1]);
1569       break;
1570    case ir_binop_logic_xor:
1571       result = supports_ints ? nir_ixor(&b, srcs[0], srcs[1])
1572                              : nir_fxor(&b, srcs[0], srcs[1]);
1573       break;
1574    case ir_binop_lshift: result = nir_ishl(&b, srcs[0], srcs[1]); break;
1575    case ir_binop_rshift:
1576       result = (out_type == GLSL_TYPE_INT) ? nir_ishr(&b, srcs[0], srcs[1])
1577                                            : nir_ushr(&b, srcs[0], srcs[1]);
1578       break;
1579    case ir_binop_imul_high:
1580       result = (out_type == GLSL_TYPE_INT) ? nir_imul_high(&b, srcs[0], srcs[1])
1581                                            : nir_umul_high(&b, srcs[0], srcs[1]);
1582       break;
1583    case ir_binop_carry:  result = nir_uadd_carry(&b, srcs[0], srcs[1]);  break;
1584    case ir_binop_borrow: result = nir_usub_borrow(&b, srcs[0], srcs[1]); break;
1585    case ir_binop_less:
1586       if (supports_ints) {
1587          if (types[0] == GLSL_TYPE_FLOAT)
1588             result = nir_flt(&b, srcs[0], srcs[1]);
1589          else if (types[0] == GLSL_TYPE_INT)
1590             result = nir_ilt(&b, srcs[0], srcs[1]);
1591          else
1592             result = nir_ult(&b, srcs[0], srcs[1]);
1593       } else {
1594          result = nir_slt(&b, srcs[0], srcs[1]);
1595       }
1596       break;
1597    case ir_binop_greater:
1598       if (supports_ints) {
1599          if (types[0] == GLSL_TYPE_FLOAT)
1600             result = nir_flt(&b, srcs[1], srcs[0]);
1601          else if (types[0] == GLSL_TYPE_INT)
1602             result = nir_ilt(&b, srcs[1], srcs[0]);
1603          else
1604             result = nir_ult(&b, srcs[1], srcs[0]);
1605       } else {
1606          result = nir_slt(&b, srcs[1], srcs[0]);
1607       }
1608       break;
1609    case ir_binop_lequal:
1610       if (supports_ints) {
1611          if (types[0] == GLSL_TYPE_FLOAT)
1612             result = nir_fge(&b, srcs[1], srcs[0]);
1613          else if (types[0] == GLSL_TYPE_INT)
1614             result = nir_ige(&b, srcs[1], srcs[0]);
1615          else
1616             result = nir_uge(&b, srcs[1], srcs[0]);
1617       } else {
1618          result = nir_slt(&b, srcs[1], srcs[0]);
1619       }
1620       break;
1621    case ir_binop_gequal:
1622       if (supports_ints) {
1623          if (types[0] == GLSL_TYPE_FLOAT)
1624             result = nir_fge(&b, srcs[0], srcs[1]);
1625          else if (types[0] == GLSL_TYPE_INT)
1626             result = nir_ige(&b, srcs[0], srcs[1]);
1627          else
1628             result = nir_uge(&b, srcs[0], srcs[1]);
1629       } else {
1630          result = nir_slt(&b, srcs[0], srcs[1]);
1631       }
1632       break;
1633    case ir_binop_equal:
1634       if (supports_ints) {
1635          if (types[0] == GLSL_TYPE_FLOAT)
1636             result = nir_feq(&b, srcs[0], srcs[1]);
1637          else
1638             result = nir_ieq(&b, srcs[0], srcs[1]);
1639       } else {
1640          result = nir_seq(&b, srcs[0], srcs[1]);
1641       }
1642       break;
1643    case ir_binop_nequal:
1644       if (supports_ints) {
1645          if (types[0] == GLSL_TYPE_FLOAT)
1646             result = nir_fne(&b, srcs[0], srcs[1]);
1647          else
1648             result = nir_ine(&b, srcs[0], srcs[1]);
1649       } else {
1650          result = nir_sne(&b, srcs[0], srcs[1]);
1651       }
1652       break;
1653    case ir_binop_all_equal:
1654       if (supports_ints) {
1655          if (types[0] == GLSL_TYPE_FLOAT) {
1656             switch (ir->operands[0]->type->vector_elements) {
1657                case 1: result = nir_feq(&b, srcs[0], srcs[1]); break;
1658                case 2: result = nir_ball_fequal2(&b, srcs[0], srcs[1]); break;
1659                case 3: result = nir_ball_fequal3(&b, srcs[0], srcs[1]); break;
1660                case 4: result = nir_ball_fequal4(&b, srcs[0], srcs[1]); break;
1661                default:
1662                   unreachable("not reached");
1663             }
1664          } else {
1665             switch (ir->operands[0]->type->vector_elements) {
1666                case 1: result = nir_ieq(&b, srcs[0], srcs[1]); break;
1667                case 2: result = nir_ball_iequal2(&b, srcs[0], srcs[1]); break;
1668                case 3: result = nir_ball_iequal3(&b, srcs[0], srcs[1]); break;
1669                case 4: result = nir_ball_iequal4(&b, srcs[0], srcs[1]); break;
1670                default:
1671                   unreachable("not reached");
1672             }
1673          }
1674       } else {
1675          switch (ir->operands[0]->type->vector_elements) {
1676             case 1: result = nir_seq(&b, srcs[0], srcs[1]); break;
1677             case 2: result = nir_fall_equal2(&b, srcs[0], srcs[1]); break;
1678             case 3: result = nir_fall_equal3(&b, srcs[0], srcs[1]); break;
1679             case 4: result = nir_fall_equal4(&b, srcs[0], srcs[1]); break;
1680             default:
1681                unreachable("not reached");
1682          }
1683       }
1684       break;
1685    case ir_binop_any_nequal:
1686       if (supports_ints) {
1687          if (types[0] == GLSL_TYPE_FLOAT) {
1688             switch (ir->operands[0]->type->vector_elements) {
1689                case 1: result = nir_fne(&b, srcs[0], srcs[1]); break;
1690                case 2: result = nir_bany_fnequal2(&b, srcs[0], srcs[1]); break;
1691                case 3: result = nir_bany_fnequal3(&b, srcs[0], srcs[1]); break;
1692                case 4: result = nir_bany_fnequal4(&b, srcs[0], srcs[1]); break;
1693                default:
1694                   unreachable("not reached");
1695             }
1696          } else {
1697             switch (ir->operands[0]->type->vector_elements) {
1698                case 1: result = nir_ine(&b, srcs[0], srcs[1]); break;
1699                case 2: result = nir_bany_inequal2(&b, srcs[0], srcs[1]); break;
1700                case 3: result = nir_bany_inequal3(&b, srcs[0], srcs[1]); break;
1701                case 4: result = nir_bany_inequal4(&b, srcs[0], srcs[1]); break;
1702                default:
1703                   unreachable("not reached");
1704             }
1705          }
1706       } else {
1707          switch (ir->operands[0]->type->vector_elements) {
1708             case 1: result = nir_sne(&b, srcs[0], srcs[1]); break;
1709             case 2: result = nir_fany_nequal2(&b, srcs[0], srcs[1]); break;
1710             case 3: result = nir_fany_nequal3(&b, srcs[0], srcs[1]); break;
1711             case 4: result = nir_fany_nequal4(&b, srcs[0], srcs[1]); break;
1712             default:
1713                unreachable("not reached");
1714          }
1715       }
1716       break;
1717    case ir_binop_dot:
1718       switch (ir->operands[0]->type->vector_elements) {
1719          case 2: result = nir_fdot2(&b, srcs[0], srcs[1]); break;
1720          case 3: result = nir_fdot3(&b, srcs[0], srcs[1]); break;
1721          case 4: result = nir_fdot4(&b, srcs[0], srcs[1]); break;
1722          default:
1723             unreachable("not reached");
1724       }
1725       break;
1726
1727    case ir_binop_ldexp: result = nir_ldexp(&b, srcs[0], srcs[1]); break;
1728    case ir_triop_fma:
1729       result = nir_ffma(&b, srcs[0], srcs[1], srcs[2]);
1730       break;
1731    case ir_triop_lrp:
1732       result = nir_flrp(&b, srcs[0], srcs[1], srcs[2]);
1733       break;
1734    case ir_triop_csel:
1735       if (supports_ints)
1736          result = nir_bcsel(&b, srcs[0], srcs[1], srcs[2]);
1737       else
1738          result = nir_fcsel(&b, srcs[0], srcs[1], srcs[2]);
1739       break;
1740    case ir_triop_bitfield_extract:
1741       result = (out_type == GLSL_TYPE_INT) ?
1742          nir_ibitfield_extract(&b, srcs[0], srcs[1], srcs[2]) :
1743          nir_ubitfield_extract(&b, srcs[0], srcs[1], srcs[2]);
1744       break;
1745    case ir_quadop_bitfield_insert:
1746       result = nir_bitfield_insert(&b, srcs[0], srcs[1], srcs[2], srcs[3]);
1747       break;
1748    case ir_quadop_vector:
1749       result = nir_vec(&b, srcs, ir->type->vector_elements);
1750       break;
1751
1752    default:
1753       unreachable("not reached");
1754    }
1755 }
1756
1757 void
1758 nir_visitor::visit(ir_swizzle *ir)
1759 {
1760    unsigned swizzle[4] = { ir->mask.x, ir->mask.y, ir->mask.z, ir->mask.w };
1761    result = nir_swizzle(&b, evaluate_rvalue(ir->val), swizzle,
1762                         ir->type->vector_elements, !supports_ints);
1763 }
1764
1765 void
1766 nir_visitor::visit(ir_texture *ir)
1767 {
1768    unsigned num_srcs;
1769    nir_texop op;
1770    switch (ir->op) {
1771    case ir_tex:
1772       op = nir_texop_tex;
1773       num_srcs = 1; /* coordinate */
1774       break;
1775
1776    case ir_txb:
1777    case ir_txl:
1778       op = (ir->op == ir_txb) ? nir_texop_txb : nir_texop_txl;
1779       num_srcs = 2; /* coordinate, bias/lod */
1780       break;
1781
1782    case ir_txd:
1783       op = nir_texop_txd; /* coordinate, dPdx, dPdy */
1784       num_srcs = 3;
1785       break;
1786
1787    case ir_txf:
1788       op = nir_texop_txf;
1789       if (ir->lod_info.lod != NULL)
1790          num_srcs = 2; /* coordinate, lod */
1791       else
1792          num_srcs = 1; /* coordinate */
1793       break;
1794
1795    case ir_txf_ms:
1796       op = nir_texop_txf_ms;
1797       num_srcs = 2; /* coordinate, sample_index */
1798       break;
1799
1800    case ir_txs:
1801       op = nir_texop_txs;
1802       if (ir->lod_info.lod != NULL)
1803          num_srcs = 1; /* lod */
1804       else
1805          num_srcs = 0;
1806       break;
1807
1808    case ir_lod:
1809       op = nir_texop_lod;
1810       num_srcs = 1; /* coordinate */
1811       break;
1812
1813    case ir_tg4:
1814       op = nir_texop_tg4;
1815       num_srcs = 1; /* coordinate */
1816       break;
1817
1818    case ir_query_levels:
1819       op = nir_texop_query_levels;
1820       num_srcs = 0;
1821       break;
1822
1823    case ir_texture_samples:
1824       op = nir_texop_texture_samples;
1825       num_srcs = 0;
1826       break;
1827
1828    case ir_samples_identical:
1829       op = nir_texop_samples_identical;
1830       num_srcs = 1; /* coordinate */
1831       break;
1832
1833    default:
1834       unreachable("not reached");
1835    }
1836
1837    if (ir->projector != NULL)
1838       num_srcs++;
1839    if (ir->shadow_comparitor != NULL)
1840       num_srcs++;
1841    if (ir->offset != NULL)
1842       num_srcs++;
1843
1844    nir_tex_instr *instr = nir_tex_instr_create(this->shader, num_srcs);
1845
1846    instr->op = op;
1847    instr->sampler_dim =
1848       (glsl_sampler_dim) ir->sampler->type->sampler_dimensionality;
1849    instr->is_array = ir->sampler->type->sampler_array;
1850    instr->is_shadow = ir->sampler->type->sampler_shadow;
1851    if (instr->is_shadow)
1852       instr->is_new_style_shadow = (ir->type->vector_elements == 1);
1853    switch (ir->type->base_type) {
1854    case GLSL_TYPE_FLOAT:
1855       instr->dest_type = nir_type_float;
1856       break;
1857    case GLSL_TYPE_INT:
1858       instr->dest_type = nir_type_int;
1859       break;
1860    case GLSL_TYPE_BOOL:
1861    case GLSL_TYPE_UINT:
1862       instr->dest_type = nir_type_uint;
1863       break;
1864    default:
1865       unreachable("not reached");
1866    }
1867
1868    instr->texture = evaluate_deref(&instr->instr, ir->sampler);
1869
1870    unsigned src_number = 0;
1871
1872    if (ir->coordinate != NULL) {
1873       instr->coord_components = ir->coordinate->type->vector_elements;
1874       instr->src[src_number].src =
1875          nir_src_for_ssa(evaluate_rvalue(ir->coordinate));
1876       instr->src[src_number].src_type = nir_tex_src_coord;
1877       src_number++;
1878    }
1879
1880    if (ir->projector != NULL) {
1881       instr->src[src_number].src =
1882          nir_src_for_ssa(evaluate_rvalue(ir->projector));
1883       instr->src[src_number].src_type = nir_tex_src_projector;
1884       src_number++;
1885    }
1886
1887    if (ir->shadow_comparitor != NULL) {
1888       instr->src[src_number].src =
1889          nir_src_for_ssa(evaluate_rvalue(ir->shadow_comparitor));
1890       instr->src[src_number].src_type = nir_tex_src_comparitor;
1891       src_number++;
1892    }
1893
1894    if (ir->offset != NULL) {
1895       /* we don't support multiple offsets yet */
1896       assert(ir->offset->type->is_vector() || ir->offset->type->is_scalar());
1897
1898       instr->src[src_number].src =
1899          nir_src_for_ssa(evaluate_rvalue(ir->offset));
1900       instr->src[src_number].src_type = nir_tex_src_offset;
1901       src_number++;
1902    }
1903
1904    switch (ir->op) {
1905    case ir_txb:
1906       instr->src[src_number].src =
1907          nir_src_for_ssa(evaluate_rvalue(ir->lod_info.bias));
1908       instr->src[src_number].src_type = nir_tex_src_bias;
1909       src_number++;
1910       break;
1911
1912    case ir_txl:
1913    case ir_txf:
1914    case ir_txs:
1915       if (ir->lod_info.lod != NULL) {
1916          instr->src[src_number].src =
1917             nir_src_for_ssa(evaluate_rvalue(ir->lod_info.lod));
1918          instr->src[src_number].src_type = nir_tex_src_lod;
1919          src_number++;
1920       }
1921       break;
1922
1923    case ir_txd:
1924       instr->src[src_number].src =
1925          nir_src_for_ssa(evaluate_rvalue(ir->lod_info.grad.dPdx));
1926       instr->src[src_number].src_type = nir_tex_src_ddx;
1927       src_number++;
1928       instr->src[src_number].src =
1929          nir_src_for_ssa(evaluate_rvalue(ir->lod_info.grad.dPdy));
1930       instr->src[src_number].src_type = nir_tex_src_ddy;
1931       src_number++;
1932       break;
1933
1934    case ir_txf_ms:
1935       instr->src[src_number].src =
1936          nir_src_for_ssa(evaluate_rvalue(ir->lod_info.sample_index));
1937       instr->src[src_number].src_type = nir_tex_src_ms_index;
1938       src_number++;
1939       break;
1940
1941    case ir_tg4:
1942       instr->component = ir->lod_info.component->as_constant()->value.u[0];
1943       break;
1944
1945    default:
1946       break;
1947    }
1948
1949    assert(src_number == num_srcs);
1950
1951    add_instr(&instr->instr, nir_tex_instr_dest_size(instr));
1952 }
1953
1954 void
1955 nir_visitor::visit(ir_constant *ir)
1956 {
1957    /*
1958     * We don't know if this variable is an an array or struct that gets
1959     * dereferenced, so do the safe thing an make it a variable with a
1960     * constant initializer and return a dereference.
1961     */
1962
1963    nir_variable *var =
1964       nir_local_variable_create(this->impl, ir->type, "const_temp");
1965    var->data.read_only = true;
1966    var->constant_initializer = constant_copy(ir, var);
1967
1968    this->deref_head = nir_deref_var_create(this->shader, var);
1969    this->deref_tail = &this->deref_head->deref;
1970 }
1971
1972 void
1973 nir_visitor::visit(ir_dereference_variable *ir)
1974 {
1975    struct hash_entry *entry =
1976       _mesa_hash_table_search(this->var_table, ir->var);
1977    assert(entry);
1978    nir_variable *var = (nir_variable *) entry->data;
1979
1980    nir_deref_var *deref = nir_deref_var_create(this->shader, var);
1981    this->deref_head = deref;
1982    this->deref_tail = &deref->deref;
1983 }
1984
1985 void
1986 nir_visitor::visit(ir_dereference_record *ir)
1987 {
1988    ir->record->accept(this);
1989
1990    int field_index = this->deref_tail->type->field_index(ir->field);
1991    assert(field_index >= 0);
1992
1993    nir_deref_struct *deref = nir_deref_struct_create(this->deref_tail, field_index);
1994    deref->deref.type = ir->type;
1995    this->deref_tail->child = &deref->deref;
1996    this->deref_tail = &deref->deref;
1997 }
1998
1999 void
2000 nir_visitor::visit(ir_dereference_array *ir)
2001 {
2002    nir_deref_array *deref = nir_deref_array_create(this->shader);
2003    deref->deref.type = ir->type;
2004
2005    ir_constant *const_index = ir->array_index->as_constant();
2006    if (const_index != NULL) {
2007       deref->deref_array_type = nir_deref_array_type_direct;
2008       deref->base_offset = const_index->value.u[0];
2009    } else {
2010       deref->deref_array_type = nir_deref_array_type_indirect;
2011       deref->indirect =
2012          nir_src_for_ssa(evaluate_rvalue(ir->array_index));
2013    }
2014
2015    ir->array->accept(this);
2016
2017    this->deref_tail->child = &deref->deref;
2018    ralloc_steal(this->deref_tail, deref);
2019    this->deref_tail = &deref->deref;
2020 }
2021
2022 void
2023 nir_visitor::visit(ir_barrier *ir)
2024 {
2025    nir_intrinsic_instr *instr =
2026       nir_intrinsic_instr_create(this->shader, nir_intrinsic_barrier);
2027    nir_builder_instr_insert(&b, &instr->instr);
2028 }