src/mesa/drivers/dri/i965/brw_nir.c

   1 /*
   2  * Copyright © 2014 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include "brw_nir.h"
  25 #include "brw_shader.h"
  26 #include "compiler/nir/glsl_to_nir.h"
  27 #include "compiler/nir/nir_builder.h"
  28 #include "program/prog_to_nir.h"
  29
  30 static bool
  31 is_input(nir_intrinsic_instr *intrin)
  32 {
  33    return intrin->intrinsic == nir_intrinsic_load_input ||
  34           intrin->intrinsic == nir_intrinsic_load_per_vertex_input;
  35 }
  36
  37 static bool
  38 is_output(nir_intrinsic_instr *intrin)
  39 {
  40    return intrin->intrinsic == nir_intrinsic_load_output ||
  41           intrin->intrinsic == nir_intrinsic_load_per_vertex_output ||
  42           intrin->intrinsic == nir_intrinsic_store_output ||
  43           intrin->intrinsic == nir_intrinsic_store_per_vertex_output;
  44 }
  45
  46 /**
  47  * In many cases, we just add the base and offset together, so there's no
  48  * reason to keep them separate.  Sometimes, combining them is essential:
  49  * if a shader only accesses part of a compound variable (such as a matrix
  50  * or array), the variable's base may not actually exist in the VUE map.
  51  *
  52  * This pass adds constant offsets to instr->const_index[0], and resets
  53  * the offset source to 0.  Non-constant offsets remain unchanged - since
  54  * we don't know what part of a compound variable is accessed, we allocate
  55  * storage for the entire thing.
  56  */
  57 struct add_const_offset_to_base_params {
  58    nir_builder b;
  59    nir_variable_mode mode;
  60 };
  61
  62 static bool
  63 add_const_offset_to_base_block(nir_block *block, void *closure)
  64 {
  65    struct add_const_offset_to_base_params *params = closure;
  66    nir_builder *b = &params->b;
  67
  68    nir_foreach_instr_safe(block, instr) {
  69       if (instr->type != nir_instr_type_intrinsic)
  70          continue;
  71
  72       nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
  73
  74       if ((params->mode == nir_var_shader_in && is_input(intrin)) ||
  75           (params->mode == nir_var_shader_out && is_output(intrin))) {
  76          nir_src *offset = nir_get_io_offset_src(intrin);
  77          nir_const_value *const_offset = nir_src_as_const_value(*offset);
  78
  79          if (const_offset) {
  80             intrin->const_index[0] += const_offset->u[0];
  81             b->cursor = nir_before_instr(&intrin->instr);
  82             nir_instr_rewrite_src(&intrin->instr, offset,
  83                                   nir_src_for_ssa(nir_imm_int(b, 0)));
  84          }
  85       }
  86    }
  87    return true;
  88 }
  89
  90 static void
  91 add_const_offset_to_base(nir_shader *nir, nir_variable_mode mode)
  92 {
  93    struct add_const_offset_to_base_params params = { .mode = mode };
  94
  95    nir_foreach_function(nir, f) {
  96       if (f->impl) {
  97          nir_builder_init(&params.b, f->impl);
  98          nir_foreach_block(f->impl, add_const_offset_to_base_block, &params);
  99       }
 100    }
 101 }
 102
 103 static bool
 104 remap_vs_attrs(nir_block *block, void *closure)
 105 {
 106    GLbitfield64 inputs_read = *((GLbitfield64 *) closure);
 107
 108    nir_foreach_instr(block, instr) {
 109       if (instr->type != nir_instr_type_intrinsic)
 110          continue;
 111
 112       nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
 113
 114       if (intrin->intrinsic == nir_intrinsic_load_input) {
 115          /* Attributes come in a contiguous block, ordered by their
 116           * gl_vert_attrib value.  That means we can compute the slot
 117           * number for an attribute by masking out the enabled attributes
 118           * before it and counting the bits.
 119           */
 120          int attr = intrin->const_index[0];
 121          int slot = _mesa_bitcount_64(inputs_read & BITFIELD64_MASK(attr));
 122
 123          intrin->const_index[0] = 4 * slot;
 124       }
 125    }
 126    return true;
 127 }
 128
 129 static bool
 130 remap_inputs_with_vue_map(nir_block *block, void *closure)
 131 {
 132    const struct brw_vue_map *vue_map = closure;
 133
 134    nir_foreach_instr(block, instr) {
 135       if (instr->type != nir_instr_type_intrinsic)
 136          continue;
 137
 138       nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
 139
 140       if (intrin->intrinsic == nir_intrinsic_load_input ||
 141           intrin->intrinsic == nir_intrinsic_load_per_vertex_input) {
 142          int vue_slot = vue_map->varying_to_slot[intrin->const_index[0]];
 143          assert(vue_slot != -1);
 144          intrin->const_index[0] = vue_slot;
 145       }
 146    }
 147    return true;
 148 }
 149
 150 struct remap_patch_urb_offsets_state {
 151    nir_builder b;
 152    struct brw_vue_map vue_map;
 153 };
 154
 155 static bool
 156 remap_patch_urb_offsets(nir_block *block, void *closure)
 157 {
 158    struct remap_patch_urb_offsets_state *state = closure;
 159
 160    nir_foreach_instr_safe(block, instr) {
 161       if (instr->type != nir_instr_type_intrinsic)
 162          continue;
 163
 164       nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
 165
 166       gl_shader_stage stage = state->b.shader->stage;
 167
 168       if ((stage == MESA_SHADER_TESS_CTRL && is_output(intrin)) ||
 169           (stage == MESA_SHADER_TESS_EVAL && is_input(intrin))) {
 170          int vue_slot = state->vue_map.varying_to_slot[intrin->const_index[0]];
 171          assert(vue_slot != -1);
 172          intrin->const_index[0] = vue_slot;
 173
 174          nir_src *vertex = nir_get_io_vertex_index_src(intrin);
 175          if (vertex) {
 176             nir_const_value *const_vertex = nir_src_as_const_value(*vertex);
 177             if (const_vertex) {
 178                intrin->const_index[0] += const_vertex->u[0] *
 179                                          state->vue_map.num_per_vertex_slots;
 180             } else {
 181                state->b.cursor = nir_before_instr(&intrin->instr);
 182
 183                /* Multiply by the number of per-vertex slots. */
 184                nir_ssa_def *vertex_offset =
 185                   nir_imul(&state->b,
 186                            nir_ssa_for_src(&state->b, *vertex, 1),
 187                            nir_imm_int(&state->b,
 188                                        state->vue_map.num_per_vertex_slots));
 189
 190                /* Add it to the existing offset */
 191                nir_src *offset = nir_get_io_offset_src(intrin);
 192                nir_ssa_def *total_offset =
 193                   nir_iadd(&state->b, vertex_offset,
 194                            nir_ssa_for_src(&state->b, *offset, 1));
 195
 196                nir_instr_rewrite_src(&intrin->instr, offset,
 197                                      nir_src_for_ssa(total_offset));
 198             }
 199          }
 200       }
 201    }
 202    return true;
 203 }
 204
 205 static void
 206 brw_nir_lower_inputs(nir_shader *nir,
 207                      const struct brw_device_info *devinfo,
 208                      bool is_scalar,
 209                      bool use_legacy_snorm_formula,
 210                      const uint8_t *vs_attrib_wa_flags)
 211 {
 212    switch (nir->stage) {
 213    case MESA_SHADER_VERTEX:
 214       /* Start with the location of the variable's base. */
 215       foreach_list_typed(nir_variable, var, node, &nir->inputs) {
 216          var->data.driver_location = var->data.location;
 217       }
 218
 219       /* Now use nir_lower_io to walk dereference chains.  Attribute arrays
 220        * are loaded as one vec4 per element (or matrix column), so we use
 221        * type_size_vec4 here.
 222        */
 223       nir_lower_io(nir, nir_var_shader_in, type_size_vec4);
 224
 225       /* This pass needs actual constants */
 226       nir_opt_constant_folding(nir);
 227
 228       add_const_offset_to_base(nir, nir_var_shader_in);
 229
 230       brw_nir_apply_attribute_workarounds(nir, use_legacy_snorm_formula,
 231                                           vs_attrib_wa_flags);
 232
 233       if (is_scalar) {
 234          /* Finally, translate VERT_ATTRIB_* values into the actual registers.
 235           *
 236           * Note that we can use nir->info.inputs_read instead of
 237           * key->inputs_read since the two are identical aside from Gen4-5
 238           * edge flag differences.
 239           */
 240          GLbitfield64 inputs_read = nir->info.inputs_read;
 241
 242          nir_foreach_function(nir, function) {
 243             if (function->impl) {
 244                nir_foreach_block(function->impl, remap_vs_attrs, &inputs_read);
 245             }
 246          }
 247       }
 248       break;
 249    case MESA_SHADER_TESS_CTRL:
 250    case MESA_SHADER_GEOMETRY: {
 251       if (!is_scalar && nir->stage == MESA_SHADER_GEOMETRY) {
 252          foreach_list_typed(nir_variable, var, node, &nir->inputs) {
 253             var->data.driver_location = var->data.location;
 254          }
 255       } else {
 256          /* The GLSL linker will have already matched up GS inputs and
 257           * the outputs of prior stages.  The driver does extend VS outputs
 258           * in some cases, but only for legacy OpenGL or Gen4-5 hardware,
 259           * neither of which offer geometry shader support.  So we can
 260           * safely ignore that.
 261           *
 262           * For SSO pipelines, we use a fixed VUE map layout based on variable
 263           * locations, so we can rely on rendezvous-by-location to make this
 264           * work.
 265           *
 266           * However, we need to ignore VARYING_SLOT_PRIMITIVE_ID, as it's not
 267           * written by previous stages and shows up via payload magic.
 268           */
 269          struct brw_vue_map input_vue_map;
 270          GLbitfield64 inputs_read =
 271             nir->info.inputs_read & ~VARYING_BIT_PRIMITIVE_ID;
 272          brw_compute_vue_map(devinfo, &input_vue_map, inputs_read,
 273                              nir->info.separate_shader ||
 274                              nir->stage == MESA_SHADER_TESS_CTRL);
 275
 276          foreach_list_typed(nir_variable, var, node, &nir->inputs) {
 277             var->data.driver_location = var->data.location;
 278          }
 279
 280          /* Inputs are stored in vec4 slots, so use type_size_vec4(). */
 281          nir_lower_io(nir, nir_var_shader_in, type_size_vec4);
 282
 283          /* This pass needs actual constants */
 284          nir_opt_constant_folding(nir);
 285
 286          add_const_offset_to_base(nir, nir_var_shader_in);
 287
 288          nir_foreach_function(nir, function) {
 289             if (function->impl) {
 290                nir_foreach_block(function->impl, remap_inputs_with_vue_map,
 291                                  &input_vue_map);
 292             }
 293          }
 294       }
 295       break;
 296    }
 297    case MESA_SHADER_TESS_EVAL: {
 298       struct remap_patch_urb_offsets_state state;
 299       brw_compute_tess_vue_map(&state.vue_map,
 300                                nir->info.inputs_read & ~VARYING_BIT_PRIMITIVE_ID,
 301                                nir->info.patch_inputs_read);
 302
 303       foreach_list_typed(nir_variable, var, node, &nir->inputs) {
 304          var->data.driver_location = var->data.location;
 305       }
 306
 307       nir_lower_io(nir, nir_var_shader_in, type_size_vec4);
 308
 309       /* This pass needs actual constants */
 310       nir_opt_constant_folding(nir);
 311
 312       add_const_offset_to_base(nir, nir_var_shader_in);
 313
 314       nir_foreach_function(nir, function) {
 315          if (function->impl) {
 316             nir_builder_init(&state.b, function->impl);
 317             nir_foreach_block(function->impl, remap_patch_urb_offsets, &state);
 318          }
 319       }
 320       break;
 321    }
 322    case MESA_SHADER_FRAGMENT:
 323       assert(is_scalar);
 324       nir_assign_var_locations(&nir->inputs, &nir->num_inputs,
 325                                type_size_scalar);
 326       break;
 327    case MESA_SHADER_COMPUTE:
 328       /* Compute shaders have no inputs. */
 329       assert(exec_list_is_empty(&nir->inputs));
 330       break;
 331    default:
 332       unreachable("unsupported shader stage");
 333    }
 334 }
 335
 336 static void
 337 brw_nir_lower_outputs(nir_shader *nir,
 338                       const struct brw_device_info *devinfo,
 339                       bool is_scalar)
 340 {
 341    switch (nir->stage) {
 342    case MESA_SHADER_VERTEX:
 343    case MESA_SHADER_TESS_EVAL:
 344    case MESA_SHADER_GEOMETRY:
 345       if (is_scalar) {
 346          nir_assign_var_locations(&nir->outputs, &nir->num_outputs,
 347                                   type_size_vec4_times_4);
 348          nir_lower_io(nir, nir_var_shader_out, type_size_vec4_times_4);
 349       } else {
 350          nir_foreach_variable(var, &nir->outputs)
 351             var->data.driver_location = var->data.location;
 352       }
 353       break;
 354    case MESA_SHADER_TESS_CTRL: {
 355       struct remap_patch_urb_offsets_state state;
 356       brw_compute_tess_vue_map(&state.vue_map, nir->info.outputs_written,
 357                                nir->info.patch_outputs_written);
 358
 359       nir_foreach_variable(var, &nir->outputs) {
 360          var->data.driver_location = var->data.location;
 361       }
 362
 363       nir_lower_io(nir, nir_var_shader_out, type_size_vec4);
 364
 365       /* This pass needs actual constants */
 366       nir_opt_constant_folding(nir);
 367
 368       add_const_offset_to_base(nir, nir_var_shader_out);
 369
 370       nir_foreach_function(nir, function) {
 371          if (function->impl) {
 372             nir_builder_init(&state.b, function->impl);
 373             nir_foreach_block(function->impl, remap_patch_urb_offsets, &state);
 374          }
 375       }
 376       break;
 377    }
 378    case MESA_SHADER_FRAGMENT:
 379       nir_assign_var_locations(&nir->outputs, &nir->num_outputs,
 380                                type_size_scalar);
 381       break;
 382    case MESA_SHADER_COMPUTE:
 383       /* Compute shaders have no outputs. */
 384       assert(exec_list_is_empty(&nir->outputs));
 385       break;
 386    default:
 387       unreachable("unsupported shader stage");
 388    }
 389 }
 390
 391 static int
 392 type_size_scalar_bytes(const struct glsl_type *type)
 393 {
 394    return type_size_scalar(type) * 4;
 395 }
 396
 397 static int
 398 type_size_vec4_bytes(const struct glsl_type *type)
 399 {
 400    return type_size_vec4(type) * 16;
 401 }
 402
 403 static void
 404 brw_nir_lower_uniforms(nir_shader *nir, bool is_scalar)
 405 {
 406    if (is_scalar) {
 407       nir_assign_var_locations(&nir->uniforms, &nir->num_uniforms,
 408                                type_size_scalar_bytes);
 409       nir_lower_io(nir, nir_var_uniform, type_size_scalar_bytes);
 410    } else {
 411       nir_assign_var_locations(&nir->uniforms, &nir->num_uniforms,
 412                                type_size_vec4_bytes);
 413       nir_lower_io(nir, nir_var_uniform, type_size_vec4_bytes);
 414    }
 415 }
 416
 417 static void
 418 brw_nir_lower_shared(nir_shader *nir)
 419 {
 420    nir_assign_var_locations(&nir->shared, &nir->num_shared,
 421                             type_size_scalar_bytes);
 422    nir_lower_io(nir, nir_var_shared, type_size_scalar_bytes);
 423 }
 424
 425 #define OPT(pass, ...) ({                                  \
 426    bool this_progress = false;                             \
 427    NIR_PASS(this_progress, nir, pass, ##__VA_ARGS__);      \
 428    if (this_progress)                                      \
 429       progress = true;                                     \
 430    this_progress;                                          \
 431 })
 432
 433 #define OPT_V(pass, ...) NIR_PASS_V(nir, pass, ##__VA_ARGS__)
 434
 435 static nir_shader *
 436 nir_optimize(nir_shader *nir, bool is_scalar)
 437 {
 438    bool progress;
 439    do {
 440       progress = false;
 441       OPT_V(nir_lower_vars_to_ssa);
 442
 443       if (is_scalar) {
 444          OPT_V(nir_lower_alu_to_scalar);
 445       }
 446
 447       OPT(nir_copy_prop);
 448
 449       if (is_scalar) {
 450          OPT_V(nir_lower_phis_to_scalar);
 451       }
 452
 453       OPT(nir_copy_prop);
 454       OPT(nir_opt_dce);
 455       OPT(nir_opt_cse);
 456       OPT(nir_opt_peephole_select);
 457       OPT(nir_opt_algebraic);
 458       OPT(nir_opt_constant_folding);
 459       OPT(nir_opt_dead_cf);
 460       OPT(nir_opt_remove_phis);
 461       OPT(nir_opt_undef);
 462    } while (progress);
 463
 464    return nir;
 465 }
 466
 467 /* Does some simple lowering and runs the standard suite of optimizations
 468  *
 469  * This is intended to be called more-or-less directly after you get the
 470  * shader out of GLSL or some other source.  While it is geared towards i965,
 471  * it is not at all generator-specific except for the is_scalar flag.  Even
 472  * there, it is safe to call with is_scalar = false for a shader that is
 473  * intended for the FS backend as long as nir_optimize is called again with
 474  * is_scalar = true to scalarize everything prior to code gen.
 475  */
 476 nir_shader *
 477 brw_preprocess_nir(nir_shader *nir, bool is_scalar)
 478 {
 479    bool progress; /* Written by OPT and OPT_V */
 480    (void)progress;
 481
 482    if (nir->stage == MESA_SHADER_GEOMETRY)
 483       OPT(nir_lower_gs_intrinsics);
 484
 485    static const nir_lower_tex_options tex_options = {
 486       .lower_txp = ~0,
 487    };
 488
 489    OPT(nir_lower_tex, &tex_options);
 490    OPT(nir_normalize_cubemap_coords);
 491
 492    OPT(nir_lower_global_vars_to_local);
 493
 494    OPT(nir_split_var_copies);
 495
 496    nir = nir_optimize(nir, is_scalar);
 497
 498    if (is_scalar) {
 499       OPT_V(nir_lower_load_const_to_scalar);
 500    }
 501
 502    /* Lower a bunch of stuff */
 503    OPT_V(nir_lower_var_copies);
 504
 505    /* Get rid of split copies */
 506    nir = nir_optimize(nir, is_scalar);
 507
 508    OPT(nir_remove_dead_variables, nir_var_local);
 509
 510    return nir;
 511 }
 512
 513 /** Lower input and output loads and stores for i965. */
 514 nir_shader *
 515 brw_nir_lower_io(nir_shader *nir,
 516                  const struct brw_device_info *devinfo,
 517                  bool is_scalar,
 518                  bool use_legacy_snorm_formula,
 519                  const uint8_t *vs_attrib_wa_flags)
 520 {
 521    bool progress; /* Written by OPT and OPT_V */
 522    (void)progress;
 523
 524    OPT_V(brw_nir_lower_inputs, devinfo, is_scalar,
 525          use_legacy_snorm_formula, vs_attrib_wa_flags);
 526    OPT_V(brw_nir_lower_outputs, devinfo, is_scalar);
 527    if (nir->stage == MESA_SHADER_COMPUTE)
 528       OPT_V(brw_nir_lower_shared);
 529    OPT_V(nir_lower_io, nir_var_all, is_scalar ? type_size_scalar : type_size_vec4);
 530
 531    return nir_optimize(nir, is_scalar);
 532 }
 533
 534 /* Prepare the given shader for codegen
 535  *
 536  * This function is intended to be called right before going into the actual
 537  * backend and is highly backend-specific.  Also, once this function has been
 538  * called on a shader, it will no longer be in SSA form so most optimizations
 539  * will not work.
 540  */
 541 nir_shader *
 542 brw_postprocess_nir(nir_shader *nir,
 543                     const struct brw_device_info *devinfo,
 544                     bool is_scalar)
 545 {
 546    bool debug_enabled =
 547       (INTEL_DEBUG & intel_debug_flag_for_shader_stage(nir->stage));
 548
 549    bool progress; /* Written by OPT and OPT_V */
 550    (void)progress;
 551
 552    if (devinfo->gen >= 6) {
 553       /* Try and fuse multiply-adds */
 554 //      OPT(brw_nir_opt_peephole_ffma);
 555    }
 556
 557    OPT(nir_opt_algebraic_late);
 558
 559    OPT(nir_lower_locals_to_regs);
 560
 561    OPT_V(nir_lower_to_source_mods);
 562    OPT(nir_copy_prop);
 563    OPT(nir_opt_dce);
 564
 565    if (unlikely(debug_enabled)) {
 566       /* Re-index SSA defs so we print more sensible numbers. */
 567       nir_foreach_function(nir, function) {
 568          if (function->impl)
 569             nir_index_ssa_defs(function->impl);
 570       }
 571
 572       fprintf(stderr, "NIR (SSA form) for %s shader:\n",
 573               _mesa_shader_stage_to_string(nir->stage));
 574       nir_print_shader(nir, stderr);
 575    }
 576
 577    OPT_V(nir_convert_from_ssa, true);
 578
 579    if (!is_scalar) {
 580       OPT_V(nir_move_vec_src_uses_to_dest);
 581       OPT(nir_lower_vec_to_movs);
 582    }
 583
 584    /* This is the last pass we run before we start emitting stuff.  It
 585     * determines when we need to insert boolean resolves on Gen <= 5.  We
 586     * run it last because it stashes data in instr->pass_flags and we don't
 587     * want that to be squashed by other NIR passes.
 588     */
 589    if (devinfo->gen <= 5)
 590       brw_nir_analyze_boolean_resolves(nir);
 591
 592    nir_sweep(nir);
 593
 594    if (unlikely(debug_enabled)) {
 595       fprintf(stderr, "NIR (final form) for %s shader:\n",
 596               _mesa_shader_stage_to_string(nir->stage));
 597       nir_print_shader(nir, stderr);
 598    }
 599
 600    return nir;
 601 }
 602
 603 nir_shader *
 604 brw_create_nir(struct brw_context *brw,
 605                const struct gl_shader_program *shader_prog,
 606                const struct gl_program *prog,
 607                gl_shader_stage stage,
 608                bool is_scalar)
 609 {
 610    struct gl_context *ctx = &brw->ctx;
 611    const struct brw_device_info *devinfo = brw->intelScreen->devinfo;
 612    const nir_shader_compiler_options *options =
 613       ctx->Const.ShaderCompilerOptions[stage].NirOptions;
 614    bool progress;
 615    nir_shader *nir;
 616
 617    /* First, lower the GLSL IR or Mesa IR to NIR */
 618    if (shader_prog) {
 619       nir = glsl_to_nir(shader_prog, stage, options);
 620    } else {
 621       nir = prog_to_nir(prog, options);
 622       OPT_V(nir_convert_to_ssa); /* turn registers into SSA */
 623    }
 624    nir_validate_shader(nir);
 625
 626    (void)progress;
 627
 628    nir = brw_preprocess_nir(nir, is_scalar);
 629
 630    OPT(nir_lower_system_values);
 631    OPT_V(brw_nir_lower_uniforms, is_scalar);
 632
 633    if (shader_prog) {
 634       OPT_V(nir_lower_samplers, shader_prog);
 635       OPT_V(nir_lower_atomics, shader_prog);
 636    }
 637
 638    if (nir->stage != MESA_SHADER_VERTEX &&
 639        nir->stage != MESA_SHADER_TESS_CTRL &&
 640        nir->stage != MESA_SHADER_TESS_EVAL) {
 641       nir = brw_nir_lower_io(nir, devinfo, is_scalar, false, NULL);
 642    }
 643
 644    return nir;
 645 }
 646
 647 nir_shader *
 648 brw_nir_apply_sampler_key(nir_shader *nir,
 649                           const struct brw_device_info *devinfo,
 650                           const struct brw_sampler_prog_key_data *key_tex,
 651                           bool is_scalar)
 652 {
 653    nir_lower_tex_options tex_options = { 0 };
 654
 655    /* Iron Lake and prior require lowering of all rectangle textures */
 656    if (devinfo->gen < 6)
 657       tex_options.lower_rect = true;
 658
 659    /* Prior to Broadwell, our hardware can't actually do GL_CLAMP */
 660    if (devinfo->gen < 8) {
 661       tex_options.saturate_s = key_tex->gl_clamp_mask[0];
 662       tex_options.saturate_t = key_tex->gl_clamp_mask[1];
 663       tex_options.saturate_r = key_tex->gl_clamp_mask[2];
 664    }
 665
 666    /* Prior to Haswell, we have to fake texture swizzle */
 667    for (unsigned s = 0; s < MAX_SAMPLERS; s++) {
 668       if (key_tex->swizzles[s] == SWIZZLE_NOOP)
 669          continue;
 670
 671       tex_options.swizzle_result |= (1 << s);
 672       for (unsigned c = 0; c < 4; c++)
 673          tex_options.swizzles[s][c] = GET_SWZ(key_tex->swizzles[s], c);
 674    }
 675
 676    if (nir_lower_tex(nir, &tex_options)) {
 677       nir_validate_shader(nir);
 678       nir = nir_optimize(nir, is_scalar);
 679    }
 680
 681    return nir;
 682 }
 683
 684 enum brw_reg_type
 685 brw_type_for_nir_type(nir_alu_type type)
 686 {
 687    switch (type) {
 688    case nir_type_uint:
 689       return BRW_REGISTER_TYPE_UD;
 690    case nir_type_bool:
 691    case nir_type_int:
 692       return BRW_REGISTER_TYPE_D;
 693    case nir_type_float:
 694       return BRW_REGISTER_TYPE_F;
 695    default:
 696       unreachable("unknown type");
 697    }
 698
 699    return BRW_REGISTER_TYPE_F;
 700 }
 701
 702 /* Returns the glsl_base_type corresponding to a nir_alu_type.
 703  * This is used by both brw_vec4_nir and brw_fs_nir.
 704  */
 705 enum glsl_base_type
 706 brw_glsl_base_type_for_nir_type(nir_alu_type type)
 707 {
 708    switch (type) {
 709    case nir_type_float:
 710       return GLSL_TYPE_FLOAT;
 711
 712    case nir_type_int:
 713       return GLSL_TYPE_INT;
 714
 715    case nir_type_uint:
 716       return GLSL_TYPE_UINT;
 717
 718    default:
 719       unreachable("bad type");
 720    }
 721 }