src/intel/compiler/brw_nir.c

   1 /*
   2  * Copyright © 2014 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include "brw_nir.h"
  25 #include "brw_shader.h"
  26 #include "common/gen_debug.h"
  27 #include "compiler/glsl_types.h"
  28 #include "compiler/nir/nir_builder.h"
  29
  30 static bool
  31 is_input(nir_intrinsic_instr *intrin)
  32 {
  33    return intrin->intrinsic == nir_intrinsic_load_input ||
  34           intrin->intrinsic == nir_intrinsic_load_per_vertex_input ||
  35           intrin->intrinsic == nir_intrinsic_load_interpolated_input;
  36 }
  37
  38 static bool
  39 is_output(nir_intrinsic_instr *intrin)
  40 {
  41    return intrin->intrinsic == nir_intrinsic_load_output ||
  42           intrin->intrinsic == nir_intrinsic_load_per_vertex_output ||
  43           intrin->intrinsic == nir_intrinsic_store_output ||
  44           intrin->intrinsic == nir_intrinsic_store_per_vertex_output;
  45 }
  46
  47 /**
  48  * In many cases, we just add the base and offset together, so there's no
  49  * reason to keep them separate.  Sometimes, combining them is essential:
  50  * if a shader only accesses part of a compound variable (such as a matrix
  51  * or array), the variable's base may not actually exist in the VUE map.
  52  *
  53  * This pass adds constant offsets to instr->const_index[0], and resets
  54  * the offset source to 0.  Non-constant offsets remain unchanged - since
  55  * we don't know what part of a compound variable is accessed, we allocate
  56  * storage for the entire thing.
  57  */
  58
  59 static bool
  60 add_const_offset_to_base_block(nir_block *block, nir_builder *b,
  61                                nir_variable_mode mode)
  62 {
  63    nir_foreach_instr_safe(instr, block) {
  64       if (instr->type != nir_instr_type_intrinsic)
  65          continue;
  66
  67       nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
  68
  69       if ((mode == nir_var_shader_in && is_input(intrin)) ||
  70           (mode == nir_var_shader_out && is_output(intrin))) {
  71          nir_src *offset = nir_get_io_offset_src(intrin);
  72          nir_const_value *const_offset = nir_src_as_const_value(*offset);
  73
  74          if (const_offset) {
  75             intrin->const_index[0] += const_offset->u32[0];
  76             b->cursor = nir_before_instr(&intrin->instr);
  77             nir_instr_rewrite_src(&intrin->instr, offset,
  78                                   nir_src_for_ssa(nir_imm_int(b, 0)));
  79          }
  80       }
  81    }
  82    return true;
  83 }
  84
  85 static void
  86 add_const_offset_to_base(nir_shader *nir, nir_variable_mode mode)
  87 {
  88    nir_foreach_function(f, nir) {
  89       if (f->impl) {
  90          nir_builder b;
  91          nir_builder_init(&b, f->impl);
  92          nir_foreach_block(block, f->impl) {
  93             add_const_offset_to_base_block(block, &b, mode);
  94          }
  95       }
  96    }
  97 }
  98
  99 static bool
 100 remap_inputs_with_vue_map(nir_block *block, const struct brw_vue_map *vue_map)
 101 {
 102    nir_foreach_instr(instr, block) {
 103       if (instr->type != nir_instr_type_intrinsic)
 104          continue;
 105
 106       nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
 107
 108       if (intrin->intrinsic == nir_intrinsic_load_input ||
 109           intrin->intrinsic == nir_intrinsic_load_per_vertex_input) {
 110          int vue_slot = vue_map->varying_to_slot[intrin->const_index[0]];
 111          assert(vue_slot != -1);
 112          intrin->const_index[0] = vue_slot;
 113       }
 114    }
 115    return true;
 116 }
 117
 118 static bool
 119 remap_tess_levels(nir_builder *b, nir_intrinsic_instr *intr,
 120                   GLenum primitive_mode)
 121 {
 122    const int location = nir_intrinsic_base(intr);
 123    const unsigned component = nir_intrinsic_component(intr);
 124    bool out_of_bounds;
 125
 126    if (location == VARYING_SLOT_TESS_LEVEL_INNER) {
 127       switch (primitive_mode) {
 128       case GL_QUADS:
 129          /* gl_TessLevelInner[0..1] lives at DWords 3-2 (reversed). */
 130          nir_intrinsic_set_base(intr, 0);
 131          nir_intrinsic_set_component(intr, 3 - component);
 132          out_of_bounds = false;
 133          break;
 134       case GL_TRIANGLES:
 135          /* gl_TessLevelInner[0] lives at DWord 4. */
 136          nir_intrinsic_set_base(intr, 1);
 137          out_of_bounds = component > 0;
 138          break;
 139       case GL_ISOLINES:
 140          out_of_bounds = true;
 141          break;
 142       default:
 143          unreachable("Bogus tessellation domain");
 144       }
 145    } else if (location == VARYING_SLOT_TESS_LEVEL_OUTER) {
 146       if (primitive_mode == GL_ISOLINES) {
 147          /* gl_TessLevelOuter[0..1] lives at DWords 6-7 (in order). */
 148          nir_intrinsic_set_base(intr, 1);
 149          nir_intrinsic_set_component(intr, 2 + nir_intrinsic_component(intr));
 150          out_of_bounds = component > 1;
 151       } else {
 152          /* Triangles use DWords 7-5 (reversed); Quads use 7-4 (reversed) */
 153          nir_intrinsic_set_base(intr, 1);
 154          nir_intrinsic_set_component(intr, 3 - nir_intrinsic_component(intr));
 155          out_of_bounds = component == 3 && primitive_mode == GL_TRIANGLES;
 156       }
 157    } else {
 158       return false;
 159    }
 160
 161    if (out_of_bounds) {
 162       if (nir_intrinsic_infos[intr->intrinsic].has_dest) {
 163          b->cursor = nir_before_instr(&intr->instr);
 164          nir_ssa_def *undef = nir_ssa_undef(b, 1, 32);
 165          nir_ssa_def_rewrite_uses(&intr->dest.ssa, nir_src_for_ssa(undef));
 166       }
 167       nir_instr_remove(&intr->instr);
 168    }
 169
 170    return true;
 171 }
 172
 173 static bool
 174 remap_patch_urb_offsets(nir_block *block, nir_builder *b,
 175                         const struct brw_vue_map *vue_map,
 176                         GLenum tes_primitive_mode)
 177 {
 178    const bool is_passthrough_tcs = b->shader->info.name &&
 179       strcmp(b->shader->info.name, "passthrough") == 0;
 180
 181    nir_foreach_instr_safe(instr, block) {
 182       if (instr->type != nir_instr_type_intrinsic)
 183          continue;
 184
 185       nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
 186
 187       gl_shader_stage stage = b->shader->stage;
 188
 189       if ((stage == MESA_SHADER_TESS_CTRL && is_output(intrin)) ||
 190           (stage == MESA_SHADER_TESS_EVAL && is_input(intrin))) {
 191
 192          if (!is_passthrough_tcs &&
 193              remap_tess_levels(b, intrin, tes_primitive_mode))
 194             continue;
 195
 196          int vue_slot = vue_map->varying_to_slot[intrin->const_index[0]];
 197          assert(vue_slot != -1);
 198          intrin->const_index[0] = vue_slot;
 199
 200          nir_src *vertex = nir_get_io_vertex_index_src(intrin);
 201          if (vertex) {
 202             nir_const_value *const_vertex = nir_src_as_const_value(*vertex);
 203             if (const_vertex) {
 204                intrin->const_index[0] += const_vertex->u32[0] *
 205                                          vue_map->num_per_vertex_slots;
 206             } else {
 207                b->cursor = nir_before_instr(&intrin->instr);
 208
 209                /* Multiply by the number of per-vertex slots. */
 210                nir_ssa_def *vertex_offset =
 211                   nir_imul(b,
 212                            nir_ssa_for_src(b, *vertex, 1),
 213                            nir_imm_int(b,
 214                                        vue_map->num_per_vertex_slots));
 215
 216                /* Add it to the existing offset */
 217                nir_src *offset = nir_get_io_offset_src(intrin);
 218                nir_ssa_def *total_offset =
 219                   nir_iadd(b, vertex_offset,
 220                            nir_ssa_for_src(b, *offset, 1));
 221
 222                nir_instr_rewrite_src(&intrin->instr, offset,
 223                                      nir_src_for_ssa(total_offset));
 224             }
 225          }
 226       }
 227    }
 228    return true;
 229 }
 230
 231 void
 232 brw_nir_lower_vs_inputs(nir_shader *nir,
 233                         bool is_scalar,
 234                         bool use_legacy_snorm_formula,
 235                         const uint8_t *vs_attrib_wa_flags)
 236 {
 237    /* Start with the location of the variable's base. */
 238    foreach_list_typed(nir_variable, var, node, &nir->inputs) {
 239       var->data.driver_location = var->data.location;
 240    }
 241
 242    /* Now use nir_lower_io to walk dereference chains.  Attribute arrays are
 243     * loaded as one vec4 or dvec4 per element (or matrix column), depending on
 244     * whether it is a double-precision type or not.
 245     */
 246    nir_lower_io(nir, nir_var_shader_in, type_size_vec4, 0);
 247
 248    /* This pass needs actual constants */
 249    nir_opt_constant_folding(nir);
 250
 251    add_const_offset_to_base(nir, nir_var_shader_in);
 252
 253    brw_nir_apply_attribute_workarounds(nir, use_legacy_snorm_formula,
 254                                        vs_attrib_wa_flags);
 255
 256    /* The last step is to remap VERT_ATTRIB_* to actual registers and we only
 257     * do that for scalar shaders at the moment.
 258     */
 259    if (!is_scalar)
 260       return;
 261
 262    nir_foreach_function(function, nir) {
 263       if (!function->impl)
 264          continue;
 265
 266       nir_foreach_block(block, function->impl) {
 267          nir_foreach_instr(instr, block) {
 268             if (instr->type != nir_instr_type_intrinsic)
 269                continue;
 270
 271             nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
 272
 273             if (intrin->intrinsic == nir_intrinsic_load_input) {
 274                /* Attributes come in a contiguous block, ordered by their
 275                 * gl_vert_attrib value.  That means we can compute the slot
 276                 * number for an attribute by masking out the enabled attributes
 277                 * before it and counting the bits.
 278                 */
 279                int attr = nir_intrinsic_base(intrin);
 280                int slot = _mesa_bitcount_64(nir->info.inputs_read &
 281                                             BITFIELD64_MASK(attr));
 282                nir_intrinsic_set_base(intrin, 4 * slot);
 283             }
 284          }
 285       }
 286    }
 287 }
 288
 289 void
 290 brw_nir_lower_vue_inputs(nir_shader *nir, bool is_scalar,
 291                          const struct brw_vue_map *vue_map)
 292 {
 293    foreach_list_typed(nir_variable, var, node, &nir->inputs) {
 294       var->data.driver_location = var->data.location;
 295    }
 296
 297    /* Inputs are stored in vec4 slots, so use type_size_vec4(). */
 298    nir_lower_io(nir, nir_var_shader_in, type_size_vec4, 0);
 299
 300    if (is_scalar || nir->stage != MESA_SHADER_GEOMETRY) {
 301       /* This pass needs actual constants */
 302       nir_opt_constant_folding(nir);
 303
 304       add_const_offset_to_base(nir, nir_var_shader_in);
 305
 306       nir_foreach_function(function, nir) {
 307          if (function->impl) {
 308             nir_foreach_block(block, function->impl) {
 309                remap_inputs_with_vue_map(block, vue_map);
 310             }
 311          }
 312       }
 313    }
 314 }
 315
 316 void
 317 brw_nir_lower_tes_inputs(nir_shader *nir, const struct brw_vue_map *vue_map)
 318 {
 319    foreach_list_typed(nir_variable, var, node, &nir->inputs) {
 320       var->data.driver_location = var->data.location;
 321    }
 322
 323    nir_lower_io(nir, nir_var_shader_in, type_size_vec4, 0);
 324
 325    /* This pass needs actual constants */
 326    nir_opt_constant_folding(nir);
 327
 328    add_const_offset_to_base(nir, nir_var_shader_in);
 329
 330    nir_foreach_function(function, nir) {
 331       if (function->impl) {
 332          nir_builder b;
 333          nir_builder_init(&b, function->impl);
 334          nir_foreach_block(block, function->impl) {
 335             remap_patch_urb_offsets(block, &b, vue_map,
 336                                     nir->info.tess.primitive_mode);
 337          }
 338       }
 339    }
 340 }
 341
 342 void
 343 brw_nir_lower_fs_inputs(nir_shader *nir,
 344                         const struct gen_device_info *devinfo,
 345                         const struct brw_wm_prog_key *key)
 346 {
 347    foreach_list_typed(nir_variable, var, node, &nir->inputs) {
 348       var->data.driver_location = var->data.location;
 349
 350       /* Apply default interpolation mode.
 351        *
 352        * Everything defaults to smooth except for the legacy GL color
 353        * built-in variables, which might be flat depending on API state.
 354        */
 355       if (var->data.interpolation == INTERP_MODE_NONE) {
 356          const bool flat = key->flat_shade &&
 357             (var->data.location == VARYING_SLOT_COL0 ||
 358              var->data.location == VARYING_SLOT_COL1);
 359
 360          var->data.interpolation = flat ? INTERP_MODE_FLAT
 361                                         : INTERP_MODE_SMOOTH;
 362       }
 363
 364       /* On Ironlake and below, there is only one interpolation mode.
 365        * Centroid interpolation doesn't mean anything on this hardware --
 366        * there is no multisampling.
 367        */
 368       if (devinfo->gen < 6) {
 369          var->data.centroid = false;
 370          var->data.sample = false;
 371       }
 372    }
 373
 374    nir_lower_io_options lower_io_options = 0;
 375    if (key->persample_interp)
 376       lower_io_options |= nir_lower_io_force_sample_interpolation;
 377
 378    nir_lower_io(nir, nir_var_shader_in, type_size_vec4, lower_io_options);
 379
 380    /* This pass needs actual constants */
 381    nir_opt_constant_folding(nir);
 382
 383    add_const_offset_to_base(nir, nir_var_shader_in);
 384 }
 385
 386 void
 387 brw_nir_lower_vue_outputs(nir_shader *nir,
 388                           bool is_scalar)
 389 {
 390    nir_foreach_variable(var, &nir->outputs) {
 391       var->data.driver_location = var->data.location;
 392    }
 393
 394    nir_lower_io(nir, nir_var_shader_out, type_size_vec4, 0);
 395 }
 396
 397 void
 398 brw_nir_lower_tcs_outputs(nir_shader *nir, const struct brw_vue_map *vue_map,
 399                           GLenum tes_primitive_mode)
 400 {
 401    nir_foreach_variable(var, &nir->outputs) {
 402       var->data.driver_location = var->data.location;
 403    }
 404
 405    nir_lower_io(nir, nir_var_shader_out, type_size_vec4, 0);
 406
 407    /* This pass needs actual constants */
 408    nir_opt_constant_folding(nir);
 409
 410    add_const_offset_to_base(nir, nir_var_shader_out);
 411
 412    nir_foreach_function(function, nir) {
 413       if (function->impl) {
 414          nir_builder b;
 415          nir_builder_init(&b, function->impl);
 416          nir_foreach_block(block, function->impl) {
 417             remap_patch_urb_offsets(block, &b, vue_map, tes_primitive_mode);
 418          }
 419       }
 420    }
 421 }
 422
 423 void
 424 brw_nir_lower_fs_outputs(nir_shader *nir)
 425 {
 426    nir_foreach_variable(var, &nir->outputs) {
 427       var->data.driver_location =
 428          SET_FIELD(var->data.index, BRW_NIR_FRAG_OUTPUT_INDEX) |
 429          SET_FIELD(var->data.location, BRW_NIR_FRAG_OUTPUT_LOCATION);
 430    }
 431
 432    nir_lower_io(nir, nir_var_shader_out, type_size_dvec4, 0);
 433 }
 434
 435 void
 436 brw_nir_lower_cs_shared(nir_shader *nir)
 437 {
 438    nir_assign_var_locations(&nir->shared, &nir->num_shared,
 439                             type_size_scalar_bytes);
 440    nir_lower_io(nir, nir_var_shared, type_size_scalar_bytes, 0);
 441 }
 442
 443 #define OPT(pass, ...) ({                                  \
 444    bool this_progress = false;                             \
 445    NIR_PASS(this_progress, nir, pass, ##__VA_ARGS__);      \
 446    if (this_progress)                                      \
 447       progress = true;                                     \
 448    this_progress;                                          \
 449 })
 450
 451 static nir_shader *
 452 nir_optimize(nir_shader *nir, const struct brw_compiler *compiler,
 453              bool is_scalar)
 454 {
 455    nir_variable_mode indirect_mask = 0;
 456    if (compiler->glsl_compiler_options[nir->stage].EmitNoIndirectInput)
 457       indirect_mask |= nir_var_shader_in;
 458    if (compiler->glsl_compiler_options[nir->stage].EmitNoIndirectOutput)
 459       indirect_mask |= nir_var_shader_out;
 460    if (compiler->glsl_compiler_options[nir->stage].EmitNoIndirectTemp)
 461       indirect_mask |= nir_var_local;
 462
 463    bool progress;
 464    do {
 465       progress = false;
 466       OPT(nir_lower_vars_to_ssa);
 467       OPT(nir_opt_copy_prop_vars);
 468
 469       if (is_scalar) {
 470          OPT(nir_lower_alu_to_scalar);
 471       }
 472
 473       OPT(nir_copy_prop);
 474
 475       if (is_scalar) {
 476          OPT(nir_lower_phis_to_scalar);
 477       }
 478
 479       OPT(nir_copy_prop);
 480       OPT(nir_opt_dce);
 481       OPT(nir_opt_cse);
 482       OPT(nir_opt_peephole_select, 0);
 483       OPT(nir_opt_algebraic);
 484       OPT(nir_opt_constant_folding);
 485       OPT(nir_opt_dead_cf);
 486       if (OPT(nir_opt_trivial_continues)) {
 487          /* If nir_opt_trivial_continues makes progress, then we need to clean
 488           * things up if we want any hope of nir_opt_if or nir_opt_loop_unroll
 489           * to make progress.
 490           */
 491          OPT(nir_copy_prop);
 492          OPT(nir_opt_dce);
 493       }
 494       OPT(nir_opt_if);
 495       if (nir->options->max_unroll_iterations != 0) {
 496          OPT(nir_opt_loop_unroll, indirect_mask);
 497       }
 498       OPT(nir_opt_remove_phis);
 499       OPT(nir_opt_undef);
 500       OPT(nir_lower_doubles, nir_lower_drcp |
 501                              nir_lower_dsqrt |
 502                              nir_lower_drsq |
 503                              nir_lower_dtrunc |
 504                              nir_lower_dfloor |
 505                              nir_lower_dceil |
 506                              nir_lower_dfract |
 507                              nir_lower_dround_even |
 508                              nir_lower_dmod);
 509       OPT(nir_lower_64bit_pack);
 510    } while (progress);
 511
 512    return nir;
 513 }
 514
 515 /* Does some simple lowering and runs the standard suite of optimizations
 516  *
 517  * This is intended to be called more-or-less directly after you get the
 518  * shader out of GLSL or some other source.  While it is geared towards i965,
 519  * it is not at all generator-specific except for the is_scalar flag.  Even
 520  * there, it is safe to call with is_scalar = false for a shader that is
 521  * intended for the FS backend as long as nir_optimize is called again with
 522  * is_scalar = true to scalarize everything prior to code gen.
 523  */
 524 nir_shader *
 525 brw_preprocess_nir(const struct brw_compiler *compiler, nir_shader *nir)
 526 {
 527    const struct gen_device_info *devinfo = compiler->devinfo;
 528    UNUSED bool progress; /* Written by OPT */
 529
 530    const bool is_scalar = compiler->scalar_stage[nir->stage];
 531
 532    if (nir->stage == MESA_SHADER_GEOMETRY)
 533       OPT(nir_lower_gs_intrinsics);
 534
 535    /* See also brw_nir_trig_workarounds.py */
 536    if (compiler->precise_trig &&
 537        !(devinfo->gen >= 10 || devinfo->is_kabylake))
 538       OPT(brw_nir_apply_trig_workarounds);
 539
 540    static const nir_lower_tex_options tex_options = {
 541       .lower_txp = ~0,
 542       .lower_txf_offset = true,
 543       .lower_rect_offset = true,
 544       .lower_txd_cube_map = true,
 545    };
 546
 547    OPT(nir_lower_tex, &tex_options);
 548    OPT(nir_normalize_cubemap_coords);
 549
 550    OPT(nir_lower_global_vars_to_local);
 551
 552    OPT(nir_split_var_copies);
 553
 554    nir = nir_optimize(nir, compiler, is_scalar);
 555
 556    if (is_scalar) {
 557       OPT(nir_lower_load_const_to_scalar);
 558    }
 559
 560    /* Lower a bunch of stuff */
 561    OPT(nir_lower_var_copies);
 562
 563    OPT(nir_lower_clip_cull_distance_arrays);
 564
 565    nir_variable_mode indirect_mask = 0;
 566    if (compiler->glsl_compiler_options[nir->stage].EmitNoIndirectInput)
 567       indirect_mask |= nir_var_shader_in;
 568    if (compiler->glsl_compiler_options[nir->stage].EmitNoIndirectOutput)
 569       indirect_mask |= nir_var_shader_out;
 570    if (compiler->glsl_compiler_options[nir->stage].EmitNoIndirectTemp)
 571       indirect_mask |= nir_var_local;
 572
 573    nir_lower_indirect_derefs(nir, indirect_mask);
 574
 575    nir_lower_int64(nir, nir_lower_imul64 |
 576                         nir_lower_isign64 |
 577                         nir_lower_divmod64);
 578
 579    /* Get rid of split copies */
 580    nir = nir_optimize(nir, compiler, is_scalar);
 581
 582    OPT(nir_remove_dead_variables, nir_var_local);
 583
 584    return nir;
 585 }
 586
 587 /* Prepare the given shader for codegen
 588  *
 589  * This function is intended to be called right before going into the actual
 590  * backend and is highly backend-specific.  Also, once this function has been
 591  * called on a shader, it will no longer be in SSA form so most optimizations
 592  * will not work.
 593  */
 594 nir_shader *
 595 brw_postprocess_nir(nir_shader *nir, const struct brw_compiler *compiler,
 596                     bool is_scalar)
 597 {
 598    const struct gen_device_info *devinfo = compiler->devinfo;
 599    bool debug_enabled =
 600       (INTEL_DEBUG & intel_debug_flag_for_shader_stage(nir->stage));
 601
 602    UNUSED bool progress; /* Written by OPT */
 603
 604
 605    do {
 606       progress = false;
 607       OPT(nir_opt_algebraic_before_ffma);
 608    } while (progress);
 609
 610    nir = nir_optimize(nir, compiler, is_scalar);
 611
 612    if (devinfo->gen >= 6) {
 613       /* Try and fuse multiply-adds */
 614       OPT(brw_nir_opt_peephole_ffma);
 615    }
 616
 617    OPT(nir_opt_algebraic_late);
 618
 619    OPT(nir_lower_to_source_mods);
 620    OPT(nir_copy_prop);
 621    OPT(nir_opt_dce);
 622    OPT(nir_opt_move_comparisons);
 623
 624    OPT(nir_lower_locals_to_regs);
 625
 626    if (unlikely(debug_enabled)) {
 627       /* Re-index SSA defs so we print more sensible numbers. */
 628       nir_foreach_function(function, nir) {
 629          if (function->impl)
 630             nir_index_ssa_defs(function->impl);
 631       }
 632
 633       fprintf(stderr, "NIR (SSA form) for %s shader:\n",
 634               _mesa_shader_stage_to_string(nir->stage));
 635       nir_print_shader(nir, stderr);
 636    }
 637
 638    OPT(nir_convert_from_ssa, true);
 639
 640    if (!is_scalar) {
 641       OPT(nir_move_vec_src_uses_to_dest);
 642       OPT(nir_lower_vec_to_movs);
 643    }
 644
 645    /* This is the last pass we run before we start emitting stuff.  It
 646     * determines when we need to insert boolean resolves on Gen <= 5.  We
 647     * run it last because it stashes data in instr->pass_flags and we don't
 648     * want that to be squashed by other NIR passes.
 649     */
 650    if (devinfo->gen <= 5)
 651       brw_nir_analyze_boolean_resolves(nir);
 652
 653    nir_sweep(nir);
 654
 655    if (unlikely(debug_enabled)) {
 656       fprintf(stderr, "NIR (final form) for %s shader:\n",
 657               _mesa_shader_stage_to_string(nir->stage));
 658       nir_print_shader(nir, stderr);
 659    }
 660
 661    return nir;
 662 }
 663
 664 nir_shader *
 665 brw_nir_apply_sampler_key(nir_shader *nir,
 666                           const struct brw_compiler *compiler,
 667                           const struct brw_sampler_prog_key_data *key_tex,
 668                           bool is_scalar)
 669 {
 670    const struct gen_device_info *devinfo = compiler->devinfo;
 671    nir_lower_tex_options tex_options = { 0 };
 672
 673    /* Iron Lake and prior require lowering of all rectangle textures */
 674    if (devinfo->gen < 6)
 675       tex_options.lower_rect = true;
 676
 677    /* Prior to Broadwell, our hardware can't actually do GL_CLAMP */
 678    if (devinfo->gen < 8) {
 679       tex_options.saturate_s = key_tex->gl_clamp_mask[0];
 680       tex_options.saturate_t = key_tex->gl_clamp_mask[1];
 681       tex_options.saturate_r = key_tex->gl_clamp_mask[2];
 682    }
 683
 684    /* Prior to Haswell, we have to fake texture swizzle */
 685    for (unsigned s = 0; s < MAX_SAMPLERS; s++) {
 686       if (key_tex->swizzles[s] == SWIZZLE_NOOP)
 687          continue;
 688
 689       tex_options.swizzle_result |= (1 << s);
 690       for (unsigned c = 0; c < 4; c++)
 691          tex_options.swizzles[s][c] = GET_SWZ(key_tex->swizzles[s], c);
 692    }
 693
 694    /* Prior to Haswell, we have to lower gradients on shadow samplers */
 695    tex_options.lower_txd_shadow = devinfo->gen < 8 && !devinfo->is_haswell;
 696
 697    tex_options.lower_y_uv_external = key_tex->y_uv_image_mask;
 698    tex_options.lower_y_u_v_external = key_tex->y_u_v_image_mask;
 699    tex_options.lower_yx_xuxv_external = key_tex->yx_xuxv_image_mask;
 700
 701    if (nir_lower_tex(nir, &tex_options)) {
 702       nir_validate_shader(nir);
 703       nir = nir_optimize(nir, compiler, is_scalar);
 704    }
 705
 706    return nir;
 707 }
 708
 709 enum brw_reg_type
 710 brw_type_for_nir_type(const struct gen_device_info *devinfo, nir_alu_type type)
 711 {
 712    switch (type) {
 713    case nir_type_uint:
 714    case nir_type_uint32:
 715       return BRW_REGISTER_TYPE_UD;
 716    case nir_type_bool:
 717    case nir_type_int:
 718    case nir_type_bool32:
 719    case nir_type_int32:
 720       return BRW_REGISTER_TYPE_D;
 721    case nir_type_float:
 722    case nir_type_float32:
 723       return BRW_REGISTER_TYPE_F;
 724    case nir_type_float64:
 725       return BRW_REGISTER_TYPE_DF;
 726    case nir_type_int64:
 727       return devinfo->gen < 8 ? BRW_REGISTER_TYPE_DF : BRW_REGISTER_TYPE_Q;
 728    case nir_type_uint64:
 729       return devinfo->gen < 8 ? BRW_REGISTER_TYPE_DF : BRW_REGISTER_TYPE_UQ;
 730    default:
 731       unreachable("unknown type");
 732    }
 733
 734    return BRW_REGISTER_TYPE_F;
 735 }
 736
 737 /* Returns the glsl_base_type corresponding to a nir_alu_type.
 738  * This is used by both brw_vec4_nir and brw_fs_nir.
 739  */
 740 enum glsl_base_type
 741 brw_glsl_base_type_for_nir_type(nir_alu_type type)
 742 {
 743    switch (type) {
 744    case nir_type_float:
 745    case nir_type_float32:
 746       return GLSL_TYPE_FLOAT;
 747
 748    case nir_type_float64:
 749       return GLSL_TYPE_DOUBLE;
 750
 751    case nir_type_int:
 752    case nir_type_int32:
 753       return GLSL_TYPE_INT;
 754
 755    case nir_type_uint:
 756    case nir_type_uint32:
 757       return GLSL_TYPE_UINT;
 758
 759    default:
 760       unreachable("bad type");
 761    }
 762 }