src/intel/compiler/brw_nir.c

   1 /*
   2  * Copyright © 2014 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include "brw_nir.h"
  25 #include "brw_shader.h"
  26 #include "common/gen_debug.h"
  27 #include "compiler/glsl_types.h"
  28 #include "compiler/nir/nir_builder.h"
  29
  30 static bool
  31 is_input(nir_intrinsic_instr *intrin)
  32 {
  33    return intrin->intrinsic == nir_intrinsic_load_input ||
  34           intrin->intrinsic == nir_intrinsic_load_per_vertex_input ||
  35           intrin->intrinsic == nir_intrinsic_load_interpolated_input;
  36 }
  37
  38 static bool
  39 is_output(nir_intrinsic_instr *intrin)
  40 {
  41    return intrin->intrinsic == nir_intrinsic_load_output ||
  42           intrin->intrinsic == nir_intrinsic_load_per_vertex_output ||
  43           intrin->intrinsic == nir_intrinsic_store_output ||
  44           intrin->intrinsic == nir_intrinsic_store_per_vertex_output;
  45 }
  46
  47 /**
  48  * In many cases, we just add the base and offset together, so there's no
  49  * reason to keep them separate.  Sometimes, combining them is essential:
  50  * if a shader only accesses part of a compound variable (such as a matrix
  51  * or array), the variable's base may not actually exist in the VUE map.
  52  *
  53  * This pass adds constant offsets to instr->const_index[0], and resets
  54  * the offset source to 0.  Non-constant offsets remain unchanged - since
  55  * we don't know what part of a compound variable is accessed, we allocate
  56  * storage for the entire thing.
  57  */
  58
  59 static bool
  60 add_const_offset_to_base_block(nir_block *block, nir_builder *b,
  61                                nir_variable_mode mode)
  62 {
  63    nir_foreach_instr_safe(instr, block) {
  64       if (instr->type != nir_instr_type_intrinsic)
  65          continue;
  66
  67       nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
  68
  69       if ((mode == nir_var_shader_in && is_input(intrin)) ||
  70           (mode == nir_var_shader_out && is_output(intrin))) {
  71          nir_src *offset = nir_get_io_offset_src(intrin);
  72          nir_const_value *const_offset = nir_src_as_const_value(*offset);
  73
  74          if (const_offset) {
  75             intrin->const_index[0] += const_offset->u32[0];
  76             b->cursor = nir_before_instr(&intrin->instr);
  77             nir_instr_rewrite_src(&intrin->instr, offset,
  78                                   nir_src_for_ssa(nir_imm_int(b, 0)));
  79          }
  80       }
  81    }
  82    return true;
  83 }
  84
  85 static void
  86 add_const_offset_to_base(nir_shader *nir, nir_variable_mode mode)
  87 {
  88    nir_foreach_function(f, nir) {
  89       if (f->impl) {
  90          nir_builder b;
  91          nir_builder_init(&b, f->impl);
  92          nir_foreach_block(block, f->impl) {
  93             add_const_offset_to_base_block(block, &b, mode);
  94          }
  95       }
  96    }
  97 }
  98
  99 static bool
 100 remap_vs_attrs(nir_block *block, shader_info *nir_info)
 101 {
 102    nir_foreach_instr(instr, block) {
 103       if (instr->type != nir_instr_type_intrinsic)
 104          continue;
 105
 106       nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
 107
 108       if (intrin->intrinsic == nir_intrinsic_load_input) {
 109          /* Attributes come in a contiguous block, ordered by their
 110           * gl_vert_attrib value.  That means we can compute the slot
 111           * number for an attribute by masking out the enabled attributes
 112           * before it and counting the bits.
 113           */
 114          int attr = intrin->const_index[0];
 115          int slot = _mesa_bitcount_64(nir_info->inputs_read &
 116                                       BITFIELD64_MASK(attr));
 117          intrin->const_index[0] = 4 * slot;
 118       }
 119    }
 120    return true;
 121 }
 122
 123 static bool
 124 remap_inputs_with_vue_map(nir_block *block, const struct brw_vue_map *vue_map)
 125 {
 126    nir_foreach_instr(instr, block) {
 127       if (instr->type != nir_instr_type_intrinsic)
 128          continue;
 129
 130       nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
 131
 132       if (intrin->intrinsic == nir_intrinsic_load_input ||
 133           intrin->intrinsic == nir_intrinsic_load_per_vertex_input) {
 134          int vue_slot = vue_map->varying_to_slot[intrin->const_index[0]];
 135          assert(vue_slot != -1);
 136          intrin->const_index[0] = vue_slot;
 137       }
 138    }
 139    return true;
 140 }
 141
 142 static bool
 143 remap_tess_levels(nir_builder *b, nir_intrinsic_instr *intr,
 144                   GLenum primitive_mode)
 145 {
 146    const int location = nir_intrinsic_base(intr);
 147    const unsigned component = nir_intrinsic_component(intr);
 148    bool out_of_bounds;
 149
 150    if (location == VARYING_SLOT_TESS_LEVEL_INNER) {
 151       switch (primitive_mode) {
 152       case GL_QUADS:
 153          /* gl_TessLevelInner[0..1] lives at DWords 3-2 (reversed). */
 154          nir_intrinsic_set_base(intr, 0);
 155          nir_intrinsic_set_component(intr, 3 - component);
 156          out_of_bounds = false;
 157          break;
 158       case GL_TRIANGLES:
 159          /* gl_TessLevelInner[0] lives at DWord 4. */
 160          nir_intrinsic_set_base(intr, 1);
 161          out_of_bounds = component > 0;
 162          break;
 163       case GL_ISOLINES:
 164          out_of_bounds = true;
 165          break;
 166       default:
 167          unreachable("Bogus tessellation domain");
 168       }
 169    } else if (location == VARYING_SLOT_TESS_LEVEL_OUTER) {
 170       if (primitive_mode == GL_ISOLINES) {
 171          /* gl_TessLevelOuter[0..1] lives at DWords 6-7 (in order). */
 172          nir_intrinsic_set_base(intr, 1);
 173          nir_intrinsic_set_component(intr, 2 + nir_intrinsic_component(intr));
 174          out_of_bounds = component > 1;
 175       } else {
 176          /* Triangles use DWords 7-5 (reversed); Quads use 7-4 (reversed) */
 177          nir_intrinsic_set_base(intr, 1);
 178          nir_intrinsic_set_component(intr, 3 - nir_intrinsic_component(intr));
 179          out_of_bounds = component == 3 && primitive_mode == GL_TRIANGLES;
 180       }
 181    } else {
 182       return false;
 183    }
 184
 185    if (out_of_bounds) {
 186       if (nir_intrinsic_infos[intr->intrinsic].has_dest) {
 187          b->cursor = nir_before_instr(&intr->instr);
 188          nir_ssa_def *undef = nir_ssa_undef(b, 1, 32);
 189          nir_ssa_def_rewrite_uses(&intr->dest.ssa, nir_src_for_ssa(undef));
 190       }
 191       nir_instr_remove(&intr->instr);
 192    }
 193
 194    return true;
 195 }
 196
 197 static bool
 198 remap_patch_urb_offsets(nir_block *block, nir_builder *b,
 199                         const struct brw_vue_map *vue_map,
 200                         GLenum tes_primitive_mode)
 201 {
 202    const bool is_passthrough_tcs = b->shader->info->name &&
 203       strcmp(b->shader->info->name, "passthrough") == 0;
 204
 205    nir_foreach_instr_safe(instr, block) {
 206       if (instr->type != nir_instr_type_intrinsic)
 207          continue;
 208
 209       nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
 210
 211       gl_shader_stage stage = b->shader->stage;
 212
 213       if ((stage == MESA_SHADER_TESS_CTRL && is_output(intrin)) ||
 214           (stage == MESA_SHADER_TESS_EVAL && is_input(intrin))) {
 215
 216          if (!is_passthrough_tcs &&
 217              remap_tess_levels(b, intrin, tes_primitive_mode))
 218             continue;
 219
 220          int vue_slot = vue_map->varying_to_slot[intrin->const_index[0]];
 221          assert(vue_slot != -1);
 222          intrin->const_index[0] = vue_slot;
 223
 224          nir_src *vertex = nir_get_io_vertex_index_src(intrin);
 225          if (vertex) {
 226             nir_const_value *const_vertex = nir_src_as_const_value(*vertex);
 227             if (const_vertex) {
 228                intrin->const_index[0] += const_vertex->u32[0] *
 229                                          vue_map->num_per_vertex_slots;
 230             } else {
 231                b->cursor = nir_before_instr(&intrin->instr);
 232
 233                /* Multiply by the number of per-vertex slots. */
 234                nir_ssa_def *vertex_offset =
 235                   nir_imul(b,
 236                            nir_ssa_for_src(b, *vertex, 1),
 237                            nir_imm_int(b,
 238                                        vue_map->num_per_vertex_slots));
 239
 240                /* Add it to the existing offset */
 241                nir_src *offset = nir_get_io_offset_src(intrin);
 242                nir_ssa_def *total_offset =
 243                   nir_iadd(b, vertex_offset,
 244                            nir_ssa_for_src(b, *offset, 1));
 245
 246                nir_instr_rewrite_src(&intrin->instr, offset,
 247                                      nir_src_for_ssa(total_offset));
 248             }
 249          }
 250       }
 251    }
 252    return true;
 253 }
 254
 255 void
 256 brw_nir_lower_vs_inputs(nir_shader *nir,
 257                         bool is_scalar,
 258                         bool use_legacy_snorm_formula,
 259                         const uint8_t *vs_attrib_wa_flags)
 260 {
 261    /* Start with the location of the variable's base. */
 262    foreach_list_typed(nir_variable, var, node, &nir->inputs) {
 263       var->data.driver_location = var->data.location;
 264    }
 265
 266    /* Now use nir_lower_io to walk dereference chains.  Attribute arrays are
 267     * loaded as one vec4 or dvec4 per element (or matrix column), depending on
 268     * whether it is a double-precision type or not.
 269     */
 270    nir_lower_io(nir, nir_var_shader_in, type_size_vec4, 0);
 271
 272    /* This pass needs actual constants */
 273    nir_opt_constant_folding(nir);
 274
 275    add_const_offset_to_base(nir, nir_var_shader_in);
 276
 277    brw_nir_apply_attribute_workarounds(nir, use_legacy_snorm_formula,
 278                                        vs_attrib_wa_flags);
 279
 280    if (is_scalar) {
 281       /* Finally, translate VERT_ATTRIB_* values into the actual registers. */
 282
 283       nir_foreach_function(function, nir) {
 284          if (function->impl) {
 285             nir_foreach_block(block, function->impl) {
 286                remap_vs_attrs(block, nir->info);
 287             }
 288          }
 289       }
 290    }
 291 }
 292
 293 void
 294 brw_nir_lower_vue_inputs(nir_shader *nir, bool is_scalar,
 295                          const struct brw_vue_map *vue_map)
 296 {
 297    foreach_list_typed(nir_variable, var, node, &nir->inputs) {
 298       var->data.driver_location = var->data.location;
 299    }
 300
 301    /* Inputs are stored in vec4 slots, so use type_size_vec4(). */
 302    nir_lower_io(nir, nir_var_shader_in, type_size_vec4, 0);
 303
 304    if (is_scalar || nir->stage != MESA_SHADER_GEOMETRY) {
 305       /* This pass needs actual constants */
 306       nir_opt_constant_folding(nir);
 307
 308       add_const_offset_to_base(nir, nir_var_shader_in);
 309
 310       nir_foreach_function(function, nir) {
 311          if (function->impl) {
 312             nir_foreach_block(block, function->impl) {
 313                remap_inputs_with_vue_map(block, vue_map);
 314             }
 315          }
 316       }
 317    }
 318 }
 319
 320 void
 321 brw_nir_lower_tes_inputs(nir_shader *nir, const struct brw_vue_map *vue_map)
 322 {
 323    foreach_list_typed(nir_variable, var, node, &nir->inputs) {
 324       var->data.driver_location = var->data.location;
 325    }
 326
 327    nir_lower_io(nir, nir_var_shader_in, type_size_vec4, 0);
 328
 329    /* This pass needs actual constants */
 330    nir_opt_constant_folding(nir);
 331
 332    add_const_offset_to_base(nir, nir_var_shader_in);
 333
 334    nir_foreach_function(function, nir) {
 335       if (function->impl) {
 336          nir_builder b;
 337          nir_builder_init(&b, function->impl);
 338          nir_foreach_block(block, function->impl) {
 339             remap_patch_urb_offsets(block, &b, vue_map,
 340                                     nir->info->tess.primitive_mode);
 341          }
 342       }
 343    }
 344 }
 345
 346 void
 347 brw_nir_lower_fs_inputs(nir_shader *nir,
 348                         const struct gen_device_info *devinfo,
 349                         const struct brw_wm_prog_key *key)
 350 {
 351    foreach_list_typed(nir_variable, var, node, &nir->inputs) {
 352       var->data.driver_location = var->data.location;
 353
 354       /* Apply default interpolation mode.
 355        *
 356        * Everything defaults to smooth except for the legacy GL color
 357        * built-in variables, which might be flat depending on API state.
 358        */
 359       if (var->data.interpolation == INTERP_MODE_NONE) {
 360          const bool flat = key->flat_shade &&
 361             (var->data.location == VARYING_SLOT_COL0 ||
 362              var->data.location == VARYING_SLOT_COL1);
 363
 364          var->data.interpolation = flat ? INTERP_MODE_FLAT
 365                                         : INTERP_MODE_SMOOTH;
 366       }
 367
 368       /* On Ironlake and below, there is only one interpolation mode.
 369        * Centroid interpolation doesn't mean anything on this hardware --
 370        * there is no multisampling.
 371        */
 372       if (devinfo->gen < 6) {
 373          var->data.centroid = false;
 374          var->data.sample = false;
 375       }
 376    }
 377
 378    nir_lower_io_options lower_io_options = 0;
 379    if (key->persample_interp)
 380       lower_io_options |= nir_lower_io_force_sample_interpolation;
 381
 382    nir_lower_io(nir, nir_var_shader_in, type_size_vec4, lower_io_options);
 383
 384    /* This pass needs actual constants */
 385    nir_opt_constant_folding(nir);
 386
 387    add_const_offset_to_base(nir, nir_var_shader_in);
 388 }
 389
 390 void
 391 brw_nir_lower_vue_outputs(nir_shader *nir,
 392                           bool is_scalar)
 393 {
 394    nir_foreach_variable(var, &nir->outputs) {
 395       var->data.driver_location = var->data.location;
 396    }
 397
 398    nir_lower_io(nir, nir_var_shader_out, type_size_vec4, 0);
 399 }
 400
 401 void
 402 brw_nir_lower_tcs_outputs(nir_shader *nir, const struct brw_vue_map *vue_map,
 403                           GLenum tes_primitive_mode)
 404 {
 405    nir_foreach_variable(var, &nir->outputs) {
 406       var->data.driver_location = var->data.location;
 407    }
 408
 409    nir_lower_io(nir, nir_var_shader_out, type_size_vec4, 0);
 410
 411    /* This pass needs actual constants */
 412    nir_opt_constant_folding(nir);
 413
 414    add_const_offset_to_base(nir, nir_var_shader_out);
 415
 416    nir_foreach_function(function, nir) {
 417       if (function->impl) {
 418          nir_builder b;
 419          nir_builder_init(&b, function->impl);
 420          nir_foreach_block(block, function->impl) {
 421             remap_patch_urb_offsets(block, &b, vue_map, tes_primitive_mode);
 422          }
 423       }
 424    }
 425 }
 426
 427 void
 428 brw_nir_lower_fs_outputs(nir_shader *nir)
 429 {
 430    nir_foreach_variable(var, &nir->outputs) {
 431       var->data.driver_location =
 432          SET_FIELD(var->data.index, BRW_NIR_FRAG_OUTPUT_INDEX) |
 433          SET_FIELD(var->data.location, BRW_NIR_FRAG_OUTPUT_LOCATION);
 434    }
 435
 436    nir_lower_io(nir, nir_var_shader_out, type_size_dvec4, 0);
 437 }
 438
 439 void
 440 brw_nir_lower_cs_shared(nir_shader *nir)
 441 {
 442    nir_assign_var_locations(&nir->shared, &nir->num_shared,
 443                             type_size_scalar_bytes);
 444    nir_lower_io(nir, nir_var_shared, type_size_scalar_bytes, 0);
 445 }
 446
 447 #define OPT(pass, ...) ({                                  \
 448    bool this_progress = false;                             \
 449    NIR_PASS(this_progress, nir, pass, ##__VA_ARGS__);      \
 450    if (this_progress)                                      \
 451       progress = true;                                     \
 452    this_progress;                                          \
 453 })
 454
 455 #define OPT_V(pass, ...) NIR_PASS_V(nir, pass, ##__VA_ARGS__)
 456
 457 static nir_shader *
 458 nir_optimize(nir_shader *nir, const struct brw_compiler *compiler,
 459              bool is_scalar)
 460 {
 461    nir_variable_mode indirect_mask = 0;
 462    if (compiler->glsl_compiler_options[nir->stage].EmitNoIndirectInput)
 463       indirect_mask |= nir_var_shader_in;
 464    if (compiler->glsl_compiler_options[nir->stage].EmitNoIndirectOutput)
 465       indirect_mask |= nir_var_shader_out;
 466    if (compiler->glsl_compiler_options[nir->stage].EmitNoIndirectTemp)
 467       indirect_mask |= nir_var_local;
 468
 469    bool progress;
 470    do {
 471       progress = false;
 472       OPT_V(nir_lower_vars_to_ssa);
 473       OPT(nir_opt_copy_prop_vars);
 474
 475       if (is_scalar) {
 476          OPT(nir_lower_alu_to_scalar);
 477       }
 478
 479       OPT(nir_copy_prop);
 480
 481       if (is_scalar) {
 482          OPT(nir_lower_phis_to_scalar);
 483       }
 484
 485       OPT(nir_copy_prop);
 486       OPT(nir_opt_dce);
 487       OPT(nir_opt_cse);
 488       OPT(nir_opt_peephole_select, 0);
 489       OPT(nir_opt_algebraic);
 490       OPT(nir_opt_constant_folding);
 491       OPT(nir_opt_dead_cf);
 492       if (OPT(nir_opt_trivial_continues)) {
 493          /* If nir_opt_trivial_continues makes progress, then we need to clean
 494           * things up if we want any hope of nir_opt_if or nir_opt_loop_unroll
 495           * to make progress.
 496           */
 497          OPT(nir_copy_prop);
 498          OPT(nir_opt_dce);
 499       }
 500       OPT(nir_opt_if);
 501       if (nir->options->max_unroll_iterations != 0) {
 502          OPT(nir_opt_loop_unroll, indirect_mask);
 503       }
 504       OPT(nir_opt_remove_phis);
 505       OPT(nir_opt_undef);
 506       OPT_V(nir_lower_doubles, nir_lower_drcp |
 507                                nir_lower_dsqrt |
 508                                nir_lower_drsq |
 509                                nir_lower_dtrunc |
 510                                nir_lower_dfloor |
 511                                nir_lower_dceil |
 512                                nir_lower_dfract |
 513                                nir_lower_dround_even |
 514                                nir_lower_dmod);
 515       OPT_V(nir_lower_64bit_pack);
 516    } while (progress);
 517
 518    return nir;
 519 }
 520
 521 /* Does some simple lowering and runs the standard suite of optimizations
 522  *
 523  * This is intended to be called more-or-less directly after you get the
 524  * shader out of GLSL or some other source.  While it is geared towards i965,
 525  * it is not at all generator-specific except for the is_scalar flag.  Even
 526  * there, it is safe to call with is_scalar = false for a shader that is
 527  * intended for the FS backend as long as nir_optimize is called again with
 528  * is_scalar = true to scalarize everything prior to code gen.
 529  */
 530 nir_shader *
 531 brw_preprocess_nir(const struct brw_compiler *compiler, nir_shader *nir)
 532 {
 533    const struct gen_device_info *devinfo = compiler->devinfo;
 534    bool progress; /* Written by OPT and OPT_V */
 535    (void)progress;
 536
 537    const bool is_scalar = compiler->scalar_stage[nir->stage];
 538
 539    if (nir->stage == MESA_SHADER_GEOMETRY)
 540       OPT(nir_lower_gs_intrinsics);
 541
 542    /* See also brw_nir_trig_workarounds.py */
 543    if (compiler->precise_trig &&
 544        !(devinfo->gen >= 10 || devinfo->is_kabylake))
 545       OPT(brw_nir_apply_trig_workarounds);
 546
 547    static const nir_lower_tex_options tex_options = {
 548       .lower_txp = ~0,
 549       .lower_txf_offset = true,
 550       .lower_rect_offset = true,
 551       .lower_txd_cube_map = true,
 552    };
 553
 554    OPT(nir_lower_tex, &tex_options);
 555    OPT(nir_normalize_cubemap_coords);
 556
 557    OPT(nir_lower_global_vars_to_local);
 558
 559    OPT(nir_split_var_copies);
 560
 561    nir = nir_optimize(nir, compiler, is_scalar);
 562
 563    if (is_scalar) {
 564       OPT_V(nir_lower_load_const_to_scalar);
 565    }
 566
 567    /* Lower a bunch of stuff */
 568    OPT_V(nir_lower_var_copies);
 569
 570    OPT_V(nir_lower_clip_cull_distance_arrays);
 571
 572    nir_variable_mode indirect_mask = 0;
 573    if (compiler->glsl_compiler_options[nir->stage].EmitNoIndirectInput)
 574       indirect_mask |= nir_var_shader_in;
 575    if (compiler->glsl_compiler_options[nir->stage].EmitNoIndirectOutput)
 576       indirect_mask |= nir_var_shader_out;
 577    if (compiler->glsl_compiler_options[nir->stage].EmitNoIndirectTemp)
 578       indirect_mask |= nir_var_local;
 579
 580    nir_lower_indirect_derefs(nir, indirect_mask);
 581
 582    nir_lower_int64(nir, nir_lower_imul64 |
 583                         nir_lower_isign64 |
 584                         nir_lower_divmod64);
 585
 586    /* Get rid of split copies */
 587    nir = nir_optimize(nir, compiler, is_scalar);
 588
 589    OPT(nir_remove_dead_variables, nir_var_local);
 590
 591    return nir;
 592 }
 593
 594 /* Prepare the given shader for codegen
 595  *
 596  * This function is intended to be called right before going into the actual
 597  * backend and is highly backend-specific.  Also, once this function has been
 598  * called on a shader, it will no longer be in SSA form so most optimizations
 599  * will not work.
 600  */
 601 nir_shader *
 602 brw_postprocess_nir(nir_shader *nir, const struct brw_compiler *compiler,
 603                     bool is_scalar)
 604 {
 605    const struct gen_device_info *devinfo = compiler->devinfo;
 606    bool debug_enabled =
 607       (INTEL_DEBUG & intel_debug_flag_for_shader_stage(nir->stage));
 608
 609    bool progress; /* Written by OPT and OPT_V */
 610    (void)progress;
 611
 612    nir = nir_optimize(nir, compiler, is_scalar);
 613
 614    if (devinfo->gen >= 6) {
 615       /* Try and fuse multiply-adds */
 616       OPT(brw_nir_opt_peephole_ffma);
 617    }
 618
 619    OPT(nir_opt_algebraic_late);
 620
 621    OPT_V(nir_lower_to_source_mods);
 622    OPT(nir_copy_prop);
 623    OPT(nir_opt_dce);
 624    OPT(nir_opt_move_comparisons);
 625
 626    OPT(nir_lower_locals_to_regs);
 627
 628    if (unlikely(debug_enabled)) {
 629       /* Re-index SSA defs so we print more sensible numbers. */
 630       nir_foreach_function(function, nir) {
 631          if (function->impl)
 632             nir_index_ssa_defs(function->impl);
 633       }
 634
 635       fprintf(stderr, "NIR (SSA form) for %s shader:\n",
 636               _mesa_shader_stage_to_string(nir->stage));
 637       nir_print_shader(nir, stderr);
 638    }
 639
 640    OPT_V(nir_convert_from_ssa, true);
 641
 642    if (!is_scalar) {
 643       OPT_V(nir_move_vec_src_uses_to_dest);
 644       OPT(nir_lower_vec_to_movs);
 645    }
 646
 647    /* This is the last pass we run before we start emitting stuff.  It
 648     * determines when we need to insert boolean resolves on Gen <= 5.  We
 649     * run it last because it stashes data in instr->pass_flags and we don't
 650     * want that to be squashed by other NIR passes.
 651     */
 652    if (devinfo->gen <= 5)
 653       brw_nir_analyze_boolean_resolves(nir);
 654
 655    nir_sweep(nir);
 656
 657    if (unlikely(debug_enabled)) {
 658       fprintf(stderr, "NIR (final form) for %s shader:\n",
 659               _mesa_shader_stage_to_string(nir->stage));
 660       nir_print_shader(nir, stderr);
 661    }
 662
 663    return nir;
 664 }
 665
 666 nir_shader *
 667 brw_nir_apply_sampler_key(nir_shader *nir,
 668                           const struct brw_compiler *compiler,
 669                           const struct brw_sampler_prog_key_data *key_tex,
 670                           bool is_scalar)
 671 {
 672    const struct gen_device_info *devinfo = compiler->devinfo;
 673    nir_lower_tex_options tex_options = { 0 };
 674
 675    /* Iron Lake and prior require lowering of all rectangle textures */
 676    if (devinfo->gen < 6)
 677       tex_options.lower_rect = true;
 678
 679    /* Prior to Broadwell, our hardware can't actually do GL_CLAMP */
 680    if (devinfo->gen < 8) {
 681       tex_options.saturate_s = key_tex->gl_clamp_mask[0];
 682       tex_options.saturate_t = key_tex->gl_clamp_mask[1];
 683       tex_options.saturate_r = key_tex->gl_clamp_mask[2];
 684    }
 685
 686    /* Prior to Haswell, we have to fake texture swizzle */
 687    for (unsigned s = 0; s < MAX_SAMPLERS; s++) {
 688       if (key_tex->swizzles[s] == SWIZZLE_NOOP)
 689          continue;
 690
 691       tex_options.swizzle_result |= (1 << s);
 692       for (unsigned c = 0; c < 4; c++)
 693          tex_options.swizzles[s][c] = GET_SWZ(key_tex->swizzles[s], c);
 694    }
 695
 696    /* Prior to Haswell, we have to lower gradients on shadow samplers */
 697    tex_options.lower_txd_shadow = devinfo->gen < 8 && !devinfo->is_haswell;
 698
 699    tex_options.lower_y_uv_external = key_tex->y_uv_image_mask;
 700    tex_options.lower_y_u_v_external = key_tex->y_u_v_image_mask;
 701    tex_options.lower_yx_xuxv_external = key_tex->yx_xuxv_image_mask;
 702
 703    if (nir_lower_tex(nir, &tex_options)) {
 704       nir_validate_shader(nir);
 705       nir = nir_optimize(nir, compiler, is_scalar);
 706    }
 707
 708    return nir;
 709 }
 710
 711 enum brw_reg_type
 712 brw_type_for_nir_type(const struct gen_device_info *devinfo, nir_alu_type type)
 713 {
 714    switch (type) {
 715    case nir_type_uint:
 716    case nir_type_uint32:
 717       return BRW_REGISTER_TYPE_UD;
 718    case nir_type_bool:
 719    case nir_type_int:
 720    case nir_type_bool32:
 721    case nir_type_int32:
 722       return BRW_REGISTER_TYPE_D;
 723    case nir_type_float:
 724    case nir_type_float32:
 725       return BRW_REGISTER_TYPE_F;
 726    case nir_type_float64:
 727       return BRW_REGISTER_TYPE_DF;
 728    case nir_type_int64:
 729       return devinfo->gen < 8 ? BRW_REGISTER_TYPE_DF : BRW_REGISTER_TYPE_Q;
 730    case nir_type_uint64:
 731       return devinfo->gen < 8 ? BRW_REGISTER_TYPE_DF : BRW_REGISTER_TYPE_UQ;
 732    default:
 733       unreachable("unknown type");
 734    }
 735
 736    return BRW_REGISTER_TYPE_F;
 737 }
 738
 739 /* Returns the glsl_base_type corresponding to a nir_alu_type.
 740  * This is used by both brw_vec4_nir and brw_fs_nir.
 741  */
 742 enum glsl_base_type
 743 brw_glsl_base_type_for_nir_type(nir_alu_type type)
 744 {
 745    switch (type) {
 746    case nir_type_float:
 747    case nir_type_float32:
 748       return GLSL_TYPE_FLOAT;
 749
 750    case nir_type_float64:
 751       return GLSL_TYPE_DOUBLE;
 752
 753    case nir_type_int:
 754    case nir_type_int32:
 755       return GLSL_TYPE_INT;
 756
 757    case nir_type_uint:
 758    case nir_type_uint32:
 759       return GLSL_TYPE_UINT;
 760
 761    default:
 762       unreachable("bad type");
 763    }
 764 }