src/gallium/drivers/radeonsi/si_shader_nir.c

   1 /*
   2  * Copyright 2017 Advanced Micro Devices, Inc.
   3  * All Rights Reserved.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * on the rights to use, copy, modify, merge, publish, distribute, sub
   9  * license, and/or sell copies of the Software, and to permit persons to whom
  10  * the Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  19  * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
  20  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  21  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  22  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  23  */
  24
  25 #include "ac_nir_to_llvm.h"
  26 #include "compiler/nir/nir.h"
  27 #include "compiler/nir/nir_builder.h"
  28 #include "compiler/nir/nir_deref.h"
  29 #include "compiler/nir_types.h"
  30 #include "si_pipe.h"
  31 #include "si_shader_internal.h"
  32 #include "tgsi/tgsi_from_mesa.h"
  33
  34 static const nir_deref_instr *tex_get_texture_deref(nir_tex_instr *instr)
  35 {
  36    for (unsigned i = 0; i < instr->num_srcs; i++) {
  37       switch (instr->src[i].src_type) {
  38       case nir_tex_src_texture_deref:
  39          return nir_src_as_deref(instr->src[i].src);
  40       default:
  41          break;
  42       }
  43    }
  44
  45    return NULL;
  46 }
  47
  48 static nir_variable *intrinsic_get_var(nir_intrinsic_instr *instr)
  49 {
  50    return nir_deref_instr_get_variable(nir_src_as_deref(instr->src[0]));
  51 }
  52
  53 static void gather_usage_helper(const nir_deref_instr **deref_ptr, unsigned location, uint8_t mask,
  54                                 uint8_t *usage_mask)
  55 {
  56    for (; *deref_ptr; deref_ptr++) {
  57       const nir_deref_instr *deref = *deref_ptr;
  58       switch (deref->deref_type) {
  59       case nir_deref_type_array: {
  60          unsigned elem_size = glsl_count_attribute_slots(deref->type, false);
  61          if (nir_src_is_const(deref->arr.index)) {
  62             location += elem_size * nir_src_as_uint(deref->arr.index);
  63          } else {
  64             unsigned array_elems = glsl_get_length(deref_ptr[-1]->type);
  65             for (unsigned i = 0; i < array_elems; i++) {
  66                gather_usage_helper(deref_ptr + 1, location + elem_size * i, mask, usage_mask);
  67             }
  68             return;
  69          }
  70          break;
  71       }
  72       case nir_deref_type_struct: {
  73          const struct glsl_type *parent_type = deref_ptr[-1]->type;
  74          unsigned index = deref->strct.index;
  75          for (unsigned i = 0; i < index; i++) {
  76             const struct glsl_type *ft = glsl_get_struct_field(parent_type, i);
  77             location += glsl_count_attribute_slots(ft, false);
  78          }
  79          break;
  80       }
  81       default:
  82          unreachable("Unhandled deref type in gather_components_used_helper");
  83       }
  84    }
  85
  86    usage_mask[location] |= mask & 0xf;
  87    if (mask & 0xf0)
  88       usage_mask[location + 1] |= (mask >> 4) & 0xf;
  89 }
  90
  91 static void gather_usage(const nir_deref_instr *deref, uint8_t mask, uint8_t *usage_mask)
  92 {
  93    nir_deref_path path;
  94    nir_deref_path_init(&path, (nir_deref_instr *)deref, NULL);
  95
  96    unsigned location_frac = path.path[0]->var->data.location_frac;
  97    if (glsl_type_is_64bit(deref->type)) {
  98       uint8_t new_mask = 0;
  99       for (unsigned i = 0; i < 4; i++) {
 100          if (mask & (1 << i))
 101             new_mask |= 0x3 << (2 * i);
 102       }
 103       mask = new_mask << location_frac;
 104    } else {
 105       mask <<= location_frac;
 106       mask &= 0xf;
 107    }
 108
 109    gather_usage_helper((const nir_deref_instr **)&path.path[1],
 110                        path.path[0]->var->data.driver_location, mask, usage_mask);
 111
 112    nir_deref_path_finish(&path);
 113 }
 114
 115 static void gather_intrinsic_load_deref_input_info(const nir_shader *nir,
 116                                                    const nir_intrinsic_instr *instr,
 117                                                    const nir_deref_instr *deref,
 118                                                    struct si_shader_info *info)
 119 {
 120    switch (nir->info.stage) {
 121    case MESA_SHADER_VERTEX:
 122       gather_usage(deref, nir_ssa_def_components_read(&instr->dest.ssa), info->input_usage_mask);
 123    default:;
 124    }
 125 }
 126
 127 static void gather_intrinsic_load_deref_output_info(const nir_shader *nir,
 128                                                     const nir_intrinsic_instr *instr,
 129                                                     nir_variable *var, struct si_shader_info *info)
 130 {
 131    assert(var && var->data.mode == nir_var_shader_out);
 132
 133    switch (nir->info.stage) {
 134    case MESA_SHADER_TESS_CTRL:
 135       if (var->data.location == VARYING_SLOT_TESS_LEVEL_INNER ||
 136           var->data.location == VARYING_SLOT_TESS_LEVEL_OUTER)
 137          info->reads_tessfactor_outputs = true;
 138       else if (var->data.patch)
 139          info->reads_perpatch_outputs = true;
 140       else
 141          info->reads_pervertex_outputs = true;
 142       break;
 143
 144    case MESA_SHADER_FRAGMENT:
 145       if (var->data.fb_fetch_output)
 146          info->uses_fbfetch = true;
 147       break;
 148    default:;
 149    }
 150 }
 151
 152 static void gather_intrinsic_store_deref_output_info(const nir_shader *nir,
 153                                                      const nir_intrinsic_instr *instr,
 154                                                      const nir_deref_instr *deref,
 155                                                      struct si_shader_info *info)
 156 {
 157    switch (nir->info.stage) {
 158    case MESA_SHADER_VERTEX:    /* needed by LS, ES */
 159    case MESA_SHADER_TESS_EVAL: /* needed by ES */
 160    case MESA_SHADER_GEOMETRY:
 161       gather_usage(deref, nir_intrinsic_write_mask(instr), info->output_usagemask);
 162       break;
 163    default:;
 164    }
 165 }
 166
 167 static void scan_instruction(const struct nir_shader *nir, struct si_shader_info *info,
 168                              nir_instr *instr)
 169 {
 170    if (instr->type == nir_instr_type_alu) {
 171       nir_alu_instr *alu = nir_instr_as_alu(instr);
 172
 173       switch (alu->op) {
 174       case nir_op_fddx:
 175       case nir_op_fddy:
 176       case nir_op_fddx_fine:
 177       case nir_op_fddy_fine:
 178       case nir_op_fddx_coarse:
 179       case nir_op_fddy_coarse:
 180          info->uses_derivatives = true;
 181          break;
 182       default:
 183          break;
 184       }
 185    } else if (instr->type == nir_instr_type_tex) {
 186       nir_tex_instr *tex = nir_instr_as_tex(instr);
 187       const nir_deref_instr *deref = tex_get_texture_deref(tex);
 188       nir_variable *var = deref ? nir_deref_instr_get_variable(deref) : NULL;
 189
 190       if (!var) {
 191          info->samplers_declared |= u_bit_consecutive(tex->sampler_index, 1);
 192       } else {
 193          if (deref->mode != nir_var_uniform || var->data.bindless)
 194             info->uses_bindless_samplers = true;
 195       }
 196
 197       switch (tex->op) {
 198       case nir_texop_tex:
 199       case nir_texop_txb:
 200       case nir_texop_lod:
 201          info->uses_derivatives = true;
 202          break;
 203       default:
 204          break;
 205       }
 206    } else if (instr->type == nir_instr_type_intrinsic) {
 207       nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
 208
 209       switch (intr->intrinsic) {
 210       case nir_intrinsic_load_front_face:
 211          info->uses_frontface = 1;
 212          break;
 213       case nir_intrinsic_load_instance_id:
 214          info->uses_instanceid = 1;
 215          break;
 216       case nir_intrinsic_load_invocation_id:
 217          info->uses_invocationid = true;
 218          break;
 219       case nir_intrinsic_load_num_work_groups:
 220          info->uses_grid_size = true;
 221          break;
 222       case nir_intrinsic_load_local_invocation_index:
 223       case nir_intrinsic_load_subgroup_id:
 224       case nir_intrinsic_load_num_subgroups:
 225          info->uses_subgroup_info = true;
 226          break;
 227       case nir_intrinsic_load_local_group_size:
 228          /* The block size is translated to IMM with a fixed block size. */
 229          if (info->properties[TGSI_PROPERTY_CS_FIXED_BLOCK_WIDTH] == 0)
 230             info->uses_block_size = true;
 231          break;
 232       case nir_intrinsic_load_local_invocation_id:
 233       case nir_intrinsic_load_work_group_id: {
 234          unsigned mask = nir_ssa_def_components_read(&intr->dest.ssa);
 235          while (mask) {
 236             unsigned i = u_bit_scan(&mask);
 237
 238             if (intr->intrinsic == nir_intrinsic_load_work_group_id)
 239                info->uses_block_id[i] = true;
 240             else
 241                info->uses_thread_id[i] = true;
 242          }
 243          break;
 244       }
 245       case nir_intrinsic_load_vertex_id:
 246          info->uses_vertexid = 1;
 247          break;
 248       case nir_intrinsic_load_vertex_id_zero_base:
 249          info->uses_vertexid_nobase = 1;
 250          break;
 251       case nir_intrinsic_load_base_vertex:
 252          info->uses_basevertex = 1;
 253          break;
 254       case nir_intrinsic_load_draw_id:
 255          info->uses_drawid = 1;
 256          break;
 257       case nir_intrinsic_load_primitive_id:
 258          info->uses_primid = 1;
 259          break;
 260       case nir_intrinsic_load_sample_mask_in:
 261          info->reads_samplemask = true;
 262          break;
 263       case nir_intrinsic_load_tess_level_inner:
 264       case nir_intrinsic_load_tess_level_outer:
 265          info->reads_tess_factors = true;
 266          break;
 267       case nir_intrinsic_bindless_image_load:
 268       case nir_intrinsic_bindless_image_size:
 269       case nir_intrinsic_bindless_image_samples:
 270          info->uses_bindless_images = true;
 271          break;
 272       case nir_intrinsic_bindless_image_store:
 273          info->uses_bindless_images = true;
 274          info->writes_memory = true;
 275          info->num_memory_instructions++; /* we only care about stores */
 276          break;
 277       case nir_intrinsic_image_deref_store:
 278          info->writes_memory = true;
 279          info->num_memory_instructions++; /* we only care about stores */
 280          break;
 281       case nir_intrinsic_bindless_image_atomic_add:
 282       case nir_intrinsic_bindless_image_atomic_imin:
 283       case nir_intrinsic_bindless_image_atomic_umin:
 284       case nir_intrinsic_bindless_image_atomic_imax:
 285       case nir_intrinsic_bindless_image_atomic_umax:
 286       case nir_intrinsic_bindless_image_atomic_and:
 287       case nir_intrinsic_bindless_image_atomic_or:
 288       case nir_intrinsic_bindless_image_atomic_xor:
 289       case nir_intrinsic_bindless_image_atomic_exchange:
 290       case nir_intrinsic_bindless_image_atomic_comp_swap:
 291          info->uses_bindless_images = true;
 292          info->writes_memory = true;
 293          info->num_memory_instructions++; /* we only care about stores */
 294          break;
 295       case nir_intrinsic_image_deref_atomic_add:
 296       case nir_intrinsic_image_deref_atomic_imin:
 297       case nir_intrinsic_image_deref_atomic_umin:
 298       case nir_intrinsic_image_deref_atomic_imax:
 299       case nir_intrinsic_image_deref_atomic_umax:
 300       case nir_intrinsic_image_deref_atomic_and:
 301       case nir_intrinsic_image_deref_atomic_or:
 302       case nir_intrinsic_image_deref_atomic_xor:
 303       case nir_intrinsic_image_deref_atomic_exchange:
 304       case nir_intrinsic_image_deref_atomic_comp_swap:
 305       case nir_intrinsic_image_deref_atomic_inc_wrap:
 306       case nir_intrinsic_image_deref_atomic_dec_wrap:
 307          info->writes_memory = true;
 308          info->num_memory_instructions++; /* we only care about stores */
 309          break;
 310       case nir_intrinsic_store_ssbo:
 311       case nir_intrinsic_ssbo_atomic_add:
 312       case nir_intrinsic_ssbo_atomic_imin:
 313       case nir_intrinsic_ssbo_atomic_umin:
 314       case nir_intrinsic_ssbo_atomic_imax:
 315       case nir_intrinsic_ssbo_atomic_umax:
 316       case nir_intrinsic_ssbo_atomic_and:
 317       case nir_intrinsic_ssbo_atomic_or:
 318       case nir_intrinsic_ssbo_atomic_xor:
 319       case nir_intrinsic_ssbo_atomic_exchange:
 320       case nir_intrinsic_ssbo_atomic_comp_swap:
 321          info->writes_memory = true;
 322          info->num_memory_instructions++; /* we only care about stores */
 323          break;
 324       case nir_intrinsic_load_color0:
 325       case nir_intrinsic_load_color1: {
 326          unsigned index = intr->intrinsic == nir_intrinsic_load_color1;
 327          uint8_t mask = nir_ssa_def_components_read(&intr->dest.ssa);
 328          info->colors_read |= mask << (index * 4);
 329          break;
 330       }
 331       case nir_intrinsic_load_barycentric_pixel:
 332       case nir_intrinsic_load_barycentric_centroid:
 333       case nir_intrinsic_load_barycentric_sample:
 334       case nir_intrinsic_load_barycentric_at_offset:   /* uses center */
 335       case nir_intrinsic_load_barycentric_at_sample: { /* uses center */
 336          unsigned mode = nir_intrinsic_interp_mode(intr);
 337
 338          if (mode == INTERP_MODE_FLAT)
 339             break;
 340
 341          if (mode == INTERP_MODE_NOPERSPECTIVE) {
 342             if (intr->intrinsic == nir_intrinsic_load_barycentric_sample)
 343                info->uses_linear_sample = true;
 344             else if (intr->intrinsic == nir_intrinsic_load_barycentric_centroid)
 345                info->uses_linear_centroid = true;
 346             else
 347                info->uses_linear_center = true;
 348
 349             if (intr->intrinsic == nir_intrinsic_load_barycentric_at_sample)
 350                info->uses_linear_opcode_interp_sample = true;
 351          } else {
 352             if (intr->intrinsic == nir_intrinsic_load_barycentric_sample)
 353                info->uses_persp_sample = true;
 354             else if (intr->intrinsic == nir_intrinsic_load_barycentric_centroid)
 355                info->uses_persp_centroid = true;
 356             else
 357                info->uses_persp_center = true;
 358
 359             if (intr->intrinsic == nir_intrinsic_load_barycentric_at_sample)
 360                info->uses_persp_opcode_interp_sample = true;
 361          }
 362          break;
 363       }
 364       case nir_intrinsic_load_deref: {
 365          nir_variable *var = intrinsic_get_var(intr);
 366          nir_variable_mode mode = var->data.mode;
 367
 368          if (mode == nir_var_shader_in) {
 369             /* PS inputs use the interpolated load intrinsics. */
 370             assert(nir->info.stage != MESA_SHADER_FRAGMENT);
 371             gather_intrinsic_load_deref_input_info(nir, intr, nir_src_as_deref(intr->src[0]), info);
 372          } else if (mode == nir_var_shader_out) {
 373             gather_intrinsic_load_deref_output_info(nir, intr, var, info);
 374          }
 375          break;
 376       }
 377       case nir_intrinsic_store_deref: {
 378          nir_variable *var = intrinsic_get_var(intr);
 379
 380          if (var->data.mode == nir_var_shader_out)
 381             gather_intrinsic_store_deref_output_info(nir, intr, nir_src_as_deref(intr->src[0]),
 382                                                      info);
 383          break;
 384       }
 385       case nir_intrinsic_interp_deref_at_centroid:
 386       case nir_intrinsic_interp_deref_at_sample:
 387       case nir_intrinsic_interp_deref_at_offset:
 388          unreachable("interp opcodes should have been lowered");
 389          break;
 390       default:
 391          break;
 392       }
 393    }
 394 }
 395
 396 static void scan_output_slot(const nir_variable *var, unsigned var_idx, unsigned component,
 397                              unsigned num_components, struct si_shader_info *info)
 398 {
 399    assert(component + num_components <= 4);
 400    assert(component < 4);
 401
 402    unsigned semantic_name, semantic_index;
 403
 404    unsigned location = var->data.location + var_idx;
 405    unsigned drv_location = var->data.driver_location + var_idx;
 406
 407    if (info->processor == PIPE_SHADER_FRAGMENT) {
 408       tgsi_get_gl_frag_result_semantic(location, &semantic_name, &semantic_index);
 409
 410       /* Adjust for dual source blending */
 411       if (var->data.index > 0) {
 412          semantic_index++;
 413       }
 414    } else {
 415       tgsi_get_gl_varying_semantic(location, true, &semantic_name, &semantic_index);
 416    }
 417
 418    ubyte usagemask = ((1 << num_components) - 1) << component;
 419
 420    unsigned gs_out_streams;
 421    if (var->data.stream & NIR_STREAM_PACKED) {
 422       gs_out_streams = var->data.stream & ~NIR_STREAM_PACKED;
 423    } else {
 424       assert(var->data.stream < 4);
 425       gs_out_streams = 0;
 426       for (unsigned j = 0; j < num_components; ++j)
 427          gs_out_streams |= var->data.stream << (2 * (component + j));
 428    }
 429
 430    unsigned streamx = gs_out_streams & 3;
 431    unsigned streamy = (gs_out_streams >> 2) & 3;
 432    unsigned streamz = (gs_out_streams >> 4) & 3;
 433    unsigned streamw = (gs_out_streams >> 6) & 3;
 434
 435    if (usagemask & TGSI_WRITEMASK_X) {
 436       info->output_streams[drv_location] |= streamx;
 437       info->num_stream_output_components[streamx]++;
 438    }
 439    if (usagemask & TGSI_WRITEMASK_Y) {
 440       info->output_streams[drv_location] |= streamy << 2;
 441       info->num_stream_output_components[streamy]++;
 442    }
 443    if (usagemask & TGSI_WRITEMASK_Z) {
 444       info->output_streams[drv_location] |= streamz << 4;
 445       info->num_stream_output_components[streamz]++;
 446    }
 447    if (usagemask & TGSI_WRITEMASK_W) {
 448       info->output_streams[drv_location] |= streamw << 6;
 449       info->num_stream_output_components[streamw]++;
 450    }
 451
 452    info->output_semantic_name[drv_location] = semantic_name;
 453    info->output_semantic_index[drv_location] = semantic_index;
 454
 455    switch (semantic_name) {
 456    case TGSI_SEMANTIC_PRIMID:
 457       info->writes_primid = true;
 458       break;
 459    case TGSI_SEMANTIC_VIEWPORT_INDEX:
 460       info->writes_viewport_index = true;
 461       break;
 462    case TGSI_SEMANTIC_LAYER:
 463       info->writes_layer = true;
 464       break;
 465    case TGSI_SEMANTIC_PSIZE:
 466       info->writes_psize = true;
 467       break;
 468    case TGSI_SEMANTIC_CLIPVERTEX:
 469       info->writes_clipvertex = true;
 470       break;
 471    case TGSI_SEMANTIC_COLOR:
 472       info->colors_written |= 1 << semantic_index;
 473       break;
 474    case TGSI_SEMANTIC_STENCIL:
 475       info->writes_stencil = true;
 476       break;
 477    case TGSI_SEMANTIC_SAMPLEMASK:
 478       info->writes_samplemask = true;
 479       break;
 480    case TGSI_SEMANTIC_EDGEFLAG:
 481       info->writes_edgeflag = true;
 482       break;
 483    case TGSI_SEMANTIC_POSITION:
 484       if (info->processor == PIPE_SHADER_FRAGMENT)
 485          info->writes_z = true;
 486       else
 487          info->writes_position = true;
 488       break;
 489    }
 490 }
 491
 492 static void scan_output_helper(const nir_variable *var, unsigned location,
 493                                const struct glsl_type *type, struct si_shader_info *info)
 494 {
 495    if (glsl_type_is_struct(type) || glsl_type_is_interface(type)) {
 496       for (unsigned i = 0; i < glsl_get_length(type); i++) {
 497          const struct glsl_type *ft = glsl_get_struct_field(type, i);
 498          scan_output_helper(var, location, ft, info);
 499          location += glsl_count_attribute_slots(ft, false);
 500       }
 501    } else if (glsl_type_is_array_or_matrix(type)) {
 502       const struct glsl_type *elem_type = glsl_get_array_element(type);
 503       unsigned num_elems = glsl_get_length(type);
 504       if (var->data.compact) {
 505          assert(glsl_type_is_scalar(elem_type));
 506          assert(glsl_get_bit_size(elem_type) == 32);
 507          unsigned component = var->data.location_frac;
 508          scan_output_slot(var, location, component, MIN2(num_elems, 4 - component), info);
 509          if (component + num_elems > 4) {
 510             scan_output_slot(var, location + 1, 0, component + num_elems - 4, info);
 511          }
 512
 513       } else {
 514          unsigned elem_count = glsl_count_attribute_slots(elem_type, false);
 515          for (unsigned i = 0; i < num_elems; i++) {
 516             scan_output_helper(var, location, elem_type, info);
 517             location += elem_count;
 518          }
 519       }
 520    } else if (glsl_type_is_dual_slot(type)) {
 521       unsigned component = var->data.location_frac;
 522       scan_output_slot(var, location, component, 4 - component, info);
 523       scan_output_slot(var, location + 1, 0, component + 2 * glsl_get_components(type) - 4, info);
 524    } else {
 525       unsigned component = var->data.location_frac;
 526       assert(glsl_type_is_vector_or_scalar(type));
 527       unsigned num_components = glsl_get_components(type);
 528       if (glsl_type_is_64bit(type))
 529          num_components *= 2;
 530       scan_output_slot(var, location, component, num_components, info);
 531    }
 532 }
 533
 534 void si_nir_scan_shader(const struct nir_shader *nir, struct si_shader_info *info)
 535 {
 536    nir_function *func;
 537    unsigned i;
 538
 539    info->processor = pipe_shader_type_from_mesa(nir->info.stage);
 540
 541    info->properties[TGSI_PROPERTY_NEXT_SHADER] = pipe_shader_type_from_mesa(nir->info.next_stage);
 542
 543    if (nir->info.stage == MESA_SHADER_VERTEX) {
 544       info->properties[TGSI_PROPERTY_VS_WINDOW_SPACE_POSITION] = nir->info.vs.window_space_position;
 545       info->properties[TGSI_PROPERTY_VS_BLIT_SGPRS_AMD] = nir->info.vs.blit_sgprs_amd;
 546    }
 547
 548    if (nir->info.stage == MESA_SHADER_TESS_CTRL) {
 549       info->properties[TGSI_PROPERTY_TCS_VERTICES_OUT] = nir->info.tess.tcs_vertices_out;
 550    }
 551
 552    if (nir->info.stage == MESA_SHADER_TESS_EVAL) {
 553       if (nir->info.tess.primitive_mode == GL_ISOLINES)
 554          info->properties[TGSI_PROPERTY_TES_PRIM_MODE] = PIPE_PRIM_LINES;
 555       else
 556          info->properties[TGSI_PROPERTY_TES_PRIM_MODE] = nir->info.tess.primitive_mode;
 557
 558       STATIC_ASSERT((TESS_SPACING_EQUAL + 1) % 3 == PIPE_TESS_SPACING_EQUAL);
 559       STATIC_ASSERT((TESS_SPACING_FRACTIONAL_ODD + 1) % 3 == PIPE_TESS_SPACING_FRACTIONAL_ODD);
 560       STATIC_ASSERT((TESS_SPACING_FRACTIONAL_EVEN + 1) % 3 == PIPE_TESS_SPACING_FRACTIONAL_EVEN);
 561
 562       info->properties[TGSI_PROPERTY_TES_SPACING] = (nir->info.tess.spacing + 1) % 3;
 563       info->properties[TGSI_PROPERTY_TES_VERTEX_ORDER_CW] = !nir->info.tess.ccw;
 564       info->properties[TGSI_PROPERTY_TES_POINT_MODE] = nir->info.tess.point_mode;
 565    }
 566
 567    if (nir->info.stage == MESA_SHADER_GEOMETRY) {
 568       info->properties[TGSI_PROPERTY_GS_INPUT_PRIM] = nir->info.gs.input_primitive;
 569       info->properties[TGSI_PROPERTY_GS_OUTPUT_PRIM] = nir->info.gs.output_primitive;
 570       info->properties[TGSI_PROPERTY_GS_MAX_OUTPUT_VERTICES] = nir->info.gs.vertices_out;
 571       info->properties[TGSI_PROPERTY_GS_INVOCATIONS] = nir->info.gs.invocations;
 572    }
 573
 574    if (nir->info.stage == MESA_SHADER_FRAGMENT) {
 575       info->properties[TGSI_PROPERTY_FS_EARLY_DEPTH_STENCIL] =
 576          nir->info.fs.early_fragment_tests | nir->info.fs.post_depth_coverage;
 577       info->properties[TGSI_PROPERTY_FS_POST_DEPTH_COVERAGE] = nir->info.fs.post_depth_coverage;
 578
 579       if (nir->info.fs.pixel_center_integer) {
 580          info->properties[TGSI_PROPERTY_FS_COORD_PIXEL_CENTER] = TGSI_FS_COORD_PIXEL_CENTER_INTEGER;
 581       }
 582
 583       if (nir->info.fs.depth_layout != FRAG_DEPTH_LAYOUT_NONE) {
 584          switch (nir->info.fs.depth_layout) {
 585          case FRAG_DEPTH_LAYOUT_ANY:
 586             info->properties[TGSI_PROPERTY_FS_DEPTH_LAYOUT] = TGSI_FS_DEPTH_LAYOUT_ANY;
 587             break;
 588          case FRAG_DEPTH_LAYOUT_GREATER:
 589             info->properties[TGSI_PROPERTY_FS_DEPTH_LAYOUT] = TGSI_FS_DEPTH_LAYOUT_GREATER;
 590             break;
 591          case FRAG_DEPTH_LAYOUT_LESS:
 592             info->properties[TGSI_PROPERTY_FS_DEPTH_LAYOUT] = TGSI_FS_DEPTH_LAYOUT_LESS;
 593             break;
 594          case FRAG_DEPTH_LAYOUT_UNCHANGED:
 595             info->properties[TGSI_PROPERTY_FS_DEPTH_LAYOUT] = TGSI_FS_DEPTH_LAYOUT_UNCHANGED;
 596             break;
 597          default:
 598             unreachable("Unknow depth layout");
 599          }
 600       }
 601    }
 602
 603    if (gl_shader_stage_is_compute(nir->info.stage)) {
 604       info->properties[TGSI_PROPERTY_CS_FIXED_BLOCK_WIDTH] = nir->info.cs.local_size[0];
 605       info->properties[TGSI_PROPERTY_CS_FIXED_BLOCK_HEIGHT] = nir->info.cs.local_size[1];
 606       info->properties[TGSI_PROPERTY_CS_FIXED_BLOCK_DEPTH] = nir->info.cs.local_size[2];
 607       info->properties[TGSI_PROPERTY_CS_USER_DATA_COMPONENTS_AMD] =
 608          nir->info.cs.user_data_components_amd;
 609    }
 610
 611    i = 0;
 612    uint64_t processed_inputs = 0;
 613    nir_foreach_variable (variable, &nir->inputs) {
 614       unsigned semantic_name, semantic_index;
 615
 616       const struct glsl_type *type = variable->type;
 617       if (nir_is_per_vertex_io(variable, nir->info.stage)) {
 618          assert(glsl_type_is_array(type));
 619          type = glsl_get_array_element(type);
 620       }
 621
 622       unsigned attrib_count =
 623          glsl_count_attribute_slots(type, nir->info.stage == MESA_SHADER_VERTEX);
 624
 625       i = variable->data.driver_location;
 626
 627       /* Vertex shader inputs don't have semantics. The state
 628        * tracker has already mapped them to attributes via
 629        * variable->data.driver_location.
 630        */
 631       if (nir->info.stage == MESA_SHADER_VERTEX)
 632          continue;
 633
 634       for (unsigned j = 0; j < attrib_count; j++, i++) {
 635
 636          if (processed_inputs & ((uint64_t)1 << i))
 637             continue;
 638
 639          processed_inputs |= ((uint64_t)1 << i);
 640
 641          tgsi_get_gl_varying_semantic(variable->data.location + j, true, &semantic_name,
 642                                       &semantic_index);
 643
 644          info->input_semantic_name[i] = semantic_name;
 645          info->input_semantic_index[i] = semantic_index;
 646
 647          if (semantic_name == TGSI_SEMANTIC_PRIMID)
 648             info->uses_primid = true;
 649
 650          if (semantic_name == TGSI_SEMANTIC_COLOR) {
 651             /* We only need this for color inputs. */
 652             if (variable->data.sample)
 653                info->input_interpolate_loc[i] = TGSI_INTERPOLATE_LOC_SAMPLE;
 654             else if (variable->data.centroid)
 655                info->input_interpolate_loc[i] = TGSI_INTERPOLATE_LOC_CENTROID;
 656             else
 657                info->input_interpolate_loc[i] = TGSI_INTERPOLATE_LOC_CENTER;
 658          }
 659
 660          enum glsl_base_type base_type = glsl_get_base_type(glsl_without_array(variable->type));
 661
 662          switch (variable->data.interpolation) {
 663          case INTERP_MODE_NONE:
 664             if (glsl_base_type_is_integer(base_type)) {
 665                info->input_interpolate[i] = TGSI_INTERPOLATE_CONSTANT;
 666                break;
 667             }
 668
 669             if (semantic_name == TGSI_SEMANTIC_COLOR) {
 670                info->input_interpolate[i] = TGSI_INTERPOLATE_COLOR;
 671                break;
 672             }
 673             /* fall-through */
 674
 675          case INTERP_MODE_SMOOTH:
 676             assert(!glsl_base_type_is_integer(base_type));
 677
 678             info->input_interpolate[i] = TGSI_INTERPOLATE_PERSPECTIVE;
 679             break;
 680
 681          case INTERP_MODE_NOPERSPECTIVE:
 682             assert(!glsl_base_type_is_integer(base_type));
 683
 684             info->input_interpolate[i] = TGSI_INTERPOLATE_LINEAR;
 685             break;
 686
 687          case INTERP_MODE_FLAT:
 688             info->input_interpolate[i] = TGSI_INTERPOLATE_CONSTANT;
 689             break;
 690          }
 691       }
 692    }
 693
 694    nir_foreach_variable (variable, &nir->outputs) {
 695       const struct glsl_type *type = variable->type;
 696       if (nir_is_per_vertex_io(variable, nir->info.stage)) {
 697          assert(glsl_type_is_array(type));
 698          type = glsl_get_array_element(type);
 699       }
 700
 701       ASSERTED unsigned attrib_count = glsl_count_attribute_slots(type, false);
 702       scan_output_helper(variable, 0, type, info);
 703
 704       unsigned loc = variable->data.location;
 705       if (nir->info.stage == MESA_SHADER_FRAGMENT && loc == FRAG_RESULT_COLOR &&
 706           nir->info.outputs_written & (1ull << loc)) {
 707          assert(attrib_count == 1);
 708          info->properties[TGSI_PROPERTY_FS_COLOR0_WRITES_ALL_CBUFS] = true;
 709       }
 710    }
 711
 712    info->num_inputs = nir->num_inputs;
 713    info->num_outputs = nir->num_outputs;
 714
 715    info->constbuf0_num_slots = nir->num_uniforms;
 716    info->shader_buffers_declared = u_bit_consecutive(0, nir->info.num_ssbos);
 717    info->const_buffers_declared = u_bit_consecutive(0, nir->info.num_ubos);
 718    info->images_declared = u_bit_consecutive(0, nir->info.num_images);
 719    info->msaa_images_declared = nir->info.msaa_images;
 720    info->image_buffers = nir->info.image_buffers;
 721    info->samplers_declared = nir->info.textures_used;
 722
 723    info->num_written_clipdistance = nir->info.clip_distance_array_size;
 724    info->num_written_culldistance = nir->info.cull_distance_array_size;
 725    info->clipdist_writemask = u_bit_consecutive(0, info->num_written_clipdistance);
 726    info->culldist_writemask = u_bit_consecutive(0, info->num_written_culldistance);
 727
 728    if (info->processor == PIPE_SHADER_FRAGMENT)
 729       info->uses_kill = nir->info.fs.uses_discard;
 730
 731    if (nir->info.stage == MESA_SHADER_TESS_CTRL) {
 732       info->tessfactors_are_def_in_all_invocs = ac_are_tessfactors_def_in_all_invocs(nir);
 733    }
 734
 735    func = (struct nir_function *)exec_list_get_head_const(&nir->functions);
 736    nir_foreach_block (block, func->impl) {
 737       nir_foreach_instr (instr, block)
 738          scan_instruction(nir, info, instr);
 739    }
 740 }
 741
 742 static void si_nir_opts(struct nir_shader *nir)
 743 {
 744    bool progress;
 745
 746    do {
 747       progress = false;
 748
 749       NIR_PASS_V(nir, nir_lower_vars_to_ssa);
 750
 751       NIR_PASS(progress, nir, nir_opt_copy_prop_vars);
 752       NIR_PASS(progress, nir, nir_opt_dead_write_vars);
 753
 754       NIR_PASS_V(nir, nir_lower_alu_to_scalar, NULL, NULL);
 755       NIR_PASS_V(nir, nir_lower_phis_to_scalar);
 756
 757       /* (Constant) copy propagation is needed for txf with offsets. */
 758       NIR_PASS(progress, nir, nir_copy_prop);
 759       NIR_PASS(progress, nir, nir_opt_remove_phis);
 760       NIR_PASS(progress, nir, nir_opt_dce);
 761       if (nir_opt_trivial_continues(nir)) {
 762          progress = true;
 763          NIR_PASS(progress, nir, nir_copy_prop);
 764          NIR_PASS(progress, nir, nir_opt_dce);
 765       }
 766       NIR_PASS(progress, nir, nir_opt_if, true);
 767       NIR_PASS(progress, nir, nir_opt_dead_cf);
 768       NIR_PASS(progress, nir, nir_opt_cse);
 769       NIR_PASS(progress, nir, nir_opt_peephole_select, 8, true, true);
 770
 771       /* Needed for algebraic lowering */
 772       NIR_PASS(progress, nir, nir_opt_algebraic);
 773       NIR_PASS(progress, nir, nir_opt_constant_folding);
 774
 775       if (!nir->info.flrp_lowered) {
 776          unsigned lower_flrp = (nir->options->lower_flrp16 ? 16 : 0) |
 777                                (nir->options->lower_flrp32 ? 32 : 0) |
 778                                (nir->options->lower_flrp64 ? 64 : 0);
 779          assert(lower_flrp);
 780          bool lower_flrp_progress = false;
 781
 782          NIR_PASS(lower_flrp_progress, nir, nir_lower_flrp, lower_flrp, false /* always_precise */,
 783                   nir->options->lower_ffma);
 784          if (lower_flrp_progress) {
 785             NIR_PASS(progress, nir, nir_opt_constant_folding);
 786             progress = true;
 787          }
 788
 789          /* Nothing should rematerialize any flrps, so we only
 790           * need to do this lowering once.
 791           */
 792          nir->info.flrp_lowered = true;
 793       }
 794
 795       NIR_PASS(progress, nir, nir_opt_undef);
 796       NIR_PASS(progress, nir, nir_opt_conditional_discard);
 797       if (nir->options->max_unroll_iterations) {
 798          NIR_PASS(progress, nir, nir_opt_loop_unroll, 0);
 799       }
 800    } while (progress);
 801 }
 802
 803 static int type_size_vec4(const struct glsl_type *type, bool bindless)
 804 {
 805    return glsl_count_attribute_slots(type, false);
 806 }
 807
 808 static void si_nir_lower_color(nir_shader *nir)
 809 {
 810    nir_function_impl *entrypoint = nir_shader_get_entrypoint(nir);
 811
 812    nir_builder b;
 813    nir_builder_init(&b, entrypoint);
 814
 815    nir_foreach_block (block, entrypoint) {
 816       nir_foreach_instr_safe (instr, block) {
 817          if (instr->type != nir_instr_type_intrinsic)
 818             continue;
 819
 820          nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
 821
 822          if (intrin->intrinsic != nir_intrinsic_load_deref)
 823             continue;
 824
 825          nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
 826          if (deref->mode != nir_var_shader_in)
 827             continue;
 828
 829          b.cursor = nir_before_instr(instr);
 830          nir_variable *var = nir_deref_instr_get_variable(deref);
 831          nir_ssa_def *def;
 832
 833          if (var->data.location == VARYING_SLOT_COL0) {
 834             def = nir_load_color0(&b);
 835          } else if (var->data.location == VARYING_SLOT_COL1) {
 836             def = nir_load_color1(&b);
 837          } else {
 838             continue;
 839          }
 840
 841          nir_ssa_def_rewrite_uses(&intrin->dest.ssa, nir_src_for_ssa(def));
 842          nir_instr_remove(instr);
 843       }
 844    }
 845 }
 846
 847 static void si_nir_lower_ps_inputs(struct nir_shader *nir)
 848 {
 849    if (nir->info.stage != MESA_SHADER_FRAGMENT)
 850       return;
 851
 852    NIR_PASS_V(nir, nir_lower_io_to_temporaries, nir_shader_get_entrypoint(nir), false, true);
 853
 854    /* Since we're doing nir_lower_io_to_temporaries late, we need
 855     * to lower all the copy_deref's introduced by
 856     * lower_io_to_temporaries before calling nir_lower_io.
 857     */
 858    NIR_PASS_V(nir, nir_split_var_copies);
 859    NIR_PASS_V(nir, nir_lower_var_copies);
 860    NIR_PASS_V(nir, nir_lower_global_vars_to_local);
 861
 862    si_nir_lower_color(nir);
 863    NIR_PASS_V(nir, nir_lower_io, nir_var_shader_in, type_size_vec4, 0);
 864
 865    /* This pass needs actual constants */
 866    NIR_PASS_V(nir, nir_opt_constant_folding);
 867    NIR_PASS_V(nir, nir_io_add_const_offset_to_base, nir_var_shader_in);
 868 }
 869
 870 void si_nir_adjust_driver_locations(struct nir_shader *nir)
 871 {
 872    /* Adjust the driver location of inputs and outputs. the gallium frontend
 873     * interprets them as slots, while the ac/nir backend interprets them
 874     * as individual components.
 875     */
 876    if (nir->info.stage != MESA_SHADER_FRAGMENT) {
 877       nir_foreach_variable (variable, &nir->inputs)
 878          variable->data.driver_location *= 4;
 879    }
 880
 881    nir_foreach_variable (variable, &nir->outputs)
 882       variable->data.driver_location *= 4;
 883 }
 884
 885 /**
 886  * Perform "lowering" operations on the NIR that are run once when the shader
 887  * selector is created.
 888  */
 889 static void si_lower_nir(struct si_screen *sscreen, struct nir_shader *nir)
 890 {
 891    /* Perform lowerings (and optimizations) of code.
 892     *
 893     * Performance considerations aside, we must:
 894     * - lower certain ALU operations
 895     * - ensure constant offsets for texture instructions are folded
 896     *   and copy-propagated
 897     */
 898
 899    static const struct nir_lower_tex_options lower_tex_options = {
 900       .lower_txp = ~0u,
 901    };
 902    NIR_PASS_V(nir, nir_lower_tex, &lower_tex_options);
 903
 904    const nir_lower_subgroups_options subgroups_options = {
 905       .subgroup_size = 64,
 906       .ballot_bit_size = 64,
 907       .lower_to_scalar = true,
 908       .lower_subgroup_masks = true,
 909       .lower_vote_trivial = false,
 910       .lower_vote_eq_to_ballot = true,
 911    };
 912    NIR_PASS_V(nir, nir_lower_subgroups, &subgroups_options);
 913
 914    /* Lower load constants to scalar and then clean up the mess */
 915    NIR_PASS_V(nir, nir_lower_load_const_to_scalar);
 916    NIR_PASS_V(nir, nir_lower_var_copies);
 917    NIR_PASS_V(nir, nir_lower_pack);
 918    NIR_PASS_V(nir, nir_opt_access);
 919    si_nir_opts(nir);
 920
 921    /* Lower large variables that are always constant with load_constant
 922     * intrinsics, which get turned into PC-relative loads from a data
 923     * section next to the shader.
 924     *
 925     * st/mesa calls finalize_nir twice, but we can't call this pass twice.
 926     */
 927    bool changed = false;
 928    if (!nir->constant_data) {
 929       NIR_PASS(changed, nir, nir_opt_large_constants, glsl_get_natural_size_align_bytes, 16);
 930    }
 931
 932    changed |= ac_lower_indirect_derefs(nir, sscreen->info.chip_class);
 933    if (changed)
 934       si_nir_opts(nir);
 935
 936    NIR_PASS_V(nir, nir_lower_bool_to_int32);
 937    NIR_PASS_V(nir, nir_remove_dead_variables, nir_var_function_temp, NULL);
 938
 939    if (sscreen->debug_flags & DBG(FS_CORRECT_DERIVS_AFTER_KILL))
 940       NIR_PASS_V(nir, nir_lower_discard_to_demote);
 941 }
 942
 943 void si_finalize_nir(struct pipe_screen *screen, void *nirptr, bool optimize)
 944 {
 945    struct si_screen *sscreen = (struct si_screen *)screen;
 946    struct nir_shader *nir = (struct nir_shader *)nirptr;
 947
 948    nir_shader_gather_info(nir, nir_shader_get_entrypoint(nir));
 949    si_nir_lower_ps_inputs(nir);
 950    si_lower_nir(sscreen, nir);
 951 }