src/amd/vulkan/radv_shader_args.c

   1 /*
   2  * Copyright © 2019 Valve Corporation.
   3  * Copyright © 2016 Red Hat.
   4  * Copyright © 2016 Bas Nieuwenhuizen
   5  *
   6  * based in part on anv driver which is:
   7  * Copyright © 2015 Intel Corporation
   8  *
   9  * Permission is hereby granted, free of charge, to any person obtaining a
  10  * copy of this software and associated documentation files (the "Software"),
  11  * to deal in the Software without restriction, including without limitation
  12  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  13  * and/or sell copies of the Software, and to permit persons to whom the
  14  * Software is furnished to do so, subject to the following conditions:
  15  *
  16  * The above copyright notice and this permission notice (including the next
  17  * paragraph) shall be included in all copies or substantial portions of the
  18  * Software.
  19  *
  20  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  21  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  23  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  24  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  25  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  26  * IN THE SOFTWARE.
  27  */
  28
  29 #include "radv_private.h"
  30 #include "radv_shader.h"
  31 #include "radv_shader_args.h"
  32
  33 static void
  34 set_loc(struct radv_userdata_info *ud_info, uint8_t *sgpr_idx,
  35         uint8_t num_sgprs)
  36 {
  37         ud_info->sgpr_idx = *sgpr_idx;
  38         ud_info->num_sgprs = num_sgprs;
  39         *sgpr_idx += num_sgprs;
  40 }
  41
  42 static void
  43 set_loc_shader(struct radv_shader_args *args, int idx, uint8_t *sgpr_idx,
  44                uint8_t num_sgprs)
  45 {
  46         struct radv_userdata_info *ud_info =
  47                 &args->shader_info->user_sgprs_locs.shader_data[idx];
  48         assert(ud_info);
  49
  50         set_loc(ud_info, sgpr_idx, num_sgprs);
  51 }
  52
  53 static void
  54 set_loc_shader_ptr(struct radv_shader_args *args, int idx, uint8_t *sgpr_idx)
  55 {
  56         bool use_32bit_pointers = idx != AC_UD_SCRATCH_RING_OFFSETS;
  57
  58         set_loc_shader(args, idx, sgpr_idx, use_32bit_pointers ? 1 : 2);
  59 }
  60
  61 static void
  62 set_loc_desc(struct radv_shader_args *args, int idx, uint8_t *sgpr_idx)
  63 {
  64         struct radv_userdata_locations *locs =
  65                 &args->shader_info->user_sgprs_locs;
  66         struct radv_userdata_info *ud_info = &locs->descriptor_sets[idx];
  67         assert(ud_info);
  68
  69         set_loc(ud_info, sgpr_idx, 1);
  70
  71         locs->descriptor_sets_enabled |= 1 << idx;
  72 }
  73
  74 struct user_sgpr_info {
  75         bool indirect_all_descriptor_sets;
  76         uint8_t remaining_sgprs;
  77 };
  78
  79 static bool needs_view_index_sgpr(struct radv_shader_args *args,
  80                                   gl_shader_stage stage)
  81 {
  82         switch (stage) {
  83         case MESA_SHADER_VERTEX:
  84                 if (args->shader_info->needs_multiview_view_index ||
  85                     (!args->options->key.vs_common_out.as_es && !args->options->key.vs_common_out.as_ls && args->options->key.has_multiview_view_index))
  86                         return true;
  87                 break;
  88         case MESA_SHADER_TESS_EVAL:
  89                 if (args->shader_info->needs_multiview_view_index || (!args->options->key.vs_common_out.as_es && args->options->key.has_multiview_view_index))
  90                         return true;
  91                 break;
  92         case MESA_SHADER_GEOMETRY:
  93         case MESA_SHADER_TESS_CTRL:
  94                 if (args->shader_info->needs_multiview_view_index)
  95                         return true;
  96                 break;
  97         default:
  98                 break;
  99         }
 100         return false;
 101 }
 102
 103 static uint8_t
 104 count_vs_user_sgprs(struct radv_shader_args *args)
 105 {
 106         uint8_t count = 0;
 107
 108         if (args->shader_info->vs.has_vertex_buffers)
 109                 count++;
 110         count += args->shader_info->vs.needs_draw_id ? 3 : 2;
 111
 112         return count;
 113 }
 114
 115 static void allocate_inline_push_consts(struct radv_shader_args *args,
 116                                         struct user_sgpr_info *user_sgpr_info)
 117 {
 118         uint8_t remaining_sgprs = user_sgpr_info->remaining_sgprs;
 119
 120         /* Only supported if shaders use push constants. */
 121         if (args->shader_info->min_push_constant_used == UINT8_MAX)
 122                 return;
 123
 124         /* Only supported if shaders don't have indirect push constants. */
 125         if (args->shader_info->has_indirect_push_constants)
 126                 return;
 127
 128         /* Only supported for 32-bit push constants. */
 129         if (!args->shader_info->has_only_32bit_push_constants)
 130                 return;
 131
 132         uint8_t num_push_consts =
 133                 (args->shader_info->max_push_constant_used -
 134                  args->shader_info->min_push_constant_used) / 4;
 135
 136         /* Check if the number of user SGPRs is large enough. */
 137         if (num_push_consts < remaining_sgprs) {
 138                 args->shader_info->num_inline_push_consts = num_push_consts;
 139         } else {
 140                 args->shader_info->num_inline_push_consts = remaining_sgprs;
 141         }
 142
 143         /* Clamp to the maximum number of allowed inlined push constants. */
 144         if (args->shader_info->num_inline_push_consts > AC_MAX_INLINE_PUSH_CONSTS)
 145                 args->shader_info->num_inline_push_consts = AC_MAX_INLINE_PUSH_CONSTS;
 146
 147         if (args->shader_info->num_inline_push_consts == num_push_consts &&
 148             !args->shader_info->loads_dynamic_offsets) {
 149                 /* Disable the default push constants path if all constants are
 150                  * inlined and if shaders don't use dynamic descriptors.
 151                  */
 152                 args->shader_info->loads_push_constants = false;
 153         }
 154
 155         args->shader_info->base_inline_push_consts =
 156                 args->shader_info->min_push_constant_used / 4;
 157 }
 158
 159 static void allocate_user_sgprs(struct radv_shader_args *args,
 160                                 gl_shader_stage stage,
 161                                 bool has_previous_stage,
 162                                 gl_shader_stage previous_stage,
 163                                 bool needs_view_index,
 164                                 struct user_sgpr_info *user_sgpr_info)
 165 {
 166         uint8_t user_sgpr_count = 0;
 167
 168         memset(user_sgpr_info, 0, sizeof(struct user_sgpr_info));
 169
 170         /* 2 user sgprs will always be allocated for scratch/rings */
 171         user_sgpr_count += 2;
 172
 173         switch (stage) {
 174         case MESA_SHADER_COMPUTE:
 175                 if (args->shader_info->cs.uses_grid_size)
 176                         user_sgpr_count += 3;
 177                 break;
 178         case MESA_SHADER_FRAGMENT:
 179                 user_sgpr_count += args->shader_info->ps.needs_sample_positions;
 180                 break;
 181         case MESA_SHADER_VERTEX:
 182                 if (!args->is_gs_copy_shader)
 183                         user_sgpr_count += count_vs_user_sgprs(args);
 184                 break;
 185         case MESA_SHADER_TESS_CTRL:
 186                 if (has_previous_stage) {
 187                         if (previous_stage == MESA_SHADER_VERTEX)
 188                                 user_sgpr_count += count_vs_user_sgprs(args);
 189                 }
 190                 break;
 191         case MESA_SHADER_TESS_EVAL:
 192                 break;
 193         case MESA_SHADER_GEOMETRY:
 194                 if (has_previous_stage) {
 195                         if (previous_stage == MESA_SHADER_VERTEX) {
 196                                 user_sgpr_count += count_vs_user_sgprs(args);
 197                         }
 198                 }
 199                 break;
 200         default:
 201                 break;
 202         }
 203
 204         if (needs_view_index)
 205                 user_sgpr_count++;
 206
 207         if (args->shader_info->loads_push_constants)
 208                 user_sgpr_count++;
 209
 210         if (args->shader_info->so.num_outputs)
 211                 user_sgpr_count++;
 212
 213         uint32_t available_sgprs = args->options->chip_class >= GFX9 && stage != MESA_SHADER_COMPUTE ? 32 : 16;
 214         uint32_t remaining_sgprs = available_sgprs - user_sgpr_count;
 215         uint32_t num_desc_set =
 216                 util_bitcount(args->shader_info->desc_set_used_mask);
 217
 218         if (remaining_sgprs < num_desc_set) {
 219                 user_sgpr_info->indirect_all_descriptor_sets = true;
 220                 user_sgpr_info->remaining_sgprs = remaining_sgprs - 1;
 221         } else {
 222                 user_sgpr_info->remaining_sgprs = remaining_sgprs - num_desc_set;
 223         }
 224
 225         allocate_inline_push_consts(args, user_sgpr_info);
 226 }
 227
 228 static void
 229 declare_global_input_sgprs(struct radv_shader_args *args,
 230                            const struct user_sgpr_info *user_sgpr_info)
 231 {
 232         /* 1 for each descriptor set */
 233         if (!user_sgpr_info->indirect_all_descriptor_sets) {
 234                 uint32_t mask = args->shader_info->desc_set_used_mask;
 235
 236                 while (mask) {
 237                         int i = u_bit_scan(&mask);
 238
 239                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_CONST_PTR,
 240                                    &args->descriptor_sets[i]);
 241                 }
 242         } else {
 243                 ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_CONST_PTR_PTR,
 244                            &args->descriptor_sets[0]);
 245         }
 246
 247         if (args->shader_info->loads_push_constants) {
 248                 /* 1 for push constants and dynamic descriptors */
 249                 ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_CONST_PTR,
 250                            &args->ac.push_constants);
 251         }
 252
 253         for (unsigned i = 0; i < args->shader_info->num_inline_push_consts; i++) {
 254                 ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT,
 255                            &args->ac.inline_push_consts[i]);
 256         }
 257         args->ac.num_inline_push_consts = args->shader_info->num_inline_push_consts;
 258         args->ac.base_inline_push_consts = args->shader_info->base_inline_push_consts;
 259
 260         if (args->shader_info->so.num_outputs) {
 261                 ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_CONST_DESC_PTR,
 262                            &args->streamout_buffers);
 263         }
 264 }
 265
 266 static void
 267 declare_vs_specific_input_sgprs(struct radv_shader_args *args,
 268                                 gl_shader_stage stage,
 269                                 bool has_previous_stage,
 270                                 gl_shader_stage previous_stage)
 271 {
 272         if (!args->is_gs_copy_shader &&
 273             (stage == MESA_SHADER_VERTEX ||
 274              (has_previous_stage && previous_stage == MESA_SHADER_VERTEX))) {
 275                 if (args->shader_info->vs.has_vertex_buffers) {
 276                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_CONST_DESC_PTR,
 277                                    &args->vertex_buffers);
 278                 }
 279                 ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT, &args->ac.base_vertex);
 280                 ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT, &args->ac.start_instance);
 281                 if (args->shader_info->vs.needs_draw_id) {
 282                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT, &args->ac.draw_id);
 283                 }
 284         }
 285 }
 286
 287 static void
 288 declare_vs_input_vgprs(struct radv_shader_args *args)
 289 {
 290         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT, &args->ac.vertex_id);
 291         if (!args->is_gs_copy_shader) {
 292                 if (args->options->key.vs_common_out.as_ls) {
 293                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT, &args->rel_auto_id);
 294                         if (args->options->chip_class >= GFX10) {
 295                                 ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT, NULL); /* user vgpr */
 296                                 ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT, &args->ac.instance_id);
 297                         } else {
 298                                 ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT, &args->ac.instance_id);
 299                                 ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT, NULL); /* unused */
 300                         }
 301                 } else {
 302                         if (args->options->chip_class >= GFX10) {
 303                                 if (args->options->key.vs_common_out.as_ngg) {
 304                                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT, NULL); /* user vgpr */
 305                                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT, NULL); /* user vgpr */
 306                                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT, &args->ac.instance_id);
 307                                 } else {
 308                                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT, NULL); /* unused */
 309                                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT, &args->vs_prim_id);
 310                                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT, &args->ac.instance_id);
 311                                 }
 312                         } else {
 313                                 ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT, &args->ac.instance_id);
 314                                 ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT, &args->vs_prim_id);
 315                                 ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT, NULL); /* unused */
 316                         }
 317                 }
 318         }
 319 }
 320
 321 static void
 322 declare_streamout_sgprs(struct radv_shader_args *args, gl_shader_stage stage)
 323 {
 324         int i;
 325
 326         if (args->options->use_ngg_streamout) {
 327                 if (stage == MESA_SHADER_TESS_EVAL)
 328                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
 329                 return;
 330         }
 331
 332         /* Streamout SGPRs. */
 333         if (args->shader_info->so.num_outputs) {
 334                 assert(stage == MESA_SHADER_VERTEX ||
 335                        stage == MESA_SHADER_TESS_EVAL);
 336
 337                 ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT, &args->streamout_config);
 338                 ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT, &args->streamout_write_idx);
 339         } else if (stage == MESA_SHADER_TESS_EVAL) {
 340                 ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
 341         }
 342
 343         /* A streamout buffer offset is loaded if the stride is non-zero. */
 344         for (i = 0; i < 4; i++) {
 345                 if (!args->shader_info->so.strides[i])
 346                         continue;
 347
 348                 ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT, &args->streamout_offset[i]);
 349         }
 350 }
 351
 352 static void
 353 declare_tes_input_vgprs(struct radv_shader_args *args)
 354 {
 355         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_FLOAT, &args->tes_u);
 356         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_FLOAT, &args->tes_v);
 357         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT, &args->tes_rel_patch_id);
 358         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT, &args->ac.tes_patch_id);
 359 }
 360
 361 static void
 362 set_global_input_locs(struct radv_shader_args *args,
 363                       const struct user_sgpr_info *user_sgpr_info,
 364                       uint8_t *user_sgpr_idx)
 365 {
 366         uint32_t mask = args->shader_info->desc_set_used_mask;
 367
 368         if (!user_sgpr_info->indirect_all_descriptor_sets) {
 369                 while (mask) {
 370                         int i = u_bit_scan(&mask);
 371
 372                         set_loc_desc(args, i, user_sgpr_idx);
 373                 }
 374         } else {
 375                 set_loc_shader_ptr(args, AC_UD_INDIRECT_DESCRIPTOR_SETS,
 376                                    user_sgpr_idx);
 377
 378                 args->shader_info->need_indirect_descriptor_sets = true;
 379         }
 380
 381         if (args->shader_info->loads_push_constants) {
 382                 set_loc_shader_ptr(args, AC_UD_PUSH_CONSTANTS, user_sgpr_idx);
 383         }
 384
 385         if (args->shader_info->num_inline_push_consts) {
 386                 set_loc_shader(args, AC_UD_INLINE_PUSH_CONSTANTS, user_sgpr_idx,
 387                                args->shader_info->num_inline_push_consts);
 388         }
 389
 390         if (args->streamout_buffers.used) {
 391                 set_loc_shader_ptr(args, AC_UD_STREAMOUT_BUFFERS,
 392                                    user_sgpr_idx);
 393         }
 394 }
 395
 396 static void
 397 set_vs_specific_input_locs(struct radv_shader_args *args,
 398                            gl_shader_stage stage, bool has_previous_stage,
 399                            gl_shader_stage previous_stage,
 400                            uint8_t *user_sgpr_idx)
 401 {
 402         if (!args->is_gs_copy_shader &&
 403             (stage == MESA_SHADER_VERTEX ||
 404              (has_previous_stage && previous_stage == MESA_SHADER_VERTEX))) {
 405                 if (args->shader_info->vs.has_vertex_buffers) {
 406                         set_loc_shader_ptr(args, AC_UD_VS_VERTEX_BUFFERS,
 407                                            user_sgpr_idx);
 408                 }
 409
 410                 unsigned vs_num = 2;
 411                 if (args->shader_info->vs.needs_draw_id)
 412                         vs_num++;
 413
 414                 set_loc_shader(args, AC_UD_VS_BASE_VERTEX_START_INSTANCE,
 415                                user_sgpr_idx, vs_num);
 416         }
 417 }
 418
 419 /* Returns whether the stage is a stage that can be directly before the GS */
 420 static bool is_pre_gs_stage(gl_shader_stage stage)
 421 {
 422         return stage == MESA_SHADER_VERTEX || stage == MESA_SHADER_TESS_EVAL;
 423 }
 424
 425 void
 426 radv_declare_shader_args(struct radv_shader_args *args,
 427                          gl_shader_stage stage,
 428                          bool has_previous_stage,
 429                          gl_shader_stage previous_stage)
 430 {
 431         struct user_sgpr_info user_sgpr_info;
 432         bool needs_view_index = needs_view_index_sgpr(args, stage);
 433
 434         if (args->options->chip_class >= GFX10) {
 435                 if (is_pre_gs_stage(stage) && args->options->key.vs_common_out.as_ngg) {
 436                         /* On GFX10, VS is merged into GS for NGG. */
 437                         previous_stage = stage;
 438                         stage = MESA_SHADER_GEOMETRY;
 439                         has_previous_stage = true;
 440                 }
 441         }
 442
 443         for (int i = 0; i < MAX_SETS; i++)
 444                 args->shader_info->user_sgprs_locs.descriptor_sets[i].sgpr_idx = -1;
 445         for (int i = 0; i < AC_UD_MAX_UD; i++)
 446                 args->shader_info->user_sgprs_locs.shader_data[i].sgpr_idx = -1;
 447
 448
 449         allocate_user_sgprs(args, stage, has_previous_stage,
 450                             previous_stage, needs_view_index, &user_sgpr_info);
 451
 452         if (args->options->explicit_scratch_args) {
 453                 ac_add_arg(&args->ac, AC_ARG_SGPR, 2, AC_ARG_CONST_DESC_PTR,
 454                            &args->ring_offsets);
 455         }
 456
 457         switch (stage) {
 458         case MESA_SHADER_COMPUTE:
 459                 declare_global_input_sgprs(args, &user_sgpr_info);
 460
 461                 if (args->shader_info->cs.uses_grid_size) {
 462                         ac_add_arg(&args->ac, AC_ARG_SGPR, 3, AC_ARG_INT,
 463                                    &args->ac.num_work_groups);
 464                 }
 465
 466                 for (int i = 0; i < 3; i++) {
 467                         if (args->shader_info->cs.uses_block_id[i]) {
 468                                 ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT,
 469                                            &args->ac.workgroup_ids[i]);
 470                         }
 471                 }
 472
 473                 if (args->shader_info->cs.uses_local_invocation_idx) {
 474                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT,
 475                                    &args->ac.tg_size);
 476                 }
 477
 478                 if (args->options->explicit_scratch_args) {
 479                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT,
 480                                    &args->scratch_offset);
 481                 }
 482
 483                 ac_add_arg(&args->ac, AC_ARG_VGPR, 3, AC_ARG_INT,
 484                            &args->ac.local_invocation_ids);
 485                 break;
 486         case MESA_SHADER_VERTEX:
 487                 declare_global_input_sgprs(args, &user_sgpr_info);
 488
 489                 declare_vs_specific_input_sgprs(args, stage, has_previous_stage,
 490                                                 previous_stage);
 491
 492                 if (needs_view_index) {
 493                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT,
 494                                    &args->ac.view_index);
 495                 }
 496
 497                 if (args->options->key.vs_common_out.as_es) {
 498                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT,
 499                                 &args->es2gs_offset);
 500                 } else if (args->options->key.vs_common_out.as_ls) {
 501                         /* no extra parameters */
 502                 } else {
 503                         declare_streamout_sgprs(args, stage);
 504                 }
 505
 506                 if (args->options->explicit_scratch_args) {
 507                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT,
 508                                    &args->scratch_offset);
 509                 }
 510
 511                 declare_vs_input_vgprs(args);
 512                 break;
 513         case MESA_SHADER_TESS_CTRL:
 514                 if (has_previous_stage) {
 515                         // First 6 system regs
 516                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT, &args->oc_lds);
 517                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT,
 518                                    &args->merged_wave_info);
 519                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT,
 520                                    &args->tess_factor_offset);
 521
 522                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT, &args->scratch_offset);
 523                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT, NULL); // unknown
 524                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT, NULL); // unknown
 525
 526                         declare_global_input_sgprs(args, &user_sgpr_info);
 527
 528                         declare_vs_specific_input_sgprs(args, stage,
 529                                                         has_previous_stage,
 530                                                         previous_stage);
 531
 532                         if (needs_view_index) {
 533                                 ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT,
 534                                            &args->ac.view_index);
 535                         }
 536
 537                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT,
 538                                   &args->ac.tcs_patch_id);
 539                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT,
 540                                    &args->ac.tcs_rel_ids);
 541
 542                         declare_vs_input_vgprs(args);
 543                 } else {
 544                         declare_global_input_sgprs(args, &user_sgpr_info);
 545
 546                         if (needs_view_index) {
 547                                 ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT,
 548                                            &args->ac.view_index);
 549                         }
 550
 551                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT, &args->oc_lds);
 552                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT,
 553                                    &args->tess_factor_offset);
 554                         if (args->options->explicit_scratch_args) {
 555                                 ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT,
 556                                            &args->scratch_offset);
 557                         }
 558                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT,
 559                                    &args->ac.tcs_patch_id);
 560                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT,
 561                                    &args->ac.tcs_rel_ids);
 562                 }
 563                 break;
 564         case MESA_SHADER_TESS_EVAL:
 565                 declare_global_input_sgprs(args, &user_sgpr_info);
 566
 567                 if (needs_view_index)
 568                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT,
 569                                 &args->ac.view_index);
 570
 571                 if (args->options->key.vs_common_out.as_es) {
 572                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT, &args->oc_lds);
 573                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
 574                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT,
 575                                 &args->es2gs_offset);
 576                 } else {
 577                         declare_streamout_sgprs(args, stage);
 578                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT, &args->oc_lds);
 579                 }
 580                 if (args->options->explicit_scratch_args) {
 581                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT,
 582                                    &args->scratch_offset);
 583                 }
 584                 declare_tes_input_vgprs(args);
 585                 break;
 586         case MESA_SHADER_GEOMETRY:
 587                 if (has_previous_stage) {
 588                         // First 6 system regs
 589                         if (args->options->key.vs_common_out.as_ngg) {
 590                                 ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT,
 591                                         &args->gs_tg_info);
 592                         } else {
 593                                 ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT,
 594                                         &args->gs2vs_offset);
 595                         }
 596
 597                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT,
 598                                    &args->merged_wave_info);
 599                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT, &args->oc_lds);
 600
 601                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT, &args->scratch_offset);
 602                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT, NULL); // unknown
 603                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT, NULL); // unknown
 604
 605                         declare_global_input_sgprs(args, &user_sgpr_info);
 606
 607                         if (previous_stage != MESA_SHADER_TESS_EVAL) {
 608                                 declare_vs_specific_input_sgprs(args, stage,
 609                                                                 has_previous_stage,
 610                                                                 previous_stage);
 611                         }
 612
 613                         if (needs_view_index) {
 614                                 ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT,
 615                                            &args->ac.view_index);
 616                         }
 617
 618                         if (args->options->key.vs_common_out.as_ngg) {
 619                                 ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT,
 620                                            &args->ngg_gs_state);
 621                         }
 622
 623                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT,
 624                                    &args->gs_vtx_offset[0]);
 625                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT,
 626                                    &args->gs_vtx_offset[2]);
 627                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT,
 628                                    &args->ac.gs_prim_id);
 629                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT,
 630                                    &args->ac.gs_invocation_id);
 631                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT,
 632                                    &args->gs_vtx_offset[4]);
 633
 634                         if (previous_stage == MESA_SHADER_VERTEX) {
 635                                 declare_vs_input_vgprs(args);
 636                         } else {
 637                                 declare_tes_input_vgprs(args);
 638                         }
 639                 } else {
 640                         declare_global_input_sgprs(args, &user_sgpr_info);
 641
 642                         if (needs_view_index) {
 643                                 ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT,
 644                                            &args->ac.view_index);
 645                         }
 646
 647                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT, &args->gs2vs_offset);
 648                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT, &args->gs_wave_id);
 649                         if (args->options->explicit_scratch_args) {
 650                                 ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT,
 651                                            &args->scratch_offset);
 652                         }
 653                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT,
 654                                    &args->gs_vtx_offset[0]);
 655                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT,
 656                                    &args->gs_vtx_offset[1]);
 657                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT,
 658                                    &args->ac.gs_prim_id);
 659                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT,
 660                                    &args->gs_vtx_offset[2]);
 661                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT,
 662                                    &args->gs_vtx_offset[3]);
 663                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT,
 664                                    &args->gs_vtx_offset[4]);
 665                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT,
 666                                    &args->gs_vtx_offset[5]);
 667                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT,
 668                                    &args->ac.gs_invocation_id);
 669                 }
 670                 break;
 671         case MESA_SHADER_FRAGMENT:
 672                 declare_global_input_sgprs(args, &user_sgpr_info);
 673
 674                 ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT, &args->ac.prim_mask);
 675                 if (args->options->explicit_scratch_args) {
 676                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT,
 677                                    &args->scratch_offset);
 678                 }
 679                 ac_add_arg(&args->ac, AC_ARG_VGPR, 2, AC_ARG_INT, &args->ac.persp_sample);
 680                 ac_add_arg(&args->ac, AC_ARG_VGPR, 2, AC_ARG_INT, &args->ac.persp_center);
 681                 ac_add_arg(&args->ac, AC_ARG_VGPR, 2, AC_ARG_INT, &args->ac.persp_centroid);
 682                 ac_add_arg(&args->ac, AC_ARG_VGPR, 3, AC_ARG_INT, &args->ac.pull_model);
 683                 ac_add_arg(&args->ac, AC_ARG_VGPR, 2, AC_ARG_INT, &args->ac.linear_sample);
 684                 ac_add_arg(&args->ac, AC_ARG_VGPR, 2, AC_ARG_INT, &args->ac.linear_center);
 685                 ac_add_arg(&args->ac, AC_ARG_VGPR, 2, AC_ARG_INT, &args->ac.linear_centroid);
 686                 ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_FLOAT, NULL);  /* line stipple tex */
 687                 ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_FLOAT, &args->ac.frag_pos[0]);
 688                 ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_FLOAT, &args->ac.frag_pos[1]);
 689                 ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_FLOAT, &args->ac.frag_pos[2]);
 690                 ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_FLOAT, &args->ac.frag_pos[3]);
 691                 ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT, &args->ac.front_face);
 692                 ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT, &args->ac.ancillary);
 693                 ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT, &args->ac.sample_coverage);
 694                 ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT, NULL);  /* fixed pt */
 695                 break;
 696         default:
 697                 unreachable("Shader stage not implemented");
 698         }
 699
 700         args->shader_info->num_input_vgprs = 0;
 701         args->shader_info->num_input_sgprs = 2;
 702         args->shader_info->num_input_sgprs += args->ac.num_sgprs_used;
 703         args->shader_info->num_input_vgprs = args->ac.num_vgprs_used;
 704
 705         uint8_t user_sgpr_idx = 0;
 706
 707         set_loc_shader_ptr(args, AC_UD_SCRATCH_RING_OFFSETS,
 708                            &user_sgpr_idx);
 709
 710         /* For merged shaders the user SGPRs start at 8, with 8 system SGPRs in front (including
 711          * the rw_buffers at s0/s1. With user SGPR0 = s8, lets restart the count from 0 */
 712         if (has_previous_stage)
 713                 user_sgpr_idx = 0;
 714
 715         set_global_input_locs(args, &user_sgpr_info, &user_sgpr_idx);
 716
 717         switch (stage) {
 718         case MESA_SHADER_COMPUTE:
 719                 if (args->shader_info->cs.uses_grid_size) {
 720                         set_loc_shader(args, AC_UD_CS_GRID_SIZE,
 721                                        &user_sgpr_idx, 3);
 722                 }
 723                 break;
 724         case MESA_SHADER_VERTEX:
 725                 set_vs_specific_input_locs(args, stage, has_previous_stage,
 726                                            previous_stage, &user_sgpr_idx);
 727                 if (args->ac.view_index.used)
 728                         set_loc_shader(args, AC_UD_VIEW_INDEX, &user_sgpr_idx, 1);
 729                 break;
 730         case MESA_SHADER_TESS_CTRL:
 731                 set_vs_specific_input_locs(args, stage, has_previous_stage,
 732                                            previous_stage, &user_sgpr_idx);
 733                 if (args->ac.view_index.used)
 734                         set_loc_shader(args, AC_UD_VIEW_INDEX, &user_sgpr_idx, 1);
 735                 break;
 736         case MESA_SHADER_TESS_EVAL:
 737                 if (args->ac.view_index.used)
 738                         set_loc_shader(args, AC_UD_VIEW_INDEX, &user_sgpr_idx, 1);
 739                 break;
 740         case MESA_SHADER_GEOMETRY:
 741                 if (has_previous_stage) {
 742                         if (previous_stage == MESA_SHADER_VERTEX)
 743                                 set_vs_specific_input_locs(args, stage,
 744                                                            has_previous_stage,
 745                                                            previous_stage,
 746                                                            &user_sgpr_idx);
 747                 }
 748                 if (args->ac.view_index.used)
 749                         set_loc_shader(args, AC_UD_VIEW_INDEX, &user_sgpr_idx, 1);
 750
 751                 if (args->ngg_gs_state.used)
 752                         set_loc_shader(args, AC_UD_NGG_GS_STATE, &user_sgpr_idx, 1);
 753                 break;
 754         case MESA_SHADER_FRAGMENT:
 755                 break;
 756         default:
 757                 unreachable("Shader stage not implemented");
 758         }
 759
 760         args->shader_info->num_user_sgprs = user_sgpr_idx;
 761 }
 762