src/amd/vulkan/radv_shader_args.c

   1 /*
   2  * Copyright © 2019 Valve Corporation.
   3  * Copyright © 2016 Red Hat.
   4  * Copyright © 2016 Bas Nieuwenhuizen
   5  *
   6  * based in part on anv driver which is:
   7  * Copyright © 2015 Intel Corporation
   8  *
   9  * Permission is hereby granted, free of charge, to any person obtaining a
  10  * copy of this software and associated documentation files (the "Software"),
  11  * to deal in the Software without restriction, including without limitation
  12  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  13  * and/or sell copies of the Software, and to permit persons to whom the
  14  * Software is furnished to do so, subject to the following conditions:
  15  *
  16  * The above copyright notice and this permission notice (including the next
  17  * paragraph) shall be included in all copies or substantial portions of the
  18  * Software.
  19  *
  20  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  21  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  23  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  24  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  25  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  26  * IN THE SOFTWARE.
  27  */
  28
  29 #include "radv_private.h"
  30 #include "radv_shader.h"
  31 #include "radv_shader_args.h"
  32
  33 static void
  34 set_loc(struct radv_userdata_info *ud_info, uint8_t *sgpr_idx,
  35         uint8_t num_sgprs)
  36 {
  37         ud_info->sgpr_idx = *sgpr_idx;
  38         ud_info->num_sgprs = num_sgprs;
  39         *sgpr_idx += num_sgprs;
  40 }
  41
  42 static void
  43 set_loc_shader(struct radv_shader_args *args, int idx, uint8_t *sgpr_idx,
  44                uint8_t num_sgprs)
  45 {
  46         struct radv_userdata_info *ud_info =
  47                 &args->shader_info->user_sgprs_locs.shader_data[idx];
  48         assert(ud_info);
  49
  50         set_loc(ud_info, sgpr_idx, num_sgprs);
  51 }
  52
  53 static void
  54 set_loc_shader_ptr(struct radv_shader_args *args, int idx, uint8_t *sgpr_idx)
  55 {
  56         bool use_32bit_pointers = idx != AC_UD_SCRATCH_RING_OFFSETS;
  57
  58         set_loc_shader(args, idx, sgpr_idx, use_32bit_pointers ? 1 : 2);
  59 }
  60
  61 static void
  62 set_loc_desc(struct radv_shader_args *args, int idx, uint8_t *sgpr_idx)
  63 {
  64         struct radv_userdata_locations *locs =
  65                 &args->shader_info->user_sgprs_locs;
  66         struct radv_userdata_info *ud_info = &locs->descriptor_sets[idx];
  67         assert(ud_info);
  68
  69         set_loc(ud_info, sgpr_idx, 1);
  70
  71         locs->descriptor_sets_enabled |= 1 << idx;
  72 }
  73
  74 struct user_sgpr_info {
  75         bool need_ring_offsets;
  76         bool indirect_all_descriptor_sets;
  77         uint8_t remaining_sgprs;
  78 };
  79
  80 static bool needs_view_index_sgpr(struct radv_shader_args *args,
  81                                   gl_shader_stage stage)
  82 {
  83         switch (stage) {
  84         case MESA_SHADER_VERTEX:
  85                 if (args->shader_info->needs_multiview_view_index ||
  86                     (!args->options->key.vs_common_out.as_es && !args->options->key.vs_common_out.as_ls && args->options->key.has_multiview_view_index))
  87                         return true;
  88                 break;
  89         case MESA_SHADER_TESS_EVAL:
  90                 if (args->shader_info->needs_multiview_view_index || (!args->options->key.vs_common_out.as_es && args->options->key.has_multiview_view_index))
  91                         return true;
  92                 break;
  93         case MESA_SHADER_GEOMETRY:
  94         case MESA_SHADER_TESS_CTRL:
  95                 if (args->shader_info->needs_multiview_view_index)
  96                         return true;
  97                 break;
  98         default:
  99                 break;
 100         }
 101         return false;
 102 }
 103
 104 static uint8_t
 105 count_vs_user_sgprs(struct radv_shader_args *args)
 106 {
 107         uint8_t count = 0;
 108
 109         if (args->shader_info->vs.has_vertex_buffers)
 110                 count++;
 111         count += args->shader_info->vs.needs_draw_id ? 3 : 2;
 112
 113         return count;
 114 }
 115
 116 static void allocate_inline_push_consts(struct radv_shader_args *args,
 117                                         struct user_sgpr_info *user_sgpr_info)
 118 {
 119         uint8_t remaining_sgprs = user_sgpr_info->remaining_sgprs;
 120
 121         /* Only supported if shaders use push constants. */
 122         if (args->shader_info->min_push_constant_used == UINT8_MAX)
 123                 return;
 124
 125         /* Only supported if shaders don't have indirect push constants. */
 126         if (args->shader_info->has_indirect_push_constants)
 127                 return;
 128
 129         /* Only supported for 32-bit push constants. */
 130         if (!args->shader_info->has_only_32bit_push_constants)
 131                 return;
 132
 133         uint8_t num_push_consts =
 134                 (args->shader_info->max_push_constant_used -
 135                  args->shader_info->min_push_constant_used) / 4;
 136
 137         /* Check if the number of user SGPRs is large enough. */
 138         if (num_push_consts < remaining_sgprs) {
 139                 args->shader_info->num_inline_push_consts = num_push_consts;
 140         } else {
 141                 args->shader_info->num_inline_push_consts = remaining_sgprs;
 142         }
 143
 144         /* Clamp to the maximum number of allowed inlined push constants. */
 145         if (args->shader_info->num_inline_push_consts > AC_MAX_INLINE_PUSH_CONSTS)
 146                 args->shader_info->num_inline_push_consts = AC_MAX_INLINE_PUSH_CONSTS;
 147
 148         if (args->shader_info->num_inline_push_consts == num_push_consts &&
 149             !args->shader_info->loads_dynamic_offsets) {
 150                 /* Disable the default push constants path if all constants are
 151                  * inlined and if shaders don't use dynamic descriptors.
 152                  */
 153                 args->shader_info->loads_push_constants = false;
 154         }
 155
 156         args->shader_info->base_inline_push_consts =
 157                 args->shader_info->min_push_constant_used / 4;
 158 }
 159
 160 static void allocate_user_sgprs(struct radv_shader_args *args,
 161                                 gl_shader_stage stage,
 162                                 bool has_previous_stage,
 163                                 gl_shader_stage previous_stage,
 164                                 bool needs_view_index,
 165                                 struct user_sgpr_info *user_sgpr_info)
 166 {
 167         uint8_t user_sgpr_count = 0;
 168
 169         memset(user_sgpr_info, 0, sizeof(struct user_sgpr_info));
 170
 171         /* until we sort out scratch/global buffers always assign ring offsets for gs/vs/es */
 172         if (stage == MESA_SHADER_GEOMETRY ||
 173             stage == MESA_SHADER_VERTEX ||
 174             stage == MESA_SHADER_TESS_CTRL ||
 175             stage == MESA_SHADER_TESS_EVAL ||
 176             args->is_gs_copy_shader)
 177                 user_sgpr_info->need_ring_offsets = true;
 178
 179         if (stage == MESA_SHADER_FRAGMENT &&
 180             args->shader_info->ps.needs_sample_positions)
 181                 user_sgpr_info->need_ring_offsets = true;
 182
 183         /* 2 user sgprs will nearly always be allocated for scratch/rings */
 184         if (args->options->supports_spill || user_sgpr_info->need_ring_offsets) {
 185                 user_sgpr_count += 2;
 186         }
 187
 188         switch (stage) {
 189         case MESA_SHADER_COMPUTE:
 190                 if (args->shader_info->cs.uses_grid_size)
 191                         user_sgpr_count += 3;
 192                 break;
 193         case MESA_SHADER_FRAGMENT:
 194                 user_sgpr_count += args->shader_info->ps.needs_sample_positions;
 195                 break;
 196         case MESA_SHADER_VERTEX:
 197                 if (!args->is_gs_copy_shader)
 198                         user_sgpr_count += count_vs_user_sgprs(args);
 199                 break;
 200         case MESA_SHADER_TESS_CTRL:
 201                 if (has_previous_stage) {
 202                         if (previous_stage == MESA_SHADER_VERTEX)
 203                                 user_sgpr_count += count_vs_user_sgprs(args);
 204                 }
 205                 break;
 206         case MESA_SHADER_TESS_EVAL:
 207                 break;
 208         case MESA_SHADER_GEOMETRY:
 209                 if (has_previous_stage) {
 210                         if (previous_stage == MESA_SHADER_VERTEX) {
 211                                 user_sgpr_count += count_vs_user_sgprs(args);
 212                         }
 213                 }
 214                 break;
 215         default:
 216                 break;
 217         }
 218
 219         if (needs_view_index)
 220                 user_sgpr_count++;
 221
 222         if (args->shader_info->loads_push_constants)
 223                 user_sgpr_count++;
 224
 225         if (args->shader_info->so.num_outputs)
 226                 user_sgpr_count++;
 227
 228         uint32_t available_sgprs = args->options->chip_class >= GFX9 && stage != MESA_SHADER_COMPUTE ? 32 : 16;
 229         uint32_t remaining_sgprs = available_sgprs - user_sgpr_count;
 230         uint32_t num_desc_set =
 231                 util_bitcount(args->shader_info->desc_set_used_mask);
 232
 233         if (remaining_sgprs < num_desc_set) {
 234                 user_sgpr_info->indirect_all_descriptor_sets = true;
 235                 user_sgpr_info->remaining_sgprs = remaining_sgprs - 1;
 236         } else {
 237                 user_sgpr_info->remaining_sgprs = remaining_sgprs - num_desc_set;
 238         }
 239
 240         allocate_inline_push_consts(args, user_sgpr_info);
 241 }
 242
 243 static void
 244 declare_global_input_sgprs(struct radv_shader_args *args,
 245                            const struct user_sgpr_info *user_sgpr_info)
 246 {
 247         /* 1 for each descriptor set */
 248         if (!user_sgpr_info->indirect_all_descriptor_sets) {
 249                 uint32_t mask = args->shader_info->desc_set_used_mask;
 250
 251                 while (mask) {
 252                         int i = u_bit_scan(&mask);
 253
 254                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_CONST_PTR,
 255                                    &args->descriptor_sets[i]);
 256                 }
 257         } else {
 258                 ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_CONST_PTR_PTR,
 259                            &args->descriptor_sets[0]);
 260         }
 261
 262         if (args->shader_info->loads_push_constants) {
 263                 /* 1 for push constants and dynamic descriptors */
 264                 ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_CONST_PTR,
 265                            &args->ac.push_constants);
 266         }
 267
 268         for (unsigned i = 0; i < args->shader_info->num_inline_push_consts; i++) {
 269                 ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT,
 270                            &args->ac.inline_push_consts[i]);
 271         }
 272         args->ac.num_inline_push_consts = args->shader_info->num_inline_push_consts;
 273         args->ac.base_inline_push_consts = args->shader_info->base_inline_push_consts;
 274
 275         if (args->shader_info->so.num_outputs) {
 276                 ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_CONST_DESC_PTR,
 277                            &args->streamout_buffers);
 278         }
 279 }
 280
 281 static void
 282 declare_vs_specific_input_sgprs(struct radv_shader_args *args,
 283                                 gl_shader_stage stage,
 284                                 bool has_previous_stage,
 285                                 gl_shader_stage previous_stage)
 286 {
 287         if (!args->is_gs_copy_shader &&
 288             (stage == MESA_SHADER_VERTEX ||
 289              (has_previous_stage && previous_stage == MESA_SHADER_VERTEX))) {
 290                 if (args->shader_info->vs.has_vertex_buffers) {
 291                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_CONST_DESC_PTR,
 292                                    &args->vertex_buffers);
 293                 }
 294                 ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT, &args->ac.base_vertex);
 295                 ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT, &args->ac.start_instance);
 296                 if (args->shader_info->vs.needs_draw_id) {
 297                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT, &args->ac.draw_id);
 298                 }
 299         }
 300 }
 301
 302 static void
 303 declare_vs_input_vgprs(struct radv_shader_args *args)
 304 {
 305         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT, &args->ac.vertex_id);
 306         if (!args->is_gs_copy_shader) {
 307                 if (args->options->key.vs_common_out.as_ls) {
 308                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT, &args->rel_auto_id);
 309                         if (args->options->chip_class >= GFX10) {
 310                                 ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT, NULL); /* user vgpr */
 311                                 ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT, &args->ac.instance_id);
 312                         } else {
 313                                 ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT, &args->ac.instance_id);
 314                                 ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT, NULL); /* unused */
 315                         }
 316                 } else {
 317                         if (args->options->chip_class >= GFX10) {
 318                                 if (args->options->key.vs_common_out.as_ngg) {
 319                                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT, NULL); /* user vgpr */
 320                                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT, NULL); /* user vgpr */
 321                                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT, &args->ac.instance_id);
 322                                 } else {
 323                                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT, NULL); /* unused */
 324                                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT, &args->vs_prim_id);
 325                                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT, &args->ac.instance_id);
 326                                 }
 327                         } else {
 328                                 ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT, &args->ac.instance_id);
 329                                 ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT, &args->vs_prim_id);
 330                                 ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT, NULL); /* unused */
 331                         }
 332                 }
 333         }
 334 }
 335
 336 static void
 337 declare_streamout_sgprs(struct radv_shader_args *args, gl_shader_stage stage)
 338 {
 339         int i;
 340
 341         if (args->options->use_ngg_streamout) {
 342                 if (stage == MESA_SHADER_TESS_EVAL)
 343                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
 344                 return;
 345         }
 346
 347         /* Streamout SGPRs. */
 348         if (args->shader_info->so.num_outputs) {
 349                 assert(stage == MESA_SHADER_VERTEX ||
 350                        stage == MESA_SHADER_TESS_EVAL);
 351
 352                 ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT, &args->streamout_config);
 353                 ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT, &args->streamout_write_idx);
 354         } else if (stage == MESA_SHADER_TESS_EVAL) {
 355                 ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
 356         }
 357
 358         /* A streamout buffer offset is loaded if the stride is non-zero. */
 359         for (i = 0; i < 4; i++) {
 360                 if (!args->shader_info->so.strides[i])
 361                         continue;
 362
 363                 ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT, &args->streamout_offset[i]);
 364         }
 365 }
 366
 367 static void
 368 declare_tes_input_vgprs(struct radv_shader_args *args)
 369 {
 370         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_FLOAT, &args->tes_u);
 371         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_FLOAT, &args->tes_v);
 372         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT, &args->tes_rel_patch_id);
 373         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT, &args->ac.tes_patch_id);
 374 }
 375
 376 static void
 377 set_global_input_locs(struct radv_shader_args *args,
 378                       const struct user_sgpr_info *user_sgpr_info,
 379                       uint8_t *user_sgpr_idx)
 380 {
 381         uint32_t mask = args->shader_info->desc_set_used_mask;
 382
 383         if (!user_sgpr_info->indirect_all_descriptor_sets) {
 384                 while (mask) {
 385                         int i = u_bit_scan(&mask);
 386
 387                         set_loc_desc(args, i, user_sgpr_idx);
 388                 }
 389         } else {
 390                 set_loc_shader_ptr(args, AC_UD_INDIRECT_DESCRIPTOR_SETS,
 391                                    user_sgpr_idx);
 392
 393                 args->shader_info->need_indirect_descriptor_sets = true;
 394         }
 395
 396         if (args->shader_info->loads_push_constants) {
 397                 set_loc_shader_ptr(args, AC_UD_PUSH_CONSTANTS, user_sgpr_idx);
 398         }
 399
 400         if (args->shader_info->num_inline_push_consts) {
 401                 set_loc_shader(args, AC_UD_INLINE_PUSH_CONSTANTS, user_sgpr_idx,
 402                                args->shader_info->num_inline_push_consts);
 403         }
 404
 405         if (args->streamout_buffers.used) {
 406                 set_loc_shader_ptr(args, AC_UD_STREAMOUT_BUFFERS,
 407                                    user_sgpr_idx);
 408         }
 409 }
 410
 411 static void
 412 set_vs_specific_input_locs(struct radv_shader_args *args,
 413                            gl_shader_stage stage, bool has_previous_stage,
 414                            gl_shader_stage previous_stage,
 415                            uint8_t *user_sgpr_idx)
 416 {
 417         if (!args->is_gs_copy_shader &&
 418             (stage == MESA_SHADER_VERTEX ||
 419              (has_previous_stage && previous_stage == MESA_SHADER_VERTEX))) {
 420                 if (args->shader_info->vs.has_vertex_buffers) {
 421                         set_loc_shader_ptr(args, AC_UD_VS_VERTEX_BUFFERS,
 422                                            user_sgpr_idx);
 423                 }
 424
 425                 unsigned vs_num = 2;
 426                 if (args->shader_info->vs.needs_draw_id)
 427                         vs_num++;
 428
 429                 set_loc_shader(args, AC_UD_VS_BASE_VERTEX_START_INSTANCE,
 430                                user_sgpr_idx, vs_num);
 431         }
 432 }
 433
 434 /* Returns whether the stage is a stage that can be directly before the GS */
 435 static bool is_pre_gs_stage(gl_shader_stage stage)
 436 {
 437         return stage == MESA_SHADER_VERTEX || stage == MESA_SHADER_TESS_EVAL;
 438 }
 439
 440 void
 441 radv_declare_shader_args(struct radv_shader_args *args,
 442                          gl_shader_stage stage,
 443                          bool has_previous_stage,
 444                          gl_shader_stage previous_stage)
 445 {
 446         struct user_sgpr_info user_sgpr_info;
 447         bool needs_view_index = needs_view_index_sgpr(args, stage);
 448
 449         if (args->options->chip_class >= GFX10) {
 450                 if (is_pre_gs_stage(stage) && args->options->key.vs_common_out.as_ngg) {
 451                         /* On GFX10, VS is merged into GS for NGG. */
 452                         previous_stage = stage;
 453                         stage = MESA_SHADER_GEOMETRY;
 454                         has_previous_stage = true;
 455                 }
 456         }
 457
 458         for (int i = 0; i < MAX_SETS; i++)
 459                 args->shader_info->user_sgprs_locs.descriptor_sets[i].sgpr_idx = -1;
 460         for (int i = 0; i < AC_UD_MAX_UD; i++)
 461                 args->shader_info->user_sgprs_locs.shader_data[i].sgpr_idx = -1;
 462
 463
 464         allocate_user_sgprs(args, stage, has_previous_stage,
 465                             previous_stage, needs_view_index, &user_sgpr_info);
 466
 467         if (user_sgpr_info.need_ring_offsets && !args->options->supports_spill) {
 468                 ac_add_arg(&args->ac, AC_ARG_SGPR, 2, AC_ARG_CONST_DESC_PTR,
 469                            &args->ring_offsets);
 470         }
 471
 472         switch (stage) {
 473         case MESA_SHADER_COMPUTE:
 474                 declare_global_input_sgprs(args, &user_sgpr_info);
 475
 476                 if (args->shader_info->cs.uses_grid_size) {
 477                         ac_add_arg(&args->ac, AC_ARG_SGPR, 3, AC_ARG_INT,
 478                                    &args->ac.num_work_groups);
 479                 }
 480
 481                 for (int i = 0; i < 3; i++) {
 482                         if (args->shader_info->cs.uses_block_id[i]) {
 483                                 ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT,
 484                                            &args->ac.workgroup_ids[i]);
 485                         }
 486                 }
 487
 488                 if (args->shader_info->cs.uses_local_invocation_idx) {
 489                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT,
 490                                    &args->ac.tg_size);
 491                 }
 492
 493                 ac_add_arg(&args->ac, AC_ARG_VGPR, 3, AC_ARG_INT,
 494                            &args->ac.local_invocation_ids);
 495                 break;
 496         case MESA_SHADER_VERTEX:
 497                 declare_global_input_sgprs(args, &user_sgpr_info);
 498
 499                 declare_vs_specific_input_sgprs(args, stage, has_previous_stage,
 500                                                 previous_stage);
 501
 502                 if (needs_view_index) {
 503                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT,
 504                                    &args->ac.view_index);
 505                 }
 506
 507                 if (args->options->key.vs_common_out.as_es) {
 508                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT,
 509                                 &args->es2gs_offset);
 510                 } else if (args->options->key.vs_common_out.as_ls) {
 511                         /* no extra parameters */
 512                 } else {
 513                         declare_streamout_sgprs(args, stage);
 514                 }
 515
 516                 declare_vs_input_vgprs(args);
 517                 break;
 518         case MESA_SHADER_TESS_CTRL:
 519                 if (has_previous_stage) {
 520                         // First 6 system regs
 521                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT, &args->oc_lds);
 522                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT,
 523                                    &args->merged_wave_info);
 524                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT,
 525                                    &args->tess_factor_offset);
 526
 527                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT, NULL); // scratch offset
 528                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT, NULL); // unknown
 529                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT, NULL); // unknown
 530
 531                         declare_global_input_sgprs(args, &user_sgpr_info);
 532
 533                         declare_vs_specific_input_sgprs(args, stage,
 534                                                         has_previous_stage,
 535                                                         previous_stage);
 536
 537                         if (needs_view_index) {
 538                                 ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT,
 539                                            &args->ac.view_index);
 540                         }
 541
 542                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT,
 543                                   &args->ac.tcs_patch_id);
 544                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT,
 545                                    &args->ac.tcs_rel_ids);
 546
 547                         declare_vs_input_vgprs(args);
 548                 } else {
 549                         declare_global_input_sgprs(args, &user_sgpr_info);
 550
 551                         if (needs_view_index) {
 552                                 ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT,
 553                                            &args->ac.view_index);
 554                         }
 555
 556                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT, &args->oc_lds);
 557                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT,
 558                                    &args->tess_factor_offset);
 559                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT,
 560                                    &args->ac.tcs_patch_id);
 561                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT,
 562                                    &args->ac.tcs_rel_ids);
 563                 }
 564                 break;
 565         case MESA_SHADER_TESS_EVAL:
 566                 declare_global_input_sgprs(args, &user_sgpr_info);
 567
 568                 if (needs_view_index)
 569                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT,
 570                                 &args->ac.view_index);
 571
 572                 if (args->options->key.vs_common_out.as_es) {
 573                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT, &args->oc_lds);
 574                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT, NULL);
 575                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT,
 576                                 &args->es2gs_offset);
 577                 } else {
 578                         declare_streamout_sgprs(args, stage);
 579                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT, &args->oc_lds);
 580                 }
 581                 declare_tes_input_vgprs(args);
 582                 break;
 583         case MESA_SHADER_GEOMETRY:
 584                 if (has_previous_stage) {
 585                         // First 6 system regs
 586                         if (args->options->key.vs_common_out.as_ngg) {
 587                                 ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT,
 588                                         &args->gs_tg_info);
 589                         } else {
 590                                 ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT,
 591                                         &args->gs2vs_offset);
 592                         }
 593
 594                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT,
 595                                    &args->merged_wave_info);
 596                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT, &args->oc_lds);
 597
 598                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT, NULL); // scratch offset
 599                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT, NULL); // unknown
 600                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT, NULL); // unknown
 601
 602                         declare_global_input_sgprs(args, &user_sgpr_info);
 603
 604                         if (previous_stage != MESA_SHADER_TESS_EVAL) {
 605                                 declare_vs_specific_input_sgprs(args, stage,
 606                                                                 has_previous_stage,
 607                                                                 previous_stage);
 608                         }
 609
 610                         if (needs_view_index) {
 611                                 ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT,
 612                                            &args->ac.view_index);
 613                         }
 614
 615                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT,
 616                                    &args->gs_vtx_offset[0]);
 617                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT,
 618                                    &args->gs_vtx_offset[2]);
 619                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT,
 620                                    &args->ac.gs_prim_id);
 621                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT,
 622                                    &args->ac.gs_invocation_id);
 623                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT,
 624                                    &args->gs_vtx_offset[4]);
 625
 626                         if (previous_stage == MESA_SHADER_VERTEX) {
 627                                 declare_vs_input_vgprs(args);
 628                         } else {
 629                                 declare_tes_input_vgprs(args);
 630                         }
 631                 } else {
 632                         declare_global_input_sgprs(args, &user_sgpr_info);
 633
 634                         if (needs_view_index) {
 635                                 ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT,
 636                                            &args->ac.view_index);
 637                         }
 638
 639                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT, &args->gs2vs_offset);
 640                         ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT, &args->gs_wave_id);
 641                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT,
 642                                    &args->gs_vtx_offset[0]);
 643                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT,
 644                                    &args->gs_vtx_offset[1]);
 645                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT,
 646                                    &args->ac.gs_prim_id);
 647                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT,
 648                                    &args->gs_vtx_offset[2]);
 649                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT,
 650                                    &args->gs_vtx_offset[3]);
 651                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT,
 652                                    &args->gs_vtx_offset[4]);
 653                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT,
 654                                    &args->gs_vtx_offset[5]);
 655                         ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT,
 656                                    &args->ac.gs_invocation_id);
 657                 }
 658                 break;
 659         case MESA_SHADER_FRAGMENT:
 660                 declare_global_input_sgprs(args, &user_sgpr_info);
 661
 662                 ac_add_arg(&args->ac, AC_ARG_SGPR, 1, AC_ARG_INT, &args->ac.prim_mask);
 663                 ac_add_arg(&args->ac, AC_ARG_VGPR, 2, AC_ARG_INT, &args->ac.persp_sample);
 664                 ac_add_arg(&args->ac, AC_ARG_VGPR, 2, AC_ARG_INT, &args->ac.persp_center);
 665                 ac_add_arg(&args->ac, AC_ARG_VGPR, 2, AC_ARG_INT, &args->ac.persp_centroid);
 666                 ac_add_arg(&args->ac, AC_ARG_VGPR, 3, AC_ARG_INT, NULL); /* persp pull model */
 667                 ac_add_arg(&args->ac, AC_ARG_VGPR, 2, AC_ARG_INT, &args->ac.linear_sample);
 668                 ac_add_arg(&args->ac, AC_ARG_VGPR, 2, AC_ARG_INT, &args->ac.linear_center);
 669                 ac_add_arg(&args->ac, AC_ARG_VGPR, 2, AC_ARG_INT, &args->ac.linear_centroid);
 670                 ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_FLOAT, NULL);  /* line stipple tex */
 671                 ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_FLOAT, &args->ac.frag_pos[0]);
 672                 ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_FLOAT, &args->ac.frag_pos[1]);
 673                 ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_FLOAT, &args->ac.frag_pos[2]);
 674                 ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_FLOAT, &args->ac.frag_pos[3]);
 675                 ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT, &args->ac.front_face);
 676                 ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT, &args->ac.ancillary);
 677                 ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT, &args->ac.sample_coverage);
 678                 ac_add_arg(&args->ac, AC_ARG_VGPR, 1, AC_ARG_INT, NULL);  /* fixed pt */
 679                 break;
 680         default:
 681                 unreachable("Shader stage not implemented");
 682         }
 683
 684         args->shader_info->num_input_vgprs = 0;
 685         args->shader_info->num_input_sgprs = args->options->supports_spill ? 2 : 0;
 686         args->shader_info->num_input_sgprs += args->ac.num_sgprs_used;
 687
 688         if (stage != MESA_SHADER_FRAGMENT)
 689                 args->shader_info->num_input_vgprs = args->ac.num_vgprs_used;
 690
 691         uint8_t user_sgpr_idx = 0;
 692
 693         if (args->options->supports_spill || user_sgpr_info.need_ring_offsets) {
 694                 set_loc_shader_ptr(args, AC_UD_SCRATCH_RING_OFFSETS,
 695                                    &user_sgpr_idx);
 696         }
 697
 698         /* For merged shaders the user SGPRs start at 8, with 8 system SGPRs in front (including
 699          * the rw_buffers at s0/s1. With user SGPR0 = s8, lets restart the count from 0 */
 700         if (has_previous_stage)
 701                 user_sgpr_idx = 0;
 702
 703         set_global_input_locs(args, &user_sgpr_info, &user_sgpr_idx);
 704
 705         switch (stage) {
 706         case MESA_SHADER_COMPUTE:
 707                 if (args->shader_info->cs.uses_grid_size) {
 708                         set_loc_shader(args, AC_UD_CS_GRID_SIZE,
 709                                        &user_sgpr_idx, 3);
 710                 }
 711                 break;
 712         case MESA_SHADER_VERTEX:
 713                 set_vs_specific_input_locs(args, stage, has_previous_stage,
 714                                            previous_stage, &user_sgpr_idx);
 715                 if (args->ac.view_index.used)
 716                         set_loc_shader(args, AC_UD_VIEW_INDEX, &user_sgpr_idx, 1);
 717                 break;
 718         case MESA_SHADER_TESS_CTRL:
 719                 set_vs_specific_input_locs(args, stage, has_previous_stage,
 720                                            previous_stage, &user_sgpr_idx);
 721                 if (args->ac.view_index.used)
 722                         set_loc_shader(args, AC_UD_VIEW_INDEX, &user_sgpr_idx, 1);
 723                 break;
 724         case MESA_SHADER_TESS_EVAL:
 725                 if (args->ac.view_index.used)
 726                         set_loc_shader(args, AC_UD_VIEW_INDEX, &user_sgpr_idx, 1);
 727                 break;
 728         case MESA_SHADER_GEOMETRY:
 729                 if (has_previous_stage) {
 730                         if (previous_stage == MESA_SHADER_VERTEX)
 731                                 set_vs_specific_input_locs(args, stage,
 732                                                            has_previous_stage,
 733                                                            previous_stage,
 734                                                            &user_sgpr_idx);
 735                 }
 736                 if (args->ac.view_index.used)
 737                         set_loc_shader(args, AC_UD_VIEW_INDEX, &user_sgpr_idx, 1);
 738                 break;
 739         case MESA_SHADER_FRAGMENT:
 740                 break;
 741         default:
 742                 unreachable("Shader stage not implemented");
 743         }
 744
 745         args->shader_info->num_user_sgprs = user_sgpr_idx;
 746 }
 747