src/amd/llvm/ac_llvm_build.h

   1 /*
   2  * Copyright 2016 Bas Nieuwenhuizen
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the
   6  * "Software"), to deal in the Software without restriction, including
   7  * without limitation the rights to use, copy, modify, merge, publish,
   8  * distribute, sub license, and/or sell copies of the Software, and to
   9  * permit persons to whom the Software is furnished to do so, subject to
  10  * the following conditions:
  11  *
  12  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  13  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  14  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  15  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  16  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  17  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  18  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  19  *
  20  * The above copyright notice and this permission notice (including the
  21  * next paragraph) shall be included in all copies or substantial portions
  22  * of the Software.
  23  *
  24  */
  25 #ifndef AC_LLVM_BUILD_H
  26 #define AC_LLVM_BUILD_H
  27
  28 #include "ac_shader_abi.h"
  29 #include "ac_shader_args.h"
  30 #include "ac_shader_util.h"
  31 #include "amd_family.h"
  32 #include "compiler/nir/nir.h"
  33 #include <llvm-c/Core.h>
  34
  35 #include <stdbool.h>
  36
  37 #ifdef __cplusplus
  38 extern "C" {
  39 #endif
  40
  41 enum
  42 {
  43    AC_ADDR_SPACE_FLAT = 0, /* Slower than global. */
  44    AC_ADDR_SPACE_GLOBAL = 1,
  45    AC_ADDR_SPACE_GDS = 2,
  46    AC_ADDR_SPACE_LDS = 3,
  47    AC_ADDR_SPACE_CONST = 4,       /* Global allowing SMEM. */
  48    AC_ADDR_SPACE_CONST_32BIT = 6, /* same as CONST, but the pointer type has 32 bits */
  49 };
  50
  51 #define AC_WAIT_LGKM   (1 << 0) /* LDS, GDS, constant, message */
  52 #define AC_WAIT_VLOAD  (1 << 1) /* VMEM load/sample instructions */
  53 #define AC_WAIT_VSTORE (1 << 2) /* VMEM store instructions */
  54
  55 struct ac_llvm_flow;
  56 struct ac_llvm_compiler;
  57 enum ac_float_mode;
  58
  59 struct ac_llvm_flow_state {
  60    struct ac_llvm_flow *stack;
  61    unsigned depth_max;
  62    unsigned depth;
  63 };
  64
  65 struct ac_llvm_context {
  66    LLVMContextRef context;
  67    LLVMModuleRef module;
  68    LLVMBuilderRef builder;
  69
  70    LLVMValueRef main_function;
  71
  72    LLVMTypeRef voidt;
  73    LLVMTypeRef i1;
  74    LLVMTypeRef i8;
  75    LLVMTypeRef i16;
  76    LLVMTypeRef i32;
  77    LLVMTypeRef i64;
  78    LLVMTypeRef i128;
  79    LLVMTypeRef intptr;
  80    LLVMTypeRef f16;
  81    LLVMTypeRef f32;
  82    LLVMTypeRef f64;
  83    LLVMTypeRef v2i16;
  84    LLVMTypeRef v4i16;
  85    LLVMTypeRef v2f16;
  86    LLVMTypeRef v4f16;
  87    LLVMTypeRef v2i32;
  88    LLVMTypeRef v3i32;
  89    LLVMTypeRef v4i32;
  90    LLVMTypeRef v2f32;
  91    LLVMTypeRef v3f32;
  92    LLVMTypeRef v4f32;
  93    LLVMTypeRef v8i32;
  94    LLVMTypeRef iN_wavemask;
  95    LLVMTypeRef iN_ballotmask;
  96
  97    LLVMValueRef i8_0;
  98    LLVMValueRef i8_1;
  99    LLVMValueRef i16_0;
 100    LLVMValueRef i16_1;
 101    LLVMValueRef i32_0;
 102    LLVMValueRef i32_1;
 103    LLVMValueRef i64_0;
 104    LLVMValueRef i64_1;
 105    LLVMValueRef i128_0;
 106    LLVMValueRef i128_1;
 107    LLVMValueRef f16_0;
 108    LLVMValueRef f16_1;
 109    LLVMValueRef f32_0;
 110    LLVMValueRef f32_1;
 111    LLVMValueRef f64_0;
 112    LLVMValueRef f64_1;
 113    LLVMValueRef i1true;
 114    LLVMValueRef i1false;
 115
 116    /* Temporary helper to implement demote_to_helper:
 117     * True = live lanes
 118     * False = demoted lanes
 119     */
 120    LLVMValueRef postponed_kill;
 121
 122    /* Since ac_nir_translate makes a local copy of ac_llvm_context, there
 123     * are two ac_llvm_contexts. Declare a pointer here, so that the control
 124     * flow stack is shared by both ac_llvm_contexts.
 125     */
 126    struct ac_llvm_flow_state *flow;
 127
 128    unsigned range_md_kind;
 129    unsigned invariant_load_md_kind;
 130    unsigned uniform_md_kind;
 131    LLVMValueRef empty_md;
 132
 133    enum chip_class chip_class;
 134    enum radeon_family family;
 135
 136    unsigned wave_size;
 137    unsigned ballot_mask_bits;
 138
 139    unsigned float_mode;
 140
 141    LLVMValueRef lds;
 142 };
 143
 144 void ac_llvm_context_init(struct ac_llvm_context *ctx, struct ac_llvm_compiler *compiler,
 145                           enum chip_class chip_class, enum radeon_family family,
 146                           enum ac_float_mode float_mode, unsigned wave_size,
 147                           unsigned ballot_mask_bits);
 148
 149 void ac_llvm_context_dispose(struct ac_llvm_context *ctx);
 150
 151 int ac_get_llvm_num_components(LLVMValueRef value);
 152
 153 int ac_get_elem_bits(struct ac_llvm_context *ctx, LLVMTypeRef type);
 154
 155 LLVMValueRef ac_llvm_extract_elem(struct ac_llvm_context *ac, LLVMValueRef value, int index);
 156
 157 unsigned ac_get_type_size(LLVMTypeRef type);
 158
 159 LLVMTypeRef ac_to_integer_type(struct ac_llvm_context *ctx, LLVMTypeRef t);
 160 LLVMValueRef ac_to_integer(struct ac_llvm_context *ctx, LLVMValueRef v);
 161 LLVMValueRef ac_to_integer_or_pointer(struct ac_llvm_context *ctx, LLVMValueRef v);
 162 LLVMTypeRef ac_to_float_type(struct ac_llvm_context *ctx, LLVMTypeRef t);
 163 LLVMValueRef ac_to_float(struct ac_llvm_context *ctx, LLVMValueRef v);
 164
 165 LLVMValueRef ac_build_intrinsic(struct ac_llvm_context *ctx, const char *name,
 166                                 LLVMTypeRef return_type, LLVMValueRef *params, unsigned param_count,
 167                                 unsigned attrib_mask);
 168
 169 void ac_build_type_name_for_intr(LLVMTypeRef type, char *buf, unsigned bufsize);
 170
 171 LLVMValueRef ac_build_phi(struct ac_llvm_context *ctx, LLVMTypeRef type, unsigned count_incoming,
 172                           LLVMValueRef *values, LLVMBasicBlockRef *blocks);
 173
 174 void ac_build_s_barrier(struct ac_llvm_context *ctx);
 175 void ac_build_optimization_barrier(struct ac_llvm_context *ctx, LLVMValueRef *pvgpr);
 176
 177 LLVMValueRef ac_build_shader_clock(struct ac_llvm_context *ctx, nir_scope scope);
 178
 179 LLVMValueRef ac_build_ballot(struct ac_llvm_context *ctx, LLVMValueRef value);
 180 LLVMValueRef ac_get_i1_sgpr_mask(struct ac_llvm_context *ctx, LLVMValueRef value);
 181
 182 LLVMValueRef ac_build_vote_all(struct ac_llvm_context *ctx, LLVMValueRef value);
 183
 184 LLVMValueRef ac_build_vote_any(struct ac_llvm_context *ctx, LLVMValueRef value);
 185
 186 LLVMValueRef ac_build_vote_eq(struct ac_llvm_context *ctx, LLVMValueRef value);
 187
 188 LLVMValueRef ac_build_varying_gather_values(struct ac_llvm_context *ctx, LLVMValueRef *values,
 189                                             unsigned value_count, unsigned component);
 190
 191 LLVMValueRef ac_build_gather_values_extended(struct ac_llvm_context *ctx, LLVMValueRef *values,
 192                                              unsigned value_count, unsigned value_stride, bool load,
 193                                              bool always_vector);
 194 LLVMValueRef ac_build_gather_values(struct ac_llvm_context *ctx, LLVMValueRef *values,
 195                                     unsigned value_count);
 196
 197 LLVMValueRef ac_extract_components(struct ac_llvm_context *ctx, LLVMValueRef value, unsigned start,
 198                                    unsigned channels);
 199
 200 LLVMValueRef ac_build_expand_to_vec4(struct ac_llvm_context *ctx, LLVMValueRef value,
 201                                      unsigned num_channels);
 202 LLVMValueRef ac_build_round(struct ac_llvm_context *ctx, LLVMValueRef value);
 203
 204 LLVMValueRef ac_build_fdiv(struct ac_llvm_context *ctx, LLVMValueRef num, LLVMValueRef den);
 205
 206 LLVMValueRef ac_build_fast_udiv(struct ac_llvm_context *ctx, LLVMValueRef num,
 207                                 LLVMValueRef multiplier, LLVMValueRef pre_shift,
 208                                 LLVMValueRef post_shift, LLVMValueRef increment);
 209 LLVMValueRef ac_build_fast_udiv_nuw(struct ac_llvm_context *ctx, LLVMValueRef num,
 210                                     LLVMValueRef multiplier, LLVMValueRef pre_shift,
 211                                     LLVMValueRef post_shift, LLVMValueRef increment);
 212 LLVMValueRef ac_build_fast_udiv_u31_d_not_one(struct ac_llvm_context *ctx, LLVMValueRef num,
 213                                               LLVMValueRef multiplier, LLVMValueRef post_shift);
 214
 215 void ac_prepare_cube_coords(struct ac_llvm_context *ctx, bool is_deriv, bool is_array, bool is_lod,
 216                             LLVMValueRef *coords_arg, LLVMValueRef *derivs_arg);
 217
 218 LLVMValueRef ac_build_fs_interp(struct ac_llvm_context *ctx, LLVMValueRef llvm_chan,
 219                                 LLVMValueRef attr_number, LLVMValueRef params, LLVMValueRef i,
 220                                 LLVMValueRef j);
 221
 222 LLVMValueRef ac_build_fs_interp_f16(struct ac_llvm_context *ctx, LLVMValueRef llvm_chan,
 223                                     LLVMValueRef attr_number, LLVMValueRef params, LLVMValueRef i,
 224                                     LLVMValueRef j);
 225
 226 LLVMValueRef ac_build_fs_interp_mov(struct ac_llvm_context *ctx, LLVMValueRef parameter,
 227                                     LLVMValueRef llvm_chan, LLVMValueRef attr_number,
 228                                     LLVMValueRef params);
 229
 230 LLVMValueRef ac_build_gep_ptr(struct ac_llvm_context *ctx, LLVMValueRef base_ptr,
 231                               LLVMValueRef index);
 232
 233 LLVMValueRef ac_build_gep0(struct ac_llvm_context *ctx, LLVMValueRef base_ptr, LLVMValueRef index);
 234 LLVMValueRef ac_build_pointer_add(struct ac_llvm_context *ctx, LLVMValueRef ptr,
 235                                   LLVMValueRef index);
 236
 237 void ac_build_indexed_store(struct ac_llvm_context *ctx, LLVMValueRef base_ptr, LLVMValueRef index,
 238                             LLVMValueRef value);
 239
 240 LLVMValueRef ac_build_load(struct ac_llvm_context *ctx, LLVMValueRef base_ptr, LLVMValueRef index);
 241 LLVMValueRef ac_build_load_invariant(struct ac_llvm_context *ctx, LLVMValueRef base_ptr,
 242                                      LLVMValueRef index);
 243 LLVMValueRef ac_build_load_to_sgpr(struct ac_llvm_context *ctx, LLVMValueRef base_ptr,
 244                                    LLVMValueRef index);
 245 LLVMValueRef ac_build_load_to_sgpr_uint_wraparound(struct ac_llvm_context *ctx,
 246                                                    LLVMValueRef base_ptr, LLVMValueRef index);
 247
 248 void ac_build_buffer_store_dword(struct ac_llvm_context *ctx, LLVMValueRef rsrc, LLVMValueRef vdata,
 249                                  unsigned num_channels, LLVMValueRef voffset, LLVMValueRef soffset,
 250                                  unsigned inst_offset, unsigned cache_policy);
 251
 252 void ac_build_buffer_store_format(struct ac_llvm_context *ctx, LLVMValueRef rsrc, LLVMValueRef data,
 253                                   LLVMValueRef vindex, LLVMValueRef voffset, unsigned cache_policy);
 254
 255 LLVMValueRef ac_build_buffer_load(struct ac_llvm_context *ctx, LLVMValueRef rsrc, int num_channels,
 256                                   LLVMValueRef vindex, LLVMValueRef voffset, LLVMValueRef soffset,
 257                                   unsigned inst_offset, unsigned cache_policy, bool can_speculate,
 258                                   bool allow_smem);
 259
 260 LLVMValueRef ac_build_buffer_load_format(struct ac_llvm_context *ctx, LLVMValueRef rsrc,
 261                                          LLVMValueRef vindex, LLVMValueRef voffset,
 262                                          unsigned num_channels, unsigned cache_policy,
 263                                          bool can_speculate, bool d16);
 264
 265 LLVMValueRef ac_build_tbuffer_load_short(struct ac_llvm_context *ctx, LLVMValueRef rsrc,
 266                                          LLVMValueRef voffset, LLVMValueRef soffset,
 267                                          LLVMValueRef immoffset, unsigned cache_policy);
 268
 269 LLVMValueRef ac_build_tbuffer_load_byte(struct ac_llvm_context *ctx, LLVMValueRef rsrc,
 270                                         LLVMValueRef voffset, LLVMValueRef soffset,
 271                                         LLVMValueRef immoffset, unsigned cache_policy);
 272
 273 LLVMValueRef ac_build_struct_tbuffer_load(struct ac_llvm_context *ctx, LLVMValueRef rsrc,
 274                                           LLVMValueRef vindex, LLVMValueRef voffset,
 275                                           LLVMValueRef soffset, LLVMValueRef immoffset,
 276                                           unsigned num_channels, unsigned dfmt, unsigned nfmt,
 277                                           unsigned cache_policy, bool can_speculate);
 278
 279 LLVMValueRef ac_build_raw_tbuffer_load(struct ac_llvm_context *ctx, LLVMValueRef rsrc,
 280                                        LLVMValueRef voffset, LLVMValueRef soffset,
 281                                        LLVMValueRef immoffset, unsigned num_channels, unsigned dfmt,
 282                                        unsigned nfmt, unsigned cache_policy, bool can_speculate);
 283
 284 /* For ac_build_fetch_format.
 285  *
 286  * Note: FLOAT must be 0 (used for convenience of encoding in radeonsi).
 287  */
 288 enum
 289 {
 290    AC_FETCH_FORMAT_FLOAT = 0,
 291    AC_FETCH_FORMAT_FIXED,
 292    AC_FETCH_FORMAT_UNORM,
 293    AC_FETCH_FORMAT_SNORM,
 294    AC_FETCH_FORMAT_USCALED,
 295    AC_FETCH_FORMAT_SSCALED,
 296    AC_FETCH_FORMAT_UINT,
 297    AC_FETCH_FORMAT_SINT,
 298 };
 299
 300 LLVMValueRef ac_build_opencoded_load_format(struct ac_llvm_context *ctx, unsigned log_size,
 301                                             unsigned num_channels, unsigned format, bool reverse,
 302                                             bool known_aligned, LLVMValueRef rsrc,
 303                                             LLVMValueRef vindex, LLVMValueRef voffset,
 304                                             LLVMValueRef soffset, unsigned cache_policy,
 305                                             bool can_speculate);
 306
 307 void ac_build_tbuffer_store_short(struct ac_llvm_context *ctx, LLVMValueRef rsrc,
 308                                   LLVMValueRef vdata, LLVMValueRef voffset, LLVMValueRef soffset,
 309                                   unsigned cache_policy);
 310
 311 void ac_build_tbuffer_store_byte(struct ac_llvm_context *ctx, LLVMValueRef rsrc, LLVMValueRef vdata,
 312                                  LLVMValueRef voffset, LLVMValueRef soffset, unsigned cache_policy);
 313
 314 void ac_build_struct_tbuffer_store(struct ac_llvm_context *ctx, LLVMValueRef rsrc,
 315                                    LLVMValueRef vdata, LLVMValueRef vindex, LLVMValueRef voffset,
 316                                    LLVMValueRef soffset, LLVMValueRef immoffset,
 317                                    unsigned num_channels, unsigned dfmt, unsigned nfmt,
 318                                    unsigned cache_policy);
 319
 320 void ac_build_raw_tbuffer_store(struct ac_llvm_context *ctx, LLVMValueRef rsrc, LLVMValueRef vdata,
 321                                 LLVMValueRef voffset, LLVMValueRef soffset, LLVMValueRef immoffset,
 322                                 unsigned num_channels, unsigned dfmt, unsigned nfmt,
 323                                 unsigned cache_policy);
 324
 325 LLVMValueRef ac_get_thread_id(struct ac_llvm_context *ctx);
 326
 327 #define AC_TID_MASK_TOP_LEFT 0xfffffffc
 328 #define AC_TID_MASK_TOP      0xfffffffd
 329 #define AC_TID_MASK_LEFT     0xfffffffe
 330
 331 LLVMValueRef ac_build_ddxy(struct ac_llvm_context *ctx, uint32_t mask, int idx, LLVMValueRef val);
 332
 333 #define AC_SENDMSG_GS           2
 334 #define AC_SENDMSG_GS_DONE      3
 335 #define AC_SENDMSG_GS_ALLOC_REQ 9
 336
 337 #define AC_SENDMSG_GS_OP_NOP      (0 << 4)
 338 #define AC_SENDMSG_GS_OP_CUT      (1 << 4)
 339 #define AC_SENDMSG_GS_OP_EMIT     (2 << 4)
 340 #define AC_SENDMSG_GS_OP_EMIT_CUT (3 << 4)
 341
 342 void ac_build_sendmsg(struct ac_llvm_context *ctx, uint32_t msg, LLVMValueRef wave_id);
 343
 344 LLVMValueRef ac_build_imsb(struct ac_llvm_context *ctx, LLVMValueRef arg, LLVMTypeRef dst_type);
 345
 346 LLVMValueRef ac_build_umsb(struct ac_llvm_context *ctx, LLVMValueRef arg, LLVMTypeRef dst_type);
 347 LLVMValueRef ac_build_fmin(struct ac_llvm_context *ctx, LLVMValueRef a, LLVMValueRef b);
 348 LLVMValueRef ac_build_fmax(struct ac_llvm_context *ctx, LLVMValueRef a, LLVMValueRef b);
 349 LLVMValueRef ac_build_imin(struct ac_llvm_context *ctx, LLVMValueRef a, LLVMValueRef b);
 350 LLVMValueRef ac_build_imax(struct ac_llvm_context *ctx, LLVMValueRef a, LLVMValueRef b);
 351 LLVMValueRef ac_build_umin(struct ac_llvm_context *ctx, LLVMValueRef a, LLVMValueRef b);
 352 LLVMValueRef ac_build_umax(struct ac_llvm_context *ctx, LLVMValueRef a, LLVMValueRef b);
 353 LLVMValueRef ac_build_clamp(struct ac_llvm_context *ctx, LLVMValueRef value);
 354
 355 struct ac_export_args {
 356    LLVMValueRef out[4];
 357    unsigned target;
 358    unsigned enabled_channels;
 359    bool compr;
 360    bool done;
 361    bool valid_mask;
 362 };
 363
 364 void ac_build_export(struct ac_llvm_context *ctx, struct ac_export_args *a);
 365
 366 void ac_build_export_null(struct ac_llvm_context *ctx);
 367
 368 enum ac_image_opcode
 369 {
 370    ac_image_sample,
 371    ac_image_gather4,
 372    ac_image_load,
 373    ac_image_load_mip,
 374    ac_image_store,
 375    ac_image_store_mip,
 376    ac_image_get_lod,
 377    ac_image_get_resinfo,
 378    ac_image_atomic,
 379    ac_image_atomic_cmpswap,
 380 };
 381
 382 enum ac_atomic_op
 383 {
 384    ac_atomic_swap,
 385    ac_atomic_add,
 386    ac_atomic_sub,
 387    ac_atomic_smin,
 388    ac_atomic_umin,
 389    ac_atomic_smax,
 390    ac_atomic_umax,
 391    ac_atomic_and,
 392    ac_atomic_or,
 393    ac_atomic_xor,
 394    ac_atomic_inc_wrap,
 395    ac_atomic_dec_wrap,
 396 };
 397
 398 /* These cache policy bits match the definitions used by the LLVM intrinsics. */
 399 enum ac_image_cache_policy
 400 {
 401    ac_glc = 1 << 0,      /* per-CU cache control */
 402    ac_slc = 1 << 1,      /* global L2 cache control */
 403    ac_dlc = 1 << 2,      /* per-shader-array cache control */
 404    ac_swizzled = 1 << 3, /* the access is swizzled, disabling load/store merging */
 405 };
 406
 407 struct ac_image_args {
 408    enum ac_image_opcode opcode : 4;
 409    enum ac_atomic_op atomic : 4; /* for the ac_image_atomic opcode */
 410    enum ac_image_dim dim : 3;
 411    unsigned dmask : 4;
 412    unsigned cache_policy : 3;
 413    bool unorm : 1;
 414    bool level_zero : 1;
 415    bool d16 : 1;        /* data and return values are 16-bit, requires GFX8+ */
 416    unsigned attributes; /* additional call-site specific AC_FUNC_ATTRs */
 417
 418    LLVMValueRef resource;
 419    LLVMValueRef sampler;
 420    LLVMValueRef data[2]; /* data[0] is source data (vector); data[1] is cmp for cmpswap */
 421    LLVMValueRef offset;
 422    LLVMValueRef bias;
 423    LLVMValueRef compare;
 424    LLVMValueRef derivs[6];
 425    LLVMValueRef coords[4];
 426    LLVMValueRef lod; // also used by ac_image_get_resinfo
 427    LLVMValueRef min_lod;
 428 };
 429
 430 LLVMValueRef ac_build_image_opcode(struct ac_llvm_context *ctx, struct ac_image_args *a);
 431 LLVMValueRef ac_build_image_get_sample_count(struct ac_llvm_context *ctx, LLVMValueRef rsrc);
 432 LLVMValueRef ac_build_cvt_pkrtz_f16(struct ac_llvm_context *ctx, LLVMValueRef args[2]);
 433 LLVMValueRef ac_build_cvt_pknorm_i16(struct ac_llvm_context *ctx, LLVMValueRef args[2]);
 434 LLVMValueRef ac_build_cvt_pknorm_u16(struct ac_llvm_context *ctx, LLVMValueRef args[2]);
 435 LLVMValueRef ac_build_cvt_pk_i16(struct ac_llvm_context *ctx, LLVMValueRef args[2], unsigned bits,
 436                                  bool hi);
 437 LLVMValueRef ac_build_cvt_pk_u16(struct ac_llvm_context *ctx, LLVMValueRef args[2], unsigned bits,
 438                                  bool hi);
 439 LLVMValueRef ac_build_wqm_vote(struct ac_llvm_context *ctx, LLVMValueRef i1);
 440 void ac_build_kill_if_false(struct ac_llvm_context *ctx, LLVMValueRef i1);
 441 LLVMValueRef ac_build_bfe(struct ac_llvm_context *ctx, LLVMValueRef input, LLVMValueRef offset,
 442                           LLVMValueRef width, bool is_signed);
 443 LLVMValueRef ac_build_imad(struct ac_llvm_context *ctx, LLVMValueRef s0, LLVMValueRef s1,
 444                            LLVMValueRef s2);
 445 LLVMValueRef ac_build_fmad(struct ac_llvm_context *ctx, LLVMValueRef s0, LLVMValueRef s1,
 446                            LLVMValueRef s2);
 447
 448 void ac_build_waitcnt(struct ac_llvm_context *ctx, unsigned wait_flags);
 449
 450 LLVMValueRef ac_build_fract(struct ac_llvm_context *ctx, LLVMValueRef src0, unsigned bitsize);
 451 LLVMValueRef ac_const_uint_vec(struct ac_llvm_context *ctx, LLVMTypeRef type, uint64_t value);
 452 LLVMValueRef ac_build_isign(struct ac_llvm_context *ctx, LLVMValueRef src0);
 453 LLVMValueRef ac_build_fsign(struct ac_llvm_context *ctx, LLVMValueRef src);
 454 LLVMValueRef ac_build_bit_count(struct ac_llvm_context *ctx, LLVMValueRef src0);
 455
 456 LLVMValueRef ac_build_bitfield_reverse(struct ac_llvm_context *ctx, LLVMValueRef src0);
 457
 458 void ac_optimize_vs_outputs(struct ac_llvm_context *ac, LLVMValueRef main_fn,
 459                             uint8_t *vs_output_param_offset, uint32_t num_outputs,
 460                             uint32_t skip_output_mask, uint8_t *num_param_exports);
 461 void ac_init_exec_full_mask(struct ac_llvm_context *ctx);
 462
 463 void ac_declare_lds_as_pointer(struct ac_llvm_context *ac);
 464 LLVMValueRef ac_lds_load(struct ac_llvm_context *ctx, LLVMValueRef dw_addr);
 465 void ac_lds_store(struct ac_llvm_context *ctx, LLVMValueRef dw_addr, LLVMValueRef value);
 466
 467 LLVMValueRef ac_find_lsb(struct ac_llvm_context *ctx, LLVMTypeRef dst_type, LLVMValueRef src0);
 468
 469 LLVMTypeRef ac_array_in_const_addr_space(LLVMTypeRef elem_type);
 470 LLVMTypeRef ac_array_in_const32_addr_space(LLVMTypeRef elem_type);
 471
 472 void ac_build_bgnloop(struct ac_llvm_context *ctx, int lable_id);
 473 void ac_build_break(struct ac_llvm_context *ctx);
 474 void ac_build_continue(struct ac_llvm_context *ctx);
 475 void ac_build_else(struct ac_llvm_context *ctx, int lable_id);
 476 void ac_build_endif(struct ac_llvm_context *ctx, int lable_id);
 477 void ac_build_endloop(struct ac_llvm_context *ctx, int lable_id);
 478 void ac_build_ifcc(struct ac_llvm_context *ctx, LLVMValueRef cond, int label_id);
 479 void ac_build_if(struct ac_llvm_context *ctx, LLVMValueRef value, int lable_id);
 480 void ac_build_uif(struct ac_llvm_context *ctx, LLVMValueRef value, int lable_id);
 481
 482 LLVMValueRef ac_build_alloca(struct ac_llvm_context *ac, LLVMTypeRef type, const char *name);
 483 LLVMValueRef ac_build_alloca_undef(struct ac_llvm_context *ac, LLVMTypeRef type, const char *name);
 484
 485 LLVMValueRef ac_cast_ptr(struct ac_llvm_context *ctx, LLVMValueRef ptr, LLVMTypeRef type);
 486
 487 LLVMValueRef ac_trim_vector(struct ac_llvm_context *ctx, LLVMValueRef value, unsigned count);
 488
 489 LLVMValueRef ac_unpack_param(struct ac_llvm_context *ctx, LLVMValueRef param, unsigned rshift,
 490                              unsigned bitwidth);
 491
 492 void ac_apply_fmask_to_sample(struct ac_llvm_context *ac, LLVMValueRef fmask, LLVMValueRef *addr,
 493                               bool is_array_tex);
 494
 495 LLVMValueRef ac_build_ds_swizzle(struct ac_llvm_context *ctx, LLVMValueRef src, unsigned mask);
 496
 497 LLVMValueRef ac_build_readlane_no_opt_barrier(struct ac_llvm_context *ctx, LLVMValueRef src,
 498                                               LLVMValueRef lane);
 499
 500 LLVMValueRef ac_build_readlane(struct ac_llvm_context *ctx, LLVMValueRef src, LLVMValueRef lane);
 501
 502 LLVMValueRef ac_build_writelane(struct ac_llvm_context *ctx, LLVMValueRef src, LLVMValueRef value,
 503                                 LLVMValueRef lane);
 504
 505 LLVMValueRef ac_build_mbcnt(struct ac_llvm_context *ctx, LLVMValueRef mask);
 506
 507 LLVMValueRef ac_build_inclusive_scan(struct ac_llvm_context *ctx, LLVMValueRef src, nir_op op);
 508
 509 LLVMValueRef ac_build_exclusive_scan(struct ac_llvm_context *ctx, LLVMValueRef src, nir_op op);
 510
 511 LLVMValueRef ac_build_reduce(struct ac_llvm_context *ctx, LLVMValueRef src, nir_op op,
 512                              unsigned cluster_size);
 513
 514 /**
 515  * Common arguments for a scan/reduce operation that accumulates per-wave
 516  * values across an entire workgroup, while respecting the order of waves.
 517  */
 518 struct ac_wg_scan {
 519    bool enable_reduce;
 520    bool enable_exclusive;
 521    bool enable_inclusive;
 522    nir_op op;
 523    LLVMValueRef src; /* clobbered! */
 524    LLVMValueRef result_reduce;
 525    LLVMValueRef result_exclusive;
 526    LLVMValueRef result_inclusive;
 527    LLVMValueRef extra;
 528    LLVMValueRef waveidx;
 529    LLVMValueRef numwaves; /* only needed for "reduce" operations */
 530
 531    /* T addrspace(LDS) pointer to the same type as value, at least maxwaves entries */
 532    LLVMValueRef scratch;
 533    unsigned maxwaves;
 534 };
 535
 536 void ac_build_wg_wavescan_top(struct ac_llvm_context *ctx, struct ac_wg_scan *ws);
 537 void ac_build_wg_wavescan_bottom(struct ac_llvm_context *ctx, struct ac_wg_scan *ws);
 538 void ac_build_wg_wavescan(struct ac_llvm_context *ctx, struct ac_wg_scan *ws);
 539
 540 void ac_build_wg_scan_top(struct ac_llvm_context *ctx, struct ac_wg_scan *ws);
 541 void ac_build_wg_scan_bottom(struct ac_llvm_context *ctx, struct ac_wg_scan *ws);
 542 void ac_build_wg_scan(struct ac_llvm_context *ctx, struct ac_wg_scan *ws);
 543
 544 LLVMValueRef ac_build_quad_swizzle(struct ac_llvm_context *ctx, LLVMValueRef src, unsigned lane0,
 545                                    unsigned lane1, unsigned lane2, unsigned lane3);
 546
 547 LLVMValueRef ac_build_shuffle(struct ac_llvm_context *ctx, LLVMValueRef src, LLVMValueRef index);
 548
 549 LLVMValueRef ac_build_frexp_exp(struct ac_llvm_context *ctx, LLVMValueRef src0, unsigned bitsize);
 550
 551 LLVMValueRef ac_build_frexp_mant(struct ac_llvm_context *ctx, LLVMValueRef src0, unsigned bitsize);
 552
 553 LLVMValueRef ac_build_canonicalize(struct ac_llvm_context *ctx, LLVMValueRef src0,
 554                                    unsigned bitsize);
 555
 556 LLVMValueRef ac_build_ddxy_interp(struct ac_llvm_context *ctx, LLVMValueRef interp_ij);
 557
 558 LLVMValueRef ac_build_load_helper_invocation(struct ac_llvm_context *ctx);
 559
 560 LLVMValueRef ac_build_is_helper_invocation(struct ac_llvm_context *ctx);
 561
 562 LLVMValueRef ac_build_call(struct ac_llvm_context *ctx, LLVMValueRef func, LLVMValueRef *args,
 563                            unsigned num_args);
 564
 565 LLVMValueRef ac_build_atomic_rmw(struct ac_llvm_context *ctx, LLVMAtomicRMWBinOp op,
 566                                  LLVMValueRef ptr, LLVMValueRef val, const char *sync_scope);
 567
 568 LLVMValueRef ac_build_atomic_cmp_xchg(struct ac_llvm_context *ctx, LLVMValueRef ptr,
 569                                       LLVMValueRef cmp, LLVMValueRef val, const char *sync_scope);
 570
 571 void ac_export_mrt_z(struct ac_llvm_context *ctx, LLVMValueRef depth, LLVMValueRef stencil,
 572                      LLVMValueRef samplemask, struct ac_export_args *args);
 573
 574 void ac_build_sendmsg_gs_alloc_req(struct ac_llvm_context *ctx, LLVMValueRef wave_id,
 575                                    LLVMValueRef vtx_cnt, LLVMValueRef prim_cnt);
 576
 577 struct ac_ngg_prim {
 578    unsigned num_vertices;
 579    LLVMValueRef isnull;
 580    LLVMValueRef index[3];
 581    LLVMValueRef edgeflag[3];
 582    LLVMValueRef passthrough;
 583 };
 584
 585 LLVMValueRef ac_pack_prim_export(struct ac_llvm_context *ctx, const struct ac_ngg_prim *prim);
 586 void ac_build_export_prim(struct ac_llvm_context *ctx, const struct ac_ngg_prim *prim);
 587
 588 static inline LLVMValueRef ac_get_arg(struct ac_llvm_context *ctx, struct ac_arg arg)
 589 {
 590    assert(arg.used);
 591    return LLVMGetParam(ctx->main_function, arg.arg_index);
 592 }
 593
 594 enum ac_llvm_calling_convention
 595 {
 596    AC_LLVM_AMDGPU_VS = 87,
 597    AC_LLVM_AMDGPU_GS = 88,
 598    AC_LLVM_AMDGPU_PS = 89,
 599    AC_LLVM_AMDGPU_CS = 90,
 600    AC_LLVM_AMDGPU_HS = 93,
 601 };
 602
 603 LLVMValueRef ac_build_main(const struct ac_shader_args *args, struct ac_llvm_context *ctx,
 604                            enum ac_llvm_calling_convention convention, const char *name,
 605                            LLVMTypeRef ret_type, LLVMModuleRef module);
 606 void ac_build_s_endpgm(struct ac_llvm_context *ctx);
 607
 608 LLVMValueRef ac_prefix_bitcount(struct ac_llvm_context *ctx, LLVMValueRef mask, LLVMValueRef index);
 609 LLVMValueRef ac_prefix_bitcount_2x64(struct ac_llvm_context *ctx, LLVMValueRef mask[2],
 610                                      LLVMValueRef index);
 611 void ac_build_triangle_strip_indices_to_triangle(struct ac_llvm_context *ctx, LLVMValueRef is_odd,
 612                                                  LLVMValueRef flatshade_first,
 613                                                  LLVMValueRef index[3]);
 614
 615 #ifdef __cplusplus
 616 }
 617 #endif
 618
 619 #endif