src/compiler/nir/nir_intrinsics.h

   1 /*
   2  * Copyright © 2014 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Connor Abbott (cwabbott0@gmail.com)
  25  *
  26  */
  27
  28 /**
  29  * This header file defines all the available intrinsics in one place. It
  30  * expands to a list of macros of the form:
  31  *
  32  * INTRINSIC(name, num_srcs, src_components, has_dest, dest_components,
  33  *              num_variables, num_indices, idx0, idx1, idx2, flags)
  34  *
  35  * Which should correspond one-to-one with the nir_intrinsic_info structure. It
  36  * is included in both ir.h to create the nir_intrinsic enum (with members of
  37  * the form nir_intrinsic_(name)) and and in opcodes.c to create
  38  * nir_intrinsic_infos, which is a const array of nir_intrinsic_info structures
  39  * for each intrinsic.
  40  */
  41
  42 #define ARR(...) { __VA_ARGS__ }
  43
  44 INTRINSIC(nop, 0, ARR(0), false, 0, 0, 0, xx, xx, xx,
  45           NIR_INTRINSIC_CAN_ELIMINATE)
  46
  47 INTRINSIC(load_var, 0, ARR(0), true, 0, 1, 0, xx, xx, xx, NIR_INTRINSIC_CAN_ELIMINATE)
  48 INTRINSIC(store_var, 1, ARR(0), false, 0, 1, 1, WRMASK, xx, xx, 0)
  49 INTRINSIC(copy_var, 0, ARR(0), false, 0, 2, 0, xx, xx, xx, 0)
  50
  51 /*
  52  * Interpolation of input.  The interp_var_at* intrinsics are similar to the
  53  * load_var intrinsic acting on a shader input except that they interpolate
  54  * the input differently.  The at_sample and at_offset intrinsics take an
  55  * additional source that is an integer sample id or a vec2 position offset
  56  * respectively.
  57  */
  58
  59 INTRINSIC(interp_var_at_centroid, 0, ARR(0), true, 0, 1, 0, xx, xx, xx,
  60           NIR_INTRINSIC_CAN_ELIMINATE | NIR_INTRINSIC_CAN_REORDER)
  61 INTRINSIC(interp_var_at_sample, 1, ARR(1), true, 0, 1, 0, xx, xx, xx,
  62           NIR_INTRINSIC_CAN_ELIMINATE | NIR_INTRINSIC_CAN_REORDER)
  63 INTRINSIC(interp_var_at_offset, 1, ARR(2), true, 0, 1, 0, xx, xx, xx,
  64           NIR_INTRINSIC_CAN_ELIMINATE | NIR_INTRINSIC_CAN_REORDER)
  65
  66 /*
  67  * Ask the driver for the size of a given buffer. It takes the buffer index
  68  * as source.
  69  */
  70 INTRINSIC(get_buffer_size, 1, ARR(1), true, 1, 0, 0, xx, xx, xx,
  71           NIR_INTRINSIC_CAN_ELIMINATE | NIR_INTRINSIC_CAN_REORDER)
  72
  73 /*
  74  * a barrier is an intrinsic with no inputs/outputs but which can't be moved
  75  * around/optimized in general
  76  */
  77 #define BARRIER(name) INTRINSIC(name, 0, ARR(0), false, 0, 0, 0, xx, xx, xx, 0)
  78
  79 BARRIER(barrier)
  80 BARRIER(discard)
  81
  82 /*
  83  * Memory barrier with semantics analogous to the memoryBarrier() GLSL
  84  * intrinsic.
  85  */
  86 BARRIER(memory_barrier)
  87
  88 /*
  89  * Shader clock intrinsic with semantics analogous to the clock2x32ARB()
  90  * GLSL intrinsic.
  91  * The latter can be used as code motion barrier, which is currently not
  92  * feasible with NIR.
  93  */
  94 INTRINSIC(shader_clock, 0, ARR(0), true, 2, 0, 0, xx, xx, xx, NIR_INTRINSIC_CAN_ELIMINATE)
  95
  96 /*
  97  * Shader ballot intrinsics with semantics analogous to the
  98  *
  99  *    ballotARB()
 100  *    readInvocationARB()
 101  *    readFirstInvocationARB()
 102  *
 103  * GLSL functions from ARB_shader_ballot.
 104  */
 105 INTRINSIC(ballot, 1, ARR(1), true, 0, 0, 0, xx, xx, xx, NIR_INTRINSIC_CAN_ELIMINATE)
 106 INTRINSIC(read_invocation, 2, ARR(0, 1), true, 0, 0, 0, xx, xx, xx, NIR_INTRINSIC_CAN_ELIMINATE)
 107 INTRINSIC(read_first_invocation, 1, ARR(0), true, 0, 0, 0, xx, xx, xx, NIR_INTRINSIC_CAN_ELIMINATE)
 108
 109 /** Additional SPIR-V ballot intrinsics
 110  *
 111  * These correspond to the SPIR-V opcodes
 112  *
 113  *    OpGroupUniformElect
 114  *    OpSubgroupFirstInvocationKHR
 115  */
 116 INTRINSIC(elect, 0, ARR(0), true, 1, 0, 0, xx, xx, xx,
 117           NIR_INTRINSIC_CAN_ELIMINATE)
 118 INTRINSIC(first_invocation, 0, ARR(0), true, 1, 0, 0, xx, xx, xx,
 119           NIR_INTRINSIC_CAN_ELIMINATE)
 120
 121 /*
 122  * Memory barrier with semantics analogous to the compute shader
 123  * groupMemoryBarrier(), memoryBarrierAtomicCounter(), memoryBarrierBuffer(),
 124  * memoryBarrierImage() and memoryBarrierShared() GLSL intrinsics.
 125  */
 126 BARRIER(group_memory_barrier)
 127 BARRIER(memory_barrier_atomic_counter)
 128 BARRIER(memory_barrier_buffer)
 129 BARRIER(memory_barrier_image)
 130 BARRIER(memory_barrier_shared)
 131
 132 /** A conditional discard, with a single boolean source. */
 133 INTRINSIC(discard_if, 1, ARR(1), false, 0, 0, 0, xx, xx, xx, 0)
 134
 135 /** ARB_shader_group_vote intrinsics */
 136 INTRINSIC(vote_any, 1, ARR(1), true, 1, 0, 0, xx, xx, xx, NIR_INTRINSIC_CAN_ELIMINATE)
 137 INTRINSIC(vote_all, 1, ARR(1), true, 1, 0, 0, xx, xx, xx, NIR_INTRINSIC_CAN_ELIMINATE)
 138 INTRINSIC(vote_feq, 1, ARR(0), true, 1, 0, 0, xx, xx, xx, NIR_INTRINSIC_CAN_ELIMINATE)
 139 INTRINSIC(vote_ieq, 1, ARR(0), true, 1, 0, 0, xx, xx, xx, NIR_INTRINSIC_CAN_ELIMINATE)
 140
 141 /** Ballot ALU operations from SPIR-V.
 142  *
 143  * These operations work like their ALU counterparts except that the operate
 144  * on a uvec4 which is treated as a 128bit integer.  Also, they are, in
 145  * general, free to ignore any bits which are above the subgroup size.
 146  */
 147 INTRINSIC(ballot_bitfield_extract, 2, ARR(4, 1), true, 1, 0,
 148           0, xx, xx, xx, NIR_INTRINSIC_CAN_ELIMINATE)
 149 INTRINSIC(ballot_bit_count_reduce, 1, ARR(4), true, 1, 0,
 150           0, xx, xx, xx, NIR_INTRINSIC_CAN_ELIMINATE)
 151 INTRINSIC(ballot_bit_count_inclusive, 1, ARR(4), true, 1, 0,
 152           0, xx, xx, xx, NIR_INTRINSIC_CAN_ELIMINATE)
 153 INTRINSIC(ballot_bit_count_exclusive, 1, ARR(4), true, 1, 0,
 154           0, xx, xx, xx, NIR_INTRINSIC_CAN_ELIMINATE)
 155 INTRINSIC(ballot_find_lsb, 1, ARR(4), true, 1, 0,
 156           0, xx, xx, xx, NIR_INTRINSIC_CAN_ELIMINATE)
 157 INTRINSIC(ballot_find_msb, 1, ARR(4), true, 1, 0,
 158           0, xx, xx, xx, NIR_INTRINSIC_CAN_ELIMINATE)
 159
 160 /** Shuffle operations from SPIR-V. */
 161 INTRINSIC(shuffle, 2, ARR(0, 1), true, 0, 0,
 162           0, xx, xx, xx, NIR_INTRINSIC_CAN_ELIMINATE)
 163 INTRINSIC(shuffle_xor, 2, ARR(0, 1), true, 0, 0,
 164           0, xx, xx, xx, NIR_INTRINSIC_CAN_ELIMINATE)
 165 INTRINSIC(shuffle_up, 2, ARR(0, 1), true, 0, 0,
 166           0, xx, xx, xx, NIR_INTRINSIC_CAN_ELIMINATE)
 167 INTRINSIC(shuffle_down, 2, ARR(0, 1), true, 0, 0,
 168           0, xx, xx, xx, NIR_INTRINSIC_CAN_ELIMINATE)
 169
 170 /** Quad operations from SPIR-V. */
 171 INTRINSIC(quad_broadcast, 2, ARR(0, 1), true, 0, 0,
 172           0, xx, xx, xx, NIR_INTRINSIC_CAN_ELIMINATE)
 173 INTRINSIC(quad_swap_horizontal, 1, ARR(0), true, 0, 0,
 174           0, xx, xx, xx, NIR_INTRINSIC_CAN_ELIMINATE)
 175 INTRINSIC(quad_swap_vertical, 1, ARR(0), true, 0, 0,
 176           0, xx, xx, xx, NIR_INTRINSIC_CAN_ELIMINATE)
 177 INTRINSIC(quad_swap_diagonal, 1, ARR(0), true, 0, 0,
 178           0, xx, xx, xx, NIR_INTRINSIC_CAN_ELIMINATE)
 179
 180 INTRINSIC(reduce, 1, ARR(0), true, 0, 0,
 181           2, REDUCTION_OP, CLUSTER_SIZE, xx, NIR_INTRINSIC_CAN_ELIMINATE)
 182 INTRINSIC(inclusive_scan, 1, ARR(0), true, 0, 0,
 183           1, REDUCTION_OP, xx, xx, NIR_INTRINSIC_CAN_ELIMINATE)
 184 INTRINSIC(exclusive_scan, 1, ARR(0), true, 0, 0,
 185           1, REDUCTION_OP, xx, xx, NIR_INTRINSIC_CAN_ELIMINATE)
 186
 187 /**
 188  * Basic Geometry Shader intrinsics.
 189  *
 190  * emit_vertex implements GLSL's EmitStreamVertex() built-in.  It takes a single
 191  * index, which is the stream ID to write to.
 192  *
 193  * end_primitive implements GLSL's EndPrimitive() built-in.
 194  */
 195 INTRINSIC(emit_vertex,   0, ARR(0), false, 0, 0, 1, STREAM_ID, xx, xx, 0)
 196 INTRINSIC(end_primitive, 0, ARR(0), false, 0, 0, 1, STREAM_ID, xx, xx, 0)
 197
 198 /**
 199  * Geometry Shader intrinsics with a vertex count.
 200  *
 201  * Alternatively, drivers may implement these intrinsics, and use
 202  * nir_lower_gs_intrinsics() to convert from the basic intrinsics.
 203  *
 204  * These maintain a count of the number of vertices emitted, as an additional
 205  * unsigned integer source.
 206  */
 207 INTRINSIC(emit_vertex_with_counter, 1, ARR(1), false, 0, 0, 1, STREAM_ID, xx, xx, 0)
 208 INTRINSIC(end_primitive_with_counter, 1, ARR(1), false, 0, 0, 1, STREAM_ID, xx, xx, 0)
 209 INTRINSIC(set_vertex_count, 1, ARR(1), false, 0, 0, 0, xx, xx, xx, 0)
 210
 211 /*
 212  * Atomic counters
 213  *
 214  * The *_var variants take an atomic_uint nir_variable, while the other,
 215  * lowered, variants take a constant buffer index and register offset.
 216  */
 217
 218 #define ATOMIC(name, flags) \
 219    INTRINSIC(name##_var, 0, ARR(0), true, 1, 1, 0, xx, xx, xx, flags) \
 220    INTRINSIC(name, 1, ARR(1), true, 1, 0, 1, BASE, xx, xx, flags)
 221 #define ATOMIC2(name) \
 222    INTRINSIC(name##_var, 1, ARR(1), true, 1, 1, 0, xx, xx, xx, 0) \
 223    INTRINSIC(name, 2, ARR(1, 1), true, 1, 0, 1, BASE, xx, xx, 0)
 224 #define ATOMIC3(name) \
 225    INTRINSIC(name##_var, 2, ARR(1, 1), true, 1, 1, 0, xx, xx, xx, 0) \
 226    INTRINSIC(name, 3, ARR(1, 1, 1), true, 1, 0, 1, BASE, xx, xx, 0)
 227
 228 ATOMIC(atomic_counter_inc, 0)
 229 ATOMIC(atomic_counter_dec, 0)
 230 ATOMIC(atomic_counter_read, NIR_INTRINSIC_CAN_ELIMINATE)
 231 ATOMIC2(atomic_counter_add)
 232 ATOMIC2(atomic_counter_min)
 233 ATOMIC2(atomic_counter_max)
 234 ATOMIC2(atomic_counter_and)
 235 ATOMIC2(atomic_counter_or)
 236 ATOMIC2(atomic_counter_xor)
 237 ATOMIC2(atomic_counter_exchange)
 238 ATOMIC3(atomic_counter_comp_swap)
 239
 240 /*
 241  * Image load, store and atomic intrinsics.
 242  *
 243  * All image intrinsics take an image target passed as a nir_variable.  Image
 244  * variables contain a number of memory and layout qualifiers that influence
 245  * the semantics of the intrinsic.
 246  *
 247  * All image intrinsics take a four-coordinate vector and a sample index as
 248  * first two sources, determining the location within the image that will be
 249  * accessed by the intrinsic.  Components not applicable to the image target
 250  * in use are undefined.  Image store takes an additional four-component
 251  * argument with the value to be written, and image atomic operations take
 252  * either one or two additional scalar arguments with the same meaning as in
 253  * the ARB_shader_image_load_store specification.
 254  */
 255 INTRINSIC(image_var_load, 2, ARR(4, 1), true, 4, 1, 0, xx, xx, xx,
 256           NIR_INTRINSIC_CAN_ELIMINATE)
 257 INTRINSIC(image_var_store, 3, ARR(4, 1, 4), false, 0, 1, 0, xx, xx, xx, 0)
 258 INTRINSIC(image_var_atomic_add, 3, ARR(4, 1, 1), true, 1, 1, 0, xx, xx, xx, 0)
 259 INTRINSIC(image_var_atomic_min, 3, ARR(4, 1, 1), true, 1, 1, 0, xx, xx, xx, 0)
 260 INTRINSIC(image_var_atomic_max, 3, ARR(4, 1, 1), true, 1, 1, 0, xx, xx, xx, 0)
 261 INTRINSIC(image_var_atomic_and, 3, ARR(4, 1, 1), true, 1, 1, 0, xx, xx, xx, 0)
 262 INTRINSIC(image_var_atomic_or, 3, ARR(4, 1, 1), true, 1, 1, 0, xx, xx, xx, 0)
 263 INTRINSIC(image_var_atomic_xor, 3, ARR(4, 1, 1), true, 1, 1, 0, xx, xx, xx, 0)
 264 INTRINSIC(image_var_atomic_exchange, 3, ARR(4, 1, 1), true, 1, 1, 0, xx, xx, xx, 0)
 265 INTRINSIC(image_var_atomic_comp_swap, 4, ARR(4, 1, 1, 1), true, 1, 1, 0, xx, xx, xx, 0)
 266 INTRINSIC(image_var_size, 0, ARR(0), true, 0, 1, 0, xx, xx, xx,
 267           NIR_INTRINSIC_CAN_ELIMINATE | NIR_INTRINSIC_CAN_REORDER)
 268 INTRINSIC(image_var_samples, 0, ARR(0), true, 1, 1, 0, xx, xx, xx,
 269           NIR_INTRINSIC_CAN_ELIMINATE | NIR_INTRINSIC_CAN_REORDER)
 270
 271 /*
 272  * Vulkan descriptor set intrinsics
 273  *
 274  * The Vulkan API uses a different binding model from GL.  In the Vulkan
 275  * API, all external resources are represented by a tuple:
 276  *
 277  * (descriptor set, binding, array index)
 278  *
 279  * where the array index is the only thing allowed to be indirect.  The
 280  * vulkan_surface_index intrinsic takes the descriptor set and binding as
 281  * its first two indices and the array index as its source.  The third
 282  * index is a nir_variable_mode in case that's useful to the backend.
 283  *
 284  * The intended usage is that the shader will call vulkan_surface_index to
 285  * get an index and then pass that as the buffer index ubo/ssbo calls.
 286  *
 287  * The vulkan_resource_reindex intrinsic takes a resource index in src0
 288  * (the result of a vulkan_resource_index or vulkan_resource_reindex) which
 289  * corresponds to the tuple (set, binding, index) and computes an index
 290  * corresponding to tuple (set, binding, idx + src1).
 291  */
 292 INTRINSIC(vulkan_resource_index, 1, ARR(1), true, 1, 0, 2,
 293           DESC_SET, BINDING, xx,
 294           NIR_INTRINSIC_CAN_ELIMINATE | NIR_INTRINSIC_CAN_REORDER)
 295 INTRINSIC(vulkan_resource_reindex, 2, ARR(1, 1), true, 1, 0, 0, xx, xx, xx,
 296           NIR_INTRINSIC_CAN_ELIMINATE | NIR_INTRINSIC_CAN_REORDER)
 297
 298 /*
 299  * variable atomic intrinsics
 300  *
 301  * All of these variable atomic memory operations read a value from memory,
 302  * compute a new value using one of the operations below, write the new value
 303  * to memory, and return the original value read.
 304  *
 305  * All operations take 1 source except CompSwap that takes 2. These sources
 306  * represent:
 307  *
 308  * 0: The data parameter to the atomic function (i.e. the value to add
 309  *    in shared_atomic_add, etc).
 310  * 1: For CompSwap only: the second data parameter.
 311  *
 312  * All operations take 1 variable deref.
 313  */
 314 INTRINSIC(var_atomic_add, 1, ARR(1), true, 1, 1, 0, xx, xx, xx, 0)
 315 INTRINSIC(var_atomic_imin, 1, ARR(1), true, 1, 1, 0, xx, xx, xx, 0)
 316 INTRINSIC(var_atomic_umin, 1, ARR(1), true, 1, 1, 0, xx, xx, xx, 0)
 317 INTRINSIC(var_atomic_imax, 1, ARR(1), true, 1, 1, 0, xx, xx, xx, 0)
 318 INTRINSIC(var_atomic_umax, 1, ARR(1), true, 1, 1, 0, xx, xx, xx, 0)
 319 INTRINSIC(var_atomic_and, 1, ARR(1), true, 1, 1, 0, xx, xx, xx, 0)
 320 INTRINSIC(var_atomic_or, 1, ARR(1), true, 1, 1, 0, xx, xx, xx, 0)
 321 INTRINSIC(var_atomic_xor, 1, ARR(1), true, 1, 1, 0, xx, xx, xx, 0)
 322 INTRINSIC(var_atomic_exchange, 1, ARR(1), true, 1, 1, 0, xx, xx, xx, 0)
 323 INTRINSIC(var_atomic_comp_swap, 2, ARR(1, 1), true, 1, 1, 0, xx, xx, xx, 0)
 324
 325 /*
 326  * SSBO atomic intrinsics
 327  *
 328  * All of the SSBO atomic memory operations read a value from memory,
 329  * compute a new value using one of the operations below, write the new
 330  * value to memory, and return the original value read.
 331  *
 332  * All operations take 3 sources except CompSwap that takes 4. These
 333  * sources represent:
 334  *
 335  * 0: The SSBO buffer index.
 336  * 1: The offset into the SSBO buffer of the variable that the atomic
 337  *    operation will operate on.
 338  * 2: The data parameter to the atomic function (i.e. the value to add
 339  *    in ssbo_atomic_add, etc).
 340  * 3: For CompSwap only: the second data parameter.
 341  */
 342 INTRINSIC(ssbo_atomic_add, 3, ARR(1, 1, 1), true, 1, 0, 0, xx, xx, xx, 0)
 343 INTRINSIC(ssbo_atomic_imin, 3, ARR(1, 1, 1), true, 1, 0, 0, xx, xx, xx, 0)
 344 INTRINSIC(ssbo_atomic_umin, 3, ARR(1, 1, 1), true, 1, 0, 0, xx, xx, xx, 0)
 345 INTRINSIC(ssbo_atomic_imax, 3, ARR(1, 1, 1), true, 1, 0, 0, xx, xx, xx, 0)
 346 INTRINSIC(ssbo_atomic_umax, 3, ARR(1, 1, 1), true, 1, 0, 0, xx, xx, xx, 0)
 347 INTRINSIC(ssbo_atomic_and, 3, ARR(1, 1, 1), true, 1, 0, 0, xx, xx, xx, 0)
 348 INTRINSIC(ssbo_atomic_or, 3, ARR(1, 1, 1), true, 1, 0, 0, xx, xx, xx, 0)
 349 INTRINSIC(ssbo_atomic_xor, 3, ARR(1, 1, 1), true, 1, 0, 0, xx, xx, xx, 0)
 350 INTRINSIC(ssbo_atomic_exchange, 3, ARR(1, 1, 1), true, 1, 0, 0, xx, xx, xx, 0)
 351 INTRINSIC(ssbo_atomic_comp_swap, 4, ARR(1, 1, 1, 1), true, 1, 0, 0, xx, xx, xx, 0)
 352
 353 /*
 354  * CS shared variable atomic intrinsics
 355  *
 356  * All of the shared variable atomic memory operations read a value from
 357  * memory, compute a new value using one of the operations below, write the
 358  * new value to memory, and return the original value read.
 359  *
 360  * All operations take 2 sources except CompSwap that takes 3. These
 361  * sources represent:
 362  *
 363  * 0: The offset into the shared variable storage region that the atomic
 364  *    operation will operate on.
 365  * 1: The data parameter to the atomic function (i.e. the value to add
 366  *    in shared_atomic_add, etc).
 367  * 2: For CompSwap only: the second data parameter.
 368  */
 369 INTRINSIC(shared_atomic_add, 2, ARR(1, 1), true, 1, 0, 1, BASE, xx, xx, 0)
 370 INTRINSIC(shared_atomic_imin, 2, ARR(1, 1), true, 1, 0, 1, BASE, xx, xx, 0)
 371 INTRINSIC(shared_atomic_umin, 2, ARR(1, 1), true, 1, 0, 1, BASE, xx, xx, 0)
 372 INTRINSIC(shared_atomic_imax, 2, ARR(1, 1), true, 1, 0, 1, BASE, xx, xx, 0)
 373 INTRINSIC(shared_atomic_umax, 2, ARR(1, 1), true, 1, 0, 1, BASE, xx, xx, 0)
 374 INTRINSIC(shared_atomic_and, 2, ARR(1, 1), true, 1, 0, 1, BASE, xx, xx, 0)
 375 INTRINSIC(shared_atomic_or, 2, ARR(1, 1), true, 1, 0, 1, BASE, xx, xx, 0)
 376 INTRINSIC(shared_atomic_xor, 2, ARR(1, 1), true, 1, 0, 1, BASE, xx, xx, 0)
 377 INTRINSIC(shared_atomic_exchange, 2, ARR(1, 1), true, 1, 0, 1, BASE, xx, xx, 0)
 378 INTRINSIC(shared_atomic_comp_swap, 3, ARR(1, 1, 1), true, 1, 0, 1, BASE, xx, xx, 0)
 379
 380 /* Used by nir_builder.h to generate loader helpers for the system values. */
 381 #ifndef DEFINE_SYSTEM_VALUE
 382 #define DEFINE_SYSTEM_VALUE(name)
 383 #endif
 384
 385 #define SYSTEM_VALUE(name, components, num_indices, idx0, idx1, idx2) \
 386    DEFINE_SYSTEM_VALUE(name) \
 387    INTRINSIC(load_##name, 0, ARR(0), true, components, 0, num_indices, \
 388    idx0, idx1, idx2, \
 389    NIR_INTRINSIC_CAN_ELIMINATE | NIR_INTRINSIC_CAN_REORDER)
 390
 391 SYSTEM_VALUE(frag_coord, 4, 0, xx, xx, xx)
 392 SYSTEM_VALUE(front_face, 1, 0, xx, xx, xx)
 393 SYSTEM_VALUE(vertex_id, 1, 0, xx, xx, xx)
 394 SYSTEM_VALUE(vertex_id_zero_base, 1, 0, xx, xx, xx)
 395 SYSTEM_VALUE(base_vertex, 1, 0, xx, xx, xx)
 396 SYSTEM_VALUE(instance_id, 1, 0, xx, xx, xx)
 397 SYSTEM_VALUE(base_instance, 1, 0, xx, xx, xx)
 398 SYSTEM_VALUE(draw_id, 1, 0, xx, xx, xx)
 399 SYSTEM_VALUE(sample_id, 1, 0, xx, xx, xx)
 400 SYSTEM_VALUE(sample_pos, 2, 0, xx, xx, xx)
 401 SYSTEM_VALUE(sample_mask_in, 1, 0, xx, xx, xx)
 402 SYSTEM_VALUE(primitive_id, 1, 0, xx, xx, xx)
 403 SYSTEM_VALUE(invocation_id, 1, 0, xx, xx, xx)
 404 SYSTEM_VALUE(tess_coord, 3, 0, xx, xx, xx)
 405 SYSTEM_VALUE(tess_level_outer, 4, 0, xx, xx, xx)
 406 SYSTEM_VALUE(tess_level_inner, 2, 0, xx, xx, xx)
 407 SYSTEM_VALUE(patch_vertices_in, 1, 0, xx, xx, xx)
 408 SYSTEM_VALUE(local_invocation_id, 3, 0, xx, xx, xx)
 409 SYSTEM_VALUE(local_invocation_index, 1, 0, xx, xx, xx)
 410 SYSTEM_VALUE(work_group_id, 3, 0, xx, xx, xx)
 411 SYSTEM_VALUE(user_clip_plane, 4, 1, UCP_ID, xx, xx)
 412 SYSTEM_VALUE(num_work_groups, 3, 0, xx, xx, xx)
 413 SYSTEM_VALUE(helper_invocation, 1, 0, xx, xx, xx)
 414 SYSTEM_VALUE(alpha_ref_float, 1, 0, xx, xx, xx)
 415 SYSTEM_VALUE(layer_id, 1, 0, xx, xx, xx)
 416 SYSTEM_VALUE(view_index, 1, 0, xx, xx, xx)
 417 SYSTEM_VALUE(subgroup_size, 1, 0, xx, xx, xx)
 418 SYSTEM_VALUE(subgroup_invocation, 1, 0, xx, xx, xx)
 419 SYSTEM_VALUE(subgroup_eq_mask, 0, 0, xx, xx, xx)
 420 SYSTEM_VALUE(subgroup_ge_mask, 0, 0, xx, xx, xx)
 421 SYSTEM_VALUE(subgroup_gt_mask, 0, 0, xx, xx, xx)
 422 SYSTEM_VALUE(subgroup_le_mask, 0, 0, xx, xx, xx)
 423 SYSTEM_VALUE(subgroup_lt_mask, 0, 0, xx, xx, xx)
 424 SYSTEM_VALUE(num_subgroups, 1, 0, xx, xx, xx)
 425 SYSTEM_VALUE(subgroup_id, 1, 0, xx, xx, xx)
 426 SYSTEM_VALUE(local_group_size, 3, 0, xx, xx, xx)
 427
 428 /* Blend constant color values.  Float values are clamped. */
 429 SYSTEM_VALUE(blend_const_color_r_float, 1, 0, xx, xx, xx)
 430 SYSTEM_VALUE(blend_const_color_g_float, 1, 0, xx, xx, xx)
 431 SYSTEM_VALUE(blend_const_color_b_float, 1, 0, xx, xx, xx)
 432 SYSTEM_VALUE(blend_const_color_a_float, 1, 0, xx, xx, xx)
 433 SYSTEM_VALUE(blend_const_color_rgba8888_unorm, 1, 0, xx, xx, xx)
 434 SYSTEM_VALUE(blend_const_color_aaaa8888_unorm, 1, 0, xx, xx, xx)
 435
 436 /**
 437  * Barycentric coordinate intrinsics.
 438  *
 439  * These set up the barycentric coordinates for a particular interpolation.
 440  * The first three are for the simple cases: pixel, centroid, or per-sample
 441  * (at gl_SampleID).  The next two handle interpolating at a specified
 442  * sample location, or interpolating with a vec2 offset,
 443  *
 444  * The interp_mode index should be either the INTERP_MODE_SMOOTH or
 445  * INTERP_MODE_NOPERSPECTIVE enum values.
 446  *
 447  * The vec2 value produced by these intrinsics is intended for use as the
 448  * barycoord source of a load_interpolated_input intrinsic.
 449  */
 450
 451 #define BARYCENTRIC(name, sources, source_components) \
 452    INTRINSIC(load_barycentric_##name, sources, ARR(source_components), \
 453              true, 2, 0, 1, INTERP_MODE, xx, xx, \
 454              NIR_INTRINSIC_CAN_ELIMINATE | NIR_INTRINSIC_CAN_REORDER)
 455
 456 /* no sources.  const_index[] = { interp_mode } */
 457 BARYCENTRIC(pixel, 0, 0)
 458 BARYCENTRIC(centroid, 0, 0)
 459 BARYCENTRIC(sample, 0, 0)
 460 /* src[] = { sample_id }.  const_index[] = { interp_mode } */
 461 BARYCENTRIC(at_sample, 1, 1)
 462 /* src[] = { offset.xy }.  const_index[] = { interp_mode } */
 463 BARYCENTRIC(at_offset, 1, 2)
 464
 465 /*
 466  * Load operations pull data from some piece of GPU memory.  All load
 467  * operations operate in terms of offsets into some piece of theoretical
 468  * memory.  Loads from externally visible memory (UBO and SSBO) simply take a
 469  * byte offset as a source.  Loads from opaque memory (uniforms, inputs, etc.)
 470  * take a base+offset pair where the base (const_index[0]) gives the location
 471  * of the start of the variable being loaded and and the offset source is a
 472  * offset into that variable.
 473  *
 474  * Uniform load operations have a second "range" index that specifies the
 475  * range (starting at base) of the data from which we are loading.  If
 476  * const_index[1] == 0, then the range is unknown.
 477  *
 478  * Some load operations such as UBO/SSBO load and per_vertex loads take an
 479  * additional source to specify which UBO/SSBO/vertex to load from.
 480  *
 481  * The exact address type depends on the lowering pass that generates the
 482  * load/store intrinsics.  Typically, this is vec4 units for things such as
 483  * varying slots and float units for fragment shader inputs.  UBO and SSBO
 484  * offsets are always in bytes.
 485  */
 486
 487 #define LOAD(name, srcs, num_indices, idx0, idx1, idx2, flags) \
 488    INTRINSIC(load_##name, srcs, ARR(1, 1, 1, 1), true, 0, 0, num_indices, idx0, idx1, idx2, flags)
 489
 490 /* src[] = { offset }. const_index[] = { base, range } */
 491 LOAD(uniform, 1, 2, BASE, RANGE, xx, NIR_INTRINSIC_CAN_ELIMINATE | NIR_INTRINSIC_CAN_REORDER)
 492 /* src[] = { buffer_index, offset }. No const_index */
 493 LOAD(ubo, 2, 0, xx, xx, xx, NIR_INTRINSIC_CAN_ELIMINATE | NIR_INTRINSIC_CAN_REORDER)
 494 /* src[] = { offset }. const_index[] = { base, component } */
 495 LOAD(input, 1, 2, BASE, COMPONENT, xx, NIR_INTRINSIC_CAN_ELIMINATE | NIR_INTRINSIC_CAN_REORDER)
 496 /* src[] = { vertex, offset }. const_index[] = { base, component } */
 497 LOAD(per_vertex_input, 2, 2, BASE, COMPONENT, xx, NIR_INTRINSIC_CAN_ELIMINATE | NIR_INTRINSIC_CAN_REORDER)
 498 /* src[] = { barycoord, offset }. const_index[] = { base, component } */
 499 INTRINSIC(load_interpolated_input, 2, ARR(2, 1), true, 0, 0,
 500           2, BASE, COMPONENT, xx,
 501           NIR_INTRINSIC_CAN_ELIMINATE | NIR_INTRINSIC_CAN_REORDER)
 502
 503 /* src[] = { buffer_index, offset }. No const_index */
 504 LOAD(ssbo, 2, 0, xx, xx, xx, NIR_INTRINSIC_CAN_ELIMINATE)
 505 /* src[] = { offset }. const_index[] = { base, component } */
 506 LOAD(output, 1, 2, BASE, COMPONENT, xx, NIR_INTRINSIC_CAN_ELIMINATE)
 507 /* src[] = { vertex, offset }. const_index[] = { base, component } */
 508 LOAD(per_vertex_output, 2, 1, BASE, COMPONENT, xx, NIR_INTRINSIC_CAN_ELIMINATE)
 509 /* src[] = { offset }. const_index[] = { base } */
 510 LOAD(shared, 1, 1, BASE, xx, xx, NIR_INTRINSIC_CAN_ELIMINATE)
 511 /* src[] = { offset }. const_index[] = { base, range } */
 512 LOAD(push_constant, 1, 2, BASE, RANGE, xx,
 513      NIR_INTRINSIC_CAN_ELIMINATE | NIR_INTRINSIC_CAN_REORDER)
 514
 515 /*
 516  * Stores work the same way as loads, except now the first source is the value
 517  * to store and the second (and possibly third) source specify where to store
 518  * the value.  SSBO and shared memory stores also have a write mask as
 519  * const_index[0].
 520  */
 521
 522 #define STORE(name, srcs, num_indices, idx0, idx1, idx2, flags) \
 523    INTRINSIC(store_##name, srcs, ARR(0, 1, 1, 1), false, 0, 0, num_indices, idx0, idx1, idx2, flags)
 524
 525 /* src[] = { value, offset }. const_index[] = { base, write_mask, component } */
 526 STORE(output, 2, 3, BASE, WRMASK, COMPONENT, 0)
 527 /* src[] = { value, vertex, offset }.
 528  * const_index[] = { base, write_mask, component }
 529  */
 530 STORE(per_vertex_output, 3, 3, BASE, WRMASK, COMPONENT, 0)
 531 /* src[] = { value, block_index, offset }. const_index[] = { write_mask } */
 532 STORE(ssbo, 3, 1, WRMASK, xx, xx, 0)
 533 /* src[] = { value, offset }. const_index[] = { base, write_mask } */
 534 STORE(shared, 2, 2, BASE, WRMASK, xx, 0)
 535
 536 LAST_INTRINSIC(store_shared)
 537
 538 #undef DEFINE_SYSTEM_VALUE
 539 #undef INTRINSIC
 540 #undef LAST_INTRINSIC