+# src[] = { value, address }.
+store("global", 2, [WRMASK, ACCESS, ALIGN_MUL, ALIGN_OFFSET])
+# src[] = { value, offset }.
+store("scratch", 2, [ALIGN_MUL, ALIGN_OFFSET, WRMASK])
+
+# IR3-specific version of most SSBO intrinsics. The only different
+# compare to the originals is that they add an extra source to hold
+# the dword-offset, which is needed by the backend code apart from
+# the byte-offset already provided by NIR in one of the sources.
+#
+# NIR lowering pass 'ir3_nir_lower_io_offset' will replace the
+# original SSBO intrinsics by these, placing the computed
+# dword-offset always in the last source.
+#
+# The float versions are not handled because those are not supported
+# by the backend.
+intrinsic("store_ssbo_ir3", src_comp=[0, 1, 1, 1],
+ indices=[WRMASK, ACCESS, ALIGN_MUL, ALIGN_OFFSET])
+intrinsic("load_ssbo_ir3", src_comp=[1, 1, 1], dest_comp=0,
+ indices=[ACCESS, ALIGN_MUL, ALIGN_OFFSET], flags=[CAN_ELIMINATE])
+intrinsic("ssbo_atomic_add_ir3", src_comp=[1, 1, 1, 1], dest_comp=1)
+intrinsic("ssbo_atomic_imin_ir3", src_comp=[1, 1, 1, 1], dest_comp=1)
+intrinsic("ssbo_atomic_umin_ir3", src_comp=[1, 1, 1, 1], dest_comp=1)
+intrinsic("ssbo_atomic_imax_ir3", src_comp=[1, 1, 1, 1], dest_comp=1)
+intrinsic("ssbo_atomic_umax_ir3", src_comp=[1, 1, 1, 1], dest_comp=1)
+intrinsic("ssbo_atomic_and_ir3", src_comp=[1, 1, 1, 1], dest_comp=1)
+intrinsic("ssbo_atomic_or_ir3", src_comp=[1, 1, 1, 1], dest_comp=1)
+intrinsic("ssbo_atomic_xor_ir3", src_comp=[1, 1, 1, 1], dest_comp=1)
+intrinsic("ssbo_atomic_exchange_ir3", src_comp=[1, 1, 1, 1], dest_comp=1)
+intrinsic("ssbo_atomic_comp_swap_ir3", src_comp=[1, 1, 1, 1, 1], dest_comp=1)
+
+# System values for freedreno geometry shaders.
+system_value("vs_primitive_stride_ir3", 1)
+system_value("vs_vertex_stride_ir3", 1)
+system_value("gs_header_ir3", 1)
+system_value("primitive_location_ir3", 1, indices=[DRIVER_LOCATION])
+
+# System values for freedreno tessellation shaders.
+system_value("hs_patch_stride_ir3", 1)
+system_value("tess_factor_base_ir3", 2)
+system_value("tess_param_base_ir3", 2)
+system_value("tcs_header_ir3", 1)
+
+# IR3-specific intrinsics for tessellation control shaders. cond_end_ir3 end
+# the shader when src0 is false and is used to narrow down the TCS shader to
+# just thread 0 before writing out tessellation levels.
+intrinsic("cond_end_ir3", src_comp=[1])
+# end_patch_ir3 is used just before thread 0 exist the TCS and presumably
+# signals the TE that the patch is complete and can be tessellated.
+intrinsic("end_patch_ir3")
+
+# IR3-specific load/store intrinsics. These access a buffer used to pass data
+# between geometry stages - perhaps it's explicit access to the vertex cache.
+
+# src[] = { value, offset }.
+store("shared_ir3", 2, [BASE, WRMASK, ALIGN_MUL, ALIGN_OFFSET])
+# src[] = { offset }.
+load("shared_ir3", 1, [BASE, ALIGN_MUL, ALIGN_OFFSET], [CAN_ELIMINATE])
+
+# IR3-specific load/store global intrinsics. They take a 64-bit base address
+# and a 32-bit offset. The hardware will add the base and the offset, which
+# saves us from doing 64-bit math on the base address.
+
+# src[] = { value, address(vec2 of hi+lo uint32_t), offset }.
+# const_index[] = { write_mask, align_mul, align_offset }
+intrinsic("store_global_ir3", [0, 2, 1], indices=[WRMASK, ACCESS, ALIGN_MUL, ALIGN_OFFSET])
+# src[] = { address(vec2 of hi+lo uint32_t), offset }.
+# const_index[] = { access, align_mul, align_offset }
+intrinsic("load_global_ir3", [2, 1], dest_comp=0, indices=[ACCESS, ALIGN_MUL, ALIGN_OFFSET], flags=[CAN_ELIMINATE])
+
+# Intrinsics used by the Midgard/Bifrost blend pipeline. These are defined
+# within a blend shader to read/write the raw value from the tile buffer,
+# without applying any format conversion in the process. If the shader needs
+# usable pixel values, it must apply format conversions itself.
+#
+# These definitions are generic, but they are explicitly vendored to prevent
+# other drivers from using them, as their semantics is defined in terms of the
+# Midgard/Bifrost hardware tile buffer and may not line up with anything sane.
+# One notable divergence is sRGB, which is asymmetric: raw_input_pan requires
+# an sRGB->linear conversion, but linear values should be written to
+# raw_output_pan and the hardware handles linear->sRGB.
+#
+# We also have format-specific Midgard intrinsics. There are rather
+# here-be-dragons. load_output_u8_as_fp16_pan does the equivalent of
+# load_raw_out_pan on an RGBA8 UNORM framebuffer followed by u2u16 -> fp16 ->
+# division by 255.
+
+# src[] = { value }
+store("raw_output_pan", 1, [])
+load("raw_output_pan", 0, [], [CAN_ELIMINATE, CAN_REORDER])
+load("output_u8_as_fp16_pan", 0, [], [CAN_ELIMINATE, CAN_REORDER])
+
+# Loads the sampler paramaters <min_lod, max_lod, lod_bias>
+# src[] = { sampler_index }
+load("sampler_lod_parameters_pan", 1, [CAN_ELIMINATE, CAN_REORDER])
+
+# V3D-specific instrinc for tile buffer color reads.
+#
+# The hardware requires that we read the samples and components of a pixel
+# in order, so we cannot eliminate or remove any loads in a sequence.
+#
+# src[] = { render_target }
+# BASE = sample index
+load("tlb_color_v3d", 1, [BASE, COMPONENT], [])
+
+# V3D-specific instrinc for per-sample tile buffer color writes.
+#
+# The driver backend needs to identify per-sample color writes and emit
+# specific code for them.
+#
+# src[] = { value, render_target }
+# BASE = sample index
+store("tlb_sample_color_v3d", 2, [BASE, COMPONENT, TYPE], [])
+
+# V3D-specific intrinsic to load the number of layers attached to
+# the target framebuffer
+intrinsic("load_fb_layers_v3d", dest_comp=1, flags=[CAN_ELIMINATE, CAN_REORDER])