ac: use new LLVM 8 intrinsic when storing 16-bit values
[mesa.git] / src / amd / common / ac_llvm_build.c
index 76047148a6a35bac8016a6e6888213a2c42c2f9b..73dd0230785b11a23cb3b2fb31454e5e89fe77e0 100644 (file)
@@ -219,6 +219,16 @@ ac_to_integer_type(struct ac_llvm_context *ctx, LLVMTypeRef t)
                return LLVMVectorType(to_integer_type_scalar(ctx, elem_type),
                                      LLVMGetVectorSize(t));
        }
+       if (LLVMGetTypeKind(t) == LLVMPointerTypeKind) {
+               switch (LLVMGetPointerAddressSpace(t)) {
+               case AC_ADDR_SPACE_GLOBAL:
+                       return ctx->i64;
+               case AC_ADDR_SPACE_LDS:
+                       return ctx->i32;
+               default:
+                       unreachable("unhandled address space");
+               }
+       }
        return to_integer_type_scalar(ctx, t);
 }
 
@@ -226,6 +236,9 @@ LLVMValueRef
 ac_to_integer(struct ac_llvm_context *ctx, LLVMValueRef v)
 {
        LLVMTypeRef type = LLVMTypeOf(v);
+       if (LLVMGetTypeKind(type) == LLVMPointerTypeKind) {
+               return LLVMBuildPtrToInt(ctx->builder, v, ac_to_integer_type(ctx, type), "");
+       }
        return LLVMBuildBitCast(ctx->builder, v, ac_to_integer_type(ctx, type), "");
 }
 
@@ -535,10 +548,11 @@ ac_build_gather_values(struct ac_llvm_context *ctx,
 /* Expand a scalar or vector to <dst_channels x type> by filling the remaining
  * channels with undef. Extract at most src_channels components from the input.
  */
-LLVMValueRef ac_build_expand(struct ac_llvm_context *ctx,
-                            LLVMValueRef value,
-                            unsigned src_channels,
-                            unsigned dst_channels)
+static LLVMValueRef
+ac_build_expand(struct ac_llvm_context *ctx,
+               LLVMValueRef value,
+               unsigned src_channels,
+               unsigned dst_channels)
 {
        LLVMTypeRef elemtype;
        LLVMValueRef chan[dst_channels];
@@ -606,7 +620,7 @@ ac_build_fdiv(struct ac_llvm_context *ctx,
         * If we do (num * (1 / den)), LLVM does:
         *    return num * v_rcp_f32(den);
         */
-       LLVMValueRef one = LLVMTypeOf(num) == ctx->f64 ? ctx->f64_1 : ctx->f32_1;
+       LLVMValueRef one = LLVMConstReal(LLVMTypeOf(num), 1.0);
        LLVMValueRef rcp = LLVMBuildFDiv(ctx->builder, one, den, "");
        LLVMValueRef ret = LLVMBuildFMul(ctx->builder, num, rcp, "");
 
@@ -905,6 +919,37 @@ ac_build_fs_interp(struct ac_llvm_context *ctx,
                                  ctx->f32, args, 5, AC_FUNC_ATTR_READNONE);
 }
 
+LLVMValueRef
+ac_build_fs_interp_f16(struct ac_llvm_context *ctx,
+                      LLVMValueRef llvm_chan,
+                      LLVMValueRef attr_number,
+                      LLVMValueRef params,
+                      LLVMValueRef i,
+                      LLVMValueRef j)
+{
+       LLVMValueRef args[6];
+       LLVMValueRef p1;
+
+       args[0] = i;
+       args[1] = llvm_chan;
+       args[2] = attr_number;
+       args[3] = ctx->i1false;
+       args[4] = params;
+
+       p1 = ac_build_intrinsic(ctx, "llvm.amdgcn.interp.p1.f16",
+                               ctx->f32, args, 5, AC_FUNC_ATTR_READNONE);
+
+       args[0] = p1;
+       args[1] = j;
+       args[2] = llvm_chan;
+       args[3] = attr_number;
+       args[4] = ctx->i1false;
+       args[5] = params;
+
+       return ac_build_intrinsic(ctx, "llvm.amdgcn.interp.p2.f16",
+                                 ctx->f16, args, 6, AC_FUNC_ATTR_READNONE);
+}
+
 LLVMValueRef
 ac_build_fs_interp_mov(struct ac_llvm_context *ctx,
                       LLVMValueRef parameter,
@@ -923,6 +968,14 @@ ac_build_fs_interp_mov(struct ac_llvm_context *ctx,
                                  ctx->f32, args, 4, AC_FUNC_ATTR_READNONE);
 }
 
+LLVMValueRef
+ac_build_gep_ptr(struct ac_llvm_context *ctx,
+                LLVMValueRef base_ptr,
+                LLVMValueRef index)
+{
+       return LLVMBuildGEP(ctx->builder, base_ptr, &index, 1, "");
+}
+
 LLVMValueRef
 ac_build_gep0(struct ac_llvm_context *ctx,
              LLVMValueRef base_ptr,
@@ -1029,6 +1082,106 @@ LLVMValueRef ac_build_load_to_sgpr_uint_wraparound(struct ac_llvm_context *ctx,
        return ac_build_load_custom(ctx, base_ptr, index, true, true, false);
 }
 
+static void
+ac_build_buffer_store_common(struct ac_llvm_context *ctx,
+                            LLVMValueRef rsrc,
+                            LLVMValueRef data,
+                            LLVMValueRef vindex,
+                            LLVMValueRef voffset,
+                            unsigned num_channels,
+                            bool glc,
+                            bool slc,
+                            bool writeonly_memory,
+                            bool use_format)
+{
+       LLVMValueRef args[] = {
+               data,
+               LLVMBuildBitCast(ctx->builder, rsrc, ctx->v4i32, ""),
+               vindex ? vindex : ctx->i32_0,
+               voffset,
+               LLVMConstInt(ctx->i1, glc, 0),
+               LLVMConstInt(ctx->i1, slc, 0)
+       };
+       unsigned func = CLAMP(num_channels, 1, 3) - 1;
+
+       const char *type_names[] = {"f32", "v2f32", "v4f32"};
+       char name[256];
+
+       if (use_format) {
+               snprintf(name, sizeof(name), "llvm.amdgcn.buffer.store.format.%s",
+                        type_names[func]);
+       } else {
+               snprintf(name, sizeof(name), "llvm.amdgcn.buffer.store.%s",
+                        type_names[func]);
+       }
+
+       ac_build_intrinsic(ctx, name, ctx->voidt, args, ARRAY_SIZE(args),
+                          ac_get_store_intr_attribs(writeonly_memory));
+}
+
+static void
+ac_build_llvm8_buffer_store_common(struct ac_llvm_context *ctx,
+                                  LLVMValueRef rsrc,
+                                  LLVMValueRef data,
+                                  LLVMValueRef vindex,
+                                  LLVMValueRef voffset,
+                                  LLVMValueRef soffset,
+                                  unsigned num_channels,
+                                  bool glc,
+                                  bool slc,
+                                  bool writeonly_memory,
+                                  bool use_format,
+                                  bool structurized)
+{
+       LLVMValueRef args[6];
+       int idx = 0;
+       args[idx++] = data;
+       args[idx++] = LLVMBuildBitCast(ctx->builder, rsrc, ctx->v4i32, "");
+       if (structurized)
+               args[idx++] = vindex ? vindex : ctx->i32_0;
+       args[idx++] = voffset ? voffset : ctx->i32_0;
+       args[idx++] = soffset ? soffset : ctx->i32_0;
+       args[idx++] = LLVMConstInt(ctx->i32, (glc ? 1 : 0) + (slc ? 2 : 0), 0);
+       unsigned func = CLAMP(num_channels, 1, 3) - 1;
+
+       const char *type_names[] = {"f32", "v2f32", "v4f32"};
+       const char *indexing_kind = structurized ? "struct" : "raw";
+       char name[256];
+
+       if (use_format) {
+               snprintf(name, sizeof(name), "llvm.amdgcn.%s.buffer.store.format.%s",
+                        indexing_kind, type_names[func]);
+       } else {
+               snprintf(name, sizeof(name), "llvm.amdgcn.%s.buffer.store.%s",
+                        indexing_kind, type_names[func]);
+       }
+
+       ac_build_intrinsic(ctx, name, ctx->voidt, args, idx,
+                          ac_get_store_intr_attribs(writeonly_memory));
+}
+
+void
+ac_build_buffer_store_format(struct ac_llvm_context *ctx,
+                            LLVMValueRef rsrc,
+                            LLVMValueRef data,
+                            LLVMValueRef vindex,
+                            LLVMValueRef voffset,
+                            unsigned num_channels,
+                            bool glc,
+                            bool writeonly_memory)
+{
+       if (HAVE_LLVM >= 0x800) {
+               ac_build_llvm8_buffer_store_common(ctx, rsrc, data, vindex,
+                                                  voffset, NULL, num_channels,
+                                                  glc, false, writeonly_memory,
+                                                  true, true);
+       } else {
+               ac_build_buffer_store_common(ctx, rsrc, data, vindex, voffset,
+                                            num_channels, glc, false,
+                                            writeonly_memory, true);
+       }
+}
+
 /* TBUFFER_STORE_FORMAT_{X,XY,XYZ,XYZW} <- the suffix is selected by num_channels=1..4.
  * The type of vdata must be one of i32 (num_channels=1), v2i32 (num_channels=2),
  * or v4i32 (num_channels=3,4).
@@ -1097,9 +1250,7 @@ ac_build_buffer_store_dword(struct ac_llvm_context *ctx,
 
                ac_build_intrinsic(ctx, name, ctx->voidt,
                                   args, ARRAY_SIZE(args),
-                                  writeonly_memory ?
-                                  AC_FUNC_ATTR_INACCESSIBLE_MEM_ONLY :
-                                  AC_FUNC_ATTR_WRITEONLY);
+                                  ac_get_store_intr_attribs(writeonly_memory));
                return;
        }
 
@@ -1128,9 +1279,7 @@ ac_build_buffer_store_dword(struct ac_llvm_context *ctx,
 
        ac_build_intrinsic(ctx, name, ctx->voidt,
                           args, ARRAY_SIZE(args),
-                          writeonly_memory ?
-                                  AC_FUNC_ATTR_INACCESSIBLE_MEM_ONLY :
-                                  AC_FUNC_ATTR_WRITEONLY);
+                          ac_get_store_intr_attribs(writeonly_memory));
 }
 
 static LLVMValueRef
@@ -1230,8 +1379,8 @@ ac_build_buffer_load(struct ac_llvm_context *ctx,
        if (soffset)
                offset = LLVMBuildAdd(ctx->builder, offset, soffset, "");
 
-       /* TODO: VI and later generations can use SMEM with GLC=1.*/
-       if (allow_smem && !glc && !slc) {
+       if (allow_smem && !slc &&
+           (!glc || (HAVE_LLVM >= 0x0800 && ctx->chip_class >= VI))) {
                assert(vindex == NULL);
 
                LLVMValueRef result[8];
@@ -1241,11 +1390,19 @@ ac_build_buffer_load(struct ac_llvm_context *ctx,
                                offset = LLVMBuildAdd(ctx->builder, offset,
                                                      LLVMConstInt(ctx->i32, 4, 0), "");
                        }
-                       LLVMValueRef args[2] = {rsrc, offset};
-                       result[i] = ac_build_intrinsic(ctx, "llvm.SI.load.const.v4i32",
-                                                      ctx->f32, args, 2,
+                       const char *intrname =
+                               HAVE_LLVM >= 0x0800 ? "llvm.amdgcn.s.buffer.load.f32"
+                                                   : "llvm.SI.load.const.v4i32";
+                       unsigned num_args = HAVE_LLVM >= 0x0800 ? 3 : 2;
+                       LLVMValueRef args[3] = {
+                               rsrc,
+                               offset,
+                               glc ? ctx->i32_1 : ctx->i32_0,
+                       };
+                       result[i] = ac_build_intrinsic(ctx, intrname,
+                                                      ctx->f32, args, num_args,
                                                       AC_FUNC_ATTR_READNONE |
-                                                      AC_FUNC_ATTR_LEGACY);
+                                                      (HAVE_LLVM < 0x0800 ? AC_FUNC_ATTR_LEGACY : 0));
                }
                if (num_channels == 1)
                        return result[0];
@@ -1255,6 +1412,14 @@ ac_build_buffer_load(struct ac_llvm_context *ctx,
                return ac_build_gather_values(ctx, result, num_channels);
        }
 
+       if (HAVE_LLVM >= 0x0800) {
+               return ac_build_llvm8_buffer_load_common(ctx, rsrc, vindex,
+                                                        offset, ctx->i32_0,
+                                                        num_channels, glc, slc,
+                                                        can_speculate, false,
+                                                        false);
+       }
+
        return ac_build_buffer_load_common(ctx, rsrc, vindex, offset,
                                           num_channels, glc, slc,
                                           can_speculate, false);
@@ -1308,32 +1473,294 @@ LLVMValueRef ac_build_buffer_load_format_gfx9_safe(struct ac_llvm_context *ctx,
                                           can_speculate, true);
 }
 
-LLVMValueRef
-ac_build_tbuffer_load_short(struct ac_llvm_context *ctx,
+static LLVMValueRef
+ac_build_llvm8_tbuffer_load(struct ac_llvm_context *ctx,
                            LLVMValueRef rsrc,
                            LLVMValueRef vindex,
                            LLVMValueRef voffset,
-                               LLVMValueRef soffset,
-                               LLVMValueRef immoffset,
-                               LLVMValueRef glc)
+                           LLVMValueRef soffset,
+                           unsigned num_channels,
+                           unsigned dfmt,
+                           unsigned nfmt,
+                           bool glc,
+                           bool slc,
+                           bool can_speculate,
+                           bool structurized)
 {
-       const char *name = "llvm.amdgcn.tbuffer.load.i32";
-       LLVMTypeRef type = ctx->i32;
-       LLVMValueRef params[] = {
-                               rsrc,
-                               vindex,
-                               voffset,
-                               soffset,
-                               immoffset,
-                               LLVMConstInt(ctx->i32, V_008F0C_BUF_DATA_FORMAT_16, false),
-                               LLVMConstInt(ctx->i32, V_008F0C_BUF_NUM_FORMAT_UINT, false),
-                               glc,
-                               ctx->i1false,
+       LLVMValueRef args[6];
+       int idx = 0;
+       args[idx++] = LLVMBuildBitCast(ctx->builder, rsrc, ctx->v4i32, "");
+       if (structurized)
+               args[idx++] = vindex ? vindex : ctx->i32_0;
+       args[idx++] = voffset ? voffset : ctx->i32_0;
+       args[idx++] = soffset ? soffset : ctx->i32_0;
+       args[idx++] = LLVMConstInt(ctx->i32, dfmt | (nfmt << 4), 0);
+       args[idx++] = LLVMConstInt(ctx->i32, (glc ? 1 : 0) + (slc ? 2 : 0), 0);
+       unsigned func = CLAMP(num_channels, 1, 3) - 1;
+
+       LLVMTypeRef types[] = {ctx->i32, ctx->v2i32, ctx->v4i32};
+       const char *type_names[] = {"i32", "v2i32", "v4i32"};
+       const char *indexing_kind = structurized ? "struct" : "raw";
+       char name[256];
+
+       snprintf(name, sizeof(name), "llvm.amdgcn.%s.tbuffer.load.%s",
+                indexing_kind, type_names[func]);
+
+       return ac_build_intrinsic(ctx, name, types[func], args,
+                                 idx,
+                                 ac_get_load_intr_attribs(can_speculate));
+}
+
+static LLVMValueRef
+ac_build_tbuffer_load(struct ac_llvm_context *ctx,
+                           LLVMValueRef rsrc,
+                           LLVMValueRef vindex,
+                           LLVMValueRef voffset,
+                           LLVMValueRef soffset,
+                           LLVMValueRef immoffset,
+                           unsigned num_channels,
+                           unsigned dfmt,
+                           unsigned nfmt,
+                           bool glc,
+                           bool slc,
+                           bool can_speculate,
+                           bool structurized) /* only matters for LLVM 8+ */
+{
+       if (HAVE_LLVM >= 0x800) {
+               voffset = LLVMBuildAdd(ctx->builder, voffset, immoffset, "");
+
+               return ac_build_llvm8_tbuffer_load(ctx, rsrc, vindex, voffset,
+                                                  soffset, num_channels,
+                                                  dfmt, nfmt, glc, slc,
+                                                  can_speculate, structurized);
+       }
+
+       LLVMValueRef args[] = {
+               rsrc,
+               vindex ? vindex : ctx->i32_0,
+               voffset,
+               soffset,
+               immoffset,
+               LLVMConstInt(ctx->i32, dfmt, false),
+               LLVMConstInt(ctx->i32, nfmt, false),
+               LLVMConstInt(ctx->i32, glc, false),
+               LLVMConstInt(ctx->i32, slc, false),
        };
-       LLVMValueRef res = ac_build_intrinsic(ctx, name, type, params, 9, 0);
+       unsigned func = CLAMP(num_channels, 1, 3) - 1;
+       LLVMTypeRef types[] = {ctx->i32, ctx->v2i32, ctx->v4i32};
+       const char *type_names[] = {"i32", "v2i32", "v4i32"};
+       char name[256];
+
+       snprintf(name, sizeof(name), "llvm.amdgcn.tbuffer.load.%s",
+                type_names[func]);
+
+       return ac_build_intrinsic(ctx, name, types[func], args, 9,
+                                 ac_get_load_intr_attribs(can_speculate));
+}
+
+LLVMValueRef
+ac_build_struct_tbuffer_load(struct ac_llvm_context *ctx,
+                            LLVMValueRef rsrc,
+                            LLVMValueRef vindex,
+                            LLVMValueRef voffset,
+                            LLVMValueRef soffset,
+                            LLVMValueRef immoffset,
+                            unsigned num_channels,
+                            unsigned dfmt,
+                            unsigned nfmt,
+                            bool glc,
+                            bool slc,
+                            bool can_speculate)
+{
+       return ac_build_tbuffer_load(ctx, rsrc, vindex, voffset, soffset,
+                                    immoffset, num_channels, dfmt, nfmt, glc,
+                                    slc, can_speculate, true);
+}
+
+LLVMValueRef
+ac_build_raw_tbuffer_load(struct ac_llvm_context *ctx,
+                         LLVMValueRef rsrc,
+                         LLVMValueRef voffset,
+                         LLVMValueRef soffset,
+                         LLVMValueRef immoffset,
+                         unsigned num_channels,
+                         unsigned dfmt,
+                         unsigned nfmt,
+                         bool glc,
+                         bool slc,
+                         bool can_speculate)
+{
+       return ac_build_tbuffer_load(ctx, rsrc, NULL, voffset, soffset,
+                                    immoffset, num_channels, dfmt, nfmt, glc,
+                                    slc, can_speculate, false);
+}
+
+LLVMValueRef
+ac_build_tbuffer_load_short(struct ac_llvm_context *ctx,
+                           LLVMValueRef rsrc,
+                           LLVMValueRef voffset,
+                           LLVMValueRef soffset,
+                           LLVMValueRef immoffset,
+                           bool glc)
+{
+       unsigned dfmt = V_008F0C_BUF_DATA_FORMAT_16;
+       unsigned nfmt = V_008F0C_BUF_NUM_FORMAT_UINT;
+       LLVMValueRef res;
+
+       res = ac_build_raw_tbuffer_load(ctx, rsrc, voffset, soffset,
+                                       immoffset, 1, dfmt, nfmt, glc, false,
+                                       false);
+
        return LLVMBuildTrunc(ctx->builder, res, ctx->i16, "");
 }
 
+static void
+ac_build_llvm8_tbuffer_store(struct ac_llvm_context *ctx,
+                            LLVMValueRef rsrc,
+                            LLVMValueRef vdata,
+                            LLVMValueRef vindex,
+                            LLVMValueRef voffset,
+                            LLVMValueRef soffset,
+                            unsigned num_channels,
+                            unsigned dfmt,
+                            unsigned nfmt,
+                            bool glc,
+                            bool slc,
+                            bool writeonly_memory,
+                            bool structurized)
+{
+       LLVMValueRef args[7];
+       int idx = 0;
+       args[idx++] = vdata;
+       args[idx++] = LLVMBuildBitCast(ctx->builder, rsrc, ctx->v4i32, "");
+       if (structurized)
+               args[idx++] = vindex ? vindex : ctx->i32_0;
+       args[idx++] = voffset ? voffset : ctx->i32_0;
+       args[idx++] = soffset ? soffset : ctx->i32_0;
+       args[idx++] = LLVMConstInt(ctx->i32, dfmt | (nfmt << 4), 0);
+       args[idx++] = LLVMConstInt(ctx->i32, (glc ? 1 : 0) + (slc ? 2 : 0), 0);
+       unsigned func = CLAMP(num_channels, 1, 3) - 1;
+
+       const char *type_names[] = {"i32", "v2i32", "v4i32"};
+       const char *indexing_kind = structurized ? "struct" : "raw";
+       char name[256];
+
+       snprintf(name, sizeof(name), "llvm.amdgcn.%s.tbuffer.store.%s",
+                indexing_kind, type_names[func]);
+
+       ac_build_intrinsic(ctx, name, ctx->voidt, args, idx,
+                          ac_get_store_intr_attribs(writeonly_memory));
+}
+
+static void
+ac_build_tbuffer_store(struct ac_llvm_context *ctx,
+                      LLVMValueRef rsrc,
+                      LLVMValueRef vdata,
+                      LLVMValueRef vindex,
+                      LLVMValueRef voffset,
+                      LLVMValueRef soffset,
+                      LLVMValueRef immoffset,
+                      unsigned num_channels,
+                      unsigned dfmt,
+                      unsigned nfmt,
+                      bool glc,
+                      bool slc,
+                      bool writeonly_memory,
+                      bool structurized) /* only matters for LLVM 8+ */
+{
+       if (HAVE_LLVM >= 0x800) {
+               voffset = LLVMBuildAdd(ctx->builder,
+                                      voffset ? voffset : ctx->i32_0,
+                                      immoffset, "");
+
+               ac_build_llvm8_tbuffer_store(ctx, rsrc, vdata, vindex, voffset,
+                                            soffset, num_channels, dfmt, nfmt,
+                                            glc, slc, writeonly_memory,
+                                            structurized);
+       } else {
+               LLVMValueRef params[] = {
+                       vdata,
+                       rsrc,
+                       vindex ? vindex : ctx->i32_0,
+                       voffset ? voffset : ctx->i32_0,
+                       soffset ? soffset : ctx->i32_0,
+                       immoffset,
+                       LLVMConstInt(ctx->i32, dfmt, false),
+                       LLVMConstInt(ctx->i32, nfmt, false),
+                       LLVMConstInt(ctx->i32, glc, false),
+                       LLVMConstInt(ctx->i32, slc, false),
+               };
+               unsigned func = CLAMP(num_channels, 1, 3) - 1;
+               const char *type_names[] = {"i32", "v2i32", "v4i32"};
+               char name[256];
+
+               snprintf(name, sizeof(name), "llvm.amdgcn.tbuffer.store.%s",
+                        type_names[func]);
+
+               ac_build_intrinsic(ctx, name, ctx->voidt, params, 10,
+                                  ac_get_store_intr_attribs(writeonly_memory));
+       }
+}
+
+void
+ac_build_struct_tbuffer_store(struct ac_llvm_context *ctx,
+                             LLVMValueRef rsrc,
+                             LLVMValueRef vdata,
+                             LLVMValueRef vindex,
+                             LLVMValueRef voffset,
+                             LLVMValueRef soffset,
+                             LLVMValueRef immoffset,
+                             unsigned num_channels,
+                             unsigned dfmt,
+                             unsigned nfmt,
+                             bool glc,
+                             bool slc,
+                             bool writeonly_memory)
+{
+       ac_build_tbuffer_store(ctx, rsrc, vdata, vindex, voffset, soffset,
+                              immoffset, num_channels, dfmt, nfmt, glc, slc,
+                              writeonly_memory, true);
+}
+
+void
+ac_build_raw_tbuffer_store(struct ac_llvm_context *ctx,
+                          LLVMValueRef rsrc,
+                          LLVMValueRef vdata,
+                          LLVMValueRef voffset,
+                          LLVMValueRef soffset,
+                          LLVMValueRef immoffset,
+                          unsigned num_channels,
+                          unsigned dfmt,
+                          unsigned nfmt,
+                          bool glc,
+                          bool slc,
+                          bool writeonly_memory)
+{
+       ac_build_tbuffer_store(ctx, rsrc, vdata, NULL, voffset, soffset,
+                              immoffset, num_channels, dfmt, nfmt, glc, slc,
+                              writeonly_memory, false);
+}
+
+void
+ac_build_tbuffer_store_short(struct ac_llvm_context *ctx,
+                            LLVMValueRef rsrc,
+                            LLVMValueRef vdata,
+                            LLVMValueRef voffset,
+                            LLVMValueRef soffset,
+                            bool glc,
+                            bool writeonly_memory)
+{
+       unsigned dfmt = V_008F0C_BUF_DATA_FORMAT_16;
+       unsigned nfmt = V_008F0C_BUF_NUM_FORMAT_UINT;
+
+       vdata = LLVMBuildBitCast(ctx->builder, vdata, ctx->i16, "");
+       vdata = LLVMBuildZExt(ctx->builder, vdata, ctx->i32, "");
+
+       ac_build_raw_tbuffer_store(ctx, rsrc, vdata, voffset, soffset,
+                                  ctx->i32_0, 1, dfmt, nfmt, glc, false,
+                                  writeonly_memory);
+}
+
 /**
  * Set range metadata on an instruction.  This can only be used on load and
  * call instructions.  If you know an instruction can only produce the values
@@ -1523,16 +1950,20 @@ ac_build_umsb(struct ac_llvm_context *ctx,
 LLVMValueRef ac_build_fmin(struct ac_llvm_context *ctx, LLVMValueRef a,
                           LLVMValueRef b)
 {
+       char name[64];
+       snprintf(name, sizeof(name), "llvm.minnum.f%d", ac_get_elem_bits(ctx, LLVMTypeOf(a)));
        LLVMValueRef args[2] = {a, b};
-       return ac_build_intrinsic(ctx, "llvm.minnum.f32", ctx->f32, args, 2,
+       return ac_build_intrinsic(ctx, name, LLVMTypeOf(a), args, 2,
                                  AC_FUNC_ATTR_READNONE);
 }
 
 LLVMValueRef ac_build_fmax(struct ac_llvm_context *ctx, LLVMValueRef a,
                           LLVMValueRef b)
 {
+       char name[64];
+       snprintf(name, sizeof(name), "llvm.maxnum.f%d", ac_get_elem_bits(ctx, LLVMTypeOf(a)));
        LLVMValueRef args[2] = {a, b};
-       return ac_build_intrinsic(ctx, "llvm.maxnum.f32", ctx->f32, args, 2,
+       return ac_build_intrinsic(ctx, name, LLVMTypeOf(a), args, 2,
                                  AC_FUNC_ATTR_READNONE);
 }
 
@@ -1559,8 +1990,9 @@ LLVMValueRef ac_build_umin(struct ac_llvm_context *ctx, LLVMValueRef a,
 
 LLVMValueRef ac_build_clamp(struct ac_llvm_context *ctx, LLVMValueRef value)
 {
-       return ac_build_fmin(ctx, ac_build_fmax(ctx, value, ctx->f32_0),
-                            ctx->f32_1);
+       LLVMTypeRef t = LLVMTypeOf(value);
+       return ac_build_fmin(ctx, ac_build_fmax(ctx, value, LLVMConstReal(t, 0.0)),
+                            LLVMConstReal(t, 1.0));
 }
 
 void ac_build_export(struct ac_llvm_context *ctx, struct ac_export_args *a)
@@ -1974,48 +2406,28 @@ LLVMValueRef ac_build_fract(struct ac_llvm_context *ctx, LLVMValueRef src0,
        char *intr;
 
        if (bitsize == 32) {
-               intr = "llvm.floor.f32";
+               intr = "llvm.amdgcn.fract.f32";
                type = ctx->f32;
        } else {
-               intr = "llvm.floor.f64";
+               intr = "llvm.amdgcn.fract.f64";
                type = ctx->f64;
        }
 
        LLVMValueRef params[] = {
                src0,
        };
-       LLVMValueRef floor = ac_build_intrinsic(ctx, intr, type, params, 1,
-                                               AC_FUNC_ATTR_READNONE);
-       return LLVMBuildFSub(ctx->builder, src0, floor, "");
+       return ac_build_intrinsic(ctx, intr, type, params, 1,
+                                 AC_FUNC_ATTR_READNONE);
 }
 
 LLVMValueRef ac_build_isign(struct ac_llvm_context *ctx, LLVMValueRef src0,
                            unsigned bitsize)
 {
-       LLVMValueRef cmp, val, zero, one;
-       LLVMTypeRef type;
-
-       switch (bitsize) {
-       case 64:
-               type = ctx->i64;
-               zero = ctx->i64_0;
-               one = ctx->i64_1;
-               break;
-       case 32:
-               type = ctx->i32;
-               zero = ctx->i32_0;
-               one = ctx->i32_1;
-               break;
-       case 16:
-               type = ctx->i16;
-               zero = ctx->i16_0;
-               one = ctx->i16_1;
-               break;
-       default:
-               unreachable(!"invalid bitsize");
-               break;
-       }
+       LLVMTypeRef type = LLVMIntTypeInContext(ctx->context, bitsize);
+       LLVMValueRef zero = LLVMConstInt(type, 0, false);
+       LLVMValueRef one = LLVMConstInt(type, 1, false);
 
+       LLVMValueRef cmp, val;
        cmp = LLVMBuildICmp(ctx->builder, LLVMIntSGT, src0, zero, "");
        val = LLVMBuildSelect(ctx->builder, cmp, one, src0, "");
        cmp = LLVMBuildICmp(ctx->builder, LLVMIntSGE, val, zero, "");