ac/nir: fix coherent global loads/stores

Signed-off-by: Rhys Perry <pendingchaos02@gmail.com>
Reviewed-by: Daniel Schürmann <daniel@schuermann.dev>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/6063>

diff --git a/src/amd/llvm/ac_nir_to_llvm.c b/src/amd/llvm/ac_nir_to_llvm.c
index 8c316e417e91ff3a0a1cf83d456f5114811b1978..1b0bdffdb4576c7cbfec31d2dbe8afa209b77e75 100644 (file)
--- a/src/amd/llvm/ac_nir_to_llvm.c
+++ b/src/amd/llvm/ac_nir_to_llvm.c
@@ -2418,6 +2418,9 @@ static LLVMValueRef visit_load_var(struct ac_nir_context *ctx,
                                LLVMValueRef offset = LLVMConstInt(ctx->ac.i32, i * stride / natural_stride, 0);
                                values[i] = LLVMBuildLoad(ctx->ac.builder,
                                                          ac_build_gep_ptr(&ctx->ac, address, offset), "");
                                LLVMValueRef offset = LLVMConstInt(ctx->ac.i32, i * stride / natural_stride, 0);
                                values[i] = LLVMBuildLoad(ctx->ac.builder,
                                                          ac_build_gep_ptr(&ctx->ac, address, offset), "");

+
+                               if (nir_intrinsic_access(instr) & (ACCESS_COHERENT | ACCESS_VOLATILE))
+                                       LLVMSetOrdering(values[i], LLVMAtomicOrderingMonotonic);

                        }
                        return ac_build_gather_values(&ctx->ac, values, instr->dest.ssa.num_components);
                } else {
                        }
                        return ac_build_gather_values(&ctx->ac, values, instr->dest.ssa.num_components);
                } else {


@@ -2425,6 +2428,9 @@ static LLVMValueRef visit_load_var(struct ac_nir_context *ctx,
                                                                LLVMGetPointerAddressSpace(LLVMTypeOf(address)));
                        address = LLVMBuildBitCast(ctx->ac.builder, address, ptr_type , "");
                        LLVMValueRef val = LLVMBuildLoad(ctx->ac.builder, address, "");
                                                                LLVMGetPointerAddressSpace(LLVMTypeOf(address)));
                        address = LLVMBuildBitCast(ctx->ac.builder, address, ptr_type , "");
                        LLVMValueRef val = LLVMBuildLoad(ctx->ac.builder, address, "");

+
+                       if (nir_intrinsic_access(instr) & (ACCESS_COHERENT | ACCESS_VOLATILE))
+                               LLVMSetOrdering(val, LLVMAtomicOrderingMonotonic);

                        return val;
                }
        }
                        return val;
                }
        }


@@ -2580,7 +2586,10 @@ visit_store_var(struct ac_nir_context *ctx,
 
                        val = LLVMBuildBitCast(ctx->ac.builder, val,
                                               LLVMGetElementType(LLVMTypeOf(address)), "");
 
                        val = LLVMBuildBitCast(ctx->ac.builder, val,
                                               LLVMGetElementType(LLVMTypeOf(address)), "");

-                       LLVMBuildStore(ctx->ac.builder, val, address);
+                       LLVMValueRef store = LLVMBuildStore(ctx->ac.builder, val, address);
+
+                       if (nir_intrinsic_access(instr) & (ACCESS_COHERENT | ACCESS_VOLATILE))
+                               LLVMSetOrdering(store, LLVMAtomicOrderingMonotonic);

                } else {
                        LLVMTypeRef val_type = LLVMTypeOf(val);
                        if (LLVMGetTypeKind(LLVMTypeOf(val)) == LLVMVectorTypeKind)
                } else {
                        LLVMTypeRef val_type = LLVMTypeOf(val);
                        if (LLVMGetTypeKind(LLVMTypeOf(val)) == LLVMVectorTypeKind)


@@ -2600,7 +2609,10 @@ visit_store_var(struct ac_nir_context *ctx,
                                                                        chan);
                                src = LLVMBuildBitCast(ctx->ac.builder, src,
                                                       LLVMGetElementType(LLVMTypeOf(ptr)), "");
                                                                        chan);
                                src = LLVMBuildBitCast(ctx->ac.builder, src,
                                                       LLVMGetElementType(LLVMTypeOf(ptr)), "");

-                               LLVMBuildStore(ctx->ac.builder, src, ptr);
+                               LLVMValueRef store = LLVMBuildStore(ctx->ac.builder, src, ptr);
+
+                               if (nir_intrinsic_access(instr) & (ACCESS_COHERENT | ACCESS_VOLATILE))
+                                       LLVMSetOrdering(store, LLVMAtomicOrderingMonotonic);

                        }
                }
                break;
                        }
                }
                break;