anv,turnip,radv,clover,glspirv: Run nir_copy_prop before nir_opt_deref

[mesa.git] / src / amd / vulkan / radv_shader.c

diff --git a/src/amd/vulkan/radv_shader.c b/src/amd/vulkan/radv_shader.c
index f928ad2be4ce4018c2bf1c4888e7b5f4d038071f..11f8fbec7cf46ea425648adeba3739f755f195b9 100644 (file)
--- a/src/amd/vulkan/radv_shader.c
+++ b/src/amd/vulkan/radv_shader.c
@@ -86,6 +86,10 @@ static const struct nir_shader_compiler_options nir_options_llvm = {
                                nir_lower_divmod64 |
                                nir_lower_minmax64 |
                                nir_lower_iabs64,
+       .lower_doubles_options = nir_lower_drcp |
+                                nir_lower_dsqrt |
+                                nir_lower_drsq |
+                                nir_lower_ddiv,
 };
 
 static const struct nir_shader_compiler_options nir_options_aco = {
@@ -122,6 +126,10 @@ static const struct nir_shader_compiler_options nir_options_aco = {
                                nir_lower_divmod64 |
                                nir_lower_minmax64 |
                                nir_lower_iabs64,
+       .lower_doubles_options = nir_lower_drcp |
+                                nir_lower_dsqrt |
+                                nir_lower_drsq |
+                                nir_lower_ddiv,
 };
 
 bool
@@ -301,8 +309,8 @@ radv_shader_compile_to_nir(struct radv_device *device,
 {
        nir_shader *nir;
        const nir_shader_compiler_options *nir_options =
-               device->physical_device->use_aco ? &nir_options_aco :
-                                                  &nir_options_llvm;
+               device->physical_device->use_llvm ? &nir_options_llvm :
+                                                   &nir_options_aco;
 
        if (module->nir) {
                /* Some things such as our meta clear/blit code will give us a NIR
@@ -357,7 +365,7 @@ radv_shader_compile_to_nir(struct radv_device *device,
                                .amd_gcn_shader = true,
                                .amd_image_gather_bias_lod = true,
                                .amd_image_read_write_lod = true,
-                               .amd_shader_ballot = device->physical_device->use_shader_ballot,
+                               .amd_shader_ballot = true,
                                .amd_shader_explicit_vertex_parameter = true,
                                .amd_trinary_minmax = true,
                                .demote_to_helper_invocation = true,
@@ -369,6 +377,7 @@ radv_shader_compile_to_nir(struct radv_device *device,
                                .draw_parameters = true,
                                .float_controls = true,
                                .float16 = device->physical_device->rad_info.has_packed_math_16bit,
+                               .float32_atomic_add = true,
                                .float64 = true,
                                .geometry_streams = true,
                                .image_ms_array = true,
@@ -422,6 +431,7 @@ radv_shader_compile_to_nir(struct radv_device *device,
                NIR_PASS_V(nir, nir_lower_variable_initializers, nir_var_function_temp);
                NIR_PASS_V(nir, nir_lower_returns);
                NIR_PASS_V(nir, nir_inline_functions);
+               NIR_PASS_V(nir, nir_copy_prop);
                NIR_PASS_V(nir, nir_opt_deref);
 
                /* Pick off the single entrypoint that we want */
@@ -450,7 +460,7 @@ radv_shader_compile_to_nir(struct radv_device *device,
                NIR_PASS_V(nir, nir_split_per_member_structs);
 
                if (nir->info.stage == MESA_SHADER_FRAGMENT &&
-                   device->physical_device->use_aco)
+                   !device->physical_device->use_llvm)
                         NIR_PASS_V(nir, nir_lower_io_to_vector, nir_var_shader_out);
                if (nir->info.stage == MESA_SHADER_FRAGMENT)
                        NIR_PASS_V(nir, nir_lower_input_attachments, true);
@@ -466,6 +476,19 @@ radv_shader_compile_to_nir(struct radv_device *device,
                NIR_PASS_V(nir, radv_nir_lower_ycbcr_textures, layout);
                if (device->instance->debug_flags & RADV_DEBUG_DISCARD_TO_DEMOTE)
                        NIR_PASS_V(nir, nir_lower_discard_to_demote);
+
+               nir_lower_doubles_options lower_doubles =
+                       nir->options->lower_doubles_options;
+
+               if (device->physical_device->rad_info.chip_class == GFX6) {
+                       /* GFX6 doesn't support v_floor_f64 and the precision
+                        * of v_fract_f64 which is used to implement 64-bit
+                        * floor is less than what Vulkan requires.
+                        */
+                       lower_doubles |= nir_lower_dfloor;
+               }
+
+               NIR_PASS_V(nir, nir_lower_doubles, NULL, lower_doubles);
        }
 
        /* Vulkan uses the separate-shader linking model */
@@ -510,6 +533,7 @@ radv_shader_compile_to_nir(struct radv_device *device,
                        .lower_vote_eq_to_ballot = 1,
                        .lower_quad_broadcast_dynamic = 1,
                        .lower_quad_broadcast_dynamic_to_const = gfx7minus,
+                       .lower_shuffle_to_swizzle_amd = 1,
                });
 
        nir_lower_load_const_to_scalar(nir);
@@ -631,7 +655,7 @@ radv_lower_fs_io(nir_shader *nir)
 }
 
 
-void *
+static void *
 radv_alloc_shader_memory(struct radv_device *device,
                         struct radv_shader_variant *shader)
 {
@@ -667,7 +691,18 @@ radv_alloc_shader_memory(struct radv_device *device,
                                             (device->physical_device->rad_info.cpdma_prefetch_writes_memory ?
                                                     0 : RADEON_FLAG_READ_ONLY),
                                             RADV_BO_PRIORITY_SHADER);
+       if (!slab->bo) {
+               free(slab);
+               return NULL;
+       }
+
        slab->ptr = (char*)device->ws->buffer_map(slab->bo);
+       if (!slab->ptr) {
+               device->ws->buffer_destroy(slab->bo);
+               free(slab);
+               return NULL;
+       }
+
        list_inithead(&slab->shaders);
 
        mtx_lock(&device->shader_slab_mutex);
@@ -1004,15 +1039,14 @@ radv_shader_variant_create(struct radv_device *device,
        radv_postprocess_config(device->physical_device, &config, &binary->info,
                                binary->stage, &variant->config);
 
-       if (radv_device_use_secure_compile(device->instance)) {
+       void *dest_ptr = radv_alloc_shader_memory(device, variant);
+       if (!dest_ptr) {
                if (binary->type == RADV_BINARY_TYPE_RTLD)
                        ac_rtld_close(&rtld_binary);
-
-               return variant;
+               free(variant);
+               return NULL;
        }
 
-       void *dest_ptr = radv_alloc_shader_memory(device, variant);
-
        if (binary->type == RADV_BINARY_TYPE_RTLD) {
                struct radv_shader_binary_rtld* bin = (struct radv_shader_binary_rtld *)binary;
                struct ac_rtld_upload_info info = {
@@ -1128,14 +1162,14 @@ shader_variant_compile(struct radv_device *device,
                                 shader_count >= 2 ? shaders[shader_count - 2]->info.stage
                                                   : MESA_SHADER_VERTEX);
 
-       if (!device->physical_device->use_aco ||
+       if (device->physical_device->use_llvm ||
            options->dump_shader || options->record_ir)
                ac_init_llvm_once();
 
-       if (device->physical_device->use_aco) {
-               aco_compile_shader(shader_count, shaders, &binary, &args);
-       } else {
+       if (device->physical_device->use_llvm) {
                llvm_compile_shader(device, shader_count, shaders, &binary, &args);
+       } else {
+               aco_compile_shader(shader_count, shaders, &binary, &args);
        }
 
        binary->info = *info;
@@ -1196,7 +1230,7 @@ radv_shader_variant_compile(struct radv_device *device,
        if (key)
                options.key = *key;
 
-       options.explicit_scratch_args = device->physical_device->use_aco;
+       options.explicit_scratch_args = !device->physical_device->use_llvm;
        options.robust_buffer_access = device->robust_buffer_access;
 
        return shader_variant_compile(device, module, shaders, shader_count, shaders[shader_count - 1]->info.stage, info,
@@ -1213,7 +1247,7 @@ radv_create_gs_copy_shader(struct radv_device *device,
 {
        struct radv_nir_compiler_options options = {0};
 
-       options.explicit_scratch_args = device->physical_device->use_aco;
+       options.explicit_scratch_args = !device->physical_device->use_llvm;
        options.key.has_multiview_view_index = multiview;
 
        return shader_variant_compile(device, NULL, &shader, 1, MESA_SHADER_VERTEX,