src/amd/vulkan/radv_meta.c

   1 /*
   2  * Copyright © 2016 Red Hat
   3  * based on intel anv code:
   4  * Copyright © 2015 Intel Corporation
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the next
  14  * paragraph) shall be included in all copies or substantial portions of the
  15  * Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  20  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  22  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  23  * IN THE SOFTWARE.
  24  */
  25
  26 #include "radv_meta.h"
  27
  28 #include <fcntl.h>
  29 #include <limits.h>
  30 #include <pwd.h>
  31 #include <sys/stat.h>
  32
  33 static void
  34 radv_meta_save_novertex(struct radv_meta_saved_state *state,
  35                const struct radv_cmd_buffer *cmd_buffer,
  36                uint32_t dynamic_mask)
  37 {
  38         state->old_pipeline = cmd_buffer->state.pipeline;
  39         state->old_descriptor_set0 = cmd_buffer->state.descriptors[0];
  40
  41         state->dynamic_mask = dynamic_mask;
  42         radv_dynamic_state_copy(&state->dynamic, &cmd_buffer->state.dynamic,
  43                                 dynamic_mask);
  44
  45         memcpy(state->push_constants, cmd_buffer->push_constants, MAX_PUSH_CONSTANTS_SIZE);
  46 }
  47
  48 void
  49 radv_meta_restore(const struct radv_meta_saved_state *state,
  50                   struct radv_cmd_buffer *cmd_buffer)
  51 {
  52         radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer), VK_PIPELINE_BIND_POINT_GRAPHICS,
  53                              radv_pipeline_to_handle(state->old_pipeline));
  54         cmd_buffer->state.descriptors[0] = state->old_descriptor_set0;
  55
  56         cmd_buffer->state.dirty |= RADV_CMD_DIRTY_PIPELINE;
  57
  58         radv_dynamic_state_copy(&cmd_buffer->state.dynamic, &state->dynamic,
  59                                 state->dynamic_mask);
  60         cmd_buffer->state.dirty |= state->dynamic_mask;
  61
  62         memcpy(cmd_buffer->push_constants, state->push_constants, MAX_PUSH_CONSTANTS_SIZE);
  63         cmd_buffer->push_constant_stages |= VK_SHADER_STAGE_ALL_GRAPHICS | VK_SHADER_STAGE_COMPUTE_BIT;
  64 }
  65
  66 void
  67 radv_meta_save_pass(struct radv_meta_saved_pass_state *state,
  68                     const struct radv_cmd_buffer *cmd_buffer)
  69 {
  70         state->pass = cmd_buffer->state.pass;
  71         state->subpass = cmd_buffer->state.subpass;
  72         state->framebuffer = cmd_buffer->state.framebuffer;
  73         state->attachments = cmd_buffer->state.attachments;
  74         state->render_area = cmd_buffer->state.render_area;
  75 }
  76
  77 void
  78 radv_meta_restore_pass(const struct radv_meta_saved_pass_state *state,
  79                        struct radv_cmd_buffer *cmd_buffer)
  80 {
  81         cmd_buffer->state.pass = state->pass;
  82         cmd_buffer->state.subpass = state->subpass;
  83         cmd_buffer->state.framebuffer = state->framebuffer;
  84         cmd_buffer->state.attachments = state->attachments;
  85         cmd_buffer->state.render_area = state->render_area;
  86         if (state->subpass)
  87                 radv_emit_framebuffer_state(cmd_buffer);
  88 }
  89
  90 void
  91 radv_meta_save_compute(struct radv_meta_saved_compute_state *state,
  92                        const struct radv_cmd_buffer *cmd_buffer,
  93                        unsigned push_constant_size)
  94 {
  95         state->old_pipeline = cmd_buffer->state.compute_pipeline;
  96         state->old_descriptor_set0 = cmd_buffer->state.descriptors[0];
  97
  98         if (push_constant_size)
  99                 memcpy(state->push_constants, cmd_buffer->push_constants, push_constant_size);
 100 }
 101
 102 void
 103 radv_meta_restore_compute(const struct radv_meta_saved_compute_state *state,
 104                           struct radv_cmd_buffer *cmd_buffer,
 105                           unsigned push_constant_size)
 106 {
 107         radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer), VK_PIPELINE_BIND_POINT_COMPUTE,
 108                              radv_pipeline_to_handle(state->old_pipeline));
 109
 110         cmd_buffer->state.descriptors[0] = state->old_descriptor_set0;
 111
 112         if (push_constant_size) {
 113                 memcpy(cmd_buffer->push_constants, state->push_constants, push_constant_size);
 114                 cmd_buffer->push_constant_stages |= VK_SHADER_STAGE_COMPUTE_BIT;
 115         }
 116 }
 117
 118 VkImageViewType
 119 radv_meta_get_view_type(const struct radv_image *image)
 120 {
 121         switch (image->type) {
 122         case VK_IMAGE_TYPE_1D: return VK_IMAGE_VIEW_TYPE_1D;
 123         case VK_IMAGE_TYPE_2D: return VK_IMAGE_VIEW_TYPE_2D;
 124         case VK_IMAGE_TYPE_3D: return VK_IMAGE_VIEW_TYPE_3D;
 125         default:
 126                 unreachable("bad VkImageViewType");
 127         }
 128 }
 129
 130 /**
 131  * When creating a destination VkImageView, this function provides the needed
 132  * VkImageViewCreateInfo::subresourceRange::baseArrayLayer.
 133  */
 134 uint32_t
 135 radv_meta_get_iview_layer(const struct radv_image *dest_image,
 136                           const VkImageSubresourceLayers *dest_subresource,
 137                           const VkOffset3D *dest_offset)
 138 {
 139         switch (dest_image->type) {
 140         case VK_IMAGE_TYPE_1D:
 141         case VK_IMAGE_TYPE_2D:
 142                 return dest_subresource->baseArrayLayer;
 143         case VK_IMAGE_TYPE_3D:
 144                 /* HACK: Vulkan does not allow attaching a 3D image to a framebuffer,
 145                  * but meta does it anyway. When doing so, we translate the
 146                  * destination's z offset into an array offset.
 147                  */
 148                 return dest_offset->z;
 149         default:
 150                 assert(!"bad VkImageType");
 151                 return 0;
 152         }
 153 }
 154
 155 static void *
 156 meta_alloc(void* _device, size_t size, size_t alignment,
 157            VkSystemAllocationScope allocationScope)
 158 {
 159         struct radv_device *device = _device;
 160         return device->alloc.pfnAllocation(device->alloc.pUserData, size, alignment,
 161                                            VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
 162 }
 163
 164 static void *
 165 meta_realloc(void* _device, void *original, size_t size, size_t alignment,
 166              VkSystemAllocationScope allocationScope)
 167 {
 168         struct radv_device *device = _device;
 169         return device->alloc.pfnReallocation(device->alloc.pUserData, original,
 170                                              size, alignment,
 171                                              VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
 172 }
 173
 174 static void
 175 meta_free(void* _device, void *data)
 176 {
 177         struct radv_device *device = _device;
 178         return device->alloc.pfnFree(device->alloc.pUserData, data);
 179 }
 180
 181 static bool
 182 radv_builtin_cache_path(char *path)
 183 {
 184         char *xdg_cache_home = getenv("XDG_CACHE_HOME");
 185         const char *suffix = "/radv_builtin_shaders";
 186         const char *suffix2 = "/.cache/radv_builtin_shaders";
 187         struct passwd pwd, *result;
 188         char path2[PATH_MAX + 1]; /* PATH_MAX is not a real max,but suffices here. */
 189
 190         if (xdg_cache_home) {
 191
 192                 if (strlen(xdg_cache_home) + strlen(suffix) > PATH_MAX)
 193                         return false;
 194
 195                 strcpy(path, xdg_cache_home);
 196                 strcat(path, suffix);
 197                 return true;
 198         }
 199
 200         getpwuid_r(getuid(), &pwd, path2, PATH_MAX - strlen(suffix2), &result);
 201         if (!result)
 202                 return false;
 203
 204         strcpy(path, pwd.pw_dir);
 205         strcat(path, "/.cache");
 206         mkdir(path, 0755);
 207
 208         strcat(path, suffix);
 209         return true;
 210 }
 211
 212 static void
 213 radv_load_meta_pipeline(struct radv_device *device)
 214 {
 215         char path[PATH_MAX + 1];
 216         struct stat st;
 217         void *data = NULL;
 218
 219         if (!radv_builtin_cache_path(path))
 220                 return;
 221
 222         int fd = open(path, O_RDONLY);
 223         if (fd < 0)
 224                 return;
 225         if (fstat(fd, &st))
 226                 goto fail;
 227         data = malloc(st.st_size);
 228         if (!data)
 229                 goto fail;
 230         if(read(fd, data, st.st_size) == -1)
 231                 goto fail;
 232
 233         radv_pipeline_cache_load(&device->meta_state.cache, data, st.st_size);
 234 fail:
 235         free(data);
 236         close(fd);
 237 }
 238
 239 static void
 240 radv_store_meta_pipeline(struct radv_device *device)
 241 {
 242         char path[PATH_MAX + 1], path2[PATH_MAX + 7];
 243         size_t size;
 244         void *data = NULL;
 245
 246         if (!device->meta_state.cache.modified)
 247                 return;
 248
 249         if (radv_GetPipelineCacheData(radv_device_to_handle(device),
 250                                       radv_pipeline_cache_to_handle(&device->meta_state.cache),
 251                                       &size, NULL))
 252                 return;
 253
 254         if (!radv_builtin_cache_path(path))
 255                 return;
 256
 257         strcpy(path2, path);
 258         strcat(path2, "XXXXXX");
 259         int fd = mkstemp(path2);//open(path, O_WRONLY | O_CREAT, 0600);
 260         if (fd < 0)
 261                 return;
 262         data = malloc(size);
 263         if (!data)
 264                 goto fail;
 265
 266         if (radv_GetPipelineCacheData(radv_device_to_handle(device),
 267                                       radv_pipeline_cache_to_handle(&device->meta_state.cache),
 268                                       &size, data))
 269                 goto fail;
 270         if(write(fd, data, size) == -1)
 271                 goto fail;
 272
 273         rename(path2, path);
 274 fail:
 275         free(data);
 276         close(fd);
 277         unlink(path2);
 278 }
 279
 280 VkResult
 281 radv_device_init_meta(struct radv_device *device)
 282 {
 283         VkResult result;
 284
 285         device->meta_state.alloc = (VkAllocationCallbacks) {
 286                 .pUserData = device,
 287                 .pfnAllocation = meta_alloc,
 288                 .pfnReallocation = meta_realloc,
 289                 .pfnFree = meta_free,
 290         };
 291
 292         device->meta_state.cache.alloc = device->meta_state.alloc;
 293         radv_pipeline_cache_init(&device->meta_state.cache, device);
 294         radv_load_meta_pipeline(device);
 295
 296         result = radv_device_init_meta_clear_state(device);
 297         if (result != VK_SUCCESS)
 298                 goto fail_clear;
 299
 300         result = radv_device_init_meta_resolve_state(device);
 301         if (result != VK_SUCCESS)
 302                 goto fail_resolve;
 303
 304         result = radv_device_init_meta_blit_state(device);
 305         if (result != VK_SUCCESS)
 306                 goto fail_blit;
 307
 308         result = radv_device_init_meta_blit2d_state(device);
 309         if (result != VK_SUCCESS)
 310                 goto fail_blit2d;
 311
 312         result = radv_device_init_meta_bufimage_state(device);
 313         if (result != VK_SUCCESS)
 314                 goto fail_bufimage;
 315
 316         result = radv_device_init_meta_depth_decomp_state(device);
 317         if (result != VK_SUCCESS)
 318                 goto fail_depth_decomp;
 319
 320         result = radv_device_init_meta_buffer_state(device);
 321         if (result != VK_SUCCESS)
 322                 goto fail_buffer;
 323
 324         result = radv_device_init_meta_query_state(device);
 325         if (result != VK_SUCCESS)
 326                 goto fail_query;
 327
 328         result = radv_device_init_meta_fast_clear_flush_state(device);
 329         if (result != VK_SUCCESS)
 330                 goto fail_fast_clear;
 331
 332         result = radv_device_init_meta_resolve_compute_state(device);
 333         if (result != VK_SUCCESS)
 334                 goto fail_resolve_compute;
 335
 336         result = radv_device_init_meta_resolve_fragment_state(device);
 337         if (result != VK_SUCCESS)
 338                 goto fail_resolve_fragment;
 339         return VK_SUCCESS;
 340
 341 fail_resolve_fragment:
 342         radv_device_finish_meta_resolve_compute_state(device);
 343 fail_resolve_compute:
 344         radv_device_finish_meta_fast_clear_flush_state(device);
 345 fail_fast_clear:
 346         radv_device_finish_meta_buffer_state(device);
 347 fail_query:
 348         radv_device_finish_meta_query_state(device);
 349 fail_buffer:
 350         radv_device_finish_meta_depth_decomp_state(device);
 351 fail_depth_decomp:
 352         radv_device_finish_meta_bufimage_state(device);
 353 fail_bufimage:
 354         radv_device_finish_meta_blit2d_state(device);
 355 fail_blit2d:
 356         radv_device_finish_meta_blit_state(device);
 357 fail_blit:
 358         radv_device_finish_meta_resolve_state(device);
 359 fail_resolve:
 360         radv_device_finish_meta_clear_state(device);
 361 fail_clear:
 362         radv_pipeline_cache_finish(&device->meta_state.cache);
 363         return result;
 364 }
 365
 366 void
 367 radv_device_finish_meta(struct radv_device *device)
 368 {
 369         radv_device_finish_meta_clear_state(device);
 370         radv_device_finish_meta_resolve_state(device);
 371         radv_device_finish_meta_blit_state(device);
 372         radv_device_finish_meta_blit2d_state(device);
 373         radv_device_finish_meta_bufimage_state(device);
 374         radv_device_finish_meta_depth_decomp_state(device);
 375         radv_device_finish_meta_query_state(device);
 376         radv_device_finish_meta_buffer_state(device);
 377         radv_device_finish_meta_fast_clear_flush_state(device);
 378         radv_device_finish_meta_resolve_compute_state(device);
 379         radv_device_finish_meta_resolve_fragment_state(device);
 380
 381         radv_store_meta_pipeline(device);
 382         radv_pipeline_cache_finish(&device->meta_state.cache);
 383 }
 384
 385 /*
 386  * The most common meta operations all want to have the viewport
 387  * reset and any scissors disabled. The rest of the dynamic state
 388  * should have no effect.
 389  */
 390 void
 391 radv_meta_save_graphics_reset_vport_scissor_novertex(struct radv_meta_saved_state *saved_state,
 392                                                      struct radv_cmd_buffer *cmd_buffer)
 393 {
 394         uint32_t dirty_state = (1 << VK_DYNAMIC_STATE_VIEWPORT) | (1 << VK_DYNAMIC_STATE_SCISSOR);
 395         radv_meta_save_novertex(saved_state, cmd_buffer, dirty_state);
 396         cmd_buffer->state.dynamic.viewport.count = 0;
 397         cmd_buffer->state.dynamic.scissor.count = 0;
 398         cmd_buffer->state.dirty |= dirty_state;
 399 }
 400
 401 nir_ssa_def *radv_meta_gen_rect_vertices_comp2(nir_builder *vs_b, nir_ssa_def *comp2)
 402 {
 403
 404         nir_intrinsic_instr *vertex_id = nir_intrinsic_instr_create(vs_b->shader, nir_intrinsic_load_vertex_id_zero_base);
 405         nir_ssa_dest_init(&vertex_id->instr, &vertex_id->dest, 1, 32, "vertexid");
 406         nir_builder_instr_insert(vs_b, &vertex_id->instr);
 407
 408         /* vertex 0 - -1.0, -1.0 */
 409         /* vertex 1 - -1.0, 1.0 */
 410         /* vertex 2 - 1.0, -1.0 */
 411         /* so channel 0 is vertex_id != 2 ? -1.0 : 1.0
 412            channel 1 is vertex id != 1 ? -1.0 : 1.0 */
 413
 414         nir_ssa_def *c0cmp = nir_ine(vs_b, &vertex_id->dest.ssa,
 415                                      nir_imm_int(vs_b, 2));
 416         nir_ssa_def *c1cmp = nir_ine(vs_b, &vertex_id->dest.ssa,
 417                                      nir_imm_int(vs_b, 1));
 418
 419         nir_ssa_def *comp[4];
 420         comp[0] = nir_bcsel(vs_b, c0cmp,
 421                             nir_imm_float(vs_b, -1.0),
 422                             nir_imm_float(vs_b, 1.0));
 423
 424         comp[1] = nir_bcsel(vs_b, c1cmp,
 425                             nir_imm_float(vs_b, -1.0),
 426                             nir_imm_float(vs_b, 1.0));
 427         comp[2] = comp2;
 428         comp[3] = nir_imm_float(vs_b, 1.0);
 429         nir_ssa_def *outvec = nir_vec(vs_b, comp, 4);
 430
 431         return outvec;
 432 }
 433
 434 nir_ssa_def *radv_meta_gen_rect_vertices(nir_builder *vs_b)
 435 {
 436         return radv_meta_gen_rect_vertices_comp2(vs_b, nir_imm_float(vs_b, 0.0));
 437 }
 438
 439 /* vertex shader that generates vertices */
 440 nir_shader *
 441 radv_meta_build_nir_vs_generate_vertices(void)
 442 {
 443         const struct glsl_type *vec4 = glsl_vec4_type();
 444
 445         nir_builder b;
 446         nir_variable *v_position;
 447
 448         nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_VERTEX, NULL);
 449         b.shader->info.name = ralloc_strdup(b.shader, "meta_vs_gen_verts");
 450
 451         nir_ssa_def *outvec = radv_meta_gen_rect_vertices(&b);
 452
 453         v_position = nir_variable_create(b.shader, nir_var_shader_out, vec4,
 454                                          "gl_Position");
 455         v_position->data.location = VARYING_SLOT_POS;
 456
 457         nir_store_var(&b, v_position, outvec, 0xf);
 458
 459         return b.shader;
 460 }
 461
 462 nir_shader *
 463 radv_meta_build_nir_fs_noop(void)
 464 {
 465         nir_builder b;
 466
 467         nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_FRAGMENT, NULL);
 468         b.shader->info.name = ralloc_asprintf(b.shader,
 469                                                "meta_noop_fs");
 470
 471         return b.shader;
 472 }
 473
 474 void radv_meta_build_resolve_shader_core(nir_builder *b,
 475                                          bool is_integer,
 476                                          int samples,
 477                                          nir_variable *input_img,
 478                                          nir_variable *color,
 479                                          nir_ssa_def *img_coord)
 480 {
 481         /* do a txf_ms on each sample */
 482         nir_ssa_def *tmp;
 483         nir_if *outer_if = NULL;
 484
 485         nir_tex_instr *tex = nir_tex_instr_create(b->shader, 2);
 486         tex->sampler_dim = GLSL_SAMPLER_DIM_MS;
 487         tex->op = nir_texop_txf_ms;
 488         tex->src[0].src_type = nir_tex_src_coord;
 489         tex->src[0].src = nir_src_for_ssa(img_coord);
 490         tex->src[1].src_type = nir_tex_src_ms_index;
 491         tex->src[1].src = nir_src_for_ssa(nir_imm_int(b, 0));
 492         tex->dest_type = nir_type_float;
 493         tex->is_array = false;
 494         tex->coord_components = 2;
 495         tex->texture = nir_deref_var_create(tex, input_img);
 496         tex->sampler = NULL;
 497
 498         nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex");
 499         nir_builder_instr_insert(b, &tex->instr);
 500
 501         tmp = &tex->dest.ssa;
 502
 503         if (!is_integer && samples > 1) {
 504                 nir_tex_instr *tex_all_same = nir_tex_instr_create(b->shader, 1);
 505                 tex_all_same->sampler_dim = GLSL_SAMPLER_DIM_MS;
 506                 tex_all_same->op = nir_texop_samples_identical;
 507                 tex_all_same->src[0].src_type = nir_tex_src_coord;
 508                 tex_all_same->src[0].src = nir_src_for_ssa(img_coord);
 509                 tex_all_same->dest_type = nir_type_float;
 510                 tex_all_same->is_array = false;
 511                 tex_all_same->coord_components = 2;
 512                 tex_all_same->texture = nir_deref_var_create(tex_all_same, input_img);
 513                 tex_all_same->sampler = NULL;
 514
 515                 nir_ssa_dest_init(&tex_all_same->instr, &tex_all_same->dest, 1, 32, "tex");
 516                 nir_builder_instr_insert(b, &tex_all_same->instr);
 517
 518                 nir_ssa_def *all_same = nir_ine(b, &tex_all_same->dest.ssa, nir_imm_int(b, 0));
 519                 nir_if *if_stmt = nir_if_create(b->shader);
 520                 if_stmt->condition = nir_src_for_ssa(all_same);
 521                 nir_cf_node_insert(b->cursor, &if_stmt->cf_node);
 522
 523                 b->cursor = nir_after_cf_list(&if_stmt->then_list);
 524                 for (int i = 1; i < samples; i++) {
 525                         nir_tex_instr *tex_add = nir_tex_instr_create(b->shader, 2);
 526                         tex_add->sampler_dim = GLSL_SAMPLER_DIM_MS;
 527                         tex_add->op = nir_texop_txf_ms;
 528                         tex_add->src[0].src_type = nir_tex_src_coord;
 529                         tex_add->src[0].src = nir_src_for_ssa(img_coord);
 530                         tex_add->src[1].src_type = nir_tex_src_ms_index;
 531                         tex_add->src[1].src = nir_src_for_ssa(nir_imm_int(b, i));
 532                         tex_add->dest_type = nir_type_float;
 533                         tex_add->is_array = false;
 534                         tex_add->coord_components = 2;
 535                         tex_add->texture = nir_deref_var_create(tex_add, input_img);
 536                         tex_add->sampler = NULL;
 537
 538                         nir_ssa_dest_init(&tex_add->instr, &tex_add->dest, 4, 32, "tex");
 539                         nir_builder_instr_insert(b, &tex_add->instr);
 540
 541                         tmp = nir_fadd(b, tmp, &tex_add->dest.ssa);
 542                 }
 543
 544                 tmp = nir_fdiv(b, tmp, nir_imm_float(b, samples));
 545                 nir_store_var(b, color, tmp, 0xf);
 546                 b->cursor = nir_after_cf_list(&if_stmt->else_list);
 547                 outer_if = if_stmt;
 548         }
 549         nir_store_var(b, color, &tex->dest.ssa, 0xf);
 550
 551         if (outer_if)
 552                 b->cursor = nir_after_cf_node(&outer_if->cf_node);
 553 }