src/amd/vulkan/radv_meta.c

   1 /*
   2  * Copyright © 2016 Red Hat
   3  * based on intel anv code:
   4  * Copyright © 2015 Intel Corporation
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the next
  14  * paragraph) shall be included in all copies or substantial portions of the
  15  * Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  20  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  22  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  23  * IN THE SOFTWARE.
  24  */
  25
  26 #include "radv_meta.h"
  27
  28 #include <fcntl.h>
  29 #include <limits.h>
  30 #include <pwd.h>
  31 #include <sys/stat.h>
  32
  33 void
  34 radv_meta_save(struct radv_meta_saved_state *state,
  35                struct radv_cmd_buffer *cmd_buffer, uint32_t flags)
  36 {
  37         VkPipelineBindPoint bind_point =
  38                 flags & RADV_META_SAVE_GRAPHICS_PIPELINE ?
  39                         VK_PIPELINE_BIND_POINT_GRAPHICS :
  40                         VK_PIPELINE_BIND_POINT_COMPUTE;
  41         struct radv_descriptor_state *descriptors_state =
  42                 radv_get_descriptors_state(cmd_buffer, bind_point);
  43
  44         assert(flags & (RADV_META_SAVE_GRAPHICS_PIPELINE |
  45                         RADV_META_SAVE_COMPUTE_PIPELINE));
  46
  47         state->flags = flags;
  48
  49         if (state->flags & RADV_META_SAVE_GRAPHICS_PIPELINE) {
  50                 assert(!(state->flags & RADV_META_SAVE_COMPUTE_PIPELINE));
  51
  52                 state->old_pipeline = cmd_buffer->state.pipeline;
  53
  54                 /* Save all viewports. */
  55                 state->viewport.count = cmd_buffer->state.dynamic.viewport.count;
  56                 typed_memcpy(state->viewport.viewports,
  57                              cmd_buffer->state.dynamic.viewport.viewports,
  58                              MAX_VIEWPORTS);
  59
  60                 /* Save all scissors. */
  61                 state->scissor.count = cmd_buffer->state.dynamic.scissor.count;
  62                 typed_memcpy(state->scissor.scissors,
  63                              cmd_buffer->state.dynamic.scissor.scissors,
  64                              MAX_SCISSORS);
  65
  66                 state->cull_mode = cmd_buffer->state.dynamic.cull_mode;
  67                 state->front_face = cmd_buffer->state.dynamic.front_face;
  68         }
  69
  70         if (state->flags & RADV_META_SAVE_SAMPLE_LOCATIONS) {
  71                 typed_memcpy(&state->sample_location,
  72                              &cmd_buffer->state.dynamic.sample_location, 1);
  73         }
  74
  75         if (state->flags & RADV_META_SAVE_COMPUTE_PIPELINE) {
  76                 assert(!(state->flags & RADV_META_SAVE_GRAPHICS_PIPELINE));
  77
  78                 state->old_pipeline = cmd_buffer->state.compute_pipeline;
  79         }
  80
  81         if (state->flags & RADV_META_SAVE_DESCRIPTORS) {
  82                 state->old_descriptor_set0 = descriptors_state->sets[0];
  83                 if (!(descriptors_state->valid & 1) || !state->old_descriptor_set0)
  84                         state->flags &= ~RADV_META_SAVE_DESCRIPTORS;
  85         }
  86
  87         if (state->flags & RADV_META_SAVE_CONSTANTS) {
  88                 memcpy(state->push_constants, cmd_buffer->push_constants,
  89                        MAX_PUSH_CONSTANTS_SIZE);
  90         }
  91
  92         if (state->flags & RADV_META_SAVE_PASS) {
  93                 state->pass = cmd_buffer->state.pass;
  94                 state->subpass = cmd_buffer->state.subpass;
  95                 state->framebuffer = cmd_buffer->state.framebuffer;
  96                 state->attachments = cmd_buffer->state.attachments;
  97                 state->render_area = cmd_buffer->state.render_area;
  98         }
  99 }
 100
 101 void
 102 radv_meta_restore(const struct radv_meta_saved_state *state,
 103                   struct radv_cmd_buffer *cmd_buffer)
 104 {
 105         VkPipelineBindPoint bind_point =
 106                 state->flags & RADV_META_SAVE_GRAPHICS_PIPELINE ?
 107                         VK_PIPELINE_BIND_POINT_GRAPHICS :
 108                         VK_PIPELINE_BIND_POINT_COMPUTE;
 109
 110         if (state->flags & RADV_META_SAVE_GRAPHICS_PIPELINE) {
 111                 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
 112                                      VK_PIPELINE_BIND_POINT_GRAPHICS,
 113                                      radv_pipeline_to_handle(state->old_pipeline));
 114
 115                 cmd_buffer->state.dirty |= RADV_CMD_DIRTY_PIPELINE;
 116
 117                 /* Restore all viewports. */
 118                 cmd_buffer->state.dynamic.viewport.count = state->viewport.count;
 119                 typed_memcpy(cmd_buffer->state.dynamic.viewport.viewports,
 120                              state->viewport.viewports,
 121                              MAX_VIEWPORTS);
 122
 123                 /* Restore all scissors. */
 124                 cmd_buffer->state.dynamic.scissor.count = state->scissor.count;
 125                 typed_memcpy(cmd_buffer->state.dynamic.scissor.scissors,
 126                              state->scissor.scissors,
 127                              MAX_SCISSORS);
 128
 129                 cmd_buffer->state.dynamic.cull_mode = state->cull_mode;
 130                 cmd_buffer->state.dynamic.front_face = state->front_face;
 131
 132                 cmd_buffer->state.dirty |= RADV_CMD_DIRTY_DYNAMIC_VIEWPORT |
 133                                            RADV_CMD_DIRTY_DYNAMIC_SCISSOR |
 134                                            RADV_CMD_DIRTY_DYNAMIC_CULL_MODE |
 135                                            RADV_CMD_DIRTY_DYNAMIC_FRONT_FACE;
 136         }
 137
 138         if (state->flags & RADV_META_SAVE_SAMPLE_LOCATIONS) {
 139                 typed_memcpy(&cmd_buffer->state.dynamic.sample_location.locations,
 140                              &state->sample_location.locations, 1);
 141
 142                 cmd_buffer->state.dirty |= RADV_CMD_DIRTY_DYNAMIC_SAMPLE_LOCATIONS;
 143         }
 144
 145         if (state->flags & RADV_META_SAVE_COMPUTE_PIPELINE) {
 146                 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
 147                                      VK_PIPELINE_BIND_POINT_COMPUTE,
 148                                      radv_pipeline_to_handle(state->old_pipeline));
 149         }
 150
 151         if (state->flags & RADV_META_SAVE_DESCRIPTORS) {
 152                 radv_set_descriptor_set(cmd_buffer, bind_point,
 153                                         state->old_descriptor_set0, 0);
 154         }
 155
 156         if (state->flags & RADV_META_SAVE_CONSTANTS) {
 157                 VkShaderStageFlags stages = VK_SHADER_STAGE_COMPUTE_BIT;
 158
 159                 if (state->flags & RADV_META_SAVE_GRAPHICS_PIPELINE)
 160                         stages |= VK_SHADER_STAGE_ALL_GRAPHICS;
 161
 162                 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer),
 163                                       VK_NULL_HANDLE, stages, 0,
 164                                       MAX_PUSH_CONSTANTS_SIZE,
 165                                       state->push_constants);
 166         }
 167
 168         if (state->flags & RADV_META_SAVE_PASS) {
 169                 cmd_buffer->state.pass = state->pass;
 170                 cmd_buffer->state.subpass = state->subpass;
 171                 cmd_buffer->state.framebuffer = state->framebuffer;
 172                 cmd_buffer->state.attachments = state->attachments;
 173                 cmd_buffer->state.render_area = state->render_area;
 174                 if (state->subpass)
 175                         cmd_buffer->state.dirty |= RADV_CMD_DIRTY_FRAMEBUFFER;
 176         }
 177 }
 178
 179 VkImageViewType
 180 radv_meta_get_view_type(const struct radv_image *image)
 181 {
 182         switch (image->type) {
 183         case VK_IMAGE_TYPE_1D: return VK_IMAGE_VIEW_TYPE_1D;
 184         case VK_IMAGE_TYPE_2D: return VK_IMAGE_VIEW_TYPE_2D;
 185         case VK_IMAGE_TYPE_3D: return VK_IMAGE_VIEW_TYPE_3D;
 186         default:
 187                 unreachable("bad VkImageViewType");
 188         }
 189 }
 190
 191 /**
 192  * When creating a destination VkImageView, this function provides the needed
 193  * VkImageViewCreateInfo::subresourceRange::baseArrayLayer.
 194  */
 195 uint32_t
 196 radv_meta_get_iview_layer(const struct radv_image *dest_image,
 197                           const VkImageSubresourceLayers *dest_subresource,
 198                           const VkOffset3D *dest_offset)
 199 {
 200         switch (dest_image->type) {
 201         case VK_IMAGE_TYPE_1D:
 202         case VK_IMAGE_TYPE_2D:
 203                 return dest_subresource->baseArrayLayer;
 204         case VK_IMAGE_TYPE_3D:
 205                 /* HACK: Vulkan does not allow attaching a 3D image to a framebuffer,
 206                  * but meta does it anyway. When doing so, we translate the
 207                  * destination's z offset into an array offset.
 208                  */
 209                 return dest_offset->z;
 210         default:
 211                 assert(!"bad VkImageType");
 212                 return 0;
 213         }
 214 }
 215
 216 static void *
 217 meta_alloc(void* _device, size_t size, size_t alignment,
 218            VkSystemAllocationScope allocationScope)
 219 {
 220         struct radv_device *device = _device;
 221         return device->vk.alloc.pfnAllocation(device->vk.alloc.pUserData, size, alignment,
 222                                            VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
 223 }
 224
 225 static void *
 226 meta_realloc(void* _device, void *original, size_t size, size_t alignment,
 227              VkSystemAllocationScope allocationScope)
 228 {
 229         struct radv_device *device = _device;
 230         return device->vk.alloc.pfnReallocation(device->vk.alloc.pUserData, original,
 231                                              size, alignment,
 232                                              VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
 233 }
 234
 235 static void
 236 meta_free(void* _device, void *data)
 237 {
 238         struct radv_device *device = _device;
 239         return device->vk.alloc.pfnFree(device->vk.alloc.pUserData, data);
 240 }
 241
 242 static bool
 243 radv_builtin_cache_path(char *path)
 244 {
 245         char *xdg_cache_home = getenv("XDG_CACHE_HOME");
 246         const char *suffix = "/radv_builtin_shaders";
 247         const char *suffix2 = "/.cache/radv_builtin_shaders";
 248         struct passwd pwd, *result;
 249         char path2[PATH_MAX + 1]; /* PATH_MAX is not a real max,but suffices here. */
 250         int ret;
 251
 252         if (xdg_cache_home) {
 253                 ret = snprintf(path, PATH_MAX + 1, "%s%s%zd",
 254                                xdg_cache_home, suffix, sizeof(void *) * 8);
 255                 return ret > 0 && ret < PATH_MAX + 1;
 256         }
 257
 258         getpwuid_r(getuid(), &pwd, path2, PATH_MAX - strlen(suffix2), &result);
 259         if (!result)
 260                 return false;
 261
 262         strcpy(path, pwd.pw_dir);
 263         strcat(path, "/.cache");
 264         if (mkdir(path, 0755) && errno != EEXIST)
 265                 return false;
 266
 267         ret = snprintf(path, PATH_MAX + 1, "%s%s%zd",
 268                        pwd.pw_dir, suffix2, sizeof(void *) * 8);
 269         return ret > 0 && ret < PATH_MAX + 1;
 270 }
 271
 272 static bool
 273 radv_load_meta_pipeline(struct radv_device *device)
 274 {
 275         char path[PATH_MAX + 1];
 276         struct stat st;
 277         void *data = NULL;
 278         bool ret = false;
 279
 280         if (!radv_builtin_cache_path(path))
 281                 return false;
 282
 283         int fd = open(path, O_RDONLY);
 284         if (fd < 0)
 285                 return false;
 286         if (fstat(fd, &st))
 287                 goto fail;
 288         data = malloc(st.st_size);
 289         if (!data)
 290                 goto fail;
 291         if(read(fd, data, st.st_size) == -1)
 292                 goto fail;
 293
 294         ret = radv_pipeline_cache_load(&device->meta_state.cache, data, st.st_size);
 295 fail:
 296         free(data);
 297         close(fd);
 298         return ret;
 299 }
 300
 301 static void
 302 radv_store_meta_pipeline(struct radv_device *device)
 303 {
 304         char path[PATH_MAX + 1], path2[PATH_MAX + 7];
 305         size_t size;
 306         void *data = NULL;
 307
 308         if (!device->meta_state.cache.modified)
 309                 return;
 310
 311         if (radv_GetPipelineCacheData(radv_device_to_handle(device),
 312                                       radv_pipeline_cache_to_handle(&device->meta_state.cache),
 313                                       &size, NULL))
 314                 return;
 315
 316         if (!radv_builtin_cache_path(path))
 317                 return;
 318
 319         strcpy(path2, path);
 320         strcat(path2, "XXXXXX");
 321         int fd = mkstemp(path2);//open(path, O_WRONLY | O_CREAT, 0600);
 322         if (fd < 0)
 323                 return;
 324         data = malloc(size);
 325         if (!data)
 326                 goto fail;
 327
 328         if (radv_GetPipelineCacheData(radv_device_to_handle(device),
 329                                       radv_pipeline_cache_to_handle(&device->meta_state.cache),
 330                                       &size, data))
 331                 goto fail;
 332         if(write(fd, data, size) == -1)
 333                 goto fail;
 334
 335         rename(path2, path);
 336 fail:
 337         free(data);
 338         close(fd);
 339         unlink(path2);
 340 }
 341
 342 VkResult
 343 radv_device_init_meta(struct radv_device *device)
 344 {
 345         VkResult result;
 346
 347         memset(&device->meta_state, 0, sizeof(device->meta_state));
 348
 349         device->meta_state.alloc = (VkAllocationCallbacks) {
 350                 .pUserData = device,
 351                 .pfnAllocation = meta_alloc,
 352                 .pfnReallocation = meta_realloc,
 353                 .pfnFree = meta_free,
 354         };
 355
 356         device->meta_state.cache.alloc = device->meta_state.alloc;
 357         radv_pipeline_cache_init(&device->meta_state.cache, device);
 358         bool loaded_cache = radv_load_meta_pipeline(device);
 359         bool on_demand = !loaded_cache;
 360
 361         mtx_init(&device->meta_state.mtx, mtx_plain);
 362
 363         result = radv_device_init_meta_clear_state(device, on_demand);
 364         if (result != VK_SUCCESS)
 365                 goto fail_clear;
 366
 367         result = radv_device_init_meta_resolve_state(device, on_demand);
 368         if (result != VK_SUCCESS)
 369                 goto fail_resolve;
 370
 371         result = radv_device_init_meta_blit_state(device, on_demand);
 372         if (result != VK_SUCCESS)
 373                 goto fail_blit;
 374
 375         result = radv_device_init_meta_blit2d_state(device, on_demand);
 376         if (result != VK_SUCCESS)
 377                 goto fail_blit2d;
 378
 379         result = radv_device_init_meta_bufimage_state(device);
 380         if (result != VK_SUCCESS)
 381                 goto fail_bufimage;
 382
 383         result = radv_device_init_meta_depth_decomp_state(device, on_demand);
 384         if (result != VK_SUCCESS)
 385                 goto fail_depth_decomp;
 386
 387         result = radv_device_init_meta_buffer_state(device);
 388         if (result != VK_SUCCESS)
 389                 goto fail_buffer;
 390
 391         result = radv_device_init_meta_query_state(device, on_demand);
 392         if (result != VK_SUCCESS)
 393                 goto fail_query;
 394
 395         result = radv_device_init_meta_fast_clear_flush_state(device, on_demand);
 396         if (result != VK_SUCCESS)
 397                 goto fail_fast_clear;
 398
 399         result = radv_device_init_meta_resolve_compute_state(device, on_demand);
 400         if (result != VK_SUCCESS)
 401                 goto fail_resolve_compute;
 402
 403         result = radv_device_init_meta_resolve_fragment_state(device, on_demand);
 404         if (result != VK_SUCCESS)
 405                 goto fail_resolve_fragment;
 406
 407         result = radv_device_init_meta_fmask_expand_state(device);
 408         if (result != VK_SUCCESS)
 409                 goto fail_fmask_expand;
 410
 411         return VK_SUCCESS;
 412
 413 fail_fmask_expand:
 414         radv_device_finish_meta_resolve_fragment_state(device);
 415 fail_resolve_fragment:
 416         radv_device_finish_meta_resolve_compute_state(device);
 417 fail_resolve_compute:
 418         radv_device_finish_meta_fast_clear_flush_state(device);
 419 fail_fast_clear:
 420         radv_device_finish_meta_query_state(device);
 421 fail_query:
 422         radv_device_finish_meta_buffer_state(device);
 423 fail_buffer:
 424         radv_device_finish_meta_depth_decomp_state(device);
 425 fail_depth_decomp:
 426         radv_device_finish_meta_bufimage_state(device);
 427 fail_bufimage:
 428         radv_device_finish_meta_blit2d_state(device);
 429 fail_blit2d:
 430         radv_device_finish_meta_blit_state(device);
 431 fail_blit:
 432         radv_device_finish_meta_resolve_state(device);
 433 fail_resolve:
 434         radv_device_finish_meta_clear_state(device);
 435 fail_clear:
 436         mtx_destroy(&device->meta_state.mtx);
 437         radv_pipeline_cache_finish(&device->meta_state.cache);
 438         return result;
 439 }
 440
 441 void
 442 radv_device_finish_meta(struct radv_device *device)
 443 {
 444         radv_device_finish_meta_clear_state(device);
 445         radv_device_finish_meta_resolve_state(device);
 446         radv_device_finish_meta_blit_state(device);
 447         radv_device_finish_meta_blit2d_state(device);
 448         radv_device_finish_meta_bufimage_state(device);
 449         radv_device_finish_meta_depth_decomp_state(device);
 450         radv_device_finish_meta_query_state(device);
 451         radv_device_finish_meta_buffer_state(device);
 452         radv_device_finish_meta_fast_clear_flush_state(device);
 453         radv_device_finish_meta_resolve_compute_state(device);
 454         radv_device_finish_meta_resolve_fragment_state(device);
 455         radv_device_finish_meta_fmask_expand_state(device);
 456
 457         radv_store_meta_pipeline(device);
 458         radv_pipeline_cache_finish(&device->meta_state.cache);
 459         mtx_destroy(&device->meta_state.mtx);
 460 }
 461
 462 nir_ssa_def *radv_meta_gen_rect_vertices_comp2(nir_builder *vs_b, nir_ssa_def *comp2)
 463 {
 464
 465         nir_intrinsic_instr *vertex_id = nir_intrinsic_instr_create(vs_b->shader, nir_intrinsic_load_vertex_id_zero_base);
 466         nir_ssa_dest_init(&vertex_id->instr, &vertex_id->dest, 1, 32, "vertexid");
 467         nir_builder_instr_insert(vs_b, &vertex_id->instr);
 468
 469         /* vertex 0 - -1.0, -1.0 */
 470         /* vertex 1 - -1.0, 1.0 */
 471         /* vertex 2 - 1.0, -1.0 */
 472         /* so channel 0 is vertex_id != 2 ? -1.0 : 1.0
 473            channel 1 is vertex id != 1 ? -1.0 : 1.0 */
 474
 475         nir_ssa_def *c0cmp = nir_ine(vs_b, &vertex_id->dest.ssa,
 476                                      nir_imm_int(vs_b, 2));
 477         nir_ssa_def *c1cmp = nir_ine(vs_b, &vertex_id->dest.ssa,
 478                                      nir_imm_int(vs_b, 1));
 479
 480         nir_ssa_def *comp[4];
 481         comp[0] = nir_bcsel(vs_b, c0cmp,
 482                             nir_imm_float(vs_b, -1.0),
 483                             nir_imm_float(vs_b, 1.0));
 484
 485         comp[1] = nir_bcsel(vs_b, c1cmp,
 486                             nir_imm_float(vs_b, -1.0),
 487                             nir_imm_float(vs_b, 1.0));
 488         comp[2] = comp2;
 489         comp[3] = nir_imm_float(vs_b, 1.0);
 490         nir_ssa_def *outvec = nir_vec(vs_b, comp, 4);
 491
 492         return outvec;
 493 }
 494
 495 nir_ssa_def *radv_meta_gen_rect_vertices(nir_builder *vs_b)
 496 {
 497         return radv_meta_gen_rect_vertices_comp2(vs_b, nir_imm_float(vs_b, 0.0));
 498 }
 499
 500 /* vertex shader that generates vertices */
 501 nir_shader *
 502 radv_meta_build_nir_vs_generate_vertices(void)
 503 {
 504         const struct glsl_type *vec4 = glsl_vec4_type();
 505
 506         nir_builder b;
 507         nir_variable *v_position;
 508
 509         nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_VERTEX, NULL);
 510         b.shader->info.name = ralloc_strdup(b.shader, "meta_vs_gen_verts");
 511
 512         nir_ssa_def *outvec = radv_meta_gen_rect_vertices(&b);
 513
 514         v_position = nir_variable_create(b.shader, nir_var_shader_out, vec4,
 515                                          "gl_Position");
 516         v_position->data.location = VARYING_SLOT_POS;
 517
 518         nir_store_var(&b, v_position, outvec, 0xf);
 519
 520         return b.shader;
 521 }
 522
 523 nir_shader *
 524 radv_meta_build_nir_fs_noop(void)
 525 {
 526         nir_builder b;
 527
 528         nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_FRAGMENT, NULL);
 529         b.shader->info.name = ralloc_asprintf(b.shader,
 530                                                "meta_noop_fs");
 531
 532         return b.shader;
 533 }
 534
 535 void radv_meta_build_resolve_shader_core(nir_builder *b,
 536                                          bool is_integer,
 537                                          int samples,
 538                                          nir_variable *input_img,
 539                                          nir_variable *color,
 540                                          nir_ssa_def *img_coord)
 541 {
 542         /* do a txf_ms on each sample */
 543         nir_ssa_def *tmp;
 544         nir_if *outer_if = NULL;
 545
 546         nir_ssa_def *input_img_deref = &nir_build_deref_var(b, input_img)->dest.ssa;
 547
 548         nir_tex_instr *tex = nir_tex_instr_create(b->shader, 3);
 549         tex->sampler_dim = GLSL_SAMPLER_DIM_MS;
 550         tex->op = nir_texop_txf_ms;
 551         tex->src[0].src_type = nir_tex_src_coord;
 552         tex->src[0].src = nir_src_for_ssa(img_coord);
 553         tex->src[1].src_type = nir_tex_src_ms_index;
 554         tex->src[1].src = nir_src_for_ssa(nir_imm_int(b, 0));
 555         tex->src[2].src_type = nir_tex_src_texture_deref;
 556         tex->src[2].src = nir_src_for_ssa(input_img_deref);
 557         tex->dest_type = nir_type_float;
 558         tex->is_array = false;
 559         tex->coord_components = 2;
 560
 561         nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex");
 562         nir_builder_instr_insert(b, &tex->instr);
 563
 564         tmp = &tex->dest.ssa;
 565
 566         if (!is_integer && samples > 1) {
 567                 nir_tex_instr *tex_all_same = nir_tex_instr_create(b->shader, 2);
 568                 tex_all_same->sampler_dim = GLSL_SAMPLER_DIM_MS;
 569                 tex_all_same->op = nir_texop_samples_identical;
 570                 tex_all_same->src[0].src_type = nir_tex_src_coord;
 571                 tex_all_same->src[0].src = nir_src_for_ssa(img_coord);
 572                 tex_all_same->src[1].src_type = nir_tex_src_texture_deref;
 573                 tex_all_same->src[1].src = nir_src_for_ssa(input_img_deref);
 574                 tex_all_same->dest_type = nir_type_float;
 575                 tex_all_same->is_array = false;
 576                 tex_all_same->coord_components = 2;
 577
 578                 nir_ssa_dest_init(&tex_all_same->instr, &tex_all_same->dest, 1, 32, "tex");
 579                 nir_builder_instr_insert(b, &tex_all_same->instr);
 580
 581                 nir_ssa_def *all_same = nir_ieq(b, &tex_all_same->dest.ssa, nir_imm_int(b, 0));
 582                 nir_if *if_stmt = nir_if_create(b->shader);
 583                 if_stmt->condition = nir_src_for_ssa(all_same);
 584                 nir_cf_node_insert(b->cursor, &if_stmt->cf_node);
 585
 586                 b->cursor = nir_after_cf_list(&if_stmt->then_list);
 587                 for (int i = 1; i < samples; i++) {
 588                         nir_tex_instr *tex_add = nir_tex_instr_create(b->shader, 3);
 589                         tex_add->sampler_dim = GLSL_SAMPLER_DIM_MS;
 590                         tex_add->op = nir_texop_txf_ms;
 591                         tex_add->src[0].src_type = nir_tex_src_coord;
 592                         tex_add->src[0].src = nir_src_for_ssa(img_coord);
 593                         tex_add->src[1].src_type = nir_tex_src_ms_index;
 594                         tex_add->src[1].src = nir_src_for_ssa(nir_imm_int(b, i));
 595                         tex_add->src[2].src_type = nir_tex_src_texture_deref;
 596                         tex_add->src[2].src = nir_src_for_ssa(input_img_deref);
 597                         tex_add->dest_type = nir_type_float;
 598                         tex_add->is_array = false;
 599                         tex_add->coord_components = 2;
 600
 601                         nir_ssa_dest_init(&tex_add->instr, &tex_add->dest, 4, 32, "tex");
 602                         nir_builder_instr_insert(b, &tex_add->instr);
 603
 604                         tmp = nir_fadd(b, tmp, &tex_add->dest.ssa);
 605                 }
 606
 607                 tmp = nir_fdiv(b, tmp, nir_imm_float(b, samples));
 608                 nir_store_var(b, color, tmp, 0xf);
 609                 b->cursor = nir_after_cf_list(&if_stmt->else_list);
 610                 outer_if = if_stmt;
 611         }
 612         nir_store_var(b, color, &tex->dest.ssa, 0xf);
 613
 614         if (outer_if)
 615                 b->cursor = nir_after_cf_node(&outer_if->cf_node);
 616 }