src/amd/vulkan/radv_meta.c

   1 /*
   2  * Copyright © 2016 Red Hat
   3  * based on intel anv code:
   4  * Copyright © 2015 Intel Corporation
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the next
  14  * paragraph) shall be included in all copies or substantial portions of the
  15  * Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  20  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  22  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  23  * IN THE SOFTWARE.
  24  */
  25
  26 #include "radv_meta.h"
  27
  28 #include <fcntl.h>
  29 #include <limits.h>
  30 #include <pwd.h>
  31 #include <sys/stat.h>
  32
  33 void
  34 radv_meta_save(struct radv_meta_saved_state *state,
  35                struct radv_cmd_buffer *cmd_buffer, uint32_t flags)
  36 {
  37         VkPipelineBindPoint bind_point =
  38                 flags & RADV_META_SAVE_GRAPHICS_PIPELINE ?
  39                         VK_PIPELINE_BIND_POINT_GRAPHICS :
  40                         VK_PIPELINE_BIND_POINT_COMPUTE;
  41         struct radv_descriptor_state *descriptors_state =
  42                 radv_get_descriptors_state(cmd_buffer, bind_point);
  43
  44         assert(flags & (RADV_META_SAVE_GRAPHICS_PIPELINE |
  45                         RADV_META_SAVE_COMPUTE_PIPELINE));
  46
  47         state->flags = flags;
  48
  49         if (state->flags & RADV_META_SAVE_GRAPHICS_PIPELINE) {
  50                 assert(!(state->flags & RADV_META_SAVE_COMPUTE_PIPELINE));
  51
  52                 state->old_pipeline = cmd_buffer->state.pipeline;
  53
  54                 /* Save all viewports. */
  55                 state->viewport.count = cmd_buffer->state.dynamic.viewport.count;
  56                 typed_memcpy(state->viewport.viewports,
  57                              cmd_buffer->state.dynamic.viewport.viewports,
  58                              MAX_VIEWPORTS);
  59
  60                 /* Save all scissors. */
  61                 state->scissor.count = cmd_buffer->state.dynamic.scissor.count;
  62                 typed_memcpy(state->scissor.scissors,
  63                              cmd_buffer->state.dynamic.scissor.scissors,
  64                              MAX_SCISSORS);
  65
  66                 /* The most common meta operations all want to have the
  67                  * viewport reset and any scissors disabled. The rest of the
  68                  * dynamic state should have no effect.
  69                  */
  70                 cmd_buffer->state.dynamic.viewport.count = 0;
  71                 cmd_buffer->state.dynamic.scissor.count = 0;
  72                 cmd_buffer->state.dirty |= 1 << VK_DYNAMIC_STATE_VIEWPORT |
  73                                            1 << VK_DYNAMIC_STATE_SCISSOR;
  74         }
  75
  76         if (state->flags & RADV_META_SAVE_COMPUTE_PIPELINE) {
  77                 assert(!(state->flags & RADV_META_SAVE_GRAPHICS_PIPELINE));
  78
  79                 state->old_pipeline = cmd_buffer->state.compute_pipeline;
  80         }
  81
  82         if (state->flags & RADV_META_SAVE_DESCRIPTORS) {
  83                 state->old_descriptor_set0 = descriptors_state->sets[0];
  84                 if (!state->old_descriptor_set0)
  85                         state->flags &= ~RADV_META_SAVE_DESCRIPTORS;
  86         }
  87
  88         if (state->flags & RADV_META_SAVE_CONSTANTS) {
  89                 memcpy(state->push_constants, cmd_buffer->push_constants,
  90                        MAX_PUSH_CONSTANTS_SIZE);
  91         }
  92
  93         if (state->flags & RADV_META_SAVE_PASS) {
  94                 state->pass = cmd_buffer->state.pass;
  95                 state->subpass = cmd_buffer->state.subpass;
  96                 state->framebuffer = cmd_buffer->state.framebuffer;
  97                 state->attachments = cmd_buffer->state.attachments;
  98                 state->render_area = cmd_buffer->state.render_area;
  99         }
 100 }
 101
 102 void
 103 radv_meta_restore(const struct radv_meta_saved_state *state,
 104                   struct radv_cmd_buffer *cmd_buffer)
 105 {
 106         VkPipelineBindPoint bind_point =
 107                 state->flags & RADV_META_SAVE_GRAPHICS_PIPELINE ?
 108                         VK_PIPELINE_BIND_POINT_GRAPHICS :
 109                         VK_PIPELINE_BIND_POINT_COMPUTE;
 110
 111         if (state->flags & RADV_META_SAVE_GRAPHICS_PIPELINE) {
 112                 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
 113                                      VK_PIPELINE_BIND_POINT_GRAPHICS,
 114                                      radv_pipeline_to_handle(state->old_pipeline));
 115
 116                 cmd_buffer->state.dirty |= RADV_CMD_DIRTY_PIPELINE;
 117
 118                 /* Restore all viewports. */
 119                 cmd_buffer->state.dynamic.viewport.count = state->viewport.count;
 120                 typed_memcpy(cmd_buffer->state.dynamic.viewport.viewports,
 121                              state->viewport.viewports,
 122                              MAX_VIEWPORTS);
 123
 124                 /* Restore all scissors. */
 125                 cmd_buffer->state.dynamic.scissor.count = state->scissor.count;
 126                 typed_memcpy(cmd_buffer->state.dynamic.scissor.scissors,
 127                              state->scissor.scissors,
 128                              MAX_SCISSORS);
 129
 130                 cmd_buffer->state.dirty |= 1 << VK_DYNAMIC_STATE_VIEWPORT |
 131                                            1 << VK_DYNAMIC_STATE_SCISSOR;
 132         }
 133
 134         if (state->flags & RADV_META_SAVE_COMPUTE_PIPELINE) {
 135                 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
 136                                      VK_PIPELINE_BIND_POINT_COMPUTE,
 137                                      radv_pipeline_to_handle(state->old_pipeline));
 138         }
 139
 140         if (state->flags & RADV_META_SAVE_DESCRIPTORS) {
 141                 radv_set_descriptor_set(cmd_buffer, bind_point,
 142                                         state->old_descriptor_set0, 0);
 143         }
 144
 145         if (state->flags & RADV_META_SAVE_CONSTANTS) {
 146                 memcpy(cmd_buffer->push_constants, state->push_constants,
 147                        MAX_PUSH_CONSTANTS_SIZE);
 148                 cmd_buffer->push_constant_stages |= VK_SHADER_STAGE_COMPUTE_BIT;
 149
 150                 if (state->flags & RADV_META_SAVE_GRAPHICS_PIPELINE) {
 151                         cmd_buffer->push_constant_stages |= VK_SHADER_STAGE_ALL_GRAPHICS;
 152                 }
 153         }
 154
 155         if (state->flags & RADV_META_SAVE_PASS) {
 156                 cmd_buffer->state.pass = state->pass;
 157                 cmd_buffer->state.subpass = state->subpass;
 158                 cmd_buffer->state.framebuffer = state->framebuffer;
 159                 cmd_buffer->state.attachments = state->attachments;
 160                 cmd_buffer->state.render_area = state->render_area;
 161                 if (state->subpass)
 162                         cmd_buffer->state.dirty |= RADV_CMD_DIRTY_FRAMEBUFFER;
 163         }
 164 }
 165
 166 VkImageViewType
 167 radv_meta_get_view_type(const struct radv_image *image)
 168 {
 169         switch (image->type) {
 170         case VK_IMAGE_TYPE_1D: return VK_IMAGE_VIEW_TYPE_1D;
 171         case VK_IMAGE_TYPE_2D: return VK_IMAGE_VIEW_TYPE_2D;
 172         case VK_IMAGE_TYPE_3D: return VK_IMAGE_VIEW_TYPE_3D;
 173         default:
 174                 unreachable("bad VkImageViewType");
 175         }
 176 }
 177
 178 /**
 179  * When creating a destination VkImageView, this function provides the needed
 180  * VkImageViewCreateInfo::subresourceRange::baseArrayLayer.
 181  */
 182 uint32_t
 183 radv_meta_get_iview_layer(const struct radv_image *dest_image,
 184                           const VkImageSubresourceLayers *dest_subresource,
 185                           const VkOffset3D *dest_offset)
 186 {
 187         switch (dest_image->type) {
 188         case VK_IMAGE_TYPE_1D:
 189         case VK_IMAGE_TYPE_2D:
 190                 return dest_subresource->baseArrayLayer;
 191         case VK_IMAGE_TYPE_3D:
 192                 /* HACK: Vulkan does not allow attaching a 3D image to a framebuffer,
 193                  * but meta does it anyway. When doing so, we translate the
 194                  * destination's z offset into an array offset.
 195                  */
 196                 return dest_offset->z;
 197         default:
 198                 assert(!"bad VkImageType");
 199                 return 0;
 200         }
 201 }
 202
 203 static void *
 204 meta_alloc(void* _device, size_t size, size_t alignment,
 205            VkSystemAllocationScope allocationScope)
 206 {
 207         struct radv_device *device = _device;
 208         return device->alloc.pfnAllocation(device->alloc.pUserData, size, alignment,
 209                                            VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
 210 }
 211
 212 static void *
 213 meta_realloc(void* _device, void *original, size_t size, size_t alignment,
 214              VkSystemAllocationScope allocationScope)
 215 {
 216         struct radv_device *device = _device;
 217         return device->alloc.pfnReallocation(device->alloc.pUserData, original,
 218                                              size, alignment,
 219                                              VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
 220 }
 221
 222 static void
 223 meta_free(void* _device, void *data)
 224 {
 225         struct radv_device *device = _device;
 226         return device->alloc.pfnFree(device->alloc.pUserData, data);
 227 }
 228
 229 static bool
 230 radv_builtin_cache_path(char *path)
 231 {
 232         char *xdg_cache_home = getenv("XDG_CACHE_HOME");
 233         const char *suffix = "/radv_builtin_shaders";
 234         const char *suffix2 = "/.cache/radv_builtin_shaders";
 235         struct passwd pwd, *result;
 236         char path2[PATH_MAX + 1]; /* PATH_MAX is not a real max,but suffices here. */
 237
 238         if (xdg_cache_home) {
 239
 240                 if (strlen(xdg_cache_home) + strlen(suffix) > PATH_MAX)
 241                         return false;
 242
 243                 strcpy(path, xdg_cache_home);
 244                 strcat(path, suffix);
 245                 return true;
 246         }
 247
 248         getpwuid_r(getuid(), &pwd, path2, PATH_MAX - strlen(suffix2), &result);
 249         if (!result)
 250                 return false;
 251
 252         strcpy(path, pwd.pw_dir);
 253         strcat(path, "/.cache");
 254         mkdir(path, 0755);
 255
 256         strcat(path, suffix);
 257         return true;
 258 }
 259
 260 static bool
 261 radv_load_meta_pipeline(struct radv_device *device)
 262 {
 263         char path[PATH_MAX + 1];
 264         struct stat st;
 265         void *data = NULL;
 266
 267         if (!radv_builtin_cache_path(path))
 268                 return false;
 269
 270         int fd = open(path, O_RDONLY);
 271         if (fd < 0)
 272                 return false;
 273         if (fstat(fd, &st))
 274                 goto fail;
 275         data = malloc(st.st_size);
 276         if (!data)
 277                 goto fail;
 278         if(read(fd, data, st.st_size) == -1)
 279                 goto fail;
 280
 281         return radv_pipeline_cache_load(&device->meta_state.cache, data, st.st_size);
 282 fail:
 283         free(data);
 284         close(fd);
 285         return false;
 286 }
 287
 288 static void
 289 radv_store_meta_pipeline(struct radv_device *device)
 290 {
 291         char path[PATH_MAX + 1], path2[PATH_MAX + 7];
 292         size_t size;
 293         void *data = NULL;
 294
 295         if (!device->meta_state.cache.modified)
 296                 return;
 297
 298         if (radv_GetPipelineCacheData(radv_device_to_handle(device),
 299                                       radv_pipeline_cache_to_handle(&device->meta_state.cache),
 300                                       &size, NULL))
 301                 return;
 302
 303         if (!radv_builtin_cache_path(path))
 304                 return;
 305
 306         strcpy(path2, path);
 307         strcat(path2, "XXXXXX");
 308         int fd = mkstemp(path2);//open(path, O_WRONLY | O_CREAT, 0600);
 309         if (fd < 0)
 310                 return;
 311         data = malloc(size);
 312         if (!data)
 313                 goto fail;
 314
 315         if (radv_GetPipelineCacheData(radv_device_to_handle(device),
 316                                       radv_pipeline_cache_to_handle(&device->meta_state.cache),
 317                                       &size, data))
 318                 goto fail;
 319         if(write(fd, data, size) == -1)
 320                 goto fail;
 321
 322         rename(path2, path);
 323 fail:
 324         free(data);
 325         close(fd);
 326         unlink(path2);
 327 }
 328
 329 VkResult
 330 radv_device_init_meta(struct radv_device *device)
 331 {
 332         VkResult result;
 333
 334         memset(&device->meta_state, 0, sizeof(device->meta_state));
 335
 336         device->meta_state.alloc = (VkAllocationCallbacks) {
 337                 .pUserData = device,
 338                 .pfnAllocation = meta_alloc,
 339                 .pfnReallocation = meta_realloc,
 340                 .pfnFree = meta_free,
 341         };
 342
 343         device->meta_state.cache.alloc = device->meta_state.alloc;
 344         radv_pipeline_cache_init(&device->meta_state.cache, device);
 345         bool loaded_cache = radv_load_meta_pipeline(device);
 346         bool on_demand = !loaded_cache;
 347
 348         mtx_init(&device->meta_state.mtx, mtx_plain);
 349
 350         result = radv_device_init_meta_clear_state(device, on_demand);
 351         if (result != VK_SUCCESS)
 352                 goto fail_clear;
 353
 354         result = radv_device_init_meta_resolve_state(device, on_demand);
 355         if (result != VK_SUCCESS)
 356                 goto fail_resolve;
 357
 358         result = radv_device_init_meta_blit_state(device, on_demand);
 359         if (result != VK_SUCCESS)
 360                 goto fail_blit;
 361
 362         result = radv_device_init_meta_blit2d_state(device, on_demand);
 363         if (result != VK_SUCCESS)
 364                 goto fail_blit2d;
 365
 366         result = radv_device_init_meta_bufimage_state(device);
 367         if (result != VK_SUCCESS)
 368                 goto fail_bufimage;
 369
 370         result = radv_device_init_meta_depth_decomp_state(device, on_demand);
 371         if (result != VK_SUCCESS)
 372                 goto fail_depth_decomp;
 373
 374         result = radv_device_init_meta_buffer_state(device);
 375         if (result != VK_SUCCESS)
 376                 goto fail_buffer;
 377
 378         result = radv_device_init_meta_query_state(device, on_demand);
 379         if (result != VK_SUCCESS)
 380                 goto fail_query;
 381
 382         result = radv_device_init_meta_fast_clear_flush_state(device, on_demand);
 383         if (result != VK_SUCCESS)
 384                 goto fail_fast_clear;
 385
 386         result = radv_device_init_meta_resolve_compute_state(device, on_demand);
 387         if (result != VK_SUCCESS)
 388                 goto fail_resolve_compute;
 389
 390         result = radv_device_init_meta_resolve_fragment_state(device, on_demand);
 391         if (result != VK_SUCCESS)
 392                 goto fail_resolve_fragment;
 393         return VK_SUCCESS;
 394
 395 fail_resolve_fragment:
 396         radv_device_finish_meta_resolve_compute_state(device);
 397 fail_resolve_compute:
 398         radv_device_finish_meta_fast_clear_flush_state(device);
 399 fail_fast_clear:
 400         radv_device_finish_meta_query_state(device);
 401 fail_query:
 402         radv_device_finish_meta_buffer_state(device);
 403 fail_buffer:
 404         radv_device_finish_meta_depth_decomp_state(device);
 405 fail_depth_decomp:
 406         radv_device_finish_meta_bufimage_state(device);
 407 fail_bufimage:
 408         radv_device_finish_meta_blit2d_state(device);
 409 fail_blit2d:
 410         radv_device_finish_meta_blit_state(device);
 411 fail_blit:
 412         radv_device_finish_meta_resolve_state(device);
 413 fail_resolve:
 414         radv_device_finish_meta_clear_state(device);
 415 fail_clear:
 416         mtx_destroy(&device->meta_state.mtx);
 417         radv_pipeline_cache_finish(&device->meta_state.cache);
 418         return result;
 419 }
 420
 421 void
 422 radv_device_finish_meta(struct radv_device *device)
 423 {
 424         radv_device_finish_meta_clear_state(device);
 425         radv_device_finish_meta_resolve_state(device);
 426         radv_device_finish_meta_blit_state(device);
 427         radv_device_finish_meta_blit2d_state(device);
 428         radv_device_finish_meta_bufimage_state(device);
 429         radv_device_finish_meta_depth_decomp_state(device);
 430         radv_device_finish_meta_query_state(device);
 431         radv_device_finish_meta_buffer_state(device);
 432         radv_device_finish_meta_fast_clear_flush_state(device);
 433         radv_device_finish_meta_resolve_compute_state(device);
 434         radv_device_finish_meta_resolve_fragment_state(device);
 435
 436         radv_store_meta_pipeline(device);
 437         radv_pipeline_cache_finish(&device->meta_state.cache);
 438         mtx_destroy(&device->meta_state.mtx);
 439 }
 440
 441 nir_ssa_def *radv_meta_gen_rect_vertices_comp2(nir_builder *vs_b, nir_ssa_def *comp2)
 442 {
 443
 444         nir_intrinsic_instr *vertex_id = nir_intrinsic_instr_create(vs_b->shader, nir_intrinsic_load_vertex_id_zero_base);
 445         nir_ssa_dest_init(&vertex_id->instr, &vertex_id->dest, 1, 32, "vertexid");
 446         nir_builder_instr_insert(vs_b, &vertex_id->instr);
 447
 448         /* vertex 0 - -1.0, -1.0 */
 449         /* vertex 1 - -1.0, 1.0 */
 450         /* vertex 2 - 1.0, -1.0 */
 451         /* so channel 0 is vertex_id != 2 ? -1.0 : 1.0
 452            channel 1 is vertex id != 1 ? -1.0 : 1.0 */
 453
 454         nir_ssa_def *c0cmp = nir_ine(vs_b, &vertex_id->dest.ssa,
 455                                      nir_imm_int(vs_b, 2));
 456         nir_ssa_def *c1cmp = nir_ine(vs_b, &vertex_id->dest.ssa,
 457                                      nir_imm_int(vs_b, 1));
 458
 459         nir_ssa_def *comp[4];
 460         comp[0] = nir_bcsel(vs_b, c0cmp,
 461                             nir_imm_float(vs_b, -1.0),
 462                             nir_imm_float(vs_b, 1.0));
 463
 464         comp[1] = nir_bcsel(vs_b, c1cmp,
 465                             nir_imm_float(vs_b, -1.0),
 466                             nir_imm_float(vs_b, 1.0));
 467         comp[2] = comp2;
 468         comp[3] = nir_imm_float(vs_b, 1.0);
 469         nir_ssa_def *outvec = nir_vec(vs_b, comp, 4);
 470
 471         return outvec;
 472 }
 473
 474 nir_ssa_def *radv_meta_gen_rect_vertices(nir_builder *vs_b)
 475 {
 476         return radv_meta_gen_rect_vertices_comp2(vs_b, nir_imm_float(vs_b, 0.0));
 477 }
 478
 479 /* vertex shader that generates vertices */
 480 nir_shader *
 481 radv_meta_build_nir_vs_generate_vertices(void)
 482 {
 483         const struct glsl_type *vec4 = glsl_vec4_type();
 484
 485         nir_builder b;
 486         nir_variable *v_position;
 487
 488         nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_VERTEX, NULL);
 489         b.shader->info.name = ralloc_strdup(b.shader, "meta_vs_gen_verts");
 490
 491         nir_ssa_def *outvec = radv_meta_gen_rect_vertices(&b);
 492
 493         v_position = nir_variable_create(b.shader, nir_var_shader_out, vec4,
 494                                          "gl_Position");
 495         v_position->data.location = VARYING_SLOT_POS;
 496
 497         nir_store_var(&b, v_position, outvec, 0xf);
 498
 499         return b.shader;
 500 }
 501
 502 nir_shader *
 503 radv_meta_build_nir_fs_noop(void)
 504 {
 505         nir_builder b;
 506
 507         nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_FRAGMENT, NULL);
 508         b.shader->info.name = ralloc_asprintf(b.shader,
 509                                                "meta_noop_fs");
 510
 511         return b.shader;
 512 }
 513
 514 void radv_meta_build_resolve_shader_core(nir_builder *b,
 515                                          bool is_integer,
 516                                          int samples,
 517                                          nir_variable *input_img,
 518                                          nir_variable *color,
 519                                          nir_ssa_def *img_coord)
 520 {
 521         /* do a txf_ms on each sample */
 522         nir_ssa_def *tmp;
 523         nir_if *outer_if = NULL;
 524
 525         nir_ssa_def *input_img_deref = &nir_build_deref_var(b, input_img)->dest.ssa;
 526
 527         nir_tex_instr *tex = nir_tex_instr_create(b->shader, 3);
 528         tex->sampler_dim = GLSL_SAMPLER_DIM_MS;
 529         tex->op = nir_texop_txf_ms;
 530         tex->src[0].src_type = nir_tex_src_coord;
 531         tex->src[0].src = nir_src_for_ssa(img_coord);
 532         tex->src[1].src_type = nir_tex_src_ms_index;
 533         tex->src[1].src = nir_src_for_ssa(nir_imm_int(b, 0));
 534         tex->src[2].src_type = nir_tex_src_texture_deref;
 535         tex->src[2].src = nir_src_for_ssa(input_img_deref);
 536         tex->dest_type = nir_type_float;
 537         tex->is_array = false;
 538         tex->coord_components = 2;
 539
 540         nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex");
 541         nir_builder_instr_insert(b, &tex->instr);
 542
 543         tmp = &tex->dest.ssa;
 544
 545         if (!is_integer && samples > 1) {
 546                 nir_tex_instr *tex_all_same = nir_tex_instr_create(b->shader, 2);
 547                 tex_all_same->sampler_dim = GLSL_SAMPLER_DIM_MS;
 548                 tex_all_same->op = nir_texop_samples_identical;
 549                 tex_all_same->src[0].src_type = nir_tex_src_coord;
 550                 tex_all_same->src[0].src = nir_src_for_ssa(img_coord);
 551                 tex_all_same->src[1].src_type = nir_tex_src_texture_deref;
 552                 tex_all_same->src[1].src = nir_src_for_ssa(input_img_deref);
 553                 tex_all_same->dest_type = nir_type_float;
 554                 tex_all_same->is_array = false;
 555                 tex_all_same->coord_components = 2;
 556
 557                 nir_ssa_dest_init(&tex_all_same->instr, &tex_all_same->dest, 1, 32, "tex");
 558                 nir_builder_instr_insert(b, &tex_all_same->instr);
 559
 560                 nir_ssa_def *all_same = nir_ieq(b, &tex_all_same->dest.ssa, nir_imm_int(b, 0));
 561                 nir_if *if_stmt = nir_if_create(b->shader);
 562                 if_stmt->condition = nir_src_for_ssa(all_same);
 563                 nir_cf_node_insert(b->cursor, &if_stmt->cf_node);
 564
 565                 b->cursor = nir_after_cf_list(&if_stmt->then_list);
 566                 for (int i = 1; i < samples; i++) {
 567                         nir_tex_instr *tex_add = nir_tex_instr_create(b->shader, 3);
 568                         tex_add->sampler_dim = GLSL_SAMPLER_DIM_MS;
 569                         tex_add->op = nir_texop_txf_ms;
 570                         tex_add->src[0].src_type = nir_tex_src_coord;
 571                         tex_add->src[0].src = nir_src_for_ssa(img_coord);
 572                         tex_add->src[1].src_type = nir_tex_src_ms_index;
 573                         tex_add->src[1].src = nir_src_for_ssa(nir_imm_int(b, i));
 574                         tex_add->src[2].src_type = nir_tex_src_texture_deref;
 575                         tex_add->src[2].src = nir_src_for_ssa(input_img_deref);
 576                         tex_add->dest_type = nir_type_float;
 577                         tex_add->is_array = false;
 578                         tex_add->coord_components = 2;
 579
 580                         nir_ssa_dest_init(&tex_add->instr, &tex_add->dest, 4, 32, "tex");
 581                         nir_builder_instr_insert(b, &tex_add->instr);
 582
 583                         tmp = nir_fadd(b, tmp, &tex_add->dest.ssa);
 584                 }
 585
 586                 tmp = nir_fdiv(b, tmp, nir_imm_float(b, samples));
 587                 nir_store_var(b, color, tmp, 0xf);
 588                 b->cursor = nir_after_cf_list(&if_stmt->else_list);
 589                 outer_if = if_stmt;
 590         }
 591         nir_store_var(b, color, &tex->dest.ssa, 0xf);
 592
 593         if (outer_if)
 594                 b->cursor = nir_after_cf_node(&outer_if->cf_node);
 595 }