src/amd/vulkan/radv_meta.c

   1 /*
   2  * Copyright © 2016 Red Hat
   3  * based on intel anv code:
   4  * Copyright © 2015 Intel Corporation
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the next
  14  * paragraph) shall be included in all copies or substantial portions of the
  15  * Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  20  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  22  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  23  * IN THE SOFTWARE.
  24  */
  25
  26 #include "radv_meta.h"
  27
  28 #include <fcntl.h>
  29 #include <limits.h>
  30 #include <pwd.h>
  31 #include <sys/stat.h>
  32
  33 void
  34 radv_meta_save(struct radv_meta_saved_state *state,
  35                struct radv_cmd_buffer *cmd_buffer, uint32_t flags)
  36 {
  37         state->flags = flags;
  38
  39         if (state->flags & RADV_META_SAVE_GRAPHICS_PIPELINE) {
  40                 state->old_pipeline = cmd_buffer->state.pipeline;
  41
  42                 /* Save all viewports. */
  43                 state->viewport.count = cmd_buffer->state.dynamic.viewport.count;
  44                 typed_memcpy(state->viewport.viewports,
  45                              cmd_buffer->state.dynamic.viewport.viewports,
  46                              MAX_VIEWPORTS);
  47
  48                 /* Save all scissors. */
  49                 state->scissor.count = cmd_buffer->state.dynamic.scissor.count;
  50                 typed_memcpy(state->scissor.scissors,
  51                              cmd_buffer->state.dynamic.scissor.scissors,
  52                              MAX_SCISSORS);
  53
  54                 /* The most common meta operations all want to have the
  55                  * viewport reset and any scissors disabled. The rest of the
  56                  * dynamic state should have no effect.
  57                  */
  58                 cmd_buffer->state.dynamic.viewport.count = 0;
  59                 cmd_buffer->state.dynamic.scissor.count = 0;
  60                 cmd_buffer->state.dirty |= 1 << VK_DYNAMIC_STATE_VIEWPORT |
  61                                            1 << VK_DYNAMIC_STATE_SCISSOR;
  62         }
  63
  64         if (state->flags & RADV_META_SAVE_DESCRIPTORS) {
  65                 state->old_descriptor_set0 = cmd_buffer->state.descriptors[0];
  66         }
  67
  68         if (state->flags & RADV_META_SAVE_CONSTANTS) {
  69                 memcpy(state->push_constants, cmd_buffer->push_constants,
  70                        MAX_PUSH_CONSTANTS_SIZE);
  71         }
  72
  73         if (state->flags & RADV_META_SAVE_PASS) {
  74                 state->pass = cmd_buffer->state.pass;
  75                 state->subpass = cmd_buffer->state.subpass;
  76                 state->framebuffer = cmd_buffer->state.framebuffer;
  77                 state->attachments = cmd_buffer->state.attachments;
  78                 state->render_area = cmd_buffer->state.render_area;
  79         }
  80 }
  81
  82 void
  83 radv_meta_restore(const struct radv_meta_saved_state *state,
  84                   struct radv_cmd_buffer *cmd_buffer)
  85 {
  86         if (state->flags & RADV_META_SAVE_GRAPHICS_PIPELINE) {
  87                 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
  88                                      VK_PIPELINE_BIND_POINT_GRAPHICS,
  89                                      radv_pipeline_to_handle(state->old_pipeline));
  90
  91                 cmd_buffer->state.dirty |= RADV_CMD_DIRTY_PIPELINE;
  92
  93                 /* Restore all viewports. */
  94                 cmd_buffer->state.dynamic.viewport.count = state->viewport.count;
  95                 typed_memcpy(cmd_buffer->state.dynamic.viewport.viewports,
  96                              state->viewport.viewports,
  97                              MAX_VIEWPORTS);
  98
  99                 /* Restore all scissors. */
 100                 cmd_buffer->state.dynamic.scissor.count = state->scissor.count;
 101                 typed_memcpy(cmd_buffer->state.dynamic.scissor.scissors,
 102                              state->scissor.scissors,
 103                              MAX_SCISSORS);
 104
 105                 cmd_buffer->state.dirty |= 1 << VK_DYNAMIC_STATE_VIEWPORT |
 106                                            1 << VK_DYNAMIC_STATE_SCISSOR;
 107         }
 108
 109         if (state->flags & RADV_META_SAVE_DESCRIPTORS) {
 110                 cmd_buffer->state.descriptors[0] = state->old_descriptor_set0;
 111                 cmd_buffer->state.descriptors_dirty |= (1 << 0);
 112         }
 113
 114         if (state->flags & RADV_META_SAVE_CONSTANTS) {
 115                 memcpy(cmd_buffer->push_constants, state->push_constants,
 116                        MAX_PUSH_CONSTANTS_SIZE);
 117                 cmd_buffer->push_constant_stages |= VK_SHADER_STAGE_ALL_GRAPHICS |
 118                                                     VK_SHADER_STAGE_COMPUTE_BIT;
 119         }
 120
 121         if (state->flags & RADV_META_SAVE_PASS) {
 122                 cmd_buffer->state.pass = state->pass;
 123                 cmd_buffer->state.subpass = state->subpass;
 124                 cmd_buffer->state.framebuffer = state->framebuffer;
 125                 cmd_buffer->state.attachments = state->attachments;
 126                 cmd_buffer->state.render_area = state->render_area;
 127                 if (state->subpass)
 128                         radv_emit_framebuffer_state(cmd_buffer);
 129         }
 130 }
 131
 132 void
 133 radv_meta_save_compute(struct radv_meta_saved_compute_state *state,
 134                        const struct radv_cmd_buffer *cmd_buffer,
 135                        unsigned push_constant_size)
 136 {
 137         state->old_pipeline = cmd_buffer->state.compute_pipeline;
 138         state->old_descriptor_set0 = cmd_buffer->state.descriptors[0];
 139         state->push_constant_size = push_constant_size;
 140
 141         if (state->push_constant_size) {
 142                 memcpy(state->push_constants, cmd_buffer->push_constants,
 143                        state->push_constant_size);
 144         }
 145 }
 146
 147 void
 148 radv_meta_restore_compute(const struct radv_meta_saved_compute_state *state,
 149                           struct radv_cmd_buffer *cmd_buffer)
 150 {
 151         radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer), VK_PIPELINE_BIND_POINT_COMPUTE,
 152                              radv_pipeline_to_handle(state->old_pipeline));
 153
 154         cmd_buffer->state.descriptors[0] = state->old_descriptor_set0;
 155
 156         if (state->push_constant_size) {
 157                 memcpy(cmd_buffer->push_constants, state->push_constants,
 158                        state->push_constant_size);
 159                 cmd_buffer->push_constant_stages |= VK_SHADER_STAGE_COMPUTE_BIT;
 160         }
 161 }
 162
 163 VkImageViewType
 164 radv_meta_get_view_type(const struct radv_image *image)
 165 {
 166         switch (image->type) {
 167         case VK_IMAGE_TYPE_1D: return VK_IMAGE_VIEW_TYPE_1D;
 168         case VK_IMAGE_TYPE_2D: return VK_IMAGE_VIEW_TYPE_2D;
 169         case VK_IMAGE_TYPE_3D: return VK_IMAGE_VIEW_TYPE_3D;
 170         default:
 171                 unreachable("bad VkImageViewType");
 172         }
 173 }
 174
 175 /**
 176  * When creating a destination VkImageView, this function provides the needed
 177  * VkImageViewCreateInfo::subresourceRange::baseArrayLayer.
 178  */
 179 uint32_t
 180 radv_meta_get_iview_layer(const struct radv_image *dest_image,
 181                           const VkImageSubresourceLayers *dest_subresource,
 182                           const VkOffset3D *dest_offset)
 183 {
 184         switch (dest_image->type) {
 185         case VK_IMAGE_TYPE_1D:
 186         case VK_IMAGE_TYPE_2D:
 187                 return dest_subresource->baseArrayLayer;
 188         case VK_IMAGE_TYPE_3D:
 189                 /* HACK: Vulkan does not allow attaching a 3D image to a framebuffer,
 190                  * but meta does it anyway. When doing so, we translate the
 191                  * destination's z offset into an array offset.
 192                  */
 193                 return dest_offset->z;
 194         default:
 195                 assert(!"bad VkImageType");
 196                 return 0;
 197         }
 198 }
 199
 200 static void *
 201 meta_alloc(void* _device, size_t size, size_t alignment,
 202            VkSystemAllocationScope allocationScope)
 203 {
 204         struct radv_device *device = _device;
 205         return device->alloc.pfnAllocation(device->alloc.pUserData, size, alignment,
 206                                            VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
 207 }
 208
 209 static void *
 210 meta_realloc(void* _device, void *original, size_t size, size_t alignment,
 211              VkSystemAllocationScope allocationScope)
 212 {
 213         struct radv_device *device = _device;
 214         return device->alloc.pfnReallocation(device->alloc.pUserData, original,
 215                                              size, alignment,
 216                                              VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
 217 }
 218
 219 static void
 220 meta_free(void* _device, void *data)
 221 {
 222         struct radv_device *device = _device;
 223         return device->alloc.pfnFree(device->alloc.pUserData, data);
 224 }
 225
 226 static bool
 227 radv_builtin_cache_path(char *path)
 228 {
 229         char *xdg_cache_home = getenv("XDG_CACHE_HOME");
 230         const char *suffix = "/radv_builtin_shaders";
 231         const char *suffix2 = "/.cache/radv_builtin_shaders";
 232         struct passwd pwd, *result;
 233         char path2[PATH_MAX + 1]; /* PATH_MAX is not a real max,but suffices here. */
 234
 235         if (xdg_cache_home) {
 236
 237                 if (strlen(xdg_cache_home) + strlen(suffix) > PATH_MAX)
 238                         return false;
 239
 240                 strcpy(path, xdg_cache_home);
 241                 strcat(path, suffix);
 242                 return true;
 243         }
 244
 245         getpwuid_r(getuid(), &pwd, path2, PATH_MAX - strlen(suffix2), &result);
 246         if (!result)
 247                 return false;
 248
 249         strcpy(path, pwd.pw_dir);
 250         strcat(path, "/.cache");
 251         mkdir(path, 0755);
 252
 253         strcat(path, suffix);
 254         return true;
 255 }
 256
 257 static void
 258 radv_load_meta_pipeline(struct radv_device *device)
 259 {
 260         char path[PATH_MAX + 1];
 261         struct stat st;
 262         void *data = NULL;
 263
 264         if (!radv_builtin_cache_path(path))
 265                 return;
 266
 267         int fd = open(path, O_RDONLY);
 268         if (fd < 0)
 269                 return;
 270         if (fstat(fd, &st))
 271                 goto fail;
 272         data = malloc(st.st_size);
 273         if (!data)
 274                 goto fail;
 275         if(read(fd, data, st.st_size) == -1)
 276                 goto fail;
 277
 278         radv_pipeline_cache_load(&device->meta_state.cache, data, st.st_size);
 279 fail:
 280         free(data);
 281         close(fd);
 282 }
 283
 284 static void
 285 radv_store_meta_pipeline(struct radv_device *device)
 286 {
 287         char path[PATH_MAX + 1], path2[PATH_MAX + 7];
 288         size_t size;
 289         void *data = NULL;
 290
 291         if (!device->meta_state.cache.modified)
 292                 return;
 293
 294         if (radv_GetPipelineCacheData(radv_device_to_handle(device),
 295                                       radv_pipeline_cache_to_handle(&device->meta_state.cache),
 296                                       &size, NULL))
 297                 return;
 298
 299         if (!radv_builtin_cache_path(path))
 300                 return;
 301
 302         strcpy(path2, path);
 303         strcat(path2, "XXXXXX");
 304         int fd = mkstemp(path2);//open(path, O_WRONLY | O_CREAT, 0600);
 305         if (fd < 0)
 306                 return;
 307         data = malloc(size);
 308         if (!data)
 309                 goto fail;
 310
 311         if (radv_GetPipelineCacheData(radv_device_to_handle(device),
 312                                       radv_pipeline_cache_to_handle(&device->meta_state.cache),
 313                                       &size, data))
 314                 goto fail;
 315         if(write(fd, data, size) == -1)
 316                 goto fail;
 317
 318         rename(path2, path);
 319 fail:
 320         free(data);
 321         close(fd);
 322         unlink(path2);
 323 }
 324
 325 VkResult
 326 radv_device_init_meta(struct radv_device *device)
 327 {
 328         VkResult result;
 329
 330         device->meta_state.alloc = (VkAllocationCallbacks) {
 331                 .pUserData = device,
 332                 .pfnAllocation = meta_alloc,
 333                 .pfnReallocation = meta_realloc,
 334                 .pfnFree = meta_free,
 335         };
 336
 337         device->meta_state.cache.alloc = device->meta_state.alloc;
 338         radv_pipeline_cache_init(&device->meta_state.cache, device);
 339         radv_load_meta_pipeline(device);
 340
 341         result = radv_device_init_meta_clear_state(device);
 342         if (result != VK_SUCCESS)
 343                 goto fail_clear;
 344
 345         result = radv_device_init_meta_resolve_state(device);
 346         if (result != VK_SUCCESS)
 347                 goto fail_resolve;
 348
 349         result = radv_device_init_meta_blit_state(device);
 350         if (result != VK_SUCCESS)
 351                 goto fail_blit;
 352
 353         result = radv_device_init_meta_blit2d_state(device);
 354         if (result != VK_SUCCESS)
 355                 goto fail_blit2d;
 356
 357         result = radv_device_init_meta_bufimage_state(device);
 358         if (result != VK_SUCCESS)
 359                 goto fail_bufimage;
 360
 361         result = radv_device_init_meta_depth_decomp_state(device);
 362         if (result != VK_SUCCESS)
 363                 goto fail_depth_decomp;
 364
 365         result = radv_device_init_meta_buffer_state(device);
 366         if (result != VK_SUCCESS)
 367                 goto fail_buffer;
 368
 369         result = radv_device_init_meta_query_state(device);
 370         if (result != VK_SUCCESS)
 371                 goto fail_query;
 372
 373         result = radv_device_init_meta_fast_clear_flush_state(device);
 374         if (result != VK_SUCCESS)
 375                 goto fail_fast_clear;
 376
 377         result = radv_device_init_meta_resolve_compute_state(device);
 378         if (result != VK_SUCCESS)
 379                 goto fail_resolve_compute;
 380
 381         result = radv_device_init_meta_resolve_fragment_state(device);
 382         if (result != VK_SUCCESS)
 383                 goto fail_resolve_fragment;
 384         return VK_SUCCESS;
 385
 386 fail_resolve_fragment:
 387         radv_device_finish_meta_resolve_compute_state(device);
 388 fail_resolve_compute:
 389         radv_device_finish_meta_fast_clear_flush_state(device);
 390 fail_fast_clear:
 391         radv_device_finish_meta_buffer_state(device);
 392 fail_query:
 393         radv_device_finish_meta_query_state(device);
 394 fail_buffer:
 395         radv_device_finish_meta_depth_decomp_state(device);
 396 fail_depth_decomp:
 397         radv_device_finish_meta_bufimage_state(device);
 398 fail_bufimage:
 399         radv_device_finish_meta_blit2d_state(device);
 400 fail_blit2d:
 401         radv_device_finish_meta_blit_state(device);
 402 fail_blit:
 403         radv_device_finish_meta_resolve_state(device);
 404 fail_resolve:
 405         radv_device_finish_meta_clear_state(device);
 406 fail_clear:
 407         radv_pipeline_cache_finish(&device->meta_state.cache);
 408         return result;
 409 }
 410
 411 void
 412 radv_device_finish_meta(struct radv_device *device)
 413 {
 414         radv_device_finish_meta_clear_state(device);
 415         radv_device_finish_meta_resolve_state(device);
 416         radv_device_finish_meta_blit_state(device);
 417         radv_device_finish_meta_blit2d_state(device);
 418         radv_device_finish_meta_bufimage_state(device);
 419         radv_device_finish_meta_depth_decomp_state(device);
 420         radv_device_finish_meta_query_state(device);
 421         radv_device_finish_meta_buffer_state(device);
 422         radv_device_finish_meta_fast_clear_flush_state(device);
 423         radv_device_finish_meta_resolve_compute_state(device);
 424         radv_device_finish_meta_resolve_fragment_state(device);
 425
 426         radv_store_meta_pipeline(device);
 427         radv_pipeline_cache_finish(&device->meta_state.cache);
 428 }
 429
 430 nir_ssa_def *radv_meta_gen_rect_vertices_comp2(nir_builder *vs_b, nir_ssa_def *comp2)
 431 {
 432
 433         nir_intrinsic_instr *vertex_id = nir_intrinsic_instr_create(vs_b->shader, nir_intrinsic_load_vertex_id_zero_base);
 434         nir_ssa_dest_init(&vertex_id->instr, &vertex_id->dest, 1, 32, "vertexid");
 435         nir_builder_instr_insert(vs_b, &vertex_id->instr);
 436
 437         /* vertex 0 - -1.0, -1.0 */
 438         /* vertex 1 - -1.0, 1.0 */
 439         /* vertex 2 - 1.0, -1.0 */
 440         /* so channel 0 is vertex_id != 2 ? -1.0 : 1.0
 441            channel 1 is vertex id != 1 ? -1.0 : 1.0 */
 442
 443         nir_ssa_def *c0cmp = nir_ine(vs_b, &vertex_id->dest.ssa,
 444                                      nir_imm_int(vs_b, 2));
 445         nir_ssa_def *c1cmp = nir_ine(vs_b, &vertex_id->dest.ssa,
 446                                      nir_imm_int(vs_b, 1));
 447
 448         nir_ssa_def *comp[4];
 449         comp[0] = nir_bcsel(vs_b, c0cmp,
 450                             nir_imm_float(vs_b, -1.0),
 451                             nir_imm_float(vs_b, 1.0));
 452
 453         comp[1] = nir_bcsel(vs_b, c1cmp,
 454                             nir_imm_float(vs_b, -1.0),
 455                             nir_imm_float(vs_b, 1.0));
 456         comp[2] = comp2;
 457         comp[3] = nir_imm_float(vs_b, 1.0);
 458         nir_ssa_def *outvec = nir_vec(vs_b, comp, 4);
 459
 460         return outvec;
 461 }
 462
 463 nir_ssa_def *radv_meta_gen_rect_vertices(nir_builder *vs_b)
 464 {
 465         return radv_meta_gen_rect_vertices_comp2(vs_b, nir_imm_float(vs_b, 0.0));
 466 }
 467
 468 /* vertex shader that generates vertices */
 469 nir_shader *
 470 radv_meta_build_nir_vs_generate_vertices(void)
 471 {
 472         const struct glsl_type *vec4 = glsl_vec4_type();
 473
 474         nir_builder b;
 475         nir_variable *v_position;
 476
 477         nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_VERTEX, NULL);
 478         b.shader->info.name = ralloc_strdup(b.shader, "meta_vs_gen_verts");
 479
 480         nir_ssa_def *outvec = radv_meta_gen_rect_vertices(&b);
 481
 482         v_position = nir_variable_create(b.shader, nir_var_shader_out, vec4,
 483                                          "gl_Position");
 484         v_position->data.location = VARYING_SLOT_POS;
 485
 486         nir_store_var(&b, v_position, outvec, 0xf);
 487
 488         return b.shader;
 489 }
 490
 491 nir_shader *
 492 radv_meta_build_nir_fs_noop(void)
 493 {
 494         nir_builder b;
 495
 496         nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_FRAGMENT, NULL);
 497         b.shader->info.name = ralloc_asprintf(b.shader,
 498                                                "meta_noop_fs");
 499
 500         return b.shader;
 501 }
 502
 503 void radv_meta_build_resolve_shader_core(nir_builder *b,
 504                                          bool is_integer,
 505                                          int samples,
 506                                          nir_variable *input_img,
 507                                          nir_variable *color,
 508                                          nir_ssa_def *img_coord)
 509 {
 510         /* do a txf_ms on each sample */
 511         nir_ssa_def *tmp;
 512         nir_if *outer_if = NULL;
 513
 514         nir_tex_instr *tex = nir_tex_instr_create(b->shader, 2);
 515         tex->sampler_dim = GLSL_SAMPLER_DIM_MS;
 516         tex->op = nir_texop_txf_ms;
 517         tex->src[0].src_type = nir_tex_src_coord;
 518         tex->src[0].src = nir_src_for_ssa(img_coord);
 519         tex->src[1].src_type = nir_tex_src_ms_index;
 520         tex->src[1].src = nir_src_for_ssa(nir_imm_int(b, 0));
 521         tex->dest_type = nir_type_float;
 522         tex->is_array = false;
 523         tex->coord_components = 2;
 524         tex->texture = nir_deref_var_create(tex, input_img);
 525         tex->sampler = NULL;
 526
 527         nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex");
 528         nir_builder_instr_insert(b, &tex->instr);
 529
 530         tmp = &tex->dest.ssa;
 531
 532         if (!is_integer && samples > 1) {
 533                 nir_tex_instr *tex_all_same = nir_tex_instr_create(b->shader, 1);
 534                 tex_all_same->sampler_dim = GLSL_SAMPLER_DIM_MS;
 535                 tex_all_same->op = nir_texop_samples_identical;
 536                 tex_all_same->src[0].src_type = nir_tex_src_coord;
 537                 tex_all_same->src[0].src = nir_src_for_ssa(img_coord);
 538                 tex_all_same->dest_type = nir_type_float;
 539                 tex_all_same->is_array = false;
 540                 tex_all_same->coord_components = 2;
 541                 tex_all_same->texture = nir_deref_var_create(tex_all_same, input_img);
 542                 tex_all_same->sampler = NULL;
 543
 544                 nir_ssa_dest_init(&tex_all_same->instr, &tex_all_same->dest, 1, 32, "tex");
 545                 nir_builder_instr_insert(b, &tex_all_same->instr);
 546
 547                 nir_ssa_def *all_same = nir_ine(b, &tex_all_same->dest.ssa, nir_imm_int(b, 0));
 548                 nir_if *if_stmt = nir_if_create(b->shader);
 549                 if_stmt->condition = nir_src_for_ssa(all_same);
 550                 nir_cf_node_insert(b->cursor, &if_stmt->cf_node);
 551
 552                 b->cursor = nir_after_cf_list(&if_stmt->then_list);
 553                 for (int i = 1; i < samples; i++) {
 554                         nir_tex_instr *tex_add = nir_tex_instr_create(b->shader, 2);
 555                         tex_add->sampler_dim = GLSL_SAMPLER_DIM_MS;
 556                         tex_add->op = nir_texop_txf_ms;
 557                         tex_add->src[0].src_type = nir_tex_src_coord;
 558                         tex_add->src[0].src = nir_src_for_ssa(img_coord);
 559                         tex_add->src[1].src_type = nir_tex_src_ms_index;
 560                         tex_add->src[1].src = nir_src_for_ssa(nir_imm_int(b, i));
 561                         tex_add->dest_type = nir_type_float;
 562                         tex_add->is_array = false;
 563                         tex_add->coord_components = 2;
 564                         tex_add->texture = nir_deref_var_create(tex_add, input_img);
 565                         tex_add->sampler = NULL;
 566
 567                         nir_ssa_dest_init(&tex_add->instr, &tex_add->dest, 4, 32, "tex");
 568                         nir_builder_instr_insert(b, &tex_add->instr);
 569
 570                         tmp = nir_fadd(b, tmp, &tex_add->dest.ssa);
 571                 }
 572
 573                 tmp = nir_fdiv(b, tmp, nir_imm_float(b, samples));
 574                 nir_store_var(b, color, tmp, 0xf);
 575                 b->cursor = nir_after_cf_list(&if_stmt->else_list);
 576                 outer_if = if_stmt;
 577         }
 578         nir_store_var(b, color, &tex->dest.ssa, 0xf);
 579
 580         if (outer_if)
 581                 b->cursor = nir_after_cf_node(&outer_if->cf_node);
 582 }