spirv: Make VertexIndex and VertexId both non-zero-based
[mesa.git] / src / compiler / spirv / vtn_cfg.c
index 6a43ef8b2dd92853c0ac6921cdee6736f78f67d9..ed1ab5d1c2c780ec1f7206bd848dd329f2780eb9 100644 (file)
 #include "vtn_private.h"
 #include "nir/nir_vla.h"
 
+static struct vtn_pointer *
+vtn_load_param_pointer(struct vtn_builder *b,
+                       struct vtn_type *param_type,
+                       uint32_t param_idx)
+{
+   struct vtn_type *ptr_type = param_type;
+   if (param_type->base_type != vtn_base_type_pointer) {
+      assert(param_type->base_type == vtn_base_type_image ||
+             param_type->base_type == vtn_base_type_sampler);
+      ptr_type = rzalloc(b, struct vtn_type);
+      ptr_type->base_type = vtn_base_type_pointer;
+      ptr_type->deref = param_type;
+      ptr_type->storage_class = SpvStorageClassUniformConstant;
+   }
+
+   return vtn_pointer_from_ssa(b, nir_load_param(&b->nb, param_idx), ptr_type);
+}
+
 static bool
 vtn_cfg_handle_prepass_instruction(struct vtn_builder *b, SpvOp opcode,
                                    const uint32_t *w, unsigned count)
 {
    switch (opcode) {
    case SpvOpFunction: {
-      assert(b->func == NULL);
+      vtn_assert(b->func == NULL);
       b->func = rzalloc(b, struct vtn_function);
 
       list_inithead(&b->func->body);
       b->func->control = w[3];
 
-      const struct glsl_type *result_type =
+      MAYBE_UNUSED const struct glsl_type *result_type =
          vtn_value(b, w[1], vtn_value_type_type)->type->type;
       struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_function);
       val->func = b->func;
 
-      const struct glsl_type *func_type =
-         vtn_value(b, w[4], vtn_value_type_type)->type->type;
+      b->func->type = vtn_value(b, w[4], vtn_value_type_type)->type;
+      const struct vtn_type *func_type = b->func->type;
 
-      assert(glsl_get_function_return_type(func_type) == result_type);
+      vtn_assert(func_type->return_type->type == result_type);
 
       nir_function *func =
          nir_function_create(b->shader, ralloc_strdup(b->shader, val->name));
 
-      func->num_params = glsl_get_length(func_type);
-      func->params = ralloc_array(b->shader, nir_parameter, func->num_params);
-      for (unsigned i = 0; i < func->num_params; i++) {
-         const struct glsl_function_param *param =
-            glsl_get_function_param(func_type, i);
-         func->params[i].type = param->type;
-         if (param->in) {
-            if (param->out) {
-               func->params[i].param_type = nir_parameter_inout;
-            } else {
-               func->params[i].param_type = nir_parameter_in;
-            }
+      unsigned num_params = func_type->length;
+      for (unsigned i = 0; i < func_type->length; i++) {
+         /* Sampled images are actually two parameters */
+         if (func_type->params[i]->base_type == vtn_base_type_sampled_image)
+            num_params++;
+      }
+
+      /* Add one parameter for the function return value */
+      if (func_type->return_type->base_type != vtn_base_type_void)
+         num_params++;
+
+      func->num_params = num_params;
+      func->params = ralloc_array(b->shader, nir_parameter, num_params);
+
+      unsigned idx = 0;
+      if (func_type->return_type->base_type != vtn_base_type_void) {
+         /* The return value is a regular pointer */
+         func->params[idx++] = (nir_parameter) {
+            .num_components = 1, .bit_size = 32,
+         };
+      }
+
+      for (unsigned i = 0; i < func_type->length; i++) {
+         if (func_type->params[i]->base_type == vtn_base_type_sampled_image) {
+            /* Sampled images are two pointer parameters */
+            func->params[idx++] = (nir_parameter) {
+               .num_components = 1, .bit_size = 32,
+            };
+            func->params[idx++] = (nir_parameter) {
+               .num_components = 1, .bit_size = 32,
+            };
+         } else if (func_type->params[i]->base_type == vtn_base_type_pointer &&
+                    func_type->params[i]->type != NULL) {
+            /* Pointers with as storage class get passed by-value */
+            assert(glsl_type_is_vector_or_scalar(func_type->params[i]->type));
+            func->params[idx++] = (nir_parameter) {
+               .num_components =
+                  glsl_get_vector_elements(func_type->params[i]->type),
+               .bit_size = glsl_get_bit_size(func_type->params[i]->type),
+            };
          } else {
-            if (param->out) {
-               func->params[i].param_type = nir_parameter_out;
-            } else {
-               assert(!"Parameter is neither in nor out");
-            }
+            /* Everything else is a regular pointer */
+            func->params[idx++] = (nir_parameter) {
+               .num_components = 1, .bit_size = 32,
+            };
          }
       }
-
-      func->return_type = glsl_get_function_return_type(func_type);
+      assert(idx == num_params);
 
       b->func->impl = nir_function_impl_create(func);
+      nir_builder_init(&b->nb, func->impl);
+      b->nb.cursor = nir_before_cf_list(&b->func->impl->body);
 
       b->func_param_idx = 0;
+
+      /* The return value is the first parameter */
+      if (func_type->return_type->base_type != vtn_base_type_void)
+         b->func_param_idx++;
       break;
    }
 
@@ -84,45 +134,54 @@ vtn_cfg_handle_prepass_instruction(struct vtn_builder *b, SpvOp opcode,
       break;
 
    case SpvOpFunctionParameter: {
-      struct vtn_value *val =
-         vtn_push_value(b, w[2], vtn_value_type_access_chain);
-
       struct vtn_type *type = vtn_value(b, w[1], vtn_value_type_type)->type;
 
-      assert(b->func_param_idx < b->func->impl->num_params);
-      nir_variable *param = b->func->impl->params[b->func_param_idx++];
-
-      assert(param->type == type->type);
-
-      /* Name the parameter so it shows up nicely in NIR */
-      param->name = ralloc_strdup(param, val->name);
+      vtn_assert(b->func_param_idx < b->func->impl->function->num_params);
 
-      struct vtn_variable *vtn_var = rzalloc(b, struct vtn_variable);
-      vtn_var->type = type;
-      vtn_var->var = param;
-      vtn_var->chain.var = vtn_var;
-      vtn_var->chain.length = 0;
-
-      struct vtn_type *without_array = type;
-      while(glsl_type_is_array(without_array->type))
-         without_array = without_array->array_element;
-
-      if (glsl_type_is_image(without_array->type)) {
-         vtn_var->mode = vtn_variable_mode_image;
-         param->interface_type = without_array->type;
-      } else if (glsl_type_is_sampler(without_array->type)) {
-         vtn_var->mode = vtn_variable_mode_sampler;
-         param->interface_type = without_array->type;
+      if (type->base_type == vtn_base_type_sampled_image) {
+         /* Sampled images are actually two parameters.  The first is the
+          * image and the second is the sampler.
+          */
+         struct vtn_value *val =
+            vtn_push_value(b, w[2], vtn_value_type_sampled_image);
+
+         val->sampled_image = ralloc(b, struct vtn_sampled_image);
+         val->sampled_image->type = type;
+
+         struct vtn_type *sampler_type = rzalloc(b, struct vtn_type);
+         sampler_type->base_type = vtn_base_type_sampler;
+         sampler_type->type = glsl_bare_sampler_type();
+
+         val->sampled_image->image =
+            vtn_load_param_pointer(b, type, b->func_param_idx++);
+         val->sampled_image->sampler =
+            vtn_load_param_pointer(b, sampler_type, b->func_param_idx++);
+      } else if (type->base_type == vtn_base_type_pointer &&
+                 type->type != NULL) {
+         /* This is a pointer with an actual storage type */
+         struct vtn_value *val =
+            vtn_push_value(b, w[2], vtn_value_type_pointer);
+         nir_ssa_def *ssa_ptr = nir_load_param(&b->nb, b->func_param_idx++);
+         val->pointer = vtn_pointer_from_ssa(b, ssa_ptr, type);
+      } else if (type->base_type == vtn_base_type_pointer ||
+                 type->base_type == vtn_base_type_image ||
+                 type->base_type == vtn_base_type_sampler) {
+         struct vtn_value *val =
+            vtn_push_value(b, w[2], vtn_value_type_pointer);
+         val->pointer =
+            vtn_load_param_pointer(b, type, b->func_param_idx++);
       } else {
-         vtn_var->mode = vtn_variable_mode_param;
+         /* We're a regular SSA value. */
+         nir_ssa_def *param_val = nir_load_param(&b->nb, b->func_param_idx++);
+         nir_deref_instr *deref =
+            nir_build_deref_cast(&b->nb, param_val, nir_var_local, type->type);
+         vtn_push_ssa(b, w[2], type, vtn_local_load(b, deref));
       }
-
-      val->access_chain = &vtn_var->chain;
       break;
    }
 
    case SpvOpLabel: {
-      assert(b->block == NULL);
+      vtn_assert(b->block == NULL);
       b->block = rzalloc(b, struct vtn_block);
       b->block->node.type = vtn_cf_node_type_block;
       b->block->label = w;
@@ -141,7 +200,7 @@ vtn_cfg_handle_prepass_instruction(struct vtn_builder *b, SpvOp opcode,
 
    case SpvOpSelectionMerge:
    case SpvOpLoopMerge:
-      assert(b->block && b->block->merge == NULL);
+      vtn_assert(b->block && b->block->merge == NULL);
       b->block->merge = w;
       break;
 
@@ -152,7 +211,7 @@ vtn_cfg_handle_prepass_instruction(struct vtn_builder *b, SpvOp opcode,
    case SpvOpReturn:
    case SpvOpReturnValue:
    case SpvOpUnreachable:
-      assert(b->block && b->block->branch == NULL);
+      vtn_assert(b->block && b->block->branch == NULL);
       b->block->branch = w;
       b->block = NULL;
       break;
@@ -168,7 +227,7 @@ vtn_cfg_handle_prepass_instruction(struct vtn_builder *b, SpvOp opcode,
 static void
 vtn_add_case(struct vtn_builder *b, struct vtn_switch *swtch,
              struct vtn_block *break_block,
-             uint32_t block_id, uint32_t val, bool is_default)
+             uint32_t block_id, uint64_t val, bool is_default)
 {
    struct vtn_block *case_block =
       vtn_value(b, block_id, vtn_value_type_block)->block;
@@ -183,7 +242,7 @@ vtn_add_case(struct vtn_builder *b, struct vtn_switch *swtch,
       list_inithead(&c->body);
       c->start_block = case_block;
       c->fallthrough = NULL;
-      nir_array_init(&c->values, b);
+      util_dynarray_init(&c->values, b);
       c->is_default = false;
       c->visited = false;
 
@@ -195,7 +254,7 @@ vtn_add_case(struct vtn_builder *b, struct vtn_switch *swtch,
    if (is_default) {
       case_block->switch_case->is_default = true;
    } else {
-      nir_array_add(&case_block->switch_case->values, uint32_t, val);
+      util_dynarray_append(&case_block->switch_case->values, uint64_t, val);
    }
 }
 
@@ -229,22 +288,23 @@ vtn_order_case(struct vtn_switch *swtch, struct vtn_case *cse)
 }
 
 static enum vtn_branch_type
-vtn_get_branch_type(struct vtn_block *block,
+vtn_get_branch_type(struct vtn_builder *b,
+                    struct vtn_block *block,
                     struct vtn_case *swcase, struct vtn_block *switch_break,
                     struct vtn_block *loop_break, struct vtn_block *loop_cont)
 {
    if (block->switch_case) {
       /* This branch is actually a fallthrough */
-      assert(swcase->fallthrough == NULL ||
-             swcase->fallthrough == block->switch_case);
+      vtn_assert(swcase->fallthrough == NULL ||
+                 swcase->fallthrough == block->switch_case);
       swcase->fallthrough = block->switch_case;
       return vtn_branch_type_switch_fallthrough;
-   } else if (block == switch_break) {
-      return vtn_branch_type_switch_break;
    } else if (block == loop_break) {
       return vtn_branch_type_loop_break;
    } else if (block == loop_cont) {
       return vtn_branch_type_loop_continue;
+   } else if (block == switch_break) {
+      return vtn_branch_type_switch_break;
    } else {
       return vtn_branch_type_none;
    }
@@ -295,11 +355,24 @@ vtn_cfg_walk_blocks(struct vtn_builder *b, struct list_head *cf_list,
          vtn_cfg_walk_blocks(b, &loop->cont_body, new_loop_cont, NULL, NULL,
                              new_loop_break, NULL, block);
 
+         enum vtn_branch_type branch_type =
+            vtn_get_branch_type(b, new_loop_break, switch_case, switch_break,
+                                loop_break, loop_cont);
+
+         if (branch_type != vtn_branch_type_none) {
+            /* Stop walking through the CFG when this inner loop's break block
+             * ends up as the same block as the outer loop's continue block
+             * because we are already going to visit it.
+             */
+            vtn_assert(branch_type == vtn_branch_type_loop_continue);
+            return;
+         }
+
          block = new_loop_break;
          continue;
       }
 
-      assert(block->node.link.next == NULL);
+      vtn_assert(block->node.link.next == NULL);
       list_addtail(&block->node.link, cf_list);
 
       switch (*block->branch & SpvOpCodeMask) {
@@ -307,7 +380,7 @@ vtn_cfg_walk_blocks(struct vtn_builder *b, struct list_head *cf_list,
          struct vtn_block *branch_block =
             vtn_value(b, block->branch[1], vtn_value_type_block)->block;
 
-         block->branch_type = vtn_get_branch_type(branch_block,
+         block->branch_type = vtn_get_branch_type(b, branch_block,
                                                   switch_case, switch_break,
                                                   loop_break, loop_cont);
 
@@ -347,17 +420,25 @@ vtn_cfg_walk_blocks(struct vtn_builder *b, struct list_head *cf_list,
             if_stmt->control = block->merge[2];
          }
 
-         if_stmt->then_type = vtn_get_branch_type(then_block,
+         if_stmt->then_type = vtn_get_branch_type(b, then_block,
                                                   switch_case, switch_break,
                                                   loop_break, loop_cont);
-         if_stmt->else_type = vtn_get_branch_type(else_block,
+         if_stmt->else_type = vtn_get_branch_type(b, else_block,
                                                   switch_case, switch_break,
                                                   loop_break, loop_cont);
 
-         if (if_stmt->then_type == vtn_branch_type_none &&
-             if_stmt->else_type == vtn_branch_type_none) {
+         if (then_block == else_block) {
+            block->branch_type = if_stmt->then_type;
+            if (block->branch_type == vtn_branch_type_none) {
+               block = then_block;
+               continue;
+            } else {
+               return;
+            }
+         } else if (if_stmt->then_type == vtn_branch_type_none &&
+                    if_stmt->else_type == vtn_branch_type_none) {
             /* Neither side of the if is something we can short-circuit. */
-            assert((*block->merge & SpvOpCodeMask) == SpvOpSelectionMerge);
+            vtn_assert((*block->merge & SpvOpCodeMask) == SpvOpSelectionMerge);
             struct vtn_block *merge_block =
                vtn_value(b, block->merge[1], vtn_value_type_block)->block;
 
@@ -369,7 +450,7 @@ vtn_cfg_walk_blocks(struct vtn_builder *b, struct list_head *cf_list,
                                 loop_break, loop_cont, merge_block);
 
             enum vtn_branch_type merge_type =
-               vtn_get_branch_type(merge_block, switch_case, switch_break,
+               vtn_get_branch_type(b, merge_block, switch_case, switch_break,
                                    loop_break, loop_cont);
             if (merge_type == vtn_branch_type_none) {
                block = merge_block;
@@ -394,11 +475,11 @@ vtn_cfg_walk_blocks(struct vtn_builder *b, struct list_head *cf_list,
             }
             continue;
          }
-         unreachable("Should have returned or continued");
+         vtn_fail("Should have returned or continued");
       }
 
       case SpvOpSwitch: {
-         assert((*block->merge & SpvOpCodeMask) == SpvOpSelectionMerge);
+         vtn_assert((*block->merge & SpvOpCodeMask) == SpvOpSelectionMerge);
          struct vtn_block *break_block =
             vtn_value(b, block->merge[1], vtn_value_type_block)->block;
 
@@ -414,35 +495,82 @@ vtn_cfg_walk_blocks(struct vtn_builder *b, struct list_head *cf_list,
          const uint32_t *branch_end =
             block->branch + (block->branch[0] >> SpvWordCountShift);
 
-         vtn_add_case(b, swtch, break_block, block->branch[2], 0, true);
-         for (const uint32_t *w = block->branch + 3; w < branch_end; w += 2)
-            vtn_add_case(b, swtch, break_block, w[1], w[0], false);
+         struct vtn_value *cond_val = vtn_untyped_value(b, block->branch[1]);
+         vtn_fail_if(!cond_val->type ||
+                     cond_val->type->base_type != vtn_base_type_scalar,
+                     "Selector of OpSelect must have a type of OpTypeInt");
+
+         nir_alu_type cond_type =
+            nir_get_nir_type_for_glsl_type(cond_val->type->type);
+         vtn_fail_if(nir_alu_type_get_base_type(cond_type) != nir_type_int &&
+                     nir_alu_type_get_base_type(cond_type) != nir_type_uint,
+                     "Selector of OpSelect must have a type of OpTypeInt");
+
+         bool is_default = true;
+         const unsigned bitsize = nir_alu_type_get_type_size(cond_type);
+         for (const uint32_t *w = block->branch + 2; w < branch_end;) {
+            uint64_t literal = 0;
+            if (!is_default) {
+               if (bitsize <= 32) {
+                  literal = *(w++);
+               } else {
+                  assert(bitsize == 64);
+                  literal = vtn_u64_literal(w);
+                  w += 2;
+               }
+            }
+
+            uint32_t block_id = *(w++);
+
+            vtn_add_case(b, swtch, break_block, block_id, literal, is_default);
+            is_default = false;
+         }
 
          /* Now, we go through and walk the blocks.  While we walk through
           * the blocks, we also gather the much-needed fall-through
           * information.
           */
          list_for_each_entry(struct vtn_case, cse, &swtch->cases, link) {
-            assert(cse->start_block != break_block);
+            vtn_assert(cse->start_block != break_block);
             vtn_cfg_walk_blocks(b, &cse->body, cse->start_block, cse,
-                                break_block, NULL, loop_cont, NULL);
+                                break_block, loop_break, loop_cont, NULL);
          }
 
          /* Finally, we walk over all of the cases one more time and put
           * them in fall-through order.
           */
-         for (const uint32_t *w = block->branch + 2; w < branch_end; w += 2) {
+         for (const uint32_t *w = block->branch + 2; w < branch_end;) {
             struct vtn_block *case_block =
                vtn_value(b, *w, vtn_value_type_block)->block;
 
+            if (bitsize <= 32) {
+               w += 2;
+            } else {
+               assert(bitsize == 64);
+               w += 3;
+            }
+
             if (case_block == break_block)
                continue;
 
-            assert(case_block->switch_case);
+            vtn_assert(case_block->switch_case);
 
             vtn_order_case(swtch, case_block->switch_case);
          }
 
+         enum vtn_branch_type branch_type =
+            vtn_get_branch_type(b, break_block, switch_case, NULL,
+                                loop_break, loop_cont);
+
+         if (branch_type != vtn_branch_type_none) {
+            /* It is possible that the break is actually the continue block
+             * for the containing loop.  In this case, we need to bail and let
+             * the loop parsing code handle the continue properly.
+             */
+            vtn_assert(branch_type == vtn_branch_type_loop_continue);
+            return;
+         }
+
          block = break_block;
          continue;
       }
@@ -451,7 +579,7 @@ vtn_cfg_walk_blocks(struct vtn_builder *b, struct list_head *cf_list,
          return;
 
       default:
-         unreachable("Unhandled opcode");
+         vtn_fail("Unhandled opcode");
       }
    }
 }
@@ -490,14 +618,13 @@ vtn_handle_phis_first_pass(struct vtn_builder *b, SpvOp opcode,
     * algorithm all over again.  It's easier if we just let
     * lower_vars_to_ssa do that for us instead of repeating it here.
     */
-   struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_ssa);
-
    struct vtn_type *type = vtn_value(b, w[1], vtn_value_type_type)->type;
    nir_variable *phi_var =
       nir_local_variable_create(b->nb.impl, type->type, "phi");
    _mesa_hash_table_insert(b->phi_table, w, phi_var);
 
-   val->ssa = vtn_local_load(b, nir_deref_var_create(b, phi_var));
+   vtn_push_ssa(b, w[2], type,
+                vtn_local_load(b, nir_build_deref_var(&b->nb, phi_var)));
 
    return true;
 }
@@ -510,17 +637,18 @@ vtn_handle_phi_second_pass(struct vtn_builder *b, SpvOp opcode,
       return true;
 
    struct hash_entry *phi_entry = _mesa_hash_table_search(b->phi_table, w);
-   assert(phi_entry);
+   vtn_assert(phi_entry);
    nir_variable *phi_var = phi_entry->data;
 
    for (unsigned i = 3; i < count; i += 2) {
-      struct vtn_ssa_value *src = vtn_ssa_value(b, w[i]);
       struct vtn_block *pred =
          vtn_value(b, w[i + 1], vtn_value_type_block)->block;
 
-      b->nb.cursor = nir_after_block_before_jump(pred->end_block);
+      b->nb.cursor = nir_after_instr(&pred->end_nop->instr);
+
+      struct vtn_ssa_value *src = vtn_ssa_value(b, w[i]);
 
-      vtn_local_store(b, src, nir_deref_var_create(b, phi_var));
+      vtn_local_store(b, src, nir_build_deref_var(&b->nb, phi_var));
    }
 
    return true;
@@ -553,7 +681,7 @@ vtn_emit_branch(struct vtn_builder *b, enum vtn_branch_type branch_type,
       break;
    }
    default:
-      unreachable("Invalid branch type");
+      vtn_fail("Invalid branch type");
    }
 }
 
@@ -576,12 +704,19 @@ vtn_emit_cf_list(struct vtn_builder *b, struct list_head *cf_list,
 
          vtn_foreach_instruction(b, block_start, block_end, handler);
 
-         block->end_block = nir_cursor_current_block(b->nb.cursor);
+         block->end_nop = nir_intrinsic_instr_create(b->nb.shader,
+                                                     nir_intrinsic_nop);
+         nir_builder_instr_insert(&b->nb, &block->end_nop->instr);
 
          if ((*block->branch & SpvOpCodeMask) == SpvOpReturnValue) {
+            vtn_fail_if(b->func->type->return_type->base_type ==
+                        vtn_base_type_void,
+                        "Return with a value from a function returning void");
             struct vtn_ssa_value *src = vtn_ssa_value(b, block->branch[1]);
-            vtn_local_store(b, src,
-                            nir_deref_var_create(b, b->impl->return_var));
+            nir_deref_instr *ret_deref =
+               nir_build_deref_cast(&b->nb, nir_load_param(&b->nb, 0),
+                                    nir_var_local, src->type);
+            vtn_local_store(b, src, ret_deref);
          }
 
          if (block->branch_type != vtn_branch_type_none) {
@@ -594,15 +729,10 @@ vtn_emit_cf_list(struct vtn_builder *b, struct list_head *cf_list,
 
       case vtn_cf_node_type_if: {
          struct vtn_if *vtn_if = (struct vtn_if *)node;
-
-         nir_if *if_stmt = nir_if_create(b->shader);
-         if_stmt->condition =
-            nir_src_for_ssa(vtn_ssa_value(b, vtn_if->condition)->def);
-         nir_cf_node_insert(b->nb.cursor, &if_stmt->cf_node);
-
          bool sw_break = false;
 
-         b->nb.cursor = nir_after_cf_list(&if_stmt->then_list);
+         nir_if *nif =
+            nir_push_if(&b->nb, vtn_ssa_value(b, vtn_if->condition)->def);
          if (vtn_if->then_type == vtn_branch_type_none) {
             vtn_emit_cf_list(b, &vtn_if->then_body,
                              switch_fall_var, &sw_break, handler);
@@ -610,7 +740,7 @@ vtn_emit_cf_list(struct vtn_builder *b, struct list_head *cf_list,
             vtn_emit_branch(b, vtn_if->then_type, switch_fall_var, &sw_break);
          }
 
-         b->nb.cursor = nir_after_cf_list(&if_stmt->else_list);
+         nir_push_else(&b->nb, nif);
          if (vtn_if->else_type == vtn_branch_type_none) {
             vtn_emit_cf_list(b, &vtn_if->else_body,
                              switch_fall_var, &sw_break, handler);
@@ -618,7 +748,7 @@ vtn_emit_cf_list(struct vtn_builder *b, struct list_head *cf_list,
             vtn_emit_branch(b, vtn_if->else_type, switch_fall_var, &sw_break);
          }
 
-         b->nb.cursor = nir_after_cf_node(&if_stmt->cf_node);
+         nir_pop_if(&b->nb, nif);
 
          /* If we encountered a switch break somewhere inside of the if,
           * then it would have been handled correctly by calling
@@ -628,13 +758,7 @@ vtn_emit_cf_list(struct vtn_builder *b, struct list_head *cf_list,
           */
          if (sw_break) {
             *has_switch_break = true;
-
-            nir_if *switch_if = nir_if_create(b->shader);
-            switch_if->condition =
-               nir_src_for_ssa(nir_load_var(&b->nb, switch_fall_var));
-            nir_cf_node_insert(b->nb.cursor, &switch_if->cf_node);
-
-            b->nb.cursor = nir_after_cf_list(&if_stmt->then_list);
+            nir_push_if(&b->nb, nir_load_var(&b->nb, switch_fall_var));
          }
          break;
       }
@@ -642,10 +766,7 @@ vtn_emit_cf_list(struct vtn_builder *b, struct list_head *cf_list,
       case vtn_cf_node_type_loop: {
          struct vtn_loop *vtn_loop = (struct vtn_loop *)node;
 
-         nir_loop *loop = nir_loop_create(b->shader);
-         nir_cf_node_insert(b->nb.cursor, &loop->cf_node);
-
-         b->nb.cursor = nir_after_cf_list(&loop->body);
+         nir_loop *loop = nir_push_loop(&b->nb);
          vtn_emit_cf_list(b, &vtn_loop->body, NULL, NULL, handler);
 
          if (!list_empty(&vtn_loop->cont_body)) {
@@ -660,20 +781,20 @@ vtn_emit_cf_list(struct vtn_builder *b, struct list_head *cf_list,
             nir_store_var(&b->nb, do_cont, nir_imm_int(&b->nb, NIR_FALSE), 1);
 
             b->nb.cursor = nir_before_cf_list(&loop->body);
-            nir_if *cont_if = nir_if_create(b->shader);
-            cont_if->condition = nir_src_for_ssa(nir_load_var(&b->nb, do_cont));
-            nir_cf_node_insert(b->nb.cursor, &cont_if->cf_node);
 
-            b->nb.cursor = nir_after_cf_list(&cont_if->then_list);
+            nir_if *cont_if =
+               nir_push_if(&b->nb, nir_load_var(&b->nb, do_cont));
+
             vtn_emit_cf_list(b, &vtn_loop->cont_body, NULL, NULL, handler);
 
-            b->nb.cursor = nir_after_cf_node(&cont_if->cf_node);
+            nir_pop_if(&b->nb, cont_if);
+
             nir_store_var(&b->nb, do_cont, nir_imm_int(&b->nb, NIR_TRUE), 1);
 
             b->has_loop_continue = true;
          }
 
-         b->nb.cursor = nir_after_cf_node(&loop->cf_node);
+         nir_pop_loop(&b->nb, loop);
          break;
       }
 
@@ -707,9 +828,9 @@ vtn_emit_cf_list(struct vtn_builder *b, struct list_head *cf_list,
             }
 
             nir_ssa_def *cond = NULL;
-            nir_array_foreach(&cse->values, uint32_t, val) {
-               nir_ssa_def *is_val =
-                  nir_ieq(&b->nb, sel, nir_imm_int(&b->nb, *val));
+            util_dynarray_foreach(&cse->values, uint64_t, val) {
+               nir_ssa_def *imm = nir_imm_intN_t(&b->nb, *val, sel->bit_size);
+               nir_ssa_def *is_val = nir_ieq(&b->nb, sel, imm);
 
                cond = cond ? nir_ior(&b->nb, cond, is_val) : is_val;
             }
@@ -717,7 +838,7 @@ vtn_emit_cf_list(struct vtn_builder *b, struct list_head *cf_list,
             any = any ? nir_ior(&b->nb, any, cond) : cond;
             conditions[i++] = cond;
          }
-         assert(i == num_cases);
+         vtn_assert(i == num_cases);
 
          /* Now we can walk the list of cases and actually emit code */
          i = 0;
@@ -725,31 +846,28 @@ vtn_emit_cf_list(struct vtn_builder *b, struct list_head *cf_list,
             /* Figure out the condition */
             nir_ssa_def *cond = conditions[i++];
             if (cse->is_default) {
-               assert(cond == NULL);
+               vtn_assert(cond == NULL);
                cond = nir_inot(&b->nb, any);
             }
             /* Take fallthrough into account */
             cond = nir_ior(&b->nb, cond, nir_load_var(&b->nb, fall_var));
 
-            nir_if *case_if = nir_if_create(b->nb.shader);
-            case_if->condition = nir_src_for_ssa(cond);
-            nir_cf_node_insert(b->nb.cursor, &case_if->cf_node);
+            nir_if *case_if = nir_push_if(&b->nb, cond);
 
             bool has_break = false;
-            b->nb.cursor = nir_after_cf_list(&case_if->then_list);
             nir_store_var(&b->nb, fall_var, nir_imm_int(&b->nb, NIR_TRUE), 1);
             vtn_emit_cf_list(b, &cse->body, fall_var, &has_break, handler);
             (void)has_break; /* We don't care */
 
-            b->nb.cursor = nir_after_cf_node(&case_if->cf_node);
+            nir_pop_if(&b->nb, case_if);
          }
-         assert(i == num_cases);
+         vtn_assert(i == num_cases);
 
          break;
       }
 
       default:
-         unreachable("Invalid CF node type");
+         vtn_fail("Invalid CF node type");
       }
    }
 }
@@ -759,6 +877,7 @@ vtn_function_emit(struct vtn_builder *b, struct vtn_function *func,
                   vtn_instruction_handler instruction_handler)
 {
    nir_builder_init(&b->nb, func->impl);
+   b->func = func;
    b->nb.cursor = nir_after_cf_list(&func->impl->body);
    b->has_loop_continue = false;
    b->phi_table = _mesa_hash_table_create(b, _mesa_hash_pointer,
@@ -775,4 +894,6 @@ vtn_function_emit(struct vtn_builder *b, struct vtn_function *func,
     */
    if (b->has_loop_continue)
       nir_repair_ssa_impl(func->impl);
+
+   func->emitted = true;
 }