anv/cmd_buffer: Move Begin/End/Execute to genX_cmd_buffer.c
[mesa.git] / src / intel / vulkan / anv_private.h
index f6ce31da1c4643858c70ac2838ed18b43e0766b5..9454e08bda78e9e7bc557c0eea574fc35fbe3f5c 100644 (file)
@@ -21,7 +21,8 @@
  * IN THE SOFTWARE.
  */
 
-#pragma once
+#ifndef ANV_PRIVATE_H
+#define ANV_PRIVATE_H
 
 #include <stdlib.h>
 #include <stdio.h>
@@ -40,7 +41,8 @@
 #define VG(x)
 #endif
 
-#include "brw_device_info.h"
+#include "common/gen_device_info.h"
+#include "blorp/blorp.h"
 #include "brw_compiler.h"
 #include "util/macros.h"
 #include "util/list.h"
@@ -52,7 +54,7 @@ typedef struct xcb_connection_t xcb_connection_t;
 typedef uint32_t xcb_visualid_t;
 typedef uint32_t xcb_window_t;
 
-struct anv_l3_config;
+struct gen_l3_config;
 
 #include <vulkan/vulkan.h>
 #include <vulkan/vulkan_intel.h>
@@ -82,6 +84,12 @@ extern "C" {
 #define MIN(a, b) ((a) < (b) ? (a) : (b))
 #define MAX(a, b) ((a) > (b) ? (a) : (b))
 
+static inline uint32_t
+align_down_npot_u32(uint32_t v, uint32_t a)
+{
+   return v - (v % a);
+}
+
 static inline uint32_t
 align_u32(uint32_t v, uint32_t a)
 {
@@ -394,9 +402,9 @@ struct anv_fixed_size_state_pool {
 };
 
 #define ANV_MIN_STATE_SIZE_LOG2 6
-#define ANV_MAX_STATE_SIZE_LOG2 10
+#define ANV_MAX_STATE_SIZE_LOG2 17
 
-#define ANV_STATE_BUCKETS (ANV_MAX_STATE_SIZE_LOG2 - ANV_MIN_STATE_SIZE_LOG2)
+#define ANV_STATE_BUCKETS (ANV_MAX_STATE_SIZE_LOG2 - ANV_MIN_STATE_SIZE_LOG2 + 1)
 
 struct anv_state_pool {
    struct anv_block_pool *block_pool;
@@ -475,19 +483,22 @@ VkResult anv_bo_pool_alloc(struct anv_bo_pool *pool, struct anv_bo *bo,
                            uint32_t size);
 void anv_bo_pool_free(struct anv_bo_pool *pool, const struct anv_bo *bo);
 
+struct anv_scratch_pool {
+   /* Indexed by Per-Thread Scratch Space number (the hardware value) and stage */
+   struct anv_bo bos[16][MESA_SHADER_STAGES];
+};
 
-void *anv_resolve_entrypoint(uint32_t index);
+void anv_scratch_pool_init(struct anv_device *device,
+                           struct anv_scratch_pool *pool);
+void anv_scratch_pool_finish(struct anv_device *device,
+                             struct anv_scratch_pool *pool);
+struct anv_bo *anv_scratch_pool_alloc(struct anv_device *device,
+                                      struct anv_scratch_pool *pool,
+                                      gl_shader_stage stage,
+                                      unsigned per_thread_scratch);
 
 extern struct anv_dispatch_table dtable;
 
-#define ANV_CALL(func) ({ \
-   if (dtable.func == NULL) { \
-      size_t idx = offsetof(struct anv_dispatch_table, func) / sizeof(void *); \
-      dtable.entrypoints[idx] = anv_resolve_entrypoint(idx); \
-   } \
-   dtable.func; \
-})
-
 static inline void *
 anv_alloc(const VkAllocationCallbacks *alloc,
           size_t size, size_t align,
@@ -542,14 +553,17 @@ struct anv_physical_device {
 
     struct anv_instance *                       instance;
     uint32_t                                    chipset_id;
-    const char *                                path;
+    char                                        path[20];
     const char *                                name;
-    const struct brw_device_info *              info;
+    struct gen_device_info                      info;
     uint64_t                                    aperture_size;
     struct brw_compiler *                       compiler;
     struct isl_device                           isl_dev;
     int                                         cmd_parser_version;
 
+    uint32_t                                    eu_total;
+    uint32_t                                    subslice_total;
+
     struct anv_wsi_interface *                  wsi[VK_ICD_WSI_PLATFORM_MAX];
 };
 
@@ -566,69 +580,6 @@ struct anv_instance {
 VkResult anv_init_wsi(struct anv_physical_device *physical_device);
 void anv_finish_wsi(struct anv_physical_device *physical_device);
 
-struct anv_meta_state {
-   VkAllocationCallbacks alloc;
-
-   /**
-    * Use array element `i` for images with `2^i` samples.
-    */
-   struct {
-      /**
-       * Pipeline N is used to clear color attachment N of the current
-       * subpass.
-       *
-       * HACK: We use one pipeline per color attachment to work around the
-       * compiler's inability to dynamically set the render target index of
-       * the render target write message.
-       */
-      struct anv_pipeline *color_pipelines[MAX_RTS];
-
-      struct anv_pipeline *depth_only_pipeline;
-      struct anv_pipeline *stencil_only_pipeline;
-      struct anv_pipeline *depthstencil_pipeline;
-   } clear[1 + MAX_SAMPLES_LOG2];
-
-   struct {
-      VkRenderPass render_pass;
-
-      /** Pipeline that blits from a 1D image. */
-      VkPipeline pipeline_1d_src;
-
-      /** Pipeline that blits from a 2D image. */
-      VkPipeline pipeline_2d_src;
-
-      /** Pipeline that blits from a 3D image. */
-      VkPipeline pipeline_3d_src;
-
-      VkPipelineLayout                          pipeline_layout;
-      VkDescriptorSetLayout                     ds_layout;
-   } blit;
-
-   struct {
-      VkRenderPass render_pass;
-
-      VkPipelineLayout                          img_p_layout;
-      VkDescriptorSetLayout                     img_ds_layout;
-      VkPipelineLayout                          buf_p_layout;
-      VkDescriptorSetLayout                     buf_ds_layout;
-
-      /* Pipelines indexed by source and destination type.  See the
-       * blit2d_src_type and blit2d_dst_type enums in anv_meta_blit2d.c to
-       * see what these mean.
-       */
-      VkPipeline pipelines[2][3];
-   } blit2d;
-
-   struct {
-      /** Pipeline [i] resolves an image with 2^(i+1) samples.  */
-      VkPipeline                                pipelines[MAX_SAMPLES_LOG2];
-
-      VkRenderPass                              pass;
-      VkPipelineLayout                          pipeline_layout;
-      VkDescriptorSetLayout                     ds_layout;
-   } resolve;
-};
-
 struct anv_queue {
     VK_LOADER_DATA                              _loader_data;
 
@@ -639,31 +590,27 @@ struct anv_queue {
 
 struct anv_pipeline_cache {
    struct anv_device *                          device;
-   struct anv_state_stream                      program_stream;
    pthread_mutex_t                              mutex;
 
-   uint32_t                                     total_size;
-   uint32_t                                     table_size;
-   uint32_t                                     kernel_count;
-   uint32_t *                                   hash_table;
+   struct hash_table *                          cache;
 };
 
 struct anv_pipeline_bind_map;
 
 void anv_pipeline_cache_init(struct anv_pipeline_cache *cache,
-                             struct anv_device *device);
+                             struct anv_device *device,
+                             bool cache_enabled);
 void anv_pipeline_cache_finish(struct anv_pipeline_cache *cache);
-uint32_t anv_pipeline_cache_search(struct anv_pipeline_cache *cache,
-                                   const unsigned char *sha1,
-                                   const struct brw_stage_prog_data **prog_data,
-                                   struct anv_pipeline_bind_map *map);
-uint32_t anv_pipeline_cache_upload_kernel(struct anv_pipeline_cache *cache,
-                                          const unsigned char *sha1,
-                                          const void *kernel,
-                                          size_t kernel_size,
-                                          const struct brw_stage_prog_data **prog_data,
-                                          size_t prog_data_size,
-                                          struct anv_pipeline_bind_map *map);
+
+struct anv_shader_bin *
+anv_pipeline_cache_search(struct anv_pipeline_cache *cache,
+                          const void *key, uint32_t key_size);
+struct anv_shader_bin *
+anv_pipeline_cache_upload_kernel(struct anv_pipeline_cache *cache,
+                                 const void *key_data, uint32_t key_size,
+                                 const void *kernel_data, uint32_t kernel_size,
+                                 const void *prog_data, uint32_t prog_data_size,
+                                 const struct anv_pipeline_bind_map *bind_map);
 
 struct anv_device {
     VK_LOADER_DATA                              _loader_data;
@@ -672,7 +619,7 @@ struct anv_device {
 
     struct anv_instance *                       instance;
     uint32_t                                    chipset_id;
-    struct brw_device_info                      info;
+    struct gen_device_info                      info;
     struct isl_device                           isl_dev;
     int                                         context_id;
     int                                         fd;
@@ -685,20 +632,21 @@ struct anv_device {
     struct anv_state_pool                       dynamic_state_pool;
 
     struct anv_block_pool                       instruction_block_pool;
-    struct anv_pipeline_cache                   default_pipeline_cache;
+    struct anv_state_pool                       instruction_state_pool;
 
     struct anv_block_pool                       surface_state_block_pool;
     struct anv_state_pool                       surface_state_pool;
 
     struct anv_bo                               workaround_bo;
 
-    struct anv_meta_state                       meta_state;
+    struct anv_pipeline_cache                   blorp_shader_cache;
+    struct blorp_context                        blorp;
 
     struct anv_state                            border_colors;
 
     struct anv_queue                            queue;
 
-    struct anv_block_pool                       scratch_block_pool;
+    struct anv_scratch_pool                     scratch_pool;
 
     uint32_t                                    default_mocs;
 
@@ -707,6 +655,8 @@ struct anv_device {
 
 void anv_device_get_cache_uuid(void *uuid);
 
+void anv_device_init_blorp(struct anv_device *device);
+void anv_device_finish_blorp(struct anv_device *device);
 
 void* anv_gem_mmap(struct anv_device *device,
                    uint32_t gem_handle, uint64_t offset, uint64_t size, uint32_t flags);
@@ -792,12 +742,9 @@ struct anv_address {
    uint32_t offset;
 };
 
-#define __gen_address_type struct anv_address
-#define __gen_user_data struct anv_batch
-
 static inline uint64_t
-__gen_combine_address(struct anv_batch *batch, void *location,
-                      const struct anv_address address, uint32_t delta)
+_anv_combine_address(struct anv_batch *batch, void *location,
+                     const struct anv_address address, uint32_t delta)
 {
    if (address.bo == NULL) {
       return address.offset + delta;
@@ -808,6 +755,10 @@ __gen_combine_address(struct anv_batch *batch, void *location,
    }
 }
 
+#define __gen_address_type struct anv_address
+#define __gen_user_data struct anv_batch
+#define __gen_combine_address _anv_combine_address
+
 /* Wrapper macros needed to work around preprocessor argument issues.  In
  * particular, arguments don't get pre-evaluated if they are concatenated.
  * This means that, if you pass GENX(3DSTATE_PS) into the emit macro, the
@@ -922,6 +873,11 @@ struct anv_vue_header {
 };
 
 struct anv_descriptor_set_binding_layout {
+#ifndef NDEBUG
+   /* The type of the descriptors in this binding */
+   VkDescriptorType type;
+#endif
+
    /* Number of array elements in this binding */
    uint16_t array_size;
 
@@ -1012,17 +968,20 @@ anv_descriptor_set_destroy(struct anv_device *device,
                            struct anv_descriptor_pool *pool,
                            struct anv_descriptor_set *set);
 
-#define ANV_DESCRIPTOR_SET_COLOR_ATTACHMENTS UINT16_MAX
+#define ANV_DESCRIPTOR_SET_COLOR_ATTACHMENTS UINT8_MAX
 
 struct anv_pipeline_binding {
    /* The descriptor set this surface corresponds to.  The special value of
     * ANV_DESCRIPTOR_SET_COLOR_ATTACHMENTS indicates that the offset refers
     * to a color attachment and not a regular descriptor.
     */
-   uint16_t set;
+   uint8_t set;
+
+   /* Binding in the descriptor set */
+   uint8_t binding;
 
-   /* Offset into the descriptor set or attachment list. */
-   uint16_t offset;
+   /* Index in the binding */
+   uint8_t index;
 };
 
 struct anv_pipeline_layout {
@@ -1036,6 +995,8 @@ struct anv_pipeline_layout {
    struct {
       bool has_dynamic_offsets;
    } stage[MESA_SHADER_STAGES];
+
+   unsigned char sha1[20];
 };
 
 struct anv_buffer {
@@ -1066,6 +1027,45 @@ enum anv_cmd_dirty_bits {
 };
 typedef uint32_t anv_cmd_dirty_mask_t;
 
+enum anv_pipe_bits {
+   ANV_PIPE_DEPTH_CACHE_FLUSH_BIT            = (1 << 0),
+   ANV_PIPE_STALL_AT_SCOREBOARD_BIT          = (1 << 1),
+   ANV_PIPE_STATE_CACHE_INVALIDATE_BIT       = (1 << 2),
+   ANV_PIPE_CONSTANT_CACHE_INVALIDATE_BIT    = (1 << 3),
+   ANV_PIPE_VF_CACHE_INVALIDATE_BIT          = (1 << 4),
+   ANV_PIPE_DATA_CACHE_FLUSH_BIT             = (1 << 5),
+   ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT     = (1 << 10),
+   ANV_PIPE_INSTRUCTION_CACHE_INVALIDATE_BIT = (1 << 11),
+   ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT    = (1 << 12),
+   ANV_PIPE_DEPTH_STALL_BIT                  = (1 << 13),
+   ANV_PIPE_CS_STALL_BIT                     = (1 << 20),
+
+   /* This bit does not exist directly in PIPE_CONTROL.  Instead it means that
+    * a flush has happened but not a CS stall.  The next time we do any sort
+    * of invalidation we need to insert a CS stall at that time.  Otherwise,
+    * we would have to CS stall on every flush which could be bad.
+    */
+   ANV_PIPE_NEEDS_CS_STALL_BIT               = (1 << 21),
+};
+
+#define ANV_PIPE_FLUSH_BITS ( \
+   ANV_PIPE_DEPTH_CACHE_FLUSH_BIT | \
+   ANV_PIPE_DATA_CACHE_FLUSH_BIT | \
+   ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT)
+
+#define ANV_PIPE_STALL_BITS ( \
+   ANV_PIPE_STALL_AT_SCOREBOARD_BIT | \
+   ANV_PIPE_DEPTH_STALL_BIT | \
+   ANV_PIPE_CS_STALL_BIT)
+
+#define ANV_PIPE_INVALIDATE_BITS ( \
+   ANV_PIPE_STATE_CACHE_INVALIDATE_BIT | \
+   ANV_PIPE_CONSTANT_CACHE_INVALIDATE_BIT | \
+   ANV_PIPE_VF_CACHE_INVALIDATE_BIT | \
+   ANV_PIPE_DATA_CACHE_FLUSH_BIT | \
+   ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT | \
+   ANV_PIPE_INSTRUCTION_CACHE_INVALIDATE_BIT)
+
 struct anv_vertex_binding {
    struct anv_buffer *                          buffer;
    VkDeviceSize                                 offset;
@@ -1160,10 +1160,11 @@ struct anv_attachment_state {
 struct anv_cmd_state {
    /* PIPELINE_SELECT.PipelineSelection */
    uint32_t                                     current_pipeline;
-   const struct anv_l3_config *                 current_l3_config;
+   const struct gen_l3_config *                 current_l3_config;
    uint32_t                                     vb_dirty;
    anv_cmd_dirty_mask_t                         dirty;
    anv_cmd_dirty_mask_t                         compute_dirty;
+   enum anv_pipe_bits                           pending_pipe_bits;
    uint32_t                                     num_workgroups_offset;
    struct anv_bo                                *num_workgroups_bo;
    VkShaderStageFlags                           descriptors_dirty;
@@ -1283,13 +1284,13 @@ void anv_cmd_buffer_add_secondary(struct anv_cmd_buffer *primary,
                                   struct anv_cmd_buffer *secondary);
 void anv_cmd_buffer_prepare_execbuf(struct anv_cmd_buffer *cmd_buffer);
 
+VkResult anv_cmd_buffer_reset(struct anv_cmd_buffer *cmd_buffer);
+
 VkResult anv_cmd_buffer_emit_binding_table(struct anv_cmd_buffer *cmd_buffer,
                                            unsigned stage, struct anv_state *bt_state);
 VkResult anv_cmd_buffer_emit_samplers(struct anv_cmd_buffer *cmd_buffer,
                                       unsigned stage, struct anv_state *state);
-uint32_t gen7_cmd_buffer_flush_descriptor_sets(struct anv_cmd_buffer *cmd_buffer);
-void gen7_cmd_buffer_emit_descriptor_pointers(struct anv_cmd_buffer *cmd_buffer,
-                                              uint32_t stages);
+uint32_t anv_cmd_buffer_flush_descriptor_sets(struct anv_cmd_buffer *cmd_buffer);
 
 struct anv_state anv_cmd_buffer_emit_dynamic(struct anv_cmd_buffer *cmd_buffer,
                                              const void *data, uint32_t size, uint32_t alignment);
@@ -1312,6 +1313,8 @@ VkResult
 anv_cmd_buffer_new_binding_table_block(struct anv_cmd_buffer *cmd_buffer);
 
 void gen8_cmd_buffer_emit_viewport(struct anv_cmd_buffer *cmd_buffer);
+void gen8_cmd_buffer_emit_depth_viewport(struct anv_cmd_buffer *cmd_buffer,
+                                         bool depth_clamp_enable);
 void gen7_cmd_buffer_emit_scissor(struct anv_cmd_buffer *cmd_buffer);
 
 void anv_cmd_buffer_emit_state_base_address(struct anv_cmd_buffer *cmd_buffer);
@@ -1319,9 +1322,6 @@ void anv_cmd_buffer_emit_state_base_address(struct anv_cmd_buffer *cmd_buffer);
 void anv_cmd_state_setup_attachments(struct anv_cmd_buffer *cmd_buffer,
                                      const VkRenderPassBeginInfo *info);
 
-void anv_cmd_buffer_set_subpass(struct anv_cmd_buffer *cmd_buffer,
-                                  struct anv_subpass *subpass);
-
 struct anv_state
 anv_cmd_buffer_push_constants(struct anv_cmd_buffer *cmd_buffer,
                               gl_shader_stage stage);
@@ -1348,11 +1348,7 @@ struct anv_event {
    struct anv_state                             state;
 };
 
-struct nir_shader;
-
 struct anv_shader_module {
-   struct nir_shader *                          nir;
-
    unsigned char                                sha1[20];
    uint32_t                                     size;
    char                                         data[0];
@@ -1361,6 +1357,7 @@ struct anv_shader_module {
 void anv_hash_shader(unsigned char *hash, const void *key, size_t key_size,
                      struct anv_shader_module *module,
                      const char *entrypoint,
+                     const struct anv_pipeline_layout *pipeline_layout,
                      const VkSpecializationInfo *spec_info);
 
 static inline gl_shader_stage
@@ -1388,13 +1385,57 @@ struct anv_pipeline_bind_map {
    uint32_t surface_count;
    uint32_t sampler_count;
    uint32_t image_count;
-   uint32_t attachment_count;
 
    struct anv_pipeline_binding *                surface_to_descriptor;
    struct anv_pipeline_binding *                sampler_to_descriptor;
-   uint32_t *                                   surface_to_attachment;
 };
 
+struct anv_shader_bin {
+   uint32_t ref_cnt;
+
+   struct anv_state kernel;
+   uint32_t kernel_size;
+
+   struct anv_pipeline_bind_map bind_map;
+
+   uint32_t prog_data_size;
+
+   /* Prog data follows, then the key, both aligned to 8-bytes */
+};
+
+struct anv_shader_bin *
+anv_shader_bin_create(struct anv_device *device,
+                      const void *key, uint32_t key_size,
+                      const void *kernel, uint32_t kernel_size,
+                      const void *prog_data, uint32_t prog_data_size,
+                      const struct anv_pipeline_bind_map *bind_map);
+
+void
+anv_shader_bin_destroy(struct anv_device *device, struct anv_shader_bin *shader);
+
+static inline void
+anv_shader_bin_ref(struct anv_shader_bin *shader)
+{
+   assert(shader->ref_cnt >= 1);
+   __sync_fetch_and_add(&shader->ref_cnt, 1);
+}
+
+static inline void
+anv_shader_bin_unref(struct anv_device *device, struct anv_shader_bin *shader)
+{
+   assert(shader->ref_cnt >= 1);
+   if (__sync_fetch_and_add(&shader->ref_cnt, -1) == 1)
+      anv_shader_bin_destroy(device, shader);
+}
+
+static inline const struct brw_stage_prog_data *
+anv_shader_bin_get_prog_data(const struct anv_shader_bin *shader)
+{
+   const void *data = shader;
+   data += align_u32(sizeof(struct anv_shader_bin), 8);
+   return data;
+}
+
 struct anv_pipeline {
    struct anv_device *                          device;
    struct anv_batch                             batch;
@@ -1404,19 +1445,13 @@ struct anv_pipeline {
    struct anv_dynamic_state                     dynamic_state;
 
    struct anv_pipeline_layout *                 layout;
-   struct anv_pipeline_bind_map                 bindings[MESA_SHADER_STAGES];
 
-   bool                                         use_repclear;
    bool                                         needs_data_cache;
 
-   const struct brw_stage_prog_data *           prog_data[MESA_SHADER_STAGES];
-   uint32_t                                     scratch_start[MESA_SHADER_STAGES];
-   uint32_t                                     total_scratch;
+   struct anv_shader_bin *                      shaders[MESA_SHADER_STAGES];
+
    struct {
-      uint32_t                                  start[MESA_SHADER_GEOMETRY + 1];
-      uint32_t                                  size[MESA_SHADER_GEOMETRY + 1];
-      uint32_t                                  entries[MESA_SHADER_GEOMETRY + 1];
-      const struct anv_l3_config *              l3_config;
+      const struct gen_l3_config *              l3_config;
       uint32_t                                  total_size;
    } urb;
 
@@ -1434,9 +1469,10 @@ struct anv_pipeline {
    bool                                         primitive_restart;
    uint32_t                                     topology;
 
-   uint32_t                                     cs_thread_width_max;
    uint32_t                                     cs_right_mask;
 
+   bool                                         depth_clamp_enable;
+
    struct {
       uint32_t                                  sf[7];
       uint32_t                                  depth_stencil_state[3];
@@ -1453,47 +1489,34 @@ struct anv_pipeline {
    } gen9;
 };
 
-static inline const struct brw_vs_prog_data *
-get_vs_prog_data(struct anv_pipeline *pipeline)
-{
-   return (const struct brw_vs_prog_data *) pipeline->prog_data[MESA_SHADER_VERTEX];
-}
-
-static inline const struct brw_gs_prog_data *
-get_gs_prog_data(struct anv_pipeline *pipeline)
+static inline bool
+anv_pipeline_has_stage(const struct anv_pipeline *pipeline,
+                       gl_shader_stage stage)
 {
-   return (const struct brw_gs_prog_data *) pipeline->prog_data[MESA_SHADER_GEOMETRY];
+   return (pipeline->active_stages & mesa_to_vk_shader_stage(stage)) != 0;
 }
 
-static inline const struct brw_wm_prog_data *
-get_wm_prog_data(struct anv_pipeline *pipeline)
-{
-   return (const struct brw_wm_prog_data *) pipeline->prog_data[MESA_SHADER_FRAGMENT];
+#define ANV_DECL_GET_PROG_DATA_FUNC(prefix, stage)                   \
+static inline const struct brw_##prefix##_prog_data *                \
+get_##prefix##_prog_data(struct anv_pipeline *pipeline)              \
+{                                                                    \
+   if (anv_pipeline_has_stage(pipeline, stage)) {                    \
+      return (const struct brw_##prefix##_prog_data *)               \
+             anv_shader_bin_get_prog_data(pipeline->shaders[stage]); \
+   } else {                                                          \
+      return NULL;                                                   \
+   }                                                                 \
 }
 
-static inline const struct brw_cs_prog_data *
-get_cs_prog_data(struct anv_pipeline *pipeline)
-{
-   return (const struct brw_cs_prog_data *) pipeline->prog_data[MESA_SHADER_COMPUTE];
-}
-
-struct anv_graphics_pipeline_create_info {
-   /**
-    * If non-negative, overrides the color attachment count of the pipeline's
-    * subpass.
-    */
-   int8_t color_attachment_count;
-
-   bool                                         use_repclear;
-   bool                                         disable_vs;
-   bool                                         use_rectlist;
-};
+ANV_DECL_GET_PROG_DATA_FUNC(vs, MESA_SHADER_VERTEX)
+ANV_DECL_GET_PROG_DATA_FUNC(gs, MESA_SHADER_GEOMETRY)
+ANV_DECL_GET_PROG_DATA_FUNC(wm, MESA_SHADER_FRAGMENT)
+ANV_DECL_GET_PROG_DATA_FUNC(cs, MESA_SHADER_COMPUTE)
 
 VkResult
 anv_pipeline_init(struct anv_pipeline *pipeline, struct anv_device *device,
                   struct anv_pipeline_cache *cache,
                   const VkGraphicsPipelineCreateInfo *pCreateInfo,
-                  const struct anv_graphics_pipeline_create_info *extra,
                   const VkAllocationCallbacks *alloc);
 
 VkResult
@@ -1504,47 +1527,30 @@ anv_pipeline_compile_cs(struct anv_pipeline *pipeline,
                         const char *entrypoint,
                         const VkSpecializationInfo *spec_info);
 
-VkResult
-anv_graphics_pipeline_create(VkDevice device,
-                             VkPipelineCache cache,
-                             const VkGraphicsPipelineCreateInfo *pCreateInfo,
-                             const struct anv_graphics_pipeline_create_info *extra,
-                             const VkAllocationCallbacks *alloc,
-                             VkPipeline *pPipeline);
-
-struct anv_format_swizzle {
-   enum isl_channel_select r:4;
-   enum isl_channel_select g:4;
-   enum isl_channel_select b:4;
-   enum isl_channel_select a:4;
-};
-
 struct anv_format {
    enum isl_format isl_format:16;
-   struct anv_format_swizzle swizzle;
+   struct isl_swizzle swizzle;
 };
 
 struct anv_format
-anv_get_format(const struct brw_device_info *devinfo, VkFormat format,
+anv_get_format(const struct gen_device_info *devinfo, VkFormat format,
                VkImageAspectFlags aspect, VkImageTiling tiling);
 
 static inline enum isl_format
-anv_get_isl_format(const struct brw_device_info *devinfo, VkFormat vk_format,
+anv_get_isl_format(const struct gen_device_info *devinfo, VkFormat vk_format,
                    VkImageAspectFlags aspect, VkImageTiling tiling)
 {
    return anv_get_format(devinfo, vk_format, aspect, tiling).isl_format;
 }
 
 void
-anv_compute_urb_partition(struct anv_pipeline *pipeline);
-
-void
-anv_setup_pipeline_l3_config(struct anv_pipeline *pipeline);
+anv_pipeline_setup_l3_config(struct anv_pipeline *pipeline, bool needs_slm);
 
 /**
  * Subsurface of an anv_image.
  */
 struct anv_surface {
+   /** Valid only if isl_surf::size > 0. */
    struct isl_surf isl;
 
    /**
@@ -1591,6 +1597,7 @@ struct anv_image {
 
       struct {
          struct anv_surface depth_surface;
+         struct anv_surface hiz_surface;
          struct anv_surface stencil_surface;
       };
    };
@@ -1618,10 +1625,10 @@ struct anv_image_view {
    struct anv_bo *bo;
    uint32_t offset; /**< Offset into bo. */
 
+   struct isl_view isl;
+
    VkImageAspectFlags aspect_mask;
    VkFormat vk_format;
-   uint32_t base_layer;
-   uint32_t base_mip;
    VkExtent3D extent; /**< Extent of VkImageViewCreateInfo::baseMipLevel. */
 
    /** RENDER_SURFACE_STATE when using image as a color render target. */
@@ -1638,7 +1645,10 @@ struct anv_image_view {
 
 struct anv_image_create_info {
    const VkImageCreateInfo *vk_info;
+
+   /** An opt-in bitmask which filters an ISL-mapping of the Vulkan tiling. */
    isl_tiling_flags_t isl_tiling_flags;
+
    uint32_t stride;
 };
 
@@ -1647,15 +1657,19 @@ VkResult anv_image_create(VkDevice _device,
                           const VkAllocationCallbacks* alloc,
                           VkImage *pImage);
 
-struct anv_surface *
-anv_image_get_surface_for_aspect_mask(struct anv_image *image,
+const struct anv_surface *
+anv_image_get_surface_for_aspect_mask(const struct anv_image *image,
                                       VkImageAspectFlags aspect_mask);
 
-void anv_image_view_init(struct anv_image_view *view,
-                         struct anv_device *device,
-                         const VkImageViewCreateInfo* pCreateInfo,
-                         struct anv_cmd_buffer *cmd_buffer,
-                         VkImageUsageFlags usage_mask);
+static inline bool
+anv_image_has_hiz(const struct anv_image *image)
+{
+   /* We must check the aspect because anv_image::hiz_surface belongs to
+    * a union.
+    */
+   return (image->aspects & VK_IMAGE_ASPECT_DEPTH_BIT) &&
+          image->hiz_surface.isl.size > 0;
+}
 
 struct anv_buffer_view {
    enum isl_format format; /**< VkBufferViewCreateInfo::format */
@@ -1669,11 +1683,6 @@ struct anv_buffer_view {
    struct brw_image_param storage_image_param;
 };
 
-void anv_buffer_view_init(struct anv_buffer_view *view,
-                          struct anv_device *device,
-                          const VkBufferViewCreateInfo* pCreateInfo,
-                          struct anv_cmd_buffer *cmd_buffer);
-
 enum isl_format
 anv_isl_format_for_descriptor_type(VkDescriptorType type);
 
@@ -1752,6 +1761,7 @@ struct anv_render_pass_attachment {
    VkFormat                                     format;
    uint32_t                                     samples;
    VkAttachmentLoadOp                           load_op;
+   VkAttachmentStoreOp                          store_op;
    VkAttachmentLoadOp                           stencil_load_op;
 };
 
@@ -1763,8 +1773,6 @@ struct anv_render_pass {
    struct anv_subpass                           subpasses[0];
 };
 
-extern struct anv_render_pass anv_meta_dummy_renderpass;
-
 struct anv_query_pool_slot {
    uint64_t begin;
    uint64_t end;
@@ -1777,14 +1785,23 @@ struct anv_query_pool {
    struct anv_bo                                bo;
 };
 
-VkResult anv_device_init_meta(struct anv_device *device);
-void anv_device_finish_meta(struct anv_device *device);
-
-void *anv_lookup_entrypoint(const char *name);
+void *anv_lookup_entrypoint(const struct gen_device_info *devinfo,
+                            const char *name);
 
 void anv_dump_image_to_ppm(struct anv_device *device,
                            struct anv_image *image, unsigned miplevel,
-                           unsigned array_layer, const char *filename);
+                           unsigned array_layer, VkImageAspectFlagBits aspect,
+                           const char *filename);
+
+enum anv_dump_action {
+   ANV_DUMP_FRAMEBUFFERS_BIT = 0x1,
+};
+
+void anv_dump_start(struct anv_device *device, enum anv_dump_action actions);
+void anv_dump_finish(void);
+
+void anv_dump_add_framebuffer(struct anv_cmd_buffer *cmd_buffer,
+                              struct anv_framebuffer *fb);
 
 #define ANV_DEFINE_HANDLE_CASTS(__anv_type, __VkType)                      \
                                                                            \
@@ -1879,3 +1896,5 @@ ANV_DEFINE_STRUCT_CASTS(anv_common, VkImageMemoryBarrier)
 #ifdef __cplusplus
 }
 #endif
+
+#endif /* ANV_PRIVATE_H */