Added few more stubs so that control reaches to DestroyDevice().
[mesa.git] / src / libre-soc / vulkan / libresoc_device.c
index d7290de696278ce56634b38aa298447d3ae89f08..546f175c7d8bc922fb199c847ee8cfdcd7822c84 100644 (file)
 #include <sys/sysinfo.h>
 #include <unistd.h>
 
+#include "util/debug.h"
+#include "util/driconf.h"
 #include "libresoc_private.h"
+#include "compiler/glsl_types.h"
 #include "vk_util.h"
 #include "vk_alloc.h"
 
+struct libresoc_deferred_queue_submission {
+       struct libresoc_queue *queue;
+       VkCommandBuffer *cmd_buffers;
+       uint32_t cmd_buffer_count;
+
+       /* Sparse bindings that happen on a queue. */
+       VkSparseBufferMemoryBindInfo *buffer_binds;
+       uint32_t buffer_bind_count;
+       VkSparseImageOpaqueMemoryBindInfo *image_opaque_binds;
+       uint32_t image_opaque_bind_count;
+
+       bool flush_caches;
+       VkShaderStageFlags wait_dst_stage_mask;
+       struct libresoc_semaphore_part **wait_semaphores;
+       uint32_t wait_semaphore_count;
+       struct libresoc_semaphore_part **signal_semaphores;
+       uint32_t signal_semaphore_count;
+       VkFence fence;
+
+       uint64_t *wait_values;
+       uint64_t *signal_values;
+
+       struct libresoc_semaphore_part *temporary_semaphore_parts;
+       uint32_t temporary_semaphore_part_count;
+
+       struct list_head queue_pending_list;
+       uint32_t submission_wait_count;
+       struct libresoc_timeline_waiter *wait_nodes;
+
+       struct list_head processing_list;
+};
+
+struct libresoc_queue_submission {
+       const VkCommandBuffer *cmd_buffers;
+       uint32_t cmd_buffer_count;
+
+       /* Sparse bindings that happen on a queue. */
+       const VkSparseBufferMemoryBindInfo *buffer_binds;
+       uint32_t buffer_bind_count;
+       const VkSparseImageOpaqueMemoryBindInfo *image_opaque_binds;
+       uint32_t image_opaque_bind_count;
+
+       bool flush_caches;
+       VkPipelineStageFlags wait_dst_stage_mask;
+       const VkSemaphore *wait_semaphores;
+       uint32_t wait_semaphore_count;
+       const VkSemaphore *signal_semaphores;
+       uint32_t signal_semaphore_count;
+       VkFence fence;
+
+       const uint64_t *wait_values;
+       uint32_t wait_value_count;
+       const uint64_t *signal_values;
+       uint32_t signal_value_count;
+};
+
+void
+libresoc_free_memory(struct libresoc_device *device,
+                const VkAllocationCallbacks* pAllocator,
+                struct libresoc_device_memory *mem)
+{
+       if (mem == NULL)
+               return;
+
+       vk_object_base_finish(&mem->base);
+       vk_free2(&device->vk.alloc, pAllocator, mem);
+}
+
+static VkResult libresoc_alloc_memory(struct libresoc_device *device,
+                                 const VkMemoryAllocateInfo*     pAllocateInfo,
+                                 const VkAllocationCallbacks*    pAllocator,
+                                 VkDeviceMemory*                 pMem)
+{
+       struct libresoc_device_memory *mem;
+       VkResult result;
+       uint32_t flags = 0;
+
+       assert(pAllocateInfo->sType == VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO);
+
+       const VkImportMemoryFdInfoKHR *import_info =
+               vk_find_struct_const(pAllocateInfo->pNext, IMPORT_MEMORY_FD_INFO_KHR);
+       const VkMemoryDedicatedAllocateInfo *dedicate_info =
+               vk_find_struct_const(pAllocateInfo->pNext, MEMORY_DEDICATED_ALLOCATE_INFO);
+       const VkExportMemoryAllocateInfo *export_info =
+               vk_find_struct_const(pAllocateInfo->pNext, EXPORT_MEMORY_ALLOCATE_INFO);
+       const VkImportMemoryHostPointerInfoEXT *host_ptr_info =
+               vk_find_struct_const(pAllocateInfo->pNext, IMPORT_MEMORY_HOST_POINTER_INFO_EXT);
+
+       const struct wsi_memory_allocate_info *wsi_info =
+               vk_find_struct_const(pAllocateInfo->pNext, WSI_MEMORY_ALLOCATE_INFO_MESA);
+
+
+       mem = vk_zalloc2(&device->vk.alloc, pAllocator, sizeof(*mem), 8,
+                         VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
+       if (mem == NULL)
+               return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
+
+       vk_object_base_init(&device->vk, &mem->base,
+                           VK_OBJECT_TYPE_DEVICE_MEMORY);
+
+       if (dedicate_info) {
+               mem->image = libresoc_image_from_handle(dedicate_info->image);
+               //mem->buffer = libresoc_buffer_from_handle(dedicate_info->buffer);
+       } else {
+               mem->image = NULL;
+               //mem->buffer = NULL;
+       }
+
+       // float priority_float = 0.5;
+       // const struct VkMemoryPriorityAllocateInfoEXT *priority_ext =
+       //      vk_find_struct_const(pAllocateInfo->pNext,
+       //                           MEMORY_PRIORITY_ALLOCATE_INFO_EXT);
+       // if (priority_ext)
+       //      priority_float = priority_ext->priority;
+
+       // unsigned priority = MIN2(LIBRESOC_BO_PRIORITY_APPLICATION_MAX - 1,
+       //                          (int)(priority_float * LIBRESOC_BO_PRIORITY_APPLICATION_MAX));
+
+       mem->user_ptr = NULL;
+       //mem->bo = NULL;
+
+
+       if (import_info) {
+               assert(import_info->handleType ==
+                      VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT ||
+                      import_info->handleType ==
+                      VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT);
+               // mem->bo = device->ws->buffer_from_fd(device->ws, import_info->fd,
+               //                                   priority, NULL);
+               // if (!mem->bo) {
+               //      result = VK_ERROR_INVALID_EXTERNAL_HANDLE;
+               //      goto fail;
+               // } else {
+               //      close(import_info->fd);
+               // }
+
+               // if (mem->image && mem->image->plane_count == 1 &&
+               //     !vk_format_is_depth_or_stencil(mem->image->vk_format)) {
+               //      struct radeon_bo_metadata metadata;
+               //      device->ws->buffer_get_metadata(mem->bo, &metadata);
+
+               //      struct libresoc_image_create_info create_info = {
+               //              .no_metadata_planes = true,
+               //              .bo_metadata = &metadata
+               //      };
+
+               //      /* This gives a basic ability to import radeonsi images
+               //       * that don't have DCC. This is not guaranteed by any
+               //       * spec and can be removed after we support modifiers. */
+               //      result = libresoc_image_create_layout(device, create_info, mem->image);
+               //      if (result != VK_SUCCESS) {
+               //              device->ws->buffer_destroy(mem->bo);
+               //              goto fail;
+               //      }
+               // }
+       } else if (host_ptr_info) {
+               // assert(host_ptr_info->handleType == VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT);
+               // mem->bo = device->ws->buffer_from_ptr(device->ws, host_ptr_info->pHostPointer,
+               //                                       pAllocateInfo->allocationSize,
+               //                                       priority);
+               // if (!mem->bo) {
+               //      result = VK_ERROR_INVALID_EXTERNAL_HANDLE;
+               //      goto fail;
+               // } else {
+               //      mem->user_ptr = host_ptr_info->pHostPointer;
+               // }
+       } else {
+               uint64_t alloc_size = align_u64(pAllocateInfo->allocationSize, 4096);
+               uint32_t heap_index;
+
+               heap_index = device->physical_device->memory_properties.memoryTypes[pAllocateInfo->memoryTypeIndex].heapIndex;
+               // domain = device->physical_device->memory_domains[pAllocateInfo->memoryTypeIndex];
+               // flags |= device->physical_device->memory_flags[pAllocateInfo->memoryTypeIndex];
+
+               // if (!dedicate_info && !import_info && (!export_info || !export_info->handleTypes)) {
+               //      flags |= RADEON_FLAG_NO_INTERPROCESS_SHARING;
+               //      if (device->use_global_bo_list) {
+               //              flags |= RADEON_FLAG_PREFER_LOCAL_BO;
+               //      }
+               // }
+
+               if (device->overallocation_disallowed) {
+                       uint64_t total_size =
+                               device->physical_device->memory_properties.memoryHeaps[heap_index].size;
+
+                       mtx_lock(&device->overallocation_mutex);
+                       if (device->allocated_memory_size[heap_index] + alloc_size > total_size) {
+                               mtx_unlock(&device->overallocation_mutex);
+                               result = VK_ERROR_OUT_OF_DEVICE_MEMORY;
+                               goto fail;
+                       }
+                       device->allocated_memory_size[heap_index] += alloc_size;
+                       mtx_unlock(&device->overallocation_mutex);
+               }
+
+               // mem->bo = device->ws->buffer_create(device->ws, alloc_size, device->physical_device->rad_info.max_alignment,
+               //                                     domain, flags, priority);
+
+               // if (!mem->bo) {
+               //      if (device->overallocation_disallowed) {
+               //              mtx_lock(&device->overallocation_mutex);
+               //              device->allocated_memory_size[heap_index] -= alloc_size;
+               //              mtx_unlock(&device->overallocation_mutex);
+               //      }
+               //      result = VK_ERROR_OUT_OF_DEVICE_MEMORY;
+               //      goto fail;
+               // }
+
+               mem->heap_index = heap_index;
+               mem->alloc_size = alloc_size;
+       }
+
+       // if (!wsi_info) {
+       //      result = libresoc_bo_list_add(device, mem->bo);
+       //      if (result != VK_SUCCESS)
+       //              goto fail;
+       // }
+
+       *pMem = libresoc_device_memory_to_handle(mem);
+
+       return VK_SUCCESS;
+
+fail:
+       libresoc_free_memory(device, pAllocator,mem);
+
+       return result;
+}
+
+VkResult libresoc_AllocateMemory(
+       VkDevice                                    _device,
+       const VkMemoryAllocateInfo*                 pAllocateInfo,
+       const VkAllocationCallbacks*                pAllocator,
+       VkDeviceMemory*                             pMem)
+{
+       LIBRESOC_FROM_HANDLE(libresoc_device, device, _device);
+       return libresoc_alloc_memory(device, pAllocateInfo, pAllocator, pMem);
+}
+
+void libresoc_FreeMemory(
+       VkDevice                                    _device,
+       VkDeviceMemory                              _mem,
+       const VkAllocationCallbacks*                pAllocator)
+{
+       LIBRESOC_FROM_HANDLE(libresoc_device, device, _device);
+       LIBRESOC_FROM_HANDLE(libresoc_device_memory, mem, _mem);
+
+       libresoc_free_memory(device, pAllocator, mem);
+}
+static VkResult
+libresoc_create_pthread_cond(pthread_cond_t *cond)
+{
+       pthread_condattr_t condattr;
+       if (pthread_condattr_init(&condattr)) {
+               return VK_ERROR_INITIALIZATION_FAILED;
+       }
+
+       if (pthread_condattr_setclock(&condattr, CLOCK_MONOTONIC)) {
+               pthread_condattr_destroy(&condattr);
+               return VK_ERROR_INITIALIZATION_FAILED;
+       }
+       if (pthread_cond_init(cond, &condattr)) {
+               pthread_condattr_destroy(&condattr);
+               return VK_ERROR_INITIALIZATION_FAILED;
+       }
+       pthread_condattr_destroy(&condattr);
+       return VK_SUCCESS;
+}
+
 VkResult
 libresoc_EnumerateInstanceExtensionProperties(const char *pLayerName,
                                           uint32_t *pPropertyCount,
@@ -80,6 +351,72 @@ static const VkAllocationCallbacks default_alloc = {
        .pfnFree = default_free_func,
 };
 
+static const struct debug_control libresoc_debug_options[] = {
+       {"nofastclears", LIBRESOC_DEBUG_NO_FAST_CLEARS},
+       {"nodcc", LIBRESOC_DEBUG_NO_DCC},
+       {"shaders", LIBRESOC_DEBUG_DUMP_SHADERS},
+       {"nocache", LIBRESOC_DEBUG_NO_CACHE},
+       {"shaderstats", LIBRESOC_DEBUG_DUMP_SHADER_STATS},
+       {"nohiz", LIBRESOC_DEBUG_NO_HIZ},
+       {"nocompute", LIBRESOC_DEBUG_NO_COMPUTE_QUEUE},
+       {"allbos", LIBRESOC_DEBUG_ALL_BOS},
+       {"noibs", LIBRESOC_DEBUG_NO_IBS},
+       {"spirv", LIBRESOC_DEBUG_DUMP_SPIRV},
+       {"vmfaults", LIBRESOC_DEBUG_VM_FAULTS},
+       {"zerovram", LIBRESOC_DEBUG_ZERO_VRAM},
+       {"syncshaders", LIBRESOC_DEBUG_SYNC_SHADERS},
+       {"preoptir", LIBRESOC_DEBUG_PREOPTIR},
+       {"nodynamicbounds", LIBRESOC_DEBUG_NO_DYNAMIC_BOUNDS},
+       {"nooutoforder", LIBRESOC_DEBUG_NO_OUT_OF_ORDER},
+       {"info", LIBRESOC_DEBUG_INFO},
+       {"errors", LIBRESOC_DEBUG_ERRORS},
+       {"startup", LIBRESOC_DEBUG_STARTUP},
+       {"checkir", LIBRESOC_DEBUG_CHECKIR},
+       {"nothreadllvm", LIBRESOC_DEBUG_NOTHREADLLVM},
+       {"nobinning", LIBRESOC_DEBUG_NOBINNING},
+       {"nongg", LIBRESOC_DEBUG_NO_NGG},
+       {"allentrypoints", LIBRESOC_DEBUG_ALL_ENTRYPOINTS},
+       {"metashaders", LIBRESOC_DEBUG_DUMP_META_SHADERS},
+       {"nomemorycache", LIBRESOC_DEBUG_NO_MEMORY_CACHE},
+       {"llvm", LIBRESOC_DEBUG_LLVM},
+       {"forcecompress", LIBRESOC_DEBUG_FORCE_COMPRESS},
+       {"nir", LIBRESOC_DEBUG_DUMP_NIR},
+       {NULL, 0}
+};
+
+const char *
+libresoc_get_debug_option_name(int id)
+{
+       assert(id < ARRAY_SIZE(libresoc_debug_options) - 1);
+       return libresoc_debug_options[id].string;
+}
+
+static const char libresoc_dri_options_xml[] =
+DRI_CONF_BEGIN
+       DRI_CONF_SECTION_PERFORMANCE
+               DRI_CONF_ADAPTIVE_SYNC("true")
+               DRI_CONF_VK_X11_OVERRIDE_MIN_IMAGE_COUNT(0)
+               DRI_CONF_VK_X11_STRICT_IMAGE_COUNT("false")
+               DRI_CONF_VK_X11_ENSURE_MIN_IMAGE_COUNT("false")
+       DRI_CONF_SECTION_END
+
+       DRI_CONF_SECTION_DEBUG
+               DRI_CONF_VK_WSI_FORCE_BGRA8_UNORM_FIRST("false")
+       DRI_CONF_SECTION_END
+DRI_CONF_END;
+
+static void  libresoc_init_dri_options(struct libresoc_instance *instance)
+{
+       driParseOptionInfo(&instance->available_dri_options, libresoc_dri_options_xml);
+       driParseConfigFiles(&instance->dri_options,
+                           &instance->available_dri_options,
+                           0, "libresoc", NULL,
+                           instance->app_info.app_name,
+                           instance->app_info.app_version,
+                           instance->engineName,
+                           instance->engineVersion);
+}
+
 VkResult
 libresoc_CreateInstance(const VkInstanceCreateInfo *pCreateInfo,
                     const VkAllocationCallbacks *pAllocator,
@@ -110,6 +447,8 @@ libresoc_CreateInstance(const VkInstanceCreateInfo *pCreateInfo,
                instance->engineVersion = app->engineVersion;
                instance->apiVersion = app->apiVersion;
        }
+       instance->debug_flags = parse_debug_string(getenv("LIBRESOC_DEBUG"),
+                                                  libresoc_debug_options);
        /*TODO : enable extensions*/
        for (uint32_t i = 0; i < pCreateInfo->enabledExtensionCount; i++) {
                int idx;
@@ -167,6 +506,8 @@ libresoc_CreateInstance(const VkInstanceCreateInfo *pCreateInfo,
        }
        instance->physical_devices_enumerated = false;
        list_inithead(&instance->physical_devices);
+       glsl_type_singleton_init_or_ref();
+       libresoc_init_dri_options(instance);
        *pInstance = libresoc_instance_to_handle(instance);
 
        return VK_SUCCESS;
@@ -182,11 +523,57 @@ libresoc_DestroyInstance(VkInstance _instance,
    /* FIXME: stub */
 }
 
+static void
+libresoc_physical_device_init_mem_types(struct libresoc_physical_device *device)
+{
+       uint64_t visible_vram_size = 65536; //TODO: some dummy value
+       uint64_t vram_size = 65536; //TODO: some dummy value
+       int vram_index = -1, visible_vram_index = -1;
+       device->memory_properties.memoryHeapCount = 0;
+       if (vram_size > 0) {
+               vram_index = device->memory_properties.memoryHeapCount++;
+               device->memory_properties.memoryHeaps[vram_index] = (VkMemoryHeap) {
+                       .size = vram_size,
+                       .flags = VK_MEMORY_HEAP_DEVICE_LOCAL_BIT,
+               };
+       }
+
+       if (visible_vram_size) {
+               visible_vram_index = device->memory_properties.memoryHeapCount++;
+               device->memory_properties.memoryHeaps[visible_vram_index] = (VkMemoryHeap) {
+                       .size = visible_vram_size,
+                       .flags = VK_MEMORY_HEAP_DEVICE_LOCAL_BIT,
+               };
+       }
+       unsigned type_count = 0;
+
+       if (vram_index >= 0 || visible_vram_index >= 0) {
+               device->memory_properties.memoryTypes[type_count++] = (VkMemoryType) {
+                       .propertyFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT,
+                       .heapIndex = vram_index >= 0 ? vram_index : visible_vram_index,
+               };
+       }
+
+       if (visible_vram_index >= 0) {
+               device->memory_properties.memoryTypes[type_count++] = (VkMemoryType) {
+                       .propertyFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT |
+                       VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
+                       VK_MEMORY_PROPERTY_HOST_COHERENT_BIT,
+                       .heapIndex = visible_vram_index,
+               };
+       }
+
+       device->memory_properties.memoryTypeCount = type_count;
+
+
+}
 static VkResult
 libresoc_physical_device_try_create(struct libresoc_instance *instance,
                                struct libresoc_physical_device **device_out)
 {
        VkResult result;
+       int fd = -1;
+       int master_fd = -1;
 
        struct libresoc_physical_device *device =
                vk_zalloc2(&instance->alloc, NULL, sizeof(*device), 8,
@@ -199,10 +586,20 @@ libresoc_physical_device_try_create(struct libresoc_instance *instance,
        device->_loader_data.loaderMagic = ICD_LOADER_MAGIC;
        device->instance = instance;
 
+       device->master_fd = master_fd;
+       device->local_fd = fd;
+       libresoc_physical_device_init_mem_types(device);
+
+       libresoc_physical_device_get_supported_extensions(device,
+                                                     &device->supported_extensions);
+       result = libresoc_init_wsi(device);
+       if (result != VK_SUCCESS) {
+               return result;
+       }
        snprintf(device->name, sizeof(device->name),
                 "LIBRE-SOC DEVICE");
        *device_out = device;
-
+//TODO: incase of failures need to deallocate and cleanup various allocation properly.
        return VK_SUCCESS;
 
 }
@@ -460,15 +857,15 @@ libresoc_GetPhysicalDeviceProperties(VkPhysicalDevice physicalDevice,
        *pProperties = (VkPhysicalDeviceProperties) {
                .apiVersion = libresoc_physical_device_api_version(pdevice),
                .driverVersion = vk_get_driver_version(),
-               .vendorID = 1,
-               .deviceID = 1,
+               .vendorID = 1, //TODO: some dummy value
+               .deviceID = 1, //TODO: dome dummy value
                .deviceType = VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU,
                .limits = limits,
                .sparseProperties = {0},
        };
 
        strcpy(pProperties->deviceName, pdevice->name);
-//     memcpy(pProperties->pipelineCacheUUID, pdevice->cache_uuid, VK_UUID_SIZE);
+       memcpy(pProperties->pipelineCacheUUID, pdevice->cache_uuid, VK_UUID_SIZE);
 }
 
 static void libresoc_get_physical_device_queue_family_properties(
@@ -533,36 +930,39 @@ libresoc_GetPhysicalDeviceMemoryProperties(VkPhysicalDevice physicalDevice,
        if (getenv("LIBRESOC_TRACE")) {
                fprintf(stderr, "GetPhysicalDEviceMemoryProperties called. \n");
        }
-   /* FIXME: stub */
-}
+       LIBRESOC_FROM_HANDLE(libresoc_physical_device, physical_device, physicalDevice);
 
-void
-libresoc_GetPhysicalDeviceFormatProperties(VkPhysicalDevice physicalDevice, VkFormat format, VkFormatProperties *pFormatProperties) {
-       if (getenv("LIBRESOC_TRACE")) {
-               fprintf(stderr, "GetPhysicalDeviceFormatProperties called. \n");
-       }
+       *pMemoryProperties = physical_device->memory_properties;
    /* FIXME: stub */
 }
 
-VkResult
-      libresoc_GetPhysicalDeviceImageFormatProperties(VkPhysicalDevice physicalDevice, VkFormat format, VkImageType type, VkImageTiling tiling, VkImageUsageFlags usage, VkImageCreateFlags flags, VkImageFormatProperties* pImageFormatProperties)
-{
-       if (getenv("LIBRESOC_TRACE")) {
-               fprintf(stderr, "GetPhysicalDEviceImageFormatProperties called. \n");
-       }
-      
-   /* FIXME: stub */
-   return VK_SUCCESS;
-}
-      void 
-      libresoc_GetPhysicalDeviceSparseImageFormatProperties(VkPhysicalDevice physicalDevice, VkFormat format, VkImageType type, VkSampleCountFlagBits samples, VkImageUsageFlags usage, VkImageTiling tiling, uint32_t* pPropertyCount, VkSparseImageFormatProperties* pProperties)
-      {
-       if (getenv("LIBRESOC_TRACE")) {
-               fprintf(stderr, "GetPhysicalDeviceSparseImageFormatProperties called. \n");
-       }
-   /* FIXME: stub */
-      }
+//void
+//libresoc_GetPhysicalDeviceFormatProperties(VkPhysicalDevice physicalDevice, VkFormat format, VkFormatProperties *pFormatProperties) {
+// 
+//     if (getenv("LIBRESOC_TRACE")) {
+//             fprintf(stderr, "GetPhysicalDeviceFormatProperties called. \n");
+//     }
+//   /* FIXME: stub */
+//}
+
+//VkResult
+//      libresoc_GetPhysicalDeviceImageFormatProperties(VkPhysicalDevice physicalDevice, VkFormat format, VkImageType type, VkImageTiling tiling, VkImageUsageFlags usage, VkImageCreateFlags flags, VkImageFormatProperties* pImageFormatProperties)
+//{
+//     if (getenv("LIBRESOC_TRACE")) {
+//             fprintf(stderr, "GetPhysicalDEviceImageFormatProperties called. \n");
+//     }
+//      
+//   /* FIXME: stub */
+//   return VK_SUCCESS;
+//}
+//      void 
+//      libresoc_GetPhysicalDeviceSparseImageFormatProperties(VkPhysicalDevice physicalDevice, VkFormat format, VkImageType type, VkSampleCountFlagBits samples, VkImageUsageFlags usage, VkImageTiling tiling, uint32_t* pPropertyCount, VkSparseImageFormatProperties* pProperties)
+//      {
+//     if (getenv("LIBRESOC_TRACE")) {
+//             fprintf(stderr, "GetPhysicalDeviceSparseImageFormatProperties called. \n");
+//     }
+//   /* FIXME: stub */
+//      }
 PFN_vkVoidFunction
 libresoc_GetInstanceProcAddr(VkInstance _instance,
                          const char *pName)
@@ -732,27 +1132,6 @@ libresoc_device_init_dispatch(struct libresoc_device *device)
        }
 }
 
-/*
-static VkResult
-libresoc_create_pthread_cond(pthread_cond_t *cond)
-{
-       pthread_condattr_t condattr;
-       if (pthread_condattr_init(&condattr)) {
-               return VK_ERROR_INITIALIZATION_FAILED;
-       }
-
-       if (pthread_condattr_setclock(&condattr, CLOCK_MONOTONIC)) {
-               pthread_condattr_destroy(&condattr);
-               return VK_ERROR_INITIALIZATION_FAILED;
-       }
-       if (pthread_cond_init(cond, &condattr)) {
-               pthread_condattr_destroy(&condattr);
-               return VK_ERROR_INITIALIZATION_FAILED;
-       }
-       pthread_condattr_destroy(&condattr);
-       return VK_SUCCESS;
-}
-*/
 static VkResult
 check_physical_device_features(VkPhysicalDevice physicalDevice,
                               const VkPhysicalDeviceFeatures *features)
@@ -790,6 +1169,16 @@ libresoc_queue_init(struct libresoc_device *device, struct libresoc_queue *queue
        queue->queue_family_index = queue_family_index;
        queue->queue_idx = idx;
        queue->flags = flags;
+       list_inithead(&queue->pending_submissions);
+       pthread_mutex_init(&queue->pending_mutex, NULL);
+
+       pthread_mutex_init(&queue->thread_mutex, NULL);
+       //queue->thread_submission = NULL;
+       queue->thread_running = queue->thread_exit = false;
+       VkResult result = libresoc_create_pthread_cond(&queue->thread_cond);
+       if (result != VK_SUCCESS)
+               return vk_error(device->instance, result);
+
        return VK_SUCCESS;
 }
 
@@ -946,14 +1335,426 @@ libresoc_DestroyDevice(VkDevice _device,
    /* FIXME: stub */
 }
 
-void
-libresoc_GetDeviceQueue(VkDevice _device,
-                    uint32_t queueNodeIndex,
-                    uint32_t queueIndex,
-                    VkQueue *pQueue)
+void libresoc_GetDeviceQueue(
+       VkDevice                                    _device,
+       uint32_t                                    queueFamilyIndex,
+       uint32_t                                    queueIndex,
+       VkQueue*                                    pQueue)
 {
        if (getenv("LIBRESOC_TRACE")) {
                fprintf(stderr, "GetDeviceQueue called. \n");
        }
-   /* FIXME: stub */
+       LIBRESOC_FROM_HANDLE(libresoc_device, device, _device);
+       struct libresoc_queue *queue;
+
+       queue = &device->queues[queueFamilyIndex][queueIndex];
+       *pQueue = libresoc_queue_to_handle(queue);
+}
+
+VkResult libresoc_QueueWaitIdle(
+       VkQueue                                     _queue)
+{
+       LIBRESOC_FROM_HANDLE(libresoc_queue, queue, _queue);
+
+       pthread_mutex_lock(&queue->pending_mutex);
+       while (!list_is_empty(&queue->pending_submissions)) {
+               pthread_cond_wait(&queue->device->timeline_cond, &queue->pending_mutex);
+       }
+       pthread_mutex_unlock(&queue->pending_mutex);
+
+       return VK_SUCCESS;
+}
+
+VkResult libresoc_DeviceWaitIdle(
+       VkDevice                                    _device)
+{
+       LIBRESOC_FROM_HANDLE(libresoc_device, device, _device);
+
+       for (unsigned i = 0; i < LIBRESOC_MAX_QUEUE_FAMILIES; i++) {
+               for (unsigned q = 0; q < device->queue_count[i]; q++) {
+                       VkResult result =
+                               libresoc_QueueWaitIdle(libresoc_queue_to_handle(&device->queues[i][q]));
+
+                       if (result != VK_SUCCESS)
+                               return result;
+               }
+       }
+       return VK_SUCCESS;
+}
+
+void libresoc_GetPhysicalDeviceProperties2(
+       VkPhysicalDevice                            physicalDevice,
+       VkPhysicalDeviceProperties2                *pProperties)
+{
+       LIBRESOC_FROM_HANDLE(libresoc_physical_device, pdevice, physicalDevice);
+       libresoc_GetPhysicalDeviceProperties(physicalDevice, &pProperties->properties);
+       //TODO: add more stuffs when required
+}
+
+void libresoc_GetPhysicalDeviceFeatures2(
+       VkPhysicalDevice                            physicalDevice,
+       VkPhysicalDeviceFeatures2                  *pFeatures)
+{
+       LIBRESOC_FROM_HANDLE(libresoc_physical_device, pdevice, physicalDevice);
+       libresoc_GetPhysicalDeviceFeatures(physicalDevice, &pFeatures->features);
+}
+
+void libresoc_GetPhysicalDeviceQueueFamilyProperties2(
+       VkPhysicalDevice                            physicalDevice,
+       uint32_t*                                   pCount,
+       VkQueueFamilyProperties2                   *pQueueFamilyProperties)
+{
+       LIBRESOC_FROM_HANDLE(libresoc_physical_device, pdevice, physicalDevice);
+       if (!pQueueFamilyProperties) {
+               libresoc_get_physical_device_queue_family_properties(pdevice, pCount, NULL);
+               return;
+       }
+       VkQueueFamilyProperties *properties[] = {
+               &pQueueFamilyProperties[0].queueFamilyProperties,
+               &pQueueFamilyProperties[1].queueFamilyProperties,
+               &pQueueFamilyProperties[2].queueFamilyProperties,
+       };
+       libresoc_get_physical_device_queue_family_properties(pdevice, pCount, properties);
+       assert(*pCount <= 3);
+}
+
+static void
+libresoc_get_memory_budget_properties(VkPhysicalDevice physicalDevice,
+                                 VkPhysicalDeviceMemoryBudgetPropertiesEXT *memoryBudget)
+{
+//TODO: stub
+}
+
+void libresoc_GetPhysicalDeviceMemoryProperties2(
+       VkPhysicalDevice                            physicalDevice,
+       VkPhysicalDeviceMemoryProperties2          *pMemoryProperties)
+{
+       libresoc_GetPhysicalDeviceMemoryProperties(physicalDevice,
+                                              &pMemoryProperties->memoryProperties);
+
+       VkPhysicalDeviceMemoryBudgetPropertiesEXT *memory_budget =
+               vk_find_struct(pMemoryProperties->pNext,
+                              PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT);
+       if (memory_budget)
+               libresoc_get_memory_budget_properties(physicalDevice, memory_budget);
+}
+
+VkResult libresoc_CreateSemaphore(
+       VkDevice                                    _device,
+       const VkSemaphoreCreateInfo*                pCreateInfo,
+       const VkAllocationCallbacks*                pAllocator,
+       VkSemaphore*                                pSemaphore)
+{
+       //TODO: minimal things as of now, add more complex code as required
+       LIBRESOC_FROM_HANDLE(libresoc_device, device, _device);
+       struct libresoc_semaphore *sem = vk_alloc2(&device->vk.alloc, pAllocator,
+                                              sizeof(*sem), 8,
+                                              VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
+       if (!sem)
+               return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
+
+       vk_object_base_init(&device->vk, &sem->base,
+                           VK_OBJECT_TYPE_SEMAPHORE);
+
+       *pSemaphore = libresoc_semaphore_to_handle(sem);
+       return VK_SUCCESS;
+}
+
+void libresoc_GetImageMemoryRequirements(
+       VkDevice                                    _device,
+       VkImage                                     _image,
+       VkMemoryRequirements*                       pMemoryRequirements)
+{
+       LIBRESOC_FROM_HANDLE(libresoc_device, device, _device);
+       LIBRESOC_FROM_HANDLE(libresoc_image, image, _image);
+
+       pMemoryRequirements->memoryTypeBits = (1u << device->physical_device->memory_properties.memoryTypeCount) - 1;
+
+       pMemoryRequirements->size = image->size;
+       pMemoryRequirements->alignment = image->alignment;
+}
+
+void libresoc_GetImageMemoryRequirements2(
+       VkDevice                                    device,
+       const VkImageMemoryRequirementsInfo2       *pInfo,
+       VkMemoryRequirements2                      *pMemoryRequirements)
+{
+       libresoc_GetImageMemoryRequirements(device, pInfo->image,
+                                        &pMemoryRequirements->memoryRequirements);
+}
+
+VkResult libresoc_BindImageMemory2(VkDevice device,
+                               uint32_t bindInfoCount,
+                               const VkBindImageMemoryInfo *pBindInfos)
+{
+       for (uint32_t i = 0; i < bindInfoCount; ++i) {
+               LIBRESOC_FROM_HANDLE(libresoc_device_memory, mem, pBindInfos[i].memory);
+               LIBRESOC_FROM_HANDLE(libresoc_image, image, pBindInfos[i].image);
+
+               if (mem) {
+                       // image->bo = mem->bo;
+                       // image->offset = pBindInfos[i].memoryOffset;
+               } else {
+                       // image->bo = NULL;
+                       // image->offset = 0;
+               }
+       }
+       return VK_SUCCESS;
+}
+
+
+VkResult libresoc_BindImageMemory(
+       VkDevice                                    device,
+       VkImage                                     image,
+       VkDeviceMemory                              memory,
+       VkDeviceSize                                memoryOffset)
+{
+       const VkBindImageMemoryInfo info = {
+               .sType = VK_STRUCTURE_TYPE_BIND_BUFFER_MEMORY_INFO,
+               .image = image,
+               .memory = memory,
+               .memoryOffset = memoryOffset
+       };
+
+       return libresoc_BindImageMemory2(device, 1, &info);
+}
+
+static VkResult libresoc_queue_submit(struct libresoc_queue *queue,
+                                  const struct libresoc_queue_submission *submission)
+{
+       return VK_SUCCESS;
+       // struct libresoc_deferred_queue_submission *deferred = NULL;
+
+       // VkResult result = libresoc_create_deferred_submission(queue, submission, &deferred);
+       // if (result != VK_SUCCESS)
+       //      return result;
+
+       // struct list_head processing_list;
+       // list_inithead(&processing_list);
+
+       // result = libresoc_queue_enqueue_submission(deferred, &processing_list);
+       // if (result != VK_SUCCESS) {
+       //      /* If anything is in the list we leak. */
+       //      assert(list_is_empty(&processing_list));
+       //      return result;
+       // }
+       // return libresoc_process_submissions(&processing_list);
+}
+
+/* Signals fence as soon as all the work currently put on queue is done. */
+static VkResult libresoc_signal_fence(struct libresoc_queue *queue,
+                              VkFence fence)
+{
+       return libresoc_queue_submit(queue, &(struct libresoc_queue_submission) {
+                       .fence = fence
+               });
+}
+
+static bool libresoc_submit_has_effects(const VkSubmitInfo *info)
+{
+       return info->commandBufferCount ||
+              info->waitSemaphoreCount ||
+              info->signalSemaphoreCount;
+}
+
+VkResult libresoc_QueueSubmit(
+       VkQueue                                     _queue,
+       uint32_t                                    submitCount,
+       const VkSubmitInfo*                         pSubmits,
+       VkFence                                     fence)
+{
+       LIBRESOC_FROM_HANDLE(libresoc_queue, queue, _queue);
+       VkResult result;
+       uint32_t fence_idx = 0;
+       bool flushed_caches = false;
+
+       if (fence != VK_NULL_HANDLE) {
+               for (uint32_t i = 0; i < submitCount; ++i)
+                       if (libresoc_submit_has_effects(pSubmits + i))
+                               fence_idx = i;
+       } else
+               fence_idx = UINT32_MAX;
+
+       for (uint32_t i = 0; i < submitCount; i++) {
+               if (!libresoc_submit_has_effects(pSubmits + i) && fence_idx != i)
+                       continue;
+
+               VkPipelineStageFlags wait_dst_stage_mask = 0;
+               for (unsigned j = 0; j < pSubmits[i].waitSemaphoreCount; ++j) {
+                       wait_dst_stage_mask |= pSubmits[i].pWaitDstStageMask[j];
+               }
+
+               const VkTimelineSemaphoreSubmitInfo *timeline_info =
+                       vk_find_struct_const(pSubmits[i].pNext, TIMELINE_SEMAPHORE_SUBMIT_INFO);
+
+               result = libresoc_queue_submit(queue, &(struct libresoc_queue_submission) {
+                               .cmd_buffers = pSubmits[i].pCommandBuffers,
+                               .cmd_buffer_count = pSubmits[i].commandBufferCount,
+                               .wait_dst_stage_mask = wait_dst_stage_mask,
+                               .flush_caches = !flushed_caches,
+                               .wait_semaphores = pSubmits[i].pWaitSemaphores,
+                               .wait_semaphore_count = pSubmits[i].waitSemaphoreCount,
+                               .signal_semaphores = pSubmits[i].pSignalSemaphores,
+                               .signal_semaphore_count = pSubmits[i].signalSemaphoreCount,
+                               .fence = i == fence_idx ? fence : VK_NULL_HANDLE,
+                               .wait_values = timeline_info ? timeline_info->pWaitSemaphoreValues : NULL,
+                               .wait_value_count = timeline_info && timeline_info->pWaitSemaphoreValues ? timeline_info->waitSemaphoreValueCount : 0,
+                               .signal_values = timeline_info ? timeline_info->pSignalSemaphoreValues : NULL,
+                               .signal_value_count = timeline_info && timeline_info->pSignalSemaphoreValues ? timeline_info->signalSemaphoreValueCount : 0,
+                       });
+               if (result != VK_SUCCESS)
+                       return result;
+
+               flushed_caches  = true;
+       }
+
+       if (fence != VK_NULL_HANDLE && !submitCount) {
+               result = libresoc_signal_fence(queue, fence);
+               if (result != VK_SUCCESS)
+                       return result;
+       }
+
+       return VK_SUCCESS;
+}
+
+VkResult libresoc_CreateFence(
+       VkDevice                                    _device,
+       const VkFenceCreateInfo*                    pCreateInfo,
+       const VkAllocationCallbacks*                pAllocator,
+       VkFence*                                    pFence)
+{
+       LIBRESOC_FROM_HANDLE(libresoc_device, device, _device);
+       const VkExportFenceCreateInfo *export =
+               vk_find_struct_const(pCreateInfo->pNext, EXPORT_FENCE_CREATE_INFO);
+       VkExternalFenceHandleTypeFlags handleTypes =
+               export ? export->handleTypes : 0;
+       struct libresoc_fence *fence;
+
+       fence = vk_zalloc2(&device->vk.alloc, pAllocator, sizeof(*fence), 8,
+                          VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
+       if (!fence)
+               return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
+
+       vk_object_base_init(&device->vk, &fence->base, VK_OBJECT_TYPE_FENCE);
+
+       // if (device->always_use_syncobj || handleTypes) {
+       //      fence->permanent.kind = LIBRESOC_FENCE_SYNCOBJ;
+
+       //      bool create_signaled = false;
+       //      if (pCreateInfo->flags & VK_FENCE_CREATE_SIGNALED_BIT)
+       //              create_signaled = true;
+
+       //      int ret = device->ws->create_syncobj(device->ws, create_signaled,
+       //                                           &fence->permanent.syncobj);
+       //      if (ret) {
+       //              libresoc_destroy_fence(device, pAllocator, fence);
+       //              return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
+       //      }
+       // } else {
+       //      fence->permanent.kind = LIBRESOC_FENCE_WINSYS;
+
+       //      fence->permanent.fence = device->ws->create_fence();
+       //      if (!fence->permanent.fence) {
+       //              vk_free2(&device->vk.alloc, pAllocator, fence);
+       //              libresoc_destroy_fence(device, pAllocator, fence);
+       //              return vk_error(device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
+       //      }
+       //      if (pCreateInfo->flags & VK_FENCE_CREATE_SIGNALED_BIT)
+       //              device->ws->signal_fence(fence->permanent.fence);
+       // }
+
+       *pFence = libresoc_fence_to_handle(fence);
+
+       return VK_SUCCESS;
+}
+
+VkResult libresoc_MapMemory(
+       VkDevice                                    _device,
+       VkDeviceMemory                              _memory,
+       VkDeviceSize                                offset,
+       VkDeviceSize                                size,
+       VkMemoryMapFlags                            flags,
+       void**                                      ppData)
+{
+       LIBRESOC_FROM_HANDLE(libresoc_device, device, _device);
+       LIBRESOC_FROM_HANDLE(libresoc_device_memory, mem, _memory);
+
+       if (mem == NULL) {
+               *ppData = NULL;
+               return VK_SUCCESS;
+       }
+
+       if (mem->user_ptr)
+               *ppData = mem->user_ptr;
+       // else
+       //      *ppData = device->ws->buffer_map(mem->bo);
+
+       if (*ppData) {
+               *ppData += offset;
+               return VK_SUCCESS;
+       }
+
+       return vk_error(device->instance, VK_ERROR_MEMORY_MAP_FAILED);
+}
+
+void libresoc_UnmapMemory(
+       VkDevice                                    _device,
+       VkDeviceMemory                              _memory)
+{
+       LIBRESOC_FROM_HANDLE(libresoc_device, device, _device);
+       LIBRESOC_FROM_HANDLE(libresoc_device_memory, mem, _memory);
+
+       if (mem == NULL)
+               return;
+
+       // if (mem->user_ptr == NULL)
+       //      device->ws->buffer_unmap(mem->bo);
+}
+
+VkResult libresoc_WaitForFences(
+       VkDevice                                    _device,
+       uint32_t                                    fenceCount,
+       const VkFence*                              pFences,
+       VkBool32                                    waitAll,
+       uint64_t                                    timeout)
+{
+       //TODO: stub
+       return VK_SUCCESS;
+}
+
+VkResult libresoc_ResetFences(VkDevice _device,
+                         uint32_t fenceCount,
+                         const VkFence *pFences)
+{
+
+       //TODO: stub
+       return VK_SUCCESS;
+}
+
+VkResult libresoc_CreateFramebuffer(
+       VkDevice                                    _device,
+       const VkFramebufferCreateInfo*              pCreateInfo,
+       const VkAllocationCallbacks*                pAllocator,
+       VkFramebuffer*                              pFramebuffer)
+{
+       //TODO: stub
+       return VK_SUCCESS;
 }
+void libresoc_DestroyBuffer(
+       VkDevice                                    _device,
+       VkBuffer                                    _buffer,
+       const VkAllocationCallbacks*                pAllocator)
+{}
+
+void libresoc_DestroyFence(
+       VkDevice                                    _device,
+       VkFence                                     _fence,
+       const VkAllocationCallbacks*                pAllocator)
+{}
+
+
+void libresoc_DestroySemaphore(
+       VkDevice                                    _device,
+       VkSemaphore                                 _semaphore,
+       const VkAllocationCallbacks*                pAllocator)
+{}