Implement libresoc_DeviceWaitIdle() libresoc_QueueWaitIdle().
[mesa.git] / src / libre-soc / vulkan / libresoc_device.c
index 089aa36cee0e27b829fc2fbaf48d41246210bf16..229fd172a0d7819579ec68af934c2c0bd5e165b2 100644 (file)
 #include <sys/sysinfo.h>
 #include <unistd.h>
 
+#include "util/debug.h"
 #include "libresoc_private.h"
 #include "vk_util.h"
+#include "vk_alloc.h"
 
 VkResult
 libresoc_EnumerateInstanceExtensionProperties(const char *pLayerName,
@@ -52,6 +54,71 @@ libresoc_EnumerateInstanceExtensionProperties(const char *pLayerName,
    return vk_outarray_status(&out);
 }
 
+static void *
+default_alloc_func(void *pUserData, size_t size, size_t align,
+                   VkSystemAllocationScope allocationScope)
+{
+       return malloc(size);
+}
+
+static void *
+default_realloc_func(void *pUserData, void *pOriginal, size_t size,
+                     size_t align, VkSystemAllocationScope allocationScope)
+{
+       return realloc(pOriginal, size);
+}
+
+static void
+default_free_func(void *pUserData, void *pMemory)
+{
+       free(pMemory);
+}
+
+static const VkAllocationCallbacks default_alloc = {
+       .pUserData = NULL,
+       .pfnAllocation = default_alloc_func,
+       .pfnReallocation = default_realloc_func,
+       .pfnFree = default_free_func,
+};
+
+static const struct debug_control libresoc_debug_options[] = {
+       {"nofastclears", LIBRESOC_DEBUG_NO_FAST_CLEARS},
+       {"nodcc", LIBRESOC_DEBUG_NO_DCC},
+       {"shaders", LIBRESOC_DEBUG_DUMP_SHADERS},
+       {"nocache", LIBRESOC_DEBUG_NO_CACHE},
+       {"shaderstats", LIBRESOC_DEBUG_DUMP_SHADER_STATS},
+       {"nohiz", LIBRESOC_DEBUG_NO_HIZ},
+       {"nocompute", LIBRESOC_DEBUG_NO_COMPUTE_QUEUE},
+       {"allbos", LIBRESOC_DEBUG_ALL_BOS},
+       {"noibs", LIBRESOC_DEBUG_NO_IBS},
+       {"spirv", LIBRESOC_DEBUG_DUMP_SPIRV},
+       {"vmfaults", LIBRESOC_DEBUG_VM_FAULTS},
+       {"zerovram", LIBRESOC_DEBUG_ZERO_VRAM},
+       {"syncshaders", LIBRESOC_DEBUG_SYNC_SHADERS},
+       {"preoptir", LIBRESOC_DEBUG_PREOPTIR},
+       {"nodynamicbounds", LIBRESOC_DEBUG_NO_DYNAMIC_BOUNDS},
+       {"nooutoforder", LIBRESOC_DEBUG_NO_OUT_OF_ORDER},
+       {"info", LIBRESOC_DEBUG_INFO},
+       {"errors", LIBRESOC_DEBUG_ERRORS},
+       {"startup", LIBRESOC_DEBUG_STARTUP},
+       {"checkir", LIBRESOC_DEBUG_CHECKIR},
+       {"nothreadllvm", LIBRESOC_DEBUG_NOTHREADLLVM},
+       {"nobinning", LIBRESOC_DEBUG_NOBINNING},
+       {"nongg", LIBRESOC_DEBUG_NO_NGG},
+       {"allentrypoints", LIBRESOC_DEBUG_ALL_ENTRYPOINTS},
+       {"metashaders", LIBRESOC_DEBUG_DUMP_META_SHADERS},
+       {"nomemorycache", LIBRESOC_DEBUG_NO_MEMORY_CACHE},
+       {"llvm", LIBRESOC_DEBUG_LLVM},
+       {"forcecompress", LIBRESOC_DEBUG_FORCE_COMPRESS},
+       {NULL, 0}
+};
+
+const char *
+libresoc_get_debug_option_name(int id)
+{
+       assert(id < ARRAY_SIZE(libresoc_debug_options) - 1);
+       return libresoc_debug_options[id].string;
+}
 VkResult
 libresoc_CreateInstance(const VkInstanceCreateInfo *pCreateInfo,
                     const VkAllocationCallbacks *pAllocator,
@@ -60,8 +127,90 @@ libresoc_CreateInstance(const VkInstanceCreateInfo *pCreateInfo,
        if (getenv("LIBRESOC_TRACE")) {
                fprintf(stderr, "CreateInstance called. \n");
        }
-   /* FIXME: stub */
-   return VK_SUCCESS;
+       struct libresoc_instance *instance;
+
+       instance = vk_zalloc2(&default_alloc, pAllocator, sizeof(*instance), 8,
+                             VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE);
+       if (!instance)
+               return vk_error(NULL, VK_ERROR_OUT_OF_HOST_MEMORY);
+
+       vk_object_base_init(NULL, &instance->base, VK_OBJECT_TYPE_INSTANCE);
+
+       if (pAllocator)
+               instance->alloc = *pAllocator;
+       else
+               instance->alloc = default_alloc;
+       if (pCreateInfo->pApplicationInfo) {
+               const VkApplicationInfo *app = pCreateInfo->pApplicationInfo;
+
+               instance->engineName =
+                       vk_strdup(&instance->alloc, app->pEngineName,
+                                 VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE);
+               instance->engineVersion = app->engineVersion;
+               instance->apiVersion = app->apiVersion;
+       }
+       instance->debug_flags = parse_debug_string(getenv("LIBRESOC_DEBUG"),
+                                                  libresoc_debug_options);
+       /*TODO : enable extensions*/
+       for (uint32_t i = 0; i < pCreateInfo->enabledExtensionCount; i++) {
+               int idx;
+               for (idx = 0; idx < LIBRESOC_INSTANCE_EXTENSION_COUNT; idx++) {
+                       if (!strcmp(pCreateInfo->ppEnabledExtensionNames[i],
+                                               libresoc_instance_extensions[idx].extensionName))
+                               break;
+               }
+
+               if (idx >= LIBRESOC_INSTANCE_EXTENSION_COUNT ||
+                               !libresoc_instance_extensions_supported.extensions[idx]) {
+                       vk_object_base_finish(&instance->base);
+                       vk_free2(&default_alloc, pAllocator, instance);
+                       return vk_error(instance, VK_ERROR_EXTENSION_NOT_PRESENT);
+               }
+
+               instance->enabled_extensions.extensions[idx] = true;
+       }
+       for (unsigned i = 0; i < ARRAY_SIZE(instance->dispatch.entrypoints); i++) {
+               /* Vulkan requires that entrypoints for extensions which have
+                * not been enabled must not be advertised.
+                */
+               if (!libresoc_instance_entrypoint_is_enabled(i, instance->apiVersion,
+                                       &instance->enabled_extensions)) {
+                       instance->dispatch.entrypoints[i] = NULL;
+               } else {
+                       instance->dispatch.entrypoints[i] =
+                               libresoc_instance_dispatch_table.entrypoints[i];
+               }
+       }
+       for (unsigned i = 0; i < ARRAY_SIZE(instance->physical_device_dispatch.entrypoints); i++) {
+               /* Vulkan requires that entrypoints for extensions which have
+                * not been enabled must not be advertised.
+                */
+               if (!libresoc_physical_device_entrypoint_is_enabled(i, instance->apiVersion,
+                                       &instance->enabled_extensions)) {
+                       instance->physical_device_dispatch.entrypoints[i] = NULL;
+               } else {
+                       instance->physical_device_dispatch.entrypoints[i] =
+                               libresoc_physical_device_dispatch_table.entrypoints[i];
+               }
+       }
+
+       for (unsigned i = 0; i < ARRAY_SIZE(instance->device_dispatch.entrypoints); i++) {
+               /* Vulkan requires that entrypoints for extensions which have
+                * not been enabled must not be advertised.
+                */
+               if (!libresoc_device_entrypoint_is_enabled(i, instance->apiVersion,
+                                       &instance->enabled_extensions, NULL)) {
+                       instance->device_dispatch.entrypoints[i] = NULL;
+               } else {
+                       instance->device_dispatch.entrypoints[i] =
+                               libresoc_device_dispatch_table.entrypoints[i];
+               }
+       }
+       instance->physical_devices_enumerated = false;
+       list_inithead(&instance->physical_devices);
+       *pInstance = libresoc_instance_to_handle(instance);
+
+       return VK_SUCCESS;
 }
 
 void
@@ -74,6 +223,53 @@ libresoc_DestroyInstance(VkInstance _instance,
    /* FIXME: stub */
 }
 
+static VkResult
+libresoc_physical_device_try_create(struct libresoc_instance *instance,
+                               struct libresoc_physical_device **device_out)
+{
+       VkResult result;
+
+       struct libresoc_physical_device *device =
+               vk_zalloc2(&instance->alloc, NULL, sizeof(*device), 8,
+                          VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE);
+       if (!device) {
+               result = vk_error(instance, VK_ERROR_OUT_OF_HOST_MEMORY);
+               return result;
+       }
+
+       device->_loader_data.loaderMagic = ICD_LOADER_MAGIC;
+       device->instance = instance;
+
+       snprintf(device->name, sizeof(device->name),
+                "LIBRE-SOC DEVICE");
+       *device_out = device;
+
+       return VK_SUCCESS;
+
+}
+static VkResult
+libresoc_enumerate_physical_devices(struct libresoc_instance *instance)
+{
+
+       if (instance->physical_devices_enumerated)
+               return VK_SUCCESS;
+
+       instance->physical_devices_enumerated = true;
+       VkResult result = VK_SUCCESS;
+       /* the driver creates a null
+        * device that allows to test the compiler without having a physical device
+        */
+       struct libresoc_physical_device *pdevice;
+
+       result = libresoc_physical_device_try_create(instance,  &pdevice);
+       if (result != VK_SUCCESS)
+               return result;
+
+       list_addtail(&pdevice->link, &instance->physical_devices);
+       return VK_SUCCESS;
+
+}
+
 VkResult
 libresoc_EnumeratePhysicalDevices(VkInstance _instance,
                               uint32_t *pPhysicalDeviceCount,
@@ -82,8 +278,21 @@ libresoc_EnumeratePhysicalDevices(VkInstance _instance,
        if (getenv("LIBRESOC_TRACE")) {
                fprintf(stderr, "EnumeratePhysicalDevices called\n");
        }
-   /* FIXME: stub */
-   return VK_SUCCESS;
+       LIBRESOC_FROM_HANDLE(libresoc_instance, instance, _instance);
+       VK_OUTARRAY_MAKE(out, pPhysicalDevices, pPhysicalDeviceCount);
+
+       VkResult result = libresoc_enumerate_physical_devices(instance);
+       if (result != VK_SUCCESS)
+               return result;
+
+       list_for_each_entry(struct libresoc_physical_device, pdevice,
+                       &instance->physical_devices, link) {
+               vk_outarray_append(&out, i) {
+                       *i = libresoc_physical_device_to_handle(pdevice);
+               }
+       }
+
+       return vk_outarray_status(&out);
 }
 
 void
@@ -93,7 +302,75 @@ libresoc_GetPhysicalDeviceFeatures(VkPhysicalDevice physicalDevice,
        if (getenv("LIBRESOC_TRACE")) {
                fprintf(stderr, "GetPhysicalDeviceFeatures called. \n");
        }
-   /* FIXME: stub */
+       //LIBRESOC_FROM_HANDLE(libresoc_physical_device, pdevice, physicalDevice);
+       memset(pFeatures, 0, sizeof(*pFeatures));
+
+       *pFeatures = (VkPhysicalDeviceFeatures) {
+               .robustBufferAccess                       = true,
+               .fullDrawIndexUint32                      = true,
+               .imageCubeArray                           = true,
+               .independentBlend                         = true,
+               .geometryShader                           = true,
+               .tessellationShader                       = true,
+               .sampleRateShading                        = true,
+               .dualSrcBlend                             = true,
+               .logicOp                                  = true,
+               .multiDrawIndirect                        = true,
+               .drawIndirectFirstInstance                = true,
+               .depthClamp                               = true,
+               .depthBiasClamp                           = true,
+               .fillModeNonSolid                         = true,
+               .depthBounds                              = true,
+               .wideLines                                = true,
+               .largePoints                              = true,
+               .alphaToOne                               = true,
+               .multiViewport                            = true,
+               .samplerAnisotropy                        = true,
+               .textureCompressionETC2                   = false,
+               .textureCompressionASTC_LDR               = false,
+               .textureCompressionBC                     = true,
+               .occlusionQueryPrecise                    = true,
+               .pipelineStatisticsQuery                  = true,
+               .vertexPipelineStoresAndAtomics           = true,
+               .fragmentStoresAndAtomics                 = true,
+               .shaderTessellationAndGeometryPointSize   = true,
+               .shaderImageGatherExtended                = true,
+               .shaderStorageImageExtendedFormats        = true,
+               .shaderStorageImageMultisample            = true,
+               .shaderUniformBufferArrayDynamicIndexing  = true,
+               .shaderSampledImageArrayDynamicIndexing   = true,
+               .shaderStorageBufferArrayDynamicIndexing  = true,
+               .shaderStorageImageArrayDynamicIndexing   = true,
+               .shaderStorageImageReadWithoutFormat      = true,
+               .shaderStorageImageWriteWithoutFormat     = true,
+               .shaderClipDistance                       = true,
+               .shaderCullDistance                       = true,
+               .shaderFloat64                            = true,
+               .shaderInt64                              = true,
+               .shaderInt16                              = true,
+               .sparseBinding                            = true,
+               .variableMultisampleRate                  = true,
+               .shaderResourceMinLod                     = true,
+               .inheritedQueries                         = true,
+       };
+}
+
+static size_t
+libresoc_max_descriptor_set_size()
+{
+       /* make sure that the entire descriptor set is addressable with a signed
+        * 32-bit int. So the sum of all limits scaled by descriptor size has to
+        * be at most 2 GiB. the combined image & samples object count as one of
+        * both. This limit is for the pipeline layout, not for the set layout, but
+        * there is no set limit, so we just set a pipeline limit. I don't think
+        * any app is going to hit this soon. */
+       return ((1ull << 31) - 16 * MAX_DYNAMIC_BUFFERS
+                            - MAX_INLINE_UNIFORM_BLOCK_SIZE * MAX_INLINE_UNIFORM_BLOCK_COUNT) /
+                 (32 /* uniform buffer, 32 due to potential space wasted on alignment */ +
+                  32 /* storage buffer, 32 due to potential space wasted on alignment */ +
+                  32 /* sampler, largest when combined with image */ +
+                  64 /* sampled image */ +
+                  64 /* storage image */);
 }
 
 void
@@ -103,7 +380,169 @@ libresoc_GetPhysicalDeviceProperties(VkPhysicalDevice physicalDevice,
        if (getenv("LIBRESOC_TRACE")) {
                fprintf(stderr, "GetPhysicalDeviceProperties called. \n");
        }
-   /* FIXME: stub */
+       LIBRESOC_FROM_HANDLE(libresoc_physical_device, pdevice, physicalDevice);
+       VkSampleCountFlags sample_counts = 0xf;
+
+       size_t max_descriptor_set_size = libresoc_max_descriptor_set_size();
+
+       VkPhysicalDeviceLimits limits = {
+               .maxImageDimension1D                      = (1 << 14),
+               .maxImageDimension2D                      = (1 << 14),
+               .maxImageDimension3D                      = (1 << 11),
+               .maxImageDimensionCube                    = (1 << 14),
+               .maxImageArrayLayers                      = (1 << 11),
+               .maxTexelBufferElements                   = UINT32_MAX,
+               .maxUniformBufferRange                    = UINT32_MAX,
+               .maxStorageBufferRange                    = UINT32_MAX,
+               .maxPushConstantsSize                     = MAX_PUSH_CONSTANTS_SIZE,
+               .maxMemoryAllocationCount                 = UINT32_MAX,
+               .maxSamplerAllocationCount                = 64 * 1024,
+               .bufferImageGranularity                   = 64, /* A cache line */
+               .sparseAddressSpaceSize                   = LIBRESOC_MAX_MEMORY_ALLOCATION_SIZE, /* buffer max size */
+               .maxBoundDescriptorSets                   = MAX_SETS,
+               .maxPerStageDescriptorSamplers            = max_descriptor_set_size,
+               .maxPerStageDescriptorUniformBuffers      = max_descriptor_set_size,
+               .maxPerStageDescriptorStorageBuffers      = max_descriptor_set_size,
+               .maxPerStageDescriptorSampledImages       = max_descriptor_set_size,
+               .maxPerStageDescriptorStorageImages       = max_descriptor_set_size,
+               .maxPerStageDescriptorInputAttachments    = max_descriptor_set_size,
+               .maxPerStageResources                     = max_descriptor_set_size,
+               .maxDescriptorSetSamplers                 = max_descriptor_set_size,
+               .maxDescriptorSetUniformBuffers           = max_descriptor_set_size,
+               .maxDescriptorSetUniformBuffersDynamic    = MAX_DYNAMIC_UNIFORM_BUFFERS,
+               .maxDescriptorSetStorageBuffers           = max_descriptor_set_size,
+               .maxDescriptorSetStorageBuffersDynamic    = MAX_DYNAMIC_STORAGE_BUFFERS,
+               .maxDescriptorSetSampledImages            = max_descriptor_set_size,
+               .maxDescriptorSetStorageImages            = max_descriptor_set_size,
+               .maxDescriptorSetInputAttachments         = max_descriptor_set_size,
+               .maxVertexInputAttributes                 = MAX_VERTEX_ATTRIBS,
+               .maxVertexInputBindings                   = MAX_VBS,
+               .maxVertexInputAttributeOffset            = 2047,
+               .maxVertexInputBindingStride              = 2048,
+               .maxVertexOutputComponents                = 128,
+               .maxTessellationGenerationLevel           = 64,
+               .maxTessellationPatchSize                 = 32,
+               .maxTessellationControlPerVertexInputComponents = 128,
+               .maxTessellationControlPerVertexOutputComponents = 128,
+               .maxTessellationControlPerPatchOutputComponents = 120,
+               .maxTessellationControlTotalOutputComponents = 4096,
+               .maxTessellationEvaluationInputComponents = 128,
+               .maxTessellationEvaluationOutputComponents = 128,
+               .maxGeometryShaderInvocations             = 127,
+               .maxGeometryInputComponents               = 64,
+               .maxGeometryOutputComponents              = 128,
+               .maxGeometryOutputVertices                = 256,
+               .maxGeometryTotalOutputComponents         = 1024,
+               .maxFragmentInputComponents               = 128,
+               .maxFragmentOutputAttachments             = 8,
+               .maxFragmentDualSrcAttachments            = 1,
+               .maxFragmentCombinedOutputResources       = 8,
+               .maxComputeSharedMemorySize               = 32768,
+               .maxComputeWorkGroupCount                 = { 65535, 65535, 65535 },
+               .maxComputeWorkGroupInvocations           = 1024,
+               .maxComputeWorkGroupSize = {
+                       1024,
+                       1024,
+                       1024
+               },
+               .subPixelPrecisionBits                    = 8,
+               .subTexelPrecisionBits                    = 8,
+               .mipmapPrecisionBits                      = 8,
+               .maxDrawIndexedIndexValue                 = UINT32_MAX,
+               .maxDrawIndirectCount                     = UINT32_MAX,
+               .maxSamplerLodBias                        = 16,
+               .maxSamplerAnisotropy                     = 16,
+               .maxViewports                             = MAX_VIEWPORTS,
+               .maxViewportDimensions                    = { (1 << 14), (1 << 14) },
+               .viewportBoundsRange                      = { INT16_MIN, INT16_MAX },
+               .viewportSubPixelBits                     = 8,
+               .minMemoryMapAlignment                    = 4096, /* A page */
+               .minTexelBufferOffsetAlignment            = 4,
+               .minUniformBufferOffsetAlignment          = 4,
+               .minStorageBufferOffsetAlignment          = 4,
+               .minTexelOffset                           = -32,
+               .maxTexelOffset                           = 31,
+               .minTexelGatherOffset                     = -32,
+               .maxTexelGatherOffset                     = 31,
+               .minInterpolationOffset                   = -2,
+               .maxInterpolationOffset                   = 2,
+               .subPixelInterpolationOffsetBits          = 8,
+               .maxFramebufferWidth                      = (1 << 14),
+               .maxFramebufferHeight                     = (1 << 14),
+               .maxFramebufferLayers                     = (1 << 10),
+               .framebufferColorSampleCounts             = sample_counts,
+               .framebufferDepthSampleCounts             = sample_counts,
+               .framebufferStencilSampleCounts           = sample_counts,
+               .framebufferNoAttachmentsSampleCounts     = sample_counts,
+               .maxColorAttachments                      = MAX_RTS,
+               .sampledImageColorSampleCounts            = sample_counts,
+               .sampledImageIntegerSampleCounts          = sample_counts,
+               .sampledImageDepthSampleCounts            = sample_counts,
+               .sampledImageStencilSampleCounts          = sample_counts,
+               .storageImageSampleCounts                 = sample_counts,
+               .maxSampleMaskWords                       = 1,
+               .timestampComputeAndGraphics              = true,
+               .timestampPeriod                          = 1000000.0 /* FIXME /pdevice->rad_info.clock_crystal_freq*/,
+               .maxClipDistances                         = 8,
+               .maxCullDistances                         = 8,
+               .maxCombinedClipAndCullDistances          = 8,
+               .discreteQueuePriorities                  = 2,
+               .pointSizeRange                           = { 0.0, 8191.875 },
+               .lineWidthRange                           = { 0.0, 8191.875 },
+               .pointSizeGranularity                     = (1.0 / 8.0),
+               .lineWidthGranularity                     = (1.0 / 8.0),
+               .strictLines                              = false, /* FINISHME */
+               .standardSampleLocations                  = true,
+               .optimalBufferCopyOffsetAlignment         = 128,
+               .optimalBufferCopyRowPitchAlignment       = 128,
+               .nonCoherentAtomSize                      = 64,
+       };
+
+       *pProperties = (VkPhysicalDeviceProperties) {
+               .apiVersion = libresoc_physical_device_api_version(pdevice),
+               .driverVersion = vk_get_driver_version(),
+               .vendorID = 1, //TODO: some dummy value
+               .deviceID = 1, //TODO: dome dummay value
+               .deviceType = VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU,
+               .limits = limits,
+               .sparseProperties = {0},
+       };
+
+       strcpy(pProperties->deviceName, pdevice->name);
+       memcpy(pProperties->pipelineCacheUUID, pdevice->cache_uuid, VK_UUID_SIZE);
+}
+
+static void libresoc_get_physical_device_queue_family_properties(
+       struct libresoc_physical_device*                pdevice,
+       uint32_t*                                   pCount,
+       VkQueueFamilyProperties**                    pQueueFamilyProperties)
+{
+       int num_queue_families = 1;
+       int idx;
+
+       if (pQueueFamilyProperties == NULL) {
+               *pCount = num_queue_families;
+               return;
+       }
+
+       if (!*pCount)
+               return;
+
+       idx = 0;
+       if (*pCount >= 1) {
+               *pQueueFamilyProperties[idx] = (VkQueueFamilyProperties) {
+                       .queueFlags = VK_QUEUE_GRAPHICS_BIT |
+                                     VK_QUEUE_COMPUTE_BIT |
+                                     VK_QUEUE_TRANSFER_BIT |
+                                     VK_QUEUE_SPARSE_BINDING_BIT,
+                       .queueCount = 1,
+                       .timestampValidBits = 64,
+                       .minImageTransferGranularity = (VkExtent3D) { 1, 1, 1 },
+               };
+               idx++;
+       }
+
+       *pCount = idx;
 }
 
 void
@@ -114,7 +553,18 @@ libresoc_GetPhysicalDeviceQueueFamilyProperties(VkPhysicalDevice physicalDevice,
        if (getenv("LIBRESOC_TRACE")) {
                fprintf(stderr, "GetPhysicalDeviceQueueFamilyProperites called. \n");
        }
-   /* FIXME: stub */
+       LIBRESOC_FROM_HANDLE(libresoc_physical_device, pdevice, physicalDevice);
+       if (!pQueueFamilyProperties) {
+               libresoc_get_physical_device_queue_family_properties(pdevice, pCount, NULL);
+               return;
+       }
+       VkQueueFamilyProperties *properties[] = {
+               pQueueFamilyProperties + 0,
+               pQueueFamilyProperties + 1,
+               pQueueFamilyProperties + 2,
+       };
+       libresoc_get_physical_device_queue_family_properties(pdevice, pCount, properties);
+       assert(*pCount <= 3);
 }
 
 void
@@ -201,7 +651,7 @@ libresoc_GetInstanceProcAddr(VkInstance _instance,
 
    idx = libresoc_get_physical_device_entrypoint_index(pName);
    if (idx >= 0)
-      return instance->physicalDevice.dispatch.entrypoints[idx];
+      return instance->physical_device_dispatch.entrypoints[idx];
 
    idx = libresoc_get_device_entrypoint_index(pName);
    if (idx >= 0)
@@ -272,7 +722,7 @@ vk_icdGetPhysicalDeviceProcAddr(VkInstance  _instance,
    if (idx < 0)
       return NULL;
 
-   return instance->physicalDevice.dispatch.entrypoints[idx];
+   return instance->physical_device_dispatch.entrypoints[idx];
 }
 
 VkResult
@@ -284,10 +734,145 @@ libresoc_EnumerateDeviceExtensionProperties(VkPhysicalDevice physicalDevice,
        if (getenv("LIBRESOC_TRACE")) {
                fprintf(stderr, "EnumerateDeviceExtensionProperties called for layer: %s \n", pLayerName);
        }
-   /* FIXME: stub */
-   return VK_SUCCESS;
+       LIBRESOC_FROM_HANDLE(libresoc_physical_device, device, physicalDevice);
+       VK_OUTARRAY_MAKE(out, pProperties, pPropertyCount);
+
+       for (int i = 0; i < LIBRESOC_DEVICE_EXTENSION_COUNT; i++) {
+               if (device->supported_extensions.extensions[i]) {
+                       vk_outarray_append(&out, prop) {
+                               *prop = libresoc_device_extensions[i];
+                       }
+               }
+       }
+
+       return vk_outarray_status(&out);
+}
+
+static void
+libresoc_device_init_dispatch(struct libresoc_device *device)
+{
+       const struct libresoc_instance *instance = device->physical_device->instance;
+       const struct libresoc_device_dispatch_table *dispatch_table_layer = NULL;
+
+       for (unsigned i = 0; i < ARRAY_SIZE(device->dispatch.entrypoints); i++) {
+               /* Vulkan requires that entrypoints for extensions which have not been
+                * enabled must not be advertised.
+                */
+               if (!libresoc_device_entrypoint_is_enabled(i, instance->apiVersion,
+                                                      &instance->enabled_extensions,
+                                                      &device->enabled_extensions)) {
+                       device->dispatch.entrypoints[i] = NULL;
+               } else if (dispatch_table_layer &&
+                          dispatch_table_layer->entrypoints[i]) {
+                       device->dispatch.entrypoints[i] =
+                               dispatch_table_layer->entrypoints[i];
+               } else {
+                       device->dispatch.entrypoints[i] =
+                               libresoc_device_dispatch_table.entrypoints[i];
+               }
+       }
 }
 
+/*
+static VkResult
+libresoc_create_pthread_cond(pthread_cond_t *cond)
+{
+       pthread_condattr_t condattr;
+       if (pthread_condattr_init(&condattr)) {
+               return VK_ERROR_INITIALIZATION_FAILED;
+       }
+
+       if (pthread_condattr_setclock(&condattr, CLOCK_MONOTONIC)) {
+               pthread_condattr_destroy(&condattr);
+               return VK_ERROR_INITIALIZATION_FAILED;
+       }
+       if (pthread_cond_init(cond, &condattr)) {
+               pthread_condattr_destroy(&condattr);
+               return VK_ERROR_INITIALIZATION_FAILED;
+       }
+       pthread_condattr_destroy(&condattr);
+       return VK_SUCCESS;
+}
+*/
+static VkResult
+check_physical_device_features(VkPhysicalDevice physicalDevice,
+                              const VkPhysicalDeviceFeatures *features)
+{
+       LIBRESOC_FROM_HANDLE(libresoc_physical_device, physical_device, physicalDevice);
+       VkPhysicalDeviceFeatures supported_features;
+       libresoc_GetPhysicalDeviceFeatures(physicalDevice, &supported_features);
+       VkBool32 *supported_feature = (VkBool32 *)&supported_features;
+       VkBool32 *enabled_feature = (VkBool32 *)features;
+       unsigned num_features = sizeof(VkPhysicalDeviceFeatures) / sizeof(VkBool32);
+       for (uint32_t i = 0; i < num_features; i++) {
+               if (enabled_feature[i] && !supported_feature[i])
+                       return vk_error(physical_device->instance, VK_ERROR_FEATURE_NOT_PRESENT);
+       }
+
+       return VK_SUCCESS;
+}
+static int libresoc_get_device_extension_index(const char *name)
+{
+       for (unsigned i = 0; i < LIBRESOC_DEVICE_EXTENSION_COUNT; ++i) {
+               if (strcmp(name, libresoc_device_extensions[i].extensionName) == 0)
+                       return i;
+       }
+       return -1;
+}
+
+static int
+libresoc_queue_init(struct libresoc_device *device, struct libresoc_queue *queue,
+               uint32_t queue_family_index, int idx,
+               VkDeviceQueueCreateFlags flags,
+               const VkDeviceQueueGlobalPriorityCreateInfoEXT *global_priority)
+{
+       queue->_loader_data.loaderMagic = ICD_LOADER_MAGIC;
+       queue->device = device;
+       queue->queue_family_index = queue_family_index;
+       queue->queue_idx = idx;
+       queue->flags = flags;
+       return VK_SUCCESS;
+}
+
+static void
+libresoc_queue_finish(struct libresoc_queue *queue)
+{
+//TODO: understand and enable following code
+//     if (queue->thread_running) {
+//             p_atomic_set(&queue->thread_exit, true);
+//             pthread_cond_broadcast(&queue->thread_cond);
+//             pthread_join(queue->submission_thread, NULL);
+//     }
+//     pthread_cond_destroy(&queue->thread_cond);
+//     pthread_mutex_destroy(&queue->pending_mutex);
+//     pthread_mutex_destroy(&queue->thread_mutex);
+//
+//     if (queue->hw_ctx)
+//             queue->device->ws->ctx_destroy(queue->hw_ctx);
+//
+//     if (queue->initial_full_flush_preamble_cs)
+//             queue->device->ws->cs_destroy(queue->initial_full_flush_preamble_cs);
+//     if (queue->initial_preamble_cs)
+//             queue->device->ws->cs_destroy(queue->initial_preamble_cs);
+//     if (queue->continue_preamble_cs)
+//             queue->device->ws->cs_destroy(queue->continue_preamble_cs);
+//     if (queue->descriptor_bo)
+//             queue->device->ws->buffer_destroy(queue->descriptor_bo);
+//     if (queue->scratch_bo)
+//             queue->device->ws->buffer_destroy(queue->scratch_bo);
+//     if (queue->esgs_ring_bo)
+//             queue->device->ws->buffer_destroy(queue->esgs_ring_bo);
+//     if (queue->gsvs_ring_bo)
+//             queue->device->ws->buffer_destroy(queue->gsvs_ring_bo);
+//     if (queue->tess_rings_bo)
+//             queue->device->ws->buffer_destroy(queue->tess_rings_bo);
+//     if (queue->gds_bo)
+//             queue->device->ws->buffer_destroy(queue->gds_bo);
+//     if (queue->gds_oa_bo)
+//             queue->device->ws->buffer_destroy(queue->gds_oa_bo);
+//     if (queue->compute_scratch_bo)
+//             queue->device->ws->buffer_destroy(queue->compute_scratch_bo);
+}
 VkResult
 libresoc_CreateDevice(VkPhysicalDevice physicalDevice,
                   const VkDeviceCreateInfo *pCreateInfo,
@@ -297,8 +882,99 @@ libresoc_CreateDevice(VkPhysicalDevice physicalDevice,
        if (getenv("LIBRESOC_TRACE")) {
                fprintf(stderr, "CreateDevice called \n");
        }
-   /* FIXME: stub */
-   return VK_SUCCESS;
+       LIBRESOC_FROM_HANDLE(libresoc_physical_device, physical_device, physicalDevice);
+       VkResult result;
+       struct libresoc_device *device;
+
+       /* Check enabled features */
+       if (pCreateInfo->pEnabledFeatures) {
+               result = check_physical_device_features(physicalDevice,
+                                                       pCreateInfo->pEnabledFeatures);
+               if (result != VK_SUCCESS)
+                       return result;
+       }
+
+       device = vk_zalloc2(&physical_device->instance->alloc, pAllocator,
+                           sizeof(*device), 8,
+                           VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
+       if (!device)
+               return vk_error(physical_device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
+
+       vk_device_init(&device->vk, pCreateInfo,
+                      &physical_device->instance->alloc, pAllocator);
+
+       device->instance = physical_device->instance;
+       device->physical_device = physical_device;
+
+       for (uint32_t i = 0; i < pCreateInfo->enabledExtensionCount; i++) {
+               const char *ext_name = pCreateInfo->ppEnabledExtensionNames[i];
+               int index = libresoc_get_device_extension_index(ext_name);
+               if (index < 0 || !physical_device->supported_extensions.extensions[index]) {
+                       vk_free(&device->vk.alloc, device);
+                       return vk_error(physical_device->instance, VK_ERROR_EXTENSION_NOT_PRESENT);
+               }
+
+               device->enabled_extensions.extensions[index] = true;
+       }
+
+       libresoc_device_init_dispatch(device);
+
+       for (unsigned i = 0; i < pCreateInfo->queueCreateInfoCount; i++) {
+               const VkDeviceQueueCreateInfo *queue_create = &pCreateInfo->pQueueCreateInfos[i];
+               uint32_t qfi = queue_create->queueFamilyIndex;
+               const VkDeviceQueueGlobalPriorityCreateInfoEXT *global_priority =
+                       vk_find_struct_const(queue_create->pNext, DEVICE_QUEUE_GLOBAL_PRIORITY_CREATE_INFO_EXT);
+
+
+               device->queues[qfi] = vk_alloc(&device->vk.alloc,
+                                              queue_create->queueCount * sizeof(struct libresoc_queue), 8, VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
+               if (!device->queues[qfi]) {
+                       result = VK_ERROR_OUT_OF_HOST_MEMORY;
+                       goto fail;
+               }
+
+               memset(device->queues[qfi], 0, queue_create->queueCount * sizeof(struct libresoc_queue));
+
+               device->queue_count[qfi] = queue_create->queueCount;
+
+               for (unsigned q = 0; q < queue_create->queueCount; q++) {
+                       result = libresoc_queue_init(device, &device->queues[qfi][q],
+                                                qfi, q, queue_create->flags,
+                                                global_priority);
+                       if (result != VK_SUCCESS)
+                               goto fail;
+               }
+       }
+
+       if (result != VK_SUCCESS)
+               goto fail;
+
+       VkPipelineCacheCreateInfo ci;
+       ci.sType = VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO;
+       ci.pNext = NULL;
+       ci.flags = 0;
+       ci.pInitialData = NULL;
+       ci.initialDataSize = 0;
+       VkPipelineCache pc;
+       result = libresoc_CreatePipelineCache(libresoc_device_to_handle(device),
+                                         &ci, NULL, &pc);
+       if (result != VK_SUCCESS)
+               goto fail;
+
+       *pDevice = libresoc_device_to_handle(device);
+       return VK_SUCCESS;
+
+fail:
+       libresoc_DestroyPipelineCache(libresoc_device_to_handle(device), pc, NULL);
+       for (unsigned i = 0; i < LIBRESOC_MAX_QUEUE_FAMILIES; i++) {
+               for (unsigned q = 0; q < device->queue_count[i]; q++)
+                       libresoc_queue_finish(&device->queues[i][q]);
+               if (device->queue_count[i])
+                       vk_free(&device->vk.alloc, device->queues[i]);
+       }
+
+       vk_free(&device->vk.alloc, device);
+       return result;
 }
 
 void
@@ -311,14 +987,49 @@ libresoc_DestroyDevice(VkDevice _device,
    /* FIXME: stub */
 }
 
-void
-libresoc_GetDeviceQueue(VkDevice _device,
-                    uint32_t queueNodeIndex,
-                    uint32_t queueIndex,
-                    VkQueue *pQueue)
+void libresoc_GetDeviceQueue(
+       VkDevice                                    _device,
+       uint32_t                                    queueFamilyIndex,
+       uint32_t                                    queueIndex,
+       VkQueue*                                    pQueue)
 {
        if (getenv("LIBRESOC_TRACE")) {
                fprintf(stderr, "GetDeviceQueue called. \n");
        }
-   /* FIXME: stub */
+       LIBRESOC_FROM_HANDLE(libresoc_device, device, _device);
+       struct libresoc_queue *queue;
+
+       queue = &device->queues[queueFamilyIndex][queueIndex];
+       *pQueue = libresoc_queue_to_handle(queue);
+}
+
+VkResult libresoc_QueueWaitIdle(
+       VkQueue                                     _queue)
+{
+       LIBRESOC_FROM_HANDLE(libresoc_queue, queue, _queue);
+
+       pthread_mutex_lock(&queue->pending_mutex);
+       while (!list_is_empty(&queue->pending_submissions)) {
+               pthread_cond_wait(&queue->device->timeline_cond, &queue->pending_mutex);
+       }
+       pthread_mutex_unlock(&queue->pending_mutex);
+
+       return VK_SUCCESS;
+}
+
+VkResult libresoc_DeviceWaitIdle(
+       VkDevice                                    _device)
+{
+       LIBRESOC_FROM_HANDLE(libresoc_device, device, _device);
+
+       for (unsigned i = 0; i < LIBRESOC_MAX_QUEUE_FAMILIES; i++) {
+               for (unsigned q = 0; q < device->queue_count[i]; q++) {
+                       VkResult result =
+                               libresoc_QueueWaitIdle(libresoc_queue_to_handle(&device->queues[i][q]));
+
+                       if (result != VK_SUCCESS)
+                               return result;
+               }
+       }
+       return VK_SUCCESS;
 }