src/vulkan/anv_device.c

   1 /*
   2  * Copyright © 2015 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include <assert.h>
  25 #include <stdbool.h>
  26 #include <string.h>
  27 #include <unistd.h>
  28 #include <fcntl.h>
  29
  30 #include "anv_private.h"
  31 #include "mesa/main/git_sha1.h"
  32 #include "util/strtod.h"
  33 #include "util/debug.h"
  34
  35 #include "gen7_pack.h"
  36
  37 struct anv_dispatch_table dtable;
  38
  39 static void
  40 compiler_debug_log(void *data, const char *fmt, ...)
  41 { }
  42
  43 static void
  44 compiler_perf_log(void *data, const char *fmt, ...)
  45 {
  46    va_list args;
  47    va_start(args, fmt);
  48
  49    if (unlikely(INTEL_DEBUG & DEBUG_PERF))
  50       vfprintf(stderr, fmt, args);
  51
  52    va_end(args);
  53 }
  54
  55 static VkResult
  56 anv_physical_device_init(struct anv_physical_device *device,
  57                          struct anv_instance *instance,
  58                          const char *path)
  59 {
  60    VkResult result;
  61    int fd;
  62
  63    fd = open(path, O_RDWR | O_CLOEXEC);
  64    if (fd < 0)
  65       return vk_errorf(VK_ERROR_INITIALIZATION_FAILED,
  66                        "failed to open %s: %m", path);
  67
  68    device->_loader_data.loaderMagic = ICD_LOADER_MAGIC;
  69    device->instance = instance;
  70    device->path = path;
  71
  72    device->chipset_id = anv_gem_get_param(fd, I915_PARAM_CHIPSET_ID);
  73    if (!device->chipset_id) {
  74       result = vk_errorf(VK_ERROR_INITIALIZATION_FAILED,
  75                          "failed to get chipset id: %m");
  76       goto fail;
  77    }
  78
  79    device->name = brw_get_device_name(device->chipset_id);
  80    device->info = brw_get_device_info(device->chipset_id);
  81    if (!device->info) {
  82       result = vk_errorf(VK_ERROR_INITIALIZATION_FAILED,
  83                          "failed to get device info");
  84       goto fail;
  85    }
  86
  87    if (device->info->is_haswell) {
  88       fprintf(stderr, "WARNING: Haswell Vulkan support is incomplete\n");
  89    } else if (device->info->gen == 7 && !device->info->is_baytrail) {
  90       fprintf(stderr, "WARNING: Ivy Bridge Vulkan support is incomplete\n");
  91    } else if (device->info->gen == 7 && device->info->is_baytrail) {
  92       fprintf(stderr, "WARNING: Bay Trail Vulkan support is incomplete\n");
  93    } else if (device->info->gen >= 8) {
  94       /* Broadwell, Cherryview, Skylake, Broxton, Kabylake is as fully
  95        * supported as anything */
  96    } else {
  97       result = vk_errorf(VK_ERROR_INCOMPATIBLE_DRIVER,
  98                          "Vulkan not yet supported on %s", device->name);
  99       goto fail;
 100    }
 101
 102    if (anv_gem_get_aperture(fd, &device->aperture_size) == -1) {
 103       result = vk_errorf(VK_ERROR_INITIALIZATION_FAILED,
 104                          "failed to get aperture size: %m");
 105       goto fail;
 106    }
 107
 108    if (!anv_gem_get_param(fd, I915_PARAM_HAS_WAIT_TIMEOUT)) {
 109       result = vk_errorf(VK_ERROR_INITIALIZATION_FAILED,
 110                          "kernel missing gem wait");
 111       goto fail;
 112    }
 113
 114    if (!anv_gem_get_param(fd, I915_PARAM_HAS_EXECBUF2)) {
 115       result = vk_errorf(VK_ERROR_INITIALIZATION_FAILED,
 116                          "kernel missing execbuf2");
 117       goto fail;
 118    }
 119
 120    if (!device->info->has_llc &&
 121        anv_gem_get_param(fd, I915_PARAM_MMAP_VERSION) < 1) {
 122       result = vk_errorf(VK_ERROR_INITIALIZATION_FAILED,
 123                          "kernel missing wc mmap");
 124       goto fail;
 125    }
 126
 127    bool swizzled = anv_gem_get_bit6_swizzle(fd, I915_TILING_X);
 128
 129    close(fd);
 130
 131    brw_process_intel_debug_variable();
 132
 133    device->compiler = brw_compiler_create(NULL, device->info);
 134    if (device->compiler == NULL) {
 135       result = vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
 136       goto fail;
 137    }
 138    device->compiler->shader_debug_log = compiler_debug_log;
 139    device->compiler->shader_perf_log = compiler_perf_log;
 140
 141    /* Default to use scalar GS on BDW+ */
 142    device->compiler->scalar_stage[MESA_SHADER_GEOMETRY] =
 143       device->info->gen >= 8 && env_var_as_boolean("INTEL_SCALAR_GS", true);
 144
 145    /* XXX: Actually detect bit6 swizzling */
 146    isl_device_init(&device->isl_dev, device->info, swizzled);
 147
 148    return VK_SUCCESS;
 149
 150 fail:
 151    close(fd);
 152    return result;
 153 }
 154
 155 static void
 156 anv_physical_device_finish(struct anv_physical_device *device)
 157 {
 158    ralloc_free(device->compiler);
 159 }
 160
 161 static const VkExtensionProperties global_extensions[] = {
 162    {
 163       .extensionName = VK_KHR_SURFACE_EXTENSION_NAME,
 164       .specVersion = 24,
 165    },
 166    {
 167       .extensionName = VK_KHR_XCB_SURFACE_EXTENSION_NAME,
 168       .specVersion = 5,
 169    },
 170 #ifdef HAVE_WAYLAND_PLATFORM
 171    {
 172       .extensionName = VK_KHR_WAYLAND_SURFACE_EXTENSION_NAME,
 173       .specVersion = 4,
 174    },
 175 #endif
 176 };
 177
 178 static const VkExtensionProperties device_extensions[] = {
 179    {
 180       .extensionName = VK_KHR_SWAPCHAIN_EXTENSION_NAME,
 181       .specVersion = 67,
 182    },
 183 };
 184
 185 static void *
 186 default_alloc_func(void *pUserData, size_t size, size_t align,
 187                    VkSystemAllocationScope allocationScope)
 188 {
 189    return malloc(size);
 190 }
 191
 192 static void *
 193 default_realloc_func(void *pUserData, void *pOriginal, size_t size,
 194                      size_t align, VkSystemAllocationScope allocationScope)
 195 {
 196    return realloc(pOriginal, size);
 197 }
 198
 199 static void
 200 default_free_func(void *pUserData, void *pMemory)
 201 {
 202    free(pMemory);
 203 }
 204
 205 static const VkAllocationCallbacks default_alloc = {
 206    .pUserData = NULL,
 207    .pfnAllocation = default_alloc_func,
 208    .pfnReallocation = default_realloc_func,
 209    .pfnFree = default_free_func,
 210 };
 211
 212 VkResult anv_CreateInstance(
 213     const VkInstanceCreateInfo*                 pCreateInfo,
 214     const VkAllocationCallbacks*                pAllocator,
 215     VkInstance*                                 pInstance)
 216 {
 217    struct anv_instance *instance;
 218
 219    assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO);
 220
 221    if (pCreateInfo->pApplicationInfo->apiVersion != VK_MAKE_VERSION(1, 0, 0))
 222       return vk_error(VK_ERROR_INCOMPATIBLE_DRIVER);
 223
 224    for (uint32_t i = 0; i < pCreateInfo->enabledExtensionCount; i++) {
 225       bool found = false;
 226       for (uint32_t j = 0; j < ARRAY_SIZE(global_extensions); j++) {
 227          if (strcmp(pCreateInfo->ppEnabledExtensionNames[i],
 228                     global_extensions[j].extensionName) == 0) {
 229             found = true;
 230             break;
 231          }
 232       }
 233       if (!found)
 234          return vk_error(VK_ERROR_EXTENSION_NOT_PRESENT);
 235    }
 236
 237    instance = anv_alloc2(&default_alloc, pAllocator, sizeof(*instance), 8,
 238                          VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE);
 239    if (!instance)
 240       return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
 241
 242    instance->_loader_data.loaderMagic = ICD_LOADER_MAGIC;
 243
 244    if (pAllocator)
 245       instance->alloc = *pAllocator;
 246    else
 247       instance->alloc = default_alloc;
 248
 249    instance->apiVersion = pCreateInfo->pApplicationInfo->apiVersion;
 250    instance->physicalDeviceCount = -1;
 251
 252    _mesa_locale_init();
 253
 254    VG(VALGRIND_CREATE_MEMPOOL(instance, 0, false));
 255
 256    anv_init_wsi(instance);
 257
 258    *pInstance = anv_instance_to_handle(instance);
 259
 260    return VK_SUCCESS;
 261 }
 262
 263 void anv_DestroyInstance(
 264     VkInstance                                  _instance,
 265     const VkAllocationCallbacks*                pAllocator)
 266 {
 267    ANV_FROM_HANDLE(anv_instance, instance, _instance);
 268
 269    if (instance->physicalDeviceCount > 0) {
 270       /* We support at most one physical device. */
 271       assert(instance->physicalDeviceCount == 1);
 272       anv_physical_device_finish(&instance->physicalDevice);
 273    }
 274
 275    anv_finish_wsi(instance);
 276
 277    VG(VALGRIND_DESTROY_MEMPOOL(instance));
 278
 279    _mesa_locale_fini();
 280
 281    anv_free(&instance->alloc, instance);
 282 }
 283
 284 VkResult anv_EnumeratePhysicalDevices(
 285     VkInstance                                  _instance,
 286     uint32_t*                                   pPhysicalDeviceCount,
 287     VkPhysicalDevice*                           pPhysicalDevices)
 288 {
 289    ANV_FROM_HANDLE(anv_instance, instance, _instance);
 290    VkResult result;
 291
 292    if (instance->physicalDeviceCount < 0) {
 293       result = anv_physical_device_init(&instance->physicalDevice,
 294                                         instance, "/dev/dri/renderD128");
 295       if (result == VK_ERROR_INCOMPATIBLE_DRIVER) {
 296          instance->physicalDeviceCount = 0;
 297       } else if (result == VK_SUCCESS) {
 298          instance->physicalDeviceCount = 1;
 299       } else {
 300          return result;
 301       }
 302    }
 303
 304    /* pPhysicalDeviceCount is an out parameter if pPhysicalDevices is NULL;
 305     * otherwise it's an inout parameter.
 306     *
 307     * The Vulkan spec (git aaed022) says:
 308     *
 309     *    pPhysicalDeviceCount is a pointer to an unsigned integer variable
 310     *    that is initialized with the number of devices the application is
 311     *    prepared to receive handles to. pname:pPhysicalDevices is pointer to
 312     *    an array of at least this many VkPhysicalDevice handles [...].
 313     *
 314     *    Upon success, if pPhysicalDevices is NULL, vkEnumeratePhysicalDevices
 315     *    overwrites the contents of the variable pointed to by
 316     *    pPhysicalDeviceCount with the number of physical devices in in the
 317     *    instance; otherwise, vkEnumeratePhysicalDevices overwrites
 318     *    pPhysicalDeviceCount with the number of physical handles written to
 319     *    pPhysicalDevices.
 320     */
 321    if (!pPhysicalDevices) {
 322       *pPhysicalDeviceCount = instance->physicalDeviceCount;
 323    } else if (*pPhysicalDeviceCount >= 1) {
 324       pPhysicalDevices[0] = anv_physical_device_to_handle(&instance->physicalDevice);
 325       *pPhysicalDeviceCount = 1;
 326    } else {
 327       *pPhysicalDeviceCount = 0;
 328    }
 329
 330    return VK_SUCCESS;
 331 }
 332
 333 void anv_GetPhysicalDeviceFeatures(
 334     VkPhysicalDevice                            physicalDevice,
 335     VkPhysicalDeviceFeatures*                   pFeatures)
 336 {
 337    anv_finishme("Get correct values for PhysicalDeviceFeatures");
 338
 339    *pFeatures = (VkPhysicalDeviceFeatures) {
 340       .robustBufferAccess                       = false,
 341       .fullDrawIndexUint32                      = false,
 342       .imageCubeArray                           = false,
 343       .independentBlend                         = false,
 344       .geometryShader                           = true,
 345       .tessellationShader                       = false,
 346       .sampleRateShading                        = false,
 347       .dualSrcBlend                             = true,
 348       .logicOp                                  = true,
 349       .multiDrawIndirect                        = false,
 350       .drawIndirectFirstInstance                = false,
 351       .depthClamp                               = false,
 352       .depthBiasClamp                           = false,
 353       .fillModeNonSolid                         = true,
 354       .depthBounds                              = false,
 355       .wideLines                                = true,
 356       .largePoints                              = true,
 357       .alphaToOne                               = true,
 358       .multiViewport                            = true,
 359       .samplerAnisotropy                        = false, /* FINISHME */
 360       .textureCompressionETC2                   = true,
 361       .textureCompressionASTC_LDR               = true,
 362       .textureCompressionBC                     = true,
 363       .occlusionQueryPrecise                    = false, /* FINISHME */
 364       .pipelineStatisticsQuery                  = true,
 365       .vertexPipelineStoresAndAtomics           = false,
 366       .fragmentStoresAndAtomics                 = true,
 367       .shaderTessellationAndGeometryPointSize   = true,
 368       .shaderImageGatherExtended                = true,
 369       .shaderStorageImageExtendedFormats        = false,
 370       .shaderStorageImageMultisample            = false,
 371       .shaderUniformBufferArrayDynamicIndexing  = true,
 372       .shaderSampledImageArrayDynamicIndexing   = false,
 373       .shaderStorageBufferArrayDynamicIndexing  = false,
 374       .shaderStorageImageArrayDynamicIndexing   = false,
 375       .shaderStorageImageReadWithoutFormat      = false,
 376       .shaderStorageImageWriteWithoutFormat     = true,
 377       .shaderClipDistance                       = false,
 378       .shaderCullDistance                       = false,
 379       .shaderFloat64                            = false,
 380       .shaderInt64                              = false,
 381       .shaderInt16                              = false,
 382       .alphaToOne                               = true,
 383       .variableMultisampleRate                  = false,
 384       .inheritedQueries                         = false,
 385    };
 386 }
 387
 388 void anv_GetPhysicalDeviceProperties(
 389     VkPhysicalDevice                            physicalDevice,
 390     VkPhysicalDeviceProperties*                 pProperties)
 391 {
 392    ANV_FROM_HANDLE(anv_physical_device, pdevice, physicalDevice);
 393    const struct brw_device_info *devinfo = pdevice->info;
 394
 395    anv_finishme("Get correct values for VkPhysicalDeviceLimits");
 396
 397    const float time_stamp_base = devinfo->gen >= 9 ? 83.333 : 80.0;
 398
 399    VkSampleCountFlags sample_counts =
 400       VK_SAMPLE_COUNT_1_BIT |
 401       VK_SAMPLE_COUNT_2_BIT |
 402       VK_SAMPLE_COUNT_4_BIT |
 403       VK_SAMPLE_COUNT_8_BIT;
 404
 405    VkPhysicalDeviceLimits limits = {
 406       .maxImageDimension1D                      = (1 << 14),
 407       .maxImageDimension2D                      = (1 << 14),
 408       .maxImageDimension3D                      = (1 << 10),
 409       .maxImageDimensionCube                    = (1 << 14),
 410       .maxImageArrayLayers                      = (1 << 10),
 411       .maxTexelBufferElements                   = (1 << 14),
 412       .maxUniformBufferRange                    = UINT32_MAX,
 413       .maxStorageBufferRange                    = UINT32_MAX,
 414       .maxPushConstantsSize                     = MAX_PUSH_CONSTANTS_SIZE,
 415       .maxMemoryAllocationCount                 = UINT32_MAX,
 416       .maxSamplerAllocationCount                = 64 * 1024,
 417       .bufferImageGranularity                   = 64, /* A cache line */
 418       .sparseAddressSpaceSize                   = 0,
 419       .maxBoundDescriptorSets                   = MAX_SETS,
 420       .maxPerStageDescriptorSamplers            = 64,
 421       .maxPerStageDescriptorUniformBuffers      = 64,
 422       .maxPerStageDescriptorStorageBuffers      = 64,
 423       .maxPerStageDescriptorSampledImages       = 64,
 424       .maxPerStageDescriptorStorageImages       = 64,
 425       .maxPerStageDescriptorInputAttachments    = 64,
 426       .maxPerStageResources                     = 128,
 427       .maxDescriptorSetSamplers                 = 256,
 428       .maxDescriptorSetUniformBuffers           = 256,
 429       .maxDescriptorSetUniformBuffersDynamic    = 256,
 430       .maxDescriptorSetStorageBuffers           = 256,
 431       .maxDescriptorSetStorageBuffersDynamic    = 256,
 432       .maxDescriptorSetSampledImages            = 256,
 433       .maxDescriptorSetStorageImages            = 256,
 434       .maxDescriptorSetInputAttachments         = 256,
 435       .maxVertexInputAttributes                 = 32,
 436       .maxVertexInputBindings                   = 32,
 437       .maxVertexInputAttributeOffset            = 256,
 438       .maxVertexInputBindingStride              = 256,
 439       .maxVertexOutputComponents                = 32,
 440       .maxTessellationGenerationLevel           = 0,
 441       .maxTessellationPatchSize                 = 0,
 442       .maxTessellationControlPerVertexInputComponents = 0,
 443       .maxTessellationControlPerVertexOutputComponents = 0,
 444       .maxTessellationControlPerPatchOutputComponents = 0,
 445       .maxTessellationControlTotalOutputComponents = 0,
 446       .maxTessellationEvaluationInputComponents = 0,
 447       .maxTessellationEvaluationOutputComponents = 0,
 448       .maxGeometryShaderInvocations             = 6,
 449       .maxGeometryInputComponents               = 16,
 450       .maxGeometryOutputComponents              = 16,
 451       .maxGeometryOutputVertices                = 16,
 452       .maxGeometryTotalOutputComponents         = 16,
 453       .maxFragmentInputComponents               = 16,
 454       .maxFragmentOutputAttachments             = 8,
 455       .maxFragmentDualSrcAttachments            = 2,
 456       .maxFragmentCombinedOutputResources       = 8,
 457       .maxComputeSharedMemorySize               = 1024,
 458       .maxComputeWorkGroupCount = {
 459          16 * devinfo->max_cs_threads,
 460          16 * devinfo->max_cs_threads,
 461          16 * devinfo->max_cs_threads,
 462       },
 463       .maxComputeWorkGroupInvocations           = 16 * devinfo->max_cs_threads,
 464       .maxComputeWorkGroupSize = {
 465          16 * devinfo->max_cs_threads,
 466          16 * devinfo->max_cs_threads,
 467          16 * devinfo->max_cs_threads,
 468       },
 469       .subPixelPrecisionBits                    = 4 /* FIXME */,
 470       .subTexelPrecisionBits                    = 4 /* FIXME */,
 471       .mipmapPrecisionBits                      = 4 /* FIXME */,
 472       .maxDrawIndexedIndexValue                 = UINT32_MAX,
 473       .maxDrawIndirectCount                     = UINT32_MAX,
 474       .maxSamplerLodBias                        = 16,
 475       .maxSamplerAnisotropy                     = 16,
 476       .maxViewports                             = MAX_VIEWPORTS,
 477       .maxViewportDimensions                    = { (1 << 14), (1 << 14) },
 478       .viewportBoundsRange                      = { -1.0, 1.0 }, /* FIXME */
 479       .viewportSubPixelBits                     = 13, /* We take a float? */
 480       .minMemoryMapAlignment                    = 4096, /* A page */
 481       .minTexelBufferOffsetAlignment            = 1,
 482       .minUniformBufferOffsetAlignment          = 1,
 483       .minStorageBufferOffsetAlignment          = 1,
 484       .minTexelOffset                           = 0, /* FIXME */
 485       .maxTexelOffset                           = 0, /* FIXME */
 486       .minTexelGatherOffset                     = 0, /* FIXME */
 487       .maxTexelGatherOffset                     = 0, /* FIXME */
 488       .minInterpolationOffset                   = 0, /* FIXME */
 489       .maxInterpolationOffset                   = 0, /* FIXME */
 490       .subPixelInterpolationOffsetBits          = 0, /* FIXME */
 491       .maxFramebufferWidth                      = (1 << 14),
 492       .maxFramebufferHeight                     = (1 << 14),
 493       .maxFramebufferLayers                     = (1 << 10),
 494       .framebufferColorSampleCounts             = sample_counts,
 495       .framebufferDepthSampleCounts             = sample_counts,
 496       .framebufferStencilSampleCounts           = sample_counts,
 497       .framebufferNoAttachmentsSampleCounts     = sample_counts,
 498       .maxColorAttachments                      = MAX_RTS,
 499       .sampledImageColorSampleCounts            = sample_counts,
 500       .sampledImageIntegerSampleCounts          = VK_SAMPLE_COUNT_1_BIT,
 501       .sampledImageDepthSampleCounts            = sample_counts,
 502       .sampledImageStencilSampleCounts          = sample_counts,
 503       .storageImageSampleCounts                 = VK_SAMPLE_COUNT_1_BIT,
 504       .maxSampleMaskWords                       = 1,
 505       .timestampComputeAndGraphics              = false,
 506       .timestampPeriod                          = time_stamp_base / (1000 * 1000 * 1000),
 507       .maxClipDistances                         = 0 /* FIXME */,
 508       .maxCullDistances                         = 0 /* FIXME */,
 509       .maxCombinedClipAndCullDistances          = 0 /* FIXME */,
 510       .discreteQueuePriorities                  = 1,
 511       .pointSizeRange                           = { 0.125, 255.875 },
 512       .lineWidthRange                           = { 0.0, 7.9921875 },
 513       .pointSizeGranularity                     = (1.0 / 8.0),
 514       .lineWidthGranularity                     = (1.0 / 128.0),
 515       .strictLines                              = false, /* FINISHME */
 516       .standardSampleLocations                  = true, /* FINISHME */
 517       .optimalBufferCopyOffsetAlignment         = 128,
 518       .optimalBufferCopyRowPitchAlignment       = 128,
 519       .nonCoherentAtomSize                      = 64,
 520    };
 521
 522    *pProperties = (VkPhysicalDeviceProperties) {
 523       .apiVersion = VK_MAKE_VERSION(1, 0, 0),
 524       .driverVersion = 1,
 525       .vendorID = 0x8086,
 526       .deviceID = pdevice->chipset_id,
 527       .deviceType = VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU,
 528       .limits = limits,
 529       .sparseProperties = {0}, /* Broadwell doesn't do sparse. */
 530    };
 531
 532    strcpy(pProperties->deviceName, pdevice->name);
 533    snprintf((char *)pProperties->pipelineCacheUUID, VK_UUID_SIZE,
 534             "anv-%s", MESA_GIT_SHA1 + 4);
 535 }
 536
 537 void anv_GetPhysicalDeviceQueueFamilyProperties(
 538     VkPhysicalDevice                            physicalDevice,
 539     uint32_t*                                   pCount,
 540     VkQueueFamilyProperties*                    pQueueFamilyProperties)
 541 {
 542    if (pQueueFamilyProperties == NULL) {
 543       *pCount = 1;
 544       return;
 545    }
 546
 547    assert(*pCount >= 1);
 548
 549    *pQueueFamilyProperties = (VkQueueFamilyProperties) {
 550       .queueFlags = VK_QUEUE_GRAPHICS_BIT |
 551                     VK_QUEUE_COMPUTE_BIT |
 552                     VK_QUEUE_TRANSFER_BIT,
 553       .queueCount = 1,
 554       .timestampValidBits = 36, /* XXX: Real value here */
 555       .minImageTransferGranularity = (VkExtent3D) { 1, 1, 1 },
 556    };
 557 }
 558
 559 void anv_GetPhysicalDeviceMemoryProperties(
 560     VkPhysicalDevice                            physicalDevice,
 561     VkPhysicalDeviceMemoryProperties*           pMemoryProperties)
 562 {
 563    ANV_FROM_HANDLE(anv_physical_device, physical_device, physicalDevice);
 564    VkDeviceSize heap_size;
 565
 566    /* Reserve some wiggle room for the driver by exposing only 75% of the
 567     * aperture to the heap.
 568     */
 569    heap_size = 3 * physical_device->aperture_size / 4;
 570
 571    if (physical_device->info->has_llc) {
 572       /* Big core GPUs share LLC with the CPU and thus one memory type can be
 573        * both cached and coherent at the same time.
 574        */
 575       pMemoryProperties->memoryTypeCount = 1;
 576       pMemoryProperties->memoryTypes[0] = (VkMemoryType) {
 577          .propertyFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT |
 578                           VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
 579                           VK_MEMORY_PROPERTY_HOST_COHERENT_BIT |
 580                           VK_MEMORY_PROPERTY_HOST_CACHED_BIT,
 581          .heapIndex = 0,
 582       };
 583    } else {
 584       /* The spec requires that we expose a host-visible, coherent memory
 585        * type, but Atom GPUs don't share LLC. Thus we offer two memory types
 586        * to give the application a choice between cached, but not coherent and
 587        * coherent but uncached (WC though).
 588        */
 589       pMemoryProperties->memoryTypeCount = 2;
 590       pMemoryProperties->memoryTypes[0] = (VkMemoryType) {
 591          .propertyFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT |
 592                           VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
 593                           VK_MEMORY_PROPERTY_HOST_COHERENT_BIT,
 594          .heapIndex = 0,
 595       };
 596       pMemoryProperties->memoryTypes[1] = (VkMemoryType) {
 597          .propertyFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT |
 598                           VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
 599                           VK_MEMORY_PROPERTY_HOST_CACHED_BIT,
 600          .heapIndex = 0,
 601       };
 602    }
 603
 604    pMemoryProperties->memoryHeapCount = 1;
 605    pMemoryProperties->memoryHeaps[0] = (VkMemoryHeap) {
 606       .size = heap_size,
 607       .flags = VK_MEMORY_HEAP_DEVICE_LOCAL_BIT,
 608    };
 609 }
 610
 611 PFN_vkVoidFunction anv_GetInstanceProcAddr(
 612     VkInstance                                  instance,
 613     const char*                                 pName)
 614 {
 615    return anv_lookup_entrypoint(pName);
 616 }
 617
 618 PFN_vkVoidFunction anv_GetDeviceProcAddr(
 619     VkDevice                                    device,
 620     const char*                                 pName)
 621 {
 622    return anv_lookup_entrypoint(pName);
 623 }
 624
 625 static VkResult
 626 anv_queue_init(struct anv_device *device, struct anv_queue *queue)
 627 {
 628    queue->_loader_data.loaderMagic = ICD_LOADER_MAGIC;
 629    queue->device = device;
 630    queue->pool = &device->surface_state_pool;
 631
 632    return VK_SUCCESS;
 633 }
 634
 635 static void
 636 anv_queue_finish(struct anv_queue *queue)
 637 {
 638 }
 639
 640 static struct anv_state
 641 anv_state_pool_emit_data(struct anv_state_pool *pool, size_t size, size_t align, const void *p)
 642 {
 643    struct anv_state state;
 644
 645    state = anv_state_pool_alloc(pool, size, align);
 646    memcpy(state.map, p, size);
 647
 648    if (!pool->block_pool->device->info.has_llc)
 649       anv_state_clflush(state);
 650
 651    return state;
 652 }
 653
 654 struct gen8_border_color {
 655    union {
 656       float float32[4];
 657       uint32_t uint32[4];
 658    };
 659    /* Pad out to 64 bytes */
 660    uint32_t _pad[12];
 661 };
 662
 663 static void
 664 anv_device_init_border_colors(struct anv_device *device)
 665 {
 666    static const struct gen8_border_color border_colors[] = {
 667       [VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK] =  { .float32 = { 0.0, 0.0, 0.0, 0.0 } },
 668       [VK_BORDER_COLOR_FLOAT_OPAQUE_BLACK] =       { .float32 = { 0.0, 0.0, 0.0, 1.0 } },
 669       [VK_BORDER_COLOR_FLOAT_OPAQUE_WHITE] =       { .float32 = { 1.0, 1.0, 1.0, 1.0 } },
 670       [VK_BORDER_COLOR_INT_TRANSPARENT_BLACK] =    { .uint32 = { 0, 0, 0, 0 } },
 671       [VK_BORDER_COLOR_INT_OPAQUE_BLACK] =         { .uint32 = { 0, 0, 0, 1 } },
 672       [VK_BORDER_COLOR_INT_OPAQUE_WHITE] =         { .uint32 = { 1, 1, 1, 1 } },
 673    };
 674
 675    device->border_colors = anv_state_pool_emit_data(&device->dynamic_state_pool,
 676                                                     sizeof(border_colors), 64,
 677                                                     border_colors);
 678 }
 679
 680 VkResult anv_CreateDevice(
 681     VkPhysicalDevice                            physicalDevice,
 682     const VkDeviceCreateInfo*                   pCreateInfo,
 683     const VkAllocationCallbacks*                pAllocator,
 684     VkDevice*                                   pDevice)
 685 {
 686    ANV_FROM_HANDLE(anv_physical_device, physical_device, physicalDevice);
 687    VkResult result;
 688    struct anv_device *device;
 689
 690    assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO);
 691
 692    for (uint32_t i = 0; i < pCreateInfo->enabledExtensionCount; i++) {
 693       bool found = false;
 694       for (uint32_t j = 0; j < ARRAY_SIZE(device_extensions); j++) {
 695          if (strcmp(pCreateInfo->ppEnabledExtensionNames[i],
 696                     device_extensions[j].extensionName) == 0) {
 697             found = true;
 698             break;
 699          }
 700       }
 701       if (!found)
 702          return vk_error(VK_ERROR_EXTENSION_NOT_PRESENT);
 703    }
 704
 705    anv_set_dispatch_devinfo(physical_device->info);
 706
 707    device = anv_alloc2(&physical_device->instance->alloc, pAllocator,
 708                        sizeof(*device), 8,
 709                        VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
 710    if (!device)
 711       return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
 712
 713    device->_loader_data.loaderMagic = ICD_LOADER_MAGIC;
 714    device->instance = physical_device->instance;
 715
 716    if (pAllocator)
 717       device->alloc = *pAllocator;
 718    else
 719       device->alloc = physical_device->instance->alloc;
 720
 721    /* XXX(chadv): Can we dup() physicalDevice->fd here? */
 722    device->fd = open(physical_device->path, O_RDWR | O_CLOEXEC);
 723    if (device->fd == -1) {
 724       result = vk_error(VK_ERROR_INITIALIZATION_FAILED);
 725       goto fail_device;
 726    }
 727
 728    device->context_id = anv_gem_create_context(device);
 729    if (device->context_id == -1) {
 730       result = vk_error(VK_ERROR_INITIALIZATION_FAILED);
 731       goto fail_fd;
 732    }
 733
 734    device->info = *physical_device->info;
 735    device->isl_dev = physical_device->isl_dev;
 736
 737    pthread_mutex_init(&device->mutex, NULL);
 738
 739    anv_bo_pool_init(&device->batch_bo_pool, device, ANV_CMD_BUFFER_BATCH_SIZE);
 740
 741    anv_block_pool_init(&device->dynamic_state_block_pool, device, 16384);
 742
 743    anv_state_pool_init(&device->dynamic_state_pool,
 744                        &device->dynamic_state_block_pool);
 745
 746    anv_block_pool_init(&device->instruction_block_pool, device, 128 * 1024);
 747    anv_pipeline_cache_init(&device->default_pipeline_cache, device);
 748
 749    anv_block_pool_init(&device->surface_state_block_pool, device, 4096);
 750
 751    anv_state_pool_init(&device->surface_state_pool,
 752                        &device->surface_state_block_pool);
 753
 754    anv_bo_init_new(&device->workaround_bo, device, 1024);
 755
 756    anv_block_pool_init(&device->scratch_block_pool, device, 0x10000);
 757
 758    anv_queue_init(device, &device->queue);
 759
 760    result = anv_device_init_meta(device);
 761    if (result != VK_SUCCESS)
 762       goto fail_fd;
 763
 764    anv_device_init_border_colors(device);
 765
 766    *pDevice = anv_device_to_handle(device);
 767
 768    return VK_SUCCESS;
 769
 770  fail_fd:
 771    close(device->fd);
 772  fail_device:
 773    anv_free(&device->alloc, device);
 774
 775    return result;
 776 }
 777
 778 void anv_DestroyDevice(
 779     VkDevice                                    _device,
 780     const VkAllocationCallbacks*                pAllocator)
 781 {
 782    ANV_FROM_HANDLE(anv_device, device, _device);
 783
 784    anv_queue_finish(&device->queue);
 785
 786    anv_device_finish_meta(device);
 787
 788 #ifdef HAVE_VALGRIND
 789    /* We only need to free these to prevent valgrind errors.  The backing
 790     * BO will go away in a couple of lines so we don't actually leak.
 791     */
 792    anv_state_pool_free(&device->dynamic_state_pool, device->border_colors);
 793 #endif
 794
 795    anv_gem_munmap(device->workaround_bo.map, device->workaround_bo.size);
 796    anv_gem_close(device, device->workaround_bo.gem_handle);
 797
 798    anv_bo_pool_finish(&device->batch_bo_pool);
 799    anv_state_pool_finish(&device->dynamic_state_pool);
 800    anv_block_pool_finish(&device->dynamic_state_block_pool);
 801    anv_block_pool_finish(&device->instruction_block_pool);
 802    anv_state_pool_finish(&device->surface_state_pool);
 803    anv_block_pool_finish(&device->surface_state_block_pool);
 804    anv_block_pool_finish(&device->scratch_block_pool);
 805
 806    close(device->fd);
 807
 808    pthread_mutex_destroy(&device->mutex);
 809
 810    anv_free(&device->alloc, device);
 811 }
 812
 813 VkResult anv_EnumerateInstanceExtensionProperties(
 814     const char*                                 pLayerName,
 815     uint32_t*                                   pPropertyCount,
 816     VkExtensionProperties*                      pProperties)
 817 {
 818    if (pProperties == NULL) {
 819       *pPropertyCount = ARRAY_SIZE(global_extensions);
 820       return VK_SUCCESS;
 821    }
 822
 823    assert(*pPropertyCount >= ARRAY_SIZE(global_extensions));
 824
 825    *pPropertyCount = ARRAY_SIZE(global_extensions);
 826    memcpy(pProperties, global_extensions, sizeof(global_extensions));
 827
 828    return VK_SUCCESS;
 829 }
 830
 831 VkResult anv_EnumerateDeviceExtensionProperties(
 832     VkPhysicalDevice                            physicalDevice,
 833     const char*                                 pLayerName,
 834     uint32_t*                                   pPropertyCount,
 835     VkExtensionProperties*                      pProperties)
 836 {
 837    if (pProperties == NULL) {
 838       *pPropertyCount = ARRAY_SIZE(device_extensions);
 839       return VK_SUCCESS;
 840    }
 841
 842    assert(*pPropertyCount >= ARRAY_SIZE(device_extensions));
 843
 844    *pPropertyCount = ARRAY_SIZE(device_extensions);
 845    memcpy(pProperties, device_extensions, sizeof(device_extensions));
 846
 847    return VK_SUCCESS;
 848 }
 849
 850 VkResult anv_EnumerateInstanceLayerProperties(
 851     uint32_t*                                   pPropertyCount,
 852     VkLayerProperties*                          pProperties)
 853 {
 854    if (pProperties == NULL) {
 855       *pPropertyCount = 0;
 856       return VK_SUCCESS;
 857    }
 858
 859    /* None supported at this time */
 860    return vk_error(VK_ERROR_LAYER_NOT_PRESENT);
 861 }
 862
 863 VkResult anv_EnumerateDeviceLayerProperties(
 864     VkPhysicalDevice                            physicalDevice,
 865     uint32_t*                                   pPropertyCount,
 866     VkLayerProperties*                          pProperties)
 867 {
 868    if (pProperties == NULL) {
 869       *pPropertyCount = 0;
 870       return VK_SUCCESS;
 871    }
 872
 873    /* None supported at this time */
 874    return vk_error(VK_ERROR_LAYER_NOT_PRESENT);
 875 }
 876
 877 void anv_GetDeviceQueue(
 878     VkDevice                                    _device,
 879     uint32_t                                    queueNodeIndex,
 880     uint32_t                                    queueIndex,
 881     VkQueue*                                    pQueue)
 882 {
 883    ANV_FROM_HANDLE(anv_device, device, _device);
 884
 885    assert(queueIndex == 0);
 886
 887    *pQueue = anv_queue_to_handle(&device->queue);
 888 }
 889
 890 VkResult anv_QueueSubmit(
 891     VkQueue                                     _queue,
 892     uint32_t                                    submitCount,
 893     const VkSubmitInfo*                         pSubmits,
 894     VkFence                                     _fence)
 895 {
 896    ANV_FROM_HANDLE(anv_queue, queue, _queue);
 897    ANV_FROM_HANDLE(anv_fence, fence, _fence);
 898    struct anv_device *device = queue->device;
 899    int ret;
 900
 901    for (uint32_t i = 0; i < submitCount; i++) {
 902       for (uint32_t j = 0; j < pSubmits[i].commandBufferCount; j++) {
 903          ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer,
 904                          pSubmits[i].pCommandBuffers[j]);
 905          assert(cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_PRIMARY);
 906
 907          ret = anv_gem_execbuffer(device, &cmd_buffer->execbuf2.execbuf);
 908          if (ret != 0) {
 909             /* We don't know the real error. */
 910             return vk_errorf(VK_ERROR_OUT_OF_DEVICE_MEMORY,
 911                              "execbuf2 failed: %m");
 912          }
 913
 914          if (fence) {
 915             ret = anv_gem_execbuffer(device, &fence->execbuf);
 916             if (ret != 0) {
 917                /* We don't know the real error. */
 918                return vk_errorf(VK_ERROR_OUT_OF_DEVICE_MEMORY,
 919                                 "execbuf2 failed: %m");
 920             }
 921          }
 922
 923          for (uint32_t k = 0; k < cmd_buffer->execbuf2.bo_count; k++)
 924             cmd_buffer->execbuf2.bos[k]->offset = cmd_buffer->execbuf2.objects[k].offset;
 925       }
 926    }
 927
 928    return VK_SUCCESS;
 929 }
 930
 931 VkResult anv_QueueWaitIdle(
 932     VkQueue                                     _queue)
 933 {
 934    ANV_FROM_HANDLE(anv_queue, queue, _queue);
 935
 936    return ANV_CALL(DeviceWaitIdle)(anv_device_to_handle(queue->device));
 937 }
 938
 939 VkResult anv_DeviceWaitIdle(
 940     VkDevice                                    _device)
 941 {
 942    ANV_FROM_HANDLE(anv_device, device, _device);
 943    struct anv_state state;
 944    struct anv_batch batch;
 945    struct drm_i915_gem_execbuffer2 execbuf;
 946    struct drm_i915_gem_exec_object2 exec2_objects[1];
 947    struct anv_bo *bo = NULL;
 948    VkResult result;
 949    int64_t timeout;
 950    int ret;
 951
 952    state = anv_state_pool_alloc(&device->dynamic_state_pool, 32, 32);
 953    bo = &device->dynamic_state_pool.block_pool->bo;
 954    batch.start = batch.next = state.map;
 955    batch.end = state.map + 32;
 956    anv_batch_emit(&batch, GEN7_MI_BATCH_BUFFER_END);
 957    anv_batch_emit(&batch, GEN7_MI_NOOP);
 958
 959    if (!device->info.has_llc)
 960       anv_state_clflush(state);
 961
 962    exec2_objects[0].handle = bo->gem_handle;
 963    exec2_objects[0].relocation_count = 0;
 964    exec2_objects[0].relocs_ptr = 0;
 965    exec2_objects[0].alignment = 0;
 966    exec2_objects[0].offset = bo->offset;
 967    exec2_objects[0].flags = 0;
 968    exec2_objects[0].rsvd1 = 0;
 969    exec2_objects[0].rsvd2 = 0;
 970
 971    execbuf.buffers_ptr = (uintptr_t) exec2_objects;
 972    execbuf.buffer_count = 1;
 973    execbuf.batch_start_offset = state.offset;
 974    execbuf.batch_len = batch.next - state.map;
 975    execbuf.cliprects_ptr = 0;
 976    execbuf.num_cliprects = 0;
 977    execbuf.DR1 = 0;
 978    execbuf.DR4 = 0;
 979
 980    execbuf.flags =
 981       I915_EXEC_HANDLE_LUT | I915_EXEC_NO_RELOC | I915_EXEC_RENDER;
 982    execbuf.rsvd1 = device->context_id;
 983    execbuf.rsvd2 = 0;
 984
 985    ret = anv_gem_execbuffer(device, &execbuf);
 986    if (ret != 0) {
 987       /* We don't know the real error. */
 988       result = vk_errorf(VK_ERROR_OUT_OF_DEVICE_MEMORY, "execbuf2 failed: %m");
 989       goto fail;
 990    }
 991
 992    timeout = INT64_MAX;
 993    ret = anv_gem_wait(device, bo->gem_handle, &timeout);
 994    if (ret != 0) {
 995       /* We don't know the real error. */
 996       result = vk_errorf(VK_ERROR_OUT_OF_DEVICE_MEMORY, "execbuf2 failed: %m");
 997       goto fail;
 998    }
 999
1000    anv_state_pool_free(&device->dynamic_state_pool, state);
1001
1002    return VK_SUCCESS;
1003
1004  fail:
1005    anv_state_pool_free(&device->dynamic_state_pool, state);
1006
1007    return result;
1008 }
1009
1010 VkResult
1011 anv_bo_init_new(struct anv_bo *bo, struct anv_device *device, uint64_t size)
1012 {
1013    bo->gem_handle = anv_gem_create(device, size);
1014    if (!bo->gem_handle)
1015       return vk_error(VK_ERROR_OUT_OF_DEVICE_MEMORY);
1016
1017    bo->map = NULL;
1018    bo->index = 0;
1019    bo->offset = 0;
1020    bo->size = size;
1021
1022    return VK_SUCCESS;
1023 }
1024
1025 VkResult anv_AllocateMemory(
1026     VkDevice                                    _device,
1027     const VkMemoryAllocateInfo*                 pAllocateInfo,
1028     const VkAllocationCallbacks*                pAllocator,
1029     VkDeviceMemory*                             pMem)
1030 {
1031    ANV_FROM_HANDLE(anv_device, device, _device);
1032    struct anv_device_memory *mem;
1033    VkResult result;
1034
1035    assert(pAllocateInfo->sType == VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO);
1036
1037    if (pAllocateInfo->allocationSize == 0) {
1038       /* Apparently, this is allowed */
1039       *pMem = VK_NULL_HANDLE;
1040       return VK_SUCCESS;
1041    }
1042
1043    /* We support exactly one memory heap. */
1044    assert(pAllocateInfo->memoryTypeIndex == 0 ||
1045           (!device->info.has_llc && pAllocateInfo->memoryTypeIndex < 2));
1046
1047    /* FINISHME: Fail if allocation request exceeds heap size. */
1048
1049    mem = anv_alloc2(&device->alloc, pAllocator, sizeof(*mem), 8,
1050                     VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
1051    if (mem == NULL)
1052       return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
1053
1054    /* The kernel is going to give us whole pages anyway */
1055    uint64_t alloc_size = align_u64(pAllocateInfo->allocationSize, 4096);
1056
1057    result = anv_bo_init_new(&mem->bo, device, alloc_size);
1058    if (result != VK_SUCCESS)
1059       goto fail;
1060
1061    mem->type_index = pAllocateInfo->memoryTypeIndex;
1062
1063    *pMem = anv_device_memory_to_handle(mem);
1064
1065    return VK_SUCCESS;
1066
1067  fail:
1068    anv_free2(&device->alloc, pAllocator, mem);
1069
1070    return result;
1071 }
1072
1073 void anv_FreeMemory(
1074     VkDevice                                    _device,
1075     VkDeviceMemory                              _mem,
1076     const VkAllocationCallbacks*                pAllocator)
1077 {
1078    ANV_FROM_HANDLE(anv_device, device, _device);
1079    ANV_FROM_HANDLE(anv_device_memory, mem, _mem);
1080
1081    if (mem == NULL)
1082       return;
1083
1084    if (mem->bo.map)
1085       anv_gem_munmap(mem->bo.map, mem->bo.size);
1086
1087    if (mem->bo.gem_handle != 0)
1088       anv_gem_close(device, mem->bo.gem_handle);
1089
1090    anv_free2(&device->alloc, pAllocator, mem);
1091 }
1092
1093 VkResult anv_MapMemory(
1094     VkDevice                                    _device,
1095     VkDeviceMemory                              _memory,
1096     VkDeviceSize                                offset,
1097     VkDeviceSize                                size,
1098     VkMemoryMapFlags                            flags,
1099     void**                                      ppData)
1100 {
1101    ANV_FROM_HANDLE(anv_device, device, _device);
1102    ANV_FROM_HANDLE(anv_device_memory, mem, _memory);
1103
1104    if (mem == NULL) {
1105       *ppData = NULL;
1106       return VK_SUCCESS;
1107    }
1108
1109    if (size == VK_WHOLE_SIZE)
1110       size = mem->bo.size - offset;
1111
1112    /* FIXME: Is this supposed to be thread safe? Since vkUnmapMemory() only
1113     * takes a VkDeviceMemory pointer, it seems like only one map of the memory
1114     * at a time is valid. We could just mmap up front and return an offset
1115     * pointer here, but that may exhaust virtual memory on 32 bit
1116     * userspace. */
1117
1118    uint32_t gem_flags = 0;
1119    if (!device->info.has_llc && mem->type_index == 0)
1120       gem_flags |= I915_MMAP_WC;
1121
1122    /* GEM will fail to map if the offset isn't 4k-aligned.  Round down. */
1123    uint64_t map_offset = offset & ~4095ull;
1124    assert(offset >= map_offset);
1125    uint64_t map_size = (offset + size) - map_offset;
1126
1127    /* Let's map whole pages */
1128    map_size = align_u64(map_size, 4096);
1129
1130    mem->map = anv_gem_mmap(device, mem->bo.gem_handle,
1131                            map_offset, map_size, gem_flags);
1132    mem->map_size = map_size;
1133
1134    *ppData = mem->map + (offset - map_offset);
1135
1136    return VK_SUCCESS;
1137 }
1138
1139 void anv_UnmapMemory(
1140     VkDevice                                    _device,
1141     VkDeviceMemory                              _memory)
1142 {
1143    ANV_FROM_HANDLE(anv_device_memory, mem, _memory);
1144
1145    if (mem == NULL)
1146       return;
1147
1148    anv_gem_munmap(mem->map, mem->map_size);
1149 }
1150
1151 static void
1152 clflush_mapped_ranges(struct anv_device         *device,
1153                       uint32_t                   count,
1154                       const VkMappedMemoryRange *ranges)
1155 {
1156    for (uint32_t i = 0; i < count; i++) {
1157       ANV_FROM_HANDLE(anv_device_memory, mem, ranges[i].memory);
1158       void *p = mem->map + (ranges[i].offset & ~CACHELINE_MASK);
1159       void *end = mem->map + ranges[i].offset + ranges[i].size;
1160
1161       while (p < end) {
1162          __builtin_ia32_clflush(p);
1163          p += CACHELINE_SIZE;
1164       }
1165    }
1166 }
1167
1168 VkResult anv_FlushMappedMemoryRanges(
1169     VkDevice                                    _device,
1170     uint32_t                                    memoryRangeCount,
1171     const VkMappedMemoryRange*                  pMemoryRanges)
1172 {
1173    ANV_FROM_HANDLE(anv_device, device, _device);
1174
1175    if (device->info.has_llc)
1176       return VK_SUCCESS;
1177
1178    /* Make sure the writes we're flushing have landed. */
1179    __builtin_ia32_sfence();
1180
1181    clflush_mapped_ranges(device, memoryRangeCount, pMemoryRanges);
1182
1183    return VK_SUCCESS;
1184 }
1185
1186 VkResult anv_InvalidateMappedMemoryRanges(
1187     VkDevice                                    _device,
1188     uint32_t                                    memoryRangeCount,
1189     const VkMappedMemoryRange*                  pMemoryRanges)
1190 {
1191    ANV_FROM_HANDLE(anv_device, device, _device);
1192
1193    if (device->info.has_llc)
1194       return VK_SUCCESS;
1195
1196    clflush_mapped_ranges(device, memoryRangeCount, pMemoryRanges);
1197
1198    /* Make sure no reads get moved up above the invalidate. */
1199    __builtin_ia32_lfence();
1200
1201    return VK_SUCCESS;
1202 }
1203
1204 void anv_GetBufferMemoryRequirements(
1205     VkDevice                                    device,
1206     VkBuffer                                    _buffer,
1207     VkMemoryRequirements*                       pMemoryRequirements)
1208 {
1209    ANV_FROM_HANDLE(anv_buffer, buffer, _buffer);
1210
1211    /* The Vulkan spec (git aaed022) says:
1212     *
1213     *    memoryTypeBits is a bitfield and contains one bit set for every
1214     *    supported memory type for the resource. The bit `1<<i` is set if and
1215     *    only if the memory type `i` in the VkPhysicalDeviceMemoryProperties
1216     *    structure for the physical device is supported.
1217     *
1218     * We support exactly one memory type.
1219     */
1220    pMemoryRequirements->memoryTypeBits = 1;
1221
1222    pMemoryRequirements->size = buffer->size;
1223    pMemoryRequirements->alignment = 16;
1224 }
1225
1226 void anv_GetImageMemoryRequirements(
1227     VkDevice                                    device,
1228     VkImage                                     _image,
1229     VkMemoryRequirements*                       pMemoryRequirements)
1230 {
1231    ANV_FROM_HANDLE(anv_image, image, _image);
1232
1233    /* The Vulkan spec (git aaed022) says:
1234     *
1235     *    memoryTypeBits is a bitfield and contains one bit set for every
1236     *    supported memory type for the resource. The bit `1<<i` is set if and
1237     *    only if the memory type `i` in the VkPhysicalDeviceMemoryProperties
1238     *    structure for the physical device is supported.
1239     *
1240     * We support exactly one memory type.
1241     */
1242    pMemoryRequirements->memoryTypeBits = 1;
1243
1244    pMemoryRequirements->size = image->size;
1245    pMemoryRequirements->alignment = image->alignment;
1246 }
1247
1248 void anv_GetImageSparseMemoryRequirements(
1249     VkDevice                                    device,
1250     VkImage                                     image,
1251     uint32_t*                                   pSparseMemoryRequirementCount,
1252     VkSparseImageMemoryRequirements*            pSparseMemoryRequirements)
1253 {
1254    stub();
1255 }
1256
1257 void anv_GetDeviceMemoryCommitment(
1258     VkDevice                                    device,
1259     VkDeviceMemory                              memory,
1260     VkDeviceSize*                               pCommittedMemoryInBytes)
1261 {
1262    *pCommittedMemoryInBytes = 0;
1263 }
1264
1265 VkResult anv_BindBufferMemory(
1266     VkDevice                                    device,
1267     VkBuffer                                    _buffer,
1268     VkDeviceMemory                              _memory,
1269     VkDeviceSize                                memoryOffset)
1270 {
1271    ANV_FROM_HANDLE(anv_device_memory, mem, _memory);
1272    ANV_FROM_HANDLE(anv_buffer, buffer, _buffer);
1273
1274    if (mem) {
1275       buffer->bo = &mem->bo;
1276       buffer->offset = memoryOffset;
1277    } else {
1278       buffer->bo = NULL;
1279       buffer->offset = 0;
1280    }
1281
1282    return VK_SUCCESS;
1283 }
1284
1285 VkResult anv_BindImageMemory(
1286     VkDevice                                    device,
1287     VkImage                                     _image,
1288     VkDeviceMemory                              _memory,
1289     VkDeviceSize                                memoryOffset)
1290 {
1291    ANV_FROM_HANDLE(anv_device_memory, mem, _memory);
1292    ANV_FROM_HANDLE(anv_image, image, _image);
1293
1294    if (mem) {
1295       image->bo = &mem->bo;
1296       image->offset = memoryOffset;
1297    } else {
1298       image->bo = NULL;
1299       image->offset = 0;
1300    }
1301
1302    return VK_SUCCESS;
1303 }
1304
1305 VkResult anv_QueueBindSparse(
1306     VkQueue                                     queue,
1307     uint32_t                                    bindInfoCount,
1308     const VkBindSparseInfo*                     pBindInfo,
1309     VkFence                                     fence)
1310 {
1311    stub_return(VK_ERROR_INCOMPATIBLE_DRIVER);
1312 }
1313
1314 VkResult anv_CreateFence(
1315     VkDevice                                    _device,
1316     const VkFenceCreateInfo*                    pCreateInfo,
1317     const VkAllocationCallbacks*                pAllocator,
1318     VkFence*                                    pFence)
1319 {
1320    ANV_FROM_HANDLE(anv_device, device, _device);
1321    struct anv_fence *fence;
1322    struct anv_batch batch;
1323    VkResult result;
1324
1325    const uint32_t fence_size = 128;
1326
1327    assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_FENCE_CREATE_INFO);
1328
1329    fence = anv_alloc2(&device->alloc, pAllocator, sizeof(*fence), 8,
1330                       VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
1331    if (fence == NULL)
1332       return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
1333
1334    result = anv_bo_init_new(&fence->bo, device, fence_size);
1335    if (result != VK_SUCCESS)
1336       goto fail;
1337
1338    fence->bo.map =
1339       anv_gem_mmap(device, fence->bo.gem_handle, 0, fence->bo.size, 0);
1340    batch.next = batch.start = fence->bo.map;
1341    batch.end = fence->bo.map + fence->bo.size;
1342    anv_batch_emit(&batch, GEN7_MI_BATCH_BUFFER_END);
1343    anv_batch_emit(&batch, GEN7_MI_NOOP);
1344
1345    if (!device->info.has_llc) {
1346       assert(((uintptr_t) fence->bo.map & CACHELINE_MASK) == 0);
1347       assert(batch.next - fence->bo.map <= CACHELINE_SIZE);
1348       __builtin_ia32_sfence();
1349       __builtin_ia32_clflush(fence->bo.map);
1350    }
1351
1352    fence->exec2_objects[0].handle = fence->bo.gem_handle;
1353    fence->exec2_objects[0].relocation_count = 0;
1354    fence->exec2_objects[0].relocs_ptr = 0;
1355    fence->exec2_objects[0].alignment = 0;
1356    fence->exec2_objects[0].offset = fence->bo.offset;
1357    fence->exec2_objects[0].flags = 0;
1358    fence->exec2_objects[0].rsvd1 = 0;
1359    fence->exec2_objects[0].rsvd2 = 0;
1360
1361    fence->execbuf.buffers_ptr = (uintptr_t) fence->exec2_objects;
1362    fence->execbuf.buffer_count = 1;
1363    fence->execbuf.batch_start_offset = 0;
1364    fence->execbuf.batch_len = batch.next - fence->bo.map;
1365    fence->execbuf.cliprects_ptr = 0;
1366    fence->execbuf.num_cliprects = 0;
1367    fence->execbuf.DR1 = 0;
1368    fence->execbuf.DR4 = 0;
1369
1370    fence->execbuf.flags =
1371       I915_EXEC_HANDLE_LUT | I915_EXEC_NO_RELOC | I915_EXEC_RENDER;
1372    fence->execbuf.rsvd1 = device->context_id;
1373    fence->execbuf.rsvd2 = 0;
1374
1375    *pFence = anv_fence_to_handle(fence);
1376
1377    return VK_SUCCESS;
1378
1379  fail:
1380    anv_free2(&device->alloc, pAllocator, fence);
1381
1382    return result;
1383 }
1384
1385 void anv_DestroyFence(
1386     VkDevice                                    _device,
1387     VkFence                                     _fence,
1388     const VkAllocationCallbacks*                pAllocator)
1389 {
1390    ANV_FROM_HANDLE(anv_device, device, _device);
1391    ANV_FROM_HANDLE(anv_fence, fence, _fence);
1392
1393    anv_gem_munmap(fence->bo.map, fence->bo.size);
1394    anv_gem_close(device, fence->bo.gem_handle);
1395    anv_free2(&device->alloc, pAllocator, fence);
1396 }
1397
1398 VkResult anv_ResetFences(
1399     VkDevice                                    _device,
1400     uint32_t                                    fenceCount,
1401     const VkFence*                              pFences)
1402 {
1403    for (uint32_t i = 0; i < fenceCount; i++) {
1404       ANV_FROM_HANDLE(anv_fence, fence, pFences[i]);
1405       fence->ready = false;
1406    }
1407
1408    return VK_SUCCESS;
1409 }
1410
1411 VkResult anv_GetFenceStatus(
1412     VkDevice                                    _device,
1413     VkFence                                     _fence)
1414 {
1415    ANV_FROM_HANDLE(anv_device, device, _device);
1416    ANV_FROM_HANDLE(anv_fence, fence, _fence);
1417    int64_t t = 0;
1418    int ret;
1419
1420    if (fence->ready)
1421       return VK_SUCCESS;
1422
1423    ret = anv_gem_wait(device, fence->bo.gem_handle, &t);
1424    if (ret == 0) {
1425       fence->ready = true;
1426       return VK_SUCCESS;
1427    }
1428
1429    return VK_NOT_READY;
1430 }
1431
1432 VkResult anv_WaitForFences(
1433     VkDevice                                    _device,
1434     uint32_t                                    fenceCount,
1435     const VkFence*                              pFences,
1436     VkBool32                                    waitAll,
1437     uint64_t                                    timeout)
1438 {
1439    ANV_FROM_HANDLE(anv_device, device, _device);
1440
1441    /* DRM_IOCTL_I915_GEM_WAIT uses a signed 64 bit timeout and is supposed
1442     * to block indefinitely timeouts <= 0.  Unfortunately, this was broken
1443     * for a couple of kernel releases.  Since there's no way to know
1444     * whether or not the kernel we're using is one of the broken ones, the
1445     * best we can do is to clamp the timeout to INT64_MAX.  This limits the
1446     * maximum timeout from 584 years to 292 years - likely not a big deal.
1447     */
1448    if (timeout > INT64_MAX)
1449       timeout = INT64_MAX;
1450
1451    int64_t t = timeout;
1452
1453    /* FIXME: handle !waitAll */
1454
1455    for (uint32_t i = 0; i < fenceCount; i++) {
1456       ANV_FROM_HANDLE(anv_fence, fence, pFences[i]);
1457       int ret = anv_gem_wait(device, fence->bo.gem_handle, &t);
1458       if (ret == -1 && errno == ETIME) {
1459          return VK_TIMEOUT;
1460       } else if (ret == -1) {
1461          /* We don't know the real error. */
1462          return vk_errorf(VK_ERROR_OUT_OF_DEVICE_MEMORY,
1463                           "gem wait failed: %m");
1464       }
1465    }
1466
1467    return VK_SUCCESS;
1468 }
1469
1470 // Queue semaphore functions
1471
1472 VkResult anv_CreateSemaphore(
1473     VkDevice                                    device,
1474     const VkSemaphoreCreateInfo*                pCreateInfo,
1475     const VkAllocationCallbacks*                pAllocator,
1476     VkSemaphore*                                pSemaphore)
1477 {
1478    /* The DRM execbuffer ioctl always execute in-oder, even between different
1479     * rings. As such, there's nothing to do for the user space semaphore.
1480     */
1481
1482    *pSemaphore = (VkSemaphore)1;
1483
1484    return VK_SUCCESS;
1485 }
1486
1487 void anv_DestroySemaphore(
1488     VkDevice                                    device,
1489     VkSemaphore                                 semaphore,
1490     const VkAllocationCallbacks*                pAllocator)
1491 {
1492 }
1493
1494 // Event functions
1495
1496 VkResult anv_CreateEvent(
1497     VkDevice                                    _device,
1498     const VkEventCreateInfo*                    pCreateInfo,
1499     const VkAllocationCallbacks*                pAllocator,
1500     VkEvent*                                    pEvent)
1501 {
1502    ANV_FROM_HANDLE(anv_device, device, _device);
1503    struct anv_state state;
1504    struct anv_event *event;
1505
1506    assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_EVENT_CREATE_INFO);
1507
1508    state = anv_state_pool_alloc(&device->dynamic_state_pool,
1509                                 sizeof(*event), 4);
1510    event = state.map;
1511    event->state = state;
1512    event->semaphore = VK_EVENT_RESET;
1513
1514    if (!device->info.has_llc) {
1515       /* Make sure the writes we're flushing have landed. */
1516       __builtin_ia32_sfence();
1517       __builtin_ia32_clflush(event);
1518    }
1519
1520    *pEvent = anv_event_to_handle(event);
1521
1522    return VK_SUCCESS;
1523 }
1524
1525 void anv_DestroyEvent(
1526     VkDevice                                    _device,
1527     VkEvent                                     _event,
1528     const VkAllocationCallbacks*                pAllocator)
1529 {
1530    ANV_FROM_HANDLE(anv_device, device, _device);
1531    ANV_FROM_HANDLE(anv_event, event, _event);
1532
1533    anv_state_pool_free(&device->dynamic_state_pool, event->state);
1534 }
1535
1536 VkResult anv_GetEventStatus(
1537     VkDevice                                    _device,
1538     VkEvent                                     _event)
1539 {
1540    ANV_FROM_HANDLE(anv_device, device, _device);
1541    ANV_FROM_HANDLE(anv_event, event, _event);
1542
1543    if (!device->info.has_llc) {
1544       /* Make sure the writes we're flushing have landed. */
1545       __builtin_ia32_clflush(event);
1546       __builtin_ia32_lfence();
1547    }
1548
1549    return event->semaphore;
1550 }
1551
1552 VkResult anv_SetEvent(
1553     VkDevice                                    _device,
1554     VkEvent                                     _event)
1555 {
1556    ANV_FROM_HANDLE(anv_device, device, _device);
1557    ANV_FROM_HANDLE(anv_event, event, _event);
1558
1559    event->semaphore = VK_EVENT_SET;
1560
1561    if (!device->info.has_llc) {
1562       /* Make sure the writes we're flushing have landed. */
1563       __builtin_ia32_sfence();
1564       __builtin_ia32_clflush(event);
1565    }
1566
1567    return VK_SUCCESS;
1568 }
1569
1570 VkResult anv_ResetEvent(
1571     VkDevice                                    _device,
1572     VkEvent                                     _event)
1573 {
1574    ANV_FROM_HANDLE(anv_device, device, _device);
1575    ANV_FROM_HANDLE(anv_event, event, _event);
1576
1577    event->semaphore = VK_EVENT_RESET;
1578
1579    if (!device->info.has_llc) {
1580       /* Make sure the writes we're flushing have landed. */
1581       __builtin_ia32_sfence();
1582       __builtin_ia32_clflush(event);
1583    }
1584
1585    return VK_SUCCESS;
1586 }
1587
1588 // Buffer functions
1589
1590 VkResult anv_CreateBuffer(
1591     VkDevice                                    _device,
1592     const VkBufferCreateInfo*                   pCreateInfo,
1593     const VkAllocationCallbacks*                pAllocator,
1594     VkBuffer*                                   pBuffer)
1595 {
1596    ANV_FROM_HANDLE(anv_device, device, _device);
1597    struct anv_buffer *buffer;
1598
1599    assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO);
1600
1601    buffer = anv_alloc2(&device->alloc, pAllocator, sizeof(*buffer), 8,
1602                        VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
1603    if (buffer == NULL)
1604       return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
1605
1606    buffer->size = pCreateInfo->size;
1607    buffer->usage = pCreateInfo->usage;
1608    buffer->bo = NULL;
1609    buffer->offset = 0;
1610
1611    *pBuffer = anv_buffer_to_handle(buffer);
1612
1613    return VK_SUCCESS;
1614 }
1615
1616 void anv_DestroyBuffer(
1617     VkDevice                                    _device,
1618     VkBuffer                                    _buffer,
1619     const VkAllocationCallbacks*                pAllocator)
1620 {
1621    ANV_FROM_HANDLE(anv_device, device, _device);
1622    ANV_FROM_HANDLE(anv_buffer, buffer, _buffer);
1623
1624    anv_free2(&device->alloc, pAllocator, buffer);
1625 }
1626
1627 void
1628 anv_fill_buffer_surface_state(struct anv_device *device, void *state,
1629                               enum isl_format format,
1630                               uint32_t offset, uint32_t range, uint32_t stride)
1631 {
1632    switch (device->info.gen) {
1633    case 7:
1634       if (device->info.is_haswell)
1635          gen75_fill_buffer_surface_state(state, format, offset, range, stride);
1636       else
1637          gen7_fill_buffer_surface_state(state, format, offset, range, stride);
1638       break;
1639    case 8:
1640       gen8_fill_buffer_surface_state(state, format, offset, range, stride);
1641       break;
1642    case 9:
1643       gen9_fill_buffer_surface_state(state, format, offset, range, stride);
1644       break;
1645    default:
1646       unreachable("unsupported gen\n");
1647    }
1648 }
1649
1650 void anv_DestroySampler(
1651     VkDevice                                    _device,
1652     VkSampler                                   _sampler,
1653     const VkAllocationCallbacks*                pAllocator)
1654 {
1655    ANV_FROM_HANDLE(anv_device, device, _device);
1656    ANV_FROM_HANDLE(anv_sampler, sampler, _sampler);
1657
1658    anv_free2(&device->alloc, pAllocator, sampler);
1659 }
1660
1661 VkResult anv_CreateFramebuffer(
1662     VkDevice                                    _device,
1663     const VkFramebufferCreateInfo*              pCreateInfo,
1664     const VkAllocationCallbacks*                pAllocator,
1665     VkFramebuffer*                              pFramebuffer)
1666 {
1667    ANV_FROM_HANDLE(anv_device, device, _device);
1668    struct anv_framebuffer *framebuffer;
1669
1670    assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO);
1671
1672    size_t size = sizeof(*framebuffer) +
1673                  sizeof(struct anv_image_view *) * pCreateInfo->attachmentCount;
1674    framebuffer = anv_alloc2(&device->alloc, pAllocator, size, 8,
1675                             VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
1676    if (framebuffer == NULL)
1677       return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
1678
1679    framebuffer->attachment_count = pCreateInfo->attachmentCount;
1680    for (uint32_t i = 0; i < pCreateInfo->attachmentCount; i++) {
1681       VkImageView _iview = pCreateInfo->pAttachments[i];
1682       framebuffer->attachments[i] = anv_image_view_from_handle(_iview);
1683    }
1684
1685    framebuffer->width = pCreateInfo->width;
1686    framebuffer->height = pCreateInfo->height;
1687    framebuffer->layers = pCreateInfo->layers;
1688
1689    *pFramebuffer = anv_framebuffer_to_handle(framebuffer);
1690
1691    return VK_SUCCESS;
1692 }
1693
1694 void anv_DestroyFramebuffer(
1695     VkDevice                                    _device,
1696     VkFramebuffer                               _fb,
1697     const VkAllocationCallbacks*                pAllocator)
1698 {
1699    ANV_FROM_HANDLE(anv_device, device, _device);
1700    ANV_FROM_HANDLE(anv_framebuffer, fb, _fb);
1701
1702    anv_free2(&device->alloc, pAllocator, fb);
1703 }
1704
1705 void vkCmdDbgMarkerBegin(
1706     VkCommandBuffer                              commandBuffer,
1707     const char*                                 pMarker)
1708    __attribute__ ((visibility ("default")));
1709
1710 void vkCmdDbgMarkerEnd(
1711    VkCommandBuffer                              commandBuffer)
1712    __attribute__ ((visibility ("default")));
1713
1714 void vkCmdDbgMarkerBegin(
1715     VkCommandBuffer                              commandBuffer,
1716     const char*                                 pMarker)
1717 {
1718 }
1719
1720 void vkCmdDbgMarkerEnd(
1721     VkCommandBuffer                              commandBuffer)
1722 {
1723 }