src/intel/vulkan/anv_pipeline.c

   1 /*
   2  * Copyright © 2015 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include <assert.h>
  25 #include <stdbool.h>
  26 #include <string.h>
  27 #include <unistd.h>
  28 #include <fcntl.h>
  29
  30 #include "util/mesa-sha1.h"
  31 #include "common/gen_l3_config.h"
  32 #include "anv_private.h"
  33 #include "brw_nir.h"
  34 #include "anv_nir.h"
  35 #include "spirv/nir_spirv.h"
  36
  37 /* Needed for SWIZZLE macros */
  38 #include "program/prog_instruction.h"
  39
  40 // Shader functions
  41
  42 VkResult anv_CreateShaderModule(
  43     VkDevice                                    _device,
  44     const VkShaderModuleCreateInfo*             pCreateInfo,
  45     const VkAllocationCallbacks*                pAllocator,
  46     VkShaderModule*                             pShaderModule)
  47 {
  48    ANV_FROM_HANDLE(anv_device, device, _device);
  49    struct anv_shader_module *module;
  50
  51    assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO);
  52    assert(pCreateInfo->flags == 0);
  53
  54    module = vk_alloc2(&device->alloc, pAllocator,
  55                        sizeof(*module) + pCreateInfo->codeSize, 8,
  56                        VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
  57    if (module == NULL)
  58       return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
  59
  60    module->size = pCreateInfo->codeSize;
  61    memcpy(module->data, pCreateInfo->pCode, module->size);
  62
  63    _mesa_sha1_compute(module->data, module->size, module->sha1);
  64
  65    *pShaderModule = anv_shader_module_to_handle(module);
  66
  67    return VK_SUCCESS;
  68 }
  69
  70 void anv_DestroyShaderModule(
  71     VkDevice                                    _device,
  72     VkShaderModule                              _module,
  73     const VkAllocationCallbacks*                pAllocator)
  74 {
  75    ANV_FROM_HANDLE(anv_device, device, _device);
  76    ANV_FROM_HANDLE(anv_shader_module, module, _module);
  77
  78    vk_free2(&device->alloc, pAllocator, module);
  79 }
  80
  81 #define SPIR_V_MAGIC_NUMBER 0x07230203
  82
  83 /* Eventually, this will become part of anv_CreateShader.  Unfortunately,
  84  * we can't do that yet because we don't have the ability to copy nir.
  85  */
  86 static nir_shader *
  87 anv_shader_compile_to_nir(struct anv_device *device,
  88                           struct anv_shader_module *module,
  89                           const char *entrypoint_name,
  90                           gl_shader_stage stage,
  91                           const VkSpecializationInfo *spec_info)
  92 {
  93    if (strcmp(entrypoint_name, "main") != 0) {
  94       anv_finishme("Multiple shaders per module not really supported");
  95    }
  96
  97    const struct brw_compiler *compiler =
  98       device->instance->physicalDevice.compiler;
  99    const nir_shader_compiler_options *nir_options =
 100       compiler->glsl_compiler_options[stage].NirOptions;
 101
 102    uint32_t *spirv = (uint32_t *) module->data;
 103    assert(spirv[0] == SPIR_V_MAGIC_NUMBER);
 104    assert(module->size % 4 == 0);
 105
 106    uint32_t num_spec_entries = 0;
 107    struct nir_spirv_specialization *spec_entries = NULL;
 108    if (spec_info && spec_info->mapEntryCount > 0) {
 109       num_spec_entries = spec_info->mapEntryCount;
 110       spec_entries = malloc(num_spec_entries * sizeof(*spec_entries));
 111       for (uint32_t i = 0; i < num_spec_entries; i++) {
 112          VkSpecializationMapEntry entry = spec_info->pMapEntries[i];
 113          const void *data = spec_info->pData + entry.offset;
 114          assert(data + entry.size <= spec_info->pData + spec_info->dataSize);
 115
 116          spec_entries[i].id = spec_info->pMapEntries[i].constantID;
 117          spec_entries[i].data = *(const uint32_t *)data;
 118       }
 119    }
 120
 121    nir_function *entry_point =
 122       spirv_to_nir(spirv, module->size / 4,
 123                    spec_entries, num_spec_entries,
 124                    stage, entrypoint_name, nir_options);
 125    nir_shader *nir = entry_point->shader;
 126    assert(nir->stage == stage);
 127    nir_validate_shader(nir);
 128
 129    free(spec_entries);
 130
 131    if (stage == MESA_SHADER_FRAGMENT) {
 132       nir_lower_wpos_center(nir);
 133       nir_validate_shader(nir);
 134    }
 135
 136    nir_lower_returns(nir);
 137    nir_validate_shader(nir);
 138
 139    nir_inline_functions(nir);
 140    nir_validate_shader(nir);
 141
 142    /* Pick off the single entrypoint that we want */
 143    foreach_list_typed_safe(nir_function, func, node, &nir->functions) {
 144       if (func != entry_point)
 145          exec_node_remove(&func->node);
 146    }
 147    assert(exec_list_length(&nir->functions) == 1);
 148    entry_point->name = ralloc_strdup(entry_point, "main");
 149
 150    nir_remove_dead_variables(nir, nir_var_shader_in);
 151    nir_remove_dead_variables(nir, nir_var_shader_out);
 152    nir_remove_dead_variables(nir, nir_var_system_value);
 153    nir_validate_shader(nir);
 154
 155    nir_propagate_invariant(nir);
 156    nir_validate_shader(nir);
 157
 158    nir_lower_io_to_temporaries(entry_point->shader, entry_point->impl,
 159                                true, false);
 160
 161    nir_lower_system_values(nir);
 162    nir_validate_shader(nir);
 163
 164    /* Vulkan uses the separate-shader linking model */
 165    nir->info->separate_shader = true;
 166
 167    nir = brw_preprocess_nir(compiler, nir);
 168
 169    nir_shader_gather_info(nir, entry_point->impl);
 170
 171    nir_variable_mode indirect_mask = 0;
 172    if (compiler->glsl_compiler_options[stage].EmitNoIndirectInput)
 173       indirect_mask |= nir_var_shader_in;
 174    if (compiler->glsl_compiler_options[stage].EmitNoIndirectOutput)
 175       indirect_mask |= nir_var_shader_out;
 176    if (compiler->glsl_compiler_options[stage].EmitNoIndirectTemp)
 177       indirect_mask |= nir_var_local;
 178
 179    nir_lower_indirect_derefs(nir, indirect_mask);
 180
 181    return nir;
 182 }
 183
 184 void anv_DestroyPipeline(
 185     VkDevice                                    _device,
 186     VkPipeline                                  _pipeline,
 187     const VkAllocationCallbacks*                pAllocator)
 188 {
 189    ANV_FROM_HANDLE(anv_device, device, _device);
 190    ANV_FROM_HANDLE(anv_pipeline, pipeline, _pipeline);
 191
 192    anv_reloc_list_finish(&pipeline->batch_relocs,
 193                          pAllocator ? pAllocator : &device->alloc);
 194    if (pipeline->blend_state.map)
 195       anv_state_pool_free(&device->dynamic_state_pool, pipeline->blend_state);
 196
 197    for (unsigned s = 0; s < MESA_SHADER_STAGES; s++) {
 198       if (pipeline->shaders[s])
 199          anv_shader_bin_unref(device, pipeline->shaders[s]);
 200    }
 201
 202    vk_free2(&device->alloc, pAllocator, pipeline);
 203 }
 204
 205 static const uint32_t vk_to_gen_primitive_type[] = {
 206    [VK_PRIMITIVE_TOPOLOGY_POINT_LIST]                    = _3DPRIM_POINTLIST,
 207    [VK_PRIMITIVE_TOPOLOGY_LINE_LIST]                     = _3DPRIM_LINELIST,
 208    [VK_PRIMITIVE_TOPOLOGY_LINE_STRIP]                    = _3DPRIM_LINESTRIP,
 209    [VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST]                 = _3DPRIM_TRILIST,
 210    [VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP]                = _3DPRIM_TRISTRIP,
 211    [VK_PRIMITIVE_TOPOLOGY_TRIANGLE_FAN]                  = _3DPRIM_TRIFAN,
 212    [VK_PRIMITIVE_TOPOLOGY_LINE_LIST_WITH_ADJACENCY]      = _3DPRIM_LINELIST_ADJ,
 213    [VK_PRIMITIVE_TOPOLOGY_LINE_STRIP_WITH_ADJACENCY]     = _3DPRIM_LINESTRIP_ADJ,
 214    [VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST_WITH_ADJACENCY]  = _3DPRIM_TRILIST_ADJ,
 215    [VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP_WITH_ADJACENCY] = _3DPRIM_TRISTRIP_ADJ,
 216 /*   [VK_PRIMITIVE_TOPOLOGY_PATCH_LIST]                = _3DPRIM_PATCHLIST_1 */
 217 };
 218
 219 static void
 220 populate_sampler_prog_key(const struct gen_device_info *devinfo,
 221                           struct brw_sampler_prog_key_data *key)
 222 {
 223    /* XXX: Handle texture swizzle on HSW- */
 224    for (int i = 0; i < MAX_SAMPLERS; i++) {
 225       /* Assume color sampler, no swizzling. (Works for BDW+) */
 226       key->swizzles[i] = SWIZZLE_XYZW;
 227    }
 228 }
 229
 230 static void
 231 populate_vs_prog_key(const struct gen_device_info *devinfo,
 232                      struct brw_vs_prog_key *key)
 233 {
 234    memset(key, 0, sizeof(*key));
 235
 236    populate_sampler_prog_key(devinfo, &key->tex);
 237
 238    /* XXX: Handle vertex input work-arounds */
 239
 240    /* XXX: Handle sampler_prog_key */
 241 }
 242
 243 static void
 244 populate_gs_prog_key(const struct gen_device_info *devinfo,
 245                      struct brw_gs_prog_key *key)
 246 {
 247    memset(key, 0, sizeof(*key));
 248
 249    populate_sampler_prog_key(devinfo, &key->tex);
 250 }
 251
 252 static void
 253 populate_wm_prog_key(const struct gen_device_info *devinfo,
 254                      const VkGraphicsPipelineCreateInfo *info,
 255                      struct brw_wm_prog_key *key)
 256 {
 257    ANV_FROM_HANDLE(anv_render_pass, render_pass, info->renderPass);
 258
 259    memset(key, 0, sizeof(*key));
 260
 261    populate_sampler_prog_key(devinfo, &key->tex);
 262
 263    /* TODO: Fill out key->input_slots_valid */
 264
 265    /* Vulkan doesn't specify a default */
 266    key->high_quality_derivatives = false;
 267
 268    /* XXX Vulkan doesn't appear to specify */
 269    key->clamp_fragment_color = false;
 270
 271    key->nr_color_regions =
 272       render_pass->subpasses[info->subpass].color_count;
 273
 274    key->replicate_alpha = key->nr_color_regions > 1 &&
 275                           info->pMultisampleState &&
 276                           info->pMultisampleState->alphaToCoverageEnable;
 277
 278    if (info->pMultisampleState && info->pMultisampleState->rasterizationSamples > 1) {
 279       /* We should probably pull this out of the shader, but it's fairly
 280        * harmless to compute it and then let dead-code take care of it.
 281        */
 282       key->persample_interp =
 283          (info->pMultisampleState->minSampleShading *
 284           info->pMultisampleState->rasterizationSamples) > 1;
 285       key->multisample_fbo = true;
 286    }
 287 }
 288
 289 static void
 290 populate_cs_prog_key(const struct gen_device_info *devinfo,
 291                      struct brw_cs_prog_key *key)
 292 {
 293    memset(key, 0, sizeof(*key));
 294
 295    populate_sampler_prog_key(devinfo, &key->tex);
 296 }
 297
 298 static nir_shader *
 299 anv_pipeline_compile(struct anv_pipeline *pipeline,
 300                      struct anv_shader_module *module,
 301                      const char *entrypoint,
 302                      gl_shader_stage stage,
 303                      const VkSpecializationInfo *spec_info,
 304                      struct brw_stage_prog_data *prog_data,
 305                      struct anv_pipeline_bind_map *map)
 306 {
 307    nir_shader *nir = anv_shader_compile_to_nir(pipeline->device,
 308                                                module, entrypoint, stage,
 309                                                spec_info);
 310    if (nir == NULL)
 311       return NULL;
 312
 313    anv_nir_lower_push_constants(nir);
 314
 315    /* Figure out the number of parameters */
 316    prog_data->nr_params = 0;
 317
 318    if (nir->num_uniforms > 0) {
 319       /* If the shader uses any push constants at all, we'll just give
 320        * them the maximum possible number
 321        */
 322       assert(nir->num_uniforms <= MAX_PUSH_CONSTANTS_SIZE);
 323       prog_data->nr_params += MAX_PUSH_CONSTANTS_SIZE / sizeof(float);
 324    }
 325
 326    if (pipeline->layout && pipeline->layout->stage[stage].has_dynamic_offsets)
 327       prog_data->nr_params += MAX_DYNAMIC_BUFFERS * 2;
 328
 329    if (nir->info->num_images > 0) {
 330       prog_data->nr_params += nir->info->num_images * BRW_IMAGE_PARAM_SIZE;
 331       pipeline->needs_data_cache = true;
 332    }
 333
 334    if (stage == MESA_SHADER_COMPUTE)
 335       ((struct brw_cs_prog_data *)prog_data)->thread_local_id_index =
 336          prog_data->nr_params++; /* The CS Thread ID uniform */
 337
 338    if (nir->info->num_ssbos > 0)
 339       pipeline->needs_data_cache = true;
 340
 341    if (prog_data->nr_params > 0) {
 342       /* XXX: I think we're leaking this */
 343       prog_data->param = (const union gl_constant_value **)
 344          malloc(prog_data->nr_params * sizeof(union gl_constant_value *));
 345
 346       /* We now set the param values to be offsets into a
 347        * anv_push_constant_data structure.  Since the compiler doesn't
 348        * actually dereference any of the gl_constant_value pointers in the
 349        * params array, it doesn't really matter what we put here.
 350        */
 351       struct anv_push_constants *null_data = NULL;
 352       if (nir->num_uniforms > 0) {
 353          /* Fill out the push constants section of the param array */
 354          for (unsigned i = 0; i < MAX_PUSH_CONSTANTS_SIZE / sizeof(float); i++)
 355             prog_data->param[i] = (const union gl_constant_value *)
 356                &null_data->client_data[i * sizeof(float)];
 357       }
 358    }
 359
 360    /* Set up dynamic offsets */
 361    anv_nir_apply_dynamic_offsets(pipeline, nir, prog_data);
 362
 363    /* Apply the actual pipeline layout to UBOs, SSBOs, and textures */
 364    if (pipeline->layout)
 365       anv_nir_apply_pipeline_layout(pipeline, nir, prog_data, map);
 366
 367    /* nir_lower_io will only handle the push constants; we need to set this
 368     * to the full number of possible uniforms.
 369     */
 370    nir->num_uniforms = prog_data->nr_params * 4;
 371
 372    return nir;
 373 }
 374
 375 static void
 376 anv_fill_binding_table(struct brw_stage_prog_data *prog_data, unsigned bias)
 377 {
 378    prog_data->binding_table.size_bytes = 0;
 379    prog_data->binding_table.texture_start = bias;
 380    prog_data->binding_table.gather_texture_start = bias;
 381    prog_data->binding_table.ubo_start = bias;
 382    prog_data->binding_table.ssbo_start = bias;
 383    prog_data->binding_table.image_start = bias;
 384 }
 385
 386 static struct anv_shader_bin *
 387 anv_pipeline_upload_kernel(struct anv_pipeline *pipeline,
 388                            struct anv_pipeline_cache *cache,
 389                            const void *key_data, uint32_t key_size,
 390                            const void *kernel_data, uint32_t kernel_size,
 391                            const void *prog_data, uint32_t prog_data_size,
 392                            const struct anv_pipeline_bind_map *bind_map)
 393 {
 394    if (cache) {
 395       return anv_pipeline_cache_upload_kernel(cache, key_data, key_size,
 396                                               kernel_data, kernel_size,
 397                                               prog_data, prog_data_size,
 398                                               bind_map);
 399    } else {
 400       return anv_shader_bin_create(pipeline->device, key_data, key_size,
 401                                    kernel_data, kernel_size,
 402                                    prog_data, prog_data_size, bind_map);
 403    }
 404 }
 405
 406
 407 static void
 408 anv_pipeline_add_compiled_stage(struct anv_pipeline *pipeline,
 409                                 gl_shader_stage stage,
 410                                 struct anv_shader_bin *shader)
 411 {
 412    pipeline->shaders[stage] = shader;
 413    pipeline->active_stages |= mesa_to_vk_shader_stage(stage);
 414 }
 415
 416 static VkResult
 417 anv_pipeline_compile_vs(struct anv_pipeline *pipeline,
 418                         struct anv_pipeline_cache *cache,
 419                         const VkGraphicsPipelineCreateInfo *info,
 420                         struct anv_shader_module *module,
 421                         const char *entrypoint,
 422                         const VkSpecializationInfo *spec_info)
 423 {
 424    const struct brw_compiler *compiler =
 425       pipeline->device->instance->physicalDevice.compiler;
 426    struct anv_pipeline_bind_map map;
 427    struct brw_vs_prog_key key;
 428    struct anv_shader_bin *bin = NULL;
 429    unsigned char sha1[20];
 430
 431    populate_vs_prog_key(&pipeline->device->info, &key);
 432
 433    if (cache) {
 434       anv_hash_shader(sha1, &key, sizeof(key), module, entrypoint,
 435                       pipeline->layout, spec_info);
 436       bin = anv_pipeline_cache_search(cache, sha1, 20);
 437    }
 438
 439    if (bin == NULL) {
 440       struct brw_vs_prog_data prog_data = { 0, };
 441       struct anv_pipeline_binding surface_to_descriptor[256];
 442       struct anv_pipeline_binding sampler_to_descriptor[256];
 443
 444       map = (struct anv_pipeline_bind_map) {
 445          .surface_to_descriptor = surface_to_descriptor,
 446          .sampler_to_descriptor = sampler_to_descriptor
 447       };
 448
 449       nir_shader *nir = anv_pipeline_compile(pipeline, module, entrypoint,
 450                                              MESA_SHADER_VERTEX, spec_info,
 451                                              &prog_data.base.base, &map);
 452       if (nir == NULL)
 453          return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
 454
 455       anv_fill_binding_table(&prog_data.base.base, 0);
 456
 457       void *mem_ctx = ralloc_context(NULL);
 458
 459       ralloc_steal(mem_ctx, nir);
 460
 461       prog_data.inputs_read = nir->info->inputs_read;
 462
 463       brw_compute_vue_map(&pipeline->device->info,
 464                           &prog_data.base.vue_map,
 465                           nir->info->outputs_written,
 466                           nir->info->separate_shader);
 467
 468       unsigned code_size;
 469       const unsigned *shader_code =
 470          brw_compile_vs(compiler, NULL, mem_ctx, &key, &prog_data, nir,
 471                         NULL, false, -1, &code_size, NULL);
 472       if (shader_code == NULL) {
 473          ralloc_free(mem_ctx);
 474          return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
 475       }
 476
 477       bin = anv_pipeline_upload_kernel(pipeline, cache, sha1, 20,
 478                                        shader_code, code_size,
 479                                        &prog_data, sizeof(prog_data), &map);
 480       if (!bin) {
 481          ralloc_free(mem_ctx);
 482          return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
 483       }
 484
 485       ralloc_free(mem_ctx);
 486    }
 487
 488    const struct brw_vs_prog_data *vs_prog_data =
 489       (const struct brw_vs_prog_data *)anv_shader_bin_get_prog_data(bin);
 490
 491    if (vs_prog_data->base.dispatch_mode == DISPATCH_MODE_SIMD8) {
 492       pipeline->vs_simd8 = bin->kernel.offset;
 493       pipeline->vs_vec4 = NO_KERNEL;
 494    } else {
 495       pipeline->vs_simd8 = NO_KERNEL;
 496       pipeline->vs_vec4 = bin->kernel.offset;
 497    }
 498
 499    anv_pipeline_add_compiled_stage(pipeline, MESA_SHADER_VERTEX, bin);
 500
 501    return VK_SUCCESS;
 502 }
 503
 504 static VkResult
 505 anv_pipeline_compile_gs(struct anv_pipeline *pipeline,
 506                         struct anv_pipeline_cache *cache,
 507                         const VkGraphicsPipelineCreateInfo *info,
 508                         struct anv_shader_module *module,
 509                         const char *entrypoint,
 510                         const VkSpecializationInfo *spec_info)
 511 {
 512    const struct brw_compiler *compiler =
 513       pipeline->device->instance->physicalDevice.compiler;
 514    struct anv_pipeline_bind_map map;
 515    struct brw_gs_prog_key key;
 516    struct anv_shader_bin *bin = NULL;
 517    unsigned char sha1[20];
 518
 519    populate_gs_prog_key(&pipeline->device->info, &key);
 520
 521    if (cache) {
 522       anv_hash_shader(sha1, &key, sizeof(key), module, entrypoint,
 523                       pipeline->layout, spec_info);
 524       bin = anv_pipeline_cache_search(cache, sha1, 20);
 525    }
 526
 527    if (bin == NULL) {
 528       struct brw_gs_prog_data prog_data = { 0, };
 529       struct anv_pipeline_binding surface_to_descriptor[256];
 530       struct anv_pipeline_binding sampler_to_descriptor[256];
 531
 532       map = (struct anv_pipeline_bind_map) {
 533          .surface_to_descriptor = surface_to_descriptor,
 534          .sampler_to_descriptor = sampler_to_descriptor
 535       };
 536
 537       nir_shader *nir = anv_pipeline_compile(pipeline, module, entrypoint,
 538                                              MESA_SHADER_GEOMETRY, spec_info,
 539                                              &prog_data.base.base, &map);
 540       if (nir == NULL)
 541          return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
 542
 543       anv_fill_binding_table(&prog_data.base.base, 0);
 544
 545       void *mem_ctx = ralloc_context(NULL);
 546
 547       ralloc_steal(mem_ctx, nir);
 548
 549       brw_compute_vue_map(&pipeline->device->info,
 550                           &prog_data.base.vue_map,
 551                           nir->info->outputs_written,
 552                           nir->info->separate_shader);
 553
 554       unsigned code_size;
 555       const unsigned *shader_code =
 556          brw_compile_gs(compiler, NULL, mem_ctx, &key, &prog_data, nir,
 557                         NULL, -1, &code_size, NULL);
 558       if (shader_code == NULL) {
 559          ralloc_free(mem_ctx);
 560          return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
 561       }
 562
 563       /* TODO: SIMD8 GS */
 564       bin = anv_pipeline_upload_kernel(pipeline, cache, sha1, 20,
 565                                        shader_code, code_size,
 566                                        &prog_data, sizeof(prog_data), &map);
 567       if (!bin) {
 568          ralloc_free(mem_ctx);
 569          return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
 570       }
 571
 572       ralloc_free(mem_ctx);
 573    }
 574
 575    pipeline->gs_kernel = bin->kernel.offset;
 576
 577    anv_pipeline_add_compiled_stage(pipeline, MESA_SHADER_GEOMETRY, bin);
 578
 579    return VK_SUCCESS;
 580 }
 581
 582 static VkResult
 583 anv_pipeline_compile_fs(struct anv_pipeline *pipeline,
 584                         struct anv_pipeline_cache *cache,
 585                         const VkGraphicsPipelineCreateInfo *info,
 586                         struct anv_shader_module *module,
 587                         const char *entrypoint,
 588                         const VkSpecializationInfo *spec_info)
 589 {
 590    const struct brw_compiler *compiler =
 591       pipeline->device->instance->physicalDevice.compiler;
 592    struct anv_pipeline_bind_map map;
 593    struct brw_wm_prog_key key;
 594    struct anv_shader_bin *bin = NULL;
 595    unsigned char sha1[20];
 596
 597    populate_wm_prog_key(&pipeline->device->info, info, &key);
 598
 599    if (cache) {
 600       anv_hash_shader(sha1, &key, sizeof(key), module, entrypoint,
 601                       pipeline->layout, spec_info);
 602       bin = anv_pipeline_cache_search(cache, sha1, 20);
 603    }
 604
 605    if (bin == NULL) {
 606       struct brw_wm_prog_data prog_data = { 0, };
 607       struct anv_pipeline_binding surface_to_descriptor[256];
 608       struct anv_pipeline_binding sampler_to_descriptor[256];
 609
 610       map = (struct anv_pipeline_bind_map) {
 611          .surface_to_descriptor = surface_to_descriptor + 8,
 612          .sampler_to_descriptor = sampler_to_descriptor
 613       };
 614
 615       nir_shader *nir = anv_pipeline_compile(pipeline, module, entrypoint,
 616                                              MESA_SHADER_FRAGMENT, spec_info,
 617                                              &prog_data.base, &map);
 618       if (nir == NULL)
 619          return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
 620
 621       unsigned num_rts = 0;
 622       struct anv_pipeline_binding rt_bindings[8];
 623       nir_function_impl *impl = nir_shader_get_entrypoint(nir);
 624       nir_foreach_variable_safe(var, &nir->outputs) {
 625          if (var->data.location < FRAG_RESULT_DATA0)
 626             continue;
 627
 628          unsigned rt = var->data.location - FRAG_RESULT_DATA0;
 629          if (rt >= key.nr_color_regions) {
 630             /* Out-of-bounds, throw it away */
 631             var->data.mode = nir_var_local;
 632             exec_node_remove(&var->node);
 633             exec_list_push_tail(&impl->locals, &var->node);
 634             continue;
 635          }
 636
 637          /* Give it a new, compacted, location */
 638          var->data.location = FRAG_RESULT_DATA0 + num_rts;
 639
 640          unsigned array_len =
 641             glsl_type_is_array(var->type) ? glsl_get_length(var->type) : 1;
 642          assert(num_rts + array_len <= 8);
 643
 644          for (unsigned i = 0; i < array_len; i++) {
 645             rt_bindings[num_rts + i] = (struct anv_pipeline_binding) {
 646                .set = ANV_DESCRIPTOR_SET_COLOR_ATTACHMENTS,
 647                .binding = 0,
 648                .index = rt + i,
 649             };
 650          }
 651
 652          num_rts += array_len;
 653       }
 654
 655       if (num_rts == 0) {
 656          /* If we have no render targets, we need a null render target */
 657          rt_bindings[0] = (struct anv_pipeline_binding) {
 658             .set = ANV_DESCRIPTOR_SET_COLOR_ATTACHMENTS,
 659             .binding = 0,
 660             .index = UINT8_MAX,
 661          };
 662          num_rts = 1;
 663       }
 664
 665       assert(num_rts <= 8);
 666       map.surface_to_descriptor -= num_rts;
 667       map.surface_count += num_rts;
 668       assert(map.surface_count <= 256);
 669       memcpy(map.surface_to_descriptor, rt_bindings,
 670              num_rts * sizeof(*rt_bindings));
 671
 672       anv_fill_binding_table(&prog_data.base, num_rts);
 673
 674       void *mem_ctx = ralloc_context(NULL);
 675
 676       ralloc_steal(mem_ctx, nir);
 677
 678       unsigned code_size;
 679       const unsigned *shader_code =
 680          brw_compile_fs(compiler, NULL, mem_ctx, &key, &prog_data, nir,
 681                         NULL, -1, -1, true, false, &code_size, NULL);
 682       if (shader_code == NULL) {
 683          ralloc_free(mem_ctx);
 684          return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
 685       }
 686
 687       bin = anv_pipeline_upload_kernel(pipeline, cache, sha1, 20,
 688                                        shader_code, code_size,
 689                                        &prog_data, sizeof(prog_data), &map);
 690       if (!bin) {
 691          ralloc_free(mem_ctx);
 692          return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
 693       }
 694
 695       ralloc_free(mem_ctx);
 696    }
 697
 698    pipeline->ps_ksp0 = bin->kernel.offset;
 699
 700    anv_pipeline_add_compiled_stage(pipeline, MESA_SHADER_FRAGMENT, bin);
 701
 702    return VK_SUCCESS;
 703 }
 704
 705 VkResult
 706 anv_pipeline_compile_cs(struct anv_pipeline *pipeline,
 707                         struct anv_pipeline_cache *cache,
 708                         const VkComputePipelineCreateInfo *info,
 709                         struct anv_shader_module *module,
 710                         const char *entrypoint,
 711                         const VkSpecializationInfo *spec_info)
 712 {
 713    const struct brw_compiler *compiler =
 714       pipeline->device->instance->physicalDevice.compiler;
 715    struct anv_pipeline_bind_map map;
 716    struct brw_cs_prog_key key;
 717    struct anv_shader_bin *bin = NULL;
 718    unsigned char sha1[20];
 719
 720    populate_cs_prog_key(&pipeline->device->info, &key);
 721
 722    if (cache) {
 723       anv_hash_shader(sha1, &key, sizeof(key), module, entrypoint,
 724                       pipeline->layout, spec_info);
 725       bin = anv_pipeline_cache_search(cache, sha1, 20);
 726    }
 727
 728    if (bin == NULL) {
 729       struct brw_cs_prog_data prog_data = { 0, };
 730       struct anv_pipeline_binding surface_to_descriptor[256];
 731       struct anv_pipeline_binding sampler_to_descriptor[256];
 732
 733       map = (struct anv_pipeline_bind_map) {
 734          .surface_to_descriptor = surface_to_descriptor,
 735          .sampler_to_descriptor = sampler_to_descriptor
 736       };
 737
 738       nir_shader *nir = anv_pipeline_compile(pipeline, module, entrypoint,
 739                                              MESA_SHADER_COMPUTE, spec_info,
 740                                              &prog_data.base, &map);
 741       if (nir == NULL)
 742          return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
 743
 744       anv_fill_binding_table(&prog_data.base, 1);
 745
 746       void *mem_ctx = ralloc_context(NULL);
 747
 748       ralloc_steal(mem_ctx, nir);
 749
 750       unsigned code_size;
 751       const unsigned *shader_code =
 752          brw_compile_cs(compiler, NULL, mem_ctx, &key, &prog_data, nir,
 753                         -1, &code_size, NULL);
 754       if (shader_code == NULL) {
 755          ralloc_free(mem_ctx);
 756          return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
 757       }
 758
 759       bin = anv_pipeline_upload_kernel(pipeline, cache, sha1, 20,
 760                                        shader_code, code_size,
 761                                        &prog_data, sizeof(prog_data), &map);
 762       if (!bin) {
 763          ralloc_free(mem_ctx);
 764          return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
 765       }
 766
 767       ralloc_free(mem_ctx);
 768    }
 769
 770    pipeline->cs_simd = bin->kernel.offset;
 771
 772    anv_pipeline_add_compiled_stage(pipeline, MESA_SHADER_COMPUTE, bin);
 773
 774    return VK_SUCCESS;
 775 }
 776
 777 /**
 778  * Copy pipeline state not marked as dynamic.
 779  * Dynamic state is pipeline state which hasn't been provided at pipeline
 780  * creation time, but is dynamically provided afterwards using various
 781  * vkCmdSet* functions.
 782  *
 783  * The set of state considered "non_dynamic" is determined by the pieces of
 784  * state that have their corresponding VkDynamicState enums omitted from
 785  * VkPipelineDynamicStateCreateInfo::pDynamicStates.
 786  *
 787  * @param[out] pipeline    Destination non_dynamic state.
 788  * @param[in]  pCreateInfo Source of non_dynamic state to be copied.
 789  */
 790 static void
 791 copy_non_dynamic_state(struct anv_pipeline *pipeline,
 792                        const VkGraphicsPipelineCreateInfo *pCreateInfo)
 793 {
 794    anv_cmd_dirty_mask_t states = ANV_CMD_DIRTY_DYNAMIC_ALL;
 795    ANV_FROM_HANDLE(anv_render_pass, pass, pCreateInfo->renderPass);
 796    struct anv_subpass *subpass = &pass->subpasses[pCreateInfo->subpass];
 797
 798    pipeline->dynamic_state = default_dynamic_state;
 799
 800    if (pCreateInfo->pDynamicState) {
 801       /* Remove all of the states that are marked as dynamic */
 802       uint32_t count = pCreateInfo->pDynamicState->dynamicStateCount;
 803       for (uint32_t s = 0; s < count; s++)
 804          states &= ~(1 << pCreateInfo->pDynamicState->pDynamicStates[s]);
 805    }
 806
 807    struct anv_dynamic_state *dynamic = &pipeline->dynamic_state;
 808
 809    /* Section 9.2 of the Vulkan 1.0.15 spec says:
 810     *
 811     *    pViewportState is [...] NULL if the pipeline
 812     *    has rasterization disabled.
 813     */
 814    if (!pCreateInfo->pRasterizationState->rasterizerDiscardEnable) {
 815       assert(pCreateInfo->pViewportState);
 816
 817       dynamic->viewport.count = pCreateInfo->pViewportState->viewportCount;
 818       if (states & (1 << VK_DYNAMIC_STATE_VIEWPORT)) {
 819          typed_memcpy(dynamic->viewport.viewports,
 820                      pCreateInfo->pViewportState->pViewports,
 821                      pCreateInfo->pViewportState->viewportCount);
 822       }
 823
 824       dynamic->scissor.count = pCreateInfo->pViewportState->scissorCount;
 825       if (states & (1 << VK_DYNAMIC_STATE_SCISSOR)) {
 826          typed_memcpy(dynamic->scissor.scissors,
 827                      pCreateInfo->pViewportState->pScissors,
 828                      pCreateInfo->pViewportState->scissorCount);
 829       }
 830    }
 831
 832    if (states & (1 << VK_DYNAMIC_STATE_LINE_WIDTH)) {
 833       assert(pCreateInfo->pRasterizationState);
 834       dynamic->line_width = pCreateInfo->pRasterizationState->lineWidth;
 835    }
 836
 837    if (states & (1 << VK_DYNAMIC_STATE_DEPTH_BIAS)) {
 838       assert(pCreateInfo->pRasterizationState);
 839       dynamic->depth_bias.bias =
 840          pCreateInfo->pRasterizationState->depthBiasConstantFactor;
 841       dynamic->depth_bias.clamp =
 842          pCreateInfo->pRasterizationState->depthBiasClamp;
 843       dynamic->depth_bias.slope =
 844          pCreateInfo->pRasterizationState->depthBiasSlopeFactor;
 845    }
 846
 847    /* Section 9.2 of the Vulkan 1.0.15 spec says:
 848     *
 849     *    pColorBlendState is [...] NULL if the pipeline has rasterization
 850     *    disabled or if the subpass of the render pass the pipeline is
 851     *    created against does not use any color attachments.
 852     */
 853    bool uses_color_att = false;
 854    for (unsigned i = 0; i < subpass->color_count; ++i) {
 855       if (subpass->color_attachments[i] != VK_ATTACHMENT_UNUSED) {
 856          uses_color_att = true;
 857          break;
 858       }
 859    }
 860
 861    if (uses_color_att &&
 862        !pCreateInfo->pRasterizationState->rasterizerDiscardEnable) {
 863       assert(pCreateInfo->pColorBlendState);
 864
 865       if (states & (1 << VK_DYNAMIC_STATE_BLEND_CONSTANTS))
 866          typed_memcpy(dynamic->blend_constants,
 867                      pCreateInfo->pColorBlendState->blendConstants, 4);
 868    }
 869
 870    /* If there is no depthstencil attachment, then don't read
 871     * pDepthStencilState. The Vulkan spec states that pDepthStencilState may
 872     * be NULL in this case. Even if pDepthStencilState is non-NULL, there is
 873     * no need to override the depthstencil defaults in
 874     * anv_pipeline::dynamic_state when there is no depthstencil attachment.
 875     *
 876     * Section 9.2 of the Vulkan 1.0.15 spec says:
 877     *
 878     *    pDepthStencilState is [...] NULL if the pipeline has rasterization
 879     *    disabled or if the subpass of the render pass the pipeline is created
 880     *    against does not use a depth/stencil attachment.
 881     */
 882    if (!pCreateInfo->pRasterizationState->rasterizerDiscardEnable &&
 883        subpass->depth_stencil_attachment != VK_ATTACHMENT_UNUSED) {
 884       assert(pCreateInfo->pDepthStencilState);
 885
 886       if (states & (1 << VK_DYNAMIC_STATE_DEPTH_BOUNDS)) {
 887          dynamic->depth_bounds.min =
 888             pCreateInfo->pDepthStencilState->minDepthBounds;
 889          dynamic->depth_bounds.max =
 890             pCreateInfo->pDepthStencilState->maxDepthBounds;
 891       }
 892
 893       if (states & (1 << VK_DYNAMIC_STATE_STENCIL_COMPARE_MASK)) {
 894          dynamic->stencil_compare_mask.front =
 895             pCreateInfo->pDepthStencilState->front.compareMask;
 896          dynamic->stencil_compare_mask.back =
 897             pCreateInfo->pDepthStencilState->back.compareMask;
 898       }
 899
 900       if (states & (1 << VK_DYNAMIC_STATE_STENCIL_WRITE_MASK)) {
 901          dynamic->stencil_write_mask.front =
 902             pCreateInfo->pDepthStencilState->front.writeMask;
 903          dynamic->stencil_write_mask.back =
 904             pCreateInfo->pDepthStencilState->back.writeMask;
 905       }
 906
 907       if (states & (1 << VK_DYNAMIC_STATE_STENCIL_REFERENCE)) {
 908          dynamic->stencil_reference.front =
 909             pCreateInfo->pDepthStencilState->front.reference;
 910          dynamic->stencil_reference.back =
 911             pCreateInfo->pDepthStencilState->back.reference;
 912       }
 913    }
 914
 915    pipeline->dynamic_state_mask = states;
 916 }
 917
 918 static void
 919 anv_pipeline_validate_create_info(const VkGraphicsPipelineCreateInfo *info)
 920 {
 921    struct anv_render_pass *renderpass = NULL;
 922    struct anv_subpass *subpass = NULL;
 923
 924    /* Assert that all required members of VkGraphicsPipelineCreateInfo are
 925     * present.  See the Vulkan 1.0.28 spec, Section 9.2 Graphics Pipelines.
 926     */
 927    assert(info->sType == VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO);
 928
 929    renderpass = anv_render_pass_from_handle(info->renderPass);
 930    assert(renderpass);
 931
 932    assert(info->subpass < renderpass->subpass_count);
 933    subpass = &renderpass->subpasses[info->subpass];
 934
 935    assert(info->stageCount >= 1);
 936    assert(info->pVertexInputState);
 937    assert(info->pInputAssemblyState);
 938    assert(info->pRasterizationState);
 939    if (!info->pRasterizationState->rasterizerDiscardEnable) {
 940       assert(info->pViewportState);
 941       assert(info->pMultisampleState);
 942
 943       if (subpass && subpass->depth_stencil_attachment != VK_ATTACHMENT_UNUSED)
 944          assert(info->pDepthStencilState);
 945
 946       if (subpass && subpass->color_count > 0)
 947          assert(info->pColorBlendState);
 948    }
 949
 950    for (uint32_t i = 0; i < info->stageCount; ++i) {
 951       switch (info->pStages[i].stage) {
 952       case VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT:
 953       case VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT:
 954          assert(info->pTessellationState);
 955          break;
 956       default:
 957          break;
 958       }
 959    }
 960 }
 961
 962 /**
 963  * Calculate the desired L3 partitioning based on the current state of the
 964  * pipeline.  For now this simply returns the conservative defaults calculated
 965  * by get_default_l3_weights(), but we could probably do better by gathering
 966  * more statistics from the pipeline state (e.g. guess of expected URB usage
 967  * and bound surfaces), or by using feed-back from performance counters.
 968  */
 969 void
 970 anv_pipeline_setup_l3_config(struct anv_pipeline *pipeline, bool needs_slm)
 971 {
 972    const struct gen_device_info *devinfo = &pipeline->device->info;
 973
 974    const struct gen_l3_weights w =
 975       gen_get_default_l3_weights(devinfo, pipeline->needs_data_cache, needs_slm);
 976
 977    pipeline->urb.l3_config = gen_get_l3_config(devinfo, w);
 978    pipeline->urb.total_size =
 979       gen_get_l3_config_urb_size(devinfo, pipeline->urb.l3_config);
 980 }
 981
 982 VkResult
 983 anv_pipeline_init(struct anv_pipeline *pipeline,
 984                   struct anv_device *device,
 985                   struct anv_pipeline_cache *cache,
 986                   const VkGraphicsPipelineCreateInfo *pCreateInfo,
 987                   const VkAllocationCallbacks *alloc)
 988 {
 989    VkResult result;
 990
 991    anv_validate {
 992       anv_pipeline_validate_create_info(pCreateInfo);
 993    }
 994
 995    if (alloc == NULL)
 996       alloc = &device->alloc;
 997
 998    pipeline->device = device;
 999    pipeline->layout = anv_pipeline_layout_from_handle(pCreateInfo->layout);
1000
1001    result = anv_reloc_list_init(&pipeline->batch_relocs, alloc);
1002    if (result != VK_SUCCESS)
1003       return result;
1004
1005    pipeline->batch.alloc = alloc;
1006    pipeline->batch.next = pipeline->batch.start = pipeline->batch_data;
1007    pipeline->batch.end = pipeline->batch.start + sizeof(pipeline->batch_data);
1008    pipeline->batch.relocs = &pipeline->batch_relocs;
1009
1010    copy_non_dynamic_state(pipeline, pCreateInfo);
1011    pipeline->depth_clamp_enable = pCreateInfo->pRasterizationState &&
1012                                   pCreateInfo->pRasterizationState->depthClampEnable;
1013
1014    pipeline->needs_data_cache = false;
1015
1016    /* When we free the pipeline, we detect stages based on the NULL status
1017     * of various prog_data pointers.  Make them NULL by default.
1018     */
1019    memset(pipeline->shaders, 0, sizeof(pipeline->shaders));
1020
1021    pipeline->vs_simd8 = NO_KERNEL;
1022    pipeline->vs_vec4 = NO_KERNEL;
1023    pipeline->gs_kernel = NO_KERNEL;
1024    pipeline->ps_ksp0 = NO_KERNEL;
1025
1026    pipeline->active_stages = 0;
1027
1028    const VkPipelineShaderStageCreateInfo *pStages[MESA_SHADER_STAGES] = { 0, };
1029    struct anv_shader_module *modules[MESA_SHADER_STAGES] = { 0, };
1030    for (uint32_t i = 0; i < pCreateInfo->stageCount; i++) {
1031       gl_shader_stage stage = ffs(pCreateInfo->pStages[i].stage) - 1;
1032       pStages[stage] = &pCreateInfo->pStages[i];
1033       modules[stage] = anv_shader_module_from_handle(pStages[stage]->module);
1034    }
1035
1036    if (modules[MESA_SHADER_VERTEX]) {
1037       result = anv_pipeline_compile_vs(pipeline, cache, pCreateInfo,
1038                                        modules[MESA_SHADER_VERTEX],
1039                                        pStages[MESA_SHADER_VERTEX]->pName,
1040                                        pStages[MESA_SHADER_VERTEX]->pSpecializationInfo);
1041       if (result != VK_SUCCESS)
1042          goto compile_fail;
1043    }
1044
1045    if (modules[MESA_SHADER_TESS_CTRL] || modules[MESA_SHADER_TESS_EVAL])
1046       anv_finishme("no tessellation support");
1047
1048    if (modules[MESA_SHADER_GEOMETRY]) {
1049       result = anv_pipeline_compile_gs(pipeline, cache, pCreateInfo,
1050                                        modules[MESA_SHADER_GEOMETRY],
1051                                        pStages[MESA_SHADER_GEOMETRY]->pName,
1052                                        pStages[MESA_SHADER_GEOMETRY]->pSpecializationInfo);
1053       if (result != VK_SUCCESS)
1054          goto compile_fail;
1055    }
1056
1057    if (modules[MESA_SHADER_FRAGMENT]) {
1058       result = anv_pipeline_compile_fs(pipeline, cache, pCreateInfo,
1059                                        modules[MESA_SHADER_FRAGMENT],
1060                                        pStages[MESA_SHADER_FRAGMENT]->pName,
1061                                        pStages[MESA_SHADER_FRAGMENT]->pSpecializationInfo);
1062       if (result != VK_SUCCESS)
1063          goto compile_fail;
1064    }
1065
1066    assert(pipeline->active_stages & VK_SHADER_STAGE_VERTEX_BIT);
1067
1068    anv_pipeline_setup_l3_config(pipeline, false);
1069
1070    const VkPipelineVertexInputStateCreateInfo *vi_info =
1071       pCreateInfo->pVertexInputState;
1072
1073    const uint64_t inputs_read = get_vs_prog_data(pipeline)->inputs_read;
1074
1075    pipeline->vb_used = 0;
1076    for (uint32_t i = 0; i < vi_info->vertexAttributeDescriptionCount; i++) {
1077       const VkVertexInputAttributeDescription *desc =
1078          &vi_info->pVertexAttributeDescriptions[i];
1079
1080       if (inputs_read & (1 << (VERT_ATTRIB_GENERIC0 + desc->location)))
1081          pipeline->vb_used |= 1 << desc->binding;
1082    }
1083
1084    for (uint32_t i = 0; i < vi_info->vertexBindingDescriptionCount; i++) {
1085       const VkVertexInputBindingDescription *desc =
1086          &vi_info->pVertexBindingDescriptions[i];
1087
1088       pipeline->binding_stride[desc->binding] = desc->stride;
1089
1090       /* Step rate is programmed per vertex element (attribute), not
1091        * binding. Set up a map of which bindings step per instance, for
1092        * reference by vertex element setup. */
1093       switch (desc->inputRate) {
1094       default:
1095       case VK_VERTEX_INPUT_RATE_VERTEX:
1096          pipeline->instancing_enable[desc->binding] = false;
1097          break;
1098       case VK_VERTEX_INPUT_RATE_INSTANCE:
1099          pipeline->instancing_enable[desc->binding] = true;
1100          break;
1101       }
1102    }
1103
1104    const VkPipelineInputAssemblyStateCreateInfo *ia_info =
1105       pCreateInfo->pInputAssemblyState;
1106    pipeline->primitive_restart = ia_info->primitiveRestartEnable;
1107    pipeline->topology = vk_to_gen_primitive_type[ia_info->topology];
1108
1109    return VK_SUCCESS;
1110
1111 compile_fail:
1112    for (unsigned s = 0; s < MESA_SHADER_STAGES; s++) {
1113       if (pipeline->shaders[s])
1114          anv_shader_bin_unref(device, pipeline->shaders[s]);
1115    }
1116
1117    anv_reloc_list_finish(&pipeline->batch_relocs, alloc);
1118
1119    return result;
1120 }