src/gallium/drivers/ilo/ilo_shader.c

   1 /*
   2  * Mesa 3-D graphics library
   3  *
   4  * Copyright (C) 2012-2013 LunarG, Inc.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included
  14  * in all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  21  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  22  * DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors:
  25  *    Chia-I Wu <olv@lunarg.com>
  26  */
  27
  28 #include "tgsi/tgsi_parse.h"
  29 #include "intel_winsys.h"
  30
  31 #include "shader/ilo_shader_internal.h"
  32 #include "ilo_state.h"
  33 #include "ilo_shader.h"
  34
  35 struct ilo_shader_cache {
  36    struct list_head shaders;
  37    struct list_head changed;
  38 };
  39
  40 /**
  41  * Create a shader cache.  A shader cache can manage shaders and upload them
  42  * to a bo as a whole.
  43  */
  44 struct ilo_shader_cache *
  45 ilo_shader_cache_create(void)
  46 {
  47    struct ilo_shader_cache *shc;
  48
  49    shc = CALLOC_STRUCT(ilo_shader_cache);
  50    if (!shc)
  51       return NULL;
  52
  53    list_inithead(&shc->shaders);
  54    list_inithead(&shc->changed);
  55
  56    return shc;
  57 }
  58
  59 /**
  60  * Destroy a shader cache.
  61  */
  62 void
  63 ilo_shader_cache_destroy(struct ilo_shader_cache *shc)
  64 {
  65    FREE(shc);
  66 }
  67
  68 /**
  69  * Add a shader to the cache.
  70  */
  71 void
  72 ilo_shader_cache_add(struct ilo_shader_cache *shc,
  73                      struct ilo_shader_state *shader)
  74 {
  75    struct ilo_shader *sh;
  76
  77    shader->cache = shc;
  78    LIST_FOR_EACH_ENTRY(sh, &shader->variants, list)
  79       sh->uploaded = false;
  80
  81    list_add(&shader->list, &shc->changed);
  82 }
  83
  84 /**
  85  * Remove a shader from the cache.
  86  */
  87 void
  88 ilo_shader_cache_remove(struct ilo_shader_cache *shc,
  89                         struct ilo_shader_state *shader)
  90 {
  91    list_del(&shader->list);
  92    shader->cache = NULL;
  93 }
  94
  95 /**
  96  * Notify the cache that a managed shader has changed.
  97  */
  98 static void
  99 ilo_shader_cache_notify_change(struct ilo_shader_cache *shc,
 100                                struct ilo_shader_state *shader)
 101 {
 102    if (shader->cache == shc) {
 103       list_del(&shader->list);
 104       list_add(&shader->list, &shc->changed);
 105    }
 106 }
 107
 108 /**
 109  * Upload a managed shader to the bo.
 110  */
 111 static int
 112 ilo_shader_cache_upload_shader(struct ilo_shader_cache *shc,
 113                                struct ilo_shader_state *shader,
 114                                struct intel_bo *bo, unsigned offset,
 115                                bool incremental)
 116 {
 117    const unsigned base = offset;
 118    struct ilo_shader *sh;
 119
 120    LIST_FOR_EACH_ENTRY(sh, &shader->variants, list) {
 121       int err;
 122
 123       if (incremental && sh->uploaded)
 124          continue;
 125
 126       /* kernels must be aligned to 64-byte */
 127       offset = align(offset, 64);
 128
 129       err = intel_bo_pwrite(bo, offset, sh->kernel_size, sh->kernel);
 130       if (unlikely(err))
 131          return -1;
 132
 133       sh->uploaded = true;
 134       sh->cache_offset = offset;
 135
 136       offset += sh->kernel_size;
 137    }
 138
 139    return (int) (offset - base);
 140 }
 141
 142 /**
 143  * Similar to ilo_shader_cache_upload(), except no upload happens.
 144  */
 145 static int
 146 ilo_shader_cache_get_upload_size(struct ilo_shader_cache *shc,
 147                                  unsigned offset,
 148                                  bool incremental)
 149 {
 150    const unsigned base = offset;
 151    struct ilo_shader_state *shader;
 152
 153    if (!incremental) {
 154       LIST_FOR_EACH_ENTRY(shader, &shc->shaders, list) {
 155          struct ilo_shader *sh;
 156
 157          /* see ilo_shader_cache_upload_shader() */
 158          LIST_FOR_EACH_ENTRY(sh, &shader->variants, list) {
 159             if (!incremental || !sh->uploaded)
 160                offset = align(offset, 64) + sh->kernel_size;
 161          }
 162       }
 163    }
 164
 165    LIST_FOR_EACH_ENTRY(shader, &shc->changed, list) {
 166       struct ilo_shader *sh;
 167
 168       /* see ilo_shader_cache_upload_shader() */
 169       LIST_FOR_EACH_ENTRY(sh, &shader->variants, list) {
 170          if (!incremental || !sh->uploaded)
 171             offset = align(offset, 64) + sh->kernel_size;
 172       }
 173    }
 174
 175    /*
 176     * From the Sandy Bridge PRM, volume 4 part 2, page 112:
 177     *
 178     *     "Due to prefetch of the instruction stream, the EUs may attempt to
 179     *      access up to 8 instructions (128 bytes) beyond the end of the
 180     *      kernel program - possibly into the next memory page.  Although
 181     *      these instructions will not be executed, software must account for
 182     *      the prefetch in order to avoid invalid page access faults."
 183     */
 184    if (offset > base)
 185       offset += 128;
 186
 187    return (int) (offset - base);
 188 }
 189
 190 /**
 191  * Upload managed shaders to the bo.  When incremental is true, only shaders
 192  * that are changed or added after the last upload are uploaded.
 193  */
 194 int
 195 ilo_shader_cache_upload(struct ilo_shader_cache *shc,
 196                         struct intel_bo *bo, unsigned offset,
 197                         bool incremental)
 198 {
 199    struct ilo_shader_state *shader, *next;
 200    int size = 0, s;
 201
 202    if (!bo)
 203       return ilo_shader_cache_get_upload_size(shc, offset, incremental);
 204
 205    if (!incremental) {
 206       LIST_FOR_EACH_ENTRY(shader, &shc->shaders, list) {
 207          s = ilo_shader_cache_upload_shader(shc, shader,
 208                bo, offset, incremental);
 209          if (unlikely(s < 0))
 210             return s;
 211
 212          size += s;
 213          offset += s;
 214       }
 215    }
 216
 217    LIST_FOR_EACH_ENTRY_SAFE(shader, next, &shc->changed, list) {
 218       s = ilo_shader_cache_upload_shader(shc, shader,
 219             bo, offset, incremental);
 220       if (unlikely(s < 0))
 221          return s;
 222
 223       size += s;
 224       offset += s;
 225
 226       list_del(&shader->list);
 227       list_add(&shader->list, &shc->shaders);
 228    }
 229
 230    return size;
 231 }
 232
 233 /**
 234  * Initialize a shader variant.
 235  */
 236 void
 237 ilo_shader_variant_init(struct ilo_shader_variant *variant,
 238                         const struct ilo_shader_info *info,
 239                         const struct ilo_context *ilo)
 240 {
 241    int num_views, i;
 242
 243    memset(variant, 0, sizeof(*variant));
 244
 245    switch (info->type) {
 246    case PIPE_SHADER_VERTEX:
 247       variant->u.vs.rasterizer_discard =
 248          ilo->rasterizer->state.rasterizer_discard;
 249       variant->u.vs.num_ucps =
 250          util_last_bit(ilo->rasterizer->state.clip_plane_enable);
 251       break;
 252    case PIPE_SHADER_GEOMETRY:
 253       variant->u.gs.rasterizer_discard =
 254          ilo->rasterizer->state.rasterizer_discard;
 255       variant->u.gs.num_inputs = ilo->vs->shader->out.count;
 256       for (i = 0; i < ilo->vs->shader->out.count; i++) {
 257          variant->u.gs.semantic_names[i] =
 258             ilo->vs->shader->out.semantic_names[i];
 259          variant->u.gs.semantic_indices[i] =
 260             ilo->vs->shader->out.semantic_indices[i];
 261       }
 262       break;
 263    case PIPE_SHADER_FRAGMENT:
 264       variant->u.fs.flatshade =
 265          (info->has_color_interp && ilo->rasterizer->state.flatshade);
 266       variant->u.fs.fb_height = (info->has_pos) ?
 267          ilo->fb.state.height : 1;
 268       variant->u.fs.num_cbufs = ilo->fb.state.nr_cbufs;
 269       break;
 270    default:
 271       assert(!"unknown shader type");
 272       break;
 273    }
 274
 275    num_views = ilo->view[info->type].count;
 276    assert(info->num_samplers <= num_views);
 277
 278    variant->num_sampler_views = info->num_samplers;
 279    for (i = 0; i < info->num_samplers; i++) {
 280       const struct pipe_sampler_view *view = ilo->view[info->type].states[i];
 281       const struct ilo_sampler_cso *sampler = ilo->sampler[info->type].cso[i];
 282
 283       if (view) {
 284          variant->sampler_view_swizzles[i].r = view->swizzle_r;
 285          variant->sampler_view_swizzles[i].g = view->swizzle_g;
 286          variant->sampler_view_swizzles[i].b = view->swizzle_b;
 287          variant->sampler_view_swizzles[i].a = view->swizzle_a;
 288       }
 289       else if (info->shadow_samplers & (1 << i)) {
 290          variant->sampler_view_swizzles[i].r = PIPE_SWIZZLE_RED;
 291          variant->sampler_view_swizzles[i].g = PIPE_SWIZZLE_RED;
 292          variant->sampler_view_swizzles[i].b = PIPE_SWIZZLE_RED;
 293          variant->sampler_view_swizzles[i].a = PIPE_SWIZZLE_ONE;
 294       }
 295       else {
 296          variant->sampler_view_swizzles[i].r = PIPE_SWIZZLE_RED;
 297          variant->sampler_view_swizzles[i].g = PIPE_SWIZZLE_GREEN;
 298          variant->sampler_view_swizzles[i].b = PIPE_SWIZZLE_BLUE;
 299          variant->sampler_view_swizzles[i].a = PIPE_SWIZZLE_ALPHA;
 300       }
 301
 302       /*
 303        * When non-nearest filter and PIPE_TEX_WRAP_CLAMP wrap mode is used,
 304        * the HW wrap mode is set to BRW_TEXCOORDMODE_CLAMP_BORDER, and we need
 305        * to manually saturate the texture coordinates.
 306        */
 307       if (sampler) {
 308          variant->saturate_tex_coords[0] |= sampler->saturate_s << i;
 309          variant->saturate_tex_coords[1] |= sampler->saturate_t << i;
 310          variant->saturate_tex_coords[2] |= sampler->saturate_r << i;
 311       }
 312    }
 313 }
 314
 315 /**
 316  * Guess the shader variant, knowing that the context may still change.
 317  */
 318 static void
 319 ilo_shader_variant_guess(struct ilo_shader_variant *variant,
 320                          const struct ilo_shader_info *info,
 321                          const struct ilo_context *ilo)
 322 {
 323    int i;
 324
 325    memset(variant, 0, sizeof(*variant));
 326
 327    switch (info->type) {
 328    case PIPE_SHADER_VERTEX:
 329       break;
 330    case PIPE_SHADER_GEOMETRY:
 331       break;
 332    case PIPE_SHADER_FRAGMENT:
 333       variant->u.fs.flatshade = false;
 334       variant->u.fs.fb_height = (info->has_pos) ?
 335          ilo->fb.state.height : 1;
 336       variant->u.fs.num_cbufs = 1;
 337       break;
 338    default:
 339       assert(!"unknown shader type");
 340       break;
 341    }
 342
 343    variant->num_sampler_views = info->num_samplers;
 344    for (i = 0; i < info->num_samplers; i++) {
 345       if (info->shadow_samplers & (1 << i)) {
 346          variant->sampler_view_swizzles[i].r = PIPE_SWIZZLE_RED;
 347          variant->sampler_view_swizzles[i].g = PIPE_SWIZZLE_RED;
 348          variant->sampler_view_swizzles[i].b = PIPE_SWIZZLE_RED;
 349          variant->sampler_view_swizzles[i].a = PIPE_SWIZZLE_ONE;
 350       }
 351       else {
 352          variant->sampler_view_swizzles[i].r = PIPE_SWIZZLE_RED;
 353          variant->sampler_view_swizzles[i].g = PIPE_SWIZZLE_GREEN;
 354          variant->sampler_view_swizzles[i].b = PIPE_SWIZZLE_BLUE;
 355          variant->sampler_view_swizzles[i].a = PIPE_SWIZZLE_ALPHA;
 356       }
 357    }
 358 }
 359
 360
 361 /**
 362  * Parse a TGSI instruction for the shader info.
 363  */
 364 static void
 365 ilo_shader_info_parse_inst(struct ilo_shader_info *info,
 366                            const struct tgsi_full_instruction *inst)
 367 {
 368    int i;
 369
 370    /* look for edgeflag passthrough */
 371    if (info->edgeflag_out >= 0 &&
 372        inst->Instruction.Opcode == TGSI_OPCODE_MOV &&
 373        inst->Dst[0].Register.File == TGSI_FILE_OUTPUT &&
 374        inst->Dst[0].Register.Index == info->edgeflag_out) {
 375
 376       assert(inst->Src[0].Register.File == TGSI_FILE_INPUT);
 377       info->edgeflag_in = inst->Src[0].Register.Index;
 378    }
 379
 380    if (inst->Instruction.Texture) {
 381       bool shadow;
 382
 383       switch (inst->Texture.Texture) {
 384       case TGSI_TEXTURE_SHADOW1D:
 385       case TGSI_TEXTURE_SHADOW2D:
 386       case TGSI_TEXTURE_SHADOWRECT:
 387       case TGSI_TEXTURE_SHADOW1D_ARRAY:
 388       case TGSI_TEXTURE_SHADOW2D_ARRAY:
 389       case TGSI_TEXTURE_SHADOWCUBE:
 390       case TGSI_TEXTURE_SHADOWCUBE_ARRAY:
 391          shadow = true;
 392          break;
 393       default:
 394          shadow = false;
 395          break;
 396       }
 397
 398       for (i = 0; i < inst->Instruction.NumSrcRegs; i++) {
 399          const struct tgsi_full_src_register *src = &inst->Src[i];
 400
 401          if (src->Register.File == TGSI_FILE_SAMPLER) {
 402             const int idx = src->Register.Index;
 403
 404             if (idx >= info->num_samplers)
 405                info->num_samplers = idx + 1;
 406
 407             if (shadow)
 408                info->shadow_samplers |= 1 << idx;
 409          }
 410       }
 411    }
 412 }
 413
 414 /**
 415  * Parse a TGSI property for the shader info.
 416  */
 417 static void
 418 ilo_shader_info_parse_prop(struct ilo_shader_info *info,
 419                            const struct tgsi_full_property *prop)
 420 {
 421    switch (prop->Property.PropertyName) {
 422    case TGSI_PROPERTY_FS_COLOR0_WRITES_ALL_CBUFS:
 423       info->fs_color0_writes_all_cbufs = prop->u[0].Data;
 424       break;
 425    default:
 426       break;
 427    }
 428 }
 429
 430 /**
 431  * Parse a TGSI declaration for the shader info.
 432  */
 433 static void
 434 ilo_shader_info_parse_decl(struct ilo_shader_info *info,
 435                            const struct tgsi_full_declaration *decl)
 436 {
 437    switch (decl->Declaration.File) {
 438    case TGSI_FILE_INPUT:
 439       if (decl->Declaration.Interpolate &&
 440           decl->Interp.Interpolate == TGSI_INTERPOLATE_COLOR)
 441          info->has_color_interp = true;
 442       if (decl->Declaration.Semantic &&
 443           decl->Semantic.Name == TGSI_SEMANTIC_POSITION)
 444          info->has_pos = true;
 445       break;
 446    case TGSI_FILE_OUTPUT:
 447       if (decl->Declaration.Semantic &&
 448           decl->Semantic.Name == TGSI_SEMANTIC_EDGEFLAG)
 449          info->edgeflag_out = decl->Range.First;
 450       break;
 451    case TGSI_FILE_SYSTEM_VALUE:
 452       if (decl->Declaration.Semantic &&
 453           decl->Semantic.Name == TGSI_SEMANTIC_INSTANCEID)
 454          info->has_instanceid = true;
 455       if (decl->Declaration.Semantic &&
 456           decl->Semantic.Name == TGSI_SEMANTIC_VERTEXID)
 457          info->has_vertexid = true;
 458       break;
 459    default:
 460       break;
 461    }
 462 }
 463
 464 static void
 465 ilo_shader_info_parse_tokens(struct ilo_shader_info *info)
 466 {
 467    struct tgsi_parse_context parse;
 468
 469    info->edgeflag_in = -1;
 470    info->edgeflag_out = -1;
 471
 472    tgsi_parse_init(&parse, info->tokens);
 473    while (!tgsi_parse_end_of_tokens(&parse)) {
 474       const union tgsi_full_token *token;
 475
 476       tgsi_parse_token(&parse);
 477       token = &parse.FullToken;
 478
 479       switch (token->Token.Type) {
 480       case TGSI_TOKEN_TYPE_DECLARATION:
 481          ilo_shader_info_parse_decl(info, &token->FullDeclaration);
 482          break;
 483       case TGSI_TOKEN_TYPE_INSTRUCTION:
 484          ilo_shader_info_parse_inst(info, &token->FullInstruction);
 485          break;
 486       case TGSI_TOKEN_TYPE_PROPERTY:
 487          ilo_shader_info_parse_prop(info, &token->FullProperty);
 488          break;
 489       default:
 490          break;
 491       }
 492    }
 493    tgsi_parse_free(&parse);
 494 }
 495
 496 /**
 497  * Create a shader state.
 498  */
 499 static struct ilo_shader_state *
 500 ilo_shader_state_create(const struct ilo_context *ilo,
 501                         int type, const void *templ)
 502 {
 503    struct ilo_shader_state *state;
 504    struct ilo_shader_variant variant;
 505
 506    state = CALLOC_STRUCT(ilo_shader_state);
 507    if (!state)
 508       return NULL;
 509
 510    state->info.dev = ilo->dev;
 511    state->info.type = type;
 512
 513    if (type == PIPE_SHADER_COMPUTE) {
 514       const struct pipe_compute_state *c =
 515          (const struct pipe_compute_state *) templ;
 516
 517       state->info.tokens = tgsi_dup_tokens(c->prog);
 518       state->info.compute.req_local_mem = c->req_local_mem;
 519       state->info.compute.req_private_mem = c->req_private_mem;
 520       state->info.compute.req_input_mem = c->req_input_mem;
 521    }
 522    else {
 523       const struct pipe_shader_state *s =
 524          (const struct pipe_shader_state *) templ;
 525
 526       state->info.tokens = tgsi_dup_tokens(s->tokens);
 527       state->info.stream_output = s->stream_output;
 528    }
 529
 530    list_inithead(&state->variants);
 531
 532    ilo_shader_info_parse_tokens(&state->info);
 533
 534    /* guess and compile now */
 535    ilo_shader_variant_guess(&variant, &state->info, ilo);
 536    if (!ilo_shader_state_use_variant(state, &variant)) {
 537       ilo_shader_destroy(state);
 538       return NULL;
 539    }
 540
 541    return state;
 542 }
 543
 544 /**
 545  * Add a compiled shader to the shader state.
 546  */
 547 static void
 548 ilo_shader_state_add_shader(struct ilo_shader_state *state,
 549                             struct ilo_shader *sh)
 550 {
 551    list_add(&sh->list, &state->variants);
 552    state->num_variants++;
 553    state->total_size += sh->kernel_size;
 554
 555    if (state->cache)
 556       ilo_shader_cache_notify_change(state->cache, state);
 557 }
 558
 559 /**
 560  * Remove a compiled shader from the shader state.
 561  */
 562 static void
 563 ilo_shader_state_remove_shader(struct ilo_shader_state *state,
 564                                struct ilo_shader *sh)
 565 {
 566    list_del(&sh->list);
 567    state->num_variants--;
 568    state->total_size -= sh->kernel_size;
 569 }
 570
 571 /**
 572  * Garbage collect shader variants in the shader state.
 573  */
 574 static void
 575 ilo_shader_state_gc(struct ilo_shader_state *state)
 576 {
 577    /* activate when the variants take up more than 4KiB of space */
 578    const int limit = 4 * 1024;
 579    struct ilo_shader *sh, *next;
 580
 581    if (state->total_size < limit)
 582       return;
 583
 584    /* remove from the tail as the most recently ones are at the head */
 585    LIST_FOR_EACH_ENTRY_SAFE_REV(sh, next, &state->variants, list) {
 586       ilo_shader_state_remove_shader(state, sh);
 587       ilo_shader_destroy_kernel(sh);
 588
 589       if (state->total_size <= limit / 2)
 590          break;
 591    }
 592 }
 593
 594 /**
 595  * Search for a shader variant.
 596  */
 597 static struct ilo_shader *
 598 ilo_shader_state_search_variant(struct ilo_shader_state *state,
 599                                 const struct ilo_shader_variant *variant)
 600 {
 601    struct ilo_shader *sh = NULL, *tmp;
 602
 603    LIST_FOR_EACH_ENTRY(tmp, &state->variants, list) {
 604       if (memcmp(&tmp->variant, variant, sizeof(*variant)) == 0) {
 605          sh = tmp;
 606          break;
 607       }
 608    }
 609
 610    return sh;
 611 }
 612
 613 static void
 614 copy_so_info(struct ilo_shader *sh,
 615              const struct pipe_stream_output_info *so_info)
 616 {
 617    unsigned i, attr;
 618
 619    if (!so_info->num_outputs)
 620       return;
 621
 622    sh->so_info = *so_info;
 623
 624    for (i = 0; i < so_info->num_outputs; i++) {
 625       /* figure out which attribute is sourced */
 626       for (attr = 0; attr < sh->out.count; attr++) {
 627          const int reg_idx = sh->out.register_indices[attr];
 628          if (reg_idx == so_info->output[i].register_index)
 629             break;
 630       }
 631
 632       if (attr < sh->out.count) {
 633          sh->so_info.output[i].register_index = attr;
 634       }
 635       else {
 636          assert(!"stream output an undefined register");
 637          sh->so_info.output[i].register_index = 0;
 638       }
 639
 640       /* PSIZE is at W channel */
 641       if (sh->out.semantic_names[attr] == TGSI_SEMANTIC_PSIZE) {
 642          assert(so_info->output[i].start_component == 0);
 643          assert(so_info->output[i].num_components == 1);
 644          sh->so_info.output[i].start_component = 3;
 645       }
 646    }
 647 }
 648
 649 /**
 650  * Add a shader variant to the shader state.
 651  */
 652 static struct ilo_shader *
 653 ilo_shader_state_add_variant(struct ilo_shader_state *state,
 654                              const struct ilo_shader_variant *variant)
 655 {
 656    struct ilo_shader *sh;
 657
 658    switch (state->info.type) {
 659    case PIPE_SHADER_VERTEX:
 660       sh = ilo_shader_compile_vs(state, variant);
 661       break;
 662    case PIPE_SHADER_FRAGMENT:
 663       sh = ilo_shader_compile_fs(state, variant);
 664       break;
 665    case PIPE_SHADER_GEOMETRY:
 666       sh = ilo_shader_compile_gs(state, variant);
 667       break;
 668    case PIPE_SHADER_COMPUTE:
 669       sh = ilo_shader_compile_cs(state, variant);
 670       break;
 671    default:
 672       sh = NULL;
 673       break;
 674    }
 675    if (!sh) {
 676       assert(!"failed to compile shader");
 677       return NULL;
 678    }
 679
 680    sh->variant = *variant;
 681
 682    copy_so_info(sh, &state->info.stream_output);
 683
 684    ilo_shader_state_add_shader(state, sh);
 685
 686    return sh;
 687 }
 688
 689 /**
 690  * Update state->shader to point to a variant.  If the variant does not exist,
 691  * it will be added first.
 692  */
 693 bool
 694 ilo_shader_state_use_variant(struct ilo_shader_state *state,
 695                              const struct ilo_shader_variant *variant)
 696 {
 697    struct ilo_shader *sh;
 698    bool construct_cso = false;
 699
 700    sh = ilo_shader_state_search_variant(state, variant);
 701    if (!sh) {
 702       ilo_shader_state_gc(state);
 703
 704       sh = ilo_shader_state_add_variant(state, variant);
 705       if (!sh)
 706          return false;
 707
 708       construct_cso = true;
 709    }
 710
 711    /* move to head */
 712    if (state->variants.next != &sh->list) {
 713       list_del(&sh->list);
 714       list_add(&sh->list, &state->variants);
 715    }
 716
 717    state->shader = sh;
 718
 719    if (construct_cso) {
 720       switch (state->info.type) {
 721       case PIPE_SHADER_VERTEX:
 722          ilo_gpe_init_vs_cso(state->info.dev, state, &sh->cso);
 723          break;
 724       case PIPE_SHADER_GEOMETRY:
 725          ilo_gpe_init_gs_cso(state->info.dev, state, &sh->cso);
 726          break;
 727       case PIPE_SHADER_FRAGMENT:
 728          ilo_gpe_init_fs_cso(state->info.dev, state, &sh->cso);
 729          break;
 730       default:
 731          break;
 732       }
 733    }
 734
 735    return true;
 736 }
 737
 738 struct ilo_shader_state *
 739 ilo_shader_create_vs(const struct ilo_dev_info *dev,
 740                      const struct pipe_shader_state *state,
 741                      const struct ilo_context *precompile)
 742 {
 743    struct ilo_shader_state *shader;
 744
 745    shader = ilo_shader_state_create(precompile, PIPE_SHADER_VERTEX, state);
 746
 747    /* states used in ilo_shader_variant_init() */
 748    shader->info.non_orthogonal_states = ILO_DIRTY_VERTEX_SAMPLER_VIEWS |
 749                                         ILO_DIRTY_RASTERIZER;
 750
 751    return shader;
 752 }
 753
 754 struct ilo_shader_state *
 755 ilo_shader_create_gs(const struct ilo_dev_info *dev,
 756                      const struct pipe_shader_state *state,
 757                      const struct ilo_context *precompile)
 758 {
 759    struct ilo_shader_state *shader;
 760
 761    shader = ilo_shader_state_create(precompile, PIPE_SHADER_GEOMETRY, state);
 762
 763    /* states used in ilo_shader_variant_init() */
 764    shader->info.non_orthogonal_states = ILO_DIRTY_GEOMETRY_SAMPLER_VIEWS |
 765                                         ILO_DIRTY_VS |
 766                                         ILO_DIRTY_RASTERIZER;
 767
 768    return shader;
 769 }
 770
 771 struct ilo_shader_state *
 772 ilo_shader_create_fs(const struct ilo_dev_info *dev,
 773                      const struct pipe_shader_state *state,
 774                      const struct ilo_context *precompile)
 775 {
 776    struct ilo_shader_state *shader;
 777
 778    shader = ilo_shader_state_create(precompile, PIPE_SHADER_FRAGMENT, state);
 779
 780    /* states used in ilo_shader_variant_init() */
 781    shader->info.non_orthogonal_states = ILO_DIRTY_FRAGMENT_SAMPLER_VIEWS |
 782                                         ILO_DIRTY_RASTERIZER |
 783                                         ILO_DIRTY_FRAMEBUFFER;
 784
 785    return shader;
 786 }
 787
 788 struct ilo_shader_state *
 789 ilo_shader_create_cs(const struct ilo_dev_info *dev,
 790                      const struct pipe_compute_state *state,
 791                      const struct ilo_context *precompile)
 792 {
 793    struct ilo_shader_state *shader;
 794
 795    shader = ilo_shader_state_create(precompile, PIPE_SHADER_COMPUTE, state);
 796
 797    shader->info.non_orthogonal_states = 0;
 798
 799    return shader;
 800 }
 801
 802 /**
 803  * Destroy a shader state.
 804  */
 805 void
 806 ilo_shader_destroy(struct ilo_shader_state *shader)
 807 {
 808    struct ilo_shader *sh, *next;
 809
 810    LIST_FOR_EACH_ENTRY_SAFE(sh, next, &shader->variants, list)
 811       ilo_shader_destroy_kernel(sh);
 812
 813    FREE((struct tgsi_token *) shader->info.tokens);
 814    FREE(shader);
 815 }
 816
 817 /**
 818  * Return the type (PIPE_SHADER_x) of the shader.
 819  */
 820 int
 821 ilo_shader_get_type(const struct ilo_shader_state *shader)
 822 {
 823    return shader->info.type;
 824 }
 825
 826 /**
 827  * Select a kernel for the given context.  This will compile a new kernel if
 828  * none of the existing kernels work with the context.
 829  *
 830  * \param ilo the context
 831  * \param dirty states of the context that are considered changed
 832  * \return true if a different kernel is selected
 833  */
 834 bool
 835 ilo_shader_select_kernel(struct ilo_shader_state *shader,
 836                          const struct ilo_context *ilo,
 837                          uint32_t dirty)
 838 {
 839    const struct ilo_shader * const cur = shader->shader;
 840    struct ilo_shader_variant variant;
 841
 842    if (!(shader->info.non_orthogonal_states & dirty))
 843       return false;
 844
 845    ilo_shader_variant_init(&variant, &shader->info, ilo);
 846    ilo_shader_state_use_variant(shader, &variant);
 847
 848    return (shader->shader != cur);
 849 }
 850
 851 /**
 852  * Return the cache offset of the selected kernel.  This must be called after
 853  * ilo_shader_select_kernel() and ilo_shader_cache_upload().
 854  */
 855 uint32_t
 856 ilo_shader_get_kernel_offset(const struct ilo_shader_state *shader)
 857 {
 858    const struct ilo_shader *kernel = shader->shader;
 859
 860    assert(kernel && kernel->uploaded);
 861
 862    return kernel->cache_offset;
 863 }
 864
 865 /**
 866  * Query a kernel parameter for the selected kernel.
 867  */
 868 int
 869 ilo_shader_get_kernel_param(const struct ilo_shader_state *shader,
 870                             enum ilo_kernel_param param)
 871 {
 872    const struct ilo_shader *kernel = shader->shader;
 873    int val;
 874
 875    assert(kernel);
 876
 877    switch (param) {
 878    case ILO_KERNEL_INPUT_COUNT:
 879       val = kernel->in.count;
 880       break;
 881    case ILO_KERNEL_OUTPUT_COUNT:
 882       val = kernel->out.count;
 883       break;
 884    case ILO_KERNEL_URB_DATA_START_REG:
 885       val = kernel->in.start_grf;
 886       break;
 887
 888    case ILO_KERNEL_VS_INPUT_INSTANCEID:
 889       val = shader->info.has_instanceid;
 890       break;
 891    case ILO_KERNEL_VS_INPUT_VERTEXID:
 892       val = shader->info.has_vertexid;
 893       break;
 894    case ILO_KERNEL_VS_INPUT_EDGEFLAG:
 895       if (shader->info.edgeflag_in >= 0) {
 896          /* we rely on the state tracker here */
 897          assert(shader->info.edgeflag_in == kernel->in.count - 1);
 898          val = true;
 899       }
 900       else {
 901          val = false;
 902       }
 903       break;
 904    case ILO_KERNEL_VS_PCB_UCP_SIZE:
 905       val = kernel->pcb.clip_state_size;
 906       break;
 907    case ILO_KERNEL_VS_GEN6_SO:
 908       val = kernel->stream_output;
 909       break;
 910    case ILO_KERNEL_VS_GEN6_SO_START_REG:
 911       val = kernel->gs_start_grf;
 912       break;
 913    case ILO_KERNEL_VS_GEN6_SO_POINT_OFFSET:
 914       val = kernel->gs_offsets[0];
 915       break;
 916    case ILO_KERNEL_VS_GEN6_SO_LINE_OFFSET:
 917       val = kernel->gs_offsets[1];
 918       break;
 919    case ILO_KERNEL_VS_GEN6_SO_TRI_OFFSET:
 920       val = kernel->gs_offsets[2];
 921       break;
 922
 923    case ILO_KERNEL_GS_DISCARD_ADJACENCY:
 924       val = kernel->in.discard_adj;
 925       break;
 926    case ILO_KERNEL_GS_GEN6_SVBI_POST_INC:
 927       val = kernel->svbi_post_inc;
 928       break;
 929
 930    case ILO_KERNEL_FS_INPUT_Z:
 931    case ILO_KERNEL_FS_INPUT_W:
 932       val = kernel->in.has_pos;
 933       break;
 934    case ILO_KERNEL_FS_OUTPUT_Z:
 935       val = kernel->out.has_pos;
 936       break;
 937    case ILO_KERNEL_FS_USE_KILL:
 938       val = kernel->has_kill;
 939       break;
 940    case ILO_KERNEL_FS_BARYCENTRIC_INTERPOLATIONS:
 941       val = kernel->in.barycentric_interpolation_mode;
 942       break;
 943    case ILO_KERNEL_FS_DISPATCH_16_OFFSET:
 944       val = 0;
 945       break;
 946
 947    default:
 948       assert(!"unknown kernel parameter");
 949       val = 0;
 950       break;
 951    }
 952
 953    return val;
 954 }
 955
 956 /**
 957  * Return the CSO of the selected kernel.
 958  */
 959 const struct ilo_shader_cso *
 960 ilo_shader_get_kernel_cso(const struct ilo_shader_state *shader)
 961 {
 962    const struct ilo_shader *kernel = shader->shader;
 963
 964    assert(kernel);
 965
 966    return &kernel->cso;
 967 }
 968
 969 /**
 970  * Return the SO info of the selected kernel.
 971  */
 972 const struct pipe_stream_output_info *
 973 ilo_shader_get_kernel_so_info(const struct ilo_shader_state *shader)
 974 {
 975    const struct ilo_shader *kernel = shader->shader;
 976
 977    assert(kernel);
 978
 979    return &kernel->so_info;
 980 }