src/gallium/drivers/v3d/v3d_program.c

   1 /*
   2  * Copyright © 2014-2017 Broadcom
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include <inttypes.h>
  25 #include "util/u_format.h"
  26 #include "util/u_math.h"
  27 #include "util/u_memory.h"
  28 #include "util/ralloc.h"
  29 #include "util/hash_table.h"
  30 #include "util/u_upload_mgr.h"
  31 #include "tgsi/tgsi_dump.h"
  32 #include "tgsi/tgsi_parse.h"
  33 #include "compiler/nir/nir.h"
  34 #include "compiler/nir/nir_builder.h"
  35 #include "nir/tgsi_to_nir.h"
  36 #include "compiler/v3d_compiler.h"
  37 #include "v3d_context.h"
  38 #include "broadcom/cle/v3d_packet_v33_pack.h"
  39 #include "mesa/state_tracker/st_glsl_types.h"
  40
  41 static struct v3d_compiled_shader *
  42 v3d_get_compiled_shader(struct v3d_context *v3d, struct v3d_key *key);
  43 static void
  44 v3d_setup_shared_precompile_key(struct v3d_uncompiled_shader *uncompiled,
  45                                 struct v3d_key *key);
  46
  47 static gl_varying_slot
  48 v3d_get_slot_for_driver_location(nir_shader *s, uint32_t driver_location)
  49 {
  50         nir_foreach_variable(var, &s->outputs) {
  51                 if (var->data.driver_location == driver_location) {
  52                         return var->data.location;
  53                 }
  54         }
  55
  56         return -1;
  57 }
  58
  59 /**
  60  * Precomputes the TRANSFORM_FEEDBACK_OUTPUT_DATA_SPEC array for the shader.
  61  *
  62  * A shader can have 16 of these specs, and each one of them can write up to
  63  * 16 dwords.  Since we allow a total of 64 transform feedback output
  64  * components (not 16 vectors), we have to group the writes of multiple
  65  * varyings together in a single data spec.
  66  */
  67 static void
  68 v3d_set_transform_feedback_outputs(struct v3d_uncompiled_shader *so,
  69                                    const struct pipe_stream_output_info *stream_output)
  70 {
  71         if (!stream_output->num_outputs)
  72                 return;
  73
  74         struct v3d_varying_slot slots[PIPE_MAX_SO_OUTPUTS * 4];
  75         int slot_count = 0;
  76
  77         for (int buffer = 0; buffer < PIPE_MAX_SO_BUFFERS; buffer++) {
  78                 uint32_t buffer_offset = 0;
  79                 uint32_t vpm_start = slot_count;
  80
  81                 for (int i = 0; i < stream_output->num_outputs; i++) {
  82                         const struct pipe_stream_output *output =
  83                                 &stream_output->output[i];
  84
  85                         if (output->output_buffer != buffer)
  86                                 continue;
  87
  88                         /* We assume that the SO outputs appear in increasing
  89                          * order in the buffer.
  90                          */
  91                         assert(output->dst_offset >= buffer_offset);
  92
  93                         /* Pad any undefined slots in the output */
  94                         for (int j = buffer_offset; j < output->dst_offset; j++) {
  95                                 slots[slot_count] =
  96                                         v3d_slot_from_slot_and_component(VARYING_SLOT_POS, 0);
  97                                 slot_count++;
  98                                 buffer_offset++;
  99                         }
 100
 101                         /* Set the coordinate shader up to output the
 102                          * components of this varying.
 103                          */
 104                         for (int j = 0; j < output->num_components; j++) {
 105                                 gl_varying_slot slot =
 106                                         v3d_get_slot_for_driver_location(so->base.ir.nir, output->register_index);
 107
 108                                 slots[slot_count] =
 109                                         v3d_slot_from_slot_and_component(slot,
 110                                                                          output->start_component + j);
 111                                 slot_count++;
 112                                 buffer_offset++;
 113                         }
 114                 }
 115
 116                 uint32_t vpm_size = slot_count - vpm_start;
 117                 if (!vpm_size)
 118                         continue;
 119
 120                 uint32_t vpm_start_offset = vpm_start + 6;
 121
 122                 while (vpm_size) {
 123                         uint32_t write_size = MIN2(vpm_size, 1 << 4);
 124
 125                         struct V3D33_TRANSFORM_FEEDBACK_OUTPUT_DATA_SPEC unpacked = {
 126                                 /* We need the offset from the coordinate shader's VPM
 127                                  * output block, which has the [X, Y, Z, W, Xs, Ys]
 128                                  * values at the start.
 129                                  */
 130                                 .first_shaded_vertex_value_to_output = vpm_start_offset,
 131                                 .number_of_consecutive_vertex_values_to_output_as_32_bit_values = write_size,
 132                                 .output_buffer_to_write_to = buffer,
 133                         };
 134
 135                         /* GFXH-1559 */
 136                         assert(unpacked.first_shaded_vertex_value_to_output != 8 ||
 137                                so->num_tf_specs != 0);
 138
 139                         assert(so->num_tf_specs != ARRAY_SIZE(so->tf_specs));
 140                         V3D33_TRANSFORM_FEEDBACK_OUTPUT_DATA_SPEC_pack(NULL,
 141                                                                        (void *)&so->tf_specs[so->num_tf_specs],
 142                                                                        &unpacked);
 143
 144                         /* If point size is being written by the shader, then
 145                          * all the VPM start offsets are shifted up by one.
 146                          * We won't know that until the variant is compiled,
 147                          * though.
 148                          */
 149                         unpacked.first_shaded_vertex_value_to_output++;
 150
 151                         /* GFXH-1559 */
 152                         assert(unpacked.first_shaded_vertex_value_to_output != 8 ||
 153                                so->num_tf_specs != 0);
 154
 155                         V3D33_TRANSFORM_FEEDBACK_OUTPUT_DATA_SPEC_pack(NULL,
 156                                                                        (void *)&so->tf_specs_psiz[so->num_tf_specs],
 157                                                                        &unpacked);
 158                         so->num_tf_specs++;
 159                         vpm_start_offset += write_size;
 160                         vpm_size -= write_size;
 161                 }
 162                 so->base.stream_output.stride[buffer] =
 163                         stream_output->stride[buffer];
 164         }
 165
 166         so->num_tf_outputs = slot_count;
 167         so->tf_outputs = ralloc_array(so->base.ir.nir, struct v3d_varying_slot,
 168                                       slot_count);
 169         memcpy(so->tf_outputs, slots, sizeof(*slots) * slot_count);
 170 }
 171
 172 static int
 173 type_size(const struct glsl_type *type)
 174 {
 175         return glsl_count_attribute_slots(type, false);
 176 }
 177
 178 static int
 179 uniforms_type_size(const struct glsl_type *type)
 180 {
 181         return st_glsl_storage_type_size(type, false);
 182 }
 183
 184 /**
 185  * Precompiles a shader variant at shader state creation time if
 186  * V3D_DEBUG=precompile is set.  Used for shader-db
 187  * (https://gitlab.freedesktop.org/mesa/shader-db)
 188  */
 189 static void
 190 v3d_shader_precompile(struct v3d_context *v3d,
 191                       struct v3d_uncompiled_shader *so)
 192 {
 193         nir_shader *s = so->base.ir.nir;
 194
 195         if (s->info.stage == MESA_SHADER_FRAGMENT) {
 196                 struct v3d_fs_key key = {
 197                         .base.shader_state = so,
 198                 };
 199
 200                 nir_foreach_variable(var, &s->outputs) {
 201                         if (var->data.location == FRAG_RESULT_COLOR) {
 202                                 key.nr_cbufs = 1;
 203                         } else if (var->data.location == FRAG_RESULT_DATA0) {
 204                                 key.nr_cbufs = MAX2(key.nr_cbufs,
 205                                                     var->data.location -
 206                                                     FRAG_RESULT_DATA0 + 1);
 207                         }
 208                 }
 209
 210                 v3d_setup_shared_precompile_key(so, &key.base);
 211                 v3d_get_compiled_shader(v3d, &key.base);
 212         } else {
 213                 struct v3d_vs_key key = {
 214                         .base.shader_state = so,
 215                 };
 216
 217                 v3d_setup_shared_precompile_key(so, &key.base);
 218
 219                 /* Compile VS: All outputs */
 220                 nir_foreach_variable(var, &s->outputs) {
 221                         unsigned array_len = MAX2(glsl_get_length(var->type), 1);
 222                         assert(array_len == 1);
 223                         (void)array_len;
 224
 225                         int slot = var->data.location;
 226                         for (int i = 0; i < glsl_get_components(var->type); i++) {
 227                                 int swiz = var->data.location_frac + i;
 228                                 key.fs_inputs[key.num_fs_inputs++] =
 229                                         v3d_slot_from_slot_and_component(slot,
 230                                                                          swiz);
 231                         }
 232                 }
 233
 234                 v3d_get_compiled_shader(v3d, &key.base);
 235
 236                 /* Compile VS bin shader: only position (XXX: include TF) */
 237                 key.is_coord = true;
 238                 key.num_fs_inputs = 0;
 239                 for (int i = 0; i < 4; i++) {
 240                         key.fs_inputs[key.num_fs_inputs++] =
 241                                 v3d_slot_from_slot_and_component(VARYING_SLOT_POS,
 242                                                                  i);
 243                 }
 244                 v3d_get_compiled_shader(v3d, &key.base);
 245         }
 246 }
 247
 248 static void *
 249 v3d_shader_state_create(struct pipe_context *pctx,
 250                         const struct pipe_shader_state *cso)
 251 {
 252         struct v3d_context *v3d = v3d_context(pctx);
 253         struct v3d_uncompiled_shader *so = CALLOC_STRUCT(v3d_uncompiled_shader);
 254         if (!so)
 255                 return NULL;
 256
 257         so->program_id = v3d->next_uncompiled_program_id++;
 258
 259         nir_shader *s;
 260
 261         if (cso->type == PIPE_SHADER_IR_NIR) {
 262                 /* The backend takes ownership of the NIR shader on state
 263                  * creation.
 264                  */
 265                 s = cso->ir.nir;
 266
 267                 NIR_PASS_V(s, nir_lower_io, nir_var_uniform,
 268                            uniforms_type_size,
 269                            (nir_lower_io_options)0);
 270         } else {
 271                 assert(cso->type == PIPE_SHADER_IR_TGSI);
 272
 273                 if (V3D_DEBUG & V3D_DEBUG_TGSI) {
 274                         fprintf(stderr, "prog %d TGSI:\n",
 275                                 so->program_id);
 276                         tgsi_dump(cso->tokens, 0);
 277                         fprintf(stderr, "\n");
 278                 }
 279                 s = tgsi_to_nir(cso->tokens, &v3d_nir_options);
 280         }
 281
 282         nir_variable_mode lower_mode = nir_var_all & ~nir_var_uniform;
 283         if (s->info.stage == MESA_SHADER_VERTEX)
 284                 lower_mode &= ~(nir_var_shader_in | nir_var_shader_out);
 285         NIR_PASS_V(s, nir_lower_io, lower_mode,
 286                    type_size,
 287                    (nir_lower_io_options)0);
 288
 289         NIR_PASS_V(s, nir_opt_global_to_local);
 290         NIR_PASS_V(s, nir_lower_regs_to_ssa);
 291         NIR_PASS_V(s, nir_normalize_cubemap_coords);
 292
 293         NIR_PASS_V(s, nir_lower_load_const_to_scalar);
 294
 295         v3d_optimize_nir(s);
 296
 297         NIR_PASS_V(s, nir_remove_dead_variables, nir_var_function_temp);
 298
 299         /* Garbage collect dead instructions */
 300         nir_sweep(s);
 301
 302         so->base.type = PIPE_SHADER_IR_NIR;
 303         so->base.ir.nir = s;
 304
 305         v3d_set_transform_feedback_outputs(so, &cso->stream_output);
 306
 307         if (V3D_DEBUG & (V3D_DEBUG_NIR |
 308                          v3d_debug_flag_for_shader_stage(s->info.stage))) {
 309                 fprintf(stderr, "%s prog %d NIR:\n",
 310                         gl_shader_stage_name(s->info.stage),
 311                         so->program_id);
 312                 nir_print_shader(s, stderr);
 313                 fprintf(stderr, "\n");
 314         }
 315
 316         if (V3D_DEBUG & V3D_DEBUG_PRECOMPILE)
 317                 v3d_shader_precompile(v3d, so);
 318
 319         return so;
 320 }
 321
 322 static void
 323 v3d_shader_debug_output(const char *message, void *data)
 324 {
 325         struct v3d_context *v3d = data;
 326
 327         pipe_debug_message(&v3d->debug, SHADER_INFO, "%s", message);
 328 }
 329
 330 static struct v3d_compiled_shader *
 331 v3d_get_compiled_shader(struct v3d_context *v3d, struct v3d_key *key)
 332 {
 333         struct v3d_uncompiled_shader *shader_state = key->shader_state;
 334         nir_shader *s = shader_state->base.ir.nir;
 335
 336         struct hash_table *ht;
 337         uint32_t key_size;
 338         if (s->info.stage == MESA_SHADER_FRAGMENT) {
 339                 ht = v3d->fs_cache;
 340                 key_size = sizeof(struct v3d_fs_key);
 341         } else {
 342                 ht = v3d->vs_cache;
 343                 key_size = sizeof(struct v3d_vs_key);
 344         }
 345
 346         struct hash_entry *entry = _mesa_hash_table_search(ht, key);
 347         if (entry)
 348                 return entry->data;
 349
 350         struct v3d_compiled_shader *shader =
 351                 rzalloc(NULL, struct v3d_compiled_shader);
 352
 353         int program_id = shader_state->program_id;
 354         int variant_id =
 355                 p_atomic_inc_return(&shader_state->compiled_variant_count);
 356         uint64_t *qpu_insts;
 357         uint32_t shader_size;
 358
 359         qpu_insts = v3d_compile(v3d->screen->compiler, key,
 360                                 &shader->prog_data.base, s,
 361                                 v3d_shader_debug_output,
 362                                 v3d,
 363                                 program_id, variant_id, &shader_size);
 364         ralloc_steal(shader, shader->prog_data.base);
 365
 366         v3d_set_shader_uniform_dirty_flags(shader);
 367
 368         if (shader_size) {
 369                 u_upload_data(v3d->state_uploader, 0, shader_size, 8,
 370                               qpu_insts, &shader->offset, &shader->resource);
 371         }
 372
 373         free(qpu_insts);
 374
 375         struct v3d_key *dup_key;
 376         dup_key = ralloc_size(shader, key_size);
 377         memcpy(dup_key, key, key_size);
 378         _mesa_hash_table_insert(ht, dup_key, shader);
 379
 380         if (shader->prog_data.base->spill_size >
 381             v3d->prog.spill_size_per_thread) {
 382                 /* Max 4 QPUs per slice, 3 slices per core. We only do single
 383                  * core so far.  This overallocates memory on smaller cores.
 384                  */
 385                 int total_spill_size =
 386                         4 * 3 * shader->prog_data.base->spill_size;
 387
 388                 v3d_bo_unreference(&v3d->prog.spill_bo);
 389                 v3d->prog.spill_bo = v3d_bo_alloc(v3d->screen,
 390                                                   total_spill_size, "spill");
 391                 v3d->prog.spill_size_per_thread =
 392                         shader->prog_data.base->spill_size;
 393         }
 394
 395         return shader;
 396 }
 397
 398 static void
 399 v3d_free_compiled_shader(struct v3d_compiled_shader *shader)
 400 {
 401         pipe_resource_reference(&shader->resource, NULL);
 402         ralloc_free(shader);
 403 }
 404
 405 static void
 406 v3d_setup_shared_key(struct v3d_context *v3d, struct v3d_key *key,
 407                      struct v3d_texture_stateobj *texstate)
 408 {
 409         const struct v3d_device_info *devinfo = &v3d->screen->devinfo;
 410
 411         for (int i = 0; i < texstate->num_textures; i++) {
 412                 struct pipe_sampler_view *sampler = texstate->textures[i];
 413                 struct v3d_sampler_view *v3d_sampler = v3d_sampler_view(sampler);
 414                 struct pipe_sampler_state *sampler_state =
 415                         texstate->samplers[i];
 416
 417                 if (!sampler)
 418                         continue;
 419
 420                 key->tex[i].return_size =
 421                         v3d_get_tex_return_size(devinfo,
 422                                                 sampler->format,
 423                                                 sampler_state->compare_mode);
 424
 425                 /* For 16-bit, we set up the sampler to always return 2
 426                  * channels (meaning no recompiles for most statechanges),
 427                  * while for 32 we actually scale the returns with channels.
 428                  */
 429                 if (key->tex[i].return_size == 16) {
 430                         key->tex[i].return_channels = 2;
 431                 } else if (devinfo->ver > 40) {
 432                         key->tex[i].return_channels = 4;
 433                 } else {
 434                         key->tex[i].return_channels =
 435                                 v3d_get_tex_return_channels(devinfo,
 436                                                             sampler->format);
 437                 }
 438
 439                 if (key->tex[i].return_size == 32 && devinfo->ver < 40) {
 440                         memcpy(key->tex[i].swizzle,
 441                                v3d_sampler->swizzle,
 442                                sizeof(v3d_sampler->swizzle));
 443                 } else {
 444                         /* For 16-bit returns, we let the sampler state handle
 445                          * the swizzle.
 446                          */
 447                         key->tex[i].swizzle[0] = PIPE_SWIZZLE_X;
 448                         key->tex[i].swizzle[1] = PIPE_SWIZZLE_Y;
 449                         key->tex[i].swizzle[2] = PIPE_SWIZZLE_Z;
 450                         key->tex[i].swizzle[3] = PIPE_SWIZZLE_W;
 451                 }
 452
 453                 if (sampler) {
 454                         key->tex[i].clamp_s =
 455                                 sampler_state->wrap_s == PIPE_TEX_WRAP_CLAMP;
 456                         key->tex[i].clamp_t =
 457                                 sampler_state->wrap_t == PIPE_TEX_WRAP_CLAMP;
 458                         key->tex[i].clamp_r =
 459                                 sampler_state->wrap_r == PIPE_TEX_WRAP_CLAMP;
 460                 }
 461         }
 462
 463         key->ucp_enables = v3d->rasterizer->base.clip_plane_enable;
 464 }
 465
 466 static void
 467 v3d_setup_shared_precompile_key(struct v3d_uncompiled_shader *uncompiled,
 468                                 struct v3d_key *key)
 469 {
 470         nir_shader *s = uncompiled->base.ir.nir;
 471
 472         for (int i = 0; i < s->info.num_textures; i++) {
 473                 key->tex[i].return_size = 16;
 474                 key->tex[i].return_channels = 2;
 475
 476                 key->tex[i].swizzle[0] = PIPE_SWIZZLE_X;
 477                 key->tex[i].swizzle[1] = PIPE_SWIZZLE_Y;
 478                 key->tex[i].swizzle[2] = PIPE_SWIZZLE_Z;
 479                 key->tex[i].swizzle[3] = PIPE_SWIZZLE_W;
 480         }
 481 }
 482
 483 static void
 484 v3d_update_compiled_fs(struct v3d_context *v3d, uint8_t prim_mode)
 485 {
 486         struct v3d_job *job = v3d->job;
 487         struct v3d_fs_key local_key;
 488         struct v3d_fs_key *key = &local_key;
 489         nir_shader *s = v3d->prog.bind_fs->base.ir.nir;
 490
 491         if (!(v3d->dirty & (VC5_DIRTY_PRIM_MODE |
 492                             VC5_DIRTY_BLEND |
 493                             VC5_DIRTY_FRAMEBUFFER |
 494                             VC5_DIRTY_ZSA |
 495                             VC5_DIRTY_RASTERIZER |
 496                             VC5_DIRTY_SAMPLE_STATE |
 497                             VC5_DIRTY_FRAGTEX |
 498                             VC5_DIRTY_UNCOMPILED_FS))) {
 499                 return;
 500         }
 501
 502         memset(key, 0, sizeof(*key));
 503         v3d_setup_shared_key(v3d, &key->base, &v3d->tex[PIPE_SHADER_FRAGMENT]);
 504         key->base.shader_state = v3d->prog.bind_fs;
 505         key->is_points = (prim_mode == PIPE_PRIM_POINTS);
 506         key->is_lines = (prim_mode >= PIPE_PRIM_LINES &&
 507                          prim_mode <= PIPE_PRIM_LINE_STRIP);
 508         key->clamp_color = v3d->rasterizer->base.clamp_fragment_color;
 509         if (v3d->blend->base.logicop_enable) {
 510                 key->logicop_func = v3d->blend->base.logicop_func;
 511         } else {
 512                 key->logicop_func = PIPE_LOGICOP_COPY;
 513         }
 514         if (job->msaa) {
 515                 key->msaa = v3d->rasterizer->base.multisample;
 516                 key->sample_coverage = (v3d->rasterizer->base.multisample &&
 517                                         v3d->sample_mask != (1 << V3D_MAX_SAMPLES) - 1);
 518                 key->sample_alpha_to_coverage = v3d->blend->base.alpha_to_coverage;
 519                 key->sample_alpha_to_one = v3d->blend->base.alpha_to_one;
 520         }
 521
 522         key->depth_enabled = (v3d->zsa->base.depth.enabled ||
 523                               v3d->zsa->base.stencil[0].enabled);
 524         if (v3d->zsa->base.alpha.enabled) {
 525                 key->alpha_test = true;
 526                 key->alpha_test_func = v3d->zsa->base.alpha.func;
 527         }
 528
 529         /* gl_FragColor's propagation to however many bound color buffers
 530          * there are means that the buffer count needs to be in the key.
 531          */
 532         key->nr_cbufs = v3d->framebuffer.nr_cbufs;
 533         key->swap_color_rb = v3d->swap_color_rb;
 534
 535         for (int i = 0; i < key->nr_cbufs; i++) {
 536                 struct pipe_surface *cbuf = v3d->framebuffer.cbufs[i];
 537                 if (!cbuf)
 538                         continue;
 539
 540                 const struct util_format_description *desc =
 541                         util_format_description(cbuf->format);
 542
 543                 if (desc->channel[0].type == UTIL_FORMAT_TYPE_FLOAT &&
 544                     desc->channel[0].size == 32) {
 545                         key->f32_color_rb |= 1 << i;
 546                 }
 547
 548                 if (s->info.fs.untyped_color_outputs) {
 549                         if (util_format_is_pure_uint(cbuf->format))
 550                                 key->uint_color_rb |= 1 << i;
 551                         else if (util_format_is_pure_sint(cbuf->format))
 552                                 key->int_color_rb |= 1 << i;
 553                 }
 554         }
 555
 556         if (key->is_points) {
 557                 key->point_sprite_mask =
 558                         v3d->rasterizer->base.sprite_coord_enable;
 559                 key->point_coord_upper_left =
 560                         (v3d->rasterizer->base.sprite_coord_mode ==
 561                          PIPE_SPRITE_COORD_UPPER_LEFT);
 562         }
 563
 564         key->light_twoside = v3d->rasterizer->base.light_twoside;
 565         key->shade_model_flat = v3d->rasterizer->base.flatshade;
 566
 567         struct v3d_compiled_shader *old_fs = v3d->prog.fs;
 568         v3d->prog.fs = v3d_get_compiled_shader(v3d, &key->base);
 569         if (v3d->prog.fs == old_fs)
 570                 return;
 571
 572         v3d->dirty |= VC5_DIRTY_COMPILED_FS;
 573
 574         if (old_fs) {
 575                 if (v3d->prog.fs->prog_data.fs->flat_shade_flags !=
 576                     old_fs->prog_data.fs->flat_shade_flags) {
 577                         v3d->dirty |= VC5_DIRTY_FLAT_SHADE_FLAGS;
 578                 }
 579
 580                 if (v3d->prog.fs->prog_data.fs->noperspective_flags !=
 581                     old_fs->prog_data.fs->noperspective_flags) {
 582                         v3d->dirty |= VC5_DIRTY_NOPERSPECTIVE_FLAGS;
 583                 }
 584
 585                 if (v3d->prog.fs->prog_data.fs->centroid_flags !=
 586                     old_fs->prog_data.fs->centroid_flags) {
 587                         v3d->dirty |= VC5_DIRTY_CENTROID_FLAGS;
 588                 }
 589         }
 590
 591         if (old_fs && memcmp(v3d->prog.fs->prog_data.fs->input_slots,
 592                              old_fs->prog_data.fs->input_slots,
 593                              sizeof(v3d->prog.fs->prog_data.fs->input_slots))) {
 594                 v3d->dirty |= VC5_DIRTY_FS_INPUTS;
 595         }
 596 }
 597
 598 static void
 599 v3d_update_compiled_vs(struct v3d_context *v3d, uint8_t prim_mode)
 600 {
 601         struct v3d_vs_key local_key;
 602         struct v3d_vs_key *key = &local_key;
 603
 604         if (!(v3d->dirty & (VC5_DIRTY_PRIM_MODE |
 605                             VC5_DIRTY_RASTERIZER |
 606                             VC5_DIRTY_VERTTEX |
 607                             VC5_DIRTY_VTXSTATE |
 608                             VC5_DIRTY_UNCOMPILED_VS |
 609                             VC5_DIRTY_FS_INPUTS))) {
 610                 return;
 611         }
 612
 613         memset(key, 0, sizeof(*key));
 614         v3d_setup_shared_key(v3d, &key->base, &v3d->tex[PIPE_SHADER_VERTEX]);
 615         key->base.shader_state = v3d->prog.bind_vs;
 616         key->num_fs_inputs = v3d->prog.fs->prog_data.fs->base.num_inputs;
 617         STATIC_ASSERT(sizeof(key->fs_inputs) ==
 618                       sizeof(v3d->prog.fs->prog_data.fs->input_slots));
 619         memcpy(key->fs_inputs, v3d->prog.fs->prog_data.fs->input_slots,
 620                sizeof(key->fs_inputs));
 621         key->clamp_color = v3d->rasterizer->base.clamp_vertex_color;
 622
 623         key->per_vertex_point_size =
 624                 (prim_mode == PIPE_PRIM_POINTS &&
 625                  v3d->rasterizer->base.point_size_per_vertex);
 626
 627         struct v3d_compiled_shader *vs =
 628                 v3d_get_compiled_shader(v3d, &key->base);
 629         if (vs != v3d->prog.vs) {
 630                 v3d->prog.vs = vs;
 631                 v3d->dirty |= VC5_DIRTY_COMPILED_VS;
 632         }
 633
 634         key->is_coord = true;
 635         /* Coord shaders only output varyings used by transform feedback. */
 636         struct v3d_uncompiled_shader *shader_state = key->base.shader_state;
 637         memcpy(key->fs_inputs, shader_state->tf_outputs,
 638                sizeof(*key->fs_inputs) * shader_state->num_tf_outputs);
 639         if (shader_state->num_tf_outputs < key->num_fs_inputs) {
 640                 memset(&key->fs_inputs[shader_state->num_tf_outputs],
 641                        0,
 642                        sizeof(*key->fs_inputs) * (key->num_fs_inputs -
 643                                                   shader_state->num_tf_outputs));
 644         }
 645         key->num_fs_inputs = shader_state->num_tf_outputs;
 646
 647         struct v3d_compiled_shader *cs =
 648                 v3d_get_compiled_shader(v3d, &key->base);
 649         if (cs != v3d->prog.cs) {
 650                 v3d->prog.cs = cs;
 651                 v3d->dirty |= VC5_DIRTY_COMPILED_CS;
 652         }
 653 }
 654
 655 void
 656 v3d_update_compiled_shaders(struct v3d_context *v3d, uint8_t prim_mode)
 657 {
 658         v3d_update_compiled_fs(v3d, prim_mode);
 659         v3d_update_compiled_vs(v3d, prim_mode);
 660 }
 661
 662 static uint32_t
 663 fs_cache_hash(const void *key)
 664 {
 665         return _mesa_hash_data(key, sizeof(struct v3d_fs_key));
 666 }
 667
 668 static uint32_t
 669 vs_cache_hash(const void *key)
 670 {
 671         return _mesa_hash_data(key, sizeof(struct v3d_vs_key));
 672 }
 673
 674 static bool
 675 fs_cache_compare(const void *key1, const void *key2)
 676 {
 677         return memcmp(key1, key2, sizeof(struct v3d_fs_key)) == 0;
 678 }
 679
 680 static bool
 681 vs_cache_compare(const void *key1, const void *key2)
 682 {
 683         return memcmp(key1, key2, sizeof(struct v3d_vs_key)) == 0;
 684 }
 685
 686 static void
 687 delete_from_cache_if_matches(struct hash_table *ht,
 688                              struct v3d_compiled_shader **last_compile,
 689                              struct hash_entry *entry,
 690                              struct v3d_uncompiled_shader *so)
 691 {
 692         const struct v3d_key *key = entry->key;
 693
 694         if (key->shader_state == so) {
 695                 struct v3d_compiled_shader *shader = entry->data;
 696                 _mesa_hash_table_remove(ht, entry);
 697
 698                 if (shader == *last_compile)
 699                         *last_compile = NULL;
 700
 701                 v3d_free_compiled_shader(shader);
 702         }
 703 }
 704
 705 static void
 706 v3d_shader_state_delete(struct pipe_context *pctx, void *hwcso)
 707 {
 708         struct v3d_context *v3d = v3d_context(pctx);
 709         struct v3d_uncompiled_shader *so = hwcso;
 710
 711         hash_table_foreach(v3d->fs_cache, entry) {
 712                 delete_from_cache_if_matches(v3d->fs_cache, &v3d->prog.fs,
 713                                              entry, so);
 714         }
 715         hash_table_foreach(v3d->vs_cache, entry) {
 716                 delete_from_cache_if_matches(v3d->vs_cache, &v3d->prog.vs,
 717                                              entry, so);
 718         }
 719
 720         ralloc_free(so->base.ir.nir);
 721         free(so);
 722 }
 723
 724 static void
 725 v3d_fp_state_bind(struct pipe_context *pctx, void *hwcso)
 726 {
 727         struct v3d_context *v3d = v3d_context(pctx);
 728         v3d->prog.bind_fs = hwcso;
 729         v3d->dirty |= VC5_DIRTY_UNCOMPILED_FS;
 730 }
 731
 732 static void
 733 v3d_vp_state_bind(struct pipe_context *pctx, void *hwcso)
 734 {
 735         struct v3d_context *v3d = v3d_context(pctx);
 736         v3d->prog.bind_vs = hwcso;
 737         v3d->dirty |= VC5_DIRTY_UNCOMPILED_VS;
 738 }
 739
 740 void
 741 v3d_program_init(struct pipe_context *pctx)
 742 {
 743         struct v3d_context *v3d = v3d_context(pctx);
 744
 745         pctx->create_vs_state = v3d_shader_state_create;
 746         pctx->delete_vs_state = v3d_shader_state_delete;
 747
 748         pctx->create_fs_state = v3d_shader_state_create;
 749         pctx->delete_fs_state = v3d_shader_state_delete;
 750
 751         pctx->bind_fs_state = v3d_fp_state_bind;
 752         pctx->bind_vs_state = v3d_vp_state_bind;
 753
 754         v3d->fs_cache = _mesa_hash_table_create(pctx, fs_cache_hash,
 755                                                 fs_cache_compare);
 756         v3d->vs_cache = _mesa_hash_table_create(pctx, vs_cache_hash,
 757                                                 vs_cache_compare);
 758 }
 759
 760 void
 761 v3d_program_fini(struct pipe_context *pctx)
 762 {
 763         struct v3d_context *v3d = v3d_context(pctx);
 764
 765         hash_table_foreach(v3d->fs_cache, entry) {
 766                 struct v3d_compiled_shader *shader = entry->data;
 767                 v3d_free_compiled_shader(shader);
 768                 _mesa_hash_table_remove(v3d->fs_cache, entry);
 769         }
 770
 771         hash_table_foreach(v3d->vs_cache, entry) {
 772                 struct v3d_compiled_shader *shader = entry->data;
 773                 v3d_free_compiled_shader(shader);
 774                 _mesa_hash_table_remove(v3d->vs_cache, entry);
 775         }
 776
 777         v3d_bo_unreference(&v3d->prog.spill_bo);
 778 }