src/gallium/drivers/vc5/vc5_program.c

   1 /*
   2  * Copyright © 2014-2017 Broadcom
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include <inttypes.h>
  25 #include "util/u_format.h"
  26 #include "util/u_math.h"
  27 #include "util/u_memory.h"
  28 #include "util/ralloc.h"
  29 #include "util/hash_table.h"
  30 #include "tgsi/tgsi_dump.h"
  31 #include "tgsi/tgsi_parse.h"
  32 #include "compiler/nir/nir.h"
  33 #include "compiler/nir/nir_builder.h"
  34 #include "nir/tgsi_to_nir.h"
  35 #include "compiler/v3d_compiler.h"
  36 #include "vc5_context.h"
  37 #include "broadcom/cle/v3d_packet_v33_pack.h"
  38
  39 static void
  40 vc5_set_transform_feedback_outputs(struct vc5_uncompiled_shader *so,
  41                                    const struct pipe_stream_output_info *stream_output)
  42 {
  43         if (!stream_output->num_outputs)
  44                 return;
  45
  46         struct v3d_varying_slot slots[PIPE_MAX_SO_OUTPUTS * 4];
  47         int slot_count = 0;
  48
  49         for (int buffer = 0; buffer < PIPE_MAX_SO_BUFFERS; buffer++) {
  50                 uint32_t buffer_offset = 0;
  51                 uint32_t vpm_start = slot_count;
  52
  53                 for (int i = 0; i < stream_output->num_outputs; i++) {
  54                         const struct pipe_stream_output *output =
  55                                 &stream_output->output[i];
  56
  57                         if (output->output_buffer != buffer)
  58                                 continue;
  59
  60                         /* We assume that the SO outputs appear in increasing
  61                          * order in the buffer.
  62                          */
  63                         assert(output->dst_offset >= buffer_offset);
  64
  65                         /* Pad any undefined slots in the output */
  66                         for (int j = buffer_offset; j < output->dst_offset; j++) {
  67                                 slots[slot_count] =
  68                                         v3d_slot_from_slot_and_component(VARYING_SLOT_POS, 0);
  69                                 slot_count++;
  70                         }
  71
  72                         /* Set the coordinate shader up to output the
  73                          * components of this varying.
  74                          */
  75                         for (int j = 0; j < output->num_components; j++) {
  76                                 slots[slot_count] =
  77                                         v3d_slot_from_slot_and_component(VARYING_SLOT_VAR0 +
  78                                                                          output->register_index,
  79                                                                          output->start_component + j);
  80                                 slot_count++;
  81                         }
  82                 }
  83
  84                 uint32_t vpm_size = slot_count - vpm_start;
  85                 if (!vpm_size)
  86                         continue;
  87
  88                 struct V3D33_TRANSFORM_FEEDBACK_OUTPUT_DATA_SPEC unpacked = {
  89                         .first_shaded_vertex_value_to_output = vpm_start,
  90                         .number_of_consecutive_vertex_values_to_output_as_32_bit_values_minus_1 = vpm_size - 1,
  91                         .output_buffer_to_write_to = buffer,
  92                 };
  93                 V3D33_TRANSFORM_FEEDBACK_OUTPUT_DATA_SPEC_pack(NULL,
  94                                                                (void *)&so->tf_specs[so->num_tf_specs++],
  95                                                                &unpacked);
  96         }
  97
  98         so->num_tf_outputs = slot_count;
  99         so->tf_outputs = ralloc_array(so->base.ir.nir, struct v3d_varying_slot,
 100                                       slot_count);
 101         memcpy(so->tf_outputs, slots, sizeof(*slots) * slot_count);
 102 }
 103
 104 static int
 105 type_size(const struct glsl_type *type)
 106 {
 107         return glsl_count_attribute_slots(type, false);
 108 }
 109
 110 static void *
 111 vc5_shader_state_create(struct pipe_context *pctx,
 112                         const struct pipe_shader_state *cso)
 113 {
 114         struct vc5_context *vc5 = vc5_context(pctx);
 115         struct vc5_uncompiled_shader *so = CALLOC_STRUCT(vc5_uncompiled_shader);
 116         if (!so)
 117                 return NULL;
 118
 119         so->program_id = vc5->next_uncompiled_program_id++;
 120
 121         nir_shader *s;
 122
 123         if (cso->type == PIPE_SHADER_IR_NIR) {
 124                 /* The backend takes ownership of the NIR shader on state
 125                  * creation.
 126                  */
 127                 s = cso->ir.nir;
 128
 129                 NIR_PASS_V(s, nir_lower_io, nir_var_all, type_size,
 130                            (nir_lower_io_options)0);
 131         } else {
 132                 assert(cso->type == PIPE_SHADER_IR_TGSI);
 133
 134                 if (V3D_DEBUG & V3D_DEBUG_TGSI) {
 135                         fprintf(stderr, "prog %d TGSI:\n",
 136                                 so->program_id);
 137                         tgsi_dump(cso->tokens, 0);
 138                         fprintf(stderr, "\n");
 139                 }
 140                 s = tgsi_to_nir(cso->tokens, &v3d_nir_options);
 141         }
 142
 143         NIR_PASS_V(s, nir_opt_global_to_local);
 144         NIR_PASS_V(s, nir_lower_regs_to_ssa);
 145         NIR_PASS_V(s, nir_normalize_cubemap_coords);
 146
 147         NIR_PASS_V(s, nir_lower_load_const_to_scalar);
 148
 149         v3d_optimize_nir(s);
 150
 151         NIR_PASS_V(s, nir_remove_dead_variables, nir_var_local);
 152
 153         /* Garbage collect dead instructions */
 154         nir_sweep(s);
 155
 156         so->base.type = PIPE_SHADER_IR_NIR;
 157         so->base.ir.nir = s;
 158
 159         vc5_set_transform_feedback_outputs(so, &cso->stream_output);
 160
 161         if (V3D_DEBUG & (V3D_DEBUG_NIR |
 162                          v3d_debug_flag_for_shader_stage(s->stage))) {
 163                 fprintf(stderr, "%s prog %d NIR:\n",
 164                         gl_shader_stage_name(s->stage),
 165                         so->program_id);
 166                 nir_print_shader(s, stderr);
 167                 fprintf(stderr, "\n");
 168         }
 169
 170         return so;
 171 }
 172
 173 static struct vc5_compiled_shader *
 174 vc5_get_compiled_shader(struct vc5_context *vc5, struct v3d_key *key)
 175 {
 176         struct vc5_uncompiled_shader *shader_state = key->shader_state;
 177         nir_shader *s = shader_state->base.ir.nir;
 178
 179         struct hash_table *ht;
 180         uint32_t key_size;
 181         if (s->stage == MESA_SHADER_FRAGMENT) {
 182                 ht = vc5->fs_cache;
 183                 key_size = sizeof(struct v3d_fs_key);
 184         } else {
 185                 ht = vc5->vs_cache;
 186                 key_size = sizeof(struct v3d_vs_key);
 187         }
 188
 189         struct hash_entry *entry = _mesa_hash_table_search(ht, key);
 190         if (entry)
 191                 return entry->data;
 192
 193         struct vc5_compiled_shader *shader =
 194                 rzalloc(NULL, struct vc5_compiled_shader);
 195
 196         int program_id = shader_state->program_id;
 197         int variant_id =
 198                 p_atomic_inc_return(&shader_state->compiled_variant_count);
 199         uint64_t *qpu_insts;
 200         uint32_t shader_size;
 201
 202         switch (s->stage) {
 203         case MESA_SHADER_VERTEX:
 204                 shader->prog_data.vs = rzalloc(shader, struct v3d_vs_prog_data);
 205
 206                 qpu_insts = v3d_compile_vs(vc5->screen->compiler,
 207                                            (struct v3d_vs_key *)key,
 208                                            shader->prog_data.vs, s,
 209                                            program_id, variant_id,
 210                                            &shader_size);
 211                 break;
 212         case MESA_SHADER_FRAGMENT:
 213                 shader->prog_data.fs = rzalloc(shader, struct v3d_fs_prog_data);
 214
 215                 qpu_insts = v3d_compile_fs(vc5->screen->compiler,
 216                                            (struct v3d_fs_key *)key,
 217                                            shader->prog_data.fs, s,
 218                                            program_id, variant_id,
 219                                            &shader_size);
 220                 break;
 221         default:
 222                 unreachable("bad stage");
 223         }
 224
 225         vc5_set_shader_uniform_dirty_flags(shader);
 226
 227         shader->bo = vc5_bo_alloc(vc5->screen, shader_size, "shader");
 228         vc5_bo_map(shader->bo);
 229         memcpy(shader->bo->map, qpu_insts, shader_size);
 230
 231         free(qpu_insts);
 232
 233         struct vc5_key *dup_key;
 234         dup_key = ralloc_size(shader, key_size);
 235         memcpy(dup_key, key, key_size);
 236         _mesa_hash_table_insert(ht, dup_key, shader);
 237
 238         return shader;
 239 }
 240
 241 static void
 242 vc5_setup_shared_key(struct vc5_context *vc5, struct v3d_key *key,
 243                      struct vc5_texture_stateobj *texstate)
 244 {
 245         for (int i = 0; i < texstate->num_textures; i++) {
 246                 struct pipe_sampler_view *sampler = texstate->textures[i];
 247                 struct vc5_sampler_view *vc5_sampler = vc5_sampler_view(sampler);
 248                 struct pipe_sampler_state *sampler_state =
 249                         texstate->samplers[i];
 250
 251                 if (!sampler)
 252                         continue;
 253
 254                 key->tex[i].return_size =
 255                         vc5_get_tex_return_size(sampler->format);
 256
 257                 /* For 16-bit, we set up the sampler to always return 2
 258                  * channels (meaning no recompiles for most statechanges),
 259                  * while for 32 we actually scale the returns with channels.
 260                  */
 261                 if (key->tex[i].return_size == 16) {
 262                         key->tex[i].return_channels = 2;
 263                 } else {
 264                         key->tex[i].return_channels =
 265                                 vc5_get_tex_return_channels(sampler->format);
 266                 }
 267
 268                 if (vc5_get_tex_return_size(sampler->format) == 32) {
 269                         memcpy(key->tex[i].swizzle,
 270                                vc5_sampler->swizzle,
 271                                sizeof(vc5_sampler->swizzle));
 272                 } else {
 273                         /* For 16-bit returns, we let the sampler state handle
 274                          * the swizzle.
 275                          */
 276                         key->tex[i].swizzle[0] = PIPE_SWIZZLE_X;
 277                         key->tex[i].swizzle[1] = PIPE_SWIZZLE_Y;
 278                         key->tex[i].swizzle[2] = PIPE_SWIZZLE_Z;
 279                         key->tex[i].swizzle[3] = PIPE_SWIZZLE_W;
 280                 }
 281
 282                 if (sampler->texture->nr_samples > 1) {
 283                         key->tex[i].msaa_width = sampler->texture->width0;
 284                         key->tex[i].msaa_height = sampler->texture->height0;
 285                 } else if (sampler){
 286                         key->tex[i].compare_mode = sampler_state->compare_mode;
 287                         key->tex[i].compare_func = sampler_state->compare_func;
 288                         key->tex[i].wrap_s = sampler_state->wrap_s;
 289                         key->tex[i].wrap_t = sampler_state->wrap_t;
 290                 }
 291         }
 292
 293         key->ucp_enables = vc5->rasterizer->base.clip_plane_enable;
 294 }
 295
 296 static void
 297 vc5_update_compiled_fs(struct vc5_context *vc5, uint8_t prim_mode)
 298 {
 299         struct vc5_job *job = vc5->job;
 300         struct v3d_fs_key local_key;
 301         struct v3d_fs_key *key = &local_key;
 302
 303         if (!(vc5->dirty & (VC5_DIRTY_PRIM_MODE |
 304                             VC5_DIRTY_BLEND |
 305                             VC5_DIRTY_FRAMEBUFFER |
 306                             VC5_DIRTY_ZSA |
 307                             VC5_DIRTY_RASTERIZER |
 308                             VC5_DIRTY_SAMPLE_MASK |
 309                             VC5_DIRTY_FRAGTEX |
 310                             VC5_DIRTY_UNCOMPILED_FS))) {
 311                 return;
 312         }
 313
 314         memset(key, 0, sizeof(*key));
 315         vc5_setup_shared_key(vc5, &key->base, &vc5->fragtex);
 316         key->base.shader_state = vc5->prog.bind_fs;
 317         key->is_points = (prim_mode == PIPE_PRIM_POINTS);
 318         key->is_lines = (prim_mode >= PIPE_PRIM_LINES &&
 319                          prim_mode <= PIPE_PRIM_LINE_STRIP);
 320         key->clamp_color = vc5->rasterizer->base.clamp_fragment_color;
 321         if (vc5->blend->logicop_enable) {
 322                 key->logicop_func = vc5->blend->logicop_func;
 323         } else {
 324                 key->logicop_func = PIPE_LOGICOP_COPY;
 325         }
 326         if (job->msaa) {
 327                 key->msaa = vc5->rasterizer->base.multisample;
 328                 key->sample_coverage = (vc5->rasterizer->base.multisample &&
 329                                         vc5->sample_mask != (1 << VC5_MAX_SAMPLES) - 1);
 330                 key->sample_alpha_to_coverage = vc5->blend->alpha_to_coverage;
 331                 key->sample_alpha_to_one = vc5->blend->alpha_to_one;
 332         }
 333
 334         key->depth_enabled = (vc5->zsa->base.depth.enabled ||
 335                               vc5->zsa->base.stencil[0].enabled);
 336         if (vc5->zsa->base.alpha.enabled) {
 337                 key->alpha_test = true;
 338                 key->alpha_test_func = vc5->zsa->base.alpha.func;
 339         }
 340
 341         if (vc5->framebuffer.cbufs[0]) {
 342                 struct pipe_surface *cbuf = vc5->framebuffer.cbufs[0];
 343                 const struct util_format_description *desc =
 344                         util_format_description(cbuf->format);
 345
 346                 key->swap_color_rb = desc->swizzle[0] == PIPE_SWIZZLE_Z;
 347         }
 348
 349         if (key->is_points) {
 350                 key->point_sprite_mask =
 351                         vc5->rasterizer->base.sprite_coord_enable;
 352                 key->point_coord_upper_left =
 353                         (vc5->rasterizer->base.sprite_coord_mode ==
 354                          PIPE_SPRITE_COORD_UPPER_LEFT);
 355         }
 356
 357         key->light_twoside = vc5->rasterizer->base.light_twoside;
 358
 359         struct vc5_compiled_shader *old_fs = vc5->prog.fs;
 360         vc5->prog.fs = vc5_get_compiled_shader(vc5, &key->base);
 361         if (vc5->prog.fs == old_fs)
 362                 return;
 363
 364         vc5->dirty |= VC5_DIRTY_COMPILED_FS;
 365
 366         if (old_fs &&
 367             (vc5->prog.fs->prog_data.fs->flat_shade_flags !=
 368              old_fs->prog_data.fs->flat_shade_flags ||
 369              (vc5->rasterizer->base.flatshade &&
 370               vc5->prog.fs->prog_data.fs->shade_model_flags !=
 371               old_fs->prog_data.fs->shade_model_flags))) {
 372                 vc5->dirty |= VC5_DIRTY_FLAT_SHADE_FLAGS;
 373         }
 374
 375         if (old_fs && memcmp(vc5->prog.fs->prog_data.fs->input_slots,
 376                              old_fs->prog_data.fs->input_slots,
 377                              sizeof(vc5->prog.fs->prog_data.fs->input_slots))) {
 378                 vc5->dirty |= VC5_DIRTY_FS_INPUTS;
 379         }
 380 }
 381
 382 static void
 383 vc5_update_compiled_vs(struct vc5_context *vc5, uint8_t prim_mode)
 384 {
 385         struct v3d_vs_key local_key;
 386         struct v3d_vs_key *key = &local_key;
 387
 388         if (!(vc5->dirty & (VC5_DIRTY_PRIM_MODE |
 389                             VC5_DIRTY_RASTERIZER |
 390                             VC5_DIRTY_VERTTEX |
 391                             VC5_DIRTY_VTXSTATE |
 392                             VC5_DIRTY_UNCOMPILED_VS |
 393                             VC5_DIRTY_FS_INPUTS))) {
 394                 return;
 395         }
 396
 397         memset(key, 0, sizeof(*key));
 398         vc5_setup_shared_key(vc5, &key->base, &vc5->verttex);
 399         key->base.shader_state = vc5->prog.bind_vs;
 400         key->num_fs_inputs = vc5->prog.fs->prog_data.fs->base.num_inputs;
 401         STATIC_ASSERT(sizeof(key->fs_inputs) ==
 402                       sizeof(vc5->prog.fs->prog_data.fs->input_slots));
 403         memcpy(key->fs_inputs, vc5->prog.fs->prog_data.fs->input_slots,
 404                sizeof(key->fs_inputs));
 405         key->clamp_color = vc5->rasterizer->base.clamp_vertex_color;
 406
 407         key->per_vertex_point_size =
 408                 (prim_mode == PIPE_PRIM_POINTS &&
 409                  vc5->rasterizer->base.point_size_per_vertex);
 410
 411         struct vc5_compiled_shader *vs =
 412                 vc5_get_compiled_shader(vc5, &key->base);
 413         if (vs != vc5->prog.vs) {
 414                 vc5->prog.vs = vs;
 415                 vc5->dirty |= VC5_DIRTY_COMPILED_VS;
 416         }
 417
 418         key->is_coord = true;
 419         /* Coord shaders only output varyings used by transform feedback. */
 420         struct vc5_uncompiled_shader *shader_state = key->base.shader_state;
 421         memcpy(key->fs_inputs, shader_state->tf_outputs,
 422                sizeof(*key->fs_inputs) * shader_state->num_tf_outputs);
 423         if (shader_state->num_tf_outputs < key->num_fs_inputs) {
 424                 memset(&key->fs_inputs[shader_state->num_tf_outputs],
 425                        0,
 426                        sizeof(*key->fs_inputs) * (key->num_fs_inputs -
 427                                                   shader_state->num_tf_outputs));
 428         }
 429         key->num_fs_inputs = shader_state->num_tf_outputs;
 430
 431         struct vc5_compiled_shader *cs =
 432                 vc5_get_compiled_shader(vc5, &key->base);
 433         if (cs != vc5->prog.cs) {
 434                 vc5->prog.cs = cs;
 435                 vc5->dirty |= VC5_DIRTY_COMPILED_CS;
 436         }
 437 }
 438
 439 void
 440 vc5_update_compiled_shaders(struct vc5_context *vc5, uint8_t prim_mode)
 441 {
 442         vc5_update_compiled_fs(vc5, prim_mode);
 443         vc5_update_compiled_vs(vc5, prim_mode);
 444 }
 445
 446 static uint32_t
 447 fs_cache_hash(const void *key)
 448 {
 449         return _mesa_hash_data(key, sizeof(struct v3d_fs_key));
 450 }
 451
 452 static uint32_t
 453 vs_cache_hash(const void *key)
 454 {
 455         return _mesa_hash_data(key, sizeof(struct v3d_vs_key));
 456 }
 457
 458 static bool
 459 fs_cache_compare(const void *key1, const void *key2)
 460 {
 461         return memcmp(key1, key2, sizeof(struct v3d_fs_key)) == 0;
 462 }
 463
 464 static bool
 465 vs_cache_compare(const void *key1, const void *key2)
 466 {
 467         return memcmp(key1, key2, sizeof(struct v3d_vs_key)) == 0;
 468 }
 469
 470 static void
 471 delete_from_cache_if_matches(struct hash_table *ht,
 472                              struct vc5_compiled_shader **last_compile,
 473                              struct hash_entry *entry,
 474                              struct vc5_uncompiled_shader *so)
 475 {
 476         const struct v3d_key *key = entry->key;
 477
 478         if (key->shader_state == so) {
 479                 struct vc5_compiled_shader *shader = entry->data;
 480                 _mesa_hash_table_remove(ht, entry);
 481                 vc5_bo_unreference(&shader->bo);
 482
 483                 if (shader == *last_compile)
 484                         *last_compile = NULL;
 485
 486                 ralloc_free(shader);
 487         }
 488 }
 489
 490 static void
 491 vc5_shader_state_delete(struct pipe_context *pctx, void *hwcso)
 492 {
 493         struct vc5_context *vc5 = vc5_context(pctx);
 494         struct vc5_uncompiled_shader *so = hwcso;
 495
 496         struct hash_entry *entry;
 497         hash_table_foreach(vc5->fs_cache, entry) {
 498                 delete_from_cache_if_matches(vc5->fs_cache, &vc5->prog.fs,
 499                                              entry, so);
 500         }
 501         hash_table_foreach(vc5->vs_cache, entry) {
 502                 delete_from_cache_if_matches(vc5->vs_cache, &vc5->prog.vs,
 503                                              entry, so);
 504         }
 505
 506         ralloc_free(so->base.ir.nir);
 507         free(so);
 508 }
 509
 510 static void
 511 vc5_fp_state_bind(struct pipe_context *pctx, void *hwcso)
 512 {
 513         struct vc5_context *vc5 = vc5_context(pctx);
 514         vc5->prog.bind_fs = hwcso;
 515         vc5->dirty |= VC5_DIRTY_UNCOMPILED_FS;
 516 }
 517
 518 static void
 519 vc5_vp_state_bind(struct pipe_context *pctx, void *hwcso)
 520 {
 521         struct vc5_context *vc5 = vc5_context(pctx);
 522         vc5->prog.bind_vs = hwcso;
 523         vc5->dirty |= VC5_DIRTY_UNCOMPILED_VS;
 524 }
 525
 526 void
 527 vc5_program_init(struct pipe_context *pctx)
 528 {
 529         struct vc5_context *vc5 = vc5_context(pctx);
 530
 531         pctx->create_vs_state = vc5_shader_state_create;
 532         pctx->delete_vs_state = vc5_shader_state_delete;
 533
 534         pctx->create_fs_state = vc5_shader_state_create;
 535         pctx->delete_fs_state = vc5_shader_state_delete;
 536
 537         pctx->bind_fs_state = vc5_fp_state_bind;
 538         pctx->bind_vs_state = vc5_vp_state_bind;
 539
 540         vc5->fs_cache = _mesa_hash_table_create(pctx, fs_cache_hash,
 541                                                 fs_cache_compare);
 542         vc5->vs_cache = _mesa_hash_table_create(pctx, vs_cache_hash,
 543                                                 vs_cache_compare);
 544 }
 545
 546 void
 547 vc5_program_fini(struct pipe_context *pctx)
 548 {
 549         struct vc5_context *vc5 = vc5_context(pctx);
 550
 551         struct hash_entry *entry;
 552         hash_table_foreach(vc5->fs_cache, entry) {
 553                 struct vc5_compiled_shader *shader = entry->data;
 554                 vc5_bo_unreference(&shader->bo);
 555                 ralloc_free(shader);
 556                 _mesa_hash_table_remove(vc5->fs_cache, entry);
 557         }
 558
 559         hash_table_foreach(vc5->vs_cache, entry) {
 560                 struct vc5_compiled_shader *shader = entry->data;
 561                 vc5_bo_unreference(&shader->bo);
 562                 ralloc_free(shader);
 563                 _mesa_hash_table_remove(vc5->vs_cache, entry);
 564         }
 565 }