X-Git-Url: https://git.libre-soc.org/?p=mesa.git;a=blobdiff_plain;f=src%2Fmesa%2Fstate_tracker%2Fst_program.c;h=f052655efcee8d76ef5734eac9d8972dd634915d;hp=f1d3f7b3ce70e1d2acdb932d1baaa2bf5aafedfc;hb=500b0735c0caa293d22fb3728cb76d267a016834;hpb=b1c4c4c7f53cde52a8f3b3ec17cd66b89bf04199 diff --git a/src/mesa/state_tracker/st_program.c b/src/mesa/state_tracker/st_program.c index f1d3f7b3ce7..f052655efce 100644 --- a/src/mesa/state_tracker/st_program.c +++ b/src/mesa/state_tracker/st_program.c @@ -1,8 +1,8 @@ /************************************************************************** - * + * * Copyright 2007 VMware, Inc. * All Rights Reserved. - * + * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the * "Software"), to deal in the Software without restriction, including @@ -10,11 +10,11 @@ * distribute, sub license, and/or sell copies of the Software, and to * permit persons to whom the Software is furnished to do so, subject to * the following conditions: - * + * * The above copyright notice and this permission notice (including the * next paragraph) shall be included in all copies or substantial portions * of the Software. - * + * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. @@ -22,7 +22,7 @@ * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - * + * **************************************************************************/ /* * Authors: @@ -32,7 +32,7 @@ #include "main/errors.h" -#include "main/imports.h" + #include "main/hash.h" #include "main/mtypes.h" #include "program/prog_parameter.h" @@ -41,6 +41,8 @@ #include "program/programopt.h" #include "compiler/nir/nir.h" +#include "compiler/nir/nir_serialize.h" +#include "draw/draw_context.h" #include "pipe/p_context.h" #include "pipe/p_defines.h" @@ -51,6 +53,8 @@ #include "tgsi/tgsi_parse.h" #include "tgsi/tgsi_ureg.h" +#include "util/u_memory.h" + #include "st_debug.h" #include "st_cb_bitmap.h" #include "st_cb_drawpixels.h" @@ -62,9 +66,12 @@ #include "st_atifs_to_tgsi.h" #include "st_nir.h" #include "st_shader_cache.h" +#include "st_util.h" #include "cso_cache/cso_context.h" +static void +destroy_program_variants(struct st_context *st, struct gl_program *target); static void set_affected_state_flags(uint64_t *states, @@ -106,7 +113,7 @@ st_set_prog_affected_state_flags(struct gl_program *prog) switch (prog->info.stage) { case MESA_SHADER_VERTEX: - states = &((struct st_vertex_program*)prog)->affected_states; + states = &((struct st_program*)prog)->affected_states; *states = ST_NEW_VS_STATE | ST_NEW_RASTERIZER | @@ -123,7 +130,7 @@ st_set_prog_affected_state_flags(struct gl_program *prog) break; case MESA_SHADER_TESS_CTRL: - states = &(st_common_program(prog))->affected_states; + states = &(st_program(prog))->affected_states; *states = ST_NEW_TCS_STATE; @@ -138,7 +145,7 @@ st_set_prog_affected_state_flags(struct gl_program *prog) break; case MESA_SHADER_TESS_EVAL: - states = &(st_common_program(prog))->affected_states; + states = &(st_program(prog))->affected_states; *states = ST_NEW_TES_STATE | ST_NEW_RASTERIZER; @@ -154,7 +161,7 @@ st_set_prog_affected_state_flags(struct gl_program *prog) break; case MESA_SHADER_GEOMETRY: - states = &(st_common_program(prog))->affected_states; + states = &(st_program(prog))->affected_states; *states = ST_NEW_GS_STATE | ST_NEW_RASTERIZER; @@ -170,7 +177,7 @@ st_set_prog_affected_state_flags(struct gl_program *prog) break; case MESA_SHADER_FRAGMENT: - states = &((struct st_fragment_program*)prog)->affected_states; + states = &((struct st_program*)prog)->affected_states; /* gl_FragCoord and glDrawPixels always use constants. */ *states = ST_NEW_FS_STATE | @@ -188,7 +195,7 @@ st_set_prog_affected_state_flags(struct gl_program *prog) break; case MESA_SHADER_COMPUTE: - states = &((struct st_common_program*)prog)->affected_states; + states = &((struct st_program*)prog)->affected_states; *states = ST_NEW_CS_STATE; @@ -207,137 +214,41 @@ st_set_prog_affected_state_flags(struct gl_program *prog) } } -static void -delete_ir(struct pipe_shader_state *ir) -{ - if (ir->tokens) { - ureg_free_tokens(ir->tokens); - ir->tokens = NULL; - } - - /* Note: Any setup of ->ir.nir that has had pipe->create_*_state called on - * it has resulted in the driver taking ownership of the NIR. Those - * callers should be NULLing out the nir field in any pipe_shader_state - * that might have this called in order to indicate that. - * - * GLSL IR and ARB programs will have set gl_program->nir to the same - * shader as ir->ir.nir, so it will be freed by _mesa_delete_program(). - */ -} - -/** - * Delete a vertex program variant. Note the caller must unlink - * the variant from the linked list. - */ -static void -delete_vp_variant(struct st_context *st, struct st_vp_variant *vpv) -{ - if (vpv->driver_shader) { - if (st->has_shareable_shaders || vpv->key.st == st) { - cso_delete_vertex_shader(st->cso_context, vpv->driver_shader); - } else { - st_save_zombie_shader(vpv->key.st, PIPE_SHADER_VERTEX, - vpv->driver_shader); - } - } - - if (vpv->draw_shader) - draw_delete_vertex_shader( st->draw, vpv->draw_shader ); - - delete_ir(&vpv->tgsi); - - free( vpv ); -} - - - -/** - * Clean out any old compilations: - */ -void -st_release_vp_variants( struct st_context *st, - struct st_vertex_program *stvp ) -{ - struct st_vp_variant *vpv; - - for (vpv = stvp->variants; vpv; ) { - struct st_vp_variant *next = vpv->next; - delete_vp_variant(st, vpv); - vpv = next; - } - - stvp->variants = NULL; - - delete_ir(&stvp->tgsi); -} - - - -/** - * Delete a fragment program variant. Note the caller must unlink - * the variant from the linked list. - */ -static void -delete_fp_variant(struct st_context *st, struct st_fp_variant *fpv) -{ - if (fpv->driver_shader) { - if (st->has_shareable_shaders || fpv->key.st == st) { - cso_delete_fragment_shader(st->cso_context, fpv->driver_shader); - } else { - st_save_zombie_shader(fpv->key.st, PIPE_SHADER_FRAGMENT, - fpv->driver_shader); - } - } - - free(fpv); -} - - -/** - * Free all variants of a fragment program. - */ -void -st_release_fp_variants(struct st_context *st, struct st_fragment_program *stfp) -{ - struct st_fp_variant *fpv; - - for (fpv = stfp->variants; fpv; ) { - struct st_fp_variant *next = fpv->next; - delete_fp_variant(st, fpv); - fpv = next; - } - - stfp->variants = NULL; - - delete_ir(&stfp->tgsi); -} - /** - * Delete a basic program variant. Note the caller must unlink - * the variant from the linked list. + * Delete a shader variant. Note the caller must unlink the variant from + * the linked list. */ static void -delete_basic_variant(struct st_context *st, struct st_basic_variant *v, - GLenum target) +delete_variant(struct st_context *st, struct st_variant *v, GLenum target) { if (v->driver_shader) { - if (st->has_shareable_shaders || v->key.st == st) { + if (target == GL_VERTEX_PROGRAM_ARB && + ((struct st_common_variant*)v)->key.is_draw_shader) { + /* Draw shader. */ + draw_delete_vertex_shader(st->draw, v->driver_shader); + } else if (st->has_shareable_shaders || v->st == st) { /* The shader's context matches the calling context, or we * don't care. */ switch (target) { + case GL_VERTEX_PROGRAM_ARB: + st->pipe->delete_vs_state(st->pipe, v->driver_shader); + break; case GL_TESS_CONTROL_PROGRAM_NV: - cso_delete_tessctrl_shader(st->cso_context, v->driver_shader); + st->pipe->delete_tcs_state(st->pipe, v->driver_shader); break; case GL_TESS_EVALUATION_PROGRAM_NV: - cso_delete_tesseval_shader(st->cso_context, v->driver_shader); + st->pipe->delete_tes_state(st->pipe, v->driver_shader); break; case GL_GEOMETRY_PROGRAM_NV: - cso_delete_geometry_shader(st->cso_context, v->driver_shader); + st->pipe->delete_gs_state(st->pipe, v->driver_shader); + break; + case GL_FRAGMENT_PROGRAM_ARB: + st->pipe->delete_fs_state(st->pipe, v->driver_shader); break; case GL_COMPUTE_PROGRAM_NV: - cso_delete_compute_shader(st->cso_context, v->driver_shader); + st->pipe->delete_compute_state(st->pipe, v->driver_shader); break; default: unreachable("bad shader type in delete_basic_variant"); @@ -346,46 +257,118 @@ delete_basic_variant(struct st_context *st, struct st_basic_variant *v, /* We can't delete a shader with a context different from the one * that created it. Add it to the creating context's zombie list. */ - enum pipe_shader_type type; - switch (target) { - case GL_TESS_CONTROL_PROGRAM_NV: - type = PIPE_SHADER_TESS_CTRL; - break; - case GL_TESS_EVALUATION_PROGRAM_NV: - type = PIPE_SHADER_TESS_EVAL; - break; - case GL_GEOMETRY_PROGRAM_NV: - type = PIPE_SHADER_GEOMETRY; - break; - default: - unreachable(""); - } - st_save_zombie_shader(v->key.st, type, v->driver_shader); + enum pipe_shader_type type = + pipe_shader_type_from_mesa(_mesa_program_enum_to_shader_stage(target)); + + st_save_zombie_shader(v->st, type, v->driver_shader); } } free(v); } +static void +st_unbind_program(struct st_context *st, struct st_program *p) +{ + /* Unbind the shader in cso_context and re-bind in st/mesa. */ + switch (p->Base.info.stage) { + case MESA_SHADER_VERTEX: + cso_set_vertex_shader_handle(st->cso_context, NULL); + st->dirty |= ST_NEW_VS_STATE; + break; + case MESA_SHADER_TESS_CTRL: + cso_set_tessctrl_shader_handle(st->cso_context, NULL); + st->dirty |= ST_NEW_TCS_STATE; + break; + case MESA_SHADER_TESS_EVAL: + cso_set_tesseval_shader_handle(st->cso_context, NULL); + st->dirty |= ST_NEW_TES_STATE; + break; + case MESA_SHADER_GEOMETRY: + cso_set_geometry_shader_handle(st->cso_context, NULL); + st->dirty |= ST_NEW_GS_STATE; + break; + case MESA_SHADER_FRAGMENT: + cso_set_fragment_shader_handle(st->cso_context, NULL); + st->dirty |= ST_NEW_FS_STATE; + break; + case MESA_SHADER_COMPUTE: + cso_set_compute_shader_handle(st->cso_context, NULL); + st->dirty |= ST_NEW_CS_STATE; + break; + default: + unreachable("invalid shader type"); + } +} /** * Free all basic program variants. */ void -st_release_basic_variants(struct st_context *st, struct st_common_program *p) +st_release_variants(struct st_context *st, struct st_program *p) { - struct st_basic_variant *v; + struct st_variant *v; + + /* If we are releasing shaders, re-bind them, because we don't + * know which shaders are bound in the driver. + */ + if (p->variants) + st_unbind_program(st, p); for (v = p->variants; v; ) { - struct st_basic_variant *next = v->next; - delete_basic_variant(st, v, p->Base.Target); + struct st_variant *next = v->next; + delete_variant(st, v, p->Base.Target); v = next; } p->variants = NULL; - delete_ir(&p->tgsi); + + if (p->state.tokens) { + ureg_free_tokens(p->state.tokens); + p->state.tokens = NULL; + } + + /* Note: Any setup of ->ir.nir that has had pipe->create_*_state called on + * it has resulted in the driver taking ownership of the NIR. Those + * callers should be NULLing out the nir field in any pipe_shader_state + * that might have this called in order to indicate that. + * + * GLSL IR and ARB programs will have set gl_program->nir to the same + * shader as ir->ir.nir, so it will be freed by _mesa_delete_program(). + */ } +/** + * Free all basic program variants and unref program. + */ +void +st_release_program(struct st_context *st, struct st_program **p) +{ + if (!*p) + return; + + destroy_program_variants(st, &((*p)->Base)); + st_reference_prog(st, p, NULL); +} + +void +st_finalize_nir_before_variants(struct nir_shader *nir) +{ + NIR_PASS_V(nir, nir_opt_access); + + NIR_PASS_V(nir, nir_split_var_copies); + NIR_PASS_V(nir, nir_lower_var_copies); + if (nir->options->lower_all_io_to_temps || + nir->options->lower_all_io_to_elements || + nir->info.stage == MESA_SHADER_VERTEX || + nir->info.stage == MESA_SHADER_GEOMETRY) { + NIR_PASS_V(nir, nir_lower_io_arrays_to_elements_no_indirects, false); + } else if (nir->info.stage == MESA_SHADER_FRAGMENT) { + NIR_PASS_V(nir, nir_lower_io_arrays_to_elements_no_indirects, true); + } + + st_nir_assign_vs_in_locations(nir); +} /** * Translate ARB (asm) program to NIR @@ -394,6 +377,7 @@ static nir_shader * st_translate_prog_to_nir(struct st_context *st, struct gl_program *prog, gl_shader_stage stage) { + struct pipe_screen *screen = st->pipe->screen; const struct gl_shader_compiler_options *options = &st->ctx->Const.ShaderCompilerOptions[stage]; @@ -402,20 +386,28 @@ st_translate_prog_to_nir(struct st_context *st, struct gl_program *prog, NIR_PASS_V(nir, nir_lower_regs_to_ssa); /* turn registers into SSA */ nir_validate_shader(nir, "after st/ptn lower_regs_to_ssa"); - NIR_PASS_V(nir, st_nir_lower_wpos_ytransform, prog, st->pipe->screen); + NIR_PASS_V(nir, st_nir_lower_wpos_ytransform, prog, screen); NIR_PASS_V(nir, nir_lower_system_values); + NIR_PASS_V(nir, nir_lower_compute_system_values, NULL); /* Optimise NIR */ NIR_PASS_V(nir, nir_opt_constant_folding); st_nir_opts(nir); - nir_validate_shader(nir, "after st/ptn NIR opts"); + st_finalize_nir_before_variants(nir); + + if (st->allow_st_finalize_nir_twice) + st_finalize_nir(st, prog, NULL, nir, true); + + nir_validate_shader(nir, "after st/glsl finalize_nir"); return nir; } void -st_prepare_vertex_program(struct st_vertex_program *stvp) +st_prepare_vertex_program(struct st_program *stp) { + struct st_vertex_program *stvp = (struct st_vertex_program *)stp; + stvp->num_inputs = 0; memset(stvp->input_to_index, ~0, sizeof(stvp->input_to_index)); memset(stvp->result_to_output, ~0, sizeof(stvp->result_to_output)); @@ -424,12 +416,12 @@ st_prepare_vertex_program(struct st_vertex_program *stvp) * and TGSI generic input indexes, plus input attrib semantic info. */ for (unsigned attr = 0; attr < VERT_ATTRIB_MAX; attr++) { - if ((stvp->Base.info.inputs_read & BITFIELD64_BIT(attr)) != 0) { + if ((stp->Base.info.inputs_read & BITFIELD64_BIT(attr)) != 0) { stvp->input_to_index[attr] = stvp->num_inputs; stvp->index_to_input[stvp->num_inputs] = attr; stvp->num_inputs++; - if ((stvp->Base.DualSlotInputs & BITFIELD64_BIT(attr)) != 0) { + if ((stp->Base.DualSlotInputs & BITFIELD64_BIT(attr)) != 0) { /* add placeholder for second part of a double attribute */ stvp->index_to_input[stvp->num_inputs] = ST_DOUBLE_ATTRIB_PLACEHOLDER; stvp->num_inputs++; @@ -443,7 +435,7 @@ st_prepare_vertex_program(struct st_vertex_program *stvp) /* Compute mapping of vertex program outputs to slots. */ unsigned num_outputs = 0; for (unsigned attr = 0; attr < VARYING_SLOT_MAX; attr++) { - if (stvp->Base.info.outputs_written & BITFIELD64_BIT(attr)) + if (stp->Base.info.outputs_written & BITFIELD64_BIT(attr)) stvp->result_to_output[attr] = num_outputs++; } /* pre-setup potentially unused edgeflag output */ @@ -468,11 +460,8 @@ st_translate_stream_output_info(struct gl_program *prog) } /* Translate stream output info. */ - struct pipe_stream_output_info *so_info = NULL; - if (prog->info.stage == MESA_SHADER_VERTEX) - so_info = &((struct st_vertex_program*)prog)->tgsi.stream_output; - else - so_info = &((struct st_common_program*)prog)->tgsi.stream_output; + struct pipe_stream_output_info *so_info = + &((struct st_program*)prog)->state.stream_output; for (unsigned i = 0; i < info->NumOutputs; i++) { so_info->output[i].register_index = @@ -495,7 +484,7 @@ st_translate_stream_output_info(struct gl_program *prog) */ bool st_translate_vertex_program(struct st_context *st, - struct st_vertex_program *stvp) + struct st_program *stp) { struct ureg_program *ureg; enum pipe_error error; @@ -504,31 +493,73 @@ st_translate_vertex_program(struct st_context *st, ubyte output_semantic_name[VARYING_SLOT_MAX] = {0}; ubyte output_semantic_index[VARYING_SLOT_MAX] = {0}; - if (stvp->Base.arb.IsPositionInvariant) - _mesa_insert_mvp_code(st->ctx, &stvp->Base); - - st_prepare_vertex_program(stvp); + if (stp->Base.arb.IsPositionInvariant) + _mesa_insert_mvp_code(st->ctx, &stp->Base); /* ARB_vp: */ - if (!stvp->glsl_to_tgsi) { - _mesa_remove_output_reads(&stvp->Base, PROGRAM_OUTPUT); + if (!stp->glsl_to_tgsi) { + _mesa_remove_output_reads(&stp->Base, PROGRAM_OUTPUT); /* This determines which states will be updated when the assembly * shader is bound. */ - stvp->affected_states = ST_NEW_VS_STATE | + stp->affected_states = ST_NEW_VS_STATE | ST_NEW_RASTERIZER | ST_NEW_VERTEX_ARRAYS; - if (stvp->Base.Parameters->NumParameters) - stvp->affected_states |= ST_NEW_VS_CONSTANTS; + if (stp->Base.Parameters->NumParameters) + stp->affected_states |= ST_NEW_VS_CONSTANTS; - /* No samplers are allowed in ARB_vp. */ + /* Translate to NIR if preferred. */ + if (PIPE_SHADER_IR_NIR == + st->pipe->screen->get_shader_param(st->pipe->screen, + PIPE_SHADER_VERTEX, + PIPE_SHADER_CAP_PREFERRED_IR)) { + assert(!stp->glsl_to_tgsi); + + if (stp->Base.nir) + ralloc_free(stp->Base.nir); + + if (stp->serialized_nir) { + free(stp->serialized_nir); + stp->serialized_nir = NULL; + } + + stp->state.type = PIPE_SHADER_IR_NIR; + stp->Base.nir = st_translate_prog_to_nir(st, &stp->Base, + MESA_SHADER_VERTEX); + + /* We must update stp->Base.info after translation and before + * st_prepare_vertex_program is called, because inputs_read + * may become outdated after NIR optimization passes. + * + * For ffvp/ARB_vp inputs_read is populated based + * on declared attributes without taking their usage into + * consideration. When creating shader variants we expect + * that their inputs_read would match the base ones for + * input mapping to work properly. + */ + nir_shader_gather_info(stp->Base.nir, + nir_shader_get_entrypoint(stp->Base.nir)); + st_nir_assign_vs_in_locations(stp->Base.nir); + stp->Base.info = stp->Base.nir->info; + + /* For st_draw_feedback, we need to generate TGSI too if draw doesn't + * use LLVM. + */ + /* TODO: Draw can't handle lowered IO. */ + if (draw_has_llvm() && !stp->Base.info.io_lowered) { + st_prepare_vertex_program(stp); + return true; + } + } } + st_prepare_vertex_program(stp); + /* Get semantic names and indices. */ for (attr = 0; attr < VARYING_SLOT_MAX; attr++) { - if (stvp->Base.info.outputs_written & BITFIELD64_BIT(attr)) { + if (stp->Base.info.outputs_written & BITFIELD64_BIT(attr)) { unsigned slot = num_outputs++; unsigned semantic_name, semantic_index; tgsi_get_gl_varying_semantic(attr, st->needs_texcoord_semantic, @@ -545,25 +576,22 @@ st_translate_vertex_program(struct st_context *st, if (ureg == NULL) return false; - if (stvp->Base.info.clip_distance_array_size) - ureg_property(ureg, TGSI_PROPERTY_NUM_CLIPDIST_ENABLED, - stvp->Base.info.clip_distance_array_size); - if (stvp->Base.info.cull_distance_array_size) - ureg_property(ureg, TGSI_PROPERTY_NUM_CULLDIST_ENABLED, - stvp->Base.info.cull_distance_array_size); + ureg_setup_shader_info(ureg, &stp->Base.info); if (ST_DEBUG & DEBUG_MESA) { - _mesa_print_program(&stvp->Base); - _mesa_print_program_parameters(st->ctx, &stvp->Base); + _mesa_print_program(&stp->Base); + _mesa_print_program_parameters(st->ctx, &stp->Base); debug_printf("\n"); } - if (stvp->glsl_to_tgsi) { + struct st_vertex_program *stvp = (struct st_vertex_program *)stp; + + if (stp->glsl_to_tgsi) { error = st_translate_program(st->ctx, PIPE_SHADER_VERTEX, ureg, - stvp->glsl_to_tgsi, - &stvp->Base, + stp->glsl_to_tgsi, + &stp->Base, /* inputs */ stvp->num_inputs, stvp->input_to_index, @@ -577,14 +605,14 @@ st_translate_vertex_program(struct st_context *st, output_semantic_name, output_semantic_index); - st_translate_stream_output_info(&stvp->Base); + st_translate_stream_output_info(&stp->Base); - free_glsl_to_tgsi_visitor(stvp->glsl_to_tgsi); + free_glsl_to_tgsi_visitor(stp->glsl_to_tgsi); } else error = st_translate_mesa_program(st->ctx, PIPE_SHADER_VERTEX, ureg, - &stvp->Base, + &stp->Base, /* inputs */ stvp->num_inputs, stvp->input_to_index, @@ -599,87 +627,158 @@ st_translate_vertex_program(struct st_context *st, if (error) { debug_printf("%s: failed to translate Mesa program:\n", __func__); - _mesa_print_program(&stvp->Base); + _mesa_print_program(&stp->Base); debug_assert(0); return false; } - stvp->tgsi.tokens = ureg_get_tokens(ureg, &stvp->num_tgsi_tokens); + stp->state.tokens = ureg_get_tokens(ureg, NULL); ureg_destroy(ureg); - if (stvp->glsl_to_tgsi) { - stvp->glsl_to_tgsi = NULL; - st_store_ir_in_disk_cache(st, &stvp->Base, false); + if (stp->glsl_to_tgsi) { + stp->glsl_to_tgsi = NULL; + st_store_ir_in_disk_cache(st, &stp->Base, false); } - /* Translate to NIR. - * - * This must be done after the translation to TGSI is done, because - * we'll pass the NIR shader to the driver and the TGSI version to - * the draw module for the select/feedback/rasterpos code. - */ - if (st->pipe->screen->get_shader_param(st->pipe->screen, - PIPE_SHADER_VERTEX, - PIPE_SHADER_CAP_PREFERRED_IR)) { - assert(!stvp->glsl_to_tgsi); - - nir_shader *nir = - st_translate_prog_to_nir(st, &stvp->Base, MESA_SHADER_VERTEX); - - if (stvp->tgsi.ir.nir) - ralloc_free(stvp->tgsi.ir.nir); - stvp->tgsi.type = PIPE_SHADER_IR_NIR; - stvp->tgsi.ir.nir = nir; - stvp->Base.nir = nir; - return true; + return stp->state.tokens != NULL; +} + +static struct nir_shader * +get_nir_shader(struct st_context *st, struct st_program *stp) +{ + if (stp->Base.nir) { + nir_shader *nir = stp->Base.nir; + + /* The first shader variant takes ownership of NIR, so that there is + * no cloning. Additional shader variants are always generated from + * serialized NIR to save memory. + */ + stp->Base.nir = NULL; + assert(stp->serialized_nir && stp->serialized_nir_size); + return nir; } - return stvp->tgsi.tokens != NULL; + struct blob_reader blob_reader; + const struct nir_shader_compiler_options *options = + st->ctx->Const.ShaderCompilerOptions[stp->Base.info.stage].NirOptions; + + blob_reader_init(&blob_reader, stp->serialized_nir, stp->serialized_nir_size); + return nir_deserialize(NULL, options, &blob_reader); +} + +static void +lower_ucp(struct st_context *st, + struct nir_shader *nir, + unsigned ucp_enables, + struct gl_program_parameter_list *params) +{ + if (nir->info.outputs_written & VARYING_BIT_CLIP_DIST0) + NIR_PASS_V(nir, nir_lower_clip_disable, ucp_enables); + else { + struct pipe_screen *screen = st->pipe->screen; + bool can_compact = screen->get_param(screen, + PIPE_CAP_NIR_COMPACT_ARRAYS); + bool use_eye = st->ctx->_Shader->CurrentProgram[MESA_SHADER_VERTEX] != NULL; + + gl_state_index16 clipplane_state[MAX_CLIP_PLANES][STATE_LENGTH]; + for (int i = 0; i < MAX_CLIP_PLANES; ++i) { + if (use_eye) { + clipplane_state[i][0] = STATE_CLIPPLANE; + clipplane_state[i][1] = i; + } else { + clipplane_state[i][0] = STATE_INTERNAL; + clipplane_state[i][1] = STATE_CLIP_INTERNAL; + clipplane_state[i][2] = i; + } + _mesa_add_state_reference(params, clipplane_state[i]); + } + + if (nir->info.stage == MESA_SHADER_VERTEX) { + NIR_PASS_V(nir, nir_lower_clip_vs, ucp_enables, + true, can_compact, clipplane_state); + } else if (nir->info.stage == MESA_SHADER_GEOMETRY) { + NIR_PASS_V(nir, nir_lower_clip_gs, ucp_enables, + can_compact, clipplane_state); + } + + NIR_PASS_V(nir, nir_lower_io_to_temporaries, + nir_shader_get_entrypoint(nir), true, false); + NIR_PASS_V(nir, nir_lower_global_vars_to_local); + } } static const gl_state_index16 depth_range_state[STATE_LENGTH] = { STATE_DEPTH_RANGE }; -static struct st_vp_variant * +static struct st_common_variant * st_create_vp_variant(struct st_context *st, - struct st_vertex_program *stvp, - const struct st_vp_variant_key *key) + struct st_program *stvp, + const struct st_common_variant_key *key) { - struct st_vp_variant *vpv = CALLOC_STRUCT(st_vp_variant); + struct st_common_variant *vpv = CALLOC_STRUCT(st_common_variant); struct pipe_context *pipe = st->pipe; + struct pipe_shader_state state = {0}; + + static const gl_state_index16 point_size_state[STATE_LENGTH] = + { STATE_INTERNAL, STATE_POINT_SIZE_CLAMPED, 0 }; struct gl_program_parameter_list *params = stvp->Base.Parameters; vpv->key = *key; - vpv->tgsi.stream_output = stvp->tgsi.stream_output; - vpv->num_inputs = stvp->num_inputs; - /* When generating a NIR program, we usually don't have TGSI tokens. - * However, we do create them for ARB_vertex_program / fixed-function VS - * programs which we may need to use with the draw module for legacy - * feedback/select emulation. If they exist, copy them. - */ - if (stvp->tgsi.tokens) - vpv->tgsi.tokens = tgsi_dup_tokens(stvp->tgsi.tokens); - - if (stvp->tgsi.type == PIPE_SHADER_IR_NIR) { - vpv->tgsi.type = PIPE_SHADER_IR_NIR; - vpv->tgsi.ir.nir = nir_shader_clone(NULL, stvp->tgsi.ir.nir); - if (key->clamp_color) - NIR_PASS_V(vpv->tgsi.ir.nir, nir_lower_clamp_color_outputs); + state.stream_output = stvp->state.stream_output; + + if (stvp->state.type == PIPE_SHADER_IR_NIR && + (!key->is_draw_shader || + /* TODO: Draw can't handle lowered IO. */ + (draw_has_llvm() && !stvp->Base.info.io_lowered))) { + bool finalize = false; + + state.type = PIPE_SHADER_IR_NIR; + state.ir.nir = get_nir_shader(st, stvp); + if (key->clamp_color) { + NIR_PASS_V(state.ir.nir, nir_lower_clamp_color_outputs); + finalize = true; + } if (key->passthrough_edgeflags) { - NIR_PASS_V(vpv->tgsi.ir.nir, nir_lower_passthrough_edgeflags); - vpv->num_inputs++; + NIR_PASS_V(state.ir.nir, nir_lower_passthrough_edgeflags); + finalize = true; + } + + if (key->lower_point_size) { + _mesa_add_state_reference(params, point_size_state); + NIR_PASS_V(state.ir.nir, nir_lower_point_size_mov, + point_size_state); + finalize = true; + } + + if (key->lower_ucp) { + lower_ucp(st, state.ir.nir, key->lower_ucp, params); + finalize = true; } - st_finalize_nir(st, &stvp->Base, stvp->shader_program, - vpv->tgsi.ir.nir); + if (finalize || !st->allow_st_finalize_nir_twice) { + st_finalize_nir(st, &stvp->Base, stvp->shader_program, state.ir.nir, + true); + + /* Some of the lowering above may have introduced new varyings */ + nir_shader_gather_info(state.ir.nir, + nir_shader_get_entrypoint(state.ir.nir)); + } + + if (ST_DEBUG & DEBUG_PRINT_IR) + nir_print_shader(state.ir.nir, stderr); + + if (key->is_draw_shader) + vpv->base.driver_shader = draw_create_vertex_shader(st->draw, &state); + else + vpv->base.driver_shader = pipe->create_vs_state(pipe, &state); - vpv->driver_shader = pipe->create_vs_state(pipe, &vpv->tgsi); - /* driver takes ownership of IR: */ - vpv->tgsi.ir.nir = NULL; return vpv; } + state.type = PIPE_SHADER_IR_TGSI; + state.tokens = tgsi_dup_tokens(stvp->state.tokens); + /* Emulate features. */ if (key->clamp_color || key->passthrough_edgeflags) { const struct tgsi_token *tokens; @@ -687,16 +786,14 @@ st_create_vp_variant(struct st_context *st, (key->clamp_color ? TGSI_EMU_CLAMP_COLOR_OUTPUTS : 0) | (key->passthrough_edgeflags ? TGSI_EMU_PASSTHROUGH_EDGEFLAG : 0); - tokens = tgsi_emulate(vpv->tgsi.tokens, flags); + tokens = tgsi_emulate(state.tokens, flags); if (tokens) { - tgsi_free_tokens(vpv->tgsi.tokens); - vpv->tgsi.tokens = tokens; - - if (key->passthrough_edgeflags) - vpv->num_inputs++; - } else + tgsi_free_tokens(state.tokens); + state.tokens = tokens; + } else { fprintf(stderr, "mesa: cannot emulate deprecated features\n"); + } } if (key->lower_depth_clamp) { @@ -704,19 +801,25 @@ st_create_vp_variant(struct st_context *st, _mesa_add_state_reference(params, depth_range_state); const struct tgsi_token *tokens; - tokens = st_tgsi_lower_depth_clamp(vpv->tgsi.tokens, depth_range_const, + tokens = st_tgsi_lower_depth_clamp(state.tokens, depth_range_const, key->clip_negative_one_to_one); - if (tokens != vpv->tgsi.tokens) - tgsi_free_tokens(vpv->tgsi.tokens); - vpv->tgsi.tokens = tokens; + if (tokens != state.tokens) + tgsi_free_tokens(state.tokens); + state.tokens = tokens; } - if (ST_DEBUG & DEBUG_TGSI) { - tgsi_dump(vpv->tgsi.tokens, 0); - debug_printf("\n"); + if (ST_DEBUG & DEBUG_PRINT_IR) + tgsi_dump(state.tokens, 0); + + if (key->is_draw_shader) + vpv->base.driver_shader = draw_create_vertex_shader(st->draw, &state); + else + vpv->base.driver_shader = pipe->create_vs_state(pipe, &state); + + if (state.tokens) { + tgsi_free_tokens(state.tokens); } - vpv->driver_shader = pipe->create_vs_state(pipe, &vpv->tgsi); return vpv; } @@ -724,15 +827,17 @@ st_create_vp_variant(struct st_context *st, /** * Find/create a vertex program variant. */ -struct st_vp_variant * +struct st_common_variant * st_get_vp_variant(struct st_context *st, - struct st_vertex_program *stvp, - const struct st_vp_variant_key *key) + struct st_program *stp, + const struct st_common_variant_key *key) { - struct st_vp_variant *vpv; + struct st_vertex_program *stvp = (struct st_vertex_program *)stp; + struct st_common_variant *vpv; /* Search for existing variant */ - for (vpv = stvp->variants; vpv; vpv = vpv->next) { + for (vpv = st_common_variant(stp->variants); vpv; + vpv = st_common_variant(vpv->base.next)) { if (memcmp(&vpv->key, key, sizeof(*key)) == 0) { break; } @@ -740,18 +845,21 @@ st_get_vp_variant(struct st_context *st, if (!vpv) { /* create now */ - vpv = st_create_vp_variant(st, stvp, key); + vpv = st_create_vp_variant(st, stp, key); if (vpv) { - for (unsigned index = 0; index < vpv->num_inputs; ++index) { - unsigned attr = stvp->index_to_input[index]; - if (attr == ST_DOUBLE_ATTRIB_PLACEHOLDER) - continue; - vpv->vert_attrib_mask |= 1u << attr; - } + vpv->base.st = key->st; + + unsigned num_inputs = stvp->num_inputs + key->passthrough_edgeflags; + for (unsigned index = 0; index < num_inputs; ++index) { + unsigned attr = stvp->index_to_input[index]; + if (attr == ST_DOUBLE_ATTRIB_PLACEHOLDER) + continue; + vpv->vert_attrib_mask |= 1u << attr; + } /* insert into list */ - vpv->next = stvp->variants; - stvp->variants = vpv; + vpv->base.next = stp->variants; + stp->variants = &vpv->base; } } @@ -764,7 +872,7 @@ st_get_vp_variant(struct st_context *st, */ bool st_translate_fragment_program(struct st_context *st, - struct st_fragment_program *stfp) + struct st_program *stfp) { /* Non-GLSL programs: */ if (!stfp->glsl_to_tgsi) { @@ -794,16 +902,20 @@ st_translate_fragment_program(struct st_context *st, /* Translate to NIR. */ if (!stfp->ati_fs && + PIPE_SHADER_IR_NIR == st->pipe->screen->get_shader_param(st->pipe->screen, PIPE_SHADER_FRAGMENT, PIPE_SHADER_CAP_PREFERRED_IR)) { nir_shader *nir = st_translate_prog_to_nir(st, &stfp->Base, MESA_SHADER_FRAGMENT); - if (stfp->tgsi.ir.nir) - ralloc_free(stfp->tgsi.ir.nir); - stfp->tgsi.type = PIPE_SHADER_IR_NIR; - stfp->tgsi.ir.nir = nir; + if (stfp->Base.nir) + ralloc_free(stfp->Base.nir); + if (stfp->serialized_nir) { + free(stfp->serialized_nir); + stfp->serialized_nir = NULL; + } + stfp->state.type = PIPE_SHADER_IR_NIR; stfp->Base.nir = nir; return true; } @@ -1041,6 +1153,8 @@ st_translate_fragment_program(struct st_context *st, if (ureg == NULL) return false; + ureg_setup_shader_info(ureg, &stfp->Base.info); + if (ST_DEBUG & DEBUG_MESA) { _mesa_print_program(&stfp->Base); _mesa_print_program_parameters(st->ctx, &stfp->Base); @@ -1049,29 +1163,6 @@ st_translate_fragment_program(struct st_context *st, if (write_all == GL_TRUE) ureg_property(ureg, TGSI_PROPERTY_FS_COLOR0_WRITES_ALL_CBUFS, 1); - if (stfp->Base.info.fs.depth_layout != FRAG_DEPTH_LAYOUT_NONE) { - switch (stfp->Base.info.fs.depth_layout) { - case FRAG_DEPTH_LAYOUT_ANY: - ureg_property(ureg, TGSI_PROPERTY_FS_DEPTH_LAYOUT, - TGSI_FS_DEPTH_LAYOUT_ANY); - break; - case FRAG_DEPTH_LAYOUT_GREATER: - ureg_property(ureg, TGSI_PROPERTY_FS_DEPTH_LAYOUT, - TGSI_FS_DEPTH_LAYOUT_GREATER); - break; - case FRAG_DEPTH_LAYOUT_LESS: - ureg_property(ureg, TGSI_PROPERTY_FS_DEPTH_LAYOUT, - TGSI_FS_DEPTH_LAYOUT_LESS); - break; - case FRAG_DEPTH_LAYOUT_UNCHANGED: - ureg_property(ureg, TGSI_PROPERTY_FS_DEPTH_LAYOUT, - TGSI_FS_DEPTH_LAYOUT_UNCHANGED); - break; - default: - assert(0); - } - } - if (stfp->glsl_to_tgsi) { st_translate_program(st->ctx, PIPE_SHADER_FRAGMENT, @@ -1124,7 +1215,7 @@ st_translate_fragment_program(struct st_context *st, fs_output_semantic_name, fs_output_semantic_index); - stfp->tgsi.tokens = ureg_get_tokens(ureg, &stfp->num_tgsi_tokens); + stfp->state.tokens = ureg_get_tokens(ureg, NULL); ureg_destroy(ureg); if (stfp->glsl_to_tgsi) { @@ -1132,17 +1223,17 @@ st_translate_fragment_program(struct st_context *st, st_store_ir_in_disk_cache(st, &stfp->Base, false); } - return stfp->tgsi.tokens != NULL; + return stfp->state.tokens != NULL; } static struct st_fp_variant * st_create_fp_variant(struct st_context *st, - struct st_fragment_program *stfp, + struct st_program *stfp, const struct st_fp_variant_key *key) { struct pipe_context *pipe = st->pipe; struct st_fp_variant *variant = CALLOC_STRUCT(st_fp_variant); - struct pipe_shader_state tgsi = {0}; + struct pipe_shader_state state = {0}; struct gl_program_parameter_list *params = stfp->Base.Parameters; static const gl_state_index16 texcoord_state[STATE_LENGTH] = { STATE_INTERNAL, STATE_CURRENT_ATTRIB, VERT_ATTRIB_TEX0 }; @@ -1156,26 +1247,40 @@ st_create_fp_variant(struct st_context *st, if (!variant) return NULL; - if (stfp->tgsi.type == PIPE_SHADER_IR_NIR) { - tgsi.type = PIPE_SHADER_IR_NIR; - tgsi.ir.nir = nir_shader_clone(NULL, stfp->tgsi.ir.nir); + if (stfp->state.type == PIPE_SHADER_IR_NIR) { + bool finalize = false; - if (key->clamp_color) - NIR_PASS_V(tgsi.ir.nir, nir_lower_clamp_color_outputs); + state.type = PIPE_SHADER_IR_NIR; + state.ir.nir = get_nir_shader(st, stfp); - if (key->lower_flatshade) - NIR_PASS_V(tgsi.ir.nir, nir_lower_flatshade); + if (key->clamp_color) { + NIR_PASS_V(state.ir.nir, nir_lower_clamp_color_outputs); + finalize = true; + } + + if (key->lower_flatshade) { + NIR_PASS_V(state.ir.nir, nir_lower_flatshade); + finalize = true; + } if (key->lower_alpha_func != COMPARE_FUNC_NEVER) { _mesa_add_state_reference(params, alpha_ref_state); - NIR_PASS_V(tgsi.ir.nir, nir_lower_alpha_test, key->lower_alpha_func, + NIR_PASS_V(state.ir.nir, nir_lower_alpha_test, key->lower_alpha_func, false, alpha_ref_state); + finalize = true; + } + + if (key->lower_two_sided_color) { + bool face_sysval = st->ctx->Const.GLSLFrontFacingIsSysVal; + NIR_PASS_V(state.ir.nir, nir_lower_two_sided_color, face_sysval); + finalize = true; } if (key->persample_shading) { - nir_shader *shader = tgsi.ir.nir; - nir_foreach_variable(var, &shader->inputs) + nir_shader *shader = state.ir.nir; + nir_foreach_shader_in_variable(var, shader) var->data.sample = true; + finalize = true; } assert(!(key->bitmap && key->drawpixels)); @@ -1188,7 +1293,8 @@ st_create_fp_variant(struct st_context *st, options.sampler = variant->bitmap_sampler; options.swizzle_xxxx = st->bitmap.tex_format == PIPE_FORMAT_R8_UNORM; - NIR_PASS_V(tgsi.ir.nir, nir_lower_bitmap, &options); + NIR_PASS_V(state.ir.nir, nir_lower_bitmap, &options); + finalize = true; } /* glDrawPixels (color only) */ @@ -1221,12 +1327,17 @@ st_create_fp_variant(struct st_context *st, memcpy(options.texcoord_state_tokens, texcoord_state, sizeof(options.texcoord_state_tokens)); - NIR_PASS_V(tgsi.ir.nir, nir_lower_drawpixels, &options); + NIR_PASS_V(state.ir.nir, nir_lower_drawpixels, &options); + finalize = true; } if (unlikely(key->external.lower_nv12 || key->external.lower_iyuv || key->external.lower_xy_uxvx || key->external.lower_yx_xuxv || key->external.lower_ayuv || key->external.lower_xyuv)) { + + st_nir_lower_samplers(pipe->screen, state.ir.nir, + stfp->shader_program, &stfp->Base); + nir_lower_tex_options options = {0}; options.lower_y_uv_external = key->external.lower_nv12; options.lower_y_u_v_external = key->external.lower_iyuv; @@ -1234,41 +1345,56 @@ st_create_fp_variant(struct st_context *st, options.lower_yx_xuxv_external = key->external.lower_yx_xuxv; options.lower_ayuv_external = key->external.lower_ayuv; options.lower_xyuv_external = key->external.lower_xyuv; - NIR_PASS_V(tgsi.ir.nir, nir_lower_tex, &options); + NIR_PASS_V(state.ir.nir, nir_lower_tex, &options); + finalize = true; } - st_finalize_nir(st, &stfp->Base, stfp->shader_program, tgsi.ir.nir); + if (finalize || !st->allow_st_finalize_nir_twice) { + st_finalize_nir(st, &stfp->Base, stfp->shader_program, state.ir.nir, + false); + } + /* This pass needs to happen *after* nir_lower_sampler */ if (unlikely(key->external.lower_nv12 || key->external.lower_iyuv || - key->external.lower_xy_uxvx || key->external.lower_yx_xuxv)) { - /* This pass needs to happen *after* nir_lower_sampler */ - NIR_PASS_V(tgsi.ir.nir, st_nir_lower_tex_src_plane, + key->external.lower_xy_uxvx || key->external.lower_yx_xuxv || + key->external.lower_ayuv || key->external.lower_xyuv)) { + NIR_PASS_V(state.ir.nir, st_nir_lower_tex_src_plane, ~stfp->Base.SamplersUsed, key->external.lower_nv12 || key->external.lower_xy_uxvx || key->external.lower_yx_xuxv, key->external.lower_iyuv); + finalize = true; } - /* Some of the lowering above may have introduced new varyings */ - nir_shader_gather_info(tgsi.ir.nir, - nir_shader_get_entrypoint(tgsi.ir.nir)); + if (finalize || !st->allow_st_finalize_nir_twice) { + /* Some of the lowering above may have introduced new varyings */ + nir_shader_gather_info(state.ir.nir, + nir_shader_get_entrypoint(state.ir.nir)); + + struct pipe_screen *screen = pipe->screen; + if (screen->finalize_nir) + screen->finalize_nir(screen, state.ir.nir, false); + } - variant->driver_shader = pipe->create_fs_state(pipe, &tgsi); + if (ST_DEBUG & DEBUG_PRINT_IR) + nir_print_shader(state.ir.nir, stderr); + + variant->base.driver_shader = pipe->create_fs_state(pipe, &state); variant->key = *key; return variant; } - tgsi.tokens = stfp->tgsi.tokens; + state.tokens = stfp->state.tokens; assert(!(key->bitmap && key->drawpixels)); /* Fix texture targets and add fog for ATI_fs */ if (stfp->ati_fs) { - const struct tgsi_token *tokens = st_fixup_atifs(tgsi.tokens, key); + const struct tgsi_token *tokens = st_fixup_atifs(state.tokens, key); if (tokens) - tgsi.tokens = tokens; + state.tokens = tokens; else fprintf(stderr, "mesa: cannot post-process ATI_fs\n"); } @@ -1280,12 +1406,12 @@ st_create_fp_variant(struct st_context *st, (key->clamp_color ? TGSI_EMU_CLAMP_COLOR_OUTPUTS : 0) | (key->persample_shading ? TGSI_EMU_FORCE_PERSAMPLE_INTERP : 0); - tokens = tgsi_emulate(tgsi.tokens, flags); + tokens = tgsi_emulate(state.tokens, flags); if (tokens) { - if (tgsi.tokens != stfp->tgsi.tokens) - tgsi_free_tokens(tgsi.tokens); - tgsi.tokens = tokens; + if (state.tokens != stfp->state.tokens) + tgsi_free_tokens(state.tokens); + state.tokens = tokens; } else fprintf(stderr, "mesa: cannot emulate deprecated features\n"); } @@ -1296,7 +1422,7 @@ st_create_fp_variant(struct st_context *st, variant->bitmap_sampler = ffs(~stfp->Base.SamplersUsed) - 1; - tokens = st_get_bitmap_shader(tgsi.tokens, + tokens = st_get_bitmap_shader(state.tokens, st->internal_target, variant->bitmap_sampler, st->needs_texcoord_semantic, @@ -1304,9 +1430,9 @@ st_create_fp_variant(struct st_context *st, PIPE_FORMAT_R8_UNORM); if (tokens) { - if (tgsi.tokens != stfp->tgsi.tokens) - tgsi_free_tokens(tgsi.tokens); - tgsi.tokens = tokens; + if (state.tokens != stfp->state.tokens) + tgsi_free_tokens(state.tokens); + state.tokens = tokens; } else fprintf(stderr, "mesa: cannot create a shader for glBitmap\n"); } @@ -1333,7 +1459,7 @@ st_create_fp_variant(struct st_context *st, texcoord_const = _mesa_add_state_reference(params, texcoord_state); - tokens = st_get_drawpix_shader(tgsi.tokens, + tokens = st_get_drawpix_shader(state.tokens, st->needs_texcoord_semantic, key->scaleAndBias, scale_const, bias_const, key->pixelMaps, @@ -1342,9 +1468,9 @@ st_create_fp_variant(struct st_context *st, texcoord_const, st->internal_target); if (tokens) { - if (tgsi.tokens != stfp->tgsi.tokens) - tgsi_free_tokens(tgsi.tokens); - tgsi.tokens = tokens; + if (state.tokens != stfp->state.tokens) + tgsi_free_tokens(state.tokens); + state.tokens = tokens; } else fprintf(stderr, "mesa: cannot create a shader for glDrawPixels\n"); } @@ -1356,16 +1482,16 @@ st_create_fp_variant(struct st_context *st, /* samplers inserted would conflict, but this should be unpossible: */ assert(!(key->bitmap || key->drawpixels)); - tokens = st_tgsi_lower_yuv(tgsi.tokens, + tokens = st_tgsi_lower_yuv(state.tokens, ~stfp->Base.SamplersUsed, key->external.lower_nv12 || key->external.lower_xy_uxvx || key->external.lower_yx_xuxv, key->external.lower_iyuv); if (tokens) { - if (tgsi.tokens != stfp->tgsi.tokens) - tgsi_free_tokens(tgsi.tokens); - tgsi.tokens = tokens; + if (state.tokens != stfp->state.tokens) + tgsi_free_tokens(state.tokens); + state.tokens = tokens; } else { fprintf(stderr, "mesa: cannot create a shader for samplerExternalOES\n"); } @@ -1375,23 +1501,21 @@ st_create_fp_variant(struct st_context *st, unsigned depth_range_const = _mesa_add_state_reference(params, depth_range_state); const struct tgsi_token *tokens; - tokens = st_tgsi_lower_depth_clamp_fs(tgsi.tokens, depth_range_const); - if (tgsi.tokens != stfp->tgsi.tokens) - tgsi_free_tokens(tgsi.tokens); - tgsi.tokens = tokens; + tokens = st_tgsi_lower_depth_clamp_fs(state.tokens, depth_range_const); + if (state.tokens != stfp->state.tokens) + tgsi_free_tokens(state.tokens); + state.tokens = tokens; } - if (ST_DEBUG & DEBUG_TGSI) { - tgsi_dump(tgsi.tokens, 0); - debug_printf("\n"); - } + if (ST_DEBUG & DEBUG_PRINT_IR) + tgsi_dump(state.tokens, 0); /* fill in variant */ - variant->driver_shader = pipe->create_fs_state(pipe, &tgsi); + variant->base.driver_shader = pipe->create_fs_state(pipe, &state); variant->key = *key; - if (tgsi.tokens != stfp->tgsi.tokens) - tgsi_free_tokens(tgsi.tokens); + if (state.tokens != stfp->state.tokens) + tgsi_free_tokens(state.tokens); return variant; } @@ -1400,13 +1524,14 @@ st_create_fp_variant(struct st_context *st, */ struct st_fp_variant * st_get_fp_variant(struct st_context *st, - struct st_fragment_program *stfp, + struct st_program *stfp, const struct st_fp_variant_key *key) { struct st_fp_variant *fpv; /* Search for existing variant */ - for (fpv = stfp->variants; fpv; fpv = fpv->next) { + for (fpv = st_fp_variant(stfp->variants); fpv; + fpv = st_fp_variant(fpv->base.next)) { if (memcmp(&fpv->key, key, sizeof(*key)) == 0) { break; } @@ -1416,6 +1541,8 @@ st_get_fp_variant(struct st_context *st, /* create new */ fpv = st_create_fp_variant(st, stfp, key); if (fpv) { + fpv->base.st = key->st; + if (key->bitmap || key->drawpixels) { /* Regular variants should always come before the * bitmap & drawpixels variants, (unless there @@ -1424,16 +1551,16 @@ st_get_fp_variant(struct st_context *st, * shader_has_one_variant is set. */ if (!stfp->variants) { - stfp->variants = fpv; + stfp->variants = &fpv->base; } else { /* insert into list after the first one */ - fpv->next = stfp->variants->next; - stfp->variants->next = fpv; + fpv->base.next = stfp->variants->next; + stfp->variants->next = &fpv->base; } } else { /* insert into list */ - fpv->next = stfp->variants; - stfp->variants = fpv; + fpv->base.next = stfp->variants; + stfp->variants = &fpv->base; } } } @@ -1447,58 +1574,17 @@ st_get_fp_variant(struct st_context *st, */ bool st_translate_common_program(struct st_context *st, - struct st_common_program *stcp) + struct st_program *stp) { - struct gl_program *prog = &stcp->Base; + struct gl_program *prog = &stp->Base; enum pipe_shader_type stage = - pipe_shader_type_from_mesa(stcp->Base.info.stage); + pipe_shader_type_from_mesa(stp->Base.info.stage); struct ureg_program *ureg = ureg_create_with_screen(stage, st->pipe->screen); if (ureg == NULL) return false; - switch (stage) { - case PIPE_SHADER_TESS_CTRL: - ureg_property(ureg, TGSI_PROPERTY_TCS_VERTICES_OUT, - stcp->Base.info.tess.tcs_vertices_out); - break; - - case PIPE_SHADER_TESS_EVAL: - if (stcp->Base.info.tess.primitive_mode == GL_ISOLINES) - ureg_property(ureg, TGSI_PROPERTY_TES_PRIM_MODE, GL_LINES); - else - ureg_property(ureg, TGSI_PROPERTY_TES_PRIM_MODE, - stcp->Base.info.tess.primitive_mode); - - STATIC_ASSERT((TESS_SPACING_EQUAL + 1) % 3 == PIPE_TESS_SPACING_EQUAL); - STATIC_ASSERT((TESS_SPACING_FRACTIONAL_ODD + 1) % 3 == - PIPE_TESS_SPACING_FRACTIONAL_ODD); - STATIC_ASSERT((TESS_SPACING_FRACTIONAL_EVEN + 1) % 3 == - PIPE_TESS_SPACING_FRACTIONAL_EVEN); - - ureg_property(ureg, TGSI_PROPERTY_TES_SPACING, - (stcp->Base.info.tess.spacing + 1) % 3); - - ureg_property(ureg, TGSI_PROPERTY_TES_VERTEX_ORDER_CW, - !stcp->Base.info.tess.ccw); - ureg_property(ureg, TGSI_PROPERTY_TES_POINT_MODE, - stcp->Base.info.tess.point_mode); - break; - - case PIPE_SHADER_GEOMETRY: - ureg_property(ureg, TGSI_PROPERTY_GS_INPUT_PRIM, - stcp->Base.info.gs.input_primitive); - ureg_property(ureg, TGSI_PROPERTY_GS_OUTPUT_PRIM, - stcp->Base.info.gs.output_primitive); - ureg_property(ureg, TGSI_PROPERTY_GS_MAX_OUTPUT_VERTICES, - stcp->Base.info.gs.vertices_out); - ureg_property(ureg, TGSI_PROPERTY_GS_INVOCATIONS, - stcp->Base.info.gs.invocations); - break; - - default: - break; - } + ureg_setup_shader_info(ureg, &stp->Base.info); ubyte inputSlotToAttr[VARYING_SLOT_TESS_MAX]; ubyte inputMapping[VARYING_SLOT_TESS_MAX]; @@ -1518,14 +1604,7 @@ st_translate_common_program(struct st_context *st, memset(inputSlotToAttr, 0, sizeof(inputSlotToAttr)); memset(inputMapping, 0, sizeof(inputMapping)); memset(outputMapping, 0, sizeof(outputMapping)); - memset(&stcp->tgsi, 0, sizeof(stcp->tgsi)); - - if (prog->info.clip_distance_array_size) - ureg_property(ureg, TGSI_PROPERTY_NUM_CLIPDIST_ENABLED, - prog->info.clip_distance_array_size); - if (prog->info.cull_distance_array_size) - ureg_property(ureg, TGSI_PROPERTY_NUM_CULLDIST_ENABLED, - prog->info.cull_distance_array_size); + memset(&stp->state, 0, sizeof(stp->state)); /* * Convert Mesa program inputs to TGSI input register semantics. @@ -1598,7 +1677,7 @@ st_translate_common_program(struct st_context *st, st_translate_program(st->ctx, stage, ureg, - stcp->glsl_to_tgsi, + stp->glsl_to_tgsi, prog, /* inputs */ num_inputs, @@ -1613,7 +1692,7 @@ st_translate_common_program(struct st_context *st, output_semantic_name, output_semantic_index); - stcp->tgsi.tokens = ureg_get_tokens(ureg, &stcp->num_tgsi_tokens); + stp->state.tokens = ureg_get_tokens(ureg, NULL); ureg_destroy(ureg); @@ -1621,18 +1700,11 @@ st_translate_common_program(struct st_context *st, st_store_ir_in_disk_cache(st, prog, false); - if ((ST_DEBUG & DEBUG_TGSI) && (ST_DEBUG & DEBUG_MESA)) { + if (ST_DEBUG & DEBUG_PRINT_IR && ST_DEBUG & DEBUG_MESA) _mesa_print_program(prog); - debug_printf("\n"); - } - if (ST_DEBUG & DEBUG_TGSI) { - tgsi_dump(stcp->tgsi.tokens, 0); - debug_printf("\n"); - } - - free_glsl_to_tgsi_visitor(stcp->glsl_to_tgsi); - stcp->glsl_to_tgsi = NULL; + free_glsl_to_tgsi_visitor(stp->glsl_to_tgsi); + stp->glsl_to_tgsi = NULL; return true; } @@ -1640,39 +1712,52 @@ st_translate_common_program(struct st_context *st, /** * Get/create a basic program variant. */ -struct st_basic_variant * -st_get_basic_variant(struct st_context *st, - struct st_common_program *prog, - const struct st_basic_variant_key *key) +struct st_variant * +st_get_common_variant(struct st_context *st, + struct st_program *prog, + const struct st_common_variant_key *key) { struct pipe_context *pipe = st->pipe; - struct st_basic_variant *v; - struct pipe_shader_state tgsi = {0}; + struct st_variant *v; + struct pipe_shader_state state = {0}; + struct gl_program_parameter_list *params = prog->Base.Parameters; /* Search for existing variant */ for (v = prog->variants; v; v = v->next) { - if (memcmp(&v->key, key, sizeof(*key)) == 0) { + if (memcmp(&st_common_variant(v)->key, key, sizeof(*key)) == 0) break; - } } if (!v) { /* create new */ - v = CALLOC_STRUCT(st_basic_variant); + v = (struct st_variant*)CALLOC_STRUCT(st_common_variant); if (v) { + if (prog->state.type == PIPE_SHADER_IR_NIR) { + bool finalize = false; + + state.type = PIPE_SHADER_IR_NIR; + state.ir.nir = get_nir_shader(st, prog); - if (prog->tgsi.type == PIPE_SHADER_IR_NIR) { - tgsi.type = PIPE_SHADER_IR_NIR; - tgsi.ir.nir = nir_shader_clone(NULL, prog->tgsi.ir.nir); + if (key->clamp_color) { + NIR_PASS_V(state.ir.nir, nir_lower_clamp_color_outputs); + finalize = true; + } - if (key->clamp_color) - NIR_PASS_V(tgsi.ir.nir, nir_lower_clamp_color_outputs); + if (key->lower_ucp) { + lower_ucp(st, state.ir.nir, key->lower_ucp, params); + finalize = true; + } - tgsi.stream_output = prog->tgsi.stream_output; + state.stream_output = prog->state.stream_output; - st_finalize_nir(st, &prog->Base, prog->shader_program, - tgsi.ir.nir); - } else { + if (finalize || !st->allow_st_finalize_nir_twice) { + st_finalize_nir(st, &prog->Base, prog->shader_program, + state.ir.nir, true); + } + + if (ST_DEBUG & DEBUG_PRINT_IR) + nir_print_shader(state.ir.nir, stderr); + } else { if (key->lower_depth_clamp) { struct gl_program_parameter_list *params = prog->Base.Parameters; @@ -1681,38 +1766,40 @@ st_get_basic_variant(struct st_context *st, const struct tgsi_token *tokens; tokens = - st_tgsi_lower_depth_clamp(prog->tgsi.tokens, + st_tgsi_lower_depth_clamp(prog->state.tokens, depth_range_const, key->clip_negative_one_to_one); - if (tokens != prog->tgsi.tokens) - tgsi_free_tokens(prog->tgsi.tokens); + if (tokens != prog->state.tokens) + tgsi_free_tokens(prog->state.tokens); - prog->tgsi.tokens = tokens; - prog->num_tgsi_tokens = tgsi_num_tokens(tokens); + prog->state.tokens = tokens; } - tgsi = prog->tgsi; + state = prog->state; + + if (ST_DEBUG & DEBUG_PRINT_IR) + tgsi_dump(state.tokens, 0); } /* fill in new variant */ switch (prog->Base.info.stage) { case MESA_SHADER_TESS_CTRL: - v->driver_shader = pipe->create_tcs_state(pipe, &tgsi); + v->driver_shader = pipe->create_tcs_state(pipe, &state); break; case MESA_SHADER_TESS_EVAL: - v->driver_shader = pipe->create_tes_state(pipe, &tgsi); + v->driver_shader = pipe->create_tes_state(pipe, &state); break; case MESA_SHADER_GEOMETRY: - v->driver_shader = pipe->create_gs_state(pipe, &tgsi); + v->driver_shader = pipe->create_gs_state(pipe, &state); break; case MESA_SHADER_COMPUTE: { struct pipe_compute_state cs = {0}; - cs.ir_type = tgsi.type; + cs.ir_type = state.type; cs.req_local_mem = prog->Base.info.cs.shared_size; - if (tgsi.type == PIPE_SHADER_IR_NIR) - cs.prog = tgsi.ir.nir; + if (state.type == PIPE_SHADER_IR_NIR) + cs.prog = state.ir.nir; else - cs.prog = tgsi.tokens; + cs.prog = state.tokens; v->driver_shader = pipe->create_compute_state(pipe, &cs); break; @@ -1723,7 +1810,8 @@ st_get_basic_variant(struct st_context *st, return NULL; } - v->key = *key; + st_common_variant(v)->key = *key; + v->st = key->st; /* insert into list */ v->next = prog->variants; @@ -1745,74 +1833,27 @@ destroy_program_variants(struct st_context *st, struct gl_program *target) if (!target || target == &_mesa_DummyProgram) return; - switch (target->Target) { - case GL_VERTEX_PROGRAM_ARB: - { - struct st_vertex_program *stvp = (struct st_vertex_program *) target; - struct st_vp_variant *vpv, **prevPtr = &stvp->variants; - - for (vpv = stvp->variants; vpv; ) { - struct st_vp_variant *next = vpv->next; - if (vpv->key.st == st) { - /* unlink from list */ - *prevPtr = next; - /* destroy this variant */ - delete_vp_variant(st, vpv); - } - else { - prevPtr = &vpv->next; - } - vpv = next; - } - } - break; - case GL_FRAGMENT_PROGRAM_ARB: - { - struct st_fragment_program *stfp = - (struct st_fragment_program *) target; - struct st_fp_variant *fpv, **prevPtr = &stfp->variants; - - for (fpv = stfp->variants; fpv; ) { - struct st_fp_variant *next = fpv->next; - if (fpv->key.st == st) { - /* unlink from list */ - *prevPtr = next; - /* destroy this variant */ - delete_fp_variant(st, fpv); - } - else { - prevPtr = &fpv->next; - } - fpv = next; + struct st_program *p = st_program(target); + struct st_variant *v, **prevPtr = &p->variants; + bool unbound = false; + + for (v = p->variants; v; ) { + struct st_variant *next = v->next; + if (v->st == st) { + if (!unbound) { + st_unbind_program(st, p); + unbound = true; } + + /* unlink from list */ + *prevPtr = next; + /* destroy this variant */ + delete_variant(st, v, target->Target); } - break; - case GL_GEOMETRY_PROGRAM_NV: - case GL_TESS_CONTROL_PROGRAM_NV: - case GL_TESS_EVALUATION_PROGRAM_NV: - case GL_COMPUTE_PROGRAM_NV: - { - struct st_common_program *p = st_common_program(target); - struct st_basic_variant *v, **prevPtr = &p->variants; - - for (v = p->variants; v; ) { - struct st_basic_variant *next = v->next; - if (v->key.st == st) { - /* unlink from list */ - *prevPtr = next; - /* destroy this variant */ - delete_basic_variant(st, v, target->Target); - } - else { - prevPtr = &v->next; - } - v = next; - } + else { + prevPtr = &v->next; } - break; - default: - _mesa_problem(NULL, "Unexpected program target 0x%x in " - "destroy_program_variants_cb()", target->Target); + v = next; } } @@ -1889,40 +1930,17 @@ st_destroy_program_variants(struct st_context *st) } -/** - * For debugging, print/dump the current vertex program. - */ -void -st_print_current_vertex_program(void) -{ - GET_CURRENT_CONTEXT(ctx); - - if (ctx->VertexProgram._Current) { - struct st_vertex_program *stvp = - (struct st_vertex_program *) ctx->VertexProgram._Current; - struct st_vp_variant *stv; - - debug_printf("Vertex program %u\n", stvp->Base.Id); - - for (stv = stvp->variants; stv; stv = stv->next) { - debug_printf("variant %p\n", stv); - tgsi_dump(stv->tgsi.tokens, 0); - } - } -} - - /** * Compile one shader variant. */ -void +static void st_precompile_shader_variant(struct st_context *st, struct gl_program *prog) { switch (prog->Target) { case GL_VERTEX_PROGRAM_ARB: { - struct st_vertex_program *p = (struct st_vertex_program *)prog; - struct st_vp_variant_key key; + struct st_program *p = (struct st_program *)prog; + struct st_common_variant_key key; memset(&key, 0, sizeof(key)); @@ -1932,7 +1950,7 @@ st_precompile_shader_variant(struct st_context *st, } case GL_FRAGMENT_PROGRAM_ARB: { - struct st_fragment_program *p = (struct st_fragment_program *)prog; + struct st_program *p = (struct st_program *)prog; struct st_fp_variant_key key; memset(&key, 0, sizeof(key)); @@ -1946,13 +1964,13 @@ st_precompile_shader_variant(struct st_context *st, case GL_TESS_EVALUATION_PROGRAM_NV: case GL_GEOMETRY_PROGRAM_NV: case GL_COMPUTE_PROGRAM_NV: { - struct st_common_program *p = st_common_program(prog); - struct st_basic_variant_key key; + struct st_program *p = st_program(prog); + struct st_common_variant_key key; memset(&key, 0, sizeof(key)); key.st = st->has_shareable_shaders ? NULL : st; - st_get_basic_variant(st, p, &key); + st_get_common_variant(st, p, &key); break; } @@ -1960,3 +1978,43 @@ st_precompile_shader_variant(struct st_context *st, assert(0); } } + +void +st_serialize_nir(struct st_program *stp) +{ + if (!stp->serialized_nir) { + struct blob blob; + size_t size; + + blob_init(&blob); + nir_serialize(&blob, stp->Base.nir, false); + blob_finish_get_buffer(&blob, &stp->serialized_nir, &size); + stp->serialized_nir_size = size; + } +} + +void +st_finalize_program(struct st_context *st, struct gl_program *prog) +{ + if (st->current_program[prog->info.stage] == prog) { + if (prog->info.stage == MESA_SHADER_VERTEX) + st->dirty |= ST_NEW_VERTEX_PROGRAM(st, (struct st_program *)prog); + else + st->dirty |= ((struct st_program *)prog)->affected_states; + } + + if (prog->nir) { + nir_sweep(prog->nir); + + /* This is only needed for ARB_vp/fp programs and when the disk cache + * is disabled. If the disk cache is enabled, GLSL programs are + * serialized in write_nir_to_cache. + */ + st_serialize_nir(st_program(prog)); + } + + /* Create Gallium shaders now instead of on demand. */ + if (ST_DEBUG & DEBUG_PRECOMPILE || + st->shader_has_one_variant[prog->info.stage]) + st_precompile_shader_variant(st, prog); +}