src/panfrost/lib/pan_blit.c

   1 /*
   2  * Copyright (C) 2020 Collabora, Ltd.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  21  * SOFTWARE.
  22  *
  23  * Authors:
  24  *   Alyssa Rosenzweig <alyssa.rosenzweig@collabora.com>
  25  */
  26
  27 #include <math.h>
  28 #include <stdio.h>
  29 #include "pan_encoder.h"
  30 #include "pan_pool.h"
  31 #include "pan_scoreboard.h"
  32 #include "pan_texture.h"
  33 #include "panfrost-quirks.h"
  34 #include "../midgard/midgard_compile.h"
  35 #include "compiler/nir/nir_builder.h"
  36 #include "util/u_math.h"
  37
  38 /* On Midgard, the native blit infrastructure (via MFBD preloads) is broken or
  39  * missing in many cases. We instead use software paths as fallbacks to
  40  * implement blits, which are done as TILER jobs. No vertex shader is
  41  * necessary since we can supply screen-space coordinates directly.
  42  *
  43  * This is primarily designed as a fallback for preloads but could be extended
  44  * for other clears/blits if needed in the future. */
  45
  46 static void
  47 panfrost_build_blit_shader(panfrost_program *program, unsigned gpu_id, gl_frag_result loc, nir_alu_type T, bool ms)
  48 {
  49         bool is_colour = loc >= FRAG_RESULT_DATA0;
  50
  51         nir_shader *shader = nir_shader_create(NULL, MESA_SHADER_FRAGMENT, &midgard_nir_options, NULL);
  52         nir_function *fn = nir_function_create(shader, "main");
  53         nir_function_impl *impl = nir_function_impl_create(fn);
  54
  55         nir_variable *c_src = nir_variable_create(shader, nir_var_shader_in, glsl_vector_type(GLSL_TYPE_FLOAT, 2), "coord");
  56         nir_variable *c_out = nir_variable_create(shader, nir_var_shader_out, glsl_vector_type(
  57                                 GLSL_TYPE_FLOAT, is_colour ? 4 : 1), "out");
  58
  59         c_src->data.location = VARYING_SLOT_TEX0;
  60         c_out->data.location = loc;
  61
  62         nir_builder _b;
  63         nir_builder *b = &_b;
  64         nir_builder_init(b, impl);
  65         b->cursor = nir_before_block(nir_start_block(impl));
  66
  67         nir_ssa_def *coord = nir_load_var(b, c_src);
  68
  69         nir_tex_instr *tex = nir_tex_instr_create(shader, ms ? 3 : 1);
  70
  71         tex->dest_type = T;
  72
  73         if (ms) {
  74                 tex->src[0].src_type = nir_tex_src_coord;
  75                 tex->src[0].src = nir_src_for_ssa(nir_f2i32(b, coord));
  76                 tex->coord_components = 2;
  77
  78                 tex->src[1].src_type = nir_tex_src_ms_index;
  79                 tex->src[1].src = nir_src_for_ssa(nir_load_sample_id(b));
  80
  81                 tex->src[2].src_type = nir_tex_src_lod;
  82                 tex->src[2].src = nir_src_for_ssa(nir_imm_int(b, 0));
  83                 tex->sampler_dim = GLSL_SAMPLER_DIM_MS;
  84                 tex->op = nir_texop_txf_ms;
  85         } else {
  86                 tex->op = nir_texop_tex;
  87
  88                 tex->src[0].src_type = nir_tex_src_coord;
  89                 tex->src[0].src = nir_src_for_ssa(coord);
  90                 tex->coord_components = 2;
  91
  92                 tex->sampler_dim = GLSL_SAMPLER_DIM_2D;
  93         }
  94
  95         nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, NULL);
  96         nir_builder_instr_insert(b, &tex->instr);
  97
  98         if (is_colour)
  99                 nir_store_var(b, c_out, &tex->dest.ssa, 0xFF);
 100         else
 101                 nir_store_var(b, c_out, nir_channel(b, &tex->dest.ssa, 0), 0xFF);
 102
 103         midgard_compile_shader_nir(shader, program, false, 0, gpu_id, false, true);
 104 }
 105
 106 /* Compile and upload all possible blit shaders ahead-of-time to reduce draw
 107  * time overhead. There's only ~30 of them at the moment, so this is fine */
 108
 109 void
 110 panfrost_init_blit_shaders(struct panfrost_device *dev)
 111 {
 112         static const struct {
 113                 gl_frag_result loc;
 114                 unsigned types;
 115         } shader_descs[] = {
 116                 { FRAG_RESULT_DEPTH,   1 << PAN_BLIT_FLOAT },
 117                 { FRAG_RESULT_STENCIL, 1 << PAN_BLIT_UINT },
 118                 { FRAG_RESULT_DATA0,  ~0 },
 119                 { FRAG_RESULT_DATA1,  ~0 },
 120                 { FRAG_RESULT_DATA2,  ~0 },
 121                 { FRAG_RESULT_DATA3,  ~0 },
 122                 { FRAG_RESULT_DATA4,  ~0 },
 123                 { FRAG_RESULT_DATA5,  ~0 },
 124                 { FRAG_RESULT_DATA6,  ~0 },
 125                 { FRAG_RESULT_DATA7,  ~0 }
 126         };
 127
 128         nir_alu_type nir_types[PAN_BLIT_NUM_TYPES] = {
 129                 nir_type_float,
 130                 nir_type_uint,
 131                 nir_type_int
 132         };
 133
 134         /* Total size = # of shaders * bytes per shader. There are
 135          * shaders for each RT (so up to DATA7 -- overestimate is
 136          * okay) and up to NUM_TYPES variants of each, * 2 for multisampling
 137          * variants. These shaders are simple enough that they should be less
 138          * than 8 quadwords each (again, overestimate is fine). */
 139
 140         unsigned offset = 0;
 141         unsigned total_size = (FRAG_RESULT_DATA7 * PAN_BLIT_NUM_TYPES)
 142                 * (8 * 16) * 2;
 143
 144         dev->blit_shaders.bo = panfrost_bo_create(dev, total_size, PAN_BO_EXECUTE);
 145
 146         /* Don't bother generating multisampling variants if we don't actually
 147          * support multisampling */
 148         bool has_ms = !(dev->quirks & MIDGARD_SFBD);
 149
 150         for (unsigned ms = 0; ms <= has_ms; ++ms) {
 151                 for (unsigned i = 0; i < ARRAY_SIZE(shader_descs); ++i) {
 152                         unsigned loc = shader_descs[i].loc;
 153
 154                         for (enum pan_blit_type T = 0; T < PAN_BLIT_NUM_TYPES; ++T) {
 155                                 if (!(shader_descs[i].types & (1 << T)))
 156                                         continue;
 157
 158                                 panfrost_program program;
 159                                 panfrost_build_blit_shader(&program, dev->gpu_id, loc,
 160                                                 nir_types[T], ms);
 161
 162                                 assert(offset + program.compiled.size < total_size);
 163                                 memcpy(dev->blit_shaders.bo->cpu + offset, program.compiled.data, program.compiled.size);
 164
 165                                 dev->blit_shaders.loads[loc][T][ms] = (dev->blit_shaders.bo->gpu + offset) | program.first_tag;
 166                                 offset += ALIGN_POT(program.compiled.size, 64);
 167                                 util_dynarray_fini(&program.compiled);
 168                         }
 169                 }
 170         }
 171 }
 172
 173 /* Add a shader-based load on Midgard (draw-time for GL). Shaders are
 174  * precached */
 175
 176 void
 177 panfrost_load_midg(
 178                 struct pan_pool *pool,
 179                 struct pan_scoreboard *scoreboard,
 180                 mali_ptr blend_shader,
 181                 mali_ptr fbd,
 182                 mali_ptr coordinates, unsigned vertex_count,
 183                 struct pan_image *image,
 184                 unsigned loc)
 185 {
 186         unsigned width = u_minify(image->width0, image->first_level);
 187         unsigned height = u_minify(image->height0, image->first_level);
 188
 189         struct panfrost_transfer viewport = panfrost_pool_alloc(pool, MALI_VIEWPORT_LENGTH);
 190         struct panfrost_transfer sampler = panfrost_pool_alloc(pool, MALI_MIDGARD_SAMPLER_LENGTH);
 191         struct panfrost_transfer varying_buf = panfrost_pool_alloc(pool, MALI_ATTRIBUTE_LENGTH);
 192
 193         pan_pack(viewport.cpu, VIEWPORT, cfg) {
 194                 cfg.scissor_maximum_x = width - 1; /* Inclusive */
 195                 cfg.scissor_maximum_y = height - 1;
 196         }
 197
 198         union mali_attr varying = {
 199                 .elements = coordinates | MALI_ATTR_LINEAR,
 200                 .stride = 4 * sizeof(float),
 201                 .size = 4 * sizeof(float) * vertex_count,
 202         };
 203
 204         pan_pack(varying_buf.cpu, ATTRIBUTE, cfg) {
 205                 cfg.buffer_index = 0;
 206                 cfg.format = (MALI_CHANNEL_R << 0) | (MALI_CHANNEL_G << 3) | (MALI_RGBA32F << 12);
 207         }
 208
 209         struct mali_stencil_packed stencil;
 210         pan_pack(&stencil, STENCIL, cfg) {
 211                 cfg.compare_function = MALI_FUNC_ALWAYS;
 212                 cfg.stencil_fail = MALI_STENCIL_OP_REPLACE;
 213                 cfg.depth_fail = MALI_STENCIL_OP_REPLACE;
 214                 cfg.depth_pass = MALI_STENCIL_OP_REPLACE;
 215         };
 216
 217         union midgard_blend replace = {
 218                 .equation = {
 219                         .rgb_mode = 0x122,
 220                         .alpha_mode = 0x122,
 221                         .color_mask = MALI_MASK_R | MALI_MASK_G | MALI_MASK_B | MALI_MASK_A,
 222                 }
 223         };
 224
 225         if (blend_shader)
 226                 replace.shader = blend_shader;
 227
 228         /* Determine the sampler type needed. Stencil is always sampled as
 229          * UINT. Pure (U)INT is always (U)INT. Everything else is FLOAT. */
 230
 231         enum pan_blit_type T =
 232                 (loc == FRAG_RESULT_STENCIL) ? PAN_BLIT_UINT :
 233                 (util_format_is_pure_uint(image->format)) ? PAN_BLIT_UINT :
 234                 (util_format_is_pure_sint(image->format)) ? PAN_BLIT_INT :
 235                 PAN_BLIT_FLOAT;
 236
 237         bool ms = image->nr_samples > 1;
 238
 239         struct mali_shader_meta shader_meta = {
 240                 .shader = pool->dev->blit_shaders.loads[loc][T][ms],
 241                 .sampler_count = 1,
 242                 .texture_count = 1,
 243                 .varying_count = 1,
 244                 .midgard1 = {
 245                         .flags_lo = 0x20,
 246                         .work_count = 4,
 247                 },
 248                 .coverage_mask = ~0,
 249                 .unknown2_3 = MALI_DEPTH_FUNC(MALI_FUNC_ALWAYS) | 0x10,
 250                 .unknown2_4 = 0x4e0,
 251                 .stencil_mask_front = ~0,
 252                 .stencil_mask_back = ~0,
 253                 .stencil_front = stencil,
 254                 .stencil_back = stencil,
 255                 .blend = {
 256                         .shader = blend_shader
 257                 }
 258         };
 259
 260         if (ms)
 261                 shader_meta.unknown2_3 |= MALI_HAS_MSAA | MALI_PER_SAMPLE;
 262         else
 263                 shader_meta.unknown2_4 |= MALI_NO_MSAA;
 264
 265         assert(shader_meta.shader);
 266
 267         if (pool->dev->quirks & MIDGARD_SFBD) {
 268                 shader_meta.unknown2_4 |= (0x10 | MALI_NO_DITHER);
 269                 shader_meta.blend = replace;
 270
 271                 if (loc < FRAG_RESULT_DATA0)
 272                         shader_meta.blend.equation.color_mask = 0x0;
 273         }
 274
 275         if (loc == FRAG_RESULT_DEPTH) {
 276                 shader_meta.midgard1.flags_lo |= MALI_WRITES_Z;
 277                 shader_meta.unknown2_3 |= MALI_DEPTH_WRITEMASK;
 278         } else if (loc == FRAG_RESULT_STENCIL) {
 279                 shader_meta.midgard1.flags_hi |= MALI_WRITES_S;
 280                 shader_meta.unknown2_4 |= MALI_STENCIL_TEST;
 281         } else {
 282                 shader_meta.midgard1.flags_lo |= MALI_EARLY_Z;
 283         }
 284
 285         /* Create the texture descriptor. We partially compute the base address
 286          * ourselves to account for layer, such that the texture descriptor
 287          * itself is for a 2D texture with array size 1 even for 3D/array
 288          * textures, removing the need to separately key the blit shaders for
 289          * 2D and 3D variants */
 290
 291         struct panfrost_transfer texture_t = panfrost_pool_alloc(pool, MALI_MIDGARD_TEXTURE_LENGTH + sizeof(mali_ptr) * 2 * MAX2(image->nr_samples, 1));
 292
 293         panfrost_new_texture(texture_t.cpu,
 294                         image->width0, image->height0,
 295                         MAX2(image->nr_samples, 1), 1,
 296                         image->format, MALI_TEXTURE_DIMENSION_2D,
 297                         image->modifier,
 298                         image->first_level, image->last_level,
 299                         0, 0,
 300                         image->nr_samples,
 301                         0,
 302                         (MALI_CHANNEL_R << 0) | (MALI_CHANNEL_G << 3) | (MALI_CHANNEL_B << 6) | (MALI_CHANNEL_A << 9),
 303                         image->bo->gpu + image->first_layer *
 304                                 panfrost_get_layer_stride(image->slices,
 305                                         image->dim == MALI_TEXTURE_DIMENSION_3D,
 306                                         image->cubemap_stride, image->first_level),
 307                         image->slices);
 308
 309         pan_pack(sampler.cpu, MIDGARD_SAMPLER, cfg)
 310                 cfg.normalized_coordinates = false;
 311
 312         struct panfrost_transfer shader_meta_t = panfrost_pool_alloc(pool, sizeof(shader_meta) + 8 * sizeof(struct midgard_blend_rt));
 313         memcpy(shader_meta_t.cpu, &shader_meta, sizeof(shader_meta));
 314
 315         for (unsigned i = 0; i < 8; ++i) {
 316                 void *dest = shader_meta_t.cpu + sizeof(shader_meta) + sizeof(struct midgard_blend_rt) * i;
 317
 318                 if (loc == (FRAG_RESULT_DATA0 + i)) {
 319                         struct midgard_blend_rt blend_rt = {
 320                                 .flags = 0x200 | MALI_BLEND_NO_DITHER,
 321                                 .blend = replace,
 322                         };
 323
 324                         if (util_format_is_srgb(image->format))
 325                                 blend_rt.flags |= MALI_BLEND_SRGB;
 326
 327                         if (blend_shader) {
 328                                 blend_rt.flags |= MALI_BLEND_MRT_SHADER;
 329                                 blend_rt.blend.shader = blend_shader;
 330                         }
 331
 332                         memcpy(dest, &blend_rt, sizeof(struct midgard_blend_rt));
 333                 } else {
 334                         memset(dest, 0x0, sizeof(struct midgard_blend_rt));
 335                 }
 336         }
 337
 338         struct midgard_payload_vertex_tiler payload = {
 339                 .prefix = {
 340                         .draw_mode = MALI_DRAW_MODE_TRIANGLES,
 341                         .unknown_draw = 0x3000,
 342                         .index_count = MALI_POSITIVE(vertex_count)
 343                 },
 344                 .postfix = {
 345                         .gl_enables = 0x7,
 346                         .position_varying = coordinates,
 347                         .textures = panfrost_pool_upload(pool, &texture_t.gpu, sizeof(texture_t.gpu)),
 348                         .sampler_descriptor = sampler.gpu,
 349                         .shader = shader_meta_t.gpu,
 350                         .varyings = panfrost_pool_upload(pool, &varying, sizeof(varying)),
 351                         .varying_meta = varying_buf.gpu,
 352                         .viewport = viewport.gpu,
 353                         .shared_memory = fbd
 354                 }
 355         };
 356
 357         panfrost_pack_work_groups_compute(&payload.prefix, 1, vertex_count, 1, 1, 1, 1, true);
 358         payload.prefix.workgroups_x_shift_3 = 6;
 359
 360         panfrost_new_job(pool, scoreboard, MALI_JOB_TYPE_TILER, false, 0, &payload, sizeof(payload), true);
 361 }