src/gallium/drivers/etnaviv/etnaviv_rs.c

   1 /*
   2  * Copyright (c) 2012-2017 Etnaviv Project
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sub license,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the
  12  * next paragraph) shall be included in all copies or substantial portions
  13  * of the Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Wladimir J. van der Laan <laanwj@gmail.com>
  25  */
  26
  27 #include "etnaviv_rs.h"
  28
  29 #include "etnaviv_clear_blit.h"
  30 #include "etnaviv_context.h"
  31 #include "etnaviv_emit.h"
  32 #include "etnaviv_format.h"
  33 #include "etnaviv_resource.h"
  34 #include "etnaviv_screen.h"
  35 #include "etnaviv_surface.h"
  36 #include "etnaviv_tiling.h"
  37 #include "etnaviv_translate.h"
  38 #include "etnaviv_util.h"
  39
  40 #include "pipe/p_defines.h"
  41 #include "pipe/p_state.h"
  42 #include "util/u_blitter.h"
  43 #include "util/u_inlines.h"
  44 #include "util/u_memory.h"
  45 #include "util/u_surface.h"
  46
  47 #include "hw/common.xml.h"
  48 #include "hw/state.xml.h"
  49 #include "hw/state_3d.xml.h"
  50
  51 #include <assert.h>
  52
  53 /* return a RS "compatible" format for use when copying */
  54 static uint32_t
  55 etna_compatible_rs_format(enum pipe_format fmt)
  56 {
  57    /* YUYV and UYVY are blocksize 4, but 2 bytes per pixel */
  58    if (fmt == PIPE_FORMAT_YUYV || fmt == PIPE_FORMAT_UYVY)
  59       return RS_FORMAT_A4R4G4B4;
  60
  61    switch (util_format_get_blocksize(fmt)) {
  62    case 2: return RS_FORMAT_A4R4G4B4;
  63    case 4: return RS_FORMAT_A8R8G8B8;
  64    default: return ETNA_NO_MATCH;
  65    }
  66 }
  67
  68 void
  69 etna_compile_rs_state(struct etna_context *ctx, struct compiled_rs_state *cs,
  70                       const struct rs_state *rs)
  71 {
  72    memset(cs, 0, sizeof(*cs));
  73
  74    /* TILED and SUPERTILED layout have their strides multiplied with 4 in RS */
  75    unsigned source_stride_shift = COND(rs->source_tiling != ETNA_LAYOUT_LINEAR, 2);
  76    unsigned dest_stride_shift = COND(rs->dest_tiling != ETNA_LAYOUT_LINEAR, 2);
  77
  78    /* tiling == ETNA_LAYOUT_MULTI_TILED or ETNA_LAYOUT_MULTI_SUPERTILED? */
  79    int source_multi = COND(rs->source_tiling & ETNA_LAYOUT_BIT_MULTI, 1);
  80    int dest_multi = COND(rs->dest_tiling & ETNA_LAYOUT_BIT_MULTI, 1);
  81
  82    /* Vivante RS needs widths to be a multiple of 16 or bad things
  83     * happen, such as scribbing over memory, or the GPU hanging,
  84     * even for non-tiled formats.  As this is serious, use abort().
  85     */
  86    if (rs->width & ETNA_RS_WIDTH_MASK)
  87       abort();
  88
  89    /* TODO could just pre-generate command buffer, would simply submit to one memcpy */
  90    cs->RS_CONFIG = VIVS_RS_CONFIG_SOURCE_FORMAT(rs->source_format) |
  91                    COND(rs->downsample_x, VIVS_RS_CONFIG_DOWNSAMPLE_X) |
  92                    COND(rs->downsample_y, VIVS_RS_CONFIG_DOWNSAMPLE_Y) |
  93                    COND(rs->source_tiling & 1, VIVS_RS_CONFIG_SOURCE_TILED) |
  94                    VIVS_RS_CONFIG_DEST_FORMAT(rs->dest_format) |
  95                    COND(rs->dest_tiling & 1, VIVS_RS_CONFIG_DEST_TILED) |
  96                    COND(rs->swap_rb, VIVS_RS_CONFIG_SWAP_RB) |
  97                    COND(rs->flip, VIVS_RS_CONFIG_FLIP);
  98
  99    cs->RS_SOURCE_STRIDE = (rs->source_stride << source_stride_shift) |
 100                           COND(rs->source_tiling & 2, VIVS_RS_SOURCE_STRIDE_TILING) |
 101                           COND(source_multi, VIVS_RS_SOURCE_STRIDE_MULTI);
 102
 103    /* Initially all pipes are set to the base address of the source and
 104     * destination buffer respectively. This will be overridden below as
 105     * necessary for the multi-pipe, multi-tiled case.
 106     */
 107    for (unsigned pipe = 0; pipe < ctx->specs.pixel_pipes; ++pipe) {
 108       cs->source[pipe].bo = rs->source;
 109       cs->source[pipe].offset = rs->source_offset;
 110       cs->source[pipe].flags = ETNA_RELOC_READ;
 111
 112       cs->dest[pipe].bo = rs->dest;
 113       cs->dest[pipe].offset = rs->dest_offset;
 114       cs->dest[pipe].flags = ETNA_RELOC_WRITE;
 115
 116       cs->RS_PIPE_OFFSET[pipe] = VIVS_RS_PIPE_OFFSET_X(0) | VIVS_RS_PIPE_OFFSET_Y(0);
 117    }
 118
 119    cs->RS_DEST_STRIDE = (rs->dest_stride << dest_stride_shift) |
 120                         COND(rs->dest_tiling & 2, VIVS_RS_DEST_STRIDE_TILING) |
 121                         COND(dest_multi, VIVS_RS_DEST_STRIDE_MULTI);
 122
 123
 124    if (source_multi)
 125       cs->source[1].offset = rs->source_offset + rs->source_stride * rs->source_padded_height / 2;
 126
 127    if (dest_multi)
 128       cs->dest[1].offset = rs->dest_offset + rs->dest_stride * rs->dest_padded_height / 2;
 129
 130    cs->RS_WINDOW_SIZE = VIVS_RS_WINDOW_SIZE_WIDTH(rs->width) |
 131                         VIVS_RS_WINDOW_SIZE_HEIGHT(rs->height);
 132
 133    /* use dual pipe mode when required */
 134    if (!ctx->specs.single_buffer && ctx->specs.pixel_pipes == 2 && !(rs->height & 7)) {
 135       cs->RS_WINDOW_SIZE = VIVS_RS_WINDOW_SIZE_WIDTH(rs->width) |
 136                               VIVS_RS_WINDOW_SIZE_HEIGHT(rs->height / 2);
 137       cs->RS_PIPE_OFFSET[1] = VIVS_RS_PIPE_OFFSET_X(0) | VIVS_RS_PIPE_OFFSET_Y(rs->height / 2);
 138    }
 139
 140    cs->RS_DITHER[0] = rs->dither[0];
 141    cs->RS_DITHER[1] = rs->dither[1];
 142    cs->RS_CLEAR_CONTROL = VIVS_RS_CLEAR_CONTROL_BITS(rs->clear_bits) | rs->clear_mode;
 143    cs->RS_FILL_VALUE[0] = rs->clear_value[0];
 144    cs->RS_FILL_VALUE[1] = rs->clear_value[1];
 145    cs->RS_FILL_VALUE[2] = rs->clear_value[2];
 146    cs->RS_FILL_VALUE[3] = rs->clear_value[3];
 147    cs->RS_EXTRA_CONFIG = VIVS_RS_EXTRA_CONFIG_AA(rs->aa) |
 148                          VIVS_RS_EXTRA_CONFIG_ENDIAN(rs->endian_mode);
 149
 150    /* If source the same as destination, and the hardware supports this,
 151     * do an in-place resolve to fill in unrendered tiles.
 152     */
 153    if (ctx->specs.single_buffer && rs->source == rs->dest &&
 154          rs->source_offset == rs->dest_offset &&
 155          rs->source_format == rs->dest_format &&
 156          rs->source_tiling == rs->dest_tiling &&
 157          (rs->source_tiling & ETNA_LAYOUT_BIT_SUPER) &&
 158          rs->source_stride == rs->dest_stride &&
 159          !rs->downsample_x && !rs->downsample_y &&
 160          !rs->swap_rb && !rs->flip &&
 161          !rs->clear_mode && rs->source_padded_width &&
 162          !rs->source_ts_compressed) {
 163       /* Total number of tiles (same as for autodisable) */
 164       cs->RS_KICKER_INPLACE = rs->tile_count;
 165    }
 166    cs->source_ts_valid = rs->source_ts_valid;
 167 }
 168
 169 /* modify the clear bits value in the compiled RS state */
 170 static void
 171 etna_modify_rs_clearbits(struct compiled_rs_state *cs, uint32_t clear_bits)
 172 {
 173    cs->RS_CLEAR_CONTROL &= ~VIVS_RS_CLEAR_CONTROL_BITS__MASK;
 174    cs->RS_CLEAR_CONTROL |= VIVS_RS_CLEAR_CONTROL_BITS(clear_bits);
 175 }
 176
 177 #define EMIT_STATE(state_name, src_value) \
 178    etna_coalsence_emit(stream, &coalesce, VIVS_##state_name, src_value)
 179
 180 #define EMIT_STATE_FIXP(state_name, src_value) \
 181    etna_coalsence_emit_fixp(stream, &coalesce, VIVS_##state_name, src_value)
 182
 183 #define EMIT_STATE_RELOC(state_name, src_value) \
 184    etna_coalsence_emit_reloc(stream, &coalesce, VIVS_##state_name, src_value)
 185
 186 /* submit RS state, without any processing and no dependence on context
 187  * except TS if this is a source-to-destination blit. */
 188 static void
 189 etna_submit_rs_state(struct etna_context *ctx,
 190                      const struct compiled_rs_state *cs)
 191 {
 192    struct etna_screen *screen = etna_screen(ctx->base.screen);
 193    struct etna_cmd_stream *stream = ctx->stream;
 194    struct etna_coalesce coalesce;
 195
 196    if (cs->RS_KICKER_INPLACE && !cs->source_ts_valid)
 197       /* Inplace resolve is no-op if TS is not configured */
 198       return;
 199
 200    ctx->stats.rs_operations++;
 201
 202    if (cs->RS_KICKER_INPLACE) {
 203       etna_cmd_stream_reserve(stream, 6);
 204       etna_coalesce_start(stream, &coalesce);
 205       /* 0/1 */ EMIT_STATE(RS_EXTRA_CONFIG, cs->RS_EXTRA_CONFIG);
 206       /* 2/3 */ EMIT_STATE(RS_SOURCE_STRIDE, cs->RS_SOURCE_STRIDE);
 207       /* 4/5 */ EMIT_STATE(RS_KICKER_INPLACE, cs->RS_KICKER_INPLACE);
 208       etna_coalesce_end(stream, &coalesce);
 209    } else if (screen->specs.pixel_pipes == 1) {
 210       etna_cmd_stream_reserve(stream, 22);
 211       etna_coalesce_start(stream, &coalesce);
 212       /* 0/1 */ EMIT_STATE(RS_CONFIG, cs->RS_CONFIG);
 213       /* 2   */ EMIT_STATE_RELOC(RS_SOURCE_ADDR, &cs->source[0]);
 214       /* 3   */ EMIT_STATE(RS_SOURCE_STRIDE, cs->RS_SOURCE_STRIDE);
 215       /* 4   */ EMIT_STATE_RELOC(RS_DEST_ADDR, &cs->dest[0]);
 216       /* 5   */ EMIT_STATE(RS_DEST_STRIDE, cs->RS_DEST_STRIDE);
 217       /* 6/7 */ EMIT_STATE(RS_WINDOW_SIZE, cs->RS_WINDOW_SIZE);
 218       /* 8/9 */ EMIT_STATE(RS_DITHER(0), cs->RS_DITHER[0]);
 219       /*10   */ EMIT_STATE(RS_DITHER(1), cs->RS_DITHER[1]);
 220       /*11 - pad */
 221       /*12/13*/ EMIT_STATE(RS_CLEAR_CONTROL, cs->RS_CLEAR_CONTROL);
 222       /*14   */ EMIT_STATE(RS_FILL_VALUE(0), cs->RS_FILL_VALUE[0]);
 223       /*15   */ EMIT_STATE(RS_FILL_VALUE(1), cs->RS_FILL_VALUE[1]);
 224       /*16   */ EMIT_STATE(RS_FILL_VALUE(2), cs->RS_FILL_VALUE[2]);
 225       /*17   */ EMIT_STATE(RS_FILL_VALUE(3), cs->RS_FILL_VALUE[3]);
 226       /*18/19*/ EMIT_STATE(RS_EXTRA_CONFIG, cs->RS_EXTRA_CONFIG);
 227       /*20/21*/ EMIT_STATE(RS_KICKER, 0xbeebbeeb);
 228       etna_coalesce_end(stream, &coalesce);
 229    } else if (screen->specs.pixel_pipes == 2) {
 230       etna_cmd_stream_reserve(stream, 34); /* worst case - both pipes multi=1 */
 231       etna_coalesce_start(stream, &coalesce);
 232       /* 0/1 */ EMIT_STATE(RS_CONFIG, cs->RS_CONFIG);
 233       /* 2/3 */ EMIT_STATE(RS_SOURCE_STRIDE, cs->RS_SOURCE_STRIDE);
 234       /* 4/5 */ EMIT_STATE(RS_DEST_STRIDE, cs->RS_DEST_STRIDE);
 235       /* 6/7 */ EMIT_STATE_RELOC(RS_PIPE_SOURCE_ADDR(0), &cs->source[0]);
 236       if (cs->RS_SOURCE_STRIDE & VIVS_RS_SOURCE_STRIDE_MULTI) {
 237          /*8 */ EMIT_STATE_RELOC(RS_PIPE_SOURCE_ADDR(1), &cs->source[1]);
 238          /*9 - pad */
 239       }
 240       /*10/11*/ EMIT_STATE_RELOC(RS_PIPE_DEST_ADDR(0), &cs->dest[0]);
 241       if (cs->RS_DEST_STRIDE & VIVS_RS_DEST_STRIDE_MULTI) {
 242          /*12*/ EMIT_STATE_RELOC(RS_PIPE_DEST_ADDR(1), &cs->dest[1]);
 243          /*13 - pad */
 244       }
 245       /*14/15*/ EMIT_STATE(RS_PIPE_OFFSET(0), cs->RS_PIPE_OFFSET[0]);
 246       /*16   */ EMIT_STATE(RS_PIPE_OFFSET(1), cs->RS_PIPE_OFFSET[1]);
 247       /*17 - pad */
 248       /*18/19*/ EMIT_STATE(RS_WINDOW_SIZE, cs->RS_WINDOW_SIZE);
 249       /*20/21*/ EMIT_STATE(RS_DITHER(0), cs->RS_DITHER[0]);
 250       /*22   */ EMIT_STATE(RS_DITHER(1), cs->RS_DITHER[1]);
 251       /*23 - pad */
 252       /*24/25*/ EMIT_STATE(RS_CLEAR_CONTROL, cs->RS_CLEAR_CONTROL);
 253       /*26   */ EMIT_STATE(RS_FILL_VALUE(0), cs->RS_FILL_VALUE[0]);
 254       /*27   */ EMIT_STATE(RS_FILL_VALUE(1), cs->RS_FILL_VALUE[1]);
 255       /*28   */ EMIT_STATE(RS_FILL_VALUE(2), cs->RS_FILL_VALUE[2]);
 256       /*29   */ EMIT_STATE(RS_FILL_VALUE(3), cs->RS_FILL_VALUE[3]);
 257       /*30/31*/ EMIT_STATE(RS_EXTRA_CONFIG, cs->RS_EXTRA_CONFIG);
 258       /*32/33*/ EMIT_STATE(RS_KICKER, 0xbeebbeeb);
 259       etna_coalesce_end(stream, &coalesce);
 260    } else {
 261       abort();
 262    }
 263 }
 264
 265 /* Generate clear command for a surface (non-fast clear case) */
 266 void
 267 etna_rs_gen_clear_surface(struct etna_context *ctx, struct etna_surface *surf,
 268                           uint32_t clear_value)
 269 {
 270    struct etna_resource *dst = etna_resource(surf->base.texture);
 271    uint32_t format;
 272
 273    switch (util_format_get_blocksizebits(surf->base.format)) {
 274    case 16:
 275       format = RS_FORMAT_A4R4G4B4;
 276       break;
 277    case 32:
 278       format = RS_FORMAT_A8R8G8B8;
 279       break;
 280    case 64:
 281       assert(ctx->specs.halti >= 2);
 282       format = RS_FORMAT_64BPP_CLEAR;
 283       break;
 284    default:
 285       unreachable("bpp not supported for clear by RS");
 286       break;
 287    }
 288
 289    /* use tiled clear if width is multiple of 16 */
 290    bool tiled_clear = (surf->surf.padded_width & ETNA_RS_WIDTH_MASK) == 0 &&
 291                       (surf->surf.padded_height & ETNA_RS_HEIGHT_MASK) == 0;
 292
 293    etna_compile_rs_state( ctx, &surf->clear_command, &(struct rs_state) {
 294       .source_format = format,
 295       .dest_format = format,
 296       .dest = dst->bo,
 297       .dest_offset = surf->surf.offset,
 298       .dest_stride = surf->surf.stride,
 299       .dest_padded_height = surf->surf.padded_height,
 300       .dest_tiling = tiled_clear ? dst->layout : ETNA_LAYOUT_LINEAR,
 301       .dither = {0xffffffff, 0xffffffff},
 302       .width = surf->surf.padded_width, /* These must be padded to 16x4 if !LINEAR, otherwise RS will hang */
 303       .height = surf->surf.padded_height,
 304       .clear_value = {clear_value},
 305       .clear_mode = VIVS_RS_CLEAR_CONTROL_MODE_ENABLED1,
 306       .clear_bits = 0xffff
 307    });
 308 }
 309
 310 static void
 311 etna_blit_clear_color_rs(struct pipe_context *pctx, struct pipe_surface *dst,
 312                       const union pipe_color_union *color)
 313 {
 314    struct etna_context *ctx = etna_context(pctx);
 315    struct etna_surface *surf = etna_surface(dst);
 316    uint32_t new_clear_value = etna_clear_blit_pack_rgba(surf->base.format, color->f);
 317
 318    if (surf->surf.ts_size) { /* TS: use precompiled clear command */
 319       ctx->framebuffer.TS_COLOR_CLEAR_VALUE = new_clear_value;
 320
 321       if (VIV_FEATURE(ctx->screen, chipMinorFeatures1, AUTO_DISABLE)) {
 322          /* Set number of color tiles to be filled */
 323          etna_set_state(ctx->stream, VIVS_TS_COLOR_AUTO_DISABLE_COUNT,
 324                         surf->surf.padded_width * surf->surf.padded_height / 16);
 325          ctx->framebuffer.TS_MEM_CONFIG |= VIVS_TS_MEM_CONFIG_COLOR_AUTO_DISABLE;
 326       }
 327
 328       surf->level->ts_valid = true;
 329       ctx->dirty |= ETNA_DIRTY_TS | ETNA_DIRTY_DERIVE_TS;
 330    } else if (unlikely(new_clear_value != surf->level->clear_value)) { /* Queue normal RS clear for non-TS surfaces */
 331       /* If clear color changed, re-generate stored command */
 332       etna_rs_gen_clear_surface(ctx, surf, new_clear_value);
 333    }
 334
 335    etna_submit_rs_state(ctx, &surf->clear_command);
 336
 337    surf->level->clear_value = new_clear_value;
 338    resource_written(ctx, surf->base.texture);
 339    etna_resource(surf->base.texture)->seqno++;
 340 }
 341
 342 static void
 343 etna_blit_clear_zs_rs(struct pipe_context *pctx, struct pipe_surface *dst,
 344                    unsigned buffers, double depth, unsigned stencil)
 345 {
 346    struct etna_context *ctx = etna_context(pctx);
 347    struct etna_surface *surf = etna_surface(dst);
 348    uint32_t new_clear_value = translate_clear_depth_stencil(surf->base.format, depth, stencil);
 349    uint32_t new_clear_bits = 0, clear_bits_depth, clear_bits_stencil;
 350
 351    /* Get the channels to clear */
 352    switch (surf->base.format) {
 353    case PIPE_FORMAT_Z16_UNORM:
 354       clear_bits_depth = 0xffff;
 355       clear_bits_stencil = 0;
 356       break;
 357    case PIPE_FORMAT_X8Z24_UNORM:
 358    case PIPE_FORMAT_S8_UINT_Z24_UNORM:
 359       clear_bits_depth = 0xeeee;
 360       clear_bits_stencil = 0x1111;
 361       break;
 362    default:
 363       clear_bits_depth = clear_bits_stencil = 0xffff;
 364       break;
 365    }
 366
 367    if (buffers & PIPE_CLEAR_DEPTH)
 368       new_clear_bits |= clear_bits_depth;
 369    if (buffers & PIPE_CLEAR_STENCIL)
 370       new_clear_bits |= clear_bits_stencil;
 371    /* FIXME: when tile status is enabled, this becomes more complex as
 372     * we may separately clear the depth from the stencil.  In this case,
 373     * we want to resolve the surface, and avoid using the tile status.
 374     * We may be better off recording the pending clear operation,
 375     * delaying the actual clear to the first use.  This way, we can merge
 376     * consecutive clears together. */
 377    if (surf->surf.ts_size) { /* TS: use precompiled clear command */
 378       /* Set new clear depth value */
 379       ctx->framebuffer.TS_DEPTH_CLEAR_VALUE = new_clear_value;
 380       if (VIV_FEATURE(ctx->screen, chipMinorFeatures1, AUTO_DISABLE)) {
 381          /* Set number of depth tiles to be filled */
 382          etna_set_state(ctx->stream, VIVS_TS_DEPTH_AUTO_DISABLE_COUNT,
 383                         surf->surf.padded_width * surf->surf.padded_height / 16);
 384          ctx->framebuffer.TS_MEM_CONFIG |= VIVS_TS_MEM_CONFIG_DEPTH_AUTO_DISABLE;
 385       }
 386
 387       surf->level->ts_valid = true;
 388       ctx->dirty |= ETNA_DIRTY_TS | ETNA_DIRTY_DERIVE_TS;
 389    } else {
 390       if (unlikely(new_clear_value != surf->level->clear_value)) { /* Queue normal RS clear for non-TS surfaces */
 391          /* If clear depth value changed, re-generate stored command */
 392          etna_rs_gen_clear_surface(ctx, surf, new_clear_value);
 393       }
 394       /* Update the channels to be cleared */
 395       etna_modify_rs_clearbits(&surf->clear_command, new_clear_bits);
 396    }
 397
 398    etna_submit_rs_state(ctx, &surf->clear_command);
 399
 400    surf->level->clear_value = new_clear_value;
 401    resource_written(ctx, surf->base.texture);
 402    etna_resource(surf->base.texture)->seqno++;
 403 }
 404
 405 static void
 406 etna_clear_rs(struct pipe_context *pctx, unsigned buffers,
 407            const union pipe_color_union *color, double depth, unsigned stencil)
 408 {
 409    struct etna_context *ctx = etna_context(pctx);
 410    mtx_lock(&ctx->lock);
 411
 412    /* Flush color and depth cache before clearing anything.
 413     * This is especially important when coming from another surface, as
 414     * otherwise it may clear part of the old surface instead. */
 415    etna_set_state(ctx->stream, VIVS_GL_FLUSH_CACHE, VIVS_GL_FLUSH_CACHE_COLOR | VIVS_GL_FLUSH_CACHE_DEPTH);
 416    etna_stall(ctx->stream, SYNC_RECIPIENT_RA, SYNC_RECIPIENT_PE);
 417
 418    /* Preparation: Flush the TS if needed. This must be done after flushing
 419     * color and depth, otherwise it can result in crashes */
 420    bool need_ts_flush = false;
 421    if ((buffers & PIPE_CLEAR_COLOR) && ctx->framebuffer_s.nr_cbufs) {
 422       struct etna_surface *surf = etna_surface(ctx->framebuffer_s.cbufs[0]);
 423       if (surf->surf.ts_size)
 424          need_ts_flush = true;
 425    }
 426    if ((buffers & PIPE_CLEAR_DEPTHSTENCIL) && ctx->framebuffer_s.zsbuf != NULL) {
 427       struct etna_surface *surf = etna_surface(ctx->framebuffer_s.zsbuf);
 428
 429       if (surf->surf.ts_size)
 430          need_ts_flush = true;
 431    }
 432
 433    if (need_ts_flush)
 434       etna_set_state(ctx->stream, VIVS_TS_FLUSH_CACHE, VIVS_TS_FLUSH_CACHE_FLUSH);
 435
 436    /* No need to set up the TS here as RS clear operations (in contrast to
 437     * resolve and copy) do not require the TS state.
 438     */
 439    if (buffers & PIPE_CLEAR_COLOR) {
 440       for (int idx = 0; idx < ctx->framebuffer_s.nr_cbufs; ++idx) {
 441          etna_blit_clear_color_rs(pctx, ctx->framebuffer_s.cbufs[idx],
 442                                &color[idx]);
 443       }
 444    }
 445
 446    /* Flush the color and depth caches before each RS clear operation
 447     * This fixes a hang on GC600. */
 448    if (buffers & PIPE_CLEAR_DEPTHSTENCIL && buffers & PIPE_CLEAR_COLOR)
 449       etna_set_state(ctx->stream, VIVS_GL_FLUSH_CACHE,
 450                      VIVS_GL_FLUSH_CACHE_COLOR | VIVS_GL_FLUSH_CACHE_DEPTH);
 451
 452    if ((buffers & PIPE_CLEAR_DEPTHSTENCIL) && ctx->framebuffer_s.zsbuf != NULL)
 453       etna_blit_clear_zs_rs(pctx, ctx->framebuffer_s.zsbuf, buffers, depth, stencil);
 454
 455    etna_stall(ctx->stream, SYNC_RECIPIENT_RA, SYNC_RECIPIENT_PE);
 456    mtx_unlock(&ctx->lock);
 457 }
 458
 459 static bool
 460 etna_manual_blit(struct etna_resource *dst, struct etna_resource_level *dst_lev,
 461                  unsigned int dst_offset, struct etna_resource *src,
 462                  struct etna_resource_level *src_lev, unsigned int src_offset,
 463                  const struct pipe_blit_info *blit_info)
 464 {
 465    void *smap, *srow, *dmap, *drow;
 466    size_t tile_size;
 467
 468    assert(src->layout == ETNA_LAYOUT_TILED);
 469    assert(dst->layout == ETNA_LAYOUT_TILED);
 470    assert(src->base.nr_samples == 0);
 471    assert(dst->base.nr_samples == 0);
 472
 473    tile_size = util_format_get_blocksize(blit_info->src.format) * 4 * 4;
 474
 475    smap = etna_bo_map(src->bo);
 476    if (!smap)
 477       return false;
 478
 479    dmap = etna_bo_map(dst->bo);
 480    if (!dmap)
 481       return false;
 482
 483    srow = smap + src_offset;
 484    drow = dmap + dst_offset;
 485
 486    etna_bo_cpu_prep(src->bo, DRM_ETNA_PREP_READ);
 487    etna_bo_cpu_prep(dst->bo, DRM_ETNA_PREP_WRITE);
 488
 489    for (int y = 0; y < blit_info->src.box.height; y += 4) {
 490       memcpy(drow, srow, tile_size * blit_info->src.box.width);
 491       srow += src_lev->stride * 4;
 492       drow += dst_lev->stride * 4;
 493    }
 494
 495    etna_bo_cpu_fini(dst->bo);
 496    etna_bo_cpu_fini(src->bo);
 497
 498    return true;
 499 }
 500
 501 static inline size_t
 502 etna_compute_tileoffset(const struct pipe_box *box, enum pipe_format format,
 503                         size_t stride, enum etna_surface_layout layout)
 504 {
 505    size_t offset;
 506    unsigned int x = box->x, y = box->y;
 507    unsigned int blocksize = util_format_get_blocksize(format);
 508
 509    switch (layout) {
 510    case ETNA_LAYOUT_LINEAR:
 511       offset = y * stride + x * blocksize;
 512       break;
 513    case ETNA_LAYOUT_MULTI_TILED:
 514       y >>= 1;
 515       /* fall-through */
 516    case ETNA_LAYOUT_TILED:
 517       assert(!(x & 0x03) && !(y & 0x03));
 518       offset = (y & ~0x03) * stride + blocksize * ((x & ~0x03) << 2);
 519       break;
 520    case ETNA_LAYOUT_MULTI_SUPERTILED:
 521       y >>= 1;
 522       /* fall-through */
 523    case ETNA_LAYOUT_SUPER_TILED:
 524       assert(!(x & 0x3f) && !(y & 0x3f));
 525       offset = (y & ~0x3f) * stride + blocksize * ((x & ~0x3f) << 6);
 526       break;
 527    default:
 528       unreachable("invalid resource layout");
 529    }
 530
 531    return offset;
 532 }
 533
 534 static inline void
 535 etna_get_rs_alignment_mask(const struct etna_context *ctx,
 536                            const enum etna_surface_layout layout,
 537                            unsigned int *width_mask, unsigned int *height_mask)
 538 {
 539    unsigned int h_align, w_align;
 540
 541    if (layout & ETNA_LAYOUT_BIT_SUPER) {
 542       w_align = 64;
 543       h_align = 64 * ctx->specs.pixel_pipes;
 544    } else {
 545       w_align = ETNA_RS_WIDTH_MASK + 1;
 546       h_align = ETNA_RS_HEIGHT_MASK + 1;
 547    }
 548
 549    *width_mask = w_align - 1;
 550    *height_mask = h_align -1;
 551 }
 552
 553 static bool msaa_config(const struct pipe_resource *src,
 554                         const struct pipe_resource *dst,
 555                         int *msaa_xscale,
 556                         int *msaa_yscale)
 557 {
 558    int src_xscale = 1, src_yscale = 1;
 559    int dst_xscale = 1, dst_yscale = 1;
 560
 561    assert(src->nr_samples <= 4);
 562    assert(dst->nr_samples <= 4);
 563
 564    translate_samples_to_xyscale(src->nr_samples, &src_xscale, &src_yscale);
 565    translate_samples_to_xyscale(dst->nr_samples, &dst_xscale, &dst_yscale);
 566
 567    /* RS does not support upscaling */
 568    if ((src_xscale < dst_xscale) || (src_yscale < dst_yscale))
 569       return false;
 570
 571    *msaa_xscale = src_xscale - dst_xscale + 1;
 572    *msaa_yscale = src_yscale - dst_yscale + 1;
 573
 574    return true;
 575 }
 576
 577 static bool
 578 etna_try_rs_blit(struct pipe_context *pctx,
 579                  const struct pipe_blit_info *blit_info)
 580 {
 581    struct etna_context *ctx = etna_context(pctx);
 582    struct etna_resource *src = etna_resource(blit_info->src.resource);
 583    struct etna_resource *dst = etna_resource(blit_info->dst.resource);
 584    struct compiled_rs_state copy_to_screen;
 585    int msaa_xscale = 1, msaa_yscale = 1;
 586
 587    /* Ensure that the level is valid */
 588    assert(blit_info->src.level <= src->base.last_level);
 589    assert(blit_info->dst.level <= dst->base.last_level);
 590
 591    if (!msaa_config(&src->base, &dst->base, &msaa_xscale, &msaa_yscale)) {
 592       DBG("upsampling not supported");
 593       return false;
 594    }
 595
 596    /* The width/height are in pixels; they do not change as a result of
 597     * multi-sampling. So, when blitting from a 4x multisampled surface
 598     * to a non-multisampled surface, the width and height will be
 599     * identical. As we do not support scaling, reject different sizes. */
 600    if (blit_info->dst.box.width != blit_info->src.box.width ||
 601        blit_info->dst.box.height != blit_info->src.box.height) {
 602       DBG("scaling requested: source %dx%d destination %dx%d",
 603           blit_info->src.box.width, blit_info->src.box.height,
 604           blit_info->dst.box.width, blit_info->dst.box.height);
 605       return false;
 606    }
 607
 608    /* No masks - RS can't copy specific channels */
 609    unsigned mask = util_format_get_mask(blit_info->dst.format);
 610    if ((blit_info->mask & mask) != mask) {
 611       DBG("sub-mask requested: 0x%02x vs format mask 0x%02x", blit_info->mask, mask);
 612       return false;
 613    }
 614
 615    /* Only support same format (used tiling/detiling) blits for now.
 616     * TODO: figure out which different-format blits are possible and test them
 617     *  - fail if swizzle needed
 618     *  - avoid trying to convert between float/int formats?
 619     */
 620    if (blit_info->src.format != blit_info->dst.format)
 621       return false;
 622
 623    uint32_t format = etna_compatible_rs_format(blit_info->dst.format);
 624    if (format == ETNA_NO_MATCH)
 625       return false;
 626
 627    if (blit_info->scissor_enable ||
 628        blit_info->dst.box.depth != blit_info->src.box.depth ||
 629        blit_info->dst.box.depth != 1) {
 630       return false;
 631    }
 632
 633    unsigned w_mask, h_mask;
 634
 635    etna_get_rs_alignment_mask(ctx, src->layout, &w_mask, &h_mask);
 636    if ((blit_info->src.box.x & w_mask) || (blit_info->src.box.y & h_mask))
 637       return false;
 638
 639    etna_get_rs_alignment_mask(ctx, dst->layout, &w_mask, &h_mask);
 640    if ((blit_info->dst.box.x & w_mask) || (blit_info->dst.box.y & h_mask))
 641       return false;
 642
 643    struct etna_resource_level *src_lev = &src->levels[blit_info->src.level];
 644    struct etna_resource_level *dst_lev = &dst->levels[blit_info->dst.level];
 645
 646    /* we may be given coordinates up to the padded width to avoid
 647     * any alignment issues with different tiling formats */
 648    assert((blit_info->src.box.x + blit_info->src.box.width) * msaa_xscale <= src_lev->padded_width);
 649    assert((blit_info->src.box.y + blit_info->src.box.height) * msaa_yscale <= src_lev->padded_height);
 650    assert(blit_info->dst.box.x + blit_info->dst.box.width <= dst_lev->padded_width);
 651    assert(blit_info->dst.box.y + blit_info->dst.box.height <= dst_lev->padded_height);
 652
 653    unsigned src_offset = src_lev->offset +
 654                          blit_info->src.box.z * src_lev->layer_stride +
 655                          etna_compute_tileoffset(&blit_info->src.box,
 656                                                  blit_info->src.format,
 657                                                  src_lev->stride,
 658                                                  src->layout);
 659    unsigned dst_offset = dst_lev->offset +
 660                          blit_info->dst.box.z * dst_lev->layer_stride +
 661                          etna_compute_tileoffset(&blit_info->dst.box,
 662                                                  blit_info->dst.format,
 663                                                  dst_lev->stride,
 664                                                  dst->layout);
 665
 666    if (src_lev->padded_width <= ETNA_RS_WIDTH_MASK ||
 667        dst_lev->padded_width <= ETNA_RS_WIDTH_MASK ||
 668        src_lev->padded_height <= ETNA_RS_HEIGHT_MASK ||
 669        dst_lev->padded_height <= ETNA_RS_HEIGHT_MASK)
 670       goto manual;
 671
 672    /* If the width is not aligned to the RS width, but is within our
 673     * padding, adjust the width to suite the RS width restriction.
 674     * Note: the RS width/height are converted to source samples here. */
 675    unsigned int width = blit_info->src.box.width * msaa_xscale;
 676    unsigned int height = blit_info->src.box.height * msaa_yscale;
 677    unsigned int w_align = ETNA_RS_WIDTH_MASK + 1;
 678    unsigned int h_align = ETNA_RS_HEIGHT_MASK + 1;
 679
 680    if (width & (w_align - 1) && width >= src_lev->width * msaa_xscale && width >= dst_lev->width)
 681       width = align(width, w_align);
 682
 683    if (height & (h_align - 1) && height >= src_lev->height * msaa_yscale && height >= dst_lev->height)
 684       height = align(height, h_align);
 685
 686    /* The padded dimensions are in samples */
 687    if (width > src_lev->padded_width ||
 688        width > dst_lev->padded_width * msaa_xscale ||
 689        height > src_lev->padded_height ||
 690        height > dst_lev->padded_height * msaa_yscale ||
 691        width & (w_align - 1) || height & (h_align - 1))
 692       goto manual;
 693
 694    mtx_lock(&ctx->lock);
 695
 696    /* Always flush color and depth cache together before resolving. This works
 697     * around artifacts that appear in some cases when scanning out a texture
 698     * directly after it has been rendered to, such as rendering an animated web
 699     * page in a QtWebEngine based WebView on GC2000. The artifacts look like
 700     * the texture sampler samples zeroes instead of texture data in a small,
 701     * irregular triangle in the lower right of each browser tile quad. Other
 702     * attempts to avoid these artifacts, including a pipeline stall before the
 703     * color flush or a TS cache flush afterwards, or flushing multiple times,
 704     * with stalls before and after each flush, have shown no effect. */
 705    if (src->base.bind & PIPE_BIND_RENDER_TARGET ||
 706        src->base.bind & PIPE_BIND_DEPTH_STENCIL) {
 707       etna_set_state(ctx->stream, VIVS_GL_FLUSH_CACHE,
 708                      VIVS_GL_FLUSH_CACHE_COLOR | VIVS_GL_FLUSH_CACHE_DEPTH);
 709       etna_stall(ctx->stream, SYNC_RECIPIENT_RA, SYNC_RECIPIENT_PE);
 710
 711       if (src_lev->ts_size && src_lev->ts_valid)
 712          etna_set_state(ctx->stream, VIVS_TS_FLUSH_CACHE, VIVS_TS_FLUSH_CACHE_FLUSH);
 713    }
 714
 715    /* Set up color TS to source surface before blit, if needed */
 716    bool source_ts_valid = false;
 717    if (src_lev->ts_size && src_lev->ts_valid) {
 718       struct etna_reloc reloc;
 719       unsigned ts_offset =
 720          src_lev->ts_offset + blit_info->src.box.z * src_lev->ts_layer_stride;
 721       uint32_t ts_mem_config = 0;
 722
 723       if (src_lev->ts_compress_fmt >= 0) {
 724          ts_mem_config |= VIVS_TS_MEM_CONFIG_COLOR_COMPRESSION |
 725                           VIVS_TS_MEM_CONFIG_COLOR_COMPRESSION_FORMAT(src_lev->ts_compress_fmt);
 726       }
 727
 728       etna_set_state(ctx->stream, VIVS_TS_MEM_CONFIG,
 729                      VIVS_TS_MEM_CONFIG_COLOR_FAST_CLEAR | ts_mem_config);
 730
 731       memset(&reloc, 0, sizeof(struct etna_reloc));
 732       reloc.bo = src->ts_bo;
 733       reloc.offset = ts_offset;
 734       reloc.flags = ETNA_RELOC_READ;
 735       etna_set_state_reloc(ctx->stream, VIVS_TS_COLOR_STATUS_BASE, &reloc);
 736
 737       memset(&reloc, 0, sizeof(struct etna_reloc));
 738       reloc.bo = src->bo;
 739       reloc.offset = src_lev->offset +
 740                      blit_info->src.box.z * src_lev->layer_stride;
 741       reloc.flags = ETNA_RELOC_READ;
 742       etna_set_state_reloc(ctx->stream, VIVS_TS_COLOR_SURFACE_BASE, &reloc);
 743
 744       etna_set_state(ctx->stream, VIVS_TS_COLOR_CLEAR_VALUE, src_lev->clear_value);
 745
 746       source_ts_valid = true;
 747    } else {
 748       etna_set_state(ctx->stream, VIVS_TS_MEM_CONFIG, 0);
 749    }
 750    ctx->dirty |= ETNA_DIRTY_TS;
 751
 752    /* Kick off RS here */
 753    etna_compile_rs_state(ctx, &copy_to_screen, &(struct rs_state) {
 754       .source_format = format,
 755       .source_tiling = src->layout,
 756       .source = src->bo,
 757       .source_offset = src_offset,
 758       .source_stride = src_lev->stride,
 759       .source_padded_width = src_lev->padded_width,
 760       .source_padded_height = src_lev->padded_height,
 761       .source_ts_valid = source_ts_valid,
 762       .source_ts_compressed = src_lev->ts_compress_fmt >= 0,
 763       .dest_format = format,
 764       .dest_tiling = dst->layout,
 765       .dest = dst->bo,
 766       .dest_offset = dst_offset,
 767       .dest_stride = dst_lev->stride,
 768       .dest_padded_height = dst_lev->padded_height,
 769       .downsample_x = msaa_xscale > 1,
 770       .downsample_y = msaa_yscale > 1,
 771       .swap_rb = translate_rb_src_dst_swap(src->base.format, dst->base.format),
 772       .dither = {0xffffffff, 0xffffffff}, // XXX dither when going from 24 to 16 bit?
 773       .clear_mode = VIVS_RS_CLEAR_CONTROL_MODE_DISABLED,
 774       .width = width,
 775       .height = height,
 776       .tile_count = src_lev->layer_stride / 64
 777    });
 778
 779    etna_submit_rs_state(ctx, &copy_to_screen);
 780    resource_read(ctx, &src->base);
 781    resource_written(ctx, &dst->base);
 782    dst->seqno++;
 783    dst_lev->ts_valid = false;
 784    ctx->dirty |= ETNA_DIRTY_DERIVE_TS;
 785    mtx_unlock(&ctx->lock);
 786
 787    return true;
 788
 789 manual:
 790    if (src->layout == ETNA_LAYOUT_TILED && dst->layout == ETNA_LAYOUT_TILED) {
 791       if ((src->status & ETNA_PENDING_WRITE) ||
 792           (dst->status & ETNA_PENDING_WRITE))
 793          pctx->flush(pctx, NULL, 0);
 794       return etna_manual_blit(dst, dst_lev, dst_offset, src, src_lev, src_offset, blit_info);
 795    }
 796
 797    return false;
 798 }
 799
 800 static void
 801 etna_blit_rs(struct pipe_context *pctx, const struct pipe_blit_info *blit_info)
 802 {
 803    /* This is a more extended version of resource_copy_region */
 804    /* TODO Some cases can be handled by RS; if not, fall back to rendering or
 805     * even CPU copy block of pixels from info->src to info->dst
 806     * (resource, level, box, format);
 807     * function is used for scaling, flipping in x and y direction (negative
 808     * width/height), format conversion, mask and filter and even a scissor rectangle
 809     *
 810     * What can the RS do for us:
 811     *   convert between tiling formats (layouts)
 812     *   downsample 2x in x and y
 813     *   convert between a limited number of pixel formats
 814     *
 815     * For the rest, fall back to util_blitter
 816     * XXX this goes wrong when source surface is supertiled. */
 817    struct etna_context *ctx = etna_context(pctx);
 818    struct pipe_blit_info info = *blit_info;
 819
 820    if (info.src.resource->nr_samples > 1 &&
 821        info.dst.resource->nr_samples <= 1 &&
 822        !util_format_is_depth_or_stencil(info.src.resource->format) &&
 823        !util_format_is_pure_integer(info.src.resource->format)) {
 824       DBG("color resolve unimplemented");
 825       return;
 826    }
 827
 828    if (etna_try_rs_blit(pctx, blit_info))
 829       return;
 830
 831    if (util_try_blit_via_copy_region(pctx, blit_info))
 832       return;
 833
 834    if (info.mask & PIPE_MASK_S) {
 835       DBG("cannot blit stencil, skipping");
 836       info.mask &= ~PIPE_MASK_S;
 837    }
 838
 839    if (!util_blitter_is_blit_supported(ctx->blitter, &info)) {
 840       DBG("blit unsupported %s -> %s",
 841           util_format_short_name(info.src.resource->format),
 842           util_format_short_name(info.dst.resource->format));
 843       return;
 844    }
 845
 846    etna_blit_save_state(ctx);
 847    util_blitter_blit(ctx->blitter, &info);
 848 }
 849
 850 void
 851 etna_clear_blit_rs_init(struct pipe_context *pctx)
 852 {
 853    DBG("etnaviv: Using RS blit engine");
 854    pctx->clear = etna_clear_rs;
 855    pctx->blit = etna_blit_rs;
 856 }