src/gallium/drivers/etnaviv/etnaviv_rs.c

   1 /*
   2  * Copyright (c) 2012-2017 Etnaviv Project
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sub license,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the
  12  * next paragraph) shall be included in all copies or substantial portions
  13  * of the Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Wladimir J. van der Laan <laanwj@gmail.com>
  25  */
  26
  27 #include "etnaviv_rs.h"
  28
  29 #include "etnaviv_clear_blit.h"
  30 #include "etnaviv_context.h"
  31 #include "etnaviv_emit.h"
  32 #include "etnaviv_format.h"
  33 #include "etnaviv_resource.h"
  34 #include "etnaviv_screen.h"
  35 #include "etnaviv_surface.h"
  36 #include "etnaviv_tiling.h"
  37 #include "etnaviv_translate.h"
  38 #include "etnaviv_util.h"
  39
  40 #include "pipe/p_defines.h"
  41 #include "pipe/p_state.h"
  42 #include "util/u_blitter.h"
  43 #include "util/u_inlines.h"
  44 #include "util/u_memory.h"
  45 #include "util/u_surface.h"
  46
  47 #include "hw/common.xml.h"
  48 #include "hw/state.xml.h"
  49 #include "hw/state_3d.xml.h"
  50
  51 #include <assert.h>
  52
  53 /* return a RS "compatible" format for use when copying */
  54 static uint32_t
  55 etna_compatible_rs_format(enum pipe_format fmt)
  56 {
  57    /* YUYV and UYVY are blocksize 4, but 2 bytes per pixel */
  58    if (fmt == PIPE_FORMAT_YUYV || fmt == PIPE_FORMAT_UYVY)
  59       return RS_FORMAT_A4R4G4B4;
  60
  61    switch (util_format_get_blocksize(fmt)) {
  62    case 2: return RS_FORMAT_A4R4G4B4;
  63    case 4: return RS_FORMAT_A8R8G8B8;
  64    default: return ETNA_NO_MATCH;
  65    }
  66 }
  67
  68 void
  69 etna_compile_rs_state(struct etna_context *ctx, struct compiled_rs_state *cs,
  70                       const struct rs_state *rs)
  71 {
  72    struct etna_screen *screen = ctx->screen;
  73
  74    memset(cs, 0, sizeof(*cs));
  75
  76    /* TILED and SUPERTILED layout have their strides multiplied with 4 in RS */
  77    unsigned source_stride_shift = COND(rs->source_tiling != ETNA_LAYOUT_LINEAR, 2);
  78    unsigned dest_stride_shift = COND(rs->dest_tiling != ETNA_LAYOUT_LINEAR, 2);
  79
  80    /* tiling == ETNA_LAYOUT_MULTI_TILED or ETNA_LAYOUT_MULTI_SUPERTILED? */
  81    int source_multi = COND(rs->source_tiling & ETNA_LAYOUT_BIT_MULTI, 1);
  82    int dest_multi = COND(rs->dest_tiling & ETNA_LAYOUT_BIT_MULTI, 1);
  83
  84    /* Vivante RS needs widths to be a multiple of 16 or bad things
  85     * happen, such as scribbing over memory, or the GPU hanging,
  86     * even for non-tiled formats.  As this is serious, use abort().
  87     */
  88    if (rs->width & ETNA_RS_WIDTH_MASK)
  89       abort();
  90
  91    /* TODO could just pre-generate command buffer, would simply submit to one memcpy */
  92    cs->RS_CONFIG = VIVS_RS_CONFIG_SOURCE_FORMAT(rs->source_format) |
  93                    COND(rs->downsample_x, VIVS_RS_CONFIG_DOWNSAMPLE_X) |
  94                    COND(rs->downsample_y, VIVS_RS_CONFIG_DOWNSAMPLE_Y) |
  95                    COND(rs->source_tiling & 1, VIVS_RS_CONFIG_SOURCE_TILED) |
  96                    VIVS_RS_CONFIG_DEST_FORMAT(rs->dest_format) |
  97                    COND(rs->dest_tiling & 1, VIVS_RS_CONFIG_DEST_TILED) |
  98                    COND(rs->swap_rb, VIVS_RS_CONFIG_SWAP_RB) |
  99                    COND(rs->flip, VIVS_RS_CONFIG_FLIP);
 100
 101    cs->RS_SOURCE_STRIDE = (rs->source_stride << source_stride_shift) |
 102                           COND(rs->source_tiling & 2, VIVS_RS_SOURCE_STRIDE_TILING) |
 103                           COND(source_multi, VIVS_RS_SOURCE_STRIDE_MULTI);
 104
 105    /* Initially all pipes are set to the base address of the source and
 106     * destination buffer respectively. This will be overridden below as
 107     * necessary for the multi-pipe, multi-tiled case.
 108     */
 109    for (unsigned pipe = 0; pipe < screen->specs.pixel_pipes; ++pipe) {
 110       cs->source[pipe].bo = rs->source;
 111       cs->source[pipe].offset = rs->source_offset;
 112       cs->source[pipe].flags = ETNA_RELOC_READ;
 113
 114       cs->dest[pipe].bo = rs->dest;
 115       cs->dest[pipe].offset = rs->dest_offset;
 116       cs->dest[pipe].flags = ETNA_RELOC_WRITE;
 117
 118       cs->RS_PIPE_OFFSET[pipe] = VIVS_RS_PIPE_OFFSET_X(0) | VIVS_RS_PIPE_OFFSET_Y(0);
 119    }
 120
 121    cs->RS_DEST_STRIDE = (rs->dest_stride << dest_stride_shift) |
 122                         COND(rs->dest_tiling & 2, VIVS_RS_DEST_STRIDE_TILING) |
 123                         COND(dest_multi, VIVS_RS_DEST_STRIDE_MULTI);
 124
 125
 126    if (source_multi)
 127       cs->source[1].offset = rs->source_offset + rs->source_stride * rs->source_padded_height / 2;
 128
 129    if (dest_multi)
 130       cs->dest[1].offset = rs->dest_offset + rs->dest_stride * rs->dest_padded_height / 2;
 131
 132    cs->RS_WINDOW_SIZE = VIVS_RS_WINDOW_SIZE_WIDTH(rs->width) |
 133                         VIVS_RS_WINDOW_SIZE_HEIGHT(rs->height);
 134
 135    /* use dual pipe mode when required */
 136    if (!screen->specs.single_buffer && screen->specs.pixel_pipes == 2 && !(rs->height & 7)) {
 137       cs->RS_WINDOW_SIZE = VIVS_RS_WINDOW_SIZE_WIDTH(rs->width) |
 138                               VIVS_RS_WINDOW_SIZE_HEIGHT(rs->height / 2);
 139       cs->RS_PIPE_OFFSET[1] = VIVS_RS_PIPE_OFFSET_X(0) | VIVS_RS_PIPE_OFFSET_Y(rs->height / 2);
 140    }
 141
 142    cs->RS_DITHER[0] = rs->dither[0];
 143    cs->RS_DITHER[1] = rs->dither[1];
 144    cs->RS_CLEAR_CONTROL = VIVS_RS_CLEAR_CONTROL_BITS(rs->clear_bits) | rs->clear_mode;
 145    cs->RS_FILL_VALUE[0] = rs->clear_value[0];
 146    cs->RS_FILL_VALUE[1] = rs->clear_value[1];
 147    cs->RS_FILL_VALUE[2] = rs->clear_value[2];
 148    cs->RS_FILL_VALUE[3] = rs->clear_value[3];
 149    cs->RS_EXTRA_CONFIG = VIVS_RS_EXTRA_CONFIG_AA(rs->aa) |
 150                          VIVS_RS_EXTRA_CONFIG_ENDIAN(rs->endian_mode);
 151
 152    /* If source the same as destination, and the hardware supports this,
 153     * do an in-place resolve to fill in unrendered tiles.
 154     */
 155    if (screen->specs.single_buffer && rs->source == rs->dest &&
 156          rs->source_offset == rs->dest_offset &&
 157          rs->source_format == rs->dest_format &&
 158          rs->source_tiling == rs->dest_tiling &&
 159          (rs->source_tiling & ETNA_LAYOUT_BIT_SUPER) &&
 160          rs->source_stride == rs->dest_stride &&
 161          !rs->downsample_x && !rs->downsample_y &&
 162          !rs->swap_rb && !rs->flip &&
 163          !rs->clear_mode && rs->source_padded_width &&
 164          !rs->source_ts_compressed) {
 165       /* Total number of tiles (same as for autodisable) */
 166       cs->RS_KICKER_INPLACE = rs->tile_count;
 167    }
 168    cs->source_ts_valid = rs->source_ts_valid;
 169 }
 170
 171 /* modify the clear bits value in the compiled RS state */
 172 static void
 173 etna_modify_rs_clearbits(struct compiled_rs_state *cs, uint32_t clear_bits)
 174 {
 175    cs->RS_CLEAR_CONTROL &= ~VIVS_RS_CLEAR_CONTROL_BITS__MASK;
 176    cs->RS_CLEAR_CONTROL |= VIVS_RS_CLEAR_CONTROL_BITS(clear_bits);
 177 }
 178
 179 #define EMIT_STATE(state_name, src_value) \
 180    etna_coalsence_emit(stream, &coalesce, VIVS_##state_name, src_value)
 181
 182 #define EMIT_STATE_FIXP(state_name, src_value) \
 183    etna_coalsence_emit_fixp(stream, &coalesce, VIVS_##state_name, src_value)
 184
 185 #define EMIT_STATE_RELOC(state_name, src_value) \
 186    etna_coalsence_emit_reloc(stream, &coalesce, VIVS_##state_name, src_value)
 187
 188 /* submit RS state, without any processing and no dependence on context
 189  * except TS if this is a source-to-destination blit. */
 190 static void
 191 etna_submit_rs_state(struct etna_context *ctx,
 192                      const struct compiled_rs_state *cs)
 193 {
 194    struct etna_screen *screen = etna_screen(ctx->base.screen);
 195    struct etna_cmd_stream *stream = ctx->stream;
 196    struct etna_coalesce coalesce;
 197
 198    if (cs->RS_KICKER_INPLACE && !cs->source_ts_valid)
 199       /* Inplace resolve is no-op if TS is not configured */
 200       return;
 201
 202    ctx->stats.rs_operations++;
 203
 204    if (cs->RS_KICKER_INPLACE) {
 205       etna_cmd_stream_reserve(stream, 6);
 206       etna_coalesce_start(stream, &coalesce);
 207       /* 0/1 */ EMIT_STATE(RS_EXTRA_CONFIG, cs->RS_EXTRA_CONFIG);
 208       /* 2/3 */ EMIT_STATE(RS_SOURCE_STRIDE, cs->RS_SOURCE_STRIDE);
 209       /* 4/5 */ EMIT_STATE(RS_KICKER_INPLACE, cs->RS_KICKER_INPLACE);
 210       etna_coalesce_end(stream, &coalesce);
 211    } else if (screen->specs.pixel_pipes == 1) {
 212       etna_cmd_stream_reserve(stream, 22);
 213       etna_coalesce_start(stream, &coalesce);
 214       /* 0/1 */ EMIT_STATE(RS_CONFIG, cs->RS_CONFIG);
 215       /* 2   */ EMIT_STATE_RELOC(RS_SOURCE_ADDR, &cs->source[0]);
 216       /* 3   */ EMIT_STATE(RS_SOURCE_STRIDE, cs->RS_SOURCE_STRIDE);
 217       /* 4   */ EMIT_STATE_RELOC(RS_DEST_ADDR, &cs->dest[0]);
 218       /* 5   */ EMIT_STATE(RS_DEST_STRIDE, cs->RS_DEST_STRIDE);
 219       /* 6/7 */ EMIT_STATE(RS_WINDOW_SIZE, cs->RS_WINDOW_SIZE);
 220       /* 8/9 */ EMIT_STATE(RS_DITHER(0), cs->RS_DITHER[0]);
 221       /*10   */ EMIT_STATE(RS_DITHER(1), cs->RS_DITHER[1]);
 222       /*11 - pad */
 223       /*12/13*/ EMIT_STATE(RS_CLEAR_CONTROL, cs->RS_CLEAR_CONTROL);
 224       /*14   */ EMIT_STATE(RS_FILL_VALUE(0), cs->RS_FILL_VALUE[0]);
 225       /*15   */ EMIT_STATE(RS_FILL_VALUE(1), cs->RS_FILL_VALUE[1]);
 226       /*16   */ EMIT_STATE(RS_FILL_VALUE(2), cs->RS_FILL_VALUE[2]);
 227       /*17   */ EMIT_STATE(RS_FILL_VALUE(3), cs->RS_FILL_VALUE[3]);
 228       /*18/19*/ EMIT_STATE(RS_EXTRA_CONFIG, cs->RS_EXTRA_CONFIG);
 229       /*20/21*/ EMIT_STATE(RS_KICKER, 0xbeebbeeb);
 230       etna_coalesce_end(stream, &coalesce);
 231    } else if (screen->specs.pixel_pipes == 2) {
 232       etna_cmd_stream_reserve(stream, 34); /* worst case - both pipes multi=1 */
 233       etna_coalesce_start(stream, &coalesce);
 234       /* 0/1 */ EMIT_STATE(RS_CONFIG, cs->RS_CONFIG);
 235       /* 2/3 */ EMIT_STATE(RS_SOURCE_STRIDE, cs->RS_SOURCE_STRIDE);
 236       /* 4/5 */ EMIT_STATE(RS_DEST_STRIDE, cs->RS_DEST_STRIDE);
 237       /* 6/7 */ EMIT_STATE_RELOC(RS_PIPE_SOURCE_ADDR(0), &cs->source[0]);
 238       if (cs->RS_SOURCE_STRIDE & VIVS_RS_SOURCE_STRIDE_MULTI) {
 239          /*8 */ EMIT_STATE_RELOC(RS_PIPE_SOURCE_ADDR(1), &cs->source[1]);
 240          /*9 - pad */
 241       }
 242       /*10/11*/ EMIT_STATE_RELOC(RS_PIPE_DEST_ADDR(0), &cs->dest[0]);
 243       if (cs->RS_DEST_STRIDE & VIVS_RS_DEST_STRIDE_MULTI) {
 244          /*12*/ EMIT_STATE_RELOC(RS_PIPE_DEST_ADDR(1), &cs->dest[1]);
 245          /*13 - pad */
 246       }
 247       /*14/15*/ EMIT_STATE(RS_PIPE_OFFSET(0), cs->RS_PIPE_OFFSET[0]);
 248       /*16   */ EMIT_STATE(RS_PIPE_OFFSET(1), cs->RS_PIPE_OFFSET[1]);
 249       /*17 - pad */
 250       /*18/19*/ EMIT_STATE(RS_WINDOW_SIZE, cs->RS_WINDOW_SIZE);
 251       /*20/21*/ EMIT_STATE(RS_DITHER(0), cs->RS_DITHER[0]);
 252       /*22   */ EMIT_STATE(RS_DITHER(1), cs->RS_DITHER[1]);
 253       /*23 - pad */
 254       /*24/25*/ EMIT_STATE(RS_CLEAR_CONTROL, cs->RS_CLEAR_CONTROL);
 255       /*26   */ EMIT_STATE(RS_FILL_VALUE(0), cs->RS_FILL_VALUE[0]);
 256       /*27   */ EMIT_STATE(RS_FILL_VALUE(1), cs->RS_FILL_VALUE[1]);
 257       /*28   */ EMIT_STATE(RS_FILL_VALUE(2), cs->RS_FILL_VALUE[2]);
 258       /*29   */ EMIT_STATE(RS_FILL_VALUE(3), cs->RS_FILL_VALUE[3]);
 259       /*30/31*/ EMIT_STATE(RS_EXTRA_CONFIG, cs->RS_EXTRA_CONFIG);
 260       /*32/33*/ EMIT_STATE(RS_KICKER, 0xbeebbeeb);
 261       etna_coalesce_end(stream, &coalesce);
 262    } else {
 263       abort();
 264    }
 265 }
 266
 267 /* Generate clear command for a surface (non-fast clear case) */
 268 void
 269 etna_rs_gen_clear_surface(struct etna_context *ctx, struct etna_surface *surf,
 270                           uint64_t clear_value)
 271 {
 272    struct etna_screen *screen = ctx->screen;
 273    struct etna_resource *dst = etna_resource(surf->base.texture);
 274    uint32_t format;
 275
 276    switch (util_format_get_blocksizebits(surf->base.format)) {
 277    case 16:
 278       format = RS_FORMAT_A4R4G4B4;
 279       break;
 280    case 32:
 281       format = RS_FORMAT_A8R8G8B8;
 282       break;
 283    case 64:
 284       assert(screen->specs.halti >= 2);
 285       format = RS_FORMAT_64BPP_CLEAR;
 286       break;
 287    default:
 288       unreachable("bpp not supported for clear by RS");
 289       break;
 290    }
 291
 292    /* use tiled clear if width is multiple of 16 */
 293    bool tiled_clear = (surf->surf.padded_width & ETNA_RS_WIDTH_MASK) == 0 &&
 294                       (surf->surf.padded_height & ETNA_RS_HEIGHT_MASK) == 0;
 295
 296    etna_compile_rs_state( ctx, &surf->clear_command, &(struct rs_state) {
 297       .source_format = format,
 298       .dest_format = format,
 299       .dest = dst->bo,
 300       .dest_offset = surf->surf.offset,
 301       .dest_stride = surf->surf.stride,
 302       .dest_padded_height = surf->surf.padded_height,
 303       .dest_tiling = tiled_clear ? dst->layout : ETNA_LAYOUT_LINEAR,
 304       .dither = {0xffffffff, 0xffffffff},
 305       .width = surf->surf.padded_width, /* These must be padded to 16x4 if !LINEAR, otherwise RS will hang */
 306       .height = surf->surf.padded_height,
 307       .clear_value = {clear_value, clear_value >> 32, clear_value, clear_value >> 32},
 308       .clear_mode = VIVS_RS_CLEAR_CONTROL_MODE_ENABLED1,
 309       .clear_bits = 0xffff
 310    });
 311 }
 312
 313 static void
 314 etna_blit_clear_color_rs(struct pipe_context *pctx, struct pipe_surface *dst,
 315                       const union pipe_color_union *color)
 316 {
 317    struct etna_context *ctx = etna_context(pctx);
 318    struct etna_surface *surf = etna_surface(dst);
 319    uint64_t new_clear_value = etna_clear_blit_pack_rgba(surf->base.format, color);
 320
 321    if (surf->surf.ts_size) { /* TS: use precompiled clear command */
 322       ctx->framebuffer.TS_COLOR_CLEAR_VALUE = new_clear_value;
 323       ctx->framebuffer.TS_COLOR_CLEAR_VALUE_EXT = new_clear_value >> 32;
 324
 325       if (VIV_FEATURE(ctx->screen, chipMinorFeatures1, AUTO_DISABLE)) {
 326          /* Set number of color tiles to be filled */
 327          etna_set_state(ctx->stream, VIVS_TS_COLOR_AUTO_DISABLE_COUNT,
 328                         surf->surf.padded_width * surf->surf.padded_height / 16);
 329          ctx->framebuffer.TS_MEM_CONFIG |= VIVS_TS_MEM_CONFIG_COLOR_AUTO_DISABLE;
 330       }
 331
 332       surf->level->ts_valid = true;
 333       ctx->dirty |= ETNA_DIRTY_TS | ETNA_DIRTY_DERIVE_TS;
 334    } else if (unlikely(new_clear_value != surf->level->clear_value)) { /* Queue normal RS clear for non-TS surfaces */
 335       /* If clear color changed, re-generate stored command */
 336       etna_rs_gen_clear_surface(ctx, surf, new_clear_value);
 337    }
 338
 339    etna_submit_rs_state(ctx, &surf->clear_command);
 340
 341    surf->level->clear_value = new_clear_value;
 342    resource_written(ctx, surf->base.texture);
 343    etna_resource(surf->base.texture)->seqno++;
 344 }
 345
 346 static void
 347 etna_blit_clear_zs_rs(struct pipe_context *pctx, struct pipe_surface *dst,
 348                    unsigned buffers, double depth, unsigned stencil)
 349 {
 350    struct etna_context *ctx = etna_context(pctx);
 351    struct etna_surface *surf = etna_surface(dst);
 352    uint32_t new_clear_value = translate_clear_depth_stencil(surf->base.format, depth, stencil);
 353    uint32_t new_clear_bits = 0, clear_bits_depth, clear_bits_stencil;
 354
 355    /* Get the channels to clear */
 356    switch (surf->base.format) {
 357    case PIPE_FORMAT_Z16_UNORM:
 358       clear_bits_depth = 0xffff;
 359       clear_bits_stencil = 0;
 360       break;
 361    case PIPE_FORMAT_X8Z24_UNORM:
 362    case PIPE_FORMAT_S8_UINT_Z24_UNORM:
 363       clear_bits_depth = 0xeeee;
 364       clear_bits_stencil = 0x1111;
 365       break;
 366    default:
 367       clear_bits_depth = clear_bits_stencil = 0xffff;
 368       break;
 369    }
 370
 371    if (buffers & PIPE_CLEAR_DEPTH)
 372       new_clear_bits |= clear_bits_depth;
 373    if (buffers & PIPE_CLEAR_STENCIL)
 374       new_clear_bits |= clear_bits_stencil;
 375    /* FIXME: when tile status is enabled, this becomes more complex as
 376     * we may separately clear the depth from the stencil.  In this case,
 377     * we want to resolve the surface, and avoid using the tile status.
 378     * We may be better off recording the pending clear operation,
 379     * delaying the actual clear to the first use.  This way, we can merge
 380     * consecutive clears together. */
 381    if (surf->surf.ts_size) { /* TS: use precompiled clear command */
 382       /* Set new clear depth value */
 383       ctx->framebuffer.TS_DEPTH_CLEAR_VALUE = new_clear_value;
 384       if (VIV_FEATURE(ctx->screen, chipMinorFeatures1, AUTO_DISABLE)) {
 385          /* Set number of depth tiles to be filled */
 386          etna_set_state(ctx->stream, VIVS_TS_DEPTH_AUTO_DISABLE_COUNT,
 387                         surf->surf.padded_width * surf->surf.padded_height / 16);
 388          ctx->framebuffer.TS_MEM_CONFIG |= VIVS_TS_MEM_CONFIG_DEPTH_AUTO_DISABLE;
 389       }
 390
 391       surf->level->ts_valid = true;
 392       ctx->dirty |= ETNA_DIRTY_TS | ETNA_DIRTY_DERIVE_TS;
 393    } else {
 394       if (unlikely(new_clear_value != surf->level->clear_value)) { /* Queue normal RS clear for non-TS surfaces */
 395          /* If clear depth value changed, re-generate stored command */
 396          etna_rs_gen_clear_surface(ctx, surf, new_clear_value);
 397       }
 398       /* Update the channels to be cleared */
 399       etna_modify_rs_clearbits(&surf->clear_command, new_clear_bits);
 400    }
 401
 402    etna_submit_rs_state(ctx, &surf->clear_command);
 403
 404    surf->level->clear_value = new_clear_value;
 405    resource_written(ctx, surf->base.texture);
 406    etna_resource(surf->base.texture)->seqno++;
 407 }
 408
 409 static void
 410 etna_clear_rs(struct pipe_context *pctx, unsigned buffers, const struct pipe_scissor_state *scissor_state,
 411            const union pipe_color_union *color, double depth, unsigned stencil)
 412 {
 413    struct etna_context *ctx = etna_context(pctx);
 414    mtx_lock(&ctx->lock);
 415
 416    /* Flush color and depth cache before clearing anything.
 417     * This is especially important when coming from another surface, as
 418     * otherwise it may clear part of the old surface instead. */
 419    etna_set_state(ctx->stream, VIVS_GL_FLUSH_CACHE, VIVS_GL_FLUSH_CACHE_COLOR | VIVS_GL_FLUSH_CACHE_DEPTH);
 420    etna_stall(ctx->stream, SYNC_RECIPIENT_RA, SYNC_RECIPIENT_PE);
 421
 422    /* Preparation: Flush the TS if needed. This must be done after flushing
 423     * color and depth, otherwise it can result in crashes */
 424    bool need_ts_flush = false;
 425    if ((buffers & PIPE_CLEAR_COLOR) && ctx->framebuffer_s.nr_cbufs) {
 426       struct etna_surface *surf = etna_surface(ctx->framebuffer_s.cbufs[0]);
 427       if (surf->surf.ts_size)
 428          need_ts_flush = true;
 429    }
 430    if ((buffers & PIPE_CLEAR_DEPTHSTENCIL) && ctx->framebuffer_s.zsbuf != NULL) {
 431       struct etna_surface *surf = etna_surface(ctx->framebuffer_s.zsbuf);
 432
 433       if (surf->surf.ts_size)
 434          need_ts_flush = true;
 435    }
 436
 437    if (need_ts_flush)
 438       etna_set_state(ctx->stream, VIVS_TS_FLUSH_CACHE, VIVS_TS_FLUSH_CACHE_FLUSH);
 439
 440    /* No need to set up the TS here as RS clear operations (in contrast to
 441     * resolve and copy) do not require the TS state.
 442     */
 443    if (buffers & PIPE_CLEAR_COLOR) {
 444       for (int idx = 0; idx < ctx->framebuffer_s.nr_cbufs; ++idx) {
 445          etna_blit_clear_color_rs(pctx, ctx->framebuffer_s.cbufs[idx],
 446                                &color[idx]);
 447       }
 448    }
 449
 450    /* Flush the color and depth caches before each RS clear operation
 451     * This fixes a hang on GC600. */
 452    if (buffers & PIPE_CLEAR_DEPTHSTENCIL && buffers & PIPE_CLEAR_COLOR)
 453       etna_set_state(ctx->stream, VIVS_GL_FLUSH_CACHE,
 454                      VIVS_GL_FLUSH_CACHE_COLOR | VIVS_GL_FLUSH_CACHE_DEPTH);
 455
 456    if ((buffers & PIPE_CLEAR_DEPTHSTENCIL) && ctx->framebuffer_s.zsbuf != NULL)
 457       etna_blit_clear_zs_rs(pctx, ctx->framebuffer_s.zsbuf, buffers, depth, stencil);
 458
 459    etna_stall(ctx->stream, SYNC_RECIPIENT_RA, SYNC_RECIPIENT_PE);
 460    mtx_unlock(&ctx->lock);
 461 }
 462
 463 static bool
 464 etna_manual_blit(struct etna_resource *dst, struct etna_resource_level *dst_lev,
 465                  unsigned int dst_offset, struct etna_resource *src,
 466                  struct etna_resource_level *src_lev, unsigned int src_offset,
 467                  const struct pipe_blit_info *blit_info)
 468 {
 469    void *smap, *srow, *dmap, *drow;
 470    size_t tile_size;
 471
 472    assert(src->layout == ETNA_LAYOUT_TILED);
 473    assert(dst->layout == ETNA_LAYOUT_TILED);
 474    assert(src->base.nr_samples == 0);
 475    assert(dst->base.nr_samples == 0);
 476
 477    tile_size = util_format_get_blocksize(blit_info->src.format) * 4 * 4;
 478
 479    smap = etna_bo_map(src->bo);
 480    if (!smap)
 481       return false;
 482
 483    dmap = etna_bo_map(dst->bo);
 484    if (!dmap)
 485       return false;
 486
 487    srow = smap + src_offset;
 488    drow = dmap + dst_offset;
 489
 490    etna_bo_cpu_prep(src->bo, DRM_ETNA_PREP_READ);
 491    etna_bo_cpu_prep(dst->bo, DRM_ETNA_PREP_WRITE);
 492
 493    for (int y = 0; y < blit_info->src.box.height; y += 4) {
 494       memcpy(drow, srow, tile_size * blit_info->src.box.width);
 495       srow += src_lev->stride * 4;
 496       drow += dst_lev->stride * 4;
 497    }
 498
 499    etna_bo_cpu_fini(dst->bo);
 500    etna_bo_cpu_fini(src->bo);
 501
 502    return true;
 503 }
 504
 505 static inline size_t
 506 etna_compute_tileoffset(const struct pipe_box *box, enum pipe_format format,
 507                         size_t stride, enum etna_surface_layout layout)
 508 {
 509    size_t offset;
 510    unsigned int x = box->x, y = box->y;
 511    unsigned int blocksize = util_format_get_blocksize(format);
 512
 513    switch (layout) {
 514    case ETNA_LAYOUT_LINEAR:
 515       offset = y * stride + x * blocksize;
 516       break;
 517    case ETNA_LAYOUT_MULTI_TILED:
 518       y >>= 1;
 519       /* fall-through */
 520    case ETNA_LAYOUT_TILED:
 521       assert(!(x & 0x03) && !(y & 0x03));
 522       offset = (y & ~0x03) * stride + blocksize * ((x & ~0x03) << 2);
 523       break;
 524    case ETNA_LAYOUT_MULTI_SUPERTILED:
 525       y >>= 1;
 526       /* fall-through */
 527    case ETNA_LAYOUT_SUPER_TILED:
 528       assert(!(x & 0x3f) && !(y & 0x3f));
 529       offset = (y & ~0x3f) * stride + blocksize * ((x & ~0x3f) << 6);
 530       break;
 531    default:
 532       unreachable("invalid resource layout");
 533    }
 534
 535    return offset;
 536 }
 537
 538 static inline void
 539 etna_get_rs_alignment_mask(const struct etna_context *ctx,
 540                            const enum etna_surface_layout layout,
 541                            unsigned int *width_mask, unsigned int *height_mask)
 542 {
 543    struct etna_screen *screen = ctx->screen;
 544    unsigned int h_align, w_align;
 545
 546    if (layout & ETNA_LAYOUT_BIT_SUPER) {
 547       w_align = 64;
 548       h_align = 64 * screen->specs.pixel_pipes;
 549    } else {
 550       w_align = ETNA_RS_WIDTH_MASK + 1;
 551       h_align = ETNA_RS_HEIGHT_MASK + 1;
 552    }
 553
 554    *width_mask = w_align - 1;
 555    *height_mask = h_align -1;
 556 }
 557
 558 static bool msaa_config(const struct pipe_resource *src,
 559                         const struct pipe_resource *dst,
 560                         int *msaa_xscale,
 561                         int *msaa_yscale)
 562 {
 563    int src_xscale = 1, src_yscale = 1;
 564    int dst_xscale = 1, dst_yscale = 1;
 565
 566    assert(src->nr_samples <= 4);
 567    assert(dst->nr_samples <= 4);
 568
 569    translate_samples_to_xyscale(src->nr_samples, &src_xscale, &src_yscale);
 570    translate_samples_to_xyscale(dst->nr_samples, &dst_xscale, &dst_yscale);
 571
 572    /* RS does not support upscaling */
 573    if ((src_xscale < dst_xscale) || (src_yscale < dst_yscale))
 574       return false;
 575
 576    *msaa_xscale = src_xscale - dst_xscale + 1;
 577    *msaa_yscale = src_yscale - dst_yscale + 1;
 578
 579    return true;
 580 }
 581
 582 static bool
 583 etna_try_rs_blit(struct pipe_context *pctx,
 584                  const struct pipe_blit_info *blit_info)
 585 {
 586    struct etna_context *ctx = etna_context(pctx);
 587    struct etna_resource *src = etna_resource(blit_info->src.resource);
 588    struct etna_resource *dst = etna_resource(blit_info->dst.resource);
 589    struct compiled_rs_state copy_to_screen;
 590    int msaa_xscale = 1, msaa_yscale = 1;
 591
 592    /* Ensure that the level is valid */
 593    assert(blit_info->src.level <= src->base.last_level);
 594    assert(blit_info->dst.level <= dst->base.last_level);
 595
 596    if (!msaa_config(&src->base, &dst->base, &msaa_xscale, &msaa_yscale)) {
 597       DBG("upsampling not supported");
 598       return false;
 599    }
 600
 601    /* The width/height are in pixels; they do not change as a result of
 602     * multi-sampling. So, when blitting from a 4x multisampled surface
 603     * to a non-multisampled surface, the width and height will be
 604     * identical. As we do not support scaling, reject different sizes. */
 605    if (blit_info->dst.box.width != blit_info->src.box.width ||
 606        blit_info->dst.box.height != blit_info->src.box.height) {
 607       DBG("scaling requested: source %dx%d destination %dx%d",
 608           blit_info->src.box.width, blit_info->src.box.height,
 609           blit_info->dst.box.width, blit_info->dst.box.height);
 610       return false;
 611    }
 612
 613    /* No masks - RS can't copy specific channels */
 614    unsigned mask = util_format_get_mask(blit_info->dst.format);
 615    if ((blit_info->mask & mask) != mask) {
 616       DBG("sub-mask requested: 0x%02x vs format mask 0x%02x", blit_info->mask, mask);
 617       return false;
 618    }
 619
 620    /* Only support same format (used tiling/detiling) blits for now.
 621     * TODO: figure out which different-format blits are possible and test them
 622     *  - fail if swizzle needed
 623     *  - avoid trying to convert between float/int formats?
 624     */
 625    if (blit_info->src.format != blit_info->dst.format)
 626       return false;
 627
 628    uint32_t format = etna_compatible_rs_format(blit_info->dst.format);
 629    if (format == ETNA_NO_MATCH)
 630       return false;
 631
 632    if (blit_info->scissor_enable ||
 633        blit_info->dst.box.depth != blit_info->src.box.depth ||
 634        blit_info->dst.box.depth != 1) {
 635       return false;
 636    }
 637
 638    unsigned w_mask, h_mask;
 639
 640    etna_get_rs_alignment_mask(ctx, src->layout, &w_mask, &h_mask);
 641    if ((blit_info->src.box.x & w_mask) || (blit_info->src.box.y & h_mask))
 642       return false;
 643
 644    etna_get_rs_alignment_mask(ctx, dst->layout, &w_mask, &h_mask);
 645    if ((blit_info->dst.box.x & w_mask) || (blit_info->dst.box.y & h_mask))
 646       return false;
 647
 648    struct etna_resource_level *src_lev = &src->levels[blit_info->src.level];
 649    struct etna_resource_level *dst_lev = &dst->levels[blit_info->dst.level];
 650
 651    /* we may be given coordinates up to the padded width to avoid
 652     * any alignment issues with different tiling formats */
 653    assert((blit_info->src.box.x + blit_info->src.box.width) * msaa_xscale <= src_lev->padded_width);
 654    assert((blit_info->src.box.y + blit_info->src.box.height) * msaa_yscale <= src_lev->padded_height);
 655    assert(blit_info->dst.box.x + blit_info->dst.box.width <= dst_lev->padded_width);
 656    assert(blit_info->dst.box.y + blit_info->dst.box.height <= dst_lev->padded_height);
 657
 658    unsigned src_offset = src_lev->offset +
 659                          blit_info->src.box.z * src_lev->layer_stride +
 660                          etna_compute_tileoffset(&blit_info->src.box,
 661                                                  blit_info->src.format,
 662                                                  src_lev->stride,
 663                                                  src->layout);
 664    unsigned dst_offset = dst_lev->offset +
 665                          blit_info->dst.box.z * dst_lev->layer_stride +
 666                          etna_compute_tileoffset(&blit_info->dst.box,
 667                                                  blit_info->dst.format,
 668                                                  dst_lev->stride,
 669                                                  dst->layout);
 670
 671    if (src_lev->padded_width <= ETNA_RS_WIDTH_MASK ||
 672        dst_lev->padded_width <= ETNA_RS_WIDTH_MASK ||
 673        src_lev->padded_height <= ETNA_RS_HEIGHT_MASK ||
 674        dst_lev->padded_height <= ETNA_RS_HEIGHT_MASK)
 675       goto manual;
 676
 677    /* If the width is not aligned to the RS width, but is within our
 678     * padding, adjust the width to suite the RS width restriction.
 679     * Note: the RS width/height are converted to source samples here. */
 680    unsigned int width = blit_info->src.box.width * msaa_xscale;
 681    unsigned int height = blit_info->src.box.height * msaa_yscale;
 682    unsigned int w_align = ETNA_RS_WIDTH_MASK + 1;
 683    unsigned int h_align = ETNA_RS_HEIGHT_MASK + 1;
 684
 685    if (width & (w_align - 1) && width >= src_lev->width * msaa_xscale && width >= dst_lev->width)
 686       width = align(width, w_align);
 687
 688    if (height & (h_align - 1) && height >= src_lev->height * msaa_yscale && height >= dst_lev->height)
 689       height = align(height, h_align);
 690
 691    /* The padded dimensions are in samples */
 692    if (width > src_lev->padded_width ||
 693        width > dst_lev->padded_width * msaa_xscale ||
 694        height > src_lev->padded_height ||
 695        height > dst_lev->padded_height * msaa_yscale ||
 696        width & (w_align - 1) || height & (h_align - 1))
 697       goto manual;
 698
 699    mtx_lock(&ctx->lock);
 700
 701    /* Always flush color and depth cache together before resolving. This works
 702     * around artifacts that appear in some cases when scanning out a texture
 703     * directly after it has been rendered to, such as rendering an animated web
 704     * page in a QtWebEngine based WebView on GC2000. The artifacts look like
 705     * the texture sampler samples zeroes instead of texture data in a small,
 706     * irregular triangle in the lower right of each browser tile quad. Other
 707     * attempts to avoid these artifacts, including a pipeline stall before the
 708     * color flush or a TS cache flush afterwards, or flushing multiple times,
 709     * with stalls before and after each flush, have shown no effect. */
 710    if (src->base.bind & PIPE_BIND_RENDER_TARGET ||
 711        src->base.bind & PIPE_BIND_DEPTH_STENCIL) {
 712       etna_set_state(ctx->stream, VIVS_GL_FLUSH_CACHE,
 713                      VIVS_GL_FLUSH_CACHE_COLOR | VIVS_GL_FLUSH_CACHE_DEPTH);
 714       etna_stall(ctx->stream, SYNC_RECIPIENT_RA, SYNC_RECIPIENT_PE);
 715
 716       if (src_lev->ts_size && src_lev->ts_valid)
 717          etna_set_state(ctx->stream, VIVS_TS_FLUSH_CACHE, VIVS_TS_FLUSH_CACHE_FLUSH);
 718    }
 719
 720    /* Set up color TS to source surface before blit, if needed */
 721    bool source_ts_valid = false;
 722    if (src_lev->ts_size && src_lev->ts_valid) {
 723       struct etna_reloc reloc;
 724       unsigned ts_offset =
 725          src_lev->ts_offset + blit_info->src.box.z * src_lev->ts_layer_stride;
 726       uint32_t ts_mem_config = 0;
 727
 728       if (src_lev->ts_compress_fmt >= 0) {
 729          ts_mem_config |= VIVS_TS_MEM_CONFIG_COLOR_COMPRESSION |
 730                           VIVS_TS_MEM_CONFIG_COLOR_COMPRESSION_FORMAT(src_lev->ts_compress_fmt);
 731       }
 732
 733       etna_set_state(ctx->stream, VIVS_TS_MEM_CONFIG,
 734                      VIVS_TS_MEM_CONFIG_COLOR_FAST_CLEAR | ts_mem_config);
 735
 736       memset(&reloc, 0, sizeof(struct etna_reloc));
 737       reloc.bo = src->ts_bo;
 738       reloc.offset = ts_offset;
 739       reloc.flags = ETNA_RELOC_READ;
 740       etna_set_state_reloc(ctx->stream, VIVS_TS_COLOR_STATUS_BASE, &reloc);
 741
 742       memset(&reloc, 0, sizeof(struct etna_reloc));
 743       reloc.bo = src->bo;
 744       reloc.offset = src_lev->offset +
 745                      blit_info->src.box.z * src_lev->layer_stride;
 746       reloc.flags = ETNA_RELOC_READ;
 747       etna_set_state_reloc(ctx->stream, VIVS_TS_COLOR_SURFACE_BASE, &reloc);
 748
 749       etna_set_state(ctx->stream, VIVS_TS_COLOR_CLEAR_VALUE, src_lev->clear_value);
 750       etna_set_state(ctx->stream, VIVS_TS_COLOR_CLEAR_VALUE_EXT, src_lev->clear_value >> 32);
 751
 752       source_ts_valid = true;
 753    } else {
 754       etna_set_state(ctx->stream, VIVS_TS_MEM_CONFIG, 0);
 755    }
 756    ctx->dirty |= ETNA_DIRTY_TS;
 757
 758    /* Kick off RS here */
 759    etna_compile_rs_state(ctx, &copy_to_screen, &(struct rs_state) {
 760       .source_format = format,
 761       .source_tiling = src->layout,
 762       .source = src->bo,
 763       .source_offset = src_offset,
 764       .source_stride = src_lev->stride,
 765       .source_padded_width = src_lev->padded_width,
 766       .source_padded_height = src_lev->padded_height,
 767       .source_ts_valid = source_ts_valid,
 768       .source_ts_compressed = src_lev->ts_compress_fmt >= 0,
 769       .dest_format = format,
 770       .dest_tiling = dst->layout,
 771       .dest = dst->bo,
 772       .dest_offset = dst_offset,
 773       .dest_stride = dst_lev->stride,
 774       .dest_padded_height = dst_lev->padded_height,
 775       .downsample_x = msaa_xscale > 1,
 776       .downsample_y = msaa_yscale > 1,
 777       .swap_rb = translate_rb_src_dst_swap(src->base.format, dst->base.format),
 778       .dither = {0xffffffff, 0xffffffff}, // XXX dither when going from 24 to 16 bit?
 779       .clear_mode = VIVS_RS_CLEAR_CONTROL_MODE_DISABLED,
 780       .width = width,
 781       .height = height,
 782       .tile_count = src_lev->layer_stride / 64
 783    });
 784
 785    etna_submit_rs_state(ctx, &copy_to_screen);
 786    resource_read(ctx, &src->base);
 787    resource_written(ctx, &dst->base);
 788    dst->seqno++;
 789    dst_lev->ts_valid = false;
 790    ctx->dirty |= ETNA_DIRTY_DERIVE_TS;
 791    mtx_unlock(&ctx->lock);
 792
 793    return true;
 794
 795 manual:
 796    if (src->layout == ETNA_LAYOUT_TILED && dst->layout == ETNA_LAYOUT_TILED) {
 797       if ((src->status & ETNA_PENDING_WRITE) ||
 798           (dst->status & ETNA_PENDING_WRITE))
 799          pctx->flush(pctx, NULL, 0);
 800       return etna_manual_blit(dst, dst_lev, dst_offset, src, src_lev, src_offset, blit_info);
 801    }
 802
 803    return false;
 804 }
 805
 806 static bool
 807 etna_blit_rs(struct pipe_context *pctx, const struct pipe_blit_info *blit_info)
 808 {
 809    /* This is a more extended version of resource_copy_region */
 810    /* TODO Some cases can be handled by RS; if not, fall back to rendering or
 811     * even CPU copy block of pixels from info->src to info->dst
 812     * (resource, level, box, format);
 813     * function is used for scaling, flipping in x and y direction (negative
 814     * width/height), format conversion, mask and filter and even a scissor rectangle
 815     *
 816     * What can the RS do for us:
 817     *   convert between tiling formats (layouts)
 818     *   downsample 2x in x and y
 819     *   convert between a limited number of pixel formats
 820     *
 821     * For the rest, fall back to util_blitter
 822     * XXX this goes wrong when source surface is supertiled. */
 823
 824    if (blit_info->src.resource->nr_samples > 1 &&
 825        blit_info->dst.resource->nr_samples <= 1 &&
 826        !util_format_is_depth_or_stencil(blit_info->src.resource->format) &&
 827        !util_format_is_pure_integer(blit_info->src.resource->format)) {
 828       DBG("color resolve unimplemented");
 829       return false;
 830    }
 831
 832    return etna_try_rs_blit(pctx, blit_info);
 833 }
 834
 835 void
 836 etna_clear_blit_rs_init(struct pipe_context *pctx)
 837 {
 838    struct etna_context *ctx = etna_context(pctx);
 839
 840    DBG("etnaviv: Using RS blit engine");
 841    pctx->clear = etna_clear_rs;
 842    ctx->blit = etna_blit_rs;
 843 }