src/gallium/drivers/etnaviv/etnaviv_rs.c

   1 /*
   2  * Copyright (c) 2012-2017 Etnaviv Project
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sub license,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the
  12  * next paragraph) shall be included in all copies or substantial portions
  13  * of the Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Wladimir J. van der Laan <laanwj@gmail.com>
  25  */
  26
  27 #include "etnaviv_rs.h"
  28
  29 #include "etnaviv_clear_blit.h"
  30 #include "etnaviv_context.h"
  31 #include "etnaviv_emit.h"
  32 #include "etnaviv_format.h"
  33 #include "etnaviv_resource.h"
  34 #include "etnaviv_screen.h"
  35 #include "etnaviv_surface.h"
  36 #include "etnaviv_tiling.h"
  37 #include "etnaviv_translate.h"
  38 #include "etnaviv_util.h"
  39
  40 #include "pipe/p_defines.h"
  41 #include "pipe/p_state.h"
  42 #include "util/u_blitter.h"
  43 #include "util/u_inlines.h"
  44 #include "util/u_memory.h"
  45 #include "util/u_surface.h"
  46
  47 #include "hw/common.xml.h"
  48 #include "hw/state.xml.h"
  49 #include "hw/state_3d.xml.h"
  50
  51 #include <assert.h>
  52
  53 void
  54 etna_compile_rs_state(struct etna_context *ctx, struct compiled_rs_state *cs,
  55                       const struct rs_state *rs)
  56 {
  57    memset(cs, 0, sizeof(*cs));
  58
  59    /* TILED and SUPERTILED layout have their strides multiplied with 4 in RS */
  60    unsigned source_stride_shift = COND(rs->source_tiling != ETNA_LAYOUT_LINEAR, 2);
  61    unsigned dest_stride_shift = COND(rs->dest_tiling != ETNA_LAYOUT_LINEAR, 2);
  62
  63    /* tiling == ETNA_LAYOUT_MULTI_TILED or ETNA_LAYOUT_MULTI_SUPERTILED? */
  64    int source_multi = COND(rs->source_tiling & ETNA_LAYOUT_BIT_MULTI, 1);
  65    int dest_multi = COND(rs->dest_tiling & ETNA_LAYOUT_BIT_MULTI, 1);
  66
  67    /* Vivante RS needs widths to be a multiple of 16 or bad things
  68     * happen, such as scribbing over memory, or the GPU hanging,
  69     * even for non-tiled formats.  As this is serious, use abort().
  70     */
  71    if (rs->width & ETNA_RS_WIDTH_MASK)
  72       abort();
  73
  74    /* TODO could just pre-generate command buffer, would simply submit to one memcpy */
  75    cs->RS_CONFIG = VIVS_RS_CONFIG_SOURCE_FORMAT(rs->source_format) |
  76                    COND(rs->downsample_x, VIVS_RS_CONFIG_DOWNSAMPLE_X) |
  77                    COND(rs->downsample_y, VIVS_RS_CONFIG_DOWNSAMPLE_Y) |
  78                    COND(rs->source_tiling & 1, VIVS_RS_CONFIG_SOURCE_TILED) |
  79                    VIVS_RS_CONFIG_DEST_FORMAT(rs->dest_format) |
  80                    COND(rs->dest_tiling & 1, VIVS_RS_CONFIG_DEST_TILED) |
  81                    COND(rs->swap_rb, VIVS_RS_CONFIG_SWAP_RB) |
  82                    COND(rs->flip, VIVS_RS_CONFIG_FLIP);
  83
  84    cs->RS_SOURCE_STRIDE = (rs->source_stride << source_stride_shift) |
  85                           COND(rs->source_tiling & 2, VIVS_RS_SOURCE_STRIDE_TILING) |
  86                           COND(source_multi, VIVS_RS_SOURCE_STRIDE_MULTI);
  87
  88    /* Initially all pipes are set to the base address of the source and
  89     * destination buffer respectively. This will be overridden below as
  90     * necessary for the multi-pipe, multi-tiled case.
  91     */
  92    for (unsigned pipe = 0; pipe < ctx->specs.pixel_pipes; ++pipe) {
  93       cs->source[pipe].bo = rs->source;
  94       cs->source[pipe].offset = rs->source_offset;
  95       cs->source[pipe].flags = ETNA_RELOC_READ;
  96
  97       cs->dest[pipe].bo = rs->dest;
  98       cs->dest[pipe].offset = rs->dest_offset;
  99       cs->dest[pipe].flags = ETNA_RELOC_WRITE;
 100
 101       cs->RS_PIPE_OFFSET[pipe] = VIVS_RS_PIPE_OFFSET_X(0) | VIVS_RS_PIPE_OFFSET_Y(0);
 102    }
 103
 104    cs->RS_DEST_STRIDE = (rs->dest_stride << dest_stride_shift) |
 105                         COND(rs->dest_tiling & 2, VIVS_RS_DEST_STRIDE_TILING) |
 106                         COND(dest_multi, VIVS_RS_DEST_STRIDE_MULTI);
 107
 108    if (ctx->specs.pixel_pipes == 1 || ctx->specs.single_buffer) {
 109       cs->RS_WINDOW_SIZE = VIVS_RS_WINDOW_SIZE_WIDTH(rs->width) |
 110                            VIVS_RS_WINDOW_SIZE_HEIGHT(rs->height);
 111    } else if (ctx->specs.pixel_pipes == 2) {
 112       assert((rs->height & 7) == 0); /* GPU hangs happen if height not 8-aligned */
 113
 114       if (source_multi)
 115          cs->source[1].offset = rs->source_offset + rs->source_stride * rs->source_padded_height / 2;
 116
 117       if (dest_multi)
 118          cs->dest[1].offset = rs->dest_offset + rs->dest_stride * rs->dest_padded_height / 2;
 119
 120       cs->RS_WINDOW_SIZE = VIVS_RS_WINDOW_SIZE_WIDTH(rs->width) |
 121                            VIVS_RS_WINDOW_SIZE_HEIGHT(rs->height / 2);
 122       cs->RS_PIPE_OFFSET[1] = VIVS_RS_PIPE_OFFSET_X(0) | VIVS_RS_PIPE_OFFSET_Y(rs->height / 2);
 123    } else {
 124       abort();
 125    }
 126
 127    cs->RS_DITHER[0] = rs->dither[0];
 128    cs->RS_DITHER[1] = rs->dither[1];
 129    cs->RS_CLEAR_CONTROL = VIVS_RS_CLEAR_CONTROL_BITS(rs->clear_bits) | rs->clear_mode;
 130    cs->RS_FILL_VALUE[0] = rs->clear_value[0];
 131    cs->RS_FILL_VALUE[1] = rs->clear_value[1];
 132    cs->RS_FILL_VALUE[2] = rs->clear_value[2];
 133    cs->RS_FILL_VALUE[3] = rs->clear_value[3];
 134    cs->RS_EXTRA_CONFIG = VIVS_RS_EXTRA_CONFIG_AA(rs->aa) |
 135                          VIVS_RS_EXTRA_CONFIG_ENDIAN(rs->endian_mode);
 136
 137    /* If source the same as destination, and the hardware supports this,
 138     * do an in-place resolve to fill in unrendered tiles.
 139     */
 140    if (ctx->specs.single_buffer && rs->source == rs->dest &&
 141          rs->source_offset == rs->dest_offset &&
 142          rs->source_format == rs->dest_format &&
 143          rs->source_tiling == rs->dest_tiling &&
 144          (rs->source_tiling & ETNA_LAYOUT_BIT_SUPER) &&
 145          rs->source_stride == rs->dest_stride &&
 146          !rs->downsample_x && !rs->downsample_y &&
 147          !rs->swap_rb && !rs->flip &&
 148          !rs->clear_mode && rs->source_padded_width) {
 149       /* Total number of tiles (same as for autodisable) */
 150       cs->RS_KICKER_INPLACE = rs->tile_count;
 151    }
 152    cs->source_ts_valid = rs->source_ts_valid;
 153 }
 154
 155 /* modify the clear bits value in the compiled RS state */
 156 static void
 157 etna_modify_rs_clearbits(struct compiled_rs_state *cs, uint32_t clear_bits)
 158 {
 159    cs->RS_CLEAR_CONTROL &= ~VIVS_RS_CLEAR_CONTROL_BITS__MASK;
 160    cs->RS_CLEAR_CONTROL |= VIVS_RS_CLEAR_CONTROL_BITS(clear_bits);
 161 }
 162
 163 #define EMIT_STATE(state_name, src_value) \
 164    etna_coalsence_emit(stream, &coalesce, VIVS_##state_name, src_value)
 165
 166 #define EMIT_STATE_FIXP(state_name, src_value) \
 167    etna_coalsence_emit_fixp(stream, &coalesce, VIVS_##state_name, src_value)
 168
 169 #define EMIT_STATE_RELOC(state_name, src_value) \
 170    etna_coalsence_emit_reloc(stream, &coalesce, VIVS_##state_name, src_value)
 171
 172 /* submit RS state, without any processing and no dependence on context
 173  * except TS if this is a source-to-destination blit. */
 174 static void
 175 etna_submit_rs_state(struct etna_context *ctx,
 176                      const struct compiled_rs_state *cs)
 177 {
 178    struct etna_screen *screen = etna_screen(ctx->base.screen);
 179    struct etna_cmd_stream *stream = ctx->stream;
 180    struct etna_coalesce coalesce;
 181
 182    if (cs->RS_KICKER_INPLACE && !cs->source_ts_valid)
 183       /* Inplace resolve is no-op if TS is not configured */
 184       return;
 185
 186    ctx->stats.rs_operations++;
 187
 188    if (cs->RS_KICKER_INPLACE) {
 189       etna_cmd_stream_reserve(stream, 6);
 190       etna_coalesce_start(stream, &coalesce);
 191       /* 0/1 */ EMIT_STATE(RS_EXTRA_CONFIG, cs->RS_EXTRA_CONFIG);
 192       /* 2/3 */ EMIT_STATE(RS_SOURCE_STRIDE, cs->RS_SOURCE_STRIDE);
 193       /* 4/5 */ EMIT_STATE(RS_KICKER_INPLACE, cs->RS_KICKER_INPLACE);
 194       etna_coalesce_end(stream, &coalesce);
 195    } else if (screen->specs.pixel_pipes == 1) {
 196       etna_cmd_stream_reserve(stream, 22);
 197       etna_coalesce_start(stream, &coalesce);
 198       /* 0/1 */ EMIT_STATE(RS_CONFIG, cs->RS_CONFIG);
 199       /* 2   */ EMIT_STATE_RELOC(RS_SOURCE_ADDR, &cs->source[0]);
 200       /* 3   */ EMIT_STATE(RS_SOURCE_STRIDE, cs->RS_SOURCE_STRIDE);
 201       /* 4   */ EMIT_STATE_RELOC(RS_DEST_ADDR, &cs->dest[0]);
 202       /* 5   */ EMIT_STATE(RS_DEST_STRIDE, cs->RS_DEST_STRIDE);
 203       /* 6/7 */ EMIT_STATE(RS_WINDOW_SIZE, cs->RS_WINDOW_SIZE);
 204       /* 8/9 */ EMIT_STATE(RS_DITHER(0), cs->RS_DITHER[0]);
 205       /*10   */ EMIT_STATE(RS_DITHER(1), cs->RS_DITHER[1]);
 206       /*11 - pad */
 207       /*12/13*/ EMIT_STATE(RS_CLEAR_CONTROL, cs->RS_CLEAR_CONTROL);
 208       /*14   */ EMIT_STATE(RS_FILL_VALUE(0), cs->RS_FILL_VALUE[0]);
 209       /*15   */ EMIT_STATE(RS_FILL_VALUE(1), cs->RS_FILL_VALUE[1]);
 210       /*16   */ EMIT_STATE(RS_FILL_VALUE(2), cs->RS_FILL_VALUE[2]);
 211       /*17   */ EMIT_STATE(RS_FILL_VALUE(3), cs->RS_FILL_VALUE[3]);
 212       /*18/19*/ EMIT_STATE(RS_EXTRA_CONFIG, cs->RS_EXTRA_CONFIG);
 213       /*20/21*/ EMIT_STATE(RS_KICKER, 0xbeebbeeb);
 214       etna_coalesce_end(stream, &coalesce);
 215    } else if (screen->specs.pixel_pipes == 2) {
 216       etna_cmd_stream_reserve(stream, 34); /* worst case - both pipes multi=1 */
 217       etna_coalesce_start(stream, &coalesce);
 218       /* 0/1 */ EMIT_STATE(RS_CONFIG, cs->RS_CONFIG);
 219       /* 2/3 */ EMIT_STATE(RS_SOURCE_STRIDE, cs->RS_SOURCE_STRIDE);
 220       /* 4/5 */ EMIT_STATE(RS_DEST_STRIDE, cs->RS_DEST_STRIDE);
 221       /* 6/7 */ EMIT_STATE_RELOC(RS_PIPE_SOURCE_ADDR(0), &cs->source[0]);
 222       if (cs->RS_SOURCE_STRIDE & VIVS_RS_SOURCE_STRIDE_MULTI) {
 223          /*8 */ EMIT_STATE_RELOC(RS_PIPE_SOURCE_ADDR(1), &cs->source[1]);
 224          /*9 - pad */
 225       }
 226       /*10/11*/ EMIT_STATE_RELOC(RS_PIPE_DEST_ADDR(0), &cs->dest[0]);
 227       if (cs->RS_DEST_STRIDE & VIVS_RS_DEST_STRIDE_MULTI) {
 228          /*12*/ EMIT_STATE_RELOC(RS_PIPE_DEST_ADDR(1), &cs->dest[1]);
 229          /*13 - pad */
 230       }
 231       /*14/15*/ EMIT_STATE(RS_PIPE_OFFSET(0), cs->RS_PIPE_OFFSET[0]);
 232       /*16   */ EMIT_STATE(RS_PIPE_OFFSET(1), cs->RS_PIPE_OFFSET[1]);
 233       /*17 - pad */
 234       /*18/19*/ EMIT_STATE(RS_WINDOW_SIZE, cs->RS_WINDOW_SIZE);
 235       /*20/21*/ EMIT_STATE(RS_DITHER(0), cs->RS_DITHER[0]);
 236       /*22   */ EMIT_STATE(RS_DITHER(1), cs->RS_DITHER[1]);
 237       /*23 - pad */
 238       /*24/25*/ EMIT_STATE(RS_CLEAR_CONTROL, cs->RS_CLEAR_CONTROL);
 239       /*26   */ EMIT_STATE(RS_FILL_VALUE(0), cs->RS_FILL_VALUE[0]);
 240       /*27   */ EMIT_STATE(RS_FILL_VALUE(1), cs->RS_FILL_VALUE[1]);
 241       /*28   */ EMIT_STATE(RS_FILL_VALUE(2), cs->RS_FILL_VALUE[2]);
 242       /*29   */ EMIT_STATE(RS_FILL_VALUE(3), cs->RS_FILL_VALUE[3]);
 243       /*30/31*/ EMIT_STATE(RS_EXTRA_CONFIG, cs->RS_EXTRA_CONFIG);
 244       /*32/33*/ EMIT_STATE(RS_KICKER, 0xbeebbeeb);
 245       etna_coalesce_end(stream, &coalesce);
 246    } else {
 247       abort();
 248    }
 249 }
 250
 251 /* Generate clear command for a surface (non-fast clear case) */
 252 void
 253 etna_rs_gen_clear_surface(struct etna_context *ctx, struct etna_surface *surf,
 254                           uint32_t clear_value)
 255 {
 256    struct etna_resource *dst = etna_resource(surf->base.texture);
 257    uint32_t format;
 258
 259    switch (util_format_get_blocksizebits(surf->base.format)) {
 260    case 16:
 261       format = RS_FORMAT_A4R4G4B4;
 262       break;
 263    case 32:
 264       format = RS_FORMAT_A8R8G8B8;
 265       break;
 266    default:
 267       format = ETNA_NO_MATCH;
 268       break;
 269    }
 270
 271    if (format == ETNA_NO_MATCH) {
 272       BUG("etna_rs_gen_clear_surface: Unhandled clear fmt %s", util_format_name(surf->base.format));
 273       format = RS_FORMAT_A8R8G8B8;
 274       assert(0);
 275    }
 276
 277    /* use tiled clear if width is multiple of 16 */
 278    bool tiled_clear = (surf->surf.padded_width & ETNA_RS_WIDTH_MASK) == 0 &&
 279                       (surf->surf.padded_height & ETNA_RS_HEIGHT_MASK) == 0;
 280
 281    etna_compile_rs_state( ctx, &surf->clear_command, &(struct rs_state) {
 282       .source_format = format,
 283       .dest_format = format,
 284       .dest = dst->bo,
 285       .dest_offset = surf->surf.offset,
 286       .dest_stride = surf->surf.stride,
 287       .dest_padded_height = surf->surf.padded_height,
 288       .dest_tiling = tiled_clear ? dst->layout : ETNA_LAYOUT_LINEAR,
 289       .dither = {0xffffffff, 0xffffffff},
 290       .width = surf->surf.padded_width, /* These must be padded to 16x4 if !LINEAR, otherwise RS will hang */
 291       .height = surf->surf.padded_height,
 292       .clear_value = {clear_value},
 293       .clear_mode = VIVS_RS_CLEAR_CONTROL_MODE_ENABLED1,
 294       .clear_bits = 0xffff
 295    });
 296 }
 297
 298 static void
 299 etna_blit_clear_color_rs(struct pipe_context *pctx, struct pipe_surface *dst,
 300                       const union pipe_color_union *color)
 301 {
 302    struct etna_context *ctx = etna_context(pctx);
 303    struct etna_surface *surf = etna_surface(dst);
 304    uint32_t new_clear_value = etna_clear_blit_pack_rgba(surf->base.format, color->f);
 305
 306    if (surf->surf.ts_size) { /* TS: use precompiled clear command */
 307       ctx->framebuffer.TS_COLOR_CLEAR_VALUE = new_clear_value;
 308
 309       if (VIV_FEATURE(ctx->screen, chipMinorFeatures1, AUTO_DISABLE)) {
 310          /* Set number of color tiles to be filled */
 311          etna_set_state(ctx->stream, VIVS_TS_COLOR_AUTO_DISABLE_COUNT,
 312                         surf->surf.padded_width * surf->surf.padded_height / 16);
 313          ctx->framebuffer.TS_MEM_CONFIG |= VIVS_TS_MEM_CONFIG_COLOR_AUTO_DISABLE;
 314       }
 315
 316       surf->level->ts_valid = true;
 317       ctx->dirty |= ETNA_DIRTY_TS | ETNA_DIRTY_DERIVE_TS;
 318    } else if (unlikely(new_clear_value != surf->level->clear_value)) { /* Queue normal RS clear for non-TS surfaces */
 319       /* If clear color changed, re-generate stored command */
 320       etna_rs_gen_clear_surface(ctx, surf, new_clear_value);
 321    }
 322
 323    etna_submit_rs_state(ctx, &surf->clear_command);
 324
 325    surf->level->clear_value = new_clear_value;
 326    resource_written(ctx, surf->base.texture);
 327    etna_resource(surf->base.texture)->seqno++;
 328 }
 329
 330 static void
 331 etna_blit_clear_zs_rs(struct pipe_context *pctx, struct pipe_surface *dst,
 332                    unsigned buffers, double depth, unsigned stencil)
 333 {
 334    struct etna_context *ctx = etna_context(pctx);
 335    struct etna_surface *surf = etna_surface(dst);
 336    uint32_t new_clear_value = translate_clear_depth_stencil(surf->base.format, depth, stencil);
 337    uint32_t new_clear_bits = 0, clear_bits_depth, clear_bits_stencil;
 338
 339    /* Get the channels to clear */
 340    switch (surf->base.format) {
 341    case PIPE_FORMAT_Z16_UNORM:
 342       clear_bits_depth = 0xffff;
 343       clear_bits_stencil = 0;
 344       break;
 345    case PIPE_FORMAT_X8Z24_UNORM:
 346    case PIPE_FORMAT_S8_UINT_Z24_UNORM:
 347       clear_bits_depth = 0xeeee;
 348       clear_bits_stencil = 0x1111;
 349       break;
 350    default:
 351       clear_bits_depth = clear_bits_stencil = 0xffff;
 352       break;
 353    }
 354
 355    if (buffers & PIPE_CLEAR_DEPTH)
 356       new_clear_bits |= clear_bits_depth;
 357    if (buffers & PIPE_CLEAR_STENCIL)
 358       new_clear_bits |= clear_bits_stencil;
 359    /* FIXME: when tile status is enabled, this becomes more complex as
 360     * we may separately clear the depth from the stencil.  In this case,
 361     * we want to resolve the surface, and avoid using the tile status.
 362     * We may be better off recording the pending clear operation,
 363     * delaying the actual clear to the first use.  This way, we can merge
 364     * consecutive clears together. */
 365    if (surf->surf.ts_size) { /* TS: use precompiled clear command */
 366       /* Set new clear depth value */
 367       ctx->framebuffer.TS_DEPTH_CLEAR_VALUE = new_clear_value;
 368       if (VIV_FEATURE(ctx->screen, chipMinorFeatures1, AUTO_DISABLE)) {
 369          /* Set number of depth tiles to be filled */
 370          etna_set_state(ctx->stream, VIVS_TS_DEPTH_AUTO_DISABLE_COUNT,
 371                         surf->surf.padded_width * surf->surf.padded_height / 16);
 372          ctx->framebuffer.TS_MEM_CONFIG |= VIVS_TS_MEM_CONFIG_DEPTH_AUTO_DISABLE;
 373       }
 374
 375       surf->level->ts_valid = true;
 376       ctx->dirty |= ETNA_DIRTY_TS | ETNA_DIRTY_DERIVE_TS;
 377    } else {
 378       if (unlikely(new_clear_value != surf->level->clear_value)) { /* Queue normal RS clear for non-TS surfaces */
 379          /* If clear depth value changed, re-generate stored command */
 380          etna_rs_gen_clear_surface(ctx, surf, new_clear_value);
 381       }
 382       /* Update the channels to be cleared */
 383       etna_modify_rs_clearbits(&surf->clear_command, new_clear_bits);
 384    }
 385
 386    etna_submit_rs_state(ctx, &surf->clear_command);
 387
 388    surf->level->clear_value = new_clear_value;
 389    resource_written(ctx, surf->base.texture);
 390    etna_resource(surf->base.texture)->seqno++;
 391 }
 392
 393 static void
 394 etna_clear_rs(struct pipe_context *pctx, unsigned buffers,
 395            const union pipe_color_union *color, double depth, unsigned stencil)
 396 {
 397    struct etna_context *ctx = etna_context(pctx);
 398
 399    /* Flush color and depth cache before clearing anything.
 400     * This is especially important when coming from another surface, as
 401     * otherwise it may clear part of the old surface instead. */
 402    etna_set_state(ctx->stream, VIVS_GL_FLUSH_CACHE, VIVS_GL_FLUSH_CACHE_COLOR | VIVS_GL_FLUSH_CACHE_DEPTH);
 403    etna_stall(ctx->stream, SYNC_RECIPIENT_RA, SYNC_RECIPIENT_PE);
 404
 405    /* Preparation: Flush the TS if needed. This must be done after flushing
 406     * color and depth, otherwise it can result in crashes */
 407    bool need_ts_flush = false;
 408    if ((buffers & PIPE_CLEAR_COLOR) && ctx->framebuffer_s.nr_cbufs) {
 409       struct etna_surface *surf = etna_surface(ctx->framebuffer_s.cbufs[0]);
 410       if (surf->surf.ts_size)
 411          need_ts_flush = true;
 412    }
 413    if ((buffers & PIPE_CLEAR_DEPTHSTENCIL) && ctx->framebuffer_s.zsbuf != NULL) {
 414       struct etna_surface *surf = etna_surface(ctx->framebuffer_s.zsbuf);
 415
 416       if (surf->surf.ts_size)
 417          need_ts_flush = true;
 418    }
 419
 420    if (need_ts_flush)
 421       etna_set_state(ctx->stream, VIVS_TS_FLUSH_CACHE, VIVS_TS_FLUSH_CACHE_FLUSH);
 422
 423    /* No need to set up the TS here as RS clear operations (in contrast to
 424     * resolve and copy) do not require the TS state.
 425     */
 426    if (buffers & PIPE_CLEAR_COLOR) {
 427       for (int idx = 0; idx < ctx->framebuffer_s.nr_cbufs; ++idx) {
 428          etna_blit_clear_color_rs(pctx, ctx->framebuffer_s.cbufs[idx],
 429                                &color[idx]);
 430       }
 431    }
 432
 433    /* Flush the color and depth caches before each RS clear operation
 434     * This fixes a hang on GC600. */
 435    if (buffers & PIPE_CLEAR_DEPTHSTENCIL && buffers & PIPE_CLEAR_COLOR)
 436       etna_set_state(ctx->stream, VIVS_GL_FLUSH_CACHE,
 437                      VIVS_GL_FLUSH_CACHE_COLOR | VIVS_GL_FLUSH_CACHE_DEPTH);
 438
 439    if ((buffers & PIPE_CLEAR_DEPTHSTENCIL) && ctx->framebuffer_s.zsbuf != NULL)
 440       etna_blit_clear_zs_rs(pctx, ctx->framebuffer_s.zsbuf, buffers, depth, stencil);
 441
 442    etna_stall(ctx->stream, SYNC_RECIPIENT_RA, SYNC_RECIPIENT_PE);
 443 }
 444
 445 static bool
 446 etna_manual_blit(struct etna_resource *dst, struct etna_resource_level *dst_lev,
 447                  unsigned int dst_offset, struct etna_resource *src,
 448                  struct etna_resource_level *src_lev, unsigned int src_offset,
 449                  const struct pipe_blit_info *blit_info)
 450 {
 451    void *smap, *srow, *dmap, *drow;
 452    size_t tile_size;
 453
 454    assert(src->layout == ETNA_LAYOUT_TILED);
 455    assert(dst->layout == ETNA_LAYOUT_TILED);
 456    assert(src->base.nr_samples == 0);
 457    assert(dst->base.nr_samples == 0);
 458
 459    tile_size = util_format_get_blocksize(blit_info->src.format) * 4 * 4;
 460
 461    smap = etna_bo_map(src->bo);
 462    if (!smap)
 463       return false;
 464
 465    dmap = etna_bo_map(dst->bo);
 466    if (!dmap)
 467       return false;
 468
 469    srow = smap + src_offset;
 470    drow = dmap + dst_offset;
 471
 472    etna_bo_cpu_prep(src->bo, DRM_ETNA_PREP_READ);
 473    etna_bo_cpu_prep(dst->bo, DRM_ETNA_PREP_WRITE);
 474
 475    for (int y = 0; y < blit_info->src.box.height; y += 4) {
 476       memcpy(drow, srow, tile_size * blit_info->src.box.width);
 477       srow += src_lev->stride * 4;
 478       drow += dst_lev->stride * 4;
 479    }
 480
 481    etna_bo_cpu_fini(dst->bo);
 482    etna_bo_cpu_fini(src->bo);
 483
 484    return true;
 485 }
 486
 487 static inline size_t
 488 etna_compute_tileoffset(const struct pipe_box *box, enum pipe_format format,
 489                         size_t stride, enum etna_surface_layout layout)
 490 {
 491    size_t offset;
 492    unsigned int x = box->x, y = box->y;
 493    unsigned int blocksize = util_format_get_blocksize(format);
 494
 495    switch (layout) {
 496    case ETNA_LAYOUT_LINEAR:
 497       offset = y * stride + x * blocksize;
 498       break;
 499    case ETNA_LAYOUT_MULTI_TILED:
 500       y >>= 1;
 501       /* fall-through */
 502    case ETNA_LAYOUT_TILED:
 503       assert(!(x & 0x03) && !(y & 0x03));
 504       offset = (y & ~0x03) * stride + blocksize * ((x & ~0x03) << 2);
 505       break;
 506    case ETNA_LAYOUT_MULTI_SUPERTILED:
 507       y >>= 1;
 508       /* fall-through */
 509    case ETNA_LAYOUT_SUPER_TILED:
 510       assert(!(x & 0x3f) && !(y & 0x3f));
 511       offset = (y & ~0x3f) * stride + blocksize * ((x & ~0x3f) << 6);
 512       break;
 513    default:
 514       unreachable("invalid resource layout");
 515    }
 516
 517    return offset;
 518 }
 519
 520 static inline void
 521 etna_get_rs_alignment_mask(const struct etna_context *ctx,
 522                            const enum etna_surface_layout layout,
 523                            unsigned int *width_mask, unsigned int *height_mask)
 524 {
 525    unsigned int h_align, w_align;
 526
 527    if (layout & ETNA_LAYOUT_BIT_SUPER) {
 528       w_align = h_align = 64;
 529    } else {
 530       w_align = ETNA_RS_WIDTH_MASK + 1;
 531       h_align = ETNA_RS_HEIGHT_MASK + 1;
 532    }
 533
 534    h_align *= ctx->screen->specs.pixel_pipes;
 535
 536    *width_mask = w_align - 1;
 537    *height_mask = h_align -1;
 538 }
 539
 540 static bool
 541 etna_try_rs_blit(struct pipe_context *pctx,
 542                  const struct pipe_blit_info *blit_info)
 543 {
 544    struct etna_context *ctx = etna_context(pctx);
 545    struct etna_resource *src = etna_resource(blit_info->src.resource);
 546    struct etna_resource *dst = etna_resource(blit_info->dst.resource);
 547    struct compiled_rs_state copy_to_screen;
 548    uint32_t ts_mem_config = 0;
 549    int msaa_xscale = 1, msaa_yscale = 1;
 550
 551    /* Ensure that the level is valid */
 552    assert(blit_info->src.level <= src->base.last_level);
 553    assert(blit_info->dst.level <= dst->base.last_level);
 554
 555    if (!translate_samples_to_xyscale(src->base.nr_samples, &msaa_xscale, &msaa_yscale, NULL))
 556       return false;
 557
 558    /* The width/height are in pixels; they do not change as a result of
 559     * multi-sampling. So, when blitting from a 4x multisampled surface
 560     * to a non-multisampled surface, the width and height will be
 561     * identical. As we do not support scaling, reject different sizes. */
 562    if (blit_info->dst.box.width != blit_info->src.box.width ||
 563        blit_info->dst.box.height != blit_info->src.box.height) {
 564       DBG("scaling requested: source %dx%d destination %dx%d",
 565           blit_info->src.box.width, blit_info->src.box.height,
 566           blit_info->dst.box.width, blit_info->dst.box.height);
 567       return false;
 568    }
 569
 570    /* No masks - RS can't copy specific channels */
 571    unsigned mask = util_format_get_mask(blit_info->dst.format);
 572    if ((blit_info->mask & mask) != mask) {
 573       DBG("sub-mask requested: 0x%02x vs format mask 0x%02x", blit_info->mask, mask);
 574       return false;
 575    }
 576
 577    unsigned src_format = etna_compatible_rs_format(blit_info->src.format);
 578    unsigned dst_format = etna_compatible_rs_format(blit_info->dst.format);
 579    if (translate_rs_format(src_format) == ETNA_NO_MATCH ||
 580        translate_rs_format(dst_format) == ETNA_NO_MATCH ||
 581        blit_info->scissor_enable ||
 582        blit_info->dst.box.depth != blit_info->src.box.depth ||
 583        blit_info->dst.box.depth != 1) {
 584       return false;
 585    }
 586
 587    unsigned w_mask, h_mask;
 588
 589    etna_get_rs_alignment_mask(ctx, src->layout, &w_mask, &h_mask);
 590    if ((blit_info->src.box.x & w_mask) || (blit_info->src.box.y & h_mask))
 591       return false;
 592
 593    etna_get_rs_alignment_mask(ctx, dst->layout, &w_mask, &h_mask);
 594    if ((blit_info->dst.box.x & w_mask) || (blit_info->dst.box.y & h_mask))
 595       return false;
 596
 597    /* Ensure that the Z coordinate is sane */
 598    if (dst->base.target != PIPE_TEXTURE_CUBE)
 599       assert(blit_info->dst.box.z == 0);
 600    if (src->base.target != PIPE_TEXTURE_CUBE)
 601       assert(blit_info->src.box.z == 0);
 602
 603    assert(blit_info->src.box.z < src->base.array_size);
 604    assert(blit_info->dst.box.z < dst->base.array_size);
 605
 606    struct etna_resource_level *src_lev = &src->levels[blit_info->src.level];
 607    struct etna_resource_level *dst_lev = &dst->levels[blit_info->dst.level];
 608
 609    /* we may be given coordinates up to the padded width to avoid
 610     * any alignment issues with different tiling formats */
 611    assert((blit_info->src.box.x + blit_info->src.box.width) * msaa_xscale <= src_lev->padded_width);
 612    assert((blit_info->src.box.y + blit_info->src.box.height) * msaa_yscale <= src_lev->padded_height);
 613    assert(blit_info->dst.box.x + blit_info->dst.box.width <= dst_lev->padded_width);
 614    assert(blit_info->dst.box.y + blit_info->dst.box.height <= dst_lev->padded_height);
 615
 616    unsigned src_offset = src_lev->offset +
 617                          blit_info->src.box.z * src_lev->layer_stride +
 618                          etna_compute_tileoffset(&blit_info->src.box,
 619                                                  blit_info->src.format,
 620                                                  src_lev->stride,
 621                                                  src->layout);
 622    unsigned dst_offset = dst_lev->offset +
 623                          blit_info->dst.box.z * dst_lev->layer_stride +
 624                          etna_compute_tileoffset(&blit_info->dst.box,
 625                                                  blit_info->dst.format,
 626                                                  dst_lev->stride,
 627                                                  dst->layout);
 628
 629    if (src_lev->padded_width <= ETNA_RS_WIDTH_MASK ||
 630        dst_lev->padded_width <= ETNA_RS_WIDTH_MASK ||
 631        src_lev->padded_height <= ETNA_RS_HEIGHT_MASK ||
 632        dst_lev->padded_height <= ETNA_RS_HEIGHT_MASK)
 633       goto manual;
 634
 635    /* If the width is not aligned to the RS width, but is within our
 636     * padding, adjust the width to suite the RS width restriction.
 637     * Note: the RS width/height are converted to source samples here. */
 638    unsigned int width = blit_info->src.box.width * msaa_xscale;
 639    unsigned int height = blit_info->src.box.height * msaa_yscale;
 640    unsigned int w_align = ETNA_RS_WIDTH_MASK + 1;
 641    unsigned int h_align = (ETNA_RS_HEIGHT_MASK + 1) * ctx->specs.pixel_pipes;
 642
 643    if (width & (w_align - 1) && width >= src_lev->width * msaa_xscale && width >= dst_lev->width)
 644       width = align(width, w_align);
 645
 646    if (height & (h_align - 1) && height >= src_lev->height * msaa_yscale && height >= dst_lev->height)
 647       height = align(height, h_align);
 648
 649    /* The padded dimensions are in samples */
 650    if (width > src_lev->padded_width ||
 651        width > dst_lev->padded_width * msaa_xscale ||
 652        height > src_lev->padded_height ||
 653        height > dst_lev->padded_height * msaa_yscale ||
 654        width & (w_align - 1) || height & (h_align - 1))
 655       goto manual;
 656
 657    if (src->base.nr_samples > 1) {
 658       uint32_t msaa_format = translate_msaa_format(src_format);
 659       assert(msaa_format != ETNA_NO_MATCH);
 660       ts_mem_config |= VIVS_TS_MEM_CONFIG_COLOR_COMPRESSION | msaa_format;
 661    }
 662
 663    /* Always flush color and depth cache together before resolving. This works
 664     * around artifacts that appear in some cases when scanning out a texture
 665     * directly after it has been rendered to, such as rendering an animated web
 666     * page in a QtWebEngine based WebView on GC2000. The artifacts look like
 667     * the texture sampler samples zeroes instead of texture data in a small,
 668     * irregular triangle in the lower right of each browser tile quad. Other
 669     * attempts to avoid these artifacts, including a pipeline stall before the
 670     * color flush or a TS cache flush afterwards, or flushing multiple times,
 671     * with stalls before and after each flush, have shown no effect. */
 672    if (src->base.bind & PIPE_BIND_RENDER_TARGET ||
 673        src->base.bind & PIPE_BIND_DEPTH_STENCIL) {
 674       etna_set_state(ctx->stream, VIVS_GL_FLUSH_CACHE,
 675                      VIVS_GL_FLUSH_CACHE_COLOR | VIVS_GL_FLUSH_CACHE_DEPTH);
 676       etna_stall(ctx->stream, SYNC_RECIPIENT_RA, SYNC_RECIPIENT_PE);
 677
 678       if (src->levels[blit_info->src.level].ts_size &&
 679           src->levels[blit_info->src.level].ts_valid)
 680          etna_set_state(ctx->stream, VIVS_TS_FLUSH_CACHE, VIVS_TS_FLUSH_CACHE_FLUSH);
 681    }
 682
 683    /* Set up color TS to source surface before blit, if needed */
 684    bool source_ts_valid = false;
 685    if (src->levels[blit_info->src.level].ts_size &&
 686        src->levels[blit_info->src.level].ts_valid) {
 687       struct etna_reloc reloc;
 688       unsigned ts_offset =
 689          src_lev->ts_offset + blit_info->src.box.z * src_lev->ts_layer_stride;
 690
 691       etna_set_state(ctx->stream, VIVS_TS_MEM_CONFIG,
 692                      VIVS_TS_MEM_CONFIG_COLOR_FAST_CLEAR | ts_mem_config);
 693
 694       memset(&reloc, 0, sizeof(struct etna_reloc));
 695       reloc.bo = src->ts_bo;
 696       reloc.offset = ts_offset;
 697       reloc.flags = ETNA_RELOC_READ;
 698       etna_set_state_reloc(ctx->stream, VIVS_TS_COLOR_STATUS_BASE, &reloc);
 699
 700       memset(&reloc, 0, sizeof(struct etna_reloc));
 701       reloc.bo = src->bo;
 702       reloc.offset = src_lev->offset +
 703                      blit_info->src.box.z * src_lev->layer_stride;
 704       reloc.flags = ETNA_RELOC_READ;
 705       etna_set_state_reloc(ctx->stream, VIVS_TS_COLOR_SURFACE_BASE, &reloc);
 706
 707       etna_set_state(ctx->stream, VIVS_TS_COLOR_CLEAR_VALUE,
 708                      src->levels[blit_info->src.level].clear_value);
 709
 710       source_ts_valid = true;
 711    } else {
 712       etna_set_state(ctx->stream, VIVS_TS_MEM_CONFIG, ts_mem_config);
 713    }
 714    ctx->dirty |= ETNA_DIRTY_TS;
 715
 716    /* Kick off RS here */
 717    etna_compile_rs_state(ctx, &copy_to_screen, &(struct rs_state) {
 718       .source_format = translate_rs_format(src_format),
 719       .source_tiling = src->layout,
 720       .source = src->bo,
 721       .source_offset = src_offset,
 722       .source_stride = src_lev->stride,
 723       .source_padded_width = src_lev->padded_width,
 724       .source_padded_height = src_lev->padded_height,
 725       .source_ts_valid = source_ts_valid,
 726       .dest_format = translate_rs_format(dst_format),
 727       .dest_tiling = dst->layout,
 728       .dest = dst->bo,
 729       .dest_offset = dst_offset,
 730       .dest_stride = dst_lev->stride,
 731       .dest_padded_height = dst_lev->padded_height,
 732       .downsample_x = msaa_xscale > 1,
 733       .downsample_y = msaa_yscale > 1,
 734       .swap_rb = translate_rb_src_dst_swap(src->base.format, dst->base.format),
 735       .dither = {0xffffffff, 0xffffffff}, // XXX dither when going from 24 to 16 bit?
 736       .clear_mode = VIVS_RS_CLEAR_CONTROL_MODE_DISABLED,
 737       .width = width,
 738       .height = height,
 739       .tile_count = src_lev->layer_stride / 64
 740    });
 741
 742    etna_submit_rs_state(ctx, &copy_to_screen);
 743    resource_read(ctx, &src->base);
 744    resource_written(ctx, &dst->base);
 745    dst->seqno++;
 746    dst->levels[blit_info->dst.level].ts_valid = false;
 747    ctx->dirty |= ETNA_DIRTY_DERIVE_TS;
 748
 749    return true;
 750
 751 manual:
 752    if (src->layout == ETNA_LAYOUT_TILED && dst->layout == ETNA_LAYOUT_TILED) {
 753       if ((src->status & ETNA_PENDING_WRITE) ||
 754           (dst->status & ETNA_PENDING_WRITE))
 755          pctx->flush(pctx, NULL, 0);
 756       return etna_manual_blit(dst, dst_lev, dst_offset, src, src_lev, src_offset, blit_info);
 757    }
 758
 759    return false;
 760 }
 761
 762 static void
 763 etna_blit_rs(struct pipe_context *pctx, const struct pipe_blit_info *blit_info)
 764 {
 765    /* This is a more extended version of resource_copy_region */
 766    /* TODO Some cases can be handled by RS; if not, fall back to rendering or
 767     * even CPU copy block of pixels from info->src to info->dst
 768     * (resource, level, box, format);
 769     * function is used for scaling, flipping in x and y direction (negative
 770     * width/height), format conversion, mask and filter and even a scissor rectangle
 771     *
 772     * What can the RS do for us:
 773     *   convert between tiling formats (layouts)
 774     *   downsample 2x in x and y
 775     *   convert between a limited number of pixel formats
 776     *
 777     * For the rest, fall back to util_blitter
 778     * XXX this goes wrong when source surface is supertiled. */
 779    struct etna_context *ctx = etna_context(pctx);
 780    struct pipe_blit_info info = *blit_info;
 781
 782    if (info.src.resource->nr_samples > 1 &&
 783        info.dst.resource->nr_samples <= 1 &&
 784        !util_format_is_depth_or_stencil(info.src.resource->format) &&
 785        !util_format_is_pure_integer(info.src.resource->format)) {
 786       DBG("color resolve unimplemented");
 787       return;
 788    }
 789
 790    if (etna_try_rs_blit(pctx, blit_info))
 791       return;
 792
 793    if (util_try_blit_via_copy_region(pctx, blit_info))
 794       return;
 795
 796    if (info.mask & PIPE_MASK_S) {
 797       DBG("cannot blit stencil, skipping");
 798       info.mask &= ~PIPE_MASK_S;
 799    }
 800
 801    if (!util_blitter_is_blit_supported(ctx->blitter, &info)) {
 802       DBG("blit unsupported %s -> %s",
 803           util_format_short_name(info.src.resource->format),
 804           util_format_short_name(info.dst.resource->format));
 805       return;
 806    }
 807
 808    etna_blit_save_state(ctx);
 809    util_blitter_blit(ctx->blitter, &info);
 810 }
 811
 812 void
 813 etna_clear_blit_rs_init(struct pipe_context *pctx)
 814 {
 815    DBG("etnaviv: Using RS blit engine");
 816    pctx->clear = etna_clear_rs;
 817    pctx->blit = etna_blit_rs;
 818 }