src/gallium/auxiliary/util/u_blitter.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2009 Marek Olšák <maraeo@gmail.com>
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the
   7  * "Software"), to deal in the Software without restriction, including
   8  * without limitation the rights to use, copy, modify, merge, publish,
   9  * distribute, sub license, and/or sell copies of the Software, and to
  10  * permit persons to whom the Software is furnished to do so, subject to
  11  * the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the
  14  * next paragraph) shall be included in all copies or substantial portions
  15  * of the Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  18  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  19  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  20  * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
  21  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  22  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  23  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  24  *
  25  **************************************************************************/
  26
  27 /**
  28  * @file
  29  * Blitter utility to facilitate acceleration of the clear, clear_render_target,
  30  * clear_depth_stencil, resource_copy_region, and blit functions.
  31  *
  32  * @author Marek Olšák
  33  */
  34
  35 #include "pipe/p_context.h"
  36 #include "pipe/p_defines.h"
  37 #include "util/u_inlines.h"
  38 #include "pipe/p_shader_tokens.h"
  39 #include "pipe/p_state.h"
  40
  41 #include "util/u_format.h"
  42 #include "util/u_memory.h"
  43 #include "util/u_math.h"
  44 #include "util/u_blitter.h"
  45 #include "util/u_draw_quad.h"
  46 #include "util/u_sampler.h"
  47 #include "util/u_simple_shaders.h"
  48 #include "util/u_surface.h"
  49 #include "util/u_texture.h"
  50 #include "util/u_upload_mgr.h"
  51
  52 #define INVALID_PTR ((void*)~0)
  53
  54 #define GET_CLEAR_BLEND_STATE_IDX(clear_buffers) \
  55    ((clear_buffers) / PIPE_CLEAR_COLOR0)
  56
  57 #define NUM_RESOLVE_FRAG_SHADERS 5 /* MSAA 2x, 4x, 8x, 16x, 32x */
  58 #define GET_MSAA_RESOLVE_FS_IDX(nr_samples) (util_logbase2(nr_samples)-1)
  59
  60 struct blitter_context_priv
  61 {
  62    struct blitter_context base;
  63
  64    struct u_upload_mgr *upload;
  65
  66    float vertices[4][2][4];   /**< {pos, color} or {pos, texcoord} */
  67
  68    /* Templates for various state objects. */
  69
  70    /* Constant state objects. */
  71    /* Vertex shaders. */
  72    void *vs; /**< Vertex shader which passes {pos, generic} to the output.*/
  73    void *vs_pos_only; /**< Vertex shader which passes pos to the output.*/
  74    void *vs_layered; /**< Vertex shader which sets LAYER = INSTANCEID. */
  75
  76    /* Fragment shaders. */
  77    void *fs_empty;
  78    void *fs_write_one_cbuf;
  79    void *fs_write_all_cbufs;
  80
  81    /* FS which outputs a color from a texture,
  82       where the index is PIPE_TEXTURE_* to be sampled. */
  83    void *fs_texfetch_col[PIPE_MAX_TEXTURE_TYPES];
  84
  85    /* FS which outputs a depth from a texture,
  86       where the index is PIPE_TEXTURE_* to be sampled. */
  87    void *fs_texfetch_depth[PIPE_MAX_TEXTURE_TYPES];
  88    void *fs_texfetch_depthstencil[PIPE_MAX_TEXTURE_TYPES];
  89    void *fs_texfetch_stencil[PIPE_MAX_TEXTURE_TYPES];
  90
  91    /* FS which outputs one sample from a multisample texture. */
  92    void *fs_texfetch_col_msaa[PIPE_MAX_TEXTURE_TYPES];
  93    void *fs_texfetch_depth_msaa[PIPE_MAX_TEXTURE_TYPES];
  94    void *fs_texfetch_depthstencil_msaa[PIPE_MAX_TEXTURE_TYPES];
  95    void *fs_texfetch_stencil_msaa[PIPE_MAX_TEXTURE_TYPES];
  96
  97    /* FS which outputs an average of all samples. */
  98    void *fs_resolve[PIPE_MAX_TEXTURE_TYPES][NUM_RESOLVE_FRAG_SHADERS][2];
  99    void *fs_resolve_sint[PIPE_MAX_TEXTURE_TYPES][NUM_RESOLVE_FRAG_SHADERS][2];
 100    void *fs_resolve_uint[PIPE_MAX_TEXTURE_TYPES][NUM_RESOLVE_FRAG_SHADERS][2];
 101
 102    /* Blend state. */
 103    void *blend[PIPE_MASK_RGBA+1]; /**< blend state with writemask */
 104    void *blend_clear[GET_CLEAR_BLEND_STATE_IDX(PIPE_CLEAR_COLOR)+1];
 105
 106    /* Depth stencil alpha state. */
 107    void *dsa_write_depth_stencil;
 108    void *dsa_write_depth_keep_stencil;
 109    void *dsa_keep_depth_stencil;
 110    void *dsa_keep_depth_write_stencil;
 111
 112    /* Vertex elements states. */
 113    void *velem_state;
 114    void *velem_state_readbuf[4]; /**< X, XY, XYZ, XYZW */
 115
 116    /* Sampler state. */
 117    void *sampler_state;
 118    void *sampler_state_linear;
 119    void *sampler_state_rect;
 120    void *sampler_state_rect_linear;
 121
 122    /* Rasterizer state. */
 123    void *rs_state, *rs_state_scissor, *rs_discard_state;
 124
 125    /* Viewport state. */
 126    struct pipe_viewport_state viewport;
 127
 128    /* Destination surface dimensions. */
 129    unsigned dst_width;
 130    unsigned dst_height;
 131
 132    boolean has_geometry_shader;
 133    boolean has_layered;
 134    boolean has_stream_out;
 135    boolean has_stencil_export;
 136    boolean has_texture_multisample;
 137    boolean cached_all_shaders;
 138
 139    /* The Draw module overrides these functions.
 140     * Always create the blitter before Draw. */
 141    void   (*bind_fs_state)(struct pipe_context *, void *);
 142    void   (*delete_fs_state)(struct pipe_context *, void *);
 143 };
 144
 145 static struct pipe_surface *
 146 util_blitter_get_next_surface_layer(struct pipe_context *pipe,
 147                                     struct pipe_surface *surf);
 148
 149 struct blitter_context *util_blitter_create(struct pipe_context *pipe)
 150 {
 151    struct blitter_context_priv *ctx;
 152    struct pipe_blend_state blend;
 153    struct pipe_depth_stencil_alpha_state dsa;
 154    struct pipe_rasterizer_state rs_state;
 155    struct pipe_sampler_state sampler_state;
 156    struct pipe_vertex_element velem[2];
 157    unsigned i;
 158
 159    ctx = CALLOC_STRUCT(blitter_context_priv);
 160    if (!ctx)
 161       return NULL;
 162
 163    ctx->base.pipe = pipe;
 164    ctx->base.draw_rectangle = util_blitter_draw_rectangle;
 165    ctx->base.get_next_surface_layer = util_blitter_get_next_surface_layer;
 166
 167    ctx->bind_fs_state = pipe->bind_fs_state;
 168    ctx->delete_fs_state = pipe->delete_fs_state;
 169
 170    /* init state objects for them to be considered invalid */
 171    ctx->base.saved_blend_state = INVALID_PTR;
 172    ctx->base.saved_dsa_state = INVALID_PTR;
 173    ctx->base.saved_rs_state = INVALID_PTR;
 174    ctx->base.saved_fs = INVALID_PTR;
 175    ctx->base.saved_vs = INVALID_PTR;
 176    ctx->base.saved_gs = INVALID_PTR;
 177    ctx->base.saved_velem_state = INVALID_PTR;
 178    ctx->base.saved_fb_state.nr_cbufs = ~0;
 179    ctx->base.saved_num_sampler_views = ~0;
 180    ctx->base.saved_num_sampler_states = ~0;
 181    ctx->base.saved_num_so_targets = ~0;
 182
 183    ctx->has_geometry_shader =
 184       pipe->screen->get_shader_param(pipe->screen, PIPE_SHADER_GEOMETRY,
 185                                      PIPE_SHADER_CAP_MAX_INSTRUCTIONS) > 0;
 186    ctx->has_stream_out =
 187       pipe->screen->get_param(pipe->screen,
 188                               PIPE_CAP_MAX_STREAM_OUTPUT_BUFFERS) != 0;
 189
 190    ctx->has_stencil_export =
 191          pipe->screen->get_param(pipe->screen,
 192                                  PIPE_CAP_SHADER_STENCIL_EXPORT);
 193
 194    ctx->has_texture_multisample =
 195       pipe->screen->get_param(pipe->screen, PIPE_CAP_TEXTURE_MULTISAMPLE);
 196
 197    /* blend state objects */
 198    memset(&blend, 0, sizeof(blend));
 199
 200    for (i = 0; i <= PIPE_MASK_RGBA; i++) {
 201       blend.rt[0].colormask = i;
 202       ctx->blend[i] = pipe->create_blend_state(pipe, &blend);
 203    }
 204
 205    /* depth stencil alpha state objects */
 206    memset(&dsa, 0, sizeof(dsa));
 207    ctx->dsa_keep_depth_stencil =
 208       pipe->create_depth_stencil_alpha_state(pipe, &dsa);
 209
 210    dsa.depth.enabled = 1;
 211    dsa.depth.writemask = 1;
 212    dsa.depth.func = PIPE_FUNC_ALWAYS;
 213    ctx->dsa_write_depth_keep_stencil =
 214       pipe->create_depth_stencil_alpha_state(pipe, &dsa);
 215
 216    dsa.stencil[0].enabled = 1;
 217    dsa.stencil[0].func = PIPE_FUNC_ALWAYS;
 218    dsa.stencil[0].fail_op = PIPE_STENCIL_OP_REPLACE;
 219    dsa.stencil[0].zpass_op = PIPE_STENCIL_OP_REPLACE;
 220    dsa.stencil[0].zfail_op = PIPE_STENCIL_OP_REPLACE;
 221    dsa.stencil[0].valuemask = 0xff;
 222    dsa.stencil[0].writemask = 0xff;
 223    ctx->dsa_write_depth_stencil =
 224       pipe->create_depth_stencil_alpha_state(pipe, &dsa);
 225
 226    dsa.depth.enabled = 0;
 227    dsa.depth.writemask = 0;
 228    ctx->dsa_keep_depth_write_stencil =
 229       pipe->create_depth_stencil_alpha_state(pipe, &dsa);
 230
 231    /* sampler state */
 232    memset(&sampler_state, 0, sizeof(sampler_state));
 233    sampler_state.wrap_s = PIPE_TEX_WRAP_CLAMP_TO_EDGE;
 234    sampler_state.wrap_t = PIPE_TEX_WRAP_CLAMP_TO_EDGE;
 235    sampler_state.wrap_r = PIPE_TEX_WRAP_CLAMP_TO_EDGE;
 236    sampler_state.normalized_coords = 1;
 237    ctx->sampler_state = pipe->create_sampler_state(pipe, &sampler_state);
 238    sampler_state.normalized_coords = 0;
 239    ctx->sampler_state_rect = pipe->create_sampler_state(pipe, &sampler_state);
 240
 241    sampler_state.min_img_filter = PIPE_TEX_FILTER_LINEAR;
 242    sampler_state.mag_img_filter = PIPE_TEX_FILTER_LINEAR;
 243    sampler_state.normalized_coords = 1;
 244    ctx->sampler_state_linear = pipe->create_sampler_state(pipe, &sampler_state);
 245    sampler_state.normalized_coords = 0;
 246    ctx->sampler_state_rect_linear = pipe->create_sampler_state(pipe, &sampler_state);
 247
 248    /* rasterizer state */
 249    memset(&rs_state, 0, sizeof(rs_state));
 250    rs_state.cull_face = PIPE_FACE_NONE;
 251    rs_state.half_pixel_center = 1;
 252    rs_state.bottom_edge_rule = 1;
 253    rs_state.flatshade = 1;
 254    rs_state.depth_clip = 1;
 255    ctx->rs_state = pipe->create_rasterizer_state(pipe, &rs_state);
 256
 257    rs_state.scissor = 1;
 258    ctx->rs_state_scissor = pipe->create_rasterizer_state(pipe, &rs_state);
 259
 260    if (ctx->has_stream_out) {
 261       rs_state.scissor = 0;
 262       rs_state.rasterizer_discard = 1;
 263       ctx->rs_discard_state = pipe->create_rasterizer_state(pipe, &rs_state);
 264    }
 265
 266    ctx->base.vb_slot = 0; /* 0 for now */
 267
 268    /* vertex elements states */
 269    memset(&velem[0], 0, sizeof(velem[0]) * 2);
 270    for (i = 0; i < 2; i++) {
 271       velem[i].src_offset = i * 4 * sizeof(float);
 272       velem[i].src_format = PIPE_FORMAT_R32G32B32A32_FLOAT;
 273       velem[i].vertex_buffer_index = ctx->base.vb_slot;
 274    }
 275    ctx->velem_state = pipe->create_vertex_elements_state(pipe, 2, &velem[0]);
 276
 277    if (ctx->has_stream_out) {
 278       static enum pipe_format formats[4] = {
 279          PIPE_FORMAT_R32_UINT,
 280          PIPE_FORMAT_R32G32_UINT,
 281          PIPE_FORMAT_R32G32B32_UINT,
 282          PIPE_FORMAT_R32G32B32A32_UINT
 283       };
 284
 285       for (i = 0; i < 4; i++) {
 286          velem[0].src_format = formats[i];
 287          velem[0].vertex_buffer_index = ctx->base.vb_slot;
 288          ctx->velem_state_readbuf[i] =
 289                pipe->create_vertex_elements_state(pipe, 1, &velem[0]);
 290       }
 291    }
 292
 293    ctx->has_layered =
 294       pipe->screen->get_param(pipe->screen, PIPE_CAP_TGSI_INSTANCEID) &&
 295       pipe->screen->get_param(pipe->screen, PIPE_CAP_TGSI_VS_LAYER_VIEWPORT);
 296
 297    /* set invariant vertex coordinates */
 298    for (i = 0; i < 4; i++)
 299       ctx->vertices[i][0][3] = 1; /*v.w*/
 300
 301    ctx->upload = u_upload_create(pipe, 65536, 4, PIPE_BIND_VERTEX_BUFFER);
 302
 303    return &ctx->base;
 304 }
 305
 306 static void bind_vs_pos_only(struct blitter_context_priv *ctx)
 307 {
 308    struct pipe_context *pipe = ctx->base.pipe;
 309
 310    if (!ctx->vs_pos_only) {
 311       struct pipe_stream_output_info so;
 312       const uint semantic_names[] = { TGSI_SEMANTIC_POSITION };
 313       const uint semantic_indices[] = { 0 };
 314
 315       memset(&so, 0, sizeof(so));
 316       so.num_outputs = 1;
 317       so.output[0].num_components = 1;
 318       so.stride[0] = 1;
 319
 320       ctx->vs_pos_only =
 321          util_make_vertex_passthrough_shader_with_so(pipe, 1, semantic_names,
 322                                                      semantic_indices, FALSE,
 323                                                      &so);
 324    }
 325
 326    pipe->bind_vs_state(pipe, ctx->vs_pos_only);
 327 }
 328
 329 static void bind_vs_passthrough(struct blitter_context_priv *ctx)
 330 {
 331    struct pipe_context *pipe = ctx->base.pipe;
 332
 333    if (!ctx->vs) {
 334       const uint semantic_names[] = { TGSI_SEMANTIC_POSITION,
 335                                       TGSI_SEMANTIC_GENERIC };
 336       const uint semantic_indices[] = { 0, 0 };
 337       ctx->vs =
 338          util_make_vertex_passthrough_shader(pipe, 2, semantic_names,
 339                                              semantic_indices, FALSE);
 340    }
 341
 342    pipe->bind_vs_state(pipe, ctx->vs);
 343 }
 344
 345 static void bind_vs_layered(struct blitter_context_priv *ctx)
 346 {
 347    struct pipe_context *pipe = ctx->base.pipe;
 348
 349    if (!ctx->vs_layered) {
 350       ctx->vs_layered = util_make_layered_clear_vertex_shader(pipe);
 351    }
 352
 353    pipe->bind_vs_state(pipe, ctx->vs_layered);
 354 }
 355
 356 static void bind_fs_empty(struct blitter_context_priv *ctx)
 357 {
 358    struct pipe_context *pipe = ctx->base.pipe;
 359
 360    if (!ctx->fs_empty) {
 361       assert(!ctx->cached_all_shaders);
 362       ctx->fs_empty = util_make_empty_fragment_shader(pipe);
 363    }
 364
 365    ctx->bind_fs_state(pipe, ctx->fs_empty);
 366 }
 367
 368 static void bind_fs_write_one_cbuf(struct blitter_context_priv *ctx)
 369 {
 370    struct pipe_context *pipe = ctx->base.pipe;
 371
 372    if (!ctx->fs_write_one_cbuf) {
 373       assert(!ctx->cached_all_shaders);
 374       ctx->fs_write_one_cbuf =
 375          util_make_fragment_passthrough_shader(pipe, TGSI_SEMANTIC_GENERIC,
 376                                                TGSI_INTERPOLATE_CONSTANT, FALSE);
 377    }
 378
 379    ctx->bind_fs_state(pipe, ctx->fs_write_one_cbuf);
 380 }
 381
 382 static void bind_fs_write_all_cbufs(struct blitter_context_priv *ctx)
 383 {
 384    struct pipe_context *pipe = ctx->base.pipe;
 385
 386    if (!ctx->fs_write_all_cbufs) {
 387       assert(!ctx->cached_all_shaders);
 388       ctx->fs_write_all_cbufs =
 389          util_make_fragment_passthrough_shader(pipe, TGSI_SEMANTIC_GENERIC,
 390                                                TGSI_INTERPOLATE_CONSTANT, TRUE);
 391    }
 392
 393    ctx->bind_fs_state(pipe, ctx->fs_write_all_cbufs);
 394 }
 395
 396 void util_blitter_destroy(struct blitter_context *blitter)
 397 {
 398    struct blitter_context_priv *ctx = (struct blitter_context_priv*)blitter;
 399    struct pipe_context *pipe = blitter->pipe;
 400    int i, j, f;
 401
 402    for (i = 0; i <= PIPE_MASK_RGBA; i++) {
 403       pipe->delete_blend_state(pipe, ctx->blend[i]);
 404    }
 405    for (i = 0; i < Elements(ctx->blend_clear); i++) {
 406       if (ctx->blend_clear[i])
 407          pipe->delete_blend_state(pipe, ctx->blend_clear[i]);
 408    }
 409    pipe->delete_depth_stencil_alpha_state(pipe, ctx->dsa_keep_depth_stencil);
 410    pipe->delete_depth_stencil_alpha_state(pipe,
 411                                           ctx->dsa_write_depth_keep_stencil);
 412    pipe->delete_depth_stencil_alpha_state(pipe, ctx->dsa_write_depth_stencil);
 413    pipe->delete_depth_stencil_alpha_state(pipe, ctx->dsa_keep_depth_write_stencil);
 414
 415    pipe->delete_rasterizer_state(pipe, ctx->rs_state);
 416    pipe->delete_rasterizer_state(pipe, ctx->rs_state_scissor);
 417    if (ctx->rs_discard_state)
 418       pipe->delete_rasterizer_state(pipe, ctx->rs_discard_state);
 419    if (ctx->vs)
 420       pipe->delete_vs_state(pipe, ctx->vs);
 421    if (ctx->vs_pos_only)
 422       pipe->delete_vs_state(pipe, ctx->vs_pos_only);
 423    if (ctx->vs_layered)
 424       pipe->delete_vs_state(pipe, ctx->vs_layered);
 425    pipe->delete_vertex_elements_state(pipe, ctx->velem_state);
 426    for (i = 0; i < 4; i++) {
 427       if (ctx->velem_state_readbuf[i]) {
 428          pipe->delete_vertex_elements_state(pipe, ctx->velem_state_readbuf[i]);
 429       }
 430    }
 431
 432    for (i = 0; i < PIPE_MAX_TEXTURE_TYPES; i++) {
 433       if (ctx->fs_texfetch_col[i])
 434          ctx->delete_fs_state(pipe, ctx->fs_texfetch_col[i]);
 435       if (ctx->fs_texfetch_depth[i])
 436          ctx->delete_fs_state(pipe, ctx->fs_texfetch_depth[i]);
 437       if (ctx->fs_texfetch_depthstencil[i])
 438          ctx->delete_fs_state(pipe, ctx->fs_texfetch_depthstencil[i]);
 439       if (ctx->fs_texfetch_stencil[i])
 440          ctx->delete_fs_state(pipe, ctx->fs_texfetch_stencil[i]);
 441
 442       if (ctx->fs_texfetch_col_msaa[i])
 443          ctx->delete_fs_state(pipe, ctx->fs_texfetch_col_msaa[i]);
 444       if (ctx->fs_texfetch_depth_msaa[i])
 445          ctx->delete_fs_state(pipe, ctx->fs_texfetch_depth_msaa[i]);
 446       if (ctx->fs_texfetch_depthstencil_msaa[i])
 447          ctx->delete_fs_state(pipe, ctx->fs_texfetch_depthstencil_msaa[i]);
 448       if (ctx->fs_texfetch_stencil_msaa[i])
 449          ctx->delete_fs_state(pipe, ctx->fs_texfetch_stencil_msaa[i]);
 450
 451       for (j = 0; j< Elements(ctx->fs_resolve[i]); j++)
 452          for (f = 0; f < 2; f++)
 453             if (ctx->fs_resolve[i][j][f])
 454                ctx->delete_fs_state(pipe, ctx->fs_resolve[i][j][f]);
 455
 456       for (j = 0; j< Elements(ctx->fs_resolve_sint[i]); j++)
 457          for (f = 0; f < 2; f++)
 458             if (ctx->fs_resolve_sint[i][j][f])
 459                ctx->delete_fs_state(pipe, ctx->fs_resolve_sint[i][j][f]);
 460
 461       for (j = 0; j< Elements(ctx->fs_resolve_uint[i]); j++)
 462          for (f = 0; f < 2; f++)
 463             if (ctx->fs_resolve_uint[i][j][f])
 464                ctx->delete_fs_state(pipe, ctx->fs_resolve_uint[i][j][f]);
 465    }
 466
 467    if (ctx->fs_empty)
 468       ctx->delete_fs_state(pipe, ctx->fs_empty);
 469    if (ctx->fs_write_one_cbuf)
 470       ctx->delete_fs_state(pipe, ctx->fs_write_one_cbuf);
 471    if (ctx->fs_write_all_cbufs)
 472       ctx->delete_fs_state(pipe, ctx->fs_write_all_cbufs);
 473
 474    pipe->delete_sampler_state(pipe, ctx->sampler_state_rect_linear);
 475    pipe->delete_sampler_state(pipe, ctx->sampler_state_rect);
 476    pipe->delete_sampler_state(pipe, ctx->sampler_state_linear);
 477    pipe->delete_sampler_state(pipe, ctx->sampler_state);
 478    u_upload_destroy(ctx->upload);
 479    FREE(ctx);
 480 }
 481
 482 void util_blitter_set_texture_multisample(struct blitter_context *blitter,
 483                                           boolean supported)
 484 {
 485    struct blitter_context_priv *ctx = (struct blitter_context_priv*)blitter;
 486
 487    ctx->has_texture_multisample = supported;
 488 }
 489
 490 static void blitter_set_running_flag(struct blitter_context_priv *ctx)
 491 {
 492    if (ctx->base.running) {
 493       _debug_printf("u_blitter:%i: Caught recursion. This is a driver bug.\n",
 494                     __LINE__);
 495    }
 496    ctx->base.running = TRUE;
 497 }
 498
 499 static void blitter_unset_running_flag(struct blitter_context_priv *ctx)
 500 {
 501    if (!ctx->base.running) {
 502       _debug_printf("u_blitter:%i: Caught recursion. This is a driver bug.\n",
 503                     __LINE__);
 504    }
 505    ctx->base.running = FALSE;
 506 }
 507
 508 static void blitter_check_saved_vertex_states(struct blitter_context_priv *ctx)
 509 {
 510    assert(ctx->base.saved_velem_state != INVALID_PTR);
 511    assert(ctx->base.saved_vs != INVALID_PTR);
 512    assert(!ctx->has_geometry_shader || ctx->base.saved_gs != INVALID_PTR);
 513    assert(!ctx->has_stream_out || ctx->base.saved_num_so_targets != ~0);
 514    assert(ctx->base.saved_rs_state != INVALID_PTR);
 515 }
 516
 517 static void blitter_restore_vertex_states(struct blitter_context_priv *ctx)
 518 {
 519    struct pipe_context *pipe = ctx->base.pipe;
 520    unsigned i;
 521
 522    /* Vertex buffer. */
 523    pipe->set_vertex_buffers(pipe, ctx->base.vb_slot, 1,
 524                             &ctx->base.saved_vertex_buffer);
 525    pipe_resource_reference(&ctx->base.saved_vertex_buffer.buffer, NULL);
 526
 527    /* Vertex elements. */
 528    pipe->bind_vertex_elements_state(pipe, ctx->base.saved_velem_state);
 529    ctx->base.saved_velem_state = INVALID_PTR;
 530
 531    /* Vertex shader. */
 532    pipe->bind_vs_state(pipe, ctx->base.saved_vs);
 533    ctx->base.saved_vs = INVALID_PTR;
 534
 535    /* Geometry shader. */
 536    if (ctx->has_geometry_shader) {
 537       pipe->bind_gs_state(pipe, ctx->base.saved_gs);
 538       ctx->base.saved_gs = INVALID_PTR;
 539    }
 540
 541    /* Stream outputs. */
 542    if (ctx->has_stream_out) {
 543       unsigned offsets[PIPE_MAX_SO_BUFFERS];
 544       for (i = 0; i < ctx->base.saved_num_so_targets; i++)
 545          offsets[i] = (unsigned)-1;
 546       pipe->set_stream_output_targets(pipe,
 547                                       ctx->base.saved_num_so_targets,
 548                                       ctx->base.saved_so_targets, offsets);
 549
 550       for (i = 0; i < ctx->base.saved_num_so_targets; i++)
 551          pipe_so_target_reference(&ctx->base.saved_so_targets[i], NULL);
 552
 553       ctx->base.saved_num_so_targets = ~0;
 554    }
 555
 556    /* Rasterizer. */
 557    pipe->bind_rasterizer_state(pipe, ctx->base.saved_rs_state);
 558    ctx->base.saved_rs_state = INVALID_PTR;
 559 }
 560
 561 static void blitter_check_saved_fragment_states(struct blitter_context_priv *ctx)
 562 {
 563    assert(ctx->base.saved_fs != INVALID_PTR);
 564    assert(ctx->base.saved_dsa_state != INVALID_PTR);
 565    assert(ctx->base.saved_blend_state != INVALID_PTR);
 566 }
 567
 568 static void blitter_restore_fragment_states(struct blitter_context_priv *ctx)
 569 {
 570    struct pipe_context *pipe = ctx->base.pipe;
 571
 572    /* Fragment shader. */
 573    ctx->bind_fs_state(pipe, ctx->base.saved_fs);
 574    ctx->base.saved_fs = INVALID_PTR;
 575
 576    /* Depth, stencil, alpha. */
 577    pipe->bind_depth_stencil_alpha_state(pipe, ctx->base.saved_dsa_state);
 578    ctx->base.saved_dsa_state = INVALID_PTR;
 579
 580    /* Blend state. */
 581    pipe->bind_blend_state(pipe, ctx->base.saved_blend_state);
 582    ctx->base.saved_blend_state = INVALID_PTR;
 583
 584    /* Sample mask. */
 585    if (ctx->base.is_sample_mask_saved) {
 586       pipe->set_sample_mask(pipe, ctx->base.saved_sample_mask);
 587       ctx->base.is_sample_mask_saved = FALSE;
 588    }
 589
 590    /* Miscellaneous states. */
 591    /* XXX check whether these are saved and whether they need to be restored
 592     * (depending on the operation) */
 593    pipe->set_stencil_ref(pipe, &ctx->base.saved_stencil_ref);
 594    pipe->set_viewport_states(pipe, 0, 1, &ctx->base.saved_viewport);
 595 }
 596
 597 static void blitter_check_saved_fb_state(struct blitter_context_priv *ctx)
 598 {
 599    assert(ctx->base.saved_fb_state.nr_cbufs != ~0);
 600 }
 601
 602 static void blitter_disable_render_cond(struct blitter_context_priv *ctx)
 603 {
 604    struct pipe_context *pipe = ctx->base.pipe;
 605
 606    if (ctx->base.saved_render_cond_query) {
 607       pipe->render_condition(pipe, NULL, FALSE, 0);
 608    }
 609 }
 610
 611 static void blitter_restore_render_cond(struct blitter_context_priv *ctx)
 612 {
 613    struct pipe_context *pipe = ctx->base.pipe;
 614
 615    if (ctx->base.saved_render_cond_query) {
 616       pipe->render_condition(pipe, ctx->base.saved_render_cond_query,
 617                              ctx->base.saved_render_cond_cond,
 618                              ctx->base.saved_render_cond_mode);
 619       ctx->base.saved_render_cond_query = NULL;
 620    }
 621 }
 622
 623 static void blitter_restore_fb_state(struct blitter_context_priv *ctx)
 624 {
 625    struct pipe_context *pipe = ctx->base.pipe;
 626
 627    pipe->set_framebuffer_state(pipe, &ctx->base.saved_fb_state);
 628    util_unreference_framebuffer_state(&ctx->base.saved_fb_state);
 629 }
 630
 631 static void blitter_check_saved_textures(struct blitter_context_priv *ctx)
 632 {
 633    assert(ctx->base.saved_num_sampler_states != ~0);
 634    assert(ctx->base.saved_num_sampler_views != ~0);
 635 }
 636
 637 static void blitter_restore_textures(struct blitter_context_priv *ctx)
 638 {
 639    struct pipe_context *pipe = ctx->base.pipe;
 640    unsigned i;
 641
 642    /* Fragment sampler states. */
 643    pipe->bind_sampler_states(pipe, PIPE_SHADER_FRAGMENT, 0,
 644                              ctx->base.saved_num_sampler_states,
 645                              ctx->base.saved_sampler_states);
 646
 647    ctx->base.saved_num_sampler_states = ~0;
 648
 649    /* Fragment sampler views. */
 650    pipe->set_sampler_views(pipe, PIPE_SHADER_FRAGMENT, 0,
 651                            ctx->base.saved_num_sampler_views,
 652                            ctx->base.saved_sampler_views);
 653
 654    for (i = 0; i < ctx->base.saved_num_sampler_views; i++)
 655       pipe_sampler_view_reference(&ctx->base.saved_sampler_views[i], NULL);
 656
 657    ctx->base.saved_num_sampler_views = ~0;
 658 }
 659
 660 static void blitter_set_rectangle(struct blitter_context_priv *ctx,
 661                                   int x1, int y1, int x2, int y2,
 662                                   float depth)
 663 {
 664    int i;
 665
 666    /* set vertex positions */
 667    ctx->vertices[0][0][0] = (float)x1 / ctx->dst_width * 2.0f - 1.0f; /*v0.x*/
 668    ctx->vertices[0][0][1] = (float)y1 / ctx->dst_height * 2.0f - 1.0f; /*v0.y*/
 669
 670    ctx->vertices[1][0][0] = (float)x2 / ctx->dst_width * 2.0f - 1.0f; /*v1.x*/
 671    ctx->vertices[1][0][1] = (float)y1 / ctx->dst_height * 2.0f - 1.0f; /*v1.y*/
 672
 673    ctx->vertices[2][0][0] = (float)x2 / ctx->dst_width * 2.0f - 1.0f; /*v2.x*/
 674    ctx->vertices[2][0][1] = (float)y2 / ctx->dst_height * 2.0f - 1.0f; /*v2.y*/
 675
 676    ctx->vertices[3][0][0] = (float)x1 / ctx->dst_width * 2.0f - 1.0f; /*v3.x*/
 677    ctx->vertices[3][0][1] = (float)y2 / ctx->dst_height * 2.0f - 1.0f; /*v3.y*/
 678
 679    for (i = 0; i < 4; i++)
 680       ctx->vertices[i][0][2] = depth; /*z*/
 681
 682    /* viewport */
 683    ctx->viewport.scale[0] = 0.5f * ctx->dst_width;
 684    ctx->viewport.scale[1] = 0.5f * ctx->dst_height;
 685    ctx->viewport.scale[2] = 1.0f;
 686    ctx->viewport.translate[0] = 0.5f * ctx->dst_width;
 687    ctx->viewport.translate[1] = 0.5f * ctx->dst_height;
 688    ctx->viewport.translate[2] = 0.0f;
 689    ctx->base.pipe->set_viewport_states(ctx->base.pipe, 0, 1, &ctx->viewport);
 690 }
 691
 692 static void blitter_set_clear_color(struct blitter_context_priv *ctx,
 693                                     const union pipe_color_union *color)
 694 {
 695    int i;
 696
 697    if (color) {
 698       for (i = 0; i < 4; i++) {
 699          uint32_t *uiverts = (uint32_t *)ctx->vertices[i][1];
 700          uiverts[0] = color->ui[0];
 701          uiverts[1] = color->ui[1];
 702          uiverts[2] = color->ui[2];
 703          uiverts[3] = color->ui[3];
 704       }
 705    } else {
 706       for (i = 0; i < 4; i++) {
 707          ctx->vertices[i][1][0] = 0;
 708          ctx->vertices[i][1][1] = 0;
 709          ctx->vertices[i][1][2] = 0;
 710          ctx->vertices[i][1][3] = 0;
 711       }
 712    }
 713 }
 714
 715 static void get_texcoords(struct pipe_sampler_view *src,
 716                           unsigned src_width0, unsigned src_height0,
 717                           int x1, int y1, int x2, int y2,
 718                           float out[4])
 719 {
 720    struct pipe_resource *tex = src->texture;
 721    unsigned level = src->u.tex.first_level;
 722    boolean normalized = tex->target != PIPE_TEXTURE_RECT &&
 723                         tex->nr_samples <= 1;
 724
 725    if (normalized) {
 726       out[0] = x1 / (float)u_minify(src_width0,  level);
 727       out[1] = y1 / (float)u_minify(src_height0, level);
 728       out[2] = x2 / (float)u_minify(src_width0,  level);
 729       out[3] = y2 / (float)u_minify(src_height0, level);
 730    } else {
 731       out[0] = (float) x1;
 732       out[1] = (float) y1;
 733       out[2] = (float) x2;
 734       out[3] = (float) y2;
 735    }
 736 }
 737
 738 static void set_texcoords_in_vertices(const float coord[4],
 739                                       float *out, unsigned stride)
 740 {
 741    out[0] = coord[0]; /*t0.s*/
 742    out[1] = coord[1]; /*t0.t*/
 743    out += stride;
 744    out[0] = coord[2]; /*t1.s*/
 745    out[1] = coord[1]; /*t1.t*/
 746    out += stride;
 747    out[0] = coord[2]; /*t2.s*/
 748    out[1] = coord[3]; /*t2.t*/
 749    out += stride;
 750    out[0] = coord[0]; /*t3.s*/
 751    out[1] = coord[3]; /*t3.t*/
 752 }
 753
 754 static void blitter_set_texcoords(struct blitter_context_priv *ctx,
 755                                   struct pipe_sampler_view *src,
 756                                   unsigned src_width0, unsigned src_height0,
 757                                   float layer, unsigned sample,
 758                                   int x1, int y1, int x2, int y2)
 759 {
 760    unsigned i;
 761    float coord[4];
 762    float face_coord[4][2];
 763
 764    get_texcoords(src, src_width0, src_height0, x1, y1, x2, y2, coord);
 765
 766    if (src->texture->target == PIPE_TEXTURE_CUBE ||
 767        src->texture->target == PIPE_TEXTURE_CUBE_ARRAY) {
 768       set_texcoords_in_vertices(coord, &face_coord[0][0], 2);
 769       util_map_texcoords2d_onto_cubemap((unsigned)layer % 6,
 770                                         /* pointer, stride in floats */
 771                                         &face_coord[0][0], 2,
 772                                         &ctx->vertices[0][1][0], 8,
 773                                         FALSE);
 774    } else {
 775       set_texcoords_in_vertices(coord, &ctx->vertices[0][1][0], 8);
 776    }
 777
 778    /* Set the layer. */
 779    switch (src->texture->target) {
 780    case PIPE_TEXTURE_3D:
 781       {
 782          float r = layer / (float)u_minify(src->texture->depth0,
 783                                            src->u.tex.first_level);
 784          for (i = 0; i < 4; i++)
 785             ctx->vertices[i][1][2] = r; /*r*/
 786       }
 787       break;
 788
 789    case PIPE_TEXTURE_1D_ARRAY:
 790       for (i = 0; i < 4; i++)
 791          ctx->vertices[i][1][1] = (float) layer; /*t*/
 792       break;
 793
 794    case PIPE_TEXTURE_2D_ARRAY:
 795       for (i = 0; i < 4; i++) {
 796          ctx->vertices[i][1][2] = (float) layer;  /*r*/
 797          ctx->vertices[i][1][3] = (float) sample; /*q*/
 798       }
 799       break;
 800
 801    case PIPE_TEXTURE_CUBE_ARRAY:
 802       for (i = 0; i < 4; i++)
 803          ctx->vertices[i][1][3] = (float) ((unsigned)layer / 6); /*w*/
 804       break;
 805
 806    case PIPE_TEXTURE_2D:
 807       for (i = 0; i < 4; i++) {
 808          ctx->vertices[i][1][3] = (float) sample; /*r*/
 809       }
 810       break;
 811
 812    default:;
 813    }
 814 }
 815
 816 static void blitter_set_dst_dimensions(struct blitter_context_priv *ctx,
 817                                        unsigned width, unsigned height)
 818 {
 819    ctx->dst_width = width;
 820    ctx->dst_height = height;
 821 }
 822
 823 static void *blitter_get_fs_texfetch_col(struct blitter_context_priv *ctx,
 824                                          enum pipe_format format,
 825                                          enum pipe_texture_target target,
 826                                          unsigned src_nr_samples,
 827                                          unsigned dst_nr_samples,
 828                                          unsigned filter)
 829 {
 830    struct pipe_context *pipe = ctx->base.pipe;
 831    unsigned tgsi_tex = util_pipe_tex_to_tgsi_tex(target, src_nr_samples);
 832
 833    assert(target < PIPE_MAX_TEXTURE_TYPES);
 834
 835    if (src_nr_samples > 1) {
 836       void **shader;
 837
 838       if (dst_nr_samples <= 1) {
 839          /* The destination has one sample, so we'll do color resolve. */
 840          boolean is_uint, is_sint;
 841          unsigned index = GET_MSAA_RESOLVE_FS_IDX(src_nr_samples);
 842
 843          is_uint = util_format_is_pure_uint(format);
 844          is_sint = util_format_is_pure_sint(format);
 845
 846          assert(filter < 2);
 847
 848          if (is_uint)
 849             shader = &ctx->fs_resolve_uint[target][index][filter];
 850          else if (is_sint)
 851             shader = &ctx->fs_resolve_sint[target][index][filter];
 852          else
 853             shader = &ctx->fs_resolve[target][index][filter];
 854
 855          if (!*shader) {
 856             assert(!ctx->cached_all_shaders);
 857             if (filter == PIPE_TEX_FILTER_LINEAR) {
 858                *shader = util_make_fs_msaa_resolve_bilinear(pipe, tgsi_tex,
 859                                                    src_nr_samples,
 860                                                    is_uint, is_sint);
 861             }
 862             else {
 863                *shader = util_make_fs_msaa_resolve(pipe, tgsi_tex,
 864                                                    src_nr_samples,
 865                                                    is_uint, is_sint);
 866             }
 867          }
 868       }
 869       else {
 870          /* The destination has multiple samples, we'll do
 871           * an MSAA->MSAA copy.
 872           */
 873          shader = &ctx->fs_texfetch_col_msaa[target];
 874
 875          /* Create the fragment shader on-demand. */
 876          if (!*shader) {
 877             assert(!ctx->cached_all_shaders);
 878             *shader = util_make_fs_blit_msaa_color(pipe, tgsi_tex);
 879          }
 880       }
 881
 882       return *shader;
 883    } else {
 884       void **shader = &ctx->fs_texfetch_col[target];
 885
 886       /* Create the fragment shader on-demand. */
 887       if (!*shader) {
 888          assert(!ctx->cached_all_shaders);
 889          *shader = util_make_fragment_tex_shader(pipe, tgsi_tex,
 890                                                  TGSI_INTERPOLATE_LINEAR);
 891       }
 892
 893       return *shader;
 894    }
 895 }
 896
 897 static INLINE
 898 void *blitter_get_fs_texfetch_depth(struct blitter_context_priv *ctx,
 899                                     enum pipe_texture_target target,
 900                                     unsigned nr_samples)
 901 {
 902    struct pipe_context *pipe = ctx->base.pipe;
 903
 904    assert(target < PIPE_MAX_TEXTURE_TYPES);
 905
 906    if (nr_samples > 1) {
 907       void **shader = &ctx->fs_texfetch_depth_msaa[target];
 908
 909       /* Create the fragment shader on-demand. */
 910       if (!*shader) {
 911          unsigned tgsi_tex;
 912          assert(!ctx->cached_all_shaders);
 913          tgsi_tex = util_pipe_tex_to_tgsi_tex(target, nr_samples);
 914          *shader = util_make_fs_blit_msaa_depth(pipe, tgsi_tex);
 915       }
 916
 917       return *shader;
 918    } else {
 919       void **shader = &ctx->fs_texfetch_depth[target];
 920
 921       /* Create the fragment shader on-demand. */
 922       if (!*shader) {
 923          unsigned tgsi_tex;
 924          assert(!ctx->cached_all_shaders);
 925          tgsi_tex = util_pipe_tex_to_tgsi_tex(target, 0);
 926          *shader =
 927             util_make_fragment_tex_shader_writedepth(pipe, tgsi_tex,
 928                                                      TGSI_INTERPOLATE_LINEAR);
 929       }
 930
 931       return *shader;
 932    }
 933 }
 934
 935 static INLINE
 936 void *blitter_get_fs_texfetch_depthstencil(struct blitter_context_priv *ctx,
 937                                            enum pipe_texture_target target,
 938                                            unsigned nr_samples)
 939 {
 940    struct pipe_context *pipe = ctx->base.pipe;
 941
 942    assert(target < PIPE_MAX_TEXTURE_TYPES);
 943
 944    if (nr_samples > 1) {
 945       void **shader = &ctx->fs_texfetch_depthstencil_msaa[target];
 946
 947       /* Create the fragment shader on-demand. */
 948       if (!*shader) {
 949          unsigned tgsi_tex;
 950          assert(!ctx->cached_all_shaders);
 951          tgsi_tex = util_pipe_tex_to_tgsi_tex(target, nr_samples);
 952          *shader = util_make_fs_blit_msaa_depthstencil(pipe, tgsi_tex);
 953       }
 954
 955       return *shader;
 956    } else {
 957       void **shader = &ctx->fs_texfetch_depthstencil[target];
 958
 959       /* Create the fragment shader on-demand. */
 960       if (!*shader) {
 961          unsigned tgsi_tex;
 962          assert(!ctx->cached_all_shaders);
 963          tgsi_tex = util_pipe_tex_to_tgsi_tex(target, 0);
 964          *shader =
 965             util_make_fragment_tex_shader_writedepthstencil(pipe, tgsi_tex,
 966                                                      TGSI_INTERPOLATE_LINEAR);
 967       }
 968
 969       return *shader;
 970    }
 971 }
 972
 973 static INLINE
 974 void *blitter_get_fs_texfetch_stencil(struct blitter_context_priv *ctx,
 975                                       enum pipe_texture_target target,
 976                                       unsigned nr_samples)
 977 {
 978    struct pipe_context *pipe = ctx->base.pipe;
 979
 980    assert(target < PIPE_MAX_TEXTURE_TYPES);
 981
 982    if (nr_samples > 1) {
 983       void **shader = &ctx->fs_texfetch_stencil_msaa[target];
 984
 985       /* Create the fragment shader on-demand. */
 986       if (!*shader) {
 987          unsigned tgsi_tex;
 988          assert(!ctx->cached_all_shaders);
 989          tgsi_tex = util_pipe_tex_to_tgsi_tex(target, nr_samples);
 990          *shader = util_make_fs_blit_msaa_stencil(pipe, tgsi_tex);
 991       }
 992
 993       return *shader;
 994    } else {
 995       void **shader = &ctx->fs_texfetch_stencil[target];
 996
 997       /* Create the fragment shader on-demand. */
 998       if (!*shader) {
 999          unsigned tgsi_tex;
1000          assert(!ctx->cached_all_shaders);
1001          tgsi_tex = util_pipe_tex_to_tgsi_tex(target, 0);
1002          *shader =
1003             util_make_fragment_tex_shader_writestencil(pipe, tgsi_tex,
1004                                                        TGSI_INTERPOLATE_LINEAR);
1005       }
1006
1007       return *shader;
1008    }
1009 }
1010
1011
1012 /**
1013  * Generate and save all fragment shaders that we will ever need for
1014  * blitting.  Drivers which use the 'draw' fallbacks will typically use
1015  * this to make sure we generate/use shaders that don't go through the
1016  * draw module's wrapper functions.
1017  */
1018 void util_blitter_cache_all_shaders(struct blitter_context *blitter)
1019 {
1020    struct blitter_context_priv *ctx = (struct blitter_context_priv*)blitter;
1021    struct pipe_context *pipe = blitter->pipe;
1022    struct pipe_screen *screen = pipe->screen;
1023    unsigned samples, j, f, target, max_samples;
1024    boolean has_arraytex, has_cubearraytex;
1025
1026    max_samples = ctx->has_texture_multisample ? 2 : 1;
1027    has_arraytex = screen->get_param(screen,
1028                                     PIPE_CAP_MAX_TEXTURE_ARRAY_LAYERS) != 0;
1029    has_cubearraytex = screen->get_param(screen,
1030                                     PIPE_CAP_CUBE_MAP_ARRAY) != 0;
1031
1032    /* It only matters if i <= 1 or > 1. */
1033    for (samples = 1; samples <= max_samples; samples++) {
1034       for (target = PIPE_TEXTURE_1D; target < PIPE_MAX_TEXTURE_TYPES; target++) {
1035          if (!has_arraytex &&
1036              (target == PIPE_TEXTURE_1D_ARRAY ||
1037               target == PIPE_TEXTURE_2D_ARRAY)) {
1038             continue;
1039          }
1040          if (!has_cubearraytex &&
1041              (target == PIPE_TEXTURE_CUBE_ARRAY))
1042             continue;
1043
1044          if (samples > 1 &&
1045              (target != PIPE_TEXTURE_2D &&
1046               target != PIPE_TEXTURE_2D_ARRAY))
1047             continue;
1048
1049          /* If samples == 1, the shaders read one texel. If samples >= 1,
1050           * they read one sample.
1051           */
1052          blitter_get_fs_texfetch_col(ctx, PIPE_FORMAT_R32_FLOAT, target,
1053                                      samples, samples, 0);
1054          blitter_get_fs_texfetch_depth(ctx, target, samples);
1055          if (ctx->has_stencil_export) {
1056             blitter_get_fs_texfetch_depthstencil(ctx, target, samples);
1057             blitter_get_fs_texfetch_stencil(ctx, target, samples);
1058          }
1059
1060          if (samples == 1)
1061             continue;
1062
1063          /* MSAA resolve shaders. */
1064          for (j = 2; j < 32; j++) {
1065             if (!screen->is_format_supported(screen, PIPE_FORMAT_R32_FLOAT,
1066                                              target, j,
1067                                              PIPE_BIND_SAMPLER_VIEW)) {
1068                continue;
1069             }
1070
1071             for (f = 0; f < 2; f++) {
1072                blitter_get_fs_texfetch_col(ctx, PIPE_FORMAT_R32_FLOAT, target,
1073                                            j, 1, f);
1074                blitter_get_fs_texfetch_col(ctx, PIPE_FORMAT_R32_UINT, target,
1075                                            j, 1, f);
1076                blitter_get_fs_texfetch_col(ctx, PIPE_FORMAT_R32_SINT, target,
1077                                            j, 1, f);
1078             }
1079          }
1080       }
1081    }
1082
1083    ctx->fs_empty = util_make_empty_fragment_shader(pipe);
1084
1085    ctx->fs_write_one_cbuf =
1086       util_make_fragment_passthrough_shader(pipe, TGSI_SEMANTIC_GENERIC,
1087                                             TGSI_INTERPOLATE_CONSTANT, FALSE);
1088
1089    ctx->fs_write_all_cbufs =
1090       util_make_fragment_passthrough_shader(pipe, TGSI_SEMANTIC_GENERIC,
1091                                             TGSI_INTERPOLATE_CONSTANT, TRUE);
1092
1093    ctx->cached_all_shaders = TRUE;
1094 }
1095
1096 static void blitter_set_common_draw_rect_state(struct blitter_context_priv *ctx,
1097                                                boolean scissor,
1098                                                boolean vs_layered)
1099 {
1100    struct pipe_context *pipe = ctx->base.pipe;
1101
1102    pipe->bind_rasterizer_state(pipe, scissor ? ctx->rs_state_scissor
1103                                              : ctx->rs_state);
1104    if (vs_layered)
1105       bind_vs_layered(ctx);
1106    else
1107       bind_vs_passthrough(ctx);
1108
1109    if (ctx->has_geometry_shader)
1110       pipe->bind_gs_state(pipe, NULL);
1111    if (ctx->has_stream_out)
1112       pipe->set_stream_output_targets(pipe, 0, NULL, NULL);
1113 }
1114
1115 static void blitter_draw(struct blitter_context_priv *ctx,
1116                          int x1, int y1, int x2, int y2, float depth,
1117                          unsigned num_instances)
1118 {
1119    struct pipe_context *pipe = ctx->base.pipe;
1120    struct pipe_vertex_buffer vb = {0};
1121
1122    blitter_set_rectangle(ctx, x1, y1, x2, y2, depth);
1123
1124    vb.stride = 8 * sizeof(float);
1125
1126    u_upload_data(ctx->upload, 0, sizeof(ctx->vertices), ctx->vertices,
1127                  &vb.buffer_offset, &vb.buffer);
1128    u_upload_unmap(ctx->upload);
1129
1130    pipe->set_vertex_buffers(pipe, ctx->base.vb_slot, 1, &vb);
1131    util_draw_arrays_instanced(pipe, PIPE_PRIM_TRIANGLE_FAN, 0, 4,
1132                               0, num_instances);
1133    pipe_resource_reference(&vb.buffer, NULL);
1134 }
1135
1136 void util_blitter_draw_rectangle(struct blitter_context *blitter,
1137                                  int x1, int y1, int x2, int y2, float depth,
1138                                  enum blitter_attrib_type type,
1139                                  const union pipe_color_union *attrib)
1140 {
1141    struct blitter_context_priv *ctx = (struct blitter_context_priv*)blitter;
1142
1143    switch (type) {
1144       case UTIL_BLITTER_ATTRIB_COLOR:
1145          blitter_set_clear_color(ctx, attrib);
1146          break;
1147
1148       case UTIL_BLITTER_ATTRIB_TEXCOORD:
1149          set_texcoords_in_vertices(attrib->f, &ctx->vertices[0][1][0], 8);
1150          break;
1151
1152       default:;
1153    }
1154
1155    blitter_draw(ctx, x1, y1, x2, y2, depth, 1);
1156 }
1157
1158 static void *get_clear_blend_state(struct blitter_context_priv *ctx,
1159                                    unsigned clear_buffers)
1160 {
1161    struct pipe_context *pipe = ctx->base.pipe;
1162    int index;
1163
1164    clear_buffers &= PIPE_CLEAR_COLOR;
1165
1166    /* Return an existing blend state. */
1167    if (!clear_buffers)
1168       return ctx->blend[0];
1169
1170    index = GET_CLEAR_BLEND_STATE_IDX(clear_buffers);
1171
1172    if (ctx->blend_clear[index])
1173       return ctx->blend_clear[index];
1174
1175    /* Create a new one. */
1176    {
1177       struct pipe_blend_state blend = {0};
1178       unsigned i;
1179
1180       blend.independent_blend_enable = 1;
1181
1182       for (i = 0; i < PIPE_MAX_COLOR_BUFS; i++) {
1183          if (clear_buffers & (PIPE_CLEAR_COLOR0 << i)) {
1184             blend.rt[i].colormask = PIPE_MASK_RGBA;
1185          }
1186       }
1187
1188       ctx->blend_clear[index] = pipe->create_blend_state(pipe, &blend);
1189    }
1190    return ctx->blend_clear[index];
1191 }
1192
1193 static void util_blitter_clear_custom(struct blitter_context *blitter,
1194                                       unsigned width, unsigned height,
1195                                       unsigned num_layers,
1196                                       unsigned clear_buffers,
1197                                       const union pipe_color_union *color,
1198                                       double depth, unsigned stencil,
1199                                       void *custom_blend, void *custom_dsa)
1200 {
1201    struct blitter_context_priv *ctx = (struct blitter_context_priv*)blitter;
1202    struct pipe_context *pipe = ctx->base.pipe;
1203    struct pipe_stencil_ref sr = { { 0 } };
1204
1205    assert(ctx->has_layered || num_layers <= 1);
1206
1207    blitter_set_running_flag(ctx);
1208    blitter_check_saved_vertex_states(ctx);
1209    blitter_check_saved_fragment_states(ctx);
1210    blitter_disable_render_cond(ctx);
1211
1212    /* bind states */
1213    if (custom_blend) {
1214       pipe->bind_blend_state(pipe, custom_blend);
1215    } else {
1216       pipe->bind_blend_state(pipe, get_clear_blend_state(ctx, clear_buffers));
1217    }
1218
1219    if (custom_dsa) {
1220       pipe->bind_depth_stencil_alpha_state(pipe, custom_dsa);
1221    } else if ((clear_buffers & PIPE_CLEAR_DEPTHSTENCIL) == PIPE_CLEAR_DEPTHSTENCIL) {
1222       pipe->bind_depth_stencil_alpha_state(pipe, ctx->dsa_write_depth_stencil);
1223    } else if (clear_buffers & PIPE_CLEAR_DEPTH) {
1224       pipe->bind_depth_stencil_alpha_state(pipe, ctx->dsa_write_depth_keep_stencil);
1225    } else if (clear_buffers & PIPE_CLEAR_STENCIL) {
1226       pipe->bind_depth_stencil_alpha_state(pipe, ctx->dsa_keep_depth_write_stencil);
1227    } else {
1228       pipe->bind_depth_stencil_alpha_state(pipe, ctx->dsa_keep_depth_stencil);
1229    }
1230
1231    sr.ref_value[0] = stencil & 0xff;
1232    pipe->set_stencil_ref(pipe, &sr);
1233
1234    pipe->bind_vertex_elements_state(pipe, ctx->velem_state);
1235    bind_fs_write_all_cbufs(ctx);
1236    pipe->set_sample_mask(pipe, ~0);
1237
1238    blitter_set_dst_dimensions(ctx, width, height);
1239
1240    if (num_layers > 1 && ctx->has_layered) {
1241       blitter_set_common_draw_rect_state(ctx, FALSE, TRUE);
1242       blitter_set_clear_color(ctx, color);
1243       blitter_draw(ctx, 0, 0, width, height, depth, num_layers);
1244    }
1245    else {
1246       blitter_set_common_draw_rect_state(ctx, FALSE, FALSE);
1247       blitter->draw_rectangle(blitter, 0, 0, width, height, (float) depth,
1248                               UTIL_BLITTER_ATTRIB_COLOR, color);
1249    }
1250
1251    blitter_restore_vertex_states(ctx);
1252    blitter_restore_fragment_states(ctx);
1253    blitter_restore_render_cond(ctx);
1254    blitter_unset_running_flag(ctx);
1255 }
1256
1257 void util_blitter_clear(struct blitter_context *blitter,
1258                         unsigned width, unsigned height, unsigned num_layers,
1259                         unsigned clear_buffers,
1260                         const union pipe_color_union *color,
1261                         double depth, unsigned stencil)
1262 {
1263    util_blitter_clear_custom(blitter, width, height, num_layers,
1264                              clear_buffers, color, depth, stencil,
1265                              NULL, NULL);
1266 }
1267
1268 void util_blitter_custom_clear_depth(struct blitter_context *blitter,
1269                                      unsigned width, unsigned height,
1270                                      double depth, void *custom_dsa)
1271 {
1272     static const union pipe_color_union color;
1273     util_blitter_clear_custom(blitter, width, height, 0, 0, &color, depth, 0,
1274                               NULL, custom_dsa);
1275 }
1276
1277 void util_blitter_default_dst_texture(struct pipe_surface *dst_templ,
1278                                       struct pipe_resource *dst,
1279                                       unsigned dstlevel,
1280                                       unsigned dstz)
1281 {
1282     memset(dst_templ, 0, sizeof(*dst_templ));
1283     dst_templ->format = util_format_linear(dst->format);
1284     dst_templ->u.tex.level = dstlevel;
1285     dst_templ->u.tex.first_layer = dstz;
1286     dst_templ->u.tex.last_layer = dstz;
1287 }
1288
1289 static struct pipe_surface *
1290 util_blitter_get_next_surface_layer(struct pipe_context *pipe,
1291                                     struct pipe_surface *surf)
1292 {
1293    struct pipe_surface dst_templ;
1294
1295    memset(&dst_templ, 0, sizeof(dst_templ));
1296    dst_templ.format = surf->format;
1297    dst_templ.u.tex.level = surf->u.tex.level;
1298    dst_templ.u.tex.first_layer = surf->u.tex.first_layer + 1;
1299    dst_templ.u.tex.last_layer = surf->u.tex.last_layer + 1;
1300
1301    return pipe->create_surface(pipe, surf->texture, &dst_templ);
1302 }
1303
1304 void util_blitter_default_src_texture(struct pipe_sampler_view *src_templ,
1305                                       struct pipe_resource *src,
1306                                       unsigned srclevel)
1307 {
1308     memset(src_templ, 0, sizeof(*src_templ));
1309     src_templ->format = util_format_linear(src->format);
1310     src_templ->u.tex.first_level = srclevel;
1311     src_templ->u.tex.last_level = srclevel;
1312     src_templ->u.tex.first_layer = 0;
1313     src_templ->u.tex.last_layer =
1314         src->target == PIPE_TEXTURE_3D ? u_minify(src->depth0, srclevel) - 1
1315                                        : src->array_size - 1;
1316     src_templ->swizzle_r = PIPE_SWIZZLE_RED;
1317     src_templ->swizzle_g = PIPE_SWIZZLE_GREEN;
1318     src_templ->swizzle_b = PIPE_SWIZZLE_BLUE;
1319     src_templ->swizzle_a = PIPE_SWIZZLE_ALPHA;
1320 }
1321
1322 static boolean is_blit_generic_supported(struct blitter_context *blitter,
1323                                          const struct pipe_resource *dst,
1324                                          enum pipe_format dst_format,
1325                                          const struct pipe_resource *src,
1326                                          enum pipe_format src_format,
1327                                          unsigned mask)
1328 {
1329    struct blitter_context_priv *ctx = (struct blitter_context_priv*)blitter;
1330    struct pipe_screen *screen = ctx->base.pipe->screen;
1331
1332    if (dst) {
1333       unsigned bind;
1334       const struct util_format_description *desc =
1335             util_format_description(dst_format);
1336       boolean dst_has_stencil = util_format_has_stencil(desc);
1337
1338       /* Stencil export must be supported for stencil copy. */
1339       if ((mask & PIPE_MASK_S) && dst_has_stencil &&
1340           !ctx->has_stencil_export) {
1341          return FALSE;
1342       }
1343
1344       if (dst_has_stencil || util_format_has_depth(desc))
1345          bind = PIPE_BIND_DEPTH_STENCIL;
1346       else
1347          bind = PIPE_BIND_RENDER_TARGET;
1348
1349       if (!screen->is_format_supported(screen, dst_format, dst->target,
1350                                        dst->nr_samples, bind)) {
1351          return FALSE;
1352       }
1353    }
1354
1355    if (src) {
1356       if (src->nr_samples > 1 && !ctx->has_texture_multisample) {
1357          return FALSE;
1358       }
1359
1360       if (!screen->is_format_supported(screen, src_format, src->target,
1361                                  src->nr_samples, PIPE_BIND_SAMPLER_VIEW)) {
1362          return FALSE;
1363       }
1364
1365       /* Check stencil sampler support for stencil copy. */
1366       if (mask & PIPE_MASK_S) {
1367          if (util_format_has_stencil(util_format_description(src_format))) {
1368             enum pipe_format stencil_format =
1369                util_format_stencil_only(src_format);
1370             assert(stencil_format != PIPE_FORMAT_NONE);
1371
1372             if (stencil_format != src_format &&
1373                 !screen->is_format_supported(screen, stencil_format,
1374                                              src->target, src->nr_samples,
1375                                              PIPE_BIND_SAMPLER_VIEW)) {
1376                return FALSE;
1377             }
1378          }
1379       }
1380    }
1381
1382    return TRUE;
1383 }
1384
1385 boolean util_blitter_is_copy_supported(struct blitter_context *blitter,
1386                                        const struct pipe_resource *dst,
1387                                        const struct pipe_resource *src)
1388 {
1389    return is_blit_generic_supported(blitter, dst, dst->format,
1390                                     src, src->format, PIPE_MASK_RGBAZS);
1391 }
1392
1393 boolean util_blitter_is_blit_supported(struct blitter_context *blitter,
1394                                        const struct pipe_blit_info *info)
1395 {
1396    return is_blit_generic_supported(blitter,
1397                                     info->dst.resource, info->dst.format,
1398                                     info->src.resource, info->src.format,
1399                                     info->mask);
1400 }
1401
1402 void util_blitter_copy_texture(struct blitter_context *blitter,
1403                                struct pipe_resource *dst,
1404                                unsigned dst_level,
1405                                unsigned dstx, unsigned dsty, unsigned dstz,
1406                                struct pipe_resource *src,
1407                                unsigned src_level,
1408                                const struct pipe_box *srcbox)
1409 {
1410    struct blitter_context_priv *ctx = (struct blitter_context_priv*)blitter;
1411    struct pipe_context *pipe = ctx->base.pipe;
1412    struct pipe_surface *dst_view, dst_templ;
1413    struct pipe_sampler_view src_templ, *src_view;
1414    struct pipe_box dstbox;
1415
1416    assert(dst && src);
1417    assert(src->target < PIPE_MAX_TEXTURE_TYPES);
1418
1419    u_box_3d(dstx, dsty, dstz, abs(srcbox->width), abs(srcbox->height),
1420             abs(srcbox->depth), &dstbox);
1421
1422    /* Initialize the surface. */
1423    util_blitter_default_dst_texture(&dst_templ, dst, dst_level, dstz);
1424    dst_view = pipe->create_surface(pipe, dst, &dst_templ);
1425
1426    /* Initialize the sampler view. */
1427    util_blitter_default_src_texture(&src_templ, src, src_level);
1428    src_view = pipe->create_sampler_view(pipe, src, &src_templ);
1429
1430    /* Copy. */
1431    util_blitter_blit_generic(blitter, dst_view, &dstbox,
1432                              src_view, srcbox, src->width0, src->height0,
1433                              PIPE_MASK_RGBAZS, PIPE_TEX_FILTER_NEAREST, NULL);
1434
1435    pipe_surface_reference(&dst_view, NULL);
1436    pipe_sampler_view_reference(&src_view, NULL);
1437 }
1438
1439 void util_blitter_blit_generic(struct blitter_context *blitter,
1440                                struct pipe_surface *dst,
1441                                const struct pipe_box *dstbox,
1442                                struct pipe_sampler_view *src,
1443                                const struct pipe_box *srcbox,
1444                                unsigned src_width0, unsigned src_height0,
1445                                unsigned mask, unsigned filter,
1446                                const struct pipe_scissor_state *scissor)
1447 {
1448    struct blitter_context_priv *ctx = (struct blitter_context_priv*)blitter;
1449    struct pipe_context *pipe = ctx->base.pipe;
1450    struct pipe_framebuffer_state fb_state;
1451    enum pipe_texture_target src_target = src->texture->target;
1452    unsigned src_samples = src->texture->nr_samples;
1453    unsigned dst_samples = dst->texture->nr_samples;
1454    boolean has_depth, has_stencil, has_color;
1455    boolean blit_stencil, blit_depth, blit_color;
1456    void *sampler_state;
1457    const struct util_format_description *src_desc =
1458          util_format_description(src->format);
1459    const struct util_format_description *dst_desc =
1460          util_format_description(dst->format);
1461
1462    has_color = src_desc->colorspace != UTIL_FORMAT_COLORSPACE_ZS &&
1463                dst_desc->colorspace != UTIL_FORMAT_COLORSPACE_ZS;
1464    has_depth = util_format_has_depth(src_desc) &&
1465                util_format_has_depth(dst_desc);
1466    has_stencil = util_format_has_stencil(src_desc) &&
1467                  util_format_has_stencil(dst_desc);
1468
1469    blit_color = has_color && (mask & PIPE_MASK_RGBA);
1470    blit_depth = has_depth && (mask & PIPE_MASK_Z);
1471    blit_stencil = has_stencil && (mask & PIPE_MASK_S) &&
1472                   ctx->has_stencil_export;
1473
1474    if (!blit_stencil && !blit_depth && !blit_color) {
1475       return;
1476    }
1477
1478    if (blit_stencil ||
1479        (dstbox->width == abs(srcbox->width) &&
1480         dstbox->height == abs(srcbox->height))) {
1481       filter = PIPE_TEX_FILTER_NEAREST;
1482    }
1483
1484    /* Check whether the states are properly saved. */
1485    blitter_set_running_flag(ctx);
1486    blitter_check_saved_vertex_states(ctx);
1487    blitter_check_saved_fragment_states(ctx);
1488    blitter_check_saved_textures(ctx);
1489    blitter_check_saved_fb_state(ctx);
1490    blitter_disable_render_cond(ctx);
1491
1492    /* Initialize framebuffer state. */
1493    fb_state.width = dst->width;
1494    fb_state.height = dst->height;
1495    fb_state.nr_cbufs = blit_depth || blit_stencil ? 0 : 1;
1496    fb_state.cbufs[0] = NULL;
1497    fb_state.zsbuf = NULL;
1498
1499    if (blit_depth || blit_stencil) {
1500       pipe->bind_blend_state(pipe, ctx->blend[0]);
1501
1502       if (blit_depth && blit_stencil) {
1503          pipe->bind_depth_stencil_alpha_state(pipe,
1504                                               ctx->dsa_write_depth_stencil);
1505          ctx->bind_fs_state(pipe,
1506                blitter_get_fs_texfetch_depthstencil(ctx, src_target,
1507                                                     src_samples));
1508       } else if (blit_depth) {
1509          pipe->bind_depth_stencil_alpha_state(pipe,
1510                                               ctx->dsa_write_depth_keep_stencil);
1511          ctx->bind_fs_state(pipe,
1512                blitter_get_fs_texfetch_depth(ctx, src_target,
1513                                              src_samples));
1514       } else { /* is_stencil */
1515          pipe->bind_depth_stencil_alpha_state(pipe,
1516                                               ctx->dsa_keep_depth_write_stencil);
1517          ctx->bind_fs_state(pipe,
1518                blitter_get_fs_texfetch_stencil(ctx, src_target,
1519                                                src_samples));
1520       }
1521
1522    } else {
1523       pipe->bind_blend_state(pipe, ctx->blend[mask & PIPE_MASK_RGBA]);
1524       pipe->bind_depth_stencil_alpha_state(pipe, ctx->dsa_keep_depth_stencil);
1525       ctx->bind_fs_state(pipe,
1526             blitter_get_fs_texfetch_col(ctx, src->format, src_target,
1527                                         src_samples, dst_samples, filter));
1528    }
1529
1530    /* Set the linear filter only for scaled color non-MSAA blits. */
1531    if (filter == PIPE_TEX_FILTER_LINEAR) {
1532       if (src_target == PIPE_TEXTURE_RECT) {
1533          sampler_state = ctx->sampler_state_rect_linear;
1534       } else {
1535          sampler_state = ctx->sampler_state_linear;
1536       }
1537    } else {
1538       if (src_target == PIPE_TEXTURE_RECT) {
1539          sampler_state = ctx->sampler_state_rect;
1540       } else {
1541          sampler_state = ctx->sampler_state;
1542       }
1543    }
1544
1545    /* Set samplers. */
1546    if (blit_depth && blit_stencil) {
1547       /* Setup two samplers, one for depth and the other one for stencil. */
1548       struct pipe_sampler_view templ;
1549       struct pipe_sampler_view *views[2];
1550       void *samplers[2] = {sampler_state, sampler_state};
1551
1552       templ = *src;
1553       templ.format = util_format_stencil_only(templ.format);
1554       assert(templ.format != PIPE_FORMAT_NONE);
1555
1556       views[0] = src;
1557       views[1] = pipe->create_sampler_view(pipe, src->texture, &templ);
1558
1559       pipe->set_sampler_views(pipe, PIPE_SHADER_FRAGMENT, 0, 2, views);
1560       pipe->bind_sampler_states(pipe, PIPE_SHADER_FRAGMENT, 0, 2, samplers);
1561
1562       pipe_sampler_view_reference(&views[1], NULL);
1563    } else if (blit_stencil) {
1564       /* Set a stencil-only sampler view for it not to sample depth instead. */
1565       struct pipe_sampler_view templ;
1566       struct pipe_sampler_view *view;
1567
1568       templ = *src;
1569       templ.format = util_format_stencil_only(templ.format);
1570       assert(templ.format != PIPE_FORMAT_NONE);
1571
1572       view = pipe->create_sampler_view(pipe, src->texture, &templ);
1573
1574       pipe->set_sampler_views(pipe, PIPE_SHADER_FRAGMENT, 0, 1, &view);
1575       pipe->bind_sampler_states(pipe, PIPE_SHADER_FRAGMENT,
1576                                 0, 1, &sampler_state);
1577
1578       pipe_sampler_view_reference(&view, NULL);
1579    } else {
1580       pipe->set_sampler_views(pipe, PIPE_SHADER_FRAGMENT, 0, 1, &src);
1581       pipe->bind_sampler_states(pipe, PIPE_SHADER_FRAGMENT,
1582                                 0, 1, &sampler_state);
1583    }
1584
1585    pipe->bind_vertex_elements_state(pipe, ctx->velem_state);
1586    if (scissor) {
1587       pipe->set_scissor_states(pipe, 0, 1, scissor);
1588    }
1589
1590    blitter_set_common_draw_rect_state(ctx, scissor != NULL, FALSE);
1591    blitter_set_dst_dimensions(ctx, dst->width, dst->height);
1592
1593    if ((src_target == PIPE_TEXTURE_1D ||
1594         src_target == PIPE_TEXTURE_2D ||
1595         src_target == PIPE_TEXTURE_RECT) &&
1596        src_samples <= 1) {
1597       /* Draw the quad with the draw_rectangle callback. */
1598
1599       /* Set texture coordinates. - use a pipe color union
1600        * for interface purposes.
1601        * XXX pipe_color_union is a wrong name since we use that to set
1602        * texture coordinates too.
1603        */
1604       union pipe_color_union coord;
1605       get_texcoords(src, src_width0, src_height0, srcbox->x, srcbox->y,
1606                     srcbox->x+srcbox->width, srcbox->y+srcbox->height, coord.f);
1607
1608       /* Set framebuffer state. */
1609       if (blit_depth || blit_stencil) {
1610          fb_state.zsbuf = dst;
1611       } else {
1612          fb_state.cbufs[0] = dst;
1613       }
1614       pipe->set_framebuffer_state(pipe, &fb_state);
1615
1616       /* Draw. */
1617       pipe->set_sample_mask(pipe, ~0);
1618       blitter->draw_rectangle(blitter, dstbox->x, dstbox->y,
1619                               dstbox->x + dstbox->width,
1620                               dstbox->y + dstbox->height, 0,
1621                               UTIL_BLITTER_ATTRIB_TEXCOORD, &coord);
1622    } else {
1623       /* Draw the quad with the generic codepath. */
1624       int dst_z;
1625       for (dst_z = 0; dst_z < dstbox->depth; dst_z++) {
1626          struct pipe_surface *old;
1627          float dst2src_scale = srcbox->depth / (float)dstbox->depth;
1628
1629          /* Scale Z properly if the blit is scaled.
1630           *
1631           * When downscaling, we want the coordinates centered, so that
1632           * mipmapping works for 3D textures. For example, when generating
1633           * a 4x4x4 level, this wouldn't average the pixels:
1634           *
1635           *   src Z:  0 1 2 3 4 5 6 7
1636           *   dst Z:  0   1   2   3
1637           *
1638           * Because the pixels are not centered below the pixels of the higher
1639           * level. Therefore, we want this:
1640           *   src Z:  0 1 2 3 4 5 6 7
1641           *   dst Z:   0   1   2   3
1642           *
1643           * dst_offset defines the offset needed for centering the pixels and
1644           * it works with any scaling (not just 2x).
1645           */
1646          float dst_offset = ((srcbox->depth - 1) -
1647                              (dstbox->depth - 1) * dst2src_scale) * 0.5;
1648          float src_z = (dst_z + dst_offset) * dst2src_scale;
1649
1650          /* Set framebuffer state. */
1651          if (blit_depth || blit_stencil) {
1652             fb_state.zsbuf = dst;
1653          } else {
1654             fb_state.cbufs[0] = dst;
1655          }
1656          pipe->set_framebuffer_state(pipe, &fb_state);
1657
1658          /* See if we need to blit a multisample or singlesample buffer. */
1659          if (src_samples == dst_samples && dst_samples > 1) {
1660             /* MSAA copy. */
1661             unsigned i, max_sample = dst_samples - 1;
1662
1663             for (i = 0; i <= max_sample; i++) {
1664                pipe->set_sample_mask(pipe, 1 << i);
1665                blitter_set_texcoords(ctx, src, src_width0, src_height0,
1666                                      srcbox->z + src_z,
1667                                      i, srcbox->x, srcbox->y,
1668                                      srcbox->x + srcbox->width,
1669                                      srcbox->y + srcbox->height);
1670                blitter_draw(ctx, dstbox->x, dstbox->y,
1671                             dstbox->x + dstbox->width,
1672                             dstbox->y + dstbox->height, 0, 1);
1673             }
1674          } else {
1675             /* Normal copy, MSAA upsampling, or MSAA resolve. */
1676             pipe->set_sample_mask(pipe, ~0);
1677             blitter_set_texcoords(ctx, src, src_width0, src_height0,
1678                                   srcbox->z + src_z, 0,
1679                                   srcbox->x, srcbox->y,
1680                                   srcbox->x + srcbox->width,
1681                                   srcbox->y + srcbox->height);
1682             blitter_draw(ctx, dstbox->x, dstbox->y,
1683                          dstbox->x + dstbox->width,
1684                          dstbox->y + dstbox->height, 0, 1);
1685          }
1686
1687          /* Get the next surface or (if this is the last iteration)
1688           * just unreference the last one. */
1689          old = dst;
1690          if (dst_z < dstbox->depth-1) {
1691             dst = ctx->base.get_next_surface_layer(ctx->base.pipe, dst);
1692          }
1693          if (dst_z) {
1694             pipe_surface_reference(&old, NULL);
1695          }
1696       }
1697    }
1698
1699    blitter_restore_vertex_states(ctx);
1700    blitter_restore_fragment_states(ctx);
1701    blitter_restore_textures(ctx);
1702    blitter_restore_fb_state(ctx);
1703    if (scissor) {
1704       pipe->set_scissor_states(pipe, 0, 1, &ctx->base.saved_scissor);
1705    }
1706    blitter_restore_render_cond(ctx);
1707    blitter_unset_running_flag(ctx);
1708 }
1709
1710 void
1711 util_blitter_blit(struct blitter_context *blitter,
1712                   const struct pipe_blit_info *info)
1713 {
1714    struct pipe_resource *dst = info->dst.resource;
1715    struct pipe_resource *src = info->src.resource;
1716    struct blitter_context_priv *ctx = (struct blitter_context_priv*)blitter;
1717    struct pipe_context *pipe = ctx->base.pipe;
1718    struct pipe_surface *dst_view, dst_templ;
1719    struct pipe_sampler_view src_templ, *src_view;
1720
1721    /* Initialize the surface. */
1722    util_blitter_default_dst_texture(&dst_templ, dst, info->dst.level,
1723                                     info->dst.box.z);
1724    dst_templ.format = info->dst.format;
1725    dst_view = pipe->create_surface(pipe, dst, &dst_templ);
1726
1727    /* Initialize the sampler view. */
1728    util_blitter_default_src_texture(&src_templ, src, info->src.level);
1729    src_templ.format = info->src.format;
1730    src_view = pipe->create_sampler_view(pipe, src, &src_templ);
1731
1732    /* Copy. */
1733    util_blitter_blit_generic(blitter, dst_view, &info->dst.box,
1734                              src_view, &info->src.box, src->width0, src->height0,
1735                              info->mask, info->filter,
1736                              info->scissor_enable ? &info->scissor : NULL);
1737
1738    pipe_surface_reference(&dst_view, NULL);
1739    pipe_sampler_view_reference(&src_view, NULL);
1740 }
1741
1742 /* Clear a region of a color surface to a constant value. */
1743 void util_blitter_clear_render_target(struct blitter_context *blitter,
1744                                       struct pipe_surface *dstsurf,
1745                                       const union pipe_color_union *color,
1746                                       unsigned dstx, unsigned dsty,
1747                                       unsigned width, unsigned height)
1748 {
1749    struct blitter_context_priv *ctx = (struct blitter_context_priv*)blitter;
1750    struct pipe_context *pipe = ctx->base.pipe;
1751    struct pipe_framebuffer_state fb_state;
1752
1753    assert(dstsurf->texture);
1754    if (!dstsurf->texture)
1755       return;
1756
1757    /* check the saved state */
1758    blitter_set_running_flag(ctx);
1759    blitter_check_saved_vertex_states(ctx);
1760    blitter_check_saved_fragment_states(ctx);
1761    blitter_check_saved_fb_state(ctx);
1762    blitter_disable_render_cond(ctx);
1763
1764    /* bind states */
1765    pipe->bind_blend_state(pipe, ctx->blend[PIPE_MASK_RGBA]);
1766    pipe->bind_depth_stencil_alpha_state(pipe, ctx->dsa_keep_depth_stencil);
1767    bind_fs_write_one_cbuf(ctx);
1768    pipe->bind_vertex_elements_state(pipe, ctx->velem_state);
1769
1770    /* set a framebuffer state */
1771    fb_state.width = dstsurf->width;
1772    fb_state.height = dstsurf->height;
1773    fb_state.nr_cbufs = 1;
1774    fb_state.cbufs[0] = dstsurf;
1775    fb_state.zsbuf = 0;
1776    pipe->set_framebuffer_state(pipe, &fb_state);
1777    pipe->set_sample_mask(pipe, ~0);
1778
1779    blitter_set_common_draw_rect_state(ctx, FALSE, FALSE);
1780    blitter_set_dst_dimensions(ctx, dstsurf->width, dstsurf->height);
1781    blitter->draw_rectangle(blitter, dstx, dsty, dstx+width, dsty+height, 0,
1782                            UTIL_BLITTER_ATTRIB_COLOR, color);
1783
1784    blitter_restore_vertex_states(ctx);
1785    blitter_restore_fragment_states(ctx);
1786    blitter_restore_fb_state(ctx);
1787    blitter_restore_render_cond(ctx);
1788    blitter_unset_running_flag(ctx);
1789 }
1790
1791 /* Clear a region of a depth stencil surface. */
1792 void util_blitter_clear_depth_stencil(struct blitter_context *blitter,
1793                                       struct pipe_surface *dstsurf,
1794                                       unsigned clear_flags,
1795                                       double depth,
1796                                       unsigned stencil,
1797                                       unsigned dstx, unsigned dsty,
1798                                       unsigned width, unsigned height)
1799 {
1800    struct blitter_context_priv *ctx = (struct blitter_context_priv*)blitter;
1801    struct pipe_context *pipe = ctx->base.pipe;
1802    struct pipe_framebuffer_state fb_state;
1803    struct pipe_stencil_ref sr = { { 0 } };
1804
1805    assert(dstsurf->texture);
1806    if (!dstsurf->texture)
1807       return;
1808
1809    /* check the saved state */
1810    blitter_set_running_flag(ctx);
1811    blitter_check_saved_vertex_states(ctx);
1812    blitter_check_saved_fragment_states(ctx);
1813    blitter_check_saved_fb_state(ctx);
1814    blitter_disable_render_cond(ctx);
1815
1816    /* bind states */
1817    pipe->bind_blend_state(pipe, ctx->blend[0]);
1818    if ((clear_flags & PIPE_CLEAR_DEPTHSTENCIL) == PIPE_CLEAR_DEPTHSTENCIL) {
1819       sr.ref_value[0] = stencil & 0xff;
1820       pipe->bind_depth_stencil_alpha_state(pipe, ctx->dsa_write_depth_stencil);
1821       pipe->set_stencil_ref(pipe, &sr);
1822    }
1823    else if (clear_flags & PIPE_CLEAR_DEPTH) {
1824       pipe->bind_depth_stencil_alpha_state(pipe, ctx->dsa_write_depth_keep_stencil);
1825    }
1826    else if (clear_flags & PIPE_CLEAR_STENCIL) {
1827       sr.ref_value[0] = stencil & 0xff;
1828       pipe->bind_depth_stencil_alpha_state(pipe, ctx->dsa_keep_depth_write_stencil);
1829       pipe->set_stencil_ref(pipe, &sr);
1830    }
1831    else
1832       /* hmm that should be illegal probably, or make it a no-op somewhere */
1833       pipe->bind_depth_stencil_alpha_state(pipe, ctx->dsa_keep_depth_stencil);
1834
1835    bind_fs_empty(ctx);
1836    pipe->bind_vertex_elements_state(pipe, ctx->velem_state);
1837
1838    /* set a framebuffer state */
1839    fb_state.width = dstsurf->width;
1840    fb_state.height = dstsurf->height;
1841    fb_state.nr_cbufs = 0;
1842    fb_state.cbufs[0] = 0;
1843    fb_state.zsbuf = dstsurf;
1844    pipe->set_framebuffer_state(pipe, &fb_state);
1845    pipe->set_sample_mask(pipe, ~0);
1846
1847    blitter_set_common_draw_rect_state(ctx, FALSE, FALSE);
1848    blitter_set_dst_dimensions(ctx, dstsurf->width, dstsurf->height);
1849    blitter->draw_rectangle(blitter, dstx, dsty, dstx+width, dsty+height,
1850                            (float) depth,
1851                            UTIL_BLITTER_ATTRIB_NONE, NULL);
1852
1853    blitter_restore_vertex_states(ctx);
1854    blitter_restore_fragment_states(ctx);
1855    blitter_restore_fb_state(ctx);
1856    blitter_restore_render_cond(ctx);
1857    blitter_unset_running_flag(ctx);
1858 }
1859
1860 /* draw a rectangle across a region using a custom dsa stage - for r600g */
1861 void util_blitter_custom_depth_stencil(struct blitter_context *blitter,
1862                                        struct pipe_surface *zsurf,
1863                                        struct pipe_surface *cbsurf,
1864                                        unsigned sample_mask,
1865                                        void *dsa_stage, float depth)
1866 {
1867    struct blitter_context_priv *ctx = (struct blitter_context_priv*)blitter;
1868    struct pipe_context *pipe = ctx->base.pipe;
1869    struct pipe_framebuffer_state fb_state;
1870
1871    assert(zsurf->texture);
1872    if (!zsurf->texture)
1873       return;
1874
1875    /* check the saved state */
1876    blitter_set_running_flag(ctx);
1877    blitter_check_saved_vertex_states(ctx);
1878    blitter_check_saved_fragment_states(ctx);
1879    blitter_check_saved_fb_state(ctx);
1880    blitter_disable_render_cond(ctx);
1881
1882    /* bind states */
1883    pipe->bind_blend_state(pipe, cbsurf ? ctx->blend[PIPE_MASK_RGBA] :
1884                                          ctx->blend[0]);
1885    pipe->bind_depth_stencil_alpha_state(pipe, dsa_stage);
1886    if (cbsurf)
1887       bind_fs_write_one_cbuf(ctx);
1888    else
1889       bind_fs_empty(ctx);
1890    pipe->bind_vertex_elements_state(pipe, ctx->velem_state);
1891
1892    /* set a framebuffer state */
1893    fb_state.width = zsurf->width;
1894    fb_state.height = zsurf->height;
1895    fb_state.nr_cbufs = 1;
1896    if (cbsurf) {
1897            fb_state.cbufs[0] = cbsurf;
1898            fb_state.nr_cbufs = 1;
1899    } else {
1900            fb_state.cbufs[0] = NULL;
1901            fb_state.nr_cbufs = 0;
1902    }
1903    fb_state.zsbuf = zsurf;
1904    pipe->set_framebuffer_state(pipe, &fb_state);
1905    pipe->set_sample_mask(pipe, sample_mask);
1906
1907    blitter_set_common_draw_rect_state(ctx, FALSE, FALSE);
1908    blitter_set_dst_dimensions(ctx, zsurf->width, zsurf->height);
1909    blitter->draw_rectangle(blitter, 0, 0, zsurf->width, zsurf->height, depth,
1910                            UTIL_BLITTER_ATTRIB_NONE, NULL);
1911
1912    blitter_restore_vertex_states(ctx);
1913    blitter_restore_fragment_states(ctx);
1914    blitter_restore_fb_state(ctx);
1915    blitter_restore_render_cond(ctx);
1916    blitter_unset_running_flag(ctx);
1917 }
1918
1919 void util_blitter_copy_buffer(struct blitter_context *blitter,
1920                               struct pipe_resource *dst,
1921                               unsigned dstx,
1922                               struct pipe_resource *src,
1923                               unsigned srcx,
1924                               unsigned size)
1925 {
1926    struct blitter_context_priv *ctx = (struct blitter_context_priv*)blitter;
1927    struct pipe_context *pipe = ctx->base.pipe;
1928    struct pipe_vertex_buffer vb;
1929    struct pipe_stream_output_target *so_target;
1930    unsigned offsets[PIPE_MAX_SO_BUFFERS] = {0};
1931
1932    if (srcx >= src->width0 ||
1933        dstx >= dst->width0) {
1934       return;
1935    }
1936    if (srcx + size > src->width0) {
1937       size = src->width0 - srcx;
1938    }
1939    if (dstx + size > dst->width0) {
1940       size = dst->width0 - dstx;
1941    }
1942
1943    /* Drivers not capable of Stream Out should not call this function
1944     * in the first place. */
1945    assert(ctx->has_stream_out);
1946
1947    /* Some alignment is required. */
1948    if (srcx % 4 != 0 || dstx % 4 != 0 || size % 4 != 0 ||
1949        !ctx->has_stream_out) {
1950       struct pipe_box box;
1951       u_box_1d(srcx, size, &box);
1952       util_resource_copy_region(pipe, dst, 0, dstx, 0, 0, src, 0, &box);
1953       return;
1954    }
1955
1956    blitter_set_running_flag(ctx);
1957    blitter_check_saved_vertex_states(ctx);
1958    blitter_disable_render_cond(ctx);
1959
1960    vb.buffer = src;
1961    vb.buffer_offset = srcx;
1962    vb.stride = 4;
1963
1964    pipe->set_vertex_buffers(pipe, ctx->base.vb_slot, 1, &vb);
1965    pipe->bind_vertex_elements_state(pipe, ctx->velem_state_readbuf[0]);
1966    bind_vs_pos_only(ctx);
1967    if (ctx->has_geometry_shader)
1968       pipe->bind_gs_state(pipe, NULL);
1969    pipe->bind_rasterizer_state(pipe, ctx->rs_discard_state);
1970
1971    so_target = pipe->create_stream_output_target(pipe, dst, dstx, size);
1972    pipe->set_stream_output_targets(pipe, 1, &so_target, offsets);
1973
1974    util_draw_arrays(pipe, PIPE_PRIM_POINTS, 0, size / 4);
1975
1976    blitter_restore_vertex_states(ctx);
1977    blitter_restore_render_cond(ctx);
1978    blitter_unset_running_flag(ctx);
1979    pipe_so_target_reference(&so_target, NULL);
1980 }
1981
1982 void util_blitter_clear_buffer(struct blitter_context *blitter,
1983                                struct pipe_resource *dst,
1984                                unsigned offset, unsigned size,
1985                                unsigned num_channels,
1986                                const union pipe_color_union *clear_value)
1987 {
1988    struct blitter_context_priv *ctx = (struct blitter_context_priv*)blitter;
1989    struct pipe_context *pipe = ctx->base.pipe;
1990    struct pipe_vertex_buffer vb = {0};
1991    struct pipe_stream_output_target *so_target;
1992    unsigned offsets[PIPE_MAX_SO_BUFFERS] = {0};
1993
1994    assert(num_channels >= 1);
1995    assert(num_channels <= 4);
1996
1997    /* IMPORTANT:  DON'T DO ANY BOUNDS CHECKING HERE!
1998     *
1999     * R600 uses this to initialize texture resources, so width0 might not be
2000     * what you think it is.
2001     */
2002
2003    /* Streamout is required. */
2004    if (!ctx->has_stream_out) {
2005       assert(!"Streamout unsupported in util_blitter_clear_buffer()");
2006       return;
2007    }
2008
2009    /* Some alignment is required. */
2010    if (offset % 4 != 0 || size % 4 != 0) {
2011       assert(!"Bad alignment in util_blitter_clear_buffer()");
2012       return;
2013    }
2014
2015    u_upload_data(ctx->upload, 0, num_channels*4, clear_value,
2016                  &vb.buffer_offset, &vb.buffer);
2017    vb.stride = 0;
2018
2019    blitter_set_running_flag(ctx);
2020    blitter_check_saved_vertex_states(ctx);
2021    blitter_disable_render_cond(ctx);
2022
2023    pipe->set_vertex_buffers(pipe, ctx->base.vb_slot, 1, &vb);
2024    pipe->bind_vertex_elements_state(pipe,
2025                                     ctx->velem_state_readbuf[num_channels-1]);
2026    bind_vs_pos_only(ctx);
2027    if (ctx->has_geometry_shader)
2028       pipe->bind_gs_state(pipe, NULL);
2029    pipe->bind_rasterizer_state(pipe, ctx->rs_discard_state);
2030
2031    so_target = pipe->create_stream_output_target(pipe, dst, offset, size);
2032    pipe->set_stream_output_targets(pipe, 1, &so_target, offsets);
2033
2034    util_draw_arrays(pipe, PIPE_PRIM_POINTS, 0, size / 4);
2035
2036    blitter_restore_vertex_states(ctx);
2037    blitter_restore_render_cond(ctx);
2038    blitter_unset_running_flag(ctx);
2039    pipe_so_target_reference(&so_target, NULL);
2040    pipe_resource_reference(&vb.buffer, NULL);
2041 }
2042
2043 /* probably radeon specific */
2044 void util_blitter_custom_resolve_color(struct blitter_context *blitter,
2045                                        struct pipe_resource *dst,
2046                                        unsigned dst_level,
2047                                        unsigned dst_layer,
2048                                        struct pipe_resource *src,
2049                                        unsigned src_layer,
2050                                        unsigned sample_mask,
2051                                        void *custom_blend,
2052                                        enum pipe_format format)
2053 {
2054    struct blitter_context_priv *ctx = (struct blitter_context_priv*)blitter;
2055    struct pipe_context *pipe = ctx->base.pipe;
2056    struct pipe_framebuffer_state fb_state;
2057    struct pipe_surface *srcsurf, *dstsurf, surf_tmpl;
2058
2059    blitter_set_running_flag(ctx);
2060    blitter_check_saved_vertex_states(ctx);
2061    blitter_check_saved_fragment_states(ctx);
2062    blitter_disable_render_cond(ctx);
2063
2064    /* bind states */
2065    pipe->bind_blend_state(pipe, custom_blend);
2066    pipe->bind_depth_stencil_alpha_state(pipe, ctx->dsa_keep_depth_stencil);
2067    pipe->bind_vertex_elements_state(pipe, ctx->velem_state);
2068    bind_fs_write_one_cbuf(ctx);
2069    pipe->set_sample_mask(pipe, sample_mask);
2070
2071    memset(&surf_tmpl, 0, sizeof(surf_tmpl));
2072    surf_tmpl.format = format;
2073    surf_tmpl.u.tex.level = dst_level;
2074    surf_tmpl.u.tex.first_layer = dst_layer;
2075    surf_tmpl.u.tex.last_layer = dst_layer;
2076
2077    dstsurf = pipe->create_surface(pipe, dst, &surf_tmpl);
2078
2079    surf_tmpl.u.tex.level = 0;
2080    surf_tmpl.u.tex.first_layer = src_layer;
2081    surf_tmpl.u.tex.last_layer = src_layer;
2082
2083    srcsurf = pipe->create_surface(pipe, src, &surf_tmpl);
2084
2085    /* set a framebuffer state */
2086    fb_state.width = src->width0;
2087    fb_state.height = src->height0;
2088    fb_state.nr_cbufs = 2;
2089    fb_state.cbufs[0] = srcsurf;
2090    fb_state.cbufs[1] = dstsurf;
2091    fb_state.zsbuf = NULL;
2092    pipe->set_framebuffer_state(pipe, &fb_state);
2093
2094    blitter_set_common_draw_rect_state(ctx, FALSE, FALSE);
2095    blitter_set_dst_dimensions(ctx, src->width0, src->height0);
2096    blitter->draw_rectangle(blitter, 0, 0, src->width0, src->height0,
2097                            0, 0, NULL);
2098    blitter_restore_fb_state(ctx);
2099    blitter_restore_vertex_states(ctx);
2100    blitter_restore_fragment_states(ctx);
2101    blitter_restore_render_cond(ctx);
2102    blitter_unset_running_flag(ctx);
2103
2104    pipe_surface_reference(&srcsurf, NULL);
2105    pipe_surface_reference(&dstsurf, NULL);
2106 }
2107
2108 void util_blitter_custom_color(struct blitter_context *blitter,
2109                                struct pipe_surface *dstsurf,
2110                                void *custom_blend)
2111 {
2112    struct blitter_context_priv *ctx = (struct blitter_context_priv*)blitter;
2113    struct pipe_context *pipe = ctx->base.pipe;
2114    struct pipe_framebuffer_state fb_state;
2115
2116    assert(dstsurf->texture);
2117    if (!dstsurf->texture)
2118       return;
2119
2120    /* check the saved state */
2121    blitter_set_running_flag(ctx);
2122    blitter_check_saved_vertex_states(ctx);
2123    blitter_check_saved_fragment_states(ctx);
2124    blitter_check_saved_fb_state(ctx);
2125    blitter_disable_render_cond(ctx);
2126
2127    /* bind states */
2128    pipe->bind_blend_state(pipe, custom_blend ? custom_blend
2129                                              : ctx->blend[PIPE_MASK_RGBA]);
2130    pipe->bind_depth_stencil_alpha_state(pipe, ctx->dsa_keep_depth_stencil);
2131    bind_fs_write_one_cbuf(ctx);
2132    pipe->bind_vertex_elements_state(pipe, ctx->velem_state);
2133    pipe->set_sample_mask(pipe, (1ull << MAX2(1, dstsurf->texture->nr_samples)) - 1);
2134
2135    /* set a framebuffer state */
2136    fb_state.width = dstsurf->width;
2137    fb_state.height = dstsurf->height;
2138    fb_state.nr_cbufs = 1;
2139    fb_state.cbufs[0] = dstsurf;
2140    fb_state.zsbuf = 0;
2141    pipe->set_framebuffer_state(pipe, &fb_state);
2142    pipe->set_sample_mask(pipe, ~0);
2143
2144    blitter_set_common_draw_rect_state(ctx, FALSE, FALSE);
2145    blitter_set_dst_dimensions(ctx, dstsurf->width, dstsurf->height);
2146    blitter->draw_rectangle(blitter, 0, 0, dstsurf->width, dstsurf->height,
2147                            0, 0, NULL);
2148
2149    blitter_restore_vertex_states(ctx);
2150    blitter_restore_fragment_states(ctx);
2151    blitter_restore_fb_state(ctx);
2152    blitter_restore_render_cond(ctx);
2153    blitter_unset_running_flag(ctx);
2154 }