src/gallium/drivers/r600/r600_blit.c

   1 /*
   2  * Copyright 2010 Jerome Glisse <glisse@freedesktop.org>
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * on the rights to use, copy, modify, merge, publish, distribute, sub
   8  * license, and/or sell copies of the Software, and to permit persons to whom
   9  * the Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  18  * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
  19  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  20  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  21  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  22  */
  23 #include "r600_pipe.h"
  24 #include "util/u_surface.h"
  25 #include "util/u_blitter.h"
  26 #include "util/u_format.h"
  27
  28 enum r600_blitter_op /* bitmask */
  29 {
  30         R600_SAVE_FRAGMENT_STATE = 1,
  31         R600_SAVE_TEXTURES       = 2,
  32         R600_SAVE_FRAMEBUFFER    = 4,
  33         R600_DISABLE_RENDER_COND = 8,
  34
  35         R600_CLEAR         = R600_SAVE_FRAGMENT_STATE,
  36
  37         R600_CLEAR_SURFACE = R600_SAVE_FRAGMENT_STATE | R600_SAVE_FRAMEBUFFER,
  38
  39         R600_COPY_BUFFER   = R600_DISABLE_RENDER_COND,
  40
  41         R600_COPY_TEXTURE  = R600_SAVE_FRAGMENT_STATE | R600_SAVE_FRAMEBUFFER | R600_SAVE_TEXTURES |
  42                              R600_DISABLE_RENDER_COND,
  43
  44         R600_DECOMPRESS    = R600_SAVE_FRAGMENT_STATE | R600_SAVE_FRAMEBUFFER | R600_DISABLE_RENDER_COND,
  45
  46         R600_COLOR_RESOLVE = R600_SAVE_FRAGMENT_STATE | R600_SAVE_FRAMEBUFFER | R600_DISABLE_RENDER_COND
  47 };
  48
  49 static void r600_blitter_begin(struct pipe_context *ctx, enum r600_blitter_op op)
  50 {
  51         struct r600_context *rctx = (struct r600_context *)ctx;
  52
  53         r600_suspend_nontimer_queries(rctx);
  54
  55         util_blitter_save_vertex_buffers(rctx->blitter,
  56                                          util_last_bit(rctx->vertex_buffer_state.enabled_mask),
  57                                          rctx->vertex_buffer_state.vb);
  58         util_blitter_save_vertex_elements(rctx->blitter, rctx->vertex_elements);
  59         util_blitter_save_vertex_shader(rctx->blitter, rctx->vs_shader);
  60         util_blitter_save_so_targets(rctx->blitter, rctx->num_so_targets,
  61                                      (struct pipe_stream_output_target**)rctx->so_targets);
  62         util_blitter_save_rasterizer(rctx->blitter, rctx->states[R600_PIPE_STATE_RASTERIZER]);
  63
  64         if (op & R600_SAVE_FRAGMENT_STATE) {
  65                 if (rctx->states[R600_PIPE_STATE_VIEWPORT]) {
  66                         util_blitter_save_viewport(rctx->blitter, &rctx->viewport);
  67                 }
  68                 util_blitter_save_fragment_shader(rctx->blitter, rctx->ps_shader);
  69                 util_blitter_save_blend(rctx->blitter, rctx->states[R600_PIPE_STATE_BLEND]);
  70                 util_blitter_save_depth_stencil_alpha(rctx->blitter, rctx->states[R600_PIPE_STATE_DSA]);
  71                 util_blitter_save_stencil_ref(rctx->blitter, &rctx->stencil_ref.pipe_state);
  72                 util_blitter_save_sample_mask(rctx->blitter, rctx->sample_mask.sample_mask);
  73         }
  74
  75         if (op & R600_SAVE_FRAMEBUFFER)
  76                 util_blitter_save_framebuffer(rctx->blitter, &rctx->framebuffer);
  77
  78         if (op & R600_SAVE_TEXTURES) {
  79                 util_blitter_save_fragment_sampler_states(
  80                         rctx->blitter, util_last_bit(rctx->samplers[PIPE_SHADER_FRAGMENT].states.enabled_mask),
  81                         (void**)rctx->samplers[PIPE_SHADER_FRAGMENT].states.states);
  82
  83                 util_blitter_save_fragment_sampler_views(
  84                         rctx->blitter, util_last_bit(rctx->samplers[PIPE_SHADER_FRAGMENT].views.enabled_mask),
  85                         (struct pipe_sampler_view**)rctx->samplers[PIPE_SHADER_FRAGMENT].views.views);
  86         }
  87
  88         if ((op & R600_DISABLE_RENDER_COND) && rctx->current_render_cond) {
  89                 rctx->saved_render_cond = rctx->current_render_cond;
  90                 rctx->saved_render_cond_mode = rctx->current_render_cond_mode;
  91                 rctx->context.render_condition(&rctx->context, NULL, 0);
  92         }
  93
  94 }
  95
  96 static void r600_blitter_end(struct pipe_context *ctx)
  97 {
  98         struct r600_context *rctx = (struct r600_context *)ctx;
  99         if (rctx->saved_render_cond) {
 100                 rctx->context.render_condition(&rctx->context,
 101                                                rctx->saved_render_cond,
 102                                                rctx->saved_render_cond_mode);
 103                 rctx->saved_render_cond = NULL;
 104         }
 105         r600_resume_nontimer_queries(rctx);
 106 }
 107
 108 static unsigned u_max_layer(struct pipe_resource *r, unsigned level)
 109 {
 110         switch (r->target) {
 111         case PIPE_TEXTURE_CUBE:
 112                 return 6 - 1;
 113         case PIPE_TEXTURE_3D:
 114                 return u_minify(r->depth0, level) - 1;
 115         case PIPE_TEXTURE_1D_ARRAY:
 116         case PIPE_TEXTURE_2D_ARRAY:
 117                 return r->array_size - 1;
 118         default:
 119                 return 0;
 120         }
 121 }
 122
 123 static unsigned u_max_sample(struct pipe_resource *r)
 124 {
 125         return r->nr_samples ? r->nr_samples - 1 : 0;
 126 }
 127
 128 void r600_blit_decompress_depth(struct pipe_context *ctx,
 129                 struct r600_texture *texture,
 130                 struct r600_texture *staging,
 131                 unsigned first_level, unsigned last_level,
 132                 unsigned first_layer, unsigned last_layer,
 133                 unsigned first_sample, unsigned last_sample)
 134 {
 135         struct r600_context *rctx = (struct r600_context *)ctx;
 136         unsigned layer, level, sample, checked_last_layer, max_layer, max_sample;
 137         struct r600_texture *flushed_depth_texture = staging ?
 138                         staging : texture->flushed_depth_texture;
 139         const struct util_format_description *desc =
 140                 util_format_description(texture->resource.b.b.format);
 141         float depth;
 142
 143         if (!staging && !texture->dirty_level_mask)
 144                 return;
 145
 146         max_sample = u_max_sample(&texture->resource.b.b);
 147
 148         /* XXX Decompressing MSAA depth textures is broken on R6xx.
 149          * There is also a hardlock if CMASK and FMASK are not present.
 150          * Just skip this until we find out how to fix it. */
 151         if (rctx->chip_class == R600 && max_sample > 0) {
 152                 texture->dirty_level_mask = 0;
 153                 return;
 154         }
 155
 156         if (rctx->family == CHIP_RV610 || rctx->family == CHIP_RV630 ||
 157             rctx->family == CHIP_RV620 || rctx->family == CHIP_RV635)
 158                 depth = 0.0f;
 159         else
 160                 depth = 1.0f;
 161
 162         /* Enable decompression in DB_RENDER_CONTROL */
 163         rctx->db_misc_state.flush_depthstencil_through_cb = true;
 164         rctx->db_misc_state.copy_depth = util_format_has_depth(desc);
 165         rctx->db_misc_state.copy_stencil = util_format_has_stencil(desc);
 166         rctx->db_misc_state.copy_sample = first_sample;
 167         r600_atom_dirty(rctx, &rctx->db_misc_state.atom);
 168
 169
 170         for (level = first_level; level <= last_level; level++) {
 171                 if (!staging && !(texture->dirty_level_mask & (1 << level)))
 172                         continue;
 173
 174                 /* The smaller the mipmap level, the less layers there are
 175                  * as far as 3D textures are concerned. */
 176                 max_layer = u_max_layer(&texture->resource.b.b, level);
 177                 checked_last_layer = last_layer < max_layer ? last_layer : max_layer;
 178
 179                 for (layer = first_layer; layer <= checked_last_layer; layer++) {
 180                         for (sample = first_sample; sample <= last_sample; sample++) {
 181                                 struct pipe_surface *zsurf, *cbsurf, surf_tmpl;
 182
 183                                 if (sample != rctx->db_misc_state.copy_sample) {
 184                                         rctx->db_misc_state.copy_sample = sample;
 185                                         r600_atom_dirty(rctx, &rctx->db_misc_state.atom);
 186                                 }
 187
 188                                 surf_tmpl.format = texture->resource.b.b.format;
 189                                 surf_tmpl.u.tex.level = level;
 190                                 surf_tmpl.u.tex.first_layer = layer;
 191                                 surf_tmpl.u.tex.last_layer = layer;
 192                                 surf_tmpl.usage = PIPE_BIND_DEPTH_STENCIL;
 193
 194                                 zsurf = ctx->create_surface(ctx, &texture->resource.b.b, &surf_tmpl);
 195
 196                                 surf_tmpl.format = flushed_depth_texture->resource.b.b.format;
 197                                 surf_tmpl.u.tex.level = level;
 198                                 surf_tmpl.u.tex.first_layer = layer;
 199                                 surf_tmpl.u.tex.last_layer = layer;
 200                                 surf_tmpl.usage = PIPE_BIND_RENDER_TARGET;
 201                                 cbsurf = ctx->create_surface(ctx,
 202                                                 &flushed_depth_texture->resource.b.b, &surf_tmpl);
 203
 204                                 r600_blitter_begin(ctx, R600_DECOMPRESS);
 205                                 util_blitter_custom_depth_stencil(rctx->blitter, zsurf, cbsurf, 1 << sample,
 206                                                                   rctx->custom_dsa_flush, depth);
 207                                 r600_blitter_end(ctx);
 208
 209                                 pipe_surface_reference(&zsurf, NULL);
 210                                 pipe_surface_reference(&cbsurf, NULL);
 211                         }
 212                 }
 213
 214                 /* The texture will always be dirty if some layers or samples aren't flushed.
 215                  * I don't think this case occurs often though. */
 216                 if (!staging &&
 217                     first_layer == 0 && last_layer == max_layer &&
 218                     first_sample == 0 && last_sample == max_sample) {
 219                         texture->dirty_level_mask &= ~(1 << level);
 220                 }
 221         }
 222
 223         /* reenable compression in DB_RENDER_CONTROL */
 224         rctx->db_misc_state.flush_depthstencil_through_cb = false;
 225         r600_atom_dirty(rctx, &rctx->db_misc_state.atom);
 226 }
 227
 228 void r600_decompress_depth_textures(struct r600_context *rctx,
 229                                struct r600_samplerview_state *textures)
 230 {
 231         unsigned i;
 232         unsigned depth_texture_mask = textures->compressed_depthtex_mask;
 233
 234         while (depth_texture_mask) {
 235                 struct pipe_sampler_view *view;
 236                 struct r600_texture *tex;
 237
 238                 i = u_bit_scan(&depth_texture_mask);
 239
 240                 view = &textures->views[i]->base;
 241                 assert(view);
 242
 243                 tex = (struct r600_texture *)view->texture;
 244                 assert(tex->is_depth && !tex->is_flushing_texture);
 245
 246                 r600_blit_decompress_depth(&rctx->context, tex, NULL,
 247                                            view->u.tex.first_level, view->u.tex.last_level,
 248                                            0, u_max_layer(&tex->resource.b.b, view->u.tex.first_level),
 249                                            0, u_max_sample(&tex->resource.b.b));
 250         }
 251 }
 252
 253 static void r600_blit_decompress_color(struct pipe_context *ctx,
 254                 struct r600_texture *rtex,
 255                 unsigned first_level, unsigned last_level,
 256                 unsigned first_layer, unsigned last_layer)
 257 {
 258         struct r600_context *rctx = (struct r600_context *)ctx;
 259         unsigned layer, level, checked_last_layer, max_layer;
 260
 261         assert(rctx->chip_class != CAYMAN);
 262
 263         if (!rtex->dirty_level_mask)
 264                 return;
 265
 266         for (level = first_level; level <= last_level; level++) {
 267                 if (!(rtex->dirty_level_mask & (1 << level)))
 268                         continue;
 269
 270                 /* The smaller the mipmap level, the less layers there are
 271                  * as far as 3D textures are concerned. */
 272                 max_layer = u_max_layer(&rtex->resource.b.b, level);
 273                 checked_last_layer = last_layer < max_layer ? last_layer : max_layer;
 274
 275                 for (layer = first_layer; layer <= checked_last_layer; layer++) {
 276                         struct pipe_surface *cbsurf, surf_tmpl;
 277
 278                         surf_tmpl.format = rtex->resource.b.b.format;
 279                         surf_tmpl.u.tex.level = level;
 280                         surf_tmpl.u.tex.first_layer = layer;
 281                         surf_tmpl.u.tex.last_layer = layer;
 282                         surf_tmpl.usage = PIPE_BIND_RENDER_TARGET;
 283                         cbsurf = ctx->create_surface(ctx, &rtex->resource.b.b, &surf_tmpl);
 284
 285                         r600_blitter_begin(ctx, R600_DECOMPRESS);
 286                         util_blitter_custom_color(rctx->blitter, cbsurf,
 287                                                   rctx->custom_blend_decompress);
 288                         r600_blitter_end(ctx);
 289
 290                         pipe_surface_reference(&cbsurf, NULL);
 291                 }
 292
 293                 /* The texture will always be dirty if some layers or samples aren't flushed.
 294                  * I don't think this case occurs often though. */
 295                 if (first_layer == 0 && last_layer == max_layer) {
 296                         rtex->dirty_level_mask &= ~(1 << level);
 297                 }
 298         }
 299 }
 300
 301 void r600_decompress_color_textures(struct r600_context *rctx,
 302                                     struct r600_samplerview_state *textures)
 303 {
 304         unsigned i;
 305         unsigned mask = textures->compressed_colortex_mask;
 306
 307         /* Cayman cannot decompress an MSAA colorbuffer,
 308          * but it can read it compressed, so skip this. */
 309         assert(rctx->chip_class != CAYMAN);
 310         if (rctx->chip_class == CAYMAN) {
 311                 return;
 312         }
 313
 314         while (mask) {
 315                 struct pipe_sampler_view *view;
 316                 struct r600_texture *tex;
 317
 318                 i = u_bit_scan(&mask);
 319
 320                 view = &textures->views[i]->base;
 321                 assert(view);
 322
 323                 tex = (struct r600_texture *)view->texture;
 324                 assert(tex->cmask_size && tex->fmask_size);
 325
 326                 r600_blit_decompress_color(&rctx->context, tex,
 327                                            view->u.tex.first_level, view->u.tex.last_level,
 328                                            0, u_max_layer(&tex->resource.b.b, view->u.tex.first_level));
 329         }
 330 }
 331
 332 static void r600_copy_first_sample(struct pipe_context *ctx,
 333                                    const struct pipe_resolve_info *info)
 334 {
 335         struct r600_context *rctx = (struct r600_context *)ctx;
 336         struct r600_texture *rsrc = (struct r600_texture*)info->src.res;
 337         struct pipe_surface *dst_view, dst_templ;
 338         struct pipe_sampler_view src_templ, *src_view;
 339         struct pipe_box box;
 340
 341         if (rsrc->is_depth && !rsrc->is_flushing_texture) {
 342                 if (!r600_init_flushed_depth_texture(ctx, info->src.res, NULL))
 343                         return; /* error */
 344
 345                 /* Decompress the first sample only. */
 346                 r600_blit_decompress_depth(ctx, rsrc, NULL,
 347                                            0, 0,
 348                                            info->src.layer, info->src.layer,
 349                                            0, 0);
 350         }
 351         if (rctx->chip_class != CAYMAN && rsrc->fmask_size && rsrc->cmask_size) {
 352                 r600_blit_decompress_color(ctx, rsrc,
 353                                            0, 0,
 354                                            info->src.layer, info->src.layer);
 355         }
 356
 357         /* this is correct for upside-down blits too */
 358         u_box_2d(info->src.x0,
 359                  info->src.y0,
 360                  info->src.x1 - info->src.x0,
 361                  info->src.y1 - info->src.y0, &box);
 362
 363         /* Initialize the surface. */
 364         util_blitter_default_dst_texture(&dst_templ, info->dst.res,
 365                                          info->dst.level, info->dst.layer, &box);
 366         dst_view = ctx->create_surface(ctx, info->dst.res, &dst_templ);
 367
 368         /* Initialize the sampler view. */
 369         util_blitter_default_src_texture(&src_templ, info->src.res, 0);
 370         src_view = ctx->create_sampler_view(ctx, info->src.res, &src_templ);
 371
 372         /* Copy the first sample into dst. */
 373         r600_blitter_begin(ctx, R600_COPY_TEXTURE);
 374         util_blitter_copy_texture_view(rctx->blitter, dst_view, ~0, info->dst.x0,
 375                                        info->dst.y0, src_view, 0, &box,
 376                                        info->src.res->width0, info->src.res->height0,
 377                                        info->mask);
 378         r600_blitter_end(ctx);
 379
 380         pipe_surface_reference(&dst_view, NULL);
 381         pipe_sampler_view_reference(&src_view, NULL);
 382 }
 383
 384 static boolean is_simple_resolve(const struct pipe_resolve_info *info)
 385 {
 386    unsigned dst_width = u_minify(info->dst.res->width0, info->dst.level);
 387    unsigned dst_height = u_minify(info->dst.res->height0, info->dst.level);
 388
 389    return info->dst.res->format == info->src.res->format &&
 390           dst_width == info->src.res->width0 &&
 391           dst_height == info->src.res->height0 &&
 392           info->dst.x0 == 0 &&
 393           info->dst.y0 == 0 &&
 394           info->dst.x1 == dst_width &&
 395           info->dst.y1 == dst_height &&
 396           info->src.x0 == 0 &&
 397           info->src.y0 == 0 &&
 398           info->src.x1 == dst_width &&
 399           info->src.y1 == dst_height;
 400 }
 401
 402 static void r600_color_resolve(struct pipe_context *ctx,
 403                                const struct pipe_resolve_info *info)
 404 {
 405         struct r600_context *rctx = (struct r600_context *)ctx;
 406         struct pipe_screen *screen = ctx->screen;
 407         struct pipe_resource *tmp, templ;
 408         struct pipe_box box;
 409         unsigned sample_mask =
 410                 rctx->chip_class == CAYMAN ? ~0 : ((1ull << MAX2(1, info->src.res->nr_samples)) - 1);
 411
 412         assert((info->mask & PIPE_MASK_RGBA) == PIPE_MASK_RGBA);
 413
 414         if (is_simple_resolve(info)) {
 415                 r600_blitter_begin(ctx, R600_COLOR_RESOLVE);
 416                 util_blitter_custom_resolve_color(rctx->blitter,
 417                                                   info->dst.res, info->dst.level, info->dst.layer,
 418                                                   info->src.res, info->src.layer,
 419                                                   sample_mask, rctx->custom_blend_resolve);
 420                 r600_blitter_end(ctx);
 421                 return;
 422         }
 423
 424         /* resolve into a temporary texture, then blit */
 425         templ.target = PIPE_TEXTURE_2D;
 426         templ.format = info->src.res->format;
 427         templ.width0 = info->src.res->width0;
 428         templ.height0 = info->src.res->height0;
 429         templ.depth0 = 1;
 430         templ.array_size = 1;
 431         templ.last_level = 0;
 432         templ.nr_samples = 0;
 433         templ.usage = PIPE_USAGE_STATIC;
 434         templ.bind = PIPE_BIND_RENDER_TARGET | PIPE_BIND_SAMPLER_VIEW;
 435         templ.flags = 0;
 436
 437         tmp = screen->resource_create(screen, &templ);
 438
 439         /* XXX use scissor, so that only the needed part of the resource is resolved */
 440         r600_blitter_begin(ctx, R600_COLOR_RESOLVE);
 441         util_blitter_custom_resolve_color(rctx->blitter,
 442                                           tmp, 0, 0,
 443                                           info->src.res, info->src.layer,
 444                                           sample_mask, rctx->custom_blend_resolve);
 445         r600_blitter_end(ctx);
 446
 447         /* this is correct for upside-down blits too */
 448         u_box_2d(info->src.x0,
 449                  info->src.y0,
 450                  info->src.x1 - info->src.x0,
 451                  info->src.y1 - info->src.y0, &box);
 452
 453         r600_blitter_begin(ctx, R600_COPY_TEXTURE);
 454         util_blitter_copy_texture(rctx->blitter, info->dst.res, info->dst.level,
 455                                   ~0, info->dst.x0, info->dst.y0, info->dst.layer,
 456                                   tmp, 0, 0, &box);
 457         r600_blitter_end(ctx);
 458
 459         pipe_resource_reference(&tmp, NULL);
 460 }
 461
 462 static void r600_resource_resolve(struct pipe_context *ctx,
 463                                   const struct pipe_resolve_info *info)
 464 {
 465         /* make sure we're doing a resolve operation */
 466         assert(info->src.res->nr_samples > 1);
 467         assert(info->dst.res->nr_samples <= 1);
 468
 469         /* limitations of multisample resources */
 470         assert(info->src.res->last_level == 0);
 471         assert(info->src.res->target == PIPE_TEXTURE_2D ||
 472                info->src.res->target == PIPE_TEXTURE_2D_ARRAY);
 473
 474         /* check if the resolve box is valid */
 475         assert(info->dst.x0 < info->dst.x1);
 476         assert(info->dst.y0 < info->dst.y1);
 477
 478         /* scaled resolve isn't allowed */
 479         assert(abs(info->dst.x0 - info->dst.x1) ==
 480                abs(info->src.x0 - info->src.x1));
 481         assert(abs(info->dst.y0 - info->dst.y1) ==
 482                abs(info->src.y0 - info->src.y1));
 483
 484         if ((info->mask & PIPE_MASK_ZS) ||
 485             util_format_is_pure_integer(info->src.res->format)) {
 486                 r600_copy_first_sample(ctx, info);
 487         } else {
 488                 r600_color_resolve(ctx, info);
 489         }
 490 }
 491
 492 static void r600_clear(struct pipe_context *ctx, unsigned buffers,
 493                        const union pipe_color_union *color,
 494                        double depth, unsigned stencil)
 495 {
 496         struct r600_context *rctx = (struct r600_context *)ctx;
 497         struct pipe_framebuffer_state *fb = &rctx->framebuffer;
 498
 499         r600_blitter_begin(ctx, R600_CLEAR);
 500         util_blitter_clear(rctx->blitter, fb->width, fb->height,
 501                            fb->nr_cbufs, buffers, fb->nr_cbufs ? fb->cbufs[0]->format : PIPE_FORMAT_NONE,
 502                            color, depth, stencil);
 503         r600_blitter_end(ctx);
 504 }
 505
 506 static void r600_clear_render_target(struct pipe_context *ctx,
 507                                      struct pipe_surface *dst,
 508                                      const union pipe_color_union *color,
 509                                      unsigned dstx, unsigned dsty,
 510                                      unsigned width, unsigned height)
 511 {
 512         struct r600_context *rctx = (struct r600_context *)ctx;
 513
 514         r600_blitter_begin(ctx, R600_CLEAR_SURFACE);
 515         util_blitter_clear_render_target(rctx->blitter, dst, color,
 516                                          dstx, dsty, width, height);
 517         r600_blitter_end(ctx);
 518 }
 519
 520 static void r600_clear_depth_stencil(struct pipe_context *ctx,
 521                                      struct pipe_surface *dst,
 522                                      unsigned clear_flags,
 523                                      double depth,
 524                                      unsigned stencil,
 525                                      unsigned dstx, unsigned dsty,
 526                                      unsigned width, unsigned height)
 527 {
 528         struct r600_context *rctx = (struct r600_context *)ctx;
 529
 530         r600_blitter_begin(ctx, R600_CLEAR_SURFACE);
 531         util_blitter_clear_depth_stencil(rctx->blitter, dst, clear_flags, depth, stencil,
 532                                          dstx, dsty, width, height);
 533         r600_blitter_end(ctx);
 534 }
 535
 536 void r600_copy_buffer(struct pipe_context *ctx, struct
 537                       pipe_resource *dst, unsigned dstx,
 538                       struct pipe_resource *src, const struct pipe_box *src_box)
 539 {
 540         struct r600_context *rctx = (struct r600_context*)ctx;
 541
 542         if (rctx->screen->has_streamout &&
 543             /* Require dword alignment. */
 544             dstx % 4 == 0 && src_box->x % 4 == 0 && src_box->width % 4 == 0) {
 545                 r600_blitter_begin(ctx, R600_COPY_BUFFER);
 546                 util_blitter_copy_buffer(rctx->blitter, dst, dstx, src, src_box->x, src_box->width);
 547                 r600_blitter_end(ctx);
 548         } else {
 549                 util_resource_copy_region(ctx, dst, 0, dstx, 0, 0, src, 0, src_box);
 550         }
 551 }
 552
 553 struct texture_orig_info {
 554         unsigned format;
 555         unsigned width0;
 556         unsigned height0;
 557         unsigned npix_x;
 558         unsigned npix_y;
 559         unsigned npix0_x;
 560         unsigned npix0_y;
 561 };
 562
 563 static void r600_compressed_to_blittable(struct pipe_resource *tex,
 564                                    unsigned level,
 565                                    struct texture_orig_info *orig)
 566 {
 567         struct r600_texture *rtex = (struct r600_texture*)tex;
 568         unsigned pixsize = util_format_get_blocksize(rtex->resource.b.b.format);
 569         int new_format;
 570         int new_height, new_width;
 571
 572         orig->format = tex->format;
 573         orig->width0 = tex->width0;
 574         orig->height0 = tex->height0;
 575         orig->npix0_x = rtex->surface.level[0].npix_x;
 576         orig->npix0_y = rtex->surface.level[0].npix_y;
 577         orig->npix_x = rtex->surface.level[level].npix_x;
 578         orig->npix_y = rtex->surface.level[level].npix_y;
 579
 580         if (pixsize == 8)
 581                 new_format = PIPE_FORMAT_R16G16B16A16_UINT; /* 64-bit block */
 582         else
 583                 new_format = PIPE_FORMAT_R32G32B32A32_UINT; /* 128-bit block */
 584
 585         new_width = util_format_get_nblocksx(tex->format, orig->width0);
 586         new_height = util_format_get_nblocksy(tex->format, orig->height0);
 587
 588         tex->width0 = new_width;
 589         tex->height0 = new_height;
 590         tex->format = new_format;
 591         rtex->surface.level[0].npix_x = util_format_get_nblocksx(orig->format, orig->npix0_x);
 592         rtex->surface.level[0].npix_y = util_format_get_nblocksy(orig->format, orig->npix0_y);
 593         rtex->surface.level[level].npix_x = util_format_get_nblocksx(orig->format, orig->npix_x);
 594         rtex->surface.level[level].npix_y = util_format_get_nblocksy(orig->format, orig->npix_y);
 595 }
 596
 597 static void r600_subsampled_2x1_32bpp_to_blittable(struct pipe_resource *tex,
 598                                                    unsigned level,
 599                                                    struct texture_orig_info *orig)
 600 {
 601         struct r600_texture *rtex = (struct r600_texture*)tex;
 602
 603         orig->format = tex->format;
 604         orig->width0 = tex->width0;
 605         orig->height0 = tex->height0;
 606         orig->npix0_x = rtex->surface.level[0].npix_x;
 607         orig->npix0_y = rtex->surface.level[0].npix_y;
 608         orig->npix_x = rtex->surface.level[level].npix_x;
 609         orig->npix_y = rtex->surface.level[level].npix_y;
 610
 611         tex->width0 = util_format_get_nblocksx(orig->format, orig->width0);
 612         tex->format = PIPE_FORMAT_R8G8B8A8_UINT;
 613         rtex->surface.level[0].npix_x = util_format_get_nblocksx(orig->format, orig->npix0_x);
 614         rtex->surface.level[level].npix_x = util_format_get_nblocksx(orig->format, orig->npix_x);
 615 }
 616
 617 static void r600_change_format(struct pipe_resource *tex,
 618                                unsigned level,
 619                                struct texture_orig_info *orig,
 620                                enum pipe_format format)
 621 {
 622         struct r600_texture *rtex = (struct r600_texture*)tex;
 623
 624         orig->format = tex->format;
 625         orig->width0 = tex->width0;
 626         orig->height0 = tex->height0;
 627         orig->npix0_x = rtex->surface.level[0].npix_x;
 628         orig->npix0_y = rtex->surface.level[0].npix_y;
 629         orig->npix_x = rtex->surface.level[level].npix_x;
 630         orig->npix_y = rtex->surface.level[level].npix_y;
 631
 632         tex->format = format;
 633 }
 634
 635 static void r600_reset_blittable_to_orig(struct pipe_resource *tex,
 636                                          unsigned level,
 637                                          struct texture_orig_info *orig)
 638 {
 639         struct r600_texture *rtex = (struct r600_texture*)tex;
 640
 641         tex->format = orig->format;
 642         tex->width0 = orig->width0;
 643         tex->height0 = orig->height0;
 644         rtex->surface.level[0].npix_x = orig->npix0_x;
 645         rtex->surface.level[0].npix_y = orig->npix0_y;
 646         rtex->surface.level[level].npix_x = orig->npix_x;
 647         rtex->surface.level[level].npix_y = orig->npix_y;
 648 }
 649
 650 static bool util_format_is_subsampled_2x1_32bpp(enum pipe_format format)
 651 {
 652         const struct util_format_description *desc = util_format_description(format);
 653
 654         return desc->layout == UTIL_FORMAT_LAYOUT_SUBSAMPLED &&
 655                desc->block.width == 2 &&
 656                desc->block.height == 1 &&
 657                desc->block.bits == 32;
 658 }
 659
 660 static void r600_resource_copy_region(struct pipe_context *ctx,
 661                                       struct pipe_resource *dst,
 662                                       unsigned dst_level,
 663                                       unsigned dstx, unsigned dsty, unsigned dstz,
 664                                       struct pipe_resource *src,
 665                                       unsigned src_level,
 666                                       const struct pipe_box *src_box)
 667 {
 668         struct r600_context *rctx = (struct r600_context *)ctx;
 669         struct r600_texture *rsrc = (struct r600_texture*)src;
 670         struct texture_orig_info orig_info[2];
 671         struct pipe_box sbox;
 672         const struct pipe_box *psbox = src_box;
 673         boolean restore_orig[2];
 674         unsigned last_sample, i;
 675
 676         memset(orig_info, 0, sizeof(orig_info));
 677
 678         /* Handle buffers first. */
 679         if (dst->target == PIPE_BUFFER && src->target == PIPE_BUFFER) {
 680                 r600_copy_buffer(ctx, dst, dstx, src, src_box);
 681                 return;
 682         }
 683
 684         assert(u_max_sample(dst) == u_max_sample(src));
 685         last_sample = u_max_sample(dst);
 686
 687         /* This must be done before entering u_blitter to avoid recursion. */
 688         if (rsrc->is_depth && !rsrc->is_flushing_texture) {
 689                 if (!r600_init_flushed_depth_texture(ctx, src, NULL))
 690                         return; /* error */
 691
 692                 r600_blit_decompress_depth(ctx, rsrc, NULL,
 693                                            src_level, src_level,
 694                                            src_box->z, src_box->z + src_box->depth - 1,
 695                                            0, u_max_sample(src));
 696         }
 697         if (rctx->chip_class != CAYMAN && rsrc->fmask_size && rsrc->cmask_size) {
 698                 r600_blit_decompress_color(ctx, rsrc, src_level, src_level,
 699                                            src_box->z, src_box->z + src_box->depth - 1);
 700         }
 701
 702         restore_orig[0] = restore_orig[1] = FALSE;
 703
 704         if (util_format_is_compressed(src->format) &&
 705             util_format_is_compressed(dst->format)) {
 706                 r600_compressed_to_blittable(src, src_level, &orig_info[0]);
 707                 restore_orig[0] = TRUE;
 708                 sbox.x = util_format_get_nblocksx(orig_info[0].format, src_box->x);
 709                 sbox.y = util_format_get_nblocksy(orig_info[0].format, src_box->y);
 710                 sbox.z = src_box->z;
 711                 sbox.width = util_format_get_nblocksx(orig_info[0].format, src_box->width);
 712                 sbox.height = util_format_get_nblocksy(orig_info[0].format, src_box->height);
 713                 sbox.depth = src_box->depth;
 714                 psbox = &sbox;
 715
 716                 r600_compressed_to_blittable(dst, dst_level, &orig_info[1]);
 717                 restore_orig[1] = TRUE;
 718                 /* translate the dst box as well */
 719                 dstx = util_format_get_nblocksx(orig_info[1].format, dstx);
 720                 dsty = util_format_get_nblocksy(orig_info[1].format, dsty);
 721         } else if (!util_blitter_is_copy_supported(rctx->blitter, dst, src,
 722                                                    PIPE_MASK_RGBAZS)) {
 723                 if (util_format_is_subsampled_2x1_32bpp(src->format) &&
 724                     util_format_is_subsampled_2x1_32bpp(dst->format)) {
 725                         r600_subsampled_2x1_32bpp_to_blittable(src, src_level, &orig_info[0]);
 726                         r600_subsampled_2x1_32bpp_to_blittable(dst, dst_level, &orig_info[1]);
 727
 728                         sbox = *src_box;
 729                         sbox.x = util_format_get_nblocksx(orig_info[0].format, src_box->x);
 730                         sbox.width = util_format_get_nblocksx(orig_info[0].format, src_box->width);
 731                         psbox = &sbox;
 732
 733                         dstx = util_format_get_nblocksx(orig_info[1].format, dstx);
 734                 } else {
 735                         unsigned blocksize = util_format_get_blocksize(src->format);
 736
 737                         switch (blocksize) {
 738                         case 1:
 739                                 r600_change_format(src, src_level, &orig_info[0],
 740                                                    PIPE_FORMAT_R8_UNORM);
 741                                 r600_change_format(dst, dst_level, &orig_info[1],
 742                                                    PIPE_FORMAT_R8_UNORM);
 743                                 break;
 744                         case 4:
 745                                 r600_change_format(src, src_level, &orig_info[0],
 746                                                    PIPE_FORMAT_R8G8B8A8_UNORM);
 747                                 r600_change_format(dst, dst_level, &orig_info[1],
 748                                                    PIPE_FORMAT_R8G8B8A8_UNORM);
 749                                 break;
 750                         default:
 751                                 fprintf(stderr, "Unhandled format %s with blocksize %u\n",
 752                                         util_format_short_name(src->format), blocksize);
 753                                 assert(0);
 754                         }
 755                 }
 756                 restore_orig[0] = TRUE;
 757                 restore_orig[1] = TRUE;
 758         }
 759
 760         /* XXX Properly implement multisample textures on Cayman. In the meantime,
 761          * copy only the first sample (which is the only one that doesn't return garbage). */
 762         if (rctx->chip_class == CAYMAN) {
 763                 r600_blitter_begin(ctx, R600_COPY_TEXTURE);
 764                 util_blitter_copy_texture(rctx->blitter, dst, dst_level, ~0, dstx, dsty, dstz,
 765                                           src, src_level, 0, psbox);
 766                 r600_blitter_end(ctx);
 767         } else {
 768                 for (i = 0; i <= last_sample; i++) {
 769                         r600_blitter_begin(ctx, R600_COPY_TEXTURE);
 770                         util_blitter_copy_texture(rctx->blitter, dst, dst_level, 1 << i, dstx, dsty, dstz,
 771                                                   src, src_level, i, psbox);
 772                         r600_blitter_end(ctx);
 773                 }
 774         }
 775
 776         if (restore_orig[0])
 777                 r600_reset_blittable_to_orig(src, src_level, &orig_info[0]);
 778
 779         if (restore_orig[1])
 780                 r600_reset_blittable_to_orig(dst, dst_level, &orig_info[1]);
 781 }
 782
 783 void r600_init_blit_functions(struct r600_context *rctx)
 784 {
 785         rctx->context.clear = r600_clear;
 786         rctx->context.clear_render_target = r600_clear_render_target;
 787         rctx->context.clear_depth_stencil = r600_clear_depth_stencil;
 788         rctx->context.resource_copy_region = r600_resource_copy_region;
 789         rctx->context.resource_resolve = r600_resource_resolve;
 790 }