src/gallium/drivers/freedreno/freedreno_resource.c

   1 /* -*- mode: C; c-file-style: "k&r"; tab-width 4; indent-tabs-mode: t; -*- */
   2
   3 /*
   4  * Copyright (C) 2012 Rob Clark <robclark@freedesktop.org>
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the next
  14  * paragraph) shall be included in all copies or substantial portions of the
  15  * Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  20  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  22  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  23  * SOFTWARE.
  24  *
  25  * Authors:
  26  *    Rob Clark <robclark@freedesktop.org>
  27  */
  28
  29 #include "util/u_format.h"
  30 #include "util/u_format_rgtc.h"
  31 #include "util/u_format_zs.h"
  32 #include "util/u_inlines.h"
  33 #include "util/u_transfer.h"
  34 #include "util/u_string.h"
  35 #include "util/u_surface.h"
  36
  37 #include "freedreno_resource.h"
  38 #include "freedreno_screen.h"
  39 #include "freedreno_surface.h"
  40 #include "freedreno_context.h"
  41 #include "freedreno_query_hw.h"
  42 #include "freedreno_util.h"
  43
  44 #include <errno.h>
  45
  46
  47 static bool
  48 pending(struct fd_resource *rsc, enum fd_resource_status status)
  49 {
  50         return (rsc->status & status) ||
  51                 (rsc->stencil && (rsc->stencil->status & status));
  52 }
  53
  54 static void
  55 fd_invalidate_resource(struct fd_context *ctx, struct pipe_resource *prsc)
  56 {
  57         int i;
  58
  59         /* Go through the entire state and see if the resource is bound
  60          * anywhere. If it is, mark the relevant state as dirty. This is called on
  61          * realloc_bo.
  62          */
  63
  64         /* Constbufs */
  65         for (i = 1; i < PIPE_MAX_CONSTANT_BUFFERS && !(ctx->dirty & FD_DIRTY_CONSTBUF); i++) {
  66                 if (ctx->constbuf[PIPE_SHADER_VERTEX].cb[i].buffer == prsc)
  67                         ctx->dirty |= FD_DIRTY_CONSTBUF;
  68                 if (ctx->constbuf[PIPE_SHADER_FRAGMENT].cb[i].buffer == prsc)
  69                         ctx->dirty |= FD_DIRTY_CONSTBUF;
  70         }
  71
  72         /* VBOs */
  73         for (i = 0; i < ctx->vtx.vertexbuf.count && !(ctx->dirty & FD_DIRTY_VTXBUF); i++) {
  74                 if (ctx->vtx.vertexbuf.vb[i].buffer == prsc)
  75                         ctx->dirty |= FD_DIRTY_VTXBUF;
  76         }
  77
  78         /* Index buffer */
  79         if (ctx->indexbuf.buffer == prsc)
  80                 ctx->dirty |= FD_DIRTY_INDEXBUF;
  81
  82         /* Textures */
  83         for (i = 0; i < ctx->verttex.num_textures && !(ctx->dirty & FD_DIRTY_VERTTEX); i++) {
  84                 if (ctx->verttex.textures[i] && (ctx->verttex.textures[i]->texture == prsc))
  85                         ctx->dirty |= FD_DIRTY_VERTTEX;
  86         }
  87         for (i = 0; i < ctx->fragtex.num_textures && !(ctx->dirty & FD_DIRTY_FRAGTEX); i++) {
  88                 if (ctx->fragtex.textures[i] && (ctx->fragtex.textures[i]->texture == prsc))
  89                         ctx->dirty |= FD_DIRTY_FRAGTEX;
  90         }
  91 }
  92
  93 static void
  94 realloc_bo(struct fd_resource *rsc, uint32_t size)
  95 {
  96         struct fd_screen *screen = fd_screen(rsc->base.b.screen);
  97         uint32_t flags = DRM_FREEDRENO_GEM_CACHE_WCOMBINE |
  98                         DRM_FREEDRENO_GEM_TYPE_KMEM; /* TODO */
  99
 100         /* if we start using things other than write-combine,
 101          * be sure to check for PIPE_RESOURCE_FLAG_MAP_COHERENT
 102          */
 103
 104         if (rsc->bo)
 105                 fd_bo_del(rsc->bo);
 106
 107         rsc->bo = fd_bo_new(screen->dev, size, flags);
 108         rsc->timestamp = 0;
 109         rsc->status = 0;
 110         rsc->pending_ctx = NULL;
 111         list_delinit(&rsc->list);
 112         util_range_set_empty(&rsc->valid_buffer_range);
 113 }
 114
 115 static unsigned
 116 fd_resource_layer_offset(struct fd_resource *rsc,
 117                                                  struct fd_resource_slice *slice,
 118                                                  unsigned layer)
 119 {
 120         if (rsc->layer_first)
 121                 return layer * rsc->layer_size;
 122         else
 123                 return layer * slice->size0;
 124 }
 125
 126 static void
 127 fd_resource_flush_z32s8(struct fd_transfer *trans, const struct pipe_box *box)
 128 {
 129         struct fd_resource *rsc = fd_resource(trans->base.resource);
 130         struct fd_resource_slice *slice = fd_resource_slice(rsc, trans->base.level);
 131         struct fd_resource_slice *sslice = fd_resource_slice(rsc->stencil, trans->base.level);
 132         enum pipe_format format = trans->base.resource->format;
 133
 134         float *depth = fd_bo_map(rsc->bo) + slice->offset +
 135                 fd_resource_layer_offset(rsc, slice, trans->base.box.z) +
 136                 (trans->base.box.y + box->y) * slice->pitch * 4 + (trans->base.box.x + box->x) * 4;
 137         uint8_t *stencil = fd_bo_map(rsc->stencil->bo) + sslice->offset +
 138                 fd_resource_layer_offset(rsc->stencil, sslice, trans->base.box.z) +
 139                 (trans->base.box.y + box->y) * sslice->pitch + trans->base.box.x + box->x;
 140
 141         if (format != PIPE_FORMAT_X32_S8X24_UINT)
 142                 util_format_z32_float_s8x24_uint_unpack_z_float(
 143                                 depth, slice->pitch * 4,
 144                                 trans->staging, trans->base.stride,
 145                                 box->width, box->height);
 146
 147         util_format_z32_float_s8x24_uint_unpack_s_8uint(
 148                         stencil, sslice->pitch,
 149                         trans->staging, trans->base.stride,
 150                         box->width, box->height);
 151 }
 152
 153 static void
 154 fd_resource_flush_rgtc(struct fd_transfer *trans, const struct pipe_box *box)
 155 {
 156         struct fd_resource *rsc = fd_resource(trans->base.resource);
 157         struct fd_resource_slice *slice = fd_resource_slice(rsc, trans->base.level);
 158         enum pipe_format format = trans->base.resource->format;
 159
 160         uint8_t *data = fd_bo_map(rsc->bo) + slice->offset +
 161                 fd_resource_layer_offset(rsc, slice, trans->base.box.z) +
 162                 ((trans->base.box.y + box->y) * slice->pitch +
 163                  trans->base.box.x + box->x) * rsc->cpp;
 164
 165         uint8_t *source = trans->staging +
 166                 util_format_get_nblocksy(format, box->y) * trans->base.stride +
 167                 util_format_get_stride(format, box->x);
 168
 169         switch (format) {
 170         case PIPE_FORMAT_RGTC1_UNORM:
 171         case PIPE_FORMAT_RGTC1_SNORM:
 172         case PIPE_FORMAT_LATC1_UNORM:
 173         case PIPE_FORMAT_LATC1_SNORM:
 174                 util_format_rgtc1_unorm_unpack_rgba_8unorm(
 175                                 data, slice->pitch * rsc->cpp,
 176                                 source, trans->base.stride,
 177                                 box->width, box->height);
 178                 break;
 179         case PIPE_FORMAT_RGTC2_UNORM:
 180         case PIPE_FORMAT_RGTC2_SNORM:
 181         case PIPE_FORMAT_LATC2_UNORM:
 182         case PIPE_FORMAT_LATC2_SNORM:
 183                 util_format_rgtc2_unorm_unpack_rgba_8unorm(
 184                                 data, slice->pitch * rsc->cpp,
 185                                 source, trans->base.stride,
 186                                 box->width, box->height);
 187                 break;
 188         default:
 189                 assert(!"Unexpected format\n");
 190                 break;
 191         }
 192 }
 193
 194 static void
 195 fd_resource_flush(struct fd_transfer *trans, const struct pipe_box *box)
 196 {
 197         enum pipe_format format = trans->base.resource->format;
 198
 199         switch (format) {
 200         case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT:
 201         case PIPE_FORMAT_X32_S8X24_UINT:
 202                 fd_resource_flush_z32s8(trans, box);
 203                 break;
 204         case PIPE_FORMAT_RGTC1_UNORM:
 205         case PIPE_FORMAT_RGTC1_SNORM:
 206         case PIPE_FORMAT_RGTC2_UNORM:
 207         case PIPE_FORMAT_RGTC2_SNORM:
 208         case PIPE_FORMAT_LATC1_UNORM:
 209         case PIPE_FORMAT_LATC1_SNORM:
 210         case PIPE_FORMAT_LATC2_UNORM:
 211         case PIPE_FORMAT_LATC2_SNORM:
 212                 fd_resource_flush_rgtc(trans, box);
 213                 break;
 214         default:
 215                 assert(!"Unexpected staging transfer type");
 216                 break;
 217         }
 218 }
 219
 220 static void fd_resource_transfer_flush_region(struct pipe_context *pctx,
 221                 struct pipe_transfer *ptrans,
 222                 const struct pipe_box *box)
 223 {
 224         struct fd_resource *rsc = fd_resource(ptrans->resource);
 225         struct fd_transfer *trans = fd_transfer(ptrans);
 226
 227         if (ptrans->resource->target == PIPE_BUFFER)
 228                 util_range_add(&rsc->valid_buffer_range,
 229                                            ptrans->box.x + box->x,
 230                                            ptrans->box.x + box->x + box->width);
 231
 232         if (trans->staging)
 233                 fd_resource_flush(trans, box);
 234 }
 235
 236 static void
 237 fd_resource_transfer_unmap(struct pipe_context *pctx,
 238                 struct pipe_transfer *ptrans)
 239 {
 240         struct fd_context *ctx = fd_context(pctx);
 241         struct fd_resource *rsc = fd_resource(ptrans->resource);
 242         struct fd_transfer *trans = fd_transfer(ptrans);
 243
 244         if (trans->staging && !(ptrans->usage & PIPE_TRANSFER_FLUSH_EXPLICIT)) {
 245                 struct pipe_box box;
 246                 u_box_2d(0, 0, ptrans->box.width, ptrans->box.height, &box);
 247                 fd_resource_flush(trans, &box);
 248         }
 249
 250         if (!(ptrans->usage & PIPE_TRANSFER_UNSYNCHRONIZED)) {
 251                 fd_bo_cpu_fini(rsc->bo);
 252                 if (rsc->stencil)
 253                         fd_bo_cpu_fini(rsc->stencil->bo);
 254         }
 255
 256         util_range_add(&rsc->valid_buffer_range,
 257                                    ptrans->box.x,
 258                                    ptrans->box.x + ptrans->box.width);
 259
 260         pipe_resource_reference(&ptrans->resource, NULL);
 261         util_slab_free(&ctx->transfer_pool, ptrans);
 262
 263         if (trans->staging)
 264                 free(trans->staging);
 265 }
 266
 267 static void *
 268 fd_resource_transfer_map(struct pipe_context *pctx,
 269                 struct pipe_resource *prsc,
 270                 unsigned level, unsigned usage,
 271                 const struct pipe_box *box,
 272                 struct pipe_transfer **pptrans)
 273 {
 274         struct fd_context *ctx = fd_context(pctx);
 275         struct fd_resource *rsc = fd_resource(prsc);
 276         struct fd_resource_slice *slice = fd_resource_slice(rsc, level);
 277         struct fd_transfer *trans;
 278         struct pipe_transfer *ptrans;
 279         enum pipe_format format = prsc->format;
 280         uint32_t op = 0;
 281         uint32_t offset;
 282         char *buf;
 283         int ret = 0;
 284
 285         DBG("prsc=%p, level=%u, usage=%x, box=%dx%d+%d,%d", prsc, level, usage,
 286                 box->width, box->height, box->x, box->y);
 287
 288         ptrans = util_slab_alloc(&ctx->transfer_pool);
 289         if (!ptrans)
 290                 return NULL;
 291
 292         /* util_slab_alloc() doesn't zero: */
 293         trans = fd_transfer(ptrans);
 294         memset(trans, 0, sizeof(*trans));
 295
 296         pipe_resource_reference(&ptrans->resource, prsc);
 297         ptrans->level = level;
 298         ptrans->usage = usage;
 299         ptrans->box = *box;
 300         ptrans->stride = util_format_get_nblocksx(format, slice->pitch) * rsc->cpp;
 301         ptrans->layer_stride = slice->size0;
 302
 303         if (usage & PIPE_TRANSFER_READ)
 304                 op |= DRM_FREEDRENO_PREP_READ;
 305
 306         if (usage & PIPE_TRANSFER_WRITE)
 307                 op |= DRM_FREEDRENO_PREP_WRITE;
 308
 309         if (usage & PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE) {
 310                 realloc_bo(rsc, fd_bo_size(rsc->bo));
 311                 if (rsc->stencil)
 312                         realloc_bo(rsc->stencil, fd_bo_size(rsc->stencil->bo));
 313                 fd_invalidate_resource(ctx, prsc);
 314         } else if ((usage & PIPE_TRANSFER_WRITE) &&
 315                            prsc->target == PIPE_BUFFER &&
 316                            !util_ranges_intersect(&rsc->valid_buffer_range,
 317                                                                           box->x, box->x + box->width)) {
 318                 /* We are trying to write to a previously uninitialized range. No need
 319                  * to wait.
 320                  */
 321         } else if (!(usage & PIPE_TRANSFER_UNSYNCHRONIZED)) {
 322                 /* If the GPU is writing to the resource, or if it is reading from the
 323                  * resource and we're trying to write to it, flush the renders.
 324                  */
 325                 if (((ptrans->usage & PIPE_TRANSFER_WRITE) &&
 326                                         pending(rsc, FD_PENDING_READ | FD_PENDING_WRITE)) ||
 327                                 pending(rsc, FD_PENDING_WRITE))
 328                         fd_context_render(pctx);
 329
 330                 /* The GPU keeps track of how the various bo's are being used, and
 331                  * will wait if necessary for the proper operation to have
 332                  * completed.
 333                  */
 334                 ret = fd_bo_cpu_prep(rsc->bo, ctx->screen->pipe, op);
 335                 if (ret)
 336                         goto fail;
 337         }
 338
 339         buf = fd_bo_map(rsc->bo);
 340         if (!buf) {
 341                 fd_resource_transfer_unmap(pctx, ptrans);
 342                 return NULL;
 343         }
 344
 345         offset = slice->offset +
 346                 box->y / util_format_get_blockheight(format) * ptrans->stride +
 347                 box->x / util_format_get_blockwidth(format) * rsc->cpp +
 348                 fd_resource_layer_offset(rsc, slice, box->z);
 349
 350         if (prsc->format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT ||
 351                 prsc->format == PIPE_FORMAT_X32_S8X24_UINT) {
 352                 assert(trans->base.box.depth == 1);
 353
 354                 trans->base.stride = trans->base.box.width * rsc->cpp * 2;
 355                 trans->staging = malloc(trans->base.stride * trans->base.box.height);
 356                 if (!trans->staging)
 357                         goto fail;
 358
 359                 /* if we're not discarding the whole range (or resource), we must copy
 360                  * the real data in.
 361                  */
 362                 if (!(usage & (PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE |
 363                                            PIPE_TRANSFER_DISCARD_RANGE))) {
 364                         struct fd_resource_slice *sslice =
 365                                 fd_resource_slice(rsc->stencil, level);
 366                         void *sbuf = fd_bo_map(rsc->stencil->bo);
 367                         if (!sbuf)
 368                                 goto fail;
 369
 370                         float *depth = (float *)(buf + slice->offset +
 371                                 fd_resource_layer_offset(rsc, slice, box->z) +
 372                                 box->y * slice->pitch * 4 + box->x * 4);
 373                         uint8_t *stencil = sbuf + sslice->offset +
 374                                 fd_resource_layer_offset(rsc->stencil, sslice, box->z) +
 375                                 box->y * sslice->pitch + box->x;
 376
 377                         if (format != PIPE_FORMAT_X32_S8X24_UINT)
 378                                 util_format_z32_float_s8x24_uint_pack_z_float(
 379                                                 trans->staging, trans->base.stride,
 380                                                 depth, slice->pitch * 4,
 381                                                 box->width, box->height);
 382
 383                         util_format_z32_float_s8x24_uint_pack_s_8uint(
 384                                         trans->staging, trans->base.stride,
 385                                         stencil, sslice->pitch,
 386                                         box->width, box->height);
 387                 }
 388
 389                 buf = trans->staging;
 390                 offset = 0;
 391         } else if (rsc->internal_format != format &&
 392                            util_format_description(format)->layout == UTIL_FORMAT_LAYOUT_RGTC) {
 393                 assert(trans->base.box.depth == 1);
 394
 395                 trans->base.stride = util_format_get_stride(
 396                                 format, trans->base.box.width);
 397                 trans->staging = malloc(
 398                                 util_format_get_2d_size(format, trans->base.stride,
 399                                                                                 trans->base.box.height));
 400                 if (!trans->staging)
 401                         goto fail;
 402
 403                 /* if we're not discarding the whole range (or resource), we must copy
 404                  * the real data in.
 405                  */
 406                 if (!(usage & (PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE |
 407                                            PIPE_TRANSFER_DISCARD_RANGE))) {
 408                         uint8_t *rgba8 = (uint8_t *)buf + slice->offset +
 409                                 fd_resource_layer_offset(rsc, slice, box->z) +
 410                                 box->y * slice->pitch * rsc->cpp + box->x * rsc->cpp;
 411
 412                         switch (format) {
 413                         case PIPE_FORMAT_RGTC1_UNORM:
 414                         case PIPE_FORMAT_RGTC1_SNORM:
 415                         case PIPE_FORMAT_LATC1_UNORM:
 416                         case PIPE_FORMAT_LATC1_SNORM:
 417                                 util_format_rgtc1_unorm_pack_rgba_8unorm(
 418                                         trans->staging, trans->base.stride,
 419                                         rgba8, slice->pitch * rsc->cpp,
 420                                         box->width, box->height);
 421                                 break;
 422                         case PIPE_FORMAT_RGTC2_UNORM:
 423                         case PIPE_FORMAT_RGTC2_SNORM:
 424                         case PIPE_FORMAT_LATC2_UNORM:
 425                         case PIPE_FORMAT_LATC2_SNORM:
 426                                 util_format_rgtc2_unorm_pack_rgba_8unorm(
 427                                         trans->staging, trans->base.stride,
 428                                         rgba8, slice->pitch * rsc->cpp,
 429                                         box->width, box->height);
 430                                 break;
 431                         default:
 432                                 assert(!"Unexpected format");
 433                                 break;
 434                         }
 435                 }
 436
 437                 buf = trans->staging;
 438                 offset = 0;
 439         }
 440
 441         *pptrans = ptrans;
 442
 443         return buf + offset;
 444
 445 fail:
 446         fd_resource_transfer_unmap(pctx, ptrans);
 447         return NULL;
 448 }
 449
 450 static void
 451 fd_resource_destroy(struct pipe_screen *pscreen,
 452                 struct pipe_resource *prsc)
 453 {
 454         struct fd_resource *rsc = fd_resource(prsc);
 455         if (rsc->bo)
 456                 fd_bo_del(rsc->bo);
 457         list_delinit(&rsc->list);
 458         util_range_destroy(&rsc->valid_buffer_range);
 459         FREE(rsc);
 460 }
 461
 462 static boolean
 463 fd_resource_get_handle(struct pipe_screen *pscreen,
 464                 struct pipe_resource *prsc,
 465                 struct winsys_handle *handle)
 466 {
 467         struct fd_resource *rsc = fd_resource(prsc);
 468
 469         return fd_screen_bo_get_handle(pscreen, rsc->bo,
 470                         rsc->slices[0].pitch * rsc->cpp, handle);
 471 }
 472
 473
 474 static const struct u_resource_vtbl fd_resource_vtbl = {
 475                 .resource_get_handle      = fd_resource_get_handle,
 476                 .resource_destroy         = fd_resource_destroy,
 477                 .transfer_map             = fd_resource_transfer_map,
 478                 .transfer_flush_region    = fd_resource_transfer_flush_region,
 479                 .transfer_unmap           = fd_resource_transfer_unmap,
 480                 .transfer_inline_write    = u_default_transfer_inline_write,
 481 };
 482
 483 static uint32_t
 484 setup_slices(struct fd_resource *rsc, uint32_t alignment, enum pipe_format format)
 485 {
 486         struct pipe_resource *prsc = &rsc->base.b;
 487         uint32_t level, size = 0;
 488         uint32_t width = prsc->width0;
 489         uint32_t height = prsc->height0;
 490         uint32_t depth = prsc->depth0;
 491         /* in layer_first layout, the level (slice) contains just one
 492          * layer (since in fact the layer contains the slices)
 493          */
 494         uint32_t layers_in_level = rsc->layer_first ? 1 : prsc->array_size;
 495
 496         for (level = 0; level <= prsc->last_level; level++) {
 497                 struct fd_resource_slice *slice = fd_resource_slice(rsc, level);
 498                 uint32_t blocks;
 499
 500                 slice->pitch = width = align(width, 32);
 501                 slice->offset = size;
 502                 blocks = util_format_get_nblocks(format, width, height);
 503                 /* 1d array and 2d array textures must all have the same layer size
 504                  * for each miplevel on a3xx. 3d textures can have different layer
 505                  * sizes for high levels, but the hw auto-sizer is buggy (or at least
 506                  * different than what this code does), so as soon as the layer size
 507                  * range gets into range, we stop reducing it.
 508                  */
 509                 if (prsc->target == PIPE_TEXTURE_3D && (
 510                                         level == 1 ||
 511                                         (level > 1 && rsc->slices[level - 1].size0 > 0xf000)))
 512                         slice->size0 = align(blocks * rsc->cpp, alignment);
 513                 else if (level == 0 || rsc->layer_first || alignment == 1)
 514                         slice->size0 = align(blocks * rsc->cpp, alignment);
 515                 else
 516                         slice->size0 = rsc->slices[level - 1].size0;
 517
 518                 size += slice->size0 * depth * layers_in_level;
 519
 520                 width = u_minify(width, 1);
 521                 height = u_minify(height, 1);
 522                 depth = u_minify(depth, 1);
 523         }
 524
 525         return size;
 526 }
 527
 528 static uint32_t
 529 slice_alignment(struct pipe_screen *pscreen, const struct pipe_resource *tmpl)
 530 {
 531         /* on a3xx, 2d array and 3d textures seem to want their
 532          * layers aligned to page boundaries:
 533          */
 534         switch (tmpl->target) {
 535         case PIPE_TEXTURE_3D:
 536         case PIPE_TEXTURE_1D_ARRAY:
 537         case PIPE_TEXTURE_2D_ARRAY:
 538                 return 4096;
 539         default:
 540                 return 1;
 541         }
 542 }
 543
 544 /**
 545  * Create a new texture object, using the given template info.
 546  */
 547 static struct pipe_resource *
 548 fd_resource_create(struct pipe_screen *pscreen,
 549                 const struct pipe_resource *tmpl)
 550 {
 551         struct fd_resource *rsc = CALLOC_STRUCT(fd_resource);
 552         struct pipe_resource *prsc = &rsc->base.b;
 553         enum pipe_format format = tmpl->format;
 554         uint32_t size;
 555
 556         DBG("target=%d, format=%s, %ux%ux%u, array_size=%u, last_level=%u, "
 557                         "nr_samples=%u, usage=%u, bind=%x, flags=%x",
 558                         tmpl->target, util_format_name(format),
 559                         tmpl->width0, tmpl->height0, tmpl->depth0,
 560                         tmpl->array_size, tmpl->last_level, tmpl->nr_samples,
 561                         tmpl->usage, tmpl->bind, tmpl->flags);
 562
 563         if (!rsc)
 564                 return NULL;
 565
 566         *prsc = *tmpl;
 567
 568         pipe_reference_init(&prsc->reference, 1);
 569         list_inithead(&rsc->list);
 570         prsc->screen = pscreen;
 571
 572         util_range_init(&rsc->valid_buffer_range);
 573
 574         rsc->base.vtbl = &fd_resource_vtbl;
 575
 576         if (format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT)
 577                 format = PIPE_FORMAT_Z32_FLOAT;
 578         else if (fd_screen(pscreen)->gpu_id < 400 &&
 579                          util_format_description(format)->layout == UTIL_FORMAT_LAYOUT_RGTC)
 580                 format = PIPE_FORMAT_R8G8B8A8_UNORM;
 581         rsc->internal_format = format;
 582         rsc->cpp = util_format_get_blocksize(format);
 583
 584         assert(rsc->cpp);
 585
 586         if (is_a4xx(fd_screen(pscreen))) {
 587                 switch (tmpl->target) {
 588                 case PIPE_TEXTURE_3D:
 589                         rsc->layer_first = false;
 590                         break;
 591                 default:
 592                         rsc->layer_first = true;
 593                         break;
 594                 }
 595         }
 596
 597         size = setup_slices(rsc, slice_alignment(pscreen, tmpl), format);
 598
 599         if (rsc->layer_first) {
 600                 rsc->layer_size = align(size, 4096);
 601                 size = rsc->layer_size * prsc->array_size;
 602         }
 603
 604         realloc_bo(rsc, size);
 605         if (!rsc->bo)
 606                 goto fail;
 607
 608         /* There is no native Z32F_S8 sampling or rendering format, so this must
 609          * be emulated via two separate textures. The depth texture still keeps
 610          * its Z32F_S8 format though, and we also keep a reference to a separate
 611          * S8 texture.
 612          */
 613         if (tmpl->format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT) {
 614                 struct pipe_resource stencil = *tmpl;
 615                 stencil.format = PIPE_FORMAT_S8_UINT;
 616                 rsc->stencil = fd_resource(fd_resource_create(pscreen, &stencil));
 617                 if (!rsc->stencil)
 618                         goto fail;
 619         }
 620
 621         return prsc;
 622 fail:
 623         fd_resource_destroy(pscreen, prsc);
 624         return NULL;
 625 }
 626
 627 /**
 628  * Create a texture from a winsys_handle. The handle is often created in
 629  * another process by first creating a pipe texture and then calling
 630  * resource_get_handle.
 631  */
 632 static struct pipe_resource *
 633 fd_resource_from_handle(struct pipe_screen *pscreen,
 634                 const struct pipe_resource *tmpl,
 635                 struct winsys_handle *handle)
 636 {
 637         struct fd_resource *rsc = CALLOC_STRUCT(fd_resource);
 638         struct fd_resource_slice *slice = &rsc->slices[0];
 639         struct pipe_resource *prsc = &rsc->base.b;
 640
 641         DBG("target=%d, format=%s, %ux%ux%u, array_size=%u, last_level=%u, "
 642                         "nr_samples=%u, usage=%u, bind=%x, flags=%x",
 643                         tmpl->target, util_format_name(tmpl->format),
 644                         tmpl->width0, tmpl->height0, tmpl->depth0,
 645                         tmpl->array_size, tmpl->last_level, tmpl->nr_samples,
 646                         tmpl->usage, tmpl->bind, tmpl->flags);
 647
 648         if (!rsc)
 649                 return NULL;
 650
 651         *prsc = *tmpl;
 652
 653         pipe_reference_init(&prsc->reference, 1);
 654         list_inithead(&rsc->list);
 655         prsc->screen = pscreen;
 656
 657         util_range_init(&rsc->valid_buffer_range);
 658
 659         rsc->bo = fd_screen_bo_from_handle(pscreen, handle, &slice->pitch);
 660         if (!rsc->bo)
 661                 goto fail;
 662
 663         rsc->base.vtbl = &fd_resource_vtbl;
 664         rsc->cpp = util_format_get_blocksize(tmpl->format);
 665         slice->pitch /= rsc->cpp;
 666
 667         assert(rsc->cpp);
 668
 669         return prsc;
 670
 671 fail:
 672         fd_resource_destroy(pscreen, prsc);
 673         return NULL;
 674 }
 675
 676 static void fd_blitter_pipe_begin(struct fd_context *ctx, bool render_cond);
 677 static void fd_blitter_pipe_end(struct fd_context *ctx);
 678
 679 /**
 680  * _copy_region using pipe (3d engine)
 681  */
 682 static bool
 683 fd_blitter_pipe_copy_region(struct fd_context *ctx,
 684                 struct pipe_resource *dst,
 685                 unsigned dst_level,
 686                 unsigned dstx, unsigned dsty, unsigned dstz,
 687                 struct pipe_resource *src,
 688                 unsigned src_level,
 689                 const struct pipe_box *src_box)
 690 {
 691         /* not until we allow rendertargets to be buffers */
 692         if (dst->target == PIPE_BUFFER || src->target == PIPE_BUFFER)
 693                 return false;
 694
 695         if (!util_blitter_is_copy_supported(ctx->blitter, dst, src))
 696                 return false;
 697
 698         fd_blitter_pipe_begin(ctx, false);
 699         util_blitter_copy_texture(ctx->blitter,
 700                         dst, dst_level, dstx, dsty, dstz,
 701                         src, src_level, src_box);
 702         fd_blitter_pipe_end(ctx);
 703
 704         return true;
 705 }
 706
 707 /**
 708  * Copy a block of pixels from one resource to another.
 709  * The resource must be of the same format.
 710  * Resources with nr_samples > 1 are not allowed.
 711  */
 712 static void
 713 fd_resource_copy_region(struct pipe_context *pctx,
 714                 struct pipe_resource *dst,
 715                 unsigned dst_level,
 716                 unsigned dstx, unsigned dsty, unsigned dstz,
 717                 struct pipe_resource *src,
 718                 unsigned src_level,
 719                 const struct pipe_box *src_box)
 720 {
 721         struct fd_context *ctx = fd_context(pctx);
 722
 723         /* TODO if we have 2d core, or other DMA engine that could be used
 724          * for simple copies and reasonably easily synchronized with the 3d
 725          * core, this is where we'd plug it in..
 726          */
 727
 728         /* try blit on 3d pipe: */
 729         if (fd_blitter_pipe_copy_region(ctx,
 730                         dst, dst_level, dstx, dsty, dstz,
 731                         src, src_level, src_box))
 732                 return;
 733
 734         /* else fallback to pure sw: */
 735         util_resource_copy_region(pctx,
 736                         dst, dst_level, dstx, dsty, dstz,
 737                         src, src_level, src_box);
 738 }
 739
 740 bool
 741 fd_render_condition_check(struct pipe_context *pctx)
 742 {
 743         struct fd_context *ctx = fd_context(pctx);
 744
 745         if (!ctx->cond_query)
 746                 return true;
 747
 748         union pipe_query_result res = { 0 };
 749         bool wait =
 750                 ctx->cond_mode != PIPE_RENDER_COND_NO_WAIT &&
 751                 ctx->cond_mode != PIPE_RENDER_COND_BY_REGION_NO_WAIT;
 752
 753         if (pctx->get_query_result(pctx, ctx->cond_query, wait, &res))
 754                         return (bool)res.u64 != ctx->cond_cond;
 755
 756         return true;
 757 }
 758
 759 /**
 760  * Optimal hardware path for blitting pixels.
 761  * Scaling, format conversion, up- and downsampling (resolve) are allowed.
 762  */
 763 static void
 764 fd_blit(struct pipe_context *pctx, const struct pipe_blit_info *blit_info)
 765 {
 766         struct fd_context *ctx = fd_context(pctx);
 767         struct pipe_blit_info info = *blit_info;
 768
 769         if (info.src.resource->nr_samples > 1 &&
 770                         info.dst.resource->nr_samples <= 1 &&
 771                         !util_format_is_depth_or_stencil(info.src.resource->format) &&
 772                         !util_format_is_pure_integer(info.src.resource->format)) {
 773                 DBG("color resolve unimplemented");
 774                 return;
 775         }
 776
 777         if (info.render_condition_enable && !fd_render_condition_check(pctx))
 778                 return;
 779
 780         if (util_try_blit_via_copy_region(pctx, &info)) {
 781                 return; /* done */
 782         }
 783
 784         if (info.mask & PIPE_MASK_S) {
 785                 DBG("cannot blit stencil, skipping");
 786                 info.mask &= ~PIPE_MASK_S;
 787         }
 788
 789         if (!util_blitter_is_blit_supported(ctx->blitter, &info)) {
 790                 DBG("blit unsupported %s -> %s",
 791                                 util_format_short_name(info.src.resource->format),
 792                                 util_format_short_name(info.dst.resource->format));
 793                 return;
 794         }
 795
 796         fd_blitter_pipe_begin(ctx, info.render_condition_enable);
 797         util_blitter_blit(ctx->blitter, &info);
 798         fd_blitter_pipe_end(ctx);
 799 }
 800
 801 static void
 802 fd_blitter_pipe_begin(struct fd_context *ctx, bool render_cond)
 803 {
 804         util_blitter_save_vertex_buffer_slot(ctx->blitter, ctx->vtx.vertexbuf.vb);
 805         util_blitter_save_vertex_elements(ctx->blitter, ctx->vtx.vtx);
 806         util_blitter_save_vertex_shader(ctx->blitter, ctx->prog.vp);
 807         util_blitter_save_so_targets(ctx->blitter, ctx->streamout.num_targets,
 808                         ctx->streamout.targets);
 809         util_blitter_save_rasterizer(ctx->blitter, ctx->rasterizer);
 810         util_blitter_save_viewport(ctx->blitter, &ctx->viewport);
 811         util_blitter_save_scissor(ctx->blitter, &ctx->scissor);
 812         util_blitter_save_fragment_shader(ctx->blitter, ctx->prog.fp);
 813         util_blitter_save_blend(ctx->blitter, ctx->blend);
 814         util_blitter_save_depth_stencil_alpha(ctx->blitter, ctx->zsa);
 815         util_blitter_save_stencil_ref(ctx->blitter, &ctx->stencil_ref);
 816         util_blitter_save_sample_mask(ctx->blitter, ctx->sample_mask);
 817         util_blitter_save_framebuffer(ctx->blitter, &ctx->framebuffer);
 818         util_blitter_save_fragment_sampler_states(ctx->blitter,
 819                         ctx->fragtex.num_samplers,
 820                         (void **)ctx->fragtex.samplers);
 821         util_blitter_save_fragment_sampler_views(ctx->blitter,
 822                         ctx->fragtex.num_textures, ctx->fragtex.textures);
 823         if (!render_cond)
 824                 util_blitter_save_render_condition(ctx->blitter,
 825                         ctx->cond_query, ctx->cond_cond, ctx->cond_mode);
 826
 827         fd_hw_query_set_stage(ctx, ctx->ring, FD_STAGE_BLIT);
 828 }
 829
 830 static void
 831 fd_blitter_pipe_end(struct fd_context *ctx)
 832 {
 833         fd_hw_query_set_stage(ctx, ctx->ring, FD_STAGE_NULL);
 834 }
 835
 836 static void
 837 fd_flush_resource(struct pipe_context *pctx, struct pipe_resource *prsc)
 838 {
 839         struct fd_resource *rsc = fd_resource(prsc);
 840
 841         if (pending(rsc, FD_PENDING_WRITE | FD_PENDING_READ))
 842                 fd_context_render(pctx);
 843 }
 844
 845 void
 846 fd_resource_screen_init(struct pipe_screen *pscreen)
 847 {
 848         pscreen->resource_create = fd_resource_create;
 849         pscreen->resource_from_handle = fd_resource_from_handle;
 850         pscreen->resource_get_handle = u_resource_get_handle_vtbl;
 851         pscreen->resource_destroy = u_resource_destroy_vtbl;
 852 }
 853
 854 void
 855 fd_resource_context_init(struct pipe_context *pctx)
 856 {
 857         pctx->transfer_map = u_transfer_map_vtbl;
 858         pctx->transfer_flush_region = u_transfer_flush_region_vtbl;
 859         pctx->transfer_unmap = u_transfer_unmap_vtbl;
 860         pctx->transfer_inline_write = u_transfer_inline_write_vtbl;
 861         pctx->create_surface = fd_create_surface;
 862         pctx->surface_destroy = fd_surface_destroy;
 863         pctx->resource_copy_region = fd_resource_copy_region;
 864         pctx->blit = fd_blit;
 865         pctx->flush_resource = fd_flush_resource;
 866 }