src/gallium/drivers/freedreno/freedreno_resource.c

   1 /* -*- mode: C; c-file-style: "k&r"; tab-width 4; indent-tabs-mode: t; -*- */
   2
   3 /*
   4  * Copyright (C) 2012 Rob Clark <robclark@freedesktop.org>
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the next
  14  * paragraph) shall be included in all copies or substantial portions of the
  15  * Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  20  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  22  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  23  * SOFTWARE.
  24  *
  25  * Authors:
  26  *    Rob Clark <robclark@freedesktop.org>
  27  */
  28
  29 #include "util/u_format.h"
  30 #include "util/u_format_rgtc.h"
  31 #include "util/u_format_zs.h"
  32 #include "util/u_inlines.h"
  33 #include "util/u_transfer.h"
  34 #include "util/u_string.h"
  35 #include "util/u_surface.h"
  36
  37 #include "freedreno_resource.h"
  38 #include "freedreno_screen.h"
  39 #include "freedreno_surface.h"
  40 #include "freedreno_context.h"
  41 #include "freedreno_query_hw.h"
  42 #include "freedreno_util.h"
  43
  44 #include <errno.h>
  45
  46
  47 static bool
  48 pending(struct fd_resource *rsc, enum fd_resource_status status)
  49 {
  50         return (rsc->status & status) ||
  51                 (rsc->stencil && (rsc->stencil->status & status));
  52 }
  53
  54 static void
  55 fd_invalidate_resource(struct fd_context *ctx, struct pipe_resource *prsc)
  56 {
  57         int i;
  58
  59         /* Go through the entire state and see if the resource is bound
  60          * anywhere. If it is, mark the relevant state as dirty. This is called on
  61          * realloc_bo.
  62          */
  63
  64         /* Constbufs */
  65         for (i = 1; i < PIPE_MAX_CONSTANT_BUFFERS && !(ctx->dirty & FD_DIRTY_CONSTBUF); i++) {
  66                 if (ctx->constbuf[PIPE_SHADER_VERTEX].cb[i].buffer == prsc)
  67                         ctx->dirty |= FD_DIRTY_CONSTBUF;
  68                 if (ctx->constbuf[PIPE_SHADER_FRAGMENT].cb[i].buffer == prsc)
  69                         ctx->dirty |= FD_DIRTY_CONSTBUF;
  70         }
  71
  72         /* VBOs */
  73         for (i = 0; i < ctx->vtx.vertexbuf.count && !(ctx->dirty & FD_DIRTY_VTXBUF); i++) {
  74                 if (ctx->vtx.vertexbuf.vb[i].buffer == prsc)
  75                         ctx->dirty |= FD_DIRTY_VTXBUF;
  76         }
  77
  78         /* Index buffer */
  79         if (ctx->indexbuf.buffer == prsc)
  80                 ctx->dirty |= FD_DIRTY_INDEXBUF;
  81
  82         /* Textures */
  83         for (i = 0; i < ctx->verttex.num_textures && !(ctx->dirty & FD_DIRTY_VERTTEX); i++) {
  84                 if (ctx->verttex.textures[i] && (ctx->verttex.textures[i]->texture == prsc))
  85                         ctx->dirty |= FD_DIRTY_VERTTEX;
  86         }
  87         for (i = 0; i < ctx->fragtex.num_textures && !(ctx->dirty & FD_DIRTY_FRAGTEX); i++) {
  88                 if (ctx->fragtex.textures[i] && (ctx->fragtex.textures[i]->texture == prsc))
  89                         ctx->dirty |= FD_DIRTY_FRAGTEX;
  90         }
  91 }
  92
  93 static void
  94 realloc_bo(struct fd_resource *rsc, uint32_t size)
  95 {
  96         struct fd_screen *screen = fd_screen(rsc->base.b.screen);
  97         uint32_t flags = DRM_FREEDRENO_GEM_CACHE_WCOMBINE |
  98                         DRM_FREEDRENO_GEM_TYPE_KMEM; /* TODO */
  99
 100         /* if we start using things other than write-combine,
 101          * be sure to check for PIPE_RESOURCE_FLAG_MAP_COHERENT
 102          */
 103
 104         if (rsc->bo)
 105                 fd_bo_del(rsc->bo);
 106
 107         rsc->bo = fd_bo_new(screen->dev, size, flags);
 108         rsc->timestamp = 0;
 109         rsc->status = 0;
 110         rsc->pending_ctx = NULL;
 111         list_delinit(&rsc->list);
 112         util_range_set_empty(&rsc->valid_buffer_range);
 113 }
 114
 115 static unsigned
 116 fd_resource_layer_offset(struct fd_resource *rsc,
 117                                                  struct fd_resource_slice *slice,
 118                                                  unsigned layer)
 119 {
 120         if (rsc->layer_first)
 121                 return layer * rsc->layer_size;
 122         else
 123                 return layer * slice->size0;
 124 }
 125
 126 static void
 127 fd_resource_flush_z32s8(struct fd_transfer *trans, const struct pipe_box *box)
 128 {
 129         struct fd_resource *rsc = fd_resource(trans->base.resource);
 130         struct fd_resource_slice *slice = fd_resource_slice(rsc, trans->base.level);
 131         struct fd_resource_slice *sslice = fd_resource_slice(rsc->stencil, trans->base.level);
 132         enum pipe_format format = trans->base.resource->format;
 133
 134         float *depth = fd_bo_map(rsc->bo) + slice->offset +
 135                 fd_resource_layer_offset(rsc, slice, trans->base.box.z) +
 136                 (trans->base.box.y + box->y) * slice->pitch * 4 + (trans->base.box.x + box->x) * 4;
 137         uint8_t *stencil = fd_bo_map(rsc->stencil->bo) + sslice->offset +
 138                 fd_resource_layer_offset(rsc->stencil, sslice, trans->base.box.z) +
 139                 (trans->base.box.y + box->y) * sslice->pitch + trans->base.box.x + box->x;
 140
 141         if (format != PIPE_FORMAT_X32_S8X24_UINT)
 142                 util_format_z32_float_s8x24_uint_unpack_z_float(
 143                                 depth, slice->pitch * 4,
 144                                 trans->staging, trans->base.stride,
 145                                 box->width, box->height);
 146
 147         util_format_z32_float_s8x24_uint_unpack_s_8uint(
 148                         stencil, sslice->pitch,
 149                         trans->staging, trans->base.stride,
 150                         box->width, box->height);
 151 }
 152
 153 static void
 154 fd_resource_flush_rgtc(struct fd_transfer *trans, const struct pipe_box *box)
 155 {
 156         struct fd_resource *rsc = fd_resource(trans->base.resource);
 157         struct fd_resource_slice *slice = fd_resource_slice(rsc, trans->base.level);
 158         enum pipe_format format = trans->base.resource->format;
 159
 160         uint8_t *data = fd_bo_map(rsc->bo) + slice->offset +
 161                 fd_resource_layer_offset(rsc, slice, trans->base.box.z) +
 162                 ((trans->base.box.y + box->y) * slice->pitch +
 163                  trans->base.box.x + box->x) * rsc->cpp;
 164
 165         uint8_t *source = trans->staging +
 166                 util_format_get_nblocksy(format, box->y) * trans->base.stride +
 167                 util_format_get_stride(format, box->x);
 168
 169         switch (format) {
 170         case PIPE_FORMAT_RGTC1_UNORM:
 171         case PIPE_FORMAT_RGTC1_SNORM:
 172         case PIPE_FORMAT_LATC1_UNORM:
 173         case PIPE_FORMAT_LATC1_SNORM:
 174                 util_format_rgtc1_unorm_unpack_rgba_8unorm(
 175                                 data, slice->pitch * rsc->cpp,
 176                                 source, trans->base.stride,
 177                                 box->width, box->height);
 178                 break;
 179         case PIPE_FORMAT_RGTC2_UNORM:
 180         case PIPE_FORMAT_RGTC2_SNORM:
 181         case PIPE_FORMAT_LATC2_UNORM:
 182         case PIPE_FORMAT_LATC2_SNORM:
 183                 util_format_rgtc2_unorm_unpack_rgba_8unorm(
 184                                 data, slice->pitch * rsc->cpp,
 185                                 source, trans->base.stride,
 186                                 box->width, box->height);
 187                 break;
 188         default:
 189                 assert(!"Unexpected format\n");
 190                 break;
 191         }
 192 }
 193
 194 static void
 195 fd_resource_flush(struct fd_transfer *trans, const struct pipe_box *box)
 196 {
 197         enum pipe_format format = trans->base.resource->format;
 198
 199         switch (format) {
 200         case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT:
 201         case PIPE_FORMAT_X32_S8X24_UINT:
 202                 fd_resource_flush_z32s8(trans, box);
 203                 break;
 204         case PIPE_FORMAT_RGTC1_UNORM:
 205         case PIPE_FORMAT_RGTC1_SNORM:
 206         case PIPE_FORMAT_RGTC2_UNORM:
 207         case PIPE_FORMAT_RGTC2_SNORM:
 208         case PIPE_FORMAT_LATC1_UNORM:
 209         case PIPE_FORMAT_LATC1_SNORM:
 210         case PIPE_FORMAT_LATC2_UNORM:
 211         case PIPE_FORMAT_LATC2_SNORM:
 212                 fd_resource_flush_rgtc(trans, box);
 213                 break;
 214         default:
 215                 assert(!"Unexpected staging transfer type");
 216                 break;
 217         }
 218 }
 219
 220 static void fd_resource_transfer_flush_region(struct pipe_context *pctx,
 221                 struct pipe_transfer *ptrans,
 222                 const struct pipe_box *box)
 223 {
 224         struct fd_resource *rsc = fd_resource(ptrans->resource);
 225         struct fd_transfer *trans = fd_transfer(ptrans);
 226
 227         if (ptrans->resource->target == PIPE_BUFFER)
 228                 util_range_add(&rsc->valid_buffer_range,
 229                                            ptrans->box.x + box->x,
 230                                            ptrans->box.x + box->x + box->width);
 231
 232         if (trans->staging)
 233                 fd_resource_flush(trans, box);
 234 }
 235
 236 static void
 237 fd_resource_transfer_unmap(struct pipe_context *pctx,
 238                 struct pipe_transfer *ptrans)
 239 {
 240         struct fd_context *ctx = fd_context(pctx);
 241         struct fd_resource *rsc = fd_resource(ptrans->resource);
 242         struct fd_transfer *trans = fd_transfer(ptrans);
 243
 244         if (trans->staging && !(ptrans->usage & PIPE_TRANSFER_FLUSH_EXPLICIT)) {
 245                 struct pipe_box box;
 246                 u_box_2d(0, 0, ptrans->box.width, ptrans->box.height, &box);
 247                 fd_resource_flush(trans, &box);
 248         }
 249
 250         if (!(ptrans->usage & PIPE_TRANSFER_UNSYNCHRONIZED)) {
 251                 fd_bo_cpu_fini(rsc->bo);
 252                 if (rsc->stencil)
 253                         fd_bo_cpu_fini(rsc->stencil->bo);
 254         }
 255
 256         util_range_add(&rsc->valid_buffer_range,
 257                                    ptrans->box.x,
 258                                    ptrans->box.x + ptrans->box.width);
 259
 260         pipe_resource_reference(&ptrans->resource, NULL);
 261         util_slab_free(&ctx->transfer_pool, ptrans);
 262
 263         if (trans->staging)
 264                 free(trans->staging);
 265 }
 266
 267 static void *
 268 fd_resource_transfer_map(struct pipe_context *pctx,
 269                 struct pipe_resource *prsc,
 270                 unsigned level, unsigned usage,
 271                 const struct pipe_box *box,
 272                 struct pipe_transfer **pptrans)
 273 {
 274         struct fd_context *ctx = fd_context(pctx);
 275         struct fd_resource *rsc = fd_resource(prsc);
 276         struct fd_resource_slice *slice = fd_resource_slice(rsc, level);
 277         struct fd_transfer *trans;
 278         struct pipe_transfer *ptrans;
 279         enum pipe_format format = prsc->format;
 280         uint32_t op = 0;
 281         uint32_t offset;
 282         char *buf;
 283         int ret = 0;
 284
 285         DBG("prsc=%p, level=%u, usage=%x, box=%dx%d+%d,%d", prsc, level, usage,
 286                 box->width, box->height, box->x, box->y);
 287
 288         ptrans = util_slab_alloc(&ctx->transfer_pool);
 289         if (!ptrans)
 290                 return NULL;
 291
 292         /* util_slab_alloc() doesn't zero: */
 293         trans = fd_transfer(ptrans);
 294         memset(trans, 0, sizeof(*trans));
 295
 296         pipe_resource_reference(&ptrans->resource, prsc);
 297         ptrans->level = level;
 298         ptrans->usage = usage;
 299         ptrans->box = *box;
 300         ptrans->stride = util_format_get_nblocksx(format, slice->pitch) * rsc->cpp;
 301         ptrans->layer_stride = slice->size0;
 302
 303         if (usage & PIPE_TRANSFER_READ)
 304                 op |= DRM_FREEDRENO_PREP_READ;
 305
 306         if (usage & PIPE_TRANSFER_WRITE)
 307                 op |= DRM_FREEDRENO_PREP_WRITE;
 308
 309         if (usage & PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE) {
 310                 realloc_bo(rsc, fd_bo_size(rsc->bo));
 311                 if (rsc->stencil)
 312                         realloc_bo(rsc->stencil, fd_bo_size(rsc->stencil->bo));
 313                 fd_invalidate_resource(ctx, prsc);
 314         } else if ((usage & PIPE_TRANSFER_WRITE) &&
 315                            prsc->target == PIPE_BUFFER &&
 316                            !util_ranges_intersect(&rsc->valid_buffer_range,
 317                                                                           box->x, box->x + box->width)) {
 318                 /* We are trying to write to a previously uninitialized range. No need
 319                  * to wait.
 320                  */
 321         } else if (!(usage & PIPE_TRANSFER_UNSYNCHRONIZED)) {
 322                 /* If the GPU is writing to the resource, or if it is reading from the
 323                  * resource and we're trying to write to it, flush the renders.
 324                  */
 325                 if (((ptrans->usage & PIPE_TRANSFER_WRITE) &&
 326                                         pending(rsc, FD_PENDING_READ | FD_PENDING_WRITE)) ||
 327                                 pending(rsc, FD_PENDING_WRITE))
 328                         fd_context_render(pctx);
 329
 330                 /* The GPU keeps track of how the various bo's are being used, and
 331                  * will wait if necessary for the proper operation to have
 332                  * completed.
 333                  */
 334                 ret = fd_bo_cpu_prep(rsc->bo, ctx->screen->pipe, op);
 335                 if (ret)
 336                         goto fail;
 337         }
 338
 339         buf = fd_bo_map(rsc->bo);
 340         if (!buf) {
 341                 fd_resource_transfer_unmap(pctx, ptrans);
 342                 return NULL;
 343         }
 344
 345         offset = slice->offset +
 346                 box->y / util_format_get_blockheight(format) * ptrans->stride +
 347                 box->x / util_format_get_blockwidth(format) * rsc->cpp +
 348                 fd_resource_layer_offset(rsc, slice, box->z);
 349
 350         if (prsc->format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT ||
 351                 prsc->format == PIPE_FORMAT_X32_S8X24_UINT) {
 352                 assert(trans->base.box.depth == 1);
 353
 354                 trans->base.stride = trans->base.box.width * rsc->cpp * 2;
 355                 trans->staging = malloc(trans->base.stride * trans->base.box.height);
 356                 if (!trans->staging)
 357                         goto fail;
 358
 359                 /* if we're not discarding the whole range (or resource), we must copy
 360                  * the real data in.
 361                  */
 362                 if (!(usage & (PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE |
 363                                            PIPE_TRANSFER_DISCARD_RANGE))) {
 364                         struct fd_resource_slice *sslice =
 365                                 fd_resource_slice(rsc->stencil, level);
 366                         void *sbuf = fd_bo_map(rsc->stencil->bo);
 367                         if (!sbuf)
 368                                 goto fail;
 369
 370                         float *depth = (float *)(buf + slice->offset +
 371                                 fd_resource_layer_offset(rsc, slice, box->z) +
 372                                 box->y * slice->pitch * 4 + box->x * 4);
 373                         uint8_t *stencil = sbuf + sslice->offset +
 374                                 fd_resource_layer_offset(rsc->stencil, sslice, box->z) +
 375                                 box->y * sslice->pitch + box->x;
 376
 377                         if (format != PIPE_FORMAT_X32_S8X24_UINT)
 378                                 util_format_z32_float_s8x24_uint_pack_z_float(
 379                                                 trans->staging, trans->base.stride,
 380                                                 depth, slice->pitch * 4,
 381                                                 box->width, box->height);
 382
 383                         util_format_z32_float_s8x24_uint_pack_s_8uint(
 384                                         trans->staging, trans->base.stride,
 385                                         stencil, sslice->pitch,
 386                                         box->width, box->height);
 387                 }
 388
 389                 buf = trans->staging;
 390                 offset = 0;
 391         } else if (rsc->internal_format != format &&
 392                            util_format_description(format)->layout == UTIL_FORMAT_LAYOUT_RGTC) {
 393                 assert(trans->base.box.depth == 1);
 394
 395                 trans->base.stride = util_format_get_stride(
 396                                 format, trans->base.box.width);
 397                 trans->staging = malloc(
 398                                 util_format_get_2d_size(format, trans->base.stride,
 399                                                                                 trans->base.box.height));
 400                 if (!trans->staging)
 401                         goto fail;
 402
 403                 /* if we're not discarding the whole range (or resource), we must copy
 404                  * the real data in.
 405                  */
 406                 if (!(usage & (PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE |
 407                                            PIPE_TRANSFER_DISCARD_RANGE))) {
 408                         uint8_t *rgba8 = (uint8_t *)buf + slice->offset +
 409                                 fd_resource_layer_offset(rsc, slice, box->z) +
 410                                 box->y * slice->pitch * rsc->cpp + box->x * rsc->cpp;
 411
 412                         switch (format) {
 413                         case PIPE_FORMAT_RGTC1_UNORM:
 414                         case PIPE_FORMAT_RGTC1_SNORM:
 415                         case PIPE_FORMAT_LATC1_UNORM:
 416                         case PIPE_FORMAT_LATC1_SNORM:
 417                                 util_format_rgtc1_unorm_pack_rgba_8unorm(
 418                                         trans->staging, trans->base.stride,
 419                                         rgba8, slice->pitch * rsc->cpp,
 420                                         box->width, box->height);
 421                                 break;
 422                         case PIPE_FORMAT_RGTC2_UNORM:
 423                         case PIPE_FORMAT_RGTC2_SNORM:
 424                         case PIPE_FORMAT_LATC2_UNORM:
 425                         case PIPE_FORMAT_LATC2_SNORM:
 426                                 util_format_rgtc2_unorm_pack_rgba_8unorm(
 427                                         trans->staging, trans->base.stride,
 428                                         rgba8, slice->pitch * rsc->cpp,
 429                                         box->width, box->height);
 430                                 break;
 431                         default:
 432                                 assert(!"Unexpected format");
 433                                 break;
 434                         }
 435                 }
 436
 437                 buf = trans->staging;
 438                 offset = 0;
 439         }
 440
 441         *pptrans = ptrans;
 442
 443         return buf + offset;
 444
 445 fail:
 446         fd_resource_transfer_unmap(pctx, ptrans);
 447         return NULL;
 448 }
 449
 450 static void
 451 fd_resource_destroy(struct pipe_screen *pscreen,
 452                 struct pipe_resource *prsc)
 453 {
 454         struct fd_resource *rsc = fd_resource(prsc);
 455         if (rsc->bo)
 456                 fd_bo_del(rsc->bo);
 457         list_delinit(&rsc->list);
 458         util_range_destroy(&rsc->valid_buffer_range);
 459         FREE(rsc);
 460 }
 461
 462 static boolean
 463 fd_resource_get_handle(struct pipe_screen *pscreen,
 464                 struct pipe_resource *prsc,
 465                 struct winsys_handle *handle)
 466 {
 467         struct fd_resource *rsc = fd_resource(prsc);
 468
 469         return fd_screen_bo_get_handle(pscreen, rsc->bo,
 470                         rsc->slices[0].pitch * rsc->cpp, handle);
 471 }
 472
 473
 474 static const struct u_resource_vtbl fd_resource_vtbl = {
 475                 .resource_get_handle      = fd_resource_get_handle,
 476                 .resource_destroy         = fd_resource_destroy,
 477                 .transfer_map             = fd_resource_transfer_map,
 478                 .transfer_flush_region    = fd_resource_transfer_flush_region,
 479                 .transfer_unmap           = fd_resource_transfer_unmap,
 480                 .transfer_inline_write    = u_default_transfer_inline_write,
 481 };
 482
 483 static uint32_t
 484 setup_slices(struct fd_resource *rsc, uint32_t alignment, enum pipe_format format)
 485 {
 486         struct pipe_resource *prsc = &rsc->base.b;
 487         uint32_t level, size = 0;
 488         uint32_t width = prsc->width0;
 489         uint32_t height = prsc->height0;
 490         uint32_t depth = prsc->depth0;
 491         /* in layer_first layout, the level (slice) contains just one
 492          * layer (since in fact the layer contains the slices)
 493          */
 494         uint32_t layers_in_level = rsc->layer_first ? 1 : prsc->array_size;
 495
 496         for (level = 0; level <= prsc->last_level; level++) {
 497                 struct fd_resource_slice *slice = fd_resource_slice(rsc, level);
 498                 uint32_t blocks;
 499
 500                 slice->pitch = width = align(width, 32);
 501                 slice->offset = size;
 502                 blocks = util_format_get_nblocks(format, width, height);
 503                 /* 1d array and 2d array textures must all have the same layer size
 504                  * for each miplevel on a3xx. 3d textures can have different layer
 505                  * sizes for high levels, but the hw auto-sizer is buggy (or at least
 506                  * different than what this code does), so as soon as the layer size
 507                  * range gets into range, we stop reducing it.
 508                  */
 509                 if (prsc->target == PIPE_TEXTURE_3D && (
 510                                         level == 1 ||
 511                                         (level > 1 && rsc->slices[level - 1].size0 > 0xf000)))
 512                         slice->size0 = align(blocks * rsc->cpp, alignment);
 513                 else if (level == 0 || rsc->layer_first || alignment == 1)
 514                         slice->size0 = align(blocks * rsc->cpp, alignment);
 515                 else
 516                         slice->size0 = rsc->slices[level - 1].size0;
 517
 518                 size += slice->size0 * depth * layers_in_level;
 519
 520                 width = u_minify(width, 1);
 521                 height = u_minify(height, 1);
 522                 depth = u_minify(depth, 1);
 523         }
 524
 525         return size;
 526 }
 527
 528 static uint32_t
 529 slice_alignment(struct pipe_screen *pscreen, const struct pipe_resource *tmpl)
 530 {
 531         /* on a3xx, 2d array and 3d textures seem to want their
 532          * layers aligned to page boundaries:
 533          */
 534         switch (tmpl->target) {
 535         case PIPE_TEXTURE_3D:
 536         case PIPE_TEXTURE_1D_ARRAY:
 537         case PIPE_TEXTURE_2D_ARRAY:
 538                 return 4096;
 539         default:
 540                 return 1;
 541         }
 542 }
 543
 544 /**
 545  * Create a new texture object, using the given template info.
 546  */
 547 static struct pipe_resource *
 548 fd_resource_create(struct pipe_screen *pscreen,
 549                 const struct pipe_resource *tmpl)
 550 {
 551         struct fd_resource *rsc = CALLOC_STRUCT(fd_resource);
 552         struct pipe_resource *prsc = &rsc->base.b;
 553         enum pipe_format format = tmpl->format;
 554         uint32_t size, alignment;
 555
 556         DBG("target=%d, format=%s, %ux%ux%u, array_size=%u, last_level=%u, "
 557                         "nr_samples=%u, usage=%u, bind=%x, flags=%x",
 558                         tmpl->target, util_format_name(format),
 559                         tmpl->width0, tmpl->height0, tmpl->depth0,
 560                         tmpl->array_size, tmpl->last_level, tmpl->nr_samples,
 561                         tmpl->usage, tmpl->bind, tmpl->flags);
 562
 563         if (!rsc)
 564                 return NULL;
 565
 566         *prsc = *tmpl;
 567
 568         pipe_reference_init(&prsc->reference, 1);
 569         list_inithead(&rsc->list);
 570         prsc->screen = pscreen;
 571
 572         util_range_init(&rsc->valid_buffer_range);
 573
 574         rsc->base.vtbl = &fd_resource_vtbl;
 575
 576         if (format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT)
 577                 format = PIPE_FORMAT_Z32_FLOAT;
 578         else if (fd_screen(pscreen)->gpu_id < 400 &&
 579                          util_format_description(format)->layout == UTIL_FORMAT_LAYOUT_RGTC)
 580                 format = PIPE_FORMAT_R8G8B8A8_UNORM;
 581         rsc->internal_format = format;
 582         rsc->cpp = util_format_get_blocksize(format);
 583
 584         assert(rsc->cpp);
 585
 586         alignment = slice_alignment(pscreen, tmpl);
 587         if (is_a4xx(fd_screen(pscreen))) {
 588                 switch (tmpl->target) {
 589                 case PIPE_TEXTURE_3D:
 590                         rsc->layer_first = false;
 591                         break;
 592                 default:
 593                         rsc->layer_first = true;
 594                         alignment = 1;
 595                         break;
 596                 }
 597         }
 598
 599         size = setup_slices(rsc, alignment, format);
 600
 601         if (rsc->layer_first) {
 602                 rsc->layer_size = align(size, 4096);
 603                 size = rsc->layer_size * prsc->array_size;
 604         }
 605
 606         realloc_bo(rsc, size);
 607         if (!rsc->bo)
 608                 goto fail;
 609
 610         /* There is no native Z32F_S8 sampling or rendering format, so this must
 611          * be emulated via two separate textures. The depth texture still keeps
 612          * its Z32F_S8 format though, and we also keep a reference to a separate
 613          * S8 texture.
 614          */
 615         if (tmpl->format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT) {
 616                 struct pipe_resource stencil = *tmpl;
 617                 stencil.format = PIPE_FORMAT_S8_UINT;
 618                 rsc->stencil = fd_resource(fd_resource_create(pscreen, &stencil));
 619                 if (!rsc->stencil)
 620                         goto fail;
 621         }
 622
 623         return prsc;
 624 fail:
 625         fd_resource_destroy(pscreen, prsc);
 626         return NULL;
 627 }
 628
 629 /**
 630  * Create a texture from a winsys_handle. The handle is often created in
 631  * another process by first creating a pipe texture and then calling
 632  * resource_get_handle.
 633  */
 634 static struct pipe_resource *
 635 fd_resource_from_handle(struct pipe_screen *pscreen,
 636                 const struct pipe_resource *tmpl,
 637                 struct winsys_handle *handle)
 638 {
 639         struct fd_resource *rsc = CALLOC_STRUCT(fd_resource);
 640         struct fd_resource_slice *slice = &rsc->slices[0];
 641         struct pipe_resource *prsc = &rsc->base.b;
 642
 643         DBG("target=%d, format=%s, %ux%ux%u, array_size=%u, last_level=%u, "
 644                         "nr_samples=%u, usage=%u, bind=%x, flags=%x",
 645                         tmpl->target, util_format_name(tmpl->format),
 646                         tmpl->width0, tmpl->height0, tmpl->depth0,
 647                         tmpl->array_size, tmpl->last_level, tmpl->nr_samples,
 648                         tmpl->usage, tmpl->bind, tmpl->flags);
 649
 650         if (!rsc)
 651                 return NULL;
 652
 653         *prsc = *tmpl;
 654
 655         pipe_reference_init(&prsc->reference, 1);
 656         list_inithead(&rsc->list);
 657         prsc->screen = pscreen;
 658
 659         util_range_init(&rsc->valid_buffer_range);
 660
 661         rsc->bo = fd_screen_bo_from_handle(pscreen, handle, &slice->pitch);
 662         if (!rsc->bo)
 663                 goto fail;
 664
 665         rsc->base.vtbl = &fd_resource_vtbl;
 666         rsc->cpp = util_format_get_blocksize(tmpl->format);
 667         slice->pitch /= rsc->cpp;
 668
 669         assert(rsc->cpp);
 670
 671         return prsc;
 672
 673 fail:
 674         fd_resource_destroy(pscreen, prsc);
 675         return NULL;
 676 }
 677
 678 static void fd_blitter_pipe_begin(struct fd_context *ctx, bool render_cond);
 679 static void fd_blitter_pipe_end(struct fd_context *ctx);
 680
 681 /**
 682  * _copy_region using pipe (3d engine)
 683  */
 684 static bool
 685 fd_blitter_pipe_copy_region(struct fd_context *ctx,
 686                 struct pipe_resource *dst,
 687                 unsigned dst_level,
 688                 unsigned dstx, unsigned dsty, unsigned dstz,
 689                 struct pipe_resource *src,
 690                 unsigned src_level,
 691                 const struct pipe_box *src_box)
 692 {
 693         /* not until we allow rendertargets to be buffers */
 694         if (dst->target == PIPE_BUFFER || src->target == PIPE_BUFFER)
 695                 return false;
 696
 697         if (!util_blitter_is_copy_supported(ctx->blitter, dst, src))
 698                 return false;
 699
 700         fd_blitter_pipe_begin(ctx, false);
 701         util_blitter_copy_texture(ctx->blitter,
 702                         dst, dst_level, dstx, dsty, dstz,
 703                         src, src_level, src_box);
 704         fd_blitter_pipe_end(ctx);
 705
 706         return true;
 707 }
 708
 709 /**
 710  * Copy a block of pixels from one resource to another.
 711  * The resource must be of the same format.
 712  * Resources with nr_samples > 1 are not allowed.
 713  */
 714 static void
 715 fd_resource_copy_region(struct pipe_context *pctx,
 716                 struct pipe_resource *dst,
 717                 unsigned dst_level,
 718                 unsigned dstx, unsigned dsty, unsigned dstz,
 719                 struct pipe_resource *src,
 720                 unsigned src_level,
 721                 const struct pipe_box *src_box)
 722 {
 723         struct fd_context *ctx = fd_context(pctx);
 724
 725         /* TODO if we have 2d core, or other DMA engine that could be used
 726          * for simple copies and reasonably easily synchronized with the 3d
 727          * core, this is where we'd plug it in..
 728          */
 729
 730         /* try blit on 3d pipe: */
 731         if (fd_blitter_pipe_copy_region(ctx,
 732                         dst, dst_level, dstx, dsty, dstz,
 733                         src, src_level, src_box))
 734                 return;
 735
 736         /* else fallback to pure sw: */
 737         util_resource_copy_region(pctx,
 738                         dst, dst_level, dstx, dsty, dstz,
 739                         src, src_level, src_box);
 740 }
 741
 742 bool
 743 fd_render_condition_check(struct pipe_context *pctx)
 744 {
 745         struct fd_context *ctx = fd_context(pctx);
 746
 747         if (!ctx->cond_query)
 748                 return true;
 749
 750         union pipe_query_result res = { 0 };
 751         bool wait =
 752                 ctx->cond_mode != PIPE_RENDER_COND_NO_WAIT &&
 753                 ctx->cond_mode != PIPE_RENDER_COND_BY_REGION_NO_WAIT;
 754
 755         if (pctx->get_query_result(pctx, ctx->cond_query, wait, &res))
 756                         return (bool)res.u64 != ctx->cond_cond;
 757
 758         return true;
 759 }
 760
 761 /**
 762  * Optimal hardware path for blitting pixels.
 763  * Scaling, format conversion, up- and downsampling (resolve) are allowed.
 764  */
 765 static void
 766 fd_blit(struct pipe_context *pctx, const struct pipe_blit_info *blit_info)
 767 {
 768         struct fd_context *ctx = fd_context(pctx);
 769         struct pipe_blit_info info = *blit_info;
 770
 771         if (info.src.resource->nr_samples > 1 &&
 772                         info.dst.resource->nr_samples <= 1 &&
 773                         !util_format_is_depth_or_stencil(info.src.resource->format) &&
 774                         !util_format_is_pure_integer(info.src.resource->format)) {
 775                 DBG("color resolve unimplemented");
 776                 return;
 777         }
 778
 779         if (info.render_condition_enable && !fd_render_condition_check(pctx))
 780                 return;
 781
 782         if (util_try_blit_via_copy_region(pctx, &info)) {
 783                 return; /* done */
 784         }
 785
 786         if (info.mask & PIPE_MASK_S) {
 787                 DBG("cannot blit stencil, skipping");
 788                 info.mask &= ~PIPE_MASK_S;
 789         }
 790
 791         if (!util_blitter_is_blit_supported(ctx->blitter, &info)) {
 792                 DBG("blit unsupported %s -> %s",
 793                                 util_format_short_name(info.src.resource->format),
 794                                 util_format_short_name(info.dst.resource->format));
 795                 return;
 796         }
 797
 798         fd_blitter_pipe_begin(ctx, info.render_condition_enable);
 799         util_blitter_blit(ctx->blitter, &info);
 800         fd_blitter_pipe_end(ctx);
 801 }
 802
 803 static void
 804 fd_blitter_pipe_begin(struct fd_context *ctx, bool render_cond)
 805 {
 806         util_blitter_save_vertex_buffer_slot(ctx->blitter, ctx->vtx.vertexbuf.vb);
 807         util_blitter_save_vertex_elements(ctx->blitter, ctx->vtx.vtx);
 808         util_blitter_save_vertex_shader(ctx->blitter, ctx->prog.vp);
 809         util_blitter_save_so_targets(ctx->blitter, ctx->streamout.num_targets,
 810                         ctx->streamout.targets);
 811         util_blitter_save_rasterizer(ctx->blitter, ctx->rasterizer);
 812         util_blitter_save_viewport(ctx->blitter, &ctx->viewport);
 813         util_blitter_save_scissor(ctx->blitter, &ctx->scissor);
 814         util_blitter_save_fragment_shader(ctx->blitter, ctx->prog.fp);
 815         util_blitter_save_blend(ctx->blitter, ctx->blend);
 816         util_blitter_save_depth_stencil_alpha(ctx->blitter, ctx->zsa);
 817         util_blitter_save_stencil_ref(ctx->blitter, &ctx->stencil_ref);
 818         util_blitter_save_sample_mask(ctx->blitter, ctx->sample_mask);
 819         util_blitter_save_framebuffer(ctx->blitter, &ctx->framebuffer);
 820         util_blitter_save_fragment_sampler_states(ctx->blitter,
 821                         ctx->fragtex.num_samplers,
 822                         (void **)ctx->fragtex.samplers);
 823         util_blitter_save_fragment_sampler_views(ctx->blitter,
 824                         ctx->fragtex.num_textures, ctx->fragtex.textures);
 825         if (!render_cond)
 826                 util_blitter_save_render_condition(ctx->blitter,
 827                         ctx->cond_query, ctx->cond_cond, ctx->cond_mode);
 828
 829         fd_hw_query_set_stage(ctx, ctx->ring, FD_STAGE_BLIT);
 830 }
 831
 832 static void
 833 fd_blitter_pipe_end(struct fd_context *ctx)
 834 {
 835         fd_hw_query_set_stage(ctx, ctx->ring, FD_STAGE_NULL);
 836 }
 837
 838 static void
 839 fd_flush_resource(struct pipe_context *pctx, struct pipe_resource *prsc)
 840 {
 841         struct fd_resource *rsc = fd_resource(prsc);
 842
 843         if (pending(rsc, FD_PENDING_WRITE | FD_PENDING_READ))
 844                 fd_context_render(pctx);
 845 }
 846
 847 void
 848 fd_resource_screen_init(struct pipe_screen *pscreen)
 849 {
 850         pscreen->resource_create = fd_resource_create;
 851         pscreen->resource_from_handle = fd_resource_from_handle;
 852         pscreen->resource_get_handle = u_resource_get_handle_vtbl;
 853         pscreen->resource_destroy = u_resource_destroy_vtbl;
 854 }
 855
 856 void
 857 fd_resource_context_init(struct pipe_context *pctx)
 858 {
 859         pctx->transfer_map = u_transfer_map_vtbl;
 860         pctx->transfer_flush_region = u_transfer_flush_region_vtbl;
 861         pctx->transfer_unmap = u_transfer_unmap_vtbl;
 862         pctx->transfer_inline_write = u_transfer_inline_write_vtbl;
 863         pctx->create_surface = fd_create_surface;
 864         pctx->surface_destroy = fd_surface_destroy;
 865         pctx->resource_copy_region = fd_resource_copy_region;
 866         pctx->blit = fd_blit;
 867         pctx->flush_resource = fd_flush_resource;
 868 }