src/gallium/drivers/freedreno/freedreno_resource.c

   1 /* -*- mode: C; c-file-style: "k&r"; tab-width 4; indent-tabs-mode: t; -*- */
   2
   3 /*
   4  * Copyright (C) 2012 Rob Clark <robclark@freedesktop.org>
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the next
  14  * paragraph) shall be included in all copies or substantial portions of the
  15  * Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  20  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  22  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  23  * SOFTWARE.
  24  *
  25  * Authors:
  26  *    Rob Clark <robclark@freedesktop.org>
  27  */
  28
  29 #include "util/u_format.h"
  30 #include "util/u_format_rgtc.h"
  31 #include "util/u_format_zs.h"
  32 #include "util/u_inlines.h"
  33 #include "util/u_transfer.h"
  34 #include "util/u_string.h"
  35 #include "util/u_surface.h"
  36
  37 #include "freedreno_resource.h"
  38 #include "freedreno_screen.h"
  39 #include "freedreno_surface.h"
  40 #include "freedreno_context.h"
  41 #include "freedreno_query_hw.h"
  42 #include "freedreno_util.h"
  43
  44 #include <errno.h>
  45
  46 /* XXX this should go away, needed for 'struct winsys_handle' */
  47 #include "state_tracker/drm_driver.h"
  48
  49 static bool
  50 pending(struct fd_resource *rsc, enum fd_resource_status status)
  51 {
  52         return (rsc->status & status) ||
  53                 (rsc->stencil && (rsc->stencil->status & status));
  54 }
  55
  56 static void
  57 fd_invalidate_resource(struct fd_context *ctx, struct pipe_resource *prsc)
  58 {
  59         int i;
  60
  61         /* Go through the entire state and see if the resource is bound
  62          * anywhere. If it is, mark the relevant state as dirty. This is called on
  63          * realloc_bo.
  64          */
  65
  66         /* Constbufs */
  67         for (i = 1; i < PIPE_MAX_CONSTANT_BUFFERS && !(ctx->dirty & FD_DIRTY_CONSTBUF); i++) {
  68                 if (ctx->constbuf[PIPE_SHADER_VERTEX].cb[i].buffer == prsc)
  69                         ctx->dirty |= FD_DIRTY_CONSTBUF;
  70                 if (ctx->constbuf[PIPE_SHADER_FRAGMENT].cb[i].buffer == prsc)
  71                         ctx->dirty |= FD_DIRTY_CONSTBUF;
  72         }
  73
  74         /* VBOs */
  75         for (i = 0; i < ctx->vtx.vertexbuf.count && !(ctx->dirty & FD_DIRTY_VTXBUF); i++) {
  76                 if (ctx->vtx.vertexbuf.vb[i].buffer == prsc)
  77                         ctx->dirty |= FD_DIRTY_VTXBUF;
  78         }
  79
  80         /* Index buffer */
  81         if (ctx->indexbuf.buffer == prsc)
  82                 ctx->dirty |= FD_DIRTY_INDEXBUF;
  83
  84         /* Textures */
  85         for (i = 0; i < ctx->verttex.num_textures && !(ctx->dirty & FD_DIRTY_VERTTEX); i++) {
  86                 if (ctx->verttex.textures[i] && (ctx->verttex.textures[i]->texture == prsc))
  87                         ctx->dirty |= FD_DIRTY_VERTTEX;
  88         }
  89         for (i = 0; i < ctx->fragtex.num_textures && !(ctx->dirty & FD_DIRTY_FRAGTEX); i++) {
  90                 if (ctx->fragtex.textures[i] && (ctx->fragtex.textures[i]->texture == prsc))
  91                         ctx->dirty |= FD_DIRTY_FRAGTEX;
  92         }
  93 }
  94
  95 static void
  96 realloc_bo(struct fd_resource *rsc, uint32_t size)
  97 {
  98         struct fd_screen *screen = fd_screen(rsc->base.b.screen);
  99         uint32_t flags = DRM_FREEDRENO_GEM_CACHE_WCOMBINE |
 100                         DRM_FREEDRENO_GEM_TYPE_KMEM; /* TODO */
 101
 102         /* if we start using things other than write-combine,
 103          * be sure to check for PIPE_RESOURCE_FLAG_MAP_COHERENT
 104          */
 105
 106         if (rsc->bo)
 107                 fd_bo_del(rsc->bo);
 108
 109         rsc->bo = fd_bo_new(screen->dev, size, flags);
 110         rsc->timestamp = 0;
 111         rsc->status = 0;
 112         rsc->pending_ctx = NULL;
 113         list_delinit(&rsc->list);
 114         util_range_set_empty(&rsc->valid_buffer_range);
 115 }
 116
 117 static unsigned
 118 fd_resource_layer_offset(struct fd_resource *rsc,
 119                                                  struct fd_resource_slice *slice,
 120                                                  unsigned layer)
 121 {
 122         if (rsc->layer_first)
 123                 return layer * rsc->layer_size;
 124         else
 125                 return layer * slice->size0;
 126 }
 127
 128 static void
 129 fd_resource_flush_z32s8(struct fd_transfer *trans, const struct pipe_box *box)
 130 {
 131         struct fd_resource *rsc = fd_resource(trans->base.resource);
 132         struct fd_resource_slice *slice = fd_resource_slice(rsc, trans->base.level);
 133         struct fd_resource_slice *sslice = fd_resource_slice(rsc->stencil, trans->base.level);
 134         enum pipe_format format = trans->base.resource->format;
 135
 136         float *depth = fd_bo_map(rsc->bo) + slice->offset +
 137                 fd_resource_layer_offset(rsc, slice, trans->base.box.z) +
 138                 (trans->base.box.y + box->y) * slice->pitch * 4 + (trans->base.box.x + box->x) * 4;
 139         uint8_t *stencil = fd_bo_map(rsc->stencil->bo) + sslice->offset +
 140                 fd_resource_layer_offset(rsc->stencil, sslice, trans->base.box.z) +
 141                 (trans->base.box.y + box->y) * sslice->pitch + trans->base.box.x + box->x;
 142
 143         if (format != PIPE_FORMAT_X32_S8X24_UINT)
 144                 util_format_z32_float_s8x24_uint_unpack_z_float(
 145                                 depth, slice->pitch * 4,
 146                                 trans->staging, trans->base.stride,
 147                                 box->width, box->height);
 148
 149         util_format_z32_float_s8x24_uint_unpack_s_8uint(
 150                         stencil, sslice->pitch,
 151                         trans->staging, trans->base.stride,
 152                         box->width, box->height);
 153 }
 154
 155 static void
 156 fd_resource_flush_rgtc(struct fd_transfer *trans, const struct pipe_box *box)
 157 {
 158         struct fd_resource *rsc = fd_resource(trans->base.resource);
 159         struct fd_resource_slice *slice = fd_resource_slice(rsc, trans->base.level);
 160         enum pipe_format format = trans->base.resource->format;
 161
 162         uint8_t *data = fd_bo_map(rsc->bo) + slice->offset +
 163                 fd_resource_layer_offset(rsc, slice, trans->base.box.z) +
 164                 ((trans->base.box.y + box->y) * slice->pitch +
 165                  trans->base.box.x + box->x) * rsc->cpp;
 166
 167         uint8_t *source = trans->staging +
 168                 util_format_get_nblocksy(format, box->y) * trans->base.stride +
 169                 util_format_get_stride(format, box->x);
 170
 171         switch (format) {
 172         case PIPE_FORMAT_RGTC1_UNORM:
 173         case PIPE_FORMAT_RGTC1_SNORM:
 174         case PIPE_FORMAT_LATC1_UNORM:
 175         case PIPE_FORMAT_LATC1_SNORM:
 176                 util_format_rgtc1_unorm_unpack_rgba_8unorm(
 177                                 data, slice->pitch * rsc->cpp,
 178                                 source, trans->base.stride,
 179                                 box->width, box->height);
 180                 break;
 181         case PIPE_FORMAT_RGTC2_UNORM:
 182         case PIPE_FORMAT_RGTC2_SNORM:
 183         case PIPE_FORMAT_LATC2_UNORM:
 184         case PIPE_FORMAT_LATC2_SNORM:
 185                 util_format_rgtc2_unorm_unpack_rgba_8unorm(
 186                                 data, slice->pitch * rsc->cpp,
 187                                 source, trans->base.stride,
 188                                 box->width, box->height);
 189                 break;
 190         default:
 191                 assert(!"Unexpected format\n");
 192                 break;
 193         }
 194 }
 195
 196 static void
 197 fd_resource_flush(struct fd_transfer *trans, const struct pipe_box *box)
 198 {
 199         enum pipe_format format = trans->base.resource->format;
 200
 201         switch (format) {
 202         case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT:
 203         case PIPE_FORMAT_X32_S8X24_UINT:
 204                 fd_resource_flush_z32s8(trans, box);
 205                 break;
 206         case PIPE_FORMAT_RGTC1_UNORM:
 207         case PIPE_FORMAT_RGTC1_SNORM:
 208         case PIPE_FORMAT_RGTC2_UNORM:
 209         case PIPE_FORMAT_RGTC2_SNORM:
 210         case PIPE_FORMAT_LATC1_UNORM:
 211         case PIPE_FORMAT_LATC1_SNORM:
 212         case PIPE_FORMAT_LATC2_UNORM:
 213         case PIPE_FORMAT_LATC2_SNORM:
 214                 fd_resource_flush_rgtc(trans, box);
 215                 break;
 216         default:
 217                 assert(!"Unexpected staging transfer type");
 218                 break;
 219         }
 220 }
 221
 222 static void fd_resource_transfer_flush_region(struct pipe_context *pctx,
 223                 struct pipe_transfer *ptrans,
 224                 const struct pipe_box *box)
 225 {
 226         struct fd_resource *rsc = fd_resource(ptrans->resource);
 227         struct fd_transfer *trans = fd_transfer(ptrans);
 228
 229         if (ptrans->resource->target == PIPE_BUFFER)
 230                 util_range_add(&rsc->valid_buffer_range,
 231                                            ptrans->box.x + box->x,
 232                                            ptrans->box.x + box->x + box->width);
 233
 234         if (trans->staging)
 235                 fd_resource_flush(trans, box);
 236 }
 237
 238 static void
 239 fd_resource_transfer_unmap(struct pipe_context *pctx,
 240                 struct pipe_transfer *ptrans)
 241 {
 242         struct fd_context *ctx = fd_context(pctx);
 243         struct fd_resource *rsc = fd_resource(ptrans->resource);
 244         struct fd_transfer *trans = fd_transfer(ptrans);
 245
 246         if (trans->staging && !(ptrans->usage & PIPE_TRANSFER_FLUSH_EXPLICIT)) {
 247                 struct pipe_box box;
 248                 u_box_2d(0, 0, ptrans->box.width, ptrans->box.height, &box);
 249                 fd_resource_flush(trans, &box);
 250         }
 251
 252         if (!(ptrans->usage & PIPE_TRANSFER_UNSYNCHRONIZED)) {
 253                 fd_bo_cpu_fini(rsc->bo);
 254                 if (rsc->stencil)
 255                         fd_bo_cpu_fini(rsc->stencil->bo);
 256         }
 257
 258         util_range_add(&rsc->valid_buffer_range,
 259                                    ptrans->box.x,
 260                                    ptrans->box.x + ptrans->box.width);
 261
 262         pipe_resource_reference(&ptrans->resource, NULL);
 263         util_slab_free(&ctx->transfer_pool, ptrans);
 264
 265         free(trans->staging);
 266 }
 267
 268 static void *
 269 fd_resource_transfer_map(struct pipe_context *pctx,
 270                 struct pipe_resource *prsc,
 271                 unsigned level, unsigned usage,
 272                 const struct pipe_box *box,
 273                 struct pipe_transfer **pptrans)
 274 {
 275         struct fd_context *ctx = fd_context(pctx);
 276         struct fd_resource *rsc = fd_resource(prsc);
 277         struct fd_resource_slice *slice = fd_resource_slice(rsc, level);
 278         struct fd_transfer *trans;
 279         struct pipe_transfer *ptrans;
 280         enum pipe_format format = prsc->format;
 281         uint32_t op = 0;
 282         uint32_t offset;
 283         char *buf;
 284         int ret = 0;
 285
 286         DBG("prsc=%p, level=%u, usage=%x, box=%dx%d+%d,%d", prsc, level, usage,
 287                 box->width, box->height, box->x, box->y);
 288
 289         ptrans = util_slab_alloc(&ctx->transfer_pool);
 290         if (!ptrans)
 291                 return NULL;
 292
 293         /* util_slab_alloc() doesn't zero: */
 294         trans = fd_transfer(ptrans);
 295         memset(trans, 0, sizeof(*trans));
 296
 297         pipe_resource_reference(&ptrans->resource, prsc);
 298         ptrans->level = level;
 299         ptrans->usage = usage;
 300         ptrans->box = *box;
 301         ptrans->stride = util_format_get_nblocksx(format, slice->pitch) * rsc->cpp;
 302         ptrans->layer_stride = rsc->layer_first ? rsc->layer_size : slice->size0;
 303
 304         if (usage & PIPE_TRANSFER_READ)
 305                 op |= DRM_FREEDRENO_PREP_READ;
 306
 307         if (usage & PIPE_TRANSFER_WRITE)
 308                 op |= DRM_FREEDRENO_PREP_WRITE;
 309
 310         if (usage & PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE) {
 311                 realloc_bo(rsc, fd_bo_size(rsc->bo));
 312                 if (rsc->stencil)
 313                         realloc_bo(rsc->stencil, fd_bo_size(rsc->stencil->bo));
 314                 fd_invalidate_resource(ctx, prsc);
 315         } else if ((usage & PIPE_TRANSFER_WRITE) &&
 316                            prsc->target == PIPE_BUFFER &&
 317                            !util_ranges_intersect(&rsc->valid_buffer_range,
 318                                                                           box->x, box->x + box->width)) {
 319                 /* We are trying to write to a previously uninitialized range. No need
 320                  * to wait.
 321                  */
 322         } else if (!(usage & PIPE_TRANSFER_UNSYNCHRONIZED)) {
 323                 /* If the GPU is writing to the resource, or if it is reading from the
 324                  * resource and we're trying to write to it, flush the renders.
 325                  */
 326                 if (((ptrans->usage & PIPE_TRANSFER_WRITE) &&
 327                                         pending(rsc, FD_PENDING_READ | FD_PENDING_WRITE)) ||
 328                                 pending(rsc, FD_PENDING_WRITE))
 329                         fd_context_render(pctx);
 330
 331                 /* The GPU keeps track of how the various bo's are being used, and
 332                  * will wait if necessary for the proper operation to have
 333                  * completed.
 334                  */
 335                 ret = fd_bo_cpu_prep(rsc->bo, ctx->screen->pipe, op);
 336                 if (ret)
 337                         goto fail;
 338         }
 339
 340         buf = fd_bo_map(rsc->bo);
 341         if (!buf)
 342                 goto fail;
 343
 344         offset = slice->offset +
 345                 box->y / util_format_get_blockheight(format) * ptrans->stride +
 346                 box->x / util_format_get_blockwidth(format) * rsc->cpp +
 347                 fd_resource_layer_offset(rsc, slice, box->z);
 348
 349         if (prsc->format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT ||
 350                 prsc->format == PIPE_FORMAT_X32_S8X24_UINT) {
 351                 assert(trans->base.box.depth == 1);
 352
 353                 trans->base.stride = trans->base.box.width * rsc->cpp * 2;
 354                 trans->staging = malloc(trans->base.stride * trans->base.box.height);
 355                 if (!trans->staging)
 356                         goto fail;
 357
 358                 /* if we're not discarding the whole range (or resource), we must copy
 359                  * the real data in.
 360                  */
 361                 if (!(usage & (PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE |
 362                                            PIPE_TRANSFER_DISCARD_RANGE))) {
 363                         struct fd_resource_slice *sslice =
 364                                 fd_resource_slice(rsc->stencil, level);
 365                         void *sbuf = fd_bo_map(rsc->stencil->bo);
 366                         if (!sbuf)
 367                                 goto fail;
 368
 369                         float *depth = (float *)(buf + slice->offset +
 370                                 fd_resource_layer_offset(rsc, slice, box->z) +
 371                                 box->y * slice->pitch * 4 + box->x * 4);
 372                         uint8_t *stencil = sbuf + sslice->offset +
 373                                 fd_resource_layer_offset(rsc->stencil, sslice, box->z) +
 374                                 box->y * sslice->pitch + box->x;
 375
 376                         if (format != PIPE_FORMAT_X32_S8X24_UINT)
 377                                 util_format_z32_float_s8x24_uint_pack_z_float(
 378                                                 trans->staging, trans->base.stride,
 379                                                 depth, slice->pitch * 4,
 380                                                 box->width, box->height);
 381
 382                         util_format_z32_float_s8x24_uint_pack_s_8uint(
 383                                         trans->staging, trans->base.stride,
 384                                         stencil, sslice->pitch,
 385                                         box->width, box->height);
 386                 }
 387
 388                 buf = trans->staging;
 389                 offset = 0;
 390         } else if (rsc->internal_format != format &&
 391                            util_format_description(format)->layout == UTIL_FORMAT_LAYOUT_RGTC) {
 392                 assert(trans->base.box.depth == 1);
 393
 394                 trans->base.stride = util_format_get_stride(
 395                                 format, trans->base.box.width);
 396                 trans->staging = malloc(
 397                                 util_format_get_2d_size(format, trans->base.stride,
 398                                                                                 trans->base.box.height));
 399                 if (!trans->staging)
 400                         goto fail;
 401
 402                 /* if we're not discarding the whole range (or resource), we must copy
 403                  * the real data in.
 404                  */
 405                 if (!(usage & (PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE |
 406                                            PIPE_TRANSFER_DISCARD_RANGE))) {
 407                         uint8_t *rgba8 = (uint8_t *)buf + slice->offset +
 408                                 fd_resource_layer_offset(rsc, slice, box->z) +
 409                                 box->y * slice->pitch * rsc->cpp + box->x * rsc->cpp;
 410
 411                         switch (format) {
 412                         case PIPE_FORMAT_RGTC1_UNORM:
 413                         case PIPE_FORMAT_RGTC1_SNORM:
 414                         case PIPE_FORMAT_LATC1_UNORM:
 415                         case PIPE_FORMAT_LATC1_SNORM:
 416                                 util_format_rgtc1_unorm_pack_rgba_8unorm(
 417                                         trans->staging, trans->base.stride,
 418                                         rgba8, slice->pitch * rsc->cpp,
 419                                         box->width, box->height);
 420                                 break;
 421                         case PIPE_FORMAT_RGTC2_UNORM:
 422                         case PIPE_FORMAT_RGTC2_SNORM:
 423                         case PIPE_FORMAT_LATC2_UNORM:
 424                         case PIPE_FORMAT_LATC2_SNORM:
 425                                 util_format_rgtc2_unorm_pack_rgba_8unorm(
 426                                         trans->staging, trans->base.stride,
 427                                         rgba8, slice->pitch * rsc->cpp,
 428                                         box->width, box->height);
 429                                 break;
 430                         default:
 431                                 assert(!"Unexpected format");
 432                                 break;
 433                         }
 434                 }
 435
 436                 buf = trans->staging;
 437                 offset = 0;
 438         }
 439
 440         *pptrans = ptrans;
 441
 442         return buf + offset;
 443
 444 fail:
 445         fd_resource_transfer_unmap(pctx, ptrans);
 446         return NULL;
 447 }
 448
 449 static void
 450 fd_resource_destroy(struct pipe_screen *pscreen,
 451                 struct pipe_resource *prsc)
 452 {
 453         struct fd_resource *rsc = fd_resource(prsc);
 454         if (rsc->bo)
 455                 fd_bo_del(rsc->bo);
 456         list_delinit(&rsc->list);
 457         util_range_destroy(&rsc->valid_buffer_range);
 458         FREE(rsc);
 459 }
 460
 461 static boolean
 462 fd_resource_get_handle(struct pipe_screen *pscreen,
 463                 struct pipe_resource *prsc,
 464                 struct winsys_handle *handle)
 465 {
 466         struct fd_resource *rsc = fd_resource(prsc);
 467
 468         return fd_screen_bo_get_handle(pscreen, rsc->bo,
 469                         rsc->slices[0].pitch * rsc->cpp, handle);
 470 }
 471
 472
 473 static const struct u_resource_vtbl fd_resource_vtbl = {
 474                 .resource_get_handle      = fd_resource_get_handle,
 475                 .resource_destroy         = fd_resource_destroy,
 476                 .transfer_map             = fd_resource_transfer_map,
 477                 .transfer_flush_region    = fd_resource_transfer_flush_region,
 478                 .transfer_unmap           = fd_resource_transfer_unmap,
 479 };
 480
 481 static uint32_t
 482 setup_slices(struct fd_resource *rsc, uint32_t alignment, enum pipe_format format)
 483 {
 484         struct pipe_resource *prsc = &rsc->base.b;
 485         enum util_format_layout layout = util_format_description(format)->layout;
 486         uint32_t level, size = 0;
 487         uint32_t width = prsc->width0;
 488         uint32_t height = prsc->height0;
 489         uint32_t depth = prsc->depth0;
 490         /* in layer_first layout, the level (slice) contains just one
 491          * layer (since in fact the layer contains the slices)
 492          */
 493         uint32_t layers_in_level = rsc->layer_first ? 1 : prsc->array_size;
 494
 495         for (level = 0; level <= prsc->last_level; level++) {
 496                 struct fd_resource_slice *slice = fd_resource_slice(rsc, level);
 497                 uint32_t blocks;
 498
 499                 if (layout == UTIL_FORMAT_LAYOUT_ASTC)
 500                         slice->pitch = width =
 501                                 util_align_npot(width, 32 * util_format_get_blockwidth(format));
 502                 else
 503                         slice->pitch = width = align(width, 32);
 504                 slice->offset = size;
 505                 blocks = util_format_get_nblocks(format, width, height);
 506                 /* 1d array and 2d array textures must all have the same layer size
 507                  * for each miplevel on a3xx. 3d textures can have different layer
 508                  * sizes for high levels, but the hw auto-sizer is buggy (or at least
 509                  * different than what this code does), so as soon as the layer size
 510                  * range gets into range, we stop reducing it.
 511                  */
 512                 if (prsc->target == PIPE_TEXTURE_3D && (
 513                                         level == 1 ||
 514                                         (level > 1 && rsc->slices[level - 1].size0 > 0xf000)))
 515                         slice->size0 = align(blocks * rsc->cpp, alignment);
 516                 else if (level == 0 || rsc->layer_first || alignment == 1)
 517                         slice->size0 = align(blocks * rsc->cpp, alignment);
 518                 else
 519                         slice->size0 = rsc->slices[level - 1].size0;
 520
 521                 size += slice->size0 * depth * layers_in_level;
 522
 523                 width = u_minify(width, 1);
 524                 height = u_minify(height, 1);
 525                 depth = u_minify(depth, 1);
 526         }
 527
 528         return size;
 529 }
 530
 531 static uint32_t
 532 slice_alignment(struct pipe_screen *pscreen, const struct pipe_resource *tmpl)
 533 {
 534         /* on a3xx, 2d array and 3d textures seem to want their
 535          * layers aligned to page boundaries:
 536          */
 537         switch (tmpl->target) {
 538         case PIPE_TEXTURE_3D:
 539         case PIPE_TEXTURE_1D_ARRAY:
 540         case PIPE_TEXTURE_2D_ARRAY:
 541                 return 4096;
 542         default:
 543                 return 1;
 544         }
 545 }
 546
 547 /**
 548  * Create a new texture object, using the given template info.
 549  */
 550 static struct pipe_resource *
 551 fd_resource_create(struct pipe_screen *pscreen,
 552                 const struct pipe_resource *tmpl)
 553 {
 554         struct fd_resource *rsc = CALLOC_STRUCT(fd_resource);
 555         struct pipe_resource *prsc = &rsc->base.b;
 556         enum pipe_format format = tmpl->format;
 557         uint32_t size, alignment;
 558
 559         DBG("target=%d, format=%s, %ux%ux%u, array_size=%u, last_level=%u, "
 560                         "nr_samples=%u, usage=%u, bind=%x, flags=%x",
 561                         tmpl->target, util_format_name(format),
 562                         tmpl->width0, tmpl->height0, tmpl->depth0,
 563                         tmpl->array_size, tmpl->last_level, tmpl->nr_samples,
 564                         tmpl->usage, tmpl->bind, tmpl->flags);
 565
 566         if (!rsc)
 567                 return NULL;
 568
 569         *prsc = *tmpl;
 570
 571         pipe_reference_init(&prsc->reference, 1);
 572         list_inithead(&rsc->list);
 573         prsc->screen = pscreen;
 574
 575         util_range_init(&rsc->valid_buffer_range);
 576
 577         rsc->base.vtbl = &fd_resource_vtbl;
 578
 579         if (format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT)
 580                 format = PIPE_FORMAT_Z32_FLOAT;
 581         else if (fd_screen(pscreen)->gpu_id < 400 &&
 582                          util_format_description(format)->layout == UTIL_FORMAT_LAYOUT_RGTC)
 583                 format = PIPE_FORMAT_R8G8B8A8_UNORM;
 584         rsc->internal_format = format;
 585         rsc->cpp = util_format_get_blocksize(format);
 586
 587         assert(rsc->cpp);
 588
 589         alignment = slice_alignment(pscreen, tmpl);
 590         if (is_a4xx(fd_screen(pscreen))) {
 591                 switch (tmpl->target) {
 592                 case PIPE_TEXTURE_3D:
 593                         rsc->layer_first = false;
 594                         break;
 595                 default:
 596                         rsc->layer_first = true;
 597                         alignment = 1;
 598                         break;
 599                 }
 600         }
 601
 602         size = setup_slices(rsc, alignment, format);
 603
 604         if (rsc->layer_first) {
 605                 rsc->layer_size = align(size, 4096);
 606                 size = rsc->layer_size * prsc->array_size;
 607         }
 608
 609         realloc_bo(rsc, size);
 610         if (!rsc->bo)
 611                 goto fail;
 612
 613         /* There is no native Z32F_S8 sampling or rendering format, so this must
 614          * be emulated via two separate textures. The depth texture still keeps
 615          * its Z32F_S8 format though, and we also keep a reference to a separate
 616          * S8 texture.
 617          */
 618         if (tmpl->format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT) {
 619                 struct pipe_resource stencil = *tmpl;
 620                 stencil.format = PIPE_FORMAT_S8_UINT;
 621                 rsc->stencil = fd_resource(fd_resource_create(pscreen, &stencil));
 622                 if (!rsc->stencil)
 623                         goto fail;
 624         }
 625
 626         return prsc;
 627 fail:
 628         fd_resource_destroy(pscreen, prsc);
 629         return NULL;
 630 }
 631
 632 /**
 633  * Create a texture from a winsys_handle. The handle is often created in
 634  * another process by first creating a pipe texture and then calling
 635  * resource_get_handle.
 636  */
 637 static struct pipe_resource *
 638 fd_resource_from_handle(struct pipe_screen *pscreen,
 639                 const struct pipe_resource *tmpl,
 640                 struct winsys_handle *handle, unsigned usage)
 641 {
 642         struct fd_resource *rsc = CALLOC_STRUCT(fd_resource);
 643         struct fd_resource_slice *slice = &rsc->slices[0];
 644         struct pipe_resource *prsc = &rsc->base.b;
 645
 646         DBG("target=%d, format=%s, %ux%ux%u, array_size=%u, last_level=%u, "
 647                         "nr_samples=%u, usage=%u, bind=%x, flags=%x",
 648                         tmpl->target, util_format_name(tmpl->format),
 649                         tmpl->width0, tmpl->height0, tmpl->depth0,
 650                         tmpl->array_size, tmpl->last_level, tmpl->nr_samples,
 651                         tmpl->usage, tmpl->bind, tmpl->flags);
 652
 653         if (!rsc)
 654                 return NULL;
 655
 656         *prsc = *tmpl;
 657
 658         pipe_reference_init(&prsc->reference, 1);
 659         list_inithead(&rsc->list);
 660         prsc->screen = pscreen;
 661
 662         util_range_init(&rsc->valid_buffer_range);
 663
 664         rsc->bo = fd_screen_bo_from_handle(pscreen, handle, &slice->pitch);
 665         if (!rsc->bo)
 666                 goto fail;
 667
 668         rsc->base.vtbl = &fd_resource_vtbl;
 669         rsc->cpp = util_format_get_blocksize(tmpl->format);
 670         slice->pitch /= rsc->cpp;
 671         slice->offset = handle->offset;
 672
 673         assert(rsc->cpp);
 674
 675         return prsc;
 676
 677 fail:
 678         fd_resource_destroy(pscreen, prsc);
 679         return NULL;
 680 }
 681
 682 static void fd_blitter_pipe_begin(struct fd_context *ctx, bool render_cond);
 683 static void fd_blitter_pipe_end(struct fd_context *ctx);
 684
 685 /**
 686  * _copy_region using pipe (3d engine)
 687  */
 688 static bool
 689 fd_blitter_pipe_copy_region(struct fd_context *ctx,
 690                 struct pipe_resource *dst,
 691                 unsigned dst_level,
 692                 unsigned dstx, unsigned dsty, unsigned dstz,
 693                 struct pipe_resource *src,
 694                 unsigned src_level,
 695                 const struct pipe_box *src_box)
 696 {
 697         /* not until we allow rendertargets to be buffers */
 698         if (dst->target == PIPE_BUFFER || src->target == PIPE_BUFFER)
 699                 return false;
 700
 701         if (!util_blitter_is_copy_supported(ctx->blitter, dst, src))
 702                 return false;
 703
 704         fd_blitter_pipe_begin(ctx, false);
 705         util_blitter_copy_texture(ctx->blitter,
 706                         dst, dst_level, dstx, dsty, dstz,
 707                         src, src_level, src_box);
 708         fd_blitter_pipe_end(ctx);
 709
 710         return true;
 711 }
 712
 713 /**
 714  * Copy a block of pixels from one resource to another.
 715  * The resource must be of the same format.
 716  * Resources with nr_samples > 1 are not allowed.
 717  */
 718 static void
 719 fd_resource_copy_region(struct pipe_context *pctx,
 720                 struct pipe_resource *dst,
 721                 unsigned dst_level,
 722                 unsigned dstx, unsigned dsty, unsigned dstz,
 723                 struct pipe_resource *src,
 724                 unsigned src_level,
 725                 const struct pipe_box *src_box)
 726 {
 727         struct fd_context *ctx = fd_context(pctx);
 728
 729         /* TODO if we have 2d core, or other DMA engine that could be used
 730          * for simple copies and reasonably easily synchronized with the 3d
 731          * core, this is where we'd plug it in..
 732          */
 733
 734         /* try blit on 3d pipe: */
 735         if (fd_blitter_pipe_copy_region(ctx,
 736                         dst, dst_level, dstx, dsty, dstz,
 737                         src, src_level, src_box))
 738                 return;
 739
 740         /* else fallback to pure sw: */
 741         util_resource_copy_region(pctx,
 742                         dst, dst_level, dstx, dsty, dstz,
 743                         src, src_level, src_box);
 744 }
 745
 746 bool
 747 fd_render_condition_check(struct pipe_context *pctx)
 748 {
 749         struct fd_context *ctx = fd_context(pctx);
 750
 751         if (!ctx->cond_query)
 752                 return true;
 753
 754         union pipe_query_result res = { 0 };
 755         bool wait =
 756                 ctx->cond_mode != PIPE_RENDER_COND_NO_WAIT &&
 757                 ctx->cond_mode != PIPE_RENDER_COND_BY_REGION_NO_WAIT;
 758
 759         if (pctx->get_query_result(pctx, ctx->cond_query, wait, &res))
 760                         return (bool)res.u64 != ctx->cond_cond;
 761
 762         return true;
 763 }
 764
 765 /**
 766  * Optimal hardware path for blitting pixels.
 767  * Scaling, format conversion, up- and downsampling (resolve) are allowed.
 768  */
 769 static void
 770 fd_blit(struct pipe_context *pctx, const struct pipe_blit_info *blit_info)
 771 {
 772         struct fd_context *ctx = fd_context(pctx);
 773         struct pipe_blit_info info = *blit_info;
 774
 775         if (info.src.resource->nr_samples > 1 &&
 776                         info.dst.resource->nr_samples <= 1 &&
 777                         !util_format_is_depth_or_stencil(info.src.resource->format) &&
 778                         !util_format_is_pure_integer(info.src.resource->format)) {
 779                 DBG("color resolve unimplemented");
 780                 return;
 781         }
 782
 783         if (info.render_condition_enable && !fd_render_condition_check(pctx))
 784                 return;
 785
 786         if (util_try_blit_via_copy_region(pctx, &info)) {
 787                 return; /* done */
 788         }
 789
 790         if (info.mask & PIPE_MASK_S) {
 791                 DBG("cannot blit stencil, skipping");
 792                 info.mask &= ~PIPE_MASK_S;
 793         }
 794
 795         if (!util_blitter_is_blit_supported(ctx->blitter, &info)) {
 796                 DBG("blit unsupported %s -> %s",
 797                                 util_format_short_name(info.src.resource->format),
 798                                 util_format_short_name(info.dst.resource->format));
 799                 return;
 800         }
 801
 802         fd_blitter_pipe_begin(ctx, info.render_condition_enable);
 803         util_blitter_blit(ctx->blitter, &info);
 804         fd_blitter_pipe_end(ctx);
 805 }
 806
 807 static void
 808 fd_blitter_pipe_begin(struct fd_context *ctx, bool render_cond)
 809 {
 810         util_blitter_save_vertex_buffer_slot(ctx->blitter, ctx->vtx.vertexbuf.vb);
 811         util_blitter_save_vertex_elements(ctx->blitter, ctx->vtx.vtx);
 812         util_blitter_save_vertex_shader(ctx->blitter, ctx->prog.vp);
 813         util_blitter_save_so_targets(ctx->blitter, ctx->streamout.num_targets,
 814                         ctx->streamout.targets);
 815         util_blitter_save_rasterizer(ctx->blitter, ctx->rasterizer);
 816         util_blitter_save_viewport(ctx->blitter, &ctx->viewport);
 817         util_blitter_save_scissor(ctx->blitter, &ctx->scissor);
 818         util_blitter_save_fragment_shader(ctx->blitter, ctx->prog.fp);
 819         util_blitter_save_blend(ctx->blitter, ctx->blend);
 820         util_blitter_save_depth_stencil_alpha(ctx->blitter, ctx->zsa);
 821         util_blitter_save_stencil_ref(ctx->blitter, &ctx->stencil_ref);
 822         util_blitter_save_sample_mask(ctx->blitter, ctx->sample_mask);
 823         util_blitter_save_framebuffer(ctx->blitter, &ctx->framebuffer);
 824         util_blitter_save_fragment_sampler_states(ctx->blitter,
 825                         ctx->fragtex.num_samplers,
 826                         (void **)ctx->fragtex.samplers);
 827         util_blitter_save_fragment_sampler_views(ctx->blitter,
 828                         ctx->fragtex.num_textures, ctx->fragtex.textures);
 829         if (!render_cond)
 830                 util_blitter_save_render_condition(ctx->blitter,
 831                         ctx->cond_query, ctx->cond_cond, ctx->cond_mode);
 832
 833         fd_hw_query_set_stage(ctx, ctx->ring, FD_STAGE_BLIT);
 834 }
 835
 836 static void
 837 fd_blitter_pipe_end(struct fd_context *ctx)
 838 {
 839         fd_hw_query_set_stage(ctx, ctx->ring, FD_STAGE_NULL);
 840 }
 841
 842 static void
 843 fd_flush_resource(struct pipe_context *pctx, struct pipe_resource *prsc)
 844 {
 845         struct fd_resource *rsc = fd_resource(prsc);
 846
 847         if (pending(rsc, FD_PENDING_WRITE | FD_PENDING_READ))
 848                 fd_context_render(pctx);
 849 }
 850
 851 void
 852 fd_resource_screen_init(struct pipe_screen *pscreen)
 853 {
 854         pscreen->resource_create = fd_resource_create;
 855         pscreen->resource_from_handle = fd_resource_from_handle;
 856         pscreen->resource_get_handle = u_resource_get_handle_vtbl;
 857         pscreen->resource_destroy = u_resource_destroy_vtbl;
 858 }
 859
 860 void
 861 fd_resource_context_init(struct pipe_context *pctx)
 862 {
 863         pctx->transfer_map = u_transfer_map_vtbl;
 864         pctx->transfer_flush_region = u_transfer_flush_region_vtbl;
 865         pctx->transfer_unmap = u_transfer_unmap_vtbl;
 866         pctx->buffer_subdata = u_default_buffer_subdata;
 867         pctx->texture_subdata = u_default_texture_subdata;
 868         pctx->create_surface = fd_create_surface;
 869         pctx->surface_destroy = fd_surface_destroy;
 870         pctx->resource_copy_region = fd_resource_copy_region;
 871         pctx->blit = fd_blit;
 872         pctx->flush_resource = fd_flush_resource;
 873 }