X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Fmesa%2Fstate_tracker%2Fst_cb_readpixels.c;h=f549852377589a5154250826569e5c420ab33ae1;hb=95415a54c007a909f0fb9d1b2567beecbd73d403;hp=d9e9a527f63ae593915cc2fe39ea7fe6df0a7aa5;hpb=1218430e1200a08cd64b6555d3fd1fd0274ad9e5;p=mesa.git diff --git a/src/mesa/state_tracker/st_cb_readpixels.c b/src/mesa/state_tracker/st_cb_readpixels.c index d9e9a527f63..f5498523775 100644 --- a/src/mesa/state_tracker/st_cb_readpixels.c +++ b/src/mesa/state_tracker/st_cb_readpixels.c @@ -1,8 +1,8 @@ /************************************************************************** - * - * Copyright 2007 Tungsten Graphics, Inc., Cedar Park, Texas. + * + * Copyright 2007 VMware, Inc. * All Rights Reserved. - * + * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the * "Software"), to deal in the Software without restriction, including @@ -10,535 +10,550 @@ * distribute, sub license, and/or sell copies of the Software, and to * permit persons to whom the Software is furnished to do so, subject to * the following conditions: - * + * * The above copyright notice and this permission notice (including the * next paragraph) shall be included in all copies or substantial portions * of the Software. - * + * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. - * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR + * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - * - **************************************************************************/ - - -/** - * glReadPixels interface to pipe * - * \author Brian Paul - */ - + **************************************************************************/ -#include "main/imports.h" #include "main/bufferobj.h" -#include "main/context.h" #include "main/image.h" +#include "main/pbo.h" -#include "pipe/p_context.h" -#include "pipe/p_defines.h" +#include "main/readpix.h" +#include "main/enums.h" +#include "main/framebuffer.h" #include "util/u_inlines.h" -#include "util/u_tile.h" +#include "util/format/u_format.h" +#include "cso_cache/cso_context.h" -#include "st_debug.h" -#include "st_context.h" +#include "st_cb_fbo.h" #include "st_atom.h" +#include "st_context.h" #include "st_cb_bitmap.h" #include "st_cb_readpixels.h" -#include "st_cb_fbo.h" +#include "st_debug.h" +#include "state_tracker/st_cb_texture.h" +#include "state_tracker/st_format.h" +#include "state_tracker/st_pbo.h" +#include "state_tracker/st_texture.h" +#include "state_tracker/st_util.h" -/** - * Special case for reading stencil buffer. - * For color/depth we use get_tile(). For stencil, map the stencil buffer. + +/* The readpixels cache caches a blitted staging texture so that back-to-back + * calls to glReadPixels with user pointers require less CPU-GPU synchronization. + * + * Assumptions: + * + * (1) Blits have high synchronization overheads, and it is beneficial to + * use a single blit of the entire framebuffer instead of many smaller + * blits (because the smaller blits cannot be batched, and we have to wait + * for the GPU after each one). + * + * (2) transfer_map implicitly involves a blit as well (for de-tiling, copy + * from VRAM, etc.), so that it is beneficial to replace the + * _mesa_readpixels path as well when possible. + * + * Change this #define to true to fill and use the cache whenever possible + * (this is inefficient and only meant for testing / debugging). */ -void -st_read_stencil_pixels(GLcontext *ctx, GLint x, GLint y, - GLsizei width, GLsizei height, - GLenum format, GLenum type, - const struct gl_pixelstore_attrib *packing, - GLvoid *pixels) +#define ALWAYS_READPIXELS_CACHE false + +static boolean +needs_integer_signed_unsigned_conversion(const struct gl_context *ctx, + GLenum format, GLenum type) { - struct gl_framebuffer *fb = ctx->ReadBuffer; - struct pipe_context *pipe = st_context(ctx)->pipe; - struct st_renderbuffer *strb = st_renderbuffer(fb->_StencilBuffer); - struct pipe_transfer *pt; - ubyte *stmap; - GLint j; - - if (strb->Base.Wrapped) { - strb = st_renderbuffer(strb->Base.Wrapped); + struct gl_renderbuffer *rb = + _mesa_get_read_renderbuffer_for_format(ctx, format); + + assert(rb); + + GLenum srcType = _mesa_get_format_datatype(rb->Format); + + if ((srcType == GL_INT && + (type == GL_UNSIGNED_INT || + type == GL_UNSIGNED_SHORT || + type == GL_UNSIGNED_BYTE)) || + (srcType == GL_UNSIGNED_INT && + (type == GL_INT || + type == GL_SHORT || + type == GL_BYTE))) { + return TRUE; } - if (st_fb_orientation(ctx->DrawBuffer) == Y_0_TOP) { - y = ctx->DrawBuffer->Height - y - height; + return FALSE; +} + +static bool +try_pbo_readpixels(struct st_context *st, struct st_renderbuffer *strb, + bool invert_y, + GLint x, GLint y, GLsizei width, GLsizei height, + enum pipe_format src_format, enum pipe_format dst_format, + const struct gl_pixelstore_attrib *pack, void *pixels) +{ + struct pipe_context *pipe = st->pipe; + struct pipe_screen *screen = pipe->screen; + struct cso_context *cso = st->cso_context; + struct pipe_surface *surface = strb->surface; + struct pipe_resource *texture = strb->texture; + const struct util_format_description *desc; + struct st_pbo_addresses addr; + struct pipe_framebuffer_state fb; + enum pipe_texture_target view_target; + bool success = false; + + if (texture->nr_samples > 1) + return false; + + if (!screen->is_format_supported(screen, dst_format, PIPE_BUFFER, 0, 0, + PIPE_BIND_SHADER_IMAGE)) + return false; + + desc = util_format_description(dst_format); + + /* Compute PBO addresses */ + addr.bytes_per_pixel = desc->block.bits / 8; + addr.xoffset = x; + addr.yoffset = y; + addr.width = width; + addr.height = height; + addr.depth = 1; + if (!st_pbo_addresses_pixelstore(st, GL_TEXTURE_2D, false, pack, pixels, &addr)) + return false; + + cso_save_state(cso, (CSO_BIT_FRAGMENT_SAMPLER_VIEWS | + CSO_BIT_FRAGMENT_SAMPLERS | + CSO_BIT_FRAGMENT_IMAGE0 | + CSO_BIT_BLEND | + CSO_BIT_VERTEX_ELEMENTS | + CSO_BIT_AUX_VERTEX_BUFFER_SLOT | + CSO_BIT_FRAMEBUFFER | + CSO_BIT_VIEWPORT | + CSO_BIT_RASTERIZER | + CSO_BIT_DEPTH_STENCIL_ALPHA | + CSO_BIT_STREAM_OUTPUTS | + (st->active_queries ? CSO_BIT_PAUSE_QUERIES : 0) | + CSO_BIT_SAMPLE_MASK | + CSO_BIT_MIN_SAMPLES | + CSO_BIT_RENDER_CONDITION | + CSO_BITS_ALL_SHADERS)); + cso_save_constant_buffer_slot0(cso, PIPE_SHADER_FRAGMENT); + + cso_set_sample_mask(cso, ~0); + cso_set_min_samples(cso, 1); + cso_set_render_condition(cso, NULL, FALSE, 0); + + /* Set up the sampler_view */ + { + struct pipe_sampler_view templ; + struct pipe_sampler_view *sampler_view; + struct pipe_sampler_state sampler = {0}; + const struct pipe_sampler_state *samplers[1] = {&sampler}; + + u_sampler_view_default_template(&templ, texture, src_format); + + switch (texture->target) { + case PIPE_TEXTURE_CUBE: + case PIPE_TEXTURE_CUBE_ARRAY: + view_target = PIPE_TEXTURE_2D_ARRAY; + break; + default: + view_target = texture->target; + break; + } + + templ.target = view_target; + templ.u.tex.first_level = surface->u.tex.level; + templ.u.tex.last_level = templ.u.tex.first_level; + + if (view_target != PIPE_TEXTURE_3D) { + templ.u.tex.first_layer = surface->u.tex.first_layer; + templ.u.tex.last_layer = templ.u.tex.first_layer; + } else { + addr.constants.layer_offset = surface->u.tex.first_layer; + } + + sampler_view = pipe->create_sampler_view(pipe, texture, &templ); + if (sampler_view == NULL) + goto fail; + + cso_set_sampler_views(cso, PIPE_SHADER_FRAGMENT, 1, &sampler_view); + + pipe_sampler_view_reference(&sampler_view, NULL); + + cso_set_samplers(cso, PIPE_SHADER_FRAGMENT, 1, samplers); } - /* Create a read transfer from the renderbuffer's texture */ + /* Set up destination image */ + { + struct pipe_image_view image; + + memset(&image, 0, sizeof(image)); + image.resource = addr.buffer; + image.format = dst_format; + image.access = PIPE_IMAGE_ACCESS_WRITE; + image.shader_access = PIPE_IMAGE_ACCESS_WRITE; + image.u.buf.offset = addr.first_element * addr.bytes_per_pixel; + image.u.buf.size = (addr.last_element - addr.first_element + 1) * + addr.bytes_per_pixel; + + cso_set_shader_images(cso, PIPE_SHADER_FRAGMENT, 0, 1, &image); + } - pt = pipe_get_transfer(pipe, strb->texture, - 0, 0, 0, /* face, level, zslice */ - PIPE_TRANSFER_READ, - x, y, width, height); + /* Set up no-attachment framebuffer */ + memset(&fb, 0, sizeof(fb)); + fb.width = surface->width; + fb.height = surface->height; + fb.samples = 1; + fb.layers = 1; + cso_set_framebuffer(cso, &fb); - /* map the stencil buffer */ - stmap = pipe_transfer_map(pipe, pt); + /* Any blend state would do. Set this just to prevent drivers having + * blend == NULL. + */ + cso_set_blend(cso, &st->pbo.upload_blend); - /* width should never be > MAX_WIDTH since we did clipping earlier */ - ASSERT(width <= MAX_WIDTH); + cso_set_viewport_dims(cso, fb.width, fb.height, invert_y); - /* process image row by row */ - for (j = 0; j < height; j++) { - GLvoid *dest; - GLstencil sValues[MAX_WIDTH]; - GLfloat zValues[MAX_WIDTH]; - GLint srcY; + if (invert_y) + st_pbo_addresses_invert_y(&addr, fb.height); - if (st_fb_orientation(ctx->DrawBuffer) == Y_0_TOP) { - srcY = height - j - 1; - } - else { - srcY = j; - } + { + struct pipe_depth_stencil_alpha_state dsa; + memset(&dsa, 0, sizeof(dsa)); + cso_set_depth_stencil_alpha(cso, &dsa); + } - /* get stencil (and Z) values */ - switch (pt->resource->format) { - case PIPE_FORMAT_S8_USCALED: - { - const ubyte *src = stmap + srcY * pt->stride; - memcpy(sValues, src, width); - } - break; - case PIPE_FORMAT_Z24_UNORM_S8_USCALED: - if (format == GL_DEPTH_STENCIL) { - const uint *src = (uint *) (stmap + srcY * pt->stride); - const GLfloat scale = 1.0f / (0xffffff); - GLint k; - for (k = 0; k < width; k++) { - sValues[k] = src[k] >> 24; - zValues[k] = (src[k] & 0xffffff) * scale; - } - } - else { - const uint *src = (uint *) (stmap + srcY * pt->stride); - GLint k; - for (k = 0; k < width; k++) { - sValues[k] = src[k] >> 24; - } - } - break; - case PIPE_FORMAT_S8_USCALED_Z24_UNORM: - if (format == GL_DEPTH_STENCIL) { - const uint *src = (uint *) (stmap + srcY * pt->stride); - const GLfloat scale = 1.0f / (0xffffff); - GLint k; - for (k = 0; k < width; k++) { - sValues[k] = src[k] & 0xff; - zValues[k] = (src[k] >> 8) * scale; - } - } - else { - const uint *src = (uint *) (stmap + srcY * pt->stride); - GLint k; - for (k = 0; k < width; k++) { - sValues[k] = src[k] & 0xff; - } - } - break; - default: - assert(0); - } + /* Set up the fragment shader */ + { + void *fs = st_pbo_get_download_fs(st, view_target, src_format, dst_format); + if (!fs) + goto fail; - /* store */ - dest = _mesa_image_address2d(packing, pixels, width, height, - format, type, j, 0); - if (format == GL_DEPTH_STENCIL) { - _mesa_pack_depth_stencil_span(ctx, width, dest, - zValues, sValues, packing); - } - else { - _mesa_pack_stencil_span(ctx, width, type, dest, sValues, packing); - } + cso_set_fragment_shader_handle(cso, fs); } - /* unmap the stencil buffer */ - pipe_transfer_unmap(pipe, pt); - pipe->transfer_destroy(pipe, pt); -} + success = st_pbo_draw(st, &addr, fb.width, fb.height); + /* Buffer written via shader images needs explicit synchronization. */ + pipe->memory_barrier(pipe, PIPE_BARRIER_ALL); -/** - * Return renderbuffer to use for reading color pixels for glRead/CopyPixel - * commands. - */ -struct st_renderbuffer * -st_get_color_read_renderbuffer(GLcontext *ctx) -{ - struct gl_framebuffer *fb = ctx->ReadBuffer; - struct st_renderbuffer *strb = - st_renderbuffer(fb->_ColorReadBuffer); +fail: + cso_restore_state(cso); + cso_restore_constant_buffer_slot0(cso, PIPE_SHADER_FRAGMENT); - return strb; + return success; } - /** - * Try to do glReadPixels in a fast manner for common cases. - * \return GL_TRUE for success, GL_FALSE for failure + * Create a staging texture and blit the requested region to it. */ -static GLboolean -st_fast_readpixels(GLcontext *ctx, struct st_renderbuffer *strb, +static struct pipe_resource * +blit_to_staging(struct st_context *st, struct st_renderbuffer *strb, + bool invert_y, GLint x, GLint y, GLsizei width, GLsizei height, - GLenum format, GLenum type, - const struct gl_pixelstore_attrib *pack, - GLvoid *dest) + GLenum format, + enum pipe_format src_format, enum pipe_format dst_format) { - enum combination { - A8R8G8B8_UNORM_TO_RGBA_UBYTE, - A8R8G8B8_UNORM_TO_RGB_UBYTE, - A8R8G8B8_UNORM_TO_BGRA_UINT - } combo; - - if (ctx->_ImageTransferState) - return GL_FALSE; - - if (strb->format == PIPE_FORMAT_B8G8R8A8_UNORM && - format == GL_RGBA && type == GL_UNSIGNED_BYTE) { - combo = A8R8G8B8_UNORM_TO_RGBA_UBYTE; - } - else if (strb->format == PIPE_FORMAT_B8G8R8A8_UNORM && - format == GL_RGB && type == GL_UNSIGNED_BYTE) { - combo = A8R8G8B8_UNORM_TO_RGB_UBYTE; - } - else if (strb->format == PIPE_FORMAT_B8G8R8A8_UNORM && - format == GL_BGRA && type == GL_UNSIGNED_INT_8_8_8_8_REV) { - combo = A8R8G8B8_UNORM_TO_BGRA_UINT; - } - else { - return GL_FALSE; + struct pipe_context *pipe = st->pipe; + struct pipe_screen *screen = pipe->screen; + struct pipe_resource dst_templ; + struct pipe_resource *dst; + struct pipe_blit_info blit; + + /* We are creating a texture of the size of the region being read back. + * Need to check for NPOT texture support. */ + if (!screen->get_param(screen, PIPE_CAP_NPOT_TEXTURES) && + (!util_is_power_of_two_or_zero(width) || + !util_is_power_of_two_or_zero(height))) + return NULL; + + /* create the destination texture */ + memset(&dst_templ, 0, sizeof(dst_templ)); + dst_templ.target = PIPE_TEXTURE_2D; + dst_templ.format = dst_format; + if (util_format_is_depth_or_stencil(dst_format)) + dst_templ.bind |= PIPE_BIND_DEPTH_STENCIL; + else + dst_templ.bind |= PIPE_BIND_RENDER_TARGET; + dst_templ.usage = PIPE_USAGE_STAGING; + + st_gl_texture_dims_to_pipe_dims(GL_TEXTURE_2D, width, height, 1, + &dst_templ.width0, &dst_templ.height0, + &dst_templ.depth0, &dst_templ.array_size); + + dst = screen->resource_create(screen, &dst_templ); + if (!dst) + return NULL; + + memset(&blit, 0, sizeof(blit)); + blit.src.resource = strb->texture; + blit.src.level = strb->surface->u.tex.level; + blit.src.format = src_format; + blit.dst.resource = dst; + blit.dst.level = 0; + blit.dst.format = dst->format; + blit.src.box.x = x; + blit.dst.box.x = 0; + blit.src.box.y = y; + blit.dst.box.y = 0; + blit.src.box.z = strb->surface->u.tex.first_layer; + blit.dst.box.z = 0; + blit.src.box.width = blit.dst.box.width = width; + blit.src.box.height = blit.dst.box.height = height; + blit.src.box.depth = blit.dst.box.depth = 1; + blit.mask = st_get_blit_mask(strb->Base._BaseFormat, format); + blit.filter = PIPE_TEX_FILTER_NEAREST; + blit.scissor_enable = FALSE; + + if (invert_y) { + blit.src.box.y = strb->Base.Height - blit.src.box.y; + blit.src.box.height = -blit.src.box.height; } - /*printf("st_fast_readpixels combo %d\n", (GLint) combo);*/ - - { - struct pipe_context *pipe = st_context(ctx)->pipe; - struct pipe_transfer *trans; - const GLubyte *map; - GLubyte *dst; - GLint row, col, dy, dstStride; - - if (st_fb_orientation(ctx->ReadBuffer) == Y_0_TOP) { - /* convert GL Y to Gallium Y */ - y = strb->texture->height0 - y - height; - } - - trans = pipe_get_transfer(pipe, strb->texture, - 0, 0, 0, /* face, level, zslice */ - PIPE_TRANSFER_READ, - x, y, width, height); - if (!trans) { - return GL_FALSE; - } + /* blit */ + st->pipe->blit(st->pipe, &blit); - map = pipe_transfer_map(pipe, trans); - if (!map) { - pipe->transfer_destroy(pipe, trans); - return GL_FALSE; - } + return dst; +} - /* We always write to the user/dest buffer from low addr to high addr - * but the read order depends on renderbuffer orientation - */ - if (st_fb_orientation(ctx->ReadBuffer) == Y_0_TOP) { - /* read source rows from bottom to top */ - y = height - 1; - dy = -1; - } - else { - /* read source rows from top to bottom */ - y = 0; - dy = 1; - } +static struct pipe_resource * +try_cached_readpixels(struct st_context *st, struct st_renderbuffer *strb, + bool invert_y, + GLsizei width, GLsizei height, + GLenum format, + enum pipe_format src_format, enum pipe_format dst_format) +{ + struct pipe_resource *src = strb->texture; + struct pipe_resource *dst = NULL; + + if (ST_DEBUG & DEBUG_NOREADPIXCACHE) + return NULL; + + /* Reset cache after invalidation or switch of parameters. */ + if (st->readpix_cache.src != src || + st->readpix_cache.dst_format != dst_format || + st->readpix_cache.level != strb->surface->u.tex.level || + st->readpix_cache.layer != strb->surface->u.tex.first_layer) { + pipe_resource_reference(&st->readpix_cache.src, src); + pipe_resource_reference(&st->readpix_cache.cache, NULL); + st->readpix_cache.dst_format = dst_format; + st->readpix_cache.level = strb->surface->u.tex.level; + st->readpix_cache.layer = strb->surface->u.tex.first_layer; + st->readpix_cache.hits = 0; + } - dst = _mesa_image_address2d(pack, dest, width, height, - format, type, 0, 0); - dstStride = _mesa_image_row_stride(pack, width, format, type); - - switch (combo) { - case A8R8G8B8_UNORM_TO_RGBA_UBYTE: - for (row = 0; row < height; row++) { - const GLubyte *src = map + y * trans->stride; - for (col = 0; col < width; col++) { - GLuint pixel = ((GLuint *) src)[col]; - dst[col*4+0] = (pixel >> 16) & 0xff; - dst[col*4+1] = (pixel >> 8) & 0xff; - dst[col*4+2] = (pixel >> 0) & 0xff; - dst[col*4+3] = (pixel >> 24) & 0xff; - } - dst += dstStride; - y += dy; + /* Decide whether to trigger the cache. */ + if (!st->readpix_cache.cache) { + if (!strb->use_readpix_cache && !ALWAYS_READPIXELS_CACHE) { + /* Heuristic: If previous successive calls read at least a fraction + * of the surface _and_ we read again, trigger the cache. + */ + unsigned threshold = MAX2(1, strb->Base.Width * strb->Base.Height / 8); + + if (st->readpix_cache.hits < threshold) { + st->readpix_cache.hits += width * height; + return NULL; } - break; - case A8R8G8B8_UNORM_TO_RGB_UBYTE: - for (row = 0; row < height; row++) { - const GLubyte *src = map + y * trans->stride; - for (col = 0; col < width; col++) { - GLuint pixel = ((GLuint *) src)[col]; - dst[col*3+0] = (pixel >> 16) & 0xff; - dst[col*3+1] = (pixel >> 8) & 0xff; - dst[col*3+2] = (pixel >> 0) & 0xff; - } - dst += dstStride; - y += dy; - } - break; - case A8R8G8B8_UNORM_TO_BGRA_UINT: - for (row = 0; row < height; row++) { - const GLubyte *src = map + y * trans->stride; - memcpy(dst, src, 4 * width); - dst += dstStride; - y += dy; - } - break; - default: - ; /* nothing */ + + strb->use_readpix_cache = true; } - pipe_transfer_unmap(pipe, trans); - pipe->transfer_destroy(pipe, trans); + /* Fill the cache */ + st->readpix_cache.cache = blit_to_staging(st, strb, invert_y, + 0, 0, + strb->Base.Width, + strb->Base.Height, format, + src_format, dst_format); } - return GL_TRUE; -} + /* Return an owning reference to stay consistent with the non-cached path */ + pipe_resource_reference(&dst, st->readpix_cache.cache); + return dst; +} /** - * Do glReadPixels by getting rows from the framebuffer transfer with - * get_tile(). Convert to requested format/type with Mesa image routines. - * Image transfer ops are done in software too. + * This uses a blit to copy the read buffer to a texture format which matches + * the format and type combo and then a fast read-back is done using memcpy. + * We can do arbitrary X/Y/Z/W/0/1 swizzling here as long as there is + * a format which matches the swizzling. + * + * If such a format isn't available, we fall back to _mesa_readpixels. + * + * NOTE: Some drivers use a blit to convert between tiled and linear + * texture layouts during texture uploads/downloads, so the blit + * we do here should be free in such cases. */ static void -st_readpixels(GLcontext *ctx, GLint x, GLint y, GLsizei width, GLsizei height, +st_ReadPixels(struct gl_context *ctx, GLint x, GLint y, + GLsizei width, GLsizei height, GLenum format, GLenum type, const struct gl_pixelstore_attrib *pack, - GLvoid *dest) + void *pixels) { struct st_context *st = st_context(ctx); + struct gl_renderbuffer *rb = + _mesa_get_read_renderbuffer_for_format(ctx, format); + struct st_renderbuffer *strb = st_renderbuffer(rb); struct pipe_context *pipe = st->pipe; - GLfloat temp[MAX_WIDTH][4]; - const GLbitfield transferOps = ctx->_ImageTransferState; - GLsizei i, j; - GLint yStep, dfStride; - GLfloat *df; - struct st_renderbuffer *strb; - struct gl_pixelstore_attrib clippedPacking = *pack; - struct pipe_transfer *trans; - - assert(ctx->ReadBuffer->Width > 0); - - st_validate_state(st); - - /* Do all needed clipping here, so that we can forget about it later */ - if (!_mesa_clip_readpixels(ctx, &x, &y, &width, &height, &clippedPacking)) { - /* The ReadPixels transfer is totally outside the window bounds */ - return; - } - + struct pipe_screen *screen = pipe->screen; + struct pipe_resource *src; + struct pipe_resource *dst = NULL; + enum pipe_format dst_format, src_format; + unsigned bind; + struct pipe_transfer *tex_xfer; + ubyte *map = NULL; + int dst_x, dst_y; + + /* Validate state (to be sure we have up-to-date framebuffer surfaces) + * and flush the bitmap cache prior to reading. */ + st_validate_state(st, ST_PIPELINE_UPDATE_FRAMEBUFFER); st_flush_bitmap_cache(st); - dest = _mesa_map_pbo_dest(ctx, &clippedPacking, dest); - if (!dest) - return; - - if (format == GL_STENCIL_INDEX || - format == GL_DEPTH_STENCIL) { - st_read_stencil_pixels(ctx, x, y, width, height, - format, type, pack, dest); - return; + if (!st->prefer_blit_based_texture_transfer) { + goto fallback; } - else if (format == GL_DEPTH_COMPONENT) { - strb = st_renderbuffer(ctx->ReadBuffer->_DepthBuffer); - if (strb->Base.Wrapped) { - strb = st_renderbuffer(strb->Base.Wrapped); - } + + /* This must be done after state validation. */ + src = strb->texture; + + /* XXX Fallback for depth-stencil formats due to an incomplete + * stencil blit implementation in some drivers. */ + if (format == GL_DEPTH_STENCIL) { + goto fallback; } - else { - /* Read color buffer */ - strb = st_get_color_read_renderbuffer(ctx); + + /* If the base internal format and the texture format don't match, we have + * to use the slow path. */ + if (rb->_BaseFormat != + _mesa_get_format_base_format(rb->Format)) { + goto fallback; } - if (!strb) - return; + if (_mesa_readpixels_needs_slow_path(ctx, format, type, GL_TRUE)) { + goto fallback; + } - /* try a fast-path readpixels before anything else */ - if (st_fast_readpixels(ctx, strb, x, y, width, height, - format, type, pack, dest)) { - /* success! */ - _mesa_unmap_pbo_dest(ctx, &clippedPacking); - return; + /* Convert the source format to what is expected by ReadPixels + * and see if it's supported. */ + src_format = util_format_linear(src->format); + src_format = util_format_luminance_to_red(src_format); + src_format = util_format_intensity_to_red(src_format); + + if (!src_format || + !screen->is_format_supported(screen, src_format, src->target, + src->nr_samples, src->nr_storage_samples, + PIPE_BIND_SAMPLER_VIEW)) { + goto fallback; } - if (format == GL_RGBA && type == GL_FLOAT) { - /* write tile(row) directly into user's buffer */ - df = (GLfloat *) _mesa_image_address2d(&clippedPacking, dest, width, - height, format, type, 0, 0); - dfStride = width * 4; + if (format == GL_DEPTH_COMPONENT || format == GL_DEPTH_STENCIL) + bind = PIPE_BIND_DEPTH_STENCIL; + else + bind = PIPE_BIND_RENDER_TARGET; + + /* Choose the destination format by finding the best match + * for the format+type combo. */ + dst_format = st_choose_matching_format(st, bind, format, type, + pack->SwapBytes); + if (dst_format == PIPE_FORMAT_NONE) { + goto fallback; } - else { - /* write tile(row) into temp row buffer */ - df = (GLfloat *) temp; - dfStride = 0; + + if (st->pbo.download_enabled && pack->BufferObj) { + if (try_pbo_readpixels(st, strb, + st_fb_orientation(ctx->ReadBuffer) == Y_0_TOP, + x, y, width, height, + src_format, dst_format, + pack, pixels)) + return; } - if (st_fb_orientation(ctx->ReadBuffer) == Y_0_TOP) { - /* convert GL Y to Gallium Y */ - y = strb->Base.Height - y - height; + if (needs_integer_signed_unsigned_conversion(ctx, format, type)) { + goto fallback; } - /* Create a read transfer from the renderbuffer's texture */ - trans = pipe_get_transfer(pipe, strb->texture, - 0, 0, 0, /* face, level, zslice */ - PIPE_TRANSFER_READ, - x, y, width, height); + /* Cache a staging texture for back-to-back ReadPixels, to avoid CPU-GPU + * synchronization overhead. + */ + dst = try_cached_readpixels(st, strb, + st_fb_orientation(ctx->ReadBuffer) == Y_0_TOP, + width, height, format, src_format, dst_format); + if (dst) { + dst_x = x; + dst_y = y; + } else { + /* See if the texture format already matches the format and type, + * in which case the memcpy-based fast path will likely be used and + * we don't have to blit. */ + if (_mesa_format_matches_format_and_type(rb->Format, format, + type, pack->SwapBytes, NULL)) { + goto fallback; + } - /* determine bottom-to-top vs. top-to-bottom order */ - if (st_fb_orientation(ctx->ReadBuffer) == Y_0_TOP) { - y = height - 1; - yStep = -1; - } - else { - y = 0; - yStep = 1; + dst = blit_to_staging(st, strb, + st_fb_orientation(ctx->ReadBuffer) == Y_0_TOP, + x, y, width, height, format, + src_format, dst_format); + if (!dst) + goto fallback; + + dst_x = 0; + dst_y = 0; } - if (ST_DEBUG & DEBUG_FALLBACK) - debug_printf("%s: fallback processing\n", __FUNCTION__); + /* map resources */ + pixels = _mesa_map_pbo_dest(ctx, pack, pixels); - /* - * Copy pixels from pipe_transfer to user memory - */ + map = pipe_transfer_map_3d(pipe, dst, 0, PIPE_TRANSFER_READ, + dst_x, dst_y, 0, width, height, 1, &tex_xfer); + if (!map) { + _mesa_unmap_pbo_dest(ctx, pack); + pipe_resource_reference(&dst, NULL); + goto fallback; + } + + /* memcpy data into a user buffer */ { - /* dest of first pixel in client memory */ - GLubyte *dst = _mesa_image_address2d(&clippedPacking, dest, width, - height, format, type, 0, 0); - /* dest row stride */ - const GLint dstStride = _mesa_image_row_stride(&clippedPacking, width, - format, type); - - if (trans->resource->format == PIPE_FORMAT_Z24_UNORM_S8_USCALED || - trans->resource->format == PIPE_FORMAT_Z24X8_UNORM) { - if (format == GL_DEPTH_COMPONENT) { - for (i = 0; i < height; i++) { - GLuint ztemp[MAX_WIDTH]; - GLfloat zfloat[MAX_WIDTH]; - const double scale = 1.0 / ((1 << 24) - 1); - pipe_get_tile_raw(pipe, trans, 0, y, width, 1, ztemp, 0); - y += yStep; - for (j = 0; j < width; j++) { - zfloat[j] = (float) (scale * (ztemp[j] & 0xffffff)); - } - _mesa_pack_depth_span(ctx, width, dst, type, - zfloat, &clippedPacking); - dst += dstStride; - } - } - else { - /* XXX: unreachable code -- should be before st_read_stencil_pixels */ - assert(format == GL_DEPTH_STENCIL_EXT); - for (i = 0; i < height; i++) { - GLuint *zshort = (GLuint *)dst; - pipe_get_tile_raw(pipe, trans, 0, y, width, 1, dst, 0); - y += yStep; - /* Reverse into 24/8 */ - for (j = 0; j < width; j++) { - zshort[j] = (zshort[j] << 8) | (zshort[j] >> 24); - } - dst += dstStride; - } - } - } - else if (trans->resource->format == PIPE_FORMAT_S8_USCALED_Z24_UNORM || - trans->resource->format == PIPE_FORMAT_X8Z24_UNORM) { - if (format == GL_DEPTH_COMPONENT) { - for (i = 0; i < height; i++) { - GLuint ztemp[MAX_WIDTH]; - GLfloat zfloat[MAX_WIDTH]; - const double scale = 1.0 / ((1 << 24) - 1); - pipe_get_tile_raw(pipe, trans, 0, y, width, 1, ztemp, 0); - y += yStep; - for (j = 0; j < width; j++) { - zfloat[j] = (float) (scale * ((ztemp[j] >> 8) & 0xffffff)); - } - _mesa_pack_depth_span(ctx, width, dst, type, - zfloat, &clippedPacking); - dst += dstStride; - } - } - else { - /* XXX: unreachable code -- should be before st_read_stencil_pixels */ - assert(format == GL_DEPTH_STENCIL_EXT); - for (i = 0; i < height; i++) { - pipe_get_tile_raw(pipe, trans, 0, y, width, 1, dst, 0); - y += yStep; - dst += dstStride; - } - } - } - else if (trans->resource->format == PIPE_FORMAT_Z16_UNORM) { - for (i = 0; i < height; i++) { - GLushort ztemp[MAX_WIDTH]; - GLfloat zfloat[MAX_WIDTH]; - const double scale = 1.0 / 0xffff; - pipe_get_tile_raw(pipe, trans, 0, y, width, 1, ztemp, 0); - y += yStep; - for (j = 0; j < width; j++) { - zfloat[j] = (float) (scale * ztemp[j]); - } - _mesa_pack_depth_span(ctx, width, dst, type, - zfloat, &clippedPacking); - dst += dstStride; - } - } - else if (trans->resource->format == PIPE_FORMAT_Z32_UNORM) { - for (i = 0; i < height; i++) { - GLuint ztemp[MAX_WIDTH]; - GLfloat zfloat[MAX_WIDTH]; - const double scale = 1.0 / 0xffffffff; - pipe_get_tile_raw(pipe, trans, 0, y, width, 1, ztemp, 0); - y += yStep; - for (j = 0; j < width; j++) { - zfloat[j] = (float) (scale * ztemp[j]); - } - _mesa_pack_depth_span(ctx, width, dst, type, - zfloat, &clippedPacking); - dst += dstStride; - } - } - else { - /* RGBA format */ - /* Do a row at a time to flip image data vertically */ - for (i = 0; i < height; i++) { - pipe_get_tile_rgba(pipe, trans, 0, y, width, 1, df); - y += yStep; - df += dfStride; - if (!dfStride) { - _mesa_pack_rgba_span_float(ctx, width, temp, format, type, dst, - &clippedPacking, transferOps); - dst += dstStride; - } + const uint bytesPerRow = width * util_format_get_blocksize(dst_format); + const int destStride = _mesa_image_row_stride(pack, width, format, type); + char *dest = _mesa_image_address2d(pack, pixels, + width, height, format, + type, 0, 0); + + if (tex_xfer->stride == bytesPerRow && destStride == bytesPerRow) { + memcpy(dest, map, bytesPerRow * height); + } else { + GLuint row; + + for (row = 0; row < (unsigned) height; row++) { + memcpy(dest, map, bytesPerRow); + map += tex_xfer->stride; + dest += destStride; } } } - pipe->transfer_destroy(pipe, trans); + pipe_transfer_unmap(pipe, tex_xfer); + _mesa_unmap_pbo_dest(ctx, pack); + pipe_resource_reference(&dst, NULL); + return; - _mesa_unmap_pbo_dest(ctx, &clippedPacking); +fallback: + _mesa_readpixels(ctx, x, y, width, height, format, type, pack, pixels); } - void st_init_readpixels_functions(struct dd_function_table *functions) { - functions->ReadPixels = st_readpixels; + functions->ReadPixels = st_ReadPixels; }