+ assert(transfer->resource);
+
+ llvmpipe_resource_unmap(transfer->resource,
+ transfer->level,
+ transfer->box.z);
+}
+
+static unsigned int
+llvmpipe_is_resource_referenced( struct pipe_context *pipe,
+ struct pipe_resource *presource,
+ unsigned level, int layer)
+{
+ struct llvmpipe_context *llvmpipe = llvmpipe_context( pipe );
+
+ if (presource->target == PIPE_BUFFER)
+ return PIPE_UNREFERENCED;
+
+ return lp_setup_is_resource_referenced(llvmpipe->setup, presource);
+}
+
+
+
+/**
+ * Create buffer which wraps user-space data.
+ */
+static struct pipe_resource *
+llvmpipe_user_buffer_create(struct pipe_screen *screen,
+ void *ptr,
+ unsigned bytes,
+ unsigned bind_flags)
+{
+ struct llvmpipe_resource *buffer;
+
+ buffer = CALLOC_STRUCT(llvmpipe_resource);
+ if(!buffer)
+ return NULL;
+
+ pipe_reference_init(&buffer->base.reference, 1);
+ buffer->base.screen = screen;
+ buffer->base.format = PIPE_FORMAT_R8_UNORM; /* ?? */
+ buffer->base.bind = bind_flags;
+ buffer->base.usage = PIPE_USAGE_IMMUTABLE;
+ buffer->base.flags = 0;
+ buffer->base.width0 = bytes;
+ buffer->base.height0 = 1;
+ buffer->base.depth0 = 1;
+ buffer->base.array_size = 1;
+ buffer->userBuffer = TRUE;
+ buffer->data = ptr;
+
+ return &buffer->base;
+}
+
+
+/**
+ * Compute size (in bytes) need to store a texture image / mipmap level,
+ * for just one cube face or one 3D texture slice
+ */
+static unsigned
+tex_image_face_size(const struct llvmpipe_resource *lpr, unsigned level,
+ enum lp_texture_layout layout)
+{
+ const unsigned width = u_minify(lpr->base.width0, level);
+ const unsigned height = u_minify(lpr->base.height0, level);
+
+ assert(layout == LP_TEX_LAYOUT_TILED ||
+ layout == LP_TEX_LAYOUT_LINEAR);
+
+ if (layout == LP_TEX_LAYOUT_TILED) {
+ /* for tiled layout, force a 32bpp format */
+ const enum pipe_format format = PIPE_FORMAT_B8G8R8A8_UNORM;
+ const unsigned block_size = util_format_get_blocksize(format);
+ const unsigned nblocksy =
+ util_format_get_nblocksy(format, align(height, TILE_SIZE));
+ const unsigned nblocksx =
+ util_format_get_nblocksx(format, align(width, TILE_SIZE));
+ const unsigned buffer_size = block_size * nblocksy * nblocksx;
+ return buffer_size;
+ }
+ else {
+ /* we already computed this */
+ return lpr->img_stride[level];
+ }
+}
+
+
+/**
+ * Compute size (in bytes) need to store a texture image / mipmap level,
+ * including all cube faces or 3D image slices
+ */
+static unsigned
+tex_image_size(const struct llvmpipe_resource *lpr, unsigned level,
+ enum lp_texture_layout layout)
+{
+ const unsigned buf_size = tex_image_face_size(lpr, level, layout);
+ return buf_size * lpr->num_slices_faces[level];
+}
+
+
+/**
+ * This function encapsulates some complicated logic for determining
+ * how to convert a tile of image data from linear layout to tiled
+ * layout, or vice versa.
+ * \param cur_layout the current tile layout
+ * \param target_layout the desired tile layout
+ * \param usage how the tile will be accessed (R/W vs. read-only, etc)
+ * \param new_layout_return returns the new layout mode
+ * \param convert_return returns TRUE if image conversion is needed
+ */
+static void
+layout_logic(enum lp_texture_layout cur_layout,
+ enum lp_texture_layout target_layout,
+ enum lp_texture_usage usage,
+ enum lp_texture_layout *new_layout_return,
+ boolean *convert)
+{
+ enum lp_texture_layout other_layout, new_layout;
+
+ *convert = FALSE;
+
+ new_layout = 99; /* debug check */
+
+ if (target_layout == LP_TEX_LAYOUT_LINEAR) {
+ other_layout = LP_TEX_LAYOUT_TILED;
+ }
+ else {
+ assert(target_layout == LP_TEX_LAYOUT_TILED);
+ other_layout = LP_TEX_LAYOUT_LINEAR;
+ }
+
+ new_layout = target_layout; /* may get changed below */
+
+ if (cur_layout == LP_TEX_LAYOUT_BOTH) {
+ if (usage == LP_TEX_USAGE_READ) {
+ new_layout = LP_TEX_LAYOUT_BOTH;
+ }
+ }
+ else if (cur_layout == other_layout) {
+ if (usage != LP_TEX_USAGE_WRITE_ALL) {
+ /* need to convert tiled data to linear or vice versa */
+ *convert = TRUE;
+
+ if (usage == LP_TEX_USAGE_READ)
+ new_layout = LP_TEX_LAYOUT_BOTH;
+ }
+ }
+ else {
+ assert(cur_layout == LP_TEX_LAYOUT_NONE ||
+ cur_layout == target_layout);
+ }
+
+ assert(new_layout == LP_TEX_LAYOUT_BOTH ||
+ new_layout == target_layout);
+
+ *new_layout_return = new_layout;
+}
+
+
+/**
+ * Return pointer to a 2D texture image/face/slice.
+ * No tiled/linear conversion is done.
+ */
+ubyte *
+llvmpipe_get_texture_image_address(struct llvmpipe_resource *lpr,
+ unsigned face_slice, unsigned level,
+ enum lp_texture_layout layout)
+{
+ struct llvmpipe_texture_image *img;
+ unsigned offset;
+
+ if (layout == LP_TEX_LAYOUT_LINEAR) {
+ img = &lpr->linear[level];
+ }
+ else {
+ assert (layout == LP_TEX_LAYOUT_TILED);
+ img = &lpr->tiled[level];
+ }
+
+ if (face_slice > 0)
+ offset = face_slice * tex_image_face_size(lpr, level, layout);
+ else
+ offset = 0;
+
+ return (ubyte *) img->data + offset;
+}
+
+
+static INLINE enum lp_texture_layout
+llvmpipe_get_texture_tile_layout(const struct llvmpipe_resource *lpr,
+ unsigned face_slice, unsigned level,
+ unsigned x, unsigned y)
+{
+ uint i;
+ assert(resource_is_texture(&lpr->base));
+ assert(x < lpr->tiles_per_row[level]);
+ i = face_slice * lpr->tiles_per_image[level]
+ + y * lpr->tiles_per_row[level] + x;
+ return lpr->layout[level][i];
+}
+
+
+static INLINE void
+llvmpipe_set_texture_tile_layout(struct llvmpipe_resource *lpr,
+ unsigned face_slice, unsigned level,
+ unsigned x, unsigned y,
+ enum lp_texture_layout layout)
+{
+ uint i;
+ assert(resource_is_texture(&lpr->base));
+ assert(x < lpr->tiles_per_row[level]);
+ i = face_slice * lpr->tiles_per_image[level]
+ + y * lpr->tiles_per_row[level] + x;
+ lpr->layout[level][i] = layout;
+}
+
+
+/**
+ * Set the layout mode for all tiles in a particular image.
+ */
+static INLINE void
+llvmpipe_set_texture_image_layout(struct llvmpipe_resource *lpr,
+ unsigned face_slice, unsigned level,
+ unsigned width_t, unsigned height_t,
+ enum lp_texture_layout layout)
+{
+ const unsigned start = face_slice * lpr->tiles_per_image[level];
+ unsigned i;
+
+ for (i = 0; i < width_t * height_t; i++) {
+ lpr->layout[level][start + i] = layout;
+ }
+}
+
+
+/**
+ * Allocate storage for a linear or tile texture image (all cube
+ * faces and all 3D slices.
+ */
+static void
+alloc_image_data(struct llvmpipe_resource *lpr, unsigned level,
+ enum lp_texture_layout layout)
+{
+ uint alignment = MAX2(16, util_cpu_caps.cacheline);
+
+ if (lpr->dt)
+ assert(level == 0);
+
+ if (layout == LP_TEX_LAYOUT_TILED) {
+ /* tiled data is stored in regular memory */
+ uint buffer_size = tex_image_size(lpr, level, layout);
+ lpr->tiled[level].data = align_malloc(buffer_size, alignment);
+ }
+ else {
+ assert(layout == LP_TEX_LAYOUT_LINEAR);
+ if (lpr->dt) {
+ /* we get the linear memory from the winsys */
+ struct llvmpipe_screen *screen = llvmpipe_screen(lpr->base.screen);
+ struct sw_winsys *winsys = screen->winsys;
+
+ lpr->linear[0].data =
+ winsys->displaytarget_map(winsys, lpr->dt,
+ PIPE_TRANSFER_READ_WRITE);
+ }
+ else {
+ /* not a display target - allocate regular memory */
+ uint buffer_size = tex_image_size(lpr, level, LP_TEX_LAYOUT_LINEAR);
+ lpr->linear[level].data = align_malloc(buffer_size, alignment);
+ }
+ }
+}
+
+
+
+/**
+ * Return pointer to texture image data (either linear or tiled layout)
+ * for a particular cube face or 3D texture slice.
+ *
+ * \param face_slice the cube face or 3D slice of interest
+ * \param usage one of LP_TEX_USAGE_READ/WRITE_ALL/READ_WRITE
+ * \param layout either LP_TEX_LAYOUT_LINEAR or _TILED or _NONE
+ */
+void *
+llvmpipe_get_texture_image(struct llvmpipe_resource *lpr,
+ unsigned face_slice, unsigned level,
+ enum lp_texture_usage usage,
+ enum lp_texture_layout layout)
+{
+ /*
+ * 'target' refers to the image which we're retrieving (either in
+ * tiled or linear layout).
+ * 'other' refers to the same image but in the other layout. (it may
+ * or may not exist.
+ */
+ struct llvmpipe_texture_image *target_img;
+ struct llvmpipe_texture_image *other_img;
+ void *target_data;
+ void *other_data;
+ const unsigned width = u_minify(lpr->base.width0, level);
+ const unsigned height = u_minify(lpr->base.height0, level);
+ const unsigned width_t = align(width, TILE_SIZE) / TILE_SIZE;
+ const unsigned height_t = align(height, TILE_SIZE) / TILE_SIZE;
+ enum lp_texture_layout other_layout;
+ boolean only_allocate;
+
+ assert(layout == LP_TEX_LAYOUT_NONE ||
+ layout == LP_TEX_LAYOUT_TILED ||
+ layout == LP_TEX_LAYOUT_LINEAR);
+
+ assert(usage == LP_TEX_USAGE_READ ||
+ usage == LP_TEX_USAGE_READ_WRITE ||
+ usage == LP_TEX_USAGE_WRITE_ALL);
+
+ /* check for the special case of layout == LP_TEX_LAYOUT_NONE */
+ if (layout == LP_TEX_LAYOUT_NONE) {
+ only_allocate = TRUE;
+ layout = LP_TEX_LAYOUT_TILED;
+ }
+ else {
+ only_allocate = FALSE;
+ }
+
+ if (lpr->dt) {
+ assert(lpr->linear[level].data);
+ }
+
+ /* which is target? which is other? */
+ if (layout == LP_TEX_LAYOUT_LINEAR) {
+ target_img = &lpr->linear[level];
+ other_img = &lpr->tiled[level];
+ other_layout = LP_TEX_LAYOUT_TILED;
+ }
+ else {
+ target_img = &lpr->tiled[level];
+ other_img = &lpr->linear[level];
+ other_layout = LP_TEX_LAYOUT_LINEAR;
+ }
+
+ target_data = target_img->data;
+ other_data = other_img->data;
+
+ if (!target_data) {
+ /* allocate memory for the target image now */
+ alloc_image_data(lpr, level, layout);
+ target_data = target_img->data;
+ }
+
+ if (face_slice > 0) {
+ unsigned target_offset, other_offset;
+
+ target_offset = face_slice * tex_image_face_size(lpr, level, layout);
+ other_offset = face_slice * tex_image_face_size(lpr, level, other_layout);
+ if (target_data) {
+ target_data = (uint8_t *) target_data + target_offset;
+ }
+ if (other_data) {
+ other_data = (uint8_t *) other_data + other_offset;
+ }
+ }
+
+ if (only_allocate) {
+ /* Just allocating tiled memory. Don't initialize it from the
+ * linear data if it exists.
+ */
+ return target_data;
+ }
+
+ if (other_data) {
+ /* may need to convert other data to the requested layout */
+ enum lp_texture_layout new_layout;
+ unsigned x, y;
+
+ /* loop over all image tiles, doing layout conversion where needed */
+ for (y = 0; y < height_t; y++) {
+ for (x = 0; x < width_t; x++) {
+ enum lp_texture_layout cur_layout =
+ llvmpipe_get_texture_tile_layout(lpr, face_slice, level, x, y);
+ boolean convert;
+
+ layout_logic(cur_layout, layout, usage, &new_layout, &convert);
+
+ if (convert && other_data && target_data) {
+ if (layout == LP_TEX_LAYOUT_TILED) {
+ lp_linear_to_tiled(other_data, target_data,
+ x * TILE_SIZE, y * TILE_SIZE,
+ TILE_SIZE, TILE_SIZE,
+ lpr->base.format,
+ lpr->row_stride[level],
+ lpr->tiles_per_row[level]);
+ }
+ else {
+ assert(layout == LP_TEX_LAYOUT_LINEAR);
+ lp_tiled_to_linear(other_data, target_data,
+ x * TILE_SIZE, y * TILE_SIZE,
+ TILE_SIZE, TILE_SIZE,
+ lpr->base.format,
+ lpr->row_stride[level],
+ lpr->tiles_per_row[level]);
+ }
+ }
+
+ if (new_layout != cur_layout)
+ llvmpipe_set_texture_tile_layout(lpr, face_slice, level, x, y,
+ new_layout);
+ }
+ }
+ }
+ else {
+ /* no other data */
+ llvmpipe_set_texture_image_layout(lpr, face_slice, level,
+ width_t, height_t, layout);
+ }
+
+ return target_data;
+}
+
+
+/**
+ * Return pointer to start of a texture image (1D, 2D, 3D, CUBE).
+ * All cube faces and 3D slices will be converted to the requested
+ * layout if needed.
+ * This is typically used when we're about to sample from a texture.
+ */
+void *
+llvmpipe_get_texture_image_all(struct llvmpipe_resource *lpr,
+ unsigned level,
+ enum lp_texture_usage usage,
+ enum lp_texture_layout layout)
+{
+ const int slices = lpr->num_slices_faces[level];
+ int slice;
+ void *map = NULL;
+
+ assert(slices > 0);
+
+ for (slice = slices - 1; slice >= 0; slice--) {
+ map = llvmpipe_get_texture_image(lpr, slice, level, usage, layout);
+ }
+
+ return map;
+}
+
+
+/**
+ * Get pointer to a linear image (not the tile!) where the tile at (x,y)
+ * is known to be in linear layout.
+ * Conversion from tiled to linear will be done if necessary.
+ * \return pointer to start of image/face (not the tile)
+ */
+ubyte *
+llvmpipe_get_texture_tile_linear(struct llvmpipe_resource *lpr,
+ unsigned face_slice, unsigned level,
+ enum lp_texture_usage usage,
+ unsigned x, unsigned y)
+{
+ struct llvmpipe_texture_image *linear_img = &lpr->linear[level];
+ enum lp_texture_layout cur_layout, new_layout;
+ const unsigned tx = x / TILE_SIZE, ty = y / TILE_SIZE;
+ boolean convert;
+ uint8_t *tiled_image, *linear_image;
+
+ assert(resource_is_texture(&lpr->base));
+ assert(x % TILE_SIZE == 0);
+ assert(y % TILE_SIZE == 0);
+
+ if (!linear_img->data) {
+ /* allocate memory for the linear image now */
+ alloc_image_data(lpr, level, LP_TEX_LAYOUT_LINEAR);
+ }
+
+ /* compute address of the slice/face of the image that contains the tile */
+ tiled_image = llvmpipe_get_texture_image_address(lpr, face_slice, level,
+ LP_TEX_LAYOUT_TILED);
+ linear_image = llvmpipe_get_texture_image_address(lpr, face_slice, level,
+ LP_TEX_LAYOUT_LINEAR);
+
+ /* get current tile layout and determine if data conversion is needed */
+ cur_layout = llvmpipe_get_texture_tile_layout(lpr, face_slice, level, tx, ty);
+
+ layout_logic(cur_layout, LP_TEX_LAYOUT_LINEAR, usage,
+ &new_layout, &convert);
+
+ if (convert && tiled_image && linear_image) {
+ lp_tiled_to_linear(tiled_image, linear_image,
+ x, y, TILE_SIZE, TILE_SIZE, lpr->base.format,
+ lpr->row_stride[level],
+ lpr->tiles_per_row[level]);
+ }
+
+ if (new_layout != cur_layout)
+ llvmpipe_set_texture_tile_layout(lpr, face_slice, level, tx, ty, new_layout);
+
+ return linear_image;
+}
+
+
+/**
+ * Get pointer to tiled data for rendering.
+ * \return pointer to the tiled data at the given tile position
+ */
+ubyte *
+llvmpipe_get_texture_tile(struct llvmpipe_resource *lpr,
+ unsigned face_slice, unsigned level,
+ enum lp_texture_usage usage,
+ unsigned x, unsigned y)
+{
+ struct llvmpipe_texture_image *tiled_img = &lpr->tiled[level];
+ enum lp_texture_layout cur_layout, new_layout;
+ const unsigned tx = x / TILE_SIZE, ty = y / TILE_SIZE;
+ boolean convert;
+ uint8_t *tiled_image, *linear_image;
+ unsigned tile_offset;
+
+ assert(x % TILE_SIZE == 0);
+ assert(y % TILE_SIZE == 0);
+
+ if (!tiled_img->data) {
+ /* allocate memory for the tiled image now */
+ alloc_image_data(lpr, level, LP_TEX_LAYOUT_TILED);
+ }
+
+ /* compute address of the slice/face of the image that contains the tile */
+ tiled_image = llvmpipe_get_texture_image_address(lpr, face_slice, level,
+ LP_TEX_LAYOUT_TILED);
+ linear_image = llvmpipe_get_texture_image_address(lpr, face_slice, level,
+ LP_TEX_LAYOUT_LINEAR);
+
+ /* get current tile layout and see if we need to convert the data */
+ cur_layout = llvmpipe_get_texture_tile_layout(lpr, face_slice, level, tx, ty);
+
+ layout_logic(cur_layout, LP_TEX_LAYOUT_TILED, usage, &new_layout, &convert);
+ if (convert && linear_image && tiled_image) {
+ lp_linear_to_tiled(linear_image, tiled_image,
+ x, y, TILE_SIZE, TILE_SIZE, lpr->base.format,
+ lpr->row_stride[level],
+ lpr->tiles_per_row[level]);
+ }
+
+ if (!tiled_image)
+ return NULL;
+
+ if (new_layout != cur_layout)
+ llvmpipe_set_texture_tile_layout(lpr, face_slice, level, tx, ty, new_layout);