X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Fgallium%2Fdrivers%2Fradeonsi%2Fsi_texture.c;h=eb17f6c82c4787117054a2e1a73c01b1d1a58be9;hb=3b143369a55d1b79f7db14dda587e18f6b27c975;hp=e55fd81526478a8d52ae0a2b9a06b4ae0c251228;hpb=20dd75a9268a1adf0a8b47267ba2c5acc55cb207;p=mesa.git

diff --git a/src/gallium/drivers/radeonsi/si_texture.c b/src/gallium/drivers/radeonsi/si_texture.c
index e55fd815264..eb17f6c82c4 100644
--- a/src/gallium/drivers/radeonsi/si_texture.c
+++ b/src/gallium/drivers/radeonsi/si_texture.c
@@ -23,9 +23,9 @@
  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include "radeonsi/si_pipe.h"
-#include "radeonsi/si_query.h"
-#include "util/u_format.h"
+#include "si_pipe.h"
+#include "si_query.h"
+#include "util/format/u_format.h"
 #include "util/u_log.h"
 #include "util/u_memory.h"
 #include "util/u_pack_color.h"
@@ -36,7 +36,9 @@
 #include <errno.h>
 #include <inttypes.h>
 #include "state_tracker/drm_driver.h"
-#include "amd/common/sid.h"
+#include "sid.h"
+#include "amd/addrlib/inc/addrinterface.h"
+#include "drm-uapi/drm_fourcc.h"
 
 static enum radeon_surf_mode
 si_choose_tiling(struct si_screen *sscreen,
@@ -128,8 +130,7 @@ static void si_copy_region_with_blit(struct pipe_context *pipe,
 	blit.dst.box.width = src_box->width;
 	blit.dst.box.height = src_box->height;
 	blit.dst.box.depth = src_box->depth;
-	blit.mask = util_format_get_mask(src->format) &
-		    util_format_get_mask(dst->format);
+	blit.mask = util_format_get_mask(dst->format);
 	blit.filter = PIPE_TEX_FILTER_NEAREST;
 
 	if (blit.mask) {
@@ -145,7 +146,7 @@ static void si_copy_to_staging_texture(struct pipe_context *ctx, struct si_trans
 	struct pipe_resource *dst = &stransfer->staging->b.b;
 	struct pipe_resource *src = transfer->resource;
 
-	if (src->nr_samples > 1) {
+	if (src->nr_samples > 1 || ((struct si_texture*)src)->is_depth) {
 		si_copy_region_with_blit(ctx, dst, 0, 0, 0, 0,
 					   src, transfer->level, &transfer->box);
 		return;
@@ -166,13 +167,18 @@ static void si_copy_from_staging_texture(struct pipe_context *ctx, struct si_tra
 
 	u_box_3d(0, 0, 0, transfer->box.width, transfer->box.height, transfer->box.depth, &sbox);
 
-	if (dst->nr_samples > 1) {
+	if (dst->nr_samples > 1 || ((struct si_texture*)dst)->is_depth) {
 		si_copy_region_with_blit(ctx, dst, transfer->level,
 					   transfer->box.x, transfer->box.y, transfer->box.z,
 					   src, 0, &sbox);
 		return;
 	}
 
+	if (util_format_is_compressed(dst->format)) {
+		sbox.width = util_format_get_nblocksx(dst->format, sbox.width);
+		sbox.height = util_format_get_nblocksx(dst->format, sbox.height);
+	}
+
 	sctx->dma_copy(ctx, dst, transfer->level,
 		       transfer->box.x, transfer->box.y, transfer->box.z,
 		       src, 0, &sbox);
@@ -222,7 +228,6 @@ static int si_init_surface(struct si_screen *sscreen,
 			   const struct pipe_resource *ptex,
 			   enum radeon_surf_mode array_mode,
 			   unsigned pitch_in_bytes_override,
-			   unsigned offset,
 			   bool is_imported,
 			   bool is_scanout,
 			   bool is_flushed_depth,
@@ -232,7 +237,7 @@ static int si_init_surface(struct si_screen *sscreen,
 		util_format_description(ptex->format);
 	bool is_depth, is_stencil;
 	int r;
-	unsigned i, bpe, flags = 0;
+	unsigned bpe, flags = 0;
 
 	is_depth = util_format_has_depth(desc);
 	is_stencil = util_format_has_stencil(desc);
@@ -248,15 +253,17 @@ static int si_init_surface(struct si_screen *sscreen,
 	if (!is_flushed_depth && is_depth) {
 		flags |= RADEON_SURF_ZBUFFER;
 
-		if (tc_compatible_htile &&
-		    (sscreen->info.chip_class >= GFX9 ||
-		     array_mode == RADEON_SURF_MODE_2D)) {
+		if (sscreen->debug_flags & DBG(NO_HYPERZ)) {
+			flags |= RADEON_SURF_NO_HTILE;
+		} else if (tc_compatible_htile &&
+			   (sscreen->info.chip_class >= GFX9 ||
+			    array_mode == RADEON_SURF_MODE_2D)) {
 			/* TC-compatible HTILE only supports Z32_FLOAT.
 			 * GFX9 also supports Z16_UNORM.
-			 * On VI, promote Z16 to Z32. DB->CB copies will convert
+			 * On GFX8, promote Z16 to Z32. DB->CB copies will convert
 			 * the format for transfers.
 			 */
-			if (sscreen->info.chip_class == VI)
+			if (sscreen->info.chip_class == GFX8)
 				bpe = 4;
 
 			flags |= RADEON_SURF_TC_COMPATIBLE_HTILE;
@@ -266,7 +273,7 @@ static int si_init_surface(struct si_screen *sscreen,
 			flags |= RADEON_SURF_SBUFFER;
 	}
 
-	if (sscreen->info.chip_class >= VI &&
+	if (sscreen->info.chip_class >= GFX8 &&
 	    (ptex->flags & SI_RESOURCE_FLAG_DISABLE_DCC ||
 	     ptex->format == PIPE_FORMAT_R9G9B9E5_FLOAT ||
 	     (ptex->nr_samples >= 2 && !sscreen->dcc_msaa_allowed)))
@@ -277,17 +284,29 @@ static int si_init_surface(struct si_screen *sscreen,
 	    bpe == 16 && ptex->nr_samples >= 2)
 		flags |= RADEON_SURF_DISABLE_DCC;
 
-	/* VI: DCC clear for 4x and 8x MSAA array textures unimplemented. */
-	if (sscreen->info.chip_class == VI &&
+	/* GFX8: DCC clear for 4x and 8x MSAA array textures unimplemented. */
+	if (sscreen->info.chip_class == GFX8 &&
 	    ptex->nr_storage_samples >= 4 &&
 	    ptex->array_size > 1)
 		flags |= RADEON_SURF_DISABLE_DCC;
 
 	/* GFX9: DCC clear for 4x and 8x MSAA textures unimplemented. */
-	if (sscreen->info.chip_class >= GFX9 &&
+	if (sscreen->info.chip_class == GFX9 &&
 	    ptex->nr_storage_samples >= 4)
 		flags |= RADEON_SURF_DISABLE_DCC;
 
+	/* TODO: GFX10: DCC causes corruption with MSAA. */
+	if (sscreen->info.chip_class >= GFX10 &&
+	    ptex->nr_storage_samples >= 2)
+		flags |= RADEON_SURF_DISABLE_DCC;
+
+	/* Shared textures must always set up DCC.
+	 * If it's not present, it will be disabled by
+	 * si_get_opaque_metadata later.
+	 */
+	if (!is_imported && (sscreen->debug_flags & DBG(NO_DCC)))
+		flags |= RADEON_SURF_DISABLE_DCC;
+
 	if (ptex->bind & PIPE_BIND_SCANOUT || is_scanout) {
 		/* This should catch bugs in gallium users setting incorrect flags. */
 		assert(ptex->nr_samples <= 1 &&
@@ -303,8 +322,16 @@ static int si_init_surface(struct si_screen *sscreen,
 		flags |= RADEON_SURF_SHAREABLE;
 	if (is_imported)
 		flags |= RADEON_SURF_IMPORTED | RADEON_SURF_SHAREABLE;
-	if (!(ptex->flags & SI_RESOURCE_FLAG_FORCE_TILING))
+	if (!(ptex->flags & SI_RESOURCE_FLAG_FORCE_MSAA_TILING))
 		flags |= RADEON_SURF_OPTIMIZE_FOR_SPACE;
+	if (sscreen->debug_flags & DBG(NO_FMASK))
+		flags |= RADEON_SURF_NO_FMASK;
+
+	if (sscreen->info.chip_class >= GFX10 &&
+	    (ptex->flags & SI_RESOURCE_FLAG_FORCE_MSAA_TILING)) {
+		flags |= RADEON_SURF_FORCE_SWIZZLE_MODE;
+		surface->u.gfx9.surf.swizzle_mode = ADDR_SW_64KB_R_X;
+	}
 
 	r = sscreen->ws->surface_init(sscreen->ws, ptex, flags, bpe,
 				      array_mode, surface);
@@ -317,55 +344,26 @@ static int si_init_surface(struct si_screen *sscreen,
 	if (sscreen->info.chip_class >= GFX9) {
 		if (pitch) {
 			surface->u.gfx9.surf_pitch = pitch;
+			if (ptex->last_level == 0)
+				surface->u.gfx9.surf.epitch = pitch - 1;
 			surface->u.gfx9.surf_slice_size =
 				(uint64_t)pitch * surface->u.gfx9.surf_height * bpe;
 		}
-		surface->u.gfx9.surf_offset = offset;
 	} else {
 		if (pitch) {
 			surface->u.legacy.level[0].nblk_x = pitch;
 			surface->u.legacy.level[0].slice_size_dw =
 				((uint64_t)pitch * surface->u.legacy.level[0].nblk_y * bpe) / 4;
 		}
-		if (offset) {
-			for (i = 0; i < ARRAY_SIZE(surface->u.legacy.level); ++i)
-				surface->u.legacy.level[i].offset += offset;
-		}
 	}
 	return 0;
 }
 
-static void si_texture_init_metadata(struct si_screen *sscreen,
-				     struct si_texture *tex,
-				     struct radeon_bo_metadata *metadata)
-{
-	struct radeon_surf *surface = &tex->surface;
-
-	memset(metadata, 0, sizeof(*metadata));
-
-	if (sscreen->info.chip_class >= GFX9) {
-		metadata->u.gfx9.swizzle_mode = surface->u.gfx9.surf.swizzle_mode;
-	} else {
-		metadata->u.legacy.microtile = surface->u.legacy.level[0].mode >= RADEON_SURF_MODE_1D ?
-					   RADEON_LAYOUT_TILED : RADEON_LAYOUT_LINEAR;
-		metadata->u.legacy.macrotile = surface->u.legacy.level[0].mode >= RADEON_SURF_MODE_2D ?
-					   RADEON_LAYOUT_TILED : RADEON_LAYOUT_LINEAR;
-		metadata->u.legacy.pipe_config = surface->u.legacy.pipe_config;
-		metadata->u.legacy.bankw = surface->u.legacy.bankw;
-		metadata->u.legacy.bankh = surface->u.legacy.bankh;
-		metadata->u.legacy.tile_split = surface->u.legacy.tile_split;
-		metadata->u.legacy.mtilea = surface->u.legacy.mtilea;
-		metadata->u.legacy.num_banks = surface->u.legacy.num_banks;
-		metadata->u.legacy.stride = surface->u.legacy.level[0].nblk_x * surface->bpe;
-		metadata->u.legacy.scanout = (surface->flags & RADEON_SURF_SCANOUT) != 0;
-	}
-}
-
-static void si_surface_import_metadata(struct si_screen *sscreen,
-				       struct radeon_surf *surf,
-				       struct radeon_bo_metadata *metadata,
-				       enum radeon_surf_mode *array_mode,
-				       bool *is_scanout)
+static void si_get_display_metadata(struct si_screen *sscreen,
+				    struct radeon_surf *surf,
+				    struct radeon_bo_metadata *metadata,
+				    enum radeon_surf_mode *array_mode,
+				    bool *is_scanout)
 {
 	if (sscreen->info.chip_class >= GFX9) {
 		if (metadata->u.gfx9.swizzle_mode > 0)
@@ -377,6 +375,11 @@ static void si_surface_import_metadata(struct si_screen *sscreen,
 			      metadata->u.gfx9.swizzle_mode % 4 == 2;
 
 		surf->u.gfx9.surf.swizzle_mode = metadata->u.gfx9.swizzle_mode;
+
+		if (metadata->u.gfx9.dcc_offset_256B) {
+			surf->u.gfx9.display_dcc_pitch_max = metadata->u.gfx9.dcc_pitch_max;
+			assert(metadata->u.gfx9.dcc_independent_64B == 1);
+		}
 	} else {
 		surf->u.legacy.pipe_config = metadata->u.legacy.pipe_config;
 		surf->u.legacy.bankw = metadata->u.legacy.bankw;
@@ -403,7 +406,7 @@ void si_eliminate_fast_color_clear(struct si_context *sctx,
 	struct pipe_context *ctx = &sctx->b;
 
 	if (ctx == sscreen->aux_context)
-		mtx_lock(&sscreen->aux_context_lock);
+		simple_mtx_lock(&sscreen->aux_context_lock);
 
 	unsigned n = sctx->num_decompress_calls;
 	ctx->flush_resource(ctx, &tex->buffer.b.b);
@@ -413,7 +416,7 @@ void si_eliminate_fast_color_clear(struct si_context *sctx,
 		ctx->flush(ctx, NULL, 0);
 
 	if (ctx == sscreen->aux_context)
-		mtx_unlock(&sscreen->aux_context_lock);
+		simple_mtx_unlock(&sscreen->aux_context_lock);
 }
 
 void si_texture_discard_cmask(struct si_screen *sscreen,
@@ -431,7 +434,7 @@ void si_texture_discard_cmask(struct si_screen *sscreen,
 	tex->cb_color_info &= ~S_028C70_FAST_CLEAR(1);
 
 	if (tex->cmask_buffer != &tex->buffer)
-	    r600_resource_reference(&tex->cmask_buffer, NULL);
+	    si_resource_reference(&tex->cmask_buffer, NULL);
 
 	tex->cmask_buffer = NULL;
 
@@ -443,21 +446,30 @@ void si_texture_discard_cmask(struct si_screen *sscreen,
 static bool si_can_disable_dcc(struct si_texture *tex)
 {
 	/* We can't disable DCC if it can be written by another process. */
-	return tex->dcc_offset &&
+	return tex->surface.dcc_offset &&
 	       (!tex->buffer.b.is_shared ||
-		!(tex->buffer.external_usage & PIPE_HANDLE_USAGE_WRITE));
+		!(tex->buffer.external_usage & PIPE_HANDLE_USAGE_FRAMEBUFFER_WRITE));
+}
+
+static void si_texture_zero_dcc_fields(struct si_texture *tex)
+{
+	tex->surface.dcc_offset = 0;
+	tex->surface.display_dcc_offset = 0;
+	tex->surface.dcc_retile_map_offset = 0;
 }
 
 static bool si_texture_discard_dcc(struct si_screen *sscreen,
 				   struct si_texture *tex)
 {
-	if (!si_can_disable_dcc(tex))
+	if (!si_can_disable_dcc(tex)) {
+		assert(tex->surface.display_dcc_offset == 0);
 		return false;
+	}
 
 	assert(tex->dcc_separate_buffer == NULL);
 
 	/* Disable DCC. */
-	tex->dcc_offset = 0;
+	si_texture_zero_dcc_fields(tex);
 
 	/* Notify all contexts about the change. */
 	p_atomic_inc(&sscreen->dirty_tex_counter);
@@ -482,7 +494,7 @@ static bool si_texture_discard_dcc(struct si_screen *sscreen,
  *   context 1 & 2 read garbage, because DCC is disabled, yet there are
  *   compressed tiled
  *
- * \param sctx  the current context if you have one, or rscreen->aux_context
+ * \param sctx  the current context if you have one, or sscreen->aux_context
  *              if you don't.
  */
 bool si_texture_disable_dcc(struct si_context *sctx,
@@ -490,18 +502,21 @@ bool si_texture_disable_dcc(struct si_context *sctx,
 {
 	struct si_screen *sscreen = sctx->screen;
 
+	if (!sctx->has_graphics)
+		return si_texture_discard_dcc(sscreen, tex);
+
 	if (!si_can_disable_dcc(tex))
 		return false;
 
 	if (&sctx->b == sscreen->aux_context)
-		mtx_lock(&sscreen->aux_context_lock);
+		simple_mtx_lock(&sscreen->aux_context_lock);
 
 	/* Decompress DCC. */
 	si_decompress_dcc(sctx, tex);
 	sctx->b.flush(&sctx->b, NULL, 0);
 
 	if (&sctx->b == sscreen->aux_context)
-		mtx_unlock(&sscreen->aux_context_lock);
+		simple_mtx_unlock(&sscreen->aux_context_lock);
 
 	return si_texture_discard_dcc(sscreen, tex);
 }
@@ -518,7 +533,7 @@ static void si_reallocate_texture_inplace(struct si_context *sctx,
 
 	templ.bind |= new_bind_flag;
 
-	if (tex->buffer.b.is_shared)
+	if (tex->buffer.b.is_shared || tex->num_planes > 1)
 		return;
 
 	if (new_bind_flag == PIPE_BIND_LINEAR) {
@@ -566,31 +581,30 @@ static void si_reallocate_texture_inplace(struct si_context *sctx,
 	tex->buffer.flags = new_tex->buffer.flags;
 
 	tex->surface = new_tex->surface;
-	tex->size = new_tex->size;
 	si_texture_reference(&tex->flushed_depth_texture,
 			     new_tex->flushed_depth_texture);
 
-	tex->fmask_offset = new_tex->fmask_offset;
-	tex->cmask_offset = new_tex->cmask_offset;
+	tex->surface.fmask_offset = new_tex->surface.fmask_offset;
+	tex->surface.cmask_offset = new_tex->surface.cmask_offset;
 	tex->cmask_base_address_reg = new_tex->cmask_base_address_reg;
 
 	if (tex->cmask_buffer == &tex->buffer)
 		tex->cmask_buffer = NULL;
 	else
-		r600_resource_reference(&tex->cmask_buffer, NULL);
+		si_resource_reference(&tex->cmask_buffer, NULL);
 
 	if (new_tex->cmask_buffer == &new_tex->buffer)
 		tex->cmask_buffer = &tex->buffer;
 	else
-		r600_resource_reference(&tex->cmask_buffer, new_tex->cmask_buffer);
+		si_resource_reference(&tex->cmask_buffer, new_tex->cmask_buffer);
 
-	tex->dcc_offset = new_tex->dcc_offset;
+	tex->surface.dcc_offset = new_tex->surface.dcc_offset;
 	tex->cb_color_info = new_tex->cb_color_info;
 	memcpy(tex->color_clear_value, new_tex->color_clear_value,
 	       sizeof(tex->color_clear_value));
 	tex->last_msaa_resolve_target_micro_mode = new_tex->last_msaa_resolve_target_micro_mode;
 
-	tex->htile_offset = new_tex->htile_offset;
+	tex->surface.htile_offset = new_tex->surface.htile_offset;
 	tex->depth_clear_value = new_tex->depth_clear_value;
 	tex->dirty_level_mask = new_tex->dirty_level_mask;
 	tex->stencil_dirty_level_mask = new_tex->stencil_dirty_level_mask;
@@ -606,16 +620,16 @@ static void si_reallocate_texture_inplace(struct si_context *sctx,
 
 	tex->separate_dcc_dirty = new_tex->separate_dcc_dirty;
 	tex->dcc_gather_statistics = new_tex->dcc_gather_statistics;
-	r600_resource_reference(&tex->dcc_separate_buffer,
+	si_resource_reference(&tex->dcc_separate_buffer,
 				new_tex->dcc_separate_buffer);
-	r600_resource_reference(&tex->last_dcc_separate_buffer,
+	si_resource_reference(&tex->last_dcc_separate_buffer,
 				new_tex->last_dcc_separate_buffer);
 
 	if (new_bind_flag == PIPE_BIND_LINEAR) {
-		assert(!tex->htile_offset);
+		assert(!tex->surface.htile_offset);
 		assert(!tex->cmask_buffer);
 		assert(!tex->surface.fmask_size);
-		assert(!tex->dcc_offset);
+		assert(!tex->surface.dcc_offset);
 		assert(!tex->is_depth);
 	}
 
@@ -629,22 +643,42 @@ static uint32_t si_get_bo_metadata_word1(struct si_screen *sscreen)
 	return (ATI_VENDOR_ID << 16) | sscreen->info.pci_id;
 }
 
-static void si_query_opaque_metadata(struct si_screen *sscreen,
-				     struct si_texture *tex,
-			             struct radeon_bo_metadata *md)
+static void si_set_tex_bo_metadata(struct si_screen *sscreen,
+				   struct si_texture *tex)
 {
+	struct radeon_surf *surface = &tex->surface;
 	struct pipe_resource *res = &tex->buffer.b.b;
-	static const unsigned char swizzle[] = {
-		PIPE_SWIZZLE_X,
-		PIPE_SWIZZLE_Y,
-		PIPE_SWIZZLE_Z,
-		PIPE_SWIZZLE_W
-	};
-	uint32_t desc[8], i;
-	bool is_array = util_texture_is_array(res->target);
+	struct radeon_bo_metadata md;
 
-	if (!sscreen->info.has_bo_metadata)
-		return;
+	memset(&md, 0, sizeof(md));
+
+	if (sscreen->info.chip_class >= GFX9) {
+		md.u.gfx9.swizzle_mode = surface->u.gfx9.surf.swizzle_mode;
+
+		if (tex->surface.dcc_offset && !tex->dcc_separate_buffer) {
+			uint64_t dcc_offset =
+				tex->surface.display_dcc_offset ? tex->surface.display_dcc_offset
+							: tex->surface.dcc_offset;
+
+			assert((dcc_offset >> 8) != 0 && (dcc_offset >> 8) < (1 << 24));
+			md.u.gfx9.dcc_offset_256B = dcc_offset >> 8;
+			md.u.gfx9.dcc_pitch_max = tex->surface.u.gfx9.display_dcc_pitch_max;
+			md.u.gfx9.dcc_independent_64B = 1;
+		}
+	} else {
+		md.u.legacy.microtile = surface->u.legacy.level[0].mode >= RADEON_SURF_MODE_1D ?
+					   RADEON_LAYOUT_TILED : RADEON_LAYOUT_LINEAR;
+		md.u.legacy.macrotile = surface->u.legacy.level[0].mode >= RADEON_SURF_MODE_2D ?
+					   RADEON_LAYOUT_TILED : RADEON_LAYOUT_LINEAR;
+		md.u.legacy.pipe_config = surface->u.legacy.pipe_config;
+		md.u.legacy.bankw = surface->u.legacy.bankw;
+		md.u.legacy.bankh = surface->u.legacy.bankh;
+		md.u.legacy.tile_split = surface->u.legacy.tile_split;
+		md.u.legacy.mtilea = surface->u.legacy.mtilea;
+		md.u.legacy.num_banks = surface->u.legacy.num_banks;
+		md.u.legacy.stride = surface->u.legacy.level[0].nblk_x * surface->bpe;
+		md.u.legacy.scanout = (surface->flags & RADEON_SURF_SCANOUT) != 0;
+	}
 
 	assert(tex->dcc_separate_buffer == NULL);
 	assert(tex->surface.fmask_size == 0);
@@ -659,12 +693,21 @@ static void si_query_opaque_metadata(struct si_screen *sscreen,
 	 * [10:10+LAST_LEVEL] = mipmap level offset bits [39:8] for each level
 	 */
 
-	md->metadata[0] = 1; /* metadata image format version 1 */
+	md.metadata[0] = 1; /* metadata image format version 1 */
 
 	/* TILE_MODE_INDEX is ambiguous without a PCI ID. */
-	md->metadata[1] = si_get_bo_metadata_word1(sscreen);
+	md.metadata[1] = si_get_bo_metadata_word1(sscreen);
 
-	si_make_texture_descriptor(sscreen, tex, true,
+	static const unsigned char swizzle[] = {
+		PIPE_SWIZZLE_X,
+		PIPE_SWIZZLE_Y,
+		PIPE_SWIZZLE_Z,
+		PIPE_SWIZZLE_W
+	};
+	bool is_array = util_texture_is_array(res->target);
+	uint32_t desc[8];
+
+	sscreen->make_texture_descriptor(sscreen, tex, true,
 				   res->target, res->format,
 				   swizzle, 0, res->last_level, 0,
 				   is_array ? res->array_size - 1 : 0,
@@ -677,58 +720,259 @@ static void si_query_opaque_metadata(struct si_screen *sscreen,
 	/* Clear the base address and set the relative DCC offset. */
 	desc[0] = 0;
 	desc[1] &= C_008F14_BASE_ADDRESS_HI;
-	desc[7] = tex->dcc_offset >> 8;
+
+	switch (sscreen->info.chip_class) {
+	case GFX6:
+	case GFX7:
+		break;
+	case GFX8:
+		desc[7] = tex->surface.dcc_offset >> 8;
+		break;
+	case GFX9:
+		desc[7] = tex->surface.dcc_offset >> 8;
+		desc[5] &= C_008F24_META_DATA_ADDRESS;
+		desc[5] |= S_008F24_META_DATA_ADDRESS(tex->surface.dcc_offset >> 40);
+		break;
+	case GFX10:
+		desc[6] &= C_00A018_META_DATA_ADDRESS_LO;
+		desc[6] |= S_00A018_META_DATA_ADDRESS_LO(tex->surface.dcc_offset >> 8);
+		desc[7] = tex->surface.dcc_offset >> 16;
+		break;
+	default:
+		assert(0);
+	}
+
 
 	/* Dwords [2:9] contain the image descriptor. */
-	memcpy(&md->metadata[2], desc, sizeof(desc));
-	md->size_metadata = 10 * 4;
+	memcpy(&md.metadata[2], desc, sizeof(desc));
+	md.size_metadata = 10 * 4;
 
 	/* Dwords [10:..] contain the mipmap level offsets. */
-	if (sscreen->info.chip_class <= VI) {
-		for (i = 0; i <= res->last_level; i++)
-			md->metadata[10+i] = tex->surface.u.legacy.level[i].offset >> 8;
+	if (sscreen->info.chip_class <= GFX8) {
+		for (unsigned i = 0; i <= res->last_level; i++)
+			md.metadata[10+i] = tex->surface.u.legacy.level[i].offset >> 8;
 
-		md->size_metadata += (1 + res->last_level) * 4;
+		md.size_metadata += (1 + res->last_level) * 4;
 	}
+
+	sscreen->ws->buffer_set_metadata(tex->buffer.buf, &md);
 }
 
-static void si_apply_opaque_metadata(struct si_screen *sscreen,
-				     struct si_texture *tex,
-			             struct radeon_bo_metadata *md)
+static bool si_read_tex_bo_metadata(struct si_screen *sscreen,
+				    struct si_texture *tex,
+				    uint64_t offset,
+				    struct radeon_bo_metadata *md)
 {
 	uint32_t *desc = &md->metadata[2];
 
-	if (sscreen->info.chip_class < VI)
-		return;
+	if (offset || /* Non-zero planes ignore metadata. */
+	    md->size_metadata < 10 * 4 || /* at least 2(header) + 8(desc) dwords */
+	    md->metadata[0] == 0 || /* invalid version number */
+	    md->metadata[1] != si_get_bo_metadata_word1(sscreen)) /* invalid PCI ID */ {
+		/* Disable DCC because it might not be enabled. */
+		si_texture_zero_dcc_fields(tex);
 
-	/* Return if DCC is enabled. The texture should be set up with it
-	 * already.
-	 */
-	if (md->size_metadata >= 10 * 4 && /* at least 2(header) + 8(desc) dwords */
-	    md->metadata[0] != 0 &&
-	    md->metadata[1] == si_get_bo_metadata_word1(sscreen) &&
+		/* Don't report an error if the texture comes from an incompatible driver,
+		 * but this might not work.
+		 */
+		return true;
+	}
+
+	/* Validate that sample counts and the number of mipmap levels match. */
+	unsigned last_level = G_008F1C_LAST_LEVEL(desc[3]);
+	unsigned type = G_008F1C_TYPE(desc[3]);
+
+	if (type == V_008F1C_SQ_RSRC_IMG_2D_MSAA ||
+	    type == V_008F1C_SQ_RSRC_IMG_2D_MSAA_ARRAY) {
+		unsigned log_samples =
+			util_logbase2(MAX2(1, tex->buffer.b.b.nr_storage_samples));
+
+		if (last_level != log_samples) {
+			fprintf(stderr, "radeonsi: invalid MSAA texture import, "
+					"metadata has log2(samples) = %u, the caller set %u\n",
+				last_level, log_samples);
+			return false;
+		}
+	} else {
+		if (last_level != tex->buffer.b.b.last_level) {
+			fprintf(stderr, "radeonsi: invalid mipmapped texture import, "
+					"metadata has last_level = %u, the caller set %u\n",
+				last_level, tex->buffer.b.b.last_level);
+			return false;
+		}
+	}
+
+	if (sscreen->info.chip_class >= GFX8 &&
 	    G_008F28_COMPRESSION_EN(desc[6])) {
-		tex->dcc_offset = (uint64_t)desc[7] << 8;
-		return;
+		/* Read DCC information.
+		 *
+		 * Some state trackers don't set the SCANOUT flag when
+		 * importing displayable images, which affects PIPE_ALIGNED
+		 * and RB_ALIGNED, so we need to recover them here.
+		 */
+		switch (sscreen->info.chip_class) {
+		case GFX8:
+			tex->surface.dcc_offset = (uint64_t)desc[7] << 8;
+			break;
+
+		case GFX9:
+			tex->surface.dcc_offset =
+				((uint64_t)desc[7] << 8) |
+				((uint64_t)G_008F24_META_DATA_ADDRESS(desc[5]) << 40);
+			tex->surface.u.gfx9.dcc.pipe_aligned =
+				G_008F24_META_PIPE_ALIGNED(desc[5]);
+			tex->surface.u.gfx9.dcc.rb_aligned =
+				G_008F24_META_RB_ALIGNED(desc[5]);
+
+			/* If DCC is unaligned, this can only be a displayable image. */
+			if (!tex->surface.u.gfx9.dcc.pipe_aligned &&
+			    !tex->surface.u.gfx9.dcc.rb_aligned)
+				tex->surface.is_displayable = true;
+			break;
+
+		case GFX10:
+			tex->surface.dcc_offset =
+				((uint64_t)G_00A018_META_DATA_ADDRESS_LO(desc[6]) << 8) |
+				((uint64_t)desc[7] << 16);
+			tex->surface.u.gfx9.dcc.pipe_aligned =
+				G_00A018_META_PIPE_ALIGNED(desc[6]);
+			break;
+
+		default:
+			assert(0);
+			return false;
+		}
+	} else {
+		/* Disable DCC. dcc_offset is always set by texture_from_handle
+		 * and must be cleared here.
+		 */
+		si_texture_zero_dcc_fields(tex);
 	}
 
-	/* Disable DCC. These are always set by texture_from_handle and must
-	 * be cleared here.
+	return true;
+}
+
+static bool si_has_displayable_dcc(struct si_texture *tex)
+{
+	struct si_screen *sscreen = (struct si_screen*)tex->buffer.b.b.screen;
+
+	if (sscreen->info.chip_class <= GFX8)
+		return false;
+
+	/* This needs a cache flush before scanout.
+	 * (it can't be scanned out and rendered to simultaneously)
 	 */
-	tex->dcc_offset = 0;
+	if (sscreen->info.use_display_dcc_unaligned &&
+	    tex->surface.dcc_offset &&
+	    !tex->surface.u.gfx9.dcc.pipe_aligned &&
+	    !tex->surface.u.gfx9.dcc.rb_aligned)
+		return true;
+
+	/* This needs an explicit flush (flush_resource). */
+	if (sscreen->info.use_display_dcc_with_retile_blit &&
+	    tex->surface.display_dcc_offset)
+		return true;
+
+	return false;
+}
+
+static bool si_resource_get_param(struct pipe_screen *screen,
+				  struct pipe_context *context,
+				  struct pipe_resource *resource,
+				  unsigned plane,
+				  unsigned layer,
+				  enum pipe_resource_param param,
+				  unsigned handle_usage,
+				  uint64_t *value)
+{
+	for (unsigned i = 0; i < plane; i++)
+		resource = resource->next;
+
+	struct si_screen *sscreen = (struct si_screen*)screen;
+	struct si_texture *tex = (struct si_texture*)resource;
+	struct winsys_handle whandle;
+
+	switch (param) {
+	case PIPE_RESOURCE_PARAM_NPLANES:
+		*value = resource->target == PIPE_BUFFER ? 1 : tex->num_planes;
+		return true;
+
+	case PIPE_RESOURCE_PARAM_STRIDE:
+		if (resource->target == PIPE_BUFFER)
+			*value = 0;
+		else if (sscreen->info.chip_class >= GFX9)
+			*value = tex->surface.u.gfx9.surf_pitch * tex->surface.bpe;
+		else
+			*value = tex->surface.u.legacy.level[0].nblk_x * tex->surface.bpe;
+		return true;
+
+	case PIPE_RESOURCE_PARAM_OFFSET:
+		if (resource->target == PIPE_BUFFER)
+			*value = 0;
+		else if (sscreen->info.chip_class >= GFX9)
+			*value = tex->surface.u.gfx9.surf_offset +
+				 layer * tex->surface.u.gfx9.surf_slice_size;
+		else
+			*value = tex->surface.u.legacy.level[0].offset +
+				 layer * (uint64_t)tex->surface.u.legacy.level[0].slice_size_dw * 4;
+		return true;
+
+	case PIPE_RESOURCE_PARAM_MODIFIER:
+		*value = DRM_FORMAT_MOD_INVALID;
+		return true;
+
+	case PIPE_RESOURCE_PARAM_HANDLE_TYPE_SHARED:
+	case PIPE_RESOURCE_PARAM_HANDLE_TYPE_KMS:
+	case PIPE_RESOURCE_PARAM_HANDLE_TYPE_FD:
+		memset(&whandle, 0, sizeof(whandle));
+
+		if (param == PIPE_RESOURCE_PARAM_HANDLE_TYPE_SHARED)
+			whandle.type = WINSYS_HANDLE_TYPE_SHARED;
+		else if (param == PIPE_RESOURCE_PARAM_HANDLE_TYPE_KMS)
+			whandle.type = WINSYS_HANDLE_TYPE_KMS;
+		else if (param == PIPE_RESOURCE_PARAM_HANDLE_TYPE_FD)
+			whandle.type = WINSYS_HANDLE_TYPE_FD;
+
+		if (!screen->resource_get_handle(screen, context, resource,
+						 &whandle, handle_usage))
+			return false;
+
+		*value = whandle.handle;
+		return true;
+	}
+	return false;
 }
 
-static boolean si_texture_get_handle(struct pipe_screen* screen,
-				     struct pipe_context *ctx,
-				     struct pipe_resource *resource,
-				     struct winsys_handle *whandle,
-				     unsigned usage)
+static void si_texture_get_info(struct pipe_screen* screen,
+				struct pipe_resource *resource,
+				unsigned *pstride,
+				unsigned *poffset)
+{
+	uint64_t value;
+
+	if (pstride) {
+		si_resource_get_param(screen, NULL, resource, 0, 0,
+				      PIPE_RESOURCE_PARAM_STRIDE, 0, &value);
+		*pstride = value;
+	}
+
+	if (poffset) {
+		si_resource_get_param(screen, NULL, resource, 0, 0,
+				      PIPE_RESOURCE_PARAM_OFFSET, 0, &value);
+		*poffset = value;
+	}
+}
+
+static bool si_texture_get_handle(struct pipe_screen* screen,
+				  struct pipe_context *ctx,
+				  struct pipe_resource *resource,
+				  struct winsys_handle *whandle,
+				  unsigned usage)
 {
 	struct si_screen *sscreen = (struct si_screen*)screen;
 	struct si_context *sctx;
-	struct r600_resource *res = r600_resource(resource);
+	struct si_resource *res = si_resource(resource);
 	struct si_texture *tex = (struct si_texture*)resource;
-	struct radeon_bo_metadata metadata;
 	bool update_metadata = false;
 	unsigned stride, offset, slice_size;
 	bool flush = false;
@@ -737,6 +981,13 @@ static boolean si_texture_get_handle(struct pipe_screen* screen,
 	sctx = (struct si_context*)(ctx ? ctx : sscreen->aux_context);
 
 	if (resource->target != PIPE_BUFFER) {
+		/* Individual planes are chained pipe_resource instances. */
+		for (unsigned i = 0; i < whandle->plane; i++) {
+			resource = resource->next;
+			res = si_resource(resource);
+			tex = (struct si_texture*)resource;
+		}
+
 		/* This is not supported now, but it might be required for OpenCL
 		 * interop in the future.
 		 */
@@ -747,8 +998,7 @@ static boolean si_texture_get_handle(struct pipe_screen* screen,
 		if (sscreen->ws->buffer_is_suballocated(res->buf) ||
 		    tex->surface.tile_swizzle ||
 		    (tex->buffer.flags & RADEON_FLAG_NO_INTERPROCESS_SHARING &&
-		     sscreen->info.has_local_buffers &&
-		     whandle->type != WINSYS_HANDLE_TYPE_KMS)) {
+		     sscreen->info.has_local_buffers)) {
 			assert(!res->b.is_shared);
 			si_reallocate_texture_inplace(sctx, tex,
 							PIPE_BIND_SHARED, false);
@@ -759,11 +1009,14 @@ static boolean si_texture_get_handle(struct pipe_screen* screen,
 			assert(tex->surface.tile_swizzle == 0);
 		}
 
-		/* Since shader image stores don't support DCC on VI,
+		/* Since shader image stores don't support DCC on GFX8,
 		 * disable it for external clients that want write
 		 * access.
 		 */
-		if (usage & PIPE_HANDLE_USAGE_WRITE && tex->dcc_offset) {
+		if ((usage & PIPE_HANDLE_USAGE_SHADER_WRITE && tex->surface.dcc_offset) ||
+		    /* Displayable DCC requires an explicit flush. */
+		    (!(usage & PIPE_HANDLE_USAGE_EXPLICIT_FLUSH) &&
+		     si_has_displayable_dcc(tex))) {
 			if (si_texture_disable_dcc(sctx, tex)) {
 				update_metadata = true;
 				/* si_texture_disable_dcc flushes the context */
@@ -772,7 +1025,7 @@ static boolean si_texture_get_handle(struct pipe_screen* screen,
 		}
 
 		if (!(usage & PIPE_HANDLE_USAGE_EXPLICIT_FLUSH) &&
-		    (tex->cmask_buffer || tex->dcc_offset)) {
+		    (tex->cmask_buffer || tex->surface.dcc_offset)) {
 			/* Eliminate fast clear (both CMASK and DCC) */
 			si_eliminate_fast_color_clear(sctx, tex);
 			/* eliminate_fast_color_clear flushes the context */
@@ -786,22 +1039,12 @@ static boolean si_texture_get_handle(struct pipe_screen* screen,
 		}
 
 		/* Set metadata. */
-		if (!res->b.is_shared || update_metadata) {
-			si_texture_init_metadata(sscreen, tex, &metadata);
-			si_query_opaque_metadata(sscreen, tex, &metadata);
-
-			sscreen->ws->buffer_set_metadata(res->buf, &metadata);
-		}
+		if ((!res->b.is_shared || update_metadata) && whandle->offset == 0)
+			si_set_tex_bo_metadata(sscreen, tex);
 
 		if (sscreen->info.chip_class >= GFX9) {
-			offset = tex->surface.u.gfx9.surf_offset;
-			stride = tex->surface.u.gfx9.surf_pitch *
-				 tex->surface.bpe;
 			slice_size = tex->surface.u.gfx9.surf_slice_size;
 		} else {
-			offset = tex->surface.u.legacy.level[0].offset;
-			stride = tex->surface.u.legacy.level[0].nblk_x *
-				 tex->surface.bpe;
 			slice_size = (uint64_t)tex->surface.u.legacy.level[0].slice_size_dw * 4;
 		}
 	} else {
@@ -837,11 +1080,11 @@ static boolean si_texture_get_handle(struct pipe_screen* screen,
 		}
 
 		/* Buffers */
-		offset = 0;
-		stride = 0;
 		slice_size = 0;
 	}
 
+	si_texture_get_info(screen, resource, &stride, &offset);
+
 	if (flush)
 		sctx->b.flush(&sctx->b, NULL, 0);
 
@@ -857,108 +1100,35 @@ static boolean si_texture_get_handle(struct pipe_screen* screen,
 		res->external_usage = usage;
 	}
 
-	return sscreen->ws->buffer_get_handle(res->buf, stride, offset,
-					      slice_size, whandle);
+	whandle->stride = stride;
+	whandle->offset = offset + slice_size * whandle->layer;
+
+	return sscreen->ws->buffer_get_handle(sscreen->ws, res->buf, whandle);
 }
 
 static void si_texture_destroy(struct pipe_screen *screen,
 			       struct pipe_resource *ptex)
 {
+	struct si_screen *sscreen = (struct si_screen*)screen;
 	struct si_texture *tex = (struct si_texture*)ptex;
-	struct r600_resource *resource = &tex->buffer;
+	struct si_resource *resource = &tex->buffer;
+
+	if (sscreen->info.chip_class >= GFX9)
+		free(tex->surface.u.gfx9.dcc_retile_map);
 
 	si_texture_reference(&tex->flushed_depth_texture, NULL);
 
 	if (tex->cmask_buffer != &tex->buffer) {
-	    r600_resource_reference(&tex->cmask_buffer, NULL);
+	    si_resource_reference(&tex->cmask_buffer, NULL);
 	}
 	pb_reference(&resource->buf, NULL);
-	r600_resource_reference(&tex->dcc_separate_buffer, NULL);
-	r600_resource_reference(&tex->last_dcc_separate_buffer, NULL);
+	si_resource_reference(&tex->dcc_separate_buffer, NULL);
+	si_resource_reference(&tex->last_dcc_separate_buffer, NULL);
 	FREE(tex);
 }
 
 static const struct u_resource_vtbl si_texture_vtbl;
 
-static void si_texture_get_htile_size(struct si_screen *sscreen,
-				      struct si_texture *tex)
-{
-	unsigned cl_width, cl_height, width, height;
-	unsigned slice_elements, slice_bytes, pipe_interleave_bytes, base_align;
-	unsigned num_pipes = sscreen->info.num_tile_pipes;
-
-	assert(sscreen->info.chip_class <= VI);
-
-	tex->surface.htile_size = 0;
-
-	if (tex->surface.u.legacy.level[0].mode == RADEON_SURF_MODE_1D &&
-	    !sscreen->info.htile_cmask_support_1d_tiling)
-		return;
-
-	/* Overalign HTILE on P2 configs to work around GPU hangs in
-	 * piglit/depthstencil-render-miplevels 585.
-	 *
-	 * This has been confirmed to help Kabini & Stoney, where the hangs
-	 * are always reproducible. I think I have seen the test hang
-	 * on Carrizo too, though it was very rare there.
-	 */
-	if (sscreen->info.chip_class >= CIK && num_pipes < 4)
-		num_pipes = 4;
-
-	switch (num_pipes) {
-	case 1:
-		cl_width = 32;
-		cl_height = 16;
-		break;
-	case 2:
-		cl_width = 32;
-		cl_height = 32;
-		break;
-	case 4:
-		cl_width = 64;
-		cl_height = 32;
-		break;
-	case 8:
-		cl_width = 64;
-		cl_height = 64;
-		break;
-	case 16:
-		cl_width = 128;
-		cl_height = 64;
-		break;
-	default:
-		assert(0);
-		return;
-	}
-
-	width = align(tex->buffer.b.b.width0, cl_width * 8);
-	height = align(tex->buffer.b.b.height0, cl_height * 8);
-
-	slice_elements = (width * height) / (8 * 8);
-	slice_bytes = slice_elements * 4;
-
-	pipe_interleave_bytes = sscreen->info.pipe_interleave_bytes;
-	base_align = num_pipes * pipe_interleave_bytes;
-
-	tex->surface.htile_alignment = base_align;
-	tex->surface.htile_size =
-		util_num_layers(&tex->buffer.b.b, 0) *
-		align(slice_bytes, base_align);
-}
-
-static void si_texture_allocate_htile(struct si_screen *sscreen,
-				      struct si_texture *tex)
-{
-	if (sscreen->info.chip_class <= VI && !tex->tc_compatible_htile)
-		si_texture_get_htile_size(sscreen, tex);
-
-	if (!tex->surface.htile_size)
-		return;
-
-	tex->htile_offset = align(tex->size, tex->surface.htile_alignment);
-	tex->size = tex->htile_offset + tex->surface.htile_size;
-}
-
 void si_print_texture_info(struct si_screen *sscreen,
 			   struct si_texture *tex, struct u_log_context *log)
 {
@@ -985,10 +1155,10 @@ void si_print_texture_info(struct si_screen *sscreen,
 			tex->surface.u.gfx9.surf.epitch,
 			tex->surface.u.gfx9.surf_pitch);
 
-		if (tex->surface.fmask_size) {
+		if (tex->surface.fmask_offset) {
 			u_log_printf(log, "  FMASK: offset=%"PRIu64", size=%"PRIu64", "
 				"alignment=%u, swmode=%u, epitch=%u\n",
-				tex->fmask_offset,
+				tex->surface.fmask_offset,
 				tex->surface.fmask_size,
 				tex->surface.fmask_alignment,
 				tex->surface.u.gfx9.fmask.swizzle_mode,
@@ -998,29 +1168,29 @@ void si_print_texture_info(struct si_screen *sscreen,
 		if (tex->cmask_buffer) {
 			u_log_printf(log, "  CMask: offset=%"PRIu64", size=%u, "
 				"alignment=%u, rb_aligned=%u, pipe_aligned=%u\n",
-				tex->cmask_offset,
+				tex->surface.cmask_offset,
 				tex->surface.cmask_size,
 				tex->surface.cmask_alignment,
 				tex->surface.u.gfx9.cmask.rb_aligned,
 				tex->surface.u.gfx9.cmask.pipe_aligned);
 		}
 
-		if (tex->htile_offset) {
+		if (tex->surface.htile_offset) {
 			u_log_printf(log, "  HTile: offset=%"PRIu64", size=%u, alignment=%u, "
 				"rb_aligned=%u, pipe_aligned=%u\n",
-				tex->htile_offset,
+				tex->surface.htile_offset,
 				tex->surface.htile_size,
 				tex->surface.htile_alignment,
 				tex->surface.u.gfx9.htile.rb_aligned,
 				tex->surface.u.gfx9.htile.pipe_aligned);
 		}
 
-		if (tex->dcc_offset) {
+		if (tex->surface.dcc_offset) {
 			u_log_printf(log, "  DCC: offset=%"PRIu64", size=%u, "
 				"alignment=%u, pitch_max=%u, num_dcc_levels=%u\n",
-				tex->dcc_offset, tex->surface.dcc_size,
+				tex->surface.dcc_offset, tex->surface.dcc_size,
 				tex->surface.dcc_alignment,
-				tex->surface.u.gfx9.dcc_pitch_max,
+				tex->surface.u.gfx9.display_dcc_pitch_max,
 				tex->surface.num_dcc_levels);
 		}
 
@@ -1040,10 +1210,10 @@ void si_print_texture_info(struct si_screen *sscreen,
 		tex->surface.u.legacy.tile_split, tex->surface.u.legacy.pipe_config,
 		(tex->surface.flags & RADEON_SURF_SCANOUT) != 0);
 
-	if (tex->surface.fmask_size)
+	if (tex->surface.fmask_offset)
 		u_log_printf(log, "  FMask: offset=%"PRIu64", size=%"PRIu64", alignment=%u, pitch_in_pixels=%u, "
 			"bankh=%u, slice_tile_max=%u, tile_mode_index=%u\n",
-			tex->fmask_offset, tex->surface.fmask_size, tex->surface.fmask_alignment,
+			tex->surface.fmask_offset, tex->surface.fmask_size, tex->surface.fmask_alignment,
 			tex->surface.u.legacy.fmask.pitch_in_pixels,
 			tex->surface.u.legacy.fmask.bankh,
 			tex->surface.u.legacy.fmask.slice_tile_max,
@@ -1052,19 +1222,19 @@ void si_print_texture_info(struct si_screen *sscreen,
 	if (tex->cmask_buffer)
 		u_log_printf(log, "  CMask: offset=%"PRIu64", size=%u, alignment=%u, "
 			"slice_tile_max=%u\n",
-			tex->cmask_offset, tex->surface.cmask_size, tex->surface.cmask_alignment,
+			tex->surface.cmask_offset, tex->surface.cmask_size, tex->surface.cmask_alignment,
 			tex->surface.u.legacy.cmask_slice_tile_max);
 
-	if (tex->htile_offset)
+	if (tex->surface.htile_offset)
 		u_log_printf(log, "  HTile: offset=%"PRIu64", size=%u, "
 			"alignment=%u, TC_compatible = %u\n",
-			tex->htile_offset, tex->surface.htile_size,
+			tex->surface.htile_offset, tex->surface.htile_size,
 			tex->surface.htile_alignment,
 			tex->tc_compatible_htile);
 
-	if (tex->dcc_offset) {
+	if (tex->surface.dcc_offset) {
 		u_log_printf(log, "  DCC: offset=%"PRIu64", size=%u, alignment=%u\n",
-			tex->dcc_offset, tex->surface.dcc_size,
+			tex->surface.dcc_offset, tex->surface.dcc_size,
 			tex->surface.dcc_alignment);
 		for (i = 0; i <= tex->buffer.b.b.last_level; i++)
 			u_log_printf(log, "  DCCLevel[%i]: enabled=%u, offset=%u, "
@@ -1109,20 +1279,35 @@ void si_print_texture_info(struct si_screen *sscreen,
 	}
 }
 
-/* Common processing for si_texture_create and si_texture_from_handle */
+/**
+ * Common function for si_texture_create and si_texture_from_handle.
+ *
+ * \param screen	screen
+ * \param base		resource template
+ * \param surface	radeon_surf
+ * \param plane0	if a non-zero plane is being created, this is the first plane
+ * \param imported_buf	from si_texture_from_handle
+ * \param offset	offset for non-zero planes or imported buffers
+ * \param alloc_size	the size to allocate if plane0 != NULL
+ * \param alignment	alignment for the allocation
+ */
 static struct si_texture *
 si_texture_create_object(struct pipe_screen *screen,
 			 const struct pipe_resource *base,
-			 struct pb_buffer *buf,
-			 struct radeon_surf *surface)
+			 const struct radeon_surf *surface,
+			 const struct si_texture *plane0,
+			 struct pb_buffer *imported_buf,
+			 uint64_t offset,
+			 uint64_t alloc_size,
+			 unsigned alignment)
 {
 	struct si_texture *tex;
-	struct r600_resource *resource;
+	struct si_resource *resource;
 	struct si_screen *sscreen = (struct si_screen*)screen;
 
 	tex = CALLOC_STRUCT(si_texture);
 	if (!tex)
-		return NULL;
+		goto error;
 
 	resource = &tex->buffer;
 	resource->b.b = *base;
@@ -1133,16 +1318,13 @@ si_texture_create_object(struct pipe_screen *screen,
 
 	/* don't include stencil-only formats which we don't support for rendering */
 	tex->is_depth = util_format_has_depth(util_format_description(tex->buffer.b.b.format));
-
 	tex->surface = *surface;
-	tex->size = tex->surface.surf_size;
-
 	tex->tc_compatible_htile = tex->surface.htile_size != 0 &&
 				   (tex->surface.flags &
 				    RADEON_SURF_TC_COMPATIBLE_HTILE);
 
 	/* TC-compatible HTILE:
-	 * - VI only supports Z32_FLOAT.
+	 * - GFX8 only supports Z32_FLOAT.
 	 * - GFX9 only supports Z32_FLOAT and Z16_UNORM. */
 	if (tex->tc_compatible_htile) {
 		if (sscreen->info.chip_class >= GFX9 &&
@@ -1166,106 +1348,193 @@ si_texture_create_object(struct pipe_screen *screen,
 	 */
 	tex->ps_draw_ratio = 0;
 
+	if (sscreen->info.chip_class >= GFX9) {
+		tex->surface.u.gfx9.surf_offset = offset;
+	} else {
+		for (unsigned i = 0; i < ARRAY_SIZE(surface->u.legacy.level); ++i)
+			tex->surface.u.legacy.level[i].offset += offset;
+	}
+
 	if (tex->is_depth) {
 		if (sscreen->info.chip_class >= GFX9) {
 			tex->can_sample_z = true;
 			tex->can_sample_s = true;
+
+			/* Stencil texturing with HTILE doesn't work
+			 * with mipmapping on Navi10-14. */
+			if ((sscreen->info.family == CHIP_NAVI10 ||
+			     sscreen->info.family == CHIP_NAVI12 ||
+			     sscreen->info.family == CHIP_NAVI14) &&
+			    base->last_level > 0)
+				tex->htile_stencil_disabled = true;
 		} else {
 			tex->can_sample_z = !tex->surface.u.legacy.depth_adjusted;
 			tex->can_sample_s = !tex->surface.u.legacy.stencil_adjusted;
 		}
 
-		if (!(base->flags & (SI_RESOURCE_FLAG_TRANSFER |
-				     SI_RESOURCE_FLAG_FLUSHED_DEPTH))) {
-			tex->db_compatible = true;
-
-			if (!(sscreen->debug_flags & DBG(NO_HYPERZ)))
-				si_texture_allocate_htile(sscreen, tex);
-		}
+		tex->db_compatible = surface->flags & RADEON_SURF_ZBUFFER;
 	} else {
-		if (base->nr_samples > 1 &&
-		    !buf &&
-		    !(sscreen->debug_flags & DBG(NO_FMASK))) {
-			/* Allocate FMASK. */
-			tex->fmask_offset = align64(tex->size,
-						     tex->surface.fmask_alignment);
-			tex->size = tex->fmask_offset + tex->surface.fmask_size;
-
-			/* Allocate CMASK. */
-			tex->cmask_offset = align64(tex->size, tex->surface.cmask_alignment);
-			tex->size = tex->cmask_offset + tex->surface.cmask_size;
+		if (tex->surface.cmask_offset) {
 			tex->cb_color_info |= S_028C70_FAST_CLEAR(1);
 			tex->cmask_buffer = &tex->buffer;
-
-			if (!tex->surface.fmask_size || !tex->surface.cmask_size) {
-				FREE(tex);
-				return NULL;
-			}
-		}
-
-		/* Shared textures must always set up DCC here.
-		 * If it's not present, it will be disabled by
-		 * apply_opaque_metadata later.
-		 */
-		if (tex->surface.dcc_size &&
-		    (buf || !(sscreen->debug_flags & DBG(NO_DCC))) &&
-		    !(tex->surface.flags & RADEON_SURF_SCANOUT)) {
-			/* Reserve space for the DCC buffer. */
-			tex->dcc_offset = align64(tex->size, tex->surface.dcc_alignment);
-			tex->size = tex->dcc_offset + tex->surface.dcc_size;
 		}
 	}
 
-	/* Now create the backing buffer. */
-	if (!buf) {
-		si_init_resource_fields(sscreen, resource, tex->size,
-					  tex->surface.surf_alignment);
-
-		if (!si_alloc_resource(sscreen, resource)) {
-			FREE(tex);
-			return NULL;
-		}
+	if (plane0) {
+		/* The buffer is shared with the first plane. */
+		resource->bo_size = plane0->buffer.bo_size;
+		resource->bo_alignment = plane0->buffer.bo_alignment;
+		resource->flags = plane0->buffer.flags;
+		resource->domains = plane0->buffer.domains;
+		resource->vram_usage = plane0->buffer.vram_usage;
+		resource->gart_usage = plane0->buffer.gart_usage;
+
+		pb_reference(&resource->buf, plane0->buffer.buf);
+		resource->gpu_address = plane0->buffer.gpu_address;
+	} else if (!(surface->flags & RADEON_SURF_IMPORTED)) {
+		/* Create the backing buffer. */
+		si_init_resource_fields(sscreen, resource, alloc_size, alignment);
+
+		if (!si_alloc_resource(sscreen, resource))
+			goto error;
 	} else {
-		resource->buf = buf;
+		resource->buf = imported_buf;
 		resource->gpu_address = sscreen->ws->buffer_get_virtual_address(resource->buf);
-		resource->bo_size = buf->size;
-		resource->bo_alignment = buf->alignment;
+		resource->bo_size = imported_buf->size;
+		resource->bo_alignment = imported_buf->alignment;
 		resource->domains = sscreen->ws->buffer_get_initial_domain(resource->buf);
 		if (resource->domains & RADEON_DOMAIN_VRAM)
-			resource->vram_usage = buf->size;
+			resource->vram_usage = resource->bo_size;
 		else if (resource->domains & RADEON_DOMAIN_GTT)
-			resource->gart_usage = buf->size;
+			resource->gart_usage = resource->bo_size;
 	}
 
 	if (tex->cmask_buffer) {
 		/* Initialize the cmask to 0xCC (= compressed state). */
 		si_screen_clear_buffer(sscreen, &tex->cmask_buffer->b.b,
-					 tex->cmask_offset, tex->surface.cmask_size,
+					 tex->surface.cmask_offset, tex->surface.cmask_size,
 					 0xCCCCCCCC);
 	}
-	if (tex->htile_offset) {
+	if (tex->surface.htile_offset) {
 		uint32_t clear_value = 0;
 
 		if (sscreen->info.chip_class >= GFX9 || tex->tc_compatible_htile)
 			clear_value = 0x0000030F;
 
 		si_screen_clear_buffer(sscreen, &tex->buffer.b.b,
-					 tex->htile_offset,
+					 tex->surface.htile_offset,
 					 tex->surface.htile_size,
 					 clear_value);
 	}
 
 	/* Initialize DCC only if the texture is not being imported. */
-	if (!buf && tex->dcc_offset) {
-		si_screen_clear_buffer(sscreen, &tex->buffer.b.b,
-					 tex->dcc_offset,
-					 tex->surface.dcc_size,
-					 0xFFFFFFFF);
+	if (!(surface->flags & RADEON_SURF_IMPORTED) && tex->surface.dcc_offset) {
+		/* Clear DCC to black for all tiles with DCC enabled.
+		 *
+		 * This fixes corruption in 3DMark Slingshot Extreme, which
+		 * uses uninitialized textures, causing corruption.
+		 */
+		if (tex->surface.num_dcc_levels == tex->buffer.b.b.last_level + 1 &&
+		    tex->buffer.b.b.nr_samples <= 2) {
+			/* Simple case - all tiles have DCC enabled. */
+			si_screen_clear_buffer(sscreen, &tex->buffer.b.b,
+					       tex->surface.dcc_offset,
+					       tex->surface.dcc_size,
+					       DCC_CLEAR_COLOR_0000);
+		} else if (sscreen->info.chip_class >= GFX9) {
+			/* Clear to uncompressed. Clearing this to black is complicated. */
+			si_screen_clear_buffer(sscreen, &tex->buffer.b.b,
+					       tex->surface.dcc_offset,
+					       tex->surface.dcc_size,
+					       DCC_UNCOMPRESSED);
+		} else {
+			/* GFX8: Initialize mipmap levels and multisamples separately. */
+			if (tex->buffer.b.b.nr_samples >= 2) {
+				/* Clearing this to black is complicated. */
+				si_screen_clear_buffer(sscreen, &tex->buffer.b.b,
+						       tex->surface.dcc_offset,
+						       tex->surface.dcc_size,
+						       DCC_UNCOMPRESSED);
+			} else {
+				/* Clear the enabled mipmap levels to black. */
+				unsigned size = 0;
+
+				for (unsigned i = 0; i < tex->surface.num_dcc_levels; i++) {
+					if (!tex->surface.u.legacy.level[i].dcc_fast_clear_size)
+						break;
+
+					size = tex->surface.u.legacy.level[i].dcc_offset +
+					       tex->surface.u.legacy.level[i].dcc_fast_clear_size;
+				}
+
+				/* Mipmap levels with DCC. */
+				if (size) {
+					si_screen_clear_buffer(sscreen, &tex->buffer.b.b,
+							       tex->surface.dcc_offset, size,
+							       DCC_CLEAR_COLOR_0000);
+				}
+				/* Mipmap levels without DCC. */
+				if (size != tex->surface.dcc_size) {
+					si_screen_clear_buffer(sscreen, &tex->buffer.b.b,
+							       tex->surface.dcc_offset + size,
+							       tex->surface.dcc_size - size,
+							       DCC_UNCOMPRESSED);
+				}
+			}
+		}
+
+		/* Initialize displayable DCC that requires the retile blit. */
+		if (tex->surface.dcc_retile_map_offset) {
+			/* Uninitialized DCC can hang the display hw.
+			 * Clear to white to indicate that. */
+			si_screen_clear_buffer(sscreen, &tex->buffer.b.b,
+					       tex->surface.display_dcc_offset,
+					       tex->surface.u.gfx9.display_dcc_size,
+					       DCC_CLEAR_COLOR_1111);
+
+			/* Upload the DCC retile map.
+			 * Use a staging buffer for the upload, because
+			 * the buffer backing the texture is unmappable.
+			 */
+			bool use_uint16 = tex->surface.u.gfx9.dcc_retile_use_uint16;
+			unsigned num_elements = tex->surface.u.gfx9.dcc_retile_num_elements;
+			struct si_resource *buf =
+				si_aligned_buffer_create(screen, 0, PIPE_USAGE_STREAM,
+							 num_elements * (use_uint16 ? 2 : 4),
+							 sscreen->info.tcc_cache_line_size);
+			uint32_t *ui = (uint32_t*)sscreen->ws->buffer_map(buf->buf, NULL,
+									  PIPE_TRANSFER_WRITE);
+			uint16_t *us = (uint16_t*)ui;
+
+			/* Upload the retile map into a staging buffer. */
+			if (use_uint16) {
+				for (unsigned i = 0; i < num_elements; i++)
+					us[i] = tex->surface.u.gfx9.dcc_retile_map[i];
+			} else {
+				for (unsigned i = 0; i < num_elements; i++)
+					ui[i] = tex->surface.u.gfx9.dcc_retile_map[i];
+			}
+
+			/* Copy the staging buffer to the buffer backing the texture. */
+			struct si_context *sctx = (struct si_context*)sscreen->aux_context;
+			struct pipe_box box;
+			u_box_1d(0, buf->b.b.width0, &box);
+
+			assert(tex->surface.dcc_retile_map_offset <= UINT_MAX);
+			simple_mtx_lock(&sscreen->aux_context_lock);
+			sctx->dma_copy(&sctx->b, &tex->buffer.b.b, 0,
+				       tex->surface.dcc_retile_map_offset, 0, 0,
+				       &buf->b.b, 0, &box);
+			sscreen->aux_context->flush(sscreen->aux_context, NULL, 0);
+			simple_mtx_unlock(&sscreen->aux_context_lock);
+
+			si_resource_reference(&buf, NULL);
+		}
 	}
 
 	/* Initialize the CMASK base register value. */
 	tex->cmask_base_address_reg =
-		(tex->buffer.gpu_address + tex->cmask_offset) >> 8;
+		(tex->buffer.gpu_address + tex->surface.cmask_offset) >> 8;
 
 	if (sscreen->debug_flags & DBG(VM)) {
 		fprintf(stderr, "VM start=0x%"PRIX64"  end=0x%"PRIX64" | Texture %ix%ix%i, %i levels, %i samples, %s\n",
@@ -1286,6 +1555,12 @@ si_texture_create_object(struct pipe_screen *screen,
 	}
 
 	return tex;
+
+error:
+	FREE(tex);
+	if (sscreen->info.chip_class >= GFX9)
+		free(surface->u.gfx9.dcc_retile_map);
+	return NULL;
 }
 
 static enum radeon_surf_mode
@@ -1293,7 +1568,7 @@ si_choose_tiling(struct si_screen *sscreen,
 		 const struct pipe_resource *templ, bool tc_compatible_htile)
 {
 	const struct util_format_description *desc = util_format_description(templ->format);
-	bool force_tiling = templ->flags & SI_RESOURCE_FLAG_FORCE_TILING;
+	bool force_tiling = templ->flags & SI_RESOURCE_FLAG_FORCE_MSAA_TILING;
 	bool is_depth_stencil = util_format_is_depth_or_stencil(templ->format) &&
 				!(templ->flags & SI_RESOURCE_FLAG_FLUSHED_DEPTH);
 
@@ -1305,10 +1580,10 @@ si_choose_tiling(struct si_screen *sscreen,
 	if (templ->flags & SI_RESOURCE_FLAG_TRANSFER)
 		return RADEON_SURF_MODE_LINEAR_ALIGNED;
 
-	/* Avoid Z/S decompress blits by forcing TC-compatible HTILE on VI,
+	/* Avoid Z/S decompress blits by forcing TC-compatible HTILE on GFX8,
 	 * which requires 2D tiling.
 	 */
-	if (sscreen->info.chip_class == VI && tc_compatible_htile)
+	if (sscreen->info.chip_class == GFX8 && tc_compatible_htile)
 		return RADEON_SURF_MODE_2D;
 
 	/* Handle common candidates for the linear mode.
@@ -1324,7 +1599,7 @@ si_choose_tiling(struct si_screen *sscreen,
 		if (desc->layout == UTIL_FORMAT_LAYOUT_SUBSAMPLED)
 			return RADEON_SURF_MODE_LINEAR_ALIGNED;
 
-		/* Cursors are linear on SI.
+		/* Cursors are linear on AMD GCN.
 		 * (XXX double-check, maybe also use RADEON_SURF_SCANOUT) */
 		if (templ->bind & PIPE_BIND_CURSOR)
 			return RADEON_SURF_MODE_LINEAR_ALIGNED;
@@ -1378,10 +1653,10 @@ struct pipe_resource *si_texture_create(struct pipe_screen *screen,
 		}
 	}
 
-	struct radeon_surf surface = {0};
-	bool is_flushed_depth = templ->flags & SI_RESOURCE_FLAG_FLUSHED_DEPTH;
+	bool is_flushed_depth = templ->flags & SI_RESOURCE_FLAG_FLUSHED_DEPTH ||
+				templ->flags & SI_RESOURCE_FLAG_TRANSFER;
 	bool tc_compatible_htile =
-		sscreen->info.chip_class >= VI &&
+		sscreen->info.chip_class >= GFX8 &&
 		/* There are issues with TC-compatible HTILE on Tonga (and
 		 * Iceland is the same design), and documented bug workarounds
 		 * don't help. For example, this fails:
@@ -1394,18 +1669,65 @@ struct pipe_resource *si_texture_create(struct pipe_screen *screen,
 		!is_flushed_depth &&
 		templ->nr_samples <= 1 && /* TC-compat HTILE is less efficient with MSAA */
 		is_zs;
-	int r;
+	enum radeon_surf_mode tile_mode = si_choose_tiling(sscreen, templ,
+							   tc_compatible_htile);
+
+	/* This allocates textures with multiple planes like NV12 in 1 buffer. */
+	enum { SI_TEXTURE_MAX_PLANES = 3 };
+	struct radeon_surf surface[SI_TEXTURE_MAX_PLANES] = {};
+	struct pipe_resource plane_templ[SI_TEXTURE_MAX_PLANES];
+	uint64_t plane_offset[SI_TEXTURE_MAX_PLANES] = {};
+	uint64_t total_size = 0;
+	unsigned max_alignment = 0;
+	unsigned num_planes = util_format_get_num_planes(templ->format);
+	assert(num_planes <= SI_TEXTURE_MAX_PLANES);
+
+	/* Compute texture or plane layouts and offsets. */
+	for (unsigned i = 0; i < num_planes; i++) {
+		plane_templ[i] = *templ;
+		plane_templ[i].format = util_format_get_plane_format(templ->format, i);
+		plane_templ[i].width0 = util_format_get_plane_width(templ->format, i, templ->width0);
+		plane_templ[i].height0 = util_format_get_plane_height(templ->format, i, templ->height0);
+
+		/* Multi-plane allocations need PIPE_BIND_SHARED, because we can't
+		 * reallocate the storage to add PIPE_BIND_SHARED, because it's
+		 * shared by 3 pipe_resources.
+		 */
+		if (num_planes > 1)
+			plane_templ[i].bind |= PIPE_BIND_SHARED;
 
-	r = si_init_surface(sscreen, &surface, templ,
-			    si_choose_tiling(sscreen, templ, tc_compatible_htile),
-			    0, 0, false, false, is_flushed_depth,
-			    tc_compatible_htile);
-	if (r) {
-		return NULL;
+		if (si_init_surface(sscreen, &surface[i], &plane_templ[i],
+				    tile_mode, 0, false, false,
+				    is_flushed_depth, tc_compatible_htile))
+			return NULL;
+
+		plane_offset[i] = align64(total_size, surface[i].surf_alignment);
+		total_size = plane_offset[i] + surface[i].total_size;
+		max_alignment = MAX2(max_alignment, surface[i].surf_alignment);
+	}
+
+	struct si_texture *plane0 = NULL, *last_plane = NULL;
+
+	for (unsigned i = 0; i < num_planes; i++) {
+		struct si_texture *tex =
+			si_texture_create_object(screen, &plane_templ[i], &surface[i],
+						 plane0, NULL, plane_offset[i],
+						 total_size, max_alignment);
+		if (!tex) {
+			si_texture_reference(&plane0, NULL);
+			return NULL;
+		}
+
+		tex->plane_index = i;
+		tex->num_planes = num_planes;
+
+		if (!last_plane)
+			plane0 = last_plane = tex;
+		else
+			last_plane->buffer.b.b.next = &tex->buffer.b.b;
 	}
 
-	return (struct pipe_resource *)
-	       si_texture_create_object(screen, templ, NULL, &surface);
+	return (struct pipe_resource *)plane0;
 }
 
 static struct pipe_resource *si_texture_from_winsys_buffer(struct si_screen *sscreen,
@@ -1423,10 +1745,14 @@ static struct pipe_resource *si_texture_from_winsys_buffer(struct si_screen *ssc
 	bool is_scanout;
 	int r;
 
+	/* Ignore metadata for non-zero planes. */
+	if (offset != 0)
+		dedicated = false;
+
 	if (dedicated) {
 		sscreen->ws->buffer_get_metadata(buf, &metadata);
-		si_surface_import_metadata(sscreen, &surface, &metadata,
-					   &array_mode, &is_scanout);
+		si_get_display_metadata(sscreen, &surface, &metadata,
+					&array_mode, &is_scanout);
 	} else {
 		/**
 		 * The bo metadata is unset for un-dedicated images. So we fall
@@ -1455,19 +1781,35 @@ static struct pipe_resource *si_texture_from_winsys_buffer(struct si_screen *ssc
 	}
 
 	r = si_init_surface(sscreen, &surface, templ,
-			    array_mode, stride, offset, true, is_scanout,
+			    array_mode, stride, true, is_scanout,
 			    false, false);
 	if (r)
 		return NULL;
 
-	tex = si_texture_create_object(&sscreen->b, templ, buf, &surface);
+	tex = si_texture_create_object(&sscreen->b, templ, &surface, NULL, buf,
+				       offset, 0, 0);
 	if (!tex)
 		return NULL;
 
 	tex->buffer.b.is_shared = true;
 	tex->buffer.external_usage = usage;
+	tex->num_planes = 1;
 
-	si_apply_opaque_metadata(sscreen, tex, &metadata);
+	if (!si_read_tex_bo_metadata(sscreen, tex, offset, &metadata)) {
+		si_texture_reference(&tex, NULL);
+		return NULL;
+	}
+
+	/* Displayable DCC requires an explicit flush. */
+	if (dedicated && offset == 0 &&
+	    !(usage & PIPE_HANDLE_USAGE_EXPLICIT_FLUSH) &&
+	    si_has_displayable_dcc(tex)) {
+		/* TODO: do we need to decompress DCC? */
+		if (si_texture_discard_dcc(sscreen, tex)) {
+			/* Update BO metadata after disabling DCC. */
+			si_set_tex_bo_metadata(sscreen, tex);
+		}
+	}
 
 	assert(tex->surface.tile_swizzle == 0);
 	return &tex->buffer.b.b;
@@ -1480,62 +1822,58 @@ static struct pipe_resource *si_texture_from_handle(struct pipe_screen *screen,
 {
 	struct si_screen *sscreen = (struct si_screen*)screen;
 	struct pb_buffer *buf = NULL;
-	unsigned stride = 0, offset = 0;
 
 	/* Support only 2D textures without mipmaps */
-	if ((templ->target != PIPE_TEXTURE_2D && templ->target != PIPE_TEXTURE_RECT) ||
-	      templ->depth0 != 1 || templ->last_level != 0)
+	if ((templ->target != PIPE_TEXTURE_2D && templ->target != PIPE_TEXTURE_RECT &&
+	     templ->target != PIPE_TEXTURE_2D_ARRAY) ||
+	      templ->last_level != 0)
 		return NULL;
 
-	buf = sscreen->ws->buffer_from_handle(sscreen->ws, whandle, &stride, &offset);
+	buf = sscreen->ws->buffer_from_handle(sscreen->ws, whandle,
+					      sscreen->info.max_alignment);
 	if (!buf)
 		return NULL;
 
-	return si_texture_from_winsys_buffer(sscreen, templ, buf, stride,
-					     offset, usage, true);
+	return si_texture_from_winsys_buffer(sscreen, templ, buf,
+					     whandle->stride, whandle->offset,
+					     usage, true);
 }
 
 bool si_init_flushed_depth_texture(struct pipe_context *ctx,
-				   struct pipe_resource *texture,
-				   struct si_texture **staging)
+				   struct pipe_resource *texture)
 {
 	struct si_texture *tex = (struct si_texture*)texture;
 	struct pipe_resource resource;
-	struct si_texture **flushed_depth_texture = staging ?
-			staging : &tex->flushed_depth_texture;
 	enum pipe_format pipe_format = texture->format;
 
-	if (!staging) {
-		if (tex->flushed_depth_texture)
-			return true; /* it's ready */
-
-		if (!tex->can_sample_z && tex->can_sample_s) {
-			switch (pipe_format) {
-			case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT:
-				/* Save memory by not allocating the S plane. */
-				pipe_format = PIPE_FORMAT_Z32_FLOAT;
-				break;
-			case PIPE_FORMAT_Z24_UNORM_S8_UINT:
-			case PIPE_FORMAT_S8_UINT_Z24_UNORM:
-				/* Save memory bandwidth by not copying the
-				 * stencil part during flush.
-				 *
-				 * This potentially increases memory bandwidth
-				 * if an application uses both Z and S texturing
-				 * simultaneously (a flushed Z24S8 texture
-				 * would be stored compactly), but how often
-				 * does that really happen?
-				 */
-				pipe_format = PIPE_FORMAT_Z24X8_UNORM;
-				break;
-			default:;
-			}
-		} else if (!tex->can_sample_s && tex->can_sample_z) {
-			assert(util_format_has_stencil(util_format_description(pipe_format)));
-
-			/* DB->CB copies to an 8bpp surface don't work. */
-			pipe_format = PIPE_FORMAT_X24S8_UINT;
+	assert(!tex->flushed_depth_texture);
+
+	if (!tex->can_sample_z && tex->can_sample_s) {
+		switch (pipe_format) {
+		case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT:
+			/* Save memory by not allocating the S plane. */
+			pipe_format = PIPE_FORMAT_Z32_FLOAT;
+			break;
+		case PIPE_FORMAT_Z24_UNORM_S8_UINT:
+		case PIPE_FORMAT_S8_UINT_Z24_UNORM:
+			/* Save memory bandwidth by not copying the
+			 * stencil part during flush.
+			 *
+			 * This potentially increases memory bandwidth
+			 * if an application uses both Z and S texturing
+			 * simultaneously (a flushed Z24S8 texture
+			 * would be stored compactly), but how often
+			 * does that really happen?
+			 */
+			pipe_format = PIPE_FORMAT_Z24X8_UNORM;
+			break;
+		default:;
 		}
+	} else if (!tex->can_sample_s && tex->can_sample_z) {
+		assert(util_format_has_stencil(util_format_description(pipe_format)));
+
+		/* DB->CB copies to an 8bpp surface don't work. */
+		pipe_format = PIPE_FORMAT_X24S8_UINT;
 	}
 
 	memset(&resource, 0, sizeof(resource));
@@ -1547,15 +1885,12 @@ bool si_init_flushed_depth_texture(struct pipe_context *ctx,
 	resource.array_size = texture->array_size;
 	resource.last_level = texture->last_level;
 	resource.nr_samples = texture->nr_samples;
-	resource.usage = staging ? PIPE_USAGE_STAGING : PIPE_USAGE_DEFAULT;
+	resource.usage = PIPE_USAGE_DEFAULT;
 	resource.bind = texture->bind & ~PIPE_BIND_DEPTH_STENCIL;
 	resource.flags = texture->flags | SI_RESOURCE_FLAG_FLUSHED_DEPTH;
 
-	if (staging)
-		resource.flags |= SI_RESOURCE_FLAG_TRANSFER;
-
-	*flushed_depth_texture = (struct si_texture *)ctx->screen->resource_create(ctx->screen, &resource);
-	if (*flushed_depth_texture == NULL) {
+	tex->flushed_depth_texture = (struct si_texture *)ctx->screen->resource_create(ctx->screen, &resource);
+	if (!tex->flushed_depth_texture) {
 		PRINT_ERR("failed to create temporary texture to hold flushed depth\n");
 		return false;
 	}
@@ -1581,6 +1916,25 @@ static void si_init_temp_resource_from_box(struct pipe_resource *res,
 	res->usage = flags & SI_RESOURCE_FLAG_TRANSFER ? PIPE_USAGE_STAGING : PIPE_USAGE_DEFAULT;
 	res->flags = flags;
 
+	if (flags & SI_RESOURCE_FLAG_TRANSFER &&
+	    util_format_is_compressed(orig->format)) {
+		/* Transfer resources are allocated with linear tiling, which is
+		 * not supported for compressed formats.
+		 */
+		unsigned blocksize =
+			util_format_get_blocksize(orig->format);
+
+		if (blocksize == 8) {
+			res->format = PIPE_FORMAT_R16G16B16A16_UINT;
+		} else {
+			assert(blocksize == 16);
+			res->format = PIPE_FORMAT_R32G32B32A32_UINT;
+		}
+
+		res->width0 = util_format_get_nblocksx(orig->format, box->width);
+		res->height0 = util_format_get_nblocksy(orig->format, box->height);
+	}
+
 	/* We must set the correct texture target and dimensions for a 3D box. */
 	if (box->depth > 1 && util_max_layer(orig, level) > 0) {
 		res->target = PIPE_TEXTURE_2D_ARRAY;
@@ -1596,6 +1950,7 @@ static bool si_can_invalidate_texture(struct si_screen *sscreen,
 				      const struct pipe_box *box)
 {
 	return !tex->buffer.b.is_shared &&
+		!(tex->surface.flags & RADEON_SURF_IMPORTED) &&
 		!(transfer_usage & PIPE_TRANSFER_READ) &&
 		tex->buffer.b.b.last_level == 0 &&
 		util_texrange_covers_whole_level(&tex->buffer.b.b, 0,
@@ -1618,11 +1973,11 @@ static void si_texture_invalidate_storage(struct si_context *sctx,
 
 	/* Initialize the CMASK base address (needed even without CMASK). */
 	tex->cmask_base_address_reg =
-		(tex->buffer.gpu_address + tex->cmask_offset) >> 8;
+		(tex->buffer.gpu_address + tex->surface.cmask_offset) >> 8;
 
 	p_atomic_inc(&sscreen->dirty_tex_counter);
 
-	sctx->num_alloc_tex_transfer_bytes += tex->size;
+	sctx->num_alloc_tex_transfer_bytes += tex->surface.total_size;
 }
 
 static void *si_texture_transfer_map(struct pipe_context *ctx,
@@ -1635,7 +1990,7 @@ static void *si_texture_transfer_map(struct pipe_context *ctx,
 	struct si_context *sctx = (struct si_context*)ctx;
 	struct si_texture *tex = (struct si_texture*)texture;
 	struct si_transfer *trans;
-	struct r600_resource *buf;
+	struct si_resource *buf;
 	unsigned offset = 0;
 	char *map;
 	bool use_staging_texture = false;
@@ -1643,8 +1998,10 @@ static void *si_texture_transfer_map(struct pipe_context *ctx,
 	assert(!(texture->flags & SI_RESOURCE_FLAG_TRANSFER));
 	assert(box->width && box->height && box->depth);
 
-	/* Depth textures use staging unconditionally. */
-	if (!tex->is_depth) {
+	if (tex->is_depth) {
+		/* Depth textures use staging unconditionally. */
+		use_staging_texture = true;
+	} else {
 		/* Degrade the tile mode if we get too many transfers on APUs.
 		 * On dGPUs, the staging texture is always faster.
 		 * Only count uploads that are at least 4x4 pixels large.
@@ -1699,68 +2056,7 @@ static void *si_texture_transfer_map(struct pipe_context *ctx,
 	trans->b.b.usage = usage;
 	trans->b.b.box = *box;
 
-	if (tex->is_depth) {
-		struct si_texture *staging_depth;
-
-		if (tex->buffer.b.b.nr_samples > 1) {
-			/* MSAA depth buffers need to be converted to single sample buffers.
-			 *
-			 * Mapping MSAA depth buffers can occur if ReadPixels is called
-			 * with a multisample GLX visual.
-			 *
-			 * First downsample the depth buffer to a temporary texture,
-			 * then decompress the temporary one to staging.
-			 *
-			 * Only the region being mapped is transfered.
-			 */
-			struct pipe_resource resource;
-
-			si_init_temp_resource_from_box(&resource, texture, box, level, 0);
-
-			if (!si_init_flushed_depth_texture(ctx, &resource, &staging_depth)) {
-				PRINT_ERR("failed to create temporary texture to hold untiled copy\n");
-				goto fail_trans;
-			}
-
-			if (usage & PIPE_TRANSFER_READ) {
-				struct pipe_resource *temp = ctx->screen->resource_create(ctx->screen, &resource);
-				if (!temp) {
-					PRINT_ERR("failed to create a temporary depth texture\n");
-					goto fail_trans;
-				}
-
-				si_copy_region_with_blit(ctx, temp, 0, 0, 0, 0, texture, level, box);
-				si_blit_decompress_depth(ctx, (struct si_texture*)temp, staging_depth,
-							 0, 0, 0, box->depth, 0, 0);
-				pipe_resource_reference(&temp, NULL);
-			}
-
-			/* Just get the strides. */
-			si_texture_get_offset(sctx->screen, staging_depth, level, NULL,
-						&trans->b.b.stride,
-						&trans->b.b.layer_stride);
-		} else {
-			/* XXX: only readback the rectangle which is being mapped? */
-			/* XXX: when discard is true, no need to read back from depth texture */
-			if (!si_init_flushed_depth_texture(ctx, texture, &staging_depth)) {
-				PRINT_ERR("failed to create temporary texture to hold untiled copy\n");
-				goto fail_trans;
-			}
-
-			si_blit_decompress_depth(ctx, tex, staging_depth,
-						 level, level,
-						 box->z, box->z + box->depth - 1,
-						 0, 0);
-
-			offset = si_texture_get_offset(sctx->screen, staging_depth,
-							 level, box,
-							 &trans->b.b.stride,
-							 &trans->b.b.layer_stride);
-		}
-
-		trans->staging = &staging_depth->buffer;
-		buf = trans->staging;
-	} else if (use_staging_texture) {
+	if (use_staging_texture) {
 		struct pipe_resource resource;
 		struct si_texture *staging;
 
@@ -1769,6 +2065,13 @@ static void *si_texture_transfer_map(struct pipe_context *ctx,
 		resource.usage = (usage & PIPE_TRANSFER_READ) ?
 			PIPE_USAGE_STAGING : PIPE_USAGE_STREAM;
 
+		/* Since depth-stencil textures don't support linear tiling,
+		 * blit from ZS to color and vice versa. u_blitter will do
+		 * the packing for these formats.
+		 */
+		if (tex->is_depth)
+			resource.format = util_blitter_get_color_format_for_zs(resource.format);
+
 		/* Create the temporary texture. */
 		staging = (struct si_texture*)ctx->screen->resource_create(ctx->screen, &resource);
 		if (!staging) {
@@ -1796,6 +2099,12 @@ static void *si_texture_transfer_map(struct pipe_context *ctx,
 		buf = &tex->buffer;
 	}
 
+	/* Always unmap texture CPU mappings on 32-bit architectures, so that
+	 * we don't run out of the CPU address space.
+	 */
+	if (sizeof(void*) == 4)
+		usage |= RADEON_TRANSFER_TEMPORARY;
+
 	if (!(map = si_buffer_map_sync_with_rings(sctx, buf, usage)))
 		goto fail_trans;
 
@@ -1803,7 +2112,7 @@ static void *si_texture_transfer_map(struct pipe_context *ctx,
 	return map + offset;
 
 fail_trans:
-	r600_resource_reference(&trans->staging, NULL);
+	si_resource_reference(&trans->staging, NULL);
 	pipe_resource_reference(&trans->b.b.resource, NULL);
 	FREE(trans);
 	return NULL;
@@ -1817,20 +2126,22 @@ static void si_texture_transfer_unmap(struct pipe_context *ctx,
 	struct pipe_resource *texture = transfer->resource;
 	struct si_texture *tex = (struct si_texture*)texture;
 
-	if ((transfer->usage & PIPE_TRANSFER_WRITE) && stransfer->staging) {
-		if (tex->is_depth && tex->buffer.b.b.nr_samples <= 1) {
-			ctx->resource_copy_region(ctx, texture, transfer->level,
-						  transfer->box.x, transfer->box.y, transfer->box.z,
-						  &stransfer->staging->b.b, transfer->level,
-						  &transfer->box);
-		} else {
-			si_copy_from_staging_texture(ctx, stransfer);
-		}
+	/* Always unmap texture CPU mappings on 32-bit architectures, so that
+	 * we don't run out of the CPU address space.
+	 */
+	if (sizeof(void*) == 4) {
+		struct si_resource *buf =
+			stransfer->staging ? stransfer->staging : &tex->buffer;
+
+		sctx->ws->buffer_unmap(buf->buf);
 	}
 
+	if ((transfer->usage & PIPE_TRANSFER_WRITE) && stransfer->staging)
+		si_copy_from_staging_texture(ctx, stransfer);
+
 	if (stransfer->staging) {
 		sctx->num_alloc_tex_transfer_bytes += stransfer->staging->buf->size;
-		r600_resource_reference(&stransfer->staging, NULL);
+		si_resource_reference(&stransfer->staging, NULL);
 	}
 
 	/* Heuristic for {upload, draw, upload, draw, ..}:
@@ -1866,7 +2177,8 @@ static const struct u_resource_vtbl si_texture_vtbl =
 
 /* Return if it's allowed to reinterpret one format as another with DCC enabled.
  */
-bool vi_dcc_formats_compatible(enum pipe_format format1,
+bool vi_dcc_formats_compatible(struct si_screen *sscreen,
+			       enum pipe_format format1,
 			       enum pipe_format format2)
 {
 	const struct util_format_description *desc1, *desc2;
@@ -1908,7 +2220,7 @@ bool vi_dcc_formats_compatible(enum pipe_format format1,
 
 	/* If the clear values are all 1 or all 0, this constraint can be
 	 * ignored. */
-	if (vi_alpha_is_on_msb(format1) != vi_alpha_is_on_msb(format2))
+	if (vi_alpha_is_on_msb(sscreen, format1) != vi_alpha_is_on_msb(sscreen, format2))
 		return false;
 
 	/* Channel types must match if the clear value of 1 is used.
@@ -1930,7 +2242,8 @@ bool vi_dcc_formats_are_incompatible(struct pipe_resource *tex,
 	struct si_texture *stex = (struct si_texture *)tex;
 
 	return vi_dcc_enabled(stex, level) &&
-	       !vi_dcc_formats_compatible(tex->format, view_format);
+	       !vi_dcc_formats_compatible((struct si_screen*)tex->screen,
+					  tex->format, view_format);
 }
 
 /* This can't be merged with the above function, because
@@ -2220,9 +2533,9 @@ void vi_separate_dcc_try_enable(struct si_context *sctx,
 	    sctx->screen->debug_flags & DBG(NO_DCC_FB))
 		return;
 
-	assert(sctx->chip_class >= VI);
+	assert(sctx->chip_class >= GFX8);
 
-	if (tex->dcc_offset)
+	if (tex->surface.dcc_offset)
 		return; /* already enabled */
 
 	/* Enable the DCC stat gathering. */
@@ -2257,7 +2570,7 @@ void vi_separate_dcc_try_enable(struct si_context *sctx,
 	}
 
 	/* dcc_offset is the absolute GPUVM address. */
-	tex->dcc_offset = tex->dcc_separate_buffer->gpu_address;
+	tex->surface.dcc_offset = tex->dcc_separate_buffer->gpu_address;
 
 	/* no need to flag anything since this is called by fast clear that
 	 * flags framebuffer state
@@ -2281,11 +2594,10 @@ void vi_separate_dcc_process_and_reset_stats(struct pipe_context *ctx,
 		union pipe_query_result result;
 
 		/* Read the results. */
-		ctx->get_query_result(ctx, sctx->dcc_stats[i].ps_stats[2],
+		struct pipe_query *query = sctx->dcc_stats[i].ps_stats[2];
+		ctx->get_query_result(ctx, query,
 				      true, &result);
-		si_query_hw_reset_buffers(sctx,
-					  (struct si_query_hw*)
-					  sctx->dcc_stats[i].ps_stats[2]);
+		si_query_buffer_reset(sctx, &((struct si_query_hw*)query)->buffer);
 
 		/* Compute the approximate number of fullscreen draws. */
 		tex->ps_draw_ratio =
@@ -2317,7 +2629,7 @@ void vi_separate_dcc_process_and_reset_stats(struct pipe_context *ctx,
 		assert(!tex->last_dcc_separate_buffer);
 		tex->last_dcc_separate_buffer = tex->dcc_separate_buffer;
 		tex->dcc_separate_buffer = NULL;
-		tex->dcc_offset = 0;
+		tex->surface.dcc_offset = 0;
 		/* no need to flag anything since this is called after
 		 * decompression that re-sets framebuffer state
 		 */
@@ -2332,13 +2644,12 @@ si_memobj_from_handle(struct pipe_screen *screen,
 	struct si_screen *sscreen = (struct si_screen*)screen;
 	struct si_memory_object *memobj = CALLOC_STRUCT(si_memory_object);
 	struct pb_buffer *buf = NULL;
-	uint32_t stride, offset;
 
 	if (!memobj)
 		return NULL;
 
 	buf = sscreen->ws->buffer_from_handle(sscreen->ws, whandle,
-					      &stride, &offset);
+					      sscreen->info.max_alignment);
 	if (!buf) {
 		free(memobj);
 		return NULL;
@@ -2346,7 +2657,7 @@ si_memobj_from_handle(struct pipe_screen *screen,
 
 	memobj->b.dedicated = dedicated;
 	memobj->buf = buf;
-	memobj->stride = stride;
+	memobj->stride = whandle->stride;
 
 	return (struct pipe_memory_object *)memobj;
 
@@ -2373,7 +2684,8 @@ si_texture_from_memobj(struct pipe_screen *screen,
 	struct pipe_resource *tex =
 		si_texture_from_winsys_buffer(sscreen, templ, memobj->buf,
 					      memobj->stride, offset,
-					      PIPE_HANDLE_USAGE_READ_WRITE,
+					      PIPE_HANDLE_USAGE_FRAMEBUFFER_WRITE |
+					      PIPE_HANDLE_USAGE_SHADER_WRITE,
 					      memobj->b.dedicated);
 	if (!tex)
 		return NULL;
@@ -2410,6 +2722,8 @@ void si_init_screen_texture_functions(struct si_screen *sscreen)
 {
 	sscreen->b.resource_from_handle = si_texture_from_handle;
 	sscreen->b.resource_get_handle = si_texture_get_handle;
+	sscreen->b.resource_get_param = si_resource_get_param;
+	sscreen->b.resource_get_info = si_texture_get_info;
 	sscreen->b.resource_from_memobj = si_texture_from_memobj;
 	sscreen->b.memobj_create_from_handle = si_memobj_from_handle;
 	sscreen->b.memobj_destroy = si_memobj_destroy;