st/mesa: fix corrupted texture levels, when adding more levels than expected
[mesa.git] / src / mesa / state_tracker / st_cb_texture.c
index 3980f5d2f51c0a6a05685e6d2cc39d794b52aad7..a2d7aa6a6d3ce1dbf4973756ffa4724bf37de52b 100644 (file)
@@ -1,8 +1,8 @@
 /**************************************************************************
- * 
+ *
  * Copyright 2007 VMware, Inc.
  * All Rights Reserved.
- * 
+ *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the
  * "Software"), to deal in the Software without restriction, including
  * distribute, sub license, and/or sell copies of the Software, and to
  * permit persons to whom the Software is furnished to do so, subject to
  * the following conditions:
- * 
+ *
  * The above copyright notice and this permission notice (including the
  * next paragraph) shall be included in all copies or substantial portions
  * of the Software.
- * 
+ *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- * 
+ *
  **************************************************************************/
 
 #include <stdio.h>
 #include "main/bufferobj.h"
 #include "main/enums.h"
+#include "main/errors.h"
 #include "main/fbobject.h"
 #include "main/formats.h"
 #include "main/format_utils.h"
 #include "main/glformats.h"
 #include "main/image.h"
-#include "main/imports.h"
+
 #include "main/macros.h"
 #include "main/mipmap.h"
 #include "main/pack.h"
 #include "main/pbo.h"
 #include "main/pixeltransfer.h"
 #include "main/texcompress.h"
+#include "main/texcompress_astc.h"
 #include "main/texcompress_etc.h"
 #include "main/texgetimage.h"
 #include "main/teximage.h"
 
 #include "state_tracker/st_debug.h"
 #include "state_tracker/st_context.h"
+#include "state_tracker/st_cb_bitmap.h"
 #include "state_tracker/st_cb_fbo.h"
 #include "state_tracker/st_cb_flush.h"
 #include "state_tracker/st_cb_texture.h"
 #include "state_tracker/st_cb_bufferobjects.h"
+#include "state_tracker/st_cb_memoryobjects.h"
 #include "state_tracker/st_format.h"
+#include "state_tracker/st_pbo.h"
 #include "state_tracker/st_texture.h"
 #include "state_tracker/st_gen_mipmap.h"
 #include "state_tracker/st_atom.h"
+#include "state_tracker/st_sampler_view.h"
+#include "state_tracker/st_util.h"
 
 #include "pipe/p_context.h"
 #include "pipe/p_defines.h"
@@ -63,7 +70,7 @@
 #include "util/u_upload_mgr.h"
 #include "pipe/p_shader_tokens.h"
 #include "util/u_tile.h"
-#include "util/u_format.h"
+#include "util/format/u_format.h"
 #include "util/u_surface.h"
 #include "util/u_sampler.h"
 #include "util/u_math.h"
@@ -147,15 +154,35 @@ static struct gl_texture_object *
 st_NewTextureObject(struct gl_context * ctx, GLuint name, GLenum target)
 {
    struct st_texture_object *obj = ST_CALLOC_STRUCT(st_texture_object);
+   if (!obj)
+      return NULL;
+
+   obj->level_override = -1;
+   obj->layer_override = -1;
+
+   /* Pre-allocate a sampler views container to save a branch in the
+    * fast path.
+    */
+   obj->sampler_views = calloc(1, sizeof(struct st_sampler_views)
+                               + sizeof(struct st_sampler_view));
+   if (!obj->sampler_views) {
+      free(obj);
+      return NULL;
+   }
+   obj->sampler_views->max = 1;
 
    DBG("%s\n", __func__);
    _mesa_initialize_texture_object(ctx, &obj->base, name, target);
 
+   simple_mtx_init(&obj->validate_mutex, mtx_plain);
+   obj->needs_validation = true;
+
    return &obj->base;
 }
 
+
 /** called via ctx->Driver.DeleteTextureObject() */
-static void 
+static void
 st_DeleteTextureObject(struct gl_context *ctx,
                        struct gl_texture_object *texObj)
 {
@@ -163,17 +190,35 @@ st_DeleteTextureObject(struct gl_context *ctx,
    struct st_texture_object *stObj = st_texture_object(texObj);
 
    pipe_resource_reference(&stObj->pt, NULL);
-   st_texture_release_all_sampler_views(st, stObj);
-   st_texture_free_sampler_views(stObj);
+   st_delete_texture_sampler_views(st, stObj);
+   simple_mtx_destroy(&stObj->validate_mutex);
    _mesa_delete_texture_object(ctx, texObj);
 }
 
+/**
+ * Called via ctx->Driver.TextureRemovedFromShared()
+ * When texture is removed from ctx->Shared->TexObjects we lose
+ * the ability to clean up views on context destruction, which may
+ * lead to dangling pointers to destroyed contexts.
+ * Release the views to prevent this.
+ */
+static void
+st_TextureReleaseAllSamplerViews(struct gl_context *ctx,
+                                 struct gl_texture_object *texObj)
+{
+   struct st_context *st = st_context(ctx);
+   struct st_texture_object *stObj = st_texture_object(texObj);
+
+   st_texture_release_all_sampler_views(st, stObj);
+}
 
 /** called via ctx->Driver.FreeTextureImageBuffer() */
 static void
 st_FreeTextureImageBuffer(struct gl_context *ctx,
                           struct gl_texture_image *texImage)
 {
+   struct st_context *st = st_context(ctx);
+   struct st_texture_object *stObj = st_texture_object(texImage->TexObject);
    struct st_texture_image *stImage = st_texture_image(texImage);
 
    DBG("%s\n", __func__);
@@ -185,6 +230,73 @@ st_FreeTextureImageBuffer(struct gl_context *ctx,
    free(stImage->transfer);
    stImage->transfer = NULL;
    stImage->num_transfers = 0;
+
+   if (stImage->compressed_data &&
+       pipe_reference(&stImage->compressed_data->reference, NULL)) {
+      free(stImage->compressed_data->ptr);
+      free(stImage->compressed_data);
+      stImage->compressed_data = NULL;
+   }
+
+   /* if the texture image is being deallocated, the structure of the
+    * texture is changing so we'll likely need a new sampler view.
+    */
+   st_texture_release_all_sampler_views(st, stObj);
+}
+
+bool
+st_astc_format_fallback(const struct st_context *st, mesa_format format)
+{
+   if (!_mesa_is_format_astc_2d(format))
+      return false;
+
+   if (format == MESA_FORMAT_RGBA_ASTC_5x5 ||
+       format == MESA_FORMAT_SRGB8_ALPHA8_ASTC_5x5)
+      return !st->has_astc_5x5_ldr;
+
+   return !st->has_astc_2d_ldr;
+}
+
+bool
+st_compressed_format_fallback(struct st_context *st, mesa_format format)
+{
+   if (format == MESA_FORMAT_ETC1_RGB8)
+      return !st->has_etc1;
+
+   if (_mesa_is_format_etc2(format))
+      return !st->has_etc2;
+
+   if (st_astc_format_fallback(st, format))
+      return true;
+
+   return false;
+}
+
+
+static void
+compressed_tex_fallback_allocate(struct st_context *st,
+                                 struct st_texture_image *stImage)
+{
+   struct gl_texture_image *texImage = &stImage->base;
+
+   if (!st_compressed_format_fallback(st, texImage->TexFormat))
+      return;
+
+   if (stImage->compressed_data &&
+       pipe_reference(&stImage->compressed_data->reference, NULL)) {
+      free(stImage->compressed_data->ptr);
+      free(stImage->compressed_data);
+   }
+
+   unsigned data_size = _mesa_format_image_size(texImage->TexFormat,
+                                                texImage->Width2,
+                                                texImage->Height2,
+                                                texImage->Depth2);
+
+   stImage->compressed_data = ST_CALLOC_STRUCT(st_compressed_data);
+   stImage->compressed_data->ptr =
+      malloc(data_size * _mesa_num_tex_faces(texImage->TexObject->Target));
+   pipe_reference_init(&stImage->compressed_data->reference, 1);
 }
 
 
@@ -198,41 +310,45 @@ st_MapTextureImage(struct gl_context *ctx,
 {
    struct st_context *st = st_context(ctx);
    struct st_texture_image *stImage = st_texture_image(texImage);
-   unsigned pipeMode;
    GLubyte *map;
    struct pipe_transfer *transfer;
 
-   pipeMode = 0x0;
-   if (mode & GL_MAP_READ_BIT)
-      pipeMode |= PIPE_TRANSFER_READ;
-   if (mode & GL_MAP_WRITE_BIT)
-      pipeMode |= PIPE_TRANSFER_WRITE;
-   if (mode & GL_MAP_INVALIDATE_RANGE_BIT)
-      pipeMode |= PIPE_TRANSFER_DISCARD_RANGE;
+   /* Check for unexpected flags */
+   assert((mode & ~(GL_MAP_READ_BIT |
+                    GL_MAP_WRITE_BIT |
+                    GL_MAP_INVALIDATE_RANGE_BIT)) == 0);
+
+   const enum pipe_transfer_usage transfer_flags =
+      st_access_flags_to_transfer_flags(mode, false);
 
-   map = st_texture_image_map(st, stImage, pipeMode, x, y, slice, w, h, 1,
+   map = st_texture_image_map(st, stImage, transfer_flags, x, y, slice, w, h, 1,
                               &transfer);
    if (map) {
-      if ((_mesa_is_format_etc2(texImage->TexFormat) && !st->has_etc2) ||
-          (texImage->TexFormat == MESA_FORMAT_ETC1_RGB8 && !st->has_etc1)) {
-         /* ETC isn't supported by gallium and it's represented
-          * by uncompressed formats. Only write transfers with precompressed
-          * data are supported by ES3, which makes this really simple.
+      if (st_compressed_format_fallback(st, texImage->TexFormat)) {
+         /* Some compressed formats don't have to be supported by drivers,
+          * and st/mesa transparently handles decompression on upload (Unmap),
+          * so that drivers don't see the compressed formats.
           *
-          * Just create a temporary storage where the ETC texture will
-          * be stored. It will be decompressed in the Unmap function.
+          * We store the compressed data (it's needed for glGetCompressedTex-
+          * Image and image copies in OES_copy_image).
           */
          unsigned z = transfer->box.z;
          struct st_texture_image_transfer *itransfer = &stImage->transfer[z];
 
-         itransfer->temp_data =
-            malloc(_mesa_format_image_size(texImage->TexFormat, w, h, 1));
-         itransfer->temp_stride =
-            _mesa_format_row_stride(texImage->TexFormat, w);
-         itransfer->map = map;
+         unsigned blk_w, blk_h;
+         _mesa_get_format_block_size(texImage->TexFormat, &blk_w, &blk_h);
+
+         unsigned y_blocks = DIV_ROUND_UP(texImage->Height2, blk_h);
+         unsigned stride = *rowStrideOut = itransfer->temp_stride =
+            _mesa_format_row_stride(texImage->TexFormat, texImage->Width2);
+         unsigned block_size = _mesa_get_format_bytes(texImage->TexFormat);
 
-         *mapOut = itransfer->temp_data;
-         *rowStrideOut = itransfer->temp_stride;
+         assert(stImage->compressed_data);
+         *mapOut = itransfer->temp_data =
+            stImage->compressed_data->ptr +
+            (z * y_blocks + (y / blk_h)) * stride +
+            (x / blk_w) * block_size;
+         itransfer->map = map;
       }
       else {
          /* supported mapping */
@@ -256,29 +372,41 @@ st_UnmapTextureImage(struct gl_context *ctx,
    struct st_context *st = st_context(ctx);
    struct st_texture_image *stImage  = st_texture_image(texImage);
 
-   if ((_mesa_is_format_etc2(texImage->TexFormat) && !st->has_etc2) ||
-       (texImage->TexFormat == MESA_FORMAT_ETC1_RGB8 && !st->has_etc1)) {
-      /* Decompress the ETC texture to the mapped one. */
+   if (st_compressed_format_fallback(st, texImage->TexFormat)) {
+      /* Decompress the compressed image on upload if the driver doesn't
+       * support the compressed format. */
       unsigned z = slice + stImage->base.Face;
       struct st_texture_image_transfer *itransfer = &stImage->transfer[z];
       struct pipe_transfer *transfer = itransfer->transfer;
 
       assert(z == transfer->box.z);
 
-      if (texImage->TexFormat == MESA_FORMAT_ETC1_RGB8) {
-         _mesa_etc1_unpack_rgba8888(itransfer->map, transfer->stride,
-                                    itransfer->temp_data,
-                                    itransfer->temp_stride,
-                                    transfer->box.width, transfer->box.height);
-      }
-      else {
-         _mesa_unpack_etc2_format(itransfer->map, transfer->stride,
-                                  itransfer->temp_data, itransfer->temp_stride,
-                                  transfer->box.width, transfer->box.height,
-                                  texImage->TexFormat);
+      if (transfer->usage & PIPE_TRANSFER_WRITE) {
+         if (texImage->TexFormat == MESA_FORMAT_ETC1_RGB8) {
+            _mesa_etc1_unpack_rgba8888(itransfer->map, transfer->stride,
+                                       itransfer->temp_data,
+                                       itransfer->temp_stride,
+                                       transfer->box.width,
+                                       transfer->box.height);
+         } else if (_mesa_is_format_etc2(texImage->TexFormat)) {
+            bool bgra = stImage->pt->format == PIPE_FORMAT_B8G8R8A8_SRGB;
+            _mesa_unpack_etc2_format(itransfer->map, transfer->stride,
+                                     itransfer->temp_data,
+                                     itransfer->temp_stride,
+                                     transfer->box.width, transfer->box.height,
+                                     texImage->TexFormat,
+                                     bgra);
+         } else if (_mesa_is_format_astc_2d(texImage->TexFormat)) {
+            _mesa_unpack_astc_2d_ldr(itransfer->map, transfer->stride,
+                                     itransfer->temp_data,
+                                     itransfer->temp_stride,
+                                     transfer->box.width, transfer->box.height,
+                                     texImage->TexFormat);
+         } else {
+            unreachable("unexpected format for a compressed format fallback");
+         }
       }
 
-      free(itransfer->temp_data);
       itransfer->temp_data = NULL;
       itransfer->temp_stride = 0;
       itransfer->map = 0;
@@ -303,13 +431,13 @@ default_bindings(struct st_context *st, enum pipe_format format)
    else
       bindings = PIPE_BIND_SAMPLER_VIEW | PIPE_BIND_RENDER_TARGET;
 
-   if (screen->is_format_supported(screen, format, target, 0, bindings))
+   if (screen->is_format_supported(screen, format, target, 0, 0, bindings))
       return bindings;
    else {
       /* Try non-sRGB. */
       format = util_format_linear(format);
 
-      if (screen->is_format_supported(screen, format, target, 0, bindings))
+      if (screen->is_format_supported(screen, format, target, 0, 0, bindings))
          return bindings;
       else
          return PIPE_BIND_SAMPLER_VIEW;
@@ -330,7 +458,7 @@ static GLboolean
 guess_base_level_size(GLenum target,
                       GLuint width, GLuint height, GLuint depth, GLuint level,
                       GLuint *width0, GLuint *height0, GLuint *depth0)
-{ 
+{
    assert(width >= 1);
    assert(height >= 1);
    assert(depth >= 1);
@@ -421,6 +549,17 @@ allocate_full_mipmap(const struct st_texture_object *stObj,
    if (stImage->base.Level > 0 || stObj->base.GenerateMipmap)
       return TRUE;
 
+   /* If the application has explicitly called glTextureParameter to set
+    * GL_TEXTURE_MAX_LEVEL, such that (max - base) > 0, then they're trying
+    * to communicate that they will have multiple miplevels.
+    *
+    * Core Mesa will initialize MaxLevel to value much larger than
+    * MAX_TEXTURE_LEVELS, so we check that to see if it's been set at all.
+    */
+   if (stObj->base.MaxLevel < MAX_TEXTURE_LEVELS &&
+       stObj->base.MaxLevel - stObj->base.BaseLevel > 0)
+      return TRUE;
+
    if (stImage->base._BaseFormat == GL_DEPTH_COMPONENT ||
        stImage->base._BaseFormat == GL_DEPTH_STENCIL_EXT)
       /* depth/stencil textures are seldom mipmapped */
@@ -434,6 +573,18 @@ allocate_full_mipmap(const struct st_texture_object *stObj,
       /* not a mipmap minification filter */
       return FALSE;
 
+   /* If the following sequence of GL calls is used:
+    *   glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, w, h, 0, GL_RGB, ...
+    *   glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
+    *
+    * we would needlessly allocate a mipmapped texture, because the initial
+    * MinFilter is GL_NEAREST_MIPMAP_LINEAR. Catch this case and don't
+    * allocate a mipmapped texture by default. This may cause texture
+    * reallocation later, but GL_NEAREST_MIPMAP_LINEAR is pretty rare.
+    */
+   if (stObj->base.Sampler.MinFilter == GL_NEAREST_MIPMAP_LINEAR)
+      return FALSE;
+
    if (stObj->base.Target == GL_TEXTURE_3D)
       /* 3D textures are seldom mipmapped */
       return FALSE;
@@ -452,26 +603,50 @@ allocate_full_mipmap(const struct st_texture_object *stObj,
  */
 static GLboolean
 guess_and_alloc_texture(struct st_context *st,
-                       struct st_texture_object *stObj,
-                       const struct st_texture_image *stImage)
+                        struct st_texture_object *stObj,
+                        const struct st_texture_image *stImage)
 {
+   const struct gl_texture_image *firstImage;
    GLuint lastLevel, width, height, depth;
    GLuint bindings;
-   GLuint ptWidth, ptHeight, ptDepth, ptLayers;
+   unsigned ptWidth;
+   uint16_t ptHeight, ptDepth, ptLayers;
    enum pipe_format fmt;
+   bool guessed_box = false;
 
    DBG("%s\n", __func__);
 
    assert(!stObj->pt);
 
-   if (!guess_base_level_size(stObj->base.Target,
-                              stImage->base.Width2,
-                              stImage->base.Height2,
-                              stImage->base.Depth2,
-                              stImage->base.Level,
-                              &width, &height, &depth)) {
+   /* If a base level image with compatible size exists, use that as our guess.
+    */
+   firstImage = _mesa_base_tex_image(&stObj->base);
+   if (firstImage &&
+       firstImage->Width2 > 0 &&
+       firstImage->Height2 > 0 &&
+       firstImage->Depth2 > 0 &&
+       guess_base_level_size(stObj->base.Target,
+                             firstImage->Width2,
+                             firstImage->Height2,
+                             firstImage->Depth2,
+                             firstImage->Level,
+                             &width, &height, &depth)) {
+      if (stImage->base.Width2 == u_minify(width, stImage->base.Level) &&
+          stImage->base.Height2 == u_minify(height, stImage->base.Level) &&
+          stImage->base.Depth2 == u_minify(depth, stImage->base.Level))
+         guessed_box = true;
+   }
+
+   if (!guessed_box)
+      guessed_box = guess_base_level_size(stObj->base.Target,
+                                          stImage->base.Width2,
+                                          stImage->base.Height2,
+                                          stImage->base.Depth2,
+                                          stImage->base.Level,
+                                          &width, &height, &depth);
+
+   if (!guessed_box) {
       /* we can't determine the image size at level=0 */
-      stObj->width0 = stObj->height0 = stObj->depth0 = 0;
       /* this is not an out of memory error */
       return GL_TRUE;
    }
@@ -496,11 +671,6 @@ guess_and_alloc_texture(struct st_context *st,
       lastLevel = 0;
    }
 
-   /* Save the level=0 dimensions */
-   stObj->width0 = width;
-   stObj->height0 = height;
-   stObj->depth0 = depth;
-
    fmt = st_mesa_format_to_pipe_format(st, stImage->base.TexFormat);
 
    bindings = default_bindings(st, fmt);
@@ -539,7 +709,6 @@ st_AllocTextureImageBuffer(struct gl_context *ctx,
    struct st_context *st = st_context(ctx);
    struct st_texture_image *stImage = st_texture_image(texImage);
    struct st_texture_object *stObj = st_texture_object(texImage->TexObject);
-   const GLuint level = texImage->Level;
    GLuint width = texImage->Width;
    GLuint height = texImage->Height;
    GLuint depth = texImage->Depth;
@@ -548,28 +717,37 @@ st_AllocTextureImageBuffer(struct gl_context *ctx,
 
    assert(!stImage->pt); /* xxx this might be wrong */
 
-   /* Look if the parent texture object has space for this image */
-   if (stObj->pt &&
-       level <= stObj->pt->last_level &&
-       st_texture_match_image(st, stObj->pt, texImage)) {
-      /* this image will fit in the existing texture object's memory */
-      pipe_resource_reference(&stImage->pt, stObj->pt);
-      return GL_TRUE;
-   }
+   stObj->needs_validation = true;
 
-   /* The parent texture object does not have space for this image */
+   compressed_tex_fallback_allocate(st, stImage);
+   const bool allowAllocateToStObj = !stObj->pt ||
+                                     stObj->pt->last_level == 0 ||
+                                     texImage->Level == 0;
 
-   pipe_resource_reference(&stObj->pt, NULL);
-   st_texture_release_all_sampler_views(st, stObj);
+   if (allowAllocateToStObj) {
+      /* Look if the parent texture object has space for this image */
+      if (stObj->pt &&
+          st_texture_match_image(st, stObj->pt, texImage)) {
+         /* this image will fit in the existing texture object's memory */
+         pipe_resource_reference(&stImage->pt, stObj->pt);
+         assert(stImage->pt);
+         return GL_TRUE;
+      }
+
+      /* The parent texture object does not have space for this image */
+
+      pipe_resource_reference(&stObj->pt, NULL);
+      st_texture_release_all_sampler_views(st, stObj);
 
-   if (!guess_and_alloc_texture(st, stObj, stImage)) {
-      /* Probably out of memory.
-       * Try flushing any pending rendering, then retry.
-       */
-      st_finish(st);
       if (!guess_and_alloc_texture(st, stObj, stImage)) {
-         _mesa_error(ctx, GL_OUT_OF_MEMORY, "glTexImage");
-         return GL_FALSE;
+         /* Probably out of memory.
+         * Try flushing any pending rendering, then retry.
+         */
+         st_finish(st);
+         if (!guess_and_alloc_texture(st, stObj, stImage)) {
+            _mesa_error(ctx, GL_OUT_OF_MEMORY, "glTexImage");
+            return GL_FALSE;
+         }
       }
    }
 
@@ -590,7 +768,8 @@ st_AllocTextureImageBuffer(struct gl_context *ctx,
       enum pipe_format format =
          st_mesa_format_to_pipe_format(st, texImage->TexFormat);
       GLuint bindings = default_bindings(st, format);
-      GLuint ptWidth, ptHeight, ptDepth, ptLayers;
+      unsigned ptWidth;
+      uint16_t ptHeight, ptDepth, ptLayers;
 
       st_gl_texture_dims_to_pipe_dims(stObj->base.Target,
                                       width, height, depth,
@@ -627,7 +806,10 @@ prep_teximage(struct gl_context *ctx, struct gl_texture_image *texImage,
       const GLuint level = texImage->Level;
       mesa_format texFormat;
 
-      _mesa_clear_texture_object(ctx, texObj);
+      assert(!st_texture_image(texImage)->pt);
+      _mesa_clear_texture_object(ctx, texObj, texImage);
+      stObj->layer_override = -1;
+      stObj->level_override = -1;
       pipe_resource_reference(&stObj->pt, NULL);
 
       /* oops, need to init this image again */
@@ -678,6 +860,7 @@ st_get_blit_mask(GLenum srcFormat, GLenum dstFormat)
 
    case GL_STENCIL_INDEX:
       switch (srcFormat) {
+      case GL_DEPTH_STENCIL:
       case GL_STENCIL_INDEX:
          return PIPE_MASK_S;
       default:
@@ -690,59 +873,6 @@ st_get_blit_mask(GLenum srcFormat, GLenum dstFormat)
    }
 }
 
-void
-st_init_pbo_upload(struct st_context *st)
-{
-   struct pipe_context *pipe = st->pipe;
-   struct pipe_screen *screen = pipe->screen;
-
-   st->pbo_upload.enabled =
-      screen->get_param(screen, PIPE_CAP_TEXTURE_BUFFER_OBJECTS) &&
-      screen->get_param(screen, PIPE_CAP_TEXTURE_BUFFER_OFFSET_ALIGNMENT) >= 1 &&
-      screen->get_shader_param(screen, PIPE_SHADER_FRAGMENT, PIPE_SHADER_CAP_INTEGERS);
-   if (!st->pbo_upload.enabled)
-      return;
-
-   st->pbo_upload.rgba_only =
-      screen->get_param(screen, PIPE_CAP_BUFFER_SAMPLER_VIEW_RGBA_ONLY);
-
-   if (screen->get_param(screen, PIPE_CAP_TGSI_INSTANCEID)) {
-      if (screen->get_param(screen, PIPE_CAP_TGSI_VS_LAYER_VIEWPORT)) {
-         st->pbo_upload.upload_layers = true;
-      } else if (screen->get_param(screen, PIPE_CAP_MAX_GEOMETRY_OUTPUT_VERTICES) >= 3) {
-         st->pbo_upload.upload_layers = true;
-         st->pbo_upload.use_gs = true;
-      }
-   }
-
-   /* Blend state */
-   memset(&st->pbo_upload.blend, 0, sizeof(struct pipe_blend_state));
-   st->pbo_upload.blend.rt[0].colormask = PIPE_MASK_RGBA;
-
-   /* Rasterizer state */
-   memset(&st->pbo_upload.raster, 0, sizeof(struct pipe_rasterizer_state));
-   st->pbo_upload.raster.half_pixel_center = 1;
-}
-
-void
-st_destroy_pbo_upload(struct st_context *st)
-{
-   if (st->pbo_upload.fs) {
-      cso_delete_fragment_shader(st->cso_context, st->pbo_upload.fs);
-      st->pbo_upload.fs = NULL;
-   }
-
-   if (st->pbo_upload.gs) {
-      cso_delete_geometry_shader(st->cso_context, st->pbo_upload.gs);
-      st->pbo_upload.gs = NULL;
-   }
-
-   if (st->pbo_upload.vs) {
-      cso_delete_vertex_shader(st->cso_context, st->pbo_upload.vs);
-      st->pbo_upload.vs = NULL;
-   }
-}
-
 /**
  * Converts format to a format with the same components, types
  * and sizes, but with the components in RGBA order.
@@ -771,6 +901,7 @@ unswizzle_format(enum pipe_format format)
    }
 }
 
+
 /**
  * Converts PIPE_FORMAT_A* to PIPE_FORMAT_R*.
  */
@@ -811,6 +942,7 @@ alpha_to_red(enum pipe_format format)
    }
 }
 
+
 /**
  * Converts PIPE_FORMAT_R*A* to PIPE_FORMAT_R*G*.
  */
@@ -851,6 +983,7 @@ red_alpha_to_red_green(enum pipe_format format)
    }
 }
 
+
 /**
  * Converts PIPE_FORMAT_L*A* to PIPE_FORMAT_R*G*.
  */
@@ -891,6 +1024,7 @@ luminance_alpha_to_red_green(enum pipe_format format)
    }
 }
 
+
 /**
  * Returns true if format is a PIPE_FORMAT_A* format, and false otherwise.
  */
@@ -900,15 +1034,16 @@ format_is_alpha(enum pipe_format format)
    const struct util_format_description *desc = util_format_description(format);
 
    if (desc->nr_channels == 1 &&
-       desc->swizzle[0] == UTIL_FORMAT_SWIZZLE_0 &&
-       desc->swizzle[1] == UTIL_FORMAT_SWIZZLE_0 &&
-       desc->swizzle[2] == UTIL_FORMAT_SWIZZLE_0 &&
-       desc->swizzle[3] == UTIL_FORMAT_SWIZZLE_X)
+       desc->swizzle[0] == PIPE_SWIZZLE_0 &&
+       desc->swizzle[1] == PIPE_SWIZZLE_0 &&
+       desc->swizzle[2] == PIPE_SWIZZLE_0 &&
+       desc->swizzle[3] == PIPE_SWIZZLE_X)
       return true;
 
    return false;
 }
 
+
 /**
  * Returns true if format is a PIPE_FORMAT_R* format, and false otherwise.
  */
@@ -918,10 +1053,10 @@ format_is_red(enum pipe_format format)
    const struct util_format_description *desc = util_format_description(format);
 
    if (desc->nr_channels == 1 &&
-       desc->swizzle[0] == UTIL_FORMAT_SWIZZLE_X &&
-       desc->swizzle[1] == UTIL_FORMAT_SWIZZLE_0 &&
-       desc->swizzle[2] == UTIL_FORMAT_SWIZZLE_0 &&
-       desc->swizzle[3] == UTIL_FORMAT_SWIZZLE_1)
+       desc->swizzle[0] == PIPE_SWIZZLE_X &&
+       desc->swizzle[1] == PIPE_SWIZZLE_0 &&
+       desc->swizzle[2] == PIPE_SWIZZLE_0 &&
+       desc->swizzle[3] == PIPE_SWIZZLE_1)
       return true;
 
    return false;
@@ -937,10 +1072,10 @@ format_is_luminance(enum pipe_format format)
    const struct util_format_description *desc = util_format_description(format);
 
    if (desc->nr_channels == 1 &&
-       desc->swizzle[0] == UTIL_FORMAT_SWIZZLE_X &&
-       desc->swizzle[1] == UTIL_FORMAT_SWIZZLE_X &&
-       desc->swizzle[2] == UTIL_FORMAT_SWIZZLE_X &&
-       desc->swizzle[3] == UTIL_FORMAT_SWIZZLE_1)
+       desc->swizzle[0] == PIPE_SWIZZLE_X &&
+       desc->swizzle[1] == PIPE_SWIZZLE_X &&
+       desc->swizzle[2] == PIPE_SWIZZLE_X &&
+       desc->swizzle[3] == PIPE_SWIZZLE_1)
       return true;
 
    return false;
@@ -955,29 +1090,31 @@ format_is_red_alpha(enum pipe_format format)
    const struct util_format_description *desc = util_format_description(format);
 
    if (desc->nr_channels == 2 &&
-       desc->swizzle[0] == UTIL_FORMAT_SWIZZLE_X &&
-       desc->swizzle[1] == UTIL_FORMAT_SWIZZLE_0 &&
-       desc->swizzle[2] == UTIL_FORMAT_SWIZZLE_0 &&
-       desc->swizzle[3] == UTIL_FORMAT_SWIZZLE_Y)
+       desc->swizzle[0] == PIPE_SWIZZLE_X &&
+       desc->swizzle[1] == PIPE_SWIZZLE_0 &&
+       desc->swizzle[2] == PIPE_SWIZZLE_0 &&
+       desc->swizzle[3] == PIPE_SWIZZLE_Y)
       return true;
 
    return false;
 }
 
+
 static bool
 format_is_swizzled_rgba(enum pipe_format format)
 {
     const struct util_format_description *desc = util_format_description(format);
 
-    if ((desc->swizzle[0] == TGSI_SWIZZLE_X || desc->swizzle[0] == UTIL_FORMAT_SWIZZLE_0) &&
-        (desc->swizzle[1] == TGSI_SWIZZLE_Y || desc->swizzle[1] == UTIL_FORMAT_SWIZZLE_0) &&
-        (desc->swizzle[2] == TGSI_SWIZZLE_Z || desc->swizzle[2] == UTIL_FORMAT_SWIZZLE_0) &&
-        (desc->swizzle[3] == TGSI_SWIZZLE_W || desc->swizzle[3] == UTIL_FORMAT_SWIZZLE_1))
+    if ((desc->swizzle[0] == TGSI_SWIZZLE_X || desc->swizzle[0] == PIPE_SWIZZLE_0) &&
+        (desc->swizzle[1] == TGSI_SWIZZLE_Y || desc->swizzle[1] == PIPE_SWIZZLE_0) &&
+        (desc->swizzle[2] == TGSI_SWIZZLE_Z || desc->swizzle[2] == PIPE_SWIZZLE_0) &&
+        (desc->swizzle[3] == TGSI_SWIZZLE_W || desc->swizzle[3] == PIPE_SWIZZLE_1))
        return false;
 
     return true;
 }
 
+
 struct format_table
 {
    unsigned char swizzle[4];
@@ -1101,10 +1238,10 @@ reinterpret_formats(enum pipe_format *src_format, enum pipe_format *dst_format)
       unsigned i;
 
       /* Make sure the format is an RGBA and not an RGBX format */
-      if (src_desc->nr_channels != 4 || src_desc->swizzle[3] == UTIL_FORMAT_SWIZZLE_1)
+      if (src_desc->nr_channels != 4 || src_desc->swizzle[3] == PIPE_SWIZZLE_1)
          return false;
 
-      if (dst_desc->nr_channels != 4 || dst_desc->swizzle[3] == UTIL_FORMAT_SWIZZLE_1)
+      if (dst_desc->nr_channels != 4 || dst_desc->swizzle[3] == PIPE_SWIZZLE_1)
          return false;
 
       for (i = 0; i < 4; i++)
@@ -1122,221 +1259,23 @@ reinterpret_formats(enum pipe_format *src_format, enum pipe_format *dst_format)
    return true;
 }
 
-static void *
-create_pbo_upload_vs(struct st_context *st)
-{
-   struct ureg_program *ureg;
-   struct ureg_src in_pos;
-   struct ureg_src in_instanceid;
-   struct ureg_dst out_pos;
-   struct ureg_dst out_layer;
-
-   ureg = ureg_create(TGSI_PROCESSOR_VERTEX);
-   if (!ureg)
-      return NULL;
-
-   in_pos = ureg_DECL_vs_input(ureg, TGSI_SEMANTIC_POSITION);
-
-   out_pos = ureg_DECL_output(ureg, TGSI_SEMANTIC_POSITION, 0);
-
-   if (st->pbo_upload.upload_layers) {
-      in_instanceid = ureg_DECL_system_value(ureg, TGSI_SEMANTIC_INSTANCEID, 0);
-
-      if (!st->pbo_upload.use_gs)
-         out_layer = ureg_DECL_output(ureg, TGSI_SEMANTIC_LAYER, 0);
-   }
-
-   /* out_pos = in_pos */
-   ureg_MOV(ureg, out_pos, in_pos);
-
-   if (st->pbo_upload.upload_layers) {
-      if (st->pbo_upload.use_gs) {
-         /* out_pos.z = i2f(gl_InstanceID) */
-         ureg_I2F(ureg, ureg_writemask(out_pos, TGSI_WRITEMASK_Z),
-                        ureg_scalar(in_instanceid, TGSI_SWIZZLE_X));
-      } else {
-         /* out_layer = gl_InstanceID */
-         ureg_MOV(ureg, out_layer, in_instanceid);
-      }
-   }
-
-   ureg_END(ureg);
-
-   return ureg_create_shader_and_destroy(ureg, st->pipe);
-}
-
-static void *
-create_pbo_upload_gs(struct st_context *st)
-{
-   static const int zero = 0;
-   struct ureg_program *ureg;
-   struct ureg_dst out_pos;
-   struct ureg_dst out_layer;
-   struct ureg_src in_pos;
-   struct ureg_src imm;
-   unsigned i;
-
-   ureg = ureg_create(TGSI_PROCESSOR_GEOMETRY);
-   if (!ureg)
-      return NULL;
-
-   ureg_property(ureg, TGSI_PROPERTY_GS_INPUT_PRIM, PIPE_PRIM_TRIANGLES);
-   ureg_property(ureg, TGSI_PROPERTY_GS_OUTPUT_PRIM, PIPE_PRIM_TRIANGLE_STRIP);
-   ureg_property(ureg, TGSI_PROPERTY_GS_MAX_OUTPUT_VERTICES, 3);
-
-   out_pos = ureg_DECL_output(ureg, TGSI_SEMANTIC_POSITION, 0);
-   out_layer = ureg_DECL_output(ureg, TGSI_SEMANTIC_LAYER, 0);
-
-   in_pos = ureg_DECL_input(ureg, TGSI_SEMANTIC_POSITION, 0, 0, 1);
-
-   imm = ureg_DECL_immediate_int(ureg, &zero, 1);
-
-   for (i = 0; i < 3; ++i) {
-      struct ureg_src in_pos_vertex = ureg_src_dimension(in_pos, i);
-
-      /* out_pos = in_pos[i] */
-      ureg_MOV(ureg, out_pos, in_pos_vertex);
-
-      /* out_layer.x = f2i(in_pos[i].z) */
-      ureg_F2I(ureg, ureg_writemask(out_layer, TGSI_WRITEMASK_X),
-                     ureg_scalar(in_pos_vertex, TGSI_SWIZZLE_Z));
-
-      ureg_EMIT(ureg, ureg_scalar(imm, TGSI_SWIZZLE_X));
-   }
-
-   ureg_END(ureg);
-
-   return ureg_create_shader_and_destroy(ureg, st->pipe);
-}
-
-static void *
-create_pbo_upload_fs(struct st_context *st)
-{
-   struct pipe_context *pipe = st->pipe;
-   struct pipe_screen *screen = pipe->screen;
-   struct ureg_program *ureg;
-   struct ureg_dst out;
-   struct ureg_src sampler;
-   struct ureg_src pos;
-   struct ureg_src layer;
-   struct ureg_src const0;
-   struct ureg_dst temp0;
-
-   ureg = ureg_create(TGSI_PROCESSOR_FRAGMENT);
-   if (!ureg)
-      return NULL;
-
-   out     = ureg_DECL_output(ureg, TGSI_SEMANTIC_COLOR, 0);
-   sampler = ureg_DECL_sampler(ureg, 0);
-   if (screen->get_param(screen, PIPE_CAP_TGSI_FS_POSITION_IS_SYSVAL)) {
-      pos = ureg_DECL_system_value(ureg, TGSI_SEMANTIC_POSITION, 0);
-   } else {
-      pos = ureg_DECL_fs_input(ureg, TGSI_SEMANTIC_POSITION, 0,
-                               TGSI_INTERPOLATE_LINEAR);
-   }
-   if (st->pbo_upload.upload_layers) {
-      layer = ureg_DECL_fs_input(ureg, TGSI_SEMANTIC_LAYER, 0,
-                                       TGSI_INTERPOLATE_CONSTANT);
-   }
-   const0  = ureg_DECL_constant(ureg, 0);
-   temp0   = ureg_DECL_temporary(ureg);
-
-   /* Note: const0 = [ -xoffset + skip_pixels, -yoffset, stride, image_height ] */
-
-   /* temp0.xy = f2i(temp0.xy) */
-   ureg_F2I(ureg, ureg_writemask(temp0, TGSI_WRITEMASK_XY),
-                  ureg_swizzle(pos,
-                               TGSI_SWIZZLE_X, TGSI_SWIZZLE_Y,
-                               TGSI_SWIZZLE_Y, TGSI_SWIZZLE_Y));
-
-   /* temp0.xy = temp0.xy + const0.xy */
-   ureg_UADD(ureg, ureg_writemask(temp0, TGSI_WRITEMASK_XY),
-                   ureg_swizzle(ureg_src(temp0),
-                                TGSI_SWIZZLE_X, TGSI_SWIZZLE_Y,
-                                TGSI_SWIZZLE_Y, TGSI_SWIZZLE_Y),
-                   ureg_swizzle(const0,
-                                TGSI_SWIZZLE_X, TGSI_SWIZZLE_Y,
-                                TGSI_SWIZZLE_Y, TGSI_SWIZZLE_Y));
-
-   /* temp0.x = const0.z * temp0.y + temp0.x */
-   ureg_UMAD(ureg, ureg_writemask(temp0, TGSI_WRITEMASK_X),
-                   ureg_scalar(const0, TGSI_SWIZZLE_Z),
-                   ureg_scalar(ureg_src(temp0), TGSI_SWIZZLE_Y),
-                   ureg_scalar(ureg_src(temp0), TGSI_SWIZZLE_X));
-
-   if (st->pbo_upload.upload_layers) {
-      /* temp0.x = const0.w * layer + temp0.x */
-      ureg_UMAD(ureg, ureg_writemask(temp0, TGSI_WRITEMASK_X),
-                      ureg_scalar(const0, TGSI_SWIZZLE_W),
-                      ureg_scalar(layer, TGSI_SWIZZLE_X),
-                      ureg_scalar(ureg_src(temp0), TGSI_SWIZZLE_X));
-   }
-
-   /* temp0.w = 0 */
-   ureg_MOV(ureg, ureg_writemask(temp0, TGSI_WRITEMASK_W), ureg_imm1u(ureg, 0));
-
-   /* out = txf(sampler, temp0.x) */
-   ureg_TXF(ureg, out, TGSI_TEXTURE_BUFFER, ureg_src(temp0), sampler);
-
-   ureg_release_temporary(ureg, temp0);
-
-   ureg_END(ureg);
-
-   return ureg_create_shader_and_destroy(ureg, pipe);
-}
-
 static bool
 try_pbo_upload_common(struct gl_context *ctx,
                       struct pipe_surface *surface,
-                      int xoffset, int yoffset,
-                      unsigned upload_width, unsigned upload_height,
-                      struct pipe_resource *buffer,
-                      enum pipe_format src_format,
-                      intptr_t buf_offset,
-                      unsigned bytes_per_pixel,
-                      unsigned stride,
-                      unsigned image_height)
+                      const struct st_pbo_addresses *addr,
+                      enum pipe_format src_format)
 {
    struct st_context *st = st_context(ctx);
    struct cso_context *cso = st->cso_context;
    struct pipe_context *pipe = st->pipe;
-   unsigned depth = surface->u.tex.last_layer - surface->u.tex.first_layer + 1;
-   unsigned skip_pixels = 0;
    bool success = false;
+   void *fs;
 
-   /* Check alignment. */
-   {
-      unsigned ofs = (buf_offset * bytes_per_pixel) % ctx->Const.TextureBufferOffsetAlignment;
-      if (ofs != 0) {
-         if (ofs % bytes_per_pixel != 0)
-            return false;
-
-         skip_pixels = ofs / bytes_per_pixel;
-         buf_offset -= skip_pixels;
-      }
-   }
-
-   /* Create the shaders */
-   if (!st->pbo_upload.vs) {
-      st->pbo_upload.vs = create_pbo_upload_vs(st);
-      if (!st->pbo_upload.vs)
-         return false;
-   }
-
-   if (depth != 1 && st->pbo_upload.use_gs && !st->pbo_upload.gs) {
-      st->pbo_upload.gs = create_pbo_upload_gs(st);
-      if (!st->pbo_upload.gs)
-         return false;
-   }
-
-   if (!st->pbo_upload.fs) {
-      st->pbo_upload.fs = create_pbo_upload_fs(st);
-      if (!st->pbo_upload.fs)
-         return false;
-   }
+   fs = st_pbo_get_upload_fs(st, src_format, surface->format);
+   if (!fs)
+      return false;
 
    cso_save_state(cso, (CSO_BIT_FRAGMENT_SAMPLER_VIEWS |
-                        CSO_BIT_FRAGMENT_SAMPLERS |
                         CSO_BIT_VERTEX_ELEMENTS |
                         CSO_BIT_AUX_VERTEX_BUFFER_SLOT |
                         CSO_BIT_FRAMEBUFFER |
@@ -1345,128 +1284,40 @@ try_pbo_upload_common(struct gl_context *ctx,
                         CSO_BIT_DEPTH_STENCIL_ALPHA |
                         CSO_BIT_RASTERIZER |
                         CSO_BIT_STREAM_OUTPUTS |
+                        (st->active_queries ? CSO_BIT_PAUSE_QUERIES : 0) |
+                        CSO_BIT_SAMPLE_MASK |
+                        CSO_BIT_MIN_SAMPLES |
+                        CSO_BIT_RENDER_CONDITION |
                         CSO_BITS_ALL_SHADERS));
    cso_save_constant_buffer_slot0(cso, PIPE_SHADER_FRAGMENT);
 
+   cso_set_sample_mask(cso, ~0);
+   cso_set_min_samples(cso, 1);
+   cso_set_render_condition(cso, NULL, FALSE, 0);
 
    /* Set up the sampler_view */
    {
-      unsigned first_element = buf_offset;
-      unsigned last_element = buf_offset + skip_pixels + upload_width - 1
-         + (upload_height - 1 + (depth - 1) * image_height) * stride;
       struct pipe_sampler_view templ;
       struct pipe_sampler_view *sampler_view;
-      struct pipe_sampler_state sampler = {0};
-      const struct pipe_sampler_state *samplers[1] = {&sampler};
-
-      /* This should be ensured by Mesa before calling our callbacks */
-      assert((last_element + 1) * bytes_per_pixel <= buffer->width0);
-
-      if (last_element - first_element > ctx->Const.MaxTextureBufferSize - 1)
-         goto fail;
 
       memset(&templ, 0, sizeof(templ));
       templ.target = PIPE_BUFFER;
       templ.format = src_format;
-      templ.u.buf.first_element = first_element;
-      templ.u.buf.last_element = last_element;
-      templ.swizzle_r = PIPE_SWIZZLE_RED;
-      templ.swizzle_g = PIPE_SWIZZLE_GREEN;
-      templ.swizzle_b = PIPE_SWIZZLE_BLUE;
-      templ.swizzle_a = PIPE_SWIZZLE_ALPHA;
-
-      sampler_view = pipe->create_sampler_view(pipe, buffer, &templ);
+      templ.u.buf.offset = addr->first_element * addr->bytes_per_pixel;
+      templ.u.buf.size = (addr->last_element - addr->first_element + 1) *
+                         addr->bytes_per_pixel;
+      templ.swizzle_r = PIPE_SWIZZLE_X;
+      templ.swizzle_g = PIPE_SWIZZLE_Y;
+      templ.swizzle_b = PIPE_SWIZZLE_Z;
+      templ.swizzle_a = PIPE_SWIZZLE_W;
+
+      sampler_view = pipe->create_sampler_view(pipe, addr->buffer, &templ);
       if (sampler_view == NULL)
          goto fail;
 
       cso_set_sampler_views(cso, PIPE_SHADER_FRAGMENT, 1, &sampler_view);
 
       pipe_sampler_view_reference(&sampler_view, NULL);
-
-      cso_set_samplers(cso, PIPE_SHADER_FRAGMENT, 1, samplers);
-   }
-
-   /* Upload vertices */
-   {
-      struct pipe_vertex_buffer vbo;
-      struct pipe_vertex_element velem;
-
-      float x0 = (float) xoffset / surface->width * 2.0f - 1.0f;
-      float y0 = (float) yoffset / surface->height * 2.0f - 1.0f;
-      float x1 = (float) (xoffset + upload_width) / surface->width * 2.0f - 1.0f;
-      float y1 = (float) (yoffset + upload_height) / surface->height * 2.0f - 1.0f;
-
-      float *verts = NULL;
-
-      vbo.user_buffer = NULL;
-      vbo.buffer = NULL;
-      vbo.stride = 2 * sizeof(float);
-
-      u_upload_alloc(st->uploader, 0, 8 * sizeof(float), 4,
-                     &vbo.buffer_offset, &vbo.buffer, (void **) &verts);
-      if (!verts)
-         goto fail;
-
-      verts[0] = x0;
-      verts[1] = y0;
-      verts[2] = x0;
-      verts[3] = y1;
-      verts[4] = x1;
-      verts[5] = y0;
-      verts[6] = x1;
-      verts[7] = y1;
-
-      u_upload_unmap(st->uploader);
-
-      velem.src_offset = 0;
-      velem.instance_divisor = 0;
-      velem.vertex_buffer_index = cso_get_aux_vertex_buffer_slot(cso);
-      velem.src_format = PIPE_FORMAT_R32G32_FLOAT;
-
-      cso_set_vertex_elements(cso, 1, &velem);
-
-      cso_set_vertex_buffers(cso, velem.vertex_buffer_index, 1, &vbo);
-
-      pipe_resource_reference(&vbo.buffer, NULL);
-   }
-
-   /* Upload constants */
-   /* Note: the user buffer must be valid until draw time */
-   struct {
-      int32_t xoffset;
-      int32_t yoffset;
-      int32_t stride;
-      int32_t image_size;
-   } constants;
-
-   {
-      struct pipe_constant_buffer cb;
-
-      constants.xoffset = -xoffset + skip_pixels;
-      constants.yoffset = -yoffset;
-      constants.stride = stride;
-      constants.image_size = stride * image_height;
-
-      if (st->constbuf_uploader) {
-         cb.buffer = NULL;
-         cb.user_buffer = NULL;
-         u_upload_data(st->constbuf_uploader, 0, sizeof(constants),
-                       ctx->Const.UniformBufferOffsetAlignment,
-                       &constants, &cb.buffer_offset, &cb.buffer);
-         if (!cb.buffer)
-            goto fail;
-
-         u_upload_unmap(st->constbuf_uploader);
-      } else {
-         cb.buffer = NULL;
-         cb.user_buffer = &constants;
-         cb.buffer_offset = 0;
-      }
-      cb.buffer_size = sizeof(constants);
-
-      cso_set_constant_buffer(cso, PIPE_SHADER_FRAGMENT, 0, &cb);
-
-      pipe_resource_reference(&cb.buffer, NULL);
    }
 
    /* Framebuffer_state */
@@ -1476,17 +1327,15 @@ try_pbo_upload_common(struct gl_context *ctx,
       fb.width = surface->width;
       fb.height = surface->height;
       fb.nr_cbufs = 1;
-      pipe_surface_reference(&fb.cbufs[0], surface);
+      fb.cbufs[0] = surface;
 
       cso_set_framebuffer(cso, &fb);
-
-      pipe_surface_reference(&fb.cbufs[0], NULL);
    }
 
    cso_set_viewport_dims(cso, surface->width, surface->height, FALSE);
 
    /* Blend state */
-   cso_set_blend(cso, &st->pbo_upload.blend);
+   cso_set_blend(cso, &st->pbo.upload_blend);
 
    /* Depth/stencil/alpha state */
    {
@@ -1495,31 +1344,10 @@ try_pbo_upload_common(struct gl_context *ctx,
       cso_set_depth_stencil_alpha(cso, &dsa);
    }
 
-   /* Rasterizer state */
-   cso_set_rasterizer(cso, &st->pbo_upload.raster);
-
-   /* Set up the shaders */
-   cso_set_vertex_shader_handle(cso, st->pbo_upload.vs);
-
-   cso_set_geometry_shader_handle(cso, depth != 1 ? st->pbo_upload.gs : NULL);
+   /* Set up the fragment shader */
+   cso_set_fragment_shader_handle(cso, fs);
 
-   cso_set_tessctrl_shader_handle(cso, NULL);
-
-   cso_set_tesseval_shader_handle(cso, NULL);
-
-   cso_set_fragment_shader_handle(cso, st->pbo_upload.fs);
-
-   /* Disable stream output */
-   cso_set_stream_outputs(cso, 0, NULL, 0);
-
-   if (depth == 1) {
-      cso_draw_arrays(cso, PIPE_PRIM_TRIANGLE_STRIP, 0, 4);
-   } else {
-      cso_draw_arrays_instanced(cso, PIPE_PRIM_TRIANGLE_STRIP,
-                                0, 4, 0, depth);
-   }
-
-   success = true;
+   success = st_pbo_draw(st, addr, surface->width, surface->height);
 
 fail:
    cso_restore_state(cso);
@@ -1528,6 +1356,7 @@ fail:
    return success;
 }
 
+
 static bool
 try_pbo_upload(struct gl_context *ctx, GLuint dims,
                struct gl_texture_image *texImage,
@@ -1545,15 +1374,13 @@ try_pbo_upload(struct gl_context *ctx, GLuint dims,
    struct pipe_context *pipe = st->pipe;
    struct pipe_screen *screen = pipe->screen;
    struct pipe_surface *surface = NULL;
+   struct st_pbo_addresses addr;
    enum pipe_format src_format;
    const struct util_format_description *desc;
    GLenum gl_target = texImage->TexObject->Target;
-   intptr_t buf_offset;
-   unsigned bytes_per_pixel;
-   unsigned stride, image_height;
    bool success;
 
-   if (!st->pbo_upload.enabled)
+   if (!st->pbo.upload_enabled)
       return false;
 
    /* From now on, we need the gallium representation of dimensions. */
@@ -1562,19 +1389,17 @@ try_pbo_upload(struct gl_context *ctx, GLuint dims,
       height = 1;
       zoffset = yoffset;
       yoffset = 0;
-      image_height = 1;
-   } else {
-      image_height = unpack->ImageHeight > 0 ? unpack->ImageHeight : height;
    }
 
-   if (depth != 1 && !st->pbo_upload.upload_layers)
+   if (depth != 1 && !st->pbo.layers)
       return false;
 
    /* Choose the source format. Initially, we do so without checking driver
     * support at all because of the remapping we later perform and because
     * at least the Radeon driver actually supports some formats for texture
     * buffers which it doesn't support for regular textures. */
-   src_format = st_choose_matching_format(st, 0, format, type, unpack->SwapBytes);
+   src_format = st_choose_matching_format(st, 0, format, type,
+                                          unpack->SwapBytes);
    if (!src_format) {
       return false;
    }
@@ -1588,7 +1413,7 @@ try_pbo_upload(struct gl_context *ctx, GLuint dims,
    if (desc->colorspace != UTIL_FORMAT_COLORSPACE_RGB)
       return false;
 
-   if (st->pbo_upload.rgba_only) {
+   if (st->pbo.rgba_only) {
       enum pipe_format orig_dst_format = dst_format;
 
       if (!reinterpret_formats(&src_format, &dst_format)) {
@@ -1597,55 +1422,33 @@ try_pbo_upload(struct gl_context *ctx, GLuint dims,
 
       if (dst_format != orig_dst_format &&
           !screen->is_format_supported(screen, dst_format, PIPE_TEXTURE_2D, 0,
-                                       PIPE_BIND_RENDER_TARGET)) {
+                                       0, PIPE_BIND_RENDER_TARGET)) {
          return false;
       }
    }
 
    if (!src_format ||
-       !screen->is_format_supported(screen, src_format, PIPE_BUFFER, 0,
+       !screen->is_format_supported(screen, src_format, PIPE_BUFFER, 0, 0,
                                     PIPE_BIND_SAMPLER_VIEW)) {
       return false;
    }
 
-   /* Check if the offset satisfies the alignment requirements */
-   buf_offset = (intptr_t) pixels;
-   bytes_per_pixel = desc->block.bits / 8;
+   /* Compute buffer addresses */
+   addr.xoffset = xoffset;
+   addr.yoffset = yoffset;
+   addr.width = width;
+   addr.height = height;
+   addr.depth = depth;
+   addr.bytes_per_pixel = desc->block.bits / 8;
 
-   if (buf_offset % bytes_per_pixel) {
+   if (!st_pbo_addresses_pixelstore(st, gl_target, dims == 3, unpack, pixels,
+                                    &addr))
       return false;
-   }
-
-   /* Convert to texels */
-   buf_offset = buf_offset / bytes_per_pixel;
-
-   /* Compute the stride, taking unpack->Alignment into account */
-   {
-       unsigned pixels_per_row = unpack->RowLength > 0 ?
-                           unpack->RowLength : width;
-       unsigned bytes_per_row = pixels_per_row * bytes_per_pixel;
-       unsigned remainder = bytes_per_row % unpack->Alignment;
-       unsigned offset_rows;
-
-       if (remainder > 0)
-          bytes_per_row += (unpack->Alignment - remainder);
-
-       if (bytes_per_row % bytes_per_pixel) {
-          return false;
-       }
-
-       stride = bytes_per_row / bytes_per_pixel;
-
-       offset_rows = unpack->SkipRows;
-       if (dims == 3)
-          offset_rows += image_height * unpack->SkipImages;
-
-       buf_offset += unpack->SkipPixels + stride * offset_rows;
-   }
 
    /* Set up the surface */
    {
-      unsigned level = stObj->pt != stImage->pt ? 0 : texImage->TexObject->MinLevel + texImage->Level;
+      unsigned level = stObj->pt != stImage->pt
+         ? 0 : texImage->TexObject->MinLevel + texImage->Level;
       unsigned max_layer = util_max_layer(texture, level);
 
       zoffset += texImage->Face + texImage->TexObject->MinLayer;
@@ -1662,18 +1465,14 @@ try_pbo_upload(struct gl_context *ctx, GLuint dims,
          return false;
    }
 
-   success = try_pbo_upload_common(ctx,  surface,
-                                   xoffset, yoffset, width, height,
-                                   st_buffer_object(unpack->BufferObj)->buffer,
-                                   src_format,
-                                   buf_offset,
-                                   bytes_per_pixel, stride, image_height);
+   success = try_pbo_upload_common(ctx, surface, &addr, src_format);
 
    pipe_surface_reference(&surface, NULL);
 
    return success;
 }
 
+
 static void
 st_TexSubImage(struct gl_context *ctx, GLuint dims,
                struct gl_texture_image *texImage,
@@ -1697,15 +1496,59 @@ st_TexSubImage(struct gl_context *ctx, GLuint dims,
    GLenum gl_target = texImage->TexObject->Target;
    unsigned bind;
    GLubyte *map;
+   unsigned dstz = texImage->Face + texImage->TexObject->MinLayer;
+   unsigned dst_level = 0;
+   bool throttled = false;
+
+   st_flush_bitmap_cache(st);
+   st_invalidate_readpix_cache(st);
+
+   if (stObj->pt == stImage->pt)
+      dst_level = texImage->TexObject->MinLevel + texImage->Level;
 
    assert(!_mesa_is_format_etc2(texImage->TexFormat) &&
+          !_mesa_is_format_astc_2d(texImage->TexFormat) &&
           texImage->TexFormat != MESA_FORMAT_ETC1_RGB8);
 
-   if (!st->prefer_blit_based_texture_transfer) {
+   if (!dst)
       goto fallback;
+
+   /* Try texture_subdata, which should be the fastest memcpy path. */
+   if (pixels &&
+       !unpack->BufferObj &&
+       _mesa_texstore_can_use_memcpy(ctx, texImage->_BaseFormat,
+                                     texImage->TexFormat, format, type,
+                                     unpack)) {
+      struct pipe_box box;
+      unsigned stride, layer_stride;
+      void *data;
+
+      stride = _mesa_image_row_stride(unpack, width, format, type);
+      layer_stride = _mesa_image_image_stride(unpack, width, height, format,
+                                              type);
+      data = _mesa_image_address(dims, unpack, pixels, width, height, format,
+                                 type, 0, 0, 0);
+
+      /* Convert to Gallium coordinates. */
+      if (gl_target == GL_TEXTURE_1D_ARRAY) {
+         zoffset = yoffset;
+         yoffset = 0;
+         depth = height;
+         height = 1;
+         layer_stride = stride;
+      }
+
+      util_throttle_memory_usage(pipe, &st->throttle,
+                                 (uint64_t) width * height * depth *
+                                 util_format_get_blocksize(dst->format));
+
+      u_box_3d(xoffset, yoffset, zoffset + dstz, width, height, depth, &box);
+      pipe->texture_subdata(pipe, dst, dst_level, 0,
+                            &box, data, stride, layer_stride);
+      return;
    }
 
-   if (!dst) {
+   if (!st->prefer_blit_based_texture_transfer) {
       goto fallback;
    }
 
@@ -1737,11 +1580,12 @@ st_TexSubImage(struct gl_context *ctx, GLuint dims,
 
    if (!dst_format ||
        !screen->is_format_supported(screen, dst_format, dst->target,
-                                    dst->nr_samples, bind)) {
+                                    dst->nr_samples, dst->nr_storage_samples,
+                                    bind)) {
       goto fallback;
    }
 
-   if (_mesa_is_bufferobj(unpack->BufferObj)) {
+   if (unpack->BufferObj) {
       if (try_pbo_upload(ctx, dims, texImage, format, type, dst_format,
                          xoffset, yoffset, zoffset,
                          width, height, depth, pixels, unpack))
@@ -1797,12 +1641,17 @@ st_TexSubImage(struct gl_context *ctx, GLuint dims,
 
    /* Check for NPOT texture support. */
    if (!screen->get_param(screen, PIPE_CAP_NPOT_TEXTURES) &&
-       (!util_is_power_of_two(src_templ.width0) ||
-        !util_is_power_of_two(src_templ.height0) ||
-        !util_is_power_of_two(src_templ.depth0))) {
+       (!util_is_power_of_two_or_zero(src_templ.width0) ||
+        !util_is_power_of_two_or_zero(src_templ.height0) ||
+        !util_is_power_of_two_or_zero(src_templ.depth0))) {
       goto fallback;
    }
 
+   util_throttle_memory_usage(pipe, &st->throttle,
+                              (uint64_t) width * height * depth *
+                              util_format_get_blocksize(src_templ.format));
+   throttled = true;
+
    /* Create the source texture. */
    src = screen->resource_create(screen, &src_templ);
    if (!src) {
@@ -1845,7 +1694,7 @@ st_TexSubImage(struct gl_context *ctx, GLuint dims,
             /* 1D array textures.
              * We need to convert gallium coords to GL coords.
              */
-            GLvoid *src = _mesa_image_address2d(unpack, pixels,
+            void *src = _mesa_image_address2d(unpack, pixels,
                                                 width, depth, format,
                                                 type, slice, 0);
             memcpy(map, src, bytesPerRow);
@@ -1854,7 +1703,7 @@ st_TexSubImage(struct gl_context *ctx, GLuint dims,
             ubyte *slice_map = map;
 
             for (row = 0; row < (unsigned) height; row++) {
-               GLvoid *src = _mesa_image_address(dims, unpack, pixels,
+               void *src = _mesa_image_address(dims, unpack, pixels,
                                                  width, height, format,
                                                  type, slice, row, 0);
                memcpy(slice_map, src, bytesPerRow);
@@ -1874,12 +1723,12 @@ st_TexSubImage(struct gl_context *ctx, GLuint dims,
    blit.src.level = 0;
    blit.src.format = src_format;
    blit.dst.resource = dst;
-   blit.dst.level = stObj->pt != stImage->pt ? 0 : texImage->TexObject->MinLevel + texImage->Level;
+   blit.dst.level = dst_level;
    blit.dst.format = dst_format;
    blit.src.box.x = blit.src.box.y = blit.src.box.z = 0;
    blit.dst.box.x = xoffset;
    blit.dst.box.y = yoffset;
-   blit.dst.box.z = zoffset + texImage->Face + texImage->TexObject->MinLayer;
+   blit.dst.box.z = zoffset + dstz;
    blit.src.box.width = blit.dst.box.width = width;
    blit.src.box.height = blit.dst.box.height = height;
    blit.src.box.depth = blit.dst.box.depth = depth;
@@ -1893,11 +1742,17 @@ st_TexSubImage(struct gl_context *ctx, GLuint dims,
    return;
 
 fallback:
+   if (!throttled) {
+      util_throttle_memory_usage(pipe, &st->throttle,
+                                 (uint64_t) width * height * depth *
+                                 _mesa_get_format_bytes(texImage->TexFormat));
+   }
    _mesa_store_texsubimage(ctx, dims, texImage, xoffset, yoffset, zoffset,
                            width, height, depth, format, type, pixels,
                            unpack);
 }
 
+
 static void
 st_TexImage(struct gl_context * ctx, GLuint dims,
             struct gl_texture_image *texImage,
@@ -1928,7 +1783,7 @@ st_CompressedTexSubImage(struct gl_context *ctx, GLuint dims,
                          struct gl_texture_image *texImage,
                          GLint x, GLint y, GLint z,
                          GLsizei w, GLsizei h, GLsizei d,
-                         GLenum format, GLsizei imageSize, const GLvoid *data)
+                         GLenum format, GLsizei imageSize, const void *data)
 {
    struct st_context *st = st_context(ctx);
    struct st_texture_image *stImage = st_texture_image(texImage);
@@ -1939,8 +1794,8 @@ st_CompressedTexSubImage(struct gl_context *ctx, GLuint dims,
    struct pipe_resource *dst = stImage->pt;
    struct pipe_surface *surface = NULL;
    struct compressed_pixelstore store;
+   struct st_pbo_addresses addr;
    enum pipe_format copy_format;
-   unsigned bytes_per_block;
    unsigned bw, bh;
    intptr_t buf_offset;
    bool success = false;
@@ -1950,30 +1805,27 @@ st_CompressedTexSubImage(struct gl_context *ctx, GLuint dims,
       goto fallback;
    }
 
-   if (!_mesa_is_bufferobj(ctx->Unpack.BufferObj))
+   if (!ctx->Unpack.BufferObj)
       goto fallback;
 
-   if ((_mesa_is_format_etc2(texImage->TexFormat) && !st->has_etc2) ||
-       (texImage->TexFormat == MESA_FORMAT_ETC1_RGB8 && !st->has_etc1)) {
-      /* ETC isn't supported and is represented by uncompressed formats. */
+   if (st_compressed_format_fallback(st, texImage->TexFormat))
       goto fallback;
-   }
 
    if (!dst) {
       goto fallback;
    }
 
-   if (!st->pbo_upload.enabled ||
+   if (!st->pbo.upload_enabled ||
        !screen->get_param(screen, PIPE_CAP_SURFACE_REINTERPRET_BLOCKS)) {
       goto fallback;
    }
 
    /* Choose the pipe format for the upload. */
-   bytes_per_block = util_format_get_blocksize(dst->format);
+   addr.bytes_per_pixel = util_format_get_blocksize(dst->format);
    bw = util_format_get_blockwidth(dst->format);
    bh = util_format_get_blockheight(dst->format);
 
-   switch (bytes_per_block) {
+   switch (addr.bytes_per_pixel) {
    case 8:
       copy_format = PIPE_FORMAT_R16G16B16A16_UINT;
       break;
@@ -1984,34 +1836,49 @@ st_CompressedTexSubImage(struct gl_context *ctx, GLuint dims,
       goto fallback;
    }
 
-   if (!screen->is_format_supported(screen, copy_format, PIPE_BUFFER, 0,
+   if (!screen->is_format_supported(screen, copy_format, PIPE_BUFFER, 0, 0,
                                     PIPE_BIND_SAMPLER_VIEW)) {
       goto fallback;
    }
 
    if (!screen->is_format_supported(screen, copy_format, dst->target,
-                                    dst->nr_samples, PIPE_BIND_RENDER_TARGET)) {
+                                    dst->nr_samples, dst->nr_storage_samples,
+                                    PIPE_BIND_RENDER_TARGET)) {
       goto fallback;
    }
 
    /* Interpret the pixelstore settings. */
    _mesa_compute_compressed_pixelstore(dims, texImage->TexFormat, w, h, d,
                                        &ctx->Unpack, &store);
-   assert(store.CopyBytesPerRow % bytes_per_block == 0);
-   assert(store.SkipBytes % bytes_per_block == 0);
+   assert(store.CopyBytesPerRow % addr.bytes_per_pixel == 0);
+   assert(store.SkipBytes % addr.bytes_per_pixel == 0);
 
    /* Compute the offset into the buffer */
    buf_offset = (intptr_t)data + store.SkipBytes;
 
-   if (buf_offset % bytes_per_block) {
+   if (buf_offset % addr.bytes_per_pixel) {
       goto fallback;
    }
 
-   buf_offset = buf_offset / bytes_per_block;
+   buf_offset = buf_offset / addr.bytes_per_pixel;
+
+   addr.xoffset = x / bw;
+   addr.yoffset = y / bh;
+   addr.width = store.CopyBytesPerRow / addr.bytes_per_pixel;
+   addr.height = store.CopyRowsPerSlice;
+   addr.depth = d;
+   addr.pixels_per_row = store.TotalBytesPerRow / addr.bytes_per_pixel;
+   addr.image_height = store.TotalRowsPerSlice;
+
+   if (!st_pbo_addresses_setup(st,
+                               st_buffer_object(ctx->Unpack.BufferObj)->buffer,
+                               buf_offset, &addr))
+      goto fallback;
 
    /* Set up the surface. */
    {
-      unsigned level = stObj->pt != stImage->pt ? 0 : texImage->TexObject->MinLevel + texImage->Level;
+      unsigned level = stObj->pt != stImage->pt
+         ? 0 : texImage->TexObject->MinLevel + texImage->Level;
       unsigned max_layer = util_max_layer(texture, level);
 
       z += texImage->Face + texImage->TexObject->MinLayer;
@@ -2028,16 +1895,7 @@ st_CompressedTexSubImage(struct gl_context *ctx, GLuint dims,
          goto fallback;
    }
 
-   success = try_pbo_upload_common(ctx, surface,
-                                   x / bw, y / bh,
-                                   store.CopyBytesPerRow / bytes_per_block,
-                                   store.CopyRowsPerSlice,
-                                   st_buffer_object(ctx->Unpack.BufferObj)->buffer,
-                                   copy_format,
-                                   buf_offset,
-                                   bytes_per_block,
-                                   store.TotalBytesPerRow / bytes_per_block,
-                                   store.TotalRowsPerSlice);
+   success = try_pbo_upload_common(ctx, surface, &addr, copy_format);
 
    pipe_surface_reference(&surface, NULL);
 
@@ -2050,10 +1908,11 @@ fallback:
                                       format, imageSize, data);
 }
 
+
 static void
 st_CompressedTexImage(struct gl_context *ctx, GLuint dims,
                       struct gl_texture_image *texImage,
-                      GLsizei imageSize, const GLvoid *data)
+                      GLsizei imageSize, const void *data)
 {
    prep_teximage(ctx, texImage, GL_NONE, GL_NONE);
 
@@ -2086,8 +1945,6 @@ st_CompressedTexImage(struct gl_context *ctx, GLuint dims,
 }
 
 
-
-
 /**
  * Called via ctx->Driver.GetTexSubImage()
  *
@@ -2106,7 +1963,7 @@ static void
 st_GetTexSubImage(struct gl_context * ctx,
                   GLint xoffset, GLint yoffset, GLint zoffset,
                   GLsizei width, GLsizei height, GLint depth,
-                  GLenum format, GLenum type, GLvoid * pixels,
+                  GLenum format, GLenum type, void * pixels,
                   struct gl_texture_image *texImage)
 {
    struct st_context *st = st_context(ctx);
@@ -2121,22 +1978,27 @@ st_GetTexSubImage(struct gl_context * ctx,
    mesa_format mesa_format;
    GLenum gl_target = texImage->TexObject->Target;
    enum pipe_texture_target pipe_target;
+   unsigned dims;
    struct pipe_blit_info blit;
-   unsigned bind = PIPE_BIND_TRANSFER_READ;
+   unsigned bind;
    struct pipe_transfer *tex_xfer;
    ubyte *map = NULL;
    boolean done = FALSE;
 
    assert(!_mesa_is_format_etc2(texImage->TexFormat) &&
+          !_mesa_is_format_astc_2d(texImage->TexFormat) &&
           texImage->TexFormat != MESA_FORMAT_ETC1_RGB8);
 
+   st_flush_bitmap_cache(st);
+
    if (!st->prefer_blit_based_texture_transfer &&
        !_mesa_is_format_compressed(texImage->TexFormat)) {
       /* Try to avoid the fallback if we're doing texture decompression here */
       goto fallback;
    }
 
-   if (!stImage->pt || !src) {
+   /* Handle non-finalized textures. */
+   if (!stImage->pt || stImage->pt != stObj->pt || !src) {
       goto fallback;
    }
 
@@ -2177,15 +2039,15 @@ st_GetTexSubImage(struct gl_context * ctx,
 
    if (!src_format ||
        !screen->is_format_supported(screen, src_format, src->target,
-                                    src->nr_samples,
+                                    src->nr_samples, src->nr_storage_samples,
                                     PIPE_BIND_SAMPLER_VIEW)) {
       goto fallback;
    }
 
    if (format == GL_DEPTH_COMPONENT || format == GL_DEPTH_STENCIL)
-      bind |= PIPE_BIND_DEPTH_STENCIL;
+      bind = PIPE_BIND_DEPTH_STENCIL;
    else
-      bind |= PIPE_BIND_RENDER_TARGET;
+      bind = PIPE_BIND_RENDER_TARGET;
 
    /* GetTexImage only returns a single face for cubemaps. */
    if (gl_target == GL_TEXTURE_CUBE_MAP) {
@@ -2196,7 +2058,7 @@ st_GetTexSubImage(struct gl_context * ctx,
    /* Choose the destination format by finding the best match
     * for the format+type combo. */
    dst_format = st_choose_matching_format(st, bind, format, type,
-                                         ctx->Pack.SwapBytes);
+                                          ctx->Pack.SwapBytes);
 
    if (dst_format == PIPE_FORMAT_NONE) {
       GLenum dst_glformat;
@@ -2217,7 +2079,26 @@ st_GetTexSubImage(struct gl_context * ctx,
       case PIPE_FORMAT_RGTC1_UNORM:
       case PIPE_FORMAT_RGTC2_UNORM:
       case PIPE_FORMAT_ETC1_RGB8:
+      case PIPE_FORMAT_ETC2_RGB8:
+      case PIPE_FORMAT_ETC2_RGB8A1:
+      case PIPE_FORMAT_ETC2_RGBA8:
+      case PIPE_FORMAT_ASTC_4x4:
+      case PIPE_FORMAT_ASTC_5x4:
+      case PIPE_FORMAT_ASTC_5x5:
+      case PIPE_FORMAT_ASTC_6x5:
+      case PIPE_FORMAT_ASTC_6x6:
+      case PIPE_FORMAT_ASTC_8x5:
+      case PIPE_FORMAT_ASTC_8x6:
+      case PIPE_FORMAT_ASTC_8x8:
+      case PIPE_FORMAT_ASTC_10x5:
+      case PIPE_FORMAT_ASTC_10x6:
+      case PIPE_FORMAT_ASTC_10x8:
+      case PIPE_FORMAT_ASTC_10x10:
+      case PIPE_FORMAT_ASTC_12x10:
+      case PIPE_FORMAT_ASTC_12x12:
       case PIPE_FORMAT_BPTC_RGBA_UNORM:
+      case PIPE_FORMAT_FXT1_RGB:
+      case PIPE_FORMAT_FXT1_RGBA:
          dst_glformat = GL_RGBA8;
          break;
       case PIPE_FORMAT_RGTC1_SNORM:
@@ -2232,13 +2113,38 @@ st_GetTexSubImage(struct gl_context * ctx,
             goto fallback;
          dst_glformat = GL_RGBA32F;
          break;
+      case PIPE_FORMAT_ETC2_R11_UNORM:
+         if (!screen->is_format_supported(screen, PIPE_FORMAT_R16_UNORM,
+                                          pipe_target, 0, 0, bind))
+            goto fallback;
+         dst_glformat = GL_R16;
+         break;
+      case PIPE_FORMAT_ETC2_R11_SNORM:
+         if (!screen->is_format_supported(screen, PIPE_FORMAT_R16_SNORM,
+                                          pipe_target, 0, 0, bind))
+            goto fallback;
+         dst_glformat = GL_R16_SNORM;
+         break;
+      case PIPE_FORMAT_ETC2_RG11_UNORM:
+         if (!screen->is_format_supported(screen, PIPE_FORMAT_R16G16_UNORM,
+                                          pipe_target, 0, 0, bind))
+            goto fallback;
+         dst_glformat = GL_RG16;
+         break;
+      case PIPE_FORMAT_ETC2_RG11_SNORM:
+         if (!screen->is_format_supported(screen, PIPE_FORMAT_R16G16_SNORM,
+                                          pipe_target, 0, 0, bind))
+            goto fallback;
+         dst_glformat = GL_RG16_SNORM;
+         break;
       default:
          assert(0);
          goto fallback;
       }
 
       dst_format = st_choose_format(st, dst_glformat, format, type,
-                                    pipe_target, 0, bind, FALSE);
+                                    pipe_target, 0, 0, bind,
+                                    false, false);
 
       if (dst_format == PIPE_FORMAT_NONE) {
          /* unable to get an rgba format!?! */
@@ -2306,6 +2212,7 @@ st_GetTexSubImage(struct gl_context * ctx,
    }
 
    mesa_format = st_pipe_format_to_mesa_format(dst_format);
+   dims = _mesa_get_texture_dimensions(gl_target);
 
    /* copy/pack data into user buffer */
    if (_mesa_format_matches_format_and_type(mesa_format, format, type,
@@ -2315,39 +2222,31 @@ st_GetTexSubImage(struct gl_context * ctx,
       GLuint row, slice;
 
       for (slice = 0; slice < depth; slice++) {
-         if (gl_target == GL_TEXTURE_1D_ARRAY) {
-            /* 1D array textures.
-             * We need to convert gallium coords to GL coords.
-             */
-            GLvoid *dest = _mesa_image_address3d(&ctx->Pack, pixels,
-                                                 width, depth, format,
-                                                 type, 0, slice, 0);
-            memcpy(dest, map, bytesPerRow);
-         }
-         else {
-            ubyte *slice_map = map;
+         ubyte *slice_map = map;
 
-            for (row = 0; row < height; row++) {
-               GLvoid *dest = _mesa_image_address3d(&ctx->Pack, pixels,
-                                                    width, height, format,
-                                                    type, slice, row, 0);
-               memcpy(dest, slice_map, bytesPerRow);
-               slice_map += tex_xfer->stride;
-            }
+         for (row = 0; row < height; row++) {
+            void *dest = _mesa_image_address(dims, &ctx->Pack, pixels,
+                                             width, height, format, type,
+                                             slice, row, 0);
+
+            memcpy(dest, slice_map, bytesPerRow);
+
+            slice_map += tex_xfer->stride;
          }
+
          map += tex_xfer->layer_stride;
       }
    }
    else {
       /* format translation via floats */
-      GLuint row, slice;
+      GLuint slice;
       GLfloat *rgba;
       uint32_t dstMesaFormat;
       int dstStride, srcStride;
 
       assert(util_format_is_compressed(src->format));
 
-      rgba = malloc(width * 4 * sizeof(GLfloat));
+      rgba = malloc(width * height * 4 * sizeof(GLfloat));
       if (!rgba) {
          goto end;
       }
@@ -2359,37 +2258,24 @@ st_GetTexSubImage(struct gl_context * ctx,
       dstStride = _mesa_image_row_stride(&ctx->Pack, width, format, type);
       srcStride = 4 * width * sizeof(GLfloat);
       for (slice = 0; slice < depth; slice++) {
-         if (gl_target == GL_TEXTURE_1D_ARRAY) {
-            /* 1D array textures.
-             * We need to convert gallium coords to GL coords.
-             */
-            GLvoid *dest = _mesa_image_address3d(&ctx->Pack, pixels,
-                                                 width, depth, format,
-                                                 type, 0, slice, 0);
-
-            /* get float[4] rgba row from surface */
-            pipe_get_tile_rgba_format(tex_xfer, map, 0, 0, width, 1,
-                                      dst_format, rgba);
-
-            _mesa_format_convert(dest, dstMesaFormat, dstStride,
-                                 rgba, RGBA32_FLOAT, srcStride,
-                                 width, 1, NULL);
-         }
-         else {
-            for (row = 0; row < height; row++) {
-               GLvoid *dest = _mesa_image_address3d(&ctx->Pack, pixels,
-                                                    width, height, format,
-                                                    type, slice, row, 0);
-
-               /* get float[4] rgba row from surface */
-               pipe_get_tile_rgba_format(tex_xfer, map, 0, row, width, 1,
-                                         dst_format, rgba);
-
-               _mesa_format_convert(dest, dstMesaFormat, dstStride,
-                                    rgba, RGBA32_FLOAT, srcStride,
-                                    width, 1, NULL);
-            }
+         void *dest = _mesa_image_address(dims, &ctx->Pack, pixels,
+                                          width, height, format, type,
+                                          slice, 0, 0);
+
+         /* get float[4] rgba row from surface */
+         pipe_get_tile_rgba(tex_xfer, map, 0, 0, width, height, dst_format,
+                            rgba);
+
+         _mesa_format_convert(dest, dstMesaFormat, dstStride,
+                              rgba, RGBA32_FLOAT, srcStride,
+                              width, height, NULL);
+
+         /* Handle byte swapping if required */
+         if (ctx->Pack.SwapBytes) {
+            _mesa_swap_bytes_2d_image(format, type, &ctx->Pack,
+                                      width, height, dest, dest);
          }
+
          map += tex_xfer->layer_stride;
       }
 
@@ -2454,6 +2340,10 @@ fallback_copy_texsubimage(struct gl_context *ctx,
                            PIPE_TRANSFER_READ,
                            srcX, srcY,
                            width, height, &src_trans);
+   if (!map) {
+      _mesa_error(ctx, GL_OUT_OF_MEMORY, "glCopyTexSubImage()");
+      return;
+   }
 
    if ((baseFormat == GL_DEPTH_COMPONENT ||
         baseFormat == GL_DEPTH_STENCIL) &&
@@ -2466,6 +2356,10 @@ fallback_copy_texsubimage(struct gl_context *ctx,
                                   destX, destY, slice,
                                   dst_width, dst_height, dst_depth,
                                   &transfer);
+   if (!texDest) {
+      _mesa_error(ctx, GL_OUT_OF_MEMORY, "glCopyTexSubImage()");
+      goto err;
+   }
 
    if (baseFormat == GL_DEPTH_COMPONENT ||
        baseFormat == GL_DEPTH_STENCIL) {
@@ -2487,20 +2381,19 @@ fallback_copy_texsubimage(struct gl_context *ctx,
       data = malloc(width * sizeof(uint));
 
       if (data) {
+         unsigned dst_stride = (stImage->pt->target == PIPE_TEXTURE_1D_ARRAY ?
+                                transfer->layer_stride : transfer->stride);
          /* To avoid a large temp memory allocation, do copy row by row */
          for (row = 0; row < height; row++, srcY += yStep) {
-            pipe_get_tile_z(src_trans, map, 0, srcY, width, 1, data);
+            util_format_unpack_z_32unorm(strb->texture->format,
+                                         data, (uint8_t *)map + src_trans->stride * srcY,
+                                         width);
             if (scaleOrBias) {
                _mesa_scale_and_bias_depth_uint(ctx, width, data);
             }
 
-            if (stImage->pt->target == PIPE_TEXTURE_1D_ARRAY) {
-               pipe_put_tile_z(transfer, texDest + row*transfer->layer_stride,
-                               0, 0, width, 1, data);
-            }
-            else {
-               pipe_put_tile_z(transfer, texDest, 0, row, width, 1, data);
-            }
+            util_format_pack_z_32unorm(stImage->pt->format,
+                                       texDest + row * dst_stride, data, width);
          }
       }
       else {
@@ -2514,7 +2407,7 @@ fallback_copy_texsubimage(struct gl_context *ctx,
       GLfloat *tempSrc =
          malloc(width * height * 4 * sizeof(GLfloat));
 
-      if (tempSrc && texDest) {
+      if (tempSrc) {
          const GLint dims = 2;
          GLint dstRowStride;
          struct gl_texture_image *texImage = &stImage->base;
@@ -2535,9 +2428,9 @@ fallback_copy_texsubimage(struct gl_context *ctx,
          /* XXX this usually involves a lot of int/float conversion.
           * try to avoid that someday.
           */
-         pipe_get_tile_rgba_format(src_trans, map, 0, 0, width, height,
-                                   util_format_linear(strb->texture->format),
-                                   tempSrc);
+         pipe_get_tile_rgba(src_trans, map, 0, 0, width, height,
+                            util_format_linear(strb->texture->format),
+                            tempSrc);
 
          /* Store into texture memory.
           * Note that this does some special things such as pixel transfer
@@ -2546,8 +2439,8 @@ fallback_copy_texsubimage(struct gl_context *ctx,
           * need to fill-in/override the alpha channel with 1.0.
           */
          _mesa_texstore(ctx, dims,
-                        texImage->_BaseFormat, 
-                        texImage->TexFormat, 
+                        texImage->_BaseFormat,
+                        texImage->TexFormat,
                         dstRowStride,
                         &texDest,
                         width, height, 1,
@@ -2562,10 +2455,38 @@ fallback_copy_texsubimage(struct gl_context *ctx,
    }
 
    st_texture_image_unmap(st, stImage, slice);
+err:
    pipe->transfer_unmap(pipe, src_trans);
 }
 
 
+static bool
+st_can_copyteximage_using_blit(const struct gl_texture_image *texImage,
+                               const struct gl_renderbuffer *rb)
+{
+   GLenum tex_baseformat = _mesa_get_format_base_format(texImage->TexFormat);
+
+   /* We don't blit to a teximage where the GL base format doesn't match the
+    * texture's chosen format, except in the case of a GL_RGB texture
+    * represented with GL_RGBA (where the alpha channel is just being
+    * dropped).
+    */
+   if (texImage->_BaseFormat != tex_baseformat &&
+       ((texImage->_BaseFormat != GL_RGB || tex_baseformat != GL_RGBA))) {
+      return false;
+   }
+
+   /* We can't blit from a RB where the GL base format doesn't match the RB's
+    * chosen format (for example, GL RGB or ALPHA with rb->Format of an RGBA
+    * type, because the other channels will be undefined).
+    */
+   if (rb->_BaseFormat != _mesa_get_format_base_format(rb->Format))
+      return false;
+
+   return true;
+}
+
+
 /**
  * Do a CopyTex[Sub]Image1/2/3D() using a hardware (blit) path if possible.
  * Note that the region to copy has already been clipped so we know we
@@ -2592,7 +2513,11 @@ st_CopyTexSubImage(struct gl_context *ctx, GLuint dims,
    unsigned bind;
    GLint srcY0, srcY1;
 
+   st_flush_bitmap_cache(st);
+   st_invalidate_readpix_cache(st);
+
    assert(!_mesa_is_format_etc2(texImage->TexFormat) &&
+          !_mesa_is_format_astc_2d(texImage->TexFormat) &&
           texImage->TexFormat != MESA_FORMAT_ETC1_RGB8);
 
    if (!strb || !strb->surface || !stImage->pt) {
@@ -2605,12 +2530,7 @@ st_CopyTexSubImage(struct gl_context *ctx, GLuint dims,
       goto fallback;
    }
 
-   /* The base internal format must match the mesa format, so make sure
-    * e.g. an RGB internal format is really allocated as RGB and not as RGBA.
-    */
-   if (texImage->_BaseFormat !=
-       _mesa_get_format_base_format(texImage->TexFormat) ||
-       rb->_BaseFormat != _mesa_get_format_base_format(rb->Format)) {
+   if (!st_can_copyteximage_using_blit(texImage, rb)) {
       goto fallback;
    }
 
@@ -2630,7 +2550,8 @@ st_CopyTexSubImage(struct gl_context *ctx, GLuint dims,
 
    if (!dst_format ||
        !screen->is_format_supported(screen, dst_format, stImage->pt->target,
-                                    stImage->pt->nr_samples, bind)) {
+                                    stImage->pt->nr_samples,
+                                    stImage->pt->nr_storage_samples, bind)) {
       goto fallback;
    }
 
@@ -2659,7 +2580,8 @@ st_CopyTexSubImage(struct gl_context *ctx, GLuint dims,
    blit.src.box.depth = 1;
    blit.dst.resource = stImage->pt;
    blit.dst.format = dst_format;
-   blit.dst.level = stObj->pt != stImage->pt ? 0 : texImage->Level + texImage->TexObject->MinLevel;
+   blit.dst.level = stObj->pt != stImage->pt
+      ? 0 : texImage->Level + texImage->TexObject->MinLevel;
    blit.dst.box.x = destX;
    blit.dst.box.y = destY;
    blit.dst.box.z = stImage->base.Face + slice + texImage->TexObject->MinLayer;
@@ -2686,13 +2608,13 @@ fallback:
  */
 static void
 copy_image_data_to_texture(struct st_context *st,
-                          struct st_texture_object *stObj,
+                           struct st_texture_object *stObj,
                            GLuint dstLevel,
-                          struct st_texture_image *stImage)
+                           struct st_texture_image *stImage)
 {
    /* debug checks */
    {
-      const struct gl_texture_image *dstImage =
+      ASSERTED const struct gl_texture_image *dstImage =
          stObj->base.Image[stImage->base.Face][dstLevel];
       assert(dstImage);
       assert(dstImage->Width == stImage->base.Width);
@@ -2735,8 +2657,9 @@ copy_image_data_to_texture(struct st_context *st,
  */
 GLboolean
 st_finalize_texture(struct gl_context *ctx,
-                   struct pipe_context *pipe,
-                   struct gl_texture_object *tObj)
+                    struct pipe_context *pipe,
+                    struct gl_texture_object *tObj,
+                    GLuint cubeMapFace)
 {
    struct st_context *st = st_context(ctx);
    struct st_texture_object *stObj = st_texture_object(tObj);
@@ -2744,46 +2667,34 @@ st_finalize_texture(struct gl_context *ctx,
    GLuint face;
    const struct st_texture_image *firstImage;
    enum pipe_format firstImageFormat;
-   GLuint ptWidth, ptHeight, ptDepth, ptLayers, ptNumSamples;
+   unsigned ptWidth;
+   uint16_t ptHeight, ptDepth, ptLayers, ptNumSamples;
 
    if (tObj->Immutable)
       return GL_TRUE;
 
-   if (_mesa_is_texture_complete(tObj, &tObj->Sampler)) {
-      /* The texture is complete and we know exactly how many mipmap levels
-       * are present/needed.  This is conditional because we may be called
-       * from the st_generate_mipmap() function when the texture object is
-       * incomplete.  In that case, we'll have set stObj->lastLevel before
-       * we get here.
-       */
-      if (stObj->base.Sampler.MinFilter == GL_LINEAR ||
-          stObj->base.Sampler.MinFilter == GL_NEAREST)
-         stObj->lastLevel = stObj->base.BaseLevel;
-      else
-         stObj->lastLevel = stObj->base._MaxLevel;
-   }
-
-   if (tObj->Target == GL_TEXTURE_BUFFER) {
-      struct st_buffer_object *st_obj = st_buffer_object(tObj->BufferObject);
+   if (tObj->_MipmapComplete)
+      stObj->lastLevel = stObj->base._MaxLevel;
+   else if (tObj->_BaseComplete)
+      stObj->lastLevel = stObj->base.BaseLevel;
 
-      if (!st_obj) {
-         pipe_resource_reference(&stObj->pt, NULL);
-         st_texture_release_all_sampler_views(st, stObj);
-         return GL_TRUE;
-      }
-
-      if (st_obj->buffer != stObj->pt) {
-         pipe_resource_reference(&stObj->pt, st_obj->buffer);
-         st_texture_release_all_sampler_views(st, stObj);
-         stObj->width0 = stObj->pt->width0 / _mesa_get_format_bytes(tObj->_BufferObjectFormat);
-         stObj->height0 = 1;
-         stObj->depth0 = 1;
-      }
+   /* Skip the loop over images in the common case of no images having
+    * changed.  But if the GL_BASE_LEVEL or GL_MAX_LEVEL change to something we
+    * haven't looked at, then we do need to look at those new images.
+    */
+   if (!stObj->needs_validation &&
+       stObj->base.BaseLevel >= stObj->validated_first_level &&
+       stObj->lastLevel <= stObj->validated_last_level) {
       return GL_TRUE;
+   }
 
+   /* If this texture comes from a window system, there is nothing else to do. */
+   if (stObj->surface_based) {
+      return GL_TRUE;
    }
 
-   firstImage = st_texture_image_const(_mesa_base_tex_image(&stObj->base));
+   firstImage = st_texture_image_const(stObj->base.Image[cubeMapFace]
+                                       [stObj->base.BaseLevel]);
    assert(firstImage);
 
    /* If both firstImage and stObj point to a texture which can contain
@@ -2798,37 +2709,64 @@ st_finalize_texture(struct gl_context *ctx,
       st_texture_release_all_sampler_views(st, stObj);
    }
 
-   /* If this texture comes from a window system, there is nothing else to do. */
-   if (stObj->surface_based) {
-      return GL_TRUE;
-   }
-
    /* Find gallium format for the Mesa texture */
    firstImageFormat =
       st_mesa_format_to_pipe_format(st, firstImage->base.TexFormat);
 
    /* Find size of level=0 Gallium mipmap image, plus number of texture layers */
    {
-      GLuint width, height, depth;
-      if (!guess_base_level_size(stObj->base.Target,
-                                 firstImage->base.Width2,
-                                 firstImage->base.Height2,
-                                 firstImage->base.Depth2,
-                                 firstImage->base.Level,
-                                 &width, &height, &depth)) {
-         width = stObj->width0;
-         height = stObj->height0;
-         depth = stObj->depth0;
+      unsigned width;
+      uint16_t height, depth;
+
+      st_gl_texture_dims_to_pipe_dims(stObj->base.Target,
+                                      firstImage->base.Width2,
+                                      firstImage->base.Height2,
+                                      firstImage->base.Depth2,
+                                      &width, &height, &depth, &ptLayers);
+
+      /* If we previously allocated a pipe texture and its sizes are
+       * compatible, use them.
+       */
+      if (stObj->pt &&
+          u_minify(stObj->pt->width0, firstImage->base.Level) == width &&
+          u_minify(stObj->pt->height0, firstImage->base.Level) == height &&
+          u_minify(stObj->pt->depth0, firstImage->base.Level) == depth) {
+         ptWidth = stObj->pt->width0;
+         ptHeight = stObj->pt->height0;
+         ptDepth = stObj->pt->depth0;
       } else {
-         /* The width/height/depth may have been previously reset in
-          * guess_and_alloc_texture. */
-         stObj->width0 = width;
-         stObj->height0 = height;
-         stObj->depth0 = depth;
+         /* Otherwise, compute a new level=0 size that is compatible with the
+          * base level image.
+          */
+         ptWidth = width > 1 ? width << firstImage->base.Level : 1;
+         ptHeight = height > 1 ? height << firstImage->base.Level : 1;
+         ptDepth = depth > 1 ? depth << firstImage->base.Level : 1;
+
+         /* If the base level image is 1x1x1, we still need to ensure that the
+          * resulting pipe texture ends up with the required number of levels
+          * in total.
+          */
+         if (ptWidth == 1 && ptHeight == 1 && ptDepth == 1) {
+            ptWidth <<= firstImage->base.Level;
+
+            if (stObj->base.Target == GL_TEXTURE_CUBE_MAP ||
+                stObj->base.Target == GL_TEXTURE_CUBE_MAP_ARRAY)
+               ptHeight = ptWidth;
+         }
+
+         /* At this point, the texture may be incomplete (mismatched cube
+          * face sizes, for example).  If that's the case, give up, but
+          * don't return GL_FALSE as that would raise an incorrect
+          * GL_OUT_OF_MEMORY error.  See Piglit fbo-incomplete-texture-03 test.
+          */
+         if (!stObj->base._BaseComplete) {
+            _mesa_test_texobj_completeness(ctx, &stObj->base);
+            if (!stObj->base._BaseComplete) {
+               return TRUE;
+            }
+         }
       }
-      /* convert GL dims to Gallium dims */
-      st_gl_texture_dims_to_pipe_dims(stObj->base.Target, width, height, depth,
-                                      &ptWidth, &ptHeight, &ptDepth, &ptLayers);
+
       ptNumSamples = firstImage->base.NumSamples;
    }
 
@@ -2850,7 +2788,7 @@ st_finalize_texture(struct gl_context *ctx,
           */
          pipe_resource_reference(&stObj->pt, NULL);
          st_texture_release_all_sampler_views(st, stObj);
-         st->dirty.st |= ST_NEW_FRAMEBUFFER;
+         st->dirty |= ST_NEW_FRAMEBUFFER;
       }
    }
 
@@ -2886,16 +2824,23 @@ st_finalize_texture(struct gl_context *ctx,
          /* Need to import images in main memory or held in other textures.
           */
          if (stImage && stObj->pt != stImage->pt) {
-            GLuint height = stObj->height0;
-            GLuint depth = stObj->depth0;
+            GLuint height;
+            GLuint depth;
 
             if (stObj->base.Target != GL_TEXTURE_1D_ARRAY)
-               height = u_minify(height, level);
+               height = u_minify(ptHeight, level);
+            else
+               height = ptLayers;
+
             if (stObj->base.Target == GL_TEXTURE_3D)
-               depth = u_minify(depth, level);
+               depth = u_minify(ptDepth, level);
+            else if (stObj->base.Target == GL_TEXTURE_CUBE_MAP)
+               depth = 1;
+            else
+               depth = ptLayers;
 
             if (level == 0 ||
-                (stImage->base.Width == u_minify(stObj->width0, level) &&
+                (stImage->base.Width == u_minify(ptWidth, level) &&
                  stImage->base.Height == height &&
                  stImage->base.Depth == depth)) {
                /* src image fits expected dest mipmap level size */
@@ -2905,49 +2850,123 @@ st_finalize_texture(struct gl_context *ctx,
       }
    }
 
+   stObj->validated_first_level = stObj->base.BaseLevel;
+   stObj->validated_last_level = stObj->lastLevel;
+   stObj->needs_validation = false;
+
    return GL_TRUE;
 }
 
 
 /**
- * Called via ctx->Driver.AllocTextureStorage() to allocate texture memory
- * for a whole mipmap stack.
+ * Allocate a new pipe_resource object
+ * width0, height0, depth0 are the dimensions of the level 0 image
+ * (the highest resolution).  last_level indicates how many mipmap levels
+ * to allocate storage for.  For non-mipmapped textures, this will be zero.
+ */
+static struct pipe_resource *
+st_texture_create_from_memory(struct st_context *st,
+                              struct st_memory_object *memObj,
+                              GLuint64 offset,
+                              enum pipe_texture_target target,
+                              enum pipe_format format,
+                              GLuint last_level,
+                              GLuint width0,
+                              GLuint height0,
+                              GLuint depth0,
+                              GLuint layers,
+                              GLuint nr_samples,
+                              GLuint bind)
+{
+   struct pipe_resource pt, *newtex;
+   struct pipe_screen *screen = st->pipe->screen;
+
+   assert(target < PIPE_MAX_TEXTURE_TYPES);
+   assert(width0 > 0);
+   assert(height0 > 0);
+   assert(depth0 > 0);
+   if (target == PIPE_TEXTURE_CUBE)
+      assert(layers == 6);
+
+   DBG("%s target %d format %s last_level %d\n", __func__,
+       (int) target, util_format_name(format), last_level);
+
+   assert(format);
+   assert(screen->is_format_supported(screen, format, target, 0, 0,
+                                      PIPE_BIND_SAMPLER_VIEW));
+
+   memset(&pt, 0, sizeof(pt));
+   pt.target = target;
+   pt.format = format;
+   pt.last_level = last_level;
+   pt.width0 = width0;
+   pt.height0 = height0;
+   pt.depth0 = depth0;
+   pt.array_size = layers;
+   pt.usage = PIPE_USAGE_DEFAULT;
+   pt.bind = bind;
+   /* only set this for OpenGL textures, not renderbuffers */
+   pt.flags = PIPE_RESOURCE_FLAG_TEXTURING_MORE_LIKELY;
+   pt.nr_samples = nr_samples;
+   pt.nr_storage_samples = nr_samples;
+
+   newtex = screen->resource_from_memobj(screen, &pt, memObj->memory, offset);
+
+   assert(!newtex || pipe_is_referenced(&newtex->reference));
+
+   return newtex;
+}
+
+
+/**
+ * Allocate texture memory for a whole mipmap stack.
+ * Note: for multisample textures if the requested sample count is not
+ * supported, we search for the next higher supported sample count.
  */
 static GLboolean
-st_AllocTextureStorage(struct gl_context *ctx,
-                       struct gl_texture_object *texObj,
-                       GLsizei levels, GLsizei width,
-                       GLsizei height, GLsizei depth)
+st_texture_storage(struct gl_context *ctx,
+                   struct gl_texture_object *texObj,
+                   GLsizei levels, GLsizei width,
+                   GLsizei height, GLsizei depth,
+                   struct gl_memory_object *memObj,
+                   GLuint64 offset)
 {
    const GLuint numFaces = _mesa_num_tex_faces(texObj->Target);
    struct gl_texture_image *texImage = texObj->Image[0][0];
    struct st_context *st = st_context(ctx);
    struct st_texture_object *stObj = st_texture_object(texObj);
+   struct st_memory_object *smObj = st_memory_object(memObj);
    struct pipe_screen *screen = st->pipe->screen;
-   GLuint ptWidth, ptHeight, ptDepth, ptLayers, bindings;
+   unsigned ptWidth, bindings;
+   uint16_t ptHeight, ptDepth, ptLayers;
    enum pipe_format fmt;
    GLint level;
    GLuint num_samples = texImage->NumSamples;
 
    assert(levels > 0);
 
-   /* Save the level=0 dimensions */
-   stObj->width0 = width;
-   stObj->height0 = height;
-   stObj->depth0 = depth;
    stObj->lastLevel = levels - 1;
 
    fmt = st_mesa_format_to_pipe_format(st, texImage->TexFormat);
 
    bindings = default_bindings(st, fmt);
 
-   /* Raise the sample count if the requested one is unsupported. */
-   if (num_samples > 1) {
+   if (num_samples > 0) {
+      /* Find msaa sample count which is actually supported.  For example,
+       * if the user requests 1x but only 4x or 8x msaa is supported, we'll
+       * choose 4x here.
+       */
+      enum pipe_texture_target ptarget = gl_target_to_pipe(texObj->Target);
       boolean found = FALSE;
 
+      if (ctx->Const.MaxSamples > 1 && num_samples == 1) {
+         /* don't try num_samples = 1 with drivers that support real msaa */
+         num_samples = 2;
+      }
+
       for (; num_samples <= ctx->Const.MaxSamples; num_samples++) {
-         if (screen->is_format_supported(screen, fmt, PIPE_TEXTURE_2D,
-                                         num_samples,
+         if (screen->is_format_supported(screen, fmt, ptarget,
+                                         num_samples, num_samples,
                                          PIPE_BIND_SAMPLER_VIEW)) {
             /* Update the sample count in gl_texture_image as well. */
             texImage->NumSamples = num_samples;
@@ -2965,15 +2984,31 @@ st_AllocTextureStorage(struct gl_context *ctx,
                                    width, height, depth,
                                    &ptWidth, &ptHeight, &ptDepth, &ptLayers);
 
-   stObj->pt = st_texture_create(st,
-                                 gl_target_to_pipe(texObj->Target),
-                                 fmt,
-                                 levels - 1,
-                                 ptWidth,
-                                 ptHeight,
-                                 ptDepth,
-                                 ptLayers, num_samples,
-                                 bindings);
+   if (smObj) {
+      stObj->pt = st_texture_create_from_memory(st,
+                                                smObj,
+                                                offset,
+                                                gl_target_to_pipe(texObj->Target),
+                                                fmt,
+                                                levels - 1,
+                                                ptWidth,
+                                                ptHeight,
+                                                ptDepth,
+                                                ptLayers, num_samples,
+                                                bindings);
+   }
+   else {
+      stObj->pt = st_texture_create(st,
+                                    gl_target_to_pipe(texObj->Target),
+                                    fmt,
+                                    levels - 1,
+                                    ptWidth,
+                                    ptHeight,
+                                    ptDepth,
+                                    ptLayers, num_samples,
+                                    bindings);
+   }
+
    if (!stObj->pt)
       return GL_FALSE;
 
@@ -2984,18 +3019,40 @@ st_AllocTextureStorage(struct gl_context *ctx,
          struct st_texture_image *stImage =
             st_texture_image(texObj->Image[face][level]);
          pipe_resource_reference(&stImage->pt, stObj->pt);
+
+         compressed_tex_fallback_allocate(st, stImage);
       }
    }
 
+   /* The texture is in a validated state, so no need to check later. */
+   stObj->needs_validation = false;
+   stObj->validated_first_level = 0;
+   stObj->validated_last_level = levels - 1;
+
    return GL_TRUE;
 }
 
+/**
+ * Called via ctx->Driver.AllocTextureStorage() to allocate texture memory
+ * for a whole mipmap stack.
+ */
+static GLboolean
+st_AllocTextureStorage(struct gl_context *ctx,
+                       struct gl_texture_object *texObj,
+                       GLsizei levels, GLsizei width,
+                       GLsizei height, GLsizei depth)
+{
+   return st_texture_storage(ctx, texObj, levels,
+                             width, height, depth,
+                             NULL, 0);
+}
+
 
 static GLboolean
 st_TestProxyTexImage(struct gl_context *ctx, GLenum target,
-                     GLint level, mesa_format format,
-                     GLint width, GLint height,
-                     GLint depth, GLint border)
+                     GLuint numLevels, GLint level,
+                     mesa_format format, GLuint numSamples,
+                     GLint width, GLint height, GLint depth)
 {
    struct st_context *st = st_context(ctx);
    struct pipe_context *pipe = st->pipe;
@@ -3017,28 +3074,34 @@ st_TestProxyTexImage(struct gl_context *ctx, GLenum target,
 
       pt.target = gl_target_to_pipe(target);
       pt.format = st_mesa_format_to_pipe_format(st, format);
+      pt.nr_samples = numSamples;
+      pt.nr_storage_samples = numSamples;
 
       st_gl_texture_dims_to_pipe_dims(target,
                                       width, height, depth,
                                       &pt.width0, &pt.height0,
                                       &pt.depth0, &pt.array_size);
 
-      if (level == 0 && (texObj->Sampler.MinFilter == GL_LINEAR ||
-                         texObj->Sampler.MinFilter == GL_NEAREST)) {
+      if (numLevels > 0) {
+         /* For immutable textures we know the final number of mip levels */
+         pt.last_level = numLevels - 1;
+      }
+      else if (level == 0 && (texObj->Sampler.MinFilter == GL_LINEAR ||
+                              texObj->Sampler.MinFilter == GL_NEAREST)) {
          /* assume just one mipmap level */
          pt.last_level = 0;
       }
       else {
          /* assume a full set of mipmaps */
-         pt.last_level = _mesa_logbase2(MAX3(width, height, depth));
+         pt.last_level = util_logbase2(MAX3(width, height, depth));
       }
 
       return pipe->screen->can_create_resource(pipe->screen, &pt);
    }
    else {
       /* Use core Mesa fallback */
-      return _mesa_test_proxy_teximage(ctx, target, level, format,
-                                       width, height, depth, border);
+      return _mesa_test_proxy_teximage(ctx, target, numLevels, level, format,
+                                       numSamples, width, height, depth);
    }
 }
 
@@ -3047,6 +3110,7 @@ st_TextureView(struct gl_context *ctx,
                struct gl_texture_object *texObj,
                struct gl_texture_object *origTexObj)
 {
+   struct st_context *st = st_context(ctx);
    struct st_texture_object *orig = st_texture_object(origTexObj);
    struct st_texture_object *tex = st_texture_object(texObj);
    struct gl_texture_image *image = texObj->Image[0][0];
@@ -3064,7 +3128,15 @@ st_TextureView(struct gl_context *ctx,
       for (face = 0; face < numFaces; face++) {
          struct st_texture_image *stImage =
             st_texture_image(texObj->Image[face][level]);
+         struct st_texture_image *origImage =
+            st_texture_image(origTexObj->Image[face][level]);
          pipe_resource_reference(&stImage->pt, tex->pt);
+         if (origImage &&
+             origImage->compressed_data) {
+            pipe_reference(NULL,
+                           &origImage->compressed_data->reference);
+            stImage->compressed_data = origImage->compressed_data;
+         }
       }
    }
 
@@ -3072,42 +3144,233 @@ st_TextureView(struct gl_context *ctx,
    tex->surface_format =
       st_mesa_format_to_pipe_format(st_context(ctx), image->TexFormat);
 
-   tex->width0 = image->Width;
-   tex->height0 = image->Height;
-   tex->depth0 = image->Depth;
    tex->lastLevel = numLevels - 1;
 
+   /* free texture sampler views.  They need to be recreated when we
+    * change the texture view parameters.
+    */
+   st_texture_release_all_sampler_views(st, tex);
+
+   /* The texture is in a validated state, so no need to check later. */
+   tex->needs_validation = false;
+   tex->validated_first_level = 0;
+   tex->validated_last_level = numLevels - 1;
+
    return GL_TRUE;
 }
 
+
+/**
+ * Find the mipmap level in 'pt' which matches the level described by
+ * 'texImage'.
+ */
+static unsigned
+find_mipmap_level(const struct gl_texture_image *texImage,
+                  const struct pipe_resource *pt)
+{
+   const GLenum target = texImage->TexObject->Target;
+   GLint texWidth = texImage->Width;
+   GLint texHeight = texImage->Height;
+   GLint texDepth = texImage->Depth;
+   unsigned level, w;
+   uint16_t h, d, layers;
+
+   st_gl_texture_dims_to_pipe_dims(target, texWidth, texHeight, texDepth,
+                                   &w, &h, &d, &layers);
+
+   for (level = 0; level <= pt->last_level; level++) {
+      if (u_minify(pt->width0, level) == w &&
+          u_minify(pt->height0, level) == h &&
+          u_minify(pt->depth0, level) == d) {
+         return level;
+      }
+   }
+
+   /* If we get here, there must be some sort of inconsistency between
+    * the Mesa texture object/images and the gallium resource.
+    */
+   debug_printf("Inconsistent textures in find_mipmap_level()\n");
+
+   return texImage->Level;
+}
+
+
 static void
 st_ClearTexSubImage(struct gl_context *ctx,
                     struct gl_texture_image *texImage,
                     GLint xoffset, GLint yoffset, GLint zoffset,
                     GLsizei width, GLsizei height, GLsizei depth,
-                    const GLvoid *clearValue)
+                    const void *clearValue)
 {
    static const char zeros[16] = {0};
+   struct gl_texture_object *texObj = texImage->TexObject;
    struct st_texture_image *stImage = st_texture_image(texImage);
    struct pipe_resource *pt = stImage->pt;
    struct st_context *st = st_context(ctx);
    struct pipe_context *pipe = st->pipe;
-   unsigned level = texImage->Level;
+   unsigned level;
    struct pipe_box box;
 
    if (!pt)
       return;
 
+   st_flush_bitmap_cache(st);
+   st_invalidate_readpix_cache(st);
+
    u_box_3d(xoffset, yoffset, zoffset + texImage->Face,
             width, height, depth, &box);
-   if (texImage->TexObject->Immutable) {
-      level += texImage->TexObject->MinLevel;
-      box.z += texImage->TexObject->MinLayer;
+   if (texObj->Immutable) {
+      /* The texture object has to be consistent (no "loose", per-image
+       * gallium resources).  If this texture is a view into another
+       * texture, we have to apply the MinLevel/Layer offsets.  If this is
+       * not a texture view, the offsets will be zero.
+       */
+      assert(stImage->pt == st_texture_object(texObj)->pt);
+      level = texImage->Level + texObj->MinLevel;
+      box.z += texObj->MinLayer;
    }
+   else {
+      /* Texture level sizes may be inconsistent.  We my have "loose",
+       * per-image gallium resources.  The texImage->Level may not match
+       * the gallium resource texture level.
+       */
+      level = find_mipmap_level(texImage, pt);
+   }
+
+   assert(level <= pt->last_level);
 
    pipe->clear_texture(pipe, pt, level, &box, clearValue ? clearValue : zeros);
 }
 
+
+/**
+ * Called via the glTexParam*() function, but only when some texture object
+ * state has actually changed.
+ */
+static void
+st_TexParameter(struct gl_context *ctx,
+                struct gl_texture_object *texObj, GLenum pname)
+{
+   struct st_context *st = st_context(ctx);
+   struct st_texture_object *stObj = st_texture_object(texObj);
+
+   switch (pname) {
+   case GL_TEXTURE_BASE_LEVEL:
+   case GL_TEXTURE_MAX_LEVEL:
+   case GL_DEPTH_TEXTURE_MODE:
+   case GL_DEPTH_STENCIL_TEXTURE_MODE:
+   case GL_TEXTURE_SRGB_DECODE_EXT:
+   case GL_TEXTURE_SWIZZLE_R:
+   case GL_TEXTURE_SWIZZLE_G:
+   case GL_TEXTURE_SWIZZLE_B:
+   case GL_TEXTURE_SWIZZLE_A:
+   case GL_TEXTURE_SWIZZLE_RGBA:
+   case GL_TEXTURE_BUFFER_SIZE:
+   case GL_TEXTURE_BUFFER_OFFSET:
+      /* changing any of these texture parameters means we must create
+       * new sampler views.
+       */
+      st_texture_release_all_sampler_views(st, stObj);
+      break;
+   default:
+      ; /* nothing */
+   }
+}
+
+static GLboolean
+st_SetTextureStorageForMemoryObject(struct gl_context *ctx,
+                                    struct gl_texture_object *texObj,
+                                    struct gl_memory_object *memObj,
+                                    GLsizei levels, GLsizei width,
+                                    GLsizei height, GLsizei depth,
+                                    GLuint64 offset)
+{
+   return st_texture_storage(ctx, texObj, levels,
+                             width, height, depth,
+                             memObj, offset);
+}
+
+static GLuint64
+st_NewTextureHandle(struct gl_context *ctx, struct gl_texture_object *texObj,
+                    struct gl_sampler_object *sampObj)
+{
+   struct st_context *st = st_context(ctx);
+   struct st_texture_object *stObj = st_texture_object(texObj);
+   struct pipe_context *pipe = st->pipe;
+   struct pipe_sampler_view *view;
+   struct pipe_sampler_state sampler = {0};
+
+   if (texObj->Target != GL_TEXTURE_BUFFER) {
+      if (!st_finalize_texture(ctx, pipe, texObj, 0))
+         return 0;
+
+      st_convert_sampler(st, texObj, sampObj, 0, &sampler);
+
+      /* TODO: Clarify the interaction of ARB_bindless_texture and EXT_texture_sRGB_decode */
+      view = st_get_texture_sampler_view_from_stobj(st, stObj, sampObj, 0, true);
+   } else {
+      view = st_get_buffer_sampler_view_from_stobj(st, stObj);
+   }
+
+   return pipe->create_texture_handle(pipe, view, &sampler);
+}
+
+
+static void
+st_DeleteTextureHandle(struct gl_context *ctx, GLuint64 handle)
+{
+   struct st_context *st = st_context(ctx);
+   struct pipe_context *pipe = st->pipe;
+
+   pipe->delete_texture_handle(pipe, handle);
+}
+
+
+static void
+st_MakeTextureHandleResident(struct gl_context *ctx, GLuint64 handle,
+                             bool resident)
+{
+   struct st_context *st = st_context(ctx);
+   struct pipe_context *pipe = st->pipe;
+
+   pipe->make_texture_handle_resident(pipe, handle, resident);
+}
+
+
+static GLuint64
+st_NewImageHandle(struct gl_context *ctx, struct gl_image_unit *imgObj)
+{
+   struct st_context *st = st_context(ctx);
+   struct pipe_context *pipe = st->pipe;
+   struct pipe_image_view image;
+
+   st_convert_image(st, imgObj, &image, GL_READ_WRITE);
+
+   return pipe->create_image_handle(pipe, &image);
+}
+
+
+static void
+st_DeleteImageHandle(struct gl_context *ctx, GLuint64 handle)
+{
+   struct st_context *st = st_context(ctx);
+   struct pipe_context *pipe = st->pipe;
+
+   pipe->delete_image_handle(pipe, handle);
+}
+
+
+static void
+st_MakeImageHandleResident(struct gl_context *ctx, GLuint64 handle,
+                           GLenum access, bool resident)
+{
+   struct st_context *st = st_context(ctx);
+   struct pipe_context *pipe = st->pipe;
+
+   pipe->make_image_handle_resident(pipe, handle, access, resident);
+}
+
+
 void
 st_init_texture_functions(struct dd_function_table *functions)
 {
@@ -3123,12 +3386,12 @@ st_init_texture_functions(struct dd_function_table *functions)
 
    /* compressed texture functions */
    functions->CompressedTexImage = st_CompressedTexImage;
-   functions->GetCompressedTexSubImage = _mesa_GetCompressedTexSubImage_sw;
 
    functions->NewTextureObject = st_NewTextureObject;
    functions->NewTextureImage = st_NewTextureImage;
    functions->DeleteTextureImage = st_DeleteTextureImage;
    functions->DeleteTexture = st_DeleteTextureObject;
+   functions->TextureRemovedFromShared = st_TextureReleaseAllSamplerViews;
    functions->AllocTextureImageBuffer = st_AllocTextureImageBuffer;
    functions->FreeTextureImageBuffer = st_FreeTextureImageBuffer;
    functions->MapTextureImage = st_MapTextureImage;
@@ -3140,4 +3403,17 @@ st_init_texture_functions(struct dd_function_table *functions)
    functions->AllocTextureStorage = st_AllocTextureStorage;
    functions->TextureView = st_TextureView;
    functions->ClearTexSubImage = st_ClearTexSubImage;
+
+   functions->TexParameter = st_TexParameter;
+
+   /* bindless functions */
+   functions->NewTextureHandle = st_NewTextureHandle;
+   functions->DeleteTextureHandle = st_DeleteTextureHandle;
+   functions->MakeTextureHandleResident = st_MakeTextureHandleResident;
+   functions->NewImageHandle = st_NewImageHandle;
+   functions->DeleteImageHandle = st_DeleteImageHandle;
+   functions->MakeImageHandleResident = st_MakeImageHandleResident;
+
+   /* external object functions */
+   functions->SetTextureStorageForMemoryObject = st_SetTextureStorageForMemoryObject;
 }