st/mesa: properly implement MapTextureImage with multiple mapped slices (v2)
[mesa.git] / src / mesa / state_tracker / st_cb_texture.c
index 1b93c9d06149f5f735381a8b05e56a195489d7de..97bba8b7da59f38c663f8c7462269da3e99121d5 100644 (file)
@@ -1,6 +1,6 @@
 /**************************************************************************
  * 
- * Copyright 2007 Tungsten Graphics, Inc., Cedar Park, Texas.
+ * Copyright 2007 VMware, Inc.
  * All Rights Reserved.
  * 
  * Permission is hereby granted, free of charge, to any person obtaining a
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
- * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
+ * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  * 
  **************************************************************************/
 
-#include "main/mfeatures.h"
 #include "main/bufferobj.h"
 #include "main/enums.h"
 #include "main/fbobject.h"
@@ -48,6 +47,7 @@
 #include "state_tracker/st_cb_fbo.h"
 #include "state_tracker/st_cb_flush.h"
 #include "state_tracker/st_cb_texture.h"
+#include "state_tracker/st_cb_bufferobjects.h"
 #include "state_tracker/st_format.h"
 #include "state_tracker/st_texture.h"
 #include "state_tracker/st_gen_mipmap.h"
@@ -58,7 +58,6 @@
 #include "util/u_inlines.h"
 #include "pipe/p_shader_tokens.h"
 #include "util/u_tile.h"
-#include "util/u_blit.h"
 #include "util/u_format.h"
 #include "util/u_surface.h"
 #include "util/u_sampler.h"
 #define DBG if (0) printf
 
 
-static enum pipe_texture_target
+enum pipe_texture_target
 gl_target_to_pipe(GLenum target)
 {
    switch (target) {
    case GL_TEXTURE_1D:
+   case GL_PROXY_TEXTURE_1D:
       return PIPE_TEXTURE_1D;
    case GL_TEXTURE_2D:
+   case GL_PROXY_TEXTURE_2D:
+   case GL_TEXTURE_EXTERNAL_OES:
+   case GL_TEXTURE_2D_MULTISAMPLE:
+   case GL_PROXY_TEXTURE_2D_MULTISAMPLE:
       return PIPE_TEXTURE_2D;
    case GL_TEXTURE_RECTANGLE_NV:
+   case GL_PROXY_TEXTURE_RECTANGLE_NV:
       return PIPE_TEXTURE_RECT;
    case GL_TEXTURE_3D:
+   case GL_PROXY_TEXTURE_3D:
       return PIPE_TEXTURE_3D;
    case GL_TEXTURE_CUBE_MAP_ARB:
+   case GL_PROXY_TEXTURE_CUBE_MAP_ARB:
+   case GL_TEXTURE_CUBE_MAP_POSITIVE_X:
+   case GL_TEXTURE_CUBE_MAP_NEGATIVE_X:
+   case GL_TEXTURE_CUBE_MAP_POSITIVE_Y:
+   case GL_TEXTURE_CUBE_MAP_NEGATIVE_Y:
+   case GL_TEXTURE_CUBE_MAP_POSITIVE_Z:
+   case GL_TEXTURE_CUBE_MAP_NEGATIVE_Z:
       return PIPE_TEXTURE_CUBE;
    case GL_TEXTURE_1D_ARRAY_EXT:
+   case GL_PROXY_TEXTURE_1D_ARRAY_EXT:
       return PIPE_TEXTURE_1D_ARRAY;
    case GL_TEXTURE_2D_ARRAY_EXT:
+   case GL_PROXY_TEXTURE_2D_ARRAY_EXT:
+   case GL_TEXTURE_2D_MULTISAMPLE_ARRAY:
+   case GL_PROXY_TEXTURE_2D_MULTISAMPLE_ARRAY:
       return PIPE_TEXTURE_2D_ARRAY;
    case GL_TEXTURE_BUFFER:
       return PIPE_BUFFER;
+   case GL_TEXTURE_CUBE_MAP_ARRAY:
+   case GL_PROXY_TEXTURE_CUBE_MAP_ARRAY:
+      return PIPE_TEXTURE_CUBE_ARRAY;
    default:
       assert(0);
       return 0;
@@ -121,7 +141,7 @@ st_NewTextureObject(struct gl_context * ctx, GLuint name, GLenum target)
    struct st_texture_object *obj = ST_CALLOC_STRUCT(st_texture_object);
 
    DBG("%s\n", __FUNCTION__);
-   _mesa_initialize_texture_object(&obj->base, name, target);
+   _mesa_initialize_texture_object(ctx, &obj->base, name, target);
 
    return &obj->base;
 }
@@ -131,28 +151,19 @@ static void
 st_DeleteTextureObject(struct gl_context *ctx,
                        struct gl_texture_object *texObj)
 {
-   struct st_context *st = st_context(ctx);
    struct st_texture_object *stObj = st_texture_object(texObj);
-   if (stObj->pt)
-      pipe_resource_reference(&stObj->pt, NULL);
-   if (stObj->sampler_view) {
-      if (stObj->sampler_view->context != st->pipe) {
-         /* Take "ownership" of this texture sampler view by setting
-          * its context pointer to this context.  This avoids potential
-          * crashes when the texture object is shared among contexts
-          * and the original/owner context has already been destroyed.
-          */
-         stObj->sampler_view->context = st->pipe;
-      }
-      pipe_sampler_view_reference(&stObj->sampler_view, NULL);
-   }
+
+   pipe_resource_reference(&stObj->pt, NULL);
+   st_texture_release_all_sampler_views(stObj);
+   st_texture_free_sampler_views(stObj);
    _mesa_delete_texture_object(ctx, texObj);
 }
 
 
 /** called via ctx->Driver.FreeTextureImageBuffer() */
 static void
-st_FreeTextureImageBuffer(struct gl_context * ctx, struct gl_texture_image *texImage)
+st_FreeTextureImageBuffer(struct gl_context *ctx,
+                          struct gl_texture_image *texImage)
 {
    struct st_texture_image *stImage = st_texture_image(texImage);
 
@@ -162,10 +173,12 @@ st_FreeTextureImageBuffer(struct gl_context * ctx, struct gl_texture_image *texI
       pipe_resource_reference(&stImage->pt, NULL);
    }
 
-   if (stImage->TexData) {
-      _mesa_align_free(stImage->TexData);
-      stImage->TexData = NULL;
-   }
+   _mesa_align_free(stImage->TexData);
+   stImage->TexData = NULL;
+
+   free(stImage->transfer);
+   stImage->transfer = NULL;
+   stImage->num_transfers = 0;
 }
 
 
@@ -181,17 +194,21 @@ st_MapTextureImage(struct gl_context *ctx,
    struct st_texture_image *stImage = st_texture_image(texImage);
    unsigned pipeMode;
    GLubyte *map;
+   struct pipe_transfer *transfer;
 
    pipeMode = 0x0;
    if (mode & GL_MAP_READ_BIT)
       pipeMode |= PIPE_TRANSFER_READ;
    if (mode & GL_MAP_WRITE_BIT)
       pipeMode |= PIPE_TRANSFER_WRITE;
+   if (mode & GL_MAP_INVALIDATE_RANGE_BIT)
+      pipeMode |= PIPE_TRANSFER_DISCARD_RANGE;
 
-   map = st_texture_image_map(st, stImage, slice, pipeMode, x, y, w, h);
+   map = st_texture_image_map(st, stImage, pipeMode, x, y, slice, w, h, 1,
+                              &transfer);
    if (map) {
       *mapOut = map;
-      *rowStrideOut = stImage->transfer->stride;
+      *rowStrideOut = transfer->stride;
    }
    else {
       *mapOut = NULL;
@@ -208,57 +225,7 @@ st_UnmapTextureImage(struct gl_context *ctx,
 {
    struct st_context *st = st_context(ctx);
    struct st_texture_image *stImage  = st_texture_image(texImage);
-   st_texture_image_unmap(st, stImage);
-}
-
-
-/**
- * From linux kernel i386 header files, copes with odd sizes better
- * than COPY_DWORDS would:
- * XXX Put this in src/mesa/main/imports.h ???
- */
-#if defined(PIPE_CC_GCC) && defined(PIPE_ARCH_X86)
-static INLINE void *
-__memcpy(void *to, const void *from, size_t n)
-{
-   int d0, d1, d2;
-   __asm__ __volatile__("rep ; movsl\n\t"
-                        "testb $2,%b4\n\t"
-                        "je 1f\n\t"
-                        "movsw\n"
-                        "1:\ttestb $1,%b4\n\t"
-                        "je 2f\n\t"
-                        "movsb\n" "2:":"=&c"(d0), "=&D"(d1), "=&S"(d2)
-                        :"0"(n / 4), "q"(n), "1"((long) to), "2"((long) from)
-                        :"memory");
-   return (to);
-}
-#else
-#define __memcpy(a,b,c) memcpy(a,b,c)
-#endif
-
-
-/**
- * The system memcpy (at least on ubuntu 5.10) has problems copying
- * to agp (writecombined) memory from a source which isn't 64-byte
- * aligned - there is a 4x performance falloff.
- *
- * The x86 __memcpy is immune to this but is slightly slower
- * (10%-ish) than the system memcpy.
- *
- * The sse_memcpy seems to have a slight cliff at 64/32 bytes, but
- * isn't much faster than x86_memcpy for agp copies.
- * 
- * TODO: switch dynamically.
- */
-static void *
-do_memcpy(void *dest, const void *src, size_t n)
-{
-   if ((((unsigned long) src) & 63) || (((unsigned long) dest) & 63)) {
-      return __memcpy(dest, src, n);
-   }
-   else
-      return memcpy(dest, src, n);
+   st_texture_image_unmap(st, stImage, slice);
 }
 
 
@@ -291,35 +258,6 @@ default_bindings(struct st_context *st, enum pipe_format format)
 }
 
 
-/** Return number of image dimensions (1, 2 or 3) for a texture target. */
-static GLuint
-get_texture_dims(GLenum target)
-{
-   switch (target) {
-   case GL_TEXTURE_1D:
-   case GL_TEXTURE_1D_ARRAY_EXT:
-   case GL_TEXTURE_BUFFER:
-      return 1;
-   case GL_TEXTURE_2D:
-   case GL_TEXTURE_CUBE_MAP_ARB:
-   case GL_TEXTURE_CUBE_MAP_POSITIVE_X_ARB:
-   case GL_TEXTURE_CUBE_MAP_NEGATIVE_X_ARB:
-   case GL_TEXTURE_CUBE_MAP_POSITIVE_Y_ARB:
-   case GL_TEXTURE_CUBE_MAP_NEGATIVE_Y_ARB:
-   case GL_TEXTURE_CUBE_MAP_POSITIVE_Z_ARB:
-   case GL_TEXTURE_CUBE_MAP_NEGATIVE_Z_ARB:
-   case GL_TEXTURE_RECTANGLE_NV:
-   case GL_TEXTURE_2D_ARRAY_EXT:
-      return 2;
-   case GL_TEXTURE_3D:
-      return 3;
-   default:
-      assert(0 && "invalid texture target in get_texture_dims()");
-      return 1;
-   }
-}
-
-
 /**
  * Given the size of a mipmap image, try to compute the size of the level=0
  * mipmap image.
@@ -334,32 +272,57 @@ guess_base_level_size(GLenum target,
                       GLuint width, GLuint height, GLuint depth, GLuint level,
                       GLuint *width0, GLuint *height0, GLuint *depth0)
 { 
-   const GLuint dims = get_texture_dims(target);
-
    assert(width >= 1);
    assert(height >= 1);
    assert(depth >= 1);
 
    if (level > 0) {
-      /* Depending on the image's size, we can't always make a guess here */
-      if ((dims >= 1 && width == 1) ||
-          (dims >= 2 && height == 1) ||
-          (dims >= 3 && depth == 1)) {
-         /* we can't determine the image size at level=0 */
-         return GL_FALSE;
-      }
+      /* Guess the size of the base level.
+       * Depending on the image's size, we can't always make a guess here.
+       */
+      switch (target) {
+      case GL_TEXTURE_1D:
+      case GL_TEXTURE_1D_ARRAY:
+         width <<= level;
+         break;
+
+      case GL_TEXTURE_2D:
+      case GL_TEXTURE_2D_ARRAY:
+         /* We can't make a good guess here, because the base level dimensions
+          * can be non-square.
+          */
+         if (width == 1 || height == 1) {
+            return GL_FALSE;
+         }
+         width <<= level;
+         height <<= level;
+         break;
+
+      case GL_TEXTURE_CUBE_MAP:
+      case GL_TEXTURE_CUBE_MAP_ARRAY:
+         width <<= level;
+         height <<= level;
+         break;
+
+      case GL_TEXTURE_3D:
+         /* We can't make a good guess here, because the base level dimensions
+          * can be non-cube.
+          */
+         if (width == 1 || height == 1 || depth == 1) {
+            return GL_FALSE;
+         }
+         width <<= level;
+         height <<= level;
+         depth <<= level;
+         break;
+
+      case GL_TEXTURE_RECTANGLE:
+         break;
 
-      /* grow the image size until we hit level = 0 */
-      while (level > 0) {
-         if (width > 1)
-            width <<= 1;
-         if (height > 1)
-            height <<= 1;
-         if (depth > 1)
-            depth <<= 1;
-         level--;
+      default:
+         assert(0);
       }
-   }      
+   }
 
    *width0 = width;
    *height0 = height;
@@ -415,6 +378,8 @@ guess_and_alloc_texture(struct st_context *st,
     */
    if ((stObj->base.Sampler.MinFilter == GL_NEAREST ||
         stObj->base.Sampler.MinFilter == GL_LINEAR ||
+        (stObj->base.BaseLevel == 0 &&
+         stObj->base.MaxLevel == 0) ||
         stImage->base._BaseFormat == GL_DEPTH_COMPONENT ||
         stImage->base._BaseFormat == GL_DEPTH_STENCIL_EXT) &&
        !stObj->base.GenerateMipmap &&
@@ -424,10 +389,8 @@ guess_and_alloc_texture(struct st_context *st,
    }
    else {
       /* alloc space for a full mipmap */
-      GLuint l2width = util_logbase2(width);
-      GLuint l2height = util_logbase2(height);
-      GLuint l2depth = util_logbase2(depth);
-      lastLevel = MAX2(MAX2(l2width, l2height), l2depth);
+      lastLevel = _mesa_get_tex_max_num_levels(stObj->base.Target,
+                                               width, height, depth) - 1;
    }
 
    /* Save the level=0 dimensions */
@@ -450,9 +413,11 @@ guess_and_alloc_texture(struct st_context *st,
                                  ptWidth,
                                  ptHeight,
                                  ptDepth,
-                                 ptLayers,
+                                 ptLayers, 0,
                                  bindings);
 
+   stObj->lastLevel = lastLevel;
+
    DBG("%s returning %d\n", __FUNCTION__, (stObj->pt != NULL));
 
    return stObj->pt != NULL;
@@ -463,25 +428,21 @@ guess_and_alloc_texture(struct st_context *st,
  * Called via ctx->Driver.AllocTextureImageBuffer().
  * If the texture object/buffer already has space for the indicated image,
  * we're done.  Otherwise, allocate memory for the new texture image.
- * XXX This function and st_TexImage() have some duplicated code.  That
- * can be cleaned up in the future.
  */
 static GLboolean
 st_AllocTextureImageBuffer(struct gl_context *ctx,
-                           struct gl_texture_image *texImage,
-                           gl_format format, GLsizei width,
-                           GLsizei height, GLsizei depth)
+                           struct gl_texture_image *texImage)
 {
    struct st_context *st = st_context(ctx);
    struct st_texture_image *stImage = st_texture_image(texImage);
    struct st_texture_object *stObj = st_texture_object(texImage->TexObject);
    const GLuint level = texImage->Level;
+   GLuint width = texImage->Width;
+   GLuint height = texImage->Height;
+   GLuint depth = texImage->Depth;
 
    DBG("%s\n", __FUNCTION__);
 
-   assert(width > 0);
-   assert(height > 0);
-   assert(depth > 0);
    assert(!stImage->TexData);
    assert(!stImage->pt); /* xxx this might be wrong */
 
@@ -497,7 +458,7 @@ st_AllocTextureImageBuffer(struct gl_context *ctx,
    /* The parent texture object does not have space for this image */
 
    pipe_resource_reference(&stObj->pt, NULL);
-   pipe_sampler_view_reference(&stObj->sampler_view, NULL);
+   st_texture_release_all_sampler_views(stObj);
 
    if (!guess_and_alloc_texture(st, stObj, stImage)) {
       /* Probably out of memory.
@@ -519,7 +480,10 @@ st_AllocTextureImageBuffer(struct gl_context *ctx,
    }
    else {
       /* Create a new, temporary texture/resource/buffer to hold this
-       * one texture image.
+       * one texture image.  Note that when we later access this image
+       * (either for mapping or copying) we'll want to always specify
+       * mipmap level=0, even if the image represents some other mipmap
+       * level.
        */
       enum pipe_format format =
          st_mesa_format_to_pipe_format(texImage->TexFormat);
@@ -537,7 +501,7 @@ st_AllocTextureImageBuffer(struct gl_context *ctx,
                                       ptWidth,
                                       ptHeight,
                                       ptDepth,
-                                      ptLayers,
+                                      ptLayers, 0,
                                       bindings);
       return stImage->pt != NULL;
    }
@@ -545,706 +509,620 @@ st_AllocTextureImageBuffer(struct gl_context *ctx,
 
 
 /**
- * Adjust pixel unpack params and image dimensions to strip off the
- * texture border.
- * Gallium doesn't support texture borders.  They've seldem been used
- * and seldom been implemented correctly anyway.
- * \param unpackNew  returns the new pixel unpack parameters
+ * Preparation prior to glTexImage.  Basically check the 'surface_based'
+ * field and switch to a "normal" tex image if necessary.
  */
 static void
-strip_texture_border(GLint border,
-                     GLint *width, GLint *height, GLint *depth,
-                     const struct gl_pixelstore_attrib *unpack,
-                     struct gl_pixelstore_attrib *unpackNew)
+prep_teximage(struct gl_context *ctx, struct gl_texture_image *texImage,
+              GLenum format, GLenum type)
 {
-   assert(border > 0);  /* sanity check */
+   struct gl_texture_object *texObj = texImage->TexObject;
+   struct st_texture_object *stObj = st_texture_object(texObj);
 
-   *unpackNew = *unpack;
+   /* switch to "normal" */
+   if (stObj->surface_based) {
+      const GLenum target = texObj->Target;
+      const GLuint level = texImage->Level;
+      mesa_format texFormat;
 
-   if (unpackNew->RowLength == 0)
-      unpackNew->RowLength = *width;
+      _mesa_clear_texture_object(ctx, texObj);
+      pipe_resource_reference(&stObj->pt, NULL);
 
-   if (depth && unpackNew->ImageHeight == 0)
-      unpackNew->ImageHeight = *height;
+      /* oops, need to init this image again */
+      texFormat = _mesa_choose_texture_format(ctx, texObj, target, level,
+                                              texImage->InternalFormat, format,
+                                              type);
 
-   unpackNew->SkipPixels += border;
-   if (height)
-      unpackNew->SkipRows += border;
-   if (depth)
-      unpackNew->SkipImages += border;
+      _mesa_init_teximage_fields(ctx, texImage,
+                                 texImage->Width, texImage->Height,
+                                 texImage->Depth, texImage->Border,
+                                 texImage->InternalFormat, texFormat);
 
-   assert(*width >= 3);
-   *width = *width - 2 * border;
-   if (height && *height >= 3)
-      *height = *height - 2 * border;
-   if (depth && *depth >= 3)
-      *depth = *depth - 2 * border;
+      stObj->surface_based = GL_FALSE;
+   }
 }
 
 
 /**
- * Do glTexImage1/2/3D().
+ * Return a writemask for the gallium blit. The parameters can be base
+ * formats or "format" from glDrawPixels/glTexImage/glGetTexImage.
  */
-static void
-st_TexImage(struct gl_context * ctx,
-            GLint dims,
-            GLenum target, GLint level,
-            GLint internalFormat,
-            GLint width, GLint height, GLint depth,
-            GLint border,
-            GLenum format, GLenum type, const void *pixels,
-            const struct gl_pixelstore_attrib *unpack,
-            struct gl_texture_object *texObj,
-            struct gl_texture_image *texImage,
-            GLsizei imageSize, GLboolean compressed_src)
+unsigned
+st_get_blit_mask(GLenum srcFormat, GLenum dstFormat)
 {
-   struct st_context *st = st_context(ctx);
-   struct st_texture_object *stObj = st_texture_object(texObj);
-   struct st_texture_image *stImage = st_texture_image(texImage);
-   GLuint dstRowStride = 0;
-   struct gl_pixelstore_attrib unpackNB;
-   enum pipe_transfer_usage transfer_usage = 0;
-   GLubyte *dstMap;
+   switch (dstFormat) {
+   case GL_DEPTH_STENCIL:
+      switch (srcFormat) {
+      case GL_DEPTH_STENCIL:
+         return PIPE_MASK_ZS;
+      case GL_DEPTH_COMPONENT:
+         return PIPE_MASK_Z;
+      case GL_STENCIL_INDEX:
+         return PIPE_MASK_S;
+      default:
+         assert(0);
+         return 0;
+      }
 
-   DBG("%s target %s level %d %dx%dx%d border %d\n", __FUNCTION__,
-       _mesa_lookup_enum_by_nr(target), level, width, height, depth, border);
+   case GL_DEPTH_COMPONENT:
+      switch (srcFormat) {
+      case GL_DEPTH_STENCIL:
+      case GL_DEPTH_COMPONENT:
+         return PIPE_MASK_Z;
+      default:
+         assert(0);
+         return 0;
+      }
 
-   /* switch to "normal" */
-   if (stObj->surface_based) {
-      gl_format texFormat;
+   case GL_STENCIL_INDEX:
+      switch (srcFormat) {
+      case GL_STENCIL_INDEX:
+         return PIPE_MASK_S;
+      default:
+         assert(0);
+         return 0;
+      }
 
-      _mesa_clear_texture_object(ctx, texObj);
-      pipe_resource_reference(&stObj->pt, NULL);
+   default:
+      return PIPE_MASK_RGBA;
+   }
+}
 
-      /* oops, need to init this image again */
-      texFormat = _mesa_choose_texture_format(ctx, texObj, target, level,
-                                              internalFormat, format, type);
 
-      _mesa_init_teximage_fields(ctx, target, texImage,
-                                 width, height, depth, border,
-                                 internalFormat, texFormat);
+static void
+st_TexSubImage(struct gl_context *ctx, GLuint dims,
+               struct gl_texture_image *texImage,
+               GLint xoffset, GLint yoffset, GLint zoffset,
+               GLint width, GLint height, GLint depth,
+               GLenum format, GLenum type, const void *pixels,
+               const struct gl_pixelstore_attrib *unpack)
+{
+   struct st_context *st = st_context(ctx);
+   struct st_texture_image *stImage = st_texture_image(texImage);
+   struct st_texture_object *stObj = st_texture_object(texImage->TexObject);
+   struct pipe_context *pipe = st->pipe;
+   struct pipe_screen *screen = pipe->screen;
+   struct pipe_resource *dst = stImage->pt;
+   struct pipe_resource *src = NULL;
+   struct pipe_resource src_templ;
+   struct pipe_transfer *transfer;
+   struct pipe_blit_info blit;
+   enum pipe_format src_format, dst_format;
+   mesa_format mesa_src_format;
+   GLenum gl_target = texImage->TexObject->Target;
+   unsigned bind;
+   GLubyte *map;
 
-      stObj->surface_based = GL_FALSE;
+   if (!st->prefer_blit_based_texture_transfer) {
+      goto fallback;
    }
 
-   /* gallium does not support texture borders, strip it off */
-   if (border) {
-      strip_texture_border(border, &width, &height, &depth, unpack, &unpackNB);
-      unpack = &unpackNB;
-      texImage->Width = width;
-      texImage->Height = height;
-      texImage->Depth = depth;
-      texImage->Border = 0;
-      border = 0;
+   if (!dst) {
+      goto fallback;
    }
-   else {
-      assert(texImage->Width == width);
-      assert(texImage->Height == height);
-      assert(texImage->Depth == depth);
-   }
-
-   stImage->base.Face = _mesa_tex_target_to_face(target);
-   stImage->base.Level = level;
 
-   /* Release the reference to a potentially orphaned buffer.   
-    * Release any old malloced memory.
-    */
-   if (stImage->pt) {
-      pipe_resource_reference(&stImage->pt, NULL);
-      assert(!stImage->TexData);
-   }
-   else if (stImage->TexData) {
-      _mesa_align_free(stImage->TexData);
+   /* XXX Fallback for depth-stencil formats due to an incomplete stencil
+    * blit implementation in some drivers. */
+   if (format == GL_DEPTH_STENCIL) {
+      goto fallback;
    }
 
-   /*
-    * See if the new image is somehow incompatible with the existing
-    * mipmap.  If so, free the old mipmap.
-    */
-   if (stObj->pt) {
-      if (level > (GLint) stObj->pt->last_level ||
-          !st_texture_match_image(stObj->pt, &stImage->base)) {
-         DBG("release it\n");
-         pipe_resource_reference(&stObj->pt, NULL);
-         assert(!stObj->pt);
-         pipe_sampler_view_reference(&stObj->sampler_view, NULL);
-      }
+   /* If the base internal format and the texture format don't match,
+    * we can't use blit-based TexSubImage. */
+   if (texImage->_BaseFormat !=
+       _mesa_get_format_base_format(texImage->TexFormat)) {
+      goto fallback;
    }
 
-   if (width == 0 || height == 0 || depth == 0) {
-      /* stop after freeing old image */
-      return;
+   /* See if the texture format already matches the format and type,
+    * in which case the memcpy-based fast path will likely be used and
+    * we don't have to blit. */
+   if (_mesa_format_matches_format_and_type(texImage->TexFormat, format,
+                                            type, unpack->SwapBytes)) {
+      goto fallback;
    }
 
-   if (!stObj->pt) {
-      if (!guess_and_alloc_texture(st, stObj, stImage)) {
-         /* Probably out of memory.
-          * Try flushing any pending rendering, then retry.
-          */
-         st_finish(st);
-         if (!guess_and_alloc_texture(st, stObj, stImage)) {
-            _mesa_error(ctx, GL_OUT_OF_MEMORY, "glTexImage");
-            return;
-         }
-      }
-   }
+   if (format == GL_DEPTH_COMPONENT || format == GL_DEPTH_STENCIL)
+      bind = PIPE_BIND_DEPTH_STENCIL;
+   else
+      bind = PIPE_BIND_RENDER_TARGET;
 
-   assert(!stImage->pt);
+   /* See if the destination format is supported.
+    * For luminance and intensity, only the red channel is stored there. */
+   dst_format = util_format_linear(dst->format);
+   dst_format = util_format_luminance_to_red(dst_format);
+   dst_format = util_format_intensity_to_red(dst_format);
 
-   /* Check if this texture image can live inside the texture object's buffer.
-    * If so, store the image there.  Otherwise the image will temporarily live
-    * in its own buffer.
-    */
-   if (stObj->pt &&
-       st_texture_match_image(stObj->pt, &stImage->base)) {
+   if (!dst_format ||
+       !screen->is_format_supported(screen, dst_format, dst->target,
+                                    dst->nr_samples, bind)) {
+      goto fallback;
+   }
 
-      pipe_resource_reference(&stImage->pt, stObj->pt);
-      assert(stImage->pt);
+   /* Choose the source format. */
+   src_format = st_choose_matching_format(screen, PIPE_BIND_SAMPLER_VIEW,
+                                          format, type, unpack->SwapBytes);
+   if (!src_format) {
+      goto fallback;
    }
 
-   if (!stImage->pt)
-      DBG("XXX: Image did not fit into texture - storing in local memory!\n");
+   mesa_src_format = st_pipe_format_to_mesa_format(src_format);
 
-   /* Pixel data may come from regular user memory or a PBO.  For the later,
-    * do bounds checking and map the PBO to read pixels data from it.
-    *
-    * XXX we should try to use a GPU-accelerated path to copy the image data
-    * from the PBO to the texture.
-    */
-   if (compressed_src) {
-      pixels = _mesa_validate_pbo_compressed_teximage(ctx, imageSize, pixels,
-                                                     unpack,
-                                                     "glCompressedTexImage");
-   }
-   else {
-      pixels = _mesa_validate_pbo_teximage(ctx, dims, width, height, 1,
-                                          format, type,
-                                          pixels, unpack, "glTexImage");
+   /* There is no reason to do this if we cannot use memcpy for the temporary
+    * source texture at least. This also takes transfer ops into account,
+    * etc. */
+   if (!_mesa_texstore_can_use_memcpy(ctx,
+                             _mesa_get_format_base_format(mesa_src_format),
+                             mesa_src_format, format, type, unpack)) {
+      goto fallback;
    }
 
-   /* for a 1D array upload the image as a series of layer with height = 1 */
-   if (target == GL_TEXTURE_1D_ARRAY) {
-      depth = height;
-      height = 1;
+   /* TexSubImage only sets a single cubemap face. */
+   if (gl_target == GL_TEXTURE_CUBE_MAP) {
+      gl_target = GL_TEXTURE_2D;
    }
 
-   /*
-    * Prepare to store the texture data.  Either map the gallium texture buffer
-    * memory or malloc space for it.
-    */
-   if (stImage->pt) {
-      if (!pixels) {
-         /* We've allocated texture resource, but have no pixel data - all done. */
-         goto done;
-      }
+   /* Initialize the source texture description. */
+   memset(&src_templ, 0, sizeof(src_templ));
+   src_templ.target = gl_target_to_pipe(gl_target);
+   src_templ.format = src_format;
+   src_templ.bind = PIPE_BIND_SAMPLER_VIEW;
+   src_templ.usage = PIPE_USAGE_STAGING;
 
-      /* Store the image in the gallium transfer object */
-      if (format == GL_DEPTH_COMPONENT &&
-          util_format_is_depth_and_stencil(stImage->pt->format))
-         transfer_usage = PIPE_TRANSFER_READ_WRITE;
-      else
-         transfer_usage = PIPE_TRANSFER_WRITE;
+   st_gl_texture_dims_to_pipe_dims(gl_target, width, height, depth,
+                                   &src_templ.width0, &src_templ.height0,
+                                   &src_templ.depth0, &src_templ.array_size);
 
-      dstMap = st_texture_image_map(st, stImage, 0,
-                                    transfer_usage, 0, 0, width, height);
-      if(stImage->transfer)
-         dstRowStride = stImage->transfer->stride;
+   /* Check for NPOT texture support. */
+   if (!screen->get_param(screen, PIPE_CAP_NPOT_TEXTURES) &&
+       (!util_is_power_of_two(src_templ.width0) ||
+        !util_is_power_of_two(src_templ.height0) ||
+        !util_is_power_of_two(src_templ.depth0))) {
+      goto fallback;
    }
-   else {
-      /* Allocate regular memory and store the image there temporarily.   */
-      GLuint imageSize = _mesa_format_image_size(texImage->TexFormat,
-                                                 width, height, depth);
-      dstRowStride = _mesa_format_row_stride(texImage->TexFormat, width);
 
-      stImage->TexData = _mesa_align_malloc(imageSize, 16);
-      dstMap = stImage->TexData;
+   /* Create the source texture. */
+   src = screen->resource_create(screen, &src_templ);
+   if (!src) {
+      goto fallback;
    }
 
-   if (!dstMap) {
-      _mesa_error(ctx, GL_OUT_OF_MEMORY, "glTexImage");
+   /* Map source pixels. */
+   pixels = _mesa_validate_pbo_teximage(ctx, dims, width, height, depth,
+                                        format, type, pixels, unpack,
+                                        "glTexSubImage");
+   if (!pixels) {
+      /* This is a GL error. */
+      pipe_resource_reference(&src, NULL);
       return;
    }
 
-   if (!pixels) {
-      /* We've allocated texture memory, but have no pixel data - all done. */
-      goto done;
+   /* From now on, we need the gallium representation of dimensions. */
+   if (gl_target == GL_TEXTURE_1D_ARRAY) {
+      depth = height;
+      height = 1;
    }
 
-   DBG("Upload image %dx%dx%d row_len %x pitch %x\n",
-       width, height, depth, width, dstRowStride);
+   map = pipe_transfer_map_3d(pipe, src, 0, PIPE_TRANSFER_WRITE, 0, 0, 0,
+                              width, height, depth, &transfer);
+   if (!map) {
+      _mesa_unmap_teximage_pbo(ctx, unpack);
+      pipe_resource_reference(&src, NULL);
+      goto fallback;
+   }
 
-   /* Copy user texture image into the mapped texture buffer.
-    */
-   if (compressed_src) {
-      const GLuint srcRowStride =
-         _mesa_format_row_stride(texImage->TexFormat, width);
-      if (dstRowStride == srcRowStride) {
-         memcpy(dstMap, pixels, imageSize);
-      }
-      else {
-         GLubyte *dst = dstMap;
-         const char *src = pixels;
-         GLuint i, bw, bh, lines;
-         _mesa_get_format_block_size(texImage->TexFormat, &bw, &bh);
-         lines = (height + bh - 1) / bh;
-
-         for (i = 0; i < lines; ++i) {
-            memcpy(dst, src, srcRowStride);
-            dst += dstRowStride;
-            src += srcRowStride;
+   /* Upload pixels (just memcpy). */
+   {
+      const uint bytesPerRow = width * util_format_get_blocksize(src_format);
+      GLuint row, slice;
+
+      for (slice = 0; slice < (unsigned) depth; slice++) {
+         if (gl_target == GL_TEXTURE_1D_ARRAY) {
+            /* 1D array textures.
+             * We need to convert gallium coords to GL coords.
+             */
+            GLvoid *src = _mesa_image_address3d(unpack, pixels,
+                                                width, depth, format,
+                                                type, 0, slice, 0);
+            memcpy(map, src, bytesPerRow);
          }
-      }
-   }
-   else {
-      const GLuint srcImageStride =
-         _mesa_image_image_stride(unpack, width, height, format, type);
-      GLint i;
-      const GLubyte *src = (const GLubyte *) pixels;
-
-      for (i = 0; i < depth; i++) {
-        if (!_mesa_texstore(ctx, dims, 
-                             texImage->_BaseFormat, 
-                             texImage->TexFormat, 
-                             0, 0, 0, /* dstX/Y/Zoffset */
-                             dstRowStride,
-                             (GLubyte **) &dstMap, /* dstSlice */
-                             width, height, 1,
-                             format, type, src, unpack)) {
-           _mesa_error(ctx, GL_OUT_OF_MEMORY, "glTexImage");
-        }
-
-        if (stImage->pt && i + 1 < depth) {
-            /* unmap this slice */
-           st_texture_image_unmap(st, stImage);
-            /* map next slice of 3D texture */
-            dstMap = st_texture_image_map(st, stImage, i + 1,
-                                          transfer_usage, 0, 0,
-                                          width, height);
-           src += srcImageStride;
-        }
+         else {
+            ubyte *slice_map = map;
+
+            for (row = 0; row < (unsigned) height; row++) {
+               GLvoid *src = _mesa_image_address3d(unpack, pixels,
+                                                   width, height, format,
+                                                   type, slice, row, 0);
+               memcpy(slice_map, src, bytesPerRow);
+               slice_map += transfer->stride;
+            }
+         }
+         map += transfer->layer_stride;
       }
    }
 
-done:
+   pipe_transfer_unmap(pipe, transfer);
    _mesa_unmap_teximage_pbo(ctx, unpack);
 
-   if (stImage->pt && stImage->transfer) {
-      st_texture_image_unmap(st, stImage);
-   }
+   /* Blit. */
+   blit.src.resource = src;
+   blit.src.level = 0;
+   blit.src.format = src_format;
+   blit.dst.resource = dst;
+   blit.dst.level = stObj->pt != stImage->pt ? 0 : texImage->Level;
+   blit.dst.format = dst_format;
+   blit.src.box.x = blit.src.box.y = blit.src.box.z = 0;
+   blit.dst.box.x = xoffset;
+   blit.dst.box.y = yoffset;
+   blit.dst.box.z = zoffset + texImage->Face;
+   blit.src.box.width = blit.dst.box.width = width;
+   blit.src.box.height = blit.dst.box.height = height;
+   blit.src.box.depth = blit.dst.box.depth = depth;
+   blit.mask = st_get_blit_mask(format, texImage->_BaseFormat);
+   blit.filter = PIPE_TEX_FILTER_NEAREST;
+   blit.scissor_enable = FALSE;
+
+   st->pipe->blit(st->pipe, &blit);
+
+   pipe_resource_reference(&src, NULL);
+   return;
+
+fallback:
+   _mesa_store_texsubimage(ctx, dims, texImage, xoffset, yoffset, zoffset,
+                           width, height, depth, format, type, pixels,
+                           unpack);
 }
 
-
 static void
-st_TexImage3D(struct gl_context * ctx,
-              GLenum target, GLint level,
-              GLint internalFormat,
-              GLint width, GLint height, GLint depth,
-              GLint border,
-              GLenum format, GLenum type, const void *pixels,
-              const struct gl_pixelstore_attrib *unpack,
-              struct gl_texture_object *texObj,
-              struct gl_texture_image *texImage)
+st_TexImage(struct gl_context * ctx, GLuint dims,
+            struct gl_texture_image *texImage,
+            GLenum format, GLenum type, const void *pixels,
+            const struct gl_pixelstore_attrib *unpack)
 {
-   st_TexImage(ctx, 3, target, level, internalFormat, width, height, depth,
-               border, format, type, pixels, unpack, texObj, texImage,
-               0, GL_FALSE);
-}
+   assert(dims == 1 || dims == 2 || dims == 3);
 
+   prep_teximage(ctx, texImage, format, type);
 
-static void
-st_TexImage2D(struct gl_context * ctx,
-              GLenum target, GLint level,
-              GLint internalFormat,
-              GLint width, GLint height, GLint border,
-              GLenum format, GLenum type, const void *pixels,
-              const struct gl_pixelstore_attrib *unpack,
-              struct gl_texture_object *texObj,
-              struct gl_texture_image *texImage)
-{
-   st_TexImage(ctx, 2, target, level, internalFormat, width, height, 1, border,
-               format, type, pixels, unpack, texObj, texImage, 0, GL_FALSE);
-}
+   if (texImage->Width == 0 || texImage->Height == 0 || texImage->Depth == 0)
+      return;
 
+   /* allocate storage for texture data */
+   if (!ctx->Driver.AllocTextureImageBuffer(ctx, texImage)) {
+      _mesa_error(ctx, GL_OUT_OF_MEMORY, "glTexImage%uD", dims);
+      return;
+   }
 
-static void
-st_TexImage1D(struct gl_context * ctx,
-              GLenum target, GLint level,
-              GLint internalFormat,
-              GLint width, GLint border,
-              GLenum format, GLenum type, const void *pixels,
-              const struct gl_pixelstore_attrib *unpack,
-              struct gl_texture_object *texObj,
-              struct gl_texture_image *texImage)
-{
-   st_TexImage(ctx, 1, target, level, internalFormat, width, 1, 1, border,
-               format, type, pixels, unpack, texObj, texImage, 0, GL_FALSE);
+   st_TexSubImage(ctx, dims, texImage, 0, 0, 0,
+                  texImage->Width, texImage->Height, texImage->Depth,
+                  format, type, pixels, unpack);
 }
 
 
 static void
-st_CompressedTexImage2D(struct gl_context *ctx, GLenum target, GLint level,
-                        GLint internalFormat,
-                        GLint width, GLint height, GLint border,
-                        GLsizei imageSize, const GLvoid *data,
-                        struct gl_texture_object *texObj,
-                        struct gl_texture_image *texImage)
+st_CompressedTexImage(struct gl_context *ctx, GLuint dims,
+                      struct gl_texture_image *texImage,
+                      GLsizei imageSize, const GLvoid *data)
 {
-   st_TexImage(ctx, 2, target, level, internalFormat, width, height, 1, border,
-               0, 0, data, &ctx->Unpack, texObj, texImage, imageSize, GL_TRUE);
+   prep_teximage(ctx, texImage, GL_NONE, GL_NONE);
+   _mesa_store_compressed_teximage(ctx, dims, texImage, imageSize, data);
 }
 
 
 
-/**
- * glGetTexImage() helper: decompress a compressed texture by rendering
- * a textured quad.  Store the results in the user's buffer.
- */
-static void
-decompress_with_blit(struct gl_context * ctx,
-                     GLenum format, GLenum type, GLvoid *pixels,
-                     struct gl_texture_image *texImage)
-{
-   struct st_context *st = st_context(ctx);
-   struct pipe_context *pipe = st->pipe;
-   struct st_texture_image *stImage = st_texture_image(texImage);
-   struct st_texture_object *stObj = st_texture_object(texImage->TexObject);
-   struct pipe_sampler_view *src_view =
-      st_get_texture_sampler_view(stObj, pipe);
-   const GLuint width = texImage->Width;
-   const GLuint height = texImage->Height;
-   struct pipe_surface *dst_surface;
-   struct pipe_resource *dst_texture;
-   struct pipe_transfer *tex_xfer;
-   unsigned bind = (PIPE_BIND_RENDER_TARGET | /* util_blit may choose to render */
-                   PIPE_BIND_TRANSFER_READ);
-
-   /* create temp / dest surface */
-   if (!util_create_rgba_surface(pipe, width, height, bind,
-                                 &dst_texture, &dst_surface)) {
-      _mesa_problem(ctx, "util_create_rgba_surface() failed "
-                    "in decompress_with_blit()");
-      return;
-   }
-
-   /* Disable conditional rendering. */
-   if (st->render_condition) {
-      pipe->render_condition(pipe, NULL, 0);
-   }
-
-   /* Choose the source mipmap level */
-   src_view->u.tex.first_level = src_view->u.tex.last_level = texImage->Level;
-
-   /* blit/render/decompress */
-   util_blit_pixels_tex(st->blit,
-                        src_view,      /* pipe_resource (src) */
-                        0, 0,             /* src x0, y0 */
-                        width, height,    /* src x1, y1 */
-                        dst_surface,      /* pipe_surface (dst) */
-                        0, 0,             /* dst x0, y0 */
-                        width, height,    /* dst x1, y1 */
-                        0.0,              /* z */
-                        PIPE_TEX_MIPFILTER_NEAREST);
-
-   /* Restore conditional rendering state. */
-   if (st->render_condition) {
-      pipe->render_condition(pipe, st->render_condition,
-                             st->condition_mode);
-   }
-
-   /* map the dst_surface so we can read from it */
-   tex_xfer = pipe_get_transfer(pipe,
-                                dst_texture, 0, 0,
-                                PIPE_TRANSFER_READ,
-                                0, 0, width, height);
-
-   pixels = _mesa_map_pbo_dest(ctx, &ctx->Pack, pixels);
-
-   /* copy/pack data into user buffer */
-   if (st_equal_formats(stImage->pt->format, format, type)) {
-      /* memcpy */
-      const uint bytesPerRow = width * util_format_get_blocksize(stImage->pt->format);
-      ubyte *map = pipe_transfer_map(pipe, tex_xfer);
-      GLuint row;
-      for (row = 0; row < height; row++) {
-         GLvoid *dest = _mesa_image_address2d(&ctx->Pack, pixels, width,
-                                              height, format, type, row, 0);
-         memcpy(dest, map, bytesPerRow);
-         map += tex_xfer->stride;
-      }
-      pipe_transfer_unmap(pipe, tex_xfer);
-   }
-   else {
-      /* format translation via floats */
-      GLuint row;
-      enum pipe_format pformat = util_format_linear(dst_texture->format);
-      for (row = 0; row < height; row++) {
-         const GLbitfield transferOps = 0x0; /* bypassed for glGetTexImage() */
-         GLfloat rgba[4 * MAX_WIDTH];
-         GLvoid *dest = _mesa_image_address2d(&ctx->Pack, pixels, width,
-                                              height, format, type, row, 0);
-
-         if (ST_DEBUG & DEBUG_FALLBACK)
-            debug_printf("%s: fallback format translation\n", __FUNCTION__);
-
-         /* get float[4] rgba row from surface */
-         pipe_get_tile_rgba_format(pipe, tex_xfer, 0, row, width, 1,
-                                   pformat, rgba);
-
-         _mesa_pack_rgba_span_float(ctx, width, (GLfloat (*)[4]) rgba, format,
-                                    type, dest, &ctx->Pack, transferOps);
-      }
-   }
-
-   _mesa_unmap_pbo_dest(ctx, &ctx->Pack);
-
-   pipe->transfer_destroy(pipe, tex_xfer);
-
-   /* destroy the temp / dest surface */
-   util_destroy_rgba_surface(dst_texture, dst_surface);
-}
-
-
 
 /**
  * Called via ctx->Driver.GetTexImage()
+ *
+ * This uses a blit to copy the texture to a texture format which matches
+ * the format and type combo and then a fast read-back is done using memcpy.
+ * We can do arbitrary X/Y/Z/W/0/1 swizzling here as long as there is
+ * a format which matches the swizzling.
+ *
+ * If such a format isn't available, it falls back to _mesa_get_teximage.
+ *
+ * NOTE: Drivers usually do a blit to convert between tiled and linear
+ *       texture layouts during texture uploads/downloads, so the blit
+ *       we do here should be free in such cases.
  */
 static void
 st_GetTexImage(struct gl_context * ctx,
                GLenum format, GLenum type, GLvoid * pixels,
                struct gl_texture_image *texImage)
 {
+   struct st_context *st = st_context(ctx);
+   struct pipe_context *pipe = st->pipe;
+   struct pipe_screen *screen = pipe->screen;
+   GLuint width = texImage->Width;
+   GLuint height = texImage->Height;
+   GLuint depth = texImage->Depth;
    struct st_texture_image *stImage = st_texture_image(texImage);
+   struct pipe_resource *src = st_texture_object(texImage->TexObject)->pt;
+   struct pipe_resource *dst = NULL;
+   struct pipe_resource dst_templ;
+   enum pipe_format dst_format, src_format;
+   mesa_format mesa_format;
+   GLenum gl_target = texImage->TexObject->Target;
+   enum pipe_texture_target pipe_target;
+   struct pipe_blit_info blit;
+   unsigned bind = PIPE_BIND_TRANSFER_READ;
+   struct pipe_transfer *tex_xfer;
+   ubyte *map = NULL;
+   boolean done = FALSE;
 
-   if (stImage->pt && util_format_is_s3tc(stImage->pt->format)) {
-      /* Need to decompress the texture.
-       * We'll do this by rendering a textured quad (which is hopefully
-       * faster than using the fallback code in texcompress.c.
-       * Note that we only expect RGBA formats (no Z/depth formats).
-       */
-      decompress_with_blit(ctx, format, type, pixels, texImage);
+   if (!st->prefer_blit_based_texture_transfer &&
+       !_mesa_is_format_compressed(texImage->TexFormat)) {
+      /* Try to avoid the fallback if we're doing texture decompression here */
+      goto fallback;
    }
-   else {
-      _mesa_get_teximage(ctx, format, type, pixels, texImage);
+
+   if (!stImage->pt || !src) {
+      goto fallback;
    }
-}
 
+   /* XXX Fallback to _mesa_get_teximage for depth-stencil formats
+    * due to an incomplete stencil blit implementation in some drivers. */
+   if (format == GL_DEPTH_STENCIL) {
+      goto fallback;
+   }
 
-static void
-st_TexSubimage(struct gl_context *ctx, GLint dims, GLenum target, GLint level,
-               GLint xoffset, GLint yoffset, GLint zoffset,
-               GLint width, GLint height, GLint depth,
-               GLenum format, GLenum type, const void *pixels,
-               const struct gl_pixelstore_attrib *packing,
-               struct gl_texture_object *texObj,
-               struct gl_texture_image *texImage)
-{
-   struct st_context *st = st_context(ctx);
-   struct st_texture_image *stImage = st_texture_image(texImage);
-   GLuint dstRowStride;
-   const GLuint srcImageStride =
-      _mesa_image_image_stride(packing, width, height, format, type);
-   GLint i;
-   const GLubyte *src;
-   /* init to silence warning only: */
-   enum pipe_transfer_usage transfer_usage = PIPE_TRANSFER_WRITE;
-   GLubyte *dstMap;
-
-   DBG("%s target %s level %d offset %d,%d %dx%d\n", __FUNCTION__,
-       _mesa_lookup_enum_by_nr(target),
-       level, xoffset, yoffset, width, height);
-
-   pixels =
-      _mesa_validate_pbo_teximage(ctx, dims, width, height, depth, format,
-                                  type, pixels, packing, "glTexSubImage2D");
-   if (!pixels)
-      return;
+   /* If the base internal format and the texture format don't match, we have
+    * to fall back to _mesa_get_teximage. */
+   if (texImage->_BaseFormat !=
+       _mesa_get_format_base_format(texImage->TexFormat)) {
+      goto fallback;
+   }
 
-   /* for a 1D array upload the image as a series of layer with height = 1 */
-   if (target == GL_TEXTURE_1D_ARRAY) {
-      depth = height;
-      height = 1;
+   /* See if the texture format already matches the format and type,
+    * in which case the memcpy-based fast path will be used. */
+   if (_mesa_format_matches_format_and_type(texImage->TexFormat, format,
+                                            type, ctx->Pack.SwapBytes)) {
+      goto fallback;
    }
 
-   /* Map buffer if necessary.  Need to lock to prevent other contexts
-    * from uploading the buffer under us.
+   /* Convert the source format to what is expected by GetTexImage
+    * and see if it's supported.
+    *
+    * This only applies to glGetTexImage:
+    * - Luminance must be returned as (L,0,0,1).
+    * - Luminance alpha must be returned as (L,0,0,A).
+    * - Intensity must be returned as (I,0,0,1)
     */
-   if (stImage->pt) {
-      if (format == GL_DEPTH_COMPONENT &&
-          util_format_is_depth_and_stencil(stImage->pt->format))
-         transfer_usage = PIPE_TRANSFER_READ_WRITE;
-      else
-         transfer_usage = PIPE_TRANSFER_WRITE;
+   src_format = util_format_linear(src->format);
+   src_format = util_format_luminance_to_red(src_format);
+   src_format = util_format_intensity_to_red(src_format);
 
-      dstMap = st_texture_image_map(st, stImage, zoffset, 
-                                    transfer_usage,
-                                    xoffset, yoffset,
-                                    width, height);
+   if (!src_format ||
+       !screen->is_format_supported(screen, src_format, src->target,
+                                    src->nr_samples,
+                                    PIPE_BIND_SAMPLER_VIEW)) {
+      goto fallback;
    }
 
-   if (!dstMap) {
-      _mesa_error(ctx, GL_OUT_OF_MEMORY, "glTexSubImage");
-      goto done;
+   if (format == GL_DEPTH_COMPONENT || format == GL_DEPTH_STENCIL)
+      bind |= PIPE_BIND_DEPTH_STENCIL;
+   else
+      bind |= PIPE_BIND_RENDER_TARGET;
+
+   /* GetTexImage only returns a single face for cubemaps. */
+   if (gl_target == GL_TEXTURE_CUBE_MAP) {
+      gl_target = GL_TEXTURE_2D;
    }
+   pipe_target = gl_target_to_pipe(gl_target);
 
-   src = (const GLubyte *) pixels;
-   dstRowStride = stImage->transfer->stride;
+   /* Choose the destination format by finding the best match
+    * for the format+type combo. */
+   dst_format = st_choose_matching_format(screen, bind, format, type,
+                                         ctx->Pack.SwapBytes);
 
-   for (i = 0; i < depth; i++) {
-      if (!_mesa_texstore(ctx, dims, texImage->_BaseFormat,
-                          texImage->TexFormat,
-                          0, 0, 0,
-                          dstRowStride,
-                          (GLubyte **) &dstMap,
-                          width, height, 1,
-                          format, type, src, packing)) {
-        _mesa_error(ctx, GL_OUT_OF_MEMORY, "glTexSubImage");
+   if (dst_format == PIPE_FORMAT_NONE) {
+      GLenum dst_glformat;
+
+      /* Fall back to _mesa_get_teximage except for compressed formats,
+       * where decompression with a blit is always preferred. */
+      if (!util_format_is_compressed(src->format)) {
+         goto fallback;
       }
 
-      if (stImage->pt && i + 1 < depth) {
-         /* unmap this slice */
-        st_texture_image_unmap(st, stImage);
-         /* map next slice of 3D texture */
-        dstMap = st_texture_image_map(st, stImage,
-                                       zoffset + i + 1,
-                                       transfer_usage,
-                                       xoffset, yoffset,
-                                       width, height);
-        src += srcImageStride;
+      /* Set the appropriate format for the decompressed texture.
+       * Luminance and sRGB formats shouldn't appear here.*/
+      switch (src_format) {
+      case PIPE_FORMAT_DXT1_RGB:
+      case PIPE_FORMAT_DXT1_RGBA:
+      case PIPE_FORMAT_DXT3_RGBA:
+      case PIPE_FORMAT_DXT5_RGBA:
+      case PIPE_FORMAT_RGTC1_UNORM:
+      case PIPE_FORMAT_RGTC2_UNORM:
+      case PIPE_FORMAT_ETC1_RGB8:
+         dst_glformat = GL_RGBA8;
+         break;
+      case PIPE_FORMAT_RGTC1_SNORM:
+      case PIPE_FORMAT_RGTC2_SNORM:
+         if (!ctx->Extensions.EXT_texture_snorm)
+            goto fallback;
+         dst_glformat = GL_RGBA8_SNORM;
+         break;
+      /* TODO: for BPTC_*FLOAT, set RGBA32F and check for ARB_texture_float */
+      default:
+         assert(0);
+         goto fallback;
       }
-   }
 
-done:
-   _mesa_unmap_teximage_pbo(ctx, packing);
+      dst_format = st_choose_format(st, dst_glformat, format, type,
+                                    pipe_target, 0, bind, FALSE);
 
-   if (stImage->pt && stImage->transfer) {
-      st_texture_image_unmap(st, stImage);
+      if (dst_format == PIPE_FORMAT_NONE) {
+         /* unable to get an rgba format!?! */
+         goto fallback;
+      }
    }
-}
 
+   /* create the destination texture */
+   memset(&dst_templ, 0, sizeof(dst_templ));
+   dst_templ.target = pipe_target;
+   dst_templ.format = dst_format;
+   dst_templ.bind = bind;
+   dst_templ.usage = PIPE_USAGE_STAGING;
 
+   st_gl_texture_dims_to_pipe_dims(gl_target, width, height, depth,
+                                   &dst_templ.width0, &dst_templ.height0,
+                                   &dst_templ.depth0, &dst_templ.array_size);
 
-static void
-st_TexSubImage3D(struct gl_context *ctx, GLenum target, GLint level,
-                 GLint xoffset, GLint yoffset, GLint zoffset,
-                 GLsizei width, GLsizei height, GLsizei depth,
-                 GLenum format, GLenum type, const GLvoid *pixels,
-                 const struct gl_pixelstore_attrib *packing,
-                 struct gl_texture_object *texObj,
-                 struct gl_texture_image *texImage)
-{
-   st_TexSubimage(ctx, 3, target, level, xoffset, yoffset, zoffset,
-                  width, height, depth, format, type,
-                  pixels, packing, texObj, texImage);
-}
+   dst = screen->resource_create(screen, &dst_templ);
+   if (!dst) {
+      goto fallback;
+   }
 
+   /* From now on, we need the gallium representation of dimensions. */
+   if (gl_target == GL_TEXTURE_1D_ARRAY) {
+      depth = height;
+      height = 1;
+   }
 
-static void
-st_TexSubImage2D(struct gl_context *ctx, GLenum target, GLint level,
-                 GLint xoffset, GLint yoffset,
-                 GLsizei width, GLsizei height,
-                 GLenum format, GLenum type, const GLvoid * pixels,
-                 const struct gl_pixelstore_attrib *packing,
-                 struct gl_texture_object *texObj,
-                 struct gl_texture_image *texImage)
-{
-   st_TexSubimage(ctx, 2, target, level, xoffset, yoffset, 0,
-                  width, height, 1, format, type,
-                  pixels, packing, texObj, texImage);
-}
+   blit.src.resource = src;
+   blit.src.level = texImage->Level;
+   blit.src.format = src_format;
+   blit.dst.resource = dst;
+   blit.dst.level = 0;
+   blit.dst.format = dst->format;
+   blit.src.box.x = blit.dst.box.x = 0;
+   blit.src.box.y = blit.dst.box.y = 0;
+   blit.src.box.z = texImage->Face;
+   blit.dst.box.z = 0;
+   blit.src.box.width = blit.dst.box.width = width;
+   blit.src.box.height = blit.dst.box.height = height;
+   blit.src.box.depth = blit.dst.box.depth = depth;
+   blit.mask = st_get_blit_mask(texImage->_BaseFormat, format);
+   blit.filter = PIPE_TEX_FILTER_NEAREST;
+   blit.scissor_enable = FALSE;
 
+   /* blit/render/decompress */
+   st->pipe->blit(st->pipe, &blit);
 
-static void
-st_TexSubImage1D(struct gl_context *ctx, GLenum target, GLint level,
-                 GLint xoffset, GLsizei width, GLenum format, GLenum type,
-                 const GLvoid * pixels,
-                 const struct gl_pixelstore_attrib *packing,
-                 struct gl_texture_object *texObj,
-                 struct gl_texture_image *texImage)
-{
-   st_TexSubimage(ctx, 1, target, level, xoffset, 0, 0, width, 1, 1,
-                  format, type, pixels, packing, texObj, texImage);
-}
+   pixels = _mesa_map_pbo_dest(ctx, &ctx->Pack, pixels);
 
+   map = pipe_transfer_map_3d(pipe, dst, 0, PIPE_TRANSFER_READ,
+                              0, 0, 0, width, height, depth, &tex_xfer);
+   if (!map) {
+      goto end;
+   }
 
-static void
-st_CompressedTexSubImage1D(struct gl_context *ctx, GLenum target, GLint level,
-                           GLint xoffset, GLsizei width,
-                           GLenum format,
-                           GLsizei imageSize, const GLvoid *data,
-                           struct gl_texture_object *texObj,
-                           struct gl_texture_image *texImage)
-{
-   assert(0);
-}
+   mesa_format = st_pipe_format_to_mesa_format(dst_format);
 
+   /* copy/pack data into user buffer */
+   if (_mesa_format_matches_format_and_type(mesa_format, format, type,
+                                            ctx->Pack.SwapBytes)) {
+      /* memcpy */
+      const uint bytesPerRow = width * util_format_get_blocksize(dst_format);
+      GLuint row, slice;
+
+      for (slice = 0; slice < depth; slice++) {
+         if (gl_target == GL_TEXTURE_1D_ARRAY) {
+            /* 1D array textures.
+             * We need to convert gallium coords to GL coords.
+             */
+            GLvoid *dest = _mesa_image_address3d(&ctx->Pack, pixels,
+                                                 width, depth, format,
+                                                 type, 0, slice, 0);
+            memcpy(dest, map, bytesPerRow);
+         }
+         else {
+            ubyte *slice_map = map;
+
+            for (row = 0; row < height; row++) {
+               GLvoid *dest = _mesa_image_address3d(&ctx->Pack, pixels,
+                                                    width, height, format,
+                                                    type, slice, row, 0);
+               memcpy(dest, slice_map, bytesPerRow);
+               slice_map += tex_xfer->stride;
+            }
+         }
+         map += tex_xfer->layer_stride;
+      }
+   }
+   else {
+      /* format translation via floats */
+      GLuint row, slice;
+      GLfloat *rgba;
 
-static void
-st_CompressedTexSubImage2D(struct gl_context *ctx, GLenum target, GLint level,
-                           GLint xoffset, GLint yoffset,
-                           GLsizei width, GLint height,
-                           GLenum format,
-                           GLsizei imageSize, const GLvoid *data,
-                           struct gl_texture_object *texObj,
-                           struct gl_texture_image *texImage)
-{
-   struct st_context *st = st_context(ctx);
-   struct st_texture_image *stImage = st_texture_image(texImage);
-   int srcBlockStride;
-   int dstBlockStride;
-   int y;
-   enum pipe_format pformat;
-   GLubyte *dstMap;
+      assert(util_format_is_compressed(src->format));
 
-   if (stImage->pt) {
-      pformat = stImage->pt->format;
-
-      dstMap = st_texture_image_map(st, stImage, 0, 
-                                    PIPE_TRANSFER_WRITE,
-                                    xoffset, yoffset,
-                                    width, height);
-      
-      srcBlockStride = util_format_get_stride(pformat, width);
-      dstBlockStride = stImage->transfer->stride;
-   } else {
-      assert(stImage->pt);
-      /* TODO find good values for block and strides */
-      /* TODO also adjust texImage->data for yoffset/xoffset */
-      return;
-   }
+      rgba = malloc(width * 4 * sizeof(GLfloat));
+      if (!rgba) {
+         goto end;
+      }
 
-   if (!dstMap) {
-      _mesa_error(ctx, GL_OUT_OF_MEMORY, "glCompressedTexSubImage");
-      return;
-   }
+      if (ST_DEBUG & DEBUG_FALLBACK)
+         debug_printf("%s: fallback format translation\n", __FUNCTION__);
+
+      for (slice = 0; slice < depth; slice++) {
+         if (gl_target == GL_TEXTURE_1D_ARRAY) {
+            /* 1D array textures.
+             * We need to convert gallium coords to GL coords.
+             */
+            GLvoid *dest = _mesa_image_address3d(&ctx->Pack, pixels,
+                                                 width, depth, format,
+                                                 type, 0, slice, 0);
+
+            /* get float[4] rgba row from surface */
+            pipe_get_tile_rgba_format(tex_xfer, map, 0, 0, width, 1,
+                                      dst_format, rgba);
+
+            _mesa_pack_rgba_span_float(ctx, width, (GLfloat (*)[4]) rgba, format,
+                                       type, dest, &ctx->Pack, 0);
+         }
+         else {
+            for (row = 0; row < height; row++) {
+               GLvoid *dest = _mesa_image_address3d(&ctx->Pack, pixels,
+                                                    width, height, format,
+                                                    type, slice, row, 0);
 
-   assert(xoffset % util_format_get_blockwidth(pformat) == 0);
-   assert(yoffset % util_format_get_blockheight(pformat) == 0);
+               /* get float[4] rgba row from surface */
+               pipe_get_tile_rgba_format(tex_xfer, map, 0, row, width, 1,
+                                         dst_format, rgba);
 
-   for (y = 0; y < height; y += util_format_get_blockheight(pformat)) {
-      /* don't need to adjust for xoffset and yoffset as st_texture_image_map does that */
-      const char *src = (const char*)data + srcBlockStride * util_format_get_nblocksy(pformat, y);
-      char *dst = (char *) dstMap + dstBlockStride * util_format_get_nblocksy(pformat, y);
-      memcpy(dst, src, util_format_get_stride(pformat, width));
-   }
+               _mesa_pack_rgba_span_float(ctx, width, (GLfloat (*)[4]) rgba, format,
+                                          type, dest, &ctx->Pack, 0);
+            }
+         }
+         map += tex_xfer->layer_stride;
+      }
 
-   if (stImage->pt && stImage->transfer) {
-      st_texture_image_unmap(st, stImage);
+      free(rgba);
    }
-}
+   done = TRUE;
 
+end:
+   if (map)
+      pipe_transfer_unmap(pipe, tex_xfer);
 
-static void
-st_CompressedTexSubImage3D(struct gl_context *ctx, GLenum target, GLint level,
-                           GLint xoffset, GLint yoffset, GLint zoffset,
-                           GLsizei width, GLint height, GLint depth,
-                           GLenum format,
-                           GLsizei imageSize, const GLvoid *data,
-                           struct gl_texture_object *texObj,
-                           struct gl_texture_image *texImage)
-{
-   assert(0);
-}
+   _mesa_unmap_pbo_dest(ctx, &ctx->Pack);
+   pipe_resource_reference(&dst, NULL);
 
+fallback:
+   if (!done) {
+      _mesa_get_teximage(ctx, format, type, pixels, texImage);
+   }
+}
 
 
 /**
@@ -1255,36 +1133,39 @@ st_CompressedTexSubImage3D(struct gl_context *ctx, GLenum target, GLint level,
  * Note: srcY=0=TOP of renderbuffer
  */
 static void
-fallback_copy_texsubimage(struct gl_context *ctx, GLenum target, GLint level,
+fallback_copy_texsubimage(struct gl_context *ctx,
                           struct st_renderbuffer *strb,
                           struct st_texture_image *stImage,
                           GLenum baseFormat,
-                          GLint destX, GLint destY, GLint destZ,
+                          GLint destX, GLint destY, GLint slice,
                           GLint srcX, GLint srcY,
                           GLsizei width, GLsizei height)
 {
    struct st_context *st = st_context(ctx);
    struct pipe_context *pipe = st->pipe;
    struct pipe_transfer *src_trans;
-   GLvoid *texDest;
+   GLubyte *texDest;
    enum pipe_transfer_usage transfer_usage;
+   void *map;
+   unsigned dst_width = width;
+   unsigned dst_height = height;
+   unsigned dst_depth = 1;
+   struct pipe_transfer *transfer;
 
    if (ST_DEBUG & DEBUG_FALLBACK)
       debug_printf("%s: fallback processing\n", __FUNCTION__);
 
-   assert(width <= MAX_WIDTH);
-
    if (st_fb_orientation(ctx->ReadBuffer) == Y_0_TOP) {
       srcY = strb->Base.Height - srcY - height;
    }
 
-   src_trans = pipe_get_transfer(pipe,
-                                 strb->texture,
-                                 strb->rtt_level,
-                                 strb->rtt_face + strb->rtt_slice,
-                                 PIPE_TRANSFER_READ,
-                                 srcX, srcY,
-                                 width, height);
+   map = pipe_transfer_map(pipe,
+                           strb->texture,
+                           strb->surface->u.tex.level,
+                           strb->surface->u.tex.first_layer,
+                           PIPE_TRANSFER_READ,
+                           srcX, srcY,
+                           width, height, &src_trans);
 
    if ((baseFormat == GL_DEPTH_COMPONENT ||
         baseFormat == GL_DEPTH_STENCIL) &&
@@ -1293,15 +1174,17 @@ fallback_copy_texsubimage(struct gl_context *ctx, GLenum target, GLint level,
    else
       transfer_usage = PIPE_TRANSFER_WRITE;
 
-   /* XXX this used to ignore destZ param */
-   texDest = st_texture_image_map(st, stImage, destZ, transfer_usage,
-                                  destX, destY, width, height);
+   texDest = st_texture_image_map(st, stImage, transfer_usage,
+                                  destX, destY, slice,
+                                  dst_width, dst_height, dst_depth,
+                                  &transfer);
 
    if (baseFormat == GL_DEPTH_COMPONENT ||
        baseFormat == GL_DEPTH_STENCIL) {
       const GLboolean scaleOrBias = (ctx->Pixel.DepthScale != 1.0F ||
                                      ctx->Pixel.DepthBias != 0.0F);
       GLint row, yStep;
+      uint *data;
 
       /* determine bottom-to-top vs. top-to-bottom order for src buffer */
       if (st_fb_orientation(ctx->ReadBuffer) == Y_0_TOP) {
@@ -1313,24 +1196,39 @@ fallback_copy_texsubimage(struct gl_context *ctx, GLenum target, GLint level,
          yStep = 1;
       }
 
-      /* To avoid a large temp memory allocation, do copy row by row */
-      for (row = 0; row < height; row++, srcY += yStep) {
-         uint data[MAX_WIDTH];
-         pipe_get_tile_z(pipe, src_trans, 0, srcY, width, 1, data);
-         if (scaleOrBias) {
-            _mesa_scale_and_bias_depth_uint(ctx, width, data);
+      data = malloc(width * sizeof(uint));
+
+      if (data) {
+         /* To avoid a large temp memory allocation, do copy row by row */
+         for (row = 0; row < height; row++, srcY += yStep) {
+            pipe_get_tile_z(src_trans, map, 0, srcY, width, 1, data);
+            if (scaleOrBias) {
+               _mesa_scale_and_bias_depth_uint(ctx, width, data);
+            }
+
+            if (stImage->pt->target == PIPE_TEXTURE_1D_ARRAY) {
+               pipe_put_tile_z(transfer, texDest + row*transfer->layer_stride,
+                               0, 0, width, 1, data);
+            }
+            else {
+               pipe_put_tile_z(transfer, texDest, 0, row, width, 1, data);
+            }
          }
-         pipe_put_tile_z(pipe, stImage->transfer, 0, row, width, 1, data);
       }
+      else {
+         _mesa_error(ctx, GL_OUT_OF_MEMORY, "glCopyTexSubImage()");
+      }
+
+      free(data);
    }
    else {
       /* RGBA format */
       GLfloat *tempSrc =
-         (GLfloat *) malloc(width * height * 4 * sizeof(GLfloat));
+         malloc(width * height * 4 * sizeof(GLfloat));
 
       if (tempSrc && texDest) {
          const GLint dims = 2;
-         const GLint dstRowStride = stImage->transfer->stride;
+         GLint dstRowStride;
          struct gl_texture_image *texImage = &stImage->base;
          struct gl_pixelstore_attrib unpack = ctx->DefaultPacking;
 
@@ -1338,11 +1236,18 @@ fallback_copy_texsubimage(struct gl_context *ctx, GLenum target, GLint level,
             unpack.Invert = GL_TRUE;
          }
 
+         if (stImage->pt->target == PIPE_TEXTURE_1D_ARRAY) {
+            dstRowStride = transfer->layer_stride;
+         }
+         else {
+            dstRowStride = transfer->stride;
+         }
+
          /* get float/RGBA image from framebuffer */
          /* XXX this usually involves a lot of int/float conversion.
           * try to avoid that someday.
           */
-         pipe_get_tile_rgba_format(pipe, src_trans, 0, 0, width, height,
+         pipe_get_tile_rgba_format(src_trans, map, 0, 0, width, height,
                                    util_format_linear(strb->texture->format),
                                    tempSrc);
 
@@ -1355,9 +1260,8 @@ fallback_copy_texsubimage(struct gl_context *ctx, GLenum target, GLint level,
          _mesa_texstore(ctx, dims,
                         texImage->_BaseFormat, 
                         texImage->TexFormat, 
-                        0, 0, 0,
                         dstRowStride,
-                        (GLubyte **) &texDest,
+                        &texDest,
                         width, height, 1,
                         GL_RGBA, GL_FLOAT, tempSrc, /* src */
                         &unpack);
@@ -1366,82 +1270,14 @@ fallback_copy_texsubimage(struct gl_context *ctx, GLenum target, GLint level,
          _mesa_error(ctx, GL_OUT_OF_MEMORY, "glTexSubImage");
       }
 
-      if (tempSrc)
-         free(tempSrc);
+      free(tempSrc);
    }
 
-   st_texture_image_unmap(st, stImage);
-   pipe->transfer_destroy(pipe, src_trans);
+   st_texture_image_unmap(st, stImage, slice);
+   pipe->transfer_unmap(pipe, src_trans);
 }
 
 
-
-/**
- * If the format of the src renderbuffer and the format of the dest
- * texture are compatible (in terms of blitting), return a TGSI writemask
- * to be used during the blit.
- * If the src/dest are incompatible, return 0.
- */
-static unsigned
-compatible_src_dst_formats(struct gl_context *ctx,
-                           const struct gl_renderbuffer *src,
-                           const struct gl_texture_image *dst)
-{
-   /* Get logical base formats for the src and dest.
-    * That is, use the user-requested formats and not the actual, device-
-    * chosen formats.
-    * For example, the user may have requested an A8 texture but the
-    * driver may actually be using an RGBA texture format.  When we
-    * copy/blit to that texture, we only want to copy the Alpha channel
-    * and not the RGB channels.
-    *
-    * Similarly, when the src FBO was created an RGB format may have been
-    * requested but the driver actually chose an RGBA format.  In that case,
-    * we don't want to copy the undefined Alpha channel to the dest texture
-    * (it should be 1.0).
-    */
-   const GLenum srcFormat = _mesa_base_fbo_format(ctx, src->InternalFormat);
-   const GLenum dstFormat = _mesa_base_tex_format(ctx, dst->InternalFormat);
-
-   /**
-    * XXX when we have red-only and red/green renderbuffers we'll need
-    * to add more cases here (or implement a general-purpose routine that
-    * queries the existance of the R,G,B,A channels in the src and dest).
-    */
-   if (srcFormat == dstFormat) {
-      /* This is the same as matching_base_formats, which should
-       * always pass, as it did previously.
-       */
-      return TGSI_WRITEMASK_XYZW;
-   }
-   else if (srcFormat == GL_RGB && dstFormat == GL_RGBA) {
-      /* Make sure that A in the dest is 1.  The actual src format
-       * may be RGBA and have undefined A values.
-       */
-      return TGSI_WRITEMASK_XYZ;
-   }
-   else if (srcFormat == GL_RGBA && dstFormat == GL_RGB) {
-      /* Make sure that A in the dest is 1.  The actual dst format
-       * may be RGBA and will need A=1 to provide proper alpha values
-       * when sampled later.
-       */
-      return TGSI_WRITEMASK_XYZ;
-   }
-   else {
-      if (ST_DEBUG & DEBUG_FALLBACK)
-         debug_printf("%s failed for src %s, dst %s\n",
-                      __FUNCTION__, 
-                      _mesa_lookup_enum_by_nr(srcFormat),
-                      _mesa_lookup_enum_by_nr(dstFormat));
-
-      /* Otherwise fail.
-       */
-      return 0;
-   }
-}
-
-
-
 /**
  * Do a CopyTex[Sub]Image1/2/3D() using a hardware (blit) path if possible.
  * Note that the region to copy has already been clipped so we know we
@@ -1450,207 +1286,106 @@ compatible_src_dst_formats(struct gl_context *ctx,
  * Note: srcY=0=Bottom of renderbuffer (GL convention)
  */
 static void
-st_copy_texsubimage(struct gl_context *ctx,
-                    GLenum target, GLint level,
-                    GLint destX, GLint destY, GLint destZ,
-                    GLint srcX, GLint srcY,
-                    GLsizei width, GLsizei height)
+st_CopyTexSubImage(struct gl_context *ctx, GLuint dims,
+                   struct gl_texture_image *texImage,
+                   GLint destX, GLint destY, GLint slice,
+                   struct gl_renderbuffer *rb,
+                   GLint srcX, GLint srcY, GLsizei width, GLsizei height)
 {
-   struct gl_texture_unit *texUnit =
-      &ctx->Texture.Unit[ctx->Texture.CurrentUnit];
-   struct gl_texture_object *texObj =
-      _mesa_select_tex_object(ctx, texUnit, target);
-   struct gl_texture_image *texImage =
-      _mesa_select_tex_image(ctx, texObj, target, level);
    struct st_texture_image *stImage = st_texture_image(texImage);
-   const GLenum texBaseFormat = texImage->_BaseFormat;
-   struct gl_framebuffer *fb = ctx->ReadBuffer;
-   struct st_renderbuffer *strb;
+   struct st_texture_object *stObj = st_texture_object(texImage->TexObject);
+   struct st_renderbuffer *strb = st_renderbuffer(rb);
    struct st_context *st = st_context(ctx);
    struct pipe_context *pipe = st->pipe;
    struct pipe_screen *screen = pipe->screen;
-   enum pipe_format dest_format, src_format;
-   GLboolean use_fallback = GL_TRUE;
-   GLboolean matching_base_formats;
-   GLuint format_writemask, sample_count;
-   struct pipe_surface *dest_surface = NULL;
+   struct pipe_blit_info blit;
+   enum pipe_format dst_format;
    GLboolean do_flip = (st_fb_orientation(ctx->ReadBuffer) == Y_0_TOP);
-
-   /* make sure finalize_textures has been called? 
-    */
-   if (0) st_validate_state(st);
-
-   /* determine if copying depth or color data */
-   if (texBaseFormat == GL_DEPTH_COMPONENT ||
-       texBaseFormat == GL_DEPTH_STENCIL) {
-      strb = st_renderbuffer(fb->_DepthBuffer);
-      if (strb->Base.Wrapped) {
-         strb = st_renderbuffer(strb->Base.Wrapped);
-      }
-   }
-   else {
-      /* texBaseFormat == GL_RGB, GL_RGBA, GL_ALPHA, etc */
-      strb = st_renderbuffer(fb->_ColorReadBuffer);
-   }
+   unsigned bind;
+   GLint srcY0, srcY1;
 
    if (!strb || !strb->surface || !stImage->pt) {
       debug_printf("%s: null strb or stImage\n", __FUNCTION__);
       return;
    }
 
-   sample_count = strb->surface->texture->nr_samples;
-   /* I believe this would be legal, presumably would need to do a resolve
-      for color, and for depth/stencil spec says to just use one of the
-      depth/stencil samples per pixel? Need some transfer clarifications. */
-   assert(sample_count < 2);
-
-   assert(strb);
-   assert(strb->surface);
-   assert(stImage->pt);
-
-   src_format = strb->surface->format;
-   dest_format = stImage->pt->format;
+   if (_mesa_texstore_needs_transfer_ops(ctx, texImage->_BaseFormat,
+                                         texImage->TexFormat)) {
+      goto fallback;
+   }
 
-   /*
-    * Determine if the src framebuffer and dest texture have the same
-    * base format.  We need this to detect a case such as the framebuffer
-    * being GL_RGBA but the texture being GL_RGB.  If the actual hardware
-    * texture format stores RGBA we need to set A=1 (overriding the
-    * framebuffer's alpha values).  We can't do that with the blit or
-    * textured-quad paths.
+   /* The base internal format must match the mesa format, so make sure
+    * e.g. an RGB internal format is really allocated as RGB and not as RGBA.
     */
-   matching_base_formats =
-      (_mesa_get_format_base_format(strb->Base.Format) ==
-       _mesa_get_format_base_format(texImage->TexFormat));
-   format_writemask = compatible_src_dst_formats(ctx, &strb->Base, texImage);
-
-   if (ctx->_ImageTransferState == 0x0) {
-
-      if (matching_base_formats &&
-          src_format == dest_format &&
-          !do_flip)
-      {
-         /* use surface_copy() / blit */
-         struct pipe_box src_box;
-         u_box_2d_zslice(srcX, srcY, strb->surface->u.tex.first_layer,
-                         width, height, &src_box);
-
-         /* for resource_copy_region(), y=0=top, always */
-         pipe->resource_copy_region(pipe,
-                                    /* dest */
-                                    stImage->pt,
-                                    stImage->base.Level,
-                                    destX, destY, destZ + stImage->base.Face,
-                                    /* src */
-                                    strb->texture,
-                                    strb->surface->u.tex.level,
-                                    &src_box);
-         use_fallback = GL_FALSE;
-      }
-      else if (format_writemask &&
-               texBaseFormat != GL_DEPTH_COMPONENT &&
-               texBaseFormat != GL_DEPTH_STENCIL &&
-               screen->is_format_supported(screen, src_format,
-                                           PIPE_TEXTURE_2D, sample_count,
-                                           PIPE_BIND_SAMPLER_VIEW) &&
-               screen->is_format_supported(screen, dest_format,
-                                           PIPE_TEXTURE_2D, 0,
-                                           PIPE_BIND_RENDER_TARGET)) {
-         /* draw textured quad to do the copy */
-         GLint srcY0, srcY1;
-         struct pipe_surface surf_tmpl;
-         memset(&surf_tmpl, 0, sizeof(surf_tmpl));
-         surf_tmpl.format = util_format_linear(stImage->pt->format);
-         surf_tmpl.usage = PIPE_BIND_RENDER_TARGET;
-         surf_tmpl.u.tex.level = stImage->base.Level;
-         surf_tmpl.u.tex.first_layer = stImage->base.Face + destZ;
-         surf_tmpl.u.tex.last_layer = stImage->base.Face + destZ;
-
-         dest_surface = pipe->create_surface(pipe, stImage->pt,
-                                             &surf_tmpl);
-
-         if (do_flip) {
-            srcY1 = strb->Base.Height - srcY - height;
-            srcY0 = srcY1 + height;
-         }
-         else {
-            srcY0 = srcY;
-            srcY1 = srcY0 + height;
-         }
-
-         /* Disable conditional rendering. */
-         if (st->render_condition) {
-            pipe->render_condition(pipe, NULL, 0);
-         }
-
-         util_blit_pixels_writemask(st->blit,
-                                    strb->texture,
-                                    strb->surface->u.tex.level,
-                                    srcX, srcY0,
-                                    srcX + width, srcY1,
-                                    strb->surface->u.tex.first_layer,
-                                    dest_surface,
-                                    destX, destY,
-                                    destX + width, destY + height,
-                                    0.0, PIPE_TEX_MIPFILTER_NEAREST,
-                                    format_writemask);
-
-         /* Restore conditional rendering state. */
-         if (st->render_condition) {
-            pipe->render_condition(pipe, st->render_condition,
-                                   st->condition_mode);
-         }
+   if (texImage->_BaseFormat !=
+       _mesa_get_format_base_format(texImage->TexFormat) ||
+       rb->_BaseFormat != _mesa_get_format_base_format(rb->Format)) {
+      goto fallback;
+   }
 
-         use_fallback = GL_FALSE;
-      }
+   /* Choose the destination format to match the TexImage behavior. */
+   dst_format = util_format_linear(stImage->pt->format);
+   dst_format = util_format_luminance_to_red(dst_format);
+   dst_format = util_format_intensity_to_red(dst_format);
 
-      if (dest_surface)
-         pipe_surface_reference(&dest_surface, NULL);
+   /* See if the destination format is supported. */
+   if (texImage->_BaseFormat == GL_DEPTH_STENCIL ||
+       texImage->_BaseFormat == GL_DEPTH_COMPONENT) {
+      bind = PIPE_BIND_DEPTH_STENCIL;
    }
-
-   if (use_fallback) {
-      /* software fallback */
-      fallback_copy_texsubimage(ctx, target, level,
-                                strb, stImage, texBaseFormat,
-                                destX, destY, destZ,
-                                srcX, srcY, width, height);
+   else {
+      bind = PIPE_BIND_RENDER_TARGET;
    }
-}
-
 
+   if (!dst_format ||
+       !screen->is_format_supported(screen, dst_format, stImage->pt->target,
+                                    stImage->pt->nr_samples, bind)) {
+      goto fallback;
+   }
 
-static void
-st_CopyTexSubImage1D(struct gl_context * ctx, GLenum target, GLint level,
-                     GLint xoffset, GLint x, GLint y, GLsizei width)
-{
-   const GLint yoffset = 0, zoffset = 0;
-   const GLsizei height = 1;
-   st_copy_texsubimage(ctx, target, level,
-                       xoffset, yoffset, zoffset,  /* destX,Y,Z */
-                       x, y, width, height);  /* src X, Y, size */
-}
-
-
-static void
-st_CopyTexSubImage2D(struct gl_context * ctx, GLenum target, GLint level,
-                     GLint xoffset, GLint yoffset,
-                     GLint x, GLint y, GLsizei width, GLsizei height)
-{
-   const GLint zoffset = 0;
-   st_copy_texsubimage(ctx, target, level,
-                       xoffset, yoffset, zoffset,  /* destX,Y,Z */
-                       x, y, width, height);  /* src X, Y, size */
-}
-
+   /* Y flipping for the main framebuffer. */
+   if (do_flip) {
+      srcY1 = strb->Base.Height - srcY - height;
+      srcY0 = srcY1 + height;
+   }
+   else {
+      srcY0 = srcY;
+      srcY1 = srcY0 + height;
+   }
 
-static void
-st_CopyTexSubImage3D(struct gl_context * ctx, GLenum target, GLint level,
-                     GLint xoffset, GLint yoffset, GLint zoffset,
-                     GLint x, GLint y, GLsizei width, GLsizei height)
-{
-   st_copy_texsubimage(ctx, target, level,
-                       xoffset, yoffset, zoffset,  /* destX,Y,Z */
-                       x, y, width, height);  /* src X, Y, size */
+   /* Blit the texture.
+    * This supports flipping, format conversions, and downsampling.
+    */
+   memset(&blit, 0, sizeof(blit));
+   blit.src.resource = strb->texture;
+   blit.src.format = util_format_linear(strb->surface->format);
+   blit.src.level = strb->surface->u.tex.level;
+   blit.src.box.x = srcX;
+   blit.src.box.y = srcY0;
+   blit.src.box.z = strb->surface->u.tex.first_layer;
+   blit.src.box.width = width;
+   blit.src.box.height = srcY1 - srcY0;
+   blit.src.box.depth = 1;
+   blit.dst.resource = stImage->pt;
+   blit.dst.format = dst_format;
+   blit.dst.level = stObj->pt != stImage->pt ? 0 : texImage->Level;
+   blit.dst.box.x = destX;
+   blit.dst.box.y = destY;
+   blit.dst.box.z = stImage->base.Face + slice;
+   blit.dst.box.width = width;
+   blit.dst.box.height = height;
+   blit.dst.box.depth = 1;
+   blit.mask = st_get_blit_mask(rb->_BaseFormat, texImage->_BaseFormat);
+   blit.filter = PIPE_TEX_FILTER_NEAREST;
+   pipe->blit(pipe, &blit);
+   return;
+
+fallback:
+   /* software fallback */
+   fallback_copy_texsubimage(ctx,
+                             strb, stImage, texImage->_BaseFormat,
+                             destX, destY, slice,
+                             srcX, srcY, width, height);
 }
 
 
@@ -1677,9 +1412,23 @@ copy_image_data_to_texture(struct st_context *st,
    if (stImage->pt) {
       /* Copy potentially with the blitter:
        */
+      GLuint src_level;
+      if (stImage->pt->last_level == 0)
+         src_level = 0;
+      else
+         src_level = stImage->base.Level;
+
+      assert(src_level <= stImage->pt->last_level);
+      assert(u_minify(stImage->pt->width0, src_level) == stImage->base.Width);
+      assert(stImage->pt->target == PIPE_TEXTURE_1D_ARRAY ||
+             u_minify(stImage->pt->height0, src_level) == stImage->base.Height);
+      assert(stImage->pt->target == PIPE_TEXTURE_2D_ARRAY ||
+             stImage->pt->target == PIPE_TEXTURE_CUBE_ARRAY ||
+             u_minify(stImage->pt->depth0, src_level) == stImage->base.Depth);
+
       st_texture_image_copy(st->pipe,
                             stObj->pt, dstLevel,  /* dest texture, level */
-                            stImage->pt, stImage->base.Level, /* src texture, level */
+                            stImage->pt, src_level, /* src texture, level */
                             stImage->base.Face);
 
       pipe_resource_reference(&stImage->pt, NULL);
@@ -1721,9 +1470,9 @@ st_finalize_texture(struct gl_context *ctx,
    GLuint face;
    struct st_texture_image *firstImage;
    enum pipe_format firstImageFormat;
-   GLuint ptWidth, ptHeight, ptDepth, ptLayers;
+   GLuint ptWidth, ptHeight, ptDepth, ptLayers, ptNumSamples;
 
-   if (stObj->base._Complete) {
+   if (_mesa_is_texture_complete(tObj, &tObj->Sampler)) {
       /* The texture is complete and we know exactly how many mipmap levels
        * are present/needed.  This is conditional because we may be called
        * from the st_generate_mipmap() function when the texture object is
@@ -1737,6 +1486,26 @@ st_finalize_texture(struct gl_context *ctx,
          stObj->lastLevel = stObj->base._MaxLevel;
    }
 
+   if (tObj->Target == GL_TEXTURE_BUFFER) {
+      struct st_buffer_object *st_obj = st_buffer_object(tObj->BufferObject);
+
+      if (!st_obj) {
+         pipe_resource_reference(&stObj->pt, NULL);
+         st_texture_release_all_sampler_views(stObj);
+         return GL_TRUE;
+      }
+
+      if (st_obj->buffer != stObj->pt) {
+         pipe_resource_reference(&stObj->pt, st_obj->buffer);
+         st_texture_release_all_sampler_views(stObj);
+         stObj->width0 = stObj->pt->width0 / _mesa_get_format_bytes(tObj->_BufferObjectFormat);
+         stObj->height0 = 1;
+         stObj->depth0 = 1;
+      }
+      return GL_TRUE;
+
+   }
+
    firstImage = st_texture_image(stObj->base.Image[0][stObj->base.BaseLevel]);
    assert(firstImage);
 
@@ -1749,7 +1518,12 @@ st_finalize_texture(struct gl_context *ctx,
        firstImage->pt != stObj->pt &&
        (!stObj->pt || firstImage->pt->last_level >= stObj->pt->last_level)) {
       pipe_resource_reference(&stObj->pt, firstImage->pt);
-      pipe_sampler_view_reference(&stObj->sampler_view, NULL);
+      st_texture_release_all_sampler_views(stObj);
+   }
+
+   /* If this texture comes from a window system, there is nothing else to do. */
+   if (stObj->surface_based) {
+      return GL_TRUE;
    }
 
    /* Find gallium format for the Mesa texture */
@@ -1771,6 +1545,7 @@ st_finalize_texture(struct gl_context *ctx,
       /* convert GL dims to Gallium dims */
       st_gl_texture_dims_to_pipe_dims(stObj->base.Target, width, height, depth,
                                       &ptWidth, &ptHeight, &ptDepth, &ptLayers);
+      ptNumSamples = firstImage->base.NumSamples;
    }
 
    /* If we already have a gallium texture, check that it matches the texture
@@ -1778,18 +1553,19 @@ st_finalize_texture(struct gl_context *ctx,
     */
    if (stObj->pt) {
       if (stObj->pt->target != gl_target_to_pipe(stObj->base.Target) ||
-          !st_sampler_compat_formats(stObj->pt->format, firstImageFormat) ||
+          stObj->pt->format != firstImageFormat ||
           stObj->pt->last_level < stObj->lastLevel ||
           stObj->pt->width0 != ptWidth ||
           stObj->pt->height0 != ptHeight ||
           stObj->pt->depth0 != ptDepth ||
+          stObj->pt->nr_samples != ptNumSamples ||
           stObj->pt->array_size != ptLayers)
       {
          /* The gallium texture does not match the Mesa texture so delete the
           * gallium texture now.  We'll make a new one below.
           */
          pipe_resource_reference(&stObj->pt, NULL);
-         pipe_sampler_view_reference(&stObj->sampler_view, NULL);
+         st_texture_release_all_sampler_views(stObj);
          st->dirty.st |= ST_NEW_FRAMEBUFFER;
       }
    }
@@ -1806,7 +1582,7 @@ st_finalize_texture(struct gl_context *ctx,
                                     ptWidth,
                                     ptHeight,
                                     ptDepth,
-                                    ptLayers,
+                                    ptLayers, ptNumSamples,
                                     bindings);
 
       if (!stObj->pt) {
@@ -1842,55 +1618,136 @@ st_finalize_texture(struct gl_context *ctx,
 
 
 /**
- * Returns pointer to a default/dummy texture.
- * This is typically used when the current shader has tex/sample instructions
- * but the user has not provided a (any) texture(s).
+ * Called via ctx->Driver.AllocTextureStorage() to allocate texture memory
+ * for a whole mipmap stack.
  */
-struct gl_texture_object *
-st_get_default_texture(struct st_context *st)
+static GLboolean
+st_AllocTextureStorage(struct gl_context *ctx,
+                       struct gl_texture_object *texObj,
+                       GLsizei levels, GLsizei width,
+                       GLsizei height, GLsizei depth)
 {
-   if (!st->default_texture) {
-      static const GLenum target = GL_TEXTURE_2D;
-      GLubyte pixels[16][16][4];
-      struct gl_texture_object *texObj;
-      struct gl_texture_image *texImg;
-      GLuint i, j;
-
-      /* The ARB_fragment_program spec says (0,0,0,1) should be returned
-       * when attempting to sample incomplete textures.
-       */
-      for (i = 0; i < 16; i++) {
-         for (j = 0; j < 16; j++) {
-            pixels[i][j][0] = 0;
-            pixels[i][j][1] = 0;
-            pixels[i][j][2] = 0;
-            pixels[i][j][3] = 255;
+   const GLuint numFaces = _mesa_num_tex_faces(texObj->Target);
+   struct gl_texture_image *texImage = texObj->Image[0][0];
+   struct st_context *st = st_context(ctx);
+   struct st_texture_object *stObj = st_texture_object(texObj);
+   struct pipe_screen *screen = st->pipe->screen;
+   GLuint ptWidth, ptHeight, ptDepth, ptLayers, bindings;
+   enum pipe_format fmt;
+   GLint level;
+   GLuint num_samples = texImage->NumSamples;
+
+   assert(levels > 0);
+
+   /* Save the level=0 dimensions */
+   stObj->width0 = width;
+   stObj->height0 = height;
+   stObj->depth0 = depth;
+   stObj->lastLevel = levels - 1;
+
+   fmt = st_mesa_format_to_pipe_format(texImage->TexFormat);
+
+   bindings = default_bindings(st, fmt);
+
+   /* Raise the sample count if the requested one is unsupported. */
+   if (num_samples > 1) {
+      boolean found = FALSE;
+
+      for (; num_samples <= ctx->Const.MaxSamples; num_samples++) {
+         if (screen->is_format_supported(screen, fmt, PIPE_TEXTURE_2D,
+                                         num_samples,
+                                         PIPE_BIND_SAMPLER_VIEW)) {
+            /* Update the sample count in gl_texture_image as well. */
+            texImage->NumSamples = num_samples;
+            found = TRUE;
+            break;
          }
       }
 
-      texObj = st->ctx->Driver.NewTextureObject(st->ctx, 0, target);
+      if (!found) {
+         return GL_FALSE;
+      }
+   }
 
-      texImg = _mesa_get_tex_image(st->ctx, texObj, target, 0);
+   st_gl_texture_dims_to_pipe_dims(texObj->Target,
+                                   width, height, depth,
+                                   &ptWidth, &ptHeight, &ptDepth, &ptLayers);
 
-      _mesa_init_teximage_fields(st->ctx, target, texImg,
-                                 16, 16, 1, 0,  /* w, h, d, border */
-                                 GL_RGBA, MESA_FORMAT_RGBA8888);
+   stObj->pt = st_texture_create(st,
+                                 gl_target_to_pipe(texObj->Target),
+                                 fmt,
+                                 levels - 1,
+                                 ptWidth,
+                                 ptHeight,
+                                 ptDepth,
+                                 ptLayers, num_samples,
+                                 bindings);
+   if (!stObj->pt)
+      return GL_FALSE;
 
-      st_TexImage(st->ctx, 2, target,
-                  0, GL_RGBA,    /* level, intformat */
-                  16, 16, 1, 0,  /* w, h, d, border */
-                  GL_RGBA, GL_UNSIGNED_BYTE, pixels,
-                  &st->ctx->DefaultPacking,
-                  texObj, texImg,
-                  0, 0);
+   /* Set image resource pointers */
+   for (level = 0; level < levels; level++) {
+      GLuint face;
+      for (face = 0; face < numFaces; face++) {
+         struct st_texture_image *stImage =
+            st_texture_image(texObj->Image[face][level]);
+         pipe_resource_reference(&stImage->pt, stObj->pt);
+      }
+   }
+
+   return GL_TRUE;
+}
 
-      texObj->Sampler.MinFilter = GL_NEAREST;
-      texObj->Sampler.MagFilter = GL_NEAREST;
-      texObj->_Complete = GL_TRUE;
 
-      st->default_texture = texObj;
+static GLboolean
+st_TestProxyTexImage(struct gl_context *ctx, GLenum target,
+                     GLint level, mesa_format format,
+                     GLint width, GLint height,
+                     GLint depth, GLint border)
+{
+   struct st_context *st = st_context(ctx);
+   struct pipe_context *pipe = st->pipe;
+
+   if (width == 0 || height == 0 || depth == 0) {
+      /* zero-sized images are legal, and always fit! */
+      return GL_TRUE;
+   }
+
+   if (pipe->screen->can_create_resource) {
+      /* Ask the gallium driver if the texture is too large */
+      struct gl_texture_object *texObj =
+         _mesa_get_current_tex_object(ctx, target);
+      struct pipe_resource pt;
+
+      /* Setup the pipe_resource object
+       */
+      memset(&pt, 0, sizeof(pt));
+
+      pt.target = gl_target_to_pipe(target);
+      pt.format = st_mesa_format_to_pipe_format(format);
+
+      st_gl_texture_dims_to_pipe_dims(target,
+                                      width, height, depth,
+                                      &pt.width0, &pt.height0,
+                                      &pt.depth0, &pt.array_size);
+
+      if (level == 0 && (texObj->Sampler.MinFilter == GL_LINEAR ||
+                         texObj->Sampler.MinFilter == GL_NEAREST)) {
+         /* assume just one mipmap level */
+         pt.last_level = 0;
+      }
+      else {
+         /* assume a full set of mipmaps */
+         pt.last_level = _mesa_logbase2(MAX3(width, height, depth));
+      }
+
+      return pipe->screen->can_create_resource(pipe->screen, &pt);
+   }
+   else {
+      /* Use core Mesa fallback */
+      return _mesa_test_proxy_teximage(ctx, target, level, format,
+                                       width, height, depth, border);
    }
-   return st->default_texture;
 }
 
 
@@ -1898,24 +1755,17 @@ void
 st_init_texture_functions(struct dd_function_table *functions)
 {
    functions->ChooseTextureFormat = st_ChooseTextureFormat;
-   functions->TexImage1D = st_TexImage1D;
-   functions->TexImage2D = st_TexImage2D;
-   functions->TexImage3D = st_TexImage3D;
-   functions->TexSubImage1D = st_TexSubImage1D;
-   functions->TexSubImage2D = st_TexSubImage2D;
-   functions->TexSubImage3D = st_TexSubImage3D;
-   functions->CompressedTexSubImage1D = st_CompressedTexSubImage1D;
-   functions->CompressedTexSubImage2D = st_CompressedTexSubImage2D;
-   functions->CompressedTexSubImage3D = st_CompressedTexSubImage3D;
-   functions->CopyTexSubImage1D = st_CopyTexSubImage1D;
-   functions->CopyTexSubImage2D = st_CopyTexSubImage2D;
-   functions->CopyTexSubImage3D = st_CopyTexSubImage3D;
+   functions->QuerySamplesForFormat = st_QuerySamplesForFormat;
+   functions->TexImage = st_TexImage;
+   functions->TexSubImage = st_TexSubImage;
+   functions->CompressedTexSubImage = _mesa_store_compressed_texsubimage;
+   functions->CopyTexSubImage = st_CopyTexSubImage;
    functions->GenerateMipmap = st_generate_mipmap;
 
    functions->GetTexImage = st_GetTexImage;
 
    /* compressed texture functions */
-   functions->CompressedTexImage2D = st_CompressedTexImage2D;
+   functions->CompressedTexImage = st_CompressedTexImage;
    functions->GetCompressedTexImage = _mesa_get_compressed_teximage;
 
    functions->NewTextureObject = st_NewTextureObject;
@@ -1927,8 +1777,8 @@ st_init_texture_functions(struct dd_function_table *functions)
    functions->MapTextureImage = st_MapTextureImage;
    functions->UnmapTextureImage = st_UnmapTextureImage;
 
-   functions->TextureMemCpy = do_memcpy;
-
    /* XXX Temporary until we can query pipe's texture sizes */
-   functions->TestProxyTexImage = _mesa_test_proxy_teximage;
+   functions->TestProxyTexImage = st_TestProxyTexImage;
+
+   functions->AllocTextureStorage = st_AllocTextureStorage;
 }