src/gallium/auxiliary/util/u_tile.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2007 Tungsten Graphics, Inc., Cedar Park, Texas.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  19  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  21  * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
  22  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  23  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  24  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27
  28 /**
  29  * RGBA/float tile get/put functions.
  30  * Usable both by drivers and state trackers.
  31  */
  32
  33
  34 #include "pipe/p_defines.h"
  35 #include "util/u_inlines.h"
  36
  37 #include "util/u_format.h"
  38 #include "util/u_math.h"
  39 #include "util/u_memory.h"
  40 #include "util/u_rect.h"
  41 #include "util/u_tile.h"
  42
  43
  44 /**
  45  * Move raw block of pixels from transfer object to user memory.
  46  */
  47 void
  48 pipe_get_tile_raw(struct pipe_context *pipe,
  49                   struct pipe_transfer *pt,
  50                   uint x, uint y, uint w, uint h,
  51                   void *dst, int dst_stride)
  52 {
  53    const void *src;
  54
  55    if (dst_stride == 0)
  56       dst_stride = util_format_get_stride(pt->resource->format, w);
  57
  58    if (u_clip_tile(x, y, &w, &h, &pt->box))
  59       return;
  60
  61    src = pipe->transfer_map(pipe, pt);
  62    assert(src);
  63    if(!src)
  64       return;
  65
  66    util_copy_rect(dst, pt->resource->format, dst_stride, 0, 0, w, h, src, pt->stride, x, y);
  67
  68    pipe->transfer_unmap(pipe, pt);
  69 }
  70
  71
  72 /**
  73  * Move raw block of pixels from user memory to transfer object.
  74  */
  75 void
  76 pipe_put_tile_raw(struct pipe_context *pipe,
  77                   struct pipe_transfer *pt,
  78                   uint x, uint y, uint w, uint h,
  79                   const void *src, int src_stride)
  80 {
  81    void *dst;
  82    enum pipe_format format = pt->resource->format;
  83
  84    if (src_stride == 0)
  85       src_stride = util_format_get_stride(format, w);
  86
  87    if (u_clip_tile(x, y, &w, &h, &pt->box))
  88       return;
  89
  90    dst = pipe->transfer_map(pipe, pt);
  91    assert(dst);
  92    if(!dst)
  93       return;
  94
  95    util_copy_rect(dst, format, pt->stride, x, y, w, h, src, src_stride, 0, 0);
  96
  97    pipe->transfer_unmap(pipe, pt);
  98 }
  99
 100
 101
 102
 103 /** Convert short in [-32768,32767] to GLfloat in [-1.0,1.0] */
 104 #define SHORT_TO_FLOAT(S)   ((2.0F * (S) + 1.0F) * (1.0F/65535.0F))
 105
 106 #define UNCLAMPED_FLOAT_TO_SHORT(us, f)  \
 107    us = ( (short) ( CLAMP((f), -1.0, 1.0) * 32767.0F) )
 108
 109
 110
 111 /*** PIPE_FORMAT_Z16_UNORM ***/
 112
 113 /**
 114  * Return each Z value as four floats in [0,1].
 115  */
 116 static void
 117 z16_get_tile_rgba(const ushort *src,
 118                   unsigned w, unsigned h,
 119                   float *p,
 120                   unsigned dst_stride)
 121 {
 122    const float scale = 1.0f / 65535.0f;
 123    unsigned i, j;
 124
 125    for (i = 0; i < h; i++) {
 126       float *pRow = p;
 127       for (j = 0; j < w; j++, pRow += 4) {
 128          pRow[0] =
 129          pRow[1] =
 130          pRow[2] =
 131          pRow[3] = *src++ * scale;
 132       }
 133       p += dst_stride;
 134    }
 135 }
 136
 137
 138
 139
 140 /*** PIPE_FORMAT_Z32_UNORM ***/
 141
 142 /**
 143  * Return each Z value as four floats in [0,1].
 144  */
 145 static void
 146 z32_get_tile_rgba(const unsigned *src,
 147                   unsigned w, unsigned h,
 148                   float *p,
 149                   unsigned dst_stride)
 150 {
 151    const double scale = 1.0 / (double) 0xffffffff;
 152    unsigned i, j;
 153
 154    for (i = 0; i < h; i++) {
 155       float *pRow = p;
 156       for (j = 0; j < w; j++, pRow += 4) {
 157          pRow[0] =
 158          pRow[1] =
 159          pRow[2] =
 160          pRow[3] = (float) (*src++ * scale);
 161       }
 162       p += dst_stride;
 163    }
 164 }
 165
 166
 167 /*** PIPE_FORMAT_Z24_UNORM_S8_USCALED ***/
 168
 169 /**
 170  * Return Z component as four float in [0,1].  Stencil part ignored.
 171  */
 172 static void
 173 s8z24_get_tile_rgba(const unsigned *src,
 174                     unsigned w, unsigned h,
 175                     float *p,
 176                     unsigned dst_stride)
 177 {
 178    const double scale = 1.0 / ((1 << 24) - 1);
 179    unsigned i, j;
 180
 181    for (i = 0; i < h; i++) {
 182       float *pRow = p;
 183       for (j = 0; j < w; j++, pRow += 4) {
 184          pRow[0] =
 185          pRow[1] =
 186          pRow[2] =
 187          pRow[3] = (float) (scale * (*src++ & 0xffffff));
 188       }
 189       p += dst_stride;
 190    }
 191 }
 192
 193
 194 /*** PIPE_FORMAT_S8_USCALED_Z24_UNORM ***/
 195
 196 /**
 197  * Return Z component as four float in [0,1].  Stencil part ignored.
 198  */
 199 static void
 200 z24s8_get_tile_rgba(const unsigned *src,
 201                     unsigned w, unsigned h,
 202                     float *p,
 203                     unsigned dst_stride)
 204 {
 205    const double scale = 1.0 / ((1 << 24) - 1);
 206    unsigned i, j;
 207
 208    for (i = 0; i < h; i++) {
 209       float *pRow = p;
 210       for (j = 0; j < w; j++, pRow += 4) {
 211          pRow[0] =
 212          pRow[1] =
 213          pRow[2] =
 214          pRow[3] = (float) (scale * (*src++ >> 8));
 215       }
 216       p += dst_stride;
 217    }
 218 }
 219
 220 /*** PIPE_FORMAT_S8X24_USCALED ***/
 221
 222 /**
 223  * Return S component as four uint32_t in [0..255].  Z part ignored.
 224  */
 225 static void
 226 s8x24_get_tile_rgba(const unsigned *src,
 227                     unsigned w, unsigned h,
 228                     float *p,
 229                     unsigned dst_stride)
 230 {
 231    unsigned i, j;
 232
 233    for (i = 0; i < h; i++) {
 234       float *pRow = p;
 235
 236       for (j = 0; j < w; j++, pRow += 4) {
 237          pRow[0] =
 238          pRow[1] =
 239          pRow[2] =
 240          pRow[3] = (float)((*src++ >> 24) & 0xff);
 241       }
 242
 243       p += dst_stride;
 244    }
 245 }
 246
 247 /*** PIPE_FORMAT_X24S8_USCALED ***/
 248
 249 /**
 250  * Return S component as four uint32_t in [0..255].  Z part ignored.
 251  */
 252 static void
 253 x24s8_get_tile_rgba(const unsigned *src,
 254                     unsigned w, unsigned h,
 255                     float *p,
 256                     unsigned dst_stride)
 257 {
 258    unsigned i, j;
 259
 260    for (i = 0; i < h; i++) {
 261       float *pRow = p;
 262       for (j = 0; j < w; j++, pRow += 4) {
 263          pRow[0] =
 264          pRow[1] =
 265          pRow[2] =
 266          pRow[3] = (float)(*src++ & 0xff);
 267       }
 268       p += dst_stride;
 269    }
 270 }
 271
 272
 273 /**
 274  * Return S component as four uint32_t in [0..255].  Z part ignored.
 275  */
 276 static void
 277 s8_get_tile_rgba(const unsigned char *src,
 278                  unsigned w, unsigned h,
 279                  float *p,
 280                  unsigned dst_stride)
 281 {
 282    unsigned i, j;
 283
 284    for (i = 0; i < h; i++) {
 285       float *pRow = p;
 286       for (j = 0; j < w; j++, pRow += 4) {
 287          pRow[0] =
 288          pRow[1] =
 289          pRow[2] =
 290          pRow[3] = (float)(*src++ & 0xff);
 291       }
 292       p += dst_stride;
 293    }
 294 }
 295
 296 /*** PIPE_FORMAT_Z32_FLOAT ***/
 297
 298 /**
 299  * Return each Z value as four floats in [0,1].
 300  */
 301 static void
 302 z32f_get_tile_rgba(const float *src,
 303                    unsigned w, unsigned h,
 304                    float *p,
 305                    unsigned dst_stride)
 306 {
 307    unsigned i, j;
 308
 309    for (i = 0; i < h; i++) {
 310       float *pRow = p;
 311       for (j = 0; j < w; j++, pRow += 4) {
 312          pRow[0] =
 313          pRow[1] =
 314          pRow[2] =
 315          pRow[3] = *src++;
 316       }
 317       p += dst_stride;
 318    }
 319 }
 320
 321
 322 void
 323 pipe_tile_raw_to_rgba(enum pipe_format format,
 324                       void *src,
 325                       uint w, uint h,
 326                       float *dst, unsigned dst_stride)
 327 {
 328    switch (format) {
 329    case PIPE_FORMAT_Z16_UNORM:
 330       z16_get_tile_rgba((ushort *) src, w, h, dst, dst_stride);
 331       break;
 332    case PIPE_FORMAT_Z32_UNORM:
 333       z32_get_tile_rgba((unsigned *) src, w, h, dst, dst_stride);
 334       break;
 335    case PIPE_FORMAT_Z24_UNORM_S8_USCALED:
 336    case PIPE_FORMAT_Z24X8_UNORM:
 337       s8z24_get_tile_rgba((unsigned *) src, w, h, dst, dst_stride);
 338       break;
 339    case PIPE_FORMAT_S8_USCALED:
 340       s8_get_tile_rgba((unsigned char *) src, w, h, dst, dst_stride);
 341       break;
 342    case PIPE_FORMAT_X24S8_USCALED:
 343       s8x24_get_tile_rgba((unsigned *) src, w, h, dst, dst_stride);
 344       break;
 345    case PIPE_FORMAT_S8_USCALED_Z24_UNORM:
 346    case PIPE_FORMAT_X8Z24_UNORM:
 347       z24s8_get_tile_rgba((unsigned *) src, w, h, dst, dst_stride);
 348       break;
 349    case PIPE_FORMAT_S8X24_USCALED:
 350       x24s8_get_tile_rgba((unsigned *) src, w, h, dst, dst_stride);
 351       break;
 352    case PIPE_FORMAT_Z32_FLOAT:
 353       z32f_get_tile_rgba((float *) src, w, h, dst, dst_stride);
 354       break;
 355    default:
 356       util_format_read_4f(format,
 357                           dst, dst_stride * sizeof(float),
 358                           src, util_format_get_stride(format, w),
 359                           0, 0, w, h);
 360    }
 361 }
 362
 363
 364 void
 365 pipe_get_tile_rgba(struct pipe_context *pipe,
 366                    struct pipe_transfer *pt,
 367                    uint x, uint y, uint w, uint h,
 368                    float *p)
 369 {
 370    unsigned dst_stride = w * 4;
 371    void *packed;
 372    enum pipe_format format = pt->resource->format;
 373
 374    if (u_clip_tile(x, y, &w, &h, &pt->box))
 375       return;
 376
 377    packed = MALLOC(util_format_get_nblocks(format, w, h) * util_format_get_blocksize(format));
 378
 379    if (!packed)
 380       return;
 381
 382    if(format == PIPE_FORMAT_UYVY || format == PIPE_FORMAT_YUYV)
 383       assert((x & 1) == 0);
 384
 385    pipe_get_tile_raw(pipe, pt, x, y, w, h, packed, 0);
 386
 387    pipe_tile_raw_to_rgba(format, packed, w, h, p, dst_stride);
 388
 389    FREE(packed);
 390 }
 391
 392
 393 void
 394 pipe_get_tile_swizzle(struct pipe_context *pipe,
 395                       struct pipe_transfer *pt,
 396                       uint x,
 397                       uint y,
 398                       uint w,
 399                       uint h,
 400                       uint swizzle_r,
 401                       uint swizzle_g,
 402                       uint swizzle_b,
 403                       uint swizzle_a,
 404                       enum pipe_format format,
 405                       float *p)
 406 {
 407    unsigned dst_stride = w * 4;
 408    void *packed;
 409    uint iy;
 410    float rgba01[6];
 411
 412    if (u_clip_tile(x, y, &w, &h, &pt->box)) {
 413       return;
 414    }
 415
 416    packed = MALLOC(util_format_get_nblocks(format, w, h) * util_format_get_blocksize(format));
 417    if (!packed) {
 418       return;
 419    }
 420
 421    if (format == PIPE_FORMAT_UYVY || format == PIPE_FORMAT_YUYV) {
 422       assert((x & 1) == 0);
 423    }
 424
 425    pipe_get_tile_raw(pipe, pt, x, y, w, h, packed, 0);
 426
 427    pipe_tile_raw_to_rgba(format, packed, w, h, p, dst_stride);
 428
 429    FREE(packed);
 430
 431    if (swizzle_r == PIPE_SWIZZLE_RED &&
 432        swizzle_g == PIPE_SWIZZLE_GREEN &&
 433        swizzle_b == PIPE_SWIZZLE_BLUE &&
 434        swizzle_a == PIPE_SWIZZLE_ALPHA) {
 435       /* no-op, skip */
 436       return;
 437    }
 438
 439    rgba01[PIPE_SWIZZLE_ZERO] = 0.0f;
 440    rgba01[PIPE_SWIZZLE_ONE] = 1.0f;
 441
 442    for (iy = 0; iy < h; iy++) {
 443       float *row = p;
 444       uint ix;
 445
 446       for (ix = 0; ix < w; ix++) {
 447          rgba01[PIPE_SWIZZLE_RED] = row[0];
 448          rgba01[PIPE_SWIZZLE_GREEN] = row[1];
 449          rgba01[PIPE_SWIZZLE_BLUE] = row[2];
 450          rgba01[PIPE_SWIZZLE_ALPHA] = row[3];
 451
 452          *row++ = rgba01[swizzle_r];
 453          *row++ = rgba01[swizzle_g];
 454          *row++ = rgba01[swizzle_b];
 455          *row++ = rgba01[swizzle_a];
 456       }
 457       p += dst_stride;
 458    }
 459 }
 460
 461
 462 void
 463 pipe_put_tile_rgba(struct pipe_context *pipe,
 464                    struct pipe_transfer *pt,
 465                    uint x, uint y, uint w, uint h,
 466                    const float *p)
 467 {
 468    unsigned src_stride = w * 4;
 469    void *packed;
 470    enum pipe_format format = pt->resource->format;
 471
 472    if (u_clip_tile(x, y, &w, &h, &pt->box))
 473       return;
 474
 475    packed = MALLOC(util_format_get_nblocks(format, w, h) * util_format_get_blocksize(format));
 476
 477    if (!packed)
 478       return;
 479
 480    switch (format) {
 481    case PIPE_FORMAT_Z16_UNORM:
 482       /*z16_put_tile_rgba((ushort *) packed, w, h, p, src_stride);*/
 483       break;
 484    case PIPE_FORMAT_Z32_UNORM:
 485       /*z32_put_tile_rgba((unsigned *) packed, w, h, p, src_stride);*/
 486       break;
 487    case PIPE_FORMAT_Z24_UNORM_S8_USCALED:
 488    case PIPE_FORMAT_Z24X8_UNORM:
 489       /*s8z24_put_tile_rgba((unsigned *) packed, w, h, p, src_stride);*/
 490       break;
 491    case PIPE_FORMAT_S8_USCALED_Z24_UNORM:
 492    case PIPE_FORMAT_X8Z24_UNORM:
 493       /*z24s8_put_tile_rgba((unsigned *) packed, w, h, p, src_stride);*/
 494       break;
 495    default:
 496       util_format_write_4f(format,
 497                            p, src_stride * sizeof(float),
 498                            packed, util_format_get_stride(format, w),
 499                            0, 0, w, h);
 500    }
 501
 502    pipe_put_tile_raw(pipe, pt, x, y, w, h, packed, 0);
 503
 504    FREE(packed);
 505 }
 506
 507
 508 /**
 509  * Get a block of Z values, converted to 32-bit range.
 510  */
 511 void
 512 pipe_get_tile_z(struct pipe_context *pipe,
 513                 struct pipe_transfer *pt,
 514                 uint x, uint y, uint w, uint h,
 515                 uint *z)
 516 {
 517    const uint dstStride = w;
 518    ubyte *map;
 519    uint *pDest = z;
 520    uint i, j;
 521    enum pipe_format format = pt->resource->format;
 522
 523    if (u_clip_tile(x, y, &w, &h, &pt->box))
 524       return;
 525
 526    map = (ubyte *)pipe->transfer_map(pipe, pt);
 527    if (!map) {
 528       assert(0);
 529       return;
 530    }
 531
 532    switch (format) {
 533    case PIPE_FORMAT_Z32_UNORM:
 534       {
 535          const uint *ptrc
 536             = (const uint *)(map  + y * pt->stride + x*4);
 537          for (i = 0; i < h; i++) {
 538             memcpy(pDest, ptrc, 4 * w);
 539             pDest += dstStride;
 540             ptrc += pt->stride/4;
 541          }
 542       }
 543       break;
 544    case PIPE_FORMAT_Z24_UNORM_S8_USCALED:
 545    case PIPE_FORMAT_Z24X8_UNORM:
 546       {
 547          const uint *ptrc
 548             = (const uint *)(map + y * pt->stride + x*4);
 549          for (i = 0; i < h; i++) {
 550             for (j = 0; j < w; j++) {
 551                /* convert 24-bit Z to 32-bit Z */
 552                pDest[j] = (ptrc[j] << 8) | ((ptrc[j] >> 16) & 0xff);
 553             }
 554             pDest += dstStride;
 555             ptrc += pt->stride/4;
 556          }
 557       }
 558       break;
 559    case PIPE_FORMAT_S8_USCALED_Z24_UNORM:
 560    case PIPE_FORMAT_X8Z24_UNORM:
 561       {
 562          const uint *ptrc
 563             = (const uint *)(map + y * pt->stride + x*4);
 564          for (i = 0; i < h; i++) {
 565             for (j = 0; j < w; j++) {
 566                /* convert 24-bit Z to 32-bit Z */
 567                pDest[j] = (ptrc[j] & 0xffffff00) | ((ptrc[j] >> 24) & 0xff);
 568             }
 569             pDest += dstStride;
 570             ptrc += pt->stride/4;
 571          }
 572       }
 573       break;
 574    case PIPE_FORMAT_Z16_UNORM:
 575       {
 576          const ushort *ptrc
 577             = (const ushort *)(map + y * pt->stride + x*2);
 578          for (i = 0; i < h; i++) {
 579             for (j = 0; j < w; j++) {
 580                /* convert 16-bit Z to 32-bit Z */
 581                pDest[j] = (ptrc[j] << 16) | ptrc[j];
 582             }
 583             pDest += dstStride;
 584             ptrc += pt->stride/2;
 585          }
 586       }
 587       break;
 588    default:
 589       assert(0);
 590    }
 591
 592    pipe->transfer_unmap(pipe, pt);
 593 }
 594
 595
 596 void
 597 pipe_put_tile_z(struct pipe_context *pipe,
 598                 struct pipe_transfer *pt,
 599                 uint x, uint y, uint w, uint h,
 600                 const uint *zSrc)
 601 {
 602    const uint srcStride = w;
 603    const uint *ptrc = zSrc;
 604    ubyte *map;
 605    uint i, j;
 606    enum pipe_format format = pt->resource->format;
 607
 608    if (u_clip_tile(x, y, &w, &h, &pt->box))
 609       return;
 610
 611    map = (ubyte *)pipe->transfer_map(pipe, pt);
 612    if (!map) {
 613       assert(0);
 614       return;
 615    }
 616
 617    switch (format) {
 618    case PIPE_FORMAT_Z32_UNORM:
 619       {
 620          uint *pDest = (uint *) (map + y * pt->stride + x*4);
 621          for (i = 0; i < h; i++) {
 622             memcpy(pDest, ptrc, 4 * w);
 623             pDest += pt->stride/4;
 624             ptrc += srcStride;
 625          }
 626       }
 627       break;
 628    case PIPE_FORMAT_Z24_UNORM_S8_USCALED:
 629       {
 630          uint *pDest = (uint *) (map + y * pt->stride + x*4);
 631          /*assert((pt->usage & PIPE_TRANSFER_READ_WRITE) == PIPE_TRANSFER_READ_WRITE);*/
 632          for (i = 0; i < h; i++) {
 633             for (j = 0; j < w; j++) {
 634                /* convert 32-bit Z to 24-bit Z, preserve stencil */
 635                pDest[j] = (pDest[j] & 0xff000000) | ptrc[j] >> 8;
 636             }
 637             pDest += pt->stride/4;
 638             ptrc += srcStride;
 639          }
 640       }
 641       break;
 642    case PIPE_FORMAT_Z24X8_UNORM:
 643       {
 644          uint *pDest = (uint *) (map + y * pt->stride + x*4);
 645          for (i = 0; i < h; i++) {
 646             for (j = 0; j < w; j++) {
 647                /* convert 32-bit Z to 24-bit Z (0 stencil) */
 648                pDest[j] = ptrc[j] >> 8;
 649             }
 650             pDest += pt->stride/4;
 651             ptrc += srcStride;
 652          }
 653       }
 654       break;
 655    case PIPE_FORMAT_S8_USCALED_Z24_UNORM:
 656       {
 657          uint *pDest = (uint *) (map + y * pt->stride + x*4);
 658          /*assert((pt->usage & PIPE_TRANSFER_READ_WRITE) == PIPE_TRANSFER_READ_WRITE);*/
 659          for (i = 0; i < h; i++) {
 660             for (j = 0; j < w; j++) {
 661                /* convert 32-bit Z to 24-bit Z, preserve stencil */
 662                pDest[j] = (pDest[j] & 0xff) | (ptrc[j] & 0xffffff00);
 663             }
 664             pDest += pt->stride/4;
 665             ptrc += srcStride;
 666          }
 667       }
 668       break;
 669    case PIPE_FORMAT_X8Z24_UNORM:
 670       {
 671          uint *pDest = (uint *) (map + y * pt->stride + x*4);
 672          for (i = 0; i < h; i++) {
 673             for (j = 0; j < w; j++) {
 674                /* convert 32-bit Z to 24-bit Z (0 stencil) */
 675                pDest[j] = ptrc[j] & 0xffffff00;
 676             }
 677             pDest += pt->stride/4;
 678             ptrc += srcStride;
 679          }
 680       }
 681       break;
 682    case PIPE_FORMAT_Z16_UNORM:
 683       {
 684          ushort *pDest = (ushort *) (map + y * pt->stride + x*2);
 685          for (i = 0; i < h; i++) {
 686             for (j = 0; j < w; j++) {
 687                /* convert 32-bit Z to 16-bit Z */
 688                pDest[j] = ptrc[j] >> 16;
 689             }
 690             pDest += pt->stride/2;
 691             ptrc += srcStride;
 692          }
 693       }
 694       break;
 695    default:
 696       assert(0);
 697    }
 698
 699    pipe->transfer_unmap(pipe, pt);
 700 }
 701
 702