src/gallium/drivers/softpipe/sp_image.c

   1 /*
   2  * Copyright 2016 Red Hat.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * on the rights to use, copy, modify, merge, publish, distribute, sub
   8  * license, and/or sell copies of the Software, and to permit persons to whom
   9  * the Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  18  * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
  19  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  20  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  21  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  22  */
  23
  24 #include "sp_context.h"
  25 #include "sp_image.h"
  26 #include "sp_texture.h"
  27
  28 #include "util/format/u_format.h"
  29
  30 /*
  31  * Get the offset into the base image
  32  * first element for a buffer or layer/level for texture.
  33  */
  34 static uint32_t
  35 get_image_offset(const struct softpipe_resource *spr,
  36                  const struct pipe_image_view *iview,
  37                  enum pipe_format format, unsigned r_coord)
  38 {
  39    int base_layer = 0;
  40
  41    if (spr->base.target == PIPE_BUFFER)
  42       return iview->u.buf.offset;
  43
  44    if (spr->base.target == PIPE_TEXTURE_1D_ARRAY ||
  45        spr->base.target == PIPE_TEXTURE_2D_ARRAY ||
  46        spr->base.target == PIPE_TEXTURE_CUBE_ARRAY ||
  47        spr->base.target == PIPE_TEXTURE_CUBE ||
  48        spr->base.target == PIPE_TEXTURE_3D)
  49       base_layer = r_coord + iview->u.tex.first_layer;
  50    return softpipe_get_tex_image_offset(spr, iview->u.tex.level, base_layer);
  51 }
  52
  53 /*
  54  * Does this texture instruction have a layer or depth parameter.
  55  */
  56 static inline bool
  57 has_layer_or_depth(unsigned tgsi_tex_instr)
  58 {
  59    return (tgsi_tex_instr == TGSI_TEXTURE_3D ||
  60            tgsi_tex_instr == TGSI_TEXTURE_CUBE ||
  61            tgsi_tex_instr == TGSI_TEXTURE_1D_ARRAY ||
  62            tgsi_tex_instr == TGSI_TEXTURE_2D_ARRAY ||
  63            tgsi_tex_instr == TGSI_TEXTURE_CUBE_ARRAY ||
  64            tgsi_tex_instr == TGSI_TEXTURE_2D_ARRAY_MSAA);
  65 }
  66
  67 /*
  68  * Is this texture instruction a single non-array coordinate.
  69  */
  70 static inline bool
  71 has_1coord(unsigned tgsi_tex_instr)
  72 {
  73    return (tgsi_tex_instr == TGSI_TEXTURE_BUFFER ||
  74            tgsi_tex_instr == TGSI_TEXTURE_1D ||
  75            tgsi_tex_instr == TGSI_TEXTURE_1D_ARRAY);
  76 }
  77
  78 /*
  79  * check the bounds vs w/h/d
  80  */
  81 static inline bool
  82 bounds_check(int width, int height, int depth,
  83              int s, int t, int r)
  84 {
  85    if (s < 0 || s >= width)
  86       return false;
  87    if (t < 0 || t >= height)
  88       return false;
  89    if (r < 0 || r >= depth)
  90       return false;
  91    return true;
  92 }
  93
  94 /*
  95  * Checks if the texture target compatible with the image resource
  96  * pipe target.
  97  */
  98 static inline bool
  99 has_compat_target(unsigned pipe_target, unsigned tgsi_target)
 100 {
 101    switch (pipe_target) {
 102    case PIPE_TEXTURE_1D:
 103       if (tgsi_target == TGSI_TEXTURE_1D)
 104          return true;
 105       break;
 106    case PIPE_TEXTURE_2D:
 107       if (tgsi_target == TGSI_TEXTURE_2D)
 108          return true;
 109       break;
 110    case PIPE_TEXTURE_RECT:
 111       if (tgsi_target == TGSI_TEXTURE_RECT)
 112          return true;
 113       break;
 114    case PIPE_TEXTURE_3D:
 115       if (tgsi_target == TGSI_TEXTURE_3D ||
 116           tgsi_target == TGSI_TEXTURE_2D)
 117          return true;
 118       break;
 119    case PIPE_TEXTURE_CUBE:
 120       if (tgsi_target == TGSI_TEXTURE_CUBE ||
 121           tgsi_target == TGSI_TEXTURE_2D)
 122          return true;
 123       break;
 124    case PIPE_TEXTURE_1D_ARRAY:
 125       if (tgsi_target == TGSI_TEXTURE_1D ||
 126           tgsi_target == TGSI_TEXTURE_1D_ARRAY)
 127          return true;
 128       break;
 129    case PIPE_TEXTURE_2D_ARRAY:
 130       if (tgsi_target == TGSI_TEXTURE_2D ||
 131           tgsi_target == TGSI_TEXTURE_2D_ARRAY)
 132          return true;
 133       break;
 134    case PIPE_TEXTURE_CUBE_ARRAY:
 135       if (tgsi_target == TGSI_TEXTURE_CUBE ||
 136           tgsi_target == TGSI_TEXTURE_CUBE_ARRAY ||
 137           tgsi_target == TGSI_TEXTURE_2D)
 138          return true;
 139       break;
 140    case PIPE_BUFFER:
 141       return (tgsi_target == TGSI_TEXTURE_BUFFER);
 142    }
 143    return false;
 144 }
 145
 146 static bool
 147 get_dimensions(const struct pipe_image_view *iview,
 148                const struct softpipe_resource *spr,
 149                unsigned tgsi_tex_instr,
 150                enum pipe_format pformat,
 151                unsigned *width,
 152                unsigned *height,
 153                unsigned *depth)
 154 {
 155    if (tgsi_tex_instr == TGSI_TEXTURE_BUFFER) {
 156       *width = iview->u.buf.size / util_format_get_blocksize(pformat);
 157       *height = 1;
 158       *depth = 1;
 159       /*
 160        * Bounds check the buffer size from the view
 161        * and the buffer size from the underlying buffer.
 162        */
 163       if (util_format_get_stride(pformat, *width) >
 164           util_format_get_stride(spr->base.format, spr->base.width0))
 165          return false;
 166    } else {
 167       unsigned level;
 168
 169       level = spr->base.target == PIPE_BUFFER ? 0 : iview->u.tex.level;
 170       *width = u_minify(spr->base.width0, level);
 171       *height = u_minify(spr->base.height0, level);
 172
 173       if (spr->base.target == PIPE_TEXTURE_3D)
 174          *depth = u_minify(spr->base.depth0, level);
 175       else
 176          *depth = spr->base.array_size;
 177
 178       /* Make sure the resource and view have compatiable formats */
 179       if (util_format_get_blocksize(pformat) >
 180           util_format_get_blocksize(spr->base.format))
 181          return false;
 182    }
 183    return true;
 184 }
 185
 186 static void
 187 fill_coords(const struct tgsi_image_params *params,
 188             unsigned index,
 189             const int s[TGSI_QUAD_SIZE],
 190             const int t[TGSI_QUAD_SIZE],
 191             const int r[TGSI_QUAD_SIZE],
 192             int *s_coord, int *t_coord, int *r_coord)
 193 {
 194    *s_coord = s[index];
 195    *t_coord = has_1coord(params->tgsi_tex_instr) ? 0 : t[index];
 196    *r_coord = has_layer_or_depth(params->tgsi_tex_instr) ?
 197       (params->tgsi_tex_instr == TGSI_TEXTURE_1D_ARRAY ? t[index] : r[index]) : 0;
 198 }
 199 /*
 200  * Implement the image LOAD operation.
 201  */
 202 static void
 203 sp_tgsi_load(const struct tgsi_image *image,
 204              const struct tgsi_image_params *params,
 205              const int s[TGSI_QUAD_SIZE],
 206              const int t[TGSI_QUAD_SIZE],
 207              const int r[TGSI_QUAD_SIZE],
 208              const int sample[TGSI_QUAD_SIZE],
 209              float rgba[TGSI_NUM_CHANNELS][TGSI_QUAD_SIZE])
 210 {
 211    struct sp_tgsi_image *sp_img = (struct sp_tgsi_image *)image;
 212    struct pipe_image_view *iview;
 213    struct softpipe_resource *spr;
 214    unsigned width, height, depth;
 215    unsigned stride;
 216    int c, j;
 217    char *data_ptr;
 218    unsigned offset = 0;
 219
 220    if (params->unit >= PIPE_MAX_SHADER_IMAGES)
 221       goto fail_write_all_zero;
 222    iview = &sp_img->sp_iview[params->unit];
 223    spr = (struct softpipe_resource *)iview->resource;
 224    if (!spr)
 225       goto fail_write_all_zero;
 226
 227    if (!has_compat_target(spr->base.target, params->tgsi_tex_instr))
 228       goto fail_write_all_zero;
 229
 230    if (!get_dimensions(iview, spr, params->tgsi_tex_instr,
 231                        params->format, &width, &height, &depth))
 232       return;
 233
 234    stride = util_format_get_stride(params->format, width);
 235
 236    for (j = 0; j < TGSI_QUAD_SIZE; j++) {
 237       int s_coord, t_coord, r_coord;
 238       bool fill_zero = false;
 239
 240       if (!(params->execmask & (1 << j)))
 241          fill_zero = true;
 242
 243       fill_coords(params, j, s, t, r, &s_coord, &t_coord, &r_coord);
 244       if (!bounds_check(width, height, depth,
 245                         s_coord, t_coord, r_coord))
 246          fill_zero = true;
 247
 248       if (fill_zero) {
 249          int nc = util_format_get_nr_components(params->format);
 250          int ival = util_format_is_pure_integer(params->format);
 251          for (c = 0; c < 4; c++) {
 252             rgba[c][j] = 0;
 253             if (c == 3 && nc < 4) {
 254                if (ival)
 255                   ((int32_t *)rgba[c])[j] = 1;
 256                else
 257                   rgba[c][j] = 1.0;
 258             }
 259          }
 260          continue;
 261       }
 262       offset = get_image_offset(spr, iview, params->format, r_coord);
 263       data_ptr = (char *)spr->data + offset;
 264
 265       uint32_t sdata[4];
 266       util_format_read_4(params->format,
 267                          sdata, 0,
 268                          data_ptr, stride,
 269                          s_coord, t_coord, 1, 1);
 270       for (c = 0; c < 4; c++)
 271          ((uint32_t *)rgba[c])[j] = sdata[c];
 272    }
 273    return;
 274 fail_write_all_zero:
 275    for (j = 0; j < TGSI_QUAD_SIZE; j++) {
 276       for (c = 0; c < 4; c++)
 277          rgba[c][j] = 0;
 278    }
 279    return;
 280 }
 281
 282 /*
 283  * Implement the image STORE operation.
 284  */
 285 static void
 286 sp_tgsi_store(const struct tgsi_image *image,
 287               const struct tgsi_image_params *params,
 288               const int s[TGSI_QUAD_SIZE],
 289               const int t[TGSI_QUAD_SIZE],
 290               const int r[TGSI_QUAD_SIZE],
 291               const int sample[TGSI_QUAD_SIZE],
 292               float rgba[TGSI_NUM_CHANNELS][TGSI_QUAD_SIZE])
 293 {
 294    struct sp_tgsi_image *sp_img = (struct sp_tgsi_image *)image;
 295    struct pipe_image_view *iview;
 296    struct softpipe_resource *spr;
 297    unsigned width, height, depth;
 298    unsigned stride;
 299    char *data_ptr;
 300    int j, c;
 301    unsigned offset = 0;
 302    unsigned pformat = params->format;
 303
 304    if (params->unit >= PIPE_MAX_SHADER_IMAGES)
 305       return;
 306    iview = &sp_img->sp_iview[params->unit];
 307    spr = (struct softpipe_resource *)iview->resource;
 308    if (!spr)
 309       return;
 310    if (!has_compat_target(spr->base.target, params->tgsi_tex_instr))
 311       return;
 312
 313    if (params->format == PIPE_FORMAT_NONE)
 314       pformat = spr->base.format;
 315
 316    if (!get_dimensions(iview, spr, params->tgsi_tex_instr,
 317                        pformat, &width, &height, &depth))
 318       return;
 319
 320    stride = util_format_get_stride(pformat, width);
 321
 322    for (j = 0; j < TGSI_QUAD_SIZE; j++) {
 323       int s_coord, t_coord, r_coord;
 324
 325       if (!(params->execmask & (1 << j)))
 326          continue;
 327
 328       fill_coords(params, j, s, t, r, &s_coord, &t_coord, &r_coord);
 329       if (!bounds_check(width, height, depth,
 330                         s_coord, t_coord, r_coord))
 331          continue;
 332
 333       offset = get_image_offset(spr, iview, pformat, r_coord);
 334       data_ptr = (char *)spr->data + offset;
 335
 336       uint32_t sdata[4];
 337       for (c = 0; c < 4; c++)
 338          sdata[c] = ((uint32_t *)rgba[c])[j];
 339       util_format_write_4(pformat, sdata, 0, data_ptr, stride,
 340                           s_coord, t_coord, 1, 1);
 341    }
 342 }
 343
 344 /*
 345  * Implement atomic operations on unsigned integers.
 346  */
 347 static void
 348 handle_op_uint(const struct pipe_image_view *iview,
 349                const struct tgsi_image_params *params,
 350                bool just_read,
 351                char *data_ptr,
 352                uint qi,
 353                unsigned stride,
 354                enum tgsi_opcode opcode,
 355                int s,
 356                int t,
 357                float rgba[TGSI_NUM_CHANNELS][TGSI_QUAD_SIZE],
 358                float rgba2[TGSI_NUM_CHANNELS][TGSI_QUAD_SIZE])
 359 {
 360    uint c;
 361    int nc = util_format_get_nr_components(params->format);
 362    unsigned sdata[4];
 363
 364    util_format_read_4(params->format,
 365                       sdata, 0,
 366                       data_ptr, stride,
 367                       s, t, 1, 1);
 368
 369    if (just_read) {
 370       for (c = 0; c < nc; c++) {
 371          ((uint32_t *)rgba[c])[qi] = sdata[c];
 372       }
 373       return;
 374    }
 375    switch (opcode) {
 376    case TGSI_OPCODE_ATOMUADD:
 377       for (c = 0; c < nc; c++) {
 378          unsigned temp = sdata[c];
 379          sdata[c] += ((uint32_t *)rgba[c])[qi];
 380          ((uint32_t *)rgba[c])[qi] = temp;
 381       }
 382       break;
 383    case TGSI_OPCODE_ATOMXCHG:
 384       for (c = 0; c < nc; c++) {
 385          unsigned temp = sdata[c];
 386          sdata[c] = ((uint32_t *)rgba[c])[qi];
 387          ((uint32_t *)rgba[c])[qi] = temp;
 388       }
 389       break;
 390    case TGSI_OPCODE_ATOMCAS:
 391       for (c = 0; c < nc; c++) {
 392          unsigned dst_x = sdata[c];
 393          unsigned cmp_x = ((uint32_t *)rgba[c])[qi];
 394          unsigned src_x = ((uint32_t *)rgba2[c])[qi];
 395          unsigned temp = sdata[c];
 396          sdata[c] = (dst_x == cmp_x) ? src_x : dst_x;
 397          ((uint32_t *)rgba[c])[qi] = temp;
 398       }
 399       break;
 400    case TGSI_OPCODE_ATOMAND:
 401       for (c = 0; c < nc; c++) {
 402          unsigned temp = sdata[c];
 403          sdata[c] &= ((uint32_t *)rgba[c])[qi];
 404          ((uint32_t *)rgba[c])[qi] = temp;
 405       }
 406       break;
 407    case TGSI_OPCODE_ATOMOR:
 408       for (c = 0; c < nc; c++) {
 409          unsigned temp = sdata[c];
 410          sdata[c] |= ((uint32_t *)rgba[c])[qi];
 411          ((uint32_t *)rgba[c])[qi] = temp;
 412       }
 413       break;
 414    case TGSI_OPCODE_ATOMXOR:
 415       for (c = 0; c < nc; c++) {
 416          unsigned temp = sdata[c];
 417          sdata[c] ^= ((uint32_t *)rgba[c])[qi];
 418          ((uint32_t *)rgba[c])[qi] = temp;
 419       }
 420       break;
 421    case TGSI_OPCODE_ATOMUMIN:
 422       for (c = 0; c < nc; c++) {
 423          unsigned dst_x = sdata[c];
 424          unsigned src_x = ((uint32_t *)rgba[c])[qi];
 425          sdata[c] = MIN2(dst_x, src_x);
 426          ((uint32_t *)rgba[c])[qi] = dst_x;
 427       }
 428       break;
 429    case TGSI_OPCODE_ATOMUMAX:
 430       for (c = 0; c < nc; c++) {
 431          unsigned dst_x = sdata[c];
 432          unsigned src_x = ((uint32_t *)rgba[c])[qi];
 433          sdata[c] = MAX2(dst_x, src_x);
 434          ((uint32_t *)rgba[c])[qi] = dst_x;
 435       }
 436       break;
 437    case TGSI_OPCODE_ATOMIMIN:
 438       for (c = 0; c < nc; c++) {
 439          int dst_x = sdata[c];
 440          int src_x = ((uint32_t *)rgba[c])[qi];
 441          sdata[c] = MIN2(dst_x, src_x);
 442          ((uint32_t *)rgba[c])[qi] = dst_x;
 443       }
 444       break;
 445    case TGSI_OPCODE_ATOMIMAX:
 446       for (c = 0; c < nc; c++) {
 447          int dst_x = sdata[c];
 448          int src_x = ((uint32_t *)rgba[c])[qi];
 449          sdata[c] = MAX2(dst_x, src_x);
 450          ((uint32_t *)rgba[c])[qi] = dst_x;
 451       }
 452       break;
 453    default:
 454       assert(!"Unexpected TGSI opcode in sp_tgsi_op");
 455       break;
 456    }
 457    util_format_write_4(params->format, sdata, 0, data_ptr, stride,
 458                        s, t, 1, 1);
 459 }
 460
 461 /*
 462  * Implement atomic operations on signed integers.
 463  */
 464 static void
 465 handle_op_int(const struct pipe_image_view *iview,
 466               const struct tgsi_image_params *params,
 467               bool just_read,
 468               char *data_ptr,
 469               uint qi,
 470               unsigned stride,
 471               enum tgsi_opcode opcode,
 472               int s,
 473               int t,
 474               float rgba[TGSI_NUM_CHANNELS][TGSI_QUAD_SIZE],
 475               float rgba2[TGSI_NUM_CHANNELS][TGSI_QUAD_SIZE])
 476 {
 477    uint c;
 478    int nc = util_format_get_nr_components(params->format);
 479    int sdata[4];
 480    util_format_read_4(params->format,
 481                       sdata, 0,
 482                       data_ptr, stride,
 483                       s, t, 1, 1);
 484
 485    if (just_read) {
 486       for (c = 0; c < nc; c++) {
 487          ((int32_t *)rgba[c])[qi] = sdata[c];
 488       }
 489       return;
 490    }
 491    switch (opcode) {
 492    case TGSI_OPCODE_ATOMUADD:
 493       for (c = 0; c < nc; c++) {
 494          int temp = sdata[c];
 495          sdata[c] += ((int32_t *)rgba[c])[qi];
 496          ((int32_t *)rgba[c])[qi] = temp;
 497       }
 498       break;
 499    case TGSI_OPCODE_ATOMXCHG:
 500       for (c = 0; c < nc; c++) {
 501          int temp = sdata[c];
 502          sdata[c] = ((int32_t *)rgba[c])[qi];
 503          ((int32_t *)rgba[c])[qi] = temp;
 504       }
 505       break;
 506    case TGSI_OPCODE_ATOMCAS:
 507       for (c = 0; c < nc; c++) {
 508          int dst_x = sdata[c];
 509          int cmp_x = ((int32_t *)rgba[c])[qi];
 510          int src_x = ((int32_t *)rgba2[c])[qi];
 511          int temp = sdata[c];
 512          sdata[c] = (dst_x == cmp_x) ? src_x : dst_x;
 513          ((int32_t *)rgba[c])[qi] = temp;
 514       }
 515       break;
 516    case TGSI_OPCODE_ATOMAND:
 517       for (c = 0; c < nc; c++) {
 518          int temp = sdata[c];
 519          sdata[c] &= ((int32_t *)rgba[c])[qi];
 520          ((int32_t *)rgba[c])[qi] = temp;
 521       }
 522       break;
 523    case TGSI_OPCODE_ATOMOR:
 524       for (c = 0; c < nc; c++) {
 525          int temp = sdata[c];
 526          sdata[c] |= ((int32_t *)rgba[c])[qi];
 527          ((int32_t *)rgba[c])[qi] = temp;
 528       }
 529       break;
 530    case TGSI_OPCODE_ATOMXOR:
 531       for (c = 0; c < nc; c++) {
 532          int temp = sdata[c];
 533          sdata[c] ^= ((int32_t *)rgba[c])[qi];
 534          ((int32_t *)rgba[c])[qi] = temp;
 535       }
 536       break;
 537    case TGSI_OPCODE_ATOMUMIN:
 538       for (c = 0; c < nc; c++) {
 539          int dst_x = sdata[c];
 540          int src_x = ((int32_t *)rgba[c])[qi];
 541          sdata[c] = MIN2(dst_x, src_x);
 542          ((int32_t *)rgba[c])[qi] = dst_x;
 543       }
 544       break;
 545    case TGSI_OPCODE_ATOMUMAX:
 546       for (c = 0; c < nc; c++) {
 547          int dst_x = sdata[c];
 548          int src_x = ((int32_t *)rgba[c])[qi];
 549          sdata[c] = MAX2(dst_x, src_x);
 550          ((int32_t *)rgba[c])[qi] = dst_x;
 551       }
 552       break;
 553    case TGSI_OPCODE_ATOMIMIN:
 554       for (c = 0; c < nc; c++) {
 555          int dst_x = sdata[c];
 556          int src_x = ((int32_t *)rgba[c])[qi];
 557          sdata[c] = MIN2(dst_x, src_x);
 558          ((int32_t *)rgba[c])[qi] = dst_x;
 559       }
 560       break;
 561    case TGSI_OPCODE_ATOMIMAX:
 562       for (c = 0; c < nc; c++) {
 563          int dst_x = sdata[c];
 564          int src_x = ((int32_t *)rgba[c])[qi];
 565          sdata[c] = MAX2(dst_x, src_x);
 566          ((int32_t *)rgba[c])[qi] = dst_x;
 567       }
 568       break;
 569    default:
 570       assert(!"Unexpected TGSI opcode in sp_tgsi_op");
 571       break;
 572    }
 573    util_format_write_4(params->format, sdata, 0, data_ptr, stride,
 574                        s, t, 1, 1);
 575 }
 576
 577 /* GLES OES_shader_image_atomic.txt allows XCHG on R32F */
 578 static void
 579 handle_op_r32f_xchg(const struct pipe_image_view *iview,
 580                     const struct tgsi_image_params *params,
 581                     bool just_read,
 582                     char *data_ptr,
 583                     uint qi,
 584                     unsigned stride,
 585                     enum tgsi_opcode opcode,
 586                     int s,
 587                     int t,
 588                     float rgba[TGSI_NUM_CHANNELS][TGSI_QUAD_SIZE])
 589 {
 590    float sdata[4];
 591    uint c;
 592    int nc = 1;
 593    util_format_read_4(params->format,
 594                       sdata, 0,
 595                       data_ptr, stride,
 596                       s, t, 1, 1);
 597    if (just_read) {
 598       for (c = 0; c < nc; c++) {
 599          ((int32_t *)rgba[c])[qi] = sdata[c];
 600       }
 601       return;
 602    }
 603
 604    for (c = 0; c < nc; c++) {
 605       int temp = sdata[c];
 606       sdata[c] = ((float *)rgba[c])[qi];
 607       ((float *)rgba[c])[qi] = temp;
 608    }
 609    util_format_write_4(params->format, sdata, 0, data_ptr, stride,
 610                        s, t, 1, 1);
 611 }
 612
 613 /*
 614  * Implement atomic image operations.
 615  */
 616 static void
 617 sp_tgsi_op(const struct tgsi_image *image,
 618            const struct tgsi_image_params *params,
 619            enum tgsi_opcode opcode,
 620            const int s[TGSI_QUAD_SIZE],
 621            const int t[TGSI_QUAD_SIZE],
 622            const int r[TGSI_QUAD_SIZE],
 623            const int sample[TGSI_QUAD_SIZE],
 624            float rgba[TGSI_NUM_CHANNELS][TGSI_QUAD_SIZE],
 625            float rgba2[TGSI_NUM_CHANNELS][TGSI_QUAD_SIZE])
 626 {
 627    struct sp_tgsi_image *sp_img = (struct sp_tgsi_image *)image;
 628    struct pipe_image_view *iview;
 629    struct softpipe_resource *spr;
 630    unsigned width, height, depth;
 631    unsigned stride;
 632    int j, c;
 633    unsigned offset;
 634    char *data_ptr;
 635
 636    if (params->unit >= PIPE_MAX_SHADER_IMAGES)
 637       return;
 638    iview = &sp_img->sp_iview[params->unit];
 639    spr = (struct softpipe_resource *)iview->resource;
 640    if (!spr)
 641       goto fail_write_all_zero;
 642    if (!has_compat_target(spr->base.target, params->tgsi_tex_instr))
 643       goto fail_write_all_zero;
 644
 645    if (!get_dimensions(iview, spr, params->tgsi_tex_instr,
 646                        params->format, &width, &height, &depth))
 647       goto fail_write_all_zero;
 648
 649    stride = util_format_get_stride(spr->base.format, width);
 650
 651    for (j = 0; j < TGSI_QUAD_SIZE; j++) {
 652       int s_coord, t_coord, r_coord;
 653       bool just_read = false;
 654
 655       fill_coords(params, j, s, t, r, &s_coord, &t_coord, &r_coord);
 656       if (!bounds_check(width, height, depth,
 657                         s_coord, t_coord, r_coord)) {
 658          int nc = util_format_get_nr_components(params->format);
 659          int ival = util_format_is_pure_integer(params->format);
 660          int c;
 661          for (c = 0; c < 4; c++) {
 662             rgba[c][j] = 0;
 663             if (c == 3 && nc < 4) {
 664                if (ival)
 665                   ((int32_t *)rgba[c])[j] = 1;
 666                else
 667                   rgba[c][j] = 1.0;
 668             }
 669          }
 670          continue;
 671       }
 672
 673       /* just readback the value for atomic if execmask isn't set */
 674       if (!(params->execmask & (1 << j))) {
 675          just_read = true;
 676       }
 677
 678       offset = get_image_offset(spr, iview, params->format, r_coord);
 679       data_ptr = (char *)spr->data + offset;
 680
 681       /* we should see atomic operations on r32 formats */
 682       if (util_format_is_pure_uint(params->format))
 683          handle_op_uint(iview, params, just_read, data_ptr, j, stride,
 684                         opcode, s_coord, t_coord, rgba, rgba2);
 685       else if (util_format_is_pure_sint(params->format))
 686          handle_op_int(iview, params, just_read, data_ptr, j, stride,
 687                        opcode, s_coord, t_coord, rgba, rgba2);
 688       else if (params->format == PIPE_FORMAT_R32_FLOAT &&
 689                opcode == TGSI_OPCODE_ATOMXCHG)
 690          handle_op_r32f_xchg(iview, params, just_read, data_ptr, j, stride,
 691                              opcode, s_coord, t_coord, rgba);
 692       else
 693          assert(0);
 694    }
 695    return;
 696 fail_write_all_zero:
 697    for (j = 0; j < TGSI_QUAD_SIZE; j++) {
 698       for (c = 0; c < 4; c++)
 699          rgba[c][j] = 0;
 700    }
 701    return;
 702 }
 703
 704 static void
 705 sp_tgsi_get_dims(const struct tgsi_image *image,
 706                  const struct tgsi_image_params *params,
 707                  int dims[4])
 708 {
 709    struct sp_tgsi_image *sp_img = (struct sp_tgsi_image *)image;
 710    struct pipe_image_view *iview;
 711    struct softpipe_resource *spr;
 712    int level;
 713
 714    if (params->unit >= PIPE_MAX_SHADER_IMAGES)
 715       return;
 716    iview = &sp_img->sp_iview[params->unit];
 717    spr = (struct softpipe_resource *)iview->resource;
 718    if (!spr)
 719       return;
 720
 721    if (params->tgsi_tex_instr == TGSI_TEXTURE_BUFFER) {
 722       dims[0] = iview->u.buf.size / util_format_get_blocksize(iview->format);
 723       dims[1] = dims[2] = dims[3] = 0;
 724       return;
 725    }
 726
 727    level = iview->u.tex.level;
 728    dims[0] = u_minify(spr->base.width0, level);
 729    switch (params->tgsi_tex_instr) {
 730    case TGSI_TEXTURE_1D_ARRAY:
 731       dims[1] = iview->u.tex.last_layer - iview->u.tex.first_layer + 1;
 732       /* fallthrough */
 733    case TGSI_TEXTURE_1D:
 734       return;
 735    case TGSI_TEXTURE_2D_ARRAY:
 736       dims[2] = iview->u.tex.last_layer - iview->u.tex.first_layer + 1;
 737       /* fallthrough */
 738    case TGSI_TEXTURE_2D:
 739    case TGSI_TEXTURE_CUBE:
 740    case TGSI_TEXTURE_RECT:
 741       dims[1] = u_minify(spr->base.height0, level);
 742       return;
 743    case TGSI_TEXTURE_3D:
 744       dims[1] = u_minify(spr->base.height0, level);
 745       dims[2] = u_minify(spr->base.depth0, level);
 746       return;
 747    case TGSI_TEXTURE_CUBE_ARRAY:
 748       dims[1] = u_minify(spr->base.height0, level);
 749       dims[2] = (iview->u.tex.last_layer - iview->u.tex.first_layer + 1) / 6;
 750       break;
 751    default:
 752       assert(!"unexpected texture target in sp_get_dims()");
 753       return;
 754    }
 755 }
 756
 757 struct sp_tgsi_image *
 758 sp_create_tgsi_image(void)
 759 {
 760    struct sp_tgsi_image *img = CALLOC_STRUCT(sp_tgsi_image);
 761    if (!img)
 762       return NULL;
 763
 764    img->base.load = sp_tgsi_load;
 765    img->base.store = sp_tgsi_store;
 766    img->base.op = sp_tgsi_op;
 767    img->base.get_dims = sp_tgsi_get_dims;
 768    return img;
 769 };