src/mesa/state_tracker/st_pbo.c

   1 /*
   2  * Copyright 2007 VMware, Inc.
   3  * Copyright 2016 Advanced Micro Devices, Inc.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * on the rights to use, copy, modify, merge, publish, distribute, sub
   9  * license, and/or sell copies of the Software, and to permit persons to whom
  10  * the Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  19  * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
  20  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  21  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  22  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  23  */
  24
  25 /**
  26  * \file
  27  *
  28  * Common helper functions for PBO up- and downloads.
  29  */
  30
  31 #include "state_tracker/st_context.h"
  32 #include "state_tracker/st_nir.h"
  33 #include "state_tracker/st_pbo.h"
  34 #include "state_tracker/st_cb_bufferobjects.h"
  35
  36 #include "pipe/p_context.h"
  37 #include "pipe/p_defines.h"
  38 #include "pipe/p_screen.h"
  39 #include "cso_cache/cso_context.h"
  40 #include "tgsi/tgsi_ureg.h"
  41 #include "util/format/u_format.h"
  42 #include "util/u_inlines.h"
  43 #include "util/u_upload_mgr.h"
  44
  45 #include "compiler/nir/nir_builder.h"
  46
  47 /* Conversion to apply in the fragment shader. */
  48 enum st_pbo_conversion {
  49    ST_PBO_CONVERT_NONE = 0,
  50    ST_PBO_CONVERT_UINT_TO_SINT,
  51    ST_PBO_CONVERT_SINT_TO_UINT,
  52
  53    ST_NUM_PBO_CONVERSIONS
  54 };
  55
  56 /* Final setup of buffer addressing information.
  57  *
  58  * buf_offset is in pixels.
  59  *
  60  * Returns false if something (e.g. alignment) prevents PBO upload/download.
  61  */
  62 bool
  63 st_pbo_addresses_setup(struct st_context *st,
  64                        struct pipe_resource *buf, intptr_t buf_offset,
  65                        struct st_pbo_addresses *addr)
  66 {
  67    unsigned skip_pixels;
  68
  69    /* Check alignment against texture buffer requirements. */
  70    {
  71       unsigned ofs = (buf_offset * addr->bytes_per_pixel) % st->ctx->Const.TextureBufferOffsetAlignment;
  72       if (ofs != 0) {
  73          if (ofs % addr->bytes_per_pixel != 0)
  74             return false;
  75
  76          skip_pixels = ofs / addr->bytes_per_pixel;
  77          buf_offset -= skip_pixels;
  78       } else {
  79          skip_pixels = 0;
  80       }
  81    }
  82
  83    assert(buf_offset >= 0);
  84
  85    addr->buffer = buf;
  86    addr->first_element = buf_offset;
  87    addr->last_element = buf_offset + skip_pixels + addr->width - 1
  88          + (addr->height - 1 + (addr->depth - 1) * addr->image_height) * addr->pixels_per_row;
  89
  90    if (addr->last_element - addr->first_element > st->ctx->Const.MaxTextureBufferSize - 1)
  91       return false;
  92
  93    /* This should be ensured by Mesa before calling our callbacks */
  94    assert((addr->last_element + 1) * addr->bytes_per_pixel <= buf->width0);
  95
  96    addr->constants.xoffset = -addr->xoffset + skip_pixels;
  97    addr->constants.yoffset = -addr->yoffset;
  98    addr->constants.stride = addr->pixels_per_row;
  99    addr->constants.image_size = addr->pixels_per_row * addr->image_height;
 100    addr->constants.layer_offset = 0;
 101
 102    return true;
 103 }
 104
 105 /* Validate and fill buffer addressing information based on GL pixelstore
 106  * attributes.
 107  *
 108  * Returns false if some aspect of the addressing (e.g. alignment) prevents
 109  * PBO upload/download.
 110  */
 111 bool
 112 st_pbo_addresses_pixelstore(struct st_context *st,
 113                             GLenum gl_target, bool skip_images,
 114                             const struct gl_pixelstore_attrib *store,
 115                             const void *pixels,
 116                             struct st_pbo_addresses *addr)
 117 {
 118    struct pipe_resource *buf = st_buffer_object(store->BufferObj)->buffer;
 119    intptr_t buf_offset = (intptr_t) pixels;
 120
 121    if (buf_offset % addr->bytes_per_pixel)
 122       return false;
 123
 124    /* Convert to texels */
 125    buf_offset = buf_offset / addr->bytes_per_pixel;
 126
 127    /* Determine image height */
 128    if (gl_target == GL_TEXTURE_1D_ARRAY) {
 129       addr->image_height = 1;
 130    } else {
 131       addr->image_height = store->ImageHeight > 0 ? store->ImageHeight : addr->height;
 132    }
 133
 134    /* Compute the stride, taking store->Alignment into account */
 135    {
 136        unsigned pixels_per_row = store->RowLength > 0 ?
 137                            store->RowLength : addr->width;
 138        unsigned bytes_per_row = pixels_per_row * addr->bytes_per_pixel;
 139        unsigned remainder = bytes_per_row % store->Alignment;
 140        unsigned offset_rows;
 141
 142        if (remainder > 0)
 143           bytes_per_row += store->Alignment - remainder;
 144
 145        if (bytes_per_row % addr->bytes_per_pixel)
 146           return false;
 147
 148        addr->pixels_per_row = bytes_per_row / addr->bytes_per_pixel;
 149
 150        offset_rows = store->SkipRows;
 151        if (skip_images)
 152           offset_rows += addr->image_height * store->SkipImages;
 153
 154        buf_offset += store->SkipPixels + addr->pixels_per_row * offset_rows;
 155    }
 156
 157    if (!st_pbo_addresses_setup(st, buf, buf_offset, addr))
 158       return false;
 159
 160    /* Support GL_PACK_INVERT_MESA */
 161    if (store->Invert) {
 162       addr->constants.xoffset += (addr->height - 1) * addr->constants.stride;
 163       addr->constants.stride = -addr->constants.stride;
 164    }
 165
 166    return true;
 167 }
 168
 169 /* For download from a framebuffer, we may have to invert the Y axis. The
 170  * setup is as follows:
 171  * - set viewport to inverted, so that the position sysval is correct for
 172  *   texel fetches
 173  * - this function adjusts the fragment shader's constant buffer to compute
 174  *   the correct destination addresses.
 175  */
 176 void
 177 st_pbo_addresses_invert_y(struct st_pbo_addresses *addr,
 178                           unsigned viewport_height)
 179 {
 180    addr->constants.xoffset +=
 181       (viewport_height - 1 + 2 * addr->constants.yoffset) * addr->constants.stride;
 182    addr->constants.stride = -addr->constants.stride;
 183 }
 184
 185 /* Setup all vertex pipeline state, rasterizer state, and fragment shader
 186  * constants, and issue the draw call for PBO upload/download.
 187  *
 188  * The caller is responsible for saving and restoring state, as well as for
 189  * setting other fragment shader state (fragment shader, samplers), and
 190  * framebuffer/viewport/DSA/blend state.
 191  */
 192 bool
 193 st_pbo_draw(struct st_context *st, const struct st_pbo_addresses *addr,
 194             unsigned surface_width, unsigned surface_height)
 195 {
 196    struct cso_context *cso = st->cso_context;
 197
 198    /* Setup vertex and geometry shaders */
 199    if (!st->pbo.vs) {
 200       st->pbo.vs = st_pbo_create_vs(st);
 201       if (!st->pbo.vs)
 202          return false;
 203    }
 204
 205    if (addr->depth != 1 && st->pbo.use_gs && !st->pbo.gs) {
 206       st->pbo.gs = st_pbo_create_gs(st);
 207       if (!st->pbo.gs)
 208          return false;
 209    }
 210
 211    cso_set_vertex_shader_handle(cso, st->pbo.vs);
 212
 213    cso_set_geometry_shader_handle(cso, addr->depth != 1 ? st->pbo.gs : NULL);
 214
 215    cso_set_tessctrl_shader_handle(cso, NULL);
 216
 217    cso_set_tesseval_shader_handle(cso, NULL);
 218
 219    /* Upload vertices */
 220    {
 221       struct pipe_vertex_buffer vbo = {0};
 222       struct cso_velems_state velem;
 223
 224       float x0 = (float) addr->xoffset / surface_width * 2.0f - 1.0f;
 225       float y0 = (float) addr->yoffset / surface_height * 2.0f - 1.0f;
 226       float x1 = (float) (addr->xoffset + addr->width) / surface_width * 2.0f - 1.0f;
 227       float y1 = (float) (addr->yoffset + addr->height) / surface_height * 2.0f - 1.0f;
 228
 229       float *verts = NULL;
 230
 231       vbo.stride = 2 * sizeof(float);
 232
 233       u_upload_alloc(st->pipe->stream_uploader, 0, 8 * sizeof(float), 4,
 234                      &vbo.buffer_offset, &vbo.buffer.resource, (void **) &verts);
 235       if (!verts)
 236          return false;
 237
 238       verts[0] = x0;
 239       verts[1] = y0;
 240       verts[2] = x0;
 241       verts[3] = y1;
 242       verts[4] = x1;
 243       verts[5] = y0;
 244       verts[6] = x1;
 245       verts[7] = y1;
 246
 247       u_upload_unmap(st->pipe->stream_uploader);
 248
 249       velem.count = 1;
 250       velem.velems[0].src_offset = 0;
 251       velem.velems[0].instance_divisor = 0;
 252       velem.velems[0].vertex_buffer_index = 0;
 253       velem.velems[0].src_format = PIPE_FORMAT_R32G32_FLOAT;
 254
 255       cso_set_vertex_elements(cso, &velem);
 256
 257       cso_set_vertex_buffers(cso, 0, 1, &vbo);
 258
 259       pipe_resource_reference(&vbo.buffer.resource, NULL);
 260    }
 261
 262    /* Upload constants */
 263    {
 264       struct pipe_constant_buffer cb;
 265
 266       cb.buffer = NULL;
 267       cb.user_buffer = &addr->constants;
 268       cb.buffer_offset = 0;
 269       cb.buffer_size = sizeof(addr->constants);
 270
 271       cso_set_constant_buffer(cso, PIPE_SHADER_FRAGMENT, 0, &cb);
 272
 273       pipe_resource_reference(&cb.buffer, NULL);
 274    }
 275
 276    /* Rasterizer state */
 277    cso_set_rasterizer(cso, &st->pbo.raster);
 278
 279    /* Disable stream output */
 280    cso_set_stream_outputs(cso, 0, NULL, 0);
 281
 282    if (addr->depth == 1) {
 283       cso_draw_arrays(cso, PIPE_PRIM_TRIANGLE_STRIP, 0, 4);
 284    } else {
 285       cso_draw_arrays_instanced(cso, PIPE_PRIM_TRIANGLE_STRIP,
 286                                 0, 4, 0, addr->depth);
 287    }
 288
 289    return true;
 290 }
 291
 292 void *
 293 st_pbo_create_vs(struct st_context *st)
 294 {
 295    struct pipe_screen *pscreen = st->pipe->screen;
 296    bool use_nir = PIPE_SHADER_IR_NIR ==
 297       pscreen->get_shader_param(pscreen, PIPE_SHADER_VERTEX,
 298                                 PIPE_SHADER_CAP_PREFERRED_IR);
 299
 300    if (use_nir) {
 301       unsigned inputs[] =  {  VERT_ATTRIB_POS, SYSTEM_VALUE_INSTANCE_ID, };
 302       unsigned outputs[] = { VARYING_SLOT_POS,       VARYING_SLOT_LAYER  };
 303
 304       return st_nir_make_passthrough_shader(st, "st/pbo VS",
 305                                             MESA_SHADER_VERTEX,
 306                                             st->pbo.layers ? 2 : 1,
 307                                             inputs, outputs, NULL, (1 << 1));
 308    }
 309
 310    struct ureg_program *ureg;
 311    struct ureg_src in_pos;
 312    struct ureg_src in_instanceid;
 313    struct ureg_dst out_pos;
 314    struct ureg_dst out_layer;
 315
 316    ureg = ureg_create(PIPE_SHADER_VERTEX);
 317    if (!ureg)
 318       return NULL;
 319
 320    in_pos = ureg_DECL_vs_input(ureg, TGSI_SEMANTIC_POSITION);
 321
 322    out_pos = ureg_DECL_output(ureg, TGSI_SEMANTIC_POSITION, 0);
 323
 324    if (st->pbo.layers) {
 325       in_instanceid = ureg_DECL_system_value(ureg, TGSI_SEMANTIC_INSTANCEID, 0);
 326
 327       if (!st->pbo.use_gs)
 328          out_layer = ureg_DECL_output(ureg, TGSI_SEMANTIC_LAYER, 0);
 329    }
 330
 331    /* out_pos = in_pos */
 332    ureg_MOV(ureg, out_pos, in_pos);
 333
 334    if (st->pbo.layers) {
 335       if (st->pbo.use_gs) {
 336          /* out_pos.z = i2f(gl_InstanceID) */
 337          ureg_I2F(ureg, ureg_writemask(out_pos, TGSI_WRITEMASK_Z),
 338                         ureg_scalar(in_instanceid, TGSI_SWIZZLE_X));
 339       } else {
 340          /* out_layer = gl_InstanceID */
 341          ureg_MOV(ureg, ureg_writemask(out_layer, TGSI_WRITEMASK_X),
 342                         ureg_scalar(in_instanceid, TGSI_SWIZZLE_X));
 343       }
 344    }
 345
 346    ureg_END(ureg);
 347
 348    return ureg_create_shader_and_destroy(ureg, st->pipe);
 349 }
 350
 351 void *
 352 st_pbo_create_gs(struct st_context *st)
 353 {
 354    static const int zero = 0;
 355    struct ureg_program *ureg;
 356    struct ureg_dst out_pos;
 357    struct ureg_dst out_layer;
 358    struct ureg_src in_pos;
 359    struct ureg_src imm;
 360    unsigned i;
 361
 362    ureg = ureg_create(PIPE_SHADER_GEOMETRY);
 363    if (!ureg)
 364       return NULL;
 365
 366    ureg_property(ureg, TGSI_PROPERTY_GS_INPUT_PRIM, PIPE_PRIM_TRIANGLES);
 367    ureg_property(ureg, TGSI_PROPERTY_GS_OUTPUT_PRIM, PIPE_PRIM_TRIANGLE_STRIP);
 368    ureg_property(ureg, TGSI_PROPERTY_GS_MAX_OUTPUT_VERTICES, 3);
 369
 370    out_pos = ureg_DECL_output(ureg, TGSI_SEMANTIC_POSITION, 0);
 371    out_layer = ureg_DECL_output(ureg, TGSI_SEMANTIC_LAYER, 0);
 372
 373    in_pos = ureg_DECL_input(ureg, TGSI_SEMANTIC_POSITION, 0, 0, 1);
 374
 375    imm = ureg_DECL_immediate_int(ureg, &zero, 1);
 376
 377    for (i = 0; i < 3; ++i) {
 378       struct ureg_src in_pos_vertex = ureg_src_dimension(in_pos, i);
 379
 380       /* out_pos = in_pos[i] */
 381       ureg_MOV(ureg, out_pos, in_pos_vertex);
 382
 383       /* out_layer.x = f2i(in_pos[i].z) */
 384       ureg_F2I(ureg, ureg_writemask(out_layer, TGSI_WRITEMASK_X),
 385                      ureg_scalar(in_pos_vertex, TGSI_SWIZZLE_Z));
 386
 387       ureg_EMIT(ureg, ureg_scalar(imm, TGSI_SWIZZLE_X));
 388    }
 389
 390    ureg_END(ureg);
 391
 392    return ureg_create_shader_and_destroy(ureg, st->pipe);
 393 }
 394
 395 static void
 396 build_conversion(struct ureg_program *ureg, const struct ureg_dst *temp,
 397                  enum st_pbo_conversion conversion)
 398 {
 399    switch (conversion) {
 400    case ST_PBO_CONVERT_SINT_TO_UINT:
 401       ureg_IMAX(ureg, *temp, ureg_src(*temp), ureg_imm1i(ureg, 0));
 402       break;
 403    case ST_PBO_CONVERT_UINT_TO_SINT:
 404       ureg_UMIN(ureg, *temp, ureg_src(*temp), ureg_imm1u(ureg, (1u << 31) - 1));
 405       break;
 406    default:
 407       /* no-op */
 408       break;
 409    }
 410 }
 411
 412 static const struct glsl_type *
 413 sampler_type_for_target(enum pipe_texture_target target)
 414 {
 415    bool is_array = target >= PIPE_TEXTURE_1D_ARRAY;
 416    static const enum glsl_sampler_dim dim[] = {
 417       [PIPE_BUFFER]             = GLSL_SAMPLER_DIM_BUF,
 418       [PIPE_TEXTURE_1D]         = GLSL_SAMPLER_DIM_1D,
 419       [PIPE_TEXTURE_2D]         = GLSL_SAMPLER_DIM_2D,
 420       [PIPE_TEXTURE_3D]         = GLSL_SAMPLER_DIM_3D,
 421       [PIPE_TEXTURE_CUBE]       = GLSL_SAMPLER_DIM_CUBE,
 422       [PIPE_TEXTURE_RECT]       = GLSL_SAMPLER_DIM_RECT,
 423       [PIPE_TEXTURE_1D_ARRAY]   = GLSL_SAMPLER_DIM_1D,
 424       [PIPE_TEXTURE_2D_ARRAY]   = GLSL_SAMPLER_DIM_2D,
 425       [PIPE_TEXTURE_CUBE_ARRAY] = GLSL_SAMPLER_DIM_CUBE,
 426    };
 427
 428    return glsl_sampler_type(dim[target], false, is_array, GLSL_TYPE_FLOAT);
 429 }
 430
 431 static void *
 432 create_fs_nir(struct st_context *st,
 433               bool download,
 434               enum pipe_texture_target target,
 435               enum st_pbo_conversion conversion)
 436 {
 437    struct pipe_screen *screen = st->pipe->screen;
 438    struct nir_builder b;
 439    const nir_shader_compiler_options *options =
 440       st->ctx->Const.ShaderCompilerOptions[MESA_SHADER_FRAGMENT].NirOptions;
 441    bool pos_is_sysval =
 442       screen->get_param(screen, PIPE_CAP_TGSI_FS_POSITION_IS_SYSVAL);
 443
 444    nir_builder_init_simple_shader(&b, NULL, MESA_SHADER_FRAGMENT, options);
 445
 446    nir_ssa_def *zero = nir_imm_int(&b, 0);
 447
 448    /* param = [ -xoffset + skip_pixels, -yoffset, stride, image_height ] */
 449    nir_variable *param_var =
 450       nir_variable_create(b.shader, nir_var_uniform, glsl_vec4_type(), "param");
 451    b.shader->num_uniforms += 4;
 452    nir_ssa_def *param = nir_load_var(&b, param_var);
 453
 454    nir_variable *fragcoord =
 455       nir_variable_create(b.shader, pos_is_sysval ? nir_var_system_value :
 456                           nir_var_shader_in, glsl_vec4_type(), "gl_FragCoord");
 457    fragcoord->data.location = pos_is_sysval ? SYSTEM_VALUE_FRAG_COORD
 458                                             : VARYING_SLOT_POS;
 459    nir_ssa_def *coord = nir_load_var(&b, fragcoord);
 460
 461    nir_ssa_def *layer = NULL;
 462    if (st->pbo.layers && (!download || target == PIPE_TEXTURE_1D_ARRAY ||
 463                                        target == PIPE_TEXTURE_2D_ARRAY ||
 464                                        target == PIPE_TEXTURE_3D ||
 465                                        target == PIPE_TEXTURE_CUBE ||
 466                                        target == PIPE_TEXTURE_CUBE_ARRAY)) {
 467       nir_variable *var = nir_variable_create(b.shader, nir_var_shader_in,
 468                                               glsl_int_type(), "gl_Layer");
 469       var->data.location = VARYING_SLOT_LAYER;
 470       var->data.interpolation = INTERP_MODE_FLAT;
 471       layer = nir_load_var(&b, var);
 472    }
 473
 474    /* offset_pos = param.xy + f2i(coord.xy) */
 475    nir_ssa_def *offset_pos =
 476       nir_iadd(&b, nir_channels(&b, param, TGSI_WRITEMASK_XY),
 477                nir_f2i32(&b, nir_channels(&b, coord, TGSI_WRITEMASK_XY)));
 478
 479    /* addr = offset_pos.x + offset_pos.y * stride */
 480    nir_ssa_def *pbo_addr =
 481       nir_iadd(&b, nir_channel(&b, offset_pos, 0),
 482                nir_imul(&b, nir_channel(&b, offset_pos, 1),
 483                         nir_channel(&b, param, 2)));
 484    if (layer) {
 485       /* pbo_addr += image_height * layer */
 486       pbo_addr = nir_iadd(&b, pbo_addr,
 487                           nir_imul(&b, layer, nir_channel(&b, param, 3)));
 488    }
 489
 490    nir_ssa_def *texcoord;
 491    if (download) {
 492       texcoord = nir_f2i32(&b, nir_channels(&b, coord, TGSI_WRITEMASK_XY));
 493
 494       if (layer) {
 495          nir_ssa_def *src_layer = layer;
 496
 497          if (target == PIPE_TEXTURE_3D) {
 498             nir_variable *layer_offset_var =
 499                nir_variable_create(b.shader, nir_var_uniform,
 500                                    glsl_int_type(), "layer_offset");
 501             b.shader->num_uniforms += 1;
 502             layer_offset_var->data.driver_location = 4;
 503             nir_ssa_def *layer_offset = nir_load_var(&b, layer_offset_var);
 504
 505             src_layer = nir_iadd(&b, layer, layer_offset);
 506          }
 507
 508          texcoord = nir_vec3(&b, nir_channel(&b, texcoord, 0),
 509                                  nir_channel(&b, texcoord, 1),
 510                                  src_layer);
 511       }
 512    } else {
 513       texcoord = pbo_addr;
 514    }
 515
 516    nir_variable *tex_var =
 517       nir_variable_create(b.shader, nir_var_uniform,
 518                           sampler_type_for_target(target), "tex");
 519    tex_var->data.explicit_binding = true;
 520    tex_var->data.binding = 0;
 521
 522    nir_deref_instr *tex_deref = nir_build_deref_var(&b, tex_var);
 523
 524    nir_tex_instr *tex = nir_tex_instr_create(b.shader, 3);
 525    tex->op = nir_texop_txf;
 526    tex->sampler_dim = glsl_get_sampler_dim(tex_var->type);
 527    tex->coord_components =
 528       glsl_get_sampler_coordinate_components(tex_var->type);
 529    tex->dest_type = nir_type_float;
 530    tex->src[0].src_type = nir_tex_src_texture_deref;
 531    tex->src[0].src = nir_src_for_ssa(&tex_deref->dest.ssa);
 532    tex->src[1].src_type = nir_tex_src_sampler_deref;
 533    tex->src[1].src = nir_src_for_ssa(&tex_deref->dest.ssa);
 534    tex->src[2].src_type = nir_tex_src_coord;
 535    tex->src[2].src = nir_src_for_ssa(texcoord);
 536    nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, NULL);
 537    nir_builder_instr_insert(&b, &tex->instr);
 538    nir_ssa_def *result = &tex->dest.ssa;
 539
 540    if (conversion == ST_PBO_CONVERT_SINT_TO_UINT)
 541       result = nir_imax(&b, result, zero);
 542    else if (conversion == ST_PBO_CONVERT_UINT_TO_SINT)
 543       result = nir_umin(&b, result, nir_imm_int(&b, (1u << 31) - 1));
 544
 545    if (download) {
 546       nir_variable *img_var =
 547          nir_variable_create(b.shader, nir_var_uniform,
 548                              glsl_image_type(GLSL_SAMPLER_DIM_BUF, false,
 549                                              GLSL_TYPE_FLOAT), "img");
 550       img_var->data.access = ACCESS_NON_READABLE;
 551       img_var->data.explicit_binding = true;
 552       img_var->data.binding = 0;
 553       nir_deref_instr *img_deref = nir_build_deref_var(&b, img_var);
 554       nir_intrinsic_instr *intrin =
 555          nir_intrinsic_instr_create(b.shader, nir_intrinsic_image_deref_store);
 556       intrin->src[0] = nir_src_for_ssa(&img_deref->dest.ssa);
 557       intrin->src[1] =
 558          nir_src_for_ssa(nir_vec4(&b, pbo_addr, zero, zero, zero));
 559       intrin->src[2] = nir_src_for_ssa(zero);
 560       intrin->src[3] = nir_src_for_ssa(result);
 561       intrin->src[4] = nir_src_for_ssa(nir_imm_int(&b, 0));
 562       intrin->num_components = 4;
 563       nir_builder_instr_insert(&b, &intrin->instr);
 564    } else {
 565       nir_variable *color =
 566          nir_variable_create(b.shader, nir_var_shader_out, glsl_vec4_type(),
 567                              "gl_FragColor");
 568       color->data.location = FRAG_RESULT_COLOR;
 569
 570       nir_store_var(&b, color, result, TGSI_WRITEMASK_XYZW);
 571    }
 572
 573    return st_nir_finish_builtin_shader(st, b.shader, download ?
 574                                        "st/pbo download FS" :
 575                                        "st/pbo upload FS");
 576 }
 577
 578 static void *
 579 create_fs_tgsi(struct st_context *st, bool download,
 580                enum pipe_texture_target target,
 581                enum st_pbo_conversion conversion)
 582 {
 583    struct pipe_context *pipe = st->pipe;
 584    struct pipe_screen *screen = pipe->screen;
 585    struct ureg_program *ureg;
 586    bool have_layer;
 587    struct ureg_dst out;
 588    struct ureg_src sampler;
 589    struct ureg_src pos;
 590    struct ureg_src layer;
 591    struct ureg_src const0;
 592    struct ureg_src const1;
 593    struct ureg_dst temp0;
 594
 595    have_layer =
 596       st->pbo.layers &&
 597       (!download || target == PIPE_TEXTURE_1D_ARRAY
 598                  || target == PIPE_TEXTURE_2D_ARRAY
 599                  || target == PIPE_TEXTURE_3D
 600                  || target == PIPE_TEXTURE_CUBE
 601                  || target == PIPE_TEXTURE_CUBE_ARRAY);
 602
 603    ureg = ureg_create(PIPE_SHADER_FRAGMENT);
 604    if (!ureg)
 605       return NULL;
 606
 607    if (!download) {
 608       out = ureg_DECL_output(ureg, TGSI_SEMANTIC_COLOR, 0);
 609    } else {
 610       struct ureg_src image;
 611
 612       /* writeonly images do not require an explicitly given format. */
 613       image = ureg_DECL_image(ureg, 0, TGSI_TEXTURE_BUFFER, PIPE_FORMAT_NONE,
 614                                     true, false);
 615       out = ureg_dst(image);
 616    }
 617
 618    sampler = ureg_DECL_sampler(ureg, 0);
 619    if (screen->get_param(screen, PIPE_CAP_TGSI_FS_POSITION_IS_SYSVAL)) {
 620       pos = ureg_DECL_system_value(ureg, TGSI_SEMANTIC_POSITION, 0);
 621    } else {
 622       pos = ureg_DECL_fs_input(ureg, TGSI_SEMANTIC_POSITION, 0,
 623                                TGSI_INTERPOLATE_LINEAR);
 624    }
 625    if (have_layer) {
 626       layer = ureg_DECL_fs_input(ureg, TGSI_SEMANTIC_LAYER, 0,
 627                                        TGSI_INTERPOLATE_CONSTANT);
 628    }
 629    const0  = ureg_DECL_constant(ureg, 0);
 630    const1  = ureg_DECL_constant(ureg, 1);
 631    temp0   = ureg_DECL_temporary(ureg);
 632
 633    /* Note: const0 = [ -xoffset + skip_pixels, -yoffset, stride, image_height ] */
 634
 635    /* temp0.xy = f2i(temp0.xy) */
 636    ureg_F2I(ureg, ureg_writemask(temp0, TGSI_WRITEMASK_XY),
 637                   ureg_swizzle(pos,
 638                                TGSI_SWIZZLE_X, TGSI_SWIZZLE_Y,
 639                                TGSI_SWIZZLE_Y, TGSI_SWIZZLE_Y));
 640
 641    /* temp0.xy = temp0.xy + const0.xy */
 642    ureg_UADD(ureg, ureg_writemask(temp0, TGSI_WRITEMASK_XY),
 643                    ureg_swizzle(ureg_src(temp0),
 644                                 TGSI_SWIZZLE_X, TGSI_SWIZZLE_Y,
 645                                 TGSI_SWIZZLE_Y, TGSI_SWIZZLE_Y),
 646                    ureg_swizzle(const0,
 647                                 TGSI_SWIZZLE_X, TGSI_SWIZZLE_Y,
 648                                 TGSI_SWIZZLE_Y, TGSI_SWIZZLE_Y));
 649
 650    /* temp0.x = const0.z * temp0.y + temp0.x */
 651    ureg_UMAD(ureg, ureg_writemask(temp0, TGSI_WRITEMASK_X),
 652                    ureg_scalar(const0, TGSI_SWIZZLE_Z),
 653                    ureg_scalar(ureg_src(temp0), TGSI_SWIZZLE_Y),
 654                    ureg_scalar(ureg_src(temp0), TGSI_SWIZZLE_X));
 655
 656    if (have_layer) {
 657       /* temp0.x = const0.w * layer + temp0.x */
 658       ureg_UMAD(ureg, ureg_writemask(temp0, TGSI_WRITEMASK_X),
 659                       ureg_scalar(const0, TGSI_SWIZZLE_W),
 660                       ureg_scalar(layer, TGSI_SWIZZLE_X),
 661                       ureg_scalar(ureg_src(temp0), TGSI_SWIZZLE_X));
 662    }
 663
 664    /* temp0.w = 0 */
 665    ureg_MOV(ureg, ureg_writemask(temp0, TGSI_WRITEMASK_W), ureg_imm1u(ureg, 0));
 666
 667    if (download) {
 668       struct ureg_dst temp1;
 669       struct ureg_src op[2];
 670
 671       temp1 = ureg_DECL_temporary(ureg);
 672
 673       /* temp1.xy = pos.xy */
 674       ureg_F2I(ureg, ureg_writemask(temp1, TGSI_WRITEMASK_XY), pos);
 675
 676       /* temp1.zw = 0 */
 677       ureg_MOV(ureg, ureg_writemask(temp1, TGSI_WRITEMASK_ZW), ureg_imm1u(ureg, 0));
 678
 679       if (have_layer) {
 680          struct ureg_dst temp1_layer =
 681             ureg_writemask(temp1, target == PIPE_TEXTURE_1D_ARRAY ? TGSI_WRITEMASK_Y
 682                                                                   : TGSI_WRITEMASK_Z);
 683
 684          /* temp1.y/z = layer */
 685          ureg_MOV(ureg, temp1_layer, ureg_scalar(layer, TGSI_SWIZZLE_X));
 686
 687          if (target == PIPE_TEXTURE_3D) {
 688             /* temp1.z += layer_offset */
 689             ureg_UADD(ureg, temp1_layer,
 690                             ureg_scalar(ureg_src(temp1), TGSI_SWIZZLE_Z),
 691                             ureg_scalar(const1, TGSI_SWIZZLE_X));
 692          }
 693       }
 694
 695       /* temp1 = txf(sampler, temp1) */
 696       ureg_TXF(ureg, temp1, util_pipe_tex_to_tgsi_tex(target, 1),
 697                      ureg_src(temp1), sampler);
 698
 699       build_conversion(ureg, &temp1, conversion);
 700
 701       /* store(out, temp0, temp1) */
 702       op[0] = ureg_src(temp0);
 703       op[1] = ureg_src(temp1);
 704       ureg_memory_insn(ureg, TGSI_OPCODE_STORE, &out, 1, op, 2, 0,
 705                              TGSI_TEXTURE_BUFFER, PIPE_FORMAT_NONE);
 706
 707       ureg_release_temporary(ureg, temp1);
 708    } else {
 709       /* out = txf(sampler, temp0.x) */
 710       ureg_TXF(ureg, temp0, TGSI_TEXTURE_BUFFER, ureg_src(temp0), sampler);
 711
 712       build_conversion(ureg, &temp0, conversion);
 713
 714       ureg_MOV(ureg, out, ureg_src(temp0));
 715    }
 716
 717    ureg_release_temporary(ureg, temp0);
 718
 719    ureg_END(ureg);
 720
 721    return ureg_create_shader_and_destroy(ureg, pipe);
 722 }
 723
 724 static void *
 725 create_fs(struct st_context *st, bool download,
 726           enum pipe_texture_target target,
 727           enum st_pbo_conversion conversion)
 728 {
 729    struct pipe_screen *pscreen = st->pipe->screen;
 730    bool use_nir = PIPE_SHADER_IR_NIR ==
 731       pscreen->get_shader_param(pscreen, PIPE_SHADER_VERTEX,
 732                                 PIPE_SHADER_CAP_PREFERRED_IR);
 733
 734    if (use_nir)
 735       return create_fs_nir(st, download, target, conversion);
 736
 737    return create_fs_tgsi(st, download, target, conversion);
 738 }
 739
 740 static enum st_pbo_conversion
 741 get_pbo_conversion(enum pipe_format src_format, enum pipe_format dst_format)
 742 {
 743    if (util_format_is_pure_uint(src_format)) {
 744       if (util_format_is_pure_sint(dst_format))
 745          return ST_PBO_CONVERT_UINT_TO_SINT;
 746    } else if (util_format_is_pure_sint(src_format)) {
 747       if (util_format_is_pure_uint(dst_format))
 748          return ST_PBO_CONVERT_SINT_TO_UINT;
 749    }
 750
 751    return ST_PBO_CONVERT_NONE;
 752 }
 753
 754 void *
 755 st_pbo_get_upload_fs(struct st_context *st,
 756                      enum pipe_format src_format,
 757                      enum pipe_format dst_format)
 758 {
 759    STATIC_ASSERT(ARRAY_SIZE(st->pbo.upload_fs) == ST_NUM_PBO_CONVERSIONS);
 760
 761    enum st_pbo_conversion conversion = get_pbo_conversion(src_format, dst_format);
 762
 763    if (!st->pbo.upload_fs[conversion])
 764       st->pbo.upload_fs[conversion] = create_fs(st, false, 0, conversion);
 765
 766    return st->pbo.upload_fs[conversion];
 767 }
 768
 769 void *
 770 st_pbo_get_download_fs(struct st_context *st, enum pipe_texture_target target,
 771                        enum pipe_format src_format,
 772                        enum pipe_format dst_format)
 773 {
 774    STATIC_ASSERT(ARRAY_SIZE(st->pbo.download_fs) == ST_NUM_PBO_CONVERSIONS);
 775    assert(target < PIPE_MAX_TEXTURE_TYPES);
 776
 777    enum st_pbo_conversion conversion = get_pbo_conversion(src_format, dst_format);
 778
 779    if (!st->pbo.download_fs[conversion][target])
 780       st->pbo.download_fs[conversion][target] = create_fs(st, true, target, conversion);
 781
 782    return st->pbo.download_fs[conversion][target];
 783 }
 784
 785 void
 786 st_init_pbo_helpers(struct st_context *st)
 787 {
 788    struct pipe_context *pipe = st->pipe;
 789    struct pipe_screen *screen = pipe->screen;
 790
 791    st->pbo.upload_enabled =
 792       screen->get_param(screen, PIPE_CAP_TEXTURE_BUFFER_OBJECTS) &&
 793       screen->get_param(screen, PIPE_CAP_TEXTURE_BUFFER_OFFSET_ALIGNMENT) >= 1 &&
 794       screen->get_shader_param(screen, PIPE_SHADER_FRAGMENT, PIPE_SHADER_CAP_INTEGERS);
 795    if (!st->pbo.upload_enabled)
 796       return;
 797
 798    st->pbo.download_enabled =
 799       st->pbo.upload_enabled &&
 800       screen->get_param(screen, PIPE_CAP_SAMPLER_VIEW_TARGET) &&
 801       screen->get_param(screen, PIPE_CAP_FRAMEBUFFER_NO_ATTACHMENT) &&
 802       screen->get_shader_param(screen, PIPE_SHADER_FRAGMENT,
 803                                        PIPE_SHADER_CAP_MAX_SHADER_IMAGES) >= 1;
 804
 805    st->pbo.rgba_only =
 806       screen->get_param(screen, PIPE_CAP_BUFFER_SAMPLER_VIEW_RGBA_ONLY);
 807
 808    if (screen->get_param(screen, PIPE_CAP_TGSI_INSTANCEID)) {
 809       if (screen->get_param(screen, PIPE_CAP_TGSI_VS_LAYER_VIEWPORT)) {
 810          st->pbo.layers = true;
 811       } else if (screen->get_param(screen, PIPE_CAP_MAX_GEOMETRY_OUTPUT_VERTICES) >= 3) {
 812          st->pbo.layers = true;
 813          st->pbo.use_gs = true;
 814       }
 815    }
 816
 817    /* Blend state */
 818    memset(&st->pbo.upload_blend, 0, sizeof(struct pipe_blend_state));
 819    st->pbo.upload_blend.rt[0].colormask = PIPE_MASK_RGBA;
 820
 821    /* Rasterizer state */
 822    memset(&st->pbo.raster, 0, sizeof(struct pipe_rasterizer_state));
 823    st->pbo.raster.half_pixel_center = 1;
 824 }
 825
 826 void
 827 st_destroy_pbo_helpers(struct st_context *st)
 828 {
 829    unsigned i;
 830
 831    for (i = 0; i < ARRAY_SIZE(st->pbo.upload_fs); ++i) {
 832       if (st->pbo.upload_fs[i]) {
 833          st->pipe->delete_fs_state(st->pipe, st->pbo.upload_fs[i]);
 834          st->pbo.upload_fs[i] = NULL;
 835       }
 836    }
 837
 838    for (i = 0; i < ARRAY_SIZE(st->pbo.download_fs); ++i) {
 839       for (unsigned j = 0; j < ARRAY_SIZE(st->pbo.download_fs[0]); ++j) {
 840          if (st->pbo.download_fs[i][j]) {
 841             st->pipe->delete_fs_state(st->pipe, st->pbo.download_fs[i][j]);
 842             st->pbo.download_fs[i][j] = NULL;
 843          }
 844       }
 845    }
 846
 847    if (st->pbo.gs) {
 848       st->pipe->delete_gs_state(st->pipe, st->pbo.gs);
 849       st->pbo.gs = NULL;
 850    }
 851
 852    if (st->pbo.vs) {
 853       st->pipe->delete_vs_state(st->pipe, st->pbo.vs);
 854       st->pbo.vs = NULL;
 855    }
 856 }