src/compiler/spirv/vtn_opencl.c

   1 /*
   2  * Copyright © 2018 Red Hat
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Rob Clark (robdclark@gmail.com)
  25  */
  26
  27 #include "math.h"
  28 #include "nir/nir_builtin_builder.h"
  29
  30 #include "vtn_private.h"
  31 #include "OpenCL.std.h"
  32
  33 typedef nir_ssa_def *(*nir_handler)(struct vtn_builder *b,
  34                                     enum OpenCLstd_Entrypoints opcode,
  35                                     unsigned num_srcs, nir_ssa_def **srcs,
  36                                     const struct glsl_type *dest_type);
  37
  38 static void
  39 handle_instr(struct vtn_builder *b, enum OpenCLstd_Entrypoints opcode,
  40              const uint32_t *w, unsigned count, nir_handler handler)
  41 {
  42    const struct glsl_type *dest_type = vtn_get_type(b, w[1])->type;
  43
  44    unsigned num_srcs = count - 5;
  45    nir_ssa_def *srcs[3] = { NULL };
  46    vtn_assert(num_srcs <= ARRAY_SIZE(srcs));
  47    for (unsigned i = 0; i < num_srcs; i++) {
  48       srcs[i] = vtn_get_nir_ssa(b, w[i + 5]);
  49    }
  50
  51    nir_ssa_def *result = handler(b, opcode, num_srcs, srcs, dest_type);
  52    if (result) {
  53       vtn_push_nir_ssa(b, w[2], result);
  54    } else {
  55       vtn_assert(dest_type == glsl_void_type());
  56    }
  57 }
  58
  59 static nir_op
  60 nir_alu_op_for_opencl_opcode(struct vtn_builder *b,
  61                              enum OpenCLstd_Entrypoints opcode)
  62 {
  63    switch (opcode) {
  64    case OpenCLstd_Fabs: return nir_op_fabs;
  65    case OpenCLstd_SAbs: return nir_op_iabs;
  66    case OpenCLstd_SAdd_sat: return nir_op_iadd_sat;
  67    case OpenCLstd_UAdd_sat: return nir_op_uadd_sat;
  68    case OpenCLstd_Ceil: return nir_op_fceil;
  69    case OpenCLstd_Cos: return nir_op_fcos;
  70    case OpenCLstd_Exp2: return nir_op_fexp2;
  71    case OpenCLstd_Log2: return nir_op_flog2;
  72    case OpenCLstd_Floor: return nir_op_ffloor;
  73    case OpenCLstd_SHadd: return nir_op_ihadd;
  74    case OpenCLstd_UHadd: return nir_op_uhadd;
  75    case OpenCLstd_Fma: return nir_op_ffma;
  76    case OpenCLstd_Fmax: return nir_op_fmax;
  77    case OpenCLstd_SMax: return nir_op_imax;
  78    case OpenCLstd_UMax: return nir_op_umax;
  79    case OpenCLstd_Fmin: return nir_op_fmin;
  80    case OpenCLstd_SMin: return nir_op_imin;
  81    case OpenCLstd_UMin: return nir_op_umin;
  82    case OpenCLstd_Fmod: return nir_op_fmod;
  83    case OpenCLstd_Mix: return nir_op_flrp;
  84    case OpenCLstd_Native_cos: return nir_op_fcos;
  85    case OpenCLstd_Native_divide: return nir_op_fdiv;
  86    case OpenCLstd_Native_exp2: return nir_op_fexp2;
  87    case OpenCLstd_Native_log2: return nir_op_flog2;
  88    case OpenCLstd_Native_powr: return nir_op_fpow;
  89    case OpenCLstd_Native_recip: return nir_op_frcp;
  90    case OpenCLstd_Native_rsqrt: return nir_op_frsq;
  91    case OpenCLstd_Native_sin: return nir_op_fsin;
  92    case OpenCLstd_Native_sqrt: return nir_op_fsqrt;
  93    case OpenCLstd_SMul_hi: return nir_op_imul_high;
  94    case OpenCLstd_UMul_hi: return nir_op_umul_high;
  95    case OpenCLstd_Popcount: return nir_op_bit_count;
  96    case OpenCLstd_Pow: return nir_op_fpow;
  97    case OpenCLstd_Remainder: return nir_op_frem;
  98    case OpenCLstd_SRhadd: return nir_op_irhadd;
  99    case OpenCLstd_URhadd: return nir_op_urhadd;
 100    case OpenCLstd_Rsqrt: return nir_op_frsq;
 101    case OpenCLstd_Sign: return nir_op_fsign;
 102    case OpenCLstd_Sin: return nir_op_fsin;
 103    case OpenCLstd_Sqrt: return nir_op_fsqrt;
 104    case OpenCLstd_SSub_sat: return nir_op_isub_sat;
 105    case OpenCLstd_USub_sat: return nir_op_usub_sat;
 106    case OpenCLstd_Trunc: return nir_op_ftrunc;
 107    case OpenCLstd_Rint: return nir_op_fround_even;
 108    /* uhm... */
 109    case OpenCLstd_UAbs: return nir_op_mov;
 110    default:
 111       vtn_fail("No NIR equivalent");
 112    }
 113 }
 114
 115 static nir_ssa_def *
 116 handle_alu(struct vtn_builder *b, enum OpenCLstd_Entrypoints opcode,
 117            unsigned num_srcs, nir_ssa_def **srcs,
 118            const struct glsl_type *dest_type)
 119 {
 120    nir_ssa_def *ret = nir_build_alu(&b->nb, nir_alu_op_for_opencl_opcode(b, opcode),
 121                                     srcs[0], srcs[1], srcs[2], NULL);
 122    if (opcode == OpenCLstd_Popcount)
 123       ret = nir_u2u(&b->nb, ret, glsl_get_bit_size(dest_type));
 124    return ret;
 125 }
 126
 127 static nir_ssa_def *
 128 handle_special(struct vtn_builder *b, enum OpenCLstd_Entrypoints opcode,
 129                unsigned num_srcs, nir_ssa_def **srcs,
 130                const struct glsl_type *dest_type)
 131 {
 132    nir_builder *nb = &b->nb;
 133
 134    switch (opcode) {
 135    case OpenCLstd_SAbs_diff:
 136       return nir_iabs_diff(nb, srcs[0], srcs[1]);
 137    case OpenCLstd_UAbs_diff:
 138       return nir_uabs_diff(nb, srcs[0], srcs[1]);
 139    case OpenCLstd_Bitselect:
 140       return nir_bitselect(nb, srcs[0], srcs[1], srcs[2]);
 141    case OpenCLstd_SMad_hi:
 142       return nir_imad_hi(nb, srcs[0], srcs[1], srcs[2]);
 143    case OpenCLstd_UMad_hi:
 144       return nir_umad_hi(nb, srcs[0], srcs[1], srcs[2]);
 145    case OpenCLstd_SMul24:
 146       return nir_imul24(nb, srcs[0], srcs[1]);
 147    case OpenCLstd_UMul24:
 148       return nir_umul24(nb, srcs[0], srcs[1]);
 149    case OpenCLstd_SMad24:
 150       return nir_imad24(nb, srcs[0], srcs[1], srcs[2]);
 151    case OpenCLstd_UMad24:
 152       return nir_umad24(nb, srcs[0], srcs[1], srcs[2]);
 153    case OpenCLstd_FClamp:
 154       return nir_fclamp(nb, srcs[0], srcs[1], srcs[2]);
 155    case OpenCLstd_SClamp:
 156       return nir_iclamp(nb, srcs[0], srcs[1], srcs[2]);
 157    case OpenCLstd_UClamp:
 158       return nir_uclamp(nb, srcs[0], srcs[1], srcs[2]);
 159    case OpenCLstd_Copysign:
 160       return nir_copysign(nb, srcs[0], srcs[1]);
 161    case OpenCLstd_Cross:
 162       if (glsl_get_components(dest_type) == 4)
 163          return nir_cross4(nb, srcs[0], srcs[1]);
 164       return nir_cross3(nb, srcs[0], srcs[1]);
 165    case OpenCLstd_Degrees:
 166       return nir_degrees(nb, srcs[0]);
 167    case OpenCLstd_Fdim:
 168       return nir_fdim(nb, srcs[0], srcs[1]);
 169    case OpenCLstd_Distance:
 170       return nir_distance(nb, srcs[0], srcs[1]);
 171    case OpenCLstd_Fast_distance:
 172       return nir_fast_distance(nb, srcs[0], srcs[1]);
 173    case OpenCLstd_Fast_length:
 174       return nir_fast_length(nb, srcs[0]);
 175    case OpenCLstd_Fast_normalize:
 176       return nir_fast_normalize(nb, srcs[0]);
 177    case OpenCLstd_Length:
 178       return nir_length(nb, srcs[0]);
 179    case OpenCLstd_Mad:
 180       return nir_fmad(nb, srcs[0], srcs[1], srcs[2]);
 181    case OpenCLstd_Maxmag:
 182       return nir_maxmag(nb, srcs[0], srcs[1]);
 183    case OpenCLstd_Minmag:
 184       return nir_minmag(nb, srcs[0], srcs[1]);
 185    case OpenCLstd_Nan:
 186       return nir_nan(nb, srcs[0]);
 187    case OpenCLstd_Nextafter:
 188       return nir_nextafter(nb, srcs[0], srcs[1]);
 189    case OpenCLstd_Normalize:
 190       return nir_normalize(nb, srcs[0]);
 191    case OpenCLstd_Radians:
 192       return nir_radians(nb, srcs[0]);
 193    case OpenCLstd_Rotate:
 194       return nir_rotate(nb, srcs[0], srcs[1]);
 195    case OpenCLstd_Smoothstep:
 196       return nir_smoothstep(nb, srcs[0], srcs[1], srcs[2]);
 197    case OpenCLstd_Clz:
 198       return nir_clz_u(nb, srcs[0]);
 199    case OpenCLstd_Select:
 200       return nir_select(nb, srcs[0], srcs[1], srcs[2]);
 201    case OpenCLstd_Step:
 202       return nir_sge(nb, srcs[1], srcs[0]);
 203    case OpenCLstd_S_Upsample:
 204    case OpenCLstd_U_Upsample:
 205       return nir_upsample(nb, srcs[0], srcs[1]);
 206    case OpenCLstd_Native_exp:
 207       return nir_fexp(nb, srcs[0]);
 208    case OpenCLstd_Native_exp10:
 209       return nir_fexp2(nb, nir_fmul_imm(nb, srcs[0], log(10) / log(2)));
 210    case OpenCLstd_Native_log:
 211       return nir_flog(nb, srcs[0]);
 212    case OpenCLstd_Native_log10:
 213       return nir_fmul_imm(nb, nir_flog2(nb, srcs[0]), log(2) / log(10));
 214    case OpenCLstd_Native_tan:
 215       return nir_ftan(nb, srcs[0]);
 216    default:
 217       vtn_fail("No NIR equivalent");
 218       return NULL;
 219    }
 220 }
 221
 222 static void
 223 _handle_v_load_store(struct vtn_builder *b, enum OpenCLstd_Entrypoints opcode,
 224                      const uint32_t *w, unsigned count, bool load)
 225 {
 226    struct vtn_type *type;
 227    if (load)
 228       type = vtn_get_type(b, w[1]);
 229    else
 230       type = vtn_get_value_type(b, w[5]);
 231    unsigned a = load ? 0 : 1;
 232
 233    const struct glsl_type *dest_type = type->type;
 234    unsigned components = glsl_get_vector_elements(dest_type);
 235
 236    nir_ssa_def *offset = vtn_get_nir_ssa(b, w[5 + a]);
 237    struct vtn_value *p = vtn_value(b, w[6 + a], vtn_value_type_pointer);
 238
 239    struct vtn_ssa_value *comps[NIR_MAX_VEC_COMPONENTS];
 240    nir_ssa_def *ncomps[NIR_MAX_VEC_COMPONENTS];
 241
 242    nir_ssa_def *moffset = nir_imul_imm(&b->nb, offset, components);
 243    nir_deref_instr *deref = vtn_pointer_to_deref(b, p->pointer);
 244
 245    for (int i = 0; i < components; i++) {
 246       nir_ssa_def *coffset = nir_iadd_imm(&b->nb, moffset, i);
 247       nir_deref_instr *arr_deref = nir_build_deref_ptr_as_array(&b->nb, deref, coffset);
 248
 249       if (load) {
 250          comps[i] = vtn_local_load(b, arr_deref, p->type->access);
 251          ncomps[i] = comps[i]->def;
 252       } else {
 253          struct vtn_ssa_value *ssa = vtn_create_ssa_value(b, glsl_scalar_type(glsl_get_base_type(dest_type)));
 254          struct vtn_ssa_value *val = vtn_ssa_value(b, w[5]);
 255          ssa->def = nir_channel(&b->nb, val->def, i);
 256          vtn_local_store(b, ssa, arr_deref, p->type->access);
 257       }
 258    }
 259    if (load) {
 260       vtn_push_nir_ssa(b, w[2], nir_vec(&b->nb, ncomps, components));
 261    }
 262 }
 263
 264 static void
 265 vtn_handle_opencl_vload(struct vtn_builder *b, enum OpenCLstd_Entrypoints opcode,
 266                         const uint32_t *w, unsigned count)
 267 {
 268    _handle_v_load_store(b, opcode, w, count, true);
 269 }
 270
 271 static void
 272 vtn_handle_opencl_vstore(struct vtn_builder *b, enum OpenCLstd_Entrypoints opcode,
 273                          const uint32_t *w, unsigned count)
 274 {
 275    _handle_v_load_store(b, opcode, w, count, false);
 276 }
 277
 278 static nir_ssa_def *
 279 handle_printf(struct vtn_builder *b, enum OpenCLstd_Entrypoints opcode,
 280               unsigned num_srcs, nir_ssa_def **srcs,
 281               const struct glsl_type *dest_type)
 282 {
 283    /* hahah, yeah, right.. */
 284    return nir_imm_int(&b->nb, -1);
 285 }
 286
 287 static nir_ssa_def *
 288 handle_round(struct vtn_builder *b, enum OpenCLstd_Entrypoints opcode,
 289              unsigned num_srcs, nir_ssa_def **srcs,
 290              const struct glsl_type *dest_type)
 291 {
 292    nir_ssa_def *src = srcs[0];
 293    nir_builder *nb = &b->nb;
 294    nir_ssa_def *half = nir_imm_floatN_t(nb, 0.5, src->bit_size);
 295    nir_ssa_def *truncated = nir_ftrunc(nb, src);
 296    nir_ssa_def *remainder = nir_fsub(nb, src, truncated);
 297
 298    return nir_bcsel(nb, nir_fge(nb, nir_fabs(nb, remainder), half),
 299                     nir_fadd(nb, truncated, nir_fsign(nb, src)), truncated);
 300 }
 301
 302 static nir_ssa_def *
 303 handle_shuffle(struct vtn_builder *b, enum OpenCLstd_Entrypoints opcode, unsigned num_srcs,
 304                nir_ssa_def **srcs, const struct glsl_type *dest_type)
 305 {
 306    struct nir_ssa_def *input = srcs[0];
 307    struct nir_ssa_def *mask = srcs[1];
 308
 309    unsigned out_elems = glsl_get_vector_elements(dest_type);
 310    nir_ssa_def *outres[NIR_MAX_VEC_COMPONENTS];
 311    unsigned in_elems = input->num_components;
 312    if (mask->bit_size != 32)
 313       mask = nir_u2u32(&b->nb, mask);
 314    mask = nir_iand(&b->nb, mask, nir_imm_intN_t(&b->nb, in_elems - 1, mask->bit_size));
 315    for (unsigned i = 0; i < out_elems; i++)
 316       outres[i] = nir_vector_extract(&b->nb, input, nir_channel(&b->nb, mask, i));
 317
 318    return nir_vec(&b->nb, outres, out_elems);
 319 }
 320
 321 static nir_ssa_def *
 322 handle_shuffle2(struct vtn_builder *b, enum OpenCLstd_Entrypoints opcode, unsigned num_srcs,
 323                 nir_ssa_def **srcs, const struct glsl_type *dest_type)
 324 {
 325    struct nir_ssa_def *input0 = srcs[0];
 326    struct nir_ssa_def *input1 = srcs[1];
 327    struct nir_ssa_def *mask = srcs[2];
 328
 329    unsigned out_elems = glsl_get_vector_elements(dest_type);
 330    nir_ssa_def *outres[NIR_MAX_VEC_COMPONENTS];
 331    unsigned in_elems = input0->num_components;
 332    unsigned total_mask = 2 * in_elems - 1;
 333    unsigned half_mask = in_elems - 1;
 334    if (mask->bit_size != 32)
 335       mask = nir_u2u32(&b->nb, mask);
 336    mask = nir_iand(&b->nb, mask, nir_imm_intN_t(&b->nb, total_mask, mask->bit_size));
 337    for (unsigned i = 0; i < out_elems; i++) {
 338       nir_ssa_def *this_mask = nir_channel(&b->nb, mask, i);
 339       nir_ssa_def *vmask = nir_iand(&b->nb, this_mask, nir_imm_intN_t(&b->nb, half_mask, mask->bit_size));
 340       nir_ssa_def *val0 = nir_vector_extract(&b->nb, input0, vmask);
 341       nir_ssa_def *val1 = nir_vector_extract(&b->nb, input1, vmask);
 342       nir_ssa_def *sel = nir_ilt(&b->nb, this_mask, nir_imm_intN_t(&b->nb, in_elems, mask->bit_size));
 343       outres[i] = nir_bcsel(&b->nb, sel, val0, val1);
 344    }
 345    return nir_vec(&b->nb, outres, out_elems);
 346 }
 347
 348 bool
 349 vtn_handle_opencl_instruction(struct vtn_builder *b, SpvOp ext_opcode,
 350                               const uint32_t *w, unsigned count)
 351 {
 352    enum OpenCLstd_Entrypoints cl_opcode = (enum OpenCLstd_Entrypoints) ext_opcode;
 353
 354    switch (cl_opcode) {
 355    case OpenCLstd_Fabs:
 356    case OpenCLstd_SAbs:
 357    case OpenCLstd_UAbs:
 358    case OpenCLstd_SAdd_sat:
 359    case OpenCLstd_UAdd_sat:
 360    case OpenCLstd_Ceil:
 361    case OpenCLstd_Cos:
 362    case OpenCLstd_Exp2:
 363    case OpenCLstd_Log2:
 364    case OpenCLstd_Floor:
 365    case OpenCLstd_Fma:
 366    case OpenCLstd_Fmax:
 367    case OpenCLstd_SHadd:
 368    case OpenCLstd_UHadd:
 369    case OpenCLstd_SMax:
 370    case OpenCLstd_UMax:
 371    case OpenCLstd_Fmin:
 372    case OpenCLstd_SMin:
 373    case OpenCLstd_UMin:
 374    case OpenCLstd_Mix:
 375    case OpenCLstd_Native_cos:
 376    case OpenCLstd_Native_divide:
 377    case OpenCLstd_Native_exp2:
 378    case OpenCLstd_Native_log2:
 379    case OpenCLstd_Native_powr:
 380    case OpenCLstd_Native_recip:
 381    case OpenCLstd_Native_rsqrt:
 382    case OpenCLstd_Native_sin:
 383    case OpenCLstd_Native_sqrt:
 384    case OpenCLstd_Fmod:
 385    case OpenCLstd_SMul_hi:
 386    case OpenCLstd_UMul_hi:
 387    case OpenCLstd_Popcount:
 388    case OpenCLstd_Pow:
 389    case OpenCLstd_Remainder:
 390    case OpenCLstd_SRhadd:
 391    case OpenCLstd_URhadd:
 392    case OpenCLstd_Rsqrt:
 393    case OpenCLstd_Sign:
 394    case OpenCLstd_Sin:
 395    case OpenCLstd_Sqrt:
 396    case OpenCLstd_SSub_sat:
 397    case OpenCLstd_USub_sat:
 398    case OpenCLstd_Trunc:
 399    case OpenCLstd_Rint:
 400       handle_instr(b, cl_opcode, w, count, handle_alu);
 401       return true;
 402    case OpenCLstd_SAbs_diff:
 403    case OpenCLstd_UAbs_diff:
 404    case OpenCLstd_SMad_hi:
 405    case OpenCLstd_UMad_hi:
 406    case OpenCLstd_SMad24:
 407    case OpenCLstd_UMad24:
 408    case OpenCLstd_SMul24:
 409    case OpenCLstd_UMul24:
 410    case OpenCLstd_Bitselect:
 411    case OpenCLstd_FClamp:
 412    case OpenCLstd_SClamp:
 413    case OpenCLstd_UClamp:
 414    case OpenCLstd_Copysign:
 415    case OpenCLstd_Cross:
 416    case OpenCLstd_Degrees:
 417    case OpenCLstd_Fdim:
 418    case OpenCLstd_Distance:
 419    case OpenCLstd_Fast_distance:
 420    case OpenCLstd_Fast_length:
 421    case OpenCLstd_Fast_normalize:
 422    case OpenCLstd_Length:
 423    case OpenCLstd_Mad:
 424    case OpenCLstd_Maxmag:
 425    case OpenCLstd_Minmag:
 426    case OpenCLstd_Nan:
 427    case OpenCLstd_Nextafter:
 428    case OpenCLstd_Normalize:
 429    case OpenCLstd_Radians:
 430    case OpenCLstd_Rotate:
 431    case OpenCLstd_Select:
 432    case OpenCLstd_Step:
 433    case OpenCLstd_Smoothstep:
 434    case OpenCLstd_S_Upsample:
 435    case OpenCLstd_U_Upsample:
 436    case OpenCLstd_Clz:
 437    case OpenCLstd_Native_exp:
 438    case OpenCLstd_Native_exp10:
 439    case OpenCLstd_Native_log:
 440    case OpenCLstd_Native_log10:
 441    case OpenCLstd_Native_tan:
 442       handle_instr(b, cl_opcode, w, count, handle_special);
 443       return true;
 444    case OpenCLstd_Vloadn:
 445       vtn_handle_opencl_vload(b, cl_opcode, w, count);
 446       return true;
 447    case OpenCLstd_Vstoren:
 448       vtn_handle_opencl_vstore(b, cl_opcode, w, count);
 449       return true;
 450    case OpenCLstd_Shuffle:
 451       handle_instr(b, cl_opcode, w, count, handle_shuffle);
 452       return true;
 453    case OpenCLstd_Shuffle2:
 454       handle_instr(b, cl_opcode, w, count, handle_shuffle2);
 455       return true;
 456    case OpenCLstd_Round:
 457       handle_instr(b, cl_opcode, w, count, handle_round);
 458       return true;
 459    case OpenCLstd_Printf:
 460       handle_instr(b, cl_opcode, w, count, handle_printf);
 461       return true;
 462    case OpenCLstd_Prefetch:
 463       /* TODO maybe add a nir instruction for this? */
 464       return true;
 465    default:
 466       vtn_fail("unhandled opencl opc: %u\n", ext_opcode);
 467       return false;
 468    }
 469 }