src/gallium/auxiliary/gallivm/lp_bld_intr.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2009 VMware, Inc.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  19  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  21  * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
  22  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  23  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  24  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27
  28
  29 /**
  30  * @file
  31  * Helpers for emiting intrinsic calls.
  32  *
  33  * LLVM vanilla IR doesn't represent all basic arithmetic operations we care
  34  * about, and it is often necessary to resort target-specific intrinsics for
  35  * performance, convenience.
  36  *
  37  * Ideally we would like to stay away from target specific intrinsics and
  38  * move all the instruction selection logic into upstream LLVM where it belongs.
  39  *
  40  * These functions are also used for calling C functions provided by us from
  41  * generated LLVM code.
  42  *
  43  * @author Jose Fonseca <jfonseca@vmware.com>
  44  */
  45
  46 #include <llvm/Config/llvm-config.h>
  47
  48 #include "util/u_debug.h"
  49 #include "util/u_string.h"
  50 #include "util/bitscan.h"
  51
  52 #include "lp_bld_const.h"
  53 #include "lp_bld_intr.h"
  54 #include "lp_bld_type.h"
  55 #include "lp_bld_pack.h"
  56 #include "lp_bld_debug.h"
  57
  58
  59 void
  60 lp_format_intrinsic(char *name,
  61                     size_t size,
  62                     const char *name_root,
  63                     LLVMTypeRef type)
  64 {
  65    unsigned length = 0;
  66    unsigned width;
  67    char c;
  68
  69    LLVMTypeKind kind = LLVMGetTypeKind(type);
  70    if (kind == LLVMVectorTypeKind) {
  71       length = LLVMGetVectorSize(type);
  72       type = LLVMGetElementType(type);
  73       kind = LLVMGetTypeKind(type);
  74    }
  75
  76    switch (kind) {
  77    case LLVMIntegerTypeKind:
  78       c = 'i';
  79       width = LLVMGetIntTypeWidth(type);
  80       break;
  81    case LLVMFloatTypeKind:
  82       c = 'f';
  83       width = 32;
  84       break;
  85    case LLVMDoubleTypeKind:
  86       c = 'f';
  87       width = 64;
  88       break;
  89    default:
  90       unreachable("unexpected LLVMTypeKind");
  91    }
  92
  93    if (length) {
  94       snprintf(name, size, "%s.v%u%c%u", name_root, length, c, width);
  95    } else {
  96       snprintf(name, size, "%s.%c%u", name_root, c, width);
  97    }
  98 }
  99
 100
 101 LLVMValueRef
 102 lp_declare_intrinsic(LLVMModuleRef module,
 103                      const char *name,
 104                      LLVMTypeRef ret_type,
 105                      LLVMTypeRef *arg_types,
 106                      unsigned num_args)
 107 {
 108    LLVMTypeRef function_type;
 109    LLVMValueRef function;
 110
 111    assert(!LLVMGetNamedFunction(module, name));
 112
 113    function_type = LLVMFunctionType(ret_type, arg_types, num_args, 0);
 114    function = LLVMAddFunction(module, name, function_type);
 115
 116    LLVMSetFunctionCallConv(function, LLVMCCallConv);
 117    LLVMSetLinkage(function, LLVMExternalLinkage);
 118
 119    assert(LLVMIsDeclaration(function));
 120
 121    return function;
 122 }
 123
 124
 125 #if LLVM_VERSION_MAJOR < 4
 126 static LLVMAttribute lp_attr_to_llvm_attr(enum lp_func_attr attr)
 127 {
 128    switch (attr) {
 129    case LP_FUNC_ATTR_ALWAYSINLINE: return LLVMAlwaysInlineAttribute;
 130    case LP_FUNC_ATTR_INREG: return LLVMInRegAttribute;
 131    case LP_FUNC_ATTR_NOALIAS: return LLVMNoAliasAttribute;
 132    case LP_FUNC_ATTR_NOUNWIND: return LLVMNoUnwindAttribute;
 133    case LP_FUNC_ATTR_READNONE: return LLVMReadNoneAttribute;
 134    case LP_FUNC_ATTR_READONLY: return LLVMReadOnlyAttribute;
 135    default:
 136       _debug_printf("Unhandled function attribute: %x\n", attr);
 137       return 0;
 138    }
 139 }
 140
 141 #else
 142
 143 static const char *attr_to_str(enum lp_func_attr attr)
 144 {
 145    switch (attr) {
 146    case LP_FUNC_ATTR_ALWAYSINLINE: return "alwaysinline";
 147    case LP_FUNC_ATTR_INREG: return "inreg";
 148    case LP_FUNC_ATTR_NOALIAS: return "noalias";
 149    case LP_FUNC_ATTR_NOUNWIND: return "nounwind";
 150    case LP_FUNC_ATTR_READNONE: return "readnone";
 151    case LP_FUNC_ATTR_READONLY: return "readonly";
 152    case LP_FUNC_ATTR_WRITEONLY: return "writeonly";
 153    case LP_FUNC_ATTR_INACCESSIBLE_MEM_ONLY: return "inaccessiblememonly";
 154    case LP_FUNC_ATTR_CONVERGENT: return "convergent";
 155    default:
 156       _debug_printf("Unhandled function attribute: %x\n", attr);
 157       return 0;
 158    }
 159 }
 160
 161 #endif
 162
 163 void
 164 lp_add_function_attr(LLVMValueRef function_or_call,
 165                      int attr_idx, enum lp_func_attr attr)
 166 {
 167
 168 #if LLVM_VERSION_MAJOR < 4
 169    LLVMAttribute llvm_attr = lp_attr_to_llvm_attr(attr);
 170    if (LLVMIsAFunction(function_or_call)) {
 171       if (attr_idx == -1) {
 172          LLVMAddFunctionAttr(function_or_call, llvm_attr);
 173       } else {
 174          LLVMAddAttribute(LLVMGetParam(function_or_call, attr_idx - 1), llvm_attr);
 175       }
 176    } else {
 177       LLVMAddInstrAttribute(function_or_call, attr_idx, llvm_attr);
 178    }
 179 #else
 180
 181    LLVMModuleRef module;
 182    if (LLVMIsAFunction(function_or_call)) {
 183       module = LLVMGetGlobalParent(function_or_call);
 184    } else {
 185       LLVMBasicBlockRef bb = LLVMGetInstructionParent(function_or_call);
 186       LLVMValueRef function = LLVMGetBasicBlockParent(bb);
 187       module = LLVMGetGlobalParent(function);
 188    }
 189    LLVMContextRef ctx = LLVMGetModuleContext(module);
 190
 191    const char *attr_name = attr_to_str(attr);
 192    unsigned kind_id = LLVMGetEnumAttributeKindForName(attr_name,
 193                                                       strlen(attr_name));
 194    LLVMAttributeRef llvm_attr = LLVMCreateEnumAttribute(ctx, kind_id, 0);
 195
 196    if (LLVMIsAFunction(function_or_call))
 197       LLVMAddAttributeAtIndex(function_or_call, attr_idx, llvm_attr);
 198    else
 199       LLVMAddCallSiteAttribute(function_or_call, attr_idx, llvm_attr);
 200 #endif
 201 }
 202
 203 static void
 204 lp_add_func_attributes(LLVMValueRef function, unsigned attrib_mask)
 205 {
 206    /* NoUnwind indicates that the intrinsic never raises a C++ exception.
 207     * Set it for all intrinsics.
 208     */
 209    attrib_mask |= LP_FUNC_ATTR_NOUNWIND;
 210    attrib_mask &= ~LP_FUNC_ATTR_LEGACY;
 211
 212    while (attrib_mask) {
 213       enum lp_func_attr attr = 1u << u_bit_scan(&attrib_mask);
 214       lp_add_function_attr(function, -1, attr);
 215    }
 216 }
 217
 218 LLVMValueRef
 219 lp_build_intrinsic(LLVMBuilderRef builder,
 220                    const char *name,
 221                    LLVMTypeRef ret_type,
 222                    LLVMValueRef *args,
 223                    unsigned num_args,
 224                    unsigned attr_mask)
 225 {
 226    LLVMModuleRef module = LLVMGetGlobalParent(LLVMGetBasicBlockParent(LLVMGetInsertBlock(builder)));
 227    LLVMValueRef function, call;
 228    bool set_callsite_attrs = LLVM_VERSION_MAJOR >= 4 &&
 229                              !(attr_mask & LP_FUNC_ATTR_LEGACY);
 230
 231    function = LLVMGetNamedFunction(module, name);
 232    if(!function) {
 233       LLVMTypeRef arg_types[LP_MAX_FUNC_ARGS];
 234       unsigned i;
 235
 236       assert(num_args <= LP_MAX_FUNC_ARGS);
 237
 238       for(i = 0; i < num_args; ++i) {
 239          assert(args[i]);
 240          arg_types[i] = LLVMTypeOf(args[i]);
 241       }
 242
 243       function = lp_declare_intrinsic(module, name, ret_type, arg_types, num_args);
 244
 245       /*
 246        * If llvm removes an intrinsic we use, we'll hit this abort (rather
 247        * than a call to address zero in the jited code).
 248        */
 249       if (LLVMGetIntrinsicID(function) == 0) {
 250          _debug_printf("llvm (version " MESA_LLVM_VERSION_STRING
 251                        ") found no intrinsic for %s, going to crash...\n",
 252                 name);
 253          abort();
 254       }
 255
 256       if (!set_callsite_attrs)
 257          lp_add_func_attributes(function, attr_mask);
 258
 259       if (gallivm_debug & GALLIVM_DEBUG_IR) {
 260          lp_debug_dump_value(function);
 261       }
 262    }
 263
 264    call = LLVMBuildCall(builder, function, args, num_args, "");
 265    if (set_callsite_attrs)
 266       lp_add_func_attributes(call, attr_mask);
 267    return call;
 268 }
 269
 270
 271 LLVMValueRef
 272 lp_build_intrinsic_unary(LLVMBuilderRef builder,
 273                          const char *name,
 274                          LLVMTypeRef ret_type,
 275                          LLVMValueRef a)
 276 {
 277    return lp_build_intrinsic(builder, name, ret_type, &a, 1, 0);
 278 }
 279
 280
 281 LLVMValueRef
 282 lp_build_intrinsic_binary(LLVMBuilderRef builder,
 283                           const char *name,
 284                           LLVMTypeRef ret_type,
 285                           LLVMValueRef a,
 286                           LLVMValueRef b)
 287 {
 288    LLVMValueRef args[2];
 289
 290    args[0] = a;
 291    args[1] = b;
 292
 293    return lp_build_intrinsic(builder, name, ret_type, args, 2, 0);
 294 }
 295
 296
 297 /**
 298  * Call intrinsic with arguments adapted to intrinsic vector length.
 299  *
 300  * Split vectors which are too large for the hw, or expand them if they
 301  * are too small, so a caller calling a function which might use intrinsics
 302  * doesn't need to do splitting/expansion on its own.
 303  * This only supports intrinsics where src and dst types match.
 304  */
 305 LLVMValueRef
 306 lp_build_intrinsic_binary_anylength(struct gallivm_state *gallivm,
 307                                     const char *name,
 308                                     struct lp_type src_type,
 309                                     unsigned intr_size,
 310                                     LLVMValueRef a,
 311                                     LLVMValueRef b)
 312 {
 313    unsigned i;
 314    struct lp_type intrin_type = src_type;
 315    LLVMBuilderRef builder = gallivm->builder;
 316    LLVMValueRef i32undef = LLVMGetUndef(LLVMInt32TypeInContext(gallivm->context));
 317    LLVMValueRef anative, bnative;
 318    unsigned intrin_length = intr_size / src_type.width;
 319
 320    intrin_type.length = intrin_length;
 321
 322    if (intrin_length > src_type.length) {
 323       LLVMValueRef elems[LP_MAX_VECTOR_LENGTH];
 324       LLVMValueRef constvec, tmp;
 325
 326       for (i = 0; i < src_type.length; i++) {
 327          elems[i] = lp_build_const_int32(gallivm, i);
 328       }
 329       for (; i < intrin_length; i++) {
 330          elems[i] = i32undef;
 331       }
 332       if (src_type.length == 1) {
 333          LLVMTypeRef elem_type = lp_build_elem_type(gallivm, intrin_type);
 334          a = LLVMBuildBitCast(builder, a, LLVMVectorType(elem_type, 1), "");
 335          b = LLVMBuildBitCast(builder, b, LLVMVectorType(elem_type, 1), "");
 336       }
 337       constvec = LLVMConstVector(elems, intrin_length);
 338       anative = LLVMBuildShuffleVector(builder, a, a, constvec, "");
 339       bnative = LLVMBuildShuffleVector(builder, b, b, constvec, "");
 340       tmp = lp_build_intrinsic_binary(builder, name,
 341                                       lp_build_vec_type(gallivm, intrin_type),
 342                                       anative, bnative);
 343       if (src_type.length > 1) {
 344          constvec = LLVMConstVector(elems, src_type.length);
 345          return LLVMBuildShuffleVector(builder, tmp, tmp, constvec, "");
 346       }
 347       else {
 348          return LLVMBuildExtractElement(builder, tmp, elems[0], "");
 349       }
 350    }
 351    else if (intrin_length < src_type.length) {
 352       unsigned num_vec = src_type.length / intrin_length;
 353       LLVMValueRef tmp[LP_MAX_VECTOR_LENGTH];
 354
 355       /* don't support arbitrary size here as this is so yuck */
 356       if (src_type.length % intrin_length) {
 357          /* FIXME: This is something which should be supported
 358           * but there doesn't seem to be any need for it currently
 359           * so crash and burn.
 360           */
 361          debug_printf("%s: should handle arbitrary vector size\n",
 362                       __FUNCTION__);
 363          assert(0);
 364          return NULL;
 365       }
 366
 367       for (i = 0; i < num_vec; i++) {
 368          anative = lp_build_extract_range(gallivm, a, i*intrin_length,
 369                                         intrin_length);
 370          bnative = lp_build_extract_range(gallivm, b, i*intrin_length,
 371                                         intrin_length);
 372          tmp[i] = lp_build_intrinsic_binary(builder, name,
 373                                             lp_build_vec_type(gallivm, intrin_type),
 374                                             anative, bnative);
 375       }
 376       return lp_build_concat(gallivm, tmp, intrin_type, num_vec);
 377    }
 378    else {
 379       return lp_build_intrinsic_binary(builder, name,
 380                                        lp_build_vec_type(gallivm, src_type),
 381                                        a, b);
 382    }
 383 }
 384
 385
 386 LLVMValueRef
 387 lp_build_intrinsic_map(struct gallivm_state *gallivm,
 388                        const char *name,
 389                        LLVMTypeRef ret_type,
 390                        LLVMValueRef *args,
 391                        unsigned num_args)
 392 {
 393    LLVMBuilderRef builder = gallivm->builder;
 394    LLVMTypeRef ret_elem_type = LLVMGetElementType(ret_type);
 395    unsigned n = LLVMGetVectorSize(ret_type);
 396    unsigned i, j;
 397    LLVMValueRef res;
 398
 399    assert(num_args <= LP_MAX_FUNC_ARGS);
 400
 401    res = LLVMGetUndef(ret_type);
 402    for(i = 0; i < n; ++i) {
 403       LLVMValueRef index = lp_build_const_int32(gallivm, i);
 404       LLVMValueRef arg_elems[LP_MAX_FUNC_ARGS];
 405       LLVMValueRef res_elem;
 406       for(j = 0; j < num_args; ++j)
 407          arg_elems[j] = LLVMBuildExtractElement(builder, args[j], index, "");
 408       res_elem = lp_build_intrinsic(builder, name, ret_elem_type, arg_elems, num_args, 0);
 409       res = LLVMBuildInsertElement(builder, res, res_elem, index, "");
 410    }
 411
 412    return res;
 413 }
 414
 415
 416 LLVMValueRef
 417 lp_build_intrinsic_map_unary(struct gallivm_state *gallivm,
 418                              const char *name,
 419                              LLVMTypeRef ret_type,
 420                              LLVMValueRef a)
 421 {
 422    return lp_build_intrinsic_map(gallivm, name, ret_type, &a, 1);
 423 }
 424
 425
 426 LLVMValueRef
 427 lp_build_intrinsic_map_binary(struct gallivm_state *gallivm,
 428                               const char *name,
 429                               LLVMTypeRef ret_type,
 430                               LLVMValueRef a,
 431                               LLVMValueRef b)
 432 {
 433    LLVMValueRef args[2];
 434
 435    args[0] = a;
 436    args[1] = b;
 437
 438    return lp_build_intrinsic_map(gallivm, name, ret_type, args, 2);
 439 }
 440
 441