src/gallium/auxiliary/gallivm/lp_bld_intr.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2009 VMware, Inc.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  19  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  21  * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
  22  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  23  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  24  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27
  28
  29 /**
  30  * @file
  31  * Helpers for emiting intrinsic calls.
  32  *
  33  * LLVM vanilla IR doesn't represent all basic arithmetic operations we care
  34  * about, and it is often necessary to resort target-specific intrinsics for
  35  * performance, convenience.
  36  *
  37  * Ideally we would like to stay away from target specific intrinsics and
  38  * move all the instruction selection logic into upstream LLVM where it belongs.
  39  *
  40  * These functions are also used for calling C functions provided by us from
  41  * generated LLVM code.
  42  *
  43  * @author Jose Fonseca <jfonseca@vmware.com>
  44  */
  45
  46
  47 #include "util/u_debug.h"
  48 #include "util/u_string.h"
  49 #include "util/bitscan.h"
  50
  51 #include "lp_bld_const.h"
  52 #include "lp_bld_intr.h"
  53 #include "lp_bld_type.h"
  54 #include "lp_bld_pack.h"
  55 #include "lp_bld_debug.h"
  56
  57
  58 void
  59 lp_format_intrinsic(char *name,
  60                     size_t size,
  61                     const char *name_root,
  62                     LLVMTypeRef type)
  63 {
  64    unsigned length = 0;
  65    unsigned width;
  66    char c;
  67
  68    LLVMTypeKind kind = LLVMGetTypeKind(type);
  69    if (kind == LLVMVectorTypeKind) {
  70       length = LLVMGetVectorSize(type);
  71       type = LLVMGetElementType(type);
  72       kind = LLVMGetTypeKind(type);
  73    }
  74
  75    switch (kind) {
  76    case LLVMIntegerTypeKind:
  77       c = 'i';
  78       width = LLVMGetIntTypeWidth(type);
  79       break;
  80    case LLVMFloatTypeKind:
  81       c = 'f';
  82       width = 32;
  83       break;
  84    case LLVMDoubleTypeKind:
  85       c = 'f';
  86       width = 64;
  87       break;
  88    default:
  89       unreachable("unexpected LLVMTypeKind");
  90    }
  91
  92    if (length) {
  93       util_snprintf(name, size, "%s.v%u%c%u", name_root, length, c, width);
  94    } else {
  95       util_snprintf(name, size, "%s.%c%u", name_root, c, width);
  96    }
  97 }
  98
  99
 100 LLVMValueRef
 101 lp_declare_intrinsic(LLVMModuleRef module,
 102                      const char *name,
 103                      LLVMTypeRef ret_type,
 104                      LLVMTypeRef *arg_types,
 105                      unsigned num_args)
 106 {
 107    LLVMTypeRef function_type;
 108    LLVMValueRef function;
 109
 110    assert(!LLVMGetNamedFunction(module, name));
 111
 112    function_type = LLVMFunctionType(ret_type, arg_types, num_args, 0);
 113    function = LLVMAddFunction(module, name, function_type);
 114
 115    LLVMSetFunctionCallConv(function, LLVMCCallConv);
 116    LLVMSetLinkage(function, LLVMExternalLinkage);
 117
 118    assert(LLVMIsDeclaration(function));
 119
 120    return function;
 121 }
 122
 123
 124 #if HAVE_LLVM < 0x0400
 125 static LLVMAttribute lp_attr_to_llvm_attr(enum lp_func_attr attr)
 126 {
 127    switch (attr) {
 128    case LP_FUNC_ATTR_ALWAYSINLINE: return LLVMAlwaysInlineAttribute;
 129    case LP_FUNC_ATTR_BYVAL: return LLVMByValAttribute;
 130    case LP_FUNC_ATTR_INREG: return LLVMInRegAttribute;
 131    case LP_FUNC_ATTR_NOALIAS: return LLVMNoAliasAttribute;
 132    case LP_FUNC_ATTR_NOUNWIND: return LLVMNoUnwindAttribute;
 133    case LP_FUNC_ATTR_READNONE: return LLVMReadNoneAttribute;
 134    case LP_FUNC_ATTR_READONLY: return LLVMReadOnlyAttribute;
 135    default:
 136       _debug_printf("Unhandled function attribute: %x\n", attr);
 137       return 0;
 138    }
 139 }
 140
 141 #else
 142
 143 static const char *attr_to_str(enum lp_func_attr attr)
 144 {
 145    switch (attr) {
 146    case LP_FUNC_ATTR_ALWAYSINLINE: return "alwaysinline";
 147    case LP_FUNC_ATTR_BYVAL: return "byval";
 148    case LP_FUNC_ATTR_INREG: return "inreg";
 149    case LP_FUNC_ATTR_NOALIAS: return "noalias";
 150    case LP_FUNC_ATTR_NOUNWIND: return "nounwind";
 151    case LP_FUNC_ATTR_READNONE: return "readnone";
 152    case LP_FUNC_ATTR_READONLY: return "readonly";
 153    default:
 154       _debug_printf("Unhandled function attribute: %x\n", attr);
 155       return 0;
 156    }
 157 }
 158
 159 #endif
 160
 161 void
 162 lp_add_function_attr(LLVMValueRef function,
 163                      int attr_idx,
 164                      enum lp_func_attr attr)
 165 {
 166
 167 #if HAVE_LLVM < 0x0400
 168    LLVMAttribute llvm_attr = lp_attr_to_llvm_attr(attr);
 169    if (attr_idx == -1) {
 170       LLVMAddFunctionAttr(function, llvm_attr);
 171    } else {
 172       LLVMAddAttribute(LLVMGetParam(function, attr_idx - 1), llvm_attr);
 173    }
 174 #else
 175    LLVMContextRef context = LLVMGetModuleContext(LLVMGetGlobalParent(function));
 176    const char *attr_name = attr_to_str(attr);
 177    unsigned kind_id = LLVMGetEnumAttributeKindForName(attr_name,
 178                                                       strlen(attr_name));
 179    LLVMAttributeRef llvm_attr = LLVMCreateEnumAttribute(context, kind_id, 0);
 180    LLVMAddAttributeAtIndex(function, attr_idx, llvm_attr);
 181 #endif
 182 }
 183
 184 LLVMValueRef
 185 lp_build_intrinsic(LLVMBuilderRef builder,
 186                    const char *name,
 187                    LLVMTypeRef ret_type,
 188                    LLVMValueRef *args,
 189                    unsigned num_args,
 190                    unsigned attr_mask)
 191 {
 192    LLVMModuleRef module = LLVMGetGlobalParent(LLVMGetBasicBlockParent(LLVMGetInsertBlock(builder)));
 193    LLVMValueRef function;
 194
 195    function = LLVMGetNamedFunction(module, name);
 196    if(!function) {
 197       LLVMTypeRef arg_types[LP_MAX_FUNC_ARGS];
 198       unsigned i;
 199
 200       assert(num_args <= LP_MAX_FUNC_ARGS);
 201
 202       for(i = 0; i < num_args; ++i) {
 203          assert(args[i]);
 204          arg_types[i] = LLVMTypeOf(args[i]);
 205       }
 206
 207       function = lp_declare_intrinsic(module, name, ret_type, arg_types, num_args);
 208
 209       /* NoUnwind indicates that the intrinsic never raises a C++ exception.
 210        * Set it for all intrinsics.
 211        */
 212       attr_mask |= LP_FUNC_ATTR_NOUNWIND;
 213
 214       while (attr_mask) {
 215          enum lp_func_attr attr = 1 << u_bit_scan(&attr_mask);
 216          lp_add_function_attr(function, -1, attr);
 217       }
 218
 219       if (gallivm_debug & GALLIVM_DEBUG_IR) {
 220          lp_debug_dump_value(function);
 221       }
 222    }
 223
 224    return LLVMBuildCall(builder, function, args, num_args, "");
 225 }
 226
 227
 228 LLVMValueRef
 229 lp_build_intrinsic_unary(LLVMBuilderRef builder,
 230                          const char *name,
 231                          LLVMTypeRef ret_type,
 232                          LLVMValueRef a)
 233 {
 234    return lp_build_intrinsic(builder, name, ret_type, &a, 1, 0);
 235 }
 236
 237
 238 LLVMValueRef
 239 lp_build_intrinsic_binary(LLVMBuilderRef builder,
 240                           const char *name,
 241                           LLVMTypeRef ret_type,
 242                           LLVMValueRef a,
 243                           LLVMValueRef b)
 244 {
 245    LLVMValueRef args[2];
 246
 247    args[0] = a;
 248    args[1] = b;
 249
 250    return lp_build_intrinsic(builder, name, ret_type, args, 2, 0);
 251 }
 252
 253
 254 /**
 255  * Call intrinsic with arguments adapted to intrinsic vector length.
 256  *
 257  * Split vectors which are too large for the hw, or expand them if they
 258  * are too small, so a caller calling a function which might use intrinsics
 259  * doesn't need to do splitting/expansion on its own.
 260  * This only supports intrinsics where src and dst types match.
 261  */
 262 LLVMValueRef
 263 lp_build_intrinsic_binary_anylength(struct gallivm_state *gallivm,
 264                                     const char *name,
 265                                     struct lp_type src_type,
 266                                     unsigned intr_size,
 267                                     LLVMValueRef a,
 268                                     LLVMValueRef b)
 269 {
 270    unsigned i;
 271    struct lp_type intrin_type = src_type;
 272    LLVMBuilderRef builder = gallivm->builder;
 273    LLVMValueRef i32undef = LLVMGetUndef(LLVMInt32TypeInContext(gallivm->context));
 274    LLVMValueRef anative, bnative;
 275    unsigned intrin_length = intr_size / src_type.width;
 276
 277    intrin_type.length = intrin_length;
 278
 279    if (intrin_length > src_type.length) {
 280       LLVMValueRef elems[LP_MAX_VECTOR_LENGTH];
 281       LLVMValueRef constvec, tmp;
 282
 283       for (i = 0; i < src_type.length; i++) {
 284          elems[i] = lp_build_const_int32(gallivm, i);
 285       }
 286       for (; i < intrin_length; i++) {
 287          elems[i] = i32undef;
 288       }
 289       if (src_type.length == 1) {
 290          LLVMTypeRef elem_type = lp_build_elem_type(gallivm, intrin_type);
 291          a = LLVMBuildBitCast(builder, a, LLVMVectorType(elem_type, 1), "");
 292          b = LLVMBuildBitCast(builder, b, LLVMVectorType(elem_type, 1), "");
 293       }
 294       constvec = LLVMConstVector(elems, intrin_length);
 295       anative = LLVMBuildShuffleVector(builder, a, a, constvec, "");
 296       bnative = LLVMBuildShuffleVector(builder, b, b, constvec, "");
 297       tmp = lp_build_intrinsic_binary(builder, name,
 298                                       lp_build_vec_type(gallivm, intrin_type),
 299                                       anative, bnative);
 300       if (src_type.length > 1) {
 301          constvec = LLVMConstVector(elems, src_type.length);
 302          return LLVMBuildShuffleVector(builder, tmp, tmp, constvec, "");
 303       }
 304       else {
 305          return LLVMBuildExtractElement(builder, tmp, elems[0], "");
 306       }
 307    }
 308    else if (intrin_length < src_type.length) {
 309       unsigned num_vec = src_type.length / intrin_length;
 310       LLVMValueRef tmp[LP_MAX_VECTOR_LENGTH];
 311
 312       /* don't support arbitrary size here as this is so yuck */
 313       if (src_type.length % intrin_length) {
 314          /* FIXME: This is something which should be supported
 315           * but there doesn't seem to be any need for it currently
 316           * so crash and burn.
 317           */
 318          debug_printf("%s: should handle arbitrary vector size\n",
 319                       __FUNCTION__);
 320          assert(0);
 321          return NULL;
 322       }
 323
 324       for (i = 0; i < num_vec; i++) {
 325          anative = lp_build_extract_range(gallivm, a, i*intrin_length,
 326                                         intrin_length);
 327          bnative = lp_build_extract_range(gallivm, b, i*intrin_length,
 328                                         intrin_length);
 329          tmp[i] = lp_build_intrinsic_binary(builder, name,
 330                                             lp_build_vec_type(gallivm, intrin_type),
 331                                             anative, bnative);
 332       }
 333       return lp_build_concat(gallivm, tmp, intrin_type, num_vec);
 334    }
 335    else {
 336       return lp_build_intrinsic_binary(builder, name,
 337                                        lp_build_vec_type(gallivm, src_type),
 338                                        a, b);
 339    }
 340 }
 341
 342
 343 LLVMValueRef
 344 lp_build_intrinsic_map(struct gallivm_state *gallivm,
 345                        const char *name,
 346                        LLVMTypeRef ret_type,
 347                        LLVMValueRef *args,
 348                        unsigned num_args)
 349 {
 350    LLVMBuilderRef builder = gallivm->builder;
 351    LLVMTypeRef ret_elem_type = LLVMGetElementType(ret_type);
 352    unsigned n = LLVMGetVectorSize(ret_type);
 353    unsigned i, j;
 354    LLVMValueRef res;
 355
 356    assert(num_args <= LP_MAX_FUNC_ARGS);
 357
 358    res = LLVMGetUndef(ret_type);
 359    for(i = 0; i < n; ++i) {
 360       LLVMValueRef index = lp_build_const_int32(gallivm, i);
 361       LLVMValueRef arg_elems[LP_MAX_FUNC_ARGS];
 362       LLVMValueRef res_elem;
 363       for(j = 0; j < num_args; ++j)
 364          arg_elems[j] = LLVMBuildExtractElement(builder, args[j], index, "");
 365       res_elem = lp_build_intrinsic(builder, name, ret_elem_type, arg_elems, num_args, 0);
 366       res = LLVMBuildInsertElement(builder, res, res_elem, index, "");
 367    }
 368
 369    return res;
 370 }
 371
 372
 373 LLVMValueRef
 374 lp_build_intrinsic_map_unary(struct gallivm_state *gallivm,
 375                              const char *name,
 376                              LLVMTypeRef ret_type,
 377                              LLVMValueRef a)
 378 {
 379    return lp_build_intrinsic_map(gallivm, name, ret_type, &a, 1);
 380 }
 381
 382
 383 LLVMValueRef
 384 lp_build_intrinsic_map_binary(struct gallivm_state *gallivm,
 385                               const char *name,
 386                               LLVMTypeRef ret_type,
 387                               LLVMValueRef a,
 388                               LLVMValueRef b)
 389 {
 390    LLVMValueRef args[2];
 391
 392    args[0] = a;
 393    args[1] = b;
 394
 395    return lp_build_intrinsic_map(gallivm, name, ret_type, args, 2);
 396 }
 397
 398