src/gallium/auxiliary/gallivm/lp_bld_misc.cpp

   1 /**************************************************************************
   2  *
   3  * Copyright 2010 VMware, Inc.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  18  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  19  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  20  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * The above copyright notice and this permission notice (including the
  23  * next paragraph) shall be included in all copies or substantial portions
  24  * of the Software.
  25  *
  26  **************************************************************************/
  27
  28
  29 /**
  30  * The purpose of this module is to expose LLVM functionality not available
  31  * through the C++ bindings.
  32  */
  33
  34
  35 // Undef these vars just to silence warnings
  36 #undef PACKAGE_BUGREPORT
  37 #undef PACKAGE_NAME
  38 #undef PACKAGE_STRING
  39 #undef PACKAGE_TARNAME
  40 #undef PACKAGE_VERSION
  41
  42
  43 #include <stddef.h>
  44
  45 #include <llvm/Config/llvm-config.h>
  46
  47 #if LLVM_VERSION_MAJOR < 7
  48 // Workaround http://llvm.org/PR23628
  49 #pragma push_macro("DEBUG")
  50 #undef DEBUG
  51 #endif
  52
  53 #include <llvm/Config/llvm-config.h>
  54 #include <llvm-c/Core.h>
  55 #include <llvm-c/Support.h>
  56 #include <llvm-c/ExecutionEngine.h>
  57 #include <llvm/Target/TargetOptions.h>
  58 #include <llvm/ExecutionEngine/ExecutionEngine.h>
  59 #include <llvm/ADT/Triple.h>
  60 #include <llvm/Analysis/TargetLibraryInfo.h>
  61 #include <llvm/ExecutionEngine/SectionMemoryManager.h>
  62 #include <llvm/Support/CommandLine.h>
  63 #include <llvm/Support/Host.h>
  64 #include <llvm/Support/PrettyStackTrace.h>
  65
  66 #include <llvm/Support/TargetSelect.h>
  67
  68 #if LLVM_VERSION_MAJOR < 11
  69 #include <llvm/IR/CallSite.h>
  70 #endif
  71 #include <llvm/IR/IRBuilder.h>
  72 #include <llvm/IR/Module.h>
  73 #include <llvm/Support/CBindingWrapping.h>
  74
  75 #include <llvm/Config/llvm-config.h>
  76 #if LLVM_USE_INTEL_JITEVENTS
  77 #include <llvm/ExecutionEngine/JITEventListener.h>
  78 #endif
  79
  80 #if LLVM_VERSION_MAJOR < 7
  81 // Workaround http://llvm.org/PR23628
  82 #pragma pop_macro("DEBUG")
  83 #endif
  84
  85 #include "c11/threads.h"
  86 #include "os/os_thread.h"
  87 #include "pipe/p_config.h"
  88 #include "util/u_debug.h"
  89 #include "util/u_cpu_detect.h"
  90
  91 #include "lp_bld_misc.h"
  92 #include "lp_bld_debug.h"
  93
  94 namespace {
  95
  96 class LLVMEnsureMultithreaded {
  97 public:
  98    LLVMEnsureMultithreaded()
  99    {
 100       if (!LLVMIsMultithreaded()) {
 101          LLVMStartMultithreaded();
 102       }
 103    }
 104 };
 105
 106 static LLVMEnsureMultithreaded lLVMEnsureMultithreaded;
 107
 108 }
 109
 110 static once_flag init_native_targets_once_flag = ONCE_FLAG_INIT;
 111
 112 static void init_native_targets()
 113 {
 114    // If we have a native target, initialize it to ensure it is linked in and
 115    // usable by the JIT.
 116    llvm::InitializeNativeTarget();
 117
 118    llvm::InitializeNativeTargetAsmPrinter();
 119
 120    llvm::InitializeNativeTargetDisassembler();
 121 #if DEBUG
 122    {
 123       char *env_llc_options = getenv("GALLIVM_LLC_OPTIONS");
 124       if (env_llc_options) {
 125          char *option;
 126          char *options[64] = {(char *) "llc"};      // Warning without cast
 127          int   n;
 128          for (n = 0, option = strtok(env_llc_options, " "); option; n++, option = strtok(NULL, " ")) {
 129             options[n + 1] = option;
 130          }
 131          if (gallivm_debug & (GALLIVM_DEBUG_IR | GALLIVM_DEBUG_ASM | GALLIVM_DEBUG_DUMP_BC)) {
 132             debug_printf("llc additional options (%d):\n", n);
 133             for (int i = 1; i <= n; i++)
 134                debug_printf("\t%s\n", options[i]);
 135             debug_printf("\n");
 136          }
 137          LLVMParseCommandLineOptions(n + 1, options, NULL);
 138       }
 139    }
 140 #endif
 141 }
 142
 143 extern "C" void
 144 lp_set_target_options(void)
 145 {
 146    /* The llvm target registry is not thread-safe, so drivers and state-trackers
 147     * that want to initialize targets should use the lp_set_target_options()
 148     * function to safely initialize targets.
 149     *
 150     * LLVM targets should be initialized before the driver or state-tracker tries
 151     * to access the registry.
 152     */
 153    call_once(&init_native_targets_once_flag, init_native_targets);
 154 }
 155
 156 extern "C"
 157 LLVMTargetLibraryInfoRef
 158 gallivm_create_target_library_info(const char *triple)
 159 {
 160    return reinterpret_cast<LLVMTargetLibraryInfoRef>(
 161    new llvm::TargetLibraryInfoImpl(
 162    llvm::Triple(triple)));
 163 }
 164
 165 extern "C"
 166 void
 167 gallivm_dispose_target_library_info(LLVMTargetLibraryInfoRef library_info)
 168 {
 169    delete reinterpret_cast<
 170    llvm::TargetLibraryInfoImpl
 171    *>(library_info);
 172 }
 173
 174
 175 typedef llvm::RTDyldMemoryManager BaseMemoryManager;
 176
 177
 178 /*
 179  * Delegating is tedious but the default manager class is hidden in an
 180  * anonymous namespace in LLVM, so we cannot just derive from it to change
 181  * its behavior.
 182  */
 183 class DelegatingJITMemoryManager : public BaseMemoryManager {
 184
 185    protected:
 186       virtual BaseMemoryManager *mgr() const = 0;
 187
 188    public:
 189       /*
 190        * From RTDyldMemoryManager
 191        */
 192       virtual uint8_t *allocateCodeSection(uintptr_t Size,
 193                                            unsigned Alignment,
 194                                            unsigned SectionID,
 195                                            llvm::StringRef SectionName) {
 196          return mgr()->allocateCodeSection(Size, Alignment, SectionID,
 197                                            SectionName);
 198       }
 199       virtual uint8_t *allocateDataSection(uintptr_t Size,
 200                                            unsigned Alignment,
 201                                            unsigned SectionID,
 202                                            llvm::StringRef SectionName,
 203                                            bool IsReadOnly) {
 204          return mgr()->allocateDataSection(Size, Alignment, SectionID,
 205                                            SectionName,
 206                                            IsReadOnly);
 207       }
 208       virtual void registerEHFrames(uint8_t *Addr, uint64_t LoadAddr, size_t Size) {
 209          mgr()->registerEHFrames(Addr, LoadAddr, Size);
 210       }
 211 #if LLVM_VERSION_MAJOR >= 5
 212       virtual void deregisterEHFrames() {
 213          mgr()->deregisterEHFrames();
 214       }
 215 #else
 216       virtual void deregisterEHFrames(uint8_t *Addr, uint64_t LoadAddr, size_t Size) {
 217          mgr()->deregisterEHFrames(Addr, LoadAddr, Size);
 218       }
 219 #endif
 220       virtual void *getPointerToNamedFunction(const std::string &Name,
 221                                               bool AbortOnFailure=true) {
 222          return mgr()->getPointerToNamedFunction(Name, AbortOnFailure);
 223       }
 224       virtual bool finalizeMemory(std::string *ErrMsg = 0) {
 225          return mgr()->finalizeMemory(ErrMsg);
 226       }
 227 };
 228
 229
 230 /*
 231  * Delegate memory management to one shared manager for more efficient use
 232  * of memory than creating a separate pool for each LLVM engine.
 233  * Keep generated code until freeGeneratedCode() is called, instead of when
 234  * memory manager is destroyed, which happens during engine destruction.
 235  * This allows additional memory savings as we don't have to keep the engine
 236  * around in order to use the code.
 237  * All methods are delegated to the shared manager except destruction and
 238  * deallocating code.  For the latter we just remember what needs to be
 239  * deallocated later.  The shared manager is deleted once it is empty.
 240  */
 241 class ShaderMemoryManager : public DelegatingJITMemoryManager {
 242
 243    BaseMemoryManager *TheMM;
 244
 245    struct GeneratedCode {
 246       typedef std::vector<void *> Vec;
 247       Vec FunctionBody, ExceptionTable;
 248       BaseMemoryManager *TheMM;
 249
 250       GeneratedCode(BaseMemoryManager *MM) {
 251          TheMM = MM;
 252       }
 253
 254       ~GeneratedCode() {
 255       }
 256    };
 257
 258    GeneratedCode *code;
 259
 260    BaseMemoryManager *mgr() const {
 261       return TheMM;
 262    }
 263
 264    public:
 265
 266       ShaderMemoryManager(BaseMemoryManager* MM) {
 267          TheMM = MM;
 268          code = new GeneratedCode(MM);
 269       }
 270
 271       virtual ~ShaderMemoryManager() {
 272          /*
 273           * 'code' is purposely not deleted.  It is the user's responsibility
 274           * to call getGeneratedCode() and freeGeneratedCode().
 275           */
 276       }
 277
 278       struct lp_generated_code *getGeneratedCode() {
 279          return (struct lp_generated_code *) code;
 280       }
 281
 282       static void freeGeneratedCode(struct lp_generated_code *code) {
 283          delete (GeneratedCode *) code;
 284       }
 285
 286       virtual void deallocateFunctionBody(void *Body) {
 287          // remember for later deallocation
 288          code->FunctionBody.push_back(Body);
 289       }
 290 };
 291
 292
 293 /**
 294  * Same as LLVMCreateJITCompilerForModule, but:
 295  * - allows using MCJIT and enabling AVX feature where available.
 296  * - set target options
 297  *
 298  * See also:
 299  * - llvm/lib/ExecutionEngine/ExecutionEngineBindings.cpp
 300  * - llvm/tools/lli/lli.cpp
 301  * - http://markmail.org/message/ttkuhvgj4cxxy2on#query:+page:1+mid:aju2dggerju3ivd3+state:results
 302  */
 303 extern "C"
 304 LLVMBool
 305 lp_build_create_jit_compiler_for_module(LLVMExecutionEngineRef *OutJIT,
 306                                         lp_generated_code **OutCode,
 307                                         LLVMModuleRef M,
 308                                         LLVMMCJITMemoryManagerRef CMM,
 309                                         unsigned OptLevel,
 310                                         char **OutError)
 311 {
 312    using namespace llvm;
 313
 314    std::string Error;
 315    EngineBuilder builder(std::unique_ptr<Module>(unwrap(M)));
 316
 317    /**
 318     * LLVM 3.1+ haven't more "extern unsigned llvm::StackAlignmentOverride" and
 319     * friends for configuring code generation options, like stack alignment.
 320     */
 321    TargetOptions options;
 322 #if defined(PIPE_ARCH_X86)
 323    options.StackAlignmentOverride = 4;
 324 #endif
 325
 326    builder.setEngineKind(EngineKind::JIT)
 327           .setErrorStr(&Error)
 328           .setTargetOptions(options)
 329           .setOptLevel((CodeGenOpt::Level)OptLevel);
 330
 331 #ifdef _WIN32
 332     /*
 333      * MCJIT works on Windows, but currently only through ELF object format.
 334      *
 335      * XXX: We could use `LLVM_HOST_TRIPLE "-elf"` but LLVM_HOST_TRIPLE has
 336      * different strings for MinGW/MSVC, so better play it safe and be
 337      * explicit.
 338      */
 339 #  ifdef _WIN64
 340     LLVMSetTarget(M, "x86_64-pc-win32-elf");
 341 #  else
 342     LLVMSetTarget(M, "i686-pc-win32-elf");
 343 #  endif
 344 #endif
 345
 346    llvm::SmallVector<std::string, 16> MAttrs;
 347
 348 #if LLVM_VERSION_MAJOR >= 4 && (defined(PIPE_ARCH_X86) || defined(PIPE_ARCH_X86_64) || defined(PIPE_ARCH_ARM))
 349    /* llvm-3.3+ implements sys::getHostCPUFeatures for Arm
 350     * and llvm-3.7+ for x86, which allows us to enable/disable
 351     * code generation based on the results of cpuid on these
 352     * architectures.
 353     */
 354    llvm::StringMap<bool> features;
 355    llvm::sys::getHostCPUFeatures(features);
 356
 357    for (StringMapIterator<bool> f = features.begin();
 358         f != features.end();
 359         ++f) {
 360       MAttrs.push_back(((*f).second ? "+" : "-") + (*f).first().str());
 361    }
 362 #elif defined(PIPE_ARCH_X86) || defined(PIPE_ARCH_X86_64)
 363    /*
 364     * We need to unset attributes because sometimes LLVM mistakenly assumes
 365     * certain features are present given the processor name.
 366     *
 367     * https://bugs.freedesktop.org/show_bug.cgi?id=92214
 368     * http://llvm.org/PR25021
 369     * http://llvm.org/PR19429
 370     * http://llvm.org/PR16721
 371     */
 372    MAttrs.push_back(util_cpu_caps.has_sse    ? "+sse"    : "-sse"   );
 373    MAttrs.push_back(util_cpu_caps.has_sse2   ? "+sse2"   : "-sse2"  );
 374    MAttrs.push_back(util_cpu_caps.has_sse3   ? "+sse3"   : "-sse3"  );
 375    MAttrs.push_back(util_cpu_caps.has_ssse3  ? "+ssse3"  : "-ssse3" );
 376    MAttrs.push_back(util_cpu_caps.has_sse4_1 ? "+sse4.1" : "-sse4.1");
 377    MAttrs.push_back(util_cpu_caps.has_sse4_2 ? "+sse4.2" : "-sse4.2");
 378    /*
 379     * AVX feature is not automatically detected from CPUID by the X86 target
 380     * yet, because the old (yet default) JIT engine is not capable of
 381     * emitting the opcodes. On newer llvm versions it is and at least some
 382     * versions (tested with 3.3) will emit avx opcodes without this anyway.
 383     */
 384    MAttrs.push_back(util_cpu_caps.has_avx  ? "+avx"  : "-avx");
 385    MAttrs.push_back(util_cpu_caps.has_f16c ? "+f16c" : "-f16c");
 386    MAttrs.push_back(util_cpu_caps.has_fma  ? "+fma"  : "-fma");
 387    MAttrs.push_back(util_cpu_caps.has_avx2 ? "+avx2" : "-avx2");
 388    /* disable avx512 and all subvariants */
 389    MAttrs.push_back("-avx512cd");
 390    MAttrs.push_back("-avx512er");
 391    MAttrs.push_back("-avx512f");
 392    MAttrs.push_back("-avx512pf");
 393    MAttrs.push_back("-avx512bw");
 394    MAttrs.push_back("-avx512dq");
 395    MAttrs.push_back("-avx512vl");
 396 #endif
 397 #if defined(PIPE_ARCH_ARM)
 398    if (!util_cpu_caps.has_neon) {
 399       MAttrs.push_back("-neon");
 400       MAttrs.push_back("-crypto");
 401       MAttrs.push_back("-vfp2");
 402    }
 403 #endif
 404
 405 #if defined(PIPE_ARCH_PPC)
 406    MAttrs.push_back(util_cpu_caps.has_altivec ? "+altivec" : "-altivec");
 407 #if (LLVM_VERSION_MAJOR < 4)
 408    /*
 409     * Make sure VSX instructions are disabled
 410     * See LLVM bugs:
 411     * https://llvm.org/bugs/show_bug.cgi?id=25503#c7 (fixed in 3.8.1)
 412     * https://llvm.org/bugs/show_bug.cgi?id=26775 (fixed in 3.8.1)
 413     * https://llvm.org/bugs/show_bug.cgi?id=33531 (fixed in 4.0)
 414     * https://llvm.org/bugs/show_bug.cgi?id=34647 (llc performance on certain unusual shader IR; intro'd in 4.0, pending as of 5.0)
 415     */
 416    if (util_cpu_caps.has_altivec) {
 417       MAttrs.push_back("-vsx");
 418    }
 419 #else
 420    /*
 421     * Bug 25503 is fixed, by the same fix that fixed
 422     * bug 26775, in versions of LLVM later than 3.8 (starting with 3.8.1).
 423     * BZ 33531 actually comprises more than one bug, all of
 424     * which are fixed in LLVM 4.0.
 425     *
 426     * With LLVM 4.0 or higher:
 427     * Make sure VSX instructions are ENABLED (if supported), unless
 428     * VSX instructions are explicitly enabled/disabled via GALLIVM_VSX=1 or 0.
 429     */
 430    if (util_cpu_caps.has_altivec) {
 431       MAttrs.push_back(util_cpu_caps.has_vsx ? "+vsx" : "-vsx");
 432    }
 433 #endif
 434 #endif
 435
 436    builder.setMAttrs(MAttrs);
 437
 438    if (gallivm_debug & (GALLIVM_DEBUG_IR | GALLIVM_DEBUG_ASM | GALLIVM_DEBUG_DUMP_BC)) {
 439       int n = MAttrs.size();
 440       if (n > 0) {
 441          debug_printf("llc -mattr option(s): ");
 442          for (int i = 0; i < n; i++)
 443             debug_printf("%s%s", MAttrs[i].c_str(), (i < n - 1) ? "," : "");
 444          debug_printf("\n");
 445       }
 446    }
 447
 448    StringRef MCPU = llvm::sys::getHostCPUName();
 449    /*
 450     * The cpu bits are no longer set automatically, so need to set mcpu manually.
 451     * Note that the MAttrs set above will be sort of ignored (since we should
 452     * not set any which would not be set by specifying the cpu anyway).
 453     * It ought to be safe though since getHostCPUName() should include bits
 454     * not only from the cpu but environment as well (for instance if it's safe
 455     * to use avx instructions which need OS support). According to
 456     * http://llvm.org/bugs/show_bug.cgi?id=19429 however if I understand this
 457     * right it may be necessary to specify older cpu (or disable mattrs) though
 458     * when not using MCJIT so no instructions are generated which the old JIT
 459     * can't handle. Not entirely sure if we really need to do anything yet.
 460     */
 461
 462 #ifdef PIPE_ARCH_PPC_64
 463    /*
 464     * Large programs, e.g. gnome-shell and firefox, may tax the addressability
 465     * of the Medium code model once dynamically generated JIT-compiled shader
 466     * programs are linked in and relocated.  Yet the default code model as of
 467     * LLVM 8 is Medium or even Small.
 468     * The cost of changing from Medium to Large is negligible:
 469     * - an additional 8-byte pointer stored immediately before the shader entrypoint;
 470     * - change an add-immediate (addis) instruction to a load (ld).
 471     */
 472    builder.setCodeModel(CodeModel::Large);
 473
 474 #if UTIL_ARCH_LITTLE_ENDIAN
 475    /*
 476     * Versions of LLVM prior to 4.0 lacked a table entry for "POWER8NVL",
 477     * resulting in (big-endian) "generic" being returned on
 478     * little-endian Power8NVL systems.  The result was that code that
 479     * attempted to load the least significant 32 bits of a 64-bit quantity
 480     * from memory loaded the wrong half.  This resulted in failures in some
 481     * Piglit tests, e.g.
 482     * .../arb_gpu_shader_fp64/execution/conversion/frag-conversion-explicit-double-uint
 483     */
 484    if (MCPU == "generic")
 485       MCPU = "pwr8";
 486 #endif
 487 #endif
 488    builder.setMCPU(MCPU);
 489    if (gallivm_debug & (GALLIVM_DEBUG_IR | GALLIVM_DEBUG_ASM | GALLIVM_DEBUG_DUMP_BC)) {
 490       debug_printf("llc -mcpu option: %s\n", MCPU.str().c_str());
 491    }
 492
 493    ShaderMemoryManager *MM = NULL;
 494    BaseMemoryManager* JMM = reinterpret_cast<BaseMemoryManager*>(CMM);
 495    MM = new ShaderMemoryManager(JMM);
 496    *OutCode = MM->getGeneratedCode();
 497
 498    builder.setMCJITMemoryManager(std::unique_ptr<RTDyldMemoryManager>(MM));
 499    MM = NULL; // ownership taken by std::unique_ptr
 500
 501    ExecutionEngine *JIT;
 502
 503    JIT = builder.create();
 504 #if LLVM_USE_INTEL_JITEVENTS
 505    JITEventListener *JEL = JITEventListener::createIntelJITEventListener();
 506    JIT->RegisterJITEventListener(JEL);
 507 #endif
 508    if (JIT) {
 509       *OutJIT = wrap(JIT);
 510       return 0;
 511    }
 512    lp_free_generated_code(*OutCode);
 513    *OutCode = 0;
 514    delete MM;
 515    *OutError = strdup(Error.c_str());
 516    return 1;
 517 }
 518
 519
 520 extern "C"
 521 void
 522 lp_free_generated_code(struct lp_generated_code *code)
 523 {
 524    ShaderMemoryManager::freeGeneratedCode(code);
 525 }
 526
 527 extern "C"
 528 LLVMMCJITMemoryManagerRef
 529 lp_get_default_memory_manager()
 530 {
 531    BaseMemoryManager *mm;
 532    mm = new llvm::SectionMemoryManager();
 533    return reinterpret_cast<LLVMMCJITMemoryManagerRef>(mm);
 534 }
 535
 536 extern "C"
 537 void
 538 lp_free_memory_manager(LLVMMCJITMemoryManagerRef memorymgr)
 539 {
 540    delete reinterpret_cast<BaseMemoryManager*>(memorymgr);
 541 }
 542
 543 extern "C" LLVMValueRef
 544 lp_get_called_value(LLVMValueRef call)
 545 {
 546         return LLVMGetCalledValue(call);
 547 }
 548
 549 extern "C" bool
 550 lp_is_function(LLVMValueRef v)
 551 {
 552         return LLVMGetValueKind(v) == LLVMFunctionValueKind;
 553 }