1 /**************************************************************************
3 * Copyright 2010 VMware, Inc.
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
18 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
19 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
20 * USE OR OTHER DEALINGS IN THE SOFTWARE.
22 * The above copyright notice and this permission notice (including the
23 * next paragraph) shall be included in all copies or substantial portions
26 **************************************************************************/
30 * The purpose of this module is to expose LLVM functionality not available
31 * through the C++ bindings.
35 // Undef these vars just to silence warnings
36 #undef PACKAGE_BUGREPORT
39 #undef PACKAGE_TARNAME
40 #undef PACKAGE_VERSION
45 #include <llvm/Config/llvm-config.h>
47 #if LLVM_VERSION_MAJOR < 7
48 // Workaround http://llvm.org/PR23628
49 #pragma push_macro("DEBUG")
53 #include <llvm/Config/llvm-config.h>
54 #include <llvm-c/Core.h>
55 #include <llvm-c/Support.h>
56 #include <llvm-c/ExecutionEngine.h>
57 #include <llvm/Target/TargetOptions.h>
58 #include <llvm/ExecutionEngine/ExecutionEngine.h>
59 #include <llvm/ADT/Triple.h>
60 #include <llvm/Analysis/TargetLibraryInfo.h>
61 #include <llvm/ExecutionEngine/SectionMemoryManager.h>
62 #include <llvm/Support/CommandLine.h>
63 #include <llvm/Support/Host.h>
64 #include <llvm/Support/PrettyStackTrace.h>
66 #include <llvm/Support/TargetSelect.h>
68 #if LLVM_VERSION_MAJOR < 11
69 #include <llvm/IR/CallSite.h>
71 #include <llvm/IR/IRBuilder.h>
72 #include <llvm/IR/Module.h>
73 #include <llvm/Support/CBindingWrapping.h>
75 #include <llvm/Config/llvm-config.h>
76 #if LLVM_USE_INTEL_JITEVENTS
77 #include <llvm/ExecutionEngine/JITEventListener.h>
80 #if LLVM_VERSION_MAJOR < 7
81 // Workaround http://llvm.org/PR23628
82 #pragma pop_macro("DEBUG")
85 #include "c11/threads.h"
86 #include "os/os_thread.h"
87 #include "pipe/p_config.h"
88 #include "util/u_debug.h"
89 #include "util/u_cpu_detect.h"
91 #include "lp_bld_misc.h"
92 #include "lp_bld_debug.h"
96 class LLVMEnsureMultithreaded
{
98 LLVMEnsureMultithreaded()
100 if (!LLVMIsMultithreaded()) {
101 LLVMStartMultithreaded();
106 static LLVMEnsureMultithreaded lLVMEnsureMultithreaded
;
110 static once_flag init_native_targets_once_flag
= ONCE_FLAG_INIT
;
112 static void init_native_targets()
114 // If we have a native target, initialize it to ensure it is linked in and
115 // usable by the JIT.
116 llvm::InitializeNativeTarget();
118 llvm::InitializeNativeTargetAsmPrinter();
120 llvm::InitializeNativeTargetDisassembler();
123 char *env_llc_options
= getenv("GALLIVM_LLC_OPTIONS");
124 if (env_llc_options
) {
126 char *options
[64] = {(char *) "llc"}; // Warning without cast
128 for (n
= 0, option
= strtok(env_llc_options
, " "); option
; n
++, option
= strtok(NULL
, " ")) {
129 options
[n
+ 1] = option
;
131 if (gallivm_debug
& (GALLIVM_DEBUG_IR
| GALLIVM_DEBUG_ASM
| GALLIVM_DEBUG_DUMP_BC
)) {
132 debug_printf("llc additional options (%d):\n", n
);
133 for (int i
= 1; i
<= n
; i
++)
134 debug_printf("\t%s\n", options
[i
]);
137 LLVMParseCommandLineOptions(n
+ 1, options
, NULL
);
144 lp_set_target_options(void)
146 /* The llvm target registry is not thread-safe, so drivers and gallium frontends
147 * that want to initialize targets should use the lp_set_target_options()
148 * function to safely initialize targets.
150 * LLVM targets should be initialized before the driver or gallium frontend tries
151 * to access the registry.
153 call_once(&init_native_targets_once_flag
, init_native_targets
);
157 LLVMTargetLibraryInfoRef
158 gallivm_create_target_library_info(const char *triple
)
160 return reinterpret_cast<LLVMTargetLibraryInfoRef
>(
161 new llvm::TargetLibraryInfoImpl(
162 llvm::Triple(triple
)));
167 gallivm_dispose_target_library_info(LLVMTargetLibraryInfoRef library_info
)
169 delete reinterpret_cast<
170 llvm::TargetLibraryInfoImpl
175 typedef llvm::RTDyldMemoryManager BaseMemoryManager
;
179 * Delegating is tedious but the default manager class is hidden in an
180 * anonymous namespace in LLVM, so we cannot just derive from it to change
183 class DelegatingJITMemoryManager
: public BaseMemoryManager
{
186 virtual BaseMemoryManager
*mgr() const = 0;
190 * From RTDyldMemoryManager
192 virtual uint8_t *allocateCodeSection(uintptr_t Size
,
195 llvm::StringRef SectionName
) {
196 return mgr()->allocateCodeSection(Size
, Alignment
, SectionID
,
199 virtual uint8_t *allocateDataSection(uintptr_t Size
,
202 llvm::StringRef SectionName
,
204 return mgr()->allocateDataSection(Size
, Alignment
, SectionID
,
208 virtual void registerEHFrames(uint8_t *Addr
, uint64_t LoadAddr
, size_t Size
) {
209 mgr()->registerEHFrames(Addr
, LoadAddr
, Size
);
211 #if LLVM_VERSION_MAJOR >= 5
212 virtual void deregisterEHFrames() {
213 mgr()->deregisterEHFrames();
216 virtual void deregisterEHFrames(uint8_t *Addr
, uint64_t LoadAddr
, size_t Size
) {
217 mgr()->deregisterEHFrames(Addr
, LoadAddr
, Size
);
220 virtual void *getPointerToNamedFunction(const std::string
&Name
,
221 bool AbortOnFailure
=true) {
222 return mgr()->getPointerToNamedFunction(Name
, AbortOnFailure
);
224 virtual bool finalizeMemory(std::string
*ErrMsg
= 0) {
225 return mgr()->finalizeMemory(ErrMsg
);
231 * Delegate memory management to one shared manager for more efficient use
232 * of memory than creating a separate pool for each LLVM engine.
233 * Keep generated code until freeGeneratedCode() is called, instead of when
234 * memory manager is destroyed, which happens during engine destruction.
235 * This allows additional memory savings as we don't have to keep the engine
236 * around in order to use the code.
237 * All methods are delegated to the shared manager except destruction and
238 * deallocating code. For the latter we just remember what needs to be
239 * deallocated later. The shared manager is deleted once it is empty.
241 class ShaderMemoryManager
: public DelegatingJITMemoryManager
{
243 BaseMemoryManager
*TheMM
;
245 struct GeneratedCode
{
246 typedef std::vector
<void *> Vec
;
247 Vec FunctionBody
, ExceptionTable
;
248 BaseMemoryManager
*TheMM
;
250 GeneratedCode(BaseMemoryManager
*MM
) {
260 BaseMemoryManager
*mgr() const {
266 ShaderMemoryManager(BaseMemoryManager
* MM
) {
268 code
= new GeneratedCode(MM
);
271 virtual ~ShaderMemoryManager() {
273 * 'code' is purposely not deleted. It is the user's responsibility
274 * to call getGeneratedCode() and freeGeneratedCode().
278 struct lp_generated_code
*getGeneratedCode() {
279 return (struct lp_generated_code
*) code
;
282 static void freeGeneratedCode(struct lp_generated_code
*code
) {
283 delete (GeneratedCode
*) code
;
286 virtual void deallocateFunctionBody(void *Body
) {
287 // remember for later deallocation
288 code
->FunctionBody
.push_back(Body
);
294 * Same as LLVMCreateJITCompilerForModule, but:
295 * - allows using MCJIT and enabling AVX feature where available.
296 * - set target options
299 * - llvm/lib/ExecutionEngine/ExecutionEngineBindings.cpp
300 * - llvm/tools/lli/lli.cpp
301 * - http://markmail.org/message/ttkuhvgj4cxxy2on#query:+page:1+mid:aju2dggerju3ivd3+state:results
305 lp_build_create_jit_compiler_for_module(LLVMExecutionEngineRef
*OutJIT
,
306 lp_generated_code
**OutCode
,
308 LLVMMCJITMemoryManagerRef CMM
,
312 using namespace llvm
;
315 EngineBuilder
builder(std::unique_ptr
<Module
>(unwrap(M
)));
318 * LLVM 3.1+ haven't more "extern unsigned llvm::StackAlignmentOverride" and
319 * friends for configuring code generation options, like stack alignment.
321 TargetOptions options
;
322 #if defined(PIPE_ARCH_X86)
323 options
.StackAlignmentOverride
= 4;
326 builder
.setEngineKind(EngineKind::JIT
)
328 .setTargetOptions(options
)
329 .setOptLevel((CodeGenOpt::Level
)OptLevel
);
333 * MCJIT works on Windows, but currently only through ELF object format.
335 * XXX: We could use `LLVM_HOST_TRIPLE "-elf"` but LLVM_HOST_TRIPLE has
336 * different strings for MinGW/MSVC, so better play it safe and be
340 LLVMSetTarget(M
, "x86_64-pc-win32-elf");
342 LLVMSetTarget(M
, "i686-pc-win32-elf");
346 llvm::SmallVector
<std::string
, 16> MAttrs
;
348 #if LLVM_VERSION_MAJOR >= 4 && (defined(PIPE_ARCH_X86) || defined(PIPE_ARCH_X86_64) || defined(PIPE_ARCH_ARM))
349 /* llvm-3.3+ implements sys::getHostCPUFeatures for Arm
350 * and llvm-3.7+ for x86, which allows us to enable/disable
351 * code generation based on the results of cpuid on these
354 llvm::StringMap
<bool> features
;
355 llvm::sys::getHostCPUFeatures(features
);
357 for (StringMapIterator
<bool> f
= features
.begin();
360 MAttrs
.push_back(((*f
).second
? "+" : "-") + (*f
).first().str());
362 #elif defined(PIPE_ARCH_X86) || defined(PIPE_ARCH_X86_64)
364 * We need to unset attributes because sometimes LLVM mistakenly assumes
365 * certain features are present given the processor name.
367 * https://bugs.freedesktop.org/show_bug.cgi?id=92214
368 * http://llvm.org/PR25021
369 * http://llvm.org/PR19429
370 * http://llvm.org/PR16721
372 MAttrs
.push_back(util_cpu_caps
.has_sse
? "+sse" : "-sse" );
373 MAttrs
.push_back(util_cpu_caps
.has_sse2
? "+sse2" : "-sse2" );
374 MAttrs
.push_back(util_cpu_caps
.has_sse3
? "+sse3" : "-sse3" );
375 MAttrs
.push_back(util_cpu_caps
.has_ssse3
? "+ssse3" : "-ssse3" );
376 MAttrs
.push_back(util_cpu_caps
.has_sse4_1
? "+sse4.1" : "-sse4.1");
377 MAttrs
.push_back(util_cpu_caps
.has_sse4_2
? "+sse4.2" : "-sse4.2");
379 * AVX feature is not automatically detected from CPUID by the X86 target
380 * yet, because the old (yet default) JIT engine is not capable of
381 * emitting the opcodes. On newer llvm versions it is and at least some
382 * versions (tested with 3.3) will emit avx opcodes without this anyway.
384 MAttrs
.push_back(util_cpu_caps
.has_avx
? "+avx" : "-avx");
385 MAttrs
.push_back(util_cpu_caps
.has_f16c
? "+f16c" : "-f16c");
386 MAttrs
.push_back(util_cpu_caps
.has_fma
? "+fma" : "-fma");
387 MAttrs
.push_back(util_cpu_caps
.has_avx2
? "+avx2" : "-avx2");
388 /* disable avx512 and all subvariants */
389 MAttrs
.push_back("-avx512cd");
390 MAttrs
.push_back("-avx512er");
391 MAttrs
.push_back("-avx512f");
392 MAttrs
.push_back("-avx512pf");
393 MAttrs
.push_back("-avx512bw");
394 MAttrs
.push_back("-avx512dq");
395 MAttrs
.push_back("-avx512vl");
397 #if defined(PIPE_ARCH_ARM)
398 if (!util_cpu_caps
.has_neon
) {
399 MAttrs
.push_back("-neon");
400 MAttrs
.push_back("-crypto");
401 MAttrs
.push_back("-vfp2");
405 #if defined(PIPE_ARCH_PPC)
406 MAttrs
.push_back(util_cpu_caps
.has_altivec
? "+altivec" : "-altivec");
407 #if (LLVM_VERSION_MAJOR < 4)
409 * Make sure VSX instructions are disabled
411 * https://llvm.org/bugs/show_bug.cgi?id=25503#c7 (fixed in 3.8.1)
412 * https://llvm.org/bugs/show_bug.cgi?id=26775 (fixed in 3.8.1)
413 * https://llvm.org/bugs/show_bug.cgi?id=33531 (fixed in 4.0)
414 * https://llvm.org/bugs/show_bug.cgi?id=34647 (llc performance on certain unusual shader IR; intro'd in 4.0, pending as of 5.0)
416 if (util_cpu_caps
.has_altivec
) {
417 MAttrs
.push_back("-vsx");
421 * Bug 25503 is fixed, by the same fix that fixed
422 * bug 26775, in versions of LLVM later than 3.8 (starting with 3.8.1).
423 * BZ 33531 actually comprises more than one bug, all of
424 * which are fixed in LLVM 4.0.
426 * With LLVM 4.0 or higher:
427 * Make sure VSX instructions are ENABLED (if supported), unless
428 * VSX instructions are explicitly enabled/disabled via GALLIVM_VSX=1 or 0.
430 if (util_cpu_caps
.has_altivec
) {
431 MAttrs
.push_back(util_cpu_caps
.has_vsx
? "+vsx" : "-vsx");
436 builder
.setMAttrs(MAttrs
);
438 if (gallivm_debug
& (GALLIVM_DEBUG_IR
| GALLIVM_DEBUG_ASM
| GALLIVM_DEBUG_DUMP_BC
)) {
439 int n
= MAttrs
.size();
441 debug_printf("llc -mattr option(s): ");
442 for (int i
= 0; i
< n
; i
++)
443 debug_printf("%s%s", MAttrs
[i
].c_str(), (i
< n
- 1) ? "," : "");
448 StringRef MCPU
= llvm::sys::getHostCPUName();
450 * The cpu bits are no longer set automatically, so need to set mcpu manually.
451 * Note that the MAttrs set above will be sort of ignored (since we should
452 * not set any which would not be set by specifying the cpu anyway).
453 * It ought to be safe though since getHostCPUName() should include bits
454 * not only from the cpu but environment as well (for instance if it's safe
455 * to use avx instructions which need OS support). According to
456 * http://llvm.org/bugs/show_bug.cgi?id=19429 however if I understand this
457 * right it may be necessary to specify older cpu (or disable mattrs) though
458 * when not using MCJIT so no instructions are generated which the old JIT
459 * can't handle. Not entirely sure if we really need to do anything yet.
462 #ifdef PIPE_ARCH_PPC_64
464 * Large programs, e.g. gnome-shell and firefox, may tax the addressability
465 * of the Medium code model once dynamically generated JIT-compiled shader
466 * programs are linked in and relocated. Yet the default code model as of
467 * LLVM 8 is Medium or even Small.
468 * The cost of changing from Medium to Large is negligible:
469 * - an additional 8-byte pointer stored immediately before the shader entrypoint;
470 * - change an add-immediate (addis) instruction to a load (ld).
472 builder
.setCodeModel(CodeModel::Large
);
474 #if UTIL_ARCH_LITTLE_ENDIAN
476 * Versions of LLVM prior to 4.0 lacked a table entry for "POWER8NVL",
477 * resulting in (big-endian) "generic" being returned on
478 * little-endian Power8NVL systems. The result was that code that
479 * attempted to load the least significant 32 bits of a 64-bit quantity
480 * from memory loaded the wrong half. This resulted in failures in some
482 * .../arb_gpu_shader_fp64/execution/conversion/frag-conversion-explicit-double-uint
484 if (MCPU
== "generic")
488 builder
.setMCPU(MCPU
);
489 if (gallivm_debug
& (GALLIVM_DEBUG_IR
| GALLIVM_DEBUG_ASM
| GALLIVM_DEBUG_DUMP_BC
)) {
490 debug_printf("llc -mcpu option: %s\n", MCPU
.str().c_str());
493 ShaderMemoryManager
*MM
= NULL
;
494 BaseMemoryManager
* JMM
= reinterpret_cast<BaseMemoryManager
*>(CMM
);
495 MM
= new ShaderMemoryManager(JMM
);
496 *OutCode
= MM
->getGeneratedCode();
498 builder
.setMCJITMemoryManager(std::unique_ptr
<RTDyldMemoryManager
>(MM
));
499 MM
= NULL
; // ownership taken by std::unique_ptr
501 ExecutionEngine
*JIT
;
503 JIT
= builder
.create();
504 #if LLVM_USE_INTEL_JITEVENTS
505 JITEventListener
*JEL
= JITEventListener::createIntelJITEventListener();
506 JIT
->RegisterJITEventListener(JEL
);
512 lp_free_generated_code(*OutCode
);
515 *OutError
= strdup(Error
.c_str());
522 lp_free_generated_code(struct lp_generated_code
*code
)
524 ShaderMemoryManager::freeGeneratedCode(code
);
528 LLVMMCJITMemoryManagerRef
529 lp_get_default_memory_manager()
531 BaseMemoryManager
*mm
;
532 mm
= new llvm::SectionMemoryManager();
533 return reinterpret_cast<LLVMMCJITMemoryManagerRef
>(mm
);
538 lp_free_memory_manager(LLVMMCJITMemoryManagerRef memorymgr
)
540 delete reinterpret_cast<BaseMemoryManager
*>(memorymgr
);
543 extern "C" LLVMValueRef
544 lp_get_called_value(LLVMValueRef call
)
546 return LLVMGetCalledValue(call
);
550 lp_is_function(LLVMValueRef v
)
552 return LLVMGetValueKind(v
) == LLVMFunctionValueKind
;