9b75676a4e295324bb71c32fc5d1c899825de36b
[mesa.git] / src / gallium / auxiliary / gallivm / lp_bld_misc.cpp
1 /**************************************************************************
2 *
3 * Copyright 2010 VMware, Inc.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
18 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
19 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
20 * USE OR OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * The above copyright notice and this permission notice (including the
23 * next paragraph) shall be included in all copies or substantial portions
24 * of the Software.
25 *
26 **************************************************************************/
27
28
29 /**
30 * The purpose of this module is to expose LLVM functionality not available
31 * through the C++ bindings.
32 */
33
34
35 // Undef these vars just to silence warnings
36 #undef PACKAGE_BUGREPORT
37 #undef PACKAGE_NAME
38 #undef PACKAGE_STRING
39 #undef PACKAGE_TARNAME
40 #undef PACKAGE_VERSION
41
42
43 #include <stddef.h>
44
45 #include <llvm/Config/llvm-config.h>
46
47 #if LLVM_VERSION_MAJOR < 7
48 // Workaround http://llvm.org/PR23628
49 #pragma push_macro("DEBUG")
50 #undef DEBUG
51 #endif
52
53 #include <llvm/Config/llvm-config.h>
54 #include <llvm-c/Core.h>
55 #include <llvm-c/Support.h>
56 #include <llvm-c/ExecutionEngine.h>
57 #include <llvm/Target/TargetOptions.h>
58 #include <llvm/ExecutionEngine/ExecutionEngine.h>
59 #include <llvm/ADT/Triple.h>
60 #include <llvm/Analysis/TargetLibraryInfo.h>
61 #include <llvm/ExecutionEngine/SectionMemoryManager.h>
62 #include <llvm/Support/CommandLine.h>
63 #include <llvm/Support/Host.h>
64 #include <llvm/Support/PrettyStackTrace.h>
65 #include <llvm/ExecutionEngine/ObjectCache.h>
66 #include <llvm/Support/TargetSelect.h>
67
68 #if LLVM_VERSION_MAJOR < 11
69 #include <llvm/IR/CallSite.h>
70 #endif
71 #include <llvm/IR/IRBuilder.h>
72 #include <llvm/IR/Module.h>
73 #include <llvm/Support/CBindingWrapping.h>
74
75 #include <llvm/Config/llvm-config.h>
76 #if LLVM_USE_INTEL_JITEVENTS
77 #include <llvm/ExecutionEngine/JITEventListener.h>
78 #endif
79
80 #if LLVM_VERSION_MAJOR < 7
81 // Workaround http://llvm.org/PR23628
82 #pragma pop_macro("DEBUG")
83 #endif
84
85 #include "c11/threads.h"
86 #include "os/os_thread.h"
87 #include "pipe/p_config.h"
88 #include "util/u_debug.h"
89 #include "util/u_cpu_detect.h"
90
91 #include "lp_bld_misc.h"
92 #include "lp_bld_debug.h"
93
94 namespace {
95
96 class LLVMEnsureMultithreaded {
97 public:
98 LLVMEnsureMultithreaded()
99 {
100 if (!LLVMIsMultithreaded()) {
101 LLVMStartMultithreaded();
102 }
103 }
104 };
105
106 static LLVMEnsureMultithreaded lLVMEnsureMultithreaded;
107
108 }
109
110 static once_flag init_native_targets_once_flag = ONCE_FLAG_INIT;
111
112 static void init_native_targets()
113 {
114 // If we have a native target, initialize it to ensure it is linked in and
115 // usable by the JIT.
116 llvm::InitializeNativeTarget();
117
118 llvm::InitializeNativeTargetAsmPrinter();
119
120 llvm::InitializeNativeTargetDisassembler();
121 #if DEBUG
122 {
123 char *env_llc_options = getenv("GALLIVM_LLC_OPTIONS");
124 if (env_llc_options) {
125 char *option;
126 char *options[64] = {(char *) "llc"}; // Warning without cast
127 int n;
128 for (n = 0, option = strtok(env_llc_options, " "); option; n++, option = strtok(NULL, " ")) {
129 options[n + 1] = option;
130 }
131 if (gallivm_debug & (GALLIVM_DEBUG_IR | GALLIVM_DEBUG_ASM | GALLIVM_DEBUG_DUMP_BC)) {
132 debug_printf("llc additional options (%d):\n", n);
133 for (int i = 1; i <= n; i++)
134 debug_printf("\t%s\n", options[i]);
135 debug_printf("\n");
136 }
137 LLVMParseCommandLineOptions(n + 1, options, NULL);
138 }
139 }
140 #endif
141 }
142
143 extern "C" void
144 lp_set_target_options(void)
145 {
146 /* The llvm target registry is not thread-safe, so drivers and gallium frontends
147 * that want to initialize targets should use the lp_set_target_options()
148 * function to safely initialize targets.
149 *
150 * LLVM targets should be initialized before the driver or gallium frontend tries
151 * to access the registry.
152 */
153 call_once(&init_native_targets_once_flag, init_native_targets);
154 }
155
156 extern "C"
157 LLVMTargetLibraryInfoRef
158 gallivm_create_target_library_info(const char *triple)
159 {
160 return reinterpret_cast<LLVMTargetLibraryInfoRef>(
161 new llvm::TargetLibraryInfoImpl(
162 llvm::Triple(triple)));
163 }
164
165 extern "C"
166 void
167 gallivm_dispose_target_library_info(LLVMTargetLibraryInfoRef library_info)
168 {
169 delete reinterpret_cast<
170 llvm::TargetLibraryInfoImpl
171 *>(library_info);
172 }
173
174
175 typedef llvm::RTDyldMemoryManager BaseMemoryManager;
176
177
178 /*
179 * Delegating is tedious but the default manager class is hidden in an
180 * anonymous namespace in LLVM, so we cannot just derive from it to change
181 * its behavior.
182 */
183 class DelegatingJITMemoryManager : public BaseMemoryManager {
184
185 protected:
186 virtual BaseMemoryManager *mgr() const = 0;
187
188 public:
189 /*
190 * From RTDyldMemoryManager
191 */
192 virtual uint8_t *allocateCodeSection(uintptr_t Size,
193 unsigned Alignment,
194 unsigned SectionID,
195 llvm::StringRef SectionName) {
196 return mgr()->allocateCodeSection(Size, Alignment, SectionID,
197 SectionName);
198 }
199 virtual uint8_t *allocateDataSection(uintptr_t Size,
200 unsigned Alignment,
201 unsigned SectionID,
202 llvm::StringRef SectionName,
203 bool IsReadOnly) {
204 return mgr()->allocateDataSection(Size, Alignment, SectionID,
205 SectionName,
206 IsReadOnly);
207 }
208 virtual void registerEHFrames(uint8_t *Addr, uint64_t LoadAddr, size_t Size) {
209 mgr()->registerEHFrames(Addr, LoadAddr, Size);
210 }
211 #if LLVM_VERSION_MAJOR >= 5
212 virtual void deregisterEHFrames() {
213 mgr()->deregisterEHFrames();
214 }
215 #else
216 virtual void deregisterEHFrames(uint8_t *Addr, uint64_t LoadAddr, size_t Size) {
217 mgr()->deregisterEHFrames(Addr, LoadAddr, Size);
218 }
219 #endif
220 virtual void *getPointerToNamedFunction(const std::string &Name,
221 bool AbortOnFailure=true) {
222 return mgr()->getPointerToNamedFunction(Name, AbortOnFailure);
223 }
224 virtual bool finalizeMemory(std::string *ErrMsg = 0) {
225 return mgr()->finalizeMemory(ErrMsg);
226 }
227 };
228
229
230 /*
231 * Delegate memory management to one shared manager for more efficient use
232 * of memory than creating a separate pool for each LLVM engine.
233 * Keep generated code until freeGeneratedCode() is called, instead of when
234 * memory manager is destroyed, which happens during engine destruction.
235 * This allows additional memory savings as we don't have to keep the engine
236 * around in order to use the code.
237 * All methods are delegated to the shared manager except destruction and
238 * deallocating code. For the latter we just remember what needs to be
239 * deallocated later. The shared manager is deleted once it is empty.
240 */
241 class ShaderMemoryManager : public DelegatingJITMemoryManager {
242
243 BaseMemoryManager *TheMM;
244
245 struct GeneratedCode {
246 typedef std::vector<void *> Vec;
247 Vec FunctionBody, ExceptionTable;
248 BaseMemoryManager *TheMM;
249
250 GeneratedCode(BaseMemoryManager *MM) {
251 TheMM = MM;
252 }
253
254 ~GeneratedCode() {
255 }
256 };
257
258 GeneratedCode *code;
259
260 BaseMemoryManager *mgr() const {
261 return TheMM;
262 }
263
264 public:
265
266 ShaderMemoryManager(BaseMemoryManager* MM) {
267 TheMM = MM;
268 code = new GeneratedCode(MM);
269 }
270
271 virtual ~ShaderMemoryManager() {
272 /*
273 * 'code' is purposely not deleted. It is the user's responsibility
274 * to call getGeneratedCode() and freeGeneratedCode().
275 */
276 }
277
278 struct lp_generated_code *getGeneratedCode() {
279 return (struct lp_generated_code *) code;
280 }
281
282 static void freeGeneratedCode(struct lp_generated_code *code) {
283 delete (GeneratedCode *) code;
284 }
285
286 virtual void deallocateFunctionBody(void *Body) {
287 // remember for later deallocation
288 code->FunctionBody.push_back(Body);
289 }
290 };
291
292 class LPObjectCache : public llvm::ObjectCache {
293 private:
294 bool has_object;
295 struct lp_cached_code *cache_out;
296 public:
297 LPObjectCache(struct lp_cached_code *cache) {
298 cache_out = cache;
299 has_object = false;
300 }
301
302 ~LPObjectCache() {
303 }
304 void notifyObjectCompiled(const llvm::Module *M, llvm::MemoryBufferRef Obj) {
305 const std::string ModuleID = M->getModuleIdentifier();
306 if (has_object)
307 fprintf(stderr, "CACHE ALREADY HAS MODULE OBJECT\n");
308 has_object = true;
309 cache_out->data_size = Obj.getBufferSize();
310 cache_out->data = malloc(cache_out->data_size);
311 memcpy(cache_out->data, Obj.getBufferStart(), cache_out->data_size);
312 }
313
314 virtual std::unique_ptr<llvm::MemoryBuffer> getObject(const llvm::Module *M) {
315 if (cache_out->data_size) {
316 return llvm::MemoryBuffer::getMemBuffer(llvm::StringRef((const char *)cache_out->data, cache_out->data_size), "", false);
317 }
318 return NULL;
319 }
320
321 };
322
323 /**
324 * Same as LLVMCreateJITCompilerForModule, but:
325 * - allows using MCJIT and enabling AVX feature where available.
326 * - set target options
327 *
328 * See also:
329 * - llvm/lib/ExecutionEngine/ExecutionEngineBindings.cpp
330 * - llvm/tools/lli/lli.cpp
331 * - http://markmail.org/message/ttkuhvgj4cxxy2on#query:+page:1+mid:aju2dggerju3ivd3+state:results
332 */
333 extern "C"
334 LLVMBool
335 lp_build_create_jit_compiler_for_module(LLVMExecutionEngineRef *OutJIT,
336 lp_generated_code **OutCode,
337 struct lp_cached_code *cache_out,
338 LLVMModuleRef M,
339 LLVMMCJITMemoryManagerRef CMM,
340 unsigned OptLevel,
341 char **OutError)
342 {
343 using namespace llvm;
344
345 std::string Error;
346 EngineBuilder builder(std::unique_ptr<Module>(unwrap(M)));
347
348 /**
349 * LLVM 3.1+ haven't more "extern unsigned llvm::StackAlignmentOverride" and
350 * friends for configuring code generation options, like stack alignment.
351 */
352 TargetOptions options;
353 #if defined(PIPE_ARCH_X86)
354 options.StackAlignmentOverride = 4;
355 #endif
356
357 builder.setEngineKind(EngineKind::JIT)
358 .setErrorStr(&Error)
359 .setTargetOptions(options)
360 .setOptLevel((CodeGenOpt::Level)OptLevel);
361
362 #ifdef _WIN32
363 /*
364 * MCJIT works on Windows, but currently only through ELF object format.
365 *
366 * XXX: We could use `LLVM_HOST_TRIPLE "-elf"` but LLVM_HOST_TRIPLE has
367 * different strings for MinGW/MSVC, so better play it safe and be
368 * explicit.
369 */
370 # ifdef _WIN64
371 LLVMSetTarget(M, "x86_64-pc-win32-elf");
372 # else
373 LLVMSetTarget(M, "i686-pc-win32-elf");
374 # endif
375 #endif
376
377 llvm::SmallVector<std::string, 16> MAttrs;
378
379 #if LLVM_VERSION_MAJOR >= 4 && (defined(PIPE_ARCH_X86) || defined(PIPE_ARCH_X86_64) || defined(PIPE_ARCH_ARM))
380 /* llvm-3.3+ implements sys::getHostCPUFeatures for Arm
381 * and llvm-3.7+ for x86, which allows us to enable/disable
382 * code generation based on the results of cpuid on these
383 * architectures.
384 */
385 llvm::StringMap<bool> features;
386 llvm::sys::getHostCPUFeatures(features);
387
388 for (StringMapIterator<bool> f = features.begin();
389 f != features.end();
390 ++f) {
391 MAttrs.push_back(((*f).second ? "+" : "-") + (*f).first().str());
392 }
393 #elif defined(PIPE_ARCH_X86) || defined(PIPE_ARCH_X86_64)
394 /*
395 * We need to unset attributes because sometimes LLVM mistakenly assumes
396 * certain features are present given the processor name.
397 *
398 * https://bugs.freedesktop.org/show_bug.cgi?id=92214
399 * http://llvm.org/PR25021
400 * http://llvm.org/PR19429
401 * http://llvm.org/PR16721
402 */
403 MAttrs.push_back(util_cpu_caps.has_sse ? "+sse" : "-sse" );
404 MAttrs.push_back(util_cpu_caps.has_sse2 ? "+sse2" : "-sse2" );
405 MAttrs.push_back(util_cpu_caps.has_sse3 ? "+sse3" : "-sse3" );
406 MAttrs.push_back(util_cpu_caps.has_ssse3 ? "+ssse3" : "-ssse3" );
407 MAttrs.push_back(util_cpu_caps.has_sse4_1 ? "+sse4.1" : "-sse4.1");
408 MAttrs.push_back(util_cpu_caps.has_sse4_2 ? "+sse4.2" : "-sse4.2");
409 /*
410 * AVX feature is not automatically detected from CPUID by the X86 target
411 * yet, because the old (yet default) JIT engine is not capable of
412 * emitting the opcodes. On newer llvm versions it is and at least some
413 * versions (tested with 3.3) will emit avx opcodes without this anyway.
414 */
415 MAttrs.push_back(util_cpu_caps.has_avx ? "+avx" : "-avx");
416 MAttrs.push_back(util_cpu_caps.has_f16c ? "+f16c" : "-f16c");
417 MAttrs.push_back(util_cpu_caps.has_fma ? "+fma" : "-fma");
418 MAttrs.push_back(util_cpu_caps.has_avx2 ? "+avx2" : "-avx2");
419 /* disable avx512 and all subvariants */
420 MAttrs.push_back("-avx512cd");
421 MAttrs.push_back("-avx512er");
422 MAttrs.push_back("-avx512f");
423 MAttrs.push_back("-avx512pf");
424 MAttrs.push_back("-avx512bw");
425 MAttrs.push_back("-avx512dq");
426 MAttrs.push_back("-avx512vl");
427 #endif
428 #if defined(PIPE_ARCH_ARM)
429 if (!util_cpu_caps.has_neon) {
430 MAttrs.push_back("-neon");
431 MAttrs.push_back("-crypto");
432 MAttrs.push_back("-vfp2");
433 }
434 #endif
435
436 #if defined(PIPE_ARCH_PPC)
437 MAttrs.push_back(util_cpu_caps.has_altivec ? "+altivec" : "-altivec");
438 #if (LLVM_VERSION_MAJOR < 4)
439 /*
440 * Make sure VSX instructions are disabled
441 * See LLVM bugs:
442 * https://llvm.org/bugs/show_bug.cgi?id=25503#c7 (fixed in 3.8.1)
443 * https://llvm.org/bugs/show_bug.cgi?id=26775 (fixed in 3.8.1)
444 * https://llvm.org/bugs/show_bug.cgi?id=33531 (fixed in 4.0)
445 * https://llvm.org/bugs/show_bug.cgi?id=34647 (llc performance on certain unusual shader IR; intro'd in 4.0, pending as of 5.0)
446 */
447 if (util_cpu_caps.has_altivec) {
448 MAttrs.push_back("-vsx");
449 }
450 #else
451 /*
452 * Bug 25503 is fixed, by the same fix that fixed
453 * bug 26775, in versions of LLVM later than 3.8 (starting with 3.8.1).
454 * BZ 33531 actually comprises more than one bug, all of
455 * which are fixed in LLVM 4.0.
456 *
457 * With LLVM 4.0 or higher:
458 * Make sure VSX instructions are ENABLED (if supported), unless
459 * VSX instructions are explicitly enabled/disabled via GALLIVM_VSX=1 or 0.
460 */
461 if (util_cpu_caps.has_altivec) {
462 MAttrs.push_back(util_cpu_caps.has_vsx ? "+vsx" : "-vsx");
463 }
464 #endif
465 #endif
466
467 builder.setMAttrs(MAttrs);
468
469 if (gallivm_debug & (GALLIVM_DEBUG_IR | GALLIVM_DEBUG_ASM | GALLIVM_DEBUG_DUMP_BC)) {
470 int n = MAttrs.size();
471 if (n > 0) {
472 debug_printf("llc -mattr option(s): ");
473 for (int i = 0; i < n; i++)
474 debug_printf("%s%s", MAttrs[i].c_str(), (i < n - 1) ? "," : "");
475 debug_printf("\n");
476 }
477 }
478
479 StringRef MCPU = llvm::sys::getHostCPUName();
480 /*
481 * The cpu bits are no longer set automatically, so need to set mcpu manually.
482 * Note that the MAttrs set above will be sort of ignored (since we should
483 * not set any which would not be set by specifying the cpu anyway).
484 * It ought to be safe though since getHostCPUName() should include bits
485 * not only from the cpu but environment as well (for instance if it's safe
486 * to use avx instructions which need OS support). According to
487 * http://llvm.org/bugs/show_bug.cgi?id=19429 however if I understand this
488 * right it may be necessary to specify older cpu (or disable mattrs) though
489 * when not using MCJIT so no instructions are generated which the old JIT
490 * can't handle. Not entirely sure if we really need to do anything yet.
491 */
492
493 #ifdef PIPE_ARCH_PPC_64
494 /*
495 * Large programs, e.g. gnome-shell and firefox, may tax the addressability
496 * of the Medium code model once dynamically generated JIT-compiled shader
497 * programs are linked in and relocated. Yet the default code model as of
498 * LLVM 8 is Medium or even Small.
499 * The cost of changing from Medium to Large is negligible:
500 * - an additional 8-byte pointer stored immediately before the shader entrypoint;
501 * - change an add-immediate (addis) instruction to a load (ld).
502 */
503 builder.setCodeModel(CodeModel::Large);
504
505 #if UTIL_ARCH_LITTLE_ENDIAN
506 /*
507 * Versions of LLVM prior to 4.0 lacked a table entry for "POWER8NVL",
508 * resulting in (big-endian) "generic" being returned on
509 * little-endian Power8NVL systems. The result was that code that
510 * attempted to load the least significant 32 bits of a 64-bit quantity
511 * from memory loaded the wrong half. This resulted in failures in some
512 * Piglit tests, e.g.
513 * .../arb_gpu_shader_fp64/execution/conversion/frag-conversion-explicit-double-uint
514 */
515 if (MCPU == "generic")
516 MCPU = "pwr8";
517 #endif
518 #endif
519 builder.setMCPU(MCPU);
520 if (gallivm_debug & (GALLIVM_DEBUG_IR | GALLIVM_DEBUG_ASM | GALLIVM_DEBUG_DUMP_BC)) {
521 debug_printf("llc -mcpu option: %s\n", MCPU.str().c_str());
522 }
523
524 ShaderMemoryManager *MM = NULL;
525 BaseMemoryManager* JMM = reinterpret_cast<BaseMemoryManager*>(CMM);
526 MM = new ShaderMemoryManager(JMM);
527 *OutCode = MM->getGeneratedCode();
528
529 builder.setMCJITMemoryManager(std::unique_ptr<RTDyldMemoryManager>(MM));
530 MM = NULL; // ownership taken by std::unique_ptr
531
532 ExecutionEngine *JIT;
533
534 JIT = builder.create();
535
536 if (cache_out) {
537 LPObjectCache *objcache = new LPObjectCache(cache_out);
538 JIT->setObjectCache(objcache);
539 cache_out->jit_obj_cache = (void *)objcache;
540 }
541
542 #if LLVM_USE_INTEL_JITEVENTS
543 JITEventListener *JEL = JITEventListener::createIntelJITEventListener();
544 JIT->RegisterJITEventListener(JEL);
545 #endif
546 if (JIT) {
547 *OutJIT = wrap(JIT);
548 return 0;
549 }
550 lp_free_generated_code(*OutCode);
551 *OutCode = 0;
552 delete MM;
553 *OutError = strdup(Error.c_str());
554 return 1;
555 }
556
557
558 extern "C"
559 void
560 lp_free_generated_code(struct lp_generated_code *code)
561 {
562 ShaderMemoryManager::freeGeneratedCode(code);
563 }
564
565 extern "C"
566 LLVMMCJITMemoryManagerRef
567 lp_get_default_memory_manager()
568 {
569 BaseMemoryManager *mm;
570 mm = new llvm::SectionMemoryManager();
571 return reinterpret_cast<LLVMMCJITMemoryManagerRef>(mm);
572 }
573
574 extern "C"
575 void
576 lp_free_memory_manager(LLVMMCJITMemoryManagerRef memorymgr)
577 {
578 delete reinterpret_cast<BaseMemoryManager*>(memorymgr);
579 }
580
581 extern "C" void
582 lp_free_objcache(void *objcache_ptr)
583 {
584 LPObjectCache *objcache = (LPObjectCache *)objcache_ptr;
585 delete objcache;
586 }
587
588 extern "C" LLVMValueRef
589 lp_get_called_value(LLVMValueRef call)
590 {
591 return LLVMGetCalledValue(call);
592 }
593
594 extern "C" bool
595 lp_is_function(LLVMValueRef v)
596 {
597 return LLVMGetValueKind(v) == LLVMFunctionValueKind;
598 }