src/gallium/state_trackers/clover/llvm/invocation.cpp

   1 //
   2 // Copyright 2012-2016 Francisco Jerez
   3 // Copyright 2012-2016 Advanced Micro Devices, Inc.
   4 // Copyright 2014-2016 Jan Vesely
   5 // Copyright 2014-2015 Serge Martin
   6 // Copyright 2015 Zoltan Gilian
   7 //
   8 // Permission is hereby granted, free of charge, to any person obtaining a
   9 // copy of this software and associated documentation files (the "Software"),
  10 // to deal in the Software without restriction, including without limitation
  11 // the rights to use, copy, modify, merge, publish, distribute, sublicense,
  12 // and/or sell copies of the Software, and to permit persons to whom the
  13 // Software is furnished to do so, subject to the following conditions:
  14 //
  15 // The above copyright notice and this permission notice shall be included in
  16 // all copies or substantial portions of the Software.
  17 //
  18 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  21 // THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  22 // OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  23 // ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  24 // OTHER DEALINGS IN THE SOFTWARE.
  25 //
  26
  27 #include "llvm/compat.hpp"
  28 #include "llvm/util.hpp"
  29 #include "core/compiler.hpp"
  30 #include "util/algorithm.hpp"
  31
  32 #include <clang/Frontend/CompilerInstance.h>
  33 #include <clang/Frontend/TextDiagnosticBuffer.h>
  34 #include <clang/Frontend/TextDiagnosticPrinter.h>
  35 #include <clang/CodeGen/CodeGenAction.h>
  36 #include <clang/Basic/TargetInfo.h>
  37 #include <llvm/Bitcode/BitstreamWriter.h>
  38 #include <llvm/Bitcode/ReaderWriter.h>
  39 #include <llvm/Linker/Linker.h>
  40 #include <llvm/IR/DiagnosticInfo.h>
  41 #include <llvm/IR/DiagnosticPrinter.h>
  42 #include <llvm/IR/DerivedTypes.h>
  43 #include <llvm/IR/LLVMContext.h>
  44 #include <llvm/IR/Module.h>
  45 #include <llvm/Support/SourceMgr.h>
  46 #include <llvm/IRReader/IRReader.h>
  47 #include <llvm/Support/CodeGen.h>
  48 #include <llvm/Support/TargetSelect.h>
  49 #include <llvm/Support/MemoryBuffer.h>
  50 #include <llvm/Support/FormattedStream.h>
  51 #include <llvm/Support/TargetRegistry.h>
  52 #include <llvm/Transforms/IPO.h>
  53 #include <llvm/Transforms/IPO/PassManagerBuilder.h>
  54 #include <llvm/Transforms/Utils/Cloning.h>
  55
  56
  57 #include <llvm/IR/DataLayout.h>
  58 #include <llvm/Target/TargetMachine.h>
  59 #include <llvm/Target/TargetOptions.h>
  60
  61 #include <llvm-c/Target.h>
  62 #include <llvm-c/TargetMachine.h>
  63 #include <llvm-c/Core.h>
  64
  65 #include "pipe/p_state.h"
  66 #include "util/u_memory.h"
  67 #include "util/u_math.h"
  68
  69 #include <iostream>
  70 #include <iomanip>
  71 #include <fstream>
  72 #include <cstdio>
  73 #include <sstream>
  74 #include <libelf.h>
  75 #include <gelf.h>
  76
  77 using namespace clover;
  78 using namespace clover::llvm;
  79
  80 using ::llvm::Function;
  81 using ::llvm::LLVMContext;
  82 using ::llvm::Module;
  83 using ::llvm::raw_string_ostream;
  84 using ::llvm::TargetMachine;
  85
  86 namespace {
  87    // XXX - Temporary hack to avoid breaking the build for the moment, will
  88    //       get rid of this later.
  89    namespace llvm {
  90       using namespace ::llvm;
  91    }
  92
  93    void
  94    init_targets() {
  95       static bool targets_initialized = false;
  96       if (!targets_initialized) {
  97          LLVMInitializeAllTargets();
  98          LLVMInitializeAllTargetInfos();
  99          LLVMInitializeAllTargetMCs();
 100          LLVMInitializeAllAsmPrinters();
 101          targets_initialized = true;
 102       }
 103    }
 104
 105    void
 106    diagnostic_handler(const ::llvm::DiagnosticInfo &di, void *data) {
 107       if (di.getSeverity() == ::llvm::DS_Error) {
 108          raw_string_ostream os { *reinterpret_cast<std::string *>(data) };
 109          ::llvm::DiagnosticPrinterRawOStream printer { os };
 110          di.print(printer);
 111          throw compile_error();
 112       }
 113    }
 114
 115    std::unique_ptr<LLVMContext>
 116    create_context(std::string &r_log) {
 117       init_targets();
 118       std::unique_ptr<LLVMContext> ctx { new LLVMContext };
 119       ctx->setDiagnosticHandler(diagnostic_handler, &r_log);
 120       return ctx;
 121    }
 122
 123    std::unique_ptr<clang::CompilerInstance>
 124    create_compiler_instance(const target &target,
 125                             const std::vector<std::string> &opts,
 126                             std::string &r_log) {
 127       std::unique_ptr<clang::CompilerInstance> c { new clang::CompilerInstance };
 128       clang::DiagnosticsEngine diag { new clang::DiagnosticIDs,
 129             new clang::DiagnosticOptions, new clang::TextDiagnosticBuffer };
 130
 131       // Parse the compiler options.  A file name should be present at the end
 132       // and must have the .cl extension in order for the CompilerInvocation
 133       // class to recognize it as an OpenCL source file.
 134       const std::vector<const char *> copts =
 135          map(std::mem_fn(&std::string::c_str), opts);
 136
 137       if (!clang::CompilerInvocation::CreateFromArgs(
 138              c->getInvocation(), copts.data(), copts.data() + copts.size(), diag))
 139          throw error(CL_INVALID_COMPILER_OPTIONS);
 140
 141       c->getTargetOpts().CPU = target.cpu;
 142       c->getTargetOpts().Triple = target.triple;
 143       c->getLangOpts().NoBuiltin = true;
 144
 145       // This is a workaround for a Clang bug which causes the number
 146       // of warnings and errors to be printed to stderr.
 147       // http://www.llvm.org/bugs/show_bug.cgi?id=19735
 148       c->getDiagnosticOpts().ShowCarets = false;
 149
 150       compat::set_lang_defaults(c->getInvocation(), c->getLangOpts(),
 151                                 clang::IK_OpenCL, ::llvm::Triple(target.triple),
 152                                 c->getPreprocessorOpts(),
 153                                 clang::LangStandard::lang_opencl11);
 154
 155       c->createDiagnostics(new clang::TextDiagnosticPrinter(
 156                               *new raw_string_ostream(r_log),
 157                               &c->getDiagnosticOpts(), true));
 158
 159       c->setTarget(clang::TargetInfo::CreateTargetInfo(
 160                            c->getDiagnostics(), c->getInvocation().TargetOpts));
 161
 162       return c;
 163    }
 164
 165    std::unique_ptr<Module>
 166    compile(LLVMContext &ctx, clang::CompilerInstance &c,
 167            const std::string &name, const std::string &source,
 168            const header_map &headers, const std::string &target,
 169            const std::string &opts, std::string &r_log) {
 170       c.getFrontendOpts().ProgramAction = clang::frontend::EmitLLVMOnly;
 171       c.getHeaderSearchOpts().UseBuiltinIncludes = true;
 172       c.getHeaderSearchOpts().UseStandardSystemIncludes = true;
 173       c.getHeaderSearchOpts().ResourceDir = CLANG_RESOURCE_DIR;
 174
 175       // Add libclc generic search path
 176       c.getHeaderSearchOpts().AddPath(LIBCLC_INCLUDEDIR,
 177                                       clang::frontend::Angled,
 178                                       false, false);
 179
 180       // Add libclc include
 181       c.getPreprocessorOpts().Includes.push_back("clc/clc.h");
 182
 183       // clc.h requires that this macro be defined:
 184       c.getPreprocessorOpts().addMacroDef("cl_clang_storage_class_specifiers");
 185       c.getPreprocessorOpts().addRemappedFile(
 186               name, ::llvm::MemoryBuffer::getMemBuffer(source).release());
 187
 188       if (headers.size()) {
 189          const std::string tmp_header_path = "/tmp/clover/";
 190
 191          c.getHeaderSearchOpts().AddPath(tmp_header_path,
 192                                          clang::frontend::Angled,
 193                                          false, false);
 194
 195          for (const auto &header : headers)
 196             c.getPreprocessorOpts().addRemappedFile(
 197                tmp_header_path + header.first,
 198                ::llvm::MemoryBuffer::getMemBuffer(header.second).release());
 199       }
 200
 201       // Tell clang to link this file before performing any
 202       // optimizations.  This is required so that we can replace calls
 203       // to the OpenCL C barrier() builtin with calls to target
 204       // intrinsics that have the noduplicate attribute.  This
 205       // attribute will prevent Clang from creating illegal uses of
 206       // barrier() (e.g. Moving barrier() inside a conditional that is
 207       // no executed by all threads) during its optimizaton passes.
 208       compat::add_link_bitcode_file(c.getCodeGenOpts(),
 209                                     LIBCLC_LIBEXECDIR + target + ".bc");
 210
 211       // Compile the code
 212       clang::EmitLLVMOnlyAction act(&ctx);
 213       if (!c.ExecuteAction(act))
 214          throw compile_error();
 215
 216       return act.takeModule();
 217    }
 218
 219    std::vector<llvm::Function *>
 220    find_kernels(llvm::Module *mod) {
 221       std::vector<llvm::Function *> kernels;
 222 #if HAVE_LLVM >= 0x0309
 223       auto &list = mod->getFunctionList();
 224       for_each(list.begin(), list.end(), [&](llvm::Function &f){
 225          if (f.getMetadata("kernel_arg_type"))
 226            kernels.push_back(&f);
 227       });
 228       return kernels;
 229 #endif
 230       const llvm::NamedMDNode *kernel_node =
 231                                  mod->getNamedMetadata("opencl.kernels");
 232       // This means there are no kernels in the program.  The spec does not
 233       // require that we return an error here, but there will be an error if
 234       // the user tries to pass this program to a clCreateKernel() call.
 235       if (!kernel_node) {
 236          return std::vector<llvm::Function *>();
 237       }
 238
 239       kernels.reserve(kernel_node->getNumOperands());
 240       for (unsigned i = 0; i < kernel_node->getNumOperands(); ++i) {
 241          kernels.push_back(llvm::mdconst::dyn_extract<llvm::Function>(
 242                                     kernel_node->getOperand(i)->getOperand(0)));
 243       }
 244       return kernels;
 245    }
 246
 247    void
 248    optimize(Module &mod, unsigned optimization_level) {
 249       compat::pass_manager pm;
 250
 251       compat::add_data_layout_pass(pm);
 252
 253       // By default, the function internalizer pass will look for a function
 254       // called "main" and then mark all other functions as internal.  Marking
 255       // functions as internal enables the optimizer to perform optimizations
 256       // like function inlining and global dead-code elimination.
 257       //
 258       // When there is no "main" function in a module, the internalize pass will
 259       // treat the module like a library, and it won't internalize any functions.
 260       // Since there is no "main" function in our kernels, we need to tell
 261       // the internalizer pass that this module is not a library by passing a
 262       // list of kernel functions to the internalizer.  The internalizer will
 263       // treat the functions in the list as "main" functions and internalize
 264       // all of the other functions.
 265       compat::add_internalize_pass(pm, map(std::mem_fn(&Function::getName),
 266                                            find_kernels(&mod)));
 267
 268       ::llvm::PassManagerBuilder pmb;
 269       pmb.OptLevel = optimization_level;
 270       pmb.LibraryInfo = new compat::target_library_info(
 271          ::llvm::Triple(mod.getTargetTriple()));
 272       pmb.populateModulePassManager(pm);
 273       pm.run(mod);
 274    }
 275
 276    // Kernel metadata
 277
 278    struct kernel_arg_md {
 279       llvm::StringRef type_name;
 280       llvm::StringRef access_qual;
 281       kernel_arg_md(llvm::StringRef type_name_, llvm::StringRef access_qual_):
 282          type_name(type_name_), access_qual(access_qual_) {}
 283    };
 284 #if HAVE_LLVM >= 0x0309
 285    std::vector<kernel_arg_md>
 286    get_kernel_arg_md(const llvm::Function *kernel_func) {
 287
 288       size_t num_args = kernel_func->getArgumentList().size();
 289
 290       auto aq = kernel_func->getMetadata("kernel_arg_access_qual");
 291       auto ty = kernel_func->getMetadata("kernel_arg_type");
 292
 293       std::vector<kernel_arg_md> res;
 294       res.reserve(num_args);
 295       for (size_t i = 0; i < num_args; ++i) {
 296          res.push_back(kernel_arg_md(
 297             llvm::cast<llvm::MDString>(ty->getOperand(i))->getString(),
 298             llvm::cast<llvm::MDString>(aq->getOperand(i))->getString()));
 299       }
 300
 301       return res;
 302    }
 303
 304 #else
 305
 306    const llvm::MDNode *
 307    get_kernel_metadata(const llvm::Function *kernel_func) {
 308       auto mod = kernel_func->getParent();
 309       auto kernels_node = mod->getNamedMetadata("opencl.kernels");
 310       if (!kernels_node) {
 311          return nullptr;
 312       }
 313
 314       const llvm::MDNode *kernel_node = nullptr;
 315       for (unsigned i = 0; i < kernels_node->getNumOperands(); ++i) {
 316          auto func = llvm::mdconst::dyn_extract<llvm::Function>(
 317                kernels_node->getOperand(i)->getOperand(0));
 318          if (func == kernel_func) {
 319             kernel_node = kernels_node->getOperand(i);
 320             break;
 321          }
 322       }
 323
 324       return kernel_node;
 325    }
 326
 327    llvm::MDNode*
 328    node_from_op_checked(const llvm::MDOperand &md_operand,
 329                         llvm::StringRef expect_name,
 330                         unsigned expect_num_args)
 331    {
 332       auto node = llvm::cast<llvm::MDNode>(md_operand);
 333       assert(node->getNumOperands() == expect_num_args &&
 334              "Wrong number of operands.");
 335
 336       auto str_node = llvm::cast<llvm::MDString>(node->getOperand(0));
 337       assert(str_node->getString() == expect_name &&
 338              "Wrong metadata node name.");
 339
 340       return node;
 341    }
 342
 343    std::vector<kernel_arg_md>
 344    get_kernel_arg_md(const llvm::Function *kernel_func) {
 345       auto num_args = kernel_func->getArgumentList().size();
 346
 347       auto kernel_node = get_kernel_metadata(kernel_func);
 348       auto aq = node_from_op_checked(kernel_node->getOperand(2),
 349                                      "kernel_arg_access_qual", num_args + 1);
 350       auto ty = node_from_op_checked(kernel_node->getOperand(3),
 351                                      "kernel_arg_type", num_args + 1);
 352
 353       std::vector<kernel_arg_md> res;
 354       res.reserve(num_args);
 355       for (unsigned i = 0; i < num_args; ++i) {
 356          res.push_back(kernel_arg_md(
 357             llvm::cast<llvm::MDString>(ty->getOperand(i+1))->getString(),
 358             llvm::cast<llvm::MDString>(aq->getOperand(i+1))->getString()));
 359       }
 360
 361       return res;
 362    }
 363 #endif
 364
 365    std::vector<module::argument>
 366    get_kernel_args(const llvm::Module *mod, const std::string &kernel_name,
 367                    const clang::CompilerInstance &c) {
 368       std::vector<module::argument> args;
 369       const auto address_spaces = c.getTarget().getAddressSpaceMap();
 370       llvm::Function *kernel_func = mod->getFunction(kernel_name);
 371       assert(kernel_func && "Kernel name not found in module.");
 372       auto arg_md = get_kernel_arg_md(kernel_func);
 373
 374       llvm::DataLayout TD(mod);
 375       llvm::Type *size_type =
 376          TD.getSmallestLegalIntType(mod->getContext(), sizeof(cl_uint) * 8);
 377
 378       for (const auto &arg: kernel_func->args()) {
 379
 380          llvm::Type *arg_type = arg.getType();
 381          const unsigned arg_store_size = TD.getTypeStoreSize(arg_type);
 382
 383          // OpenCL 1.2 specification, Ch. 6.1.5: "A built-in data
 384          // type that is not a power of two bytes in size must be
 385          // aligned to the next larger power of two".  We need this
 386          // alignment for three element vectors, which have
 387          // non-power-of-2 store size.
 388          const unsigned arg_api_size = util_next_power_of_two(arg_store_size);
 389
 390          llvm::Type *target_type = arg_type->isIntegerTy() ?
 391                TD.getSmallestLegalIntType(mod->getContext(), arg_store_size * 8)
 392                : arg_type;
 393          unsigned target_size = TD.getTypeStoreSize(target_type);
 394          unsigned target_align = TD.getABITypeAlignment(target_type);
 395
 396          llvm::StringRef type_name = arg_md[arg.getArgNo()].type_name;
 397          llvm::StringRef access_qual = arg_md[arg.getArgNo()].access_qual;
 398
 399          // Image
 400          const bool is_image2d = type_name == "image2d_t";
 401          const bool is_image3d = type_name == "image3d_t";
 402          if (is_image2d || is_image3d) {
 403             const bool is_write_only = access_qual == "write_only";
 404             const bool is_read_only = access_qual == "read_only";
 405
 406             enum module::argument::type marg_type;
 407             if (is_image2d && is_read_only) {
 408                marg_type = module::argument::image2d_rd;
 409             } else if (is_image2d && is_write_only) {
 410                marg_type = module::argument::image2d_wr;
 411             } else if (is_image3d && is_read_only) {
 412                marg_type = module::argument::image3d_rd;
 413             } else if (is_image3d && is_write_only) {
 414                marg_type = module::argument::image3d_wr;
 415             } else {
 416                assert(0 && "Wrong image access qualifier");
 417             }
 418
 419             args.push_back(module::argument(marg_type,
 420                                             arg_store_size, target_size,
 421                                             target_align,
 422                                             module::argument::zero_ext));
 423             continue;
 424          }
 425
 426          // Image size implicit argument
 427          if (type_name == "__llvm_image_size") {
 428             args.push_back(module::argument(module::argument::scalar,
 429                                             sizeof(cl_uint),
 430                                             TD.getTypeStoreSize(size_type),
 431                                             TD.getABITypeAlignment(size_type),
 432                                             module::argument::zero_ext,
 433                                             module::argument::image_size));
 434             continue;
 435          }
 436
 437          // Image format implicit argument
 438          if (type_name == "__llvm_image_format") {
 439             args.push_back(module::argument(module::argument::scalar,
 440                                             sizeof(cl_uint),
 441                                             TD.getTypeStoreSize(size_type),
 442                                             TD.getABITypeAlignment(size_type),
 443                                             module::argument::zero_ext,
 444                                             module::argument::image_format));
 445             continue;
 446          }
 447
 448          // Other types
 449          if (llvm::isa<llvm::PointerType>(arg_type) && arg.hasByValAttr()) {
 450             arg_type =
 451                   llvm::dyn_cast<llvm::PointerType>(arg_type)->getElementType();
 452          }
 453
 454          if (arg_type->isPointerTy()) {
 455             unsigned address_space = llvm::cast<llvm::PointerType>(arg_type)->getAddressSpace();
 456             if (address_space == address_spaces[clang::LangAS::opencl_local
 457                                                      - clang::LangAS::Offset]) {
 458                args.push_back(module::argument(module::argument::local,
 459                                                arg_api_size, target_size,
 460                                                target_align,
 461                                                module::argument::zero_ext));
 462             } else {
 463                // XXX: Correctly handle constant address space.  There is no
 464                // way for r600g to pass a handle for constant buffers back
 465                // to clover like it can for global buffers, so
 466                // creating constant arguments will break r600g.  For now,
 467                // continue treating constant buffers as global buffers
 468                // until we can come up with a way to create handles for
 469                // constant buffers.
 470                args.push_back(module::argument(module::argument::global,
 471                                                arg_api_size, target_size,
 472                                                target_align,
 473                                                module::argument::zero_ext));
 474            }
 475
 476          } else {
 477             llvm::AttributeSet attrs = kernel_func->getAttributes();
 478             enum module::argument::ext_type ext_type =
 479                   (attrs.hasAttribute(arg.getArgNo() + 1,
 480                                      llvm::Attribute::SExt) ?
 481                    module::argument::sign_ext :
 482                    module::argument::zero_ext);
 483
 484             args.push_back(
 485                module::argument(module::argument::scalar, arg_api_size,
 486                                 target_size, target_align, ext_type));
 487          }
 488       }
 489
 490       // Append implicit arguments.  XXX - The types, ordering and
 491       // vector size of the implicit arguments should depend on the
 492       // target according to the selected calling convention.
 493       args.push_back(
 494          module::argument(module::argument::scalar, sizeof(cl_uint),
 495                           TD.getTypeStoreSize(size_type),
 496                           TD.getABITypeAlignment(size_type),
 497                           module::argument::zero_ext,
 498                           module::argument::grid_dimension));
 499
 500       args.push_back(
 501          module::argument(module::argument::scalar, sizeof(cl_uint),
 502                           TD.getTypeStoreSize(size_type),
 503                           TD.getABITypeAlignment(size_type),
 504                           module::argument::zero_ext,
 505                           module::argument::grid_offset));
 506
 507       return args;
 508    }
 509
 510    module::section
 511    make_text_section(const std::vector<char> &code) {
 512       const pipe_llvm_program_header header { uint32_t(code.size()) };
 513       module::section text { 0, module::section::text, header.num_bytes, {} };
 514
 515       text.data.insert(text.data.end(), reinterpret_cast<const char *>(&header),
 516                        reinterpret_cast<const char *>(&header) + sizeof(header));
 517       text.data.insert(text.data.end(), code.begin(), code.end());
 518
 519       return text;
 520    }
 521
 522    module
 523    build_module_common(const Module &mod,
 524                        const std::vector<char> &code,
 525                        const std::map<std::string,
 526                                       unsigned> &offsets,
 527                        const clang::CompilerInstance &c) {
 528       module m;
 529
 530       for (const auto &name : map(std::mem_fn(&Function::getName),
 531                                   find_kernels(const_cast<Module *>(&mod)))) {
 532          if (offsets.count(name))
 533             m.syms.emplace_back(name, 0, offsets.at(name),
 534                                 get_kernel_args(&mod, name, c));
 535       }
 536
 537       m.secs.push_back(make_text_section(code));
 538       return m;
 539    }
 540
 541    module
 542    build_module_llvm(llvm::Module *mod,
 543                      const clang::CompilerInstance &c) {
 544       llvm::SmallVector<char, 1024> llvm_bitcode;
 545       llvm::raw_svector_ostream bitcode_ostream(llvm_bitcode);
 546       llvm::BitstreamWriter writer(llvm_bitcode);
 547       llvm::WriteBitcodeToFile(mod, bitcode_ostream);
 548 #if HAVE_LLVM < 0x0308
 549       bitcode_ostream.flush();
 550 #endif
 551
 552       std::map<std::string, unsigned> offsets;
 553       unsigned i = 0;
 554
 555       for (const auto &name : map(std::mem_fn(&::llvm::Function::getName),
 556                                   find_kernels(mod)))
 557          offsets[name] = i++;
 558
 559       return build_module_common(*mod, { llvm_bitcode.begin(),
 560                                          llvm_bitcode.end() },
 561                                  offsets, c);
 562    }
 563
 564    std::vector<char>
 565    emit_code(::llvm::Module &mod, const target &target,
 566              TargetMachine::CodeGenFileType ft,
 567              std::string &r_log) {
 568       std::string err;
 569       auto t = ::llvm::TargetRegistry::lookupTarget(target.triple, err);
 570       if (!t)
 571          fail(r_log, compile_error(), err);
 572
 573       std::unique_ptr<TargetMachine> tm {
 574          t->createTargetMachine(target.triple, target.cpu, "", {},
 575                                 compat::default_reloc_model,
 576                                 ::llvm::CodeModel::Default,
 577                                 ::llvm::CodeGenOpt::Default) };
 578       if (!tm)
 579          fail(r_log, compile_error(),
 580               "Could not create TargetMachine: " + target.triple);
 581
 582       ::llvm::SmallVector<char, 1024> data;
 583
 584       {
 585          compat::pass_manager pm;
 586          ::llvm::raw_svector_ostream os { data };
 587          compat::raw_ostream_to_emit_file fos { os };
 588
 589          mod.setDataLayout(compat::get_data_layout(*tm));
 590          tm->Options.MCOptions.AsmVerbose =
 591             (ft == TargetMachine::CGFT_AssemblyFile);
 592
 593          if (tm->addPassesToEmitFile(pm, fos, ft))
 594             fail(r_log, compile_error(), "TargetMachine can't emit this file");
 595
 596          pm.run(mod);
 597       }
 598
 599       return { data.begin(), data.end() };
 600    }
 601
 602    std::vector<char>
 603    compile_native(llvm::Module *mod, const target &target,
 604                   std::string &r_log) {
 605       if (has_flag(debug::native)) {
 606          std::unique_ptr<llvm::Module> cmod { CloneModule(mod) };
 607          debug::log(".asm", as_string(
 608                        emit_code(*cmod, target,
 609                                  TargetMachine::CGFT_AssemblyFile, r_log)));
 610       }
 611
 612       return emit_code(*mod, target, TargetMachine::CGFT_ObjectFile, r_log);
 613    }
 614
 615    namespace elf {
 616       std::unique_ptr<Elf, int (*)(Elf *)>
 617       get(const std::vector<char> &code) {
 618          // One of the libelf implementations
 619          // (http://www.mr511.de/software/english.htm) requires calling
 620          // elf_version() before elf_memory().
 621          elf_version(EV_CURRENT);
 622          return { elf_memory(const_cast<char *>(code.data()), code.size()),
 623                   elf_end };
 624       }
 625
 626       Elf_Scn *
 627       get_symbol_table(Elf *elf) {
 628          size_t section_str_index;
 629          elf_getshdrstrndx(elf, &section_str_index);
 630
 631          for (Elf_Scn *s = elf_nextscn(elf, NULL); s; s = elf_nextscn(elf, s)) {
 632             GElf_Shdr header;
 633             if (gelf_getshdr(s, &header) != &header)
 634                return nullptr;
 635
 636             if (!std::strcmp(elf_strptr(elf, section_str_index, header.sh_name),
 637                              ".symtab"))
 638                return s;
 639          }
 640
 641          return nullptr;
 642       }
 643
 644       std::map<std::string, unsigned>
 645       get_symbol_offsets(Elf *elf, Elf_Scn *symtab) {
 646          Elf_Data *const symtab_data = elf_getdata(symtab, NULL);
 647          GElf_Shdr header;
 648          if (gelf_getshdr(symtab, &header) != &header)
 649             return {};
 650
 651          std::map<std::string, unsigned> symbol_offsets;
 652          GElf_Sym symbol;
 653          unsigned i = 0;
 654
 655          while (GElf_Sym *s = gelf_getsym(symtab_data, i++, &symbol)) {
 656             const char *name = elf_strptr(elf, header.sh_link, s->st_name);
 657             symbol_offsets[name] = s->st_value;
 658          }
 659
 660          return symbol_offsets;
 661       }
 662    }
 663
 664    std::map<std::string, unsigned>
 665    get_symbol_offsets(const std::vector<char> &code,
 666                       std::string &r_log) {
 667       const auto elf = elf::get(code);
 668       const auto symtab = elf::get_symbol_table(elf.get());
 669       if (!symtab)
 670          fail(r_log, compile_error(), "Unable to find symbol table.");
 671
 672       return elf::get_symbol_offsets(elf.get(), symtab);
 673    }
 674
 675    module
 676    build_module_native(std::vector<char> &code,
 677                        llvm::Module *mod,
 678                        const clang::CompilerInstance &c,
 679                        std::string &r_log) {
 680       return build_module_common(*mod, code,
 681                                  get_symbol_offsets(code, r_log), c);
 682    }
 683 } // End anonymous namespace
 684
 685 module
 686 clover::compile_program_llvm(const std::string &source,
 687                              const header_map &headers,
 688                              enum pipe_shader_ir ir,
 689                              const std::string &target,
 690                              const std::string &opts,
 691                              std::string &r_log) {
 692    if (has_flag(debug::clc))
 693       debug::log(".cl", "// Build options: " + opts + '\n' + source);
 694
 695    auto ctx = create_context(r_log);
 696    // The input file name must have the .cl extension in order for the
 697    // CompilerInvocation class to recognize it as an OpenCL source file.
 698    const auto c = create_compiler_instance(target, tokenize(opts + " input.cl"),
 699                                            r_log);
 700    auto mod = compile(*ctx, *c, "input.cl", source, headers, target, opts, r_log);
 701
 702    optimize(*mod, c->getCodeGenOpts().OptimizationLevel);
 703
 704    if (has_flag(debug::llvm)) {
 705       std::string log;
 706       raw_string_ostream s_log(log);
 707       mod->print(s_log, NULL);
 708       s_log.flush();
 709       debug::log(".ll", log);
 710     }
 711
 712    module m;
 713    // Build the clover::module
 714    switch (ir) {
 715       case PIPE_SHADER_IR_NIR:
 716       case PIPE_SHADER_IR_TGSI:
 717          //XXX: Handle TGSI, NIR
 718          assert(0);
 719          m = module();
 720          break;
 721       case PIPE_SHADER_IR_LLVM:
 722          m = build_module_llvm(&*mod, *c);
 723          break;
 724       case PIPE_SHADER_IR_NATIVE: {
 725          std::vector<char> code = compile_native(&*mod, target, r_log);
 726          m = build_module_native(code, &*mod, *c, r_log);
 727          break;
 728       }
 729    }
 730
 731    return m;
 732 }