src/gallium/state_trackers/clover/llvm/invocation.cpp

   1 //
   2 // Copyright 2012-2016 Francisco Jerez
   3 // Copyright 2012-2016 Advanced Micro Devices, Inc.
   4 // Copyright 2014-2016 Jan Vesely
   5 // Copyright 2014-2015 Serge Martin
   6 // Copyright 2015 Zoltan Gilian
   7 //
   8 // Permission is hereby granted, free of charge, to any person obtaining a
   9 // copy of this software and associated documentation files (the "Software"),
  10 // to deal in the Software without restriction, including without limitation
  11 // the rights to use, copy, modify, merge, publish, distribute, sublicense,
  12 // and/or sell copies of the Software, and to permit persons to whom the
  13 // Software is furnished to do so, subject to the following conditions:
  14 //
  15 // The above copyright notice and this permission notice shall be included in
  16 // all copies or substantial portions of the Software.
  17 //
  18 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  21 // THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  22 // OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  23 // ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  24 // OTHER DEALINGS IN THE SOFTWARE.
  25 //
  26
  27 #include "llvm/compat.hpp"
  28 #include "core/compiler.hpp"
  29 #include "util/algorithm.hpp"
  30
  31 #include <clang/Frontend/CompilerInstance.h>
  32 #include <clang/Frontend/TextDiagnosticBuffer.h>
  33 #include <clang/Frontend/TextDiagnosticPrinter.h>
  34 #include <clang/CodeGen/CodeGenAction.h>
  35 #include <clang/Basic/TargetInfo.h>
  36 #include <llvm/Bitcode/BitstreamWriter.h>
  37 #include <llvm/Bitcode/ReaderWriter.h>
  38 #include <llvm/Linker/Linker.h>
  39 #include <llvm/IR/DiagnosticInfo.h>
  40 #include <llvm/IR/DiagnosticPrinter.h>
  41 #include <llvm/IR/DerivedTypes.h>
  42 #include <llvm/IR/LLVMContext.h>
  43 #include <llvm/IR/Module.h>
  44 #include <llvm/Support/SourceMgr.h>
  45 #include <llvm/IRReader/IRReader.h>
  46 #include <llvm/Support/CodeGen.h>
  47 #include <llvm/Support/TargetSelect.h>
  48 #include <llvm/Support/MemoryBuffer.h>
  49 #include <llvm/Support/FormattedStream.h>
  50 #include <llvm/Support/TargetRegistry.h>
  51 #include <llvm/Transforms/IPO.h>
  52 #include <llvm/Transforms/IPO/PassManagerBuilder.h>
  53 #include <llvm/Transforms/Utils/Cloning.h>
  54
  55
  56 #include <llvm/IR/DataLayout.h>
  57 #include <llvm/Target/TargetMachine.h>
  58 #include <llvm/Target/TargetOptions.h>
  59
  60 #include <llvm-c/Target.h>
  61 #include <llvm-c/TargetMachine.h>
  62 #include <llvm-c/Core.h>
  63
  64 #include "pipe/p_state.h"
  65 #include "util/u_memory.h"
  66 #include "util/u_math.h"
  67
  68 #include <iostream>
  69 #include <iomanip>
  70 #include <fstream>
  71 #include <cstdio>
  72 #include <sstream>
  73 #include <libelf.h>
  74 #include <gelf.h>
  75
  76 using namespace clover;
  77 using namespace clover::llvm;
  78
  79 using ::llvm::Function;
  80 using ::llvm::LLVMContext;
  81 using ::llvm::Module;
  82 using ::llvm::raw_string_ostream;
  83
  84 namespace {
  85    // XXX - Temporary hack to avoid breaking the build for the moment, will
  86    //       get rid of this later.
  87    namespace llvm {
  88       using namespace ::llvm;
  89    }
  90
  91    void debug_log(const std::string &msg, const std::string &suffix) {
  92       const char *dbg_file = debug_get_option("CLOVER_DEBUG_FILE", "stderr");
  93       if (!strcmp("stderr", dbg_file)) {
  94          std::cerr << msg;
  95        } else {
  96         std::ofstream file(dbg_file + suffix, std::ios::app);
  97         file << msg;
  98        }
  99    }
 100
 101    inline std::vector<std::string>
 102    tokenize(const std::string &s) {
 103       std::vector<std::string> ss;
 104       std::istringstream iss(s);
 105       std::string t;
 106
 107       while (getline(iss, t, ' '))
 108          ss.push_back(t);
 109
 110       return ss;
 111    }
 112
 113    struct target {
 114       target(const std::string &s) :
 115          cpu(s.begin(), s.begin() + s.find_first_of("-")),
 116          triple(s.begin() + s.find_first_of("-") + 1, s.end()) {}
 117
 118       std::string cpu;
 119       std::string triple;
 120    };
 121
 122    void
 123    init_targets() {
 124       static bool targets_initialized = false;
 125       if (!targets_initialized) {
 126          LLVMInitializeAllTargets();
 127          LLVMInitializeAllTargetInfos();
 128          LLVMInitializeAllTargetMCs();
 129          LLVMInitializeAllAsmPrinters();
 130          targets_initialized = true;
 131       }
 132    }
 133
 134    void
 135    diagnostic_handler(const llvm::DiagnosticInfo &di, void *data) {
 136       if (di.getSeverity() == llvm::DS_Error) {
 137          std::string message = *(std::string*)data;
 138          llvm::raw_string_ostream stream(message);
 139          llvm::DiagnosticPrinterRawOStream dp(stream);
 140          di.print(dp);
 141          stream.flush();
 142          *(std::string*)data = message;
 143
 144          throw compile_error();
 145       }
 146    }
 147
 148    std::unique_ptr<LLVMContext>
 149    create_context(std::string &r_log) {
 150       init_targets();
 151       std::unique_ptr<LLVMContext> ctx { new LLVMContext };
 152       ctx->setDiagnosticHandler(diagnostic_handler, &r_log);
 153       return ctx;
 154    }
 155
 156    std::unique_ptr<clang::CompilerInstance>
 157    create_compiler_instance(const target &target,
 158                             const std::vector<std::string> &opts,
 159                             std::string &r_log) {
 160       std::unique_ptr<clang::CompilerInstance> c { new clang::CompilerInstance };
 161       clang::DiagnosticsEngine diag { new clang::DiagnosticIDs,
 162             new clang::DiagnosticOptions, new clang::TextDiagnosticBuffer };
 163
 164       // Parse the compiler options.  A file name should be present at the end
 165       // and must have the .cl extension in order for the CompilerInvocation
 166       // class to recognize it as an OpenCL source file.
 167       const std::vector<const char *> copts =
 168          map(std::mem_fn(&std::string::c_str), opts);
 169
 170       if (!clang::CompilerInvocation::CreateFromArgs(
 171              c->getInvocation(), copts.data(), copts.data() + copts.size(), diag))
 172          throw error(CL_INVALID_COMPILER_OPTIONS);
 173
 174       c->getTargetOpts().CPU = target.cpu;
 175       c->getTargetOpts().Triple = target.triple;
 176       c->getLangOpts().NoBuiltin = true;
 177
 178       // This is a workaround for a Clang bug which causes the number
 179       // of warnings and errors to be printed to stderr.
 180       // http://www.llvm.org/bugs/show_bug.cgi?id=19735
 181       c->getDiagnosticOpts().ShowCarets = false;
 182
 183       compat::set_lang_defaults(c->getInvocation(), c->getLangOpts(),
 184                                 clang::IK_OpenCL, ::llvm::Triple(target.triple),
 185                                 c->getPreprocessorOpts(),
 186                                 clang::LangStandard::lang_opencl11);
 187
 188       c->createDiagnostics(new clang::TextDiagnosticPrinter(
 189                               *new raw_string_ostream(r_log),
 190                               &c->getDiagnosticOpts(), true));
 191
 192       c->setTarget(clang::TargetInfo::CreateTargetInfo(
 193                            c->getDiagnostics(), c->getInvocation().TargetOpts));
 194
 195       return c;
 196    }
 197
 198    llvm::Module *
 199    compile_llvm(LLVMContext &ctx, clang::CompilerInstance &c,
 200                 const std::string &source, const header_map &headers,
 201                 const std::string &name, const std::string &target,
 202                 const std::string &opts, std::string &r_log) {
 203       clang::EmitLLVMOnlyAction act(&ctx);
 204
 205       c.getFrontendOpts().ProgramAction = clang::frontend::EmitLLVMOnly;
 206       c.getHeaderSearchOpts().UseBuiltinIncludes = true;
 207       c.getHeaderSearchOpts().UseStandardSystemIncludes = true;
 208       c.getHeaderSearchOpts().ResourceDir = CLANG_RESOURCE_DIR;
 209
 210       // Add libclc generic search path
 211       c.getHeaderSearchOpts().AddPath(LIBCLC_INCLUDEDIR,
 212                                       clang::frontend::Angled,
 213                                       false, false
 214                                       );
 215
 216       // Add libclc include
 217       c.getPreprocessorOpts().Includes.push_back("clc/clc.h");
 218
 219       // clc.h requires that this macro be defined:
 220       c.getPreprocessorOpts().addMacroDef("cl_clang_storage_class_specifiers");
 221
 222       c.getPreprocessorOpts().addRemappedFile(name,
 223                                               llvm::MemoryBuffer::getMemBuffer(source).release());
 224
 225       if (headers.size()) {
 226          const std::string tmp_header_path = "/tmp/clover/";
 227
 228          c.getHeaderSearchOpts().AddPath(tmp_header_path,
 229                                          clang::frontend::Angled,
 230                                          false, false
 231                                          );
 232
 233          for (header_map::const_iterator it = headers.begin();
 234               it != headers.end(); ++it) {
 235             const std::string path = tmp_header_path + std::string(it->first);
 236             c.getPreprocessorOpts().addRemappedFile(path,
 237                     llvm::MemoryBuffer::getMemBuffer(it->second.c_str()).release());
 238          }
 239       }
 240
 241       // Tell clang to link this file before performing any
 242       // optimizations.  This is required so that we can replace calls
 243       // to the OpenCL C barrier() builtin with calls to target
 244       // intrinsics that have the noduplicate attribute.  This
 245       // attribute will prevent Clang from creating illegal uses of
 246       // barrier() (e.g. Moving barrier() inside a conditional that is
 247       // no executed by all threads) during its optimizaton passes.
 248       compat::add_link_bitcode_file(c.getCodeGenOpts(),
 249                                     LIBCLC_LIBEXECDIR + target + ".bc");
 250
 251       // Compile the code
 252       if (!c.ExecuteAction(act))
 253          throw compile_error();
 254
 255       return act.takeModule().release();
 256    }
 257
 258    std::vector<llvm::Function *>
 259    find_kernels(llvm::Module *mod) {
 260       std::vector<llvm::Function *> kernels;
 261 #if HAVE_LLVM >= 0x0309
 262       auto &list = mod->getFunctionList();
 263       for_each(list.begin(), list.end(), [&](llvm::Function &f){
 264          if (f.getMetadata("kernel_arg_type"))
 265            kernels.push_back(&f);
 266       });
 267       return kernels;
 268 #endif
 269       const llvm::NamedMDNode *kernel_node =
 270                                  mod->getNamedMetadata("opencl.kernels");
 271       // This means there are no kernels in the program.  The spec does not
 272       // require that we return an error here, but there will be an error if
 273       // the user tries to pass this program to a clCreateKernel() call.
 274       if (!kernel_node) {
 275          return std::vector<llvm::Function *>();
 276       }
 277
 278       kernels.reserve(kernel_node->getNumOperands());
 279       for (unsigned i = 0; i < kernel_node->getNumOperands(); ++i) {
 280          kernels.push_back(llvm::mdconst::dyn_extract<llvm::Function>(
 281                                     kernel_node->getOperand(i)->getOperand(0)));
 282       }
 283       return kernels;
 284    }
 285
 286    void
 287    optimize(llvm::Module *mod, unsigned optimization_level) {
 288       compat::pass_manager PM;
 289
 290       compat::add_data_layout_pass(PM);
 291
 292       // By default, the function internalizer pass will look for a function
 293       // called "main" and then mark all other functions as internal.  Marking
 294       // functions as internal enables the optimizer to perform optimizations
 295       // like function inlining and global dead-code elimination.
 296       //
 297       // When there is no "main" function in a module, the internalize pass will
 298       // treat the module like a library, and it won't internalize any functions.
 299       // Since there is no "main" function in our kernels, we need to tell
 300       // the internalizer pass that this module is not a library by passing a
 301       // list of kernel functions to the internalizer.  The internalizer will
 302       // treat the functions in the list as "main" functions and internalize
 303       // all of the other functions.
 304       compat::add_internalize_pass(PM, map(std::mem_fn(&Function::getName),
 305                                            find_kernels(mod)));
 306
 307       llvm::PassManagerBuilder PMB;
 308       PMB.OptLevel = optimization_level;
 309       PMB.LibraryInfo = new compat::target_library_info(
 310          llvm::Triple(mod->getTargetTriple()));
 311       PMB.populateModulePassManager(PM);
 312       PM.run(*mod);
 313    }
 314
 315    // Kernel metadata
 316
 317    struct kernel_arg_md {
 318       llvm::StringRef type_name;
 319       llvm::StringRef access_qual;
 320       kernel_arg_md(llvm::StringRef type_name_, llvm::StringRef access_qual_):
 321          type_name(type_name_), access_qual(access_qual_) {}
 322    };
 323 #if HAVE_LLVM >= 0x0309
 324    std::vector<kernel_arg_md>
 325    get_kernel_arg_md(const llvm::Function *kernel_func) {
 326
 327       size_t num_args = kernel_func->getArgumentList().size();
 328
 329       auto aq = kernel_func->getMetadata("kernel_arg_access_qual");
 330       auto ty = kernel_func->getMetadata("kernel_arg_type");
 331
 332       std::vector<kernel_arg_md> res;
 333       res.reserve(num_args);
 334       for (size_t i = 0; i < num_args; ++i) {
 335          res.push_back(kernel_arg_md(
 336             llvm::cast<llvm::MDString>(ty->getOperand(i))->getString(),
 337             llvm::cast<llvm::MDString>(aq->getOperand(i))->getString()));
 338       }
 339
 340       return res;
 341    }
 342
 343 #else
 344
 345    const llvm::MDNode *
 346    get_kernel_metadata(const llvm::Function *kernel_func) {
 347       auto mod = kernel_func->getParent();
 348       auto kernels_node = mod->getNamedMetadata("opencl.kernels");
 349       if (!kernels_node) {
 350          return nullptr;
 351       }
 352
 353       const llvm::MDNode *kernel_node = nullptr;
 354       for (unsigned i = 0; i < kernels_node->getNumOperands(); ++i) {
 355          auto func = llvm::mdconst::dyn_extract<llvm::Function>(
 356                kernels_node->getOperand(i)->getOperand(0));
 357          if (func == kernel_func) {
 358             kernel_node = kernels_node->getOperand(i);
 359             break;
 360          }
 361       }
 362
 363       return kernel_node;
 364    }
 365
 366    llvm::MDNode*
 367    node_from_op_checked(const llvm::MDOperand &md_operand,
 368                         llvm::StringRef expect_name,
 369                         unsigned expect_num_args)
 370    {
 371       auto node = llvm::cast<llvm::MDNode>(md_operand);
 372       assert(node->getNumOperands() == expect_num_args &&
 373              "Wrong number of operands.");
 374
 375       auto str_node = llvm::cast<llvm::MDString>(node->getOperand(0));
 376       assert(str_node->getString() == expect_name &&
 377              "Wrong metadata node name.");
 378
 379       return node;
 380    }
 381
 382    std::vector<kernel_arg_md>
 383    get_kernel_arg_md(const llvm::Function *kernel_func) {
 384       auto num_args = kernel_func->getArgumentList().size();
 385
 386       auto kernel_node = get_kernel_metadata(kernel_func);
 387       auto aq = node_from_op_checked(kernel_node->getOperand(2),
 388                                      "kernel_arg_access_qual", num_args + 1);
 389       auto ty = node_from_op_checked(kernel_node->getOperand(3),
 390                                      "kernel_arg_type", num_args + 1);
 391
 392       std::vector<kernel_arg_md> res;
 393       res.reserve(num_args);
 394       for (unsigned i = 0; i < num_args; ++i) {
 395          res.push_back(kernel_arg_md(
 396             llvm::cast<llvm::MDString>(ty->getOperand(i+1))->getString(),
 397             llvm::cast<llvm::MDString>(aq->getOperand(i+1))->getString()));
 398       }
 399
 400       return res;
 401    }
 402 #endif
 403
 404    std::vector<module::argument>
 405    get_kernel_args(const llvm::Module *mod, const std::string &kernel_name,
 406                    const clang::CompilerInstance &c) {
 407       std::vector<module::argument> args;
 408       const auto address_spaces = c.getTarget().getAddressSpaceMap();
 409       llvm::Function *kernel_func = mod->getFunction(kernel_name);
 410       assert(kernel_func && "Kernel name not found in module.");
 411       auto arg_md = get_kernel_arg_md(kernel_func);
 412
 413       llvm::DataLayout TD(mod);
 414       llvm::Type *size_type =
 415          TD.getSmallestLegalIntType(mod->getContext(), sizeof(cl_uint) * 8);
 416
 417       for (const auto &arg: kernel_func->args()) {
 418
 419          llvm::Type *arg_type = arg.getType();
 420          const unsigned arg_store_size = TD.getTypeStoreSize(arg_type);
 421
 422          // OpenCL 1.2 specification, Ch. 6.1.5: "A built-in data
 423          // type that is not a power of two bytes in size must be
 424          // aligned to the next larger power of two".  We need this
 425          // alignment for three element vectors, which have
 426          // non-power-of-2 store size.
 427          const unsigned arg_api_size = util_next_power_of_two(arg_store_size);
 428
 429          llvm::Type *target_type = arg_type->isIntegerTy() ?
 430                TD.getSmallestLegalIntType(mod->getContext(), arg_store_size * 8)
 431                : arg_type;
 432          unsigned target_size = TD.getTypeStoreSize(target_type);
 433          unsigned target_align = TD.getABITypeAlignment(target_type);
 434
 435          llvm::StringRef type_name = arg_md[arg.getArgNo()].type_name;
 436          llvm::StringRef access_qual = arg_md[arg.getArgNo()].access_qual;
 437
 438          // Image
 439          const bool is_image2d = type_name == "image2d_t";
 440          const bool is_image3d = type_name == "image3d_t";
 441          if (is_image2d || is_image3d) {
 442             const bool is_write_only = access_qual == "write_only";
 443             const bool is_read_only = access_qual == "read_only";
 444
 445             enum module::argument::type marg_type;
 446             if (is_image2d && is_read_only) {
 447                marg_type = module::argument::image2d_rd;
 448             } else if (is_image2d && is_write_only) {
 449                marg_type = module::argument::image2d_wr;
 450             } else if (is_image3d && is_read_only) {
 451                marg_type = module::argument::image3d_rd;
 452             } else if (is_image3d && is_write_only) {
 453                marg_type = module::argument::image3d_wr;
 454             } else {
 455                assert(0 && "Wrong image access qualifier");
 456             }
 457
 458             args.push_back(module::argument(marg_type,
 459                                             arg_store_size, target_size,
 460                                             target_align,
 461                                             module::argument::zero_ext));
 462             continue;
 463          }
 464
 465          // Image size implicit argument
 466          if (type_name == "__llvm_image_size") {
 467             args.push_back(module::argument(module::argument::scalar,
 468                                             sizeof(cl_uint),
 469                                             TD.getTypeStoreSize(size_type),
 470                                             TD.getABITypeAlignment(size_type),
 471                                             module::argument::zero_ext,
 472                                             module::argument::image_size));
 473             continue;
 474          }
 475
 476          // Image format implicit argument
 477          if (type_name == "__llvm_image_format") {
 478             args.push_back(module::argument(module::argument::scalar,
 479                                             sizeof(cl_uint),
 480                                             TD.getTypeStoreSize(size_type),
 481                                             TD.getABITypeAlignment(size_type),
 482                                             module::argument::zero_ext,
 483                                             module::argument::image_format));
 484             continue;
 485          }
 486
 487          // Other types
 488          if (llvm::isa<llvm::PointerType>(arg_type) && arg.hasByValAttr()) {
 489             arg_type =
 490                   llvm::dyn_cast<llvm::PointerType>(arg_type)->getElementType();
 491          }
 492
 493          if (arg_type->isPointerTy()) {
 494             unsigned address_space = llvm::cast<llvm::PointerType>(arg_type)->getAddressSpace();
 495             if (address_space == address_spaces[clang::LangAS::opencl_local
 496                                                      - clang::LangAS::Offset]) {
 497                args.push_back(module::argument(module::argument::local,
 498                                                arg_api_size, target_size,
 499                                                target_align,
 500                                                module::argument::zero_ext));
 501             } else {
 502                // XXX: Correctly handle constant address space.  There is no
 503                // way for r600g to pass a handle for constant buffers back
 504                // to clover like it can for global buffers, so
 505                // creating constant arguments will break r600g.  For now,
 506                // continue treating constant buffers as global buffers
 507                // until we can come up with a way to create handles for
 508                // constant buffers.
 509                args.push_back(module::argument(module::argument::global,
 510                                                arg_api_size, target_size,
 511                                                target_align,
 512                                                module::argument::zero_ext));
 513            }
 514
 515          } else {
 516             llvm::AttributeSet attrs = kernel_func->getAttributes();
 517             enum module::argument::ext_type ext_type =
 518                   (attrs.hasAttribute(arg.getArgNo() + 1,
 519                                      llvm::Attribute::SExt) ?
 520                    module::argument::sign_ext :
 521                    module::argument::zero_ext);
 522
 523             args.push_back(
 524                module::argument(module::argument::scalar, arg_api_size,
 525                                 target_size, target_align, ext_type));
 526          }
 527       }
 528
 529       // Append implicit arguments.  XXX - The types, ordering and
 530       // vector size of the implicit arguments should depend on the
 531       // target according to the selected calling convention.
 532       args.push_back(
 533          module::argument(module::argument::scalar, sizeof(cl_uint),
 534                           TD.getTypeStoreSize(size_type),
 535                           TD.getABITypeAlignment(size_type),
 536                           module::argument::zero_ext,
 537                           module::argument::grid_dimension));
 538
 539       args.push_back(
 540          module::argument(module::argument::scalar, sizeof(cl_uint),
 541                           TD.getTypeStoreSize(size_type),
 542                           TD.getABITypeAlignment(size_type),
 543                           module::argument::zero_ext,
 544                           module::argument::grid_offset));
 545
 546       return args;
 547    }
 548
 549    module
 550    build_module_llvm(llvm::Module *mod,
 551                      const clang::CompilerInstance &c) {
 552
 553       module m;
 554       struct pipe_llvm_program_header header;
 555
 556       llvm::SmallVector<char, 1024> llvm_bitcode;
 557       llvm::raw_svector_ostream bitcode_ostream(llvm_bitcode);
 558       llvm::BitstreamWriter writer(llvm_bitcode);
 559       llvm::WriteBitcodeToFile(mod, bitcode_ostream);
 560 #if HAVE_LLVM < 0x0308
 561       bitcode_ostream.flush();
 562 #endif
 563
 564       const std::vector<llvm::Function *> kernels = find_kernels(mod);
 565       for (unsigned i = 0; i < kernels.size(); ++i) {
 566          std::string kernel_name = kernels[i]->getName();
 567          std::vector<module::argument> args =
 568                get_kernel_args(mod, kernel_name, c);
 569
 570          m.syms.push_back(module::symbol(kernel_name, 0, i, args ));
 571       }
 572
 573       header.num_bytes = llvm_bitcode.size();
 574       std::vector<char> data;
 575       data.insert(data.end(), (char*)(&header),
 576                               (char*)(&header) + sizeof(header));
 577       data.insert(data.end(), llvm_bitcode.begin(),
 578                                   llvm_bitcode.end());
 579       m.secs.push_back(module::section(0, module::section::text,
 580                                        header.num_bytes, data));
 581
 582       return m;
 583    }
 584
 585    void
 586    emit_code(LLVMTargetMachineRef tm, LLVMModuleRef mod,
 587              LLVMCodeGenFileType file_type,
 588              LLVMMemoryBufferRef *out_buffer,
 589              std::string &r_log) {
 590       LLVMBool err;
 591       char *err_message = NULL;
 592
 593       err = LLVMTargetMachineEmitToMemoryBuffer(tm, mod, file_type,
 594                                                 &err_message, out_buffer);
 595
 596       if (err) {
 597          r_log = std::string(err_message);
 598       }
 599
 600       LLVMDisposeMessage(err_message);
 601
 602       if (err) {
 603          throw compile_error();
 604       }
 605    }
 606
 607    std::vector<char>
 608    compile_native(const llvm::Module *mod, const target &t, bool dump_asm,
 609                   std::string &r_log) {
 610
 611       std::string log;
 612       LLVMTargetRef target;
 613       char *error_message;
 614       LLVMMemoryBufferRef out_buffer;
 615       unsigned buffer_size;
 616       const char *buffer_data;
 617       LLVMModuleRef mod_ref = wrap(mod);
 618
 619       if (LLVMGetTargetFromTriple(t.triple.c_str(), &target, &error_message)) {
 620          r_log = std::string(error_message);
 621          LLVMDisposeMessage(error_message);
 622          throw compile_error();
 623       }
 624
 625       LLVMTargetMachineRef tm = LLVMCreateTargetMachine(
 626             target, t.triple.c_str(), t.cpu.c_str(), "",
 627             LLVMCodeGenLevelDefault, LLVMRelocDefault, LLVMCodeModelDefault);
 628
 629       if (!tm) {
 630          r_log = "Could not create TargetMachine: " + t.triple;
 631          throw compile_error();
 632       }
 633
 634       if (dump_asm) {
 635          LLVMSetTargetMachineAsmVerbosity(tm, true);
 636 #if HAVE_LLVM >= 0x0308
 637          LLVMModuleRef debug_mod = wrap(llvm::CloneModule(mod).release());
 638 #else
 639          LLVMModuleRef debug_mod = wrap(llvm::CloneModule(mod));
 640 #endif
 641          emit_code(tm, debug_mod, LLVMAssemblyFile, &out_buffer, r_log);
 642          buffer_size = LLVMGetBufferSize(out_buffer);
 643          buffer_data = LLVMGetBufferStart(out_buffer);
 644          debug_log(std::string(buffer_data, buffer_size), ".asm");
 645
 646          LLVMSetTargetMachineAsmVerbosity(tm, false);
 647          LLVMDisposeMemoryBuffer(out_buffer);
 648          LLVMDisposeModule(debug_mod);
 649       }
 650
 651       emit_code(tm, mod_ref, LLVMObjectFile, &out_buffer, r_log);
 652
 653       buffer_size = LLVMGetBufferSize(out_buffer);
 654       buffer_data = LLVMGetBufferStart(out_buffer);
 655
 656       std::vector<char> code(buffer_data, buffer_data + buffer_size);
 657
 658       LLVMDisposeMemoryBuffer(out_buffer);
 659       LLVMDisposeTargetMachine(tm);
 660
 661       return code;
 662    }
 663
 664    std::map<std::string, unsigned>
 665    get_kernel_offsets(std::vector<char> &code,
 666                       const std::vector<llvm::Function *> &kernels,
 667                       std::string &r_log) {
 668
 669       // One of the libelf implementations
 670       // (http://www.mr511.de/software/english.htm) requires calling
 671       // elf_version() before elf_memory().
 672       //
 673       elf_version(EV_CURRENT);
 674
 675       Elf *elf = elf_memory(&code[0], code.size());
 676       size_t section_str_index;
 677       elf_getshdrstrndx(elf, &section_str_index);
 678       Elf_Scn *section = NULL;
 679       Elf_Scn *symtab = NULL;
 680       GElf_Shdr symtab_header;
 681
 682       // Find the symbol table
 683       try {
 684          while ((section = elf_nextscn(elf, section))) {
 685             const char *name;
 686             if (gelf_getshdr(section, &symtab_header) != &symtab_header) {
 687                r_log = "Failed to read ELF section header.";
 688                throw compile_error();
 689             }
 690             name = elf_strptr(elf, section_str_index, symtab_header.sh_name);
 691            if (!strcmp(name, ".symtab")) {
 692                symtab = section;
 693                break;
 694            }
 695          }
 696          if (!symtab) {
 697             r_log = "Unable to find symbol table.";
 698             throw compile_error();
 699          }
 700       } catch (compile_error &e) {
 701          elf_end(elf);
 702          throw e;
 703       }
 704
 705
 706       // Extract symbol information from the table
 707       Elf_Data *symtab_data = NULL;
 708       GElf_Sym *symbol;
 709       GElf_Sym s;
 710
 711       std::map<std::string, unsigned> kernel_offsets;
 712       symtab_data = elf_getdata(symtab, symtab_data);
 713
 714       // Determine the offsets for each kernel
 715       for (int i = 0; (symbol = gelf_getsym(symtab_data, i, &s)); i++) {
 716          char *name = elf_strptr(elf, symtab_header.sh_link, symbol->st_name);
 717          for (std::vector<llvm::Function*>::const_iterator it = kernels.begin(),
 718               e = kernels.end(); it != e; ++it) {
 719             llvm::Function *f = *it;
 720             if (f->getName() == std::string(name))
 721                kernel_offsets[f->getName()] = symbol->st_value;
 722          }
 723       }
 724       elf_end(elf);
 725       return kernel_offsets;
 726    }
 727
 728    module
 729    build_module_native(std::vector<char> &code,
 730                        llvm::Module *mod,
 731                        const clang::CompilerInstance &c,
 732                        std::string &r_log) {
 733
 734       const std::vector<llvm::Function *> kernels = find_kernels(mod);
 735
 736       std::map<std::string, unsigned> kernel_offsets =
 737             get_kernel_offsets(code, kernels, r_log);
 738
 739       // Begin building the clover module
 740       module m;
 741       struct pipe_llvm_program_header header;
 742
 743       // Store the generated ELF binary in the module's text section.
 744       header.num_bytes = code.size();
 745       std::vector<char> data;
 746       data.insert(data.end(), (char*)(&header),
 747                               (char*)(&header) + sizeof(header));
 748       data.insert(data.end(), code.begin(), code.end());
 749       m.secs.push_back(module::section(0, module::section::text,
 750                                        header.num_bytes, data));
 751
 752       for (std::map<std::string, unsigned>::iterator i = kernel_offsets.begin(),
 753            e = kernel_offsets.end(); i != e; ++i) {
 754          std::vector<module::argument> args = get_kernel_args(mod, i->first, c);
 755          m.syms.push_back(module::symbol(i->first, 0, i->second, args ));
 756       }
 757
 758       return m;
 759    }
 760
 761 #define DBG_CLC  (1 << 0)
 762 #define DBG_LLVM (1 << 1)
 763 #define DBG_ASM  (1 << 2)
 764
 765    unsigned
 766    get_debug_flags() {
 767       static const struct debug_named_value debug_options[] = {
 768          {"clc", DBG_CLC, "Dump the OpenCL C code for all kernels."},
 769          {"llvm", DBG_LLVM, "Dump the generated LLVM IR for all kernels."},
 770          {"asm", DBG_ASM, "Dump kernel assembly code for targets specifying "
 771           "PIPE_SHADER_IR_NATIVE"},
 772          DEBUG_NAMED_VALUE_END // must be last
 773       };
 774       static const unsigned debug_flags =
 775          debug_get_flags_option("CLOVER_DEBUG", debug_options, 0);
 776
 777       return debug_flags;
 778    }
 779
 780 } // End anonymous namespace
 781
 782 module
 783 clover::compile_program_llvm(const std::string &source,
 784                              const header_map &headers,
 785                              enum pipe_shader_ir ir,
 786                              const std::string &target,
 787                              const std::string &opts,
 788                              std::string &r_log) {
 789    if (get_debug_flags() & DBG_CLC)
 790       debug_log("// Build options: " + opts + '\n' + source, ".cl");
 791
 792    auto ctx = create_context(r_log);
 793    // The input file name must have the .cl extension in order for the
 794    // CompilerInvocation class to recognize it as an OpenCL source file.
 795    const auto c = create_compiler_instance(target, tokenize(opts + " input.cl"),
 796                                            r_log);
 797    Module *mod = compile_llvm(*ctx, *c, source, headers, "input.cl",
 798                               target, opts, r_log);
 799
 800    optimize(mod, c->getCodeGenOpts().OptimizationLevel);
 801
 802    if (get_debug_flags() & DBG_LLVM) {
 803       std::string log;
 804       raw_string_ostream s_log(log);
 805       mod->print(s_log, NULL);
 806       s_log.flush();
 807       debug_log(log, ".ll");
 808     }
 809
 810    module m;
 811    // Build the clover::module
 812    switch (ir) {
 813       case PIPE_SHADER_IR_NIR:
 814       case PIPE_SHADER_IR_TGSI:
 815          //XXX: Handle TGSI, NIR
 816          assert(0);
 817          m = module();
 818          break;
 819       case PIPE_SHADER_IR_LLVM:
 820          m = build_module_llvm(mod, *c);
 821          break;
 822       case PIPE_SHADER_IR_NATIVE: {
 823          std::vector<char> code = compile_native(mod, target,
 824                                                  get_debug_flags() & DBG_ASM,
 825                                                  r_log);
 826          m = build_module_native(code, mod, *c, r_log);
 827          break;
 828       }
 829    }
 830    // The user takes ownership of the module.
 831    delete mod;
 832    return m;
 833 }