src/gallium/state_trackers/clover/llvm/invocation.cpp

   1 //
   2 // Copyright 2012-2016 Francisco Jerez
   3 // Copyright 2012-2016 Advanced Micro Devices, Inc.
   4 // Copyright 2014-2016 Jan Vesely
   5 // Copyright 2014-2015 Serge Martin
   6 // Copyright 2015 Zoltan Gilian
   7 //
   8 // Permission is hereby granted, free of charge, to any person obtaining a
   9 // copy of this software and associated documentation files (the "Software"),
  10 // to deal in the Software without restriction, including without limitation
  11 // the rights to use, copy, modify, merge, publish, distribute, sublicense,
  12 // and/or sell copies of the Software, and to permit persons to whom the
  13 // Software is furnished to do so, subject to the following conditions:
  14 //
  15 // The above copyright notice and this permission notice shall be included in
  16 // all copies or substantial portions of the Software.
  17 //
  18 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  21 // THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  22 // OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  23 // ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  24 // OTHER DEALINGS IN THE SOFTWARE.
  25 //
  26
  27 #include "llvm/compat.hpp"
  28 #include "core/compiler.hpp"
  29 #include "util/algorithm.hpp"
  30
  31 #include <clang/Frontend/CompilerInstance.h>
  32 #include <clang/Frontend/TextDiagnosticBuffer.h>
  33 #include <clang/Frontend/TextDiagnosticPrinter.h>
  34 #include <clang/CodeGen/CodeGenAction.h>
  35 #include <clang/Basic/TargetInfo.h>
  36 #include <llvm/Bitcode/BitstreamWriter.h>
  37 #include <llvm/Bitcode/ReaderWriter.h>
  38 #include <llvm/Linker/Linker.h>
  39 #include <llvm/IR/DiagnosticInfo.h>
  40 #include <llvm/IR/DiagnosticPrinter.h>
  41 #include <llvm/IR/DerivedTypes.h>
  42 #include <llvm/IR/LLVMContext.h>
  43 #include <llvm/IR/Module.h>
  44 #include <llvm/Support/SourceMgr.h>
  45 #include <llvm/IRReader/IRReader.h>
  46 #include <llvm/Support/CodeGen.h>
  47 #include <llvm/Support/TargetSelect.h>
  48 #include <llvm/Support/MemoryBuffer.h>
  49 #include <llvm/Support/FormattedStream.h>
  50 #include <llvm/Support/TargetRegistry.h>
  51 #include <llvm/Transforms/IPO.h>
  52 #include <llvm/Transforms/IPO/PassManagerBuilder.h>
  53 #include <llvm/Transforms/Utils/Cloning.h>
  54
  55
  56 #include <llvm/IR/DataLayout.h>
  57 #include <llvm/Target/TargetMachine.h>
  58 #include <llvm/Target/TargetOptions.h>
  59
  60 #include <llvm-c/Target.h>
  61 #include <llvm-c/TargetMachine.h>
  62 #include <llvm-c/Core.h>
  63
  64 #include "pipe/p_state.h"
  65 #include "util/u_memory.h"
  66 #include "util/u_math.h"
  67
  68 #include <iostream>
  69 #include <iomanip>
  70 #include <fstream>
  71 #include <cstdio>
  72 #include <sstream>
  73 #include <libelf.h>
  74 #include <gelf.h>
  75
  76 using namespace clover;
  77 using namespace clover::llvm;
  78
  79 using ::llvm::Function;
  80 using ::llvm::Module;
  81 using ::llvm::raw_string_ostream;
  82
  83 namespace {
  84    // XXX - Temporary hack to avoid breaking the build for the moment, will
  85    //       get rid of this later.
  86    namespace llvm {
  87       using namespace ::llvm;
  88    }
  89
  90    void debug_log(const std::string &msg, const std::string &suffix) {
  91       const char *dbg_file = debug_get_option("CLOVER_DEBUG_FILE", "stderr");
  92       if (!strcmp("stderr", dbg_file)) {
  93          std::cerr << msg;
  94        } else {
  95         std::ofstream file(dbg_file + suffix, std::ios::app);
  96         file << msg;
  97        }
  98    }
  99
 100    inline std::vector<std::string>
 101    tokenize(const std::string &s) {
 102       std::vector<std::string> ss;
 103       std::istringstream iss(s);
 104       std::string t;
 105
 106       while (getline(iss, t, ' '))
 107          ss.push_back(t);
 108
 109       return ss;
 110    }
 111
 112    struct target {
 113       target(const std::string &s) :
 114          cpu(s.begin(), s.begin() + s.find_first_of("-")),
 115          triple(s.begin() + s.find_first_of("-") + 1, s.end()) {}
 116
 117       std::string cpu;
 118       std::string triple;
 119    };
 120
 121    std::unique_ptr<clang::CompilerInstance>
 122    create_compiler_instance(const target &target,
 123                             const std::vector<std::string> &opts,
 124                             std::string &r_log) {
 125       std::unique_ptr<clang::CompilerInstance> c { new clang::CompilerInstance };
 126       clang::DiagnosticsEngine diag { new clang::DiagnosticIDs,
 127             new clang::DiagnosticOptions, new clang::TextDiagnosticBuffer };
 128
 129       // Parse the compiler options.  A file name should be present at the end
 130       // and must have the .cl extension in order for the CompilerInvocation
 131       // class to recognize it as an OpenCL source file.
 132       const std::vector<const char *> copts =
 133          map(std::mem_fn(&std::string::c_str), opts);
 134
 135       if (!clang::CompilerInvocation::CreateFromArgs(
 136              c->getInvocation(), copts.data(), copts.data() + copts.size(), diag))
 137          throw error(CL_INVALID_COMPILER_OPTIONS);
 138
 139       c->getTargetOpts().CPU = target.cpu;
 140       c->getTargetOpts().Triple = target.triple;
 141       c->getLangOpts().NoBuiltin = true;
 142
 143       // This is a workaround for a Clang bug which causes the number
 144       // of warnings and errors to be printed to stderr.
 145       // http://www.llvm.org/bugs/show_bug.cgi?id=19735
 146       c->getDiagnosticOpts().ShowCarets = false;
 147
 148       compat::set_lang_defaults(c->getInvocation(), c->getLangOpts(),
 149                                 clang::IK_OpenCL, ::llvm::Triple(target.triple),
 150                                 c->getPreprocessorOpts(),
 151                                 clang::LangStandard::lang_opencl11);
 152
 153       c->createDiagnostics(new clang::TextDiagnosticPrinter(
 154                               *new raw_string_ostream(r_log),
 155                               &c->getDiagnosticOpts(), true));
 156
 157       c->setTarget(clang::TargetInfo::CreateTargetInfo(
 158                            c->getDiagnostics(), c->getInvocation().TargetOpts));
 159
 160       return c;
 161    }
 162
 163    llvm::Module *
 164    compile_llvm(llvm::LLVMContext &llvm_ctx, const std::string &source,
 165                 const header_map &headers, const std::string &name,
 166                 const std::string &target, const std::string &opts,
 167                 clang::LangAS::Map &address_spaces, unsigned &optimization_level,
 168                 std::string &r_log) {
 169       clang::EmitLLVMOnlyAction act(&llvm_ctx);
 170
 171       std::unique_ptr<clang::CompilerInstance> pc =
 172          create_compiler_instance(target, tokenize(opts + " " + name), r_log);
 173       auto &c = *pc;
 174       c.getFrontendOpts().ProgramAction = clang::frontend::EmitLLVMOnly;
 175       c.getHeaderSearchOpts().UseBuiltinIncludes = true;
 176       c.getHeaderSearchOpts().UseStandardSystemIncludes = true;
 177       c.getHeaderSearchOpts().ResourceDir = CLANG_RESOURCE_DIR;
 178
 179       // Add libclc generic search path
 180       c.getHeaderSearchOpts().AddPath(LIBCLC_INCLUDEDIR,
 181                                       clang::frontend::Angled,
 182                                       false, false
 183                                       );
 184
 185       // Add libclc include
 186       c.getPreprocessorOpts().Includes.push_back("clc/clc.h");
 187
 188       // clc.h requires that this macro be defined:
 189       c.getPreprocessorOpts().addMacroDef("cl_clang_storage_class_specifiers");
 190
 191       c.getPreprocessorOpts().addRemappedFile(name,
 192                                               llvm::MemoryBuffer::getMemBuffer(source).release());
 193
 194       if (headers.size()) {
 195          const std::string tmp_header_path = "/tmp/clover/";
 196
 197          c.getHeaderSearchOpts().AddPath(tmp_header_path,
 198                                          clang::frontend::Angled,
 199                                          false, false
 200                                          );
 201
 202          for (header_map::const_iterator it = headers.begin();
 203               it != headers.end(); ++it) {
 204             const std::string path = tmp_header_path + std::string(it->first);
 205             c.getPreprocessorOpts().addRemappedFile(path,
 206                     llvm::MemoryBuffer::getMemBuffer(it->second.c_str()).release());
 207          }
 208       }
 209
 210       // Tell clang to link this file before performing any
 211       // optimizations.  This is required so that we can replace calls
 212       // to the OpenCL C barrier() builtin with calls to target
 213       // intrinsics that have the noduplicate attribute.  This
 214       // attribute will prevent Clang from creating illegal uses of
 215       // barrier() (e.g. Moving barrier() inside a conditional that is
 216       // no executed by all threads) during its optimizaton passes.
 217       compat::add_link_bitcode_file(c.getCodeGenOpts(),
 218                                     LIBCLC_LIBEXECDIR + target + ".bc");
 219
 220       optimization_level = c.getCodeGenOpts().OptimizationLevel;
 221
 222       // Compile the code
 223       if (!c.ExecuteAction(act))
 224          throw compile_error();
 225
 226       // Get address spaces map to be able to find kernel argument address space
 227       memcpy(address_spaces, c.getTarget().getAddressSpaceMap(),
 228                                                         sizeof(address_spaces));
 229
 230       return act.takeModule().release();
 231    }
 232
 233    std::vector<llvm::Function *>
 234    find_kernels(llvm::Module *mod) {
 235       std::vector<llvm::Function *> kernels;
 236 #if HAVE_LLVM >= 0x0309
 237       auto &list = mod->getFunctionList();
 238       for_each(list.begin(), list.end(), [&](llvm::Function &f){
 239          if (f.getMetadata("kernel_arg_type"))
 240            kernels.push_back(&f);
 241       });
 242       return kernels;
 243 #endif
 244       const llvm::NamedMDNode *kernel_node =
 245                                  mod->getNamedMetadata("opencl.kernels");
 246       // This means there are no kernels in the program.  The spec does not
 247       // require that we return an error here, but there will be an error if
 248       // the user tries to pass this program to a clCreateKernel() call.
 249       if (!kernel_node) {
 250          return std::vector<llvm::Function *>();
 251       }
 252
 253       kernels.reserve(kernel_node->getNumOperands());
 254       for (unsigned i = 0; i < kernel_node->getNumOperands(); ++i) {
 255          kernels.push_back(llvm::mdconst::dyn_extract<llvm::Function>(
 256                                     kernel_node->getOperand(i)->getOperand(0)));
 257       }
 258       return kernels;
 259    }
 260
 261    void
 262    optimize(llvm::Module *mod, unsigned optimization_level) {
 263       compat::pass_manager PM;
 264
 265       compat::add_data_layout_pass(PM);
 266
 267       // By default, the function internalizer pass will look for a function
 268       // called "main" and then mark all other functions as internal.  Marking
 269       // functions as internal enables the optimizer to perform optimizations
 270       // like function inlining and global dead-code elimination.
 271       //
 272       // When there is no "main" function in a module, the internalize pass will
 273       // treat the module like a library, and it won't internalize any functions.
 274       // Since there is no "main" function in our kernels, we need to tell
 275       // the internalizer pass that this module is not a library by passing a
 276       // list of kernel functions to the internalizer.  The internalizer will
 277       // treat the functions in the list as "main" functions and internalize
 278       // all of the other functions.
 279       compat::add_internalize_pass(PM, map(std::mem_fn(&Function::getName),
 280                                            find_kernels(mod)));
 281
 282       llvm::PassManagerBuilder PMB;
 283       PMB.OptLevel = optimization_level;
 284       PMB.LibraryInfo = new compat::target_library_info(
 285          llvm::Triple(mod->getTargetTriple()));
 286       PMB.populateModulePassManager(PM);
 287       PM.run(*mod);
 288    }
 289
 290    // Kernel metadata
 291
 292    struct kernel_arg_md {
 293       llvm::StringRef type_name;
 294       llvm::StringRef access_qual;
 295       kernel_arg_md(llvm::StringRef type_name_, llvm::StringRef access_qual_):
 296          type_name(type_name_), access_qual(access_qual_) {}
 297    };
 298 #if HAVE_LLVM >= 0x0309
 299    std::vector<kernel_arg_md>
 300    get_kernel_arg_md(const llvm::Function *kernel_func) {
 301
 302       size_t num_args = kernel_func->getArgumentList().size();
 303
 304       auto aq = kernel_func->getMetadata("kernel_arg_access_qual");
 305       auto ty = kernel_func->getMetadata("kernel_arg_type");
 306
 307       std::vector<kernel_arg_md> res;
 308       res.reserve(num_args);
 309       for (size_t i = 0; i < num_args; ++i) {
 310          res.push_back(kernel_arg_md(
 311             llvm::cast<llvm::MDString>(ty->getOperand(i))->getString(),
 312             llvm::cast<llvm::MDString>(aq->getOperand(i))->getString()));
 313       }
 314
 315       return res;
 316    }
 317
 318 #else
 319
 320    const llvm::MDNode *
 321    get_kernel_metadata(const llvm::Function *kernel_func) {
 322       auto mod = kernel_func->getParent();
 323       auto kernels_node = mod->getNamedMetadata("opencl.kernels");
 324       if (!kernels_node) {
 325          return nullptr;
 326       }
 327
 328       const llvm::MDNode *kernel_node = nullptr;
 329       for (unsigned i = 0; i < kernels_node->getNumOperands(); ++i) {
 330          auto func = llvm::mdconst::dyn_extract<llvm::Function>(
 331                kernels_node->getOperand(i)->getOperand(0));
 332          if (func == kernel_func) {
 333             kernel_node = kernels_node->getOperand(i);
 334             break;
 335          }
 336       }
 337
 338       return kernel_node;
 339    }
 340
 341    llvm::MDNode*
 342    node_from_op_checked(const llvm::MDOperand &md_operand,
 343                         llvm::StringRef expect_name,
 344                         unsigned expect_num_args)
 345    {
 346       auto node = llvm::cast<llvm::MDNode>(md_operand);
 347       assert(node->getNumOperands() == expect_num_args &&
 348              "Wrong number of operands.");
 349
 350       auto str_node = llvm::cast<llvm::MDString>(node->getOperand(0));
 351       assert(str_node->getString() == expect_name &&
 352              "Wrong metadata node name.");
 353
 354       return node;
 355    }
 356
 357    std::vector<kernel_arg_md>
 358    get_kernel_arg_md(const llvm::Function *kernel_func) {
 359       auto num_args = kernel_func->getArgumentList().size();
 360
 361       auto kernel_node = get_kernel_metadata(kernel_func);
 362       auto aq = node_from_op_checked(kernel_node->getOperand(2),
 363                                      "kernel_arg_access_qual", num_args + 1);
 364       auto ty = node_from_op_checked(kernel_node->getOperand(3),
 365                                      "kernel_arg_type", num_args + 1);
 366
 367       std::vector<kernel_arg_md> res;
 368       res.reserve(num_args);
 369       for (unsigned i = 0; i < num_args; ++i) {
 370          res.push_back(kernel_arg_md(
 371             llvm::cast<llvm::MDString>(ty->getOperand(i+1))->getString(),
 372             llvm::cast<llvm::MDString>(aq->getOperand(i+1))->getString()));
 373       }
 374
 375       return res;
 376    }
 377 #endif
 378
 379    std::vector<module::argument>
 380    get_kernel_args(const llvm::Module *mod, const std::string &kernel_name,
 381                    const clang::LangAS::Map &address_spaces) {
 382
 383       std::vector<module::argument> args;
 384       llvm::Function *kernel_func = mod->getFunction(kernel_name);
 385       assert(kernel_func && "Kernel name not found in module.");
 386       auto arg_md = get_kernel_arg_md(kernel_func);
 387
 388       llvm::DataLayout TD(mod);
 389       llvm::Type *size_type =
 390          TD.getSmallestLegalIntType(mod->getContext(), sizeof(cl_uint) * 8);
 391
 392       for (const auto &arg: kernel_func->args()) {
 393
 394          llvm::Type *arg_type = arg.getType();
 395          const unsigned arg_store_size = TD.getTypeStoreSize(arg_type);
 396
 397          // OpenCL 1.2 specification, Ch. 6.1.5: "A built-in data
 398          // type that is not a power of two bytes in size must be
 399          // aligned to the next larger power of two".  We need this
 400          // alignment for three element vectors, which have
 401          // non-power-of-2 store size.
 402          const unsigned arg_api_size = util_next_power_of_two(arg_store_size);
 403
 404          llvm::Type *target_type = arg_type->isIntegerTy() ?
 405                TD.getSmallestLegalIntType(mod->getContext(), arg_store_size * 8)
 406                : arg_type;
 407          unsigned target_size = TD.getTypeStoreSize(target_type);
 408          unsigned target_align = TD.getABITypeAlignment(target_type);
 409
 410          llvm::StringRef type_name = arg_md[arg.getArgNo()].type_name;
 411          llvm::StringRef access_qual = arg_md[arg.getArgNo()].access_qual;
 412
 413          // Image
 414          const bool is_image2d = type_name == "image2d_t";
 415          const bool is_image3d = type_name == "image3d_t";
 416          if (is_image2d || is_image3d) {
 417             const bool is_write_only = access_qual == "write_only";
 418             const bool is_read_only = access_qual == "read_only";
 419
 420             enum module::argument::type marg_type;
 421             if (is_image2d && is_read_only) {
 422                marg_type = module::argument::image2d_rd;
 423             } else if (is_image2d && is_write_only) {
 424                marg_type = module::argument::image2d_wr;
 425             } else if (is_image3d && is_read_only) {
 426                marg_type = module::argument::image3d_rd;
 427             } else if (is_image3d && is_write_only) {
 428                marg_type = module::argument::image3d_wr;
 429             } else {
 430                assert(0 && "Wrong image access qualifier");
 431             }
 432
 433             args.push_back(module::argument(marg_type,
 434                                             arg_store_size, target_size,
 435                                             target_align,
 436                                             module::argument::zero_ext));
 437             continue;
 438          }
 439
 440          // Image size implicit argument
 441          if (type_name == "__llvm_image_size") {
 442             args.push_back(module::argument(module::argument::scalar,
 443                                             sizeof(cl_uint),
 444                                             TD.getTypeStoreSize(size_type),
 445                                             TD.getABITypeAlignment(size_type),
 446                                             module::argument::zero_ext,
 447                                             module::argument::image_size));
 448             continue;
 449          }
 450
 451          // Image format implicit argument
 452          if (type_name == "__llvm_image_format") {
 453             args.push_back(module::argument(module::argument::scalar,
 454                                             sizeof(cl_uint),
 455                                             TD.getTypeStoreSize(size_type),
 456                                             TD.getABITypeAlignment(size_type),
 457                                             module::argument::zero_ext,
 458                                             module::argument::image_format));
 459             continue;
 460          }
 461
 462          // Other types
 463          if (llvm::isa<llvm::PointerType>(arg_type) && arg.hasByValAttr()) {
 464             arg_type =
 465                   llvm::dyn_cast<llvm::PointerType>(arg_type)->getElementType();
 466          }
 467
 468          if (arg_type->isPointerTy()) {
 469             unsigned address_space = llvm::cast<llvm::PointerType>(arg_type)->getAddressSpace();
 470             if (address_space == address_spaces[clang::LangAS::opencl_local
 471                                                      - clang::LangAS::Offset]) {
 472                args.push_back(module::argument(module::argument::local,
 473                                                arg_api_size, target_size,
 474                                                target_align,
 475                                                module::argument::zero_ext));
 476             } else {
 477                // XXX: Correctly handle constant address space.  There is no
 478                // way for r600g to pass a handle for constant buffers back
 479                // to clover like it can for global buffers, so
 480                // creating constant arguments will break r600g.  For now,
 481                // continue treating constant buffers as global buffers
 482                // until we can come up with a way to create handles for
 483                // constant buffers.
 484                args.push_back(module::argument(module::argument::global,
 485                                                arg_api_size, target_size,
 486                                                target_align,
 487                                                module::argument::zero_ext));
 488            }
 489
 490          } else {
 491             llvm::AttributeSet attrs = kernel_func->getAttributes();
 492             enum module::argument::ext_type ext_type =
 493                   (attrs.hasAttribute(arg.getArgNo() + 1,
 494                                      llvm::Attribute::SExt) ?
 495                    module::argument::sign_ext :
 496                    module::argument::zero_ext);
 497
 498             args.push_back(
 499                module::argument(module::argument::scalar, arg_api_size,
 500                                 target_size, target_align, ext_type));
 501          }
 502       }
 503
 504       // Append implicit arguments.  XXX - The types, ordering and
 505       // vector size of the implicit arguments should depend on the
 506       // target according to the selected calling convention.
 507       args.push_back(
 508          module::argument(module::argument::scalar, sizeof(cl_uint),
 509                           TD.getTypeStoreSize(size_type),
 510                           TD.getABITypeAlignment(size_type),
 511                           module::argument::zero_ext,
 512                           module::argument::grid_dimension));
 513
 514       args.push_back(
 515          module::argument(module::argument::scalar, sizeof(cl_uint),
 516                           TD.getTypeStoreSize(size_type),
 517                           TD.getABITypeAlignment(size_type),
 518                           module::argument::zero_ext,
 519                           module::argument::grid_offset));
 520
 521       return args;
 522    }
 523
 524    module
 525    build_module_llvm(llvm::Module *mod,
 526                      clang::LangAS::Map& address_spaces) {
 527
 528       module m;
 529       struct pipe_llvm_program_header header;
 530
 531       llvm::SmallVector<char, 1024> llvm_bitcode;
 532       llvm::raw_svector_ostream bitcode_ostream(llvm_bitcode);
 533       llvm::BitstreamWriter writer(llvm_bitcode);
 534       llvm::WriteBitcodeToFile(mod, bitcode_ostream);
 535 #if HAVE_LLVM < 0x0308
 536       bitcode_ostream.flush();
 537 #endif
 538
 539       const std::vector<llvm::Function *> kernels = find_kernels(mod);
 540       for (unsigned i = 0; i < kernels.size(); ++i) {
 541          std::string kernel_name = kernels[i]->getName();
 542          std::vector<module::argument> args =
 543                get_kernel_args(mod, kernel_name, address_spaces);
 544
 545          m.syms.push_back(module::symbol(kernel_name, 0, i, args ));
 546       }
 547
 548       header.num_bytes = llvm_bitcode.size();
 549       std::vector<char> data;
 550       data.insert(data.end(), (char*)(&header),
 551                               (char*)(&header) + sizeof(header));
 552       data.insert(data.end(), llvm_bitcode.begin(),
 553                                   llvm_bitcode.end());
 554       m.secs.push_back(module::section(0, module::section::text,
 555                                        header.num_bytes, data));
 556
 557       return m;
 558    }
 559
 560    void
 561    emit_code(LLVMTargetMachineRef tm, LLVMModuleRef mod,
 562              LLVMCodeGenFileType file_type,
 563              LLVMMemoryBufferRef *out_buffer,
 564              std::string &r_log) {
 565       LLVMBool err;
 566       char *err_message = NULL;
 567
 568       err = LLVMTargetMachineEmitToMemoryBuffer(tm, mod, file_type,
 569                                                 &err_message, out_buffer);
 570
 571       if (err) {
 572          r_log = std::string(err_message);
 573       }
 574
 575       LLVMDisposeMessage(err_message);
 576
 577       if (err) {
 578          throw compile_error();
 579       }
 580    }
 581
 582    std::vector<char>
 583    compile_native(const llvm::Module *mod, const target &t, bool dump_asm,
 584                   std::string &r_log) {
 585
 586       std::string log;
 587       LLVMTargetRef target;
 588       char *error_message;
 589       LLVMMemoryBufferRef out_buffer;
 590       unsigned buffer_size;
 591       const char *buffer_data;
 592       LLVMModuleRef mod_ref = wrap(mod);
 593
 594       if (LLVMGetTargetFromTriple(t.triple.c_str(), &target, &error_message)) {
 595          r_log = std::string(error_message);
 596          LLVMDisposeMessage(error_message);
 597          throw compile_error();
 598       }
 599
 600       LLVMTargetMachineRef tm = LLVMCreateTargetMachine(
 601             target, t.triple.c_str(), t.cpu.c_str(), "",
 602             LLVMCodeGenLevelDefault, LLVMRelocDefault, LLVMCodeModelDefault);
 603
 604       if (!tm) {
 605          r_log = "Could not create TargetMachine: " + t.triple;
 606          throw compile_error();
 607       }
 608
 609       if (dump_asm) {
 610          LLVMSetTargetMachineAsmVerbosity(tm, true);
 611 #if HAVE_LLVM >= 0x0308
 612          LLVMModuleRef debug_mod = wrap(llvm::CloneModule(mod).release());
 613 #else
 614          LLVMModuleRef debug_mod = wrap(llvm::CloneModule(mod));
 615 #endif
 616          emit_code(tm, debug_mod, LLVMAssemblyFile, &out_buffer, r_log);
 617          buffer_size = LLVMGetBufferSize(out_buffer);
 618          buffer_data = LLVMGetBufferStart(out_buffer);
 619          debug_log(std::string(buffer_data, buffer_size), ".asm");
 620
 621          LLVMSetTargetMachineAsmVerbosity(tm, false);
 622          LLVMDisposeMemoryBuffer(out_buffer);
 623          LLVMDisposeModule(debug_mod);
 624       }
 625
 626       emit_code(tm, mod_ref, LLVMObjectFile, &out_buffer, r_log);
 627
 628       buffer_size = LLVMGetBufferSize(out_buffer);
 629       buffer_data = LLVMGetBufferStart(out_buffer);
 630
 631       std::vector<char> code(buffer_data, buffer_data + buffer_size);
 632
 633       LLVMDisposeMemoryBuffer(out_buffer);
 634       LLVMDisposeTargetMachine(tm);
 635
 636       return code;
 637    }
 638
 639    std::map<std::string, unsigned>
 640    get_kernel_offsets(std::vector<char> &code,
 641                       const std::vector<llvm::Function *> &kernels,
 642                       std::string &r_log) {
 643
 644       // One of the libelf implementations
 645       // (http://www.mr511.de/software/english.htm) requires calling
 646       // elf_version() before elf_memory().
 647       //
 648       elf_version(EV_CURRENT);
 649
 650       Elf *elf = elf_memory(&code[0], code.size());
 651       size_t section_str_index;
 652       elf_getshdrstrndx(elf, &section_str_index);
 653       Elf_Scn *section = NULL;
 654       Elf_Scn *symtab = NULL;
 655       GElf_Shdr symtab_header;
 656
 657       // Find the symbol table
 658       try {
 659          while ((section = elf_nextscn(elf, section))) {
 660             const char *name;
 661             if (gelf_getshdr(section, &symtab_header) != &symtab_header) {
 662                r_log = "Failed to read ELF section header.";
 663                throw compile_error();
 664             }
 665             name = elf_strptr(elf, section_str_index, symtab_header.sh_name);
 666            if (!strcmp(name, ".symtab")) {
 667                symtab = section;
 668                break;
 669            }
 670          }
 671          if (!symtab) {
 672             r_log = "Unable to find symbol table.";
 673             throw compile_error();
 674          }
 675       } catch (compile_error &e) {
 676          elf_end(elf);
 677          throw e;
 678       }
 679
 680
 681       // Extract symbol information from the table
 682       Elf_Data *symtab_data = NULL;
 683       GElf_Sym *symbol;
 684       GElf_Sym s;
 685
 686       std::map<std::string, unsigned> kernel_offsets;
 687       symtab_data = elf_getdata(symtab, symtab_data);
 688
 689       // Determine the offsets for each kernel
 690       for (int i = 0; (symbol = gelf_getsym(symtab_data, i, &s)); i++) {
 691          char *name = elf_strptr(elf, symtab_header.sh_link, symbol->st_name);
 692          for (std::vector<llvm::Function*>::const_iterator it = kernels.begin(),
 693               e = kernels.end(); it != e; ++it) {
 694             llvm::Function *f = *it;
 695             if (f->getName() == std::string(name))
 696                kernel_offsets[f->getName()] = symbol->st_value;
 697          }
 698       }
 699       elf_end(elf);
 700       return kernel_offsets;
 701    }
 702
 703    module
 704    build_module_native(std::vector<char> &code,
 705                        llvm::Module *mod,
 706                        const clang::LangAS::Map &address_spaces,
 707                        std::string &r_log) {
 708
 709       const std::vector<llvm::Function *> kernels = find_kernels(mod);
 710
 711       std::map<std::string, unsigned> kernel_offsets =
 712             get_kernel_offsets(code, kernels, r_log);
 713
 714       // Begin building the clover module
 715       module m;
 716       struct pipe_llvm_program_header header;
 717
 718       // Store the generated ELF binary in the module's text section.
 719       header.num_bytes = code.size();
 720       std::vector<char> data;
 721       data.insert(data.end(), (char*)(&header),
 722                               (char*)(&header) + sizeof(header));
 723       data.insert(data.end(), code.begin(), code.end());
 724       m.secs.push_back(module::section(0, module::section::text,
 725                                        header.num_bytes, data));
 726
 727       for (std::map<std::string, unsigned>::iterator i = kernel_offsets.begin(),
 728            e = kernel_offsets.end(); i != e; ++i) {
 729          std::vector<module::argument> args =
 730                get_kernel_args(mod, i->first, address_spaces);
 731          m.syms.push_back(module::symbol(i->first, 0, i->second, args ));
 732       }
 733
 734       return m;
 735    }
 736
 737    void
 738    diagnostic_handler(const llvm::DiagnosticInfo &di, void *data) {
 739       if (di.getSeverity() == llvm::DS_Error) {
 740          std::string message = *(std::string*)data;
 741          llvm::raw_string_ostream stream(message);
 742          llvm::DiagnosticPrinterRawOStream dp(stream);
 743          di.print(dp);
 744          stream.flush();
 745          *(std::string*)data = message;
 746
 747          throw compile_error();
 748       }
 749    }
 750
 751    void
 752    init_targets() {
 753       static bool targets_initialized = false;
 754       if (!targets_initialized) {
 755          LLVMInitializeAllTargets();
 756          LLVMInitializeAllTargetInfos();
 757          LLVMInitializeAllTargetMCs();
 758          LLVMInitializeAllAsmPrinters();
 759          targets_initialized = true;
 760       }
 761    }
 762
 763 #define DBG_CLC  (1 << 0)
 764 #define DBG_LLVM (1 << 1)
 765 #define DBG_ASM  (1 << 2)
 766
 767    unsigned
 768    get_debug_flags() {
 769       static const struct debug_named_value debug_options[] = {
 770          {"clc", DBG_CLC, "Dump the OpenCL C code for all kernels."},
 771          {"llvm", DBG_LLVM, "Dump the generated LLVM IR for all kernels."},
 772          {"asm", DBG_ASM, "Dump kernel assembly code for targets specifying "
 773           "PIPE_SHADER_IR_NATIVE"},
 774          DEBUG_NAMED_VALUE_END // must be last
 775       };
 776       static const unsigned debug_flags =
 777          debug_get_flags_option("CLOVER_DEBUG", debug_options, 0);
 778
 779       return debug_flags;
 780    }
 781
 782 } // End anonymous namespace
 783
 784 module
 785 clover::compile_program_llvm(const std::string &source,
 786                              const header_map &headers,
 787                              enum pipe_shader_ir ir,
 788                              const std::string &target,
 789                              const std::string &opts,
 790                              std::string &r_log) {
 791
 792    init_targets();
 793
 794    clang::LangAS::Map address_spaces;
 795    ::llvm::LLVMContext llvm_ctx;
 796    unsigned optimization_level;
 797
 798    llvm_ctx.setDiagnosticHandler(diagnostic_handler, &r_log);
 799
 800    if (get_debug_flags() & DBG_CLC)
 801       debug_log("// Build options: " + opts + '\n' + source, ".cl");
 802
 803    // The input file name must have the .cl extension in order for the
 804    // CompilerInvocation class to recognize it as an OpenCL source file.
 805    Module *mod = compile_llvm(llvm_ctx, source, headers, "input.cl",
 806                               target, opts, address_spaces,
 807                               optimization_level, r_log);
 808
 809    optimize(mod, optimization_level);
 810
 811    if (get_debug_flags() & DBG_LLVM) {
 812       std::string log;
 813       raw_string_ostream s_log(log);
 814       mod->print(s_log, NULL);
 815       s_log.flush();
 816       debug_log(log, ".ll");
 817     }
 818
 819    module m;
 820    // Build the clover::module
 821    switch (ir) {
 822       case PIPE_SHADER_IR_NIR:
 823       case PIPE_SHADER_IR_TGSI:
 824          //XXX: Handle TGSI, NIR
 825          assert(0);
 826          m = module();
 827          break;
 828       case PIPE_SHADER_IR_LLVM:
 829          m = build_module_llvm(mod, address_spaces);
 830          break;
 831       case PIPE_SHADER_IR_NATIVE: {
 832          std::vector<char> code = compile_native(mod, target,
 833                                                  get_debug_flags() & DBG_ASM,
 834                                                  r_log);
 835          m = build_module_native(code, mod, address_spaces, r_log);
 836          break;
 837       }
 838    }
 839    // The user takes ownership of the module.
 840    delete mod;
 841    return m;
 842 }