src/gallium/state_trackers/clover/llvm/invocation.cpp

   1 //
   2 // Copyright 2012-2016 Francisco Jerez
   3 // Copyright 2012-2016 Advanced Micro Devices, Inc.
   4 // Copyright 2014-2016 Jan Vesely
   5 // Copyright 2014-2015 Serge Martin
   6 // Copyright 2015 Zoltan Gilian
   7 //
   8 // Permission is hereby granted, free of charge, to any person obtaining a
   9 // copy of this software and associated documentation files (the "Software"),
  10 // to deal in the Software without restriction, including without limitation
  11 // the rights to use, copy, modify, merge, publish, distribute, sublicense,
  12 // and/or sell copies of the Software, and to permit persons to whom the
  13 // Software is furnished to do so, subject to the following conditions:
  14 //
  15 // The above copyright notice and this permission notice shall be included in
  16 // all copies or substantial portions of the Software.
  17 //
  18 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  21 // THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  22 // OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  23 // ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  24 // OTHER DEALINGS IN THE SOFTWARE.
  25 //
  26
  27 #include "llvm/compat.hpp"
  28 #include "core/compiler.hpp"
  29
  30 #include <clang/Frontend/CompilerInstance.h>
  31 #include <clang/Frontend/TextDiagnosticBuffer.h>
  32 #include <clang/Frontend/TextDiagnosticPrinter.h>
  33 #include <clang/CodeGen/CodeGenAction.h>
  34 #include <clang/Basic/TargetInfo.h>
  35 #include <llvm/Bitcode/BitstreamWriter.h>
  36 #include <llvm/Bitcode/ReaderWriter.h>
  37 #include <llvm/Linker/Linker.h>
  38 #include <llvm/IR/DiagnosticInfo.h>
  39 #include <llvm/IR/DiagnosticPrinter.h>
  40 #include <llvm/IR/DerivedTypes.h>
  41 #include <llvm/IR/LLVMContext.h>
  42 #include <llvm/IR/Module.h>
  43 #include <llvm/Support/SourceMgr.h>
  44 #include <llvm/IRReader/IRReader.h>
  45 #include <llvm/Support/CodeGen.h>
  46 #include <llvm/Support/TargetSelect.h>
  47 #include <llvm/Support/MemoryBuffer.h>
  48 #include <llvm/Support/FormattedStream.h>
  49 #include <llvm/Support/TargetRegistry.h>
  50 #include <llvm/Transforms/IPO.h>
  51 #include <llvm/Transforms/IPO/PassManagerBuilder.h>
  52 #include <llvm/Transforms/Utils/Cloning.h>
  53
  54
  55 #include <llvm/IR/DataLayout.h>
  56 #include <llvm/Target/TargetMachine.h>
  57 #include <llvm/Target/TargetOptions.h>
  58
  59 #include <llvm-c/Target.h>
  60 #include <llvm-c/TargetMachine.h>
  61 #include <llvm-c/Core.h>
  62
  63 #include "pipe/p_state.h"
  64 #include "util/u_memory.h"
  65 #include "util/u_math.h"
  66
  67 #include <iostream>
  68 #include <iomanip>
  69 #include <fstream>
  70 #include <cstdio>
  71 #include <sstream>
  72 #include <libelf.h>
  73 #include <gelf.h>
  74
  75 using namespace clover;
  76 using namespace clover::llvm;
  77
  78 using ::llvm::Function;
  79 using ::llvm::Module;
  80 using ::llvm::raw_string_ostream;
  81
  82 namespace {
  83    // XXX - Temporary hack to avoid breaking the build for the moment, will
  84    //       get rid of this later.
  85    namespace llvm {
  86       using namespace ::llvm;
  87    }
  88
  89    void debug_log(const std::string &msg, const std::string &suffix) {
  90       const char *dbg_file = debug_get_option("CLOVER_DEBUG_FILE", "stderr");
  91       if (!strcmp("stderr", dbg_file)) {
  92          std::cerr << msg;
  93        } else {
  94         std::ofstream file(dbg_file + suffix, std::ios::app);
  95         file << msg;
  96        }
  97    }
  98
  99    struct target {
 100       target(const std::string &s) :
 101          cpu(s.begin(), s.begin() + s.find_first_of("-")),
 102          triple(s.begin() + s.find_first_of("-") + 1, s.end()) {}
 103
 104       std::string cpu;
 105       std::string triple;
 106    };
 107
 108    llvm::Module *
 109    compile_llvm(llvm::LLVMContext &llvm_ctx, const std::string &source,
 110                 const header_map &headers, const std::string &name,
 111                 const std::string &target, const std::string &opts,
 112                 clang::LangAS::Map &address_spaces, unsigned &optimization_level,
 113                 std::string &r_log) {
 114
 115       clang::CompilerInstance c;
 116       clang::EmitLLVMOnlyAction act(&llvm_ctx);
 117       std::string log;
 118       llvm::raw_string_ostream s_log(log);
 119
 120       // Parse the compiler options:
 121       std::vector<std::string> opts_array;
 122       std::istringstream ss(opts);
 123
 124       while (!ss.eof()) {
 125          std::string opt;
 126          getline(ss, opt, ' ');
 127          opts_array.push_back(opt);
 128       }
 129
 130       opts_array.push_back(name);
 131
 132       std::vector<const char *> opts_carray;
 133       for (unsigned i = 0; i < opts_array.size(); i++) {
 134          opts_carray.push_back(opts_array.at(i).c_str());
 135       }
 136
 137       llvm::IntrusiveRefCntPtr<clang::DiagnosticIDs> DiagID;
 138       llvm::IntrusiveRefCntPtr<clang::DiagnosticOptions> DiagOpts;
 139       clang::TextDiagnosticBuffer *DiagsBuffer;
 140
 141       DiagID = new clang::DiagnosticIDs();
 142       DiagOpts = new clang::DiagnosticOptions();
 143       DiagsBuffer = new clang::TextDiagnosticBuffer();
 144
 145       clang::DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagsBuffer);
 146       bool Success;
 147
 148       Success = clang::CompilerInvocation::CreateFromArgs(c.getInvocation(),
 149                                         opts_carray.data(),
 150                                         opts_carray.data() + opts_carray.size(),
 151                                         Diags);
 152       if (!Success) {
 153          throw error(CL_INVALID_COMPILER_OPTIONS);
 154       }
 155       c.getFrontendOpts().ProgramAction = clang::frontend::EmitLLVMOnly;
 156       c.getHeaderSearchOpts().UseBuiltinIncludes = true;
 157       c.getHeaderSearchOpts().UseStandardSystemIncludes = true;
 158       c.getHeaderSearchOpts().ResourceDir = CLANG_RESOURCE_DIR;
 159
 160       // Add libclc generic search path
 161       c.getHeaderSearchOpts().AddPath(LIBCLC_INCLUDEDIR,
 162                                       clang::frontend::Angled,
 163                                       false, false
 164                                       );
 165
 166       // Add libclc include
 167       c.getPreprocessorOpts().Includes.push_back("clc/clc.h");
 168
 169       // clc.h requires that this macro be defined:
 170       c.getPreprocessorOpts().addMacroDef("cl_clang_storage_class_specifiers");
 171
 172       c.getLangOpts().NoBuiltin = true;
 173       const struct target t { target };
 174       c.getTargetOpts().Triple = t.triple;
 175       c.getTargetOpts().CPU = t.cpu;
 176
 177       // This is a workaround for a Clang bug which causes the number
 178       // of warnings and errors to be printed to stderr.
 179       // http://www.llvm.org/bugs/show_bug.cgi?id=19735
 180       c.getDiagnosticOpts().ShowCarets = false;
 181
 182       compat::set_lang_defaults(c.getInvocation(), c.getLangOpts(),
 183                                 clang::IK_OpenCL, ::llvm::Triple(t.triple),
 184                                 c.getPreprocessorOpts(),
 185                                 clang::LangStandard::lang_opencl11);
 186
 187       c.createDiagnostics(
 188                           new clang::TextDiagnosticPrinter(
 189                                  s_log,
 190                                  &c.getDiagnosticOpts()));
 191
 192       c.getPreprocessorOpts().addRemappedFile(name,
 193                                               llvm::MemoryBuffer::getMemBuffer(source).release());
 194
 195       if (headers.size()) {
 196          const std::string tmp_header_path = "/tmp/clover/";
 197
 198          c.getHeaderSearchOpts().AddPath(tmp_header_path,
 199                                          clang::frontend::Angled,
 200                                          false, false
 201                                          );
 202
 203          for (header_map::const_iterator it = headers.begin();
 204               it != headers.end(); ++it) {
 205             const std::string path = tmp_header_path + std::string(it->first);
 206             c.getPreprocessorOpts().addRemappedFile(path,
 207                     llvm::MemoryBuffer::getMemBuffer(it->second.c_str()).release());
 208          }
 209       }
 210
 211       // Tell clang to link this file before performing any
 212       // optimizations.  This is required so that we can replace calls
 213       // to the OpenCL C barrier() builtin with calls to target
 214       // intrinsics that have the noduplicate attribute.  This
 215       // attribute will prevent Clang from creating illegal uses of
 216       // barrier() (e.g. Moving barrier() inside a conditional that is
 217       // no executed by all threads) during its optimizaton passes.
 218       compat::add_link_bitcode_file(c.getCodeGenOpts(),
 219                                     LIBCLC_LIBEXECDIR + target + ".bc");
 220
 221       optimization_level = c.getCodeGenOpts().OptimizationLevel;
 222
 223       // Compile the code
 224       bool ExecSuccess = c.ExecuteAction(act);
 225       r_log = log;
 226
 227       if (!ExecSuccess)
 228          throw compile_error();
 229
 230       // Get address spaces map to be able to find kernel argument address space
 231       memcpy(address_spaces, c.getTarget().getAddressSpaceMap(),
 232                                                         sizeof(address_spaces));
 233
 234       return act.takeModule().release();
 235    }
 236
 237    std::vector<llvm::Function *>
 238    find_kernels(llvm::Module *mod) {
 239       std::vector<llvm::Function *> kernels;
 240 #if HAVE_LLVM >= 0x0309
 241       auto &list = mod->getFunctionList();
 242       for_each(list.begin(), list.end(), [&](llvm::Function &f){
 243          if (f.getMetadata("kernel_arg_type"))
 244            kernels.push_back(&f);
 245       });
 246       return kernels;
 247 #endif
 248       const llvm::NamedMDNode *kernel_node =
 249                                  mod->getNamedMetadata("opencl.kernels");
 250       // This means there are no kernels in the program.  The spec does not
 251       // require that we return an error here, but there will be an error if
 252       // the user tries to pass this program to a clCreateKernel() call.
 253       if (!kernel_node) {
 254          return std::vector<llvm::Function *>();
 255       }
 256
 257       kernels.reserve(kernel_node->getNumOperands());
 258       for (unsigned i = 0; i < kernel_node->getNumOperands(); ++i) {
 259          kernels.push_back(llvm::mdconst::dyn_extract<llvm::Function>(
 260                                     kernel_node->getOperand(i)->getOperand(0)));
 261       }
 262       return kernels;
 263    }
 264
 265    void
 266    optimize(llvm::Module *mod, unsigned optimization_level) {
 267       compat::pass_manager PM;
 268
 269       compat::add_data_layout_pass(PM);
 270
 271       // By default, the function internalizer pass will look for a function
 272       // called "main" and then mark all other functions as internal.  Marking
 273       // functions as internal enables the optimizer to perform optimizations
 274       // like function inlining and global dead-code elimination.
 275       //
 276       // When there is no "main" function in a module, the internalize pass will
 277       // treat the module like a library, and it won't internalize any functions.
 278       // Since there is no "main" function in our kernels, we need to tell
 279       // the internalizer pass that this module is not a library by passing a
 280       // list of kernel functions to the internalizer.  The internalizer will
 281       // treat the functions in the list as "main" functions and internalize
 282       // all of the other functions.
 283       compat::add_internalize_pass(PM, map(std::mem_fn(&Function::getName),
 284                                            find_kernels(mod)));
 285
 286       llvm::PassManagerBuilder PMB;
 287       PMB.OptLevel = optimization_level;
 288       PMB.LibraryInfo = new compat::target_library_info(
 289          llvm::Triple(mod->getTargetTriple()));
 290       PMB.populateModulePassManager(PM);
 291       PM.run(*mod);
 292    }
 293
 294    // Kernel metadata
 295
 296    struct kernel_arg_md {
 297       llvm::StringRef type_name;
 298       llvm::StringRef access_qual;
 299       kernel_arg_md(llvm::StringRef type_name_, llvm::StringRef access_qual_):
 300          type_name(type_name_), access_qual(access_qual_) {}
 301    };
 302 #if HAVE_LLVM >= 0x0309
 303    std::vector<kernel_arg_md>
 304    get_kernel_arg_md(const llvm::Function *kernel_func) {
 305
 306       size_t num_args = kernel_func->getArgumentList().size();
 307
 308       auto aq = kernel_func->getMetadata("kernel_arg_access_qual");
 309       auto ty = kernel_func->getMetadata("kernel_arg_type");
 310
 311       std::vector<kernel_arg_md> res;
 312       res.reserve(num_args);
 313       for (size_t i = 0; i < num_args; ++i) {
 314          res.push_back(kernel_arg_md(
 315             llvm::cast<llvm::MDString>(ty->getOperand(i))->getString(),
 316             llvm::cast<llvm::MDString>(aq->getOperand(i))->getString()));
 317       }
 318
 319       return res;
 320    }
 321
 322 #else
 323
 324    const llvm::MDNode *
 325    get_kernel_metadata(const llvm::Function *kernel_func) {
 326       auto mod = kernel_func->getParent();
 327       auto kernels_node = mod->getNamedMetadata("opencl.kernels");
 328       if (!kernels_node) {
 329          return nullptr;
 330       }
 331
 332       const llvm::MDNode *kernel_node = nullptr;
 333       for (unsigned i = 0; i < kernels_node->getNumOperands(); ++i) {
 334          auto func = llvm::mdconst::dyn_extract<llvm::Function>(
 335                kernels_node->getOperand(i)->getOperand(0));
 336          if (func == kernel_func) {
 337             kernel_node = kernels_node->getOperand(i);
 338             break;
 339          }
 340       }
 341
 342       return kernel_node;
 343    }
 344
 345    llvm::MDNode*
 346    node_from_op_checked(const llvm::MDOperand &md_operand,
 347                         llvm::StringRef expect_name,
 348                         unsigned expect_num_args)
 349    {
 350       auto node = llvm::cast<llvm::MDNode>(md_operand);
 351       assert(node->getNumOperands() == expect_num_args &&
 352              "Wrong number of operands.");
 353
 354       auto str_node = llvm::cast<llvm::MDString>(node->getOperand(0));
 355       assert(str_node->getString() == expect_name &&
 356              "Wrong metadata node name.");
 357
 358       return node;
 359    }
 360
 361    std::vector<kernel_arg_md>
 362    get_kernel_arg_md(const llvm::Function *kernel_func) {
 363       auto num_args = kernel_func->getArgumentList().size();
 364
 365       auto kernel_node = get_kernel_metadata(kernel_func);
 366       auto aq = node_from_op_checked(kernel_node->getOperand(2),
 367                                      "kernel_arg_access_qual", num_args + 1);
 368       auto ty = node_from_op_checked(kernel_node->getOperand(3),
 369                                      "kernel_arg_type", num_args + 1);
 370
 371       std::vector<kernel_arg_md> res;
 372       res.reserve(num_args);
 373       for (unsigned i = 0; i < num_args; ++i) {
 374          res.push_back(kernel_arg_md(
 375             llvm::cast<llvm::MDString>(ty->getOperand(i+1))->getString(),
 376             llvm::cast<llvm::MDString>(aq->getOperand(i+1))->getString()));
 377       }
 378
 379       return res;
 380    }
 381 #endif
 382
 383    std::vector<module::argument>
 384    get_kernel_args(const llvm::Module *mod, const std::string &kernel_name,
 385                    const clang::LangAS::Map &address_spaces) {
 386
 387       std::vector<module::argument> args;
 388       llvm::Function *kernel_func = mod->getFunction(kernel_name);
 389       assert(kernel_func && "Kernel name not found in module.");
 390       auto arg_md = get_kernel_arg_md(kernel_func);
 391
 392       llvm::DataLayout TD(mod);
 393       llvm::Type *size_type =
 394          TD.getSmallestLegalIntType(mod->getContext(), sizeof(cl_uint) * 8);
 395
 396       for (const auto &arg: kernel_func->args()) {
 397
 398          llvm::Type *arg_type = arg.getType();
 399          const unsigned arg_store_size = TD.getTypeStoreSize(arg_type);
 400
 401          // OpenCL 1.2 specification, Ch. 6.1.5: "A built-in data
 402          // type that is not a power of two bytes in size must be
 403          // aligned to the next larger power of two".  We need this
 404          // alignment for three element vectors, which have
 405          // non-power-of-2 store size.
 406          const unsigned arg_api_size = util_next_power_of_two(arg_store_size);
 407
 408          llvm::Type *target_type = arg_type->isIntegerTy() ?
 409                TD.getSmallestLegalIntType(mod->getContext(), arg_store_size * 8)
 410                : arg_type;
 411          unsigned target_size = TD.getTypeStoreSize(target_type);
 412          unsigned target_align = TD.getABITypeAlignment(target_type);
 413
 414          llvm::StringRef type_name = arg_md[arg.getArgNo()].type_name;
 415          llvm::StringRef access_qual = arg_md[arg.getArgNo()].access_qual;
 416
 417          // Image
 418          const bool is_image2d = type_name == "image2d_t";
 419          const bool is_image3d = type_name == "image3d_t";
 420          if (is_image2d || is_image3d) {
 421             const bool is_write_only = access_qual == "write_only";
 422             const bool is_read_only = access_qual == "read_only";
 423
 424             enum module::argument::type marg_type;
 425             if (is_image2d && is_read_only) {
 426                marg_type = module::argument::image2d_rd;
 427             } else if (is_image2d && is_write_only) {
 428                marg_type = module::argument::image2d_wr;
 429             } else if (is_image3d && is_read_only) {
 430                marg_type = module::argument::image3d_rd;
 431             } else if (is_image3d && is_write_only) {
 432                marg_type = module::argument::image3d_wr;
 433             } else {
 434                assert(0 && "Wrong image access qualifier");
 435             }
 436
 437             args.push_back(module::argument(marg_type,
 438                                             arg_store_size, target_size,
 439                                             target_align,
 440                                             module::argument::zero_ext));
 441             continue;
 442          }
 443
 444          // Image size implicit argument
 445          if (type_name == "__llvm_image_size") {
 446             args.push_back(module::argument(module::argument::scalar,
 447                                             sizeof(cl_uint),
 448                                             TD.getTypeStoreSize(size_type),
 449                                             TD.getABITypeAlignment(size_type),
 450                                             module::argument::zero_ext,
 451                                             module::argument::image_size));
 452             continue;
 453          }
 454
 455          // Image format implicit argument
 456          if (type_name == "__llvm_image_format") {
 457             args.push_back(module::argument(module::argument::scalar,
 458                                             sizeof(cl_uint),
 459                                             TD.getTypeStoreSize(size_type),
 460                                             TD.getABITypeAlignment(size_type),
 461                                             module::argument::zero_ext,
 462                                             module::argument::image_format));
 463             continue;
 464          }
 465
 466          // Other types
 467          if (llvm::isa<llvm::PointerType>(arg_type) && arg.hasByValAttr()) {
 468             arg_type =
 469                   llvm::dyn_cast<llvm::PointerType>(arg_type)->getElementType();
 470          }
 471
 472          if (arg_type->isPointerTy()) {
 473             unsigned address_space = llvm::cast<llvm::PointerType>(arg_type)->getAddressSpace();
 474             if (address_space == address_spaces[clang::LangAS::opencl_local
 475                                                      - clang::LangAS::Offset]) {
 476                args.push_back(module::argument(module::argument::local,
 477                                                arg_api_size, target_size,
 478                                                target_align,
 479                                                module::argument::zero_ext));
 480             } else {
 481                // XXX: Correctly handle constant address space.  There is no
 482                // way for r600g to pass a handle for constant buffers back
 483                // to clover like it can for global buffers, so
 484                // creating constant arguments will break r600g.  For now,
 485                // continue treating constant buffers as global buffers
 486                // until we can come up with a way to create handles for
 487                // constant buffers.
 488                args.push_back(module::argument(module::argument::global,
 489                                                arg_api_size, target_size,
 490                                                target_align,
 491                                                module::argument::zero_ext));
 492            }
 493
 494          } else {
 495             llvm::AttributeSet attrs = kernel_func->getAttributes();
 496             enum module::argument::ext_type ext_type =
 497                   (attrs.hasAttribute(arg.getArgNo() + 1,
 498                                      llvm::Attribute::SExt) ?
 499                    module::argument::sign_ext :
 500                    module::argument::zero_ext);
 501
 502             args.push_back(
 503                module::argument(module::argument::scalar, arg_api_size,
 504                                 target_size, target_align, ext_type));
 505          }
 506       }
 507
 508       // Append implicit arguments.  XXX - The types, ordering and
 509       // vector size of the implicit arguments should depend on the
 510       // target according to the selected calling convention.
 511       args.push_back(
 512          module::argument(module::argument::scalar, sizeof(cl_uint),
 513                           TD.getTypeStoreSize(size_type),
 514                           TD.getABITypeAlignment(size_type),
 515                           module::argument::zero_ext,
 516                           module::argument::grid_dimension));
 517
 518       args.push_back(
 519          module::argument(module::argument::scalar, sizeof(cl_uint),
 520                           TD.getTypeStoreSize(size_type),
 521                           TD.getABITypeAlignment(size_type),
 522                           module::argument::zero_ext,
 523                           module::argument::grid_offset));
 524
 525       return args;
 526    }
 527
 528    module
 529    build_module_llvm(llvm::Module *mod,
 530                      clang::LangAS::Map& address_spaces) {
 531
 532       module m;
 533       struct pipe_llvm_program_header header;
 534
 535       llvm::SmallVector<char, 1024> llvm_bitcode;
 536       llvm::raw_svector_ostream bitcode_ostream(llvm_bitcode);
 537       llvm::BitstreamWriter writer(llvm_bitcode);
 538       llvm::WriteBitcodeToFile(mod, bitcode_ostream);
 539 #if HAVE_LLVM < 0x0308
 540       bitcode_ostream.flush();
 541 #endif
 542
 543       const std::vector<llvm::Function *> kernels = find_kernels(mod);
 544       for (unsigned i = 0; i < kernels.size(); ++i) {
 545          std::string kernel_name = kernels[i]->getName();
 546          std::vector<module::argument> args =
 547                get_kernel_args(mod, kernel_name, address_spaces);
 548
 549          m.syms.push_back(module::symbol(kernel_name, 0, i, args ));
 550       }
 551
 552       header.num_bytes = llvm_bitcode.size();
 553       std::vector<char> data;
 554       data.insert(data.end(), (char*)(&header),
 555                               (char*)(&header) + sizeof(header));
 556       data.insert(data.end(), llvm_bitcode.begin(),
 557                                   llvm_bitcode.end());
 558       m.secs.push_back(module::section(0, module::section::text,
 559                                        header.num_bytes, data));
 560
 561       return m;
 562    }
 563
 564    void
 565    emit_code(LLVMTargetMachineRef tm, LLVMModuleRef mod,
 566              LLVMCodeGenFileType file_type,
 567              LLVMMemoryBufferRef *out_buffer,
 568              std::string &r_log) {
 569       LLVMBool err;
 570       char *err_message = NULL;
 571
 572       err = LLVMTargetMachineEmitToMemoryBuffer(tm, mod, file_type,
 573                                                 &err_message, out_buffer);
 574
 575       if (err) {
 576          r_log = std::string(err_message);
 577       }
 578
 579       LLVMDisposeMessage(err_message);
 580
 581       if (err) {
 582          throw compile_error();
 583       }
 584    }
 585
 586    std::vector<char>
 587    compile_native(const llvm::Module *mod, const target &t, bool dump_asm,
 588                   std::string &r_log) {
 589
 590       std::string log;
 591       LLVMTargetRef target;
 592       char *error_message;
 593       LLVMMemoryBufferRef out_buffer;
 594       unsigned buffer_size;
 595       const char *buffer_data;
 596       LLVMModuleRef mod_ref = wrap(mod);
 597
 598       if (LLVMGetTargetFromTriple(t.triple.c_str(), &target, &error_message)) {
 599          r_log = std::string(error_message);
 600          LLVMDisposeMessage(error_message);
 601          throw compile_error();
 602       }
 603
 604       LLVMTargetMachineRef tm = LLVMCreateTargetMachine(
 605             target, t.triple.c_str(), t.cpu.c_str(), "",
 606             LLVMCodeGenLevelDefault, LLVMRelocDefault, LLVMCodeModelDefault);
 607
 608       if (!tm) {
 609          r_log = "Could not create TargetMachine: " + t.triple;
 610          throw compile_error();
 611       }
 612
 613       if (dump_asm) {
 614          LLVMSetTargetMachineAsmVerbosity(tm, true);
 615 #if HAVE_LLVM >= 0x0308
 616          LLVMModuleRef debug_mod = wrap(llvm::CloneModule(mod).release());
 617 #else
 618          LLVMModuleRef debug_mod = wrap(llvm::CloneModule(mod));
 619 #endif
 620          emit_code(tm, debug_mod, LLVMAssemblyFile, &out_buffer, r_log);
 621          buffer_size = LLVMGetBufferSize(out_buffer);
 622          buffer_data = LLVMGetBufferStart(out_buffer);
 623          debug_log(std::string(buffer_data, buffer_size), ".asm");
 624
 625          LLVMSetTargetMachineAsmVerbosity(tm, false);
 626          LLVMDisposeMemoryBuffer(out_buffer);
 627          LLVMDisposeModule(debug_mod);
 628       }
 629
 630       emit_code(tm, mod_ref, LLVMObjectFile, &out_buffer, r_log);
 631
 632       buffer_size = LLVMGetBufferSize(out_buffer);
 633       buffer_data = LLVMGetBufferStart(out_buffer);
 634
 635       std::vector<char> code(buffer_data, buffer_data + buffer_size);
 636
 637       LLVMDisposeMemoryBuffer(out_buffer);
 638       LLVMDisposeTargetMachine(tm);
 639
 640       return code;
 641    }
 642
 643    std::map<std::string, unsigned>
 644    get_kernel_offsets(std::vector<char> &code,
 645                       const std::vector<llvm::Function *> &kernels,
 646                       std::string &r_log) {
 647
 648       // One of the libelf implementations
 649       // (http://www.mr511.de/software/english.htm) requires calling
 650       // elf_version() before elf_memory().
 651       //
 652       elf_version(EV_CURRENT);
 653
 654       Elf *elf = elf_memory(&code[0], code.size());
 655       size_t section_str_index;
 656       elf_getshdrstrndx(elf, &section_str_index);
 657       Elf_Scn *section = NULL;
 658       Elf_Scn *symtab = NULL;
 659       GElf_Shdr symtab_header;
 660
 661       // Find the symbol table
 662       try {
 663          while ((section = elf_nextscn(elf, section))) {
 664             const char *name;
 665             if (gelf_getshdr(section, &symtab_header) != &symtab_header) {
 666                r_log = "Failed to read ELF section header.";
 667                throw compile_error();
 668             }
 669             name = elf_strptr(elf, section_str_index, symtab_header.sh_name);
 670            if (!strcmp(name, ".symtab")) {
 671                symtab = section;
 672                break;
 673            }
 674          }
 675          if (!symtab) {
 676             r_log = "Unable to find symbol table.";
 677             throw compile_error();
 678          }
 679       } catch (compile_error &e) {
 680          elf_end(elf);
 681          throw e;
 682       }
 683
 684
 685       // Extract symbol information from the table
 686       Elf_Data *symtab_data = NULL;
 687       GElf_Sym *symbol;
 688       GElf_Sym s;
 689
 690       std::map<std::string, unsigned> kernel_offsets;
 691       symtab_data = elf_getdata(symtab, symtab_data);
 692
 693       // Determine the offsets for each kernel
 694       for (int i = 0; (symbol = gelf_getsym(symtab_data, i, &s)); i++) {
 695          char *name = elf_strptr(elf, symtab_header.sh_link, symbol->st_name);
 696          for (std::vector<llvm::Function*>::const_iterator it = kernels.begin(),
 697               e = kernels.end(); it != e; ++it) {
 698             llvm::Function *f = *it;
 699             if (f->getName() == std::string(name))
 700                kernel_offsets[f->getName()] = symbol->st_value;
 701          }
 702       }
 703       elf_end(elf);
 704       return kernel_offsets;
 705    }
 706
 707    module
 708    build_module_native(std::vector<char> &code,
 709                        llvm::Module *mod,
 710                        const clang::LangAS::Map &address_spaces,
 711                        std::string &r_log) {
 712
 713       const std::vector<llvm::Function *> kernels = find_kernels(mod);
 714
 715       std::map<std::string, unsigned> kernel_offsets =
 716             get_kernel_offsets(code, kernels, r_log);
 717
 718       // Begin building the clover module
 719       module m;
 720       struct pipe_llvm_program_header header;
 721
 722       // Store the generated ELF binary in the module's text section.
 723       header.num_bytes = code.size();
 724       std::vector<char> data;
 725       data.insert(data.end(), (char*)(&header),
 726                               (char*)(&header) + sizeof(header));
 727       data.insert(data.end(), code.begin(), code.end());
 728       m.secs.push_back(module::section(0, module::section::text,
 729                                        header.num_bytes, data));
 730
 731       for (std::map<std::string, unsigned>::iterator i = kernel_offsets.begin(),
 732            e = kernel_offsets.end(); i != e; ++i) {
 733          std::vector<module::argument> args =
 734                get_kernel_args(mod, i->first, address_spaces);
 735          m.syms.push_back(module::symbol(i->first, 0, i->second, args ));
 736       }
 737
 738       return m;
 739    }
 740
 741    void
 742    diagnostic_handler(const llvm::DiagnosticInfo &di, void *data) {
 743       if (di.getSeverity() == llvm::DS_Error) {
 744          std::string message = *(std::string*)data;
 745          llvm::raw_string_ostream stream(message);
 746          llvm::DiagnosticPrinterRawOStream dp(stream);
 747          di.print(dp);
 748          stream.flush();
 749          *(std::string*)data = message;
 750
 751          throw compile_error();
 752       }
 753    }
 754
 755    void
 756    init_targets() {
 757       static bool targets_initialized = false;
 758       if (!targets_initialized) {
 759          LLVMInitializeAllTargets();
 760          LLVMInitializeAllTargetInfos();
 761          LLVMInitializeAllTargetMCs();
 762          LLVMInitializeAllAsmPrinters();
 763          targets_initialized = true;
 764       }
 765    }
 766
 767 #define DBG_CLC  (1 << 0)
 768 #define DBG_LLVM (1 << 1)
 769 #define DBG_ASM  (1 << 2)
 770
 771    unsigned
 772    get_debug_flags() {
 773       static const struct debug_named_value debug_options[] = {
 774          {"clc", DBG_CLC, "Dump the OpenCL C code for all kernels."},
 775          {"llvm", DBG_LLVM, "Dump the generated LLVM IR for all kernels."},
 776          {"asm", DBG_ASM, "Dump kernel assembly code for targets specifying "
 777           "PIPE_SHADER_IR_NATIVE"},
 778          DEBUG_NAMED_VALUE_END // must be last
 779       };
 780       static const unsigned debug_flags =
 781          debug_get_flags_option("CLOVER_DEBUG", debug_options, 0);
 782
 783       return debug_flags;
 784    }
 785
 786 } // End anonymous namespace
 787
 788 module
 789 clover::compile_program_llvm(const std::string &source,
 790                              const header_map &headers,
 791                              enum pipe_shader_ir ir,
 792                              const std::string &target,
 793                              const std::string &opts,
 794                              std::string &r_log) {
 795
 796    init_targets();
 797
 798    clang::LangAS::Map address_spaces;
 799    ::llvm::LLVMContext llvm_ctx;
 800    unsigned optimization_level;
 801
 802    llvm_ctx.setDiagnosticHandler(diagnostic_handler, &r_log);
 803
 804    if (get_debug_flags() & DBG_CLC)
 805       debug_log("// Build options: " + opts + '\n' + source, ".cl");
 806
 807    // The input file name must have the .cl extension in order for the
 808    // CompilerInvocation class to recognize it as an OpenCL source file.
 809    Module *mod = compile_llvm(llvm_ctx, source, headers, "input.cl",
 810                               target, opts, address_spaces,
 811                               optimization_level, r_log);
 812
 813    optimize(mod, optimization_level);
 814
 815    if (get_debug_flags() & DBG_LLVM) {
 816       std::string log;
 817       raw_string_ostream s_log(log);
 818       mod->print(s_log, NULL);
 819       s_log.flush();
 820       debug_log(log, ".ll");
 821     }
 822
 823    module m;
 824    // Build the clover::module
 825    switch (ir) {
 826       case PIPE_SHADER_IR_NIR:
 827       case PIPE_SHADER_IR_TGSI:
 828          //XXX: Handle TGSI, NIR
 829          assert(0);
 830          m = module();
 831          break;
 832       case PIPE_SHADER_IR_LLVM:
 833          m = build_module_llvm(mod, address_spaces);
 834          break;
 835       case PIPE_SHADER_IR_NATIVE: {
 836          std::vector<char> code = compile_native(mod, target,
 837                                                  get_debug_flags() & DBG_ASM,
 838                                                  r_log);
 839          m = build_module_native(code, mod, address_spaces, r_log);
 840          break;
 841       }
 842    }
 843    // The user takes ownership of the module.
 844    delete mod;
 845    return m;
 846 }