src/amd/llvm/ac_llvm_util.c

   1 /*
   2  * Copyright 2014 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the
   6  * "Software"), to deal in the Software without restriction, including
   7  * without limitation the rights to use, copy, modify, merge, publish,
   8  * distribute, sub license, and/or sell copies of the Software, and to
   9  * permit persons to whom the Software is furnished to do so, subject to
  10  * the following conditions:
  11  *
  12  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  13  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  14  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  15  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  16  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  17  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  18  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  19  *
  20  * The above copyright notice and this permission notice (including the
  21  * next paragraph) shall be included in all copies or substantial portions
  22  * of the Software.
  23  *
  24  */
  25 /* based on pieces from si_pipe.c and radeon_llvm_emit.c */
  26 #include "ac_llvm_util.h"
  27
  28 #include "ac_llvm_build.h"
  29 #include "c11/threads.h"
  30 #include "gallivm/lp_bld_misc.h"
  31 #include "util/bitscan.h"
  32 #include "util/u_math.h"
  33 #include <llvm-c/Core.h>
  34 #include <llvm-c/Support.h>
  35 #include <llvm-c/Transforms/IPO.h>
  36 #include <llvm-c/Transforms/Scalar.h>
  37 #include <llvm-c/Transforms/Utils.h>
  38
  39 #include <assert.h>
  40 #include <stdio.h>
  41 #include <string.h>
  42
  43 static void ac_init_llvm_target()
  44 {
  45    LLVMInitializeAMDGPUTargetInfo();
  46    LLVMInitializeAMDGPUTarget();
  47    LLVMInitializeAMDGPUTargetMC();
  48    LLVMInitializeAMDGPUAsmPrinter();
  49
  50    /* For inline assembly. */
  51    LLVMInitializeAMDGPUAsmParser();
  52
  53    /* For ACO disassembly. */
  54    LLVMInitializeAMDGPUDisassembler();
  55
  56    /* Workaround for bug in llvm 4.0 that causes image intrinsics
  57     * to disappear.
  58     * https://reviews.llvm.org/D26348
  59     *
  60     * "mesa" is the prefix for error messages.
  61     *
  62     * -global-isel-abort=2 is a no-op unless global isel has been enabled.
  63     * This option tells the backend to fall-back to SelectionDAG and print
  64     * a diagnostic message if global isel fails.
  65     */
  66    const char *argv[] = {
  67       "mesa",
  68       "-simplifycfg-sink-common=false",
  69       "-global-isel-abort=2",
  70 #if LLVM_VERSION_MAJOR >= 10
  71       /* Atomic optimizations require LLVM 10.0 for gfx10 support. */
  72       "-amdgpu-atomic-optimizations=true",
  73 #endif
  74 #if LLVM_VERSION_MAJOR >= 11
  75       /* This was disabled by default in: https://reviews.llvm.org/D77228 */
  76       "-structurizecfg-skip-uniform-regions",
  77 #endif
  78    };
  79    LLVMParseCommandLineOptions(ARRAY_SIZE(argv), argv, NULL);
  80 }
  81
  82 PUBLIC void ac_init_shared_llvm_once(void)
  83 {
  84    static once_flag ac_init_llvm_target_once_flag = ONCE_FLAG_INIT;
  85    call_once(&ac_init_llvm_target_once_flag, ac_init_llvm_target);
  86 }
  87
  88 #if !LLVM_IS_SHARED
  89 static once_flag ac_init_static_llvm_target_once_flag = ONCE_FLAG_INIT;
  90 static void ac_init_static_llvm_once(void)
  91 {
  92    call_once(&ac_init_static_llvm_target_once_flag, ac_init_llvm_target);
  93 }
  94 #endif
  95
  96 void ac_init_llvm_once(void)
  97 {
  98 #if LLVM_IS_SHARED
  99    ac_init_shared_llvm_once();
 100 #else
 101    ac_init_static_llvm_once();
 102 #endif
 103 }
 104
 105 static LLVMTargetRef ac_get_llvm_target(const char *triple)
 106 {
 107    LLVMTargetRef target = NULL;
 108    char *err_message = NULL;
 109
 110    if (LLVMGetTargetFromTriple(triple, &target, &err_message)) {
 111       fprintf(stderr, "Cannot find target for triple %s ", triple);
 112       if (err_message) {
 113          fprintf(stderr, "%s\n", err_message);
 114       }
 115       LLVMDisposeMessage(err_message);
 116       return NULL;
 117    }
 118    return target;
 119 }
 120
 121 const char *ac_get_llvm_processor_name(enum radeon_family family)
 122 {
 123    switch (family) {
 124    case CHIP_TAHITI:
 125       return "tahiti";
 126    case CHIP_PITCAIRN:
 127       return "pitcairn";
 128    case CHIP_VERDE:
 129       return "verde";
 130    case CHIP_OLAND:
 131       return "oland";
 132    case CHIP_HAINAN:
 133       return "hainan";
 134    case CHIP_BONAIRE:
 135       return "bonaire";
 136    case CHIP_KABINI:
 137       return "kabini";
 138    case CHIP_KAVERI:
 139       return "kaveri";
 140    case CHIP_HAWAII:
 141       return "hawaii";
 142    case CHIP_TONGA:
 143       return "tonga";
 144    case CHIP_ICELAND:
 145       return "iceland";
 146    case CHIP_CARRIZO:
 147       return "carrizo";
 148    case CHIP_FIJI:
 149       return "fiji";
 150    case CHIP_STONEY:
 151       return "stoney";
 152    case CHIP_POLARIS10:
 153       return "polaris10";
 154    case CHIP_POLARIS11:
 155    case CHIP_POLARIS12:
 156    case CHIP_VEGAM:
 157       return "polaris11";
 158    case CHIP_VEGA10:
 159       return "gfx900";
 160    case CHIP_RAVEN:
 161       return "gfx902";
 162    case CHIP_VEGA12:
 163       return "gfx904";
 164    case CHIP_VEGA20:
 165       return "gfx906";
 166    case CHIP_RAVEN2:
 167    case CHIP_RENOIR:
 168       return "gfx909";
 169    case CHIP_ARCTURUS:
 170       return "gfx908";
 171    case CHIP_NAVI10:
 172       return "gfx1010";
 173    case CHIP_NAVI12:
 174       return "gfx1011";
 175    case CHIP_NAVI14:
 176       return "gfx1012";
 177    case CHIP_SIENNA_CICHLID:
 178    case CHIP_NAVY_FLOUNDER:
 179       return "gfx1030";
 180    default:
 181       return "";
 182    }
 183 }
 184
 185 static LLVMTargetMachineRef ac_create_target_machine(enum radeon_family family,
 186                                                      enum ac_target_machine_options tm_options,
 187                                                      LLVMCodeGenOptLevel level,
 188                                                      const char **out_triple)
 189 {
 190    assert(family >= CHIP_TAHITI);
 191    char features[256];
 192    const char *triple = (tm_options & AC_TM_SUPPORTS_SPILL) ? "amdgcn-mesa-mesa3d" : "amdgcn--";
 193    LLVMTargetRef target = ac_get_llvm_target(triple);
 194
 195    snprintf(features, sizeof(features), "+DumpCode%s%s%s%s%s",
 196             LLVM_VERSION_MAJOR >= 11 ? "" : ",-fp32-denormals,+fp64-denormals",
 197             family >= CHIP_NAVI10 && !(tm_options & AC_TM_WAVE32)
 198                ? ",+wavefrontsize64,-wavefrontsize32"
 199                : "",
 200             family <= CHIP_NAVI14 && tm_options & AC_TM_FORCE_ENABLE_XNACK ? ",+xnack" : "",
 201             family <= CHIP_NAVI14 && tm_options & AC_TM_FORCE_DISABLE_XNACK ? ",-xnack" : "",
 202             tm_options & AC_TM_PROMOTE_ALLOCA_TO_SCRATCH ? ",-promote-alloca" : "");
 203
 204    LLVMTargetMachineRef tm =
 205       LLVMCreateTargetMachine(target, triple, ac_get_llvm_processor_name(family), features, level,
 206                               LLVMRelocDefault, LLVMCodeModelDefault);
 207
 208    if (out_triple)
 209       *out_triple = triple;
 210    if (tm_options & AC_TM_ENABLE_GLOBAL_ISEL)
 211       ac_enable_global_isel(tm);
 212    return tm;
 213 }
 214
 215 static LLVMPassManagerRef ac_create_passmgr(LLVMTargetLibraryInfoRef target_library_info,
 216                                             bool check_ir)
 217 {
 218    LLVMPassManagerRef passmgr = LLVMCreatePassManager();
 219    if (!passmgr)
 220       return NULL;
 221
 222    if (target_library_info)
 223       LLVMAddTargetLibraryInfo(target_library_info, passmgr);
 224
 225    if (check_ir)
 226       LLVMAddVerifierPass(passmgr);
 227    LLVMAddAlwaysInlinerPass(passmgr);
 228    /* Normally, the pass manager runs all passes on one function before
 229     * moving onto another. Adding a barrier no-op pass forces the pass
 230     * manager to run the inliner on all functions first, which makes sure
 231     * that the following passes are only run on the remaining non-inline
 232     * function, so it removes useless work done on dead inline functions.
 233     */
 234    ac_llvm_add_barrier_noop_pass(passmgr);
 235    /* This pass should eliminate all the load and store instructions. */
 236    LLVMAddPromoteMemoryToRegisterPass(passmgr);
 237    LLVMAddScalarReplAggregatesPass(passmgr);
 238    LLVMAddLICMPass(passmgr);
 239    LLVMAddAggressiveDCEPass(passmgr);
 240    LLVMAddCFGSimplificationPass(passmgr);
 241    /* This is recommended by the instruction combining pass. */
 242    LLVMAddEarlyCSEMemSSAPass(passmgr);
 243    LLVMAddInstructionCombiningPass(passmgr);
 244    return passmgr;
 245 }
 246
 247 static const char *attr_to_str(enum ac_func_attr attr)
 248 {
 249    switch (attr) {
 250    case AC_FUNC_ATTR_ALWAYSINLINE:
 251       return "alwaysinline";
 252    case AC_FUNC_ATTR_INREG:
 253       return "inreg";
 254    case AC_FUNC_ATTR_NOALIAS:
 255       return "noalias";
 256    case AC_FUNC_ATTR_NOUNWIND:
 257       return "nounwind";
 258    case AC_FUNC_ATTR_READNONE:
 259       return "readnone";
 260    case AC_FUNC_ATTR_READONLY:
 261       return "readonly";
 262    case AC_FUNC_ATTR_WRITEONLY:
 263       return "writeonly";
 264    case AC_FUNC_ATTR_INACCESSIBLE_MEM_ONLY:
 265       return "inaccessiblememonly";
 266    case AC_FUNC_ATTR_CONVERGENT:
 267       return "convergent";
 268    default:
 269       fprintf(stderr, "Unhandled function attribute: %x\n", attr);
 270       return 0;
 271    }
 272 }
 273
 274 void ac_add_function_attr(LLVMContextRef ctx, LLVMValueRef function, int attr_idx,
 275                           enum ac_func_attr attr)
 276 {
 277    const char *attr_name = attr_to_str(attr);
 278    unsigned kind_id = LLVMGetEnumAttributeKindForName(attr_name, strlen(attr_name));
 279    LLVMAttributeRef llvm_attr = LLVMCreateEnumAttribute(ctx, kind_id, 0);
 280
 281    if (LLVMIsAFunction(function))
 282       LLVMAddAttributeAtIndex(function, attr_idx, llvm_attr);
 283    else
 284       LLVMAddCallSiteAttribute(function, attr_idx, llvm_attr);
 285 }
 286
 287 void ac_add_func_attributes(LLVMContextRef ctx, LLVMValueRef function, unsigned attrib_mask)
 288 {
 289    attrib_mask |= AC_FUNC_ATTR_NOUNWIND;
 290    attrib_mask &= ~AC_FUNC_ATTR_LEGACY;
 291
 292    while (attrib_mask) {
 293       enum ac_func_attr attr = 1u << u_bit_scan(&attrib_mask);
 294       ac_add_function_attr(ctx, function, -1, attr);
 295    }
 296 }
 297
 298 void ac_dump_module(LLVMModuleRef module)
 299 {
 300    char *str = LLVMPrintModuleToString(module);
 301    fprintf(stderr, "%s", str);
 302    LLVMDisposeMessage(str);
 303 }
 304
 305 void ac_llvm_add_target_dep_function_attr(LLVMValueRef F, const char *name, unsigned value)
 306 {
 307    char str[16];
 308
 309    snprintf(str, sizeof(str), "0x%x", value);
 310    LLVMAddTargetDependentFunctionAttr(F, name, str);
 311 }
 312
 313 void ac_llvm_set_workgroup_size(LLVMValueRef F, unsigned size)
 314 {
 315    if (!size)
 316       return;
 317
 318    char str[32];
 319    snprintf(str, sizeof(str), "%u,%u", size, size);
 320    LLVMAddTargetDependentFunctionAttr(F, "amdgpu-flat-work-group-size", str);
 321 }
 322
 323 unsigned ac_count_scratch_private_memory(LLVMValueRef function)
 324 {
 325    unsigned private_mem_vgprs = 0;
 326
 327    /* Process all LLVM instructions. */
 328    LLVMBasicBlockRef bb = LLVMGetFirstBasicBlock(function);
 329    while (bb) {
 330       LLVMValueRef next = LLVMGetFirstInstruction(bb);
 331
 332       while (next) {
 333          LLVMValueRef inst = next;
 334          next = LLVMGetNextInstruction(next);
 335
 336          if (LLVMGetInstructionOpcode(inst) != LLVMAlloca)
 337             continue;
 338
 339          LLVMTypeRef type = LLVMGetElementType(LLVMTypeOf(inst));
 340          /* No idea why LLVM aligns allocas to 4 elements. */
 341          unsigned alignment = LLVMGetAlignment(inst);
 342          unsigned dw_size = align(ac_get_type_size(type) / 4, alignment);
 343          private_mem_vgprs += dw_size;
 344       }
 345       bb = LLVMGetNextBasicBlock(bb);
 346    }
 347
 348    return private_mem_vgprs;
 349 }
 350
 351 bool ac_init_llvm_compiler(struct ac_llvm_compiler *compiler, enum radeon_family family,
 352                            enum ac_target_machine_options tm_options)
 353 {
 354    const char *triple;
 355    memset(compiler, 0, sizeof(*compiler));
 356
 357    compiler->tm = ac_create_target_machine(family, tm_options, LLVMCodeGenLevelDefault, &triple);
 358    if (!compiler->tm)
 359       return false;
 360
 361    if (tm_options & AC_TM_CREATE_LOW_OPT) {
 362       compiler->low_opt_tm =
 363          ac_create_target_machine(family, tm_options, LLVMCodeGenLevelLess, NULL);
 364       if (!compiler->low_opt_tm)
 365          goto fail;
 366    }
 367
 368    if (family >= CHIP_NAVI10) {
 369       assert(!(tm_options & AC_TM_CREATE_LOW_OPT));
 370       compiler->tm_wave32 =
 371          ac_create_target_machine(family, tm_options | AC_TM_WAVE32, LLVMCodeGenLevelDefault, NULL);
 372       if (!compiler->tm_wave32)
 373          goto fail;
 374    }
 375
 376    compiler->target_library_info = ac_create_target_library_info(triple);
 377    if (!compiler->target_library_info)
 378       goto fail;
 379
 380    compiler->passmgr =
 381       ac_create_passmgr(compiler->target_library_info, tm_options & AC_TM_CHECK_IR);
 382    if (!compiler->passmgr)
 383       goto fail;
 384
 385    return true;
 386 fail:
 387    ac_destroy_llvm_compiler(compiler);
 388    return false;
 389 }
 390
 391 void ac_destroy_llvm_compiler(struct ac_llvm_compiler *compiler)
 392 {
 393    ac_destroy_llvm_passes(compiler->passes);
 394    ac_destroy_llvm_passes(compiler->passes_wave32);
 395    ac_destroy_llvm_passes(compiler->low_opt_passes);
 396
 397    if (compiler->passmgr)
 398       LLVMDisposePassManager(compiler->passmgr);
 399    if (compiler->target_library_info)
 400       ac_dispose_target_library_info(compiler->target_library_info);
 401    if (compiler->low_opt_tm)
 402       LLVMDisposeTargetMachine(compiler->low_opt_tm);
 403    if (compiler->tm)
 404       LLVMDisposeTargetMachine(compiler->tm);
 405    if (compiler->tm_wave32)
 406       LLVMDisposeTargetMachine(compiler->tm_wave32);
 407 }