src/amd/llvm/ac_llvm_util.c

   1 /*
   2  * Copyright 2014 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the
   6  * "Software"), to deal in the Software without restriction, including
   7  * without limitation the rights to use, copy, modify, merge, publish,
   8  * distribute, sub license, and/or sell copies of the Software, and to
   9  * permit persons to whom the Software is furnished to do so, subject to
  10  * the following conditions:
  11  *
  12  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  13  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  14  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  15  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  16  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  17  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  18  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  19  *
  20  * The above copyright notice and this permission notice (including the
  21  * next paragraph) shall be included in all copies or substantial portions
  22  * of the Software.
  23  *
  24  */
  25 /* based on pieces from si_pipe.c and radeon_llvm_emit.c */
  26 #include "ac_llvm_util.h"
  27 #include "ac_llvm_build.h"
  28 #include "util/bitscan.h"
  29 #include <llvm-c/Core.h>
  30 #include <llvm-c/Support.h>
  31 #include <llvm-c/Transforms/IPO.h>
  32 #include <llvm-c/Transforms/Scalar.h>
  33 #include <llvm-c/Transforms/Utils.h>
  34 #include "c11/threads.h"
  35 #include "gallivm/lp_bld_misc.h"
  36 #include "util/u_math.h"
  37
  38 #include <assert.h>
  39 #include <stdio.h>
  40 #include <string.h>
  41
  42 static void ac_init_llvm_target()
  43 {
  44         LLVMInitializeAMDGPUTargetInfo();
  45         LLVMInitializeAMDGPUTarget();
  46         LLVMInitializeAMDGPUTargetMC();
  47         LLVMInitializeAMDGPUAsmPrinter();
  48
  49         /* For inline assembly. */
  50         LLVMInitializeAMDGPUAsmParser();
  51
  52         /* For ACO disassembly. */
  53         LLVMInitializeAMDGPUDisassembler();
  54
  55         /* Workaround for bug in llvm 4.0 that causes image intrinsics
  56          * to disappear.
  57          * https://reviews.llvm.org/D26348
  58          *
  59          * "mesa" is the prefix for error messages.
  60          *
  61          * -global-isel-abort=2 is a no-op unless global isel has been enabled.
  62          * This option tells the backend to fall-back to SelectionDAG and print
  63          * a diagnostic message if global isel fails.
  64          */
  65         const char *argv[] = {
  66                 "mesa",
  67                 "-simplifycfg-sink-common=false",
  68                 "-global-isel-abort=2",
  69 #if LLVM_VERSION_MAJOR >= 10
  70                 /* Atomic optimizations require LLVM 10.0 for gfx10 support. */
  71                 "-amdgpu-atomic-optimizations=true",
  72 #endif
  73 #if LLVM_VERSION_MAJOR >= 11
  74                 /* This was disabled by default in: https://reviews.llvm.org/D77228 */
  75                 "-structurizecfg-skip-uniform-regions",
  76 #endif
  77         };
  78         LLVMParseCommandLineOptions(ARRAY_SIZE(argv), argv, NULL);
  79 }
  80
  81 PUBLIC void ac_init_shared_llvm_once(void)
  82 {
  83         static once_flag ac_init_llvm_target_once_flag = ONCE_FLAG_INIT;
  84         call_once(&ac_init_llvm_target_once_flag, ac_init_llvm_target);
  85 }
  86
  87 #if !LLVM_IS_SHARED
  88 static once_flag ac_init_static_llvm_target_once_flag = ONCE_FLAG_INIT;
  89 static void ac_init_static_llvm_once(void)
  90 {
  91         call_once(&ac_init_static_llvm_target_once_flag, ac_init_llvm_target);
  92 }
  93 #endif
  94
  95 void ac_init_llvm_once(void)
  96 {
  97 #if LLVM_IS_SHARED
  98         ac_init_shared_llvm_once();
  99 #else
 100         ac_init_static_llvm_once();
 101 #endif
 102 }
 103
 104 static LLVMTargetRef ac_get_llvm_target(const char *triple)
 105 {
 106         LLVMTargetRef target = NULL;
 107         char *err_message = NULL;
 108
 109         if (LLVMGetTargetFromTriple(triple, &target, &err_message)) {
 110                 fprintf(stderr, "Cannot find target for triple %s ", triple);
 111                 if (err_message) {
 112                         fprintf(stderr, "%s\n", err_message);
 113                 }
 114                 LLVMDisposeMessage(err_message);
 115                 return NULL;
 116         }
 117         return target;
 118 }
 119
 120 const char *ac_get_llvm_processor_name(enum radeon_family family)
 121 {
 122         switch (family) {
 123         case CHIP_TAHITI:
 124                 return "tahiti";
 125         case CHIP_PITCAIRN:
 126                 return "pitcairn";
 127         case CHIP_VERDE:
 128                 return "verde";
 129         case CHIP_OLAND:
 130                 return "oland";
 131         case CHIP_HAINAN:
 132                 return "hainan";
 133         case CHIP_BONAIRE:
 134                 return "bonaire";
 135         case CHIP_KABINI:
 136                 return "kabini";
 137         case CHIP_KAVERI:
 138                 return "kaveri";
 139         case CHIP_HAWAII:
 140                 return "hawaii";
 141         case CHIP_TONGA:
 142                 return "tonga";
 143         case CHIP_ICELAND:
 144                 return "iceland";
 145         case CHIP_CARRIZO:
 146                 return "carrizo";
 147         case CHIP_FIJI:
 148                 return "fiji";
 149         case CHIP_STONEY:
 150                 return "stoney";
 151         case CHIP_POLARIS10:
 152                 return "polaris10";
 153         case CHIP_POLARIS11:
 154         case CHIP_POLARIS12:
 155         case CHIP_VEGAM:
 156                 return "polaris11";
 157         case CHIP_VEGA10:
 158                 return "gfx900";
 159         case CHIP_RAVEN:
 160                 return "gfx902";
 161         case CHIP_VEGA12:
 162                 return "gfx904";
 163         case CHIP_VEGA20:
 164                 return "gfx906";
 165         case CHIP_RAVEN2:
 166         case CHIP_RENOIR:
 167                 return "gfx909";
 168         case CHIP_ARCTURUS:
 169                 return "gfx908";
 170         case CHIP_NAVI10:
 171                 return "gfx1010";
 172         case CHIP_NAVI12:
 173                 return "gfx1011";
 174         case CHIP_NAVI14:
 175                 return "gfx1012";
 176         case CHIP_SIENNA_CICHLID:
 177         case CHIP_NAVY_FLOUNDER:
 178                 return "gfx1030";
 179         default:
 180                 return "";
 181         }
 182 }
 183
 184 static LLVMTargetMachineRef ac_create_target_machine(enum radeon_family family,
 185                                                      enum ac_target_machine_options tm_options,
 186                                                      LLVMCodeGenOptLevel level,
 187                                                      const char **out_triple)
 188 {
 189         assert(family >= CHIP_TAHITI);
 190         char features[256];
 191         const char *triple = (tm_options & AC_TM_SUPPORTS_SPILL) ? "amdgcn-mesa-mesa3d" : "amdgcn--";
 192         LLVMTargetRef target = ac_get_llvm_target(triple);
 193
 194         snprintf(features, sizeof(features),
 195                  "+DumpCode%s%s%s%s%s",
 196                  LLVM_VERSION_MAJOR >= 11 ? "" : ",-fp32-denormals,+fp64-denormals",
 197                  family >= CHIP_NAVI10 && !(tm_options & AC_TM_WAVE32) ?
 198                          ",+wavefrontsize64,-wavefrontsize32" : "",
 199                  family <= CHIP_NAVI14 && tm_options & AC_TM_FORCE_ENABLE_XNACK ? ",+xnack" : "",
 200                  family <= CHIP_NAVI14 && tm_options & AC_TM_FORCE_DISABLE_XNACK ? ",-xnack" : "",
 201                  tm_options & AC_TM_PROMOTE_ALLOCA_TO_SCRATCH ? ",-promote-alloca" : "");
 202
 203         LLVMTargetMachineRef tm = LLVMCreateTargetMachine(
 204                                      target,
 205                                      triple,
 206                                      ac_get_llvm_processor_name(family),
 207                                      features,
 208                                      level,
 209                                      LLVMRelocDefault,
 210                                      LLVMCodeModelDefault);
 211
 212         if (out_triple)
 213                 *out_triple = triple;
 214         if (tm_options & AC_TM_ENABLE_GLOBAL_ISEL)
 215                 ac_enable_global_isel(tm);
 216         return tm;
 217 }
 218
 219 static LLVMPassManagerRef ac_create_passmgr(LLVMTargetLibraryInfoRef target_library_info,
 220                                             bool check_ir)
 221 {
 222         LLVMPassManagerRef passmgr = LLVMCreatePassManager();
 223         if (!passmgr)
 224                 return NULL;
 225
 226         if (target_library_info)
 227                 LLVMAddTargetLibraryInfo(target_library_info,
 228                                          passmgr);
 229
 230         if (check_ir)
 231                 LLVMAddVerifierPass(passmgr);
 232         LLVMAddAlwaysInlinerPass(passmgr);
 233         /* Normally, the pass manager runs all passes on one function before
 234          * moving onto another. Adding a barrier no-op pass forces the pass
 235          * manager to run the inliner on all functions first, which makes sure
 236          * that the following passes are only run on the remaining non-inline
 237          * function, so it removes useless work done on dead inline functions.
 238          */
 239         ac_llvm_add_barrier_noop_pass(passmgr);
 240         /* This pass should eliminate all the load and store instructions. */
 241         LLVMAddPromoteMemoryToRegisterPass(passmgr);
 242         LLVMAddScalarReplAggregatesPass(passmgr);
 243         LLVMAddLICMPass(passmgr);
 244         LLVMAddAggressiveDCEPass(passmgr);
 245         LLVMAddCFGSimplificationPass(passmgr);
 246         /* This is recommended by the instruction combining pass. */
 247         LLVMAddEarlyCSEMemSSAPass(passmgr);
 248         LLVMAddInstructionCombiningPass(passmgr);
 249         return passmgr;
 250 }
 251
 252 static const char *attr_to_str(enum ac_func_attr attr)
 253 {
 254    switch (attr) {
 255    case AC_FUNC_ATTR_ALWAYSINLINE: return "alwaysinline";
 256    case AC_FUNC_ATTR_INREG: return "inreg";
 257    case AC_FUNC_ATTR_NOALIAS: return "noalias";
 258    case AC_FUNC_ATTR_NOUNWIND: return "nounwind";
 259    case AC_FUNC_ATTR_READNONE: return "readnone";
 260    case AC_FUNC_ATTR_READONLY: return "readonly";
 261    case AC_FUNC_ATTR_WRITEONLY: return "writeonly";
 262    case AC_FUNC_ATTR_INACCESSIBLE_MEM_ONLY: return "inaccessiblememonly";
 263    case AC_FUNC_ATTR_CONVERGENT: return "convergent";
 264    default:
 265            fprintf(stderr, "Unhandled function attribute: %x\n", attr);
 266            return 0;
 267    }
 268 }
 269
 270 void
 271 ac_add_function_attr(LLVMContextRef ctx, LLVMValueRef function,
 272                      int attr_idx, enum ac_func_attr attr)
 273 {
 274    const char *attr_name = attr_to_str(attr);
 275    unsigned kind_id = LLVMGetEnumAttributeKindForName(attr_name,
 276                                                       strlen(attr_name));
 277    LLVMAttributeRef llvm_attr = LLVMCreateEnumAttribute(ctx, kind_id, 0);
 278
 279    if (LLVMIsAFunction(function))
 280       LLVMAddAttributeAtIndex(function, attr_idx, llvm_attr);
 281    else
 282       LLVMAddCallSiteAttribute(function, attr_idx, llvm_attr);
 283 }
 284
 285 void ac_add_func_attributes(LLVMContextRef ctx, LLVMValueRef function,
 286                             unsigned attrib_mask)
 287 {
 288         attrib_mask |= AC_FUNC_ATTR_NOUNWIND;
 289         attrib_mask &= ~AC_FUNC_ATTR_LEGACY;
 290
 291         while (attrib_mask) {
 292                 enum ac_func_attr attr = 1u << u_bit_scan(&attrib_mask);
 293                 ac_add_function_attr(ctx, function, -1, attr);
 294         }
 295 }
 296
 297 void
 298 ac_dump_module(LLVMModuleRef module)
 299 {
 300         char *str = LLVMPrintModuleToString(module);
 301         fprintf(stderr, "%s", str);
 302         LLVMDisposeMessage(str);
 303 }
 304
 305 void
 306 ac_llvm_add_target_dep_function_attr(LLVMValueRef F,
 307                                      const char *name, unsigned value)
 308 {
 309         char str[16];
 310
 311         snprintf(str, sizeof(str), "0x%x", value);
 312         LLVMAddTargetDependentFunctionAttr(F, name, str);
 313 }
 314
 315 void ac_llvm_set_workgroup_size(LLVMValueRef F, unsigned size)
 316 {
 317         if (!size)
 318                 return;
 319
 320         char str[32];
 321         snprintf(str, sizeof(str), "%u,%u", size, size);
 322         LLVMAddTargetDependentFunctionAttr(F, "amdgpu-flat-work-group-size", str);
 323 }
 324
 325 unsigned
 326 ac_count_scratch_private_memory(LLVMValueRef function)
 327 {
 328         unsigned private_mem_vgprs = 0;
 329
 330         /* Process all LLVM instructions. */
 331         LLVMBasicBlockRef bb = LLVMGetFirstBasicBlock(function);
 332         while (bb) {
 333                 LLVMValueRef next = LLVMGetFirstInstruction(bb);
 334
 335                 while (next) {
 336                         LLVMValueRef inst = next;
 337                         next = LLVMGetNextInstruction(next);
 338
 339                         if (LLVMGetInstructionOpcode(inst) != LLVMAlloca)
 340                                 continue;
 341
 342                         LLVMTypeRef type = LLVMGetElementType(LLVMTypeOf(inst));
 343                         /* No idea why LLVM aligns allocas to 4 elements. */
 344                         unsigned alignment = LLVMGetAlignment(inst);
 345                         unsigned dw_size = align(ac_get_type_size(type) / 4, alignment);
 346                         private_mem_vgprs += dw_size;
 347                 }
 348                 bb = LLVMGetNextBasicBlock(bb);
 349         }
 350
 351         return private_mem_vgprs;
 352 }
 353
 354 bool
 355 ac_init_llvm_compiler(struct ac_llvm_compiler *compiler,
 356                       enum radeon_family family,
 357                       enum ac_target_machine_options tm_options)
 358 {
 359         const char *triple;
 360         memset(compiler, 0, sizeof(*compiler));
 361
 362         compiler->tm = ac_create_target_machine(family, tm_options,
 363                                                 LLVMCodeGenLevelDefault,
 364                                                 &triple);
 365         if (!compiler->tm)
 366                 return false;
 367
 368         if (tm_options & AC_TM_CREATE_LOW_OPT) {
 369                 compiler->low_opt_tm =
 370                         ac_create_target_machine(family, tm_options,
 371                                                  LLVMCodeGenLevelLess, NULL);
 372                 if (!compiler->low_opt_tm)
 373                         goto fail;
 374         }
 375
 376         if (family >= CHIP_NAVI10) {
 377                 assert(!(tm_options & AC_TM_CREATE_LOW_OPT));
 378                 compiler->tm_wave32 = ac_create_target_machine(family,
 379                                                                tm_options | AC_TM_WAVE32,
 380                                                                LLVMCodeGenLevelDefault,
 381                                                                NULL);
 382                 if (!compiler->tm_wave32)
 383                         goto fail;
 384         }
 385
 386         compiler->target_library_info =
 387                 ac_create_target_library_info(triple);
 388         if (!compiler->target_library_info)
 389                 goto fail;
 390
 391         compiler->passmgr = ac_create_passmgr(compiler->target_library_info,
 392                                               tm_options & AC_TM_CHECK_IR);
 393         if (!compiler->passmgr)
 394                 goto fail;
 395
 396         return true;
 397 fail:
 398         ac_destroy_llvm_compiler(compiler);
 399         return false;
 400 }
 401
 402 void
 403 ac_destroy_llvm_compiler(struct ac_llvm_compiler *compiler)
 404 {
 405         ac_destroy_llvm_passes(compiler->passes);
 406         ac_destroy_llvm_passes(compiler->passes_wave32);
 407         ac_destroy_llvm_passes(compiler->low_opt_passes);
 408
 409         if (compiler->passmgr)
 410                 LLVMDisposePassManager(compiler->passmgr);
 411         if (compiler->target_library_info)
 412                 ac_dispose_target_library_info(compiler->target_library_info);
 413         if (compiler->low_opt_tm)
 414                 LLVMDisposeTargetMachine(compiler->low_opt_tm);
 415         if (compiler->tm)
 416                 LLVMDisposeTargetMachine(compiler->tm);
 417         if (compiler->tm_wave32)
 418                 LLVMDisposeTargetMachine(compiler->tm_wave32);
 419 }