src/amd/common/ac_rtld.c

   1 /*
   2  * Copyright 2014-2019 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  21  * SOFTWARE.
  22  */
  23
  24 #include "ac_rtld.h"
  25
  26 #include <gelf.h>
  27 #include <libelf.h>
  28 #include <stdarg.h>
  29 #include <stdio.h>
  30 #include <stdlib.h>
  31 #include <string.h>
  32
  33 #include "ac_binary.h"
  34 #include "ac_gpu_info.h"
  35 #include "util/u_dynarray.h"
  36 #include "util/u_math.h"
  37
  38 // Old distributions may not have this enum constant
  39 #define MY_EM_AMDGPU 224
  40
  41 #ifndef STT_AMDGPU_LDS
  42 #define STT_AMDGPU_LDS 13
  43 #endif
  44
  45 #ifndef R_AMDGPU_NONE
  46 #define R_AMDGPU_NONE 0
  47 #define R_AMDGPU_ABS32_LO 1
  48 #define R_AMDGPU_ABS32_HI 2
  49 #define R_AMDGPU_ABS64 3
  50 #define R_AMDGPU_REL32 4
  51 #define R_AMDGPU_REL64 5
  52 #define R_AMDGPU_ABS32 6
  53 #define R_AMDGPU_GOTPCREL 7
  54 #define R_AMDGPU_GOTPCREL32_LO 8
  55 #define R_AMDGPU_GOTPCREL32_HI 9
  56 #define R_AMDGPU_REL32_LO 10
  57 #define R_AMDGPU_REL32_HI 11
  58 #define R_AMDGPU_RELATIVE64 13
  59 #endif
  60
  61 /* For the UMR disassembler. */
  62 #define DEBUGGER_END_OF_CODE_MARKER     0xbf9f0000 /* invalid instruction */
  63 #define DEBUGGER_NUM_MARKERS            5
  64
  65 struct ac_rtld_section {
  66         bool is_rx : 1;
  67         bool is_pasted_text : 1;
  68         uint64_t offset;
  69         const char *name;
  70 };
  71
  72 struct ac_rtld_part {
  73         Elf *elf;
  74         struct ac_rtld_section *sections;
  75         unsigned num_sections;
  76 };
  77
  78 static void report_erroraf(const char *fmt, va_list va)
  79 {
  80         char *msg;
  81         int ret = asprintf(&msg, fmt, va);
  82         if (ret < 0)
  83                 msg = "(asprintf failed)";
  84
  85         fprintf(stderr, "ac_rtld error: %s\n", msg);
  86
  87         if (ret >= 0)
  88                 free(msg);
  89 }
  90
  91 static void report_errorf(const char *fmt, ...) PRINTFLIKE(1, 2);
  92
  93 static void report_errorf(const char *fmt, ...)
  94 {
  95         va_list va;
  96         va_start(va, fmt);
  97         report_erroraf(fmt, va);
  98         va_end(va);
  99 }
 100
 101 static void report_elf_errorf(const char *fmt, ...) PRINTFLIKE(1, 2);
 102
 103 static void report_elf_errorf(const char *fmt, ...)
 104 {
 105         va_list va;
 106         va_start(va, fmt);
 107         report_erroraf(fmt, va);
 108         va_end(va);
 109
 110         fprintf(stderr, "ELF error: %s\n", elf_errmsg(elf_errno()));
 111 }
 112
 113 /**
 114  * Find a symbol in a dynarray of struct ac_rtld_symbol by \p name and shader
 115  * \p part_idx.
 116  */
 117 static const struct ac_rtld_symbol *find_symbol(const struct util_dynarray *symbols,
 118                                                 const char *name, unsigned part_idx)
 119 {
 120         util_dynarray_foreach(symbols, struct ac_rtld_symbol, symbol) {
 121                 if ((symbol->part_idx == ~0u || symbol->part_idx == part_idx) &&
 122                     !strcmp(name, symbol->name))
 123                         return symbol;
 124         }
 125         return 0;
 126 }
 127
 128 static int compare_symbol_by_align(const void *lhsp, const void *rhsp)
 129 {
 130         const struct ac_rtld_symbol *lhs = lhsp;
 131         const struct ac_rtld_symbol *rhs = rhsp;
 132         if (rhs->align > lhs->align)
 133                 return 1;
 134         if (rhs->align < lhs->align)
 135                 return -1;
 136         return 0;
 137 }
 138
 139 /**
 140  * Sort the given symbol list by decreasing alignment and assign offsets.
 141  */
 142 static bool layout_symbols(struct ac_rtld_symbol *symbols, unsigned num_symbols,
 143                            uint64_t *ptotal_size)
 144 {
 145         qsort(symbols, num_symbols, sizeof(*symbols), compare_symbol_by_align);
 146
 147         uint64_t total_size = *ptotal_size;
 148
 149         for (unsigned i = 0; i < num_symbols; ++i) {
 150                 struct ac_rtld_symbol *s = &symbols[i];
 151                 assert(util_is_power_of_two_nonzero(s->align));
 152
 153                 total_size = align64(total_size, s->align);
 154                 s->offset = total_size;
 155
 156                 if (total_size + s->size < total_size) {
 157                         report_errorf("%s: size overflow", __FUNCTION__);
 158                         return false;
 159                 }
 160
 161                 total_size += s->size;
 162         }
 163
 164         *ptotal_size = total_size;
 165         return true;
 166 }
 167
 168 /**
 169  * Read LDS symbols from the given \p section of the ELF of \p part and append
 170  * them to the LDS symbols list.
 171  *
 172  * Shared LDS symbols are filtered out.
 173  */
 174 static bool read_private_lds_symbols(struct ac_rtld_binary *binary,
 175                                      unsigned part_idx,
 176                                      Elf_Scn *section,
 177                                      uint32_t *lds_end_align)
 178 {
 179 #define report_elf_if(cond) \
 180         do { \
 181                 if ((cond)) { \
 182                         report_errorf(#cond); \
 183                         return false; \
 184                 } \
 185         } while (false)
 186
 187         struct ac_rtld_part *part = &binary->parts[part_idx];
 188         Elf64_Shdr *shdr = elf64_getshdr(section);
 189         uint32_t strtabidx = shdr->sh_link;
 190         Elf_Data *symbols_data = elf_getdata(section, NULL);
 191         report_elf_if(!symbols_data);
 192
 193         const Elf64_Sym *symbol = symbols_data->d_buf;
 194         size_t num_symbols = symbols_data->d_size / sizeof(Elf64_Sym);
 195
 196         for (size_t j = 0; j < num_symbols; ++j, ++symbol) {
 197                 if (ELF64_ST_TYPE(symbol->st_info) != STT_AMDGPU_LDS)
 198                         continue;
 199
 200                 report_elf_if(symbol->st_size > 1u << 29);
 201
 202                 struct ac_rtld_symbol s = {};
 203                 s.name = elf_strptr(part->elf, strtabidx, symbol->st_name);
 204                 s.size = symbol->st_size;
 205                 s.align = MIN2(1u << (symbol->st_other >> 3), 1u << 16);
 206                 s.part_idx = part_idx;
 207
 208                 if (!strcmp(s.name, "__lds_end")) {
 209                         report_elf_if(s.size != 0);
 210                         *lds_end_align = MAX2(*lds_end_align, s.align);
 211                         continue;
 212                 }
 213
 214                 const struct ac_rtld_symbol *shared =
 215                         find_symbol(&binary->lds_symbols, s.name, part_idx);
 216                 if (shared) {
 217                         report_elf_if(s.align > shared->align);
 218                         report_elf_if(s.size > shared->size);
 219                         continue;
 220                 }
 221
 222                 util_dynarray_append(&binary->lds_symbols, struct ac_rtld_symbol, s);
 223         }
 224
 225         return true;
 226
 227 #undef report_elf_if
 228 }
 229
 230 /**
 231  * Open a binary consisting of one or more shader parts.
 232  *
 233  * \param binary the uninitialized struct
 234  * \param i binary opening parameters
 235  */
 236 bool ac_rtld_open(struct ac_rtld_binary *binary,
 237                   struct ac_rtld_open_info i)
 238 {
 239         /* One of the libelf implementations
 240          * (http://www.mr511.de/software/english.htm) requires calling
 241          * elf_version() before elf_memory().
 242          */
 243         elf_version(EV_CURRENT);
 244
 245         memset(binary, 0, sizeof(*binary));
 246         memcpy(&binary->options, &i.options, sizeof(binary->options));
 247         binary->num_parts = i.num_parts;
 248         binary->parts = calloc(sizeof(*binary->parts), i.num_parts);
 249         if (!binary->parts)
 250                 return false;
 251
 252         uint64_t pasted_text_size = 0;
 253         uint64_t rx_align = 1;
 254         uint64_t rx_size = 0;
 255
 256 #define report_if(cond) \
 257         do { \
 258                 if ((cond)) { \
 259                         report_errorf(#cond); \
 260                         goto fail; \
 261                 } \
 262         } while (false)
 263 #define report_elf_if(cond) \
 264         do { \
 265                 if ((cond)) { \
 266                         report_elf_errorf(#cond); \
 267                         goto fail; \
 268                 } \
 269         } while (false)
 270
 271         /* Copy and layout shared LDS symbols. */
 272         if (i.num_shared_lds_symbols) {
 273                 if (!util_dynarray_resize(&binary->lds_symbols, struct ac_rtld_symbol,
 274                                           i.num_shared_lds_symbols))
 275                         goto fail;
 276
 277                 memcpy(binary->lds_symbols.data, i.shared_lds_symbols, binary->lds_symbols.size);
 278         }
 279
 280         util_dynarray_foreach(&binary->lds_symbols, struct ac_rtld_symbol, symbol)
 281                 symbol->part_idx = ~0u;
 282
 283         unsigned max_lds_size = i.info->chip_class >= GFX7 ? 64 * 1024 : 32 * 1024;
 284         uint64_t shared_lds_size = 0;
 285         if (!layout_symbols(binary->lds_symbols.data, i.num_shared_lds_symbols, &shared_lds_size))
 286                 goto fail;
 287         report_if(shared_lds_size > max_lds_size);
 288         binary->lds_size = shared_lds_size;
 289
 290         /* First pass over all parts: open ELFs, pre-determine the placement of
 291          * sections in the memory image, and collect and layout private LDS symbols. */
 292         uint32_t lds_end_align = 0;
 293
 294         if (binary->options.halt_at_entry)
 295                 pasted_text_size += 4;
 296
 297         for (unsigned part_idx = 0; part_idx < i.num_parts; ++part_idx) {
 298                 struct ac_rtld_part *part = &binary->parts[part_idx];
 299                 unsigned part_lds_symbols_begin =
 300                         util_dynarray_num_elements(&binary->lds_symbols, struct ac_rtld_symbol);
 301
 302                 part->elf = elf_memory((char *)i.elf_ptrs[part_idx], i.elf_sizes[part_idx]);
 303                 report_elf_if(!part->elf);
 304
 305                 const Elf64_Ehdr *ehdr = elf64_getehdr(part->elf);
 306                 report_elf_if(!ehdr);
 307                 report_if(ehdr->e_machine != MY_EM_AMDGPU);
 308
 309                 size_t section_str_index;
 310                 size_t num_shdrs;
 311                 report_elf_if(elf_getshdrstrndx(part->elf, &section_str_index) < 0);
 312                 report_elf_if(elf_getshdrnum(part->elf, &num_shdrs) < 0);
 313
 314                 part->num_sections = num_shdrs;
 315                 part->sections = calloc(sizeof(*part->sections), num_shdrs);
 316                 report_if(!part->sections);
 317
 318                 Elf_Scn *section = NULL;
 319                 while ((section = elf_nextscn(part->elf, section))) {
 320                         Elf64_Shdr *shdr = elf64_getshdr(section);
 321                         struct ac_rtld_section *s = &part->sections[elf_ndxscn(section)];
 322                         s->name = elf_strptr(part->elf, section_str_index, shdr->sh_name);
 323                         report_elf_if(!s->name);
 324
 325                         /* Cannot actually handle linked objects yet */
 326                         report_elf_if(shdr->sh_addr != 0);
 327
 328                         /* Alignment must be 0 or a power of two */
 329                         report_elf_if(shdr->sh_addralign & (shdr->sh_addralign - 1));
 330                         uint64_t sh_align = MAX2(shdr->sh_addralign, 1);
 331
 332                         if (shdr->sh_flags & SHF_ALLOC &&
 333                             shdr->sh_type != SHT_NOTE) {
 334                                 report_if(shdr->sh_flags & SHF_WRITE);
 335
 336                                 s->is_rx = true;
 337
 338                                 if (shdr->sh_flags & SHF_EXECINSTR) {
 339                                         report_elf_if(shdr->sh_size & 3);
 340
 341                                         if (!strcmp(s->name, ".text"))
 342                                                 s->is_pasted_text = true;
 343                                 }
 344
 345                                 if (s->is_pasted_text) {
 346                                         s->offset = pasted_text_size;
 347                                         pasted_text_size += shdr->sh_size;
 348                                 } else {
 349                                         rx_align = align(rx_align, sh_align);
 350                                         rx_size = align(rx_size, sh_align);
 351                                         s->offset = rx_size;
 352                                         rx_size += shdr->sh_size;
 353                                 }
 354                         } else if (shdr->sh_type == SHT_SYMTAB) {
 355                                 if (!read_private_lds_symbols(binary, part_idx, section, &lds_end_align))
 356                                         goto fail;
 357                         }
 358                 }
 359
 360                 uint64_t part_lds_size = shared_lds_size;
 361                 if (!layout_symbols(
 362                         util_dynarray_element(&binary->lds_symbols, struct ac_rtld_symbol, part_lds_symbols_begin),
 363                         util_dynarray_num_elements(&binary->lds_symbols, struct ac_rtld_symbol) - part_lds_symbols_begin,
 364                         &part_lds_size))
 365                         goto fail;
 366                 binary->lds_size = MAX2(binary->lds_size, part_lds_size);
 367         }
 368
 369         binary->rx_end_markers = pasted_text_size;
 370         pasted_text_size += 4 * DEBUGGER_NUM_MARKERS;
 371
 372         /* __lds_end is a special symbol that points at the end of the memory
 373          * occupied by other LDS symbols. Its alignment is taken as the
 374          * maximum of its alignment over all shader parts where it occurs.
 375          */
 376         if (lds_end_align) {
 377                 binary->lds_size = align(binary->lds_size, lds_end_align);
 378
 379                 struct ac_rtld_symbol *lds_end =
 380                         util_dynarray_grow(&binary->lds_symbols, struct ac_rtld_symbol, 1);
 381                 lds_end->name = "__lds_end";
 382                 lds_end->size = 0;
 383                 lds_end->align = lds_end_align;
 384                 lds_end->offset = binary->lds_size;
 385                 lds_end->part_idx = ~0u;
 386         }
 387
 388         report_elf_if(binary->lds_size > max_lds_size);
 389
 390         /* Second pass: Adjust offsets of non-pasted text sections. */
 391         binary->rx_size = pasted_text_size;
 392         binary->rx_size = align(binary->rx_size, rx_align);
 393
 394         for (unsigned part_idx = 0; part_idx < i.num_parts; ++part_idx) {
 395                 struct ac_rtld_part *part = &binary->parts[part_idx];
 396                 size_t num_shdrs;
 397                 elf_getshdrnum(part->elf, &num_shdrs);
 398
 399                 for (unsigned j = 0; j < num_shdrs; ++j) {
 400                         struct ac_rtld_section *s = &part->sections[j];
 401                         if (s->is_rx && !s->is_pasted_text)
 402                                 s->offset += binary->rx_size;
 403                 }
 404         }
 405
 406         binary->rx_size += rx_size;
 407
 408         return true;
 409
 410 #undef report_if
 411 #undef report_elf_if
 412
 413 fail:
 414         ac_rtld_close(binary);
 415         return false;
 416 }
 417
 418 void ac_rtld_close(struct ac_rtld_binary *binary)
 419 {
 420         for (unsigned i = 0; i < binary->num_parts; ++i) {
 421                 struct ac_rtld_part *part = &binary->parts[i];
 422                 free(part->sections);
 423                 elf_end(part->elf);
 424         }
 425
 426         util_dynarray_fini(&binary->lds_symbols);
 427         free(binary->parts);
 428         binary->parts = NULL;
 429         binary->num_parts = 0;
 430 }
 431
 432 static bool get_section_by_name(struct ac_rtld_part *part, const char *name,
 433                                 const char **data, size_t *nbytes)
 434 {
 435         for (unsigned i = 0; i < part->num_sections; ++i) {
 436                 struct ac_rtld_section *s = &part->sections[i];
 437                 if (s->name && !strcmp(name, s->name)) {
 438                         Elf_Scn *target_scn = elf_getscn(part->elf, i);
 439                         Elf_Data *target_data = elf_getdata(target_scn, NULL);
 440                         if (!target_data) {
 441                                 report_elf_errorf("ac_rtld: get_section_by_name: elf_getdata");
 442                                 return false;
 443                         }
 444
 445                         *data = target_data->d_buf;
 446                         *nbytes = target_data->d_size;
 447                         return true;
 448                 }
 449         }
 450         return false;
 451 }
 452
 453 bool ac_rtld_get_section_by_name(struct ac_rtld_binary *binary, const char *name,
 454                                  const char **data, size_t *nbytes)
 455 {
 456         assert(binary->num_parts == 1);
 457         return get_section_by_name(&binary->parts[0], name, data, nbytes);
 458 }
 459
 460 bool ac_rtld_read_config(struct ac_rtld_binary *binary,
 461                          struct ac_shader_config *config)
 462 {
 463         for (unsigned i = 0; i < binary->num_parts; ++i) {
 464                 struct ac_rtld_part *part = &binary->parts[i];
 465                 const char *config_data;
 466                 size_t config_nbytes;
 467
 468                 if (!get_section_by_name(part, ".AMDGPU.config",
 469                                          &config_data, &config_nbytes))
 470                         return false;
 471
 472                 /* TODO: be precise about scratch use? */
 473                 struct ac_shader_config c = {};
 474                 ac_parse_shader_binary_config(config_data, config_nbytes, true, &c);
 475
 476                 config->num_sgprs = MAX2(config->num_sgprs, c.num_sgprs);
 477                 config->num_vgprs = MAX2(config->num_vgprs, c.num_vgprs);
 478                 config->spilled_sgprs = MAX2(config->spilled_sgprs, c.spilled_sgprs);
 479                 config->spilled_vgprs = MAX2(config->spilled_vgprs, c.spilled_vgprs);
 480                 config->scratch_bytes_per_wave = MAX2(config->scratch_bytes_per_wave,
 481                                                       c.scratch_bytes_per_wave);
 482
 483                 assert(i == 0 || config->float_mode == c.float_mode);
 484                 config->float_mode = c.float_mode;
 485
 486                 /* SPI_PS_INPUT_ENA/ADDR can't be combined. Only the value from
 487                  * the main shader part is used. */
 488                 assert(config->spi_ps_input_ena == 0 &&
 489                        config->spi_ps_input_addr == 0);
 490                 config->spi_ps_input_ena = c.spi_ps_input_ena;
 491                 config->spi_ps_input_addr = c.spi_ps_input_addr;
 492
 493                 /* TODO: consistently use LDS symbols for this */
 494                 config->lds_size = MAX2(config->lds_size, c.lds_size);
 495
 496                 /* TODO: Should we combine these somehow? It's currently only
 497                  * used for radeonsi's compute, where multiple parts aren't used. */
 498                 assert(config->rsrc1 == 0 && config->rsrc2 == 0);
 499                 config->rsrc1 = c.rsrc1;
 500                 config->rsrc2 = c.rsrc2;
 501         }
 502
 503         return true;
 504 }
 505
 506 static bool resolve_symbol(const struct ac_rtld_upload_info *u,
 507                            unsigned part_idx, const Elf64_Sym *sym,
 508                            const char *name, uint64_t *value)
 509 {
 510         if (sym->st_shndx == SHN_UNDEF) {
 511                 const struct ac_rtld_symbol *lds_sym =
 512                         find_symbol(&u->binary->lds_symbols, name, part_idx);
 513
 514                 if (lds_sym) {
 515                         *value = lds_sym->offset;
 516                         return true;
 517                 }
 518
 519                 /* TODO: resolve from other parts */
 520
 521                 if (u->get_external_symbol(u->cb_data, name, value))
 522                         return true;
 523
 524                 report_errorf("symbol %s: unknown", name);
 525                 return false;
 526         }
 527
 528         struct ac_rtld_part *part = &u->binary->parts[part_idx];
 529         if (sym->st_shndx >= part->num_sections) {
 530                 report_errorf("symbol %s: section out of bounds", name);
 531                 return false;
 532         }
 533
 534         struct ac_rtld_section *s = &part->sections[sym->st_shndx];
 535         if (!s->is_rx) {
 536                 report_errorf("symbol %s: bad section", name);
 537                 return false;
 538         }
 539
 540         uint64_t section_base = u->rx_va + s->offset;
 541
 542         *value = section_base + sym->st_value;
 543         return true;
 544 }
 545
 546 static bool apply_relocs(const struct ac_rtld_upload_info *u,
 547                          unsigned part_idx, const Elf64_Shdr *reloc_shdr,
 548                          const Elf_Data *reloc_data)
 549 {
 550 #define report_if(cond) \
 551         do { \
 552                 if ((cond)) { \
 553                         report_errorf(#cond); \
 554                         return false; \
 555                 } \
 556         } while (false)
 557 #define report_elf_if(cond) \
 558         do { \
 559                 if ((cond)) { \
 560                         report_elf_errorf(#cond); \
 561                         return false; \
 562                 } \
 563         } while (false)
 564
 565         struct ac_rtld_part *part = &u->binary->parts[part_idx];
 566         Elf_Scn *target_scn = elf_getscn(part->elf, reloc_shdr->sh_info);
 567         report_elf_if(!target_scn);
 568
 569         Elf_Data *target_data = elf_getdata(target_scn, NULL);
 570         report_elf_if(!target_data);
 571
 572         Elf_Scn *symbols_scn = elf_getscn(part->elf, reloc_shdr->sh_link);
 573         report_elf_if(!symbols_scn);
 574
 575         Elf64_Shdr *symbols_shdr = elf64_getshdr(symbols_scn);
 576         report_elf_if(!symbols_shdr);
 577         uint32_t strtabidx = symbols_shdr->sh_link;
 578
 579         Elf_Data *symbols_data = elf_getdata(symbols_scn, NULL);
 580         report_elf_if(!symbols_data);
 581
 582         const Elf64_Sym *symbols = symbols_data->d_buf;
 583         size_t num_symbols = symbols_data->d_size / sizeof(Elf64_Sym);
 584
 585         struct ac_rtld_section *s = &part->sections[reloc_shdr->sh_info];
 586         report_if(!s->is_rx);
 587
 588         const char *orig_base = target_data->d_buf;
 589         char *dst_base = u->rx_ptr + s->offset;
 590         uint64_t va_base = u->rx_va + s->offset;
 591
 592         Elf64_Rel *rel = reloc_data->d_buf;
 593         size_t num_relocs = reloc_data->d_size / sizeof(*rel);
 594         for (size_t i = 0; i < num_relocs; ++i, ++rel) {
 595                 size_t r_sym = ELF64_R_SYM(rel->r_info);
 596                 unsigned r_type = ELF64_R_TYPE(rel->r_info);
 597
 598                 const char *orig_ptr = orig_base + rel->r_offset;
 599                 char *dst_ptr = dst_base + rel->r_offset;
 600                 uint64_t va = va_base + rel->r_offset;
 601
 602                 uint64_t symbol;
 603                 uint64_t addend;
 604
 605                 if (r_sym == STN_UNDEF) {
 606                         symbol = 0;
 607                 } else {
 608                         report_elf_if(r_sym >= num_symbols);
 609
 610                         const Elf64_Sym *sym = &symbols[r_sym];
 611                         const char *symbol_name =
 612                                 elf_strptr(part->elf, strtabidx, sym->st_name);
 613                         report_elf_if(!symbol_name);
 614
 615                         if (!resolve_symbol(u, part_idx, sym, symbol_name, &symbol))
 616                                 return false;
 617                 }
 618
 619                 /* TODO: Should we also support .rela sections, where the
 620                  * addend is part of the relocation record? */
 621
 622                 /* Load the addend from the ELF instead of the destination,
 623                  * because the destination may be in VRAM. */
 624                 switch (r_type) {
 625                 case R_AMDGPU_ABS32:
 626                 case R_AMDGPU_ABS32_LO:
 627                 case R_AMDGPU_ABS32_HI:
 628                 case R_AMDGPU_REL32:
 629                 case R_AMDGPU_REL32_LO:
 630                 case R_AMDGPU_REL32_HI:
 631                         addend = *(const uint32_t *)orig_ptr;
 632                         break;
 633                 case R_AMDGPU_ABS64:
 634                 case R_AMDGPU_REL64:
 635                         addend = *(const uint64_t *)orig_ptr;
 636                         break;
 637                 default:
 638                         report_errorf("unsupported r_type == %u", r_type);
 639                         return false;
 640                 }
 641
 642                 uint64_t abs = symbol + addend;
 643
 644                 switch (r_type) {
 645                 case R_AMDGPU_ABS32:
 646                         assert((uint32_t)abs == abs);
 647                 case R_AMDGPU_ABS32_LO:
 648                         *(uint32_t *)dst_ptr = util_cpu_to_le32(abs);
 649                         break;
 650                 case R_AMDGPU_ABS32_HI:
 651                         *(uint32_t *)dst_ptr = util_cpu_to_le32(abs >> 32);
 652                         break;
 653                 case R_AMDGPU_ABS64:
 654                         *(uint64_t *)dst_ptr = util_cpu_to_le64(abs);
 655                         break;
 656                 case R_AMDGPU_REL32:
 657                         assert((int64_t)(int32_t)(abs - va) == (int64_t)(abs - va));
 658                 case R_AMDGPU_REL32_LO:
 659                         *(uint32_t *)dst_ptr = util_cpu_to_le32(abs - va);
 660                         break;
 661                 case R_AMDGPU_REL32_HI:
 662                         *(uint32_t *)dst_ptr = util_cpu_to_le32((abs - va) >> 32);
 663                         break;
 664                 case R_AMDGPU_REL64:
 665                         *(uint64_t *)dst_ptr = util_cpu_to_le64(abs - va);
 666                         break;
 667                 default:
 668                         unreachable("bad r_type");
 669                 }
 670         }
 671
 672         return true;
 673
 674 #undef report_if
 675 #undef report_elf_if
 676 }
 677
 678 /**
 679  * Upload the binary or binaries to the provided GPU buffers, including
 680  * relocations.
 681  */
 682 bool ac_rtld_upload(struct ac_rtld_upload_info *u)
 683 {
 684 #define report_if(cond) \
 685         do { \
 686                 if ((cond)) { \
 687                         report_errorf(#cond); \
 688                         return false; \
 689                 } \
 690         } while (false)
 691 #define report_elf_if(cond) \
 692         do { \
 693                 if ((cond)) { \
 694                         report_errorf(#cond); \
 695                         return false; \
 696                 } \
 697         } while (false)
 698
 699         if (u->binary->options.halt_at_entry) {
 700                 /* s_sethalt 1 */
 701                 *(uint32_t *)u->rx_ptr = util_cpu_to_le32(0xbf8d0001);
 702         }
 703
 704         /* First pass: upload raw section data and lay out private LDS symbols. */
 705         for (unsigned i = 0; i < u->binary->num_parts; ++i) {
 706                 struct ac_rtld_part *part = &u->binary->parts[i];
 707
 708                 Elf_Scn *section = NULL;
 709                 while ((section = elf_nextscn(part->elf, section))) {
 710                         Elf64_Shdr *shdr = elf64_getshdr(section);
 711                         struct ac_rtld_section *s = &part->sections[elf_ndxscn(section)];
 712
 713                         if (!s->is_rx)
 714                                 continue;
 715
 716                         report_if(shdr->sh_type != SHT_PROGBITS);
 717
 718                         Elf_Data *data = elf_getdata(section, NULL);
 719                         report_elf_if(!data || data->d_size != shdr->sh_size);
 720                         memcpy(u->rx_ptr + s->offset, data->d_buf, shdr->sh_size);
 721                 }
 722         }
 723
 724         if (u->binary->rx_end_markers) {
 725                 uint32_t *dst = (uint32_t *)(u->rx_ptr + u->binary->rx_end_markers);
 726                 for (unsigned i = 0; i < DEBUGGER_NUM_MARKERS; ++i)
 727                         *dst++ = util_cpu_to_le32(DEBUGGER_END_OF_CODE_MARKER);
 728         }
 729
 730         /* Second pass: handle relocations, overwriting uploaded data where
 731          * appropriate. */
 732         for (unsigned i = 0; i < u->binary->num_parts; ++i) {
 733                 struct ac_rtld_part *part = &u->binary->parts[i];
 734                 Elf_Scn *section = NULL;
 735                 while ((section = elf_nextscn(part->elf, section))) {
 736                         Elf64_Shdr *shdr = elf64_getshdr(section);
 737                         if (shdr->sh_type == SHT_REL) {
 738                                 Elf_Data *relocs = elf_getdata(section, NULL);
 739                                 report_elf_if(!relocs || relocs->d_size != shdr->sh_size);
 740                                 if (!apply_relocs(u, i, shdr, relocs))
 741                                         return false;
 742                         } else if (shdr->sh_type == SHT_RELA) {
 743                                 report_errorf("SHT_RELA not supported");
 744                                 return false;
 745                         }
 746                 }
 747         }
 748
 749         return true;
 750
 751 #undef report_if
 752 #undef report_elf_if
 753 }