src/freedreno/ir3/ir3_context.c

   1 /*
   2  * Copyright (C) 2015-2018 Rob Clark <robclark@freedesktop.org>
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  21  * SOFTWARE.
  22  *
  23  * Authors:
  24  *    Rob Clark <robclark@freedesktop.org>
  25  */
  26
  27 #include "ir3_compiler.h"
  28 #include "ir3_context.h"
  29 #include "ir3_image.h"
  30 #include "ir3_shader.h"
  31 #include "ir3_nir.h"
  32
  33 struct ir3_context *
  34 ir3_context_init(struct ir3_compiler *compiler,
  35                 struct ir3_shader_variant *so)
  36 {
  37         struct ir3_context *ctx = rzalloc(NULL, struct ir3_context);
  38
  39         if (compiler->gpu_id >= 400) {
  40                 if (so->type == MESA_SHADER_VERTEX) {
  41                         ctx->astc_srgb = so->key.vastc_srgb;
  42                 } else if (so->type == MESA_SHADER_FRAGMENT) {
  43                         ctx->astc_srgb = so->key.fastc_srgb;
  44                 }
  45
  46         } else {
  47                 if (so->type == MESA_SHADER_VERTEX) {
  48                         ctx->samples = so->key.vsamples;
  49                 } else if (so->type == MESA_SHADER_FRAGMENT) {
  50                         ctx->samples = so->key.fsamples;
  51                 }
  52         }
  53
  54         if (compiler->gpu_id >= 600) {
  55                 ctx->funcs = &ir3_a6xx_funcs;
  56         } else if (compiler->gpu_id >= 400) {
  57                 ctx->funcs = &ir3_a4xx_funcs;
  58         }
  59
  60         ctx->compiler = compiler;
  61         ctx->so = so;
  62         ctx->def_ht = _mesa_hash_table_create(ctx,
  63                         _mesa_hash_pointer, _mesa_key_pointer_equal);
  64         ctx->block_ht = _mesa_hash_table_create(ctx,
  65                         _mesa_hash_pointer, _mesa_key_pointer_equal);
  66
  67         /* TODO: maybe generate some sort of bitmask of what key
  68          * lowers vs what shader has (ie. no need to lower
  69          * texture clamp lowering if no texture sample instrs)..
  70          * although should be done further up the stack to avoid
  71          * creating duplicate variants..
  72          */
  73
  74         ctx->s = nir_shader_clone(ctx, so->shader->nir);
  75         if (ir3_key_lowers_nir(&so->key))
  76                 ir3_optimize_nir(so->shader, ctx->s, &so->key);
  77
  78         /* this needs to be the last pass run, so do this here instead of
  79          * in ir3_optimize_nir():
  80          */
  81         NIR_PASS_V(ctx->s, nir_lower_bool_to_int32);
  82         NIR_PASS_V(ctx->s, nir_lower_locals_to_regs);
  83
  84         /* We want to lower nir_op_imul as late as possible, to catch also
  85          * those generated by earlier passes (e.g, nir_lower_locals_to_regs).
  86          * However, we want a final swing of a few passes to have a chance
  87          * at optimizing the result.
  88          */
  89         bool progress = false;
  90         NIR_PASS(progress, ctx->s, ir3_nir_lower_imul);
  91         if (progress) {
  92                 NIR_PASS_V(ctx->s, nir_opt_algebraic);
  93                 NIR_PASS_V(ctx->s, nir_opt_copy_prop_vars);
  94                 NIR_PASS_V(ctx->s, nir_opt_dead_write_vars);
  95                 NIR_PASS_V(ctx->s, nir_opt_dce);
  96                 NIR_PASS_V(ctx->s, nir_opt_constant_folding);
  97         }
  98
  99         NIR_PASS_V(ctx->s, nir_convert_from_ssa, true);
 100
 101         if (ir3_shader_debug & IR3_DBG_DISASM) {
 102                 DBG("dump nir%dv%d: type=%d, k={cts=%u,hp=%u}",
 103                         so->shader->id, so->id, so->type,
 104                         so->key.color_two_side, so->key.half_precision);
 105                 nir_print_shader(ctx->s, stdout);
 106         }
 107
 108         if (shader_debug_enabled(so->type)) {
 109                 fprintf(stderr, "NIR (final form) for %s shader:\n",
 110                         _mesa_shader_stage_to_string(so->type));
 111                 nir_print_shader(ctx->s, stderr);
 112         }
 113
 114         ir3_ibo_mapping_init(&so->image_mapping, ctx->s->info.num_textures);
 115
 116         return ctx;
 117 }
 118
 119 void
 120 ir3_context_free(struct ir3_context *ctx)
 121 {
 122         ralloc_free(ctx);
 123 }
 124
 125 /*
 126  * Misc helpers
 127  */
 128
 129 /* allocate a n element value array (to be populated by caller) and
 130  * insert in def_ht
 131  */
 132 struct ir3_instruction **
 133 ir3_get_dst_ssa(struct ir3_context *ctx, nir_ssa_def *dst, unsigned n)
 134 {
 135         struct ir3_instruction **value =
 136                 ralloc_array(ctx->def_ht, struct ir3_instruction *, n);
 137         _mesa_hash_table_insert(ctx->def_ht, dst, value);
 138         return value;
 139 }
 140
 141 struct ir3_instruction **
 142 ir3_get_dst(struct ir3_context *ctx, nir_dest *dst, unsigned n)
 143 {
 144         struct ir3_instruction **value;
 145
 146         if (dst->is_ssa) {
 147                 value = ir3_get_dst_ssa(ctx, &dst->ssa, n);
 148         } else {
 149                 value = ralloc_array(ctx, struct ir3_instruction *, n);
 150         }
 151
 152         /* NOTE: in non-ssa case, we don't really need to store last_dst
 153          * but this helps us catch cases where put_dst() call is forgotten
 154          */
 155         compile_assert(ctx, !ctx->last_dst);
 156         ctx->last_dst = value;
 157         ctx->last_dst_n = n;
 158
 159         return value;
 160 }
 161
 162 struct ir3_instruction * const *
 163 ir3_get_src(struct ir3_context *ctx, nir_src *src)
 164 {
 165         if (src->is_ssa) {
 166                 struct hash_entry *entry;
 167                 entry = _mesa_hash_table_search(ctx->def_ht, src->ssa);
 168                 compile_assert(ctx, entry);
 169                 return entry->data;
 170         } else {
 171                 nir_register *reg = src->reg.reg;
 172                 struct ir3_array *arr = ir3_get_array(ctx, reg);
 173                 unsigned num_components = arr->r->num_components;
 174                 struct ir3_instruction *addr = NULL;
 175                 struct ir3_instruction **value =
 176                         ralloc_array(ctx, struct ir3_instruction *, num_components);
 177
 178                 if (src->reg.indirect)
 179                         addr = ir3_get_addr(ctx, ir3_get_src(ctx, src->reg.indirect)[0],
 180                                         reg->num_components);
 181
 182                 for (unsigned i = 0; i < num_components; i++) {
 183                         unsigned n = src->reg.base_offset * reg->num_components + i;
 184                         compile_assert(ctx, n < arr->length);
 185                         value[i] = ir3_create_array_load(ctx, arr, n, addr, reg->bit_size);
 186                 }
 187
 188                 return value;
 189         }
 190 }
 191
 192 void
 193 ir3_put_dst(struct ir3_context *ctx, nir_dest *dst)
 194 {
 195         unsigned bit_size = nir_dest_bit_size(*dst);
 196
 197         /* add extra mov if dst value is HIGH reg.. in some cases not all
 198          * instructions can read from HIGH regs, in cases where they can
 199          * ir3_cp will clean up the extra mov:
 200          */
 201         for (unsigned i = 0; i < ctx->last_dst_n; i++) {
 202                 if (!ctx->last_dst[i])
 203                         continue;
 204                 if (ctx->last_dst[i]->regs[0]->flags & IR3_REG_HIGH) {
 205                         ctx->last_dst[i] = ir3_MOV(ctx->block, ctx->last_dst[i], TYPE_U32);
 206                 }
 207         }
 208
 209         if (bit_size < 32) {
 210                 for (unsigned i = 0; i < ctx->last_dst_n; i++) {
 211                         struct ir3_instruction *dst = ctx->last_dst[i];
 212                         dst->regs[0]->flags |= IR3_REG_HALF;
 213                         if (ctx->last_dst[i]->opc == OPC_META_FO)
 214                                 dst->regs[1]->instr->regs[0]->flags |= IR3_REG_HALF;
 215                 }
 216         }
 217
 218         if (!dst->is_ssa) {
 219                 nir_register *reg = dst->reg.reg;
 220                 struct ir3_array *arr = ir3_get_array(ctx, reg);
 221                 unsigned num_components = ctx->last_dst_n;
 222                 struct ir3_instruction *addr = NULL;
 223
 224                 if (dst->reg.indirect)
 225                         addr = ir3_get_addr(ctx, ir3_get_src(ctx, dst->reg.indirect)[0],
 226                                         reg->num_components);
 227
 228                 for (unsigned i = 0; i < num_components; i++) {
 229                         unsigned n = dst->reg.base_offset * reg->num_components + i;
 230                         compile_assert(ctx, n < arr->length);
 231                         if (!ctx->last_dst[i])
 232                                 continue;
 233                         ir3_create_array_store(ctx, arr, n, ctx->last_dst[i], addr);
 234                 }
 235
 236                 ralloc_free(ctx->last_dst);
 237         }
 238
 239         ctx->last_dst = NULL;
 240         ctx->last_dst_n = 0;
 241 }
 242
 243 struct ir3_instruction *
 244 ir3_create_collect(struct ir3_context *ctx, struct ir3_instruction *const *arr,
 245                 unsigned arrsz)
 246 {
 247         struct ir3_block *block = ctx->block;
 248         struct ir3_instruction *collect;
 249
 250         if (arrsz == 0)
 251                 return NULL;
 252
 253         unsigned flags = arr[0]->regs[0]->flags & IR3_REG_HALF;
 254
 255         collect = ir3_instr_create2(block, OPC_META_FI, 1 + arrsz);
 256         ir3_reg_create(collect, 0, flags);     /* dst */
 257         for (unsigned i = 0; i < arrsz; i++) {
 258                 struct ir3_instruction *elem = arr[i];
 259
 260                 /* Since arrays are pre-colored in RA, we can't assume that
 261                  * things will end up in the right place.  (Ie. if a collect
 262                  * joins elements from two different arrays.)  So insert an
 263                  * extra mov.
 264                  *
 265                  * We could possibly skip this if all the collected elements
 266                  * are contiguous elements in a single array.. not sure how
 267                  * likely that is to happen.
 268                  *
 269                  * Fixes a problem with glamor shaders, that in effect do
 270                  * something like:
 271                  *
 272                  *   if (foo)
 273                  *     texcoord = ..
 274                  *   else
 275                  *     texcoord = ..
 276                  *   color = texture2D(tex, texcoord);
 277                  *
 278                  * In this case, texcoord will end up as nir registers (which
 279                  * translate to ir3 array's of length 1.  And we can't assume
 280                  * the two (or more) arrays will get allocated in consecutive
 281                  * scalar registers.
 282                  *
 283                  */
 284                 if (elem->regs[0]->flags & IR3_REG_ARRAY) {
 285                         type_t type = (flags & IR3_REG_HALF) ? TYPE_U16 : TYPE_U32;
 286                         elem = ir3_MOV(block, elem, type);
 287                 }
 288
 289                 compile_assert(ctx, (elem->regs[0]->flags & IR3_REG_HALF) == flags);
 290                 ir3_reg_create(collect, 0, IR3_REG_SSA | flags)->instr = elem;
 291         }
 292
 293         collect->regs[0]->wrmask = MASK(arrsz);
 294
 295         return collect;
 296 }
 297
 298 /* helper for instructions that produce multiple consecutive scalar
 299  * outputs which need to have a split/fanout meta instruction inserted
 300  */
 301 void
 302 ir3_split_dest(struct ir3_block *block, struct ir3_instruction **dst,
 303                 struct ir3_instruction *src, unsigned base, unsigned n)
 304 {
 305         struct ir3_instruction *prev = NULL;
 306
 307         if ((n == 1) && (src->regs[0]->wrmask == 0x1)) {
 308                 dst[0] = src;
 309                 return;
 310         }
 311
 312         unsigned flags = src->regs[0]->flags & (IR3_REG_HALF | IR3_REG_HIGH);
 313
 314         for (int i = 0, j = 0; i < n; i++) {
 315                 struct ir3_instruction *split = ir3_instr_create(block, OPC_META_FO);
 316                 ir3_reg_create(split, 0, IR3_REG_SSA | flags);
 317                 ir3_reg_create(split, 0, IR3_REG_SSA | flags)->instr = src;
 318                 split->fo.off = i + base;
 319
 320                 if (prev) {
 321                         split->cp.left = prev;
 322                         split->cp.left_cnt++;
 323                         prev->cp.right = split;
 324                         prev->cp.right_cnt++;
 325                 }
 326                 prev = split;
 327
 328                 if (src->regs[0]->wrmask & (1 << (i + base)))
 329                         dst[j++] = split;
 330         }
 331 }
 332
 333 NORETURN void
 334 ir3_context_error(struct ir3_context *ctx, const char *format, ...)
 335 {
 336         struct hash_table *errors = NULL;
 337         va_list ap;
 338         va_start(ap, format);
 339         if (ctx->cur_instr) {
 340                 errors = _mesa_hash_table_create(NULL,
 341                                 _mesa_hash_pointer,
 342                                 _mesa_key_pointer_equal);
 343                 char *msg = ralloc_vasprintf(errors, format, ap);
 344                 _mesa_hash_table_insert(errors, ctx->cur_instr, msg);
 345         } else {
 346                 _debug_vprintf(format, ap);
 347         }
 348         va_end(ap);
 349         nir_print_shader_annotated(ctx->s, stdout, errors);
 350         ralloc_free(errors);
 351         ctx->error = true;
 352         unreachable("");
 353 }
 354
 355 static struct ir3_instruction *
 356 create_addr(struct ir3_block *block, struct ir3_instruction *src, int align)
 357 {
 358         struct ir3_instruction *instr, *immed;
 359
 360         /* TODO in at least some cases, the backend could probably be
 361          * made clever enough to propagate IR3_REG_HALF..
 362          */
 363         instr = ir3_COV(block, src, TYPE_U32, TYPE_S16);
 364         instr->regs[0]->flags |= IR3_REG_HALF;
 365
 366         switch(align){
 367         case 1:
 368                 /* src *= 1: */
 369                 break;
 370         case 2:
 371                 /* src *= 2     => src <<= 1: */
 372                 immed = create_immed(block, 1);
 373                 immed->regs[0]->flags |= IR3_REG_HALF;
 374
 375                 instr = ir3_SHL_B(block, instr, 0, immed, 0);
 376                 instr->regs[0]->flags |= IR3_REG_HALF;
 377                 instr->regs[1]->flags |= IR3_REG_HALF;
 378                 break;
 379         case 3:
 380                 /* src *= 3: */
 381                 immed = create_immed(block, 3);
 382                 immed->regs[0]->flags |= IR3_REG_HALF;
 383
 384                 instr = ir3_MULL_U(block, instr, 0, immed, 0);
 385                 instr->regs[0]->flags |= IR3_REG_HALF;
 386                 instr->regs[1]->flags |= IR3_REG_HALF;
 387                 break;
 388         case 4:
 389                 /* src *= 4 => src <<= 2: */
 390                 immed = create_immed(block, 2);
 391                 immed->regs[0]->flags |= IR3_REG_HALF;
 392
 393                 instr = ir3_SHL_B(block, instr, 0, immed, 0);
 394                 instr->regs[0]->flags |= IR3_REG_HALF;
 395                 instr->regs[1]->flags |= IR3_REG_HALF;
 396                 break;
 397         default:
 398                 unreachable("bad align");
 399                 return NULL;
 400         }
 401
 402         instr = ir3_MOV(block, instr, TYPE_S16);
 403         instr->regs[0]->num = regid(REG_A0, 0);
 404         instr->regs[0]->flags |= IR3_REG_HALF;
 405         instr->regs[1]->flags |= IR3_REG_HALF;
 406
 407         return instr;
 408 }
 409
 410 /* caches addr values to avoid generating multiple cov/shl/mova
 411  * sequences for each use of a given NIR level src as address
 412  */
 413 struct ir3_instruction *
 414 ir3_get_addr(struct ir3_context *ctx, struct ir3_instruction *src, int align)
 415 {
 416         struct ir3_instruction *addr;
 417         unsigned idx = align - 1;
 418
 419         compile_assert(ctx, idx < ARRAY_SIZE(ctx->addr_ht));
 420
 421         if (!ctx->addr_ht[idx]) {
 422                 ctx->addr_ht[idx] = _mesa_hash_table_create(ctx,
 423                                 _mesa_hash_pointer, _mesa_key_pointer_equal);
 424         } else {
 425                 struct hash_entry *entry;
 426                 entry = _mesa_hash_table_search(ctx->addr_ht[idx], src);
 427                 if (entry)
 428                         return entry->data;
 429         }
 430
 431         addr = create_addr(ctx->block, src, align);
 432         _mesa_hash_table_insert(ctx->addr_ht[idx], src, addr);
 433
 434         return addr;
 435 }
 436
 437 struct ir3_instruction *
 438 ir3_get_predicate(struct ir3_context *ctx, struct ir3_instruction *src)
 439 {
 440         struct ir3_block *b = ctx->block;
 441         struct ir3_instruction *cond;
 442
 443         /* NOTE: only cmps.*.* can write p0.x: */
 444         cond = ir3_CMPS_S(b, src, 0, create_immed(b, 0), 0);
 445         cond->cat2.condition = IR3_COND_NE;
 446
 447         /* condition always goes in predicate register: */
 448         cond->regs[0]->num = regid(REG_P0, 0);
 449
 450         return cond;
 451 }
 452
 453 /*
 454  * Array helpers
 455  */
 456
 457 void
 458 ir3_declare_array(struct ir3_context *ctx, nir_register *reg)
 459 {
 460         struct ir3_array *arr = rzalloc(ctx, struct ir3_array);
 461         arr->id = ++ctx->num_arrays;
 462         /* NOTE: sometimes we get non array regs, for example for arrays of
 463          * length 1.  See fs-const-array-of-struct-of-array.shader_test.  So
 464          * treat a non-array as if it was an array of length 1.
 465          *
 466          * It would be nice if there was a nir pass to convert arrays of
 467          * length 1 to ssa.
 468          */
 469         arr->length = reg->num_components * MAX2(1, reg->num_array_elems);
 470         compile_assert(ctx, arr->length > 0);
 471         arr->r = reg;
 472         list_addtail(&arr->node, &ctx->ir->array_list);
 473 }
 474
 475 struct ir3_array *
 476 ir3_get_array(struct ir3_context *ctx, nir_register *reg)
 477 {
 478         list_for_each_entry (struct ir3_array, arr, &ctx->ir->array_list, node) {
 479                 if (arr->r == reg)
 480                         return arr;
 481         }
 482         ir3_context_error(ctx, "bogus reg: %s\n", reg->name);
 483         return NULL;
 484 }
 485
 486 /* relative (indirect) if address!=NULL */
 487 struct ir3_instruction *
 488 ir3_create_array_load(struct ir3_context *ctx, struct ir3_array *arr, int n,
 489                 struct ir3_instruction *address, unsigned bitsize)
 490 {
 491         struct ir3_block *block = ctx->block;
 492         struct ir3_instruction *mov;
 493         struct ir3_register *src;
 494         unsigned flags = 0;
 495
 496         mov = ir3_instr_create(block, OPC_MOV);
 497         if (bitsize < 32) {
 498                 mov->cat1.src_type = TYPE_U16;
 499                 mov->cat1.dst_type = TYPE_U16;
 500                 flags |= IR3_REG_HALF;
 501         } else {
 502                 mov->cat1.src_type = TYPE_U32;
 503                 mov->cat1.dst_type = TYPE_U32;
 504         }
 505
 506         mov->barrier_class = IR3_BARRIER_ARRAY_R;
 507         mov->barrier_conflict = IR3_BARRIER_ARRAY_W;
 508         ir3_reg_create(mov, 0, flags);
 509         src = ir3_reg_create(mov, 0, IR3_REG_ARRAY |
 510                         COND(address, IR3_REG_RELATIV) | flags);
 511         src->instr = arr->last_write;
 512         src->size  = arr->length;
 513         src->array.id = arr->id;
 514         src->array.offset = n;
 515
 516         if (address)
 517                 ir3_instr_set_address(mov, address);
 518
 519         return mov;
 520 }
 521
 522 /* relative (indirect) if address!=NULL */
 523 void
 524 ir3_create_array_store(struct ir3_context *ctx, struct ir3_array *arr, int n,
 525                 struct ir3_instruction *src, struct ir3_instruction *address)
 526 {
 527         struct ir3_block *block = ctx->block;
 528         struct ir3_instruction *mov;
 529         struct ir3_register *dst;
 530
 531         /* if not relative store, don't create an extra mov, since that
 532          * ends up being difficult for cp to remove.
 533          */
 534         if (!address) {
 535                 dst = src->regs[0];
 536
 537                 src->barrier_class |= IR3_BARRIER_ARRAY_W;
 538                 src->barrier_conflict |= IR3_BARRIER_ARRAY_R | IR3_BARRIER_ARRAY_W;
 539
 540                 dst->flags |= IR3_REG_ARRAY;
 541                 dst->instr = arr->last_write;
 542                 dst->size = arr->length;
 543                 dst->array.id = arr->id;
 544                 dst->array.offset = n;
 545
 546                 arr->last_write = src;
 547
 548                 array_insert(block, block->keeps, src);
 549
 550                 return;
 551         }
 552
 553         mov = ir3_instr_create(block, OPC_MOV);
 554         mov->cat1.src_type = TYPE_U32;
 555         mov->cat1.dst_type = TYPE_U32;
 556         mov->barrier_class = IR3_BARRIER_ARRAY_W;
 557         mov->barrier_conflict = IR3_BARRIER_ARRAY_R | IR3_BARRIER_ARRAY_W;
 558         dst = ir3_reg_create(mov, 0, IR3_REG_ARRAY |
 559                         COND(address, IR3_REG_RELATIV));
 560         dst->instr = arr->last_write;
 561         dst->size  = arr->length;
 562         dst->array.id = arr->id;
 563         dst->array.offset = n;
 564         ir3_reg_create(mov, 0, IR3_REG_SSA)->instr = src;
 565
 566         if (address)
 567                 ir3_instr_set_address(mov, address);
 568
 569         arr->last_write = mov;
 570
 571         /* the array store may only matter to something in an earlier
 572          * block (ie. loops), but since arrays are not in SSA, depth
 573          * pass won't know this.. so keep all array stores:
 574          */
 575         array_insert(block, block->keeps, mov);
 576 }