src/freedreno/ir3/ir3_context.c

   1 /*
   2  * Copyright (C) 2015-2018 Rob Clark <robclark@freedesktop.org>
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  21  * SOFTWARE.
  22  *
  23  * Authors:
  24  *    Rob Clark <robclark@freedesktop.org>
  25  */
  26
  27 #include "ir3_compiler.h"
  28 #include "ir3_context.h"
  29 #include "ir3_image.h"
  30 #include "ir3_shader.h"
  31 #include "ir3_nir.h"
  32
  33 struct ir3_context *
  34 ir3_context_init(struct ir3_compiler *compiler,
  35                 struct ir3_shader_variant *so)
  36 {
  37         struct ir3_context *ctx = rzalloc(NULL, struct ir3_context);
  38
  39         if (compiler->gpu_id >= 400) {
  40                 if (so->type == MESA_SHADER_VERTEX) {
  41                         ctx->astc_srgb = so->key.vastc_srgb;
  42                 } else if (so->type == MESA_SHADER_FRAGMENT) {
  43                         ctx->astc_srgb = so->key.fastc_srgb;
  44                 }
  45
  46         } else {
  47                 if (so->type == MESA_SHADER_VERTEX) {
  48                         ctx->samples = so->key.vsamples;
  49                 } else if (so->type == MESA_SHADER_FRAGMENT) {
  50                         ctx->samples = so->key.fsamples;
  51                 }
  52         }
  53
  54         if (compiler->gpu_id >= 600) {
  55                 ctx->funcs = &ir3_a6xx_funcs;
  56         } else if (compiler->gpu_id >= 400) {
  57                 ctx->funcs = &ir3_a4xx_funcs;
  58         }
  59
  60         ctx->compiler = compiler;
  61         ctx->so = so;
  62         ctx->def_ht = _mesa_hash_table_create(ctx,
  63                         _mesa_hash_pointer, _mesa_key_pointer_equal);
  64         ctx->block_ht = _mesa_hash_table_create(ctx,
  65                         _mesa_hash_pointer, _mesa_key_pointer_equal);
  66
  67         /* TODO: maybe generate some sort of bitmask of what key
  68          * lowers vs what shader has (ie. no need to lower
  69          * texture clamp lowering if no texture sample instrs)..
  70          * although should be done further up the stack to avoid
  71          * creating duplicate variants..
  72          */
  73
  74         ctx->s = nir_shader_clone(ctx, so->shader->nir);
  75         if (ir3_key_lowers_nir(&so->key))
  76                 ir3_optimize_nir(so->shader, ctx->s, &so->key);
  77
  78         /* this needs to be the last pass run, so do this here instead of
  79          * in ir3_optimize_nir():
  80          */
  81         NIR_PASS_V(ctx->s, nir_lower_bool_to_int32);
  82         NIR_PASS_V(ctx->s, nir_lower_locals_to_regs);
  83
  84         /* We want to lower nir_op_imul as late as possible, to catch also
  85          * those generated by earlier passes (e.g, nir_lower_locals_to_regs).
  86          * However, we want a final swing of a few passes to have a chance
  87          * at optimizing the result.
  88          */
  89         bool progress = false;
  90         NIR_PASS(progress, ctx->s, ir3_nir_lower_imul);
  91         if (progress) {
  92                 NIR_PASS_V(ctx->s, nir_opt_algebraic);
  93                 NIR_PASS_V(ctx->s, nir_opt_copy_prop_vars);
  94                 NIR_PASS_V(ctx->s, nir_opt_dead_write_vars);
  95                 NIR_PASS_V(ctx->s, nir_opt_dce);
  96                 NIR_PASS_V(ctx->s, nir_opt_constant_folding);
  97         }
  98
  99         /* Enable the texture pre-fetch feature only a4xx onwards.  But
 100          * only enable it on generations that have been tested:
 101          */
 102         if ((so->type == MESA_SHADER_FRAGMENT) && (compiler->gpu_id >= 600))
 103                 NIR_PASS_V(ctx->s, ir3_nir_lower_tex_prefetch);
 104
 105         NIR_PASS_V(ctx->s, nir_convert_from_ssa, true);
 106
 107         if (ir3_shader_debug & IR3_DBG_DISASM) {
 108                 DBG("dump nir%dv%d: type=%d, k={cts=%u,hp=%u}",
 109                         so->shader->id, so->id, so->type,
 110                         so->key.color_two_side, so->key.half_precision);
 111                 nir_print_shader(ctx->s, stdout);
 112         }
 113
 114         if (shader_debug_enabled(so->type)) {
 115                 fprintf(stderr, "NIR (final form) for %s shader:\n",
 116                         _mesa_shader_stage_to_string(so->type));
 117                 nir_print_shader(ctx->s, stderr);
 118         }
 119
 120         ir3_ibo_mapping_init(&so->image_mapping, ctx->s->info.num_textures);
 121
 122         return ctx;
 123 }
 124
 125 void
 126 ir3_context_free(struct ir3_context *ctx)
 127 {
 128         ralloc_free(ctx);
 129 }
 130
 131 /*
 132  * Misc helpers
 133  */
 134
 135 /* allocate a n element value array (to be populated by caller) and
 136  * insert in def_ht
 137  */
 138 struct ir3_instruction **
 139 ir3_get_dst_ssa(struct ir3_context *ctx, nir_ssa_def *dst, unsigned n)
 140 {
 141         struct ir3_instruction **value =
 142                 ralloc_array(ctx->def_ht, struct ir3_instruction *, n);
 143         _mesa_hash_table_insert(ctx->def_ht, dst, value);
 144         return value;
 145 }
 146
 147 struct ir3_instruction **
 148 ir3_get_dst(struct ir3_context *ctx, nir_dest *dst, unsigned n)
 149 {
 150         struct ir3_instruction **value;
 151
 152         if (dst->is_ssa) {
 153                 value = ir3_get_dst_ssa(ctx, &dst->ssa, n);
 154         } else {
 155                 value = ralloc_array(ctx, struct ir3_instruction *, n);
 156         }
 157
 158         /* NOTE: in non-ssa case, we don't really need to store last_dst
 159          * but this helps us catch cases where put_dst() call is forgotten
 160          */
 161         compile_assert(ctx, !ctx->last_dst);
 162         ctx->last_dst = value;
 163         ctx->last_dst_n = n;
 164
 165         return value;
 166 }
 167
 168 struct ir3_instruction * const *
 169 ir3_get_src(struct ir3_context *ctx, nir_src *src)
 170 {
 171         if (src->is_ssa) {
 172                 struct hash_entry *entry;
 173                 entry = _mesa_hash_table_search(ctx->def_ht, src->ssa);
 174                 compile_assert(ctx, entry);
 175                 return entry->data;
 176         } else {
 177                 nir_register *reg = src->reg.reg;
 178                 struct ir3_array *arr = ir3_get_array(ctx, reg);
 179                 unsigned num_components = arr->r->num_components;
 180                 struct ir3_instruction *addr = NULL;
 181                 struct ir3_instruction **value =
 182                         ralloc_array(ctx, struct ir3_instruction *, num_components);
 183
 184                 if (src->reg.indirect)
 185                         addr = ir3_get_addr(ctx, ir3_get_src(ctx, src->reg.indirect)[0],
 186                                         reg->num_components);
 187
 188                 for (unsigned i = 0; i < num_components; i++) {
 189                         unsigned n = src->reg.base_offset * reg->num_components + i;
 190                         compile_assert(ctx, n < arr->length);
 191                         value[i] = ir3_create_array_load(ctx, arr, n, addr, reg->bit_size);
 192                 }
 193
 194                 return value;
 195         }
 196 }
 197
 198 void
 199 ir3_put_dst(struct ir3_context *ctx, nir_dest *dst)
 200 {
 201         unsigned bit_size = nir_dest_bit_size(*dst);
 202
 203         /* add extra mov if dst value is HIGH reg.. in some cases not all
 204          * instructions can read from HIGH regs, in cases where they can
 205          * ir3_cp will clean up the extra mov:
 206          */
 207         for (unsigned i = 0; i < ctx->last_dst_n; i++) {
 208                 if (!ctx->last_dst[i])
 209                         continue;
 210                 if (ctx->last_dst[i]->regs[0]->flags & IR3_REG_HIGH) {
 211                         ctx->last_dst[i] = ir3_MOV(ctx->block, ctx->last_dst[i], TYPE_U32);
 212                 }
 213         }
 214
 215         if (bit_size < 32) {
 216                 for (unsigned i = 0; i < ctx->last_dst_n; i++) {
 217                         struct ir3_instruction *dst = ctx->last_dst[i];
 218                         dst->regs[0]->flags |= IR3_REG_HALF;
 219                         if (ctx->last_dst[i]->opc == OPC_META_FO)
 220                                 dst->regs[1]->instr->regs[0]->flags |= IR3_REG_HALF;
 221                 }
 222         }
 223
 224         if (!dst->is_ssa) {
 225                 nir_register *reg = dst->reg.reg;
 226                 struct ir3_array *arr = ir3_get_array(ctx, reg);
 227                 unsigned num_components = ctx->last_dst_n;
 228                 struct ir3_instruction *addr = NULL;
 229
 230                 if (dst->reg.indirect)
 231                         addr = ir3_get_addr(ctx, ir3_get_src(ctx, dst->reg.indirect)[0],
 232                                         reg->num_components);
 233
 234                 for (unsigned i = 0; i < num_components; i++) {
 235                         unsigned n = dst->reg.base_offset * reg->num_components + i;
 236                         compile_assert(ctx, n < arr->length);
 237                         if (!ctx->last_dst[i])
 238                                 continue;
 239                         ir3_create_array_store(ctx, arr, n, ctx->last_dst[i], addr);
 240                 }
 241
 242                 ralloc_free(ctx->last_dst);
 243         }
 244
 245         ctx->last_dst = NULL;
 246         ctx->last_dst_n = 0;
 247 }
 248
 249 struct ir3_instruction *
 250 ir3_create_collect(struct ir3_context *ctx, struct ir3_instruction *const *arr,
 251                 unsigned arrsz)
 252 {
 253         struct ir3_block *block = ctx->block;
 254         struct ir3_instruction *collect;
 255
 256         if (arrsz == 0)
 257                 return NULL;
 258
 259         unsigned flags = arr[0]->regs[0]->flags & IR3_REG_HALF;
 260
 261         collect = ir3_instr_create2(block, OPC_META_FI, 1 + arrsz);
 262         ir3_reg_create(collect, 0, flags);     /* dst */
 263         for (unsigned i = 0; i < arrsz; i++) {
 264                 struct ir3_instruction *elem = arr[i];
 265
 266                 /* Since arrays are pre-colored in RA, we can't assume that
 267                  * things will end up in the right place.  (Ie. if a collect
 268                  * joins elements from two different arrays.)  So insert an
 269                  * extra mov.
 270                  *
 271                  * We could possibly skip this if all the collected elements
 272                  * are contiguous elements in a single array.. not sure how
 273                  * likely that is to happen.
 274                  *
 275                  * Fixes a problem with glamor shaders, that in effect do
 276                  * something like:
 277                  *
 278                  *   if (foo)
 279                  *     texcoord = ..
 280                  *   else
 281                  *     texcoord = ..
 282                  *   color = texture2D(tex, texcoord);
 283                  *
 284                  * In this case, texcoord will end up as nir registers (which
 285                  * translate to ir3 array's of length 1.  And we can't assume
 286                  * the two (or more) arrays will get allocated in consecutive
 287                  * scalar registers.
 288                  *
 289                  */
 290                 if (elem->regs[0]->flags & IR3_REG_ARRAY) {
 291                         type_t type = (flags & IR3_REG_HALF) ? TYPE_U16 : TYPE_U32;
 292                         elem = ir3_MOV(block, elem, type);
 293                 }
 294
 295                 compile_assert(ctx, (elem->regs[0]->flags & IR3_REG_HALF) == flags);
 296                 ir3_reg_create(collect, 0, IR3_REG_SSA | flags)->instr = elem;
 297         }
 298
 299         collect->regs[0]->wrmask = MASK(arrsz);
 300
 301         return collect;
 302 }
 303
 304 /* helper for instructions that produce multiple consecutive scalar
 305  * outputs which need to have a split/fanout meta instruction inserted
 306  */
 307 void
 308 ir3_split_dest(struct ir3_block *block, struct ir3_instruction **dst,
 309                 struct ir3_instruction *src, unsigned base, unsigned n)
 310 {
 311         struct ir3_instruction *prev = NULL;
 312
 313         if ((n == 1) && (src->regs[0]->wrmask == 0x1)) {
 314                 dst[0] = src;
 315                 return;
 316         }
 317
 318         unsigned flags = src->regs[0]->flags & (IR3_REG_HALF | IR3_REG_HIGH);
 319
 320         for (int i = 0, j = 0; i < n; i++) {
 321                 struct ir3_instruction *split = ir3_instr_create(block, OPC_META_FO);
 322                 ir3_reg_create(split, 0, IR3_REG_SSA | flags);
 323                 ir3_reg_create(split, 0, IR3_REG_SSA | flags)->instr = src;
 324                 split->fo.off = i + base;
 325
 326                 if (prev) {
 327                         split->cp.left = prev;
 328                         split->cp.left_cnt++;
 329                         prev->cp.right = split;
 330                         prev->cp.right_cnt++;
 331                 }
 332                 prev = split;
 333
 334                 if (src->regs[0]->wrmask & (1 << (i + base)))
 335                         dst[j++] = split;
 336         }
 337 }
 338
 339 NORETURN void
 340 ir3_context_error(struct ir3_context *ctx, const char *format, ...)
 341 {
 342         struct hash_table *errors = NULL;
 343         va_list ap;
 344         va_start(ap, format);
 345         if (ctx->cur_instr) {
 346                 errors = _mesa_hash_table_create(NULL,
 347                                 _mesa_hash_pointer,
 348                                 _mesa_key_pointer_equal);
 349                 char *msg = ralloc_vasprintf(errors, format, ap);
 350                 _mesa_hash_table_insert(errors, ctx->cur_instr, msg);
 351         } else {
 352                 _debug_vprintf(format, ap);
 353         }
 354         va_end(ap);
 355         nir_print_shader_annotated(ctx->s, stdout, errors);
 356         ralloc_free(errors);
 357         ctx->error = true;
 358         unreachable("");
 359 }
 360
 361 static struct ir3_instruction *
 362 create_addr(struct ir3_block *block, struct ir3_instruction *src, int align)
 363 {
 364         struct ir3_instruction *instr, *immed;
 365
 366         /* TODO in at least some cases, the backend could probably be
 367          * made clever enough to propagate IR3_REG_HALF..
 368          */
 369         instr = ir3_COV(block, src, TYPE_U32, TYPE_S16);
 370         instr->regs[0]->flags |= IR3_REG_HALF;
 371
 372         switch(align){
 373         case 1:
 374                 /* src *= 1: */
 375                 break;
 376         case 2:
 377                 /* src *= 2     => src <<= 1: */
 378                 immed = create_immed(block, 1);
 379                 immed->regs[0]->flags |= IR3_REG_HALF;
 380
 381                 instr = ir3_SHL_B(block, instr, 0, immed, 0);
 382                 instr->regs[0]->flags |= IR3_REG_HALF;
 383                 instr->regs[1]->flags |= IR3_REG_HALF;
 384                 break;
 385         case 3:
 386                 /* src *= 3: */
 387                 immed = create_immed(block, 3);
 388                 immed->regs[0]->flags |= IR3_REG_HALF;
 389
 390                 instr = ir3_MULL_U(block, instr, 0, immed, 0);
 391                 instr->regs[0]->flags |= IR3_REG_HALF;
 392                 instr->regs[1]->flags |= IR3_REG_HALF;
 393                 break;
 394         case 4:
 395                 /* src *= 4 => src <<= 2: */
 396                 immed = create_immed(block, 2);
 397                 immed->regs[0]->flags |= IR3_REG_HALF;
 398
 399                 instr = ir3_SHL_B(block, instr, 0, immed, 0);
 400                 instr->regs[0]->flags |= IR3_REG_HALF;
 401                 instr->regs[1]->flags |= IR3_REG_HALF;
 402                 break;
 403         default:
 404                 unreachable("bad align");
 405                 return NULL;
 406         }
 407
 408         instr = ir3_MOV(block, instr, TYPE_S16);
 409         instr->regs[0]->num = regid(REG_A0, 0);
 410         instr->regs[0]->flags |= IR3_REG_HALF;
 411         instr->regs[1]->flags |= IR3_REG_HALF;
 412
 413         return instr;
 414 }
 415
 416 /* caches addr values to avoid generating multiple cov/shl/mova
 417  * sequences for each use of a given NIR level src as address
 418  */
 419 struct ir3_instruction *
 420 ir3_get_addr(struct ir3_context *ctx, struct ir3_instruction *src, int align)
 421 {
 422         struct ir3_instruction *addr;
 423         unsigned idx = align - 1;
 424
 425         compile_assert(ctx, idx < ARRAY_SIZE(ctx->addr_ht));
 426
 427         if (!ctx->addr_ht[idx]) {
 428                 ctx->addr_ht[idx] = _mesa_hash_table_create(ctx,
 429                                 _mesa_hash_pointer, _mesa_key_pointer_equal);
 430         } else {
 431                 struct hash_entry *entry;
 432                 entry = _mesa_hash_table_search(ctx->addr_ht[idx], src);
 433                 if (entry)
 434                         return entry->data;
 435         }
 436
 437         addr = create_addr(ctx->block, src, align);
 438         _mesa_hash_table_insert(ctx->addr_ht[idx], src, addr);
 439
 440         return addr;
 441 }
 442
 443 struct ir3_instruction *
 444 ir3_get_predicate(struct ir3_context *ctx, struct ir3_instruction *src)
 445 {
 446         struct ir3_block *b = ctx->block;
 447         struct ir3_instruction *cond;
 448
 449         /* NOTE: only cmps.*.* can write p0.x: */
 450         cond = ir3_CMPS_S(b, src, 0, create_immed(b, 0), 0);
 451         cond->cat2.condition = IR3_COND_NE;
 452
 453         /* condition always goes in predicate register: */
 454         cond->regs[0]->num = regid(REG_P0, 0);
 455
 456         return cond;
 457 }
 458
 459 /*
 460  * Array helpers
 461  */
 462
 463 void
 464 ir3_declare_array(struct ir3_context *ctx, nir_register *reg)
 465 {
 466         struct ir3_array *arr = rzalloc(ctx, struct ir3_array);
 467         arr->id = ++ctx->num_arrays;
 468         /* NOTE: sometimes we get non array regs, for example for arrays of
 469          * length 1.  See fs-const-array-of-struct-of-array.shader_test.  So
 470          * treat a non-array as if it was an array of length 1.
 471          *
 472          * It would be nice if there was a nir pass to convert arrays of
 473          * length 1 to ssa.
 474          */
 475         arr->length = reg->num_components * MAX2(1, reg->num_array_elems);
 476         compile_assert(ctx, arr->length > 0);
 477         arr->r = reg;
 478         list_addtail(&arr->node, &ctx->ir->array_list);
 479 }
 480
 481 struct ir3_array *
 482 ir3_get_array(struct ir3_context *ctx, nir_register *reg)
 483 {
 484         list_for_each_entry (struct ir3_array, arr, &ctx->ir->array_list, node) {
 485                 if (arr->r == reg)
 486                         return arr;
 487         }
 488         ir3_context_error(ctx, "bogus reg: %s\n", reg->name);
 489         return NULL;
 490 }
 491
 492 /* relative (indirect) if address!=NULL */
 493 struct ir3_instruction *
 494 ir3_create_array_load(struct ir3_context *ctx, struct ir3_array *arr, int n,
 495                 struct ir3_instruction *address, unsigned bitsize)
 496 {
 497         struct ir3_block *block = ctx->block;
 498         struct ir3_instruction *mov;
 499         struct ir3_register *src;
 500         unsigned flags = 0;
 501
 502         mov = ir3_instr_create(block, OPC_MOV);
 503         if (bitsize < 32) {
 504                 mov->cat1.src_type = TYPE_U16;
 505                 mov->cat1.dst_type = TYPE_U16;
 506                 flags |= IR3_REG_HALF;
 507         } else {
 508                 mov->cat1.src_type = TYPE_U32;
 509                 mov->cat1.dst_type = TYPE_U32;
 510         }
 511
 512         mov->barrier_class = IR3_BARRIER_ARRAY_R;
 513         mov->barrier_conflict = IR3_BARRIER_ARRAY_W;
 514         ir3_reg_create(mov, 0, flags);
 515         src = ir3_reg_create(mov, 0, IR3_REG_ARRAY |
 516                         COND(address, IR3_REG_RELATIV) | flags);
 517         src->instr = arr->last_write;
 518         src->size  = arr->length;
 519         src->array.id = arr->id;
 520         src->array.offset = n;
 521
 522         if (address)
 523                 ir3_instr_set_address(mov, address);
 524
 525         return mov;
 526 }
 527
 528 /* relative (indirect) if address!=NULL */
 529 void
 530 ir3_create_array_store(struct ir3_context *ctx, struct ir3_array *arr, int n,
 531                 struct ir3_instruction *src, struct ir3_instruction *address)
 532 {
 533         struct ir3_block *block = ctx->block;
 534         struct ir3_instruction *mov;
 535         struct ir3_register *dst;
 536
 537         /* if not relative store, don't create an extra mov, since that
 538          * ends up being difficult for cp to remove.
 539          *
 540          * Also, don't skip the mov if the src is meta (like fanout/split),
 541          * since that creates a situation that RA can't really handle properly.
 542          */
 543         if (!address && !is_meta(src)) {
 544                 dst = src->regs[0];
 545
 546                 src->barrier_class |= IR3_BARRIER_ARRAY_W;
 547                 src->barrier_conflict |= IR3_BARRIER_ARRAY_R | IR3_BARRIER_ARRAY_W;
 548
 549                 dst->flags |= IR3_REG_ARRAY;
 550                 dst->instr = arr->last_write;
 551                 dst->size = arr->length;
 552                 dst->array.id = arr->id;
 553                 dst->array.offset = n;
 554
 555                 arr->last_write = src;
 556
 557                 array_insert(block, block->keeps, src);
 558
 559                 return;
 560         }
 561
 562         mov = ir3_instr_create(block, OPC_MOV);
 563         mov->cat1.src_type = TYPE_U32;
 564         mov->cat1.dst_type = TYPE_U32;
 565         mov->barrier_class = IR3_BARRIER_ARRAY_W;
 566         mov->barrier_conflict = IR3_BARRIER_ARRAY_R | IR3_BARRIER_ARRAY_W;
 567         dst = ir3_reg_create(mov, 0, IR3_REG_ARRAY |
 568                         COND(address, IR3_REG_RELATIV));
 569         dst->instr = arr->last_write;
 570         dst->size  = arr->length;
 571         dst->array.id = arr->id;
 572         dst->array.offset = n;
 573         ir3_reg_create(mov, 0, IR3_REG_SSA)->instr = src;
 574
 575         if (address)
 576                 ir3_instr_set_address(mov, address);
 577
 578         arr->last_write = mov;
 579
 580         /* the array store may only matter to something in an earlier
 581          * block (ie. loops), but since arrays are not in SSA, depth
 582          * pass won't know this.. so keep all array stores:
 583          */
 584         array_insert(block, block->keeps, mov);
 585 }