src/freedreno/ir3/ir3_context.c

   1 /*
   2  * Copyright (C) 2015-2018 Rob Clark <robclark@freedesktop.org>
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  21  * SOFTWARE.
  22  *
  23  * Authors:
  24  *    Rob Clark <robclark@freedesktop.org>
  25  */
  26
  27 #include "ir3_compiler.h"
  28 #include "ir3_context.h"
  29 #include "ir3_image.h"
  30 #include "ir3_shader.h"
  31 #include "ir3_nir.h"
  32
  33 struct ir3_context *
  34 ir3_context_init(struct ir3_compiler *compiler,
  35                 struct ir3_shader_variant *so)
  36 {
  37         struct ir3_context *ctx = rzalloc(NULL, struct ir3_context);
  38
  39         if (compiler->gpu_id >= 400) {
  40                 if (so->type == MESA_SHADER_VERTEX) {
  41                         ctx->astc_srgb = so->key.vastc_srgb;
  42                 } else if (so->type == MESA_SHADER_FRAGMENT) {
  43                         ctx->astc_srgb = so->key.fastc_srgb;
  44                 }
  45
  46         } else {
  47                 if (so->type == MESA_SHADER_VERTEX) {
  48                         ctx->samples = so->key.vsamples;
  49                 } else if (so->type == MESA_SHADER_FRAGMENT) {
  50                         ctx->samples = so->key.fsamples;
  51                 }
  52         }
  53
  54         if (compiler->gpu_id >= 600) {
  55                 ctx->funcs = &ir3_a6xx_funcs;
  56         } else if (compiler->gpu_id >= 400) {
  57                 ctx->funcs = &ir3_a4xx_funcs;
  58         }
  59
  60         ctx->compiler = compiler;
  61         ctx->so = so;
  62         ctx->def_ht = _mesa_hash_table_create(ctx,
  63                         _mesa_hash_pointer, _mesa_key_pointer_equal);
  64         ctx->block_ht = _mesa_hash_table_create(ctx,
  65                         _mesa_hash_pointer, _mesa_key_pointer_equal);
  66
  67         /* TODO: maybe generate some sort of bitmask of what key
  68          * lowers vs what shader has (ie. no need to lower
  69          * texture clamp lowering if no texture sample instrs)..
  70          * although should be done further up the stack to avoid
  71          * creating duplicate variants..
  72          */
  73
  74         if (ir3_key_lowers_nir(&so->key)) {
  75                 nir_shader *s = nir_shader_clone(ctx, so->shader->nir);
  76                 ctx->s = ir3_optimize_nir(so->shader, s, &so->key);
  77         } else {
  78                 /* fast-path for shader key that lowers nothing in NIR: */
  79                 ctx->s = nir_shader_clone(ctx, so->shader->nir);
  80         }
  81
  82         /* this needs to be the last pass run, so do this here instead of
  83          * in ir3_optimize_nir():
  84          */
  85         NIR_PASS_V(ctx->s, nir_lower_bool_to_int32);
  86         NIR_PASS_V(ctx->s, nir_lower_locals_to_regs);
  87
  88         /* We want to lower nir_op_imul as late as possible, to catch also
  89          * those generated by earlier passes (e.g, nir_lower_locals_to_regs).
  90          * However, we want a final swing of a few passes to have a chance
  91          * at optimizing the result.
  92          */
  93         bool progress = false;
  94         NIR_PASS(progress, ctx->s, ir3_nir_lower_imul);
  95         if (progress) {
  96                 NIR_PASS_V(ctx->s, nir_opt_algebraic);
  97                 NIR_PASS_V(ctx->s, nir_opt_copy_prop_vars);
  98                 NIR_PASS_V(ctx->s, nir_opt_dead_write_vars);
  99                 NIR_PASS_V(ctx->s, nir_opt_dce);
 100                 NIR_PASS_V(ctx->s, nir_opt_constant_folding);
 101         }
 102
 103         NIR_PASS_V(ctx->s, nir_convert_from_ssa, true);
 104
 105         if (ir3_shader_debug & IR3_DBG_DISASM) {
 106                 DBG("dump nir%dv%d: type=%d, k={cts=%u,hp=%u}",
 107                         so->shader->id, so->id, so->type,
 108                         so->key.color_two_side, so->key.half_precision);
 109                 nir_print_shader(ctx->s, stdout);
 110         }
 111
 112         if (shader_debug_enabled(so->type)) {
 113                 fprintf(stderr, "NIR (final form) for %s shader:\n",
 114                         _mesa_shader_stage_to_string(so->type));
 115                 nir_print_shader(ctx->s, stderr);
 116         }
 117
 118         ir3_ibo_mapping_init(&so->image_mapping, ctx->s->info.num_textures);
 119
 120         return ctx;
 121 }
 122
 123 void
 124 ir3_context_free(struct ir3_context *ctx)
 125 {
 126         ralloc_free(ctx);
 127 }
 128
 129 /*
 130  * Misc helpers
 131  */
 132
 133 /* allocate a n element value array (to be populated by caller) and
 134  * insert in def_ht
 135  */
 136 struct ir3_instruction **
 137 ir3_get_dst_ssa(struct ir3_context *ctx, nir_ssa_def *dst, unsigned n)
 138 {
 139         struct ir3_instruction **value =
 140                 ralloc_array(ctx->def_ht, struct ir3_instruction *, n);
 141         _mesa_hash_table_insert(ctx->def_ht, dst, value);
 142         return value;
 143 }
 144
 145 struct ir3_instruction **
 146 ir3_get_dst(struct ir3_context *ctx, nir_dest *dst, unsigned n)
 147 {
 148         struct ir3_instruction **value;
 149
 150         if (dst->is_ssa) {
 151                 value = ir3_get_dst_ssa(ctx, &dst->ssa, n);
 152         } else {
 153                 value = ralloc_array(ctx, struct ir3_instruction *, n);
 154         }
 155
 156         /* NOTE: in non-ssa case, we don't really need to store last_dst
 157          * but this helps us catch cases where put_dst() call is forgotten
 158          */
 159         compile_assert(ctx, !ctx->last_dst);
 160         ctx->last_dst = value;
 161         ctx->last_dst_n = n;
 162
 163         return value;
 164 }
 165
 166 struct ir3_instruction * const *
 167 ir3_get_src(struct ir3_context *ctx, nir_src *src)
 168 {
 169         if (src->is_ssa) {
 170                 struct hash_entry *entry;
 171                 entry = _mesa_hash_table_search(ctx->def_ht, src->ssa);
 172                 compile_assert(ctx, entry);
 173                 return entry->data;
 174         } else {
 175                 nir_register *reg = src->reg.reg;
 176                 struct ir3_array *arr = ir3_get_array(ctx, reg);
 177                 unsigned num_components = arr->r->num_components;
 178                 struct ir3_instruction *addr = NULL;
 179                 struct ir3_instruction **value =
 180                         ralloc_array(ctx, struct ir3_instruction *, num_components);
 181
 182                 if (src->reg.indirect)
 183                         addr = ir3_get_addr(ctx, ir3_get_src(ctx, src->reg.indirect)[0],
 184                                         reg->num_components);
 185
 186                 for (unsigned i = 0; i < num_components; i++) {
 187                         unsigned n = src->reg.base_offset * reg->num_components + i;
 188                         compile_assert(ctx, n < arr->length);
 189                         value[i] = ir3_create_array_load(ctx, arr, n, addr, reg->bit_size);
 190                 }
 191
 192                 return value;
 193         }
 194 }
 195
 196 void
 197 ir3_put_dst(struct ir3_context *ctx, nir_dest *dst)
 198 {
 199         unsigned bit_size = nir_dest_bit_size(*dst);
 200
 201         /* add extra mov if dst value is HIGH reg.. in some cases not all
 202          * instructions can read from HIGH regs, in cases where they can
 203          * ir3_cp will clean up the extra mov:
 204          */
 205         for (unsigned i = 0; i < ctx->last_dst_n; i++) {
 206                 if (!ctx->last_dst[i])
 207                         continue;
 208                 if (ctx->last_dst[i]->regs[0]->flags & IR3_REG_HIGH) {
 209                         ctx->last_dst[i] = ir3_MOV(ctx->block, ctx->last_dst[i], TYPE_U32);
 210                 }
 211         }
 212
 213         if (bit_size < 32) {
 214                 for (unsigned i = 0; i < ctx->last_dst_n; i++) {
 215                         struct ir3_instruction *dst = ctx->last_dst[i];
 216                         dst->regs[0]->flags |= IR3_REG_HALF;
 217                         if (ctx->last_dst[i]->opc == OPC_META_FO)
 218                                 dst->regs[1]->instr->regs[0]->flags |= IR3_REG_HALF;
 219                 }
 220         }
 221
 222         if (!dst->is_ssa) {
 223                 nir_register *reg = dst->reg.reg;
 224                 struct ir3_array *arr = ir3_get_array(ctx, reg);
 225                 unsigned num_components = ctx->last_dst_n;
 226                 struct ir3_instruction *addr = NULL;
 227
 228                 if (dst->reg.indirect)
 229                         addr = ir3_get_addr(ctx, ir3_get_src(ctx, dst->reg.indirect)[0],
 230                                         reg->num_components);
 231
 232                 for (unsigned i = 0; i < num_components; i++) {
 233                         unsigned n = dst->reg.base_offset * reg->num_components + i;
 234                         compile_assert(ctx, n < arr->length);
 235                         if (!ctx->last_dst[i])
 236                                 continue;
 237                         ir3_create_array_store(ctx, arr, n, ctx->last_dst[i], addr);
 238                 }
 239
 240                 ralloc_free(ctx->last_dst);
 241         }
 242
 243         ctx->last_dst = NULL;
 244         ctx->last_dst_n = 0;
 245 }
 246
 247 struct ir3_instruction *
 248 ir3_create_collect(struct ir3_context *ctx, struct ir3_instruction *const *arr,
 249                 unsigned arrsz)
 250 {
 251         struct ir3_block *block = ctx->block;
 252         struct ir3_instruction *collect;
 253
 254         if (arrsz == 0)
 255                 return NULL;
 256
 257         unsigned flags = arr[0]->regs[0]->flags & IR3_REG_HALF;
 258
 259         collect = ir3_instr_create2(block, OPC_META_FI, 1 + arrsz);
 260         ir3_reg_create(collect, 0, flags);     /* dst */
 261         for (unsigned i = 0; i < arrsz; i++) {
 262                 struct ir3_instruction *elem = arr[i];
 263
 264                 /* Since arrays are pre-colored in RA, we can't assume that
 265                  * things will end up in the right place.  (Ie. if a collect
 266                  * joins elements from two different arrays.)  So insert an
 267                  * extra mov.
 268                  *
 269                  * We could possibly skip this if all the collected elements
 270                  * are contiguous elements in a single array.. not sure how
 271                  * likely that is to happen.
 272                  *
 273                  * Fixes a problem with glamor shaders, that in effect do
 274                  * something like:
 275                  *
 276                  *   if (foo)
 277                  *     texcoord = ..
 278                  *   else
 279                  *     texcoord = ..
 280                  *   color = texture2D(tex, texcoord);
 281                  *
 282                  * In this case, texcoord will end up as nir registers (which
 283                  * translate to ir3 array's of length 1.  And we can't assume
 284                  * the two (or more) arrays will get allocated in consecutive
 285                  * scalar registers.
 286                  *
 287                  */
 288                 if (elem->regs[0]->flags & IR3_REG_ARRAY) {
 289                         type_t type = (flags & IR3_REG_HALF) ? TYPE_U16 : TYPE_U32;
 290                         elem = ir3_MOV(block, elem, type);
 291                 }
 292
 293                 compile_assert(ctx, (elem->regs[0]->flags & IR3_REG_HALF) == flags);
 294                 ir3_reg_create(collect, 0, IR3_REG_SSA | flags)->instr = elem;
 295         }
 296
 297         collect->regs[0]->wrmask = MASK(arrsz);
 298
 299         return collect;
 300 }
 301
 302 /* helper for instructions that produce multiple consecutive scalar
 303  * outputs which need to have a split/fanout meta instruction inserted
 304  */
 305 void
 306 ir3_split_dest(struct ir3_block *block, struct ir3_instruction **dst,
 307                 struct ir3_instruction *src, unsigned base, unsigned n)
 308 {
 309         struct ir3_instruction *prev = NULL;
 310
 311         if ((n == 1) && (src->regs[0]->wrmask == 0x1)) {
 312                 dst[0] = src;
 313                 return;
 314         }
 315
 316         unsigned flags = src->regs[0]->flags & (IR3_REG_HALF | IR3_REG_HIGH);
 317
 318         for (int i = 0, j = 0; i < n; i++) {
 319                 struct ir3_instruction *split = ir3_instr_create(block, OPC_META_FO);
 320                 ir3_reg_create(split, 0, IR3_REG_SSA | flags);
 321                 ir3_reg_create(split, 0, IR3_REG_SSA | flags)->instr = src;
 322                 split->fo.off = i + base;
 323
 324                 if (prev) {
 325                         split->cp.left = prev;
 326                         split->cp.left_cnt++;
 327                         prev->cp.right = split;
 328                         prev->cp.right_cnt++;
 329                 }
 330                 prev = split;
 331
 332                 if (src->regs[0]->wrmask & (1 << (i + base)))
 333                         dst[j++] = split;
 334         }
 335 }
 336
 337 NORETURN void
 338 ir3_context_error(struct ir3_context *ctx, const char *format, ...)
 339 {
 340         struct hash_table *errors = NULL;
 341         va_list ap;
 342         va_start(ap, format);
 343         if (ctx->cur_instr) {
 344                 errors = _mesa_hash_table_create(NULL,
 345                                 _mesa_hash_pointer,
 346                                 _mesa_key_pointer_equal);
 347                 char *msg = ralloc_vasprintf(errors, format, ap);
 348                 _mesa_hash_table_insert(errors, ctx->cur_instr, msg);
 349         } else {
 350                 _debug_vprintf(format, ap);
 351         }
 352         va_end(ap);
 353         nir_print_shader_annotated(ctx->s, stdout, errors);
 354         ralloc_free(errors);
 355         ctx->error = true;
 356         unreachable("");
 357 }
 358
 359 static struct ir3_instruction *
 360 create_addr(struct ir3_block *block, struct ir3_instruction *src, int align)
 361 {
 362         struct ir3_instruction *instr, *immed;
 363
 364         /* TODO in at least some cases, the backend could probably be
 365          * made clever enough to propagate IR3_REG_HALF..
 366          */
 367         instr = ir3_COV(block, src, TYPE_U32, TYPE_S16);
 368         instr->regs[0]->flags |= IR3_REG_HALF;
 369
 370         switch(align){
 371         case 1:
 372                 /* src *= 1: */
 373                 break;
 374         case 2:
 375                 /* src *= 2     => src <<= 1: */
 376                 immed = create_immed(block, 1);
 377                 immed->regs[0]->flags |= IR3_REG_HALF;
 378
 379                 instr = ir3_SHL_B(block, instr, 0, immed, 0);
 380                 instr->regs[0]->flags |= IR3_REG_HALF;
 381                 instr->regs[1]->flags |= IR3_REG_HALF;
 382                 break;
 383         case 3:
 384                 /* src *= 3: */
 385                 immed = create_immed(block, 3);
 386                 immed->regs[0]->flags |= IR3_REG_HALF;
 387
 388                 instr = ir3_MULL_U(block, instr, 0, immed, 0);
 389                 instr->regs[0]->flags |= IR3_REG_HALF;
 390                 instr->regs[1]->flags |= IR3_REG_HALF;
 391                 break;
 392         case 4:
 393                 /* src *= 4 => src <<= 2: */
 394                 immed = create_immed(block, 2);
 395                 immed->regs[0]->flags |= IR3_REG_HALF;
 396
 397                 instr = ir3_SHL_B(block, instr, 0, immed, 0);
 398                 instr->regs[0]->flags |= IR3_REG_HALF;
 399                 instr->regs[1]->flags |= IR3_REG_HALF;
 400                 break;
 401         default:
 402                 unreachable("bad align");
 403                 return NULL;
 404         }
 405
 406         instr = ir3_MOV(block, instr, TYPE_S16);
 407         instr->regs[0]->num = regid(REG_A0, 0);
 408         instr->regs[0]->flags |= IR3_REG_HALF;
 409         instr->regs[1]->flags |= IR3_REG_HALF;
 410
 411         return instr;
 412 }
 413
 414 /* caches addr values to avoid generating multiple cov/shl/mova
 415  * sequences for each use of a given NIR level src as address
 416  */
 417 struct ir3_instruction *
 418 ir3_get_addr(struct ir3_context *ctx, struct ir3_instruction *src, int align)
 419 {
 420         struct ir3_instruction *addr;
 421         unsigned idx = align - 1;
 422
 423         compile_assert(ctx, idx < ARRAY_SIZE(ctx->addr_ht));
 424
 425         if (!ctx->addr_ht[idx]) {
 426                 ctx->addr_ht[idx] = _mesa_hash_table_create(ctx,
 427                                 _mesa_hash_pointer, _mesa_key_pointer_equal);
 428         } else {
 429                 struct hash_entry *entry;
 430                 entry = _mesa_hash_table_search(ctx->addr_ht[idx], src);
 431                 if (entry)
 432                         return entry->data;
 433         }
 434
 435         addr = create_addr(ctx->block, src, align);
 436         _mesa_hash_table_insert(ctx->addr_ht[idx], src, addr);
 437
 438         return addr;
 439 }
 440
 441 struct ir3_instruction *
 442 ir3_get_predicate(struct ir3_context *ctx, struct ir3_instruction *src)
 443 {
 444         struct ir3_block *b = ctx->block;
 445         struct ir3_instruction *cond;
 446
 447         /* NOTE: only cmps.*.* can write p0.x: */
 448         cond = ir3_CMPS_S(b, src, 0, create_immed(b, 0), 0);
 449         cond->cat2.condition = IR3_COND_NE;
 450
 451         /* condition always goes in predicate register: */
 452         cond->regs[0]->num = regid(REG_P0, 0);
 453
 454         return cond;
 455 }
 456
 457 /*
 458  * Array helpers
 459  */
 460
 461 void
 462 ir3_declare_array(struct ir3_context *ctx, nir_register *reg)
 463 {
 464         struct ir3_array *arr = rzalloc(ctx, struct ir3_array);
 465         arr->id = ++ctx->num_arrays;
 466         /* NOTE: sometimes we get non array regs, for example for arrays of
 467          * length 1.  See fs-const-array-of-struct-of-array.shader_test.  So
 468          * treat a non-array as if it was an array of length 1.
 469          *
 470          * It would be nice if there was a nir pass to convert arrays of
 471          * length 1 to ssa.
 472          */
 473         arr->length = reg->num_components * MAX2(1, reg->num_array_elems);
 474         compile_assert(ctx, arr->length > 0);
 475         arr->r = reg;
 476         list_addtail(&arr->node, &ctx->ir->array_list);
 477 }
 478
 479 struct ir3_array *
 480 ir3_get_array(struct ir3_context *ctx, nir_register *reg)
 481 {
 482         list_for_each_entry (struct ir3_array, arr, &ctx->ir->array_list, node) {
 483                 if (arr->r == reg)
 484                         return arr;
 485         }
 486         ir3_context_error(ctx, "bogus reg: %s\n", reg->name);
 487         return NULL;
 488 }
 489
 490 /* relative (indirect) if address!=NULL */
 491 struct ir3_instruction *
 492 ir3_create_array_load(struct ir3_context *ctx, struct ir3_array *arr, int n,
 493                 struct ir3_instruction *address, unsigned bitsize)
 494 {
 495         struct ir3_block *block = ctx->block;
 496         struct ir3_instruction *mov;
 497         struct ir3_register *src;
 498         unsigned flags = 0;
 499
 500         mov = ir3_instr_create(block, OPC_MOV);
 501         if (bitsize < 32) {
 502                 mov->cat1.src_type = TYPE_U16;
 503                 mov->cat1.dst_type = TYPE_U16;
 504                 flags |= IR3_REG_HALF;
 505         } else {
 506                 mov->cat1.src_type = TYPE_U32;
 507                 mov->cat1.dst_type = TYPE_U32;
 508         }
 509
 510         mov->barrier_class = IR3_BARRIER_ARRAY_R;
 511         mov->barrier_conflict = IR3_BARRIER_ARRAY_W;
 512         ir3_reg_create(mov, 0, flags);
 513         src = ir3_reg_create(mov, 0, IR3_REG_ARRAY |
 514                         COND(address, IR3_REG_RELATIV) | flags);
 515         src->instr = arr->last_write;
 516         src->size  = arr->length;
 517         src->array.id = arr->id;
 518         src->array.offset = n;
 519
 520         if (address)
 521                 ir3_instr_set_address(mov, address);
 522
 523         return mov;
 524 }
 525
 526 /* relative (indirect) if address!=NULL */
 527 void
 528 ir3_create_array_store(struct ir3_context *ctx, struct ir3_array *arr, int n,
 529                 struct ir3_instruction *src, struct ir3_instruction *address)
 530 {
 531         struct ir3_block *block = ctx->block;
 532         struct ir3_instruction *mov;
 533         struct ir3_register *dst;
 534
 535         /* if not relative store, don't create an extra mov, since that
 536          * ends up being difficult for cp to remove.
 537          */
 538         if (!address) {
 539                 dst = src->regs[0];
 540
 541                 src->barrier_class |= IR3_BARRIER_ARRAY_W;
 542                 src->barrier_conflict |= IR3_BARRIER_ARRAY_R | IR3_BARRIER_ARRAY_W;
 543
 544                 dst->flags |= IR3_REG_ARRAY;
 545                 dst->instr = arr->last_write;
 546                 dst->size = arr->length;
 547                 dst->array.id = arr->id;
 548                 dst->array.offset = n;
 549
 550                 arr->last_write = src;
 551
 552                 array_insert(block, block->keeps, src);
 553
 554                 return;
 555         }
 556
 557         mov = ir3_instr_create(block, OPC_MOV);
 558         mov->cat1.src_type = TYPE_U32;
 559         mov->cat1.dst_type = TYPE_U32;
 560         mov->barrier_class = IR3_BARRIER_ARRAY_W;
 561         mov->barrier_conflict = IR3_BARRIER_ARRAY_R | IR3_BARRIER_ARRAY_W;
 562         dst = ir3_reg_create(mov, 0, IR3_REG_ARRAY |
 563                         COND(address, IR3_REG_RELATIV));
 564         dst->instr = arr->last_write;
 565         dst->size  = arr->length;
 566         dst->array.id = arr->id;
 567         dst->array.offset = n;
 568         ir3_reg_create(mov, 0, IR3_REG_SSA)->instr = src;
 569
 570         if (address)
 571                 ir3_instr_set_address(mov, address);
 572
 573         arr->last_write = mov;
 574
 575         /* the array store may only matter to something in an earlier
 576          * block (ie. loops), but since arrays are not in SSA, depth
 577          * pass won't know this.. so keep all array stores:
 578          */
 579         array_insert(block, block->keeps, mov);
 580 }