src/freedreno/ir3/ir3_context.c

   1 /*
   2  * Copyright (C) 2015-2018 Rob Clark <robclark@freedesktop.org>
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  21  * SOFTWARE.
  22  *
  23  * Authors:
  24  *    Rob Clark <robclark@freedesktop.org>
  25  */
  26
  27 #include "util/u_math.h"
  28
  29 #include "ir3_compiler.h"
  30 #include "ir3_context.h"
  31 #include "ir3_shader.h"
  32 #include "ir3_nir.h"
  33
  34 struct ir3_context *
  35 ir3_context_init(struct ir3_compiler *compiler,
  36                 struct ir3_shader_variant *so)
  37 {
  38         struct ir3_context *ctx = rzalloc(NULL, struct ir3_context);
  39
  40         if (compiler->gpu_id >= 400) {
  41                 if (so->type == MESA_SHADER_VERTEX) {
  42                         ctx->astc_srgb = so->key.vastc_srgb;
  43                 } else if (so->type == MESA_SHADER_FRAGMENT) {
  44                         ctx->astc_srgb = so->key.fastc_srgb;
  45                 }
  46
  47         } else {
  48                 if (so->type == MESA_SHADER_VERTEX) {
  49                         ctx->samples = so->key.vsamples;
  50                 } else if (so->type == MESA_SHADER_FRAGMENT) {
  51                         ctx->samples = so->key.fsamples;
  52                 }
  53         }
  54
  55         ctx->compiler = compiler;
  56         ctx->so = so;
  57         ctx->def_ht = _mesa_hash_table_create(ctx,
  58                         _mesa_hash_pointer, _mesa_key_pointer_equal);
  59         ctx->block_ht = _mesa_hash_table_create(ctx,
  60                         _mesa_hash_pointer, _mesa_key_pointer_equal);
  61
  62         /* TODO: maybe generate some sort of bitmask of what key
  63          * lowers vs what shader has (ie. no need to lower
  64          * texture clamp lowering if no texture sample instrs)..
  65          * although should be done further up the stack to avoid
  66          * creating duplicate variants..
  67          */
  68
  69         if (ir3_key_lowers_nir(&so->key)) {
  70                 nir_shader *s = nir_shader_clone(ctx, so->shader->nir);
  71                 ctx->s = ir3_optimize_nir(so->shader, s, &so->key);
  72         } else {
  73                 /* fast-path for shader key that lowers nothing in NIR: */
  74                 ctx->s = so->shader->nir;
  75         }
  76
  77         /* this needs to be the last pass run, so do this here instead of
  78          * in ir3_optimize_nir():
  79          */
  80         NIR_PASS_V(ctx->s, nir_lower_locals_to_regs);
  81         NIR_PASS_V(ctx->s, nir_convert_from_ssa, true);
  82
  83         if (ir3_shader_debug & IR3_DBG_DISASM) {
  84                 DBG("dump nir%dv%d: type=%d, k={cts=%u,hp=%u}",
  85                         so->shader->id, so->id, so->type,
  86                         so->key.color_two_side, so->key.half_precision);
  87                 nir_print_shader(ctx->s, stdout);
  88         }
  89
  90         if (shader_debug_enabled(so->type)) {
  91                 fprintf(stderr, "NIR (final form) for %s shader:\n",
  92                         _mesa_shader_stage_to_string(so->type));
  93                 nir_print_shader(ctx->s, stderr);
  94         }
  95
  96         ir3_nir_scan_driver_consts(ctx->s, &so->const_layout);
  97
  98         so->num_uniforms = ctx->s->num_uniforms;
  99         so->num_ubos = ctx->s->info.num_ubos;
 100
 101         /* Layout of constant registers, each section aligned to vec4.  Note
 102          * that pointer size (ubo, etc) changes depending on generation.
 103          *
 104          *    user consts
 105          *    UBO addresses
 106          *    SSBO sizes
 107          *    if (vertex shader) {
 108          *        driver params (IR3_DP_*)
 109          *        if (stream_output.num_outputs > 0)
 110          *           stream-out addresses
 111          *    }
 112          *    immediates
 113          *
 114          * Immediates go last mostly because they are inserted in the CP pass
 115          * after the nir -> ir3 frontend.
 116          */
 117         unsigned constoff = align(ctx->s->num_uniforms, 4);
 118         unsigned ptrsz = ir3_pointer_size(ctx);
 119
 120         memset(&so->constbase, ~0, sizeof(so->constbase));
 121
 122         if (so->num_ubos > 0) {
 123                 so->constbase.ubo = constoff;
 124                 constoff += align(ctx->s->info.num_ubos * ptrsz, 4) / 4;
 125         }
 126
 127         if (so->const_layout.ssbo_size.count > 0) {
 128                 unsigned cnt = so->const_layout.ssbo_size.count;
 129                 so->constbase.ssbo_sizes = constoff;
 130                 constoff += align(cnt, 4) / 4;
 131         }
 132
 133         if (so->const_layout.image_dims.count > 0) {
 134                 unsigned cnt = so->const_layout.image_dims.count;
 135                 so->constbase.image_dims = constoff;
 136                 constoff += align(cnt, 4) / 4;
 137         }
 138
 139         unsigned num_driver_params = 0;
 140         if (so->type == MESA_SHADER_VERTEX) {
 141                 num_driver_params = IR3_DP_VS_COUNT;
 142         } else if (so->type == MESA_SHADER_COMPUTE) {
 143                 num_driver_params = IR3_DP_CS_COUNT;
 144         }
 145
 146         so->constbase.driver_param = constoff;
 147         constoff += align(num_driver_params, 4) / 4;
 148
 149         if ((so->type == MESA_SHADER_VERTEX) &&
 150                         (compiler->gpu_id < 500) &&
 151                         so->shader->stream_output.num_outputs > 0) {
 152                 so->constbase.tfbo = constoff;
 153                 constoff += align(IR3_MAX_SO_BUFFERS * ptrsz, 4) / 4;
 154         }
 155
 156         so->constbase.immediate = constoff;
 157
 158         return ctx;
 159 }
 160
 161 void
 162 ir3_context_free(struct ir3_context *ctx)
 163 {
 164         ralloc_free(ctx);
 165 }
 166
 167 /*
 168  * Misc helpers
 169  */
 170
 171 /* allocate a n element value array (to be populated by caller) and
 172  * insert in def_ht
 173  */
 174 struct ir3_instruction **
 175 ir3_get_dst_ssa(struct ir3_context *ctx, nir_ssa_def *dst, unsigned n)
 176 {
 177         struct ir3_instruction **value =
 178                 ralloc_array(ctx->def_ht, struct ir3_instruction *, n);
 179         _mesa_hash_table_insert(ctx->def_ht, dst, value);
 180         return value;
 181 }
 182
 183 struct ir3_instruction **
 184 ir3_get_dst(struct ir3_context *ctx, nir_dest *dst, unsigned n)
 185 {
 186         struct ir3_instruction **value;
 187
 188         if (dst->is_ssa) {
 189                 value = ir3_get_dst_ssa(ctx, &dst->ssa, n);
 190         } else {
 191                 value = ralloc_array(ctx, struct ir3_instruction *, n);
 192         }
 193
 194         /* NOTE: in non-ssa case, we don't really need to store last_dst
 195          * but this helps us catch cases where put_dst() call is forgotten
 196          */
 197         compile_assert(ctx, !ctx->last_dst);
 198         ctx->last_dst = value;
 199         ctx->last_dst_n = n;
 200
 201         return value;
 202 }
 203
 204 struct ir3_instruction * const *
 205 ir3_get_src(struct ir3_context *ctx, nir_src *src)
 206 {
 207         if (src->is_ssa) {
 208                 struct hash_entry *entry;
 209                 entry = _mesa_hash_table_search(ctx->def_ht, src->ssa);
 210                 compile_assert(ctx, entry);
 211                 return entry->data;
 212         } else {
 213                 nir_register *reg = src->reg.reg;
 214                 struct ir3_array *arr = ir3_get_array(ctx, reg);
 215                 unsigned num_components = arr->r->num_components;
 216                 struct ir3_instruction *addr = NULL;
 217                 struct ir3_instruction **value =
 218                         ralloc_array(ctx, struct ir3_instruction *, num_components);
 219
 220                 if (src->reg.indirect)
 221                         addr = ir3_get_addr(ctx, ir3_get_src(ctx, src->reg.indirect)[0],
 222                                         reg->num_components);
 223
 224                 for (unsigned i = 0; i < num_components; i++) {
 225                         unsigned n = src->reg.base_offset * reg->num_components + i;
 226                         compile_assert(ctx, n < arr->length);
 227                         value[i] = ir3_create_array_load(ctx, arr, n, addr);
 228                 }
 229
 230                 return value;
 231         }
 232 }
 233
 234 void
 235 put_dst(struct ir3_context *ctx, nir_dest *dst)
 236 {
 237         unsigned bit_size = nir_dest_bit_size(*dst);
 238
 239         if (bit_size < 32) {
 240                 for (unsigned i = 0; i < ctx->last_dst_n; i++) {
 241                         struct ir3_instruction *dst = ctx->last_dst[i];
 242                         dst->regs[0]->flags |= IR3_REG_HALF;
 243                         if (ctx->last_dst[i]->opc == OPC_META_FO)
 244                                 dst->regs[1]->instr->regs[0]->flags |= IR3_REG_HALF;
 245                 }
 246         }
 247
 248         if (!dst->is_ssa) {
 249                 nir_register *reg = dst->reg.reg;
 250                 struct ir3_array *arr = ir3_get_array(ctx, reg);
 251                 unsigned num_components = ctx->last_dst_n;
 252                 struct ir3_instruction *addr = NULL;
 253
 254                 if (dst->reg.indirect)
 255                         addr = ir3_get_addr(ctx, ir3_get_src(ctx, dst->reg.indirect)[0],
 256                                         reg->num_components);
 257
 258                 for (unsigned i = 0; i < num_components; i++) {
 259                         unsigned n = dst->reg.base_offset * reg->num_components + i;
 260                         compile_assert(ctx, n < arr->length);
 261                         if (!ctx->last_dst[i])
 262                                 continue;
 263                         ir3_create_array_store(ctx, arr, n, ctx->last_dst[i], addr);
 264                 }
 265
 266                 ralloc_free(ctx->last_dst);
 267         }
 268         ctx->last_dst = NULL;
 269         ctx->last_dst_n = 0;
 270 }
 271
 272 struct ir3_instruction *
 273 ir3_create_collect(struct ir3_context *ctx, struct ir3_instruction *const *arr,
 274                 unsigned arrsz)
 275 {
 276         struct ir3_block *block = ctx->block;
 277         struct ir3_instruction *collect;
 278
 279         if (arrsz == 0)
 280                 return NULL;
 281
 282         unsigned flags = arr[0]->regs[0]->flags & IR3_REG_HALF;
 283
 284         collect = ir3_instr_create2(block, OPC_META_FI, 1 + arrsz);
 285         ir3_reg_create(collect, 0, flags);     /* dst */
 286         for (unsigned i = 0; i < arrsz; i++) {
 287                 struct ir3_instruction *elem = arr[i];
 288
 289                 /* Since arrays are pre-colored in RA, we can't assume that
 290                  * things will end up in the right place.  (Ie. if a collect
 291                  * joins elements from two different arrays.)  So insert an
 292                  * extra mov.
 293                  *
 294                  * We could possibly skip this if all the collected elements
 295                  * are contiguous elements in a single array.. not sure how
 296                  * likely that is to happen.
 297                  *
 298                  * Fixes a problem with glamor shaders, that in effect do
 299                  * something like:
 300                  *
 301                  *   if (foo)
 302                  *     texcoord = ..
 303                  *   else
 304                  *     texcoord = ..
 305                  *   color = texture2D(tex, texcoord);
 306                  *
 307                  * In this case, texcoord will end up as nir registers (which
 308                  * translate to ir3 array's of length 1.  And we can't assume
 309                  * the two (or more) arrays will get allocated in consecutive
 310                  * scalar registers.
 311                  *
 312                  */
 313                 if (elem->regs[0]->flags & IR3_REG_ARRAY) {
 314                         type_t type = (flags & IR3_REG_HALF) ? TYPE_U16 : TYPE_U32;
 315                         elem = ir3_MOV(block, elem, type);
 316                 }
 317
 318                 compile_assert(ctx, (elem->regs[0]->flags & IR3_REG_HALF) == flags);
 319                 ir3_reg_create(collect, 0, IR3_REG_SSA | flags)->instr = elem;
 320         }
 321
 322         return collect;
 323 }
 324
 325 /* helper for instructions that produce multiple consecutive scalar
 326  * outputs which need to have a split/fanout meta instruction inserted
 327  */
 328 void
 329 ir3_split_dest(struct ir3_block *block, struct ir3_instruction **dst,
 330                 struct ir3_instruction *src, unsigned base, unsigned n)
 331 {
 332         struct ir3_instruction *prev = NULL;
 333
 334         if ((n == 1) && (src->regs[0]->wrmask == 0x1)) {
 335                 dst[0] = src;
 336                 return;
 337         }
 338
 339         for (int i = 0, j = 0; i < n; i++) {
 340                 struct ir3_instruction *split = ir3_instr_create(block, OPC_META_FO);
 341                 ir3_reg_create(split, 0, IR3_REG_SSA);
 342                 ir3_reg_create(split, 0, IR3_REG_SSA)->instr = src;
 343                 split->fo.off = i + base;
 344
 345                 if (prev) {
 346                         split->cp.left = prev;
 347                         split->cp.left_cnt++;
 348                         prev->cp.right = split;
 349                         prev->cp.right_cnt++;
 350                 }
 351                 prev = split;
 352
 353                 if (src->regs[0]->wrmask & (1 << (i + base)))
 354                         dst[j++] = split;
 355         }
 356 }
 357
 358 void
 359 ir3_context_error(struct ir3_context *ctx, const char *format, ...)
 360 {
 361         struct hash_table *errors = NULL;
 362         va_list ap;
 363         va_start(ap, format);
 364         if (ctx->cur_instr) {
 365                 errors = _mesa_hash_table_create(NULL,
 366                                 _mesa_hash_pointer,
 367                                 _mesa_key_pointer_equal);
 368                 char *msg = ralloc_vasprintf(errors, format, ap);
 369                 _mesa_hash_table_insert(errors, ctx->cur_instr, msg);
 370         } else {
 371                 _debug_vprintf(format, ap);
 372         }
 373         va_end(ap);
 374         nir_print_shader_annotated(ctx->s, stdout, errors);
 375         ralloc_free(errors);
 376         ctx->error = true;
 377         debug_assert(0);
 378 }
 379
 380 static struct ir3_instruction *
 381 create_addr(struct ir3_block *block, struct ir3_instruction *src, int align)
 382 {
 383         struct ir3_instruction *instr, *immed;
 384
 385         /* TODO in at least some cases, the backend could probably be
 386          * made clever enough to propagate IR3_REG_HALF..
 387          */
 388         instr = ir3_COV(block, src, TYPE_U32, TYPE_S16);
 389         instr->regs[0]->flags |= IR3_REG_HALF;
 390
 391         switch(align){
 392         case 1:
 393                 /* src *= 1: */
 394                 break;
 395         case 2:
 396                 /* src *= 2     => src <<= 1: */
 397                 immed = create_immed(block, 1);
 398                 immed->regs[0]->flags |= IR3_REG_HALF;
 399
 400                 instr = ir3_SHL_B(block, instr, 0, immed, 0);
 401                 instr->regs[0]->flags |= IR3_REG_HALF;
 402                 instr->regs[1]->flags |= IR3_REG_HALF;
 403                 break;
 404         case 3:
 405                 /* src *= 3: */
 406                 immed = create_immed(block, 3);
 407                 immed->regs[0]->flags |= IR3_REG_HALF;
 408
 409                 instr = ir3_MULL_U(block, instr, 0, immed, 0);
 410                 instr->regs[0]->flags |= IR3_REG_HALF;
 411                 instr->regs[1]->flags |= IR3_REG_HALF;
 412                 break;
 413         case 4:
 414                 /* src *= 4 => src <<= 2: */
 415                 immed = create_immed(block, 2);
 416                 immed->regs[0]->flags |= IR3_REG_HALF;
 417
 418                 instr = ir3_SHL_B(block, instr, 0, immed, 0);
 419                 instr->regs[0]->flags |= IR3_REG_HALF;
 420                 instr->regs[1]->flags |= IR3_REG_HALF;
 421                 break;
 422         default:
 423                 unreachable("bad align");
 424                 return NULL;
 425         }
 426
 427         instr = ir3_MOV(block, instr, TYPE_S16);
 428         instr->regs[0]->num = regid(REG_A0, 0);
 429         instr->regs[0]->flags |= IR3_REG_HALF;
 430         instr->regs[1]->flags |= IR3_REG_HALF;
 431
 432         return instr;
 433 }
 434
 435 /* caches addr values to avoid generating multiple cov/shl/mova
 436  * sequences for each use of a given NIR level src as address
 437  */
 438 struct ir3_instruction *
 439 ir3_get_addr(struct ir3_context *ctx, struct ir3_instruction *src, int align)
 440 {
 441         struct ir3_instruction *addr;
 442         unsigned idx = align - 1;
 443
 444         compile_assert(ctx, idx < ARRAY_SIZE(ctx->addr_ht));
 445
 446         if (!ctx->addr_ht[idx]) {
 447                 ctx->addr_ht[idx] = _mesa_hash_table_create(ctx,
 448                                 _mesa_hash_pointer, _mesa_key_pointer_equal);
 449         } else {
 450                 struct hash_entry *entry;
 451                 entry = _mesa_hash_table_search(ctx->addr_ht[idx], src);
 452                 if (entry)
 453                         return entry->data;
 454         }
 455
 456         addr = create_addr(ctx->block, src, align);
 457         _mesa_hash_table_insert(ctx->addr_ht[idx], src, addr);
 458
 459         return addr;
 460 }
 461
 462 struct ir3_instruction *
 463 ir3_get_predicate(struct ir3_context *ctx, struct ir3_instruction *src)
 464 {
 465         struct ir3_block *b = ctx->block;
 466         struct ir3_instruction *cond;
 467
 468         /* NOTE: only cmps.*.* can write p0.x: */
 469         cond = ir3_CMPS_S(b, src, 0, create_immed(b, 0), 0);
 470         cond->cat2.condition = IR3_COND_NE;
 471
 472         /* condition always goes in predicate register: */
 473         cond->regs[0]->num = regid(REG_P0, 0);
 474
 475         return cond;
 476 }
 477
 478 /*
 479  * Array helpers
 480  */
 481
 482 void
 483 ir3_declare_array(struct ir3_context *ctx, nir_register *reg)
 484 {
 485         struct ir3_array *arr = rzalloc(ctx, struct ir3_array);
 486         arr->id = ++ctx->num_arrays;
 487         /* NOTE: sometimes we get non array regs, for example for arrays of
 488          * length 1.  See fs-const-array-of-struct-of-array.shader_test.  So
 489          * treat a non-array as if it was an array of length 1.
 490          *
 491          * It would be nice if there was a nir pass to convert arrays of
 492          * length 1 to ssa.
 493          */
 494         arr->length = reg->num_components * MAX2(1, reg->num_array_elems);
 495         compile_assert(ctx, arr->length > 0);
 496         arr->r = reg;
 497         list_addtail(&arr->node, &ctx->ir->array_list);
 498 }
 499
 500 struct ir3_array *
 501 ir3_get_array(struct ir3_context *ctx, nir_register *reg)
 502 {
 503         list_for_each_entry (struct ir3_array, arr, &ctx->ir->array_list, node) {
 504                 if (arr->r == reg)
 505                         return arr;
 506         }
 507         ir3_context_error(ctx, "bogus reg: %s\n", reg->name);
 508         return NULL;
 509 }
 510
 511 /* relative (indirect) if address!=NULL */
 512 struct ir3_instruction *
 513 ir3_create_array_load(struct ir3_context *ctx, struct ir3_array *arr, int n,
 514                 struct ir3_instruction *address)
 515 {
 516         struct ir3_block *block = ctx->block;
 517         struct ir3_instruction *mov;
 518         struct ir3_register *src;
 519
 520         mov = ir3_instr_create(block, OPC_MOV);
 521         mov->cat1.src_type = TYPE_U32;
 522         mov->cat1.dst_type = TYPE_U32;
 523         mov->barrier_class = IR3_BARRIER_ARRAY_R;
 524         mov->barrier_conflict = IR3_BARRIER_ARRAY_W;
 525         ir3_reg_create(mov, 0, 0);
 526         src = ir3_reg_create(mov, 0, IR3_REG_ARRAY |
 527                         COND(address, IR3_REG_RELATIV));
 528         src->instr = arr->last_write;
 529         src->size  = arr->length;
 530         src->array.id = arr->id;
 531         src->array.offset = n;
 532
 533         if (address)
 534                 ir3_instr_set_address(mov, address);
 535
 536         return mov;
 537 }
 538
 539 /* relative (indirect) if address!=NULL */
 540 void
 541 ir3_create_array_store(struct ir3_context *ctx, struct ir3_array *arr, int n,
 542                 struct ir3_instruction *src, struct ir3_instruction *address)
 543 {
 544         struct ir3_block *block = ctx->block;
 545         struct ir3_instruction *mov;
 546         struct ir3_register *dst;
 547
 548         /* if not relative store, don't create an extra mov, since that
 549          * ends up being difficult for cp to remove.
 550          */
 551         if (!address) {
 552                 dst = src->regs[0];
 553
 554                 src->barrier_class |= IR3_BARRIER_ARRAY_W;
 555                 src->barrier_conflict |= IR3_BARRIER_ARRAY_R | IR3_BARRIER_ARRAY_W;
 556
 557                 dst->flags |= IR3_REG_ARRAY;
 558                 dst->instr = arr->last_write;
 559                 dst->size = arr->length;
 560                 dst->array.id = arr->id;
 561                 dst->array.offset = n;
 562
 563                 arr->last_write = src;
 564
 565                 array_insert(block, block->keeps, src);
 566
 567                 return;
 568         }
 569
 570         mov = ir3_instr_create(block, OPC_MOV);
 571         mov->cat1.src_type = TYPE_U32;
 572         mov->cat1.dst_type = TYPE_U32;
 573         mov->barrier_class = IR3_BARRIER_ARRAY_W;
 574         mov->barrier_conflict = IR3_BARRIER_ARRAY_R | IR3_BARRIER_ARRAY_W;
 575         dst = ir3_reg_create(mov, 0, IR3_REG_ARRAY |
 576                         COND(address, IR3_REG_RELATIV));
 577         dst->instr = arr->last_write;
 578         dst->size  = arr->length;
 579         dst->array.id = arr->id;
 580         dst->array.offset = n;
 581         ir3_reg_create(mov, 0, IR3_REG_SSA)->instr = src;
 582
 583         if (address)
 584                 ir3_instr_set_address(mov, address);
 585
 586         arr->last_write = mov;
 587
 588         /* the array store may only matter to something in an earlier
 589          * block (ie. loops), but since arrays are not in SSA, depth
 590          * pass won't know this.. so keep all array stores:
 591          */
 592         array_insert(block, block->keeps, mov);
 593 }