src/freedreno/ir3/ir3_legalize.c

   1 /*
   2  * Copyright (C) 2014 Rob Clark <robclark@freedesktop.org>
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  21  * SOFTWARE.
  22  *
  23  * Authors:
  24  *    Rob Clark <robclark@freedesktop.org>
  25  */
  26
  27 #include "util/ralloc.h"
  28 #include "util/u_math.h"
  29
  30 #include "ir3.h"
  31 #include "ir3_compiler.h"
  32
  33 /*
  34  * Legalize:
  35  *
  36  * We currently require that scheduling ensures that we have enough nop's
  37  * in all the right places.  The legalize step mostly handles fixing up
  38  * instruction flags ((ss)/(sy)/(ei)), and collapses sequences of nop's
  39  * into fewer nop's w/ rpt flag.
  40  */
  41
  42 struct ir3_legalize_ctx {
  43         struct ir3_compiler *compiler;
  44         int num_samp;
  45         bool has_ssbo;
  46         int max_bary;
  47 };
  48
  49 struct ir3_legalize_state {
  50         regmask_t needs_ss;
  51         regmask_t needs_ss_war;       /* write after read */
  52         regmask_t needs_sy;
  53 };
  54
  55 struct ir3_legalize_block_data {
  56         bool valid;
  57         struct ir3_legalize_state state;
  58 };
  59
  60 /* We want to evaluate each block from the position of any other
  61  * predecessor block, in order that the flags set are the union of
  62  * all possible program paths.
  63  *
  64  * To do this, we need to know the output state (needs_ss/ss_war/sy)
  65  * of all predecessor blocks.  The tricky thing is loops, which mean
  66  * that we can't simply recursively process each predecessor block
  67  * before legalizing the current block.
  68  *
  69  * How we handle that is by looping over all the blocks until the
  70  * results converge.  If the output state of a given block changes
  71  * in a given pass, this means that all successor blocks are not
  72  * yet fully legalized.
  73  */
  74
  75 static bool
  76 legalize_block(struct ir3_legalize_ctx *ctx, struct ir3_block *block)
  77 {
  78         struct ir3_legalize_block_data *bd = block->data;
  79
  80         if (bd->valid)
  81                 return false;
  82
  83         struct ir3_instruction *last_input = NULL;
  84         struct ir3_instruction *last_rel = NULL;
  85         struct ir3_instruction *last_n = NULL;
  86         struct list_head instr_list;
  87         struct ir3_legalize_state prev_state = bd->state;
  88         struct ir3_legalize_state *state = &bd->state;
  89
  90         /* our input state is the OR of all predecessor blocks' state: */
  91         for (unsigned i = 0; i < block->predecessors_count; i++) {
  92                 struct ir3_legalize_block_data *pbd = block->predecessors[i]->data;
  93                 struct ir3_legalize_state *pstate = &pbd->state;
  94
  95                 /* Our input (ss)/(sy) state is based on OR'ing the output
  96                  * state of all our predecessor blocks
  97                  */
  98                 regmask_or(&state->needs_ss,
  99                                 &state->needs_ss, &pstate->needs_ss);
 100                 regmask_or(&state->needs_ss_war,
 101                                 &state->needs_ss_war, &pstate->needs_ss_war);
 102                 regmask_or(&state->needs_sy,
 103                                 &state->needs_sy, &pstate->needs_sy);
 104         }
 105
 106         /* remove all the instructions from the list, we'll be adding
 107          * them back in as we go
 108          */
 109         list_replace(&block->instr_list, &instr_list);
 110         list_inithead(&block->instr_list);
 111
 112         list_for_each_entry_safe (struct ir3_instruction, n, &instr_list, node) {
 113                 struct ir3_register *reg;
 114                 unsigned i;
 115
 116                 n->flags &= ~(IR3_INSTR_SS | IR3_INSTR_SY);
 117
 118                 if (is_meta(n))
 119                         continue;
 120
 121                 if (is_input(n)) {
 122                         struct ir3_register *inloc = n->regs[1];
 123                         assert(inloc->flags & IR3_REG_IMMED);
 124                         ctx->max_bary = MAX2(ctx->max_bary, inloc->iim_val);
 125                 }
 126
 127                 if (last_n && is_barrier(last_n))
 128                         n->flags |= IR3_INSTR_SS | IR3_INSTR_SY;
 129
 130                 /* NOTE: consider dst register too.. it could happen that
 131                  * texture sample instruction (for example) writes some
 132                  * components which are unused.  A subsequent instruction
 133                  * that writes the same register can race w/ the sam instr
 134                  * resulting in undefined results:
 135                  */
 136                 for (i = 0; i < n->regs_count; i++) {
 137                         reg = n->regs[i];
 138
 139                         if (reg_gpr(reg)) {
 140
 141                                 /* TODO: we probably only need (ss) for alu
 142                                  * instr consuming sfu result.. need to make
 143                                  * some tests for both this and (sy)..
 144                                  */
 145                                 if (regmask_get(&state->needs_ss, reg)) {
 146                                         n->flags |= IR3_INSTR_SS;
 147                                         regmask_init(&state->needs_ss_war);
 148                                         regmask_init(&state->needs_ss);
 149                                 }
 150
 151                                 if (regmask_get(&state->needs_sy, reg)) {
 152                                         n->flags |= IR3_INSTR_SY;
 153                                         regmask_init(&state->needs_sy);
 154                                 }
 155                         }
 156
 157                         /* TODO: is it valid to have address reg loaded from a
 158                          * relative src (ie. mova a0, c<a0.x+4>)?  If so, the
 159                          * last_rel check below should be moved ahead of this:
 160                          */
 161                         if (reg->flags & IR3_REG_RELATIV)
 162                                 last_rel = n;
 163                 }
 164
 165                 if (n->regs_count > 0) {
 166                         reg = n->regs[0];
 167                         if (regmask_get(&state->needs_ss_war, reg)) {
 168                                 n->flags |= IR3_INSTR_SS;
 169                                 regmask_init(&state->needs_ss_war);
 170                                 regmask_init(&state->needs_ss);
 171                         }
 172
 173                         if (last_rel && (reg->num == regid(REG_A0, 0))) {
 174                                 last_rel->flags |= IR3_INSTR_UL;
 175                                 last_rel = NULL;
 176                         }
 177                 }
 178
 179                 /* cat5+ does not have an (ss) bit, if needed we need to
 180                  * insert a nop to carry the sync flag.  Would be kinda
 181                  * clever if we were aware of this during scheduling, but
 182                  * this should be a pretty rare case:
 183                  */
 184                 if ((n->flags & IR3_INSTR_SS) && (opc_cat(n->opc) >= 5)) {
 185                         struct ir3_instruction *nop;
 186                         nop = ir3_NOP(block);
 187                         nop->flags |= IR3_INSTR_SS;
 188                         n->flags &= ~IR3_INSTR_SS;
 189                 }
 190
 191                 /* need to be able to set (ss) on first instruction: */
 192                 if (list_empty(&block->instr_list) && (opc_cat(n->opc) >= 5))
 193                         ir3_NOP(block);
 194
 195                 if (is_nop(n) && !list_empty(&block->instr_list)) {
 196                         struct ir3_instruction *last = list_last_entry(&block->instr_list,
 197                                         struct ir3_instruction, node);
 198                         if (is_nop(last) && (last->repeat < 5)) {
 199                                 last->repeat++;
 200                                 last->flags |= n->flags;
 201                                 continue;
 202                         }
 203
 204                         /* NOTE: I think the nopN encoding works for a5xx and
 205                          * probably a4xx, but not a3xx.  So far only tested on
 206                          * a6xx.
 207                          */
 208                         if ((ctx->compiler->gpu_id >= 600) && !n->flags && (last->nop < 3) &&
 209                                         ((opc_cat(last->opc) == 2) || (opc_cat(last->opc) == 3))) {
 210                                 last->nop++;
 211                                 continue;
 212                         }
 213                 }
 214
 215                 list_addtail(&n->node, &block->instr_list);
 216
 217                 if (is_sfu(n))
 218                         regmask_set(&state->needs_ss, n->regs[0]);
 219
 220                 if (is_tex(n)) {
 221                         /* this ends up being the # of samp instructions.. but that
 222                          * is ok, everything else only cares whether it is zero or
 223                          * not.  We do this here, rather than when we encounter a
 224                          * SAMP decl, because (especially in binning pass shader)
 225                          * the samp instruction(s) could get eliminated if the
 226                          * result is not used.
 227                          */
 228                         ctx->num_samp = MAX2(ctx->num_samp, n->cat5.samp + 1);
 229                         regmask_set(&state->needs_sy, n->regs[0]);
 230                 } else if (n->opc == OPC_RESINFO) {
 231                         regmask_set(&state->needs_ss, n->regs[0]);
 232                         ir3_NOP(block)->flags |= IR3_INSTR_SS;
 233                 } else if (is_load(n)) {
 234                         /* seems like ldlv needs (ss) bit instead??  which is odd but
 235                          * makes a bunch of flat-varying tests start working on a4xx.
 236                          */
 237                         if ((n->opc == OPC_LDLV) || (n->opc == OPC_LDL))
 238                                 regmask_set(&state->needs_ss, n->regs[0]);
 239                         else
 240                                 regmask_set(&state->needs_sy, n->regs[0]);
 241                 } else if (is_atomic(n->opc)) {
 242                         if (n->flags & IR3_INSTR_G) {
 243                                 if (ctx->compiler->gpu_id >= 600) {
 244                                         /* New encoding, returns  result via second src: */
 245                                         regmask_set(&state->needs_sy, n->regs[3]);
 246                                 } else {
 247                                         regmask_set(&state->needs_sy, n->regs[0]);
 248                                 }
 249                         } else {
 250                                 regmask_set(&state->needs_ss, n->regs[0]);
 251                         }
 252                 }
 253
 254                 if (is_ssbo(n->opc) || (is_atomic(n->opc) && (n->flags & IR3_INSTR_G)))
 255                         ctx->has_ssbo = true;
 256
 257                 /* both tex/sfu appear to not always immediately consume
 258                  * their src register(s):
 259                  */
 260                 if (is_tex(n) || is_sfu(n) || is_mem(n)) {
 261                         foreach_src(reg, n) {
 262                                 if (reg_gpr(reg))
 263                                         regmask_set(&state->needs_ss_war, reg);
 264                         }
 265                 }
 266
 267                 if (is_input(n))
 268                         last_input = n;
 269
 270                 last_n = n;
 271         }
 272
 273         if (last_input) {
 274                 /* special hack.. if using ldlv to bypass interpolation,
 275                  * we need to insert a dummy bary.f on which we can set
 276                  * the (ei) flag:
 277                  */
 278                 if (is_mem(last_input) && (last_input->opc == OPC_LDLV)) {
 279                         struct ir3_instruction *baryf;
 280
 281                         /* (ss)bary.f (ei)r63.x, 0, r0.x */
 282                         baryf = ir3_instr_create(block, OPC_BARY_F);
 283                         baryf->flags |= IR3_INSTR_SS;
 284                         ir3_reg_create(baryf, regid(63, 0), 0);
 285                         ir3_reg_create(baryf, 0, IR3_REG_IMMED)->iim_val = 0;
 286                         ir3_reg_create(baryf, regid(0, 0), 0);
 287
 288                         /* insert the dummy bary.f after last_input: */
 289                         list_delinit(&baryf->node);
 290                         list_add(&baryf->node, &last_input->node);
 291
 292                         last_input = baryf;
 293                 }
 294                 last_input->regs[0]->flags |= IR3_REG_EI;
 295         }
 296
 297         if (last_rel)
 298                 last_rel->flags |= IR3_INSTR_UL;
 299
 300         bd->valid = true;
 301
 302         if (memcmp(&prev_state, state, sizeof(*state))) {
 303                 /* our output state changed, this invalidates all of our
 304                  * successors:
 305                  */
 306                 for (unsigned i = 0; i < ARRAY_SIZE(block->successors); i++) {
 307                         if (!block->successors[i])
 308                                 break;
 309                         struct ir3_legalize_block_data *pbd = block->successors[i]->data;
 310                         pbd->valid = false;
 311                 }
 312         }
 313
 314         return true;
 315 }
 316
 317 /* NOTE: branch instructions are always the last instruction(s)
 318  * in the block.  We take advantage of this as we resolve the
 319  * branches, since "if (foo) break;" constructs turn into
 320  * something like:
 321  *
 322  *   block3 {
 323  *      ...
 324  *      0029:021: mov.s32s32 r62.x, r1.y
 325  *      0082:022: br !p0.x, target=block5
 326  *      0083:023: br p0.x, target=block4
 327  *      // succs: if _[0029:021: mov.s32s32] block4; else block5;
 328  *   }
 329  *   block4 {
 330  *      0084:024: jump, target=block6
 331  *      // succs: block6;
 332  *   }
 333  *   block5 {
 334  *      0085:025: jump, target=block7
 335  *      // succs: block7;
 336  *   }
 337  *
 338  * ie. only instruction in block4/block5 is a jump, so when
 339  * resolving branches we can easily detect this by checking
 340  * that the first instruction in the target block is itself
 341  * a jump, and setup the br directly to the jump's target
 342  * (and strip back out the now unreached jump)
 343  *
 344  * TODO sometimes we end up with things like:
 345  *
 346  *    br !p0.x, #2
 347  *    br p0.x, #12
 348  *    add.u r0.y, r0.y, 1
 349  *
 350  * If we swapped the order of the branches, we could drop one.
 351  */
 352 static struct ir3_block *
 353 resolve_dest_block(struct ir3_block *block)
 354 {
 355         /* special case for last block: */
 356         if (!block->successors[0])
 357                 return block;
 358
 359         /* NOTE that we may or may not have inserted the jump
 360          * in the target block yet, so conditions to resolve
 361          * the dest to the dest block's successor are:
 362          *
 363          *   (1) successor[1] == NULL &&
 364          *   (2) (block-is-empty || only-instr-is-jump)
 365          */
 366         if (block->successors[1] == NULL) {
 367                 if (list_empty(&block->instr_list)) {
 368                         return block->successors[0];
 369                 } else if (list_length(&block->instr_list) == 1) {
 370                         struct ir3_instruction *instr = list_first_entry(
 371                                         &block->instr_list, struct ir3_instruction, node);
 372                         if (instr->opc == OPC_JUMP)
 373                                 return block->successors[0];
 374                 }
 375         }
 376         return block;
 377 }
 378
 379 static bool
 380 resolve_jump(struct ir3_instruction *instr)
 381 {
 382         struct ir3_block *tblock =
 383                 resolve_dest_block(instr->cat0.target);
 384         struct ir3_instruction *target;
 385
 386         if (tblock != instr->cat0.target) {
 387                 list_delinit(&instr->cat0.target->node);
 388                 instr->cat0.target = tblock;
 389                 return true;
 390         }
 391
 392         target = list_first_entry(&tblock->instr_list,
 393                                 struct ir3_instruction, node);
 394
 395         /* TODO maybe a less fragile way to do this.  But we are expecting
 396          * a pattern from sched_block() that looks like:
 397          *
 398          *   br !p0.x, #else-block
 399          *   br p0.x, #if-block
 400          *
 401          * if the first branch target is +2, or if 2nd branch target is +1
 402          * then we can just drop the jump.
 403          */
 404         unsigned next_block;
 405         if (instr->cat0.inv == true)
 406                 next_block = 2;
 407         else
 408                 next_block = 1;
 409
 410         if ((!target) || (target->ip == (instr->ip + next_block))) {
 411                 list_delinit(&instr->node);
 412                 return true;
 413         } else {
 414                 instr->cat0.immed =
 415                         (int)target->ip - (int)instr->ip;
 416         }
 417         return false;
 418 }
 419
 420 /* resolve jumps, removing jumps/branches to immediately following
 421  * instruction which we end up with from earlier stages.  Since
 422  * removing an instruction can invalidate earlier instruction's
 423  * branch offsets, we need to do this iteratively until no more
 424  * branches are removed.
 425  */
 426 static bool
 427 resolve_jumps(struct ir3 *ir)
 428 {
 429         list_for_each_entry (struct ir3_block, block, &ir->block_list, node)
 430                 list_for_each_entry (struct ir3_instruction, instr, &block->instr_list, node)
 431                         if (is_flow(instr) && instr->cat0.target)
 432                                 if (resolve_jump(instr))
 433                                         return true;
 434
 435         return false;
 436 }
 437
 438 /* we want to mark points where divergent flow control re-converges
 439  * with (jp) flags.  For now, since we don't do any optimization for
 440  * things that start out as a 'do {} while()', re-convergence points
 441  * will always be a branch or jump target.  Note that this is overly
 442  * conservative, since unconditional jump targets are not convergence
 443  * points, we are just assuming that the other path to reach the jump
 444  * target was divergent.  If we were clever enough to optimize the
 445  * jump at end of a loop back to a conditional branch into a single
 446  * conditional branch, ie. like:
 447  *
 448  *    add.f r1.w, r0.x, (neg)(r)c2.x   <= loop start
 449  *    mul.f r1.z, r1.z, r0.x
 450  *    mul.f r1.y, r1.y, r0.x
 451  *    mul.f r0.z, r1.x, r0.x
 452  *    mul.f r0.w, r0.y, r0.x
 453  *    cmps.f.ge r0.x, (r)c2.y, (r)r1.w
 454  *    add.s r0.x, (r)r0.x, (r)-1
 455  *    sel.f32 r0.x, (r)c3.y, (r)r0.x, c3.x
 456  *    cmps.f.eq p0.x, r0.x, c3.y
 457  *    mov.f32f32 r0.x, r1.w
 458  *    mov.f32f32 r0.y, r0.w
 459  *    mov.f32f32 r1.x, r0.z
 460  *    (rpt2)nop
 461  *    br !p0.x, #-13
 462  *    (jp)mul.f r0.x, c263.y, r1.y
 463  *
 464  * Then we'd have to be more clever, as the convergence point is no
 465  * longer a branch or jump target.
 466  */
 467 static void
 468 mark_convergence_points(struct ir3 *ir)
 469 {
 470         list_for_each_entry (struct ir3_block, block, &ir->block_list, node) {
 471                 list_for_each_entry (struct ir3_instruction, instr, &block->instr_list, node) {
 472                         if (is_flow(instr) && instr->cat0.target) {
 473                                 struct ir3_instruction *target =
 474                                         list_first_entry(&instr->cat0.target->instr_list,
 475                                                         struct ir3_instruction, node);
 476                                 target->flags |= IR3_INSTR_JP;
 477                         }
 478                 }
 479         }
 480 }
 481
 482 void
 483 ir3_legalize(struct ir3 *ir, int *num_samp, bool *has_ssbo, int *max_bary)
 484 {
 485         struct ir3_legalize_ctx *ctx = rzalloc(ir, struct ir3_legalize_ctx);
 486         bool progress;
 487
 488         ctx->max_bary = -1;
 489         ctx->compiler = ir->compiler;
 490
 491         /* allocate per-block data: */
 492         list_for_each_entry (struct ir3_block, block, &ir->block_list, node) {
 493                 block->data = rzalloc(ctx, struct ir3_legalize_block_data);
 494         }
 495
 496         /* process each block: */
 497         do {
 498                 progress = false;
 499                 list_for_each_entry (struct ir3_block, block, &ir->block_list, node) {
 500                         progress |= legalize_block(ctx, block);
 501                 }
 502         } while (progress);
 503
 504         *num_samp = ctx->num_samp;
 505         *has_ssbo = ctx->has_ssbo;
 506         *max_bary = ctx->max_bary;
 507
 508         do {
 509                 ir3_count_instructions(ir);
 510         } while(resolve_jumps(ir));
 511
 512         mark_convergence_points(ir);
 513
 514         ralloc_free(ctx);
 515 }