src/freedreno/ir3/ir3_group.c

   1 /*
   2  * Copyright (C) 2014 Rob Clark <robclark@freedesktop.org>
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  21  * SOFTWARE.
  22  *
  23  * Authors:
  24  *    Rob Clark <robclark@freedesktop.org>
  25  */
  26
  27 #include "ir3.h"
  28
  29 /*
  30  * Find/group instruction neighbors:
  31  */
  32
  33 /* bleh.. we need to do the same group_n() thing for both inputs/outputs
  34  * (where we have a simple instr[] array), and fanin nodes (where we have
  35  * an extra indirection via reg->instr).
  36  */
  37 struct group_ops {
  38         struct ir3_instruction *(*get)(void *arr, int idx);
  39         void (*insert_mov)(void *arr, int idx, struct ir3_instruction *instr);
  40 };
  41
  42 static struct ir3_instruction *arr_get(void *arr, int idx)
  43 {
  44         return ((struct ir3_instruction **)arr)[idx];
  45 }
  46 static void arr_insert_mov_out(void *arr, int idx, struct ir3_instruction *instr)
  47 {
  48         ((struct ir3_instruction **)arr)[idx] =
  49                         ir3_MOV(instr->block, instr, TYPE_F32);
  50 }
  51 static void arr_insert_mov_in(void *arr, int idx, struct ir3_instruction *instr)
  52 {
  53         /* so, we can't insert a mov in front of a meta:in.. and the downstream
  54          * instruction already has a pointer to 'instr'.  So we cheat a bit and
  55          * morph the meta:in instruction into a mov and insert a new meta:in
  56          * in front.
  57          */
  58         struct ir3_instruction *in;
  59
  60         debug_assert(instr->regs_count == 1);
  61
  62         in = ir3_instr_create(instr->block, OPC_META_INPUT);
  63         ir3_reg_create(in, instr->regs[0]->num, 0);
  64
  65         /* create src reg for meta:in and fixup to now be a mov: */
  66         ir3_reg_create(instr, 0, IR3_REG_SSA)->instr = in;
  67         instr->opc = OPC_MOV;
  68         instr->cat1.src_type = TYPE_F32;
  69         instr->cat1.dst_type = TYPE_F32;
  70
  71         ((struct ir3_instruction **)arr)[idx] = in;
  72 }
  73 static struct group_ops arr_ops_out = { arr_get, arr_insert_mov_out };
  74 static struct group_ops arr_ops_in = { arr_get, arr_insert_mov_in };
  75
  76 static struct ir3_instruction *instr_get(void *arr, int idx)
  77 {
  78         return ssa(((struct ir3_instruction *)arr)->regs[idx+1]);
  79 }
  80 static void
  81 instr_insert_mov(void *arr, int idx, struct ir3_instruction *instr)
  82 {
  83         ((struct ir3_instruction *)arr)->regs[idx+1]->instr =
  84                         ir3_MOV(instr->block, instr, TYPE_F32);
  85 }
  86 static struct group_ops instr_ops = { instr_get, instr_insert_mov };
  87
  88 /* verify that cur != instr, but cur is also not in instr's neighbor-list: */
  89 static bool
  90 in_neighbor_list(struct ir3_instruction *instr, struct ir3_instruction *cur, int pos)
  91 {
  92         int idx = 0;
  93
  94         if (!instr)
  95                 return false;
  96
  97         if (instr == cur)
  98                 return true;
  99
 100         for (instr = ir3_neighbor_first(instr); instr; instr = instr->cp.right)
 101                 if ((idx++ != pos) && (instr == cur))
 102                         return true;
 103
 104         return false;
 105 }
 106
 107 static void
 108 group_n(struct group_ops *ops, void *arr, unsigned n)
 109 {
 110         unsigned i, j;
 111
 112         /* first pass, figure out what has conflicts and needs a mov
 113          * inserted.  Do this up front, before starting to setup
 114          * left/right neighbor pointers.  Trying to do it in a single
 115          * pass could result in a situation where we can't even setup
 116          * the mov's right neighbor ptr if the next instr also needs
 117          * a mov.
 118          */
 119 restart:
 120         for (i = 0; i < n; i++) {
 121                 struct ir3_instruction *instr = ops->get(arr, i);
 122                 if (instr) {
 123                         struct ir3_instruction *left = (i > 0) ? ops->get(arr, i - 1) : NULL;
 124                         struct ir3_instruction *right = (i < (n-1)) ? ops->get(arr, i + 1) : NULL;
 125                         bool conflict;
 126
 127                         /* check for left/right neighbor conflicts: */
 128                         conflict = conflicts(instr->cp.left, left) ||
 129                                 conflicts(instr->cp.right, right);
 130
 131                         /* Mixing array elements and higher register classes
 132                          * (ie. groups) doesn't really work out in RA.  See:
 133                          *
 134                          * https://trello.com/c/DqeDkeVf/156-bug-with-stk-70frag
 135                          */
 136                         if (instr->regs[0]->flags & IR3_REG_ARRAY)
 137                                 conflict = true;
 138
 139                         /* we also can't have an instr twice in the group: */
 140                         for (j = i + 1; (j < n) && !conflict; j++)
 141                                 if (in_neighbor_list(ops->get(arr, j), instr, i))
 142                                         conflict = true;
 143
 144                         if (conflict) {
 145                                 ops->insert_mov(arr, i, instr);
 146                                 /* inserting the mov may have caused a conflict
 147                                  * against the previous:
 148                                  */
 149                                 goto restart;
 150                         }
 151                 }
 152         }
 153
 154         /* second pass, now that we've inserted mov's, fixup left/right
 155          * neighbors.  This is guaranteed to succeed, since by definition
 156          * the newly inserted mov's cannot conflict with anything.
 157          */
 158         for (i = 0; i < n; i++) {
 159                 struct ir3_instruction *instr = ops->get(arr, i);
 160                 if (instr) {
 161                         struct ir3_instruction *left = (i > 0) ? ops->get(arr, i - 1) : NULL;
 162                         struct ir3_instruction *right = (i < (n-1)) ? ops->get(arr, i + 1) : NULL;
 163
 164                         debug_assert(!conflicts(instr->cp.left, left));
 165                         if (left) {
 166                                 instr->cp.left_cnt++;
 167                                 instr->cp.left = left;
 168                         }
 169
 170                         debug_assert(!conflicts(instr->cp.right, right));
 171                         if (right) {
 172                                 instr->cp.right_cnt++;
 173                                 instr->cp.right = right;
 174                         }
 175                 }
 176         }
 177 }
 178
 179 static void
 180 instr_find_neighbors(struct ir3_instruction *instr)
 181 {
 182         struct ir3_instruction *src;
 183
 184         if (ir3_instr_check_mark(instr))
 185                 return;
 186
 187         if (instr->opc == OPC_META_FI)
 188                 group_n(&instr_ops, instr, instr->regs_count - 1);
 189
 190         foreach_ssa_src(src, instr)
 191                 instr_find_neighbors(src);
 192 }
 193
 194 /* a bit of sadness.. we can't have "holes" in inputs from PoV of
 195  * register assignment, they still need to be grouped together.  So
 196  * we need to insert dummy/padding instruction for grouping, and
 197  * then take it back out again before anyone notices.
 198  */
 199 static void
 200 pad_and_group_input(struct ir3_instruction **input, unsigned n)
 201 {
 202         int i, mask = 0;
 203         struct ir3_block *block = NULL;
 204
 205         for (i = n - 1; i >= 0; i--) {
 206                 struct ir3_instruction *instr = input[i];
 207                 if (instr) {
 208                         block = instr->block;
 209                 } else if (block) {
 210                         instr = ir3_NOP(block);
 211                         ir3_reg_create(instr, 0, IR3_REG_SSA);    /* dummy dst */
 212                         input[i] = instr;
 213                         mask |= (1 << i);
 214                 }
 215         }
 216
 217         group_n(&arr_ops_in, input, n);
 218
 219         for (i = 0; i < n; i++) {
 220                 if (mask & (1 << i))
 221                         input[i] = NULL;
 222         }
 223 }
 224
 225 static void
 226 find_neighbors(struct ir3 *ir)
 227 {
 228         unsigned i;
 229
 230         /* shader inputs/outputs themselves must be contiguous as well:
 231          *
 232          * NOTE: group inputs first, since we only insert mov's
 233          * *before* the conflicted instr (and that would go badly
 234          * for inputs).  By doing inputs first, we should never
 235          * have a conflict on inputs.. pushing any conflict to
 236          * resolve to the outputs, for stuff like:
 237          *
 238          *     MOV OUT[n], IN[m].wzyx
 239          *
 240          * NOTE: we assume here inputs/outputs are grouped in vec4.
 241          * This logic won't quite cut it if we don't align smaller
 242          * on vec4 boundaries
 243          */
 244         for (i = 0; i < ir->ninputs; i += 4)
 245                 pad_and_group_input(&ir->inputs[i], 4);
 246         for (i = 0; i < ir->noutputs; i += 4)
 247                 group_n(&arr_ops_out, &ir->outputs[i], 4);
 248
 249         for (i = 0; i < ir->noutputs; i++) {
 250                 if (ir->outputs[i]) {
 251                         struct ir3_instruction *instr = ir->outputs[i];
 252                         instr_find_neighbors(instr);
 253                 }
 254         }
 255
 256         list_for_each_entry (struct ir3_block, block, &ir->block_list, node) {
 257                 for (i = 0; i < block->keeps_count; i++) {
 258                         struct ir3_instruction *instr = block->keeps[i];
 259                         instr_find_neighbors(instr);
 260                 }
 261
 262                 /* We also need to account for if-condition: */
 263                 if (block->condition)
 264                         instr_find_neighbors(block->condition);
 265         }
 266 }
 267
 268 void
 269 ir3_group(struct ir3 *ir)
 270 {
 271         ir3_clear_mark(ir);
 272         find_neighbors(ir);
 273 }