src/panfrost/midgard/midgard_address.c

   1 /*
   2  * Copyright (C) 2019 Collabora, Ltd.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  21  * SOFTWARE.
  22  *
  23  * Authors (Collabora):
  24  *    Alyssa Rosenzweig <alyssa.rosenzweig@collabora.com>
  25  */
  26
  27 #include "compiler.h"
  28
  29 /* Midgard's generic load/store instructions, particularly to implement SSBOs
  30  * and globals, have support for address arithmetic natively. In particularly,
  31  * they take two indirect arguments A, B and two immediates #s, #c, calculating
  32  * the address:
  33  *
  34  *      A + (zext?(B) << #s) + #c
  35  *
  36  * This allows for fast indexing into arrays. This file tries to pattern match the offset in NIR with this form to reduce pressure on the ALU pipe.
  37  */
  38
  39 struct mir_address {
  40         nir_ssa_scalar A;
  41         nir_ssa_scalar B;
  42
  43         bool zext;
  44         unsigned shift;
  45         unsigned bias;
  46 };
  47
  48 /* Matches a constant in either slot and moves it to the bias */
  49
  50 static void
  51 mir_match_constant(struct mir_address *address)
  52 {
  53         if (address->A.def && nir_ssa_scalar_is_const(address->A)) {
  54                 address->bias += nir_ssa_scalar_as_uint(address->A);
  55                 address->A.def = NULL;
  56         }
  57
  58         if (address->B.def && nir_ssa_scalar_is_const(address->B)) {
  59                 address->bias += nir_ssa_scalar_as_uint(address->B);
  60                 address->B.def = NULL;
  61         }
  62 }
  63
  64 /* Matches an iadd when there is a free slot or constant */
  65
  66 static void
  67 mir_match_iadd(struct mir_address *address, bool first_free)
  68 {
  69         if (!address->B.def || !nir_ssa_scalar_is_alu(address->B))
  70                 return;
  71
  72         nir_op op = nir_ssa_scalar_alu_op(address->B);
  73
  74         if (op != nir_op_iadd) return;
  75
  76         nir_ssa_scalar op1 = nir_ssa_scalar_chase_alu_src(address->B, 0);
  77         nir_ssa_scalar op2 = nir_ssa_scalar_chase_alu_src(address->B, 1);
  78
  79         if (nir_ssa_scalar_is_const(op1)) {
  80                 address->bias += nir_ssa_scalar_as_uint(op1);
  81                 address->B = op2;
  82         } else if (nir_ssa_scalar_is_const(op2)) {
  83                 address->bias += nir_ssa_scalar_as_uint(op2);
  84                 address->B = op1;
  85         } else if (first_free && !address->A.def) {
  86                 address->A = op1;
  87                 address->B = op2;
  88         }
  89 }
  90
  91 /* Matches u2u64 and sets zext */
  92
  93 static void
  94 mir_match_u2u64(struct mir_address *address)
  95 {
  96         if (!address->B.def || !nir_ssa_scalar_is_alu(address->B))
  97                 return;
  98
  99         nir_op op = nir_ssa_scalar_alu_op(address->B);
 100         if (op != nir_op_u2u64) return;
 101         nir_ssa_scalar arg = nir_ssa_scalar_chase_alu_src(address->B, 0);
 102
 103         address->B = arg;
 104         address->zext = true;
 105 }
 106
 107 /* Matches ishl to shift */
 108
 109 static void
 110 mir_match_ishl(struct mir_address *address)
 111 {
 112         if (!address->B.def || !nir_ssa_scalar_is_alu(address->B))
 113                 return;
 114
 115         nir_op op = nir_ssa_scalar_alu_op(address->B);
 116         if (op != nir_op_ishl) return;
 117         nir_ssa_scalar op1 = nir_ssa_scalar_chase_alu_src(address->B, 0);
 118         nir_ssa_scalar op2 = nir_ssa_scalar_chase_alu_src(address->B, 1);
 119
 120         if (!nir_ssa_scalar_is_const(op2)) return;
 121
 122         unsigned shift = nir_ssa_scalar_as_uint(op2);
 123         if (shift > 0x7) return;
 124
 125         address->B = op1;
 126         address->shift = shift;
 127 }
 128
 129 /* Strings through mov which can happen from NIR vectorization */
 130
 131 static void
 132 mir_match_mov(struct mir_address *address)
 133 {
 134         if (address->A.def && nir_ssa_scalar_is_alu(address->A)) {
 135                 nir_op op = nir_ssa_scalar_alu_op(address->A);
 136
 137                 if (op == nir_op_mov)
 138                         address->A = nir_ssa_scalar_chase_alu_src(address->A, 0);
 139         }
 140
 141         if (address->B.def && nir_ssa_scalar_is_alu(address->B)) {
 142                 nir_op op = nir_ssa_scalar_alu_op(address->B);
 143
 144                 if (op == nir_op_mov)
 145                         address->B = nir_ssa_scalar_chase_alu_src(address->B, 0);
 146         }
 147 }
 148
 149 /* Tries to pattern match into mir_address */
 150
 151 static struct mir_address
 152 mir_match_offset(nir_ssa_def *offset, bool first_free)
 153 {
 154         struct mir_address address = {
 155                 .B = { .def = offset }
 156         };
 157
 158         mir_match_mov(&address);
 159         mir_match_constant(&address);
 160         mir_match_mov(&address);
 161         mir_match_iadd(&address, first_free);
 162         mir_match_mov(&address);
 163         mir_match_u2u64(&address);
 164         mir_match_mov(&address);
 165         mir_match_ishl(&address);
 166
 167         return address;
 168 }
 169
 170 void
 171 mir_set_offset(compiler_context *ctx, midgard_instruction *ins, nir_src *offset, bool is_shared)
 172 {
 173         for(unsigned i = 0; i < 16; ++i) {
 174                 ins->swizzle[1][i] = 0;
 175                 ins->swizzle[2][i] = 0;
 176         }
 177
 178         if (!offset->is_ssa) {
 179                 ins->load_store.arg_1 |= is_shared ? 0x6E : 0x7E;
 180                 ins->src[2] = nir_src_index(ctx, offset);
 181                 ins->src_types[2] = nir_type_uint | nir_src_bit_size(*offset);
 182                 return;
 183         }
 184
 185         struct mir_address match = mir_match_offset(offset->ssa, !is_shared);
 186
 187         if (match.A.def) {
 188                 ins->src[1] = nir_ssa_index(match.A.def);
 189                 ins->swizzle[1][0] = match.A.comp;
 190                 ins->src_types[1] = nir_type_uint | match.A.def->bit_size;
 191         } else
 192                 ins->load_store.arg_1 |= is_shared ? 0x6E : 0x7E;
 193
 194         if (match.B.def) {
 195                 ins->src[2] = nir_ssa_index(match.B.def);
 196                 ins->swizzle[2][0] = match.B.comp;
 197                 ins->src_types[2] = nir_type_uint | match.B.def->bit_size;
 198         } else
 199                 ins->load_store.arg_2 = 0x1E;
 200
 201         if (match.zext)
 202                 ins->load_store.arg_1 |= 0x80;
 203
 204         assert(match.shift <= 7);
 205         ins->load_store.arg_2 |= (match.shift) << 5;
 206
 207         ins->constants.u32[0] = match.bias;
 208 }