src/panfrost/bifrost/bi_pack.c

   1 /*
   2  * Copyright (C) 2020 Collabora, Ltd.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  21  * SOFTWARE.
  22  */
  23
  24 #include "compiler.h"
  25
  26 #define RETURN_PACKED(str) { \
  27         uint64_t temp = 0; \
  28         memcpy(&temp, &str, sizeof(str)); \
  29         return temp; \
  30 }
  31
  32 /* This file contains the final passes of the compiler. Running after
  33  * scheduling and RA, the IR is now finalized, so we need to emit it to actual
  34  * bits on the wire (as well as fixup branches) */
  35
  36 static uint64_t
  37 bi_pack_header(bi_clause *clause, bi_clause *next, bool is_fragment)
  38 {
  39         struct bifrost_header header = {
  40                 .back_to_back = clause->back_to_back,
  41                 .no_end_of_shader = (next != NULL),
  42                 .elide_writes = is_fragment,
  43                 .branch_cond = clause->branch_conditional,
  44                 .datareg_writebarrier = clause->data_register_write_barrier,
  45                 .datareg = clause->data_register,
  46                 .scoreboard_deps = next ? next->dependencies : 0,
  47                 .scoreboard_index = clause->scoreboard_id,
  48                 .clause_type = clause->clause_type,
  49                 .next_clause_type = next ? next->clause_type : 0,
  50         };
  51
  52         uint64_t u = 0;
  53         memcpy(&u, &header, sizeof(header));
  54         return u;
  55 }
  56
  57 /* Represents the assignment of ports for a given bundle */
  58
  59 struct bi_registers {
  60         /* Register to assign to each port */
  61         unsigned port[4];
  62
  63         /* Read ports can be disabled */
  64         bool enabled[2];
  65
  66         /* Should we write FMA? what about ADD? If only a single port is
  67          * enabled it is in port 2, else ADD/FMA is 2/3 respectively */
  68         bool write_fma, write_add;
  69
  70         /* Should we read with port 3? */
  71         bool read_port3;
  72
  73         /* Packed uniform/constant */
  74         uint8_t uniform_constant;
  75
  76         /* Whether writes are actually for the last instruction */
  77         bool first_instruction;
  78 };
  79
  80 static inline void
  81 bi_print_ports(struct bi_registers *regs)
  82 {
  83         for (unsigned i = 0; i < 2; ++i) {
  84                 if (regs->enabled[i])
  85                         printf("port %u: %u\n", i, regs->port[i]);
  86         }
  87
  88         if (regs->write_fma || regs->write_add) {
  89                 printf("port 2 (%s): %u\n",
  90                                 regs->write_fma ? "FMA" : "ADD",
  91                                 regs->port[2]);
  92         }
  93
  94         if ((regs->write_fma && regs->write_add) || regs->read_port3) {
  95                 printf("port 3 (%s): %u\n",
  96                                 regs->read_port3 ? "read" : "ADD",
  97                                 regs->port[3]);
  98         }
  99 }
 100
 101 /* The uniform/constant slot allows loading a contiguous 64-bit immediate or
 102  * pushed uniform per bundle. Figure out which one we need in the bundle (the
 103  * scheduler needs to ensure we only have one type per bundle), validate
 104  * everything, and rewrite away the register/uniform indices to use 3-bit
 105  * sources directly. */
 106
 107 static unsigned
 108 bi_lookup_constant(bi_clause *clause, uint64_t cons, bool *hi, bool b64)
 109 {
 110         uint64_t want = (cons >> 4);
 111
 112         for (unsigned i = 0; i < clause->constant_count; ++i) {
 113                 /* Only check top 60-bits since that's what's actually embedded
 114                  * in the clause, the bottom 4-bits are bundle-inline */
 115
 116                 unsigned candidates[2] = {
 117                         clause->constants[i] >> 4,
 118                         clause->constants[i] >> 36
 119                 };
 120
 121                 if (!b64)
 122                         candidates[0] &= 0xFFFFFFFF;
 123
 124                 if (candidates[0] == want)
 125                         return i;
 126
 127                 if (candidates[1] == want && !b64) {
 128                         *hi = true;
 129                         return i;
 130                 }
 131         }
 132
 133         unreachable("Invalid constant accessed");
 134 }
 135
 136 static unsigned
 137 bi_constant_field(unsigned idx)
 138 {
 139         assert(idx <= 5);
 140
 141         const unsigned values[] = {
 142                 4, 5, 6, 7, 2, 3
 143         };
 144
 145         return values[idx] << 4;
 146 }
 147
 148 static bool
 149 bi_assign_uniform_constant_single(
 150                 struct bi_registers *regs,
 151                 bi_clause *clause,
 152                 bi_instruction *ins, bool assigned, bool fast_zero)
 153 {
 154         if (!ins)
 155                 return assigned;
 156
 157         bi_foreach_src(ins, s) {
 158                 if (s == 0 && (ins->type == BI_LOAD_VAR_ADDRESS || ins->type == BI_LOAD_ATTR)) continue;
 159
 160                 if (ins->src[s] & BIR_INDEX_CONSTANT) {
 161                         bool hi = false;
 162                         bool b64 = nir_alu_type_get_type_size(ins->src_types[s]) > 32;
 163                         uint64_t cons = bi_get_immediate(ins, ins->src[s]);
 164                         unsigned idx = bi_lookup_constant(clause, cons, &hi, b64);
 165                         unsigned f = bi_constant_field(idx) | (cons & 0xF);
 166
 167                         if (assigned && regs->uniform_constant != f)
 168                                 unreachable("Mismatched uniform/const field: imm");
 169
 170                         regs->uniform_constant = f;
 171                         ins->src[s] = BIR_INDEX_PASS | (hi ? BIFROST_SRC_CONST_HI : BIFROST_SRC_CONST_LO);
 172                         assigned = true;
 173                 } else if (ins->src[s] & BIR_INDEX_ZERO && (ins->type == BI_LOAD_UNIFORM || ins->type == BI_LOAD_VAR)) {
 174                         /* XXX: HACK UNTIL WE HAVE HI MATCHING DUE TO OVERFLOW XXX */
 175                         ins->src[s] = BIR_INDEX_PASS | BIFROST_SRC_CONST_HI;
 176                 } else if (ins->src[s] & BIR_INDEX_ZERO && !fast_zero) {
 177                         /* FMAs have a fast zero port, ADD needs to use the
 178                          * uniform/const port's special 0 mode handled here */
 179                         unsigned f = 0;
 180
 181                         if (assigned && regs->uniform_constant != f)
 182                                 unreachable("Mismatched uniform/const field: 0");
 183
 184                         regs->uniform_constant = f;
 185                         ins->src[s] = BIR_INDEX_PASS | BIFROST_SRC_CONST_LO;
 186                         assigned = true;
 187                 } else if (s & BIR_INDEX_UNIFORM) {
 188                         unreachable("Push uniforms not implemented yet");
 189                 }
 190         }
 191
 192         return assigned;
 193 }
 194
 195 static void
 196 bi_assign_uniform_constant(
 197                 bi_clause *clause,
 198                 struct bi_registers *regs,
 199                 bi_bundle bundle)
 200 {
 201         bool assigned =
 202                 bi_assign_uniform_constant_single(regs, clause, bundle.fma, false, true);
 203
 204         bi_assign_uniform_constant_single(regs, clause, bundle.add, assigned, false);
 205 }
 206
 207 /* Assigns a port for reading, before anything is written */
 208
 209 static void
 210 bi_assign_port_read(struct bi_registers *regs, unsigned src)
 211 {
 212         /* We only assign for registers */
 213         if (!(src & BIR_INDEX_REGISTER))
 214                 return;
 215
 216         unsigned reg = src & ~BIR_INDEX_REGISTER;
 217
 218         /* Check if we already assigned the port */
 219         for (unsigned i = 0; i <= 1; ++i) {
 220                 if (regs->port[i] == reg && regs->enabled[i])
 221                         return;
 222         }
 223
 224         if (regs->port[3] == reg && regs->read_port3)
 225                 return;
 226
 227         /* Assign it now */
 228
 229         for (unsigned i = 0; i <= 1; ++i) {
 230                 if (!regs->enabled[i]) {
 231                         regs->port[i] = reg;
 232                         regs->enabled[i] = true;
 233                         return;
 234                 }
 235         }
 236
 237         if (!regs->read_port3) {
 238                 regs->port[3] = reg;
 239                 regs->read_port3 = true;
 240                 return;
 241         }
 242
 243         bi_print_ports(regs);
 244         unreachable("Failed to find a free port for src");
 245 }
 246
 247 static struct bi_registers
 248 bi_assign_ports(bi_bundle now, bi_bundle prev)
 249 {
 250         struct bi_registers regs = { 0 };
 251
 252         /* We assign ports for the main register mechanism. Special ops
 253          * use the data registers, which has its own mechanism entirely
 254          * and thus gets skipped over here. */
 255
 256         unsigned read_dreg = now.add &&
 257                 bi_class_props[now.add->type] & BI_DATA_REG_SRC;
 258
 259         unsigned write_dreg = prev.add &&
 260                 bi_class_props[prev.add->type] & BI_DATA_REG_DEST;
 261
 262         /* First, assign reads */
 263
 264         if (now.fma)
 265                 bi_foreach_src(now.fma, src)
 266                         bi_assign_port_read(&regs, now.fma->src[src]);
 267
 268         if (now.add) {
 269                 bi_foreach_src(now.add, src) {
 270                         if (!(src == 0 && read_dreg))
 271                                 bi_assign_port_read(&regs, now.add->src[src]);
 272                 }
 273         }
 274
 275         /* Next, assign writes */
 276
 277         if (prev.fma && prev.fma->dest & BIR_INDEX_REGISTER) {
 278                 regs.port[2] = prev.fma->dest & ~BIR_INDEX_REGISTER;
 279                 regs.write_fma = true;
 280         }
 281
 282         if (prev.add && prev.add->dest & BIR_INDEX_REGISTER && !write_dreg) {
 283                 unsigned r = prev.add->dest & ~BIR_INDEX_REGISTER;
 284
 285                 if (regs.write_fma) {
 286                         /* Scheduler constraint: cannot read 3 and write 2 */
 287                         assert(!regs.read_port3);
 288                         regs.port[3] = r;
 289                 } else {
 290                         regs.port[2] = r;
 291                 }
 292
 293                 regs.write_add = true;
 294         }
 295
 296         /* Finally, ensure port 1 > port 0 for the 63-x trick to function */
 297
 298         if (regs.enabled[0] && regs.enabled[1] && regs.port[1] < regs.port[0]) {
 299                 unsigned temp = regs.port[0];
 300                 regs.port[0] = regs.port[1];
 301                 regs.port[1] = temp;
 302         }
 303
 304         return regs;
 305 }
 306
 307 /* Determines the register control field, ignoring the first? flag */
 308
 309 static enum bifrost_reg_control
 310 bi_pack_register_ctrl_lo(struct bi_registers r)
 311 {
 312         if (r.write_fma) {
 313                 if (r.write_add) {
 314                         assert(!r.read_port3);
 315                         return BIFROST_WRITE_ADD_P2_FMA_P3;
 316                 } else {
 317                         if (r.read_port3)
 318                                 return BIFROST_WRITE_FMA_P2_READ_P3;
 319                         else
 320                                 return BIFROST_WRITE_FMA_P2;
 321                 }
 322         } else if (r.write_add) {
 323                 if (r.read_port3)
 324                         return BIFROST_WRITE_ADD_P2_READ_P3;
 325                 else
 326                         return BIFROST_WRITE_ADD_P2;
 327         } else if (r.read_port3)
 328                 return BIFROST_READ_P3;
 329         else
 330                 return BIFROST_REG_NONE;
 331 }
 332
 333 /* Ditto but account for the first? flag this time */
 334
 335 static enum bifrost_reg_control
 336 bi_pack_register_ctrl(struct bi_registers r)
 337 {
 338         enum bifrost_reg_control ctrl = bi_pack_register_ctrl_lo(r);
 339
 340         if (r.first_instruction) {
 341                 if (ctrl == BIFROST_REG_NONE)
 342                         ctrl = BIFROST_FIRST_NONE;
 343                 else if (ctrl == BIFROST_WRITE_FMA_P2_READ_P3)
 344                         ctrl = BIFROST_FIRST_WRITE_FMA_P2_READ_P3;
 345                 else
 346                         ctrl |= BIFROST_FIRST_NONE;
 347         }
 348
 349         return ctrl;
 350 }
 351
 352 static uint64_t
 353 bi_pack_registers(struct bi_registers regs)
 354 {
 355         enum bifrost_reg_control ctrl = bi_pack_register_ctrl(regs);
 356         struct bifrost_regs s;
 357         uint64_t packed = 0;
 358
 359         if (regs.enabled[1]) {
 360                 /* Gotta save that bit!~ Required by the 63-x trick */
 361                 assert(regs.port[1] > regs.port[0]);
 362                 assert(regs.enabled[0]);
 363
 364                 /* Do the 63-x trick, see docs/disasm */
 365                 if (regs.port[0] > 31) {
 366                         regs.port[0] = 63 - regs.port[0];
 367                         regs.port[1] = 63 - regs.port[1];
 368                 }
 369
 370                 assert(regs.port[0] <= 31);
 371                 assert(regs.port[1] <= 63);
 372
 373                 s.ctrl = ctrl;
 374                 s.reg1 = regs.port[1];
 375                 s.reg0 = regs.port[0];
 376         } else {
 377                 /* Port 1 disabled, so set to zero and use port 1 for ctrl */
 378                 s.reg1 = ctrl << 2;
 379
 380                 if (regs.enabled[0]) {
 381                         /* Bit 0 upper bit of port 0 */
 382                         s.reg1 |= (regs.port[0] >> 5);
 383
 384                         /* Rest of port 0 in usual spot */
 385                         s.reg0 = (regs.port[0] & 0b11111);
 386                 } else {
 387                         /* Bit 1 set if port 0 also disabled */
 388                         s.reg1 |= (1 << 1);
 389                 }
 390         }
 391
 392         /* When port 3 isn't used, we have to set it to port 2, and vice versa,
 393          * or INSTR_INVALID_ENC is raised. The reason is unknown. */
 394
 395         bool has_port2 = regs.write_fma || regs.write_add;
 396         bool has_port3 = regs.read_port3 || (regs.write_fma && regs.write_add);
 397
 398         if (!has_port3)
 399                 regs.port[3] = regs.port[2];
 400
 401         if (!has_port2)
 402                 regs.port[2] = regs.port[3];
 403
 404         s.reg3 = regs.port[3];
 405         s.reg2 = regs.port[2];
 406         s.uniform_const = regs.uniform_constant;
 407
 408         memcpy(&packed, &s, sizeof(s));
 409         return packed;
 410 }
 411
 412 static void
 413 bi_set_data_register(bi_clause *clause, unsigned idx)
 414 {
 415         assert(idx & BIR_INDEX_REGISTER);
 416         unsigned reg = idx & ~BIR_INDEX_REGISTER;
 417         assert(reg <= 63);
 418         clause->data_register = reg;
 419 }
 420
 421 static void
 422 bi_read_data_register(bi_clause *clause, bi_instruction *ins)
 423 {
 424         bi_set_data_register(clause, ins->src[0]);
 425 }
 426
 427 static void
 428 bi_write_data_register(bi_clause *clause, bi_instruction *ins)
 429 {
 430         bi_set_data_register(clause, ins->dest);
 431 }
 432
 433 static enum bifrost_packed_src
 434 bi_get_src_reg_port(struct bi_registers *regs, unsigned src)
 435 {
 436         unsigned reg = src & ~BIR_INDEX_REGISTER;
 437
 438         if (regs->port[0] == reg && regs->enabled[0])
 439                 return BIFROST_SRC_PORT0;
 440         else if (regs->port[1] == reg && regs->enabled[1])
 441                 return BIFROST_SRC_PORT1;
 442         else if (regs->port[3] == reg && regs->read_port3)
 443                 return BIFROST_SRC_PORT3;
 444         else
 445                 unreachable("Tried to access register with no port");
 446 }
 447
 448 static enum bifrost_packed_src
 449 bi_get_src(bi_instruction *ins, struct bi_registers *regs, unsigned s, bool is_fma)
 450 {
 451         unsigned src = ins->src[s];
 452
 453         if (src & BIR_INDEX_REGISTER)
 454                 return bi_get_src_reg_port(regs, src);
 455         else if (src & BIR_INDEX_ZERO && is_fma)
 456                 return BIFROST_SRC_STAGE;
 457         else if (src & BIR_INDEX_PASS)
 458                 return src & ~BIR_INDEX_PASS;
 459         else
 460                 unreachable("Unknown src");
 461 }
 462
 463 /* Constructs a packed 2-bit swizzle for a 16-bit vec2 source. Source must be
 464  * 16-bit and written components must correspond to valid swizzles (component x
 465  * or y). */
 466
 467 static unsigned
 468 bi_swiz16(bi_instruction *ins, unsigned src)
 469 {
 470         assert(nir_alu_type_get_type_size(ins->src_types[src]) == 16);
 471         unsigned swizzle = 0;
 472
 473         for (unsigned c = 0; c < 2; ++c) {
 474                 if (!bi_writes_component(ins, src)) continue;
 475
 476                 unsigned k = ins->swizzle[src][c];
 477                 assert(k < 1);
 478                 swizzle |= (k << c);
 479         }
 480
 481         return swizzle;
 482 }
 483
 484 static unsigned
 485 bi_pack_fma_fma(bi_instruction *ins, struct bi_registers *regs)
 486 {
 487         /* (-a)(-b) = ab, so we only need one negate bit */
 488         bool negate_mul = ins->src_neg[0] ^ ins->src_neg[1];
 489
 490         if (ins->dest_type == nir_type_float32) {
 491                 struct bifrost_fma_fma pack = {
 492                         .src0 = bi_get_src(ins, regs, 0, true),
 493                         .src1 = bi_get_src(ins, regs, 1, true),
 494                         .src2 = bi_get_src(ins, regs, 2, true),
 495                         .src0_abs = ins->src_abs[0],
 496                         .src1_abs = ins->src_abs[1],
 497                         .src2_abs = ins->src_abs[2],
 498                         .src0_neg = negate_mul,
 499                         .src2_neg = ins->src_neg[2],
 500                         .outmod = ins->outmod,
 501                         .roundmode = ins->roundmode,
 502                         .op = BIFROST_FMA_OP_FMA
 503                 };
 504
 505                 RETURN_PACKED(pack);
 506         } else if (ins->dest_type == nir_type_float16) {
 507                 struct bifrost_fma_fma16 pack = {
 508                         .src0 = bi_get_src(ins, regs, 0, true),
 509                         .src1 = bi_get_src(ins, regs, 1, true),
 510                         .src2 = bi_get_src(ins, regs, 2, true),
 511                         .swizzle_0 = bi_swiz16(ins, 0),
 512                         .swizzle_1 = bi_swiz16(ins, 1),
 513                         .swizzle_2 = bi_swiz16(ins, 2),
 514                         .src0_neg = negate_mul,
 515                         .src2_neg = ins->src_neg[2],
 516                         .outmod = ins->outmod,
 517                         .roundmode = ins->roundmode,
 518                         .op = BIFROST_FMA_OP_FMA16
 519                 };
 520
 521                 RETURN_PACKED(pack);
 522         } else {
 523                 unreachable("Invalid fma dest type");
 524         }
 525 }
 526
 527 static unsigned
 528 bi_pack_fma_addmin_f32(bi_instruction *ins, struct bi_registers *regs)
 529 {
 530         unsigned op =
 531                 (ins->type == BI_ADD) ? BIFROST_FMA_OP_FADD32 :
 532                 (ins->op.minmax == BI_MINMAX_MIN) ? BIFROST_FMA_OP_FMIN32 :
 533                 BIFROST_FMA_OP_FMAX32;
 534
 535         struct bifrost_fma_add pack = {
 536                 .src0 = bi_get_src(ins, regs, 0, true),
 537                 .src1 = bi_get_src(ins, regs, 1, true),
 538                 .src0_abs = ins->src_abs[0],
 539                 .src1_abs = ins->src_abs[1],
 540                 .src0_neg = ins->src_neg[0],
 541                 .src1_neg = ins->src_neg[1],
 542                 .unk = 0x0,
 543                 .outmod = ins->outmod,
 544                 .roundmode = (ins->type == BI_ADD) ? ins->roundmode : ins->minmax,
 545                 .op = op
 546         };
 547
 548         RETURN_PACKED(pack);
 549 }
 550
 551 static unsigned
 552 bi_pack_fma_addmin_f16(bi_instruction *ins, struct bi_registers *regs)
 553 {
 554         unsigned op =
 555                 (ins->type == BI_ADD) ? BIFROST_FMA_OP_FADD16 :
 556                 (ins->op.minmax == BI_MINMAX_MIN) ? BIFROST_FMA_OP_FMIN16 :
 557                 BIFROST_FMA_OP_FMAX16;
 558
 559         /* Absolute values are packed in a quirky way. Let k = src1 < src0. Let
 560          * l be an auxiliary bit we encode. Then the hardware determines:
 561          *
 562          *      abs0 = l || k
 563          *      abs1 = l && k
 564          *
 565          * Since add/min/max are commutative, this saves a bit by using the
 566          * order of the operands as a bit (k). To pack this, first note:
 567          *
 568          *      (l && k) implies (l || k).
 569          *
 570          * That is, if the second argument is abs'd, then the first argument
 571          * also has abs. So there are three cases:
 572          *
 573          * Case 0: Neither src has absolute value. Then we have l = k = 0.
 574          *
 575          * Case 1: Exactly one src has absolute value. Assign that source to
 576          * src0 and the other source to src1. Compute k = src1 < src0 based on
 577          * that assignment. Then l = ~k.
 578          *
 579          * Case 2: Both sources have absolute value. Then we have l = k = 1.
 580          * Note to force k = 1 requires that (src1 < src0) OR (src0 < src1).
 581          * That is, this encoding is only valid if src1 and src0 are distinct.
 582          * This is a scheduling restriction (XXX); if an op of this type
 583          * requires both identical sources to have abs value, then we must
 584          * schedule to ADD (which does not use this ordering trick).
 585          */
 586
 587         unsigned abs_0 = ins->src_abs[0], abs_1 = ins->src_abs[1];
 588         unsigned src_0 = bi_get_src(ins, regs, 0, true);
 589         unsigned src_1 = bi_get_src(ins, regs, 1, true);
 590         bool l = false;
 591         bool flip = false;
 592
 593         if (!abs_0 && !abs_1) {
 594                 /* Force k = 0 <===> NOT(src1 < src0) */
 595                 flip = (src_1 < src_0);
 596         } else if (abs_0 && !abs_1) {
 597                 l = src_1 >= src_0;
 598         } else if (abs_1 && !abs_0) {
 599                 flip = true;
 600                 l = src_0 >= src_1;
 601         } else {
 602                 flip = (src_0 >= src_1);
 603                 l = true;
 604         }
 605
 606         struct bifrost_fma_add_minmax16 pack = {
 607                 .src0 = flip ? src_1 : src_0,
 608                 .src1 = flip ? src_0 : src_1,
 609                 .src0_neg = ins->src_neg[flip ? 1 : 0],
 610                 .src1_neg = ins->src_neg[flip ? 0 : 1],
 611                 .abs1 = l,
 612                 .outmod = ins->outmod,
 613                 .mode = (ins->type == BI_ADD) ? ins->roundmode : ins->minmax,
 614                 .op = op
 615         };
 616
 617         RETURN_PACKED(pack);
 618 }
 619
 620 static unsigned
 621 bi_pack_fma_addmin(bi_instruction *ins, struct bi_registers *regs)
 622 {
 623         if (ins->dest_type == nir_type_float32)
 624                 return bi_pack_fma_addmin_f32(ins, regs);
 625         else if(ins->dest_type == nir_type_float16)
 626                 return bi_pack_fma_addmin_f16(ins, regs);
 627         else
 628                 unreachable("Unknown FMA/ADD type");
 629 }
 630
 631 static unsigned
 632 bi_pack_fma_1src(bi_instruction *ins, struct bi_registers *regs, unsigned op)
 633 {
 634         struct bifrost_fma_inst pack = {
 635                 .src0 = bi_get_src(ins, regs, 0, true),
 636                 .op = op
 637         };
 638
 639         RETURN_PACKED(pack);
 640 }
 641
 642 static unsigned
 643 bi_pack_add_1src(bi_instruction *ins, struct bi_registers *regs, unsigned op)
 644 {
 645         struct bifrost_add_inst pack = {
 646                 .src0 = bi_get_src(ins, regs, 0, true),
 647                 .op = op
 648         };
 649
 650         RETURN_PACKED(pack);
 651 }
 652
 653 static enum bifrost_csel_cond
 654 bi_cond_to_csel(enum bi_cond cond, bool *flip, bool *invert, nir_alu_type T)
 655 {
 656         nir_alu_type B = nir_alu_type_get_base_type(T);
 657         unsigned idx = (B == nir_type_float) ? 0 :
 658                 ((B == nir_type_int) ? 1 : 2);
 659
 660         switch (cond){
 661         case BI_COND_LT:
 662                 *flip = true;
 663         case BI_COND_GT: {
 664                 const enum bifrost_csel_cond ops[] = {
 665                         BIFROST_FGT_F,
 666                         BIFROST_IGT_I,
 667                         BIFROST_UGT_I
 668                 };
 669
 670                 return ops[idx];
 671         }
 672         case BI_COND_LE:
 673                 *flip = true;
 674         case BI_COND_GE: {
 675                 const enum bifrost_csel_cond ops[] = {
 676                         BIFROST_FGE_F,
 677                         BIFROST_IGE_I,
 678                         BIFROST_UGE_I
 679                 };
 680
 681                 return ops[idx];
 682         }
 683         case BI_COND_NE:
 684                 *invert = true;
 685         case BI_COND_EQ: {
 686                 const enum bifrost_csel_cond ops[] = {
 687                         BIFROST_FEQ_F,
 688                         BIFROST_IEQ_F,
 689                         BIFROST_IEQ_F /* sign is irrelevant */
 690                 };
 691
 692                 return ops[idx];
 693         }
 694         default:
 695                 unreachable("Invalid op for csel");
 696         }
 697 }
 698
 699 static unsigned
 700 bi_pack_fma_csel(bi_instruction *ins, struct bi_registers *regs)
 701 {
 702         /* TODO: Use csel3 as well */
 703         bool flip = false, invert = false;
 704
 705         enum bifrost_csel_cond cond =
 706                 bi_cond_to_csel(ins->csel_cond, &flip, &invert, ins->src_types[0]);
 707
 708         unsigned size = nir_alu_type_get_type_size(ins->dest_type);
 709
 710         unsigned cmp_0 = (flip ? 1 : 0);
 711         unsigned cmp_1 = (flip ? 0 : 1);
 712         unsigned res_0 = (invert ? 3 : 2);
 713         unsigned res_1 = (invert ? 2 : 3);
 714
 715         struct bifrost_csel4 pack = {
 716                 .src0 = bi_get_src(ins, regs, cmp_0, true),
 717                 .src1 = bi_get_src(ins, regs, cmp_1, true),
 718                 .src2 = bi_get_src(ins, regs, res_0, true),
 719                 .src3 = bi_get_src(ins, regs, res_1, true),
 720                 .cond = cond,
 721                 .op = (size == 16) ? BIFROST_FMA_OP_CSEL4_V16 :
 722                         BIFROST_FMA_OP_CSEL4
 723         };
 724
 725         RETURN_PACKED(pack);
 726 }
 727
 728 /* We have a single convert opcode in the IR but a number of opcodes that could
 729  * come out. In particular we have native opcodes for:
 730  *
 731  * [ui]16 --> [fui]32           -- int16_to_32
 732  * f16     --> f32              -- float16_to_32
 733  * f32     --> f16              -- float32_to_16
 734  * f32     --> [ui]32           -- float32_to_int
 735  * [ui]32  --> f32              -- int_to_float32
 736  * [fui]16 --> [fui]16          -- f2i_i2f16
 737  */
 738
 739 static unsigned
 740 bi_pack_fma_convert(bi_instruction *ins, struct bi_registers *regs)
 741 {
 742         nir_alu_type from_base = nir_alu_type_get_base_type(ins->src_types[0]);
 743         unsigned from_size = nir_alu_type_get_type_size(ins->src_types[0]);
 744         bool from_unsigned = from_base == nir_type_uint;
 745
 746         nir_alu_type to_base = nir_alu_type_get_base_type(ins->dest_type);
 747         unsigned to_size = nir_alu_type_get_type_size(ins->dest_type);
 748         bool to_unsigned = to_base == nir_type_uint;
 749
 750         /* Sanity check */
 751         assert((from_base != to_base) || (from_size != to_size));
 752         assert((MAX2(from_size, to_size) / MIN2(from_size, to_size)) <= 2);
 753
 754         if (from_size == 16 && to_size == 16) {
 755                 /* f2i_i2f16 */
 756                 unreachable("i16 not yet implemented");
 757         } else if (from_size == 32 && to_size == 32) {
 758                 unsigned op = 0;
 759
 760                 if (from_base == nir_type_float) {
 761                         op = BIFROST_FMA_FLOAT32_TO_INT(to_unsigned);
 762                 } else {
 763                         op = BIFROST_FMA_INT_TO_FLOAT32(from_unsigned);
 764                 }
 765
 766                 return bi_pack_fma_1src(ins, regs, op);
 767         } else if (from_size == 16 && to_size == 32) {
 768                 bool from_y = ins->swizzle[0][0];
 769
 770                 if (from_base == nir_type_float) {
 771                         return bi_pack_fma_1src(ins, regs,
 772                                         BIFROST_FMA_FLOAT16_TO_32(from_y));
 773                 } else {
 774                         unreachable("i16 not yet implemented");
 775                 }
 776         } else if (from_size == 32 && to_size == 16) {
 777                 if (from_base == nir_type_float) {
 778                         /* TODO: second vectorized source? */
 779                         struct bifrost_fma_2src pack = {
 780                                 .src0 = bi_get_src(ins, regs, 0, true),
 781                                 .src1 = BIFROST_SRC_STAGE, /* 0 */
 782                                 .op = BIFROST_FMA_FLOAT32_TO_16
 783                         };
 784
 785                         RETURN_PACKED(pack);
 786                 } else {
 787                         unreachable("i16 not yet implemented");
 788                 }
 789         }
 790
 791         unreachable("Unknown convert");
 792 }
 793
 794 static unsigned
 795 bi_pack_fma(bi_clause *clause, bi_bundle bundle, struct bi_registers *regs)
 796 {
 797         if (!bundle.fma)
 798                 return BIFROST_FMA_NOP;
 799
 800         switch (bundle.fma->type) {
 801         case BI_ADD:
 802                 return bi_pack_fma_addmin(bundle.fma, regs);
 803         case BI_CMP:
 804         case BI_BITWISE:
 805                 return BIFROST_FMA_NOP;
 806         case BI_CONVERT:
 807                 return bi_pack_fma_convert(bundle.fma, regs);
 808         case BI_CSEL:
 809                 return bi_pack_fma_csel(bundle.fma, regs);
 810         case BI_FMA:
 811                 return bi_pack_fma_fma(bundle.fma, regs);
 812         case BI_FREXP:
 813         case BI_ISUB:
 814                 return BIFROST_FMA_NOP;
 815         case BI_MINMAX:
 816                 return bi_pack_fma_addmin(bundle.fma, regs);
 817         case BI_MOV:
 818                 return bi_pack_fma_1src(bundle.fma, regs, BIFROST_FMA_OP_MOV);
 819         case BI_SHIFT:
 820         case BI_SWIZZLE:
 821         case BI_ROUND:
 822                 return BIFROST_FMA_NOP;
 823         default:
 824                 unreachable("Cannot encode class as FMA");
 825         }
 826 }
 827
 828 static unsigned
 829 bi_pack_add_ld_vary(bi_clause *clause, bi_instruction *ins, struct bi_registers *regs)
 830 {
 831         unsigned size = nir_alu_type_get_type_size(ins->dest_type);
 832         assert(size == 32 || size == 16);
 833
 834         unsigned op = (size == 32) ?
 835                 BIFROST_ADD_OP_LD_VAR_32 :
 836                 BIFROST_ADD_OP_LD_VAR_16;
 837
 838         unsigned cmask = bi_from_bytemask(ins->writemask, size / 8);
 839         unsigned channels = util_bitcount(cmask);
 840         assert(cmask == ((1 << channels) - 1));
 841
 842         unsigned packed_addr = 0;
 843
 844         if (ins->src[0] & BIR_INDEX_CONSTANT) {
 845                 /* Direct uses address field directly */
 846                 packed_addr = bi_get_immediate(ins, ins->src[0]);
 847                 assert(packed_addr < 0b1000);
 848         } else {
 849                 /* Indirect gets an extra source */
 850                 packed_addr = bi_get_src(ins, regs, 0, false) | 0b11000;
 851         }
 852
 853         /* The destination is thrown in the data register */
 854         assert(ins->dest & BIR_INDEX_REGISTER);
 855         clause->data_register = ins->dest & ~BIR_INDEX_REGISTER;
 856
 857         assert(channels >= 1 && channels <= 4);
 858
 859         struct bifrost_ld_var pack = {
 860                 .src0 = bi_get_src(ins, regs, 1, false),
 861                 .addr = packed_addr,
 862                 .channels = MALI_POSITIVE(channels),
 863                 .interp_mode = ins->load_vary.interp_mode,
 864                 .reuse = ins->load_vary.reuse,
 865                 .flat = ins->load_vary.flat,
 866                 .op = op
 867         };
 868
 869         RETURN_PACKED(pack);
 870 }
 871
 872 static unsigned
 873 bi_pack_add_2src(bi_instruction *ins, struct bi_registers *regs, unsigned op)
 874 {
 875         struct bifrost_add_2src pack = {
 876                 .src0 = bi_get_src(ins, regs, 0, true),
 877                 .src1 = bi_get_src(ins, regs, 1, true),
 878                 .op = op
 879         };
 880
 881         RETURN_PACKED(pack);
 882 }
 883
 884 static unsigned
 885 bi_pack_add_addmin_f32(bi_instruction *ins, struct bi_registers *regs)
 886 {
 887         unsigned op =
 888                 (ins->type == BI_ADD) ? BIFROST_ADD_OP_FADD32 :
 889                 (ins->op.minmax == BI_MINMAX_MIN) ? BIFROST_ADD_OP_FMIN32 :
 890                 BIFROST_ADD_OP_FMAX32;
 891
 892         struct bifrost_add_faddmin pack = {
 893                 .src0 = bi_get_src(ins, regs, 0, true),
 894                 .src1 = bi_get_src(ins, regs, 1, true),
 895                 .src0_abs = ins->src_abs[0],
 896                 .src1_abs = ins->src_abs[1],
 897                 .src0_neg = ins->src_neg[0],
 898                 .src1_neg = ins->src_neg[1],
 899                 .outmod = ins->outmod,
 900                 .mode = (ins->type == BI_ADD) ? ins->roundmode : ins->minmax,
 901                 .op = op
 902         };
 903
 904         RETURN_PACKED(pack);
 905 }
 906
 907 static unsigned
 908 bi_pack_add_addmin(bi_instruction *ins, struct bi_registers *regs)
 909 {
 910         if (ins->dest_type == nir_type_float32)
 911                 return bi_pack_add_addmin_f32(ins, regs);
 912         else if(ins->dest_type == nir_type_float16)
 913                 unreachable("todo");
 914                 //return bi_pack_add_addmin_f16(ins, regs);
 915         else
 916                 unreachable("Unknown FMA/ADD type");
 917 }
 918
 919 static unsigned
 920 bi_pack_add_ld_ubo(bi_clause *clause, bi_instruction *ins, struct bi_registers *regs)
 921 {
 922         unsigned components = bi_load32_components(ins);
 923
 924         const unsigned ops[4] = {
 925                 BIFROST_ADD_OP_LD_UBO_1,
 926                 BIFROST_ADD_OP_LD_UBO_2,
 927                 BIFROST_ADD_OP_LD_UBO_3,
 928                 BIFROST_ADD_OP_LD_UBO_4
 929         };
 930
 931         bi_write_data_register(clause, ins);
 932         return bi_pack_add_2src(ins, regs, ops[components - 1]);
 933 }
 934
 935 static enum bifrost_ldst_type
 936 bi_pack_ldst_type(nir_alu_type T)
 937 {
 938         switch (T) {
 939         case nir_type_float16: return BIFROST_LDST_F16;
 940         case nir_type_float32: return BIFROST_LDST_F32;
 941         case nir_type_int32:   return BIFROST_LDST_I32;
 942         case nir_type_uint32:  return BIFROST_LDST_U32;
 943         default: unreachable("Invalid type loaded");
 944         }
 945 }
 946
 947 static unsigned
 948 bi_pack_add_ld_var_addr(bi_clause *clause, bi_instruction *ins, struct bi_registers *regs)
 949 {
 950         struct bifrost_ld_var_addr pack = {
 951                 .src0 = bi_get_src(ins, regs, 1, false),
 952                 .src1 = bi_get_src(ins, regs, 2, false),
 953                 .location = bi_get_immediate(ins, ins->src[0]),
 954                 .type = bi_pack_ldst_type(ins->src_types[3]),
 955                 .op = BIFROST_ADD_OP_LD_VAR_ADDR
 956         };
 957
 958         bi_write_data_register(clause, ins);
 959         RETURN_PACKED(pack);
 960 }
 961
 962 static unsigned
 963 bi_pack_add_ld_attr(bi_clause *clause, bi_instruction *ins, struct bi_registers *regs)
 964 {
 965         struct bifrost_ld_attr pack = {
 966                 .src0 = bi_get_src(ins, regs, 1, false),
 967                 .src1 = bi_get_src(ins, regs, 2, false),
 968                 .location = bi_get_immediate(ins, ins->src[0]),
 969                 .channels = MALI_POSITIVE(bi_load32_components(ins)),
 970                 .type = bi_pack_ldst_type(ins->dest_type),
 971                 .op = BIFROST_ADD_OP_LD_ATTR
 972         };
 973
 974         bi_write_data_register(clause, ins);
 975         RETURN_PACKED(pack);
 976 }
 977
 978 static unsigned
 979 bi_pack_add_st_vary(bi_clause *clause, bi_instruction *ins, struct bi_registers *regs)
 980 {
 981         assert(ins->store_channels >= 1 && ins->store_channels <= 4);
 982
 983         struct bifrost_st_vary pack = {
 984                 .src0 = bi_get_src(ins, regs, 1, false),
 985                 .src1 = bi_get_src(ins, regs, 2, false),
 986                 .src2 = bi_get_src(ins, regs, 3, false),
 987                 .channels = MALI_POSITIVE(ins->store_channels),
 988                 .op = BIFROST_ADD_OP_ST_VAR
 989         };
 990
 991         bi_read_data_register(clause, ins);
 992         RETURN_PACKED(pack);
 993 }
 994
 995 static unsigned
 996 bi_pack_add_atest(bi_clause *clause, bi_instruction *ins, struct bi_registers *regs)
 997 {
 998         /* TODO: fp16 */
 999         assert(ins->src_types[1] == nir_type_float32);
1000
1001         struct bifrost_add_atest pack = {
1002                 .src0 = bi_get_src(ins, regs, 0, false),
1003                 .src1 = bi_get_src(ins, regs, 1, false),
1004                 .component = 1, /* Set for fp32 */
1005                 .op = BIFROST_ADD_OP_ATEST,
1006         };
1007
1008         /* Despite *also* writing with the usual mechanism... quirky and
1009          * perhaps unnecessary, but let's match the blob */
1010         clause->data_register = ins->dest & ~BIR_INDEX_REGISTER;
1011
1012         RETURN_PACKED(pack);
1013 }
1014
1015 static unsigned
1016 bi_pack_add_blend(bi_instruction *ins, struct bi_registers *regs)
1017 {
1018         struct bifrost_add_inst pack = {
1019                 .src0 = bi_get_src(ins, regs, 0, false),
1020                 .op = BIFROST_ADD_OP_BLEND
1021         };
1022
1023         /* TODO: Pack location in uniform_const */
1024         assert(ins->blend_location == 0);
1025
1026         RETURN_PACKED(pack);
1027 }
1028
1029 static unsigned
1030 bi_pack_add_special(bi_instruction *ins, struct bi_registers *regs)
1031 {
1032         unsigned op = 0;
1033         bool fp16 = ins->dest_type == nir_type_float16;
1034         bool Y = ins->swizzle[0][0];
1035
1036         if (ins->op.special == BI_SPECIAL_FRCP) {
1037                 op = fp16 ?
1038                         (Y ? BIFROST_ADD_OP_FRCP_FAST_F16_Y :
1039                         BIFROST_ADD_OP_FRCP_FAST_F16_X) :
1040                         BIFROST_ADD_OP_FRCP_FAST_F32;
1041         } else {
1042                 op = fp16 ?
1043                         (Y ? BIFROST_ADD_OP_FRSQ_FAST_F16_Y :
1044                         BIFROST_ADD_OP_FRSQ_FAST_F16_X) :
1045                         BIFROST_ADD_OP_FRSQ_FAST_F32;
1046
1047         }
1048
1049         return bi_pack_add_1src(ins, regs, op);
1050 }
1051
1052 static unsigned
1053 bi_pack_add(bi_clause *clause, bi_bundle bundle, struct bi_registers *regs)
1054 {
1055         if (!bundle.add)
1056                 return BIFROST_ADD_NOP;
1057
1058         switch (bundle.add->type) {
1059         case BI_ADD:
1060                 return BIFROST_ADD_NOP;
1061         case BI_ATEST:
1062                 return bi_pack_add_atest(clause, bundle.add, regs);
1063         case BI_BRANCH:
1064         case BI_CMP:
1065                 return BIFROST_ADD_NOP;
1066         case BI_BLEND:
1067                 return bi_pack_add_blend(bundle.add, regs);
1068         case BI_BITWISE:
1069         case BI_CONVERT:
1070         case BI_DISCARD:
1071         case BI_FREXP:
1072         case BI_ISUB:
1073         case BI_LOAD:
1074                 return BIFROST_ADD_NOP;
1075         case BI_LOAD_ATTR:
1076                 return bi_pack_add_ld_attr(clause, bundle.add, regs);
1077         case BI_LOAD_UNIFORM:
1078                 return bi_pack_add_ld_ubo(clause, bundle.add, regs);
1079         case BI_LOAD_VAR:
1080                 return bi_pack_add_ld_vary(clause, bundle.add, regs);
1081         case BI_LOAD_VAR_ADDRESS:
1082                 return bi_pack_add_ld_var_addr(clause, bundle.add, regs);
1083         case BI_MINMAX:
1084                 return bi_pack_add_addmin(bundle.add, regs);
1085         case BI_MOV:
1086         case BI_SHIFT:
1087         case BI_STORE:
1088                 return BIFROST_ADD_NOP;
1089         case BI_STORE_VAR:
1090                 return bi_pack_add_st_vary(clause, bundle.add, regs);
1091         case BI_SPECIAL:
1092                 return bi_pack_add_special(bundle.add, regs);
1093         case BI_SWIZZLE:
1094         case BI_TEX:
1095         case BI_ROUND:
1096                 return BIFROST_ADD_NOP;
1097         default:
1098                 unreachable("Cannot encode class as ADD");
1099         }
1100 }
1101
1102 struct bi_packed_bundle {
1103         uint64_t lo;
1104         uint64_t hi;
1105 };
1106
1107 static struct bi_packed_bundle
1108 bi_pack_bundle(bi_clause *clause, bi_bundle bundle, bi_bundle prev, bool first_bundle)
1109 {
1110         struct bi_registers regs = bi_assign_ports(bundle, prev);
1111         bi_assign_uniform_constant(clause, &regs, bundle);
1112         regs.first_instruction = first_bundle;
1113
1114         uint64_t reg = bi_pack_registers(regs);
1115         uint64_t fma = bi_pack_fma(clause, bundle, &regs);
1116         uint64_t add = bi_pack_add(clause, bundle, &regs);
1117
1118         struct bi_packed_bundle packed = {
1119                 .lo = reg | (fma << 35) | ((add & 0b111111) << 58),
1120                 .hi = add >> 6
1121         };
1122
1123         return packed;
1124 }
1125
1126 /* Packs the next two constants as a dedicated constant quadword at the end of
1127  * the clause, returning the number packed. */
1128
1129 static unsigned
1130 bi_pack_constants(bi_context *ctx, bi_clause *clause,
1131                 unsigned index,
1132                 struct util_dynarray *emission)
1133 {
1134         /* After these two, are we done? Determines tag */
1135         bool done = clause->constant_count <= (index + 2);
1136         bool only = clause->constant_count <= (index + 1);
1137
1138         /* TODO: Pos */
1139         assert(index == 0 && clause->bundle_count == 1);
1140
1141         struct bifrost_fmt_constant quad = {
1142                 .pos = 0, /* TODO */
1143                 .tag = done ? BIFROST_FMTC_FINAL : BIFROST_FMTC_CONSTANTS,
1144                 .imm_1 = clause->constants[index + 0] >> 4,
1145                 .imm_2 = only ? 0 : clause->constants[index + 1] >> 4
1146         };
1147
1148         /* XXX: On G71, Connor observed that the difference of the top 4 bits
1149          * of the second constant with the first must be less than 8, otherwise
1150          * we have to swap them. I am not able to reproduce this on G52,
1151          * further investigation needed. Possibly an errata. XXX */
1152
1153         util_dynarray_append(emission, struct bifrost_fmt_constant, quad);
1154
1155         return 2;
1156 }
1157
1158 static void
1159 bi_pack_clause(bi_context *ctx, bi_clause *clause, bi_clause *next,
1160                 struct util_dynarray *emission)
1161 {
1162         struct bi_packed_bundle ins_1 = bi_pack_bundle(clause, clause->bundles[0], clause->bundles[0], true);
1163         assert(clause->bundle_count == 1);
1164
1165         /* Used to decide if we elide writes */
1166         bool is_fragment = ctx->stage == MESA_SHADER_FRAGMENT;
1167
1168         /* State for packing constants throughout */
1169         unsigned constant_index = 0;
1170
1171         struct bifrost_fmt1 quad_1 = {
1172                 .tag = clause->constant_count ? BIFROST_FMT1_CONSTANTS : BIFROST_FMT1_FINAL,
1173                 .header = bi_pack_header(clause, next, is_fragment),
1174                 .ins_1 = ins_1.lo,
1175                 .ins_2 = ins_1.hi & ((1 << 11) - 1),
1176                 .ins_0 = (ins_1.hi >> 11) & 0b111,
1177         };
1178
1179         util_dynarray_append(emission, struct bifrost_fmt1, quad_1);
1180
1181         /* Pack the remaining constants */
1182
1183         while (constant_index < clause->constant_count) {
1184                 constant_index += bi_pack_constants(ctx, clause,
1185                                 constant_index, emission);
1186         }
1187 }
1188
1189 static bi_clause *
1190 bi_next_clause(bi_context *ctx, pan_block *block, bi_clause *clause)
1191 {
1192         /* Try the next clause in this block */
1193         if (clause->link.next != &((bi_block *) block)->clauses)
1194                 return list_first_entry(&(clause->link), bi_clause, link);
1195
1196         /* Try the next block, or the one after that if it's empty, etc .*/
1197         pan_block *next_block = pan_next_block(block);
1198
1199         bi_foreach_block_from(ctx, next_block, block) {
1200                 bi_block *blk = (bi_block *) block;
1201
1202                 if (!list_is_empty(&blk->clauses))
1203                         return list_first_entry(&(blk->clauses), bi_clause, link);
1204         }
1205
1206         return NULL;
1207 }
1208
1209 void
1210 bi_pack(bi_context *ctx, struct util_dynarray *emission)
1211 {
1212         util_dynarray_init(emission, NULL);
1213
1214         bi_foreach_block(ctx, _block) {
1215                 bi_block *block = (bi_block *) _block;
1216
1217                 bi_foreach_clause_in_block(block, clause) {
1218                         bi_clause *next = bi_next_clause(ctx, _block, clause);
1219                         bi_pack_clause(ctx, clause, next, emission);
1220                 }
1221         }
1222 }