src/intel/compiler/brw_eu.h

   1 /*
   2  Copyright (C) Intel Corp.  2006.  All Rights Reserved.
   3  Intel funded Tungsten Graphics to
   4  develop this 3D driver.
   5
   6  Permission is hereby granted, free of charge, to any person obtaining
   7  a copy of this software and associated documentation files (the
   8  "Software"), to deal in the Software without restriction, including
   9  without limitation the rights to use, copy, modify, merge, publish,
  10  distribute, sublicense, and/or sell copies of the Software, and to
  11  permit persons to whom the Software is furnished to do so, subject to
  12  the following conditions:
  13
  14  The above copyright notice and this permission notice (including the
  15  next paragraph) shall be included in all copies or substantial
  16  portions of the Software.
  17
  18  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  19  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  21  IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
  22  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  23  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  24  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25
  26  **********************************************************************/
  27  /*
  28   * Authors:
  29   *   Keith Whitwell <keithw@vmware.com>
  30   */
  31
  32
  33 #ifndef BRW_EU_H
  34 #define BRW_EU_H
  35
  36 #include <stdbool.h>
  37 #include <stdio.h>
  38 #include "brw_inst.h"
  39 #include "brw_eu_defines.h"
  40 #include "brw_reg.h"
  41 #include "brw_disasm_info.h"
  42
  43 #ifdef __cplusplus
  44 extern "C" {
  45 #endif
  46
  47 #define BRW_EU_MAX_INSN_STACK 5
  48
  49 struct brw_insn_state {
  50    /* One of BRW_EXECUTE_* */
  51    unsigned exec_size:3;
  52
  53    /* Group in units of channels */
  54    unsigned group:5;
  55
  56    /* Compression control on gen4-5 */
  57    bool compressed:1;
  58
  59    /* One of BRW_MASK_* */
  60    unsigned mask_control:1;
  61
  62    /* Scheduling info for Gen12+ */
  63    struct tgl_swsb swsb;
  64
  65    bool saturate:1;
  66
  67    /* One of BRW_ALIGN_* */
  68    unsigned access_mode:1;
  69
  70    /* One of BRW_PREDICATE_* */
  71    enum brw_predicate predicate:4;
  72
  73    bool pred_inv:1;
  74
  75    /* Flag subreg.  Bottom bit is subreg, top bit is reg */
  76    unsigned flag_subreg:2;
  77
  78    bool acc_wr_control:1;
  79 };
  80
  81
  82 /* A helper for accessing the last instruction emitted.  This makes it easy
  83  * to set various bits on an instruction without having to create temporary
  84  * variable and assign the emitted instruction to those.
  85  */
  86 #define brw_last_inst (&p->store[p->nr_insn - 1])
  87
  88 struct brw_codegen {
  89    brw_inst *store;
  90    int store_size;
  91    unsigned nr_insn;
  92    unsigned int next_insn_offset;
  93
  94    void *mem_ctx;
  95
  96    /* Allow clients to push/pop instruction state:
  97     */
  98    struct brw_insn_state stack[BRW_EU_MAX_INSN_STACK];
  99    struct brw_insn_state *current;
 100
 101    /** Whether or not the user wants automatic exec sizes
 102     *
 103     * If true, codegen will try to automatically infer the exec size of an
 104     * instruction from the width of the destination register.  If false, it
 105     * will take whatever is set by brw_set_default_exec_size verbatim.
 106     *
 107     * This is set to true by default in brw_init_codegen.
 108     */
 109    bool automatic_exec_sizes;
 110
 111    bool single_program_flow;
 112    const struct gen_device_info *devinfo;
 113
 114    /* Control flow stacks:
 115     * - if_stack contains IF and ELSE instructions which must be patched
 116     *   (and popped) once the matching ENDIF instruction is encountered.
 117     *
 118     *   Just store the instruction pointer(an index).
 119     */
 120    int *if_stack;
 121    int if_stack_depth;
 122    int if_stack_array_size;
 123
 124    /**
 125     * loop_stack contains the instruction pointers of the starts of loops which
 126     * must be patched (and popped) once the matching WHILE instruction is
 127     * encountered.
 128     */
 129    int *loop_stack;
 130    /**
 131     * pre-gen6, the BREAK and CONT instructions had to tell how many IF/ENDIF
 132     * blocks they were popping out of, to fix up the mask stack.  This tracks
 133     * the IF/ENDIF nesting in each current nested loop level.
 134     */
 135    int *if_depth_in_loop;
 136    int loop_stack_depth;
 137    int loop_stack_array_size;
 138 };
 139
 140 void brw_pop_insn_state( struct brw_codegen *p );
 141 void brw_push_insn_state( struct brw_codegen *p );
 142 unsigned brw_get_default_exec_size(struct brw_codegen *p);
 143 unsigned brw_get_default_group(struct brw_codegen *p);
 144 unsigned brw_get_default_access_mode(struct brw_codegen *p);
 145 struct tgl_swsb brw_get_default_swsb(struct brw_codegen *p);
 146 void brw_set_default_exec_size(struct brw_codegen *p, unsigned value);
 147 void brw_set_default_mask_control( struct brw_codegen *p, unsigned value );
 148 void brw_set_default_saturate( struct brw_codegen *p, bool enable );
 149 void brw_set_default_access_mode( struct brw_codegen *p, unsigned access_mode );
 150 void brw_inst_set_compression(const struct gen_device_info *devinfo,
 151                               brw_inst *inst, bool on);
 152 void brw_set_default_compression(struct brw_codegen *p, bool on);
 153 void brw_inst_set_group(const struct gen_device_info *devinfo,
 154                         brw_inst *inst, unsigned group);
 155 void brw_set_default_group(struct brw_codegen *p, unsigned group);
 156 void brw_set_default_compression_control(struct brw_codegen *p, enum brw_compression c);
 157 void brw_set_default_predicate_control(struct brw_codegen *p, enum brw_predicate pc);
 158 void brw_set_default_predicate_inverse(struct brw_codegen *p, bool predicate_inverse);
 159 void brw_set_default_flag_reg(struct brw_codegen *p, int reg, int subreg);
 160 void brw_set_default_acc_write_control(struct brw_codegen *p, unsigned value);
 161 void brw_set_default_swsb(struct brw_codegen *p, struct tgl_swsb value);
 162
 163 void brw_init_codegen(const struct gen_device_info *, struct brw_codegen *p,
 164                       void *mem_ctx);
 165 int brw_disassemble_inst(FILE *file, const struct gen_device_info *devinfo,
 166                          const struct brw_inst *inst, bool is_compacted);
 167 void brw_disassemble(const struct gen_device_info *devinfo,
 168                      const void *assembly, int start, int end, FILE *out);
 169 const unsigned *brw_get_program( struct brw_codegen *p, unsigned *sz );
 170
 171 bool brw_try_override_assembly(struct brw_codegen *p, int start_offset,
 172                                const char *identifier);
 173
 174 brw_inst *brw_next_insn(struct brw_codegen *p, unsigned opcode);
 175 void brw_set_dest(struct brw_codegen *p, brw_inst *insn, struct brw_reg dest);
 176 void brw_set_src0(struct brw_codegen *p, brw_inst *insn, struct brw_reg reg);
 177
 178 void gen6_resolve_implied_move(struct brw_codegen *p,
 179                                struct brw_reg *src,
 180                                unsigned msg_reg_nr);
 181
 182 /* Helpers for regular instructions:
 183  */
 184 #define ALU1(OP)                                \
 185 brw_inst *brw_##OP(struct brw_codegen *p,       \
 186               struct brw_reg dest,              \
 187               struct brw_reg src0);
 188
 189 #define ALU2(OP)                                \
 190 brw_inst *brw_##OP(struct brw_codegen *p,       \
 191               struct brw_reg dest,              \
 192               struct brw_reg src0,              \
 193               struct brw_reg src1);
 194
 195 #define ALU3(OP)                                \
 196 brw_inst *brw_##OP(struct brw_codegen *p,       \
 197               struct brw_reg dest,              \
 198               struct brw_reg src0,              \
 199               struct brw_reg src1,              \
 200               struct brw_reg src2);
 201
 202 #define ROUND(OP) \
 203 void brw_##OP(struct brw_codegen *p, struct brw_reg dest, struct brw_reg src0);
 204
 205 ALU1(MOV)
 206 ALU2(SEL)
 207 ALU1(NOT)
 208 ALU2(AND)
 209 ALU2(OR)
 210 ALU2(XOR)
 211 ALU2(SHR)
 212 ALU2(SHL)
 213 ALU1(DIM)
 214 ALU2(ASR)
 215 ALU2(ROL)
 216 ALU2(ROR)
 217 ALU3(CSEL)
 218 ALU1(F32TO16)
 219 ALU1(F16TO32)
 220 ALU2(ADD)
 221 ALU2(AVG)
 222 ALU2(MUL)
 223 ALU1(FRC)
 224 ALU1(RNDD)
 225 ALU2(MAC)
 226 ALU2(MACH)
 227 ALU1(LZD)
 228 ALU2(DP4)
 229 ALU2(DPH)
 230 ALU2(DP3)
 231 ALU2(DP2)
 232 ALU2(LINE)
 233 ALU2(PLN)
 234 ALU3(MAD)
 235 ALU3(LRP)
 236 ALU1(BFREV)
 237 ALU3(BFE)
 238 ALU2(BFI1)
 239 ALU3(BFI2)
 240 ALU1(FBH)
 241 ALU1(FBL)
 242 ALU1(CBIT)
 243 ALU2(ADDC)
 244 ALU2(SUBB)
 245 ALU2(MAC)
 246
 247 ROUND(RNDZ)
 248 ROUND(RNDE)
 249
 250 #undef ALU1
 251 #undef ALU2
 252 #undef ALU3
 253 #undef ROUND
 254
 255
 256 /* Helpers for SEND instruction:
 257  */
 258
 259 /**
 260  * Construct a message descriptor immediate with the specified common
 261  * descriptor controls.
 262  */
 263 static inline uint32_t
 264 brw_message_desc(const struct gen_device_info *devinfo,
 265                  unsigned msg_length,
 266                  unsigned response_length,
 267                  bool header_present)
 268 {
 269    if (devinfo->gen >= 5) {
 270       return (SET_BITS(msg_length, 28, 25) |
 271               SET_BITS(response_length, 24, 20) |
 272               SET_BITS(header_present, 19, 19));
 273    } else {
 274       return (SET_BITS(msg_length, 23, 20) |
 275               SET_BITS(response_length, 19, 16));
 276    }
 277 }
 278
 279 static inline unsigned
 280 brw_message_desc_mlen(const struct gen_device_info *devinfo, uint32_t desc)
 281 {
 282    if (devinfo->gen >= 5)
 283       return GET_BITS(desc, 28, 25);
 284    else
 285       return GET_BITS(desc, 23, 20);
 286 }
 287
 288 static inline unsigned
 289 brw_message_desc_rlen(const struct gen_device_info *devinfo, uint32_t desc)
 290 {
 291    if (devinfo->gen >= 5)
 292       return GET_BITS(desc, 24, 20);
 293    else
 294       return GET_BITS(desc, 19, 16);
 295 }
 296
 297 static inline bool
 298 brw_message_desc_header_present(ASSERTED const struct gen_device_info *devinfo,
 299                                 uint32_t desc)
 300 {
 301    assert(devinfo->gen >= 5);
 302    return GET_BITS(desc, 19, 19);
 303 }
 304
 305 static inline unsigned
 306 brw_message_ex_desc(UNUSED const struct gen_device_info *devinfo,
 307                     unsigned ex_msg_length)
 308 {
 309    return SET_BITS(ex_msg_length, 9, 6);
 310 }
 311
 312 static inline unsigned
 313 brw_message_ex_desc_ex_mlen(UNUSED const struct gen_device_info *devinfo,
 314                             uint32_t ex_desc)
 315 {
 316    return GET_BITS(ex_desc, 9, 6);
 317 }
 318
 319 /**
 320  * Construct a message descriptor immediate with the specified sampler
 321  * function controls.
 322  */
 323 static inline uint32_t
 324 brw_sampler_desc(const struct gen_device_info *devinfo,
 325                  unsigned binding_table_index,
 326                  unsigned sampler,
 327                  unsigned msg_type,
 328                  unsigned simd_mode,
 329                  unsigned return_format)
 330 {
 331    const unsigned desc = (SET_BITS(binding_table_index, 7, 0) |
 332                           SET_BITS(sampler, 11, 8));
 333    if (devinfo->gen >= 7)
 334       return (desc | SET_BITS(msg_type, 16, 12) |
 335               SET_BITS(simd_mode, 18, 17));
 336    else if (devinfo->gen >= 5)
 337       return (desc | SET_BITS(msg_type, 15, 12) |
 338               SET_BITS(simd_mode, 17, 16));
 339    else if (devinfo->is_g4x)
 340       return desc | SET_BITS(msg_type, 15, 12);
 341    else
 342       return (desc | SET_BITS(return_format, 13, 12) |
 343               SET_BITS(msg_type, 15, 14));
 344 }
 345
 346 static inline unsigned
 347 brw_sampler_desc_binding_table_index(UNUSED const struct gen_device_info *devinfo,
 348                                      uint32_t desc)
 349 {
 350    return GET_BITS(desc, 7, 0);
 351 }
 352
 353 static inline unsigned
 354 brw_sampler_desc_sampler(UNUSED const struct gen_device_info *devinfo, uint32_t desc)
 355 {
 356    return GET_BITS(desc, 11, 8);
 357 }
 358
 359 static inline unsigned
 360 brw_sampler_desc_msg_type(const struct gen_device_info *devinfo, uint32_t desc)
 361 {
 362    if (devinfo->gen >= 7)
 363       return GET_BITS(desc, 16, 12);
 364    else if (devinfo->gen >= 5 || devinfo->is_g4x)
 365       return GET_BITS(desc, 15, 12);
 366    else
 367       return GET_BITS(desc, 15, 14);
 368 }
 369
 370 static inline unsigned
 371 brw_sampler_desc_simd_mode(const struct gen_device_info *devinfo, uint32_t desc)
 372 {
 373    assert(devinfo->gen >= 5);
 374    if (devinfo->gen >= 7)
 375       return GET_BITS(desc, 18, 17);
 376    else
 377       return GET_BITS(desc, 17, 16);
 378 }
 379
 380 static  inline unsigned
 381 brw_sampler_desc_return_format(ASSERTED const struct gen_device_info *devinfo,
 382                                uint32_t desc)
 383 {
 384    assert(devinfo->gen == 4 && !devinfo->is_g4x);
 385    return GET_BITS(desc, 13, 12);
 386 }
 387
 388 /**
 389  * Construct a message descriptor for the dataport
 390  */
 391 static inline uint32_t
 392 brw_dp_desc(const struct gen_device_info *devinfo,
 393             unsigned binding_table_index,
 394             unsigned msg_type,
 395             unsigned msg_control)
 396 {
 397    /* Prior to gen6, things are too inconsistent; use the dp_read/write_desc
 398     * helpers instead.
 399     */
 400    assert(devinfo->gen >= 6);
 401    const unsigned desc = SET_BITS(binding_table_index, 7, 0);
 402    if (devinfo->gen >= 8) {
 403       return (desc | SET_BITS(msg_control, 13, 8) |
 404               SET_BITS(msg_type, 18, 14));
 405    } else if (devinfo->gen >= 7) {
 406       return (desc | SET_BITS(msg_control, 13, 8) |
 407               SET_BITS(msg_type, 17, 14));
 408    } else {
 409       return (desc | SET_BITS(msg_control, 12, 8) |
 410               SET_BITS(msg_type, 16, 13));
 411    }
 412 }
 413
 414 static inline unsigned
 415 brw_dp_desc_binding_table_index(UNUSED const struct gen_device_info *devinfo,
 416                                 uint32_t desc)
 417 {
 418    return GET_BITS(desc, 7, 0);
 419 }
 420
 421 static inline unsigned
 422 brw_dp_desc_msg_type(const struct gen_device_info *devinfo, uint32_t desc)
 423 {
 424    assert(devinfo->gen >= 6);
 425    if (devinfo->gen >= 8)
 426       return GET_BITS(desc, 18, 14);
 427    else if (devinfo->gen >= 7)
 428       return GET_BITS(desc, 17, 14);
 429    else
 430       return GET_BITS(desc, 16, 13);
 431 }
 432
 433 static inline unsigned
 434 brw_dp_desc_msg_control(const struct gen_device_info *devinfo, uint32_t desc)
 435 {
 436    assert(devinfo->gen >= 6);
 437    if (devinfo->gen >= 7)
 438       return GET_BITS(desc, 13, 8);
 439    else
 440       return GET_BITS(desc, 12, 8);
 441 }
 442
 443 /**
 444  * Construct a message descriptor immediate with the specified dataport read
 445  * function controls.
 446  */
 447 static inline uint32_t
 448 brw_dp_read_desc(const struct gen_device_info *devinfo,
 449                  unsigned binding_table_index,
 450                  unsigned msg_control,
 451                  unsigned msg_type,
 452                  unsigned target_cache)
 453 {
 454    if (devinfo->gen >= 6)
 455       return brw_dp_desc(devinfo, binding_table_index, msg_type, msg_control);
 456    else if (devinfo->gen >= 5 || devinfo->is_g4x)
 457       return (SET_BITS(binding_table_index, 7, 0) |
 458               SET_BITS(msg_control, 10, 8) |
 459               SET_BITS(msg_type, 13, 11) |
 460               SET_BITS(target_cache, 15, 14));
 461    else
 462       return (SET_BITS(binding_table_index, 7, 0) |
 463               SET_BITS(msg_control, 11, 8) |
 464               SET_BITS(msg_type, 13, 12) |
 465               SET_BITS(target_cache, 15, 14));
 466 }
 467
 468 static inline unsigned
 469 brw_dp_read_desc_msg_type(const struct gen_device_info *devinfo, uint32_t desc)
 470 {
 471    if (devinfo->gen >= 6)
 472       return brw_dp_desc_msg_type(devinfo, desc);
 473    else if (devinfo->gen >= 5 || devinfo->is_g4x)
 474       return GET_BITS(desc, 13, 11);
 475    else
 476       return GET_BITS(desc, 13, 12);
 477 }
 478
 479 static inline unsigned
 480 brw_dp_read_desc_msg_control(const struct gen_device_info *devinfo,
 481                              uint32_t desc)
 482 {
 483    if (devinfo->gen >= 6)
 484       return brw_dp_desc_msg_control(devinfo, desc);
 485    else if (devinfo->gen >= 5 || devinfo->is_g4x)
 486       return GET_BITS(desc, 10, 8);
 487    else
 488       return GET_BITS(desc, 11, 8);
 489 }
 490
 491 /**
 492  * Construct a message descriptor immediate with the specified dataport write
 493  * function controls.
 494  */
 495 static inline uint32_t
 496 brw_dp_write_desc(const struct gen_device_info *devinfo,
 497                   unsigned binding_table_index,
 498                   unsigned msg_control,
 499                   unsigned msg_type,
 500                   unsigned last_render_target,
 501                   unsigned send_commit_msg)
 502 {
 503    assert(devinfo->gen <= 6 || !send_commit_msg);
 504    if (devinfo->gen >= 6)
 505       return brw_dp_desc(devinfo, binding_table_index, msg_type, msg_control) |
 506              SET_BITS(last_render_target, 12, 12) |
 507              SET_BITS(send_commit_msg, 17, 17);
 508    else
 509       return (SET_BITS(binding_table_index, 7, 0) |
 510               SET_BITS(msg_control, 11, 8) |
 511               SET_BITS(last_render_target, 11, 11) |
 512               SET_BITS(msg_type, 14, 12) |
 513               SET_BITS(send_commit_msg, 15, 15));
 514 }
 515
 516 static inline unsigned
 517 brw_dp_write_desc_msg_type(const struct gen_device_info *devinfo,
 518                            uint32_t desc)
 519 {
 520    if (devinfo->gen >= 6)
 521       return brw_dp_desc_msg_type(devinfo, desc);
 522    else
 523       return GET_BITS(desc, 14, 12);
 524 }
 525
 526 static inline unsigned
 527 brw_dp_write_desc_msg_control(const struct gen_device_info *devinfo,
 528                               uint32_t desc)
 529 {
 530    if (devinfo->gen >= 6)
 531       return brw_dp_desc_msg_control(devinfo, desc);
 532    else
 533       return GET_BITS(desc, 11, 8);
 534 }
 535
 536 static inline bool
 537 brw_dp_write_desc_last_render_target(const struct gen_device_info *devinfo,
 538                                      uint32_t desc)
 539 {
 540    if (devinfo->gen >= 6)
 541       return GET_BITS(desc, 12, 12);
 542    else
 543       return GET_BITS(desc, 11, 11);
 544 }
 545
 546 static inline bool
 547 brw_dp_write_desc_write_commit(const struct gen_device_info *devinfo,
 548                                uint32_t desc)
 549 {
 550    assert(devinfo->gen <= 6);
 551    if (devinfo->gen >= 6)
 552       return GET_BITS(desc, 17, 17);
 553    else
 554       return GET_BITS(desc, 15, 15);
 555 }
 556
 557 /**
 558  * Construct a message descriptor immediate with the specified dataport
 559  * surface function controls.
 560  */
 561 static inline uint32_t
 562 brw_dp_surface_desc(const struct gen_device_info *devinfo,
 563                     unsigned msg_type,
 564                     unsigned msg_control)
 565 {
 566    assert(devinfo->gen >= 7);
 567    /* We'll OR in the binding table index later */
 568    return brw_dp_desc(devinfo, 0, msg_type, msg_control);
 569 }
 570
 571 static inline uint32_t
 572 brw_dp_untyped_atomic_desc(const struct gen_device_info *devinfo,
 573                            unsigned exec_size, /**< 0 for SIMD4x2 */
 574                            unsigned atomic_op,
 575                            bool response_expected)
 576 {
 577    assert(exec_size <= 8 || exec_size == 16);
 578
 579    unsigned msg_type;
 580    if (devinfo->gen >= 8 || devinfo->is_haswell) {
 581       if (exec_size > 0) {
 582          msg_type = HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP;
 583       } else {
 584          msg_type = HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP_SIMD4X2;
 585       }
 586    } else {
 587       msg_type = GEN7_DATAPORT_DC_UNTYPED_ATOMIC_OP;
 588    }
 589
 590    const unsigned msg_control =
 591       SET_BITS(atomic_op, 3, 0) |
 592       SET_BITS(0 < exec_size && exec_size <= 8, 4, 4) |
 593       SET_BITS(response_expected, 5, 5);
 594
 595    return brw_dp_surface_desc(devinfo, msg_type, msg_control);
 596 }
 597
 598 static inline uint32_t
 599 brw_dp_untyped_atomic_float_desc(const struct gen_device_info *devinfo,
 600                                  unsigned exec_size,
 601                                  unsigned atomic_op,
 602                                  bool response_expected)
 603 {
 604    assert(exec_size <= 8 || exec_size == 16);
 605    assert(devinfo->gen >= 9);
 606
 607    assert(exec_size > 0);
 608    const unsigned msg_type = GEN9_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_FLOAT_OP;
 609
 610    const unsigned msg_control =
 611       SET_BITS(atomic_op, 1, 0) |
 612       SET_BITS(exec_size <= 8, 4, 4) |
 613       SET_BITS(response_expected, 5, 5);
 614
 615    return brw_dp_surface_desc(devinfo, msg_type, msg_control);
 616 }
 617
 618 static inline unsigned
 619 brw_mdc_cmask(unsigned num_channels)
 620 {
 621    /* See also MDC_CMASK in the SKL PRM Vol 2d. */
 622    return 0xf & (0xf << num_channels);
 623 }
 624
 625 static inline uint32_t
 626 brw_dp_untyped_surface_rw_desc(const struct gen_device_info *devinfo,
 627                                unsigned exec_size, /**< 0 for SIMD4x2 */
 628                                unsigned num_channels,
 629                                bool write)
 630 {
 631    assert(exec_size <= 8 || exec_size == 16);
 632
 633    unsigned msg_type;
 634    if (write) {
 635       if (devinfo->gen >= 8 || devinfo->is_haswell) {
 636          msg_type = HSW_DATAPORT_DC_PORT1_UNTYPED_SURFACE_WRITE;
 637       } else {
 638          msg_type = GEN7_DATAPORT_DC_UNTYPED_SURFACE_WRITE;
 639       }
 640    } else {
 641       /* Read */
 642       if (devinfo->gen >= 8 || devinfo->is_haswell) {
 643          msg_type = HSW_DATAPORT_DC_PORT1_UNTYPED_SURFACE_READ;
 644       } else {
 645          msg_type = GEN7_DATAPORT_DC_UNTYPED_SURFACE_READ;
 646       }
 647    }
 648
 649    /* SIMD4x2 is only valid for read messages on IVB; use SIMD8 instead */
 650    if (write && devinfo->gen == 7 && !devinfo->is_haswell && exec_size == 0)
 651       exec_size = 8;
 652
 653    /* See also MDC_SM3 in the SKL PRM Vol 2d. */
 654    const unsigned simd_mode = exec_size == 0 ? 0 : /* SIMD4x2 */
 655                               exec_size <= 8 ? 2 : 1;
 656
 657    const unsigned msg_control =
 658       SET_BITS(brw_mdc_cmask(num_channels), 3, 0) |
 659       SET_BITS(simd_mode, 5, 4);
 660
 661    return brw_dp_surface_desc(devinfo, msg_type, msg_control);
 662 }
 663
 664 static inline unsigned
 665 brw_mdc_ds(unsigned bit_size)
 666 {
 667    switch (bit_size) {
 668    case 8:
 669       return GEN7_BYTE_SCATTERED_DATA_ELEMENT_BYTE;
 670    case 16:
 671       return GEN7_BYTE_SCATTERED_DATA_ELEMENT_WORD;
 672    case 32:
 673       return GEN7_BYTE_SCATTERED_DATA_ELEMENT_DWORD;
 674    default:
 675       unreachable("Unsupported bit_size for byte scattered messages");
 676    }
 677 }
 678
 679 static inline uint32_t
 680 brw_dp_byte_scattered_rw_desc(const struct gen_device_info *devinfo,
 681                               unsigned exec_size,
 682                               unsigned bit_size,
 683                               bool write)
 684 {
 685    assert(exec_size <= 8 || exec_size == 16);
 686
 687    assert(devinfo->gen > 7 || devinfo->is_haswell);
 688    const unsigned msg_type =
 689       write ? HSW_DATAPORT_DC_PORT0_BYTE_SCATTERED_WRITE :
 690               HSW_DATAPORT_DC_PORT0_BYTE_SCATTERED_READ;
 691
 692    assert(exec_size > 0);
 693    const unsigned msg_control =
 694       SET_BITS(exec_size == 16, 0, 0) |
 695       SET_BITS(brw_mdc_ds(bit_size), 3, 2);
 696
 697    return brw_dp_surface_desc(devinfo, msg_type, msg_control);
 698 }
 699
 700 static inline uint32_t
 701 brw_dp_dword_scattered_rw_desc(const struct gen_device_info *devinfo,
 702                                unsigned exec_size,
 703                                bool write)
 704 {
 705    assert(exec_size == 8 || exec_size == 16);
 706
 707    unsigned msg_type;
 708    if (write) {
 709       if (devinfo->gen >= 6) {
 710          msg_type = GEN6_DATAPORT_WRITE_MESSAGE_DWORD_SCATTERED_WRITE;
 711       } else {
 712          msg_type = BRW_DATAPORT_WRITE_MESSAGE_DWORD_SCATTERED_WRITE;
 713       }
 714    } else {
 715       if (devinfo->gen >= 7) {
 716          msg_type = GEN7_DATAPORT_DC_DWORD_SCATTERED_READ;
 717       } else if (devinfo->gen > 4 || devinfo->is_g4x) {
 718          msg_type = G45_DATAPORT_READ_MESSAGE_DWORD_SCATTERED_READ;
 719       } else {
 720          msg_type = BRW_DATAPORT_READ_MESSAGE_DWORD_SCATTERED_READ;
 721       }
 722    }
 723
 724    const unsigned msg_control =
 725       SET_BITS(1, 1, 1) | /* Legacy SIMD Mode */
 726       SET_BITS(exec_size == 16, 0, 0);
 727
 728    return brw_dp_surface_desc(devinfo, msg_type, msg_control);
 729 }
 730
 731 static inline uint32_t
 732 brw_dp_a64_untyped_surface_rw_desc(const struct gen_device_info *devinfo,
 733                                    unsigned exec_size, /**< 0 for SIMD4x2 */
 734                                    unsigned num_channels,
 735                                    bool write)
 736 {
 737    assert(exec_size <= 8 || exec_size == 16);
 738    assert(devinfo->gen >= 8);
 739
 740    unsigned msg_type =
 741       write ? GEN8_DATAPORT_DC_PORT1_A64_UNTYPED_SURFACE_WRITE :
 742               GEN8_DATAPORT_DC_PORT1_A64_UNTYPED_SURFACE_READ;
 743
 744    /* See also MDC_SM3 in the SKL PRM Vol 2d. */
 745    const unsigned simd_mode = exec_size == 0 ? 0 : /* SIMD4x2 */
 746                               exec_size <= 8 ? 2 : 1;
 747
 748    const unsigned msg_control =
 749       SET_BITS(brw_mdc_cmask(num_channels), 3, 0) |
 750       SET_BITS(simd_mode, 5, 4);
 751
 752    return brw_dp_desc(devinfo, BRW_BTI_STATELESS, msg_type, msg_control);
 753 }
 754
 755 /**
 756  * Calculate the data size (see MDC_A64_DS in the "Structures" volume of the
 757  * Skylake PRM).
 758  */
 759 static inline uint32_t
 760 brw_mdc_a64_ds(unsigned elems)
 761 {
 762    switch (elems) {
 763    case 1:  return 0;
 764    case 2:  return 1;
 765    case 4:  return 2;
 766    case 8:  return 3;
 767    default:
 768       unreachable("Unsupported elmeent count for A64 scattered message");
 769    }
 770 }
 771
 772 static inline uint32_t
 773 brw_dp_a64_byte_scattered_rw_desc(const struct gen_device_info *devinfo,
 774                                   unsigned exec_size, /**< 0 for SIMD4x2 */
 775                                   unsigned bit_size,
 776                                   bool write)
 777 {
 778    assert(exec_size <= 8 || exec_size == 16);
 779    assert(devinfo->gen >= 8);
 780
 781    unsigned msg_type =
 782       write ? GEN8_DATAPORT_DC_PORT1_A64_SCATTERED_WRITE :
 783               GEN9_DATAPORT_DC_PORT1_A64_SCATTERED_READ;
 784
 785    const unsigned msg_control =
 786       SET_BITS(GEN8_A64_SCATTERED_SUBTYPE_BYTE, 1, 0) |
 787       SET_BITS(brw_mdc_a64_ds(bit_size / 8), 3, 2) |
 788       SET_BITS(exec_size == 16, 4, 4);
 789
 790    return brw_dp_desc(devinfo, BRW_BTI_STATELESS, msg_type, msg_control);
 791 }
 792
 793 static inline uint32_t
 794 brw_dp_a64_untyped_atomic_desc(const struct gen_device_info *devinfo,
 795                                ASSERTED unsigned exec_size, /**< 0 for SIMD4x2 */
 796                                unsigned bit_size,
 797                                unsigned atomic_op,
 798                                bool response_expected)
 799 {
 800    assert(exec_size == 8);
 801    assert(devinfo->gen >= 8);
 802    assert(bit_size == 32 || bit_size == 64);
 803
 804    const unsigned msg_type = GEN8_DATAPORT_DC_PORT1_A64_UNTYPED_ATOMIC_OP;
 805
 806    const unsigned msg_control =
 807       SET_BITS(atomic_op, 3, 0) |
 808       SET_BITS(bit_size == 64, 4, 4) |
 809       SET_BITS(response_expected, 5, 5);
 810
 811    return brw_dp_desc(devinfo, BRW_BTI_STATELESS, msg_type, msg_control);
 812 }
 813
 814 static inline uint32_t
 815 brw_dp_a64_untyped_atomic_float_desc(const struct gen_device_info *devinfo,
 816                                      ASSERTED unsigned exec_size,
 817                                      unsigned atomic_op,
 818                                      bool response_expected)
 819 {
 820    assert(exec_size == 8);
 821    assert(devinfo->gen >= 9);
 822
 823    assert(exec_size > 0);
 824    const unsigned msg_type = GEN9_DATAPORT_DC_PORT1_A64_UNTYPED_ATOMIC_FLOAT_OP;
 825
 826    const unsigned msg_control =
 827       SET_BITS(atomic_op, 1, 0) |
 828       SET_BITS(response_expected, 5, 5);
 829
 830    return brw_dp_desc(devinfo, BRW_BTI_STATELESS, msg_type, msg_control);
 831 }
 832
 833 static inline uint32_t
 834 brw_dp_typed_atomic_desc(const struct gen_device_info *devinfo,
 835                          unsigned exec_size,
 836                          unsigned exec_group,
 837                          unsigned atomic_op,
 838                          bool response_expected)
 839 {
 840    assert(exec_size > 0 || exec_group == 0);
 841    assert(exec_group % 8 == 0);
 842
 843    unsigned msg_type;
 844    if (devinfo->gen >= 8 || devinfo->is_haswell) {
 845       if (exec_size == 0) {
 846          msg_type = HSW_DATAPORT_DC_PORT1_TYPED_ATOMIC_OP_SIMD4X2;
 847       } else {
 848          msg_type = HSW_DATAPORT_DC_PORT1_TYPED_ATOMIC_OP;
 849       }
 850    } else {
 851       /* SIMD4x2 typed surface R/W messages only exist on HSW+ */
 852       assert(exec_size > 0);
 853       msg_type = GEN7_DATAPORT_RC_TYPED_ATOMIC_OP;
 854    }
 855
 856    const bool high_sample_mask = (exec_group / 8) % 2 == 1;
 857
 858    const unsigned msg_control =
 859       SET_BITS(atomic_op, 3, 0) |
 860       SET_BITS(high_sample_mask, 4, 4) |
 861       SET_BITS(response_expected, 5, 5);
 862
 863    return brw_dp_surface_desc(devinfo, msg_type, msg_control);
 864 }
 865
 866 static inline uint32_t
 867 brw_dp_typed_surface_rw_desc(const struct gen_device_info *devinfo,
 868                              unsigned exec_size,
 869                              unsigned exec_group,
 870                              unsigned num_channels,
 871                              bool write)
 872 {
 873    assert(exec_size > 0 || exec_group == 0);
 874    assert(exec_group % 8 == 0);
 875
 876    /* Typed surface reads and writes don't support SIMD16 */
 877    assert(exec_size <= 8);
 878
 879    unsigned msg_type;
 880    if (write) {
 881       if (devinfo->gen >= 8 || devinfo->is_haswell) {
 882          msg_type = HSW_DATAPORT_DC_PORT1_TYPED_SURFACE_WRITE;
 883       } else {
 884          msg_type = GEN7_DATAPORT_RC_TYPED_SURFACE_WRITE;
 885       }
 886    } else {
 887       if (devinfo->gen >= 8 || devinfo->is_haswell) {
 888          msg_type = HSW_DATAPORT_DC_PORT1_TYPED_SURFACE_READ;
 889       } else {
 890          msg_type = GEN7_DATAPORT_RC_TYPED_SURFACE_READ;
 891       }
 892    }
 893
 894    /* See also MDC_SG3 in the SKL PRM Vol 2d. */
 895    unsigned msg_control;
 896    if (devinfo->gen >= 8 || devinfo->is_haswell) {
 897       /* See also MDC_SG3 in the SKL PRM Vol 2d. */
 898       const unsigned slot_group = exec_size == 0 ? 0 : /* SIMD4x2 */
 899                                   1 + ((exec_group / 8) % 2);
 900
 901       msg_control =
 902          SET_BITS(brw_mdc_cmask(num_channels), 3, 0) |
 903          SET_BITS(slot_group, 5, 4);
 904    } else {
 905       /* SIMD4x2 typed surface R/W messages only exist on HSW+ */
 906       assert(exec_size > 0);
 907       const unsigned slot_group = ((exec_group / 8) % 2);
 908
 909       msg_control =
 910          SET_BITS(brw_mdc_cmask(num_channels), 3, 0) |
 911          SET_BITS(slot_group, 5, 5);
 912    }
 913
 914    return brw_dp_surface_desc(devinfo, msg_type, msg_control);
 915 }
 916
 917 /**
 918  * Construct a message descriptor immediate with the specified pixel
 919  * interpolator function controls.
 920  */
 921 static inline uint32_t
 922 brw_pixel_interp_desc(UNUSED const struct gen_device_info *devinfo,
 923                       unsigned msg_type,
 924                       bool noperspective,
 925                       unsigned simd_mode,
 926                       unsigned slot_group)
 927 {
 928    return (SET_BITS(slot_group, 11, 11) |
 929            SET_BITS(msg_type, 13, 12) |
 930            SET_BITS(!!noperspective, 14, 14) |
 931            SET_BITS(simd_mode, 16, 16));
 932 }
 933
 934 void brw_urb_WRITE(struct brw_codegen *p,
 935                    struct brw_reg dest,
 936                    unsigned msg_reg_nr,
 937                    struct brw_reg src0,
 938                    enum brw_urb_write_flags flags,
 939                    unsigned msg_length,
 940                    unsigned response_length,
 941                    unsigned offset,
 942                    unsigned swizzle);
 943
 944 /**
 945  * Send message to shared unit \p sfid with a possibly indirect descriptor \p
 946  * desc.  If \p desc is not an immediate it will be transparently loaded to an
 947  * address register using an OR instruction.
 948  */
 949 void
 950 brw_send_indirect_message(struct brw_codegen *p,
 951                           unsigned sfid,
 952                           struct brw_reg dst,
 953                           struct brw_reg payload,
 954                           struct brw_reg desc,
 955                           unsigned desc_imm,
 956                           bool eot);
 957
 958 void
 959 brw_send_indirect_split_message(struct brw_codegen *p,
 960                                 unsigned sfid,
 961                                 struct brw_reg dst,
 962                                 struct brw_reg payload0,
 963                                 struct brw_reg payload1,
 964                                 struct brw_reg desc,
 965                                 unsigned desc_imm,
 966                                 struct brw_reg ex_desc,
 967                                 unsigned ex_desc_imm,
 968                                 bool eot);
 969
 970 void brw_ff_sync(struct brw_codegen *p,
 971                    struct brw_reg dest,
 972                    unsigned msg_reg_nr,
 973                    struct brw_reg src0,
 974                    bool allocate,
 975                    unsigned response_length,
 976                    bool eot);
 977
 978 void brw_svb_write(struct brw_codegen *p,
 979                    struct brw_reg dest,
 980                    unsigned msg_reg_nr,
 981                    struct brw_reg src0,
 982                    unsigned binding_table_index,
 983                    bool   send_commit_msg);
 984
 985 brw_inst *brw_fb_WRITE(struct brw_codegen *p,
 986                        struct brw_reg payload,
 987                        struct brw_reg implied_header,
 988                        unsigned msg_control,
 989                        unsigned binding_table_index,
 990                        unsigned msg_length,
 991                        unsigned response_length,
 992                        bool eot,
 993                        bool last_render_target,
 994                        bool header_present);
 995
 996 brw_inst *gen9_fb_READ(struct brw_codegen *p,
 997                        struct brw_reg dst,
 998                        struct brw_reg payload,
 999                        unsigned binding_table_index,
1000                        unsigned msg_length,
1001                        unsigned response_length,
1002                        bool per_sample);
1003
1004 void brw_SAMPLE(struct brw_codegen *p,
1005                 struct brw_reg dest,
1006                 unsigned msg_reg_nr,
1007                 struct brw_reg src0,
1008                 unsigned binding_table_index,
1009                 unsigned sampler,
1010                 unsigned msg_type,
1011                 unsigned response_length,
1012                 unsigned msg_length,
1013                 unsigned header_present,
1014                 unsigned simd_mode,
1015                 unsigned return_format);
1016
1017 void brw_adjust_sampler_state_pointer(struct brw_codegen *p,
1018                                       struct brw_reg header,
1019                                       struct brw_reg sampler_index);
1020
1021 void gen4_math(struct brw_codegen *p,
1022                struct brw_reg dest,
1023                unsigned function,
1024                unsigned msg_reg_nr,
1025                struct brw_reg src,
1026                unsigned precision );
1027
1028 void gen6_math(struct brw_codegen *p,
1029                struct brw_reg dest,
1030                unsigned function,
1031                struct brw_reg src0,
1032                struct brw_reg src1);
1033
1034 void brw_oword_block_read(struct brw_codegen *p,
1035                           struct brw_reg dest,
1036                           struct brw_reg mrf,
1037                           uint32_t offset,
1038                           uint32_t bind_table_index);
1039
1040 unsigned brw_scratch_surface_idx(const struct brw_codegen *p);
1041
1042 void brw_oword_block_read_scratch(struct brw_codegen *p,
1043                                   struct brw_reg dest,
1044                                   struct brw_reg mrf,
1045                                   int num_regs,
1046                                   unsigned offset);
1047
1048 void brw_oword_block_write_scratch(struct brw_codegen *p,
1049                                    struct brw_reg mrf,
1050                                    int num_regs,
1051                                    unsigned offset);
1052
1053 void gen7_block_read_scratch(struct brw_codegen *p,
1054                              struct brw_reg dest,
1055                              int num_regs,
1056                              unsigned offset);
1057
1058 void brw_shader_time_add(struct brw_codegen *p,
1059                          struct brw_reg payload,
1060                          uint32_t surf_index);
1061
1062 /**
1063  * Return the generation-specific jump distance scaling factor.
1064  *
1065  * Given the number of instructions to jump, we need to scale by
1066  * some number to obtain the actual jump distance to program in an
1067  * instruction.
1068  */
1069 static inline unsigned
1070 brw_jump_scale(const struct gen_device_info *devinfo)
1071 {
1072    /* Broadwell measures jump targets in bytes. */
1073    if (devinfo->gen >= 8)
1074       return 16;
1075
1076    /* Ironlake and later measure jump targets in 64-bit data chunks (in order
1077     * (to support compaction), so each 128-bit instruction requires 2 chunks.
1078     */
1079    if (devinfo->gen >= 5)
1080       return 2;
1081
1082    /* Gen4 simply uses the number of 128-bit instructions. */
1083    return 1;
1084 }
1085
1086 void brw_barrier(struct brw_codegen *p, struct brw_reg src);
1087
1088 /* If/else/endif.  Works by manipulating the execution flags on each
1089  * channel.
1090  */
1091 brw_inst *brw_IF(struct brw_codegen *p, unsigned execute_size);
1092 brw_inst *gen6_IF(struct brw_codegen *p, enum brw_conditional_mod conditional,
1093                   struct brw_reg src0, struct brw_reg src1);
1094
1095 void brw_ELSE(struct brw_codegen *p);
1096 void brw_ENDIF(struct brw_codegen *p);
1097
1098 /* DO/WHILE loops:
1099  */
1100 brw_inst *brw_DO(struct brw_codegen *p, unsigned execute_size);
1101
1102 brw_inst *brw_WHILE(struct brw_codegen *p);
1103
1104 brw_inst *brw_BREAK(struct brw_codegen *p);
1105 brw_inst *brw_CONT(struct brw_codegen *p);
1106 brw_inst *gen6_HALT(struct brw_codegen *p);
1107
1108 /* Forward jumps:
1109  */
1110 void brw_land_fwd_jump(struct brw_codegen *p, int jmp_insn_idx);
1111
1112 brw_inst *brw_JMPI(struct brw_codegen *p, struct brw_reg index,
1113                    unsigned predicate_control);
1114
1115 void brw_NOP(struct brw_codegen *p);
1116
1117 void brw_WAIT(struct brw_codegen *p);
1118
1119 void brw_SYNC(struct brw_codegen *p, enum tgl_sync_function func);
1120
1121 /* Special case: there is never a destination, execution size will be
1122  * taken from src0:
1123  */
1124 void brw_CMP(struct brw_codegen *p,
1125              struct brw_reg dest,
1126              unsigned conditional,
1127              struct brw_reg src0,
1128              struct brw_reg src1);
1129
1130 void
1131 brw_untyped_atomic(struct brw_codegen *p,
1132                    struct brw_reg dst,
1133                    struct brw_reg payload,
1134                    struct brw_reg surface,
1135                    unsigned atomic_op,
1136                    unsigned msg_length,
1137                    bool response_expected,
1138                    bool header_present);
1139
1140 void
1141 brw_untyped_surface_read(struct brw_codegen *p,
1142                          struct brw_reg dst,
1143                          struct brw_reg payload,
1144                          struct brw_reg surface,
1145                          unsigned msg_length,
1146                          unsigned num_channels);
1147
1148 void
1149 brw_untyped_surface_write(struct brw_codegen *p,
1150                           struct brw_reg payload,
1151                           struct brw_reg surface,
1152                           unsigned msg_length,
1153                           unsigned num_channels,
1154                           bool header_present);
1155
1156 void
1157 brw_memory_fence(struct brw_codegen *p,
1158                  struct brw_reg dst,
1159                  struct brw_reg src,
1160                  enum opcode send_op,
1161                  bool stall,
1162                  unsigned bti);
1163
1164 void
1165 brw_pixel_interpolator_query(struct brw_codegen *p,
1166                              struct brw_reg dest,
1167                              struct brw_reg mrf,
1168                              bool noperspective,
1169                              unsigned mode,
1170                              struct brw_reg data,
1171                              unsigned msg_length,
1172                              unsigned response_length);
1173
1174 void
1175 brw_find_live_channel(struct brw_codegen *p,
1176                       struct brw_reg dst,
1177                       struct brw_reg mask);
1178
1179 void
1180 brw_broadcast(struct brw_codegen *p,
1181               struct brw_reg dst,
1182               struct brw_reg src,
1183               struct brw_reg idx);
1184
1185 void
1186 brw_float_controls_mode(struct brw_codegen *p,
1187                         unsigned mode, unsigned mask);
1188
1189 /***********************************************************************
1190  * brw_eu_util.c:
1191  */
1192
1193 void brw_copy_indirect_to_indirect(struct brw_codegen *p,
1194                                    struct brw_indirect dst_ptr,
1195                                    struct brw_indirect src_ptr,
1196                                    unsigned count);
1197
1198 void brw_copy_from_indirect(struct brw_codegen *p,
1199                             struct brw_reg dst,
1200                             struct brw_indirect ptr,
1201                             unsigned count);
1202
1203 void brw_copy4(struct brw_codegen *p,
1204                struct brw_reg dst,
1205                struct brw_reg src,
1206                unsigned count);
1207
1208 void brw_copy8(struct brw_codegen *p,
1209                struct brw_reg dst,
1210                struct brw_reg src,
1211                unsigned count);
1212
1213 void brw_math_invert( struct brw_codegen *p,
1214                       struct brw_reg dst,
1215                       struct brw_reg src);
1216
1217 void brw_set_src1(struct brw_codegen *p, brw_inst *insn, struct brw_reg reg);
1218
1219 void brw_set_desc_ex(struct brw_codegen *p, brw_inst *insn,
1220                      unsigned desc, unsigned ex_desc);
1221
1222 static inline void
1223 brw_set_desc(struct brw_codegen *p, brw_inst *insn, unsigned desc)
1224 {
1225    brw_set_desc_ex(p, insn, desc, 0);
1226 }
1227
1228 void brw_set_uip_jip(struct brw_codegen *p, int start_offset);
1229
1230 enum brw_conditional_mod brw_negate_cmod(enum brw_conditional_mod cmod);
1231 enum brw_conditional_mod brw_swap_cmod(enum brw_conditional_mod cmod);
1232
1233 /* brw_eu_compact.c */
1234 void brw_init_compaction_tables(const struct gen_device_info *devinfo);
1235 void brw_compact_instructions(struct brw_codegen *p, int start_offset,
1236                               struct disasm_info *disasm);
1237 void brw_uncompact_instruction(const struct gen_device_info *devinfo,
1238                                brw_inst *dst, brw_compact_inst *src);
1239 bool brw_try_compact_instruction(const struct gen_device_info *devinfo,
1240                                  brw_compact_inst *dst, const brw_inst *src);
1241
1242 void brw_debug_compact_uncompact(const struct gen_device_info *devinfo,
1243                                  brw_inst *orig, brw_inst *uncompacted);
1244
1245 /* brw_eu_validate.c */
1246 bool brw_validate_instructions(const struct gen_device_info *devinfo,
1247                                const void *assembly, int start_offset, int end_offset,
1248                                struct disasm_info *disasm);
1249
1250 static inline int
1251 next_offset(const struct gen_device_info *devinfo, void *store, int offset)
1252 {
1253    brw_inst *insn = (brw_inst *)((char *)store + offset);
1254
1255    if (brw_inst_cmpt_control(devinfo, insn))
1256       return offset + 8;
1257    else
1258       return offset + 16;
1259 }
1260
1261 struct opcode_desc {
1262    unsigned ir;
1263    unsigned hw;
1264    const char *name;
1265    int nsrc;
1266    int ndst;
1267    int gens;
1268 };
1269
1270 const struct opcode_desc *
1271 brw_opcode_desc(const struct gen_device_info *devinfo, enum opcode opcode);
1272
1273 const struct opcode_desc *
1274 brw_opcode_desc_from_hw(const struct gen_device_info *devinfo, unsigned hw);
1275
1276 static inline unsigned
1277 brw_opcode_encode(const struct gen_device_info *devinfo, enum opcode opcode)
1278 {
1279    return brw_opcode_desc(devinfo, opcode)->hw;
1280 }
1281
1282 static inline enum opcode
1283 brw_opcode_decode(const struct gen_device_info *devinfo, unsigned hw)
1284 {
1285    const struct opcode_desc *desc = brw_opcode_desc_from_hw(devinfo, hw);
1286    return desc ? (enum opcode)desc->ir : BRW_OPCODE_ILLEGAL;
1287 }
1288
1289 static inline void
1290 brw_inst_set_opcode(const struct gen_device_info *devinfo,
1291                     brw_inst *inst, enum opcode opcode)
1292 {
1293    brw_inst_set_hw_opcode(devinfo, inst, brw_opcode_encode(devinfo, opcode));
1294 }
1295
1296 static inline enum opcode
1297 brw_inst_opcode(const struct gen_device_info *devinfo, const brw_inst *inst)
1298 {
1299    return brw_opcode_decode(devinfo, brw_inst_hw_opcode(devinfo, inst));
1300 }
1301
1302 static inline bool
1303 is_3src(const struct gen_device_info *devinfo, enum opcode opcode)
1304 {
1305    const struct opcode_desc *desc = brw_opcode_desc(devinfo, opcode);
1306    return desc && desc->nsrc == 3;
1307 }
1308
1309 /** Maximum SEND message length */
1310 #define BRW_MAX_MSG_LENGTH 15
1311
1312 /** First MRF register used by pull loads */
1313 #define FIRST_SPILL_MRF(gen) ((gen) == 6 ? 21 : 13)
1314
1315 /** First MRF register used by spills */
1316 #define FIRST_PULL_LOAD_MRF(gen) ((gen) == 6 ? 16 : 13)
1317
1318 #ifdef __cplusplus
1319 }
1320 #endif
1321
1322 #endif