src/intel/compiler/brw_eu.h

   1 /*
   2  Copyright (C) Intel Corp.  2006.  All Rights Reserved.
   3  Intel funded Tungsten Graphics to
   4  develop this 3D driver.
   5
   6  Permission is hereby granted, free of charge, to any person obtaining
   7  a copy of this software and associated documentation files (the
   8  "Software"), to deal in the Software without restriction, including
   9  without limitation the rights to use, copy, modify, merge, publish,
  10  distribute, sublicense, and/or sell copies of the Software, and to
  11  permit persons to whom the Software is furnished to do so, subject to
  12  the following conditions:
  13
  14  The above copyright notice and this permission notice (including the
  15  next paragraph) shall be included in all copies or substantial
  16  portions of the Software.
  17
  18  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  19  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  21  IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
  22  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  23  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  24  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25
  26  **********************************************************************/
  27  /*
  28   * Authors:
  29   *   Keith Whitwell <keithw@vmware.com>
  30   */
  31
  32
  33 #ifndef BRW_EU_H
  34 #define BRW_EU_H
  35
  36 #include <stdbool.h>
  37 #include <stdio.h>
  38 #include "brw_inst.h"
  39 #include "brw_eu_defines.h"
  40 #include "brw_reg.h"
  41 #include "brw_disasm_info.h"
  42
  43 #ifdef __cplusplus
  44 extern "C" {
  45 #endif
  46
  47 #define BRW_EU_MAX_INSN_STACK 5
  48
  49 struct brw_insn_state {
  50    /* One of BRW_EXECUTE_* */
  51    unsigned exec_size:3;
  52
  53    /* Group in units of channels */
  54    unsigned group:5;
  55
  56    /* Compression control on gen4-5 */
  57    bool compressed:1;
  58
  59    /* One of BRW_MASK_* */
  60    unsigned mask_control:1;
  61
  62    /* Scheduling info for Gen12+ */
  63    struct tgl_swsb swsb;
  64
  65    bool saturate:1;
  66
  67    /* One of BRW_ALIGN_* */
  68    unsigned access_mode:1;
  69
  70    /* One of BRW_PREDICATE_* */
  71    enum brw_predicate predicate:4;
  72
  73    bool pred_inv:1;
  74
  75    /* Flag subreg.  Bottom bit is subreg, top bit is reg */
  76    unsigned flag_subreg:2;
  77
  78    bool acc_wr_control:1;
  79 };
  80
  81
  82 /* A helper for accessing the last instruction emitted.  This makes it easy
  83  * to set various bits on an instruction without having to create temporary
  84  * variable and assign the emitted instruction to those.
  85  */
  86 #define brw_last_inst (&p->store[p->nr_insn - 1])
  87
  88 struct brw_codegen {
  89    brw_inst *store;
  90    int store_size;
  91    unsigned nr_insn;
  92    unsigned int next_insn_offset;
  93
  94    void *mem_ctx;
  95
  96    /* Allow clients to push/pop instruction state:
  97     */
  98    struct brw_insn_state stack[BRW_EU_MAX_INSN_STACK];
  99    struct brw_insn_state *current;
 100
 101    /** Whether or not the user wants automatic exec sizes
 102     *
 103     * If true, codegen will try to automatically infer the exec size of an
 104     * instruction from the width of the destination register.  If false, it
 105     * will take whatever is set by brw_set_default_exec_size verbatim.
 106     *
 107     * This is set to true by default in brw_init_codegen.
 108     */
 109    bool automatic_exec_sizes;
 110
 111    bool single_program_flow;
 112    const struct gen_device_info *devinfo;
 113
 114    /* Control flow stacks:
 115     * - if_stack contains IF and ELSE instructions which must be patched
 116     *   (and popped) once the matching ENDIF instruction is encountered.
 117     *
 118     *   Just store the instruction pointer(an index).
 119     */
 120    int *if_stack;
 121    int if_stack_depth;
 122    int if_stack_array_size;
 123
 124    /**
 125     * loop_stack contains the instruction pointers of the starts of loops which
 126     * must be patched (and popped) once the matching WHILE instruction is
 127     * encountered.
 128     */
 129    int *loop_stack;
 130    /**
 131     * pre-gen6, the BREAK and CONT instructions had to tell how many IF/ENDIF
 132     * blocks they were popping out of, to fix up the mask stack.  This tracks
 133     * the IF/ENDIF nesting in each current nested loop level.
 134     */
 135    int *if_depth_in_loop;
 136    int loop_stack_depth;
 137    int loop_stack_array_size;
 138 };
 139
 140 void brw_pop_insn_state( struct brw_codegen *p );
 141 void brw_push_insn_state( struct brw_codegen *p );
 142 unsigned brw_get_default_exec_size(struct brw_codegen *p);
 143 unsigned brw_get_default_group(struct brw_codegen *p);
 144 unsigned brw_get_default_access_mode(struct brw_codegen *p);
 145 struct tgl_swsb brw_get_default_swsb(struct brw_codegen *p);
 146 void brw_set_default_exec_size(struct brw_codegen *p, unsigned value);
 147 void brw_set_default_mask_control( struct brw_codegen *p, unsigned value );
 148 void brw_set_default_saturate( struct brw_codegen *p, bool enable );
 149 void brw_set_default_access_mode( struct brw_codegen *p, unsigned access_mode );
 150 void brw_inst_set_compression(const struct gen_device_info *devinfo,
 151                               brw_inst *inst, bool on);
 152 void brw_set_default_compression(struct brw_codegen *p, bool on);
 153 void brw_inst_set_group(const struct gen_device_info *devinfo,
 154                         brw_inst *inst, unsigned group);
 155 void brw_set_default_group(struct brw_codegen *p, unsigned group);
 156 void brw_set_default_compression_control(struct brw_codegen *p, enum brw_compression c);
 157 void brw_set_default_predicate_control(struct brw_codegen *p, enum brw_predicate pc);
 158 void brw_set_default_predicate_inverse(struct brw_codegen *p, bool predicate_inverse);
 159 void brw_set_default_flag_reg(struct brw_codegen *p, int reg, int subreg);
 160 void brw_set_default_acc_write_control(struct brw_codegen *p, unsigned value);
 161 void brw_set_default_swsb(struct brw_codegen *p, struct tgl_swsb value);
 162
 163 void brw_init_codegen(const struct gen_device_info *, struct brw_codegen *p,
 164                       void *mem_ctx);
 165 int brw_disassemble_inst(FILE *file, const struct gen_device_info *devinfo,
 166                          const struct brw_inst *inst, bool is_compacted);
 167 void brw_disassemble(const struct gen_device_info *devinfo,
 168                      const void *assembly, int start, int end, FILE *out);
 169 const unsigned *brw_get_program( struct brw_codegen *p, unsigned *sz );
 170
 171 bool brw_try_override_assembly(struct brw_codegen *p, int start_offset,
 172                                const char *identifier);
 173
 174 brw_inst *brw_next_insn(struct brw_codegen *p, unsigned opcode);
 175 void brw_set_dest(struct brw_codegen *p, brw_inst *insn, struct brw_reg dest);
 176 void brw_set_src0(struct brw_codegen *p, brw_inst *insn, struct brw_reg reg);
 177
 178 void gen6_resolve_implied_move(struct brw_codegen *p,
 179                                struct brw_reg *src,
 180                                unsigned msg_reg_nr);
 181
 182 /* Helpers for regular instructions:
 183  */
 184 #define ALU1(OP)                                \
 185 brw_inst *brw_##OP(struct brw_codegen *p,       \
 186               struct brw_reg dest,              \
 187               struct brw_reg src0);
 188
 189 #define ALU2(OP)                                \
 190 brw_inst *brw_##OP(struct brw_codegen *p,       \
 191               struct brw_reg dest,              \
 192               struct brw_reg src0,              \
 193               struct brw_reg src1);
 194
 195 #define ALU3(OP)                                \
 196 brw_inst *brw_##OP(struct brw_codegen *p,       \
 197               struct brw_reg dest,              \
 198               struct brw_reg src0,              \
 199               struct brw_reg src1,              \
 200               struct brw_reg src2);
 201
 202 ALU1(MOV)
 203 ALU2(SEL)
 204 ALU1(NOT)
 205 ALU2(AND)
 206 ALU2(OR)
 207 ALU2(XOR)
 208 ALU2(SHR)
 209 ALU2(SHL)
 210 ALU1(DIM)
 211 ALU2(ASR)
 212 ALU2(ROL)
 213 ALU2(ROR)
 214 ALU3(CSEL)
 215 ALU1(F32TO16)
 216 ALU1(F16TO32)
 217 ALU2(ADD)
 218 ALU2(AVG)
 219 ALU2(MUL)
 220 ALU1(FRC)
 221 ALU1(RNDD)
 222 ALU1(RNDE)
 223 ALU1(RNDZ)
 224 ALU2(MAC)
 225 ALU2(MACH)
 226 ALU1(LZD)
 227 ALU2(DP4)
 228 ALU2(DPH)
 229 ALU2(DP3)
 230 ALU2(DP2)
 231 ALU2(LINE)
 232 ALU2(PLN)
 233 ALU3(MAD)
 234 ALU3(LRP)
 235 ALU1(BFREV)
 236 ALU3(BFE)
 237 ALU2(BFI1)
 238 ALU3(BFI2)
 239 ALU1(FBH)
 240 ALU1(FBL)
 241 ALU1(CBIT)
 242 ALU2(ADDC)
 243 ALU2(SUBB)
 244 ALU2(MAC)
 245
 246 #undef ALU1
 247 #undef ALU2
 248 #undef ALU3
 249
 250
 251 /* Helpers for SEND instruction:
 252  */
 253
 254 /**
 255  * Construct a message descriptor immediate with the specified common
 256  * descriptor controls.
 257  */
 258 static inline uint32_t
 259 brw_message_desc(const struct gen_device_info *devinfo,
 260                  unsigned msg_length,
 261                  unsigned response_length,
 262                  bool header_present)
 263 {
 264    if (devinfo->gen >= 5) {
 265       return (SET_BITS(msg_length, 28, 25) |
 266               SET_BITS(response_length, 24, 20) |
 267               SET_BITS(header_present, 19, 19));
 268    } else {
 269       return (SET_BITS(msg_length, 23, 20) |
 270               SET_BITS(response_length, 19, 16));
 271    }
 272 }
 273
 274 static inline unsigned
 275 brw_message_desc_mlen(const struct gen_device_info *devinfo, uint32_t desc)
 276 {
 277    if (devinfo->gen >= 5)
 278       return GET_BITS(desc, 28, 25);
 279    else
 280       return GET_BITS(desc, 23, 20);
 281 }
 282
 283 static inline unsigned
 284 brw_message_desc_rlen(const struct gen_device_info *devinfo, uint32_t desc)
 285 {
 286    if (devinfo->gen >= 5)
 287       return GET_BITS(desc, 24, 20);
 288    else
 289       return GET_BITS(desc, 19, 16);
 290 }
 291
 292 static inline bool
 293 brw_message_desc_header_present(ASSERTED const struct gen_device_info *devinfo,
 294                                 uint32_t desc)
 295 {
 296    assert(devinfo->gen >= 5);
 297    return GET_BITS(desc, 19, 19);
 298 }
 299
 300 static inline unsigned
 301 brw_message_ex_desc(UNUSED const struct gen_device_info *devinfo,
 302                     unsigned ex_msg_length)
 303 {
 304    return SET_BITS(ex_msg_length, 9, 6);
 305 }
 306
 307 static inline unsigned
 308 brw_message_ex_desc_ex_mlen(UNUSED const struct gen_device_info *devinfo,
 309                             uint32_t ex_desc)
 310 {
 311    return GET_BITS(ex_desc, 9, 6);
 312 }
 313
 314 /**
 315  * Construct a message descriptor immediate with the specified sampler
 316  * function controls.
 317  */
 318 static inline uint32_t
 319 brw_sampler_desc(const struct gen_device_info *devinfo,
 320                  unsigned binding_table_index,
 321                  unsigned sampler,
 322                  unsigned msg_type,
 323                  unsigned simd_mode,
 324                  unsigned return_format)
 325 {
 326    const unsigned desc = (SET_BITS(binding_table_index, 7, 0) |
 327                           SET_BITS(sampler, 11, 8));
 328    if (devinfo->gen >= 7)
 329       return (desc | SET_BITS(msg_type, 16, 12) |
 330               SET_BITS(simd_mode, 18, 17));
 331    else if (devinfo->gen >= 5)
 332       return (desc | SET_BITS(msg_type, 15, 12) |
 333               SET_BITS(simd_mode, 17, 16));
 334    else if (devinfo->is_g4x)
 335       return desc | SET_BITS(msg_type, 15, 12);
 336    else
 337       return (desc | SET_BITS(return_format, 13, 12) |
 338               SET_BITS(msg_type, 15, 14));
 339 }
 340
 341 static inline unsigned
 342 brw_sampler_desc_binding_table_index(UNUSED const struct gen_device_info *devinfo,
 343                                      uint32_t desc)
 344 {
 345    return GET_BITS(desc, 7, 0);
 346 }
 347
 348 static inline unsigned
 349 brw_sampler_desc_sampler(UNUSED const struct gen_device_info *devinfo, uint32_t desc)
 350 {
 351    return GET_BITS(desc, 11, 8);
 352 }
 353
 354 static inline unsigned
 355 brw_sampler_desc_msg_type(const struct gen_device_info *devinfo, uint32_t desc)
 356 {
 357    if (devinfo->gen >= 7)
 358       return GET_BITS(desc, 16, 12);
 359    else if (devinfo->gen >= 5 || devinfo->is_g4x)
 360       return GET_BITS(desc, 15, 12);
 361    else
 362       return GET_BITS(desc, 15, 14);
 363 }
 364
 365 static inline unsigned
 366 brw_sampler_desc_simd_mode(const struct gen_device_info *devinfo, uint32_t desc)
 367 {
 368    assert(devinfo->gen >= 5);
 369    if (devinfo->gen >= 7)
 370       return GET_BITS(desc, 18, 17);
 371    else
 372       return GET_BITS(desc, 17, 16);
 373 }
 374
 375 static  inline unsigned
 376 brw_sampler_desc_return_format(ASSERTED const struct gen_device_info *devinfo,
 377                                uint32_t desc)
 378 {
 379    assert(devinfo->gen == 4 && !devinfo->is_g4x);
 380    return GET_BITS(desc, 13, 12);
 381 }
 382
 383 /**
 384  * Construct a message descriptor for the dataport
 385  */
 386 static inline uint32_t
 387 brw_dp_desc(const struct gen_device_info *devinfo,
 388             unsigned binding_table_index,
 389             unsigned msg_type,
 390             unsigned msg_control)
 391 {
 392    /* Prior to gen6, things are too inconsistent; use the dp_read/write_desc
 393     * helpers instead.
 394     */
 395    assert(devinfo->gen >= 6);
 396    const unsigned desc = SET_BITS(binding_table_index, 7, 0);
 397    if (devinfo->gen >= 8) {
 398       return (desc | SET_BITS(msg_control, 13, 8) |
 399               SET_BITS(msg_type, 18, 14));
 400    } else if (devinfo->gen >= 7) {
 401       return (desc | SET_BITS(msg_control, 13, 8) |
 402               SET_BITS(msg_type, 17, 14));
 403    } else {
 404       return (desc | SET_BITS(msg_control, 12, 8) |
 405               SET_BITS(msg_type, 16, 13));
 406    }
 407 }
 408
 409 static inline unsigned
 410 brw_dp_desc_binding_table_index(UNUSED const struct gen_device_info *devinfo,
 411                                 uint32_t desc)
 412 {
 413    return GET_BITS(desc, 7, 0);
 414 }
 415
 416 static inline unsigned
 417 brw_dp_desc_msg_type(const struct gen_device_info *devinfo, uint32_t desc)
 418 {
 419    assert(devinfo->gen >= 6);
 420    if (devinfo->gen >= 8)
 421       return GET_BITS(desc, 18, 14);
 422    else if (devinfo->gen >= 7)
 423       return GET_BITS(desc, 17, 14);
 424    else
 425       return GET_BITS(desc, 16, 13);
 426 }
 427
 428 static inline unsigned
 429 brw_dp_desc_msg_control(const struct gen_device_info *devinfo, uint32_t desc)
 430 {
 431    assert(devinfo->gen >= 6);
 432    if (devinfo->gen >= 7)
 433       return GET_BITS(desc, 13, 8);
 434    else
 435       return GET_BITS(desc, 12, 8);
 436 }
 437
 438 /**
 439  * Construct a message descriptor immediate with the specified dataport read
 440  * function controls.
 441  */
 442 static inline uint32_t
 443 brw_dp_read_desc(const struct gen_device_info *devinfo,
 444                  unsigned binding_table_index,
 445                  unsigned msg_control,
 446                  unsigned msg_type,
 447                  unsigned target_cache)
 448 {
 449    if (devinfo->gen >= 6)
 450       return brw_dp_desc(devinfo, binding_table_index, msg_type, msg_control);
 451    else if (devinfo->gen >= 5 || devinfo->is_g4x)
 452       return (SET_BITS(binding_table_index, 7, 0) |
 453               SET_BITS(msg_control, 10, 8) |
 454               SET_BITS(msg_type, 13, 11) |
 455               SET_BITS(target_cache, 15, 14));
 456    else
 457       return (SET_BITS(binding_table_index, 7, 0) |
 458               SET_BITS(msg_control, 11, 8) |
 459               SET_BITS(msg_type, 13, 12) |
 460               SET_BITS(target_cache, 15, 14));
 461 }
 462
 463 static inline unsigned
 464 brw_dp_read_desc_msg_type(const struct gen_device_info *devinfo, uint32_t desc)
 465 {
 466    if (devinfo->gen >= 6)
 467       return brw_dp_desc_msg_type(devinfo, desc);
 468    else if (devinfo->gen >= 5 || devinfo->is_g4x)
 469       return GET_BITS(desc, 13, 11);
 470    else
 471       return GET_BITS(desc, 13, 12);
 472 }
 473
 474 static inline unsigned
 475 brw_dp_read_desc_msg_control(const struct gen_device_info *devinfo,
 476                              uint32_t desc)
 477 {
 478    if (devinfo->gen >= 6)
 479       return brw_dp_desc_msg_control(devinfo, desc);
 480    else if (devinfo->gen >= 5 || devinfo->is_g4x)
 481       return GET_BITS(desc, 10, 8);
 482    else
 483       return GET_BITS(desc, 11, 8);
 484 }
 485
 486 /**
 487  * Construct a message descriptor immediate with the specified dataport write
 488  * function controls.
 489  */
 490 static inline uint32_t
 491 brw_dp_write_desc(const struct gen_device_info *devinfo,
 492                   unsigned binding_table_index,
 493                   unsigned msg_control,
 494                   unsigned msg_type,
 495                   unsigned last_render_target,
 496                   unsigned send_commit_msg)
 497 {
 498    assert(devinfo->gen <= 6 || !send_commit_msg);
 499    if (devinfo->gen >= 6)
 500       return brw_dp_desc(devinfo, binding_table_index, msg_type, msg_control) |
 501              SET_BITS(last_render_target, 12, 12) |
 502              SET_BITS(send_commit_msg, 17, 17);
 503    else
 504       return (SET_BITS(binding_table_index, 7, 0) |
 505               SET_BITS(msg_control, 11, 8) |
 506               SET_BITS(last_render_target, 11, 11) |
 507               SET_BITS(msg_type, 14, 12) |
 508               SET_BITS(send_commit_msg, 15, 15));
 509 }
 510
 511 static inline unsigned
 512 brw_dp_write_desc_msg_type(const struct gen_device_info *devinfo,
 513                            uint32_t desc)
 514 {
 515    if (devinfo->gen >= 6)
 516       return brw_dp_desc_msg_type(devinfo, desc);
 517    else
 518       return GET_BITS(desc, 14, 12);
 519 }
 520
 521 static inline unsigned
 522 brw_dp_write_desc_msg_control(const struct gen_device_info *devinfo,
 523                               uint32_t desc)
 524 {
 525    if (devinfo->gen >= 6)
 526       return brw_dp_desc_msg_control(devinfo, desc);
 527    else
 528       return GET_BITS(desc, 11, 8);
 529 }
 530
 531 static inline bool
 532 brw_dp_write_desc_last_render_target(const struct gen_device_info *devinfo,
 533                                      uint32_t desc)
 534 {
 535    if (devinfo->gen >= 6)
 536       return GET_BITS(desc, 12, 12);
 537    else
 538       return GET_BITS(desc, 11, 11);
 539 }
 540
 541 static inline bool
 542 brw_dp_write_desc_write_commit(const struct gen_device_info *devinfo,
 543                                uint32_t desc)
 544 {
 545    assert(devinfo->gen <= 6);
 546    if (devinfo->gen >= 6)
 547       return GET_BITS(desc, 17, 17);
 548    else
 549       return GET_BITS(desc, 15, 15);
 550 }
 551
 552 /**
 553  * Construct a message descriptor immediate with the specified dataport
 554  * surface function controls.
 555  */
 556 static inline uint32_t
 557 brw_dp_surface_desc(const struct gen_device_info *devinfo,
 558                     unsigned msg_type,
 559                     unsigned msg_control)
 560 {
 561    assert(devinfo->gen >= 7);
 562    /* We'll OR in the binding table index later */
 563    return brw_dp_desc(devinfo, 0, msg_type, msg_control);
 564 }
 565
 566 static inline uint32_t
 567 brw_dp_untyped_atomic_desc(const struct gen_device_info *devinfo,
 568                            unsigned exec_size, /**< 0 for SIMD4x2 */
 569                            unsigned atomic_op,
 570                            bool response_expected)
 571 {
 572    assert(exec_size <= 8 || exec_size == 16);
 573
 574    unsigned msg_type;
 575    if (devinfo->gen >= 8 || devinfo->is_haswell) {
 576       if (exec_size > 0) {
 577          msg_type = HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP;
 578       } else {
 579          msg_type = HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP_SIMD4X2;
 580       }
 581    } else {
 582       msg_type = GEN7_DATAPORT_DC_UNTYPED_ATOMIC_OP;
 583    }
 584
 585    const unsigned msg_control =
 586       SET_BITS(atomic_op, 3, 0) |
 587       SET_BITS(0 < exec_size && exec_size <= 8, 4, 4) |
 588       SET_BITS(response_expected, 5, 5);
 589
 590    return brw_dp_surface_desc(devinfo, msg_type, msg_control);
 591 }
 592
 593 static inline uint32_t
 594 brw_dp_untyped_atomic_float_desc(const struct gen_device_info *devinfo,
 595                                  unsigned exec_size,
 596                                  unsigned atomic_op,
 597                                  bool response_expected)
 598 {
 599    assert(exec_size <= 8 || exec_size == 16);
 600    assert(devinfo->gen >= 9);
 601
 602    assert(exec_size > 0);
 603    const unsigned msg_type = GEN9_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_FLOAT_OP;
 604
 605    const unsigned msg_control =
 606       SET_BITS(atomic_op, 1, 0) |
 607       SET_BITS(exec_size <= 8, 4, 4) |
 608       SET_BITS(response_expected, 5, 5);
 609
 610    return brw_dp_surface_desc(devinfo, msg_type, msg_control);
 611 }
 612
 613 static inline unsigned
 614 brw_mdc_cmask(unsigned num_channels)
 615 {
 616    /* See also MDC_CMASK in the SKL PRM Vol 2d. */
 617    return 0xf & (0xf << num_channels);
 618 }
 619
 620 static inline uint32_t
 621 brw_dp_untyped_surface_rw_desc(const struct gen_device_info *devinfo,
 622                                unsigned exec_size, /**< 0 for SIMD4x2 */
 623                                unsigned num_channels,
 624                                bool write)
 625 {
 626    assert(exec_size <= 8 || exec_size == 16);
 627
 628    unsigned msg_type;
 629    if (write) {
 630       if (devinfo->gen >= 8 || devinfo->is_haswell) {
 631          msg_type = HSW_DATAPORT_DC_PORT1_UNTYPED_SURFACE_WRITE;
 632       } else {
 633          msg_type = GEN7_DATAPORT_DC_UNTYPED_SURFACE_WRITE;
 634       }
 635    } else {
 636       /* Read */
 637       if (devinfo->gen >= 8 || devinfo->is_haswell) {
 638          msg_type = HSW_DATAPORT_DC_PORT1_UNTYPED_SURFACE_READ;
 639       } else {
 640          msg_type = GEN7_DATAPORT_DC_UNTYPED_SURFACE_READ;
 641       }
 642    }
 643
 644    /* SIMD4x2 is only valid for read messages on IVB; use SIMD8 instead */
 645    if (write && devinfo->gen == 7 && !devinfo->is_haswell && exec_size == 0)
 646       exec_size = 8;
 647
 648    /* See also MDC_SM3 in the SKL PRM Vol 2d. */
 649    const unsigned simd_mode = exec_size == 0 ? 0 : /* SIMD4x2 */
 650                               exec_size <= 8 ? 2 : 1;
 651
 652    const unsigned msg_control =
 653       SET_BITS(brw_mdc_cmask(num_channels), 3, 0) |
 654       SET_BITS(simd_mode, 5, 4);
 655
 656    return brw_dp_surface_desc(devinfo, msg_type, msg_control);
 657 }
 658
 659 static inline unsigned
 660 brw_mdc_ds(unsigned bit_size)
 661 {
 662    switch (bit_size) {
 663    case 8:
 664       return GEN7_BYTE_SCATTERED_DATA_ELEMENT_BYTE;
 665    case 16:
 666       return GEN7_BYTE_SCATTERED_DATA_ELEMENT_WORD;
 667    case 32:
 668       return GEN7_BYTE_SCATTERED_DATA_ELEMENT_DWORD;
 669    default:
 670       unreachable("Unsupported bit_size for byte scattered messages");
 671    }
 672 }
 673
 674 static inline uint32_t
 675 brw_dp_byte_scattered_rw_desc(const struct gen_device_info *devinfo,
 676                               unsigned exec_size,
 677                               unsigned bit_size,
 678                               bool write)
 679 {
 680    assert(exec_size <= 8 || exec_size == 16);
 681
 682    assert(devinfo->gen > 7 || devinfo->is_haswell);
 683    const unsigned msg_type =
 684       write ? HSW_DATAPORT_DC_PORT0_BYTE_SCATTERED_WRITE :
 685               HSW_DATAPORT_DC_PORT0_BYTE_SCATTERED_READ;
 686
 687    assert(exec_size > 0);
 688    const unsigned msg_control =
 689       SET_BITS(exec_size == 16, 0, 0) |
 690       SET_BITS(brw_mdc_ds(bit_size), 3, 2);
 691
 692    return brw_dp_surface_desc(devinfo, msg_type, msg_control);
 693 }
 694
 695 static inline uint32_t
 696 brw_dp_dword_scattered_rw_desc(const struct gen_device_info *devinfo,
 697                                unsigned exec_size,
 698                                bool write)
 699 {
 700    assert(exec_size == 8 || exec_size == 16);
 701
 702    unsigned msg_type;
 703    if (write) {
 704       if (devinfo->gen >= 6) {
 705          msg_type = GEN6_DATAPORT_WRITE_MESSAGE_DWORD_SCATTERED_WRITE;
 706       } else {
 707          msg_type = BRW_DATAPORT_WRITE_MESSAGE_DWORD_SCATTERED_WRITE;
 708       }
 709    } else {
 710       if (devinfo->gen >= 7) {
 711          msg_type = GEN7_DATAPORT_DC_DWORD_SCATTERED_READ;
 712       } else if (devinfo->gen > 4 || devinfo->is_g4x) {
 713          msg_type = G45_DATAPORT_READ_MESSAGE_DWORD_SCATTERED_READ;
 714       } else {
 715          msg_type = BRW_DATAPORT_READ_MESSAGE_DWORD_SCATTERED_READ;
 716       }
 717    }
 718
 719    const unsigned msg_control =
 720       SET_BITS(1, 1, 1) | /* Legacy SIMD Mode */
 721       SET_BITS(exec_size == 16, 0, 0);
 722
 723    return brw_dp_surface_desc(devinfo, msg_type, msg_control);
 724 }
 725
 726 static inline uint32_t
 727 brw_dp_a64_untyped_surface_rw_desc(const struct gen_device_info *devinfo,
 728                                    unsigned exec_size, /**< 0 for SIMD4x2 */
 729                                    unsigned num_channels,
 730                                    bool write)
 731 {
 732    assert(exec_size <= 8 || exec_size == 16);
 733    assert(devinfo->gen >= 8);
 734
 735    unsigned msg_type =
 736       write ? GEN8_DATAPORT_DC_PORT1_A64_UNTYPED_SURFACE_WRITE :
 737               GEN8_DATAPORT_DC_PORT1_A64_UNTYPED_SURFACE_READ;
 738
 739    /* See also MDC_SM3 in the SKL PRM Vol 2d. */
 740    const unsigned simd_mode = exec_size == 0 ? 0 : /* SIMD4x2 */
 741                               exec_size <= 8 ? 2 : 1;
 742
 743    const unsigned msg_control =
 744       SET_BITS(brw_mdc_cmask(num_channels), 3, 0) |
 745       SET_BITS(simd_mode, 5, 4);
 746
 747    return brw_dp_desc(devinfo, BRW_BTI_STATELESS, msg_type, msg_control);
 748 }
 749
 750 /**
 751  * Calculate the data size (see MDC_A64_DS in the "Structures" volume of the
 752  * Skylake PRM).
 753  */
 754 static inline uint32_t
 755 brw_mdc_a64_ds(unsigned elems)
 756 {
 757    switch (elems) {
 758    case 1:  return 0;
 759    case 2:  return 1;
 760    case 4:  return 2;
 761    case 8:  return 3;
 762    default:
 763       unreachable("Unsupported elmeent count for A64 scattered message");
 764    }
 765 }
 766
 767 static inline uint32_t
 768 brw_dp_a64_byte_scattered_rw_desc(const struct gen_device_info *devinfo,
 769                                   unsigned exec_size, /**< 0 for SIMD4x2 */
 770                                   unsigned bit_size,
 771                                   bool write)
 772 {
 773    assert(exec_size <= 8 || exec_size == 16);
 774    assert(devinfo->gen >= 8);
 775
 776    unsigned msg_type =
 777       write ? GEN8_DATAPORT_DC_PORT1_A64_SCATTERED_WRITE :
 778               GEN9_DATAPORT_DC_PORT1_A64_SCATTERED_READ;
 779
 780    const unsigned msg_control =
 781       SET_BITS(GEN8_A64_SCATTERED_SUBTYPE_BYTE, 1, 0) |
 782       SET_BITS(brw_mdc_a64_ds(bit_size / 8), 3, 2) |
 783       SET_BITS(exec_size == 16, 4, 4);
 784
 785    return brw_dp_desc(devinfo, BRW_BTI_STATELESS, msg_type, msg_control);
 786 }
 787
 788 static inline uint32_t
 789 brw_dp_a64_untyped_atomic_desc(const struct gen_device_info *devinfo,
 790                                ASSERTED unsigned exec_size, /**< 0 for SIMD4x2 */
 791                                unsigned bit_size,
 792                                unsigned atomic_op,
 793                                bool response_expected)
 794 {
 795    assert(exec_size == 8);
 796    assert(devinfo->gen >= 8);
 797    assert(bit_size == 32 || bit_size == 64);
 798
 799    const unsigned msg_type = GEN8_DATAPORT_DC_PORT1_A64_UNTYPED_ATOMIC_OP;
 800
 801    const unsigned msg_control =
 802       SET_BITS(atomic_op, 3, 0) |
 803       SET_BITS(bit_size == 64, 4, 4) |
 804       SET_BITS(response_expected, 5, 5);
 805
 806    return brw_dp_desc(devinfo, BRW_BTI_STATELESS, msg_type, msg_control);
 807 }
 808
 809 static inline uint32_t
 810 brw_dp_a64_untyped_atomic_float_desc(const struct gen_device_info *devinfo,
 811                                      ASSERTED unsigned exec_size,
 812                                      unsigned atomic_op,
 813                                      bool response_expected)
 814 {
 815    assert(exec_size == 8);
 816    assert(devinfo->gen >= 9);
 817
 818    assert(exec_size > 0);
 819    const unsigned msg_type = GEN9_DATAPORT_DC_PORT1_A64_UNTYPED_ATOMIC_FLOAT_OP;
 820
 821    const unsigned msg_control =
 822       SET_BITS(atomic_op, 1, 0) |
 823       SET_BITS(response_expected, 5, 5);
 824
 825    return brw_dp_desc(devinfo, BRW_BTI_STATELESS, msg_type, msg_control);
 826 }
 827
 828 static inline uint32_t
 829 brw_dp_typed_atomic_desc(const struct gen_device_info *devinfo,
 830                          unsigned exec_size,
 831                          unsigned exec_group,
 832                          unsigned atomic_op,
 833                          bool response_expected)
 834 {
 835    assert(exec_size > 0 || exec_group == 0);
 836    assert(exec_group % 8 == 0);
 837
 838    unsigned msg_type;
 839    if (devinfo->gen >= 8 || devinfo->is_haswell) {
 840       if (exec_size == 0) {
 841          msg_type = HSW_DATAPORT_DC_PORT1_TYPED_ATOMIC_OP_SIMD4X2;
 842       } else {
 843          msg_type = HSW_DATAPORT_DC_PORT1_TYPED_ATOMIC_OP;
 844       }
 845    } else {
 846       /* SIMD4x2 typed surface R/W messages only exist on HSW+ */
 847       assert(exec_size > 0);
 848       msg_type = GEN7_DATAPORT_RC_TYPED_ATOMIC_OP;
 849    }
 850
 851    const bool high_sample_mask = (exec_group / 8) % 2 == 1;
 852
 853    const unsigned msg_control =
 854       SET_BITS(atomic_op, 3, 0) |
 855       SET_BITS(high_sample_mask, 4, 4) |
 856       SET_BITS(response_expected, 5, 5);
 857
 858    return brw_dp_surface_desc(devinfo, msg_type, msg_control);
 859 }
 860
 861 static inline uint32_t
 862 brw_dp_typed_surface_rw_desc(const struct gen_device_info *devinfo,
 863                              unsigned exec_size,
 864                              unsigned exec_group,
 865                              unsigned num_channels,
 866                              bool write)
 867 {
 868    assert(exec_size > 0 || exec_group == 0);
 869    assert(exec_group % 8 == 0);
 870
 871    /* Typed surface reads and writes don't support SIMD16 */
 872    assert(exec_size <= 8);
 873
 874    unsigned msg_type;
 875    if (write) {
 876       if (devinfo->gen >= 8 || devinfo->is_haswell) {
 877          msg_type = HSW_DATAPORT_DC_PORT1_TYPED_SURFACE_WRITE;
 878       } else {
 879          msg_type = GEN7_DATAPORT_RC_TYPED_SURFACE_WRITE;
 880       }
 881    } else {
 882       if (devinfo->gen >= 8 || devinfo->is_haswell) {
 883          msg_type = HSW_DATAPORT_DC_PORT1_TYPED_SURFACE_READ;
 884       } else {
 885          msg_type = GEN7_DATAPORT_RC_TYPED_SURFACE_READ;
 886       }
 887    }
 888
 889    /* See also MDC_SG3 in the SKL PRM Vol 2d. */
 890    unsigned msg_control;
 891    if (devinfo->gen >= 8 || devinfo->is_haswell) {
 892       /* See also MDC_SG3 in the SKL PRM Vol 2d. */
 893       const unsigned slot_group = exec_size == 0 ? 0 : /* SIMD4x2 */
 894                                   1 + ((exec_group / 8) % 2);
 895
 896       msg_control =
 897          SET_BITS(brw_mdc_cmask(num_channels), 3, 0) |
 898          SET_BITS(slot_group, 5, 4);
 899    } else {
 900       /* SIMD4x2 typed surface R/W messages only exist on HSW+ */
 901       assert(exec_size > 0);
 902       const unsigned slot_group = ((exec_group / 8) % 2);
 903
 904       msg_control =
 905          SET_BITS(brw_mdc_cmask(num_channels), 3, 0) |
 906          SET_BITS(slot_group, 5, 5);
 907    }
 908
 909    return brw_dp_surface_desc(devinfo, msg_type, msg_control);
 910 }
 911
 912 /**
 913  * Construct a message descriptor immediate with the specified pixel
 914  * interpolator function controls.
 915  */
 916 static inline uint32_t
 917 brw_pixel_interp_desc(UNUSED const struct gen_device_info *devinfo,
 918                       unsigned msg_type,
 919                       bool noperspective,
 920                       unsigned simd_mode,
 921                       unsigned slot_group)
 922 {
 923    return (SET_BITS(slot_group, 11, 11) |
 924            SET_BITS(msg_type, 13, 12) |
 925            SET_BITS(!!noperspective, 14, 14) |
 926            SET_BITS(simd_mode, 16, 16));
 927 }
 928
 929 void brw_urb_WRITE(struct brw_codegen *p,
 930                    struct brw_reg dest,
 931                    unsigned msg_reg_nr,
 932                    struct brw_reg src0,
 933                    enum brw_urb_write_flags flags,
 934                    unsigned msg_length,
 935                    unsigned response_length,
 936                    unsigned offset,
 937                    unsigned swizzle);
 938
 939 /**
 940  * Send message to shared unit \p sfid with a possibly indirect descriptor \p
 941  * desc.  If \p desc is not an immediate it will be transparently loaded to an
 942  * address register using an OR instruction.
 943  */
 944 void
 945 brw_send_indirect_message(struct brw_codegen *p,
 946                           unsigned sfid,
 947                           struct brw_reg dst,
 948                           struct brw_reg payload,
 949                           struct brw_reg desc,
 950                           unsigned desc_imm,
 951                           bool eot);
 952
 953 void
 954 brw_send_indirect_split_message(struct brw_codegen *p,
 955                                 unsigned sfid,
 956                                 struct brw_reg dst,
 957                                 struct brw_reg payload0,
 958                                 struct brw_reg payload1,
 959                                 struct brw_reg desc,
 960                                 unsigned desc_imm,
 961                                 struct brw_reg ex_desc,
 962                                 unsigned ex_desc_imm,
 963                                 bool eot);
 964
 965 void brw_ff_sync(struct brw_codegen *p,
 966                    struct brw_reg dest,
 967                    unsigned msg_reg_nr,
 968                    struct brw_reg src0,
 969                    bool allocate,
 970                    unsigned response_length,
 971                    bool eot);
 972
 973 void brw_svb_write(struct brw_codegen *p,
 974                    struct brw_reg dest,
 975                    unsigned msg_reg_nr,
 976                    struct brw_reg src0,
 977                    unsigned binding_table_index,
 978                    bool   send_commit_msg);
 979
 980 brw_inst *brw_fb_WRITE(struct brw_codegen *p,
 981                        struct brw_reg payload,
 982                        struct brw_reg implied_header,
 983                        unsigned msg_control,
 984                        unsigned binding_table_index,
 985                        unsigned msg_length,
 986                        unsigned response_length,
 987                        bool eot,
 988                        bool last_render_target,
 989                        bool header_present);
 990
 991 brw_inst *gen9_fb_READ(struct brw_codegen *p,
 992                        struct brw_reg dst,
 993                        struct brw_reg payload,
 994                        unsigned binding_table_index,
 995                        unsigned msg_length,
 996                        unsigned response_length,
 997                        bool per_sample);
 998
 999 void brw_SAMPLE(struct brw_codegen *p,
1000                 struct brw_reg dest,
1001                 unsigned msg_reg_nr,
1002                 struct brw_reg src0,
1003                 unsigned binding_table_index,
1004                 unsigned sampler,
1005                 unsigned msg_type,
1006                 unsigned response_length,
1007                 unsigned msg_length,
1008                 unsigned header_present,
1009                 unsigned simd_mode,
1010                 unsigned return_format);
1011
1012 void brw_adjust_sampler_state_pointer(struct brw_codegen *p,
1013                                       struct brw_reg header,
1014                                       struct brw_reg sampler_index);
1015
1016 void gen4_math(struct brw_codegen *p,
1017                struct brw_reg dest,
1018                unsigned function,
1019                unsigned msg_reg_nr,
1020                struct brw_reg src,
1021                unsigned precision );
1022
1023 void gen6_math(struct brw_codegen *p,
1024                struct brw_reg dest,
1025                unsigned function,
1026                struct brw_reg src0,
1027                struct brw_reg src1);
1028
1029 void brw_oword_block_read(struct brw_codegen *p,
1030                           struct brw_reg dest,
1031                           struct brw_reg mrf,
1032                           uint32_t offset,
1033                           uint32_t bind_table_index);
1034
1035 unsigned brw_scratch_surface_idx(const struct brw_codegen *p);
1036
1037 void brw_oword_block_read_scratch(struct brw_codegen *p,
1038                                   struct brw_reg dest,
1039                                   struct brw_reg mrf,
1040                                   int num_regs,
1041                                   unsigned offset);
1042
1043 void brw_oword_block_write_scratch(struct brw_codegen *p,
1044                                    struct brw_reg mrf,
1045                                    int num_regs,
1046                                    unsigned offset);
1047
1048 void gen7_block_read_scratch(struct brw_codegen *p,
1049                              struct brw_reg dest,
1050                              int num_regs,
1051                              unsigned offset);
1052
1053 void brw_shader_time_add(struct brw_codegen *p,
1054                          struct brw_reg payload,
1055                          uint32_t surf_index);
1056
1057 /**
1058  * Return the generation-specific jump distance scaling factor.
1059  *
1060  * Given the number of instructions to jump, we need to scale by
1061  * some number to obtain the actual jump distance to program in an
1062  * instruction.
1063  */
1064 static inline unsigned
1065 brw_jump_scale(const struct gen_device_info *devinfo)
1066 {
1067    /* Broadwell measures jump targets in bytes. */
1068    if (devinfo->gen >= 8)
1069       return 16;
1070
1071    /* Ironlake and later measure jump targets in 64-bit data chunks (in order
1072     * (to support compaction), so each 128-bit instruction requires 2 chunks.
1073     */
1074    if (devinfo->gen >= 5)
1075       return 2;
1076
1077    /* Gen4 simply uses the number of 128-bit instructions. */
1078    return 1;
1079 }
1080
1081 void brw_barrier(struct brw_codegen *p, struct brw_reg src);
1082
1083 /* If/else/endif.  Works by manipulating the execution flags on each
1084  * channel.
1085  */
1086 brw_inst *brw_IF(struct brw_codegen *p, unsigned execute_size);
1087 brw_inst *gen6_IF(struct brw_codegen *p, enum brw_conditional_mod conditional,
1088                   struct brw_reg src0, struct brw_reg src1);
1089
1090 void brw_ELSE(struct brw_codegen *p);
1091 void brw_ENDIF(struct brw_codegen *p);
1092
1093 /* DO/WHILE loops:
1094  */
1095 brw_inst *brw_DO(struct brw_codegen *p, unsigned execute_size);
1096
1097 brw_inst *brw_WHILE(struct brw_codegen *p);
1098
1099 brw_inst *brw_BREAK(struct brw_codegen *p);
1100 brw_inst *brw_CONT(struct brw_codegen *p);
1101 brw_inst *gen6_HALT(struct brw_codegen *p);
1102
1103 /* Forward jumps:
1104  */
1105 void brw_land_fwd_jump(struct brw_codegen *p, int jmp_insn_idx);
1106
1107 brw_inst *brw_JMPI(struct brw_codegen *p, struct brw_reg index,
1108                    unsigned predicate_control);
1109
1110 void brw_NOP(struct brw_codegen *p);
1111
1112 void brw_WAIT(struct brw_codegen *p);
1113
1114 void brw_SYNC(struct brw_codegen *p, enum tgl_sync_function func);
1115
1116 /* Special case: there is never a destination, execution size will be
1117  * taken from src0:
1118  */
1119 void brw_CMP(struct brw_codegen *p,
1120              struct brw_reg dest,
1121              unsigned conditional,
1122              struct brw_reg src0,
1123              struct brw_reg src1);
1124
1125 void
1126 brw_untyped_atomic(struct brw_codegen *p,
1127                    struct brw_reg dst,
1128                    struct brw_reg payload,
1129                    struct brw_reg surface,
1130                    unsigned atomic_op,
1131                    unsigned msg_length,
1132                    bool response_expected,
1133                    bool header_present);
1134
1135 void
1136 brw_untyped_surface_read(struct brw_codegen *p,
1137                          struct brw_reg dst,
1138                          struct brw_reg payload,
1139                          struct brw_reg surface,
1140                          unsigned msg_length,
1141                          unsigned num_channels);
1142
1143 void
1144 brw_untyped_surface_write(struct brw_codegen *p,
1145                           struct brw_reg payload,
1146                           struct brw_reg surface,
1147                           unsigned msg_length,
1148                           unsigned num_channels,
1149                           bool header_present);
1150
1151 unsigned
1152 brw_memory_fence(struct brw_codegen *p,
1153                  struct brw_reg dst,
1154                  struct brw_reg src,
1155                  enum opcode send_op,
1156                  bool stall,
1157                  unsigned bti);
1158
1159 void
1160 brw_pixel_interpolator_query(struct brw_codegen *p,
1161                              struct brw_reg dest,
1162                              struct brw_reg mrf,
1163                              bool noperspective,
1164                              unsigned mode,
1165                              struct brw_reg data,
1166                              unsigned msg_length,
1167                              unsigned response_length);
1168
1169 void
1170 brw_find_live_channel(struct brw_codegen *p,
1171                       struct brw_reg dst,
1172                       struct brw_reg mask);
1173
1174 void
1175 brw_broadcast(struct brw_codegen *p,
1176               struct brw_reg dst,
1177               struct brw_reg src,
1178               struct brw_reg idx);
1179
1180 void
1181 brw_float_controls_mode(struct brw_codegen *p,
1182                         unsigned mode, unsigned mask);
1183
1184 /***********************************************************************
1185  * brw_eu_util.c:
1186  */
1187
1188 void brw_copy_indirect_to_indirect(struct brw_codegen *p,
1189                                    struct brw_indirect dst_ptr,
1190                                    struct brw_indirect src_ptr,
1191                                    unsigned count);
1192
1193 void brw_copy_from_indirect(struct brw_codegen *p,
1194                             struct brw_reg dst,
1195                             struct brw_indirect ptr,
1196                             unsigned count);
1197
1198 void brw_copy4(struct brw_codegen *p,
1199                struct brw_reg dst,
1200                struct brw_reg src,
1201                unsigned count);
1202
1203 void brw_copy8(struct brw_codegen *p,
1204                struct brw_reg dst,
1205                struct brw_reg src,
1206                unsigned count);
1207
1208 void brw_math_invert( struct brw_codegen *p,
1209                       struct brw_reg dst,
1210                       struct brw_reg src);
1211
1212 void brw_set_src1(struct brw_codegen *p, brw_inst *insn, struct brw_reg reg);
1213
1214 void brw_set_desc_ex(struct brw_codegen *p, brw_inst *insn,
1215                      unsigned desc, unsigned ex_desc);
1216
1217 static inline void
1218 brw_set_desc(struct brw_codegen *p, brw_inst *insn, unsigned desc)
1219 {
1220    brw_set_desc_ex(p, insn, desc, 0);
1221 }
1222
1223 void brw_set_uip_jip(struct brw_codegen *p, int start_offset);
1224
1225 enum brw_conditional_mod brw_negate_cmod(enum brw_conditional_mod cmod);
1226 enum brw_conditional_mod brw_swap_cmod(enum brw_conditional_mod cmod);
1227
1228 /* brw_eu_compact.c */
1229 void brw_init_compaction_tables(const struct gen_device_info *devinfo);
1230 void brw_compact_instructions(struct brw_codegen *p, int start_offset,
1231                               struct disasm_info *disasm);
1232 void brw_uncompact_instruction(const struct gen_device_info *devinfo,
1233                                brw_inst *dst, brw_compact_inst *src);
1234 bool brw_try_compact_instruction(const struct gen_device_info *devinfo,
1235                                  brw_compact_inst *dst, const brw_inst *src);
1236
1237 void brw_debug_compact_uncompact(const struct gen_device_info *devinfo,
1238                                  brw_inst *orig, brw_inst *uncompacted);
1239
1240 /* brw_eu_validate.c */
1241 bool brw_validate_instruction(const struct gen_device_info *devinfo,
1242                               const brw_inst *inst, int offset,
1243                               struct disasm_info *disasm);
1244 bool brw_validate_instructions(const struct gen_device_info *devinfo,
1245                                const void *assembly, int start_offset, int end_offset,
1246                                struct disasm_info *disasm);
1247
1248 static inline int
1249 next_offset(const struct gen_device_info *devinfo, void *store, int offset)
1250 {
1251    brw_inst *insn = (brw_inst *)((char *)store + offset);
1252
1253    if (brw_inst_cmpt_control(devinfo, insn))
1254       return offset + 8;
1255    else
1256       return offset + 16;
1257 }
1258
1259 struct opcode_desc {
1260    unsigned ir;
1261    unsigned hw;
1262    const char *name;
1263    int nsrc;
1264    int ndst;
1265    int gens;
1266 };
1267
1268 const struct opcode_desc *
1269 brw_opcode_desc(const struct gen_device_info *devinfo, enum opcode opcode);
1270
1271 const struct opcode_desc *
1272 brw_opcode_desc_from_hw(const struct gen_device_info *devinfo, unsigned hw);
1273
1274 static inline unsigned
1275 brw_opcode_encode(const struct gen_device_info *devinfo, enum opcode opcode)
1276 {
1277    return brw_opcode_desc(devinfo, opcode)->hw;
1278 }
1279
1280 static inline enum opcode
1281 brw_opcode_decode(const struct gen_device_info *devinfo, unsigned hw)
1282 {
1283    const struct opcode_desc *desc = brw_opcode_desc_from_hw(devinfo, hw);
1284    return desc ? (enum opcode)desc->ir : BRW_OPCODE_ILLEGAL;
1285 }
1286
1287 static inline void
1288 brw_inst_set_opcode(const struct gen_device_info *devinfo,
1289                     brw_inst *inst, enum opcode opcode)
1290 {
1291    brw_inst_set_hw_opcode(devinfo, inst, brw_opcode_encode(devinfo, opcode));
1292 }
1293
1294 static inline enum opcode
1295 brw_inst_opcode(const struct gen_device_info *devinfo, const brw_inst *inst)
1296 {
1297    return brw_opcode_decode(devinfo, brw_inst_hw_opcode(devinfo, inst));
1298 }
1299
1300 static inline bool
1301 is_3src(const struct gen_device_info *devinfo, enum opcode opcode)
1302 {
1303    const struct opcode_desc *desc = brw_opcode_desc(devinfo, opcode);
1304    return desc && desc->nsrc == 3;
1305 }
1306
1307 /** Maximum SEND message length */
1308 #define BRW_MAX_MSG_LENGTH 15
1309
1310 /** First MRF register used by pull loads */
1311 #define FIRST_SPILL_MRF(gen) ((gen) == 6 ? 21 : 13)
1312
1313 /** First MRF register used by spills */
1314 #define FIRST_PULL_LOAD_MRF(gen) ((gen) == 6 ? 16 : 13)
1315
1316 #ifdef __cplusplus
1317 }
1318 #endif
1319
1320 #endif