src/intel/compiler/brw_eu.h

   1 /*
   2  Copyright (C) Intel Corp.  2006.  All Rights Reserved.
   3  Intel funded Tungsten Graphics to
   4  develop this 3D driver.
   5
   6  Permission is hereby granted, free of charge, to any person obtaining
   7  a copy of this software and associated documentation files (the
   8  "Software"), to deal in the Software without restriction, including
   9  without limitation the rights to use, copy, modify, merge, publish,
  10  distribute, sublicense, and/or sell copies of the Software, and to
  11  permit persons to whom the Software is furnished to do so, subject to
  12  the following conditions:
  13
  14  The above copyright notice and this permission notice (including the
  15  next paragraph) shall be included in all copies or substantial
  16  portions of the Software.
  17
  18  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  19  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  21  IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
  22  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  23  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  24  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25
  26  **********************************************************************/
  27  /*
  28   * Authors:
  29   *   Keith Whitwell <keithw@vmware.com>
  30   */
  31
  32
  33 #ifndef BRW_EU_H
  34 #define BRW_EU_H
  35
  36 #include <stdbool.h>
  37 #include <stdio.h>
  38 #include "brw_inst.h"
  39 #include "brw_compiler.h"
  40 #include "brw_eu_defines.h"
  41 #include "brw_reg.h"
  42 #include "brw_disasm_info.h"
  43
  44 #ifdef __cplusplus
  45 extern "C" {
  46 #endif
  47
  48 #define BRW_EU_MAX_INSN_STACK 5
  49
  50 struct brw_insn_state {
  51    /* One of BRW_EXECUTE_* */
  52    unsigned exec_size:3;
  53
  54    /* Group in units of channels */
  55    unsigned group:5;
  56
  57    /* Compression control on gen4-5 */
  58    bool compressed:1;
  59
  60    /* One of BRW_MASK_* */
  61    unsigned mask_control:1;
  62
  63    /* Scheduling info for Gen12+ */
  64    struct tgl_swsb swsb;
  65
  66    bool saturate:1;
  67
  68    /* One of BRW_ALIGN_* */
  69    unsigned access_mode:1;
  70
  71    /* One of BRW_PREDICATE_* */
  72    enum brw_predicate predicate:4;
  73
  74    bool pred_inv:1;
  75
  76    /* Flag subreg.  Bottom bit is subreg, top bit is reg */
  77    unsigned flag_subreg:2;
  78
  79    bool acc_wr_control:1;
  80 };
  81
  82
  83 /* A helper for accessing the last instruction emitted.  This makes it easy
  84  * to set various bits on an instruction without having to create temporary
  85  * variable and assign the emitted instruction to those.
  86  */
  87 #define brw_last_inst (&p->store[p->nr_insn - 1])
  88
  89 struct brw_codegen {
  90    brw_inst *store;
  91    int store_size;
  92    unsigned nr_insn;
  93    unsigned int next_insn_offset;
  94
  95    void *mem_ctx;
  96
  97    /* Allow clients to push/pop instruction state:
  98     */
  99    struct brw_insn_state stack[BRW_EU_MAX_INSN_STACK];
 100    struct brw_insn_state *current;
 101
 102    /** Whether or not the user wants automatic exec sizes
 103     *
 104     * If true, codegen will try to automatically infer the exec size of an
 105     * instruction from the width of the destination register.  If false, it
 106     * will take whatever is set by brw_set_default_exec_size verbatim.
 107     *
 108     * This is set to true by default in brw_init_codegen.
 109     */
 110    bool automatic_exec_sizes;
 111
 112    bool single_program_flow;
 113    const struct gen_device_info *devinfo;
 114
 115    /* Control flow stacks:
 116     * - if_stack contains IF and ELSE instructions which must be patched
 117     *   (and popped) once the matching ENDIF instruction is encountered.
 118     *
 119     *   Just store the instruction pointer(an index).
 120     */
 121    int *if_stack;
 122    int if_stack_depth;
 123    int if_stack_array_size;
 124
 125    /**
 126     * loop_stack contains the instruction pointers of the starts of loops which
 127     * must be patched (and popped) once the matching WHILE instruction is
 128     * encountered.
 129     */
 130    int *loop_stack;
 131    /**
 132     * pre-gen6, the BREAK and CONT instructions had to tell how many IF/ENDIF
 133     * blocks they were popping out of, to fix up the mask stack.  This tracks
 134     * the IF/ENDIF nesting in each current nested loop level.
 135     */
 136    int *if_depth_in_loop;
 137    int loop_stack_depth;
 138    int loop_stack_array_size;
 139
 140    struct brw_shader_reloc *relocs;
 141    int num_relocs;
 142    int reloc_array_size;
 143 };
 144
 145 struct brw_label {
 146    int offset;
 147    int number;
 148    struct brw_label *next;
 149 };
 150
 151 void brw_pop_insn_state( struct brw_codegen *p );
 152 void brw_push_insn_state( struct brw_codegen *p );
 153 unsigned brw_get_default_exec_size(struct brw_codegen *p);
 154 unsigned brw_get_default_group(struct brw_codegen *p);
 155 unsigned brw_get_default_access_mode(struct brw_codegen *p);
 156 struct tgl_swsb brw_get_default_swsb(struct brw_codegen *p);
 157 void brw_set_default_exec_size(struct brw_codegen *p, unsigned value);
 158 void brw_set_default_mask_control( struct brw_codegen *p, unsigned value );
 159 void brw_set_default_saturate( struct brw_codegen *p, bool enable );
 160 void brw_set_default_access_mode( struct brw_codegen *p, unsigned access_mode );
 161 void brw_inst_set_compression(const struct gen_device_info *devinfo,
 162                               brw_inst *inst, bool on);
 163 void brw_set_default_compression(struct brw_codegen *p, bool on);
 164 void brw_inst_set_group(const struct gen_device_info *devinfo,
 165                         brw_inst *inst, unsigned group);
 166 void brw_set_default_group(struct brw_codegen *p, unsigned group);
 167 void brw_set_default_compression_control(struct brw_codegen *p, enum brw_compression c);
 168 void brw_set_default_predicate_control(struct brw_codegen *p, enum brw_predicate pc);
 169 void brw_set_default_predicate_inverse(struct brw_codegen *p, bool predicate_inverse);
 170 void brw_set_default_flag_reg(struct brw_codegen *p, int reg, int subreg);
 171 void brw_set_default_acc_write_control(struct brw_codegen *p, unsigned value);
 172 void brw_set_default_swsb(struct brw_codegen *p, struct tgl_swsb value);
 173
 174 void brw_init_codegen(const struct gen_device_info *, struct brw_codegen *p,
 175                       void *mem_ctx);
 176 bool brw_has_jip(const struct gen_device_info *devinfo, enum opcode opcode);
 177 bool brw_has_uip(const struct gen_device_info *devinfo, enum opcode opcode);
 178 const struct brw_label *brw_find_label(const struct brw_label *root, int offset);
 179 void brw_create_label(struct brw_label **labels, int offset, void *mem_ctx);
 180 int brw_disassemble_inst(FILE *file, const struct gen_device_info *devinfo,
 181                          const struct brw_inst *inst, bool is_compacted,
 182                          int offset, const struct brw_label *root_label);
 183 const struct brw_label *brw_label_assembly(const struct gen_device_info *devinfo,
 184                                            const void *assembly, int start, int end,
 185                                            void *mem_ctx);
 186 void brw_disassemble_with_labels(const struct gen_device_info *devinfo,
 187                                  const void *assembly, int start, int end, FILE *out);
 188 void brw_disassemble(const struct gen_device_info *devinfo,
 189                      const void *assembly, int start, int end,
 190                      const struct brw_label *root_label, FILE *out);
 191 const struct brw_shader_reloc *brw_get_shader_relocs(struct brw_codegen *p,
 192                                                      unsigned *num_relocs);
 193 const unsigned *brw_get_program( struct brw_codegen *p, unsigned *sz );
 194
 195 bool brw_try_override_assembly(struct brw_codegen *p, int start_offset,
 196                                const char *identifier);
 197
 198 void brw_realign(struct brw_codegen *p, unsigned align);
 199 int brw_append_data(struct brw_codegen *p, void *data,
 200                     unsigned size, unsigned align);
 201 brw_inst *brw_next_insn(struct brw_codegen *p, unsigned opcode);
 202 void brw_set_dest(struct brw_codegen *p, brw_inst *insn, struct brw_reg dest);
 203 void brw_set_src0(struct brw_codegen *p, brw_inst *insn, struct brw_reg reg);
 204
 205 void gen6_resolve_implied_move(struct brw_codegen *p,
 206                                struct brw_reg *src,
 207                                unsigned msg_reg_nr);
 208
 209 /* Helpers for regular instructions:
 210  */
 211 #define ALU1(OP)                                \
 212 brw_inst *brw_##OP(struct brw_codegen *p,       \
 213               struct brw_reg dest,              \
 214               struct brw_reg src0);
 215
 216 #define ALU2(OP)                                \
 217 brw_inst *brw_##OP(struct brw_codegen *p,       \
 218               struct brw_reg dest,              \
 219               struct brw_reg src0,              \
 220               struct brw_reg src1);
 221
 222 #define ALU3(OP)                                \
 223 brw_inst *brw_##OP(struct brw_codegen *p,       \
 224               struct brw_reg dest,              \
 225               struct brw_reg src0,              \
 226               struct brw_reg src1,              \
 227               struct brw_reg src2);
 228
 229 ALU1(MOV)
 230 ALU2(SEL)
 231 ALU1(NOT)
 232 ALU2(AND)
 233 ALU2(OR)
 234 ALU2(XOR)
 235 ALU2(SHR)
 236 ALU2(SHL)
 237 ALU1(DIM)
 238 ALU2(ASR)
 239 ALU2(ROL)
 240 ALU2(ROR)
 241 ALU3(CSEL)
 242 ALU1(F32TO16)
 243 ALU1(F16TO32)
 244 ALU2(ADD)
 245 ALU2(AVG)
 246 ALU2(MUL)
 247 ALU1(FRC)
 248 ALU1(RNDD)
 249 ALU1(RNDE)
 250 ALU1(RNDU)
 251 ALU1(RNDZ)
 252 ALU2(MAC)
 253 ALU2(MACH)
 254 ALU1(LZD)
 255 ALU2(DP4)
 256 ALU2(DPH)
 257 ALU2(DP3)
 258 ALU2(DP2)
 259 ALU2(LINE)
 260 ALU2(PLN)
 261 ALU3(MAD)
 262 ALU3(LRP)
 263 ALU1(BFREV)
 264 ALU3(BFE)
 265 ALU2(BFI1)
 266 ALU3(BFI2)
 267 ALU1(FBH)
 268 ALU1(FBL)
 269 ALU1(CBIT)
 270 ALU2(ADDC)
 271 ALU2(SUBB)
 272 ALU2(MAC)
 273
 274 #undef ALU1
 275 #undef ALU2
 276 #undef ALU3
 277
 278
 279 /* Helpers for SEND instruction:
 280  */
 281
 282 /**
 283  * Construct a message descriptor immediate with the specified common
 284  * descriptor controls.
 285  */
 286 static inline uint32_t
 287 brw_message_desc(const struct gen_device_info *devinfo,
 288                  unsigned msg_length,
 289                  unsigned response_length,
 290                  bool header_present)
 291 {
 292    if (devinfo->gen >= 5) {
 293       return (SET_BITS(msg_length, 28, 25) |
 294               SET_BITS(response_length, 24, 20) |
 295               SET_BITS(header_present, 19, 19));
 296    } else {
 297       return (SET_BITS(msg_length, 23, 20) |
 298               SET_BITS(response_length, 19, 16));
 299    }
 300 }
 301
 302 static inline unsigned
 303 brw_message_desc_mlen(const struct gen_device_info *devinfo, uint32_t desc)
 304 {
 305    if (devinfo->gen >= 5)
 306       return GET_BITS(desc, 28, 25);
 307    else
 308       return GET_BITS(desc, 23, 20);
 309 }
 310
 311 static inline unsigned
 312 brw_message_desc_rlen(const struct gen_device_info *devinfo, uint32_t desc)
 313 {
 314    if (devinfo->gen >= 5)
 315       return GET_BITS(desc, 24, 20);
 316    else
 317       return GET_BITS(desc, 19, 16);
 318 }
 319
 320 static inline bool
 321 brw_message_desc_header_present(ASSERTED const struct gen_device_info *devinfo,
 322                                 uint32_t desc)
 323 {
 324    assert(devinfo->gen >= 5);
 325    return GET_BITS(desc, 19, 19);
 326 }
 327
 328 static inline unsigned
 329 brw_message_ex_desc(UNUSED const struct gen_device_info *devinfo,
 330                     unsigned ex_msg_length)
 331 {
 332    return SET_BITS(ex_msg_length, 9, 6);
 333 }
 334
 335 static inline unsigned
 336 brw_message_ex_desc_ex_mlen(UNUSED const struct gen_device_info *devinfo,
 337                             uint32_t ex_desc)
 338 {
 339    return GET_BITS(ex_desc, 9, 6);
 340 }
 341
 342 static inline uint32_t
 343 brw_urb_desc(const struct gen_device_info *devinfo,
 344              unsigned msg_type,
 345              bool per_slot_offset_present,
 346              bool channel_mask_present,
 347              unsigned global_offset)
 348 {
 349    if (devinfo->gen >= 8) {
 350       return (SET_BITS(per_slot_offset_present, 17, 17) |
 351               SET_BITS(channel_mask_present, 15, 15) |
 352               SET_BITS(global_offset, 14, 4) |
 353               SET_BITS(msg_type, 3, 0));
 354    } else if (devinfo->gen >= 7) {
 355       assert(!channel_mask_present);
 356       return (SET_BITS(per_slot_offset_present, 16, 16) |
 357               SET_BITS(global_offset, 13, 3) |
 358               SET_BITS(msg_type, 3, 0));
 359    } else {
 360       unreachable("unhandled URB write generation");
 361    }
 362 }
 363
 364 static inline uint32_t
 365 brw_urb_desc_msg_type(ASSERTED const struct gen_device_info *devinfo,
 366                       uint32_t desc)
 367 {
 368    assert(devinfo->gen >= 7);
 369    return GET_BITS(desc, 3, 0);
 370 }
 371
 372 /**
 373  * Construct a message descriptor immediate with the specified sampler
 374  * function controls.
 375  */
 376 static inline uint32_t
 377 brw_sampler_desc(const struct gen_device_info *devinfo,
 378                  unsigned binding_table_index,
 379                  unsigned sampler,
 380                  unsigned msg_type,
 381                  unsigned simd_mode,
 382                  unsigned return_format)
 383 {
 384    const unsigned desc = (SET_BITS(binding_table_index, 7, 0) |
 385                           SET_BITS(sampler, 11, 8));
 386    if (devinfo->gen >= 7)
 387       return (desc | SET_BITS(msg_type, 16, 12) |
 388               SET_BITS(simd_mode, 18, 17));
 389    else if (devinfo->gen >= 5)
 390       return (desc | SET_BITS(msg_type, 15, 12) |
 391               SET_BITS(simd_mode, 17, 16));
 392    else if (devinfo->is_g4x)
 393       return desc | SET_BITS(msg_type, 15, 12);
 394    else
 395       return (desc | SET_BITS(return_format, 13, 12) |
 396               SET_BITS(msg_type, 15, 14));
 397 }
 398
 399 static inline unsigned
 400 brw_sampler_desc_binding_table_index(UNUSED const struct gen_device_info *devinfo,
 401                                      uint32_t desc)
 402 {
 403    return GET_BITS(desc, 7, 0);
 404 }
 405
 406 static inline unsigned
 407 brw_sampler_desc_sampler(UNUSED const struct gen_device_info *devinfo, uint32_t desc)
 408 {
 409    return GET_BITS(desc, 11, 8);
 410 }
 411
 412 static inline unsigned
 413 brw_sampler_desc_msg_type(const struct gen_device_info *devinfo, uint32_t desc)
 414 {
 415    if (devinfo->gen >= 7)
 416       return GET_BITS(desc, 16, 12);
 417    else if (devinfo->gen >= 5 || devinfo->is_g4x)
 418       return GET_BITS(desc, 15, 12);
 419    else
 420       return GET_BITS(desc, 15, 14);
 421 }
 422
 423 static inline unsigned
 424 brw_sampler_desc_simd_mode(const struct gen_device_info *devinfo, uint32_t desc)
 425 {
 426    assert(devinfo->gen >= 5);
 427    if (devinfo->gen >= 7)
 428       return GET_BITS(desc, 18, 17);
 429    else
 430       return GET_BITS(desc, 17, 16);
 431 }
 432
 433 static  inline unsigned
 434 brw_sampler_desc_return_format(ASSERTED const struct gen_device_info *devinfo,
 435                                uint32_t desc)
 436 {
 437    assert(devinfo->gen == 4 && !devinfo->is_g4x);
 438    return GET_BITS(desc, 13, 12);
 439 }
 440
 441 /**
 442  * Construct a message descriptor for the dataport
 443  */
 444 static inline uint32_t
 445 brw_dp_desc(const struct gen_device_info *devinfo,
 446             unsigned binding_table_index,
 447             unsigned msg_type,
 448             unsigned msg_control)
 449 {
 450    /* Prior to gen6, things are too inconsistent; use the dp_read/write_desc
 451     * helpers instead.
 452     */
 453    assert(devinfo->gen >= 6);
 454    const unsigned desc = SET_BITS(binding_table_index, 7, 0);
 455    if (devinfo->gen >= 8) {
 456       return (desc | SET_BITS(msg_control, 13, 8) |
 457               SET_BITS(msg_type, 18, 14));
 458    } else if (devinfo->gen >= 7) {
 459       return (desc | SET_BITS(msg_control, 13, 8) |
 460               SET_BITS(msg_type, 17, 14));
 461    } else {
 462       return (desc | SET_BITS(msg_control, 12, 8) |
 463               SET_BITS(msg_type, 16, 13));
 464    }
 465 }
 466
 467 static inline unsigned
 468 brw_dp_desc_binding_table_index(UNUSED const struct gen_device_info *devinfo,
 469                                 uint32_t desc)
 470 {
 471    return GET_BITS(desc, 7, 0);
 472 }
 473
 474 static inline unsigned
 475 brw_dp_desc_msg_type(const struct gen_device_info *devinfo, uint32_t desc)
 476 {
 477    assert(devinfo->gen >= 6);
 478    if (devinfo->gen >= 8)
 479       return GET_BITS(desc, 18, 14);
 480    else if (devinfo->gen >= 7)
 481       return GET_BITS(desc, 17, 14);
 482    else
 483       return GET_BITS(desc, 16, 13);
 484 }
 485
 486 static inline unsigned
 487 brw_dp_desc_msg_control(const struct gen_device_info *devinfo, uint32_t desc)
 488 {
 489    assert(devinfo->gen >= 6);
 490    if (devinfo->gen >= 7)
 491       return GET_BITS(desc, 13, 8);
 492    else
 493       return GET_BITS(desc, 12, 8);
 494 }
 495
 496 /**
 497  * Construct a message descriptor immediate with the specified dataport read
 498  * function controls.
 499  */
 500 static inline uint32_t
 501 brw_dp_read_desc(const struct gen_device_info *devinfo,
 502                  unsigned binding_table_index,
 503                  unsigned msg_control,
 504                  unsigned msg_type,
 505                  unsigned target_cache)
 506 {
 507    if (devinfo->gen >= 6)
 508       return brw_dp_desc(devinfo, binding_table_index, msg_type, msg_control);
 509    else if (devinfo->gen >= 5 || devinfo->is_g4x)
 510       return (SET_BITS(binding_table_index, 7, 0) |
 511               SET_BITS(msg_control, 10, 8) |
 512               SET_BITS(msg_type, 13, 11) |
 513               SET_BITS(target_cache, 15, 14));
 514    else
 515       return (SET_BITS(binding_table_index, 7, 0) |
 516               SET_BITS(msg_control, 11, 8) |
 517               SET_BITS(msg_type, 13, 12) |
 518               SET_BITS(target_cache, 15, 14));
 519 }
 520
 521 static inline unsigned
 522 brw_dp_read_desc_msg_type(const struct gen_device_info *devinfo, uint32_t desc)
 523 {
 524    if (devinfo->gen >= 6)
 525       return brw_dp_desc_msg_type(devinfo, desc);
 526    else if (devinfo->gen >= 5 || devinfo->is_g4x)
 527       return GET_BITS(desc, 13, 11);
 528    else
 529       return GET_BITS(desc, 13, 12);
 530 }
 531
 532 static inline unsigned
 533 brw_dp_read_desc_msg_control(const struct gen_device_info *devinfo,
 534                              uint32_t desc)
 535 {
 536    if (devinfo->gen >= 6)
 537       return brw_dp_desc_msg_control(devinfo, desc);
 538    else if (devinfo->gen >= 5 || devinfo->is_g4x)
 539       return GET_BITS(desc, 10, 8);
 540    else
 541       return GET_BITS(desc, 11, 8);
 542 }
 543
 544 /**
 545  * Construct a message descriptor immediate with the specified dataport write
 546  * function controls.
 547  */
 548 static inline uint32_t
 549 brw_dp_write_desc(const struct gen_device_info *devinfo,
 550                   unsigned binding_table_index,
 551                   unsigned msg_control,
 552                   unsigned msg_type,
 553                   unsigned last_render_target,
 554                   unsigned send_commit_msg)
 555 {
 556    assert(devinfo->gen <= 6 || !send_commit_msg);
 557    if (devinfo->gen >= 6)
 558       return brw_dp_desc(devinfo, binding_table_index, msg_type, msg_control) |
 559              SET_BITS(last_render_target, 12, 12) |
 560              SET_BITS(send_commit_msg, 17, 17);
 561    else
 562       return (SET_BITS(binding_table_index, 7, 0) |
 563               SET_BITS(msg_control, 11, 8) |
 564               SET_BITS(last_render_target, 11, 11) |
 565               SET_BITS(msg_type, 14, 12) |
 566               SET_BITS(send_commit_msg, 15, 15));
 567 }
 568
 569 static inline unsigned
 570 brw_dp_write_desc_msg_type(const struct gen_device_info *devinfo,
 571                            uint32_t desc)
 572 {
 573    if (devinfo->gen >= 6)
 574       return brw_dp_desc_msg_type(devinfo, desc);
 575    else
 576       return GET_BITS(desc, 14, 12);
 577 }
 578
 579 static inline unsigned
 580 brw_dp_write_desc_msg_control(const struct gen_device_info *devinfo,
 581                               uint32_t desc)
 582 {
 583    if (devinfo->gen >= 6)
 584       return brw_dp_desc_msg_control(devinfo, desc);
 585    else
 586       return GET_BITS(desc, 11, 8);
 587 }
 588
 589 static inline bool
 590 brw_dp_write_desc_last_render_target(const struct gen_device_info *devinfo,
 591                                      uint32_t desc)
 592 {
 593    if (devinfo->gen >= 6)
 594       return GET_BITS(desc, 12, 12);
 595    else
 596       return GET_BITS(desc, 11, 11);
 597 }
 598
 599 static inline bool
 600 brw_dp_write_desc_write_commit(const struct gen_device_info *devinfo,
 601                                uint32_t desc)
 602 {
 603    assert(devinfo->gen <= 6);
 604    if (devinfo->gen >= 6)
 605       return GET_BITS(desc, 17, 17);
 606    else
 607       return GET_BITS(desc, 15, 15);
 608 }
 609
 610 /**
 611  * Construct a message descriptor immediate with the specified dataport
 612  * surface function controls.
 613  */
 614 static inline uint32_t
 615 brw_dp_surface_desc(const struct gen_device_info *devinfo,
 616                     unsigned msg_type,
 617                     unsigned msg_control)
 618 {
 619    assert(devinfo->gen >= 7);
 620    /* We'll OR in the binding table index later */
 621    return brw_dp_desc(devinfo, 0, msg_type, msg_control);
 622 }
 623
 624 static inline uint32_t
 625 brw_dp_untyped_atomic_desc(const struct gen_device_info *devinfo,
 626                            unsigned exec_size, /**< 0 for SIMD4x2 */
 627                            unsigned atomic_op,
 628                            bool response_expected)
 629 {
 630    assert(exec_size <= 8 || exec_size == 16);
 631
 632    unsigned msg_type;
 633    if (devinfo->gen >= 8 || devinfo->is_haswell) {
 634       if (exec_size > 0) {
 635          msg_type = HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP;
 636       } else {
 637          msg_type = HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP_SIMD4X2;
 638       }
 639    } else {
 640       msg_type = GEN7_DATAPORT_DC_UNTYPED_ATOMIC_OP;
 641    }
 642
 643    const unsigned msg_control =
 644       SET_BITS(atomic_op, 3, 0) |
 645       SET_BITS(0 < exec_size && exec_size <= 8, 4, 4) |
 646       SET_BITS(response_expected, 5, 5);
 647
 648    return brw_dp_surface_desc(devinfo, msg_type, msg_control);
 649 }
 650
 651 static inline uint32_t
 652 brw_dp_untyped_atomic_float_desc(const struct gen_device_info *devinfo,
 653                                  unsigned exec_size,
 654                                  unsigned atomic_op,
 655                                  bool response_expected)
 656 {
 657    assert(exec_size <= 8 || exec_size == 16);
 658    assert(devinfo->gen >= 9);
 659
 660    assert(exec_size > 0);
 661    const unsigned msg_type = GEN9_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_FLOAT_OP;
 662
 663    const unsigned msg_control =
 664       SET_BITS(atomic_op, 1, 0) |
 665       SET_BITS(exec_size <= 8, 4, 4) |
 666       SET_BITS(response_expected, 5, 5);
 667
 668    return brw_dp_surface_desc(devinfo, msg_type, msg_control);
 669 }
 670
 671 static inline unsigned
 672 brw_mdc_cmask(unsigned num_channels)
 673 {
 674    /* See also MDC_CMASK in the SKL PRM Vol 2d. */
 675    return 0xf & (0xf << num_channels);
 676 }
 677
 678 static inline uint32_t
 679 brw_dp_untyped_surface_rw_desc(const struct gen_device_info *devinfo,
 680                                unsigned exec_size, /**< 0 for SIMD4x2 */
 681                                unsigned num_channels,
 682                                bool write)
 683 {
 684    assert(exec_size <= 8 || exec_size == 16);
 685
 686    unsigned msg_type;
 687    if (write) {
 688       if (devinfo->gen >= 8 || devinfo->is_haswell) {
 689          msg_type = HSW_DATAPORT_DC_PORT1_UNTYPED_SURFACE_WRITE;
 690       } else {
 691          msg_type = GEN7_DATAPORT_DC_UNTYPED_SURFACE_WRITE;
 692       }
 693    } else {
 694       /* Read */
 695       if (devinfo->gen >= 8 || devinfo->is_haswell) {
 696          msg_type = HSW_DATAPORT_DC_PORT1_UNTYPED_SURFACE_READ;
 697       } else {
 698          msg_type = GEN7_DATAPORT_DC_UNTYPED_SURFACE_READ;
 699       }
 700    }
 701
 702    /* SIMD4x2 is only valid for read messages on IVB; use SIMD8 instead */
 703    if (write && devinfo->gen == 7 && !devinfo->is_haswell && exec_size == 0)
 704       exec_size = 8;
 705
 706    /* See also MDC_SM3 in the SKL PRM Vol 2d. */
 707    const unsigned simd_mode = exec_size == 0 ? 0 : /* SIMD4x2 */
 708                               exec_size <= 8 ? 2 : 1;
 709
 710    const unsigned msg_control =
 711       SET_BITS(brw_mdc_cmask(num_channels), 3, 0) |
 712       SET_BITS(simd_mode, 5, 4);
 713
 714    return brw_dp_surface_desc(devinfo, msg_type, msg_control);
 715 }
 716
 717 static inline unsigned
 718 brw_mdc_ds(unsigned bit_size)
 719 {
 720    switch (bit_size) {
 721    case 8:
 722       return GEN7_BYTE_SCATTERED_DATA_ELEMENT_BYTE;
 723    case 16:
 724       return GEN7_BYTE_SCATTERED_DATA_ELEMENT_WORD;
 725    case 32:
 726       return GEN7_BYTE_SCATTERED_DATA_ELEMENT_DWORD;
 727    default:
 728       unreachable("Unsupported bit_size for byte scattered messages");
 729    }
 730 }
 731
 732 static inline uint32_t
 733 brw_dp_byte_scattered_rw_desc(const struct gen_device_info *devinfo,
 734                               unsigned exec_size,
 735                               unsigned bit_size,
 736                               bool write)
 737 {
 738    assert(exec_size <= 8 || exec_size == 16);
 739
 740    assert(devinfo->gen > 7 || devinfo->is_haswell);
 741    const unsigned msg_type =
 742       write ? HSW_DATAPORT_DC_PORT0_BYTE_SCATTERED_WRITE :
 743               HSW_DATAPORT_DC_PORT0_BYTE_SCATTERED_READ;
 744
 745    assert(exec_size > 0);
 746    const unsigned msg_control =
 747       SET_BITS(exec_size == 16, 0, 0) |
 748       SET_BITS(brw_mdc_ds(bit_size), 3, 2);
 749
 750    return brw_dp_surface_desc(devinfo, msg_type, msg_control);
 751 }
 752
 753 static inline uint32_t
 754 brw_dp_dword_scattered_rw_desc(const struct gen_device_info *devinfo,
 755                                unsigned exec_size,
 756                                bool write)
 757 {
 758    assert(exec_size == 8 || exec_size == 16);
 759
 760    unsigned msg_type;
 761    if (write) {
 762       if (devinfo->gen >= 6) {
 763          msg_type = GEN6_DATAPORT_WRITE_MESSAGE_DWORD_SCATTERED_WRITE;
 764       } else {
 765          msg_type = BRW_DATAPORT_WRITE_MESSAGE_DWORD_SCATTERED_WRITE;
 766       }
 767    } else {
 768       if (devinfo->gen >= 7) {
 769          msg_type = GEN7_DATAPORT_DC_DWORD_SCATTERED_READ;
 770       } else if (devinfo->gen > 4 || devinfo->is_g4x) {
 771          msg_type = G45_DATAPORT_READ_MESSAGE_DWORD_SCATTERED_READ;
 772       } else {
 773          msg_type = BRW_DATAPORT_READ_MESSAGE_DWORD_SCATTERED_READ;
 774       }
 775    }
 776
 777    const unsigned msg_control =
 778       SET_BITS(1, 1, 1) | /* Legacy SIMD Mode */
 779       SET_BITS(exec_size == 16, 0, 0);
 780
 781    return brw_dp_surface_desc(devinfo, msg_type, msg_control);
 782 }
 783
 784 static inline uint32_t
 785 brw_dp_a64_untyped_surface_rw_desc(const struct gen_device_info *devinfo,
 786                                    unsigned exec_size, /**< 0 for SIMD4x2 */
 787                                    unsigned num_channels,
 788                                    bool write)
 789 {
 790    assert(exec_size <= 8 || exec_size == 16);
 791    assert(devinfo->gen >= 8);
 792
 793    unsigned msg_type =
 794       write ? GEN8_DATAPORT_DC_PORT1_A64_UNTYPED_SURFACE_WRITE :
 795               GEN8_DATAPORT_DC_PORT1_A64_UNTYPED_SURFACE_READ;
 796
 797    /* See also MDC_SM3 in the SKL PRM Vol 2d. */
 798    const unsigned simd_mode = exec_size == 0 ? 0 : /* SIMD4x2 */
 799                               exec_size <= 8 ? 2 : 1;
 800
 801    const unsigned msg_control =
 802       SET_BITS(brw_mdc_cmask(num_channels), 3, 0) |
 803       SET_BITS(simd_mode, 5, 4);
 804
 805    return brw_dp_desc(devinfo, GEN8_BTI_STATELESS_NON_COHERENT,
 806                       msg_type, msg_control);
 807 }
 808
 809 /**
 810  * Calculate the data size (see MDC_A64_DS in the "Structures" volume of the
 811  * Skylake PRM).
 812  */
 813 static inline uint32_t
 814 brw_mdc_a64_ds(unsigned elems)
 815 {
 816    switch (elems) {
 817    case 1:  return 0;
 818    case 2:  return 1;
 819    case 4:  return 2;
 820    case 8:  return 3;
 821    default:
 822       unreachable("Unsupported elmeent count for A64 scattered message");
 823    }
 824 }
 825
 826 static inline uint32_t
 827 brw_dp_a64_byte_scattered_rw_desc(const struct gen_device_info *devinfo,
 828                                   unsigned exec_size, /**< 0 for SIMD4x2 */
 829                                   unsigned bit_size,
 830                                   bool write)
 831 {
 832    assert(exec_size <= 8 || exec_size == 16);
 833    assert(devinfo->gen >= 8);
 834
 835    unsigned msg_type =
 836       write ? GEN8_DATAPORT_DC_PORT1_A64_SCATTERED_WRITE :
 837               GEN9_DATAPORT_DC_PORT1_A64_SCATTERED_READ;
 838
 839    const unsigned msg_control =
 840       SET_BITS(GEN8_A64_SCATTERED_SUBTYPE_BYTE, 1, 0) |
 841       SET_BITS(brw_mdc_a64_ds(bit_size / 8), 3, 2) |
 842       SET_BITS(exec_size == 16, 4, 4);
 843
 844    return brw_dp_desc(devinfo, GEN8_BTI_STATELESS_NON_COHERENT,
 845                       msg_type, msg_control);
 846 }
 847
 848 static inline uint32_t
 849 brw_dp_a64_untyped_atomic_desc(const struct gen_device_info *devinfo,
 850                                ASSERTED unsigned exec_size, /**< 0 for SIMD4x2 */
 851                                unsigned bit_size,
 852                                unsigned atomic_op,
 853                                bool response_expected)
 854 {
 855    assert(exec_size == 8);
 856    assert(devinfo->gen >= 8);
 857    assert(bit_size == 32 || bit_size == 64);
 858
 859    const unsigned msg_type = GEN8_DATAPORT_DC_PORT1_A64_UNTYPED_ATOMIC_OP;
 860
 861    const unsigned msg_control =
 862       SET_BITS(atomic_op, 3, 0) |
 863       SET_BITS(bit_size == 64, 4, 4) |
 864       SET_BITS(response_expected, 5, 5);
 865
 866    return brw_dp_desc(devinfo, GEN8_BTI_STATELESS_NON_COHERENT,
 867                       msg_type, msg_control);
 868 }
 869
 870 static inline uint32_t
 871 brw_dp_a64_untyped_atomic_float_desc(const struct gen_device_info *devinfo,
 872                                      ASSERTED unsigned exec_size,
 873                                      unsigned atomic_op,
 874                                      bool response_expected)
 875 {
 876    assert(exec_size == 8);
 877    assert(devinfo->gen >= 9);
 878
 879    assert(exec_size > 0);
 880    const unsigned msg_type = GEN9_DATAPORT_DC_PORT1_A64_UNTYPED_ATOMIC_FLOAT_OP;
 881
 882    const unsigned msg_control =
 883       SET_BITS(atomic_op, 1, 0) |
 884       SET_BITS(response_expected, 5, 5);
 885
 886    return brw_dp_desc(devinfo, GEN8_BTI_STATELESS_NON_COHERENT,
 887                       msg_type, msg_control);
 888 }
 889
 890 static inline uint32_t
 891 brw_dp_typed_atomic_desc(const struct gen_device_info *devinfo,
 892                          unsigned exec_size,
 893                          unsigned exec_group,
 894                          unsigned atomic_op,
 895                          bool response_expected)
 896 {
 897    assert(exec_size > 0 || exec_group == 0);
 898    assert(exec_group % 8 == 0);
 899
 900    unsigned msg_type;
 901    if (devinfo->gen >= 8 || devinfo->is_haswell) {
 902       if (exec_size == 0) {
 903          msg_type = HSW_DATAPORT_DC_PORT1_TYPED_ATOMIC_OP_SIMD4X2;
 904       } else {
 905          msg_type = HSW_DATAPORT_DC_PORT1_TYPED_ATOMIC_OP;
 906       }
 907    } else {
 908       /* SIMD4x2 typed surface R/W messages only exist on HSW+ */
 909       assert(exec_size > 0);
 910       msg_type = GEN7_DATAPORT_RC_TYPED_ATOMIC_OP;
 911    }
 912
 913    const bool high_sample_mask = (exec_group / 8) % 2 == 1;
 914
 915    const unsigned msg_control =
 916       SET_BITS(atomic_op, 3, 0) |
 917       SET_BITS(high_sample_mask, 4, 4) |
 918       SET_BITS(response_expected, 5, 5);
 919
 920    return brw_dp_surface_desc(devinfo, msg_type, msg_control);
 921 }
 922
 923 static inline uint32_t
 924 brw_dp_typed_surface_rw_desc(const struct gen_device_info *devinfo,
 925                              unsigned exec_size,
 926                              unsigned exec_group,
 927                              unsigned num_channels,
 928                              bool write)
 929 {
 930    assert(exec_size > 0 || exec_group == 0);
 931    assert(exec_group % 8 == 0);
 932
 933    /* Typed surface reads and writes don't support SIMD16 */
 934    assert(exec_size <= 8);
 935
 936    unsigned msg_type;
 937    if (write) {
 938       if (devinfo->gen >= 8 || devinfo->is_haswell) {
 939          msg_type = HSW_DATAPORT_DC_PORT1_TYPED_SURFACE_WRITE;
 940       } else {
 941          msg_type = GEN7_DATAPORT_RC_TYPED_SURFACE_WRITE;
 942       }
 943    } else {
 944       if (devinfo->gen >= 8 || devinfo->is_haswell) {
 945          msg_type = HSW_DATAPORT_DC_PORT1_TYPED_SURFACE_READ;
 946       } else {
 947          msg_type = GEN7_DATAPORT_RC_TYPED_SURFACE_READ;
 948       }
 949    }
 950
 951    /* See also MDC_SG3 in the SKL PRM Vol 2d. */
 952    unsigned msg_control;
 953    if (devinfo->gen >= 8 || devinfo->is_haswell) {
 954       /* See also MDC_SG3 in the SKL PRM Vol 2d. */
 955       const unsigned slot_group = exec_size == 0 ? 0 : /* SIMD4x2 */
 956                                   1 + ((exec_group / 8) % 2);
 957
 958       msg_control =
 959          SET_BITS(brw_mdc_cmask(num_channels), 3, 0) |
 960          SET_BITS(slot_group, 5, 4);
 961    } else {
 962       /* SIMD4x2 typed surface R/W messages only exist on HSW+ */
 963       assert(exec_size > 0);
 964       const unsigned slot_group = ((exec_group / 8) % 2);
 965
 966       msg_control =
 967          SET_BITS(brw_mdc_cmask(num_channels), 3, 0) |
 968          SET_BITS(slot_group, 5, 5);
 969    }
 970
 971    return brw_dp_surface_desc(devinfo, msg_type, msg_control);
 972 }
 973
 974 /**
 975  * Construct a message descriptor immediate with the specified pixel
 976  * interpolator function controls.
 977  */
 978 static inline uint32_t
 979 brw_pixel_interp_desc(UNUSED const struct gen_device_info *devinfo,
 980                       unsigned msg_type,
 981                       bool noperspective,
 982                       unsigned simd_mode,
 983                       unsigned slot_group)
 984 {
 985    return (SET_BITS(slot_group, 11, 11) |
 986            SET_BITS(msg_type, 13, 12) |
 987            SET_BITS(!!noperspective, 14, 14) |
 988            SET_BITS(simd_mode, 16, 16));
 989 }
 990
 991 void brw_urb_WRITE(struct brw_codegen *p,
 992                    struct brw_reg dest,
 993                    unsigned msg_reg_nr,
 994                    struct brw_reg src0,
 995                    enum brw_urb_write_flags flags,
 996                    unsigned msg_length,
 997                    unsigned response_length,
 998                    unsigned offset,
 999                    unsigned swizzle);
1000
1001 /**
1002  * Send message to shared unit \p sfid with a possibly indirect descriptor \p
1003  * desc.  If \p desc is not an immediate it will be transparently loaded to an
1004  * address register using an OR instruction.
1005  */
1006 void
1007 brw_send_indirect_message(struct brw_codegen *p,
1008                           unsigned sfid,
1009                           struct brw_reg dst,
1010                           struct brw_reg payload,
1011                           struct brw_reg desc,
1012                           unsigned desc_imm,
1013                           bool eot);
1014
1015 void
1016 brw_send_indirect_split_message(struct brw_codegen *p,
1017                                 unsigned sfid,
1018                                 struct brw_reg dst,
1019                                 struct brw_reg payload0,
1020                                 struct brw_reg payload1,
1021                                 struct brw_reg desc,
1022                                 unsigned desc_imm,
1023                                 struct brw_reg ex_desc,
1024                                 unsigned ex_desc_imm,
1025                                 bool eot);
1026
1027 void brw_ff_sync(struct brw_codegen *p,
1028                    struct brw_reg dest,
1029                    unsigned msg_reg_nr,
1030                    struct brw_reg src0,
1031                    bool allocate,
1032                    unsigned response_length,
1033                    bool eot);
1034
1035 void brw_svb_write(struct brw_codegen *p,
1036                    struct brw_reg dest,
1037                    unsigned msg_reg_nr,
1038                    struct brw_reg src0,
1039                    unsigned binding_table_index,
1040                    bool   send_commit_msg);
1041
1042 brw_inst *brw_fb_WRITE(struct brw_codegen *p,
1043                        struct brw_reg payload,
1044                        struct brw_reg implied_header,
1045                        unsigned msg_control,
1046                        unsigned binding_table_index,
1047                        unsigned msg_length,
1048                        unsigned response_length,
1049                        bool eot,
1050                        bool last_render_target,
1051                        bool header_present);
1052
1053 brw_inst *gen9_fb_READ(struct brw_codegen *p,
1054                        struct brw_reg dst,
1055                        struct brw_reg payload,
1056                        unsigned binding_table_index,
1057                        unsigned msg_length,
1058                        unsigned response_length,
1059                        bool per_sample);
1060
1061 void brw_SAMPLE(struct brw_codegen *p,
1062                 struct brw_reg dest,
1063                 unsigned msg_reg_nr,
1064                 struct brw_reg src0,
1065                 unsigned binding_table_index,
1066                 unsigned sampler,
1067                 unsigned msg_type,
1068                 unsigned response_length,
1069                 unsigned msg_length,
1070                 unsigned header_present,
1071                 unsigned simd_mode,
1072                 unsigned return_format);
1073
1074 void brw_adjust_sampler_state_pointer(struct brw_codegen *p,
1075                                       struct brw_reg header,
1076                                       struct brw_reg sampler_index);
1077
1078 void gen4_math(struct brw_codegen *p,
1079                struct brw_reg dest,
1080                unsigned function,
1081                unsigned msg_reg_nr,
1082                struct brw_reg src,
1083                unsigned precision );
1084
1085 void gen6_math(struct brw_codegen *p,
1086                struct brw_reg dest,
1087                unsigned function,
1088                struct brw_reg src0,
1089                struct brw_reg src1);
1090
1091 void brw_oword_block_read(struct brw_codegen *p,
1092                           struct brw_reg dest,
1093                           struct brw_reg mrf,
1094                           uint32_t offset,
1095                           uint32_t bind_table_index);
1096
1097 unsigned brw_scratch_surface_idx(const struct brw_codegen *p);
1098
1099 void brw_oword_block_read_scratch(struct brw_codegen *p,
1100                                   struct brw_reg dest,
1101                                   struct brw_reg mrf,
1102                                   int num_regs,
1103                                   unsigned offset);
1104
1105 void brw_oword_block_write_scratch(struct brw_codegen *p,
1106                                    struct brw_reg mrf,
1107                                    int num_regs,
1108                                    unsigned offset);
1109
1110 void gen7_block_read_scratch(struct brw_codegen *p,
1111                              struct brw_reg dest,
1112                              int num_regs,
1113                              unsigned offset);
1114
1115 void brw_shader_time_add(struct brw_codegen *p,
1116                          struct brw_reg payload,
1117                          uint32_t surf_index);
1118
1119 /**
1120  * Return the generation-specific jump distance scaling factor.
1121  *
1122  * Given the number of instructions to jump, we need to scale by
1123  * some number to obtain the actual jump distance to program in an
1124  * instruction.
1125  */
1126 static inline unsigned
1127 brw_jump_scale(const struct gen_device_info *devinfo)
1128 {
1129    /* Broadwell measures jump targets in bytes. */
1130    if (devinfo->gen >= 8)
1131       return 16;
1132
1133    /* Ironlake and later measure jump targets in 64-bit data chunks (in order
1134     * (to support compaction), so each 128-bit instruction requires 2 chunks.
1135     */
1136    if (devinfo->gen >= 5)
1137       return 2;
1138
1139    /* Gen4 simply uses the number of 128-bit instructions. */
1140    return 1;
1141 }
1142
1143 void brw_barrier(struct brw_codegen *p, struct brw_reg src);
1144
1145 /* If/else/endif.  Works by manipulating the execution flags on each
1146  * channel.
1147  */
1148 brw_inst *brw_IF(struct brw_codegen *p, unsigned execute_size);
1149 brw_inst *gen6_IF(struct brw_codegen *p, enum brw_conditional_mod conditional,
1150                   struct brw_reg src0, struct brw_reg src1);
1151
1152 void brw_ELSE(struct brw_codegen *p);
1153 void brw_ENDIF(struct brw_codegen *p);
1154
1155 /* DO/WHILE loops:
1156  */
1157 brw_inst *brw_DO(struct brw_codegen *p, unsigned execute_size);
1158
1159 brw_inst *brw_WHILE(struct brw_codegen *p);
1160
1161 brw_inst *brw_BREAK(struct brw_codegen *p);
1162 brw_inst *brw_CONT(struct brw_codegen *p);
1163 brw_inst *brw_HALT(struct brw_codegen *p);
1164
1165 /* Forward jumps:
1166  */
1167 void brw_land_fwd_jump(struct brw_codegen *p, int jmp_insn_idx);
1168
1169 brw_inst *brw_JMPI(struct brw_codegen *p, struct brw_reg index,
1170                    unsigned predicate_control);
1171
1172 void brw_NOP(struct brw_codegen *p);
1173
1174 void brw_WAIT(struct brw_codegen *p);
1175
1176 void brw_SYNC(struct brw_codegen *p, enum tgl_sync_function func);
1177
1178 /* Special case: there is never a destination, execution size will be
1179  * taken from src0:
1180  */
1181 void brw_CMP(struct brw_codegen *p,
1182              struct brw_reg dest,
1183              unsigned conditional,
1184              struct brw_reg src0,
1185              struct brw_reg src1);
1186
1187 void
1188 brw_untyped_atomic(struct brw_codegen *p,
1189                    struct brw_reg dst,
1190                    struct brw_reg payload,
1191                    struct brw_reg surface,
1192                    unsigned atomic_op,
1193                    unsigned msg_length,
1194                    bool response_expected,
1195                    bool header_present);
1196
1197 void
1198 brw_untyped_surface_read(struct brw_codegen *p,
1199                          struct brw_reg dst,
1200                          struct brw_reg payload,
1201                          struct brw_reg surface,
1202                          unsigned msg_length,
1203                          unsigned num_channels);
1204
1205 void
1206 brw_untyped_surface_write(struct brw_codegen *p,
1207                           struct brw_reg payload,
1208                           struct brw_reg surface,
1209                           unsigned msg_length,
1210                           unsigned num_channels,
1211                           bool header_present);
1212
1213 void
1214 brw_memory_fence(struct brw_codegen *p,
1215                  struct brw_reg dst,
1216                  struct brw_reg src,
1217                  enum opcode send_op,
1218                  enum brw_message_target sfid,
1219                  bool commit_enable,
1220                  unsigned bti);
1221
1222 void
1223 brw_pixel_interpolator_query(struct brw_codegen *p,
1224                              struct brw_reg dest,
1225                              struct brw_reg mrf,
1226                              bool noperspective,
1227                              unsigned mode,
1228                              struct brw_reg data,
1229                              unsigned msg_length,
1230                              unsigned response_length);
1231
1232 void
1233 brw_find_live_channel(struct brw_codegen *p,
1234                       struct brw_reg dst,
1235                       struct brw_reg mask);
1236
1237 void
1238 brw_broadcast(struct brw_codegen *p,
1239               struct brw_reg dst,
1240               struct brw_reg src,
1241               struct brw_reg idx);
1242
1243 void
1244 brw_float_controls_mode(struct brw_codegen *p,
1245                         unsigned mode, unsigned mask);
1246
1247 void
1248 brw_update_reloc_imm(const struct gen_device_info *devinfo,
1249                      brw_inst *inst,
1250                      uint32_t value);
1251
1252 void
1253 brw_MOV_reloc_imm(struct brw_codegen *p,
1254                   struct brw_reg dst,
1255                   enum brw_reg_type src_type,
1256                   uint32_t id);
1257
1258 /***********************************************************************
1259  * brw_eu_util.c:
1260  */
1261
1262 void brw_copy_indirect_to_indirect(struct brw_codegen *p,
1263                                    struct brw_indirect dst_ptr,
1264                                    struct brw_indirect src_ptr,
1265                                    unsigned count);
1266
1267 void brw_copy_from_indirect(struct brw_codegen *p,
1268                             struct brw_reg dst,
1269                             struct brw_indirect ptr,
1270                             unsigned count);
1271
1272 void brw_copy4(struct brw_codegen *p,
1273                struct brw_reg dst,
1274                struct brw_reg src,
1275                unsigned count);
1276
1277 void brw_copy8(struct brw_codegen *p,
1278                struct brw_reg dst,
1279                struct brw_reg src,
1280                unsigned count);
1281
1282 void brw_math_invert( struct brw_codegen *p,
1283                       struct brw_reg dst,
1284                       struct brw_reg src);
1285
1286 void brw_set_src1(struct brw_codegen *p, brw_inst *insn, struct brw_reg reg);
1287
1288 void brw_set_desc_ex(struct brw_codegen *p, brw_inst *insn,
1289                      unsigned desc, unsigned ex_desc);
1290
1291 static inline void
1292 brw_set_desc(struct brw_codegen *p, brw_inst *insn, unsigned desc)
1293 {
1294    brw_set_desc_ex(p, insn, desc, 0);
1295 }
1296
1297 void brw_set_uip_jip(struct brw_codegen *p, int start_offset);
1298
1299 enum brw_conditional_mod brw_negate_cmod(enum brw_conditional_mod cmod);
1300 enum brw_conditional_mod brw_swap_cmod(enum brw_conditional_mod cmod);
1301
1302 /* brw_eu_compact.c */
1303 void brw_compact_instructions(struct brw_codegen *p, int start_offset,
1304                               struct disasm_info *disasm);
1305 void brw_uncompact_instruction(const struct gen_device_info *devinfo,
1306                                brw_inst *dst, brw_compact_inst *src);
1307 bool brw_try_compact_instruction(const struct gen_device_info *devinfo,
1308                                  brw_compact_inst *dst, const brw_inst *src);
1309
1310 void brw_debug_compact_uncompact(const struct gen_device_info *devinfo,
1311                                  brw_inst *orig, brw_inst *uncompacted);
1312
1313 /* brw_eu_validate.c */
1314 bool brw_validate_instruction(const struct gen_device_info *devinfo,
1315                               const brw_inst *inst, int offset,
1316                               struct disasm_info *disasm);
1317 bool brw_validate_instructions(const struct gen_device_info *devinfo,
1318                                const void *assembly, int start_offset, int end_offset,
1319                                struct disasm_info *disasm);
1320
1321 static inline int
1322 next_offset(const struct gen_device_info *devinfo, void *store, int offset)
1323 {
1324    brw_inst *insn = (brw_inst *)((char *)store + offset);
1325
1326    if (brw_inst_cmpt_control(devinfo, insn))
1327       return offset + 8;
1328    else
1329       return offset + 16;
1330 }
1331
1332 struct opcode_desc {
1333    unsigned ir;
1334    unsigned hw;
1335    const char *name;
1336    int nsrc;
1337    int ndst;
1338    int gens;
1339 };
1340
1341 const struct opcode_desc *
1342 brw_opcode_desc(const struct gen_device_info *devinfo, enum opcode opcode);
1343
1344 const struct opcode_desc *
1345 brw_opcode_desc_from_hw(const struct gen_device_info *devinfo, unsigned hw);
1346
1347 static inline unsigned
1348 brw_opcode_encode(const struct gen_device_info *devinfo, enum opcode opcode)
1349 {
1350    return brw_opcode_desc(devinfo, opcode)->hw;
1351 }
1352
1353 static inline enum opcode
1354 brw_opcode_decode(const struct gen_device_info *devinfo, unsigned hw)
1355 {
1356    const struct opcode_desc *desc = brw_opcode_desc_from_hw(devinfo, hw);
1357    return desc ? (enum opcode)desc->ir : BRW_OPCODE_ILLEGAL;
1358 }
1359
1360 static inline void
1361 brw_inst_set_opcode(const struct gen_device_info *devinfo,
1362                     brw_inst *inst, enum opcode opcode)
1363 {
1364    brw_inst_set_hw_opcode(devinfo, inst, brw_opcode_encode(devinfo, opcode));
1365 }
1366
1367 static inline enum opcode
1368 brw_inst_opcode(const struct gen_device_info *devinfo, const brw_inst *inst)
1369 {
1370    return brw_opcode_decode(devinfo, brw_inst_hw_opcode(devinfo, inst));
1371 }
1372
1373 static inline bool
1374 is_3src(const struct gen_device_info *devinfo, enum opcode opcode)
1375 {
1376    const struct opcode_desc *desc = brw_opcode_desc(devinfo, opcode);
1377    return desc && desc->nsrc == 3;
1378 }
1379
1380 /** Maximum SEND message length */
1381 #define BRW_MAX_MSG_LENGTH 15
1382
1383 /** First MRF register used by pull loads */
1384 #define FIRST_SPILL_MRF(gen) ((gen) == 6 ? 21 : 13)
1385
1386 /** First MRF register used by spills */
1387 #define FIRST_PULL_LOAD_MRF(gen) ((gen) == 6 ? 16 : 13)
1388
1389 #ifdef __cplusplus
1390 }
1391 #endif
1392
1393 #endif