src/intel/compiler/brw_eu.h

   1 /*
   2  Copyright (C) Intel Corp.  2006.  All Rights Reserved.
   3  Intel funded Tungsten Graphics to
   4  develop this 3D driver.
   5
   6  Permission is hereby granted, free of charge, to any person obtaining
   7  a copy of this software and associated documentation files (the
   8  "Software"), to deal in the Software without restriction, including
   9  without limitation the rights to use, copy, modify, merge, publish,
  10  distribute, sublicense, and/or sell copies of the Software, and to
  11  permit persons to whom the Software is furnished to do so, subject to
  12  the following conditions:
  13
  14  The above copyright notice and this permission notice (including the
  15  next paragraph) shall be included in all copies or substantial
  16  portions of the Software.
  17
  18  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  19  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  21  IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
  22  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  23  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  24  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25
  26  **********************************************************************/
  27  /*
  28   * Authors:
  29   *   Keith Whitwell <keithw@vmware.com>
  30   */
  31
  32
  33 #ifndef BRW_EU_H
  34 #define BRW_EU_H
  35
  36 #include <stdbool.h>
  37 #include <stdio.h>
  38 #include "brw_inst.h"
  39 #include "brw_eu_defines.h"
  40 #include "brw_reg.h"
  41 #include "brw_disasm_info.h"
  42
  43 #ifdef __cplusplus
  44 extern "C" {
  45 #endif
  46
  47 #define BRW_EU_MAX_INSN_STACK 5
  48
  49 struct brw_insn_state {
  50    /* One of BRW_EXECUTE_* */
  51    unsigned exec_size:3;
  52
  53    /* Group in units of channels */
  54    unsigned group:5;
  55
  56    /* Compression control on gen4-5 */
  57    bool compressed:1;
  58
  59    /* One of BRW_MASK_* */
  60    unsigned mask_control:1;
  61
  62    bool saturate:1;
  63
  64    /* One of BRW_ALIGN_* */
  65    unsigned access_mode:1;
  66
  67    /* One of BRW_PREDICATE_* */
  68    enum brw_predicate predicate:4;
  69
  70    bool pred_inv:1;
  71
  72    /* Flag subreg.  Bottom bit is subreg, top bit is reg */
  73    unsigned flag_subreg:2;
  74
  75    bool acc_wr_control:1;
  76 };
  77
  78
  79 /* A helper for accessing the last instruction emitted.  This makes it easy
  80  * to set various bits on an instruction without having to create temporary
  81  * variable and assign the emitted instruction to those.
  82  */
  83 #define brw_last_inst (&p->store[p->nr_insn - 1])
  84
  85 struct brw_codegen {
  86    brw_inst *store;
  87    int store_size;
  88    unsigned nr_insn;
  89    unsigned int next_insn_offset;
  90
  91    void *mem_ctx;
  92
  93    /* Allow clients to push/pop instruction state:
  94     */
  95    struct brw_insn_state stack[BRW_EU_MAX_INSN_STACK];
  96    struct brw_insn_state *current;
  97
  98    /** Whether or not the user wants automatic exec sizes
  99     *
 100     * If true, codegen will try to automatically infer the exec size of an
 101     * instruction from the width of the destination register.  If false, it
 102     * will take whatever is set by brw_set_default_exec_size verbatim.
 103     *
 104     * This is set to true by default in brw_init_codegen.
 105     */
 106    bool automatic_exec_sizes;
 107
 108    bool single_program_flow;
 109    const struct gen_device_info *devinfo;
 110
 111    /* Control flow stacks:
 112     * - if_stack contains IF and ELSE instructions which must be patched
 113     *   (and popped) once the matching ENDIF instruction is encountered.
 114     *
 115     *   Just store the instruction pointer(an index).
 116     */
 117    int *if_stack;
 118    int if_stack_depth;
 119    int if_stack_array_size;
 120
 121    /**
 122     * loop_stack contains the instruction pointers of the starts of loops which
 123     * must be patched (and popped) once the matching WHILE instruction is
 124     * encountered.
 125     */
 126    int *loop_stack;
 127    /**
 128     * pre-gen6, the BREAK and CONT instructions had to tell how many IF/ENDIF
 129     * blocks they were popping out of, to fix up the mask stack.  This tracks
 130     * the IF/ENDIF nesting in each current nested loop level.
 131     */
 132    int *if_depth_in_loop;
 133    int loop_stack_depth;
 134    int loop_stack_array_size;
 135 };
 136
 137 void brw_pop_insn_state( struct brw_codegen *p );
 138 void brw_push_insn_state( struct brw_codegen *p );
 139 unsigned brw_get_default_exec_size(struct brw_codegen *p);
 140 unsigned brw_get_default_group(struct brw_codegen *p);
 141 unsigned brw_get_default_access_mode(struct brw_codegen *p);
 142 void brw_set_default_exec_size(struct brw_codegen *p, unsigned value);
 143 void brw_set_default_mask_control( struct brw_codegen *p, unsigned value );
 144 void brw_set_default_saturate( struct brw_codegen *p, bool enable );
 145 void brw_set_default_access_mode( struct brw_codegen *p, unsigned access_mode );
 146 void brw_inst_set_compression(const struct gen_device_info *devinfo,
 147                               brw_inst *inst, bool on);
 148 void brw_set_default_compression(struct brw_codegen *p, bool on);
 149 void brw_inst_set_group(const struct gen_device_info *devinfo,
 150                         brw_inst *inst, unsigned group);
 151 void brw_set_default_group(struct brw_codegen *p, unsigned group);
 152 void brw_set_default_compression_control(struct brw_codegen *p, enum brw_compression c);
 153 void brw_set_default_predicate_control( struct brw_codegen *p, unsigned pc );
 154 void brw_set_default_predicate_inverse(struct brw_codegen *p, bool predicate_inverse);
 155 void brw_set_default_flag_reg(struct brw_codegen *p, int reg, int subreg);
 156 void brw_set_default_acc_write_control(struct brw_codegen *p, unsigned value);
 157
 158 void brw_init_codegen(const struct gen_device_info *, struct brw_codegen *p,
 159                       void *mem_ctx);
 160 int brw_disassemble_inst(FILE *file, const struct gen_device_info *devinfo,
 161                          const struct brw_inst *inst, bool is_compacted);
 162 void brw_disassemble(const struct gen_device_info *devinfo,
 163                      const void *assembly, int start, int end, FILE *out);
 164 const unsigned *brw_get_program( struct brw_codegen *p, unsigned *sz );
 165
 166 brw_inst *brw_next_insn(struct brw_codegen *p, unsigned opcode);
 167 void brw_set_dest(struct brw_codegen *p, brw_inst *insn, struct brw_reg dest);
 168 void brw_set_src0(struct brw_codegen *p, brw_inst *insn, struct brw_reg reg);
 169
 170 void gen6_resolve_implied_move(struct brw_codegen *p,
 171                                struct brw_reg *src,
 172                                unsigned msg_reg_nr);
 173
 174 /* Helpers for regular instructions:
 175  */
 176 #define ALU1(OP)                                \
 177 brw_inst *brw_##OP(struct brw_codegen *p,       \
 178               struct brw_reg dest,              \
 179               struct brw_reg src0);
 180
 181 #define ALU2(OP)                                \
 182 brw_inst *brw_##OP(struct brw_codegen *p,       \
 183               struct brw_reg dest,              \
 184               struct brw_reg src0,              \
 185               struct brw_reg src1);
 186
 187 #define ALU3(OP)                                \
 188 brw_inst *brw_##OP(struct brw_codegen *p,       \
 189               struct brw_reg dest,              \
 190               struct brw_reg src0,              \
 191               struct brw_reg src1,              \
 192               struct brw_reg src2);
 193
 194 #define ROUND(OP) \
 195 void brw_##OP(struct brw_codegen *p, struct brw_reg dest, struct brw_reg src0);
 196
 197 ALU1(MOV)
 198 ALU2(SEL)
 199 ALU1(NOT)
 200 ALU2(AND)
 201 ALU2(OR)
 202 ALU2(XOR)
 203 ALU2(SHR)
 204 ALU2(SHL)
 205 ALU1(DIM)
 206 ALU2(ASR)
 207 ALU3(CSEL)
 208 ALU1(F32TO16)
 209 ALU1(F16TO32)
 210 ALU2(ADD)
 211 ALU2(AVG)
 212 ALU2(MUL)
 213 ALU1(FRC)
 214 ALU1(RNDD)
 215 ALU2(MAC)
 216 ALU2(MACH)
 217 ALU1(LZD)
 218 ALU2(DP4)
 219 ALU2(DPH)
 220 ALU2(DP3)
 221 ALU2(DP2)
 222 ALU2(LINE)
 223 ALU2(PLN)
 224 ALU3(MAD)
 225 ALU3(LRP)
 226 ALU1(BFREV)
 227 ALU3(BFE)
 228 ALU2(BFI1)
 229 ALU3(BFI2)
 230 ALU1(FBH)
 231 ALU1(FBL)
 232 ALU1(CBIT)
 233 ALU2(ADDC)
 234 ALU2(SUBB)
 235 ALU2(MAC)
 236
 237 ROUND(RNDZ)
 238 ROUND(RNDE)
 239
 240 #undef ALU1
 241 #undef ALU2
 242 #undef ALU3
 243 #undef ROUND
 244
 245
 246 /* Helpers for SEND instruction:
 247  */
 248
 249 /**
 250  * Construct a message descriptor immediate with the specified common
 251  * descriptor controls.
 252  */
 253 static inline uint32_t
 254 brw_message_desc(const struct gen_device_info *devinfo,
 255                  unsigned msg_length,
 256                  unsigned response_length,
 257                  bool header_present)
 258 {
 259    if (devinfo->gen >= 5) {
 260       return (SET_BITS(msg_length, 28, 25) |
 261               SET_BITS(response_length, 24, 20) |
 262               SET_BITS(header_present, 19, 19));
 263    } else {
 264       return (SET_BITS(msg_length, 23, 20) |
 265               SET_BITS(response_length, 19, 16));
 266    }
 267 }
 268
 269 static inline unsigned
 270 brw_message_desc_mlen(const struct gen_device_info *devinfo, uint32_t desc)
 271 {
 272    if (devinfo->gen >= 5)
 273       return GET_BITS(desc, 28, 25);
 274    else
 275       return GET_BITS(desc, 23, 20);
 276 }
 277
 278 static inline unsigned
 279 brw_message_desc_rlen(const struct gen_device_info *devinfo, uint32_t desc)
 280 {
 281    if (devinfo->gen >= 5)
 282       return GET_BITS(desc, 24, 20);
 283    else
 284       return GET_BITS(desc, 19, 16);
 285 }
 286
 287 static inline bool
 288 brw_message_desc_header_present(const struct gen_device_info *devinfo,
 289                                 uint32_t desc)
 290 {
 291    assert(devinfo->gen >= 5);
 292    return GET_BITS(desc, 19, 19);
 293 }
 294
 295 static inline unsigned
 296 brw_message_ex_desc(const struct gen_device_info *devinfo,
 297                     unsigned ex_msg_length)
 298 {
 299    return SET_BITS(ex_msg_length, 9, 6);
 300 }
 301
 302 static inline unsigned
 303 brw_message_ex_desc_ex_mlen(const struct gen_device_info *devinfo,
 304                             uint32_t ex_desc)
 305 {
 306    return GET_BITS(ex_desc, 9, 6);
 307 }
 308
 309 /**
 310  * Construct a message descriptor immediate with the specified sampler
 311  * function controls.
 312  */
 313 static inline uint32_t
 314 brw_sampler_desc(const struct gen_device_info *devinfo,
 315                  unsigned binding_table_index,
 316                  unsigned sampler,
 317                  unsigned msg_type,
 318                  unsigned simd_mode,
 319                  unsigned return_format)
 320 {
 321    const unsigned desc = (SET_BITS(binding_table_index, 7, 0) |
 322                           SET_BITS(sampler, 11, 8));
 323    if (devinfo->gen >= 7)
 324       return (desc | SET_BITS(msg_type, 16, 12) |
 325               SET_BITS(simd_mode, 18, 17));
 326    else if (devinfo->gen >= 5)
 327       return (desc | SET_BITS(msg_type, 15, 12) |
 328               SET_BITS(simd_mode, 17, 16));
 329    else if (devinfo->is_g4x)
 330       return desc | SET_BITS(msg_type, 15, 12);
 331    else
 332       return (desc | SET_BITS(return_format, 13, 12) |
 333               SET_BITS(msg_type, 15, 14));
 334 }
 335
 336 static inline unsigned
 337 brw_sampler_desc_binding_table_index(const struct gen_device_info *devinfo,
 338                                      uint32_t desc)
 339 {
 340    return GET_BITS(desc, 7, 0);
 341 }
 342
 343 static inline unsigned
 344 brw_sampler_desc_sampler(const struct gen_device_info *devinfo, uint32_t desc)
 345 {
 346    return GET_BITS(desc, 11, 8);
 347 }
 348
 349 static inline unsigned
 350 brw_sampler_desc_msg_type(const struct gen_device_info *devinfo, uint32_t desc)
 351 {
 352    if (devinfo->gen >= 7)
 353       return GET_BITS(desc, 16, 12);
 354    else if (devinfo->gen >= 5 || devinfo->is_g4x)
 355       return GET_BITS(desc, 15, 12);
 356    else
 357       return GET_BITS(desc, 15, 14);
 358 }
 359
 360 static inline unsigned
 361 brw_sampler_desc_simd_mode(const struct gen_device_info *devinfo, uint32_t desc)
 362 {
 363    assert(devinfo->gen >= 5);
 364    if (devinfo->gen >= 7)
 365       return GET_BITS(desc, 18, 17);
 366    else
 367       return GET_BITS(desc, 17, 16);
 368 }
 369
 370 static  inline unsigned
 371 brw_sampler_desc_return_format(const struct gen_device_info *devinfo,
 372                                uint32_t desc)
 373 {
 374    assert(devinfo->gen == 4 && !devinfo->is_g4x);
 375    return GET_BITS(desc, 13, 12);
 376 }
 377
 378 /**
 379  * Construct a message descriptor for the dataport
 380  */
 381 static inline uint32_t
 382 brw_dp_desc(const struct gen_device_info *devinfo,
 383             unsigned binding_table_index,
 384             unsigned msg_type,
 385             unsigned msg_control)
 386 {
 387    /* Prior to gen6, things are too inconsistent; use the dp_read/write_desc
 388     * helpers instead.
 389     */
 390    assert(devinfo->gen >= 6);
 391    const unsigned desc = SET_BITS(binding_table_index, 7, 0);
 392    if (devinfo->gen >= 8) {
 393       return (desc | SET_BITS(msg_control, 13, 8) |
 394               SET_BITS(msg_type, 18, 14));
 395    } else if (devinfo->gen >= 7) {
 396       return (desc | SET_BITS(msg_control, 13, 8) |
 397               SET_BITS(msg_type, 17, 14));
 398    } else {
 399       return (desc | SET_BITS(msg_control, 12, 8) |
 400               SET_BITS(msg_type, 16, 13));
 401    }
 402 }
 403
 404 static inline unsigned
 405 brw_dp_desc_binding_table_index(const struct gen_device_info *devinfo,
 406                                 uint32_t desc)
 407 {
 408    return GET_BITS(desc, 7, 0);
 409 }
 410
 411 static inline unsigned
 412 brw_dp_desc_msg_type(const struct gen_device_info *devinfo, uint32_t desc)
 413 {
 414    assert(devinfo->gen >= 6);
 415    if (devinfo->gen >= 8)
 416       return GET_BITS(desc, 18, 14);
 417    else if (devinfo->gen >= 7)
 418       return GET_BITS(desc, 17, 14);
 419    else
 420       return GET_BITS(desc, 16, 13);
 421 }
 422
 423 static inline unsigned
 424 brw_dp_desc_msg_control(const struct gen_device_info *devinfo, uint32_t desc)
 425 {
 426    assert(devinfo->gen >= 6);
 427    if (devinfo->gen >= 7)
 428       return GET_BITS(desc, 13, 8);
 429    else
 430       return GET_BITS(desc, 12, 8);
 431 }
 432
 433 /**
 434  * Construct a message descriptor immediate with the specified dataport read
 435  * function controls.
 436  */
 437 static inline uint32_t
 438 brw_dp_read_desc(const struct gen_device_info *devinfo,
 439                  unsigned binding_table_index,
 440                  unsigned msg_control,
 441                  unsigned msg_type,
 442                  unsigned target_cache)
 443 {
 444    if (devinfo->gen >= 6)
 445       return brw_dp_desc(devinfo, binding_table_index, msg_type, msg_control);
 446    else if (devinfo->gen >= 5 || devinfo->is_g4x)
 447       return (SET_BITS(binding_table_index, 7, 0) |
 448               SET_BITS(msg_control, 10, 8) |
 449               SET_BITS(msg_type, 13, 11) |
 450               SET_BITS(target_cache, 15, 14));
 451    else
 452       return (SET_BITS(binding_table_index, 7, 0) |
 453               SET_BITS(msg_control, 11, 8) |
 454               SET_BITS(msg_type, 13, 12) |
 455               SET_BITS(target_cache, 15, 14));
 456 }
 457
 458 static inline unsigned
 459 brw_dp_read_desc_msg_type(const struct gen_device_info *devinfo, uint32_t desc)
 460 {
 461    if (devinfo->gen >= 6)
 462       return brw_dp_desc_msg_type(devinfo, desc);
 463    else if (devinfo->gen >= 5 || devinfo->is_g4x)
 464       return GET_BITS(desc, 13, 11);
 465    else
 466       return GET_BITS(desc, 13, 12);
 467 }
 468
 469 static inline unsigned
 470 brw_dp_read_desc_msg_control(const struct gen_device_info *devinfo,
 471                              uint32_t desc)
 472 {
 473    if (devinfo->gen >= 6)
 474       return brw_dp_desc_msg_control(devinfo, desc);
 475    else if (devinfo->gen >= 5 || devinfo->is_g4x)
 476       return GET_BITS(desc, 10, 8);
 477    else
 478       return GET_BITS(desc, 11, 8);
 479 }
 480
 481 /**
 482  * Construct a message descriptor immediate with the specified dataport write
 483  * function controls.
 484  */
 485 static inline uint32_t
 486 brw_dp_write_desc(const struct gen_device_info *devinfo,
 487                   unsigned binding_table_index,
 488                   unsigned msg_control,
 489                   unsigned msg_type,
 490                   unsigned last_render_target,
 491                   unsigned send_commit_msg)
 492 {
 493    assert(devinfo->gen <= 6 || !send_commit_msg);
 494    if (devinfo->gen >= 6)
 495       return brw_dp_desc(devinfo, binding_table_index, msg_type, msg_control) |
 496              SET_BITS(last_render_target, 12, 12) |
 497              SET_BITS(send_commit_msg, 17, 17);
 498    else
 499       return (SET_BITS(binding_table_index, 7, 0) |
 500               SET_BITS(msg_control, 11, 8) |
 501               SET_BITS(last_render_target, 11, 11) |
 502               SET_BITS(msg_type, 14, 12) |
 503               SET_BITS(send_commit_msg, 15, 15));
 504 }
 505
 506 static inline unsigned
 507 brw_dp_write_desc_msg_type(const struct gen_device_info *devinfo,
 508                            uint32_t desc)
 509 {
 510    if (devinfo->gen >= 6)
 511       return brw_dp_desc_msg_type(devinfo, desc);
 512    else
 513       return GET_BITS(desc, 14, 12);
 514 }
 515
 516 static inline unsigned
 517 brw_dp_write_desc_msg_control(const struct gen_device_info *devinfo,
 518                               uint32_t desc)
 519 {
 520    if (devinfo->gen >= 6)
 521       return brw_dp_desc_msg_control(devinfo, desc);
 522    else
 523       return GET_BITS(desc, 11, 8);
 524 }
 525
 526 static inline bool
 527 brw_dp_write_desc_last_render_target(const struct gen_device_info *devinfo,
 528                                      uint32_t desc)
 529 {
 530    if (devinfo->gen >= 6)
 531       return GET_BITS(desc, 12, 12);
 532    else
 533       return GET_BITS(desc, 11, 11);
 534 }
 535
 536 static inline bool
 537 brw_dp_write_desc_write_commit(const struct gen_device_info *devinfo,
 538                                uint32_t desc)
 539 {
 540    assert(devinfo->gen <= 6);
 541    if (devinfo->gen >= 6)
 542       return GET_BITS(desc, 17, 17);
 543    else
 544       return GET_BITS(desc, 15, 15);
 545 }
 546
 547 /**
 548  * Construct a message descriptor immediate with the specified dataport
 549  * surface function controls.
 550  */
 551 static inline uint32_t
 552 brw_dp_surface_desc(const struct gen_device_info *devinfo,
 553                     unsigned msg_type,
 554                     unsigned msg_control)
 555 {
 556    assert(devinfo->gen >= 7);
 557    /* We'll OR in the binding table index later */
 558    return brw_dp_desc(devinfo, 0, msg_type, msg_control);
 559 }
 560
 561 static inline uint32_t
 562 brw_dp_untyped_atomic_desc(const struct gen_device_info *devinfo,
 563                            unsigned exec_size, /**< 0 for SIMD4x2 */
 564                            unsigned atomic_op,
 565                            bool response_expected)
 566 {
 567    assert(exec_size <= 8 || exec_size == 16);
 568
 569    unsigned msg_type;
 570    if (devinfo->gen >= 8 || devinfo->is_haswell) {
 571       if (exec_size > 0) {
 572          msg_type = HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP;
 573       } else {
 574          msg_type = HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP_SIMD4X2;
 575       }
 576    } else {
 577       msg_type = GEN7_DATAPORT_DC_UNTYPED_ATOMIC_OP;
 578    }
 579
 580    const unsigned msg_control =
 581       SET_BITS(atomic_op, 3, 0) |
 582       SET_BITS(0 < exec_size && exec_size <= 8, 4, 4) |
 583       SET_BITS(response_expected, 5, 5);
 584
 585    return brw_dp_surface_desc(devinfo, msg_type, msg_control);
 586 }
 587
 588 static inline uint32_t
 589 brw_dp_untyped_atomic_float_desc(const struct gen_device_info *devinfo,
 590                                  unsigned exec_size,
 591                                  unsigned atomic_op,
 592                                  bool response_expected)
 593 {
 594    assert(exec_size <= 8 || exec_size == 16);
 595    assert(devinfo->gen >= 9);
 596
 597    assert(exec_size > 0);
 598    const unsigned msg_type = GEN9_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_FLOAT_OP;
 599
 600    const unsigned msg_control =
 601       SET_BITS(atomic_op, 1, 0) |
 602       SET_BITS(exec_size <= 8, 4, 4) |
 603       SET_BITS(response_expected, 5, 5);
 604
 605    return brw_dp_surface_desc(devinfo, msg_type, msg_control);
 606 }
 607
 608 static inline unsigned
 609 brw_mdc_cmask(unsigned num_channels)
 610 {
 611    /* See also MDC_CMASK in the SKL PRM Vol 2d. */
 612    return 0xf & (0xf << num_channels);
 613 }
 614
 615 static inline uint32_t
 616 brw_dp_untyped_surface_rw_desc(const struct gen_device_info *devinfo,
 617                                unsigned exec_size, /**< 0 for SIMD4x2 */
 618                                unsigned num_channels,
 619                                bool write)
 620 {
 621    assert(exec_size <= 8 || exec_size == 16);
 622
 623    unsigned msg_type;
 624    if (write) {
 625       if (devinfo->gen >= 8 || devinfo->is_haswell) {
 626          msg_type = HSW_DATAPORT_DC_PORT1_UNTYPED_SURFACE_WRITE;
 627       } else {
 628          msg_type = GEN7_DATAPORT_DC_UNTYPED_SURFACE_WRITE;
 629       }
 630    } else {
 631       /* Read */
 632       if (devinfo->gen >= 8 || devinfo->is_haswell) {
 633          msg_type = HSW_DATAPORT_DC_PORT1_UNTYPED_SURFACE_READ;
 634       } else {
 635          msg_type = GEN7_DATAPORT_DC_UNTYPED_SURFACE_READ;
 636       }
 637    }
 638
 639    /* SIMD4x2 is only valid for read messages on IVB; use SIMD8 instead */
 640    if (write && devinfo->gen == 7 && !devinfo->is_haswell && exec_size == 0)
 641       exec_size = 8;
 642
 643    /* See also MDC_SM3 in the SKL PRM Vol 2d. */
 644    const unsigned simd_mode = exec_size == 0 ? 0 : /* SIMD4x2 */
 645                               exec_size <= 8 ? 2 : 1;
 646
 647    const unsigned msg_control =
 648       SET_BITS(brw_mdc_cmask(num_channels), 3, 0) |
 649       SET_BITS(simd_mode, 5, 4);
 650
 651    return brw_dp_surface_desc(devinfo, msg_type, msg_control);
 652 }
 653
 654 static inline unsigned
 655 brw_mdc_ds(unsigned bit_size)
 656 {
 657    switch (bit_size) {
 658    case 8:
 659       return GEN7_BYTE_SCATTERED_DATA_ELEMENT_BYTE;
 660    case 16:
 661       return GEN7_BYTE_SCATTERED_DATA_ELEMENT_WORD;
 662    case 32:
 663       return GEN7_BYTE_SCATTERED_DATA_ELEMENT_DWORD;
 664    default:
 665       unreachable("Unsupported bit_size for byte scattered messages");
 666    }
 667 }
 668
 669 static inline uint32_t
 670 brw_dp_byte_scattered_rw_desc(const struct gen_device_info *devinfo,
 671                               unsigned exec_size,
 672                               unsigned bit_size,
 673                               bool write)
 674 {
 675    assert(exec_size <= 8 || exec_size == 16);
 676
 677    assert(devinfo->gen > 7 || devinfo->is_haswell);
 678    const unsigned msg_type =
 679       write ? HSW_DATAPORT_DC_PORT0_BYTE_SCATTERED_WRITE :
 680               HSW_DATAPORT_DC_PORT0_BYTE_SCATTERED_READ;
 681
 682    assert(exec_size > 0);
 683    const unsigned msg_control =
 684       SET_BITS(exec_size == 16, 0, 0) |
 685       SET_BITS(brw_mdc_ds(bit_size), 3, 2);
 686
 687    return brw_dp_surface_desc(devinfo, msg_type, msg_control);
 688 }
 689
 690 static inline uint32_t
 691 brw_dp_typed_atomic_desc(const struct gen_device_info *devinfo,
 692                          unsigned exec_size,
 693                          unsigned exec_group,
 694                          unsigned atomic_op,
 695                          bool response_expected)
 696 {
 697    assert(exec_size > 0 || exec_group == 0);
 698    assert(exec_group % 8 == 0);
 699
 700    unsigned msg_type;
 701    if (devinfo->gen >= 8 || devinfo->is_haswell) {
 702       if (exec_size == 0) {
 703          msg_type = HSW_DATAPORT_DC_PORT1_TYPED_ATOMIC_OP_SIMD4X2;
 704       } else {
 705          msg_type = HSW_DATAPORT_DC_PORT1_TYPED_ATOMIC_OP;
 706       }
 707    } else {
 708       /* SIMD4x2 typed surface R/W messages only exist on HSW+ */
 709       assert(exec_size > 0);
 710       msg_type = GEN7_DATAPORT_RC_TYPED_ATOMIC_OP;
 711    }
 712
 713    const bool high_sample_mask = (exec_group / 8) % 2 == 1;
 714
 715    const unsigned msg_control =
 716       SET_BITS(atomic_op, 3, 0) |
 717       SET_BITS(high_sample_mask, 4, 4) |
 718       SET_BITS(response_expected, 5, 5);
 719
 720    return brw_dp_surface_desc(devinfo, msg_type, msg_control);
 721 }
 722
 723 static inline uint32_t
 724 brw_dp_typed_surface_rw_desc(const struct gen_device_info *devinfo,
 725                              unsigned exec_size,
 726                              unsigned exec_group,
 727                              unsigned num_channels,
 728                              bool write)
 729 {
 730    assert(exec_size > 0 || exec_group == 0);
 731    assert(exec_group % 8 == 0);
 732
 733    /* Typed surface reads and writes don't support SIMD16 */
 734    assert(exec_size <= 8);
 735
 736    unsigned msg_type;
 737    if (write) {
 738       if (devinfo->gen >= 8 || devinfo->is_haswell) {
 739          msg_type = HSW_DATAPORT_DC_PORT1_TYPED_SURFACE_WRITE;
 740       } else {
 741          msg_type = GEN7_DATAPORT_RC_TYPED_SURFACE_WRITE;
 742       }
 743    } else {
 744       if (devinfo->gen >= 8 || devinfo->is_haswell) {
 745          msg_type = HSW_DATAPORT_DC_PORT1_TYPED_SURFACE_READ;
 746       } else {
 747          msg_type = GEN7_DATAPORT_RC_TYPED_SURFACE_READ;
 748       }
 749    }
 750
 751    /* See also MDC_SG3 in the SKL PRM Vol 2d. */
 752    unsigned msg_control;
 753    if (devinfo->gen >= 8 || devinfo->is_haswell) {
 754       /* See also MDC_SG3 in the SKL PRM Vol 2d. */
 755       const unsigned slot_group = exec_size == 0 ? 0 : /* SIMD4x2 */
 756                                   1 + ((exec_group / 8) % 2);
 757
 758       msg_control =
 759          SET_BITS(brw_mdc_cmask(num_channels), 3, 0) |
 760          SET_BITS(slot_group, 5, 4);
 761    } else {
 762       /* SIMD4x2 typed surface R/W messages only exist on HSW+ */
 763       assert(exec_size > 0);
 764       const unsigned slot_group = ((exec_group / 8) % 2);
 765
 766       msg_control =
 767          SET_BITS(brw_mdc_cmask(num_channels), 3, 0) |
 768          SET_BITS(slot_group, 5, 5);
 769    }
 770
 771    return brw_dp_surface_desc(devinfo, msg_type, msg_control);
 772 }
 773
 774 /**
 775  * Construct a message descriptor immediate with the specified pixel
 776  * interpolator function controls.
 777  */
 778 static inline uint32_t
 779 brw_pixel_interp_desc(UNUSED const struct gen_device_info *devinfo,
 780                       unsigned msg_type,
 781                       bool noperspective,
 782                       unsigned simd_mode,
 783                       unsigned slot_group)
 784 {
 785    return (SET_BITS(slot_group, 11, 11) |
 786            SET_BITS(msg_type, 13, 12) |
 787            SET_BITS(!!noperspective, 14, 14) |
 788            SET_BITS(simd_mode, 16, 16));
 789 }
 790
 791 void brw_urb_WRITE(struct brw_codegen *p,
 792                    struct brw_reg dest,
 793                    unsigned msg_reg_nr,
 794                    struct brw_reg src0,
 795                    enum brw_urb_write_flags flags,
 796                    unsigned msg_length,
 797                    unsigned response_length,
 798                    unsigned offset,
 799                    unsigned swizzle);
 800
 801 /**
 802  * Send message to shared unit \p sfid with a possibly indirect descriptor \p
 803  * desc.  If \p desc is not an immediate it will be transparently loaded to an
 804  * address register using an OR instruction.
 805  */
 806 void
 807 brw_send_indirect_message(struct brw_codegen *p,
 808                           unsigned sfid,
 809                           struct brw_reg dst,
 810                           struct brw_reg payload,
 811                           struct brw_reg desc,
 812                           unsigned desc_imm);
 813
 814 void
 815 brw_send_indirect_split_message(struct brw_codegen *p,
 816                                 unsigned sfid,
 817                                 struct brw_reg dst,
 818                                 struct brw_reg payload0,
 819                                 struct brw_reg payload1,
 820                                 struct brw_reg desc,
 821                                 unsigned desc_imm,
 822                                 struct brw_reg ex_desc,
 823                                 unsigned ex_desc_imm);
 824
 825 void brw_ff_sync(struct brw_codegen *p,
 826                    struct brw_reg dest,
 827                    unsigned msg_reg_nr,
 828                    struct brw_reg src0,
 829                    bool allocate,
 830                    unsigned response_length,
 831                    bool eot);
 832
 833 void brw_svb_write(struct brw_codegen *p,
 834                    struct brw_reg dest,
 835                    unsigned msg_reg_nr,
 836                    struct brw_reg src0,
 837                    unsigned binding_table_index,
 838                    bool   send_commit_msg);
 839
 840 brw_inst *brw_fb_WRITE(struct brw_codegen *p,
 841                        struct brw_reg payload,
 842                        struct brw_reg implied_header,
 843                        unsigned msg_control,
 844                        unsigned binding_table_index,
 845                        unsigned msg_length,
 846                        unsigned response_length,
 847                        bool eot,
 848                        bool last_render_target,
 849                        bool header_present);
 850
 851 brw_inst *gen9_fb_READ(struct brw_codegen *p,
 852                        struct brw_reg dst,
 853                        struct brw_reg payload,
 854                        unsigned binding_table_index,
 855                        unsigned msg_length,
 856                        unsigned response_length,
 857                        bool per_sample);
 858
 859 void brw_SAMPLE(struct brw_codegen *p,
 860                 struct brw_reg dest,
 861                 unsigned msg_reg_nr,
 862                 struct brw_reg src0,
 863                 unsigned binding_table_index,
 864                 unsigned sampler,
 865                 unsigned msg_type,
 866                 unsigned response_length,
 867                 unsigned msg_length,
 868                 unsigned header_present,
 869                 unsigned simd_mode,
 870                 unsigned return_format);
 871
 872 void brw_adjust_sampler_state_pointer(struct brw_codegen *p,
 873                                       struct brw_reg header,
 874                                       struct brw_reg sampler_index);
 875
 876 void gen4_math(struct brw_codegen *p,
 877                struct brw_reg dest,
 878                unsigned function,
 879                unsigned msg_reg_nr,
 880                struct brw_reg src,
 881                unsigned precision );
 882
 883 void gen6_math(struct brw_codegen *p,
 884                struct brw_reg dest,
 885                unsigned function,
 886                struct brw_reg src0,
 887                struct brw_reg src1);
 888
 889 void brw_oword_block_read(struct brw_codegen *p,
 890                           struct brw_reg dest,
 891                           struct brw_reg mrf,
 892                           uint32_t offset,
 893                           uint32_t bind_table_index);
 894
 895 unsigned brw_scratch_surface_idx(const struct brw_codegen *p);
 896
 897 void brw_oword_block_read_scratch(struct brw_codegen *p,
 898                                   struct brw_reg dest,
 899                                   struct brw_reg mrf,
 900                                   int num_regs,
 901                                   unsigned offset);
 902
 903 void brw_oword_block_write_scratch(struct brw_codegen *p,
 904                                    struct brw_reg mrf,
 905                                    int num_regs,
 906                                    unsigned offset);
 907
 908 void gen7_block_read_scratch(struct brw_codegen *p,
 909                              struct brw_reg dest,
 910                              int num_regs,
 911                              unsigned offset);
 912
 913 void brw_shader_time_add(struct brw_codegen *p,
 914                          struct brw_reg payload,
 915                          uint32_t surf_index);
 916
 917 /**
 918  * Return the generation-specific jump distance scaling factor.
 919  *
 920  * Given the number of instructions to jump, we need to scale by
 921  * some number to obtain the actual jump distance to program in an
 922  * instruction.
 923  */
 924 static inline unsigned
 925 brw_jump_scale(const struct gen_device_info *devinfo)
 926 {
 927    /* Broadwell measures jump targets in bytes. */
 928    if (devinfo->gen >= 8)
 929       return 16;
 930
 931    /* Ironlake and later measure jump targets in 64-bit data chunks (in order
 932     * (to support compaction), so each 128-bit instruction requires 2 chunks.
 933     */
 934    if (devinfo->gen >= 5)
 935       return 2;
 936
 937    /* Gen4 simply uses the number of 128-bit instructions. */
 938    return 1;
 939 }
 940
 941 void brw_barrier(struct brw_codegen *p, struct brw_reg src);
 942
 943 /* If/else/endif.  Works by manipulating the execution flags on each
 944  * channel.
 945  */
 946 brw_inst *brw_IF(struct brw_codegen *p, unsigned execute_size);
 947 brw_inst *gen6_IF(struct brw_codegen *p, enum brw_conditional_mod conditional,
 948                   struct brw_reg src0, struct brw_reg src1);
 949
 950 void brw_ELSE(struct brw_codegen *p);
 951 void brw_ENDIF(struct brw_codegen *p);
 952
 953 /* DO/WHILE loops:
 954  */
 955 brw_inst *brw_DO(struct brw_codegen *p, unsigned execute_size);
 956
 957 brw_inst *brw_WHILE(struct brw_codegen *p);
 958
 959 brw_inst *brw_BREAK(struct brw_codegen *p);
 960 brw_inst *brw_CONT(struct brw_codegen *p);
 961 brw_inst *gen6_HALT(struct brw_codegen *p);
 962
 963 /* Forward jumps:
 964  */
 965 void brw_land_fwd_jump(struct brw_codegen *p, int jmp_insn_idx);
 966
 967 brw_inst *brw_JMPI(struct brw_codegen *p, struct brw_reg index,
 968                    unsigned predicate_control);
 969
 970 void brw_NOP(struct brw_codegen *p);
 971
 972 void brw_WAIT(struct brw_codegen *p);
 973
 974 /* Special case: there is never a destination, execution size will be
 975  * taken from src0:
 976  */
 977 void brw_CMP(struct brw_codegen *p,
 978              struct brw_reg dest,
 979              unsigned conditional,
 980              struct brw_reg src0,
 981              struct brw_reg src1);
 982
 983 void
 984 brw_untyped_atomic(struct brw_codegen *p,
 985                    struct brw_reg dst,
 986                    struct brw_reg payload,
 987                    struct brw_reg surface,
 988                    unsigned atomic_op,
 989                    unsigned msg_length,
 990                    bool response_expected,
 991                    bool header_present);
 992
 993 void
 994 brw_untyped_surface_read(struct brw_codegen *p,
 995                          struct brw_reg dst,
 996                          struct brw_reg payload,
 997                          struct brw_reg surface,
 998                          unsigned msg_length,
 999                          unsigned num_channels);
1000
1001 void
1002 brw_untyped_surface_write(struct brw_codegen *p,
1003                           struct brw_reg payload,
1004                           struct brw_reg surface,
1005                           unsigned msg_length,
1006                           unsigned num_channels,
1007                           bool header_present);
1008
1009 void
1010 brw_typed_atomic(struct brw_codegen *p,
1011                  struct brw_reg dst,
1012                  struct brw_reg payload,
1013                  struct brw_reg surface,
1014                  unsigned atomic_op,
1015                  unsigned msg_length,
1016                  bool response_expected,
1017                  bool header_present);
1018
1019 void
1020 brw_typed_surface_read(struct brw_codegen *p,
1021                        struct brw_reg dst,
1022                        struct brw_reg payload,
1023                        struct brw_reg surface,
1024                        unsigned msg_length,
1025                        unsigned num_channels,
1026                        bool header_present);
1027
1028 void
1029 brw_typed_surface_write(struct brw_codegen *p,
1030                         struct brw_reg payload,
1031                         struct brw_reg surface,
1032                         unsigned msg_length,
1033                         unsigned num_channels,
1034                         bool header_present);
1035
1036 void
1037 brw_memory_fence(struct brw_codegen *p,
1038                  struct brw_reg dst,
1039                  enum opcode send_op);
1040
1041 void
1042 brw_pixel_interpolator_query(struct brw_codegen *p,
1043                              struct brw_reg dest,
1044                              struct brw_reg mrf,
1045                              bool noperspective,
1046                              unsigned mode,
1047                              struct brw_reg data,
1048                              unsigned msg_length,
1049                              unsigned response_length);
1050
1051 void
1052 brw_find_live_channel(struct brw_codegen *p,
1053                       struct brw_reg dst,
1054                       struct brw_reg mask);
1055
1056 void
1057 brw_broadcast(struct brw_codegen *p,
1058               struct brw_reg dst,
1059               struct brw_reg src,
1060               struct brw_reg idx);
1061
1062 void
1063 brw_rounding_mode(struct brw_codegen *p,
1064                   enum brw_rnd_mode mode);
1065
1066 /***********************************************************************
1067  * brw_eu_util.c:
1068  */
1069
1070 void brw_copy_indirect_to_indirect(struct brw_codegen *p,
1071                                    struct brw_indirect dst_ptr,
1072                                    struct brw_indirect src_ptr,
1073                                    unsigned count);
1074
1075 void brw_copy_from_indirect(struct brw_codegen *p,
1076                             struct brw_reg dst,
1077                             struct brw_indirect ptr,
1078                             unsigned count);
1079
1080 void brw_copy4(struct brw_codegen *p,
1081                struct brw_reg dst,
1082                struct brw_reg src,
1083                unsigned count);
1084
1085 void brw_copy8(struct brw_codegen *p,
1086                struct brw_reg dst,
1087                struct brw_reg src,
1088                unsigned count);
1089
1090 void brw_math_invert( struct brw_codegen *p,
1091                       struct brw_reg dst,
1092                       struct brw_reg src);
1093
1094 void brw_set_src1(struct brw_codegen *p, brw_inst *insn, struct brw_reg reg);
1095
1096 void brw_set_desc_ex(struct brw_codegen *p, brw_inst *insn,
1097                      unsigned desc, unsigned ex_desc);
1098
1099 static inline void
1100 brw_set_desc(struct brw_codegen *p, brw_inst *insn, unsigned desc)
1101 {
1102    brw_set_desc_ex(p, insn, desc, 0);
1103 }
1104
1105 void brw_set_uip_jip(struct brw_codegen *p, int start_offset);
1106
1107 enum brw_conditional_mod brw_negate_cmod(uint32_t cmod);
1108 enum brw_conditional_mod brw_swap_cmod(uint32_t cmod);
1109
1110 /* brw_eu_compact.c */
1111 void brw_init_compaction_tables(const struct gen_device_info *devinfo);
1112 void brw_compact_instructions(struct brw_codegen *p, int start_offset,
1113                               struct disasm_info *disasm);
1114 void brw_uncompact_instruction(const struct gen_device_info *devinfo,
1115                                brw_inst *dst, brw_compact_inst *src);
1116 bool brw_try_compact_instruction(const struct gen_device_info *devinfo,
1117                                  brw_compact_inst *dst, const brw_inst *src);
1118
1119 void brw_debug_compact_uncompact(const struct gen_device_info *devinfo,
1120                                  brw_inst *orig, brw_inst *uncompacted);
1121
1122 /* brw_eu_validate.c */
1123 bool brw_validate_instructions(const struct gen_device_info *devinfo,
1124                                const void *assembly, int start_offset, int end_offset,
1125                                struct disasm_info *disasm);
1126
1127 static inline int
1128 next_offset(const struct gen_device_info *devinfo, void *store, int offset)
1129 {
1130    brw_inst *insn = (brw_inst *)((char *)store + offset);
1131
1132    if (brw_inst_cmpt_control(devinfo, insn))
1133       return offset + 8;
1134    else
1135       return offset + 16;
1136 }
1137
1138 struct opcode_desc {
1139    /* The union is an implementation detail used by brw_opcode_desc() to handle
1140     * opcodes that have been reused for different instructions across hardware
1141     * generations.
1142     *
1143     * The gens field acts as a tag. If it is non-zero, name points to a string
1144     * containing the instruction mnemonic. If it is zero, the table field is
1145     * valid and either points to a secondary opcode_desc table with 'size'
1146     * elements or is NULL and no such instruction exists for the opcode.
1147     */
1148    union {
1149       struct {
1150          char    *name;
1151          int      nsrc;
1152       };
1153       struct {
1154          const struct opcode_desc *table;
1155          unsigned size;
1156       };
1157    };
1158    int      ndst;
1159    int      gens;
1160 };
1161
1162 const struct opcode_desc *
1163 brw_opcode_desc(const struct gen_device_info *devinfo, enum opcode opcode);
1164
1165 static inline bool
1166 is_3src(const struct gen_device_info *devinfo, enum opcode opcode)
1167 {
1168    const struct opcode_desc *desc = brw_opcode_desc(devinfo, opcode);
1169    return desc && desc->nsrc == 3;
1170 }
1171
1172 /** Maximum SEND message length */
1173 #define BRW_MAX_MSG_LENGTH 15
1174
1175 /** First MRF register used by pull loads */
1176 #define FIRST_SPILL_MRF(gen) ((gen) == 6 ? 21 : 13)
1177
1178 /** First MRF register used by spills */
1179 #define FIRST_PULL_LOAD_MRF(gen) ((gen) == 6 ? 16 : 13)
1180
1181 #ifdef __cplusplus
1182 }
1183 #endif
1184
1185 #endif