src/intel/compiler/brw_eu.h

   1 /*
   2  Copyright (C) Intel Corp.  2006.  All Rights Reserved.
   3  Intel funded Tungsten Graphics to
   4  develop this 3D driver.
   5
   6  Permission is hereby granted, free of charge, to any person obtaining
   7  a copy of this software and associated documentation files (the
   8  "Software"), to deal in the Software without restriction, including
   9  without limitation the rights to use, copy, modify, merge, publish,
  10  distribute, sublicense, and/or sell copies of the Software, and to
  11  permit persons to whom the Software is furnished to do so, subject to
  12  the following conditions:
  13
  14  The above copyright notice and this permission notice (including the
  15  next paragraph) shall be included in all copies or substantial
  16  portions of the Software.
  17
  18  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  19  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  21  IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
  22  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  23  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  24  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25
  26  **********************************************************************/
  27  /*
  28   * Authors:
  29   *   Keith Whitwell <keithw@vmware.com>
  30   */
  31
  32
  33 #ifndef BRW_EU_H
  34 #define BRW_EU_H
  35
  36 #include <stdbool.h>
  37 #include <stdio.h>
  38 #include "brw_inst.h"
  39 #include "brw_eu_defines.h"
  40 #include "brw_reg.h"
  41 #include "brw_disasm_info.h"
  42
  43 #ifdef __cplusplus
  44 extern "C" {
  45 #endif
  46
  47 #define BRW_EU_MAX_INSN_STACK 5
  48
  49 struct brw_insn_state {
  50    /* One of BRW_EXECUTE_* */
  51    unsigned exec_size:3;
  52
  53    /* Group in units of channels */
  54    unsigned group:5;
  55
  56    /* Compression control on gen4-5 */
  57    bool compressed:1;
  58
  59    /* One of BRW_MASK_* */
  60    unsigned mask_control:1;
  61
  62    bool saturate:1;
  63
  64    /* One of BRW_ALIGN_* */
  65    unsigned access_mode:1;
  66
  67    /* One of BRW_PREDICATE_* */
  68    enum brw_predicate predicate:4;
  69
  70    bool pred_inv:1;
  71
  72    /* Flag subreg.  Bottom bit is subreg, top bit is reg */
  73    unsigned flag_subreg:2;
  74
  75    bool acc_wr_control:1;
  76 };
  77
  78
  79 /* A helper for accessing the last instruction emitted.  This makes it easy
  80  * to set various bits on an instruction without having to create temporary
  81  * variable and assign the emitted instruction to those.
  82  */
  83 #define brw_last_inst (&p->store[p->nr_insn - 1])
  84
  85 struct brw_codegen {
  86    brw_inst *store;
  87    int store_size;
  88    unsigned nr_insn;
  89    unsigned int next_insn_offset;
  90
  91    void *mem_ctx;
  92
  93    /* Allow clients to push/pop instruction state:
  94     */
  95    struct brw_insn_state stack[BRW_EU_MAX_INSN_STACK];
  96    struct brw_insn_state *current;
  97
  98    /** Whether or not the user wants automatic exec sizes
  99     *
 100     * If true, codegen will try to automatically infer the exec size of an
 101     * instruction from the width of the destination register.  If false, it
 102     * will take whatever is set by brw_set_default_exec_size verbatim.
 103     *
 104     * This is set to true by default in brw_init_codegen.
 105     */
 106    bool automatic_exec_sizes;
 107
 108    bool single_program_flow;
 109    const struct gen_device_info *devinfo;
 110
 111    /* Control flow stacks:
 112     * - if_stack contains IF and ELSE instructions which must be patched
 113     *   (and popped) once the matching ENDIF instruction is encountered.
 114     *
 115     *   Just store the instruction pointer(an index).
 116     */
 117    int *if_stack;
 118    int if_stack_depth;
 119    int if_stack_array_size;
 120
 121    /**
 122     * loop_stack contains the instruction pointers of the starts of loops which
 123     * must be patched (and popped) once the matching WHILE instruction is
 124     * encountered.
 125     */
 126    int *loop_stack;
 127    /**
 128     * pre-gen6, the BREAK and CONT instructions had to tell how many IF/ENDIF
 129     * blocks they were popping out of, to fix up the mask stack.  This tracks
 130     * the IF/ENDIF nesting in each current nested loop level.
 131     */
 132    int *if_depth_in_loop;
 133    int loop_stack_depth;
 134    int loop_stack_array_size;
 135 };
 136
 137 void brw_pop_insn_state( struct brw_codegen *p );
 138 void brw_push_insn_state( struct brw_codegen *p );
 139 unsigned brw_get_default_exec_size(struct brw_codegen *p);
 140 unsigned brw_get_default_group(struct brw_codegen *p);
 141 unsigned brw_get_default_access_mode(struct brw_codegen *p);
 142 void brw_set_default_exec_size(struct brw_codegen *p, unsigned value);
 143 void brw_set_default_mask_control( struct brw_codegen *p, unsigned value );
 144 void brw_set_default_saturate( struct brw_codegen *p, bool enable );
 145 void brw_set_default_access_mode( struct brw_codegen *p, unsigned access_mode );
 146 void brw_inst_set_compression(const struct gen_device_info *devinfo,
 147                               brw_inst *inst, bool on);
 148 void brw_set_default_compression(struct brw_codegen *p, bool on);
 149 void brw_inst_set_group(const struct gen_device_info *devinfo,
 150                         brw_inst *inst, unsigned group);
 151 void brw_set_default_group(struct brw_codegen *p, unsigned group);
 152 void brw_set_default_compression_control(struct brw_codegen *p, enum brw_compression c);
 153 void brw_set_default_predicate_control( struct brw_codegen *p, unsigned pc );
 154 void brw_set_default_predicate_inverse(struct brw_codegen *p, bool predicate_inverse);
 155 void brw_set_default_flag_reg(struct brw_codegen *p, int reg, int subreg);
 156 void brw_set_default_acc_write_control(struct brw_codegen *p, unsigned value);
 157
 158 void brw_init_codegen(const struct gen_device_info *, struct brw_codegen *p,
 159                       void *mem_ctx);
 160 int brw_disassemble_inst(FILE *file, const struct gen_device_info *devinfo,
 161                          const struct brw_inst *inst, bool is_compacted);
 162 void brw_disassemble(const struct gen_device_info *devinfo,
 163                      const void *assembly, int start, int end, FILE *out);
 164 const unsigned *brw_get_program( struct brw_codegen *p, unsigned *sz );
 165
 166 brw_inst *brw_next_insn(struct brw_codegen *p, unsigned opcode);
 167 void brw_set_dest(struct brw_codegen *p, brw_inst *insn, struct brw_reg dest);
 168 void brw_set_src0(struct brw_codegen *p, brw_inst *insn, struct brw_reg reg);
 169
 170 void gen6_resolve_implied_move(struct brw_codegen *p,
 171                                struct brw_reg *src,
 172                                unsigned msg_reg_nr);
 173
 174 /* Helpers for regular instructions:
 175  */
 176 #define ALU1(OP)                                \
 177 brw_inst *brw_##OP(struct brw_codegen *p,       \
 178               struct brw_reg dest,              \
 179               struct brw_reg src0);
 180
 181 #define ALU2(OP)                                \
 182 brw_inst *brw_##OP(struct brw_codegen *p,       \
 183               struct brw_reg dest,              \
 184               struct brw_reg src0,              \
 185               struct brw_reg src1);
 186
 187 #define ALU3(OP)                                \
 188 brw_inst *brw_##OP(struct brw_codegen *p,       \
 189               struct brw_reg dest,              \
 190               struct brw_reg src0,              \
 191               struct brw_reg src1,              \
 192               struct brw_reg src2);
 193
 194 #define ROUND(OP) \
 195 void brw_##OP(struct brw_codegen *p, struct brw_reg dest, struct brw_reg src0);
 196
 197 ALU1(MOV)
 198 ALU2(SEL)
 199 ALU1(NOT)
 200 ALU2(AND)
 201 ALU2(OR)
 202 ALU2(XOR)
 203 ALU2(SHR)
 204 ALU2(SHL)
 205 ALU1(DIM)
 206 ALU2(ASR)
 207 ALU3(CSEL)
 208 ALU1(F32TO16)
 209 ALU1(F16TO32)
 210 ALU2(ADD)
 211 ALU2(AVG)
 212 ALU2(MUL)
 213 ALU1(FRC)
 214 ALU1(RNDD)
 215 ALU2(MAC)
 216 ALU2(MACH)
 217 ALU1(LZD)
 218 ALU2(DP4)
 219 ALU2(DPH)
 220 ALU2(DP3)
 221 ALU2(DP2)
 222 ALU2(LINE)
 223 ALU2(PLN)
 224 ALU3(MAD)
 225 ALU3(LRP)
 226 ALU1(BFREV)
 227 ALU3(BFE)
 228 ALU2(BFI1)
 229 ALU3(BFI2)
 230 ALU1(FBH)
 231 ALU1(FBL)
 232 ALU1(CBIT)
 233 ALU2(ADDC)
 234 ALU2(SUBB)
 235 ALU2(MAC)
 236
 237 ROUND(RNDZ)
 238 ROUND(RNDE)
 239
 240 #undef ALU1
 241 #undef ALU2
 242 #undef ALU3
 243 #undef ROUND
 244
 245
 246 /* Helpers for SEND instruction:
 247  */
 248
 249 /**
 250  * Construct a message descriptor immediate with the specified common
 251  * descriptor controls.
 252  */
 253 static inline uint32_t
 254 brw_message_desc(const struct gen_device_info *devinfo,
 255                  unsigned msg_length,
 256                  unsigned response_length,
 257                  bool header_present)
 258 {
 259    if (devinfo->gen >= 5) {
 260       return (SET_BITS(msg_length, 28, 25) |
 261               SET_BITS(response_length, 24, 20) |
 262               SET_BITS(header_present, 19, 19));
 263    } else {
 264       return (SET_BITS(msg_length, 23, 20) |
 265               SET_BITS(response_length, 19, 16));
 266    }
 267 }
 268
 269 static inline unsigned
 270 brw_message_desc_mlen(const struct gen_device_info *devinfo, uint32_t desc)
 271 {
 272    if (devinfo->gen >= 5)
 273       return GET_BITS(desc, 28, 25);
 274    else
 275       return GET_BITS(desc, 23, 20);
 276 }
 277
 278 static inline unsigned
 279 brw_message_desc_rlen(const struct gen_device_info *devinfo, uint32_t desc)
 280 {
 281    if (devinfo->gen >= 5)
 282       return GET_BITS(desc, 24, 20);
 283    else
 284       return GET_BITS(desc, 19, 16);
 285 }
 286
 287 static inline bool
 288 brw_message_desc_header_present(const struct gen_device_info *devinfo,
 289                                 uint32_t desc)
 290 {
 291    assert(devinfo->gen >= 5);
 292    return GET_BITS(desc, 19, 19);
 293 }
 294
 295 static inline unsigned
 296 brw_message_ex_desc(const struct gen_device_info *devinfo,
 297                     unsigned ex_msg_length)
 298 {
 299    return SET_BITS(ex_msg_length, 9, 6);
 300 }
 301
 302 static inline unsigned
 303 brw_message_ex_desc_ex_mlen(const struct gen_device_info *devinfo,
 304                             uint32_t ex_desc)
 305 {
 306    return GET_BITS(ex_desc, 9, 6);
 307 }
 308
 309 /**
 310  * Construct a message descriptor immediate with the specified sampler
 311  * function controls.
 312  */
 313 static inline uint32_t
 314 brw_sampler_desc(const struct gen_device_info *devinfo,
 315                  unsigned binding_table_index,
 316                  unsigned sampler,
 317                  unsigned msg_type,
 318                  unsigned simd_mode,
 319                  unsigned return_format)
 320 {
 321    const unsigned desc = (SET_BITS(binding_table_index, 7, 0) |
 322                           SET_BITS(sampler, 11, 8));
 323    if (devinfo->gen >= 7)
 324       return (desc | SET_BITS(msg_type, 16, 12) |
 325               SET_BITS(simd_mode, 18, 17));
 326    else if (devinfo->gen >= 5)
 327       return (desc | SET_BITS(msg_type, 15, 12) |
 328               SET_BITS(simd_mode, 17, 16));
 329    else if (devinfo->is_g4x)
 330       return desc | SET_BITS(msg_type, 15, 12);
 331    else
 332       return (desc | SET_BITS(return_format, 13, 12) |
 333               SET_BITS(msg_type, 15, 14));
 334 }
 335
 336 static inline unsigned
 337 brw_sampler_desc_binding_table_index(const struct gen_device_info *devinfo,
 338                                      uint32_t desc)
 339 {
 340    return GET_BITS(desc, 7, 0);
 341 }
 342
 343 static inline unsigned
 344 brw_sampler_desc_sampler(const struct gen_device_info *devinfo, uint32_t desc)
 345 {
 346    return GET_BITS(desc, 11, 8);
 347 }
 348
 349 static inline unsigned
 350 brw_sampler_desc_msg_type(const struct gen_device_info *devinfo, uint32_t desc)
 351 {
 352    if (devinfo->gen >= 7)
 353       return GET_BITS(desc, 16, 12);
 354    else if (devinfo->gen >= 5 || devinfo->is_g4x)
 355       return GET_BITS(desc, 15, 12);
 356    else
 357       return GET_BITS(desc, 15, 14);
 358 }
 359
 360 static inline unsigned
 361 brw_sampler_desc_simd_mode(const struct gen_device_info *devinfo, uint32_t desc)
 362 {
 363    assert(devinfo->gen >= 5);
 364    if (devinfo->gen >= 7)
 365       return GET_BITS(desc, 18, 17);
 366    else
 367       return GET_BITS(desc, 17, 16);
 368 }
 369
 370 static  inline unsigned
 371 brw_sampler_desc_return_format(const struct gen_device_info *devinfo,
 372                                uint32_t desc)
 373 {
 374    assert(devinfo->gen == 4 && !devinfo->is_g4x);
 375    return GET_BITS(desc, 13, 12);
 376 }
 377
 378 /**
 379  * Construct a message descriptor for the dataport
 380  */
 381 static inline uint32_t
 382 brw_dp_desc(const struct gen_device_info *devinfo,
 383             unsigned binding_table_index,
 384             unsigned msg_type,
 385             unsigned msg_control)
 386 {
 387    /* Prior to gen6, things are too inconsistent; use the dp_read/write_desc
 388     * helpers instead.
 389     */
 390    assert(devinfo->gen >= 6);
 391    const unsigned desc = SET_BITS(binding_table_index, 7, 0);
 392    if (devinfo->gen >= 8) {
 393       return (desc | SET_BITS(msg_control, 13, 8) |
 394               SET_BITS(msg_type, 18, 14));
 395    } else if (devinfo->gen >= 7) {
 396       return (desc | SET_BITS(msg_control, 13, 8) |
 397               SET_BITS(msg_type, 17, 14));
 398    } else {
 399       return (desc | SET_BITS(msg_control, 12, 8) |
 400               SET_BITS(msg_type, 16, 13));
 401    }
 402 }
 403
 404 static inline unsigned
 405 brw_dp_desc_binding_table_index(const struct gen_device_info *devinfo,
 406                                 uint32_t desc)
 407 {
 408    return GET_BITS(desc, 7, 0);
 409 }
 410
 411 static inline unsigned
 412 brw_dp_desc_msg_type(const struct gen_device_info *devinfo, uint32_t desc)
 413 {
 414    assert(devinfo->gen >= 6);
 415    if (devinfo->gen >= 8)
 416       return GET_BITS(desc, 18, 14);
 417    else if (devinfo->gen >= 7)
 418       return GET_BITS(desc, 17, 14);
 419    else
 420       return GET_BITS(desc, 16, 13);
 421 }
 422
 423 static inline unsigned
 424 brw_dp_desc_msg_control(const struct gen_device_info *devinfo, uint32_t desc)
 425 {
 426    assert(devinfo->gen >= 6);
 427    if (devinfo->gen >= 7)
 428       return GET_BITS(desc, 13, 8);
 429    else
 430       return GET_BITS(desc, 12, 8);
 431 }
 432
 433 /**
 434  * Construct a message descriptor immediate with the specified dataport read
 435  * function controls.
 436  */
 437 static inline uint32_t
 438 brw_dp_read_desc(const struct gen_device_info *devinfo,
 439                  unsigned binding_table_index,
 440                  unsigned msg_control,
 441                  unsigned msg_type,
 442                  unsigned target_cache)
 443 {
 444    if (devinfo->gen >= 6)
 445       return brw_dp_desc(devinfo, binding_table_index, msg_type, msg_control);
 446    else if (devinfo->gen >= 5 || devinfo->is_g4x)
 447       return (SET_BITS(binding_table_index, 7, 0) |
 448               SET_BITS(msg_control, 10, 8) |
 449               SET_BITS(msg_type, 13, 11) |
 450               SET_BITS(target_cache, 15, 14));
 451    else
 452       return (SET_BITS(binding_table_index, 7, 0) |
 453               SET_BITS(msg_control, 11, 8) |
 454               SET_BITS(msg_type, 13, 12) |
 455               SET_BITS(target_cache, 15, 14));
 456 }
 457
 458 static inline unsigned
 459 brw_dp_read_desc_msg_type(const struct gen_device_info *devinfo, uint32_t desc)
 460 {
 461    if (devinfo->gen >= 6)
 462       return brw_dp_desc_msg_type(devinfo, desc);
 463    else if (devinfo->gen >= 5 || devinfo->is_g4x)
 464       return GET_BITS(desc, 13, 11);
 465    else
 466       return GET_BITS(desc, 13, 12);
 467 }
 468
 469 static inline unsigned
 470 brw_dp_read_desc_msg_control(const struct gen_device_info *devinfo,
 471                              uint32_t desc)
 472 {
 473    if (devinfo->gen >= 6)
 474       return brw_dp_desc_msg_control(devinfo, desc);
 475    else if (devinfo->gen >= 5 || devinfo->is_g4x)
 476       return GET_BITS(desc, 10, 8);
 477    else
 478       return GET_BITS(desc, 11, 8);
 479 }
 480
 481 /**
 482  * Construct a message descriptor immediate with the specified dataport write
 483  * function controls.
 484  */
 485 static inline uint32_t
 486 brw_dp_write_desc(const struct gen_device_info *devinfo,
 487                   unsigned binding_table_index,
 488                   unsigned msg_control,
 489                   unsigned msg_type,
 490                   unsigned last_render_target,
 491                   unsigned send_commit_msg)
 492 {
 493    assert(devinfo->gen <= 6 || !send_commit_msg);
 494    if (devinfo->gen >= 6)
 495       return brw_dp_desc(devinfo, binding_table_index, msg_type, msg_control) |
 496              SET_BITS(last_render_target, 12, 12) |
 497              SET_BITS(send_commit_msg, 17, 17);
 498    else
 499       return (SET_BITS(binding_table_index, 7, 0) |
 500               SET_BITS(msg_control, 11, 8) |
 501               SET_BITS(last_render_target, 11, 11) |
 502               SET_BITS(msg_type, 14, 12) |
 503               SET_BITS(send_commit_msg, 15, 15));
 504 }
 505
 506 static inline unsigned
 507 brw_dp_write_desc_msg_type(const struct gen_device_info *devinfo,
 508                            uint32_t desc)
 509 {
 510    if (devinfo->gen >= 6)
 511       return brw_dp_desc_msg_type(devinfo, desc);
 512    else
 513       return GET_BITS(desc, 14, 12);
 514 }
 515
 516 static inline unsigned
 517 brw_dp_write_desc_msg_control(const struct gen_device_info *devinfo,
 518                               uint32_t desc)
 519 {
 520    if (devinfo->gen >= 6)
 521       return brw_dp_desc_msg_control(devinfo, desc);
 522    else
 523       return GET_BITS(desc, 11, 8);
 524 }
 525
 526 static inline bool
 527 brw_dp_write_desc_last_render_target(const struct gen_device_info *devinfo,
 528                                      uint32_t desc)
 529 {
 530    if (devinfo->gen >= 6)
 531       return GET_BITS(desc, 12, 12);
 532    else
 533       return GET_BITS(desc, 11, 11);
 534 }
 535
 536 static inline bool
 537 brw_dp_write_desc_write_commit(const struct gen_device_info *devinfo,
 538                                uint32_t desc)
 539 {
 540    assert(devinfo->gen <= 6);
 541    if (devinfo->gen >= 6)
 542       return GET_BITS(desc, 17, 17);
 543    else
 544       return GET_BITS(desc, 15, 15);
 545 }
 546
 547 /**
 548  * Construct a message descriptor immediate with the specified dataport
 549  * surface function controls.
 550  */
 551 static inline uint32_t
 552 brw_dp_surface_desc(const struct gen_device_info *devinfo,
 553                     unsigned msg_type,
 554                     unsigned msg_control)
 555 {
 556    assert(devinfo->gen >= 7);
 557    /* We'll OR in the binding table index later */
 558    return brw_dp_desc(devinfo, 0, msg_type, msg_control);
 559 }
 560
 561 static inline uint32_t
 562 brw_dp_untyped_atomic_desc(const struct gen_device_info *devinfo,
 563                            unsigned exec_size, /**< 0 for SIMD4x2 */
 564                            unsigned atomic_op,
 565                            bool response_expected)
 566 {
 567    assert(exec_size <= 8 || exec_size == 16);
 568
 569    unsigned msg_type;
 570    if (devinfo->gen >= 8 || devinfo->is_haswell) {
 571       if (exec_size > 0) {
 572          msg_type = HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP;
 573       } else {
 574          msg_type = HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP_SIMD4X2;
 575       }
 576    } else {
 577       msg_type = GEN7_DATAPORT_DC_UNTYPED_ATOMIC_OP;
 578    }
 579
 580    const unsigned msg_control =
 581       SET_BITS(atomic_op, 3, 0) |
 582       SET_BITS(0 < exec_size && exec_size <= 8, 4, 4) |
 583       SET_BITS(response_expected, 5, 5);
 584
 585    return brw_dp_surface_desc(devinfo, msg_type, msg_control);
 586 }
 587
 588 static inline uint32_t
 589 brw_dp_untyped_atomic_float_desc(const struct gen_device_info *devinfo,
 590                                  unsigned exec_size,
 591                                  unsigned atomic_op,
 592                                  bool response_expected)
 593 {
 594    assert(exec_size <= 8 || exec_size == 16);
 595    assert(devinfo->gen >= 9);
 596
 597    assert(exec_size > 0);
 598    const unsigned msg_type = GEN9_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_FLOAT_OP;
 599
 600    const unsigned msg_control =
 601       SET_BITS(atomic_op, 1, 0) |
 602       SET_BITS(exec_size <= 8, 4, 4) |
 603       SET_BITS(response_expected, 5, 5);
 604
 605    return brw_dp_surface_desc(devinfo, msg_type, msg_control);
 606 }
 607
 608 static inline unsigned
 609 brw_mdc_cmask(unsigned num_channels)
 610 {
 611    /* See also MDC_CMASK in the SKL PRM Vol 2d. */
 612    return 0xf & (0xf << num_channels);
 613 }
 614
 615 static inline uint32_t
 616 brw_dp_untyped_surface_rw_desc(const struct gen_device_info *devinfo,
 617                                unsigned exec_size, /**< 0 for SIMD4x2 */
 618                                unsigned num_channels,
 619                                bool write)
 620 {
 621    assert(exec_size <= 8 || exec_size == 16);
 622
 623    unsigned msg_type;
 624    if (write) {
 625       if (devinfo->gen >= 8 || devinfo->is_haswell) {
 626          msg_type = HSW_DATAPORT_DC_PORT1_UNTYPED_SURFACE_WRITE;
 627       } else {
 628          msg_type = GEN7_DATAPORT_DC_UNTYPED_SURFACE_WRITE;
 629       }
 630    } else {
 631       /* Read */
 632       if (devinfo->gen >= 8 || devinfo->is_haswell) {
 633          msg_type = HSW_DATAPORT_DC_PORT1_UNTYPED_SURFACE_READ;
 634       } else {
 635          msg_type = GEN7_DATAPORT_DC_UNTYPED_SURFACE_READ;
 636       }
 637    }
 638
 639    /* SIMD4x2 is only valid for read messages on IVB; use SIMD8 instead */
 640    if (write && devinfo->gen == 7 && !devinfo->is_haswell && exec_size == 0)
 641       exec_size = 8;
 642
 643    /* See also MDC_SM3 in the SKL PRM Vol 2d. */
 644    const unsigned simd_mode = exec_size == 0 ? 0 : /* SIMD4x2 */
 645                               exec_size <= 8 ? 2 : 1;
 646
 647    const unsigned msg_control =
 648       SET_BITS(brw_mdc_cmask(num_channels), 3, 0) |
 649       SET_BITS(simd_mode, 5, 4);
 650
 651    return brw_dp_surface_desc(devinfo, msg_type, msg_control);
 652 }
 653
 654 static inline unsigned
 655 brw_mdc_ds(unsigned bit_size)
 656 {
 657    switch (bit_size) {
 658    case 8:
 659       return GEN7_BYTE_SCATTERED_DATA_ELEMENT_BYTE;
 660    case 16:
 661       return GEN7_BYTE_SCATTERED_DATA_ELEMENT_WORD;
 662    case 32:
 663       return GEN7_BYTE_SCATTERED_DATA_ELEMENT_DWORD;
 664    default:
 665       unreachable("Unsupported bit_size for byte scattered messages");
 666    }
 667 }
 668
 669 static inline uint32_t
 670 brw_dp_byte_scattered_rw_desc(const struct gen_device_info *devinfo,
 671                               unsigned exec_size,
 672                               unsigned bit_size,
 673                               bool write)
 674 {
 675    assert(exec_size <= 8 || exec_size == 16);
 676
 677    assert(devinfo->gen > 7 || devinfo->is_haswell);
 678    const unsigned msg_type =
 679       write ? HSW_DATAPORT_DC_PORT0_BYTE_SCATTERED_WRITE :
 680               HSW_DATAPORT_DC_PORT0_BYTE_SCATTERED_READ;
 681
 682    assert(exec_size > 0);
 683    const unsigned msg_control =
 684       SET_BITS(exec_size == 16, 0, 0) |
 685       SET_BITS(brw_mdc_ds(bit_size), 3, 2);
 686
 687    return brw_dp_surface_desc(devinfo, msg_type, msg_control);
 688 }
 689
 690 static inline uint32_t
 691 brw_dp_a64_untyped_surface_rw_desc(const struct gen_device_info *devinfo,
 692                                    unsigned exec_size, /**< 0 for SIMD4x2 */
 693                                    unsigned num_channels,
 694                                    bool write)
 695 {
 696    assert(exec_size <= 8 || exec_size == 16);
 697    assert(devinfo->gen >= 8);
 698
 699    unsigned msg_type =
 700       write ? GEN8_DATAPORT_DC_PORT1_A64_UNTYPED_SURFACE_WRITE :
 701               GEN8_DATAPORT_DC_PORT1_A64_UNTYPED_SURFACE_READ;
 702
 703    /* See also MDC_SM3 in the SKL PRM Vol 2d. */
 704    const unsigned simd_mode = exec_size == 0 ? 0 : /* SIMD4x2 */
 705                               exec_size <= 8 ? 2 : 1;
 706
 707    const unsigned msg_control =
 708       SET_BITS(brw_mdc_cmask(num_channels), 3, 0) |
 709       SET_BITS(simd_mode, 5, 4);
 710
 711    return brw_dp_desc(devinfo, BRW_BTI_STATELESS, msg_type, msg_control);
 712 }
 713
 714 /**
 715  * Calculate the data size (see MDC_A64_DS in the "Structures" volume of the
 716  * Skylake PRM).
 717  */
 718 static inline uint32_t
 719 brw_mdc_a64_ds(unsigned elems)
 720 {
 721    switch (elems) {
 722    case 1:  return 0;
 723    case 2:  return 1;
 724    case 4:  return 2;
 725    case 8:  return 3;
 726    default:
 727       unreachable("Unsupported elmeent count for A64 scattered message");
 728    }
 729 }
 730
 731 static inline uint32_t
 732 brw_dp_a64_byte_scattered_rw_desc(const struct gen_device_info *devinfo,
 733                                   unsigned exec_size, /**< 0 for SIMD4x2 */
 734                                   unsigned bit_size,
 735                                   bool write)
 736 {
 737    assert(exec_size <= 8 || exec_size == 16);
 738    assert(devinfo->gen >= 8);
 739
 740    unsigned msg_type =
 741       write ? GEN8_DATAPORT_DC_PORT1_A64_SCATTERED_WRITE :
 742               GEN9_DATAPORT_DC_PORT1_A64_SCATTERED_READ;
 743
 744    const unsigned msg_control =
 745       SET_BITS(GEN8_A64_SCATTERED_SUBTYPE_BYTE, 1, 0) |
 746       SET_BITS(brw_mdc_a64_ds(bit_size / 8), 3, 2) |
 747       SET_BITS(exec_size == 16, 4, 4);
 748
 749    return brw_dp_desc(devinfo, BRW_BTI_STATELESS, msg_type, msg_control);
 750 }
 751
 752 static inline uint32_t
 753 brw_dp_typed_atomic_desc(const struct gen_device_info *devinfo,
 754                          unsigned exec_size,
 755                          unsigned exec_group,
 756                          unsigned atomic_op,
 757                          bool response_expected)
 758 {
 759    assert(exec_size > 0 || exec_group == 0);
 760    assert(exec_group % 8 == 0);
 761
 762    unsigned msg_type;
 763    if (devinfo->gen >= 8 || devinfo->is_haswell) {
 764       if (exec_size == 0) {
 765          msg_type = HSW_DATAPORT_DC_PORT1_TYPED_ATOMIC_OP_SIMD4X2;
 766       } else {
 767          msg_type = HSW_DATAPORT_DC_PORT1_TYPED_ATOMIC_OP;
 768       }
 769    } else {
 770       /* SIMD4x2 typed surface R/W messages only exist on HSW+ */
 771       assert(exec_size > 0);
 772       msg_type = GEN7_DATAPORT_RC_TYPED_ATOMIC_OP;
 773    }
 774
 775    const bool high_sample_mask = (exec_group / 8) % 2 == 1;
 776
 777    const unsigned msg_control =
 778       SET_BITS(atomic_op, 3, 0) |
 779       SET_BITS(high_sample_mask, 4, 4) |
 780       SET_BITS(response_expected, 5, 5);
 781
 782    return brw_dp_surface_desc(devinfo, msg_type, msg_control);
 783 }
 784
 785 static inline uint32_t
 786 brw_dp_typed_surface_rw_desc(const struct gen_device_info *devinfo,
 787                              unsigned exec_size,
 788                              unsigned exec_group,
 789                              unsigned num_channels,
 790                              bool write)
 791 {
 792    assert(exec_size > 0 || exec_group == 0);
 793    assert(exec_group % 8 == 0);
 794
 795    /* Typed surface reads and writes don't support SIMD16 */
 796    assert(exec_size <= 8);
 797
 798    unsigned msg_type;
 799    if (write) {
 800       if (devinfo->gen >= 8 || devinfo->is_haswell) {
 801          msg_type = HSW_DATAPORT_DC_PORT1_TYPED_SURFACE_WRITE;
 802       } else {
 803          msg_type = GEN7_DATAPORT_RC_TYPED_SURFACE_WRITE;
 804       }
 805    } else {
 806       if (devinfo->gen >= 8 || devinfo->is_haswell) {
 807          msg_type = HSW_DATAPORT_DC_PORT1_TYPED_SURFACE_READ;
 808       } else {
 809          msg_type = GEN7_DATAPORT_RC_TYPED_SURFACE_READ;
 810       }
 811    }
 812
 813    /* See also MDC_SG3 in the SKL PRM Vol 2d. */
 814    unsigned msg_control;
 815    if (devinfo->gen >= 8 || devinfo->is_haswell) {
 816       /* See also MDC_SG3 in the SKL PRM Vol 2d. */
 817       const unsigned slot_group = exec_size == 0 ? 0 : /* SIMD4x2 */
 818                                   1 + ((exec_group / 8) % 2);
 819
 820       msg_control =
 821          SET_BITS(brw_mdc_cmask(num_channels), 3, 0) |
 822          SET_BITS(slot_group, 5, 4);
 823    } else {
 824       /* SIMD4x2 typed surface R/W messages only exist on HSW+ */
 825       assert(exec_size > 0);
 826       const unsigned slot_group = ((exec_group / 8) % 2);
 827
 828       msg_control =
 829          SET_BITS(brw_mdc_cmask(num_channels), 3, 0) |
 830          SET_BITS(slot_group, 5, 5);
 831    }
 832
 833    return brw_dp_surface_desc(devinfo, msg_type, msg_control);
 834 }
 835
 836 /**
 837  * Construct a message descriptor immediate with the specified pixel
 838  * interpolator function controls.
 839  */
 840 static inline uint32_t
 841 brw_pixel_interp_desc(UNUSED const struct gen_device_info *devinfo,
 842                       unsigned msg_type,
 843                       bool noperspective,
 844                       unsigned simd_mode,
 845                       unsigned slot_group)
 846 {
 847    return (SET_BITS(slot_group, 11, 11) |
 848            SET_BITS(msg_type, 13, 12) |
 849            SET_BITS(!!noperspective, 14, 14) |
 850            SET_BITS(simd_mode, 16, 16));
 851 }
 852
 853 void brw_urb_WRITE(struct brw_codegen *p,
 854                    struct brw_reg dest,
 855                    unsigned msg_reg_nr,
 856                    struct brw_reg src0,
 857                    enum brw_urb_write_flags flags,
 858                    unsigned msg_length,
 859                    unsigned response_length,
 860                    unsigned offset,
 861                    unsigned swizzle);
 862
 863 /**
 864  * Send message to shared unit \p sfid with a possibly indirect descriptor \p
 865  * desc.  If \p desc is not an immediate it will be transparently loaded to an
 866  * address register using an OR instruction.
 867  */
 868 void
 869 brw_send_indirect_message(struct brw_codegen *p,
 870                           unsigned sfid,
 871                           struct brw_reg dst,
 872                           struct brw_reg payload,
 873                           struct brw_reg desc,
 874                           unsigned desc_imm);
 875
 876 void
 877 brw_send_indirect_split_message(struct brw_codegen *p,
 878                                 unsigned sfid,
 879                                 struct brw_reg dst,
 880                                 struct brw_reg payload0,
 881                                 struct brw_reg payload1,
 882                                 struct brw_reg desc,
 883                                 unsigned desc_imm,
 884                                 struct brw_reg ex_desc,
 885                                 unsigned ex_desc_imm);
 886
 887 void brw_ff_sync(struct brw_codegen *p,
 888                    struct brw_reg dest,
 889                    unsigned msg_reg_nr,
 890                    struct brw_reg src0,
 891                    bool allocate,
 892                    unsigned response_length,
 893                    bool eot);
 894
 895 void brw_svb_write(struct brw_codegen *p,
 896                    struct brw_reg dest,
 897                    unsigned msg_reg_nr,
 898                    struct brw_reg src0,
 899                    unsigned binding_table_index,
 900                    bool   send_commit_msg);
 901
 902 brw_inst *brw_fb_WRITE(struct brw_codegen *p,
 903                        struct brw_reg payload,
 904                        struct brw_reg implied_header,
 905                        unsigned msg_control,
 906                        unsigned binding_table_index,
 907                        unsigned msg_length,
 908                        unsigned response_length,
 909                        bool eot,
 910                        bool last_render_target,
 911                        bool header_present);
 912
 913 brw_inst *gen9_fb_READ(struct brw_codegen *p,
 914                        struct brw_reg dst,
 915                        struct brw_reg payload,
 916                        unsigned binding_table_index,
 917                        unsigned msg_length,
 918                        unsigned response_length,
 919                        bool per_sample);
 920
 921 void brw_SAMPLE(struct brw_codegen *p,
 922                 struct brw_reg dest,
 923                 unsigned msg_reg_nr,
 924                 struct brw_reg src0,
 925                 unsigned binding_table_index,
 926                 unsigned sampler,
 927                 unsigned msg_type,
 928                 unsigned response_length,
 929                 unsigned msg_length,
 930                 unsigned header_present,
 931                 unsigned simd_mode,
 932                 unsigned return_format);
 933
 934 void brw_adjust_sampler_state_pointer(struct brw_codegen *p,
 935                                       struct brw_reg header,
 936                                       struct brw_reg sampler_index);
 937
 938 void gen4_math(struct brw_codegen *p,
 939                struct brw_reg dest,
 940                unsigned function,
 941                unsigned msg_reg_nr,
 942                struct brw_reg src,
 943                unsigned precision );
 944
 945 void gen6_math(struct brw_codegen *p,
 946                struct brw_reg dest,
 947                unsigned function,
 948                struct brw_reg src0,
 949                struct brw_reg src1);
 950
 951 void brw_oword_block_read(struct brw_codegen *p,
 952                           struct brw_reg dest,
 953                           struct brw_reg mrf,
 954                           uint32_t offset,
 955                           uint32_t bind_table_index);
 956
 957 unsigned brw_scratch_surface_idx(const struct brw_codegen *p);
 958
 959 void brw_oword_block_read_scratch(struct brw_codegen *p,
 960                                   struct brw_reg dest,
 961                                   struct brw_reg mrf,
 962                                   int num_regs,
 963                                   unsigned offset);
 964
 965 void brw_oword_block_write_scratch(struct brw_codegen *p,
 966                                    struct brw_reg mrf,
 967                                    int num_regs,
 968                                    unsigned offset);
 969
 970 void gen7_block_read_scratch(struct brw_codegen *p,
 971                              struct brw_reg dest,
 972                              int num_regs,
 973                              unsigned offset);
 974
 975 void brw_shader_time_add(struct brw_codegen *p,
 976                          struct brw_reg payload,
 977                          uint32_t surf_index);
 978
 979 /**
 980  * Return the generation-specific jump distance scaling factor.
 981  *
 982  * Given the number of instructions to jump, we need to scale by
 983  * some number to obtain the actual jump distance to program in an
 984  * instruction.
 985  */
 986 static inline unsigned
 987 brw_jump_scale(const struct gen_device_info *devinfo)
 988 {
 989    /* Broadwell measures jump targets in bytes. */
 990    if (devinfo->gen >= 8)
 991       return 16;
 992
 993    /* Ironlake and later measure jump targets in 64-bit data chunks (in order
 994     * (to support compaction), so each 128-bit instruction requires 2 chunks.
 995     */
 996    if (devinfo->gen >= 5)
 997       return 2;
 998
 999    /* Gen4 simply uses the number of 128-bit instructions. */
1000    return 1;
1001 }
1002
1003 void brw_barrier(struct brw_codegen *p, struct brw_reg src);
1004
1005 /* If/else/endif.  Works by manipulating the execution flags on each
1006  * channel.
1007  */
1008 brw_inst *brw_IF(struct brw_codegen *p, unsigned execute_size);
1009 brw_inst *gen6_IF(struct brw_codegen *p, enum brw_conditional_mod conditional,
1010                   struct brw_reg src0, struct brw_reg src1);
1011
1012 void brw_ELSE(struct brw_codegen *p);
1013 void brw_ENDIF(struct brw_codegen *p);
1014
1015 /* DO/WHILE loops:
1016  */
1017 brw_inst *brw_DO(struct brw_codegen *p, unsigned execute_size);
1018
1019 brw_inst *brw_WHILE(struct brw_codegen *p);
1020
1021 brw_inst *brw_BREAK(struct brw_codegen *p);
1022 brw_inst *brw_CONT(struct brw_codegen *p);
1023 brw_inst *gen6_HALT(struct brw_codegen *p);
1024
1025 /* Forward jumps:
1026  */
1027 void brw_land_fwd_jump(struct brw_codegen *p, int jmp_insn_idx);
1028
1029 brw_inst *brw_JMPI(struct brw_codegen *p, struct brw_reg index,
1030                    unsigned predicate_control);
1031
1032 void brw_NOP(struct brw_codegen *p);
1033
1034 void brw_WAIT(struct brw_codegen *p);
1035
1036 /* Special case: there is never a destination, execution size will be
1037  * taken from src0:
1038  */
1039 void brw_CMP(struct brw_codegen *p,
1040              struct brw_reg dest,
1041              unsigned conditional,
1042              struct brw_reg src0,
1043              struct brw_reg src1);
1044
1045 void
1046 brw_untyped_atomic(struct brw_codegen *p,
1047                    struct brw_reg dst,
1048                    struct brw_reg payload,
1049                    struct brw_reg surface,
1050                    unsigned atomic_op,
1051                    unsigned msg_length,
1052                    bool response_expected,
1053                    bool header_present);
1054
1055 void
1056 brw_untyped_surface_read(struct brw_codegen *p,
1057                          struct brw_reg dst,
1058                          struct brw_reg payload,
1059                          struct brw_reg surface,
1060                          unsigned msg_length,
1061                          unsigned num_channels);
1062
1063 void
1064 brw_untyped_surface_write(struct brw_codegen *p,
1065                           struct brw_reg payload,
1066                           struct brw_reg surface,
1067                           unsigned msg_length,
1068                           unsigned num_channels,
1069                           bool header_present);
1070
1071 void
1072 brw_typed_atomic(struct brw_codegen *p,
1073                  struct brw_reg dst,
1074                  struct brw_reg payload,
1075                  struct brw_reg surface,
1076                  unsigned atomic_op,
1077                  unsigned msg_length,
1078                  bool response_expected,
1079                  bool header_present);
1080
1081 void
1082 brw_typed_surface_read(struct brw_codegen *p,
1083                        struct brw_reg dst,
1084                        struct brw_reg payload,
1085                        struct brw_reg surface,
1086                        unsigned msg_length,
1087                        unsigned num_channels,
1088                        bool header_present);
1089
1090 void
1091 brw_typed_surface_write(struct brw_codegen *p,
1092                         struct brw_reg payload,
1093                         struct brw_reg surface,
1094                         unsigned msg_length,
1095                         unsigned num_channels,
1096                         bool header_present);
1097
1098 void
1099 brw_memory_fence(struct brw_codegen *p,
1100                  struct brw_reg dst,
1101                  enum opcode send_op);
1102
1103 void
1104 brw_pixel_interpolator_query(struct brw_codegen *p,
1105                              struct brw_reg dest,
1106                              struct brw_reg mrf,
1107                              bool noperspective,
1108                              unsigned mode,
1109                              struct brw_reg data,
1110                              unsigned msg_length,
1111                              unsigned response_length);
1112
1113 void
1114 brw_find_live_channel(struct brw_codegen *p,
1115                       struct brw_reg dst,
1116                       struct brw_reg mask);
1117
1118 void
1119 brw_broadcast(struct brw_codegen *p,
1120               struct brw_reg dst,
1121               struct brw_reg src,
1122               struct brw_reg idx);
1123
1124 void
1125 brw_rounding_mode(struct brw_codegen *p,
1126                   enum brw_rnd_mode mode);
1127
1128 /***********************************************************************
1129  * brw_eu_util.c:
1130  */
1131
1132 void brw_copy_indirect_to_indirect(struct brw_codegen *p,
1133                                    struct brw_indirect dst_ptr,
1134                                    struct brw_indirect src_ptr,
1135                                    unsigned count);
1136
1137 void brw_copy_from_indirect(struct brw_codegen *p,
1138                             struct brw_reg dst,
1139                             struct brw_indirect ptr,
1140                             unsigned count);
1141
1142 void brw_copy4(struct brw_codegen *p,
1143                struct brw_reg dst,
1144                struct brw_reg src,
1145                unsigned count);
1146
1147 void brw_copy8(struct brw_codegen *p,
1148                struct brw_reg dst,
1149                struct brw_reg src,
1150                unsigned count);
1151
1152 void brw_math_invert( struct brw_codegen *p,
1153                       struct brw_reg dst,
1154                       struct brw_reg src);
1155
1156 void brw_set_src1(struct brw_codegen *p, brw_inst *insn, struct brw_reg reg);
1157
1158 void brw_set_desc_ex(struct brw_codegen *p, brw_inst *insn,
1159                      unsigned desc, unsigned ex_desc);
1160
1161 static inline void
1162 brw_set_desc(struct brw_codegen *p, brw_inst *insn, unsigned desc)
1163 {
1164    brw_set_desc_ex(p, insn, desc, 0);
1165 }
1166
1167 void brw_set_uip_jip(struct brw_codegen *p, int start_offset);
1168
1169 enum brw_conditional_mod brw_negate_cmod(uint32_t cmod);
1170 enum brw_conditional_mod brw_swap_cmod(uint32_t cmod);
1171
1172 /* brw_eu_compact.c */
1173 void brw_init_compaction_tables(const struct gen_device_info *devinfo);
1174 void brw_compact_instructions(struct brw_codegen *p, int start_offset,
1175                               struct disasm_info *disasm);
1176 void brw_uncompact_instruction(const struct gen_device_info *devinfo,
1177                                brw_inst *dst, brw_compact_inst *src);
1178 bool brw_try_compact_instruction(const struct gen_device_info *devinfo,
1179                                  brw_compact_inst *dst, const brw_inst *src);
1180
1181 void brw_debug_compact_uncompact(const struct gen_device_info *devinfo,
1182                                  brw_inst *orig, brw_inst *uncompacted);
1183
1184 /* brw_eu_validate.c */
1185 bool brw_validate_instructions(const struct gen_device_info *devinfo,
1186                                const void *assembly, int start_offset, int end_offset,
1187                                struct disasm_info *disasm);
1188
1189 static inline int
1190 next_offset(const struct gen_device_info *devinfo, void *store, int offset)
1191 {
1192    brw_inst *insn = (brw_inst *)((char *)store + offset);
1193
1194    if (brw_inst_cmpt_control(devinfo, insn))
1195       return offset + 8;
1196    else
1197       return offset + 16;
1198 }
1199
1200 struct opcode_desc {
1201    /* The union is an implementation detail used by brw_opcode_desc() to handle
1202     * opcodes that have been reused for different instructions across hardware
1203     * generations.
1204     *
1205     * The gens field acts as a tag. If it is non-zero, name points to a string
1206     * containing the instruction mnemonic. If it is zero, the table field is
1207     * valid and either points to a secondary opcode_desc table with 'size'
1208     * elements or is NULL and no such instruction exists for the opcode.
1209     */
1210    union {
1211       struct {
1212          char    *name;
1213          int      nsrc;
1214       };
1215       struct {
1216          const struct opcode_desc *table;
1217          unsigned size;
1218       };
1219    };
1220    int      ndst;
1221    int      gens;
1222 };
1223
1224 const struct opcode_desc *
1225 brw_opcode_desc(const struct gen_device_info *devinfo, enum opcode opcode);
1226
1227 static inline bool
1228 is_3src(const struct gen_device_info *devinfo, enum opcode opcode)
1229 {
1230    const struct opcode_desc *desc = brw_opcode_desc(devinfo, opcode);
1231    return desc && desc->nsrc == 3;
1232 }
1233
1234 /** Maximum SEND message length */
1235 #define BRW_MAX_MSG_LENGTH 15
1236
1237 /** First MRF register used by pull loads */
1238 #define FIRST_SPILL_MRF(gen) ((gen) == 6 ? 21 : 13)
1239
1240 /** First MRF register used by spills */
1241 #define FIRST_PULL_LOAD_MRF(gen) ((gen) == 6 ? 16 : 13)
1242
1243 #ifdef __cplusplus
1244 }
1245 #endif
1246
1247 #endif