src/intel/compiler/brw_eu.h

   1 /*
   2  Copyright (C) Intel Corp.  2006.  All Rights Reserved.
   3  Intel funded Tungsten Graphics to
   4  develop this 3D driver.
   5
   6  Permission is hereby granted, free of charge, to any person obtaining
   7  a copy of this software and associated documentation files (the
   8  "Software"), to deal in the Software without restriction, including
   9  without limitation the rights to use, copy, modify, merge, publish,
  10  distribute, sublicense, and/or sell copies of the Software, and to
  11  permit persons to whom the Software is furnished to do so, subject to
  12  the following conditions:
  13
  14  The above copyright notice and this permission notice (including the
  15  next paragraph) shall be included in all copies or substantial
  16  portions of the Software.
  17
  18  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  19  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  21  IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
  22  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  23  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  24  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25
  26  **********************************************************************/
  27  /*
  28   * Authors:
  29   *   Keith Whitwell <keithw@vmware.com>
  30   */
  31
  32
  33 #ifndef BRW_EU_H
  34 #define BRW_EU_H
  35
  36 #include <stdbool.h>
  37 #include <stdio.h>
  38 #include "brw_inst.h"
  39 #include "brw_eu_defines.h"
  40 #include "brw_reg.h"
  41 #include "brw_disasm_info.h"
  42
  43 #ifdef __cplusplus
  44 extern "C" {
  45 #endif
  46
  47 #define BRW_EU_MAX_INSN_STACK 5
  48
  49 struct brw_insn_state {
  50    /* One of BRW_EXECUTE_* */
  51    unsigned exec_size:3;
  52
  53    /* Group in units of channels */
  54    unsigned group:5;
  55
  56    /* Compression control on gen4-5 */
  57    bool compressed:1;
  58
  59    /* One of BRW_MASK_* */
  60    unsigned mask_control:1;
  61
  62    bool saturate:1;
  63
  64    /* One of BRW_ALIGN_* */
  65    unsigned access_mode:1;
  66
  67    /* One of BRW_PREDICATE_* */
  68    enum brw_predicate predicate:4;
  69
  70    bool pred_inv:1;
  71
  72    /* Flag subreg.  Bottom bit is subreg, top bit is reg */
  73    unsigned flag_subreg:2;
  74
  75    bool acc_wr_control:1;
  76 };
  77
  78
  79 /* A helper for accessing the last instruction emitted.  This makes it easy
  80  * to set various bits on an instruction without having to create temporary
  81  * variable and assign the emitted instruction to those.
  82  */
  83 #define brw_last_inst (&p->store[p->nr_insn - 1])
  84
  85 struct brw_codegen {
  86    brw_inst *store;
  87    int store_size;
  88    unsigned nr_insn;
  89    unsigned int next_insn_offset;
  90
  91    void *mem_ctx;
  92
  93    /* Allow clients to push/pop instruction state:
  94     */
  95    struct brw_insn_state stack[BRW_EU_MAX_INSN_STACK];
  96    struct brw_insn_state *current;
  97
  98    /** Whether or not the user wants automatic exec sizes
  99     *
 100     * If true, codegen will try to automatically infer the exec size of an
 101     * instruction from the width of the destination register.  If false, it
 102     * will take whatever is set by brw_set_default_exec_size verbatim.
 103     *
 104     * This is set to true by default in brw_init_codegen.
 105     */
 106    bool automatic_exec_sizes;
 107
 108    bool single_program_flow;
 109    const struct gen_device_info *devinfo;
 110
 111    /* Control flow stacks:
 112     * - if_stack contains IF and ELSE instructions which must be patched
 113     *   (and popped) once the matching ENDIF instruction is encountered.
 114     *
 115     *   Just store the instruction pointer(an index).
 116     */
 117    int *if_stack;
 118    int if_stack_depth;
 119    int if_stack_array_size;
 120
 121    /**
 122     * loop_stack contains the instruction pointers of the starts of loops which
 123     * must be patched (and popped) once the matching WHILE instruction is
 124     * encountered.
 125     */
 126    int *loop_stack;
 127    /**
 128     * pre-gen6, the BREAK and CONT instructions had to tell how many IF/ENDIF
 129     * blocks they were popping out of, to fix up the mask stack.  This tracks
 130     * the IF/ENDIF nesting in each current nested loop level.
 131     */
 132    int *if_depth_in_loop;
 133    int loop_stack_depth;
 134    int loop_stack_array_size;
 135 };
 136
 137 void brw_pop_insn_state( struct brw_codegen *p );
 138 void brw_push_insn_state( struct brw_codegen *p );
 139 unsigned brw_get_default_exec_size(struct brw_codegen *p);
 140 unsigned brw_get_default_group(struct brw_codegen *p);
 141 unsigned brw_get_default_access_mode(struct brw_codegen *p);
 142 void brw_set_default_exec_size(struct brw_codegen *p, unsigned value);
 143 void brw_set_default_mask_control( struct brw_codegen *p, unsigned value );
 144 void brw_set_default_saturate( struct brw_codegen *p, bool enable );
 145 void brw_set_default_access_mode( struct brw_codegen *p, unsigned access_mode );
 146 void brw_inst_set_compression(const struct gen_device_info *devinfo,
 147                               brw_inst *inst, bool on);
 148 void brw_set_default_compression(struct brw_codegen *p, bool on);
 149 void brw_inst_set_group(const struct gen_device_info *devinfo,
 150                         brw_inst *inst, unsigned group);
 151 void brw_set_default_group(struct brw_codegen *p, unsigned group);
 152 void brw_set_default_compression_control(struct brw_codegen *p, enum brw_compression c);
 153 void brw_set_default_predicate_control( struct brw_codegen *p, unsigned pc );
 154 void brw_set_default_predicate_inverse(struct brw_codegen *p, bool predicate_inverse);
 155 void brw_set_default_flag_reg(struct brw_codegen *p, int reg, int subreg);
 156 void brw_set_default_acc_write_control(struct brw_codegen *p, unsigned value);
 157
 158 void brw_init_codegen(const struct gen_device_info *, struct brw_codegen *p,
 159                       void *mem_ctx);
 160 int brw_disassemble_inst(FILE *file, const struct gen_device_info *devinfo,
 161                          const struct brw_inst *inst, bool is_compacted);
 162 void brw_disassemble(const struct gen_device_info *devinfo,
 163                      const void *assembly, int start, int end, FILE *out);
 164 const unsigned *brw_get_program( struct brw_codegen *p, unsigned *sz );
 165
 166 brw_inst *brw_next_insn(struct brw_codegen *p, unsigned opcode);
 167 void brw_set_dest(struct brw_codegen *p, brw_inst *insn, struct brw_reg dest);
 168 void brw_set_src0(struct brw_codegen *p, brw_inst *insn, struct brw_reg reg);
 169
 170 void gen6_resolve_implied_move(struct brw_codegen *p,
 171                                struct brw_reg *src,
 172                                unsigned msg_reg_nr);
 173
 174 /* Helpers for regular instructions:
 175  */
 176 #define ALU1(OP)                                \
 177 brw_inst *brw_##OP(struct brw_codegen *p,       \
 178               struct brw_reg dest,              \
 179               struct brw_reg src0);
 180
 181 #define ALU2(OP)                                \
 182 brw_inst *brw_##OP(struct brw_codegen *p,       \
 183               struct brw_reg dest,              \
 184               struct brw_reg src0,              \
 185               struct brw_reg src1);
 186
 187 #define ALU3(OP)                                \
 188 brw_inst *brw_##OP(struct brw_codegen *p,       \
 189               struct brw_reg dest,              \
 190               struct brw_reg src0,              \
 191               struct brw_reg src1,              \
 192               struct brw_reg src2);
 193
 194 #define ROUND(OP) \
 195 void brw_##OP(struct brw_codegen *p, struct brw_reg dest, struct brw_reg src0);
 196
 197 ALU1(MOV)
 198 ALU2(SEL)
 199 ALU1(NOT)
 200 ALU2(AND)
 201 ALU2(OR)
 202 ALU2(XOR)
 203 ALU2(SHR)
 204 ALU2(SHL)
 205 ALU1(DIM)
 206 ALU2(ASR)
 207 ALU2(ROL)
 208 ALU2(ROR)
 209 ALU3(CSEL)
 210 ALU1(F32TO16)
 211 ALU1(F16TO32)
 212 ALU2(ADD)
 213 ALU2(AVG)
 214 ALU2(MUL)
 215 ALU1(FRC)
 216 ALU1(RNDD)
 217 ALU2(MAC)
 218 ALU2(MACH)
 219 ALU1(LZD)
 220 ALU2(DP4)
 221 ALU2(DPH)
 222 ALU2(DP3)
 223 ALU2(DP2)
 224 ALU2(LINE)
 225 ALU2(PLN)
 226 ALU3(MAD)
 227 ALU3(LRP)
 228 ALU1(BFREV)
 229 ALU3(BFE)
 230 ALU2(BFI1)
 231 ALU3(BFI2)
 232 ALU1(FBH)
 233 ALU1(FBL)
 234 ALU1(CBIT)
 235 ALU2(ADDC)
 236 ALU2(SUBB)
 237 ALU2(MAC)
 238
 239 ROUND(RNDZ)
 240 ROUND(RNDE)
 241
 242 #undef ALU1
 243 #undef ALU2
 244 #undef ALU3
 245 #undef ROUND
 246
 247
 248 /* Helpers for SEND instruction:
 249  */
 250
 251 /**
 252  * Construct a message descriptor immediate with the specified common
 253  * descriptor controls.
 254  */
 255 static inline uint32_t
 256 brw_message_desc(const struct gen_device_info *devinfo,
 257                  unsigned msg_length,
 258                  unsigned response_length,
 259                  bool header_present)
 260 {
 261    if (devinfo->gen >= 5) {
 262       return (SET_BITS(msg_length, 28, 25) |
 263               SET_BITS(response_length, 24, 20) |
 264               SET_BITS(header_present, 19, 19));
 265    } else {
 266       return (SET_BITS(msg_length, 23, 20) |
 267               SET_BITS(response_length, 19, 16));
 268    }
 269 }
 270
 271 static inline unsigned
 272 brw_message_desc_mlen(const struct gen_device_info *devinfo, uint32_t desc)
 273 {
 274    if (devinfo->gen >= 5)
 275       return GET_BITS(desc, 28, 25);
 276    else
 277       return GET_BITS(desc, 23, 20);
 278 }
 279
 280 static inline unsigned
 281 brw_message_desc_rlen(const struct gen_device_info *devinfo, uint32_t desc)
 282 {
 283    if (devinfo->gen >= 5)
 284       return GET_BITS(desc, 24, 20);
 285    else
 286       return GET_BITS(desc, 19, 16);
 287 }
 288
 289 static inline bool
 290 brw_message_desc_header_present(MAYBE_UNUSED const struct gen_device_info *devinfo,
 291                                 uint32_t desc)
 292 {
 293    assert(devinfo->gen >= 5);
 294    return GET_BITS(desc, 19, 19);
 295 }
 296
 297 static inline unsigned
 298 brw_message_ex_desc(UNUSED const struct gen_device_info *devinfo,
 299                     unsigned ex_msg_length)
 300 {
 301    return SET_BITS(ex_msg_length, 9, 6);
 302 }
 303
 304 static inline unsigned
 305 brw_message_ex_desc_ex_mlen(UNUSED const struct gen_device_info *devinfo,
 306                             uint32_t ex_desc)
 307 {
 308    return GET_BITS(ex_desc, 9, 6);
 309 }
 310
 311 /**
 312  * Construct a message descriptor immediate with the specified sampler
 313  * function controls.
 314  */
 315 static inline uint32_t
 316 brw_sampler_desc(const struct gen_device_info *devinfo,
 317                  unsigned binding_table_index,
 318                  unsigned sampler,
 319                  unsigned msg_type,
 320                  unsigned simd_mode,
 321                  unsigned return_format)
 322 {
 323    const unsigned desc = (SET_BITS(binding_table_index, 7, 0) |
 324                           SET_BITS(sampler, 11, 8));
 325    if (devinfo->gen >= 7)
 326       return (desc | SET_BITS(msg_type, 16, 12) |
 327               SET_BITS(simd_mode, 18, 17));
 328    else if (devinfo->gen >= 5)
 329       return (desc | SET_BITS(msg_type, 15, 12) |
 330               SET_BITS(simd_mode, 17, 16));
 331    else if (devinfo->is_g4x)
 332       return desc | SET_BITS(msg_type, 15, 12);
 333    else
 334       return (desc | SET_BITS(return_format, 13, 12) |
 335               SET_BITS(msg_type, 15, 14));
 336 }
 337
 338 static inline unsigned
 339 brw_sampler_desc_binding_table_index(UNUSED const struct gen_device_info *devinfo,
 340                                      uint32_t desc)
 341 {
 342    return GET_BITS(desc, 7, 0);
 343 }
 344
 345 static inline unsigned
 346 brw_sampler_desc_sampler(UNUSED const struct gen_device_info *devinfo, uint32_t desc)
 347 {
 348    return GET_BITS(desc, 11, 8);
 349 }
 350
 351 static inline unsigned
 352 brw_sampler_desc_msg_type(const struct gen_device_info *devinfo, uint32_t desc)
 353 {
 354    if (devinfo->gen >= 7)
 355       return GET_BITS(desc, 16, 12);
 356    else if (devinfo->gen >= 5 || devinfo->is_g4x)
 357       return GET_BITS(desc, 15, 12);
 358    else
 359       return GET_BITS(desc, 15, 14);
 360 }
 361
 362 static inline unsigned
 363 brw_sampler_desc_simd_mode(const struct gen_device_info *devinfo, uint32_t desc)
 364 {
 365    assert(devinfo->gen >= 5);
 366    if (devinfo->gen >= 7)
 367       return GET_BITS(desc, 18, 17);
 368    else
 369       return GET_BITS(desc, 17, 16);
 370 }
 371
 372 static  inline unsigned
 373 brw_sampler_desc_return_format(MAYBE_UNUSED const struct gen_device_info *devinfo,
 374                                uint32_t desc)
 375 {
 376    assert(devinfo->gen == 4 && !devinfo->is_g4x);
 377    return GET_BITS(desc, 13, 12);
 378 }
 379
 380 /**
 381  * Construct a message descriptor for the dataport
 382  */
 383 static inline uint32_t
 384 brw_dp_desc(const struct gen_device_info *devinfo,
 385             unsigned binding_table_index,
 386             unsigned msg_type,
 387             unsigned msg_control)
 388 {
 389    /* Prior to gen6, things are too inconsistent; use the dp_read/write_desc
 390     * helpers instead.
 391     */
 392    assert(devinfo->gen >= 6);
 393    const unsigned desc = SET_BITS(binding_table_index, 7, 0);
 394    if (devinfo->gen >= 8) {
 395       return (desc | SET_BITS(msg_control, 13, 8) |
 396               SET_BITS(msg_type, 18, 14));
 397    } else if (devinfo->gen >= 7) {
 398       return (desc | SET_BITS(msg_control, 13, 8) |
 399               SET_BITS(msg_type, 17, 14));
 400    } else {
 401       return (desc | SET_BITS(msg_control, 12, 8) |
 402               SET_BITS(msg_type, 16, 13));
 403    }
 404 }
 405
 406 static inline unsigned
 407 brw_dp_desc_binding_table_index(UNUSED const struct gen_device_info *devinfo,
 408                                 uint32_t desc)
 409 {
 410    return GET_BITS(desc, 7, 0);
 411 }
 412
 413 static inline unsigned
 414 brw_dp_desc_msg_type(const struct gen_device_info *devinfo, uint32_t desc)
 415 {
 416    assert(devinfo->gen >= 6);
 417    if (devinfo->gen >= 8)
 418       return GET_BITS(desc, 18, 14);
 419    else if (devinfo->gen >= 7)
 420       return GET_BITS(desc, 17, 14);
 421    else
 422       return GET_BITS(desc, 16, 13);
 423 }
 424
 425 static inline unsigned
 426 brw_dp_desc_msg_control(const struct gen_device_info *devinfo, uint32_t desc)
 427 {
 428    assert(devinfo->gen >= 6);
 429    if (devinfo->gen >= 7)
 430       return GET_BITS(desc, 13, 8);
 431    else
 432       return GET_BITS(desc, 12, 8);
 433 }
 434
 435 /**
 436  * Construct a message descriptor immediate with the specified dataport read
 437  * function controls.
 438  */
 439 static inline uint32_t
 440 brw_dp_read_desc(const struct gen_device_info *devinfo,
 441                  unsigned binding_table_index,
 442                  unsigned msg_control,
 443                  unsigned msg_type,
 444                  unsigned target_cache)
 445 {
 446    if (devinfo->gen >= 6)
 447       return brw_dp_desc(devinfo, binding_table_index, msg_type, msg_control);
 448    else if (devinfo->gen >= 5 || devinfo->is_g4x)
 449       return (SET_BITS(binding_table_index, 7, 0) |
 450               SET_BITS(msg_control, 10, 8) |
 451               SET_BITS(msg_type, 13, 11) |
 452               SET_BITS(target_cache, 15, 14));
 453    else
 454       return (SET_BITS(binding_table_index, 7, 0) |
 455               SET_BITS(msg_control, 11, 8) |
 456               SET_BITS(msg_type, 13, 12) |
 457               SET_BITS(target_cache, 15, 14));
 458 }
 459
 460 static inline unsigned
 461 brw_dp_read_desc_msg_type(const struct gen_device_info *devinfo, uint32_t desc)
 462 {
 463    if (devinfo->gen >= 6)
 464       return brw_dp_desc_msg_type(devinfo, desc);
 465    else if (devinfo->gen >= 5 || devinfo->is_g4x)
 466       return GET_BITS(desc, 13, 11);
 467    else
 468       return GET_BITS(desc, 13, 12);
 469 }
 470
 471 static inline unsigned
 472 brw_dp_read_desc_msg_control(const struct gen_device_info *devinfo,
 473                              uint32_t desc)
 474 {
 475    if (devinfo->gen >= 6)
 476       return brw_dp_desc_msg_control(devinfo, desc);
 477    else if (devinfo->gen >= 5 || devinfo->is_g4x)
 478       return GET_BITS(desc, 10, 8);
 479    else
 480       return GET_BITS(desc, 11, 8);
 481 }
 482
 483 /**
 484  * Construct a message descriptor immediate with the specified dataport write
 485  * function controls.
 486  */
 487 static inline uint32_t
 488 brw_dp_write_desc(const struct gen_device_info *devinfo,
 489                   unsigned binding_table_index,
 490                   unsigned msg_control,
 491                   unsigned msg_type,
 492                   unsigned last_render_target,
 493                   unsigned send_commit_msg)
 494 {
 495    assert(devinfo->gen <= 6 || !send_commit_msg);
 496    if (devinfo->gen >= 6)
 497       return brw_dp_desc(devinfo, binding_table_index, msg_type, msg_control) |
 498              SET_BITS(last_render_target, 12, 12) |
 499              SET_BITS(send_commit_msg, 17, 17);
 500    else
 501       return (SET_BITS(binding_table_index, 7, 0) |
 502               SET_BITS(msg_control, 11, 8) |
 503               SET_BITS(last_render_target, 11, 11) |
 504               SET_BITS(msg_type, 14, 12) |
 505               SET_BITS(send_commit_msg, 15, 15));
 506 }
 507
 508 static inline unsigned
 509 brw_dp_write_desc_msg_type(const struct gen_device_info *devinfo,
 510                            uint32_t desc)
 511 {
 512    if (devinfo->gen >= 6)
 513       return brw_dp_desc_msg_type(devinfo, desc);
 514    else
 515       return GET_BITS(desc, 14, 12);
 516 }
 517
 518 static inline unsigned
 519 brw_dp_write_desc_msg_control(const struct gen_device_info *devinfo,
 520                               uint32_t desc)
 521 {
 522    if (devinfo->gen >= 6)
 523       return brw_dp_desc_msg_control(devinfo, desc);
 524    else
 525       return GET_BITS(desc, 11, 8);
 526 }
 527
 528 static inline bool
 529 brw_dp_write_desc_last_render_target(const struct gen_device_info *devinfo,
 530                                      uint32_t desc)
 531 {
 532    if (devinfo->gen >= 6)
 533       return GET_BITS(desc, 12, 12);
 534    else
 535       return GET_BITS(desc, 11, 11);
 536 }
 537
 538 static inline bool
 539 brw_dp_write_desc_write_commit(const struct gen_device_info *devinfo,
 540                                uint32_t desc)
 541 {
 542    assert(devinfo->gen <= 6);
 543    if (devinfo->gen >= 6)
 544       return GET_BITS(desc, 17, 17);
 545    else
 546       return GET_BITS(desc, 15, 15);
 547 }
 548
 549 /**
 550  * Construct a message descriptor immediate with the specified dataport
 551  * surface function controls.
 552  */
 553 static inline uint32_t
 554 brw_dp_surface_desc(const struct gen_device_info *devinfo,
 555                     unsigned msg_type,
 556                     unsigned msg_control)
 557 {
 558    assert(devinfo->gen >= 7);
 559    /* We'll OR in the binding table index later */
 560    return brw_dp_desc(devinfo, 0, msg_type, msg_control);
 561 }
 562
 563 static inline uint32_t
 564 brw_dp_untyped_atomic_desc(const struct gen_device_info *devinfo,
 565                            unsigned exec_size, /**< 0 for SIMD4x2 */
 566                            unsigned atomic_op,
 567                            bool response_expected)
 568 {
 569    assert(exec_size <= 8 || exec_size == 16);
 570
 571    unsigned msg_type;
 572    if (devinfo->gen >= 8 || devinfo->is_haswell) {
 573       if (exec_size > 0) {
 574          msg_type = HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP;
 575       } else {
 576          msg_type = HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP_SIMD4X2;
 577       }
 578    } else {
 579       msg_type = GEN7_DATAPORT_DC_UNTYPED_ATOMIC_OP;
 580    }
 581
 582    const unsigned msg_control =
 583       SET_BITS(atomic_op, 3, 0) |
 584       SET_BITS(0 < exec_size && exec_size <= 8, 4, 4) |
 585       SET_BITS(response_expected, 5, 5);
 586
 587    return brw_dp_surface_desc(devinfo, msg_type, msg_control);
 588 }
 589
 590 static inline uint32_t
 591 brw_dp_untyped_atomic_float_desc(const struct gen_device_info *devinfo,
 592                                  unsigned exec_size,
 593                                  unsigned atomic_op,
 594                                  bool response_expected)
 595 {
 596    assert(exec_size <= 8 || exec_size == 16);
 597    assert(devinfo->gen >= 9);
 598
 599    assert(exec_size > 0);
 600    const unsigned msg_type = GEN9_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_FLOAT_OP;
 601
 602    const unsigned msg_control =
 603       SET_BITS(atomic_op, 1, 0) |
 604       SET_BITS(exec_size <= 8, 4, 4) |
 605       SET_BITS(response_expected, 5, 5);
 606
 607    return brw_dp_surface_desc(devinfo, msg_type, msg_control);
 608 }
 609
 610 static inline unsigned
 611 brw_mdc_cmask(unsigned num_channels)
 612 {
 613    /* See also MDC_CMASK in the SKL PRM Vol 2d. */
 614    return 0xf & (0xf << num_channels);
 615 }
 616
 617 static inline uint32_t
 618 brw_dp_untyped_surface_rw_desc(const struct gen_device_info *devinfo,
 619                                unsigned exec_size, /**< 0 for SIMD4x2 */
 620                                unsigned num_channels,
 621                                bool write)
 622 {
 623    assert(exec_size <= 8 || exec_size == 16);
 624
 625    unsigned msg_type;
 626    if (write) {
 627       if (devinfo->gen >= 8 || devinfo->is_haswell) {
 628          msg_type = HSW_DATAPORT_DC_PORT1_UNTYPED_SURFACE_WRITE;
 629       } else {
 630          msg_type = GEN7_DATAPORT_DC_UNTYPED_SURFACE_WRITE;
 631       }
 632    } else {
 633       /* Read */
 634       if (devinfo->gen >= 8 || devinfo->is_haswell) {
 635          msg_type = HSW_DATAPORT_DC_PORT1_UNTYPED_SURFACE_READ;
 636       } else {
 637          msg_type = GEN7_DATAPORT_DC_UNTYPED_SURFACE_READ;
 638       }
 639    }
 640
 641    /* SIMD4x2 is only valid for read messages on IVB; use SIMD8 instead */
 642    if (write && devinfo->gen == 7 && !devinfo->is_haswell && exec_size == 0)
 643       exec_size = 8;
 644
 645    /* See also MDC_SM3 in the SKL PRM Vol 2d. */
 646    const unsigned simd_mode = exec_size == 0 ? 0 : /* SIMD4x2 */
 647                               exec_size <= 8 ? 2 : 1;
 648
 649    const unsigned msg_control =
 650       SET_BITS(brw_mdc_cmask(num_channels), 3, 0) |
 651       SET_BITS(simd_mode, 5, 4);
 652
 653    return brw_dp_surface_desc(devinfo, msg_type, msg_control);
 654 }
 655
 656 static inline unsigned
 657 brw_mdc_ds(unsigned bit_size)
 658 {
 659    switch (bit_size) {
 660    case 8:
 661       return GEN7_BYTE_SCATTERED_DATA_ELEMENT_BYTE;
 662    case 16:
 663       return GEN7_BYTE_SCATTERED_DATA_ELEMENT_WORD;
 664    case 32:
 665       return GEN7_BYTE_SCATTERED_DATA_ELEMENT_DWORD;
 666    default:
 667       unreachable("Unsupported bit_size for byte scattered messages");
 668    }
 669 }
 670
 671 static inline uint32_t
 672 brw_dp_byte_scattered_rw_desc(const struct gen_device_info *devinfo,
 673                               unsigned exec_size,
 674                               unsigned bit_size,
 675                               bool write)
 676 {
 677    assert(exec_size <= 8 || exec_size == 16);
 678
 679    assert(devinfo->gen > 7 || devinfo->is_haswell);
 680    const unsigned msg_type =
 681       write ? HSW_DATAPORT_DC_PORT0_BYTE_SCATTERED_WRITE :
 682               HSW_DATAPORT_DC_PORT0_BYTE_SCATTERED_READ;
 683
 684    assert(exec_size > 0);
 685    const unsigned msg_control =
 686       SET_BITS(exec_size == 16, 0, 0) |
 687       SET_BITS(brw_mdc_ds(bit_size), 3, 2);
 688
 689    return brw_dp_surface_desc(devinfo, msg_type, msg_control);
 690 }
 691
 692 static inline uint32_t
 693 brw_dp_a64_untyped_surface_rw_desc(const struct gen_device_info *devinfo,
 694                                    unsigned exec_size, /**< 0 for SIMD4x2 */
 695                                    unsigned num_channels,
 696                                    bool write)
 697 {
 698    assert(exec_size <= 8 || exec_size == 16);
 699    assert(devinfo->gen >= 8);
 700
 701    unsigned msg_type =
 702       write ? GEN8_DATAPORT_DC_PORT1_A64_UNTYPED_SURFACE_WRITE :
 703               GEN8_DATAPORT_DC_PORT1_A64_UNTYPED_SURFACE_READ;
 704
 705    /* See also MDC_SM3 in the SKL PRM Vol 2d. */
 706    const unsigned simd_mode = exec_size == 0 ? 0 : /* SIMD4x2 */
 707                               exec_size <= 8 ? 2 : 1;
 708
 709    const unsigned msg_control =
 710       SET_BITS(brw_mdc_cmask(num_channels), 3, 0) |
 711       SET_BITS(simd_mode, 5, 4);
 712
 713    return brw_dp_desc(devinfo, BRW_BTI_STATELESS, msg_type, msg_control);
 714 }
 715
 716 /**
 717  * Calculate the data size (see MDC_A64_DS in the "Structures" volume of the
 718  * Skylake PRM).
 719  */
 720 static inline uint32_t
 721 brw_mdc_a64_ds(unsigned elems)
 722 {
 723    switch (elems) {
 724    case 1:  return 0;
 725    case 2:  return 1;
 726    case 4:  return 2;
 727    case 8:  return 3;
 728    default:
 729       unreachable("Unsupported elmeent count for A64 scattered message");
 730    }
 731 }
 732
 733 static inline uint32_t
 734 brw_dp_a64_byte_scattered_rw_desc(const struct gen_device_info *devinfo,
 735                                   unsigned exec_size, /**< 0 for SIMD4x2 */
 736                                   unsigned bit_size,
 737                                   bool write)
 738 {
 739    assert(exec_size <= 8 || exec_size == 16);
 740    assert(devinfo->gen >= 8);
 741
 742    unsigned msg_type =
 743       write ? GEN8_DATAPORT_DC_PORT1_A64_SCATTERED_WRITE :
 744               GEN9_DATAPORT_DC_PORT1_A64_SCATTERED_READ;
 745
 746    const unsigned msg_control =
 747       SET_BITS(GEN8_A64_SCATTERED_SUBTYPE_BYTE, 1, 0) |
 748       SET_BITS(brw_mdc_a64_ds(bit_size / 8), 3, 2) |
 749       SET_BITS(exec_size == 16, 4, 4);
 750
 751    return brw_dp_desc(devinfo, BRW_BTI_STATELESS, msg_type, msg_control);
 752 }
 753
 754 static inline uint32_t
 755 brw_dp_a64_untyped_atomic_desc(const struct gen_device_info *devinfo,
 756                                MAYBE_UNUSED unsigned exec_size, /**< 0 for SIMD4x2 */
 757                                unsigned bit_size,
 758                                unsigned atomic_op,
 759                                bool response_expected)
 760 {
 761    assert(exec_size == 8);
 762    assert(devinfo->gen >= 8);
 763    assert(bit_size == 32 || bit_size == 64);
 764
 765    const unsigned msg_type = GEN8_DATAPORT_DC_PORT1_A64_UNTYPED_ATOMIC_OP;
 766
 767    const unsigned msg_control =
 768       SET_BITS(atomic_op, 3, 0) |
 769       SET_BITS(bit_size == 64, 4, 4) |
 770       SET_BITS(response_expected, 5, 5);
 771
 772    return brw_dp_desc(devinfo, BRW_BTI_STATELESS, msg_type, msg_control);
 773 }
 774
 775 static inline uint32_t
 776 brw_dp_a64_untyped_atomic_float_desc(const struct gen_device_info *devinfo,
 777                                      MAYBE_UNUSED unsigned exec_size,
 778                                      unsigned atomic_op,
 779                                      bool response_expected)
 780 {
 781    assert(exec_size == 8);
 782    assert(devinfo->gen >= 9);
 783
 784    assert(exec_size > 0);
 785    const unsigned msg_type = GEN9_DATAPORT_DC_PORT1_A64_UNTYPED_ATOMIC_FLOAT_OP;
 786
 787    const unsigned msg_control =
 788       SET_BITS(atomic_op, 1, 0) |
 789       SET_BITS(response_expected, 5, 5);
 790
 791    return brw_dp_desc(devinfo, BRW_BTI_STATELESS, msg_type, msg_control);
 792 }
 793
 794 static inline uint32_t
 795 brw_dp_typed_atomic_desc(const struct gen_device_info *devinfo,
 796                          unsigned exec_size,
 797                          unsigned exec_group,
 798                          unsigned atomic_op,
 799                          bool response_expected)
 800 {
 801    assert(exec_size > 0 || exec_group == 0);
 802    assert(exec_group % 8 == 0);
 803
 804    unsigned msg_type;
 805    if (devinfo->gen >= 8 || devinfo->is_haswell) {
 806       if (exec_size == 0) {
 807          msg_type = HSW_DATAPORT_DC_PORT1_TYPED_ATOMIC_OP_SIMD4X2;
 808       } else {
 809          msg_type = HSW_DATAPORT_DC_PORT1_TYPED_ATOMIC_OP;
 810       }
 811    } else {
 812       /* SIMD4x2 typed surface R/W messages only exist on HSW+ */
 813       assert(exec_size > 0);
 814       msg_type = GEN7_DATAPORT_RC_TYPED_ATOMIC_OP;
 815    }
 816
 817    const bool high_sample_mask = (exec_group / 8) % 2 == 1;
 818
 819    const unsigned msg_control =
 820       SET_BITS(atomic_op, 3, 0) |
 821       SET_BITS(high_sample_mask, 4, 4) |
 822       SET_BITS(response_expected, 5, 5);
 823
 824    return brw_dp_surface_desc(devinfo, msg_type, msg_control);
 825 }
 826
 827 static inline uint32_t
 828 brw_dp_typed_surface_rw_desc(const struct gen_device_info *devinfo,
 829                              unsigned exec_size,
 830                              unsigned exec_group,
 831                              unsigned num_channels,
 832                              bool write)
 833 {
 834    assert(exec_size > 0 || exec_group == 0);
 835    assert(exec_group % 8 == 0);
 836
 837    /* Typed surface reads and writes don't support SIMD16 */
 838    assert(exec_size <= 8);
 839
 840    unsigned msg_type;
 841    if (write) {
 842       if (devinfo->gen >= 8 || devinfo->is_haswell) {
 843          msg_type = HSW_DATAPORT_DC_PORT1_TYPED_SURFACE_WRITE;
 844       } else {
 845          msg_type = GEN7_DATAPORT_RC_TYPED_SURFACE_WRITE;
 846       }
 847    } else {
 848       if (devinfo->gen >= 8 || devinfo->is_haswell) {
 849          msg_type = HSW_DATAPORT_DC_PORT1_TYPED_SURFACE_READ;
 850       } else {
 851          msg_type = GEN7_DATAPORT_RC_TYPED_SURFACE_READ;
 852       }
 853    }
 854
 855    /* See also MDC_SG3 in the SKL PRM Vol 2d. */
 856    unsigned msg_control;
 857    if (devinfo->gen >= 8 || devinfo->is_haswell) {
 858       /* See also MDC_SG3 in the SKL PRM Vol 2d. */
 859       const unsigned slot_group = exec_size == 0 ? 0 : /* SIMD4x2 */
 860                                   1 + ((exec_group / 8) % 2);
 861
 862       msg_control =
 863          SET_BITS(brw_mdc_cmask(num_channels), 3, 0) |
 864          SET_BITS(slot_group, 5, 4);
 865    } else {
 866       /* SIMD4x2 typed surface R/W messages only exist on HSW+ */
 867       assert(exec_size > 0);
 868       const unsigned slot_group = ((exec_group / 8) % 2);
 869
 870       msg_control =
 871          SET_BITS(brw_mdc_cmask(num_channels), 3, 0) |
 872          SET_BITS(slot_group, 5, 5);
 873    }
 874
 875    return brw_dp_surface_desc(devinfo, msg_type, msg_control);
 876 }
 877
 878 /**
 879  * Construct a message descriptor immediate with the specified pixel
 880  * interpolator function controls.
 881  */
 882 static inline uint32_t
 883 brw_pixel_interp_desc(UNUSED const struct gen_device_info *devinfo,
 884                       unsigned msg_type,
 885                       bool noperspective,
 886                       unsigned simd_mode,
 887                       unsigned slot_group)
 888 {
 889    return (SET_BITS(slot_group, 11, 11) |
 890            SET_BITS(msg_type, 13, 12) |
 891            SET_BITS(!!noperspective, 14, 14) |
 892            SET_BITS(simd_mode, 16, 16));
 893 }
 894
 895 void brw_urb_WRITE(struct brw_codegen *p,
 896                    struct brw_reg dest,
 897                    unsigned msg_reg_nr,
 898                    struct brw_reg src0,
 899                    enum brw_urb_write_flags flags,
 900                    unsigned msg_length,
 901                    unsigned response_length,
 902                    unsigned offset,
 903                    unsigned swizzle);
 904
 905 /**
 906  * Send message to shared unit \p sfid with a possibly indirect descriptor \p
 907  * desc.  If \p desc is not an immediate it will be transparently loaded to an
 908  * address register using an OR instruction.
 909  */
 910 void
 911 brw_send_indirect_message(struct brw_codegen *p,
 912                           unsigned sfid,
 913                           struct brw_reg dst,
 914                           struct brw_reg payload,
 915                           struct brw_reg desc,
 916                           unsigned desc_imm,
 917                           bool eot);
 918
 919 void
 920 brw_send_indirect_split_message(struct brw_codegen *p,
 921                                 unsigned sfid,
 922                                 struct brw_reg dst,
 923                                 struct brw_reg payload0,
 924                                 struct brw_reg payload1,
 925                                 struct brw_reg desc,
 926                                 unsigned desc_imm,
 927                                 struct brw_reg ex_desc,
 928                                 unsigned ex_desc_imm,
 929                                 bool eot);
 930
 931 void brw_ff_sync(struct brw_codegen *p,
 932                    struct brw_reg dest,
 933                    unsigned msg_reg_nr,
 934                    struct brw_reg src0,
 935                    bool allocate,
 936                    unsigned response_length,
 937                    bool eot);
 938
 939 void brw_svb_write(struct brw_codegen *p,
 940                    struct brw_reg dest,
 941                    unsigned msg_reg_nr,
 942                    struct brw_reg src0,
 943                    unsigned binding_table_index,
 944                    bool   send_commit_msg);
 945
 946 brw_inst *brw_fb_WRITE(struct brw_codegen *p,
 947                        struct brw_reg payload,
 948                        struct brw_reg implied_header,
 949                        unsigned msg_control,
 950                        unsigned binding_table_index,
 951                        unsigned msg_length,
 952                        unsigned response_length,
 953                        bool eot,
 954                        bool last_render_target,
 955                        bool header_present);
 956
 957 brw_inst *gen9_fb_READ(struct brw_codegen *p,
 958                        struct brw_reg dst,
 959                        struct brw_reg payload,
 960                        unsigned binding_table_index,
 961                        unsigned msg_length,
 962                        unsigned response_length,
 963                        bool per_sample);
 964
 965 void brw_SAMPLE(struct brw_codegen *p,
 966                 struct brw_reg dest,
 967                 unsigned msg_reg_nr,
 968                 struct brw_reg src0,
 969                 unsigned binding_table_index,
 970                 unsigned sampler,
 971                 unsigned msg_type,
 972                 unsigned response_length,
 973                 unsigned msg_length,
 974                 unsigned header_present,
 975                 unsigned simd_mode,
 976                 unsigned return_format);
 977
 978 void brw_adjust_sampler_state_pointer(struct brw_codegen *p,
 979                                       struct brw_reg header,
 980                                       struct brw_reg sampler_index);
 981
 982 void gen4_math(struct brw_codegen *p,
 983                struct brw_reg dest,
 984                unsigned function,
 985                unsigned msg_reg_nr,
 986                struct brw_reg src,
 987                unsigned precision );
 988
 989 void gen6_math(struct brw_codegen *p,
 990                struct brw_reg dest,
 991                unsigned function,
 992                struct brw_reg src0,
 993                struct brw_reg src1);
 994
 995 void brw_oword_block_read(struct brw_codegen *p,
 996                           struct brw_reg dest,
 997                           struct brw_reg mrf,
 998                           uint32_t offset,
 999                           uint32_t bind_table_index);
1000
1001 unsigned brw_scratch_surface_idx(const struct brw_codegen *p);
1002
1003 void brw_oword_block_read_scratch(struct brw_codegen *p,
1004                                   struct brw_reg dest,
1005                                   struct brw_reg mrf,
1006                                   int num_regs,
1007                                   unsigned offset);
1008
1009 void brw_oword_block_write_scratch(struct brw_codegen *p,
1010                                    struct brw_reg mrf,
1011                                    int num_regs,
1012                                    unsigned offset);
1013
1014 void gen7_block_read_scratch(struct brw_codegen *p,
1015                              struct brw_reg dest,
1016                              int num_regs,
1017                              unsigned offset);
1018
1019 void brw_shader_time_add(struct brw_codegen *p,
1020                          struct brw_reg payload,
1021                          uint32_t surf_index);
1022
1023 /**
1024  * Return the generation-specific jump distance scaling factor.
1025  *
1026  * Given the number of instructions to jump, we need to scale by
1027  * some number to obtain the actual jump distance to program in an
1028  * instruction.
1029  */
1030 static inline unsigned
1031 brw_jump_scale(const struct gen_device_info *devinfo)
1032 {
1033    /* Broadwell measures jump targets in bytes. */
1034    if (devinfo->gen >= 8)
1035       return 16;
1036
1037    /* Ironlake and later measure jump targets in 64-bit data chunks (in order
1038     * (to support compaction), so each 128-bit instruction requires 2 chunks.
1039     */
1040    if (devinfo->gen >= 5)
1041       return 2;
1042
1043    /* Gen4 simply uses the number of 128-bit instructions. */
1044    return 1;
1045 }
1046
1047 void brw_barrier(struct brw_codegen *p, struct brw_reg src);
1048
1049 /* If/else/endif.  Works by manipulating the execution flags on each
1050  * channel.
1051  */
1052 brw_inst *brw_IF(struct brw_codegen *p, unsigned execute_size);
1053 brw_inst *gen6_IF(struct brw_codegen *p, enum brw_conditional_mod conditional,
1054                   struct brw_reg src0, struct brw_reg src1);
1055
1056 void brw_ELSE(struct brw_codegen *p);
1057 void brw_ENDIF(struct brw_codegen *p);
1058
1059 /* DO/WHILE loops:
1060  */
1061 brw_inst *brw_DO(struct brw_codegen *p, unsigned execute_size);
1062
1063 brw_inst *brw_WHILE(struct brw_codegen *p);
1064
1065 brw_inst *brw_BREAK(struct brw_codegen *p);
1066 brw_inst *brw_CONT(struct brw_codegen *p);
1067 brw_inst *gen6_HALT(struct brw_codegen *p);
1068
1069 /* Forward jumps:
1070  */
1071 void brw_land_fwd_jump(struct brw_codegen *p, int jmp_insn_idx);
1072
1073 brw_inst *brw_JMPI(struct brw_codegen *p, struct brw_reg index,
1074                    unsigned predicate_control);
1075
1076 void brw_NOP(struct brw_codegen *p);
1077
1078 void brw_WAIT(struct brw_codegen *p);
1079
1080 /* Special case: there is never a destination, execution size will be
1081  * taken from src0:
1082  */
1083 void brw_CMP(struct brw_codegen *p,
1084              struct brw_reg dest,
1085              unsigned conditional,
1086              struct brw_reg src0,
1087              struct brw_reg src1);
1088
1089 void
1090 brw_untyped_atomic(struct brw_codegen *p,
1091                    struct brw_reg dst,
1092                    struct brw_reg payload,
1093                    struct brw_reg surface,
1094                    unsigned atomic_op,
1095                    unsigned msg_length,
1096                    bool response_expected,
1097                    bool header_present);
1098
1099 void
1100 brw_untyped_surface_read(struct brw_codegen *p,
1101                          struct brw_reg dst,
1102                          struct brw_reg payload,
1103                          struct brw_reg surface,
1104                          unsigned msg_length,
1105                          unsigned num_channels);
1106
1107 void
1108 brw_untyped_surface_write(struct brw_codegen *p,
1109                           struct brw_reg payload,
1110                           struct brw_reg surface,
1111                           unsigned msg_length,
1112                           unsigned num_channels,
1113                           bool header_present);
1114
1115 void
1116 brw_memory_fence(struct brw_codegen *p,
1117                  struct brw_reg dst,
1118                  struct brw_reg src,
1119                  enum opcode send_op,
1120                  bool stall,
1121                  unsigned bti);
1122
1123 void
1124 brw_pixel_interpolator_query(struct brw_codegen *p,
1125                              struct brw_reg dest,
1126                              struct brw_reg mrf,
1127                              bool noperspective,
1128                              unsigned mode,
1129                              struct brw_reg data,
1130                              unsigned msg_length,
1131                              unsigned response_length);
1132
1133 void
1134 brw_find_live_channel(struct brw_codegen *p,
1135                       struct brw_reg dst,
1136                       struct brw_reg mask);
1137
1138 void
1139 brw_broadcast(struct brw_codegen *p,
1140               struct brw_reg dst,
1141               struct brw_reg src,
1142               struct brw_reg idx);
1143
1144 void
1145 brw_rounding_mode(struct brw_codegen *p,
1146                   enum brw_rnd_mode mode);
1147
1148 /***********************************************************************
1149  * brw_eu_util.c:
1150  */
1151
1152 void brw_copy_indirect_to_indirect(struct brw_codegen *p,
1153                                    struct brw_indirect dst_ptr,
1154                                    struct brw_indirect src_ptr,
1155                                    unsigned count);
1156
1157 void brw_copy_from_indirect(struct brw_codegen *p,
1158                             struct brw_reg dst,
1159                             struct brw_indirect ptr,
1160                             unsigned count);
1161
1162 void brw_copy4(struct brw_codegen *p,
1163                struct brw_reg dst,
1164                struct brw_reg src,
1165                unsigned count);
1166
1167 void brw_copy8(struct brw_codegen *p,
1168                struct brw_reg dst,
1169                struct brw_reg src,
1170                unsigned count);
1171
1172 void brw_math_invert( struct brw_codegen *p,
1173                       struct brw_reg dst,
1174                       struct brw_reg src);
1175
1176 void brw_set_src1(struct brw_codegen *p, brw_inst *insn, struct brw_reg reg);
1177
1178 void brw_set_desc_ex(struct brw_codegen *p, brw_inst *insn,
1179                      unsigned desc, unsigned ex_desc);
1180
1181 static inline void
1182 brw_set_desc(struct brw_codegen *p, brw_inst *insn, unsigned desc)
1183 {
1184    brw_set_desc_ex(p, insn, desc, 0);
1185 }
1186
1187 void brw_set_uip_jip(struct brw_codegen *p, int start_offset);
1188
1189 enum brw_conditional_mod brw_negate_cmod(uint32_t cmod);
1190 enum brw_conditional_mod brw_swap_cmod(uint32_t cmod);
1191
1192 /* brw_eu_compact.c */
1193 void brw_init_compaction_tables(const struct gen_device_info *devinfo);
1194 void brw_compact_instructions(struct brw_codegen *p, int start_offset,
1195                               struct disasm_info *disasm);
1196 void brw_uncompact_instruction(const struct gen_device_info *devinfo,
1197                                brw_inst *dst, brw_compact_inst *src);
1198 bool brw_try_compact_instruction(const struct gen_device_info *devinfo,
1199                                  brw_compact_inst *dst, const brw_inst *src);
1200
1201 void brw_debug_compact_uncompact(const struct gen_device_info *devinfo,
1202                                  brw_inst *orig, brw_inst *uncompacted);
1203
1204 /* brw_eu_validate.c */
1205 bool brw_validate_instructions(const struct gen_device_info *devinfo,
1206                                const void *assembly, int start_offset, int end_offset,
1207                                struct disasm_info *disasm);
1208
1209 static inline int
1210 next_offset(const struct gen_device_info *devinfo, void *store, int offset)
1211 {
1212    brw_inst *insn = (brw_inst *)((char *)store + offset);
1213
1214    if (brw_inst_cmpt_control(devinfo, insn))
1215       return offset + 8;
1216    else
1217       return offset + 16;
1218 }
1219
1220 struct opcode_desc {
1221    /* The union is an implementation detail used by brw_opcode_desc() to handle
1222     * opcodes that have been reused for different instructions across hardware
1223     * generations.
1224     *
1225     * The gens field acts as a tag. If it is non-zero, name points to a string
1226     * containing the instruction mnemonic. If it is zero, the table field is
1227     * valid and either points to a secondary opcode_desc table with 'size'
1228     * elements or is NULL and no such instruction exists for the opcode.
1229     */
1230    union {
1231       struct {
1232          char    *name;
1233          int      nsrc;
1234       };
1235       struct {
1236          const struct opcode_desc *table;
1237          unsigned size;
1238       };
1239    };
1240    int      ndst;
1241    int      gens;
1242 };
1243
1244 const struct opcode_desc *
1245 brw_opcode_desc(const struct gen_device_info *devinfo, enum opcode opcode);
1246
1247 static inline bool
1248 is_3src(const struct gen_device_info *devinfo, enum opcode opcode)
1249 {
1250    const struct opcode_desc *desc = brw_opcode_desc(devinfo, opcode);
1251    return desc && desc->nsrc == 3;
1252 }
1253
1254 /** Maximum SEND message length */
1255 #define BRW_MAX_MSG_LENGTH 15
1256
1257 /** First MRF register used by pull loads */
1258 #define FIRST_SPILL_MRF(gen) ((gen) == 6 ? 21 : 13)
1259
1260 /** First MRF register used by spills */
1261 #define FIRST_PULL_LOAD_MRF(gen) ((gen) == 6 ? 16 : 13)
1262
1263 #ifdef __cplusplus
1264 }
1265 #endif
1266
1267 #endif