2 Copyright (C) Intel Corp. 2006. All Rights Reserved.
3 Intel funded Tungsten Graphics to
4 develop this 3D driver.
6 Permission is hereby granted, free of charge, to any person obtaining
7 a copy of this software and associated documentation files (the
8 "Software"), to deal in the Software without restriction, including
9 without limitation the rights to use, copy, modify, merge, publish,
10 distribute, sublicense, and/or sell copies of the Software, and to
11 permit persons to whom the Software is furnished to do so, subject to
12 the following conditions:
14 The above copyright notice and this permission notice (including the
15 next paragraph) shall be included in all copies or substantial
16 portions of the Software.
18 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
19 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
21 IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
22 LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
23 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
24 WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **********************************************************************/
29 * Keith Whitwell <keithw@vmware.com>
39 #include "brw_eu_defines.h"
41 #include "brw_disasm_info.h"
47 #define BRW_EU_MAX_INSN_STACK 5
49 struct brw_insn_state
{
50 /* One of BRW_EXECUTE_* */
53 /* Group in units of channels */
56 /* Compression control on gen4-5 */
59 /* One of BRW_MASK_* */
60 unsigned mask_control
:1;
62 /* Scheduling info for Gen12+ */
67 /* One of BRW_ALIGN_* */
68 unsigned access_mode
:1;
70 /* One of BRW_PREDICATE_* */
71 enum brw_predicate predicate
:4;
75 /* Flag subreg. Bottom bit is subreg, top bit is reg */
76 unsigned flag_subreg
:2;
78 bool acc_wr_control
:1;
82 /* A helper for accessing the last instruction emitted. This makes it easy
83 * to set various bits on an instruction without having to create temporary
84 * variable and assign the emitted instruction to those.
86 #define brw_last_inst (&p->store[p->nr_insn - 1])
92 unsigned int next_insn_offset
;
96 /* Allow clients to push/pop instruction state:
98 struct brw_insn_state stack
[BRW_EU_MAX_INSN_STACK
];
99 struct brw_insn_state
*current
;
101 /** Whether or not the user wants automatic exec sizes
103 * If true, codegen will try to automatically infer the exec size of an
104 * instruction from the width of the destination register. If false, it
105 * will take whatever is set by brw_set_default_exec_size verbatim.
107 * This is set to true by default in brw_init_codegen.
109 bool automatic_exec_sizes
;
111 bool single_program_flow
;
112 const struct gen_device_info
*devinfo
;
114 /* Control flow stacks:
115 * - if_stack contains IF and ELSE instructions which must be patched
116 * (and popped) once the matching ENDIF instruction is encountered.
118 * Just store the instruction pointer(an index).
122 int if_stack_array_size
;
125 * loop_stack contains the instruction pointers of the starts of loops which
126 * must be patched (and popped) once the matching WHILE instruction is
131 * pre-gen6, the BREAK and CONT instructions had to tell how many IF/ENDIF
132 * blocks they were popping out of, to fix up the mask stack. This tracks
133 * the IF/ENDIF nesting in each current nested loop level.
135 int *if_depth_in_loop
;
136 int loop_stack_depth
;
137 int loop_stack_array_size
;
143 struct brw_label
*next
;
146 void brw_pop_insn_state( struct brw_codegen
*p
);
147 void brw_push_insn_state( struct brw_codegen
*p
);
148 unsigned brw_get_default_exec_size(struct brw_codegen
*p
);
149 unsigned brw_get_default_group(struct brw_codegen
*p
);
150 unsigned brw_get_default_access_mode(struct brw_codegen
*p
);
151 struct tgl_swsb
brw_get_default_swsb(struct brw_codegen
*p
);
152 void brw_set_default_exec_size(struct brw_codegen
*p
, unsigned value
);
153 void brw_set_default_mask_control( struct brw_codegen
*p
, unsigned value
);
154 void brw_set_default_saturate( struct brw_codegen
*p
, bool enable
);
155 void brw_set_default_access_mode( struct brw_codegen
*p
, unsigned access_mode
);
156 void brw_inst_set_compression(const struct gen_device_info
*devinfo
,
157 brw_inst
*inst
, bool on
);
158 void brw_set_default_compression(struct brw_codegen
*p
, bool on
);
159 void brw_inst_set_group(const struct gen_device_info
*devinfo
,
160 brw_inst
*inst
, unsigned group
);
161 void brw_set_default_group(struct brw_codegen
*p
, unsigned group
);
162 void brw_set_default_compression_control(struct brw_codegen
*p
, enum brw_compression c
);
163 void brw_set_default_predicate_control(struct brw_codegen
*p
, enum brw_predicate pc
);
164 void brw_set_default_predicate_inverse(struct brw_codegen
*p
, bool predicate_inverse
);
165 void brw_set_default_flag_reg(struct brw_codegen
*p
, int reg
, int subreg
);
166 void brw_set_default_acc_write_control(struct brw_codegen
*p
, unsigned value
);
167 void brw_set_default_swsb(struct brw_codegen
*p
, struct tgl_swsb value
);
169 void brw_init_codegen(const struct gen_device_info
*, struct brw_codegen
*p
,
171 bool brw_has_jip(const struct gen_device_info
*devinfo
, enum opcode opcode
);
172 bool brw_has_uip(const struct gen_device_info
*devinfo
, enum opcode opcode
);
173 const struct brw_label
*brw_find_label(const struct brw_label
*root
, int offset
);
174 void brw_create_label(struct brw_label
**labels
, int offset
, void *mem_ctx
);
175 int brw_disassemble_inst(FILE *file
, const struct gen_device_info
*devinfo
,
176 const struct brw_inst
*inst
, bool is_compacted
,
177 int offset
, const struct brw_label
*root_label
);
178 const struct brw_label
*brw_label_assembly(const struct gen_device_info
*devinfo
,
179 const void *assembly
, int start
, int end
,
181 void brw_disassemble_with_labels(const struct gen_device_info
*devinfo
,
182 const void *assembly
, int start
, int end
, FILE *out
);
183 void brw_disassemble(const struct gen_device_info
*devinfo
,
184 const void *assembly
, int start
, int end
,
185 const struct brw_label
*root_label
, FILE *out
);
186 const unsigned *brw_get_program( struct brw_codegen
*p
, unsigned *sz
);
188 bool brw_try_override_assembly(struct brw_codegen
*p
, int start_offset
,
189 const char *identifier
);
191 void brw_realign(struct brw_codegen
*p
, unsigned align
);
192 int brw_append_data(struct brw_codegen
*p
, void *data
,
193 unsigned size
, unsigned align
);
194 brw_inst
*brw_next_insn(struct brw_codegen
*p
, unsigned opcode
);
195 void brw_set_dest(struct brw_codegen
*p
, brw_inst
*insn
, struct brw_reg dest
);
196 void brw_set_src0(struct brw_codegen
*p
, brw_inst
*insn
, struct brw_reg reg
);
198 void gen6_resolve_implied_move(struct brw_codegen
*p
,
200 unsigned msg_reg_nr
);
202 /* Helpers for regular instructions:
205 brw_inst *brw_##OP(struct brw_codegen *p, \
206 struct brw_reg dest, \
207 struct brw_reg src0);
210 brw_inst *brw_##OP(struct brw_codegen *p, \
211 struct brw_reg dest, \
212 struct brw_reg src0, \
213 struct brw_reg src1);
216 brw_inst *brw_##OP(struct brw_codegen *p, \
217 struct brw_reg dest, \
218 struct brw_reg src0, \
219 struct brw_reg src1, \
220 struct brw_reg src2);
272 /* Helpers for SEND instruction:
276 * Construct a message descriptor immediate with the specified common
277 * descriptor controls.
279 static inline uint32_t
280 brw_message_desc(const struct gen_device_info
*devinfo
,
282 unsigned response_length
,
285 if (devinfo
->gen
>= 5) {
286 return (SET_BITS(msg_length
, 28, 25) |
287 SET_BITS(response_length
, 24, 20) |
288 SET_BITS(header_present
, 19, 19));
290 return (SET_BITS(msg_length
, 23, 20) |
291 SET_BITS(response_length
, 19, 16));
295 static inline unsigned
296 brw_message_desc_mlen(const struct gen_device_info
*devinfo
, uint32_t desc
)
298 if (devinfo
->gen
>= 5)
299 return GET_BITS(desc
, 28, 25);
301 return GET_BITS(desc
, 23, 20);
304 static inline unsigned
305 brw_message_desc_rlen(const struct gen_device_info
*devinfo
, uint32_t desc
)
307 if (devinfo
->gen
>= 5)
308 return GET_BITS(desc
, 24, 20);
310 return GET_BITS(desc
, 19, 16);
314 brw_message_desc_header_present(ASSERTED
const struct gen_device_info
*devinfo
,
317 assert(devinfo
->gen
>= 5);
318 return GET_BITS(desc
, 19, 19);
321 static inline unsigned
322 brw_message_ex_desc(UNUSED
const struct gen_device_info
*devinfo
,
323 unsigned ex_msg_length
)
325 return SET_BITS(ex_msg_length
, 9, 6);
328 static inline unsigned
329 brw_message_ex_desc_ex_mlen(UNUSED
const struct gen_device_info
*devinfo
,
332 return GET_BITS(ex_desc
, 9, 6);
335 static inline uint32_t
336 brw_urb_desc(const struct gen_device_info
*devinfo
,
338 bool per_slot_offset_present
,
339 bool channel_mask_present
,
340 unsigned global_offset
)
342 if (devinfo
->gen
>= 8) {
343 return (SET_BITS(per_slot_offset_present
, 17, 17) |
344 SET_BITS(channel_mask_present
, 15, 15) |
345 SET_BITS(global_offset
, 14, 4) |
346 SET_BITS(msg_type
, 3, 0));
347 } else if (devinfo
->gen
>= 7) {
348 assert(!channel_mask_present
);
349 return (SET_BITS(per_slot_offset_present
, 16, 16) |
350 SET_BITS(global_offset
, 13, 3) |
351 SET_BITS(msg_type
, 3, 0));
353 unreachable("unhandled URB write generation");
357 static inline uint32_t
358 brw_urb_desc_msg_type(ASSERTED
const struct gen_device_info
*devinfo
,
361 assert(devinfo
->gen
>= 7);
362 return GET_BITS(desc
, 3, 0);
366 * Construct a message descriptor immediate with the specified sampler
369 static inline uint32_t
370 brw_sampler_desc(const struct gen_device_info
*devinfo
,
371 unsigned binding_table_index
,
375 unsigned return_format
)
377 const unsigned desc
= (SET_BITS(binding_table_index
, 7, 0) |
378 SET_BITS(sampler
, 11, 8));
379 if (devinfo
->gen
>= 7)
380 return (desc
| SET_BITS(msg_type
, 16, 12) |
381 SET_BITS(simd_mode
, 18, 17));
382 else if (devinfo
->gen
>= 5)
383 return (desc
| SET_BITS(msg_type
, 15, 12) |
384 SET_BITS(simd_mode
, 17, 16));
385 else if (devinfo
->is_g4x
)
386 return desc
| SET_BITS(msg_type
, 15, 12);
388 return (desc
| SET_BITS(return_format
, 13, 12) |
389 SET_BITS(msg_type
, 15, 14));
392 static inline unsigned
393 brw_sampler_desc_binding_table_index(UNUSED
const struct gen_device_info
*devinfo
,
396 return GET_BITS(desc
, 7, 0);
399 static inline unsigned
400 brw_sampler_desc_sampler(UNUSED
const struct gen_device_info
*devinfo
, uint32_t desc
)
402 return GET_BITS(desc
, 11, 8);
405 static inline unsigned
406 brw_sampler_desc_msg_type(const struct gen_device_info
*devinfo
, uint32_t desc
)
408 if (devinfo
->gen
>= 7)
409 return GET_BITS(desc
, 16, 12);
410 else if (devinfo
->gen
>= 5 || devinfo
->is_g4x
)
411 return GET_BITS(desc
, 15, 12);
413 return GET_BITS(desc
, 15, 14);
416 static inline unsigned
417 brw_sampler_desc_simd_mode(const struct gen_device_info
*devinfo
, uint32_t desc
)
419 assert(devinfo
->gen
>= 5);
420 if (devinfo
->gen
>= 7)
421 return GET_BITS(desc
, 18, 17);
423 return GET_BITS(desc
, 17, 16);
426 static inline unsigned
427 brw_sampler_desc_return_format(ASSERTED
const struct gen_device_info
*devinfo
,
430 assert(devinfo
->gen
== 4 && !devinfo
->is_g4x
);
431 return GET_BITS(desc
, 13, 12);
435 * Construct a message descriptor for the dataport
437 static inline uint32_t
438 brw_dp_desc(const struct gen_device_info
*devinfo
,
439 unsigned binding_table_index
,
441 unsigned msg_control
)
443 /* Prior to gen6, things are too inconsistent; use the dp_read/write_desc
446 assert(devinfo
->gen
>= 6);
447 const unsigned desc
= SET_BITS(binding_table_index
, 7, 0);
448 if (devinfo
->gen
>= 8) {
449 return (desc
| SET_BITS(msg_control
, 13, 8) |
450 SET_BITS(msg_type
, 18, 14));
451 } else if (devinfo
->gen
>= 7) {
452 return (desc
| SET_BITS(msg_control
, 13, 8) |
453 SET_BITS(msg_type
, 17, 14));
455 return (desc
| SET_BITS(msg_control
, 12, 8) |
456 SET_BITS(msg_type
, 16, 13));
460 static inline unsigned
461 brw_dp_desc_binding_table_index(UNUSED
const struct gen_device_info
*devinfo
,
464 return GET_BITS(desc
, 7, 0);
467 static inline unsigned
468 brw_dp_desc_msg_type(const struct gen_device_info
*devinfo
, uint32_t desc
)
470 assert(devinfo
->gen
>= 6);
471 if (devinfo
->gen
>= 8)
472 return GET_BITS(desc
, 18, 14);
473 else if (devinfo
->gen
>= 7)
474 return GET_BITS(desc
, 17, 14);
476 return GET_BITS(desc
, 16, 13);
479 static inline unsigned
480 brw_dp_desc_msg_control(const struct gen_device_info
*devinfo
, uint32_t desc
)
482 assert(devinfo
->gen
>= 6);
483 if (devinfo
->gen
>= 7)
484 return GET_BITS(desc
, 13, 8);
486 return GET_BITS(desc
, 12, 8);
490 * Construct a message descriptor immediate with the specified dataport read
493 static inline uint32_t
494 brw_dp_read_desc(const struct gen_device_info
*devinfo
,
495 unsigned binding_table_index
,
496 unsigned msg_control
,
498 unsigned target_cache
)
500 if (devinfo
->gen
>= 6)
501 return brw_dp_desc(devinfo
, binding_table_index
, msg_type
, msg_control
);
502 else if (devinfo
->gen
>= 5 || devinfo
->is_g4x
)
503 return (SET_BITS(binding_table_index
, 7, 0) |
504 SET_BITS(msg_control
, 10, 8) |
505 SET_BITS(msg_type
, 13, 11) |
506 SET_BITS(target_cache
, 15, 14));
508 return (SET_BITS(binding_table_index
, 7, 0) |
509 SET_BITS(msg_control
, 11, 8) |
510 SET_BITS(msg_type
, 13, 12) |
511 SET_BITS(target_cache
, 15, 14));
514 static inline unsigned
515 brw_dp_read_desc_msg_type(const struct gen_device_info
*devinfo
, uint32_t desc
)
517 if (devinfo
->gen
>= 6)
518 return brw_dp_desc_msg_type(devinfo
, desc
);
519 else if (devinfo
->gen
>= 5 || devinfo
->is_g4x
)
520 return GET_BITS(desc
, 13, 11);
522 return GET_BITS(desc
, 13, 12);
525 static inline unsigned
526 brw_dp_read_desc_msg_control(const struct gen_device_info
*devinfo
,
529 if (devinfo
->gen
>= 6)
530 return brw_dp_desc_msg_control(devinfo
, desc
);
531 else if (devinfo
->gen
>= 5 || devinfo
->is_g4x
)
532 return GET_BITS(desc
, 10, 8);
534 return GET_BITS(desc
, 11, 8);
538 * Construct a message descriptor immediate with the specified dataport write
541 static inline uint32_t
542 brw_dp_write_desc(const struct gen_device_info
*devinfo
,
543 unsigned binding_table_index
,
544 unsigned msg_control
,
546 unsigned last_render_target
,
547 unsigned send_commit_msg
)
549 assert(devinfo
->gen
<= 6 || !send_commit_msg
);
550 if (devinfo
->gen
>= 6)
551 return brw_dp_desc(devinfo
, binding_table_index
, msg_type
, msg_control
) |
552 SET_BITS(last_render_target
, 12, 12) |
553 SET_BITS(send_commit_msg
, 17, 17);
555 return (SET_BITS(binding_table_index
, 7, 0) |
556 SET_BITS(msg_control
, 11, 8) |
557 SET_BITS(last_render_target
, 11, 11) |
558 SET_BITS(msg_type
, 14, 12) |
559 SET_BITS(send_commit_msg
, 15, 15));
562 static inline unsigned
563 brw_dp_write_desc_msg_type(const struct gen_device_info
*devinfo
,
566 if (devinfo
->gen
>= 6)
567 return brw_dp_desc_msg_type(devinfo
, desc
);
569 return GET_BITS(desc
, 14, 12);
572 static inline unsigned
573 brw_dp_write_desc_msg_control(const struct gen_device_info
*devinfo
,
576 if (devinfo
->gen
>= 6)
577 return brw_dp_desc_msg_control(devinfo
, desc
);
579 return GET_BITS(desc
, 11, 8);
583 brw_dp_write_desc_last_render_target(const struct gen_device_info
*devinfo
,
586 if (devinfo
->gen
>= 6)
587 return GET_BITS(desc
, 12, 12);
589 return GET_BITS(desc
, 11, 11);
593 brw_dp_write_desc_write_commit(const struct gen_device_info
*devinfo
,
596 assert(devinfo
->gen
<= 6);
597 if (devinfo
->gen
>= 6)
598 return GET_BITS(desc
, 17, 17);
600 return GET_BITS(desc
, 15, 15);
604 * Construct a message descriptor immediate with the specified dataport
605 * surface function controls.
607 static inline uint32_t
608 brw_dp_surface_desc(const struct gen_device_info
*devinfo
,
610 unsigned msg_control
)
612 assert(devinfo
->gen
>= 7);
613 /* We'll OR in the binding table index later */
614 return brw_dp_desc(devinfo
, 0, msg_type
, msg_control
);
617 static inline uint32_t
618 brw_dp_untyped_atomic_desc(const struct gen_device_info
*devinfo
,
619 unsigned exec_size
, /**< 0 for SIMD4x2 */
621 bool response_expected
)
623 assert(exec_size
<= 8 || exec_size
== 16);
626 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
) {
628 msg_type
= HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP
;
630 msg_type
= HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP_SIMD4X2
;
633 msg_type
= GEN7_DATAPORT_DC_UNTYPED_ATOMIC_OP
;
636 const unsigned msg_control
=
637 SET_BITS(atomic_op
, 3, 0) |
638 SET_BITS(0 < exec_size
&& exec_size
<= 8, 4, 4) |
639 SET_BITS(response_expected
, 5, 5);
641 return brw_dp_surface_desc(devinfo
, msg_type
, msg_control
);
644 static inline uint32_t
645 brw_dp_untyped_atomic_float_desc(const struct gen_device_info
*devinfo
,
648 bool response_expected
)
650 assert(exec_size
<= 8 || exec_size
== 16);
651 assert(devinfo
->gen
>= 9);
653 assert(exec_size
> 0);
654 const unsigned msg_type
= GEN9_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_FLOAT_OP
;
656 const unsigned msg_control
=
657 SET_BITS(atomic_op
, 1, 0) |
658 SET_BITS(exec_size
<= 8, 4, 4) |
659 SET_BITS(response_expected
, 5, 5);
661 return brw_dp_surface_desc(devinfo
, msg_type
, msg_control
);
664 static inline unsigned
665 brw_mdc_cmask(unsigned num_channels
)
667 /* See also MDC_CMASK in the SKL PRM Vol 2d. */
668 return 0xf & (0xf << num_channels
);
671 static inline uint32_t
672 brw_dp_untyped_surface_rw_desc(const struct gen_device_info
*devinfo
,
673 unsigned exec_size
, /**< 0 for SIMD4x2 */
674 unsigned num_channels
,
677 assert(exec_size
<= 8 || exec_size
== 16);
681 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
) {
682 msg_type
= HSW_DATAPORT_DC_PORT1_UNTYPED_SURFACE_WRITE
;
684 msg_type
= GEN7_DATAPORT_DC_UNTYPED_SURFACE_WRITE
;
688 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
) {
689 msg_type
= HSW_DATAPORT_DC_PORT1_UNTYPED_SURFACE_READ
;
691 msg_type
= GEN7_DATAPORT_DC_UNTYPED_SURFACE_READ
;
695 /* SIMD4x2 is only valid for read messages on IVB; use SIMD8 instead */
696 if (write
&& devinfo
->gen
== 7 && !devinfo
->is_haswell
&& exec_size
== 0)
699 /* See also MDC_SM3 in the SKL PRM Vol 2d. */
700 const unsigned simd_mode
= exec_size
== 0 ? 0 : /* SIMD4x2 */
701 exec_size
<= 8 ? 2 : 1;
703 const unsigned msg_control
=
704 SET_BITS(brw_mdc_cmask(num_channels
), 3, 0) |
705 SET_BITS(simd_mode
, 5, 4);
707 return brw_dp_surface_desc(devinfo
, msg_type
, msg_control
);
710 static inline unsigned
711 brw_mdc_ds(unsigned bit_size
)
715 return GEN7_BYTE_SCATTERED_DATA_ELEMENT_BYTE
;
717 return GEN7_BYTE_SCATTERED_DATA_ELEMENT_WORD
;
719 return GEN7_BYTE_SCATTERED_DATA_ELEMENT_DWORD
;
721 unreachable("Unsupported bit_size for byte scattered messages");
725 static inline uint32_t
726 brw_dp_byte_scattered_rw_desc(const struct gen_device_info
*devinfo
,
731 assert(exec_size
<= 8 || exec_size
== 16);
733 assert(devinfo
->gen
> 7 || devinfo
->is_haswell
);
734 const unsigned msg_type
=
735 write
? HSW_DATAPORT_DC_PORT0_BYTE_SCATTERED_WRITE
:
736 HSW_DATAPORT_DC_PORT0_BYTE_SCATTERED_READ
;
738 assert(exec_size
> 0);
739 const unsigned msg_control
=
740 SET_BITS(exec_size
== 16, 0, 0) |
741 SET_BITS(brw_mdc_ds(bit_size
), 3, 2);
743 return brw_dp_surface_desc(devinfo
, msg_type
, msg_control
);
746 static inline uint32_t
747 brw_dp_dword_scattered_rw_desc(const struct gen_device_info
*devinfo
,
751 assert(exec_size
== 8 || exec_size
== 16);
755 if (devinfo
->gen
>= 6) {
756 msg_type
= GEN6_DATAPORT_WRITE_MESSAGE_DWORD_SCATTERED_WRITE
;
758 msg_type
= BRW_DATAPORT_WRITE_MESSAGE_DWORD_SCATTERED_WRITE
;
761 if (devinfo
->gen
>= 7) {
762 msg_type
= GEN7_DATAPORT_DC_DWORD_SCATTERED_READ
;
763 } else if (devinfo
->gen
> 4 || devinfo
->is_g4x
) {
764 msg_type
= G45_DATAPORT_READ_MESSAGE_DWORD_SCATTERED_READ
;
766 msg_type
= BRW_DATAPORT_READ_MESSAGE_DWORD_SCATTERED_READ
;
770 const unsigned msg_control
=
771 SET_BITS(1, 1, 1) | /* Legacy SIMD Mode */
772 SET_BITS(exec_size
== 16, 0, 0);
774 return brw_dp_surface_desc(devinfo
, msg_type
, msg_control
);
777 static inline uint32_t
778 brw_dp_a64_untyped_surface_rw_desc(const struct gen_device_info
*devinfo
,
779 unsigned exec_size
, /**< 0 for SIMD4x2 */
780 unsigned num_channels
,
783 assert(exec_size
<= 8 || exec_size
== 16);
784 assert(devinfo
->gen
>= 8);
787 write
? GEN8_DATAPORT_DC_PORT1_A64_UNTYPED_SURFACE_WRITE
:
788 GEN8_DATAPORT_DC_PORT1_A64_UNTYPED_SURFACE_READ
;
790 /* See also MDC_SM3 in the SKL PRM Vol 2d. */
791 const unsigned simd_mode
= exec_size
== 0 ? 0 : /* SIMD4x2 */
792 exec_size
<= 8 ? 2 : 1;
794 const unsigned msg_control
=
795 SET_BITS(brw_mdc_cmask(num_channels
), 3, 0) |
796 SET_BITS(simd_mode
, 5, 4);
798 return brw_dp_desc(devinfo
, GEN8_BTI_STATELESS_NON_COHERENT
,
799 msg_type
, msg_control
);
803 * Calculate the data size (see MDC_A64_DS in the "Structures" volume of the
806 static inline uint32_t
807 brw_mdc_a64_ds(unsigned elems
)
815 unreachable("Unsupported elmeent count for A64 scattered message");
819 static inline uint32_t
820 brw_dp_a64_byte_scattered_rw_desc(const struct gen_device_info
*devinfo
,
821 unsigned exec_size
, /**< 0 for SIMD4x2 */
825 assert(exec_size
<= 8 || exec_size
== 16);
826 assert(devinfo
->gen
>= 8);
829 write
? GEN8_DATAPORT_DC_PORT1_A64_SCATTERED_WRITE
:
830 GEN9_DATAPORT_DC_PORT1_A64_SCATTERED_READ
;
832 const unsigned msg_control
=
833 SET_BITS(GEN8_A64_SCATTERED_SUBTYPE_BYTE
, 1, 0) |
834 SET_BITS(brw_mdc_a64_ds(bit_size
/ 8), 3, 2) |
835 SET_BITS(exec_size
== 16, 4, 4);
837 return brw_dp_desc(devinfo
, GEN8_BTI_STATELESS_NON_COHERENT
,
838 msg_type
, msg_control
);
841 static inline uint32_t
842 brw_dp_a64_untyped_atomic_desc(const struct gen_device_info
*devinfo
,
843 ASSERTED
unsigned exec_size
, /**< 0 for SIMD4x2 */
846 bool response_expected
)
848 assert(exec_size
== 8);
849 assert(devinfo
->gen
>= 8);
850 assert(bit_size
== 32 || bit_size
== 64);
852 const unsigned msg_type
= GEN8_DATAPORT_DC_PORT1_A64_UNTYPED_ATOMIC_OP
;
854 const unsigned msg_control
=
855 SET_BITS(atomic_op
, 3, 0) |
856 SET_BITS(bit_size
== 64, 4, 4) |
857 SET_BITS(response_expected
, 5, 5);
859 return brw_dp_desc(devinfo
, GEN8_BTI_STATELESS_NON_COHERENT
,
860 msg_type
, msg_control
);
863 static inline uint32_t
864 brw_dp_a64_untyped_atomic_float_desc(const struct gen_device_info
*devinfo
,
865 ASSERTED
unsigned exec_size
,
867 bool response_expected
)
869 assert(exec_size
== 8);
870 assert(devinfo
->gen
>= 9);
872 assert(exec_size
> 0);
873 const unsigned msg_type
= GEN9_DATAPORT_DC_PORT1_A64_UNTYPED_ATOMIC_FLOAT_OP
;
875 const unsigned msg_control
=
876 SET_BITS(atomic_op
, 1, 0) |
877 SET_BITS(response_expected
, 5, 5);
879 return brw_dp_desc(devinfo
, GEN8_BTI_STATELESS_NON_COHERENT
,
880 msg_type
, msg_control
);
883 static inline uint32_t
884 brw_dp_typed_atomic_desc(const struct gen_device_info
*devinfo
,
888 bool response_expected
)
890 assert(exec_size
> 0 || exec_group
== 0);
891 assert(exec_group
% 8 == 0);
894 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
) {
895 if (exec_size
== 0) {
896 msg_type
= HSW_DATAPORT_DC_PORT1_TYPED_ATOMIC_OP_SIMD4X2
;
898 msg_type
= HSW_DATAPORT_DC_PORT1_TYPED_ATOMIC_OP
;
901 /* SIMD4x2 typed surface R/W messages only exist on HSW+ */
902 assert(exec_size
> 0);
903 msg_type
= GEN7_DATAPORT_RC_TYPED_ATOMIC_OP
;
906 const bool high_sample_mask
= (exec_group
/ 8) % 2 == 1;
908 const unsigned msg_control
=
909 SET_BITS(atomic_op
, 3, 0) |
910 SET_BITS(high_sample_mask
, 4, 4) |
911 SET_BITS(response_expected
, 5, 5);
913 return brw_dp_surface_desc(devinfo
, msg_type
, msg_control
);
916 static inline uint32_t
917 brw_dp_typed_surface_rw_desc(const struct gen_device_info
*devinfo
,
920 unsigned num_channels
,
923 assert(exec_size
> 0 || exec_group
== 0);
924 assert(exec_group
% 8 == 0);
926 /* Typed surface reads and writes don't support SIMD16 */
927 assert(exec_size
<= 8);
931 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
) {
932 msg_type
= HSW_DATAPORT_DC_PORT1_TYPED_SURFACE_WRITE
;
934 msg_type
= GEN7_DATAPORT_RC_TYPED_SURFACE_WRITE
;
937 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
) {
938 msg_type
= HSW_DATAPORT_DC_PORT1_TYPED_SURFACE_READ
;
940 msg_type
= GEN7_DATAPORT_RC_TYPED_SURFACE_READ
;
944 /* See also MDC_SG3 in the SKL PRM Vol 2d. */
945 unsigned msg_control
;
946 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
) {
947 /* See also MDC_SG3 in the SKL PRM Vol 2d. */
948 const unsigned slot_group
= exec_size
== 0 ? 0 : /* SIMD4x2 */
949 1 + ((exec_group
/ 8) % 2);
952 SET_BITS(brw_mdc_cmask(num_channels
), 3, 0) |
953 SET_BITS(slot_group
, 5, 4);
955 /* SIMD4x2 typed surface R/W messages only exist on HSW+ */
956 assert(exec_size
> 0);
957 const unsigned slot_group
= ((exec_group
/ 8) % 2);
960 SET_BITS(brw_mdc_cmask(num_channels
), 3, 0) |
961 SET_BITS(slot_group
, 5, 5);
964 return brw_dp_surface_desc(devinfo
, msg_type
, msg_control
);
968 * Construct a message descriptor immediate with the specified pixel
969 * interpolator function controls.
971 static inline uint32_t
972 brw_pixel_interp_desc(UNUSED
const struct gen_device_info
*devinfo
,
978 return (SET_BITS(slot_group
, 11, 11) |
979 SET_BITS(msg_type
, 13, 12) |
980 SET_BITS(!!noperspective
, 14, 14) |
981 SET_BITS(simd_mode
, 16, 16));
984 void brw_urb_WRITE(struct brw_codegen
*p
,
988 enum brw_urb_write_flags flags
,
990 unsigned response_length
,
995 * Send message to shared unit \p sfid with a possibly indirect descriptor \p
996 * desc. If \p desc is not an immediate it will be transparently loaded to an
997 * address register using an OR instruction.
1000 brw_send_indirect_message(struct brw_codegen
*p
,
1003 struct brw_reg payload
,
1004 struct brw_reg desc
,
1009 brw_send_indirect_split_message(struct brw_codegen
*p
,
1012 struct brw_reg payload0
,
1013 struct brw_reg payload1
,
1014 struct brw_reg desc
,
1016 struct brw_reg ex_desc
,
1017 unsigned ex_desc_imm
,
1020 void brw_ff_sync(struct brw_codegen
*p
,
1021 struct brw_reg dest
,
1022 unsigned msg_reg_nr
,
1023 struct brw_reg src0
,
1025 unsigned response_length
,
1028 void brw_svb_write(struct brw_codegen
*p
,
1029 struct brw_reg dest
,
1030 unsigned msg_reg_nr
,
1031 struct brw_reg src0
,
1032 unsigned binding_table_index
,
1033 bool send_commit_msg
);
1035 brw_inst
*brw_fb_WRITE(struct brw_codegen
*p
,
1036 struct brw_reg payload
,
1037 struct brw_reg implied_header
,
1038 unsigned msg_control
,
1039 unsigned binding_table_index
,
1040 unsigned msg_length
,
1041 unsigned response_length
,
1043 bool last_render_target
,
1044 bool header_present
);
1046 brw_inst
*gen9_fb_READ(struct brw_codegen
*p
,
1048 struct brw_reg payload
,
1049 unsigned binding_table_index
,
1050 unsigned msg_length
,
1051 unsigned response_length
,
1054 void brw_SAMPLE(struct brw_codegen
*p
,
1055 struct brw_reg dest
,
1056 unsigned msg_reg_nr
,
1057 struct brw_reg src0
,
1058 unsigned binding_table_index
,
1061 unsigned response_length
,
1062 unsigned msg_length
,
1063 unsigned header_present
,
1065 unsigned return_format
);
1067 void brw_adjust_sampler_state_pointer(struct brw_codegen
*p
,
1068 struct brw_reg header
,
1069 struct brw_reg sampler_index
);
1071 void gen4_math(struct brw_codegen
*p
,
1072 struct brw_reg dest
,
1074 unsigned msg_reg_nr
,
1076 unsigned precision
);
1078 void gen6_math(struct brw_codegen
*p
,
1079 struct brw_reg dest
,
1081 struct brw_reg src0
,
1082 struct brw_reg src1
);
1084 void brw_oword_block_read(struct brw_codegen
*p
,
1085 struct brw_reg dest
,
1088 uint32_t bind_table_index
);
1090 unsigned brw_scratch_surface_idx(const struct brw_codegen
*p
);
1092 void brw_oword_block_read_scratch(struct brw_codegen
*p
,
1093 struct brw_reg dest
,
1098 void brw_oword_block_write_scratch(struct brw_codegen
*p
,
1103 void gen7_block_read_scratch(struct brw_codegen
*p
,
1104 struct brw_reg dest
,
1108 void brw_shader_time_add(struct brw_codegen
*p
,
1109 struct brw_reg payload
,
1110 uint32_t surf_index
);
1113 * Return the generation-specific jump distance scaling factor.
1115 * Given the number of instructions to jump, we need to scale by
1116 * some number to obtain the actual jump distance to program in an
1119 static inline unsigned
1120 brw_jump_scale(const struct gen_device_info
*devinfo
)
1122 /* Broadwell measures jump targets in bytes. */
1123 if (devinfo
->gen
>= 8)
1126 /* Ironlake and later measure jump targets in 64-bit data chunks (in order
1127 * (to support compaction), so each 128-bit instruction requires 2 chunks.
1129 if (devinfo
->gen
>= 5)
1132 /* Gen4 simply uses the number of 128-bit instructions. */
1136 void brw_barrier(struct brw_codegen
*p
, struct brw_reg src
);
1138 /* If/else/endif. Works by manipulating the execution flags on each
1141 brw_inst
*brw_IF(struct brw_codegen
*p
, unsigned execute_size
);
1142 brw_inst
*gen6_IF(struct brw_codegen
*p
, enum brw_conditional_mod conditional
,
1143 struct brw_reg src0
, struct brw_reg src1
);
1145 void brw_ELSE(struct brw_codegen
*p
);
1146 void brw_ENDIF(struct brw_codegen
*p
);
1150 brw_inst
*brw_DO(struct brw_codegen
*p
, unsigned execute_size
);
1152 brw_inst
*brw_WHILE(struct brw_codegen
*p
);
1154 brw_inst
*brw_BREAK(struct brw_codegen
*p
);
1155 brw_inst
*brw_CONT(struct brw_codegen
*p
);
1156 brw_inst
*brw_HALT(struct brw_codegen
*p
);
1160 void brw_land_fwd_jump(struct brw_codegen
*p
, int jmp_insn_idx
);
1162 brw_inst
*brw_JMPI(struct brw_codegen
*p
, struct brw_reg index
,
1163 unsigned predicate_control
);
1165 void brw_NOP(struct brw_codegen
*p
);
1167 void brw_WAIT(struct brw_codegen
*p
);
1169 void brw_SYNC(struct brw_codegen
*p
, enum tgl_sync_function func
);
1171 /* Special case: there is never a destination, execution size will be
1174 void brw_CMP(struct brw_codegen
*p
,
1175 struct brw_reg dest
,
1176 unsigned conditional
,
1177 struct brw_reg src0
,
1178 struct brw_reg src1
);
1181 brw_untyped_atomic(struct brw_codegen
*p
,
1183 struct brw_reg payload
,
1184 struct brw_reg surface
,
1186 unsigned msg_length
,
1187 bool response_expected
,
1188 bool header_present
);
1191 brw_untyped_surface_read(struct brw_codegen
*p
,
1193 struct brw_reg payload
,
1194 struct brw_reg surface
,
1195 unsigned msg_length
,
1196 unsigned num_channels
);
1199 brw_untyped_surface_write(struct brw_codegen
*p
,
1200 struct brw_reg payload
,
1201 struct brw_reg surface
,
1202 unsigned msg_length
,
1203 unsigned num_channels
,
1204 bool header_present
);
1207 brw_memory_fence(struct brw_codegen
*p
,
1210 enum opcode send_op
,
1211 enum brw_message_target sfid
,
1216 brw_pixel_interpolator_query(struct brw_codegen
*p
,
1217 struct brw_reg dest
,
1221 struct brw_reg data
,
1222 unsigned msg_length
,
1223 unsigned response_length
);
1226 brw_find_live_channel(struct brw_codegen
*p
,
1228 struct brw_reg mask
);
1231 brw_broadcast(struct brw_codegen
*p
,
1234 struct brw_reg idx
);
1237 brw_float_controls_mode(struct brw_codegen
*p
,
1238 unsigned mode
, unsigned mask
);
1240 /***********************************************************************
1244 void brw_copy_indirect_to_indirect(struct brw_codegen
*p
,
1245 struct brw_indirect dst_ptr
,
1246 struct brw_indirect src_ptr
,
1249 void brw_copy_from_indirect(struct brw_codegen
*p
,
1251 struct brw_indirect ptr
,
1254 void brw_copy4(struct brw_codegen
*p
,
1259 void brw_copy8(struct brw_codegen
*p
,
1264 void brw_math_invert( struct brw_codegen
*p
,
1266 struct brw_reg src
);
1268 void brw_set_src1(struct brw_codegen
*p
, brw_inst
*insn
, struct brw_reg reg
);
1270 void brw_set_desc_ex(struct brw_codegen
*p
, brw_inst
*insn
,
1271 unsigned desc
, unsigned ex_desc
);
1274 brw_set_desc(struct brw_codegen
*p
, brw_inst
*insn
, unsigned desc
)
1276 brw_set_desc_ex(p
, insn
, desc
, 0);
1279 void brw_set_uip_jip(struct brw_codegen
*p
, int start_offset
);
1281 enum brw_conditional_mod
brw_negate_cmod(enum brw_conditional_mod cmod
);
1282 enum brw_conditional_mod
brw_swap_cmod(enum brw_conditional_mod cmod
);
1284 /* brw_eu_compact.c */
1285 void brw_compact_instructions(struct brw_codegen
*p
, int start_offset
,
1286 struct disasm_info
*disasm
);
1287 void brw_uncompact_instruction(const struct gen_device_info
*devinfo
,
1288 brw_inst
*dst
, brw_compact_inst
*src
);
1289 bool brw_try_compact_instruction(const struct gen_device_info
*devinfo
,
1290 brw_compact_inst
*dst
, const brw_inst
*src
);
1292 void brw_debug_compact_uncompact(const struct gen_device_info
*devinfo
,
1293 brw_inst
*orig
, brw_inst
*uncompacted
);
1295 /* brw_eu_validate.c */
1296 bool brw_validate_instruction(const struct gen_device_info
*devinfo
,
1297 const brw_inst
*inst
, int offset
,
1298 struct disasm_info
*disasm
);
1299 bool brw_validate_instructions(const struct gen_device_info
*devinfo
,
1300 const void *assembly
, int start_offset
, int end_offset
,
1301 struct disasm_info
*disasm
);
1304 next_offset(const struct gen_device_info
*devinfo
, void *store
, int offset
)
1306 brw_inst
*insn
= (brw_inst
*)((char *)store
+ offset
);
1308 if (brw_inst_cmpt_control(devinfo
, insn
))
1314 struct opcode_desc
{
1323 const struct opcode_desc
*
1324 brw_opcode_desc(const struct gen_device_info
*devinfo
, enum opcode opcode
);
1326 const struct opcode_desc
*
1327 brw_opcode_desc_from_hw(const struct gen_device_info
*devinfo
, unsigned hw
);
1329 static inline unsigned
1330 brw_opcode_encode(const struct gen_device_info
*devinfo
, enum opcode opcode
)
1332 return brw_opcode_desc(devinfo
, opcode
)->hw
;
1335 static inline enum opcode
1336 brw_opcode_decode(const struct gen_device_info
*devinfo
, unsigned hw
)
1338 const struct opcode_desc
*desc
= brw_opcode_desc_from_hw(devinfo
, hw
);
1339 return desc
? (enum opcode
)desc
->ir
: BRW_OPCODE_ILLEGAL
;
1343 brw_inst_set_opcode(const struct gen_device_info
*devinfo
,
1344 brw_inst
*inst
, enum opcode opcode
)
1346 brw_inst_set_hw_opcode(devinfo
, inst
, brw_opcode_encode(devinfo
, opcode
));
1349 static inline enum opcode
1350 brw_inst_opcode(const struct gen_device_info
*devinfo
, const brw_inst
*inst
)
1352 return brw_opcode_decode(devinfo
, brw_inst_hw_opcode(devinfo
, inst
));
1356 is_3src(const struct gen_device_info
*devinfo
, enum opcode opcode
)
1358 const struct opcode_desc
*desc
= brw_opcode_desc(devinfo
, opcode
);
1359 return desc
&& desc
->nsrc
== 3;
1362 /** Maximum SEND message length */
1363 #define BRW_MAX_MSG_LENGTH 15
1365 /** First MRF register used by pull loads */
1366 #define FIRST_SPILL_MRF(gen) ((gen) == 6 ? 21 : 13)
1368 /** First MRF register used by spills */
1369 #define FIRST_PULL_LOAD_MRF(gen) ((gen) == 6 ? 16 : 13)