2 Copyright (C) Intel Corp. 2006. All Rights Reserved.
3 Intel funded Tungsten Graphics to
4 develop this 3D driver.
6 Permission is hereby granted, free of charge, to any person obtaining
7 a copy of this software and associated documentation files (the
8 "Software"), to deal in the Software without restriction, including
9 without limitation the rights to use, copy, modify, merge, publish,
10 distribute, sublicense, and/or sell copies of the Software, and to
11 permit persons to whom the Software is furnished to do so, subject to
12 the following conditions:
14 The above copyright notice and this permission notice (including the
15 next paragraph) shall be included in all copies or substantial
16 portions of the Software.
18 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
19 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
21 IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
22 LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
23 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
24 WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **********************************************************************/
29 * Keith Whitwell <keithw@vmware.com>
39 #include "brw_compiler.h"
40 #include "brw_eu_defines.h"
42 #include "brw_disasm_info.h"
48 #define BRW_EU_MAX_INSN_STACK 5
50 struct brw_insn_state
{
51 /* One of BRW_EXECUTE_* */
54 /* Group in units of channels */
57 /* Compression control on gen4-5 */
60 /* One of BRW_MASK_* */
61 unsigned mask_control
:1;
63 /* Scheduling info for Gen12+ */
68 /* One of BRW_ALIGN_* */
69 unsigned access_mode
:1;
71 /* One of BRW_PREDICATE_* */
72 enum brw_predicate predicate
:4;
76 /* Flag subreg. Bottom bit is subreg, top bit is reg */
77 unsigned flag_subreg
:2;
79 bool acc_wr_control
:1;
83 /* A helper for accessing the last instruction emitted. This makes it easy
84 * to set various bits on an instruction without having to create temporary
85 * variable and assign the emitted instruction to those.
87 #define brw_last_inst (&p->store[p->nr_insn - 1])
93 unsigned int next_insn_offset
;
97 /* Allow clients to push/pop instruction state:
99 struct brw_insn_state stack
[BRW_EU_MAX_INSN_STACK
];
100 struct brw_insn_state
*current
;
102 /** Whether or not the user wants automatic exec sizes
104 * If true, codegen will try to automatically infer the exec size of an
105 * instruction from the width of the destination register. If false, it
106 * will take whatever is set by brw_set_default_exec_size verbatim.
108 * This is set to true by default in brw_init_codegen.
110 bool automatic_exec_sizes
;
112 bool single_program_flow
;
113 const struct gen_device_info
*devinfo
;
115 /* Control flow stacks:
116 * - if_stack contains IF and ELSE instructions which must be patched
117 * (and popped) once the matching ENDIF instruction is encountered.
119 * Just store the instruction pointer(an index).
123 int if_stack_array_size
;
126 * loop_stack contains the instruction pointers of the starts of loops which
127 * must be patched (and popped) once the matching WHILE instruction is
132 * pre-gen6, the BREAK and CONT instructions had to tell how many IF/ENDIF
133 * blocks they were popping out of, to fix up the mask stack. This tracks
134 * the IF/ENDIF nesting in each current nested loop level.
136 int *if_depth_in_loop
;
137 int loop_stack_depth
;
138 int loop_stack_array_size
;
140 struct brw_shader_reloc
*relocs
;
142 int reloc_array_size
;
148 struct brw_label
*next
;
151 void brw_pop_insn_state( struct brw_codegen
*p
);
152 void brw_push_insn_state( struct brw_codegen
*p
);
153 unsigned brw_get_default_exec_size(struct brw_codegen
*p
);
154 unsigned brw_get_default_group(struct brw_codegen
*p
);
155 unsigned brw_get_default_access_mode(struct brw_codegen
*p
);
156 struct tgl_swsb
brw_get_default_swsb(struct brw_codegen
*p
);
157 void brw_set_default_exec_size(struct brw_codegen
*p
, unsigned value
);
158 void brw_set_default_mask_control( struct brw_codegen
*p
, unsigned value
);
159 void brw_set_default_saturate( struct brw_codegen
*p
, bool enable
);
160 void brw_set_default_access_mode( struct brw_codegen
*p
, unsigned access_mode
);
161 void brw_inst_set_compression(const struct gen_device_info
*devinfo
,
162 brw_inst
*inst
, bool on
);
163 void brw_set_default_compression(struct brw_codegen
*p
, bool on
);
164 void brw_inst_set_group(const struct gen_device_info
*devinfo
,
165 brw_inst
*inst
, unsigned group
);
166 void brw_set_default_group(struct brw_codegen
*p
, unsigned group
);
167 void brw_set_default_compression_control(struct brw_codegen
*p
, enum brw_compression c
);
168 void brw_set_default_predicate_control(struct brw_codegen
*p
, enum brw_predicate pc
);
169 void brw_set_default_predicate_inverse(struct brw_codegen
*p
, bool predicate_inverse
);
170 void brw_set_default_flag_reg(struct brw_codegen
*p
, int reg
, int subreg
);
171 void brw_set_default_acc_write_control(struct brw_codegen
*p
, unsigned value
);
172 void brw_set_default_swsb(struct brw_codegen
*p
, struct tgl_swsb value
);
174 void brw_init_codegen(const struct gen_device_info
*, struct brw_codegen
*p
,
176 bool brw_has_jip(const struct gen_device_info
*devinfo
, enum opcode opcode
);
177 bool brw_has_uip(const struct gen_device_info
*devinfo
, enum opcode opcode
);
178 const struct brw_label
*brw_find_label(const struct brw_label
*root
, int offset
);
179 void brw_create_label(struct brw_label
**labels
, int offset
, void *mem_ctx
);
180 int brw_disassemble_inst(FILE *file
, const struct gen_device_info
*devinfo
,
181 const struct brw_inst
*inst
, bool is_compacted
,
182 int offset
, const struct brw_label
*root_label
);
183 const struct brw_label
*brw_label_assembly(const struct gen_device_info
*devinfo
,
184 const void *assembly
, int start
, int end
,
186 void brw_disassemble_with_labels(const struct gen_device_info
*devinfo
,
187 const void *assembly
, int start
, int end
, FILE *out
);
188 void brw_disassemble(const struct gen_device_info
*devinfo
,
189 const void *assembly
, int start
, int end
,
190 const struct brw_label
*root_label
, FILE *out
);
191 const struct brw_shader_reloc
*brw_get_shader_relocs(struct brw_codegen
*p
,
192 unsigned *num_relocs
);
193 const unsigned *brw_get_program( struct brw_codegen
*p
, unsigned *sz
);
195 bool brw_try_override_assembly(struct brw_codegen
*p
, int start_offset
,
196 const char *identifier
);
198 void brw_realign(struct brw_codegen
*p
, unsigned align
);
199 int brw_append_data(struct brw_codegen
*p
, void *data
,
200 unsigned size
, unsigned align
);
201 brw_inst
*brw_next_insn(struct brw_codegen
*p
, unsigned opcode
);
202 void brw_set_dest(struct brw_codegen
*p
, brw_inst
*insn
, struct brw_reg dest
);
203 void brw_set_src0(struct brw_codegen
*p
, brw_inst
*insn
, struct brw_reg reg
);
205 void gen6_resolve_implied_move(struct brw_codegen
*p
,
207 unsigned msg_reg_nr
);
209 /* Helpers for regular instructions:
212 brw_inst *brw_##OP(struct brw_codegen *p, \
213 struct brw_reg dest, \
214 struct brw_reg src0);
217 brw_inst *brw_##OP(struct brw_codegen *p, \
218 struct brw_reg dest, \
219 struct brw_reg src0, \
220 struct brw_reg src1);
223 brw_inst *brw_##OP(struct brw_codegen *p, \
224 struct brw_reg dest, \
225 struct brw_reg src0, \
226 struct brw_reg src1, \
227 struct brw_reg src2);
279 /* Helpers for SEND instruction:
283 * Construct a message descriptor immediate with the specified common
284 * descriptor controls.
286 static inline uint32_t
287 brw_message_desc(const struct gen_device_info
*devinfo
,
289 unsigned response_length
,
292 if (devinfo
->gen
>= 5) {
293 return (SET_BITS(msg_length
, 28, 25) |
294 SET_BITS(response_length
, 24, 20) |
295 SET_BITS(header_present
, 19, 19));
297 return (SET_BITS(msg_length
, 23, 20) |
298 SET_BITS(response_length
, 19, 16));
302 static inline unsigned
303 brw_message_desc_mlen(const struct gen_device_info
*devinfo
, uint32_t desc
)
305 if (devinfo
->gen
>= 5)
306 return GET_BITS(desc
, 28, 25);
308 return GET_BITS(desc
, 23, 20);
311 static inline unsigned
312 brw_message_desc_rlen(const struct gen_device_info
*devinfo
, uint32_t desc
)
314 if (devinfo
->gen
>= 5)
315 return GET_BITS(desc
, 24, 20);
317 return GET_BITS(desc
, 19, 16);
321 brw_message_desc_header_present(ASSERTED
const struct gen_device_info
*devinfo
,
324 assert(devinfo
->gen
>= 5);
325 return GET_BITS(desc
, 19, 19);
328 static inline unsigned
329 brw_message_ex_desc(UNUSED
const struct gen_device_info
*devinfo
,
330 unsigned ex_msg_length
)
332 return SET_BITS(ex_msg_length
, 9, 6);
335 static inline unsigned
336 brw_message_ex_desc_ex_mlen(UNUSED
const struct gen_device_info
*devinfo
,
339 return GET_BITS(ex_desc
, 9, 6);
342 static inline uint32_t
343 brw_urb_desc(const struct gen_device_info
*devinfo
,
345 bool per_slot_offset_present
,
346 bool channel_mask_present
,
347 unsigned global_offset
)
349 if (devinfo
->gen
>= 8) {
350 return (SET_BITS(per_slot_offset_present
, 17, 17) |
351 SET_BITS(channel_mask_present
, 15, 15) |
352 SET_BITS(global_offset
, 14, 4) |
353 SET_BITS(msg_type
, 3, 0));
354 } else if (devinfo
->gen
>= 7) {
355 assert(!channel_mask_present
);
356 return (SET_BITS(per_slot_offset_present
, 16, 16) |
357 SET_BITS(global_offset
, 13, 3) |
358 SET_BITS(msg_type
, 3, 0));
360 unreachable("unhandled URB write generation");
364 static inline uint32_t
365 brw_urb_desc_msg_type(ASSERTED
const struct gen_device_info
*devinfo
,
368 assert(devinfo
->gen
>= 7);
369 return GET_BITS(desc
, 3, 0);
373 * Construct a message descriptor immediate with the specified sampler
376 static inline uint32_t
377 brw_sampler_desc(const struct gen_device_info
*devinfo
,
378 unsigned binding_table_index
,
382 unsigned return_format
)
384 const unsigned desc
= (SET_BITS(binding_table_index
, 7, 0) |
385 SET_BITS(sampler
, 11, 8));
386 if (devinfo
->gen
>= 7)
387 return (desc
| SET_BITS(msg_type
, 16, 12) |
388 SET_BITS(simd_mode
, 18, 17));
389 else if (devinfo
->gen
>= 5)
390 return (desc
| SET_BITS(msg_type
, 15, 12) |
391 SET_BITS(simd_mode
, 17, 16));
392 else if (devinfo
->is_g4x
)
393 return desc
| SET_BITS(msg_type
, 15, 12);
395 return (desc
| SET_BITS(return_format
, 13, 12) |
396 SET_BITS(msg_type
, 15, 14));
399 static inline unsigned
400 brw_sampler_desc_binding_table_index(UNUSED
const struct gen_device_info
*devinfo
,
403 return GET_BITS(desc
, 7, 0);
406 static inline unsigned
407 brw_sampler_desc_sampler(UNUSED
const struct gen_device_info
*devinfo
, uint32_t desc
)
409 return GET_BITS(desc
, 11, 8);
412 static inline unsigned
413 brw_sampler_desc_msg_type(const struct gen_device_info
*devinfo
, uint32_t desc
)
415 if (devinfo
->gen
>= 7)
416 return GET_BITS(desc
, 16, 12);
417 else if (devinfo
->gen
>= 5 || devinfo
->is_g4x
)
418 return GET_BITS(desc
, 15, 12);
420 return GET_BITS(desc
, 15, 14);
423 static inline unsigned
424 brw_sampler_desc_simd_mode(const struct gen_device_info
*devinfo
, uint32_t desc
)
426 assert(devinfo
->gen
>= 5);
427 if (devinfo
->gen
>= 7)
428 return GET_BITS(desc
, 18, 17);
430 return GET_BITS(desc
, 17, 16);
433 static inline unsigned
434 brw_sampler_desc_return_format(ASSERTED
const struct gen_device_info
*devinfo
,
437 assert(devinfo
->gen
== 4 && !devinfo
->is_g4x
);
438 return GET_BITS(desc
, 13, 12);
442 * Construct a message descriptor for the dataport
444 static inline uint32_t
445 brw_dp_desc(const struct gen_device_info
*devinfo
,
446 unsigned binding_table_index
,
448 unsigned msg_control
)
450 /* Prior to gen6, things are too inconsistent; use the dp_read/write_desc
453 assert(devinfo
->gen
>= 6);
454 const unsigned desc
= SET_BITS(binding_table_index
, 7, 0);
455 if (devinfo
->gen
>= 8) {
456 return (desc
| SET_BITS(msg_control
, 13, 8) |
457 SET_BITS(msg_type
, 18, 14));
458 } else if (devinfo
->gen
>= 7) {
459 return (desc
| SET_BITS(msg_control
, 13, 8) |
460 SET_BITS(msg_type
, 17, 14));
462 return (desc
| SET_BITS(msg_control
, 12, 8) |
463 SET_BITS(msg_type
, 16, 13));
467 static inline unsigned
468 brw_dp_desc_binding_table_index(UNUSED
const struct gen_device_info
*devinfo
,
471 return GET_BITS(desc
, 7, 0);
474 static inline unsigned
475 brw_dp_desc_msg_type(const struct gen_device_info
*devinfo
, uint32_t desc
)
477 assert(devinfo
->gen
>= 6);
478 if (devinfo
->gen
>= 8)
479 return GET_BITS(desc
, 18, 14);
480 else if (devinfo
->gen
>= 7)
481 return GET_BITS(desc
, 17, 14);
483 return GET_BITS(desc
, 16, 13);
486 static inline unsigned
487 brw_dp_desc_msg_control(const struct gen_device_info
*devinfo
, uint32_t desc
)
489 assert(devinfo
->gen
>= 6);
490 if (devinfo
->gen
>= 7)
491 return GET_BITS(desc
, 13, 8);
493 return GET_BITS(desc
, 12, 8);
497 * Construct a message descriptor immediate with the specified dataport read
500 static inline uint32_t
501 brw_dp_read_desc(const struct gen_device_info
*devinfo
,
502 unsigned binding_table_index
,
503 unsigned msg_control
,
505 unsigned target_cache
)
507 if (devinfo
->gen
>= 6)
508 return brw_dp_desc(devinfo
, binding_table_index
, msg_type
, msg_control
);
509 else if (devinfo
->gen
>= 5 || devinfo
->is_g4x
)
510 return (SET_BITS(binding_table_index
, 7, 0) |
511 SET_BITS(msg_control
, 10, 8) |
512 SET_BITS(msg_type
, 13, 11) |
513 SET_BITS(target_cache
, 15, 14));
515 return (SET_BITS(binding_table_index
, 7, 0) |
516 SET_BITS(msg_control
, 11, 8) |
517 SET_BITS(msg_type
, 13, 12) |
518 SET_BITS(target_cache
, 15, 14));
521 static inline unsigned
522 brw_dp_read_desc_msg_type(const struct gen_device_info
*devinfo
, uint32_t desc
)
524 if (devinfo
->gen
>= 6)
525 return brw_dp_desc_msg_type(devinfo
, desc
);
526 else if (devinfo
->gen
>= 5 || devinfo
->is_g4x
)
527 return GET_BITS(desc
, 13, 11);
529 return GET_BITS(desc
, 13, 12);
532 static inline unsigned
533 brw_dp_read_desc_msg_control(const struct gen_device_info
*devinfo
,
536 if (devinfo
->gen
>= 6)
537 return brw_dp_desc_msg_control(devinfo
, desc
);
538 else if (devinfo
->gen
>= 5 || devinfo
->is_g4x
)
539 return GET_BITS(desc
, 10, 8);
541 return GET_BITS(desc
, 11, 8);
545 * Construct a message descriptor immediate with the specified dataport write
548 static inline uint32_t
549 brw_dp_write_desc(const struct gen_device_info
*devinfo
,
550 unsigned binding_table_index
,
551 unsigned msg_control
,
553 unsigned last_render_target
,
554 unsigned send_commit_msg
)
556 assert(devinfo
->gen
<= 6 || !send_commit_msg
);
557 if (devinfo
->gen
>= 6)
558 return brw_dp_desc(devinfo
, binding_table_index
, msg_type
, msg_control
) |
559 SET_BITS(last_render_target
, 12, 12) |
560 SET_BITS(send_commit_msg
, 17, 17);
562 return (SET_BITS(binding_table_index
, 7, 0) |
563 SET_BITS(msg_control
, 11, 8) |
564 SET_BITS(last_render_target
, 11, 11) |
565 SET_BITS(msg_type
, 14, 12) |
566 SET_BITS(send_commit_msg
, 15, 15));
569 static inline unsigned
570 brw_dp_write_desc_msg_type(const struct gen_device_info
*devinfo
,
573 if (devinfo
->gen
>= 6)
574 return brw_dp_desc_msg_type(devinfo
, desc
);
576 return GET_BITS(desc
, 14, 12);
579 static inline unsigned
580 brw_dp_write_desc_msg_control(const struct gen_device_info
*devinfo
,
583 if (devinfo
->gen
>= 6)
584 return brw_dp_desc_msg_control(devinfo
, desc
);
586 return GET_BITS(desc
, 11, 8);
590 brw_dp_write_desc_last_render_target(const struct gen_device_info
*devinfo
,
593 if (devinfo
->gen
>= 6)
594 return GET_BITS(desc
, 12, 12);
596 return GET_BITS(desc
, 11, 11);
600 brw_dp_write_desc_write_commit(const struct gen_device_info
*devinfo
,
603 assert(devinfo
->gen
<= 6);
604 if (devinfo
->gen
>= 6)
605 return GET_BITS(desc
, 17, 17);
607 return GET_BITS(desc
, 15, 15);
611 * Construct a message descriptor immediate with the specified dataport
612 * surface function controls.
614 static inline uint32_t
615 brw_dp_surface_desc(const struct gen_device_info
*devinfo
,
617 unsigned msg_control
)
619 assert(devinfo
->gen
>= 7);
620 /* We'll OR in the binding table index later */
621 return brw_dp_desc(devinfo
, 0, msg_type
, msg_control
);
624 static inline uint32_t
625 brw_dp_untyped_atomic_desc(const struct gen_device_info
*devinfo
,
626 unsigned exec_size
, /**< 0 for SIMD4x2 */
628 bool response_expected
)
630 assert(exec_size
<= 8 || exec_size
== 16);
633 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
) {
635 msg_type
= HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP
;
637 msg_type
= HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP_SIMD4X2
;
640 msg_type
= GEN7_DATAPORT_DC_UNTYPED_ATOMIC_OP
;
643 const unsigned msg_control
=
644 SET_BITS(atomic_op
, 3, 0) |
645 SET_BITS(0 < exec_size
&& exec_size
<= 8, 4, 4) |
646 SET_BITS(response_expected
, 5, 5);
648 return brw_dp_surface_desc(devinfo
, msg_type
, msg_control
);
651 static inline uint32_t
652 brw_dp_untyped_atomic_float_desc(const struct gen_device_info
*devinfo
,
655 bool response_expected
)
657 assert(exec_size
<= 8 || exec_size
== 16);
658 assert(devinfo
->gen
>= 9);
660 assert(exec_size
> 0);
661 const unsigned msg_type
= GEN9_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_FLOAT_OP
;
663 const unsigned msg_control
=
664 SET_BITS(atomic_op
, 1, 0) |
665 SET_BITS(exec_size
<= 8, 4, 4) |
666 SET_BITS(response_expected
, 5, 5);
668 return brw_dp_surface_desc(devinfo
, msg_type
, msg_control
);
671 static inline unsigned
672 brw_mdc_cmask(unsigned num_channels
)
674 /* See also MDC_CMASK in the SKL PRM Vol 2d. */
675 return 0xf & (0xf << num_channels
);
678 static inline uint32_t
679 brw_dp_untyped_surface_rw_desc(const struct gen_device_info
*devinfo
,
680 unsigned exec_size
, /**< 0 for SIMD4x2 */
681 unsigned num_channels
,
684 assert(exec_size
<= 8 || exec_size
== 16);
688 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
) {
689 msg_type
= HSW_DATAPORT_DC_PORT1_UNTYPED_SURFACE_WRITE
;
691 msg_type
= GEN7_DATAPORT_DC_UNTYPED_SURFACE_WRITE
;
695 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
) {
696 msg_type
= HSW_DATAPORT_DC_PORT1_UNTYPED_SURFACE_READ
;
698 msg_type
= GEN7_DATAPORT_DC_UNTYPED_SURFACE_READ
;
702 /* SIMD4x2 is only valid for read messages on IVB; use SIMD8 instead */
703 if (write
&& devinfo
->gen
== 7 && !devinfo
->is_haswell
&& exec_size
== 0)
706 /* See also MDC_SM3 in the SKL PRM Vol 2d. */
707 const unsigned simd_mode
= exec_size
== 0 ? 0 : /* SIMD4x2 */
708 exec_size
<= 8 ? 2 : 1;
710 const unsigned msg_control
=
711 SET_BITS(brw_mdc_cmask(num_channels
), 3, 0) |
712 SET_BITS(simd_mode
, 5, 4);
714 return brw_dp_surface_desc(devinfo
, msg_type
, msg_control
);
717 static inline unsigned
718 brw_mdc_ds(unsigned bit_size
)
722 return GEN7_BYTE_SCATTERED_DATA_ELEMENT_BYTE
;
724 return GEN7_BYTE_SCATTERED_DATA_ELEMENT_WORD
;
726 return GEN7_BYTE_SCATTERED_DATA_ELEMENT_DWORD
;
728 unreachable("Unsupported bit_size for byte scattered messages");
732 static inline uint32_t
733 brw_dp_byte_scattered_rw_desc(const struct gen_device_info
*devinfo
,
738 assert(exec_size
<= 8 || exec_size
== 16);
740 assert(devinfo
->gen
> 7 || devinfo
->is_haswell
);
741 const unsigned msg_type
=
742 write
? HSW_DATAPORT_DC_PORT0_BYTE_SCATTERED_WRITE
:
743 HSW_DATAPORT_DC_PORT0_BYTE_SCATTERED_READ
;
745 assert(exec_size
> 0);
746 const unsigned msg_control
=
747 SET_BITS(exec_size
== 16, 0, 0) |
748 SET_BITS(brw_mdc_ds(bit_size
), 3, 2);
750 return brw_dp_surface_desc(devinfo
, msg_type
, msg_control
);
753 static inline uint32_t
754 brw_dp_dword_scattered_rw_desc(const struct gen_device_info
*devinfo
,
758 assert(exec_size
== 8 || exec_size
== 16);
762 if (devinfo
->gen
>= 6) {
763 msg_type
= GEN6_DATAPORT_WRITE_MESSAGE_DWORD_SCATTERED_WRITE
;
765 msg_type
= BRW_DATAPORT_WRITE_MESSAGE_DWORD_SCATTERED_WRITE
;
768 if (devinfo
->gen
>= 7) {
769 msg_type
= GEN7_DATAPORT_DC_DWORD_SCATTERED_READ
;
770 } else if (devinfo
->gen
> 4 || devinfo
->is_g4x
) {
771 msg_type
= G45_DATAPORT_READ_MESSAGE_DWORD_SCATTERED_READ
;
773 msg_type
= BRW_DATAPORT_READ_MESSAGE_DWORD_SCATTERED_READ
;
777 const unsigned msg_control
=
778 SET_BITS(1, 1, 1) | /* Legacy SIMD Mode */
779 SET_BITS(exec_size
== 16, 0, 0);
781 return brw_dp_surface_desc(devinfo
, msg_type
, msg_control
);
784 static inline uint32_t
785 brw_dp_a64_untyped_surface_rw_desc(const struct gen_device_info
*devinfo
,
786 unsigned exec_size
, /**< 0 for SIMD4x2 */
787 unsigned num_channels
,
790 assert(exec_size
<= 8 || exec_size
== 16);
791 assert(devinfo
->gen
>= 8);
794 write
? GEN8_DATAPORT_DC_PORT1_A64_UNTYPED_SURFACE_WRITE
:
795 GEN8_DATAPORT_DC_PORT1_A64_UNTYPED_SURFACE_READ
;
797 /* See also MDC_SM3 in the SKL PRM Vol 2d. */
798 const unsigned simd_mode
= exec_size
== 0 ? 0 : /* SIMD4x2 */
799 exec_size
<= 8 ? 2 : 1;
801 const unsigned msg_control
=
802 SET_BITS(brw_mdc_cmask(num_channels
), 3, 0) |
803 SET_BITS(simd_mode
, 5, 4);
805 return brw_dp_desc(devinfo
, GEN8_BTI_STATELESS_NON_COHERENT
,
806 msg_type
, msg_control
);
810 * Calculate the data size (see MDC_A64_DS in the "Structures" volume of the
813 static inline uint32_t
814 brw_mdc_a64_ds(unsigned elems
)
822 unreachable("Unsupported elmeent count for A64 scattered message");
826 static inline uint32_t
827 brw_dp_a64_byte_scattered_rw_desc(const struct gen_device_info
*devinfo
,
828 unsigned exec_size
, /**< 0 for SIMD4x2 */
832 assert(exec_size
<= 8 || exec_size
== 16);
833 assert(devinfo
->gen
>= 8);
836 write
? GEN8_DATAPORT_DC_PORT1_A64_SCATTERED_WRITE
:
837 GEN9_DATAPORT_DC_PORT1_A64_SCATTERED_READ
;
839 const unsigned msg_control
=
840 SET_BITS(GEN8_A64_SCATTERED_SUBTYPE_BYTE
, 1, 0) |
841 SET_BITS(brw_mdc_a64_ds(bit_size
/ 8), 3, 2) |
842 SET_BITS(exec_size
== 16, 4, 4);
844 return brw_dp_desc(devinfo
, GEN8_BTI_STATELESS_NON_COHERENT
,
845 msg_type
, msg_control
);
848 static inline uint32_t
849 brw_dp_a64_untyped_atomic_desc(const struct gen_device_info
*devinfo
,
850 ASSERTED
unsigned exec_size
, /**< 0 for SIMD4x2 */
853 bool response_expected
)
855 assert(exec_size
== 8);
856 assert(devinfo
->gen
>= 8);
857 assert(bit_size
== 32 || bit_size
== 64);
859 const unsigned msg_type
= GEN8_DATAPORT_DC_PORT1_A64_UNTYPED_ATOMIC_OP
;
861 const unsigned msg_control
=
862 SET_BITS(atomic_op
, 3, 0) |
863 SET_BITS(bit_size
== 64, 4, 4) |
864 SET_BITS(response_expected
, 5, 5);
866 return brw_dp_desc(devinfo
, GEN8_BTI_STATELESS_NON_COHERENT
,
867 msg_type
, msg_control
);
870 static inline uint32_t
871 brw_dp_a64_untyped_atomic_float_desc(const struct gen_device_info
*devinfo
,
872 ASSERTED
unsigned exec_size
,
874 bool response_expected
)
876 assert(exec_size
== 8);
877 assert(devinfo
->gen
>= 9);
879 assert(exec_size
> 0);
880 const unsigned msg_type
= GEN9_DATAPORT_DC_PORT1_A64_UNTYPED_ATOMIC_FLOAT_OP
;
882 const unsigned msg_control
=
883 SET_BITS(atomic_op
, 1, 0) |
884 SET_BITS(response_expected
, 5, 5);
886 return brw_dp_desc(devinfo
, GEN8_BTI_STATELESS_NON_COHERENT
,
887 msg_type
, msg_control
);
890 static inline uint32_t
891 brw_dp_typed_atomic_desc(const struct gen_device_info
*devinfo
,
895 bool response_expected
)
897 assert(exec_size
> 0 || exec_group
== 0);
898 assert(exec_group
% 8 == 0);
901 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
) {
902 if (exec_size
== 0) {
903 msg_type
= HSW_DATAPORT_DC_PORT1_TYPED_ATOMIC_OP_SIMD4X2
;
905 msg_type
= HSW_DATAPORT_DC_PORT1_TYPED_ATOMIC_OP
;
908 /* SIMD4x2 typed surface R/W messages only exist on HSW+ */
909 assert(exec_size
> 0);
910 msg_type
= GEN7_DATAPORT_RC_TYPED_ATOMIC_OP
;
913 const bool high_sample_mask
= (exec_group
/ 8) % 2 == 1;
915 const unsigned msg_control
=
916 SET_BITS(atomic_op
, 3, 0) |
917 SET_BITS(high_sample_mask
, 4, 4) |
918 SET_BITS(response_expected
, 5, 5);
920 return brw_dp_surface_desc(devinfo
, msg_type
, msg_control
);
923 static inline uint32_t
924 brw_dp_typed_surface_rw_desc(const struct gen_device_info
*devinfo
,
927 unsigned num_channels
,
930 assert(exec_size
> 0 || exec_group
== 0);
931 assert(exec_group
% 8 == 0);
933 /* Typed surface reads and writes don't support SIMD16 */
934 assert(exec_size
<= 8);
938 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
) {
939 msg_type
= HSW_DATAPORT_DC_PORT1_TYPED_SURFACE_WRITE
;
941 msg_type
= GEN7_DATAPORT_RC_TYPED_SURFACE_WRITE
;
944 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
) {
945 msg_type
= HSW_DATAPORT_DC_PORT1_TYPED_SURFACE_READ
;
947 msg_type
= GEN7_DATAPORT_RC_TYPED_SURFACE_READ
;
951 /* See also MDC_SG3 in the SKL PRM Vol 2d. */
952 unsigned msg_control
;
953 if (devinfo
->gen
>= 8 || devinfo
->is_haswell
) {
954 /* See also MDC_SG3 in the SKL PRM Vol 2d. */
955 const unsigned slot_group
= exec_size
== 0 ? 0 : /* SIMD4x2 */
956 1 + ((exec_group
/ 8) % 2);
959 SET_BITS(brw_mdc_cmask(num_channels
), 3, 0) |
960 SET_BITS(slot_group
, 5, 4);
962 /* SIMD4x2 typed surface R/W messages only exist on HSW+ */
963 assert(exec_size
> 0);
964 const unsigned slot_group
= ((exec_group
/ 8) % 2);
967 SET_BITS(brw_mdc_cmask(num_channels
), 3, 0) |
968 SET_BITS(slot_group
, 5, 5);
971 return brw_dp_surface_desc(devinfo
, msg_type
, msg_control
);
975 * Construct a message descriptor immediate with the specified pixel
976 * interpolator function controls.
978 static inline uint32_t
979 brw_pixel_interp_desc(UNUSED
const struct gen_device_info
*devinfo
,
985 return (SET_BITS(slot_group
, 11, 11) |
986 SET_BITS(msg_type
, 13, 12) |
987 SET_BITS(!!noperspective
, 14, 14) |
988 SET_BITS(simd_mode
, 16, 16));
991 void brw_urb_WRITE(struct brw_codegen
*p
,
995 enum brw_urb_write_flags flags
,
997 unsigned response_length
,
1002 * Send message to shared unit \p sfid with a possibly indirect descriptor \p
1003 * desc. If \p desc is not an immediate it will be transparently loaded to an
1004 * address register using an OR instruction.
1007 brw_send_indirect_message(struct brw_codegen
*p
,
1010 struct brw_reg payload
,
1011 struct brw_reg desc
,
1016 brw_send_indirect_split_message(struct brw_codegen
*p
,
1019 struct brw_reg payload0
,
1020 struct brw_reg payload1
,
1021 struct brw_reg desc
,
1023 struct brw_reg ex_desc
,
1024 unsigned ex_desc_imm
,
1027 void brw_ff_sync(struct brw_codegen
*p
,
1028 struct brw_reg dest
,
1029 unsigned msg_reg_nr
,
1030 struct brw_reg src0
,
1032 unsigned response_length
,
1035 void brw_svb_write(struct brw_codegen
*p
,
1036 struct brw_reg dest
,
1037 unsigned msg_reg_nr
,
1038 struct brw_reg src0
,
1039 unsigned binding_table_index
,
1040 bool send_commit_msg
);
1042 brw_inst
*brw_fb_WRITE(struct brw_codegen
*p
,
1043 struct brw_reg payload
,
1044 struct brw_reg implied_header
,
1045 unsigned msg_control
,
1046 unsigned binding_table_index
,
1047 unsigned msg_length
,
1048 unsigned response_length
,
1050 bool last_render_target
,
1051 bool header_present
);
1053 brw_inst
*gen9_fb_READ(struct brw_codegen
*p
,
1055 struct brw_reg payload
,
1056 unsigned binding_table_index
,
1057 unsigned msg_length
,
1058 unsigned response_length
,
1061 void brw_SAMPLE(struct brw_codegen
*p
,
1062 struct brw_reg dest
,
1063 unsigned msg_reg_nr
,
1064 struct brw_reg src0
,
1065 unsigned binding_table_index
,
1068 unsigned response_length
,
1069 unsigned msg_length
,
1070 unsigned header_present
,
1072 unsigned return_format
);
1074 void brw_adjust_sampler_state_pointer(struct brw_codegen
*p
,
1075 struct brw_reg header
,
1076 struct brw_reg sampler_index
);
1078 void gen4_math(struct brw_codegen
*p
,
1079 struct brw_reg dest
,
1081 unsigned msg_reg_nr
,
1083 unsigned precision
);
1085 void gen6_math(struct brw_codegen
*p
,
1086 struct brw_reg dest
,
1088 struct brw_reg src0
,
1089 struct brw_reg src1
);
1091 void brw_oword_block_read(struct brw_codegen
*p
,
1092 struct brw_reg dest
,
1095 uint32_t bind_table_index
);
1097 unsigned brw_scratch_surface_idx(const struct brw_codegen
*p
);
1099 void brw_oword_block_read_scratch(struct brw_codegen
*p
,
1100 struct brw_reg dest
,
1105 void brw_oword_block_write_scratch(struct brw_codegen
*p
,
1110 void gen7_block_read_scratch(struct brw_codegen
*p
,
1111 struct brw_reg dest
,
1115 void brw_shader_time_add(struct brw_codegen
*p
,
1116 struct brw_reg payload
,
1117 uint32_t surf_index
);
1120 * Return the generation-specific jump distance scaling factor.
1122 * Given the number of instructions to jump, we need to scale by
1123 * some number to obtain the actual jump distance to program in an
1126 static inline unsigned
1127 brw_jump_scale(const struct gen_device_info
*devinfo
)
1129 /* Broadwell measures jump targets in bytes. */
1130 if (devinfo
->gen
>= 8)
1133 /* Ironlake and later measure jump targets in 64-bit data chunks (in order
1134 * (to support compaction), so each 128-bit instruction requires 2 chunks.
1136 if (devinfo
->gen
>= 5)
1139 /* Gen4 simply uses the number of 128-bit instructions. */
1143 void brw_barrier(struct brw_codegen
*p
, struct brw_reg src
);
1145 /* If/else/endif. Works by manipulating the execution flags on each
1148 brw_inst
*brw_IF(struct brw_codegen
*p
, unsigned execute_size
);
1149 brw_inst
*gen6_IF(struct brw_codegen
*p
, enum brw_conditional_mod conditional
,
1150 struct brw_reg src0
, struct brw_reg src1
);
1152 void brw_ELSE(struct brw_codegen
*p
);
1153 void brw_ENDIF(struct brw_codegen
*p
);
1157 brw_inst
*brw_DO(struct brw_codegen
*p
, unsigned execute_size
);
1159 brw_inst
*brw_WHILE(struct brw_codegen
*p
);
1161 brw_inst
*brw_BREAK(struct brw_codegen
*p
);
1162 brw_inst
*brw_CONT(struct brw_codegen
*p
);
1163 brw_inst
*brw_HALT(struct brw_codegen
*p
);
1167 void brw_land_fwd_jump(struct brw_codegen
*p
, int jmp_insn_idx
);
1169 brw_inst
*brw_JMPI(struct brw_codegen
*p
, struct brw_reg index
,
1170 unsigned predicate_control
);
1172 void brw_NOP(struct brw_codegen
*p
);
1174 void brw_WAIT(struct brw_codegen
*p
);
1176 void brw_SYNC(struct brw_codegen
*p
, enum tgl_sync_function func
);
1178 /* Special case: there is never a destination, execution size will be
1181 void brw_CMP(struct brw_codegen
*p
,
1182 struct brw_reg dest
,
1183 unsigned conditional
,
1184 struct brw_reg src0
,
1185 struct brw_reg src1
);
1188 brw_untyped_atomic(struct brw_codegen
*p
,
1190 struct brw_reg payload
,
1191 struct brw_reg surface
,
1193 unsigned msg_length
,
1194 bool response_expected
,
1195 bool header_present
);
1198 brw_untyped_surface_read(struct brw_codegen
*p
,
1200 struct brw_reg payload
,
1201 struct brw_reg surface
,
1202 unsigned msg_length
,
1203 unsigned num_channels
);
1206 brw_untyped_surface_write(struct brw_codegen
*p
,
1207 struct brw_reg payload
,
1208 struct brw_reg surface
,
1209 unsigned msg_length
,
1210 unsigned num_channels
,
1211 bool header_present
);
1214 brw_memory_fence(struct brw_codegen
*p
,
1217 enum opcode send_op
,
1218 enum brw_message_target sfid
,
1223 brw_pixel_interpolator_query(struct brw_codegen
*p
,
1224 struct brw_reg dest
,
1228 struct brw_reg data
,
1229 unsigned msg_length
,
1230 unsigned response_length
);
1233 brw_find_live_channel(struct brw_codegen
*p
,
1235 struct brw_reg mask
);
1238 brw_broadcast(struct brw_codegen
*p
,
1241 struct brw_reg idx
);
1244 brw_float_controls_mode(struct brw_codegen
*p
,
1245 unsigned mode
, unsigned mask
);
1248 brw_update_reloc_imm(const struct gen_device_info
*devinfo
,
1253 brw_MOV_reloc_imm(struct brw_codegen
*p
,
1255 enum brw_reg_type src_type
,
1258 /***********************************************************************
1262 void brw_copy_indirect_to_indirect(struct brw_codegen
*p
,
1263 struct brw_indirect dst_ptr
,
1264 struct brw_indirect src_ptr
,
1267 void brw_copy_from_indirect(struct brw_codegen
*p
,
1269 struct brw_indirect ptr
,
1272 void brw_copy4(struct brw_codegen
*p
,
1277 void brw_copy8(struct brw_codegen
*p
,
1282 void brw_math_invert( struct brw_codegen
*p
,
1284 struct brw_reg src
);
1286 void brw_set_src1(struct brw_codegen
*p
, brw_inst
*insn
, struct brw_reg reg
);
1288 void brw_set_desc_ex(struct brw_codegen
*p
, brw_inst
*insn
,
1289 unsigned desc
, unsigned ex_desc
);
1292 brw_set_desc(struct brw_codegen
*p
, brw_inst
*insn
, unsigned desc
)
1294 brw_set_desc_ex(p
, insn
, desc
, 0);
1297 void brw_set_uip_jip(struct brw_codegen
*p
, int start_offset
);
1299 enum brw_conditional_mod
brw_negate_cmod(enum brw_conditional_mod cmod
);
1300 enum brw_conditional_mod
brw_swap_cmod(enum brw_conditional_mod cmod
);
1302 /* brw_eu_compact.c */
1303 void brw_compact_instructions(struct brw_codegen
*p
, int start_offset
,
1304 struct disasm_info
*disasm
);
1305 void brw_uncompact_instruction(const struct gen_device_info
*devinfo
,
1306 brw_inst
*dst
, brw_compact_inst
*src
);
1307 bool brw_try_compact_instruction(const struct gen_device_info
*devinfo
,
1308 brw_compact_inst
*dst
, const brw_inst
*src
);
1310 void brw_debug_compact_uncompact(const struct gen_device_info
*devinfo
,
1311 brw_inst
*orig
, brw_inst
*uncompacted
);
1313 /* brw_eu_validate.c */
1314 bool brw_validate_instruction(const struct gen_device_info
*devinfo
,
1315 const brw_inst
*inst
, int offset
,
1316 struct disasm_info
*disasm
);
1317 bool brw_validate_instructions(const struct gen_device_info
*devinfo
,
1318 const void *assembly
, int start_offset
, int end_offset
,
1319 struct disasm_info
*disasm
);
1322 next_offset(const struct gen_device_info
*devinfo
, void *store
, int offset
)
1324 brw_inst
*insn
= (brw_inst
*)((char *)store
+ offset
);
1326 if (brw_inst_cmpt_control(devinfo
, insn
))
1332 struct opcode_desc
{
1341 const struct opcode_desc
*
1342 brw_opcode_desc(const struct gen_device_info
*devinfo
, enum opcode opcode
);
1344 const struct opcode_desc
*
1345 brw_opcode_desc_from_hw(const struct gen_device_info
*devinfo
, unsigned hw
);
1347 static inline unsigned
1348 brw_opcode_encode(const struct gen_device_info
*devinfo
, enum opcode opcode
)
1350 return brw_opcode_desc(devinfo
, opcode
)->hw
;
1353 static inline enum opcode
1354 brw_opcode_decode(const struct gen_device_info
*devinfo
, unsigned hw
)
1356 const struct opcode_desc
*desc
= brw_opcode_desc_from_hw(devinfo
, hw
);
1357 return desc
? (enum opcode
)desc
->ir
: BRW_OPCODE_ILLEGAL
;
1361 brw_inst_set_opcode(const struct gen_device_info
*devinfo
,
1362 brw_inst
*inst
, enum opcode opcode
)
1364 brw_inst_set_hw_opcode(devinfo
, inst
, brw_opcode_encode(devinfo
, opcode
));
1367 static inline enum opcode
1368 brw_inst_opcode(const struct gen_device_info
*devinfo
, const brw_inst
*inst
)
1370 return brw_opcode_decode(devinfo
, brw_inst_hw_opcode(devinfo
, inst
));
1374 is_3src(const struct gen_device_info
*devinfo
, enum opcode opcode
)
1376 const struct opcode_desc
*desc
= brw_opcode_desc(devinfo
, opcode
);
1377 return desc
&& desc
->nsrc
== 3;
1380 /** Maximum SEND message length */
1381 #define BRW_MAX_MSG_LENGTH 15
1383 /** First MRF register used by pull loads */
1384 #define FIRST_SPILL_MRF(gen) ((gen) == 6 ? 21 : 13)
1386 /** First MRF register used by spills */
1387 #define FIRST_PULL_LOAD_MRF(gen) ((gen) == 6 ? 16 : 13)