2 * Copyright © 2012 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 /** @file brw_eu_compact.c
26 * Instruction compaction is a feature of gm45 and newer hardware that allows
27 * for a smaller instruction encoding.
29 * The instruction cache is on the order of 32KB, and many programs generate
30 * far more instructions than that. The instruction cache is built to barely
31 * keep up with instruction dispatch abaility in cache hit cases -- L1
32 * instruction cache misses that still hit in the next level could limit
33 * throughput by around 50%.
35 * The idea of instruction compaction is that most instructions use a tiny
36 * subset of the GPU functionality, so we can encode what would be a 16 byte
37 * instruction in 8 bytes using some lookup tables for various fields.
40 #include "brw_context.h"
42 #include "intel_asm_annotation.h"
44 static const uint32_t gen6_control_index_table
[32] = {
79 static const uint32_t gen6_datatype_table
[32] = {
100 0b001011011000101100,
101 0b001011010110100101,
102 0b001011110110100101,
103 0b001111011110111101,
104 0b001111011110111100,
105 0b001111011110111101,
106 0b001111011110011101,
107 0b001111011110111110,
108 0b001000000000100001,
109 0b001000000000100010,
110 0b001001111111011101,
111 0b001000001110111110,
114 static const uint16_t gen6_subreg_table
[32] = {
149 static const uint16_t gen6_src_index_table
[32] = {
184 static const uint32_t gen7_control_index_table
[32] = {
185 0b0000000000000000010,
186 0b0000100000000000000,
187 0b0000100000000000001,
188 0b0000100000000000010,
189 0b0000100000000000011,
190 0b0000100000000000100,
191 0b0000100000000000101,
192 0b0000100000000000111,
193 0b0000100000000001000,
194 0b0000100000000001001,
195 0b0000100000000001101,
196 0b0000110000000000000,
197 0b0000110000000000001,
198 0b0000110000000000010,
199 0b0000110000000000011,
200 0b0000110000000000100,
201 0b0000110000000000101,
202 0b0000110000000000111,
203 0b0000110000000001001,
204 0b0000110000000001101,
205 0b0000110000000010000,
206 0b0000110000100000000,
207 0b0001000000000000000,
208 0b0001000000000000010,
209 0b0001000000000000100,
210 0b0001000000100000000,
211 0b0010110000000000000,
212 0b0010110000000010000,
213 0b0011000000000000000,
214 0b0011000000100000000,
215 0b0101000000000000000,
216 0b0101000000100000000
219 static const uint32_t gen7_datatype_table
[32] = {
220 0b001000000000000001,
221 0b001000000000100000,
222 0b001000000000100001,
223 0b001000000001100001,
224 0b001000000010111101,
225 0b001000001011111101,
226 0b001000001110100001,
227 0b001000001110100101,
228 0b001000001110111101,
229 0b001000010000100001,
230 0b001000110000100000,
231 0b001000110000100001,
232 0b001001010010100101,
233 0b001001110010100100,
234 0b001001110010100101,
235 0b001111001110111101,
236 0b001111011110011101,
237 0b001111011110111100,
238 0b001111011110111101,
239 0b001111111110111100,
240 0b000000001000001100,
241 0b001000000000111101,
242 0b001000000010100101,
243 0b001000010000100000,
244 0b001001010010100100,
245 0b001001110010000100,
246 0b001010010100001001,
247 0b001101111110111101,
248 0b001111111110111101,
249 0b001011110110101100,
250 0b001010010100101000,
254 static const uint16_t gen7_subreg_table
[32] = {
289 static const uint16_t gen7_src_index_table
[32] = {
324 static const uint32_t *control_index_table
;
325 static const uint32_t *datatype_table
;
326 static const uint16_t *subreg_table
;
327 static const uint16_t *src_index_table
;
330 set_control_index(struct brw_context
*brw
, brw_compact_inst
*dst
, brw_inst
*src
)
332 uint32_t uncompacted
= /* 17b/SNB; 19b/IVB+ */
333 (brw_inst_bits(src
, 31, 31) << 16) | /* 1b */
334 (brw_inst_bits(src
, 23, 8)); /* 16b */
336 /* On gen7, the flag register and subregister numbers are integrated into
340 uncompacted
|= brw_inst_bits(src
, 90, 89) << 17; /* 2b */
342 for (int i
= 0; i
< 32; i
++) {
343 if (control_index_table
[i
] == uncompacted
) {
344 brw_compact_inst_set_control_index(dst
, i
);
353 set_datatype_index(struct brw_context
*brw
, brw_compact_inst
*dst
,
356 uint32_t uncompacted
= /* 18b */
357 (brw_inst_bits(src
, 63, 61) << 15) | /* 3b */
358 (brw_inst_bits(src
, 46, 32)); /* 15b */
360 for (int i
= 0; i
< 32; i
++) {
361 if (datatype_table
[i
] == uncompacted
) {
362 brw_compact_inst_set_datatype_index(dst
, i
);
371 set_subreg_index(struct brw_context
*brw
, brw_compact_inst
*dst
, brw_inst
*src
,
374 uint16_t uncompacted
= /* 15b */
375 (brw_inst_bits(src
, 52, 48) << 0) | /* 5b */
376 (brw_inst_bits(src
, 68, 64) << 5); /* 5b */
379 uncompacted
|= brw_inst_bits(src
, 100, 96) << 10; /* 5b */
381 for (int i
= 0; i
< 32; i
++) {
382 if (subreg_table
[i
] == uncompacted
) {
383 brw_compact_inst_set_subreg_index(dst
, i
);
392 get_src_index(uint16_t uncompacted
,
395 for (int i
= 0; i
< 32; i
++) {
396 if (src_index_table
[i
] == uncompacted
) {
406 set_src0_index(struct brw_context
*brw
, brw_compact_inst
*dst
, brw_inst
*src
)
409 uint16_t uncompacted
= brw_inst_bits(src
, 88, 77); /* 12b */
411 if (!get_src_index(uncompacted
, &compacted
))
414 brw_compact_inst_set_src0_index(dst
, compacted
);
420 set_src1_index(struct brw_context
*brw
, brw_compact_inst
*dst
, brw_inst
*src
,
426 compacted
= (brw_inst_imm_ud(brw
, src
) >> 8) & 0x1f;
428 uint16_t uncompacted
= brw_inst_bits(src
, 120, 109); /* 12b */
430 if (!get_src_index(uncompacted
, &compacted
))
434 brw_compact_inst_set_src1_index(dst
, compacted
);
439 /* Compacted instructions have 12-bits for immediate sources, and a 13th bit
440 * that's replicated through the high 20 bits.
442 * Effectively this means we get 12-bit integers, 0.0f, and some limited uses
443 * of packed vectors as compactable immediates.
446 is_compactable_immediate(unsigned imm
)
448 /* We get the low 12 bits as-is. */
451 /* We get one bit replicated through the top 20 bits. */
452 return imm
== 0 || imm
== 0xfffff000;
456 * Tries to compact instruction src into dst.
458 * It doesn't modify dst unless src is compactable, which is relied on by
459 * brw_compact_instructions().
462 brw_try_compact_instruction(struct brw_context
*brw
, brw_compact_inst
*dst
,
465 brw_compact_inst temp
;
467 if (brw_inst_opcode(brw
, src
) == BRW_OPCODE_IF
||
468 brw_inst_opcode(brw
, src
) == BRW_OPCODE_ELSE
||
469 brw_inst_opcode(brw
, src
) == BRW_OPCODE_ENDIF
||
470 brw_inst_opcode(brw
, src
) == BRW_OPCODE_HALT
||
471 brw_inst_opcode(brw
, src
) == BRW_OPCODE_DO
||
472 brw_inst_opcode(brw
, src
) == BRW_OPCODE_WHILE
) {
473 /* FINISHME: The fixup code below, and brw_set_uip_jip and friends, needs
474 * to be able to handle compacted flow control instructions..
480 brw_inst_src0_reg_file(brw
, src
) == BRW_IMMEDIATE_VALUE
||
481 brw_inst_src1_reg_file(brw
, src
) == BRW_IMMEDIATE_VALUE
;
482 if (is_immediate
&& !is_compactable_immediate(brw_inst_imm_ud(brw
, src
))) {
486 memset(&temp
, 0, sizeof(temp
));
488 brw_compact_inst_set_opcode(&temp
, brw_inst_opcode(brw
, src
));
489 brw_compact_inst_set_debug_control(&temp
, brw_inst_debug_control(brw
, src
));
490 if (!set_control_index(brw
, &temp
, src
))
492 if (!set_datatype_index(brw
, &temp
, src
))
494 if (!set_subreg_index(brw
, &temp
, src
, is_immediate
))
496 brw_compact_inst_set_acc_wr_control(&temp
,
497 brw_inst_acc_wr_control(brw
, src
));
498 brw_compact_inst_set_cond_modifier(&temp
, brw_inst_cond_modifier(brw
, src
));
500 brw_compact_inst_set_flag_subreg_nr(&temp
,
501 brw_inst_flag_subreg_nr(brw
, src
));
502 brw_compact_inst_set_cmpt_control(&temp
, true);
503 if (!set_src0_index(brw
, &temp
, src
))
505 if (!set_src1_index(brw
, &temp
, src
, is_immediate
))
507 brw_compact_inst_set_dst_reg_nr(&temp
, brw_inst_dst_da_reg_nr(brw
, src
));
508 brw_compact_inst_set_src0_reg_nr(&temp
, brw_inst_src0_da_reg_nr(brw
, src
));
510 brw_compact_inst_set_src1_reg_nr(&temp
, brw_inst_imm_ud(brw
, src
) & 0xff);
512 brw_compact_inst_set_src1_reg_nr(&temp
,
513 brw_inst_src1_da_reg_nr(brw
, src
));
522 set_uncompacted_control(struct brw_context
*brw
, brw_inst
*dst
,
523 brw_compact_inst
*src
)
525 uint32_t uncompacted
=
526 control_index_table
[brw_compact_inst_control_index(src
)];
528 brw_inst_set_bits(dst
, 31, 31, (uncompacted
>> 16) & 0x1);
529 brw_inst_set_bits(dst
, 23, 8, (uncompacted
& 0xffff));
532 brw_inst_set_bits(dst
, 90, 89, uncompacted
>> 17);
536 set_uncompacted_datatype(struct brw_context
*brw
, brw_inst
*dst
,
537 brw_compact_inst
*src
)
539 uint32_t uncompacted
= datatype_table
[brw_compact_inst_datatype_index(src
)];
541 brw_inst_set_bits(dst
, 63, 61, (uncompacted
>> 15));
542 brw_inst_set_bits(dst
, 46, 32, (uncompacted
& 0x7fff));
546 set_uncompacted_subreg(struct brw_context
*brw
, brw_inst
*dst
,
547 brw_compact_inst
*src
)
549 uint16_t uncompacted
= subreg_table
[brw_compact_inst_subreg_index(src
)];
551 brw_inst_set_bits(dst
, 100, 96, (uncompacted
>> 10));
552 brw_inst_set_bits(dst
, 68, 64, (uncompacted
>> 5) & 0x1f);
553 brw_inst_set_bits(dst
, 52, 48, (uncompacted
>> 0) & 0x1f);
557 set_uncompacted_src0(struct brw_context
*brw
, brw_inst
*dst
,
558 brw_compact_inst
*src
)
560 uint32_t compacted
= brw_compact_inst_src0_index(src
);
561 uint16_t uncompacted
= src_index_table
[compacted
];
563 brw_inst_set_bits(dst
, 88, 77, uncompacted
);
567 set_uncompacted_src1(struct brw_context
*brw
, brw_inst
*dst
,
568 brw_compact_inst
*src
, bool is_immediate
)
571 signed high5
= brw_compact_inst_src1_index(src
);
572 /* Replicate top bit of src1_index into high 20 bits of the immediate. */
573 brw_inst_set_imm_ud(brw
, dst
, (high5
<< 27) >> 19);
575 uint16_t uncompacted
= src_index_table
[brw_compact_inst_src1_index(src
)];
577 brw_inst_set_bits(dst
, 120, 109, uncompacted
);
582 brw_uncompact_instruction(struct brw_context
*brw
, brw_inst
*dst
,
583 brw_compact_inst
*src
)
585 memset(dst
, 0, sizeof(*dst
));
587 brw_inst_set_opcode(brw
, dst
, brw_compact_inst_opcode(src
));
588 brw_inst_set_debug_control(brw
, dst
, brw_compact_inst_debug_control(src
));
590 set_uncompacted_control(brw
, dst
, src
);
591 set_uncompacted_datatype(brw
, dst
, src
);
593 /* src0/1 register file fields are in the datatype table. */
594 bool is_immediate
= brw_inst_src0_reg_file(brw
, dst
) == BRW_IMMEDIATE_VALUE
||
595 brw_inst_src1_reg_file(brw
, dst
) == BRW_IMMEDIATE_VALUE
;
597 set_uncompacted_subreg(brw
, dst
, src
);
598 brw_inst_set_acc_wr_control(brw
, dst
, brw_compact_inst_acc_wr_control(src
));
599 brw_inst_set_cond_modifier(brw
, dst
, brw_compact_inst_cond_modifier(src
));
601 brw_inst_set_flag_subreg_nr(brw
, dst
,
602 brw_compact_inst_flag_subreg_nr(src
));
603 set_uncompacted_src0(brw
, dst
, src
);
604 set_uncompacted_src1(brw
, dst
, src
, is_immediate
);
605 brw_inst_set_dst_da_reg_nr(brw
, dst
, brw_compact_inst_dst_reg_nr(src
));
606 brw_inst_set_src0_da_reg_nr(brw
, dst
, brw_compact_inst_src0_reg_nr(src
));
608 brw_inst_set_imm_ud(brw
, dst
,
609 brw_inst_imm_ud(brw
, dst
) |
610 brw_compact_inst_src1_reg_nr(src
));
612 brw_inst_set_src1_da_reg_nr(brw
, dst
, brw_compact_inst_src1_reg_nr(src
));
616 void brw_debug_compact_uncompact(struct brw_context
*brw
,
618 brw_inst
*uncompacted
)
620 fprintf(stderr
, "Instruction compact/uncompact changed (gen%d):\n",
623 fprintf(stderr
, " before: ");
624 brw_disassemble_inst(stderr
, brw
, orig
, true);
626 fprintf(stderr
, " after: ");
627 brw_disassemble_inst(stderr
, brw
, uncompacted
, false);
629 uint32_t *before_bits
= (uint32_t *)orig
;
630 uint32_t *after_bits
= (uint32_t *)uncompacted
;
631 fprintf(stderr
, " changed bits:\n");
632 for (int i
= 0; i
< 128; i
++) {
633 uint32_t before
= before_bits
[i
/ 32] & (1 << (i
& 31));
634 uint32_t after
= after_bits
[i
/ 32] & (1 << (i
& 31));
636 if (before
!= after
) {
637 fprintf(stderr
, " bit %d, %s to %s\n", i
,
638 before
? "set" : "unset",
639 after
? "set" : "unset");
645 compacted_between(int old_ip
, int old_target_ip
, int *compacted_counts
)
647 int this_compacted_count
= compacted_counts
[old_ip
];
648 int target_compacted_count
= compacted_counts
[old_target_ip
];
649 return target_compacted_count
- this_compacted_count
;
653 update_uip_jip(struct brw_context
*brw
, brw_inst
*insn
,
654 int this_old_ip
, int *compacted_counts
)
656 int jip
= brw_inst_jip(brw
, insn
);
657 jip
-= compacted_between(this_old_ip
, this_old_ip
+ jip
, compacted_counts
);
658 brw_inst_set_jip(brw
, insn
, jip
);
660 if (brw_inst_opcode(brw
, insn
) == BRW_OPCODE_ENDIF
||
661 brw_inst_opcode(brw
, insn
) == BRW_OPCODE_WHILE
)
664 int uip
= brw_inst_uip(brw
, insn
);
665 uip
-= compacted_between(this_old_ip
, this_old_ip
+ uip
, compacted_counts
);
666 brw_inst_set_uip(brw
, insn
, uip
);
670 brw_init_compaction_tables(struct brw_context
*brw
)
672 assert(gen6_control_index_table
[ARRAY_SIZE(gen6_control_index_table
) - 1] != 0);
673 assert(gen6_datatype_table
[ARRAY_SIZE(gen6_datatype_table
) - 1] != 0);
674 assert(gen6_subreg_table
[ARRAY_SIZE(gen6_subreg_table
) - 1] != 0);
675 assert(gen6_src_index_table
[ARRAY_SIZE(gen6_src_index_table
) - 1] != 0);
676 assert(gen7_control_index_table
[ARRAY_SIZE(gen7_control_index_table
) - 1] != 0);
677 assert(gen7_datatype_table
[ARRAY_SIZE(gen7_datatype_table
) - 1] != 0);
678 assert(gen7_subreg_table
[ARRAY_SIZE(gen7_subreg_table
) - 1] != 0);
679 assert(gen7_src_index_table
[ARRAY_SIZE(gen7_src_index_table
) - 1] != 0);
683 control_index_table
= gen7_control_index_table
;
684 datatype_table
= gen7_datatype_table
;
685 subreg_table
= gen7_subreg_table
;
686 src_index_table
= gen7_src_index_table
;
689 control_index_table
= gen6_control_index_table
;
690 datatype_table
= gen6_datatype_table
;
691 subreg_table
= gen6_subreg_table
;
692 src_index_table
= gen6_src_index_table
;
700 brw_compact_instructions(struct brw_compile
*p
, int start_offset
,
701 int num_annotations
, struct annotation
*annotation
)
703 struct brw_context
*brw
= p
->brw
;
704 void *store
= p
->store
+ start_offset
/ 16;
705 /* For an instruction at byte offset 8*i before compaction, this is the number
706 * of compacted instructions that preceded it.
708 int compacted_counts
[(p
->next_insn_offset
- start_offset
) / 8];
709 /* For an instruction at byte offset 8*i after compaction, this is the
710 * 8-byte offset it was at before compaction.
712 int old_ip
[(p
->next_insn_offset
- start_offset
) / 8];
714 if (brw
->gen
< 6 || brw
->gen
>= 8)
719 int compacted_count
= 0;
720 for (src_offset
= 0; src_offset
< p
->next_insn_offset
- start_offset
;) {
721 brw_inst
*src
= store
+ src_offset
;
722 void *dst
= store
+ offset
;
724 old_ip
[offset
/ 8] = src_offset
/ 8;
725 compacted_counts
[src_offset
/ 8] = compacted_count
;
727 brw_inst saved
= *src
;
729 if (!brw_inst_cmpt_control(brw
, src
) &&
730 brw_try_compact_instruction(brw
, dst
, src
)) {
734 brw_inst uncompacted
;
735 brw_uncompact_instruction(brw
, &uncompacted
, dst
);
736 if (memcmp(&saved
, &uncompacted
, sizeof(uncompacted
))) {
737 brw_debug_compact_uncompact(brw
, &saved
, &uncompacted
);
744 int size
= brw_inst_cmpt_control(brw
, src
) ? 8 : 16;
746 /* It appears that the end of thread SEND instruction needs to be
747 * aligned, or the GPU hangs.
749 if ((brw_inst_opcode(brw
, src
) == BRW_OPCODE_SEND
||
750 brw_inst_opcode(brw
, src
) == BRW_OPCODE_SENDC
) &&
751 brw_inst_eot(brw
, src
) &&
753 brw_compact_inst
*align
= store
+ offset
;
754 memset(align
, 0, sizeof(*align
));
755 brw_compact_inst_set_opcode(align
, BRW_OPCODE_NOP
);
756 brw_compact_inst_set_cmpt_control(align
, true);
758 old_ip
[offset
/ 8] = src_offset
/ 8;
759 dst
= store
+ offset
;
762 /* If we didn't compact this intruction, we need to move it down into
765 if (offset
!= src_offset
) {
766 memmove(dst
, src
, size
);
773 /* Fix up control flow offsets. */
774 p
->next_insn_offset
= start_offset
+ offset
;
775 for (offset
= 0; offset
< p
->next_insn_offset
- start_offset
;) {
776 brw_inst
*insn
= store
+ offset
;
777 int this_old_ip
= old_ip
[offset
/ 8];
778 int this_compacted_count
= compacted_counts
[this_old_ip
];
779 int target_old_ip
, target_compacted_count
;
781 switch (brw_inst_opcode(brw
, insn
)) {
782 case BRW_OPCODE_BREAK
:
783 case BRW_OPCODE_CONTINUE
:
784 case BRW_OPCODE_HALT
:
785 update_uip_jip(brw
, insn
, this_old_ip
, compacted_counts
);
789 case BRW_OPCODE_ELSE
:
790 case BRW_OPCODE_ENDIF
:
791 case BRW_OPCODE_WHILE
:
793 int gen6_jump_count
= brw_inst_gen6_jump_count(brw
, insn
);
794 target_old_ip
= this_old_ip
+ gen6_jump_count
;
795 target_compacted_count
= compacted_counts
[target_old_ip
];
796 gen6_jump_count
-= (target_compacted_count
- this_compacted_count
);
797 brw_inst_set_gen6_jump_count(brw
, insn
, gen6_jump_count
);
799 update_uip_jip(brw
, insn
, this_old_ip
, compacted_counts
);
804 offset
= next_offset(brw
, store
, offset
);
807 /* p->nr_insn is counting the number of uncompacted instructions still, so
808 * divide. We do want to be sure there's a valid instruction in any
809 * alignment padding, so that the next compression pass (for the FS 8/16
810 * compile passes) parses correctly.
812 if (p
->next_insn_offset
& 8) {
813 brw_compact_inst
*align
= store
+ offset
;
814 memset(align
, 0, sizeof(*align
));
815 brw_compact_inst_set_opcode(align
, BRW_OPCODE_NOP
);
816 brw_compact_inst_set_cmpt_control(align
, true);
817 p
->next_insn_offset
+= 8;
819 p
->nr_insn
= p
->next_insn_offset
/ 16;
821 /* Update the instruction offsets for each annotation. */
823 for (int offset
= 0, i
= 0; i
< num_annotations
; i
++) {
824 while (start_offset
+ old_ip
[offset
/ 8] * 8 != annotation
[i
].offset
) {
825 assert(start_offset
+ old_ip
[offset
/ 8] * 8 <
826 annotation
[i
].offset
);
827 offset
= next_offset(brw
, store
, offset
);
830 annotation
[i
].offset
= start_offset
+ offset
;
832 offset
= next_offset(brw
, store
, offset
);
835 annotation
[num_annotations
].offset
= p
->next_insn_offset
;